User-agent: * Disallow: / Disallow: /download/ Disallow: /images/ Disallow: /img/ Disallow: /include/ Disallow: /cgi-bin/ Disallow: /webstat/ # User-agent: peerbot Disallow: /favicon.ico # User-agent: MSIECrawler Disallow: / # User-agent: Googlebot-Image Disallow: / # User-agent: googlebot Disallow: / # User-agent: psbot Disallow: / # User-agent: NetMechanic Disallow: / # User-agent: ia_archiver Disallow: / # User-agent: sitecheck.internetseer.com Disallow: / # User-agent: LinkWalker Disallow: / # User-agent: FreeFind Disallow: / # User-agent: MondoSearch Disallow: / # User-agent: Spiderline Disallow: / # User-agent: fusionbot Disallow: / User-agent: WebReaper Disallow: / User-agent: NPBot Disallow: / # # #Examples #The following allows all robots to visit all files because the wildcard "*" #specifies all robots. #User-agent: * #Disallow: # #This one keeps all robots out. #User-agent: * #Disallow: / # #The next one bars all robots from the cgi-bin and images directories: #User-agent: * #Disallow: /cgi-bin/ #Disallow: /images/ # #This one bans Roverdog from all files on the server: User-agent: Roverdog Disallow: / # #This one bans keeps googlebot from getting at the cheese.htm file: #User-agent: googlebot #Disallow: index.html #