# copied from http://www.robotstxt.org/wc/exclusion-admin.html # ------------------------------------------------------------ # # To exclude all robots from the entire server # User-agent: * # Disallow: / # # To allow all robots complete access # User-agent: * # Disallow: # Or create an empty "/robots.txt" file. # # To exclude all robots from part of the server # User-agent: * # Disallow: /cgi-bin/ # Disallow: /tmp/ # Disallow: /private/ # # To exclude a single robot # User-agent: BadBot # Disallow: / # # To allow a single robot # User-agent: WebCrawler # Disallow: # ### User-agent: * ### Disallow: / # User-agent: * Disallow: