#Here are examples: #To exclude all robots from the entire server #User-agent: * #Disallow: / #To allow all robots complete access #User-agent: * #Disallow: #(or just create an empty "/robots.txt" file) #To exclude all robots from part of the server #User-agent: * #Disallow: /Bin/ #Disallow: /DesktopModules/ #Disallow: /SomeOtherFolder/ #To exclude a single robot #User-agent: BadBot #Disallow: / #To allow a single robot #User-agent: Google #Disallow: User-agent: * Disallow: /currentpage Disallow: /SearchResults Disallow: /DesktopModules User-agent: Slurp Crawl-delay: 5