User-Agent: * Allow: / # Hits many times per second, not acceptable # http://www.nameprotect.com/botinfo.html User-agent: NPBot Disallow: / # Downloads gazillions of pages with no public benefit, not acceptable # http://www.webreaper.net/ User-agent: WebReaper Disallow: / # Friendly, low-speed bots are always welcome to view all pages. User-agent: * Disallow: /gallery Disallow: /images Disallow: /js Disallow: /styles Disallow: /store_old Disallow: 404_Not_Found.html Disallow: /password Disallow: /Rotherbrook Crawl-delay: 2