# Syntax at http://en.wikipedia.org/wiki/Robots.txt
# Try to limit Yahoo Slurp indexing
User-agent: Slurp
Crawl-delay: 20               # only allow one connection every 20 secs
Request-rate: 1/30m		# one document every 30 minutes
Visit-time: 0600-1200		# and then only between 1 am to 7 am EST
# Rest of the search engines
User-agent: *
Disallow: /catalog/cookie_usage.php
Disallow: /catalog/product_reviews_write.php
Disallow: /catalog/login.php
Disallow: /catalog/admin/
Disallow: /catalog/cache/
Disallow: /admin/