# robots.txt
User-agent: ia_archiver
Disallow: /

User-agent: Slurp
Disallow: /ccp5/ 
Crawl-delay: 300 


User-agent: *
Disallow: 
Disallow: /ccp5/
Disallow: /images/
Disallow: /delete_me/
Disallow: /javascripts/
Disallow: /catalog/
Disallow: /products/