# $Id: robots.txt 613 2007-08-19 03:18:11Z px $ # #Sitemap: # User-agent: ia_archiver Crawl-delay: 60 Disallow: # I'm not a shoppping website. Don't Index me. User-agent: BecomeBot Disallow:/ # Other stuff User-agent: * Crawl-delay: 240 #Disallow:/