# robots.txt for OCPanteras.com # "Any empty [Disallow] value, indicates that all URLs can be retrieved. # At least one Disallow field needs to be present in a record." User-agent: * Disallow: /cgi-bin Disallow: /gt8kat Disallow: /swapmeet Disallow: topoc.webprj ##User-agent: Mediapartners-Google* ##Disallow: ##User-agent: Googlebot ##Disallow: /comments.pl ##Disallow: /users.pl ##Disallow: /zoo.pl ##Disallow: comments.pl ##Disallow: users.pl ##Disallow: zoo.pl ##Disallow: /~ ##Disallow: ~ ##User-agent: Slurp ##Crawl-delay: 10 ##Disallow: ##User-agent: Yahoo-NewsCrawler* ##Disallow: ##User-Agent: msnbot ##Crawl-delay: 100 ##Disallow: