# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file

# allow google
User-agent: Googlebot
Crawl-delay: 120 
Disallow: /cgi-bin/perlfect/search/search.pl

# allow msn 
User-agent: msnbot
Crawl-delay: 120
Disallow: /cgi-bin/perlfect/search/search.pl

# allow live.com msn 
User-agent: MSNBot
Crawl-delay: 120
#Disallow: /
Disallow: /cgi-bin/perlfect/search/search.pl

# allow yahoo
User-agent: Slurp
Crawl-delay: 120
Disallow: /cgi-bin/perlfect/search/search.pl

# allow googlemini (kagisearch)
User-agent: gsa-crawler-kagi-search
Crawl-delay: 10
Disallow: /cgi-bin/perlfect/search/search.pl

# everyone else go away
User-agent: *
Crawl-delay: 120
Disallow: /