# Allowed crawlers

User-agent: Atomz/1.0
Disallow:

User-agent: GigaBot/2.0
Disallow:

User-agent: Googlebot/2.X (+http://www.googlebot.com/bot.html)
Disallow:

User-agent: Slurp/2.0  # Inktomi
Disallow:

User-agent: Mozilla/2.0 (compatible; Ask Jeeves/Teoma)
Disallow:

User-agent: Link Validity Check From: http://www.w3dir.com/cgi-bin (Using: Hot Links SQL by Mrcgiguy.com)
Disallow:

User-agent: LinkWalker
Disallow:

User-agent: MSNBOT/0.1 (http://search.msn.com/msnbot.htm)
Disallow:

User-agent: TurnitinBot/2.1 (http://www.turnitin.com/robot/crawlerinfo.html)
Disallow:

User-agent: VoilaBot 1.2
Disallow:

User-agent: WISENutbot (looksmart)
Disallow:

# Disallowed crawlers

User-agent: Wget/1.9+cvs-stable (Red Hat modified)
Disallow: /

User-agent: ia_archiver  # Alexa Web crawler
Disallow: /

User-agent: MSIECrawler
Disallow: /

User-agent: crawl
Disallow: /

User-agent: psbot/0.1 (+http://www.picsearch.com/bot.html)
Disallow: /

User-agent: robot
Disallow: /

User-agent: spider
Disallow: /