User-agent: Slurp
Crawl-delay: 10

User-agent: msnbot
Crawl-Delay: 10

User-agent: OmniExplorer_Bot
Disallow: /

# advertising-related bots:
# added 7-11-07 CP
User-agent: Mediapartners-Google*
Disallow: /

# Hits many times per second, not acceptable
# http://www.nameprotect.com/botinfo.html
# added 7-11-07 CP
User-agent: NPBot
Disallow: /

# A capture bot, downloads gazillions of pages with no public benefit
# http://www.webreaper.net/
# added 7-11-07 CP
User-agent: WebReaper
Disallow: /

User-agent: *
Disallow: /testing/
Disallow: /misc/
Disallow: /cgi-bin/
Disallow: /caspsamp/
Disallow: /caspdoc/
Disallow: /images/
Disallow: /pdf/
Disallow: /misc/acd/
Disallow: /hr/survey/
Disallow: /pw/traffic/counts/
Disallow: /tourism-cvb/eleads/
Disallow: /tourism-cvb/inquiry/
Disallow: /agenda/minutesarchive/
Disallow: /pd/boaagenda/
Disallow: /pd/pzagenda/
Disallow: /includes/
Disallow: /_vti_cnf/
Disallow: /pub/gis/
Disallow: /pub/strmwater/
Disallow: /pub/purchasing/
Disallow: /pub/pd/
# Dont spider the following because they are symbolic links - avoiding duplication
Disallow: /4-H/
Disallow: /aghort/
Disallow: /billpay/
Disallow: /budget/
Disallow: /building/
Disallow: /citizensacademy/
Disallow: /commsrvs/
Disallow: /coopext/
Disallow: /ctst/
Disallow: /dac/
Disallow: /employment/
Disallow: /engineering/
Disallow: /family/
Disallow: /gis/
Disallow: /grants/
Disallow: /homerule/
Disallow: /horses/
Disallow: /msbu/
Disallow: /mg/
Disallow: /museum/
Disallow: /natland/
Disallow: /parks/
Disallow: /planning/
Disallow: /pubwrks/
Disallow: /purchasing/
Disallow: /pwa/
Disallow: /reclaimed/
Disallow: /roads/
Disallow: /sbureau/
Disallow: /scea/
Disallow: /scinet/
Disallow: /sgtv/
Disallow: /solidwaste/
Disallow: /stormwater/
Disallow: /traffic/
Disallow: /trails/