# robots, goaway
#this bans robots from our cgi and othe places they should not be

User-agent: googlebot
Disallow: /cgi/
Disallow: /backdoor/
Disallow: /homepages/
Disallow: /images/

User-agent: *
Disallow: /cgi/
Disallow: /backdoor/
Disallow: /homepages/
Disallow: /images/

User-agent: Mozilla/3.01 (hotwired-test/0.1)
Disallow: /cgi/
Disallow: /backdoor/
Disallow: /homepages/
Disallow: /images/

User-agent: Slurp
Disallow: /cgi/
Disallow: /backdoor/
Disallow: /homepages/
Disallow: /images/

User-agent: Ultraseek
Disallow: /cgi/
Disallow: /backdoor/
Disallow: /homepages/
Disallow: /images/

User-agent: smallbear
Disallow: /cgi/
Disallow: /backdoor/
Disallow: /homepages/
Disallow: /images/

User-agent: WebFerret
Disallow:

User-agent: 'Ahoy! The Homepage Finder'
Disallow:

User-agent: Arachnophilia
Disallow:

User-agent: ArchitextSpider
Disallow:

User-agent: ASpider/0.09
Disallow:

User-agent: AURESYS/1.0
Disallow:

User-agent: BackRub/*.*
Disallow:

User-agent: Big Brother
Disallow:

User-agent: BlackWidow
Disallow:

User-agent: BSpider/1.0 libwww-perl/0.40
Disallow:

User-agent: CACTVS Chemistry Spider
Disallow:

User-agent: Digimarc CGIReader/1.0
Disallow:

User-agent: Checkbot/x.xx LWP/5.x
Disallow:

User-agent: CMC/0.01
Disallow:

User-agent: combine/0.0
Disallow:

User-agent: conceptbot/0.3
Disallow:

User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0
Disallow:

User-agent: root/0.1
Disallow:

User-agent: CS-HKUST-IndexServer/1.0
Disallow:

User-agent: CyberSpyder/2.1
Disallow:

User-agent: Deweb/1.01
Disallow:

User-agent: DragonBot/1.0 libwww/5.0
Disallow:

User-agent: EIT-Link-Verifier-Robot/0.2
Disallow:

User-agent: Emacs-w3/v[0-9\.]+
Disallow:

User-agent: EmailSiphon
Disallow:

User-agent: EMC Spider
Disallow:

User-agent: explorersearch
Disallow:

User-agent: Explorer
Disallow:

User-agent: ExtractorPro
Disallow:

User-agent: FelixIDE/1.0
Disallow:

User-agent: Hazel's Ferret Web hopper,
Disallow:

User-agent: ESIRover v1.0
Disallow:

User-agent: fido/0.9 Harvest/1.4.pl2
Disallow:

User-agent: Hämähäkki/0.2
Disallow:

User-agent: KIT-Fireball/2.0 libwww/5.0a
Disallow:

User-agent: Fish-Search-Robot
Disallow:

User-agent: Mozilla/2.0 (compatible fouineur v2.0; fouineur.9bit.qc.ca)
Disallow:

User-agent: Robot du CRIM 1.0a
Disallow:

User-agent: Freecrawl
Disallow:

User-agent: FunnelWeb-1.0
Disallow:

User-agent: gcreep/1.0
Disallow:

User-agent: ???
Disallow:

User-agent: GetURL.rexx v1.05
Disallow:

User-agent: Golem/1.1
Disallow:

User-agent: Gromit/1.0
Disallow:

User-agent: Gulliver/1.1
Disallow:

User-agent: yes
Disallow:

User-agent: AITCSRobot/1.1
Disallow:

User-agent: wired-digital-newsbot/1.5
Disallow:

User-agent: htdig/3.0b3
Disallow:

User-agent: HTMLgobble v2.2
Disallow:

User-agent: no
Disallow:

User-agent: IBM_Planetwide,
Disallow:

User-agent: gestaltIconoclast/1.0 libwww-FM/2.17
Disallow:

User-agent: INGRID/0.1
Disallow:

User-agent: IncyWincy/1.0b1
Disallow:

User-agent: Informant
Disallow:

User-agent: InfoSeek Robot 1.0
Disallow:

User-agent: Infoseek Sidewinder
Disallow:

User-agent: InfoSpiders/0.1
Disallow: