User-Agent: htdig-3.1.5
User-Agent: Harvest-NG
User-Agent: Harvest
Disallow: /cgi-bin/
Disallow: /private_html/
Disallow: /Liverpool/Private/
Disallow: /pd-tutorials/
# Allow says Paul hagan (Corp Comme) - PDM 02/02/2009
#Disallow: /xjtlu/

User-Agent: W3C-checklink
User-Agent: Lycos
User-Agent: Lycos_Spider_(T-Rex)
User-Agent: Gulliver
User-Agent: Scooter
User-Agent: InfoSeek
User-Agent: Ultraseek
User-Agent: fido
User-Agent: NetMechanic
User-Agent: ArchitextSpider
User-Agent: Mirago
User-Agent: Slurp
User-Agent: Googlebot
User-Agent: Google
User-Agent: WebCrawler
User-Agent: ia_archiver
User-Agent: gsa-crawler
User-Agent: msnbot
User-Agent: Connotate
Disallow: /cgi-bin/
Disallow: /private_html/
Disallow: /local_html/
Disallow: /Admin/
Disallow: /harvest/
Disallow: /Harvest/
Disallow: /Liverpool/Private/
Disallow: /pd-tutorials/
Disallow: /international/Clearing/
Disallow: /intranet/
Disallow: /wwwstats/
Disallow: /WWWstats/
Disallow: /xjtlu/

User-Agent: BioCrawler
Disallow: /a
Disallow: /b
Disallow: /ca
Disallow: /cb
Disallow: /cc
Disallow: /cd
Disallow: /ce
Disallow: /cf
Disallow: /cg
Disallow: /ch
Disallow: /cia
Disallow: /cib
Disallow: /cic
Disallow: /cid
Disallow: /cie
Disallow: /cif
Disallow: /cig
Disallow: /cih
Disallow: /cii
Disallow: /cij
Disallow: /cik
Disallow: /cim
Disallow: /cin
Disallow: /cio
Disallow: /cip
Disallow: /ciq
Disallow: /cir
Disallow: /cis
Disallow: /cit
Disallow: /ciu
Disallow: /civ
Disallow: /ciw
Disallow: /cix
Disallow: /ciy
Disallow: /ciz
Disallow: /cj
Disallow: /ck
Disallow: /cl
Disallow: /cm
Disallow: /cn
Disallow: /co
Disallow: /cp
Disallow: /cq
Disallow: /cr
Disallow: /cs
Disallow: /ct
Disallow: /cu
Disallow: /cv
Disallow: /cw
Disallow: /cx
Disallow: /cy
Disallow: /cz
Disallow: /d
Disallow: /e
Disallow: /f
Disallow: /g
Disallow: /h
Disallow: /i
Disallow: /j
Disallow: /k
Disallow: /l
Disallow: /m
Disallow: /n
Disallow: /o
Disallow: /p
Disallow: /q
Disallow: /r
Disallow: /s
Disallow: /t
Disallow: /u
Disallow: /v
Disallow: /w
Disallow: /x
Disallow: /y
Disallow: /z
Disallow: /A
Disallow: /B
Disallow: /C
Disallow: /D
Disallow: /E
Disallow: /F
Disallow: /G
Disallow: /H
Disallow: /I
Disallow: /J
Disallow: /K
Disallow: /L
Disallow: /M
Disallow: /N
Disallow: /O
Disallow: /P
Disallow: /Q
Disallow: /R
Disallow: /S
Disallow: /T
Disallow: /U
Disallow: /V
Disallow: /W
Disallow: /X
Disallow: /Y
Disallow: /z
Disallow: /cgi-bin/
Disallow: /private_html/
Disallow: /local_html/
Disallow: /Admin/
Disallow: /harvest/
Disallow: /Harvest/
Disallow: /Liverpool/Private/
Disallow: /pd-tutorials/

User-agent: Xenu Link Sleuth
Disallow: /

User-agent: *
Disallow: /