User-agent: *
Disallow: /~slaven/cpantestersmatrix.cgi
# Don't want automatic comments
Disallow: /cgi-bin/mapserver_comment.cgi
# These are quite large and unlikely to be usable for bots
Disallow: /cgi-bin/bbbike-snapshot.cgi
Disallow: /cgi-bin/bbbike-data.cgi

User-Agent: feedfinder
Disallow: /

User-Agent: msnbot
Disallow: /cgi-bin/A327B109991F487BB01E1A38C1F3A941_00000

# We don't have online shops here,
# and the robot generated too many 404
User-Agent: TwengaBot-Discover
Disallow: /

# Crawls ftp URLs badly and does not have the advertised /robot page
User-Agent: SiteBot
Disallow: /