# /robots.txt file for http://webcrawler.com/
# mail webmaster@webcrawler.com for constructive criticism

# User-agent: webcrawler
# Disallow:

# User-agent: lycra
# Disallow: /

User-agent: *
Disallow: /asp
Disallow: /asp_client
Disallow: /data
Disallow: /count
Disallow: /customers
Disallow: /css
Disallow: /db
Disallow: /eastwest
Disallow: /footer
Disallow: /images
Disallow: /ip
Disallow: /media120
Disallow: /nitrokaviar
Disallow: /swf
Disallow: /system
Disallow: /test
Disallow: /test2
Disallow: /rob
Disallow: /lockpick
Disallow: /project
Disallow: /projects