# /robots.txt file for http://webcrawler.com/
# mail webmaster@webcrawler.com for constructive criticism

# User-agent: webcrawler
# Disallow:

# User-agent: lycra
# Disallow: /

User-agent: *
Disallow: /asp
Disallow: /asp_client
Disallow: /data
Disallow: /count
Disallow: /css
Disallow: /customer
Disallow: /db
Disallow: /eastwest
Disallow: /flow
Disallow: /footer
Disallow: /forums
Disallow: /ftp
Disallow: /images
Disallow: /ip
Disallow: /login
Disallow: /mail
Disallow: /media120
Disallow: /mediax
Disallow: /msi
Disallow: /musichelp
Disallow: /nitrokaviar
Disallow: /project
Disallow: /projects
Disallow: /public
Disallow: /rob
Disallow: /spack
Disallow: /support
Disallow: /system
Disallow: /test
Disallow: /tid
Disallow: /upload
Disallow: /video