# /robots.txt file for http://webcrawler.com/
# mail jason@idmstudio.co.za for constructive criticism

User-agent: *
Disallow: /catalogs
Disallow: /Connections
Disallow: /data.files
Disallow: /extras
Disallow: /images
Disallow: /info
Disallow: /js
Disallow: /myadmin
Disallow: /nl
Disallow: /scripts
Disallow: /search
Disallow: /stats
Disallow: /tinymce
Disallow: /temp