# /robots.txt file for http://webcrawler.com/
# mail webmaster@webcrawler.com for constructive criticism

User-agent: *
Disallow: /stats
Disallow: /*.htm
Disallow: /php_uploads
Disallow: /cgi
Disallow: /photographs/slideshow*
Disallow: /slideshow*
Disallow: /wedding

User-agent: Googlebot
Disallow: /*.php
Disallow: /*.js
Disallow: /*.inc
Disallow: /*.css
Disallow: /*.py
Disallow: /*.wmv
Disallow: /*.cgi
Disallow: /*.xhtml
Disallow: /*.txt
Disallow: /*/index.htm
Disallow: /google*
Disallow: /wedding

#User-agent: Googlebot-Image
#Disallow:
#Allow: /*