# All robots will spider the domain



User-agent: *

Disallow: /*?
Disallow: /*.php
Disallow: /*.htm$