# Do not remove the Crawl-delay directive It is needed to prevent dos
# conditions caused by certain robots, like msn/bing etc

User-agent: *
Crawl-delay: 30

User-agent: Googlebot