# Allow only major search spiders
User-agent: *
Crawl-delay: 10


# Block Directories for all spiders
User-agent: *
Disallow: /ads/
Disallow: /cgi-bin/
Disallow: /scripts/