User-agent:  *
Crawl-delay: 60

 
# Disallow all directories and files within
Disallow: /cgi-bin/
Disallow: /wp-admin/
Disallow: /wp-includes/
 
# Disallow all files ending with these extensions
Disallow: /*.js$
Disallow: /*.inc$
Disallow: /*.css$
 
# Disallow parsing indididual post feeds and trackbacks..
Disallow: /trackback/
Disallow: /feed/
Disallow: /comment-page*

# Disallow others
Disallow: /site/

User-agent: twiceler
Disallow: /

User-agent: Slurp
Crawl-delay: 300

User-agent: Yahoo!-AdCrawler
Disallow: /