User-agent:  *
Crawl-delay: 60

 
# Disallow all directories and files within
Disallow: /cgi-bin/
Disallow: /wp-admin/
Disallow: /wp-includes/
 
# Disallow all files ending with these extensions
Disallow: /*.js$
Disallow: /*.inc$
Disallow: /*.css$
 
# Disallow parsing indididual post feeds and trackbacks..
Disallow: /trackback/
Disallow: /feed/
Disallow: /comment-page-*
Disallow: /comments/feed
Disallow: /feed/$
Disallow: /*/feed/$
Disallow: /*/feed/rss/$
Disallow: /*/trackback/$
Disallow: /*/*/feed/$
Disallow: /*/*/feed/rss/$
Disallow: /*/*/trackback/$
Disallow: /*/*/*/feed/$
Disallow: /*/*/*/feed/rss/$
Disallow: /*/*/*/trackback/$


User-agent: Mediapartners-Google
Allow: /
 
User-agent: Adsbot-Google
Allow: /
 
User-agent: Googlebot-Image
Allow: /
 
User-agent: Googlebot-Mobile
Allow: /

User-agent: twiceler
Disallow: /

User-agent: Slurp
Crawl-delay: 300

User-agent: Yahoo!-AdCrawler
Disallow: /