# Allow only major search spiders    
# GOOGLE BOTS
User-agent: Mediapartners-Google
Disallow:
Crawl-delay: 30

User-agent: Googlebot
Disallow:
Crawl-delay: 30

User-agent: Adsbot-Google
Disallow:
Crawl-delay: 30

User-agent: Googlebot-Image
Disallow:
Crawl-delay: 30

User-agent: Googlebot-Mobile
Disallow:
Crawl-delay: 30

#MICROSOFT BOTS

User-agent: msnbot
Disallow:
Crawl-delay: 30

User-agent: bingbot
Disallow:
Crawl-delay: 30

#YAHOO BOTS

User-agent: Slurp
Disallow:
Crawl-delay: 30

User-agent: Yahoo! Slurp
Disallow:
Crawl-delay: 30

User-agent: moget
User-agent: ichiro
Disallow: /

User-agent: Yandex
Disallow: /

User-agent: sogou spider
Disallow: /

User-agent: YoudaoBot
Disallow: /

User-agent: Baiduspider
User-agent: Baiduspider-video
User-agent: Baiduspider-image
Disallow: /

User-agent: NaverBot
User-agent: Yeti
Disallow: /

# Block all other spiders
User-agent: *
Disallow: /

User-agent: *
Disallow: /siteadmin/
Disallow: /cache/
Disallow: /include/
Disallow: /language/
Disallow: /media/
Disallow: /tmp/
Disallow: /classes/
Disallow: /templates/
Disallow: /scripts/
Disallow: /addons/