User-agent: *
Allow: /
Crawl-delay: 5

User-agent: bingbot
Crawl-delay: 1

User-agent: SemrushBot
Disallow: /

User-agent: dotbot
Disallow: /

# www.sentibot.eu
User-agent: sentibot
Disallow: /
User-agent: SentiBot
Disallow: /

# Block  MegaIndex.ru
User-agent: MegaIndex.ru/2.0
Disallow: /
User-agent: MegaIndex.ru
Disallow: /
User-agent: megaIndex.ru
Disallow: /

User-Agent: MJ12bot
Disallow: /
User-Agent: MJ12bot/v1.4.3
Disallow: /

# Yandex tends to be rather aggressive
User-agent: YandexBot
Crawl-delay: 10

User-agent: AhrefsBot
Disallow: /

User-agent: Baiduspider
User-agent: Baiduspider-video
User-agent: Baiduspider-image
Disallow: /