#2021-10-01 updated
#allowed crawls
User-agent: Googlebot
Crawl-delay: 1

User-agent: Amazonbot
Crawl-delay: 1

User-agent: ia_archiver
Crawl-delay: 1

User-agent: alexabot
Crawl-delay: 1

User-agent: Bingbot
Crawl-delay: 30

User-agent: Baiduspider
Crawl-delay: 30

User-agent: googlebot-image
Crawl-delay: 3

User-agent: googlebot-mobile
Crawl-delay: 1

#disallow crawls
User-agent: Sosospider
Disallow: /
User-agent: sogou spider
Disallow: /
User-agent: YodaoBot
Disallow: /
User-agent: Slurp
Disallow: /
User-agent: Teoma
Disallow: /
User-agent: twiceler
Disallow: /
User-agent: Scrubby
Disallow: /
User-agent: Robozilla
Disallow: /
User-agent: Gigabot
Disallow: /
User-agent: yahoo-mmcrawler
Disallow: /
User-agent: yahoo-blogs/v3.9
Disallow: /
User-agent: psbot
Disallow: /
User-agent: grapeshot
Disallow: /
User-agent: Semrushbot
Disallow: /
User-agent: Semrushbot-SA
Disallow: /
User-agent: Semrushbot-SI
Disallow: /
User-agent: 008
Disallow: /
User-agent: Facebot
Disallow: /
User-agent: Twitterbot
Disallow: /