#Disallow Scrapers (non-search-engine content crawlers)
User-agent: ahrefsbot
Disallow: /
User-agent: AhrefsBot
Disallow: /

User-agent: XoviBot
Disallow: /

User-agent: MegaIndex.ru
Disallow: /

User-agent: dotbot
Disallow: /
User-agent: DotBot
Disallow: /

User-agent: spbot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: Spiderbot/Nutch-1.7
Disallow: /
User-agent: Spiderbot
Disallow: /

User-agent: SEOkicks-Robot
Disallow: /

#Specific exclusions for everyone else
User-agent: *
Disallow: /cgi-bin/
Disallow: /BV.shopcart.epl
Disallow: /dl/
Disallow: /docs/
Disallow: /web_product/
Disallow: /media-server/
Disallow: /wander/

Sitemap: https://www.bibliovault.org/sitemaps/sitemap.xml