# # robots.txt # # This file is to prevent the crawling and indexing of certain parts # of your site by web crawlers and spiders run by sites like Yahoo! # and Google. By telling these "robots" where not to go on your site, # you save bandwidth and server resources. # # This file will be ignored unless it is at the root of your host: # Used: http://example.com/robots.txt # Ignored: http://example.com/site/robots.txt # # For more information about the robots.txt standard, see: # http://www.robotstxt.org/robotstxt.html User-agent: * # CSS, JS, Images Allow: /core/*.css$ Allow: /core/*.css? Allow: /core/*.js$ Allow: /core/*.js? Allow: /core/*.gif Allow: /core/*.jpg Allow: /core/*.jpeg Allow: /core/*.png Allow: /core/*.svg Allow: /profiles/*.css$ Allow: /profiles/*.css? Allow: /profiles/*.js$ Allow: /profiles/*.js? Allow: /profiles/*.gif Allow: /profiles/*.jpg Allow: /profiles/*.jpeg Allow: /profiles/*.png Allow: /profiles/*.svg # Directories Disallow: /core/ Disallow: /profiles/ # Files Disallow: /README.md Disallow: /composer/Metapackage/README.txt Disallow: /composer/Plugin/ProjectMessage/README.md Disallow: /composer/Plugin/Scaffold/README.md Disallow: /composer/Plugin/VendorHardening/README.txt Disallow: /composer/Template/README.txt Disallow: /modules/README.txt Disallow: /sites/README.txt Disallow: /themes/README.txt Disallow: /web.config # Paths (clean URLs) Disallow: /admin/ Disallow: /comment/reply/ Disallow: /filter/tips Disallow: /node/add/ Disallow: /search/ Disallow: /user/register Disallow: /user/password Disallow: /user/login Disallow: /user/logout Disallow: /media/oembed Disallow: /*/media/oembed # Paths (no clean URLs) Disallow: /index.php/admin/ Disallow: /index.php/comment/reply/ Disallow: /index.php/filter/tips Disallow: /index.php/node/add/ Disallow: /index.php/search/ Disallow: /index.php/user/password Disallow: /index.php/user/register Disallow: /index.php/user/login Disallow: /index.php/user/logout Disallow: /index.php/media/oembed Disallow: /index.php/*/media/oembed # # Additional robots.txt rules v2.4 (2026-09-01) // BBE # # AI policy: block TRAINING/dataset crawlers (traffic, no return); # deliberately ALLOW answer/search bots (visibility in AI assistants): # OAI-SearchBot, ChatGPT-User, Claude-User, Claude-SearchBot, PerplexityBot, # Perplexity-User, DuckAssistBot, Applebot, Meta-ExternalFetcher. # Google-Extended is deliberately ALLOWED: it also gates Gemini grounding, # and we prefer AI visibility over the Gemini training opt-out. # # ---------------------------------------------------- # Crawl-budget rules: these extend core's "User-agent: *" # group and must stay before the first User-agent line below. # ---------------------------------------------------- Disallow: /index.php* Disallow: /*page=* Disallow: /search? Disallow: /*/search/ Disallow: /*/search? Disallow: /*?f[* Disallow: /*&f[* Disallow: /*?f%5B* Disallow: /*&f%5B* # ---------------------------------------------------- # AI / dataset / large-scale collectors (block) # ---------------------------------------------------- User-agent: AI2Bot Disallow: / User-agent: AI2Bot-Dolma Disallow: / User-agent: anthropic-ai Disallow: / User-agent: Applebot-Extended Disallow: / User-agent: Bytespider Disallow: / User-agent: CCBot Disallow: / User-agent: ClaudeBot Disallow: / User-agent: Claude-Web Disallow: / User-agent: CommonCrawl Disallow: / User-agent: cohere-ai Disallow: / User-agent: cohere-training-data-crawler Disallow: / User-agent: Diffbot Disallow: / User-agent: FirecrawlAgent Disallow: / User-agent: GoogleOther Disallow: / User-agent: GPTBot Disallow: / User-agent: ImagesiftBot Disallow: / User-agent: img2dataset Disallow: / User-agent: Kangaroo Bot Disallow: / User-agent: Meta-ExternalAgent Disallow: / User-agent: Omgili Disallow: / User-agent: Omgilibot Disallow: / User-agent: PanguBot Disallow: / User-agent: Reflectionbot Disallow: / User-agent: TikTokSpider Disallow: / User-agent: Timpibot Disallow: / User-agent: TurnitinBot Disallow: / User-agent: VelenPublicWebCrawler Disallow: / User-agent: Webzio Disallow: / User-agent: webzio-extended Disallow: / # ---------------------------------------------------- # Competitive SEO / backlink tooling (block) # ---------------------------------------------------- User-agent: AhrefsBot Disallow: / User-agent: AhrefsSiteAudit Disallow: / User-agent: Babbar Disallow: / User-agent: Barkrowler Disallow: / User-agent: BLEXBot Disallow: / User-agent: cognitiveseo Disallow: / User-agent: DataForSeoBot Disallow: / User-agent: Dataprovider.com Disallow: / User-agent: DomainStatsBot Disallow: / User-agent: DotBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: OnCrawl Disallow: / User-agent: Pandalytics Disallow: / User-agent: Qwantify Disallow: / User-agent: Screaming Frog SEO Spider Disallow: / User-agent: SemrushBot Disallow: / User-agent: SemrushBot-BA Disallow: / User-agent: SemrushBot-SA Disallow: / User-agent: SemrushBot-SI Disallow: / User-agent: SERankingBacklinksBot Disallow: / User-agent: serpstatbot Disallow: / User-agent: SiteAuditBot Disallow: / User-agent: SpyFu Disallow: / User-agent: Website-info.net-Robot Disallow: / # ---------------------------------------------------- # Monitoring / mention / misc crawlers you don't use (block) # ---------------------------------------------------- User-agent: AliyunSecBot Disallow: / User-agent: Amazing-SearchBot Disallow: / User-agent: AwarioBot Disallow: / User-agent: AwarioSmartBot Disallow: / User-agent: IbouBot Disallow: / User-agent: ips-agent Disallow: / User-agent: Siteimprove Disallow: / User-agent: SiteimproveBot Disallow: / User-agent: stepstoneCrawlBot Disallow: / User-agent: TrendictionBot Disallow: / User-agent: ubermetrics-technologies.com Disallow: / User-agent: VeryHip Disallow: / User-agent: wpbot Disallow: / User-agent: ZoominfoBot Disallow: / # ---------------------------------------------------- # Search engines you did NOT ask to keep (block) # ---------------------------------------------------- User-agent: BaiduSpider Disallow: / User-agent: BaiduSpider-image Disallow: / User-agent: BaiduSpider-mobile Disallow: / User-agent: BaiduSpider-news Disallow: / User-agent: BaiduSpider-video Disallow: / User-agent: coccocbot Disallow: / User-agent: coccocbot-image Disallow: / User-agent: coccocbot-web Disallow: / User-agent: crawl.sogou.com Disallow: / User-agent: MojeekBot Disallow: / User-agent: PetalBot Disallow: / User-agent: SeznamBot Disallow: / User-agent: sogouspider Disallow: / User-agent: Sogou web spider Disallow: / User-agent: YandexBot Disallow: / User-agent: YandexImages Disallow: / User-agent: Yeti Disallow: / # ---------------------------------------------------- # Old agents still common in the wild (block) # ---------------------------------------------------- User-agent: com.plumanalytics Disallow: / User-agent: crawler4j Disallow: / User-agent: heritrix Disallow: / User-agent: plumanalytics Disallow: / User-agent: ripz Disallow: / User-agent: Scrapy Disallow: / User-agent: siteripz Disallow: /