# AI crawlers (GPTBot, ClaudeBot, PerplexityBot, etc.) are welcome — see /llms.txt
# The rules below apply to all agents; there are no AI-specific blocks.

User-agent: *

# OG/social images live under /api/ — allow them explicitly (longest-match wins)
Allow: /api/og-image
# The retired bot-trap URL: a permanent 301 to the real page for everyone.
# Crawlable so an engine still holding one of these URLs sees the redirect
# and moves the listing to the page. It is never content.
Allow: /api/bot-challenge

# Block System Directories
Disallow: /includes/
Disallow: /api/
Disallow: /admin/
Disallow: /og-cache/
Disallow: /translations/
Disallow: /backups/
Disallow: /twitterbot/
Disallow: /matomo/

# Block search result pages (allow /search landing, block ?q= results)
Disallow: /search?

# Block the crawler traps: UI state and facets carried in query strings.
# Every URL below is already either noindex (/ai?q=) or canonicalised to a
# clean path that is itself in the sitemap, so crawling them returns nothing
# and costs everything: in Aug 2026 these three shapes were ~70% of all origin
# traffic, /ai? alone being 61%. They are combinatorial — one link per verse
# crossed with 39 languages, 66 books and the menu-open flag.
# NOTE: `?lang=` is deliberately NOT blocked. It is the real language mechanism
# for the pages that carry no /xx path prefix (/ai, /fathers/<slug>), so a rule
# against it would deindex every non-English edition of them.
Disallow: /ai?
Disallow: /*langmenu=
Disallow: /fathers/*?*book=
# `?father=` is a .htaccess rewrite capture echoed back out, never a reader
# control (the chips use `book=`), and the slug is already in the path — so
# /fathers/x?father=x is /fathers/x wearing a disguise. Safe to block outright:
# all 46,823 sitemap URLs are clean paths, so no canonical URL carries a query.
Disallow: /*?*father=

# Block Log and Config Extensions (Wildcards)
Disallow: /*.log$
Disallow: /*.ini$

Sitemap: https://trulyrandomverse.com/sitemap.xml