# Public marketing surface is crawlable. Authenticated areas (dashboard,
# portal, API endpoints, generated email web views) are explicitly disallowed
# so search engines and AI crawlers do not index per-merchant content.
#
# Build 34: AI crawlers explicitly listed. We want maximum visibility in
# both classic search and generative-search citations.

User-agent: *
Allow: /
Allow: /site/
Allow: /pricing/
Allow: /docs/
Disallow: /onboard/
Disallow: /dashboard/
Disallow: /portal/
Disallow: /admin/
Allow: /api/public/
Disallow: /api/
Disallow: /alert/
Disallow: /digest/
Disallow: /gaps/
Disallow: /enhanced/

User-agent: Googlebot
Allow: /
Disallow: /onboard/
Disallow: /dashboard/
Disallow: /portal/
Disallow: /admin/
Disallow: /api/
Disallow: /alert/
Disallow: /digest/
Disallow: /gaps/
Disallow: /enhanced/

User-agent: Bingbot
Allow: /
Disallow: /onboard/
Disallow: /dashboard/
Disallow: /portal/
Disallow: /admin/
Disallow: /api/
Disallow: /alert/
Disallow: /digest/
Disallow: /gaps/
Disallow: /enhanced/

# THE ANSWER ENGINES, ONE GROUP (DR-2026-09-28-the-evidence-goes-out).
# A crawler that matches a named group ignores the * group entirely, so this
# group repeats every internal line of the * group above, the /api/public/
# allowance included: the public reads are the evidence these engines cite.
# OAI-SearchBot and PerplexityBot used to have groups of their own that said
# only "Allow: /", which opened every internal path above to them, so for them
# this narrows access; Claude-SearchBot fell to the * group and keeps what it
# had. Googlebot and Bingbot keep their own groups above, exactly as they were,
# without the allowance: widening them is not this change's to make
# (DR-2026-09-04-access-control-class).
# tests/unit/site/search-and-answer-engines.test.ts holds all five.
User-agent: OAI-SearchBot
User-agent: Claude-SearchBot
User-agent: PerplexityBot
Allow: /
Disallow: /onboard/
Disallow: /dashboard/
Disallow: /portal/
Disallow: /admin/
Allow: /api/public/
Disallow: /api/
Disallow: /alert/
Disallow: /digest/
Disallow: /gaps/
Disallow: /enhanced/

User-agent: Google-Extended
Allow: /

User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# The /benchmarks reference layer is generated from the observed corpus, so its
# routes and their lastmod dates move as the crawl advances. It publishes its
# own sitemap rather than being frozen into the static one; both are listed
# here, which is how a crawler is meant to discover more than one.
Sitemap: https://myoddly.com/sitemap.xml
Sitemap: https://myoddly.com/benchmarks/sitemap.xml