# Aetherya — robots.txt (2026-07-10)
#
# Sanctioned agent surfaces:
#   /agent           — agent portal and directory
#   /llms.txt        — LLM-readable index
#   /llms-full.txt   — full agent-readable content
#   /openapi.json    — OpenAPI specification
#   /.well-known/    — discovery files (API catalog, MCP card)
#   /api/public/     — unauthenticated read-only endpoints
#   /sitemap.xml     — sitemap
#
# Browser automation against /dashboard is unsupported.
# The sanctioned path is the documented API and MCP server.
#
# Content Signals:
#   search=yes, ai-input=yes, ai-train=no

# ── General crawlers ─────────────────────────────────────────────────────────
User-agent: *
Allow: /
Disallow: /dashboard
Disallow: /dashboard/
Disallow: /checkout
Disallow: /checkout/
Disallow: /invite/
Disallow: /shared/
Disallow: /redeem
Disallow: /partners/byb
Disallow: /partners/byb/
Content-Signal: search=yes, ai-input=yes, ai-train=no

# ── User-initiated browsing agents ───────────────────────────────────────────
User-agent: Claude-User
Allow: /
Disallow: /dashboard
Disallow: /dashboard/
Disallow: /checkout
Disallow: /checkout/

User-agent: ChatGPT-User
Allow: /
Disallow: /dashboard
Disallow: /dashboard/
Disallow: /checkout
Disallow: /checkout/

User-agent: Google-Agent
Allow: /
Disallow: /dashboard
Disallow: /dashboard/
Disallow: /checkout
Disallow: /checkout/

User-agent: Perplexity-User
Allow: /
Disallow: /dashboard
Disallow: /dashboard/
Disallow: /checkout
Disallow: /checkout/

# ── Social unfurlers ─────────────────────────────────────────────────────────
# Shared result pages are meant to be posted. These agents only fetch a page to
# render its preview card, so they are allowed into /shared/ even though general
# crawlers are not. The pages themselves also send `noindex`
# (app/shared/layout.tsx), so allowing the unfurl does not put them in search.
User-agent: Twitterbot
Allow: /shared/
Allow: /

User-agent: facebookexternalhit
Allow: /shared/
Allow: /

User-agent: LinkedInBot
Allow: /shared/
Allow: /

User-agent: Slackbot-LinkExpanding
Allow: /shared/
Allow: /

User-agent: Slackbot
Allow: /shared/
Allow: /

User-agent: Discordbot
Allow: /shared/
Allow: /

User-agent: TelegramBot
Allow: /shared/
Allow: /

User-agent: WhatsApp
Allow: /shared/
Allow: /

# ── Training crawlers (public pages allowed; see Content Signals) ────────────
User-agent: ClaudeBot
Allow: /
Disallow: /dashboard
Disallow: /dashboard/

User-agent: GPTBot
Allow: /
Disallow: /dashboard
Disallow: /dashboard/

User-agent: Google-Extended
Allow: /
Disallow: /dashboard
Disallow: /dashboard/

Sitemap: https://www.aetherya.ai/sitemap.xml