# robots.txt for quotewise.io (Authoritative Domain)
#
# quotewise.io is the canonical domain for all quote content, API access,
# and AI/LLM integrations. All other domains canonicalize to quotewise.io.

# ============================================
# AI & LLM Crawler Guidelines (IETF AIPREF / Content Signals)
# ============================================
# We welcome responsible AI access for quote verification.
#
# Domain Authority:
#   quotewise.io = Authoritative domain for AI, API, and professional use
#   quotosaurus.com = Consumer discovery (canonicalized to quotewise.io)
#   api.quotewise.io = REST API for integrations
#   mcp.quotewise.io = MCP server for AI agents
#
# For structured API access optimized for LLMs:
#   - llms.txt: https://quotewise.io/llms.txt
#   - Full docs: https://quotewise.io/llms-full.txt
#   - REST API: https://api.quotewise.io/docs/
#
# Commercial AI training or large-scale harvesting?
# Contact: q@quotewise.io
# ============================================

# Content Signals (IETF AIPREF draft / Cloudflare Content Signals)
# As a condition of accessing this website, you agree to abide by
# the following content signals:
# (a) If a content-signal = yes, you may collect content for the
#     corresponding use.
# (b) If a content-signal = no, you may not collect content for
#     the corresponding use.
# (c) If absent, the website operator neither grants nor restricts
#     permission via content signal for that use.
#
# Categories:
#   search: building search index, returning hyperlinks/excerpts
#   ai-input: RAG, grounding, real-time AI answers (NOT training)
#   ai-train: training or fine-tuning AI models

# Policy is by ROLE, not by operator. Three roles, one rule each:
#   corpus acquisition (training)  -> Disallow. Declining ai-train is only
#                                     advisory; for real fetchers we enforce it.
#   search / citation index        -> Allow. This is how an assistant cites us,
#                                     and it is our distribution channel.
#   user-initiated fetch           -> Allow, unrestricted. A human asked.
#
# Measured over 7 days (Cloudflare AI Audit, 2026-08-08): 352.5k crawler
# requests produced 360 referrals, 979:1. Training crawlers returned nothing and
# have no mechanism to. Alexa, asked directly for Quotewise quotes, returns no
# results — Amazon does not surface the corpus even on its own products.

# ROLE 1 — Corpus acquisition. No citation surface, no referral path.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: CCBot
User-agent: Bytespider
Content-Signal: ai-train=no
Disallow: /

# Training-policy tokens. Not fetchers — Googlebot and Applebot do the
# fetching; these govern downstream training use only, so this costs no
# search visibility and reduces no request volume.
User-agent: Google-Extended
Content-Signal: ai-train=no
Disallow: /

# ROLE 2 — Search / citation index. Answers here link back, so they stay.
User-agent: OAI-SearchBot
User-agent: Claude-SearchBot
User-agent: PerplexityBot
Content-Signal: ai-train=no, search=yes, ai-input=yes
Crawl-delay: 5
Allow: /
# Content sections: derived/duplicate pages, expensive to render.
# Must be repeated here — named groups do NOT inherit the wildcard group (RFC 9309).
Disallow: /tags/
Disallow: /sources/
Disallow: /similar/
Disallow: /q/*/similar/
Disallow: /search/
Disallow: /admin/
Disallow: /accounts/
Disallow: /api/
Disallow: /htmx/
Disallow: /collections/private/

# ROLE 3 — User-initiated. A person asked an assistant to fetch this page, so
# it is readership, not extraction. Unrestricted, including /sources/: the
# provenance layer is the reason they were sent here.
User-agent: ChatGPT-User
User-agent: Claude-User
User-agent: Perplexity-User
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /
Disallow: /admin/
Disallow: /accounts/
Disallow: /htmx/
Disallow: /collections/private/

# Meta Crawlers - Differential Policy
User-agent: facebookexternalhit
User-agent: meta-webindexer
User-agent: meta-externalfetcher
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /
# Content sections: derived/duplicate pages, expensive to render.
# Must be repeated here — named groups do NOT inherit the wildcard group (RFC 9309).
Disallow: /tags/
Disallow: /sources/
Disallow: /similar/
Disallow: /q/*/similar/
Disallow: /search/
Disallow: /admin/
Disallow: /accounts/
Disallow: /api/
Disallow: /htmx/
Disallow: /collections/private/

User-agent: meta-externalagent
Content-Signal: ai-train=no
Disallow: /

# Zero-referral extraction crawlers.
# Measured over 7 days (Cloudflare AI Audit, 2026-08-08): Amazon 28.72k requests
# and 0 referrals. Amazon's crawlers feed Rufus and Alexa answers, which do not
# link back, so the corpus leaves and nothing returns. Compare Google 250:1,
# Bing 449:1, DuckDuckGo 0.6:1 — all kept.
User-agent: Amazonbot
User-agent: Amzn-SearchBot
Content-Signal: ai-train=no
Disallow: /

# Apple: Applebot itself powers Siri and Spotlight and can refer traffic, so it
# stays on the standard policy below. Applebot-Extended is the AI-training
# fetcher only, and training is declined site-wide.
User-agent: Applebot-Extended
Content-Signal: ai-train=no
Disallow: /

# Baidu: 1.07k requests and 0 referrals over the same window, against an
# English-language corpus with no Chinese-market audience.
User-agent: Baiduspider
Content-Signal: ai-train=no
Disallow: /

# Standard crawlers
User-agent: *
Content-Signal: ai-train=no, search=yes, ai-input=yes

# Allow indexing of main content
Allow: /
Allow: /o/
Allow: /q/
Allow: /privacy/
Allow: /terms/
Allow: /plans/
Allow: /developers/

# Disallow specific content sections
# Disallow: /originators/ - handled with noindex, follow tags header block on template
# Disallow: /trending/
Disallow: /tags/
Disallow: /sources/
Disallow: /similar/
Disallow: /q/*/similar/
Disallow: /search/

# Disallow admin, accounts, API endpoints, and other non-user-facing areas
Disallow: /admin/
Disallow: /accounts/
Disallow: /api/
Disallow: /oembed/
Disallow: /htmx/
Disallow: /qrawler/
Disallow: /invitations/
Disallow: /moderator/
Disallow: /collections/private/
Disallow: /404/
# Block search params (consistent with /search/ being blocked)
Disallow: /*?q=*
Disallow: /*?search_text=*
Disallow: /*?search_type=*

# Block sort/ordering (same content, different presentation)
Disallow: /*?sort=*
Disallow: /*?ordering=*
Disallow: /*?method=*

# Block UI preferences and limits
Disallow: /*?filter=*
Disallow: /*?paginate_by=*
Disallow: /*?limit=*

# Block auth/session parameters
Disallow: /*?next=*
Disallow: /*?auth=*
Disallow: /*?new=*
Disallow: /*?username=*
Disallow: /*?current=*
Disallow: /*?collection_slug=*

# Pagination (?page=) is NOT blocked - Google needs to crawl these
# Letter filter (?letter=) is NOT blocked - unique content subsets

# Sitemap location (authoritative domain)
Sitemap: https://quotewise.io/sitemap.xml