# ===========================================
# Laconote - Robots.txt
# ===========================================
# Policy: allow retrieval (what earns citations and referral traffic), block
# training (what only feeds model weights). They are different crawlers at
# every vendor -- GPTBot trains, OAI-SearchBot retrieves.
#
# Consecutive User-agent lines share one group (RFC 9309), and a crawler reads
# only its most-specific matching group -- it does NOT inherit from '*'. So both
# the app-route Disallow list AND the Content-Signal reservation are repeated in
# every allowed group on purpose: a signal left only in '*' would be read by
# none of the crawlers we deliberately let in.
#
# There is deliberately no 'Allow:' line anywhere, of any form. RFC parsers take
# the longest match, so 'Allow: /' looks harmless -- but CPython's
# urllib.robotparser before 3.14, and anything modelled on it, is first-match and
# hands out whatever the Allow covers. Absence of a rule already means allowed,
# so an Allow line here can only ever cost us private routes.
#
# BLOCK groups come first on purpose: a parser that substring-matches product
# tokens would otherwise bind 'Applebot-Extended' to the 'Applebot' group. This
# helps first-match parsers and would hurt a last-match one; CPython, the
# canonical naive parser, is first-match, so this is the right trade. No ordering
# satisfies both -- do not 'fix' it back.
#
# This file is one half of the policy. The edge enforces the other half via a
# WAF custom rule (Security > Security rules > custom rules, 'AI Crawl Control -
# Block AI bots by User Agent'), which 403s Amazonbot, Bytespider, CCBot,
# ClaudeBot, GPTBot and meta-externalagent everywhere except /robots.txt -- the
# exception matters, or a blocked bot could never read this policy. Keep the two
# in agreement: a bot allowed here and 403'd there is invited in and shut out.
#
# Cloudflare's managed robots.txt is currently ON and PREPENDS its own groups.
# Our block set is a superset of its per-bot list, so every named agent resolves
# the same either way -- but it also prepends 'User-agent: *' with 'Allow: /',
# and CPython up to 3.13 keeps only the FIRST '*' group and discards ours
# entirely, leaking all 22 private prefixes below. (3.14 merges per-agent
# instead, which is why the leak stops there.) Turning managed off is therefore
# preferable, and this file no longer needs it: it emits its own Content-Signal.

# --- BLOCK: OpenAI / Anthropic model training. Blocking these does NOT remove Laconote
# from ChatGPT Search or Claude web search -- those use the *-SearchBot and
# *-User agents allowed below. (anthropic-ai is a legacy token Anthropic no
# longer documents; kept because it costs nothing.)
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
Disallow: /

# --- BLOCK: Google / Apple AI training tokens. These NEVER appear in server logs --
# Google states Google-Extended has no separate request user agent, and
# Applebot-Extended does not crawl. robots.txt is the only place they work,
# which is why the edge WAF rule cannot cover them and this file must.
# TRADE-OFF: Google-Extended also gates Gemini app and Vertex AI grounding,
# so blocking it forfeits Gemini citations. Deliberate, to keep ai-train=no
# consistent. Google Search and AI Overviews are unaffected: they crawl as
# Googlebot, which is allowed below.
User-agent: Google-Extended
User-agent: Applebot-Extended
Disallow: /

# --- BLOCK: Training corpora and bulk scrapers. Amazonbot is here on Amazon's own
# wording -- it 'may be used to train Amazon AI models' -- unlike Amzn-SearchBot
# and Amzn-User, which state they do not crawl for generative AI training and
# are allowed below. CloudflareBrowserRenderingCrawler is listed so our block
# set stays a superset of Cloudflare's managed one; without it, turning managed
# robots.txt off would silently un-block it.
User-agent: CCBot
User-agent: Bytespider
User-agent: Amazonbot
User-agent: meta-externalagent
User-agent: cohere-ai
User-agent: AI2Bot
User-agent: Timpibot
User-agent: Diffbot
User-agent: omgili
User-agent: omgilibot
User-agent: PanguBot
User-agent: Webzio-Extended
User-agent: CloudflareBrowserRenderingCrawler
Disallow: /

# --- ALLOW: OpenAI -- search index and user-initiated fetch (ChatGPT Search citations)
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
Content-Signal: search=yes,ai-train=no,use=reference
Disallow: /dashboard
Disallow: /upload
Disallow: /record
Disallow: /projects
Disallow: /shared-with-me
Disallow: /trash
Disallow: /settings
Disallow: /chat
Disallow: /meeting/
Disallow: /calendar
Disallow: /team/
Disallow: /payment/
Disallow: /s/
Disallow: /shared/
Disallow: /p/
Disallow: /login
Disallow: /register
Disallow: /verify-email
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /restore-account
Disallow: /oauth/consent

# --- ALLOW: Anthropic -- search index and user-initiated fetch (Claude web search)
User-agent: Claude-SearchBot
User-agent: Claude-User
Content-Signal: search=yes,ai-train=no,use=reference
Disallow: /dashboard
Disallow: /upload
Disallow: /record
Disallow: /projects
Disallow: /shared-with-me
Disallow: /trash
Disallow: /settings
Disallow: /chat
Disallow: /meeting/
Disallow: /calendar
Disallow: /team/
Disallow: /payment/
Disallow: /s/
Disallow: /shared/
Disallow: /p/
Disallow: /login
Disallow: /register
Disallow: /verify-email
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /restore-account
Disallow: /oauth/consent

# --- ALLOW: Perplexity -- index and user-initiated fetch
User-agent: PerplexityBot
User-agent: Perplexity-User
Content-Signal: search=yes,ai-train=no,use=reference
Disallow: /dashboard
Disallow: /upload
Disallow: /record
Disallow: /projects
Disallow: /shared-with-me
Disallow: /trash
Disallow: /settings
Disallow: /chat
Disallow: /meeting/
Disallow: /calendar
Disallow: /team/
Disallow: /payment/
Disallow: /s/
Disallow: /shared/
Disallow: /p/
Disallow: /login
Disallow: /register
Disallow: /verify-email
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /restore-account
Disallow: /oauth/consent

# --- ALLOW: Meta -- Meta AI search index and user-initiated fetch. Meta-ExternalFetcher
# fetches individual links at a user's request, the same class as ChatGPT-User,
# so it belongs here and not with meta-externalagent, which does train.
User-agent: Meta-WebIndexer
User-agent: Meta-ExternalFetcher
Content-Signal: search=yes,ai-train=no,use=reference
Disallow: /dashboard
Disallow: /upload
Disallow: /record
Disallow: /projects
Disallow: /shared-with-me
Disallow: /trash
Disallow: /settings
Disallow: /chat
Disallow: /meeting/
Disallow: /calendar
Disallow: /team/
Disallow: /payment/
Disallow: /s/
Disallow: /shared/
Disallow: /p/
Disallow: /login
Disallow: /register
Disallow: /verify-email
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /restore-account
Disallow: /oauth/consent

# --- ALLOW: Amazon -- search experiences and Alexa answers. Both document that they do
# not crawl content for generative AI model training.
User-agent: Amzn-SearchBot
User-agent: Amzn-User
Content-Signal: search=yes,ai-train=no,use=reference
Disallow: /dashboard
Disallow: /upload
Disallow: /record
Disallow: /projects
Disallow: /shared-with-me
Disallow: /trash
Disallow: /settings
Disallow: /chat
Disallow: /meeting/
Disallow: /calendar
Disallow: /team/
Disallow: /payment/
Disallow: /s/
Disallow: /shared/
Disallow: /p/
Disallow: /login
Disallow: /register
Disallow: /verify-email
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /restore-account
Disallow: /oauth/consent

# --- ALLOW: DuckDuckGo -- real-time answer assistant (not the DuckDuckBot index crawler)
User-agent: DuckAssistBot
Content-Signal: search=yes,ai-train=no,use=reference
Disallow: /dashboard
Disallow: /upload
Disallow: /record
Disallow: /projects
Disallow: /shared-with-me
Disallow: /trash
Disallow: /settings
Disallow: /chat
Disallow: /meeting/
Disallow: /calendar
Disallow: /team/
Disallow: /payment/
Disallow: /s/
Disallow: /shared/
Disallow: /p/
Disallow: /login
Disallow: /register
Disallow: /verify-email
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /restore-account
Disallow: /oauth/consent

# --- ALLOW: Classic search engines. Google AI Overviews and AI Mode are served from the
# regular Googlebot crawl, so this is what keeps Laconote eligible there.
User-agent: Googlebot
User-agent: Bingbot
User-agent: Applebot
Content-Signal: search=yes,ai-train=no,use=reference
Disallow: /dashboard
Disallow: /upload
Disallow: /record
Disallow: /projects
Disallow: /shared-with-me
Disallow: /trash
Disallow: /settings
Disallow: /chat
Disallow: /meeting/
Disallow: /calendar
Disallow: /team/
Disallow: /payment/
Disallow: /s/
Disallow: /shared/
Disallow: /p/
Disallow: /login
Disallow: /register
Disallow: /verify-email
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /restore-account
Disallow: /oauth/consent

# --- Everything else: allowed, app and token routes closed.
# Content-Signal is an express reservation of rights under Article 4 of EU
# Directive 2019/790.
User-agent: *
Content-Signal: search=yes,ai-train=no,use=reference
Disallow: /dashboard
Disallow: /upload
Disallow: /record
Disallow: /projects
Disallow: /shared-with-me
Disallow: /trash
Disallow: /settings
Disallow: /chat
Disallow: /meeting/
Disallow: /calendar
Disallow: /team/
Disallow: /payment/
Disallow: /s/
Disallow: /shared/
Disallow: /p/
Disallow: /login
Disallow: /register
Disallow: /verify-email
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /restore-account
Disallow: /oauth/consent

Sitemap: https://laconote.com/sitemap.xml

# LLM-readable site summaries. No engine has committed to reading these; they
# are cheap to keep and pointed at from here rather than relied upon.
# https://laconote.com/llms.txt
# https://laconote.com/llms-full.txt