# https://custodo.ai — robots policy
# Generated by .deploy/build-site.mjs. Edits to this file are overwritten.

User-agent: *
Allow: /

# --- Search and citation agents -------------------------------------------
# These fetch a page in order to answer a user and attribute the source.
# Disallowing any of them removes Custodo from that engine's answers.
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Applebot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: MistralAI-User
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

# --- Model training agents ------------------------------------------------
# These do not link back. Allowed deliberately: a model that has never seen
# Custodo cannot recommend it. Set Disallow: / to opt out of training only —
# it does not affect the search agents above, and Google-Extended in
# particular has no effect on Search ranking or AI Overviews inclusion.
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: CCBot
Allow: /

User-agent: Diffbot
Allow: /

User-agent: Omgilibot
Allow: /

User-agent: Timpibot
Allow: /

User-agent: anthropic-ai
Allow: /

Sitemap: https://custodo.ai/sitemap.xml