# Castro Coverage crawler policy (updated 2026-10-21).
# Search engines and answer engines may read and cite the site. Bulk AI-training crawlers are blocked.
# robots.txt only stops crawlers that obey it; terms.html section 6 and the TDM-Reservation header
# state the same rule for everyone else. Copyright notice and terms: https://castrocoverage.com/terms.html#intellectual-property

User-agent: *
Allow: /

# Answer engines, AI search, and user-triggered fetches: allowed (they cite and link to the site)
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

# Bulk AI-training and data-harvesting crawlers: not allowed
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: cohere-training-data-crawler
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: omgili
Disallow: /

User-agent: omgilibot
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: PanguBot
Disallow: /

Sitemap: https://castrocoverage.com/sitemap.xml