# Crypto Horse Club — robots.txt
# Effective: 2026-05-07
#
# Policy summary:
#
#   1. General search engines (Google, Bing, DuckDuckGo, Apple, etc.)
#      and AI search/answer crawlers (OAI-SearchBot, ChatGPT-User,
#      PerplexityBot, Bingbot for Copilot, etc.) are WELCOME and
#      explicitly ALLOWED. We want CHC to surface when someone asks
#      an AI assistant about crypto trading mentorship.
#
#   2. AI TRAINING crawlers — bots whose purpose is to scrape content
#      to bake into model weights, fine-tunes, or persistent embedding
#      indexes — are BLOCKED. The Horse Power EULA Section 13 carries
#      the contractual reservation; this file is the machine-readable
#      equivalent.
#
#   3. /api/, /webhooks/, /admin*, and authenticated/transactional
#      paths are blocked for everyone — no SEO value, no point
#      exposing them.

# ====================================================================
# Default — everyone (general crawlers + AI search/answer)
# Block authenticated and transactional paths only.
# ====================================================================
User-agent: *
Allow: /
Allow: /signup.html
Allow: /img/
Allow: /css/
Allow: /legal/

Disallow: /api/
Disallow: /webhooks/
Disallow: /r/
Disallow: /healthz
Disallow: /version
Disallow: /login.html
Disallow: /admin-login.html
Disallow: /admin.html
Disallow: /dashboard.html
Disallow: /account.html
Disallow: /thank-you.html
Disallow: /buy.html

# ====================================================================
# AI TRAINING crawlers — blocked.
# Bots whose primary or stated purpose is to gather data for inclusion
# in machine-learning model training corpora.
# ====================================================================

# OpenAI training (distinct from OAI-SearchBot which is allowed below)
User-agent: GPTBot
Disallow: /

# Google's generative-AI training crawler (distinct from Googlebot,
# which we allow because it powers normal Search AND AI Overviews).
User-agent: Google-Extended
Disallow: /

# Anthropic training
User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

# Common Crawl — feeds many open-source training datasets
User-agent: CCBot
Disallow: /

User-agent: cohere-ai
Disallow: /

# ByteDance / TikTok
User-agent: Bytespider
Disallow: /

# Meta / Facebook training crawlers
User-agent: FacebookBot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Meta-ExternalFetcher
Disallow: /

# Apple's training-only crawler (the regular Applebot is allowed below
# because it powers Siri Suggestions and Apple Intelligence retrieval)
User-agent: Applebot-Extended
Disallow: /

# Amazon
User-agent: Amazonbot
Disallow: /

# Allen Institute / Diffbot / aggregators
User-agent: AI2Bot
Disallow: /

User-agent: Ai2Bot-Dolma
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Omgili
Disallow: /

User-agent: OmgiliBot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: PanguBot
Disallow: /

User-agent: Sidetrade indexer bot
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: VelenPublicWebCrawler
Disallow: /

User-agent: webzio-extended
Disallow: /

User-agent: DuckAssistBot
Disallow: /

User-agent: YouBot
Disallow: /

User-agent: Scrapy
Disallow: /

# ====================================================================
# AI SEARCH / ANSWER crawlers — explicitly ALLOWED.
# Listed here to make the policy explicit (User-agent:* already
# permits them, but explicit beats implicit).
# ====================================================================

# OpenAI ChatGPT search (NOT GPTBot)
User-agent: OAI-SearchBot
Allow: /

# OpenAI user-initiated browsing
User-agent: ChatGPT-User
Allow: /

# Anthropic Claude when serving a human user's in-session request
User-agent: Claude-User
Allow: /

# Perplexity index + answer
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Apple Search + Siri Suggestions + Apple Intelligence (live retrieval)
User-agent: Applebot
Allow: /

# Microsoft Bing — powers Bing search, Copilot, and Edge sidebar
User-agent: Bingbot
Allow: /

# Google Search — feeds AI Overviews and standard search results
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /

# Yahoo
User-agent: Slurp
Allow: /

# Yandex
User-agent: YandexBot
Allow: /

# Baidu
User-agent: Baiduspider
Allow: /

# ====================================================================
# Sitemap pointer
# ====================================================================
Sitemap: https://membership.cryptohorseclub.com/sitemap.xml
Sitemap: https://affiliate.cryptohorseclub.com/sitemap.xml