# Standard web crawlers — full access
User-agent: *
Disallow: /api/
Allow: /

# --- Generative/AI crawlers — explicitly allowed for GEO (Bölüm 8) ---
# OpenAI: used to train GPT models
User-agent: GPTBot
Allow: /
# OpenAI: powers real-time ChatGPT search citations
User-agent: OAI-SearchBot
Allow: /
# OpenAI: fetches a page when a user pastes/asks about a specific URL in ChatGPT
User-agent: ChatGPT-User
Allow: /
# Anthropic: used to train Claude models
User-agent: ClaudeBot
Allow: /
# Anthropic: fetches a page on a user's behalf inside Claude
User-agent: Claude-User
Allow: /
# Anthropic: powers Claude's live web search citations
User-agent: Claude-SearchBot
Allow: /
# Perplexity: training crawler
User-agent: PerplexityBot
Allow: /
# Perplexity: fetches a page on a user's behalf for live answers
User-agent: Perplexity-User
Allow: /
# Google: powers Search Generative Experience / AI Overviews training use
User-agent: Google-Extended
Allow: /
# Apple: powers Apple Intelligence / Siri answer training
User-agent: Applebot-Extended
Allow: /
# Amazon: general-purpose crawler, incl. Alexa/shopping answers
User-agent: Amazonbot
Allow: /
# ByteDance (TikTok/Doubao): training crawler
User-agent: Bytespider
Allow: /
# Common Crawl: open dataset used by many downstream LLMs
User-agent: CCbot
Allow: /
# Meta: training crawler for Meta AI
User-agent: Meta-ExternalAgent
Allow: /
# Cohere: training crawler
User-agent: cohere-ai
Allow: /
# You.com: training/answer crawler
User-agent: YouBot
Allow: /
# Diffbot: structured-data extraction, feeds multiple downstream AI products
User-agent: Diffbot
Allow: /

Sitemap: https://www.cookrangeapp.com/sitemap-index.xml