# robots.txt - thosuadien.com

# Allow all bots by default - except internal component templates
User-agent: *
Allow: /
Disallow: /components/

# === SEARCH ENGINES ===

# Google
User-agent: Googlebot
Allow: /

# Google Images
User-agent: Googlebot-Image
Allow: /

# Bing
User-agent: Bingbot
Allow: /

# Coc Coc (popular Vietnamese search engine)
User-agent: coccocbot-web
Allow: /

# Yandex
User-agent: YandexBot
Allow: /

# === AI BOTS - GEO (Generative Engine Optimization) ===
# Fully allowed so content can be crawled, trained on, and cited by AI

# OpenAI / ChatGPT (training)
User-agent: GPTBot
Allow: /

# OpenAI / ChatGPT (fetches when a user asks directly)
User-agent: ChatGPT-User
Allow: /

# OpenAI / ChatGPT Search (cites sources in search results)
User-agent: OAI-SearchBot
Allow: /

# Google Gemini / AI Overviews / Bard
User-agent: Google-Extended
Allow: /

# Anthropic / Claude (training - legacy)
User-agent: anthropic-ai
Allow: /

# Anthropic / Claude (data crawling)
User-agent: ClaudeBot
Allow: /

# Anthropic / Claude (fetches when a user asks directly)
User-agent: Claude-User
Allow: /

# Anthropic / Claude Search (cites sources)
User-agent: Claude-SearchBot
Allow: /

# Perplexity (data crawling)
User-agent: PerplexityBot
Allow: /

# Perplexity (fetches when a user asks directly)
User-agent: Perplexity-User
Allow: /

# Meta AI (training)
User-agent: Meta-ExternalAgent
Allow: /

# Meta AI (real-time fetch when a user asks)
User-agent: Meta-ExternalFetcher
Allow: /

# Facebook (link previews, article sharing)
User-agent: FacebookBot
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# Apple / Siri / Apple Intelligence
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# ByteDance / TikTok
User-agent: Bytespider
Allow: /

# Common Crawl (AI training dataset - reused by many LLMs)
User-agent: CCBot
Allow: /

# Diffbot (feeds structured data to many AI tools)
User-agent: Diffbot
Allow: /

# Amazon / Alexa
User-agent: Amazonbot
Allow: /

# You.com
User-agent: YouBot
Allow: /

# Sitemap
Sitemap: https://thosuadien.com/sitemap.xml