# robots.txt — bestaeoskill.com
# Updated 2026-05-01
# Policy: explicitly allow all major AI crawlers and traditional search engines.
# We want to be cited. Block only obvious abuse vectors.

# ============== Traditional search ==============
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: YandexBot
Allow: /

# ============== Google AI surfaces ==============
# Powers Bard / Gemini training and grounding signals.
User-agent: Google-Extended
Allow: /

# Powers AI Overviews and Search Generative Experience.
User-agent: GoogleOther
Allow: /

# ============== OpenAI ==============
# ChatGPT browsing and training.
User-agent: GPTBot
Allow: /

# ChatGPT real-time browsing for user queries.
User-agent: ChatGPT-User
Allow: /

# OpenAI search index (SearchGPT).
User-agent: OAI-SearchBot
Allow: /

# ============== Anthropic ==============
# Claude search/browse.
User-agent: ClaudeBot
Allow: /

# Anthropic web crawler.
User-agent: anthropic-ai
Allow: /

# Claude.ai user-initiated fetches.
User-agent: Claude-Web
Allow: /

# Claude-User: real-time fetch via Claude tool use.
User-agent: Claude-User
Allow: /

# Claude-SearchBot.
User-agent: Claude-SearchBot
Allow: /

# ============== Perplexity ==============
User-agent: PerplexityBot
Allow: /

# Perplexity user-initiated fetches.
User-agent: Perplexity-User
Allow: /

# ============== Other AI search ==============
# You.com.
User-agent: YouBot
Allow: /

# Cohere.
User-agent: cohere-ai
Allow: /

# Meta / Llama.
User-agent: FacebookBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

# Apple Intelligence.
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Mistral.
User-agent: MistralAI-User
Allow: /

# Common Crawl (used by many LLMs).
User-agent: CCBot
Allow: /

# Bytedance.
User-agent: Bytespider
Allow: /

# Diffbot.
User-agent: Diffbot
Allow: /

# Amazon AI.
User-agent: Amazonbot
Allow: /

# ============== Default ==============
User-agent: *
Allow: /
Disallow: /admin/
Disallow: /api/

# ============== Sitemap & AI-discovery ==============
Sitemap: https://bestaeoskill.com/sitemap.xml

# llms.txt — llmstxt.org spec
# Sites can crawl: https://bestaeoskill.com/llms.txt
# Full content: https://bestaeoskill.com/llms-full.txt
# AI permissions: https://bestaeoskill.com/.well-known/ai.txt
# RSS: https://bestaeoskill.com/feed.xml