# ============================================================================
# robots.txt — https://indexchecker.online
# Generated from sitemap architecture (/robots-llms-from-sitemap)
# ============================================================================

# Default rule: Allow all legitimate crawlers
User-agent: *
Allow: /
Disallow: /api/
Disallow: /_astro/
Disallow: /404

# ============================================================================
# Major Search Engines — Full Access
# ============================================================================
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

# ============================================================================
# AI / LLM Crawlers — Selective Access Policy (RFC 9309 Precedence)
# Allow AI search & citation bots that drive attribution traffic
# ============================================================================

# OpenAI (ChatGPT Search & browsing — provides source links)
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Google AI (Gemini grounding & AI Overviews — drives search citations)
User-agent: Google-Extended
Allow: /

# Anthropic (Claude web search & citations)
User-agent: Anthropic-ai
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

# Perplexity AI (citation-first answer engine)
User-agent: PerplexityBot
Allow: /

# Apple Intelligence
User-agent: Applebot-Extended
Allow: /

# ============================================================================
# Non-Attributing Bulk Scrapers — Blocked
# ============================================================================

# Common Crawl (bulk training corpus without attribution)
User-agent: CCBot
Disallow: /

# ByteDance scraper
User-agent: Bytespider
Disallow: /

# Huawei PetalBot aggressive crawler
User-agent: PetalBot
Disallow: /

# ============================================================================
# Sitemaps & LLM Discovery
# ============================================================================
Sitemap: https://indexchecker.online/sitemap-index.xml

# LLM Specification Files (https://llmstxt.org)
# LLMs-Full: https://indexchecker.online/llms-full.txt
# LLMs: https://indexchecker.online/llms.txt