# Empire Enthusiast - Robots.txt
# AI-Optimized for autonomous agents and search crawlers

# General Rules
User-agent: *
Allow: /
Allow: /pages/
Allow: /api/
Allow: /images/
Disallow: /office.html
Disallow: /leads/
Disallow: /*.ps1$
Disallow: /*.py$

# Sitemap
Sitemap: https://empireenthusiast.com/sitemap.xml

# LLM/AI Agent Information
# See /llms.txt for detailed AI agent guidance

# OpenAI GPT Crawler
User-agent: GPTBot
Allow: /
Allow: /api/
Allow: /llms.txt

# Anthropic Claude Crawler
User-agent: ClaudeBot
Allow: /
Allow: /api/
Allow: /llms.txt

User-agent: Claude-Web
Allow: /
Allow: /api/
Allow: /llms.txt

# Google AI
User-agent: Google-Extended
Allow: /
Allow: /api/

# Bing/Microsoft AI
User-agent: Bingbot
Allow: /

# Common AI Research Crawlers
User-agent: CCBot
Allow: /

User-agent: ChatGPT-User
Allow: /
Allow: /api/

User-agent: PerplexityBot
Allow: /
Allow: /api/

User-agent: Bytespider
Allow: /

# Standard Search Engines
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /images/

User-agent: bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

# Crawl Delay Recommendations
# AI agents are encouraged to respect rate limits
# Suggested: 1 request per second for bulk operations

# API Access Points for AI Agents
# GET /api/knowledge.json - Full knowledge graph
# GET /api/articles.json - Article metadata
# GET /llms.txt - AI agent documentation
# GET /sitemap.xml - Site structure
