# Robots.txt for HowToCSV
# Optimized for search engines and LLM crawlers

# All Bots (Default)
User-agent: *
Allow: /
Crawl-delay: 0.5

# Host
Host: https://howtocsv.com

# Sitemaps
Sitemap: https://howtocsv.com/sitemap.xml
Sitemap: https://howtocsv.com/llm-sitemap.xml

# LLM-optimized content
# Special files for AI/LLM crawlers with comprehensive content
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /ai.txt
Allow: /ai.json
Allow: /.well-known/ai-plugin.json

# OpenAI GPTBot
User-agent: GPTBot
Allow: /
Crawl-delay: 0.5

# Google Gemini Bot
User-agent: Google-Extended
Allow: /
Crawl-delay: 0.5

# Anthropic Claude Bot
User-agent: anthropic-ai
Allow: /
Crawl-delay: 0.5

# Common Crawl (used by many LLMs)
User-agent: CCBot
Allow: /
Crawl-delay: 1

# Bing Bot (used by Microsoft Copilot)
User-agent: bingbot
Allow: /
Crawl-delay: 0.5

# Meta AI
User-agent: FacebookBot
Allow: /
Crawl-delay: 1

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Crawl-delay: 0.5

# Cohere
User-agent: cohere-ai
Allow: /
Crawl-delay: 0.5