# robots.txt for thingstodoinbismarck.com

# General crawlers
User-agent: *
Allow: /
Crawl-delay: 1

# Block API endpoints from crawling
Disallow: /api/

# AI/LLM Crawlers - Explicitly allow for training and browsing
# OpenAI GPT
User-agent: GPTBot
Allow: /

# Google Bard/Gemini
User-agent: Google-Extended
Allow: /

# Anthropic Claude
User-agent: anthropic-ai
Allow: /

# Common Crawl (used by many AI systems)
User-agent: CCBot
Allow: /

# Cohere AI
User-agent: cohere-ai
Allow: /

# Meta AI
User-agent: FacebookBot
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Sitemaps
Sitemap: https://thingstodoinbismarck.com/sitemap.xml

# LLM-optimized content index
# See https://thingstodoinbismarck.com/llms.txt for curated content guide