# MeilleursStructures.com - Robots.txt
# Comparateur indépendant de produits structurés

# Allow all crawlers (search engines & AI)
User-agent: *
Allow: /
Disallow: /cdn-cgi/
Disallow: /go/

# Google
User-agent: Googlebot
Allow: /

# Bing
User-agent: Bingbot
Allow: /

# OpenAI GPTBot - Allow for AI training and ChatGPT
User-agent: GPTBot
Allow: /

# Google Gemini / Bard
User-agent: Google-Extended
Allow: /

# Anthropic Claude
User-agent: anthropic-ai
Allow: /
User-agent: Claude-Web
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Meta AI
User-agent: FacebookBot
Allow: /

# Apple (Siri, Spotlight)
User-agent: Applebot
Allow: /

# Common Crawl (used by many AI models)
User-agent: CCBot
Allow: /

# Cohere AI
User-agent: cohere-ai
Allow: /

# Amazon / Alexa
User-agent: Amazonbot
Allow: /

# Sitemap
Sitemap: https://meilleursstructures.com/sitemap-index.xml