# robots.txt for madiot.fr/knowledge_base/ in short:
#
# /public/      is open to search engines and ai crawlers
# /search-only/ is open to search engines, closed to ai
# /ai-only/     is open to ai crawlers, closed to search engines
# /private/     is closed to everyone


# default

User-agent: *
Disallow: /private/

Allow: /public/
Allow: /search-only/
Allow: /ai-only/


# no search in AI

User-agent: Googlebot
Disallow: /ai-only/
Allow: /search-only/
Allow: /public/

User-agent: Bingbot
Disallow: /ai-only/
Allow: /search-only/
Allow: /public/

User-agent: Slurp
Disallow: /ai-only/
Allow: /search-only/
Allow: /public/

User-agent: DuckDuckBot
Disallow: /ai-only/
Allow: /search-only/
Allow: /public/

User-agent: Baiduspider
Disallow: /ai-only/

User-agent: Yandex
Disallow: /ai-only/


# no AI in search

# OpenAI
User-agent: GPTBot
Disallow: /search-only/

User-agent: ChatGPT-User
Disallow: /search-only/

# Anthropic
User-agent: ClaudeBot
Disallow: /search-only/

User-agent: anthropic-ai
Disallow: /search-only/

# Google AI training (separate from Google search indexing)
User-agent: Google-Extended
Disallow: /search-only/

# Perplexity
User-agent: PerplexityBot
Disallow: /search-only/

# Common Crawl
User-agent: CCBot
Disallow: /search-only/

# ByteDance / TikTok crawler
User-agent: Bytespider
Disallow: /search-only/

# Amazon AI crawler
User-agent: Amazonbot
Disallow: /search-only/

# Apple AI training crawler
User-agent: Applebot-Extended
Disallow: /search-only/

# Meta AI
User-agent: Meta-ExternalAgent
Disallow: /search-only/

User-agent: meta-externalfetcher
Disallow: /search-only/


# no

User-agent: *
Disallow: /private/


# throttling

User-agent: Bingbot
Crawl-delay: 2

User-agent: Yandex
Crawl-delay: 2