# 121 Group AI and search crawler policy.
#
# We sell GEO (Generative Engine Optimisation), so the crawlers that decide
# whether we get cited in AI answers are explicitly welcome. The ones that only
# bulk-harvest for training datasets, with no citation path back to us, are not.
#
# Cloudflare's managed robots.txt block was disabled on this zone so that this
# file is the single source of truth. See 121group-relaunch-golive memory.

# --- Search engines ---
User-agent: Googlebot
Allow: /
Disallow: /api/
Disallow: /cdn-cgi/

User-agent: bingbot
Allow: /

# --- AI assistants and answer engines: ALLOWED, these cite sources ---
# OpenAI: search index + user-initiated fetches
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: GPTBot
Allow: /

# Anthropic
User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Google AI Overviews and Gemini grounding
User-agent: Google-Extended
Allow: /

# Microsoft Copilot
User-agent: msnbot
Allow: /

# Apple Intelligence
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Meta AI
User-agent: meta-externalagent
Allow: /

# --- Bulk dataset scrapers: no citation, no referral, no benefit ---
User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

# --- Everything else ---
User-agent: *
Allow: /
Disallow: /api/
Disallow: /cdn-cgi/

Sitemap: https://121group.io/sitemap.xml