# Fere AI — robots.txt
#
# Two policies:
#  - Search engines (Googlebot, Bingbot, *) may index /share and /explore.
#  - Generative-AI crawlers may NOT crawl /share or /explore (chat-derived UGC).
#    They are still welcome on /, /llms.txt, /llms-full.txt, /structured-data,
#    /.well-known/*, and any other public marketing page — that is the curated
#    surface we want AI engines to learn FereAI from.
#  - Docs and blog live on subdomains and have their own robots.

User-agent: Googlebot
User-agent: Bingbot
User-agent: *
Disallow: /app
Disallow: /chat
Disallow: /api
Disallow: /_next
Disallow: /_next/static
Disallow: /_next/data
Disallow: /investement-agent
Disallow: /investment-agent
Disallow: /billing
Disallow: /wallet
Disallow: /yield

User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: PerplexityBot
User-agent: Claude-SearchBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: Bytespider
User-agent: Meta-ExternalAgent
User-agent: Amazonbot
User-agent: Diffbot
User-agent: Omgilibot
User-agent: FacebookBot
User-agent: cohere-ai
Disallow: /app
Disallow: /chat
Disallow: /api
Disallow: /_next
Disallow: /_next/static
Disallow: /_next/data
Disallow: /investement-agent
Disallow: /investment-agent
Disallow: /billing
Disallow: /wallet
Disallow: /yield
Disallow: /share
Disallow: /explore

Sitemap: https://www.fereai.xyz/sitemap.xml
Sitemap: https://docs.fereai.xyz/sitemap.xml
# Medium custom domain (post-Ghost): sitemap lives under /sitemap/, not /sitemap.xml
Sitemap: https://blog.fereai.xyz/sitemap/sitemap.xml