User-agent: *
Allow: /

# Private / app surfaces — keep aligned with client/src/lib/useSEO.ts pathImpliesNoIndex
Disallow: /dashboard/
Disallow: /admin/
# Public agent API + discovery files stay crawlable (longest match beats Disallow: /api/)
Allow: /api/agent/
Allow: /openapi.json
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /auth/
Disallow: /checkout/
Disallow: /plan-change/
Disallow: /quote/
Disallow: /login
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /marketing/

# ── AI / LLM crawlers — explicitly welcome on marketing surfaces ──
# OpenAI (ChatGPT search + GPT training)
User-agent: GPTBot
Allow: /
Disallow: /dashboard/
Disallow: /admin/
Allow: /api/agent/
Allow: /openapi.json
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /auth/
Disallow: /checkout/
Disallow: /plan-change/
Disallow: /quote/
Disallow: /login
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /marketing/

User-agent: OAI-SearchBot
Allow: /
Disallow: /dashboard/
Disallow: /admin/
Allow: /api/agent/
Allow: /openapi.json
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /auth/
Disallow: /checkout/
Disallow: /plan-change/
Disallow: /quote/
Disallow: /login
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /marketing/

User-agent: ChatGPT-User
Allow: /
Disallow: /dashboard/
Disallow: /admin/
Allow: /api/agent/
Allow: /openapi.json
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /auth/
Disallow: /checkout/
Disallow: /plan-change/
Disallow: /quote/
Disallow: /login
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /marketing/

# Anthropic (Claude)
User-agent: ClaudeBot
Allow: /
Disallow: /dashboard/
Disallow: /admin/
Allow: /api/agent/
Allow: /openapi.json
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /auth/
Disallow: /checkout/
Disallow: /plan-change/
Disallow: /quote/
Disallow: /login
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /marketing/

User-agent: anthropic-ai
Allow: /
Disallow: /dashboard/
Disallow: /admin/
Allow: /api/agent/
Allow: /openapi.json
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /auth/
Disallow: /checkout/
Disallow: /plan-change/
Disallow: /quote/
Disallow: /login
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /marketing/

User-agent: Claude-Web
Allow: /
Disallow: /dashboard/
Disallow: /admin/
Allow: /api/agent/
Allow: /openapi.json
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /auth/
Disallow: /checkout/
Disallow: /plan-change/
Disallow: /quote/
Disallow: /login
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /marketing/

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /dashboard/
Disallow: /admin/
Allow: /api/agent/
Allow: /openapi.json
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /auth/
Disallow: /checkout/
Disallow: /plan-change/
Disallow: /quote/
Disallow: /login
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /marketing/

User-agent: Perplexity-User
Allow: /
Disallow: /dashboard/
Disallow: /admin/
Allow: /api/agent/
Allow: /openapi.json
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /auth/
Disallow: /checkout/
Disallow: /plan-change/
Disallow: /quote/
Disallow: /login
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /marketing/

# Google AI (Gemini, AI Overviews — separate from regular Googlebot)
User-agent: Google-Extended
Allow: /
Disallow: /dashboard/
Disallow: /admin/
Allow: /api/agent/
Allow: /openapi.json
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /auth/
Disallow: /checkout/
Disallow: /plan-change/
Disallow: /quote/
Disallow: /login
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /marketing/

# Apple Intelligence
User-agent: Applebot-Extended
Allow: /
Disallow: /dashboard/
Disallow: /admin/
Allow: /api/agent/
Allow: /openapi.json
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /auth/
Disallow: /checkout/
Disallow: /plan-change/
Disallow: /quote/
Disallow: /login
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /marketing/

# Common Crawl (used by many LLM training pipelines)
User-agent: CCBot
Allow: /
Disallow: /dashboard/
Disallow: /admin/
Allow: /api/agent/
Allow: /openapi.json
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /auth/
Disallow: /checkout/
Disallow: /plan-change/
Disallow: /quote/
Disallow: /login
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /marketing/

# You.com / Phind / DuckAssist / Bytespider — generic AI search crawlers
User-agent: YouBot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Bytespider
Allow: /

User-agent: Diffbot
Allow: /

Sitemap: https://www.whiteglovecontent.com/sitemap.xml

# Blog RSS feeds — auto-generated from /blog seed posts
# Newer crawlers (Bing, Yandex, AI search engines) discover content via feed declarations.
# Sitemap: https://www.whiteglovecontent.com/feed.xml
# Sitemap: https://www.whiteglovecontent.com/feed.json