User-agent: *
Allow: /

Disallow: /dashboard
Disallow: /login
Disallow: /register
Disallow: /account
Disallow: /api-keys
Disallow: /playground
Disallow: /credits
Disallow: /_debug
Disallow: /brand

Sitemap: https://spider.cloud/sitemap-index.xml
Sitemap: https://spider.cloud/sitemap-scrapers.xml

# Docs for language models, in the llms.txt format (llmstxt.org). robots.txt
# has no directive for these, so they stay comments. The homepage links the
# index from its head.
# https://spider.cloud/llms.txt
# https://spider.cloud/llms-full.txt

# AI Search & Assistant Crawlers
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: YouBot
Allow: /

User-agent: PhindBot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Meta-ExternalAgent
Allow: /