--- a/robots.txt +++ b/robots.txt @@ -1,35 +1,44 @@ # robots.txt for doany.ai -# Last updated: 2025-11-14 +# Last updated: 2026-04-12 +# Allow standard search engines +User-agent: Googlebot +Allow: / + +User-agent: Bingbot +Allow: / + +# Allow AI search crawlers — these platforms cite sources in AI-generated answers +User-agent: GPTBot +Allow: / + +User-agent: ChatGPT-User +Allow: / + +User-agent: PerplexityBot +Allow: / + +User-agent: ClaudeBot +Allow: / + +User-agent: anthropic-ai +Allow: / + +User-agent: Google-Extended +Allow: / + +# Block training-only crawlers (don't contribute to search citations) +User-agent: CCBot +Disallow: / + +User-agent: Bytespider +Disallow: / + +# General User-agent: * Allow: / Disallow: /admin/ Disallow: /internal/ Disallow: /api/v1/ -# Block AI training crawlers -User-agent: GPTBot -Disallow: / - -User-agent: ChatGPT-User -Disallow: / - -User-agent: CCBot -Disallow: / - -User-agent: Google-Extended -Disallow: / - -User-agent: PerplexityBot -Disallow: / - -User-agent: ClaudeBot -Disallow: / - -User-agent: anthropic-ai -Disallow: / - -User-agent: Bytespider -Disallow: / - Sitemap: https://doany.ai/sitemap.xml