User-agent: googlebot-news
Allow: /blog
Sitemap: https://scrapfly.io/blog/sitemap.xml

# AI search agents — public docs, marketing, and blog are open
# These crawlers typically resolve user prompts in real time
User-agent: ChatGPT-User
Allow: /
Disallow: /workspace/
Disallow: /dashboard
Disallow: /dashboard/

User-agent: OAI-SearchBot
Allow: /
Disallow: /workspace/
Disallow: /dashboard
Disallow: /dashboard/

User-agent: PerplexityBot
Allow: /
Disallow: /workspace/
Disallow: /dashboard
Disallow: /dashboard/

User-agent: Perplexity-User
Allow: /
Disallow: /workspace/
Disallow: /dashboard
Disallow: /dashboard/

User-agent: Claude-Web
Allow: /
Disallow: /workspace/
Disallow: /dashboard
Disallow: /dashboard/

User-agent: ClaudeBot
Allow: /
Disallow: /workspace/
Disallow: /dashboard
Disallow: /dashboard/

User-agent: Google-Extended
Allow: /
Disallow: /workspace/
Disallow: /dashboard
Disallow: /dashboard/

User-agent: Applebot-Extended
Allow: /
Disallow: /workspace/
Disallow: /dashboard
Disallow: /dashboard/

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: Omgili
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Meta-ExternalFetcher
Disallow: /

User-agent: *
Disallow: /workspace/*
Disallow: /dashboard
Disallow: /dashboard/*
Disallow: /api.scrapfly.io
Disallow: /*?*key=
Disallow: /*?*iframe=1
Disallow: /blog/ghost/
Disallow: /blog/p/
Disallow: /blog/email/
Disallow: /blog/r/
# .md variants of /docs/* pages are the same content as the canonical HTML
# (served as text/markdown for LLM consumption). They carry X-Robots-Tag:
# noindex on the response, but disallow them here too so SEO crawlers
# don't waste crawl-budget fetching them. AI-search agents (listed above)
# remain free to fetch them — they have their own Allow: / rules.
Disallow: /docs/*.md$
Disallow: /docs/*.md?
Disallow: /docs/llms.txt
Disallow: /*?view=markdown
Sitemap: https://scrapfly.io/sitemap.xml