Crawler API

from scrapfly import ScrapflyClient, CrawlerConfig, Crawl
client = ScrapflyClient(key="API KEY")

crawl = Crawl(
    client,
    CrawlerConfig(
        url='https://example.com',
        # Only follow links matching this pattern
        include_only_paths=['/blog/*', '/docs/*'],
        # Skip these paths entirely
        exclude_paths=['/tags/*', '/author/*'],
        # Stay on the same host
        follow_external_links=False,
        page_limit=200,
        max_depth=5,
    )
).crawl().wait()

print(f"Crawled {crawl.status().state.urls_visited} pages")

import { ScrapflyClient, CrawlerConfig, Crawl } from 'jsr:@scrapfly/scrapfly-sdk';

const client = new ScrapflyClient({ key: "API KEY" });

const crawl = new Crawl(
  client,
  new CrawlerConfig({
    url: 'https://example.com',
    // Only follow links matching this pattern
    include_only_paths: ['/blog/*', '/docs/*'],
    // Skip these paths entirely
    exclude_paths: ['/tags/*', '/author/*'],
    // Stay on the same host
    follow_external_links: false,
    page_limit: 200,
    max_depth: 5,
  })
);

await crawl.start();
await crawl.wait();

const status = await crawl.status();
console.log(`Crawled ${status.state.urls_visited} pages`);

curl -X POST "https://api.scrapfly.io/crawl?key=$SCRAPFLY_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "url": "https://example.com",
    "include_only_paths": ["/blog/*", "/docs/*"],
    "exclude_paths": ["/tags/*", "/author/*"],
    "follow_external_links": false,
    "page_limit": 200,
    "max_depth": 5
  }'

from scrapfly import ScrapflyClient, CrawlerConfig, Crawl
client = ScrapflyClient(key="API KEY")

crawl = Crawl(
    client,
    CrawlerConfig(
        url='https://web-scraping.dev/products',
        page_limit=50,
        # Emit markdown + clean_html for every page,
        # ready for LLM / RAG pipelines.
        content_formats=['markdown', 'clean_html'],
    )
).crawl().wait()

# Stream markdown for every visited page via a URL pattern
for page in crawl.read_iter(pattern='*', format='markdown'):
    print(page.url, len(page.content), 'chars')

import { ScrapflyClient, CrawlerConfig, Crawl } from 'jsr:@scrapfly/scrapfly-sdk';

const client = new ScrapflyClient({ key: "API KEY" });

const crawl = new Crawl(
  client,
  new CrawlerConfig({
    url: 'https://web-scraping.dev/products',
    page_limit: 50,
    // Emit markdown + clean_html for every page,
    // ready for LLM / RAG pipelines.
    content_formats: ['markdown', 'clean_html'],
  })
);

await crawl.start();
await crawl.wait();

// Read markdown for the seed URL
const md = await crawl.read('https://web-scraping.dev/products', 'markdown');
console.log(`markdown length=${md?.length ?? 0} chars`);

curl -X POST "https://api.scrapfly.io/crawl?key=$SCRAPFLY_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "url": "https://web-scraping.dev/products",
    "page_limit": 50,
    "content_formats": ["markdown", "clean_html"]
  }'

from scrapfly import ScrapflyClient, CrawlerConfig, Crawl
client = ScrapflyClient(key="API KEY")

# Fire-and-forget: register a webhook in the Scrapfly dashboard
# ("my-crawl-endpoint"), then pass its name here.
crawl = Crawl(
    client,
    CrawlerConfig(
        url='https://web-scraping.dev/products',
        page_limit=500,
        webhook_name='my-crawl-endpoint',
        webhook_events=['crawler_url_visited', 'crawler_finished'],
    )
).crawl()  # returns immediately, no .wait() needed

print(f"Crawl started: {crawl.uuid}")

import { ScrapflyClient, CrawlerConfig, Crawl } from 'jsr:@scrapfly/scrapfly-sdk';

const client = new ScrapflyClient({ key: "API KEY" });

// Fire-and-forget: register a webhook in the Scrapfly dashboard
// ("my-crawl-endpoint"), then pass its name here.
const crawl = new Crawl(
  client,
  new CrawlerConfig({
    url: 'https://web-scraping.dev/products',
    page_limit: 500,
    webhook_name: 'my-crawl-endpoint',
    webhook_events: ['crawler_url_visited', 'crawler_finished'],
  })
);

await crawl.start();  // returns immediately, no .wait() needed
console.log(`Crawl started: ${crawl.uuid}`);

curl -X POST "https://api.scrapfly.io/crawl?key=$SCRAPFLY_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "url": "https://web-scraping.dev/products",
    "page_limit": 500,
    "webhook_name": "my-crawl-endpoint",
    "webhook_events": ["crawler_url_visited", "crawler_finished"]
  }'