Retrieving Crawler Results

# Homepage

Welcome to our site! We provide the best products and services for your needs.

## Our Services

- Web Development
- Mobile Apps
- Cloud Solutions

Contact us today to get started!

HTTP/1.1 200 OK
Content-Type: multipart/related; boundary=custom123
Content-Location: https://example.com

--custom123
Content-Type: text/markdown

# Homepage

Welcome to our site...
--custom123
Content-Type: text/plain

Homepage

Welcome to our site...
--custom123--

HTTP/1.1 200 OK
Content-Type: multipart/related; boundary=abc123
X-Scrapfly-Requested-URLs: 3
X-Scrapfly-Found-URLs: 3

--abc123
Content-Type: text/markdown
Content-Location: https://example.com/page1

# Page 1

Content here...
--abc123
Content-Type: text/plain
Content-Location: https://example.com/page1

Page 1 Content here...
--abc123
Content-Type: text/markdown
Content-Location: https://example.com/page2

# Page 2

Different content...
--abc123--

{
  "contents": {
    "https://example.com": "# Homepage\n\nWelcome to our site...",
    "https://example.com/about": "# About Us\n\nWe are a company...",
    "https://example.com/contact": "# Contact\n\nReach us at..."
  }
}

import requests
import time

API_KEY = ""
BASE_URL = "https://api.scrapfly.io"

# Step 1: Create crawler
response = requests.post(
    f"{BASE_URL}/crawl",
    params={"key": API_KEY},
    json={
        "url": "https://web-scraping.dev/products",
        "page_limit": 25
    }
)
crawler_data = response.json()
uuid = crawler_data["crawler_uuid"]
print(f"Crawler UUID: {uuid}")

# Step 2: Poll status until complete
while True:
    response = requests.get(
        f"{BASE_URL}/crawl/{uuid}/status",
        params={"key": API_KEY}
    )
    status = response.json()

    is_finished = status.get("is_finished", False)
    is_success = status.get("is_success", False)

    print(f"Status check: is_finished={is_finished}, is_success={is_success}")

    if is_finished:
        if is_success:
            print("Crawler completed successfully!")
            break
        else:
            print("Crawler failed!")
            exit(1)

    time.sleep(5)

# Step 3: Download results
print("Downloading WARC artifact...")
warc_response = requests.get(
    f"{BASE_URL}/crawl/{uuid}/artifact",
    params={"key": API_KEY, "type": "warc"}
)
with open("crawl.warc.gz", "wb") as f:
    f.write(warc_response.content)

print("Getting markdown content...")
content_response = requests.get(
    f"{BASE_URL}/crawl/{uuid}/contents",
    params={"key": API_KEY, "format": "markdown"}
)
with open("content.json", "w") as f:
    f.write(content_response.text)

print("Done!")

const API_KEY = "";
const BASE_URL = "https://api.scrapfly.io";

async function runCrawler() {
    // Step 1: Create crawler
    const createResponse = await fetch(`${BASE_URL}/crawl?key=${API_KEY}`, {
        method: "POST",
        headers: {
            "Content-Type": "application/json"
        },
        body: JSON.stringify({
            url: "https://web-scraping.dev/products",
            page_limit: 25
        })
    });

    const crawlerData = await createResponse.json();
    const uuid = crawlerData.crawler_uuid;
    console.log(`Crawler UUID: ${uuid}`);

    // Step 2: Poll status until complete
    while (true) {
        const statusResponse = await fetch(
            `${BASE_URL}/crawl/${uuid}/status?key=${API_KEY}`
        );
        const status = await statusResponse.json();

        const isFinished = status.is_finished || false;
        const isSuccess = status.is_success || false;

        console.log(`Status check: is_finished=${isFinished}, is_success=${isSuccess}`);

        if (isFinished) {
            if (isSuccess) {
                console.log("Crawler completed successfully!");
                break;
            } else {
                console.log("Crawler failed!");
                process.exit(1);
            }
        }

        await new Promise(resolve => setTimeout(resolve, 5000));
    }

    // Step 3: Download results
    console.log("Downloading WARC artifact...");
    const warcResponse = await fetch(
        `${BASE_URL}/crawl/${uuid}/artifact?key=${API_KEY}&type=warc`
    );
    const warcBlob = await warcResponse.blob();
    // In Node.js, use fs.writeFileSync to save
    // In browser, use URL.createObjectURL to download

    console.log("Getting markdown content...");
    const contentResponse = await fetch(
        `${BASE_URL}/crawl/${uuid}/contents?key=${API_KEY}&format=markdown`
    );
    const content = await contentResponse.json();
    // Save content.json to file

    console.log("Done!");
}

runCrawler().catch(console.error);