Octoparse to Scrapfly Migration Guide

"""
Scrapfly Product Scraper
Replaces Octoparse point-and-click workflow with cloud-native API
"""
from scrapfly import ScrapflyClient, ScrapeConfig, ExtractionConfig
import json

# Initialize client (no desktop app needed!)
client = ScrapflyClient(key="YOUR_API_KEY")

def scrape_product_page(url: str) -> dict:
    """Scrape a single product page with anti-bot bypass"""

    # Fetch the page with Cloudflare bypass
    result = client.scrape(ScrapeConfig(
        url=url,
        render_js=True,      # Render JavaScript (like Octoparse)
        asp=True,            # Anti-bot bypass (Octoparse can't do this)
        country="us",        # Geo-targeting
        # Handle "Load More" button if needed:
        js_scenario=[
            {"wait_for_selector": {"selector": ".product-details"}},
            {"click": {"selector": ".show-more", "ignore_if_not_visible": True}},
            {"wait": 1000}
        ]
    ))

    # Extract product data automatically (no manual selectors!)
    extracted = client.extract(ExtractionConfig(
        content=result.content,
        content_type="text/html",
        extraction_model="product"
    ))

    return extracted.data

def scrape_product_listing(listing_url: str, max_pages: int = 5) -> list:
    """Scrape multiple products from listing page with pagination"""

    products = []

    for page in range(1, max_pages + 1):
        url = f"{listing_url}?page={page}"
        print(f"Scraping page {page}...")

        result = client.scrape(ScrapeConfig(
            url=url,
            render_js=True,
            asp=True
        ))

        # Extract all products on this page
        extracted = client.extract(ExtractionConfig(
            content=result.content,
            content_type="text/html",
            extraction_model="product_listing"
        ))

        if not extracted.data:
            break  # No more products

        products.extend(extracted.data)

    return products

# Run the scraper
if __name__ == "__main__":
    # Single product
    product = scrape_product_page("https://web-scraping.dev/product/123")
    print(json.dumps(product, indent=2))

    # Product listing (replaces Octoparse pagination)
    products = scrape_product_listing("https://web-scraping.dev/products")
    print(f"Scraped {len(products)} products")

    # Export to JSON (replaces Octoparse CSV export)
    with open("products.json", "w") as f:
        json.dump(products, f, indent=2)