Playwright Integration

View as markdown

Playwright is Microsoft's modern browser automation library supporting Chromium, Firefox, and WebKit. Connect it to Scrapfly Cloud Browser for scalable automation with built-in proxies and fingerprinting.

Beta Feature: Cloud Browser is currently in beta.

Installation & Quick Start

Install Playwright and connect to Cloud Browser:

Installation

npm install playwright

Quick Start Example

const { chromium } = require('playwright');

const API_KEY = '';
const BROWSER_WS = `wss://browser.scrapfly.io?api_key=${API_KEY}&proxy_pool=datacenter&os=linux`;

async function run() {
    let browser = null;
    try {
        // Connect to Cloud Browser
        browser = await chromium.connectOverCDP(BROWSER_WS);

        const context = await browser.newContext();
        const page = await context.newPage();

        // Navigate and interact
        await page.goto('https://web-scraping.dev');
        const title = await page.title();
        console.log('Page title:', title);

        // Take a screenshot
        await page.screenshot({ path: 'screenshot.png' });
    } finally {
        if (browser) await browser.close();
    }
}

run();

Installation

pip install playwright

Quick Start Example

from playwright.sync_api import sync_playwright

API_KEY = ''
BROWSER_WS = f'wss://browser.scrapfly.io?api_key={API_KEY}&proxy_pool=datacenter&os=linux'

def run():
    with sync_playwright() as p:
        browser = None
        try:
            # Connect to Cloud Browser
            browser = p.chromium.connect_over_cdp(BROWSER_WS)

            context = browser.new_context()
            page = context.new_page()

            # Navigate and interact
            page.goto('https://web-scraping.dev')
            print('Page title:', page.title())

            # Take a screenshot
            page.screenshot(path='screenshot.png')
        finally:
            if browser:
                browser.close()

run()

Connection Parameters

Configure your Cloud Browser connection with these WebSocket URL parameters:

Parameter	Required	Description
`api_key`	Required	Your Scrapfly API key
`proxy_pool`	Optional	`datacenter` (default) or `residential`
`os`	Optional	OS fingerprint: `linux`, `windows`, `macos`
`session`	Optional	Session ID for persistent browser state
`country`	Optional	Proxy country code (ISO 3166-1 alpha-2)

Data Extraction

Extract data from a dynamic page:

const { chromium } = require('playwright');

const API_KEY = '';
const BROWSER_WS = `wss://browser.scrapfly.io?api_key=${API_KEY}&proxy_pool=datacenter`;

async function scrapeProducts() {
    let browser = null;
    try {
        browser = await chromium.connectOverCDP(BROWSER_WS);
        const context = await browser.newContext();
        const page = await context.newPage();

        // Navigate to the page
        await page.goto('https://web-scraping.dev/products', {
            waitUntil: 'networkidle',
        });

        // Extract product data using Playwright locators
        const products = await page.locator('.product').evaluateAll(elements =>
            elements.map(el => ({
                title: el.querySelector('.product-title')?.textContent?.trim(),
                price: el.querySelector('.product-price')?.textContent?.trim(),
                url: el.querySelector('a')?.href,
            }))
        );

        console.log('Products:', products);
        return products;
    } finally {
        if (browser) await browser.close();
    }
}

scrapeProducts();

Form Interaction

Fill forms and handle login flows:

const { chromium } = require('playwright');

const API_KEY = '';
const BROWSER_WS = `wss://browser.scrapfly.io?api_key=${API_KEY}&proxy_pool=datacenter`;

async function login() {
    let browser = null;
    try {
        browser = await chromium.connectOverCDP(BROWSER_WS);
        const context = await browser.newContext();
        const page = await context.newPage();

        await page.goto('https://web-scraping.dev/login');

        // Fill the login form using Playwright's fill method
        await page.fill('#username', 'myuser');
        await page.fill('#password', 'mypassword');

        // Click submit and wait for navigation
        await Promise.all([
            page.waitForNavigation(),
            page.click('#submit-button'),
        ]);

        // Check if login was successful
        const isLoggedIn = await page.locator('.user-profile').count() > 0;
        console.log('Login successful:', isLoggedIn);
    } finally {
        if (browser) await browser.close();
    }
}

login();

Session Persistence

Maintain browser state across connections using the session parameter:

const { chromium } = require('playwright');

const API_KEY = '';
const SESSION_ID = 'my-persistent-session';

// First connection: Login and set cookies
async function firstConnection() {
    let browser = null;
    try {
        browser = await chromium.connectOverCDP(
            `wss://browser.scrapfly.io?api_key=${API_KEY}&session=${SESSION_ID}`
        );

        const context = await browser.newContext();
        const page = await context.newPage();
        await page.goto('https://web-scraping.dev/login');
        // ... perform login ...
    } finally {
        if (browser) await browser.close();  // Session is preserved
    }
}

// Second connection: Reuse the logged-in session
async function secondConnection() {
    let browser = null;
    try {
        browser = await chromium.connectOverCDP(
            `wss://browser.scrapfly.io?api_key=${API_KEY}&session=${SESSION_ID}`
        );

        const context = await browser.newContext();
        const page = await context.newPage();
        await page.goto('https://web-scraping.dev/dashboard');
        // Already logged in from previous session!
    } finally {
        if (browser) await browser.close();
    }
}

Proxy Options

Proxy Pool	Use Case	Cost
`datacenter`	General scraping, high speed, lower cost	1 credits/30s + 2 credits/MB
`residential`	Protected sites, geo-targeting, anti-bot bypass	1 credits/30s + 10 credits/MB

Error Handling

Always handle errors and ensure proper browser cleanup to prevent billing leaks:

const { chromium } = require('playwright');

const API_KEY = '';
const BROWSER_WS = `wss://browser.scrapfly.io?api_key=${API_KEY}&proxy_pool=datacenter`;

async function scrapeWithErrorHandling() {
    let browser = null;
    try {
        browser = await chromium.connectOverCDP(BROWSER_WS);
        const context = await browser.newContext();
        const page = await context.newPage();

        await page.goto('https://web-scraping.dev/products');

        // Your scraping logic here
        const data = await page.locator('.product').evaluateAll(elements =>
            elements.map(el => ({
                title: el.querySelector('.product-title')?.textContent?.trim(),
            }))
        );

        return data;
    } catch (error) {
        console.error('Scraping failed:', error.message);
        throw error;
    } finally {
        // Always close browser to stop billing
        if (browser) await browser.close();
    }
}

scrapeWithErrorHandling()
    .then(data => console.log('Success:', data))
    .catch(error => console.error('Failed:', error));

Important: Failing to close the browser will keep the session running and continue billing. Always use try/finally blocks.

Best Practices

Use connectOverCDP - Playwright's CDP connection method for remote browsers
Handle disconnects - Wrap connections in try/catch
Close browsers - Always call browser.close() to stop billing
Use sessions wisely - Reuse sessions for multi-step flows
Leverage Playwright features - Use locators, auto-waiting, and built-in assertions

Playwright Integration

Installation & Quick Start

Installation

Quick Start Example

Installation

Quick Start Example

Connection Parameters

Data Extraction

Form Interaction

Session Persistence

Proxy Options

Error Handling

Best Practices

Related

Summary