# The Best Web Scraping API ## Collect data from any website with our web scraping API. 98% success rate against Cloudflare, DataDome, Akamai, and 17 other anti-bot vendors. No proxies, browsers, or CAPTCHA solvers to manage. - **Anti-bot bypass built in.** 20+ vendors handled (Cloudflare, DataDome, Akamai, PerimeterX, Kasada, F5, AWS WAF, more) with one `asp=true` flag. - **Scales with you.** Datacenter and residential proxy pools with geo-targeting, managed sessions, 100+ concurrent requests on custom plans, no infra to manage. [ Get Free API Key ](https://scrapfly.io/register) [ Developer Docs ](https://scrapfly.io/docs/scrape-api) 1,000 free credits. No credit card required. '; var lanesEl = el.querySelector('[data-lanes]'); // Engine slug → marketing-source SVG (filled silhouette, FA-style). // The two icon URLs are pre-resolved by Twig's asset() helper and // stored on the wrapper as data-attrs, so the picks up the // correct CDN-prefixed URL in prod and the same-origin URL in dev // without the JS having to know about it. var engineIcons = { CURLIUM: el.dataset.curliumIcon, SCRAPIUM: el.dataset.scrapiumIcon }; function engineIconSrc (engineName) { return engineIcons[engineName] || ''; } function buildLane (idx, initialStep) { var job = pickJob(); var laneEl = document.createElement('div'); laneEl.className = 'anim-scrape__lane'; laneEl.innerHTML = '' + '' + '' + job.country + '/' + job.pool + '' + '`' + job.path + '`' + '' + stageNames\[initialStep\] + '' + '' + ' ' + ' '; return { el: laneEl, engineIconEl: laneEl.querySelector('[data-engine-icon]'), geoEl: laneEl.querySelector('[data-geo]'), pathEl: laneEl.querySelector('[data-path]'), stageEl: laneEl.querySelector('[data-stage-text]'), elapsedEl: laneEl.querySelector('[data-elapsed]'), barEl: laneEl.querySelector('[data-bar]'), step: initialStep, cumul: 0, job: job }; } // Three lanes, each starting at a different stage so they look // genuinely parallel from the first frame (no staggered fade-in). var lanes = [buildLane(0, 0), buildLane(1, 1), buildLane(2, 2)]; lanes.forEach(function (l) { lanesEl.appendChild(l.el); }); // Live RPS readout — sums lane completions over a rolling window. var statRpsEl = el.querySelector('[data-rps]'); var completionTimes = []; function updateRps () { var now = Date.now(); completionTimes = completionTimes.filter(function (t) { return now - t < 1000; }); statRpsEl.textContent = completionTimes.length === 0 ? '—' : String(completionTimes.length * 3); } function tickLane (lane) { var s = lane.step; var deltas = lane.job.deltas; if (s < deltas.length) { var d = deltas[s][0] + Math.random() * (deltas[s][1] - deltas[s][0]); lane.cumul += d; lane.stageEl.textContent = stageNames[s]; lane.elapsedEl.textContent = Math.round(lane.cumul) + 'ms'; } lane.barEl.setAttribute('data-progress', String(s)); lane.step++; if (lane.step > deltas.length) { // Cycle complete — record completion, then re-roll the entire // job (path + engine + geo + pool) so each lane keeps showing // realistic Scrapfly variety rather than reusing the same // engine/path forever. completionTimes.push(Date.now()); lane.step = 0; lane.cumul = 0; lane.job = pickJob(); lane.pathEl.textContent = lane.job.path; lane.geoEl.textContent = lane.job.country + '/' + lane.job.pool; lane.engineIconEl.src = engineIconSrc(lane.job.engine); lane.engineIconEl.alt = lane.job.engine; lane.engineIconEl.title = lane.job.engine; lane.stageEl.textContent = stageNames[0]; lane.elapsedEl.textContent = ''; } } // Each lane ticks on its own interval — staggered so they don't // synchronize over time (~480ms ± per-lane jitter). var intervals = lanes.map(function (lane, i) { // Phase-stagger first tick by 160ms × lane index. setTimeout(function () { tickLane(lane); }, i * 160); return setInterval(function () { tickLane(lane); }, 460 + i * 40); }); var rpsInterval = setInterval(updateRps, 250); // Return the first interval so the existing teardown contract holds. // (No teardown is currently invoked, but stay symmetrical with the // other drivers that all return one setInterval handle.) void intervals; void rpsInterval; return intervals[0]; }, browser: function (el) { el.innerHTML = 'CDP EVENTS ' + ' '; var feed = el.querySelector('[data-feed]'); // Each event carries a realistic [minDelta, maxDelta] in ms — the // gap from the *previous* event in the same request flight. Numbers // mirror what Chrome DevTools shows on a real CDP trace: tens of ms // between network events, hundreds for DOM/load milestones, ~10ms // for Input dispatch. Hardcoded timestamps are dropped from detail // strings so the feed-time column is the single source of truth. var events = [ ['Network.requestWillBeSent', 'GET web-scraping.dev/abc', [15, 25]], ['Page.frameStartedLoading', 'frame=main', [5, 15]], ['Network.responseReceived', 'status=200, type=document', [40, 120]], ['Page.domContentEventFired', 'frame=main', [180, 320]], ['Runtime.executionContextCreated', 'origin=web-scraping.dev', [10, 25]], ['DOM.documentUpdated', 'nodes=1,284', [20, 60]], ['Page.loadEventFired', 'frame=main', [120, 240]], ['Network.dataReceived', '124.3 KB', [15, 45]], ['Input.dispatchMouseEvent', 'click (842, 316)', [5, 15]] ]; var i = 0; // tCdp is the simulated CDP clock in ms, NOT wall-clock time. It // resets at the start of each request flight (every full cycle of // events) so the feed reads as a fresh trace, not a 30-minute // session log. var tCdp = 0; function tick () { var idx = i % events.length; if (idx === 0) tCdp = 0; var ev = events[idx]; var jitter = ev[2][0] + Math.random() * (ev[2][1] - ev[2][0]); tCdp += jitter; var dt = Math.round(tCdp) + 'ms'; var li = document.createElement('li'); li.innerHTML = '' + dt + '' + '' + ev\[0\] + '' + '' + ev\[1\] + ''; feed.insertBefore(li, feed.firstChild); while (feed.children.length > 6) feed.removeChild(feed.lastChild); i++; } for (var k = 0; k < 4; k++) tick(); return setInterval(tick, 950); }, screenshot: function (el) { el.innerHTML = 'CAPTURING ' + '' + '' + ' ' + ' ' + ' ' + '' + 'PNG' + 'JPEG' + 'WEBP' + 'FULL PAGE' + ' ' + ' '; var fmts = el.querySelectorAll('[data-fmt]'); var shutter = el.querySelector('[data-shutter]'); var spec = el.querySelector('[data-spec]'); var elapsed = el.querySelector('[data-elapsed]'); // Each format combines a realistic viewport spec, capture latency, // and resulting payload size. Numbers cross-checked against // Scrapfly screenshot benchmarks: PNG/JPEG/WEBP on 1920×1080 land // 180-400ms; full-page on a long article scrolls + stitches and // takes 700-1200ms. var presets = [ { dim: '1920×1080', size: '184 KB', latencyMs: [180, 320] }, { dim: '1920×1080', size: '92 KB', latencyMs: [160, 260] }, { dim: '1920×1080', size: '76 KB', latencyMs: [200, 360] }, { dim: '1920×6840', size: '1.4 MB', latencyMs: [780, 1180] } ]; var step = 0; var anim = null; function tick () { var p = presets[step]; var latency = Math.round(p.latencyMs[0] + Math.random() * (p.latencyMs[1] - p.latencyMs[0])); spec.textContent = p.dim + ' • ' + p.size; elapsed.textContent = latency + 'ms'; // Web Animations API for the shutter sweep — replaces a CSS // transition + offsetWidth-reflow restart trick. Each tick we // cancel the previous animation and run a fresh one; WAAPI keeps // the work on the compositor thread, so no main-thread reflow. if (anim) anim.cancel(); anim = shutter.animate( [{ width: '0%' }, { width: '100%' }], { duration: latency, easing: 'cubic-bezier(.2,.8,.2,1)', fill: 'forwards' } ); fmts.forEach(function (f, i) { f.classList.toggle('anim-screenshot__format--active', i === step); }); step = (step + 1) % fmts.length; } tick(); return setInterval(tick, 1500); }, extract: function (el) { el.innerHTML = 'SCHEMA HYDRATION ' + '' + '{ name: \_\_\_\_\_\_\_\_\_\_\_\_, ' + ' price: \_\_\_\_\_\_\_\_\_\_\_\_, ' + ' in\_stock: \_\_\_\_, ' + ' rating: \_\_\_\_ }' + ' '; var records = [ { name: '"Widget Pro"', price: '49.99', in_stock: 'true', rating: '4.7' }, { name: '"Acme Runner"', price: '129.00', in_stock: 'true', rating: '4.3' }, { name: '"Vintage Chair"', price: '340.00', in_stock: 'false', rating: '4.9' }, { name: '"Coffee Grinder"', price: '89.50', in_stock: 'true', rating: '4.6' } ]; var keys = ['name', 'price', 'in_stock', 'rating']; var stat = el.querySelector('[data-stat]'); // Counter that ticks up each completed record so the panel reads // as "ongoing batch extraction" rather than a single shot demo. var totalRecords = 0; var rec = 0, step = 0; function tick () { var key = keys[step]; var field = el.querySelector('[data-field="' + key + '"]'); if (field) { field.textContent = records[rec % records.length][key]; field.className = 'v v-new'; } step++; if (step >= keys.length) { step = 0; rec++; totalRecords++; if (stat) stat.textContent = totalRecords.toLocaleString('en-US') + ' records • ~340ms/rec'; setTimeout(function () { keys.forEach(function (k) { var f = el.querySelector('[data-field="' + k + '"]'); if (!f) return; f.textContent = k === 'in_stock' || k === 'rating' ? '____' : '____________'; f.className = 'pending'; }); }, 600); } } // Faster field reveal — 250ms feels like a template extraction // (regex/CSS), not a slow LLM dribble. Total per-record: ~1s. return setInterval(tick, 250); }, crawl: function (el) { el.innerHTML = '' + '**0 urls discovered**' + 'depth 1/5 • 0 req/s' + ' ' + '``` web-scraping.dev/ ``` '; var countEl = el.querySelector('[data-count]'); var depthEl = el.querySelector('[data-depth]'); var rpsEl = el.querySelector('[data-rps]'); var treeEl = el.querySelector('[data-tree]'); var branches = [ '├─ /products (1,284 pages)', '│ ├─ /products/shoes (392)', '│ ├─ /products/bags (218)', '│ └─ /products/accessories (674)', '├─ /articles (3,902 pages)', '│ ├─ /articles/2024/', '│ └─ /articles/2025/', '├─ /reviews (8,401)', '└─ /sitemap.xml' ]; // Counter starts plausible, climbs by realistic-per-tick batches // (~10 req/s sustained = 65/tick at 650ms cadence; we vary per // tick to read as live discovery rather than a clock). var count = 1, branchIdx = 0, depth = 1; function tick () { var batch = 50 + Math.floor(Math.random() * 60); count += batch; countEl.textContent = count.toLocaleString('en-US'); // RPS oscillates around 8-15 — the typical Scrapfly crawler // throttle for a single seed under default politeness. rpsEl.textContent = String(8 + Math.floor(Math.random() * 8)); if (branchIdx < branches.length) { treeEl.innerHTML += '\n' + branches[branchIdx]; branchIdx++; depth = Math.min(5, 1 + Math.floor(branchIdx / 2)); depthEl.textContent = String(depth); } else { setTimeout(function () { treeEl.innerHTML = 'web-scraping.dev/'; branchIdx = 0; depth = 1; count = 1; depthEl.textContent = '1'; countEl.textContent = '1'; }, 1400); branchIdx = branches.length + 1; } } return setInterval(tick, 650); } }; document.querySelectorAll('[data-hero-anim]').forEach(function (el) { var kind = el.getAttribute('data-hero-anim'); var driver = drivers[kind]; if (driver) driver(el); }); })(); --- ## 5B+ scrapes / month ## 98% bypass success ## 20+ antibot vendors defeated ## 55k+ developers --- // ANTI-BOT STACK## The stack that actually defeats blocking. Blocking is the scraper's real bill. Credits matter less than the targets you can't reach. The Web Scraping API ships `asp=true`, the composed result of two proprietary low-layer products we build and patch in-house: **[Curlium](https://scrapfly.io/curlium)** for byte-perfect Chrome on the wire, and **[Scrapium](https://scrapfly.io/scrapium)** for anti-detect Chromium at the C++ level. Proven at 5B+ scrapes/month on protected websites. [ View ASP docs ](https://scrapfly.io/docs/scrape-api/anti-scraping-protection) [ Browse 20+ bypasses ](https://scrapfly.io/bypass) [ // CURLIUM **HTTP client, byte-perfect Chrome** Patched curl fork with BoringSSL + nghttp3 + ngtcp2. JA4, HTTP/2 SETTINGS, QUIC transport params match reference Chrome exactly. 10-50x faster than a browser. ](https://scrapfly.io/curlium) [ // SCRAPIUM **Chromium, anti-detect at the kernel** Blink/V8 patched directly. 4,000+ fingerprint signals (WebGL, Canvas, AudioContext, Navigator, WebRTC, fonts) coherent across every surface. Drive with Playwright, Puppeteer, Selenium. ](https://scrapfly.io/scrapium) --- CAPABILITIES## One API, Every Obstacle Anti-bot, proxies, browsers, extraction. All composable, all on one endpoint. ### Anti-Bot Bypass Add `asp=true` to any scrape request. Scrapfly detects the active anti-bot vendor, builds a coherent browser fingerprint (TLS, HTTP/2, JS runtime, behavioral signals), solves the challenge, and replays the original request transparently. Vendor detection is automatic. Failed challenge retries do not cost credits. ASP may auto-upgrade the proxy pool to residential when required; the final cost reflects the pool actually used. **[JA3/JA4](https://scrapfly.io/web-scraping-tools/ja3-fingerprint)** TLS fingerprint **[HTTP/2](https://scrapfly.io/web-scraping-tools/http2-fingerprint)** SETTINGS frame **Behavioral** mouse + scroll **Retries** free on fail [Cloudflare](https://scrapfly.io/bypass/cloudflare) [DataDome](https://scrapfly.io/bypass/datadome) [Akamai](https://scrapfly.io/bypass/akamai) [PerimeterX](https://scrapfly.io/bypass/perimeterx) [Kasada](https://scrapfly.io/bypass/kasada) [Imperva](https://scrapfly.io/bypass/incapsula) [F5](https://scrapfly.io/bypass/f5) [AWS WAF](https://scrapfly.io/bypass/aws-waf) **Curlium** HTTP fingerprint **Scrapium** browser identity **190+ countries** proxy coverage **Challenge solver** server-side [View ASP docs →](https://scrapfly.io/docs/scrape-api/anti-scraping-protection) ### Request Pipeline Every `/scrape` call flows through a composable pipeline. Each layer activates only when its parameter is set; unused layers add zero latency. **Your Code** SDK or raw HTTP, one endpoint **Scrapfly Edge** routing, auth, cost accounting, rate limits **Curlium / Scrapium** HTTP or browser, byte-perfect Chrome fingerprint **Proxy Pool** datacenter or residential, auto-rotated, geo-pinned **Challenge Solver** Turnstile, CAPTCHA, sensor data - server-side, free retries **Target** sees a real browser from a real location **Response** HTML, browser\_data, extracted\_data, screenshots, cost headers ### Proxy Pools, Built In Two pools available by default. Datacenter (`public_datacenter_pool`) is the default and costs 1 credit per request. Residential (`public_residential_pool`) routes through ISP-assigned IPs, required for most bot-protected targets. Set the `country` parameter to pin requests to a specific country. The pool rotates IPs, cools them, and excludes underperforming exits automatically, with no separate proxy dashboard to manage. **Datacenter** 1 credit/req **Residential** geo-targeted **Auto rotation** cooled, pooled 190+ countries Session sticky IP cooling No proxy mgmt Auto-upgrade country param [View proxy docs →](https://scrapfly.io/docs/scrape-api/proxy) ### Cloud Browser Rendering Set `render_js=true` to route the request through a dedicated Scrapium browser session (patched Chromium, optimized for scraping). The response includes fully rendered HTML plus a `browser_data` object containing all XHR calls, localStorage, sessionStorage, WebSocket frames, and browser file downloads. Block until an element appears using `wait_for_selector` (CSS or XPath, 15s timeout) or a fixed delay via `rendering_wait` (ms, max 25s). Chain browser actions step by step with `js_scenario`: click, fill, scroll, wait\_for\_navigation, and more. [render\_js](https://scrapfly.io/docs/scrape-api/javascript-rendering) [js\_scenario](https://scrapfly.io/docs/scrape-api/javascript-scenario) [wait\_for\_selector](https://scrapfly.io/docs/scrape-api/javascript-rendering) [rendering\_wait](https://scrapfly.io/docs/scrape-api/javascript-rendering) **Click** js\_scenario **Fill** form input **Scroll** lazy load **Downloads** browser\_data [View JS rendering docs →](https://scrapfly.io/docs/scrape-api/javascript-rendering) ### Scrape and Extract in One Call Pass an extraction parameter alongside your scrape config and receive structured JSON in `result.extracted_data`, without a second API call. Three methods: `extraction_model` uses a pre-trained AI model (product, article, review, job listing); `extraction_template` accepts a base64-encoded JSON template of CSS, XPath, and regex rules that you define; `extraction_prompt` passes the page content to an LLM with a natural-language question. If a page is unstable or heavily protected, scrape first and submit the stored HTML to the dedicated Extraction API separately. [extraction\_model](https://scrapfly.io/docs/extraction-api/automatic-ai) [extraction\_template](https://scrapfly.io/docs/extraction-api/rules-and-template) [extraction\_prompt](https://scrapfly.io/docs/extraction-api/llm-prompt) [extracted\_data](https://scrapfly.io/docs/scrape-api/extraction) [View extraction docs →](https://scrapfly.io/docs/scrape-api/extraction) ### Per-Request Debug and Replay Add `debug=true` to any request. Scrapfly stores the full response for the duration of your log retention window. When `render_js=true` is also set, a screenshot is captured automatically. The response includes a `response_url` for direct download and a `content_replay_url` to re-run extraction against the cached content without making a new scrape. Every response carries cost details in the `X-Scrapfly-Api-Cost` header and remaining quota in `X-Scrapfly-Remaining-Api-Credit`. Debug costs +2 credits in the live environment; free in test. **Debug** stored trace **Replay** no extra scrape **Cost** per-header **Screenshot** auto on render\_js [View debug docs →](https://scrapfly.io/docs/scrape-api/debug) ### Screenshots Pass `screenshots[name]=fullpage` for a full-page capture or a CSS selector to target a specific element. Up to 10 named screenshots per request. Screenshot URLs land in `result.screenshots`. Requires `render_js=true`. fullpage CSS selector load_images dark_mode block_banners high_quality [View screenshot docs →](https://scrapfly.io/docs/scrape-api/screenshot) ### Batch Scraping `POST /scrape/batch` accepts up to 100 scrape configs in one request. Results stream back as `multipart/mixed` parts as each scrape completes, so you start consuming data at the speed of the fastest result, not the slowest. Each part is the same JSON shape as a single `/scrape` response. Each config must carry a unique `correlation_id`. Requires a paid plan. Up to 100 configs per batch multipart/mixed streaming correlation_id per config [View batch docs →](https://scrapfly.io/docs/scrape-api/batch) ### Server-Side Cache Add `cache=true` to store the scraped response server-side. Subsequent requests matching the same URL and config fingerprint return the cached copy at 0 credits. Default TTL is 24 hours; extend to 7 days with `cache_ttl` (max 604800 seconds). Force-clear a cached entry with `cache_clear=true`. Cache is isolated per project and environment. Unlike HTTP caches, it supports POST, PUT, and PATCH methods. 0credits on hit **7 days**max TTL [View cache docs →](https://scrapfly.io/docs/scrape-api/cache) ### Proxy Mode Use Scrapfly as a standard HTTP/HTTPS forward proxy without touching the REST API. Scraping options encode into the proxy username as dash-separated key-value pairs; your API key is the password. Designed for tools that only accept a proxy URL: Screaming Frog, Apify, Crawlee, and similar platforms. Not a replacement for the REST API in custom code. [View proxy mode docs →](https://scrapfly.io/docs/scrape-api/proxy-mode) ### Block Detection API `POST /classify` runs the same anti-bot detection pipeline used on every live scrape against an HTTP response you already hold. Pass the URL, status code, response headers, and body. Get back `blocked: true/false` and the vendor name (cloudflare, datadome, akamai, and others). Useful for auditing cached pages, deciding whether to retry with `asp=true`, or verifying responses from your own infrastructure. blocked: true/false vendor name POST /classify [View classify docs →](https://scrapfly.io/docs/scrape-api/classify) ### Pay Only for Successful Scrapes Failed scrapes (target 5xx, ASP challenge failures, network timeouts) are not billed. A fairness policy applies: if more than 30% of your traffic returns eligible error codes within a one-hour window, billing resumes for that window. Use `cost_budget` to set a maximum credit cap per request and prevent a runaway call from draining quota before it completes. 0credits on fail **cost\_budget**per-request cap [View billing docs →](https://scrapfly.io/docs/scrape-api/billing) ### Async Webhooks Add `webhook_name=your_hook` to enqueue a scrape without holding a connection open. Scrapfly returns a job UUID immediately in `context.job.uuid`. When the scrape finishes, Scrapfly POSTs the full response body to your endpoint, signed with HMAC-SHA256 in `X-Scrapfly-Webhook-Signature`. Delivery retries on failure: 30s, 1m, 5m, 30m, 1h, then 1 day. After 100 consecutive failures the webhook disables automatically. **HMAC-SHA256** signed payload **6-step retry** 30s to 1 day **Job UUID** instant response **Auto-disable** 100 failures [View webhook docs →](https://scrapfly.io/docs/scrape-api/webhook) ### Persistent Sessions Pass a `session` key and requests share cookies, referrer, navigation history, and a sticky proxy IP. Works across both plain HTTP and browser (`render_js`) requests in the same named session, so you can establish auth state with a browser call and continue without it. Sessions expire 7 days after last use. After 30 seconds idle the proxy IP may rotate depending on the pool. Cannot be used together with the cache feature. Shared cookies Sticky proxy IP 7-day TTL localStorage [View session docs →](https://scrapfly.io/docs/scrape-api/session) --- CODE## Scrape Any URL in Three Lines Pick a workflow, pick a language. Every example runs against a real Scrapfly endpoint. [ Anti-Bot Bypass ](#wsa-strat-asp) [ Browser Rendering ](#wsa-strat-browsers) [ Browser Scenarios ](#wsa-strat-browser-control) [ Residential Proxies ](#wsa-strat-proxies) [ Persistent Sessions ](#wsa-strat-session) [ Custom Requests ](#wsa-strat-customize) Set `asp=true` and the API handles Cloudflare, DataDome, Akamai, and more. See the full [bypass catalog](https://scrapfly.io/"/bypass"). Python TypeScript HTTP / cURL Go ``` from scrapfly import ScrapeConfig, ScrapflyClient, ScrapeApiResponse client = ScrapflyClient(key="API KEY") api_response: ScrapeApiResponse = client.scrape( ScrapeConfig( url='https://httpbin.dev/html', # bypass anti-scraping protection asp=True ) ) print(api_response.result) ``` ``` import { ScrapflyClient, ScrapeConfig } from 'jsr:@scrapfly/scrapfly-sdk'; const client = new ScrapflyClient({ key: "API KEY" }); let api_result = await client.scrape( new ScrapeConfig({ url: 'https://httpbin.dev/html', // bypass anti-scraping protection asp: true, }) ); console.log(api_result.result); ``` ``` http https://api.scrapfly.io/scrape \ key==$SCRAPFLY_KEY \ url==https://httpbin.dev/html \ asp==true ``` ``` package main import ( "fmt" "github.com/scrapfly/go-scrapfly" ) func main() { client, _ := scrapfly.New("API KEY") result, _ := client.Scrape(&scrapfly.ScrapeConfig{ URL: "https://httpbin.dev/html", // bypass anti-scraping protection ASP: true, }) fmt.Println(result.Result.Content) } ``` [ Python SDK docs → ](https://scrapfly.io/docs/sdk/python) [ TypeScript SDK docs → ](https://scrapfly.io/docs/sdk/typescript) [ HTTP API docs → ](https://scrapfly.io/docs) [ Go SDK docs → ](https://scrapfly.io/docs/sdk/python) Render JavaScript with `render_js=true`. Under the hood, [Scrapium](https://scrapfly.io/"/scrapium") drives the session. Python TypeScript HTTP / cURL ``` from scrapfly import ScrapeConfig, ScrapflyClient, ScrapeApiResponse client = ScrapflyClient(key="API KEY") api_response: ScrapeApiResponse = client.scrape( ScrapeConfig( url='https://web-scraping.dev/reviews', # enable the use of cloud browers render_js=True, # wait for specific element to appear wait_for_selector=".review", # or wait set amount of time rendering_wait=3_000, # 3 seconds ) ) print(api_response.result) ``` ``` import { ScrapflyClient, ScrapeConfig } from 'jsr:@scrapfly/scrapfly-sdk'; const client = new ScrapflyClient({ key: "API KEY" }); let api_result = await client.scrape( new ScrapeConfig({ url: 'https://web-scraping.dev/reviews', // enable the use of cloud browers render_js: true, // wait for specific element to appear wait_for_selector: ".review", // or wait set amount of time rendering_wait: 3_000, // 3 seconds }) ); console.log(JSON.stringify(api_result.result)); ``` ``` http https://api.scrapfly.io/scrape \ key==$SCRAPFLY_KEY \ url==https://web-scraping.dev/reviews \ render_js==true \ wait_for_selector==.review \ rendering_wait==3000 ``` [ Python SDK docs → ](https://scrapfly.io/docs/sdk/python) [ TypeScript SDK docs → ](https://scrapfly.io/docs/sdk/typescript) [ HTTP API docs → ](https://scrapfly.io/docs) Click, fill, wait for selectors inside the same call with `js_scenario`. Best for login flows and multi-step forms. Python TypeScript HTTP / cURL ``` from scrapfly import ScrapeConfig, ScrapflyClient, ScrapeApiResponse client = ScrapflyClient(key="API KEY") api_response: ScrapeApiResponse = client.scrape( ScrapeConfig( url='https://web-scraping.dev/login', # enable browsers for this request render_js = True, # describe your control flow js_scenario = [ {"fill": {"selector": "input[name=username]", "value":"user123"}}, {"fill": {"selector": "input[name=password]", "value":"password"}}, {"click": {"selector": "button[type='submit']"}}, {"wait_for_navigation": {"timeout": 5000}} ] ) ) print(api_response.result) ``` ``` import { ScrapflyClient, ScrapeConfig } from 'jsr:@scrapfly/scrapfly-sdk'; const client = new ScrapflyClient({ key: "API KEY" }); let api_result = await client.scrape( new ScrapeConfig({ url: 'https://web-scraping.dev/reviews', // enable browsers for this request render_js: true, // describe your control flow js_scenario: [ {"fill": {"selector": "input[name=username]", "value":"user123"}}, {"fill": {"selector": "input[name=password]", "value":"password"}}, {"click": {"selector": "button[type='submit']"}}, {"wait_for_navigation": {"timeout": 5000}} ] }) ); console.log(JSON.stringify(api_result.result)); ``` ``` http https://api.scrapfly.io/scrape \ key==$SCRAPFLY_KEY \ url==https://web-scraping.dev/login \ render_js==true \ js_scenario==Ww0KCXsiZmlsbCI6IHsic2VsZWN0b3IiOiAiaW5wdXRbbmFtZT11c2VybmFtZV0iLCAidmFsdWUiOiJ1c2VyMTIzIn19LA0KCXsiZmlsbCI6IHsic2VsZWN0b3IiOiAiaW5wdXRbbmFtZT1wYXNzd29yZF0iLCAidmFsdWUiOiJwYXNzd29yZCJ9fSwNCgl7ImNsaWNrIjogeyJzZWxlY3RvciI6ICJidXR0b25bdHlwZT0nc3VibWl0J10ifX0sDQoJeyJ3YWl0X2Zvcl9uYXZpZ2F0aW9uIjogeyJ0aW1lb3V0IjogNTAwMH19DQpd # note: js scenario has to be base64 encoded ``` [ Python SDK docs → ](https://scrapfly.io/docs/sdk/python) [ TypeScript SDK docs → ](https://scrapfly.io/docs/sdk/typescript) [ HTTP API docs → ](https://scrapfly.io/docs) 190+ country codes, residential or datacenter pool, session stickiness. No per-vendor account setup. Python TypeScript HTTP / cURL ``` from scrapfly import ScrapeConfig, ScrapflyClient, ScrapeApiResponse client = ScrapflyClient(key="API KEY") api_response: ScrapeApiResponse = client.scrape( ScrapeConfig( url='https://httpbin.dev/html', # choose proxy countries country="US,CA", # residential or datacenter proxies proxy_pool="public_residential_pool" ) ) print(api_response.result) ``` ``` import { ScrapflyClient, ScrapeConfig } from 'jsr:@scrapfly/scrapfly-sdk'; const client = new ScrapflyClient({ key: "API KEY" }); let api_result = await client.scrape( new ScrapeConfig({ url: 'https://web-scraping.dev/product/1', // choose proxy countries country: "US,CA", // residential or datacenter proxies proxy_pool: "public_residential_pool" }) ); console.log(api_result.result); ``` ``` http https://api.scrapfly.io/scrape \ key==$SCRAPFLY_KEY \ url==https://httpbin.dev/html \ country=="US,CA" \ proxy_pool=="public_residential_pool" ``` [ Python SDK docs → ](https://scrapfly.io/docs/sdk/python) [ TypeScript SDK docs → ](https://scrapfly.io/docs/sdk/typescript) [ HTTP API docs → ](https://scrapfly.io/docs) Share cookies and auth state across multiple calls. Same proxy exit, same browser profile. Python TypeScript HTTP / cURL ``` from scrapfly import ScrapeConfig, ScrapflyClient, ScrapeApiResponse client = ScrapflyClient(key="API KEY") api_response: ScrapeApiResponse = client.scrape( ScrapeConfig( url='https://web-scraping.dev/product/1', # add unique identifier to start a session session="mysession123", ) ) # resume session api_response2: ScrapeApiResponse = client.scrape( ScrapeConfig( url='https://web-scraping.dev/product/1', session="mysession123", # sessions can be shared between browser and http requests # render_js = True, # enable browser for this session ) ) print(api_response2.result) ``` ``` import { ScrapflyClient, ScrapeConfig } from 'jsr:@scrapfly/scrapfly-sdk'; const client = new ScrapflyClient({ key: "API KEY" }); let api_result = await client.scrape( new ScrapeConfig({ url: 'https://web-scraping.dev/product/1', // add unique identifier to start a session session: "mysession123", }) ); // resume session let api_result2 = await client.scrape( new ScrapeConfig({ url: 'https://web-scraping.dev/product/1', session: "mysession123", // sessions can be shared between browser and http requests // render_js: true, // enable browser for this session }) ); console.log(JSON.stringify(api_result2.result)); ``` ``` # start session http https://api.scrapfly.io/scrape \ key==$SCRAPFLY_KEY \ url==https://web-scraping.dev/product/1 \ session=mysession123 # resume session http https://api.scrapfly.io/scrape \ key==$SCRAPFLY_KEY \ url==https://web-scraping.dev/product/1 \ session=mysession123 ``` [ Python SDK docs → ](https://scrapfly.io/docs/sdk/python) [ TypeScript SDK docs → ](https://scrapfly.io/docs/sdk/typescript) [ HTTP API docs → ](https://scrapfly.io/docs) Custom headers, HTTP methods, POST bodies, timeouts. The API is a superset of curl. Python TypeScript HTTP / cURL ``` from scrapfly import ScrapeConfig, ScrapflyClient, ScrapeApiResponse client = ScrapflyClient(key="API KEY") api_response: ScrapeApiResponse = client.scrape( ScrapeConfig( url='http://httpbin.dev/post', # change method to POST, GET, PUT etc. method="POST", # POST data data={ "key": "value" }, # send custom headers headers={ "Authorization": "Basic ABC", "Content-Type": "application/json", } ) ) print(api_response.result) ``` ``` import { ScrapflyClient, ScrapeConfig } from 'jsr:@scrapfly/scrapfly-sdk'; const client = new ScrapflyClient({ key: "API KEY" }); let api_result = await client.scrape( new ScrapeConfig({ url: 'https://httpbin.dev/post', // change method to POST, GET, PUT etc. method: "POST", // POST data data: { "key": "value" }, // send custom headers headers: { "Authorization": "Basic ABC", "Content-Type": "application/json", } }) ); console.log(api_result.result); ``` ``` http POST https://api.scrapfly.io/scrape \ key==$SCRAPFLY_KEY \ url==https://httpbin.dev/post \ headers[myheader]==myvalue \ key=value ``` [ Python SDK docs → ](https://scrapfly.io/docs/sdk/python) [ TypeScript SDK docs → ](https://scrapfly.io/docs/sdk/typescript) [ HTTP API docs → ](https://scrapfly.io/docs) --- LEARN## Docs, Tools, And Ready-Made Scrapers Everything you need to go from zero to production data pipeline. ### API Reference Every parameter, every response field, with runnable code examples. [ Developer Docs → ](https://scrapfly.io/docs/scrape-api) ### Academy Interactive courses on web scraping, anti-bot, and data extraction. [ Start learning → ](https://scrapfly.io/academy) ### Open-Source Scrapers 40+ production-ready scrapers on GitHub. Copy, paste, customize. [ Explore repo → ](https://github.com/scrapfly/scrapfly-scrapers) ### Developer Tools cURL-to-Python, JA3 checker, selector tester, HTTP/2 fingerprint. [ Browse tools → ](https://scrapfly.io/web-scraping-tools) --- // INTEGRATIONS## Seamlessly integrate with frameworks & platforms Plug Scrapfly into your favorite tools, or build custom workflows with our first-class SDKs. ### No-code automation [ Zapier ](https://scrapfly.io/integration/zapier) [ Make ](https://scrapfly.io/integration/make) [ n8n ](https://scrapfly.io/integration/n8n) ### LLM & RAG frameworks [ LlamaIndex ](https://scrapfly.io/integration/llamaindex) [ LangChain ](https://scrapfly.io/integration/langchain) [ CrewAI ](https://scrapfly.io/integration/crewai) ### First-class SDKs [ Python pip install scrapfly-sdk ](https://scrapfly.io/docs/sdk/python) [ TypeScript Node, Deno, Bun ](https://scrapfly.io/docs/sdk/typescript) [ Go go get scrapfly-sdk ](https://scrapfly.io/docs/sdk/golang) [ Rust cargo add scrapfly-sdk ](https://scrapfly.io/docs/sdk/rust) [ Scrapy Full-feature extension ](https://scrapfly.io/docs/sdk/scrapy) [ See all integrations ](https://scrapfly.io/integration) --- FAQ## Frequently Asked Questions ### What makes the Scrapfly Web Scraping API different? Every feature ships on one endpoint: anti-bot bypass, residential proxies, cloud browsers, AI extraction, async mode, webhooks, full observability. You don't pay for failed scrapes, and every request gives you a full HAR waterfall and replay link. Most competitors charge for retries and force you to debug with curl, tcpdump, and hope. ### How does anti-bot bypass actually work? Enable `asp=true`. Behind the scenes the API picks the right TLS fingerprint (JA3/JA4), browser profile, proxy location, and challenge-handling strategy for the target. Cloudflare Turnstile, DataDome slider, Akamai sensor data, PerimeterX Press-and-Hold, Kasada proof-of-work, and more are solved server-side. You get the page, not the block. ### Do you pass residential proxy cost through or mark it up? Residential proxy traffic costs a flat number of API credits per request regardless of data size. Per-GB bandwidth tiers are published in the dashboard before you commit, and the Proxy Saver feature automatically picks the cheapest pool that still bypasses the target. ### Can I run concurrent requests? Yes. Free plan includes 5 concurrent, and plans scale to 100 concurrent on Enterprise. Hit the limit and you get a 429 with a retry-after, no silent drops. Higher concurrency is available on custom plans. ### What happens to credits if the request fails? Failed requests don't consume credits. The API only bills when a target returns a usable response. Timeouts, upstream 5xx, and challenges we couldn't solve are all free retries. ### Is web scraping legal? Scraping publicly accessible data is legal in most jurisdictions (Meta v. Bright Data and hiQ v. LinkedIn have established strong precedent). You're responsible for respecting robots.txt, rate limits, and target terms of service. See [our legal overview](https://scrapfly.io/"/is-web-scraping-legal") for details. ### How do I debug a failing scrape? Every response includes a `log_url` that opens the full request in the dashboard: response body, headers, cookies, rendered HTML, screenshots, HAR waterfall. One click replays the exact request with the same parameters. No guesswork, no tcpdump. --- // PRICING## Transparent, usage-based pricing One plan covers the full Scrapfly platform. Pick a monthly credit budget; every API shares the same credit pool. No per-product lock-in, no surprise line items. **Free tier**1,000 free credits on signup. No credit card required. **Pay on success**You only pay for successful requests. Failed calls are free. **No lock-in**Upgrade, downgrade, or cancel anytime. No contract. [ See pricing ](https://scrapfly.io/pricing) [ Start free ](https://scrapfly.io/register) // UNBUNDLE### Need more control? We unbundle the stack. The Web Scraping API is the batteries-included product. Each layer is also available standalone: [Curlium](https://scrapfly.io/curlium) for raw HTTP with perfect fingerprints, [Scrapium](https://scrapfly.io/scrapium) for stealth Chromium you drive with Playwright, [Browser API](https://scrapfly.io/products/cloud-browser-api) for hosted headless Chrome, [Extraction API](https://scrapfly.io/products/extraction-api) for structured output from any HTML. [Get Free API Key](https://scrapfly.io/register)1,000 free credits. No card.