feat(playwright-crawler): add BrowserFetchHttpClient

docs/guides/http-clients.mdx

@@ -8,9 +8,11 @@ import ApiLink from '@site/src/components/ApiLink';
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
+import CodeBlock from '@theme/CodeBlock';
 
 import CheerioGotScrapingExample from '!!raw-loader!roa-loader!./http-clients/cheerio-got-scraping-example.ts';
 import CheerioImpitExample from '!!raw-loader!roa-loader!./http-clients/cheerio-impit-example.ts';
+import CheerioBrowserFetchExample from '!!raw-loader!./http-clients/cheerio-browser-fetch-example.ts';
 
 HTTP clients are utilized by HTTP-based crawlers (e.g., <ApiLink to="cheerio-crawler/class/CheerioCrawler">`CheerioCrawler`</ApiLink>) to communicate with web servers. They use external HTTP libraries for communication rather than a browser. Examples of such libraries include [`impit`](https://apify.github.io/impit/) or [`got-scraping`](https://github.com/apify/got-scraping/). After retrieving page content, an HTML parsing library is typically used to facilitate data extraction. Examples of such libraries include [Cheerio](https://cheerio.js.org/), [`jsdom`](https://github.com/jsdom/jsdom) or [`linkedom`](https://github.com/WebReflection/linkedom). These crawlers are faster than browser-based crawlers but generally cannot execute client-side JavaScript.
 
@@ -76,6 +78,20 @@ For <ApiLink to="impit-client/class/ImpitHttpClient">`ImpitHttpClient`</ApiLink>
 npm i @crawlee/impit-client
 ```
 
+## Sending requests from a logged-in browser
+
+Some sites only serve content to a browser that has logged in or passed an anti-bot check, and block HTTP clients because of their TLS fingerprint or missing cookies. <ApiLink to="playwright-crawler/class/BrowserFetchHttpClient">`BrowserFetchHttpClient`</ApiLink> sends each request with `fetch()` from a page of a Playwright browser pool, so an HTTP-based crawler gets the browser's fingerprint while keeping its request queue, retries, statistics and sessions.
+
+<CodeBlock language="ts">{CheerioBrowserFetchExample}</CodeBlock>
+
+Every session gets its own page for each origin (the scheme, host and port of the URL), opened by the pool for that session, so it uses the session's proxy and fingerprint. The page shows an empty document of the origin, without loading the site, and the cookies of the session are added to it before each request and saved back to the session after it. Before the first request of a session, `login` gets a new page of the session to log in with. With the default `blockedStatusCodes`, a `401` response retires the session, its pages are closed and the pool retires their browser, and the request is retried with a new session that logs in again. The `use-until-failure` session reuse strategy keeps using one session until then. With the default strategy, the session pool starts a new session for each of the first requests, so each of them gets a new page and logs in. Keep in mind that:
+
+- pages of different sessions share cookies, unless the pool opens every page in its own browser context with `launchContext.useIncognitoPages`,
+- up to `maxOpenPages` pages are kept open (20 by default), and the least recently used idle ones are closed,
+- redirects to another origin fail, as the browser treats them as cross-origin requests,
+- the `proxyUrl` and `ignoreTlsErrors` options are ignored, as the pool decides them for the session,
+- the browser sends only a few HTTP/1.1 requests to one host at a time (6 in Chromium), and the others wait in the browser, counting towards the navigation timeout, so keep `maxConcurrency` around that for sites without HTTP/2.
+
 ## Creating custom HTTP clients
 
 Crawlee provides an interface, <ApiLink to="types/interface/BaseHttpClient">`BaseHttpClient`</ApiLink>, which defines the interface that all HTTP clients must implement. This allows you to create custom HTTP clients tailored to your specific requirements.

docs/guides/http-clients/cheerio-browser-fetch-example.ts

@@ -0,0 +1,25 @@
+import { BrowserFetchHttpClient, CheerioCrawler, playwrightBrowserPool, SessionPool } from 'crawlee';
+
+const browserPool = playwrightBrowserPool();
+
+const crawler = new CheerioCrawler({
+    httpClient: new BrowserFetchHttpClient({
+        browserPool,
+        // Runs before the first request of every session
+        async login(page) {
+            await page.goto('https://example.com/login');
+            await page.fill('#username', process.env.USERNAME!);
+            await page.fill('#password', process.env.PASSWORD!);
+            await page.click('button[type=submit]');
+            await page.waitForURL('https://example.com/account');
+        },
+    }),
+    // Keep using a logged-in session until the site rejects it
+    sessionPool: new SessionPool({ sessionReuseStrategy: 'use-until-failure' }),
+    async requestHandler({ $, request }) {
+        console.log(`${request.url}: ${$('title').text()}`);
+    },
+});
+
+await crawler.run(['https://example.com/account']);
+await browserPool.destroy();

docs/public-api/crawlee-playwright.api.md

@@ -6,6 +6,7 @@
 
 import type { AnyNode } from 'domhandler';
 import type { Awaitable } from '@crawlee/types';
+import { BaseHttpClient } from '@crawlee/http-client';
 import { BasicCrawler } from '@crawlee/basic';
 import type { BasicCrawlerOptions } from '@crawlee/basic';
 import type { BatchAddRequestsResult } from '@crawlee/types';
@@ -22,6 +23,7 @@ import type { BrowserType } from 'playwright';
 import { Cheerio } from 'cheerio';
 import { CheerioAPI } from 'cheerio';
 import { Configuration } from '@crawlee/browser';
+import type { CrawleeLogger } from '@crawlee/types';
 import type { CrawlerRemoteBrowserOptions } from '@crawlee/browser';
 import type { CrawlingContext } from '@crawlee/basic';
 import { CrawlingRequest } from '@crawlee/browser';
@@ -30,7 +32,9 @@ import type { Download } from 'playwright';
 import type { EnqueueLinksOptions } from '@crawlee/basic';
 import type { GetUserDataFromRequest } from '@crawlee/browser';
 import type { GetUserDataFromRequest as GetUserDataFromRequest_2 } from '@crawlee/basic';
+import type { IBrowserPool } from '@crawlee/types';
 import { IRequestManager } from '@crawlee/browser';
+import type { ISession } from '@crawlee/types';
 import type { LaunchOptions } from 'playwright';
 import type { LoadedRequest } from '@crawlee/browser';
 import type { Page } from 'playwright';
@@ -45,6 +49,7 @@ import type { RouterRoutes } from '@crawlee/browser';
 import type { RouterRoutes as RouterRoutes_2 } from '@crawlee/basic';
 import type { RouteSchemas } from '@crawlee/browser';
 import type { RoutesFromSchemas } from '@crawlee/browser';
+import type { SendRequestOptions } from '@crawlee/types';
 import type { SkippedRequestCallback } from '@crawlee/browser';
 import type { StorageTransactionView } from '@crawlee/core';
 import type { UrlPatternInput } from '@crawlee/browser';
@@ -147,6 +152,24 @@ interface BlockRequestsOptions {
     urlPatterns?: string[];
 }
 
+// @public
+export class BrowserFetchHttpClient extends BaseHttpClient {
+    constructor(options: BrowserFetchHttpClientOptions);
+    // (undocumented)
+    protected fetch(): Promise<Response>;
+    // (undocumented)
+    sendRequest(request: Request, options?: SendRequestOptions): Promise<Response>;
+}
+
+// @public (undocumented)
+export interface BrowserFetchHttpClientOptions {
+    browserPool: IBrowserPool<Page>;
+    // (undocumented)
+    logger?: CrawleeLogger;
+    login?: (page: Page, session?: ISession) => Promise<void>;
+    maxOpenPages?: number;
+}
+
 // Not exported by the entry point; reachable only as a referenced type.
 // @public (undocumented)
 type ClickOptions = Parameters<Page['click']>[1];

packages/playwright-crawler/package.json

@@ -67,7 +67,7 @@
         "zod": "catalog:"
     },
     "peerDependencies": {
-        "playwright": "*"
+        "playwright": ">=1.52.0"
     },
     "peerDependenciesMeta": {
         "playwright": {

packages/playwright-crawler/src/index.ts

@@ -4,6 +4,7 @@ export * from './internals/playwright-crawler.js';
 export { launchPlaywright } from './internals/playwright-launcher.js';
 export type { PlaywrightLaunchContext } from './internals/playwright-launcher.js';
 export * from './internals/adaptive-playwright-crawler.js';
+export { BrowserFetchHttpClient, type BrowserFetchHttpClientOptions } from './internals/browser-fetch-http-client.js';
 export { RenderingTypePredictor } from './internals/utils/rendering-type-prediction.js';
 
 export * as playwrightUtils from './internals/utils/playwright-utils.js';