diff --git a/README.md b/README.md index 957188fc7d..826dee29f8 100644 --- a/README.md +++ b/README.md @@ -703,6 +703,67 @@ firecrawl crawl https://example.com --wait -o crawl-results.json --pretty --- +### `batch-scrape` - Scrape a list of URLs + +Scrape many known URLs as one async job. Unlike multi-URL `scrape`, every requested format is kept and results come back as JSON. + +```bash +# Start a batch scrape (returns job ID) +firecrawl batch-scrape https://example.com https://example.org + +# Read URLs from a file (one per line; blank lines and # comments ignored) +firecrawl batch-scrape --urls-file urls.txt + +# Wait for results +firecrawl batch-scrape --urls-file urls.txt --format markdown,links --wait --progress + +# Check status (returns documents scraped so far) +firecrawl batch-scrape + +# Cancel a job +firecrawl batch-scrape --cancel +``` + +#### Batch Scrape Options + +| Option | Description | +| ------------------------------ | ----------------------------------------------- | +| `--urls-file ` | File with one URL per line | +| `-f, --format ` | Output format(s), comma-separated | +| `--only-main-content` | Extract only main content | +| `--wait-for ` | Wait time before scraping each page | +| `--max-age ` | Maximum age of cached content in milliseconds | +| `--schema ` | JSON schema for extraction (adds `json` format) | +| `--schema-file ` | Path to JSON schema file | +| `--prompt ` | Extraction prompt (adds `json` format) | +| `--scrape-options ` | JSON scrape options applied to every URL | +| `--scrape-options-file ` | Path to scrape options JSON file | +| `--wait` | Wait for the job to complete | +| `--progress` | Show progress while waiting | +| `--timeout ` | Timeout when waiting | +| `--poll-interval ` | Status check interval (default: 2) | +| `--webhook ` | Webhook URL or configuration | +| `--max-concurrency ` | Max concurrent scrapes for this job | +| `--ignore-invalid-urls` | Skip invalid URLs instead of failing the job | +| `--status` | Check status of a job by job ID | +| `--cancel` | Cancel an active job by job ID | +| `-o, --output ` | Save output to file | +| `--pretty` | Pretty print JSON output | + +#### Examples + +```bash +# Extract the same fields from every page +firecrawl batch-scrape --urls-file products.txt --wait \ + --schema '{"type":"object","properties":{"name":{"type":"string"},"price":{"type":"string"}}}' \ + -o products.json --pretty + +# Notify a webhook instead of waiting +firecrawl batch-scrape --urls-file urls.txt --webhook https://example.com/hooks/firecrawl +``` + +--- + ### `credit-usage` - Check your credits ```bash @@ -1026,6 +1087,9 @@ firecrawl https://example.com --format markdown,links,images ```bash # Just pass multiple URLs -- results are saved to .firecrawl/ firecrawl scrape https://firecrawl.dev https://firecrawl.dev/blog https://docs.firecrawl.dev + +# Many URLs, other formats, or JSON extraction: run one batch job +firecrawl batch-scrape --urls-file urls.txt --format markdown,links --wait -o results.json ``` ### Combine with other tools diff --git a/package.json b/package.json index f5c1fdf948..641fcce562 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "firecrawl-cli", - "version": "1.26.17", + "version": "1.26.18", "publishConfig": { "tag": "latest" }, diff --git a/skills/firecrawl-scrape/SKILL.md b/skills/firecrawl-scrape/SKILL.md index 975ed57dee..6d2b3d7c04 100644 --- a/skills/firecrawl-scrape/SKILL.md +++ b/skills/firecrawl-scrape/SKILL.md @@ -24,7 +24,7 @@ firecrawl scrape "" --only-main-content -o .firecrawl/page.md # Wait for JS to render, then scrape firecrawl scrape "" --wait-for 3000 -o .firecrawl/page.md -# Multiple URLs (markdown only; each saved to .firecrawl/; -o is ignored) +# A few URLs (markdown only; each saved to .firecrawl/; -o is ignored) firecrawl scrape https://example.com https://example.com/blog https://example.com/docs # Get markdown and links together @@ -38,6 +38,40 @@ Run `firecrawl scrape --help` for the full option list. **Done when:** the page content or provider result has been checked for errors and inspected in bounded sections to answer the request. Preserve source links and disclose partial results. +## Structured JSON from a page + +Pass a JSON schema with the `json` format to get typed fields instead of markdown: + +```bash +firecrawl scrape "" --format json \ + --schema '{"type":"object","properties":{"plan":{"type":"string"},"price":{"type":"string"}},"required":["plan"]}' \ + -o .firecrawl/pricing.json + +# Large schemas: keep them in a file +firecrawl scrape "" --format json --schema-file schema.json -o .firecrawl/page.json +``` + +The result is under `.json` in the output. `--schema` only applies when `json` is in `--format`; add other formats with commas (`--format markdown,json`). For the same schema across many URLs, use `batch-scrape --schema` below. + +## Many known URLs + +Use `firecrawl batch-scrape` for a list of known URLs when you need formats other than markdown, JSON extraction, a webhook, or a job you can check later. Keep the multi-URL `scrape` above for a few pages of markdown. + +```bash +# Start a job (prints a job ID), then check it later +firecrawl batch-scrape https://example.com/a https://example.com/b +firecrawl batch-scrape + +# Wait and save every document as JSON (urls.txt: one URL per line, # comments ok) +firecrawl batch-scrape --urls-file urls.txt --format markdown,links --wait -o .firecrawl/batch.json + +# Same extraction for every URL (--schema or --prompt adds the json format) +firecrawl batch-scrape --urls-file urls.txt --wait \ + --schema '{"type":"object","properties":{"title":{"type":"string"}}}' -o .firecrawl/batch.json +``` + +Results are in `.data[]`, one document per URL with `metadata.sourceURL`. Add `--webhook ` to get notified, `--ignore-invalid-urls` to skip bad entries, and `--cancel` with a job ID to stop a job. Run `firecrawl batch-scrape --help` for all options. + ## Find tools, inspect inputs, and get help Use the CLI help to check supported options rather than guessing: @@ -98,7 +132,7 @@ The cap applies to each PDF, not the whole command or total credits. Extra forma - **Prefer plain scrape over `--query`.** Scrape to a file, then use `grep`, `head`, or read the markdown directly — you can search and reason over the full content yourself. Use `--query` only when you want a single targeted answer without saving the page (costs 5 extra credits). - **Scrape handles static pages and JS-rendered SPAs.** Escalate to `interact` when the page needs interaction (clicks, form fills, pagination) or scrape misses content. -- Multiple URLs are scraped concurrently — check `firecrawl --status` for your concurrency limit. This mode saves markdown only and ignores `-o`; other requested formats are dropped. If markdown wasn't requested, the whole JSON response is written into the `.md` file. +- Multiple URLs are scraped concurrently — check `firecrawl --status` for your concurrency limit. This mode saves markdown only and ignores `-o`; other requested formats are dropped. If markdown wasn't requested, the whole JSON response is written into the `.md` file. Use `batch-scrape` when you need other formats or JSON extraction. - Single format outputs raw content. Multiple formats (e.g., `--format markdown,links`) output JSON. - Always quote URLs — shell interprets `?` and `&` as special characters. - Naming convention: `.firecrawl/{site}-{path}.md` diff --git a/skills/firecrawl/SKILL.md b/skills/firecrawl/SKILL.md index 9818ce1191..ef0359e3e4 100644 --- a/skills/firecrawl/SKILL.md +++ b/skills/firecrawl/SKILL.md @@ -39,6 +39,7 @@ Follow this escalation pattern: | Answer a coding question | `developer` | Issues, merged PRs, READMEs, and docs — not a general web page | | Find US law or regulation | `gov` | US statutes, regulations, codes, and court opinions | | Get a page's content | `scrape` | Have a URL, page is static or JS-rendered | +| Scrape a list of known URLs | `batch-scrape` | Many URLs, non-markdown formats, JSON extraction, or a webhook | | Find URLs within a site | `map` | Need to locate a specific subpage | | Bulk extract a site section | `crawl` | Need many pages (e.g., all /docs/) | | AI-powered data extraction | `agent` | Need structured data from complex sites | @@ -78,6 +79,7 @@ A client context/output error does not prove the provider failed. Keep the reque - **Finding research papers (biomedical, clinical, or scientific literature; PubMed, bioRxiv, medRxiv, arXiv)** -> [firecrawl-research-index](../firecrawl-research-index/SKILL.md). Use the paper index instead of scraping PubMed or Google Scholar by hand; `search --categories research` is a website filter, not the paper index. - **Answering a library, API, error, or known-bug question from issues, merged PRs, READMEs, or docs** -> [firecrawl-developer-index](../firecrawl-developer-index/SKILL.md) - **Scraping a known URL** -> [firecrawl-scrape](../firecrawl-scrape/SKILL.md) +- **Scraping a list of known URLs as one job (batch scrape) or extracting JSON with a schema** -> [firecrawl-scrape](../firecrawl-scrape/SKILL.md) - **Finding URLs on a known site** -> [firecrawl-map](../firecrawl-map/SKILL.md) - **Bulk extraction from a docs section or site** -> [firecrawl-crawl](../firecrawl-crawl/SKILL.md) - **AI-powered structured extraction from complex sites** -> [firecrawl-agent](../firecrawl-agent/SKILL.md) diff --git a/src/__tests__/cli-aliases.test.ts b/src/__tests__/cli-aliases.test.ts index d39949cea2..d6ac5eba3c 100644 --- a/src/__tests__/cli-aliases.test.ts +++ b/src/__tests__/cli-aliases.test.ts @@ -22,6 +22,7 @@ describe('CLI compatibility aliases', { timeout: 30000 }, () => { scrape.handleAllScrapeCommand = (_url, options) => print(options); require('./dist/commands/parse').handleParseCommand = print; require('./dist/commands/crawl').handleCrawlCommand = print; + require('./dist/commands/batch-scrape').handleBatchScrapeCommand = print; require('./dist/commands/agent').handleAgentCommand = print; process.argv = [process.execPath, ${JSON.stringify(cliPath)}, ...${JSON.stringify(args)}]; require(${JSON.stringify(cliPath)}); @@ -103,7 +104,7 @@ describe('CLI compatibility aliases', { timeout: 30000 }, () => { expect(alias.stdout).toContain('"formats":["markdown","links"]'); }); - testWithBuiltCli.each(['crawl', 'agent'])( + testWithBuiltCli.each(['crawl', 'batch-scrape', 'agent'])( 'preserves %s job status', (command) => { const result = run([ diff --git a/src/__tests__/commands/batch-scrape.test.ts b/src/__tests__/commands/batch-scrape.test.ts new file mode 100644 index 0000000000..88281c8227 --- /dev/null +++ b/src/__tests__/commands/batch-scrape.test.ts @@ -0,0 +1,430 @@ +/** + * Tests for batch-scrape command + */ + +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { + collectBatchScrapeUrls, + executeBatchScrape, +} from '../../commands/batch-scrape'; +import { getClient } from '../../utils/client'; +import { initializeConfig } from '../../utils/config'; +import { setupTest, teardownTest } from '../utils/mock-client'; + +vi.mock('../../utils/client', async () => { + const actual = await vi.importActual('../../utils/client'); + return { + ...actual, + getClient: vi.fn(), + }; +}); + +const JOB_ID = '550e8400-e29b-41d4-a716-446655440000'; +const URLS = ['https://example.com', 'https://example.org']; + +describe('collectBatchScrapeUrls', () => { + let tmpDir: string; + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'batch-scrape-test-')); + }); + + afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + it('combines positional URLs with a URLs file, skipping blanks and comments', () => { + const file = path.join(tmpDir, 'urls.txt'); + fs.writeFileSync( + file, + '# docs\nhttps://example.org/a\n\n example.org/b \r\n# end\nhttps://example.com\n' + ); + + expect(collectBatchScrapeUrls(['https://example.com'], file)).toEqual([ + 'https://example.com', + 'https://example.org/a', + 'https://example.org/b', + ]); + }); + + it('normalizes positional URLs without a file', () => { + expect(collectBatchScrapeUrls(['example.com'])).toEqual([ + 'https://example.com', + ]); + }); + + it('throws a readable error for a missing file', () => { + expect(() => + collectBatchScrapeUrls([], path.join(tmpDir, 'missing.txt')) + ).toThrow('Could not read --urls-file'); + }); +}); + +describe('executeBatchScrape', () => { + let mockClient: any; + + beforeEach(() => { + setupTest(); + initializeConfig({ + apiKey: 'test-api-key', + apiUrl: 'https://api.firecrawl.dev', + }); + + mockClient = { + startBatchScrape: vi.fn(), + getBatchScrapeStatus: vi.fn(), + batchScrape: vi.fn(), + cancelBatchScrape: vi.fn(), + }; + vi.mocked(getClient).mockReturnValue(mockClient as any); + }); + + afterEach(() => { + teardownTest(); + vi.clearAllMocks(); + }); + + describe('start (no wait)', () => { + it('starts a batch job and returns the job ID', async () => { + mockClient.startBatchScrape.mockResolvedValue({ + id: JOB_ID, + url: `https://api.firecrawl.dev/v2/batch/scrape/${JOB_ID}`, + }); + + const result = await executeBatchScrape({ urls: URLS }); + + expect(mockClient.startBatchScrape).toHaveBeenCalledWith(URLS, { + options: { onlyMainContent: false }, + integration: 'cli', + }); + expect(mockClient.batchScrape).not.toHaveBeenCalled(); + expect(result).toEqual({ + success: true, + data: { + jobId: JOB_ID, + url: `https://api.firecrawl.dev/v2/batch/scrape/${JOB_ID}`, + status: 'processing', + }, + }); + }); + + it('passes job options and reports invalid URLs', async () => { + mockClient.startBatchScrape.mockResolvedValue({ + id: JOB_ID, + url: 'u', + invalidURLs: ['not a url'], + }); + + const result = await executeBatchScrape({ + urls: URLS, + webhook: 'https://hooks.example.com/batch', + maxConcurrency: 3, + ignoreInvalidUrls: true, + }); + + expect(mockClient.startBatchScrape).toHaveBeenCalledWith( + URLS, + expect.objectContaining({ + webhook: 'https://hooks.example.com/batch', + maxConcurrency: 3, + ignoreInvalidURLs: true, + }) + ); + expect(result.data.invalidURLs).toEqual(['not a url']); + }); + + it('merges --scrape-options under explicit flags', async () => { + mockClient.startBatchScrape.mockResolvedValue({ id: JOB_ID, url: 'u' }); + + await executeBatchScrape({ + urls: URLS, + format: 'markdown,links', + onlyMainContent: true, + waitFor: 1000, + maxAge: 60000, + scrapeOptions: { + formats: ['html'], + waitFor: 5, + mobile: true, + }, + }); + + expect(mockClient.startBatchScrape.mock.calls[0][1].options).toEqual({ + formats: ['markdown', 'links'], + onlyMainContent: true, + waitFor: 1000, + maxAge: 60000, + mobile: true, + }); + }); + + it('requires at least one URL', async () => { + const result = await executeBatchScrape({ urls: [] }); + + expect(result.success).toBe(false); + expect(result.error).toContain('At least one URL is required'); + expect(mockClient.startBatchScrape).not.toHaveBeenCalled(); + }); + }); + + describe('json format', () => { + const schema = { + type: 'object', + properties: { title: { type: 'string' } }, + }; + + it('uses only the json format when --schema is passed without --format', async () => { + mockClient.startBatchScrape.mockResolvedValue({ id: JOB_ID, url: 'u' }); + + await executeBatchScrape({ urls: URLS, schema }); + + expect( + mockClient.startBatchScrape.mock.calls[0][1].options.formats + ).toEqual([{ type: 'json', schema }]); + }); + + it('adds the json format next to requested formats for --prompt', async () => { + mockClient.startBatchScrape.mockResolvedValue({ id: JOB_ID, url: 'u' }); + + await executeBatchScrape({ + urls: URLS, + format: 'markdown', + prompt: 'Extract the title', + }); + + expect( + mockClient.startBatchScrape.mock.calls[0][1].options.formats + ).toEqual(['markdown', { type: 'json', prompt: 'Extract the title' }]); + }); + + it('fills an explicit json format with schema and prompt', async () => { + mockClient.startBatchScrape.mockResolvedValue({ id: JOB_ID, url: 'u' }); + + await executeBatchScrape({ + urls: URLS, + format: 'json,links', + schema, + prompt: 'Extract the title', + }); + + expect( + mockClient.startBatchScrape.mock.calls[0][1].options.formats + ).toEqual([ + { type: 'json', schema, prompt: 'Extract the title' }, + 'links', + ]); + }); + + it('keeps --scrape-options formats when adding the json format', async () => { + mockClient.startBatchScrape.mockResolvedValue({ id: JOB_ID, url: 'u' }); + + await executeBatchScrape({ + urls: URLS, + schema, + scrapeOptions: { + formats: ['markdown', { type: 'json', prompt: 'old' }, 'links'], + }, + }); + + expect( + mockClient.startBatchScrape.mock.calls[0][1].options.formats + ).toEqual(['markdown', { type: 'json', schema }, 'links']); + }); + + it('rejects a json format from --scrape-options without a schema or prompt', async () => { + const result = await executeBatchScrape({ + urls: URLS, + scrapeOptions: { formats: ['markdown', 'json'] }, + }); + + expect(result.success).toBe(false); + expect(result.error).toBe( + 'The json format requires --schema, --schema-file, or --prompt' + ); + expect(mockClient.startBatchScrape).not.toHaveBeenCalled(); + }); + + it('rejects the json format without a schema or prompt', async () => { + const result = await executeBatchScrape({ urls: URLS, format: 'json' }); + + expect(result).toEqual({ + success: false, + error: 'The json format requires --schema, --schema-file, or --prompt', + }); + expect(mockClient.startBatchScrape).not.toHaveBeenCalled(); + }); + }); + + describe('status', () => { + it('checks status when given a job ID', async () => { + const job = { + id: JOB_ID, + status: 'completed', + completed: 2, + total: 2, + data: [{ markdown: '# A' }, { markdown: '# B' }], + }; + mockClient.getBatchScrapeStatus.mockResolvedValue(job); + + const result = await executeBatchScrape({ jobId: JOB_ID }); + + expect(mockClient.getBatchScrapeStatus).toHaveBeenCalledWith(JOB_ID); + expect(mockClient.startBatchScrape).not.toHaveBeenCalled(); + expect(result).toEqual({ success: true, data: job }); + }); + + it('requires a job ID for --status', async () => { + const result = await executeBatchScrape({ urls: URLS, status: true }); + + expect(result).toEqual({ + success: false, + error: 'Status mode requires a job ID', + }); + expect(mockClient.getBatchScrapeStatus).not.toHaveBeenCalled(); + expect(mockClient.startBatchScrape).not.toHaveBeenCalled(); + }); + + it('returns API errors', async () => { + mockClient.getBatchScrapeStatus.mockRejectedValue( + new Error('Job not found') + ); + + const result = await executeBatchScrape({ jobId: JOB_ID }); + + expect(result).toEqual({ success: false, error: 'Job not found' }); + }); + }); + + describe('wait', () => { + const job = { + id: JOB_ID, + status: 'completed', + completed: 2, + total: 2, + data: [{ markdown: '# A' }, { markdown: '# B' }], + }; + + it('uses the SDK waiter with poll interval and timeout', async () => { + mockClient.batchScrape.mockResolvedValue(job); + + const result = await executeBatchScrape({ + urls: URLS, + wait: true, + timeout: 60, + }); + + expect(mockClient.batchScrape).toHaveBeenCalledWith(URLS, { + options: { onlyMainContent: false }, + integration: 'cli', + pollInterval: 2, + timeout: 60, + }); + expect(mockClient.startBatchScrape).not.toHaveBeenCalled(); + expect(result).toEqual({ success: true, data: job }); + }); + + it('polls with progress until the job finishes', async () => { + const stderr = vi + .spyOn(process.stderr, 'write') + .mockImplementation(() => true); + mockClient.startBatchScrape.mockResolvedValue({ id: JOB_ID, url: 'u' }); + mockClient.getBatchScrapeStatus + .mockResolvedValueOnce({ ...job, status: 'scraping', completed: 1 }) + .mockResolvedValueOnce({ ...job, data: [] }) + .mockResolvedValueOnce(job); + + const result = await executeBatchScrape({ + urls: URLS, + wait: true, + progress: true, + pollInterval: 0.001, + }); + + expect(mockClient.getBatchScrapeStatus).toHaveBeenNthCalledWith( + 1, + JOB_ID, + { autoPaginate: false } + ); + expect(mockClient.getBatchScrapeStatus).toHaveBeenLastCalledWith(JOB_ID); + expect(result).toEqual({ success: true, data: job }); + expect(stderr).toHaveBeenCalledWith( + expect.stringContaining('Progress: 1/2 pages (scraping)') + ); + stderr.mockRestore(); + }); + + it('times out the progress loop with a hint naming the job ID', async () => { + const stderr = vi + .spyOn(process.stderr, 'write') + .mockImplementation(() => true); + const now = vi.spyOn(Date, 'now'); + now.mockReturnValueOnce(0).mockReturnValue(2000); + mockClient.startBatchScrape.mockResolvedValue({ id: JOB_ID, url: 'u' }); + mockClient.getBatchScrapeStatus.mockResolvedValue({ + ...job, + status: 'scraping', + completed: 1, + }); + + const result = await executeBatchScrape({ + urls: URLS, + wait: true, + progress: true, + pollInterval: 0.001, + timeout: 1, + }); + + expect(mockClient.getBatchScrapeStatus).toHaveBeenCalledTimes(1); + expect(result.success).toBe(false); + expect(result.error).toContain('Timeout after 1 seconds'); + expect(result.error).toContain(`firecrawl batch-scrape ${JOB_ID}`); + now.mockRestore(); + stderr.mockRestore(); + }); + }); + + describe('cancel', () => { + it('cancels a job by ID', async () => { + mockClient.cancelBatchScrape.mockResolvedValue(true); + + const result = await executeBatchScrape({ + jobId: JOB_ID, + cancel: true, + }); + + expect(mockClient.cancelBatchScrape).toHaveBeenCalledWith(JOB_ID); + expect(mockClient.getBatchScrapeStatus).not.toHaveBeenCalled(); + expect(result).toEqual({ + success: true, + data: { id: JOB_ID, status: 'cancelled' }, + }); + }); + + it('reports a failed cancellation', async () => { + mockClient.cancelBatchScrape.mockResolvedValue(false); + + const result = await executeBatchScrape({ + jobId: JOB_ID, + cancel: true, + }); + + expect(result).toEqual({ + success: false, + error: `Failed to cancel batch scrape job ${JOB_ID}`, + }); + }); + + it('requires a job ID', async () => { + const result = await executeBatchScrape({ urls: URLS, cancel: true }); + + expect(result).toEqual({ + success: false, + error: 'Cancel mode requires a job ID', + }); + expect(mockClient.cancelBatchScrape).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/src/commands/batch-scrape.ts b/src/commands/batch-scrape.ts new file mode 100644 index 0000000000..65d2ddde91 --- /dev/null +++ b/src/commands/batch-scrape.ts @@ -0,0 +1,250 @@ +/** + * Batch scrape command implementation + */ + +import { readFileSync } from 'fs'; +import type { + BatchScrapeCommandOptions, + BatchScrapeResult, +} from '../types/batch-scrape'; +import { getClient } from '../utils/client'; +import { parseFormats } from '../utils/options'; +import { writeOutput } from '../utils/output'; +import { normalizeUrl } from '../utils/url'; + +const TERMINAL_STATUSES = ['completed', 'failed', 'cancelled']; + +/** + * Collect URLs from positional arguments and an optional file with one URL + * per line. Blank lines and lines starting with # are ignored. + */ +export function collectBatchScrapeUrls( + positional: string[], + urlsFile?: string +): string[] { + let fromFile: string[] = []; + if (urlsFile) { + let content: string; + try { + content = readFileSync(urlsFile, 'utf-8'); + } catch { + throw new Error(`Could not read --urls-file: ${urlsFile}`); + } + fromFile = content + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => line && !line.startsWith('#')); + } + + return [...new Set([...positional, ...fromFile].map(normalizeUrl))]; +} + +/** + * Build the scrape options applied to every URL. Explicit flags override + * --scrape-options; --schema and --prompt add the json format. + */ +export function buildBatchScrapeOptions( + options: BatchScrapeCommandOptions +): Record { + const { schema, prompt } = options; + const wantsJson = schema !== undefined || prompt !== undefined; + const scrapeOptions: Record = { + onlyMainContent: false, + ...options.scrapeOptions, + }; + + const baseFormats: unknown[] | undefined = options.format + ? parseFormats(options.format) + : Array.isArray(scrapeOptions.formats) + ? scrapeOptions.formats + : undefined; + const formats = baseFormats ?? (wantsJson ? [] : undefined); + if (formats) { + if (formats.includes('json') && !wantsJson) { + throw new Error( + 'The json format requires --schema, --schema-file, or --prompt' + ); + } + if (wantsJson) { + const isJson = (format: unknown) => + format === 'json' || + (typeof format === 'object' && + format !== null && + (format as { type?: unknown }).type === 'json'); + const jsonFormat = { + type: 'json', + ...(schema !== undefined ? { schema } : {}), + ...(prompt !== undefined ? { prompt } : {}), + }; + const jsonIndex = formats.findIndex(isJson); + const merged = formats.filter((format) => !isJson(format)); + merged.splice( + jsonIndex === -1 ? merged.length : jsonIndex, + 0, + jsonFormat + ); + scrapeOptions.formats = merged; + } else { + scrapeOptions.formats = formats; + } + } + + if (options.onlyMainContent) { + scrapeOptions.onlyMainContent = true; + } + if (options.waitFor !== undefined) { + scrapeOptions.waitFor = options.waitFor; + } + if (options.maxAge !== undefined) { + scrapeOptions.maxAge = options.maxAge; + } + + return scrapeOptions; +} + +/** + * Execute batch scrape command + */ +export async function executeBatchScrape( + options: BatchScrapeCommandOptions +): Promise { + try { + const app = getClient({ apiKey: options.apiKey, apiUrl: options.apiUrl }); + const { jobId, urls = [], timeout } = options; + + if ((options.cancel || options.status) && !jobId) { + return { + success: false, + error: `${options.cancel ? 'Cancel' : 'Status'} mode requires a job ID`, + }; + } + + if (options.cancel && jobId) { + const cancelled = await app.cancelBatchScrape(jobId); + if (!cancelled) { + return { + success: false, + error: `Failed to cancel batch scrape job ${jobId}`, + }; + } + return { success: true, data: { id: jobId, status: 'cancelled' } }; + } + + if (jobId) { + return { success: true, data: await app.getBatchScrapeStatus(jobId) }; + } + + if (urls.length === 0) { + return { + success: false, + error: + 'At least one URL is required. Pass URLs as arguments or use --urls-file.', + }; + } + + const batchOptions: any = { + options: buildBatchScrapeOptions(options), + integration: 'cli', + }; + if (options.webhook) { + batchOptions.webhook = options.webhook; + } + if (options.maxConcurrency !== undefined) { + batchOptions.maxConcurrency = options.maxConcurrency; + } + if (options.ignoreInvalidUrls) { + batchOptions.ignoreInvalidURLs = true; + } + + if (!options.wait) { + const response = await app.startBatchScrape(urls, batchOptions); + return { + success: true, + data: { + jobId: response.id, + url: response.url, + status: 'processing', + ...(response.invalidURLs?.length + ? { invalidURLs: response.invalidURLs } + : {}), + }, + }; + } + + const pollInterval = options.pollInterval ?? 2; + + if (!options.progress) { + const job = await app.batchScrape(urls, { + ...batchOptions, + pollInterval, + ...(timeout !== undefined ? { timeout } : {}), + }); + return { success: true, data: job }; + } + + const response = await app.startBatchScrape(urls, batchOptions); + process.stderr.write(`Batch scraping ${urls.length} URLs...\n`); + process.stderr.write(`Job ID: ${response.id}\n`); + + const startTime = Date.now(); + while (true) { + await new Promise((resolve) => setTimeout(resolve, pollInterval * 1000)); + + const status = await app.getBatchScrapeStatus(response.id, { + autoPaginate: false, + }); + process.stderr.write( + `\rProgress: ${status.completed}/${status.total} pages (${status.status})` + ); + + if (TERMINAL_STATUSES.includes(status.status)) { + process.stderr.write('\n'); + return { + success: true, + data: await app.getBatchScrapeStatus(response.id), + }; + } + + if (timeout !== undefined && Date.now() - startTime > timeout * 1000) { + process.stderr.write('\n'); + return { + success: false, + error: `Timeout after ${timeout} seconds. Batch scrape still in progress; check it with: firecrawl batch-scrape ${response.id}`, + }; + } + } + } catch (error) { + return { + success: false, + error: error instanceof Error ? error.message : 'Unknown error occurred', + }; + } +} + +/** + * Handle batch scrape command output + */ +export async function handleBatchScrapeCommand( + options: BatchScrapeCommandOptions +): Promise { + const result = await executeBatchScrape(options); + + if (!result.success) { + console.error('Error:', result.error); + process.exit(1); + } + + const started = result.data && 'jobId' in result.data; + const payload = started ? { success: true, data: result.data } : result.data; + const outputContent = options.pretty + ? JSON.stringify(payload, null, 2) + : JSON.stringify(payload); + + writeOutput(outputContent, options.output, !!options.output); + + if (started) { + process.stderr.write( + `Check status: firecrawl batch-scrape ${result.data.jobId}\n` + ); + } +} diff --git a/src/index.ts b/src/index.ts index 59fd3957b9..b03ea06c4d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -23,6 +23,10 @@ import { initializeConfig, updateConfig } from './utils/config'; import { configure, viewConfig } from './commands/config'; import { handleCreditUsageCommand } from './commands/credit-usage'; import { handleCrawlCommand } from './commands/crawl'; +import { + collectBatchScrapeUrls, + handleBatchScrapeCommand, +} from './commands/batch-scrape'; import { handleMapCommand } from './commands/map'; import { handleParseCommand } from './commands/parse'; import { createMonitorCommand } from './commands/monitor'; @@ -98,6 +102,7 @@ initializeConfig(); const AUTH_REQUIRED_COMMANDS = [ 'download', 'crawl', + 'batch-scrape', 'map', 'feedback', 'search-feedback', @@ -765,6 +770,157 @@ function createCrawlCommand(): Command { return crawlCmd; } +/** + * Create and configure the batch-scrape command + */ +function createBatchScrapeCommand(): Command { + const batchCmd = new Command('batch-scrape') + .description( + 'Scrape a list of known URLs as one async job (returns a job ID unless --wait)' + ) + .argument('[urls-or-job-id...]', 'URLs to scrape, or a job ID to check') + .option( + '--urls-file ', + 'File with one URL per line (blank lines and # comments ignored)' + ) + .option( + '-f, --format ', + 'Output format(s), comma-separated (default: markdown). Available: markdown, html, rawHtml, links, images, screenshot, summary, changeTracking, json, attributes, branding' + ) + .option('--only-main-content', 'Include only main content', false) + .option( + '--wait-for ', + 'Wait time before scraping in milliseconds', + parseInt + ) + .option( + '--max-age ', + 'Maximum age of cached content in milliseconds', + parseInt + ) + .option( + '--schema ', + 'JSON schema for structured extraction (adds the json format)' + ) + .option('--schema-file ', 'Path to JSON schema file') + .option( + '--prompt ', + 'Extraction prompt for the json format (adds the json format)' + ) + .option( + '--scrape-options ', + 'JSON scrape options applied to every URL (explicit flags win)' + ) + .option('--scrape-options-file ', 'Path to scrape options JSON file') + .option('--status', 'Check status of existing batch scrape job', false) + .option('--cancel', 'Cancel active batch scrape job by job ID', false) + .option('--wait', 'Wait for the job to complete and return results', false) + .option( + '--poll-interval ', + 'Polling interval in seconds when waiting (default: 2)', + parseFloat + ) + .option( + '--timeout ', + 'Timeout in seconds when waiting (default: no timeout)', + parseFloat + ) + .option('--progress', 'Show progress while waiting', false) + .option('--webhook ', 'Webhook URL or webhook configuration') + .option( + '--max-concurrency ', + 'Maximum concurrent scrapes for this job', + parseInt + ) + .option( + '--ignore-invalid-urls', + 'Skip invalid URLs instead of failing the job', + false + ) + .option( + '-k, --api-key ', + 'Firecrawl API key (overrides global --api-key)' + ) + .option('--api-url ', 'API URL (overrides global --api-url)') + .option('-o, --output ', 'Output file path (default: stdout)') + .option('--pretty', 'Pretty print JSON output', false) + .action(async (positional: string[], options, command: Command) => { + const jobId = + positional.length === 1 && isJobId(positional[0]) + ? positional[0] + : undefined; + + let urls: string[] = []; + let schema: Record | undefined; + let scrapeOptions: Record | undefined; + let webhook: string | Record | undefined; + + try { + if (!jobId) { + urls = collectBatchScrapeUrls(positional, options.urlsFile); + } + schema = + parseJsonObject(options.schema, undefined, '--schema') ?? + parseJsonObject(undefined, options.schemaFile, '--schema-file'); + scrapeOptions = + parseJsonObject( + options.scrapeOptions, + undefined, + '--scrape-options' + ) ?? + parseJsonObject( + undefined, + options.scrapeOptionsFile, + '--scrape-options-file' + ); + webhook = parseWebhookOption(options.webhook, '--webhook'); + } catch (error) { + console.error( + 'Error:', + error instanceof Error ? error.message : String(error) + ); + process.exit(1); + } + + await handleBatchScrapeCommand({ + urls, + jobId, + status: options.status || command.parent?.opts().status, + cancel: options.cancel, + wait: options.wait, + pollInterval: options.pollInterval, + timeout: options.timeout, + progress: options.progress, + output: options.output, + pretty: options.pretty, + apiKey: options.apiKey, + apiUrl: options.apiUrl, + format: options.format, + onlyMainContent: options.onlyMainContent, + waitFor: options.waitFor, + maxAge: options.maxAge, + schema, + prompt: options.prompt, + scrapeOptions, + webhook, + maxConcurrency: options.maxConcurrency, + ignoreInvalidUrls: options.ignoreInvalidUrls, + }); + }); + + batchCmd.addHelpText( + 'after', + ` +Examples: + firecrawl batch-scrape https://example.com https://example.org + firecrawl batch-scrape --urls-file urls.txt --format markdown,links --wait -o .firecrawl/batch.json + firecrawl batch-scrape --urls-file urls.txt --wait --schema '{"type":"object","properties":{"title":{"type":"string"}}}' + firecrawl batch-scrape +` + ); + return addFormatsAlias(batchCmd); +} + /** * Create and configure the map command */ @@ -2394,6 +2550,7 @@ Examples: // Add core commands to main program program.addCommand(createCrawlCommand()); +program.addCommand(createBatchScrapeCommand()); program.addCommand(createMapCommand()); program.addCommand(createParseCommand()); program.addCommand(createMonitorCommand()); diff --git a/src/types/batch-scrape.ts b/src/types/batch-scrape.ts new file mode 100644 index 0000000000..3573b7c0e9 --- /dev/null +++ b/src/types/batch-scrape.ts @@ -0,0 +1,56 @@ +/** + * Types for batch-scrape command + */ + +export interface BatchScrapeCommandOptions { + /** API key for Firecrawl */ + apiKey?: string; + /** API URL for Firecrawl */ + apiUrl?: string; + /** URLs to scrape (ignored when jobId is set) */ + urls?: string[]; + /** Existing batch scrape job ID for status checks or cancellation */ + jobId?: string; + /** Check status of an existing batch scrape job */ + status?: boolean; + /** Cancel an existing batch scrape job */ + cancel?: boolean; + /** Wait for the batch scrape to complete */ + wait?: boolean; + /** Polling interval in seconds when waiting */ + pollInterval?: number; + /** Timeout in seconds when waiting */ + timeout?: number; + /** Show progress while waiting */ + progress?: boolean; + /** Output file path */ + output?: string; + /** Pretty print JSON output */ + pretty?: boolean; + /** Comma-separated output formats */ + format?: string; + /** Include only main content */ + onlyMainContent?: boolean; + /** Wait time before scraping (ms) */ + waitFor?: number; + /** Maximum age of cached content (ms) */ + maxAge?: number; + /** JSON schema for the json format */ + schema?: Record; + /** Extraction prompt for the json format */ + prompt?: string; + /** Scrape options applied to every URL, overridden by explicit flags */ + scrapeOptions?: Record; + /** Webhook URL or webhook config */ + webhook?: string | Record; + /** Maximum concurrent scrapes for this job */ + maxConcurrency?: number; + /** Skip invalid URLs instead of failing the whole job */ + ignoreInvalidUrls?: boolean; +} + +export interface BatchScrapeResult { + success: boolean; + data?: any; + error?: string; +}