diff --git a/README.md b/README.md index 2cdcac66aa..e2a468eb40 100644 --- a/README.md +++ b/README.md @@ -1063,6 +1063,12 @@ To disable telemetry, set the environment variable: export FIRECRAWL_NO_TELEMETRY=1 ``` +Requests from the CLI are tagged `integration: cli`. A tool that wraps the CLI can tag them as its own by setting `FIRECRAWL_INTEGRATION`: the CLI sends the value, trimmed, as the `X-Integration` header on every API request (a value with control or non-ASCII characters cannot be a header and is not sent), and the API attributes the request to it when it is one of its integration names or any value starting with an underscore. Any other value is ignored by the API and never makes a request fail. + +```bash +export FIRECRAWL_INTEGRATION=_my-tool +``` + --- ## Experimental diff --git a/package.json b/package.json index 266a22e404..4338745752 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "firecrawl-cli", - "version": "1.26.14", + "version": "1.26.15", "publishConfig": { "tag": "latest" }, diff --git a/src/__tests__/commands/doctor.test.ts b/src/__tests__/commands/doctor.test.ts index a66a98adf8..29737b1038 100644 --- a/src/__tests__/commands/doctor.test.ts +++ b/src/__tests__/commands/doctor.test.ts @@ -158,6 +158,27 @@ describe('runChecks', () => { return c; } + it('sends FIRECRAWL_INTEGRATION as X-Integration on its API pings', async () => { + vi.stubEnv('FIRECRAWL_INTEGRATION', 'hermes'); + initializeConfig({ + apiKey: 'fc-abc123def456', + apiUrl: 'https://api.firecrawl.dev', + }); + stubFetch({}); + try { + await runChecks({}); + } finally { + vi.unstubAllEnvs(); + } + const apiCalls = mockFetch.mock.calls.filter(([url]) => + String(url).startsWith('https://api.firecrawl.dev/') + ); + expect(apiCalls).toHaveLength(2); + for (const [, init] of apiCalls) { + expect(init.headers['X-Integration']).toBe('hermes'); + } + }); + it('fails the API Key check when no key is configured', async () => { stubFetch({}); const { checks } = await runChecks({}); @@ -400,6 +421,25 @@ describe('runSupportAsk', () => { ); }); + it('sends FIRECRAWL_INTEGRATION as X-Integration to support Ask', async () => { + vi.stubEnv('FIRECRAWL_INTEGRATION', 'hermes'); + initializeConfig({ + apiKey: 'fc-test', + apiUrl: 'https://api.firecrawl.dev', + }); + mockFetch.mockResolvedValue({ + ok: true, + status: 200, + json: async () => ({ answer: 'Try increasing waitFor.' }), + }); + try { + await runSupportAsk({ jobId, query: 'why is it empty?' }); + } finally { + vi.unstubAllEnvs(); + } + expect(mockFetch.mock.calls[0][1].headers['X-Integration']).toBe('hermes'); + }); + it('does not call support Ask without an API key', async () => { const exitCode = await runSupportAsk({ jobId }); diff --git a/src/__tests__/commands/feedback.test.ts b/src/__tests__/commands/feedback.test.ts index cb5d0542b8..2e523b6484 100644 --- a/src/__tests__/commands/feedback.test.ts +++ b/src/__tests__/commands/feedback.test.ts @@ -7,6 +7,7 @@ import { parsePageNumbersArg, } from '../../commands/feedback'; import { + executeSearchFeedback, parseValuableResultsArg, type ValuableResultInput, } from '../../commands/search-feedback'; @@ -159,6 +160,44 @@ describe('executeEndpointFeedback', () => { }); }); + it('sends FIRECRAWL_INTEGRATION as X-Integration on endpoint and search feedback', async () => { + vi.stubEnv('FIRECRAWL_INTEGRATION', 'rtk'); + mockFetch.mockResolvedValue({ + ok: true, + status: 200, + json: async () => ({ success: true, feedbackId: 'feedback-1' }), + }); + + try { + await executeEndpointFeedback({ + endpoint: 'scrape', + jobId: '0193f6c5-1234-7890-abcd-1234567890ab', + rating: 'good', + }); + await executeSearchFeedback({ searchId: 'search-1', rating: 'good' }); + } finally { + vi.unstubAllEnvs(); + } + + const [endpointUrl, endpointRequest] = mockFetch.mock.calls[0]; + expect(endpointUrl).toBe('https://api.firecrawl.dev/v2/feedback'); + expect(endpointRequest.headers['X-Integration']).toBe('rtk'); + expect(JSON.parse(endpointRequest.body)).toMatchObject({ + origin: 'cli', + integration: 'cli', + }); + const [searchUrl, searchRequest] = mockFetch.mock.calls[1]; + expect(searchUrl).toBe( + 'https://api.firecrawl.dev/v2/search/search-1/feedback' + ); + expect(searchRequest.headers['X-Integration']).toBe('rtk'); + expect(JSON.parse(searchRequest.body)).toEqual({ + rating: 'good', + origin: 'cli', + integration: 'cli', + }); + }); + it('rejects --valuable-results on non-search endpoints', async () => { const result = await executeEndpointFeedback({ endpoint: 'scrape', diff --git a/src/__tests__/utils/integration.test.ts b/src/__tests__/utils/integration.test.ts new file mode 100644 index 0000000000..d2d998b567 --- /dev/null +++ b/src/__tests__/utils/integration.test.ts @@ -0,0 +1,110 @@ +/** + * Tests for the X-Integration header + */ + +import http from 'http'; +import type { AddressInfo } from 'net'; +import { + afterAll, + afterEach, + beforeAll, + describe, + expect, + it, + vi, +} from 'vitest'; +import { getClient, keylessRequest, resetClient } from '../../utils/client'; +import { initializeConfig, resetConfig } from '../../utils/config'; +import { integrationHeaders } from '../../utils/integration'; + +describe('integrationHeaders', () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it('sends nothing when FIRECRAWL_INTEGRATION is unset or blank', () => { + vi.stubEnv('FIRECRAWL_INTEGRATION', undefined); + expect(integrationHeaders()).toEqual({}); + vi.stubEnv('FIRECRAWL_INTEGRATION', ' '); + expect(integrationHeaders()).toEqual({}); + }); + + it.each(['rtk', 'hermes', '_rtk', 'not a valid tag!'])( + 'sends %j as is, with no list to match', + (value) => { + vi.stubEnv('FIRECRAWL_INTEGRATION', value); + expect(integrationHeaders()).toEqual({ 'X-Integration': value }); + } + ); + + it('trims surrounding whitespace', () => { + vi.stubEnv('FIRECRAWL_INTEGRATION', ' hermes\n'); + expect(integrationHeaders()).toEqual({ 'X-Integration': 'hermes' }); + }); + + it.each(['bad\nvalue', 'tag\u2713'])( + 'drops %j, which cannot be sent as a header', + (value) => { + vi.stubEnv('FIRECRAWL_INTEGRATION', value); + expect(integrationHeaders()).toEqual({}); + } + ); +}); + +describe('X-Integration on the wire', () => { + const received: http.IncomingHttpHeaders[] = []; + let server: http.Server; + let apiUrl: string; + + beforeAll(async () => { + server = http.createServer((req, res) => { + received.push(req.headers); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ success: true, data: {} })); + }); + await new Promise((resolve) => + server.listen(0, '127.0.0.1', resolve) + ); + apiUrl = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + }); + + afterAll(async () => { + await new Promise((resolve) => server.close(resolve)); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + received.length = 0; + resetClient(); + resetConfig(); + }); + + it.each(['rtk', 'hermes', '_rtk'])( + 'puts %j on SDK client requests while the body keeps integration cli', + async (value) => { + vi.stubEnv('FIRECRAWL_INTEGRATION', value); + const client = getClient({ apiKey: 'fc-test', apiUrl }); + await (client as any).http.post('/v2/scrape', { integration: 'cli' }); + await (client as any).http.get('/v2/search/developer?integration=cli'); + expect(received.map((h) => h['x-integration'])).toEqual([value, value]); + } + ); + + it('lets a request with a junk value succeed', async () => { + vi.stubEnv('FIRECRAWL_INTEGRATION', 'not a valid tag!'); + initializeConfig({ apiUrl }); + await expect( + keylessRequest('/v2/scrape', { url: 'https://example.com' }) + ).resolves.toEqual({ success: true, data: {} }); + expect(received[0]['x-integration']).toBe('not a valid tag!'); + }); + + it('lets a request succeed without the header when the value cannot be sent', async () => { + vi.stubEnv('FIRECRAWL_INTEGRATION', 'bad\nvalue'); + const client = getClient({ apiKey: 'fc-test', apiUrl }); + await expect( + (client as any).http.post('/v2/scrape', { integration: 'cli' }) + ).resolves.toMatchObject({ status: 200 }); + expect(received[0]['x-integration']).toBeUndefined(); + }); +}); diff --git a/src/commands/credit-usage.ts b/src/commands/credit-usage.ts index 117d921003..7b3c02be4f 100644 --- a/src/commands/credit-usage.ts +++ b/src/commands/credit-usage.ts @@ -6,6 +6,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { getConfig, validateConfig } from '../utils/config'; import { getClient } from '../utils/client'; +import { integrationHeaders } from '../utils/integration'; export interface CreditUsageResult { success: boolean; @@ -58,6 +59,7 @@ export async function executeCreditUsage( headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json', + ...integrationHeaders(), }, }); diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index c6839fffd6..aea91af301 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -17,6 +17,7 @@ import { isJobId } from '../utils/job'; import { detectAgents, type AgentDetection } from '../utils/agents'; import { compareVersions, getLatestVersion } from '../utils/npm-registry'; import { getAuthSource, type AuthSource } from './status'; +import { integrationHeaders } from '../utils/integration'; type CheckStatus = 'pass' | 'warn' | 'fail'; type DoctorAuthSource = AuthSource | 'flag'; @@ -121,6 +122,7 @@ async function pingCreditUsage( headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json', + ...integrationHeaders(), }, }); const durationMs = Date.now() - start; @@ -156,6 +158,7 @@ async function fetchQueueStatus( headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json', + ...integrationHeaders(), }, }); if (!response.ok) { @@ -698,6 +701,7 @@ export async function runSupportAsk(options: DoctorOptions): Promise { headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json', + ...integrationHeaders(), }, body: JSON.stringify({ question, jobId }), }); diff --git a/src/commands/feedback.ts b/src/commands/feedback.ts index 63b5332696..215626403d 100644 --- a/src/commands/feedback.ts +++ b/src/commands/feedback.ts @@ -11,6 +11,7 @@ import { type ValuableResultInput, type ValuableSourceInput, } from './search-feedback'; +import { integrationHeaders } from '../utils/integration'; export type EndpointFeedbackEndpoint = 'search' | 'scrape' | 'parse' | 'map'; @@ -313,6 +314,7 @@ export async function executeEndpointFeedback( headers: { ...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}), 'Content-Type': 'application/json', + ...integrationHeaders(), }, body: JSON.stringify(body), }); diff --git a/src/commands/interact.ts b/src/commands/interact.ts index 6a99b36dbe..c42b810b12 100644 --- a/src/commands/interact.ts +++ b/src/commands/interact.ts @@ -11,6 +11,7 @@ import { clearInteractSession, } from '../utils/interact-session'; import { writeOutput } from '../utils/output'; +import { integrationHeaders } from '../utils/integration'; export interface InteractExecuteOptions { scrapeId?: string; @@ -60,7 +61,7 @@ function buildHeaders(apiKey: string | undefined, keyless: boolean) { headers.Authorization = `Bearer ${apiKey}`; } if (keyless) Object.assign(headers, KEYLESS_CLI_HEADERS); - return headers; + return { ...headers, ...integrationHeaders() }; } /** diff --git a/src/commands/list.ts b/src/commands/list.ts index 044c4cf23b..829a2be6cf 100644 --- a/src/commands/list.ts +++ b/src/commands/list.ts @@ -11,6 +11,7 @@ import { } from './alexandria'; import { writeOutput } from '../utils/output'; import { getApiKey, getConfig } from '../utils/config'; +import { integrationHeaders } from '../utils/integration'; type Selectors = Record; type ListOptions = AlexandriaOptions & { @@ -97,6 +98,7 @@ async function requestCategories( headers: { Authorization: `Bearer ${getApiKey(options.apiKey)}`, 'X-Request-ID': requestId, + ...integrationHeaders(), }, signal: AbortSignal.timeout(getConfig().timeoutMs ?? 30000), redirect: 'error', diff --git a/src/commands/monitor.ts b/src/commands/monitor.ts index 7cf26cf61f..f0eb018286 100644 --- a/src/commands/monitor.ts +++ b/src/commands/monitor.ts @@ -20,6 +20,7 @@ import * as fs from 'fs'; import { Command } from 'commander'; import { getConfig, validateConfig } from '../utils/config'; import { writeOutput } from '../utils/output'; +import { integrationHeaders } from '../utils/integration'; const DEFAULT_API_URL = 'https://api.firecrawl.dev'; @@ -63,6 +64,7 @@ async function monitorRequest( const headers: Record = { Authorization: `Bearer ${apiKey}`, 'X-Origin': 'cli', + ...integrationHeaders(), }; if (init.body !== undefined) headers['Content-Type'] = 'application/json'; diff --git a/src/commands/parse.ts b/src/commands/parse.ts index 7e7e7ab8ba..cd852b23c6 100644 --- a/src/commands/parse.ts +++ b/src/commands/parse.ts @@ -14,6 +14,7 @@ import type { ScrapeFormat } from '../types/scrape'; import { getClient, isKeylessMode, KEYLESS_CLI_HEADERS } from '../utils/client'; import { getConfig, validateConfig } from '../utils/config'; import { handleScrapeOutput } from '../utils/output'; +import { integrationHeaders } from '../utils/integration'; const DEFAULT_API_URL = 'https://api.firecrawl.dev'; @@ -186,12 +187,11 @@ export async function executeParse( method: 'POST', // Multipart options are parsed after auth, so the header carries the // CLI origin to the keyless check. - headers: - !keyless && apiKey - ? { Authorization: `Bearer ${apiKey}` } - : keyless - ? { ...KEYLESS_CLI_HEADERS } - : undefined, + headers: { + ...(!keyless && apiKey ? { Authorization: `Bearer ${apiKey}` } : {}), + ...(keyless ? KEYLESS_CLI_HEADERS : {}), + ...integrationHeaders(), + }, body: form, }); diff --git a/src/commands/search-feedback.ts b/src/commands/search-feedback.ts index f99994766a..66dcdf0fb7 100644 --- a/src/commands/search-feedback.ts +++ b/src/commands/search-feedback.ts @@ -1,5 +1,6 @@ import { getConfig, validateConfig } from '../utils/config'; import { getClient } from '../utils/client'; +import { integrationHeaders } from '../utils/integration'; export type SearchFeedbackRating = 'good' | 'bad' | 'partial'; @@ -156,6 +157,7 @@ export async function executeSearchFeedback( headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json', + ...integrationHeaders(), }, body: JSON.stringify(body), }); diff --git a/src/commands/status.ts b/src/commands/status.ts index b1c1ca15f8..b9a81cc2a1 100644 --- a/src/commands/status.ts +++ b/src/commands/status.ts @@ -10,6 +10,7 @@ import { isAuthenticated } from '../utils/auth'; import { getConfig, validateConfig } from '../utils/config'; import { loadCredentials } from '../utils/credentials'; import { isSearchFeedbackDisabledLocally } from './search-feedback'; +import { integrationHeaders } from '../utils/integration'; export type AuthSource = 'env' | 'stored' | 'none'; @@ -81,6 +82,7 @@ async function fetchQueueStatus( headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json', + ...integrationHeaders(), }, }); @@ -107,6 +109,7 @@ async function fetchCreditUsage( headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json', + ...integrationHeaders(), }, }); diff --git a/src/commands/terms.ts b/src/commands/terms.ts index 7b03e1bedc..52dd0f97e8 100644 --- a/src/commands/terms.ts +++ b/src/commands/terms.ts @@ -1,6 +1,7 @@ import { Command } from 'commander'; import { getApiKey, getConfig } from '../utils/config'; import { writeOutput } from '../utils/output'; +import { integrationHeaders } from '../utils/integration'; type TermsOptions = { apiKey?: string; @@ -42,6 +43,7 @@ export async function requestTerms( headers: { Authorization: `Bearer ${key}`, 'Content-Type': 'application/json', + ...integrationHeaders(), }, redirect: 'error', signal: AbortSignal.timeout(getConfig().timeoutMs ?? 30000), diff --git a/src/utils/client.ts b/src/utils/client.ts index d61d16e89a..0ce35d37cd 100644 --- a/src/utils/client.ts +++ b/src/utils/client.ts @@ -13,6 +13,7 @@ import { updateConfig, type GlobalConfig, } from './config'; +import { integrationHeaders, withIntegrationHeader } from './integration'; let clientInstance: Firecrawl | null = null; @@ -32,7 +33,8 @@ export function isKeylessMode(apiKey?: string, apiUrl?: string): boolean { /** * Headers for keyless requests. The API reads X-Origin to attribute keyless * use, and a keyless prompt's signup link, to the CLI; requests without a body - * (GET research and developer lookups, interact stop) carry nothing else. + * (GET research and developer lookups, interact stop) carry only this and + * X-Integration. * Keyless error messages are printed as the API sends them: their * firecrawl.dev/k/ link already resolves to CLI attribution. */ @@ -47,7 +49,11 @@ export async function keylessRequest( const apiUrl = (getConfig().apiUrl || DEFAULT_API_URL).replace(/\/$/, ''); const response = await fetch(`${apiUrl}${path}`, { method: 'POST', - headers: { 'Content-Type': 'application/json', ...KEYLESS_CLI_HEADERS }, + headers: { + 'Content-Type': 'application/json', + ...KEYLESS_CLI_HEADERS, + ...integrationHeaders(), + }, body: JSON.stringify(body), }); const json: any = await response.json().catch(() => ({})); @@ -63,7 +69,11 @@ export async function keylessGet(path: string): Promise { const apiUrl = (getConfig().apiUrl || DEFAULT_API_URL).replace(/\/$/, ''); const response = await fetch(`${apiUrl}${path}`, { method: 'GET', - headers: { 'Content-Type': 'application/json', ...KEYLESS_CLI_HEADERS }, + headers: { + 'Content-Type': 'application/json', + ...KEYLESS_CLI_HEADERS, + ...integrationHeaders(), + }, }); const json: any = await response.json().catch(() => ({})); if (!response.ok) { @@ -128,7 +138,7 @@ export function getClient( backoffFactor: options.backoffFactor ?? config.backoffFactor, }; - return new Firecrawl(clientOptions); + return withIntegrationHeader(new Firecrawl(clientOptions)); } // Return singleton instance or create one @@ -144,7 +154,7 @@ export function getClient( backoffFactor: config.backoffFactor, }; - clientInstance = new Firecrawl(clientOptions); + clientInstance = withIntegrationHeader(new Firecrawl(clientOptions)); } return clientInstance; diff --git a/src/utils/integration.ts b/src/utils/integration.ts new file mode 100644 index 0000000000..18227b5a16 --- /dev/null +++ b/src/utils/integration.ts @@ -0,0 +1,24 @@ +import type { Firecrawl } from 'firecrawl'; + +/** + * FIRECRAWL_INTEGRATION as an X-Integration header, for every API request. + * The API ignores a value it does not recognize, so the CLI keeps no list; it + * only drops values that cannot be sent as a header at all. + */ +export function integrationHeaders(): Record { + const tag = process.env.FIRECRAWL_INTEGRATION?.trim(); + return tag && /^[\x20-\x7e]+$/.test(tag) ? { 'X-Integration': tag } : {}; +} + +/** Adds the integration header to every request an SDK client sends. */ +export function withIntegrationHeader(client: T): T { + const defaults = ( + client as unknown as { + http?: { + instance?: { defaults?: { headers?: Record } }; + }; + } + ).http?.instance?.defaults?.headers; + if (defaults) Object.assign(defaults, integrationHeaders()); + return client; +} diff --git a/vitest.config.mjs b/vitest.config.mjs index d8ea2abfd3..6bb69ea622 100644 --- a/vitest.config.mjs +++ b/vitest.config.mjs @@ -5,6 +5,7 @@ export default defineConfig({ globals: true, environment: 'node', include: ['src/**/*.test.ts'], + env: { FIRECRAWL_INTEGRATION: '' }, coverage: { provider: 'v8', reporter: ['text', 'json', 'html'],