fix(crawl): wait out rate-limited status polls and page results only once the crawl ends (#570)

* fix(crawl): wait out rate-limited status polls and page results only once the crawl ends

* fix(crawl): reuse the terminal poll as the first result page, honor HTTP-date Retry-After, assert the wait

---------

Co-authored-by: erikengervall <5392475+erikengervall@users.noreply.github.com>

Author: erikengervall

src/server.ts

@@ -1897,7 +1897,16 @@ For a programming question, add \`categories: ["developer"]\`; its hits return i
       `/v2/crawl/${encodeURIComponent(jobId)}`,
       originHeaders(origin)
     );
-    const body = (res?.data ?? {}) as any;
+    return collectCrawlPages(client, jobId, origin, (res?.data ?? {}) as any);
+  }
+
+  /** A crawl status body with its remaining result pages appended. */
+  async function collectCrawlPages(
+    client: FirecrawlApp,
+    jobId: string,
+    origin: string,
+    body: any
+  ): Promise<Record<string, unknown>> {
     const initialDocs = Array.isArray(body.data) ? body.data : [];
 
     if (!body.next) {
@@ -1954,18 +1963,38 @@ For a programming question, add \`categories: ["developer"]\`; its hits return i
   ): Promise<Record<string, unknown>> {
     const startedAt = Date.now();
     for (;;) {
-      const status = await getCrawlStatusWithOrigin(client, jobId, origin);
-      if (
-        ['completed', 'failed', 'cancelled'].includes(String(status.status ?? ''))
-      ) {
-        return status;
+      let waitMs = Math.max(1000, pollInterval * 1000);
+      try {
+        // Poll the first page only; result pages are read once the crawl ends.
+        const res = await (client as any).http.get(
+          `/v2/crawl/${encodeURIComponent(jobId)}`,
+          originHeaders(origin)
+        );
+        if (
+          ['completed', 'failed', 'cancelled'].includes(String(res?.data?.status ?? ''))
+        ) {
+          return await collectCrawlPages(client, jobId, origin, res.data);
+        }
+      } catch (error) {
+        const response = (error as { response?: { status?: number; headers?: any; data?: any } })
+          ?.response;
+        if (response?.status !== 429) throw error;
+        // A rate-limited status read says nothing about the crawl: wait it out.
+        const retryAfter = String(
+          response.headers?.['retry-after'] ?? response.data?.retry_after_seconds ?? ''
+        ).trim();
+        const retryAfterMs = /^\d+$/.test(retryAfter)
+          ? Number(retryAfter) * 1000
+          : Date.parse(retryAfter) - Date.now();
+        waitMs = Math.min(
+          60_000,
+          Math.max(waitMs, Number.isFinite(retryAfterMs) ? retryAfterMs : 10_000)
+        );
       }
       if (timeout != null && Date.now() - startedAt > timeout * 1000) {
         throw new Error(`Crawl job ${jobId} did not complete within ${timeout}s`);
       }
-      await new Promise((resolve) =>
-        setTimeout(resolve, Math.max(1000, pollInterval * 1000))
-      );
+      await new Promise((resolve) => setTimeout(resolve, waitMs));
     }
   }
 

tests/mcp-crawl-polling.test.mjs

@@ -0,0 +1,98 @@
+import assert from 'node:assert/strict';
+import { createServer } from 'node:http';
+import test from 'node:test';
+import {
+  getFreePort,
+  parseSseJson,
+  spawnServer,
+  stopChild,
+  waitForHealth,
+} from './helpers/exchange-mcp.mjs';
+
+const CRAWL_ID = 'crawl-1';
+
+async function startCrawlApi() {
+  const requests = [];
+  let statusReads = 0;
+  const server = createServer(async (req, res) => {
+    for await (const _chunk of req);
+    requests.push({ url: req.url, at: Date.now() });
+    const base = `http://127.0.0.1:${server.address().port}`;
+    const json = (status, payload, headers = {}) => {
+      res.writeHead(status, { 'content-type': 'application/json', ...headers });
+      res.end(JSON.stringify(payload));
+    };
+    if (req.method === 'POST' && req.url === '/v2/crawl') {
+      return json(200, { success: true, id: CRAWL_ID });
+    }
+    if (req.url === `/v2/crawl/${CRAWL_ID}?skip=1`) {
+      return json(200, { success: true, data: [{ markdown: 'page two' }], next: null });
+    }
+    if (req.url === `/v2/crawl/${CRAWL_ID}`) {
+      statusReads += 1;
+      if (statusReads === 2) {
+        return json(429, { success: false, error: 'Rate limit exceeded.', retry_after_seconds: 1 }, { 'retry-after': '1' });
+      }
+      const done = statusReads >= 3;
+      return json(200, {
+        success: true,
+        status: done ? 'completed' : 'scraping',
+        completed: done ? 2 : 1,
+        total: 2,
+        data: [{ markdown: 'page one' }],
+        next: `${base}/v2/crawl/${CRAWL_ID}?skip=1`,
+      });
+    }
+    json(404, { success: false, error: `Unhandled ${req.method} ${req.url}` });
+  });
+  await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve));
+  return {
+    requests,
+    url: `http://127.0.0.1:${server.address().port}`,
+    close: () => new Promise((resolve) => server.close(resolve)),
+  };
+}
+
+test('crawl waits out a rate-limited status read and pages results only once it ends', async (t) => {
+  const api = await startCrawlApi();
+  t.after(() => api.close());
+  const port = await getFreePort();
+  const child = spawnServer({
+    CLOUD_SERVICE: 'true',
+    HTTP_STREAMABLE_SERVER: 'true',
+    FASTMCP_ENDPOINT: '/v2/mcp',
+    FIRECRAWL_API_KEY: '',
+    FIRECRAWL_OAUTH_TOKEN: '',
+    FIRECRAWL_API_URL: api.url,
+    FIRECRAWL_MCP_SEARCH_ENABLED: 'false',
+    PORT: String(port),
+  });
+  t.after(() => stopChild(child));
+  await waitForHealth(port, child);
+
+  const response = await fetch(`http://127.0.0.1:${port}/v2/mcp`, {
+    body: JSON.stringify({
+      id: 1,
+      jsonrpc: '2.0',
+      method: 'tools/call',
+      params: { name: 'firecrawl_crawl', arguments: { url: 'https://example.com' } },
+    }),
+    headers: {
+      accept: 'application/json, text/event-stream',
+      'content-type': 'application/json',
+      'x-api-key': 'fc-hosted-test',
+    },
+    method: 'POST',
+    signal: AbortSignal.timeout(20_000),
+  });
+  assert.equal(response.status, 200);
+  const message = parseSseJson(await response.text());
+  assert.notEqual(message.result?.isError, true, JSON.stringify(message));
+  const result = JSON.parse(message.result.content[0].text);
+  assert.equal(result.status, 'completed');
+  assert.deepEqual(result.data.map((doc) => doc.markdown), ['page one', 'page two']);
+  const statusReads = api.requests.filter(({ url }) => url === `/v2/crawl/${CRAWL_ID}`);
+  assert.equal(statusReads.length, 3);
+  assert.ok(statusReads[2].at - statusReads[1].at >= 1000, 'the read after a 429 waits for Retry-After');
+  assert.equal(api.requests.filter(({ url }) => url.endsWith('?skip=1')).length, 1);
+});