fix: Respect robots.txt by default in website-content-crawler

src/const.ts

@@ -161,6 +161,15 @@ If the exact URL cannot be fetched (e.g. a non-http(s) scheme), say so rather th
 export const WEB_FETCH_URL_SCHEME_NOTE =
     'http(s) URLs only - for any other scheme (e.g. ftp:), tell the user this tool cannot fetch it rather than substituting a different URL.';
 
+/**
+ * Input defaults this server applies instead of the Actor's own, keyed by Actor full name. The served schema
+ * shows them and a run starts with them unless the caller sets the field. The platform fills an omitted field
+ * from the Actor's schema, where website-content-crawler's respectRobotsTxtFile defaults to false (#1574).
+ */
+export const SERVED_INPUT_DEFAULTS: Record<string, Record<string, unknown>> = {
+    'apify/website-content-crawler': { respectRobotsTxtFile: true },
+};
+
 export const defaults = {
     actors: [RAG_WEB_BROWSER, WEB_FETCH],
 };

src/tools/actor_input_schema.ts

@@ -7,6 +7,7 @@ import {
     RAG_WEB_BROWSER_WHITELISTED_FIELDS,
     WEB_FETCH,
     WEB_FETCH_URL_SCHEME_NOTE,
+    SERVED_INPUT_DEFAULTS,
 } from '../const.js';
 import { SchemaTooLargeError } from '../errors.js';
 import type { ActorInputSchema, SchemaProperties } from '../types.js';
@@ -210,10 +211,24 @@ export function buildActorInputSchema(actorFullName: string, input: ActorInputSc
         finalSchema.properties.url.description += ` ${WEB_FETCH_URL_SCHEME_NOTE}`;
     }
 
+    applyServedInputDefaults(actorFullName, finalSchema.properties);
+
     finalSchema.$id = getToolSchemaID(actorFullName);
     return { inputSchema: finalSchema };
 }
 
+/** Shows `SERVED_INPUT_DEFAULTS` in the served schema. */
+export function applyServedInputDefaults(actorFullName: string, properties: Record<string, SchemaProperties>): void {
+    for (const [field, value] of Object.entries(SERVED_INPUT_DEFAULTS[actorFullName] ?? {})) {
+        if (properties[field]) properties[field].default = value;
+    }
+}
+
+/** The input a run starts with: `SERVED_INPUT_DEFAULTS` under the caller's fields. */
+export function buildActorRunInput(actorFullName: string, input: Record<string, unknown>): Record<string, unknown> {
+    return { ...SERVED_INPUT_DEFAULTS[actorFullName], ...input };
+}
+
 /**
  * Returns a shallow-cloned input schema that keeps only whitelisted properties
  * and filters the required array accordingly. All other top-level fields are preserved.

src/tools/actors/actor_executor.ts

@@ -6,6 +6,7 @@ import { isActorInputValidationError } from '../../utils/apify_errors.js';
 import { getConsoleLinkContext } from '../../utils/console_link.js';
 import { redactSkyfirePayId } from '../../utils/logging.js';
 import { buildInvalidInputTexts, respondUserError } from '../../utils/mcp.js';
+import { buildActorRunInput } from '../actor_input_schema.js';
 import { buildGetActorRunResponse } from '../runs/get_actor_run.js';
 import { abortRunOnSignal, CALL_ACTOR_WAIT_SECS_DEFAULT, fetchActorRunData } from './actor_run_response.js';
 
@@ -43,7 +44,9 @@ export const actorExecutor: ActorExecutor = {
 
         let actorRun;
         try {
-            actorRun = await apifyClient.actor(actorFullName).start(actorInput, params.callOptions);
+            actorRun = await apifyClient
+                .actor(actorFullName)
+                .start(buildActorRunInput(actorFullName, actorInput), params.callOptions);
         } catch (error) {
             // Platform can reject input our own AJV gate passed (see isActorInputValidationError).
             if (!isActorInputValidationError(error)) throw error;

src/tools/actors/call_actor.ts

@@ -42,7 +42,8 @@ import {
 } from '../../utils/mcp.js';
 import { buildPermissionApprovalTexts } from '../../utils/payment_errors.js';
 import { classifyFailureCategory, extractAjvErrorDetails } from '../../utils/tool_status.js';
-import { extractActorId } from '../../utils/tools.js';
+import { extractActorId, getToolFullName } from '../../utils/tools.js';
+import { buildActorRunInput } from '../actor_input_schema.js';
 import { actorNameToToolName, isActorBlockedUnderPaymentProvider } from '../actor_tool_naming.js';
 import { buildGetActorRunResponse } from '../runs/get_actor_run.js';
 import { actorRunOutputSchema } from '../structured_output_schemas.js';
@@ -667,7 +668,9 @@ export async function executeCallActor(toolArgs: InternalToolArgs): Promise<Tool
 
         if (abortSignal?.aborted) return respondAborted();
 
-        const actorRun = await apifyClient.actor(baseActorName).start(input, callOptions);
+        const actorRun = await apifyClient
+            .actor(baseActorName)
+            .start(buildActorRunInput(getToolFullName(resolution.actor), input), callOptions);
         log.debug('Started Actor run', {
             actorName: baseActorName,
             runId: actorRun.id,

src/tools/widgets/call_actor_widget.ts

@@ -9,7 +9,8 @@ import type { InternalToolArgs, ToolDescriptionContext, ToolEntry, ToolInputSche
 import { ALL_TOOLS_PRESENT, TOOL_TYPE } from '../../types.js';
 import { compileSchema } from '../../utils/ajv.js';
 import { respondServerError } from '../../utils/mcp.js';
-import { extractActorId } from '../../utils/tools.js';
+import { extractActorId, getToolFullName } from '../../utils/tools.js';
+import { buildActorRunInput } from '../actor_input_schema.js';
 import { buildStartRunWidgetResponse } from '../actors/actor_run_response.js';
 import {
     buildCallActorErrorResponse,
@@ -126,7 +127,10 @@ export const callActorWidget: ToolEntry = Object.freeze({
             const { apifyClient } = toolArgs;
 
             const actorClient = apifyClient.actor(baseActorName);
-            const actorRun = await actorClient.start(input, callOptions);
+            const actorRun = await actorClient.start(
+                buildActorRunInput(getToolFullName(resolution.actor), input),
+                callOptions,
+            );
             log.debug('Started Actor run (widget)', {
                 actorName: baseActorName,
                 runId: actorRun.id,

src/utils/actor_details.ts

@@ -5,7 +5,7 @@ import { CODE_RUNTIME_ACTOR_NAME } from '../const.js';
 import { ActorLoadError } from '../errors.js';
 import { connectMCPClient } from '../mcp/client.js';
 import type { PaymentProvider } from '../payments/types.js';
-import { filterSchemaProperties, shortenProperties } from '../tools/actor_input_schema.js';
+import { applyServedInputDefaults, filterSchemaProperties, shortenProperties } from '../tools/actor_input_schema.js';
 import type { Actor, ActorCardOptions, ActorInputSchema, ActorStoreList, StructuredActorCard } from '../types.js';
 import { ACTOR_TOOL_MODE } from '../types.js';
 import { getActorToolResolutionCached } from './actor.js';
@@ -111,6 +111,7 @@ export async function fetchActorDetails(
         }) as ActorInputSchema;
         inputSchema.properties = filterSchemaProperties(inputSchema.properties);
         inputSchema.properties = shortenProperties(inputSchema.properties);
+        applyServedInputDefaults(`${actorInfo.username}/${actorInfo.name}`, inputSchema.properties);
         const actorCard = formatActorToActorCard(actorInfoWithPicture, cardOptions);
         const actorCardStructured = formatActorToStructuredCard(actorInfoWithPicture, cardOptions);
         return {

tests/unit/tools.actor_executor.test.ts

@@ -148,6 +148,35 @@ describe('actorExecutor', () => {
         });
     });
 
+    // The Actor defaults respectRobotsTxtFile to false; its tool sends true unless the caller sets it (#1574).
+    describe('website-content-crawler respectRobotsTxtFile', () => {
+        const WEBSITE_CONTENT_CRAWLER = 'apify/website-content-crawler';
+        const START_URLS = [{ url: 'https://example.com' }];
+
+        it.each([
+            ['an omitted', true, {}],
+            ['an explicit false', false, { respectRobotsTxtFile: false }],
+            ['an explicit true', true, { respectRobotsTxtFile: true }],
+        ])('sends %s field as %s', async (_label, expected, field) => {
+            const { params, spies } = buildParams(
+                { startUrls: START_URLS, ...field },
+                { actorFullName: WEBSITE_CONTENT_CRAWLER },
+            );
+
+            await actorExecutor.executeActorTool(params);
+
+            expect(spies.startInput).toEqual({ startUrls: START_URLS, respectRobotsTxtFile: expected });
+        });
+
+        it('leaves the input of other Actors unchanged', async () => {
+            const { params, spies } = buildParams({ startUrls: START_URLS });
+
+            await actorExecutor.executeActorTool(params);
+
+            expect(spies.startInput).toEqual({ startUrls: START_URLS });
+        });
+    });
+
     describe('platform input-validation error', () => {
         function invalidInputError(message: string): ApifyApiError {
             return new ApifyApiError(

tests/unit/tools.actor_tools_factory.test.ts

@@ -193,6 +193,23 @@ describe('getNormalActorsAsTools()', () => {
         });
     });
 
+    // Regression: #1574 — the Actor defaults respectRobotsTxtFile to false for API runs.
+    it('serves respectRobotsTxtFile with default true only for apify/website-content-crawler', async () => {
+        const respectRobotsTxtFile = { type: 'boolean', default: false, prefill: true } as const;
+        const [crawler, other] = await getNormalActorsAsTools([
+            createMockActorInfo('apify/website-content-crawler', { inputProperties: { respectRobotsTxtFile } }),
+            createMockActorInfo('apify/cheerio-scraper', { inputProperties: { respectRobotsTxtFile } }),
+        ]);
+
+        expect(getToolPublicFieldOnly(crawler).inputSchema.properties?.respectRobotsTxtFile).toMatchObject({
+            type: 'boolean',
+            default: true,
+        });
+        expect(getToolPublicFieldOnly(other).inputSchema.properties?.respectRobotsTxtFile).toMatchObject({
+            default: false,
+        });
+    });
+
     // Regression: #1532 — the RAG description named apify/web-fetch when the session did not serve it.
     it('names apify/web-fetch in the RAG description only when its tool is served', async () => {
         const [tool] = await getNormalActorsAsTools([createMockActorInfo(RAG_WEB_BROWSER)]);

tests/unit/tools.call_actor.run_input.test.ts

@@ -0,0 +1,95 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { callActor } from '../../src/tools/actors/call_actor.js';
+import { callActorWidget } from '../../src/tools/widgets/call_actor_widget.js';
+import type { HelperTool, ToolEntry } from '../../src/types.js';
+import { ACTOR_TOOL_MODE, TOOL_TYPE } from '../../src/types.js';
+import { getActorToolResolutionCached } from '../../src/utils/actor.js';
+import { mockApifyClient, stubToolCallContext } from './helpers/tool_context.js';
+
+/** call-actor and call-actor-widget start a website-content-crawler run with the same input as its direct tool (#1574). */
+vi.mock('../../src/utils/actor.js', () => ({
+    getActorToolResolutionCached: vi.fn(),
+}));
+
+vi.mock('../../src/tools/actors/actor_tools_factory.js', async () => {
+    const actual = await vi.importActual<Record<string, unknown>>('../../src/tools/actors/actor_tools_factory.js');
+    return { ...actual, getActorsAsTools: vi.fn() };
+});
+
+const { getActorsAsTools } = await import('../../src/tools/actors/actor_tools_factory.js');
+
+const WEBSITE_CONTENT_CRAWLER = 'apify/website-content-crawler';
+const START_URLS = [{ url: 'https://example.com' }];
+
+const MOCK_RUN = {
+    id: 'run-1',
+    actId: 'actor-id-wcc',
+    status: 'RUNNING',
+    startedAt: new Date('2026-10-14T10:00:00.000Z'),
+    defaultDatasetId: 'dataset-id-1',
+    defaultKeyValueStoreId: 'kv-id-1',
+};
+
+function mockActorTool(actorFullName: string): ToolEntry {
+    return {
+        type: TOOL_TYPE.ACTOR,
+        name: actorFullName.replace('/', '--'),
+        actorId: 'actor-id',
+        actorFullName,
+        description: actorFullName,
+        inputSchema: { type: 'object', properties: {}, additionalProperties: true } as never,
+        ajvValidate: (() => true) as never,
+    } as unknown as ToolEntry;
+}
+
+describe.each([
+    ['call-actor', callActor, { waitSecs: 0 }],
+    ['call-actor-widget', callActorWidget, {}],
+])('%s run input', (_name, tool, extraArgs) => {
+    const callWithInput = async (actorFullName: string, input: Record<string, unknown>) => {
+        vi.mocked(getActorToolResolutionCached).mockResolvedValue({ toolMode: ACTOR_TOOL_MODE.RUN, actorFullName });
+        vi.mocked(getActorsAsTools).mockResolvedValue({ tools: [mockActorTool(actorFullName)], errors: [] });
+        const startSpy = vi.fn().mockResolvedValue(MOCK_RUN);
+        const client = mockApifyClient({ actor: () => ({ start: startSpy }) });
+        await (tool as HelperTool).call(stubToolCallContext({ actor: actorFullName, input, ...extraArgs }, client));
+        return startSpy.mock.calls[0]?.[0];
+    };
+
+    beforeEach(() => {
+        vi.mocked(getActorToolResolutionCached).mockReset();
+        vi.mocked(getActorsAsTools).mockReset();
+    });
+
+    it.each([
+        ['an omitted', true, {}],
+        ['an explicit false', false, { respectRobotsTxtFile: false }],
+    ])('sends %s website-content-crawler respectRobotsTxtFile as %s', async (_label, expected, field) => {
+        expect(await callWithInput(WEBSITE_CONTENT_CRAWLER, { startUrls: START_URLS, ...field })).toEqual({
+            startUrls: START_URLS,
+            respectRobotsTxtFile: expected,
+        });
+    });
+
+    it('sends respectRobotsTxtFile as true when website-content-crawler is named by its ID', async () => {
+        vi.mocked(getActorToolResolutionCached).mockResolvedValue({
+            toolMode: ACTOR_TOOL_MODE.RUN,
+            actorFullName: WEBSITE_CONTENT_CRAWLER,
+        });
+        vi.mocked(getActorsAsTools).mockResolvedValue({ tools: [mockActorTool(WEBSITE_CONTENT_CRAWLER)], errors: [] });
+        const startSpy = vi.fn().mockResolvedValue(MOCK_RUN);
+        const client = mockApifyClient({ actor: () => ({ start: startSpy }) });
+
+        await (tool as HelperTool).call(
+            stubToolCallContext({ actor: 'aYG0l9s7dbB7j3gbS', input: { startUrls: START_URLS }, ...extraArgs }, client),
+        );
+
+        expect(startSpy.mock.calls[0]?.[0]).toEqual({ startUrls: START_URLS, respectRobotsTxtFile: true });
+    });
+
+    it('leaves the input of other Actors unchanged', async () => {
+        expect(await callWithInput('apify/cheerio-scraper', { startUrls: START_URLS })).toEqual({
+            startUrls: START_URLS,
+        });
+    });
+});

tests/unit/utils.actor_details.test.ts

@@ -181,6 +181,35 @@ describe('fetchActorDetails()', () => {
 
         await expect(fetchActorDetails(client, 'apify/instagram-scraper')).rejects.toMatchObject({ statusCode: 401 });
     });
+
+    // Regression: #1574 — call-actor sends respectRobotsTxtFile true when omitted, so the schema it points to says so.
+    it.each([
+        ['website-content-crawler', true],
+        ['cheerio-scraper', false],
+    ])('serves the respectRobotsTxtFile default of apify/%s as %s', async (name, expected) => {
+        const respectRobotsTxtFile = {
+            title: 'Respect robots.txt',
+            description: 'If enabled, the crawler consults robots.txt.',
+            type: 'boolean',
+            default: false,
+            prefill: true,
+        };
+        const client = mockApifyClient({
+            token: 'test-token',
+            actor: () => ({
+                get: async () => ({ id: 'actor-id', username: 'apify', name, stats: {} }),
+                defaultBuild: async () => ({
+                    get: async () => ({
+                        actorDefinition: { input: { type: 'object', properties: { respectRobotsTxtFile } } },
+                    }),
+                }),
+            }),
+        });
+
+        const result = await fetchActorDetails(client, 'actor-id');
+
+        expect(result?.inputSchema.properties.respectRobotsTxtFile).toMatchObject({ default: expected });
+    });
 });
 
 describe('getMcpToolsMessage()', () => {