diff --git a/src/const.ts b/src/const.ts index a268caf85..db287fa8c 100644 --- a/src/const.ts +++ b/src/const.ts @@ -161,6 +161,15 @@ If the exact URL cannot be fetched (e.g. a non-http(s) scheme), say so rather th export const WEB_FETCH_URL_SCHEME_NOTE = 'http(s) URLs only - for any other scheme (e.g. ftp:), tell the user this tool cannot fetch it rather than substituting a different URL.'; +/** + * Input defaults this server applies instead of the Actor's own, keyed by Actor full name. The served schema + * shows them and a run starts with them unless the caller sets the field. The platform fills an omitted field + * from the Actor's schema, where website-content-crawler's respectRobotsTxtFile defaults to false (#1574). + */ +export const SERVED_INPUT_DEFAULTS: Record> = { + 'apify/website-content-crawler': { respectRobotsTxtFile: true }, +}; + export const defaults = { actors: [RAG_WEB_BROWSER, WEB_FETCH], }; diff --git a/src/tools/actor_input_schema.ts b/src/tools/actor_input_schema.ts index ce0b8a69c..0cb6c4b4e 100644 --- a/src/tools/actor_input_schema.ts +++ b/src/tools/actor_input_schema.ts @@ -7,6 +7,7 @@ import { RAG_WEB_BROWSER_WHITELISTED_FIELDS, WEB_FETCH, WEB_FETCH_URL_SCHEME_NOTE, + SERVED_INPUT_DEFAULTS, } from '../const.js'; import { SchemaTooLargeError } from '../errors.js'; import type { ActorInputSchema, SchemaProperties } from '../types.js'; @@ -210,10 +211,24 @@ export function buildActorInputSchema(actorFullName: string, input: ActorInputSc finalSchema.properties.url.description += ` ${WEB_FETCH_URL_SCHEME_NOTE}`; } + applyServedInputDefaults(actorFullName, finalSchema.properties); + finalSchema.$id = getToolSchemaID(actorFullName); return { inputSchema: finalSchema }; } +/** Shows `SERVED_INPUT_DEFAULTS` in the served schema. */ +export function applyServedInputDefaults(actorFullName: string, properties: Record): void { + for (const [field, value] of Object.entries(SERVED_INPUT_DEFAULTS[actorFullName] ?? {})) { + if (properties[field]) properties[field].default = value; + } +} + +/** The input a run starts with: `SERVED_INPUT_DEFAULTS` under the caller's fields. */ +export function buildActorRunInput(actorFullName: string, input: Record): Record { + return { ...SERVED_INPUT_DEFAULTS[actorFullName], ...input }; +} + /** * Returns a shallow-cloned input schema that keeps only whitelisted properties * and filters the required array accordingly. All other top-level fields are preserved. diff --git a/src/tools/actors/actor_executor.ts b/src/tools/actors/actor_executor.ts index 6e6d380a7..ea08aff55 100644 --- a/src/tools/actors/actor_executor.ts +++ b/src/tools/actors/actor_executor.ts @@ -6,6 +6,7 @@ import { isActorInputValidationError } from '../../utils/apify_errors.js'; import { getConsoleLinkContext } from '../../utils/console_link.js'; import { redactSkyfirePayId } from '../../utils/logging.js'; import { buildInvalidInputTexts, respondUserError } from '../../utils/mcp.js'; +import { buildActorRunInput } from '../actor_input_schema.js'; import { buildGetActorRunResponse } from '../runs/get_actor_run.js'; import { abortRunOnSignal, CALL_ACTOR_WAIT_SECS_DEFAULT, fetchActorRunData } from './actor_run_response.js'; @@ -43,7 +44,9 @@ export const actorExecutor: ActorExecutor = { let actorRun; try { - actorRun = await apifyClient.actor(actorFullName).start(actorInput, params.callOptions); + actorRun = await apifyClient + .actor(actorFullName) + .start(buildActorRunInput(actorFullName, actorInput), params.callOptions); } catch (error) { // Platform can reject input our own AJV gate passed (see isActorInputValidationError). if (!isActorInputValidationError(error)) throw error; diff --git a/src/tools/actors/call_actor.ts b/src/tools/actors/call_actor.ts index 1f35b6415..56feb686e 100644 --- a/src/tools/actors/call_actor.ts +++ b/src/tools/actors/call_actor.ts @@ -42,7 +42,8 @@ import { } from '../../utils/mcp.js'; import { buildPermissionApprovalTexts } from '../../utils/payment_errors.js'; import { classifyFailureCategory, extractAjvErrorDetails } from '../../utils/tool_status.js'; -import { extractActorId } from '../../utils/tools.js'; +import { extractActorId, getToolFullName } from '../../utils/tools.js'; +import { buildActorRunInput } from '../actor_input_schema.js'; import { actorNameToToolName, isActorBlockedUnderPaymentProvider } from '../actor_tool_naming.js'; import { buildGetActorRunResponse } from '../runs/get_actor_run.js'; import { actorRunOutputSchema } from '../structured_output_schemas.js'; @@ -667,7 +668,9 @@ export async function executeCallActor(toolArgs: InternalToolArgs): Promise { }); }); + // The Actor defaults respectRobotsTxtFile to false; its tool sends true unless the caller sets it (#1574). + describe('website-content-crawler respectRobotsTxtFile', () => { + const WEBSITE_CONTENT_CRAWLER = 'apify/website-content-crawler'; + const START_URLS = [{ url: 'https://example.com' }]; + + it.each([ + ['an omitted', true, {}], + ['an explicit false', false, { respectRobotsTxtFile: false }], + ['an explicit true', true, { respectRobotsTxtFile: true }], + ])('sends %s field as %s', async (_label, expected, field) => { + const { params, spies } = buildParams( + { startUrls: START_URLS, ...field }, + { actorFullName: WEBSITE_CONTENT_CRAWLER }, + ); + + await actorExecutor.executeActorTool(params); + + expect(spies.startInput).toEqual({ startUrls: START_URLS, respectRobotsTxtFile: expected }); + }); + + it('leaves the input of other Actors unchanged', async () => { + const { params, spies } = buildParams({ startUrls: START_URLS }); + + await actorExecutor.executeActorTool(params); + + expect(spies.startInput).toEqual({ startUrls: START_URLS }); + }); + }); + describe('platform input-validation error', () => { function invalidInputError(message: string): ApifyApiError { return new ApifyApiError( diff --git a/tests/unit/tools.actor_tools_factory.test.ts b/tests/unit/tools.actor_tools_factory.test.ts index 072a871be..08700bfa9 100644 --- a/tests/unit/tools.actor_tools_factory.test.ts +++ b/tests/unit/tools.actor_tools_factory.test.ts @@ -193,6 +193,23 @@ describe('getNormalActorsAsTools()', () => { }); }); + // Regression: #1574 — the Actor defaults respectRobotsTxtFile to false for API runs. + it('serves respectRobotsTxtFile with default true only for apify/website-content-crawler', async () => { + const respectRobotsTxtFile = { type: 'boolean', default: false, prefill: true } as const; + const [crawler, other] = await getNormalActorsAsTools([ + createMockActorInfo('apify/website-content-crawler', { inputProperties: { respectRobotsTxtFile } }), + createMockActorInfo('apify/cheerio-scraper', { inputProperties: { respectRobotsTxtFile } }), + ]); + + expect(getToolPublicFieldOnly(crawler).inputSchema.properties?.respectRobotsTxtFile).toMatchObject({ + type: 'boolean', + default: true, + }); + expect(getToolPublicFieldOnly(other).inputSchema.properties?.respectRobotsTxtFile).toMatchObject({ + default: false, + }); + }); + // Regression: #1532 — the RAG description named apify/web-fetch when the session did not serve it. it('names apify/web-fetch in the RAG description only when its tool is served', async () => { const [tool] = await getNormalActorsAsTools([createMockActorInfo(RAG_WEB_BROWSER)]); diff --git a/tests/unit/tools.call_actor.run_input.test.ts b/tests/unit/tools.call_actor.run_input.test.ts new file mode 100644 index 000000000..a28289f7d --- /dev/null +++ b/tests/unit/tools.call_actor.run_input.test.ts @@ -0,0 +1,95 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { callActor } from '../../src/tools/actors/call_actor.js'; +import { callActorWidget } from '../../src/tools/widgets/call_actor_widget.js'; +import type { HelperTool, ToolEntry } from '../../src/types.js'; +import { ACTOR_TOOL_MODE, TOOL_TYPE } from '../../src/types.js'; +import { getActorToolResolutionCached } from '../../src/utils/actor.js'; +import { mockApifyClient, stubToolCallContext } from './helpers/tool_context.js'; + +/** call-actor and call-actor-widget start a website-content-crawler run with the same input as its direct tool (#1574). */ +vi.mock('../../src/utils/actor.js', () => ({ + getActorToolResolutionCached: vi.fn(), +})); + +vi.mock('../../src/tools/actors/actor_tools_factory.js', async () => { + const actual = await vi.importActual>('../../src/tools/actors/actor_tools_factory.js'); + return { ...actual, getActorsAsTools: vi.fn() }; +}); + +const { getActorsAsTools } = await import('../../src/tools/actors/actor_tools_factory.js'); + +const WEBSITE_CONTENT_CRAWLER = 'apify/website-content-crawler'; +const START_URLS = [{ url: 'https://example.com' }]; + +const MOCK_RUN = { + id: 'run-1', + actId: 'actor-id-wcc', + status: 'RUNNING', + startedAt: new Date('2026-10-14T10:00:00.000Z'), + defaultDatasetId: 'dataset-id-1', + defaultKeyValueStoreId: 'kv-id-1', +}; + +function mockActorTool(actorFullName: string): ToolEntry { + return { + type: TOOL_TYPE.ACTOR, + name: actorFullName.replace('/', '--'), + actorId: 'actor-id', + actorFullName, + description: actorFullName, + inputSchema: { type: 'object', properties: {}, additionalProperties: true } as never, + ajvValidate: (() => true) as never, + } as unknown as ToolEntry; +} + +describe.each([ + ['call-actor', callActor, { waitSecs: 0 }], + ['call-actor-widget', callActorWidget, {}], +])('%s run input', (_name, tool, extraArgs) => { + const callWithInput = async (actorFullName: string, input: Record) => { + vi.mocked(getActorToolResolutionCached).mockResolvedValue({ toolMode: ACTOR_TOOL_MODE.RUN, actorFullName }); + vi.mocked(getActorsAsTools).mockResolvedValue({ tools: [mockActorTool(actorFullName)], errors: [] }); + const startSpy = vi.fn().mockResolvedValue(MOCK_RUN); + const client = mockApifyClient({ actor: () => ({ start: startSpy }) }); + await (tool as HelperTool).call(stubToolCallContext({ actor: actorFullName, input, ...extraArgs }, client)); + return startSpy.mock.calls[0]?.[0]; + }; + + beforeEach(() => { + vi.mocked(getActorToolResolutionCached).mockReset(); + vi.mocked(getActorsAsTools).mockReset(); + }); + + it.each([ + ['an omitted', true, {}], + ['an explicit false', false, { respectRobotsTxtFile: false }], + ])('sends %s website-content-crawler respectRobotsTxtFile as %s', async (_label, expected, field) => { + expect(await callWithInput(WEBSITE_CONTENT_CRAWLER, { startUrls: START_URLS, ...field })).toEqual({ + startUrls: START_URLS, + respectRobotsTxtFile: expected, + }); + }); + + it('sends respectRobotsTxtFile as true when website-content-crawler is named by its ID', async () => { + vi.mocked(getActorToolResolutionCached).mockResolvedValue({ + toolMode: ACTOR_TOOL_MODE.RUN, + actorFullName: WEBSITE_CONTENT_CRAWLER, + }); + vi.mocked(getActorsAsTools).mockResolvedValue({ tools: [mockActorTool(WEBSITE_CONTENT_CRAWLER)], errors: [] }); + const startSpy = vi.fn().mockResolvedValue(MOCK_RUN); + const client = mockApifyClient({ actor: () => ({ start: startSpy }) }); + + await (tool as HelperTool).call( + stubToolCallContext({ actor: 'aYG0l9s7dbB7j3gbS', input: { startUrls: START_URLS }, ...extraArgs }, client), + ); + + expect(startSpy.mock.calls[0]?.[0]).toEqual({ startUrls: START_URLS, respectRobotsTxtFile: true }); + }); + + it('leaves the input of other Actors unchanged', async () => { + expect(await callWithInput('apify/cheerio-scraper', { startUrls: START_URLS })).toEqual({ + startUrls: START_URLS, + }); + }); +}); diff --git a/tests/unit/utils.actor_details.test.ts b/tests/unit/utils.actor_details.test.ts index 61466add9..683d58353 100644 --- a/tests/unit/utils.actor_details.test.ts +++ b/tests/unit/utils.actor_details.test.ts @@ -181,6 +181,35 @@ describe('fetchActorDetails()', () => { await expect(fetchActorDetails(client, 'apify/instagram-scraper')).rejects.toMatchObject({ statusCode: 401 }); }); + + // Regression: #1574 — call-actor sends respectRobotsTxtFile true when omitted, so the schema it points to says so. + it.each([ + ['website-content-crawler', true], + ['cheerio-scraper', false], + ])('serves the respectRobotsTxtFile default of apify/%s as %s', async (name, expected) => { + const respectRobotsTxtFile = { + title: 'Respect robots.txt', + description: 'If enabled, the crawler consults robots.txt.', + type: 'boolean', + default: false, + prefill: true, + }; + const client = mockApifyClient({ + token: 'test-token', + actor: () => ({ + get: async () => ({ id: 'actor-id', username: 'apify', name, stats: {} }), + defaultBuild: async () => ({ + get: async () => ({ + actorDefinition: { input: { type: 'object', properties: { respectRobotsTxtFile } } }, + }), + }), + }), + }); + + const result = await fetchActorDetails(client, 'actor-id'); + + expect(result?.inputSchema.properties.respectRobotsTxtFile).toMatchObject({ default: expected }); + }); }); describe('getMcpToolsMessage()', () => {