From aaccf327b626304ac0391c25e37683afe4867346 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sun, 25 Oct 2026 22:31:09 +0000 Subject: [PATCH 01/11] Bind workflow service operations to trusted host authority --- CHANGELOG.md | 12 + .../workflow-service-calls-architecture.md | 48 ++ src/Providers/WorkflowServiceProvider.php | 6 + .../WorkflowServiceOperationAuthorizer.php | 26 + ...ServiceOperationAuthorizationException.php | 15 + src/V2/Support/DefaultServiceControlPlane.php | 17 +- src/V2/Support/DefaultWorkflowTaskBridge.php | 88 +++- ...DenyWorkflowServiceOperationAuthorizer.php | 18 + .../Support/RunServiceOperationAuthorizer.php | 23 + src/V2/Support/ServiceCallPrincipal.php | 4 +- src/V2/Support/WorkflowExecutor.php | 28 +- .../WorkflowServiceOperationAuthority.php | 138 +++++ tests/Support/NexusCallerDeliveryScenario.php | 37 ++ ...WorkflowTaskBridgeServiceOperationTest.php | 22 +- .../WorkflowExecutorServiceOperationTest.php | 10 + .../WorkflowServiceOperationAuthorityTest.php | 471 ++++++++++++++++++ 16 files changed, 913 insertions(+), 50 deletions(-) create mode 100644 src/V2/Contracts/WorkflowServiceOperationAuthorizer.php create mode 100644 src/V2/Exceptions/ServiceOperationAuthorizationException.php create mode 100644 src/V2/Support/DenyWorkflowServiceOperationAuthorizer.php create mode 100644 src/V2/Support/RunServiceOperationAuthorizer.php create mode 100644 src/V2/Support/WorkflowServiceOperationAuthority.php create mode 100644 tests/Unit/V2/WorkflowServiceOperationAuthorityTest.php diff --git a/CHANGELOG.md b/CHANGELOG.md index b8f7185f3..0e8792048 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## Unreleased + +- Require a trusted host authorizer for workflow service operations. Refuse + worker-supplied principal overrides, foreign caller namespaces and service + call IDs or idempotency keys owned by another run, instance or operation. + Preserve explicit embedded null namespaces instead of using ambient config. +- Service-operation hosts must bind `WorkflowServiceOperationAuthorizer` before + upgrading. The default refuses service operations; explicitly trusted embedded + hosts can opt into `RunServiceOperationAuthorizer` for same-namespace calls + with an unprivileged workflow principal. Authenticated servers supply their + own current principal and authorize requested target scope. + ## 2.13.1 - 2026-10-24 - Cancel the live task when a retried activity receives cooperative cancellation. diff --git a/docs/architecture/workflow-service-calls-architecture.md b/docs/architecture/workflow-service-calls-architecture.md index 9c94198b5..773daf32d 100644 --- a/docs/architecture/workflow-service-calls-architecture.md +++ b/docs/architecture/workflow-service-calls-architecture.md @@ -352,6 +352,54 @@ of the five reasons, and a Cancelled call MUST carry `ServiceCallFailureReason::Cancellation` even though its terminal status is Cancelled rather than Failed. +## Workflow operation authority + +Workflow service operations require a host binding for +`Workflow\V2\Contracts\WorkflowServiceOperationAuthorizer`. The default binding +refuses them with `service_operation_authority_invalid`. This applies to embedded +execution and external worker completions, including retained-claim prefixes. +Upgrade the host binding before accepting service-operation work. + +An authorizer receives the locked caller run and normalized command intent. +It must validate the requested target scope against trusted host/run context, +then return a `ServiceCallPrincipal` derived from that context. It must not +reconstruct authority from the command's principal fields or metadata. It is an +authorization check, so it must not reserve admission budgets or perform +external effects; a transaction retry or preflight may invoke it again. + +An embedded host that explicitly trusts its own workflow code to invoke +services within each run's namespace can opt into the provided workload policy: + +```php +$app->bind( + \Workflow\V2\Contracts\WorkflowServiceOperationAuthorizer::class, + \Workflow\V2\Support\RunServiceOperationAuthorizer::class, +); +``` + +That policy permits only the run's namespace. Its principal is +`workflow:`, method `workflow`, with no roles, tenant or claims. It does +not satisfy a catalog policy that requires privileged roles. A server with +authenticated workers instead binds an authorizer that checks its current +authentication/namespace policy and returns the authenticated worker principal. +Authorized cross-namespace routing remains possible through that trusted host +binding; a worker's requested `namespace` alone grants no permission. + +Caller namespace and workflow run/instance identity always come from the locked +run. A mismatched non-null `caller_namespace` or non-null `principal_*` override +is refused before applying sibling commands. Unknown fields do not grant +authority. Explicit null run namespaces stay null even when the process has a +configured default namespace. + +An explicit `service_call_id` or an existing idempotency-key match must bind to +the same caller namespace, run and instance, the authorized target namespace, +and the exact endpoint/service/operation names. Existing rows are checked under +locks before replay or dispatch. A missing or foreign explicit call, or a +foreign key match, refuses the worker turn with `service_call_not_owned` and +leaves the lease and durable rows unchanged. New default idempotency keys are +checked by the same rule. Server adapters retain their current-policy replay +checks before handing the call to Core. + ## Observability surface The observability surface for a cross-namespace service call MUST be diff --git a/src/Providers/WorkflowServiceProvider.php b/src/Providers/WorkflowServiceProvider.php index d20f4a748..9efa531fb 100644 --- a/src/Providers/WorkflowServiceProvider.php +++ b/src/Providers/WorkflowServiceProvider.php @@ -43,6 +43,7 @@ use Workflow\V2\Contracts\ServiceBoundaryPolicy; use Workflow\V2\Contracts\ServiceControlPlane; use Workflow\V2\Contracts\WorkflowControlPlane; +use Workflow\V2\Contracts\WorkflowServiceOperationAuthorizer; use Workflow\V2\Contracts\WorkflowTaskBridge; use Workflow\V2\Models\WorkflowHistoryEvent; use Workflow\V2\Models\WorkflowLink; @@ -63,6 +64,7 @@ use Workflow\V2\Support\DefaultServiceControlPlane; use Workflow\V2\Support\DefaultWorkflowControlPlane; use Workflow\V2\Support\DefaultWorkflowTaskBridge; +use Workflow\V2\Support\DenyWorkflowServiceOperationAuthorizer; use Workflow\V2\Support\ExecutionTracing; use Workflow\V2\Support\HistoryProjectionMaintenanceFallback; use Workflow\V2\Support\InMemoryTaskFairnessState; @@ -132,6 +134,10 @@ static function ($app): HistoryProjectionMaintenanceRole { ); $this->app->singletonIf(ServiceControlPlane::class, DefaultServiceControlPlane::class); + $this->app->singletonIf( + WorkflowServiceOperationAuthorizer::class, + DenyWorkflowServiceOperationAuthorizer::class + ); $this->app->singletonIf(SchedulerRole::class, DefaultSchedulerRole::class); diff --git a/src/V2/Contracts/WorkflowServiceOperationAuthorizer.php b/src/V2/Contracts/WorkflowServiceOperationAuthorizer.php new file mode 100644 index 000000000..4219b7c66 --- /dev/null +++ b/src/V2/Contracts/WorkflowServiceOperationAuthorizer.php @@ -0,0 +1,26 @@ + $command + */ + public function authorize(WorkflowRun $run, array $command): ServiceCallPrincipal; +} diff --git a/src/V2/Exceptions/ServiceOperationAuthorizationException.php b/src/V2/Exceptions/ServiceOperationAuthorizationException.php new file mode 100644 index 000000000..63b6c8462 --- /dev/null +++ b/src/V2/Exceptions/ServiceOperationAuthorizationException.php @@ -0,0 +1,15 @@ +newCall(); $principal = $this->principal($options); - $callerNamespace = $this->namespaceOption($options, 'caller_namespace') - ?? ($call instanceof WorkflowServiceCall ? $this->stringFrom($call->caller_namespace) : null) - ?? $this->configuredNamespace(); + $callerNamespace = array_key_exists('caller_namespace', $options) + ? $this->namespaceOption($options, 'caller_namespace') + : (($call instanceof WorkflowServiceCall ? $this->stringFrom($call->caller_namespace) : null) + ?? $this->configuredNamespace()); if ($call->accepted_at === null) { $this->stampPending( @@ -467,9 +468,13 @@ private function emptyCallShape(): array */ private function targetNamespace(array $options): ?string { - return $this->namespaceOption($options, 'target_namespace') - ?? $this->namespaceOption($options, 'namespace') - ?? $this->configuredNamespace(); + foreach (['target_namespace', 'namespace'] as $field) { + if (array_key_exists($field, $options)) { + return $this->namespaceOption($options, $field); + } + } + + return $this->configuredNamespace(); } /** diff --git a/src/V2/Support/DefaultWorkflowTaskBridge.php b/src/V2/Support/DefaultWorkflowTaskBridge.php index 69e3e6594..45e1e7b6b 100644 --- a/src/V2/Support/DefaultWorkflowTaskBridge.php +++ b/src/V2/Support/DefaultWorkflowTaskBridge.php @@ -39,6 +39,7 @@ use Workflow\V2\Enums\UpdateStatus; use Workflow\V2\Exceptions\ActivityTimeoutException; use Workflow\V2\Exceptions\HistoryEventShapeMismatchException; +use Workflow\V2\Exceptions\ServiceOperationAuthorizationException; use Workflow\V2\Models\ActivityAttempt; use Workflow\V2\Models\ActivityExecution; use Workflow\V2\Models\WorkflowChildCall; @@ -1234,7 +1235,10 @@ public function complete(string $taskId, array $commands): array ]; } - return DB::transaction(function () use ($taskId, $parsed): array { + return self::serviceOperationTransaction(fn (): array => DB::transaction(function () use ( + $taskId, + $parsed + ): array { /** @var WorkflowTask|null $task */ $task = ConfiguredV2Models::query('task_model', WorkflowTask::class) ->lockForUpdate() @@ -1334,6 +1338,17 @@ public function complete(string $taskId, array $commands): array $sequence = WorkflowStepHistory::nextDurableCommandSequence($run); $createdTaskIds = []; + $serviceError = $this->serviceOperationAuthorityError($run, $parsed['non_terminal'], $sequence); + if ($serviceError !== null) { + return [ + 'completed' => false, + 'task_id' => $taskId, + 'workflow_run_id' => $run->id, + 'run_status' => $run->status->value, + 'created_task_ids' => [], + 'reason' => $serviceError, + ]; + } $scopeError = $this->validateCancellationScopeMembership($run, $parsed['non_terminal'], $sequence); if ($scopeError !== null) { return [ @@ -1427,7 +1442,14 @@ public function complete(string $taskId, array $commands): array 'created_task_ids' => $createdTaskIds, 'reason' => null, ]; - }); + }), static fn (string $reason): array => [ + 'completed' => false, + 'task_id' => $taskId, + 'workflow_run_id' => null, + 'run_status' => null, + 'created_task_ids' => [], + 'reason' => $reason, + ]); } /** @@ -1534,7 +1556,7 @@ private function checkpointCommands( return $refused('task_not_found'); } - return DB::transaction(function () use ( + return self::serviceOperationTransaction(fn (): array => DB::transaction(function () use ( $snapshot, $taskId, $leaseOwner, @@ -1571,6 +1593,10 @@ private function checkpointCommands( || ($run->run_deadline_at !== null && now()->gte($run->run_deadline_at))) { return $refused('run_deadline_expired'); } + $serviceError = $this->serviceOperationAuthorityError($run, $parsed['non_terminal'], $startSequence); + if ($serviceError !== null) { + return $refused($serviceError); + } $payload = is_array($task->payload) ? $task->payload : []; $receiptKey = $scopePrefix ? 'portable_scope_checkpoint' : ($group ? 'portable_local_group_checkpoint' : 'portable_local_checkpoint'); @@ -1732,7 +1758,7 @@ private function checkpointCommands( 'duplicate' => false, 'reason' => null, ]; - }, 5); + }, 5), $refused); } /** @@ -3522,17 +3548,13 @@ private function applyStartServiceOperation( $payloadCodec = is_string($command['payload_codec'] ?? null) && $command['payload_codec'] !== '' ? $command['payload_codec'] : ($run->payload_codec ?? CodecRegistry::defaultCodec()); + $options = $this->serviceOperationControlPlaneOptions($run, $command, $sequence, $payloadCodec); $namespace = is_string($run->namespace) ? $run->namespace : null; $requestPayload = isset($command['request_payload']) && is_string($command['request_payload']) ? ExternalPayloads::externalizeForNamespace($command['request_payload'], $payloadCodec, $namespace) : null; $surface = $this->serviceControlPlane() - ->execute( - $command['endpoint_name'], - $command['service_name'], - $command['operation_name'], - $this->serviceOperationControlPlaneOptions($run, $command, $sequence, $payloadCodec), - ); + ->execute($command['endpoint_name'], $command['service_name'], $command['operation_name'], $options); $eventType = self::serviceOperationEventTypeForSurface($surface); $event = WorkflowHistoryEvent::record($run, $eventType, $this->serviceOperationEventPayload( @@ -3580,6 +3602,7 @@ private function serviceOperationControlPlaneOptions( int $sequence, string $payloadCodec, ): array { + $command['idempotency_key'] ??= $this->defaultServiceOperationIdempotencyKey($run, $sequence); $metadata = is_array($command['metadata'] ?? null) ? $command['metadata'] : []; $metadata = [ 'caller_sdk_language' => self::metadataString($metadata, 'caller_sdk_language') ?? 'workflow-php', @@ -3588,8 +3611,7 @@ private function serviceOperationControlPlaneOptions( 'workflow_sequence' => $sequence, ] + $metadata; - return array_filter([ - 'namespace' => self::nonEmptyString($command['namespace'] ?? null) ?? $run->namespace, + return WorkflowServiceOperationAuthority::options($run, $command) + array_filter([ 'arguments' => $this->serviceOperationRequestArguments($command, $payloadCodec), 'payload_blob' => self::nonEmptyString($command['request_payload'] ?? null), 'payload_codec' => $payloadCodec, @@ -3601,7 +3623,6 @@ private function serviceOperationControlPlaneOptions( 'wait_timeout_seconds' => is_int($command['wait_timeout_seconds'] ?? null) ? $command['wait_timeout_seconds'] : null, - 'caller_namespace' => self::nonEmptyString($command['caller_namespace'] ?? null) ?? $run->namespace, 'caller_workflow_instance_id' => $run->workflow_instance_id, 'caller_workflow_run_id' => $run->id, 'target_workflow_instance_id' => self::nonEmptyString($command['target_workflow_instance_id'] ?? null), @@ -3617,14 +3638,45 @@ private function serviceOperationControlPlaneOptions( 'duplicate_start_policy' => self::nonEmptyString($command['duplicate_start_policy'] ?? null), 'metadata' => $metadata, 'request_payload_reference' => self::nonEmptyString($command['request_payload_reference'] ?? null), - 'principal_subject' => self::nonEmptyString($command['principal_subject'] ?? null), - 'principal_method' => self::nonEmptyString($command['principal_method'] ?? null), - 'principal_roles' => is_array($command['principal_roles'] ?? null) ? $command['principal_roles'] : null, - 'principal_tenant' => self::nonEmptyString($command['principal_tenant'] ?? null), - 'principal_claims' => is_array($command['principal_claims'] ?? null) ? $command['principal_claims'] : null, ], static fn (mixed $value): bool => $value !== null); } + /** + * @param list $commands + */ + private function serviceOperationAuthorityError(WorkflowRun $run, array $commands, int $sequence): ?string + { + try { + foreach ($commands as $command) { + if ($command['type'] === 'start_service_operation') { + $command['idempotency_key'] ??= $this->defaultServiceOperationIdempotencyKey($run, $sequence); + WorkflowServiceOperationAuthority::options($run, $command); + } + if ($command['type'] !== 'cancel_selection_operation') { + ++$sequence; + } + } + } catch (ServiceOperationAuthorizationException $exception) { + return $exception->getMessage(); + } + + return null; + } + + /** + * @param callable(): array $transaction + * @param callable(string): array $refused + * @return array + */ + private static function serviceOperationTransaction(callable $transaction, callable $refused): array + { + try { + return $transaction(); + } catch (ServiceOperationAuthorizationException $exception) { + return $refused($exception->getMessage()); + } + } + /** * @param array $command */ diff --git a/src/V2/Support/DenyWorkflowServiceOperationAuthorizer.php b/src/V2/Support/DenyWorkflowServiceOperationAuthorizer.php new file mode 100644 index 000000000..b69438204 --- /dev/null +++ b/src/V2/Support/DenyWorkflowServiceOperationAuthorizer.php @@ -0,0 +1,18 @@ +namespace; + if ($target !== $run->namespace) { + throw new ServiceOperationAuthorizationException(); + } + + return new ServiceCallPrincipal(subject: 'workflow:' . $run->id, method: 'workflow'); + } +} diff --git a/src/V2/Support/ServiceCallPrincipal.php b/src/V2/Support/ServiceCallPrincipal.php index 4e2ef5e6c..9987006ed 100644 --- a/src/V2/Support/ServiceCallPrincipal.php +++ b/src/V2/Support/ServiceCallPrincipal.php @@ -11,8 +11,8 @@ * the audit row so accepted *and* rejected calls share the same * audit shape. This object is intentionally narrow — only the fields * that survive into the durable audit row — so it can be reconstructed - * cleanly from server-side `Principal`, from worker-supplied identity - * headers, or from a synthetic "system" caller. + * cleanly from a server-authenticated `Principal` or an explicitly + * authorized embedded workload. Worker command fields are not identity. * * Privacy boundary: this DTO carries identity fields (subject, method, * roles, tenant, optional opaque claims). Payload material — request diff --git a/src/V2/Support/WorkflowExecutor.php b/src/V2/Support/WorkflowExecutor.php index 5a5ab5882..4490c2797 100644 --- a/src/V2/Support/WorkflowExecutor.php +++ b/src/V2/Support/WorkflowExecutor.php @@ -5969,14 +5969,16 @@ private function recordServiceOperationEvent( : CodecRegistry::defaultCodec()); $namespace = is_string($run->namespace) ? $run->namespace : null; $serializedRequest = Serializer::serializeWithCodec($payloadCodec, $call->requestPayload); + $options = $this->serviceOperationControlPlaneOptions( + $run, + $sequence, + $call, + $payloadCodec, + $serializedRequest + ); $storedRequest = ExternalPayloads::externalizeForNamespace($serializedRequest, $payloadCodec, $namespace); $surface = $this->serviceControlPlane() - ->execute( - $call->endpointName, - $call->serviceName, - $call->operationName, - $this->serviceOperationControlPlaneOptions($run, $sequence, $call, $payloadCodec, $serializedRequest), - ); + ->execute($call->endpointName, $call->serviceName, $call->operationName, $options); $eventType = self::serviceOperationEventTypeForSurface($surface); $event = WorkflowHistoryEvent::record( @@ -6001,6 +6003,7 @@ private function serviceOperationControlPlaneOptions( string $serializedRequest, ): array { $options = $call->options?->toCommandOptions() ?? []; + $options['idempotency_key'] ??= $this->defaultServiceOperationIdempotencyKey($run, $sequence); $metadata = is_array($options['metadata'] ?? null) ? $options['metadata'] : []; $metadata = [ 'caller_sdk_language' => 'workflow-php', @@ -6009,8 +6012,11 @@ private function serviceOperationControlPlaneOptions( 'workflow_sequence' => $sequence, ] + $metadata; - return array_filter([ - 'namespace' => self::nonEmptyString($options['namespace'] ?? null) ?? $run->namespace, + return WorkflowServiceOperationAuthority::options($run, $options + [ + 'endpoint_name' => $call->endpointName, + 'service_name' => $call->serviceName, + 'operation_name' => $call->operationName, + ]) + array_filter([ 'arguments' => $call->requestPayload, 'payload_blob' => $serializedRequest, 'payload_codec' => $payloadCodec, @@ -6022,7 +6028,6 @@ private function serviceOperationControlPlaneOptions( 'wait_timeout_seconds' => is_int($options['wait_timeout_seconds'] ?? null) ? $options['wait_timeout_seconds'] : null, - 'caller_namespace' => self::nonEmptyString($options['caller_namespace'] ?? null) ?? $run->namespace, 'caller_workflow_instance_id' => $run->workflow_instance_id, 'caller_workflow_run_id' => $run->id, 'target_workflow_instance_id' => self::nonEmptyString($options['target_workflow_instance_id'] ?? null), @@ -6038,11 +6043,6 @@ private function serviceOperationControlPlaneOptions( 'duplicate_start_policy' => self::nonEmptyString($options['duplicate_start_policy'] ?? null), 'metadata' => $metadata, 'request_payload_reference' => self::nonEmptyString($options['request_payload_reference'] ?? null), - 'principal_subject' => self::nonEmptyString($options['principal_subject'] ?? null), - 'principal_method' => self::nonEmptyString($options['principal_method'] ?? null), - 'principal_roles' => is_array($options['principal_roles'] ?? null) ? $options['principal_roles'] : null, - 'principal_tenant' => self::nonEmptyString($options['principal_tenant'] ?? null), - 'principal_claims' => is_array($options['principal_claims'] ?? null) ? $options['principal_claims'] : null, ], static fn (mixed $value): bool => $value !== null); } diff --git a/src/V2/Support/WorkflowServiceOperationAuthority.php b/src/V2/Support/WorkflowServiceOperationAuthority.php new file mode 100644 index 000000000..715b2db50 --- /dev/null +++ b/src/V2/Support/WorkflowServiceOperationAuthority.php @@ -0,0 +1,138 @@ + $command + * @return array + */ + public static function options(WorkflowRun $run, array $command): array + { + foreach (['principal', 'principal_subject', 'principal_method', 'principal_roles', + 'principal_tenant', 'principal_claims'] as $field) { + if (array_key_exists($field, $command)) { + throw new ServiceOperationAuthorizationException(); + } + } + if (isset($command['caller_namespace']) && $command['caller_namespace'] !== $run->namespace) { + throw new ServiceOperationAuthorizationException(); + } + + $principal = app(WorkflowServiceOperationAuthorizer::class)->authorize($run, $command); + $namespace = $command['namespace'] ?? $run->namespace; + self::checkReference($run, $command, $namespace); + + // Preserve explicit null namespaces: null is the embedded namespace, + // not permission to use ambient configuration or any existing call. + return [ + 'namespace' => $namespace, + 'caller_namespace' => $run->namespace, + 'caller_workflow_instance_id' => $run->workflow_instance_id, + 'caller_workflow_run_id' => $run->id, + 'principal' => $principal, + ]; + } + + /** + * @param array $command + */ + private static function checkReference(WorkflowRun $run, array $command, ?string $namespace): void + { + $callId = $command['service_call_id'] ?? null; + $key = $command['idempotency_key'] ?? null; + if ($callId === null && $key === null) { + return; + } + + $query = ConfiguredV2Models::query('service_call_model', WorkflowServiceCall::class); + if ($callId !== null) { + $query->whereKey($callId); + } else { + $query->where('target_namespace', $namespace) + ->where('endpoint_name', strtolower(trim($command['endpoint_name']))) + ->where('service_name', strtolower(trim($command['service_name']))) + ->where('operation_name', strtolower(trim($command['operation_name']))) + ->where('idempotency_key', $key) + ->oldest('created_at') + ->oldest('id'); + } + /** @var WorkflowServiceCall|null $call */ + $call = $query->lockForUpdate() + ->first(); + if ($call === null && $callId === null) { + return; + } + if ($call === null || $call->namespace !== $namespace || $call->target_namespace !== $namespace + || $call->caller_namespace !== $run->namespace + || $call->caller_workflow_instance_id !== $run->workflow_instance_id + || $call->caller_workflow_run_id !== $run->id + || $call->endpoint_name !== strtolower(trim($command['endpoint_name'])) + || $call->service_name !== strtolower(trim($command['service_name'])) + || $call->operation_name !== strtolower(trim($command['operation_name']))) { + throw new ServiceOperationAuthorizationException('service_call_not_owned'); + } + self::checkCatalog($call, $namespace); + } + + private static function checkCatalog(WorkflowServiceCall $call, ?string $namespace): void + { + $resolutionFailure = $call->status === ServiceCallStatus::Failed->value + && $call->outcome === ServiceCallOutcome::RejectedNotFound + && ($call->outcome_metadata['failure_reason'] ?? null) === 'resolution_failure'; + foreach ([ + [ + 'service_endpoint_model', WorkflowServiceEndpoint::class, $call->workflow_service_endpoint_id, + [ + 'endpoint_name' => $call->endpoint_name, + ]], + [ + 'service_model', WorkflowService::class, $call->workflow_service_id, + [ + 'workflow_service_endpoint_id' => $call->workflow_service_endpoint_id, + 'service_name' => $call->service_name, + ]], + [ + 'service_operation_model', WorkflowServiceOperation::class, $call->workflow_service_operation_id, + [ + 'workflow_service_endpoint_id' => $call->workflow_service_endpoint_id, + 'workflow_service_id' => $call->workflow_service_id, + 'operation_name' => $call->operation_name, + ]], + ] as [$role, $model, $id, $attributes]) { + // A retained resolution failure may have no catalog component. + // Known components still have to be this call's scoped objects. + if ($id === 'unresolved' && $resolutionFailure) { + continue; + } + $row = ConfiguredV2Models::query($role, $model)->where('namespace', $namespace) + ->lockForUpdate() + ->find($id); + if ($row === null) { + throw new ServiceOperationAuthorizationException('service_call_not_owned'); + } + foreach ([ + 'id' => $id, + 'namespace' => $namespace, + ] + $attributes as $field => $expected) { + if ($row->getAttribute($field) !== $expected) { + throw new ServiceOperationAuthorizationException('service_call_not_owned'); + } + } + } + } +} diff --git a/tests/Support/NexusCallerDeliveryScenario.php b/tests/Support/NexusCallerDeliveryScenario.php index 5aea820a1..d8278dc34 100644 --- a/tests/Support/NexusCallerDeliveryScenario.php +++ b/tests/Support/NexusCallerDeliveryScenario.php @@ -10,10 +10,12 @@ use Workflow\Serializers\Serializer; use Workflow\V2\Contracts\ServiceControlPlane; use Workflow\V2\Contracts\WorkflowControlPlane; +use Workflow\V2\Contracts\WorkflowServiceOperationAuthorizer; use Workflow\V2\Contracts\WorkflowTaskBridge; use Workflow\V2\Enums\HistoryEventType; use Workflow\V2\Enums\RunStatus; use Workflow\V2\Enums\TaskStatus; +use Workflow\V2\Exceptions\ServiceOperationAuthorizationException; use Workflow\V2\Jobs\RunWorkflowTask; use Workflow\V2\Models\WorkflowHistoryEvent; use Workflow\V2\Models\WorkflowInstance; @@ -24,6 +26,7 @@ use Workflow\V2\Models\WorkflowServiceOperation; use Workflow\V2\Models\WorkflowTask; use Workflow\V2\Support\DefaultServiceControlPlane; +use Workflow\V2\Support\ServiceCallPrincipal; use Workflow\V2\Support\WorkflowFiberRunner; use Workflow\V2\TaskWatchdog; use Workflow\V2\Workflow; @@ -37,6 +40,7 @@ trait NexusCallerDeliveryScenario protected function assertNexusCallerSdkAttribution(array $fixture): void { $previous = $this->app->make(ServiceControlPlane::class); + $previousAuthority = $this->installNexusServiceAuthority(); $previousDispatchMode = config('workflows.v2.task_dispatch_mode'); config([ 'workflows.v2.task_dispatch_mode' => 'poll', @@ -96,6 +100,9 @@ protected function assertNexusCallerSdkAttribution(array $fixture): void 'workflows.v2.task_dispatch_mode' => $previousDispatchMode, ]); $this->app->instance(ServiceControlPlane::class, $previous); + if ($previousAuthority !== null) { + $this->app->instance(WorkflowServiceOperationAuthorizer::class, $previousAuthority); + } } } @@ -113,6 +120,7 @@ protected function assertWaitingNexusCallerReceivesResult( ): void { $previousDispatchMode = config('workflows.v2.task_dispatch_mode'); $previousControlPlane = $this->app->make(ServiceControlPlane::class); + $previousAuthority = $this->installNexusServiceAuthority(); $previousNow = Carbon::getTestNow(); config([ 'workflows.v2.task_dispatch_mode' => 'poll', @@ -243,6 +251,35 @@ protected function assertWaitingNexusCallerReceivesResult( 'workflows.v2.task_dispatch_mode' => $previousDispatchMode, ]); $this->app->instance(ServiceControlPlane::class, $previousControlPlane); + if ($previousAuthority !== null) { + $this->app->instance(WorkflowServiceOperationAuthorizer::class, $previousAuthority); + } } } + + private function installNexusServiceAuthority(): ?object + { + // The retained consumer also runs against the pre-authorizer base. + if (! interface_exists(WorkflowServiceOperationAuthorizer::class)) { + return null; + } + $previous = $this->app->make(WorkflowServiceOperationAuthorizer::class); + $this->app->instance( + WorkflowServiceOperationAuthorizer::class, + new class() implements WorkflowServiceOperationAuthorizer { + public function authorize(WorkflowRun $run, array $command): ServiceCallPrincipal + { + $target = $command['namespace'] ?? $run->namespace; + $operation = [$command['endpoint_name'], $command['service_name'], $command['operation_name']]; + $allowed = ($target === 'caller' && $operation === ['billing', 'invoices', 'create']) + || ($target === 'service' && $operation === ['php-service', 'greeter', 'greet']); + if ($run->namespace !== 'caller' || ! $allowed) { + throw new ServiceOperationAuthorizationException(); + } + return new ServiceCallPrincipal(subject: 'workflow:' . $run->id, method: 'workflow'); + } + } + ); + return $previous; + } } diff --git a/tests/Unit/V2/DefaultWorkflowTaskBridgeServiceOperationTest.php b/tests/Unit/V2/DefaultWorkflowTaskBridgeServiceOperationTest.php index 7dc2e92bf..d5173165a 100644 --- a/tests/Unit/V2/DefaultWorkflowTaskBridgeServiceOperationTest.php +++ b/tests/Unit/V2/DefaultWorkflowTaskBridgeServiceOperationTest.php @@ -13,6 +13,7 @@ use Workflow\Serializers\Serializer; use Workflow\V2\Attributes\Type; use Workflow\V2\Contracts\ServiceControlPlane; +use Workflow\V2\Contracts\WorkflowServiceOperationAuthorizer; use Workflow\V2\Contracts\WorkflowTaskBridge; use Workflow\V2\Enums\HistoryEventType; use Workflow\V2\Enums\RunStatus; @@ -23,6 +24,8 @@ use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowRunSummary; use Workflow\V2\Models\WorkflowTask; +use Workflow\V2\Support\RunServiceOperationAuthorizer; +use Workflow\V2\Support\ServiceCallPrincipal; use Workflow\V2\Support\WorkerCompatibilityFleet; use Workflow\V2\Workflow; use Workflow\V2\WorkflowStub; @@ -38,6 +41,7 @@ final class DefaultWorkflowTaskBridgeServiceOperationTest extends TestCase protected function setUp(): void { parent::setUp(); + $this->app->instance(WorkflowServiceOperationAuthorizer::class, new RunServiceOperationAuthorizer()); Carbon::setTestNow('2026-10-08T02:00:00Z'); config([ 'workflows.v2.task_dispatch_mode' => 'poll', @@ -304,9 +308,8 @@ public function testServiceOptionsPreserveTypedValuesAndAuthoritativeCallerMetad ]; $payload = Serializer::serialize($arguments); $options = [ - 'namespace' => 'billing', - 'caller_namespace' => 'finance', - 'service_call_id' => 'requested-service-call', + 'namespace' => 'caller', + 'caller_namespace' => 'caller', 'idempotency_key' => 'invoice-request-42', 'mode_override' => ' SYNC ', 'wait_for' => ' COMPLETED ', @@ -327,13 +330,6 @@ public function testServiceOptionsPreserveTypedValuesAndAuthoritativeCallerMetad ], 'duplicate_start_policy' => 'return_existing_active', 'request_payload_reference' => 'invoice-request-reference', - 'principal_subject' => 'user:billing-clerk', - 'principal_method' => 'token', - 'principal_roles' => ['invoice:create', 'invoice:read'], - 'principal_tenant' => 'customer-a', - 'principal_claims' => [ - 'approved' => false, - ], 'metadata' => [ 'caller_sdk_language' => 'sdk-python', 'caller_workflow_instance_id' => 'forged-owner', @@ -368,7 +364,13 @@ public function testServiceOptionsPreserveTypedValuesAndAuthoritativeCallerMetad 'service_sdk_language' => 'rust', 'trace_id' => 'trace-42', ]; + $this->assertInstanceOf(ServiceCallPrincipal::class, $forwarded['principal']); + $forwarded['principal'] = $forwarded['principal']->toAuditArray(); $expected += [ + 'principal' => (new ServiceCallPrincipal( + subject: 'workflow:' . $this->run->id, + method: 'workflow' + ))->toAuditArray(), 'arguments' => $arguments, 'payload_blob' => $payload, 'payload_codec' => 'avro', diff --git a/tests/Unit/V2/WorkflowExecutorServiceOperationTest.php b/tests/Unit/V2/WorkflowExecutorServiceOperationTest.php index 843b1f784..f4b03591f 100644 --- a/tests/Unit/V2/WorkflowExecutorServiceOperationTest.php +++ b/tests/Unit/V2/WorkflowExecutorServiceOperationTest.php @@ -15,6 +15,7 @@ use Workflow\Serializers\Serializer; use Workflow\V2\Attributes\Type; use Workflow\V2\Contracts\ServiceControlPlane; +use Workflow\V2\Contracts\WorkflowServiceOperationAuthorizer; use Workflow\V2\Enums\HistoryEventType; use Workflow\V2\Enums\TaskStatus; use Workflow\V2\Exceptions\RestoredWorkflowException; @@ -26,7 +27,9 @@ use Workflow\V2\Models\WorkflowRunSummary; use Workflow\V2\Models\WorkflowTask; use Workflow\V2\Support\DefaultWorkflowTaskBridge; +use Workflow\V2\Support\RunServiceOperationAuthorizer; use Workflow\V2\Support\RunTimelineProjector; +use Workflow\V2\Support\ServiceCallPrincipal; use Workflow\V2\Support\ServiceOperationOptions; use Workflow\V2\Support\ServiceOperationResult; use Workflow\V2\Support\WorkerCompatibilityFleet; @@ -39,6 +42,7 @@ final class WorkflowExecutorServiceOperationTest extends TestCase protected function setUp(): void { parent::setUp(); + $this->app->instance(WorkflowServiceOperationAuthorizer::class, new RunServiceOperationAuthorizer()); Carbon::setTestNow('2026-10-08 17:10:00'); Queue::fake(); config() @@ -107,6 +111,10 @@ public function testPublicExecutionRecordsTheBoundaryOutcomeAndReplaysWithoutCal $this->assertCount(1, $service->executions); $request = Serializer::serializeWithCodec('avro', ExecutorServiceOperationWorkflow::REQUEST); $expectedOptions = [ + 'principal' => (new ServiceCallPrincipal( + subject: 'workflow:' . $run->id, + method: 'workflow' + ))->toAuditArray(), 'namespace' => 'service-caller', 'arguments' => ExecutorServiceOperationWorkflow::REQUEST, 'payload_blob' => $request, @@ -128,6 +136,8 @@ public function testPublicExecutionRecordsTheBoundaryOutcomeAndReplaysWithoutCal 'wait_for' => 'accepted', ]; } + $this->assertInstanceOf(ServiceCallPrincipal::class, $service->executions[0][3]['principal']); + $service->executions[0][3]['principal'] = $service->executions[0][3]['principal']->toAuditArray(); $this->assertSame( $this->ordered(['billing', 'invoices', 'create', $expectedOptions]), $this->ordered($service->executions[0]) diff --git a/tests/Unit/V2/WorkflowServiceOperationAuthorityTest.php b/tests/Unit/V2/WorkflowServiceOperationAuthorityTest.php new file mode 100644 index 000000000..af47acd99 --- /dev/null +++ b/tests/Unit/V2/WorkflowServiceOperationAuthorityTest.php @@ -0,0 +1,471 @@ +app->instance(WorkflowServiceOperationAuthorizer::class, new RunServiceOperationAuthorizer()); + Carbon::setTestNow('2026-10-25T21:00:00Z'); + config([ + 'workflows.v2.task_dispatch_mode' => 'poll', + 'workflows.v2.compatibility.current' => 'authority-test', + 'workflows.v2.compatibility.supported' => ['authority-test'], + 'workflows.v2.namespace' => null, + ]); + WorkerCompatibilityFleet::clear(); + Queue::fake(); + $workflow = WorkflowStub::make(ServiceAuthorityWorkflow::class, 'authority-owner', 'caller'); + $this->assertTrue($workflow->attemptStart()->accepted()); + $this->run = WorkflowRun::query()->findOrFail($workflow->runId()); + $this->task = WorkflowTask::query()->where('workflow_run_id', $this->run->id)->sole(); + $this->bridge = $this->app->make(WorkflowTaskBridge::class); + $this->assertTrue($this->bridge->claimStatus($this->task->id, 'authority-worker')['claimed']); + } + + protected function tearDown(): void + { + Carbon::setTestNow(); + WorkerCompatibilityFleet::clear(); + parent::tearDown(); + } + + /** + * @param array $options + */ + #[DataProvider('forgedAuthority')] + public function testWorkerAuthorityOverridesRefuseTheWholeCompletion(array $options): void + { + $before = $this->records(); + $result = $this->bridge->complete($this->task->id, [ + [ + 'type' => 'start_timer', + 'delay_seconds' => 60, + ], + $this->command() + $options, + ]); + + $this->assertFalse($result['completed'], json_encode($result, JSON_THROW_ON_ERROR)); + $this->assertSame('service_operation_authority_invalid', $result['reason']); + $this->assertSame($before, $this->records()); + } + + /** + * @return iterable}> + */ + public static function forgedAuthority(): iterable + { + yield 'foreign namespace' => [[ + 'namespace' => 'foreign', + ]]; + yield 'foreign caller' => [[ + 'caller_namespace' => 'foreign', + ]]; + yield 'principal subject' => [[ + 'principal_subject' => 'administrator', + ]]; + yield 'principal method' => [[ + 'principal_method' => 'system', + ]]; + yield 'principal roles' => [[ + 'principal_roles' => ['admin'], + ]]; + yield 'principal tenant' => [[ + 'principal_tenant' => 'foreign', + ]]; + yield 'principal claims' => [[ + 'principal_claims' => [ + 'administrator' => true, + ], + ]]; + } + + #[DataProvider('foreignCalls')] + public function testForeignCallReferencesRefuseBeforeAnyCommandWrites( + string $status, + string $mismatch, + bool $idempotency, + ): void { + $call = $this->serviceCall($status); + $call->forceFill(match ($mismatch) { + 'run' => [ + 'caller_workflow_run_id' => 'another-run', + ], + 'instance' => [ + 'caller_workflow_instance_id' => 'another-owner', + ], + 'caller' => [ + 'caller_namespace' => 'another-caller', + ], + 'catalog' => [ + 'operation_name' => 'another-operation', + ], + 'namespace' => [ + 'namespace' => 'foreign', + 'target_namespace' => 'foreign', + ], + 'endpoint id' => [ + 'workflow_service_endpoint_id' => 'foreign-endpoint', + ], + 'service id' => [ + 'workflow_service_id' => 'foreign-service', + ], + 'operation id' => [ + 'workflow_service_operation_id' => 'foreign-operation', + ], + })->save(); + $options = $idempotency ? [ + 'idempotency_key' => $call->idempotency_key, + ] + : [ + 'service_call_id' => $call->id, + ]; + $before = $this->records(); + $result = $this->bridge->complete($this->task->id, [ + [ + 'type' => 'start_timer', + 'delay_seconds' => 60, + ], + $this->command() + $options, + ]); + + $this->assertFalse($result['completed'], json_encode($result, JSON_THROW_ON_ERROR)); + $this->assertSame('service_call_not_owned', $result['reason']); + $this->assertSame($before, $this->records()); + } + + /** + * @return iterable + */ + public static function foreignCalls(): iterable + { + foreach (['pending', 'started', 'completed'] as $status) { + foreach ([ + 'run', + 'instance', + 'caller', + 'catalog', + 'namespace', + 'endpoint id', + 'service id', + 'operation id', + ] as $mismatch) { + yield $status . ' ' . $mismatch => [$status, $mismatch, false]; + } + foreach (['run', 'instance', 'caller'] as $mismatch) { + yield $status . ' idempotency ' . $mismatch => [$status, $mismatch, true]; + } + } + } + + #[DataProvider('ownedCalls')] + public function testOwnedCallReplayRemainsUsable(string $status, bool $idempotency): void + { + $call = $this->serviceCall($status); + $result = $this->bridge->complete($this->task->id, [ + $this->command() + ($idempotency + ? [ + 'idempotency_key' => $call->idempotency_key, + ] : [ + 'service_call_id' => $call->id, + ])]); + $this->assertTrue($result['completed'], json_encode($result, JSON_THROW_ON_ERROR)); + $this->assertSame(1, WorkflowServiceCall::query()->count()); + $this->assertSame($this->run->id, $call->fresh()->caller_workflow_run_id); + } + + /** + * @return iterable + */ + public static function ownedCalls(): iterable + { + foreach (['started', 'completed'] as $status) { + yield $status . ' by id' => [$status, false]; + yield $status . ' by key' => [$status, true]; + } + } + + public function testNewCallsUseTheRunWorkloadPrincipal(): void + { + $result = $this->bridge->complete($this->task->id, [$this->command()]); + $this->assertTrue($result['completed'], json_encode($result, JSON_THROW_ON_ERROR)); + $call = WorkflowServiceCall::query()->sole(); + $this->assertSame('caller', $call->namespace); + $this->assertSame('caller', $call->caller_namespace); + $this->assertSame('workflow:' . $this->run->id, $call->caller_principal_subject); + $this->assertSame('workflow', $call->caller_principal_method); + $this->assertNull($call->caller_principal_roles); + $this->assertNull($call->caller_principal_tenant); + $this->assertNull($call->caller_principal_claims); + } + + public function testMissingTrustedContextRefusesBeforeAnyCommandWrites(): void + { + $this->app->forgetInstance(WorkflowServiceOperationAuthorizer::class); + $this->assertInstanceOf( + DenyWorkflowServiceOperationAuthorizer::class, + $this->app->make(WorkflowServiceOperationAuthorizer::class) + ); + $before = $this->records(); + $result = $this->bridge->complete($this->task->id, [ + [ + 'type' => 'start_timer', + 'delay_seconds' => 60, + ], + $this->command(), + ]); + $this->assertFalse($result['completed']); + $this->assertSame('service_operation_authority_invalid', $result['reason']); + $this->assertSame($before, $this->records()); + } + + public function testNullRunNamespaceDoesNotUseAmbientConfiguration(): void + { + $this->run->instance->forceFill([ + 'namespace' => null, + ])->save(); + $this->run->forceFill([ + 'namespace' => null, + ])->save(); + $this->task->forceFill([ + 'namespace' => null, + ])->save(); + config([ + 'workflows.v2.namespace' => 'ambient-namespace', + ]); + + $result = $this->bridge->complete($this->task->id, [$this->command()]); + $this->assertTrue($result['completed'], json_encode($result, JSON_THROW_ON_ERROR)); + $call = WorkflowServiceCall::query()->sole(); + $this->assertNull($call->namespace); + $this->assertNull($call->target_namespace); + $this->assertNull($call->caller_namespace); + $this->assertSame($this->run->id, $call->caller_workflow_run_id); + } + + public function testDefaultIdempotencyKeyCannotReplayAnotherRunsCall(): void + { + $call = $this->serviceCall('completed'); + $call->forceFill([ + 'caller_workflow_run_id' => 'another-run', + 'idempotency_key' => 'workflow-service-operation:' . $this->run->workflow_instance_id . ':' . $this->run->id . ':2', + ])->save(); + $before = $this->records(); + $result = $this->bridge->complete($this->task->id, [ + [ + 'type' => 'start_timer', + 'delay_seconds' => 60, + ], + $this->command(), + ]); + $this->assertFalse($result['completed']); + $this->assertSame('service_call_not_owned', $result['reason']); + $this->assertSame($before, $this->records()); + } + + public function testRetainedClaimPrefixRefusesForeignCallBeforeWritingItsSibling(): void + { + $call = $this->serviceCall('completed'); + $call->forceFill([ + 'caller_workflow_instance_id' => 'another-instance', + ])->save(); + $before = $this->records(); + $result = $this->app->make(\Workflow\V2\Contracts\PreparedLocalActivityTaskBridge::class) + ->checkpointLocalActivityPrefix( + $this->task->id, + 'authority-worker', + $this->task->fresh() +->attempt_count, + 'authority-checkpoint', + 1, + [ + [ + 'type' => 'start_timer', + 'delay_seconds' => 60, + ], + $this->command() + [ + 'service_call_id' => $call->id, + ], + ], + '1.20' + ); + $this->assertFalse($result['checkpointed'], json_encode($result, JSON_THROW_ON_ERROR)); + $this->assertSame('service_call_not_owned', $result['reason']); + $this->assertSame($before, $this->records()); + } + + public function testDuplicatePrefixStillRequiresCurrentTrustedAuthority(): void + { + $call = $this->serviceCall('completed'); + $bridge = $this->app->make(\Workflow\V2\Contracts\PreparedLocalActivityTaskBridge::class); + $commands = [ + $this->command() + [ + 'service_call_id' => $call->id, + ]]; + $attempt = $this->task->fresh() +->attempt_count; + $first = $bridge->checkpointLocalActivityPrefix( + $this->task->id, + 'authority-worker', + $attempt, + 'authority-checkpoint', + 1, + $commands, + '1.20' + ); + $this->assertTrue($first['checkpointed'], json_encode($first, JSON_THROW_ON_ERROR)); + $before = $this->records(); + $this->app->instance(WorkflowServiceOperationAuthorizer::class, new DenyWorkflowServiceOperationAuthorizer()); + $duplicate = $bridge->checkpointLocalActivityPrefix( + $this->task->id, + 'authority-worker', + $attempt, + 'authority-checkpoint', + 1, + $commands, + '1.20' + ); + $this->assertFalse($duplicate['checkpointed'], json_encode($duplicate, JSON_THROW_ON_ERROR)); + $this->assertSame('service_operation_authority_invalid', $duplicate['reason']); + $this->assertSame($before, $this->records()); + } + + public function testOwnedResolutionFailureCanBeReplayed(): void + { + $first = $this->bridge->complete($this->task->id, [$this->command()]); + $this->assertTrue($first['completed']); + $call = WorkflowServiceCall::query()->sole(); + $before = $call->getRawOriginal(); + $this->assertSame('failed', $call->status); + $task = WorkflowTask::query()->whereKey($first['created_task_ids'][0])->sole(); + $this->assertTrue($this->bridge->claimStatus($task->id, 'authority-worker')['claimed']); + $replay = $this->bridge->complete($task->id, [ + $this->command() + [ + 'service_call_id' => $call->id, + ]]); + $this->assertTrue($replay['completed'], json_encode($replay, JSON_THROW_ON_ERROR)); + $this->assertSame(1, WorkflowServiceCall::query()->count()); + $this->assertSame($before, $call->fresh()->getRawOriginal()); + } + + /** + * @return array + */ + private function command(): array + { + return [ + 'type' => 'start_service_operation', + 'endpoint_name' => 'billing', + 'service_name' => 'invoices', + 'operation_name' => 'create', + ]; + } + + private function serviceCall(string $status): WorkflowServiceCall + { + $endpoint = WorkflowServiceEndpoint::query()->create([ + 'namespace' => 'caller', + 'endpoint_name' => 'billing', + ]); + $service = WorkflowService::query()->create([ + 'namespace' => 'caller', + 'workflow_service_endpoint_id' => $endpoint->id, + 'service_name' => 'invoices', + ]); + $operation = WorkflowServiceOperation::query()->create([ + 'namespace' => 'caller', + 'workflow_service_endpoint_id' => $endpoint->id, + 'workflow_service_id' => $service->id, + 'operation_name' => 'create', + 'operation_mode' => 'async', + 'handler_binding_kind' => ServiceCallBindingKind::WorkflowRun->value, + 'handler_target_reference' => 'service-authority-workflow', + ]); + return WorkflowServiceCall::query()->create([ + 'namespace' => 'caller', + 'target_namespace' => 'caller', + 'caller_namespace' => 'caller', + 'caller_workflow_instance_id' => $this->run->workflow_instance_id, + 'caller_workflow_run_id' => $this->run->id, + 'workflow_service_endpoint_id' => $endpoint->id, + 'workflow_service_id' => $service->id, + 'workflow_service_operation_id' => $operation->id, + 'endpoint_name' => 'billing', + 'service_name' => 'invoices', + 'operation_name' => 'create', + 'operation_mode' => 'async', + 'status' => $status, + 'outcome' => $status === 'completed' ? ServiceCallOutcome::Completed->value + : ServiceCallOutcome::Accepted->value, + 'resolved_binding_kind' => ServiceCallBindingKind::WorkflowRun->value, + 'idempotency_key' => 'shared-idempotency-key', + 'accepted_at' => $status === ServiceCallStatus::Pending->value ? null : now(), + 'started_at' => $status === ServiceCallStatus::Started->value ? now() : null, + 'completed_at' => $status === ServiceCallStatus::Completed->value ? now() : null, + ]); + } + + /** + * @return array>> + */ + private function records(): array + { + $records = []; + foreach ([WorkflowInstance::class, WorkflowRun::class, WorkflowTask::class, WorkflowCommand::class, + WorkflowHistoryEvent::class, WorkflowRunSummary::class, WorkflowServiceCall::class, + WorkflowServiceEndpoint::class, WorkflowService::class, WorkflowServiceOperation::class, + WorkflowTimer::class] as $model) { + $records[$model] = $model::query()->orderBy('id')->get()->map( + static fn (Model $row): array => $row->getRawOriginal(), + )->all(); + } + return $records; + } +} + +#[Type('service-authority-workflow')] +final class ServiceAuthorityWorkflow extends Workflow +{ + public function handle(): string + { + return 'done'; + } +} From d902242ae737b871e24e889979dc5cac78082fdb Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sun, 25 Oct 2026 22:36:06 +0000 Subject: [PATCH 02/11] Retain service authority refusal in the replay corpus --- .../V2EmbeddedReplayRegressionCorpusTest.php | 6 +++ ...flow-service-operation-host-authority.json | 48 +++++++++++++++++++ tests/Support/NexusCallerDeliveryScenario.php | 41 ++++++++++++++++ 3 files changed, 95 insertions(+) create mode 100644 tests/Fixtures/V2/ReplayRegression/workflow-service-operation-host-authority.json diff --git a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php index b2e788402..ccb65115a 100644 --- a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php +++ b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php @@ -330,6 +330,12 @@ public function testFixturesExecuteThroughDeclaredReplayConsumers(): void continue; } + if ($fixture['id'] === 'workflow-service-operation-host-authority') { + $this->clearWorkflowState(); + $this->assertNexusWorkerCannotSupplyAuthority($fixture); + continue; + } + if ($fixture['id'] === 'nexus-waiting-caller-terminal-delivery') { $this->clearWorkflowState(); $this->assertWaitingNexusCallerReceivesResult( diff --git a/tests/Fixtures/V2/ReplayRegression/workflow-service-operation-host-authority.json b/tests/Fixtures/V2/ReplayRegression/workflow-service-operation-host-authority.json new file mode 100644 index 000000000..4f01b6bbf --- /dev/null +++ b/tests/Fixtures/V2/ReplayRegression/workflow-service-operation-host-authority.json @@ -0,0 +1,48 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "workflow-service-operation-host-authority", + "protocol_version": "1.19", + "bindings": ["php"], + "consumers": ["workflow-fiber-runner", "workflow-executor"], + "workflow": { + "type": "Tests\\Fixtures\\V2\\TestNexusWaitingCallerWorkflow", + "arguments": ["world"], + "payload_codec": "avro" + }, + "history": [ + {"sequence": 1, "event_type": "WorkflowStarted", "payload": {}}, + { + "sequence": 2, + "event_type": "ServiceCallStarted", + "payload": { + "sequence": 1, + "service_call_id": "authority-recorded-call", + "endpoint_name": "php-service", + "service_name": "greeter", + "operation_name": "greet", + "status": "started", + "operation_mode": "sync", + "wait_for": "completed", + "service_call": {"service_call_id": "authority-recorded-call", "status": "started"} + } + } + ], + "authority": { + "commands": [ + {"type": "start_timer", "delay_seconds": 60}, + { + "type": "start_service_operation", + "endpoint_name": "billing", + "service_name": "invoices", + "operation_name": "create", + "namespace": "foreign", + "caller_namespace": "foreign", + "principal_subject": "administrator", + "principal_roles": ["admin"] + } + ], + "reason": "service_operation_authority_invalid" + }, + "expected": {"completed": false, "result": null, "commands": []} +} diff --git a/tests/Support/NexusCallerDeliveryScenario.php b/tests/Support/NexusCallerDeliveryScenario.php index d8278dc34..390fcd3f7 100644 --- a/tests/Support/NexusCallerDeliveryScenario.php +++ b/tests/Support/NexusCallerDeliveryScenario.php @@ -17,14 +17,17 @@ use Workflow\V2\Enums\TaskStatus; use Workflow\V2\Exceptions\ServiceOperationAuthorizationException; use Workflow\V2\Jobs\RunWorkflowTask; +use Workflow\V2\Models\WorkflowCommand; use Workflow\V2\Models\WorkflowHistoryEvent; use Workflow\V2\Models\WorkflowInstance; use Workflow\V2\Models\WorkflowRun; +use Workflow\V2\Models\WorkflowRunSummary; use Workflow\V2\Models\WorkflowService; use Workflow\V2\Models\WorkflowServiceCall; use Workflow\V2\Models\WorkflowServiceEndpoint; use Workflow\V2\Models\WorkflowServiceOperation; use Workflow\V2\Models\WorkflowTask; +use Workflow\V2\Models\WorkflowTimer; use Workflow\V2\Support\DefaultServiceControlPlane; use Workflow\V2\Support\ServiceCallPrincipal; use Workflow\V2\Support\WorkflowFiberRunner; @@ -34,6 +37,44 @@ trait NexusCallerDeliveryScenario { + /** + * @param array $fixture + */ + protected function assertNexusWorkerCannotSupplyAuthority(array $fixture): void + { + $previousDispatchMode = config('workflows.v2.task_dispatch_mode'); + config([ + 'workflows.v2.task_dispatch_mode' => 'poll', + ]); + try { + $workflow = WorkflowStub::make($fixture['workflow']['type'], 'authority-corpus-caller', 'caller'); + $this->assertTrue($workflow->attemptStart(...$fixture['workflow']['arguments'])->accepted()); + $task = WorkflowTask::query()->where('workflow_run_id', $workflow->runId())->sole(); + $bridge = $this->app->make(WorkflowTaskBridge::class); + $this->assertTrue($bridge->claimStatus($task->id, 'authority-corpus-worker')['claimed']); + $models = [WorkflowInstance::class, WorkflowRun::class, WorkflowTask::class, WorkflowTimer::class, + WorkflowCommand::class, WorkflowHistoryEvent::class, WorkflowRunSummary::class, WorkflowServiceCall::class]; + $snapshot = static function () use ($models): array { + $rows = []; + foreach ($models as $model) { + $rows[$model] = $model::query()->orderBy('id')->get()->map( + static fn ($row): array => $row->getRawOriginal(), + )->all(); + } + return $rows; + }; + $before = $snapshot(); + $result = $bridge->complete($task->id, $fixture['authority']['commands']); + $this->assertFalse($result['completed'], json_encode($result, JSON_THROW_ON_ERROR)); + $this->assertSame($fixture['authority']['reason'], $result['reason']); + $this->assertSame($before, $snapshot()); + } finally { + config([ + 'workflows.v2.task_dispatch_mode' => $previousDispatchMode, + ]); + } + } + /** * @param array $fixture */ From fce3ac4b4448fc6b3835d338725a0d37d29d2af3 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Mon, 26 Oct 2026 00:54:29 +0000 Subject: [PATCH 03/11] Bind trusted hosts in service operation tests --- tests/Unit/V2/LinkedServiceCallCloserTest.php | 23 +++++++++++++++++++ .../Unit/V2/NexusCancellationDeliveryTest.php | 19 +++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/tests/Unit/V2/LinkedServiceCallCloserTest.php b/tests/Unit/V2/LinkedServiceCallCloserTest.php index 173906cbd..efd4cef15 100644 --- a/tests/Unit/V2/LinkedServiceCallCloserTest.php +++ b/tests/Unit/V2/LinkedServiceCallCloserTest.php @@ -15,6 +15,7 @@ use Workflow\V2\Contracts\ExternalPayloadStoragePolicy; use Workflow\V2\Contracts\ServiceControlPlane; use Workflow\V2\Contracts\WorkflowControlPlane; +use Workflow\V2\Contracts\WorkflowServiceOperationAuthorizer; use Workflow\V2\Contracts\WorkflowTaskBridge; use Workflow\V2\Enums\HistoryEventType; use Workflow\V2\Enums\RunStatus; @@ -22,6 +23,7 @@ use Workflow\V2\Enums\ServiceCallOutcome; use Workflow\V2\Enums\TaskStatus; use Workflow\V2\Enums\TaskType; +use Workflow\V2\Exceptions\ServiceOperationAuthorizationException; use Workflow\V2\Jobs\RunWorkflowTask; use Workflow\V2\Models\WorkflowHistoryEvent; use Workflow\V2\Models\WorkflowInstance; @@ -36,6 +38,7 @@ use Workflow\V2\Support\ExternalPayloadStorage; use Workflow\V2\Support\LinkedServiceCallCloser; use Workflow\V2\Support\ServiceCallDeadline; +use Workflow\V2\Support\ServiceCallPrincipal; use Workflow\V2\Support\ServiceCallView; use Workflow\V2\Support\ServiceCallWorkflowDelivery; use Workflow\V2\Support\ServiceOperationOptions; @@ -456,6 +459,7 @@ public function testEmbeddedCallerWaitsAndThenReplaysTheActualServiceResult(): v 'workflow_run_id' => $service->id, ]); $this->registerWorkflowService(); + $this->registerTrustedServiceOperationHost(); $this->app->instance(ServiceControlPlane::class, new DefaultServiceControlPlane($control)); $workflow = WorkflowStub::make(NexusWaitingCallerWorkflow::class, 'waiting-caller', 'caller'); $this->assertTrue($workflow->attemptStart()->accepted()); @@ -997,6 +1001,25 @@ private function registerWorkflowService(): void ]); } + private function registerTrustedServiceOperationHost(): void + { + $this->app->instance( + WorkflowServiceOperationAuthorizer::class, + new class() implements WorkflowServiceOperationAuthorizer { + public function authorize(WorkflowRun $run, array $command): ServiceCallPrincipal + { + if ($run->namespace !== 'caller' || ($command['namespace'] ?? null) !== 'service' + || [$command['endpoint_name'], $command['service_name'], $command['operation_name']] + !== ['php-service', 'greeter', 'greet']) { + throw new ServiceOperationAuthorizationException(); + } + + return new ServiceCallPrincipal(subject: 'workflow:' . $run->id, method: 'workflow'); + } + } + ); + } + private function createRun(string $namespace = 'service'): WorkflowRun { $instance = WorkflowInstance::query()->create([ diff --git a/tests/Unit/V2/NexusCancellationDeliveryTest.php b/tests/Unit/V2/NexusCancellationDeliveryTest.php index e5e5aa11a..794f3a22b 100644 --- a/tests/Unit/V2/NexusCancellationDeliveryTest.php +++ b/tests/Unit/V2/NexusCancellationDeliveryTest.php @@ -11,20 +11,24 @@ use Tests\Fixtures\V2\TestNexusWaitingCallerWorkflow; use Tests\TestCase; use Workflow\Serializers\Serializer; +use Workflow\V2\Contracts\WorkflowServiceOperationAuthorizer; use Workflow\V2\Contracts\WorkflowTaskBridge; use Workflow\V2\Enums\HistoryEventType; use Workflow\V2\Enums\RunStatus; use Workflow\V2\Enums\TaskStatus; use Workflow\V2\Enums\TaskType; +use Workflow\V2\Exceptions\ServiceOperationAuthorizationException; use Workflow\V2\Exceptions\WorkflowCancellationRequestedException; use Workflow\V2\Jobs\RunTimerTask; use Workflow\V2\Jobs\RunWorkflowTask; +use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowService; use Workflow\V2\Models\WorkflowServiceCall; use Workflow\V2\Models\WorkflowServiceEndpoint; use Workflow\V2\Models\WorkflowServiceOperation; use Workflow\V2\Models\WorkflowTask; use Workflow\V2\Support\CooperativeCancellationDelivery; +use Workflow\V2\Support\ServiceCallPrincipal; use Workflow\V2\Support\ServiceOperationCancellation; use Workflow\V2\Support\WorkflowFiberRunner; use Workflow\V2\WorkflowStub; @@ -40,6 +44,21 @@ protected function setUp(): void 'workflows.v2.compatibility.current' => 'build-a', 'workflows.v2.compatibility.supported' => ['build-a'], ]); + $this->app->instance( + WorkflowServiceOperationAuthorizer::class, + new class() implements WorkflowServiceOperationAuthorizer { + public function authorize(WorkflowRun $run, array $command): ServiceCallPrincipal + { + if ($run->namespace !== 'caller' || ($command['namespace'] ?? null) !== 'service' + || [$command['endpoint_name'], $command['service_name'], $command['operation_name']] + !== ['php-service', 'greeter', 'greet']) { + throw new ServiceOperationAuthorizationException(); + } + + return new ServiceCallPrincipal(subject: 'workflow:' . $run->id, method: 'workflow'); + } + } + ); Carbon::setTestNow('2026-10-12T08:00:00Z'); } From 2a7511501bb91ba85e3ff250f65c14ae01ba4c93 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Mon, 26 Oct 2026 02:52:09 +0000 Subject: [PATCH 04/11] Schedule workflow service calls outside completion locks --- docs/api-stability.md | 25 + .../workflow-service-calls-architecture.md | 80 +- src/Commands/V2RepairPassCommand.php | 18 +- src/V2/Contracts/MatchingRole.php | 3 + src/V2/Enums/HistoryEventType.php | 1 + src/V2/Jobs/RunServiceCall.php | 43 ++ src/V2/Models/WorkflowServiceCall.php | 9 + src/V2/Models/WorkflowServiceCallDispatch.php | 49 ++ .../CooperativeCancellationDelivery.php | 2 + src/V2/Support/DefaultServiceControlPlane.php | 208 ++++- src/V2/Support/DefaultWorkflowTaskBridge.php | 25 +- src/V2/Support/EmbeddedV2HistoryImport.php | 2 + .../Support/HistoryEventPayloadContract.php | 32 + src/V2/Support/HistoryTimeline.php | 6 + src/V2/Support/QueryStateReplayer.php | 18 +- src/V2/Support/ServiceCallAttemptClaim.php | 28 + .../Support/ServiceCallCancellationOrigin.php | 16 +- src/V2/Support/ServiceCallScheduler.php | 709 ++++++++++++++++++ .../Support/ServiceCallWorkflowDelivery.php | 27 +- src/V2/Support/ServiceExecutionContract.php | 24 + .../Support/ServiceOperationCancellation.php | 38 +- src/V2/Support/ServiceOperationOptions.php | 8 + src/V2/Support/UpdateWaitPolicy.php | 52 +- src/V2/Support/WorkerProtocolVersion.php | 12 +- src/V2/Support/WorkflowCommandNormalizer.php | 10 + src/V2/Support/WorkflowExecutor.php | 45 +- src/V2/Support/WorkflowFiberRunner.php | 21 +- .../WorkflowServiceOperationAuthority.php | 2 + src/V2/Support/WorkflowStepHistory.php | 2 + src/V2/TaskWatchdog.php | 24 + src/config/workflows.php | 13 + ...workflow_service_call_dispatches_table.php | 56 ++ .../Unit/Commands/V2RepairPassCommandTest.php | 30 + .../Providers/WorkflowServiceProviderTest.php | 6 + ...amespaceServicePolicyDocumentationTest.php | 2 + ...ServiceControlPlaneWorkflowBindingTest.php | 44 +- ...WorkflowTaskBridgeServiceOperationTest.php | 30 +- tests/Unit/V2/LinkedServiceCallCloserTest.php | 39 +- .../Unit/V2/NexusCancellationDeliveryTest.php | 43 ++ ...QueryStateReplayerServiceOperationTest.php | 20 + tests/Unit/V2/ServiceCallSchedulerTest.php | 541 +++++++++++++ .../V2/ServiceCommandOptionValidationTest.php | 17 + .../Unit/V2/ServiceExecutionContractTest.php | 15 +- tests/Unit/V2/UpdateWaitPolicyTest.php | 58 ++ tests/Unit/V2/WorkerProtocolVersionTest.php | 8 +- .../WorkflowExecutorServiceOperationTest.php | 21 + tests/Unit/V2/WorkflowFiberRunnerTest.php | 44 ++ tests/Unit/migrations/MigrationsTest.php | 9 + 48 files changed, 2465 insertions(+), 70 deletions(-) create mode 100644 src/V2/Jobs/RunServiceCall.php create mode 100644 src/V2/Models/WorkflowServiceCallDispatch.php create mode 100644 src/V2/Support/ServiceCallAttemptClaim.php create mode 100644 src/V2/Support/ServiceCallScheduler.php create mode 100644 src/migrations/2026_10_26_000100_create_workflow_service_call_dispatches_table.php create mode 100644 tests/Unit/V2/ServiceCallSchedulerTest.php create mode 100644 tests/Unit/V2/UpdateWaitPolicyTest.php diff --git a/docs/api-stability.md b/docs/api-stability.md index 6fb855196..304a8ae4d 100644 --- a/docs/api-stability.md +++ b/docs/api-stability.md @@ -203,6 +203,31 @@ list, command-kind map, generated identity prefixes, path rules, or structural size checks; compatible grammar evolution belongs in the Workflow package and is consumed through those methods. +### Durable service-operation scheduling + +Worker protocol `1.21` adds the `ServiceOperationScheduled` history event for a +workflow-authored service call whose effect will run after the completion +transaction commits. Its payload uses the same service-call identity, address, +codec and result-surface fields as `ServiceCallStarted`, plus the required +boolean `workflow_resume_on_admission`. A `false` value is durable evidence that +the workflow remains blocked for a terminal service-call event even though the +scheduler's internal handler wait is `accepted`. + +Hosts must not synthesize `ServiceCallStarted` before a handler attempt is +claimed. They must version-gate the new scheduled-event behavior at +`WorkerProtocolVersion::DURABLE_SERVICE_OPERATION_SCHEDULER_MINIMUM_PROTOCOL_VERSION` +and keep workers below that floor away from histories that require the event. +The original `start_service_operation` command remains available from its 1.13 +floor; the scheduled execution semantics require 1.21. + +`WorkflowServiceCallDispatch` is a documented persisted model. Its table and +state/attempt/claim/revision/repair fields are additive schema and observability +surface. `ServiceCallScheduler`, `ServiceCallAttemptClaim`, and the +`RunServiceCall` job are internal execution helpers: their PHP method and +constructor signatures are not covered by the public semver guarantee. The +machine-readable durable-record and scheduling rules are published by +`ServiceExecutionContract`. + ## Activity attempt authority An activity task's authority ends at its current lease expiry or an applicable diff --git a/docs/architecture/workflow-service-calls-architecture.md b/docs/architecture/workflow-service-calls-architecture.md index 0355612ed..559e149c3 100644 --- a/docs/architecture/workflow-service-calls-architecture.md +++ b/docs/architecture/workflow-service-calls-architecture.md @@ -206,6 +206,71 @@ The contract is symmetric: a Sync caller always has the same durable record as an Async caller. Sync mode is a delivery preference on top of the durable record, never an alternative to it. +## Durable scheduling, claims, and retries + +A service operation authored by a workflow is accepted inside the workflow-task +completion transaction, but its handler effect is not executed there. That +transaction may only persist the accepted `workflow_service_calls` row, the +immutable accepted handler-binding snapshot, a +`workflow_service_call_dispatches` row, and a `ServiceOperationScheduled` +history event. It MUST NOT invoke a workflow/activity/carrier handler, enqueue +work before commit, wait for an update result, or sleep for retry backoff. + +`workflow_service_call_dispatches` is the durable scheduler record. Its state is +`scheduled`, `claimed`, or `completed`; `attempt`, `max_attempts`, and +`available_at` carry the retry schedule. `claim_generation` plus `claim_token` +form the compare-and-swap fence. `dispatch_attempted_at`, +`repair_available_at`, and `claim_expires_at` make a lost enqueue or crashed +worker discoverable by the normal v2 repair pass. + +The completion transaction registers queue publication for after commit. Queue +delivery is at least once, so an enqueue failure, duplicate job, or worker crash +does not change the durable schedule. `TaskWatchdog` republishes due schedules, +lost dispatches, and expired claims. Operators may tune +`workflows.v2.service_call_dispatch.claim_lease_seconds` and +`repair_after_seconds`; disabling the repair pass while scheduled calls exist +violates the liveness contract. + +A runnable attempt claims in one database transaction and in a fixed lock order: +service-call row, dispatch row, current caller-run row, endpoint, service, then +operation. The claim reconstructs the principal recorded when the call was +accepted, checks that the current caller is still active and has no cooperative +cancellation request, re-runs +`authorizeReplay()` against the current catalog policy, and stores hashes of the +authorized catalog and policy revisions. Authority is current at claim time; +the actual handler kind, binding and target reference remain the immutable +accepted snapshot. A later catalog edit may revoke a call but cannot redirect +an already accepted call to a different handler. + +An explicit authorization refusal terminally rejects the call before any +effect. A transient policy or storage failure is not an authorization decision: +the claim transaction rolls back and leaves the durable schedule repairable. + +The handler effect runs only after that claim transaction commits. Every attempt +receives the stable identity +`workflow-service-call::attempt:`. Handler adapters +MUST propagate or otherwise enforce that identity because a lease may expire +after an effect began and at-least-once recovery may invoke the attempt again. +Only the worker holding the current generation and token may persist handler +progress or complete the attempt. Retry backoff is stored in `available_at`; no +worker thread sleeps while holding a call, caller, catalog, or workflow-task +lock. + +`ServiceOperationScheduled.workflow_resume_on_admission` records the authored +delivery decision separately from the scheduler's internal accepted-only +dispatch. `true` lets an explicit accepted/async caller resume on the scheduled +event. `false` keeps a synchronous caller suspended until a terminal +`ServiceCallCompleted`, `ServiceCallFailed`, or `ServiceCallCancelled` event is +delivered and remains the authority for cooperative cancellation ownership. The +internal `wait_for=accepted` surface MUST NOT by itself resume a synchronous +workflow or erase its authored completion wait. + +All service-operation wait-time inputs are bounded to 60 seconds. Direct calls +made outside a workflow completion transaction may still use the existing +inline control-plane path, but they receive the same bound. Inside a completion +transaction the handler-facing wait is always `accepted`, and any longer wait +is represented by durable history and a later workflow task. + ## Linked target references Every accepted call carries a `resolved_binding_kind` (matching the @@ -466,6 +531,10 @@ render: `payload_codec`); - the **timing** (`accepted_at`, `started_at`, `completed_at`, `failed_at`, `cancelled_at`). +- when a handler attempt is pending, the **dispatch state** (`state`, + `attempt`, `max_attempts`, `available_at`), current claim generation and + lease expiry, last authorized catalog/policy revisions, and next repair time + from `workflow_service_call_dispatches`. No part of the observability surface MAY require raw transport logs. The transport that carried the call between namespaces is an @@ -477,6 +546,9 @@ The following components are bound by this contract: - `WorkflowServiceCall` model — the Eloquent surface over `workflow_service_calls`. +- `WorkflowServiceCallDispatch` model — the durable schedule, attempt lease, + authorization revision and repair surface over + `workflow_service_call_dispatches`. - `WorkflowServiceOperation` — defines the contract-layer policies every accepted call snaps. - `WorkflowServiceEndpoint` — owns admission of calls in a target @@ -516,8 +588,12 @@ them is permitted to rely on transport-only state. `tests/Unit/V2/ServiceExecutionContractTest.php` verifies the machine-readable service execution manifest against the runtime enums, -control-plane interface, and durable record fields. Migration and model -tests verify the persisted service-call shape. +control-plane interface, and durable call/dispatch record fields. +`ServiceCallSchedulerTest` covers commit/rollback, lost enqueue, expired +claims, stale claim fencing, current-policy revocation, transient-policy +rollback, immutable bindings, admission release, deadlines, terminal callers, +the 60-second ingress bound and persisted backoff without lock-held sleep. +Migration and model tests verify the persisted service-call shape. ## Changing this contract diff --git a/src/Commands/V2RepairPassCommand.php b/src/Commands/V2RepairPassCommand.php index 583946647..2774e26b0 100644 --- a/src/Commands/V2RepairPassCommand.php +++ b/src/Commands/V2RepairPassCommand.php @@ -86,6 +86,7 @@ private function runOnce(?bool $respectThrottleOverride): int return $report['existing_task_failures'] === [] && $report['missing_run_failures'] === [] && ($report['service_call_timeout_failures'] ?? []) === [] + && ($report['service_call_dispatch_failures'] ?? []) === [] ? self::SUCCESS : self::FAILURE; } @@ -173,7 +174,10 @@ private function resolveSleepSeconds(): int * activity_timeout_failures: list, * service_call_timeout_candidates?: int, * service_call_timeouts_enforced?: int, - * service_call_timeout_failures?: list + * service_call_timeout_failures?: list, + * service_call_dispatch_candidates?: int, + * service_call_dispatches_enqueued?: int, + * service_call_dispatch_failures?: list * } $report */ private function renderHumanReport(array $report): void @@ -237,6 +241,18 @@ private function renderHumanReport(array $report): void sprintf('Service call [%s] failed timeout enforcement: %s', $failure['call_id'], $failure['message']) ); } + $this->line(sprintf( + 'Service call dispatch repair: %d enqueued from %d candidates.', + $report['service_call_dispatches_enqueued'] ?? 0, + $report['service_call_dispatch_candidates'] ?? 0, + )); + foreach (($report['service_call_dispatch_failures'] ?? []) as $failure) { + $this->error(sprintf( + 'Service call [%s] failed dispatch repair: %s', + $failure['call_id'], + $failure['message'], + )); + } } private function stringOption(string $name): ?string diff --git a/src/V2/Contracts/MatchingRole.php b/src/V2/Contracts/MatchingRole.php index 3d1326810..a0594a4a5 100644 --- a/src/V2/Contracts/MatchingRole.php +++ b/src/V2/Contracts/MatchingRole.php @@ -41,6 +41,9 @@ public function wake(?string $connection = null, ?string $queue = null): void; * service_call_timeout_candidates?: int, * service_call_timeouts_enforced?: int, * service_call_timeout_failures?: list + * service_call_dispatch_candidates?: int, + * service_call_dispatches_enqueued?: int, + * service_call_dispatch_failures?: list * } */ public function runPass( diff --git a/src/V2/Enums/HistoryEventType.php b/src/V2/Enums/HistoryEventType.php index aba689e00..4594e418b 100644 --- a/src/V2/Enums/HistoryEventType.php +++ b/src/V2/Enums/HistoryEventType.php @@ -18,6 +18,7 @@ enum HistoryEventType: string case ChildRunFailed = 'ChildRunFailed'; case ChildRunCancelled = 'ChildRunCancelled'; case ChildRunTerminated = 'ChildRunTerminated'; + case ServiceOperationScheduled = 'ServiceOperationScheduled'; case ServiceCallStarted = 'ServiceCallStarted'; case ServiceCallCompleted = 'ServiceCallCompleted'; case ServiceCallFailed = 'ServiceCallFailed'; diff --git a/src/V2/Jobs/RunServiceCall.php b/src/V2/Jobs/RunServiceCall.php new file mode 100644 index 000000000..6dda68239 --- /dev/null +++ b/src/V2/Jobs/RunServiceCall.php @@ -0,0 +1,43 @@ +afterCommit(); + } + + public function handle(ServiceControlPlane $controlPlane): void + { + if (! $controlPlane instanceof DefaultServiceControlPlane) { + throw new RuntimeException('scheduled_service_call_executor_unavailable'); + } + $claim = ServiceCallScheduler::claim($this->serviceCallId); + + if ($claim !== null) { + $controlPlane->executeScheduledAttempt($claim); + } + } +} diff --git a/src/V2/Models/WorkflowServiceCall.php b/src/V2/Models/WorkflowServiceCall.php index 79ace9bf1..4e69efde5 100644 --- a/src/V2/Models/WorkflowServiceCall.php +++ b/src/V2/Models/WorkflowServiceCall.php @@ -7,6 +7,7 @@ use Illuminate\Database\Eloquent\Concerns\HasUlids; use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Relations\BelongsTo; +use Illuminate\Database\Eloquent\Relations\HasOne; use Workflow\Traits\ResolvesStorageConnection; use Workflow\V2\Enums\ServiceCallOutcome; use Workflow\V2\Support\ConfiguredV2Models; @@ -111,4 +112,12 @@ public function linkedUpdate(): BelongsTo 'linked_workflow_update_id', ); } + + public function dispatch(): HasOne + { + return $this->hasOne( + ConfiguredV2Models::resolve('service_call_dispatch_model', WorkflowServiceCallDispatch::class), + 'service_call_id', + ); + } } diff --git a/src/V2/Models/WorkflowServiceCallDispatch.php b/src/V2/Models/WorkflowServiceCallDispatch.php new file mode 100644 index 000000000..32aedc0d8 --- /dev/null +++ b/src/V2/Models/WorkflowServiceCallDispatch.php @@ -0,0 +1,49 @@ + 'array', + 'attempt' => 'integer', + 'max_attempts' => 'integer', + 'claim_generation' => 'integer', + 'available_at' => UtcScheduleTimestamp::class, + 'claimed_at' => 'datetime', + 'claim_expires_at' => UtcScheduleTimestamp::class, + 'dispatch_attempted_at' => 'datetime', + 'repair_available_at' => UtcScheduleTimestamp::class, + 'completed_at' => 'datetime', + ]; + + public function call(): BelongsTo + { + return $this->belongsTo( + ConfiguredV2Models::resolve('service_call_model', WorkflowServiceCall::class), + 'service_call_id', + ); + } +} diff --git a/src/V2/Support/CooperativeCancellationDelivery.php b/src/V2/Support/CooperativeCancellationDelivery.php index 7d0354377..1f898ec95 100644 --- a/src/V2/Support/CooperativeCancellationDelivery.php +++ b/src/V2/Support/CooperativeCancellationDelivery.php @@ -307,6 +307,7 @@ public static function callKindAt(WorkflowRun $run, int $sequence): ?string HistoryEventType::ConditionWaitOpened => 'condition', HistoryEventType::SignalWaitOpened => 'signal', HistoryEventType::ChildWorkflowScheduled => 'child', + HistoryEventType::ServiceOperationScheduled, HistoryEventType::ServiceCallStarted => 'service_operation', HistoryEventType::TimerScheduled => match ($event->payload['timer_kind'] ?? null) { 'condition_timeout' => 'condition', @@ -341,6 +342,7 @@ private static function validateRootOperationMembership(WorkflowRun $run, int $s HistoryEventType::ActivityScheduled => 'activity', HistoryEventType::TimerScheduled => 'timer', HistoryEventType::ChildWorkflowScheduled => 'child_workflow', + HistoryEventType::ServiceOperationScheduled, HistoryEventType::ServiceCallStarted => 'service_operation', HistoryEventType::ConditionWaitOpened, HistoryEventType::SignalWaitOpened => '', default => null, diff --git a/src/V2/Support/DefaultServiceControlPlane.php b/src/V2/Support/DefaultServiceControlPlane.php index 0315ff63d..830c753a4 100644 --- a/src/V2/Support/DefaultServiceControlPlane.php +++ b/src/V2/Support/DefaultServiceControlPlane.php @@ -38,6 +38,8 @@ final class DefaultServiceControlPlane implements ServiceControlPlane private const WORKFLOW_QUERY_HANDLER_BINDING = 'workflow.v2.service_control_plane.workflow_query_handler'; + private ?ServiceCallAttemptClaim $scheduledClaim = null; + public function __construct( private ?WorkflowControlPlane $workflowControlPlane = null, private ?ServiceBoundaryPolicy $boundaryPolicy = null, @@ -54,6 +56,11 @@ public function execute( string $operationName, array $options = [] ): array { + if (array_key_exists('wait_timeout_seconds', $options)) { + $options['wait_timeout_seconds'] = UpdateWaitPolicy::serviceOperationTimeoutSeconds( + $options['wait_timeout_seconds'], + ); + } $namespace = $this->targetNamespace($options); $endpointKey = $this->contractName($endpointName); $serviceKey = $this->contractName($serviceName); @@ -288,10 +295,16 @@ public function execute( ]; } - $handler = $this->dispatchHandler($call, $operation, $boundaryRequest, $options); + $scheduled = $this->shouldScheduleHandler($call); + $handler = $scheduled + ? $this->scheduleHandler($call, $operation, $options) + : $this->dispatchHandler($call, $operation, $boundaryRequest, $options); $call->refresh(); - return $this->serialize($call) + [ + return $this->serialize($call) + ($scheduled ? [ + 'wait_for' => UpdateWaitPolicy::WAIT_FOR_ACCEPTED, + 'retry_scheduled' => true, + ] : []) + [ 'accepted' => $this->acceptedShape($call), 'idempotent_replay' => false, 'handler' => $handler, @@ -299,6 +312,50 @@ public function execute( ]; } + public function executeScheduledAttempt(ServiceCallAttemptClaim $claim): void + { + if (! ServiceCallScheduler::ownsClaim($claim)) { + return; + } + + $this->scheduledClaim = $claim; + try { + $startedAt = now(); + $result = $this->dispatchHandlerAttempt( + $claim->call, + $claim->operation, + $claim->request, + $claim->options, + $claim->attempt, + $claim->maxAttempts, + ); + $claim->call->refresh(); + $retry = $this->shouldRetryServiceCall($result, $claim->attempt, $claim->maxAttempts); + $backoff = $retry + ? $this->serviceCallBackoffSeconds($claim->call->retry_policy, $claim->attempt) + : null; + $recorded = ServiceCallScheduler::completeAttempt( + $claim, + $this->serviceCallAttemptEntry( + $claim->call, + $result, + $claim->attempt, + $startedAt, + $retry, + $backoff, + ), + $retry, + $backoff, + ); + $claim->call->refresh(); + if ($recorded && ! $retry && $this->isFailed($claim->call)) { + $this->releaseBoundaryAdmission($claim->request); + } + } finally { + $this->scheduledClaim = null; + } + } + /** * @param array $options * @return array @@ -964,6 +1021,36 @@ private function dispatchHandler( return $result; } + /** + * @param array $options + * @return array + */ + private function scheduleHandler( + WorkflowServiceCall $call, + WorkflowServiceOperation $operation, + array $options, + ): array { + $maxAttempts = $this->maxServiceCallAttempts($call->retry_policy); + ServiceCallScheduler::schedule($call, $operation, $options, $maxAttempts); + + return [ + 'accepted' => true, + 'kind' => $call->resolved_binding_kind, + 'wait_for' => UpdateWaitPolicy::WAIT_FOR_ACCEPTED, + 'retry_scheduled' => true, + 'service_call_attempt' => 1, + 'service_call_max_attempts' => $maxAttempts, + ]; + } + + private function shouldScheduleHandler(WorkflowServiceCall $call): bool + { + return is_string($call->caller_workflow_run_id) + && $call->caller_workflow_run_id !== '' + && $call->getConnection() + ->transactionLevel() > 0; + } + /** * @param array $options * @return array|null @@ -1292,6 +1379,10 @@ private function dispatchWorkflowUpdate( $commandOptions['wait_for'] ??= $call->operation_mode === ServiceCallOperationMode::Sync->value ? 'completed' : 'accepted'; + if ($call->getConnection()->transactionLevel() > 0) { + $commandOptions['wait_for'] = UpdateWaitPolicy::WAIT_FOR_ACCEPTED; + unset($commandOptions['wait_timeout_seconds']); + } try { $result = $this->workflows() @@ -1319,7 +1410,7 @@ private function dispatchWorkflowUpdate( ]; if (($result['update_status'] ?? null) === 'completed') { - $this->markCompleted( + $completed = $this->markCompleted( $call, ServiceCallBindingKind::WorkflowUpdate->value, $updateId ?? $updateName, @@ -1328,7 +1419,9 @@ private function dispatchWorkflowUpdate( $updateId, $metadata, ); - $this->releaseBoundaryAdmission($request); + if ($completed) { + $this->releaseBoundaryAdmission($request); + } } else { $this->markStarted( $call, @@ -1501,7 +1594,7 @@ private function dispatchWorkflowQuery( if (($result['success'] ?? false) === true) { $runId = $this->stringFrom($result['run_id'] ?? null); - $this->markCompleted( + $completed = $this->markCompleted( $call, ServiceCallBindingKind::WorkflowQuery->value, $runId ?? $instanceId, @@ -1513,7 +1606,9 @@ private function dispatchWorkflowQuery( 'control_plane' => $this->publicResult($result), ], ); - $this->releaseBoundaryAdmission($request); + if ($completed) { + $this->releaseBoundaryAdmission($request); + } return [ 'accepted' => true, @@ -1612,6 +1707,7 @@ private function dispatchActivityExecution( 'activity_execution_id' => $activityExecutionId, 'activity_class' => $activityClass, 'activity_type' => $activityType, + 'service_call_idempotency_key' => $options['service_call_idempotency_key'] ?? null, ]; foreach (['connection', 'queue'] as $key) { @@ -1679,6 +1775,7 @@ private function dispatchInvocableCarrierRequest( $metadata = [ 'carrier_request_id' => $carrierRequestId, 'carrier_endpoint' => $carrierEndpoint, + 'service_call_idempotency_key' => $options['service_call_idempotency_key'] ?? null, ]; if ($carrierHandler !== null) { @@ -1733,13 +1830,34 @@ private function stampBindingResolution( ?string $updateId, array $metadata, ): void { - $call->resolved_binding_kind = $bindingKind; - $call->resolved_target_reference = $targetReference; - $call->linked_workflow_instance_id = $instanceId; - $call->linked_workflow_run_id = $runId; - $call->linked_workflow_update_id = $updateId; - $call->metadata = $this->mergeMetadata($call->metadata, $metadata); - $call->save(); + $call->getConnection() + ->transaction(function () use ( + $call, + $bindingKind, + $targetReference, + $instanceId, + $runId, + $updateId, + $metadata, + ): void { + /** @var WorkflowServiceCall $locked */ + $locked = ConfiguredV2Models::query('service_call_model', WorkflowServiceCall::class) + ->lockForUpdate() + ->findOrFail($call->id); + if (! $this->scheduledClaimOwnsLockedCall($locked) + || ServiceCallDeadline::expireLocked($locked, now()) + || $this->isTerminal($locked)) { + return; + } + $locked->resolved_binding_kind = $bindingKind; + $locked->resolved_target_reference = $targetReference; + $locked->linked_workflow_instance_id = $instanceId; + $locked->linked_workflow_run_id = $runId; + $locked->linked_workflow_update_id = $updateId; + $locked->metadata = $this->mergeMetadata($locked->metadata, $metadata); + $locked->save(); + }); + $call->refresh(); } /** @@ -1753,7 +1871,8 @@ private function markStarted( ?string $runId, ?string $updateId, array $metadata, - ): void { + ): bool { + $recorded = false; $call->getConnection() ->transaction(function () use ( $call, @@ -1763,11 +1882,15 @@ private function markStarted( $runId, $updateId, $metadata, + &$recorded, ): void { /** @var WorkflowServiceCall $locked */ $locked = ConfiguredV2Models::query('service_call_model', WorkflowServiceCall::class) ->lockForUpdate() ->findOrFail($call->id); + if (! $this->scheduledClaimOwnsLockedCall($locked)) { + return; + } ServiceCallDeadline::expireLocked($locked, now()); $locked->resolved_binding_kind ??= $bindingKind; if (! $this->isTerminal($locked) @@ -1794,9 +1917,14 @@ private function markStarted( } $locked->save(); + $recorded = true; }); $call->refresh(); - LinkedServiceCallCloser::reconcile($call); + if ($recorded) { + LinkedServiceCallCloser::reconcile($call); + } + + return $recorded; } /** @@ -1810,7 +1938,8 @@ private function markCompleted( ?string $runId, ?string $updateId, array $metadata, - ): void { + ): bool { + $recorded = false; $call->getConnection() ->transaction(function () use ( $call, @@ -1820,12 +1949,14 @@ private function markCompleted( $runId, $updateId, $metadata, + &$recorded, ): void { /** @var WorkflowServiceCall $locked */ $locked = ConfiguredV2Models::query('service_call_model', WorkflowServiceCall::class) ->lockForUpdate() ->findOrFail($call->id); - if (ServiceCallDeadline::expireLocked($locked, now()) || $this->isTerminal($locked)) { + if (! $this->scheduledClaimOwnsLockedCall($locked) + || ServiceCallDeadline::expireLocked($locked, now()) || $this->isTerminal($locked)) { return; } $call = $locked; @@ -1847,8 +1978,11 @@ private function markCompleted( $call->failure_message = null; $call->save(); ServiceCallWorkflowDelivery::enqueue($call); + $recorded = true; }); $call->refresh(); + + return $recorded; } /** @@ -1878,6 +2012,15 @@ private function markHandlerFailure( $locked = ConfiguredV2Models::query('service_call_model', WorkflowServiceCall::class) ->lockForUpdate() ->findOrFail($call->id); + if (! $this->scheduledClaimOwnsLockedCall($locked)) { + return [ + 'accepted' => false, + 'kind' => $locked->resolved_binding_kind, + 'reason' => 'stale_service_call_claim', + 'message' => 'The scheduled service-call claim is no longer current.', + 'retry_scheduled' => false, + ]; + } if (ServiceCallDeadline::expireLocked($locked, now()) || $this->isTerminal($locked)) { return [ 'accepted' => false, @@ -2042,6 +2185,13 @@ private function workflowStartOptions(WorkflowServiceCall $call, array $binding, $startOptions['payload_codec'] = $call->payload_codec; } + foreach (['service_call_id', 'service_call_attempt', 'service_call_max_attempts', + 'service_call_idempotency_key'] as $key) { + if (array_key_exists($key, $options)) { + $startOptions[$key] = $options[$key]; + } + } + return $startOptions; } @@ -2061,7 +2211,9 @@ private function workflowCommandOptions(WorkflowServiceCall $call, array $bindin $commandOptions['arguments'] = $arguments; } - foreach (['payload_codec', 'payload_blob', 'wait_for', 'wait_timeout_seconds'] as $key) { + foreach (['payload_codec', 'payload_blob', 'wait_for', 'wait_timeout_seconds', + 'service_call_id', 'service_call_attempt', 'service_call_max_attempts', + 'service_call_idempotency_key'] as $key) { $value = $options[$key] ?? $binding[$key] ?? null; if ($value !== null) { @@ -2072,6 +2224,26 @@ private function workflowCommandOptions(WorkflowServiceCall $call, array $bindin return $commandOptions; } + private function scheduledClaimOwnsLockedCall(WorkflowServiceCall $call): bool + { + if (! $this->scheduledClaim instanceof ServiceCallAttemptClaim) { + return true; + } + if ($call->id !== $this->scheduledClaim->call->id) { + return false; + } + + /** @var \Workflow\V2\Models\WorkflowServiceCallDispatch|null $dispatch */ + $dispatch = ConfiguredV2Models::query( + 'service_call_dispatch_model', + \Workflow\V2\Models\WorkflowServiceCallDispatch::class, + )->lockForUpdate() + ->find($call->id); + + return $dispatch instanceof \Workflow\V2\Models\WorkflowServiceCallDispatch + && ServiceCallScheduler::ownsLockedClaim($dispatch, $this->scheduledClaim); + } + /** * @param array $result * @return array diff --git a/src/V2/Support/DefaultWorkflowTaskBridge.php b/src/V2/Support/DefaultWorkflowTaskBridge.php index 469de9969..eaffa04d7 100644 --- a/src/V2/Support/DefaultWorkflowTaskBridge.php +++ b/src/V2/Support/DefaultWorkflowTaskBridge.php @@ -3647,6 +3647,7 @@ private function applyStartServiceOperation( $sequence, $command, $surface, + $eventType, $requestPayload, $payloadCodec, ), $task); @@ -3779,6 +3780,10 @@ private function defaultServiceOperationIdempotencyKey(WorkflowRun $run, int $se */ private static function serviceOperationEventTypeForSurface(array $surface): HistoryEventType { + if (($surface['retry_scheduled'] ?? false) === true) { + return HistoryEventType::ServiceOperationScheduled; + } + $status = self::nonEmptyString($surface['status'] ?? null); return match ($status) { @@ -3800,10 +3805,19 @@ private static function serviceOperationEventIsWorkflowVisible( array $command, array $surface, ): bool { - if ($eventType !== HistoryEventType::ServiceCallStarted) { + if (! in_array($eventType, [ + HistoryEventType::ServiceOperationScheduled, + HistoryEventType::ServiceCallStarted, + ], true)) { return true; } + if ($eventType === HistoryEventType::ServiceOperationScheduled) { + return self::nonEmptyString($command['wait_for'] ?? null) === 'accepted' + || self::nonEmptyString($command['mode_override'] ?? null) === 'async' + || self::nonEmptyString($surface['operation_mode'] ?? null) === 'async'; + } + return self::nonEmptyString($command['wait_for'] ?? null) === 'accepted' || self::nonEmptyString($command['mode_override'] ?? null) === 'async' || self::nonEmptyString($surface['wait_for'] ?? null) === 'accepted' @@ -3820,6 +3834,7 @@ private function serviceOperationEventPayload( int $sequence, array $command, array $surface, + HistoryEventType $eventType, ?string $requestPayload, string $payloadCodec, ): array { @@ -3850,6 +3865,9 @@ private function serviceOperationEventPayload( ?? self::nonEmptyString($command['mode_override'] ?? null), 'wait_for' => self::nonEmptyString($surface['wait_for'] ?? null) ?? self::nonEmptyString($command['wait_for'] ?? null), + 'workflow_resume_on_admission' => $eventType === HistoryEventType::ServiceOperationScheduled + ? self::serviceOperationEventIsWorkflowVisible($eventType, $command, $surface) + : null, 'status' => self::nonEmptyString($surface['status'] ?? null), 'outcome' => self::nonEmptyString($surface['outcome'] ?? null), 'resolved_binding_kind' => self::nonEmptyString($surface['resolved_binding_kind'] ?? null), @@ -6697,6 +6715,11 @@ private static function normalizeStartServiceOperationCommand(array $command): ? return null; } + if ($waitTimeoutSeconds !== null + && $waitTimeoutSeconds > UpdateWaitPolicy::MAX_COMPLETION_TIMEOUT_SECONDS) { + return null; + } + return array_filter([ 'type' => 'start_service_operation', 'endpoint_name' => $endpointName, diff --git a/src/V2/Support/EmbeddedV2HistoryImport.php b/src/V2/Support/EmbeddedV2HistoryImport.php index b5819988c..7eb51c997 100644 --- a/src/V2/Support/EmbeddedV2HistoryImport.php +++ b/src/V2/Support/EmbeddedV2HistoryImport.php @@ -52,6 +52,7 @@ final class EmbeddedV2HistoryImport 'ActivityCompleted' => ['result'], 'ChildRunCompleted' => ['output', 'result'], 'MemoUpserted' => ['entries', 'merged'], + 'ServiceOperationScheduled' => ['request_payload', 'response_payload'], 'ServiceCallStarted' => ['request_payload', 'response_payload'], 'ServiceCallCompleted' => ['request_payload', 'response_payload'], 'ServiceCallFailed' => ['request_payload', 'response_payload'], @@ -97,6 +98,7 @@ final class EmbeddedV2HistoryImport * @var list */ private const SERVICE_CALL_EVENT_TYPES = [ + 'ServiceOperationScheduled', 'ServiceCallStarted', 'ServiceCallCompleted', 'ServiceCallFailed', diff --git a/src/V2/Support/HistoryEventPayloadContract.php b/src/V2/Support/HistoryEventPayloadContract.php index f4e54b8f8..dab941828 100644 --- a/src/V2/Support/HistoryEventPayloadContract.php +++ b/src/V2/Support/HistoryEventPayloadContract.php @@ -748,6 +748,34 @@ final class HistoryEventPayloadContract 'parallel_group_index', 'parallel_group_path', ], + 'ServiceOperationScheduled' => [ + 'sequence', + 'service_call_id', + 'workflow_instance_id', + 'workflow_run_id', + 'caller_workflow_instance_id', + 'caller_workflow_run_id', + 'caller_sdk_language', + 'endpoint_name', + 'service_name', + 'operation_name', + 'service_sdk_language', + 'request_payload', + 'response_payload', + 'payload_codec', + 'operation_mode', + 'wait_for', + 'workflow_resume_on_admission', + 'status', + 'outcome', + 'resolved_binding_kind', + 'resolved_target_reference', + 'linked_workflow_instance_id', + 'linked_workflow_run_id', + 'linked_workflow_update_id', + 'service_call', + 'response_or_failure_surface', + ], 'ServiceCallStarted' => [ 'sequence', 'service_call_id', @@ -765,6 +793,7 @@ final class HistoryEventPayloadContract 'payload_codec', 'operation_mode', 'wait_for', + 'workflow_resume_on_admission', 'status', 'outcome', 'resolved_binding_kind', @@ -792,6 +821,7 @@ final class HistoryEventPayloadContract 'payload_codec', 'operation_mode', 'wait_for', + 'workflow_resume_on_admission', 'status', 'outcome', 'resolved_binding_kind', @@ -819,6 +849,7 @@ final class HistoryEventPayloadContract 'payload_codec', 'operation_mode', 'wait_for', + 'workflow_resume_on_admission', 'status', 'outcome', 'resolved_binding_kind', @@ -851,6 +882,7 @@ final class HistoryEventPayloadContract 'payload_codec', 'operation_mode', 'wait_for', + 'workflow_resume_on_admission', 'status', 'outcome', 'resolved_binding_kind', diff --git a/src/V2/Support/HistoryTimeline.php b/src/V2/Support/HistoryTimeline.php index 6519357ba..359edfcaf 100644 --- a/src/V2/Support/HistoryTimeline.php +++ b/src/V2/Support/HistoryTimeline.php @@ -436,6 +436,7 @@ private static function kindFor(HistoryEventType $eventType): string HistoryEventType::ChildCancellationRequested, HistoryEventType::ChildCancellationResolved, HistoryEventType::ChildRunTerminated => 'child', + HistoryEventType::ServiceOperationScheduled, HistoryEventType::ServiceCallStarted, HistoryEventType::ServiceCallCompleted, HistoryEventType::ServiceCallFailed, @@ -536,6 +537,10 @@ private static function summaryFor( : sprintf('Child workflow %s failed: %s.', $childLabel, $message), HistoryEventType::ChildRunCancelled => sprintf('Child workflow %s cancelled.', $childLabel), HistoryEventType::ChildRunTerminated => sprintf('Child workflow %s terminated.', $childLabel), + HistoryEventType::ServiceOperationScheduled => sprintf( + 'Service operation %s scheduled.', + self::stringValue($payload['operation_name'] ?? null) ?? 'unknown', + ), HistoryEventType::ServiceCallStarted => sprintf( 'Service operation %s started.', self::stringValue($payload['operation_name'] ?? null) ?? 'unknown', @@ -1282,6 +1287,7 @@ private static function sourceKindFor(WorkflowHistoryEvent $event): string HistoryEventType::ChildRunFailed, HistoryEventType::ChildRunCancelled, HistoryEventType::ChildRunTerminated => 'child_workflow_run', + HistoryEventType::ServiceOperationScheduled, HistoryEventType::ServiceCallStarted, HistoryEventType::ServiceCallCompleted, HistoryEventType::ServiceCallFailed, diff --git a/src/V2/Support/QueryStateReplayer.php b/src/V2/Support/QueryStateReplayer.php index 8437413c2..54f840ad9 100644 --- a/src/V2/Support/QueryStateReplayer.php +++ b/src/V2/Support/QueryStateReplayer.php @@ -491,7 +491,10 @@ public function replayState(WorkflowRun $run): ReplayState } if ( - $serviceEvent->event_type === HistoryEventType::ServiceCallStarted + in_array($serviceEvent->event_type, [ + HistoryEventType::ServiceOperationScheduled, + HistoryEventType::ServiceCallStarted, + ], true) && ! self::serviceOperationStartedEventIsVisible($serviceEvent, $current) ) { $this->syncWorkflowCursor($workflow, $sequence + 1); @@ -1750,6 +1753,7 @@ private function serviceOperationEvent(WorkflowRun $run, int $sequence): ?Workfl static fn (WorkflowHistoryEvent $event): bool => in_array( $event->event_type, [ + HistoryEventType::ServiceOperationScheduled, HistoryEventType::ServiceCallStarted, HistoryEventType::ServiceCallCompleted, HistoryEventType::ServiceCallFailed, @@ -1821,7 +1825,10 @@ private static function serviceOperationStartedEventIsVisible( WorkflowHistoryEvent $event, ServiceOperationCall $call, ): bool { - if ($event->event_type !== HistoryEventType::ServiceCallStarted) { + if (! in_array($event->event_type, [ + HistoryEventType::ServiceOperationScheduled, + HistoryEventType::ServiceCallStarted, + ], true)) { return true; } @@ -1829,6 +1836,13 @@ private static function serviceOperationStartedEventIsVisible( ? $event->payload['service_call'] : []; + if ($event->event_type === HistoryEventType::ServiceOperationScheduled) { + return ($event->payload['workflow_resume_on_admission'] ?? null) === true + || ($call->options?->shouldResumeOnAdmission() ?? false) + || self::nonEmptyString($event->payload['operation_mode'] ?? null) === 'async' + || self::nonEmptyString($surface['operation_mode'] ?? null) === 'async'; + } + return ($call->options?->shouldResumeOnAdmission() ?? false) || self::nonEmptyString($event->payload['wait_for'] ?? null) === 'accepted' || self::nonEmptyString($event->payload['operation_mode'] ?? null) === 'async' diff --git a/src/V2/Support/ServiceCallAttemptClaim.php b/src/V2/Support/ServiceCallAttemptClaim.php new file mode 100644 index 000000000..762ad225f --- /dev/null +++ b/src/V2/Support/ServiceCallAttemptClaim.php @@ -0,0 +1,28 @@ + $options + */ + public function __construct( + public readonly WorkflowServiceCall $call, + public readonly WorkflowServiceOperation $operation, + public readonly ServiceBoundaryRequest $request, + public readonly array $options, + public readonly int $attempt, + public readonly int $maxAttempts, + public readonly int $generation, + public readonly string $token, + public readonly string $catalogRevision, + public readonly string $policyRevision, + ) { + } +} diff --git a/src/V2/Support/ServiceCallCancellationOrigin.php b/src/V2/Support/ServiceCallCancellationOrigin.php index 98b7721d5..1878a00be 100644 --- a/src/V2/Support/ServiceCallCancellationOrigin.php +++ b/src/V2/Support/ServiceCallCancellationOrigin.php @@ -79,14 +79,17 @@ public static function validateOwnership( $caller->loadMissing('historyEvents'); } $started = $caller->historyEvents->first(static fn (WorkflowHistoryEvent $event): bool => - $event->event_type === HistoryEventType::ServiceCallStarted + in_array($event->event_type, [ + HistoryEventType::ServiceOperationScheduled, + HistoryEventType::ServiceCallStarted, + ], true) && $event->workflow_run_id === $caller->id && ($event->payload['service_call_id'] ?? null) === $call->id); if (! $started instanceof WorkflowHistoryEvent || ! is_int($started->payload['sequence'] ?? null) || $started->payload['sequence'] < 1 || ($started->payload['operation_mode'] ?? null) !== 'sync' - || ($started->payload['wait_for'] ?? null) !== 'completed') { + || ! self::waitsForCompletion($started)) { return 'nexus_cancellation_original_admission_missing'; } @@ -105,4 +108,13 @@ public static function validateTargetCreation(WorkflowServiceCall $call): ?strin return null; } + + private static function waitsForCompletion(WorkflowHistoryEvent $event): bool + { + if ($event->event_type === HistoryEventType::ServiceOperationScheduled) { + return ($event->payload['workflow_resume_on_admission'] ?? null) === false; + } + + return ($event->payload['wait_for'] ?? null) === 'completed'; + } } diff --git a/src/V2/Support/ServiceCallScheduler.php b/src/V2/Support/ServiceCallScheduler.php new file mode 100644 index 000000000..c8fd049ba --- /dev/null +++ b/src/V2/Support/ServiceCallScheduler.php @@ -0,0 +1,709 @@ + $options + */ + public static function schedule( + WorkflowServiceCall $call, + WorkflowServiceOperation $operation, + array $options, + int $maxAttempts, + ): void { + self::assertSharedConnection($call); + $dispatch = self::dispatchQuery() + ->lockForUpdate() + ->find($call->id); + + if (! $dispatch instanceof WorkflowServiceCallDispatch) { + /** @var WorkflowServiceCallDispatch $dispatch */ + $dispatch = self::dispatchQuery()->create([ + 'service_call_id' => $call->id, + 'state' => self::STATE_SCHEDULED, + 'dispatch_options' => self::persistableOptions($operation, $options), + 'attempt' => 1, + 'max_attempts' => max(1, $maxAttempts), + 'available_at' => now(), + 'claim_generation' => 0, + ]); + } + + if ($dispatch->state === self::STATE_COMPLETED) { + return; + } + + $connection = $call->getConnection(); + $connection->afterCommit(static function () use ($call): void { + self::enqueue($call->id); + }); + } + + public static function enqueue(string $serviceCallId): bool + { + $availableAt = self::withLockedDispatch($serviceCallId, static function ( + WorkflowServiceCall $call, + WorkflowServiceCallDispatch $dispatch, + ): ?CarbonInterface { + if (self::terminal($call) || $dispatch->state === self::STATE_COMPLETED) { + self::completeDispatch($dispatch); + + return null; + } + + if ($dispatch->state === self::STATE_CLAIMED + && $dispatch->claim_expires_at !== null + && $dispatch->claim_expires_at->isFuture()) { + return null; + } + + $now = now(); + $dispatch->forceFill([ + 'dispatch_attempted_at' => $now, + 'repair_available_at' => $now->copy() + ->addSeconds(self::repairAfterSeconds()), + ])->save(); + + return $dispatch->available_at; + }); + + if (! $availableAt instanceof CarbonInterface) { + return false; + } + + $job = RunServiceCall::dispatch($serviceCallId); + if ($availableAt->isFuture()) { + $job->delay($availableAt); + } + + return true; + } + + public static function claim(string $serviceCallId): ?ServiceCallAttemptClaim + { + $callModel = ConfiguredV2Models::query('service_call_model', WorkflowServiceCall::class)->getModel(); + $connection = $callModel->getConnection(); + + return $connection->transaction(static function () use ($serviceCallId): ?ServiceCallAttemptClaim { + /** @var WorkflowServiceCall|null $call */ + $call = ConfiguredV2Models::query('service_call_model', WorkflowServiceCall::class) + ->lockForUpdate() + ->find($serviceCallId); + if (! $call instanceof WorkflowServiceCall) { + return null; + } + self::assertSharedConnection($call); + + /** @var WorkflowServiceCallDispatch|null $dispatch */ + $dispatch = self::dispatchQuery() + ->lockForUpdate() + ->find($serviceCallId); + if (! $dispatch instanceof WorkflowServiceCallDispatch || $dispatch->state === self::STATE_COMPLETED) { + return null; + } + if (self::terminal($call)) { + self::completeDispatch($dispatch); + + return null; + } + if (ServiceCallDeadline::expireLocked($call, now())) { + self::completeDispatch($dispatch); + + return null; + } + if ($dispatch->available_at->isFuture()) { + return null; + } + if ($dispatch->state === self::STATE_CLAIMED + && ($dispatch->claim_expires_at === null || $dispatch->claim_expires_at->isFuture())) { + return null; + } + + $run = self::lockedCallerRun($call); + if (! $run instanceof WorkflowRun + || $run->status->isTerminal() + || is_string($run->cancellation_request_command_id)) { + self::cancelForTerminalCaller($call, $dispatch, $run); + + return null; + } + + try { + [$endpoint, $service, $operation] = self::lockedCatalog($call); + $principal = self::acceptedPrincipal($call); + $scope = [ + 'namespace' => $call->target_namespace, + 'caller_namespace' => $run->namespace, + 'caller_workflow_instance_id' => $run->workflow_instance_id, + 'caller_workflow_run_id' => $run->id, + 'endpoint_name' => $call->endpoint_name, + 'service_name' => $call->service_name, + 'operation_name' => $call->operation_name, + ]; + WorkflowServiceOperationAuthority::authorizeExistingCall($call, $principal, $scope); + } catch (ServiceOperationAuthorizationException $failure) { + self::failAuthorization($call, $dispatch, $failure->getMessage()); + + return null; + } + + $catalogRevision = self::revision([ + 'endpoint' => self::catalogRevisionFields($endpoint), + 'service' => self::catalogRevisionFields($service), + 'operation' => self::catalogRevisionFields($operation), + ]); + $policyRevision = self::revision([ + 'endpoint' => $endpoint->boundary_policy, + 'service' => $service->boundary_policy, + 'operation' => $operation->boundary_policy, + ]); + $generation = (int) $dispatch->claim_generation + 1; + $token = hash('sha256', (string) Str::uuid()); + $now = now(); + $dispatch->forceFill([ + 'state' => self::STATE_CLAIMED, + 'claim_generation' => $generation, + 'claim_token' => $token, + 'claimed_at' => $now, + 'claim_expires_at' => $now->copy() + ->addSeconds(self::claimLeaseSeconds()), + 'authorized_catalog_revision' => $catalogRevision, + 'authorized_policy_revision' => $policyRevision, + 'repair_available_at' => null, + 'last_error' => null, + ])->save(); + + $options = is_array($dispatch->dispatch_options) ? $dispatch->dispatch_options : []; + $options['service_call_id'] = $call->id; + $options['service_call_attempt'] = $dispatch->attempt; + $options['service_call_max_attempts'] = $dispatch->max_attempts; + $options['service_call_idempotency_key'] = self::attemptIdentity($call->id, $dispatch->attempt); + + $effectOperation = clone $operation; + $effectOperation->forceFill([ + 'handler_binding_kind' => $options['accepted_handler_binding_kind'] ?? null, + 'handler_binding' => $options['accepted_handler_binding'] ?? null, + 'handler_target_reference' => $options['accepted_handler_target_reference'] ?? null, + 'operation_mode' => $options['accepted_operation_mode'] ?? $call->operation_mode, + ]); + + return new ServiceCallAttemptClaim( + call: $call, + operation: $effectOperation, + request: self::boundaryRequest($call, $principal, $endpoint, $service, $operation), + options: $options, + attempt: $dispatch->attempt, + maxAttempts: $dispatch->max_attempts, + generation: $generation, + token: $token, + catalogRevision: $catalogRevision, + policyRevision: $policyRevision, + ); + }); + } + + /** + * @param array $attemptEntry + */ + public static function completeAttempt( + ServiceCallAttemptClaim $claim, + array $attemptEntry, + bool $retry, + ?float $backoffSeconds, + ): bool { + $reenqueue = false; + $recorded = self::withLockedDispatch($claim->call->id, static function ( + WorkflowServiceCall $call, + WorkflowServiceCallDispatch $dispatch, + ) use ($claim, $attemptEntry, $retry, $backoffSeconds, &$reenqueue): bool { + if (! self::ownsLockedClaim($dispatch, $claim)) { + return false; + } + + $metadata = is_array($call->metadata) ? $call->metadata : []; + $attempts = isset($metadata['service_call_attempts']) && is_array($metadata['service_call_attempts']) + ? array_values(array_filter($metadata['service_call_attempts'], 'is_array')) + : []; + $attempts[] = $attemptEntry; + $metadata['service_call_attempts'] = $attempts; + $metadata['retry_attempt_count'] = count($attempts); + $call->metadata = $metadata; + if ($call->status === ServiceCallStatus::Failed->value) { + $outcomeMetadata = is_array($call->outcome_metadata) ? $call->outcome_metadata : []; + $outcomeMetadata['service_call_attempts'] = $attempts; + $outcomeMetadata['retry_attempt_count'] = count($attempts); + $call->outcome_metadata = $outcomeMetadata; + } + + if ($retry && ! self::terminal($call) && $dispatch->attempt < $dispatch->max_attempts) { + $availableAt = now() + ->addMilliseconds((int) ceil(max(0.0, $backoffSeconds ?? 0.0) * 1000)); + $dispatch->forceFill([ + 'state' => self::STATE_SCHEDULED, + 'attempt' => $dispatch->attempt + 1, + 'available_at' => $availableAt, + 'claim_token' => null, + 'claimed_at' => null, + 'claim_expires_at' => null, + 'dispatch_attempted_at' => null, + 'repair_available_at' => null, + 'completed_at' => null, + ])->save(); + $reenqueue = true; + } else { + self::completeDispatch($dispatch); + } + + $call->save(); + + return true; + }); + + if ($recorded === true && $reenqueue) { + self::enqueue($claim->call->id); + } + + return $recorded === true; + } + + public static function ownsLockedClaim( + WorkflowServiceCallDispatch $dispatch, + ServiceCallAttemptClaim $claim, + ): bool { + return $dispatch->state === self::STATE_CLAIMED + && $dispatch->claim_generation === $claim->generation + && hash_equals((string) $dispatch->claim_token, $claim->token); + } + + public static function ownsClaim(ServiceCallAttemptClaim $claim): bool + { + return self::withLockedDispatch($claim->call->id, static function ( + WorkflowServiceCall $call, + WorkflowServiceCallDispatch $dispatch, + ) use ($claim): bool { + return ! self::terminal($call) && self::ownsLockedClaim($dispatch, $claim); + }) === true; + } + + /** + * @return list + */ + public static function dueIds( + int $limit, + array $runIds = [], + ?string $instanceId = null, + ?string $connection = null, + ?string $queue = null, + ): array { + $now = now() + ->utc() + ->format('Y-m-d H:i:s.u'); + + $query = self::dispatchQuery() + ->where(static function ($query) use ($now): void { + $query->where(static function ($scheduled) use ($now): void { + $scheduled->where('state', self::STATE_SCHEDULED) + ->where('available_at', '<=', $now) + ->where(static function ($repair) use ($now): void { + $repair->whereNull('repair_available_at') + ->orWhere('repair_available_at', '<=', $now); + }); + })->orWhere(static function ($claimed) use ($now): void { + $claimed->where('state', self::STATE_CLAIMED) + ->whereNotNull('claim_expires_at') + ->where('claim_expires_at', '<=', $now); + }); + }); + if ($runIds !== []) { + $query->whereHas('call', static fn ($call) => $call->whereIn('caller_workflow_run_id', $runIds)); + } + if ($instanceId !== null) { + $query->whereHas( + 'call', + static fn ($call) => $call->where('caller_workflow_instance_id', $instanceId), + ); + } + if ($connection !== null || $queue !== null) { + $query->whereHas('call.callerRun', static function ($run) use ($connection, $queue): void { + if ($connection !== null) { + $run->where('connection', $connection); + } + if ($queue !== null) { + $run->where('queue', $queue); + } + }); + } + + return $query + ->orderBy('available_at') + ->orderBy('service_call_id') + ->limit(max(1, $limit)) + ->pluck('service_call_id') + ->all(); + } + + /** + * @return array{candidates: int, enqueued: int, failures: list} + */ + public static function repairDue( + int $limit, + array $runIds = [], + ?string $instanceId = null, + ?string $connection = null, + ?string $queue = null, + ): array { + $ids = self::dueIds($limit, $runIds, $instanceId, $connection, $queue); + $report = [ + 'candidates' => count($ids), + 'enqueued' => 0, + 'failures' => [], + ]; + foreach ($ids as $id) { + try { + if (self::enqueue($id)) { + $report['enqueued']++; + } + } catch (Throwable $failure) { + report($failure); + $report['failures'][] = [ + 'call_id' => $id, + 'message' => $failure->getMessage(), + ]; + } + } + + return $report; + } + + public static function attemptIdentity(string $serviceCallId, int $attempt): string + { + return sprintf('workflow-service-call:%s:attempt:%d', $serviceCallId, $attempt); + } + + private static function lockedCallerRun(WorkflowServiceCall $call): ?WorkflowRun + { + if (! is_string($call->caller_workflow_run_id) || $call->caller_workflow_run_id === '') { + return null; + } + + /** @var WorkflowRun|null $run */ + $run = ConfiguredV2Models::query('run_model', WorkflowRun::class) + ->lockForUpdate() + ->find($call->caller_workflow_run_id); + if (! $run instanceof WorkflowRun + || $run->id !== $call->caller_workflow_run_id + || $run->workflow_instance_id !== $call->caller_workflow_instance_id + || $run->namespace !== $call->caller_namespace) { + return null; + } + + return $run; + } + + /** + * @return array{WorkflowServiceEndpoint, WorkflowService, WorkflowServiceOperation} + */ + private static function lockedCatalog(WorkflowServiceCall $call): array + { + /** @var WorkflowServiceEndpoint|null $endpoint */ + $endpoint = ConfiguredV2Models::query('service_endpoint_model', WorkflowServiceEndpoint::class) + ->lockForUpdate() + ->find($call->workflow_service_endpoint_id); + /** @var WorkflowService|null $service */ + $service = ConfiguredV2Models::query('service_model', WorkflowService::class) + ->lockForUpdate() + ->find($call->workflow_service_id); + /** @var WorkflowServiceOperation|null $operation */ + $operation = ConfiguredV2Models::query('service_operation_model', WorkflowServiceOperation::class) + ->lockForUpdate() + ->find($call->workflow_service_operation_id); + if (! $endpoint instanceof WorkflowServiceEndpoint + || ! $service instanceof WorkflowService + || ! $operation instanceof WorkflowServiceOperation) { + throw new ServiceOperationAuthorizationException('service_call_not_owned'); + } + + return [$endpoint, $service, $operation]; + } + + private static function acceptedPrincipal(WorkflowServiceCall $call): ServiceCallPrincipal + { + if (! is_string($call->caller_principal_subject) || trim($call->caller_principal_subject) === '' + || ! is_string($call->caller_principal_method) || trim($call->caller_principal_method) === '') { + throw new ServiceOperationAuthorizationException('service_operation_replay_authority_incomplete'); + } + if ($call->caller_principal_roles !== null && ! is_array($call->caller_principal_roles)) { + throw new ServiceOperationAuthorizationException('service_operation_replay_authority_incomplete'); + } + if ($call->caller_principal_claims !== null && ! is_array($call->caller_principal_claims)) { + throw new ServiceOperationAuthorizationException('service_operation_replay_authority_incomplete'); + } + + return ServiceCallPrincipal::fromAuditArray([ + 'subject' => $call->caller_principal_subject, + 'method' => $call->caller_principal_method, + 'roles' => $call->caller_principal_roles, + 'tenant' => $call->caller_principal_tenant, + 'claims' => $call->caller_principal_claims, + ]); + } + + private static function boundaryRequest( + WorkflowServiceCall $call, + ServiceCallPrincipal $principal, + WorkflowServiceEndpoint $endpoint, + WorkflowService $service, + WorkflowServiceOperation $operation, + ): ServiceBoundaryRequest { + return new ServiceBoundaryRequest( + principal: $principal, + callerNamespace: $call->caller_namespace, + targetNamespace: $call->target_namespace ?? '', + endpointName: $call->endpoint_name, + serviceName: $call->service_name, + operationName: $call->operation_name, + operationMode: ServiceCallOperationMode::tryFromCatalog($call->operation_mode) + ?? ServiceCallOperationMode::Async, + resolvedBindingKind: $call->resolved_binding_kind, + resolvedTargetReference: $call->resolved_target_reference, + callerWorkflowInstanceId: $call->caller_workflow_instance_id, + callerWorkflowRunId: $call->caller_workflow_run_id, + linkedWorkflowInstanceId: $call->linked_workflow_instance_id, + linkedWorkflowRunId: $call->linked_workflow_run_id, + linkedWorkflowUpdateId: $call->linked_workflow_update_id, + idempotencyKey: $call->idempotency_key, + endpointBoundaryPolicy: is_array($endpoint->boundary_policy) ? $endpoint->boundary_policy : [], + serviceBoundaryPolicy: is_array($service->boundary_policy) ? $service->boundary_policy : [], + operationBoundaryPolicy: is_array($operation->boundary_policy) ? $operation->boundary_policy : [], + deadlinePolicy: is_array($call->deadline_policy) ? $call->deadline_policy : null, + idempotencyPolicy: is_array($call->idempotency_policy) ? $call->idempotency_policy : null, + cancellationPolicy: is_array($call->cancellation_policy) ? $call->cancellation_policy : null, + retryPolicy: is_array($call->retry_policy) ? $call->retry_policy : null, + ); + } + + private static function cancelForTerminalCaller( + WorkflowServiceCall $call, + WorkflowServiceCallDispatch $dispatch, + ?WorkflowRun $run, + ): void { + $cancellationRequested = $run instanceof WorkflowRun + && is_string($run->cancellation_request_command_id); + $reason = $cancellationRequested ? 'caller_cancellation_requested' : 'caller_run_terminal'; + $message = $cancellationRequested + ? 'Caller workflow cancellation was requested before the service-call attempt was claimed.' + : ($run instanceof WorkflowRun + ? sprintf('Caller workflow run is terminal (%s).', $run->status->value) + : 'Caller workflow authority is unavailable.'); + $call->forceFill([ + 'status' => ServiceCallStatus::Cancelled->value, + 'outcome' => ServiceCallOutcome::Cancelled->value, + 'outcome_category' => ServiceCallOutcome::Cancelled->category(), + 'outcome_reason' => $reason, + 'outcome_message' => $message, + 'failure_message' => $message, + 'outcome_metadata' => [ + 'failure_reason' => ServiceCallFailureReason::Cancellation->value, + 'caller_run_status' => $run?->status->value, + 'cancellation_request_command_id' => $cancellationRequested + ? $run->cancellation_request_command_id + : null, + ], + 'cancelled_at' => now(), + ])->save(); + self::completeDispatch($dispatch); + if (! $cancellationRequested) { + ServiceCallWorkflowDelivery::enqueue($call); + } + } + + private static function failAuthorization( + WorkflowServiceCall $call, + WorkflowServiceCallDispatch $dispatch, + string $reason, + ): void { + $reason = trim($reason) !== '' ? $reason : 'service_operation_authority_invalid'; + $call->forceFill([ + 'status' => ServiceCallStatus::Failed->value, + 'outcome' => ServiceCallOutcome::RejectedForbidden->value, + 'outcome_category' => ServiceCallOutcome::RejectedForbidden->category(), + 'outcome_reason' => $reason, + 'outcome_message' => $reason, + 'failure_message' => $reason, + 'outcome_metadata' => [ + 'failure_reason' => ServiceCallFailureReason::PolicyRejection->value, + ], + 'failed_at' => now(), + ])->save(); + $dispatch->last_error = $reason; + self::completeDispatch($dispatch); + ServiceCallWorkflowDelivery::enqueue($call); + } + + private static function completeDispatch(WorkflowServiceCallDispatch $dispatch): void + { + $dispatch->forceFill([ + 'state' => self::STATE_COMPLETED, + 'claim_token' => null, + 'claimed_at' => null, + 'claim_expires_at' => null, + 'repair_available_at' => null, + 'completed_at' => $dispatch->completed_at ?? now(), + ])->save(); + } + + private static function terminal(WorkflowServiceCall $call): bool + { + return in_array($call->status, [ + ServiceCallStatus::Completed->value, + ServiceCallStatus::Failed->value, + ServiceCallStatus::Cancelled->value, + ], true); + } + + private static function assertSharedConnection(WorkflowServiceCall $call): void + { + $dispatch = self::dispatchQuery()->getModel(); + if ($dispatch->getConnection() !== $call->getConnection()) { + throw new \RuntimeException('service_operation_storage_connection_mismatch'); + } + } + + /** + * @param callable(WorkflowServiceCall, WorkflowServiceCallDispatch): mixed $callback + */ + private static function withLockedDispatch(string $serviceCallId, callable $callback): mixed + { + $callModel = ConfiguredV2Models::query('service_call_model', WorkflowServiceCall::class)->getModel(); + + return $callModel->getConnection() + ->transaction(static function () use ($serviceCallId, $callback): mixed { + /** @var WorkflowServiceCall|null $call */ + $call = ConfiguredV2Models::query('service_call_model', WorkflowServiceCall::class) + ->lockForUpdate() + ->find($serviceCallId); + if (! $call instanceof WorkflowServiceCall) { + return null; + } + self::assertSharedConnection($call); + /** @var WorkflowServiceCallDispatch|null $dispatch */ + $dispatch = self::dispatchQuery() + ->lockForUpdate() + ->find($serviceCallId); + if (! $dispatch instanceof WorkflowServiceCallDispatch) { + return null; + } + + return $callback($call, $dispatch); + }); + } + + /** + * @param array $options + * @return array + */ + private static function persistableOptions(WorkflowServiceOperation $operation, array $options): array + { + $persisted = []; + foreach ([ + 'arguments', 'payload_blob', 'payload_codec', 'target_workflow_instance_id', + 'target_workflow_run_id', 'workflow_instance_id', 'connection', 'queue', 'business_key', + 'labels', 'memo', 'search_attributes', 'duplicate_start_policy', 'request_payload_reference', + 'activity_execution_id', 'owning_workflow_instance_id', 'owning_workflow_run_id', + 'carrier_request_id', 'bound_workflow_instance_id', + ] as $key) { + $value = $options[$key] ?? null; + if (is_string($value) || is_int($value) || is_float($value) || is_bool($value) || is_array($value)) { + $persisted[$key] = $value; + } + } + $persisted['wait_for'] = UpdateWaitPolicy::WAIT_FOR_ACCEPTED; + $persisted['accepted_handler_binding_kind'] = $operation->handler_binding_kind; + $persisted['accepted_handler_binding'] = is_array($operation->handler_binding) + ? $operation->handler_binding + : []; + $persisted['accepted_handler_target_reference'] = $operation->handler_target_reference; + $persisted['accepted_operation_mode'] = $operation->operation_mode; + + return $persisted; + } + + /** + * @return array + */ + private static function catalogRevisionFields(object $model): array + { + return [ + 'id' => $model->getAttribute('id'), + 'updated_at' => $model->getRawOriginal('updated_at'), + 'boundary_policy' => $model->getAttribute('boundary_policy'), + 'handler_binding_kind' => $model->getAttribute('handler_binding_kind'), + 'handler_binding' => $model->getAttribute('handler_binding'), + 'handler_target_reference' => $model->getAttribute('handler_target_reference'), + ]; + } + + private static function revision(array $value): string + { + self::sortRecursive($value); + + return hash('sha256', json_encode($value, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES)); + } + + private static function sortRecursive(array &$value): void + { + if (! array_is_list($value)) { + ksort($value); + } + foreach ($value as &$child) { + if (is_array($child)) { + self::sortRecursive($child); + } + } + } + + private static function claimLeaseSeconds(): int + { + return max(1, (int) config('workflows.v2.service_call_dispatch.claim_lease_seconds', 60)); + } + + private static function repairAfterSeconds(): int + { + return max(1, (int) config('workflows.v2.service_call_dispatch.repair_after_seconds', 5)); + } + + /** + * @return Builder + */ + private static function dispatchQuery(): Builder + { + return ConfiguredV2Models::query('service_call_dispatch_model', WorkflowServiceCallDispatch::class); + } +} diff --git a/src/V2/Support/ServiceCallWorkflowDelivery.php b/src/V2/Support/ServiceCallWorkflowDelivery.php index 532aa6668..09e9ed742 100644 --- a/src/V2/Support/ServiceCallWorkflowDelivery.php +++ b/src/V2/Support/ServiceCallWorkflowDelivery.php @@ -18,7 +18,10 @@ final class ServiceCallWorkflowDelivery { public static function forAdmission(WorkflowRun $run, WorkflowHistoryEvent $event): void { - if ($event->event_type !== HistoryEventType::ServiceCallStarted + if (! in_array($event->event_type, [ + HistoryEventType::ServiceOperationScheduled, + HistoryEventType::ServiceCallStarted, + ], true) || ! is_string($event->payload['service_call_id'] ?? null)) { return; } @@ -65,7 +68,10 @@ public static function enqueue(WorkflowServiceCall $call): void $startedEvents = ConfiguredV2Models::query('history_event_model', WorkflowHistoryEvent::class) ->where('workflow_run_id', $caller->id) - ->where('event_type', HistoryEventType::ServiceCallStarted->value) + ->whereIn('event_type', [ + HistoryEventType::ServiceOperationScheduled->value, + HistoryEventType::ServiceCallStarted->value, + ]) ->where('payload->service_call_id', $locked->id) ->orderBy('sequence') ->get(); @@ -75,11 +81,13 @@ public static function enqueue(WorkflowServiceCall $call): void } $payload = $started->payload ?? []; $surface = is_array($payload['service_call'] ?? null) ? $payload['service_call'] : []; - if (! is_int($payload['sequence'] ?? null) - || ($payload['operation_mode'] ?? null) === 'async' - || ($payload['wait_for'] ?? null) === 'accepted' - || ($surface['operation_mode'] ?? null) === 'async' - || ($surface['wait_for'] ?? null) === 'accepted') { + $resumesOnAdmission = $started->event_type === HistoryEventType::ServiceOperationScheduled + ? ($payload['workflow_resume_on_admission'] ?? false) === true + : (($payload['operation_mode'] ?? null) === 'async' + || ($payload['wait_for'] ?? null) === 'accepted' + || ($surface['operation_mode'] ?? null) === 'async' + || ($surface['wait_for'] ?? null) === 'accepted'); + if (! is_int($payload['sequence'] ?? null) || $resumesOnAdmission) { continue; } @@ -151,7 +159,10 @@ public static function recordForClaim(WorkflowTask $task, WorkflowRun $run): voi /** @var WorkflowHistoryEvent|null $started */ $started = ConfiguredV2Models::query('history_event_model', WorkflowHistoryEvent::class) ->where('workflow_run_id', $run->id) - ->where('event_type', HistoryEventType::ServiceCallStarted->value) + ->whereIn('event_type', [ + HistoryEventType::ServiceOperationScheduled->value, + HistoryEventType::ServiceCallStarted->value, + ]) ->find($taskPayload['service_call_started_history_event_id']); if ($call === null || ! self::isTerminal($call) || $started === null || ($started->payload['service_call_id'] ?? null) !== $call->id diff --git a/src/V2/Support/ServiceExecutionContract.php b/src/V2/Support/ServiceExecutionContract.php index 1f00548a3..a9174e53a 100644 --- a/src/V2/Support/ServiceExecutionContract.php +++ b/src/V2/Support/ServiceExecutionContract.php @@ -12,6 +12,7 @@ use Workflow\V2\Enums\ServiceCallStatus; use Workflow\V2\Models\WorkflowService; use Workflow\V2\Models\WorkflowServiceCall; +use Workflow\V2\Models\WorkflowServiceCallDispatch; use Workflow\V2\Models\WorkflowServiceEndpoint; use Workflow\V2\Models\WorkflowServiceOperation; @@ -234,6 +235,20 @@ private static function durableRecords(): array 'target_cancellation_is_separate' => true, ], ], + 'dispatches' => [ + 'model' => WorkflowServiceCallDispatch::class, + 'table' => 'workflow_service_call_dispatches', + 'identity_fields' => ['service_call_id'], + 'states' => [ + ServiceCallScheduler::STATE_SCHEDULED, + ServiceCallScheduler::STATE_CLAIMED, + ServiceCallScheduler::STATE_COMPLETED, + ], + 'attempt_fields' => ['attempt', 'max_attempts', 'available_at'], + 'claim_fence_fields' => ['claim_generation', 'claim_token', 'claimed_at', 'claim_expires_at'], + 'authorization_revision_fields' => ['authorized_catalog_revision', 'authorized_policy_revision'], + 'repair_fields' => ['dispatch_attempted_at', 'repair_available_at', 'last_error'], + ], ]; } @@ -357,6 +372,15 @@ private static function executionRules(): array . 'handler binding before dispatching into the selected execution lane.', 'routing_boundary' => 'Service execution selects the durable capability; workflow and activity routing ' . 'remains owned by the selected execution lane.', + 'transaction_dispatch' => 'A workflow completion transaction persists the accepted call, immutable ' + . 'handler binding snapshot, dispatch row and ServiceOperationScheduled history before commit. ' + . 'It does not execute a handler or sleep for retry backoff.', + 'claim_authority' => 'Each runnable attempt locks the call, caller run and current catalog, reconstructs ' + . 'the accepted principal, reauthorizes current replay policy and records catalog and policy revisions.', + 'attempt_fence' => 'Handler effects execute outside the claim transaction with a stable per-attempt ' + . 'idempotency identity. Only the current generation and token may persist attempt progress.', + 'repair_delivery' => 'Queue delivery is at least once. Due schedules, lost enqueue attempts and expired ' + . 'claims remain durable repair candidates until a terminal compare-and-swap completes the dispatch.', 'transport_logs' => 'Raw transport logs are diagnostic only and are not required for lifecycle, ' . 'outcome, or observability state.', ]; diff --git a/src/V2/Support/ServiceOperationCancellation.php b/src/V2/Support/ServiceOperationCancellation.php index 69a1e188d..8235f7b87 100644 --- a/src/V2/Support/ServiceOperationCancellation.php +++ b/src/V2/Support/ServiceOperationCancellation.php @@ -6,6 +6,7 @@ use LogicException; use Workflow\V2\Enums\HistoryEventType; +use Workflow\V2\Enums\ServiceCallStatus; use Workflow\V2\Enums\TaskStatus; use Workflow\V2\Models\WorkflowHistoryEvent; use Workflow\V2\Models\WorkflowRun; @@ -26,9 +27,12 @@ public static function unsupportedReason(WorkflowRun $run, ?string $incomingRoot } foreach ($run->historyEvents as $event) { $payload = $event->payload; - if ($event->event_type !== HistoryEventType::ServiceCallStarted + if (! in_array($event->event_type, [ + HistoryEventType::ServiceOperationScheduled, + HistoryEventType::ServiceCallStarted, + ], true) || ($payload['operation_mode'] ?? null) !== 'sync' - || ($payload['wait_for'] ?? null) !== 'completed') { + || ! self::waitsForCompletion($event)) { continue; } $sequence = $payload['sequence'] ?? null; @@ -50,10 +54,13 @@ public static function unsupportedReason(WorkflowRun $run, ?string $incomingRoot ->where('namespace', $call->target_namespace) ->where('workflow_instance_id', $call->linked_workflow_instance_id) ->find($call->linked_workflow_run_id); - $reason = $call === null || $target === null ? 'nexus_cancellation_owner_unavailable' - : (ServiceCallCancellationOrigin::validateOwnership($call, $run, $target) - ?? ServiceCallCancellationOrigin::validateTargetCreation($call)); - if ($reason === null && ! $target->status->isTerminal() + $reason = $call === null ? 'nexus_cancellation_owner_unavailable' + : ($target === null && $call->status === ServiceCallStatus::Accepted->value + ? null + : ($target === null ? 'nexus_cancellation_owner_unavailable' + : (ServiceCallCancellationOrigin::validateOwnership($call, $run, $target) + ?? ServiceCallCancellationOrigin::validateTargetCreation($call)))); + if ($reason === null && $target instanceof WorkflowRun && ! $target->status->isTerminal() && is_string($target->cancellation_request_command_id) && CooperativeCancellationDelivery::context($target)?->rootRequestId !== $incomingRootRequestId) { $reason = 'nexus_cancellation_target_root_conflict'; @@ -92,6 +99,11 @@ public static function prepare(WorkflowRun $caller, WorkflowTask $task, int $seq ->where('namespace', $call->target_namespace) ->where('workflow_instance_id', $call->linked_workflow_instance_id) ->find($call->linked_workflow_run_id); + if ($target === null + && $call->status === ServiceCallStatus::Cancelled->value + && $call->outcome_reason === 'caller_cancellation_requested') { + return false; + } if ($context === null || $target === null) { throw new LogicException('nexus_cancellation_owner_unavailable'); } @@ -181,7 +193,10 @@ private static function callAt(WorkflowRun $run, int $sequence): ?WorkflowServic $run->loadMissing('historyEvents'); } $started = $run->historyEvents->first(static fn (WorkflowHistoryEvent $event): bool => - $event->event_type === HistoryEventType::ServiceCallStarted + in_array($event->event_type, [ + HistoryEventType::ServiceOperationScheduled, + HistoryEventType::ServiceCallStarted, + ], true) && ($event->payload['sequence'] ?? null) === $sequence); if ($started === null) { return null; @@ -198,4 +213,13 @@ private static function callAt(WorkflowRun $run, int $sequence): ?WorkflowServic return $call; } + + private static function waitsForCompletion(WorkflowHistoryEvent $event): bool + { + if ($event->event_type === HistoryEventType::ServiceOperationScheduled) { + return ($event->payload['workflow_resume_on_admission'] ?? null) === false; + } + + return ($event->payload['wait_for'] ?? null) === 'completed'; + } } diff --git a/src/V2/Support/ServiceOperationOptions.php b/src/V2/Support/ServiceOperationOptions.php index 40b456968..012f86fa5 100644 --- a/src/V2/Support/ServiceOperationOptions.php +++ b/src/V2/Support/ServiceOperationOptions.php @@ -70,6 +70,14 @@ public function __construct( if ($this->waitTimeoutSeconds !== null && $this->waitTimeoutSeconds < 0) { throw new InvalidArgumentException('Service operation wait timeout must be zero or greater.'); } + + if ($this->waitTimeoutSeconds !== null + && $this->waitTimeoutSeconds > UpdateWaitPolicy::MAX_COMPLETION_TIMEOUT_SECONDS) { + throw new InvalidArgumentException(sprintf( + 'Service operation wait timeout must not exceed %d seconds.', + UpdateWaitPolicy::MAX_COMPLETION_TIMEOUT_SECONDS, + )); + } } public static function asyncAccepted(?string $idempotencyKey = null): self diff --git a/src/V2/Support/UpdateWaitPolicy.php b/src/V2/Support/UpdateWaitPolicy.php index 4c6f2dc34..ef76d1c0b 100644 --- a/src/V2/Support/UpdateWaitPolicy.php +++ b/src/V2/Support/UpdateWaitPolicy.php @@ -10,6 +10,8 @@ final class UpdateWaitPolicy { public const COMPLETION_TIMEOUT_SECONDS = 10; + public const MAX_COMPLETION_TIMEOUT_SECONDS = 60; + public const POLL_INTERVAL_MILLISECONDS = 50; public const WAIT_FOR_ACCEPTED = 'accepted'; @@ -20,9 +22,12 @@ final class UpdateWaitPolicy public static function completionTimeoutSeconds(): int { - return self::configuredPositiveInt( - 'workflows.v2.update_wait.completion_timeout_seconds', - self::COMPLETION_TIMEOUT_SECONDS, + return min( + self::configuredPositiveInt( + 'workflows.v2.update_wait.completion_timeout_seconds', + self::COMPLETION_TIMEOUT_SECONDS, + ), + self::MAX_COMPLETION_TIMEOUT_SECONDS, ); } @@ -88,7 +93,19 @@ public static function requestedTimeoutSeconds(mixed $value): ?int } if ((is_int($value) || (is_string($value) && ctype_digit($value))) && (int) $value >= 1) { - return (int) $value; + $seconds = (int) $value; + + if ($seconds <= self::MAX_COMPLETION_TIMEOUT_SECONDS) { + return $seconds; + } + + throw ValidationException::withMessages([ + 'wait_timeout_seconds' => [ + 'The wait_timeout_seconds field must not exceed ' + . self::MAX_COMPLETION_TIMEOUT_SECONDS + . ' seconds.', + ], + ]); } throw ValidationException::withMessages([ @@ -96,6 +113,33 @@ public static function requestedTimeoutSeconds(mixed $value): ?int ]); } + public static function serviceOperationTimeoutSeconds(mixed $value): ?int + { + if ($value === null) { + return null; + } + + if (! is_int($value) || $value < 0) { + throw ValidationException::withMessages([ + 'wait_timeout_seconds' => [ + 'The wait_timeout_seconds field must be a non-negative integer when provided.', + ], + ]); + } + + if ($value > self::MAX_COMPLETION_TIMEOUT_SECONDS) { + throw ValidationException::withMessages([ + 'wait_timeout_seconds' => [ + 'The wait_timeout_seconds field must not exceed ' + . self::MAX_COMPLETION_TIMEOUT_SECONDS + . ' seconds.', + ], + ]); + } + + return $value; + } + private static function configuredPositiveInt(string $key, int $default): int { $value = config($key, $default); diff --git a/src/V2/Support/WorkerProtocolVersion.php b/src/V2/Support/WorkerProtocolVersion.php index a42706222..e46d9ccc3 100644 --- a/src/V2/Support/WorkerProtocolVersion.php +++ b/src/V2/Support/WorkerProtocolVersion.php @@ -30,7 +30,7 @@ final class WorkerProtocolVersion * pagination semantics). Bump the minor for additive changes (new * optional fields, new non-terminal command types). */ - public const VERSION = '1.20'; + public const VERSION = '1.21'; /** * Worker registration capability for server-routed workflow query @@ -166,6 +166,8 @@ final class WorkerProtocolVersion public const DURABLE_SELECTION_MINIMUM_PROTOCOL_VERSION = '1.19'; + public const DURABLE_SERVICE_OPERATION_SCHEDULER_MINIMUM_PROTOCOL_VERSION = '1.21'; + private const QUERY_TASKS_MINIMUM_PROTOCOL_VERSION = '1.8'; private const UPSERT_SEARCH_ATTRIBUTES_MINIMUM_PROTOCOL_VERSION = '1.8'; @@ -806,8 +808,15 @@ public static function serviceOperationCommandShape(): array ], 'wait_timeout_seconds' => [ 'shape' => 'non-negative integer', + 'maximum' => UpdateWaitPolicy::MAX_COMPLETION_TIMEOUT_SECONDS, 'required' => false, ], + 'durable_scheduler' => [ + 'minimum_protocol_version' => self::DURABLE_SERVICE_OPERATION_SCHEDULER_MINIMUM_PROTOCOL_VERSION, + 'admission_event' => 'ServiceOperationScheduled', + 'resume_visibility_field' => 'workflow_resume_on_admission', + 'effect_delivery' => 'after_commit_at_least_once', + ], 'metadata' => [ 'shape' => 'array', 'reserved_conformance_keys' => [ @@ -818,6 +827,7 @@ public static function serviceOperationCommandShape(): array ], ], 'service_call_result_events' => [ + 'ServiceOperationScheduled', 'ServiceCallStarted', 'ServiceCallCompleted', 'ServiceCallFailed', diff --git a/src/V2/Support/WorkflowCommandNormalizer.php b/src/V2/Support/WorkflowCommandNormalizer.php index 24e7b343d..4aa66d898 100644 --- a/src/V2/Support/WorkflowCommandNormalizer.php +++ b/src/V2/Support/WorkflowCommandNormalizer.php @@ -900,6 +900,16 @@ public static function normalize(array $commands, ?string $protocolVersion = nul $index, $errors, ); + if ($waitTimeoutSeconds !== null + && $waitTimeoutSeconds > UpdateWaitPolicy::MAX_COMPLETION_TIMEOUT_SECONDS) { + $errors["commands.{$index}.wait_timeout_seconds"] = [ + 'Workflow task command field [wait_timeout_seconds] must not exceed ' + . UpdateWaitPolicy::MAX_COMPLETION_TIMEOUT_SECONDS + . ' seconds.', + ]; + + continue; + } $normalized[] = array_filter([ 'type' => $type, diff --git a/src/V2/Support/WorkflowExecutor.php b/src/V2/Support/WorkflowExecutor.php index 24e423123..8fa42278a 100644 --- a/src/V2/Support/WorkflowExecutor.php +++ b/src/V2/Support/WorkflowExecutor.php @@ -1487,7 +1487,10 @@ private function executeServiceOperation( $task, $sequence, 'service_operation', - $serviceEvent?->event_type === HistoryEventType::ServiceCallStarted ? null : $serviceEvent, + in_array($serviceEvent?->event_type, [ + HistoryEventType::ServiceOperationScheduled, + HistoryEventType::ServiceCallStarted, + ], true) ? null : $serviceEvent, $workflowExecution ); if ($cancellationDelivery === CancellationDeliveryState::Waiting) { @@ -1505,10 +1508,10 @@ private function executeServiceOperation( return null; } } - if ($serviceEvent->event_type === HistoryEventType::ServiceCallStarted && ! self::serviceOperationStartedEventIsVisible( - $serviceEvent, - $current - )) { + if (in_array($serviceEvent->event_type, [ + HistoryEventType::ServiceOperationScheduled, + HistoryEventType::ServiceCallStarted, + ], true) && ! self::serviceOperationStartedEventIsVisible($serviceEvent, $current)) { $this->syncWorkflowCursor($workflow, $sequence + 1); return $this->waitForNextResumeSource($run, $task); } @@ -5931,6 +5934,7 @@ private function serviceOperationEvent(WorkflowRun $run, int $sequence): ?Workfl static fn (WorkflowHistoryEvent $event): bool => in_array( $event->event_type, [ + HistoryEventType::ServiceOperationScheduled, HistoryEventType::ServiceCallStarted, HistoryEventType::ServiceCallCompleted, HistoryEventType::ServiceCallFailed, @@ -5984,7 +5988,15 @@ private function recordServiceOperationEvent( $event = WorkflowHistoryEvent::record( $run, $eventType, - $this->serviceOperationEventPayload($run, $sequence, $call, $surface, $storedRequest, $payloadCodec), + $this->serviceOperationEventPayload( + $run, + $sequence, + $call, + $surface, + $eventType, + $storedRequest, + $payloadCodec, + ), $task, ); $run->historyEvents->push($event); @@ -6062,6 +6074,10 @@ private function defaultServiceOperationIdempotencyKey(WorkflowRun $run, int $se */ private static function serviceOperationEventTypeForSurface(array $surface): HistoryEventType { + if (($surface['retry_scheduled'] ?? false) === true) { + return HistoryEventType::ServiceOperationScheduled; + } + $status = self::nonEmptyString($surface['status'] ?? null); return match ($status) { @@ -6078,7 +6094,10 @@ private static function serviceOperationStartedEventIsVisible( WorkflowHistoryEvent $event, ServiceOperationCall $call, ): bool { - if ($event->event_type !== HistoryEventType::ServiceCallStarted) { + if (! in_array($event->event_type, [ + HistoryEventType::ServiceOperationScheduled, + HistoryEventType::ServiceCallStarted, + ], true)) { return true; } @@ -6086,6 +6105,13 @@ private static function serviceOperationStartedEventIsVisible( ? $event->payload['service_call'] : []; + if ($event->event_type === HistoryEventType::ServiceOperationScheduled) { + return ($event->payload['workflow_resume_on_admission'] ?? null) === true + || self::serviceOperationResumesOnAdmission($call) + || self::nonEmptyString($event->payload['operation_mode'] ?? null) === 'async' + || self::nonEmptyString($surface['operation_mode'] ?? null) === 'async'; + } + return self::serviceOperationResumesOnAdmission($call) || self::nonEmptyString($event->payload['wait_for'] ?? null) === 'accepted' || self::nonEmptyString($event->payload['operation_mode'] ?? null) === 'async' @@ -6107,6 +6133,7 @@ private function serviceOperationEventPayload( int $sequence, ServiceOperationCall $call, array $surface, + HistoryEventType $eventType, ?string $storedRequest, string $payloadCodec, ): array { @@ -6137,6 +6164,10 @@ private function serviceOperationEventPayload( ?? $call->options?->modeOverride, 'wait_for' => self::nonEmptyString($surface['wait_for'] ?? null) ?? $call->options?->waitFor, + 'workflow_resume_on_admission' => $eventType === HistoryEventType::ServiceOperationScheduled + ? (self::serviceOperationResumesOnAdmission($call) + || self::nonEmptyString($surface['operation_mode'] ?? null) === 'async') + : null, 'status' => self::nonEmptyString($surface['status'] ?? null), 'outcome' => self::nonEmptyString($surface['outcome'] ?? null), 'resolved_binding_kind' => self::nonEmptyString($surface['resolved_binding_kind'] ?? null), diff --git a/src/V2/Support/WorkflowFiberRunner.php b/src/V2/Support/WorkflowFiberRunner.php index 3a8adf87b..27386a9f9 100644 --- a/src/V2/Support/WorkflowFiberRunner.php +++ b/src/V2/Support/WorkflowFiberRunner.php @@ -1627,6 +1627,7 @@ private static function hasWorkflowCommandSequence(?string $type): bool 'ChildRunFailed', 'ChildRunCancelled', 'ChildRunTerminated', + 'ServiceOperationScheduled', 'ServiceCallStarted', 'ServiceCallCompleted', 'ServiceCallFailed', @@ -2201,6 +2202,7 @@ private static function indexRecordedServiceOperationOutcomes( $type = self::eventType($event); if (! in_array($type, [ + 'ServiceOperationScheduled', 'ServiceCallStarted', 'ServiceCallCompleted', 'ServiceCallFailed', @@ -2252,8 +2254,8 @@ private static function indexRecordedServiceOperationOutcomes( self::serviceOperationResponsePayload($payload, $fallbackCodec, $namespace), ), 'recorded_at' => $recordedAt, - 'admission_visible' => $type === 'ServiceCallStarted' - ? self::serviceOperationStartedPayloadIsVisible($payload, $surface) + 'admission_visible' => in_array($type, ['ServiceOperationScheduled', 'ServiceCallStarted'], true) + ? self::serviceOperationStartedPayloadIsVisible($type, $payload, $surface) : true, ]; } @@ -2276,8 +2278,17 @@ private static function serviceOperationStartedOutcomeIsVisible( * @param array $payload * @param array $surface */ - private static function serviceOperationStartedPayloadIsVisible(array $payload, array $surface): bool - { + private static function serviceOperationStartedPayloadIsVisible( + string $type, + array $payload, + array $surface, + ): bool { + if ($type === 'ServiceOperationScheduled') { + return ($payload['workflow_resume_on_admission'] ?? null) === true + || self::stringValue($payload['operation_mode'] ?? null) === 'async' + || self::stringValue($surface['operation_mode'] ?? null) === 'async'; + } + return self::stringValue($payload['wait_for'] ?? null) === 'accepted' || self::stringValue($payload['operation_mode'] ?? null) === 'async' || self::stringValue($surface['wait_for'] ?? null) === 'accepted' @@ -2315,7 +2326,7 @@ private static function indexOpenServiceOperationWaits(array $historyEvents): ar $open = []; foreach ($historyEvents as $event) { - if (self::eventType($event) !== 'ServiceCallStarted') { + if (! in_array(self::eventType($event), ['ServiceOperationScheduled', 'ServiceCallStarted'], true)) { continue; } diff --git a/src/V2/Support/WorkflowServiceOperationAuthority.php b/src/V2/Support/WorkflowServiceOperationAuthority.php index 96c6b752f..b1b0a03ca 100644 --- a/src/V2/Support/WorkflowServiceOperationAuthority.php +++ b/src/V2/Support/WorkflowServiceOperationAuthority.php @@ -14,6 +14,7 @@ use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowService; use Workflow\V2\Models\WorkflowServiceCall; +use Workflow\V2\Models\WorkflowServiceCallDispatch; use Workflow\V2\Models\WorkflowServiceEndpoint; use Workflow\V2\Models\WorkflowServiceOperation; @@ -34,6 +35,7 @@ public static function options(WorkflowRun $run, array $command): array } foreach ([ ['service_call_model', WorkflowServiceCall::class], + ['service_call_dispatch_model', WorkflowServiceCallDispatch::class], ['service_endpoint_model', WorkflowServiceEndpoint::class], ['service_model', WorkflowService::class], ['service_operation_model', WorkflowServiceOperation::class], diff --git a/src/V2/Support/WorkflowStepHistory.php b/src/V2/Support/WorkflowStepHistory.php index c4804de93..9a5525f29 100644 --- a/src/V2/Support/WorkflowStepHistory.php +++ b/src/V2/Support/WorkflowStepHistory.php @@ -65,6 +65,7 @@ final class WorkflowStepHistory HistoryEventType::ChildRunFailed, HistoryEventType::ChildRunCancelled, HistoryEventType::ChildRunTerminated, + HistoryEventType::ServiceOperationScheduled, HistoryEventType::ServiceCallStarted, HistoryEventType::ServiceCallCompleted, HistoryEventType::ServiceCallFailed, @@ -449,6 +450,7 @@ private static function eventMatchesShape(WorkflowHistoryEvent $event, string $e HistoryEventType::ChildRunTerminated, ], true), self::SERVICE_OPERATION => in_array($event->event_type, [ + HistoryEventType::ServiceOperationScheduled, HistoryEventType::ServiceCallStarted, HistoryEventType::ServiceCallCompleted, HistoryEventType::ServiceCallFailed, diff --git a/src/V2/TaskWatchdog.php b/src/V2/TaskWatchdog.php index b1eee0308..23c83e156 100644 --- a/src/V2/TaskWatchdog.php +++ b/src/V2/TaskWatchdog.php @@ -21,11 +21,13 @@ use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowRunSummary; use Workflow\V2\Models\WorkflowServiceCall; +use Workflow\V2\Models\WorkflowServiceCallDispatch; use Workflow\V2\Models\WorkflowTask; use Workflow\V2\Support\ActivityTimeoutEnforcer; use Workflow\V2\Support\ConfiguredV2Models; use Workflow\V2\Support\RequestedTaskScope; use Workflow\V2\Support\ServiceCallDeadline; +use Workflow\V2\Support\ServiceCallScheduler; use Workflow\V2\Support\TaskCompatibility; use Workflow\V2\Support\TaskDispatcher; use Workflow\V2\Support\TaskRepair; @@ -86,6 +88,9 @@ public static function wake(?string $connection = null, ?string $queue = null): * service_call_timeout_candidates: int, * service_call_timeouts_enforced: int, * service_call_timeout_failures: list + * service_call_dispatch_candidates: int, + * service_call_dispatches_enqueued: int, + * service_call_dispatch_failures: list * } */ public static function runPass( @@ -214,6 +219,17 @@ public static function runPass( } } + $serviceCallDispatch = ServiceCallScheduler::repairDue( + TaskRepairPolicy::scanLimit(), + $runIds, + $instanceId, + $connection, + $queue, + ); + $report['service_call_dispatch_candidates'] = $serviceCallDispatch['candidates']; + $report['service_call_dispatches_enqueued'] = $serviceCallDispatch['enqueued']; + $report['service_call_dispatch_failures'] = $serviceCallDispatch['failures']; + $activityTimeoutIds = ActivityTimeoutEnforcer::expiredExecutionIds( TaskRepairPolicy::scanLimit(), $connection, @@ -430,6 +446,9 @@ private static function recoverMissingTask(string $runId): array * service_call_timeout_candidates: int, * service_call_timeouts_enforced: int, * service_call_timeout_failures: list + * service_call_dispatch_candidates: int, + * service_call_dispatches_enqueued: int, + * service_call_dispatch_failures: list * } */ private static function emptyReport( @@ -465,6 +484,9 @@ private static function emptyReport( 'service_call_timeout_candidates' => 0, 'service_call_timeouts_enforced' => 0, 'service_call_timeout_failures' => [], + 'service_call_dispatch_candidates' => 0, + 'service_call_dispatches_enqueued' => 0, + 'service_call_dispatch_failures' => [], ]; } @@ -531,6 +553,8 @@ private static function tablesReady(): bool new WorkflowRun(), new WorkflowRunSummary(), new ActivityExecution(), + new WorkflowServiceCall(), + new WorkflowServiceCallDispatch(), ] as $model) { // The models honour workflows.storage.connection; ask that connection, // not the application default, whether their tables exist. diff --git a/src/config/workflows.php b/src/config/workflows.php index 946c0d540..d554eb7d1 100644 --- a/src/config/workflows.php +++ b/src/config/workflows.php @@ -85,6 +85,7 @@ 'service_model' => Workflow\V2\Models\WorkflowService::class, 'service_operation_model' => Workflow\V2\Models\WorkflowServiceOperation::class, 'service_call_model' => Workflow\V2\Models\WorkflowServiceCall::class, + 'service_call_dispatch_model' => Workflow\V2\Models\WorkflowServiceCallDispatch::class, 'types' => [ 'workflows' => [ // 'billing.invoice-sync' => App\Workflows\InvoiceSyncWorkflow::class, @@ -221,6 +222,18 @@ 50 ), ], + 'service_call_dispatch' => [ + 'claim_lease_seconds' => (int) Env::dw( + 'DW_V2_SERVICE_CALL_CLAIM_LEASE_SECONDS', + 'WORKFLOW_V2_SERVICE_CALL_CLAIM_LEASE_SECONDS', + 60 + ), + 'repair_after_seconds' => (int) Env::dw( + 'DW_V2_SERVICE_CALL_REPAIR_AFTER_SECONDS', + 'WORKFLOW_V2_SERVICE_CALL_REPAIR_AFTER_SECONDS', + 5 + ), + ], 'guardrails' => [ 'boot' => env('DW_V2_GUARDRAILS_BOOT', 'warn'), ], diff --git a/src/migrations/2026_10_26_000100_create_workflow_service_call_dispatches_table.php b/src/migrations/2026_10_26_000100_create_workflow_service_call_dispatches_table.php new file mode 100644 index 000000000..5017512a2 --- /dev/null +++ b/src/migrations/2026_10_26_000100_create_workflow_service_call_dispatches_table.php @@ -0,0 +1,56 @@ +string('service_call_id', 26) + ->primary(); + $table->string('state', 32); + $table->json('dispatch_options'); + $table->unsignedInteger('attempt') + ->default(1); + $table->unsignedInteger('max_attempts') + ->default(1); + $table->timestamp('available_at', 6); + $table->unsignedBigInteger('claim_generation') + ->default(0); + $table->char('claim_token', 64) + ->nullable(); + $table->timestamp('claimed_at', 6) + ->nullable(); + $table->timestamp('claim_expires_at', 6) + ->nullable(); + $table->char('authorized_catalog_revision', 64) + ->nullable(); + $table->char('authorized_policy_revision', 64) + ->nullable(); + $table->timestamp('dispatch_attempted_at', 6) + ->nullable(); + $table->timestamp('repair_available_at', 6) + ->nullable(); + $table->timestamp('completed_at', 6) + ->nullable(); + $table->text('last_error') + ->nullable(); + $table->timestamps(6); + + $table->index(['state', 'available_at'], 'wf_service_dispatch_due_idx'); + $table->index(['state', 'claim_expires_at'], 'wf_service_dispatch_claim_idx'); + $table->index(['state', 'repair_available_at'], 'wf_service_dispatch_repair_idx'); + }); + } + + public function down(): void + { + Schema::dropIfExists(self::TABLE); + } +}; diff --git a/tests/Unit/Commands/V2RepairPassCommandTest.php b/tests/Unit/Commands/V2RepairPassCommandTest.php index 5abe9104c..632e26755 100644 --- a/tests/Unit/Commands/V2RepairPassCommandTest.php +++ b/tests/Unit/Commands/V2RepairPassCommandTest.php @@ -78,6 +78,9 @@ public function testItIgnoresTheLoopThrottleByDefaultAndRepairsMissingSignalTask 'service_call_timeout_candidates' => 0, 'service_call_timeouts_enforced' => 0, 'service_call_timeout_failures' => [], + 'service_call_dispatch_candidates' => 0, + 'service_call_dispatches_enqueued' => 0, + 'service_call_dispatch_failures' => [], ]; $this->artisan('workflow:v2:repair-pass', [ @@ -147,6 +150,9 @@ public function testRespectThrottleOptionSkipsRepairWhenTheLoopThrottleIsHeld(): 'service_call_timeout_candidates' => 0, 'service_call_timeouts_enforced' => 0, 'service_call_timeout_failures' => [], + 'service_call_dispatch_candidates' => 0, + 'service_call_dispatches_enqueued' => 0, + 'service_call_dispatch_failures' => [], ]; $this->artisan('workflow:v2:repair-pass', [ @@ -348,6 +354,9 @@ public function testRunIdScopeRepairsOnlyTheSelectedMissingTaskRun(): void 'service_call_timeout_candidates' => 0, 'service_call_timeouts_enforced' => 0, 'service_call_timeout_failures' => [], + 'service_call_dispatch_candidates' => 0, + 'service_call_dispatches_enqueued' => 0, + 'service_call_dispatch_failures' => [], ]; $this->artisan('workflow:v2:repair-pass', [ @@ -407,6 +416,9 @@ public function testInstanceIdScopeRepairsOnlyTheSelectedInstanceTasks(): void 'service_call_timeout_candidates' => 0, 'service_call_timeouts_enforced' => 0, 'service_call_timeout_failures' => [], + 'service_call_dispatch_candidates' => 0, + 'service_call_dispatches_enqueued' => 0, + 'service_call_dispatch_failures' => [], ]; $this->artisan('workflow:v2:repair-pass', [ @@ -483,6 +495,9 @@ public function testConnectionAndQueueScopeRepairsOnlyMatchingExistingAndMissing 'service_call_timeout_candidates' => 0, 'service_call_timeouts_enforced' => 0, 'service_call_timeout_failures' => [], + 'service_call_dispatch_candidates' => 0, + 'service_call_dispatches_enqueued' => 0, + 'service_call_dispatch_failures' => [], ]; $this->artisan('workflow:v2:repair-pass', [ @@ -585,6 +600,9 @@ public function testCommaSeparatedQueueScopeRepairsEveryRequestedQueue(): void 'service_call_timeout_candidates' => 0, 'service_call_timeouts_enforced' => 0, 'service_call_timeout_failures' => [], + 'service_call_dispatch_candidates' => 0, + 'service_call_dispatches_enqueued' => 0, + 'service_call_dispatch_failures' => [], ]; $this->artisan('workflow:v2:repair-pass', [ @@ -662,6 +680,9 @@ public function testConnectionAndQueueScopeRepairsOnlyMatchingDeadlineExpiredRun 'service_call_timeout_candidates' => 0, 'service_call_timeouts_enforced' => 0, 'service_call_timeout_failures' => [], + 'service_call_dispatch_candidates' => 0, + 'service_call_dispatches_enqueued' => 0, + 'service_call_dispatch_failures' => [], ]; $this->artisan('workflow:v2:repair-pass', [ @@ -772,6 +793,9 @@ public function testConnectionAndQueueScopeEnforcesOnlyMatchingActivityTimeouts( 'service_call_timeout_candidates' => 0, 'service_call_timeouts_enforced' => 0, 'service_call_timeout_failures' => [], + 'service_call_dispatch_candidates' => 0, + 'service_call_dispatches_enqueued' => 0, + 'service_call_dispatch_failures' => [], ]; $this->artisan('workflow:v2:repair-pass', [ @@ -835,6 +859,9 @@ public function testLoopModeRunsTheRequestedNumberOfIterationsAndForcesThrottleA 'service_call_timeout_candidates' => 0, 'service_call_timeouts_enforced' => 0, 'service_call_timeout_failures' => [], + 'service_call_dispatch_candidates' => 0, + 'service_call_dispatches_enqueued' => 0, + 'service_call_dispatch_failures' => [], ]; $secondIteration = [ @@ -862,6 +889,9 @@ public function testLoopModeRunsTheRequestedNumberOfIterationsAndForcesThrottleA 'service_call_timeout_candidates' => 0, 'service_call_timeouts_enforced' => 0, 'service_call_timeout_failures' => [], + 'service_call_dispatch_candidates' => 0, + 'service_call_dispatches_enqueued' => 0, + 'service_call_dispatch_failures' => [], ]; $this->artisan('workflow:v2:repair-pass', [ diff --git a/tests/Unit/Providers/WorkflowServiceProviderTest.php b/tests/Unit/Providers/WorkflowServiceProviderTest.php index 60b03fb4c..66cfdff5e 100644 --- a/tests/Unit/Providers/WorkflowServiceProviderTest.php +++ b/tests/Unit/Providers/WorkflowServiceProviderTest.php @@ -205,9 +205,15 @@ public function testProviderMergesV2DefaultsIntoLegacyPublishedConfig(): void config('workflows.v2.service_operation_model') ); $this->assertSame(\Workflow\V2\Models\WorkflowServiceCall::class, config('workflows.v2.service_call_model')); + $this->assertSame( + \Workflow\V2\Models\WorkflowServiceCallDispatch::class, + config('workflows.v2.service_call_dispatch_model'), + ); $this->assertSame(30, config('workflows.v2.compatibility.heartbeat_ttl_seconds')); $this->assertSame(10, config('workflows.v2.update_wait.completion_timeout_seconds')); $this->assertSame(50, config('workflows.v2.update_wait.poll_interval_milliseconds')); + $this->assertSame(60, config('workflows.v2.service_call_dispatch.claim_lease_seconds')); + $this->assertSame(5, config('workflows.v2.service_call_dispatch.repair_after_seconds')); $this->assertSame(300, config('workflows.v2.workflow_task_lease_seconds')); $this->assertSame(3, config('workflows.v2.task_repair.redispatch_after_seconds')); $this->assertSame(5, config('workflows.v2.task_repair.loop_throttle_seconds')); diff --git a/tests/Unit/V2/CrossNamespaceServicePolicyDocumentationTest.php b/tests/Unit/V2/CrossNamespaceServicePolicyDocumentationTest.php index c1c7deffb..7519a5dcd 100644 --- a/tests/Unit/V2/CrossNamespaceServicePolicyDocumentationTest.php +++ b/tests/Unit/V2/CrossNamespaceServicePolicyDocumentationTest.php @@ -10,6 +10,7 @@ use Workflow\V2\Enums\ServiceCallOutcome; use Workflow\V2\Models\WorkflowService; use Workflow\V2\Models\WorkflowServiceCall; +use Workflow\V2\Models\WorkflowServiceCallDispatch; use Workflow\V2\Models\WorkflowServiceEndpoint; use Workflow\V2\Models\WorkflowServiceOperation; use Workflow\V2\Support\PayloadEnvelopeResolver; @@ -63,6 +64,7 @@ public function testServiceModelsUseTheirContractTables(): void $this->assertSame('workflow_services', (new WorkflowService())->getTable()); $this->assertSame('workflow_service_operations', (new WorkflowServiceOperation())->getTable()); $this->assertSame('workflow_service_calls', (new WorkflowServiceCall())->getTable()); + $this->assertSame('workflow_service_call_dispatches', (new WorkflowServiceCallDispatch())->getTable()); } public function testServiceCallModelCastsPolicySnapshotColumns(): void diff --git a/tests/Unit/V2/DefaultServiceControlPlaneWorkflowBindingTest.php b/tests/Unit/V2/DefaultServiceControlPlaneWorkflowBindingTest.php index 1748db248..ac8554b29 100644 --- a/tests/Unit/V2/DefaultServiceControlPlaneWorkflowBindingTest.php +++ b/tests/Unit/V2/DefaultServiceControlPlaneWorkflowBindingTest.php @@ -167,7 +167,7 @@ public function testLateCancellationCannotReplaceTheOriginalTimeoutWithoutPriorP } #[DataProvider('updateOutcomes')] - public function testUpdateBindingPreservesOptionsAndDurableOutcomeWithoutRedispatch( + public function testUpdateBindingDoesNotWaitForCompletionInsideTheIdempotencyTransaction( ServiceCallOperationMode $mode, bool $completed, bool $returnedIdentity, @@ -245,8 +245,7 @@ static function (string $instanceId, string $name, array $options) use ( 'arguments' => $arguments, 'payload_codec' => 'avro', 'payload_blob' => $payload, - 'wait_for' => $mode === ServiceCallOperationMode::Sync ? 'completed' : 'accepted', - 'wait_timeout_seconds' => 0, + 'wait_for' => 'accepted', ], $forwarded[2]); $call = WorkflowServiceCall::query()->sole(); @@ -310,6 +309,45 @@ static function (string $instanceId, string $name, array $options) use ( Queue::assertNothingPushed(); } + public function testUpdateBindingCanWaitForCompletionOutsideATransaction(): void + { + $this->catalog(ServiceCallBindingKind::WorkflowUpdate, ServiceCallOperationMode::Sync, [ + 'workflow_instance_id' => 'bound-invoice-owner', + 'update_name' => 'approve', + ]); + /** @var WorkflowControlPlane&MockInterface $workflows */ + $workflows = Mockery::mock(WorkflowControlPlane::class); + $workflows->shouldReceive('update') + ->once() + ->withArgs(static function (string $instanceId, string $name, array $options): bool { + $thisConnection = WorkflowServiceCall::query()->getModel()->getConnection(); + + return $instanceId === 'bound-invoice-owner' + && $name === 'approve' + && $thisConnection->transactionLevel() === 0 + && $options['wait_for'] === 'completed' + && $options['wait_timeout_seconds'] === 5; + }) + ->andReturn([ + 'accepted' => true, + 'update_status' => 'completed', + 'workflow_instance_id' => 'bound-invoice-owner', + 'run_id' => 'returned-run', + 'update_id' => 'returned-update', + ]); + + $result = (new DefaultServiceControlPlane($workflows))->execute('billing', 'invoices', 'approve', [ + 'namespace' => 'billing', + 'caller_namespace' => 'finance', + 'wait_timeout_seconds' => 5, + ]); + + $this->assertTrue($result['accepted']); + $this->assertSame(ServiceCallStatus::Completed->value, $result['status']); + $this->assertSame(ServiceCallOutcome::Completed->value, $result['outcome']); + Queue::assertNothingPushed(); + } + /** * @return iterable */ diff --git a/tests/Unit/V2/DefaultWorkflowTaskBridgeServiceOperationTest.php b/tests/Unit/V2/DefaultWorkflowTaskBridgeServiceOperationTest.php index d5173165a..9209afa7b 100644 --- a/tests/Unit/V2/DefaultWorkflowTaskBridgeServiceOperationTest.php +++ b/tests/Unit/V2/DefaultWorkflowTaskBridgeServiceOperationTest.php @@ -134,6 +134,9 @@ public function testServiceOutcomeRecordsDurableEvidenceAndOnlyResumesVisibleRes $this->assertSameJsonObject($surface, $event->payload['response_or_failure_surface']); $this->assertSame($this->run->id, $event->payload['caller_workflow_run_id']); $this->assertSame($payload, $event->payload['request_payload']); + if ($eventType === HistoryEventType::ServiceOperationScheduled) { + $this->assertSame($resume, $event->payload['workflow_resume_on_admission']); + } $response = $surface['response_payload'] ?? $surface['result'] ?? null; if ($response === null) { $this->assertArrayNotHasKey('response_payload', $event->payload); @@ -277,6 +280,31 @@ public static function serviceOutcomes(): iterable yield 'surface async mode' => [$started + [ 'operation_mode' => 'async', ], [], HistoryEventType::ServiceCallStarted, true, null, null]; + $scheduled = $base + [ + 'status' => 'accepted', + 'outcome' => 'accepted', + 'operation_mode' => 'sync', + 'wait_for' => 'accepted', + 'retry_scheduled' => true, + ]; + yield 'scheduled sync call remains blocked' => [ + $scheduled, [], HistoryEventType::ServiceOperationScheduled, false, null, null, + ]; + yield 'scheduled authored accepted wait resumes' => [ + $scheduled, [ + 'wait_for' => 'accepted', + ], HistoryEventType::ServiceOperationScheduled, true, null, null, + ]; + yield 'scheduled async operation resumes' => [ + array_replace($scheduled, [ + 'operation_mode' => 'async', + ]), + [], + HistoryEventType::ServiceOperationScheduled, + true, + null, + null, + ]; } public function testServiceOptionsPreserveTypedValuesAndAuthoritativeCallerMetadata(): void @@ -521,7 +549,7 @@ public static function invalidServiceCommands(): iterable foreach ([ 'mode_override' => ['queued', 42], 'wait_for' => ['applied', []], - 'wait_timeout_seconds' => [-1, 1.5, '0'], + 'wait_timeout_seconds' => [-1, 1.5, '0', 61], ] as $field => $values) { foreach ($values as $value) { yield $field . ' ' . json_encode($value, JSON_THROW_ON_ERROR) => [[ diff --git a/tests/Unit/V2/LinkedServiceCallCloserTest.php b/tests/Unit/V2/LinkedServiceCallCloserTest.php index efd4cef15..77737507f 100644 --- a/tests/Unit/V2/LinkedServiceCallCloserTest.php +++ b/tests/Unit/V2/LinkedServiceCallCloserTest.php @@ -24,6 +24,7 @@ use Workflow\V2\Enums\TaskStatus; use Workflow\V2\Enums\TaskType; use Workflow\V2\Exceptions\ServiceOperationAuthorizationException; +use Workflow\V2\Jobs\RunServiceCall; use Workflow\V2\Jobs\RunWorkflowTask; use Workflow\V2\Models\WorkflowHistoryEvent; use Workflow\V2\Models\WorkflowInstance; @@ -255,8 +256,14 @@ public function testUnlinkedCallsRemainExpiryCandidatesInAQueueScopedBackgroundP $this->assertSame(ServiceCallOutcome::TimedOut, $call->outcome); } - public function testTerminalServiceOutcomeResumesItsOriginalWaitingCallerOnce(): void - { + /** + * @param array $admission + */ + #[DataProvider('waitingAdmissionEvents')] + public function testTerminalServiceOutcomeResumesItsOriginalWaitingCallerOnce( + HistoryEventType $admissionType, + array $admission, + ): void { config([ 'workflows.v2.task_dispatch_mode' => 'poll', ]); @@ -267,11 +274,10 @@ public function testTerminalServiceOutcomeResumesItsOriginalWaitingCallerOnce(): 'caller_workflow_run_id' => $caller->id, 'operation_mode' => 'sync', ]); - WorkflowHistoryEvent::record($caller, HistoryEventType::ServiceCallStarted, [ + WorkflowHistoryEvent::record($caller, $admissionType, [ 'sequence' => 7, 'service_call_id' => $call->id, 'operation_mode' => 'sync', - 'wait_for' => 'completed', 'endpoint_name' => $call->endpoint_name, 'service_name' => $call->service_name, 'operation_name' => $call->operation_name, @@ -280,7 +286,7 @@ public function testTerminalServiceOutcomeResumesItsOriginalWaitingCallerOnce(): 'service_call' => (new DefaultServiceControlPlane())->describeCall($call->id, [ 'namespace' => 'service', ]), - ]); + ] + $admission); $blob = Serializer::serializeWithCodec('avro', [ 'greeting' => 'hello world', ]); @@ -338,6 +344,17 @@ public function testTerminalServiceOutcomeResumesItsOriginalWaitingCallerOnce(): $this->assertSame(2, WorkflowHistoryEvent::query()->where('workflow_run_id', $caller->id)->count()); } + public static function waitingAdmissionEvents(): iterable + { + yield 'handler started' => [HistoryEventType::ServiceCallStarted, [ + 'wait_for' => 'completed', + ]]; + yield 'durably scheduled' => [HistoryEventType::ServiceOperationScheduled, [ + 'wait_for' => 'accepted', + 'workflow_resume_on_admission' => false, + ]]; + } + public function testAlreadyCompletedServiceResumesWhenAdmissionHistoryArrivesLate(): void { config([ @@ -469,7 +486,16 @@ public function testEmbeddedCallerWaitsAndThenReplaysTheActualServiceResult(): v $this->assertSame(RunStatus::Waiting, $run->refresh()->status); $this->assertNull($workflow->refresh()->output()); $call = WorkflowServiceCall::query()->where('caller_workflow_run_id', $run->id)->sole(); - $this->assertSame('started', $call->status); + $this->assertSame('accepted', $call->status); + $scheduled = WorkflowHistoryEvent::query() + ->where('workflow_run_id', $run->id) + ->where('event_type', HistoryEventType::ServiceOperationScheduled->value) + ->sole(); + $this->assertFalse($scheduled->payload['workflow_resume_on_admission']); + $this->app->call([new RunServiceCall($call->id), 'handle']); + $this->assertSame('started', $call->refresh()->status); + $this->assertSame($service->workflow_instance_id, $call->linked_workflow_instance_id); + $this->assertSame($service->id, $call->linked_workflow_run_id); $expected = [ 'greeting' => 'hello world', @@ -478,6 +504,7 @@ public function testEmbeddedCallerWaitsAndThenReplaysTheActualServiceResult(): v 'output' => Serializer::serializeWithCodec('avro', $expected), 'payload_codec' => 'avro', ]); + $this->assertSame('completed', $call->refresh()->status); $resume = WorkflowTask::query() ->where('workflow_run_id', $run->id) ->where('status', TaskStatus::Ready->value) diff --git a/tests/Unit/V2/NexusCancellationDeliveryTest.php b/tests/Unit/V2/NexusCancellationDeliveryTest.php index 794f3a22b..327eb1997 100644 --- a/tests/Unit/V2/NexusCancellationDeliveryTest.php +++ b/tests/Unit/V2/NexusCancellationDeliveryTest.php @@ -19,6 +19,7 @@ use Workflow\V2\Enums\TaskType; use Workflow\V2\Exceptions\ServiceOperationAuthorizationException; use Workflow\V2\Exceptions\WorkflowCancellationRequestedException; +use Workflow\V2\Jobs\RunServiceCall; use Workflow\V2\Jobs\RunTimerTask; use Workflow\V2\Jobs\RunWorkflowTask; use Workflow\V2\Models\WorkflowRun; @@ -93,6 +94,10 @@ public function testOriginalCallerWaitsForTargetCleanupAndColdReplayKeepsItsBoun $this->runTask($initial); } $call = WorkflowServiceCall::query()->where('caller_workflow_run_id', $caller->runId())->sole(); + $this->assertSame('accepted', $call->status); + $this->assertNull($call->linked_workflow_run_id); + $this->runServiceCall($call); + $call->refresh(); $this->assertSame('started', $call->status); // Changing the catalog cannot revoke the original call's stored policy. WorkflowServiceOperation::query()->update([ @@ -176,6 +181,12 @@ public function testOriginalCallerWaitsForTargetCleanupAndColdReplayKeepsItsBoun } $this->assertSame( 1, + $caller->run() + ->historyEvents() + ->where('event_type', HistoryEventType::ServiceOperationScheduled->value)->count() + ); + $this->assertSame( + 0, $caller->run() ->historyEvents() ->where('event_type', HistoryEventType::ServiceCallStarted->value)->count() @@ -225,6 +236,8 @@ public function testReturnedExistingRunDoesNotBecomeOwnedByTheCaller(): void $caller->start('world'); $this->runTask($this->ready($caller)); $call = WorkflowServiceCall::query()->where('caller_workflow_run_id', $caller->runId())->sole(); + $this->runServiceCall($call); + $call->refresh(); $this->assertSame('returned_existing_active', $call->metadata['control_plane']['outcome']); $this->assertSame($target->runId(), $call->linked_workflow_run_id); $result = $caller->attemptRequestCancellation('caller cleanup', 30); @@ -258,6 +271,29 @@ public function testCancellationBeforeAdmissionDispatchesNoCallOrTarget(): void $this->assertSame($request->requestId, $delivery->payload['workflow_command_id']); } + public function testCancellationAfterAdmissionButBeforeClaimStartsNoTarget(): void + { + $this->catalog(); + $caller = WorkflowStub::make(TestNexusWaitingCallerWorkflow::class, 'nexus-caller', 'caller'); + $caller->start('world'); + $this->runTask($this->ready($caller)); + $call = WorkflowServiceCall::query()->where('caller_workflow_run_id', $caller->runId())->sole(); + $this->assertSame('accepted', $call->status); + $this->assertNull($call->linked_workflow_run_id); + + $request = $caller->attemptRequestCancellation('cancel before scheduled effect', 30); + $this->assertTrue($request->accepted()); + $this->runServiceCall($call); + $this->assertSame('cancelled', $call->refresh()->status); + $this->assertSame('caller_cancellation_requested', $call->outcome_reason); + $this->assertSame(0, WorkflowRun::query()->where('namespace', 'service')->count()); + $this->assertSame(1, $caller->run()->tasks()->where('status', TaskStatus::Ready->value)->count()); + + $this->runTask($this->ready($caller)); + $this->assertSame(RunStatus::Cancelled, $caller->run()->fresh()->status); + $this->assertSame(0, WorkflowRun::query()->where('namespace', 'service')->count()); + } + public function testIndependentTargetRequestRefusesANewCallerRootBeforeCommit(): void { $this->catalog(); @@ -265,6 +301,8 @@ public function testIndependentTargetRequestRefusesANewCallerRootBeforeCommit(): $caller->start('world'); $this->runTask($this->ready($caller)); $call = WorkflowServiceCall::query()->where('caller_workflow_run_id', $caller->runId())->sole(); + $this->runServiceCall($call); + $call->refresh(); $target = WorkflowStub::loadRun($call->linked_workflow_run_id); $original = $target->requestCancellation('independent owner', 20) ->cancellationContext(); @@ -323,4 +361,9 @@ private function runTask(WorkflowTask $task): void $job = $task->task_type === TaskType::Timer ? new RunTimerTask($task->id) : new RunWorkflowTask($task->id); $this->app->call([$job, 'handle']); } + + private function runServiceCall(WorkflowServiceCall $call): void + { + $this->app->call([new RunServiceCall($call->id), 'handle']); + } } diff --git a/tests/Unit/V2/QueryStateReplayerServiceOperationTest.php b/tests/Unit/V2/QueryStateReplayerServiceOperationTest.php index ad0fe252b..a5bab9a24 100644 --- a/tests/Unit/V2/QueryStateReplayerServiceOperationTest.php +++ b/tests/Unit/V2/QueryStateReplayerServiceOperationTest.php @@ -55,6 +55,26 @@ public static function pendingPrefixes(): array ]; } + public function testScheduledSyncCallDoesNotResumeFromInternalAcceptedDispatch(): void + { + $run = $this->createRun('sync'); + $this->appendEvent($run, HistoryEventType::ServiceOperationScheduled, [ + 'operation_mode' => 'sync', + 'wait_for' => 'accepted', + 'workflow_resume_on_admission' => false, + 'service_call' => [ + 'operation_mode' => 'sync', + 'wait_for' => 'accepted', + ], + ] + $this->servicePayload()); + + $this->assertReadOnlyState($run, [ + 'stage' => 'waiting-for-service', + 'result' => null, + 'failure' => null, + ], ServiceOperationCall::class, 1); + } + #[DataProvider('visibleAdmissions')] public function testAcceptedAdmissionRestoresAQueryableResult(string $mode, array $metadata): void { diff --git a/tests/Unit/V2/ServiceCallSchedulerTest.php b/tests/Unit/V2/ServiceCallSchedulerTest.php new file mode 100644 index 000000000..1cae08cf5 --- /dev/null +++ b/tests/Unit/V2/ServiceCallSchedulerTest.php @@ -0,0 +1,541 @@ +assertTrue($workflow->attemptStart()->accepted()); + $this->run = WorkflowRun::query()->findOrFail($workflow->runId()); + Queue::fake(); + } + + protected function tearDown(): void + { + Carbon::setTestNow(); + parent::tearDown(); + } + + public function testCompletionTransactionOnlyPersistsScheduleAndEnqueuesAfterCommit(): void + { + $operation = $this->catalog([ + 'max_attempts' => 3, + 'backoff_seconds' => 3_600, + ]); + /** @var WorkflowControlPlane&MockInterface $workflows */ + $workflows = Mockery::mock(WorkflowControlPlane::class); + $workflows->shouldNotReceive('update'); + $control = new DefaultServiceControlPlane($workflows); + + $started = hrtime(true); + DB::transaction(function () use ($control): void { + $result = $control->execute('billing', 'invoices', 'approve', $this->callOptions()); + + $this->assertTrue($result['accepted']); + $this->assertSame('accepted', $result['status']); + $this->assertSame('accepted', $result['wait_for']); + $this->assertTrue($result['retry_scheduled']); + $this->assertSame('accepted', $result['handler']['wait_for']); + $this->assertTrue($result['handler']['retry_scheduled']); + $this->assertSame(1, WorkflowServiceCallDispatch::query()->count()); + Queue::assertNothingPushed(); + }); + $elapsedMilliseconds = (hrtime(true) - $started) / 1_000_000; + + $call = WorkflowServiceCall::query()->sole(); + $dispatch = WorkflowServiceCallDispatch::query()->sole(); + $this->assertSame($operation->id, $call->workflow_service_operation_id); + $this->assertSame(ServiceCallScheduler::STATE_SCHEDULED, $dispatch->state); + $this->assertSame(1, $dispatch->attempt); + $this->assertSame(3, $dispatch->max_attempts); + $this->assertLessThan(1_000, $elapsedMilliseconds); + Queue::assertPushed(RunServiceCall::class, static fn (RunServiceCall $job): bool => + $job->serviceCallId === $call->id); + } + + public function testRollbackLeavesNoCallScheduleOrRunnableJob(): void + { + $this->catalog(); + /** @var WorkflowControlPlane&MockInterface $workflows */ + $workflows = Mockery::mock(WorkflowControlPlane::class); + $workflows->shouldNotReceive('update'); + $control = new DefaultServiceControlPlane($workflows); + + DB::beginTransaction(); + try { + $control->execute('billing', 'invoices', 'approve', $this->callOptions()); + throw new RuntimeException('rollback control'); + } catch (RuntimeException $failure) { + $this->assertSame('rollback control', $failure->getMessage()); + DB::rollBack(); + } + + $this->assertSame(0, WorkflowServiceCall::query()->count()); + $this->assertSame(0, WorkflowServiceCallDispatch::query()->count()); + Queue::assertNothingPushed(); + } + + public function testExcessiveWaitIsRejectedBeforeCallOrSchedulePersistence(): void + { + $this->catalog(); + /** @var WorkflowControlPlane&MockInterface $workflows */ + $workflows = Mockery::mock(WorkflowControlPlane::class); + $workflows->shouldNotReceive('update'); + $control = new DefaultServiceControlPlane($workflows); + + try { + DB::transaction(fn (): array => $control->execute( + 'billing', + 'invoices', + 'approve', + array_replace($this->callOptions(), [ + 'wait_timeout_seconds' => 61, + ]), + )); + } catch (ValidationException $exception) { + $this->assertSame([ + 'wait_timeout_seconds' => ['The wait_timeout_seconds field must not exceed 60 seconds.'], + ], $exception->errors()); + $this->assertSame(0, WorkflowServiceCall::query()->count()); + $this->assertSame(0, WorkflowServiceCallDispatch::query()->count()); + Queue::assertNothingPushed(); + + return; + } + + $this->fail('An excessive wait must be rejected before scheduler persistence.'); + } + + public function testClaimUsesCurrentPolicyButExecutesTheImmutableAcceptedBindingOutsideTransactions(): void + { + $operation = $this->catalog(); + $this->schedule(); + $operation->forceFill([ + 'handler_binding' => [ + 'workflow_instance_id' => 'changed-target', + 'update_name' => 'changed-update', + ], + 'handler_target_reference' => 'changed-update', + ])->save(); + + $claim = ServiceCallScheduler::claim(WorkflowServiceCall::query()->sole()->id); + $this->assertNotNull($claim); + $this->assertSame('accepted-target', $claim->operation->handler_binding['workflow_instance_id']); + $this->assertSame('approve', $claim->operation->handler_binding['update_name']); + $this->assertSame(1, $claim->attempt); + $this->assertSame(1, $claim->generation); + $this->assertSame(64, strlen($claim->catalogRevision)); + $this->assertSame(64, strlen($claim->policyRevision)); + $claimedDispatch = WorkflowServiceCallDispatch::query()->sole(); + $this->assertSame($claim->catalogRevision, $claimedDispatch->authorized_catalog_revision); + $this->assertSame($claim->policyRevision, $claimedDispatch->authorized_policy_revision); + + /** @var WorkflowControlPlane&MockInterface $workflows */ + $workflows = Mockery::mock(WorkflowControlPlane::class); + $workflows->shouldReceive('update') + ->once() + ->withArgs(static function (string $instanceId, string $name, array $options) use ($claim): bool { + return DB::connection()->transactionLevel() === 0 + && $instanceId === 'accepted-target' + && $name === 'approve' + && $options['wait_for'] === 'accepted' + && $options['service_call_id'] === $claim->call->id + && $options['service_call_attempt'] === 1 + && $options['service_call_max_attempts'] === 1 + && $options['service_call_idempotency_key'] === ServiceCallScheduler::attemptIdentity( + $claim->call->id, + 1, + ); + }) + ->andReturn([ + 'accepted' => true, + 'update_status' => 'accepted', + 'workflow_instance_id' => 'accepted-target', + 'run_id' => 'target-run', + 'update_id' => 'target-update', + ]); + + (new DefaultServiceControlPlane($workflows))->executeScheduledAttempt($claim); + + $call = WorkflowServiceCall::query()->sole(); + $dispatch = WorkflowServiceCallDispatch::query()->sole(); + $this->assertSame(ServiceCallStatus::Started->value, $call->status); + $this->assertSame('accepted-target', $call->linked_workflow_instance_id); + $this->assertSame('target-update', $call->linked_workflow_update_id); + $this->assertSame(ServiceCallScheduler::STATE_COMPLETED, $dispatch->state); + $this->assertCount(1, $call->metadata['service_call_attempts']); + $this->assertFalse($call->metadata['service_call_attempts'][0]['retry_scheduled']); + } + + public function testRetryBackoffIsPersistedWithoutSleepingOrHoldingTheCompletionLock(): void + { + $this->catalog([ + 'max_attempts' => 2, + 'backoff_seconds' => 3_600, + ]); + $this->schedule(); + $call = WorkflowServiceCall::query()->sole(); + $claim = ServiceCallScheduler::claim($call->id); + $this->assertNotNull($claim); + Queue::fake(); + + /** @var WorkflowControlPlane&MockInterface $workflows */ + $workflows = Mockery::mock(WorkflowControlPlane::class); + $workflows->shouldReceive('update') + ->once() + ->andThrow(new RuntimeException('retryable update failure')); + + $started = hrtime(true); + (new DefaultServiceControlPlane($workflows))->executeScheduledAttempt($claim); + $elapsedMilliseconds = (hrtime(true) - $started) / 1_000_000; + + $dispatch = WorkflowServiceCallDispatch::query()->sole(); + $this->assertLessThan(1_000, $elapsedMilliseconds); + $this->assertSame(ServiceCallScheduler::STATE_SCHEDULED, $dispatch->state); + $this->assertSame(2, $dispatch->attempt); + $this->assertTrue($dispatch->available_at->equalTo(now()->addHour())); + $this->assertSame(ServiceCallStatus::Started->value, $call->refresh()->status); + $this->assertTrue($call->metadata['service_call_attempts'][0]['retry_scheduled']); + $this->assertSame(3_600, $call->metadata['service_call_attempts'][0]['scheduled_backoff_seconds']); + Queue::assertPushed(RunServiceCall::class, static fn (RunServiceCall $job): bool => + $job->serviceCallId === $call->id); + } + + public function testFinalScheduledFailureReleasesBoundaryAdmission(): void + { + $this->catalog(); + $policy = new DefaultServiceBoundaryPolicy([ + 'concurrency' => [ + 'max_in_flight' => 1, + 'sync_only' => false, + ], + ]); + $this->schedule($policy); + $claim = ServiceCallScheduler::claim(WorkflowServiceCall::query()->sole()->id); + $this->assertNotNull($claim); + + /** @var WorkflowControlPlane&MockInterface $workflows */ + $workflows = Mockery::mock(WorkflowControlPlane::class); + $workflows->shouldReceive('update') + ->once() + ->andReturn([ + 'accepted' => false, + 'reason' => 'target_unavailable', + 'message' => 'Target is unavailable.', + ]); + (new DefaultServiceControlPlane($workflows, $policy))->executeScheduledAttempt($claim); + + Queue::fake(); + /** @var WorkflowControlPlane&MockInterface $nextWorkflows */ + $nextWorkflows = Mockery::mock(WorkflowControlPlane::class); + $nextWorkflows->shouldNotReceive('update'); + $next = DB::transaction(fn (): array => (new DefaultServiceControlPlane($nextWorkflows, $policy))->execute( + 'billing', + 'invoices', + 'approve', + array_replace($this->callOptions(), [ + 'idempotency_key' => 'scheduler-call-after-failure', + ]), + )); + + $this->assertTrue($next['accepted']); + $this->assertSame(2, WorkflowServiceCall::query()->count()); + $this->assertSame(2, WorkflowServiceCallDispatch::query()->count()); + Queue::assertPushed(RunServiceCall::class, 1); + } + + public function testExpiredClaimCanBeRepairedAndAStaleWorkerCannotAdvanceIt(): void + { + $this->catalog(); + $this->schedule(); + $call = WorkflowServiceCall::query()->sole(); + $first = ServiceCallScheduler::claim($call->id); + $this->assertNotNull($first); + WorkflowServiceCallDispatch::query()->whereKey($call->id)->update([ + 'claim_expires_at' => now() + ->subSecond(), + ]); + $second = ServiceCallScheduler::claim($call->id); + $this->assertNotNull($second); + $this->assertSame($first->generation + 1, $second->generation); + $this->assertNotSame($first->token, $second->token); + + /** @var WorkflowControlPlane&MockInterface $workflows */ + $workflows = Mockery::mock(WorkflowControlPlane::class); + $workflows->shouldNotReceive('update'); + (new DefaultServiceControlPlane($workflows))->executeScheduledAttempt($first); + + $dispatch = WorkflowServiceCallDispatch::query()->sole(); + $this->assertSame(ServiceCallScheduler::STATE_CLAIMED, $dispatch->state); + $this->assertSame($second->generation, $dispatch->claim_generation); + $this->assertSame($second->token, $dispatch->claim_token); + $this->assertSame(ServiceCallStatus::Accepted->value, $call->fresh()->status); + } + + public function testRepairReenqueuesALostDueDispatch(): void + { + $this->catalog(); + $this->schedule(); + $call = WorkflowServiceCall::query()->sole(); + Queue::fake(); + WorkflowServiceCallDispatch::query()->whereKey($call->id)->update([ + 'available_at' => now() + ->subSecond(), + 'dispatch_attempted_at' => now() + ->subMinute(), + 'repair_available_at' => now() + ->subSecond(), + ]); + + $this->assertSame([], ServiceCallScheduler::dueIds(10, ['another-run'])); + $this->assertSame([$call->id], ServiceCallScheduler::dueIds(10, [$this->run->id])); + $this->assertSame( + [$call->id], + ServiceCallScheduler::dueIds( + 10, + instanceId: $this->run->workflow_instance_id, + connection: $this->run->connection, + queue: $this->run->queue, + ), + ); + + $report = ServiceCallScheduler::repairDue(10); + + $this->assertSame(1, $report['candidates']); + $this->assertSame(1, $report['enqueued']); + $this->assertSame([], $report['failures']); + Queue::assertPushed(RunServiceCall::class, static fn (RunServiceCall $job): bool => + $job->serviceCallId === $call->id); + } + + public function testRepairReenqueuesAnExpiredCrashClaimAndTheNextWorkerAdvancesItsGeneration(): void + { + $this->catalog(); + $this->schedule(); + $call = WorkflowServiceCall::query()->sole(); + $crashed = ServiceCallScheduler::claim($call->id); + $this->assertNotNull($crashed); + Queue::fake(); + Carbon::setTestNow(now()->addSeconds(61)); + + $report = ServiceCallScheduler::repairDue(10); + + $this->assertSame(1, $report['candidates']); + $this->assertSame(1, $report['enqueued']); + Queue::assertPushed(RunServiceCall::class, static fn (RunServiceCall $job): bool => + $job->serviceCallId === $call->id); + $replacement = ServiceCallScheduler::claim($call->id); + $this->assertNotNull($replacement); + $this->assertSame($crashed->generation + 1, $replacement->generation); + $this->assertNotSame($crashed->token, $replacement->token); + } + + public function testCurrentPolicyRevocationFailsTheCallBeforeEffect(): void + { + $this->catalog(); + $policy = new DefaultServiceBoundaryPolicy(); + $this->app->instance(ServiceBoundaryPolicy::class, $policy); + $this->schedule($policy); + $policy->setRules([ + 'default_action' => 'deny', + ]); + + $claim = ServiceCallScheduler::claim(WorkflowServiceCall::query()->sole()->id); + + $this->assertNull($claim); + $call = WorkflowServiceCall::query()->sole(); + $this->assertSame(ServiceCallStatus::Failed->value, $call->status); + $this->assertSame(ServiceCallOutcome::RejectedForbidden, $call->outcome); + $this->assertSame('default_action_denied', $call->outcome_reason); + $this->assertSame(ServiceCallScheduler::STATE_COMPLETED, WorkflowServiceCallDispatch::query()->sole()->state); + } + + public function testTransientReplayPolicyFailureRollsBackClaimForRepair(): void + { + $this->catalog(); + $this->schedule(new DefaultServiceBoundaryPolicy()); + /** @var ServiceBoundaryReplayPolicy&MockInterface $policy */ + $policy = Mockery::mock(ServiceBoundaryReplayPolicy::class); + $policy->shouldReceive('authorizeReplay') + ->once() + ->andThrow(new RuntimeException('policy backend unavailable')); + app() + ->instance(ServiceBoundaryPolicy::class, $policy); + + try { + ServiceCallScheduler::claim(WorkflowServiceCall::query()->sole()->id); + } catch (RuntimeException $failure) { + $this->assertSame('policy backend unavailable', $failure->getMessage()); + $this->assertSame(ServiceCallStatus::Accepted->value, WorkflowServiceCall::query()->sole()->status); + $dispatch = WorkflowServiceCallDispatch::query()->sole(); + $this->assertSame(ServiceCallScheduler::STATE_SCHEDULED, $dispatch->state); + $this->assertSame(0, $dispatch->claim_generation); + $this->assertNull($dispatch->claim_token); + + return; + } + + $this->fail('A transient replay-policy failure must escape so the durable repair path can retry it.'); + } + + public function testTerminalCallerFencesAttempts(): void + { + $this->catalog(); + $this->schedule(); + $call = WorkflowServiceCall::query()->sole(); + $this->run->forceFill([ + 'status' => RunStatus::Completed->value, + 'closed_at' => now(), + ])->save(); + + $this->assertNull(ServiceCallScheduler::claim($call->id)); + $this->assertSame(ServiceCallStatus::Cancelled->value, $call->fresh()->status); + $this->assertSame(ServiceCallScheduler::STATE_COMPLETED, WorkflowServiceCallDispatch::query()->sole()->state); + } + + public function testCallerCancellationRequestedAfterCommitFencesTheEffect(): void + { + $this->catalog(); + $this->schedule(); + $call = WorkflowServiceCall::query()->sole(); + $request = WorkflowStub::loadRun($this->run->id) + ->attemptRequestCancellation('cancel before scheduled effect', 30); + + $this->assertTrue($request->accepted()); + $this->assertNull(ServiceCallScheduler::claim($call->id)); + $call->refresh(); + $this->assertSame(ServiceCallStatus::Cancelled->value, $call->status); + $this->assertSame(ServiceCallOutcome::Cancelled, $call->outcome); + $this->assertSame('caller_cancellation_requested', $call->outcome_reason); + $this->assertSame($request->commandId(), $call->outcome_metadata['cancellation_request_command_id']); + $this->assertSame(ServiceCallScheduler::STATE_COMPLETED, WorkflowServiceCallDispatch::query()->sole()->state); + } + + public function testExpiredCallDeadlineFencesAttempts(): void + { + $this->catalog(); + $this->schedule(); + $call = WorkflowServiceCall::query()->sole(); + $call->forceFill([ + 'call_deadline_at' => now() + ->subSecond(), + ])->save(); + + $this->assertNull(ServiceCallScheduler::claim($call->id)); + $this->assertSame(ServiceCallStatus::Failed->value, $call->fresh()->status); + $this->assertSame(ServiceCallOutcome::TimedOut, $call->fresh()->outcome); + $this->assertSame(ServiceCallScheduler::STATE_COMPLETED, WorkflowServiceCallDispatch::query()->sole()->state); + } + + private function schedule(?DefaultServiceBoundaryPolicy $policy = null): array + { + /** @var WorkflowControlPlane&MockInterface $workflows */ + $workflows = Mockery::mock(WorkflowControlPlane::class); + $workflows->shouldNotReceive('update'); + $control = new DefaultServiceControlPlane($workflows, $policy); + + return DB::transaction(fn (): array => $control->execute( + 'billing', + 'invoices', + 'approve', + $this->callOptions(), + )); + } + + /** + * @param array|null $retryPolicy + */ + private function catalog(?array $retryPolicy = null): WorkflowServiceOperation + { + $endpoint = WorkflowServiceEndpoint::query()->create([ + 'namespace' => 'billing', + 'endpoint_name' => 'billing', + ]); + $service = WorkflowService::query()->create([ + 'namespace' => 'billing', + 'workflow_service_endpoint_id' => $endpoint->id, + 'service_name' => 'invoices', + ]); + + return WorkflowServiceOperation::query()->create([ + 'namespace' => 'billing', + 'workflow_service_endpoint_id' => $endpoint->id, + 'workflow_service_id' => $service->id, + 'operation_name' => 'approve', + 'operation_mode' => ServiceCallOperationMode::Sync->value, + 'handler_binding_kind' => ServiceCallBindingKind::WorkflowUpdate->value, + 'handler_binding' => [ + 'workflow_instance_id' => 'accepted-target', + 'update_name' => 'approve', + ], + 'handler_target_reference' => 'approve', + 'retry_policy' => $retryPolicy, + ]); + } + + /** + * @return array + */ + private function callOptions(): array + { + return [ + 'namespace' => 'billing', + 'caller_namespace' => 'caller', + 'caller_workflow_instance_id' => $this->run->workflow_instance_id, + 'caller_workflow_run_id' => $this->run->id, + 'principal' => new ServiceCallPrincipal(subject: 'workflow:' . $this->run->id, method: 'workflow'), + 'idempotency_key' => 'scheduler-call', + 'wait_for' => 'completed', + 'wait_timeout_seconds' => 60, + ]; + } +} + +#[Type('service-scheduler-caller')] +final class ServiceSchedulerCallerWorkflow extends Workflow +{ + public function handle(): string + { + return 'done'; + } +} diff --git a/tests/Unit/V2/ServiceCommandOptionValidationTest.php b/tests/Unit/V2/ServiceCommandOptionValidationTest.php index f6f1796b2..7a8317234 100644 --- a/tests/Unit/V2/ServiceCommandOptionValidationTest.php +++ b/tests/Unit/V2/ServiceCommandOptionValidationTest.php @@ -5,13 +5,23 @@ namespace Tests\Unit\V2; use Illuminate\Validation\ValidationException; +use InvalidArgumentException; use PHPUnit\Framework\Attributes\DataProvider; use Tests\NonDatabaseTestCase; use Workflow\Serializers\Serializer; +use Workflow\V2\Support\ServiceOperationOptions; use Workflow\V2\Support\WorkflowCommandNormalizer; final class ServiceCommandOptionValidationTest extends NonDatabaseTestCase { + public function testTypedOptionsRejectAWaitAboveTheProtocolMaximum(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Service operation wait timeout must not exceed 60 seconds.'); + + new ServiceOperationOptions(waitTimeoutSeconds: 61); + } + public function testServiceMetadataAndPrincipalOptionsRetainTheirTypedValues(): void { $base = $this->command(); @@ -164,6 +174,13 @@ public static function invalidOptions(): iterable ], ]]; } + yield 'wait exceeds maximum' => [[ + 'wait_timeout_seconds' => 61, + ], [ + 'commands.1.wait_timeout_seconds' => [ + 'Workflow task command field [wait_timeout_seconds] must not exceed 60 seconds.', + ], + ]]; foreach ([ 'string' => 'reader', 'boolean' => false, diff --git a/tests/Unit/V2/ServiceExecutionContractTest.php b/tests/Unit/V2/ServiceExecutionContractTest.php index 73502018e..1b73c4241 100644 --- a/tests/Unit/V2/ServiceExecutionContractTest.php +++ b/tests/Unit/V2/ServiceExecutionContractTest.php @@ -12,6 +12,7 @@ use Workflow\V2\Enums\ServiceCallOperationMode; use Workflow\V2\Enums\ServiceCallOutcome; use Workflow\V2\Enums\ServiceCallStatus; +use Workflow\V2\Models\WorkflowServiceCallDispatch; use Workflow\V2\Support\ServiceExecutionContract; /** @@ -195,7 +196,7 @@ public function testManifestNamesDurableServiceRecordsAndGuaranteedObservability $manifest = ServiceExecutionContract::manifest(); $this->assertSame( - ['endpoints', 'services', 'operations', 'calls'], + ['endpoints', 'services', 'operations', 'calls', 'dispatches'], array_keys($manifest['durable_records']), ); $this->assertSame( @@ -204,6 +205,14 @@ public function testManifestNamesDurableServiceRecordsAndGuaranteedObservability ); $this->assertContains('resolved_binding_kind', $manifest['durable_records']['calls']['link_fields']); $this->assertContains('resolved_target_reference', $manifest['observability']['guaranteed_fields']); + $dispatch = $manifest['durable_records']['dispatches']; + $this->assertSame(WorkflowServiceCallDispatch::class, $dispatch['model']); + $this->assertSame('workflow_service_call_dispatches', $dispatch['table']); + $this->assertSame(['scheduled', 'claimed', 'completed'], $dispatch['states']); + $this->assertSame( + ['claim_generation', 'claim_token', 'claimed_at', 'claim_expires_at'], + $dispatch['claim_fence_fields'] + ); } public function testExecutionRulesRequireDurableCallIdAndFailClosedUnknownBindings(): void @@ -212,6 +221,10 @@ public function testExecutionRulesRequireDurableCallIdAndFailClosedUnknownBindin $this->assertStringContainsString('durable service-call id', $rules['durable_call_id']); $this->assertStringContainsString('fail closed', $rules['unknown_binding_kind']); + $this->assertStringContainsString('does not execute a handler', $rules['transaction_dispatch']); + $this->assertStringContainsString('current catalog', $rules['claim_authority']); + $this->assertStringContainsString('generation and token', $rules['attempt_fence']); + $this->assertStringContainsString('at least once', $rules['repair_delivery']); $this->assertStringContainsString('Raw transport logs are diagnostic only', $rules['transport_logs']); } } diff --git a/tests/Unit/V2/UpdateWaitPolicyTest.php b/tests/Unit/V2/UpdateWaitPolicyTest.php new file mode 100644 index 000000000..25bb07511 --- /dev/null +++ b/tests/Unit/V2/UpdateWaitPolicyTest.php @@ -0,0 +1,58 @@ +set('workflows.v2.update_wait.completion_timeout_seconds', 61); + + $this->assertSame(60, UpdateWaitPolicy::completionTimeoutSeconds()); + $this->assertSame(60, UpdateWaitPolicy::snapshot()['completion_timeout_seconds']); + } + + public function testRequestedCompletionTimeoutAcceptsOnlyTheBoundedPositiveRange(): void + { + $this->assertNull(UpdateWaitPolicy::requestedTimeoutSeconds(null)); + $this->assertSame(1, UpdateWaitPolicy::requestedTimeoutSeconds(1)); + $this->assertSame(60, UpdateWaitPolicy::requestedTimeoutSeconds('60')); + + try { + UpdateWaitPolicy::requestedTimeoutSeconds(61); + } catch (ValidationException $exception) { + $this->assertSame([ + 'wait_timeout_seconds' => ['The wait_timeout_seconds field must not exceed 60 seconds.'], + ], $exception->errors()); + + return; + } + + $this->fail('A completion wait above the maximum must be refused.'); + } + + public function testServiceOperationTimeoutAcceptsOnlyTheBoundedNonNegativeRange(): void + { + $this->assertNull(UpdateWaitPolicy::serviceOperationTimeoutSeconds(null)); + $this->assertSame(0, UpdateWaitPolicy::serviceOperationTimeoutSeconds(0)); + $this->assertSame(60, UpdateWaitPolicy::serviceOperationTimeoutSeconds(60)); + + try { + UpdateWaitPolicy::serviceOperationTimeoutSeconds(61); + } catch (ValidationException $exception) { + $this->assertSame([ + 'wait_timeout_seconds' => ['The wait_timeout_seconds field must not exceed 60 seconds.'], + ], $exception->errors()); + + return; + } + + $this->fail('A service-operation wait above the maximum must be refused.'); + } +} diff --git a/tests/Unit/V2/WorkerProtocolVersionTest.php b/tests/Unit/V2/WorkerProtocolVersionTest.php index f6a72abb7..49d1c8031 100644 --- a/tests/Unit/V2/WorkerProtocolVersionTest.php +++ b/tests/Unit/V2/WorkerProtocolVersionTest.php @@ -19,7 +19,7 @@ public function testVersionIsNonEmptyString(): void public function testVersionTracksServiceOperationCommandShape(): void { - $this->assertSame('1.20', WorkerProtocolVersion::VERSION); + $this->assertSame('1.21', WorkerProtocolVersion::VERSION); $this->assertContains('start_service_operation', WorkerProtocolVersion::nonTerminalCommandTypes()); $this->assertSame(0, WorkerProtocolVersion::longPollSemantics()['min_timeout_seconds']); } @@ -379,6 +379,12 @@ public function testDescribeIncludesServiceOperationCommandShape(): void $this->assertSame('start_service_operation', $shape['type']); $this->assertSame('non_terminal_command', $shape['category']); $this->assertSame('1.13', $shape['minimum_protocol_version']); + $this->assertSame(60, $shape['wait_timeout_seconds']['maximum']); + $this->assertSame( + WorkerProtocolVersion::DURABLE_SERVICE_OPERATION_SCHEDULER_MINIMUM_PROTOCOL_VERSION, + $shape['durable_scheduler']['minimum_protocol_version'], + ); + $this->assertSame('ServiceOperationScheduled', $shape['durable_scheduler']['admission_event']); $this->assertSame( ['type', 'endpoint_name', 'service_name', 'operation_name', 'request_payload'], $shape['required_fields'], diff --git a/tests/Unit/V2/WorkflowExecutorServiceOperationTest.php b/tests/Unit/V2/WorkflowExecutorServiceOperationTest.php index f4b03591f..5009f8c64 100644 --- a/tests/Unit/V2/WorkflowExecutorServiceOperationTest.php +++ b/tests/Unit/V2/WorkflowExecutorServiceOperationTest.php @@ -143,6 +143,7 @@ public function testPublicExecutionRecordsTheBoundaryOutcomeAndReplaysWithoutCal $this->ordered($service->executions[0]) ); $event = WorkflowHistoryEvent::query()->where('workflow_run_id', $run->id)->whereIn('event_type', [ + HistoryEventType::ServiceOperationScheduled->value, HistoryEventType::ServiceCallStarted->value, HistoryEventType::ServiceCallCompleted->value, HistoryEventType::ServiceCallFailed->value, @@ -193,6 +194,9 @@ public function testPublicExecutionRecordsTheBoundaryOutcomeAndReplaysWithoutCal 'wait_for' => 'accepted', ]; } + if ($eventType === HistoryEventType::ServiceOperationScheduled) { + $expectedPayload['workflow_resume_on_admission'] = ! $waiting; + } if ($failure !== null) { $exception = [ 'class' => RuntimeException::class, @@ -293,6 +297,23 @@ public static function outcomes(): iterable 'operation_mode' => 'async', 'wait_for' => 'accepted', ], [], null, HistoryEventType::ServiceCallStarted, false, null]; + yield 'scheduled sync admission waits for terminal history' => [[ + 'status' => 'accepted', + 'outcome' => 'accepted', + 'operation_mode' => 'sync', + 'wait_for' => 'accepted', + 'retry_scheduled' => true, + ], [], null, HistoryEventType::ServiceOperationScheduled, true, null]; + yield 'scheduled authored async admission resumes' => [[ + 'status' => 'accepted', + 'outcome' => 'accepted', + 'operation_mode' => 'async', + 'wait_for' => 'accepted', + 'retry_scheduled' => true, + ], [ + 'modeOverride' => 'async', + 'waitFor' => 'accepted', + ], null, HistoryEventType::ServiceOperationScheduled, false, null]; yield 'explicit accepted admission' => [[ 'status' => 'accepted', 'outcome' => 'accepted', diff --git a/tests/Unit/V2/WorkflowFiberRunnerTest.php b/tests/Unit/V2/WorkflowFiberRunnerTest.php index 652ef8add..bfcadfe86 100644 --- a/tests/Unit/V2/WorkflowFiberRunnerTest.php +++ b/tests/Unit/V2/WorkflowFiberRunnerTest.php @@ -2014,6 +2014,50 @@ public function testRunnerWaitsForStartedServiceOperationWhenAdmissionIsNotWorkf $this->assertInstanceOf(ServiceOperationCall::class, $waiting->yielded); } + public function testRunnerWaitsForScheduledSyncOperationDespiteInternalAcceptedDispatch(): void + { + $waiting = WorkflowFiberRunner::forClass( + WorkerProtocolRunnerServiceOperationWorkflow::class, + 'workflow-1', + 'run-1', + [], + 'avro', + [[ + 'sequence' => 1, + 'event_type' => 'WorkflowStarted', + 'payload' => [], + 'recorded_at' => '2026-05-12T10:11:12+00:00', + ], [ + 'sequence' => 2, + 'event_type' => 'ServiceOperationScheduled', + 'payload' => [ + 'sequence' => 1, + 'service_call_id' => 'svc-sync-scheduled-1', + 'endpoint_name' => 'payments', + 'service_name' => 'PythonPayments', + 'operation_name' => 'authorize', + 'operation_mode' => 'sync', + 'wait_for' => 'accepted', + 'workflow_resume_on_admission' => false, + 'status' => 'accepted', + 'outcome' => 'accepted', + 'service_call' => [ + 'service_call_id' => 'svc-sync-scheduled-1', + 'operation_mode' => 'sync', + 'wait_for' => 'accepted', + 'status' => 'accepted', + ], + ], + 'recorded_at' => '2026-05-12T10:11:13+00:00', + ]], + )->step(); + + $this->assertFalse($waiting->completed); + $this->assertNull($waiting->command); + $this->assertSame([], $waiting->commands); + $this->assertInstanceOf(ServiceOperationCall::class, $waiting->yielded); + } + public function testRunnerThrowsTypedServiceOperationFailureIntoWorkflow(): void { $completed = WorkflowFiberRunner::forClass( diff --git a/tests/Unit/migrations/MigrationsTest.php b/tests/Unit/migrations/MigrationsTest.php index 07d6f2f61..0f137e345 100644 --- a/tests/Unit/migrations/MigrationsTest.php +++ b/tests/Unit/migrations/MigrationsTest.php @@ -232,6 +232,7 @@ public function testDownMethodsDropTables(): void $this->assertTrue(Schema::hasTable('workflow_services')); $this->assertTrue(Schema::hasTable('workflow_service_operations')); $this->assertTrue(Schema::hasTable('workflow_service_calls')); + $this->assertTrue(Schema::hasTable('workflow_service_call_dispatches')); $this->assertTrue(Schema::hasColumn('workflow_service_endpoints', 'boundary_policy')); $this->assertTrue(Schema::hasColumn('workflow_services', 'boundary_policy')); $this->assertTrue(Schema::hasColumn('workflow_service_operations', 'boundary_policy')); @@ -248,6 +249,9 @@ public function testDownMethodsDropTables(): void $this->assertTrue(Schema::hasColumn('workflow_service_calls', 'caller_principal_roles')); $this->assertTrue(Schema::hasColumn('workflow_service_calls', 'caller_principal_tenant')); $this->assertTrue(Schema::hasColumn('workflow_service_calls', 'caller_principal_claims')); + $this->assertTrue(Schema::hasColumn('workflow_service_call_dispatches', 'claim_generation')); + $this->assertTrue(Schema::hasColumn('workflow_service_call_dispatches', 'authorized_catalog_revision')); + $this->assertTrue(Schema::hasColumn('workflow_service_call_dispatches', 'repair_available_at')); $this->assertTrue(Schema::hasColumn('activity_executions', 'current_attempt_id')); $this->assertTrue(Schema::hasColumn('workflow_history_events', 'workflow_command_id')); $this->assertTrue(Schema::hasColumn('workflow_run_summaries', 'sort_timestamp')); @@ -279,6 +283,9 @@ public function testDownMethodsDropTables(): void $this->assertIndexExists('workflow_run_summaries', 'wfrs_namespace_type_sort_idx'); $this->assertIndexExists('workflow_run_summaries', 'wfrs_namespace_status_sort_idx'); $this->assertIndexExists('workflow_tasks', 'workflow_tasks_namespace_queue_status_idx'); + $this->assertIndexExists('workflow_service_call_dispatches', 'wf_service_dispatch_due_idx'); + $this->assertIndexExists('workflow_service_call_dispatches', 'wf_service_dispatch_claim_idx'); + $this->assertIndexExists('workflow_service_call_dispatches', 'wf_service_dispatch_repair_idx'); $this->assertIndexExists( 'workflow_child_projection_repairs', 'workflow_child_projection_repairs_drain_idx', @@ -311,6 +318,7 @@ public function testDownMethodsDropTables(): void $this->assertFalse(Schema::hasTable('workflow_services')); $this->assertFalse(Schema::hasTable('workflow_service_operations')); $this->assertFalse(Schema::hasTable('workflow_service_calls')); + $this->assertFalse(Schema::hasTable('workflow_service_call_dispatches')); } /** @@ -588,6 +596,7 @@ private static function sqliteWorkflowTables(): array 'workflow_services', 'workflow_service_operations', 'workflow_service_calls', + 'workflow_service_call_dispatches', ]; } } From 111a526f3e9682eaf2f911e311d765400ddb61d9 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Mon, 26 Oct 2026 03:42:00 +0000 Subject: [PATCH 05/11] Fence scheduled service effects across crash recovery --- .../workflow-service-calls-architecture.md | 26 +- src/V2/Models/WorkflowCommand.php | 11 +- .../Support/DefaultWorkflowControlPlane.php | 38 ++- src/V2/Support/ServiceCallEffectIdentity.php | 38 +++ src/V2/Support/ServiceCallScheduler.php | 114 +++++++- src/V2/WorkflowStub.php | 20 ++ src/config/workflows.php | 5 + ...workflow_service_call_dispatches_table.php | 3 + .../Providers/WorkflowServiceProviderTest.php | 1 + .../Unit/V2/ServiceCallEffectIdentityTest.php | 138 ++++++++++ tests/Unit/V2/ServiceCallSchedulerTest.php | 243 ++++++++++++++++++ 11 files changed, 625 insertions(+), 12 deletions(-) create mode 100644 src/V2/Support/ServiceCallEffectIdentity.php create mode 100644 tests/Unit/V2/ServiceCallEffectIdentityTest.php diff --git a/docs/architecture/workflow-service-calls-architecture.md b/docs/architecture/workflow-service-calls-architecture.md index 559e149c3..641c9e53b 100644 --- a/docs/architecture/workflow-service-calls-architecture.md +++ b/docs/architecture/workflow-service-calls-architecture.md @@ -229,7 +229,13 @@ does not change the durable schedule. `TaskWatchdog` republishes due schedules, lost dispatches, and expired claims. Operators may tune `workflows.v2.service_call_dispatch.claim_lease_seconds` and `repair_after_seconds`; disabling the repair pass while scheduled calls exist -violates the liveness contract. +violates the liveness contract. The dispatch table intentionally has no fixed +foreign key because the service-call model and table are configurable. The same +repair pass removes only dispatches whose configured call is missing and whose +age exceeds `orphan_grace_seconds` (one hour by default and as an enforced +minimum), in deterministic, bounded primary-key batches. It rechecks absence +and age in the delete query so a concurrently restored or custom-model call is +not removed. A runnable attempt claims in one database transaction and in a fixed lock order: service-call row, dispatch row, current caller-run row, endpoint, service, then @@ -240,7 +246,11 @@ cancellation request, re-runs authorized catalog and policy revisions. Authority is current at claim time; the actual handler kind, binding and target reference remain the immutable accepted snapshot. A later catalog edit may revoke a call but cannot redirect -an already accepted call to a different handler. +an already accepted call to a different handler. When an activity or carrier +binding does not provide its target id, the first authorized claim creates that +id and persists it in the dispatch row in the same generation/token-fenced +transaction. Lease replacement reuses the persisted id; stale generations +cannot replace it. An explicit authorization refusal terminally rejects the call before any effect. A transient policy or storage failure is not an authorization decision: @@ -251,10 +261,20 @@ receives the stable identity `workflow-service-call::attempt:`. Handler adapters MUST propagate or otherwise enforce that identity because a lease may expire after an effect began and at-least-once recovery may invoke the attempt again. +The built-in workflow start, update, and signal adapters consume it as their +durable command request identity, so a repeated attempt returns the original +command/effect receipt rather than creating another effect. Activity and carrier +bindings currently resolve and persist a stable target reference without +launching an external effect in this control plane; a future execution lane must +consume the same attempt identity. Only the worker holding the current generation and token may persist handler progress or complete the attempt. Retry backoff is stored in `available_at`; no worker thread sleeps while holding a call, caller, catalog, or workflow-task -lock. +lock. Immediately before invoking an effect, the worker rechecks current claim +ownership, the call deadline, and the caller run's terminal/cancellation state; +a stale worker exits without touching the replacement claim, while a terminal +call, expired deadline, or closed/cancelling caller completes the dispatch +without invoking the handler. `ServiceOperationScheduled.workflow_resume_on_admission` records the authored delivery decision separately from the scheduler's internal accepted-only diff --git a/src/V2/Models/WorkflowCommand.php b/src/V2/Models/WorkflowCommand.php index 5d7064571..2498e7ef2 100644 --- a/src/V2/Models/WorkflowCommand.php +++ b/src/V2/Models/WorkflowCommand.php @@ -18,6 +18,7 @@ use Workflow\V2\Support\ConfiguredV2Models; use Workflow\V2\Support\ExternalPayloads; use Workflow\V2\Support\MessageStreamCursor; +use Workflow\V2\Support\ServiceCallEffectIdentity; use Workflow\V2\Support\UpdateRequestIdentity; class WorkflowCommand extends Model @@ -506,13 +507,21 @@ private static function withRequestIdentity(WorkflowInstance $instance, array $a $isUpdate = $commandType === CommandType::Update || $commandType === CommandType::Update->value; $usesRequestId = $commandType === CommandType::Redrive || $commandType === CommandType::Redrive->value; + $context = is_array($attributes['context'] ?? null) ? $attributes['context'] : []; + $serviceCallIdentity = ServiceCallEffectIdentity::fromContext($context); if ($isUpdate) { - $context = is_array($attributes['context'] ?? null) ? $attributes['context'] : []; $context['request'] = is_array($context['request'] ?? null) ? $context['request'] : []; $context['request']['request_id'] = UpdateRequestIdentity::fromContext($context); $attributes['context'] = $context; $attributes['request_id'] = UpdateRequestIdentity::storageToken($instance, $context); + } elseif ($serviceCallIdentity !== null && in_array($commandType, [ + CommandType::Start, + CommandType::Start->value, + CommandType::Signal, + CommandType::Signal->value, + ], true)) { + $attributes['request_id'] = ServiceCallEffectIdentity::storageToken($serviceCallIdentity); } elseif ($usesRequestId) { $request = is_array($attributes['context']['request'] ?? null) ? $attributes['context']['request'] diff --git a/src/V2/Support/DefaultWorkflowControlPlane.php b/src/V2/Support/DefaultWorkflowControlPlane.php index f411f7f4d..3d343bead 100644 --- a/src/V2/Support/DefaultWorkflowControlPlane.php +++ b/src/V2/Support/DefaultWorkflowControlPlane.php @@ -392,6 +392,10 @@ public function start(string $workflowType, ?string $instanceId = null, array $o $priority = TaskPriority::normalize($options['priority'] ?? null); $fairnessKey = TaskFairnessKey::normalize($options['fairness_key'] ?? null); $fairnessWeight = TaskFairnessKey::normalizeWeight($options['fairness_weight'] ?? null); + $serviceCallIdentity = ServiceCallEffectIdentity::fromOptions($options); + $serviceCallToken = $serviceCallIdentity === null + ? null + : ServiceCallEffectIdentity::storageToken($serviceCallIdentity); $workflowClass = $resolvedClass ?? $workflowType; $externalFingerprint = is_string($options['external_workflow_definition_fingerprint'] ?? null) @@ -427,6 +431,7 @@ public function start(string $workflowType, ?string $instanceId = null, array $o $priority, $fairnessKey, $fairnessWeight, + $serviceCallToken, &$command, &$task, &$instance, @@ -437,6 +442,21 @@ public function start(string $workflowType, ?string $instanceId = null, array $o $instance = $this->resolveOrCreateInstance($workflowType, $workflowClass, $instanceId, $namespace); + if ($serviceCallToken !== null) { + /** @var WorkflowCommand|null $existing */ + $existing = WorkflowCommand::query() + ->where('workflow_instance_id', $instance->id) + ->where('command_type', CommandType::Start->value) + ->where('request_id', $serviceCallToken) + ->lockForUpdate() + ->first(); + if ($existing instanceof WorkflowCommand) { + $command = $existing; + + return; + } + } + $currentRun = CurrentRunResolver::forInstance($instance, lockForUpdate: true); CurrentRunResolver::syncPointer($instance, $currentRun); @@ -1329,9 +1349,23 @@ private function commandContext(array $options): CommandContext { $commandContext = $options['command_context'] ?? null; - return $commandContext instanceof CommandContext + if ($commandContext instanceof CommandContext) { + return $commandContext; + } + + $commandContext = CommandContext::controlPlane(); + $serviceCallIdentity = ServiceCallEffectIdentity::fromOptions($options); + + return $serviceCallIdentity === null ? $commandContext - : CommandContext::controlPlane(); + : $commandContext->with([ + 'request' => [ + 'request_id' => $serviceCallIdentity, + ], + 'service_call' => [ + 'idempotency_key' => $serviceCallIdentity, + ], + ]); } private function configuredWorkflowValidationMessage(string $workflowType): ?string diff --git a/src/V2/Support/ServiceCallEffectIdentity.php b/src/V2/Support/ServiceCallEffectIdentity.php new file mode 100644 index 000000000..1773b7965 --- /dev/null +++ b/src/V2/Support/ServiceCallEffectIdentity.php @@ -0,0 +1,38 @@ + $options + */ + public static function fromOptions(array $options): ?string + { + return self::nonEmptyString($options['service_call_idempotency_key'] ?? null); + } + + /** + * @param array $context + */ + public static function fromContext(array $context): ?string + { + $serviceCall = $context['service_call'] ?? null; + + return is_array($serviceCall) + ? self::nonEmptyString($serviceCall['idempotency_key'] ?? null) + : null; + } + + public static function storageToken(string $identity): string + { + return 'service-call-v1:' . hash('sha256', $identity); + } + + private static function nonEmptyString(mixed $value): ?string + { + return is_string($value) && $value !== '' ? $value : null; + } +} diff --git a/src/V2/Support/ServiceCallScheduler.php b/src/V2/Support/ServiceCallScheduler.php index c8fd049ba..a2bae5a74 100644 --- a/src/V2/Support/ServiceCallScheduler.php +++ b/src/V2/Support/ServiceCallScheduler.php @@ -8,6 +8,7 @@ use Illuminate\Database\Eloquent\Builder; use Illuminate\Support\Str; use Throwable; +use Workflow\V2\Enums\ServiceCallBindingKind; use Workflow\V2\Enums\ServiceCallFailureReason; use Workflow\V2\Enums\ServiceCallOperationMode; use Workflow\V2\Enums\ServiceCallOutcome; @@ -29,6 +30,8 @@ final class ServiceCallScheduler public const STATE_COMPLETED = 'completed'; + private const MIN_ORPHAN_GRACE_SECONDS = 3_600; + /** * @param array $options */ @@ -187,8 +190,11 @@ public static function claim(string $serviceCallId): ?ServiceCallAttemptClaim $generation = (int) $dispatch->claim_generation + 1; $token = hash('sha256', (string) Str::uuid()); $now = now(); + $options = is_array($dispatch->dispatch_options) ? $dispatch->dispatch_options : []; + $options = self::withClaimTargetIdentifiers($options); $dispatch->forceFill([ 'state' => self::STATE_CLAIMED, + 'dispatch_options' => $options, 'claim_generation' => $generation, 'claim_token' => $token, 'claimed_at' => $now, @@ -200,7 +206,6 @@ public static function claim(string $serviceCallId): ?ServiceCallAttemptClaim 'last_error' => null, ])->save(); - $options = is_array($dispatch->dispatch_options) ? $dispatch->dispatch_options : []; $options['service_call_id'] = $call->id; $options['service_call_attempt'] = $dispatch->attempt; $options['service_call_max_attempts'] = $dispatch->max_attempts; @@ -308,7 +313,25 @@ public static function ownsClaim(ServiceCallAttemptClaim $claim): bool WorkflowServiceCall $call, WorkflowServiceCallDispatch $dispatch, ) use ($claim): bool { - return ! self::terminal($call) && self::ownsLockedClaim($dispatch, $claim); + if (self::terminal($call) || ! self::ownsLockedClaim($dispatch, $claim)) { + return false; + } + if (ServiceCallDeadline::expireLocked($call, now())) { + self::completeDispatch($dispatch); + + return false; + } + + $run = self::lockedCallerRun($call); + if (! $run instanceof WorkflowRun + || $run->status->isTerminal() + || is_string($run->cancellation_request_command_id)) { + self::cancelForTerminalCaller($call, $dispatch, $run); + + return false; + } + + return true; }) === true; } @@ -379,6 +402,7 @@ public static function repairDue( ?string $connection = null, ?string $queue = null, ): array { + self::cleanupOrphans($limit); $ids = self::dueIds($limit, $runIds, $instanceId, $connection, $queue); $report = [ 'candidates' => count($ids), @@ -402,6 +426,29 @@ public static function repairDue( return $report; } + public static function cleanupOrphans(int $limit): int + { + $cutoff = now() + ->subSeconds(self::orphanGraceSeconds()) + ->utc() + ->format('Y-m-d H:i:s.u'); + $ids = self::dispatchQuery() + ->whereDoesntHave('call') + ->where('created_at', '<=', $cutoff) + ->orderBy('service_call_id') + ->limit(max(1, $limit)) + ->pluck('service_call_id') + ->all(); + + return $ids === [] + ? 0 + : self::dispatchQuery() + ->whereKey($ids) + ->whereDoesntHave('call') + ->where('created_at', '<=', $cutoff) + ->delete(); + } + public static function attemptIdentity(string $serviceCallId, int $attempt): string { return sprintf('workflow-service-call:%s:attempt:%d', $serviceCallId, $attempt); @@ -518,7 +565,7 @@ private static function cancelForTerminalCaller( && is_string($run->cancellation_request_command_id); $reason = $cancellationRequested ? 'caller_cancellation_requested' : 'caller_run_terminal'; $message = $cancellationRequested - ? 'Caller workflow cancellation was requested before the service-call attempt was claimed.' + ? 'Caller workflow cancellation was requested before the service-call effect was invoked.' : ($run instanceof WorkflowRun ? sprintf('Caller workflow run is terminal (%s).', $run->status->value) : 'Caller workflow authority is unavailable.'); @@ -646,15 +693,62 @@ private static function persistableOptions(WorkflowServiceOperation $operation, } $persisted['wait_for'] = UpdateWaitPolicy::WAIT_FOR_ACCEPTED; $persisted['accepted_handler_binding_kind'] = $operation->handler_binding_kind; - $persisted['accepted_handler_binding'] = is_array($operation->handler_binding) - ? $operation->handler_binding - : []; + $binding = is_array($operation->handler_binding) ? $operation->handler_binding : []; + $persisted['accepted_handler_binding'] = $binding; $persisted['accepted_handler_target_reference'] = $operation->handler_target_reference; $persisted['accepted_operation_mode'] = $operation->operation_mode; + $kind = ServiceCallBindingKind::tryFromHandlerBindingKind((string) $operation->handler_binding_kind); + if ($kind === ServiceCallBindingKind::WorkflowRun + && ! self::hasString($binding, ['workflow_instance_id', 'instance_id']) + && ! self::hasString($persisted, ['workflow_instance_id'])) { + $persisted['workflow_instance_id'] = (string) Str::ulid(); + } + return $persisted; } + /** + * @param array $options + * @return array + */ + private static function withClaimTargetIdentifiers(array $options): array + { + $binding = is_array($options['accepted_handler_binding'] ?? null) + ? $options['accepted_handler_binding'] + : []; + $kind = ServiceCallBindingKind::tryFromHandlerBindingKind( + (string) ($options['accepted_handler_binding_kind'] ?? ''), + ); + if ($kind === ServiceCallBindingKind::ActivityExecution + && ! self::hasString($binding, ['activity_execution_id']) + && ! self::hasString($options, ['activity_execution_id'])) { + $options['activity_execution_id'] = (string) Str::ulid(); + } + if ($kind === ServiceCallBindingKind::InvocableCarrierRequest + && ! self::hasString($binding, ['carrier_request_id']) + && ! self::hasString($options, ['carrier_request_id'])) { + $options['carrier_request_id'] = (string) Str::ulid(); + } + + return $options; + } + + /** + * @param array $values + * @param list $keys + */ + private static function hasString(array $values, array $keys): bool + { + foreach ($keys as $key) { + if (is_string($values[$key] ?? null) && trim($values[$key]) !== '') { + return true; + } + } + + return false; + } + /** * @return array */ @@ -699,6 +793,14 @@ private static function repairAfterSeconds(): int return max(1, (int) config('workflows.v2.service_call_dispatch.repair_after_seconds', 5)); } + private static function orphanGraceSeconds(): int + { + return max( + self::MIN_ORPHAN_GRACE_SECONDS, + (int) config('workflows.v2.service_call_dispatch.orphan_grace_seconds', 3_600), + ); + } + /** * @return Builder */ diff --git a/src/V2/WorkflowStub.php b/src/V2/WorkflowStub.php index 04c1a54ce..28d2524e5 100644 --- a/src/V2/WorkflowStub.php +++ b/src/V2/WorkflowStub.php @@ -73,6 +73,7 @@ use Workflow\V2\Support\SearchAttributeUpsertService; use Workflow\V2\Support\SelectedRunLocator; use Workflow\V2\Support\ServiceCallCancellationOrigin; +use Workflow\V2\Support\ServiceCallEffectIdentity; use Workflow\V2\Support\ServiceOperationCancellation; use Workflow\V2\Support\SignalWaits; use Workflow\V2\Support\SqliteTransaction; @@ -2880,6 +2881,25 @@ private function attemptSignalInternal( ->lockForUpdate() ->findOrFail($this->instance->id); + $serviceCallIdentity = ServiceCallEffectIdentity::fromContext( + $this->resolvedCommandContext() + ->attributes()['context'], + ); + if ($serviceCallIdentity !== null) { + /** @var WorkflowCommand|null $existing */ + $existing = WorkflowCommand::query() + ->where('workflow_instance_id', $instance->id) + ->where('command_type', CommandType::Signal->value) + ->where('request_id', ServiceCallEffectIdentity::storageToken($serviceCallIdentity)) + ->lockForUpdate() + ->first(); + if ($existing instanceof WorkflowCommand) { + $command = $existing; + + return; + } + } + $target = $this->signalRunOrRejection($instance, $name, $arguments, $payloadCodec); if ($target instanceof WorkflowCommand) { $command = $target; diff --git a/src/config/workflows.php b/src/config/workflows.php index d554eb7d1..71219fe04 100644 --- a/src/config/workflows.php +++ b/src/config/workflows.php @@ -233,6 +233,11 @@ 'WORKFLOW_V2_SERVICE_CALL_REPAIR_AFTER_SECONDS', 5 ), + 'orphan_grace_seconds' => (int) Env::dw( + 'DW_V2_SERVICE_CALL_ORPHAN_GRACE_SECONDS', + 'WORKFLOW_V2_SERVICE_CALL_ORPHAN_GRACE_SECONDS', + 3600 + ), ], 'guardrails' => [ 'boot' => env('DW_V2_GUARDRAILS_BOOT', 'warn'), diff --git a/src/migrations/2026_10_26_000100_create_workflow_service_call_dispatches_table.php b/src/migrations/2026_10_26_000100_create_workflow_service_call_dispatches_table.php index 5017512a2..97dbb8eb4 100644 --- a/src/migrations/2026_10_26_000100_create_workflow_service_call_dispatches_table.php +++ b/src/migrations/2026_10_26_000100_create_workflow_service_call_dispatches_table.php @@ -12,6 +12,9 @@ public function up(): void { Schema::create(self::TABLE, static function (Blueprint $table): void { + // The service-call model/table is configurable, so a fixed foreign + // key would reject supported custom storage. Repair passes remove + // old missing-call dispatches in bounded primary-key batches instead. $table->string('service_call_id', 26) ->primary(); $table->string('state', 32); diff --git a/tests/Unit/Providers/WorkflowServiceProviderTest.php b/tests/Unit/Providers/WorkflowServiceProviderTest.php index 66cfdff5e..f7a3f9b13 100644 --- a/tests/Unit/Providers/WorkflowServiceProviderTest.php +++ b/tests/Unit/Providers/WorkflowServiceProviderTest.php @@ -214,6 +214,7 @@ public function testProviderMergesV2DefaultsIntoLegacyPublishedConfig(): void $this->assertSame(50, config('workflows.v2.update_wait.poll_interval_milliseconds')); $this->assertSame(60, config('workflows.v2.service_call_dispatch.claim_lease_seconds')); $this->assertSame(5, config('workflows.v2.service_call_dispatch.repair_after_seconds')); + $this->assertSame(3600, config('workflows.v2.service_call_dispatch.orphan_grace_seconds')); $this->assertSame(300, config('workflows.v2.workflow_task_lease_seconds')); $this->assertSame(3, config('workflows.v2.task_repair.redispatch_after_seconds')); $this->assertSame(5, config('workflows.v2.task_repair.loop_throttle_seconds')); diff --git a/tests/Unit/V2/ServiceCallEffectIdentityTest.php b/tests/Unit/V2/ServiceCallEffectIdentityTest.php new file mode 100644 index 000000000..6dfb82b20 --- /dev/null +++ b/tests/Unit/V2/ServiceCallEffectIdentityTest.php @@ -0,0 +1,138 @@ + 'workflow-service-call:call:attempt:1', + ]; + + $first = $control->start(ServiceCallEffectWorkflow::class, 'service-effect-start', $options); + $second = $control->start(ServiceCallEffectWorkflow::class, 'service-effect-start', $options); + + $this->assertTrue($first['started']); + $this->assertTrue($second['started']); + $this->assertSame($first['workflow_run_id'], $second['workflow_run_id']); + $this->assertSame(1, WorkflowRun::query()->count()); + $command = WorkflowCommand::query() + ->where('command_type', CommandType::Start->value) + ->sole(); + $this->assertSame( + ServiceCallEffectIdentity::storageToken($options['service_call_idempotency_key']), + $command->request_id, + ); + } + + public function testWorkflowUpdateConsumesTheStableServiceCallIdentity(): void + { + $workflow = $this->waitingWorkflow('service-effect-update'); + $control = new DefaultWorkflowControlPlane(); + $options = [ + 'arguments' => ['first'], + 'wait_for' => 'accepted', + 'service_call_idempotency_key' => 'workflow-service-call:update:attempt:1', + ]; + + $first = $control->update($workflow->id(), 'change', $options); + $second = $control->update($workflow->id(), 'change', array_replace($options, [ + 'arguments' => ['ignored'], + ])); + + $this->assertTrue($first['accepted']); + $this->assertTrue($second['accepted']); + $this->assertSame($first['update_id'], $second['update_id']); + $this->assertSame(1, WorkflowUpdate::query()->count()); + $command = WorkflowCommand::query() + ->where('command_type', CommandType::Update->value) + ->sole(); + $this->assertSame($options['service_call_idempotency_key'], $command->requestId()); + $this->assertSame( + $options['service_call_idempotency_key'], + $command->context['service_call']['idempotency_key'], + ); + } + + public function testWorkflowSignalConsumesTheStableServiceCallIdentity(): void + { + $workflow = $this->waitingWorkflow('service-effect-signal'); + $control = new DefaultWorkflowControlPlane(); + $options = [ + 'service_call_idempotency_key' => 'workflow-service-call:signal:attempt:1', + ]; + + $first = $control->signal($workflow->id(), 'finish', $options); + $second = $control->signal($workflow->id(), 'finish', $options); + + $this->assertTrue($first['accepted']); + $this->assertTrue($second['accepted']); + $this->assertSame($first['workflow_command_id'], $second['workflow_command_id']); + $this->assertSame(1, WorkflowSignal::query()->count()); + $command = WorkflowCommand::query() + ->where('command_type', CommandType::Signal->value) + ->sole(); + $this->assertSame( + ServiceCallEffectIdentity::storageToken($options['service_call_idempotency_key']), + $command->request_id, + ); + } + + private function waitingWorkflow(string $instanceId): WorkflowStub + { + $workflow = WorkflowStub::make(ServiceCallEffectWorkflow::class, $instanceId); + $workflow->start(); + $task = WorkflowTask::query() + ->where('workflow_run_id', $workflow->runId()) + ->sole(); + $result = $this->app->make(DefaultWorkflowTaskBridge::class)->execute($task->id); + $this->assertSame('waiting', $result['run_status']); + Queue::fake(); + + return $workflow; + } +} + +#[Type('service-call-effect')] +#[Signal('finish')] +final class ServiceCallEffectWorkflow extends Workflow +{ + public function handle(): void + { + signal('finish'); + } + + #[UpdateMethod] + public function change(string $value): string + { + return $value; + } +} diff --git a/tests/Unit/V2/ServiceCallSchedulerTest.php b/tests/Unit/V2/ServiceCallSchedulerTest.php index 1cae08cf5..98fc9c7fa 100644 --- a/tests/Unit/V2/ServiceCallSchedulerTest.php +++ b/tests/Unit/V2/ServiceCallSchedulerTest.php @@ -4,12 +4,15 @@ namespace Tests\Unit\V2; +use Illuminate\Database\Schema\Blueprint; use Illuminate\Support\Carbon; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Queue; +use Illuminate\Support\Facades\Schema; use Illuminate\Validation\ValidationException; use Mockery; use Mockery\MockInterface; +use PHPUnit\Framework\Attributes\DataProvider; use RuntimeException; use Tests\TestCase; use Workflow\V2\Attributes\Type; @@ -239,6 +242,20 @@ public function testRetryBackoffIsPersistedWithoutSleepingOrHoldingTheCompletion $this->assertSame(3_600, $call->metadata['service_call_attempts'][0]['scheduled_backoff_seconds']); Queue::assertPushed(RunServiceCall::class, static fn (RunServiceCall $job): bool => $job->serviceCallId === $call->id); + + Carbon::setTestNow(now()->addHour()); + $finalClaim = ServiceCallScheduler::claim($call->id); + $this->assertNotNull($finalClaim); + $workflows->shouldReceive('update') + ->once() + ->andThrow(new RuntimeException('final update failure')); + (new DefaultServiceControlPlane($workflows))->executeScheduledAttempt($finalClaim); + + $this->assertSame(ServiceCallScheduler::STATE_COMPLETED, $dispatch->fresh()->state); + $call->refresh(); + $this->assertSame(ServiceCallStatus::Failed->value, $call->status); + $this->assertCount(2, $call->metadata['service_call_attempts']); + $this->assertFalse($call->metadata['service_call_attempts'][1]['retry_scheduled']); } public function testFinalScheduledFailureReleasesBoundaryAdmission(): void @@ -299,6 +316,10 @@ public function testExpiredClaimCanBeRepairedAndAStaleWorkerCannotAdvanceIt(): v $this->assertNotNull($second); $this->assertSame($first->generation + 1, $second->generation); $this->assertNotSame($first->token, $second->token); + $this->assertSame( + $first->options['service_call_idempotency_key'], + $second->options['service_call_idempotency_key'], + ); /** @var WorkflowControlPlane&MockInterface $workflows */ $workflows = Mockery::mock(WorkflowControlPlane::class); @@ -312,6 +333,18 @@ public function testExpiredClaimCanBeRepairedAndAStaleWorkerCannotAdvanceIt(): v $this->assertSame(ServiceCallStatus::Accepted->value, $call->fresh()->status); } + public function testAnUnexpiredClaimCannotBeClaimedConcurrently(): void + { + $this->catalog(); + $this->schedule(); + $call = WorkflowServiceCall::query()->sole(); + $first = ServiceCallScheduler::claim($call->id); + + $this->assertNotNull($first); + $this->assertNull(ServiceCallScheduler::claim($call->id)); + $this->assertSame($first->token, WorkflowServiceCallDispatch::query()->sole()->claim_token); + } + public function testRepairReenqueuesALostDueDispatch(): void { $this->catalog(); @@ -390,6 +423,34 @@ public function testCurrentPolicyRevocationFailsTheCallBeforeEffect(): void $this->assertSame(ServiceCallScheduler::STATE_COMPLETED, WorkflowServiceCallDispatch::query()->sole()->state); } + public function testRemovedCatalogOperationFailsTheCallBeforeEffect(): void + { + $operation = $this->catalog(); + $this->schedule(); + $operation->delete(); + + $this->assertNull(ServiceCallScheduler::claim(WorkflowServiceCall::query()->sole()->id)); + $call = WorkflowServiceCall::query()->sole(); + $this->assertSame(ServiceCallStatus::Failed->value, $call->status); + $this->assertSame('service_call_not_owned', $call->outcome_reason); + $this->assertSame(ServiceCallScheduler::STATE_COMPLETED, WorkflowServiceCallDispatch::query()->sole()->state); + } + + public function testIncompleteAcceptedAuthorityFailsClosedBeforeEffect(): void + { + $this->catalog(); + $this->schedule(); + $call = WorkflowServiceCall::query()->sole(); + $call->forceFill([ + 'caller_principal_method' => null, + ])->save(); + + $this->assertNull(ServiceCallScheduler::claim($call->id)); + $this->assertSame(ServiceCallStatus::Failed->value, $call->fresh()->status); + $this->assertSame('service_operation_replay_authority_incomplete', $call->fresh()->outcome_reason); + $this->assertSame(ServiceCallScheduler::STATE_COMPLETED, WorkflowServiceCallDispatch::query()->sole()->state); + } + public function testTransientReplayPolicyFailureRollsBackClaimForRepair(): void { $this->catalog(); @@ -467,6 +528,174 @@ public function testExpiredCallDeadlineFencesAttempts(): void $this->assertSame(ServiceCallScheduler::STATE_COMPLETED, WorkflowServiceCallDispatch::query()->sole()->state); } + public function testCancellationAfterClaimStillFencesTheEffect(): void + { + $this->catalog(); + $this->schedule(); + $call = WorkflowServiceCall::query()->sole(); + $claim = ServiceCallScheduler::claim($call->id); + $this->assertNotNull($claim); + $request = WorkflowStub::loadRun($this->run->id) + ->attemptRequestCancellation('cancel after claim', 30); + $this->assertTrue($request->accepted()); + + /** @var WorkflowControlPlane&MockInterface $workflows */ + $workflows = Mockery::mock(WorkflowControlPlane::class); + $workflows->shouldNotReceive('update'); + (new DefaultServiceControlPlane($workflows))->executeScheduledAttempt($claim); + + $this->assertSame(ServiceCallStatus::Cancelled->value, $call->fresh()->status); + $this->assertSame('caller_cancellation_requested', $call->fresh()->outcome_reason); + $this->assertSame(ServiceCallScheduler::STATE_COMPLETED, WorkflowServiceCallDispatch::query()->sole()->state); + } + + public function testTerminalCallerAfterClaimStillFencesTheEffect(): void + { + $this->catalog(); + $this->schedule(); + $call = WorkflowServiceCall::query()->sole(); + $claim = ServiceCallScheduler::claim($call->id); + $this->assertNotNull($claim); + $this->run->forceFill([ + 'status' => RunStatus::Completed->value, + 'closed_at' => now(), + ])->save(); + + /** @var WorkflowControlPlane&MockInterface $workflows */ + $workflows = Mockery::mock(WorkflowControlPlane::class); + $workflows->shouldNotReceive('update'); + (new DefaultServiceControlPlane($workflows))->executeScheduledAttempt($claim); + + $this->assertSame(ServiceCallStatus::Cancelled->value, $call->fresh()->status); + $this->assertSame('caller_run_terminal', $call->fresh()->outcome_reason); + $this->assertSame(ServiceCallScheduler::STATE_COMPLETED, WorkflowServiceCallDispatch::query()->sole()->state); + } + + public function testDeadlineAfterClaimStillFencesTheEffect(): void + { + $this->catalog(); + $this->schedule(); + $call = WorkflowServiceCall::query()->sole(); + $claim = ServiceCallScheduler::claim($call->id); + $this->assertNotNull($claim); + $call->forceFill([ + 'call_deadline_at' => now() + ->subSecond(), + ])->save(); + + /** @var WorkflowControlPlane&MockInterface $workflows */ + $workflows = Mockery::mock(WorkflowControlPlane::class); + $workflows->shouldNotReceive('update'); + (new DefaultServiceControlPlane($workflows))->executeScheduledAttempt($claim); + + $this->assertSame(ServiceCallStatus::Failed->value, $call->fresh()->status); + $this->assertSame(ServiceCallOutcome::TimedOut, $call->fresh()->outcome); + $this->assertSame(ServiceCallScheduler::STATE_COMPLETED, WorkflowServiceCallDispatch::query()->sole()->state); + } + + public static function generatedTargetReferences(): iterable + { + yield 'workflow start' => [ServiceCallBindingKind::WorkflowRun, 'workflow_instance_id', false]; + yield 'activity' => [ServiceCallBindingKind::ActivityExecution, 'activity_execution_id', true]; + yield 'carrier' => [ServiceCallBindingKind::InvocableCarrierRequest, 'carrier_request_id', true]; + } + + #[DataProvider('generatedTargetReferences')] + public function testGeneratedEffectTargetIsFrozenAcrossCrashRedelivery( + ServiceCallBindingKind $kind, + string $option, + bool $generatedAtClaim, + ): void { + $operation = $this->catalog(); + $operation->forceFill([ + 'handler_binding_kind' => $kind->value, + 'handler_binding' => [], + 'handler_target_reference' => $kind === ServiceCallBindingKind::WorkflowRun + ? ServiceSchedulerTargetWorkflow::class + : null, + ])->save(); + $this->schedule(); + $call = WorkflowServiceCall::query()->sole(); + $scheduled = WorkflowServiceCallDispatch::query()->sole(); + if ($generatedAtClaim) { + $this->assertArrayNotHasKey($option, $scheduled->dispatch_options); + } else { + $this->assertIsString($scheduled->dispatch_options[$option] ?? null); + } + $first = ServiceCallScheduler::claim($call->id); + $this->assertNotNull($first); + $target = $first->options[$option] ?? null; + $this->assertIsString($target); + $this->assertNotSame('', $target); + $this->assertSame($target, $scheduled->fresh()->dispatch_options[$option]); + WorkflowServiceCallDispatch::query()->whereKey($call->id)->update([ + 'claim_expires_at' => now() + ->subSecond(), + ]); + + $replacement = ServiceCallScheduler::claim($call->id); + + $this->assertNotNull($replacement); + $this->assertSame($target, $replacement->options[$option]); + $this->assertSame( + ServiceCallScheduler::attemptIdentity($call->id, 1), + $replacement->options['service_call_idempotency_key'], + ); + $this->assertFalse(ServiceCallScheduler::completeAttempt($first, [], false, null)); + $this->assertSame($target, $scheduled->fresh()->dispatch_options[$option]); + } + + public function testRepairUsesConfiguredCallRelationAndDeletesOnlyAnAgedBoundedBatchOfOrphans(): void + { + config()->set('workflows.v2.service_call_dispatch.orphan_grace_seconds', 0); + $this->catalog(); + $this->schedule(); + $live = WorkflowServiceCallDispatch::query()->sole(); + $live->forceFill([ + 'created_at' => now() + ->subHours(2), + ])->save(); + foreach ([ + '01JAZZZZZZZZZZZZZZZZZZZZZ1' => now() + ->subHours(2), + '01JAZZZZZZZZZZZZZZZZZZZZZ2' => now() + ->subHours(2), + '01JAZZZZZZZZZZZZZZZZZZZZZ3' => now() + ->subMinutes(30), + ] as $id => $createdAt) { + WorkflowServiceCallDispatch::query()->create([ + 'service_call_id' => $id, + 'state' => ServiceCallScheduler::STATE_SCHEDULED, + 'dispatch_options' => [], + 'attempt' => 1, + 'max_attempts' => 1, + 'available_at' => now(), + 'claim_generation' => 0, + 'created_at' => $createdAt, + 'updated_at' => $createdAt, + ]); + } + Schema::create('scheduler_configured_service_calls', static function (Blueprint $table): void { + $table->string('id', 26) + ->primary(); + }); + DB::table('scheduler_configured_service_calls') + ->insert([ + 'id' => $live->service_call_id, + ]); + config() + ->set('workflows.v2.service_call_model', SchedulerConfiguredServiceCall::class); + + $this->assertSame(1, ServiceCallScheduler::cleanupOrphans(1)); + $this->assertFalse(WorkflowServiceCallDispatch::query()->whereKey('01JAZZZZZZZZZZZZZZZZZZZZZ1')->exists()); + $this->assertSame(1, ServiceCallScheduler::cleanupOrphans(1)); + $this->assertFalse(WorkflowServiceCallDispatch::query()->whereKey('01JAZZZZZZZZZZZZZZZZZZZZZ2')->exists()); + $this->assertSame(0, ServiceCallScheduler::cleanupOrphans(1)); + $this->assertSame(0, ServiceCallScheduler::cleanupOrphans(1)); + $this->assertTrue($live->fresh()->exists); + $this->assertTrue(WorkflowServiceCallDispatch::query()->whereKey('01JAZZZZZZZZZZZZZZZZZZZZZ3')->exists()); + } + private function schedule(?DefaultServiceBoundaryPolicy $policy = null): array { /** @var WorkflowControlPlane&MockInterface $workflows */ @@ -539,3 +768,17 @@ public function handle(): string return 'done'; } } + +#[Type('service-scheduler-target')] +final class ServiceSchedulerTargetWorkflow extends Workflow +{ + public function handle(): string + { + return 'target'; + } +} + +final class SchedulerConfiguredServiceCall extends WorkflowServiceCall +{ + protected $table = 'scheduler_configured_service_calls'; +} From c0b9f97042b1e840a7ce5aa974d04bbe0206bb68 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Mon, 26 Oct 2026 03:47:38 +0000 Subject: [PATCH 06/11] Run service scheduler tests on async queue --- tests/Unit/V2/ServiceCallEffectIdentityTest.php | 2 ++ tests/Unit/V2/ServiceCallSchedulerTest.php | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tests/Unit/V2/ServiceCallEffectIdentityTest.php b/tests/Unit/V2/ServiceCallEffectIdentityTest.php index 6dfb82b20..f7313ccfe 100644 --- a/tests/Unit/V2/ServiceCallEffectIdentityTest.php +++ b/tests/Unit/V2/ServiceCallEffectIdentityTest.php @@ -27,6 +27,8 @@ final class ServiceCallEffectIdentityTest extends TestCase protected function setUp(): void { parent::setUp(); + config() + ->set('queue.default', 'redis'); Queue::fake(); } diff --git a/tests/Unit/V2/ServiceCallSchedulerTest.php b/tests/Unit/V2/ServiceCallSchedulerTest.php index 98fc9c7fa..5fe7a49c4 100644 --- a/tests/Unit/V2/ServiceCallSchedulerTest.php +++ b/tests/Unit/V2/ServiceCallSchedulerTest.php @@ -45,6 +45,8 @@ final class ServiceCallSchedulerTest extends TestCase protected function setUp(): void { parent::setUp(); + config() + ->set('queue.default', 'redis'); Carbon::setTestNow('2026-10-26T01:30:00Z'); Queue::fake(); $workflow = WorkflowStub::make(ServiceSchedulerCallerWorkflow::class, 'scheduler-caller', 'caller'); From 336f10b969abbd27c8987072f671fdd595abfd69 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Mon, 26 Oct 2026 04:11:32 +0000 Subject: [PATCH 07/11] Reserve scheduled workflow effect identities --- .../workflow-service-calls-architecture.md | 21 +- src/V2/Models/WorkflowCommand.php | 20 +- src/V2/Support/DefaultServiceControlPlane.php | 21 +- .../Support/DefaultWorkflowControlPlane.php | 25 +-- src/V2/Support/ServiceCallEffectIdentity.php | 57 ++++- src/V2/Support/ServiceCallScheduler.php | 5 +- src/V2/WorkflowStub.php | 14 +- .../Unit/V2/ServiceCallEffectIdentityTest.php | 204 ++++++++++++++++-- tests/Unit/V2/ServiceCallSchedulerTest.php | 70 +++--- 9 files changed, 344 insertions(+), 93 deletions(-) diff --git a/docs/architecture/workflow-service-calls-architecture.md b/docs/architecture/workflow-service-calls-architecture.md index 641c9e53b..5b16b2fd7 100644 --- a/docs/architecture/workflow-service-calls-architecture.md +++ b/docs/architecture/workflow-service-calls-architecture.md @@ -233,9 +233,9 @@ violates the liveness contract. The dispatch table intentionally has no fixed foreign key because the service-call model and table are configurable. The same repair pass removes only dispatches whose configured call is missing and whose age exceeds `orphan_grace_seconds` (one hour by default and as an enforced -minimum), in deterministic, bounded primary-key batches. It rechecks absence -and age in the delete query so a concurrently restored or custom-model call is -not removed. +minimum), in deterministic primary-key batches capped at 100 rows. It rechecks +absence and age in the delete query so a concurrently restored or custom-model +call is not removed. A runnable attempt claims in one database transaction and in a fixed lock order: service-call row, dispatch row, current caller-run row, endpoint, service, then @@ -261,12 +261,15 @@ receives the stable identity `workflow-service-call::attempt:`. Handler adapters MUST propagate or otherwise enforce that identity because a lease may expire after an effect began and at-least-once recovery may invoke the attempt again. -The built-in workflow start, update, and signal adapters consume it as their -durable command request identity, so a repeated attempt returns the original -command/effect receipt rather than creating another effect. Activity and carrier -bindings currently resolve and persist a stable target reference without -launching an external effect in this control plane; a future execution lane must -consume the same attempt identity. +The built-in workflow start, update, and signal adapters receive a reserved, +in-memory capability issued only after the current attempt claim passes its +generation/token fence. They derive the durable command identity from that +capability; raw public options or command context cannot reserve the scheduler +identity. A repeated attempt returns the original command/effect receipt rather +than creating another effect. Activity and carrier bindings currently resolve +and persist a stable target reference without launching an external effect in +this control plane; a future execution lane must consume the same attempt +identity. Only the worker holding the current generation and token may persist handler progress or complete the attempt. Retry backoff is stored in `available_at`; no worker thread sleeps while holding a call, caller, catalog, or workflow-task diff --git a/src/V2/Models/WorkflowCommand.php b/src/V2/Models/WorkflowCommand.php index 2498e7ef2..1d3285ada 100644 --- a/src/V2/Models/WorkflowCommand.php +++ b/src/V2/Models/WorkflowCommand.php @@ -509,19 +509,27 @@ private static function withRequestIdentity(WorkflowInstance $instance, array $a || $commandType === CommandType::Redrive->value; $context = is_array($attributes['context'] ?? null) ? $attributes['context'] : []; $serviceCallIdentity = ServiceCallEffectIdentity::fromContext($context); + if ($serviceCallIdentity instanceof ServiceCallEffectIdentity) { + $context = $serviceCallIdentity->persistentContext($context); + $attributes['context'] = $context; + } if ($isUpdate) { - $context['request'] = is_array($context['request'] ?? null) ? $context['request'] : []; - $context['request']['request_id'] = UpdateRequestIdentity::fromContext($context); - $attributes['context'] = $context; - $attributes['request_id'] = UpdateRequestIdentity::storageToken($instance, $context); - } elseif ($serviceCallIdentity !== null && in_array($commandType, [ + if ($serviceCallIdentity instanceof ServiceCallEffectIdentity) { + $attributes['request_id'] = $serviceCallIdentity->storageToken(); + } else { + $context['request'] = is_array($context['request'] ?? null) ? $context['request'] : []; + $context['request']['request_id'] = UpdateRequestIdentity::fromContext($context); + $attributes['context'] = $context; + $attributes['request_id'] = UpdateRequestIdentity::storageToken($instance, $context); + } + } elseif ($serviceCallIdentity instanceof ServiceCallEffectIdentity && in_array($commandType, [ CommandType::Start, CommandType::Start->value, CommandType::Signal, CommandType::Signal->value, ], true)) { - $attributes['request_id'] = ServiceCallEffectIdentity::storageToken($serviceCallIdentity); + $attributes['request_id'] = $serviceCallIdentity->storageToken(); } elseif ($usesRequestId) { $request = is_array($attributes['context']['request'] ?? null) ? $attributes['context']['request'] diff --git a/src/V2/Support/DefaultServiceControlPlane.php b/src/V2/Support/DefaultServiceControlPlane.php index 830c753a4..37e7a3624 100644 --- a/src/V2/Support/DefaultServiceControlPlane.php +++ b/src/V2/Support/DefaultServiceControlPlane.php @@ -314,18 +314,21 @@ public function execute( public function executeScheduledAttempt(ServiceCallAttemptClaim $claim): void { - if (! ServiceCallScheduler::ownsClaim($claim)) { + $effectIdentity = ServiceCallEffectIdentity::forCurrentClaim($claim); + if (! $effectIdentity instanceof ServiceCallEffectIdentity) { return; } $this->scheduledClaim = $claim; try { $startedAt = now(); + $options = $claim->options; + $options[ServiceCallEffectIdentity::OPTION] = $effectIdentity; $result = $this->dispatchHandlerAttempt( $claim->call, $claim->operation, $claim->request, - $claim->options, + $options, $claim->attempt, $claim->maxAttempts, ); @@ -2185,12 +2188,15 @@ private function workflowStartOptions(WorkflowServiceCall $call, array $binding, $startOptions['payload_codec'] = $call->payload_codec; } - foreach (['service_call_id', 'service_call_attempt', 'service_call_max_attempts', - 'service_call_idempotency_key'] as $key) { + foreach (['service_call_id', 'service_call_attempt', 'service_call_max_attempts'] as $key) { if (array_key_exists($key, $options)) { $startOptions[$key] = $options[$key]; } } + $effectIdentity = ServiceCallEffectIdentity::fromOptions($options); + if ($effectIdentity instanceof ServiceCallEffectIdentity) { + $startOptions[ServiceCallEffectIdentity::OPTION] = $effectIdentity; + } return $startOptions; } @@ -2212,14 +2218,17 @@ private function workflowCommandOptions(WorkflowServiceCall $call, array $bindin } foreach (['payload_codec', 'payload_blob', 'wait_for', 'wait_timeout_seconds', - 'service_call_id', 'service_call_attempt', 'service_call_max_attempts', - 'service_call_idempotency_key'] as $key) { + 'service_call_id', 'service_call_attempt', 'service_call_max_attempts'] as $key) { $value = $options[$key] ?? $binding[$key] ?? null; if ($value !== null) { $commandOptions[$key] = $value; } } + $effectIdentity = ServiceCallEffectIdentity::fromOptions($options); + if ($effectIdentity instanceof ServiceCallEffectIdentity) { + $commandOptions[ServiceCallEffectIdentity::OPTION] = $effectIdentity; + } return $commandOptions; } diff --git a/src/V2/Support/DefaultWorkflowControlPlane.php b/src/V2/Support/DefaultWorkflowControlPlane.php index 3d343bead..578e5c341 100644 --- a/src/V2/Support/DefaultWorkflowControlPlane.php +++ b/src/V2/Support/DefaultWorkflowControlPlane.php @@ -393,9 +393,9 @@ public function start(string $workflowType, ?string $instanceId = null, array $o $fairnessKey = TaskFairnessKey::normalize($options['fairness_key'] ?? null); $fairnessWeight = TaskFairnessKey::normalizeWeight($options['fairness_weight'] ?? null); $serviceCallIdentity = ServiceCallEffectIdentity::fromOptions($options); - $serviceCallToken = $serviceCallIdentity === null + $serviceCallToken = ! $serviceCallIdentity instanceof ServiceCallEffectIdentity ? null - : ServiceCallEffectIdentity::storageToken($serviceCallIdentity); + : $serviceCallIdentity->storageToken(); $workflowClass = $resolvedClass ?? $workflowType; $externalFingerprint = is_string($options['external_workflow_definition_fingerprint'] ?? null) @@ -444,7 +444,7 @@ public function start(string $workflowType, ?string $instanceId = null, array $o if ($serviceCallToken !== null) { /** @var WorkflowCommand|null $existing */ - $existing = WorkflowCommand::query() + $existing = ConfiguredV2Models::query('command_model', WorkflowCommand::class) ->where('workflow_instance_id', $instance->id) ->where('command_type', CommandType::Start->value) ->where('request_id', $serviceCallToken) @@ -1347,24 +1347,15 @@ private function commandArguments(array $options): array */ private function commandContext(array $options): CommandContext { - $commandContext = $options['command_context'] ?? null; - - if ($commandContext instanceof CommandContext) { - return $commandContext; - } - - $commandContext = CommandContext::controlPlane(); + $commandContext = ($options['command_context'] ?? null) instanceof CommandContext + ? $options['command_context'] + : CommandContext::controlPlane(); $serviceCallIdentity = ServiceCallEffectIdentity::fromOptions($options); - return $serviceCallIdentity === null + return ! $serviceCallIdentity instanceof ServiceCallEffectIdentity ? $commandContext : $commandContext->with([ - 'request' => [ - 'request_id' => $serviceCallIdentity, - ], - 'service_call' => [ - 'idempotency_key' => $serviceCallIdentity, - ], + 'service_call' => $serviceCallIdentity->commandContext(), ]); } diff --git a/src/V2/Support/ServiceCallEffectIdentity.php b/src/V2/Support/ServiceCallEffectIdentity.php index 1773b7965..9d3876dce 100644 --- a/src/V2/Support/ServiceCallEffectIdentity.php +++ b/src/V2/Support/ServiceCallEffectIdentity.php @@ -6,33 +6,70 @@ final class ServiceCallEffectIdentity { + public const OPTION = '__service_call_effect_identity'; + + private const CONTEXT_KEY = '__effect_identity'; + + private function __construct( + private readonly string $identity, + ) { + } + + public static function forCurrentClaim(ServiceCallAttemptClaim $claim): ?self + { + return ServiceCallScheduler::ownsClaim($claim) + ? new self(ServiceCallScheduler::attemptIdentity($claim->call->id, $claim->attempt)) + : null; + } + /** * @param array $options */ - public static function fromOptions(array $options): ?string + public static function fromOptions(array $options): ?self { - return self::nonEmptyString($options['service_call_idempotency_key'] ?? null); + $identity = $options[self::OPTION] ?? null; + + return $identity instanceof self ? $identity : null; } /** * @param array $context */ - public static function fromContext(array $context): ?string + public static function fromContext(array $context): ?self { $serviceCall = $context['service_call'] ?? null; + $identity = is_array($serviceCall) ? ($serviceCall[self::CONTEXT_KEY] ?? null) : null; - return is_array($serviceCall) - ? self::nonEmptyString($serviceCall['idempotency_key'] ?? null) - : null; + return $identity instanceof self ? $identity : null; } - public static function storageToken(string $identity): string + /** + * @return array + */ + public function commandContext(): array { - return 'service-call-v1:' . hash('sha256', $identity); + return [ + 'idempotency_key' => $this->identity, + self::CONTEXT_KEY => $this, + ]; + } + + /** + * @param array $context + * @return array + */ + public function persistentContext(array $context): array + { + $serviceCall = is_array($context['service_call'] ?? null) ? $context['service_call'] : []; + unset($serviceCall[self::CONTEXT_KEY]); + $serviceCall['idempotency_key'] = $this->identity; + $context['service_call'] = $serviceCall; + + return $context; } - private static function nonEmptyString(mixed $value): ?string + public function storageToken(): string { - return is_string($value) && $value !== '' ? $value : null; + return 'service-call-v1:' . hash('sha256', $this->identity); } } diff --git a/src/V2/Support/ServiceCallScheduler.php b/src/V2/Support/ServiceCallScheduler.php index a2bae5a74..ac8e00a4f 100644 --- a/src/V2/Support/ServiceCallScheduler.php +++ b/src/V2/Support/ServiceCallScheduler.php @@ -32,6 +32,8 @@ final class ServiceCallScheduler private const MIN_ORPHAN_GRACE_SECONDS = 3_600; + private const ORPHAN_CLEANUP_BATCH_CAP = 100; + /** * @param array $options */ @@ -303,6 +305,7 @@ public static function ownsLockedClaim( ServiceCallAttemptClaim $claim, ): bool { return $dispatch->state === self::STATE_CLAIMED + && $dispatch->attempt === $claim->attempt && $dispatch->claim_generation === $claim->generation && hash_equals((string) $dispatch->claim_token, $claim->token); } @@ -436,7 +439,7 @@ public static function cleanupOrphans(int $limit): int ->whereDoesntHave('call') ->where('created_at', '<=', $cutoff) ->orderBy('service_call_id') - ->limit(max(1, $limit)) + ->limit(max(1, min($limit, self::ORPHAN_CLEANUP_BATCH_CAP))) ->pluck('service_call_id') ->all(); diff --git a/src/V2/WorkflowStub.php b/src/V2/WorkflowStub.php index 28d2524e5..f4323c0da 100644 --- a/src/V2/WorkflowStub.php +++ b/src/V2/WorkflowStub.php @@ -2704,7 +2704,15 @@ private function existingUpdateForRequest(WorkflowInstance $instance): array { $context = $this->resolvedCommandContext() ->attributes()['context'] ?? []; - $command = UpdateRequestIdentity::find($instance, $context); + $serviceCallIdentity = ServiceCallEffectIdentity::fromContext($context); + $command = $serviceCallIdentity instanceof ServiceCallEffectIdentity + ? ConfiguredV2Models::query('command_model', WorkflowCommand::class) + ->where('workflow_instance_id', $instance->id) + ->where('command_type', CommandType::Update->value) + ->where('request_id', $serviceCallIdentity->storageToken()) + ->lockForUpdate() + ->first() + : UpdateRequestIdentity::find($instance, $context); if (! $command instanceof WorkflowCommand) { return [null, null]; @@ -2887,10 +2895,10 @@ private function attemptSignalInternal( ); if ($serviceCallIdentity !== null) { /** @var WorkflowCommand|null $existing */ - $existing = WorkflowCommand::query() + $existing = ConfiguredV2Models::query('command_model', WorkflowCommand::class) ->where('workflow_instance_id', $instance->id) ->where('command_type', CommandType::Signal->value) - ->where('request_id', ServiceCallEffectIdentity::storageToken($serviceCallIdentity)) + ->where('request_id', $serviceCallIdentity->storageToken()) ->lockForUpdate() ->first(); if ($existing instanceof WorkflowCommand) { diff --git a/tests/Unit/V2/ServiceCallEffectIdentityTest.php b/tests/Unit/V2/ServiceCallEffectIdentityTest.php index f7313ccfe..3c5bb9b45 100644 --- a/tests/Unit/V2/ServiceCallEffectIdentityTest.php +++ b/tests/Unit/V2/ServiceCallEffectIdentityTest.php @@ -4,21 +4,32 @@ namespace Tests\Unit\V2; +use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Queue; use Tests\TestCase; use Workflow\UpdateMethod; use Workflow\V2\Attributes\Signal; use Workflow\V2\Attributes\Type; +use Workflow\V2\CommandContext; use Workflow\V2\Enums\CommandType; +use Workflow\V2\Enums\ServiceCallBindingKind; +use Workflow\V2\Enums\ServiceCallOperationMode; use Workflow\V2\Models\WorkflowCommand; use Workflow\V2\Models\WorkflowRun; +use Workflow\V2\Models\WorkflowService; +use Workflow\V2\Models\WorkflowServiceCall; +use Workflow\V2\Models\WorkflowServiceEndpoint; +use Workflow\V2\Models\WorkflowServiceOperation; use Workflow\V2\Models\WorkflowSignal; use Workflow\V2\Models\WorkflowTask; use Workflow\V2\Models\WorkflowUpdate; use function Workflow\V2\signal; +use Workflow\V2\Support\DefaultServiceControlPlane; use Workflow\V2\Support\DefaultWorkflowControlPlane; use Workflow\V2\Support\DefaultWorkflowTaskBridge; use Workflow\V2\Support\ServiceCallEffectIdentity; +use Workflow\V2\Support\ServiceCallPrincipal; +use Workflow\V2\Support\ServiceCallScheduler; use Workflow\V2\Workflow; use Workflow\V2\WorkflowStub; @@ -34,9 +45,11 @@ protected function setUp(): void public function testWorkflowStartConsumesTheStableServiceCallIdentity(): void { + $caller = $this->waitingWorkflow('service-effect-start-caller'); + $identity = $this->currentClaimIdentity($caller); $control = new DefaultWorkflowControlPlane(); $options = [ - 'service_call_idempotency_key' => 'workflow-service-call:call:attempt:1', + ServiceCallEffectIdentity::OPTION => $identity, ]; $first = $control->start(ServiceCallEffectWorkflow::class, 'service-effect-start', $options); @@ -45,24 +58,23 @@ public function testWorkflowStartConsumesTheStableServiceCallIdentity(): void $this->assertTrue($first['started']); $this->assertTrue($second['started']); $this->assertSame($first['workflow_run_id'], $second['workflow_run_id']); - $this->assertSame(1, WorkflowRun::query()->count()); + $this->assertSame(1, WorkflowRun::query() ->where('workflow_instance_id', 'service-effect-start') ->count()); $command = WorkflowCommand::query() + ->where('workflow_instance_id', 'service-effect-start') ->where('command_type', CommandType::Start->value) ->sole(); - $this->assertSame( - ServiceCallEffectIdentity::storageToken($options['service_call_idempotency_key']), - $command->request_id, - ); + $this->assertSame($identity->storageToken(), $command->request_id); } public function testWorkflowUpdateConsumesTheStableServiceCallIdentity(): void { $workflow = $this->waitingWorkflow('service-effect-update'); + $identity = $this->currentClaimIdentity($workflow); $control = new DefaultWorkflowControlPlane(); $options = [ 'arguments' => ['first'], 'wait_for' => 'accepted', - 'service_call_idempotency_key' => 'workflow-service-call:update:attempt:1', + ServiceCallEffectIdentity::OPTION => $identity, ]; $first = $control->update($workflow->id(), 'change', $options); @@ -75,21 +87,22 @@ public function testWorkflowUpdateConsumesTheStableServiceCallIdentity(): void $this->assertSame($first['update_id'], $second['update_id']); $this->assertSame(1, WorkflowUpdate::query()->count()); $command = WorkflowCommand::query() + ->where('workflow_instance_id', $workflow->id()) ->where('command_type', CommandType::Update->value) ->sole(); - $this->assertSame($options['service_call_idempotency_key'], $command->requestId()); - $this->assertSame( - $options['service_call_idempotency_key'], - $command->context['service_call']['idempotency_key'], - ); + $this->assertSame($identity->storageToken(), $command->requestId()); + $auditIdentity = $identity->commandContext()['idempotency_key']; + $this->assertSame($auditIdentity, $command->context['service_call']['idempotency_key']); + $this->assertCount(1, $command->context['service_call']); } public function testWorkflowSignalConsumesTheStableServiceCallIdentity(): void { $workflow = $this->waitingWorkflow('service-effect-signal'); + $identity = $this->currentClaimIdentity($workflow); $control = new DefaultWorkflowControlPlane(); $options = [ - 'service_call_idempotency_key' => 'workflow-service-call:signal:attempt:1', + ServiceCallEffectIdentity::OPTION => $identity, ]; $first = $control->signal($workflow->id(), 'finish', $options); @@ -100,12 +113,169 @@ public function testWorkflowSignalConsumesTheStableServiceCallIdentity(): void $this->assertSame($first['workflow_command_id'], $second['workflow_command_id']); $this->assertSame(1, WorkflowSignal::query()->count()); $command = WorkflowCommand::query() + ->where('workflow_instance_id', $workflow->id()) ->where('command_type', CommandType::Signal->value) ->sole(); - $this->assertSame( - ServiceCallEffectIdentity::storageToken($options['service_call_idempotency_key']), - $command->request_id, - ); + $this->assertSame($identity->storageToken(), $command->request_id); + } + + public function testRawDirectOptionsCannotClaimReservedStartUpdateOrSignalIdentity(): void + { + $rawIdentity = 'workflow-service-call:predictable:attempt:1'; + $rawOptions = [ + 'service_call_idempotency_key' => $rawIdentity, + ServiceCallEffectIdentity::OPTION => [ + 'idempotency_key' => $rawIdentity, + ], + ]; + $control = new DefaultWorkflowControlPlane(); + + $start = $control->start(ServiceCallEffectWorkflow::class, 'raw-effect-start', $rawOptions); + $duplicateStart = $control->start(ServiceCallEffectWorkflow::class, 'raw-effect-start', $rawOptions); + $this->assertTrue($start['started']); + $this->assertFalse($duplicateStart['started']); + $startCommands = WorkflowCommand::query() + ->where('workflow_instance_id', 'raw-effect-start') + ->where('command_type', CommandType::Start->value) + ->get(); + $this->assertCount(2, $startCommands); + $this->assertTrue($startCommands->every(static fn (WorkflowCommand $command): bool => + $command->request_id === null)); + + $updateWorkflow = $this->waitingWorkflow('raw-effect-update'); + $updateOptions = $rawOptions + [ + 'arguments' => ['first'], + 'wait_for' => 'accepted', + ]; + $firstUpdate = $control->update($updateWorkflow->id(), 'change', $updateOptions); + $secondUpdate = $control->update($updateWorkflow->id(), 'change', array_replace($updateOptions, [ + 'arguments' => ['second'], + ])); + $this->assertNotSame($firstUpdate['update_id'], $secondUpdate['update_id']); + $this->assertSame(2, WorkflowUpdate::query()->count()); + + $signalWorkflow = $this->waitingWorkflow('raw-effect-signal'); + $firstSignal = $control->signal($signalWorkflow->id(), 'finish', $rawOptions); + $secondSignal = $control->signal($signalWorkflow->id(), 'finish', $rawOptions); + $this->assertNotSame($firstSignal['workflow_command_id'], $secondSignal['workflow_command_id']); + $this->assertSame(2, WorkflowSignal::query()->count()); + } + + public function testCraftedCommandContextCannotClaimReservedSignalIdentity(): void + { + $workflow = $this->waitingWorkflow('context-effect-signal'); + $context = CommandContext::controlPlane() + ->with([ + 'service_call' => [ + 'idempotency_key' => 'workflow-service-call:predictable:attempt:1', + ], + ]); + $control = new DefaultWorkflowControlPlane(); + + $first = $control->signal($workflow->id(), 'finish', [ + 'command_context' => $context, + ]); + $second = $control->signal($workflow->id(), 'finish', [ + 'command_context' => $context, + ]); + + $this->assertNotSame($first['workflow_command_id'], $second['workflow_command_id']); + $commands = WorkflowCommand::query() + ->where('workflow_instance_id', $workflow->id()) + ->where('command_type', CommandType::Signal->value) + ->get(); + $this->assertCount(2, $commands); + $this->assertTrue($commands->every(static fn (WorkflowCommand $command): bool => + $command->request_id === null)); + } + + public function testPredictableRawIdentityCannotPreseedTheScheduledUpdateReceipt(): void + { + $workflow = $this->waitingWorkflow('preseed-effect-update'); + $identity = $this->currentClaimIdentity($workflow); + $rawIdentity = $identity->commandContext()['idempotency_key']; + $attackerContext = CommandContext::controlPlane() + ->with([ + 'request' => [ + 'request_id' => $rawIdentity, + ], + 'service_call' => [ + 'idempotency_key' => $rawIdentity, + ], + ]); + $control = new DefaultWorkflowControlPlane(); + $attacker = $control->update($workflow->id(), 'change', [ + 'arguments' => ['attacker'], + 'wait_for' => 'accepted', + 'command_context' => $attackerContext, + ]); + $scheduledOptions = [ + 'arguments' => ['legitimate'], + 'wait_for' => 'accepted', + ServiceCallEffectIdentity::OPTION => $identity, + ]; + + $legitimate = $control->update($workflow->id(), 'change', $scheduledOptions); + $retry = $control->update($workflow->id(), 'change', array_replace($scheduledOptions, [ + 'arguments' => ['ignored retry'], + ])); + + $this->assertNotSame($attacker['update_id'], $legitimate['update_id']); + $this->assertSame($legitimate['update_id'], $retry['update_id']); + $this->assertSame(2, WorkflowUpdate::query()->count()); + $tokens = WorkflowCommand::query() + ->where('workflow_instance_id', $workflow->id()) + ->where('command_type', CommandType::Update->value) + ->orderBy('created_at') + ->pluck('request_id') + ->all(); + $this->assertCount(2, $tokens); + $this->assertStringStartsWith('update-v1:', $tokens[0]); + $this->assertSame($identity->storageToken(), $tokens[1]); + } + + private function currentClaimIdentity(WorkflowStub $caller): ServiceCallEffectIdentity + { + $run = WorkflowRun::query()->findOrFail($caller->runId()); + $endpoint = WorkflowServiceEndpoint::query()->create([ + 'namespace' => 'billing', + 'endpoint_name' => 'billing', + ]); + $service = WorkflowService::query()->create([ + 'namespace' => 'billing', + 'workflow_service_endpoint_id' => $endpoint->id, + 'service_name' => 'effects', + ]); + WorkflowServiceOperation::query()->create([ + 'namespace' => 'billing', + 'workflow_service_endpoint_id' => $endpoint->id, + 'workflow_service_id' => $service->id, + 'operation_name' => 'apply', + 'operation_mode' => ServiceCallOperationMode::Sync->value, + 'handler_binding_kind' => ServiceCallBindingKind::WorkflowUpdate->value, + 'handler_binding' => [ + 'workflow_instance_id' => $caller->id(), + 'update_name' => 'change', + ], + 'handler_target_reference' => 'change', + ]); + $control = new DefaultServiceControlPlane(new DefaultWorkflowControlPlane()); + DB::transaction(static fn (): array => $control->execute('billing', 'effects', 'apply', [ + 'namespace' => 'billing', + 'caller_namespace' => $run->namespace, + 'caller_workflow_instance_id' => $caller->id(), + 'caller_workflow_run_id' => $run->id, + 'principal' => new ServiceCallPrincipal(subject: 'workflow:' . $run->id, method: 'workflow'), + 'idempotency_key' => 'effect-identity-' . $caller->id(), + 'wait_for' => 'completed', + 'wait_timeout_seconds' => 60, + ])); + $claim = ServiceCallScheduler::claim(WorkflowServiceCall::query()->sole()->id); + $this->assertNotNull($claim); + $identity = ServiceCallEffectIdentity::forCurrentClaim($claim); + $this->assertInstanceOf(ServiceCallEffectIdentity::class, $identity); + + return $identity; } private function waitingWorkflow(string $instanceId): WorkflowStub diff --git a/tests/Unit/V2/ServiceCallSchedulerTest.php b/tests/Unit/V2/ServiceCallSchedulerTest.php index 5fe7a49c4..c72f93e00 100644 --- a/tests/Unit/V2/ServiceCallSchedulerTest.php +++ b/tests/Unit/V2/ServiceCallSchedulerTest.php @@ -33,6 +33,7 @@ use Workflow\V2\Models\WorkflowServiceOperation; use Workflow\V2\Support\DefaultServiceBoundaryPolicy; use Workflow\V2\Support\DefaultServiceControlPlane; +use Workflow\V2\Support\ServiceCallEffectIdentity; use Workflow\V2\Support\ServiceCallPrincipal; use Workflow\V2\Support\ServiceCallScheduler; use Workflow\V2\Workflow; @@ -187,10 +188,12 @@ public function testClaimUsesCurrentPolicyButExecutesTheImmutableAcceptedBinding && $options['service_call_id'] === $claim->call->id && $options['service_call_attempt'] === 1 && $options['service_call_max_attempts'] === 1 - && $options['service_call_idempotency_key'] === ServiceCallScheduler::attemptIdentity( - $claim->call->id, - 1, - ); + && ! array_key_exists('service_call_idempotency_key', $options) + && ServiceCallEffectIdentity::fromOptions($options)?->storageToken() + === 'service-call-v1:' . hash('sha256', ServiceCallScheduler::attemptIdentity( + $claim->call->id, + 1, + )); }) ->andReturn([ 'accepted' => true, @@ -322,6 +325,11 @@ public function testExpiredClaimCanBeRepairedAndAStaleWorkerCannotAdvanceIt(): v $first->options['service_call_idempotency_key'], $second->options['service_call_idempotency_key'], ); + $this->assertNull(ServiceCallEffectIdentity::forCurrentClaim($first)); + $this->assertInstanceOf( + ServiceCallEffectIdentity::class, + ServiceCallEffectIdentity::forCurrentClaim($second), + ); /** @var WorkflowControlPlane&MockInterface $workflows */ $workflows = Mockery::mock(WorkflowControlPlane::class); @@ -657,26 +665,40 @@ public function testRepairUsesConfiguredCallRelationAndDeletesOnlyAnAgedBoundedB 'created_at' => now() ->subHours(2), ])->save(); - foreach ([ - '01JAZZZZZZZZZZZZZZZZZZZZZ1' => now() - ->subHours(2), - '01JAZZZZZZZZZZZZZZZZZZZZZ2' => now() - ->subHours(2), - '01JAZZZZZZZZZZZZZZZZZZZZZ3' => now() - ->subMinutes(30), - ] as $id => $createdAt) { - WorkflowServiceCallDispatch::query()->create([ + $orphanIds = []; + $rows = []; + $old = now() + ->subHours(2); + foreach (range(1, 101) as $sequence) { + $id = '01JB' . str_pad((string) $sequence, 22, '0', STR_PAD_LEFT); + $orphanIds[] = $id; + $rows[] = [ 'service_call_id' => $id, 'state' => ServiceCallScheduler::STATE_SCHEDULED, - 'dispatch_options' => [], + 'dispatch_options' => '[]', 'attempt' => 1, 'max_attempts' => 1, 'available_at' => now(), 'claim_generation' => 0, - 'created_at' => $createdAt, - 'updated_at' => $createdAt, - ]); + 'created_at' => $old, + 'updated_at' => $old, + ]; } + $recentId = '01JC' . str_repeat('0', 22); + $recent = now() + ->subMinutes(30); + $rows[] = [ + 'service_call_id' => $recentId, + 'state' => ServiceCallScheduler::STATE_SCHEDULED, + 'dispatch_options' => '[]', + 'attempt' => 1, + 'max_attempts' => 1, + 'available_at' => now(), + 'claim_generation' => 0, + 'created_at' => $recent, + 'updated_at' => $recent, + ]; + WorkflowServiceCallDispatch::query()->insert($rows); Schema::create('scheduler_configured_service_calls', static function (Blueprint $table): void { $table->string('id', 26) ->primary(); @@ -688,14 +710,14 @@ public function testRepairUsesConfiguredCallRelationAndDeletesOnlyAnAgedBoundedB config() ->set('workflows.v2.service_call_model', SchedulerConfiguredServiceCall::class); - $this->assertSame(1, ServiceCallScheduler::cleanupOrphans(1)); - $this->assertFalse(WorkflowServiceCallDispatch::query()->whereKey('01JAZZZZZZZZZZZZZZZZZZZZZ1')->exists()); - $this->assertSame(1, ServiceCallScheduler::cleanupOrphans(1)); - $this->assertFalse(WorkflowServiceCallDispatch::query()->whereKey('01JAZZZZZZZZZZZZZZZZZZZZZ2')->exists()); - $this->assertSame(0, ServiceCallScheduler::cleanupOrphans(1)); - $this->assertSame(0, ServiceCallScheduler::cleanupOrphans(1)); + $this->assertSame(100, ServiceCallScheduler::cleanupOrphans(PHP_INT_MAX)); + $this->assertFalse(WorkflowServiceCallDispatch::query()->whereKey($orphanIds[0])->exists()); + $this->assertTrue(WorkflowServiceCallDispatch::query()->whereKey($orphanIds[100])->exists()); + $this->assertSame(1, ServiceCallScheduler::cleanupOrphans(PHP_INT_MAX)); + $this->assertFalse(WorkflowServiceCallDispatch::query()->whereKey($orphanIds[100])->exists()); + $this->assertSame(0, ServiceCallScheduler::cleanupOrphans(PHP_INT_MAX)); $this->assertTrue($live->fresh()->exists); - $this->assertTrue(WorkflowServiceCallDispatch::query()->whereKey('01JAZZZZZZZZZZZZZZZZZZZZZ3')->exists()); + $this->assertTrue(WorkflowServiceCallDispatch::query()->whereKey($recentId)->exists()); } private function schedule(?DefaultServiceBoundaryPolicy $policy = null): array From 64abfc0dac01a420d5f19bd9107e45c4ed40ca7f Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Mon, 26 Oct 2026 04:21:56 +0000 Subject: [PATCH 08/11] Keep service effect capabilities opaque across recovery --- .../workflow-service-calls-architecture.md | 17 ++- src/V2/Support/ServiceCallAttemptClaim.php | 52 ++++++- src/V2/Support/ServiceCallEffectIdentity.php | 51 +++++-- src/V2/Support/ServiceCallScheduler.php | 33 ++++- .../Unit/V2/ServiceCallEffectIdentityTest.php | 128 +++++++++++++++++- tests/Unit/V2/ServiceCallSchedulerTest.php | 40 ++++-- 6 files changed, 292 insertions(+), 29 deletions(-) diff --git a/docs/architecture/workflow-service-calls-architecture.md b/docs/architecture/workflow-service-calls-architecture.md index 5b16b2fd7..9ccd92137 100644 --- a/docs/architecture/workflow-service-calls-architecture.md +++ b/docs/architecture/workflow-service-calls-architecture.md @@ -263,13 +263,16 @@ MUST propagate or otherwise enforce that identity because a lease may expire after an effect began and at-least-once recovery may invoke the attempt again. The built-in workflow start, update, and signal adapters receive a reserved, in-memory capability issued only after the current attempt claim passes its -generation/token fence. They derive the durable command identity from that -capability; raw public options or command context cannot reserve the scheduler -identity. A repeated attempt returns the original command/effect receipt rather -than creating another effect. Activity and carrier bindings currently resolve -and persist a stable target reference without launching an external effect in -this control plane; a future execution lane must consume the same attempt -identity. +generation/token fence. Its opaque, scheduler-generated command token is stable +across claim generations for one attempt and changes for the next retry attempt. +The capability cannot be serialized or meaningfully debugged, and only its +non-secret audit identity is retained in command context. Raw public options, +command context, and low-level request ids cannot reserve the scheduler token. +A repeated or crash-reacquired attempt therefore returns the original +command/effect receipt rather than creating another effect. Activity and carrier +bindings currently resolve and persist a stable target reference without +launching an external effect in this control plane; a future execution lane must +consume the same attempt identity. Only the worker holding the current generation and token may persist handler progress or complete the attempt. Retry backoff is stored in `available_at`; no worker thread sleeps while holding a call, caller, catalog, or workflow-task diff --git a/src/V2/Support/ServiceCallAttemptClaim.php b/src/V2/Support/ServiceCallAttemptClaim.php index 762ad225f..5027398c5 100644 --- a/src/V2/Support/ServiceCallAttemptClaim.php +++ b/src/V2/Support/ServiceCallAttemptClaim.php @@ -4,10 +4,12 @@ namespace Workflow\V2\Support; +use JsonSerializable; +use LogicException; use Workflow\V2\Models\WorkflowServiceCall; use Workflow\V2\Models\WorkflowServiceOperation; -final class ServiceCallAttemptClaim +final class ServiceCallAttemptClaim implements JsonSerializable { /** * @param array $options @@ -23,6 +25,54 @@ public function __construct( public readonly string $token, public readonly string $catalogRevision, public readonly string $policyRevision, + #[\SensitiveParameter] + private readonly string $effectToken, ) { } + + /** + * @return array + */ + public function __debugInfo(): array + { + return [ + 'call' => $this->call, + 'operation' => $this->operation, + 'request' => $this->request, + 'options' => $this->options, + 'attempt' => $this->attempt, + 'maxAttempts' => $this->maxAttempts, + 'generation' => $this->generation, + 'token' => '[redacted]', + 'catalogRevision' => $this->catalogRevision, + 'policyRevision' => $this->policyRevision, + 'effectToken' => '[redacted]', + ]; + } + + /** + * @return array + */ + public function __serialize(): array + { + throw new LogicException('Service-call attempt claims cannot be serialized.'); + } + + public function effectToken(): string + { + return $this->effectToken; + } + + /** + * @return array{service_call_id: string, attempt: int, max_attempts: int, generation: int} + */ + public function jsonSerialize(): array + { + return [ + 'service_call_id' => $this->call->id, + 'attempt' => $this->attempt, + 'max_attempts' => $this->maxAttempts, + 'generation' => $this->generation, + ]; + } } diff --git a/src/V2/Support/ServiceCallEffectIdentity.php b/src/V2/Support/ServiceCallEffectIdentity.php index 9d3876dce..26a984ddc 100644 --- a/src/V2/Support/ServiceCallEffectIdentity.php +++ b/src/V2/Support/ServiceCallEffectIdentity.php @@ -4,22 +4,52 @@ namespace Workflow\V2\Support; -final class ServiceCallEffectIdentity +use JsonSerializable; +use LogicException; + +final class ServiceCallEffectIdentity implements JsonSerializable { public const OPTION = '__service_call_effect_identity'; private const CONTEXT_KEY = '__effect_identity'; private function __construct( - private readonly string $identity, + private readonly string $auditIdentity, + #[\SensitiveParameter] + private readonly string $storageToken, ) { } + /** + * @return array{capability: string} + */ + public function __debugInfo(): array + { + return [ + 'capability' => '[opaque]', + ]; + } + + /** + * @return array + */ + public function __serialize(): array + { + throw new LogicException('Service-call effect identities cannot be serialized.'); + } + public static function forCurrentClaim(ServiceCallAttemptClaim $claim): ?self { - return ServiceCallScheduler::ownsClaim($claim) - ? new self(ServiceCallScheduler::attemptIdentity($claim->call->id, $claim->attempt)) - : null; + if (! ServiceCallScheduler::ownsClaim($claim)) { + return null; + } + + $token = $claim->effectToken(); + if (preg_match('/^service-call-v1:[a-f0-9]{64}$/D', $token) !== 1) { + return null; + } + + return new self(ServiceCallScheduler::attemptIdentity($claim->call->id, $claim->attempt), $token); } /** @@ -49,7 +79,7 @@ public static function fromContext(array $context): ?self public function commandContext(): array { return [ - 'idempotency_key' => $this->identity, + 'idempotency_key' => $this->auditIdentity, self::CONTEXT_KEY => $this, ]; } @@ -62,7 +92,7 @@ public function persistentContext(array $context): array { $serviceCall = is_array($context['service_call'] ?? null) ? $context['service_call'] : []; unset($serviceCall[self::CONTEXT_KEY]); - $serviceCall['idempotency_key'] = $this->identity; + $serviceCall['idempotency_key'] = $this->auditIdentity; $context['service_call'] = $serviceCall; return $context; @@ -70,6 +100,11 @@ public function persistentContext(array $context): array public function storageToken(): string { - return 'service-call-v1:' . hash('sha256', $this->identity); + return $this->storageToken; + } + + public function jsonSerialize(): never + { + throw new LogicException('Service-call effect identities cannot be JSON serialized.'); } } diff --git a/src/V2/Support/ServiceCallScheduler.php b/src/V2/Support/ServiceCallScheduler.php index ac8e00a4f..82dd94857 100644 --- a/src/V2/Support/ServiceCallScheduler.php +++ b/src/V2/Support/ServiceCallScheduler.php @@ -34,6 +34,8 @@ final class ServiceCallScheduler private const ORPHAN_CLEANUP_BATCH_CAP = 100; + private const EFFECT_TOKEN_OPTION = '__service_call_effect_token'; + /** * @param array $options */ @@ -193,7 +195,7 @@ public static function claim(string $serviceCallId): ?ServiceCallAttemptClaim $token = hash('sha256', (string) Str::uuid()); $now = now(); $options = is_array($dispatch->dispatch_options) ? $dispatch->dispatch_options : []; - $options = self::withClaimTargetIdentifiers($options); + $options = self::withEffectToken(self::withClaimTargetIdentifiers($options)); $dispatch->forceFill([ 'state' => self::STATE_CLAIMED, 'dispatch_options' => $options, @@ -208,6 +210,8 @@ public static function claim(string $serviceCallId): ?ServiceCallAttemptClaim 'last_error' => null, ])->save(); + $effectToken = $options[self::EFFECT_TOKEN_OPTION]; + unset($options[self::EFFECT_TOKEN_OPTION]); $options['service_call_id'] = $call->id; $options['service_call_attempt'] = $dispatch->attempt; $options['service_call_max_attempts'] = $dispatch->max_attempts; @@ -232,6 +236,7 @@ public static function claim(string $serviceCallId): ?ServiceCallAttemptClaim token: $token, catalogRevision: $catalogRevision, policyRevision: $policyRevision, + effectToken: $effectToken, ); }); } @@ -272,8 +277,11 @@ public static function completeAttempt( if ($retry && ! self::terminal($call) && $dispatch->attempt < $dispatch->max_attempts) { $availableAt = now() ->addMilliseconds((int) ceil(max(0.0, $backoffSeconds ?? 0.0) * 1000)); + $options = is_array($dispatch->dispatch_options) ? $dispatch->dispatch_options : []; + $options[self::EFFECT_TOKEN_OPTION] = self::newEffectToken(); $dispatch->forceFill([ 'state' => self::STATE_SCHEDULED, + 'dispatch_options' => $options, 'attempt' => $dispatch->attempt + 1, 'available_at' => $availableAt, 'claim_token' => null, @@ -619,8 +627,11 @@ private static function failAuthorization( private static function completeDispatch(WorkflowServiceCallDispatch $dispatch): void { + $options = is_array($dispatch->dispatch_options) ? $dispatch->dispatch_options : []; + unset($options[self::EFFECT_TOKEN_OPTION]); $dispatch->forceFill([ 'state' => self::STATE_COMPLETED, + 'dispatch_options' => $options, 'claim_token' => null, 'claimed_at' => null, 'claim_expires_at' => null, @@ -700,6 +711,7 @@ private static function persistableOptions(WorkflowServiceOperation $operation, $persisted['accepted_handler_binding'] = $binding; $persisted['accepted_handler_target_reference'] = $operation->handler_target_reference; $persisted['accepted_operation_mode'] = $operation->operation_mode; + $persisted[self::EFFECT_TOKEN_OPTION] = self::newEffectToken(); $kind = ServiceCallBindingKind::tryFromHandlerBindingKind((string) $operation->handler_binding_kind); if ($kind === ServiceCallBindingKind::WorkflowRun @@ -711,6 +723,25 @@ private static function persistableOptions(WorkflowServiceOperation $operation, return $persisted; } + /** + * @param array $options + * @return array + */ + private static function withEffectToken(array $options): array + { + $token = $options[self::EFFECT_TOKEN_OPTION] ?? null; + if (! is_string($token) || preg_match('/^service-call-v1:[a-f0-9]{64}$/D', $token) !== 1) { + $options[self::EFFECT_TOKEN_OPTION] = self::newEffectToken(); + } + + return $options; + } + + private static function newEffectToken(): string + { + return 'service-call-v1:' . bin2hex(random_bytes(32)); + } + /** * @param array $options * @return array diff --git a/tests/Unit/V2/ServiceCallEffectIdentityTest.php b/tests/Unit/V2/ServiceCallEffectIdentityTest.php index 3c5bb9b45..170366cfc 100644 --- a/tests/Unit/V2/ServiceCallEffectIdentityTest.php +++ b/tests/Unit/V2/ServiceCallEffectIdentityTest.php @@ -6,18 +6,22 @@ use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Queue; +use LogicException; use Tests\TestCase; use Workflow\UpdateMethod; use Workflow\V2\Attributes\Signal; use Workflow\V2\Attributes\Type; use Workflow\V2\CommandContext; +use Workflow\V2\Enums\CommandStatus; use Workflow\V2\Enums\CommandType; use Workflow\V2\Enums\ServiceCallBindingKind; use Workflow\V2\Enums\ServiceCallOperationMode; use Workflow\V2\Models\WorkflowCommand; +use Workflow\V2\Models\WorkflowInstance; use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowService; use Workflow\V2\Models\WorkflowServiceCall; +use Workflow\V2\Models\WorkflowServiceCallDispatch; use Workflow\V2\Models\WorkflowServiceEndpoint; use Workflow\V2\Models\WorkflowServiceOperation; use Workflow\V2\Models\WorkflowSignal; @@ -27,6 +31,7 @@ use Workflow\V2\Support\DefaultServiceControlPlane; use Workflow\V2\Support\DefaultWorkflowControlPlane; use Workflow\V2\Support\DefaultWorkflowTaskBridge; +use Workflow\V2\Support\ServiceCallAttemptClaim; use Workflow\V2\Support\ServiceCallEffectIdentity; use Workflow\V2\Support\ServiceCallPrincipal; use Workflow\V2\Support\ServiceCallScheduler; @@ -124,6 +129,7 @@ public function testRawDirectOptionsCannotClaimReservedStartUpdateOrSignalIdenti $rawIdentity = 'workflow-service-call:predictable:attempt:1'; $rawOptions = [ 'service_call_idempotency_key' => $rawIdentity, + 'request_id' => 'service-call-v1:' . hash('sha256', $rawIdentity), ServiceCallEffectIdentity::OPTION => [ 'idempotency_key' => $rawIdentity, ], @@ -234,7 +240,127 @@ public function testPredictableRawIdentityCannotPreseedTheScheduledUpdateReceipt $this->assertSame($identity->storageToken(), $tokens[1]); } + public function testLowLevelCommandRequestIdCannotReserveTheOldCanonicalIdentity(): void + { + $workflow = $this->waitingWorkflow('low-level-effect-injection'); + $instance = WorkflowInstance::query()->findOrFail($workflow->id()); + $run = WorkflowRun::query()->findOrFail($workflow->runId()); + $rawIdentity = ServiceCallScheduler::attemptIdentity('01JATTACKERCONTROLLED00000', 1); + $guessedReservedToken = 'service-call-v1:' . hash('sha256', $rawIdentity); + $context = [ + 'request' => [ + 'request_id' => $rawIdentity, + ], + 'service_call' => [ + 'idempotency_key' => $rawIdentity, + ], + ]; + + $start = WorkflowCommand::record($instance, $run, [ + 'command_type' => CommandType::Start->value, + 'status' => CommandStatus::Accepted->value, + 'context' => $context, + 'request_id' => $guessedReservedToken, + ]); + $update = WorkflowCommand::record($instance, $run, [ + 'command_type' => CommandType::Update->value, + 'status' => CommandStatus::Accepted->value, + 'context' => $context, + 'request_id' => $guessedReservedToken, + ]); + $signal = WorkflowCommand::record($instance, $run, [ + 'command_type' => CommandType::Signal->value, + 'status' => CommandStatus::Accepted->value, + 'context' => $context, + 'request_id' => $guessedReservedToken, + ]); + + $this->assertNull($start->request_id); + $this->assertStringStartsWith('update-v1:', $update->request_id); + $this->assertNotSame($guessedReservedToken, $update->request_id); + $this->assertNull($signal->request_id); + } + + public function testEffectCommittedBeforeClaimReacquisitionKeepsOneOpaqueReceipt(): void + { + $workflow = $this->waitingWorkflow('reacquired-effect-update'); + [$firstClaim, $firstIdentity] = $this->currentClaim($workflow); + $control = new DefaultWorkflowControlPlane(); + $auditIdentity = $firstIdentity->commandContext()['idempotency_key']; + $predictableToken = 'service-call-v1:' . hash('sha256', $auditIdentity); + $first = $control->update($workflow->id(), 'change', [ + 'arguments' => ['committed before crash'], + 'wait_for' => 'accepted', + ServiceCallEffectIdentity::OPTION => $firstIdentity, + ]); + + $this->assertNotSame($predictableToken, $firstIdentity->storageToken()); + $identityDebug = print_r($firstIdentity, true); + $this->assertStringNotContainsString($auditIdentity, $identityDebug); + $this->assertStringNotContainsString($firstIdentity->storageToken(), $identityDebug); + $this->assertStringNotContainsString($firstClaim->token, $identityDebug); + try { + json_encode($firstIdentity, JSON_THROW_ON_ERROR); + $this->fail('The effect identity capability must not be JSON serializable.'); + } catch (LogicException $failure) { + $this->assertSame('Service-call effect identities cannot be JSON serialized.', $failure->getMessage()); + } + try { + serialize($firstIdentity); + $this->fail('The effect identity capability must not be serializable.'); + } catch (LogicException $failure) { + $this->assertSame('Service-call effect identities cannot be serialized.', $failure->getMessage()); + } + $claimDebug = print_r($firstClaim, true); + $this->assertStringNotContainsString($firstClaim->token, $claimDebug); + $this->assertStringNotContainsString($firstIdentity->storageToken(), $claimDebug); + $this->assertStringNotContainsString($firstClaim->token, json_encode($firstClaim, JSON_THROW_ON_ERROR)); + try { + serialize($firstClaim); + $this->fail('The service-call attempt claim must not be serializable.'); + } catch (LogicException $failure) { + $this->assertSame('Service-call attempt claims cannot be serialized.', $failure->getMessage()); + } + + // The target effect committed, but the service-call result did not. + WorkflowServiceCallDispatch::query()->whereKey($firstClaim->call->id)->update([ + 'claim_expires_at' => now() + ->subSecond(), + ]); + $replacement = ServiceCallScheduler::claim($firstClaim->call->id); + $this->assertNotNull($replacement); + $this->assertNotSame($firstClaim->token, $replacement->token); + $replacementIdentity = ServiceCallEffectIdentity::forCurrentClaim($replacement); + $this->assertInstanceOf(ServiceCallEffectIdentity::class, $replacementIdentity); + $this->assertSame($firstIdentity->storageToken(), $replacementIdentity->storageToken()); + + $retry = $control->update($workflow->id(), 'change', [ + 'arguments' => ['must not replace committed arguments'], + 'wait_for' => 'accepted', + ServiceCallEffectIdentity::OPTION => $replacementIdentity, + ]); + + $this->assertSame($first['update_id'], $retry['update_id']); + $this->assertSame(1, WorkflowUpdate::query()->count()); + $command = WorkflowCommand::query() + ->where('workflow_instance_id', $workflow->id()) + ->where('command_type', CommandType::Update->value) + ->sole(); + $this->assertSame($replacementIdentity->storageToken(), $command->request_id); + $this->assertSame($auditIdentity, $command->context['service_call']['idempotency_key']); + $this->assertStringNotContainsString($firstClaim->token, json_encode($command->toArray(), JSON_THROW_ON_ERROR)); + $this->assertFalse(ServiceCallScheduler::completeAttempt($firstClaim, [], false, null)); + } + private function currentClaimIdentity(WorkflowStub $caller): ServiceCallEffectIdentity + { + return $this->currentClaim($caller)[1]; + } + + /** + * @return array{ServiceCallAttemptClaim, ServiceCallEffectIdentity} + */ + private function currentClaim(WorkflowStub $caller): array { $run = WorkflowRun::query()->findOrFail($caller->runId()); $endpoint = WorkflowServiceEndpoint::query()->create([ @@ -275,7 +401,7 @@ private function currentClaimIdentity(WorkflowStub $caller): ServiceCallEffectId $identity = ServiceCallEffectIdentity::forCurrentClaim($claim); $this->assertInstanceOf(ServiceCallEffectIdentity::class, $identity); - return $identity; + return [$claim, $identity]; } private function waitingWorkflow(string $instanceId): WorkflowStub diff --git a/tests/Unit/V2/ServiceCallSchedulerTest.php b/tests/Unit/V2/ServiceCallSchedulerTest.php index c72f93e00..d1b5339c8 100644 --- a/tests/Unit/V2/ServiceCallSchedulerTest.php +++ b/tests/Unit/V2/ServiceCallSchedulerTest.php @@ -175,12 +175,23 @@ public function testClaimUsesCurrentPolicyButExecutesTheImmutableAcceptedBinding $claimedDispatch = WorkflowServiceCallDispatch::query()->sole(); $this->assertSame($claim->catalogRevision, $claimedDispatch->authorized_catalog_revision); $this->assertSame($claim->policyRevision, $claimedDispatch->authorized_policy_revision); + $effectIdentity = ServiceCallEffectIdentity::forCurrentClaim($claim); + $this->assertInstanceOf(ServiceCallEffectIdentity::class, $effectIdentity); + $effectToken = $effectIdentity->storageToken(); + $this->assertStringStartsWith('service-call-v1:', $effectToken); + $this->assertNotSame( + 'service-call-v1:' . hash('sha256', ServiceCallScheduler::attemptIdentity($claim->call->id, 1)), + $effectToken, + ); /** @var WorkflowControlPlane&MockInterface $workflows */ $workflows = Mockery::mock(WorkflowControlPlane::class); $workflows->shouldReceive('update') ->once() - ->withArgs(static function (string $instanceId, string $name, array $options) use ($claim): bool { + ->withArgs(static function (string $instanceId, string $name, array $options) use ( + $claim, + $effectToken, + ): bool { return DB::connection()->transactionLevel() === 0 && $instanceId === 'accepted-target' && $name === 'approve' @@ -189,11 +200,7 @@ public function testClaimUsesCurrentPolicyButExecutesTheImmutableAcceptedBinding && $options['service_call_attempt'] === 1 && $options['service_call_max_attempts'] === 1 && ! array_key_exists('service_call_idempotency_key', $options) - && ServiceCallEffectIdentity::fromOptions($options)?->storageToken() - === 'service-call-v1:' . hash('sha256', ServiceCallScheduler::attemptIdentity( - $claim->call->id, - 1, - )); + && ServiceCallEffectIdentity::fromOptions($options)?->storageToken() === $effectToken; }) ->andReturn([ 'accepted' => true, @@ -225,6 +232,8 @@ public function testRetryBackoffIsPersistedWithoutSleepingOrHoldingTheCompletion $call = WorkflowServiceCall::query()->sole(); $claim = ServiceCallScheduler::claim($call->id); $this->assertNotNull($claim); + $firstIdentity = ServiceCallEffectIdentity::forCurrentClaim($claim); + $this->assertInstanceOf(ServiceCallEffectIdentity::class, $firstIdentity); Queue::fake(); /** @var WorkflowControlPlane&MockInterface $workflows */ @@ -251,12 +260,20 @@ public function testRetryBackoffIsPersistedWithoutSleepingOrHoldingTheCompletion Carbon::setTestNow(now()->addHour()); $finalClaim = ServiceCallScheduler::claim($call->id); $this->assertNotNull($finalClaim); + $finalIdentity = ServiceCallEffectIdentity::forCurrentClaim($finalClaim); + $this->assertInstanceOf(ServiceCallEffectIdentity::class, $finalIdentity); + $this->assertNotSame($firstIdentity->storageToken(), $finalIdentity->storageToken()); $workflows->shouldReceive('update') ->once() ->andThrow(new RuntimeException('final update failure')); (new DefaultServiceControlPlane($workflows))->executeScheduledAttempt($finalClaim); - $this->assertSame(ServiceCallScheduler::STATE_COMPLETED, $dispatch->fresh()->state); + $dispatch->refresh(); + $this->assertSame(ServiceCallScheduler::STATE_COMPLETED, $dispatch->state); + $this->assertStringNotContainsString( + $finalIdentity->storageToken(), + json_encode($dispatch->dispatch_options, JSON_THROW_ON_ERROR), + ); $call->refresh(); $this->assertSame(ServiceCallStatus::Failed->value, $call->status); $this->assertCount(2, $call->metadata['service_call_attempts']); @@ -313,6 +330,8 @@ public function testExpiredClaimCanBeRepairedAndAStaleWorkerCannotAdvanceIt(): v $call = WorkflowServiceCall::query()->sole(); $first = ServiceCallScheduler::claim($call->id); $this->assertNotNull($first); + $firstIdentity = ServiceCallEffectIdentity::forCurrentClaim($first); + $this->assertInstanceOf(ServiceCallEffectIdentity::class, $firstIdentity); WorkflowServiceCallDispatch::query()->whereKey($call->id)->update([ 'claim_expires_at' => now() ->subSecond(), @@ -326,10 +345,9 @@ public function testExpiredClaimCanBeRepairedAndAStaleWorkerCannotAdvanceIt(): v $second->options['service_call_idempotency_key'], ); $this->assertNull(ServiceCallEffectIdentity::forCurrentClaim($first)); - $this->assertInstanceOf( - ServiceCallEffectIdentity::class, - ServiceCallEffectIdentity::forCurrentClaim($second), - ); + $secondIdentity = ServiceCallEffectIdentity::forCurrentClaim($second); + $this->assertInstanceOf(ServiceCallEffectIdentity::class, $secondIdentity); + $this->assertSame($firstIdentity->storageToken(), $secondIdentity->storageToken()); /** @var WorkflowControlPlane&MockInterface $workflows */ $workflows = Mockery::mock(WorkflowControlPlane::class); From 495e9774a91a4dc8ccd4811a2b44b140ac3e2d06 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Mon, 26 Oct 2026 04:38:51 +0000 Subject: [PATCH 09/11] Bind scheduled effect capabilities to claims --- src/V2/Support/DefaultServiceControlPlane.php | 131 +++++++++++------- src/V2/Support/ServiceCallScheduler.php | 7 +- .../Unit/V2/ServiceCallEffectIdentityTest.php | 56 ++++++++ tests/Unit/V2/ServiceCallSchedulerTest.php | 19 +++ 4 files changed, 162 insertions(+), 51 deletions(-) diff --git a/src/V2/Support/DefaultServiceControlPlane.php b/src/V2/Support/DefaultServiceControlPlane.php index 37e7a3624..ea66faee4 100644 --- a/src/V2/Support/DefaultServiceControlPlane.php +++ b/src/V2/Support/DefaultServiceControlPlane.php @@ -38,8 +38,6 @@ final class DefaultServiceControlPlane implements ServiceControlPlane private const WORKFLOW_QUERY_HANDLER_BINDING = 'workflow.v2.service_control_plane.workflow_query_handler'; - private ?ServiceCallAttemptClaim $scheduledClaim = null; - public function __construct( private ?WorkflowControlPlane $workflowControlPlane = null, private ?ServiceBoundaryPolicy $boundaryPolicy = null, @@ -319,43 +317,32 @@ public function executeScheduledAttempt(ServiceCallAttemptClaim $claim): void return; } - $this->scheduledClaim = $claim; - try { - $startedAt = now(); - $options = $claim->options; - $options[ServiceCallEffectIdentity::OPTION] = $effectIdentity; - $result = $this->dispatchHandlerAttempt( - $claim->call, - $claim->operation, - $claim->request, - $options, - $claim->attempt, - $claim->maxAttempts, - ); - $claim->call->refresh(); - $retry = $this->shouldRetryServiceCall($result, $claim->attempt, $claim->maxAttempts); - $backoff = $retry - ? $this->serviceCallBackoffSeconds($claim->call->retry_policy, $claim->attempt) - : null; - $recorded = ServiceCallScheduler::completeAttempt( - $claim, - $this->serviceCallAttemptEntry( - $claim->call, - $result, - $claim->attempt, - $startedAt, - $retry, - $backoff, - ), - $retry, - $backoff, - ); - $claim->call->refresh(); - if ($recorded && ! $retry && $this->isFailed($claim->call)) { - $this->releaseBoundaryAdmission($claim->request); - } - } finally { - $this->scheduledClaim = null; + $startedAt = now(); + $options = $claim->options; + $options[ServiceCallEffectIdentity::OPTION] = $effectIdentity; + $result = $this->dispatchHandlerAttempt( + $claim->call, + $claim->operation, + $claim->request, + $options, + $claim->attempt, + $claim->maxAttempts, + $claim, + ); + $claim->call->refresh(); + $retry = $this->shouldRetryServiceCall($result, $claim->attempt, $claim->maxAttempts); + $backoff = $retry + ? $this->serviceCallBackoffSeconds($claim->call->retry_policy, $claim->attempt) + : null; + $recorded = ServiceCallScheduler::completeAttempt( + $claim, + $this->serviceCallAttemptEntry($claim->call, $result, $claim->attempt, $startedAt, $retry, $backoff), + $retry, + $backoff, + ); + $claim->call->refresh(); + if ($recorded && ! $retry && $this->isFailed($claim->call)) { + $this->releaseBoundaryAdmission($claim->request); } } @@ -1065,6 +1052,7 @@ private function dispatchHandlerAttempt( array $options, int $attempt, int $maxAttempts, + ?ServiceCallAttemptClaim $claim = null, ): ?array { $kind = $this->bindingKind($operation); @@ -1076,6 +1064,7 @@ private function dispatchHandlerAttempt( $options, $attempt, $maxAttempts, + $claim, ), ServiceCallBindingKind::WorkflowUpdate->value, 'update_workflow' => $this->dispatchWorkflowUpdate( $call, @@ -1084,6 +1073,7 @@ private function dispatchHandlerAttempt( $options, $attempt, $maxAttempts, + $claim, ), ServiceCallBindingKind::WorkflowSignal->value, 'signal_workflow' => $this->dispatchWorkflowSignal( $call, @@ -1092,6 +1082,7 @@ private function dispatchHandlerAttempt( $options, $attempt, $maxAttempts, + $claim, ), ServiceCallBindingKind::WorkflowQuery->value, 'query_workflow' => $this->dispatchWorkflowQuery( $call, @@ -1100,6 +1091,7 @@ private function dispatchHandlerAttempt( $options, $attempt, $maxAttempts, + $claim, ), ServiceCallBindingKind::ActivityExecution->value => $this->dispatchActivityExecution( $call, @@ -1108,6 +1100,7 @@ private function dispatchHandlerAttempt( $options, $attempt, $maxAttempts, + $claim, ), ServiceCallBindingKind::InvocableCarrierRequest->value, 'invocable_http' => $this->dispatchInvocableCarrierRequest( $call, @@ -1116,6 +1109,7 @@ private function dispatchHandlerAttempt( $options, $attempt, $maxAttempts, + $claim, ), default => null, }; @@ -1278,6 +1272,7 @@ private function dispatchWorkflowRun( array $options, int $attempt, int $maxAttempts, + ?ServiceCallAttemptClaim $claim, ): array { $binding = $this->handlerBinding($operation); $workflowType = $this->bindingString($binding, ['workflow_type', 'workflow_class', 'type']) @@ -1291,6 +1286,7 @@ private function dispatchWorkflowRun( 'Workflow start binding has no workflow type.', attempt: $attempt, maxAttempts: $maxAttempts, + claim: $claim, ); } @@ -1311,6 +1307,7 @@ private function dispatchWorkflowRun( ], $attempt, $maxAttempts, + $claim, ); } @@ -1326,6 +1323,7 @@ private function dispatchWorkflowRun( 'workflow_type' => $workflowType, 'control_plane' => $this->publicResult($result), ], + $claim, ); return [ @@ -1344,6 +1342,7 @@ private function dispatchWorkflowRun( ], $attempt, $maxAttempts, + $claim, ); } @@ -1358,6 +1357,7 @@ private function dispatchWorkflowUpdate( array $options, int $attempt, int $maxAttempts, + ?ServiceCallAttemptClaim $claim, ): array { $binding = $this->handlerBinding($operation); $instanceId = $this->bindingString($binding, ['workflow_instance_id', 'instance_id', 'target_instance_id']) @@ -1375,6 +1375,7 @@ private function dispatchWorkflowUpdate( 'Workflow update binding has no workflow instance id.', attempt: $attempt, maxAttempts: $maxAttempts, + claim: $claim, ); } @@ -1401,6 +1402,7 @@ private function dispatchWorkflowUpdate( ], $attempt, $maxAttempts, + $claim, ); } @@ -1421,6 +1423,7 @@ private function dispatchWorkflowUpdate( $runId, $updateId, $metadata, + $claim, ); if ($completed) { $this->releaseBoundaryAdmission($request); @@ -1434,6 +1437,7 @@ private function dispatchWorkflowUpdate( $runId, $updateId, $metadata, + $claim, ); } @@ -1453,6 +1457,7 @@ private function dispatchWorkflowUpdate( ], $attempt, $maxAttempts, + $claim, ); } @@ -1467,6 +1472,7 @@ private function dispatchWorkflowSignal( array $options, int $attempt, int $maxAttempts, + ?ServiceCallAttemptClaim $claim, ): array { $binding = $this->handlerBinding($operation); $instanceId = $this->bindingString($binding, ['workflow_instance_id', 'instance_id', 'target_instance_id']) @@ -1484,6 +1490,7 @@ private function dispatchWorkflowSignal( 'Workflow signal binding has no workflow instance id.', attempt: $attempt, maxAttempts: $maxAttempts, + claim: $claim, ); } @@ -1501,6 +1508,7 @@ private function dispatchWorkflowSignal( ], $attempt, $maxAttempts, + $claim, ); } @@ -1518,6 +1526,7 @@ private function dispatchWorkflowSignal( 'signal_name' => $signalName, 'control_plane' => $this->publicResult($result), ], + $claim, ); return [ @@ -1536,6 +1545,7 @@ private function dispatchWorkflowSignal( ], $attempt, $maxAttempts, + $claim, ); } @@ -1550,6 +1560,7 @@ private function dispatchWorkflowQuery( array $options, int $attempt, int $maxAttempts, + ?ServiceCallAttemptClaim $claim, ): array { $binding = $this->handlerBinding($operation); $instanceId = $this->bindingString($binding, ['workflow_instance_id', 'instance_id', 'target_instance_id']) @@ -1567,6 +1578,7 @@ private function dispatchWorkflowQuery( 'Workflow query binding has no workflow instance id.', attempt: $attempt, maxAttempts: $maxAttempts, + claim: $claim, ); } @@ -1591,6 +1603,7 @@ private function dispatchWorkflowQuery( ], $attempt, $maxAttempts, + $claim, ); } @@ -1608,6 +1621,7 @@ private function dispatchWorkflowQuery( 'query_name' => $queryName, 'control_plane' => $this->publicResult($result), ], + $claim, ); if ($completed) { $this->releaseBoundaryAdmission($request); @@ -1629,6 +1643,7 @@ private function dispatchWorkflowQuery( ], $attempt, $maxAttempts, + $claim, ); } @@ -1679,6 +1694,7 @@ private function dispatchActivityExecution( array $options, int $attempt, int $maxAttempts, + ?ServiceCallAttemptClaim $claim, ): array { $binding = $this->handlerBinding($operation); $activityClass = $this->bindingString($binding, ['activity_class', 'class']) @@ -1694,6 +1710,7 @@ private function dispatchActivityExecution( 'Activity binding has no activity class or type.', attempt: $attempt, maxAttempts: $maxAttempts, + claim: $claim, ); } @@ -1728,6 +1745,7 @@ private function dispatchActivityExecution( $linkedRunId, null, $metadata, + $claim, ); return [ @@ -1750,6 +1768,7 @@ private function dispatchInvocableCarrierRequest( array $options, int $attempt, int $maxAttempts, + ?ServiceCallAttemptClaim $claim, ): array { $binding = $this->handlerBinding($operation); $carrierEndpoint = $this->bindingString($binding, ['carrier_endpoint', 'endpoint', 'url']) @@ -1765,6 +1784,7 @@ private function dispatchInvocableCarrierRequest( 'Invocable carrier binding has no endpoint or carrier reference.', attempt: $attempt, maxAttempts: $maxAttempts, + claim: $claim, ); } @@ -1797,6 +1817,7 @@ private function dispatchInvocableCarrierRequest( null, null, $metadata, + $claim, ); return [ @@ -1832,6 +1853,7 @@ private function stampBindingResolution( ?string $runId, ?string $updateId, array $metadata, + ?ServiceCallAttemptClaim $claim, ): void { $call->getConnection() ->transaction(function () use ( @@ -1842,12 +1864,13 @@ private function stampBindingResolution( $runId, $updateId, $metadata, + $claim, ): void { /** @var WorkflowServiceCall $locked */ $locked = ConfiguredV2Models::query('service_call_model', WorkflowServiceCall::class) ->lockForUpdate() ->findOrFail($call->id); - if (! $this->scheduledClaimOwnsLockedCall($locked) + if (! $this->scheduledClaimOwnsLockedCall($locked, $claim) || ServiceCallDeadline::expireLocked($locked, now()) || $this->isTerminal($locked)) { return; @@ -1874,6 +1897,7 @@ private function markStarted( ?string $runId, ?string $updateId, array $metadata, + ?ServiceCallAttemptClaim $claim, ): bool { $recorded = false; $call->getConnection() @@ -1885,13 +1909,14 @@ private function markStarted( $runId, $updateId, $metadata, + $claim, &$recorded, ): void { /** @var WorkflowServiceCall $locked */ $locked = ConfiguredV2Models::query('service_call_model', WorkflowServiceCall::class) ->lockForUpdate() ->findOrFail($call->id); - if (! $this->scheduledClaimOwnsLockedCall($locked)) { + if (! $this->scheduledClaimOwnsLockedCall($locked, $claim)) { return; } ServiceCallDeadline::expireLocked($locked, now()); @@ -1941,6 +1966,7 @@ private function markCompleted( ?string $runId, ?string $updateId, array $metadata, + ?ServiceCallAttemptClaim $claim, ): bool { $recorded = false; $call->getConnection() @@ -1952,13 +1978,14 @@ private function markCompleted( $runId, $updateId, $metadata, + $claim, &$recorded, ): void { /** @var WorkflowServiceCall $locked */ $locked = ConfiguredV2Models::query('service_call_model', WorkflowServiceCall::class) ->lockForUpdate() ->findOrFail($call->id); - if (! $this->scheduledClaimOwnsLockedCall($locked) + if (! $this->scheduledClaimOwnsLockedCall($locked, $claim) || ServiceCallDeadline::expireLocked($locked, now()) || $this->isTerminal($locked)) { return; } @@ -2000,6 +2027,7 @@ private function markHandlerFailure( array $metadata = [], int $attempt = 1, int $maxAttempts = 1, + ?ServiceCallAttemptClaim $claim = null, ): array { $result = $call->getConnection() ->transaction(function () use ( @@ -2009,13 +2037,14 @@ private function markHandlerFailure( $message, $metadata, $attempt, - $maxAttempts + $maxAttempts, + $claim, ): array { /** @var WorkflowServiceCall $locked */ $locked = ConfiguredV2Models::query('service_call_model', WorkflowServiceCall::class) ->lockForUpdate() ->findOrFail($call->id); - if (! $this->scheduledClaimOwnsLockedCall($locked)) { + if (! $this->scheduledClaimOwnsLockedCall($locked, $claim)) { return [ 'accepted' => false, 'kind' => $locked->resolved_binding_kind, @@ -2041,7 +2070,7 @@ private function markHandlerFailure( $message, $metadata, $attempt, - $maxAttempts + $maxAttempts, ); }); $call->refresh(); @@ -2233,12 +2262,14 @@ private function workflowCommandOptions(WorkflowServiceCall $call, array $bindin return $commandOptions; } - private function scheduledClaimOwnsLockedCall(WorkflowServiceCall $call): bool - { - if (! $this->scheduledClaim instanceof ServiceCallAttemptClaim) { + private function scheduledClaimOwnsLockedCall( + WorkflowServiceCall $call, + ?ServiceCallAttemptClaim $claim, + ): bool { + if (! $claim instanceof ServiceCallAttemptClaim) { return true; } - if ($call->id !== $this->scheduledClaim->call->id) { + if ($call->id !== $claim->call->id) { return false; } @@ -2250,7 +2281,7 @@ private function scheduledClaimOwnsLockedCall(WorkflowServiceCall $call): bool ->find($call->id); return $dispatch instanceof \Workflow\V2\Models\WorkflowServiceCallDispatch - && ServiceCallScheduler::ownsLockedClaim($dispatch, $this->scheduledClaim); + && ServiceCallScheduler::ownsLockedClaim($dispatch, $claim); } /** diff --git a/src/V2/Support/ServiceCallScheduler.php b/src/V2/Support/ServiceCallScheduler.php index 82dd94857..53a2536e4 100644 --- a/src/V2/Support/ServiceCallScheduler.php +++ b/src/V2/Support/ServiceCallScheduler.php @@ -312,10 +312,15 @@ public static function ownsLockedClaim( WorkflowServiceCallDispatch $dispatch, ServiceCallAttemptClaim $claim, ): bool { + $options = is_array($dispatch->dispatch_options) ? $dispatch->dispatch_options : []; + $effectToken = $options[self::EFFECT_TOKEN_OPTION] ?? null; + return $dispatch->state === self::STATE_CLAIMED && $dispatch->attempt === $claim->attempt && $dispatch->claim_generation === $claim->generation - && hash_equals((string) $dispatch->claim_token, $claim->token); + && hash_equals((string) $dispatch->claim_token, $claim->token) + && is_string($effectToken) + && hash_equals($effectToken, $claim->effectToken()); } public static function ownsClaim(ServiceCallAttemptClaim $claim): bool diff --git a/tests/Unit/V2/ServiceCallEffectIdentityTest.php b/tests/Unit/V2/ServiceCallEffectIdentityTest.php index 170366cfc..96e41056c 100644 --- a/tests/Unit/V2/ServiceCallEffectIdentityTest.php +++ b/tests/Unit/V2/ServiceCallEffectIdentityTest.php @@ -281,6 +281,62 @@ public function testLowLevelCommandRequestIdCannotReserveTheOldCanonicalIdentity $this->assertNull($signal->request_id); } + public function testLowLevelCommandRequestIdCannotReserveAnActualOpaqueEffectToken(): void + { + $workflow = $this->waitingWorkflow('low-level-opaque-effect-injection'); + $identity = $this->currentClaimIdentity($workflow); + $instance = WorkflowInstance::query()->findOrFail($workflow->id()); + $run = WorkflowRun::query()->findOrFail($workflow->runId()); + $effectToken = $identity->storageToken(); + $context = [ + 'request' => [ + 'request_id' => $effectToken, + ], + 'service_call' => [ + 'idempotency_key' => $identity->commandContext()['idempotency_key'], + ], + ]; + + $start = WorkflowCommand::record($instance, $run, [ + 'command_type' => CommandType::Start->value, + 'status' => CommandStatus::Accepted->value, + 'context' => $context, + 'request_id' => $effectToken, + ]); + $update = WorkflowCommand::record($instance, $run, [ + 'command_type' => CommandType::Update->value, + 'status' => CommandStatus::Accepted->value, + 'context' => $context, + 'request_id' => $effectToken, + ]); + $signal = WorkflowCommand::record($instance, $run, [ + 'command_type' => CommandType::Signal->value, + 'status' => CommandStatus::Accepted->value, + 'context' => $context, + 'request_id' => $effectToken, + ]); + + $this->assertNull($start->request_id); + $this->assertStringStartsWith('update-v1:', $update->request_id); + $this->assertNotSame($effectToken, $update->request_id); + $this->assertNull($signal->request_id); + + $legitimate = (new DefaultWorkflowControlPlane())->update($workflow->id(), 'change', [ + 'arguments' => ['legitimate'], + 'wait_for' => 'accepted', + ServiceCallEffectIdentity::OPTION => $identity, + ]); + + $this->assertTrue($legitimate['accepted']); + $this->assertSame(1, WorkflowUpdate::query()->count()); + $legitimateCommand = WorkflowCommand::query() + ->where('workflow_instance_id', $workflow->id()) + ->where('command_type', CommandType::Update->value) + ->where('request_id', $effectToken) + ->sole(); + $this->assertSame($effectToken, $legitimateCommand->request_id); + } + public function testEffectCommittedBeforeClaimReacquisitionKeepsOneOpaqueReceipt(): void { $workflow = $this->waitingWorkflow('reacquired-effect-update'); diff --git a/tests/Unit/V2/ServiceCallSchedulerTest.php b/tests/Unit/V2/ServiceCallSchedulerTest.php index d1b5339c8..8c2e3efcf 100644 --- a/tests/Unit/V2/ServiceCallSchedulerTest.php +++ b/tests/Unit/V2/ServiceCallSchedulerTest.php @@ -33,6 +33,7 @@ use Workflow\V2\Models\WorkflowServiceOperation; use Workflow\V2\Support\DefaultServiceBoundaryPolicy; use Workflow\V2\Support\DefaultServiceControlPlane; +use Workflow\V2\Support\ServiceCallAttemptClaim; use Workflow\V2\Support\ServiceCallEffectIdentity; use Workflow\V2\Support\ServiceCallPrincipal; use Workflow\V2\Support\ServiceCallScheduler; @@ -348,6 +349,24 @@ public function testExpiredClaimCanBeRepairedAndAStaleWorkerCannotAdvanceIt(): v $secondIdentity = ServiceCallEffectIdentity::forCurrentClaim($second); $this->assertInstanceOf(ServiceCallEffectIdentity::class, $secondIdentity); $this->assertSame($firstIdentity->storageToken(), $secondIdentity->storageToken()); + $differentEffectToken = $secondIdentity->storageToken() === 'service-call-v1:' . str_repeat('f', 64) + ? 'service-call-v1:' . str_repeat('e', 64) + : 'service-call-v1:' . str_repeat('f', 64); + $forged = new ServiceCallAttemptClaim( + call: $second->call, + operation: $second->operation, + request: $second->request, + options: $second->options, + attempt: $second->attempt, + maxAttempts: $second->maxAttempts, + generation: $second->generation, + token: $second->token, + catalogRevision: $second->catalogRevision, + policyRevision: $second->policyRevision, + effectToken: $differentEffectToken, + ); + $this->assertFalse(ServiceCallScheduler::ownsClaim($forged)); + $this->assertNull(ServiceCallEffectIdentity::forCurrentClaim($forged)); /** @var WorkflowControlPlane&MockInterface $workflows */ $workflows = Mockery::mock(WorkflowControlPlane::class); From 7445a7ef22d70455b25880143f367b54a47d0208 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Mon, 26 Oct 2026 05:25:47 +0000 Subject: [PATCH 10/11] Align service scheduler regression contracts --- tests/Support/NexusCallerDeliveryScenario.php | 5 ++++- tests/Unit/V2/EmbeddedContinuationTest.php | 13 ++++++++++--- tests/Unit/V2/LinkedServiceCallCloserTest.php | 2 ++ tests/Unit/V2/SurfaceStabilityContractTest.php | 3 ++- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/tests/Support/NexusCallerDeliveryScenario.php b/tests/Support/NexusCallerDeliveryScenario.php index 390fcd3f7..9e0621fe1 100644 --- a/tests/Support/NexusCallerDeliveryScenario.php +++ b/tests/Support/NexusCallerDeliveryScenario.php @@ -16,6 +16,7 @@ use Workflow\V2\Enums\RunStatus; use Workflow\V2\Enums\TaskStatus; use Workflow\V2\Exceptions\ServiceOperationAuthorizationException; +use Workflow\V2\Jobs\RunServiceCall; use Workflow\V2\Jobs\RunWorkflowTask; use Workflow\V2\Models\WorkflowCommand; use Workflow\V2\Models\WorkflowHistoryEvent; @@ -226,7 +227,9 @@ protected function assertWaitingNexusCallerReceivesResult( $this->assertSame(RunStatus::Waiting, $run->refresh()->status); $this->assertNull($workflow->refresh()->output()); $call = WorkflowServiceCall::query()->where('caller_workflow_run_id', $run->id)->sole(); - $this->assertSame('started', $call->status); + $this->assertSame('accepted', $call->status); + $this->app->call([new RunServiceCall($call->id), 'handle']); + $this->assertSame('started', $call->refresh()->status); if ($deadlinePolicy !== null) { $this->assertSame('2026-10-11T00:00:01.123456Z', $call->call_deadline_at?->toJSON()); diff --git a/tests/Unit/V2/EmbeddedContinuationTest.php b/tests/Unit/V2/EmbeddedContinuationTest.php index 0d5144c53..85ff0cec6 100644 --- a/tests/Unit/V2/EmbeddedContinuationTest.php +++ b/tests/Unit/V2/EmbeddedContinuationTest.php @@ -39,6 +39,7 @@ use Workflow\V2\Enums\HistoryEventType; use Workflow\V2\Exceptions\ServiceOperationAuthorizationException; use Workflow\V2\Exceptions\WorkflowEntryArgumentsMismatchException; +use Workflow\V2\Jobs\RunServiceCall; use Workflow\V2\Jobs\RunWorkflowTask; use Workflow\V2\Models\WorkflowHistoryEvent; use Workflow\V2\Models\WorkflowInstance; @@ -865,7 +866,7 @@ public static function fallbackOperations(): iterable 'child' => [HistoryEventType::ChildWorkflowScheduled, 'child-value'], 'condition' => [HistoryEventType::ConditionWaitOpened, true], 'continue' => [HistoryEventType::WorkflowContinuedAsNew, 'continued'], - 'nexus' => [HistoryEventType::ServiceCallStarted, [ + 'nexus' => [HistoryEventType::ServiceOperationScheduled, [ 'integer' => 9007199254740993, 'optional' => null, 'unicode' => 'Nexus λ', @@ -926,6 +927,11 @@ public function testExcludedOperationsReplayBeforeTheirFirstDurableWrite( ->map->getRawOriginal() ->all(); $this->workOnce(); + if ($serviceRun !== null) { + $call = WorkflowServiceCall::query()->where('caller_workflow_run_id', $run->id)->sole(); + $this->assertSame('accepted', $call->status); + $this->app->call([new RunServiceCall($call->id), 'handle']); + } $this->assertNotEmpty($atWrite); $this->assertSame([2], array_values(array_unique($atWrite))); $this->assertSame(0, app(WorkflowContinuationCache::class)->snapshot()['entries']); @@ -965,11 +971,12 @@ public function testExcludedOperationsReplayBeforeTheirFirstDurableWrite( $this->assertCount(1, $events->where('event_type', HistoryEventType::ActivityCompleted)); if ($serviceRun !== null) { $this->assertSame(1, $backendStarts); - $this->assertCount(1, $events->where('event_type', HistoryEventType::ServiceCallStarted)); + $this->assertCount(1, $events->where('event_type', HistoryEventType::ServiceOperationScheduled)); + $this->assertCount(0, $events->where('event_type', HistoryEventType::ServiceCallStarted)); $this->assertCount(1, $events->where('event_type', HistoryEventType::ServiceCallCompleted)); $this->assertSame( 2, - $events->firstWhere('event_type', HistoryEventType::ServiceCallStarted)->payload['sequence'] + $events->firstWhere('event_type', HistoryEventType::ServiceOperationScheduled)->payload['sequence'] ); $this->assertSame( 2, diff --git a/tests/Unit/V2/LinkedServiceCallCloserTest.php b/tests/Unit/V2/LinkedServiceCallCloserTest.php index 77737507f..0c899797d 100644 --- a/tests/Unit/V2/LinkedServiceCallCloserTest.php +++ b/tests/Unit/V2/LinkedServiceCallCloserTest.php @@ -5,6 +5,7 @@ namespace Tests\Unit\V2; use Illuminate\Support\Carbon; +use Illuminate\Support\Facades\Queue; use Mockery; use PHPUnit\Framework\Attributes\DataProvider; use RuntimeException; @@ -463,6 +464,7 @@ public function testFailureAndCancellationResumeWithTheActualTerminalCause(): vo public function testEmbeddedCallerWaitsAndThenReplaysTheActualServiceResult(): void { + Queue::fake(); config([ 'workflows.v2.task_dispatch_mode' => 'poll', ]); diff --git a/tests/Unit/V2/SurfaceStabilityContractTest.php b/tests/Unit/V2/SurfaceStabilityContractTest.php index 5f922cc05..504e6409a 100644 --- a/tests/Unit/V2/SurfaceStabilityContractTest.php +++ b/tests/Unit/V2/SurfaceStabilityContractTest.php @@ -145,7 +145,7 @@ public function testWorkerProtocolNegotiationIncludesRustProtocolAndFailsClosed( $negotiation = $manifest['surface_families']['worker_protocol']['negotiation']; $this->assertSame('worker_protocol.version', $negotiation['advertised_version_path']); - $this->assertSame('1.20', $negotiation['default_advertised_version']); + $this->assertSame('1.21', $negotiation['default_advertised_version']); $this->assertSame( 'same_major_and_minor_less_than_or_equal_to_advertised', $negotiation['request_header_rule'], @@ -154,6 +154,7 @@ public function testWorkerProtocolNegotiationIncludesRustProtocolAndFailsClosed( $this->assertContains('1.2', $negotiation['accepted_request_versions_by_default']); $this->assertSame('1.19', $negotiation['accepted_request_versions_by_default'][19]); $this->assertSame('1.20', $negotiation['accepted_request_versions_by_default'][20]); + $this->assertSame('1.21', $negotiation['accepted_request_versions_by_default'][21]); $this->assertSame('advertised_version', $negotiation['response_version']); $this->assertSame( ['missing_header', 'malformed_version', 'different_major', 'minor_greater_than_advertised'], From 23dc794fd90818145ee490b8544dbc2aeffa4e6e Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Mon, 26 Oct 2026 06:08:01 +0000 Subject: [PATCH 11/11] Version scheduler corpus assertions --- tests/Support/NexusCallerDeliveryScenario.php | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/Support/NexusCallerDeliveryScenario.php b/tests/Support/NexusCallerDeliveryScenario.php index 9e0621fe1..a6705d7fb 100644 --- a/tests/Support/NexusCallerDeliveryScenario.php +++ b/tests/Support/NexusCallerDeliveryScenario.php @@ -31,6 +31,7 @@ use Workflow\V2\Models\WorkflowTimer; use Workflow\V2\Support\DefaultServiceControlPlane; use Workflow\V2\Support\ServiceCallPrincipal; +use Workflow\V2\Support\WorkerProtocolVersion; use Workflow\V2\Support\WorkflowFiberRunner; use Workflow\V2\TaskWatchdog; use Workflow\V2\Workflow; @@ -227,9 +228,13 @@ protected function assertWaitingNexusCallerReceivesResult( $this->assertSame(RunStatus::Waiting, $run->refresh()->status); $this->assertNull($workflow->refresh()->output()); $call = WorkflowServiceCall::query()->where('caller_workflow_run_id', $run->id)->sole(); - $this->assertSame('accepted', $call->status); - $this->app->call([new RunServiceCall($call->id), 'handle']); - $this->assertSame('started', $call->refresh()->status); + if (version_compare(WorkerProtocolVersion::VERSION, '1.21', '<')) { + $this->assertSame('started', $call->status); + } else { + $this->assertSame('accepted', $call->status); + $this->app->call([new RunServiceCall($call->id), 'handle']); + $this->assertSame('started', $call->refresh()->status); + } if ($deadlinePolicy !== null) { $this->assertSame('2026-10-11T00:00:01.123456Z', $call->call_deadline_at?->toJSON());