From 2dd3c5ecd3b5e27ebc6b0bd047672b36113f1c9d Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:38:20 +0000 Subject: [PATCH 01/10] Retain Nexus workflow results and guard namespace collisions Refs #947 and durable-workflow/server#794. The linked lifecycle and argument follow-through remain separate source work on this branch. --- src/V2/Support/DefaultServiceControlPlane.php | 67 +++++++++- .../Support/DefaultWorkflowControlPlane.php | 12 +- ...ServiceControlPlaneWorkflowBindingTest.php | 122 ++++++++++++++++++ 3 files changed, 191 insertions(+), 10 deletions(-) diff --git a/src/V2/Support/DefaultServiceControlPlane.php b/src/V2/Support/DefaultServiceControlPlane.php index 40e2293e..dc8e8093 100644 --- a/src/V2/Support/DefaultServiceControlPlane.php +++ b/src/V2/Support/DefaultServiceControlPlane.php @@ -9,6 +9,7 @@ use Illuminate\Support\Str; use Throwable; use Workflow\Serializers\CodecRegistry; +use Workflow\Serializers\Serializer; use Workflow\V2\Contracts\ServiceBoundaryPolicy; use Workflow\V2\Contracts\ServiceControlPlane; use Workflow\V2\Contracts\WorkflowControlPlane; @@ -1428,6 +1429,7 @@ private function dispatchWorkflowUpdate( $updateId, $metadata, $claim, + $this->workflowResultEnvelope($result, $call), ); if ($completed) { $this->releaseBoundaryAdmission($request); @@ -1626,6 +1628,7 @@ private function dispatchWorkflowQuery( 'control_plane' => $this->publicResult($result), ], $claim, + $this->workflowResultEnvelope($result, $call), ); if ($completed) { $this->releaseBoundaryAdmission($request); @@ -1971,6 +1974,7 @@ private function markCompleted( ?string $updateId, array $metadata, ?ServiceCallAttemptClaim $claim, + ?array $resultEnvelope = null, ): bool { $recorded = false; $call->getConnection() @@ -1983,6 +1987,7 @@ private function markCompleted( $updateId, $metadata, $claim, + $resultEnvelope, &$recorded, ): void { /** @var WorkflowServiceCall $locked */ @@ -2005,6 +2010,25 @@ private function markCompleted( $call->linked_workflow_instance_id = $instanceId; $call->linked_workflow_run_id = $runId; $call->linked_workflow_update_id = $updateId; + if ($resultEnvelope !== null) { + $codec = $resultEnvelope['codec']; + if (is_string($resultEnvelope['blob'] ?? null)) { + $metadata['nexus_result'] = [ + 'codec' => $codec, + 'blob' => $resultEnvelope['blob'], + ]; + $call->output_payload_reference = null; + } elseif (is_array($resultEnvelope['external_storage'] ?? null)) { + $reference = $resultEnvelope['external_storage']; + $call->output_payload_reference = $reference['uri']; + unset($reference['uri']); + $metadata['nexus_result'] = [ + 'codec' => $codec, + 'external_reference' => $reference, + ]; + } + $call->payload_codec = $codec; + } $call->metadata = $this->mergeMetadata($call->metadata, $metadata); $call->started_at ??= now(); $call->completed_at ??= now(); @@ -2209,11 +2233,14 @@ private function workflowStartOptions(WorkflowServiceCall $call, array $binding, } } - $arguments = $this->stringOption($options, 'arguments') - ?? $this->stringOption($options, 'payload_blob') - ?? $this->bindingString($binding, ['arguments', 'payload_blob']); + $arguments = $options['arguments'] ?? $options['payload_blob'] + ?? $binding['arguments'] ?? $binding['payload_blob'] ?? null; + + if (is_array($arguments)) { + $arguments = Serializer::serializeWithCodec($call->payload_codec, $arguments); + } - if ($arguments !== null) { + if (is_string($arguments)) { $startOptions['arguments'] = $arguments; } @@ -2288,6 +2315,38 @@ private function scheduledClaimOwnsLockedCall( && ServiceCallScheduler::ownsLockedClaim($dispatch, $claim); } + /** + * @param array $result + * @return array{codec: string, blob?: string, external_storage?: array}|null + */ + private function workflowResultEnvelope(array $result, WorkflowServiceCall $call): ?array + { + $envelope = $result['result_envelope'] ?? null; + if (is_array($envelope)) { + $codec = CodecRegistry::canonicalize($envelope['codec'] ?? $call->payload_codec); + if (is_string($envelope['blob'] ?? null)) { + return [ + 'codec' => $codec, + 'blob' => $envelope['blob'], + ]; + } + if (is_string($envelope['external_storage']['uri'] ?? null)) { + return [ + 'codec' => $codec, + 'external_storage' => $envelope['external_storage'], + ]; + } + } + if (array_key_exists('result', $result)) { + $codec = CodecRegistry::canonicalize($call->payload_codec); + return [ + 'codec' => $codec, + 'blob' => Serializer::serializeWithCodec($codec, $result['result']), + ]; + } + return null; + } + /** * @param array $result * @return array diff --git a/src/V2/Support/DefaultWorkflowControlPlane.php b/src/V2/Support/DefaultWorkflowControlPlane.php index 578e5c34..72ca28c6 100644 --- a/src/V2/Support/DefaultWorkflowControlPlane.php +++ b/src/V2/Support/DefaultWorkflowControlPlane.php @@ -1229,19 +1229,19 @@ private function resolveOrCreateInstance( ->lockForUpdate() ->findOrFail($instanceId); - if ($instance->workflow_type !== $workflowType) { + if ($instance->namespace !== $namespace) { throw new LogicException(sprintf( - 'Workflow instance [%s] is reserved for durable type [%s] and cannot be reused for [%s].', + 'Workflow instance [%s] cannot be reused with a different namespace.', $instanceId, - $instance->workflow_type, - $workflowType, )); } - if ($instance->namespace !== $namespace) { + if ($instance->workflow_type !== $workflowType) { throw new LogicException(sprintf( - 'Workflow instance [%s] cannot be reused with a different namespace.', + 'Workflow instance [%s] is reserved for durable type [%s] and cannot be reused for [%s].', $instanceId, + $instance->workflow_type, + $workflowType, )); } diff --git a/tests/Unit/V2/DefaultServiceControlPlaneWorkflowBindingTest.php b/tests/Unit/V2/DefaultServiceControlPlaneWorkflowBindingTest.php index ac8554b2..805c1c05 100644 --- a/tests/Unit/V2/DefaultServiceControlPlaneWorkflowBindingTest.php +++ b/tests/Unit/V2/DefaultServiceControlPlaneWorkflowBindingTest.php @@ -27,6 +27,7 @@ use Workflow\V2\Support\DefaultServiceControlPlane; use Workflow\V2\Support\ServiceBoundaryDecision; use Workflow\V2\Support\ServiceBoundaryRequest; +use Workflow\V2\Support\ServiceCallResultEnvelope; final class DefaultServiceControlPlaneWorkflowBindingTest extends TestCase { @@ -43,6 +44,127 @@ protected function tearDown(): void parent::tearDown(); } + #[DataProvider('completedWorkflowValues')] + public function testCompletedWorkflowBindingsRetainResultsOutsidePublicControlMetadata( + ServiceCallBindingKind $kind, + mixed $value, + bool $external, + ): void { + $this->catalog($kind, ServiceCallOperationMode::Sync, [ + 'workflow_instance_id' => 'result-owner', + ], 'answer'); + $blob = Serializer::serializeWithCodec('avro', $value); + $envelope = $external ? [ + 'codec' => 'avro', + 'external_storage' => [ + 'uri' => 'fixture://result', + 'codec' => 'avro', + 'sha256' => hash('sha256', $blob), + 'size_bytes' => strlen($blob), + ], + ] : [ + 'codec' => 'avro', + 'blob' => $blob, + ]; + $workflows = Mockery::mock(WorkflowControlPlane::class); + $workflows->shouldReceive($kind === ServiceCallBindingKind::WorkflowQuery ? 'query' : 'update') + ->once() + ->andReturn([ + 'accepted' => true, + 'success' => true, + 'update_status' => 'completed', + 'result' => $value, + 'result_envelope' => $envelope, + ]); + $control = new DefaultServiceControlPlane($workflows); + $response = $control->execute('billing', 'invoices', 'approve', [ + 'namespace' => 'billing', + ]); + $this->assertSame('completed', $response['status']); + $call = WorkflowServiceCall::query()->sole(); + $this->assertEquals($envelope, ServiceCallResultEnvelope::fromCall($call)); + $this->assertArrayNotHasKey('result', $call->metadata['control_plane']); + $this->assertArrayNotHasKey('result_envelope', $call->metadata['control_plane']); + $this->assertSame($external ? 'fixture://result' : null, $call->output_payload_reference); + $before = $call->getRawOriginal(); + $control->execute('billing', 'invoices', 'approve', [ + 'namespace' => 'billing', + 'service_call_id' => $call->id, + ]); + $this->assertSame($before, $call->fresh()->getRawOriginal()); + } + + public static function completedWorkflowValues(): iterable + { + foreach ([ServiceCallBindingKind::WorkflowQuery, ServiceCallBindingKind::WorkflowUpdate] as $kind) { + foreach ([ + null, false, 0, '', [ + 'answer' => ['Ω', null, 9223372036854775807], + ]] as $index => $value) { + yield $kind->value . '-' . $index => [$kind, $value, false]; + } + yield $kind->value . '-external' => [$kind, [ + 'large' => str_repeat('Ω', 2048), + ], true]; + } + } + + public function testWorkflowStartRetainsJsonListArguments(): void + { + $this->catalog(ServiceCallBindingKind::WorkflowRun, ServiceCallOperationMode::Async, [ + 'workflow_type' => 'sdk.bound-order', + 'queue' => 'orders', + ]); + $result = (new DefaultServiceControlPlane())->execute('billing', 'invoices', 'approve', [ + 'namespace' => 'billing', + 'arguments' => [7, 'order Ω', null], + ]); + $this->assertTrue($result['accepted']); + $run = \Workflow\V2\Models\WorkflowRun::query()->findOrFail($result['linked_workflow_run_id']); + $this->assertIsString($run->arguments); + $this->assertSame( + [7, 'order Ω', null], + Serializer::unserializeWithCodec($run->payload_codec, $run->arguments) + ); + } + + #[DataProvider('foreignWorkflowTypes')] + public function testWorkflowStartRefusesForeignNamespaceBeforeDisclosingItsType(string $foreignType): void + { + $foreign = \Workflow\V2\Models\WorkflowInstance::query()->create([ + 'id' => 'foreign-order', + 'namespace' => 'private-owner', + 'workflow_type' => $foreignType, + 'workflow_class' => $foreignType, + 'run_count' => 0, + ]); + $before = $foreign->fresh() + ->getRawOriginal(); + $this->catalog(ServiceCallBindingKind::WorkflowRun, ServiceCallOperationMode::Async, [ + 'workflow_type' => 'sdk.bound-order', + 'workflow_instance_id' => $foreign->id, + 'queue' => 'orders', + ]); + $result = (new DefaultServiceControlPlane())->execute('billing', 'invoices', 'approve', [ + 'namespace' => 'billing', + 'caller_namespace' => 'billing', + ]); + $this->assertFalse($result['accepted']); + $this->assertSame( + 'Workflow instance [foreign-order] cannot be reused with a different namespace.', + $result['message'] + ); + $this->assertSame($before, $foreign->fresh()->getRawOriginal()); + $this->assertStringNotContainsString('secret.foreign-type', json_encode($result, JSON_THROW_ON_ERROR)); + $this->assertStringNotContainsString('secret.foreign-type', WorkflowServiceCall::query()->sole()->toJson()); + } + + public static function foreignWorkflowTypes(): iterable + { + yield 'same type' => ['sdk.bound-order']; + yield 'different type' => ['secret.foreign-type']; + } + public function testDeferredDispatchAndDuplicatesRetainTheOriginalDeadlineAfterCatalogEdits(): void { Carbon::setTestNow('2026-10-08T03:00:00.123456Z'); From f7ebfa30cb58dec40f2675e272b51dfca2284523 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Mon, 26 Oct 2026 23:18:19 +0000 Subject: [PATCH 02/10] Preserve encoded Nexus arguments and close linked workflow timeouts --- src/V2/Support/DefaultServiceControlPlane.php | 40 ++++++-- src/V2/Support/LinkedServiceCallCloser.php | 7 +- ...ServiceControlPlaneWorkflowBindingTest.php | 96 +++++++++++++++++++ tests/Unit/V2/LinkedServiceCallCloserTest.php | 24 +++++ 4 files changed, 159 insertions(+), 8 deletions(-) diff --git a/src/V2/Support/DefaultServiceControlPlane.php b/src/V2/Support/DefaultServiceControlPlane.php index dc8e8093..80053cfb 100644 --- a/src/V2/Support/DefaultServiceControlPlane.php +++ b/src/V2/Support/DefaultServiceControlPlane.php @@ -7,6 +7,7 @@ use Carbon\CarbonImmutable; use DateTimeInterface; use Illuminate\Support\Str; +use Illuminate\Validation\ValidationException; use Throwable; use Workflow\Serializers\CodecRegistry; use Workflow\Serializers\Serializer; @@ -2272,12 +2273,7 @@ private function workflowCommandOptions(WorkflowServiceCall $call, array $bindin 'namespace' => $call->target_namespace, ]; - $arguments = $this->arrayOption($options, 'arguments') ?: $this->arrayAt($binding, 'arguments'); - if ($arguments !== []) { - $commandOptions['arguments'] = $arguments; - } - - foreach (['payload_codec', 'payload_blob', 'wait_for', 'wait_timeout_seconds', + foreach (['payload_codec', 'wait_for', 'wait_timeout_seconds', 'service_call_id', 'service_call_attempt', 'service_call_max_attempts'] as $key) { $value = $options[$key] ?? $binding[$key] ?? null; @@ -2285,6 +2281,32 @@ private function workflowCommandOptions(WorkflowServiceCall $call, array $bindin $commandOptions[$key] = $value; } } + $callerArguments = $options['arguments'] ?? $options['payload_blob'] ?? null; + $arguments = $callerArguments ?? $binding['arguments'] ?? $binding['payload_blob'] ?? []; + $codec = CodecRegistry::canonicalize($commandOptions['payload_codec'] ?? $call->payload_codec); + if (is_string($arguments)) { + // Serialized input belongs to the service request's namespace. + // caller_namespace is a policy identity, not a storage authority. + $blob = ExternalPayloads::resolveStoredPayload($arguments, $codec, $call->target_namespace); + $commandOptions['payload_blob'] = $arguments; + $arguments = Serializer::unserializeWithCodec($codec, $blob); + if (! is_array($arguments)) { + throw ValidationException::withMessages([ + 'arguments' => ['Workflow arguments must decode to an array.'], + ]); + } + } elseif (is_array($arguments) + && (($options['payload_blob'] ?? $binding['payload_blob'] ?? null) !== null)) { + // An explicit argument list overrides catalog defaults. Keep the + // receipt and stored signal payload derived from that same value. + $commandOptions['payload_blob'] = Serializer::serializeWithCodec($codec, $arguments); + } + if (is_array($arguments) && ($arguments !== [] || $callerArguments !== null || array_key_exists( + 'arguments', + $binding + ))) { + $commandOptions['arguments'] = $arguments; + } $effectIdentity = ServiceCallEffectIdentity::fromOptions($options); if ($effectIdentity instanceof ServiceCallEffectIdentity) { $commandOptions[ServiceCallEffectIdentity::OPTION] = $effectIdentity; @@ -2677,6 +2699,12 @@ private function mergeMetadata(mixed $metadata, ?array $extra): ?array $base = $this->arrayValue($metadata); $extra ??= []; + // A result has one representation; merging can retain a stale inline + // blob beside a new external reference (or the reverse). + if (array_key_exists('nexus_result', $extra)) { + unset($base['nexus_result']); + } + if ($base === [] && $extra === []) { return null; } diff --git a/src/V2/Support/LinkedServiceCallCloser.php b/src/V2/Support/LinkedServiceCallCloser.php index 6c4f8b80..351abc4d 100644 --- a/src/V2/Support/LinkedServiceCallCloser.php +++ b/src/V2/Support/LinkedServiceCallCloser.php @@ -60,8 +60,10 @@ public static function closeForTerminalEvent(WorkflowRun $run, WorkflowHistoryEv ServiceCallOutcome::Cancelled, 'cancelled_at', ], - HistoryEventType::WorkflowFailed, HistoryEventType::WorkflowTerminated => [ - ServiceCallStatus::Failed, ServiceCallOutcome::HandlerFailed, 'failed_at', + HistoryEventType::WorkflowFailed, HistoryEventType::WorkflowTerminated, HistoryEventType::WorkflowTimedOut => [ + ServiceCallStatus::Failed, + ServiceCallOutcome::HandlerFailed, + 'failed_at', ], default => null, }; @@ -148,6 +150,7 @@ private static function terminalEventTypes(): array HistoryEventType::WorkflowFailed->value, HistoryEventType::WorkflowCancelled->value, HistoryEventType::WorkflowTerminated->value, + HistoryEventType::WorkflowTimedOut->value, ]; } } diff --git a/tests/Unit/V2/DefaultServiceControlPlaneWorkflowBindingTest.php b/tests/Unit/V2/DefaultServiceControlPlaneWorkflowBindingTest.php index 805c1c05..b0427c28 100644 --- a/tests/Unit/V2/DefaultServiceControlPlaneWorkflowBindingTest.php +++ b/tests/Unit/V2/DefaultServiceControlPlaneWorkflowBindingTest.php @@ -10,8 +10,10 @@ use Mockery\MockInterface; use PHPUnit\Framework\Attributes\DataProvider; use RuntimeException; +use Tests\Fixtures\V2\RecordingExternalPayloadStorage; use Tests\TestCase; use Workflow\Serializers\Serializer; +use Workflow\V2\Contracts\ExternalPayloadStoragePolicy; use Workflow\V2\Contracts\ServiceBoundaryPolicy; use Workflow\V2\Contracts\WorkflowControlPlane; use Workflow\V2\Enums\ServiceCallBindingKind; @@ -25,6 +27,8 @@ use Workflow\V2\Models\WorkflowServiceOperation; use Workflow\V2\Support\DefaultServiceBoundaryPolicy; use Workflow\V2\Support\DefaultServiceControlPlane; +use Workflow\V2\Support\ExternalPayloads; +use Workflow\V2\Support\ExternalPayloadStorage; use Workflow\V2\Support\ServiceBoundaryDecision; use Workflow\V2\Support\ServiceBoundaryRequest; use Workflow\V2\Support\ServiceCallResultEnvelope; @@ -79,6 +83,18 @@ public function testCompletedWorkflowBindingsRetainResultsOutsidePublicControlMe $control = new DefaultServiceControlPlane($workflows); $response = $control->execute('billing', 'invoices', 'approve', [ 'namespace' => 'billing', + 'metadata' => [ + 'trace' => 'keep-me', + 'nexus_result' => $external ? [ + 'codec' => 'avro', + 'blob' => Serializer::serializeWithCodec('avro', 'stale'), + ] : [ + 'codec' => 'avro', + 'external_reference' => [ + 'sha256' => str_repeat('0', 64), + ], + ], + ], ]); $this->assertSame('completed', $response['status']); $call = WorkflowServiceCall::query()->sole(); @@ -86,6 +102,8 @@ public function testCompletedWorkflowBindingsRetainResultsOutsidePublicControlMe $this->assertArrayNotHasKey('result', $call->metadata['control_plane']); $this->assertArrayNotHasKey('result_envelope', $call->metadata['control_plane']); $this->assertSame($external ? 'fixture://result' : null, $call->output_payload_reference); + $this->assertSame('keep-me', $call->metadata['trace']); + $this->assertArrayNotHasKey($external ? 'blob' : 'external_reference', $call->metadata['nexus_result']); $before = $call->getRawOriginal(); $control->execute('billing', 'invoices', 'approve', [ 'namespace' => 'billing', @@ -165,6 +183,84 @@ public static function foreignWorkflowTypes(): iterable yield 'different type' => ['secret.foreign-type']; } + #[DataProvider('workflowCommandArguments')] + public function testWorkflowCommandBindingsPreserveArgumentsInTheirCommandAndTargetRecord( + ServiceCallBindingKind $kind, + string $wireShape, + ): void { + config()->set('queue.default', 'database'); + $target = (new \Workflow\V2\Support\DefaultWorkflowControlPlane())->start( + \Tests\Fixtures\V2\TestUpdateWorkflow::class, + 'encoded-target', + [ + 'namespace' => 'billing', + 'queue' => 'orders', + 'arguments' => Serializer::serialize([]), + ], + ); + $this->app->call([new \Workflow\V2\Jobs\RunWorkflowTask($target['task_id']), 'handle']); + $isUpdate = $kind === ServiceCallBindingKind::WorkflowUpdate; + $arguments = $isUpdate ? [true, 'source Ω'] : ['name Ω']; + $this->catalog($kind, ServiceCallOperationMode::Async, [ + 'workflow_instance_id' => $target['workflow_instance_id'], + ], $isUpdate ? 'approve' : 'name-provided'); + $value = $wireShape === 'array' ? $arguments : Serializer::serializeWithCodec('avro', $arguments); + if (str_starts_with($wireShape, 'external_')) { + $storage = new RecordingExternalPayloadStorage(threshold: 1); + $policy = Mockery::mock(ExternalPayloadStoragePolicy::class); + $policy->shouldReceive('driverFor') + ->andReturnUsing( + static fn (?string $namespace) => $namespace === 'billing' ? $storage->driverFor($namespace) : null, + ); + $policy->shouldReceive('thresholdBytesFor') + ->andReturn(1); + $this->app->instance(ExternalPayloadStoragePolicy::class, $policy); + $value = ExternalPayloads::externalizeForNamespace($value, 'avro', 'billing'); + ExternalPayloadStorage::flushVerifiedCache(); + } + $result = (new DefaultServiceControlPlane())->execute('billing', 'invoices', 'approve', [ + 'namespace' => 'billing', + 'caller_namespace' => 'finance', + 'payload_codec' => 'avro', + str_ends_with($wireShape, 'payload_blob') ? 'payload_blob' : 'arguments' => $value, + 'wait_for' => 'accepted', + ]); + $this->assertTrue($result['accepted'], $result['reason'] ?? 'Workflow command binding was refused.'); + $model = $isUpdate ? \Workflow\V2\Models\WorkflowUpdate::class : \Workflow\V2\Models\WorkflowSignal::class; + $record = $model::query()->sole(); + $recordBlob = ExternalPayloads::resolveStoredPayload($record->arguments, $record->payload_codec, 'billing'); + $this->assertSame($arguments, Serializer::unserializeWithCodec($record->payload_codec, $recordBlob)); + $command = \Workflow\V2\Models\WorkflowCommand::query() + ->where('command_type', $isUpdate ? 'update' : 'signal') + ->sole(); + if (! $isUpdate && str_starts_with($wireShape, 'external_')) { + $commandBlob = ExternalPayloads::resolveStoredPayload( + $command->payload, + $command->payload_codec, + 'billing' + ); + $this->assertSame($arguments, Serializer::unserializeWithCodec($command->payload_codec, $commandBlob)); + $this->assertSame('name-provided', $command->context['signal_name']); + } else { + $this->assertSame($arguments, $command->payloadValues(['arguments'])['arguments']); + } + } + + public static function workflowCommandArguments(): iterable + { + foreach ([ServiceCallBindingKind::WorkflowSignal, ServiceCallBindingKind::WorkflowUpdate] as $kind) { + foreach ([ + 'array', + 'arguments', + 'payload_blob', + 'external_arguments', + 'external_payload_blob', + ] as $wireShape) { + yield $kind->value . '-' . $wireShape => [$kind, $wireShape]; + } + } + } + public function testDeferredDispatchAndDuplicatesRetainTheOriginalDeadlineAfterCatalogEdits(): void { Carbon::setTestNow('2026-10-08T03:00:00.123456Z'); diff --git a/tests/Unit/V2/LinkedServiceCallCloserTest.php b/tests/Unit/V2/LinkedServiceCallCloserTest.php index 0c899797..ca16001c 100644 --- a/tests/Unit/V2/LinkedServiceCallCloserTest.php +++ b/tests/Unit/V2/LinkedServiceCallCloserTest.php @@ -773,6 +773,7 @@ public function testFailureCancellationAndTerminationCloseWithoutLosingTypedReas [HistoryEventType::WorkflowFailed, 'failed', ServiceCallOutcome::HandlerFailed, 'failed_at'], [HistoryEventType::WorkflowCancelled, 'cancelled', ServiceCallOutcome::Cancelled, 'cancelled_at'], [HistoryEventType::WorkflowTerminated, 'failed', ServiceCallOutcome::HandlerFailed, 'failed_at'], + [HistoryEventType::WorkflowTimedOut, 'failed', ServiceCallOutcome::HandlerFailed, 'failed_at'], ] as [$type, $status, $outcome, $timestamp]) { $run = $this->createRun(); $call = $this->createCall($run); @@ -794,6 +795,29 @@ public function testFailureCancellationAndTerminationCloseWithoutLosingTypedReas } } + public function testLinkedWorkflowTimeoutBeforeAdmissionIsReconciledOnceWithItsTypedCause(): void + { + $run = $this->createRun(); + $event = WorkflowHistoryEvent::record($run, HistoryEventType::WorkflowTimedOut, [ + 'exception_class' => \Workflow\V2\Exceptions\WorkflowTimeoutException::class, + 'message' => 'Workflow run timeout elapsed.', + 'timeout_kind' => 'run', + ]); + $call = $this->createCall($run); + LinkedServiceCallCloser::reconcile($call); + $this->assertSame('failed', $call->status); + $this->assertSame(ServiceCallOutcome::HandlerFailed, $call->outcome); + $this->assertTrue($event->recorded_at->equalTo($call->failed_at)); + $this->assertSame( + \Workflow\V2\Exceptions\WorkflowTimeoutException::class, + $call->outcome_metadata['service_error_type'] + ); + $this->assertSame('Workflow run timeout elapsed.', $call->failure_message); + $before = $call->getRawOriginal(); + LinkedServiceCallCloser::reconcile($call); + $this->assertSame($before, $call->fresh()->getRawOriginal()); + } + public function testUnrelatedNamespaceRunInstanceAndBindingRemainOpen(): void { $run = $this->createRun(); From a8bfe821b7c6a721bc61bb2b5695c68833b845f2 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Mon, 26 Oct 2026 23:24:05 +0000 Subject: [PATCH 03/10] Retain linked cancellation outcomes across terminal races --- src/V2/Support/DefaultServiceControlPlane.php | 34 +++-- tests/Unit/V2/LinkedServiceCallCloserTest.php | 141 ++++++++++++++++++ 2 files changed, 165 insertions(+), 10 deletions(-) diff --git a/src/V2/Support/DefaultServiceControlPlane.php b/src/V2/Support/DefaultServiceControlPlane.php index 80053cfb..92515c09 100644 --- a/src/V2/Support/DefaultServiceControlPlane.php +++ b/src/V2/Support/DefaultServiceControlPlane.php @@ -403,32 +403,47 @@ public function cancelCall(string $serviceCallId, array $options = []): array ]; } - $metadata = $this->arrayValue($call->metadata); + $linkedCancel = null; if ( ($this->arrayValue($call->cancellation_policy)['propagate_to_linked_workflow'] ?? false) === true && is_string($call->linked_workflow_instance_id) && $call->linked_workflow_instance_id !== '' ) { - $result = $this->workflows() - ->cancel($call->linked_workflow_instance_id, [ - 'namespace' => $call->target_namespace, - 'reason' => $this->stringOption($options, 'reason') ?? 'service_call_cancelled', - ]); - - $metadata['linked_cancel'] = $this->publicResult($result); + try { + $result = $this->workflows() + ->cancel($call->linked_workflow_instance_id, [ + 'namespace' => $call->target_namespace, + 'reason' => $this->stringOption($options, 'reason') ?? 'service_call_cancelled', + ]); + $linkedCancel = $this->publicResult($result); + } catch (Throwable $exception) { + $linkedCancel = [ + 'accepted' => false, + 'reason' => 'workflow_cancel_exception', + 'exception_type' => get_class($exception), + ]; + } } $cancelled = $call->getConnection() - ->transaction(function () use ($call, $metadata, $options): bool { + ->transaction(function () use ($call, $linkedCancel, $options): bool { /** @var WorkflowServiceCall $locked */ $locked = ConfiguredV2Models::query('service_call_model', WorkflowServiceCall::class) ->lockForUpdate() ->findOrFail($call->id); + if ($linkedCancel !== null) { + $metadata = $this->arrayValue($locked->metadata); + $metadata['linked_cancel'] = $linkedCancel; + $locked->metadata = $metadata; + } if (ServiceCallDeadline::expireLocked($locked, now())) { return false; } if ($this->isTerminal($locked)) { + if ($locked->isDirty('metadata')) { + $locked->save(); + } return $locked->status === ServiceCallStatus::Cancelled->value; } $call = $locked; @@ -441,7 +456,6 @@ public function cancelCall(string $serviceCallId, array $options = []): array $call->outcome_metadata = [ 'failure_reason' => ServiceCallFailureReason::Cancellation->value, ]; - $call->metadata = $metadata === [] ? null : $metadata; $call->cancelled_at ??= $now; $call->failure_message = $this->stringOption($options, 'reason') ?? $call->failure_message; $call->save(); diff --git a/tests/Unit/V2/LinkedServiceCallCloserTest.php b/tests/Unit/V2/LinkedServiceCallCloserTest.php index ca16001c..c6f88222 100644 --- a/tests/Unit/V2/LinkedServiceCallCloserTest.php +++ b/tests/Unit/V2/LinkedServiceCallCloserTest.php @@ -818,6 +818,147 @@ public function testLinkedWorkflowTimeoutBeforeAdmissionIsReconciledOnceWithItsT $this->assertSame($before, $call->fresh()->getRawOriginal()); } + #[DataProvider('cancellationTerminalWinners')] + public function testLinkedCancellationKeepsItsReceiptAndTheTerminalWinner(HistoryEventType $terminal): void + { + $run = $this->createRun(); + $call = $this->createCall($run, [ + 'cancellation_policy' => [ + 'propagate_to_linked_workflow' => true, + ], + 'metadata' => [ + 'trace' => 'keep-me', + ], + ]); + $workflows = Mockery::mock(WorkflowControlPlane::class); + $workflows->shouldReceive('cancel') + ->once() + ->andReturnUsing(static function () use ($run, $terminal): array { + $payload = $terminal === HistoryEventType::WorkflowCompleted + ? [ + 'output' => Serializer::serializeWithCodec('avro', false), + 'payload_codec' => 'avro', + ] + : [ + 'message' => 'target won', + ]; + WorkflowHistoryEvent::record($run, $terminal, $payload); + return [ + 'accepted' => true, + 'command_id' => 'linked-cancel', + 'result' => 'private-result', + ]; + }); + $control = new DefaultServiceControlPlane($workflows); + $result = $control->cancelCall($call->id, [ + 'namespace' => 'service', + ]); + $call->refresh(); + $this->assertSame($terminal === HistoryEventType::WorkflowCancelled, $result['accepted']); + $this->assertSame(match ($terminal) { + HistoryEventType::WorkflowCancelled => 'cancelled', + HistoryEventType::WorkflowCompleted => 'completed', + default => 'failed', + }, $call->status); + $this->assertSame('keep-me', $call->metadata['trace']); + $this->assertNotEmpty($call->metadata['linked_terminal_event_id']); + $this->assertSame([ + 'accepted' => true, + 'command_id' => 'linked-cancel', + ], $call->metadata['linked_cancel']); + if ($terminal === HistoryEventType::WorkflowCompleted) { + $this->assertSame(Serializer::serializeWithCodec('avro', false), $call->metadata['nexus_result']['blob']); + } + $before = $call->getRawOriginal(); + $control->cancelCall($call->id, [ + 'namespace' => 'service', + ]); + $this->assertSame($before, $call->fresh()->getRawOriginal()); + } + + public static function cancellationTerminalWinners(): iterable + { + foreach ([ + HistoryEventType::WorkflowCompleted, + HistoryEventType::WorkflowCancelled, + HistoryEventType::WorkflowFailed, + ] as $terminal) { + yield $terminal->value => [$terminal]; + } + } + + public function testLinkedCancellationFailureIsTypedWithoutLosingTheLocalCancellation(): void + { + $run = $this->createRun(); + $call = $this->createCall($run, [ + 'cancellation_policy' => [ + 'propagate_to_linked_workflow' => true, + ], + ]); + $workflows = Mockery::mock(WorkflowControlPlane::class); + $workflows->shouldReceive('cancel') + ->once() + ->andThrow(new RuntimeException('private backend detail')); + $control = new DefaultServiceControlPlane($workflows); + $result = $control->cancelCall($call->id, [ + 'namespace' => 'service', + ]); + $this->assertTrue($result['accepted']); + $call->refresh(); + $this->assertSame('cancelled', $call->status); + $this->assertSame([ + 'accepted' => false, + 'reason' => 'workflow_cancel_exception', + 'exception_type' => RuntimeException::class, + ], $call->metadata['linked_cancel']); + $before = $call->getRawOriginal(); + $control->cancelCall($call->id, [ + 'namespace' => 'service', + ]); + $this->assertSame($before, $call->fresh()->getRawOriginal()); + } + + public function testDeadlineStillWinsWhileLinkedCancellationIsInFlight(): void + { + Carbon::setTestNow('2026-10-11T10:00:00.123456Z'); + try { + $run = $this->createRun(); + $deadline = now() + ->addSecond(); + $call = $this->createCall($run, [ + 'cancellation_policy' => [ + 'propagate_to_linked_workflow' => true, + ], + 'call_deadline_at' => $deadline, + ]); + $workflows = Mockery::mock(WorkflowControlPlane::class); + $workflows->shouldReceive('cancel') + ->once() + ->andReturnUsing(static function () use ($run, $deadline): array { + Carbon::setTestNow($deadline); + WorkflowHistoryEvent::record($run, HistoryEventType::WorkflowCancelled, [ + 'message' => 'late cancellation', + ]); + return [ + 'accepted' => true, + ]; + }); + $result = (new DefaultServiceControlPlane($workflows))->cancelCall($call->id, [ + 'namespace' => 'service', + ]); + $this->assertFalse($result['accepted']); + $call->refresh(); + $this->assertSame(ServiceCallOutcome::TimedOut, $call->outcome); + $this->assertSame([ + 'accepted' => true, + ], $call->metadata['linked_cancel']); + $this->assertTrue($deadline->equalTo($call->failed_at)); + $this->assertNull($call->cancelled_at); + } finally { + Carbon::setTestNow(); + } + } + public function testUnrelatedNamespaceRunInstanceAndBindingRemainOpen(): void { $run = $this->createRun(); From a5b5671ad78f6591c4fb60968984f9cc07282eeb Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Mon, 26 Oct 2026 23:36:51 +0000 Subject: [PATCH 04/10] Follow canonical Nexus workflow continuations with bounded repair --- .../workflow-service-calls-architecture.md | 23 ++ src/V2/Support/LinkedServiceCallCloser.php | 173 +++++++++++-- src/V2/TaskWatchdog.php | 25 +- .../Unit/Commands/V2RepairPassCommandTest.php | 30 +++ .../V2/LinkedServiceCallContinuationTest.php | 242 ++++++++++++++++++ 5 files changed, 475 insertions(+), 18 deletions(-) create mode 100644 tests/Unit/V2/LinkedServiceCallContinuationTest.php diff --git a/docs/architecture/workflow-service-calls-architecture.md b/docs/architecture/workflow-service-calls-architecture.md index 6a175e52..2e30b590 100644 --- a/docs/architecture/workflow-service-calls-architecture.md +++ b/docs/architecture/workflow-service-calls-architecture.md @@ -336,6 +336,21 @@ reference. A second handler binding for the same call (for example when the caller retries with a different idempotency key) is a new service-call row with its own durable id. +A WorkflowRun binding follows the logical execution through canonical +`WorkflowContinuedAsNew` history. Its current linked run and resolved target +reference advance only to the declared next run, in the same namespace, +instance and workflow type, with the next run number. Selecting an instance's +current run or an unrelated redrive is insufficient. The call id, acceptance +time and original call deadline remain unchanged. The first run and latest +continuation event remain inspectable in call metadata. Continuation keeps the +call open; the final run's completion, cancellation or typed failure closes it. +A linked workflow timeout is a typed handler failure; expiry of the call's own +deadline remains the `timed_out` outcome and takes precedence once reached. + +One reconciliation read follows at most 32 continuation hops. The existing +repair pass resumes longer chains, within its normal candidate and queue/run +scope limits. No polling client is required to finish that reconciliation. + ## Deadline, cancellation, retry, and idempotency Every operation declares the policies it accepts at the contract @@ -378,6 +393,14 @@ or Waterline surface inspecting the call row can render the snapped deadline, cancellation, retry, and idempotency policy without consulting the live operation registry. +For the built-in `propagate_to_linked_workflow` cancellation option, propagation +is best-effort and its public receipt is retained in `metadata.linked_cancel`. +A propagation exception records `workflow_cancel_exception` and its exception +type while still applying the local cancellation. A linked terminal event or +the original call deadline that wins during propagation is preserved, together +with its result/failure and the propagation receipt. A later duplicate cannot +replace that terminal outcome or send another propagation request. + ## Reference-based payload storage Result and failure payloads on a service call follow the same diff --git a/src/V2/Support/LinkedServiceCallCloser.php b/src/V2/Support/LinkedServiceCallCloser.php index 351abc4d..eaa07865 100644 --- a/src/V2/Support/LinkedServiceCallCloser.php +++ b/src/V2/Support/LinkedServiceCallCloser.php @@ -4,6 +4,7 @@ namespace Workflow\V2\Support; +use Throwable; use Workflow\V2\Enums\HistoryEventType; use Workflow\V2\Enums\ServiceCallBindingKind; use Workflow\V2\Enums\ServiceCallOutcome; @@ -14,6 +15,8 @@ final class LinkedServiceCallCloser { + private const CONTINUATION_HOPS_PER_PASS = 32; + public static function reconcile(WorkflowServiceCall $call): void { if (in_array($call->status, ['pending', 'accepted', 'started'], true)) { @@ -26,29 +29,45 @@ public static function reconcile(WorkflowServiceCall $call): void return; } - /** @var WorkflowRun|null $run */ - $run = ConfiguredV2Models::query('run_model', WorkflowRun::class) - ->where('namespace', $call->target_namespace) - ->where('workflow_instance_id', $call->linked_workflow_instance_id) - ->find($call->linked_workflow_run_id); - if ($run === null) { - return; - } - - /** @var WorkflowHistoryEvent|null $event */ - $event = ConfiguredV2Models::query('history_event_model', WorkflowHistoryEvent::class) - ->where('workflow_run_id', $run->id) - ->whereIn('event_type', self::terminalEventTypes()) - ->orderBy('sequence') - ->first(); - if ($event !== null) { + for ($hop = 0; $hop < self::CONTINUATION_HOPS_PER_PASS; $hop++) { + /** @var WorkflowRun|null $run */ + $run = ConfiguredV2Models::query('run_model', WorkflowRun::class) + ->where('namespace', $call->target_namespace) + ->where('workflow_instance_id', $call->linked_workflow_instance_id) + ->find($call->linked_workflow_run_id); + if ($run === null) { + return; + } + /** @var WorkflowHistoryEvent|null $event */ + $event = ConfiguredV2Models::query('history_event_model', WorkflowHistoryEvent::class) + ->where('workflow_run_id', $run->id) + ->whereIn( + 'event_type', + [...self::terminalEventTypes(), HistoryEventType::WorkflowContinuedAsNew->value] + ) + ->orderBy('sequence') + ->first(); + if ($event === null) { + return; + } self::closeForTerminalEvent($run, $event); $call->refresh(); + if (! in_array($call->status, ['pending', 'accepted', 'started'], true) + || $call->linked_workflow_run_id === $run->id) { + return; + } } } public static function closeForTerminalEvent(WorkflowRun $run, WorkflowHistoryEvent $event): void { + if ($event->workflow_run_id !== $run->id) { + return; + } + if ($event->event_type === HistoryEventType::WorkflowContinuedAsNew) { + self::advanceForContinuation($run, $event); + return; + } $terminal = match ($event->event_type) { HistoryEventType::WorkflowCompleted => [ ServiceCallStatus::Completed, @@ -140,6 +159,128 @@ public static function closeForTerminalEvent(WorkflowRun $run, WorkflowHistoryEv }); } + /** + * Catch up long chains without making an API read traverse unbounded history. + * @return array{candidates: int, reconciled: int, failures: list} + */ + public static function repairDue( + int $limit, + array $runIds = [], + ?string $instanceId = null, + ?string $connection = null, + ?string $queue = null + ): array { + $query = ConfiguredV2Models::query('service_call_model', WorkflowServiceCall::class) + ->where('resolved_binding_kind', ServiceCallBindingKind::WorkflowRun->value) + ->whereIn('status', ['pending', 'accepted', 'started']) + ->whereHas('linkedRun', static function ($run): void { + $run->whereHas('historyEvents', static function ($history): void { + $history->whereIn( + 'event_type', + [...self::terminalEventTypes(), HistoryEventType::WorkflowContinuedAsNew->value] + ); + }); + }); + if ($runIds !== []) { + $query->where(static function ($scope) use ($runIds): void { + $scope->whereIn('caller_workflow_run_id', $runIds) + ->orWhereIn('linked_workflow_run_id', $runIds); + }); + } + if ($instanceId !== null) { + $query->where(static function ($scope) use ($instanceId): void { + $scope->where('caller_workflow_instance_id', $instanceId) + ->orWhere('linked_workflow_instance_id', $instanceId); + }); + } + if ($connection !== null || $queue !== null) { + $runScope = static function ($run) use ($connection, $queue): void { + if ($connection !== null) { + $run->where('connection', $connection); + } + if ($queue !== null) { + $run->where('queue', $queue); + } + }; + $query->where(static function ($scope) use ($runScope): void { + $scope->whereHas('callerRun', $runScope) + ->orWhereHas('linkedRun', $runScope); + }); + } + $calls = $query->orderBy('updated_at') + ->orderBy('id') + ->limit(max(1, $limit)) + ->get(); + $report = [ + 'candidates' => $calls->count(), + 'reconciled' => 0, + 'failures' => [], + ]; + foreach ($calls as $call) { + if (! $call instanceof WorkflowServiceCall) { + continue; + } + try { + $before = [$call->status, $call->linked_workflow_run_id]; + self::reconcile($call); + if ($before !== [$call->status, $call->linked_workflow_run_id]) { + $report['reconciled']++; + } + } catch (Throwable $failure) { + report($failure); + $report['failures'][] = [ + 'call_id' => $call->id, + 'message' => $failure->getMessage(), + ]; + } + } + return $report; + } + + private static function advanceForContinuation(WorkflowRun $run, WorkflowHistoryEvent $event): void + { + $nextId = $event->payload['continued_to_run_id'] ?? null; + if (! is_string($nextId) || $nextId === '') { + return; + } + /** @var WorkflowRun|null $next */ + $next = ConfiguredV2Models::query('run_model', WorkflowRun::class) + ->where('namespace', $run->namespace) + ->where('workflow_instance_id', $run->workflow_instance_id) + ->where('workflow_type', $run->workflow_type) + ->where('run_number', $run->run_number + 1) + ->find($nextId); + if ($next === null || ($event->payload['continued_to_run_number'] ?? null) !== $next->run_number) { + return; + } + ConfiguredV2Models::query('service_call_model', WorkflowServiceCall::class) + ->getModel() + ->getConnection() + ->transaction(static function () use ($run, $next, $event): void { + $calls = ConfiguredV2Models::query('service_call_model', WorkflowServiceCall::class) + ->where('target_namespace', $run->namespace) + ->where('linked_workflow_instance_id', $run->workflow_instance_id) + ->where('linked_workflow_run_id', $run->id) + ->where('resolved_binding_kind', ServiceCallBindingKind::WorkflowRun->value) + ->whereIn('status', ['pending', 'accepted', 'started']) + ->lockForUpdate() + ->get(); + foreach ($calls as $call) { + if (! $call instanceof WorkflowServiceCall || ServiceCallDeadline::expireLocked($call, now())) { + continue; + } + $metadata = is_array($call->metadata) ? $call->metadata : []; + $metadata['linked_initial_workflow_run_id'] ??= $run->id; + $metadata['linked_continuation_event_id'] = $event->id; + $call->forceFill([ + 'linked_workflow_run_id' => $next->id, + 'resolved_target_reference' => $next->id, + 'metadata' => $metadata, + ])->save(); + } + }); + } + /** * @return list */ diff --git a/src/V2/TaskWatchdog.php b/src/V2/TaskWatchdog.php index 23c83e15..96fdada7 100644 --- a/src/V2/TaskWatchdog.php +++ b/src/V2/TaskWatchdog.php @@ -25,6 +25,7 @@ use Workflow\V2\Models\WorkflowTask; use Workflow\V2\Support\ActivityTimeoutEnforcer; use Workflow\V2\Support\ConfiguredV2Models; +use Workflow\V2\Support\LinkedServiceCallCloser; use Workflow\V2\Support\RequestedTaskScope; use Workflow\V2\Support\ServiceCallDeadline; use Workflow\V2\Support\ServiceCallScheduler; @@ -90,7 +91,10 @@ public static function wake(?string $connection = null, ?string $queue = null): * service_call_timeout_failures: list * service_call_dispatch_candidates: int, * service_call_dispatches_enqueued: int, - * service_call_dispatch_failures: list + * service_call_dispatch_failures: list, + * service_call_linked_candidates: int, + * service_call_links_reconciled: int, + * service_call_linked_failures: list * } */ public static function runPass( @@ -230,6 +234,17 @@ public static function runPass( $report['service_call_dispatches_enqueued'] = $serviceCallDispatch['enqueued']; $report['service_call_dispatch_failures'] = $serviceCallDispatch['failures']; + $linkedCalls = LinkedServiceCallCloser::repairDue( + TaskRepairPolicy::scanLimit(), + $runIds, + $instanceId, + $connection, + $queue + ); + $report['service_call_linked_candidates'] = $linkedCalls['candidates']; + $report['service_call_links_reconciled'] = $linkedCalls['reconciled']; + $report['service_call_linked_failures'] = $linkedCalls['failures']; + $activityTimeoutIds = ActivityTimeoutEnforcer::expiredExecutionIds( TaskRepairPolicy::scanLimit(), $connection, @@ -448,7 +463,10 @@ private static function recoverMissingTask(string $runId): array * service_call_timeout_failures: list * service_call_dispatch_candidates: int, * service_call_dispatches_enqueued: int, - * service_call_dispatch_failures: list + * service_call_dispatch_failures: list, + * service_call_linked_candidates: int, + * service_call_links_reconciled: int, + * service_call_linked_failures: list * } */ private static function emptyReport( @@ -487,6 +505,9 @@ private static function emptyReport( 'service_call_dispatch_candidates' => 0, 'service_call_dispatches_enqueued' => 0, 'service_call_dispatch_failures' => [], + 'service_call_linked_candidates' => 0, + 'service_call_links_reconciled' => 0, + 'service_call_linked_failures' => [], ]; } diff --git a/tests/Unit/Commands/V2RepairPassCommandTest.php b/tests/Unit/Commands/V2RepairPassCommandTest.php index 632e2675..321b0761 100644 --- a/tests/Unit/Commands/V2RepairPassCommandTest.php +++ b/tests/Unit/Commands/V2RepairPassCommandTest.php @@ -81,6 +81,9 @@ public function testItIgnoresTheLoopThrottleByDefaultAndRepairsMissingSignalTask 'service_call_dispatch_candidates' => 0, 'service_call_dispatches_enqueued' => 0, 'service_call_dispatch_failures' => [], + 'service_call_linked_candidates' => 0, + 'service_call_links_reconciled' => 0, + 'service_call_linked_failures' => [], ]; $this->artisan('workflow:v2:repair-pass', [ @@ -153,6 +156,9 @@ public function testRespectThrottleOptionSkipsRepairWhenTheLoopThrottleIsHeld(): 'service_call_dispatch_candidates' => 0, 'service_call_dispatches_enqueued' => 0, 'service_call_dispatch_failures' => [], + 'service_call_linked_candidates' => 0, + 'service_call_links_reconciled' => 0, + 'service_call_linked_failures' => [], ]; $this->artisan('workflow:v2:repair-pass', [ @@ -357,6 +363,9 @@ public function testRunIdScopeRepairsOnlyTheSelectedMissingTaskRun(): void 'service_call_dispatch_candidates' => 0, 'service_call_dispatches_enqueued' => 0, 'service_call_dispatch_failures' => [], + 'service_call_linked_candidates' => 0, + 'service_call_links_reconciled' => 0, + 'service_call_linked_failures' => [], ]; $this->artisan('workflow:v2:repair-pass', [ @@ -419,6 +428,9 @@ public function testInstanceIdScopeRepairsOnlyTheSelectedInstanceTasks(): void 'service_call_dispatch_candidates' => 0, 'service_call_dispatches_enqueued' => 0, 'service_call_dispatch_failures' => [], + 'service_call_linked_candidates' => 0, + 'service_call_links_reconciled' => 0, + 'service_call_linked_failures' => [], ]; $this->artisan('workflow:v2:repair-pass', [ @@ -498,6 +510,9 @@ public function testConnectionAndQueueScopeRepairsOnlyMatchingExistingAndMissing 'service_call_dispatch_candidates' => 0, 'service_call_dispatches_enqueued' => 0, 'service_call_dispatch_failures' => [], + 'service_call_linked_candidates' => 0, + 'service_call_links_reconciled' => 0, + 'service_call_linked_failures' => [], ]; $this->artisan('workflow:v2:repair-pass', [ @@ -603,6 +618,9 @@ public function testCommaSeparatedQueueScopeRepairsEveryRequestedQueue(): void 'service_call_dispatch_candidates' => 0, 'service_call_dispatches_enqueued' => 0, 'service_call_dispatch_failures' => [], + 'service_call_linked_candidates' => 0, + 'service_call_links_reconciled' => 0, + 'service_call_linked_failures' => [], ]; $this->artisan('workflow:v2:repair-pass', [ @@ -683,6 +701,9 @@ public function testConnectionAndQueueScopeRepairsOnlyMatchingDeadlineExpiredRun 'service_call_dispatch_candidates' => 0, 'service_call_dispatches_enqueued' => 0, 'service_call_dispatch_failures' => [], + 'service_call_linked_candidates' => 0, + 'service_call_links_reconciled' => 0, + 'service_call_linked_failures' => [], ]; $this->artisan('workflow:v2:repair-pass', [ @@ -796,6 +817,9 @@ public function testConnectionAndQueueScopeEnforcesOnlyMatchingActivityTimeouts( 'service_call_dispatch_candidates' => 0, 'service_call_dispatches_enqueued' => 0, 'service_call_dispatch_failures' => [], + 'service_call_linked_candidates' => 0, + 'service_call_links_reconciled' => 0, + 'service_call_linked_failures' => [], ]; $this->artisan('workflow:v2:repair-pass', [ @@ -862,6 +886,9 @@ public function testLoopModeRunsTheRequestedNumberOfIterationsAndForcesThrottleA 'service_call_dispatch_candidates' => 0, 'service_call_dispatches_enqueued' => 0, 'service_call_dispatch_failures' => [], + 'service_call_linked_candidates' => 0, + 'service_call_links_reconciled' => 0, + 'service_call_linked_failures' => [], ]; $secondIteration = [ @@ -892,6 +919,9 @@ public function testLoopModeRunsTheRequestedNumberOfIterationsAndForcesThrottleA 'service_call_dispatch_candidates' => 0, 'service_call_dispatches_enqueued' => 0, 'service_call_dispatch_failures' => [], + 'service_call_linked_candidates' => 0, + 'service_call_links_reconciled' => 0, + 'service_call_linked_failures' => [], ]; $this->artisan('workflow:v2:repair-pass', [ diff --git a/tests/Unit/V2/LinkedServiceCallContinuationTest.php b/tests/Unit/V2/LinkedServiceCallContinuationTest.php new file mode 100644 index 00000000..8e5d3ca1 --- /dev/null +++ b/tests/Unit/V2/LinkedServiceCallContinuationTest.php @@ -0,0 +1,242 @@ +createRun(); + $call = $this->createServiceCall($run); + $deadline = $call->call_deadline_at->toJSON(); + $accepted = $call->accepted_at->toJSON(); + $next = $this->continueRun($run); + $this->assertSame('started', $call->refresh()->status); + $this->assertSame($next->id, $call->linked_workflow_run_id); + $this->assertSame($next->id, $call->resolved_target_reference); + $this->assertSame($run->id, $call->metadata['linked_initial_workflow_run_id']); + $this->assertSame($deadline, $call->call_deadline_at->toJSON()); + $this->assertSame($accepted, $call->accepted_at->toJSON()); + $this->assertSame(1, WorkflowServiceCall::query()->count()); + WorkflowHistoryEvent::record($next, HistoryEventType::WorkflowCompleted, [ + 'output' => Serializer::serializeWithCodec('avro', [ + 'answer' => false, + 'source' => 'Ω', + ]), + 'payload_codec' => 'avro', + ]); + $this->assertSame('completed', $call->refresh()->status); + $this->assertSame([ + 'answer' => false, + 'source' => 'Ω', + ], Serializer::unserializeWithCodec('avro', $call->metadata['nexus_result']['blob'])); + $before = $call->getRawOriginal(); + LinkedServiceCallCloser::reconcile($call); + $this->assertSame($before, $call->fresh()->getRawOriginal()); + } + + public function testContinuationAndCompletionBeforeAdmissionAreReconciledFromCanonicalHistory(): void + { + Queue::fake(); + $run = $this->createRun(); + $next = $this->continueRun($run); + WorkflowHistoryEvent::record($next, HistoryEventType::WorkflowCompleted, [ + 'output' => Serializer::serializeWithCodec('avro', null), + 'payload_codec' => 'avro', + ]); + $call = $this->createServiceCall($run); + LinkedServiceCallCloser::reconcile($call); + $this->assertSame('completed', $call->status); + $this->assertSame($next->id, $call->linked_workflow_run_id); + $this->assertSame(Serializer::serializeWithCodec('avro', null), $call->metadata['nexus_result']['blob']); + } + + public function testLongContinuationHistoryIsBoundedAndTheNormalRepairPassFinishesIt(): void + { + Queue::fake(); + $first = $this->createRun(); + $last = $first; + for ($i = 0; $i < 40; $i++) { + $last = $this->continueRun($last); + } + WorkflowHistoryEvent::record($last, HistoryEventType::WorkflowCompleted, [ + 'output' => Serializer::serializeWithCodec('avro', 'done'), + 'payload_codec' => 'avro', + ]); + $call = $this->createServiceCall($first); + LinkedServiceCallCloser::reconcile($call); + $this->assertSame('started', $call->status); + $this->assertNotSame($first->id, $call->linked_workflow_run_id); + $this->assertNotSame($last->id, $call->linked_workflow_run_id); + $report = TaskWatchdog::runPass('sync', 'default', runIds: [$call->linked_workflow_run_id]); + $this->assertSame([], $report['service_call_linked_failures']); + $this->assertSame(1, $report['service_call_links_reconciled']); + $this->assertSame('completed', $call->refresh()->status); + $this->assertSame($last->id, $call->linked_workflow_run_id); + } + + #[DataProvider('invalidSuccessors')] + public function testContinuationCannotFollowAnUnrelatedRun(string $changedField, mixed $value): void + { + Queue::fake(); + $run = $this->createRun(); + $next = $this->createRun(); + if ($changedField === 'workflow_instance_id') { + $value = $next->workflow_instance_id; + } + $next->forceFill([ + 'workflow_instance_id' => $run->workflow_instance_id, + 'run_number' => 2, + $changedField => $value, + ])->save(); + $call = $this->createServiceCall($run); + WorkflowHistoryEvent::record($run, HistoryEventType::WorkflowContinuedAsNew, [ + 'continued_to_run_id' => $next->id, + 'continued_to_run_number' => 2, + 'closed_reason' => 'continued', + ]); + LinkedServiceCallCloser::reconcile($call); + $this->assertSame('started', $call->refresh()->status); + $this->assertSame($run->id, $call->linked_workflow_run_id); + $this->assertNull($call->completed_at); + } + + public function testAReplacementCurrentRunIsNotACanonicalContinuation(): void + { + Queue::fake(); + $run = $this->createRun(); + $replacement = $run->replicate(); + $replacement->forceFill([ + 'run_number' => 2, + ])->save(); + $run->instance->forceFill([ + 'current_run_id' => $replacement->id, + 'run_count' => 2, + ])->save(); + $call = $this->createServiceCall($run); + WorkflowHistoryEvent::record($run, HistoryEventType::WorkflowFailed, [ + 'exception_class' => 'OriginalFailure', + 'message' => 'original failed', + ]); + WorkflowHistoryEvent::record($replacement, HistoryEventType::WorkflowCompleted, [ + 'output' => Serializer::serializeWithCodec('avro', 'unrelated replacement'), + 'payload_codec' => 'avro', + ]); + LinkedServiceCallCloser::reconcile($call); + $this->assertSame('failed', $call->refresh()->status); + $this->assertSame($run->id, $call->linked_workflow_run_id); + $this->assertSame('OriginalFailure', $call->outcome_metadata['service_error_type']); + $this->assertArrayNotHasKey('nexus_result', $call->metadata); + } + + public static function invalidSuccessors(): iterable + { + yield 'namespace' => ['namespace', 'foreign']; + yield 'instance' => ['workflow_instance_id', 'foreign-instance']; + yield 'type' => ['workflow_type', 'foreign-type']; + yield 'backwards run number' => ['run_number', 0]; + } + + private function createRun(): WorkflowRun + { + $instance = WorkflowInstance::query()->create([ + 'namespace' => 'service', + 'workflow_type' => 'nexus.service', + 'workflow_class' => 'nexus.service', + 'run_count' => 1, + ]); + return WorkflowRun::query()->create([ + 'workflow_instance_id' => $instance->id, + 'namespace' => 'service', + 'run_number' => 1, + 'workflow_type' => 'nexus.service', + 'workflow_class' => 'nexus.service', + 'status' => RunStatus::Waiting, + 'payload_codec' => 'avro', + 'arguments' => Serializer::serializeWithCodec('avro', []), + 'connection' => 'sync', + 'queue' => 'default', + 'started_at' => now(), + ]); + } + + private function createServiceCall(WorkflowRun $run): WorkflowServiceCall + { + return WorkflowServiceCall::query()->create([ + 'namespace' => 'service', + 'target_namespace' => 'service', + 'caller_namespace' => 'caller', + 'endpoint_name' => 'endpoint', + 'service_name' => 'service', + 'operation_name' => 'operation', + 'resolved_binding_kind' => 'workflow_run', + 'resolved_target_reference' => $run->id, + 'linked_workflow_instance_id' => $run->workflow_instance_id, + 'linked_workflow_run_id' => $run->id, + 'status' => 'started', + 'outcome' => 'accepted', + 'operation_mode' => 'sync', + 'payload_codec' => 'avro', + 'accepted_at' => now(), + 'started_at' => now(), + 'call_deadline_at' => now() + ->addHour(), + ]); + } + + private function continueRun(WorkflowRun $run): WorkflowRun + { + return $run->getConnection() + ->transaction(static function () use ($run): WorkflowRun { + $next = $run->replicate(); + $next->forceFill([ + 'run_number' => $run->run_number + 1, + 'last_history_sequence' => 0, + ])->save(); + $run->instance->forceFill([ + 'current_run_id' => $next->id, + 'run_count' => $next->run_number, + ])->save(); + $link = WorkflowLink::query()->create([ + 'link_type' => 'continue_as_new', + 'sequence' => 1, + 'is_primary_parent' => true, + 'parent_workflow_instance_id' => $run->workflow_instance_id, + 'parent_workflow_run_id' => $run->id, + 'child_workflow_instance_id' => $next->workflow_instance_id, + 'child_workflow_run_id' => $next->id, + ]); + $task = WorkflowTask::query()->create([ + 'workflow_run_id' => $run->id, + 'namespace' => $run->namespace, + 'task_type' => TaskType::Workflow, + 'status' => TaskStatus::Completed, + 'connection' => 'sync', + 'queue' => 'default', + ]); + ContinuedRunMessageHandoff::closePredecessor($run, $next, $task, 1, $link->id, now()); + return $next; + }); + } +} From 459579f5d17c7c6f0e169643d255e952c663883c Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Mon, 26 Oct 2026 23:58:19 +0000 Subject: [PATCH 05/10] Close workflow-bound service updates and durable signal receipts --- composer.json | 2 +- .../workflow-service-calls-architecture.md | 20 ++ src/V2/Support/DefaultServiceControlPlane.php | 9 +- src/V2/Support/LinkedServiceCallCloser.php | 91 +++++++- .../V2/DefaultServiceControlPlaneTest.php | 2 +- .../V2/ServiceCallWorkflowLifecycleTest.php | 195 ++++++++++++++++++ 6 files changed, 308 insertions(+), 11 deletions(-) create mode 100644 tests/Unit/V2/ServiceCallWorkflowLifecycleTest.php diff --git a/composer.json b/composer.json index 3460fd41..5b43d2fd 100644 --- a/composer.json +++ b/composer.json @@ -87,7 +87,7 @@ "dev-main": "2.0.x-dev" }, "durable-workflow": { - "product-train": "2.15.0", + "product-train": "2.15.1", "laravel-embedded-upgrade-contract": "resources/laravel-embedded-upgrade-contract.json", "laravel-dependency-security-policy": "resources/laravel-dependency-security-policy.json" }, diff --git a/docs/architecture/workflow-service-calls-architecture.md b/docs/architecture/workflow-service-calls-architecture.md index 2e30b590..ff0d17ac 100644 --- a/docs/architecture/workflow-service-calls-architecture.md +++ b/docs/architecture/workflow-service-calls-architecture.md @@ -351,6 +351,26 @@ One reconciliation read follows at most 32 continuation hops. The existing repair pass resumes longer chains, within its normal candidate and queue/run scope limits. No polling client is required to finish that reconciliation. +### Workflow binding completion + +| Binding | Successful completion | While awaiting completion | +|---------|-----------------------|---------------------------| +| WorkflowRun | The final run in its canonical continuation chain completes, preserving its result. | The call remains Started; run failure, timeout, cancellation and the original call deadline retain their typed outcomes. | +| WorkflowUpdate | The accepted update records its actual `UpdateCompleted` result. | Accepted-only admission remains Started; later completion/failure is delivered by history, including a completion committed before the call records its update link. | +| WorkflowSignal | The signal's durable command receipt is accepted; the result is Avro null. | Completion confirms receipt. The target may still be waiting and the signal may still be Received rather than Applied. | +| WorkflowQuery | The host returns the query result. | Host query admission and worker-wait budgets apply; typed query failure closes the call without waiting for workflow completion. | + +Update failure/rejection closes only the linked operation; it does not require +the target workflow itself to finish. The immutable update id follows an +instance-scoped update handed to a continued run. Results preserve null, false, +zero, empty and nested values, and use the same inline/external storage +contract for all completing bindings. The reserved `nexus_result` representation +is replaced atomically rather than recursively merged with old result metadata. +JSON argument lists and Avro-encoded argument lists deliver the same values to +the target and its durable command receipt. Opaque serialized argument +references resolve in the target service request's namespace; caller namespace +policy metadata grants no storage authority. + ## Deadline, cancellation, retry, and idempotency Every operation declares the policies it accepts at the contract diff --git a/src/V2/Support/DefaultServiceControlPlane.php b/src/V2/Support/DefaultServiceControlPlane.php index 92515c09..3a27e306 100644 --- a/src/V2/Support/DefaultServiceControlPlane.php +++ b/src/V2/Support/DefaultServiceControlPlane.php @@ -1536,7 +1536,7 @@ private function dispatchWorkflowSignal( if (($result['accepted'] ?? false) === true) { $commandId = $this->stringFrom($result['workflow_command_id'] ?? null); - $this->markStarted( + $completed = $this->markCompleted( $call, ServiceCallBindingKind::WorkflowSignal->value, $commandId ?? $signalName, @@ -1548,7 +1548,14 @@ private function dispatchWorkflowSignal( 'control_plane' => $this->publicResult($result), ], $claim, + [ + 'codec' => $call->payload_codec, + 'blob' => Serializer::serializeWithCodec($call->payload_codec, null), + ], ); + if ($completed) { + $this->releaseBoundaryAdmission($request); + } return [ 'accepted' => true, diff --git a/src/V2/Support/LinkedServiceCallCloser.php b/src/V2/Support/LinkedServiceCallCloser.php index eaa07865..a1248aec 100644 --- a/src/V2/Support/LinkedServiceCallCloser.php +++ b/src/V2/Support/LinkedServiceCallCloser.php @@ -12,6 +12,7 @@ use Workflow\V2\Models\WorkflowHistoryEvent; use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowServiceCall; +use Workflow\V2\Models\WorkflowUpdate; final class LinkedServiceCallCloser { @@ -23,6 +24,11 @@ public static function reconcile(WorkflowServiceCall $call): void ServiceCallDeadline::expire($call); } ServiceCallWorkflowDelivery::enqueue($call); + if ($call->resolved_binding_kind === ServiceCallBindingKind::WorkflowUpdate->value + && in_array($call->status, ['pending', 'accepted', 'started'], true)) { + self::reconcileUpdate($call); + return; + } if ($call->resolved_binding_kind !== ServiceCallBindingKind::WorkflowRun->value || ! in_array($call->status, ['pending', 'accepted', 'started'], true) || ! is_string($call->linked_workflow_run_id)) { @@ -68,6 +74,21 @@ public static function closeForTerminalEvent(WorkflowRun $run, WorkflowHistoryEv self::advanceForContinuation($run, $event); return; } + $update = null; + if (in_array($event->event_type, [HistoryEventType::UpdateCompleted, HistoryEventType::UpdateRejected], true)) { + $updateId = $event->payload['update_id'] ?? null; + if (! is_string($updateId) || $updateId === '') { + return; + } + /** @var WorkflowUpdate|null $update */ + $update = ConfiguredV2Models::query('update_model', WorkflowUpdate::class) + ->where('workflow_instance_id', $run->workflow_instance_id) + ->where('workflow_run_id', $run->id) + ->find($updateId); + if ($update === null) { + return; + } + } $terminal = match ($event->event_type) { HistoryEventType::WorkflowCompleted => [ ServiceCallStatus::Completed, @@ -84,6 +105,15 @@ public static function closeForTerminalEvent(WorkflowRun $run, WorkflowHistoryEv ServiceCallOutcome::HandlerFailed, 'failed_at', ], + HistoryEventType::UpdateCompleted => ($event->payload['failure_id'] ?? null) !== null + || isset($event->payload['exception_class']) || isset($event->payload['exception_type']) + ? [ServiceCallStatus::Failed, ServiceCallOutcome::HandlerFailed, 'failed_at'] + : [ServiceCallStatus::Completed, ServiceCallOutcome::Completed, 'completed_at'], + HistoryEventType::UpdateRejected => [ + ServiceCallStatus::Failed, + ServiceCallOutcome::HandlerFailed, + 'failed_at', + ], default => null, }; if ($terminal === null || $event->workflow_run_id !== $run->id) { @@ -93,14 +123,21 @@ public static function closeForTerminalEvent(WorkflowRun $run, WorkflowHistoryEv ConfiguredV2Models::query('service_call_model', WorkflowServiceCall::class) ->getModel() ->getConnection() - ->transaction(static function () use ($run, $event, $terminal): void { - $calls = ConfiguredV2Models::query('service_call_model', WorkflowServiceCall::class) + ->transaction(static function () use ($run, $event, $terminal, $update): void { + $query = ConfiguredV2Models::query('service_call_model', WorkflowServiceCall::class) ->where('target_namespace', $run->namespace) ->where('linked_workflow_instance_id', $run->workflow_instance_id) - ->where('linked_workflow_run_id', $run->id) - ->where('resolved_binding_kind', ServiceCallBindingKind::WorkflowRun->value) - ->whereIn('status', ['pending', 'accepted', 'started']) - ->lockForUpdate() + ->whereIn('status', ['pending', 'accepted', 'started']); + if ($update !== null) { + // The immutable update id also follows an instance-scoped + // update handed to a canonical continued run. + $query->where('resolved_binding_kind', ServiceCallBindingKind::WorkflowUpdate->value) + ->where('linked_workflow_update_id', $update->id); + } else { + $query->where('linked_workflow_run_id', $run->id) + ->where('resolved_binding_kind', ServiceCallBindingKind::WorkflowRun->value); + } + $calls = $query->lockForUpdate() ->get(); foreach ($calls as $call) { @@ -121,11 +158,16 @@ public static function closeForTerminalEvent(WorkflowRun $run, WorkflowHistoryEv 'outcome_reason' => $outcome->value, $timestamp => $event->recorded_at, ]; + if ($update !== null) { + $values['linked_workflow_run_id'] = $run->id; + $values['resolved_target_reference'] = $update->id; + } if ($status === ServiceCallStatus::Completed) { - $codec = $payload['payload_codec'] ?? $run->output_payload_codec ?? $call->payload_codec; - $output = $payload['output'] ?? null; + $codec = $payload['payload_codec'] ?? $update?->payload_codec ?? $run->output_payload_codec ?? $call->payload_codec; + $output = $payload[$update !== null ? 'result' : 'output'] ?? null; if (is_string($output)) { + $values['output_payload_reference'] = null; $metadata['nexus_result'] = [ 'codec' => $codec, 'blob' => $output, @@ -237,6 +279,39 @@ public static function repairDue( return $report; } + private static function reconcileUpdate(WorkflowServiceCall $call): void + { + if (! is_string($call->linked_workflow_update_id)) { + return; + } + /** @var WorkflowUpdate|null $update */ + $update = ConfiguredV2Models::query('update_model', WorkflowUpdate::class) + ->where('workflow_instance_id', $call->linked_workflow_instance_id) + ->find($call->linked_workflow_update_id); + if ($update === null) { + return; + } + /** @var WorkflowRun|null $run */ + $run = ConfiguredV2Models::query('run_model', WorkflowRun::class) + ->where('namespace', $call->target_namespace) + ->where('workflow_instance_id', $call->linked_workflow_instance_id) + ->find($update->workflow_run_id); + if ($run === null) { + return; + } + /** @var WorkflowHistoryEvent|null $event */ + $event = ConfiguredV2Models::query('history_event_model', WorkflowHistoryEvent::class) + ->where('workflow_run_id', $run->id) + ->where('payload->update_id', $update->id) + ->whereIn('event_type', [HistoryEventType::UpdateCompleted->value, HistoryEventType::UpdateRejected->value]) + ->orderBy('sequence') + ->first(); + if ($event !== null) { + self::closeForTerminalEvent($run, $event); + $call->refresh(); + } + } + private static function advanceForContinuation(WorkflowRun $run, WorkflowHistoryEvent $event): void { $nextId = $event->payload['continued_to_run_id'] ?? null; diff --git a/tests/Unit/V2/DefaultServiceControlPlaneTest.php b/tests/Unit/V2/DefaultServiceControlPlaneTest.php index c4719acd..85bdfc1a 100644 --- a/tests/Unit/V2/DefaultServiceControlPlaneTest.php +++ b/tests/Unit/V2/DefaultServiceControlPlaneTest.php @@ -331,7 +331,7 @@ public function testWorkflowSignalBindingDispatchesThroughWorkflowControlPlaneAn ]); $this->assertTrue($result['accepted']); - $this->assertSame(ServiceCallStatus::Started->value, $result['status']); + $this->assertSame(ServiceCallStatus::Completed->value, $result['status']); $this->assertSame(ServiceCallBindingKind::WorkflowSignal->value, $result['resolved_binding_kind']); $this->assertSame('command-signal-1', $result['resolved_target_reference']); $this->assertSame('invoice-42', $result['linked_workflow_instance_id']); diff --git a/tests/Unit/V2/ServiceCallWorkflowLifecycleTest.php b/tests/Unit/V2/ServiceCallWorkflowLifecycleTest.php new file mode 100644 index 00000000..a1cf3a6e --- /dev/null +++ b/tests/Unit/V2/ServiceCallWorkflowLifecycleTest.php @@ -0,0 +1,195 @@ +set('queue.default', 'database'); + $started = (new DefaultWorkflowControlPlane())->start(TestUpdateWorkflow::class, 'lifecycle-target', [ + 'namespace' => 'billing', + 'queue' => 'orders', + 'arguments' => Serializer::serialize([]), + ]); + $this->target = WorkflowRun::query()->findOrFail($started['workflow_run_id']); + $this->app->call([new RunWorkflowTask($started['task_id']), 'handle']); + } + + #[DataProvider('updates')] + public function testAcceptedUpdatesCloseOnTheirActualTerminalEvent( + string $name, + array $arguments, + bool $failed, + bool $lateLink, + bool $external = false + ): void { + if ($external) { + $this->app->instance( + ExternalPayloadStoragePolicy::class, + new RecordingExternalPayloadStorage(threshold: 32) + ); + } + $this->operation('workflow_update', $name); + $workflows = new DefaultWorkflowControlPlane(); + if ($lateLink) { + $adapter = Mockery::mock(WorkflowControlPlane::class); + $adapter->shouldReceive('update') + ->once() + ->andReturnUsing(function (string $id, string $name, array $options) use ($workflows): array { + $result = $workflows->update($id, $name, $options); + $this->runReadyTargetTask(); + return $result; + }); + $workflows = $adapter; + } + $result = (new DefaultServiceControlPlane($workflows))->execute('billing', 'service', 'operation', [ + 'namespace' => 'billing', + 'arguments' => $arguments, + 'wait_for' => 'accepted', + ]); + $this->assertSame(! ($failed && $lateLink), $result['accepted']); + $call = WorkflowServiceCall::query()->sole(); + $foreign = null; + if (! $lateLink) { + $this->assertSame('started', $call->status); + $foreign = $call->replicate(); + $foreign->forceFill([ + 'namespace' => 'foreign', + 'target_namespace' => 'foreign', + ])->save(); + $this->runReadyTargetTask(); + } + $this->assertSame($failed ? 'failed' : 'completed', $call->refresh()->status); + $this->assertNotNull($call->linked_workflow_update_id); + if ($failed) { + $this->assertSame(\RuntimeException::class, $call->outcome_metadata['service_error_type']); + $this->assertSame('fixture update failure', $call->failure_message); + $this->assertNull(ServiceCallResultEnvelope::fromCall($call)); + } else { + $envelope = ServiceCallResultEnvelope::fromCall($call); + $this->assertNotNull($envelope); + $blob = ExternalPayloads::payloadBlob($envelope, $envelope['codec'], 'billing'); + $this->assertIsString($blob); + $value = Serializer::unserializeWithCodec($envelope['codec'], $blob); + $this->assertSame($external, $call->output_payload_reference !== null); + $this->assertSame($name === 'largeResult' ? [ + 'length' => 2048, + 'value' => str_repeat('u', 2048), + ] : [ + 'approved' => true, + 'events' => ['started', 'approved:yes:source Ω'], + ], $value); + } + if ($foreign !== null) { + $this->assertSame('started', $foreign->refresh()->status); + $this->assertArrayNotHasKey('nexus_result', $foreign->metadata); + } + $this->assertFalse($this->target->refresh()->status->isTerminal()); + } + + public static function updates(): iterable + { + foreach ([false, true] as $late) { + yield 'approval-' . (int) $late => ['approve', [true, 'source Ω'], false, $late]; + yield 'large-' . (int) $late => ['largeResult', [2048], false, $late]; + yield 'failure-' . (int) $late => ['explode', ['fixture update failure'], true, $late]; + yield 'external-' . (int) $late => ['largeResult', [2048], false, $late, true]; + } + } + + public function testSignalCompletesWithADurableReceiptBeforeTheWorkflowAppliesIt(): void + { + $this->operation('workflow_signal', 'name-provided'); + $control = new DefaultServiceControlPlane(); + $result = $control->execute('billing', 'service', 'operation', [ + 'namespace' => 'billing', + 'arguments' => ['name Ω'], + 'wait_for' => 'accepted', + ]); + $this->assertTrue($result['accepted']); + $call = WorkflowServiceCall::query()->sole(); + $this->assertSame('completed', $call->status); + $this->assertFalse($this->target->refresh()->status->isTerminal()); + $signal = WorkflowSignal::query()->sole(); + $this->assertSame('received', $signal->status->value); + $this->assertSame($signal->workflow_command_id, $call->resolved_target_reference); + $envelope = ServiceCallResultEnvelope::fromCall($call); + $this->assertSame([ + 'codec' => 'avro', + 'blob' => Serializer::serializeWithCodec('avro', null), + ], $envelope); + $before = $call->getRawOriginal(); + $control->execute('billing', 'service', 'operation', [ + 'namespace' => 'billing', + 'service_call_id' => $call->id, + ]); + $this->assertSame($before, $call->fresh()->getRawOriginal()); + $this->assertSame(1, WorkflowSignal::query()->count()); + } + + private function runReadyTargetTask(): void + { + $tasks = WorkflowTask::query()->where('workflow_run_id', $this->target->id)->where( + 'status', + TaskStatus::Ready->value + )->get(); + $this->assertCount(1, $tasks); + $this->app->call([new RunWorkflowTask($tasks->sole()->id), 'handle']); + } + + private function operation(string $kind, string $name): void + { + $endpoint = WorkflowServiceEndpoint::query()->create([ + 'namespace' => 'billing', + 'endpoint_name' => 'billing', + ]); + $service = WorkflowService::query()->create([ + 'namespace' => 'billing', + 'workflow_service_endpoint_id' => $endpoint->id, + 'service_name' => 'service', + ]); + WorkflowServiceOperation::query()->create([ + 'namespace' => 'billing', + 'workflow_service_endpoint_id' => $endpoint->id, + 'workflow_service_id' => $service->id, + 'operation_name' => 'operation', + 'operation_mode' => 'async', + 'handler_binding_kind' => $kind, + 'handler_target_reference' => $name, + 'handler_binding' => [ + 'workflow_instance_id' => $this->target->workflow_instance_id, + ], + ]); + } +} From 90e80990bdc6004d14bc96c32864f4d754335594 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:11:53 +0000 Subject: [PATCH 06/10] Authorize HTTP call replays and preserve issued handler identities --- src/V2/Support/DefaultServiceControlPlane.php | 20 ++- tests/Unit/V2/HttpServiceCallReplayTest.php | 158 ++++++++++++++++++ 2 files changed, 176 insertions(+), 2 deletions(-) create mode 100644 tests/Unit/V2/HttpServiceCallReplayTest.php diff --git a/src/V2/Support/DefaultServiceControlPlane.php b/src/V2/Support/DefaultServiceControlPlane.php index 3a27e306..d1ef68c3 100644 --- a/src/V2/Support/DefaultServiceControlPlane.php +++ b/src/V2/Support/DefaultServiceControlPlane.php @@ -150,7 +150,8 @@ public function execute( if ( $call instanceof WorkflowServiceCall && $explicitCallId !== null - && (string) $call->status === ServiceCallStatus::Started->value + && ((string) $call->status === ServiceCallStatus::Started->value + || $this->hasIssuedHandler($call)) ) { return $this->serialize($call) + [ 'accepted' => true, @@ -719,7 +720,8 @@ private function authorizeWorkflowCall( ): void { $run = $this->stringOption($options, 'caller_workflow_run_id'); $instance = $this->stringOption($options, 'caller_workflow_instance_id'); - if ($run === null && $instance === null) { + if ($run === null && $instance === null + && ! ($options['principal'] ?? null) instanceof ServiceCallPrincipal) { return; } WorkflowServiceOperationAuthority::authorizeExistingCall($call, $this->principal($options), [ @@ -734,6 +736,20 @@ private function authorizeWorkflowCall( ], $this->boundaryPolicy()); } + private function hasIssuedHandler(WorkflowServiceCall $call): bool + { + // Accepted also represents an issued activity/carrier that its lane + // has not started yet. A catalog target alone is only a template. + $field = match ((string) $call->resolved_binding_kind) { + ServiceCallBindingKind::ActivityExecution->value => 'activity_execution_id', + ServiceCallBindingKind::InvocableCarrierRequest->value => 'carrier_request_id', + default => null, + }; + $metadata = $this->arrayValue($call->metadata); + + return $field !== null && $this->stringFrom($metadata[$field] ?? null) !== null; + } + /** * @param array $options */ diff --git a/tests/Unit/V2/HttpServiceCallReplayTest.php b/tests/Unit/V2/HttpServiceCallReplayTest.php new file mode 100644 index 00000000..3f6a638b --- /dev/null +++ b/tests/Unit/V2/HttpServiceCallReplayTest.php @@ -0,0 +1,158 @@ +catalog('activity_execution'); + $control = new DefaultServiceControlPlane(); + $control->execute('billing', 'invoices', 'create', $this->executionOptions()); + $call = WorkflowServiceCall::query()->sole(); + $before = $call->getRawOriginal(); + try { + $control->execute('billing', 'invoices', 'create', array_replace($this->executionOptions(), [ + 'caller_namespace' => 'foreign', + ], $explicit ? [ + 'service_call_id' => $call->id, + ] : [])); + $this->fail('An HTTP caller must not reuse another caller namespace\'s key.'); + } catch (ServiceOperationAuthorizationException $exception) { + $this->assertSame('service_call_not_owned', $exception->getMessage()); + } + $this->assertSame($before, $call->fresh()->getRawOriginal()); + $this->assertSame(1, WorkflowServiceCall::query()->count()); + } + + #[DataProvider('handlerReplayPaths')] + public function testIssuedHandlerIdentityIsImmutableOnEveryReplayPath(string $kind, bool $explicit): void + { + $operation = $this->catalog($kind); + $control = new DefaultServiceControlPlane(); + $control->execute('billing', 'invoices', 'create', $this->executionOptions()); + $call = WorkflowServiceCall::query()->sole(); + $this->assertSame('accepted', $call->status); + $before = $call->getRawOriginal(); + $operation->forceFill([ + 'handler_target_reference' => 'replacement-target', + ])->save(); + $result = $control->execute('billing', 'invoices', 'create', array_replace( + $this->executionOptions(), + $explicit ? [ + 'service_call_id' => $call->id, + ] : [], + )); + $this->assertTrue($result['accepted']); + $this->assertSame($call->id, $result['service_call_id']); + $this->assertSame($before, $call->fresh()->getRawOriginal()); + $this->assertSame(1, WorkflowServiceCall::query()->count()); + } + + #[DataProvider('replayPaths')] + public function testHttpReplayChecksCurrentPolicyWithoutReadmittingTheCall(bool $explicit): void + { + $operation = $this->catalog('activity_execution'); + $control = new DefaultServiceControlPlane(); + $control->execute('billing', 'invoices', 'create', $this->executionOptions()); + $call = WorkflowServiceCall::query()->sole(); + $before = $call->getRawOriginal(); + $operation->forceFill([ + 'boundary_policy' => [ + 'deny_callers' => ['client'], + ], + ])->save(); + $this->expectException(ServiceOperationAuthorizationException::class); + try { + $control->execute('billing', 'invoices', 'create', array_replace( + $this->executionOptions(), + $explicit ? [ + 'service_call_id' => $call->id, + ] : [], + )); + } finally { + $this->assertSame($before, $call->fresh()->getRawOriginal()); + } + } + + public function testAnAdmittedRowWithoutAnIssuedHandlerCanStillDispatchExactlyOnce(): void + { + $this->catalog('activity_execution'); + $control = new DefaultServiceControlPlane(); + $admitted = $control->execute('billing', 'invoices', 'create', $this->executionOptions() + [ + 'dispatch_handler' => false, + ]); + $call = WorkflowServiceCall::query()->sole(); + $this->assertArrayNotHasKey('activity_execution_id', $call->metadata ?? []); + $options = $this->executionOptions() + [ + 'service_call_id' => $admitted['service_call_id'], + ]; + $result = $control->execute('billing', 'invoices', 'create', $options); + $this->assertTrue($result['accepted']); + $call->refresh(); + $this->assertNotEmpty($call->metadata['activity_execution_id']); + $issued = $call->getRawOriginal(); + $control->execute('billing', 'invoices', 'create', $options); + $this->assertSame($issued, $call->fresh()->getRawOriginal()); + } + + public static function replayPaths(): iterable + { + yield 'key' => [false]; + yield 'explicit call' => [true]; + } + + public static function handlerReplayPaths(): iterable + { + foreach (['activity_execution', 'invocable_carrier_request'] as $kind) { + foreach ([false, true] as $explicit) { + yield $kind . '-' . (int) $explicit => [$kind, $explicit]; + } + } + } + + private function executionOptions(): array + { + return [ + 'namespace' => 'billing', + 'caller_namespace' => 'client', + 'principal' => new ServiceCallPrincipal(subject: 'http-client', method: 'token', roles: ['worker']), + 'idempotency_key' => 'stable-key', + ]; + } + + private function catalog(string $kind): WorkflowServiceOperation + { + $endpoint = WorkflowServiceEndpoint::query()->create([ + 'namespace' => 'billing', + 'endpoint_name' => 'billing', + ]); + $service = WorkflowService::query()->create([ + 'namespace' => 'billing', + 'workflow_service_endpoint_id' => $endpoint->id, + 'service_name' => 'invoices', + ]); + return WorkflowServiceOperation::query()->create([ + 'namespace' => 'billing', + 'workflow_service_endpoint_id' => $endpoint->id, + 'workflow_service_id' => $service->id, + 'operation_name' => 'create', + 'operation_mode' => 'async', + 'handler_binding_kind' => $kind, + 'handler_target_reference' => 'original-target', + ]); + } +} From 9142c06975fdd1a2f9f5cd3214fd3e7594b37443 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:15:54 +0000 Subject: [PATCH 07/10] Normalize nested Avro results when closing linked service calls --- src/V2/Support/LinkedServiceCallCloser.php | 8 +++ tests/Unit/V2/LinkedServiceCallCloserTest.php | 67 +++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/src/V2/Support/LinkedServiceCallCloser.php b/src/V2/Support/LinkedServiceCallCloser.php index a1248aec..28983a40 100644 --- a/src/V2/Support/LinkedServiceCallCloser.php +++ b/src/V2/Support/LinkedServiceCallCloser.php @@ -5,6 +5,7 @@ namespace Workflow\V2\Support; use Throwable; +use Workflow\Serializers\CodecRegistry; use Workflow\V2\Enums\HistoryEventType; use Workflow\V2\Enums\ServiceCallBindingKind; use Workflow\V2\Enums\ServiceCallOutcome; @@ -166,6 +167,13 @@ public static function closeForTerminalEvent(WorkflowRun $run, WorkflowHistoryEv if ($status === ServiceCallStatus::Completed) { $codec = $payload['payload_codec'] ?? $update?->payload_codec ?? $run->output_payload_codec ?? $call->payload_codec; $output = $payload[$update !== null ? 'result' : 'output'] ?? null; + if (is_array($output) && array_key_exists('codec', $output)) { + $codec = $output['codec']; + if (array_key_exists('blob', $output)) { + $output = $output['blob']; + } + } + $codec = CodecRegistry::canonicalize($codec); if (is_string($output)) { $values['output_payload_reference'] = null; $metadata['nexus_result'] = [ diff --git a/tests/Unit/V2/LinkedServiceCallCloserTest.php b/tests/Unit/V2/LinkedServiceCallCloserTest.php index c6f88222..438d66cf 100644 --- a/tests/Unit/V2/LinkedServiceCallCloserTest.php +++ b/tests/Unit/V2/LinkedServiceCallCloserTest.php @@ -6,6 +6,7 @@ use Illuminate\Support\Carbon; use Illuminate\Support\Facades\Queue; +use Illuminate\Support\Str; use Mockery; use PHPUnit\Framework\Attributes\DataProvider; use RuntimeException; @@ -35,6 +36,7 @@ use Workflow\V2\Models\WorkflowServiceEndpoint; use Workflow\V2\Models\WorkflowServiceOperation; use Workflow\V2\Models\WorkflowTask; +use Workflow\V2\Models\WorkflowUpdate; use Workflow\V2\Support\DefaultServiceControlPlane; use Workflow\V2\Support\ExternalPayloads; use Workflow\V2\Support\ExternalPayloadStorage; @@ -1022,6 +1024,71 @@ public function testExternalResultKeepsIntegrityMetadataWithoutCopyingProviderUr ], $view['result_envelope']); } + #[DataProvider('nestedUpdateHistoryResults')] + public function testNestedAvroUpdateHistoryResultIsRetainedOnImmediateAndLateLinks(bool $external, bool $late): void + { + $run = $this->createRun(); + $blob = Serializer::serializeWithCodec('avro', [ + 'answer' => false, + 'source' => 'Ω', + ]); + $update = WorkflowUpdate::query()->create([ + 'workflow_command_id' => (string) Str::ulid(), + 'workflow_instance_id' => $run->workflow_instance_id, + 'workflow_run_id' => $run->id, + 'update_name' => 'answer', + 'status' => 'completed', + 'payload_codec' => 'avro', + ]); + $reference = [ + 'uri' => 'fixture://nested-result', + 'codec' => 'avro', + 'sha256' => hash('sha256', $blob), + 'size_bytes' => strlen($blob), + ]; + $envelope = [ + 'codec' => 'avro', + ] + ($external ? [ + 'external_storage' => $reference, + ] : [ + 'blob' => $blob, + ]); + $overrides = [ + 'resolved_binding_kind' => ServiceCallBindingKind::WorkflowUpdate->value, + 'linked_workflow_update_id' => $update->id, + 'output_payload_reference' => 'fixture://stale', + 'metadata' => [ + 'trace' => 'keep-me', + 'nexus_result' => [ + 'codec' => 'avro', + 'blob' => 'stale', + ], + ], + ]; + $call = $late ? null : $this->createCall($run, $overrides); + WorkflowHistoryEvent::record($run, HistoryEventType::UpdateCompleted, [ + 'update_id' => $update->id, + 'result' => $envelope, + ]); + $call ??= $this->createCall($run, $overrides); + LinkedServiceCallCloser::reconcile($call); + $call->refresh(); + $this->assertSame('completed', $call->status); + $this->assertSame('keep-me', $call->metadata['trace']); + $this->assertSame($external ? $reference['uri'] : null, $call->output_payload_reference); + $this->assertSameJsonObject($envelope, \Workflow\V2\Support\ServiceCallResultEnvelope::fromCall($call)); + $this->assertArrayNotHasKey($external ? 'blob' : 'external_reference', $call->metadata['nexus_result']); + } + + public static function nestedUpdateHistoryResults(): iterable + { + foreach ([false, true] as $external) { + foreach ([false, true] as $late) { + yield (int) $external . '-' . (int) $late => [$external, $late]; + } + } + } + #[DataProvider('externalCallerNamespaces')] public function testExternalTerminalDeliveryBelongsToTheCallerNamespace(string $namespace): void { From 93899982228237ad8fea25c18cabeda094ba48b4 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Tue, 27 Oct 2026 02:29:21 +0000 Subject: [PATCH 08/10] Prepare and finalize a canonical service admission within one writer transaction --- .../workflow-service-calls-architecture.md | 9 ++ .../Support/ServiceBoundaryAuditRecorder.php | 61 +++++++++- .../V2/ServiceBoundaryAuditRecorderTest.php | 114 ++++++++++++++++++ 3 files changed, 181 insertions(+), 3 deletions(-) diff --git a/docs/architecture/workflow-service-calls-architecture.md b/docs/architecture/workflow-service-calls-architecture.md index ff0d17ac..cbc8bc25 100644 --- a/docs/architecture/workflow-service-calls-architecture.md +++ b/docs/architecture/workflow-service-calls-architecture.md @@ -413,6 +413,15 @@ or Waterline surface inspecting the call row can render the snapped deadline, cancellation, retry, and idempotency policy without consulting the live operation registry. +Hosts that claim the canonical idempotency key during admission may use +`ServiceBoundaryAuditRecorder::make()` to prepare an unsaved row. Claim the key +inside the writer transaction, then finalize that same newly inserted pending +row with `record($request, $decision, $call)`. Finalization refuses a committed, +reloaded, already admitted, or differently scoped row. It keeps the canonical +identity and clears provisional decision timestamps before recording the final +allow or denial. Authorize an existing canonical winner as a replay before +reserving fresh rate or concurrency capacity; do not overwrite or re-admit it. + For the built-in `propagate_to_linked_workflow` cancellation option, propagation is best-effort and its public receipt is retained in `metadata.linked_cancel`. A propagation exception records `workflow_cancel_exception` and its exception diff --git a/src/V2/Support/ServiceBoundaryAuditRecorder.php b/src/V2/Support/ServiceBoundaryAuditRecorder.php index 859f8595..5fd592f8 100644 --- a/src/V2/Support/ServiceBoundaryAuditRecorder.php +++ b/src/V2/Support/ServiceBoundaryAuditRecorder.php @@ -5,6 +5,7 @@ namespace Workflow\V2\Support; use Illuminate\Database\Eloquent\Model; +use LogicException; use Workflow\V2\Enums\ServiceCallStatus; use Workflow\V2\Models\WorkflowService; use Workflow\V2\Models\WorkflowServiceCall; @@ -21,12 +22,64 @@ final class ServiceBoundaryAuditRecorder public function record( ServiceBoundaryRequest $request, ServiceBoundaryDecision $decision, + ?WorkflowServiceCall $call = null, + ): WorkflowServiceCall { + if ($call !== null && $call->exists + && (! $call->wasRecentlyCreated || $call->getConnection()->transactionLevel() === 0 + || $call->status !== ServiceCallStatus::Pending->value)) { + throw new LogicException('Only a newly inserted, uncommitted pending admission can be finalized.'); + } + if ($call !== null && $call->exists) { + foreach ([ + 'namespace' => $request->targetNamespace, + 'target_namespace' => $request->targetNamespace, + 'endpoint_name' => $request->endpointName, + 'service_name' => $request->serviceName, + 'operation_name' => $request->operationName, + 'idempotency_key' => $request->idempotencyKey, + 'caller_namespace' => $request->callerNamespace, + 'caller_workflow_instance_id' => $request->callerWorkflowInstanceId, + 'caller_workflow_run_id' => $request->callerWorkflowRunId, + 'caller_principal_subject' => $request->principal->subject, + 'caller_principal_method' => $request->principal->method, + 'caller_principal_tenant' => $request->principal->tenant, + ] as $field => $expected) { + if ($call->getAttribute($field) !== $expected) { + throw new LogicException('The prepared admission does not belong to this request.'); + } + } + if ($call->idempotency_scope_hash !== null && ($request->idempotencyKey === null + || $call->idempotency_scope_hash !== ServiceCallIdempotency::digest([ + $request->targetNamespace, $request->endpointName, $request->serviceName, + $request->operationName, $request->idempotencyKey, + ]))) { + throw new LogicException('The prepared admission has a different canonical key.'); + } + } + $call = $this->populate($request, $decision, $call); + $call->save(); + + return $call; + } + + /** + * Build an unsaved admission for a host's canonical-key transaction. + */ + public function make(ServiceBoundaryRequest $request, ServiceBoundaryDecision $decision): WorkflowServiceCall + { + return $this->populate($request, $decision); + } + + private function populate( + ServiceBoundaryRequest $request, + ServiceBoundaryDecision $decision, + ?WorkflowServiceCall $call = null, ): WorkflowServiceCall { /** @var class-string $modelClass */ $modelClass = ConfiguredV2Models::resolve('service_call_model', WorkflowServiceCall::class); /** @var WorkflowServiceCall $call */ - $call = new $modelClass(); + $call ??= new $modelClass(); $endpointId = $this->endpointId($request); $serviceId = $endpointId !== null ? $this->serviceId($request, $endpointId) : null; @@ -78,6 +131,10 @@ public function record( : $request->principal->claims; $now = now(); + $call->accepted_at = null; + $call->call_deadline_at = null; + $call->failed_at = null; + $call->failure_message = null; if ($decision->isAllowed()) { $call->accepted_at = $now; $call->call_deadline_at = ServiceCallDeadlinePolicy::resolve($request->deadlinePolicy, $now); @@ -86,8 +143,6 @@ public function record( $call->failure_message = $decision->message; } - $call->save(); - return $call; } diff --git a/tests/Unit/V2/ServiceBoundaryAuditRecorderTest.php b/tests/Unit/V2/ServiceBoundaryAuditRecorderTest.php index 88ac9f78..14b14627 100644 --- a/tests/Unit/V2/ServiceBoundaryAuditRecorderTest.php +++ b/tests/Unit/V2/ServiceBoundaryAuditRecorderTest.php @@ -5,7 +5,10 @@ namespace Tests\Unit\V2; use Illuminate\Support\Carbon; +use Illuminate\Support\Facades\DB; use InvalidArgumentException; +use LogicException; +use PHPUnit\Framework\Attributes\DataProvider; use Tests\TestCase; use Workflow\V2\Enums\ServiceCallBindingKind; use Workflow\V2\Enums\ServiceCallOperationMode; @@ -17,11 +20,122 @@ use Workflow\V2\Support\ServiceBoundaryAuditRecorder; use Workflow\V2\Support\ServiceBoundaryDecision; use Workflow\V2\Support\ServiceBoundaryRequest; +use Workflow\V2\Support\ServiceCallIdempotency; use Workflow\V2\Support\ServiceCallPrincipal; use Workflow\V2\TaskWatchdog; final class ServiceBoundaryAuditRecorderTest extends TestCase { + public function testPreparedAdmissionDoesNotPersistBeforeItsHostClaimsTheCanonicalKey(): void + { + [, , $operation] = $this->seedCatalog(); + $call = (new ServiceBoundaryAuditRecorder())->make( + $this->request($operation), + ServiceBoundaryDecision::allow() + ); + $this->assertFalse($call->exists); + $this->assertSame(0, WorkflowServiceCall::query()->count()); + $this->assertSame($operation->id, $call->workflow_service_operation_id); + $this->assertSame('analytics', $call->caller_namespace); + } + + #[DataProvider('finalAdmissionDecisions')] + public function testTheCanonicalPendingRowIsFinalizedWithoutAnotherAuditIdentity(bool $allowed): void + { + [, , $operation] = $this->seedCatalog(); + $request = $this->request($operation); + $recorder = new ServiceBoundaryAuditRecorder(); + $deny = ServiceBoundaryDecision::denyNamespacePolicy(reason: 'caller_namespace_denied', message: 'Denied.'); + DB::transaction(function () use ($request, $recorder, $deny, $allowed): void { + $call = $recorder->make($request, $allowed ? $deny : ServiceBoundaryDecision::allow()); + $call->status = 'pending'; + $call = ServiceCallIdempotency::save($call, ['finance', 'billing', 'invoicing', 'create', 'invoice-1']); + $id = $call->id; + $digest = $call->idempotency_scope_hash; + $result = $recorder->record($request, $allowed ? ServiceBoundaryDecision::allow() : $deny, $call); + $this->assertSame($id, $result->id); + $this->assertSame($digest, $result->idempotency_scope_hash); + $this->assertSame('invoice-1', $result->idempotency_key); + $this->assertSame($allowed ? 'accepted' : 'failed', $result->status); + $this->assertSame($allowed, $result->accepted_at !== null); + $this->assertSame(! $allowed, $result->failed_at !== null); + $this->assertSame($allowed ? null : 'Denied.', $result->failure_message); + $this->assertSame(1, WorkflowServiceCall::query()->count()); + }); + $this->assertSame(1, WorkflowServiceCall::query()->count()); + } + + public static function finalAdmissionDecisions(): iterable + { + yield 'allow' => [true]; + yield 'deny' => [false]; + } + + public function testACommittedAdmissionCannotBeRewrittenThroughThePreparedRowApi(): void + { + [, , $operation] = $this->seedCatalog(); + $request = $this->request($operation); + $recorder = new ServiceBoundaryAuditRecorder(); + $call = $recorder->record($request, ServiceBoundaryDecision::allow())->refresh(); + $before = $call->getRawOriginal(); + $this->expectException(LogicException::class); + try { + $recorder->record($request, ServiceBoundaryDecision::denyAuthorization(reason: 'denied'), $call); + } finally { + $this->assertSame($before, $call->fresh()->getRawOriginal()); + } + } + + public function testAnotherCallerCannotFinalizeThePreparedCanonicalRow(): void + { + [, , $operation] = $this->seedCatalog(); + $recorder = new ServiceBoundaryAuditRecorder(); + $this->expectException(LogicException::class); + try { + DB::transaction(function () use ($recorder, $operation): void { + $call = $recorder->make($this->request($operation), ServiceBoundaryDecision::allow()); + $call->status = 'pending'; + $call = ServiceCallIdempotency::save($call, ['finance', 'billing', 'invoicing', 'create', 'invoice-1']); + $recorder->record( + $this->request($operation, callerNamespace: 'foreign'), + ServiceBoundaryDecision::allow(), + $call + ); + }); + } finally { + $this->assertSame(0, WorkflowServiceCall::query()->count()); + } + } + + #[DataProvider('committedPendingRows')] + public function testACommittedPendingRowCannotBeAdoptedAsANewAdmission(bool $reload): void + { + [, , $operation] = $this->seedCatalog(); + $recorder = new ServiceBoundaryAuditRecorder(); + $request = $this->request($operation); + $call = $recorder->make($request, ServiceBoundaryDecision::allow()); + $call->status = 'pending'; + $call->save(); + $call->refresh(); + $before = $call->getRawOriginal(); + $this->expectException(LogicException::class); + try { + if ($reload) { + DB::transaction(static fn () => $recorder->record($request, ServiceBoundaryDecision::allow(), $call->fresh())); + } else { + $recorder->record($request, ServiceBoundaryDecision::allow(), $call); + } + } finally { + $this->assertSame($before, $call->fresh()->getRawOriginal()); + } + } + + public static function committedPendingRows(): iterable + { + yield 'retained model after commit' => [false]; + yield 'reloaded in a later transaction' => [true]; + } + public function testAdmissionPersistsTheOriginalDeadlineBeforeHandlerDispatchAndRepairExpiresIt(): void { Carbon::setTestNow('2026-10-11T00:00:00.123456Z'); From 2a0e9466428d0bd8b9eab1d2609d798d7412d952 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Tue, 27 Oct 2026 03:46:16 +0000 Subject: [PATCH 09/10] Format the prepared-admission transaction assertion --- tests/Unit/V2/ServiceBoundaryAuditRecorderTest.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/Unit/V2/ServiceBoundaryAuditRecorderTest.php b/tests/Unit/V2/ServiceBoundaryAuditRecorderTest.php index 14b14627..108fd52c 100644 --- a/tests/Unit/V2/ServiceBoundaryAuditRecorderTest.php +++ b/tests/Unit/V2/ServiceBoundaryAuditRecorderTest.php @@ -121,7 +121,9 @@ public function testACommittedPendingRowCannotBeAdoptedAsANewAdmission(bool $rel $this->expectException(LogicException::class); try { if ($reload) { - DB::transaction(static fn () => $recorder->record($request, ServiceBoundaryDecision::allow(), $call->fresh())); + DB::transaction( + static fn () => $recorder->record($request, ServiceBoundaryDecision::allow(), $call->fresh()) + ); } else { $recorder->record($request, ServiceBoundaryDecision::allow(), $call); } From 1c3f95e5ad6defd0b3cfed9c0416fd7f896eac79 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Tue, 27 Oct 2026 04:37:04 +0000 Subject: [PATCH 10/10] Preserve typed replay-policy refusals after ownership checks --- .../workflow-service-calls-architecture.md | 3 + ...ServiceOperationAuthorizationException.php | 7 ++- .../WorkflowServiceOperationAuthority.php | 1 + .../WorkflowServiceOperationAuthorityTest.php | 62 +++++++++++++++++++ 4 files changed, 71 insertions(+), 2 deletions(-) diff --git a/docs/architecture/workflow-service-calls-architecture.md b/docs/architecture/workflow-service-calls-architecture.md index cbc8bc25..28a51ff0 100644 --- a/docs/architecture/workflow-service-calls-architecture.md +++ b/docs/architecture/workflow-service-calls-architecture.md @@ -421,6 +421,9 @@ reloaded, already admitted, or differently scoped row. It keeps the canonical identity and clears provisional decision timestamps before recording the final allow or denial. Authorize an existing canonical winner as a replay before reserving fresh rate or concurrency capacity; do not overwrite or re-admit it. +An existing-call authorization exception carries `boundaryDecision` only when +the owned, scoped catalog's current replay policy returned a denial. Hosts may +preserve that typed rejection; ownership/catalog failures have no decision. For the built-in `propagate_to_linked_workflow` cancellation option, propagation is best-effort and its public receipt is retained in `metadata.linked_cancel`. diff --git a/src/V2/Exceptions/ServiceOperationAuthorizationException.php b/src/V2/Exceptions/ServiceOperationAuthorizationException.php index 63b6c846..b2bd51ed 100644 --- a/src/V2/Exceptions/ServiceOperationAuthorizationException.php +++ b/src/V2/Exceptions/ServiceOperationAuthorizationException.php @@ -5,11 +5,14 @@ namespace Workflow\V2\Exceptions; use RuntimeException; +use Workflow\V2\Support\ServiceBoundaryDecision; final class ServiceOperationAuthorizationException extends RuntimeException { - public function __construct(string $reason = 'service_operation_authority_invalid') - { + public function __construct( + string $reason = 'service_operation_authority_invalid', + public readonly ?ServiceBoundaryDecision $boundaryDecision = null, + ) { parent::__construct($reason); } } diff --git a/src/V2/Support/WorkflowServiceOperationAuthority.php b/src/V2/Support/WorkflowServiceOperationAuthority.php index b1b0a03c..72827c37 100644 --- a/src/V2/Support/WorkflowServiceOperationAuthority.php +++ b/src/V2/Support/WorkflowServiceOperationAuthority.php @@ -217,6 +217,7 @@ private static function checkCatalog( $decision instanceof ServiceBoundaryDecision ? ($decision->reason ?? 'service_operation_authority_invalid') : 'service_operation_replay_authority_unavailable', + boundaryDecision: $decision instanceof ServiceBoundaryDecision ? $decision : null, ); } } diff --git a/tests/Unit/V2/WorkflowServiceOperationAuthorityTest.php b/tests/Unit/V2/WorkflowServiceOperationAuthorityTest.php index 4542e789..85903386 100644 --- a/tests/Unit/V2/WorkflowServiceOperationAuthorityTest.php +++ b/tests/Unit/V2/WorkflowServiceOperationAuthorityTest.php @@ -570,6 +570,68 @@ public function authorizeReplay(ServiceBoundaryRequest $request): ServiceBoundar $this->assertSame($before, $this->records()); } + #[DataProvider('typedReplayRefusals')] + public function testReplayPolicyRefusalRetainsItsTypedDecisionAfterOwnershipChecks(string $kind): void + { + $call = $this->serviceCall('completed'); + $decision = $kind === 'circuit' + ? ServiceBoundaryDecision::denyCircuitOpen(retryAfterSeconds: 17) + : ServiceBoundaryDecision::denyAuthorization('fixture_current_policy_denied'); + $policy = new class($decision) implements ServiceBoundaryPolicy { + public function __construct( + private readonly ServiceBoundaryDecision $decision + ) { + } + + public function evaluate(ServiceBoundaryRequest $request): ServiceBoundaryDecision + { + throw new RuntimeException('Replay must not reserve quota.'); + } + + public function authorizeReplay(ServiceBoundaryRequest $request): ServiceBoundaryDecision + { + return $this->decision; + } + }; + $scope = [ + 'namespace' => 'caller', + 'caller_namespace' => 'caller', + 'caller_workflow_instance_id' => $this->run->workflow_instance_id, + 'caller_workflow_run_id' => $this->run->id, + 'endpoint_name' => 'billing', + 'service_name' => 'invoices', + 'operation_name' => 'create', + ]; + if ($kind === 'foreign') { + $scope['caller_workflow_run_id'] = 'foreign-run'; + } + $before = $this->records(); + try { + WorkflowServiceOperationAuthority::authorizeExistingCall( + $call, + new ServiceCallPrincipal('fixture', 'token'), + $scope, + $policy + ); + $this->fail('A denied replay must still throw an authorization exception.'); + } catch (ServiceOperationAuthorizationException $exception) { + $this->assertTrue(property_exists($exception, 'boundaryDecision')); + $this->assertSame($kind === 'foreign' ? null : $decision, $exception->boundaryDecision); + $this->assertSame( + $kind === 'foreign' ? 'service_call_not_owned' : $decision->reason, + $exception->getMessage() + ); + } + $this->assertSame($before, $this->records()); + } + + public static function typedReplayRefusals(): iterable + { + yield 'circuit policy' => ['circuit']; + yield 'authorization policy' => ['authorization']; + yield 'foreign caller never receives policy decision' => ['foreign']; + } + #[DataProvider('concurrentInsertions')] public function testConcurrentKeyInsertionIsReauthorizedAtTheEffectingLookup( string $pauseAt,