Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,17 @@
},
"path": "./src/step/step_with_retry.py"
},
{
"name": "Step with Pause Every",
"description": "Experimental: pause the execution after every third failed attempt of a step, until an operator resumes it",
"handler": "step_with_pause_every.handler",
"integration": false,
"durableConfig": {
"RetentionPeriodInDays": 7,
"ExecutionTimeout": 300
},
"path": "./src/step/step_with_pause_every.py"
},
{
"name": "Custom SerDes Round-Trip",
"description": "One function proving the first-run/replay result-equality guarantee with a non-identity SerDes across step, wait_for_condition, and run_in_child_context (normal, virtual, large-payload)",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
"""Pause the execution after every third failed attempt of a step.

Experimental: RetryStrategyConfig.pause_every may change or be removed in a
future release.

The function's execution role needs lambda:PauseDurableExecution on its own
durable executions. A function that uses a customer managed AWS KMS key also
needs kms:Decrypt on that key. Without these permissions, the step logs a
warning and retries as usual.
An operator resumes a paused execution with ResumeDurableExecution.
"""

from typing import Any

from aws_durable_execution_sdk_python.config import StepConfig
from aws_durable_execution_sdk_python.context import (
DurableContext,
StepContext,
durable_step,
)
from aws_durable_execution_sdk_python.execution import durable_execution
from aws_durable_execution_sdk_python.retries import (
RetryStrategyConfig,
create_retry_strategy,
)


@durable_step
def call_partner_api(step_context: StepContext) -> str:
# Fail on the first four attempts and succeed on attempt 5.
if step_context.attempt < 5:
msg = f"Partner API unavailable on attempt {step_context.attempt}"
raise RuntimeError(msg)
return "Partner API answered"


@durable_execution
def handler(_event: Any, context: DurableContext) -> str:
config = StepConfig(
retry_strategy=create_retry_strategy(
RetryStrategyConfig(
max_attempts=10,
retryable_error_types=[RuntimeError],
# Pause after attempts 3, 6, 9. An operator can inspect the
# error and resume the execution when the partner API is back.
pause_every=3,
)
),
)

result: str = context.step(call_partner_api(), config=config)
return result
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
"""Tests for the step_with_pause_every example.

The local runner pauses the execution when the step asks it to. The test
resumes it, like an operator would with ResumeDurableExecution.
"""

import threading
import time

from aws_durable_execution_sdk_python_testing.runner import (
DurableFunctionTestRunner,
)

from aws_durable_execution_sdk_python.execution import InvocationStatus
from src.step import step_with_pause_every
from test.conftest import deserialize_operation_payload


def _count_events(runner: DurableFunctionTestRunner, arn: str, event_type: str) -> int:
return sum(
1
for event in runner.get_execution_history(arn).events
if event.event_type == event_type
)


def _wait_for_failed_attempts(
runner: DurableFunctionTestRunner, arn: str, count: int
) -> None:
deadline = time.monotonic() + 10
while time.monotonic() < deadline:
if _count_events(runner, arn, "StepFailed") >= count:
return
threading.Event().wait(0.01)
assert _count_events(runner, arn, "StepFailed") >= count


def test_step_with_pause_every_pauses_after_third_attempt():
"""The execution pauses after attempt 3 and finishes after resume."""
with DurableFunctionTestRunner(
handler=step_with_pause_every.handler, execution_timeout=30
) as runner:
arn = runner.run_async(input="{}")

_wait_for_failed_attempts(runner, arn, 3)
threading.Event().wait(0.3)
# Paused: attempt 4 does not start until the execution is resumed.
assert _count_events(runner, arn, "StepFailed") == 3
assert _count_events(runner, arn, "ExecutionSucceeded") == 0

runner.resume_execution(arn)
result = runner.wait_for_result(arn)

assert result.status is InvocationStatus.SUCCEEDED
assert deserialize_operation_payload(result.result) == "Partner API answered"
assert result.get_step("call_partner_api").attempt == 5
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,22 @@ def get_execution_state(
next_marker=None,
)

def pause_execution(self, durable_execution_arn: str) -> None:
"""Experimental; may change or be removed in a future release.

Pause an execution without waiting for its invocation to stop.

The running invocation's next checkpoint is answered without a
token, like Lambda does after PauseDurableExecution. Completed
executions are unchanged. An already paused execution stays as it is,
because Execution.pause() does nothing then.
"""
execution: Execution = self._store.load(durable_execution_arn)
if execution.is_complete:
return
execution.pause()
self._store.update(execution)


def _maybe_replay_cached(
execution: Execution,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
"""An in-memory service client, that can replace the boto lambda service client."""

from __future__ import annotations

import datetime
from typing import TYPE_CHECKING

from aws_durable_execution_sdk_python.lambda_service import (
CheckpointOutput,
Expand All @@ -14,12 +17,21 @@
CheckpointProcessor,
)
from aws_durable_execution_sdk_python_testing.worker.checkpoint_tasks import (
CallableTask,
CheckpointTask,
GetStateTask,
)
from aws_durable_execution_sdk_python_testing.worker.registry import ExecutionRegistry


if TYPE_CHECKING:
# Imported for typing only, so the client still loads with a core SDK
# that predates pause support.
from aws_durable_execution_sdk_python.lambda_service import (
PauseExecutionRequest,
)


class InMemoryServiceClient(DurableServiceClient):
"""An in-memory service client, that can replace the boto lambda service client."""

Expand Down Expand Up @@ -55,6 +67,25 @@ def get_execution_state(
)
return self._registry.submit(durable_execution_arn, task).result()

def pause_durable_execution(
self,
durable_execution_arn: str,
request: PauseExecutionRequest, # noqa: ARG002
) -> None:
"""Experimental; may change or be removed in a future release.

Pause the execution, like the Lambda PauseDurableExecution API.

Returns without waiting for the running invocation to stop. Its next
checkpoint is answered without a token. Resume the execution with
DurableFunctionTestRunner.resume_execution(). The client token and
the reason are not recorded.
"""
task: CallableTask[None] = CallableTask(
lambda: self._checkpoint_processor.pause_execution(durable_execution_arn)
)
self._registry.submit(durable_execution_arn, task).result()

def stop(self, execution_arn: str, payload: bytes | None) -> datetime.datetime: # noqa: ARG002
# TODO: implement
# Return current time for in-memory testing
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -536,3 +536,33 @@ def test_paused_checkpoint_retry_is_rejected_even_after_resume(
):
processor.process_checkpoint(token, [], retry_client_token)
assert store.load(arn).to_json_dict() == before_retry


def test_pause_execution_answers_next_checkpoint_without_token() -> None:
processor, store, execution, token = _make_processor_with_started_execution()

processor.pause_execution(execution.durable_execution_arn)

assert store.load(execution.durable_execution_arn).is_paused
assert processor.process_checkpoint(token, [], "c1").checkpoint_token is None


def test_pause_execution_leaves_existing_pause_unchanged() -> None:
processor, store, execution, _token = _make_processor_with_started_execution()
execution.pause()
execution.defer_invocation()
store.save(execution)

processor.pause_execution(execution.durable_execution_arn)

assert store.load(execution.durable_execution_arn).has_deferred_invocation


def test_pause_execution_leaves_completed_execution_unchanged() -> None:
processor, store, execution, _token = _make_processor_with_started_execution()
execution.complete_success(result=None)
store.save(execution)

processor.pause_execution(execution.durable_execution_arn)

assert not store.load(execution.durable_execution_arn).is_paused
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
OperationAction,
OperationType,
OperationUpdate,
PausableDurableServiceClient,
PauseExecutionRequest,
StateOutput,
)

Expand Down Expand Up @@ -107,3 +109,19 @@ def test_stop_with_none_payload():
result = client.stop("arn:aws:states:us-east-1:123456789012:execution:test", None)

assert isinstance(result, datetime.datetime)


def test_pause_durable_execution_delegates_to_processor():
"""pause_durable_execution pauses through the processor on the lane."""
processor = Mock()
client = _client(processor)
request = PauseExecutionRequest(client_token="token", reason="why") # noqa: S106

client.pause_durable_execution("arn", request)

processor.pause_execution.assert_called_once_with("arn")


def test_in_memory_client_supports_pause():
"""The SDK finds the pause method on the in-memory client."""
assert isinstance(_client(Mock()), PausableDurableServiceClient)
Loading
Loading