Skip to content

Commit 264fc4e

Browse files
committed
feat(sdk): add distributed map operation
Add distributed map, a durable operation that processes a large set of items in separate processor Lambda functions run by the service, and returns the run's outcome to the parent durable function. - DurableContext.distributedMap starts a run and waits for it to finish. It returns a summary of the counts and status, or a result that also holds each item's output or error. The run is checkpointed, so it is started once and its outcome is reused on replay. - Items come from an inline list, S3 objects (JSON, JSON Lines or CSV), or a custom reader function. Configuration covers the processor, batch size, concurrency, failure tolerance and S3 result destinations, and is validated before the start checkpoint. - DistributedMapHandlers wraps a customer's function as a processor or reader Lambda handler, with durable and non-durable variants. The wrappers handle the service's event and response formats, and pass the run ARN and item ids to the customer's function.
1 parent 29dc23a commit 264fc4e

45 files changed

Lines changed: 6298 additions & 5 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎sdk/pom.xml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,13 @@
6969
<version>${slf4j.version}</version>
7070
<scope>test</scope>
7171
</dependency>
72+
<!-- The Lambda Java runtime's event serializer, the version aws-lambda-java-runtime-interface-client 2.12.2 uses -->
73+
<dependency>
74+
<groupId>com.amazonaws</groupId>
75+
<artifactId>aws-lambda-java-serialization</artifactId>
76+
<version>1.4.1</version>
77+
<scope>test</scope>
78+
</dependency>
7279
</dependencies>
7380

7481
<build>

‎sdk/src/main/java/software/amazon/lambda/durable/DurableContext.java‎

Lines changed: 211 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@
99
import java.util.function.Function;
1010
import java.util.function.Supplier;
1111
import software.amazon.lambda.durable.config.CallbackConfig;
12+
import software.amazon.lambda.durable.config.DistributedMapConfig;
13+
import software.amazon.lambda.durable.config.DistributedMapProcessor;
14+
import software.amazon.lambda.durable.config.DistributedMapSource;
1215
import software.amazon.lambda.durable.config.InvokeConfig;
1316
import software.amazon.lambda.durable.config.MapConfig;
1417
import software.amazon.lambda.durable.config.ParallelConfig;
@@ -19,6 +22,8 @@
1922
import software.amazon.lambda.durable.config.WithRetryConfig;
2023
import software.amazon.lambda.durable.context.BaseContext;
2124
import software.amazon.lambda.durable.exception.ChildContextFailedException;
25+
import software.amazon.lambda.durable.model.DistributedMapResult;
26+
import software.amazon.lambda.durable.model.DistributedMapSummary;
2227
import software.amazon.lambda.durable.model.MapResult;
2328
import software.amazon.lambda.durable.model.WaitForConditionResult;
2429

@@ -544,6 +549,212 @@ default ParallelDurableFuture parallel(String name) {
544549
*/
545550
ParallelDurableFuture parallel(String name, ParallelConfig config);
546551

552+
// =============== distributedMap ================
553+
554+
/**
555+
* Runs a distributed map with default config, blocking until the run reaches a terminal state.
556+
*
557+
* @param maxConcurrency the maximum number of items processed concurrently. Required, with no service default.
558+
*/
559+
default <I> DistributedMapSummary distributedMap(
560+
String name, DistributedMapSource<I> source, DistributedMapProcessor processor, int maxConcurrency) {
561+
return distributedMapAsync(
562+
name,
563+
source,
564+
processor,
565+
maxConcurrency,
566+
DistributedMapConfig.builder().build())
567+
.get();
568+
}
569+
570+
/**
571+
* Runs a distributed map, blocking until the run reaches a terminal state.
572+
*
573+
* @param maxConcurrency the maximum number of items processed concurrently. Required, with no service default.
574+
*/
575+
default <I> DistributedMapSummary distributedMap(
576+
String name,
577+
DistributedMapSource<I> source,
578+
DistributedMapProcessor processor,
579+
int maxConcurrency,
580+
DistributedMapConfig config) {
581+
return distributedMapAsync(name, source, processor, maxConcurrency, config)
582+
.get();
583+
}
584+
585+
/**
586+
* Runs a distributed map that collects per-item results with a TypeToken, blocking until the run reaches a terminal
587+
* state.
588+
*
589+
* @param maxConcurrency the maximum number of items processed concurrently. Required, with no service default.
590+
*/
591+
default <I, O> DistributedMapResult<O> distributedMap(
592+
String name,
593+
DistributedMapSource<I> source,
594+
DistributedMapProcessor processor,
595+
int maxConcurrency,
596+
TypeToken<O> resultType,
597+
DistributedMapConfig config) {
598+
return distributedMapAsync(name, source, processor, maxConcurrency, resultType, config)
599+
.get();
600+
}
601+
602+
/**
603+
* Runs a distributed map that collects per-item results with a result Class, blocking until the run reaches a
604+
* terminal state.
605+
*
606+
* @param maxConcurrency the maximum number of items processed concurrently. Required, with no service default.
607+
*/
608+
default <I, O> DistributedMapResult<O> distributedMap(
609+
String name,
610+
DistributedMapSource<I> source,
611+
DistributedMapProcessor processor,
612+
int maxConcurrency,
613+
Class<O> resultType,
614+
DistributedMapConfig config) {
615+
return distributedMapAsync(name, source, processor, maxConcurrency, TypeToken.get(resultType), config)
616+
.get();
617+
}
618+
619+
/**
620+
* Runs a distributed map that collects per-item results with a result Class and default config, blocking until the
621+
* run reaches a terminal state.
622+
*
623+
* @param maxConcurrency the maximum number of items processed concurrently. Required, with no service default.
624+
*/
625+
default <I, O> DistributedMapResult<O> distributedMap(
626+
String name,
627+
DistributedMapSource<I> source,
628+
DistributedMapProcessor processor,
629+
int maxConcurrency,
630+
Class<O> resultType) {
631+
return distributedMapAsync(
632+
name,
633+
source,
634+
processor,
635+
maxConcurrency,
636+
TypeToken.get(resultType),
637+
DistributedMapConfig.builder().build())
638+
.get();
639+
}
640+
641+
/**
642+
* Runs a distributed map that collects per-item results with a TypeToken and default config, blocking until the run
643+
* reaches a terminal state.
644+
*
645+
* @param maxConcurrency the maximum number of items processed concurrently. Required, with no service default.
646+
*/
647+
default <I, O> DistributedMapResult<O> distributedMap(
648+
String name,
649+
DistributedMapSource<I> source,
650+
DistributedMapProcessor processor,
651+
int maxConcurrency,
652+
TypeToken<O> resultType) {
653+
return distributedMapAsync(
654+
name,
655+
source,
656+
processor,
657+
maxConcurrency,
658+
resultType,
659+
DistributedMapConfig.builder().build())
660+
.get();
661+
}
662+
663+
/**
664+
* Asynchronously runs a distributed map with default config.
665+
*
666+
* @param maxConcurrency the maximum number of items processed concurrently. Required, with no service default.
667+
*/
668+
default <I> DurableFuture<DistributedMapSummary> distributedMapAsync(
669+
String name, DistributedMapSource<I> source, DistributedMapProcessor processor, int maxConcurrency) {
670+
return distributedMapAsync(
671+
name,
672+
source,
673+
processor,
674+
maxConcurrency,
675+
DistributedMapConfig.builder().build());
676+
}
677+
678+
/**
679+
* Asynchronously runs a distributed map.
680+
*
681+
* @param maxConcurrency the maximum number of items processed concurrently. Required, with no service default.
682+
*/
683+
<I> DurableFuture<DistributedMapSummary> distributedMapAsync(
684+
String name,
685+
DistributedMapSource<I> source,
686+
DistributedMapProcessor processor,
687+
int maxConcurrency,
688+
DistributedMapConfig config);
689+
690+
/**
691+
* Asynchronously runs a distributed map that collects per-item results with a TypeToken.
692+
*
693+
* @param maxConcurrency the maximum number of items processed concurrently. Required, with no service default.
694+
*/
695+
<I, O> DurableFuture<DistributedMapResult<O>> distributedMapAsync(
696+
String name,
697+
DistributedMapSource<I> source,
698+
DistributedMapProcessor processor,
699+
int maxConcurrency,
700+
TypeToken<O> resultType,
701+
DistributedMapConfig config);
702+
703+
/**
704+
* Asynchronously runs a distributed map that collects per-item results with a result Class.
705+
*
706+
* @param maxConcurrency the maximum number of items processed concurrently. Required, with no service default.
707+
*/
708+
default <I, O> DurableFuture<DistributedMapResult<O>> distributedMapAsync(
709+
String name,
710+
DistributedMapSource<I> source,
711+
DistributedMapProcessor processor,
712+
int maxConcurrency,
713+
Class<O> resultType,
714+
DistributedMapConfig config) {
715+
return distributedMapAsync(name, source, processor, maxConcurrency, TypeToken.get(resultType), config);
716+
}
717+
718+
/**
719+
* Asynchronously runs a distributed map that collects per-item results with a result Class and default config.
720+
*
721+
* @param maxConcurrency the maximum number of items processed concurrently. Required, with no service default.
722+
*/
723+
default <I, O> DurableFuture<DistributedMapResult<O>> distributedMapAsync(
724+
String name,
725+
DistributedMapSource<I> source,
726+
DistributedMapProcessor processor,
727+
int maxConcurrency,
728+
Class<O> resultType) {
729+
return distributedMapAsync(
730+
name,
731+
source,
732+
processor,
733+
maxConcurrency,
734+
TypeToken.get(resultType),
735+
DistributedMapConfig.builder().build());
736+
}
737+
738+
/**
739+
* Asynchronously runs a distributed map that collects per-item results with a TypeToken and default config.
740+
*
741+
* @param maxConcurrency the maximum number of items processed concurrently. Required, with no service default.
742+
*/
743+
default <I, O> DurableFuture<DistributedMapResult<O>> distributedMapAsync(
744+
String name,
745+
DistributedMapSource<I> source,
746+
DistributedMapProcessor processor,
747+
int maxConcurrency,
748+
TypeToken<O> resultType) {
749+
return distributedMapAsync(
750+
name,
751+
source,
752+
processor,
753+
maxConcurrency,
754+
resultType,
755+
DistributedMapConfig.builder().build());
756+
}
757+
547758
/**
548759
* Executes a submitter function and waits for an external callback, blocking until the callback completes.
549760
*
Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
package software.amazon.lambda.durable.config;
4+
5+
import java.time.Duration;
6+
import software.amazon.lambda.durable.serde.SerDes;
7+
8+
/** Optional configuration for a distributed map run. */
9+
public class DistributedMapConfig {
10+
private static final long MAX_TIMEOUT_SECONDS = 7776000; // 90 days
11+
12+
private final DistributedMapDestination destination;
13+
private final CompletionConfig completionConfig;
14+
private final Duration timeout;
15+
private final SerDes resultSerDes;
16+
17+
private DistributedMapConfig(Builder builder) {
18+
if (builder.timeout != null
19+
&& (builder.timeout.toSeconds() <= 0 || builder.timeout.toSeconds() > MAX_TIMEOUT_SECONDS)) {
20+
throw new IllegalArgumentException(
21+
"timeout must be positive and at most 90 days, got: " + builder.timeout.toSeconds() + "s");
22+
}
23+
this.destination = builder.destination;
24+
this.completionConfig = builder.completionConfig;
25+
this.timeout = builder.timeout;
26+
this.resultSerDes = builder.resultSerDes;
27+
}
28+
29+
public DistributedMapDestination destination() {
30+
return destination;
31+
}
32+
33+
public CompletionConfig completionConfig() {
34+
return completionConfig;
35+
}
36+
37+
public Duration timeout() {
38+
return timeout;
39+
}
40+
41+
public SerDes resultSerDes() {
42+
return resultSerDes;
43+
}
44+
45+
public static Builder builder() {
46+
return new Builder();
47+
}
48+
49+
public Builder toBuilder() {
50+
var builder = new Builder();
51+
builder.destination = destination;
52+
builder.completionConfig = completionConfig;
53+
builder.timeout = timeout;
54+
builder.resultSerDes = resultSerDes;
55+
return builder;
56+
}
57+
58+
/** Builder for DistributedMapConfig. */
59+
public static class Builder {
60+
private DistributedMapDestination destination;
61+
private CompletionConfig completionConfig;
62+
private Duration timeout;
63+
private SerDes resultSerDes;
64+
65+
private Builder() {}
66+
67+
public Builder destination(DistributedMapDestination destination) {
68+
this.destination = destination;
69+
return this;
70+
}
71+
72+
public Builder completionConfig(CompletionConfig completionConfig) {
73+
this.completionConfig = completionConfig;
74+
return this;
75+
}
76+
77+
public Builder timeout(Duration timeout) {
78+
this.timeout = timeout;
79+
return this;
80+
}
81+
82+
public Builder resultSerDes(SerDes resultSerDes) {
83+
this.resultSerDes = resultSerDes;
84+
return this;
85+
}
86+
87+
public DistributedMapConfig build() {
88+
return new DistributedMapConfig(this);
89+
}
90+
}
91+
92+
/**
93+
* Failure-tolerance configuration for a distributed map run. {@code toleratedFailurePercentage} is a number from 0
94+
* through 100 and may be fractional.
95+
*/
96+
public record CompletionConfig(
97+
Integer toleratedFailureCount, Double toleratedFailurePercentage, Integer minimumSampleSize) {
98+
99+
public CompletionConfig {
100+
if (toleratedFailureCount != null && toleratedFailurePercentage != null) {
101+
throw new IllegalArgumentException(
102+
"toleratedFailureCount and toleratedFailurePercentage are mutually exclusive");
103+
}
104+
if (minimumSampleSize != null && toleratedFailurePercentage == null) {
105+
throw new IllegalArgumentException("minimumSampleSize is only valid with toleratedFailurePercentage");
106+
}
107+
// Omitting the config means zero tolerated failures, so an empty one is rejected rather than omitted.
108+
if (toleratedFailureCount == null && toleratedFailurePercentage == null) {
109+
throw new IllegalArgumentException(
110+
"a completion config must set toleratedFailureCount or toleratedFailurePercentage");
111+
}
112+
if (toleratedFailureCount != null && toleratedFailureCount < 0) {
113+
throw new IllegalArgumentException(
114+
"toleratedFailureCount must be non-negative, got: " + toleratedFailureCount);
115+
}
116+
if (toleratedFailurePercentage != null && !Double.isFinite(toleratedFailurePercentage)) {
117+
throw new IllegalArgumentException(
118+
"toleratedFailurePercentage must be a finite number, got: " + toleratedFailurePercentage);
119+
}
120+
if (toleratedFailurePercentage != null
121+
&& (toleratedFailurePercentage < 0 || toleratedFailurePercentage > 100)) {
122+
throw new IllegalArgumentException(
123+
"toleratedFailurePercentage must be between 0 and 100, got: " + toleratedFailurePercentage);
124+
}
125+
if (minimumSampleSize != null && minimumSampleSize < 1) {
126+
throw new IllegalArgumentException("minimumSampleSize must be at least 1, got: " + minimumSampleSize);
127+
}
128+
}
129+
130+
/** Stops the run after more than this many items permanently fail. */
131+
public static CompletionConfig failureCount(int count) {
132+
return new CompletionConfig(count, null, null);
133+
}
134+
135+
/** Abort once the failure rate exceeds this percentage, a number from 0 through 100 that may be fractional. */
136+
public static CompletionConfig failurePercentage(double percentage) {
137+
return new CompletionConfig(null, percentage, null);
138+
}
139+
140+
/**
141+
* Abort once the failure rate exceeds this percentage, a number from 0 through 100 that may be fractional,
142+
* gated by a minimum sample size.
143+
*/
144+
public static CompletionConfig failurePercentage(double percentage, int minimumSampleSize) {
145+
return new CompletionConfig(null, percentage, minimumSampleSize);
146+
}
147+
}
148+
}

0 commit comments

Comments
 (0)