Skip to content

Commit 55b31a9

Browse files
committed
ci: merge main and refresh shared runner pins
Keep the OTel caller identical to main. Pin the four remaining callers to merged shared-CI code and test their CodeBuild runner labels.
2 parents 9363d2a + 0ceaf04 commit 55b31a9

99 files changed

Lines changed: 12488 additions & 517 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/lambda-layer-publish.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
[layer]
2-
sdk-version = "2.0.0"
2+
sdk-version = "2.1.0"
Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
"""Install and verify built artifacts without silently using editable sources."""
2+
3+
from __future__ import annotations
4+
5+
import argparse
6+
import ast
7+
import hashlib
8+
import importlib.metadata
9+
import json
10+
import subprocess
11+
import sys
12+
import zipfile
13+
from pathlib import Path
14+
15+
16+
ROOT = Path(__file__).resolve().parents[2]
17+
CORE = "aws-durable-execution-sdk-python"
18+
OTEL = CORE + "-otel"
19+
20+
21+
def built_wheel(package: str) -> Path:
22+
directory = ROOT / "packages" / package
23+
module = package.replace("-", "_")
24+
about = ast.parse((directory / "src" / module / "__about__.py").read_text())
25+
version = next(
26+
ast.literal_eval(node.value)
27+
for node in about.body
28+
if isinstance(node, ast.Assign)
29+
and any(
30+
isinstance(target, ast.Name) and target.id == "__version__"
31+
for target in node.targets
32+
)
33+
)
34+
wheels = list((directory / "dist").glob(f"{module}-{version}-*.whl"))
35+
if len(wheels) != 1:
36+
raise ValueError(f"Build exactly one {package} {version} wheel first: {wheels}")
37+
return wheels[0]
38+
39+
40+
def verify(wheel: Path, package: str) -> None:
41+
installed = importlib.metadata.distribution(package)
42+
direct = json.loads(installed.read_text("direct_url.json") or "{}")
43+
assert not direct.get("dir_info", {}).get("editable"), direct
44+
digest = hashlib.sha256(wheel.read_bytes()).hexdigest()
45+
assert direct["archive_info"]["hashes"]["sha256"] == digest, direct
46+
module = package.replace("-", "_")
47+
with zipfile.ZipFile(wheel) as archive:
48+
sources = [
49+
name
50+
for name in archive.namelist()
51+
if name.startswith(module + "/") and name.endswith(".py")
52+
]
53+
assert sources
54+
for name in sources:
55+
path = Path(installed.locate_file(name)).resolve()
56+
assert "site-packages" in path.parts, path
57+
assert path.read_bytes() == archive.read(name), path
58+
print(
59+
json.dumps(
60+
{
61+
"package": package,
62+
"version": installed.version,
63+
"wheel": str(wheel),
64+
"sha256": digest,
65+
"verified_sources": len(sources),
66+
}
67+
)
68+
)
69+
70+
71+
def main() -> None:
72+
parser = argparse.ArgumentParser(description=__doc__)
73+
parser.add_argument("--legacy-plugin", action="store_true")
74+
args = parser.parse_args()
75+
packages = [CORE] if args.legacy_plugin else [CORE, OTEL]
76+
wheels = [built_wheel(package) for package in packages]
77+
subprocess.run(
78+
[
79+
sys.executable,
80+
"-m",
81+
"pip",
82+
"install",
83+
"--no-index",
84+
"--no-deps",
85+
"--force-reinstall",
86+
*map(str, wheels),
87+
],
88+
check=True,
89+
)
90+
for wheel, package in zip(wheels, packages, strict=True):
91+
verify(wheel, package)
92+
if args.legacy_plugin:
93+
assert importlib.metadata.version(OTEL) == "1.0.0"
94+
import aws_durable_execution_sdk_python_otel as otel
95+
96+
assert "site-packages" in Path(otel.__file__).resolve().parts
97+
subprocess.run([sys.executable, "-m", "pip", "check"], check=True)
98+
99+
100+
if __name__ == "__main__":
101+
main()

‎.github/scripts/js_examples/run.py‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,15 +61,24 @@
6161
TESTING_SOURCE = REPO_ROOT / "packages/aws-durable-execution-sdk-python-testing/src"
6262
JS_SDK_URL = "https://github.com/aws/aws-durable-execution-sdk-js.git"
6363
EXAMPLES_REL = Path("packages/aws-durable-execution-sdk-js-examples")
64-
# The examples depend on these workspaces by "*". The root "npm run build"
64+
# The examples depend on these local workspaces. The root "npm run build"
6565
# also builds the insight tools and the VS Code extension, which the harness
6666
# does not need, so the script builds only these, in dependency order.
6767
JS_BUILD_WORKSPACES = (
6868
"packages/aws-durable-execution-sdk-js",
6969
"packages/aws-durable-execution-sdk-js-testing",
7070
"packages/aws-durable-execution-sdk-js-otel",
71+
"packages/aws-durable-execution-sdk-js-extras",
72+
"packages/aws-durable-execution-sdk-js-microvm-worker",
7173
"packages/aws-durable-execution-sdk-js-examples",
7274
)
75+
# Older JS refs predate these packages; their examples do not need them.
76+
OPTIONAL_JS_BUILD_WORKSPACES = frozenset(
77+
{
78+
"packages/aws-durable-execution-sdk-js-extras",
79+
"packages/aws-durable-execution-sdk-js-microvm-worker",
80+
}
81+
)
7382
# otel examples export spans to an OpenTelemetry collector. The harness does
7483
# not run one, so these examples cannot pass here. They are not selected.
7584
EXCLUDED_DIRS = ("/otel/",)
@@ -202,6 +211,11 @@ def build_js_sdk(js_dir: Path, *, force: bool) -> None:
202211
[npm, "ci", "--no-audit", "--no-fund"], cwd=js_dir, env=env, check=True
203212
)
204213
for workspace in JS_BUILD_WORKSPACES:
214+
if (
215+
workspace in OPTIONAL_JS_BUILD_WORKSPACES
216+
and not (js_dir / workspace / "package.json").is_file()
217+
):
218+
continue
205219
log(f"npm run build -w {workspace}")
206220
subprocess.run(
207221
[npm, "run", "build", "-w", workspace], cwd=js_dir, env=env, check=True

‎.github/scripts/tests/test_js_examples.py‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -623,6 +623,58 @@ def test_source_fingerprint_changes_with_local_edits(tmp_path: Path) -> None:
623623
assert run.source_fingerprint(tmp_path) != untracked
624624

625625

626+
@pytest.mark.parametrize("has_microvm_packages", [False, True])
627+
def test_build_produces_example_workspace_dependencies(
628+
tmp_path: Path,
629+
monkeypatch: pytest.MonkeyPatch,
630+
has_microvm_packages: bool,
631+
) -> None:
632+
core = "packages/aws-durable-execution-sdk-js"
633+
testing = "packages/aws-durable-execution-sdk-js-testing"
634+
otel = "packages/aws-durable-execution-sdk-js-otel"
635+
extras = "packages/aws-durable-execution-sdk-js-extras"
636+
worker = "packages/aws-durable-execution-sdk-js-microvm-worker"
637+
examples_package = "packages/aws-durable-execution-sdk-js-examples"
638+
packages = [core, testing, otel, examples_package]
639+
if has_microvm_packages:
640+
packages.extend([extras, worker])
641+
for package in packages:
642+
directory = tmp_path / package
643+
directory.mkdir(parents=True)
644+
(directory / "package.json").write_text("{}")
645+
646+
built: list[str] = []
647+
648+
def execute(args: list[str], **_kwargs: Any) -> subprocess.CompletedProcess[str]:
649+
if args[1] == "ci":
650+
return subprocess.CompletedProcess(args, 0)
651+
workspace = args[-1]
652+
assert (tmp_path / workspace / "package.json").is_file()
653+
if workspace == worker:
654+
assert testing in built
655+
if workspace == examples_package and has_microvm_packages:
656+
# The generator imports extras/microvm and typechecks the worker.
657+
# A successful examples build requires both artifacts to exist.
658+
assert extras in built
659+
assert worker in built
660+
built.append(workspace)
661+
return subprocess.CompletedProcess(args, 0)
662+
663+
stamp = tmp_path / "built-stamp"
664+
monkeypatch.setattr(run, "_head", lambda _: "test-head")
665+
monkeypatch.setattr(run, "state_file", lambda *_: stamp)
666+
monkeypatch.setattr(run, "source_fingerprint", lambda _: "fingerprint")
667+
monkeypatch.setattr(run.shutil, "which", lambda _: "/fake/npm")
668+
monkeypatch.setattr(run.subprocess, "run", execute)
669+
670+
run.build_js_sdk(tmp_path, force=True)
671+
672+
assert built[-1] == examples_package
673+
assert stamp.read_text() == "fingerprint\n"
674+
if not has_microvm_packages:
675+
assert built == [core, testing, otel, examples_package]
676+
677+
626678
def test_proxy_creates_the_dump_directory(tmp_path: Path) -> None:
627679
dump_dir = tmp_path / "new" / "dump"
628680
proxy = invoke_proxy.ProxyServer(0, "http://127.0.0.1:1", dump_dir)

‎.github/scripts/tests/test_opentelemetry_conformance_workflow.py‎

Lines changed: 37 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
from pathlib import Path
22

3+
import yaml
4+
35

46
WORKFLOW_PATH = (
57
Path(__file__).parents[2] / "workflows" / "opentelemetry-conformance-tests.yml"
68
)
7-
EXAMPLES_DIR = (
8-
".build/durable-sdk/packages/aws-durable-execution-sdk-python-conformance-tests-otel"
9-
)
9+
EXAMPLES_DIR = ".build/durable-sdk/packages/aws-durable-execution-sdk-python-conformance-tests-otel"
1010

1111

1212
def test_opentelemetry_conformance_caller_uses_current_workflow_contract() -> None:
@@ -24,6 +24,7 @@ def test_opentelemetry_conformance_caller_uses_current_workflow_contract() -> No
2424

2525
for configuration in (
2626
"language: python",
27+
"runs_on: codebuild-github-actions-runner-${{ github.run_id }}-${{ github.run_attempt }}",
2728
"resource_prefix: p",
2829
"sdk_repository: aws/aws-durable-execution-sdk-python",
2930
"sdk_ref: ${{ github.event.pull_request.head.sha || github.sha }}",
@@ -66,8 +67,39 @@ def test_opentelemetry_conformance_runs_when_the_handlers_change() -> None:
6667
workflow = WORKFLOW_PATH.read_text()
6768

6869
trigger_path = (
69-
" - "
70-
'"packages/aws-durable-execution-sdk-python-conformance-tests-otel/**"'
70+
' - "packages/aws-durable-execution-sdk-python-conformance-tests-otel/**"'
7171
)
7272
# Once for pull_request, once for push.
7373
assert workflow.count(trigger_path) == 2
74+
75+
76+
def test_opentelemetry_conformance_queues_complete_runs() -> None:
77+
workflow = yaml.safe_load(WORKFLOW_PATH.read_text())
78+
79+
assert workflow["concurrency"] == {
80+
"group": "otel-conformance-tests",
81+
"cancel-in-progress": False,
82+
"queue": "max",
83+
}
84+
85+
86+
def test_cloud_tests_queue_each_shared_runtime_stack() -> None:
87+
workflow = yaml.safe_load(WORKFLOW_PATH.with_name("cloud-tests.yml").read_text())
88+
89+
assert workflow["jobs"]["example-tests"]["concurrency"] == {
90+
"group": "cloud-tests-${{ matrix.python-prefix }}",
91+
"cancel-in-progress": False,
92+
"queue": "max",
93+
}
94+
95+
96+
def test_general_conformance_keeps_all_pending_runs() -> None:
97+
workflow = yaml.safe_load(
98+
WORKFLOW_PATH.with_name("conformance-tests.yml").read_text()
99+
)
100+
101+
assert workflow["concurrency"] == {
102+
"group": "conformance-tests-global",
103+
"cancel-in-progress": False,
104+
"queue": "max",
105+
}
Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
from pathlib import Path
2+
3+
import pytest
4+
import yaml
5+
6+
7+
@pytest.mark.parametrize(
8+
("filename", "job_id"),
9+
[
10+
("ai-pr-review.yml", "ai-pr-review"),
11+
("issue-triage.yml", "triage"),
12+
("notify.yml", "notify"),
13+
("stale-issue-closer.yml", "close-stale"),
14+
],
15+
)
16+
def test_shared_workflow_caller_passes_codebuild_runner(
17+
filename: str, job_id: str
18+
) -> None:
19+
path = Path(__file__).parents[2] / "workflows" / filename
20+
workflow = yaml.safe_load(path.read_text())
21+
22+
assert workflow["jobs"][job_id]["with"]["runs-on"] == (
23+
"codebuild-github-actions-runner-${{ github.run_id }}-${{ github.run_attempt }}"
24+
)

‎.github/scripts/type-checks.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,10 @@ mypy --install-types --non-interactive \
1717
packages/aws-durable-execution-sdk-python-insight/src/aws_durable_execution_sdk_python_insight \
1818
packages/aws-durable-execution-sdk-python-insight/tests
1919

20+
mypy --install-types --non-interactive \
21+
packages/aws-durable-execution-sdk-python-microvm-worker/src/aws_durable_execution_sdk_python_microvm_worker \
22+
packages/aws-durable-execution-sdk-python-microvm-worker/tests
23+
2024
# comment out this for now as there are many type check errors in this package
2125
#mypy --install-types --non-interactive \
2226
# packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing \

0 commit comments

Comments
 (0)