Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 85 additions & 0 deletions .github/scripts/package_release_bundles.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
#!/usr/bin/env python3
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
"""Bundle the exact signed Maven publication outputs for archive-based consumers.

Run after maven_publish.sh, which deploys with the publishing profile and so leaves
every file submitted to Sonatype in each module's target directory: the flattened
POM, the three JARs, and a detached GPG signature for each of them.
"""

import argparse
import hashlib
import json
from pathlib import Path
import re
import xml.etree.ElementTree as ET
import zipfile


ROOT = Path(__file__).resolve().parents[2]
GROUP_ID = "software.amazon.lambda.durable"
MODULES = {
"sdk": "aws-durable-execution-sdk-java",
"sdk-testing": "aws-durable-execution-sdk-java-testing",
"otel-plugin": "aws-durable-execution-sdk-java-plugin-otel",
"telemetry-exporter": "aws-durable-execution-sdk-java-plugin-telemetry-exporter",
}
SUFFIXES = (".pom", ".jar", "-sources.jar", "-javadoc.jar")
NAMESPACE = {"m": "http://maven.apache.org/POM/4.0.0"}
# A published release version: semantic, optionally a prerelease, never a snapshot.
RELEASE_VERSION = re.compile(r"[0-9]+\.[0-9]+\.[0-9]+(?:-(?!SNAPSHOT\Z)[0-9A-Za-z][0-9A-Za-z.-]*)?")


def publication_names(artifact, version):
"""Return every file Maven publishes for a module, then their detached signatures."""
names = [f"{artifact}-{version}{suffix}" for suffix in SUFFIXES]
return names + [f"{name}.asc" for name in names]


def package_bundle(root, module, artifact, version, output):
target = root / module / "target"
files = {name: (target / name).read_bytes()
for name in publication_names(artifact, version)}

project = ET.fromstring(files[f"{artifact}-{version}.pom"])
coordinates = tuple(project.findtext("m:" + key, namespaces=NAMESPACE)
for key in ("groupId", "artifactId", "version"))
if coordinates != (GROUP_ID, artifact, version):
raise ValueError(f"Unexpected Maven coordinates in {module}: {coordinates}")

manifest = {
"groupId": GROUP_ID,
"artifactId": artifact,
"version": version,
"sha256": {name: hashlib.sha256(data).hexdigest() for name, data in files.items()},
}
files["manifest.json"] = (json.dumps(manifest, indent=2, sort_keys=True) + "\n").encode()

destination = output / f"{artifact}-{version}-bundle.zip"
with zipfile.ZipFile(destination, "w") as bundle:
for name, data in sorted(files.items()):
entry = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0))
entry.create_system = 3
entry.compress_type = zipfile.ZIP_DEFLATED
entry.external_attr = 0o100644 << 16
bundle.writestr(entry, data)
return destination


def main(root=ROOT):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("version")
parser.add_argument("output", type=Path)
args = parser.parse_args()
if not RELEASE_VERSION.fullmatch(args.version):
parser.error("version must be a release version such as 3.0.1")
args.output.mkdir(parents=True, exist_ok=True)
for stale in args.output.glob("*-bundle.zip"):
stale.unlink()
for module, artifact in MODULES.items():
print(package_bundle(root, module, artifact, args.version, args.output))


if __name__ == "__main__":
main()
149 changes: 149 additions & 0 deletions .github/scripts/test_release_bundles.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
#!/usr/bin/env python3
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0

import contextlib
import hashlib
import io
import json
from pathlib import Path
import re
import subprocess
import sys
import tempfile
import unittest
from unittest import mock
import xml.etree.ElementTree as ET
import zipfile

from package_release_bundles import (GROUP_ID, MODULES, NAMESPACE, RELEASE_VERSION, ROOT,
main, package_bundle, publication_names)


VERSION = "3.0.1"
SCRIPT = Path(__file__).with_name("package_release_bundles.py")


class ReleaseBundlesTest(unittest.TestCase):
def setUp(self):
temporary = tempfile.TemporaryDirectory()
self.addCleanup(temporary.cleanup)
self.root = Path(temporary.name)
self.output = self.root / "bundles"
self.output.mkdir()
for module, artifact in MODULES.items():
target = self.root / module / "target"
target.mkdir(parents=True)
(target / f"{artifact}-{VERSION}.pom").write_text(
'<project xmlns="http://maven.apache.org/POM/4.0.0">'
f"<groupId>{GROUP_ID}</groupId><artifactId>{artifact}</artifactId>"
f"<version>{VERSION}</version></project>\n")
for classifier in ("", "-sources", "-javadoc"):
with zipfile.ZipFile(target / f"{artifact}-{VERSION}{classifier}.jar", "w") as jar:
jar.writestr("fixture", (artifact + classifier).encode())
# The publishing profile signs every file above, so each has a detached signature.
for published in sorted(target.iterdir()):
published.with_name(published.name + ".asc").write_text(
f"-----BEGIN PGP SIGNATURE-----\n{published.name}\n-----END PGP SIGNATURE-----\n")

def test_preserves_every_signed_maven_artifact_and_records_checksums(self):
for module, artifact in MODULES.items():
with self.subTest(module=module):
output = package_bundle(self.root, module, artifact, VERSION, self.output)
published = publication_names(artifact, VERSION)
with zipfile.ZipFile(output) as bundle:
manifest = json.loads(bundle.read("manifest.json"))
self.assertEqual((manifest["groupId"], manifest["artifactId"], manifest["version"]),
(GROUP_ID, artifact, VERSION))
self.assertEqual(bundle.namelist(), sorted(published + ["manifest.json"]))
self.assertEqual(sorted(manifest["sha256"]), sorted(published))
for filename, digest in manifest["sha256"].items():
original = (self.root / module / "target" / filename).read_bytes()
self.assertEqual(bundle.read(filename), original)
self.assertEqual(hashlib.sha256(original).hexdigest(), digest)

def test_normalizes_every_zip_entry(self):
module, artifact = next(iter(MODULES.items()))
path = package_bundle(self.root, module, artifact, VERSION, self.output)
with zipfile.ZipFile(path) as bundle:
for entry in bundle.infolist():
with self.subTest(entry=entry.filename):
self.assertEqual(entry.date_time, (1980, 1, 1, 0, 0, 0))
self.assertEqual(entry.create_system, 3)
self.assertEqual(entry.compress_type, zipfile.ZIP_DEFLATED)
self.assertEqual(entry.external_attr >> 16, 0o100644)

def test_bundle_is_reproducible(self):
module, artifact = next(iter(MODULES.items()))
with mock.patch("zipfile.time.localtime", return_value=(2001, 1, 1, 0, 0, 0, 0, 1, -1)):
path = package_bundle(self.root, module, artifact, VERSION, self.output)
first = path.read_bytes()
with mock.patch("zipfile.time.localtime", return_value=(2031, 1, 1, 0, 0, 0, 0, 1, -1)):
path = package_bundle(self.root, module, artifact, VERSION, self.output)
self.assertEqual(first, path.read_bytes())

def test_rejects_wrong_coordinate(self):
module, artifact = next(iter(MODULES.items()))
pom = self.root / module / "target" / f"{artifact}-{VERSION}.pom"
original = pom.read_text()
for field, expected in (("groupId", GROUP_ID), ("artifactId", artifact), ("version", VERSION)):
with self.subTest(field=field):
pom.write_text(original.replace(f"<{field}>{expected}</{field}>",
f"<{field}>wrong</{field}>"))
try:
with self.assertRaises(ValueError):
package_bundle(self.root, module, artifact, VERSION, self.output)
finally:
pom.write_text(original)

def test_any_missing_file_fails_before_writing_bundle(self):
module, artifact = next(iter(MODULES.items()))
target = self.root / module / "target"
for name in publication_names(artifact, VERSION):
with self.subTest(missing=name):
missing = target / name
data = missing.read_bytes()
missing.unlink()
try:
with self.assertRaises(FileNotFoundError):
package_bundle(self.root, module, artifact, VERSION, self.output)
self.assertEqual(list(self.output.iterdir()), [])
finally:
missing.write_bytes(data)

def test_modules_match_the_modules_maven_publishes(self):
published = re.findall(r"-pl (\S+)", (ROOT / ".github/scripts/maven_publish.sh").read_text())
self.assertEqual(sorted(MODULES), sorted(published))
for module, artifact in MODULES.items():
with self.subTest(module=module):
pom = ET.parse(ROOT / module / "pom.xml")
self.assertEqual(pom.findtext("m:artifactId", namespaces=NAMESPACE), artifact)
self.assertEqual(pom.findtext("m:parent/m:groupId", namespaces=NAMESPACE), GROUP_ID)

def test_release_version_accepts_releases_and_prereleases_only(self):
for version in ("3.0.1", "3.0.1-rc1", "10.20.30-beta.2"):
self.assertTrue(RELEASE_VERSION.fullmatch(version), version)
for version in ("3.0.2-SNAPSHOT", "../3.0.1", "3.0", "3.0.1-", "3.0.1 "):
self.assertFalse(RELEASE_VERSION.fullmatch(version), version)

def test_command_generates_every_bundle_and_clears_stale_output(self):
stale = self.output / f"{next(iter(MODULES.values()))}-2.0.0-bundle.zip"
stale.write_bytes(b"left over from an earlier version")
with mock.patch.object(sys, "argv", [SCRIPT.name, VERSION, str(self.output)]), \
contextlib.redirect_stdout(io.StringIO()):
main(self.root)
self.assertEqual(sorted(path.name for path in self.output.iterdir()),
sorted(f"{artifact}-{VERSION}-bundle.zip" for artifact in MODULES.values()))

def test_command_rejects_a_version_that_is_not_a_release(self):
for version in ("../3.0.1", "3.0.2-SNAPSHOT"):
with self.subTest(version=version):
result = subprocess.run([sys.executable, str(SCRIPT), version, str(self.output)],
capture_output=True, text=True)
self.assertNotEqual(result.returncode, 0)
self.assertIn("version must be a release version", result.stderr)
self.assertEqual(list(self.output.iterdir()), [])


if __name__ == "__main__":
unittest.main()
10 changes: 10 additions & 0 deletions .github/workflows/publish_maven.yml
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,16 @@ jobs:
"telemetry-exporter/target/aws-durable-execution-sdk-java-plugin-telemetry-exporter-${RELEASE_VERSION}.jar" \
--clobber

# Packaged after the JARs are attached so that a bundling failure cannot
# withhold the release assets that the JAR upload above already published.
- name: Package Maven release bundles
run: python3 .github/scripts/package_release_bundles.py "$RELEASE_VERSION" release-bundles

- name: Upload GitHub release bundles
env:
GH_TOKEN: ${{ github.token }}
run: gh release upload "$RELEASE_TAG" release-bundles/*-bundle.zip --clobber

- name: Checkout default branch
uses: actions/checkout@v7
with:
Expand Down
13 changes: 13 additions & 0 deletions .github/workflows/test-release-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,22 @@ on:
paths:
- '.github/workflows/publish_maven.yml'
- '.github/workflows/test-release-build.yml'
- '.github/scripts/maven_publish.sh'
- '.github/scripts/set_release_timestamp.sh'
- '.github/scripts/test_release_timestamp.py'
- '.github/scripts/package_release_bundles.py'
- '.github/scripts/test_release_bundles.py'
- 'pom.xml'
push:
branches: [main]
paths:
- '.github/workflows/publish_maven.yml'
- '.github/workflows/test-release-build.yml'
- '.github/scripts/maven_publish.sh'
- '.github/scripts/set_release_timestamp.sh'
- '.github/scripts/test_release_timestamp.py'
- '.github/scripts/package_release_bundles.py'
- '.github/scripts/test_release_bundles.py'
- 'pom.xml'
workflow_dispatch:

Expand All @@ -34,3 +40,10 @@ jobs:
cache: maven
- name: Test release timestamp and Javadoc reproducibility
run: python3 .github/scripts/test_release_timestamp.py

release-bundles:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
- name: Test Maven release bundles
run: python3 .github/scripts/test_release_bundles.py
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -47,3 +47,4 @@ samconfig.toml.bak

# OSS
.flattened-pom.xml
release-bundles/
15 changes: 13 additions & 2 deletions RELEASE.md
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,13 @@ The publication workflow:
2. Builds, signs, and uploads the SDK, testing library, OpenTelemetry plugin,
and Durable Execution Telemetry Exporter to Sonatype Central Portal.
3. Uploads the four JARs to the existing GitHub release.
4. Opens a pull request for the next development version. A final release
4. Attaches one ZIP bundle per module to the same release. Each bundle holds
every file that module published to Sonatype: the POM, the runtime, source,
and Javadoc JARs, and a detached GPG signature for each of them. It also
holds a `manifest.json` recording the Maven coordinates and a SHA-256 for
every bundled file. The standalone JARs stay available, so archive consumers
and Maven consumers get the same artifacts.
5. Opens a pull request for the next development version. A final release
increments the patch version, so `2.1.1` produces `2.1.2-SNAPSHOT`. A
prerelease keeps the same base version, so `2.1.1-rc1` produces
`2.1.1-SNAPSHOT` until the final `2.1.1` release.
Expand All @@ -62,7 +68,12 @@ After **Publish Maven Release** succeeds:
The workflow uses `autoPublish=false`, so this manual action is required.
4. Confirm that the GitHub release contains the expected JARs and that the
artifacts are available in Maven Central.
5. Review, approve, and merge the next development version pull request.
5. Confirm that the GitHub release also contains the four `*-bundle.zip`
assets and that each one is for the release version. A bundle verifies on
its own: extract it, check the SHA-256 of every file against
`manifest.json`, then verify a signature against the release signing key
with `gpg --verify <file>.asc <file>`.
6. Review, approve, and merge the next development version pull request.

## 4. Update the shared documentation

Expand Down
Loading