From f107ded4ea84a67cddd4f9e4277c695ea0f551e3 Mon Sep 17 00:00:00 2001 From: Volv G Date: Tue, 29 Sep 2026 18:25:50 -0700 Subject: [PATCH 1/2] Add `pipelines decompile`: YAML -> Python pipeline source (v0.1.31) --- README.md | 250 ++ .../tangle-cli/src/tangle_cli/__init__.py | 2 +- .../src/tangle_cli/pipeline_decompiler.py | 3200 +++++++++++++++++ .../src/tangle_cli/pipeline_hydrator.py | 5 + .../src/tangle_cli/pipelines_cli.py | 130 + .../src/tangle_cli/schema_validation.py | 11 +- pyproject.toml | 2 +- tests/test_packaging.py | 2 +- tests/test_pipeline_decompiler.py | 2031 +++++++++++ tests/test_pipelines_cli.py | 129 + uv.lock | 2 +- 11 files changed, 5758 insertions(+), 6 deletions(-) create mode 100644 packages/tangle-cli/src/tangle_cli/pipeline_decompiler.py create mode 100644 tests/test_pipeline_decompiler.py diff --git a/README.md b/README.md index ad99c53..c6dea38 100644 --- a/README.md +++ b/README.md @@ -545,6 +545,256 @@ flags are the same as `pipeline-runs submit`; the extra compile-tier flags are `--pipeline`, repeatable `--override KEY=VALUE`, and repeatable `--image ID=REF`. Use `pipelines compile` instead when the compiled YAML itself is what you want. +##### Decompiling YAML back to Python + +`pipelines decompile` is the reverse direction: it turns a pipeline YAML — +**hydrated or dehydrated** — into `python_pipeline` source, so an existing YAML +pipeline can be adopted into Python authoring without being retyped. + +```bash +uv run tangle sdk pipelines decompile pipeline.yaml -o pipeline.py +``` + +**Dehydration is part of decompiling, not an option.** The command always +produces canonical pinned references before generating any code, using the +same `PipelineDehydrator` as `pipelines dehydrate --mode digest`. A dehydrated +input is resolved first and then dehydrated; a hydrated input goes to that same +dehydrator directly. Both gates below run on that portable form: each leaf +behind a resolve fragment listing a verified published digest first, when the +library has one, and then a local copy. + +What is written then depends on `--local-fallbacks`: + +- **By default**, a leaf the library verified becomes `ref(digest=...)` — its + exact published content, with no local copy — so **hydrating the output + needs the component library**. A leaf with no published digest becomes + `ref(url="resolve://./.leaves.yaml#")` with its only + copy in `.leaves/`; when every leaf is published, neither is + written. +- **With `--local-fallbacks`** (`include_local_fallbacks=True`), every leaf + keeps the portable form — digest first, local copy behind it — so the + script compiles and hydrates with the library unreachable. + +(The sidecar is not `.components.yaml`: that is where +`compile .py -o .yaml` writes its own `@task` sidecar. An existing +`.components*` artifact is never touched, as it may be the compiler's.) + +**Task layout uses `.with_position()`.** A task's `editor.position` — on an +ordinary ref, a recovered `@task` or a parent `subpipeline(...)` card — is +written as `.with_position(x, y[, width=..., height=...])`, after any other +annotations, but only when upstream's own rendering reproduces the stored +string exactly. Anything else (the editor's compact `{"x":1,"y":2}`, reordered +keys, extra fields, non-finite, boolean or null values, a non-canonical +exponent like `1e3`, a number too large to convert) stays +verbatim in `.with_annotations(...)`, so no value is ever re-rendered. Graph +inputs and outputs use the same rule through `graph_input(...)` / +`graph_output(...)`'s `position=(x, y)`, applied after `annotations=`; as that +takes x and y only, a position with `width`/`height` stays in `annotations=`. + +**Every graph becomes Python.** Each nested graph, at any depth, becomes its +own `@pipeline` function in a companion `_subgraphs.py`, which the +root script imports. The parent calls it through +`subpipeline(fn).named()(...)`, keeping task IDs, arguments, edges and +annotations. Identical graphs share one function. A graph whose name is already +taken by different content gets a short content-digest suffix, so the naming is +deterministic. Functions are defined deepest first, and `compile .py` +works with no `--pipeline` flag. Only leaf components go through the resolve +sidecar. A nested-graph task that sets `isEnabled` or `executionOptions` is +refused with `unsupported-subgraph-task`: Tangle does not apply those to +graph-component tasks, and `subpipeline(...)` rejects them. A cycle of +component references is refused with `cyclic-component-reference`. Because the +script imports its companion by name, a pipeline with nested graphs needs an +output name that makes `_subgraphs` a valid ASCII Python module name (e.g. +`gen.py`, not `foo-bar.py`); anything else is refused with +`unsupported-output-name` before anything is written. A canonical URL is +deliberately *not* kept: its contents can change underneath the reference. See +[Resolve-config fallback entries](#resolve-config-fallback-entries) for the +sidecar format. + +**Python components become `@task` functions** (unless `--no-python-tasks`). +A leaf the Python component generator made — carrying its markers, the full +`python_original_code`, `tangle_cli_generation_function_name` and +`tangle_cli_generation_mode: inline` — is written as source instead of a pinned +YAML leaf. Each recorded module goes byte-for-byte into its own directory, +`_tasks//.py`, next to a TOML file holding its +`python_dependencies`; `_tasks/__init__.py` exports it. The root +and the companion import it by name, so a leaf they share is defined once. +Nothing is inferred from the container command. + +- A plain function is wrapped once, in + `task(image=, mode="inline", dependencies_from=...)`. +- Source that is already `python_pipeline` authoring — one literal + `@task(...)` from an unaliased `from tangle_cli.python_pipeline import task`, + with only `image`, `image_id`, `mode="inline"`, a relative `.toml` + `dependencies_from` and `unwrap` — is re-exported as it is, never wrapped a + second time. Its own `dependencies_from` path is honoured inside its + directory; without one, `.toml` is what the generator's discovery + finds first, so a `pyproject.toml` further up cannot change the list. +- `image=` must equal the recorded image. **`image_id=` is kept as the source's + intent: compile resolves it in your environment** (`--image ID=REF`, then + registered defaults), which may give a different image than the input + recorded. Use `--no-python-tasks` for the digest-exact component. +- `@task(unwrap=...)` inputs are checked by name: every input must be exactly + one dict parameter's (`dict`, or `Dict` from `typing`) `__`, and every task using it must pass + exactly those inputs. The call is written back as `param={key: ...}` in the + spec's input order. Their types are derived again at compile. + +This is a source transform only: **the decompiler never imports, runs, +regenerates, compiles or hydrates recovered source** — nor any input's local +Python: a dehydrated input whose components resolve through +`local_from_python` (a compiled `@task` pipeline) is refused with +`unsupported-local-python-source`, whatever the cwd or `--trusted-source` +trust. So is any other resolver kind that may build or run code — +`local_from_docker` and the other docker/container spellings, or any kind a +downstream package registers — with `unsupported-executable-component-source`, +and a `template_file` component config, whose Jinja could reach Python, with +`unsupported-template-component-source`: decompile resolves only digests, +names, http(s) URLs and plain files, through the built-in resolvers and URI +readers; a URI scheme only a downstream reader handles is refused the same way. Run `tangle sdk pipelines hydrate` on such a pipeline first — that is +where trusted sources execute — and decompile the hydrated YAML, which carries +the recorded source for recovery. Candidates are checked +statically, against the generator's own rules — the source parses, binds the +recorded function exactly once at module scope; the docstring yields the +spec's name, description and metadata annotations; every input and output +names one parameter. Anything `@task` cannot express, or that cannot be +checked from the text — bundle mode, a resolve root, a custom name or +annotation, any other decorator or decorator setting, a relative import or +star import, any container field beyond `image`/`command`/`args` or +input/output field the generator does not write, a name rebound anywhere at +module scope — stays a pinned YAML leaf, and is only counted in the output. +Both gates run on the all-YAML form; the final script is then checked to differ +from that verified form only at the converted call sites. + +What that does **not** prove is the component Tangle rebuilds: parameter types, +the generated container program (which follows the installed generator's +version), an `image_id`'s resolution, digests and provenance annotations +(`git_*`, the code and YAML paths) are produced later, when `compile` imports +`_tasks` and hydrate, submit or run execute it. Review that package +before compiling; its top-level imports must be installed, and hydration +executes it only from the current directory tree or a `--trusted-source`. +Converted components are no longer digest-pinned — use `--no-python-tasks` for +an output whose every component is. The tasks package follows the same +one-to-one output name rule as the companion. + +Because canonicalization resolves components, the command needs whatever access +the input's references require. If any reference cannot be resolved the whole +operation is refused; a pipeline is never emitted with some components pinned +and others left dangling. + +**Two independent gates** must both pass before anything is written: + +| Gate | Question | How | +| --- | --- | --- | +| (a) | Did pinning preserve the pipeline? | The pinned document is resolved again from the script's own directory, through its sidecar, and must yield the same component specs hydration produced. Every leaf task must use exactly `resolve://./.leaves.yaml#`, and every sidecar entry must be a digest or a local copy inside `.leaves/`. An absolute, traversing, foreign or mutable locator is refused even when it happens to resolve. | +| (b) | Does the Python reproduce the pinned document? | The generated source is recompiled and must match the pinned document's *semantic digest* (`compute_spec_digest`, sha256 of sorted-key YAML). | + +Both are needed because they check different things. The generated Python +names only a sidecar fragment, never the component's contents, so gate (b) +alone cannot tell a correct pin from a wrong one. Both run on the portable +form; the default output then replaces each verified leaf's fragment with its +already-verified `ref(digest=...)`, a substitution checked statically to be +the only difference from the verified source. + +**Exactness is scoped to seed time.** Decompiling is a one-time seeding +transform: it reproduces the *exact* component every input reference names. +Canonicalization and both parts of gate (a) resolve with deprecation successors +**disabled**, so a deprecated digest is seeded and verified as its own content, +not as its successor's. After that, ordinary `pipelines hydrate`, +`pipeline-runs submit` and `run` follow successors as they always do. Taking up +an upgrade that changes a component is the maintainer's decision, not +something decompile makes on their behalf. + +Pass `--no-verify` to skip gate (b) — which removes the only proof the *source* is +faithful. Gate (a) always runs. + +`-o/--output` is always replaced, with no existence check and no `--force`. +The script, its `_subgraphs.py` companion (when there are nested graphs), +its `.leaves.yaml` sidecar and its `.leaves/` bundle are +staged together and published only after both gates pass. Gate (b) verifies +each generated `@pipeline` function on its own, including which child each +`subpipeline(...)` call targets. A refusal +therefore leaves any existing output and its previous products untouched, and +no staging directory is left behind. Publication itself is renames only, with +the script renamed last (replacing a symlink rather than writing through it). +Previous products are moved aside rather than deleted, so a failed rename at +any boundary restores the whole previous generation. A stale companion, +sidecar or bundle from an earlier run of the same output is replaced +wholesale, or removed when the new run no longer needs it; it is never merged +into. + +**What it accepts.** A pipeline document in either shape: *dehydrated* (what +`pipelines compile` emits and `pipelines validate` accepts) or *hydrated* (with +inline `componentRef.spec`s, including nested graphs). The input is classified +and validated against the contract matching its own shape **before** anything +else looks at it, because resolution would otherwise launder an unsupported +document into one that passes a later check. Inline component `text`, legacy +keys and compile-time `{%` templates remain out of scope in both shapes and are +refused up front — `text` in particular is dropped by hydration, so accepting it +would silently generate a script missing part of its input. + +Both shapes are canonicalized before codegen, so the input's shape does not +change the *pipeline* that is generated. It can still change the **locators**, +because a dehydrated reference carries provenance an inline spec does not: the +hydrator records the digest of the reference's source text, and a component +published under that digest gets it as its verified primary. An inline +`componentRef.spec` has no source text, so it is addressed by its sorted-spec +digest and gets only its local copy. Equivalent inputs therefore agree +exactly when the lookup outcome agrees — always for an unpublished component, +and not necessarily for a published one. + +**What it does not guarantee.** Not byte identity in general. Comments, key +order and quoting style are not recoverable. Task order *is* preserved: ready +tasks are emitted one at a time in the document's own order, so a document +whose tasks are already in dependency order keeps that order exactly, and only +a document that lists a consumer before its producer gets reordered. When the +input was itself produced by `pipelines compile` there is therefore nothing +left to lose and the bytes do match — that fixpoint is covered by a test over +the repo's example pipelines, exercised through the in-memory +`decompile_pipeline` API, since the file command re-pins locators by design and +so cannot preserve bytes. + +**Nothing echoes document values — on any channel.** A schema failure reports +the structural location only, taken from the validator's own path metadata +rather than parsed out of its message, because the rejected value may be a +credential in an argument or annotation. A parse failure reports line and +column only. Locators in resolution errors *are* shown — they are what you +must fix — with any URL userinfo or query string redacted. Resolution and +pinning run with a silent logger, so component names, locators and the source +lines of a malformed referenced component never reach stdout or stderr +either; progress is reported by this command instead. + +**Two comparator normalizations** are applied before digests are compared, and +both are reported on the command line when they fire: + +| | Rule | Why | +| --- | --- | --- | +| N1 | An input with `default` and no explicit `optional` compares as `optional: true`. | `graph_input()` always writes `optional` alongside a `default`. | +| N2 | An empty block at a known optional location (`metadata`, `inputs`, `outputs`, `outputValues`, or a task's `annotations`, `arguments`, `executionOptions`) compares as absent. | Such a block declares nothing, and the authoring API omits the key rather than writing an empty one. A block with any content is never dropped. | + +Neither rule edits the generated source or the input; they exist only inside +the comparator. + +**Scope.** `pipelines decompile` accepts hydrated or dehydrated input (see +*What it accepts*). Inline component text, legacy keys, `{%`/`{#` compile-time +templates, `cfg` lifting and schedule/deploy metadata are out of scope and fail +closed with a diagnostic code. The in-memory `decompile_pipeline` API takes +only a dehydrated document, whose nested graphs are therefore references rather +than inline graphs. Refusal messages name the construct and its location and +never echo the value, which may be a secret. + +**Safety.** Verification imports the generated file, so decompiling an +untrusted export executes code derived from it (never recovered Python source, +which is only copied). Every recovered value is built +as an `ast` literal node rather than interpolated as text, no input text is +placed in a comment or docstring, and the finished source is re-parsed and +checked against a node allowlist before it is imported. + +One practical note: the compiler resolves relative component URLs against the +*compiled YAML's* directory, not the script's. Extracted components therefore +travel with the script, and recompiling generated source has to target the +directory holding them — compile the script in place. The same applies to a +relative `resolve://./.leaves.yaml` ref, which is preserved verbatim. + The two value tiers are distinct and easy to confuse: | Flag | Tier | Meaning | diff --git a/packages/tangle-cli/src/tangle_cli/__init__.py b/packages/tangle-cli/src/tangle_cli/__init__.py index e78b9c0..d203387 100644 --- a/packages/tangle-cli/src/tangle_cli/__init__.py +++ b/packages/tangle-cli/src/tangle_cli/__init__.py @@ -14,6 +14,6 @@ try: __version__ = metadata_version("tangle-cli") except PackageNotFoundError: - __version__ = "0.1.30" + __version__ = "0.1.31" __all__ = ["TangleDynamicDiscoveryClient", "__version__"] diff --git a/packages/tangle-cli/src/tangle_cli/pipeline_decompiler.py b/packages/tangle-cli/src/tangle_cli/pipeline_decompiler.py new file mode 100644 index 0000000..f6840f2 --- /dev/null +++ b/packages/tangle-cli/src/tangle_cli/pipeline_decompiler.py @@ -0,0 +1,3200 @@ +"""Dehydrated pipeline YAML → deterministic ``python_pipeline`` source. + +Reverses :mod:`tangle_cli.python_pipeline.emit` for the *dehydrated* shape +only, using the public authoring API. Every recovered value becomes an +``ast`` literal node, never interpolated text, and the result is proven by +recompiling it and comparing semantic digests. + +Fails closed: anything this module cannot represent exactly raises +:class:`DecompileError` with a diagnostic code, and never echoes the value. +""" +from __future__ import annotations + +import ast +import builtins +import contextlib +import copy +import hashlib +import keyword +import os +import re +import shutil +import tempfile +import unicodedata +from collections.abc import Collection, Iterable, Iterator, Mapping, Sequence +from dataclasses import dataclass, field, replace +from pathlib import Path, PurePosixPath +from typing import Any + +from .utils import compute_spec_digest + +# schema_validation pulls in jsonschema, which the "expert, no optional +# deps" install path does not have. This module is reachable from +# pipelines_cli at CLI build time, so the import stays inside the two +# functions that validate -- matching pipelines.py, which defers the same +# import for the same reason. + +__all__ = [ + "DecompileError", + "DecompileResult", + "decompile_pipeline", + "decompile_pipeline_file", +] + +# Header written above generated source. Fixed text only: nothing derived +# from the input document may reach a comment or docstring, because a +# newline in untrusted text would break out of it into executable code. +_HEADER = ( + "Generated by `tangle sdk pipelines decompile`. Do not edit by hand.\n\n" + "Recompiling this file reproduces the source pipeline's dehydrated\n" + "YAML up to semantic identity (sorted-key digest equality), not byte\n" + "identity: comments, key order and task order are not recoverable.\n" +) + +_IMPORTS = ("dynamic_secret", "graph_input", "graph_output", "pipeline", "raw", "ref") + +# Reserved ``CallableRef.__call__`` keywords. A component input that +# genuinely uses one of these names cannot travel through ``**kwargs`` -- +# it would bind the task-metadata parameter instead -- so it is routed +# through ``.bind()``, which keeps it a component argument. +_RESERVED_CALL_KEYWORDS = frozenset( + {"is_enabled", "execution_options", "max_cache_staleness"} +) + +# Locator keys in canonical emission order (emit.py ``_emit_component_ref``). +_LOCATOR_KEYS = ("url", "name", "digest") + +# Execution-option groups/fields the backend actually models (emit.py +# ``_EXECUTION_OPTION_FIELDS``). Anything else is silently dropped server +# side, so it must not survive a round trip unnoticed. +_EXECUTION_OPTION_FIELDS: dict[str, frozenset[str]] = { + "cachingStrategy": frozenset({"maxCacheStaleness"}), + "retryStrategy": frozenset({"maxRetries"}), +} + +_RUNTIME_SENTINEL = "{{" +_COMPILE_TIME_TOKENS = ("{%", "{#") + + +class DecompileError(Exception): + """A dehydrated document this decompiler refuses to represent. + + Carries a stable ``code`` so callers can branch on the refusal reason. + Messages name the construct and its location; they never echo the + offending value, which may be a secret or a config leak. + """ + + def __init__(self, code: str, message: str) -> None: + super().__init__(f"[{code}] {message}") + self.code = code + self.message = message + + +@dataclass(frozen=True) +class CanonicalComponents: + """The same pipeline before and after re-pinning its components.""" + + hydrated: dict[str, Any] + pinned: dict[str, Any] + + +@dataclass(frozen=True) +class DecompileResult: + """Outcome of a decompile, including how strongly it was verified.""" + + source: str + pipeline_name: str + task_count: int + verified: bool + normalizations: tuple[str, ...] = () + output_path: Path | None = None + components_dir: Path | None = None + component_files: tuple[str, ...] = () + resolve_config: Path | None = None + # ``_subgraphs.py``: every nested graph, as a Python @pipeline + subgraph_module: Path | None = None + # ``_tasks/``: leaves recovered as Python @task functions + tasks_package: Path | None = None + python_tasks: int = 0 + # Leaves carrying the generator markers, converted or not + python_candidates: int = 0 + # Converted leaves whose image compile resolves from ``image_id``, and + # whose inputs ``@task(unwrap=...)`` derives from the call site + image_id_tasks: int = 0 + unwrap_tasks: int = 0 + # Leaves referenced by their published digest alone (no local copy) + digest_refs: int = 0 + + +# --------------------------------------------------------------------------- +# Public entry points + + +def decompile_pipeline( + data: Mapping[str, Any], + *, + verify: bool = True, +) -> DecompileResult: + """Return deterministic Python source reproducing ``data``. + + Args: + data: A dehydrated pipeline dict (the shape ``pipelines compile`` + writes). Hydrated documents are refused. + verify: Recompile the generated source in a temporary directory + and require semantic-digest equality with ``data``. + + Raises: + DecompileError: on any construct outside the supported subset, or + when verification does not reproduce the input. + """ + document = _require_mapping(data, "pipeline document") + exempt_paths = assert_dehydrated_input(document) + + plan = _plan(document, exempt_paths) + source = _render(plan) + _assert_generated_source_is_safe(source) + + verified = False + if verify: + _verify_round_trip(source, document, plan.pipeline_name) + verified = True + + return DecompileResult( + source=source, + pipeline_name=plan.pipeline_name, + task_count=len(plan.tasks), + verified=verified, + normalizations=plan.normalizations, + ) + + +# A nested-graph boundary, as it appears in its parent's per-graph document. +# ``file://`` keeps the document schema-valid; every real leaf ref is a +# ``resolve://`` sidecar ref by the time codegen runs, so this cannot collide. +_SUBPIPELINE_MARKER = "file://./__decompiled_subpipeline__/" + + +@dataclass +class _GraphUnit: + """One graph to emit as a ``@pipeline`` function.""" + + function_name: str + document: dict[str, Any] + # task id -> child unit's function name, for each nested-graph boundary + calls: dict[str, str] + + +def _split_graphs(pinned: Mapping[str, Any], root_function: str) -> tuple[_GraphUnit, list[_GraphUnit]]: + """Give every nested graph, at any depth, its own unit -- deepest first. + + A boundary is replaced by a marker naming its child unit, so each unit is + an ordinary per-graph document to plan and verify on its own. Identical + graphs share one unit; a name already taken by different content gets a + short content-digest suffix, so naming is deterministic. + """ + by_digest: dict[str, _GraphUnit] = {} + children: list[_GraphUnit] = [] + taken = {root_function} + + def visit(spec: Mapping[str, Any]) -> tuple[dict[str, Any], dict[str, str]]: + document = copy.deepcopy(dict(spec)) + calls: dict[str, str] = {} + tasks = document.get("implementation", {}).get("graph", {}).get("tasks") or {} + for task_id, body in tasks.items(): + ref = body.get("componentRef") if isinstance(body, Mapping) else None + if not isinstance(ref, Mapping) or not _is_graph_spec(ref.get("spec")): + continue + for key in ("isEnabled", "executionOptions"): + if key in body: + raise DecompileError( + "unsupported-subgraph-task", + f"task {task_id} sets {key} on a nested graph. Tangle does " + "not apply it to graph-component tasks, so it has no " + "subpipeline(...) equivalent.", + ) + child = unit(ref["spec"]) + body["componentRef"] = {"url": f"{_SUBPIPELINE_MARKER}{child.function_name}.yaml"} + calls[str(task_id)] = child.function_name + return document, calls + + def unit(spec: Mapping[str, Any]) -> _GraphUnit: + document, calls = visit(spec) + digest = compute_spec_digest(document) + if digest in by_digest: + return by_digest[digest] + name = _allocate_function_name(str(document.get("name") or ""), digest, taken) + by_digest[digest] = made = _GraphUnit(name, document, calls) + children.append(made) + return made + + document, calls = visit(pinned) + return _GraphUnit(root_function, document, calls), children + + +# Names a generated child function may never take: every authoring helper the +# module imports or an author would expect to (so a later hand edit cannot be +# shadowed either), and the prefixes of the locals inside every function body. +_RESERVED_GLOBAL_NAMES = frozenset({*_IMPORTS, "subpipeline", "task", "In", "Out"}) +_LOCAL_PREFIXES = ("input_", "task_") + + +def _allocate_function_name(raw_name: str, digest: str, taken: set[str]) -> str: + """The one allocator for generated child ``@pipeline`` function names. + + Keywords and builtins are already avoided by ``_python_identifier``. A name + clashing with a helper or a local namespace is moved to ``graph_*``, and + uniqueness is checked in a loop: another graph may literally be NAMED like + a suffixed candidate. Deterministic, and independent of verification, so + ``--no-verify`` can never emit a definition that silently replaces another. + """ + base = _python_identifier(raw_name, prefix="subgraph") + if base.startswith(_LOCAL_PREFIXES) or base in _RESERVED_GLOBAL_NAMES: + base = f"graph_{base}" + name, attempt = base, 1 + while name in taken or name in _RESERVED_GLOBAL_NAMES: + name = f"{base}_{digest[:8]}" if attempt == 1 else f"{base}_{digest[:8]}_{attempt}" + attempt += 1 + taken.add(name) + return name + + +def _companion_module_name(output: Path) -> str | None: + """``_subgraphs``, or None when that is not an importable module name. + + Deliberately not sanitized: mapping ``foo-bar`` and ``foo_bar`` to one + companion would let either output silently replace the other's. + """ + name = f"{output.stem}_subgraphs" + # ASCII only: a non-ASCII module name can fail to import where the + # filesystem stores a different Unicode normalization of the file name. + return name if name.isascii() and name.isidentifier() and not keyword.iskeyword(name) else None + + +# --------------------------------------------------------------------------- +# Python task recovery: a static source transform. Nothing recovered is ever +# imported, executed, regenerated, compiled or hydrated here; Tangle does that +# later, when the maintainer compiles the generated script. + +# Markers a leaf must carry, verbatim, to be recognised as generator output. +_PYTHON_MARKERS = {"cloud_pipelines.net": "true", "components new regenerate python-function-component": "true"} +# Annotations the generator writes itself: the markers this recovery consumes, +# and provenance Tangle re-derives from wherever the source is compiled. +_GENERATOR_ANNOTATIONS = frozenset({ + *_PYTHON_MARKERS, + "python_original_code", "python_original_code_path", "python_dependencies", + "tangle_cli_generation_function_name", "tangle_cli_generation_mode", + "tangle_cli_generation_dependencies_from", "component_yaml_path", + "git_relative_dir", "git_local_branch", "git_local_sha", + "git_remote_url", "git_remote_branch", "git_remote_sha", +}) + + +@dataclass(frozen=True) +class _PythonTask: + """What ``@task`` needs to let Tangle rebuild one leaf from its own source.""" + + function: str + source: str + # The recorded container image; for ``image_id`` sources, only what it + # resolved to then -- compile re-resolves it in the maintainer's environment. + image: str + dependencies: tuple[str, ...] + module_hint: str + # Set when the source carries its own literal ``@task(...)``: it is then + # re-exported as-is, never wrapped a second time. + decorated: bool = False + image_id: str | None = None + dependencies_from: str | None = None + # unwrap parameter -> its synthetic input keys, in spec input order + unwrap: tuple[tuple[str, tuple[str, ...]], ...] = () + + +def _has_python_markers(spec: Any) -> bool: + annotations = _leaf_annotations(spec) + return all(annotations.get(key) == value for key, value in _PYTHON_MARKERS.items()) and "python_original_code" in annotations + + +def _leaf_annotations(spec: Any) -> Mapping[str, Any]: + metadata = spec.get("metadata") if isinstance(spec, Mapping) else None + annotations = metadata.get("annotations") if isinstance(metadata, Mapping) else None + return annotations if isinstance(annotations, Mapping) else {} + + +def _python_task(spec: Any) -> _PythonTask | None: + """The ``@task`` form of a generator-made leaf, or None to keep it YAML. + + Static only, and conservative: anything ``@task`` cannot express (bundle + mode, a resolve root, custom names or annotations, any decorator but one + literal ``@task(...)``) or that cannot be checked from the source text + alone stays a pinned YAML leaf. + """ + import json + + from .component_from_func import _python_name_to_component_name, extract_docstring_metadata + + if not _has_python_markers(spec) or set(spec) - {"name", "description", "metadata", "inputs", "outputs", "implementation"}: + return None + annotations = _leaf_annotations(spec) + source, function = annotations.get("python_original_code"), annotations.get("tangle_cli_generation_function_name") + container = spec.get("implementation", {}).get("container") if isinstance(spec.get("implementation"), Mapping) else None + image = container.get("image") if isinstance(container, Mapping) else None + if ( + set(spec["metadata"]) != {"annotations"} + # Exactly what the generator writes: anything else (env, fileOutputs, + # ...) has no @task form and would be silently lost. + or set(spec["implementation"]) != {"container"} + or not isinstance(container, Mapping) + or set(container) - {"image", "command", "args"} + or not isinstance(source, str) + or not isinstance(function, str) + or function.startswith("_") + or annotations.get("tangle_cli_generation_mode") != "inline" + or not isinstance(image, str) + or not image + ): + return None + try: + dependencies = json.loads(annotations.get("python_dependencies", "[]")) + tree = ast.parse(source) + except (ValueError, SyntaxError, TypeError): + return None + if not isinstance(dependencies, list) or not all(isinstance(d, str) for d in dependencies): + return None + bound = [node for name, node in _module_scope_bindings(tree.body) if name == function] + if len(bound) != 1 or bound[0] not in tree.body or not isinstance(bound[0], ast.FunctionDef): + return None + # A relative import would resolve against the generated package instead. + if _rebinds_dynamically(tree) or any(isinstance(n, ast.ImportFrom) and n.level for n in ast.walk(tree)): + return None + node = bound[0] + config = _task_decorator(tree, node) if node.decorator_list else {"image": image} + if config is None or not _task_config_is_supported(config, image): + return None + # The generator's own derivations, from the same text it would read. + docstring = ast.get_docstring(node) or "" + metadata = extract_docstring_metadata(docstring) if docstring else {} + name = metadata.get("name") or _python_name_to_component_name(function) + if spec.get("name") != name or spec.get("description") != (_docstring_description(docstring) or f"{name} component"): + return None + custom = {k: v for k, v in annotations.items() if k not in _GENERATOR_ANNOTATIONS} + if custom != {k: v for k, v in metadata.items() if k not in ("name", "description")}: + return None + unwrap_names = config.get("unwrap", ()) + unwrap = _interface_matches(node, spec, (unwrap_names,) if isinstance(unwrap_names, str) else tuple(unwrap_names), tree) + if unwrap is None: + return None + hint = PurePosixPath(str(annotations.get("python_original_code_path") or function)).stem + return _PythonTask( + function, + source, + image, + tuple(dependencies), + hint, + decorated=bool(node.decorator_list), + image_id=config.get("image_id"), + dependencies_from=config.get("dependencies_from"), + unwrap=tuple(unwrap.items()), + ) + + +# The literal ``@task(...)`` settings that map statically. Anything else (env, +# annotations, resolve_root, ...) keeps the leaf YAML. +_TASK_DECORATOR_KEYWORDS = frozenset({"image", "image_id", "mode", "dependencies_from", "unwrap"}) +_UNWRAP_KEY = re.compile(r"[A-Za-z0-9_-]+") + + +def _task_decorator(tree: ast.Module, node: ast.FunctionDef) -> dict[str, Any] | None: + """The literal keywords of the function's one ``@task(...)``, or None. + + Only the exact authoring form: a single call of ``task``, bound once, by a + top-level unaliased ``from tangle_cli.python_pipeline import task``. + """ + if len(node.decorator_list) != 1: + return None + decorator = node.decorator_list[0] + if not (isinstance(decorator, ast.Call) and isinstance(decorator.func, ast.Name) and decorator.func.id == "task" and not decorator.args): + return None + bindings = [n for name, n in _module_scope_bindings(tree.body) if name == "task"] + if not ( + len(bindings) == 1 + and bindings[0] in tree.body + and isinstance(bindings[0], ast.ImportFrom) + and bindings[0].module == "tangle_cli.python_pipeline" + and bindings[0].level == 0 + # The very alias that binds ``task`` must import ``task``: one import + # may also hold ``task as other, ref as task``. + and any(alias.name == "task" and (alias.asname or alias.name) == "task" for alias in bindings[0].names) + ): + return None + config: dict[str, Any] = {} + for keyword_ in decorator.keywords: + if keyword_.arg not in _TASK_DECORATOR_KEYWORDS or keyword_.arg in config: + return None + try: + config[keyword_.arg] = ast.literal_eval(keyword_.value) + except (ValueError, TypeError, SyntaxError, MemoryError, RecursionError): + return None + return config + + +def _task_config_is_supported(config: Mapping[str, Any], recorded_image: str) -> bool: + """``image`` must equal the recorded one; ``image_id`` is kept as source + intent (compile resolves it). Mode inline, dependencies a safe relative + ``.toml`` path, unwrap a set of names.""" + image, image_id = config.get("image"), config.get("image_id") + if (image is None) == (image_id is None): + return False + if image is not None and image != recorded_image: + return False + if image_id is not None and not (isinstance(image_id, str) and image_id): + return False + if config.get("mode", "inline") != "inline": + return False + dependencies_from = config.get("dependencies_from") + if dependencies_from is not None and not _safe_relative_toml(dependencies_from): + return False + unwrap = config.get("unwrap", ()) + unwrap = (unwrap,) if isinstance(unwrap, str) else unwrap + return ( + isinstance(unwrap, (list, tuple)) + and all(isinstance(u, str) and u and u not in _RESERVED_CALL_KEYWORDS for u in unwrap) + and len(set(unwrap)) == len(unwrap) + ) + + +def _safe_relative_toml(value: Any) -> bool: + if not isinstance(value, str) or not value or "\\" in value: + return False + path = PurePosixPath(value) + # A ``*.py`` directory would collide with the module directory's own + # ``__init__.py`` / ``.py`` files. + return ( + not path.is_absolute() + and bool(path.parts) + and ".." not in path.parts + and path.suffix == ".toml" + # casefold: a case-insensitive filesystem treats __INIT__.PY as __init__.py + and not any(part.casefold().endswith(".py") for part in path.parts[:-1]) + ) + + +def _module_scope_bindings(statements: Sequence[ast.stmt]) -> Iterator[tuple[str, ast.AST]]: + """Every module-scope binding, through if/try/for/with/match bodies, but + not into function or class scopes. Over-approximate on purpose: a name + bound inside a comprehension is counted too, and only costs a conversion.""" + for statement in statements: + if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + yield statement.name, statement + # Everything but the body is evaluated at module scope: decorators, + # bases and keywords, parameter defaults and annotations, returns. + for name, value in ast.iter_fields(statement): + if name != "body": + for child in value if isinstance(value, list) else [value]: + if isinstance(child, ast.AST): + yield from _expression_bindings(child, statement) + continue + if isinstance(statement, (ast.Import, ast.ImportFrom)): + for alias in statement.names: + yield (alias.asname or alias.name).split(".")[0], statement + continue + for child in ast.iter_child_nodes(statement): + if isinstance(child, ast.stmt): + yield from _module_scope_bindings([child]) + elif isinstance(child, ast.ExceptHandler): + if child.name: + yield child.name, statement + if child.type: + yield from _expression_bindings(child.type, statement) + yield from _module_scope_bindings(child.body) + elif isinstance(child, ast.match_case): + yield from _expression_bindings(child.pattern, statement) + if child.guard: + yield from _expression_bindings(child.guard, statement) + yield from _module_scope_bindings(child.body) + else: + yield from _expression_bindings(child, statement) + + +def _expression_bindings(node: ast.AST, owner: ast.AST) -> Iterator[tuple[str, ast.AST]]: + for child in ast.walk(node): + if isinstance(child, ast.Name) and isinstance(child.ctx, (ast.Store, ast.Del)): + yield child.id, owner + elif isinstance(child, (ast.MatchAs, ast.MatchStar)) and child.name: + yield child.name, owner + elif isinstance(child, ast.MatchMapping) and child.rest: + yield child.rest, owner + + +# Ways to rebind a module global without a binding statement. Seeing any of +# them anywhere means the static contract cannot hold. +_DYNAMIC_REBINDING = frozenset({"globals", "vars", "exec", "eval", "setattr", "delattr", "__import__", "__dict__", "modules"}) + + +def _rebinds_dynamically(tree: ast.AST) -> bool: + return any( + (isinstance(n, ast.Name) and n.id in _DYNAMIC_REBINDING) + or (isinstance(n, ast.Attribute) and n.attr in _DYNAMIC_REBINDING) + or isinstance(n, (ast.Global, ast.Nonlocal)) + # Binds names only importing the module could list. + or (isinstance(n, ast.ImportFrom) and any(alias.name == "*" for alias in n.names)) + for n in ast.walk(tree) + ) + + +def _docstring_description(docstring: str) -> str | None: + """The generator's description: the docstring summary, without Metadata.""" + import docstring_parser + + description = docstring_parser.parse(docstring).description + if not description: + return None + lines = [] + for line in description.split("\n"): + if line.strip().lower() == "metadata:": + break + lines.append(line) + return "\n".join(lines).strip() or None + + +def _interface_matches( + node: ast.FunctionDef, spec: Mapping[str, Any], unwrap: Sequence[str] = (), tree: ast.Module | None = None +) -> dict[str, tuple[str, ...]] | None: + """Unwrap parameter -> synthetic keys when every input/output is accounted + for, else None. + + Names only: types come from annotations only evaluating the source could + resolve. A fixed parameter names exactly one input/output; every other + input must be exactly one unwrap parameter's ``__`` (keys as + ``@task`` allows). Outputs no parameter names are allowed only for a + function with a return annotation, which the generator turns into outputs. + """ + arguments = node.args + if arguments.vararg or arguments.kwarg: + return None + names = [] + # The keys the generator writes per input/output; any other (annotations, + # ...) has no @task form and would be lost. + for kind, written in (("inputs", {"name", "type", "description", "default", "optional"}), ("outputs", {"name", "type", "description"})): + entries = spec.get(kind) or [] + if not isinstance(entries, list) or not all( + isinstance(e, Mapping) and isinstance(e.get("name"), str) and not set(e) - written for e in entries + ): + return None + names.append([e["name"] for e in entries]) + inputs, outputs = names + if len(set(inputs + outputs)) != len(inputs + outputs): + return None + parameters = {p.arg: p for p in (*arguments.posonlyargs, *arguments.args, *arguments.kwonlyargs)} + for name in unwrap: + # Unwrap needs a dict parameter, and flattening leaves no input of its + # own (an output may share its name: another parameter's ``_path``). + annotation = parameters[name].annotation if name in parameters else None + if name not in parameters or name in inputs or not _is_dict_annotation(annotation, tree): + return None + claimed: set[str] = set() + for parameter in parameters: + if parameter in unwrap: + continue + candidates = {parameter, *(parameter[: -len(s)] for s in ("_path", "_file") if parameter.endswith(s))} + hits = candidates & set(inputs + outputs) + if len(hits) != 1 or hits & claimed: + return None + claimed |= hits + synthetic: dict[str, list[str]] = {name: [] for name in unwrap} + for name in inputs: + owners = [u for u in unwrap if name.startswith(f"{u}__") and _UNWRAP_KEY.fullmatch(name[len(u) + 2:])] + if name in claimed: + if owners: + return None # a fixed input that also reads as synthetic is ambiguous + continue + if len(owners) != 1: + return None + synthetic[owners[0]].append(name[len(owners[0]) + 2:]) + if any(not keys for keys in synthetic.values()): + return None # @task refuses an empty unwrap dict + if not (set(outputs) <= claimed or node.returns is not None): + return None + return {name: tuple(keys) for name, keys in synthetic.items()} + + +def _is_dict_annotation(annotation: ast.expr | None, tree: ast.Module | None) -> bool: + """None, ``dict`` / ``dict[...]``, or ``Dict`` / ``Dict[...]`` from a + top-level ``from typing import Dict`` -- each name provably unrebound.""" + if annotation is None: + return True + base = annotation.value if isinstance(annotation, ast.Subscript) else annotation + if not isinstance(base, ast.Name) or base.id not in ("dict", "Dict"): + return False + bindings = [n for name, n in _module_scope_bindings(tree.body) if name == base.id] if tree is not None else [] + if base.id == "dict": + return not bindings + return ( + len(bindings) == 1 + and bindings[0] in tree.body # type: ignore[union-attr] + and isinstance(bindings[0], ast.ImportFrom) + and bindings[0].module == "typing" + and bindings[0].level == 0 + and any(alias.name == "Dict" and alias.asname is None for alias in bindings[0].names) + ) + + +def _tasks_package_name(output: Path) -> str | None: + """``_tasks``, under the same one-to-one rule as the companion.""" + name = f"{output.stem}_tasks" + return name if name.isascii() and name.isidentifier() and not keyword.iskeyword(name) else None + + +@dataclass(frozen=True) +class _EmittedTask: + name: str + module: str + task: _PythonTask + + +def _allocate_python_tasks(recovered: Mapping[str, _PythonTask], taken: set[str]) -> dict[str, _EmittedTask]: + """url -> emitted task. One module per distinct (source, function, config). + + Task names share the module namespace with every generated function, via + the same allocator -- and so do the subpackage names: importing + ``.`` binds ```` on the package, which would replace a task + exported under that name. + """ + import json + + by_key: dict[str, _EmittedTask] = {} + emitted: dict[str, _EmittedTask] = {} + for url in sorted(recovered): + task = recovered[url] + key = json.dumps([ + task.source, task.function, task.image, task.dependencies, + task.decorated, task.image_id, task.dependencies_from, task.unwrap, + ]) + if key not in by_key: + digest = hashlib.sha256(key.encode("utf-8")).hexdigest() + by_key[key] = _EmittedTask( + _allocate_function_name(task.function, digest, taken), + _allocate_function_name(task.module_hint, digest, taken), + task, + ) + emitted[url] = by_key[key] + return emitted + + +_TASKS_HEADER = ( + "Generated by `tangle sdk pipelines decompile`.\n\n" + "Each subpackage holds python_original_code recovered byte-for-byte from\n" + "the input pipeline YAML. The decompiler never imported or ran it;\n" + "`tangle sdk pipelines compile` imports it, and hydrate/submit/run execute\n" + "it to rebuild each component. Review it before compiling.\n" +) + + +def _task_module(emitted: _EmittedTask) -> str: + """``.``: each recovered module has its own directory, so a + source's own relative ``dependencies_from`` cannot collide with another's.""" + return f"{emitted.module}.{emitted.module}" + + +def _dependencies_path(emitted: _EmittedTask) -> str: + """Where the recorded dependencies go, relative to the module's directory. + + A wrapped function names it explicitly; a decorated source's own literal + ``dependencies_from`` is honoured; with none, ``.toml`` is the file + the generator's own discovery picks first, pinning the recorded list + rather than whatever pyproject.toml it would otherwise find. + """ + if not emitted.task.decorated: + return f"{emitted.module}.deps.toml" + return emitted.task.dependencies_from or f"{emitted.module}.toml" + + +def _render_tasks_package(tasks: Sequence[_EmittedTask]) -> str: + """``__init__.py``: a decorated source is re-exported as it is; anything + else gets one ``task(...)`` wrapper.""" + body: list[ast.stmt] = [] + if any(not emitted.task.decorated for emitted in tasks): + body.append(ast.ImportFrom(module="tangle_cli.python_pipeline", names=[ast.alias(name="task", asname=None)], level=0)) + wrappers: list[ast.stmt] = [] + for index, emitted in enumerate(tasks): + alias = emitted.name if emitted.task.decorated else f"_recovered_{index}" + body.append(ast.ImportFrom(module=_task_module(emitted), names=[ast.alias(name=emitted.task.function, asname=alias)], level=1)) + if emitted.task.decorated: + continue + keywords = [ + ast.keyword(arg="image", value=_literal(emitted.task.image)), + ast.keyword(arg="mode", value=_literal("inline")), + # Always, even when empty: pins "no dependencies" rather than let + # the generator discover some unrelated pyproject.toml. + ast.keyword(arg="dependencies_from", value=_literal(_dependencies_path(emitted))), + ] + wrappers.append( + ast.Assign( + targets=[ast.Name(id=emitted.name, ctx=ast.Store())], + value=ast.Call( + func=ast.Call(func=ast.Name(id="task", ctx=ast.Load()), args=[], keywords=keywords), + args=[ast.Name(id=alias, ctx=ast.Load())], + keywords=[], + ), + type_comment=None, + ) + ) + module = ast.Module(body=[*body, *wrappers], type_ignores=[]) + ast.fix_missing_locations(module) + return f'"""{_TASKS_HEADER}"""\n{ast.unparse(module)}\n' + + +def _dependencies_toml(dependencies: Sequence[str]) -> str: + return "[project]\ndependencies = [" + ", ".join(_toml_string(d) for d in dependencies) + "]\n" + + +def _toml_string(value: str) -> str: + """A TOML basic string. Not ``json.dumps``: its surrogate-pair escapes for + non-BMP characters are not Unicode scalar values, which TOML rejects.""" + escaped = [] + for character in value: + if character in '"\\': + escaped.append("\\" + character) + elif ord(character) < 0x20 or ord(character) == 0x7F: + escaped.append(f"\\u{ord(character):04X}") + else: + escaped.append(character) + return '"' + "".join(escaped) + '"' + + +def _write_tasks_package(directory: Path, tasks: Sequence[_EmittedTask]) -> None: + directory.mkdir() + (directory / "__init__.py").write_text(_render_tasks_package(tasks), encoding="utf-8") + for emitted in tasks: + module = directory / emitted.module + module.mkdir() + (module / "__init__.py").write_text("", encoding="utf-8") + (module / f"{emitted.module}.py").write_bytes(emitted.task.source.encode("utf-8")) + dependencies = module / _dependencies_path(emitted) + dependencies.parent.mkdir(parents=True, exist_ok=True) + dependencies.write_text(_dependencies_toml(emitted.task.dependencies), encoding="utf-8") + + +def _assert_tasks_package(directory: Path, tasks: Sequence[_EmittedTask]) -> None: + """Static checks of what was written: text, imports, wrappers, dependencies.""" + try: + import tomllib + except ModuleNotFoundError: # pragma: no cover - Python 3.10 + import tomli as tomllib # type: ignore[no-redef] + + modules = {_task_module(emitted) for emitted in tasks} + tree = ast.parse((directory / "__init__.py").read_text(encoding="utf-8")) + imported = {} + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom) and not ( + (node.module == "tangle_cli.python_pipeline" and node.level == 0) or (node.module in modules and node.level == 1) + ): + raise DecompileError("codegen-unsafe-node", "the generated tasks package imports an unexpected module.") + if not isinstance(node, _ALLOWED_NODES): + raise DecompileError("codegen-unsafe-node", f"the generated tasks package contains an unexpected {type(node).__name__} node.") + if isinstance(node, ast.ImportFrom) and node.level == 1: + imported[node.names[0].asname] = (node.module, node.names[0].name) + wrapped = { + node.targets[0].id: (imported[node.value.args[0].id], {k.arg: ast.literal_eval(k.value) for k in node.value.func.keywords}) + for node in tree.body + if isinstance(node, ast.Assign) + } + for emitted in tasks: + task = emitted.task + origin = (_task_module(emitted), task.function) + if task.decorated: + ok = imported.get(emitted.name) == origin and emitted.name not in wrapped + else: + ok = wrapped.get(emitted.name) == (origin, {"image": task.image, "mode": "inline", "dependencies_from": _dependencies_path(emitted)}) + module = directory / emitted.module + source = (module / f"{emitted.module}.py").read_bytes() + dependencies = tomllib.loads((module / _dependencies_path(emitted)).read_text(encoding="utf-8")) + if ( + not ok + or (module / "__init__.py").read_bytes() + or source != task.source.encode("utf-8") + or dependencies.get("project", {}).get("dependencies") != list(task.dependencies) + or not any(isinstance(n, ast.FunctionDef) and n.name == task.function for n in ast.parse(source).body) + ): + raise DecompileError( + "verification-invalid-output", + "the generated tasks package does not carry exactly the recorded " + "source, function, image and dependencies.", + ) + + +def _decompile_units( + root: _GraphUnit, + children: list[_GraphUnit], + companion: str, + *, + verify: bool, + python_tasks: Mapping[str, _EmittedTask] | None = None, + tasks_package: str | None = None, + digest_refs: Mapping[str, str] | None = None, +) -> tuple[DecompileResult, str | None]: + """Render the root module and, when there are nested graphs, its companion. + + Gate (b) verifies the all-YAML-leaf form. Recovered Python tasks are then + substituted, and the result is checked statically to differ from that + verified form only at those call sites: nothing recovered is ever run here. + """ + plans = [] + for graph in (root, *children): + plan = _plan(graph.document, assert_dehydrated_input(graph.document)) + plan.function_name = graph.function_name + plan.subpipelines = dict(graph.calls) + plans.append(plan) + root_plan, child_plans = plans[0], plans[1:] + owner = companion if child_plans else None + + def render(package: str | None) -> tuple[str, str | None]: + companion_source = _render_module(child_plans, tasks_package=package) if child_plans else None + root_source = _render_module([root_plan], owner, tuple(sorted(set(root.calls.values()))), package) + _assert_generated_source_is_safe(root_source, owner, package) + if companion_source is not None: + _assert_generated_source_is_safe(companion_source, package) + return root_source, companion_source + + root_source, companion_source = render(None) + if verify: + _verify_units(root_source, companion_source, companion, [root, *children]) + if python_tasks or digest_refs: + python_tasks = python_tasks or {} + names = {url: emitted.name for url, emitted in python_tasks.items()} + unwraps = {url: dict(emitted.task.unwrap) for url, emitted in python_tasks.items() if emitted.task.unwrap} + for plan in plans: + plan.python_tasks = names + plan.python_unwrap = unwraps + plan.digest_refs = dict(digest_refs or {}) + verified = (root_source, companion_source) + root_source, companion_source = render(tasks_package if python_tasks else None) + unwrap_by_name = {names[url]: frozenset(unwrap) for url, unwrap in unwraps.items()} + for before, after in zip(verified, (root_source, companion_source)): + if before is not None: + _assert_only_task_substitutions( + before, after or "", names, tasks_package or "", unwrap_by_name, digest_refs or {} + ) + return ( + DecompileResult( + source=root_source, + pipeline_name=root_plan.pipeline_name, + task_count=len(root_plan.tasks), + verified=verify, + normalizations=tuple(dict.fromkeys(note for plan in plans for note in plan.normalizations)), + ), + companion_source, + ) + + +def _verify_units(root_source: str, companion_source: str | None, companion: str, units: list[_GraphUnit]) -> None: + """Gate (b) per graph: each function must trace back to exactly its unit. + + A boundary traces as a ``subpipeline://pending`` task; it is mapped back to + the marker naming the child it actually calls, so calling the wrong child + fails the comparison too. The root imports its companion by NAME, which + caches it in ``sys.modules``; it is evicted afterwards, or a later run in + this process would import this run's stale companion instead of its own. + """ + import sys + + with tempfile.TemporaryDirectory() as tmp: + root_path = Path(tmp) / "decompiled_pipeline.py" + root_path.write_text(root_source, encoding="utf-8") + companion_path = Path(tmp) / f"{companion}.py" + if companion_source is not None: + companion_path.write_text(companion_source, encoding="utf-8") + try: + for index, graph in enumerate(units): + module_path = root_path if index == 0 else companion_path + rebuilt, exempt_paths, calls = _trace_function(module_path, graph.function_name) + tasks = rebuilt.get("implementation", {}).get("graph", {}).get("tasks", {}) + for task_id, child in calls: + if task_id in tasks: + tasks[task_id]["componentRef"] = {"url": f"{_SUBPIPELINE_MARKER}{child}.yaml"} + _compare_rebuilt(rebuilt, exempt_paths, graph.document) + finally: + sys.modules.pop(companion, None) + + +def _assert_only_task_substitutions( + before: str, + after: str, + names: Mapping[str, str], + package: str, + unwrap: Mapping[str, frozenset[str]] | None = None, + digests: Mapping[str, str] | None = None, +) -> None: + """``after`` must be the verified ``before`` with each converted + ``ref(url=...)`` replaced by its task name, plus that task import -- and, + for an unwrap task, its flattened arguments regrouped into dicts, which + are flattened back here before comparing (keyword order aside).""" + unwrap = unwrap or {} + + class Substitute(ast.NodeTransformer): + def visit_Call(self, node: ast.Call) -> ast.AST: + self.generic_visit(node) + url = _ref_url(node) + if url is not None and url in names: + return ast.Name(id=names[url], ctx=ast.Load()) + if url is not None and url in (digests or {}): + return _digest_ref(digests[url]) # type: ignore[index] + return node + + class Flatten(ast.NodeTransformer): + def __init__(self, regroup: bool) -> None: + self.regroup = regroup + + def visit_Call(self, node: ast.Call) -> ast.AST: + self.generic_visit(node) + params = unwrap.get(_call_root(node) or "") + if params is None: + return node + keywords = [] + for item in node.keywords: + if self.regroup and item.arg in params and isinstance(item.value, ast.Dict): + for key, value in zip(item.value.keys, item.value.values): + flat = f"{item.arg}__{key.value}" if isinstance(key, ast.Constant) else "" + keywords.append( + ast.keyword(arg=flat, value=value) + if flat.isidentifier() and not keyword.iskeyword(flat) + else ast.keyword(arg=None, value=ast.Dict(keys=[ast.Constant(value=flat)], values=[value])) + ) + else: + keywords.append(item) + node.keywords = sorted(keywords, key=lambda k: k.arg or ast.dump(k.value)) + return node + + expected = Flatten(regroup=False).visit(Substitute().visit(ast.parse(before))) + actual = ast.parse(after) + actual.body = [n for n in actual.body if not (isinstance(n, ast.ImportFrom) and n.module == package)] + actual = Flatten(regroup=True).visit(actual) + # The header docstring says which components are not proven; nothing else + # may differ. + for tree in (expected, actual): + tree.body = [n for i, n in enumerate(tree.body) if not (i == 0 and isinstance(n, ast.Expr))] + if ast.dump(expected) != ast.dump(actual): + raise DecompileError( + "verification-invalid-output", + "substituting recovered Python tasks changed more than their call sites.", + ) + + +def _digest_ref(digest: str) -> ast.Call: + return ast.Call(func=ast.Name(id="ref", ctx=ast.Load()), args=[], keywords=[ast.keyword(arg="digest", value=_literal(digest))]) + + +def _call_root(node: ast.AST) -> str | None: + """The name a ``name.named(...)...(...)`` call chain starts from.""" + while isinstance(node, (ast.Call, ast.Attribute)): + node = node.func if isinstance(node, ast.Call) else node.value + return node.id if isinstance(node, ast.Name) else None + + +def _unwrap_calls_fit(pinned: Mapping[str, Any], recovered: dict[str, _PythonTask]) -> None: + """Keep an unwrap leaf YAML unless EVERY task using it passes exactly its + synthetic inputs: ``@task`` derives them from the call-site dict, so a + missing, extra or bare one would compile to a different component.""" + + def visit(document: Mapping[str, Any]) -> None: + tasks = document.get("implementation", {}).get("graph", {}).get("tasks") + for body in (tasks or {}).values() if isinstance(tasks, Mapping) else (): + ref = body.get("componentRef") if isinstance(body, Mapping) else None + if not isinstance(ref, Mapping): + continue + if _is_graph_spec(ref.get("spec")): + visit(ref["spec"]) + continue + task = recovered.get(ref.get("url")) + if task is None or not task.unwrap: + continue + arguments = {str(k) for k in (body.get("arguments") or {})} + for name, keys in task.unwrap: + synthetic = {f"{name}__{key}" for key in keys} + if name in arguments or {a for a in arguments if a.startswith(f"{name}__")} != synthetic: + recovered.pop(ref["url"], None) + break + + visit(pinned) + + +def decompile_pipeline_file( + pipeline_path: Path, + output: Path, + *, + verify: bool = True, + client: Any = None, + python_tasks: bool = True, + include_local_fallbacks: bool = False, +) -> DecompileResult: + """Decompile ``pipeline_path`` and write the source to ``output``. + + Accepts hydrated or dehydrated pipeline YAML. Every graph becomes Python: + the root in ``output``, each nested graph as a ``@pipeline`` in the + companion ``_subgraphs.py``. Every leaf component is resolved and + re-pinned through the ``.leaves.yaml`` sidecar, with a local copy + in ``.leaves/``. All products are staged and gated before any is + published, and any failure aborts the whole operation. + + With ``python_tasks`` (the default), a leaf the Python component generator + made, whose source is recorded in full, becomes a ``@task`` in + ``_tasks/`` instead: a static source transform that never runs that + source. ``python_tasks=False`` keeps every leaf a digest-pinned YAML leaf. + + A leaf the component library verified is referenced by its published + digest alone, so hydration needs the library; ``include_local_fallbacks`` + also keeps a local copy behind each one, for offline-resilient output. A + leaf with no published digest is always written locally. + + ``output`` is always replaced, and a stale components directory from a + previous run of the same output is replaced wholesale rather than merged. + """ + document = _parse_input(pipeline_path) + if not isinstance(document, Mapping): + raise DecompileError( + "unsupported-document", + f"{pipeline_path} does not contain a YAML mapping.", + ) + + # Classify and validate the ORIGINAL under its own contract, before + # resolution can launder an unsupported shape into a supported one. + assert_supported_input(document) + + # Lexically absolute, never ``resolve()``: resolving would follow a + # symlink at the output path and overwrite its target, where every other + # write here replaces the link itself. + output = Path(os.path.abspath(output)) + # Never ``.components*``: ``compile .py -o .yaml`` writes + # its own ``@task`` sidecar there, and would replace this one. An existing + # ``.components*`` artifact is never touched, as it may be the compiler's. + assets_name = f"{output.stem}.leaves" + sidecar_name = f"{output.stem}.leaves.yaml" + companion = _companion_module_name(output) + tasks_package = _tasks_package_name(output) + # Stage the script, its resolve sidecar and its assets in the layout they + # will be published into, so the relative URLs written now are the ones + # that will be correct after publication. + with _staging_directory(output.parent) as stage: + canonical = canonicalize_components( + document, + base_dir=pipeline_path.parent, + components_dir=stage / assets_name, + output_file=stage / output.name, + client=client, + sidecar_name=sidecar_name, + ) + # Gate (a): pinning preserved the pipeline. + assert_pinning_preserved( + canonical, script_dir=stage, sidecar=sidecar_name, client=client + ) + # Every nested graph becomes its own @pipeline in the companion module; + # gate (b) then proves each generated function reproduces its graph. + root, children = _split_graphs( + canonical.pinned, _python_identifier(str(canonical.pinned.get("name") or ""), prefix="pipeline") + ) + if children and companion is None: + raise DecompileError( + "unsupported-output-name", + "this pipeline has nested graphs, which are written to a " + "companion _subgraphs.py that the script imports by name, " + f"so the output name must make a valid Python module name; " + f"{output.name!r} does not.", + ) + recovered, candidates = _recover_python_tasks(canonical) if python_tasks else ({}, _count_python_candidates(canonical)) + # Before the name check: it applies only when something still converts. + _unwrap_calls_fit(canonical.pinned, recovered) + if recovered and tasks_package is None: + raise DecompileError( + "unsupported-output-name", + "this pipeline has Python components, which are written to a " + "_tasks package that the script imports by name, so the " + f"output name must make a valid Python module name; {output.name!r} " + "does not. Rename the output, or pass --no-python-tasks.", + ) + emitted = _allocate_python_tasks(recovered, {root.function_name, *(c.function_name for c in children)}) + # After both gates, which ran on the portable form: every digest here + # was verified online and every copy offline. Dropping the copy is a + # static substitution, checked like the Python tasks. + digests = {} if include_local_fallbacks else _verified_digests(stage / sidecar_name, sidecar_name, set(emitted)) + result, companion_source = _decompile_units( + root, children, companion or "", verify=verify, python_tasks=emitted, tasks_package=tasks_package, + digest_refs=digests, + ) + + staged_script = stage / output.name + staged_script.write_text(result.source, encoding="utf-8") + if companion_source is not None: + (stage / f"{companion}.py").write_text(companion_source, encoding="utf-8") + if emitted: + distinct = list({e.name: e for e in emitted.values()}.values()) + _write_tasks_package(stage / str(tasks_package), distinct) + _assert_tasks_package(stage / str(tasks_package), distinct) + if emitted or digests: + _prune_sidecar(stage, sidecar_name, set(emitted) | set(digests)) + _assert_remaining_leaves_resolve(stage, sidecar_name, [result.source, companion_source or ""]) + staged_assets = stage / assets_name + # Recursive: subgraphs are extracted into their own subdirectory. + files = ( + tuple( + sorted( + p.relative_to(staged_assets).as_posix() + for p in staged_assets.rglob("*") + if p.is_file() + ) + ) + if staged_assets.is_dir() + else () + ) + + published = _publish( + staged_script, + output, + [ + # Listed even when unused, so a stale companion is removed. + *([(stage / f"{companion}.py", output.parent / f"{companion}.py")] if companion else []), + *([(stage / tasks_package, output.parent / tasks_package)] if tasks_package else []), + (stage / sidecar_name, output.parent / sidecar_name), + (staged_assets, output.parent / assets_name), + ], + ) + + return replace( + result, + output_path=output, + components_dir=output.parent / assets_name if output.parent / assets_name in published else None, + component_files=files, + resolve_config=output.parent / sidecar_name if output.parent / sidecar_name in published else None, + subgraph_module=output.parent / f"{companion}.py" if companion and output.parent / f"{companion}.py" in published else None, + tasks_package=output.parent / tasks_package if tasks_package and output.parent / tasks_package in published else None, + python_tasks=len(emitted), + python_candidates=candidates, + image_id_tasks=sum(1 for e in emitted.values() if e.task.image_id is not None), + unwrap_tasks=sum(1 for e in emitted.values() if e.task.unwrap), + digest_refs=len(digests), + ) + + +def _verified_digests(sidecar: Path, sidecar_name: str, converted: set[str]) -> dict[str, str]: + """sidecar URL -> published digest, for every fragment whose only entries + are a library-verified digest and its local copy (the dehydrator writes the + digest only once the library returned that spec).""" + import yaml + + if not sidecar.is_file(): + return {} + digests = {} + for fragment, entries in (yaml.safe_load(sidecar.read_text(encoding="utf-8")) or {}).items(): + url = f"resolve://./{sidecar_name}#{fragment}" + if ( + url not in converted + and isinstance(entries, list) + and len(entries) == 2 + and isinstance(entries[0], Mapping) + and set(entries[0]) == {"digest", "fallback_on_error"} + and isinstance(entries[0]["digest"], str) + and isinstance(entries[1], Mapping) + and set(entries[1]) == {"local"} + ): + digests[url] = entries[0]["digest"] + return digests + + +def _count_python_candidates(canonical: CanonicalComponents) -> int: + return len({ref["url"] for _, ref, spec in _paired_leaves(canonical) if _has_python_markers(spec)}) + + +def _recover_python_tasks(canonical: CanonicalComponents) -> tuple[dict[str, _PythonTask], int]: + """sidecar URL -> recoverable task, and how many leaves carry the markers.""" + recovered: dict[str, _PythonTask] = {} + for _, ref, spec in _paired_leaves(canonical): + task = _python_task(spec) + if task is not None: + recovered[ref["url"]] = task + return recovered, _count_python_candidates(canonical) + + +def _paired_leaves(canonical: CanonicalComponents) -> Iterator[tuple[tuple[str, ...], Mapping[str, Any], Any]]: + """Each pinned leaf ref with the resolved spec it was pinned from.""" + specs = {path: ref.get("spec") for path, ref in _pinned_locators(canonical.hydrated)} + for path, ref in _pinned_locators(canonical.pinned): + if isinstance(ref.get("url"), str): + yield path, ref, specs.get(path) + + +def _prune_sidecar(stage: Path, sidecar_name: str, converted: set[str]) -> None: + """Drop the fragments, and local copies, only converted leaves used.""" + import yaml + + from .utils import dump_yaml + + prefix = f"resolve://./{sidecar_name}#" + sidecar = stage / sidecar_name + manifest = yaml.safe_load(sidecar.read_text(encoding="utf-8")) or {} + kept = {k: v for k, v in manifest.items() if f"{prefix}{k}" not in converted} + local = { + str(PurePosixPath(entry["local"])) + for entries in kept.values() + for entry in entries + if isinstance(entry, Mapping) and isinstance(entry.get("local"), str) + } + assets = stage / sidecar_name[: -len(".yaml")] + for path in sorted(assets.rglob("*"), reverse=True) if assets.is_dir() else (): + if path.is_file() and path.relative_to(stage).as_posix() not in local: + path.unlink() + elif path.is_dir() and not any(path.iterdir()): + path.rmdir() + if kept: + sidecar.write_text(dump_yaml(kept), encoding="utf-8") + else: + sidecar.unlink() + + +def _ref_url(node: ast.AST) -> str | None: + """The literal URL of a generated ``ref(url=...)`` call, else None.""" + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "ref" + and not node.args + and len(node.keywords) == 1 + and node.keywords[0].arg == "url" + and isinstance(node.keywords[0].value, ast.Constant) + and isinstance(node.keywords[0].value.value, str) + ): + return node.keywords[0].value.value + return None + + +def _assert_remaining_leaves_resolve(stage: Path, sidecar_name: str, sources: Sequence[str]) -> None: + """Every YAML leaf left in the generated code names a fragment that is + still defined, whose local copies are all still there.""" + import yaml + + prefix = f"resolve://./{sidecar_name}#" + sidecar = stage / sidecar_name + manifest = (yaml.safe_load(sidecar.read_text(encoding="utf-8")) or {}) if sidecar.is_file() else {} + for source in sources: + for node in ast.walk(ast.parse(source)) if source else (): + # Only generated ``ref(url=...)`` locators: the same text in an + # argument, default or annotation is user data, not a reference. + url = _ref_url(node) + if url is not None and url.startswith(prefix): + entries = manifest.get(url[len(prefix):]) + if not entries or not all( + (stage / entry["local"]).is_file() for entry in entries if isinstance(entry, Mapping) and "local" in entry + ): + raise DecompileError( + "pinning-incomplete", + "removing the converted Python components from the sidecar " + "left a YAML leaf without its pinned copy.", + ) + + +@contextlib.contextmanager +def _staging_directory(parent: Path) -> Iterator[Path]: + """A scratch directory beside the destination, always cleaned up. + + Beside the destination rather than in the system temp area so that + publication is a rename within one filesystem. + """ + parent.mkdir(parents=True, exist_ok=True) + stage = Path(tempfile.mkdtemp(prefix=".decompile-", dir=parent)) + try: + yield stage + finally: + shutil.rmtree(stage, ignore_errors=True) + + +def _publish( + staged_script: Path, script: Path, sidecars: list[tuple[Path, Path]] +) -> list[Path]: + """Move the staged script and its sidecars into place as one transaction. + + POSIX offers no multi-path atomic rename, so the guarantee is that + everything fallible -- resolution, pinning, codegen and both gates -- has + already happened, and that a rename which fails anyway is rolled back + across EVERY product: the previous script keeps the resolve config and + components it was published with. Previous sidecars are moved aside, not + deleted, until the script rename -- always last -- has succeeded; one the + new run no longer needs is removed then, never merged into. + + ``sidecars`` are ``(staged, destination)`` pairs, files or directories. + Returns the destinations actually published. + """ + sidelined: list[tuple[Path, Path]] = [] + placed: list[tuple[Path, Path]] = [] + try: + for staged, destination in sidecars: + if destination.exists() or destination.is_symlink(): + aside = destination.with_name(f"{destination.name}.replaced-{os.getpid()}") + os.replace(destination, aside) + sidelined.append((destination, aside)) + wanted = staged.is_file() or (staged.is_dir() and any(staged.iterdir())) + if wanted: + os.replace(staged, destination) + placed.append((staged, destination)) + os.replace(staged_script, script) + except OSError: + # Undo newest-first so a failed publication leaves the previous + # generation together, not mismatched. + for staged, destination in reversed(placed): + os.replace(destination, staged) + for destination, aside in reversed(sidelined): + os.replace(aside, destination) + raise + + for _, aside in sidelined: + if aside.is_dir() and not aside.is_symlink(): + shutil.rmtree(aside, ignore_errors=True) + else: + with contextlib.suppress(OSError): + aside.unlink() + return [destination for _, destination in placed] + + +def _parse_input(pipeline_path: Path) -> Any: + """Read and parse the input, reporting position instead of content. + + A YAML error renders the offending token into its message, and nothing + in an unparsed file has been validated yet. ``utils.parse_yaml_string`` + is bypassed deliberately: it prints the offending source line and then + returns ``None``, which would both leak and lose the position. + """ + import yaml + + try: + return yaml.safe_load(pipeline_path.read_text(encoding="utf-8")) + except Exception as exc: + mark = getattr(exc, "problem_mark", None) + where = ( + f" at line {mark.line + 1}, column {mark.column + 1}" + if mark is not None + else "" + ) + raise DecompileError( + "unreadable-input", + f"{pipeline_path} could not be read as YAML{where}.", + ) from exc + + +def assert_supported_input(data: Mapping[str, Any]) -> bool: + """Validate a hydrated or dehydrated input under its own contract. + + Returns True when the document is hydrated. Classification happens before + resolution because resolution would otherwise launder an unsupported + shape -- inlining a spec that was already inline, or dropping a ref's + sibling keys -- into something a later check would accept. + """ + document = _require_mapping(data, "pipeline document") + _refuse_inline_component_text(document) + if _has_inline_component_spec(document): + _assert_valid_hydrated_input(document) + return True + assert_dehydrated_input(document) + return False + + +def _iter_component_refs(document: Mapping[str, Any]) -> Iterator[tuple[str, Mapping[str, Any]]]: + """Yield ``(path, componentRef)`` for every task at any graph depth.""" + + def visit(tasks: Any, prefix: str) -> Iterator[tuple[str, Mapping[str, Any]]]: + if not isinstance(tasks, Mapping): + return + for task_id, body in tasks.items(): + if not isinstance(body, Mapping): + continue + where = f"{prefix}{task_id}" + ref = body.get("componentRef") + if isinstance(ref, Mapping): + yield where, ref + spec = ref.get("spec") + if isinstance(spec, Mapping): + nested = spec.get("implementation") + if isinstance(nested, Mapping): + graph = nested.get("graph") + if isinstance(graph, Mapping): + yield from visit(graph.get("tasks"), f"{where}.") + + implementation = document.get("implementation") + if isinstance(implementation, Mapping): + graph = implementation.get("graph") + if isinstance(graph, Mapping): + yield from visit(graph.get("tasks"), "") + + +def _has_inline_component_spec(document: Mapping[str, Any]) -> bool: + return any(isinstance(ref.get("spec"), Mapping) for _, ref in _iter_component_refs(document)) + + +def _refuse_inline_component_text(document: Mapping[str, Any]) -> None: + """Refuse ``componentRef.text``, which neither contract can represent. + + The general schema still permits it, and hydration drops it silently, so + accepting it would generate a script missing part of its input. + """ + # Key presence, not truthiness: an empty `text` is still a legacy key the + # generated script cannot represent. + carrying = [where for where, ref in _iter_component_refs(document) if "text" in ref] + if carrying: + raise DecompileError( + "unsupported-document", + "inline componentRef.text is out of scope for this command; it " + "cannot be expressed in Python authoring and would be dropped. " + f"Found at: {', '.join(sorted(carrying))}.", + ) + + +def _assert_valid_hydrated_input(document: Mapping[str, Any]) -> None: + """Validate an inline-spec document against the general pipeline schema. + + Uses structured error paths rather than the formatted messages, which + render the rejected value. + """ + import jsonschema + + from .pipeline_validation import _should_ignore_schema_error, load_pipeline_schema + from .schema_validation import ( + SchemaValidationError, + assert_no_template_delimiters, + ) + + # The general schema permits compile-time templates; this command does + # not. Checked here rather than downstream so an unrendered template + # cannot be laundered away by resolution first. + try: + assert_no_template_delimiters(document, _runtime_sentinel_paths(document)) + except SchemaValidationError as exc: + raise DecompileError( + "unsupported-document", + "input still contains compile-time template delimiters, which " + f"are out of scope for this command. {_redacted_validation_reason(exc)}", + ) from exc + + validator = jsonschema.Draft7Validator(load_pipeline_schema()) + errors = [e for e in validator.iter_errors(document) if not _should_ignore_schema_error(e)] + if not errors: + return + best = min(errors, key=lambda e: list(e.absolute_path)) + location = ".".join(str(p) for p in best.absolute_path) or "root" + raise DecompileError( + "unsupported-document", + "input is not a valid pipeline document. " + f"The document is invalid at {location}.", + ) + + +def assert_dehydrated_input(data: Mapping[str, Any]) -> frozenset[str]: + """Require a dehydrated pipeline document and return its exempt paths. + + Runs before anything else touches the document. Hydration happily inlines + a spec that was already inline and drops sibling keys such as a ref's + literal ``text``, so validating only the canonicalized result would let an + out-of-scope input through and silently discard part of it. + """ + from .schema_validation import SchemaValidationError, validate_dehydrated_pipeline + + document = _require_mapping(data, "pipeline document") + exempt_paths = _runtime_sentinel_paths(document) + try: + validate_dehydrated_pipeline(document, exempt_paths) + except SchemaValidationError as exc: + raise DecompileError( + "unsupported-document", + "input is not a valid dehydrated pipeline document. Hydrated " + "specs, inline component text, legacy keys and compile-time " + "templates are all out of scope for this command. " + # Only the location, never the validator's rendering of the + # offending value: an argument or annotation can hold a secret. + f"{_redacted_validation_reason(exc)}", + ) from exc + return exempt_paths + + +def _redacted_validation_reason(exc: Exception) -> str: + """Location of a schema failure with the offending value removed. + + The validator renders the rejected value into its message, which may be a + secret. The location is read from the exception's structured ``location`` + -- built from the document's own keys -- and the message text is never + parsed: a rejected value shaped like ``" at something: "`` would + otherwise be reported as if it were the path. + """ + location = getattr(exc, "location", None) + if isinstance(location, str) and location: + return f"The document is invalid at {location}." + return "Run `tangle sdk pipelines validate` for the full report." + + +_CREDENTIAL_IN_URL = re.compile(r"(?P[A-Za-z][A-Za-z0-9+.-]*://)[^/\s@]*@") +_URL_QUERY = re.compile(r"(?P[A-Za-z][A-Za-z0-9+.-]*://\S*?)\?\S*") + + +class _QuietLogger: + """Drops upstream progress text instead of forwarding it. + + Hydration and dehydration render document content into their own output + -- locator URLs, and via ``utils.parse_yaml_string`` the offending source + line of a malformed component -- which this command must not echo. The + chosen diagnostics carry everything the reader needs. + """ + + def info(self, msg: str) -> None: + pass + + def warn(self, msg: str) -> None: + pass + + def error(self, msg: str) -> None: + pass + + +def _safe_detail(exc: Exception) -> str: + """An upstream message with credential-bearing URL parts removed. + + Locators are kept because they are what the reader must fix, but a URL + can carry userinfo or a signed query, and these messages are built from + unvalidated document text. + """ + detail = _CREDENTIAL_IN_URL.sub(r"\g[redacted]@", str(exc)) + return _URL_QUERY.sub(r"\g?[redacted]", detail) + + +# The resolver kinds decompile may use: each only reads a file or the network. +# They are pinned to the built-in implementations, so a downstream +# re-registration cannot swap code in. EVERY other kind -- local_from_python, +# the docker/container spellings, anything a downstream package registers -- +# is refused, never run, and a kind no one registered is already refused by +# the hydrator itself: a new resolver kind fails closed until it is listed here. +_STATIC_RESOLVER_KINDS = { + "digest": "_resolve_digest", + "name": "_resolve_name", + "url": "_resolve_url", + "file": "_resolve_file", + "resolve": "_resolve_resolve", + "http": "_resolve_http", + "https": "_resolve_http", + "local": "_resolve_local", +} + + +_STATIC_URI_READERS = {"http": "_read_http_uri", "https": "_read_http_uri"} + + +class _StaticResolution: + """Resolver overrides for every decompiler hydrator, whatever the cwd or + ``--trusted-source`` trust. Refusals are recorded, so the code stays + stable even where a ``fallback_on_error`` entry swallows the error.""" + + def __init__(self) -> None: + self.refused: DecompileError | None = None + + def resolvers(self) -> dict[str, Any]: + from . import pipeline_hydrator as hydrator_module + + # The hydrator's own list of resolve-entry kinds, so a spelling it + # learns later is refused here too. + known = hydrator_module.PipelineHydrator._resolve_entry_kinds(_KindsOnly()) # type: ignore[arg-type] + overrides: dict[str, Any] = {kind: self._refusal(kind) for kind in {*hydrator_module.COMPONENT_RESOLVERS, *known}} + overrides.update({kind: getattr(hydrator_module, name) for kind, name in _STATIC_RESOLVER_KINDS.items()}) + return overrides + + def uri_readers(self) -> dict[str, Any]: + """The same rule for URI schemes: only the built-in network readers, + pinned; every other registered reader is refused, never called.""" + from . import pipeline_hydrator as hydrator_module + + overrides: dict[str, Any] = {scheme: self._refusal(scheme) for scheme in hydrator_module.URI_READERS} + overrides.update({scheme: getattr(hydrator_module, name) for scheme, name in _STATIC_URI_READERS.items()}) + return overrides + + def _refusal(self, kind: str) -> Any: + error = _LOCAL_PYTHON if kind == "local_from_python" else _EXECUTABLE_SOURCE + + def refuse(*_: Any) -> Any: + self.refused = error + raise error + + return refuse + + def check(self) -> None: + if self.refused is not None: + raise self.refused + + +class _KindsOnly: + component_resolvers: dict[str, Any] = {} + + +def _static_hydrator(guard: _StaticResolution, **options: Any) -> Any: + """A hydrator that resolves without running anything. + + Besides the resolver allowlist, a local file or fallback copy may be a + ``template_file`` config, which the hydrator renders with full Jinja -- + able to reach Python builtins -- so rendering is refused too. Decompiler + only: the shared hydrator keeps rendering for ``pipelines hydrate``. + """ + from .pipeline_hydrator import PipelineHydrator + + class _StaticHydrator(PipelineHydrator): + def _render_template_config(self, *args: Any, **kwargs: Any) -> Any: + guard.refused = _TEMPLATE_SOURCE + raise _TEMPLATE_SOURCE + + return _StaticHydrator(component_resolvers=guard.resolvers(), uri_readers=guard.uri_readers(), **options) + + +_HYDRATE_FIRST = ( + "Run `tangle sdk pipelines hydrate` on it first -- that is where trusted " + "sources are executed -- then decompile the hydrated YAML." +) +_LOCAL_PYTHON = DecompileError( + "unsupported-local-python-source", + "this pipeline references a local Python component (local_from_python), " + f"and resolving it would run that source; decompile never does. {_HYDRATE_FIRST}", +) +_TEMPLATE_SOURCE = DecompileError( + "unsupported-template-component-source", + "this pipeline references a template_file component config, and rendering " + f"it would evaluate a Jinja template; decompile never does. {_HYDRATE_FIRST}", +) +_EXECUTABLE_SOURCE = DecompileError( + "unsupported-executable-component-source", + "this pipeline references a component whose resolver may build or run " + f"code (anything but a digest, name, URL or file); decompile never does. {_HYDRATE_FIRST}", +) + + +def canonicalize_components( + document: Mapping[str, Any], + *, + base_dir: Path | None, + components_dir: Path, + output_file: Path, + client: Any = None, + sidecar_name: str | None = None, +) -> CanonicalComponents: + """Resolve every componentRef, then re-pin it through a resolve sidecar. + + Hydration inlines each spec; the shared DIGEST dehydration contract then + puts every task on ``resolve://./.components.yaml#`` (or + ``sidecar_name``, beside it), whose entries are a verified published digest + (when there is one) followed by a local copy extracted into ``components_dir``. + + Any failure aborts the whole operation. The hydrator raises for a single + reference it cannot resolve, but a ref carrying several locators falls back + to the first one and returns unresolved instead, so the post-check between + the two stages is what stops that case from being re-emitted silently. + """ + from .pipeline_dehydrator import DehydrateChoice, PipelineDehydrator + from .pipeline_hydrator import HydrationError + + # Seed-time exactness: resolve the exact referenced content. Following a + # deprecation successor here would seed a different component than the + # input names; later hydrate/submit/run still upgrade as usual. + guard = _StaticResolution() + hydrator = _static_hydrator(guard, client=client, logger=_QuietLogger(), upgrade_deprecated=False) + try: + hydrated = hydrator.resolve_components( + copy.deepcopy(dict(document)), base_dir + ) + except DecompileError: + raise + except HydrationError as exc: + guard.check() + # The hydrator located no content for a reference: a missing file or a + # component the library does not have. Distinguished from the failure + # below because the fix is the document, not the environment. + raise DecompileError("unresolved-component", _safe_detail(exc)) from exc + except RecursionError as exc: + guard.check() + # The hydrator has no cycle guard of its own, so a component that + # (transitively) references itself recurses until Python stops it. + raise DecompileError( + "cyclic-component-reference", + "resolving this pipeline's components recursed without end: its " + "component references form a cycle, which no pipeline can express.", + ) from exc + except Exception as exc: + guard.check() + raise DecompileError( + "component-resolution-failed", + "could not reach the component library to resolve this pipeline's " + f"references ({type(exc).__name__}): {_safe_detail(exc)}", + ) from exc + + guard.check() + _assert_every_component_resolved(hydrated) + + # Leaves only: every nested graph stays inline so codegen can turn it into + # a Python @pipeline, while the components inside it are pinned in place. + dehydrator = PipelineDehydrator( + {"": DehydrateChoice.DIGEST}, + extract_subgraphs=False, + components_dir=components_dir, + output_file=output_file, + client=client, + interactive=False, + logger=_QuietLogger(), + ) + try: + pinned = dehydrator.dehydrate(hydrated) + except Exception as exc: + raise DecompileError( + "component-pinning-failed", + "could not pin the pipeline's resolved components: " + f"{_safe_detail(exc)}", + ) from exc + if sidecar_name is not None: + _rename_sidecar(pinned, output_file, sidecar_name) + return CanonicalComponents(hydrated=hydrated, pinned=pinned) + + +def _rename_sidecar(pinned: Mapping[str, Any], output_file: Path, sidecar_name: str) -> None: + """Move the dehydrator's ``.components.yaml`` and retarget every leaf. + + Gate (a) runs afterwards, against the renamed sidecar, so a missed + reference fails as ``pinning-incomplete`` rather than shipping. + """ + written = output_file.with_name(f"{output_file.stem}.components.yaml") + if written.is_file(): + os.replace(written, output_file.with_name(sidecar_name)) + old, new = f"resolve://./{written.name}#", f"resolve://./{sidecar_name}#" + for _, ref in _pinned_locators(pinned): + url = ref.get("url") + if isinstance(url, str) and url.startswith(old): + ref["url"] = new + url[len(old):] + + +def _assert_every_component_resolved(document: Mapping[str, Any]) -> None: + """Refuse the whole document if any componentRef lacks a resolved spec.""" + + unresolved: list[str] = [] + + def visit(tasks: Any, prefix: str) -> None: + if not isinstance(tasks, Mapping): + return + for task_id, body in tasks.items(): + if not isinstance(body, Mapping): + continue + where = f"{prefix}{task_id}" + ref = body.get("componentRef") + if isinstance(ref, Mapping): + spec = ref.get("spec") + if not isinstance(spec, Mapping) or not spec: + unresolved.append(where) + else: + visit( + spec.get("implementation", {}).get("graph", {}).get("tasks"), + f"{where}/", + ) + + visit(document.get("implementation", {}).get("graph", {}).get("tasks"), "") + if unresolved: + listed = ", ".join(sorted(unresolved)) + raise DecompileError( + "unresolved-component", + "every component must resolve before a pipeline can be pinned; " + f"these did not: {listed}. Check the component library credentials " + "and that any file:// references exist relative to the input.", + ) + + +def assert_pinning_preserved( + canonical: CanonicalComponents, *, script_dir: Path, sidecar: str, client: Any = None +) -> None: + """Gate (a): the pinned document still denotes the hydrated pipeline. + + Resolves the pinned document the way a consumer would -- from the + generated script's own directory -- and requires the result to carry the + same component specs hydration produced. Locators are stripped before + comparison because they legitimately differ: an extracted file is sorted + YAML, so its text digest is not the original's. + + This is the only oracle that survives pinning. Once a task is on a bare + digest the generated Python no longer mentions the component's contents, + so gate (b) alone cannot tell a correct pin from a wrong one. + """ + + guard = _StaticResolution() + try: + rehydrated = _static_hydrator( + guard, client=client, logger=_QuietLogger(), upgrade_deprecated=False + ).resolve_components(copy.deepcopy(canonical.pinned), script_dir) + except Exception as exc: + guard.check() + raise DecompileError( + "pinning-unresolvable", + "the pinned document could not be resolved back from the " + f"generated script's directory: {_safe_detail(exc)}", + ) from exc + + guard.check() + # No resolution re-check here: an unresolved ref reduces to a null spec, + # which the comparison below already fails on. + before = _specs_only(canonical.hydrated) + after = _specs_only(rehydrated) + if before != after: + raise DecompileError( + "pinning-content-mismatch", + "pinning changed what the pipeline resolves to. " + f"{_shape_difference(before, after)}", + ) + + _assert_local_fallback_is_complete(canonical.pinned, before, script_dir) + _assert_sidecar_locators(canonical.pinned, script_dir, sidecar) + + +class _UnreachableLibrary: + """A component library that is down: every remote lookup raises. + + Generated primaries are ``fallback_on_error``, so under this client each one + falls through to its local copy -- which a healthy primary would otherwise + short-circuit, leaving the copy the script needs offline never checked. + """ + + def _unreachable(self, *args: Any, **kwargs: Any) -> Any: + raise ConnectionError("component library deliberately unreachable") + + resolve_digest = get_component_spec = _unreachable + find_existing_components = list_published_component_infos = _unreachable + + +def _assert_local_fallback_is_complete(pinned: Mapping[str, Any], expected: Any, script_dir: Path) -> None: + """Gate (a), offline: the local copies alone must reproduce the pipeline. + + Runs in addition to the online pass, never instead of it: a healthy remote + primary short-circuits that pass, so it never reads the local copies. + """ + + guard = _StaticResolution() + try: + # upgrade_deprecated is moot with no library, but stated for intent. + offline = _static_hydrator( + guard, client=_UnreachableLibrary(), logger=_QuietLogger(), upgrade_deprecated=False + ).resolve_components( + copy.deepcopy(dict(pinned)), script_dir + ) + except Exception as exc: + guard.check() + raise DecompileError( + "pinning-incomplete", + "the local fallback copies could not be resolved with the component " + "library unreachable, so the generated script would not work offline: " + f"{_safe_detail(exc)}", + ) from exc + guard.check() + # A missing copy never gets here: a generated ref has one locator, so the + # hydrator raises for it above rather than leaving a null spec. + actual = _specs_only(offline) + if actual != expected: + raise DecompileError( + "pinning-content-mismatch", + "a local fallback copy resolves to different content than the " + f"pipeline, so the script would change meaning offline. {_shape_difference(expected, actual)}", + ) + + +def _assert_sidecar_locators(pinned: Mapping[str, Any], script_dir: Path, sidecar: str) -> None: + """Every task must use the generated sidecar, and it must stay portable. + + Accepts only ``resolve://./#`` naming a fragment the + sidecar defines, whose entries are a published digest or a local copy + inside ``.leaves/``. Anything else -- an absolute or traversing + path, a foreign manifest, a mutable URL -- would not survive moving the + generated script, or would resolve to something gate (a) never checked. + """ + import yaml + + prefix = f"resolve://./{sidecar}#" + assets = sidecar[: -len(".yaml")] + manifest_path = script_dir / sidecar + manifest = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) if manifest_path.is_file() else {} + if not isinstance(manifest, Mapping): + manifest = {} + + for path, ref in _pinned_locators(pinned): + where = "/".join(path) + url = ref.get("url") if set(ref) == {"url"} else None + fragment = url[len(prefix):] if isinstance(url, str) and url.startswith(prefix) else None + # A fragment the sidecar does not define never gets here: resolving the + # pinned document through this same sidecar has already failed. + if not fragment: + raise DecompileError( + "pinning-incomplete", + f"task {where} is not pinned through the generated {sidecar} " + "sidecar, so it would not survive moving the generated script.", + ) + for entry in manifest.get(fragment) or (): + if not _portable_sidecar_entry(entry, assets): + raise DecompileError( + "pinning-incomplete", + f"task {where} resolves through a sidecar entry that is not a " + f"published digest or a local copy inside {assets}/.", + ) + + +def _portable_sidecar_entry(entry: Any, assets: str) -> bool: + if not isinstance(entry, Mapping): + return False + if set(entry) <= {"digest", "fallback_on_error"} and isinstance(entry.get("digest"), str): + return True + local = entry.get("local") + if set(entry) != {"local"} or not isinstance(local, str) or not local.startswith("./"): + return False + parts = PurePosixPath(local[2:]).parts + return len(parts) >= 2 and parts[0] == assets and ".." not in parts + + +def _specs_only(document: Any) -> Any: + """``document`` with every componentRef reduced to its resolved spec.""" + + if isinstance(document, Mapping): + out = {} + for key, value in document.items(): + if key == "componentRef" and isinstance(value, Mapping): + out[key] = {"spec": _specs_only(value.get("spec"))} + else: + out[key] = _specs_only(value) + return out + if isinstance(document, list): + return [_specs_only(item) for item in document] + return document + + +def _pinned_locators(document: Mapping[str, Any], path: tuple[str, ...] = ()) -> Iterator[tuple[tuple[str, ...], Any]]: + """Every leaf componentRef at every depth, with its task-id path. + + An inline graph boundary is not a locator -- it becomes a Python + subpipeline -- but the leaves inside it are, and must be portable too. + Paths are tuples, never joined keys: task ids may contain ``/``, so a + root ``a/b`` and a nested ``a`` -> ``b`` must not collapse into one. + """ + tasks = document.get("implementation", {}).get("graph", {}).get("tasks") + if not isinstance(tasks, Mapping): + return + for task_id, body in tasks.items(): + ref = body.get("componentRef") if isinstance(body, Mapping) else None + if not isinstance(ref, Mapping): + continue + if _is_graph_spec(ref.get("spec")): + yield from _pinned_locators(ref["spec"], (*path, str(task_id))) + else: + yield (*path, str(task_id)), ref + + +def _is_graph_spec(spec: Any) -> bool: + return isinstance(spec, Mapping) and isinstance(spec.get("implementation"), Mapping) and "graph" in spec["implementation"] + + +# --------------------------------------------------------------------------- +# Plan (IR) + + +@dataclass +class _Plan: + pipeline_name: str + function_name: str + description: str | None + labels: dict[str, Any] + annotations: dict[str, Any] + inputs: list[dict[str, Any]] + outputs: list[dict[str, Any]] + output_values: dict[str, Any] + tasks: list[tuple[str, dict[str, Any]]] + names: dict[str, str] + exempt_paths: frozenset[str] + normalizations: tuple[str, ...] = () + input_names: set[str] = field(default_factory=set) + # task id -> generated @pipeline function for a nested graph boundary + subpipelines: dict[str, str] = field(default_factory=dict) + # sidecar URL -> recovered @task name imported from the tasks package + python_tasks: dict[str, str] = field(default_factory=dict) + # sidecar URL -> unwrap parameter -> synthetic keys, for regrouping + python_unwrap: dict[str, dict[str, tuple[str, ...]]] = field(default_factory=dict) + # sidecar URL -> verified published digest, referenced directly + digest_refs: dict[str, str] = field(default_factory=dict) + + +def _plan(document: Mapping[str, Any], exempt_paths: Collection[str]) -> _Plan: + name = document.get("name") + if not isinstance(name, str) or not name: + raise DecompileError( + "unsupported-document", "pipeline document has no usable 'name'." + ) + + metadata = _require_mapping(document.get("metadata") or {}, "metadata") + labels = dict(_require_mapping(metadata.get("labels") or {}, "metadata.labels")) + annotations = dict( + _require_mapping(metadata.get("annotations") or {}, "metadata.annotations") + ) + + inputs = [ + _check_input(entry, index) + for index, entry in enumerate(_as_list(document.get("inputs"), "inputs")) + ] + outputs = [ + _require_mapping(entry, f"outputs[{index}]") + for index, entry in enumerate(_as_list(document.get("outputs"), "outputs")) + ] + for index, entry in enumerate(outputs): + _check_io_annotations(entry, f"outputs[{index}]") + + graph = _require_mapping( + _require_mapping( + document.get("implementation") or {}, "implementation" + ).get("graph") + or {}, + "implementation.graph", + ) + raw_tasks = _require_mapping(graph.get("tasks") or {}, "implementation.graph.tasks") + output_values = _require_mapping( + graph.get("outputValues") or {}, "implementation.graph.outputValues" + ) + + input_names = {str(entry["name"]) for entry in inputs} + for task_id, body in raw_tasks.items(): + _check_task(str(task_id), _require_mapping(body, f"task {task_id!r}")) + + ordered = _topological_order(raw_tasks) + names = _allocate_names( + [str(entry["name"]) for entry in inputs], [task_id for task_id, _ in ordered] + ) + + normalizations: list[str] = [] + if any("default" in entry and "optional" not in entry for entry in inputs): + normalizations.append( + "N1: a graph input carrying 'default' but no explicit 'optional' " + "is compared as optional: true, because graph_input() always " + "writes optional alongside a default." + ) + if _normalize_for_comparison(document) != _deep_copy(document): + normalizations.append( + "N2: an EMPTY block at a known optional location (metadata, " + "inputs, outputs, outputValues, or a task's annotations, " + "arguments or executionOptions) is compared as absent. Such a " + "block declares nothing and the authoring API omits the key " + "entirely, so it cannot survive a round trip. A block with any " + "content is never dropped." + ) + + return _Plan( + pipeline_name=name, + function_name=_python_identifier(name, prefix="pipeline"), + description=_optional_str(document.get("description"), "description"), + labels=labels, + annotations=annotations, + inputs=inputs, + outputs=outputs, + output_values=dict(output_values), + tasks=ordered, + names=names, + exempt_paths=frozenset(exempt_paths), + normalizations=tuple(normalizations), + input_names=input_names, + ) + + +# --------------------------------------------------------------------------- +# Refusals + + +def _check_input(entry: Any, index: int) -> dict[str, Any]: + spec = dict(_require_mapping(entry, f"inputs[{index}]")) + name = spec.get("name") + if not isinstance(name, str) or not name: + raise DecompileError( + "unsupported-document", f"inputs[{index}] has no usable 'name'." + ) + if "default" in spec and spec["default"] is None: + raise DecompileError( + "unsupported-input-default", + f"graph input {name!r} declares a null default. graph_input() " + "refuses default=None; omit the key to declare no default.", + ) + if "value" in spec: + raise DecompileError( + "unsupported-input-value", + f"graph input {name!r} carries a baked 'value', which the " + "authoring API cannot express.", + ) + _check_io_annotations(spec, f"graph input {name!r}") + return spec + + +def _check_io_annotations(spec: Mapping[str, Any], where: str) -> None: + """Pre-check what ``CALLER_ANNOTATION_POLICY`` enforces at authoring time. + + graph_input/graph_output annotations run through the caller policy, so + a non-string value, a reserved ``system/`` prefix or an embedded + template delimiter would raise deep inside the generated module. Catch + it here, where the diagnostic can name the YAML location. + """ + annotations = spec.get("annotations") + if annotations is None: + return + for key, value in _require_mapping(annotations, f"{where} annotations").items(): + if not isinstance(value, str): + raise DecompileError( + "unsupported-io-annotations", + f"{where} annotation {key!r} is {type(value).__name__}; the " + "authoring API accepts string annotation values only.", + ) + if str(key).startswith("system/"): + raise DecompileError( + "unsupported-io-annotations", + f"{where} annotation {key!r} uses the reserved 'system/' " + "prefix, which the authoring API refuses.", + ) + + +def _check_task(task_id: str, body: Mapping[str, Any]) -> None: + component_ref = _require_mapping( + body.get("componentRef") or {}, f"task {task_id!r} componentRef" + ) + if not any(key in component_ref for key in _LOCATOR_KEYS): + raise DecompileError( + "unsupported-component-ref", + f"task {task_id!r} has a componentRef with no url, name or " + "digest. Inline and hydrated specs are out of scope.", + ) + + for key, value in _require_mapping( + body.get("arguments") or {}, f"task {task_id!r} arguments" + ).items(): + _check_argument(task_id, str(key), value) + + if "isEnabled" in body: + _check_is_enabled(task_id, body["isEnabled"]) + + if "executionOptions" in body: + _check_execution_options(task_id, body["executionOptions"]) + + +def _check_argument(task_id: str, key: str, value: Any) -> None: + where = f"task {task_id!r} argument {key!r}" + if isinstance(value, str): + return + if not isinstance(value, Mapping): + raise DecompileError( + "unsupported-argument", + f"{where} is {type(value).__name__}; dehydrated arguments are " + "strings or a graphInput/taskOutput/dynamicData wrapper.", + ) + if "taskOutput" in value: + spec = _require_mapping(value["taskOutput"], where) + _check_edge_keys(spec, {"taskId", "outputName"}, "taskOutput", where) + output_name = spec.get("outputName") + if isinstance(output_name, str) and output_name.startswith("_"): + raise DecompileError( + "unsupported-task-output-name", + f"{where} reads an output whose name starts with '_'. The " + "task-output proxy treats leading-underscore names as " + "framework-internal and refuses them.", + ) + return + if "graphInput" in value: + _check_edge_keys( + _require_mapping(value["graphInput"], where), + {"inputName"}, + "graphInput", + where, + ) + return + if "dynamicData" in value: + return + raise DecompileError( + "unsupported-argument", + f"{where} uses an unrecognized argument wrapper.", + ) + + +def _check_edge_keys( + spec: Mapping[str, Any], allowed: set[str], kind: str, where: str +) -> None: + """Refuse an edge wrapper carrying keys the emitter would silently drop. + + ``emit.py`` writes exactly ``{taskId, outputName}`` / ``{inputName}``, + so an extra key such as a ``graphInput.type`` hint is real data that + cannot survive. Dropping it quietly would make the round trip lossy in + a way the digest gate catches but nobody could explain, so it is named + here instead. + """ + extra = sorted(set(spec) - allowed) + if extra: + raise DecompileError( + "unsupported-edge-fields", + f"{where} has a {kind} wrapper carrying {', '.join(extra)}, " + f"which the authoring API does not model: it emits only " + f"{', '.join(sorted(allowed))}.", + ) + + +def _check_is_enabled(task_id: str, value: Any) -> None: + if isinstance(value, str): + return + if isinstance(value, Mapping) and ("taskOutput" in value or "graphInput" in value): + return + raise DecompileError( + "unsupported-is-enabled", + f"task {task_id!r} has an isEnabled of type " + f"{type(value).__name__}; only a string, graphInput or taskOutput " + "is supported.", + ) + + +def _check_execution_options(task_id: str, value: Any) -> None: + options = _require_mapping(value, f"task {task_id!r} executionOptions") + for group, fields in options.items(): + allowed = _EXECUTION_OPTION_FIELDS.get(str(group)) + if allowed is None: + raise DecompileError( + "unsupported-execution-options", + f"task {task_id!r} declares the executionOptions group " + f"{group!r}, which the backend does not model and would " + "drop silently.", + ) + for key in _require_mapping( + fields, f"task {task_id!r} executionOptions.{group}" + ): + if str(key) not in allowed: + raise DecompileError( + "unsupported-execution-options", + f"task {task_id!r} declares executionOptions." + f"{group}.{key}, which the backend does not model and " + "would drop silently.", + ) + + +# --------------------------------------------------------------------------- +# Ordering and naming + + +def _topological_order( + raw_tasks: Mapping[str, Any], +) -> list[tuple[str, dict[str, Any]]]: + """Order tasks so every producer precedes its consumers. + + Python source is sequential, so a task must be assigned to a variable + before another task can reference it. Ties keep the document's own + order, which makes an already-topological input round-trip unchanged + and makes the output a pure function of the input. + """ + tasks = {str(k): _require_mapping(v, f"task {k!r}") for k, v in raw_tasks.items()} + position = {task_id: index for index, task_id in enumerate(tasks)} + + dependencies: dict[str, set[str]] = {} + for task_id, body in tasks.items(): + deps = { + producer + for producer in _referenced_task_ids(body) + if producer in tasks and producer != task_id + } + dependencies[task_id] = deps + + ordered: list[tuple[str, dict[str, Any]]] = [] + emitted: set[str] = set() + remaining = set(tasks) + while remaining: + ready = min( + (t for t in remaining if dependencies[t] <= emitted), + key=lambda t: position[t], + default=None, + ) + if ready is None: + raise DecompileError( + "unsupported-task-graph", + "the task graph contains a cycle, so no sequential Python " + "source can express it.", + ) + # One task per round, not the whole ready batch: emitting the batch + # would put an independent later task ahead of a dependent earlier + # one, reordering input that was already topological. + ordered.append((ready, dict(tasks[ready]))) + emitted.add(ready) + remaining.discard(ready) + return ordered + + +def _referenced_task_ids(body: Mapping[str, Any]) -> set[str]: + found: set[str] = set() + candidates: list[Any] = list( + _require_mapping(body.get("arguments") or {}, "arguments").values() + ) + if "isEnabled" in body: + candidates.append(body["isEnabled"]) + for value in candidates: + if isinstance(value, Mapping) and isinstance( + value.get("taskOutput"), Mapping + ): + task_id = value["taskOutput"].get("taskId") + if isinstance(task_id, str): + found.add(task_id) + return found + + +def _ascii_words(value: str) -> list[str]: + normalized = ( + unicodedata.normalize("NFKD", value).encode("ascii", "ignore").decode("ascii") + ) + return [part.lower() for part in re.findall(r"[A-Za-z0-9]+", normalized)] + + +def _python_identifier(value: str, *, prefix: str) -> str: + """Port of the Airflow converter's identifier rule. + + Ported rather than imported: that module lives downstream in + tangle-deploy, and this package must not depend on it. + """ + candidate = "_".join(_ascii_words(value)) or prefix + if candidate[0].isdigit(): + candidate = f"{prefix}_{candidate}" + # Builtins as well as keywords: a module-level ``def getattr(...)`` shadows + # the builtin for the whole module, and the emitter uses ``getattr`` to + # reach outputs whose names are not identifiers. + if ( + keyword.iskeyword(candidate) + or candidate in _IMPORTS + or candidate == "subpipeline" + or hasattr(builtins, candidate) + ): + candidate = f"{candidate}_" + return candidate + + +def _allocate_names( + input_names: Sequence[str], task_ids: Sequence[str] +) -> dict[str, str]: + """Allocate one flat namespace for graph inputs and tasks. + + Inputs and tasks share the function body's local scope, so they are + allocated together: ``"Foo"`` as both an input name and a task id must + not collapse onto one variable. Keys are prefixed by kind so the two + namespaces stay distinguishable in the returned map. + """ + allocated: set[str] = set(_IMPORTS) + result: dict[str, str] = {} + for kind, values, prefix in ( + ("input", input_names, "input"), + ("task", task_ids, "task"), + ): + for value in values: + base = f"{prefix}_{_python_identifier(value, prefix=prefix)}" + candidate = base + attempt = 0 + while candidate in allocated: + attempt += 1 + digest = hashlib.sha256( + f"{value}:{attempt}".encode("utf-8") + ).hexdigest()[:8] + candidate = f"{base}_{digest}" + allocated.add(candidate) + result[f"{kind}:{value}"] = candidate + return result + + +# --------------------------------------------------------------------------- +# Codegen (ast only) + + +def _render(plan: _Plan) -> str: + return _render_module([plan]) + + +def _render_module( + plans: list[_Plan], companion: str | None = None, companion_names: tuple[str, ...] = (), tasks_package: str | None = None +) -> str: + """One module of ``@pipeline`` functions, defined in dependency order. + + ``subpipeline`` is imported only when used, so a document with no nested + graphs renders exactly as it always has. + """ + names = list(_IMPORTS) + if any(plan.subpipelines for plan in plans): + names = sorted([*names, "subpipeline"]) + body: list[ast.stmt] = [ + ast.ImportFrom( + module="tangle_cli.python_pipeline", + names=[ast.alias(name=n, asname=None) for n in names], + level=0, + ) + ] + if companion and companion_names: + body.append( + ast.ImportFrom( + module=companion, + names=[ast.alias(name=n, asname=None) for n in companion_names], + level=0, + ) + ) + task_names = sorted({ + plan.python_tasks[body_["componentRef"]["url"]] + for plan in plans + for _, body_ in plan.tasks + if set(body_.get("componentRef") or {}) == {"url"} and body_["componentRef"]["url"] in plan.python_tasks + }) + if tasks_package and task_names: + body.append( + ast.ImportFrom(module=tasks_package, names=[ast.alias(name=n, asname=None) for n in task_names], level=0) + ) + body.extend(_pipeline_function(plan) for plan in plans) + module = ast.Module(body=body, type_ignores=[]) + ast.fix_missing_locations(module) + rendered = ast.unparse(module) + header = _HEADER + if tasks_package and task_names: + header += ( + f"\nExcept the Python components imported from {tasks_package}: Tangle\n" + "rebuilds those from their recovered source, which is not proven here.\n" + ) + return f'"""{header}"""\n{rendered}\n' + + +def _pipeline_function(plan: _Plan) -> ast.FunctionDef: + decorator_kwargs = [ast.keyword(arg="name", value=_literal(plan.pipeline_name))] + if plan.description is not None: + decorator_kwargs.append( + ast.keyword(arg="description", value=_literal(plan.description)) + ) + if plan.labels: + decorator_kwargs.append(ast.keyword(arg="labels", value=_literal(plan.labels))) + if plan.annotations: + # Passed as a mapping, never via flow_direction=: that keyword is + # applied last and would overwrite an authored + # editor.flow-direction rather than reproduce it. + decorator_kwargs.append( + ast.keyword(arg="annotations", value=_literal(plan.annotations)) + ) + + statements: list[ast.stmt] = [] + for spec in plan.inputs: + statements.append(_graph_input_statement(plan, spec)) + for task_id, task_body in plan.tasks: + statements.append(_task_statement(plan, task_id, task_body)) + for spec in plan.outputs: + statements.append(_graph_output_statement(plan, spec)) + if not statements: + statements.append(ast.Pass()) + + return ast.FunctionDef( + name=plan.function_name, + args=ast.arguments( + posonlyargs=[], + args=[], + vararg=None, + kwonlyargs=[], + kw_defaults=[], + kwarg=None, + defaults=[], + ), + body=statements, + decorator_list=[ + ast.Call( + func=ast.Name(id="pipeline", ctx=ast.Load()), + args=[], + keywords=decorator_kwargs, + ) + ], + returns=ast.Constant(value=None), + type_comment=None, + ) + + +def _graph_input_statement(plan: _Plan, spec: Mapping[str, Any]) -> ast.stmt: + name = str(spec["name"]) + args: list[ast.expr] = [_literal(name)] + if "type" in spec: + args.append(_literal(spec["type"])) + + keywords: list[ast.keyword] = [] + if "description" in spec: + keywords.append( + ast.keyword(arg="description", value=_literal(spec["description"])) + ) + if "default" in spec: + # The original default is already the schema's string form. Passing + # it verbatim avoids serialize_input_default re-rendering it (True + # -> "True", dicts -> sorted JSON), which would change the bytes. + keywords.append(ast.keyword(arg="default", value=_literal(spec["default"]))) + if "optional" in spec: + keywords.append(ast.keyword(arg="optional", value=_literal(spec["optional"]))) + keywords.extend(_graph_io_annotation_keywords(spec)) + + return ast.Assign( + targets=[ast.Name(id=plan.names[f"input:{name}"], ctx=ast.Store())], + value=ast.Call( + func=ast.Name(id="graph_input", ctx=ast.Load()), + args=args, + keywords=keywords, + ), + type_comment=None, + ) + + +def _graph_io_annotation_keywords(spec: Mapping[str, Any]) -> list[ast.keyword]: + """``annotations=`` and, for an exactly reproducible x/y-only + editor.position, ``position=(x, y)`` -- which graph I/O applies after the + mapping, so it stays the last write. Anything else (width/height included: + ``position=`` takes x/y only) stays verbatim in ``annotations=``.""" + annotations = dict(_require_mapping(spec.get("annotations") or {}, "graph I/O annotations")) + position = _position_sugar(annotations.get(_POSITION_ANNOTATION)) + if position is not None and set(position) != {"x", "y"}: + position = None + rest = {k: v for k, v in annotations.items() if k != _POSITION_ANNOTATION} if position else annotations + keywords = [ast.keyword(arg="annotations", value=_literal(rest))] if rest else [] + if position: + keywords.append( + ast.keyword(arg="position", value=ast.Tuple(elts=[_literal(position["x"]), _literal(position["y"])], ctx=ast.Load())) + ) + return keywords + + +def _graph_output_statement(plan: _Plan, spec: Mapping[str, Any]) -> ast.stmt: + name = str(spec["name"]) + edge = plan.output_values.get(name) + if edge is None: + raise DecompileError( + "unsupported-output", + f"graph output {name!r} has no entry in outputValues; the " + "authoring API wires every output to a task output or input.", + ) + args: list[ast.expr] = [_literal(name), _edge_expression(plan, edge, name)] + if "type" in spec: + args.append(_literal(spec["type"])) + + keywords: list[ast.keyword] = [] + if "description" in spec: + keywords.append( + ast.keyword(arg="description", value=_literal(spec["description"])) + ) + keywords.extend(_graph_io_annotation_keywords(spec)) + + return ast.Expr( + value=ast.Call( + func=ast.Name(id="graph_output", ctx=ast.Load()), + args=args, + keywords=keywords, + ) + ) + + +def _task_statement(plan: _Plan, task_id: str, body: Mapping[str, Any]) -> ast.stmt: + component_ref = _require_mapping(body["componentRef"], "componentRef") + call: ast.expr + if task_id in plan.subpipelines: + call = ast.Call( + func=ast.Name(id="subpipeline", ctx=ast.Load()), + args=[ast.Name(id=plan.subpipelines[task_id], ctx=ast.Load())], + keywords=[], + ) + elif set(component_ref) == {"url"} and component_ref["url"] in plan.python_tasks: + call = ast.Name(id=plan.python_tasks[component_ref["url"]], ctx=ast.Load()) + elif set(component_ref) == {"url"} and component_ref["url"] in plan.digest_refs: + call = _digest_ref(plan.digest_refs[component_ref["url"]]) + else: + locator_keywords = [ + ast.keyword(arg=key, value=_literal(component_ref[key])) + for key in _LOCATOR_KEYS + if key in component_ref + ] + call = ast.Call(func=ast.Name(id="ref", ctx=ast.Load()), args=[], keywords=locator_keywords) + # .named() always: the LHS-derived id is a Title Case transform of the + # variable name, which cannot reproduce an arbitrary task id. + call = _method(call, "named", [_literal(task_id)]) + + annotations = dict(_require_mapping(body.get("annotations") or {}, "task annotations")) + position = _position_sugar(annotations.get(_POSITION_ANNOTATION)) + rest = {k: v for k, v in annotations.items() if k != _POSITION_ANNOTATION} if position else annotations + if rest: + call = _method(call, "with_annotations", [_literal(rest)]) + if position: + # Last, so editor.position is its final write whatever else is set. + call = ast.Call( + func=ast.Attribute(value=call, attr="with_position", ctx=ast.Load()), + args=[_literal(position["x"]), _literal(position["y"])], + keywords=[ast.keyword(arg=k, value=_literal(position[k])) for k in ("width", "height") if k in position], + ) + + arguments = _require_mapping(body.get("arguments") or {}, "arguments") + bind_pairs: list[tuple[str, ast.expr]] = [] + call_keywords: list[ast.keyword] = [] + task_path = f"implementation.graph.tasks.{task_id}" + # A recovered @task(unwrap=...) takes each flattened ``__`` + # input back as one ``param={key: ...}`` dict, keys in spec input order, + # at the position of the first of them. + converted = set(component_ref) == {"url"} and component_ref["url"] in plan.python_tasks and task_id not in plan.subpipelines + unwrap = plan.python_unwrap.get(component_ref["url"], {}) if converted else {} + owner = {f"{name}__{key}": name for name, keys in unwrap.items() for key in keys} + grouped: set[str] = set() + for key, value in arguments.items(): + key = str(key) + if key in owner: + name = owner[key] + if name in grouped: + continue + grouped.add(name) + expression = ast.Dict( + keys=[_literal(k) for k in unwrap[name]], + values=[ + _argument_expression(plan, arguments[f"{name}__{k}"], f"{task_path}.arguments.{name}__{k}", task_id, f"{name}__{k}") + for k in unwrap[name] + ], + ) + key = name + else: + expression = _argument_expression( + plan, value, f"{task_path}.arguments.{key}", task_id, key + ) + if key in _RESERVED_CALL_KEYWORDS: + # Would bind the task-metadata parameter instead of the + # component input; .bind() keeps it an argument. + bind_pairs.append((key, expression)) + elif key.isidentifier() and not keyword.iskeyword(key): + call_keywords.append(ast.keyword(arg=key, value=expression)) + else: + # A **{...} segment at this position keeps the original + # argument order rather than hoisting odd keys to the end. + call_keywords.append( + ast.keyword( + arg=None, + value=ast.Dict(keys=[_literal(key)], values=[expression]), + ) + ) + if bind_pairs: + call = ast.Call( + func=ast.Attribute(value=call, attr="bind", ctx=ast.Load()), + args=[], + keywords=[ast.keyword(arg=k, value=v) for k, v in bind_pairs], + ) + + if "isEnabled" in body: + call_keywords.append( + ast.keyword( + arg="is_enabled", + value=_is_enabled_expression(plan, body["isEnabled"], task_id), + ) + ) + execution_options = _require_mapping( + body.get("executionOptions") or {}, "executionOptions" + ) + if execution_options: + # An EMPTY mapping is skipped under N2: the authoring API rejects + # execution_options={} outright, and the block declares nothing. + call_keywords.append( + ast.keyword( + arg="execution_options", value=_literal(execution_options) + ) + ) + + return ast.Assign( + targets=[ast.Name(id=plan.names[f"task:{task_id}"], ctx=ast.Store())], + value=ast.Call(func=call, args=[], keywords=call_keywords), + type_comment=None, + ) + + +_POSITION_ANNOTATION = "editor.position" + + +def _position_sugar(value: Any) -> dict[str, Any] | None: + """The coordinates ``.with_position`` re-renders to exactly ``value``, or None. + + Only when upstream ``position_annotation_value`` reproduces the very same + string: a compact, reordered, non-finite, boolean, null or extra-keyed + payload would be rewritten, so it stays in ``.with_annotations`` verbatim, + as does one it cannot convert at all (an overlong int, deep nesting). + """ + import json + + from .editor_layout import position_annotation_value + + if not isinstance(value, str): + return None + try: + position = json.loads(value) + except (ValueError, RecursionError): + return None + if not isinstance(position, dict) or not {"x", "y"} <= set(position) <= {"x", "y", "width", "height"}: + return None + try: + rendered = position_annotation_value( + position["x"], position["y"], width=position.get("width"), height=position.get("height") + ) + except (ValueError, OverflowError): # an int too large for a float + return None + return position if rendered == value else None + + +def _argument_expression( + plan: _Plan, value: Any, path: str, task_id: str, key: str +) -> ast.expr: + if isinstance(value, str): + if path in plan.exempt_paths: + return ast.Call( + func=ast.Name(id="raw", ctx=ast.Load()), + args=[_literal(value)], + keywords=[], + ) + return _literal(value) + + mapping = _require_mapping(value, f"task {task_id!r} argument {key!r}") + if "taskOutput" in mapping or "graphInput" in mapping: + return _edge_expression(plan, mapping, f"task {task_id!r} argument {key!r}") + if "dynamicData" in mapping: + return _dynamic_data_expression( + mapping["dynamicData"], f"task {task_id!r} argument {key!r}" + ) + raise DecompileError( + "unsupported-argument", + f"task {task_id!r} argument {key!r} uses an unrecognized wrapper.", + ) + + +def _edge_expression(plan: _Plan, value: Any, where: str) -> ast.expr: + mapping = _require_mapping(value, where) + if "taskOutput" in mapping: + spec = _require_mapping(mapping["taskOutput"], where) + task_id = str(spec.get("taskId", "")) + output_name = str(spec.get("outputName", "")) + variable = plan.names.get(f"task:{task_id}") + if variable is None: + raise DecompileError( + "unsupported-document", + f"{where} references task {task_id!r}, which the document " + "does not define.", + ) + target = ast.Name(id=variable, ctx=ast.Load()) + if ( + output_name.isidentifier() + and not output_name.startswith("_") + # A keyword is a valid identifier but not a valid attribute: + # ``task.class`` does not parse. + and not keyword.iskeyword(output_name) + ): + return ast.Attribute(value=target, attr=output_name, ctx=ast.Load()) + # getattr keeps an arbitrary output name a string literal instead + # of forcing it through the identifier grammar. + return ast.Call( + func=ast.Name(id="getattr", ctx=ast.Load()), + args=[target, _literal(output_name)], + keywords=[], + ) + if "graphInput" in mapping: + spec = _require_mapping(mapping["graphInput"], where) + input_name = str(spec.get("inputName", "")) + variable = plan.names.get(f"input:{input_name}") + if variable is None: + raise DecompileError( + "unsupported-document", + f"{where} references graph input {input_name!r}, which the " + "document does not declare.", + ) + return ast.Name(id=variable, ctx=ast.Load()) + raise DecompileError("unsupported-argument", f"{where} is not a usable edge.") + + +def _dynamic_data_expression(value: Any, where: str) -> ast.expr: + mapping = _require_mapping(value, where) + secret = mapping.get("secret") + if set(mapping) != {"secret"} or not isinstance(secret, Mapping): + raise DecompileError( + "unsupported-dynamic-data", + f"{where} uses a dynamicData kind other than 'secret'; only " + "dynamic_secret() exists in the authoring API.", + ) + secret_name = secret.get("name") + if set(secret) != {"name"} or not isinstance(secret_name, str) or not secret_name: + raise DecompileError( + "unsupported-dynamic-data", + f"{where} has a dynamicData.secret without a single usable " + "'name' field.", + ) + return ast.Call( + func=ast.Name(id="dynamic_secret", ctx=ast.Load()), + args=[_literal(secret_name)], + keywords=[], + ) + + +def _is_enabled_expression(plan: _Plan, value: Any, task_id: str) -> ast.expr: + if isinstance(value, str): + # Emitted as the exact string. A bool would be re-serialized to + # "true"/"false", which is already what the document holds. + return _literal(value) + return _edge_expression(plan, value, f"task {task_id!r} isEnabled") + + +def _method(value: ast.expr, name: str, args: list[ast.expr]) -> ast.expr: + return ast.Call( + func=ast.Attribute(value=value, attr=name, ctx=ast.Load()), + args=args, + keywords=[], + ) + + +def _literal(value: Any) -> ast.expr: + """Build a literal node for a recovered YAML value. + + Everything reaching generated source goes through here, so no input + text is ever interpolated: ``ast.unparse`` owns the quoting and + escaping, which is what makes a hostile name or description inert. + """ + if isinstance(value, Mapping): + return ast.Dict( + keys=[_literal(k) for k in value], + values=[_literal(v) for v in value.values()], + ) + if isinstance(value, (list, tuple)): + return ast.List(elts=[_literal(v) for v in value], ctx=ast.Load()) + if isinstance(value, (str, bool, int, float)) or value is None: + return ast.Constant(value=value) + raise DecompileError( + "unsupported-value", + f"cannot render a {type(value).__name__} as a Python literal.", + ) + + +# Node types generated source may legally contain. The generator is the +# only producer, so a node outside this set means a codegen bug, not a +# hostile input -- but the file is about to be imported and executed, so +# it is checked rather than assumed. +_ALLOWED_NODES: tuple[type[ast.AST], ...] = ( + ast.Module, + ast.ImportFrom, + ast.alias, + ast.FunctionDef, + ast.arguments, + ast.Assign, + ast.Expr, + ast.Pass, + ast.Call, + ast.Attribute, + ast.Name, + ast.Constant, + ast.Dict, + ast.List, + ast.Tuple, # graph I/O position=(x, y) + ast.keyword, + ast.Load, + ast.Store, +) + + +def _is_negative_number(node: ast.UnaryOp) -> bool: + return ( + isinstance(node.op, ast.USub) + and isinstance(node.operand, ast.Constant) + and isinstance(node.operand.value, (int, float)) + and not isinstance(node.operand.value, bool) + ) + + +def _assert_generated_source_is_safe(source: str, *modules: str | None) -> None: + """Parse the generated source and allowlist every node in it. + + Imports are pinned too: only the authoring package, plus the generated + companion and tasks modules when there are any, since this code is about + to run. + """ + try: + tree = ast.parse(source) + except SyntaxError as exc: # pragma: no cover - a codegen bug + raise DecompileError( + "codegen-syntax-error", + f"generated source is not valid Python: {exc.msg}", + ) from exc + allowed_modules = {"tangle_cli.python_pipeline", *(m for m in modules if m)} + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom) and (node.module not in allowed_modules or node.level): + raise DecompileError( + "codegen-unsafe-node", + "generated source imports an unexpected module. Refusing to " + "emit code that is about to be executed.", + ) + if isinstance(node, ast.UnaryOp) and _is_negative_number(node): + continue # ast.unparse writes -5 as USub(5) + if isinstance(node, ast.USub): + continue + if not isinstance(node, _ALLOWED_NODES): + raise DecompileError( + "codegen-unsafe-node", + f"generated source contains an unexpected {type(node).__name__} " + "node. Refusing to emit code that is about to be executed.", + ) + + +# --------------------------------------------------------------------------- +# Verification + + +def _verify_round_trip( + source: str, original: Mapping[str, Any], pipeline_name: str +) -> None: + """Recompile the generated source and require semantic identity. + + Uses trace+emit rather than the full compile driver: the driver's + asset policy resolves relative ``file://`` and ``resolve://`` refs + against the output directory, which most corpus documents would fail + purely because the referenced files are not beside a temp directory. + """ + with tempfile.TemporaryDirectory() as tmp: + module_path = Path(tmp) / "decompiled_pipeline.py" + module_path.write_text(source, encoding="utf-8") + rebuilt, exempt_paths, _ = _trace_function(module_path, None) + _compare_rebuilt(rebuilt, exempt_paths, original) + + +def _trace_function(module_path: Path, function_name: str | None) -> tuple[dict[str, Any], Any, list[tuple[str, str]]]: + """Trace+emit one ``@pipeline`` function; also return its subpipeline calls.""" + from .python_pipeline import emit, trace + from .python_pipeline.cfg import Cfg + from .python_pipeline.errors import CompileError + + try: + with _scrubbed_config_environment(): + from .pipeline_compiler import _load_pipeline_fn + + pipeline_fn = _load_pipeline_fn(module_path, function_name) + builder = trace.trace_pipeline(pipeline_fn, cfg=Cfg({}), inputs={}) + rebuilt, exempt_paths = emit.emit_pipeline(builder) + except CompileError as exc: + raise DecompileError( + "verification-compile-failed", + "the generated source did not compile back to a pipeline: " + f"{_safe_detail(exc)}", + ) from exc + calls = [(str(task_id), ref.child.fn.__name__) for task_id, ref in builder.task_refs_for_subpipelines] + return rebuilt, exempt_paths, calls + + +def _compare_rebuilt(rebuilt: dict[str, Any], exempt_paths: Any, original: Mapping[str, Any]) -> None: + from .schema_validation import SchemaValidationError, validate_dehydrated_pipeline + + try: + validate_dehydrated_pipeline(rebuilt, exempt_paths) + except SchemaValidationError as exc: + raise DecompileError( + "verification-invalid-output", + "the recompiled pipeline failed dehydrated validation. " + f"{_redacted_validation_reason(exc)}", + ) from exc + + normalized_original = _normalize_for_comparison(original) + normalized_rebuilt = _normalize_for_comparison(rebuilt) + expected = compute_spec_digest(normalized_original) + actual = compute_spec_digest(normalized_rebuilt) + if expected != actual: + raise DecompileError( + "verification-digest-mismatch", + "the recompiled pipeline is not semantically identical to the " + f"input (expected digest {expected}, got {actual}). " + # Diff the NORMALIZED documents: diffing the raw ones reports + # every N1/N2 difference as a discrepancy and buries the real + # one in noise. + f"{_shape_difference(normalized_original, normalized_rebuilt)}", + ) + + +def _normalize_for_comparison(document: Mapping[str, Any]) -> dict[str, Any]: + """Apply the one documented comparator normalization, N1. + + N1: a graph input carrying ``default`` but no explicit ``optional`` + compares as ``optional: true``. ``graph_input()`` always writes + ``optional`` alongside a ``default``, so the key reappears on the + round trip; the two documents mean the same thing. This is a + comparator rule only -- it never edits generated source or output. + """ + normalized = _deep_copy(document) + for entry in normalized.get("inputs", []) or []: + if isinstance(entry, dict) and "default" in entry and "optional" not in entry: + entry["optional"] = True + + # N2: drop vacuous empty blocks, at KNOWN optional locations only. A + # blanket "drop every empty container" would also hide real loss + # somewhere the emitter should have written content, so each location + # is named. + _drop_if_vacuous(normalized, ("metadata", "labels")) + _drop_if_vacuous(normalized, ("metadata", "annotations")) + _drop_if_vacuous(normalized, ("metadata",)) + _drop_if_vacuous(normalized, ("inputs",)) + _drop_if_vacuous(normalized, ("outputs",)) + + graph = _graph_of(normalized) + if graph is not None: + _drop_if_vacuous(graph, ("outputValues",)) + tasks = graph.get("tasks") + if isinstance(tasks, Mapping): + for body in tasks.values(): + if isinstance(body, dict): + _drop_if_vacuous(body, ("annotations",)) + _drop_if_vacuous(body, ("arguments",)) + _drop_if_vacuous(body, ("executionOptions",)) + _drop_if_vacuous(graph, ("tasks",)) + return normalized + + +def _drop_if_vacuous(container: Any, path: tuple[str, ...]) -> None: + """Remove ``path`` from ``container`` when it holds nothing at all. + + ``None``, ``{}`` and ``[]`` all mean "declares nothing" here, and the + authoring API writes no key in that case. + """ + for key in path[:-1]: + container = container.get(key) if isinstance(container, dict) else None + if container is None: + return + if not isinstance(container, dict): + return + leaf = path[-1] + if leaf in container and not container[leaf]: + container.pop(leaf) + + +def _graph_of(document: Any) -> dict[str, Any] | None: + implementation = document.get("implementation") if isinstance( + document, Mapping + ) else None + if not isinstance(implementation, dict): + return None + graph = implementation.get("graph") + return graph if isinstance(graph, dict) else None + + +def _shape_difference(original: Mapping[str, Any], rebuilt: Mapping[str, Any]) -> str: + """Describe *where* two documents diverge without echoing any value.""" + original_paths = set(_value_paths(original)) + rebuilt_paths = set(_value_paths(rebuilt)) + missing = sorted(original_paths - rebuilt_paths)[:5] + added = sorted(rebuilt_paths - original_paths)[:5] + if not missing and not added: + return "Key sets match, so at least one value differs." + parts = [] + if missing: + parts.append(f"missing from output: {', '.join(missing)}") + if added: + parts.append(f"added by output: {', '.join(added)}") + return "; ".join(parts) + "." + + +def _value_paths(value: Any, prefix: str = "") -> Iterable[str]: + if isinstance(value, Mapping): + for key, child in value.items(): + path = f"{prefix}.{key}" if prefix else str(key) + yield path + yield from _value_paths(child, path) + elif isinstance(value, list): + for index, child in enumerate(value): + yield from _value_paths(child, f"{prefix}[{index}]") + + +def _deep_copy(value: Any) -> Any: + if isinstance(value, Mapping): + return {k: _deep_copy(v) for k, v in value.items()} + if isinstance(value, list): + return [_deep_copy(v) for v in value] + return value + + +@contextlib.contextmanager +def _scrubbed_config_environment() -> Any: + """Hide ambient config from the verification trace. + + ``TANGLE_ROOT_CONFIG`` would let a developer's local config bind + ``cfg`` values into the traced graph, so verification could pass or + fail based on the machine it ran on. Generated source never emits + ``cfg``, so the variable can only distort the result. + """ + saved = os.environ.pop("TANGLE_ROOT_CONFIG", None) + try: + yield + finally: + if saved is not None: + os.environ["TANGLE_ROOT_CONFIG"] = saved + + +# --------------------------------------------------------------------------- +# Template-delimiter pre-pass + + +def _runtime_sentinel_paths(document: Mapping[str, Any]) -> frozenset[str]: + """Collect argument paths holding a legitimate ``{{...}}`` sentinel. + + Only *argument* string values qualify, and only when they carry no + ``{%``/``{#`` token: those are compile-time Jinja, never runtime + substitution, and ``raw()`` refuses them. Exempting such a value would + hide a leaked template rather than preserve a placeholder, so they are + left for the validator to reject. + """ + paths: set[str] = set() + + def visit(container: Any, prefix: str) -> None: + """Walk tasks at any depth, building full schema paths. + + Nested graphs only appear in hydrated input, where a sentinel is just + as legitimate as at the root; collecting the root alone would refuse a + document whose dehydrated equivalent is accepted. + """ + if not isinstance(container, Mapping): + return + graph = container.get("graph") + if not isinstance(graph, Mapping): + return + tasks = graph.get("tasks") + if not isinstance(tasks, Mapping): + return + for task_id, body in tasks.items(): + if not isinstance(body, Mapping): + continue + task_path = f"{prefix}.graph.tasks.{task_id}" + arguments = body.get("arguments") + if isinstance(arguments, Mapping): + for key, value in arguments.items(): + if not isinstance(value, str) or _RUNTIME_SENTINEL not in value: + continue + if any(token in value for token in _COMPILE_TIME_TOKENS): + continue + paths.add(f"{task_path}.arguments.{key}") + ref = body.get("componentRef") + if isinstance(ref, Mapping): + spec = ref.get("spec") + if isinstance(spec, Mapping): + visit( + spec.get("implementation"), + f"{task_path}.componentRef.spec.implementation", + ) + + visit(document.get("implementation"), "implementation") + return frozenset(paths) + + +# --------------------------------------------------------------------------- +# Small shared helpers + + +def _require_mapping(value: Any, where: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise DecompileError( + "unsupported-document", + f"expected {where} to be a mapping; got {type(value).__name__}.", + ) + return value + + +def _as_list(value: Any, where: str) -> list[Any]: + if value is None: + return [] + if not isinstance(value, list): + raise DecompileError( + "unsupported-document", + f"expected {where} to be a list; got {type(value).__name__}.", + ) + return value + + +def _optional_str(value: Any, where: str) -> str | None: + if value is None: + return None + if not isinstance(value, str): + raise DecompileError( + "unsupported-document", + f"expected {where} to be a string; got {type(value).__name__}.", + ) + return value diff --git a/packages/tangle-cli/src/tangle_cli/pipeline_hydrator.py b/packages/tangle-cli/src/tangle_cli/pipeline_hydrator.py index c2a1d5d..47cc15a 100644 --- a/packages/tangle-cli/src/tangle_cli/pipeline_hydrator.py +++ b/packages/tangle-cli/src/tangle_cli/pipeline_hydrator.py @@ -235,6 +235,11 @@ def __init__( "header": header, "include_env_credentials": include_env_credentials, } + # Only an EXPLICIT logger is forwarded to an internally created client. + # Passing the default one would override the client's own verbosity + # gating and make ordinary non-verbose runs start logging. + if logger is not None: + self._client_options["logger"] = logger self.cache: dict[str, Any] = {} self.upgrade_deprecated = upgrade_deprecated self.verbose = verbose diff --git a/packages/tangle-cli/src/tangle_cli/pipelines_cli.py b/packages/tangle-cli/src/tangle_cli/pipelines_cli.py index a87cb18..671b12b 100644 --- a/packages/tangle-cli/src/tangle_cli/pipelines_cli.py +++ b/packages/tangle-cli/src/tangle_cli/pipelines_cli.py @@ -23,6 +23,7 @@ TokenOption, ) from .logger import logger_for_log_type +from .pipeline_decompiler import DecompileError, decompile_pipeline_file from .pipelines import ( PipelineValidationError, compile_pipeline_file, @@ -393,6 +394,135 @@ def pipelines_compile( print(f"warning: {warning}") +@app.command(name="decompile") +def pipelines_decompile( + pipeline_path: pathlib.Path, + *, + output: Annotated[ + pathlib.Path, + Parameter( + name="--output", + alias="-o", + help=( + "Output path for the generated Python pipeline source. " + "Always replaced, atomically." + ), + ), + ], + verify: Annotated[ + bool, + Parameter( + name="--verify", + help=( + "Recompile the generated source and require semantic " + "equality with the input. Disabling this removes the only " + "proof the output is faithful." + ), + ), + ] = True, + python_tasks: Annotated[ + bool, + Parameter( + name="--python-tasks", + help=( + "Write components the Python component generator made, whose " + "source is recorded in full, as @task functions in " + "_tasks/. The source is copied, never run; compiling the " + "script later imports it. --no-python-tasks keeps every " + "component a digest-pinned YAML leaf." + ), + ), + ] = True, + local_fallbacks: Annotated[ + bool, + Parameter( + name="--local-fallbacks", + negative="", + help=( + "Also keep a local copy of every component the library " + "verified, behind its published digest, so the output " + "compiles and hydrates with the library unreachable. By " + "default those components are referenced by digest alone." + ), + ), + ] = False, +) -> None: + """Generate Python pipeline source from a pipeline YAML. + + Accepts hydrated or dehydrated YAML. Components are always resolved and + re-pinned before code generation. + """ + + try: + result = decompile_pipeline_file( + pipeline_path, output, verify=verify, python_tasks=python_tasks, include_local_fallbacks=local_fallbacks + ) + except DecompileError as exc: + raise SystemExit(str(exc)) from exc + + print( + f"Decompiled {pipeline_path} -> {result.output_path} " + f"({result.task_count} task(s))." + ) + if result.subgraph_module is not None: + print(f"Nested graphs are @pipeline functions in {result.subgraph_module}; keep it beside the script.") + if result.resolve_config is not None: + print( + f"Components resolve through {result.resolve_config}, with " + f"{len(result.component_files)} local copy(ies) in " + f"{result.components_dir}. Keep both beside the script: the " + "compiler resolves relative component URLs against the compiled " + "YAML's directory, so compile the script in place." + ) + if result.digest_refs: + print( + f"{result.digest_refs} component(s) are referenced by their published digest " + "alone: hydrating the output needs the component library. Pass " + "--local-fallbacks to also keep local copies for offline use." + ) + if result.tasks_package is not None: + print( + f"{result.python_tasks} Python component(s) are @task functions in " + f"{result.tasks_package}, holding source recovered verbatim from the " + "input. It was not run here: `compile` imports it, and hydrate/submit/" + "run execute it to rebuild each component (from the current " + "directory tree or a --trusted-source), so review it first. Those " + "components are no longer digest-pinned, and their rebuilt specs " + "carry new provenance annotations." + ) + if result.image_id_tasks: + print( + f"{result.image_id_tasks} of them set @task(image_id=...): compile resolves it " + "in your environment (--image ID=REF, then registered defaults) and may " + "pick a different image than the input recorded. Use --no-python-tasks " + "for the digest-exact components." + ) + if result.unwrap_tasks: + print( + f"{result.unwrap_tasks} of them use @task(unwrap=...): the flattened inputs' " + "names were checked, their types are derived again at compile." + ) + kept = result.python_candidates - result.python_tasks + if kept > 0: + print(f"{kept} component(s) with recorded Python source stayed YAML leaves.") + if result.verified: + print( + "Verified: recompiling the output reproduces the input's " + "semantic digest. This is not byte identity \u2014 comments, key " + "order and task order are not recoverable." + + ( + " For the Python components this proves the pipeline around " + "them, not the components Tangle will rebuild from source." + if result.tasks_package is not None + else "" + ) + ) + else: + print("warning: --no-verify was passed; the output is unproven.") + for note in result.normalizations: + print(f"normalization applied when comparing: {note}") + + @app.command(name="layout") def pipelines_layout( pipeline_path: pathlib.Path, diff --git a/packages/tangle-cli/src/tangle_cli/schema_validation.py b/packages/tangle-cli/src/tangle_cli/schema_validation.py index 0011e7d..c2081e8 100644 --- a/packages/tangle-cli/src/tangle_cli/schema_validation.py +++ b/packages/tangle-cli/src/tangle_cli/schema_validation.py @@ -60,9 +60,15 @@ class SchemaValidationError(ValueError): """Raised when a dehydrated pipeline fails schema/contract validation. The compiler wraps this in a ``CompileError`` so the CLI exits 1 with - a friendly message; tests may assert on it directly. + a friendly message; tests may assert on it directly. ``location`` carries + the structural path when one is known, so a caller that must not echo the + rejected value has an alternative to parsing the message. """ + def __init__(self, *args: Any, location: str | None = None) -> None: + super().__init__(*args) + self.location = location + def _schema_path() -> Path: """Locate the packaged dehydrated schema JSON. @@ -127,7 +133,8 @@ def validate_dehydrated_data(data: Mapping[str, Any]) -> None: ) raise SchemaValidationError( f"dehydrated pipeline failed schema validation at {location}: " - f"{best.message}" + f"{best.message}", + location=location, ) diff --git a/pyproject.toml b/pyproject.toml index 3a6b0ed..61264eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "tangle-cli" -version = "0.1.30" +version = "0.1.31" description = "CLI for Tangle, the open-source ML pipeline orchestration platform" readme = "README.md" authors = [ diff --git a/tests/test_packaging.py b/tests/test_packaging.py index 75c5369..990d27b 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -183,7 +183,7 @@ def test_tangle_cli_wheel_supports_expert_no_deps_import_path_without_tangle_api requires_dist = [line for line in metadata.splitlines() if line.startswith("Requires-Dist: ")] assert not any(name.startswith("tangle_api/") for name in names) assert "tangle_cli/openapi/openapi.json" not in names - assert "Version: 0.1.30" in metadata + assert "Version: 0.1.31" in metadata assert "Requires-Dist: tangle-api==0.1.1" in requires_dist assert not any("extra == 'native'" in line for line in requires_dist) assert "Provides-Extra: native" in metadata diff --git a/tests/test_pipeline_decompiler.py b/tests/test_pipeline_decompiler.py new file mode 100644 index 0000000..2e6cdfe --- /dev/null +++ b/tests/test_pipeline_decompiler.py @@ -0,0 +1,2031 @@ +"""YAML -> Python decompile: the load-bearing public contracts. + +Codegen is proven by round-trip verification (the generated source must +recompile to the same semantic digest). The file entry point adds original +input validation, DIGEST resolve-sidecar canonicalization, two resolution +gates (online and forced-offline), and three-artifact publication. +""" +from __future__ import annotations + +import ast +import contextlib +import copy +import io +import os +import pathlib +import re +import shutil +from pathlib import Path +from typing import Any + +import pytest + +from tangle_cli.pipeline_compiler import compile_pipeline +from tangle_cli.pipeline_decompiler import ( + CanonicalComponents, + DecompileError, + _UnreachableLibrary, + assert_pinning_preserved, + decompile_pipeline, + decompile_pipeline_file, +) +from tangle_cli.pipeline_hydrator import PipelineHydrator +from tangle_cli.utils import compute_spec_digest, compute_text_digest, dump_yaml, parse_yaml_string + +DIGEST = "a" * 64 + + +# --------------------------------------------------------------------------- +# Builders + + +def _doc(tasks: dict[str, Any] | None = None, *, graph: dict[str, Any] | None = None, **top: Any) -> dict[str, Any]: + """A dehydrated document; ``tasks`` defaults to one named-component task.""" + body = {"tasks": tasks if tasks is not None else {"Only Task": {"componentRef": {"name": "comp"}}}} + body.update(graph or {}) + return {"name": "Base Pipeline", "implementation": {"graph": body}, **top} + + +def _ref(**ref: Any) -> dict[str, Any]: + return {"componentRef": ref} + + +def _out(task: str, output: str) -> dict[str, Any]: + return {"taskOutput": {"taskId": task, "outputName": output}} + + +def _leaf(marker: str = "payload") -> dict[str, Any]: + return {"name": "Leaf", "implementation": {"container": {"image": "example/image:latest", "command": [marker]}}} + + +def _graph_spec(name: str, tasks: dict[str, Any]) -> dict[str, Any]: + return {"name": name, "implementation": {"graph": {"tasks": tasks}}} + + +def _write(path: Path, data: Any) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(dump_yaml(data), encoding="utf-8") + return path + + +def _pipeline(tmp_path: Path, task: dict[str, Any], name: str = "source.yaml") -> Path: + return _write(tmp_path / name, _graph_spec("Outer", {"a": task})) + + +def _local_leaf_source(tmp_path: Path, marker: str = "payload") -> Path: + """A dehydrated pipeline whose one component is a local file.""" + _write(tmp_path / "leaf.yaml", _leaf(marker)) + return _pipeline(tmp_path, _ref(url="file://./leaf.yaml")) + + +class _Library: + """An offline component library: ``published`` digest -> spec.""" + + def __init__(self, published: dict[str, Any] | None = None, successors: dict[str, str] | None = None) -> None: + self.published = published or {} + self.successors = successors or {} + + def resolve_digest(self, digest: str) -> str: + return self.successors.get(digest, digest) + + def get_component_spec(self, digest: str) -> dict[str, Any]: + if digest not in self.published: + raise KeyError(digest) + return copy.deepcopy(self.published[digest]) + + def find_existing_components(self, names: list[str] | None = None, **kwargs: Any) -> list[Any]: + from tangle_cli.models import ComponentInfo + + return [ + ComponentInfo(name=name, digest=digest, version=(spec.get("metadata", {}).get("annotations", {}).get("version"))) + for name in names or () + for digest, spec in self.published.items() + if spec.get("name") == name + ] + + +def _published(spec: dict[str, Any]) -> tuple[str, _Library]: + digest = compute_text_digest(dump_yaml(spec)) + return digest, _Library({digest: spec}) + + +def _captured(fn: Any) -> tuple[str, str]: + """Run ``fn``; return (combined stdout+stderr, DecompileError message or "").""" + buffer, message = io.StringIO(), "" + with contextlib.redirect_stdout(buffer), contextlib.redirect_stderr(buffer): + try: + fn() + except DecompileError as exc: + message = str(exc) + return buffer.getvalue(), message + + +def _fragment(script: Path, task: str = "a") -> list[dict[str, Any]]: + """The sidecar entries a task of a PUBLISHED (or staged) script resolves through.""" + sidecar = script.with_name(f"{script.stem}.leaves.yaml") + manifest = parse_yaml_string(sidecar.read_text(encoding="utf-8")) + prefix = f"resolve://./{sidecar.name}#" + for line in script.read_text(encoding="utf-8").splitlines(): + if prefix in line: + return manifest[line.split(prefix, 1)[1].split("'")[0].split('"')[0]] + raise AssertionError("no sidecar reference in the script") + + +def _snapshot(directory: Path) -> dict[str, bytes]: + return {p.relative_to(directory).as_posix(): p.read_bytes() for p in directory.rglob("*") if p.is_file()} + + +def _leaf_specs(document: dict[str, Any]) -> list[str]: + found: list[str] = [] + for task in document["implementation"]["graph"]["tasks"].values(): + spec = task["componentRef"]["spec"] + if "graph" in spec["implementation"]: + found += _leaf_specs(spec) + else: + found.append(compute_spec_digest({k: v for k, v in spec.items() if not str(k).startswith("_")})) + return found + + +# --------------------------------------------------------------------------- +# Codegen: every construct round-trips (verification proves the digest) + +CONSTRUCTS = [ + pytest.param(_doc(), ["ref(name='comp')", ".named('Only Task')"], [], id="minimal"), + pytest.param( + _doc(description="what it does", metadata={"labels": {"team": "search"}, "annotations": {"editor.flow-direction": "LR"}}), + ["description='what it does'", "labels={'team': 'search'}", "annotations={'editor.flow-direction': 'LR'}"], + # flow_direction= is applied after annotations, so it would overwrite an authored value. + ["flow_direction="], + id="root-metadata", + ), + pytest.param( + _doc({"T": _ref(url="file://./c.yaml", name="comp", digest=DIGEST)}), + ["url=", "name=", "digest="], + [], + id="all-locators", + ), + pytest.param( + _doc({"T": _ref(url="resolve://./gen.yaml#frag", digest=DIGEST)}), ["url=", "digest="], [], id="resolve-locator" + ), + pytest.param( + _doc( + { + "Producer": _ref(name="p"), + "Consumer": { + **_ref(name="c"), + "arguments": { + "plain": "literal", + "sentinel": "select {{input_1}}", + "from_input": {"graphInput": {"inputName": "In One"}}, + "from_task": _out("Producer", "rows"), + "token": {"dynamicData": {"secret": {"name": "API_KEY"}}}, + }, + }, + }, + inputs=[{"name": "In One", "type": "String"}], + ), + [ + "plain='literal'", + "raw('select {{input_1}}')", + "from_input=input_in_one", + "from_task=task_producer.rows", + "token=dynamic_secret('API_KEY')", + ], + [], + id="argument-shapes", + ), + pytest.param( + _doc( + { + "T": { + "annotations": {"service_account": "sa@example.com"}, + **_ref(name="c"), + "isEnabled": "false", + "executionOptions": {"cachingStrategy": {"maxCacheStaleness": "P0D"}, "retryStrategy": {"maxRetries": 3}}, + } + } + ), + ["with_annotations({'service_account': 'sa@example.com'})", "is_enabled='false'", "maxCacheStaleness"], + [], + id="task-metadata", + ), + pytest.param( + _doc({"Gate": _ref(name="g"), "Gated": {**_ref(name="c"), "isEnabled": _out("Gate", "ok")}}), + ["is_enabled=task_gate.ok"], + [], + id="is-enabled-edge", + ), + pytest.param( + _doc( + outputs=[{"name": "Result", "type": "String", "description": "the result"}], + graph={"outputValues": {"Result": _out("Only Task", "out")}}, + ), + ["graph_output('Result', task_only_task.out, 'String'"], + [], + id="graph-outputs", + ), + pytest.param( + _doc( + inputs=[ + { + "name": "Pipeline Creation Time", + "type": "String", + "description": "when the run started", + "default": "1970-01-01", + "optional": True, + "annotations": {"editor.position": '{"x":1,"y":2}'}, + }, + {"name": "Maybe", "type": "String", "optional": True}, + {"name": "Flag", "type": "Boolean", "default": "True"}, + ] + ), + ["graph_input('Pipeline Creation Time', 'String'", "description='when the run started'", "optional=True", "default='True'"], + ["In["], + id="inputs-all-fields", + ), + pytest.param( + # A compact editor.position must pass through verbatim: re-rendering via + # json.dumps would rewrite roughly half the corpus. + _doc( + {"T": {"annotations": {"editor.position": '{"x":10,"y":20}'}, **_ref(name="c")}}, + inputs=[{"name": "In", "type": "String", "annotations": {"editor.position": '{"x":10,"y":20}'}}], + ), + ['{"x":10,"y":20}'], + ["with_position", "position=("], + id="compact-position", + ), + pytest.param( + _doc({"Load 2024 Data!": _ref(name="c")}, inputs=[{"name": "Thing", "type": "String"}]), + [".named('Load 2024 Data!')", "input_thing ="], + [], + id="non-identifier-task-id", + ), + pytest.param( + _doc({"Thing": _ref(name="c")}, inputs=[{"name": "Thing", "type": "String"}]), + ["input_thing =", "task_thing ="], + [], + id="input-and-task-share-a-name", + ), + pytest.param( + _doc({"T": {**_ref(name="c"), "arguments": {"from": "x", "class": "y", "odd-key": "z"}}}), + ["**{'from': 'x'}", "**{'class': 'y'}", "**{'odd-key': 'z'}"], + [], + id="keyword-argument-keys", + ), + pytest.param( + # A real component input named is_enabled must not become the condition. + _doc({"T": {**_ref(name="c"), "arguments": {"is_enabled": "a component input"}, "isEnabled": "true"}}), + [".bind(is_enabled='a component input')", "is_enabled='true'"], + [], + id="is-enabled-input-is-bound", + ), + pytest.param( + _doc({"P": _ref(name="p"), "C": {**_ref(name="c"), "arguments": {"x": _out("P", "rows written"), "y": _out("P", "class")}}}), + ["getattr(task_p, 'rows written')", "getattr(task_p, 'class')"], + [], + id="non-identifier-and-keyword-output-names", + ), +] + + +@pytest.mark.parametrize(("document", "present", "absent"), CONSTRUCTS) +def test_every_construct_round_trips(document: dict[str, Any], present: list[str], absent: list[str]) -> None: + source = decompile_pipeline(document).source + for needle in present: + assert needle in source, needle + for needle in absent: + assert needle not in source, needle + + +def test_inputs_keep_document_order_and_distinct_names_stay_distinct() -> None: + """Inputs all go through graph_input in document order (list order is part + of the digest), and names collapsing to one identifier stay distinct.""" + document = _doc( + {"my task": _ref(name="a"), "My Task": _ref(name="b"), "MY-TASK": _ref(name="c")}, + inputs=[{"name": "b", "type": "String"}, {"name": "a", "type": "String"}], + ) + source = decompile_pipeline(document).source + assert source.index("graph_input('b'") < source.index("graph_input('a'") + assigned = [n.targets[0].id for n in ast.walk(ast.parse(source)) if isinstance(n, ast.Assign)] + assert len([a for a in assigned if a.startswith("task_")]) == len({a for a in assigned if a.startswith("task_")}) == 3 + + +@pytest.mark.parametrize( + ("tasks", "order"), + [ + # Consumer listed before its producer: reordered. + ({"Consumer": {**_ref(name="c"), "arguments": {"x": _out("Producer", "o")}}, "Producer": _ref(name="p")}, ["task_producer", "task_consumer"]), + # Already topological: document order kept, even for an independent task. + ({"Zebra": _ref(name="z"), "Alpha": _ref(name="a")}, ["task_zebra", "task_alpha"]), + ({"A": _ref(name="a"), "B": {**_ref(name="b"), "arguments": {"x": _out("A", "o")}}, "C": _ref(name="c")}, ["task_a", "task_b", "task_c"]), + ], + ids=["producer-first", "topological-kept", "independent-does-not-jump"], +) +def test_task_order_is_topological_with_document_order_ties(tasks: dict[str, Any], order: list[str]) -> None: + source = decompile_pipeline(_doc(tasks)).source + positions = [source.index(f"{name} =") for name in order] + assert positions == sorted(positions) + + +@pytest.mark.parametrize( + "hostile", + [ + '"""\nimport os\nos.system("id")\n#', + "'; import os; os.system('id'); '", + "\\x27 + __import__('os').system('id') + \\x27", + "line one\nline two", + "#!/bin/sh\nrm -rf /", + ], +) +def test_hostile_text_cannot_escape_into_code(hostile: str) -> None: + """Every recovered value is an ast.Constant: the generated file is executed + during verification, so escaping a literal would be code execution.""" + tree = ast.parse( + decompile_pipeline( + {"name": hostile, "description": hostile, "implementation": {"graph": {"tasks": {hostile: {**_ref(name="c"), "arguments": {"a": hostile}}}}}} + ).source + ) + assert hostile in {n.value for n in ast.walk(tree) if isinstance(n, ast.Constant)} + assert hostile not in ast.get_docstring(tree) + assert [type(n).__name__ for n in tree.body] == ["Expr", "ImportFrom", "FunctionDef"] + + +def test_a_pipeline_named_after_a_builtin_does_not_shadow_it() -> None: + """``def getattr(...)`` at module level would break the emitter's own getattr use.""" + document = _doc( + {"a": {"componentRef": {"digest": DIGEST}, "arguments": {}}}, + graph={"outputValues": {"r": _out("a", "odd-name")}}, + outputs=[{"name": "r", "type": "String"}], + ) + document["name"] = "getattr" + result = decompile_pipeline(document) + assert result.verified + assert "def getattr_(" in result.source + + +def test_output_is_deterministic_and_independent_of_key_order() -> None: + forward = _doc(description="d", metadata={"labels": {"a": "1"}}, inputs=[{"name": "i", "type": "String"}]) + shuffled = dict(reversed(list(forward.items()))) + assert decompile_pipeline(forward).source == decompile_pipeline(forward).source == decompile_pipeline(shuffled).source + + +# --------------------------------------------------------------------------- +# In-memory refusals: each names what it cannot faithfully express + +REFUSALS = [ + pytest.param(_doc({"A": {**_ref(name="a"), "arguments": {"x": _out("B", "o")}}, "B": {**_ref(name="b"), "arguments": {"x": _out("A", "o")}}}), "unsupported-task-graph", id="cycle"), + pytest.param(_doc({"T": _ref(name="c", spec={"implementation": {"container": {}}})}), "unsupported-document", id="hydrated-spec"), + # A leaked compile-time template is never swept into the raw() runtime-sentinel exemption. + pytest.param(_doc({"T": {**_ref(name="c"), "arguments": {"a": "{% for x in y %}{{ x }}{% endfor %}"}}}), "unsupported-document", id="jinja-statement"), + # The schema permits these; the backend silently drops them. + pytest.param(_doc({"T": {**_ref(name="c"), "executionOptions": {"retryStrategy": {"maxRetries": 1, "backoff": "60s"}}}}), "unsupported-execution-options", id="retry-backoff"), + pytest.param(_doc({"T": {**_ref(name="c"), "executionOptions": {"timeoutStrategy": {}}}}), "unsupported-execution-options", id="empty-unmodeled-group"), + pytest.param(_doc({"P": _ref(name="p"), "C": {**_ref(name="c"), "arguments": {"x": _out("P", "_hidden")}}}), "unsupported-task-output-name", id="private-output"), + pytest.param(_doc(inputs=[{"name": "i", "type": "String", "default": None}]), "unsupported-input-default", id="null-default"), + pytest.param(_doc(inputs=[{"name": "i", "type": "String", "annotations": {"system/x": "y"}}]), "unsupported-io-annotations", id="system-annotation"), + # N2 covers empty containers, never nulls. + pytest.param(_doc(outputs=None), "unsupported-document", id="null-block"), + pytest.param( + _doc({"Only Task": _ref(name="comp"), "R": {**_ref(name="r"), "arguments": {"x": {"graphInput": {"inputName": "In", "type": "String"}}}}}, inputs=[{"name": "In", "type": "String"}]), + "unsupported-edge-fields", + id="edge-wrapper-extra-field", + ), +] + + +@pytest.mark.parametrize(("document", "code"), REFUSALS) +def test_unrepresentable_documents_are_refused_by_code(document: dict[str, Any], code: str) -> None: + with pytest.raises(DecompileError) as excinfo: + decompile_pipeline(document) + assert excinfo.value.code == code + + +def test_a_refusal_reports_structure_never_the_offending_value() -> None: + """Diagnostics name a location from structured metadata, never the value: + a planted value shaped like a location must not leak into the message.""" + planted = " at SYNTHETIC_SCHEMA_VALUE_136: " + document = _doc({"Task With Spaces": {**_ref(digest=DIGEST), "arguments": {"token": {"malformed": planted}}}}) + streams, message = _captured(lambda: decompile_pipeline(document, verify=False)) + assert "SYNTHETIC_SCHEMA_VALUE_136" not in message + streams + assert "tasks.Task With Spaces.arguments.token" in message + + +def test_verification_is_what_catches_an_unfaithful_emission(monkeypatch: pytest.MonkeyPatch) -> None: + """Drop a field during codegen and the digest gate must fire.""" + import tangle_cli.pipeline_decompiler as decompiler + + original = decompiler._graph_input_statement + monkeypatch.setattr(decompiler, "_graph_input_statement", lambda plan, spec: original(plan, {k: v for k, v in spec.items() if k != "description"})) + with pytest.raises(DecompileError) as excinfo: + decompile_pipeline(_doc(inputs=[{"name": "i", "type": "String", "description": "dropped"}])) + assert excinfo.value.code == "verification-digest-mismatch" + assert "description" in str(excinfo.value) + + +@pytest.mark.parametrize( + ("optional", "reported"), [(None, True), (True, False)], ids=["implied-optional", "explicit-optional"] +) +def test_n1_is_reported_only_when_a_default_implies_optional(optional: bool | None, reported: bool) -> None: + field = {"name": "i", "type": "String", "default": "x", **({"optional": optional} if optional else {})} + result = decompile_pipeline(_doc(inputs=[field])) + assert result.verified + assert any("N1" in note for note in result.normalizations) is reported + + +def _task(d: dict[str, Any]) -> dict[str, Any]: + return d["implementation"]["graph"]["tasks"]["Only Task"] + + +@pytest.mark.parametrize( + "mutate", + [ + pytest.param(lambda d: d.update(metadata={}), id="metadata"), + pytest.param(lambda d: d.update(inputs=[]), id="inputs"), + pytest.param(lambda d: d.update(outputs=[]), id="outputs"), + pytest.param(lambda d: d["implementation"]["graph"].update(outputValues={}), id="outputValues"), + pytest.param(lambda d: _task(d).update(arguments={}), id="arguments"), + pytest.param(lambda d: _task(d).update(annotations={}), id="annotations"), + pytest.param(lambda d: _task(d).update(executionOptions={}), id="executionOptions"), + ], +) +def test_n2_treats_a_vacuous_block_as_absent(mutate: Any) -> None: + document = _doc() + mutate(document) + result = decompile_pipeline(document) + assert result.verified + assert any("N2" in note for note in result.normalizations) + + +def test_n2_never_drops_a_block_that_has_content() -> None: + document = _doc( + {"Only Task": {"annotations": {"k": "v"}, **_ref(name="comp"), "arguments": {"a": "b"}, "executionOptions": {"cachingStrategy": {"maxCacheStaleness": "P0D"}}}}, + metadata={"labels": {"a": "1"}}, + ) + result = decompile_pipeline(document) + assert result.normalizations == () + assert "labels={'a': '1'}" in result.source and "a='b'" in result.source + + +EXAMPLES = sorted((pathlib.Path(__file__).resolve().parents[1] / "examples" / "python_pipeline").glob("*.py")) + + +def test_the_examples_directory_is_not_empty() -> None: + assert EXAMPLES + + +@pytest.mark.parametrize("example", EXAMPLES, ids=lambda p: p.stem) +def test_compiling_a_decompiled_example_reproduces_the_original_bytes(example: Path, tmp_path: Path) -> None: + """compile -> decompile -> compile is a byte fixpoint on compiler output. + + Uses the in-memory API: the file command re-pins every component, which + rewrites locators by design. Both compiles share one directory because a + ``@task`` compiles to a relative sidecar ref resolved against the output. + """ + workdir = tmp_path / "out" + workdir.mkdir() + compile_pipeline(example, workdir / "p.yaml") + original = (workdir / "p.yaml").read_text(encoding="utf-8") + generated = tmp_path / "generated.py" + generated.write_text(decompile_pipeline(parse_yaml_string(original)).source, encoding="utf-8") + compile_pipeline(generated, workdir / "p2.yaml") + assert (workdir / "p2.yaml").read_text(encoding="utf-8") == original + + +# --------------------------------------------------------------------------- +# File entry point: the ORIGINAL input is validated before anything resolves + + +def _unresolvable_sibling(tasks: dict[str, Any]) -> dict[str, Any]: + return {**tasks, "unresolvable": _ref(url="file://./definitely-missing.yaml")} + + +def _nested_with(leaf_task: dict[str, Any]) -> dict[str, Any]: + return _ref(spec=_graph_spec("Inner", {"leaf": leaf_task})) + + +class _Unreachable(_Library): + def get_component_spec(self, digest: str) -> Any: + raise ConnectionError("simulated outage") + + +INPUT_REFUSALS = [ + # Legacy inline text: hydration would drop it silently. Refused on key presence, at any depth. + pytest.param(_graph_spec("O", {"a": _ref(spec=_leaf(), text="name: Legacy\n")}), None, "unsupported-document", id="legacy-text"), + pytest.param(_graph_spec("O", {"a": _ref(spec=_leaf(), text="")}), None, "unsupported-document", id="legacy-text-empty"), + pytest.param(_graph_spec("O", {"sub": _nested_with(_ref(spec=_leaf(), text="x"))}), None, "unsupported-document", id="legacy-text-nested"), + # Compile-time templates: refused in both shapes, at any depth. + pytest.param(_graph_spec("O", {"a": {**_ref(spec=_leaf()), "arguments": {"x": "{% if y %}"}}}), None, "unsupported-document", id="template-hydrated"), + pytest.param(_graph_spec("O", {"sub": _nested_with({**_ref(spec=_leaf()), "arguments": {"q": "{% if x %}1{% endif %}"}})}), None, "unsupported-document", id="template-nested"), + pytest.param(_graph_spec("O", {"a": {**_ref(name="c"), "arguments": {"x": "{% if y %}"}}}), None, "unsupported-document", id="template-dehydrated"), + # Each of these ALSO references an unresolvable component: the contract + # error must win, proving validation ran before any resolution. + pytest.param({**_graph_spec("O", _unresolvable_sibling({"a": _ref(name="c")})), "bogusRootKey": 1}, None, "unsupported-document", id="unknown-key-before-resolution"), + pytest.param({**_graph_spec("O", _unresolvable_sibling({"a": _ref(spec=_leaf())})), "inputs": "not-a-list"}, None, "unsupported-document", id="hydrated-bad-shape-before-resolution"), + # Resolution failures are split by who can fix them. + pytest.param(_graph_spec("O", {"a": _ref(url="file://./missing.yaml")}), None, "unresolved-component", id="missing-file"), + pytest.param(_graph_spec("O", {"a": _ref(name="Missing")}), _Library(), "unresolved-component", id="unpublished-name"), + # A locator-less top-level ref never reaches resolution: the schema requires one. + pytest.param(_graph_spec("O", {"a": {"componentRef": {}}}), None, "unsupported-document", id="empty-ref"), + pytest.param(_graph_spec("O", {"a": _ref(url="")}), None, "unsupported-document", id="falsy-url"), + pytest.param(_graph_spec("O", {"a": _ref(digest=DIGEST)}), _Unreachable(), "component-resolution-failed", id="library-unreachable"), +] + + +@pytest.mark.parametrize(("document", "client", "code"), INPUT_REFUSALS) +def test_an_unsupported_or_unresolvable_input_publishes_nothing(tmp_path: Path, document: dict[str, Any], client: Any, code: str) -> None: + source = _write(tmp_path / "in.yaml", document) + with pytest.raises(DecompileError) as excinfo: + decompile_pipeline_file(source, tmp_path / "out.py", client=client) + assert excinfo.value.code == code + assert sorted(p.name for p in tmp_path.iterdir()) == ["in.yaml"] + + +def test_a_silently_unresolved_nested_component_is_refused(tmp_path: Path) -> None: + """The one resolution failure that raises nothing: the hydrator returns a + locator-less NESTED ref untouched. Input validation never sees a + referenced file, so only the post-resolution check can catch it.""" + _write(tmp_path / "outer.yaml", _graph_spec("Outer", {"inner": {"componentRef": {}}})) + source = _pipeline(tmp_path, _ref(url="file://./outer.yaml"), name="in.yaml") + with pytest.raises(DecompileError) as excinfo: + decompile_pipeline_file(source, tmp_path / "out.py") + assert excinfo.value.code == "unresolved-component" + assert "inner" in excinfo.value.message + assert not (tmp_path / "out.py").exists() + + +def test_an_unparseable_input_reports_position_not_content(tmp_path: Path) -> None: + source = tmp_path / "bad.yaml" + source.write_text("name: ok\n\tSYNTHETIC_TOKEN_136: [unclosed\n", encoding="utf-8") + with pytest.raises(DecompileError) as excinfo: + decompile_pipeline_file(source, tmp_path / "out.py") + assert excinfo.value.code == "unreadable-input" + assert "SYNTHETIC_TOKEN_136" not in str(excinfo.value) + + +@pytest.mark.parametrize("nested", [False, True], ids=["flat", "nested-runtime-sentinel"]) +def test_equivalent_hydrated_and_dehydrated_inputs_decompile_identically(tmp_path: Path, nested: bool) -> None: + """Both shapes are canonicalized before codegen. A ``{{input_1}}`` runtime + sentinel is legitimate at any depth, so nesting must not break parity.""" + leaf_task = {**_ref(spec=_leaf()), "arguments": {"q": "SELECT {{input_1}}"}} if nested else _ref(spec=_leaf()) + inner = _graph_spec("Inner", {"leaf": leaf_task}) + hydrated = _pipeline(tmp_path / "h", _ref(spec=inner) if nested else leaf_task) + dehydrated_dir = tmp_path / "d" + _write(dehydrated_dir / "c.yaml", inner if nested else _leaf()) + dehydrated = _pipeline(dehydrated_dir, _ref(url="file://./c.yaml")) + + one = decompile_pipeline_file(hydrated, hydrated.parent / "out.py") + two = decompile_pipeline_file(dehydrated, dehydrated.parent / "out.py") + assert one.verified and two.verified + assert one.source == two.source + assert one.component_files == two.component_files + + +# --------------------------------------------------------------------------- +# DIGEST canonicalization: every task goes through the resolve sidecar + +def _versioned(marker: str, version: str) -> dict[str, Any]: + spec = _leaf(marker) + spec["metadata"] = {"annotations": {"version": version}} + return spec + + +def _sidecar_case(kind: str, tmp_path: Path) -> tuple[Path, Any, str | None]: + """Return (source, client, expected verified primary digest or None).""" + if kind == "unpublished-file": + return _local_leaf_source(tmp_path), None, None + if kind == "published-file": + text = dump_yaml(_leaf()) + (tmp_path / "leaf.yaml").write_text(text, encoding="utf-8") + digest = compute_text_digest(text) + return _pipeline(tmp_path, _ref(url="file://./leaf.yaml")), _Library({digest: _leaf()}), digest + if kind == "name-only": + digest, library = _published(_leaf()) + return _pipeline(tmp_path, _ref(name="Leaf")), library, digest + if kind == "name-highest-version": + old, new = _versioned("old", "1.0.0"), _versioned("new", "2.0.0") + library = _Library({compute_text_digest(dump_yaml(s)): s for s in (old, new)}) + return _pipeline(tmp_path, _ref(name="Leaf")), library, compute_text_digest(dump_yaml(new)) + if kind == "inline-spec-has-no-text-provenance": + # A documented limit: the library indexes the source TEXT digest, which + # an inline spec does not have, so it gets only its local copy. + _, library = _published(_leaf()) + return _pipeline(tmp_path, _ref(spec=_leaf())), library, None + if kind == "canonical-url-is-not-kept": + spec = {**_leaf(), "metadata": {"annotations": {"canonical_location": "https://example.test/leaf.yaml"}}} + _write(tmp_path / "leaf.yaml", spec) + return _pipeline(tmp_path, _ref(url="file://./leaf.yaml")), None, None + raise AssertionError(kind) + + +@pytest.mark.parametrize( + "kind", + ["unpublished-file", "published-file", "name-only", "name-highest-version", "inline-spec-has-no-text-provenance", "canonical-url-is-not-kept"], +) +def test_a_verified_leaf_is_referenced_by_digest_alone_by_default(tmp_path: Path, kind: str) -> None: + """Published: ref(digest=...) and no local copy at all. Unpublished: the + local copy is the only representation, so it is always written.""" + source, client, primary = _sidecar_case(kind, tmp_path) + output = tmp_path / "out" / "gen.py" + + result = decompile_pipeline_file(source, output, client=client) + + script = output.read_text(encoding="utf-8") + assert result.verified and result.digest_refs == (1 if primary else 0) + if primary: + assert f"ref(digest={primary!r})" in script and "resolve://" not in script + assert not output.with_name("gen.leaves.yaml").exists() and not output.with_name("gen.leaves").exists() + assert result.resolve_config is None and result.components_dir is None + else: + assert [set(e) for e in _fragment(output)] == [{"local"}] + + +@pytest.mark.parametrize( + "kind", + ["unpublished-file", "published-file", "name-only", "name-highest-version", "inline-spec-has-no-text-provenance", "canonical-url-is-not-kept"], +) +def test_local_fallbacks_keep_a_verified_primary_and_a_local_copy(tmp_path: Path, kind: str) -> None: + source, client, primary = _sidecar_case(kind, tmp_path) + output = tmp_path / "out" / "gen.py" + + result = decompile_pipeline_file(source, output, client=client, include_local_fallbacks=True) + + assert result.digest_refs == 0 + entries = _fragment(output) + local = entries[-1] + assert set(local) == {"local"} and local["local"].startswith("./gen.leaves/") + assert (output.parent / local["local"]).is_file() + assert entries[:-1] == ([{"digest": primary, "fallback_on_error": True}] if primary else []) + for text in (output.read_text(encoding="utf-8"), output.with_name("gen.leaves.yaml").read_text(encoding="utf-8")): + assert "https://" not in text and str(tmp_path) not in text, "no mutable or absolute locator" + assert result.verified and result.resolve_config == output.with_name("gen.leaves.yaml") + + +def test_one_component_shares_a_copy_and_same_named_components_stay_distinct(tmp_path: Path) -> None: + _write(tmp_path / "first.yaml", _leaf("first")) + _write(tmp_path / "second.yaml", _leaf("second")) + tasks = {"a": _ref(url="file://./first.yaml"), "b": _ref(url="file://./second.yaml"), "c": _ref(url="file://./first.yaml")} + source = _write(tmp_path / "in.yaml", _graph_spec("O", tasks)) + + result = decompile_pipeline_file(source, tmp_path / "out.py") + + assert len(result.component_files) == 2, "same content shares one copy; distinct content never collapses" + bodies = sorted((tmp_path / "out.leaves" / f).read_text(encoding="utf-8") for f in result.component_files) + assert "first" in bodies[0] and "second" in bodies[1] + + +# --------------------------------------------------------------------------- +# Three-artifact publication: script, .leaves.yaml, .leaves/ + + +def _published_leaf_source(tmp_path: Path, *, nested: bool) -> tuple[Path, _Library]: + """An inline leaf carrying a published digest the library verifies.""" + digest, library = _published(_leaf("published-leaf")) + task = _ref(digest=digest, spec=_leaf("published-leaf")) + return _pipeline(tmp_path, _nested_with(task) if nested else task), library + + +@pytest.mark.parametrize("nested", [False, True], ids=["root", "nested"]) +def test_the_three_products_relocate_compile_and_rehydrate_offline(tmp_path: Path, nested: bool) -> None: + """Move all three products, compile from the moved script, and hydrate + through the moved sidecar with the component library unreachable.""" + source, library = _published_leaf_source(tmp_path, nested=nested) + work = tmp_path / "work" + + result = decompile_pipeline_file(source, work / "gen.py", client=library, include_local_fallbacks=True) + + assert result.verified + assert (result.subgraph_module is not None) is nested + companion = {"gen_subgraphs.py"} if nested else set() + assert set(_snapshot(work)) == {"gen.py", "gen.leaves.yaml", *companion, *(f"gen.leaves/{f}" for f in result.component_files)} + moved = tmp_path / "moved" + shutil.copytree(work, moved) + shutil.rmtree(work) + # The natural invocation: .yaml beside the script, where a @task + # sidecar would be written as .components.yaml. + compile_pipeline(moved / "gen.py", moved / "gen.yaml") + hydrated = PipelineHydrator(client=_UnreachableLibrary()).hydrate_file(moved / "gen.yaml").data + assert _leaf_specs(hydrated) == [compute_spec_digest(_leaf("published-leaf"))] + + +@pytest.mark.parametrize("nested", [False, True], ids=["root", "nested"]) +def test_mixed_default_output_relocates_and_rehydrates_exactly(tmp_path: Path, nested: bool) -> None: + """A published leaf (twice, deduplicated) by digest, an unpublished one + local only: moved, compiled, and hydrated through the library, every spec + is the input's -- and a stale copy of the published leaf is not kept.""" + published = _leaf("published-leaf") + digest, library = _published(published) + _write(tmp_path / "local.yaml", _leaf("local-only")) + tasks = {"a": _ref(digest=digest, spec=published), "b": _ref(digest=digest, spec=published), "c": _ref(url="file://./local.yaml")} + document = _graph_spec("Outer", tasks) + source = _write(tmp_path / "in.yaml", _graph_spec("Outer", {"n": _ref(spec=_graph_spec("Inner", tasks))}) if nested else document) + work = tmp_path / "work" + decompile_pipeline_file(source, work / "gen.py", client=library, include_local_fallbacks=True) + + result = decompile_pipeline_file(source, work / "gen.py", client=library) + + generated = "".join(p.read_text(encoding="utf-8") for p in work.glob("*.py")) + assert result.verified and result.digest_refs == 1 and generated.count(f"ref(digest={digest!r})") == 2 + assert [p.name for p in (work / "gen.leaves").iterdir()] == [f for f in result.component_files] and len(result.component_files) == 1 + assert "local-only" in (work / "gen.leaves" / result.component_files[0]).read_text(encoding="utf-8") + assert len(parse_yaml_string((work / "gen.leaves.yaml").read_text(encoding="utf-8"))) == 1 + moved = tmp_path / "moved" + shutil.move(str(work), moved) + compile_pipeline(moved / "gen.py", moved / "gen.yaml") + hydrated = PipelineHydrator(client=library).hydrate_file(moved / "gen.yaml").data + assert sorted(_leaf_specs(hydrated)) == sorted(compute_spec_digest(_leaf(m)) for m in ("published-leaf", "published-leaf", "local-only")) + + +@pytest.mark.parametrize("existing", ["file", "absolute-symlink", "relative-symlink", "stale-products"]) +def test_the_output_is_always_replaced_never_written_through_or_merged(tmp_path: Path, existing: str) -> None: + source = _local_leaf_source(tmp_path) + output = tmp_path / "gen.py" + target = tmp_path / "target.py" + target.write_text("# untouched\n", encoding="utf-8") + if existing == "file": + output.write_text("# stale\n", encoding="utf-8") + elif existing.endswith("symlink"): + output.symlink_to(target if existing == "absolute-symlink" else Path("target.py")) + else: + _write(tmp_path / "gen.leaves" / "stale.yaml", {"name": "stale"}) + _write(tmp_path / "gen.leaves.yaml", {"stale-fragment": []}) + # The compiler's (or a user's) artifacts under the OLD name are not ours. + _write(tmp_path / "gen.components" / "theirs.yaml", {"name": "theirs"}) + _write(tmp_path / "gen.components.yaml", {"their-fragment": []}) + foreign = {k: v for k, v in _snapshot(tmp_path).items() if ".components" in k} + + decompile_pipeline_file(source, output) + + assert {k: v for k, v in _snapshot(tmp_path).items() if ".components" in k} == foreign + + assert not output.is_symlink() and "# stale" not in output.read_text(encoding="utf-8") + assert target.read_text(encoding="utf-8") == "# untouched\n", "a symlink is replaced, never followed" + assert not (tmp_path / "gen.leaves" / "stale.yaml").exists() + assert "stale-fragment" not in parse_yaml_string((tmp_path / "gen.leaves.yaml").read_text(encoding="utf-8")) + assert not list(tmp_path.glob("*.replaced-*")) and not list(tmp_path.glob(".decompile-*")) + + +@pytest.mark.parametrize("fault", ["refused-input", "gen_subgraphs.py", "gen_tasks", "gen.leaves.yaml", "gen.leaves", "gen.py"]) +def test_any_failure_leaves_the_previous_generation_exactly_intact(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, fault: str) -> None: + """No partial destination state: a refusal, or a failed rename at any of the + five boundaries, leaves the previous script with its own companion, tasks + package, sidecar and bundle.""" + import tangle_cli.pipeline_decompiler as module + + python = _python_leaf(tmp_path) + + def nested(marker: str) -> Path: + child = _graph_spec(f"Child {marker}", {"leaf": _ref(spec=_leaf(marker))}) + return _write(tmp_path / f"{marker}.yaml", _graph_spec("O", {"a": _ref(spec=child), "py": _ref(spec=python)})) + + output = tmp_path / "gen.py" + decompile_pipeline_file(nested("previous"), output) + assert {"gen_subgraphs.py", "gen_tasks/__init__.py", "gen.leaves.yaml"} <= set(_snapshot(tmp_path)), "all five products" + source = nested("next") + if fault == "refused-input": + source = _write(tmp_path / "bad.yaml", _graph_spec("O", {"a": _ref(url="file://./missing.yaml")})) + expected: type[BaseException] = DecompileError + else: + real_replace = os.replace + reached: list[Path] = [] + + def fail_at_boundary(src: Any, dst: Any) -> Any: + # The PUBLICATION rename from staging into the destination -- not a + # rename inside staging, nor the rollback restoring the aside copy. + if Path(dst) == tmp_path / fault and Path(src).parent.name.startswith(".decompile-"): + reached.append(Path(dst)) + raise PermissionError("denied") + return real_replace(src, dst) + + monkeypatch.setattr(module.os, "replace", fail_at_boundary) + expected = PermissionError + before = _snapshot(tmp_path) + + with pytest.raises(expected): + decompile_pipeline_file(source, output) + monkeypatch.undo() + + assert fault == "refused-input" or reached == [tmp_path / fault], "the fault fired at the publication boundary" + assert _snapshot(tmp_path) == before, fault + assert not list(tmp_path.glob("*.replaced-*")) and not list(tmp_path.glob(".decompile-*")) + + +# --------------------------------------------------------------------------- +# Gate (a): the pinned document must reproduce the pipeline, online AND offline + +SIDECAR_PIN = {"url": "resolve://./out.leaves.yaml#leaf"} + + +def _pinned_pair(script_dir: Path, pinned_ref: dict[str, Any], manifest: dict[str, Any] | None = None) -> CanonicalComponents: + """A hydrated/pinned pair with a sidecar that is correct by construction.""" + _write(script_dir / "out.leaves" / "leaf.yaml", _leaf()) + _write(script_dir / "out.leaves.yaml", manifest if manifest is not None else {"leaf": [{"local": "./out.leaves/leaf.yaml"}]}) + return CanonicalComponents( + hydrated=_graph_spec("Outer", {"a": _ref(name="Leaf", spec=_leaf())}), + pinned=_graph_spec("Outer", {"a": {"componentRef": pinned_ref}}), + ) + + +@pytest.mark.parametrize( + ("case", "code"), + [ + ("generated-sidecar", None), + # Each of these resolves correctly TODAY -- content comparison alone + # cannot catch them -- but breaks once the script moves. + ("absolute-sidecar", "pinning-incomplete"), + ("traversal", "pinning-incomplete"), + ("foreign-manifest", "pinning-incomplete"), + ("bare-file", "pinning-incomplete"), + ("entry-escapes-bundle", "pinning-incomplete"), + ("missing-sidecar", "pinning-unresolvable"), + ("undefined-fragment", "pinning-unresolvable"), + ("content-differs", "pinning-content-mismatch"), + ], +) +def test_gate_a_accepts_only_the_exact_portable_generated_sidecar(tmp_path: Path, case: str, code: str | None) -> None: + script_dir = tmp_path / "s" + manifest = {"leaf": [{"local": "./elsewhere/leaf.yaml"}]} if case == "entry-escapes-bundle" else None + canonical = _pinned_pair(script_dir, SIDECAR_PIN, manifest) + _pinned_pair(tmp_path, SIDECAR_PIN) # a twin in the parent, so traversal resolves + _write(script_dir / "elsewhere" / "leaf.yaml", _leaf()) + shutil.copy(script_dir / "out.leaves.yaml", script_dir / "other.yaml") + pins = { + "absolute-sidecar": {"url": f"resolve://{script_dir / 'out.leaves.yaml'}#leaf"}, + "traversal": {"url": "resolve://./../out.leaves.yaml#leaf"}, + "foreign-manifest": {"url": "resolve://./other.yaml#leaf"}, + "bare-file": {"url": "file://./out.leaves/leaf.yaml"}, + "undefined-fragment": {"url": "resolve://./out.leaves.yaml#never-generated"}, + } + if case in pins: + canonical.pinned["implementation"]["graph"]["tasks"]["a"]["componentRef"] = pins[case] + if case == "missing-sidecar": + (script_dir / "out.leaves.yaml").unlink() + if case == "content-differs": + _write(script_dir / "out.leaves" / "leaf.yaml", _leaf("different")) + + check = lambda: assert_pinning_preserved(canonical, script_dir=script_dir, sidecar="out.leaves.yaml") # noqa: E731 + if code is None: + check() + return + with pytest.raises(DecompileError) as excinfo: + check() + assert excinfo.value.code == code + + +def _sabotage_copies(monkeypatch: pytest.MonkeyPatch, marker: str, content: str | None) -> None: + """Replace -- or drop -- only the local copies whose text contains ``marker``.""" + from tangle_cli.pipeline_dehydrator import PipelineDehydrator + + original = PipelineDehydrator._write_text + + def sabotaged(self, destination, text, *, kind="output"): # type: ignore[no-untyped-def] + if kind != "component" or marker not in text: + return original(self, destination, text, kind=kind) + return original(self, destination, content, kind=kind) if content is not None else None + + monkeypatch.setattr(PipelineDehydrator, "_write_text", sabotaged) + + +@pytest.mark.parametrize("nested", [False, True], ids=["root", "nested"]) +@pytest.mark.parametrize( + ("published", "damage", "code"), + [ + # A healthy remote primary short-circuits online resolution, so only the + # forced-offline pass can see these; they would ship broken otherwise. + (True, "missing", "pinning-incomplete"), + (True, "corrupted", "pinning-content-mismatch"), + # With no primary, the local copy IS the online path. + (False, "missing", "pinning-unresolvable"), + (False, "corrupted", "pinning-content-mismatch"), + ], +) +def test_a_bad_local_fallback_is_refused_and_nothing_is_replaced( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, nested: bool, published: bool, damage: str, code: str +) -> None: + if published: + source, library = _published_leaf_source(tmp_path, nested=nested) + else: + leaf_task = _ref(spec=_leaf("published-leaf")) + source, library = _pipeline(tmp_path, _nested_with(leaf_task) if nested else leaf_task), None + output = tmp_path / "gen.py" + decompile_pipeline_file(source, output, client=library) + before = _snapshot(tmp_path) + + _sabotage_copies(monkeypatch, "published-leaf", None if damage == "missing" else dump_yaml(_leaf("CORRUPTED"))) + with pytest.raises(DecompileError) as excinfo: + decompile_pipeline_file(source, output, client=library) + + assert excinfo.value.code == code + assert _snapshot(tmp_path) == before + assert not list(tmp_path.glob(".decompile-*")) + + +def _scores(extra_optional_input: bool) -> dict[str, Any]: + """A sanitized stand-in for a real CompositeScores regression. + + Its successor adds one optional input, whose ``if/isPresent`` block lands + before the ``outputPath`` items and so shifts the whole command array. + """ + def optional(name: str) -> dict[str, Any]: + return {"if": {"cond": {"isPresent": name}, "then": [f"--{name}", {"inputValue": name}]}} + + inputs = [{"name": n, "type": "String"} for n in ("run_id", "table_a", "table_b")] + inputs.append({"name": "out_table", "type": "String", "optional": True}) + command: list[Any] = ["python", "-m", "--run_id", {"inputValue": "run_id"}, "--table_a", {"inputValue": "table_a"}] + command += ["--table_b", {"inputValue": "table_b"}, optional("out_table")] + if extra_optional_input: + inputs.append({"name": "new_flag", "type": "String", "optional": True}) + command.append(optional("new_flag")) + command += ["--o1", {"outputPath": "o1"}, "--o2", {"outputPath": "o2"}] + return { + "name": "CompositeScores", + "inputs": inputs, + "outputs": [{"name": "o1"}, {"name": "o2"}], + "implementation": {"container": {"image": "example/scores:1", "command": command}}, + } + + +@pytest.mark.parametrize("successor", ["different", "identical"]) +@pytest.mark.parametrize("shape", ["hydrated", "dehydrated-digest"]) +def test_seed_time_exactness_ignores_deprecation_successors(tmp_path: Path, shape: str, successor: str) -> None: + """Decompile is a one-time seeding transform: it reproduces the EXACT + component the input references. A deprecation successor -- even one that + changes the command -- is neither seeded nor used to judge the output; + later hydrate/submit/run still upgrade, which is the maintainer's call.""" + seeded = _scores(extra_optional_input=False) + newer = _scores(extra_optional_input=successor == "different") + digest, successor_digest = compute_text_digest(dump_yaml(seeded)), "5" * 64 + library = _Library({digest: seeded, successor_digest: newer}, successors={digest: successor_digest}) + leaf = _ref(digest=digest, name="CompositeScores", spec=seeded) if shape == "hydrated" else _ref(digest=digest) + source = _pipeline(tmp_path, _nested_with(leaf)) + output = tmp_path / "out" / "gen.py" + + assert f"ref(digest={digest!r})" in decompile_pipeline_file(source, tmp_path / "default" / "gen.py", client=library).source + ( + tmp_path / "default" / "gen_subgraphs.py" + ).read_text(encoding="utf-8"), "the seeded digest, not its successor" + result = decompile_pipeline_file(source, output, client=library, include_local_fallbacks=True) + + assert result.verified + manifest = parse_yaml_string(output.with_name("gen.leaves.yaml").read_text(encoding="utf-8")) + leaf_entries = next(v for v in manifest.values() if v and "digest" in v[0]) + assert leaf_entries[0] == {"digest": digest, "fallback_on_error": True}, "the seeded digest, not its successor" + local = parse_yaml_string((output.parent / leaf_entries[1]["local"]).read_text(encoding="utf-8")) + assert compute_spec_digest(local) == compute_spec_digest(seeded), "the local copy is the exact seeded content" + + +# --------------------------------------------------------------------------- +# Privacy: no untrusted document or component value reaches any output channel + + +@pytest.mark.parametrize( + "case", + ["url-userinfo", "url-signed-query", "malformed-child-yaml", "successful-pin-names"], +) +def test_no_document_value_reaches_the_message_or_the_streams(tmp_path: Path, case: str) -> None: + marker = "SYNTHETIC_SECRET_136" + if case == "url-userinfo": + source = _pipeline(tmp_path, _ref(url=f"file://user:{marker}@/nope.yaml")) + elif case == "url-signed-query": + source = _pipeline(tmp_path, _ref(url=f"https://example.invalid/c.yaml?sig={marker}")) + elif case == "malformed-child-yaml": + (tmp_path / "leaf.yaml").write_text(f"name: L\n\t{marker}: [x\n", encoding="utf-8") + source = _pipeline(tmp_path, _ref(url="file://./leaf.yaml")) + else: + _write(tmp_path / "leaf.yaml", {**_leaf(), "name": marker}) + source = _pipeline(tmp_path, _ref(url="file://./leaf.yaml")) + + streams, message = _captured(lambda: decompile_pipeline_file(source, tmp_path / "o.py")) + + assert marker not in message and marker.lower() not in streams.lower() + if case.startswith("url-"): + assert "[redacted]" in message, "the locator is still reported, minus its credentials" + if case == "successful-pin-names": + assert message == "", "this run must succeed, so the pin path really ran" + + +def test_a_verbose_api_client_does_not_echo_the_fetched_spec(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """The quiet logger must reach the API client the hydrator builds internally.""" + import json + + import requests + + import tangle_cli.client as client_module + + spec = {**_leaf(), "description": "SYNTHETIC_VERBOSE_VALUE_136"} + digest = compute_text_digest(dump_yaml(spec)) + payload = {"spec": spec, "digest": digest} + body = json.dumps(payload) + + class _Response: + status_code = 200 + headers = {"Content-Type": "application/json"} + text = body + content = body.encode("utf-8") + + def json(self) -> dict[str, Any]: + return payload + + def raise_for_status(self) -> None: + pass + + monkeypatch.setenv("TANGLE_VERBOSE", "1") + monkeypatch.setenv("TANGLE_API_URL", "https://review.invalid") + monkeypatch.setattr(requests.Session, "request", lambda self, m, u, **k: _Response()) + monkeypatch.setattr(client_module.TangleApiClient, "resolve_digest", lambda self, d: d) + output = tmp_path / "o.py" + + streams, message = _captured(lambda: decompile_pipeline_file(_pipeline(tmp_path, _ref(digest=digest)), output)) + + assert message == "" and output.is_file(), "the fetch path must actually run" + assert "SYNTHETIC_VERBOSE_VALUE_136" not in streams + + +def test_only_an_explicit_logger_is_forwarded_to_internal_api_clients() -> None: + """Forwarding the DEFAULT logger would override the client's own verbosity + gating and make ordinary runs start logging. No public seam exposes this, + so it is asserted on the option the hydrator passes to the client.""" + quiet = object() + assert "logger" not in PipelineHydrator()._client_options + assert PipelineHydrator(logger=quiet)._client_options.get("logger") is quiet # type: ignore[arg-type] + + +# --------------------------------------------------------------------------- +# Every graph becomes Python: nested graphs are @pipeline functions in the +# companion _subgraphs.py, called through subpipeline(...) + + +@pytest.fixture(autouse=True) +def _forget_generated_companions() -> Any: + """The compiler imports a companion by NAME, so evict any from a previous + test -- otherwise its cached module would silently stand in for this one.""" + import sys + + def evict() -> None: + for name in [m for m in sys.modules if m.endswith("_subgraphs")]: + sys.modules.pop(name) + + evict() + yield + evict() + + +def _io_leaf(marker: str) -> dict[str, Any]: + return { + "name": "Leaf", + "inputs": [{"name": "x", "type": "String"}], + "outputs": [{"name": "y", "type": "String"}], + "implementation": {"container": {"image": "i", "command": [marker, {"inputValue": "x"}, {"outputPath": "y"}]}}, + } + + +def _io_graph(name: str, tasks: dict[str, Any], *, output_from: str | None = None, inputs: tuple[str, ...] = ()) -> dict[str, Any]: + spec = _graph_spec(name, tasks) + if inputs: + spec["inputs"] = [{"name": n, "type": "String"} for n in inputs] + if output_from: + spec["outputs"] = [{"name": "out", "type": "String"}] + spec["implementation"]["graph"]["outputValues"] = {"out": _out(output_from, "y")} + return spec + + +def _nested_document() -> dict[str, Any]: + """Root -> Mid -> Inner (shared by two tasks, chained by an edge), plus two + same-name 'Judge' graphs whose content differs.""" + inner = _io_graph("Inner", {"leaf": {**_ref(spec=_io_leaf("inner")), "arguments": {"x": {"graphInput": {"inputName": "seed"}}}}}, output_from="leaf", inputs=("seed",)) + judge_a = _io_graph("Judge", {"leaf": {**_ref(spec=_io_leaf("judge-a")), "arguments": {"x": "1"}}}, output_from="leaf") + judge_b = _io_graph("Judge", {"leaf": {**_ref(spec=_io_leaf("judge-b")), "arguments": {"x": "1"}}}, output_from="leaf") + mid = _graph_spec("Mid", { + "i1": {**_ref(spec=inner), "arguments": {"seed": "s1"}, "annotations": {"k": "v"}}, + "i2": {**_ref(spec=inner), "arguments": {"seed": _out("i1", "out")}}, + "j": _ref(spec=judge_a), + }) + return _graph_spec("Root", {"mid": _ref(spec=mid), "judge": _ref(spec=judge_b)}) + + +def _functions(path: Path) -> list[str]: + return [n.name for n in ast.parse(path.read_text(encoding="utf-8")).body if isinstance(n, ast.FunctionDef)] + + +def test_every_nested_graph_becomes_a_python_pipeline_and_round_trips(tmp_path: Path) -> None: + document = _nested_document() + source = _write(tmp_path / "in.yaml", document) + work = tmp_path / "work" + + result = decompile_pipeline_file(source, work / "gen.py") + + assert result.verified and result.subgraph_module == work / "gen_subgraphs.py" + assert _functions(work / "gen.py") == ["root"] + children = _functions(work / "gen_subgraphs.py") + assert children[:3] == ["inner", "judge", "mid"], "deepest first, so every call is to an earlier definition" + assert len(children) == 4 and children[3].startswith("judge_"), "same name, different content: suffixed" + assert (work / "gen_subgraphs.py").read_text(encoding="utf-8").count("subpipeline(inner)") == 2, "a shared graph is one function" + bundle = [parse_yaml_string((work / "gen.leaves" / f).read_text(encoding="utf-8")) for f in result.component_files] + assert all("container" in spec["implementation"] for spec in bundle), "no graph is left as YAML" + + again = tmp_path / "again" + decompile_pipeline_file(source, again / "gen.py") + for name in ("gen.py", "gen_subgraphs.py"): + assert (again / name).read_text(encoding="utf-8") == (work / name).read_text(encoding="utf-8"), "deterministic" + + moved = tmp_path / "moved" + shutil.copytree(work, moved) + shutil.rmtree(work) + compile_pipeline(moved / "gen.py", moved / "compiled.yaml") + hydrated = PipelineHydrator(client=_UnreachableLibrary()).hydrate_file(moved / "compiled.yaml").data + assert sorted(_leaf_specs(hydrated)) == sorted(_leaf_specs(document)) + + +@pytest.mark.parametrize(("key", "value"), [("isEnabled", "true"), ("executionOptions", {"cachingStrategy": {"maxCacheStaleness": "P0D"}})]) +def test_graph_task_metadata_that_subpipeline_cannot_express_is_refused(tmp_path: Path, key: str, value: Any) -> None: + """Tangle does not apply these to graph-component tasks, and subpipeline(...) rejects them.""" + boundary = {**_ref(spec=_graph_spec("Child", {"leaf": _ref(spec=_leaf())})), key: value} + source = _write(tmp_path / "in.yaml", _graph_spec("Root", {"sub": boundary})) + with pytest.raises(DecompileError) as excinfo: + decompile_pipeline_file(source, tmp_path / "out.py") + assert excinfo.value.code == "unsupported-subgraph-task" + assert sorted(p.name for p in tmp_path.iterdir()) == ["in.yaml"] + + +def test_a_cycle_between_referenced_graphs_is_refused_by_name(tmp_path: Path) -> None: + _write(tmp_path / "a.yaml", _graph_spec("A", {"t": _ref(url="file://./b.yaml")})) + _write(tmp_path / "b.yaml", _graph_spec("B", {"t": _ref(url="file://./a.yaml")})) + source = _write(tmp_path / "in.yaml", _graph_spec("Root", {"t": _ref(url="file://./a.yaml")})) + with pytest.raises(DecompileError) as excinfo: + decompile_pipeline_file(source, tmp_path / "out" / "gen.py") + assert excinfo.value.code == "cyclic-component-reference" + assert not (tmp_path / "out" / "gen.py").exists() + + +def test_a_real_shape_twelve_graph_pipeline_decompiles_to_python(tmp_path: Path) -> None: + """The sanitized shape of a real export: 6 scrape graphs (staging and + production) each holding a judge graph, where two same-name judges differ + only by provenance -- 12 graph instances, 9 distinct graphs.""" + def judge(name: str, provenance: str) -> dict[str, Any]: + spec = _graph_spec(name, {"score": _ref(spec=_leaf(f"score-{name}")), "rank": _ref(spec=_leaf("rank"))}) + return {**spec, "metadata": {"annotations": {"component_yaml_path": provenance}}} + + variants = [("Judge Evaluate v5.0", "judge-47d4.yaml"), ("Judge Evaluate v5.0", "judge-f899.yaml"), ("Judge Evaluate storefront_v2.0", "sf.yaml")] + tasks = { + f"{env} {index}": _ref(spec=_graph_spec(f"{env} scrape {index}", {"scrape": _ref(spec=_leaf(f"scrape-{env}-{index}")), "judge": _ref(spec=judge(*variant))})) + for env in ("Staging", "Production") + for index, variant in enumerate(variants) + } + document = _graph_spec("Daily Pulse - Real Time", tasks) + work = tmp_path / "work" + + result = decompile_pipeline_file(_write(tmp_path / "in.yaml", document), work / "gen.py") + + assert result.verified + children = _functions(work / "gen_subgraphs.py") + assert len(children) == 9 + assert sum(name.startswith("judge_evaluate_v5_0") for name in children) == 2, "provenance-only difference: two functions" + compile_pipeline(work / "gen.py", work / "compiled.yaml") + hydrated = PipelineHydrator(client=_UnreachableLibrary()).hydrate_file(work / "compiled.yaml").data + assert sorted(_leaf_specs(hydrated)) == sorted(_leaf_specs(document)) + + + +def test_consecutive_runs_in_one_process_never_see_a_stale_companion(tmp_path: Path) -> None: + """The root imports its companion by name, so a companion cached by an + earlier run -- lacking this run's child functions -- must not be reused.""" + import sys + + for index, child in enumerate(("Child", "Other")): + directory = tmp_path / f"run{index}" + document = _graph_spec("Root", {"sub": _ref(spec=_graph_spec(child, {"leaf": _ref(spec=_leaf(child))}))}) + assert decompile_pipeline_file(_write(directory / "in.yaml", document), directory / "gen.py").verified + assert "gen_subgraphs" not in sys.modules, "nothing lingers into a later run" + + +# --------------------------------------------------------------------------- +# The generated module namespace: one allocator, and one-to-one companion names + + +def _child(name: str, marker: str) -> dict[str, Any]: + return _ref(spec=_graph_spec(name, {"leaf": _ref(spec=_leaf(marker))})) + + +def _namespace_case(case: str) -> dict[str, Any]: + if case == "task-local": + return _graph_spec("Root", {"Load": _child("Task Load", "a")}) + if case == "input-local": + return {**_graph_spec("Root", {"t": _child("Input Seed", "a")}), "inputs": [{"name": "seed", "type": "String"}]} + if case == "helper-names": + return _graph_spec("Root", {f"t{i}": _child(name, name) for i, name in enumerate(("ref", "subpipeline", "task", "pipeline", "In", "Graph Input"))}) + if case == "adversarial-suffix": + # Filled in by the test: a graph literally NAMED like the suffix the + # second 'Judge' is given. + return _graph_spec("Root", {"a": _child("Judge", "a"), "c": _child("Judge", "b")}) + if case == "nested-child-calls-child": + return _graph_spec("Root", {"m": _ref(spec=_graph_spec("Mid", {"Load": _child("Task Load", "a")}))}) + raise AssertionError(case) + + +@pytest.mark.parametrize("case", ["task-local", "input-local", "helper-names", "adversarial-suffix", "nested-child-calls-child"]) +def test_child_function_names_never_collide_in_the_generated_module(tmp_path: Path, case: str) -> None: + """No child may be shadowed by a local, shadow a helper, or silently replace + another definition -- with verification on OR off, and stably.""" + document = _namespace_case(case) + if case == "adversarial-suffix": + probe = tmp_path / "probe" + decompile_pipeline_file(_write(probe / "in.yaml", document), probe / "gen.py") + (suffixed,) = [n for n in _functions(probe / "gen_subgraphs.py") if n != "judge"] + tasks = document["implementation"]["graph"]["tasks"] + document["implementation"]["graph"]["tasks"] = {"a": tasks["a"], "b": _child(suffixed, "literal"), "c": tasks["c"]} + source = _write(tmp_path / "in.yaml", document) + for label, verify in (("verified", True), ("unverified", False), ("repeat", True)): + directory = tmp_path / label + result = decompile_pipeline_file(source, directory / "gen.py", verify=verify) + assert result.verified is verify + names = _functions(directory / "gen_subgraphs.py") + assert len(names) == len(set(names)), f"{label}: a definition would replace another" + assert not {"ref", "subpipeline", "task", "pipeline", "In", "Out", "graph_input"} & set(names) + assert not [n for n in names if n.startswith(("input_", "task_"))] + assert (tmp_path / "repeat" / "gen_subgraphs.py").read_text(encoding="utf-8") == (tmp_path / "verified" / "gen_subgraphs.py").read_text(encoding="utf-8") + compile_pipeline(tmp_path / "verified" / "gen.py", tmp_path / "verified" / "compiled.yaml") + hydrated = PipelineHydrator(client=_UnreachableLibrary()).hydrate_file(tmp_path / "verified" / "compiled.yaml").data + assert sorted(_leaf_specs(hydrated)) == sorted(_leaf_specs(document)) + + +@pytest.mark.parametrize(("stem", "nested", "accepted"), [ + ("foo-bar", True, False), # would alias foo_bar's companion + ("1pipe", True, False), + ("my.pipeline", True, False), + ("résumé", True, False), # non-ASCII module names may not import + ("class", True, True), # class_subgraphs is a valid module name + ("foo-bar", False, True), # no nested graphs, no companion: any name +]) +def test_a_companion_name_must_be_a_one_to_one_module_name(tmp_path: Path, stem: str, nested: bool, accepted: bool) -> None: + decompile_pipeline_file(_write(tmp_path / "a.yaml", _graph_spec("First", {"t": _child("First", "first")})), tmp_path / "foo_bar.py") + document = _graph_spec("Second", {"t": _child("Second", "second")} if nested else {"t": _ref(spec=_leaf("second"))}) + source = _write(tmp_path / "b.yaml", document) + before = _snapshot(tmp_path) + + if not accepted: + with pytest.raises(DecompileError) as excinfo: + decompile_pipeline_file(source, tmp_path / f"{stem}.py") + assert excinfo.value.code == "unsupported-output-name" + assert _snapshot(tmp_path) == before, "the neighbouring output and its companion are untouched" + return + assert decompile_pipeline_file(source, tmp_path / f"{stem}.py").verified + compile_pipeline(tmp_path / "foo_bar.py", tmp_path / "first.yaml") # the neighbour still compiles + + +def test_every_leaf_is_retargeted_once_even_when_slash_joined_paths_coincide(tmp_path: Path) -> None: + """Task ids may contain ``/``: four distinct leaves whose ids all join to + ``a/b/c`` must each be retargeted exactly once.""" + deepest = _graph_spec("Deepest", {"c": _ref(spec=_leaf("a>b>c"))}) + document = _graph_spec("Root", { + "a/b/c": _ref(spec=_leaf("a/b/c")), + "a": _ref(spec=_graph_spec("A", {"b/c": _ref(spec=_leaf("a>b/c")), "b": _ref(spec=deepest)})), + "a/b": _ref(spec=_graph_spec("AB", {"c": _ref(spec=_leaf("a/b>c"))})), + }) + source = _write(tmp_path / "in.yaml", document) + + result = decompile_pipeline_file(source, tmp_path / "gen.py") + + assert result.verified + generated = (tmp_path / "gen.py").read_text(encoding="utf-8") + (tmp_path / "gen_subgraphs.py").read_text(encoding="utf-8") + fragments = re.findall(r"resolve://\./gen\.leaves\.yaml#([^'\"]+)", generated) + assert len(fragments) == len(set(fragments)) == 4, "every leaf once, none duplicated or omitted" + assert ".components.yaml" not in generated + manifest = parse_yaml_string((tmp_path / "gen.leaves.yaml").read_text(encoding="utf-8")) + assert set(fragments) == set(manifest) + + +# --------------------------------------------------------------------------- +# Python components: a static source transform; Tangle compiles them later + +_GREET = '''import pathlib +pathlib.Path(__file__).with_name("SENTINEL").write_text("imported") + + +def greet(name: str, shout: bool = False): + """Greet someone. + + Metadata: + Name: Greet + Version: 1.0.0 + """ + print(name.upper() if shout else name) +''' + + +def _python_leaf(tmp_path: Path, source: str = _GREET, *, dependencies: tuple[str, ...] = ()) -> dict[str, Any]: + """A leaf made by the real Python component generator (which imports the + source -- this is the test authoring it, never the decompiler).""" + from tangle_cli.component_generator import regenerate_yaml + + authoring = tmp_path / "authoring" + authoring.mkdir(exist_ok=True) + (authoring / "greet_task.py").write_text(source, encoding="utf-8") + deps = authoring / "deps.toml" + deps.write_text("[project]\ndependencies = [" + ", ".join(f'"{d}"' for d in dependencies) + "]\n", encoding="utf-8") + assert regenerate_yaml( + python_file=authoring / "greet_task.py", output_path=authoring / "greet.yaml", image="python:3.12", function_name="greet", dependencies_from=deps + ) + leaf = parse_yaml_string((authoring / "greet.yaml").read_text(encoding="utf-8")) + shutil.rmtree(authoring) + return leaf + + +def _provenance_free(spec: dict[str, Any]) -> dict[str, Any]: + """What Tangle must rebuild exactly: everything but generator provenance.""" + from tangle_cli.pipeline_decompiler import _GENERATOR_ANNOTATIONS + + provenance = _GENERATOR_ANNOTATIONS - {"python_original_code", "python_dependencies", "tangle_cli_generation_function_name", "tangle_cli_generation_mode"} + copied = copy.deepcopy(spec) + annotations = copied["metadata"]["annotations"] + for key in provenance & set(annotations) - {"cloud_pipelines.net", "components new regenerate python-function-component"}: + del annotations[key] + return copied + + +def test_python_components_become_tasks_without_running_their_source(tmp_path: Path) -> None: + """Root and a child share one recovered task (renamed away from the child + function ``greet``); a plain leaf stays pinned YAML. Nothing recovered runs + until Tangle compiles, after which the rebuilt specs match but for + provenance.""" + leaf = _python_leaf(tmp_path, dependencies=("requests==2.31.0", "demo @ https://example.invalid/\U0001F600/demo.whl")) + plain = _leaf("plain") + child = _graph_spec("Greet", {"inner": {**_ref(spec=leaf), "arguments": {"name": "b"}}}) + document = _graph_spec("Root", { + "a": {**_ref(spec=leaf), "arguments": {"name": "a", "shout": "true"}}, + "c": _ref(spec=child), + "p": _ref(spec=plain), + }) + work = tmp_path / "work" + + result = decompile_pipeline_file(_write(tmp_path / "in.yaml", document), work / "gen.py") + + assert result.verified and (result.python_tasks, result.python_candidates) == (1, 1) + assert not list(tmp_path.rglob("SENTINEL")), "decompile never imports recovered source" + package = work / "gen_tasks" + assert result.tasks_package == package + assert (package / "greet_task" / "greet_task.py").read_bytes() == leaf["metadata"]["annotations"]["python_original_code"].encode("utf-8") + assert "requests==2.31.0" in (package / "greet_task" / "greet_task.deps.toml").read_text(encoding="utf-8") + assert _functions(work / "gen_subgraphs.py") == ["greet"] + root_source, companion_source = (work / "gen.py").read_text(encoding="utf-8"), (work / "gen_subgraphs.py").read_text(encoding="utf-8") + (task_name,) = re.findall(r"from gen_tasks import (\w+)", root_source) + assert task_name != "greet" and f"from gen_tasks import {task_name}" in companion_source + manifest = parse_yaml_string((work / "gen.leaves.yaml").read_text(encoding="utf-8")) + assert len(manifest) == 1 and root_source.count("resolve://./gen.leaves.yaml#") == 1, "only the plain leaf is pinned" + assert len(list((work / "gen.leaves").iterdir())) == 1, "the converted leaf's local copy is pruned" + + moved = tmp_path / "moved" + shutil.move(str(work), moved) + compile_pipeline(moved / "gen.py", moved / "gen.yaml") + assert (moved / "gen_tasks" / "greet_task" / "SENTINEL").is_file(), "Tangle's compile is what imports it" + # Hydrate runs it too, so it needs explicit trust outside the cwd tree. + hydrated = PipelineHydrator(client=_UnreachableLibrary(), trusted_python_sources=[moved]).hydrate_file(moved / "gen.yaml").data + tasks = hydrated["implementation"]["graph"]["tasks"] + for spec in (tasks["a"]["componentRef"]["spec"], tasks["c"]["componentRef"]["spec"]["implementation"]["graph"]["tasks"]["inner"]["componentRef"]["spec"]): + assert _provenance_free(spec) == _provenance_free(leaf) + assert tasks["a"]["arguments"] == {"name": "a", "shout": "true"} + assert tasks["p"]["componentRef"]["spec"] == plain + + +def test_a_sidecar_like_argument_is_data_not_a_reference(tmp_path: Path) -> None: + """Only generated ref(url=...) locators are checked after pruning; the same + text as an ordinary argument value is user data.""" + leaf = _python_leaf(tmp_path) + data = "resolve://./gen.leaves.yaml#business-data" + source = _write(tmp_path / "in.yaml", _graph_spec("Root", {"a": {**_ref(spec=leaf), "arguments": {"name": data}}})) + + result = decompile_pipeline_file(source, tmp_path / "gen.py") + + assert result.verified and result.python_tasks == 1 + assert repr(data) in (tmp_path / "gen.py").read_text(encoding="utf-8") + + +@pytest.mark.parametrize("dependency", ["plain==1.0", "astral \U0001F600", 'quote " and back\\slash', "control \x01 and del \x7f", "é BMP"]) +def test_dependencies_round_trip_through_toml(dependency: str) -> None: + from tangle_cli.component_from_func import tomllib + from tangle_cli.pipeline_decompiler import _dependencies_toml + + assert tomllib.loads(_dependencies_toml([dependency, "second"]))["project"]["dependencies"] == [dependency, "second"] + + +def test_no_python_tasks_keeps_every_leaf_pinned_and_removes_a_stale_package(tmp_path: Path) -> None: + leaf = _python_leaf(tmp_path) + source = _write(tmp_path / "in.yaml", _graph_spec("Root", {"a": _ref(spec=leaf)})) + decompile_pipeline_file(source, tmp_path / "gen.py") + assert (tmp_path / "gen_tasks").is_dir() + + result = decompile_pipeline_file(source, tmp_path / "gen.py", python_tasks=False) + + assert result.verified and (result.python_tasks, result.python_candidates, result.tasks_package) == (0, 1, None) + assert not (tmp_path / "gen_tasks").exists() and "gen_tasks" not in (tmp_path / "gen.py").read_text(encoding="utf-8") + assert len(parse_yaml_string((tmp_path / "gen.leaves.yaml").read_text(encoding="utf-8"))) == 1 + + with pytest.raises(DecompileError) as excinfo: + decompile_pipeline_file(source, tmp_path / "foo-bar.py") + assert excinfo.value.code == "unsupported-output-name" + assert decompile_pipeline_file(source, tmp_path / "foo-bar.py", python_tasks=False).verified + assert not list(tmp_path.rglob("SENTINEL")) + + +def _unprovable(case: str, leaf: dict[str, Any]) -> dict[str, Any]: + leaf = copy.deepcopy(leaf) + annotations = leaf["metadata"]["annotations"] + code = annotations["python_original_code"] + if case == "decorated": + annotations["python_original_code"] = code.replace("def greet(", "@staticmethod\ndef greet(") + elif case == "bundle-mode": + annotations["tangle_cli_generation_mode"] = "bundle" + elif case == "custom-name": + leaf["name"] = "Renamed" + elif case == "custom-annotation": + annotations["owner"] = "someone" + elif case == "synthetic-input": + leaf["inputs"].append({"name": "unwrapped_field", "type": "String"}) + elif case == "missing-marker": + del annotations["components new regenerate python-function-component"] + elif case == "resolve-root": + annotations["tangle_cli_generation_resolve_root"] = "src" + elif case == "unparseable": + annotations["python_original_code"] = code + "\ndef broken(:\n" + elif case == "rebound": + annotations["python_original_code"] = code + "\ngreet = print\n" + elif case == "conditional-import": + annotations["python_original_code"] = code + '\nif __name__.startswith("gen_tasks."):\n import math as greet\n' + elif case == "try-def": + annotations["python_original_code"] = code + "\ntry:\n pass\nexcept ImportError:\n def greet(name):\n pass\n" + elif case == "walrus": + annotations["python_original_code"] = code + "\nif (greet := print):\n pass\n" + elif case == "class-bases": + annotations["python_original_code"] = code + '\nclass Shadow((greet := str) if __name__.startswith("gen_tasks.") else object):\n pass\n' + elif case == "def-annotation": + annotations["python_original_code"] = code + '\ndef helper(x: ((greet := str) if __name__.startswith("gen_tasks.") else str)):\n pass\n' + elif case == "match-guard": + annotations["python_original_code"] = code + '\nmatch __name__.startswith("gen_tasks."):\n case True if (greet := str):\n pass\n' + elif case == "except-type": + annotations["python_original_code"] = code + '\ntry:\n raise Exception\nexcept (greet := Exception):\n pass\n' + elif case == "star-import": + annotations["python_original_code"] = code + '\nif __name__.startswith("gen_tasks."):\n from math import *\n' + elif case == "globals": + annotations["python_original_code"] = code + '\nglobals()["greet"] = print\n' + elif case == "container-env": + leaf["implementation"]["container"]["env"] = {"IMPORTANT_SETTING": "required-value"} + elif case == "input-annotations": + leaf["inputs"][0]["annotations"] = {"editor.position": "1"} + elif case == "async": + annotations["python_original_code"] = code.replace("def greet(", "async def greet(") + else: + raise AssertionError(case) + return leaf + + +@pytest.mark.parametrize("case", [ + "decorated", "bundle-mode", "custom-name", "custom-annotation", "synthetic-input", + "missing-marker", "resolve-root", "unparseable", "rebound", "async", + "conditional-import", "try-def", "walrus", "globals", "container-env", "input-annotations", + "class-bases", "def-annotation", "match-guard", "except-type", "star-import", +]) +def test_python_leaves_the_static_contract_cannot_prove_stay_pinned_yaml(tmp_path: Path, case: str) -> None: + leaf = _unprovable(case, _python_leaf(tmp_path)) + source = _write(tmp_path / "in.yaml", _graph_spec("Root", {"a": _ref(spec=leaf)})) + + result = decompile_pipeline_file(source, tmp_path / "gen.py") + + assert result.verified and result.python_tasks == 0 and result.tasks_package is None + assert result.python_candidates == (0 if case == "missing-marker" else 1) + assert not (tmp_path / "gen_tasks").exists() and not list(tmp_path.rglob("SENTINEL")) + + +# --------------------------------------------------------------------------- +# Decorated @task sources: literal image_id/unwrap, re-exported as they are + +_SCORE = '''import pathlib +from typing import Dict + +from cloud_pipelines.components import OutputPath +from tangle_cli.python_pipeline import task + +pathlib.Path(__file__).with_name("SENTINEL").write_text("imported") + + +@task(image_id="scoring", unwrap="settings") +def score(settings: Dict[str, str], settings_path: OutputPath("Text")) -> str: + """Score a batch. + + Metadata: + Name: Score + Version: 2.0.0 + """ + open(settings_path, "w").write(str(settings)) + return "ok" +''' +_SCORE_CALL = "score.named('Score')(settings={'alpha': '1', 'beta': '2', 'gamma-x': '3'})" + + +def _authored_task(tmp_path: Path, source: str = _SCORE, *, files: dict[str, str] | None = None, module: str = "score_task") -> dict[str, Any]: + """A task body exactly as python_pipeline authoring produces it: compiled + and hydrated by Tangle itself, which runs the source (the test authoring + it -- never the decompiler).""" + import sys + + authoring = tmp_path / f"authoring-{module}" + authoring.mkdir() + (authoring / f"{module}.py").write_text(source, encoding="utf-8") + for name, text in (files or {}).items(): + (authoring / name).write_text(text, encoding="utf-8") + (authoring / "pipe.py").write_text( + f"from tangle_cli.python_pipeline import pipeline\nfrom {module} import score\n\n" + f"@pipeline(name='Authored')\ndef authored() -> None:\n {_SCORE_CALL}\n", + encoding="utf-8", + ) + sys.modules.pop(module, None) + try: + compile_pipeline(authoring / "pipe.py", authoring / "pipe.yaml", image_overrides={"scoring": "python:3.12"}) + hydrated = PipelineHydrator(trusted_python_sources=[str(authoring)]).hydrate_file(authoring / "pipe.yaml").data + finally: + sys.modules.pop(module, None) + shutil.rmtree(authoring) + body = hydrated["implementation"]["graph"]["tasks"]["Score"] + return {"componentRef": {"spec": body["componentRef"]["spec"]}, "arguments": body["arguments"]} + + +def test_a_real_shape_image_id_unwrap_task_converts_without_running_its_source(tmp_path: Path) -> None: + """The shape of the real marked leaf: @task(image_id=..., unwrap=...) from + a typing.Dict parameter, all inputs synthetic, outputs from an OutputPath + (named like the unwrap parameter) and the return. Shared by the + root and a child named like it; re-exported verbatim; image_id left for + the maintainer's compile to resolve.""" + import sys + + task = _authored_task(tmp_path) + document = _graph_spec("Root", {"Score": task, "c": _ref(spec=_graph_spec("Score", {"Score": copy.deepcopy(task)}))}) + source = _write(tmp_path / "in.yaml", document) + work = tmp_path / "work" + + result = decompile_pipeline_file(source, work / "gen.py") + + assert result.verified and (result.python_tasks, result.image_id_tasks, result.unwrap_tasks) == (1, 1, 1) + assert not list(tmp_path.rglob("SENTINEL")) + module = work / "gen_tasks" / "score_task" + assert (module / "score_task.py").read_text(encoding="utf-8") == _SCORE, "byte-verbatim, decorator included" + init = (work / "gen_tasks" / "__init__.py").read_text(encoding="utf-8") + assert "task(" not in init.split('"""')[2], "re-exported, never wrapped a second time" + (name,) = re.findall(r"from gen_tasks import (\w+)", (work / "gen.py").read_text(encoding="utf-8")) + assert _functions(work / "gen_subgraphs.py") == ["score"] and name != "score", "one namespace with the child function" + assert "settings={'alpha': '1', 'beta': '2', 'gamma-x': '3'}" in (work / "gen.py").read_text(encoding="utf-8") + assert not (work / "gen.leaves.yaml").exists(), "nothing left to pin" + + moved = tmp_path / "moved" + shutil.move(str(work), moved) + for image in ("python:3.12", "python:3.13-slim"): + sys.modules.pop("gen_tasks", None) + compile_pipeline(moved / "gen.py", moved / "gen.yaml", image_overrides={"scoring": image}) + hydrated = PipelineHydrator(trusted_python_sources=[str(moved)]).hydrate_file(moved / "gen.yaml").data + tasks = hydrated["implementation"]["graph"]["tasks"] + rebuilt = tasks["Score"]["componentRef"]["spec"] + assert rebuilt["implementation"]["container"]["image"] == image, "compile resolves image_id in ITS environment" + if image == "python:3.12": + assert _provenance_free(rebuilt) == _provenance_free(task["componentRef"]["spec"]) + assert tasks["Score"]["arguments"] == task["arguments"] + sys.modules.pop("gen_tasks", None) + assert (moved / "gen_tasks" / "score_task" / "SENTINEL").is_file() + + exact = decompile_pipeline_file(source, tmp_path / "exact" / "gen.py", python_tasks=False) + assert exact.verified and exact.python_tasks == 0 and exact.python_candidates == 1 + + +def _decorator_variant(case: str) -> str: + replace = { + "aliased": ("import task\n", "import task as t\n", "@task(", "@t("), + "alias-swap": ("import task\n", "import task as real_task, ref as task\n"), + "attribute": ("from tangle_cli.python_pipeline import task\n", "import tangle_cli.python_pipeline as pp\n", "@task(", "@pp.task("), + "two-decorators": ("@task(", "@staticmethod\n@task("), + "star-kwargs": ('unwrap="settings")', 'unwrap="settings", **{})'), + "dynamic-value": ('image_id="scoring"', "image_id=IMAGE"), + "unsupported-kwarg": ('unwrap="settings")', 'unwrap="settings", annotations={"a": "b"})'), + "both-images": ('image_id="scoring"', 'image_id="scoring", image="python:3.12"'), + "wrong-image": ('image_id="scoring"', 'image="python:3.11"'), + "bundle-mode": ('unwrap="settings")', 'unwrap="settings", mode="bundle")'), + "absolute-deps": ('unwrap="settings")', 'unwrap="settings", dependencies_from="/etc/deps.toml")'), + "parent-deps": ('unwrap="settings")', 'unwrap="settings", dependencies_from="../deps.toml")'), + "py-directory-deps": ('unwrap="settings")', 'unwrap="settings", dependencies_from="__init__.py/deps.toml")'), + "py-directory-deps-case": ('unwrap="settings")', 'unwrap="settings", dependencies_from="__INIT__.PY/deps.toml")'), + "non-toml-deps": ('unwrap="settings")', 'unwrap="settings", dependencies_from="requirements.txt")'), + "not-a-dict": ("settings: Dict[str, str]", "settings: list"), + "dict-from-elsewhere": ("from typing import Dict\n", "from collections import OrderedDict as Dict\n"), + "dict-rebound": ("\n\n@task(", "\nDict = list\n\n\n@task("), + "rebound-task": ("\n\n@task(", "\ntask = task\n\n\n@task("), + }[case] + source = _SCORE + for old, new in zip(replace[::2], replace[1::2]): + assert old in source, case + source = source.replace(old, new, 1) + return source + + +@pytest.mark.parametrize("case", [ + "aliased", "alias-swap", "attribute", "two-decorators", "star-kwargs", "dynamic-value", "unsupported-kwarg", "both-images", + "wrong-image", "bundle-mode", "absolute-deps", "parent-deps", "py-directory-deps", "py-directory-deps-case", "non-toml-deps", "not-a-dict", "rebound-task", + "dict-from-elsewhere", "dict-rebound", +]) +def test_a_decorator_the_static_contract_cannot_prove_stays_pinned_yaml(tmp_path: Path, case: str) -> None: + """Only the recorded source changes: it is all the decompiler reads.""" + task = _authored_task(tmp_path) + task["componentRef"]["spec"]["metadata"]["annotations"]["python_original_code"] = _decorator_variant(case) + source = _write(tmp_path / "in.yaml", _graph_spec("Root", {"Score": task})) + + result = decompile_pipeline_file(source, tmp_path / "gen.py") + + assert result.verified and (result.python_tasks, result.python_candidates) == (0, 1) + assert not (tmp_path / "gen_tasks").exists() and not list(tmp_path.rglob("SENTINEL")) + + +@pytest.mark.parametrize("case", ["missing-synthetic", "extra-synthetic", "bare-unwrap", "one-call-site-of-two"]) +def test_unwrap_call_sites_that_would_compile_differently_stay_yaml(tmp_path: Path, case: str) -> None: + task = _authored_task(tmp_path) + other = copy.deepcopy(task) + arguments = (other if case == "one-call-site-of-two" else task)["arguments"] + if case in ("missing-synthetic", "one-call-site-of-two"): + del arguments["settings__beta"] + elif case == "extra-synthetic": + arguments["settings__zeta"] = "9" + else: + arguments["settings"] = "{}" + tasks = {"Score": task, **({"Other": other} if case == "one-call-site-of-two" else {})} + + source = _write(tmp_path / "in.yaml", _graph_spec("Root", tasks)) + result = decompile_pipeline_file(source, tmp_path / "gen.py") + + assert result.verified and result.python_tasks == 0 and not (tmp_path / "gen_tasks").exists() + # Nothing converts, so no package name is needed either. + assert decompile_pipeline_file(source, tmp_path / "foo-bar.py").verified + + +def test_decorated_sources_with_the_same_relative_dependencies_do_not_collide(tmp_path: Path) -> None: + """Each module has its own directory, so two ``dependencies_from= + "pyproject.toml"`` keep their own recorded lists.""" + import sys + + tasks = {} + for marker, dependency in (("first", "alpha==1"), ("second", "beta==2")): + variant = _SCORE.replace('unwrap="settings")', 'unwrap="settings", dependencies_from="pyproject.toml")').replace("Score a batch.", f"Score a {marker} batch.") + task = _authored_task(tmp_path, variant, files={"pyproject.toml": f'[project]\ndependencies = ["{dependency}"]\n'}, module=f"score_{marker}") + tasks[marker] = task + + result = decompile_pipeline_file(_write(tmp_path / "in.yaml", _graph_spec("Root", tasks)), tmp_path / "work" / "gen.py") + + assert result.verified and result.python_tasks == 2 + lists = sorted((p.parent.name, p.read_text(encoding="utf-8")) for p in (tmp_path / "work" / "gen_tasks").rglob("pyproject.toml")) + assert [name for name, _ in lists] == ["score_first", "score_second"] + assert "alpha==1" in lists[0][1] and "beta==2" in lists[1][1] + sys.modules.pop("gen_tasks", None) + compile_pipeline(tmp_path / "work" / "gen.py", tmp_path / "work" / "gen.yaml", image_overrides={"scoring": "python:3.12"}) + sys.modules.pop("gen_tasks", None) + hydrated = PipelineHydrator(trusted_python_sources=[str(tmp_path / "work")]).hydrate_file(tmp_path / "work" / "gen.yaml").data + for marker in ("first", "second"): + rebuilt = hydrated["implementation"]["graph"]["tasks"][marker]["componentRef"]["spec"] + assert _provenance_free(rebuilt) == _provenance_free(tasks[marker]["componentRef"]["spec"]) + + +def test_a_decorated_source_without_dependencies_from_keeps_its_recorded_list(tmp_path: Path) -> None: + """Its dependencies were discovered beside the source; ``.toml`` is + what discovery finds first, so a maintainer's own pyproject.toml further up + cannot replace the recorded list.""" + import sys + + task = _authored_task(tmp_path, files={"pyproject.toml": '[project]\ndependencies = ["recorded==1"]\n'}) + assert "recorded==1" in task["componentRef"]["spec"]["metadata"]["annotations"]["python_dependencies"] + work = tmp_path / "work" + work.mkdir() + (work / "pyproject.toml").write_text('[project]\ndependencies = ["decoy==9"]\n', encoding="utf-8") + + assert decompile_pipeline_file(_write(tmp_path / "in.yaml", _graph_spec("Root", {"Score": task})), work / "gen.py").python_tasks == 1 + + sys.modules.pop("gen_tasks", None) + compile_pipeline(work / "gen.py", work / "gen.yaml", image_overrides={"scoring": "python:3.12"}) + sys.modules.pop("gen_tasks", None) + hydrated = PipelineHydrator(trusted_python_sources=[str(work)]).hydrate_file(work / "gen.yaml").data + rebuilt = hydrated["implementation"]["graph"]["tasks"]["Score"]["componentRef"]["spec"] + assert _provenance_free(rebuilt) == _provenance_free(task["componentRef"]["spec"]) + + +def test_task_and_subpackage_names_never_shadow_each_other(tmp_path: Path) -> None: + """Function ``foo`` in ``bar.py`` and ``bar`` in ``foo.py``: importing a + subpackage binds its name on the package, so neither may also be a task + exported under that name.""" + import sys + + tasks = {} + for function, module in (("foo", "bar"), ("bar", "foo")): + source = _SCORE.replace("def score(", f"def {function}(") + f"\n\nscore = {function}\n" + tasks[module] = _authored_task(tmp_path, source, module=module) + work = tmp_path / "work" + + result = decompile_pipeline_file(_write(tmp_path / "in.yaml", _graph_spec("Root", tasks)), work / "gen.py") + + assert result.verified and result.python_tasks == 2 + subpackages = {p.name for p in (work / "gen_tasks").iterdir() if p.is_dir()} + exported = set(re.findall(r"from gen_tasks import (.+)", (work / "gen.py").read_text(encoding="utf-8"))[0].split(", ")) + assert len(subpackages) == 2 and not subpackages & exported + sys.modules.pop("gen_tasks", None) + compile_pipeline(work / "gen.py", work / "gen.yaml", image_overrides={"scoring": "python:3.12"}) + sys.modules.pop("gen_tasks", None) + + +# --------------------------------------------------------------------------- +# Decompile never runs local Python, not even to resolve its input + + +@pytest.mark.parametrize("trust", ["cwd", "trusted-source"]) +def test_a_local_from_python_input_is_refused_without_running_it(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, trust: str) -> None: + """A compiled @task pipeline resolves through local_from_python, which runs + the source. Neither cwd trust nor an explicit trusted source makes + decompile do that; the previous output is untouched.""" + import sys + + from tangle_cli.hydration_trust import register_trusted_python_source + + authoring = tmp_path / "authoring" + authoring.mkdir() + (authoring / "score_task.py").write_text(_SCORE, encoding="utf-8") + (authoring / "pipe.py").write_text( + f"from tangle_cli.python_pipeline import pipeline\nfrom score_task import score\n\n@pipeline(name='Authored')\ndef authored() -> None:\n {_SCORE_CALL}\n", + encoding="utf-8", + ) + sys.modules.pop("score_task", None) + compile_pipeline(authoring / "pipe.py", authoring / "pipe.yaml", image_overrides={"scoring": "python:3.12"}) + sys.modules.pop("score_task", None) + (authoring / "SENTINEL").unlink() + if trust == "cwd": + monkeypatch.chdir(authoring) + else: + monkeypatch.setattr("tangle_cli.hydration_trust._TRUSTED_PYTHON_SOURCES", []) + register_trusted_python_source(authoring) + output = tmp_path / "out" / "gen.py" + output.parent.mkdir() + output.write_text("# previous\n", encoding="utf-8") + + with pytest.raises(DecompileError) as excinfo: + decompile_pipeline_file(authoring / "pipe.yaml", output, client=_UnreachableLibrary()) + + assert excinfo.value.code == "unsupported-local-python-source" + assert "tangle sdk pipelines hydrate" in str(excinfo.value) and "score" not in str(excinfo.value) + assert not (authoring / "SENTINEL").exists(), "the source never ran" + assert output.read_text(encoding="utf-8") == "# previous\n" + assert not list(output.parent.glob(".decompile-*")) + + +def test_hydrating_first_then_decompiling_recovers_the_task_statically(tmp_path: Path) -> None: + """The documented path: the explicit, trust-gated hydrate runs the source; + decompile then recovers the task from the hydrated YAML without running it.""" + task = _authored_task(tmp_path) # compiled AND hydrated by Tangle + source = _write(tmp_path / "hydrated.yaml", _graph_spec("Root", {"Score": task})) + + result = decompile_pipeline_file(source, tmp_path / "gen.py") + + assert result.verified and result.python_tasks == 1 and not list(tmp_path.rglob("SENTINEL")) + + +def test_a_local_from_python_entry_is_refused_even_behind_a_fallback(tmp_path: Path) -> None: + """A fallback_on_error entry would swallow the refusal and fall through to + a copy; the input still names local Python, so it is still refused.""" + (tmp_path / "score_task.py").write_text(_SCORE, encoding="utf-8") + _write(tmp_path / "leaf.yaml", _leaf("copy")) + _write(tmp_path / "c.yaml", {"f": [ + {"local_from_python": {"file": "score_task.py", "function": "score", "image": "python:3.12"}, "fallback_on_error": True}, + {"local": "./leaf.yaml"}, + ]}) + source = _write(tmp_path / "in.yaml", _graph_spec("Root", {"a": _ref(url="resolve://./c.yaml#f")})) + + with pytest.raises(DecompileError) as excinfo: + decompile_pipeline_file(source, tmp_path / "out" / "gen.py", client=_UnreachableLibrary()) + + assert excinfo.value.code == "unsupported-local-python-source" + assert not list(tmp_path.rglob("SENTINEL")) + + +_EXECUTABLE_KINDS = ["local_from_docker", "local_from_container", "from_docker", "from_container", "from-docker", "from-container", "downstream_build"] + + +@pytest.mark.parametrize("fallback", [False, True], ids=["primary", "behind-fallback"]) +@pytest.mark.parametrize("kind", _EXECUTABLE_KINDS) +def test_every_non_static_resolver_kind_is_refused_without_running( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, kind: str, fallback: bool +) -> None: + """As a downstream package would register them: a resolver that runs code. + Decompile refuses each one -- even behind a fallback -- and never calls it.""" + import tangle_cli.pipeline_hydrator as hydrator_module + + ran: list[str] = [] + monkeypatch.setitem(hydrator_module.COMPONENT_RESOLVERS, kind, lambda *a: ran.append(kind) or ("d" * 64, _leaf("built"))) + _write(tmp_path / "leaf.yaml", _leaf("copy")) + entries = [{kind: {"image": "example"}, **({"fallback_on_error": True} if fallback else {})}, *([{"local": "./leaf.yaml"}] if fallback else [])] + _write(tmp_path / "c.yaml", {"f": entries}) + source = _write(tmp_path / "in.yaml", _graph_spec("Root", {"a": _ref(url="resolve://./c.yaml#f")})) + output = tmp_path / "out" / "gen.py" + output.parent.mkdir() + output.write_text("# previous\n", encoding="utf-8") + + with pytest.raises(DecompileError) as excinfo: + decompile_pipeline_file(source, output, client=_UnreachableLibrary()) + + assert excinfo.value.code == "unsupported-executable-component-source" + assert "tangle sdk pipelines hydrate" in str(excinfo.value) and "example" not in str(excinfo.value) + assert not ran, "the resolver never ran" + assert output.read_text(encoding="utf-8") == "# previous\n" and not list(output.parent.glob(".decompile-*")) + + +def test_a_downstream_override_of_a_static_kind_is_not_used(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Allowed kinds are pinned to the built-in readers, so re-registering one + cannot swap code into decompile.""" + import tangle_cli.pipeline_hydrator as hydrator_module + + ran: list[str] = [] + for kind in ("file", "url", "local", "resolve"): + monkeypatch.setitem(hydrator_module.COMPONENT_RESOLVERS, kind, lambda *a, k=kind: ran.append(k) or None) + + assert decompile_pipeline_file(_local_leaf_source(tmp_path), tmp_path / "gen.py").verified + assert not ran + + +def _template_leaf(directory: Path) -> Path: + """A template_file config whose Jinja reaches Python builtins.""" + sentinel = directory / "SENTINEL" + (directory / "leaf.j2").write_text( + "# {{ cycler.__init__.__globals__.__builtins__.open(sentinel, 'w').write('ran') }}\n" + + dump_yaml(_leaf("templated")), + encoding="utf-8", + ) + return _write(directory / "leaf.yaml", {"template_file": "leaf.j2", "sentinel": str(sentinel)}) + + +@pytest.mark.parametrize("route", ["file-ref", "sidecar-local", "behind-fallback"]) +def test_a_template_component_is_refused_without_rendering(tmp_path: Path, route: str) -> None: + _template_leaf(tmp_path) + if route == "file-ref": + ref = _ref(url="file://./leaf.yaml") + else: + _write(tmp_path / "plain.yaml", _leaf("plain")) + entries = [{"local": "./leaf.yaml"}] if route == "sidecar-local" else [{"local": "./leaf.yaml", "fallback_on_error": True}, {"local": "./plain.yaml"}] + _write(tmp_path / "c.yaml", {"f": entries}) + ref = _ref(url="resolve://./c.yaml#f") + source = _write(tmp_path / "in.yaml", _graph_spec("Root", {"a": ref})) + output = tmp_path / "out" / "gen.py" + output.parent.mkdir() + output.write_text("# previous\n", encoding="utf-8") + + with pytest.raises(DecompileError) as excinfo: + decompile_pipeline_file(source, output, client=_UnreachableLibrary()) + + assert excinfo.value.code == "unsupported-template-component-source" + assert not (tmp_path / "SENTINEL").exists(), "the template never rendered" + assert output.read_text(encoding="utf-8") == "# previous\n" + + +def test_hydrating_a_template_first_then_decompiling_succeeds(tmp_path: Path) -> None: + """The explicit hydrate renders it; decompile then reads plain YAML.""" + _template_leaf(tmp_path) + source = _write(tmp_path / "in.yaml", _graph_spec("Root", {"a": _ref(url="file://./leaf.yaml")})) + hydrated = PipelineHydrator(client=_UnreachableLibrary()).hydrate_file(source).data + (tmp_path / "SENTINEL").unlink() + + result = decompile_pipeline_file(_write(tmp_path / "hydrated.yaml", hydrated), tmp_path / "gen.py", client=_UnreachableLibrary()) + + assert result.verified and not (tmp_path / "SENTINEL").exists() + + +@pytest.mark.parametrize("fallback", [False, True], ids=["primary", "behind-fallback"]) +def test_a_downstream_uri_reader_is_refused_without_running(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, fallback: bool) -> None: + """URI schemes follow the same allowlist as resolver kinds: a reader only + a downstream package registered is never called.""" + import tangle_cli.pipeline_hydrator as hydrator_module + + ran: list[str] = [] + monkeypatch.setitem(hydrator_module.URI_READERS, "downstream-build", lambda *a: ran.append("read") or dump_yaml(_leaf("built"))) + _write(tmp_path / "plain.yaml", _leaf("plain")) + entry = {"url": "downstream-build://./leaf.yaml", **({"fallback_on_error": True} if fallback else {})} + _write(tmp_path / "c.yaml", {"f": [entry, *([{"local": "./plain.yaml"}] if fallback else [])]}) + source = _write(tmp_path / "in.yaml", _graph_spec("Root", {"a": _ref(url="resolve://./c.yaml#f")})) + + with pytest.raises(DecompileError) as excinfo: + decompile_pipeline_file(source, tmp_path / "out" / "gen.py", client=_UnreachableLibrary()) + + assert excinfo.value.code == "unsupported-executable-component-source" and not ran + + +def test_a_downstream_override_of_the_https_reader_is_not_used(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """https stays the built-in reader (faked here, so nothing goes on the network) + for a remote resolve config.""" + import tangle_cli.pipeline_hydrator as hydrator_module + + ran: list[str] = [] + leaf = _write(tmp_path / "leaf.yaml", _leaf("fetched")) + manifest = dump_yaml({"f": [{"url": f"file://{leaf}"}]}) + monkeypatch.setattr(hydrator_module, "_read_http_uri", lambda *a: manifest) + monkeypatch.setitem(hydrator_module.URI_READERS, "https", lambda *a: ran.append("override") or manifest) + source = _write(tmp_path / "in.yaml", _graph_spec("Root", {"a": _ref(url="resolve://https://example.invalid/c.yaml#f")})) + + result = decompile_pipeline_file(source, tmp_path / "gen.py", client=_UnreachableLibrary()) + + assert result.verified and not ran + assert "fetched" in "".join(p.read_text(encoding="utf-8") for p in (tmp_path / "gen.leaves").rglob("*.yaml")) + + +# --------------------------------------------------------------------------- +# editor.position: .with_position() only when it reproduces the exact string + + +@pytest.mark.parametrize(("annotations", "expected"), [ + pytest.param({"editor.position": '{"x": 10, "y": 20}'}, ".named('T').with_position(10, 20)(", id="position-only"), + pytest.param({"editor.position": '{"x": -5, "y": 2.5}'}, ".with_position(-5, 2.5)(", id="negative-and-float"), + pytest.param( + {"editor.position": '{"x": 1, "y": 2, "width": 300, "height": 80}'}, + ".with_position(1, 2, width=300, height=80)(", + id="width-height", + ), + pytest.param({"editor.position": '{"x": 1e+20, "y": 2}'}, ".with_position(1e+20, 2)(", id="canonical-exponent"), + pytest.param( + {"owner": "me", "editor.position": '{"x": 1, "y": 2}', "z": "last"}, + ".with_annotations({'owner': 'me', 'z': 'last'}).with_position(1, 2)(", + id="mixed-position-last", + ), +]) +def test_an_exactly_reproducible_position_becomes_with_position(annotations: dict[str, Any], expected: str) -> None: + result = decompile_pipeline(_doc({"T": {"annotations": annotations, **_ref(name="c")}})) + + assert result.verified and expected in result.source and "editor.position" not in result.source + + +@pytest.mark.parametrize("value", [ + pytest.param('{"x":10,"y":20}', id="compact"), + pytest.param('{"y": 2, "x": 1}', id="reordered"), + pytest.param('{"x": 1, "y": 2, "z": 3}', id="extra-key"), + pytest.param('{"x": 1}', id="missing-y"), + pytest.param('{"x": NaN, "y": 2}', id="non-finite"), + pytest.param('{"x": true, "y": 2}', id="boolean"), + pytest.param('{"x": 1, "y": 2, "width": null}', id="null-width"), + pytest.param('{"x": 1, "x": 3, "y": 2}', id="duplicate-key"), + pytest.param('{"x": 1e3, "y": 2}', id="non-canonical-exponent"), + pytest.param('{"x": ' + "1" * 400 + ', "y": 2}', id="int-overflows-float"), + pytest.param('{"x": 1, "y": 2, "width": ' + "9" * 400 + "}", id="width-overflows-float"), + pytest.param("[" * 5000 + "]" * 5000, id="too-deep"), + pytest.param("not json", id="malformed"), + pytest.param("[1, 2]", id="not-an-object"), +]) +def test_a_position_with_position_would_rewrite_stays_verbatim(value: str) -> None: + result = decompile_pipeline(_doc({"T": {"annotations": {"editor.position": value, "k": "v"}, **_ref(name="c")}})) + + assert result.verified and "with_position" not in result.source + assert "'k': 'v'" in result.source + assert any(isinstance(n, ast.Constant) and n.value == value for n in ast.walk(ast.parse(result.source))), "kept verbatim" + + +def test_a_subpipeline_parent_card_and_its_child_internals_position_separately(tmp_path: Path) -> None: + child = _graph_spec("Child", {"leaf": {"annotations": {"editor.position": '{"x": 7, "y": 8}'}, **_ref(spec=_leaf("c"))}}) + document = _graph_spec("Root", {"n": {"annotations": {"editor.position": '{"x": 1, "y": 2}'}, **_ref(spec=child)}}) + + result = decompile_pipeline_file(_write(tmp_path / "in.yaml", document), tmp_path / "gen.py") + + assert result.verified + assert "subpipeline(child).named('n').with_position(1, 2)(" in (tmp_path / "gen.py").read_text(encoding="utf-8") + assert ".named('leaf').with_position(7, 8)(" in (tmp_path / "gen_subgraphs.py").read_text(encoding="utf-8") + + +def test_a_recovered_python_task_takes_its_position_too(tmp_path: Path) -> None: + task = {"annotations": {"editor.position": '{"x": 3, "y": 4}'}, **_ref(spec=_python_leaf(tmp_path)), "arguments": {"name": "a"}} + + result = decompile_pipeline_file(_write(tmp_path / "in.yaml", _graph_spec("Root", {"g": task})), tmp_path / "gen.py") + + assert result.verified and result.python_tasks == 1 + assert re.search(r"= \w+\.named\('g'\)\.with_position\(3, 4\)\(name='a'\)", (tmp_path / "gen.py").read_text(encoding="utf-8")) + + +def _io_doc(annotations: dict[str, Any]) -> dict[str, Any]: + return _doc( + {"T": _ref(name="c")}, + inputs=[{"name": "In", "type": "String", "annotations": dict(annotations)}], + outputs=[{"name": "Out", "type": "String", "annotations": dict(annotations)}], + graph={"outputValues": {"Out": {"graphInput": {"inputName": "In"}}}}, + ) + + +@pytest.mark.parametrize(("annotations", "expected"), [ + pytest.param({"editor.position": '{"x": 10, "y": 20}'}, "position=(10, 20))", id="position-only"), + pytest.param({"editor.position": '{"x": -5, "y": 2.5}'}, "position=(-5, 2.5))", id="negative-and-float"), + pytest.param({"note": "n", "editor.position": '{"x": 1, "y": 2}'}, "annotations={'note': 'n'}, position=(1, 2))", id="mixed"), +]) +def test_an_exact_x_y_graph_io_position_becomes_position_kwarg(annotations: dict[str, Any], expected: str) -> None: + result = decompile_pipeline(_io_doc(annotations)) + + assert result.verified and result.source.count(expected) == 2, "both graph_input and graph_output" + assert "editor.position" not in result.source + + +@pytest.mark.parametrize("value", [ + pytest.param('{"x": 1, "y": 2, "width": 300, "height": 80}', id="width-height-unsupported-by-position"), + pytest.param('{"x":10,"y":20}', id="compact"), + pytest.param('{"y": 2, "x": 1}', id="reordered"), + pytest.param('{"x": ' + "1" * 400 + ', "y": 2}', id="overflow"), + pytest.param("not json", id="malformed"), +]) +def test_a_graph_io_position_position_cannot_reproduce_stays_verbatim(value: str) -> None: + result = decompile_pipeline(_io_doc({"editor.position": value, "k": "v"})) + + assert result.verified and "position=" not in result.source + assert sum(isinstance(n, ast.Constant) and n.value == value for n in ast.walk(ast.parse(result.source))) == 2 diff --git a/tests/test_pipelines_cli.py b/tests/test_pipelines_cli.py index e026ab0..2a5e1c2 100644 --- a/tests/test_pipelines_cli.py +++ b/tests/test_pipelines_cli.py @@ -2245,3 +2245,132 @@ def test_shared_compile_flag_parsers(): for bad in (["nope"], ["id="], ["=ref"]): with pytest.raises(SystemExit): parse_image_overrides(bad) + + +# --------------------------------------------------------------------------- +# pipelines decompile + + +def _decompilable(tmp_path: Path, component_name: str = "Leaf") -> Path: + """A dehydrated pipeline whose one component is a local file.""" + (tmp_path / "leaf.yaml").write_text( + yaml.safe_dump( + { + "name": component_name, + "implementation": {"container": {"image": "i", "command": ["p"]}}, + }, + sort_keys=False, + ), + encoding="utf-8", + ) + source = tmp_path / "pipeline.yaml" + return _write_pipeline( + source, + { + "name": "Outer", + "implementation": { + "graph": {"tasks": {"a": {"componentRef": {"url": "file://./leaf.yaml"}}}} + }, + }, + ) + + +def test_pipelines_decompile_help_lists_the_pinning_flag(capsys): + app = cli.build_app() + + run_app(app, ["sdk", "pipelines", "decompile", "--help"]) + + output = capsys.readouterr().out + assert "--verify" in output + assert "--pin-components" not in output, "canonicalization is not optional" + + +def test_pipelines_decompile_writes_a_script_and_its_components(tmp_path: Path, capsys): + source = _decompilable(tmp_path) + output = tmp_path / "generated.py" + app = cli.build_app() + + run_app(app, ["sdk", "pipelines", "decompile", str(source), "--output", str(output)]) + + printed = capsys.readouterr().out + assert output.is_file() + assert (tmp_path / "generated.leaves").is_dir() + assert (tmp_path / "generated.leaves.yaml").is_file() + assert "resolve://./generated.leaves.yaml#" in output.read_text(encoding="utf-8") + assert "generated.leaves.yaml" in printed + assert "Verified:" in printed + assert "compile the script in place" in printed, "the relative-URL caveat is surfaced" + + +@pytest.mark.parametrize("flag", [[], ["--no-python-tasks"]]) +def test_pipelines_decompile_python_tasks_flag(tmp_path: Path, capsys, flag: list[str]): + from test_pipeline_decompiler import _python_leaf + + source = _write_pipeline(tmp_path / "in.yaml", {"name": "Outer", "implementation": {"graph": {"tasks": {"a": {"componentRef": {"spec": _python_leaf(tmp_path)}}}}}}) + output = tmp_path / "generated.py" + + run_app(cli.build_app(), ["sdk", "pipelines", "decompile", str(source), "--output", str(output), *flag]) + + printed = capsys.readouterr().out + converted = not flag + assert (tmp_path / "generated_tasks").is_dir() == converted + assert ("review it first" in printed) == converted, "the later-execution caveat is surfaced" + assert ("stayed YAML leaves" in printed) == (not converted) + + +def test_pipelines_decompile_discloses_image_id_resolution(tmp_path: Path, capsys): + from test_pipeline_decompiler import _authored_task + + source = _write_pipeline(tmp_path / "in.yaml", {"name": "Outer", "implementation": {"graph": {"tasks": {"Score": _authored_task(tmp_path)}}}}) + + run_app(cli.build_app(), ["sdk", "pipelines", "decompile", str(source), "--output", str(tmp_path / "generated.py")]) + + printed = capsys.readouterr().out + assert "may pick a different image" in printed and "--no-python-tasks" in printed + assert "unwrap" in printed + + +@pytest.mark.parametrize("flag", [[], ["--local-fallbacks"]]) +def test_pipelines_decompile_local_fallbacks_flag(tmp_path: Path, capsys, monkeypatch, flag: list[str]): + import tangle_cli.pipelines_cli as pipelines_cli + from test_pipeline_decompiler import _leaf, _published, _ref + + published = _leaf("published") + digest, library = _published(published) + real = pipelines_cli.decompile_pipeline_file + monkeypatch.setattr(pipelines_cli, "decompile_pipeline_file", lambda *a, **k: real(*a, client=library, **k)) + source = _write_pipeline(tmp_path / "in.yaml", {"name": "Outer", "implementation": {"graph": {"tasks": {"a": _ref(digest=digest, spec=published)}}}}) + + run_app(cli.build_app(), ["sdk", "pipelines", "decompile", str(source), "--output", str(tmp_path / "generated.py"), *flag]) + + printed = capsys.readouterr().out + assert (tmp_path / "generated.leaves.yaml").is_file() == bool(flag) + assert ("needs the component library" in printed) == (not flag) + + +def test_pipelines_decompile_exits_nonzero_without_echoing_values(tmp_path: Path): + """A refusal must fail the command and must not print document content.""" + source = _write_pipeline( + tmp_path / "bad.yaml", + { + "name": "Outer", + "implementation": { + "graph": { + "tasks": { + "a": { + "componentRef": {"digest": "a" * 64}, + "arguments": {"token": {"malformed": "SYNTHETIC_CLI_SECRET bad"}}, + } + } + } + }, + }, + ) + app = cli.build_app() + + with pytest.raises(SystemExit) as excinfo: + app(["sdk", "pipelines", "decompile", str(source), "--output", str(tmp_path / "o.py")]) + + assert excinfo.value.code not in (0, None) + assert "SYNTHETIC_CLI_SECRET" not in str(excinfo.value.code) + assert not (tmp_path / "o.py").exists() diff --git a/uv.lock b/uv.lock index 2b8f7ad..50b6e9c 100644 --- a/uv.lock +++ b/uv.lock @@ -2083,7 +2083,7 @@ requires-dist = [{ name = "pydantic", specifier = ">=2.0" }] [[package]] name = "tangle-cli" -version = "0.1.30" +version = "0.1.31" source = { editable = "." } dependencies = [ { name = "cloud-pipelines" }, From 2f0320870bb30e50b45d89bcd04a417c29aa0d84 Mon Sep 17 00:00:00 2001 From: Volv G Date: Fri, 2 Oct 2026 19:45:16 -0700 Subject: [PATCH 2/2] Keep decompiler tests off the network A test that passes no client fell back to a real component-library client, which retries for about a minute per lookup where the network is blocked (CI) and silently answers where it is not. Client creation now yields an unreachable library in those tests; explicit clients are unaffected. Test-only. --- tests/test_pipeline_decompiler.py | 11 +++++++++++ tests/test_pipelines_cli.py | 15 +++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/tests/test_pipeline_decompiler.py b/tests/test_pipeline_decompiler.py index 2e6cdfe..a8b22b0 100644 --- a/tests/test_pipeline_decompiler.py +++ b/tests/test_pipeline_decompiler.py @@ -1048,6 +1048,17 @@ def test_only_an_explicit_logger_is_forwarded_to_internal_api_clients() -> None: assert PipelineHydrator(logger=quiet)._client_options.get("logger") is quiet # type: ignore[arg-type] +@pytest.fixture(autouse=True) +def _no_ambient_component_library(monkeypatch: pytest.MonkeyPatch) -> None: + """A test that passes no client must never reach a real component library: + the API client retries for about a minute per lookup where the network is + blocked (CI), and silently answers where it is not. Explicit clients are + unaffected.""" + from tangle_cli.handler import TangleCliHandler + + monkeypatch.setattr(TangleCliHandler, "_create_client", lambda self: _UnreachableLibrary()) + + # --------------------------------------------------------------------------- # Every graph becomes Python: nested graphs are @pipeline functions in the # companion _subgraphs.py, called through subpipeline(...) diff --git a/tests/test_pipelines_cli.py b/tests/test_pipelines_cli.py index 2a5e1c2..36f6f8c 100644 --- a/tests/test_pipelines_cli.py +++ b/tests/test_pipelines_cli.py @@ -2275,6 +2275,16 @@ def _decompilable(tmp_path: Path, component_name: str = "Leaf") -> Path: ) +@pytest.fixture +def no_ambient_component_library(monkeypatch): + """Decompile tests must never reach a real component library (see + test_pipeline_decompiler._no_ambient_component_library).""" + from tangle_cli.handler import TangleCliHandler + from tangle_cli.pipeline_decompiler import _UnreachableLibrary + + monkeypatch.setattr(TangleCliHandler, "_create_client", lambda self: _UnreachableLibrary()) + + def test_pipelines_decompile_help_lists_the_pinning_flag(capsys): app = cli.build_app() @@ -2285,6 +2295,7 @@ def test_pipelines_decompile_help_lists_the_pinning_flag(capsys): assert "--pin-components" not in output, "canonicalization is not optional" +@pytest.mark.usefixtures("no_ambient_component_library") def test_pipelines_decompile_writes_a_script_and_its_components(tmp_path: Path, capsys): source = _decompilable(tmp_path) output = tmp_path / "generated.py" @@ -2302,6 +2313,7 @@ def test_pipelines_decompile_writes_a_script_and_its_components(tmp_path: Path, assert "compile the script in place" in printed, "the relative-URL caveat is surfaced" +@pytest.mark.usefixtures("no_ambient_component_library") @pytest.mark.parametrize("flag", [[], ["--no-python-tasks"]]) def test_pipelines_decompile_python_tasks_flag(tmp_path: Path, capsys, flag: list[str]): from test_pipeline_decompiler import _python_leaf @@ -2318,6 +2330,7 @@ def test_pipelines_decompile_python_tasks_flag(tmp_path: Path, capsys, flag: lis assert ("stayed YAML leaves" in printed) == (not converted) +@pytest.mark.usefixtures("no_ambient_component_library") def test_pipelines_decompile_discloses_image_id_resolution(tmp_path: Path, capsys): from test_pipeline_decompiler import _authored_task @@ -2330,6 +2343,7 @@ def test_pipelines_decompile_discloses_image_id_resolution(tmp_path: Path, capsy assert "unwrap" in printed +@pytest.mark.usefixtures("no_ambient_component_library") @pytest.mark.parametrize("flag", [[], ["--local-fallbacks"]]) def test_pipelines_decompile_local_fallbacks_flag(tmp_path: Path, capsys, monkeypatch, flag: list[str]): import tangle_cli.pipelines_cli as pipelines_cli @@ -2348,6 +2362,7 @@ def test_pipelines_decompile_local_fallbacks_flag(tmp_path: Path, capsys, monkey assert ("needs the component library" in printed) == (not flag) +@pytest.mark.usefixtures("no_ambient_component_library") def test_pipelines_decompile_exits_nonzero_without_echoing_values(tmp_path: Path): """A refusal must fail the command and must not print document content.""" source = _write_pipeline(