From 364daa7d915a20ed50b1a01128ca0c1ac24a5c2a Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Sat, 17 Oct 2026 19:25:50 +0000 Subject: [PATCH 01/11] fix: retain unfinished checkpoints at Deep Scan deadlines --- .../scripts/deep_scan_workbench.py | 10 +- .../scripts/workbench_saved_results.py | 56 +++++-- .../tests/test_deep_scan_stop_conditions.py | 154 +++++++++++++++++- 3 files changed, 202 insertions(+), 18 deletions(-) diff --git a/plugins/codex-security/scripts/deep_scan_workbench.py b/plugins/codex-security/scripts/deep_scan_workbench.py index cebf561caa..a2b9e82370 100644 --- a/plugins/codex-security/scripts/deep_scan_workbench.py +++ b/plugins/codex-security/scripts/deep_scan_workbench.py @@ -1758,9 +1758,9 @@ def finish_deep_scan_locked( """, (scan_id,), ).fetchone() + discovery_deadline = args.terminal_reason == "capped" and deep_scan_deadline_reached(run) zero_discovery_deadline = ( - args.terminal_reason == "capped" - and deep_scan_deadline_reached(run) + discovery_deadline and run["completion_sequence"] == 0 and ( standard_scan_manifest @@ -1793,8 +1793,10 @@ def finish_deep_scan_locked( ) promotion = promote_staged_file(staged_manifest_path, manifest_path) with ( - saved_results.preserve_deep_deadline_draft(dependencies(), connection, scan) - if zero_discovery_deadline and standard_scan_manifest + saved_results.preserve_deep_deadline_draft( + dependencies(), connection, scan, preserve_reduction=not zero_discovery_deadline + ) + if discovery_deadline and standard_scan_manifest else nullcontext() ): timestamp = dependencies().now() diff --git a/plugins/codex-security/scripts/workbench_saved_results.py b/plugins/codex-security/scripts/workbench_saved_results.py index a772585491..48ab74e6ad 100644 --- a/plugins/codex-security/scripts/workbench_saved_results.py +++ b/plugins/codex-security/scripts/workbench_saved_results.py @@ -908,6 +908,7 @@ def merge_saved_results( allow_frozen_legacy_parent: bool = False, frozen_model_source: str | None = None, selected_model_source: list[str] | None = None, + preserve_canonical_parent: bool = False, ) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]] | None: """Read only bound parent/worker files; return an unsealed loss-preserving union.""" initial_warnings = set(warnings) @@ -932,7 +933,7 @@ def merge_saved_results( warnings.append(f"Could not read the saved parent draft: {exc}") parent_manifest = None parent = None - if parent_manifest is not None and parent is not None: + if parent_manifest is not None and parent is not None and not preserve_canonical_parent: parent_scan = parent_manifest["scan"] try: previous_head, _, head_modified = _read_saved_result( @@ -1017,8 +1018,9 @@ def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None for saved_directory in (directory, head_snapshots): paths.update(dict.fromkeys(_checkpoint_paths(scan_dir, saved_directory), worker_id)) - paths["checkpoint-head.json"] = None - checkpoints("checkpoints", None) + if not preserve_canonical_parent: + paths["checkpoint-head.json"] = None + checkpoints("checkpoints", None) for worker in workers: try: outputs = _worker_outputs(scan_dir, worker) @@ -1310,6 +1312,7 @@ def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None ) findings: list[dict[str, Any]] = [] finding_positions: dict[str, int] = {} + canonical_finding_keys: set[str] = set() represented: dict[str, str | None] = {} represented_candidates: dict[tuple[str, str, Any, Any, Any], str | None] = {} represented_history: dict[str, set[str]] = {} @@ -1754,6 +1757,8 @@ def valid_finding(value: Any) -> bool: continue if valid_finding(finding): finding_positions.setdefault(_finding_key(finding), len(findings)) + if preserve_canonical_parent: + canonical_finding_keys.add(_finding_key(finding)) findings.append(finding) continue if relative != "parent" and parent and value in parent["findings"]: @@ -1826,8 +1831,10 @@ def valid_finding(value: Any) -> bool: if key in finding_positions: retained = findings[finding_positions[key]] if finding != retained: - if not represented_by_parent and _finding_strength(finding) > _finding_strength( - retained + if ( + key not in canonical_finding_keys + and not represented_by_parent + and _finding_strength(finding) > _finding_strength(retained) ): previous = copy.deepcopy(retained) previous_history = previous["provenance"].pop("previousFindings", []) @@ -2063,31 +2070,56 @@ def _restore_published_outputs( @contextmanager -def preserve_deep_deadline_draft(db: Any, connection: Any, scan: Any) -> Iterator[None]: - """Recover a settled, zero-accepted Deep deadline without sealing the parent.""" +def preserve_deep_deadline_draft( + db: Any, connection: Any, scan: Any, *, preserve_reduction: bool +) -> Iterator[None]: + """Recover unfinished reviews at a settled Deep deadline without sealing the parent.""" scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"])) snapshots = _snapshot_published_outputs(scan_dir) warnings = json.loads(scan["completion_warnings_json"]) binding = db.workbench_completion_binding(scan, db.now()) + workers = connection.execute( + "SELECT * FROM deep_scan_workers WHERE scan_id = ? ORDER BY created_at, id", + (scan["id"],), + ).fetchall() + if preserve_reduction: + # Completed reviews already went through reduction. Only add evidence + # from unfinished reviews; raw accepted drafts must not undo that result. + workers = [ + worker + for worker in workers + if worker["kind"] == "discovery" and worker["status"] == "canceled" + ] + if not workers: + yield + return with preserve_parent_head_on_error(scan_dir): try: documents = merge_saved_results( scan_dir, scan["id"], binding, - connection.execute( - "SELECT * FROM deep_scan_workers WHERE scan_id = ? ORDER BY created_at, id", - (scan["id"],), - ).fetchall(), + workers, warnings, - stopped=True, + stopped=not preserve_reduction, reason=( "The configured discovery time limit elapsed before any source review completed. " "Saved findings and pending review were preserved without a completed reduction." ), + preserve_canonical_parent=preserve_reduction, ) if documents is None: raise ContractError("Deep Scan deadline has no saved parent draft.") + if preserve_reduction: + documents[2]["deferred"].append( + { + "id": "scan-deadline", + "reason": ( + "The configured discovery time limit elapsed. Saved findings and " + "pending work from unfinished source reviews were preserved." + ), + } + ) documents[0]["scan"].pop("complete", None) # Validate recovered evidence against the current finding contract, # scope, source lineage, and receipts before admitting this draft. diff --git a/plugins/codex-security/tests/test_deep_scan_stop_conditions.py b/plugins/codex-security/tests/test_deep_scan_stop_conditions.py index 62f5268f0a..192a4b39b0 100644 --- a/plugins/codex-security/tests/test_deep_scan_stop_conditions.py +++ b/plugins/codex-security/tests/test_deep_scan_stop_conditions.py @@ -1,5 +1,6 @@ from __future__ import annotations +import copy import json import os import sqlite3 @@ -125,6 +126,148 @@ def test_saturated_finish_retains_failed_discovery_diagnostic( assert_published_aggregate(scan) +@pytest.mark.parametrize("stop", ["deadline", "run-limit", "saturated"]) +def test_unfinished_checkpoints_after_a_completed_reduction( + workbench_api, workbench_db, saturated_scan, stop +): + scan = saturated_scan + result = add_worker(workbench_db, scan, status="canceled") + recovered = copy.deepcopy(scan.findings[0]) + recovered["identity"]["anchor"] = "unfinished-upload-review" + recovered["title"] = "Separate upload handler bypasses containment" + recovered["provenance"] = {"source": "local_plugin"} + duplicate = copy.deepcopy(scan.findings[0]) + duplicate["severity"]["level"] = "critical" + duplicate["remediation"] = "Unreduced remediation from an unfinished review." + pending = { + "id": "pending-upload", + "reason": "The remaining upload paths need review.", + "paths": ["subdir/extract.py"], + "surfaceIds": ["upload"], + } + checkpoint = write_checkpoint( + result.parent / "checkpoints", + saved_draft( + scan.scan_id, + findings=[recovered, duplicate], + deferred=[pending], + surfaces=[ + { + "id": "upload", + "label": "Upload handler", + "disposition": "needs_follow_up", + "receiptRefs": [], + } + ], + ), + ) + source_bytes = checkpoint.read_bytes() + # Earlier parent, discovery, and reducer drafts must not undo reduction. + discarded = copy.deepcopy(recovered) + discarded["identity"]["anchor"] = "discarded-during-reduction" + for directory in ( + scan.scan_dir, + scan.scan_dir / "workers" / scan.completed_worker_id, + scan.scan_dir / "workers" / scan.reducer_id, + ): + discarded_checkpoint = write_checkpoint( + directory / "checkpoints", + saved_draft(scan.scan_id, findings=[discarded]), + ) + if directory == scan.scan_dir: + head = directory / "checkpoint-head.json" + head.write_text(json.dumps({"checkpoint": discarded_checkpoint.name})) + observed = ( + max( + (directory / name).stat().st_mtime_ns + for name in ("scan-manifest.json", "findings.json", "coverage.json") + ) + + 1_000_000_000 + ) + for path in (discarded_checkpoint, head): + os.utime(path, ns=(observed, observed)) + with workbench_db: + if stop == "deadline": + workbench_db.execute( + "UPDATE deep_scan_runs SET created_at = '2000-01-01T00:00:00+00:00' " + "WHERE scan_id = ?", + (scan.scan_id,), + ) + elif stop == "run-limit": + workbench_db.execute( + "UPDATE deep_scan_runs SET discovery_runs_dispatched = max_discovery_runs " + "WHERE scan_id = ?", + (scan.scan_id,), + ) + + reason = "saturated" if stop == "saturated" else "capped" + finished = finish(workbench_api, workbench_db, scan, terminal_reason=reason) + + assert finished["status"] == "succeeded" + worker = next(item for item in finished["workers"] if item["id"] == result.parent.name) + assert worker["status"] == "canceled" + assert worker["completionSequence"] is None + row = workbench_db.execute( + "SELECT status, seal_manifest_digest FROM scans WHERE id = ?", (scan.scan_id,) + ).fetchone() + assert tuple(row) == ("running", None) + manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text()) + assert "sealedAt" not in manifest["scan"] + completed = complete(workbench_api, workbench_db, scan) + assert completed["progress"]["status"] == "complete" + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + expected_anchors = {scan.findings[0]["identity"]["anchor"]} + if stop == "deadline": + expected_anchors.add(recovered["identity"]["anchor"]) + retained = next(item for item in findings if item["title"] == recovered["title"]) + assert retained["provenance"]["workerId"] == result.parent.name + assert ( + checkpoint.relative_to(scan.scan_dir).as_posix() in manifest["scan"]["preservedSources"] + ) + assert {item["identity"]["anchor"] for item in findings} == expected_anchors + canonical = next(item for item in findings if item["title"] == scan.findings[0]["title"]) + for field in ("severity", "confidence", "remediation"): + assert canonical[field] == scan.findings[0][field] + assert ( + canonical["provenance"]["sourceFindings"] + == scan.findings[0]["provenance"]["sourceFindings"] + ) + if stop == "deadline": + assert any( + item["remediation"] == duplicate["remediation"] + for item in canonical["provenance"]["previousFindings"] + ) + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert any(item["reason"] == pending["reason"] for item in coverage["deferred"]) == ( + stop == "deadline" + ) + assert any(item["disposition"] == "needs_follow_up" for item in coverage["surfaces"]) == ( + stop == "deadline" + ) + assert checkpoint.read_bytes() == source_bytes + names = ("scan-manifest.json", "findings.json", "coverage.json", "report.md") + sealed = {name: (scan.scan_dir / name).read_bytes() for name in names} + finish(workbench_api, workbench_db, scan, terminal_reason=reason) + complete(workbench_api, workbench_db, scan) + assert {name: (scan.scan_dir / name).read_bytes() for name in names} == sealed + + +def test_deadline_after_all_reviews_complete_keeps_reduced_coverage( + workbench_api, workbench_db, saturated_scan +): + scan = saturated_scan + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_runs SET created_at = '2000-01-01T00:00:00+00:00' WHERE scan_id = ?", + (scan.scan_id,), + ) + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + complete(workbench_api, workbench_db, scan) + + assert_published_aggregate(scan) + + @pytest.mark.parametrize( ("terminal_reason", "kind"), [("saturated", "setup"), ("saturated", "dedup"), ("capped", "discovery")], @@ -156,14 +299,21 @@ def test_finish_preserves_other_worker_failure_checks( @pytest.mark.parametrize("existing_head", [False, True]) +@pytest.mark.parametrize("with_reduction", [False, True], ids=["zero-accepted", "reduced"]) def test_rejected_deadline_publication_does_not_leave_parent_observations( - workbench_api, workbench_db, publication_scan, existing_head + workbench_api, workbench_db, publication_scan, request, existing_head, with_reduction ): - scan = publication_scan() + scan = request.getfixturevalue("saturated_scan") if with_reduction else publication_scan() canonical_names = ("scan-manifest.json", "findings.json", "coverage.json") for name in canonical_names: os.utime(scan.scan_dir / name, ns=(1_000_000_000, 1_000_000_000)) pending = {"id": "authorization-review", "reason": "Review the remaining authorization path."} + if with_reduction: + result = add_worker(workbench_db, scan, status="canceled") + write_checkpoint( + result.parent / "checkpoints", + saved_draft(scan.scan_id, deferred=[pending]), + ) if existing_head: checkpoint = write_checkpoint( scan.scan_dir / "checkpoints", From 5d2ff41d1405f65fd8953c926fb1d6b4eff115d4 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Sat, 17 Oct 2026 19:51:20 +0000 Subject: [PATCH 02/11] fix: retain valid deadline checkpoint artifacts --- .../scripts/workbench_saved_results.py | 60 ++++ .../tests/test_deep_scan_stop_conditions.py | 293 ++++++++++++++++++ 2 files changed, 353 insertions(+) diff --git a/plugins/codex-security/scripts/workbench_saved_results.py b/plugins/codex-security/scripts/workbench_saved_results.py index 48ab74e6ad..b25ba82c44 100644 --- a/plugins/codex-security/scripts/workbench_saved_results.py +++ b/plugins/codex-security/scripts/workbench_saved_results.py @@ -31,6 +31,7 @@ _read_scan_local_json_with_metadata, _recover_unsealed_findings, _remove_scan_local_file_if_exists, + _require_portable_relative_path, _validate_completion_binding, _validate_resolved_deferred, _validate_schema_node, @@ -1097,6 +1098,51 @@ def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None if _checkpoint_head_directory(relative) is not None: saved_heads[relative] = draft["checkpoint"] continue + if preserve_canonical_parent and worker_id is not None: + # Recover imported records without relaxing canonical parent validation. + recovered_findings = [] + for index, finding in enumerate(draft["findings"]): + _ensure_finding_identity(finding) + recovered = {"scanId": scan_id, "findings": [finding]} + source_warnings: list[str] = [] + _recover_unsealed_findings( + {"scan": {"id": scan_id, "target": binding["target"]}}, + recovered, + Path(__file__).resolve().parent.parent / "schemas", + scan_dir, + source_warnings, + ) + recovered_findings.extend(recovered["findings"]) + warnings.extend( + f"{relative} finding {index + 1}: {warning}" for warning in source_warnings + ) + draft["findings"] = recovered_findings + directory = Path(relative).parent + if directory.name == "checkpoints": + directory = directory.parent + for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"): + rows = draft["coverage"].get(field) + if not isinstance(rows, list): + continue + for index, row in enumerate(rows): + if field == "openQuestions" and isinstance(row, str): + row = rows[index] = {"question": row} + if not isinstance(row, dict): + continue + row["workerId"] = worker_id + refs = row.get("receiptRefs") + if not isinstance(refs, list): + continue + for ref_index, ref in enumerate(refs): + if not isinstance(ref, str): + continue + try: + normalized = _require_portable_relative_path(ref, "receiptRef") + except ContractError: + # Keep malformed references for the finalizer's diagnostics. + continue + if normalized.startswith("artifacts/"): + refs[ref_index] = f"{directory.as_posix()}/{normalized}" # Recovery expects coverage, but reducer results only contain findings # and context. Add an empty value after hashing the original result. sources.append((relative, {"coverage": {}, **draft}, worker_id)) @@ -1985,6 +2031,7 @@ def valid_finding(value: Any) -> bool: finding.setdefault("provenance", {})["preservedIdentity"] = copy.deepcopy(identity) identity["instance"] = f"{identity.get('instance', 'saved')}-{variant[:16]}" identities[key] = variant + recovered_worker_ids = {worker_id for _, _, worker_id in sources if worker_id is not None} for field in ("surfaces", "explicitExclusions", "deferred"): used: set[str] = set() items = coverage.setdefault(field, []) @@ -1999,6 +2046,18 @@ def valid_finding(value: Any) -> bool: if not isinstance(item["id"], str): # Preserve malformed rows for per-record recovery, including frozen replay. continue + if ( + preserve_canonical_parent + and isinstance(worker_id := item.get("workerId"), str) + and worker_id in recovered_worker_ids + ): + # Resolve worker-local candidate/deferred history before scoping IDs. + item["id"] = f"{worker_id}/{item['id']}" + if isinstance(item.get("surfaceIds"), list): + item["surfaceIds"] = [ + f"{worker_id}/{surface_id}" if isinstance(surface_id, str) else surface_id + for surface_id in item["surfaceIds"] + ] if item["id"] in used: item["id"] = f"{item['id']}-{_digest(item)[:16]}" used.add(item["id"]) @@ -2128,6 +2187,7 @@ def preserve_deep_deadline_draft( expected_coverage_mode=db.expected_coverage_mode(scan), completion_binding=binding, recover_unsealed=True, + allow_missing_writeups=preserve_reduction, draft_documents=documents, ) warnings.extend(prepared.warnings) diff --git a/plugins/codex-security/tests/test_deep_scan_stop_conditions.py b/plugins/codex-security/tests/test_deep_scan_stop_conditions.py index 192a4b39b0..d585058c4c 100644 --- a/plugins/codex-security/tests/test_deep_scan_stop_conditions.py +++ b/plugins/codex-security/tests/test_deep_scan_stop_conditions.py @@ -73,6 +73,299 @@ def finish(workbench_api, connection, scan, *, terminal_reason="saturated"): )["deepScan"] +@pytest.fixture +def deadline_scan(saturated_scan, workbench_db): + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_runs SET created_at = '2000-01-01T00:00:00+00:00' WHERE scan_id = ?", + (saturated_scan.scan_id,), + ) + return saturated_scan + + +def canceled_discovery_output(connection, scan, *, attempt=1): + result = add_worker(connection, scan, status="canceled") + worker_id = result.parent.name + output = scan.scan_dir / "artifacts" / "deep_discovery" / "workers" / worker_id / "output" + output.parent.mkdir(parents=True) + result.parent.rename(output) + with connection: + connection.execute( + "UPDATE deep_scan_workers SET artifact_dir = ?, prompt_path = ?, " + "result_manifest_path = NULL, attempt = ? WHERE id = ?", + (str(output), str(output.parent / "prompt.md"), attempt, worker_id), + ) + return worker_id, output + + +@pytest.mark.parametrize("missing_writeup", [False, True], ids=["present", "missing"]) +def test_deadline_preserves_optional_reduced_writeup_behavior( + workbench_api, workbench_db, deadline_scan, missing_writeup +): + scan = deadline_scan + report_path = "findings/reduced/reduced.md" + scan.findings[0]["writeup"] = {"reportPath": report_path} + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": scan.findings})) + if not missing_writeup: + report = scan.scan_dir / report_path + report.parent.mkdir(parents=True) + report.write_text("# Reduced finding\n") + _, output = canceled_discovery_output(workbench_db, scan) + write_checkpoint( + output / "checkpoints", + saved_draft(scan.scan_id, deferred=[{"reason": "Review the remaining upload paths."}]), + ) + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + completed = complete(workbench_api, workbench_db, scan) + + assert completed["progress"]["status"] == "complete" + finding = json.loads((scan.scan_dir / "findings.json").read_text())["findings"][0] + assert finding["title"] == scan.findings[0]["title"] + assert ("writeup" not in finding) == missing_writeup + assert any("Skipped missing writeup" in warning for warning in completed["warnings"]) == ( + missing_writeup + ) + + +def test_deadline_recovers_valid_findings_from_an_invalid_archived_retry( + workbench_api, workbench_db, deadline_scan +): + scan = deadline_scan + worker_id, output = canceled_discovery_output(workbench_db, scan, attempt=2) + archived = output.parent / "attempts" / "attempt-1" + archived.mkdir(parents=True) + valid = copy.deepcopy(scan.findings[0]) + valid["identity"]["anchor"] = "valid-archived-finding" + valid["title"] = "Valid finding saved before retry" + valid["provenance"] = {"source": "local_plugin"} + invalid = copy.deepcopy(valid) + invalid["identity"]["anchor"] = "invalid-archived-finding" + invalid.pop("remediation") + result = archived / "result.json" + result.write_text( + json.dumps(saved_draft(scan.scan_id, complete=True, findings=[valid, invalid])) + ) + original = result.read_bytes() + write_checkpoint( + output / "checkpoints", + saved_draft(scan.scan_id, deferred=[{"reason": "Retry review remains unfinished."}]), + ) + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + completed = complete(workbench_api, workbench_db, scan) + + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert {item["identity"]["anchor"] for item in findings} == { + scan.findings[0]["identity"]["anchor"], + valid["identity"]["anchor"], + } + recovered = next(item for item in findings if item["title"] == valid["title"]) + assert recovered["provenance"]["workerId"] == worker_id + assert any("remediation" in warning for warning in completed["warnings"]) + manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"] + assert manifest["preservedSources"][result.relative_to(scan.scan_dir).as_posix()] == ( + workbench_api["saved_results"]._digest(json.loads(original)) + ) + assert result.read_bytes() == original + + +@pytest.mark.parametrize("invalid", ["finding", "unsafe-writeup"]) +def test_deadline_does_not_recover_invalid_reduced_findings( + workbench_api, workbench_db, deadline_scan, invalid +): + from finalize_scan_contract import ContractError + + scan = deadline_scan + if invalid == "finding": + scan.findings[0].pop("remediation") + else: + scan.findings[0]["writeup"] = {"reportPath": "../invalid.md"} + findings_path = scan.scan_dir / "findings.json" + findings_path.write_text(json.dumps({"findings": scan.findings})) + original = findings_path.read_bytes() + _, output = canceled_discovery_output(workbench_db, scan) + write_checkpoint(output / "checkpoints", saved_draft(scan.scan_id)) + + with pytest.raises(ContractError, match="remediation|reportPath"): + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + + assert findings_path.read_bytes() == original + assert ( + workbench_db.execute( + "SELECT status FROM deep_scan_runs WHERE scan_id = ?", (scan.scan_id,) + ).fetchone()[0] + == "running" + ) + + +def test_deadline_preserves_worker_findings_with_optional_and_shared_identity( + workbench_api, workbench_db, deadline_scan +): + scan = deadline_scan + _, output = canceled_discovery_output(workbench_db, scan) + no_identity = copy.deepcopy(scan.findings[0]) + no_identity.pop("identity") + no_identity["title"] = "Worker finding without authored identity" + no_identity["provenance"] = {"source": "local_plugin"} + first = copy.deepcopy(no_identity) + first["title"] = "Shared semantic anchor at distinct locations" + first["identity"] = {"anchor": "shared-worker-anchor"} + first["locations"] = [{"path": "src/first.py", "startLine": 1}] + second = copy.deepcopy(first) + second["locations"] = [{"path": "src/second.py", "startLine": 1}] + malformed = copy.deepcopy(first) + malformed.pop("remediation") + write_checkpoint( + output / "checkpoints", + saved_draft(scan.scan_id, findings=[no_identity, first, second, malformed]), + ) + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + completed = complete(workbench_api, workbench_db, scan) + + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert len(findings) == 4 + assert any(item["title"] == no_identity["title"] for item in findings) + shared = [item for item in findings if item["identity"]["anchor"] == "shared-worker-anchor"] + assert {item["locations"][0]["path"] for item in shared} == {"src/first.py", "src/second.py"} + assert any("remediation" in warning for warning in completed["warnings"]) + + +def test_deadline_resolves_worker_local_deferred_ids_before_scoping( + workbench_api, workbench_db, deadline_scan +): + scan = deadline_scan + _, output = canceled_discovery_output(workbench_db, scan) + previous = write_checkpoint( + output / "checkpoints", + saved_draft(scan.scan_id, deferred=[{"id": "review", "reason": "Review pending."}]), + ) + os.utime(previous, ns=(100, 100)) + current = saved_draft(scan.scan_id, complete=True) + current["coverage"]["resolvedDeferred"] = [{"id": "review", "reason": "Review completed."}] + latest = write_checkpoint(output / "checkpoints", current) + os.utime(latest, ns=(200, 200)) + (output / "checkpoint-head.json").write_text(json.dumps({"checkpoint": latest.name})) + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + complete(workbench_api, workbench_db, scan) + + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert not any(item["reason"] == "Review pending." for item in coverage["deferred"]) + + +def test_deadline_does_not_turn_invalid_worker_receipts_into_parent_artifacts( + workbench_api, workbench_db, deadline_scan +): + scan = deadline_scan + _, output = canceled_discovery_output(workbench_db, scan) + (output / "notes.txt").write_text("Not a worker receipt under artifacts/.\n") + write_checkpoint( + output / "checkpoints", + saved_draft( + scan.scan_id, + surfaces=[ + { + "id": "entry", + "label": "Unverified review", + "disposition": "no_issue_found", + "receiptRefs": ["notes.txt", "../review.txt", "/artifacts/review.txt"], + } + ], + ), + ) + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + completed = complete(workbench_api, workbench_db, scan) + + surface = json.loads((scan.scan_dir / "coverage.json").read_text())["surfaces"][0] + assert surface["receiptRefs"] == [] + assert surface["disposition"] == "needs_follow_up" + assert ( + len( + [ + warning + for warning in completed["warnings"] + if "malformed coverage receipt" in warning + ] + ) + == 3 + ) + + +@pytest.mark.parametrize("archived", [False, True], ids=["current", "archived"]) +def test_deadline_preserves_worker_receipts_and_surface_links( + workbench_api, workbench_db, deadline_scan, archived +): + scan = deadline_scan + canonical_receipt = scan.scan_dir / "artifacts" / "reduced.txt" + canonical_receipt.parent.mkdir() + canonical_receipt.write_text("Reduced review evidence.\n") + canonical_surface = { + "id": f"{scan.completed_worker_id}/entry", + "workerId": scan.completed_worker_id, + "label": "Reduced review", + "disposition": "no_issue_found", + "receiptRefs": ["artifacts/reduced.txt"], + } + scan.coverage["surfaces"] = [canonical_surface] + (scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage)) + expected = {} + for label in ("upload", "download"): + worker_id, output = canceled_discovery_output( + workbench_db, scan, attempt=2 if archived else 1 + ) + source = output.parent / "attempts" / "attempt-1" if archived else output + receipt = source / "artifacts" / "review.txt" + receipt.parent.mkdir(parents=True) + receipt.write_text(f"Reviewed {label}.\n") + if archived: + current_receipt = output / "artifacts" / "review.txt" + current_receipt.parent.mkdir(parents=True) + current_receipt.write_text("Different current-attempt evidence.\n") + checkpoint = write_checkpoint( + source / "checkpoints", + saved_draft( + scan.scan_id, + surfaces=[ + { + "id": "entry", + "label": label, + "disposition": "no_issue_found", + "receiptRefs": ["artifacts/review.txt"], + } + ], + deferred=[{"id": "follow-up", "reason": label, "surfaceIds": ["entry"]}], + ), + ) + expected[label] = (worker_id, receipt, checkpoint, checkpoint.read_bytes()) + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + complete(workbench_api, workbench_db, scan) + + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert len(coverage["surfaces"]) == 3 + assert canonical_surface in coverage["surfaces"] + sources = json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"][ + "preservedSources" + ] + for label, (worker_id, receipt, checkpoint, original) in expected.items(): + surface = next(item for item in coverage["surfaces"] if item["label"] == label) + deferred = next(item for item in coverage["deferred"] if item["reason"] == label) + assert surface["id"] == f"{worker_id}/entry" + assert deferred["id"] == f"{worker_id}/follow-up" + assert deferred["surfaceIds"] == [surface["id"]] + assert surface["workerId"] == deferred["workerId"] == worker_id + assert surface["disposition"] == "no_issue_found" + assert surface["receiptRefs"] == [receipt.relative_to(scan.scan_dir).as_posix()] + assert (scan.scan_dir / surface["receiptRefs"][0]).read_text() == f"Reviewed {label}.\n" + assert sources[checkpoint.relative_to(scan.scan_dir).as_posix()] == workbench_api[ + "saved_results" + ]._digest(json.loads(original)) + assert checkpoint.read_bytes() == original + + def test_saturated_finish_cancels_unfinished_discovery_without_using_its_drafts( workbench_api, workbench_db, saturated_scan ): From f16400ad7f25b5ce5fc78eaece4bd39880ecce2f Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Sat, 17 Oct 2026 20:12:23 +0000 Subject: [PATCH 03/11] fix: isolate deadline recovery from canonical validation --- .../scripts/finalize_scan_contract.py | 11 +- .../scripts/workbench_saved_results.py | 79 +++- .../tests/test_deep_scan_deadline_recovery.py | 346 ++++++++++++++++++ 3 files changed, 427 insertions(+), 9 deletions(-) create mode 100644 plugins/codex-security/tests/test_deep_scan_deadline_recovery.py diff --git a/plugins/codex-security/scripts/finalize_scan_contract.py b/plugins/codex-security/scripts/finalize_scan_contract.py index f32249e2dd..286c461a85 100644 --- a/plugins/codex-security/scripts/finalize_scan_contract.py +++ b/plugins/codex-security/scripts/finalize_scan_contract.py @@ -1217,6 +1217,8 @@ def _recover_unsealed_coverage( scan_dir: Path, warnings: list[str], discarded_findings: list[str], + *, + worker_draft: bool = False, ) -> None: schema = _read_json(schema_dir / "coverage.schema.json") properties = _require_dict(schema, "properties", "coverage.schema") @@ -1249,7 +1251,7 @@ def _recover_unsealed_coverage( raise ContractError(f"{context}: expected an object") if field == "surfaces": surface_id = _require_str(item, "id", context) - if surface_id in surface_ids: + if surface_id in surface_ids and not worker_draft: raise ContractError(f"{context}.id: duplicate surface id") disposition = item.get("disposition") surface_recovered = False @@ -1310,6 +1312,13 @@ def _recover_unsealed_coverage( if discarded_findings: for surface in coverage["surfaces"]: + if ( + worker_draft + and surface.get("candidateId") + and surface["disposition"] in {"rejected", "not_applicable"} + ): + # A valid explicit outcome can still resolve that worker's history. + continue surface["disposition"] = "needs_follow_up" coverage["deferred"].extend( {"id": f"discarded-finding-{index}", "reason": warning} diff --git a/plugins/codex-security/scripts/workbench_saved_results.py b/plugins/codex-security/scripts/workbench_saved_results.py index b25ba82c44..61f8743592 100644 --- a/plugins/codex-security/scripts/workbench_saved_results.py +++ b/plugins/codex-security/scripts/workbench_saved_results.py @@ -29,6 +29,7 @@ _read_scan_local_json, _read_scan_local_json_bytes, _read_scan_local_json_with_metadata, + _recover_unsealed_coverage, _recover_unsealed_findings, _remove_scan_local_file_if_exists, _require_portable_relative_path, @@ -1085,6 +1086,7 @@ def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None if latest_reducer not in frozen_source_digests: latest_reducer = None + discarded_by_source: dict[str, list[str]] = {} for relative, worker_id in paths.items(): try: draft, digest, observed = _read_saved_result( @@ -1100,26 +1102,36 @@ def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None continue if preserve_canonical_parent and worker_id is not None: # Recover imported records without relaxing canonical parent validation. + directory = Path(relative).parent + if directory.name == "checkpoints": + directory = directory.parent recovered_findings = [] for index, finding in enumerate(draft["findings"]): _ensure_finding_identity(finding) recovered = {"scanId": scan_id, "findings": [finding]} source_warnings: list[str] = [] - _recover_unsealed_findings( + discarded = _recover_unsealed_findings( {"scan": {"id": scan_id, "target": binding["target"]}}, recovered, Path(__file__).resolve().parent.parent / "schemas", - scan_dir, + scan_dir / directory, source_warnings, ) + if discarded: + discarded_by_source.setdefault(relative, []).extend(discarded) + for recovered_finding in recovered["findings"]: + writeup = recovered_finding.pop("writeup", None) + if writeup is not None: + source_warnings.append( + "Retained worker writeup at " + f"{directory.as_posix()}/{writeup['reportPath']}; " + "the parent report renders this finding inline." + ) recovered_findings.extend(recovered["findings"]) warnings.extend( f"{relative} finding {index + 1}: {warning}" for warning in source_warnings ) draft["findings"] = recovered_findings - directory = Path(relative).parent - if directory.name == "checkpoints": - directory = directory.parent for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"): rows = draft["coverage"].get(field) if not isinstance(rows, list): @@ -1266,6 +1278,31 @@ def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None row["id"] = identity reserved.add(identity) + if preserve_canonical_parent: + for relative, draft, worker_id in sources: + if worker_id is None: + continue + for field in ("surfaces", "explicitExclusions", "deferred"): + rows = draft["coverage"].get(field) + for row in rows if isinstance(rows, list) else []: + if isinstance(row, dict): + row.setdefault("id", _saved_coverage_id(row)) + if field == "surfaces": + row.setdefault("receiptRefs", []) + source_warnings = [] + _recover_unsealed_coverage( + draft["coverage"], + Path(__file__).resolve().parent.parent / "schemas", + scan_dir, + source_warnings, + discarded_by_source.get(relative, []), + # Reconcile local IDs and valid outcomes against worker history below. + worker_draft=True, + ) + for row in draft["coverage"]["deferred"]: + row["workerId"] = worker_id + warnings.extend(f"{relative}: {warning}" for warning in source_warnings) + if parent is None and not sources: return None if ( @@ -1347,11 +1384,15 @@ def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None {"question": item.strip()} if isinstance(item, str) else item for item in coverage["openQuestions"] ] + # Closure reconciliation can reuse original parent rows instead of their copies. canonical_rows = ( { id(item) + for parent_coverage in (coverage, parent["coverage"]) for field in ("surfaces", "explicitExclusions", "deferred") - for item in (coverage.get(field) if isinstance(coverage.get(field), list) else []) + for item in ( + parent_coverage.get(field) if isinstance(parent_coverage.get(field), list) else [] + ) } if parent_is_canonical else set() @@ -1684,6 +1725,7 @@ def valid_finding(value: Any) -> bool: worker_id is not None and relative in selected_observations and draft.get("complete") is not False + and relative not in discarded_by_source ): order = source_order[relative] selected_terminal_orders[worker_id] = max( @@ -1691,7 +1733,11 @@ def valid_finding(value: Any) -> bool: ) terminal_worker_orders: dict[str | None, tuple[int, int]] = {} for relative, draft, worker_id in sources: - if relative in current_results and draft.get("complete") is not False: + if ( + relative in current_results + and draft.get("complete") is not False + and relative not in discarded_by_source + ): order = source_order[relative] terminal_worker_orders[worker_id] = max( terminal_worker_orders.get(worker_id, order), order @@ -2035,7 +2081,9 @@ def valid_finding(value: Any) -> bool: for field in ("surfaces", "explicitExclusions", "deferred"): used: set[str] = set() items = coverage.setdefault(field, []) + normalized_items = [] for item in items if isinstance(items, list) else []: + normalized_items.append(item) if not isinstance(item, dict): continue if id(item) in canonical_rows: @@ -2058,11 +2106,20 @@ def valid_finding(value: Any) -> bool: f"{worker_id}/{surface_id}" if isinstance(surface_id, str) else surface_id for surface_id in item["surfaceIds"] ] + if item in normalized_items[:-1]: + # A previous finish may have published files before its DB commit. + normalized_items.pop() + continue if item["id"] in used: item["id"] = f"{item['id']}-{_digest(item)[:16]}" + if preserve_canonical_parent and item in normalized_items[:-1]: + normalized_items.pop() + continue used.add(item["id"]) if field == "surfaces": item.setdefault("receiptRefs", []) + if preserve_canonical_parent and isinstance(items, list): + coverage[field] = normalized_items coverage.pop("completeness", None) if stopped: if not isinstance(coverage.get("deferred"), list): @@ -2152,6 +2209,12 @@ def preserve_deep_deadline_draft( if not workers: yield return + _prepare_scan_finalization( + scan_dir, + expected_coverage_mode=db.expected_coverage_mode(scan), + completion_binding=binding, + allow_missing_writeups=True, + ) with preserve_parent_head_on_error(scan_dir): try: documents = merge_saved_results( @@ -2186,7 +2249,7 @@ def preserve_deep_deadline_draft( scan_dir, expected_coverage_mode=db.expected_coverage_mode(scan), completion_binding=binding, - recover_unsealed=True, + recover_unsealed=not preserve_reduction, allow_missing_writeups=preserve_reduction, draft_documents=documents, ) diff --git a/plugins/codex-security/tests/test_deep_scan_deadline_recovery.py b/plugins/codex-security/tests/test_deep_scan_deadline_recovery.py new file mode 100644 index 0000000000..403c4ee39f --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_deadline_recovery.py @@ -0,0 +1,346 @@ +from __future__ import annotations + +import copy +import json +import os +import sqlite3 +import traceback + +import pytest +from test_deep_scan_stop_conditions import ( + canceled_discovery_output, + complete, + finish, +) +from test_deep_scan_stop_conditions import ( + deadline_scan as deadline_scan, +) +from test_deep_scan_stop_conditions import ( + publication_scan as publication_scan, +) +from test_deep_scan_stop_conditions import ( + saturated_scan as saturated_scan, +) +from workbench_test_support import saved_draft, write_checkpoint + + +def worker_finding(scan, anchor): + finding = copy.deepcopy(scan.findings[0]) + finding["identity"] = {"anchor": anchor} + finding["title"] = anchor + finding["provenance"] = {"source": "local_plugin"} + return finding + + +def test_discarded_worker_finding_leaves_its_review_pending( + workbench_api, workbench_db, deadline_scan +): + scan = deadline_scan + canonical = { + "id": "accepted", + "label": "Accepted review", + "disposition": "reported", + "receiptRefs": [], + } + scan.coverage["surfaces"] = [canonical] + (scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage)) + worker_id, output = canceled_discovery_output(workbench_db, scan) + invalid = worker_finding(scan, "incomplete-worker-finding") + invalid.pop("remediation") + checkpoint = write_checkpoint( + output / "checkpoints", + saved_draft( + scan.scan_id, + findings=[invalid], + surfaces=[ + { + "id": "entry", + "label": "Worker review", + "disposition": "reported", + "receiptRefs": [], + } + ], + ), + ) + original = checkpoint.read_bytes() + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + completed = complete(workbench_api, workbench_db, scan) + + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert canonical in coverage["surfaces"] + surface = next(row for row in coverage["surfaces"] if row.get("workerId") == worker_id) + assert surface["disposition"] == "needs_follow_up" + assert any( + row.get("workerId") == worker_id and "remediation" in row["reason"] + for row in coverage["deferred"] + ) + assert completed["progress"]["status"] == "complete" + assert len(json.loads((scan.scan_dir / "findings.json").read_text())["findings"]) == 1 + assert checkpoint.read_bytes() == original + + +@pytest.mark.parametrize( + "invalid_terminal", [False, True], ids=["valid-terminal", "invalid-terminal"] +) +def test_terminal_worker_authority_requires_valid_findings( + workbench_api, workbench_db, deadline_scan, invalid_terminal +): + scan = deadline_scan + _, output = canceled_discovery_output(workbench_db, scan) + previous = worker_finding(scan, "earlier-valid-finding") + checkpoint = write_checkpoint( + output / "checkpoints", + saved_draft( + scan.scan_id, + findings=[previous], + deferred=[{"id": "review", "reason": "Review pending."}], + ), + ) + os.utime(checkpoint, ns=(100, 100)) + sibling = worker_finding(scan, "valid-terminal-sibling") + terminal_findings = [sibling] + if invalid_terminal: + invalid = copy.deepcopy(previous) + invalid.pop("remediation") + terminal_findings.append(invalid) + result = output / "result.json" + result.write_text( + json.dumps( + saved_draft( + scan.scan_id, + complete=True, + findings=terminal_findings, + closures=[{"id": "review", "reason": "Review completed."}], + ) + ) + ) + os.utime(result, ns=(200, 200)) + original = {path: path.read_bytes() for path in (checkpoint, result)} + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + complete(workbench_api, workbench_db, scan) + + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert any(row["title"] == previous["title"] for row in findings) == invalid_terminal + assert any(row["title"] == sibling["title"] for row in findings) + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert not any(row["reason"] == "Review pending." for row in coverage["deferred"]) + assert {path: path.read_bytes() for path in original} == original + + +def test_malformed_worker_finding_preserves_a_valid_sibling_outcome( + workbench_api, workbench_db, deadline_scan +): + scan = deadline_scan + worker_id, output = canceled_discovery_output(workbench_db, scan) + earlier = worker_finding(scan, "rejected-candidate") + earlier["provenance"]["candidateId"] = "candidate" + checkpoint = write_checkpoint( + output / "checkpoints", saved_draft(scan.scan_id, findings=[earlier]) + ) + os.utime(checkpoint, ns=(100, 100)) + invalid = worker_finding(scan, "malformed-sibling") + invalid.pop("remediation") + outcome = { + "id": "review", + "label": "Candidate reviewed", + "candidateId": "candidate", + "disposition": "rejected", + "receiptRefs": [], + } + result = output / "result.json" + result.write_text( + json.dumps(saved_draft(scan.scan_id, complete=True, findings=[invalid], surfaces=[outcome])) + ) + os.utime(result, ns=(200, 200)) + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + complete(workbench_api, workbench_db, scan) + + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert all(row["title"] != earlier["title"] for row in findings) + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert ( + next(row for row in coverage["surfaces"] if row.get("workerId") == worker_id)["disposition"] + == "rejected" + ) + assert any("remediation" in row["reason"] for row in coverage["deferred"]) + + +@pytest.mark.parametrize("invalid", ["surfaces", "missing-receipt"]) +def test_deadline_keeps_canonical_coverage_validation_strict( + workbench_api, workbench_db, deadline_scan, invalid +): + from finalize_scan_contract import ContractError + + scan = deadline_scan + scan.coverage["surfaces"] = ( + "invalid" + if invalid == "surfaces" + else [ + { + "id": "accepted", + "label": "Accepted review", + "disposition": "no_issue_found", + "receiptRefs": ["artifacts/missing.txt"], + } + ] + ) + coverage_file = scan.scan_dir / "coverage.json" + coverage_file.write_text(json.dumps(scan.coverage)) + original = coverage_file.read_bytes() + _, output = canceled_discovery_output(workbench_db, scan) + write_checkpoint(output / "checkpoints", saved_draft(scan.scan_id)) + + with pytest.raises(ContractError, match="surfaces|receiptRefs"): + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + + assert coverage_file.read_bytes() == original + assert ( + workbench_db.execute( + "SELECT status FROM deep_scan_runs WHERE scan_id = ?", (scan.scan_id,) + ).fetchone()[0] + == "running" + ) + + +@pytest.mark.parametrize("worker_writeup_exists", [False, True], ids=["missing", "present"]) +def test_worker_writeups_remain_bound_to_their_source_directory( + workbench_api, workbench_db, deadline_scan, worker_writeup_exists +): + scan = deadline_scan + report_path = "findings/review/review.md" + scan.findings[0]["writeup"] = {"reportPath": report_path} + parent_writeup = scan.scan_dir / report_path + parent_writeup.parent.mkdir(parents=True) + parent_writeup.write_text("# Accepted parent writeup\n") + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": scan.findings})) + originals = {parent_writeup: parent_writeup.read_bytes()} + sources = [] + for archived in (False, True): + _, output = canceled_discovery_output(workbench_db, scan, attempt=2 if archived else 1) + source = output.parent / "attempts" / "attempt-1" if archived else output + sources.append(source.relative_to(scan.scan_dir).as_posix()) + finding = worker_finding(scan, "archived-worker" if archived else "current-worker") + if worker_writeup_exists: + writeup = source / report_path + writeup.parent.mkdir(parents=True) + writeup.write_text(f"# Separate worker writeup: {finding['title']}\n") + originals[writeup] = writeup.read_bytes() + checkpoint = write_checkpoint( + source / "checkpoints", saved_draft(scan.scan_id, findings=[finding]) + ) + originals[checkpoint] = checkpoint.read_bytes() + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + completed = complete(workbench_api, workbench_db, scan) + + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert len(findings) == 3 + assert next(row for row in findings if row["title"] == scan.findings[0]["title"])[ + "writeup" + ] == {"reportPath": report_path} + assert all( + "writeup" not in row for row in findings if row["title"] != scan.findings[0]["title"] + ) + writeup_warnings = [warning for warning in completed["warnings"] if "writeup" in warning] + assert len(writeup_warnings) == 2 + expected = ( + "Retained worker writeup at" if worker_writeup_exists else "Skipped malformed writeup" + ) + assert all(expected in warning for warning in writeup_warnings) + assert all(any(source in warning for warning in writeup_warnings) for source in sources) + report = (scan.scan_dir / "report.md").read_text() + assert "archived-worker" in report and "current-worker" in report + assert {path: path.read_bytes() for path in originals} == originals + + +@pytest.mark.skipif( + not hasattr(os, "fork"), reason="Crash boundary uses a forked workbench process." +) +@pytest.mark.parametrize("colliding_ids", [False, True], ids=["distinct-ids", "legacy-collision"]) +@pytest.mark.parametrize( + "parent_closure", [False, True], ids=["no-parent-closure", "parent-closure"] +) +def test_deadline_recovery_is_idempotent_after_crash_before_commit( + workbench_api, workbench_db, deadline_scan, tmp_path, colliding_ids, parent_closure +): + scan = deadline_scan + if parent_closure: + scan.coverage["resolvedDeferred"] = [ + {"id": "prior-review", "reason": "Earlier review completed."} + ] + (scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage)) + database = tmp_path / "crash-recovery.db" + connection = sqlite3.connect(database) + connection.row_factory = sqlite3.Row + workbench_db.backup(connection) + worker_id, output = canceled_discovery_output(connection, scan) + surfaces = [ + { + "id": "entry", + "label": "Worker review", + "disposition": "needs_follow_up", + "receiptRefs": [], + } + ] + if colliding_ids: + surfaces.append({**surfaces[0], "label": "Separate worker review"}) + write_checkpoint( + output / "checkpoints", + saved_draft( + scan.scan_id, + surfaces=surfaces, + findings=[worker_finding(scan, "saved-before-crash"), copy.deepcopy(scan.findings[0])], + deferred=[{"id": "review", "reason": "Review pending.", "surfaceIds": ["entry"]}], + ), + ) + connection.close() + child = os.fork() + if child == 0: + connection = sqlite3.connect(database) + connection.row_factory = sqlite3.Row + + class CrashBeforeCommit: + def __getattr__(self, name): + return getattr(connection, name) + + def commit(self): + os._exit(0) + + try: + finish(workbench_api, CrashBeforeCommit(), scan, terminal_reason="capped") + except BaseException: + traceback.print_exc() + os._exit(1) + _, status = os.waitpid(child, 0) + assert os.waitstatus_to_exitcode(status) == 0 + before = json.loads((scan.scan_dir / "coverage.json").read_text()) + findings_before = json.loads((scan.scan_dir / "findings.json").read_text()) + assert len([row for row in before["surfaces"] if row.get("workerId") == worker_id]) == len( + surfaces + ) + with sqlite3.connect(database) as connection: + connection.row_factory = sqlite3.Row + assert ( + connection.execute( + "SELECT status FROM deep_scan_runs WHERE scan_id = ?", (scan.scan_id,) + ).fetchone()[0] + == "running" + ) + finish(workbench_api, connection, scan, terminal_reason="capped") + complete(workbench_api, connection, scan) + + after = json.loads((scan.scan_dir / "coverage.json").read_text()) + findings_after = json.loads((scan.scan_dir / "findings.json").read_text()) + for document in (findings_before, findings_after): + for finding in document["findings"]: + # Existing recovery can materialize an empty optional history list. + if finding["provenance"].get("previousFindings") == []: + finding["provenance"].pop("previousFindings") + assert findings_after == findings_before + for field in ("surfaces", "deferred"): + assert [row for row in after[field] if row.get("workerId") == worker_id] == [ + row for row in before[field] if row.get("workerId") == worker_id + ] From bc404ea4775c94a846af583bf2cba5112b057241 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Sat, 17 Oct 2026 20:40:23 +0000 Subject: [PATCH 04/11] fix: preserve deadline coverage and replay identity --- .../scripts/finalize_scan_contract.py | 21 +- .../scripts/workbench_saved_results.py | 103 ++++++-- .../test_deep_scan_deadline_contracts.py | 248 ++++++++++++++++++ .../tests/test_deep_scan_deadline_recovery.py | 49 +++- 4 files changed, 387 insertions(+), 34 deletions(-) create mode 100644 plugins/codex-security/tests/test_deep_scan_deadline_contracts.py diff --git a/plugins/codex-security/scripts/finalize_scan_contract.py b/plugins/codex-security/scripts/finalize_scan_contract.py index 286c461a85..22f8d12a6e 100644 --- a/plugins/codex-security/scripts/finalize_scan_contract.py +++ b/plugins/codex-security/scripts/finalize_scan_contract.py @@ -1162,6 +1162,7 @@ def _recover_unsealed_findings( f"Skipped malformed {auxiliary} for finding {index + 1}: {exc}." ) _validate_schema_node(finding, finding_schema, context, schema) + _require_safe_json_value(finding, context) except ContractError as exc: warning = f"Skipped malformed finding {index + 1}: {exc}." warnings.append(warning) @@ -1219,9 +1220,10 @@ def _recover_unsealed_coverage( discarded_findings: list[str], *, worker_draft: bool = False, -) -> None: +) -> set[str]: schema = _read_json(schema_dir / "coverage.schema.json") properties = _require_dict(schema, "properties", "coverage.schema") + discarded_fields: set[str] = set() if ( coverage.get("mode") == "deep_repository" and coverage.get("inventoryStrategy") != "repository" @@ -1230,16 +1232,24 @@ def _recover_unsealed_coverage( warnings.append("Recovered malformed Deep Scan inventory strategy.") surface_ids: set[str] = set() - for field, label in ( + fields = [ ("surfaces", "coverage surface"), ("explicitExclusions", "coverage exclusion"), ("deferred", "deferred coverage item"), - ): + ] + if worker_draft: + if "openQuestions" in coverage: + _normalize_unsealed_open_questions(coverage) + fields.append(("openQuestions", "coverage open question")) + if "resolvedDeferred" in coverage: + fields.append(("resolvedDeferred", "resolved deferred item")) + for field, label in fields: array_schema = _require_dict(properties, field, "coverage.schema.properties") item_schema = _require_dict(array_schema, "items", f"coverage.schema.properties.{field}") items = coverage.get(field) if not isinstance(items, list): warnings.append(f"Skipped malformed {label} records: expected an array.") + discarded_fields.add(field) coverage[field] = [] continue @@ -1300,8 +1310,10 @@ def _recover_unsealed_coverage( if field == "deferred": _validate_deferred_paths(item, context) _validate_schema_node(item, item_schema, context) + _require_safe_json_value(item, context) except ContractError as exc: warnings.append(f"Skipped malformed {label} {index + 1}: {exc}.") + discarded_fields.add(field) continue if field == "surfaces": @@ -1333,9 +1345,12 @@ def _recover_unsealed_coverage( "coverage.resolvedDeferred", ) _validate_resolved_deferred(coverage) + _require_safe_json_value(coverage["resolvedDeferred"], "coverage.resolvedDeferred") except ContractError as exc: coverage.pop("resolvedDeferred") warnings.append(f"Skipped malformed resolved deferred work: {exc}.") + discarded_fields.add("resolvedDeferred") + return discarded_fields def _recover_unsealed_hardening( diff --git a/plugins/codex-security/scripts/workbench_saved_results.py b/plugins/codex-security/scripts/workbench_saved_results.py index 61f8743592..9207b8329c 100644 --- a/plugins/codex-security/scripts/workbench_saved_results.py +++ b/plugins/codex-security/scripts/workbench_saved_results.py @@ -33,6 +33,7 @@ _recover_unsealed_findings, _remove_scan_local_file_if_exists, _require_portable_relative_path, + _require_safe_json_value, _validate_completion_binding, _validate_resolved_deferred, _validate_schema_node, @@ -745,6 +746,7 @@ def _generic_surface_updates( surface_schema: dict[str, Any], deferred_rows: dict[str, list[Any]], ambiguous_deferred: set[tuple[str | None, str]], + canonical_rows: set[int], ) -> tuple[set[int], list[dict[str, Any]]]: if not closed_deferred and not reopened_generic: return set(), [] @@ -855,6 +857,8 @@ def linked(row: dict[str, Any], identity: str | None) -> bool: ) if update not in updates: updates.append(update) + if id(latest_surface) in canonical_rows: + canonical_rows.add(id(update)) return replaced, updates @@ -1102,6 +1106,26 @@ def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None continue if preserve_canonical_parent and worker_id is not None: # Recover imported records without relaxing canonical parent validation. + if "threatModel" in draft: + try: + model = draft["threatModel"] + if isinstance(model, dict): + model = {**model, "origin": "recovered"} + schema = _read_json( + Path(__file__).resolve().parent.parent + / "schemas" + / "scan-manifest.schema.json" + ) + _validate_schema_node( + model, + schema["properties"]["scan"]["properties"]["threatModel"], + "threatModel", + schema, + ) + _require_safe_json_value(model, "threatModel") + except ContractError as exc: + draft.pop("threatModel") + warnings.append(f"{relative}: Skipped malformed threat model: {exc}.") directory = Path(relative).parent if directory.name == "checkpoints": directory = directory.parent @@ -1278,6 +1302,7 @@ def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None row["id"] = identity reserved.add(identity) + discarded_coverage_sources: set[str] = set() if preserve_canonical_parent: for relative, draft, worker_id in sources: if worker_id is None: @@ -1290,7 +1315,7 @@ def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None if field == "surfaces": row.setdefault("receiptRefs", []) source_warnings = [] - _recover_unsealed_coverage( + discarded_fields = _recover_unsealed_coverage( draft["coverage"], Path(__file__).resolve().parent.parent / "schemas", scan_dir, @@ -1299,6 +1324,13 @@ def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None # Reconcile local IDs and valid outcomes against worker history below. worker_draft=True, ) + if discarded_fields & { + "surfaces", + "explicitExclusions", + "deferred", + "resolvedDeferred", + }: + discarded_coverage_sources.add(relative) for row in draft["coverage"]["deferred"]: row["workerId"] = worker_id warnings.extend(f"{relative}: {warning}" for warning in source_warnings) @@ -1388,7 +1420,9 @@ def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None canonical_rows = ( { id(item) - for parent_coverage in (coverage, parent["coverage"]) + for parent_coverage in ( + (coverage, parent["coverage"]) if preserve_canonical_parent else (coverage,) + ) for field in ("surfaces", "explicitExclusions", "deferred") for item in ( parent_coverage.get(field) if isinstance(parent_coverage.get(field), list) else [] @@ -1644,6 +1678,7 @@ def valid_finding(value: Any) -> bool: coverage_schema["surfaces"]["items"], deferred_rows, ambiguous_deferred, + canonical_rows, ) replaced_rows = { "surfaces": replaced_surfaces, @@ -1726,12 +1761,14 @@ def valid_finding(value: Any) -> bool: and relative in selected_observations and draft.get("complete") is not False and relative not in discarded_by_source + and relative not in discarded_coverage_sources ): order = source_order[relative] selected_terminal_orders[worker_id] = max( selected_terminal_orders.get(worker_id, order), order ) terminal_worker_orders: dict[str | None, tuple[int, int]] = {} + terminal_coverage_orders: dict[str | None, tuple[int, int]] = {} for relative, draft, worker_id in sources: if ( relative in current_results @@ -1742,6 +1779,10 @@ def valid_finding(value: Any) -> bool: terminal_worker_orders[worker_id] = max( terminal_worker_orders.get(worker_id, order), order ) + if relative not in discarded_coverage_sources: + terminal_coverage_orders[worker_id] = max( + terminal_coverage_orders.get(worker_id, order), order + ) for relative, draft, worker_id in all_sources: worker_result_order = terminal_worker_orders.get(worker_id) selected_coverage_superseded = worker_id in selected_terminal_orders and ( @@ -1771,14 +1812,27 @@ def valid_finding(value: Any) -> bool: or source_order[relative] < worker_result_order ) ) + worker_coverage_order = terminal_coverage_orders.get(worker_id) + coverage_superseded = ( + superseded + if worker_id is None + else ( + relative not in current_results + and worker_coverage_order is not None + and ( + relative not in selected_observations + or source_order[relative] < worker_coverage_order + ) + ) + ) # A failed result write can leave pending work outside the accepted result. accepted_order = ( (0, parent_modified) if worker_id is None and parent is not None - else worker_result_order + else worker_coverage_order ) retain_pending = ( - superseded + coverage_superseded and (worker_id in headed_workers or (worker_id is None and parent_heads)) and accepted_order is not None and source_order[relative] >= accepted_order @@ -1789,7 +1843,12 @@ def valid_finding(value: Any) -> bool: and not stopped and all(valid_finding(finding) for finding in (parent["findings"] if parent else [])) ) - if skip_superseded_findings and not selected_candidates and not retain_pending: + if ( + skip_superseded_findings + and coverage_superseded + and not selected_candidates + and not retain_pending + ): continue if ( not skip_superseded_findings @@ -1967,10 +2026,10 @@ def valid_finding(value: Any) -> bool: continue finding_positions[key] = len(findings) findings.append(finding) - if superseded and not selected_candidates and not retain_pending: + if coverage_superseded and not selected_candidates and not retain_pending: continue for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"): - if superseded and field not in {"surfaces", "explicitExclusions", "deferred"}: + if coverage_superseded and field not in {"surfaces", "explicitExclusions", "deferred"}: continue items = draft["coverage"].get(field, []) if field == "deferred": @@ -1995,7 +2054,7 @@ def valid_finding(value: Any) -> bool: ): continue # A selected outcome must retain its evidence even if its result write failed. - if superseded and not ( + if coverage_superseded and not ( isinstance(item, dict) and ( (field == "deferred" and retain_pending) @@ -2144,9 +2203,11 @@ def coverage_for_comparison(db: Any, scan: Any) -> dict[str, Any]: return prepared.coverage -def _snapshot_published_outputs(scan_dir: Path) -> dict[str, tuple[bytes, os.stat_result] | None]: +def _snapshot_published_outputs( + scan_dir: Path, relative_paths: tuple[str, ...] = _PUBLISHED_OUTPUTS +) -> dict[str, tuple[bytes, os.stat_result] | None]: snapshots: dict[str, tuple[bytes, os.stat_result] | None] = {} - for relative in _PUBLISHED_OUTPUTS: + for relative in relative_paths: descriptor = -1 try: descriptor = open_scan_local_file_descriptor( @@ -2191,7 +2252,6 @@ def preserve_deep_deadline_draft( ) -> Iterator[None]: """Recover unfinished reviews at a settled Deep deadline without sealing the parent.""" scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"])) - snapshots = _snapshot_published_outputs(scan_dir) warnings = json.loads(scan["completion_warnings_json"]) binding = db.workbench_completion_binding(scan, db.now()) workers = connection.execute( @@ -2215,6 +2275,9 @@ def preserve_deep_deadline_draft( completion_binding=binding, allow_missing_writeups=True, ) + snapshots = _snapshot_published_outputs( + scan_dir, ("findings.json", "coverage.json", "scan-manifest.json") + ) with preserve_parent_head_on_error(scan_dir): try: documents = merge_saved_results( @@ -2233,15 +2296,15 @@ def preserve_deep_deadline_draft( if documents is None: raise ContractError("Deep Scan deadline has no saved parent draft.") if preserve_reduction: - documents[2]["deferred"].append( - { - "id": "scan-deadline", - "reason": ( - "The configured discovery time limit elapsed. Saved findings and " - "pending work from unfinished source reviews were preserved." - ), - } - ) + marker = { + "id": "scan-deadline", + "reason": ( + "The configured discovery time limit elapsed. Saved findings and " + "pending work from unfinished source reviews were preserved." + ), + } + if marker not in documents[2]["deferred"]: + documents[2]["deferred"].append(marker) documents[0]["scan"].pop("complete", None) # Validate recovered evidence against the current finding contract, # scope, source lineage, and receipts before admitting this draft. diff --git a/plugins/codex-security/tests/test_deep_scan_deadline_contracts.py b/plugins/codex-security/tests/test_deep_scan_deadline_contracts.py new file mode 100644 index 0000000000..3c7e54b9a4 --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_deadline_contracts.py @@ -0,0 +1,248 @@ +from __future__ import annotations + +import copy +import json +import os + +import pytest +from test_deep_scan_deadline_recovery import worker_finding +from test_deep_scan_stop_conditions import canceled_discovery_output, complete, finish +from test_deep_scan_stop_conditions import deadline_scan as deadline_scan +from test_deep_scan_stop_conditions import publication_scan as publication_scan +from test_deep_scan_stop_conditions import saturated_scan as saturated_scan +from workbench_test_support import saved_draft, write_checkpoint + + +@pytest.mark.parametrize( + "model", + [ + {"assets": []}, + {"summary": "Invalid " + chr(0xD800)}, + {"summary": "Reviewed boundaries."}, + {"format": "markdown", "content": "# Reviewed boundaries"}, + ], +) +def test_deadline_recovers_optional_worker_model_without_losing_findings( + workbench_api, workbench_db, deadline_scan, model +): + scan = deadline_scan + _, output = canceled_discovery_output(workbench_db, scan) + finding = worker_finding(scan, "worker-model-finding") + draft = {**saved_draft(scan.scan_id, findings=[finding]), "threatModel": model} + checkpoint = write_checkpoint(output / "checkpoints", draft) + original = checkpoint.read_bytes() + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + completed = complete(workbench_api, workbench_db, scan) + + assert len(json.loads((scan.scan_dir / "findings.json").read_text())["findings"]) == 2 + manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"] + invalid = "assets" in model or chr(0xD800) in model.get("summary", "") + if invalid: + assert "threatModel" not in manifest + assert any( + "threat model" in warning and checkpoint.name in warning + for warning in completed["warnings"] + ) + else: + assert manifest["threatModel"] == {**model, "origin": "recovered"} + assert checkpoint.read_bytes() == original + + +def test_deadline_still_rejects_malformed_canonical_model( + workbench_api, workbench_db, deadline_scan +): + from finalize_scan_contract import ContractError + + scan = deadline_scan + manifest_file = scan.scan_dir / "scan-manifest.json" + manifest = json.loads(manifest_file.read_text()) + manifest["scan"]["threatModel"] = {"assets": []} + manifest_file.write_text(json.dumps(manifest)) + _, output = canceled_discovery_output(workbench_db, scan) + write_checkpoint(output / "checkpoints", saved_draft(scan.scan_id)) + with pytest.raises(ContractError, match="threatModel"): + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + + +@pytest.mark.parametrize( + "field", + ["finding", "surfaces", "explicitExclusions", "deferred", "openQuestions", "resolvedDeferred"], +) +@pytest.mark.parametrize("malformed", [False, True], ids=["valid-unicode", "lone-surrogate"]) +def test_deadline_applies_existing_unicode_contract_per_worker_record( + workbench_api, workbench_db, deadline_scan, field, malformed +): + scan = deadline_scan + _, output = canceled_discovery_output(workbench_db, scan) + text = "Review " + (chr(0xD800) if malformed else "Résumé 😀") + finding = worker_finding(scan, "unicode-worker-finding") + draft = saved_draft(scan.scan_id, findings=[finding]) + if field == "finding": + finding["summary"] = text + elif field == "surfaces": + draft["coverage"][field] = [ + {"id": "entry", "label": text, "disposition": "reported", "receiptRefs": []} + ] + elif field == "openQuestions": + draft["coverage"][field] = [text] + elif field == "explicitExclusions": + draft["coverage"][field] = [{"pattern": "vendor/", "reason": text}] + else: + draft["coverage"][field] = [{"id": "review", "reason": text}] + if field == "resolvedDeferred": + draft["complete"] = True + previous = write_checkpoint( + output / "checkpoints", + saved_draft(scan.scan_id, deferred=[{"id": "review", "reason": "Review pending."}]), + ) + os.utime(previous, ns=(100, 100)) + checkpoint = write_checkpoint(output / "checkpoints", draft) + os.utime(checkpoint, ns=(200, 200)) + if field == "resolvedDeferred": + (output / "checkpoint-head.json").write_text(json.dumps({"checkpoint": checkpoint.name})) + original = checkpoint.read_bytes() + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + completed = complete(workbench_api, workbench_db, scan) + + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert len(findings) == (1 if malformed and field == "finding" else 2) + assert any("well-formed Unicode" in warning for warning in completed["warnings"]) == malformed + if field == "resolvedDeferred": + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert any(row["reason"] == "Review pending." for row in coverage["deferred"]) == malformed + elif not malformed: + document = ( + findings + if field == "finding" + else json.loads((scan.scan_dir / "coverage.json").read_text()) + ) + assert text in json.dumps(document, ensure_ascii=False) + assert checkpoint.read_bytes() == original + + +@pytest.mark.parametrize( + ("terminal_findings", "malformed", "selected_head"), + [ + (False, None, False), + (True, None, False), + (False, "surface", False), + (True, "surface", False), + (False, "closure", False), + (True, "closure", False), + (True, "surface", True), + ], + ids=[ + "valid-coverage-empty-findings", + "valid-coverage-retained-finding", + "malformed-surface-empty-findings", + "malformed-surface-retained-finding", + "malformed-closure-empty-findings", + "malformed-closure-retained-finding", + "malformed-selected-head", + ], +) +def test_malformed_terminal_coverage_preserves_only_earlier_coverage( + workbench_api, workbench_db, deadline_scan, terminal_findings, malformed, selected_head +): + scan = deadline_scan + worker_id, output = canceled_discovery_output(workbench_db, scan) + finding = worker_finding(scan, "earlier-worker-finding") + surface = { + "id": "entry", + "label": "Unfinished review", + "disposition": "needs_follow_up", + "receiptRefs": [], + } + checkpoint = write_checkpoint( + output / "checkpoints", + saved_draft( + scan.scan_id, + findings=[finding], + surfaces=[surface], + deferred=[ + {"id": "review", "reason": "Review remains.", "surfaceIds": ["entry"]}, + {"id": "finished", "reason": "Separate review remains."}, + ], + ), + ) + os.utime(checkpoint, ns=(100, 100)) + invalid_surface = copy.deepcopy(surface) + invalid_surface.pop("label") + closures = [{"id": "finished", "reason": "Separate review completed."}] + if malformed == "closure": + closures.append({"id": "review", "reason": "Invalid " + chr(0xD800)}) + terminal = saved_draft( + scan.scan_id, + complete=True, + findings=[finding] if terminal_findings else [], + surfaces=[invalid_surface] if malformed == "surface" else [], + closures=closures, + ) + if selected_head: + result = write_checkpoint(output / "checkpoints", terminal) + (output / "checkpoint-head.json").write_text(json.dumps({"checkpoint": result.name})) + else: + result = output / "result.json" + result.write_text(json.dumps(terminal)) + os.utime(result, ns=(200, 200)) + original = {path: path.read_bytes() for path in (checkpoint, result)} + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + complete(workbench_api, workbench_db, scan) + + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert any(row["title"] == finding["title"] for row in findings) == terminal_findings + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert any( + row.get("workerId") == worker_id and row["label"] == surface["label"] + for row in coverage["surfaces"] + ) == bool(malformed) + assert any(row["reason"] == "Review remains." for row in coverage["deferred"]) == bool( + malformed + ) + assert not any(row["reason"] == "Separate review remains." for row in coverage["deferred"]) + assert {path: path.read_bytes() for path in original} == original + + +@pytest.mark.parametrize("canceled_worker", [False, True]) +def test_optional_export_failure_does_not_block_deadline( + workbench_api, workbench_db, deadline_scan, tmp_path, capsys, canceled_worker +): + scan = deadline_scan + sentinel = tmp_path / "untouched.sarif" + sentinel.write_text("Preserve this file.\n") + export = scan.scan_dir / "exports" / "results.sarif" + export.parent.mkdir() + export.symlink_to(sentinel) + if canceled_worker: + _, output = canceled_discovery_output(workbench_db, scan) + write_checkpoint( + output / "checkpoints", + saved_draft(scan.scan_id, findings=[worker_finding(scan, "saved-worker-finding")]), + ) + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + completed = complete(workbench_api, workbench_db, scan) + + assert completed["progress"]["status"] == "complete" + assert "automatic SARIF export failed" in capsys.readouterr().err + assert export.is_symlink() + assert sentinel.read_text() == "Preserve this file.\n" + + +def test_standard_parent_closure_preserves_unrelated_task_without_id( + workbench_api, workbench_db, publication_scan +): + scan = publication_scan(mode="standard") + scan.coverage["resolvedDeferred"] = [{"id": "finished", "reason": "Review completed."}] + scan.coverage["deferred"] = [{"reason": "Unrelated pending review."}] + (scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage)) + + complete(workbench_api, workbench_db, scan) + + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert len(coverage["deferred"]) == 1 + assert coverage["deferred"][0]["id"] + assert coverage["deferred"][0]["reason"] == "Unrelated pending review." diff --git a/plugins/codex-security/tests/test_deep_scan_deadline_recovery.py b/plugins/codex-security/tests/test_deep_scan_deadline_recovery.py index 403c4ee39f..91a8a7dd14 100644 --- a/plugins/codex-security/tests/test_deep_scan_deadline_recovery.py +++ b/plugins/codex-security/tests/test_deep_scan_deadline_recovery.py @@ -259,12 +259,34 @@ def test_worker_writeups_remain_bound_to_their_source_directory( @pytest.mark.skipif( not hasattr(os, "fork"), reason="Crash boundary uses a forked workbench process." ) -@pytest.mark.parametrize("colliding_ids", [False, True], ids=["distinct-ids", "legacy-collision"]) @pytest.mark.parametrize( - "parent_closure", [False, True], ids=["no-parent-closure", "parent-closure"] + ("colliding_ids", "parent_closure", "reviewed", "worker_like_id"), + [ + (False, False, False, False), + (True, False, False, False), + (False, True, False, False), + (True, True, False, False), + (False, True, True, False), + (False, True, True, True), + ], + ids=[ + "pending", + "legacy-collision", + "parent-closure", + "parent-closure-with-collision", + "reviewed-surface", + "reviewed-surface-with-worker-like-id", + ], ) def test_deadline_recovery_is_idempotent_after_crash_before_commit( - workbench_api, workbench_db, deadline_scan, tmp_path, colliding_ids, parent_closure + workbench_api, + workbench_db, + deadline_scan, + tmp_path, + colliding_ids, + parent_closure, + reviewed, + worker_like_id, ): scan = deadline_scan if parent_closure: @@ -277,11 +299,12 @@ def test_deadline_recovery_is_idempotent_after_crash_before_commit( connection.row_factory = sqlite3.Row workbench_db.backup(connection) worker_id, output = canceled_discovery_output(connection, scan) + surface_id = f"{worker_id}/entry" if worker_like_id else "entry" surfaces = [ { - "id": "entry", + "id": surface_id, "label": "Worker review", - "disposition": "needs_follow_up", + "disposition": "no_issue_found" if reviewed else "needs_follow_up", "receiptRefs": [], } ] @@ -292,8 +315,12 @@ def test_deadline_recovery_is_idempotent_after_crash_before_commit( saved_draft( scan.scan_id, surfaces=surfaces, - findings=[worker_finding(scan, "saved-before-crash"), copy.deepcopy(scan.findings[0])], - deferred=[{"id": "review", "reason": "Review pending.", "surfaceIds": ["entry"]}], + findings=[] + if reviewed + else [worker_finding(scan, "saved-before-crash"), copy.deepcopy(scan.findings[0])], + deferred=[] + if reviewed + else [{"id": "review", "reason": "Review pending.", "surfaceIds": [surface_id]}], ), ) connection.close() @@ -321,6 +348,9 @@ def commit(self): assert len([row for row in before["surfaces"] if row.get("workerId") == worker_id]) == len( surfaces ) + if reviewed: + assert before["surfaces"][0]["id"] == f"{worker_id}/{surface_id}" + assert len([row for row in before["deferred"] if row["id"] == "scan-deadline"]) == 1 with sqlite3.connect(database) as connection: connection.row_factory = sqlite3.Row assert ( @@ -340,7 +370,4 @@ def commit(self): if finding["provenance"].get("previousFindings") == []: finding["provenance"].pop("previousFindings") assert findings_after == findings_before - for field in ("surfaces", "deferred"): - assert [row for row in after[field] if row.get("workerId") == worker_id] == [ - row for row in before[field] if row.get("workerId") == worker_id - ] + assert after == before From 4438db85b39cd43d3afbbcf455e156fa1dbc222b Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Sat, 17 Oct 2026 20:59:30 +0000 Subject: [PATCH 05/11] fix: stabilize recovered deadline coverage --- .../scripts/finalize_scan_contract.py | 2 + .../scripts/workbench_saved_results.py | 8 +- .../test_deep_scan_deadline_contracts.py | 84 ++++++++++++- .../tests/test_deep_scan_deadline_recovery.py | 119 +++++++++++++++--- 4 files changed, 190 insertions(+), 23 deletions(-) diff --git a/plugins/codex-security/scripts/finalize_scan_contract.py b/plugins/codex-security/scripts/finalize_scan_contract.py index 22f8d12a6e..104cf49e2d 100644 --- a/plugins/codex-security/scripts/finalize_scan_contract.py +++ b/plugins/codex-security/scripts/finalize_scan_contract.py @@ -1261,6 +1261,7 @@ def _recover_unsealed_coverage( raise ContractError(f"{context}: expected an object") if field == "surfaces": surface_id = _require_str(item, "id", context) + _require_str(item, "label", context) if surface_id in surface_ids and not worker_draft: raise ContractError(f"{context}.id: duplicate surface id") disposition = item.get("disposition") @@ -1306,6 +1307,7 @@ def _recover_unsealed_coverage( item["receiptRefs"] = recovered_receipts if surface_recovered: item["disposition"] = "needs_follow_up" + discarded_fields.add(field) if field == "deferred": _validate_deferred_paths(item, context) diff --git a/plugins/codex-security/scripts/workbench_saved_results.py b/plugins/codex-security/scripts/workbench_saved_results.py index 9207b8329c..e133bf1543 100644 --- a/plugins/codex-security/scripts/workbench_saved_results.py +++ b/plugins/codex-security/scripts/workbench_saved_results.py @@ -1923,9 +1923,15 @@ def valid_finding(value: Any) -> bool: "rejected", "not_applicable", }: + if preserve_canonical_parent: + rejected_history.setdefault((worker_id, candidate_id), []).append(finding) surfaces = coverage.get("surfaces") for item in surfaces if isinstance(surfaces, list) else []: - if isinstance(item, dict) and item.get("candidateId") == candidate_id: + if ( + isinstance(item, dict) + and item.get("candidateId") == candidate_id + and (not preserve_canonical_parent or item.get("workerId") == worker_id) + ): if not isinstance(item.get("previousFindings"), list): item["previousFindings"] = [] history = item["previousFindings"] diff --git a/plugins/codex-security/tests/test_deep_scan_deadline_contracts.py b/plugins/codex-security/tests/test_deep_scan_deadline_contracts.py index 3c7e54b9a4..a208beb93a 100644 --- a/plugins/codex-security/tests/test_deep_scan_deadline_contracts.py +++ b/plugins/codex-security/tests/test_deep_scan_deadline_contracts.py @@ -65,6 +65,41 @@ def test_deadline_still_rejects_malformed_canonical_model( finish(workbench_api, workbench_db, scan, terminal_reason="capped") +@pytest.mark.parametrize("label", [" \t ", " Reviewed Résumé 😀 "]) +def test_deadline_applies_existing_nonblank_surface_label_contract( + workbench_api, workbench_db, deadline_scan, label +): + scan = deadline_scan + worker_id, output = canceled_discovery_output(workbench_db, scan) + checkpoint = write_checkpoint( + output / "checkpoints", + saved_draft( + scan.scan_id, + findings=[worker_finding(scan, "worker-label-finding")], + surfaces=[ + {"id": "entry", "label": label, "disposition": "reported", "receiptRefs": []} + ], + ), + ) + original = checkpoint.read_bytes() + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + completed = complete(workbench_api, workbench_db, scan) + + assert len(json.loads((scan.scan_dir / "findings.json").read_text())["findings"]) == 2 + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + surfaces = [row for row in coverage["surfaces"] if row.get("workerId") == worker_id] + assert len(surfaces) == bool(label.strip()) + if surfaces: + assert surfaces[0]["label"] == label + else: + assert any( + "label: expected a non-empty string" in warning and checkpoint.name in warning + for warning in completed["warnings"] + ) + assert checkpoint.read_bytes() == original + + @pytest.mark.parametrize( "field", ["finding", "surfaces", "explicitExclusions", "deferred", "openQuestions", "resolvedDeferred"], @@ -132,6 +167,12 @@ def test_deadline_applies_existing_unicode_contract_per_worker_record( (False, "closure", False), (True, "closure", False), (True, "surface", True), + (False, "missing-receipt", False), + (True, "missing-receipt", False), + (False, "disposition", False), + (True, "disposition", False), + (False, "receipt-array", False), + (True, "receipt-array", False), ], ids=[ "valid-coverage-empty-findings", @@ -141,6 +182,12 @@ def test_deadline_applies_existing_unicode_contract_per_worker_record( "malformed-closure-empty-findings", "malformed-closure-retained-finding", "malformed-selected-head", + "missing-receipt-empty-findings", + "missing-receipt-retained-finding", + "disposition-empty-findings", + "disposition-retained-finding", + "receipt-array-empty-findings", + "receipt-array-retained-finding", ], ) def test_malformed_terminal_coverage_preserves_only_earlier_coverage( @@ -149,6 +196,8 @@ def test_malformed_terminal_coverage_preserves_only_earlier_coverage( scan = deadline_scan worker_id, output = canceled_discovery_output(workbench_db, scan) finding = worker_finding(scan, "earlier-worker-finding") + rejected = worker_finding(scan, "rejected-sibling") + rejected["provenance"]["candidateId"] = "rejected-sibling" surface = { "id": "entry", "label": "Unfinished review", @@ -159,7 +208,7 @@ def test_malformed_terminal_coverage_preserves_only_earlier_coverage( output / "checkpoints", saved_draft( scan.scan_id, - findings=[finding], + findings=[finding, rejected], surfaces=[surface], deferred=[ {"id": "review", "reason": "Review remains.", "surfaceIds": ["entry"]}, @@ -169,7 +218,16 @@ def test_malformed_terminal_coverage_preserves_only_earlier_coverage( ) os.utime(checkpoint, ns=(100, 100)) invalid_surface = copy.deepcopy(surface) - invalid_surface.pop("label") + if malformed == "surface": + invalid_surface.pop("label") + else: + invalid_surface.update(id="latest", label="Latest review", disposition="no_issue_found") + if malformed == "missing-receipt": + invalid_surface["receiptRefs"] = ["artifacts/missing.txt"] + elif malformed == "disposition": + invalid_surface["disposition"] = "unknown" + elif malformed == "receipt-array": + invalid_surface["receiptRefs"] = "invalid" closures = [{"id": "finished", "reason": "Separate review completed."}] if malformed == "closure": closures.append({"id": "review", "reason": "Invalid " + chr(0xD800)}) @@ -177,7 +235,16 @@ def test_malformed_terminal_coverage_preserves_only_earlier_coverage( scan.scan_id, complete=True, findings=[finding] if terminal_findings else [], - surfaces=[invalid_surface] if malformed == "surface" else [], + surfaces=[ + { + "id": "sibling-outcome", + "candidateId": "rejected-sibling", + "label": "Sibling reviewed", + "disposition": "rejected", + "receiptRefs": [], + }, + *([invalid_surface] if malformed not in (None, "closure") else []), + ], closures=closures, ) if selected_head: @@ -194,6 +261,7 @@ def test_malformed_terminal_coverage_preserves_only_earlier_coverage( findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] assert any(row["title"] == finding["title"] for row in findings) == terminal_findings + assert all(row["title"] != rejected["title"] for row in findings) coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) assert any( row.get("workerId") == worker_id and row["label"] == surface["label"] @@ -203,6 +271,16 @@ def test_malformed_terminal_coverage_preserves_only_earlier_coverage( malformed ) assert not any(row["reason"] == "Separate review remains." for row in coverage["deferred"]) + assert ( + next(row for row in coverage["surfaces"] if row["label"] == "Sibling reviewed")[ + "disposition" + ] + == "rejected" + ) + if malformed in {"missing-receipt", "disposition", "receipt-array"}: + latest = next(row for row in coverage["surfaces"] if row["label"] == "Latest review") + assert latest["disposition"] == "needs_follow_up" + assert latest["receiptRefs"] == [] assert {path: path.read_bytes() for path in original} == original diff --git a/plugins/codex-security/tests/test_deep_scan_deadline_recovery.py b/plugins/codex-security/tests/test_deep_scan_deadline_recovery.py index 91a8a7dd14..1bd3946ba9 100644 --- a/plugins/codex-security/tests/test_deep_scan_deadline_recovery.py +++ b/plugins/codex-security/tests/test_deep_scan_deadline_recovery.py @@ -1,6 +1,7 @@ from __future__ import annotations import copy +import hashlib import json import os import sqlite3 @@ -32,6 +33,28 @@ def worker_finding(scan, anchor): return finding +def interrupt_finish_before_commit(workbench_api, database, scan): + child = os.fork() + if child == 0: + connection = sqlite3.connect(database) + connection.row_factory = sqlite3.Row + + class CrashBeforeCommit: + def __getattr__(self, name): + return getattr(connection, name) + + def commit(self): + os._exit(0) + + try: + finish(workbench_api, CrashBeforeCommit(), scan, terminal_reason="capped") + except BaseException: + traceback.print_exc() + os._exit(1) + _, status = os.waitpid(child, 0) + assert os.waitstatus_to_exitcode(status) == 0 + + def test_discarded_worker_finding_leaves_its_review_pending( workbench_api, workbench_db, deadline_scan ): @@ -324,25 +347,7 @@ def test_deadline_recovery_is_idempotent_after_crash_before_commit( ), ) connection.close() - child = os.fork() - if child == 0: - connection = sqlite3.connect(database) - connection.row_factory = sqlite3.Row - - class CrashBeforeCommit: - def __getattr__(self, name): - return getattr(connection, name) - - def commit(self): - os._exit(0) - - try: - finish(workbench_api, CrashBeforeCommit(), scan, terminal_reason="capped") - except BaseException: - traceback.print_exc() - os._exit(1) - _, status = os.waitpid(child, 0) - assert os.waitstatus_to_exitcode(status) == 0 + interrupt_finish_before_commit(workbench_api, database, scan) before = json.loads((scan.scan_dir / "coverage.json").read_text()) findings_before = json.loads((scan.scan_dir / "findings.json").read_text()) assert len([row for row in before["surfaces"] if row.get("workerId") == worker_id]) == len( @@ -371,3 +376,79 @@ def commit(self): finding["provenance"].pop("previousFindings") assert findings_after == findings_before assert after == before + + +@pytest.mark.skipif( + not hasattr(os, "fork"), reason="Crash boundary uses a forked workbench process." +) +@pytest.mark.parametrize("earlier_checkpoint_first", [False, True]) +@pytest.mark.parametrize("colliding_ids", [False, True]) +def test_deadline_rejection_history_is_stable_across_repeated_crashes( + workbench_api, workbench_db, deadline_scan, tmp_path, earlier_checkpoint_first, colliding_ids +): + scan = deadline_scan + worker_id, output = canceled_discovery_output(workbench_db, scan) + finding = worker_finding(scan, "reviewed-candidate") + finding["provenance"]["candidateId"] = "candidate" + # Check both hash filename orders without adding invalid or unrelated checkpoints. + for attempt in range(1000): + finding["title"] = f"Candidate review {attempt}" + earlier = saved_draft(scan.scan_id, findings=[finding]) + outcomes = [ + { + "id": "entry", + "candidateId": "candidate", + "finding": copy.deepcopy(finding), + "label": "Candidate reviewed", + "disposition": "rejected", + "receiptRefs": [], + } + ] + if colliding_ids: + outcomes.append({**copy.deepcopy(outcomes[0]), "label": "Separate observation"}) + terminal = saved_draft(scan.scan_id, complete=True, surfaces=outcomes) + earlier_hash = hashlib.sha256(json.dumps(earlier).encode()).hexdigest() + terminal_hash = hashlib.sha256(json.dumps(terminal).encode()).hexdigest() + if (earlier_hash < terminal_hash) == earlier_checkpoint_first: + break + else: + pytest.fail("Could not select checkpoint filename order") + old_checkpoint = write_checkpoint(output / "checkpoints", earlier) + checkpoint = write_checkpoint(output / "checkpoints", terminal) + assert (old_checkpoint.name < checkpoint.name) == earlier_checkpoint_first + os.utime(old_checkpoint, ns=(100, 100)) + os.utime(checkpoint, ns=(200, 200)) + head = output / "checkpoint-head.json" + head.write_text(json.dumps({"checkpoint": checkpoint.name})) + (output / "result.json").unlink(missing_ok=True) + originals = {path: path.read_bytes() for path in (old_checkpoint, checkpoint, head)} + database = tmp_path / "repeated-crash.db" + with sqlite3.connect(database) as connection: + workbench_db.backup(connection) + + publications = [] + for _ in range(3): + interrupt_finish_before_commit(workbench_api, database, scan) + publications.append(json.loads((scan.scan_dir / "coverage.json").read_text())) + with sqlite3.connect(database) as connection: + assert ( + connection.execute( + "SELECT status FROM deep_scan_runs WHERE scan_id = ?", (scan.scan_id,) + ).fetchone()[0] + == "running" + ) + with sqlite3.connect(database) as connection: + connection.row_factory = sqlite3.Row + finish(workbench_api, connection, scan, terminal_reason="capped") + complete(workbench_api, connection, scan) + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert all(publication == coverage for publication in publications) + surfaces = [row for row in coverage["surfaces"] if row.get("workerId") == worker_id] + assert {row["label"] for row in surfaces} == {row["label"] for row in outcomes} + assert len(surfaces) == len({row["id"] for row in surfaces}) == len(outcomes) + assert all(len(row["previousFindings"]) == 1 for row in surfaces) + assert all(row["previousFindings"][0]["title"] == finding["title"] for row in surfaces) + assert all(row["disposition"] == "rejected" for row in surfaces) + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert len(findings) == 1 and findings[0]["title"] == scan.findings[0]["title"] + assert {path: path.read_bytes() for path in originals} == originals From 48ba18626cdf5a2018f3e902fdb9afd3f5a9c131 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Sat, 17 Oct 2026 21:25:08 +0000 Subject: [PATCH 06/11] fix: preserve deadline checkpoint contracts and source links --- .../scripts/finalize_scan_contract.py | 2 + .../scripts/workbench_saved_results.py | 197 ++++++-- .../test_deep_scan_deadline_contracts.py | 272 +++++++++++ .../test_deep_scan_deadline_surface_links.py | 428 ++++++++++++++++++ 4 files changed, 867 insertions(+), 32 deletions(-) create mode 100644 plugins/codex-security/tests/test_deep_scan_deadline_surface_links.py diff --git a/plugins/codex-security/scripts/finalize_scan_contract.py b/plugins/codex-security/scripts/finalize_scan_contract.py index 104cf49e2d..71e98e9453 100644 --- a/plugins/codex-security/scripts/finalize_scan_contract.py +++ b/plugins/codex-security/scripts/finalize_scan_contract.py @@ -1312,6 +1312,8 @@ def _recover_unsealed_coverage( if field == "deferred": _validate_deferred_paths(item, context) _validate_schema_node(item, item_schema, context) + if field == "resolvedDeferred": + _validate_resolved_deferred({"resolvedDeferred": [item]}) _require_safe_json_value(item, context) except ContractError as exc: warnings.append(f"Skipped malformed {label} {index + 1}: {exc}.") diff --git a/plugins/codex-security/scripts/workbench_saved_results.py b/plugins/codex-security/scripts/workbench_saved_results.py index e133bf1543..69ed6ca0ad 100644 --- a/plugins/codex-security/scripts/workbench_saved_results.py +++ b/plugins/codex-security/scripts/workbench_saved_results.py @@ -35,6 +35,7 @@ _require_portable_relative_path, _require_safe_json_value, _validate_completion_binding, + _validate_current_findings, _validate_resolved_deferred, _validate_schema_node, _write_prepared_scan_finalization, @@ -736,6 +737,24 @@ def _deferred_candidate_id( return identity +def _inherit_coverage_sources( + row: Any, + originals: list[Any], + sources: dict[int, tuple[dict[str, Any], list[str]]], +) -> None: + if not isinstance(row, dict): + return + for original in originals: + saved = sources.get(id(original)) + if saved is None: + continue + # Retain the object as well as its ID while copies are reconciled. + paths = sources.setdefault(id(row), (row, []))[1] + for relative in saved[1]: + if relative not in paths: + paths.append(relative) + + def _generic_surface_updates( sources: list[tuple[str, dict[str, Any], str | None]], source_order: dict[str, tuple[int, int]], @@ -747,6 +766,7 @@ def _generic_surface_updates( deferred_rows: dict[str, list[Any]], ambiguous_deferred: set[tuple[str | None, str]], canonical_rows: set[int], + coverage_sources: dict[int, tuple[dict[str, Any], list[str]]], ) -> tuple[set[int], list[dict[str, Any]]]: if not closed_deferred and not reopened_generic: return set(), [] @@ -849,16 +869,21 @@ def linked(row: dict[str, Any], identity: str | None) -> bool: _validate_schema_node(update, surface_schema, "coverage.surfaces") except ContractError: continue - replaced.update( - id(row) + replaced_sources = [ + row for _, row in matches if reopening or row.get("disposition") in {"needs_follow_up", surface.get("disposition")} - ) + ] + replaced.update(id(row) for row in replaced_sources) if update not in updates: updates.append(update) - if id(latest_surface) in canonical_rows: - canonical_rows.add(id(update)) + retained = updates[updates.index(update)] + _inherit_coverage_sources( + retained, [latest_surface, *replaced_sources], coverage_sources + ) + if id(latest_surface) in canonical_rows: + canonical_rows.add(id(retained)) return replaced, updates @@ -1106,16 +1131,18 @@ def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None continue if preserve_canonical_parent and worker_id is not None: # Recover imported records without relaxing canonical parent validation. + if "complete" in draft and not isinstance(draft["complete"], bool): + draft["complete"] = False + warnings.append( + f"{relative}: Invalid complete marker; retained as a progress checkpoint." + ) + schema_dir = Path(__file__).resolve().parent.parent / "schemas" if "threatModel" in draft: try: model = draft["threatModel"] if isinstance(model, dict): model = {**model, "origin": "recovered"} - schema = _read_json( - Path(__file__).resolve().parent.parent - / "schemas" - / "scan-manifest.schema.json" - ) + schema = _read_json(schema_dir / "scan-manifest.schema.json") _validate_schema_node( model, schema["properties"]["scan"]["properties"]["threatModel"], @@ -1137,10 +1164,18 @@ def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None discarded = _recover_unsealed_findings( {"scan": {"id": scan_id, "target": binding["target"]}}, recovered, - Path(__file__).resolve().parent.parent / "schemas", + schema_dir, scan_dir / directory, source_warnings, ) + if recovered["findings"]: + try: + _validate_current_findings(recovered, schema_dir) + except ContractError as exc: + warning = f"Skipped malformed finding: {exc}." + source_warnings.append(warning) + discarded.append(warning) + recovered["findings"] = [] if discarded: discarded_by_source.setdefault(relative, []).extend(discarded) for recovered_finding in recovered["findings"]: @@ -1302,6 +1337,17 @@ def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None row["id"] = identity reserved.add(identity) + # A discarded local surface must not fall back to another source's reused ID. + surface_ids_by_source: dict[tuple[str, str], list[str]] = { + (relative, row["id"]): [] + for relative, draft, _ in all_sources + for row in ( + draft["coverage"].get("surfaces", []) + if isinstance(draft["coverage"].get("surfaces"), list) + else [] + ) + if isinstance(row, dict) and isinstance(row.get("id"), str) + } discarded_coverage_sources: set[str] = set() if preserve_canonical_parent: for relative, draft, worker_id in sources: @@ -1431,6 +1477,35 @@ def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None if parent_is_canonical else set() ) + # Links use source-local IDs until all retained surfaces receive their final IDs. + coverage_sources: dict[int, tuple[dict[str, Any], list[str]]] = {} + for relative, draft, _ in all_sources: + for field in ("surfaces", "deferred"): + rows = draft["coverage"].get(field) + for row in rows if isinstance(rows, list) else []: + if isinstance(row, dict): + paths_for_row = coverage_sources.setdefault(id(row), (row, []))[1] + if relative not in paths_for_row: + paths_for_row.append(relative) + if parent: + for field in ("surfaces", "deferred"): + rows, originals = coverage.get(field), parent["coverage"].get(field) + if isinstance(rows, list) and isinstance(originals, list): + for row, original in zip(rows, originals): + _inherit_coverage_sources(row, [original], coverage_sources) + + def retain_coverage_row(output: list[Any], item: Any, *originals: Any) -> Any: + for retained in output: + if retained == item: + break + else: + retained = copy.deepcopy(item) + output.append(retained) + _inherit_coverage_sources(retained, [item, *originals], coverage_sources) + if any(id(original) in canonical_rows for original in (item, *originals)): + canonical_rows.add(id(retained)) + return retained + findings: list[dict[str, Any]] = [] finding_positions: dict[str, int] = {} canonical_finding_keys: set[str] = set() @@ -1462,6 +1537,20 @@ def valid_finding(value: Any) -> bool: deferred_rows = { relative: _deferred_rows(draft["coverage"]) for relative, draft, _ in all_sources } + # Accepted heads can replace equal tasks before they reach output coalescing. + equivalent_tasks: dict[str | None, list[dict[str, Any]]] = {} + for relative, _, owner in all_sources: + known = equivalent_tasks.setdefault(owner, []) + for row in deferred_rows[relative]: + if not isinstance(row, dict): + continue + for previous in known: + if previous == row: + _inherit_coverage_sources(previous, [row], coverage_sources) + coverage_sources[id(row)] = (row, coverage_sources[id(previous)][1]) + break + else: + known.append(row) # A legacy source can contain independent tasks with the same explicit ID. # Later rewrites cannot make an ID-only closure identify one of those tasks. ambiguous_deferred: set[tuple[str | None, str]] = set() @@ -1679,6 +1768,7 @@ def valid_finding(value: Any) -> bool: deferred_rows, ambiguous_deferred, canonical_rows, + coverage_sources, ) replaced_rows = { "surfaces": replaced_surfaces, @@ -1702,8 +1792,7 @@ def valid_finding(value: Any) -> bool: coverage[field] = [row for row in coverage[field] if row not in removed_parent] if isinstance(coverage.get("surfaces"), list): for surface in surface_updates: - if surface not in coverage["surfaces"]: - coverage["surfaces"].append(surface) + retain_coverage_row(coverage["surfaces"], surface) for surface in coverage["surfaces"]: if isinstance(surface, dict): surface.setdefault("receiptRefs", []) @@ -1717,8 +1806,8 @@ def valid_finding(value: Any) -> bool: ) in resolved: continue pending = coverage.setdefault("deferred", []) - if isinstance(pending, list) and item not in pending: - pending.append(copy.deepcopy(item)) + if isinstance(pending, list): + retain_coverage_row(pending, item) ambiguous_surface_ids = { (owner, identity) for owner, row in reopened_rows @@ -1752,8 +1841,7 @@ def valid_finding(value: Any) -> bool: and isinstance(coverage.get("surfaces"), list) ): retained_surface = {**surface, "receiptRefs": surface.get("receiptRefs", [])} - if retained_surface not in coverage["surfaces"]: - coverage["surfaces"].append(copy.deepcopy(retained_surface)) + retain_coverage_row(coverage["surfaces"], retained_surface, surface) selected_terminal_orders: dict[str, tuple[int, int]] = {} for relative, draft, worker_id in sources: if ( @@ -2048,6 +2136,7 @@ def valid_finding(value: Any) -> bool: # recovery and warnings rather than silently changing its contract. continue for item in items: + original_item = item # A selected terminal checkpoint replaces ordinary coverage, while # candidate evidence and pending tasks retain their own reconciliation. if ( @@ -2119,15 +2208,20 @@ def valid_finding(value: Any) -> bool: item = {**item, "receiptRefs": item.get("receiptRefs", [])} if isinstance(item, dict) and "id" not in item: semantic_item = dict(item) - if any( - isinstance(existing, dict) - and {key: value for key, value in existing.items() if key != "id"} - == semantic_item - for existing in output - ): + existing = next( + ( + existing + for existing in output + if isinstance(existing, dict) + and {key: value for key, value in existing.items() if key != "id"} + == semantic_item + ), + None, + ) + if existing is not None: + _inherit_coverage_sources(existing, [original_item], coverage_sources) continue - if item not in output: - output.append(copy.deepcopy(item)) + retain_coverage_row(output, item, original_item) identities: dict[str, str] = {} for finding in findings: @@ -2143,6 +2237,16 @@ def valid_finding(value: Any) -> bool: identity["instance"] = f"{identity.get('instance', 'saved')}-{variant[:16]}" identities[key] = variant recovered_worker_ids = {worker_id for _, _, worker_id in sources if worker_id is not None} + surface_positions: dict[str, int] = {} + + def remember_surface_id(item: dict[str, Any], local_id: str) -> None: + final_id = item["id"] + surface_positions.setdefault(final_id, len(surface_positions)) + for relative in coverage_sources.get(id(item), (item, []))[1]: + targets = surface_ids_by_source.setdefault((relative, local_id), []) + if final_id not in targets: + targets.append(final_id) + for field in ("surfaces", "explicitExclusions", "deferred"): used: set[str] = set() items = coverage.setdefault(field, []) @@ -2154,35 +2258,64 @@ def valid_finding(value: Any) -> bool: if id(item) in canonical_rows: if isinstance(item.get("id"), str): used.add(item["id"]) + if field == "surfaces": + remember_surface_id(item, item["id"]) continue item.setdefault("id", _saved_coverage_id(item)) if not isinstance(item["id"], str): # Preserve malformed rows for per-record recovery, including frozen replay. continue - if ( + local_id = item["id"] + scoped = ( preserve_canonical_parent and isinstance(worker_id := item.get("workerId"), str) and worker_id in recovered_worker_ids - ): + ) + if scoped: # Resolve worker-local candidate/deferred history before scoping IDs. item["id"] = f"{worker_id}/{item['id']}" - if isinstance(item.get("surfaceIds"), list): - item["surfaceIds"] = [ - f"{worker_id}/{surface_id}" if isinstance(surface_id, str) else surface_id - for surface_id in item["surfaceIds"] - ] + if isinstance(item.get("surfaceIds"), list): + links: list[Any] = [] + for surface_id in item["surfaceIds"]: + targets: list[str] = [] + declared = False + if field == "deferred" and isinstance(surface_id, str): + for relative in coverage_sources.get(id(item), (item, []))[1]: + source_key = (relative, surface_id) + declared |= source_key in surface_ids_by_source + for target in surface_ids_by_source.get(source_key, []): + if target not in targets: + targets.append(target) + if targets: + targets.sort(key=surface_positions.__getitem__) + for target in targets: + if target not in links: + links.append(target) + elif not declared: + links.append( + f"{worker_id}/{surface_id}" + if scoped and isinstance(surface_id, str) + else surface_id + ) + item["surfaceIds"] = links + if scoped: if item in normalized_items[:-1]: # A previous finish may have published files before its DB commit. + if field == "surfaces": + remember_surface_id(item, local_id) normalized_items.pop() continue if item["id"] in used: item["id"] = f"{item['id']}-{_digest(item)[:16]}" if preserve_canonical_parent and item in normalized_items[:-1]: + if field == "surfaces": + remember_surface_id(item, local_id) normalized_items.pop() continue used.add(item["id"]) if field == "surfaces": item.setdefault("receiptRefs", []) + remember_surface_id(item, local_id) if preserve_canonical_parent and isinstance(items, list): coverage[field] = normalized_items coverage.pop("completeness", None) diff --git a/plugins/codex-security/tests/test_deep_scan_deadline_contracts.py b/plugins/codex-security/tests/test_deep_scan_deadline_contracts.py index a208beb93a..789fd62a5e 100644 --- a/plugins/codex-security/tests/test_deep_scan_deadline_contracts.py +++ b/plugins/codex-security/tests/test_deep_scan_deadline_contracts.py @@ -13,6 +13,194 @@ from workbench_test_support import saved_draft, write_checkpoint +@pytest.mark.parametrize("source", ["result", "checkpoint", "archived-checkpoint"]) +@pytest.mark.parametrize( + "marker", + [ + pytest.param("false", id="string-false"), + pytest.param(None, id="null"), + pytest.param(0, id="zero"), + pytest.param(1, id="one"), + pytest.param(False, id="incomplete"), + pytest.param(True, id="complete"), + pytest.param("omitted", id="omitted"), + ], +) +def test_deadline_requires_boolean_completion_before_superseding_worker_history( + workbench_api, workbench_db, deadline_scan, source, marker +): + scan = deadline_scan + _, output = canceled_discovery_output(workbench_db, scan, attempt=2) + if source == "archived-checkpoint": + output = output.parent / "attempts" / "attempt-1" + earlier = worker_finding(scan, "earlier-worker-finding") + rejected = worker_finding(scan, "rejected-worker-finding") + rejected["provenance"]["candidateId"] = "rejected-worker-finding" + checkpoint = write_checkpoint( + output / "checkpoints", + saved_draft( + scan.scan_id, + findings=[earlier, rejected], + deferred=[{"id": "review", "reason": "Review remains."}], + ), + ) + os.utime(checkpoint, ns=(100, 100)) + sibling = worker_finding(scan, "latest-valid-sibling") + latest = saved_draft( + scan.scan_id, + complete=marker, + findings=[sibling], + surfaces=[ + { + "id": "rejected-outcome", + "candidateId": "rejected-worker-finding", + "label": "Rejected after review", + "disposition": "rejected", + "receiptRefs": [], + } + ], + closures=[{"id": "review", "reason": "Review completed."}], + ) + if marker == "omitted": + latest.pop("complete") + if source == "result": + result = output / "result.json" + result.write_text(json.dumps(latest)) + else: + result = write_checkpoint(output / "checkpoints", latest) + head = output / "checkpoint-head.json" + head.write_text(json.dumps({"checkpoint": result.name})) + os.utime(head, ns=(300, 300)) + os.utime(result, ns=(200, 200)) + original = {path: path.read_bytes() for path in (checkpoint, result)} + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + completed = complete(workbench_api, workbench_db, scan) + + terminal = marker is True or marker == "omitted" + invalid = not isinstance(marker, bool) and marker != "omitted" + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + titles = {finding["title"] for finding in findings} + assert sibling["title"] in titles + assert rejected["title"] not in titles + if not terminal or source == "result": + assert (earlier["title"] in titles) == (not terminal) + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert any(row["reason"] == "Review remains." for row in coverage["deferred"]) == (not terminal) + assert ( + any( + "Invalid complete marker" in warning and result.name in warning + for warning in completed["warnings"] + ) + == invalid + ) + manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"] + for path, contents in original.items(): + assert path.read_bytes() == contents + assert manifest["preservedSources"][path.relative_to(scan.scan_dir).as_posix()] == ( + workbench_api["saved_results"]._digest(json.loads(contents)) + ) + + +@pytest.mark.parametrize("source", ["result", "checkpoint", "archived-result"]) +@pytest.mark.parametrize("invalid", ["rationale", "code_evidence", "sourceFindingIds"]) +def test_deadline_validates_each_recovered_finding_against_the_completed_contract( + workbench_api, workbench_db, deadline_scan, source, invalid +): + scan = deadline_scan + worker_id, output = canceled_discovery_output(workbench_db, scan, attempt=2) + if source == "archived-result": + output = output.parent / "attempts" / "attempt-1" + earlier = worker_finding(scan, "earlier-worker-finding") + rejected = worker_finding(scan, "rejected-worker-finding") + rejected["provenance"]["candidateId"] = "rejected-worker-finding" + checkpoint = write_checkpoint( + output / "checkpoints", + saved_draft( + scan.scan_id, + findings=[earlier, rejected], + deferred=[ + {"id": "review", "reason": "Review remains."}, + {"id": "finished", "reason": "Separate review remains."}, + ], + ), + ) + os.utime(checkpoint, ns=(100, 100)) + sibling = worker_finding(scan, "valid-worker-sibling") + malformed = worker_finding(scan, "malformed-worker-finding") + malformed["provenance"]["candidateId"] = "malformed-worker-finding" + if invalid == "rationale": + malformed["severity"]["rationale"] = " \t " + elif invalid == "code_evidence": + malformed["code_evidence"] = [] + else: + malformed["provenance"]["sourceFindingIds"] = ["source-finding", "source-finding"] + latest = saved_draft( + scan.scan_id, + complete=True, + findings=[malformed, sibling], + surfaces=[ + { + "id": "reported-outcome", + "candidateId": "malformed-worker-finding", + "label": "Unverified finding", + "disposition": "reported", + "receiptRefs": [], + }, + { + "id": "rejected-outcome", + "candidateId": "rejected-worker-finding", + "label": "Rejected after review", + "disposition": "rejected", + "receiptRefs": [], + }, + ], + closures=[{"id": "finished", "reason": "Separate review completed."}], + ) + if source == "checkpoint": + result = write_checkpoint(output / "checkpoints", latest) + head = output / "checkpoint-head.json" + head.write_text(json.dumps({"checkpoint": result.name})) + os.utime(head, ns=(300, 300)) + else: + result = output / "result.json" + result.write_text(json.dumps(latest)) + os.utime(result, ns=(200, 200)) + original = {path: path.read_bytes() for path in (checkpoint, result)} + if source == "archived-result": + selected = write_checkpoint(output / "checkpoints", latest) + os.utime(selected, ns=(200, 200)) + head = output / "checkpoint-head.json" + head.write_text(json.dumps({"checkpoint": selected.name})) + os.utime(head, ns=(300, 300)) + original[selected] = selected.read_bytes() + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + completed = complete(workbench_api, workbench_db, scan) + + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert {finding["title"] for finding in findings} == { + scan.findings[0]["title"], + earlier["title"], + sibling["title"], + } + retained = next(finding for finding in findings if finding["title"] == sibling["title"]) + assert retained["provenance"]["workerId"] == worker_id + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert any(row["reason"] == "Review remains." for row in coverage["deferred"]) + assert not any(row["reason"] == "Separate review remains." for row in coverage["deferred"]) + surfaces = {row["label"]: row for row in coverage["surfaces"]} + assert surfaces["Unverified finding"]["disposition"] == "needs_follow_up" + assert surfaces["Rejected after review"]["disposition"] == "rejected" + assert any(invalid in warning and result.name in warning for warning in completed["warnings"]) + manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"] + for path, contents in original.items(): + assert path.read_bytes() == contents + assert manifest["preservedSources"][path.relative_to(scan.scan_dir).as_posix()] == ( + workbench_api["saved_results"]._digest(json.loads(contents)) + ) + + @pytest.mark.parametrize( "model", [ @@ -65,6 +253,29 @@ def test_deadline_still_rejects_malformed_canonical_model( finish(workbench_api, workbench_db, scan, terminal_reason="capped") +def test_deadline_still_rejects_canonical_findings_outside_the_completed_contract( + workbench_api, workbench_db, deadline_scan +): + from finalize_scan_contract import ContractError + + scan = deadline_scan + finding = worker_finding(scan, "valid-worker-finding") + scan.findings[0]["severity"]["rationale"] = " \t " + findings_path = scan.scan_dir / "findings.json" + findings_path.write_text(json.dumps({"findings": scan.findings})) + original = findings_path.read_bytes() + _, output = canceled_discovery_output(workbench_db, scan) + write_checkpoint( + output / "checkpoints", + saved_draft(scan.scan_id, findings=[finding]), + ) + + with pytest.raises(ContractError, match="rationale"): + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + + assert findings_path.read_bytes() == original + + @pytest.mark.parametrize("label", [" \t ", " Reviewed Résumé 😀 "]) def test_deadline_applies_existing_nonblank_surface_label_contract( workbench_api, workbench_db, deadline_scan, label @@ -157,6 +368,67 @@ def test_deadline_applies_existing_unicode_contract_per_worker_record( assert checkpoint.read_bytes() == original +@pytest.mark.parametrize( + "reason", + [pytest.param("\u0085", id="historical-nel"), pytest.param("Review completed.", id="valid")], +) +def test_deadline_recovers_historical_closure_text_without_losing_a_valid_sibling( + workbench_api, workbench_db, deadline_scan, reason +): + scan = deadline_scan + _, output = canceled_discovery_output(workbench_db, scan) + previous = write_checkpoint( + output / "checkpoints", + saved_draft( + scan.scan_id, + deferred=[ + {"id": "review", "reason": "First review pending."}, + {"id": "sibling", "reason": "Valid sibling review pending."}, + ], + ), + ) + os.utime(previous, ns=(100, 100)) + draft = saved_draft( + scan.scan_id, + complete=True, + closures=[ + {"id": "review", "reason": reason}, + {"id": "sibling", "reason": "Valid sibling review completed."}, + ], + ) + checkpoint = write_checkpoint(output / "checkpoints", draft) + result = output / "result.json" + result.write_text(json.dumps(draft)) + head = output / "checkpoint-head.json" + head.write_text(json.dumps({"checkpoint": checkpoint.name})) + for path in (checkpoint, result, head): + os.utime(path, ns=(200, 200)) + original = {path: path.read_bytes() for path in (previous, checkpoint, result)} + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + completed = complete(workbench_api, workbench_db, scan) + + invalid = not reason.strip() + assert completed["progress"]["status"] == "complete" + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + pending = {row["reason"] for row in coverage["deferred"]} + assert ("First review pending." in pending) == invalid + assert "Valid sibling review pending." not in pending + report = (scan.scan_dir / "report.md").read_text() + assert ("First review pending." in report) == invalid + assert "Valid sibling review pending." not in report + assert any("expected a non-empty string" in warning for warning in completed["warnings"]) == ( + invalid + ) + manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"] + assert manifest["sealedAt"] + for path, contents in original.items(): + assert path.read_bytes() == contents + assert manifest["preservedSources"][path.relative_to(scan.scan_dir).as_posix()] == ( + workbench_api["saved_results"]._digest(json.loads(contents)) + ) + + @pytest.mark.parametrize( ("terminal_findings", "malformed", "selected_head"), [ diff --git a/plugins/codex-security/tests/test_deep_scan_deadline_surface_links.py b/plugins/codex-security/tests/test_deep_scan_deadline_surface_links.py new file mode 100644 index 0000000000..37d7725299 --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_deadline_surface_links.py @@ -0,0 +1,428 @@ +from __future__ import annotations + +import json +import os +import sqlite3 +import subprocess +import sys +from pathlib import Path + +import pytest +from test_deep_scan_stop_conditions import canceled_discovery_output, complete, finish +from test_deep_scan_stop_conditions import deadline_scan as deadline_scan +from test_deep_scan_stop_conditions import publication_scan as publication_scan +from test_deep_scan_stop_conditions import saturated_scan as saturated_scan +from workbench_test_support import ( + replay_saved_results, + saved_binding, + saved_discovery_worker, + saved_draft, + write_checkpoint, +) + + +def surface(label, identity="entry", disposition="needs_follow_up"): + return {"id": identity, "label": label, "disposition": disposition, "receiptRefs": []} + + +def pending(identity, reason, surface_id="entry"): + return {"id": identity, "reason": reason, "surfaceIds": [surface_id]} + + +def interrupt_in_fresh_python(database, scan, hash_seed): + script = Path(__file__).resolve().parents[1] / "scripts" / "workbench_db.py" + result = subprocess.run( + [ + sys.executable, + "-B", + "-c", + """ +import runpy +import sys +from pathlib import Path +from types import SimpleNamespace +sys.path.insert(0, str(Path(sys.argv[1]).parents[1] / "tests")) +from test_deep_scan_deadline_recovery import interrupt_finish_before_commit + +api = runpy.run_path(sys.argv[1], run_name="surface_link_recovery_test") +api["deep_scan"]._dependencies = SimpleNamespace( + **api, preserve_stopped_results=api["preserve_stopped_results_after_transition"] +) +scan = SimpleNamespace(scan_id=sys.argv[3], scan_dir=Path(sys.argv[4])) +interrupt_finish_before_commit(api, Path(sys.argv[2]), scan) +""", + str(script), + str(database), + scan.scan_id, + str(scan.scan_dir), + ], + cwd=Path(__file__).parent, + env={**os.environ, "PYTHONHASHSEED": str(hash_seed)}, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + + +@pytest.mark.skipif( + not hasattr(os, "fork"), reason="Crash boundary uses a forked workbench process." +) +@pytest.mark.parametrize("parent_collision", [False, True]) +def test_deadline_surface_links_follow_sources_across_repeated_crashes( + workbench_api, workbench_db, deadline_scan, tmp_path, parent_collision +): + scan = deadline_scan + worker_id, output = canceled_discovery_output(workbench_db, scan, attempt=2) + archived = output.parent / "attempts" / "attempt-01" + shared = pending("shared-review", "Review all retained observations.") + checkpoints = [] + for index, (source, label) in enumerate( + [(archived, "download"), (output, "upload"), (output, "export")], start=1 + ): + # Each is a valid raw input checkpoint saved before the writer merges history. + checkpoint = write_checkpoint( + source / "checkpoints", + saved_draft( + scan.scan_id, + surfaces=[surface(label)], + deferred=[pending(f"{label}-review", f"Review {label}."), shared], + ), + ) + os.utime(checkpoint, ns=(index * 100, index * 100)) + checkpoints.append(checkpoint) + canonical_id = f"{worker_id}/entry" if parent_collision else "parent-entry" + canonical_surface = surface("Canonical parent", canonical_id) + canonical_pending = pending("parent-review", "Parent review remains.", canonical_id) + scan.coverage.update(surfaces=[canonical_surface], deferred=[canonical_pending]) + (scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage)) + originals = {path: path.read_bytes() for path in checkpoints} + database = tmp_path / "surface-links.db" + with sqlite3.connect(database) as connection: + workbench_db.backup(connection) + + publications = [] + for hash_seed in (1, 17, 83): + interrupt_in_fresh_python(database, scan, hash_seed) + publications.append(json.loads((scan.scan_dir / "coverage.json").read_text())) + with sqlite3.connect(database) as connection: + assert ( + connection.execute( + "SELECT status FROM deep_scan_runs WHERE scan_id = ?", (scan.scan_id,) + ).fetchone()[0] + == "running" + ) + with sqlite3.connect(database) as connection: + connection.row_factory = sqlite3.Row + finish(workbench_api, connection, scan, terminal_reason="capped") + completed = complete(workbench_api, connection, scan) + + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert completed["progress"]["status"] == "complete" + assert all(publication == coverage for publication in publications) + assert canonical_surface in coverage["surfaces"] + assert canonical_pending in coverage["deferred"] + surfaces = [row for row in coverage["surfaces"] if row.get("workerId") == worker_id] + assert {row["label"] for row in surfaces} == {"download", "upload", "export"} + assert len(surfaces) == len({row["id"] for row in surfaces}) == 3 + tasks = [row for row in coverage["deferred"] if row.get("workerId") == worker_id] + for row in surfaces: + task = next(item for item in tasks if item["reason"] == f"Review {row['label']}.") + assert task["surfaceIds"] == [row["id"]] + shared_tasks = [row for row in tasks if row["reason"] == shared["reason"]] + assert len(shared_tasks) == 1 + assert shared_tasks[0]["surfaceIds"] == [row["id"] for row in surfaces] + assert {path: path.read_bytes() for path in originals} == originals + + +@pytest.mark.parametrize("with_reduction", [False, True], ids=["zero-accepted", "reduced"]) +def test_deadline_retains_all_targets_of_one_legacy_surface_id( + workbench_api, workbench_db, publication_scan, request, with_reduction +): + scan = request.getfixturevalue("deadline_scan") if with_reduction else publication_scan() + if not with_reduction: + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', phase = 'discovery', " + "created_at = '2000-01-01T00:00:00+00:00', manifest_path = NULL, " + "terminal_reason = NULL, completed_at = NULL WHERE scan_id = ?", + (scan.scan_id,), + ) + _, output = canceled_discovery_output(workbench_db, scan) + checkpoint = write_checkpoint( + output / "checkpoints", + saved_draft( + scan.scan_id, + surfaces=[surface("download"), surface("upload")], + deferred=[pending("review", "Review both legacy observations.")], + ), + ) + original = checkpoint.read_bytes() + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + complete(workbench_api, workbench_db, scan) + + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + surfaces = [row for row in coverage["surfaces"] if row["label"] in {"download", "upload"}] + assert len(surfaces) == len({row["id"] for row in surfaces}) == 2 + task = next( + row for row in coverage["deferred"] if row["reason"] == "Review both legacy observations." + ) + assert task["surfaceIds"] == [row["id"] for row in surfaces] + assert checkpoint.read_bytes() == original + + +def test_deadline_deferred_only_checkpoint_keeps_prior_surface_link( + workbench_api, workbench_db, deadline_scan +): + scan = deadline_scan + worker_id, output = canceled_discovery_output(workbench_db, scan) + earlier = write_checkpoint( + output / "checkpoints", + saved_draft(scan.scan_id, surfaces=[surface("Earlier review", "prior")]), + ) + current = write_checkpoint( + output / "checkpoints", + saved_draft(scan.scan_id, deferred=[pending("review", "Review prior surface.", "prior")]), + ) + os.utime(earlier, ns=(100, 100)) + os.utime(current, ns=(200, 200)) + originals = {path: path.read_bytes() for path in (earlier, current)} + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + complete(workbench_api, workbench_db, scan) + + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + retained = next(row for row in coverage["surfaces"] if row["label"] == "Earlier review") + task = next(row for row in coverage["deferred"] if row["reason"] == "Review prior surface.") + assert task["surfaceIds"] == [retained["id"]] == [f"{worker_id}/prior"] + assert {path: path.read_bytes() for path in originals} == originals + + +def test_deadline_reopened_generic_surface_copy_retains_its_link( + workbench_api, workbench_db, deadline_scan +): + scan = deadline_scan + worker_id, output = canceled_discovery_output(workbench_db, scan) + canonical = surface("Canonical parent", f"{worker_id}/entry") + scan.coverage["surfaces"] = [canonical] + (scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage)) + drafts = [ + saved_draft( + scan.scan_id, + surfaces=[surface("Worker review")], + deferred=[pending("review", "Initial review pending.")], + ), + saved_draft( + scan.scan_id, + complete=True, + surfaces=[surface("Worker review", disposition="no_issue_found")], + closures=[{"id": "review", "reason": "Initial review completed."}], + ), + saved_draft( + scan.scan_id, + surfaces=[surface("Worker review")], + deferred=[pending("review", "Review reopened.")], + ), + ] + originals = {} + for index, draft in enumerate(drafts, start=1): + checkpoint = write_checkpoint(output / "checkpoints", draft) + os.utime(checkpoint, ns=(index * 100, index * 100)) + originals[checkpoint] = checkpoint.read_bytes() + head = output / "checkpoint-head.json" + head.write_text(json.dumps({"checkpoint": checkpoint.name})) + originals[head] = head.read_bytes() + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + complete(workbench_api, workbench_db, scan) + + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + retained = [row for row in coverage["surfaces"] if row.get("workerId") == worker_id] + assert len(retained) == 1 + assert retained[0]["disposition"] == "needs_follow_up" + task = next(row for row in coverage["deferred"] if row["reason"] == "Review reopened.") + assert task["surfaceIds"] == [retained[0]["id"]] + assert canonical in coverage["surfaces"] + assert {path: path.read_bytes() for path in originals} == originals + + +def test_stopped_recovery_preserves_source_link_union(workbench_api, tmp_path): + saved = workbench_api["saved_results"] + scan_id = "surface-links" + binding = saved_binding("deep_repository") + output = tmp_path / "worker" / "output" + archived = output.parent / "attempts" / "attempt-01" + shared = pending("review", "Review retained observations.") + originals = {} + for source, label in ((archived, "download"), (output, "upload")): + checkpoint = write_checkpoint( + source / "checkpoints", + saved_draft(scan_id, surfaces=[surface(label)], deferred=[shared]), + ) + originals[checkpoint] = checkpoint.read_bytes() + workers = [saved_discovery_worker(output, "worker", 2)] + + documents = saved.merge_saved_results( + tmp_path, scan_id, binding, workers, [], stopped=True, reason="interrupted" + ) + replay = replay_saved_results(saved, documents, tmp_path, scan_id, binding, workers) + + assert replay[2] == documents[2] + coverage = documents[2] + assert {row["label"] for row in coverage["surfaces"]} == {"download", "upload"} + tasks = [row for row in coverage["deferred"] if row["reason"] == shared["reason"]] + assert len(tasks) == 1 + assert tasks[0]["surfaceIds"] == [row["id"] for row in coverage["surfaces"]] + assert {path: path.read_bytes() for path in originals} == originals + + +@pytest.mark.skipif( + not hasattr(os, "fork"), reason="Crash boundary uses a forked workbench process." +) +@pytest.mark.parametrize("changed_task", [False, True], ids=["copied-task", "updated-task"]) +def test_deadline_retains_equal_task_origins_from_selected_worker_result( + workbench_api, workbench_db, deadline_scan, tmp_path, changed_task +): + scan = deadline_scan + worker_id, output = canceled_discovery_output(workbench_db, scan, attempt=2) + archived = output.parent / "attempts" / "attempt-01" + previous_task = pending("download-review", "Download review pending.") + archived_draft = saved_draft( + scan.scan_id, surfaces=[surface("download")], deferred=[previous_task] + ) + checkpoint = write_checkpoint(archived / "checkpoints", archived_draft) + os.utime(checkpoint, ns=(100, 100)) + archived_head = archived / "checkpoint-head.json" + archived_head.write_text(json.dumps({"checkpoint": checkpoint.name})) + os.utime(archived_head, ns=(110, 110)) + archived_result = archived / "result.json" + archived_result.write_bytes(checkpoint.read_bytes()) + os.utime(archived_result, ns=(120, 120)) + current_task = pending("upload-review", "Upload review pending.") + retained_task = ( + pending("download-review", "Download review updated.") if changed_task else previous_task + ) + incoming = saved_draft( + scan.scan_id, + surfaces=[surface("upload")], + deferred=[current_task, retained_task] if changed_task else [current_task], + ) + raw_checkpoint = write_checkpoint(output / "checkpoints", incoming) + os.utime(raw_checkpoint, ns=(200, 200)) + # The writer preserves the old pending task while publishing its latest surface. + merged = saved_draft( + scan.scan_id, surfaces=[surface("upload")], deferred=[current_task, retained_task] + ) + selected = write_checkpoint(output / "checkpoints", merged) + os.utime(selected, ns=(210, 210)) + head = output / "checkpoint-head.json" + head.write_text(json.dumps({"checkpoint": selected.name})) + os.utime(head, ns=(220, 220)) + result = output / "result.json" + result.write_bytes(selected.read_bytes()) + os.utime(result, ns=(230, 230)) + originals = {path: path.read_bytes() for path in output.parent.rglob("*") if path.is_file()} + database = tmp_path / "selected-result.db" + with sqlite3.connect(database) as connection: + workbench_db.backup(connection) + + publications = [] + for hash_seed in (1, 17, 83): + interrupt_in_fresh_python(database, scan, hash_seed) + publications.append(json.loads((scan.scan_dir / "coverage.json").read_text())) + with sqlite3.connect(database) as connection: + connection.row_factory = sqlite3.Row + finish(workbench_api, connection, scan, terminal_reason="capped") + complete(workbench_api, connection, scan) + + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert all(publication == coverage for publication in publications) + surfaces = [row for row in coverage["surfaces"] if row.get("workerId") == worker_id] + assert {row["label"] for row in surfaces} == {"download", "upload"} + upload = next(row for row in surfaces if row["label"] == "upload") + tasks = [row for row in coverage["deferred"] if row.get("workerId") == worker_id] + assert len(tasks) == 2 + current = next(row for row in tasks if row["reason"] == current_task["reason"]) + retained = next(row for row in tasks if row["reason"] == retained_task["reason"]) + assert current["surfaceIds"] == [upload["id"]] + assert retained["surfaceIds"] == ( + [upload["id"]] if changed_task else [row["id"] for row in surfaces] + ) + assert {path: path.read_bytes() for path in originals} == originals + + +@pytest.mark.skipif( + not hasattr(os, "fork"), reason="Crash boundary uses a forked workbench process." +) +@pytest.mark.parametrize( + "selected_result", [False, True], ids=["raw-checkpoint", "selected-result"] +) +def test_deadline_dropped_legacy_surface_does_not_rebind_its_pending_task( + workbench_api, workbench_db, deadline_scan, tmp_path, selected_result +): + scan = deadline_scan + worker_id, output = canceled_discovery_output(workbench_db, scan, attempt=2) + archived = output.parent / "attempts" / "attempt-01" + download_task = pending("download-review", "Download review pending.") + archived_checkpoint = write_checkpoint( + archived / "checkpoints", + saved_draft(scan.scan_id, surfaces=[surface("download")], deferred=[download_task]), + ) + os.utime(archived_checkpoint, ns=(100, 100)) + archived_result = archived / "result.json" + archived_result.write_bytes(archived_checkpoint.read_bytes()) + os.utime(archived_result, ns=(110, 110)) + upload_task = pending("upload-review", "Upload review pending.") + # Historical artifacts can contain labels rejected during finalization. + malformed_surface = surface("\u0085") + raw_checkpoint = write_checkpoint( + output / "checkpoints", + saved_draft(scan.scan_id, surfaces=[malformed_surface], deferred=[upload_task]), + ) + os.utime(raw_checkpoint, ns=(200, 200)) + if selected_result: + selected = write_checkpoint( + output / "checkpoints", + saved_draft( + scan.scan_id, + surfaces=[malformed_surface], + deferred=[upload_task, download_task], + ), + ) + os.utime(selected, ns=(210, 210)) + head = output / "checkpoint-head.json" + head.write_text(json.dumps({"checkpoint": selected.name})) + os.utime(head, ns=(220, 220)) + result = output / "result.json" + result.write_bytes(selected.read_bytes()) + os.utime(result, ns=(230, 230)) + originals = {path: path.read_bytes() for path in output.parent.rglob("*") if path.is_file()} + database = tmp_path / "dropped-surface.db" + with sqlite3.connect(database) as connection: + workbench_db.backup(connection) + + publications = [] + for hash_seed in (1, 17, 83): + interrupt_in_fresh_python(database, scan, hash_seed) + publications.append(json.loads((scan.scan_dir / "coverage.json").read_text())) + with sqlite3.connect(database) as connection: + connection.row_factory = sqlite3.Row + finish(workbench_api, connection, scan, terminal_reason="capped") + completed = complete(workbench_api, connection, scan) + + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert completed["progress"]["status"] == "complete" + assert all(publication == coverage for publication in publications) + surfaces = [row for row in coverage["surfaces"] if row.get("workerId") == worker_id] + assert len(surfaces) == 1 and surfaces[0]["label"] == "download" + tasks = [row for row in coverage["deferred"] if row.get("workerId") == worker_id] + retained_upload = next(row for row in tasks if row["reason"] == upload_task["reason"]) + retained_download = next(row for row in tasks if row["reason"] == download_task["reason"]) + assert retained_upload["surfaceIds"] == [] + assert retained_download["surfaceIds"] == [surfaces[0]["id"]] + assert any( + raw_checkpoint.relative_to(scan.scan_dir).as_posix() in warning + and "coverage.surfaces[0].label: expected a non-empty string" in warning + for warning in completed["warnings"] + ) + assert {path: path.read_bytes() for path in originals} == originals From 1bab58f51094af7e2c8123f18f4c977bfc0afdca Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Sat, 17 Oct 2026 21:56:37 +0000 Subject: [PATCH 07/11] fix: preserve recovered coverage links and outcome validation --- .../scripts/finalize_scan_contract.py | 9 +- .../scripts/workbench_saved_results.py | 55 +++++- .../test_deep_scan_deadline_surface_links.py | 175 ++++++++++++++++++ .../tests/test_workbench_checkpoint_heads.py | 17 +- ...t_workbench_recovery_outcome_validation.py | 145 +++++++++++++++ 5 files changed, 386 insertions(+), 15 deletions(-) create mode 100644 plugins/codex-security/tests/test_workbench_recovery_outcome_validation.py diff --git a/plugins/codex-security/scripts/finalize_scan_contract.py b/plugins/codex-security/scripts/finalize_scan_contract.py index 71e98e9453..53f77de54d 100644 --- a/plugins/codex-security/scripts/finalize_scan_contract.py +++ b/plugins/codex-security/scripts/finalize_scan_contract.py @@ -1261,7 +1261,8 @@ def _recover_unsealed_coverage( raise ContractError(f"{context}: expected an object") if field == "surfaces": surface_id = _require_str(item, "id", context) - _require_str(item, "label", context) + if worker_draft: + _require_str(item, "label", context) if surface_id in surface_ids and not worker_draft: raise ContractError(f"{context}.id: duplicate surface id") disposition = item.get("disposition") @@ -1314,7 +1315,8 @@ def _recover_unsealed_coverage( _validate_schema_node(item, item_schema, context) if field == "resolvedDeferred": _validate_resolved_deferred({"resolvedDeferred": [item]}) - _require_safe_json_value(item, context) + if worker_draft: + _require_safe_json_value(item, context) except ContractError as exc: warnings.append(f"Skipped malformed {label} {index + 1}: {exc}.") discarded_fields.add(field) @@ -1349,7 +1351,8 @@ def _recover_unsealed_coverage( "coverage.resolvedDeferred", ) _validate_resolved_deferred(coverage) - _require_safe_json_value(coverage["resolvedDeferred"], "coverage.resolvedDeferred") + if worker_draft: + _require_safe_json_value(coverage["resolvedDeferred"], "coverage.resolvedDeferred") except ContractError as exc: coverage.pop("resolvedDeferred") warnings.append(f"Skipped malformed resolved deferred work: {exc}.") diff --git a/plugins/codex-security/scripts/workbench_saved_results.py b/plugins/codex-security/scripts/workbench_saved_results.py index 69ed6ca0ad..3036836199 100644 --- a/plugins/codex-security/scripts/workbench_saved_results.py +++ b/plugins/codex-security/scripts/workbench_saved_results.py @@ -1479,6 +1479,8 @@ def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None ) # Links use source-local IDs until all retained surfaces receive their final IDs. coverage_sources: dict[int, tuple[dict[str, Any], list[str]]] = {} + source_owners = {relative: owner for relative, _, owner in all_sources} + retained_surface_rows: set[int] = set() for relative, draft, _ in all_sources: for field in ("surfaces", "deferred"): rows = draft["coverage"].get(field) @@ -1493,6 +1495,8 @@ def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None if isinstance(rows, list) and isinstance(originals, list): for row, original in zip(rows, originals): _inherit_coverage_sources(row, [original], coverage_sources) + if field == "surfaces": + retained_surface_rows.update(id(original) for original in originals) def retain_coverage_row(output: list[Any], item: Any, *originals: Any) -> Any: for retained in output: @@ -1502,6 +1506,8 @@ def retain_coverage_row(output: list[Any], item: Any, *originals: Any) -> Any: retained = copy.deepcopy(item) output.append(retained) _inherit_coverage_sources(retained, [item, *originals], coverage_sources) + if output is coverage.get("surfaces"): + retained_surface_rows.update(id(original) for original in (item, *originals)) if any(id(original) in canonical_rows for original in (item, *originals)): canonical_rows.add(id(retained)) return retained @@ -2220,6 +2226,8 @@ def valid_finding(value: Any) -> bool: ) if existing is not None: _inherit_coverage_sources(existing, [original_item], coverage_sources) + if field == "surfaces": + retained_surface_rows.add(id(original_item)) continue retain_coverage_row(output, item, original_item) @@ -2238,6 +2246,7 @@ def valid_finding(value: Any) -> bool: identities[key] = variant recovered_worker_ids = {worker_id for _, _, worker_id in sources if worker_id is not None} surface_positions: dict[str, int] = {} + resolved_surface_targets: dict[tuple[str | None, str], list[str]] = {} def remember_surface_id(item: dict[str, Any], local_id: str) -> None: final_id = item["id"] @@ -2246,6 +2255,13 @@ def remember_surface_id(item: dict[str, Any], local_id: str) -> None: targets = surface_ids_by_source.setdefault((relative, local_id), []) if final_id not in targets: targets.append(final_id) + candidate_id = item.get("candidateId") + if isinstance(candidate_id, str): + key = (source_owners[relative], candidate_id) + if key in resolved and resolved[key] == item.get("disposition"): + targets = resolved_surface_targets.setdefault(key, []) + if final_id not in targets: + targets.append(final_id) for field in ("surfaces", "explicitExclusions", "deferred"): used: set[str] = set() @@ -2282,10 +2298,22 @@ def remember_surface_id(item: dict[str, Any], local_id: str) -> None: if field == "deferred" and isinstance(surface_id, str): for relative in coverage_sources.get(id(item), (item, []))[1]: source_key = (relative, surface_id) - declared |= source_key in surface_ids_by_source - for target in surface_ids_by_source.get(source_key, []): - if target not in targets: - targets.append(target) + history_keys = [source_key] + if source_key not in surface_ids_by_source: + # A deferred-only checkpoint can refer to an earlier + # observation from this logical worker, including retries. + # Reused IDs can denote several retained observations. + history_keys = [ + (historical, surface_id) + for historical, _, owner in all_sources + if owner == source_owners[relative] + and source_order[historical] <= source_order[relative] + ] + for key in history_keys: + declared |= key in surface_ids_by_source + for target in surface_ids_by_source.get(key, []): + if target not in targets: + targets.append(target) if targets: targets.sort(key=surface_positions.__getitem__) for target in targets: @@ -2318,6 +2346,25 @@ def remember_surface_id(item: dict[str, Any], local_id: str) -> None: remember_surface_id(item, local_id) if preserve_canonical_parent and isinstance(items, list): coverage[field] = normalized_items + if field == "surfaces": + # Candidate resolution replaces pending surfaces independently of generic + # tasks. Preserve their original links, even when the outcome changes ID + # or a distinct retained surface shares the old source-local ID. + for relative, draft, owner in all_sources: + originals = draft["coverage"].get("surfaces", []) + for original in originals if isinstance(originals, list) else []: + if ( + not isinstance(original, dict) + or id(original) in retained_surface_rows + or original.get("disposition") != "needs_follow_up" + or not isinstance(local_id := original.get("id"), str) + or not isinstance(candidate_id := original.get("candidateId"), str) + ): + continue + targets = surface_ids_by_source.setdefault((relative, local_id), []) + for target in resolved_surface_targets.get((owner, candidate_id), []): + if target not in targets: + targets.append(target) coverage.pop("completeness", None) if stopped: if not isinstance(coverage.get("deferred"), list): diff --git a/plugins/codex-security/tests/test_deep_scan_deadline_surface_links.py b/plugins/codex-security/tests/test_deep_scan_deadline_surface_links.py index 37d7725299..13b6939046 100644 --- a/plugins/codex-security/tests/test_deep_scan_deadline_surface_links.py +++ b/plugins/codex-security/tests/test_deep_scan_deadline_surface_links.py @@ -426,3 +426,178 @@ def test_deadline_dropped_legacy_surface_does_not_rebind_its_pending_task( for warning in completed["warnings"] ) assert {path: path.read_bytes() for path in originals} == originals + + +@pytest.mark.skipif( + not hasattr(os, "fork"), reason="Crash boundary uses a forked workbench process." +) +@pytest.mark.parametrize( + "history", + ["parent-collision", "current-checkpoints", "retry-checkpoints", "coalesced-history"], +) +def test_deadline_inherited_surface_links_follow_historical_observations( + workbench_api, workbench_db, deadline_scan, tmp_path, history +): + scan = deadline_scan + worker_id, output = canceled_discovery_output( + workbench_db, scan, attempt=2 if history == "retry-checkpoints" else 1 + ) + archived = output.parent / "attempts" / "attempt-01" + first_output = archived if history == "retry-checkpoints" else output + earlier = write_checkpoint( + first_output / "checkpoints", + saved_draft(scan.scan_id, surfaces=[surface("Earlier observation")]), + ) + os.utime(earlier, ns=(100, 100)) + expected_labels = {"Earlier observation"} + if history != "parent-collision": + newer = write_checkpoint( + output / "checkpoints", + saved_draft(scan.scan_id, surfaces=[surface("Newer observation")]), + ) + os.utime(newer, ns=(200, 200)) + expected_labels.add("Newer observation") + inherited_task = pending("history-review", "Review the inherited surface.") + unknown_task = pending("unknown-review", "Review an unresolved reference.", "not-declared") + deferred_only = write_checkpoint( + output / "checkpoints", + saved_draft(scan.scan_id, deferred=[inherited_task, unknown_task]), + ) + os.utime(deferred_only, ns=(300, 300)) + if history == "coalesced-history": + # The merged checkpoint copies a task whose earlier raw source inherited history. + selected = write_checkpoint( + output / "checkpoints", + saved_draft( + scan.scan_id, + surfaces=[surface("Newer observation")], + deferred=[inherited_task, unknown_task], + ), + ) + os.utime(selected, ns=(400, 400)) + head = output / "checkpoint-head.json" + head.write_text(json.dumps({"checkpoint": selected.name})) + os.utime(head, ns=(410, 410)) + if history == "current-checkpoints": + # An interrupted later write cannot identify an earlier task's target. + future = write_checkpoint( + output / "checkpoints", + saved_draft(scan.scan_id, surfaces=[surface("Later observation")]), + ) + os.utime(future, ns=(400, 400)) + canonical_id = f"{worker_id}/entry" if history == "parent-collision" else "parent-entry" + canonical_surface = surface("Canonical parent", canonical_id) + canonical_task = pending("parent-review", "Parent review remains.", canonical_id) + scan.coverage.update(surfaces=[canonical_surface], deferred=[canonical_task]) + (scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage)) + originals = {path: path.read_bytes() for path in output.parent.rglob("*") if path.is_file()} + database = tmp_path / "historical-links.db" + with sqlite3.connect(database) as connection: + workbench_db.backup(connection) + + publications = [] + for hash_seed in (1, 17, 83): + interrupt_in_fresh_python(database, scan, hash_seed) + publications.append(json.loads((scan.scan_dir / "coverage.json").read_text())) + with sqlite3.connect(database) as connection: + connection.row_factory = sqlite3.Row + finish(workbench_api, connection, scan, terminal_reason="capped") + completed = complete(workbench_api, connection, scan) + + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert completed["progress"]["status"] == "complete" + assert all(publication == coverage for publication in publications) + assert canonical_surface in coverage["surfaces"] + assert canonical_task in coverage["deferred"] + worker_surfaces = [row for row in coverage["surfaces"] if row.get("workerId") == worker_id] + historical = [row for row in worker_surfaces if row["label"] in expected_labels] + assert {row["label"] for row in historical} == expected_labels + assert len(historical) == len(expected_labels) + if history == "current-checkpoints": + assert any(row["label"] == "Later observation" for row in worker_surfaces) + tasks = [row for row in coverage["deferred"] if row.get("workerId") == worker_id] + inherited = next(row for row in tasks if row["reason"] == inherited_task["reason"]) + unknown = next(row for row in tasks if row["reason"] == unknown_task["reason"]) + assert inherited["surfaceIds"] == [row["id"] for row in historical] + assert unknown["surfaceIds"] == [f"{worker_id}/not-declared"] + assert {path: path.read_bytes() for path in originals} == originals + + +@pytest.mark.skipif( + not hasattr(os, "fork"), reason="Crash boundary uses a forked workbench process." +) +@pytest.mark.parametrize( + ("changed_surface_id", "legacy_duplicate_id"), + [(False, False), (True, False), (True, True)], + ids=["same-id", "changed-id", "legacy-duplicate-id"], +) +def test_deadline_candidate_supersession_preserves_independent_surface_links( + workbench_api, workbench_db, deadline_scan, tmp_path, changed_surface_id, legacy_duplicate_id +): + scan = deadline_scan + worker_id, output = canceled_discovery_output(workbench_db, scan) + canonical_id = f"{worker_id}/entry" + canonical_surface = surface("Canonical parent", canonical_id) + canonical_task = pending("parent-review", "Parent review remains.", canonical_id) + scan.coverage.update(surfaces=[canonical_surface], deferred=[canonical_task]) + (scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage)) + independent = pending("independent-review", "Review another caller.") + candidate_surface = {**surface("Candidate review"), "candidateId": "candidate"} + earlier_surfaces = [candidate_surface] + if legacy_duplicate_id: + # Legacy artifacts may reuse an ID for a candidate and an independent observation. + earlier_surfaces.append(surface("Independent observation")) + earlier = write_checkpoint( + output / "checkpoints", + saved_draft(scan.scan_id, surfaces=earlier_surfaces, deferred=[independent]), + ) + os.utime(earlier, ns=(100, 100)) + outcome_id = "outcome" if changed_surface_id else "entry" + outcome = { + **surface("Candidate reviewed", outcome_id, "rejected"), + "candidateId": "candidate", + } + raw_checkpoint = write_checkpoint( + output / "checkpoints", saved_draft(scan.scan_id, complete=True, surfaces=[outcome]) + ) + os.utime(raw_checkpoint, ns=(200, 200)) + # The selected merged checkpoint retains independent work before result publication. + selected = write_checkpoint( + output / "checkpoints", + saved_draft(scan.scan_id, complete=True, surfaces=[outcome], deferred=[independent]), + ) + os.utime(selected, ns=(210, 210)) + head = output / "checkpoint-head.json" + head.write_text(json.dumps({"checkpoint": selected.name})) + os.utime(head, ns=(220, 220)) + originals = {path: path.read_bytes() for path in output.rglob("*") if path.is_file()} + database = tmp_path / "candidate-links.db" + with sqlite3.connect(database) as connection: + workbench_db.backup(connection) + + publications = [] + for hash_seed in (1, 17, 83): + interrupt_in_fresh_python(database, scan, hash_seed) + publications.append(json.loads((scan.scan_dir / "coverage.json").read_text())) + with sqlite3.connect(database) as connection: + connection.row_factory = sqlite3.Row + finish(workbench_api, connection, scan, terminal_reason="capped") + completed = complete(workbench_api, connection, scan) + + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert completed["progress"]["status"] == "complete" + assert all(publication == coverage for publication in publications) + assert canonical_surface in coverage["surfaces"] + assert canonical_task in coverage["deferred"] + retained = [row for row in coverage["surfaces"] if row.get("workerId") == worker_id] + assert len(retained) == (2 if legacy_duplicate_id else 1) + outcome = next(row for row in retained if row["label"] == "Candidate reviewed") + assert outcome["disposition"] == "rejected" + if legacy_duplicate_id: + observation = next(row for row in retained if row["label"] == "Independent observation") + assert observation["disposition"] == "needs_follow_up" + assert all(row["id"] != canonical_id for row in retained) + tasks = [row for row in coverage["deferred"] if row.get("workerId") == worker_id] + assert len(tasks) == 1 and tasks[0]["reason"] == independent["reason"] + assert tasks[0]["surfaceIds"] == [row["id"] for row in retained] + assert {path: path.read_bytes() for path in originals} == originals diff --git a/plugins/codex-security/tests/test_workbench_checkpoint_heads.py b/plugins/codex-security/tests/test_workbench_checkpoint_heads.py index 9fdb08db91..6ad64a7e85 100644 --- a/plugins/codex-security/tests/test_workbench_checkpoint_heads.py +++ b/plugins/codex-security/tests/test_workbench_checkpoint_heads.py @@ -417,14 +417,15 @@ def test_recovery_keeps_generic_work_that_collides_with_candidate_aliases( repeated = replay_saved_results(saved, replay, tmp_path, scan_id, binding, workers) assert first[2] == replay[2] == repeated[2] for result in (first, replay, repeated): - assert [row for row in result[2]["deferred"] if row["id"] != "scan-stopped"] == [generic] - assert ( - sum( - row["label"] == "Generic review" and row["disposition"] == "needs_follow_up" - for row in result[2]["surfaces"] - ) - == 1 - ) + pending_surfaces = [ + row + for row in result[2]["surfaces"] + if row["label"] == "Generic review" and row["disposition"] == "needs_follow_up" + ] + assert len(pending_surfaces) == 1 + assert [row for row in result[2]["deferred"] if row["id"] != "scan-stopped"] == [ + {**generic, "surfaceIds": [pending_surfaces[0]["id"]]} + ] assert any( row.get("candidateId") == "candidate-a" and row["disposition"] == outcome for row in result[2]["surfaces"] diff --git a/plugins/codex-security/tests/test_workbench_recovery_outcome_validation.py b/plugins/codex-security/tests/test_workbench_recovery_outcome_validation.py new file mode 100644 index 0000000000..502bf5b9fa --- /dev/null +++ b/plugins/codex-security/tests/test_workbench_recovery_outcome_validation.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +import copy +import json +import os +from types import SimpleNamespace + +import pytest +from test_deep_scan_successful_publication import complete +from test_deep_scan_successful_publication import publication_scan as publication_scan +from workbench_test_support import saved_draft, write_checkpoint + + +@pytest.mark.parametrize( + ("label", "error"), + [ + pytest.param(None, None, id="absent-rejection"), + pytest.param("Reviewed candidate", None, id="valid-rejection"), + pytest.param("Invalid \ud800", "well-formed Unicode", id="surrogate-label"), + pytest.param("\u0085", "label: expected a non-empty string", id="nel-label"), + pytest.param(" ", "label: expected a non-empty string", id="blank-label"), + ], +) +def test_stopped_scan_requires_valid_rejection_evidence_before_sealing( + workbench_api, workbench_db, publication_scan, label, error +): + scan = publication_scan(mode="standard") + report_path = scan.scan_dir / "report.md" + report_path.unlink() + finding = copy.deepcopy(scan.findings[0]) + finding["provenance"] = {"source": "local_plugin", "candidateId": "candidate-a"} + checkpoint = write_checkpoint( + scan.scan_dir / "checkpoints", saved_draft(scan.scan_id, findings=[finding]) + ) + os.utime(checkpoint, ns=(100, 100)) + checkpoint_bytes = checkpoint.read_bytes() + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + scan.coverage["surfaces"] = ( + [] + if label is None + else [ + { + "id": "rejection", + "candidateId": "candidate-a", + "label": label, + "disposition": "rejected", + "receiptRefs": [], + } + ] + ) + coverage_path = scan.scan_dir / "coverage.json" + coverage_path.write_text(json.dumps(scan.coverage)) + coverage_bytes = coverage_path.read_bytes() + + workbench_api["saved_results"].fail_scan( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + SimpleNamespace( + scan_id=scan.scan_id, + cost_json=None, + claim_token=None, + message="Synthetic interruption", + ), + ) + + row = workbench_db.execute( + "SELECT status, completion_warnings_json FROM scans WHERE id = ?", (scan.scan_id,) + ).fetchone() + assert row["status"] == "failed" + assert checkpoint.read_bytes() == checkpoint_bytes + manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"] + warnings = json.loads(row["completion_warnings_json"]) + if error: + assert not manifest.get("sealedAt") + assert not report_path.exists() + assert coverage_path.read_bytes() == coverage_bytes + assert any( + "result publication needs follow-up" in warning and error in warning + for warning in warnings + ) + else: + assert manifest["sealedAt"] + assert report_path.is_file() + assert warnings == [] + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert [item["title"] for item in findings] == ([finding["title"]] if label is None else []) + surfaces = json.loads(coverage_path.read_text())["surfaces"] + assert [(item.get("candidateId"), item["disposition"]) for item in surfaces] == ( + [] if label is None else [("candidate-a", "rejected")] + ) + + +@pytest.mark.parametrize( + ("reason", "invalid"), + [ + pytest.param("Review completed.", False, id="valid-closure"), + pytest.param("Invalid \ud800", True, id="surrogate-reason"), + ], +) +def test_standard_completion_requires_valid_closure_evidence_before_sealing( + workbench_api, workbench_db, publication_scan, reason, invalid +): + scan = publication_scan(mode="standard") + report_path = scan.scan_dir / "report.md" + report_path.unlink() + checkpoint = write_checkpoint( + scan.scan_dir / "checkpoints", + saved_draft( + scan.scan_id, + deferred=[{"id": "review", "reason": "Review authorization remains pending."}], + ), + ) + os.utime(checkpoint, ns=(100, 100)) + checkpoint_bytes = checkpoint.read_bytes() + scan.coverage["deferred"] = [] + closure = {"id": "review", "reason": reason} + scan.coverage["resolvedDeferred"] = [closure] + coverage_path = scan.scan_dir / "coverage.json" + coverage_path.write_text(json.dumps(scan.coverage)) + coverage_bytes = coverage_path.read_bytes() + + if invalid: + with pytest.raises(SystemExit, match="well-formed Unicode"): + complete(workbench_api, workbench_db, scan) + else: + completed = complete(workbench_api, workbench_db, scan) + assert completed["warnings"] == [] + + assert checkpoint.read_bytes() == checkpoint_bytes + status = workbench_db.execute( + "SELECT status FROM scans WHERE id = ?", (scan.scan_id,) + ).fetchone()[0] + manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"] + if invalid: + assert status == "running" + assert not manifest.get("sealedAt") + assert not report_path.exists() + assert coverage_path.read_bytes() == coverage_bytes + else: + assert status == "complete" + assert manifest["sealedAt"] + assert report_path.is_file() + coverage = json.loads(coverage_path.read_text()) + assert coverage["deferred"] == [] + assert coverage["resolvedDeferred"] == [closure] From ada1ff7097a00999e744ff5e4f16ed1bd19714ca Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Tue, 20 Oct 2026 17:56:21 +0000 Subject: [PATCH 08/11] fix(deep-scan): preserve retained surface link identity --- plugins/codex-security/plugin-files.json | 2 +- .../retained_coverage.py} | 5 +- .../scripts/workbench/saved_sources.py | 2 +- .../scripts/workbench_saved_results.py | 95 ++++++++++--------- 4 files changed, 55 insertions(+), 49 deletions(-) rename plugins/codex-security/scripts/{workbench_saved_coverage.py => workbench/retained_coverage.py} (99%) diff --git a/plugins/codex-security/plugin-files.json b/plugins/codex-security/plugin-files.json index 56bbf8465f..9518bedfa0 100644 --- a/plugins/codex-security/plugin-files.json +++ b/plugins/codex-security/plugin-files.json @@ -95,7 +95,7 @@ "scripts/workbench_publication.py", "scripts/workbench_remediation.py", "scripts/workbench_saved_results.py", - "scripts/workbench_saved_coverage.py", + "scripts/workbench/retained_coverage.py", "scripts/workbench_scan_history.py", "scripts/workbench_scan_start.py", "scripts/workbench_scan_usage.py", diff --git a/plugins/codex-security/scripts/workbench_saved_coverage.py b/plugins/codex-security/scripts/workbench/retained_coverage.py similarity index 99% rename from plugins/codex-security/scripts/workbench_saved_coverage.py rename to plugins/codex-security/scripts/workbench/retained_coverage.py index c98e8c33b1..e012c28795 100644 --- a/plugins/codex-security/scripts/workbench_saved_coverage.py +++ b/plugins/codex-security/scripts/workbench/retained_coverage.py @@ -7,13 +7,10 @@ import json import os import re -import sys from collections.abc import Callable, Iterator from pathlib import Path from typing import Any -sys.path.insert(0, str(Path(__file__).resolve().parent)) - from finalize_scan_contract import ( ContractError, _read_json, @@ -185,7 +182,7 @@ def _merge_tied_parent_observations( if isinstance(identity, str) } closure_schema = _read_json( - Path(__file__).resolve().parent.parent / "schemas" / "coverage.schema.json" + Path(__file__).resolve().parents[2] / "schemas" / "coverage.schema.json" )["properties"]["resolvedDeferred"] closures = {} for observed in (current, previous): diff --git a/plugins/codex-security/scripts/workbench/saved_sources.py b/plugins/codex-security/scripts/workbench/saved_sources.py index cf5c70ab65..ecff801312 100644 --- a/plugins/codex-security/scripts/workbench/saved_sources.py +++ b/plugins/codex-security/scripts/workbench/saved_sources.py @@ -12,7 +12,7 @@ _read_scan_local_json_with_metadata, write_scan_local_bytes, ) -from workbench_saved_coverage import _digest, _encoded +from workbench.retained_coverage import _digest, _encoded def _checkpoint_head_directory(relative: str) -> Path | None: diff --git a/plugins/codex-security/scripts/workbench_saved_results.py b/plugins/codex-security/scripts/workbench_saved_results.py index ac7874485d..057f1657dd 100644 --- a/plugins/codex-security/scripts/workbench_saved_results.py +++ b/plugins/codex-security/scripts/workbench_saved_results.py @@ -69,6 +69,45 @@ ) +from workbench.retained_coverage import ( + _deferred_candidate_id as _deferred_candidate_id, +) +from workbench.retained_coverage import ( + _deferred_rows as _deferred_rows, +) +from workbench.retained_coverage import ( + _digest as _digest, +) +from workbench.retained_coverage import ( + _encoded as _encoded, +) +from workbench.retained_coverage import ( + _ensure_finding_identity as _ensure_finding_identity, +) +from workbench.retained_coverage import ( + _finding_content as _finding_content, +) +from workbench.retained_coverage import ( + _finding_key as _finding_key, +) +from workbench.retained_coverage import ( + _generic_surface_updates as _generic_surface_updates, +) +from workbench.retained_coverage import ( + _inherit_coverage_sources as _inherit_coverage_sources, +) +from workbench.retained_coverage import ( + _merge_tied_parent_observations as _merge_tied_parent_observations, +) +from workbench.retained_coverage import ( + _resolved_deferred_rows as _resolved_deferred_rows, +) +from workbench.retained_coverage import ( + _retained_findings as _retained_findings, +) +from workbench.retained_coverage import ( + _worker_candidate_key as _worker_candidate_key, +) from workbench.saved_sources import ( _capture_saved_source as _capture_saved_source, ) @@ -96,45 +135,6 @@ from workbench.saved_sources import ( _source_digests as _source_digests, ) -from workbench_saved_coverage import ( - _deferred_candidate_id as _deferred_candidate_id, -) -from workbench_saved_coverage import ( - _deferred_rows as _deferred_rows, -) -from workbench_saved_coverage import ( - _digest as _digest, -) -from workbench_saved_coverage import ( - _encoded as _encoded, -) -from workbench_saved_coverage import ( - _ensure_finding_identity as _ensure_finding_identity, -) -from workbench_saved_coverage import ( - _finding_content as _finding_content, -) -from workbench_saved_coverage import ( - _finding_key as _finding_key, -) -from workbench_saved_coverage import ( - _generic_surface_updates as _generic_surface_updates, -) -from workbench_saved_coverage import ( - _inherit_coverage_sources as _inherit_coverage_sources, -) -from workbench_saved_coverage import ( - _merge_tied_parent_observations as _merge_tied_parent_observations, -) -from workbench_saved_coverage import ( - _resolved_deferred_rows as _resolved_deferred_rows, -) -from workbench_saved_coverage import ( - _retained_findings as _retained_findings, -) -from workbench_saved_coverage import ( - _worker_candidate_key as _worker_candidate_key, -) def threat_model_fields(db: Any, scan: sqlite3.Row) -> dict[str, Any]: @@ -906,8 +906,8 @@ def append_coverage_row(output: list[Any], item: Any, owner: str | None, field: output.append(retained) retained = output[output.index(retained)] record_coverage_owner(retained, item, owner, field) + _inherit_coverage_sources(retained, [item], coverage_sources) if preserve_canonical_parent: - _inherit_coverage_sources(retained, [item], coverage_sources) if id(item) in canonical_rows: canonical_rows.add(id(retained)) if field == "surfaces": @@ -2040,10 +2040,19 @@ def coverage_row_is_retained( def remember_surface_id(item: dict[str, Any], local_id: str) -> None: final_id = item["id"] surface_positions.setdefault(final_id, len(surface_positions)) + shadowed_alias = ( + project_worker_coverage + and id(item) not in named_surface_outputs + and (named := worker_surfaces.get((worker_row_owners.get(id(item)), local_id))) + is not None + and id(named) in named_surface_outputs + ) for relative in coverage_sources.get(id(item), (item, []))[1]: - targets = surface_ids_by_source.setdefault((relative, local_id), []) - if final_id not in targets: - targets.append(final_id) + # A candidate-derived ID cannot claim a link explicitly naming another surface. + if not shadowed_alias: + targets = surface_ids_by_source.setdefault((relative, local_id), []) + if final_id not in targets: + targets.append(final_id) candidate_id = item.get("candidateId") if isinstance(candidate_id, str): key = (source_owners[relative], candidate_id) From ffe80c4b01226d32aeffe977a242457866eb873c Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Thu, 22 Oct 2026 21:21:32 +0000 Subject: [PATCH 09/11] fix(deep-scan): preserve tasks that share the deadline marker ID --- .../scripts/workbench_saved_results.py | 6 +- .../tests/test_deep_scan_deadline_recovery.py | 58 +++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/plugins/codex-security/scripts/workbench_saved_results.py b/plugins/codex-security/scripts/workbench_saved_results.py index 3e13ccdd9e..24182d4810 100644 --- a/plugins/codex-security/scripts/workbench_saved_results.py +++ b/plugins/codex-security/scripts/workbench_saved_results.py @@ -2354,7 +2354,11 @@ def preserve_deep_deadline_draft( "pending work from unfinished source reviews were preserved." ), } - if marker not in documents[2]["deferred"]: + if not any( + marker["id"] in (row.get("id"), row.get("candidateId")) + for key in ("deferred", "resolvedDeferred") + for row in documents[2].get(key, []) + ): documents[2]["deferred"].append(marker) documents[0]["scan"].pop("complete", None) # Validate recovered evidence against the current finding contract, diff --git a/plugins/codex-security/tests/test_deep_scan_deadline_recovery.py b/plugins/codex-security/tests/test_deep_scan_deadline_recovery.py index 1bd3946ba9..d86b711de5 100644 --- a/plugins/codex-security/tests/test_deep_scan_deadline_recovery.py +++ b/plugins/codex-security/tests/test_deep_scan_deadline_recovery.py @@ -33,6 +33,64 @@ def worker_finding(scan, anchor): return finding +@pytest.mark.parametrize( + ("task_state", "task_id", "marker_count"), + [ + ("resolvedDeferred", "scan-deadline", 0), + ("resolvedDeferred", "prior-review", 1), + ("deferred", "scan-deadline", 0), + ], +) +def test_deadline_completion_preserves_existing_tasks_and_recovered_findings( + workbench_api, workbench_db, deadline_scan, task_state, task_id, marker_count +): + scan = deadline_scan + task = {"id": task_id, "reason": "Existing review outcome."} + scan.coverage[task_state] = [task] + (scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage)) + worker_id, output = canceled_discovery_output(workbench_db, scan) + recovered = worker_finding(scan, "saved-before-deadline") + checkpoint = write_checkpoint( + output / "checkpoints", + saved_draft( + scan.scan_id, + findings=[recovered], + deferred=[{"id": "unfinished-review", "reason": "Review remains unfinished."}], + ), + ) + original_checkpoint = checkpoint.read_bytes() + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + completed = complete(workbench_api, workbench_db, scan) + + assert completed["progress"]["status"] == "complete" + assert ( + workbench_db.execute("SELECT status FROM scans WHERE id = ?", (scan.scan_id,)).fetchone()[0] + == "complete" + ) + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert task in coverage[task_state] + if task_state == "resolvedDeferred": + assert coverage[task_state] == [task] + assert any( + row.get("workerId") == worker_id and row["reason"] == "Review remains unfinished." + for row in coverage["deferred"] + ) + assert ( + sum(row.get("id") == "scan-deadline" and row != task for row in coverage["deferred"]) + == marker_count + ) + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert {row["title"] for row in findings} == {scan.findings[0]["title"], recovered["title"]} + assert ( + next(row for row in findings if row["title"] == recovered["title"])["provenance"][ + "workerId" + ] + == worker_id + ) + assert checkpoint.read_bytes() == original_checkpoint + + def interrupt_finish_before_commit(workbench_api, database, scan): child = os.fork() if child == 0: From 225c47055906b64c48f2c034b1c06498ebd0c831 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Thu, 22 Oct 2026 21:22:52 +0000 Subject: [PATCH 10/11] test: close native configuration fixture before preflight reply --- .../tests-ts/cli-skill-native-config.test.ts | 100 +++++++++++++++++- .../tests-ts/fixtures/skill-native-config.mjs | 38 ++++++- 2 files changed, 133 insertions(+), 5 deletions(-) diff --git a/sdk/typescript/tests-ts/cli-skill-native-config.test.ts b/sdk/typescript/tests-ts/cli-skill-native-config.test.ts index 007ce942a0..59a63a8cf8 100644 --- a/sdk/typescript/tests-ts/cli-skill-native-config.test.ts +++ b/sdk/typescript/tests-ts/cli-skill-native-config.test.ts @@ -1,6 +1,13 @@ import { execFileSync, spawn } from "node:child_process"; import { once } from "node:events"; -import { mkdir, mkdtemp, realpath, rm, writeFile } from "node:fs/promises"; +import { + mkdir, + mkdtemp, + readFile, + realpath, + rm, + writeFile, +} from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -18,6 +25,97 @@ import { } from "../src/runtime.js"; import { capture } from "./cli-fixtures.js"; +test.each([ + ["codex_security_deep_scan", true, false], + ["codex_security_deep_scan", true, true], + ["codex_security_deep_scan", false, false], + ["codex-security", true, false], +] as const)( + "native configuration fixture owns preflight shutdown: client=%s, layers=%p, error=%p", + async (clientName, includeLayers, error) => { + const root = await mkdtemp(join(tmpdir(), "native-config-lifetime-")); + const exited = join(root, "native-exited"); + const response = { + id: 2, + ...(error + ? { error: { code: -32603, message: "synthetic configuration error" } } + : { + result: { + config: { model: "synthetic-model", daybreak: true }, + origins: { daybreak: { name: { type: "user" } } }, + layers: [{ name: { type: "user" }, config: { daybreak: true } }], + }, + }), + }; + const child = spawn( + process.execPath, + [ + fileURLToPath( + new URL("./fixtures/skill-native-config.mjs", import.meta.url), + ), + "-e", + `const { createInterface } = require("node:readline"); + const { writeFileSync } = require("node:fs"); + createInterface({ input: process.stdin }).on("line", line => { + const request = JSON.parse(line); + const response = request.method === "config/read" + ? ${JSON.stringify(response)} + : { id: request.id, result: {} }; + process.stdout.write(JSON.stringify(response) + "\\n"); + }); + process.on("exit", () => writeFileSync(${JSON.stringify(exited)}, "exited"));`, + ], + { + env: { ...process.env, SYNTHETIC_NATIVE_CODEX: process.execPath }, + stdio: ["pipe", "pipe", "pipe"], + windowsHide: true, + }, + ); + const closed = once(child, "close"); + const lines = createInterface({ input: child.stdout }); + const timeout = setTimeout(() => child.kill(), 10_000); + try { + child.stdin.write( + JSON.stringify({ + id: 1, + method: "initialize", + params: { clientInfo: { name: clientName } }, + }) + "\n", + ); + await once(lines, "line"); + child.stdin.write( + JSON.stringify({ + id: 2, + method: "config/read", + params: { includeLayers }, + }) + "\n", + ); + const [line] = await once(lines, "line"); + expect(line).toBe(JSON.stringify(response)); + if (clientName === "codex_security_deep_scan" && includeLayers) { + expect(await readFile(exited, "utf8")).toBe("exited"); + child.stdin.end(); + child.kill("SIGTERM"); + } else { + await expect(readFile(exited, "utf8")).rejects.toHaveProperty( + "code", + "ENOENT", + ); + child.stdin.end(); + } + await closed; + expect(await readFile(exited, "utf8")).toBe("exited"); + } finally { + child.stdin.end(); + await closed; + clearTimeout(timeout); + lines.close(); + child.stderr.destroy(); + await rm(root, { recursive: true, force: true }); + } + }, +); + test.each([0, 23])( "native configuration fixture preserves early exit %p with its input still open", async (exitCode) => { diff --git a/sdk/typescript/tests-ts/fixtures/skill-native-config.mjs b/sdk/typescript/tests-ts/fixtures/skill-native-config.mjs index 39b478635d..c22736c97e 100644 --- a/sdk/typescript/tests-ts/fixtures/skill-native-config.mjs +++ b/sdk/typescript/tests-ts/fixtures/skill-native-config.mjs @@ -10,11 +10,16 @@ const child = spawn(process.env.SYNTHETIC_NATIVE_CODEX, process.argv.slice(2), { }); child.stderr.pipe(process.stderr); const input = createInterface({ input: process.stdin }); -child.once("close", (code) => { - process.exitCode = code ?? 1; - input.close(); - process.stdin.destroy(); +const childClosed = new Promise((resolve) => { + child.once("close", (code) => { + process.exitCode = code ?? 1; + input.close(); + process.stdin.destroy(); + resolve(); + }); }); +let preflightClient = false; +let preflightConfigId; let turnRequest; let modelProvider; let initialConfig; @@ -24,6 +29,15 @@ await Promise.all([ for await (const line of input) { const request = JSON.parse(line); if (request.id !== undefined) methods.set(request.id, request.method); + if (request.method === "initialize") + preflightClient = + request.params.clientInfo.name === "codex_security_deep_scan"; + if ( + preflightClient && + request.method === "config/read" && + request.params.includeLayers === true + ) + preflightConfigId = request.id; if (request.method === "thread/start") { daybreakEnabled = request.params.daybreakEnabled ?? null; } @@ -45,8 +59,19 @@ await Promise.all([ child.stdin.end(); })(), (async () => { + let preflightResponse; for await (const line of createInterface({ input: child.stdout })) { const response = JSON.parse(line); + if ( + preflightConfigId !== undefined && + response.id === preflightConfigId + ) { + // The one-shot origins reader terminates this proxy after the response. + // Close its native child first, including on Windows where SIGTERM is forced. + preflightResponse = line; + child.stdin.end(); + continue; + } if (methods.get(response.id) === "thread/start") modelProvider = response.result?.modelProvider; if (methods.get(response.id) === "config/read") @@ -77,5 +102,10 @@ await Promise.all([ threadId, ); } + if (preflightResponse !== undefined) { + await childClosed; + process.stdout.write(preflightResponse + "\n"); + } })(), + childClosed, ]); From 1ef4ecb6c1ee71cb3da5d0c5390526a53046d1bb Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Fri, 23 Oct 2026 05:38:55 +0000 Subject: [PATCH 11/11] refactor(imports): separate report readers from source providers --- sdk/typescript/src/cli.ts | 3 ++- sdk/typescript/src/external-findings-import.ts | 13 ++++++++++--- .../src/external-findings-publish.ts | 18 +++++++++++++++--- sdk/typescript/src/external-findings-read.ts | 18 +++++++++++++++++- .../tests-ts/external-findings-import.test.ts | 5 +++++ .../tests-ts/external-findings-publish.test.ts | 9 ++++++--- .../tests-ts/semgrep-findings-local.test.ts | 4 ++++ 7 files changed, 59 insertions(+), 11 deletions(-) diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts index 1782624838..cdf6d89701 100644 --- a/sdk/typescript/src/cli.ts +++ b/sdk/typescript/src/cli.ts @@ -112,6 +112,7 @@ import { } from "./external-findings-publish.js"; import { prepareLocalExternalImport } from "./external-findings-import.js"; import { listLocalExternalFindings } from "./external-findings-local.js"; +import { EXTERNAL_FINDINGS_READERS } from "./external-findings-read.js"; import type { ExternalFindings } from "./external-findings.js"; import { publishPatchReview, @@ -3553,7 +3554,7 @@ export async function main( "Local repository path for --to local; authorized repository ID or URL for --to cloud.", ), provider: z - .enum(["wiz", "snyk", "semgrep", "endor", "github"]) + .enum(EXTERNAL_FINDINGS_READERS) .describe("Provider that produced these findings."), sourceRepositoryUrl: optionValue("--source-repository-url") .optional() diff --git a/sdk/typescript/src/external-findings-import.ts b/sdk/typescript/src/external-findings-import.ts index dd4391256a..27b997c86b 100644 --- a/sdk/typescript/src/external-findings-import.ts +++ b/sdk/typescript/src/external-findings-import.ts @@ -3,7 +3,10 @@ import { cloudRepositoryUrlKey, repositoryUrlKey, } from "./external-import-contract.js"; -import { readExternalFindings } from "./external-findings-read.js"; +import { + EXTERNAL_FINDINGS_PROVIDERS, + readExternalFindings, +} from "./external-findings-read.js"; import { prepareNormalizedLocalImport, type LocalExternalImportDependencies, @@ -21,8 +24,12 @@ export async function prepareLocalExternalImport( () => readExternalFindings(path, options), dependencies.signal, ); + const normalizedOptions = { + ...options, + provider: EXTERNAL_FINDINGS_PROVIDERS[options.provider], + }; let sourceRepositories = parsed.sourceRepositories ?? []; - if (options.provider === "semgrep") { + if (normalizedOptions.provider === "semgrep") { // The Cloud reader allows custom GitHub casing. Local provenance must also // agree with the caller's source URL when the checkout has no remote. const source = options.sourceRepositoryUrl!; @@ -47,7 +54,7 @@ export async function prepareLocalExternalImport( })), ]; } - return prepareNormalizedLocalImport(parsed.findings, options, { + return prepareNormalizedLocalImport(parsed.findings, normalizedOptions, { ...dependencies, sourceRepositories: [ ...(dependencies.sourceRepositories ?? []), diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index c617b82346..5f5b83daf6 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -25,6 +25,7 @@ import { validatePreparedOutputDir, } from "./runtime.js"; import { + EXTERNAL_FINDINGS_PROVIDERS, readExternalFindings, type ExternalFindingsReaderOptions, } from "./external-findings-read.js"; @@ -61,7 +62,10 @@ export interface NormalizedExternalPublicationOptions { } /** File readers accept the vendor formats installed in this SDK. */ -export interface ExternalPublicationOptions extends NormalizedExternalPublicationOptions { +export interface ExternalPublicationOptions extends Omit< + NormalizedExternalPublicationOptions, + "provider" +> { provider: ExternalFindingsReaderOptions["provider"]; sourceRepositoryUrl?: string; } @@ -347,11 +351,19 @@ export async function prepareExternalPublication( dependencies: ExternalPublicationDependencies = {}, ): Promise { const parsed = await readExternalFindings(path, options); + const normalizedOptions = { + ...options, + provider: EXTERNAL_FINDINGS_PROVIDERS[options.provider], + }; return parsed.publication - ? preparePublicationInput(parsed.publication, options, dependencies) + ? preparePublicationInput( + parsed.publication, + normalizedOptions, + dependencies, + ) : prepareNormalizedExternalPublication( parsed.findings, - options, + normalizedOptions, dependencies, ); } diff --git a/sdk/typescript/src/external-findings-read.ts b/sdk/typescript/src/external-findings-read.ts index 00230860e2..99fbcb0d57 100644 --- a/sdk/typescript/src/external-findings-read.ts +++ b/sdk/typescript/src/external-findings-read.ts @@ -3,6 +3,7 @@ import type { ExternalPublicationInput, ExternalSourceRepository, } from "./external-findings.js"; +import type { Provider } from "./external-import-models.js"; import { wizCsvRepositoryUrl } from "./wiz-csv.js"; import { readSnykFindings, snykRepositoryUrls } from "./snyk-findings.js"; import { readSemgrepFindings } from "./semgrep-findings.js"; @@ -12,9 +13,24 @@ import { isGitHubCloudHost } from "./git-remote.js"; import { isRecord } from "./record.js"; import { readVendorFindingsForPublication } from "./wiz-findings.js"; +/** File reader names mapped to their stored and Cloud source provider. */ +export const EXTERNAL_FINDINGS_PROVIDERS = { + wiz: "wiz", + snyk: "snyk", + semgrep: "semgrep", + endor: "endor", + github: "github", +} as const satisfies Record; + +export type ExternalFindingsReader = keyof typeof EXTERNAL_FINDINGS_PROVIDERS; + +export const EXTERNAL_FINDINGS_READERS = Object.keys( + EXTERNAL_FINDINGS_PROVIDERS, +) as ExternalFindingsReader[]; + /** File readers available in this SDK, shared by local and Cloud imports. */ export interface ExternalFindingsReaderOptions { - provider: "wiz" | "snyk" | "semgrep" | "endor" | "github"; + provider: ExternalFindingsReader; sourceRepositoryUrl?: string; } diff --git a/sdk/typescript/tests-ts/external-findings-import.test.ts b/sdk/typescript/tests-ts/external-findings-import.test.ts index 4218cfa887..6b753da288 100644 --- a/sdk/typescript/tests-ts/external-findings-import.test.ts +++ b/sdk/typescript/tests-ts/external-findings-import.test.ts @@ -96,6 +96,10 @@ test("Wiz JSON imports retain parsed evidence through SQLite readback and update dependencies, ); expect(prepared.preview.findings).toEqual(parsed.findings); + expect(prepared.preview.source).toEqual({ + provider: "wiz", + source_key: options.sourceKey, + }); await expect( access(join(fixture.stateDirectory, "workbench.sqlite3")), ).rejects.toThrow(); @@ -107,6 +111,7 @@ test("Wiz JSON imports retain parsed evidence through SQLite readback and update ); expect(stored).toEqual(first.findings); expect(stored[0]).toMatchObject({ + provider: "wiz", sourceFindingId: "package-occurrence", assessment: { state: "not_assessed" }, version: 1, diff --git a/sdk/typescript/tests-ts/external-findings-publish.test.ts b/sdk/typescript/tests-ts/external-findings-publish.test.ts index 63233bbe55..57d278b122 100644 --- a/sdk/typescript/tests-ts/external-findings-publish.test.ts +++ b/sdk/typescript/tests-ts/external-findings-publish.test.ts @@ -3463,9 +3463,12 @@ test("Semgrep publication preserves identity across JSON and SARIF with explicit sourceKey: "organization/project/main", sourceRepositoryUrl: "https://github.com/example/project", }; - const first = await ( - await prepareExternalPublication(f.file, settings, f.deps) - ).publish(); + const prepared = await prepareExternalPublication(f.file, settings, f.deps); + expect(prepared.preview.source).toEqual({ + provider: "semgrep", + source_key: settings.sourceKey, + }); + const first = await prepared.publish(); expect(first.counts.created).toBe(1); expect(first.verified).toBe(1); const original = structuredClone([...f.reports.values()][0]!); diff --git a/sdk/typescript/tests-ts/semgrep-findings-local.test.ts b/sdk/typescript/tests-ts/semgrep-findings-local.test.ts index c3559c9cda..de22cb8a81 100644 --- a/sdk/typescript/tests-ts/semgrep-findings-local.test.ts +++ b/sdk/typescript/tests-ts/semgrep-findings-local.test.ts @@ -196,6 +196,10 @@ test.each(formats)( ); expect(prepared.preview.findings).toHaveLength(1); expect(prepared.preview.excluded).toEqual([]); + expect(prepared.preview.source).toEqual({ + provider: "semgrep", + source_key: options.sourceKey, + }); const imported = await prepared.import(); expect(imported.counts).toEqual({ created: 1, updated: 0, unchanged: 0 });