diff --git a/plugins/codex-security/plugin-files.json b/plugins/codex-security/plugin-files.json index b34ac93cf..3b8c90052 100644 --- a/plugins/codex-security/plugin-files.json +++ b/plugins/codex-security/plugin-files.json @@ -82,6 +82,7 @@ "scripts/workbench/runtime_toml.py", "scripts/workbench/saved_coverage.py", "scripts/workbench/saved_outputs.py", + "scripts/workbench/saved_sources.py", "scripts/workbench/storage.py", "scripts/workbench_cli.py", "scripts/workbench_constants.py", @@ -95,6 +96,7 @@ "scripts/workbench_publication.py", "scripts/workbench_remediation.py", "scripts/workbench_saved_results.py", + "scripts/workbench/retained_coverage.py", "scripts/workbench_scan_history.py", "scripts/workbench_scan_start.py", "scripts/workbench_scan_usage.py", diff --git a/plugins/codex-security/scripts/deep_scan_workbench.py b/plugins/codex-security/scripts/deep_scan_workbench.py index 8ae81df50..ed71e008d 100644 --- a/plugins/codex-security/scripts/deep_scan_workbench.py +++ b/plugins/codex-security/scripts/deep_scan_workbench.py @@ -1747,9 +1747,9 @@ def finish_deep_scan_locked( """, (scan_id,), ).fetchone() + discovery_deadline = args.terminal_reason == "capped" and deep_scan_deadline_reached(run) zero_discovery_deadline = ( - args.terminal_reason == "capped" - and deep_scan_deadline_reached(run) + discovery_deadline and run["completion_sequence"] == 0 and ( standard_scan_manifest @@ -1781,9 +1781,13 @@ def finish_deep_scan_locked( kind="file", ) promotion = promote_staged_file(staged_manifest_path, manifest_path) - if zero_discovery_deadline: + if zero_discovery_deadline or (discovery_deadline and standard_scan_manifest): deadline_publication = saved_results.preserve_deep_deadline_draft( - dependencies(), connection, scan, allow_empty=not standard_scan_manifest + dependencies(), + connection, + scan, + preserve_reduction=not zero_discovery_deadline, + allow_empty=not standard_scan_manifest, ) timestamp = dependencies().now() connection.execute( diff --git a/plugins/codex-security/scripts/finalize_scan_contract.py b/plugins/codex-security/scripts/finalize_scan_contract.py index 8442910fa..ca7f16859 100644 --- a/plugins/codex-security/scripts/finalize_scan_contract.py +++ b/plugins/codex-security/scripts/finalize_scan_contract.py @@ -1211,6 +1211,7 @@ def _recover_unsealed_findings( f"Skipped malformed {auxiliary} for finding {index + 1}: {exc}." ) _validate_schema_node(finding, finding_schema, context, schema) + _require_safe_json_value(finding, context) if current_contract: _validate_current_findings({"findings": [finding]}, schema_dir) except ContractError as exc: @@ -1268,9 +1269,12 @@ def _recover_unsealed_coverage( scan_dir: Path, warnings: list[str], discarded_findings: list[str], -) -> None: + *, + worker_draft: bool = False, +) -> set[str]: schema = _read_json(schema_dir / "coverage.schema.json") properties = _require_dict(schema, "properties", "coverage.schema") + discarded_fields: set[str] = set() if ( coverage.get("mode") == "deep_repository" and coverage.get("inventoryStrategy") != "repository" @@ -1280,16 +1284,24 @@ def _recover_unsealed_coverage( surface_ids: set[str] = set() receipt_paths: dict[str, str] = {} - for field, label in ( + fields = [ ("surfaces", "coverage surface"), ("explicitExclusions", "coverage exclusion"), ("deferred", "deferred coverage item"), - ): + ] + if worker_draft: + if "openQuestions" in coverage: + _normalize_unsealed_open_questions(coverage) + fields.append(("openQuestions", "coverage open question")) + if "resolvedDeferred" in coverage: + fields.append(("resolvedDeferred", "resolved deferred item")) + for field, label in fields: array_schema = _require_dict(properties, field, "coverage.schema.properties") item_schema = _require_dict(array_schema, "items", f"coverage.schema.properties.{field}") items = coverage.get(field) if not isinstance(items, list): warnings.append(f"Skipped malformed {label} records: expected an array.") + discarded_fields.add(field) coverage[field] = [] continue @@ -1302,7 +1314,9 @@ def _recover_unsealed_coverage( if field == "surfaces": surface_receipt_paths = dict(receipt_paths) surface_id = _require_str(item, "id", context) - if surface_id in surface_ids: + if worker_draft: + _require_str(item, "label", context) + if surface_id in surface_ids and not worker_draft: raise ContractError(f"{context}.id: duplicate surface id") disposition = item.get("disposition") surface_recovered = False @@ -1352,12 +1366,18 @@ def _recover_unsealed_coverage( item["receiptRefs"] = recovered_receipts if surface_recovered: item["disposition"] = "needs_follow_up" + discarded_fields.add(field) if field == "deferred": _validate_deferred_paths(item, context) _validate_schema_node(item, item_schema, context) + if field == "resolvedDeferred": + _validate_resolved_deferred({"resolvedDeferred": [item]}) + if worker_draft: + _require_safe_json_value(item, context) except ContractError as exc: warnings.append(f"Skipped malformed {label} {index + 1}: {exc}.") + discarded_fields.add(field) continue if field == "surfaces": @@ -1369,6 +1389,13 @@ def _recover_unsealed_coverage( if discarded_findings: for surface in coverage["surfaces"]: + if ( + worker_draft + and surface.get("candidateId") + and surface["disposition"] in {"rejected", "not_applicable"} + ): + # A valid explicit outcome can still resolve that worker's history. + continue surface["disposition"] = "needs_follow_up" coverage["deferred"].extend( {"id": f"discarded-finding-{index}", "reason": warning} @@ -1383,9 +1410,13 @@ def _recover_unsealed_coverage( "coverage.resolvedDeferred", ) _validate_resolved_deferred(coverage) + if worker_draft: + _require_safe_json_value(coverage["resolvedDeferred"], "coverage.resolvedDeferred") except ContractError as exc: coverage.pop("resolvedDeferred") warnings.append(f"Skipped malformed resolved deferred work: {exc}.") + discarded_fields.add("resolvedDeferred") + return discarded_fields def _recover_unsealed_hardening( diff --git a/plugins/codex-security/scripts/workbench/retained_coverage.py b/plugins/codex-security/scripts/workbench/retained_coverage.py new file mode 100644 index 000000000..fdf1a95f8 --- /dev/null +++ b/plugins/codex-security/scripts/workbench/retained_coverage.py @@ -0,0 +1,511 @@ +"""Reconcile retained coverage observations during saved-result recovery.""" + +from __future__ import annotations + +import copy +import hashlib +import json +import os +import re +from collections.abc import Callable, Iterator +from pathlib import Path +from typing import Any + +from finalize_scan_contract import ( + ContractError, + _read_json, + _require_portable_relative_path, + _require_str, + _validate_resolved_deferred, + _validate_schema_node, + finding_candidate_id, + open_scan_local_file_descriptor, +) +from workbench.saved_coverage import _project_worker_coverage_row, _saved_coverage_id + + +def _encoded(value: Any) -> bytes: + return json.dumps( + value, ensure_ascii=True, allow_nan=False, sort_keys=True, separators=(",", ":") + ).encode() + + +def _digest(value: Any) -> str: + return hashlib.sha256(_encoded(value)).hexdigest() + + +def _finding_key(finding: dict[str, Any]) -> str: + # Wording and evidence may improve between checkpoints; distinct source locations + # must not collide merely because two workers chose the same semantic identity. + provenance = finding.get("provenance") + identity = ( + provenance.get("preservedIdentity", finding.get("identity")) + if isinstance(provenance, dict) + else finding.get("identity") + ) + if not isinstance(identity, dict): + normalized = dict(finding) + normalized.pop("identity", None) + _ensure_finding_identity(normalized) + identity = normalized["identity"] + locations = finding.get("locations", []) + if not isinstance(locations, list): + locations = [] + return _digest( + [ + finding.get("ruleId"), + identity, + sorted( + ( + ( + location.get("path"), + location.get("startLine"), + location.get("endLine", location.get("startLine")), + ) + for location in locations + if isinstance(location, dict) + ), + key=_encoded, + ), + ] + ) + + +def _worker_candidate_key( + worker_id: str, candidate_id: str, finding: dict[str, Any] +) -> tuple[str, str, Any, Any, Any]: + """Identify one worker-local candidate without merging unrelated locations.""" + provenance = finding.get("provenance") + identity = ( + provenance.get("preservedIdentity", finding.get("identity")) + if isinstance(provenance, dict) + else finding.get("identity") + ) + if not isinstance(identity, dict): + normalized = dict(finding) + _ensure_finding_identity(normalized) + identity = normalized.get("identity") + anchor = identity.get("anchor") if isinstance(identity, dict) else None + instance = identity.get("instance") if isinstance(identity, dict) else None + return worker_id, candidate_id, finding.get("ruleId"), anchor, instance + + +def _finding_content(finding: dict[str, Any]) -> dict[str, Any]: + """Return substantive finding content without generated identity or provenance.""" + return { + key: value + for key, value in finding.items() + if key not in {"findingId", "occurrenceId", "fingerprints", "identity", "provenance"} + } + + +def _ensure_finding_identity(finding: Any, *, candidate_only: bool = False) -> None: + if not isinstance(finding, dict) or "identity" in finding: + return + if candidate_only and not finding_candidate_id(finding): + return + extensions = finding.get("extensions") + source = str( + (extensions.get("candidateId") if isinstance(extensions, dict) else None) + or finding.get("title") + or "finding" + ) + anchor = re.sub(r"[^a-z0-9._/-]+", "-", source.lower()).strip("._/-") or "finding" + finding["identity"] = {"anchor": anchor} + + +def _retained_findings(finding: dict[str, Any]) -> Iterator[dict[str, Any]]: + """Yield canonical and historical findings without trusting candidate IDs.""" + pending = [finding] + seen: set[int] = set() + while pending: + current = pending.pop() + marker = id(current) + if marker in seen: + continue + seen.add(marker) + yield current + provenance = current.get("provenance") + if not isinstance(provenance, dict): + continue + previous = provenance.get("previousFindings") + if isinstance(previous, list): + pending.extend(item for item in reversed(previous) if isinstance(item, dict)) + sources = provenance.get("sourceFindings") + if isinstance(sources, list): + pending.extend( + source["finding"] + for source in reversed(sources) + if isinstance(source, dict) and isinstance(source.get("finding"), dict) + ) + + +def _append_previous_finding( + item: dict[str, Any], finding: dict[str, Any], *, clone: bool = False +) -> None: + if not isinstance(item.get("previousFindings"), list): + item["previousFindings"] = [] + history = item["previousFindings"] + if not any( + isinstance(previous, dict) + and _finding_key(previous) == _finding_key(finding) + and _finding_content(previous) == _finding_content(finding) + for previous in history + ): + history.append(copy.deepcopy(finding) if clone else finding) + + +def _has_candidate_metadata(row: dict[str, Any]) -> bool: + return any(key in row for key in ("candidateId", "candidate", "finding")) + + +def _coverage_rows(coverage: dict[str, Any], field: str) -> list[Any]: + rows = coverage.get(field, []) + return rows if isinstance(rows, list) else [] + + +def _resolved_deferred_rows(draft: dict[str, Any], schema: dict[str, Any]) -> list[dict[str, Any]]: + if draft.get("complete") is False: + return [] + coverage = draft["coverage"] + rows = coverage.get("resolvedDeferred", []) + try: + # Invalid closure metadata cannot discard the evidence it names. + _validate_schema_node(rows, schema, "coverage.resolvedDeferred") + _validate_resolved_deferred({**coverage, "deferred": _coverage_rows(coverage, "deferred")}) + except ContractError: + return [] + return rows + + +def _merge_tied_parent_observations( + current: dict[str, Any], previous: dict[str, Any] +) -> dict[str, Any]: + merged = copy.deepcopy(current) + for finding in previous["findings"]: + if finding not in merged["findings"]: + merged["findings"].append(copy.deepcopy(finding)) + coverage = merged["coverage"] + for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"): + output = coverage.setdefault(field, []) + if isinstance(output, list): + for row in _coverage_rows(previous["coverage"], field): + if row not in output: + output.append(copy.deepcopy(row)) + pending_ids = { + identity + for row in _coverage_rows(coverage, "deferred") + if isinstance(row, dict) + for identity in (row.get("id"), row.get("candidateId")) + if isinstance(identity, str) + } + closure_schema = _read_json( + Path(__file__).resolve().parents[2] / "schemas" / "coverage.schema.json" + )["properties"]["resolvedDeferred"] + closures = {} + for observed in (current, previous): + for row in _resolved_deferred_rows(observed, closure_schema): + if row["id"] not in pending_ids: + closures.setdefault(row["id"], copy.deepcopy(row)) + coverage.pop("resolvedDeferred", None) + if closures: + coverage["resolvedDeferred"] = list(closures.values()) + if current.get("complete") is False or previous.get("complete") is False: + merged["complete"] = False + coverage.pop("completeness", None) + return merged + + +def _inherit_coverage_sources( + row: Any, + originals: list[Any], + sources: dict[int, tuple[dict[str, Any], list[str]]], +) -> None: + if not isinstance(row, dict): + return + for original in originals: + saved = sources.get(id(original)) + if saved is None: + continue + # Retain the object as well as its ID while copies are reconciled. + paths = sources.setdefault(id(row), (row, []))[1] + for relative in saved[1]: + if relative not in paths: + paths.append(relative) + + +def _generic_surface_updates( + sources: list[tuple[str, dict[str, Any], str | None]], + source_order: dict[str, tuple[int, int]], + closed_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]], + active_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]], + resolved_candidates: dict[tuple[str | None, str], str], + reopened_generic: set[tuple[str | None, str]], + surface_schema: dict[str, Any], + deferred_rows: dict[str, list[Any]], + ambiguous_deferred: set[tuple[str | None, str]], + canonical_rows: set[int], + coverage_sources: dict[int, tuple[dict[str, Any], list[str]]], + *, + surface_is_retained: Callable[[str, str | None, dict[str, Any]], bool], + eligible_receipt_sources: set[str], + receipt_root: Path | None = None, + receipt_warnings: list[str] | None = None, + fallback_surfaces: list[dict[str, Any]] | None = None, +) -> tuple[set[int], list[tuple[str | None, dict[str, Any], set[int]]]]: + if not closed_deferred and not reopened_generic and receipt_root is None: + return set(), [] + + def receipt_error(ref: Any) -> ContractError | None: + if receipt_root is None or not isinstance(ref, str): + return ContractError("Saved coverage receipt is unavailable.") + try: + descriptor = open_scan_local_file_descriptor( + receipt_root, ref, "Saved coverage receipt" + ) + os.close(descriptor) + except ContractError as exc: + return exc + return None + + def linked(row: dict[str, Any], identity: str | None) -> bool: + surface_ids = row.get("surfaceIds", []) + return identity is not None and ( + row.get("id") == identity or (isinstance(surface_ids, list) and identity in surface_ids) + ) + + saved_surfaces: dict[tuple[str | None, str], list[tuple[str, dict[str, Any]]]] = {} + for relative, draft, owner in sources: + for row in _coverage_rows(draft["coverage"], "surfaces"): + if isinstance(row, dict) and isinstance(row.get("id"), str): + saved_surfaces.setdefault((owner, row["id"]), []).append((relative, row)) + replaced: set[int] = set() + updates: list[tuple[str | None, dict[str, Any], set[int]]] = [] + # Current selections across all surfaces take precedence over historical refs. + receipt_paths: dict[str, str] = {} + retained_surfaces: dict[tuple[str | None, str], dict[str, Any]] = {} + if receipt_root is not None: + receipt_surfaces: list[tuple[str | None, dict[str, Any]]] = [] + for key, matches in saved_surfaces.items(): + retained = [ + match for match in matches if surface_is_retained(match[0], key[0], match[1]) + ] + if not retained: + continue + latest_source, latest_surface = max(retained, key=lambda match: source_order[match[0]]) + retained_surfaces[key] = latest_surface + receipt_surfaces.extend( + (key[0], row) + for relative, row in retained + if source_order[relative] == source_order[latest_source] + ) + for relative, draft, owner in sources: + for row in _coverage_rows(draft["coverage"], "surfaces"): + if ( + isinstance(row, dict) + and "id" not in row + and surface_is_retained(relative, owner, row) + ): + receipt_surfaces.append((owner, row)) + receipt_surfaces.extend((None, row) for row in fallback_surfaces or []) + for owner, latest_surface in receipt_surfaces: + # Recovery repairs disposition and receipts before validating row + # metadata. A row discarded there cannot reserve a receipt spelling. + metadata = _project_worker_coverage_row(latest_surface, owner, "surfaces") + metadata.setdefault("id", _saved_coverage_id(metadata)) + metadata.update(disposition="needs_follow_up", receiptRefs=[]) + try: + _require_str(metadata, "id", "coverage.surfaces") + _validate_schema_node(metadata, surface_schema, "coverage.surfaces") + except ContractError: + continue + refs = latest_surface.get("receiptRefs", []) + for ref in refs if isinstance(refs, list) else []: + if receipt_error(ref) is None: + normalized = _require_portable_relative_path(ref, "Saved coverage receipt") + receipt_paths.setdefault(normalized.lower(), normalized) + for relative, draft, owner in sources: + closed_ids = { + identity + for (saved_owner, identity), (_, _, source) in closed_deferred.items() + if saved_owner == owner and source == relative + } + reopened_ids = { + identity + for (saved_owner, identity), (_, _, source) in active_deferred.items() + if saved_owner == owner and source == relative and (owner, identity) in reopened_generic + } + if not closed_ids and not reopened_ids and receipt_root is None: + continue + for surface in _coverage_rows(draft["coverage"], "surfaces"): + if not isinstance(surface, dict): + continue + reopening = surface.get("disposition") == "needs_follow_up" + work_ids = reopened_ids if reopening else closed_ids + repair_only = not work_ids + if repair_only and receipt_root is None: + continue + identity = surface.get("id") + if not isinstance(identity, str): + continue + matches = saved_surfaces[(owner, identity)] + # Receipt repair alone cannot close or reopen a saved surface. + if repair_only and any( + row.get("disposition") != surface.get("disposition") for _, row in matches + ): + continue + # Older writers could assign one ID to distinct surfaces in a draft. + # A closure cannot identify which of those observations it replaces. + by_source = dict(matches) + if any(row != by_source[saved_path] for saved_path, row in matches): + continue + if any(_has_candidate_metadata(row) for _, row in matches): + continue + if any( + row is not surface + and source_order[saved_path] >= source_order[relative] + and row.get("disposition") != surface.get("disposition") + for saved_path, row in matches + ): + continue + + if not reopening and any( + saved_owner == owner + and ( + not isinstance(row.get("id") or row.get("candidateId"), str) + or ( + isinstance(row.get("id"), str) + and (owner, row["id"]) in ambiguous_deferred + and not _has_candidate_metadata(row) + ) + ) + and linked(row, identity) + for saved_path, _, saved_owner in sources + for row in deferred_rows[saved_path] + if isinstance(row, dict) + ): + continue + if not reopening and any( + saved_owner == owner + and (owner, deferred_id) not in closed_deferred + and ( + (candidate_id := _deferred_candidate_id(row, owner, ambiguous_deferred)) is None + or (owner, candidate_id) not in resolved_candidates + ) + and linked(row, identity) + for (saved_owner, deferred_id), (_, row, _) in active_deferred.items() + ): + continue + # An accepted checkpoint can update a saved surface by ID without + # optional surfaceIds links on its generic task. + receipt_matches = [ + (saved_path, row) + for saved_path, row in matches + if receipt_root is None + or saved_path in eligible_receipt_sources + or surface_is_retained(saved_path, owner, row) + ] + if not receipt_matches: + continue + latest_surface = max(receipt_matches, key=lambda match: source_order[match[0]])[1] + target = retained_surfaces.get((owner, identity)) if repair_only else latest_surface + if target is None: + continue + update = copy.deepcopy(target) + # Historical evidence can repair a retained row without promoting + # an unselected observation's label, notes, or other metadata. + refs = copy.deepcopy(latest_surface.get("receiptRefs", [])) + update["receiptRefs"] = refs + replaced_receipt = False + replacement_paths = dict(receipt_paths) + if isinstance(refs, list): + repaired = ( + not reopening and bool(refs) and all(receipt_error(ref) is None for ref in refs) + ) + if repaired: + for ref in refs: + normalized = _require_portable_relative_path(ref, "Saved coverage receipt") + key = normalized.lower() + if key in replacement_paths and replacement_paths[key] != normalized: + repaired = False + break + replacement_paths[key] = normalized + for _, row in receipt_matches: + previous_refs = row.get("receiptRefs", []) + for ref in previous_refs if isinstance(previous_refs, list) else []: + if ref not in refs: + failure = receipt_error(ref) if repaired else None + replaceable = failure is not None and isinstance( + failure.__cause__, FileNotFoundError + ) + if repaired and isinstance(ref, str): + try: + normalized = _require_portable_relative_path( + ref, "Saved coverage receipt" + ) + except ContractError as error: + failure = error + replaceable = True + else: + key = normalized.lower() + previous = replacement_paths.get(key) + if previous is not None and previous != normalized: + failure = ContractError( + f"Saved coverage receipt: duplicate artifact path: {previous} and {normalized}" + ) + replaceable = True + elif not replaceable: + replacement_paths[key] = normalized + if replaceable: + replaced_receipt = True + if receipt_warnings is not None: + receipt_warnings.append( + f"Replaced invalid saved coverage receipt {ref}: {failure}" + ) + continue + refs.append(ref) + if repair_only and not replaced_receipt: + continue + try: + _validate_schema_node(update, surface_schema, "coverage.surfaces") + except ContractError: + continue + replaced_sources = { + id(row) + for _, row in matches + if reopening + or row.get("disposition") in {"needs_follow_up", surface.get("disposition")} + } + replaced.update(replaced_sources) + existing = next( + (entry for entry in updates if entry[0] == owner and entry[1] == update), None + ) + if existing is None: + receipt_paths.update(replacement_paths) + updates.append((owner, update, replaced_sources)) + else: + existing[2].update(replaced_sources) + retained = next( + entry[1] for entry in updates if entry[0] == owner and entry[1] == update + ) + _inherit_coverage_sources( + retained, + [latest_surface, *(row for _, row in matches if id(row) in replaced_sources)], + coverage_sources, + ) + if id(latest_surface) in canonical_rows: + canonical_rows.add(id(retained)) + return replaced, updates + + +def _deferred_candidate_id( + row: dict[str, Any], + owner: str | None, + ambiguous_deferred: set[tuple[str | None, str]], +) -> str | None: + identity = row.get("candidateId") or row.get("id") + if not isinstance(identity, str): + return None + if (owner, identity) in ambiguous_deferred and not _has_candidate_metadata(row): + return None + return identity diff --git a/plugins/codex-security/scripts/workbench/saved_sources.py b/plugins/codex-security/scripts/workbench/saved_sources.py new file mode 100644 index 000000000..585ebdf60 --- /dev/null +++ b/plugins/codex-security/scripts/workbench/saved_sources.py @@ -0,0 +1,169 @@ +"""Read immutable saved-result observations and their frozen ordering.""" + +from __future__ import annotations + +import json +import os +import re +from pathlib import Path +from typing import Any + +from finalize_scan_contract import ( + ContractError, + _read_scan_local_json_with_metadata, + write_scan_local_bytes, +) +from workbench.retained_coverage import _digest, _encoded + + +def _checkpoint_head_directory(relative: str) -> Path | None: + path = Path(relative) + if path.name == "checkpoint-head.json": + return path.parent + if path.parent.name == "checkpoint-heads": + return path.parent.parent + return None + + +def _capture_saved_source( + scan_dir: Path, + relative: str, + scan_id: str, + *, + kind: str | None = None, + snapshot_head: bool = True, + write: bool = True, + created_paths: set[str] | None = None, +) -> dict[str, tuple[str, int]]: + if not snapshot_head or Path(relative).name != "checkpoint-head.json": + _, digest, observed = _read_saved_result(scan_dir, relative, scan_id, kind=kind) + return {relative: (digest, observed)} + head, _, observed = _read_saved_result(scan_dir, relative, scan_id) + observation = {"checkpoint": head["checkpoint"], "observedAtNs": str(observed)} + directory = Path(relative).parent + selected = (directory / "checkpoints" / observation["checkpoint"]).as_posix() + _, selected_digest, selected_time = _read_saved_result(scan_dir, selected, scan_id) + digest = _digest(observation) + snapshot = (directory / "checkpoint-heads" / f"{digest}.json").as_posix() + # Capture the selected file even if the worker created it after directory enumeration. + if write and not (scan_dir / snapshot).exists(): + if created_paths is not None and not os.path.lexists(scan_dir / snapshot): + created_paths.add(snapshot) + write_scan_local_bytes(scan_dir, snapshot, _encoded(observation)) + return { + snapshot: (digest, int(observation["observedAtNs"])), + selected: (selected_digest, selected_time), + } + + +def _is_source_order_snapshot(relative: str) -> bool: + path = Path(relative) + return path.parent == Path("source-order") and bool( + re.fullmatch(r"[0-9a-f]{64}\.json", path.name) + ) + + +def _read_saved_result( + scan_dir: Path, relative: str, scan_id: str, *, kind: str | None = None +) -> tuple[dict[str, Any], str, int]: + draft, _, metadata = _read_scan_local_json_with_metadata( + scan_dir, relative, "Saved scan checkpoint" + ) + directory = _checkpoint_head_directory(relative) + if directory is not None: + checkpoint = draft.get("checkpoint") + if not isinstance(checkpoint, str) or not re.fullmatch(r"[0-9a-f]{64}\.json", checkpoint): + raise ContractError("checkpoint head does not name a saved checkpoint") + _read_saved_result(scan_dir, (directory / "checkpoints" / checkpoint).as_posix(), scan_id) + if Path(relative).name == "checkpoint-head.json": + return draft, _digest([draft, metadata.st_mtime_ns]), metadata.st_mtime_ns + observed = draft.get("observedAtNs") + if not isinstance(observed, str) or not re.fullmatch(r"-?[0-9]+", observed): + raise ContractError("checkpoint head has no observation time") + return draft, _digest(draft), int(observed) + if draft.get("scanId") != scan_id: + raise ContractError("checkpoint belongs to a different scan") + if not _is_source_order_snapshot(relative) and ( + not isinstance(draft.get("findings"), list) + or not isinstance(draft.get("coverage", {} if kind == "dedup" else None), dict) + ): + raise ContractError("checkpoint has no semantic findings or coverage") + return draft, _digest(draft), metadata.st_mtime_ns + + +def _frozen_source_times(scan_dir: Path, scan_id: str, sources: dict[str, str]) -> dict[str, int]: + times: dict[str, int] = {} + snapshots = [path for path in sources if _is_source_order_snapshot(path)] + for path in snapshots: + record, digest, _ = _read_saved_result(scan_dir, path, scan_id) + if digest != sources[path] or not isinstance(record.get("sources"), dict): + raise ContractError("saved source ordering changed after the scan stopped") + for relative, observation in record["sources"].items(): + if ( + not isinstance(observation, dict) + or relative not in sources + or observation.get("digest") != sources[relative] + or not isinstance(observation.get("observedAtNs"), str) + or not re.fullmatch(r"-?[0-9]+", observation["observedAtNs"]) + ): + raise ContractError("saved source ordering does not match its frozen sources") + observed = int(observation["observedAtNs"]) + if relative in times and times[relative] != observed: + raise ContractError("saved source ordering has conflicting observations") + times[relative] = observed + if snapshots and sources.keys() - set(snapshots) - times.keys(): + raise ContractError("saved source ordering is incomplete") + return times + + +def _freeze_source_times( + scan_dir: Path, + scan_id: str, + sources: dict[str, str], + times: dict[str, int], + *, + created_paths: set[str] | None = None, +) -> None: + # Identical result rewrites must not change the order of frozen review evidence. + observations = { + path: {"digest": digest, "observedAtNs": str(times[path])} + for path, digest in sources.items() + if not _is_source_order_snapshot(path) + } + if not observations: + return + record = {"scanId": scan_id, "sources": observations} + digest = _digest(record) + path = f"source-order/{digest}.json" + if not (scan_dir / path).exists(): + if created_paths is not None and not os.path.lexists(scan_dir / path): + created_paths.add(path) + write_scan_local_bytes(scan_dir, path, _encoded(record)) + if _read_saved_result(scan_dir, path, scan_id)[1] != digest: + raise ContractError("saved source ordering does not match its digest") + sources[path] = digest + + +def _source_digests(value: Any, error: str) -> dict[str, str]: + if not isinstance(value, dict) or not all( + isinstance(relative, str) and isinstance(digest, str) for relative, digest in value.items() + ): + raise ContractError(error) + return value + + +def _retained_source_state(value: Any) -> tuple[dict[str, str], str | None]: + if isinstance(value, dict) and isinstance(value.get("sources"), dict): + sources = _source_digests( + value["sources"], "Saved stopped-scan source digests are malformed." + ) + model_source = value.get("threatModelSource") + if not isinstance(model_source, str) or model_source not in sources: + raise ContractError("Saved stopped-scan model source is outside its checkpoint set.") + return sources, model_source + return _source_digests(value, "Saved stopped-scan source digests are malformed."), None + + +def _encode_retained_sources(sources: dict[str, str], model_source: list[str]) -> str: + state = {"sources": sources, "threatModelSource": model_source[0]} if model_source else sources + return json.dumps(state, sort_keys=True) diff --git a/plugins/codex-security/scripts/workbench_saved_results.py b/plugins/codex-security/scripts/workbench_saved_results.py index cd709d050..d9a94ff31 100644 --- a/plugins/codex-security/scripts/workbench_saved_results.py +++ b/plugins/codex-security/scripts/workbench_saved_results.py @@ -12,7 +12,7 @@ import sqlite3 import stat import sys -from collections.abc import Callable, Iterator +from collections.abc import Iterator from contextlib import contextmanager from pathlib import Path from typing import Any @@ -29,15 +29,13 @@ _read_saved_threat_model, _read_scan_local_json, _read_scan_local_json_bytes, - _read_scan_local_json_with_metadata, _recover_unsealed_coverage, _recover_unsealed_findings, _remove_scan_local_file_if_exists, _require_portable_relative_path, _require_safe_json_value, - _require_str, _validate_completion_binding, - _validate_resolved_deferred, + _validate_current_findings, _validate_schema_node, _write_prepared_scan_finalization, finalize_scan, @@ -64,6 +62,80 @@ ) +from workbench.retained_coverage import ( + _append_previous_finding as _append_previous_finding, +) +from workbench.retained_coverage import ( + _coverage_rows as _coverage_rows, +) +from workbench.retained_coverage import ( + _deferred_candidate_id as _deferred_candidate_id, +) +from workbench.retained_coverage import ( + _digest as _digest, +) +from workbench.retained_coverage import ( + _encoded as _encoded, +) +from workbench.retained_coverage import ( + _ensure_finding_identity as _ensure_finding_identity, +) +from workbench.retained_coverage import ( + _finding_content as _finding_content, +) +from workbench.retained_coverage import ( + _finding_key as _finding_key, +) +from workbench.retained_coverage import ( + _generic_surface_updates as _generic_surface_updates, +) +from workbench.retained_coverage import ( + _has_candidate_metadata as _has_candidate_metadata, +) +from workbench.retained_coverage import ( + _inherit_coverage_sources as _inherit_coverage_sources, +) +from workbench.retained_coverage import ( + _merge_tied_parent_observations as _merge_tied_parent_observations, +) +from workbench.retained_coverage import ( + _resolved_deferred_rows as _resolved_deferred_rows, +) +from workbench.retained_coverage import ( + _retained_findings as _retained_findings, +) +from workbench.retained_coverage import ( + _worker_candidate_key as _worker_candidate_key, +) +from workbench.saved_sources import ( + _capture_saved_source as _capture_saved_source, +) +from workbench.saved_sources import ( + _checkpoint_head_directory as _checkpoint_head_directory, +) +from workbench.saved_sources import ( + _encode_retained_sources as _encode_retained_sources, +) +from workbench.saved_sources import ( + _freeze_source_times as _freeze_source_times, +) +from workbench.saved_sources import ( + _frozen_source_times as _frozen_source_times, +) +from workbench.saved_sources import ( + _is_source_order_snapshot as _is_source_order_snapshot, +) +from workbench.saved_sources import ( + _read_saved_result as _read_saved_result, +) +from workbench.saved_sources import ( + _retained_source_state as _retained_source_state, +) +from workbench.saved_sources import ( + _source_digests as _source_digests, +) + + def threat_model_fields(db: Any, scan: sqlite3.Row) -> dict[str, Any]: scan_dir = Path(scan["scan_dir"]) fields: dict[str, Any] = {"threatModelAvailable": False} @@ -119,16 +191,6 @@ def refresh_completed_scan( return db.scan_context(connection, scan["id"]) -def _encoded(value: Any) -> bytes: - return json.dumps( - value, ensure_ascii=True, allow_nan=False, sort_keys=True, separators=(",", ":") - ).encode() - - -def _digest(value: Any) -> str: - return hashlib.sha256(_encoded(value)).hexdigest() - - def _children(scan_dir: Path, relative: str) -> list[str]: cursor = scan_dir for part in Path(relative).parts: @@ -215,134 +277,6 @@ def _saved_result_paths(scan_dir: Path, workers: list[Any]) -> Iterator[tuple[st continue -def _checkpoint_head_directory(relative: str) -> Path | None: - path = Path(relative) - if path.name == "checkpoint-head.json": - return path.parent - if path.parent.name == "checkpoint-heads": - return path.parent.parent - return None - - -def _capture_saved_source( - scan_dir: Path, - relative: str, - scan_id: str, - *, - kind: str | None = None, - snapshot_head: bool = True, - write: bool = True, - created_paths: set[str] | None = None, -) -> dict[str, tuple[str, int]]: - if not snapshot_head or Path(relative).name != "checkpoint-head.json": - _, digest, observed = _read_saved_result(scan_dir, relative, scan_id, kind=kind) - return {relative: (digest, observed)} - head, _, observed = _read_saved_result(scan_dir, relative, scan_id) - observation = {"checkpoint": head["checkpoint"], "observedAtNs": str(observed)} - directory = Path(relative).parent - selected = (directory / "checkpoints" / observation["checkpoint"]).as_posix() - _, selected_digest, selected_time = _read_saved_result(scan_dir, selected, scan_id) - digest = _digest(observation) - snapshot = (directory / "checkpoint-heads" / f"{digest}.json").as_posix() - # Capture the selected file even if the worker created it after directory enumeration. - if write and not (scan_dir / snapshot).exists(): - if created_paths is not None and not os.path.lexists(scan_dir / snapshot): - created_paths.add(snapshot) - write_scan_local_bytes(scan_dir, snapshot, _encoded(observation)) - return { - snapshot: (digest, int(observation["observedAtNs"])), - selected: (selected_digest, selected_time), - } - - -def _is_source_order_snapshot(relative: str) -> bool: - path = Path(relative) - return path.parent == Path("source-order") and bool( - re.fullmatch(r"[0-9a-f]{64}\.json", path.name) - ) - - -def _read_saved_result( - scan_dir: Path, relative: str, scan_id: str, *, kind: str | None = None -) -> tuple[dict[str, Any], str, int]: - draft, _, metadata = _read_scan_local_json_with_metadata( - scan_dir, relative, "Saved scan checkpoint" - ) - directory = _checkpoint_head_directory(relative) - if directory is not None: - checkpoint = draft.get("checkpoint") - if not isinstance(checkpoint, str) or not re.fullmatch(r"[0-9a-f]{64}\.json", checkpoint): - raise ContractError("checkpoint head does not name a saved checkpoint") - _read_saved_result(scan_dir, (directory / "checkpoints" / checkpoint).as_posix(), scan_id) - if Path(relative).name == "checkpoint-head.json": - return draft, _digest([draft, metadata.st_mtime_ns]), metadata.st_mtime_ns - observed = draft.get("observedAtNs") - if not isinstance(observed, str) or not re.fullmatch(r"-?[0-9]+", observed): - raise ContractError("checkpoint head has no observation time") - return draft, _digest(draft), int(observed) - if draft.get("scanId") != scan_id: - raise ContractError("checkpoint belongs to a different scan") - if not _is_source_order_snapshot(relative) and ( - not isinstance(draft.get("findings"), list) - or not isinstance(draft.get("coverage", {} if kind == "dedup" else None), dict) - ): - raise ContractError("checkpoint has no semantic findings or coverage") - return draft, _digest(draft), metadata.st_mtime_ns - - -def _frozen_source_times(scan_dir: Path, scan_id: str, sources: dict[str, str]) -> dict[str, int]: - times: dict[str, int] = {} - snapshots = [path for path in sources if _is_source_order_snapshot(path)] - for path in snapshots: - record, digest, _ = _read_saved_result(scan_dir, path, scan_id) - if digest != sources[path] or not isinstance(record.get("sources"), dict): - raise ContractError("saved source ordering changed after the scan stopped") - for relative, observation in record["sources"].items(): - if ( - not isinstance(observation, dict) - or relative not in sources - or observation.get("digest") != sources[relative] - or not isinstance(observation.get("observedAtNs"), str) - or not re.fullmatch(r"-?[0-9]+", observation["observedAtNs"]) - ): - raise ContractError("saved source ordering does not match its frozen sources") - observed = int(observation["observedAtNs"]) - if relative in times and times[relative] != observed: - raise ContractError("saved source ordering has conflicting observations") - times[relative] = observed - if snapshots and sources.keys() - set(snapshots) - times.keys(): - raise ContractError("saved source ordering is incomplete") - return times - - -def _freeze_source_times( - scan_dir: Path, - scan_id: str, - sources: dict[str, str], - times: dict[str, int], - *, - created_paths: set[str] | None = None, -) -> None: - # Identical result rewrites must not change the order of frozen review evidence. - observations = { - path: {"digest": digest, "observedAtNs": str(times[path])} - for path, digest in sources.items() - if not _is_source_order_snapshot(path) - } - if not observations: - return - record = {"scanId": scan_id, "sources": observations} - digest = _digest(record) - path = f"source-order/{digest}.json" - if not (scan_dir / path).exists(): - if created_paths is not None and not os.path.lexists(scan_dir / path): - created_paths.add(path) - write_scan_local_bytes(scan_dir, path, _encoded(record)) - if _read_saved_result(scan_dir, path, scan_id)[1] != digest: - raise ContractError("saved source ordering does not match its digest") - sources[path] = digest - - def _parent_scan_draft( scan_id: str, parent_scan: dict[str, Any], @@ -382,31 +316,6 @@ def _read_saved_parent_result( return manifest, _parent_scan_draft(scan_id, parent_scan, findings, coverage) -def _source_digests(value: Any, error: str) -> dict[str, str]: - if not isinstance(value, dict) or not all( - isinstance(relative, str) and isinstance(digest, str) for relative, digest in value.items() - ): - raise ContractError(error) - return value - - -def _retained_source_state(value: Any) -> tuple[dict[str, str], str | None]: - if isinstance(value, dict) and isinstance(value.get("sources"), dict): - sources = _source_digests( - value["sources"], "Saved stopped-scan source digests are malformed." - ) - model_source = value.get("threatModelSource") - if not isinstance(model_source, str) or model_source not in sources: - raise ContractError("Saved stopped-scan model source is outside its checkpoint set.") - return sources, model_source - return _source_digests(value, "Saved stopped-scan source digests are malformed."), None - - -def _encode_retained_sources(sources: dict[str, str], model_source: list[str]) -> str: - state = {"sources": sources, "threatModelSource": model_source[0]} if model_source else sources - return json.dumps(state, sort_keys=True) - - def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool: try: scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"])) @@ -562,453 +471,6 @@ def scan_results_recovery_needed(db: Any, connection: Any, scan: Any) -> bool: return _saved_results_changed(db, connection, scan) -def _finding_key(finding: dict[str, Any]) -> str: - # Wording and evidence may improve between checkpoints; distinct source locations - # must not collide merely because two workers chose the same semantic identity. - provenance = finding.get("provenance") - identity = ( - provenance.get("preservedIdentity", finding.get("identity")) - if isinstance(provenance, dict) - else finding.get("identity") - ) - if not isinstance(identity, dict): - normalized = dict(finding) - normalized.pop("identity", None) - _ensure_finding_identity(normalized) - identity = normalized["identity"] - locations = finding.get("locations", []) - if not isinstance(locations, list): - locations = [] - return _digest( - [ - finding.get("ruleId"), - identity, - sorted( - ( - ( - location.get("path"), - location.get("startLine"), - location.get("endLine", location.get("startLine")), - ) - for location in locations - if isinstance(location, dict) - ), - key=_encoded, - ), - ] - ) - - -def _worker_candidate_key( - worker_id: str, candidate_id: str, finding: dict[str, Any] -) -> tuple[str, str, Any, Any, Any]: - """Identify one worker-local candidate without merging unrelated locations.""" - provenance = finding.get("provenance") - identity = ( - provenance.get("preservedIdentity", finding.get("identity")) - if isinstance(provenance, dict) - else finding.get("identity") - ) - if not isinstance(identity, dict): - normalized = dict(finding) - _ensure_finding_identity(normalized) - identity = normalized.get("identity") - anchor = identity.get("anchor") if isinstance(identity, dict) else None - instance = identity.get("instance") if isinstance(identity, dict) else None - return worker_id, candidate_id, finding.get("ruleId"), anchor, instance - - -def _finding_content(finding: dict[str, Any]) -> dict[str, Any]: - """Return substantive finding content without generated identity or provenance.""" - return { - key: value - for key, value in finding.items() - if key not in {"findingId", "occurrenceId", "fingerprints", "identity", "provenance"} - } - - -def _ensure_finding_identity(finding: Any, *, candidate_only: bool = False) -> None: - if not isinstance(finding, dict) or "identity" in finding: - return - if candidate_only and not finding_candidate_id(finding): - return - extensions = finding.get("extensions") - source = str( - (extensions.get("candidateId") if isinstance(extensions, dict) else None) - or finding.get("title") - or "finding" - ) - anchor = re.sub(r"[^a-z0-9._/-]+", "-", source.lower()).strip("._/-") or "finding" - finding["identity"] = {"anchor": anchor} - - -def _retained_findings(finding: dict[str, Any]) -> Iterator[dict[str, Any]]: - """Yield canonical and historical findings without trusting candidate IDs.""" - pending = [finding] - seen: set[int] = set() - while pending: - current = pending.pop() - marker = id(current) - if marker in seen: - continue - seen.add(marker) - yield current - provenance = current.get("provenance") - if not isinstance(provenance, dict): - continue - previous = provenance.get("previousFindings") - if isinstance(previous, list): - pending.extend(item for item in reversed(previous) if isinstance(item, dict)) - sources = provenance.get("sourceFindings") - if isinstance(sources, list): - pending.extend( - source["finding"] - for source in reversed(sources) - if isinstance(source, dict) and isinstance(source.get("finding"), dict) - ) - - -def _append_previous_finding( - item: dict[str, Any], finding: dict[str, Any], *, clone: bool = False -) -> None: - if not isinstance(item.get("previousFindings"), list): - item["previousFindings"] = [] - history = item["previousFindings"] - if not any( - isinstance(previous, dict) - and _finding_key(previous) == _finding_key(finding) - and _finding_content(previous) == _finding_content(finding) - for previous in history - ): - history.append(copy.deepcopy(finding) if clone else finding) - - -def _has_candidate_metadata(row: dict[str, Any]) -> bool: - return any(key in row for key in ("candidateId", "candidate", "finding")) - - -def _coverage_rows(coverage: dict[str, Any], field: str) -> list[Any]: - rows = coverage.get(field, []) - return rows if isinstance(rows, list) else [] - - -def _resolved_deferred_rows(draft: dict[str, Any], schema: dict[str, Any]) -> list[dict[str, Any]]: - if draft.get("complete") is False: - return [] - coverage = draft["coverage"] - rows = coverage.get("resolvedDeferred", []) - try: - # Invalid closure metadata cannot discard the evidence it names. - _validate_schema_node(rows, schema, "coverage.resolvedDeferred") - _validate_resolved_deferred({**coverage, "deferred": _coverage_rows(coverage, "deferred")}) - except ContractError: - return [] - return rows - - -def _merge_tied_parent_observations( - current: dict[str, Any], previous: dict[str, Any] -) -> dict[str, Any]: - merged = copy.deepcopy(current) - for finding in previous["findings"]: - if finding not in merged["findings"]: - merged["findings"].append(copy.deepcopy(finding)) - coverage = merged["coverage"] - for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"): - output = coverage.setdefault(field, []) - if isinstance(output, list): - for row in _coverage_rows(previous["coverage"], field): - if row not in output: - output.append(copy.deepcopy(row)) - pending_ids = { - identity - for row in _coverage_rows(coverage, "deferred") - if isinstance(row, dict) - for identity in (row.get("id"), row.get("candidateId")) - if isinstance(identity, str) - } - closure_schema = _read_json( - Path(__file__).resolve().parent.parent / "schemas" / "coverage.schema.json" - )["properties"]["resolvedDeferred"] - closures = {} - for observed in (current, previous): - for row in _resolved_deferred_rows(observed, closure_schema): - if row["id"] not in pending_ids: - closures.setdefault(row["id"], copy.deepcopy(row)) - coverage.pop("resolvedDeferred", None) - if closures: - coverage["resolvedDeferred"] = list(closures.values()) - if current.get("complete") is False or previous.get("complete") is False: - merged["complete"] = False - coverage.pop("completeness", None) - return merged - - -def _deferred_candidate_id( - row: dict[str, Any], - owner: str | None, - ambiguous_deferred: set[tuple[str | None, str]], -) -> str | None: - identity = row.get("candidateId") or row.get("id") - if not isinstance(identity, str): - return None - if (owner, identity) in ambiguous_deferred and not _has_candidate_metadata(row): - return None - return identity - - -def _generic_surface_updates( - sources: list[tuple[str, dict[str, Any], str | None]], - source_order: dict[str, tuple[int, int]], - closed_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]], - active_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]], - resolved_candidates: dict[tuple[str | None, str], str], - reopened_generic: set[tuple[str | None, str]], - surface_schema: dict[str, Any], - deferred_rows: dict[str, list[Any]], - ambiguous_deferred: set[tuple[str | None, str]], - *, - surface_is_retained: Callable[[str, str | None, dict[str, Any]], bool], - eligible_receipt_sources: set[str], - receipt_root: Path | None = None, - receipt_warnings: list[str] | None = None, - fallback_surfaces: list[dict[str, Any]] | None = None, -) -> tuple[set[int], list[tuple[str | None, dict[str, Any], set[int]]]]: - if not closed_deferred and not reopened_generic and receipt_root is None: - return set(), [] - - def receipt_error(ref: Any) -> ContractError | None: - if receipt_root is None or not isinstance(ref, str): - return ContractError("Saved coverage receipt is unavailable.") - try: - descriptor = open_scan_local_file_descriptor( - receipt_root, ref, "Saved coverage receipt" - ) - os.close(descriptor) - except ContractError as exc: - return exc - return None - - def linked(row: dict[str, Any], identity: str | None) -> bool: - surface_ids = row.get("surfaceIds", []) - return identity is not None and ( - row.get("id") == identity or (isinstance(surface_ids, list) and identity in surface_ids) - ) - - saved_surfaces: dict[tuple[str | None, str], list[tuple[str, dict[str, Any]]]] = {} - for relative, draft, owner in sources: - for row in _coverage_rows(draft["coverage"], "surfaces"): - if isinstance(row, dict) and isinstance(row.get("id"), str): - saved_surfaces.setdefault((owner, row["id"]), []).append((relative, row)) - replaced: set[int] = set() - updates: list[tuple[str | None, dict[str, Any], set[int]]] = [] - # Current selections across all surfaces take precedence over historical refs. - receipt_paths: dict[str, str] = {} - retained_surfaces: dict[tuple[str | None, str], dict[str, Any]] = {} - if receipt_root is not None: - receipt_surfaces: list[tuple[str | None, dict[str, Any]]] = [] - for key, matches in saved_surfaces.items(): - retained = [ - match for match in matches if surface_is_retained(match[0], key[0], match[1]) - ] - if not retained: - continue - latest_source, latest_surface = max(retained, key=lambda match: source_order[match[0]]) - retained_surfaces[key] = latest_surface - receipt_surfaces.extend( - (key[0], row) - for relative, row in retained - if source_order[relative] == source_order[latest_source] - ) - for relative, draft, owner in sources: - for row in _coverage_rows(draft["coverage"], "surfaces"): - if ( - isinstance(row, dict) - and "id" not in row - and surface_is_retained(relative, owner, row) - ): - receipt_surfaces.append((owner, row)) - receipt_surfaces.extend((None, row) for row in fallback_surfaces or []) - for owner, latest_surface in receipt_surfaces: - # Recovery repairs disposition and receipts before validating row - # metadata. A row discarded there cannot reserve a receipt spelling. - metadata = _project_worker_coverage_row(latest_surface, owner, "surfaces") - metadata.setdefault("id", _saved_coverage_id(metadata)) - metadata.update(disposition="needs_follow_up", receiptRefs=[]) - try: - _require_str(metadata, "id", "coverage.surfaces") - _validate_schema_node(metadata, surface_schema, "coverage.surfaces") - except ContractError: - continue - refs = latest_surface.get("receiptRefs", []) - for ref in refs if isinstance(refs, list) else []: - if receipt_error(ref) is None: - normalized = _require_portable_relative_path(ref, "Saved coverage receipt") - receipt_paths.setdefault(normalized.lower(), normalized) - for relative, draft, owner in sources: - closed_ids = { - identity - for (saved_owner, identity), (_, _, source) in closed_deferred.items() - if saved_owner == owner and source == relative - } - reopened_ids = { - identity - for (saved_owner, identity), (_, _, source) in active_deferred.items() - if saved_owner == owner and source == relative and (owner, identity) in reopened_generic - } - if not closed_ids and not reopened_ids and receipt_root is None: - continue - for surface in _coverage_rows(draft["coverage"], "surfaces"): - if not isinstance(surface, dict): - continue - reopening = surface.get("disposition") == "needs_follow_up" - work_ids = reopened_ids if reopening else closed_ids - repair_only = not work_ids - if repair_only and receipt_root is None: - continue - identity = surface.get("id") - if not isinstance(identity, str): - continue - matches = saved_surfaces[(owner, identity)] - # Receipt repair alone cannot close or reopen a saved surface. - if repair_only and any( - row.get("disposition") != surface.get("disposition") for _, row in matches - ): - continue - # Older writers could assign one ID to distinct surfaces in a draft. - # A closure cannot identify which of those observations it replaces. - by_source = dict(matches) - if any(row != by_source[saved_path] for saved_path, row in matches): - continue - if any(_has_candidate_metadata(row) for _, row in matches): - continue - if any( - row is not surface - and source_order[saved_path] >= source_order[relative] - and row.get("disposition") != surface.get("disposition") - for saved_path, row in matches - ): - continue - - if not reopening and any( - saved_owner == owner - and ( - not isinstance(row.get("id") or row.get("candidateId"), str) - or ( - isinstance(row.get("id"), str) - and (owner, row["id"]) in ambiguous_deferred - and not _has_candidate_metadata(row) - ) - ) - and linked(row, identity) - for saved_path, _, saved_owner in sources - for row in deferred_rows[saved_path] - if isinstance(row, dict) - ): - continue - if not reopening and any( - saved_owner == owner - and (owner, deferred_id) not in closed_deferred - and ( - (candidate_id := _deferred_candidate_id(row, owner, ambiguous_deferred)) is None - or (owner, candidate_id) not in resolved_candidates - ) - and linked(row, identity) - for (saved_owner, deferred_id), (_, row, _) in active_deferred.items() - ): - continue - # An accepted checkpoint can update a saved surface by ID without - # optional surfaceIds links on its generic task. - receipt_matches = [ - (saved_path, row) - for saved_path, row in matches - if receipt_root is None - or saved_path in eligible_receipt_sources - or surface_is_retained(saved_path, owner, row) - ] - if not receipt_matches: - continue - latest_surface = max(receipt_matches, key=lambda match: source_order[match[0]])[1] - target = retained_surfaces.get((owner, identity)) if repair_only else latest_surface - if target is None: - continue - update = copy.deepcopy(target) - # Historical evidence can repair a retained row without promoting - # an unselected observation's label, notes, or other metadata. - refs = copy.deepcopy(latest_surface.get("receiptRefs", [])) - update["receiptRefs"] = refs - replaced_receipt = False - replacement_paths = dict(receipt_paths) - if isinstance(refs, list): - repaired = ( - not reopening and bool(refs) and all(receipt_error(ref) is None for ref in refs) - ) - if repaired: - for ref in refs: - normalized = _require_portable_relative_path(ref, "Saved coverage receipt") - key = normalized.lower() - if key in replacement_paths and replacement_paths[key] != normalized: - repaired = False - break - replacement_paths[key] = normalized - for _, row in receipt_matches: - previous_refs = row.get("receiptRefs", []) - for ref in previous_refs if isinstance(previous_refs, list) else []: - if ref not in refs: - failure = receipt_error(ref) if repaired else None - replaceable = failure is not None and isinstance( - failure.__cause__, FileNotFoundError - ) - if repaired and isinstance(ref, str): - try: - normalized = _require_portable_relative_path( - ref, "Saved coverage receipt" - ) - except ContractError as error: - failure = error - replaceable = True - else: - key = normalized.lower() - previous = replacement_paths.get(key) - if previous is not None and previous != normalized: - failure = ContractError( - f"Saved coverage receipt: duplicate artifact path: {previous} and {normalized}" - ) - replaceable = True - elif not replaceable: - replacement_paths[key] = normalized - if replaceable: - replaced_receipt = True - if receipt_warnings is not None: - receipt_warnings.append( - f"Replaced invalid saved coverage receipt {ref}: {failure}" - ) - continue - refs.append(ref) - if repair_only and not replaced_receipt: - continue - try: - _validate_schema_node(update, surface_schema, "coverage.surfaces") - except ContractError: - continue - replaced_sources = { - id(row) - for _, row in matches - if reopening - or row.get("disposition") in {"needs_follow_up", surface.get("disposition")} - } - replaced.update(replaced_sources) - existing = next( - (entry for entry in updates if entry[0] == owner and entry[1] == update), None - ) - if existing is None: - receipt_paths.update(replacement_paths) - updates.append((owner, update, replaced_sources)) - else: - existing[2].update(replaced_sources) - return replaced, updates - - @contextmanager def preserve_parent_head_on_error(scan_dir: Path) -> Iterator[None]: """Keep rejected completion attempts from becoming accepted parent observations.""" @@ -1061,6 +523,7 @@ def merge_saved_results( allow_frozen_legacy_parent: bool = False, frozen_model_source: str | None = None, selected_model_source: list[str] | None = None, + preserve_canonical_parent: bool = False, project_worker_coverage: bool = False, created_paths: set[str] | None = None, current_contract: bool = False, @@ -1088,7 +551,7 @@ def merge_saved_results( warnings.append(f"Could not read the saved parent draft: {exc}") parent_manifest = None parent = None - if parent_manifest is not None and parent is not None: + if parent_manifest is not None and parent is not None and not preserve_canonical_parent: parent_scan = parent_manifest["scan"] try: previous_head, _, head_modified = _read_saved_result( @@ -1177,8 +640,9 @@ def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None for saved_directory in (directory, head_snapshots): paths.update(dict.fromkeys(_checkpoint_paths(scan_dir, saved_directory), worker_id)) - paths["checkpoint-head.json"] = None - checkpoints("checkpoints", None) + if not preserve_canonical_parent: + paths["checkpoint-head.json"] = None + checkpoints("checkpoints", None) for worker in workers: try: outputs = _worker_outputs(scan_dir, worker) @@ -1244,6 +708,7 @@ def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None if latest_reducer not in frozen_source_digests: latest_reducer = None + discarded_by_source: dict[str, list[str]] = {} for relative, worker_id in paths.items(): try: draft, digest, observed = _read_saved_result( @@ -1257,6 +722,91 @@ def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None if _checkpoint_head_directory(relative) is not None: saved_heads[relative] = draft["checkpoint"] continue + if preserve_canonical_parent and worker_id is not None: + # Recover imported records without relaxing canonical parent validation. + if "complete" in draft and not isinstance(draft["complete"], bool): + draft["complete"] = False + warnings.append( + f"{relative}: Invalid complete marker; retained as a progress checkpoint." + ) + schema_dir = Path(__file__).resolve().parent.parent / "schemas" + if "threatModel" in draft: + try: + model = draft["threatModel"] + if isinstance(model, dict): + model = {**model, "origin": "recovered"} + schema = _read_json(schema_dir / "scan-manifest.schema.json") + _validate_schema_node( + model, + schema["properties"]["scan"]["properties"]["threatModel"], + "threatModel", + schema, + ) + _require_safe_json_value(model, "threatModel") + except ContractError as exc: + draft.pop("threatModel") + warnings.append(f"{relative}: Skipped malformed threat model: {exc}.") + directory = Path(relative).parent + if directory.name == "checkpoints": + directory = directory.parent + recovered_findings = [] + for index, finding in enumerate(draft["findings"]): + _ensure_finding_identity(finding) + recovered = {"scanId": scan_id, "findings": [finding]} + source_warnings: list[str] = [] + discarded = _recover_unsealed_findings( + {"scan": {"id": scan_id, "target": binding["target"]}}, + recovered, + schema_dir, + scan_dir / directory, + source_warnings, + ) + if recovered["findings"]: + try: + _validate_current_findings(recovered, schema_dir) + except ContractError as exc: + warning = f"Skipped malformed finding: {exc}." + source_warnings.append(warning) + discarded.append(warning) + recovered["findings"] = [] + if discarded: + discarded_by_source.setdefault(relative, []).extend(discarded) + for recovered_finding in recovered["findings"]: + writeup = recovered_finding.pop("writeup", None) + if writeup is not None: + source_warnings.append( + "Retained worker writeup at " + f"{directory.as_posix()}/{writeup['reportPath']}; " + "the parent report renders this finding inline." + ) + recovered_findings.extend(recovered["findings"]) + warnings.extend( + f"{relative} finding {index + 1}: {warning}" for warning in source_warnings + ) + draft["findings"] = recovered_findings + for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"): + rows = draft["coverage"].get(field) + if not isinstance(rows, list): + continue + for index, row in enumerate(rows): + if field == "openQuestions" and isinstance(row, str): + row = rows[index] = {"question": row} + if not isinstance(row, dict): + continue + row["workerId"] = worker_id + refs = row.get("receiptRefs") + if not isinstance(refs, list): + continue + for ref_index, ref in enumerate(refs): + if not isinstance(ref, str): + continue + try: + normalized = _require_portable_relative_path(ref, "receiptRef") + except ContractError: + # Keep malformed references for the finalizer's diagnostics. + continue + if normalized.startswith("artifacts/"): + refs[ref_index] = f"{directory.as_posix()}/{normalized}" # Recovery expects coverage, but reducer results only contain findings # and context. Add an empty value after hashing the original result. sources.append((relative, {"coverage": {}, **draft}, worker_id)) @@ -1335,9 +885,12 @@ def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None if parent is None and latest_reducer is not None: parent = drafts_by_path[latest_reducer] - project_worker_coverage, surface_links = _prepare_worker_coverage_recovery( - sources, parent, source_order, enabled=project_worker_coverage - ) + if preserve_canonical_parent: + project_worker_coverage, surface_links = False, [] + else: + project_worker_coverage, surface_links = _prepare_worker_coverage_recovery( + sources, parent, source_order, enabled=project_worker_coverage + ) # Keep the actual retained row even when recovery adds previous findings. linked_surface_sources = {id(surface) for _, _, surface, _, _ in surface_links} linked_surface_outputs: dict[int, dict[str, Any]] = {} @@ -1360,6 +913,12 @@ def append_coverage_row(output: list[Any], item: Any, owner: str | None, field: output.append(retained) retained = output[output.index(retained)] record_coverage_owner(retained, item, owner, field) + _inherit_coverage_sources(retained, [item], coverage_sources) + if preserve_canonical_parent: + if id(item) in canonical_rows: + canonical_rows.add(id(retained)) + if field == "surfaces": + retained_surface_rows.add(id(item)) return retained def linked_tasks( @@ -1439,6 +998,50 @@ def linked_surface(surface, outputs, named_outputs): row["id"] = identity reserved.add(identity) + # A discarded local surface must not fall back to another source's reused ID. + surface_ids_by_source: dict[tuple[str, str], list[str]] = { + (relative, row["id"]): [] + for relative, draft, _ in all_sources + for row in ( + draft["coverage"].get("surfaces", []) + if isinstance(draft["coverage"].get("surfaces"), list) + else [] + ) + if isinstance(row, dict) and isinstance(row.get("id"), str) + } + discarded_coverage_sources: set[str] = set() + if preserve_canonical_parent: + for relative, draft, worker_id in sources: + if worker_id is None: + continue + for field in ("surfaces", "explicitExclusions", "deferred"): + rows = draft["coverage"].get(field) + for row in rows if isinstance(rows, list) else []: + if isinstance(row, dict): + row.setdefault("id", _saved_coverage_id(row)) + if field == "surfaces": + row.setdefault("receiptRefs", []) + source_warnings = [] + discarded_fields = _recover_unsealed_coverage( + draft["coverage"], + Path(__file__).resolve().parent.parent / "schemas", + scan_dir, + source_warnings, + discarded_by_source.get(relative, []), + # Reconcile local IDs and valid outcomes against worker history below. + worker_draft=True, + ) + if discarded_fields & { + "surfaces", + "explicitExclusions", + "deferred", + "resolvedDeferred", + }: + discarded_coverage_sources.add(relative) + for row in draft["coverage"]["deferred"]: + row["workerId"] = worker_id + warnings.extend(f"{relative}: {warning}" for warning in source_warnings) + if parent is None and not sources: return None if ( @@ -1557,17 +1160,43 @@ def recovered_model( {"question": item.strip()} if isinstance(item, str) else item for item in coverage["openQuestions"] ] + # Closure reconciliation can reuse original parent rows instead of their copies. canonical_rows = ( { id(item) + for parent_coverage in ( + (coverage, parent["coverage"]) if preserve_canonical_parent else (coverage,) + ) for field in ("surfaces", "explicitExclusions", "deferred") - for item in _coverage_rows(coverage, field) + for item in _coverage_rows(parent_coverage, field) } if parent_is_canonical else set() ) + # Links use source-local IDs until all retained surfaces receive their final IDs. + coverage_sources: dict[int, tuple[dict[str, Any], list[str]]] = {} + source_owners = {relative: owner for relative, _, owner in all_sources} + retained_surface_rows: set[int] = set() + for relative, draft, _ in all_sources: + for field in ("surfaces", "deferred"): + rows = draft["coverage"].get(field) + for row in rows if isinstance(rows, list) else []: + if isinstance(row, dict): + paths_for_row = coverage_sources.setdefault(id(row), (row, []))[1] + if relative not in paths_for_row: + paths_for_row.append(relative) + if parent: + for field in ("surfaces", "deferred"): + rows, originals = coverage.get(field), parent["coverage"].get(field) + if isinstance(rows, list) and isinstance(originals, list): + for row, original in zip(rows, originals): + _inherit_coverage_sources(row, [original], coverage_sources) + if field == "surfaces": + retained_surface_rows.update(id(original) for original in originals) + findings: list[dict[str, Any]] = [] finding_positions: dict[str, int] = {} + canonical_finding_keys: set[str] = set() represented: dict[str, str | None] = {} represented_candidates: dict[tuple[str, str, Any, Any, Any], str | None] = {} represented_history: dict[str, set[str]] = {} @@ -1598,6 +1227,20 @@ def valid_finding(value: Any) -> bool: relative: _coverage_rows(draft["coverage"], "deferred") for relative, draft, _ in all_sources } + # Accepted heads can replace equal tasks before they reach output coalescing. + equivalent_tasks: dict[str | None, list[dict[str, Any]]] = {} + for relative, _, owner in all_sources: + known = equivalent_tasks.setdefault(owner, []) + for row in deferred_rows[relative]: + if not isinstance(row, dict): + continue + for previous in known: + if previous == row: + _inherit_coverage_sources(previous, [row], coverage_sources) + coverage_sources[id(row)] = (row, coverage_sources[id(previous)][1]) + break + else: + known.append(row) # A legacy source can contain independent tasks with the same explicit ID. # Later rewrites cannot make an ID-only closure identify one of those tasks. ambiguous_deferred: set[tuple[str | None, str]] = set() @@ -1808,23 +1451,34 @@ def valid_finding(value: Any) -> bool: worker_id is not None and relative in selected_observations and draft.get("complete") is not False + and relative not in discarded_by_source + and relative not in discarded_coverage_sources ): order = source_order[relative] selected_terminal_orders[worker_id] = max( selected_terminal_orders.get(worker_id, order), order ) terminal_worker_orders: dict[str | None, tuple[int, int]] = {} + terminal_coverage_orders: dict[str | None, tuple[int, int]] = {} for relative, draft, worker_id in sources: - if relative in current_results and draft.get("complete") is not False: + if ( + relative in current_results + and draft.get("complete") is not False + and relative not in discarded_by_source + ): order = source_order[relative] terminal_worker_orders[worker_id] = max( terminal_worker_orders.get(worker_id, order), order ) + if relative not in discarded_coverage_sources: + terminal_coverage_orders[worker_id] = max( + terminal_coverage_orders.get(worker_id, order), order + ) def source_coverage_selection( relative: str, worker_id: str | None, - ) -> tuple[bool, set[str], bool, bool]: + ) -> tuple[bool, set[str], bool, bool, bool]: worker_result_order = terminal_worker_orders.get(worker_id) selected_coverage_superseded = worker_id in selected_terminal_orders and ( source_order[relative] < selected_terminal_orders[worker_id] @@ -1853,28 +1507,53 @@ def source_coverage_selection( or source_order[relative] < worker_result_order ) ) + worker_coverage_order = terminal_coverage_orders.get(worker_id) + coverage_superseded = ( + superseded + if worker_id is None + else ( + relative not in current_results + and worker_coverage_order is not None + and ( + relative not in selected_observations + or source_order[relative] < worker_coverage_order + ) + ) + ) # A failed result write can leave pending work outside the accepted result. accepted_order = ( (0, parent_modified) if worker_id is None and parent is not None - else worker_result_order + else worker_coverage_order ) retain_pending = ( - superseded + coverage_superseded and (worker_id in headed_workers or (worker_id is None and parent_heads)) and accepted_order is not None and source_order[relative] >= accepted_order and any(isinstance(row, dict) for row in deferred_rows[relative]) ) - return selected_coverage_superseded, selected_candidates, superseded, retain_pending + return ( + selected_coverage_superseded, + selected_candidates, + superseded, + retain_pending, + coverage_superseded, + ) def coverage_row_is_retained( field: str, item: Any, worker_id: str | None, - selection: tuple[bool, set[str], bool, bool], + selection: tuple[bool, set[str], bool, bool, bool], ) -> bool: - selected_coverage_superseded, selected_candidates, superseded, retain_pending = selection + ( + selected_coverage_superseded, + selected_candidates, + superseded, + retain_pending, + coverage_superseded, + ) = selection # A selected terminal checkpoint replaces ordinary coverage, while # candidate evidence and pending tasks retain their own reconciliation. if ( @@ -1884,7 +1563,7 @@ def coverage_row_is_retained( ): return False # A selected outcome must retain its evidence even if its result write failed. - if superseded and not ( + if coverage_superseded and not ( isinstance(item, dict) and ( (field == "deferred" and retain_pending) @@ -2016,6 +1695,8 @@ def ambiguous_pending_surface(row: dict[str, Any], owner: str | None) -> bool: coverage_schema["surfaces"]["items"], deferred_rows, ambiguous_deferred, + canonical_rows, + coverage_sources, receipt_root=scan_dir if project_worker_coverage else None, receipt_warnings=warnings, fallback_surfaces=fallback_surfaces, @@ -2065,21 +1746,34 @@ def ambiguous_pending_surface(row: dict[str, Any], owner: str | None) -> bool: and ambiguous_pending_surface(surface, owner) and isinstance(coverage.get("surfaces"), list) ): - append_coverage_row( + retained = append_coverage_row( coverage["surfaces"], {**surface, "receiptRefs": surface.get("receiptRefs", [])}, owner, "surfaces", ) + _inherit_coverage_sources(retained, [surface], coverage_sources) + retained_surface_rows.add(id(surface)) for relative, draft, worker_id in all_sources: selection = source_coverage_selection(relative, worker_id) - selected_coverage_superseded, selected_candidates, superseded, retain_pending = selection + ( + selected_coverage_superseded, + selected_candidates, + superseded, + retain_pending, + coverage_superseded, + ) = selection skip_superseded_findings = ( superseded and not stopped and all(valid_finding(finding) for finding in (parent["findings"] if parent else [])) ) - if skip_superseded_findings and not selected_candidates and not retain_pending: + if ( + skip_superseded_findings + and coverage_superseded + and not selected_candidates + and not retain_pending + ): continue if ( not skip_superseded_findings @@ -2139,6 +1833,8 @@ def ambiguous_pending_surface(row: dict[str, Any], owner: str | None) -> bool: continue if valid_finding(finding): finding_positions.setdefault(_finding_key(finding), len(findings)) + if preserve_canonical_parent: + canonical_finding_keys.add(_finding_key(finding)) findings.append(finding) continue if relative != "parent" and parent and value in parent["findings"]: @@ -2152,8 +1848,14 @@ def ambiguous_pending_surface(row: dict[str, Any], owner: str | None) -> bool: "rejected", "not_applicable", }: + if preserve_canonical_parent: + rejected_history.setdefault((worker_id, candidate_id), []).append(finding) for item in _coverage_rows(coverage, "surfaces"): - if isinstance(item, dict) and item.get("candidateId") == candidate_id: + if ( + isinstance(item, dict) + and item.get("candidateId") == candidate_id + and (not preserve_canonical_parent or item.get("workerId") == worker_id) + ): _append_previous_finding(item, finding) continue for key in ("findingId", "occurrenceId", "fingerprints"): @@ -2201,8 +1903,10 @@ def ambiguous_pending_surface(row: dict[str, Any], owner: str | None) -> bool: if key in finding_positions: retained = findings[finding_positions[key]] if finding != retained: - if not represented_by_parent and _finding_strength(finding) > _finding_strength( - retained + if ( + key not in canonical_finding_keys + and not represented_by_parent + and _finding_strength(finding) > _finding_strength(retained) ): previous = copy.deepcopy(retained) previous_history = previous["provenance"].pop("previousFindings", []) @@ -2243,10 +1947,10 @@ def ambiguous_pending_surface(row: dict[str, Any], owner: str | None) -> bool: continue finding_positions[key] = len(findings) findings.append(finding) - if superseded and not selected_candidates and not retain_pending: + if coverage_superseded and not selected_candidates and not retain_pending: continue for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"): - if superseded and field not in {"surfaces", "explicitExclusions", "deferred"}: + if coverage_superseded and field not in {"surfaces", "explicitExclusions", "deferred"}: continue items = draft["coverage"].get(field, []) if field == "deferred": @@ -2259,7 +1963,7 @@ def ambiguous_pending_surface(row: dict[str, Any], owner: str | None) -> bool: # recovery and warnings rather than silently changing its contract. continue for item in items: - source_item = item + source_item = original_item = item if not coverage_row_is_retained(field, item, worker_id, selection): continue if id(item) in replaced_rows.get(field, ()): @@ -2302,8 +2006,14 @@ def ambiguous_pending_surface(row: dict[str, Any], owner: str | None) -> bool: worker_row_owners[id(existing)] = worker_id if field == "surfaces" and id(source_item) in linked_surface_sources: linked_surface_outputs[id(source_item)] = existing + _inherit_coverage_sources(existing, [original_item], coverage_sources) + if field == "surfaces": + retained_surface_rows.add(id(original_item)) continue retained = append_coverage_row(output, item, worker_id, field) + _inherit_coverage_sources(retained, [original_item], coverage_sources) + if field == "surfaces": + retained_surface_rows.add(id(original_item)) if field == "surfaces" and id(source_item) in linked_surface_sources: linked_surface_outputs[id(source_item)] = retained @@ -2320,6 +2030,34 @@ def ambiguous_pending_surface(row: dict[str, Any], owner: str | None) -> bool: finding.setdefault("provenance", {})["preservedIdentity"] = copy.deepcopy(identity) identity["instance"] = f"{identity.get('instance', 'saved')}-{variant[:16]}" identities[key] = variant + recovered_worker_ids = {worker_id for _, _, worker_id in sources if worker_id is not None} + surface_positions: dict[str, int] = {} + resolved_surface_targets: dict[tuple[str | None, str], list[str]] = {} + + def remember_surface_id(item: dict[str, Any], local_id: str) -> None: + final_id = item["id"] + surface_positions.setdefault(final_id, len(surface_positions)) + shadowed_alias = ( + project_worker_coverage + and id(item) not in named_surface_outputs + and (named := worker_surfaces.get((worker_row_owners.get(id(item)), local_id))) + is not None + and id(named) in named_surface_outputs + ) + for relative in coverage_sources.get(id(item), (item, []))[1]: + # A candidate-derived ID cannot claim a link explicitly naming another surface. + if not shadowed_alias: + targets = surface_ids_by_source.setdefault((relative, local_id), []) + if final_id not in targets: + targets.append(final_id) + candidate_id = item.get("candidateId") + if isinstance(candidate_id, str): + key = (source_owners[relative], candidate_id) + if key in resolved and resolved[key] == item.get("disposition"): + targets = resolved_surface_targets.setdefault(key, []) + if final_id not in targets: + targets.append(final_id) + # Bind links to retained output objects before their IDs become unique. # Candidate metadata can give an anonymous row the same preliminary ID as # another surface, so only the final surface ID can be used for its links. @@ -2387,17 +2125,73 @@ def ambiguous_pending_surface(row: dict[str, Any], owner: str | None) -> bool: reserved = { row["id"] for row in items if isinstance(row, dict) and isinstance(row.get("id"), str) } + normalized_items = [] for item in items: + normalized_items.append(item) if not isinstance(item, dict): continue if id(item) in canonical_rows: if isinstance(item.get("id"), str): used.add(item["id"]) + if field == "surfaces": + remember_surface_id(item, item["id"]) continue item.setdefault("id", _saved_coverage_id(item)) if not isinstance(item["id"], str): # Preserve malformed rows for per-record recovery, including frozen replay. continue + local_id = item["id"] + scoped = ( + preserve_canonical_parent + and isinstance(worker_id := item.get("workerId"), str) + and worker_id in recovered_worker_ids + ) + if scoped: + # Resolve worker-local candidate/deferred history before scoping IDs. + item["id"] = f"{worker_id}/{item['id']}" + if isinstance(item.get("surfaceIds"), list): + links: list[Any] = [] + for surface_id in item["surfaceIds"]: + targets: list[str] = [] + declared = False + if field == "deferred" and isinstance(surface_id, str): + for relative in coverage_sources.get(id(item), (item, []))[1]: + source_key = (relative, surface_id) + history_keys = [source_key] + if source_key not in surface_ids_by_source: + # A deferred-only checkpoint can refer to an earlier + # observation from this logical worker, including retries. + # Reused IDs can denote several retained observations. + history_keys = [ + (historical, surface_id) + for historical, _, owner in all_sources + if owner == source_owners[relative] + and source_order[historical] <= source_order[relative] + ] + for key in history_keys: + declared |= key in surface_ids_by_source + for target in surface_ids_by_source.get(key, []): + if target not in targets: + targets.append(target) + if targets: + targets.sort(key=surface_positions.__getitem__) + for target in targets: + if target not in links: + links.append(target) + elif not declared: + links.append( + f"{worker_id}/{surface_id}" + if scoped and isinstance(surface_id, str) + else surface_id + ) + item["surfaceIds"] = links + if scoped: + if item in normalized_items[:-1]: + # A previous finish may have published files before its DB commit. + if field == "surfaces": + remember_surface_id(item, local_id) + normalized_items.pop() + continue if item["id"] in used: if project_worker_coverage: base = item["id"] @@ -2407,9 +2201,36 @@ def ambiguous_pending_surface(row: dict[str, Any], owner: str | None) -> bool: suffix += 1 else: item["id"] = f"{item['id']}-{_digest(item)[:16]}" + if preserve_canonical_parent and item in normalized_items[:-1]: + if field == "surfaces": + remember_surface_id(item, local_id) + normalized_items.pop() + continue used.add(item["id"]) if field == "surfaces": item.setdefault("receiptRefs", []) + remember_surface_id(item, local_id) + if preserve_canonical_parent and isinstance(items, list): + coverage[field] = normalized_items + if field == "surfaces": + # Candidate resolution replaces pending surfaces independently of generic + # tasks. Preserve their original links, even when the outcome changes ID + # or a distinct retained surface shares the old source-local ID. + for relative, draft, owner in all_sources: + originals = draft["coverage"].get("surfaces", []) + for original in originals if isinstance(originals, list) else []: + if ( + not isinstance(original, dict) + or id(original) in retained_surface_rows + or original.get("disposition") != "needs_follow_up" + or not isinstance(local_id := original.get("id"), str) + or not isinstance(candidate_id := original.get("candidateId"), str) + ): + continue + targets = surface_ids_by_source.setdefault((relative, local_id), []) + for target in resolved_surface_targets.get((owner, candidate_id), []): + if target not in targets: + targets.append(target) for task, links in retained_surface_links.values(): task["surfaceIds"] = [ links[identity]["id"] if isinstance(identity, str) and identity in links else identity @@ -2440,27 +2261,50 @@ def coverage_for_comparison(db: Any, scan: Any) -> dict[str, Any]: def preserve_deep_deadline_draft( - db: Any, connection: Any, scan: Any, *, allow_empty: bool = False + db: Any, + connection: Any, + scan: Any, + *, + preserve_reduction: bool = False, + allow_empty: bool = False, ) -> tuple[Path, dict[str, tuple[bytes, os.stat_result] | None], set[str]] | None: - """Recover a settled, zero-accepted Deep deadline without sealing the parent.""" + """Recover unfinished reviews at a settled Deep deadline without sealing the parent.""" scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"])) + warnings = json.loads(scan["completion_warnings_json"]) + binding = db.workbench_completion_binding(scan, db.now()) + workers = connection.execute( + "SELECT * FROM deep_scan_workers WHERE scan_id = ? ORDER BY created_at, id", + (scan["id"],), + ).fetchall() + if preserve_reduction: + # Completed reviews already went through reduction. Only add evidence + # from unfinished reviews; raw accepted drafts must not undo that result. + workers = [ + worker + for worker in workers + if worker["kind"] == "discovery" and worker["status"] == "canceled" + ] + if not workers: + return None + _prepare_scan_finalization( + scan_dir, + expected_coverage_mode=db.expected_coverage_mode(scan), + completion_binding=binding, + allow_missing_writeups=True, + ) snapshots = _snapshot_published_outputs( scan_dir, ("findings.json", "coverage.json", "scan-manifest.json", "checkpoint-head.json") ) - warnings = json.loads(scan["completion_warnings_json"]) - binding = db.workbench_completion_binding(scan, db.now()) created_paths: set[str] = set() try: documents = merge_saved_results( scan_dir, scan["id"], binding, - connection.execute( - "SELECT * FROM deep_scan_workers WHERE scan_id = ? ORDER BY created_at, id", - (scan["id"],), - ).fetchall(), + workers, warnings, - stopped=True, + stopped=not preserve_reduction, + preserve_canonical_parent=preserve_reduction, project_worker_coverage=True, created_paths=created_paths, current_contract=True, @@ -2478,28 +2322,46 @@ def preserve_deep_deadline_draft( (json.dumps(list(dict.fromkeys(warnings))), scan["id"]), ) return None + if preserve_reduction: + marker = { + "id": "scan-deadline", + "reason": ( + "The configured discovery time limit elapsed. Saved findings and " + "pending work from unfinished source reviews were preserved." + ), + } + if not any( + marker["id"] in (row.get("id"), row.get("candidateId")) + for key in ("deferred", "resolvedDeferred") + for row in documents[2].get(key, []) + ): + documents[2]["deferred"].append(marker) documents[0]["scan"].pop("complete", None) - # A capped deadline can retain interrupted records even though its - # completion binding requires strict validation of completed findings. expected_coverage_mode = db.expected_coverage_mode(scan) - _populate_unsealed_manifest_envelope(documents[0], documents[0]["scan"], binding) - _populate_unsealed_artifact_envelope(*documents, binding) - if expected_coverage_mode == "deep_repository": - documents[2]["inventoryStrategy"] = "repository" - elif expected_coverage_mode == "scoped_path": - documents[2]["inventoryStrategy"] = "scoped_path" - schema_dir = Path(__file__).resolve().parent.parent / "schemas" - discarded_findings = _recover_unsealed_findings( - documents[0], documents[1], schema_dir, scan_dir, warnings, current_contract=True - ) - _recover_unsealed_coverage(documents[2], schema_dir, scan_dir, warnings, discarded_findings) + if not preserve_reduction: + # A capped deadline can retain interrupted records even though its + # completion binding requires strict validation of completed findings. + _populate_unsealed_manifest_envelope(documents[0], documents[0]["scan"], binding) + _populate_unsealed_artifact_envelope(*documents, binding) + if expected_coverage_mode == "deep_repository": + documents[2]["inventoryStrategy"] = "repository" + elif expected_coverage_mode == "scoped_path": + documents[2]["inventoryStrategy"] = "scoped_path" + schema_dir = Path(__file__).resolve().parent.parent / "schemas" + discarded_findings = _recover_unsealed_findings( + documents[0], documents[1], schema_dir, scan_dir, warnings, current_contract=True + ) + _recover_unsealed_coverage( + documents[2], schema_dir, scan_dir, warnings, discarded_findings + ) # Reuse stopped-result recovery's per-record validation, scope checks, # source lineage, and receipt validation before admitting this draft. prepared = _prepare_scan_finalization( scan_dir, expected_coverage_mode=expected_coverage_mode, completion_binding=binding, - recover_unsealed=True, + recover_unsealed=not preserve_reduction, + allow_missing_writeups=preserve_reduction, draft_documents=documents, finding_scope=db.requested_finding_scope(scan), ) diff --git a/plugins/codex-security/tests/test_deep_coverage_recovery.py b/plugins/codex-security/tests/test_deep_coverage_recovery.py index 74a91492c..1eb8982bf 100644 --- a/plugins/codex-security/tests/test_deep_coverage_recovery.py +++ b/plugins/codex-security/tests/test_deep_coverage_recovery.py @@ -1033,6 +1033,8 @@ def surface(identity, disposition, receipt): schema, {"old": [task], "current": []}, set(), + canonical_rows=set(), + coverage_sources={}, surface_is_retained=lambda _relative, _owner, _row: True, eligible_receipt_sources={"old", "current"}, receipt_root=tmp_path, diff --git a/plugins/codex-security/tests/test_deep_deadline_recovery_edges.py b/plugins/codex-security/tests/test_deep_deadline_recovery_edges.py index 7f494f9b4..08b897f1f 100644 --- a/plugins/codex-security/tests/test_deep_deadline_recovery_edges.py +++ b/plugins/codex-security/tests/test_deep_deadline_recovery_edges.py @@ -431,6 +431,8 @@ def commit(self): def test_mid_merge_failure_removes_only_created_recovery_evidence( tmp_path: Path, monkeypatch, workbench_api, write_kind: str ): + from workbench import saved_sources + scan = deadline_fixture(tmp_path, "valid") canonical_parent(scan, saved_draft(scan.scan_id, findings=[scan.finding])) before = { @@ -458,6 +460,7 @@ def fail_during_merge(directory, relative, contents): raise RuntimeError("Synthetic merge failure") monkeypatch.setattr(saved, "write_scan_local_bytes", fail_during_merge) + monkeypatch.setattr(saved_sources, "write_scan_local_bytes", fail_during_merge) with sqlite3.connect(scan.database) as connection: connection.row_factory = sqlite3.Row record = workbench_api["require_scan"](connection, scan.scan_id) @@ -476,6 +479,8 @@ def fail_during_merge(directory, relative, contents): def test_failed_head_snapshot_does_not_interrupt_publication_rollback( tmp_path, monkeypatch, workbench_api, snapshot_failure ): + from workbench import saved_sources + scan = deadline_fixture(tmp_path, "valid") canonical_parent(scan, saved_draft(scan.scan_id, findings=[scan.finding])) before = { @@ -495,6 +500,7 @@ def fail_before_snapshot_directory(directory, relative, contents): write(directory, relative, contents) monkeypatch.setattr(saved, "write_scan_local_bytes", fail_before_snapshot_directory) + monkeypatch.setattr(saved_sources, "write_scan_local_bytes", fail_before_snapshot_directory) with sqlite3.connect(scan.database) as connection: connection.row_factory = sqlite3.Row connection.execute( diff --git a/plugins/codex-security/tests/test_deep_scan_deadline_contracts.py b/plugins/codex-security/tests/test_deep_scan_deadline_contracts.py new file mode 100644 index 000000000..789fd62a5 --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_deadline_contracts.py @@ -0,0 +1,598 @@ +from __future__ import annotations + +import copy +import json +import os + +import pytest +from test_deep_scan_deadline_recovery import worker_finding +from test_deep_scan_stop_conditions import canceled_discovery_output, complete, finish +from test_deep_scan_stop_conditions import deadline_scan as deadline_scan +from test_deep_scan_stop_conditions import publication_scan as publication_scan +from test_deep_scan_stop_conditions import saturated_scan as saturated_scan +from workbench_test_support import saved_draft, write_checkpoint + + +@pytest.mark.parametrize("source", ["result", "checkpoint", "archived-checkpoint"]) +@pytest.mark.parametrize( + "marker", + [ + pytest.param("false", id="string-false"), + pytest.param(None, id="null"), + pytest.param(0, id="zero"), + pytest.param(1, id="one"), + pytest.param(False, id="incomplete"), + pytest.param(True, id="complete"), + pytest.param("omitted", id="omitted"), + ], +) +def test_deadline_requires_boolean_completion_before_superseding_worker_history( + workbench_api, workbench_db, deadline_scan, source, marker +): + scan = deadline_scan + _, output = canceled_discovery_output(workbench_db, scan, attempt=2) + if source == "archived-checkpoint": + output = output.parent / "attempts" / "attempt-1" + earlier = worker_finding(scan, "earlier-worker-finding") + rejected = worker_finding(scan, "rejected-worker-finding") + rejected["provenance"]["candidateId"] = "rejected-worker-finding" + checkpoint = write_checkpoint( + output / "checkpoints", + saved_draft( + scan.scan_id, + findings=[earlier, rejected], + deferred=[{"id": "review", "reason": "Review remains."}], + ), + ) + os.utime(checkpoint, ns=(100, 100)) + sibling = worker_finding(scan, "latest-valid-sibling") + latest = saved_draft( + scan.scan_id, + complete=marker, + findings=[sibling], + surfaces=[ + { + "id": "rejected-outcome", + "candidateId": "rejected-worker-finding", + "label": "Rejected after review", + "disposition": "rejected", + "receiptRefs": [], + } + ], + closures=[{"id": "review", "reason": "Review completed."}], + ) + if marker == "omitted": + latest.pop("complete") + if source == "result": + result = output / "result.json" + result.write_text(json.dumps(latest)) + else: + result = write_checkpoint(output / "checkpoints", latest) + head = output / "checkpoint-head.json" + head.write_text(json.dumps({"checkpoint": result.name})) + os.utime(head, ns=(300, 300)) + os.utime(result, ns=(200, 200)) + original = {path: path.read_bytes() for path in (checkpoint, result)} + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + completed = complete(workbench_api, workbench_db, scan) + + terminal = marker is True or marker == "omitted" + invalid = not isinstance(marker, bool) and marker != "omitted" + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + titles = {finding["title"] for finding in findings} + assert sibling["title"] in titles + assert rejected["title"] not in titles + if not terminal or source == "result": + assert (earlier["title"] in titles) == (not terminal) + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert any(row["reason"] == "Review remains." for row in coverage["deferred"]) == (not terminal) + assert ( + any( + "Invalid complete marker" in warning and result.name in warning + for warning in completed["warnings"] + ) + == invalid + ) + manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"] + for path, contents in original.items(): + assert path.read_bytes() == contents + assert manifest["preservedSources"][path.relative_to(scan.scan_dir).as_posix()] == ( + workbench_api["saved_results"]._digest(json.loads(contents)) + ) + + +@pytest.mark.parametrize("source", ["result", "checkpoint", "archived-result"]) +@pytest.mark.parametrize("invalid", ["rationale", "code_evidence", "sourceFindingIds"]) +def test_deadline_validates_each_recovered_finding_against_the_completed_contract( + workbench_api, workbench_db, deadline_scan, source, invalid +): + scan = deadline_scan + worker_id, output = canceled_discovery_output(workbench_db, scan, attempt=2) + if source == "archived-result": + output = output.parent / "attempts" / "attempt-1" + earlier = worker_finding(scan, "earlier-worker-finding") + rejected = worker_finding(scan, "rejected-worker-finding") + rejected["provenance"]["candidateId"] = "rejected-worker-finding" + checkpoint = write_checkpoint( + output / "checkpoints", + saved_draft( + scan.scan_id, + findings=[earlier, rejected], + deferred=[ + {"id": "review", "reason": "Review remains."}, + {"id": "finished", "reason": "Separate review remains."}, + ], + ), + ) + os.utime(checkpoint, ns=(100, 100)) + sibling = worker_finding(scan, "valid-worker-sibling") + malformed = worker_finding(scan, "malformed-worker-finding") + malformed["provenance"]["candidateId"] = "malformed-worker-finding" + if invalid == "rationale": + malformed["severity"]["rationale"] = " \t " + elif invalid == "code_evidence": + malformed["code_evidence"] = [] + else: + malformed["provenance"]["sourceFindingIds"] = ["source-finding", "source-finding"] + latest = saved_draft( + scan.scan_id, + complete=True, + findings=[malformed, sibling], + surfaces=[ + { + "id": "reported-outcome", + "candidateId": "malformed-worker-finding", + "label": "Unverified finding", + "disposition": "reported", + "receiptRefs": [], + }, + { + "id": "rejected-outcome", + "candidateId": "rejected-worker-finding", + "label": "Rejected after review", + "disposition": "rejected", + "receiptRefs": [], + }, + ], + closures=[{"id": "finished", "reason": "Separate review completed."}], + ) + if source == "checkpoint": + result = write_checkpoint(output / "checkpoints", latest) + head = output / "checkpoint-head.json" + head.write_text(json.dumps({"checkpoint": result.name})) + os.utime(head, ns=(300, 300)) + else: + result = output / "result.json" + result.write_text(json.dumps(latest)) + os.utime(result, ns=(200, 200)) + original = {path: path.read_bytes() for path in (checkpoint, result)} + if source == "archived-result": + selected = write_checkpoint(output / "checkpoints", latest) + os.utime(selected, ns=(200, 200)) + head = output / "checkpoint-head.json" + head.write_text(json.dumps({"checkpoint": selected.name})) + os.utime(head, ns=(300, 300)) + original[selected] = selected.read_bytes() + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + completed = complete(workbench_api, workbench_db, scan) + + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert {finding["title"] for finding in findings} == { + scan.findings[0]["title"], + earlier["title"], + sibling["title"], + } + retained = next(finding for finding in findings if finding["title"] == sibling["title"]) + assert retained["provenance"]["workerId"] == worker_id + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert any(row["reason"] == "Review remains." for row in coverage["deferred"]) + assert not any(row["reason"] == "Separate review remains." for row in coverage["deferred"]) + surfaces = {row["label"]: row for row in coverage["surfaces"]} + assert surfaces["Unverified finding"]["disposition"] == "needs_follow_up" + assert surfaces["Rejected after review"]["disposition"] == "rejected" + assert any(invalid in warning and result.name in warning for warning in completed["warnings"]) + manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"] + for path, contents in original.items(): + assert path.read_bytes() == contents + assert manifest["preservedSources"][path.relative_to(scan.scan_dir).as_posix()] == ( + workbench_api["saved_results"]._digest(json.loads(contents)) + ) + + +@pytest.mark.parametrize( + "model", + [ + {"assets": []}, + {"summary": "Invalid " + chr(0xD800)}, + {"summary": "Reviewed boundaries."}, + {"format": "markdown", "content": "# Reviewed boundaries"}, + ], +) +def test_deadline_recovers_optional_worker_model_without_losing_findings( + workbench_api, workbench_db, deadline_scan, model +): + scan = deadline_scan + _, output = canceled_discovery_output(workbench_db, scan) + finding = worker_finding(scan, "worker-model-finding") + draft = {**saved_draft(scan.scan_id, findings=[finding]), "threatModel": model} + checkpoint = write_checkpoint(output / "checkpoints", draft) + original = checkpoint.read_bytes() + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + completed = complete(workbench_api, workbench_db, scan) + + assert len(json.loads((scan.scan_dir / "findings.json").read_text())["findings"]) == 2 + manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"] + invalid = "assets" in model or chr(0xD800) in model.get("summary", "") + if invalid: + assert "threatModel" not in manifest + assert any( + "threat model" in warning and checkpoint.name in warning + for warning in completed["warnings"] + ) + else: + assert manifest["threatModel"] == {**model, "origin": "recovered"} + assert checkpoint.read_bytes() == original + + +def test_deadline_still_rejects_malformed_canonical_model( + workbench_api, workbench_db, deadline_scan +): + from finalize_scan_contract import ContractError + + scan = deadline_scan + manifest_file = scan.scan_dir / "scan-manifest.json" + manifest = json.loads(manifest_file.read_text()) + manifest["scan"]["threatModel"] = {"assets": []} + manifest_file.write_text(json.dumps(manifest)) + _, output = canceled_discovery_output(workbench_db, scan) + write_checkpoint(output / "checkpoints", saved_draft(scan.scan_id)) + with pytest.raises(ContractError, match="threatModel"): + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + + +def test_deadline_still_rejects_canonical_findings_outside_the_completed_contract( + workbench_api, workbench_db, deadline_scan +): + from finalize_scan_contract import ContractError + + scan = deadline_scan + finding = worker_finding(scan, "valid-worker-finding") + scan.findings[0]["severity"]["rationale"] = " \t " + findings_path = scan.scan_dir / "findings.json" + findings_path.write_text(json.dumps({"findings": scan.findings})) + original = findings_path.read_bytes() + _, output = canceled_discovery_output(workbench_db, scan) + write_checkpoint( + output / "checkpoints", + saved_draft(scan.scan_id, findings=[finding]), + ) + + with pytest.raises(ContractError, match="rationale"): + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + + assert findings_path.read_bytes() == original + + +@pytest.mark.parametrize("label", [" \t ", " Reviewed Résumé 😀 "]) +def test_deadline_applies_existing_nonblank_surface_label_contract( + workbench_api, workbench_db, deadline_scan, label +): + scan = deadline_scan + worker_id, output = canceled_discovery_output(workbench_db, scan) + checkpoint = write_checkpoint( + output / "checkpoints", + saved_draft( + scan.scan_id, + findings=[worker_finding(scan, "worker-label-finding")], + surfaces=[ + {"id": "entry", "label": label, "disposition": "reported", "receiptRefs": []} + ], + ), + ) + original = checkpoint.read_bytes() + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + completed = complete(workbench_api, workbench_db, scan) + + assert len(json.loads((scan.scan_dir / "findings.json").read_text())["findings"]) == 2 + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + surfaces = [row for row in coverage["surfaces"] if row.get("workerId") == worker_id] + assert len(surfaces) == bool(label.strip()) + if surfaces: + assert surfaces[0]["label"] == label + else: + assert any( + "label: expected a non-empty string" in warning and checkpoint.name in warning + for warning in completed["warnings"] + ) + assert checkpoint.read_bytes() == original + + +@pytest.mark.parametrize( + "field", + ["finding", "surfaces", "explicitExclusions", "deferred", "openQuestions", "resolvedDeferred"], +) +@pytest.mark.parametrize("malformed", [False, True], ids=["valid-unicode", "lone-surrogate"]) +def test_deadline_applies_existing_unicode_contract_per_worker_record( + workbench_api, workbench_db, deadline_scan, field, malformed +): + scan = deadline_scan + _, output = canceled_discovery_output(workbench_db, scan) + text = "Review " + (chr(0xD800) if malformed else "Résumé 😀") + finding = worker_finding(scan, "unicode-worker-finding") + draft = saved_draft(scan.scan_id, findings=[finding]) + if field == "finding": + finding["summary"] = text + elif field == "surfaces": + draft["coverage"][field] = [ + {"id": "entry", "label": text, "disposition": "reported", "receiptRefs": []} + ] + elif field == "openQuestions": + draft["coverage"][field] = [text] + elif field == "explicitExclusions": + draft["coverage"][field] = [{"pattern": "vendor/", "reason": text}] + else: + draft["coverage"][field] = [{"id": "review", "reason": text}] + if field == "resolvedDeferred": + draft["complete"] = True + previous = write_checkpoint( + output / "checkpoints", + saved_draft(scan.scan_id, deferred=[{"id": "review", "reason": "Review pending."}]), + ) + os.utime(previous, ns=(100, 100)) + checkpoint = write_checkpoint(output / "checkpoints", draft) + os.utime(checkpoint, ns=(200, 200)) + if field == "resolvedDeferred": + (output / "checkpoint-head.json").write_text(json.dumps({"checkpoint": checkpoint.name})) + original = checkpoint.read_bytes() + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + completed = complete(workbench_api, workbench_db, scan) + + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert len(findings) == (1 if malformed and field == "finding" else 2) + assert any("well-formed Unicode" in warning for warning in completed["warnings"]) == malformed + if field == "resolvedDeferred": + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert any(row["reason"] == "Review pending." for row in coverage["deferred"]) == malformed + elif not malformed: + document = ( + findings + if field == "finding" + else json.loads((scan.scan_dir / "coverage.json").read_text()) + ) + assert text in json.dumps(document, ensure_ascii=False) + assert checkpoint.read_bytes() == original + + +@pytest.mark.parametrize( + "reason", + [pytest.param("\u0085", id="historical-nel"), pytest.param("Review completed.", id="valid")], +) +def test_deadline_recovers_historical_closure_text_without_losing_a_valid_sibling( + workbench_api, workbench_db, deadline_scan, reason +): + scan = deadline_scan + _, output = canceled_discovery_output(workbench_db, scan) + previous = write_checkpoint( + output / "checkpoints", + saved_draft( + scan.scan_id, + deferred=[ + {"id": "review", "reason": "First review pending."}, + {"id": "sibling", "reason": "Valid sibling review pending."}, + ], + ), + ) + os.utime(previous, ns=(100, 100)) + draft = saved_draft( + scan.scan_id, + complete=True, + closures=[ + {"id": "review", "reason": reason}, + {"id": "sibling", "reason": "Valid sibling review completed."}, + ], + ) + checkpoint = write_checkpoint(output / "checkpoints", draft) + result = output / "result.json" + result.write_text(json.dumps(draft)) + head = output / "checkpoint-head.json" + head.write_text(json.dumps({"checkpoint": checkpoint.name})) + for path in (checkpoint, result, head): + os.utime(path, ns=(200, 200)) + original = {path: path.read_bytes() for path in (previous, checkpoint, result)} + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + completed = complete(workbench_api, workbench_db, scan) + + invalid = not reason.strip() + assert completed["progress"]["status"] == "complete" + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + pending = {row["reason"] for row in coverage["deferred"]} + assert ("First review pending." in pending) == invalid + assert "Valid sibling review pending." not in pending + report = (scan.scan_dir / "report.md").read_text() + assert ("First review pending." in report) == invalid + assert "Valid sibling review pending." not in report + assert any("expected a non-empty string" in warning for warning in completed["warnings"]) == ( + invalid + ) + manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"] + assert manifest["sealedAt"] + for path, contents in original.items(): + assert path.read_bytes() == contents + assert manifest["preservedSources"][path.relative_to(scan.scan_dir).as_posix()] == ( + workbench_api["saved_results"]._digest(json.loads(contents)) + ) + + +@pytest.mark.parametrize( + ("terminal_findings", "malformed", "selected_head"), + [ + (False, None, False), + (True, None, False), + (False, "surface", False), + (True, "surface", False), + (False, "closure", False), + (True, "closure", False), + (True, "surface", True), + (False, "missing-receipt", False), + (True, "missing-receipt", False), + (False, "disposition", False), + (True, "disposition", False), + (False, "receipt-array", False), + (True, "receipt-array", False), + ], + ids=[ + "valid-coverage-empty-findings", + "valid-coverage-retained-finding", + "malformed-surface-empty-findings", + "malformed-surface-retained-finding", + "malformed-closure-empty-findings", + "malformed-closure-retained-finding", + "malformed-selected-head", + "missing-receipt-empty-findings", + "missing-receipt-retained-finding", + "disposition-empty-findings", + "disposition-retained-finding", + "receipt-array-empty-findings", + "receipt-array-retained-finding", + ], +) +def test_malformed_terminal_coverage_preserves_only_earlier_coverage( + workbench_api, workbench_db, deadline_scan, terminal_findings, malformed, selected_head +): + scan = deadline_scan + worker_id, output = canceled_discovery_output(workbench_db, scan) + finding = worker_finding(scan, "earlier-worker-finding") + rejected = worker_finding(scan, "rejected-sibling") + rejected["provenance"]["candidateId"] = "rejected-sibling" + surface = { + "id": "entry", + "label": "Unfinished review", + "disposition": "needs_follow_up", + "receiptRefs": [], + } + checkpoint = write_checkpoint( + output / "checkpoints", + saved_draft( + scan.scan_id, + findings=[finding, rejected], + surfaces=[surface], + deferred=[ + {"id": "review", "reason": "Review remains.", "surfaceIds": ["entry"]}, + {"id": "finished", "reason": "Separate review remains."}, + ], + ), + ) + os.utime(checkpoint, ns=(100, 100)) + invalid_surface = copy.deepcopy(surface) + if malformed == "surface": + invalid_surface.pop("label") + else: + invalid_surface.update(id="latest", label="Latest review", disposition="no_issue_found") + if malformed == "missing-receipt": + invalid_surface["receiptRefs"] = ["artifacts/missing.txt"] + elif malformed == "disposition": + invalid_surface["disposition"] = "unknown" + elif malformed == "receipt-array": + invalid_surface["receiptRefs"] = "invalid" + closures = [{"id": "finished", "reason": "Separate review completed."}] + if malformed == "closure": + closures.append({"id": "review", "reason": "Invalid " + chr(0xD800)}) + terminal = saved_draft( + scan.scan_id, + complete=True, + findings=[finding] if terminal_findings else [], + surfaces=[ + { + "id": "sibling-outcome", + "candidateId": "rejected-sibling", + "label": "Sibling reviewed", + "disposition": "rejected", + "receiptRefs": [], + }, + *([invalid_surface] if malformed not in (None, "closure") else []), + ], + closures=closures, + ) + if selected_head: + result = write_checkpoint(output / "checkpoints", terminal) + (output / "checkpoint-head.json").write_text(json.dumps({"checkpoint": result.name})) + else: + result = output / "result.json" + result.write_text(json.dumps(terminal)) + os.utime(result, ns=(200, 200)) + original = {path: path.read_bytes() for path in (checkpoint, result)} + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + complete(workbench_api, workbench_db, scan) + + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert any(row["title"] == finding["title"] for row in findings) == terminal_findings + assert all(row["title"] != rejected["title"] for row in findings) + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert any( + row.get("workerId") == worker_id and row["label"] == surface["label"] + for row in coverage["surfaces"] + ) == bool(malformed) + assert any(row["reason"] == "Review remains." for row in coverage["deferred"]) == bool( + malformed + ) + assert not any(row["reason"] == "Separate review remains." for row in coverage["deferred"]) + assert ( + next(row for row in coverage["surfaces"] if row["label"] == "Sibling reviewed")[ + "disposition" + ] + == "rejected" + ) + if malformed in {"missing-receipt", "disposition", "receipt-array"}: + latest = next(row for row in coverage["surfaces"] if row["label"] == "Latest review") + assert latest["disposition"] == "needs_follow_up" + assert latest["receiptRefs"] == [] + assert {path: path.read_bytes() for path in original} == original + + +@pytest.mark.parametrize("canceled_worker", [False, True]) +def test_optional_export_failure_does_not_block_deadline( + workbench_api, workbench_db, deadline_scan, tmp_path, capsys, canceled_worker +): + scan = deadline_scan + sentinel = tmp_path / "untouched.sarif" + sentinel.write_text("Preserve this file.\n") + export = scan.scan_dir / "exports" / "results.sarif" + export.parent.mkdir() + export.symlink_to(sentinel) + if canceled_worker: + _, output = canceled_discovery_output(workbench_db, scan) + write_checkpoint( + output / "checkpoints", + saved_draft(scan.scan_id, findings=[worker_finding(scan, "saved-worker-finding")]), + ) + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + completed = complete(workbench_api, workbench_db, scan) + + assert completed["progress"]["status"] == "complete" + assert "automatic SARIF export failed" in capsys.readouterr().err + assert export.is_symlink() + assert sentinel.read_text() == "Preserve this file.\n" + + +def test_standard_parent_closure_preserves_unrelated_task_without_id( + workbench_api, workbench_db, publication_scan +): + scan = publication_scan(mode="standard") + scan.coverage["resolvedDeferred"] = [{"id": "finished", "reason": "Review completed."}] + scan.coverage["deferred"] = [{"reason": "Unrelated pending review."}] + (scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage)) + + complete(workbench_api, workbench_db, scan) + + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert len(coverage["deferred"]) == 1 + assert coverage["deferred"][0]["id"] + assert coverage["deferred"][0]["reason"] == "Unrelated pending review." diff --git a/plugins/codex-security/tests/test_deep_scan_deadline_recovery.py b/plugins/codex-security/tests/test_deep_scan_deadline_recovery.py new file mode 100644 index 000000000..d86b711de --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_deadline_recovery.py @@ -0,0 +1,512 @@ +from __future__ import annotations + +import copy +import hashlib +import json +import os +import sqlite3 +import traceback + +import pytest +from test_deep_scan_stop_conditions import ( + canceled_discovery_output, + complete, + finish, +) +from test_deep_scan_stop_conditions import ( + deadline_scan as deadline_scan, +) +from test_deep_scan_stop_conditions import ( + publication_scan as publication_scan, +) +from test_deep_scan_stop_conditions import ( + saturated_scan as saturated_scan, +) +from workbench_test_support import saved_draft, write_checkpoint + + +def worker_finding(scan, anchor): + finding = copy.deepcopy(scan.findings[0]) + finding["identity"] = {"anchor": anchor} + finding["title"] = anchor + finding["provenance"] = {"source": "local_plugin"} + return finding + + +@pytest.mark.parametrize( + ("task_state", "task_id", "marker_count"), + [ + ("resolvedDeferred", "scan-deadline", 0), + ("resolvedDeferred", "prior-review", 1), + ("deferred", "scan-deadline", 0), + ], +) +def test_deadline_completion_preserves_existing_tasks_and_recovered_findings( + workbench_api, workbench_db, deadline_scan, task_state, task_id, marker_count +): + scan = deadline_scan + task = {"id": task_id, "reason": "Existing review outcome."} + scan.coverage[task_state] = [task] + (scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage)) + worker_id, output = canceled_discovery_output(workbench_db, scan) + recovered = worker_finding(scan, "saved-before-deadline") + checkpoint = write_checkpoint( + output / "checkpoints", + saved_draft( + scan.scan_id, + findings=[recovered], + deferred=[{"id": "unfinished-review", "reason": "Review remains unfinished."}], + ), + ) + original_checkpoint = checkpoint.read_bytes() + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + completed = complete(workbench_api, workbench_db, scan) + + assert completed["progress"]["status"] == "complete" + assert ( + workbench_db.execute("SELECT status FROM scans WHERE id = ?", (scan.scan_id,)).fetchone()[0] + == "complete" + ) + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert task in coverage[task_state] + if task_state == "resolvedDeferred": + assert coverage[task_state] == [task] + assert any( + row.get("workerId") == worker_id and row["reason"] == "Review remains unfinished." + for row in coverage["deferred"] + ) + assert ( + sum(row.get("id") == "scan-deadline" and row != task for row in coverage["deferred"]) + == marker_count + ) + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert {row["title"] for row in findings} == {scan.findings[0]["title"], recovered["title"]} + assert ( + next(row for row in findings if row["title"] == recovered["title"])["provenance"][ + "workerId" + ] + == worker_id + ) + assert checkpoint.read_bytes() == original_checkpoint + + +def interrupt_finish_before_commit(workbench_api, database, scan): + child = os.fork() + if child == 0: + connection = sqlite3.connect(database) + connection.row_factory = sqlite3.Row + + class CrashBeforeCommit: + def __getattr__(self, name): + return getattr(connection, name) + + def commit(self): + os._exit(0) + + try: + finish(workbench_api, CrashBeforeCommit(), scan, terminal_reason="capped") + except BaseException: + traceback.print_exc() + os._exit(1) + _, status = os.waitpid(child, 0) + assert os.waitstatus_to_exitcode(status) == 0 + + +def test_discarded_worker_finding_leaves_its_review_pending( + workbench_api, workbench_db, deadline_scan +): + scan = deadline_scan + canonical = { + "id": "accepted", + "label": "Accepted review", + "disposition": "reported", + "receiptRefs": [], + } + scan.coverage["surfaces"] = [canonical] + (scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage)) + worker_id, output = canceled_discovery_output(workbench_db, scan) + invalid = worker_finding(scan, "incomplete-worker-finding") + invalid.pop("remediation") + checkpoint = write_checkpoint( + output / "checkpoints", + saved_draft( + scan.scan_id, + findings=[invalid], + surfaces=[ + { + "id": "entry", + "label": "Worker review", + "disposition": "reported", + "receiptRefs": [], + } + ], + ), + ) + original = checkpoint.read_bytes() + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + completed = complete(workbench_api, workbench_db, scan) + + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert canonical in coverage["surfaces"] + surface = next(row for row in coverage["surfaces"] if row.get("workerId") == worker_id) + assert surface["disposition"] == "needs_follow_up" + assert any( + row.get("workerId") == worker_id and "remediation" in row["reason"] + for row in coverage["deferred"] + ) + assert completed["progress"]["status"] == "complete" + assert len(json.loads((scan.scan_dir / "findings.json").read_text())["findings"]) == 1 + assert checkpoint.read_bytes() == original + + +@pytest.mark.parametrize( + "invalid_terminal", [False, True], ids=["valid-terminal", "invalid-terminal"] +) +def test_terminal_worker_authority_requires_valid_findings( + workbench_api, workbench_db, deadline_scan, invalid_terminal +): + scan = deadline_scan + _, output = canceled_discovery_output(workbench_db, scan) + previous = worker_finding(scan, "earlier-valid-finding") + checkpoint = write_checkpoint( + output / "checkpoints", + saved_draft( + scan.scan_id, + findings=[previous], + deferred=[{"id": "review", "reason": "Review pending."}], + ), + ) + os.utime(checkpoint, ns=(100, 100)) + sibling = worker_finding(scan, "valid-terminal-sibling") + terminal_findings = [sibling] + if invalid_terminal: + invalid = copy.deepcopy(previous) + invalid.pop("remediation") + terminal_findings.append(invalid) + result = output / "result.json" + result.write_text( + json.dumps( + saved_draft( + scan.scan_id, + complete=True, + findings=terminal_findings, + closures=[{"id": "review", "reason": "Review completed."}], + ) + ) + ) + os.utime(result, ns=(200, 200)) + original = {path: path.read_bytes() for path in (checkpoint, result)} + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + complete(workbench_api, workbench_db, scan) + + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert any(row["title"] == previous["title"] for row in findings) == invalid_terminal + assert any(row["title"] == sibling["title"] for row in findings) + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert not any(row["reason"] == "Review pending." for row in coverage["deferred"]) + assert {path: path.read_bytes() for path in original} == original + + +def test_malformed_worker_finding_preserves_a_valid_sibling_outcome( + workbench_api, workbench_db, deadline_scan +): + scan = deadline_scan + worker_id, output = canceled_discovery_output(workbench_db, scan) + earlier = worker_finding(scan, "rejected-candidate") + earlier["provenance"]["candidateId"] = "candidate" + checkpoint = write_checkpoint( + output / "checkpoints", saved_draft(scan.scan_id, findings=[earlier]) + ) + os.utime(checkpoint, ns=(100, 100)) + invalid = worker_finding(scan, "malformed-sibling") + invalid.pop("remediation") + outcome = { + "id": "review", + "label": "Candidate reviewed", + "candidateId": "candidate", + "disposition": "rejected", + "receiptRefs": [], + } + result = output / "result.json" + result.write_text( + json.dumps(saved_draft(scan.scan_id, complete=True, findings=[invalid], surfaces=[outcome])) + ) + os.utime(result, ns=(200, 200)) + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + complete(workbench_api, workbench_db, scan) + + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert all(row["title"] != earlier["title"] for row in findings) + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert ( + next(row for row in coverage["surfaces"] if row.get("workerId") == worker_id)["disposition"] + == "rejected" + ) + assert any("remediation" in row["reason"] for row in coverage["deferred"]) + + +@pytest.mark.parametrize("invalid", ["surfaces", "missing-receipt"]) +def test_deadline_keeps_canonical_coverage_validation_strict( + workbench_api, workbench_db, deadline_scan, invalid +): + from finalize_scan_contract import ContractError + + scan = deadline_scan + scan.coverage["surfaces"] = ( + "invalid" + if invalid == "surfaces" + else [ + { + "id": "accepted", + "label": "Accepted review", + "disposition": "no_issue_found", + "receiptRefs": ["artifacts/missing.txt"], + } + ] + ) + coverage_file = scan.scan_dir / "coverage.json" + coverage_file.write_text(json.dumps(scan.coverage)) + original = coverage_file.read_bytes() + _, output = canceled_discovery_output(workbench_db, scan) + write_checkpoint(output / "checkpoints", saved_draft(scan.scan_id)) + + with pytest.raises(ContractError, match="surfaces|receiptRefs"): + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + + assert coverage_file.read_bytes() == original + assert ( + workbench_db.execute( + "SELECT status FROM deep_scan_runs WHERE scan_id = ?", (scan.scan_id,) + ).fetchone()[0] + == "running" + ) + + +@pytest.mark.parametrize("worker_writeup_exists", [False, True], ids=["missing", "present"]) +def test_worker_writeups_remain_bound_to_their_source_directory( + workbench_api, workbench_db, deadline_scan, worker_writeup_exists +): + scan = deadline_scan + report_path = "findings/review/review.md" + scan.findings[0]["writeup"] = {"reportPath": report_path} + parent_writeup = scan.scan_dir / report_path + parent_writeup.parent.mkdir(parents=True) + parent_writeup.write_text("# Accepted parent writeup\n") + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": scan.findings})) + originals = {parent_writeup: parent_writeup.read_bytes()} + sources = [] + for archived in (False, True): + _, output = canceled_discovery_output(workbench_db, scan, attempt=2 if archived else 1) + source = output.parent / "attempts" / "attempt-1" if archived else output + sources.append(source.relative_to(scan.scan_dir).as_posix()) + finding = worker_finding(scan, "archived-worker" if archived else "current-worker") + if worker_writeup_exists: + writeup = source / report_path + writeup.parent.mkdir(parents=True) + writeup.write_text(f"# Separate worker writeup: {finding['title']}\n") + originals[writeup] = writeup.read_bytes() + checkpoint = write_checkpoint( + source / "checkpoints", saved_draft(scan.scan_id, findings=[finding]) + ) + originals[checkpoint] = checkpoint.read_bytes() + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + completed = complete(workbench_api, workbench_db, scan) + + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert len(findings) == 3 + assert next(row for row in findings if row["title"] == scan.findings[0]["title"])[ + "writeup" + ] == {"reportPath": report_path} + assert all( + "writeup" not in row for row in findings if row["title"] != scan.findings[0]["title"] + ) + writeup_warnings = [warning for warning in completed["warnings"] if "writeup" in warning] + assert len(writeup_warnings) == 2 + expected = ( + "Retained worker writeup at" if worker_writeup_exists else "Skipped malformed writeup" + ) + assert all(expected in warning for warning in writeup_warnings) + assert all(any(source in warning for warning in writeup_warnings) for source in sources) + report = (scan.scan_dir / "report.md").read_text() + assert "archived-worker" in report and "current-worker" in report + assert {path: path.read_bytes() for path in originals} == originals + + +@pytest.mark.skipif( + not hasattr(os, "fork"), reason="Crash boundary uses a forked workbench process." +) +@pytest.mark.parametrize( + ("colliding_ids", "parent_closure", "reviewed", "worker_like_id"), + [ + (False, False, False, False), + (True, False, False, False), + (False, True, False, False), + (True, True, False, False), + (False, True, True, False), + (False, True, True, True), + ], + ids=[ + "pending", + "legacy-collision", + "parent-closure", + "parent-closure-with-collision", + "reviewed-surface", + "reviewed-surface-with-worker-like-id", + ], +) +def test_deadline_recovery_is_idempotent_after_crash_before_commit( + workbench_api, + workbench_db, + deadline_scan, + tmp_path, + colliding_ids, + parent_closure, + reviewed, + worker_like_id, +): + scan = deadline_scan + if parent_closure: + scan.coverage["resolvedDeferred"] = [ + {"id": "prior-review", "reason": "Earlier review completed."} + ] + (scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage)) + database = tmp_path / "crash-recovery.db" + connection = sqlite3.connect(database) + connection.row_factory = sqlite3.Row + workbench_db.backup(connection) + worker_id, output = canceled_discovery_output(connection, scan) + surface_id = f"{worker_id}/entry" if worker_like_id else "entry" + surfaces = [ + { + "id": surface_id, + "label": "Worker review", + "disposition": "no_issue_found" if reviewed else "needs_follow_up", + "receiptRefs": [], + } + ] + if colliding_ids: + surfaces.append({**surfaces[0], "label": "Separate worker review"}) + write_checkpoint( + output / "checkpoints", + saved_draft( + scan.scan_id, + surfaces=surfaces, + findings=[] + if reviewed + else [worker_finding(scan, "saved-before-crash"), copy.deepcopy(scan.findings[0])], + deferred=[] + if reviewed + else [{"id": "review", "reason": "Review pending.", "surfaceIds": [surface_id]}], + ), + ) + connection.close() + interrupt_finish_before_commit(workbench_api, database, scan) + before = json.loads((scan.scan_dir / "coverage.json").read_text()) + findings_before = json.loads((scan.scan_dir / "findings.json").read_text()) + assert len([row for row in before["surfaces"] if row.get("workerId") == worker_id]) == len( + surfaces + ) + if reviewed: + assert before["surfaces"][0]["id"] == f"{worker_id}/{surface_id}" + assert len([row for row in before["deferred"] if row["id"] == "scan-deadline"]) == 1 + with sqlite3.connect(database) as connection: + connection.row_factory = sqlite3.Row + assert ( + connection.execute( + "SELECT status FROM deep_scan_runs WHERE scan_id = ?", (scan.scan_id,) + ).fetchone()[0] + == "running" + ) + finish(workbench_api, connection, scan, terminal_reason="capped") + complete(workbench_api, connection, scan) + + after = json.loads((scan.scan_dir / "coverage.json").read_text()) + findings_after = json.loads((scan.scan_dir / "findings.json").read_text()) + for document in (findings_before, findings_after): + for finding in document["findings"]: + # Existing recovery can materialize an empty optional history list. + if finding["provenance"].get("previousFindings") == []: + finding["provenance"].pop("previousFindings") + assert findings_after == findings_before + assert after == before + + +@pytest.mark.skipif( + not hasattr(os, "fork"), reason="Crash boundary uses a forked workbench process." +) +@pytest.mark.parametrize("earlier_checkpoint_first", [False, True]) +@pytest.mark.parametrize("colliding_ids", [False, True]) +def test_deadline_rejection_history_is_stable_across_repeated_crashes( + workbench_api, workbench_db, deadline_scan, tmp_path, earlier_checkpoint_first, colliding_ids +): + scan = deadline_scan + worker_id, output = canceled_discovery_output(workbench_db, scan) + finding = worker_finding(scan, "reviewed-candidate") + finding["provenance"]["candidateId"] = "candidate" + # Check both hash filename orders without adding invalid or unrelated checkpoints. + for attempt in range(1000): + finding["title"] = f"Candidate review {attempt}" + earlier = saved_draft(scan.scan_id, findings=[finding]) + outcomes = [ + { + "id": "entry", + "candidateId": "candidate", + "finding": copy.deepcopy(finding), + "label": "Candidate reviewed", + "disposition": "rejected", + "receiptRefs": [], + } + ] + if colliding_ids: + outcomes.append({**copy.deepcopy(outcomes[0]), "label": "Separate observation"}) + terminal = saved_draft(scan.scan_id, complete=True, surfaces=outcomes) + earlier_hash = hashlib.sha256(json.dumps(earlier).encode()).hexdigest() + terminal_hash = hashlib.sha256(json.dumps(terminal).encode()).hexdigest() + if (earlier_hash < terminal_hash) == earlier_checkpoint_first: + break + else: + pytest.fail("Could not select checkpoint filename order") + old_checkpoint = write_checkpoint(output / "checkpoints", earlier) + checkpoint = write_checkpoint(output / "checkpoints", terminal) + assert (old_checkpoint.name < checkpoint.name) == earlier_checkpoint_first + os.utime(old_checkpoint, ns=(100, 100)) + os.utime(checkpoint, ns=(200, 200)) + head = output / "checkpoint-head.json" + head.write_text(json.dumps({"checkpoint": checkpoint.name})) + (output / "result.json").unlink(missing_ok=True) + originals = {path: path.read_bytes() for path in (old_checkpoint, checkpoint, head)} + database = tmp_path / "repeated-crash.db" + with sqlite3.connect(database) as connection: + workbench_db.backup(connection) + + publications = [] + for _ in range(3): + interrupt_finish_before_commit(workbench_api, database, scan) + publications.append(json.loads((scan.scan_dir / "coverage.json").read_text())) + with sqlite3.connect(database) as connection: + assert ( + connection.execute( + "SELECT status FROM deep_scan_runs WHERE scan_id = ?", (scan.scan_id,) + ).fetchone()[0] + == "running" + ) + with sqlite3.connect(database) as connection: + connection.row_factory = sqlite3.Row + finish(workbench_api, connection, scan, terminal_reason="capped") + complete(workbench_api, connection, scan) + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert all(publication == coverage for publication in publications) + surfaces = [row for row in coverage["surfaces"] if row.get("workerId") == worker_id] + assert {row["label"] for row in surfaces} == {row["label"] for row in outcomes} + assert len(surfaces) == len({row["id"] for row in surfaces}) == len(outcomes) + assert all(len(row["previousFindings"]) == 1 for row in surfaces) + assert all(row["previousFindings"][0]["title"] == finding["title"] for row in surfaces) + assert all(row["disposition"] == "rejected" for row in surfaces) + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert len(findings) == 1 and findings[0]["title"] == scan.findings[0]["title"] + assert {path: path.read_bytes() for path in originals} == originals diff --git a/plugins/codex-security/tests/test_deep_scan_deadline_surface_links.py b/plugins/codex-security/tests/test_deep_scan_deadline_surface_links.py new file mode 100644 index 000000000..13b693904 --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_deadline_surface_links.py @@ -0,0 +1,603 @@ +from __future__ import annotations + +import json +import os +import sqlite3 +import subprocess +import sys +from pathlib import Path + +import pytest +from test_deep_scan_stop_conditions import canceled_discovery_output, complete, finish +from test_deep_scan_stop_conditions import deadline_scan as deadline_scan +from test_deep_scan_stop_conditions import publication_scan as publication_scan +from test_deep_scan_stop_conditions import saturated_scan as saturated_scan +from workbench_test_support import ( + replay_saved_results, + saved_binding, + saved_discovery_worker, + saved_draft, + write_checkpoint, +) + + +def surface(label, identity="entry", disposition="needs_follow_up"): + return {"id": identity, "label": label, "disposition": disposition, "receiptRefs": []} + + +def pending(identity, reason, surface_id="entry"): + return {"id": identity, "reason": reason, "surfaceIds": [surface_id]} + + +def interrupt_in_fresh_python(database, scan, hash_seed): + script = Path(__file__).resolve().parents[1] / "scripts" / "workbench_db.py" + result = subprocess.run( + [ + sys.executable, + "-B", + "-c", + """ +import runpy +import sys +from pathlib import Path +from types import SimpleNamespace +sys.path.insert(0, str(Path(sys.argv[1]).parents[1] / "tests")) +from test_deep_scan_deadline_recovery import interrupt_finish_before_commit + +api = runpy.run_path(sys.argv[1], run_name="surface_link_recovery_test") +api["deep_scan"]._dependencies = SimpleNamespace( + **api, preserve_stopped_results=api["preserve_stopped_results_after_transition"] +) +scan = SimpleNamespace(scan_id=sys.argv[3], scan_dir=Path(sys.argv[4])) +interrupt_finish_before_commit(api, Path(sys.argv[2]), scan) +""", + str(script), + str(database), + scan.scan_id, + str(scan.scan_dir), + ], + cwd=Path(__file__).parent, + env={**os.environ, "PYTHONHASHSEED": str(hash_seed)}, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + + +@pytest.mark.skipif( + not hasattr(os, "fork"), reason="Crash boundary uses a forked workbench process." +) +@pytest.mark.parametrize("parent_collision", [False, True]) +def test_deadline_surface_links_follow_sources_across_repeated_crashes( + workbench_api, workbench_db, deadline_scan, tmp_path, parent_collision +): + scan = deadline_scan + worker_id, output = canceled_discovery_output(workbench_db, scan, attempt=2) + archived = output.parent / "attempts" / "attempt-01" + shared = pending("shared-review", "Review all retained observations.") + checkpoints = [] + for index, (source, label) in enumerate( + [(archived, "download"), (output, "upload"), (output, "export")], start=1 + ): + # Each is a valid raw input checkpoint saved before the writer merges history. + checkpoint = write_checkpoint( + source / "checkpoints", + saved_draft( + scan.scan_id, + surfaces=[surface(label)], + deferred=[pending(f"{label}-review", f"Review {label}."), shared], + ), + ) + os.utime(checkpoint, ns=(index * 100, index * 100)) + checkpoints.append(checkpoint) + canonical_id = f"{worker_id}/entry" if parent_collision else "parent-entry" + canonical_surface = surface("Canonical parent", canonical_id) + canonical_pending = pending("parent-review", "Parent review remains.", canonical_id) + scan.coverage.update(surfaces=[canonical_surface], deferred=[canonical_pending]) + (scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage)) + originals = {path: path.read_bytes() for path in checkpoints} + database = tmp_path / "surface-links.db" + with sqlite3.connect(database) as connection: + workbench_db.backup(connection) + + publications = [] + for hash_seed in (1, 17, 83): + interrupt_in_fresh_python(database, scan, hash_seed) + publications.append(json.loads((scan.scan_dir / "coverage.json").read_text())) + with sqlite3.connect(database) as connection: + assert ( + connection.execute( + "SELECT status FROM deep_scan_runs WHERE scan_id = ?", (scan.scan_id,) + ).fetchone()[0] + == "running" + ) + with sqlite3.connect(database) as connection: + connection.row_factory = sqlite3.Row + finish(workbench_api, connection, scan, terminal_reason="capped") + completed = complete(workbench_api, connection, scan) + + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert completed["progress"]["status"] == "complete" + assert all(publication == coverage for publication in publications) + assert canonical_surface in coverage["surfaces"] + assert canonical_pending in coverage["deferred"] + surfaces = [row for row in coverage["surfaces"] if row.get("workerId") == worker_id] + assert {row["label"] for row in surfaces} == {"download", "upload", "export"} + assert len(surfaces) == len({row["id"] for row in surfaces}) == 3 + tasks = [row for row in coverage["deferred"] if row.get("workerId") == worker_id] + for row in surfaces: + task = next(item for item in tasks if item["reason"] == f"Review {row['label']}.") + assert task["surfaceIds"] == [row["id"]] + shared_tasks = [row for row in tasks if row["reason"] == shared["reason"]] + assert len(shared_tasks) == 1 + assert shared_tasks[0]["surfaceIds"] == [row["id"] for row in surfaces] + assert {path: path.read_bytes() for path in originals} == originals + + +@pytest.mark.parametrize("with_reduction", [False, True], ids=["zero-accepted", "reduced"]) +def test_deadline_retains_all_targets_of_one_legacy_surface_id( + workbench_api, workbench_db, publication_scan, request, with_reduction +): + scan = request.getfixturevalue("deadline_scan") if with_reduction else publication_scan() + if not with_reduction: + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', phase = 'discovery', " + "created_at = '2000-01-01T00:00:00+00:00', manifest_path = NULL, " + "terminal_reason = NULL, completed_at = NULL WHERE scan_id = ?", + (scan.scan_id,), + ) + _, output = canceled_discovery_output(workbench_db, scan) + checkpoint = write_checkpoint( + output / "checkpoints", + saved_draft( + scan.scan_id, + surfaces=[surface("download"), surface("upload")], + deferred=[pending("review", "Review both legacy observations.")], + ), + ) + original = checkpoint.read_bytes() + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + complete(workbench_api, workbench_db, scan) + + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + surfaces = [row for row in coverage["surfaces"] if row["label"] in {"download", "upload"}] + assert len(surfaces) == len({row["id"] for row in surfaces}) == 2 + task = next( + row for row in coverage["deferred"] if row["reason"] == "Review both legacy observations." + ) + assert task["surfaceIds"] == [row["id"] for row in surfaces] + assert checkpoint.read_bytes() == original + + +def test_deadline_deferred_only_checkpoint_keeps_prior_surface_link( + workbench_api, workbench_db, deadline_scan +): + scan = deadline_scan + worker_id, output = canceled_discovery_output(workbench_db, scan) + earlier = write_checkpoint( + output / "checkpoints", + saved_draft(scan.scan_id, surfaces=[surface("Earlier review", "prior")]), + ) + current = write_checkpoint( + output / "checkpoints", + saved_draft(scan.scan_id, deferred=[pending("review", "Review prior surface.", "prior")]), + ) + os.utime(earlier, ns=(100, 100)) + os.utime(current, ns=(200, 200)) + originals = {path: path.read_bytes() for path in (earlier, current)} + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + complete(workbench_api, workbench_db, scan) + + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + retained = next(row for row in coverage["surfaces"] if row["label"] == "Earlier review") + task = next(row for row in coverage["deferred"] if row["reason"] == "Review prior surface.") + assert task["surfaceIds"] == [retained["id"]] == [f"{worker_id}/prior"] + assert {path: path.read_bytes() for path in originals} == originals + + +def test_deadline_reopened_generic_surface_copy_retains_its_link( + workbench_api, workbench_db, deadline_scan +): + scan = deadline_scan + worker_id, output = canceled_discovery_output(workbench_db, scan) + canonical = surface("Canonical parent", f"{worker_id}/entry") + scan.coverage["surfaces"] = [canonical] + (scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage)) + drafts = [ + saved_draft( + scan.scan_id, + surfaces=[surface("Worker review")], + deferred=[pending("review", "Initial review pending.")], + ), + saved_draft( + scan.scan_id, + complete=True, + surfaces=[surface("Worker review", disposition="no_issue_found")], + closures=[{"id": "review", "reason": "Initial review completed."}], + ), + saved_draft( + scan.scan_id, + surfaces=[surface("Worker review")], + deferred=[pending("review", "Review reopened.")], + ), + ] + originals = {} + for index, draft in enumerate(drafts, start=1): + checkpoint = write_checkpoint(output / "checkpoints", draft) + os.utime(checkpoint, ns=(index * 100, index * 100)) + originals[checkpoint] = checkpoint.read_bytes() + head = output / "checkpoint-head.json" + head.write_text(json.dumps({"checkpoint": checkpoint.name})) + originals[head] = head.read_bytes() + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + complete(workbench_api, workbench_db, scan) + + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + retained = [row for row in coverage["surfaces"] if row.get("workerId") == worker_id] + assert len(retained) == 1 + assert retained[0]["disposition"] == "needs_follow_up" + task = next(row for row in coverage["deferred"] if row["reason"] == "Review reopened.") + assert task["surfaceIds"] == [retained[0]["id"]] + assert canonical in coverage["surfaces"] + assert {path: path.read_bytes() for path in originals} == originals + + +def test_stopped_recovery_preserves_source_link_union(workbench_api, tmp_path): + saved = workbench_api["saved_results"] + scan_id = "surface-links" + binding = saved_binding("deep_repository") + output = tmp_path / "worker" / "output" + archived = output.parent / "attempts" / "attempt-01" + shared = pending("review", "Review retained observations.") + originals = {} + for source, label in ((archived, "download"), (output, "upload")): + checkpoint = write_checkpoint( + source / "checkpoints", + saved_draft(scan_id, surfaces=[surface(label)], deferred=[shared]), + ) + originals[checkpoint] = checkpoint.read_bytes() + workers = [saved_discovery_worker(output, "worker", 2)] + + documents = saved.merge_saved_results( + tmp_path, scan_id, binding, workers, [], stopped=True, reason="interrupted" + ) + replay = replay_saved_results(saved, documents, tmp_path, scan_id, binding, workers) + + assert replay[2] == documents[2] + coverage = documents[2] + assert {row["label"] for row in coverage["surfaces"]} == {"download", "upload"} + tasks = [row for row in coverage["deferred"] if row["reason"] == shared["reason"]] + assert len(tasks) == 1 + assert tasks[0]["surfaceIds"] == [row["id"] for row in coverage["surfaces"]] + assert {path: path.read_bytes() for path in originals} == originals + + +@pytest.mark.skipif( + not hasattr(os, "fork"), reason="Crash boundary uses a forked workbench process." +) +@pytest.mark.parametrize("changed_task", [False, True], ids=["copied-task", "updated-task"]) +def test_deadline_retains_equal_task_origins_from_selected_worker_result( + workbench_api, workbench_db, deadline_scan, tmp_path, changed_task +): + scan = deadline_scan + worker_id, output = canceled_discovery_output(workbench_db, scan, attempt=2) + archived = output.parent / "attempts" / "attempt-01" + previous_task = pending("download-review", "Download review pending.") + archived_draft = saved_draft( + scan.scan_id, surfaces=[surface("download")], deferred=[previous_task] + ) + checkpoint = write_checkpoint(archived / "checkpoints", archived_draft) + os.utime(checkpoint, ns=(100, 100)) + archived_head = archived / "checkpoint-head.json" + archived_head.write_text(json.dumps({"checkpoint": checkpoint.name})) + os.utime(archived_head, ns=(110, 110)) + archived_result = archived / "result.json" + archived_result.write_bytes(checkpoint.read_bytes()) + os.utime(archived_result, ns=(120, 120)) + current_task = pending("upload-review", "Upload review pending.") + retained_task = ( + pending("download-review", "Download review updated.") if changed_task else previous_task + ) + incoming = saved_draft( + scan.scan_id, + surfaces=[surface("upload")], + deferred=[current_task, retained_task] if changed_task else [current_task], + ) + raw_checkpoint = write_checkpoint(output / "checkpoints", incoming) + os.utime(raw_checkpoint, ns=(200, 200)) + # The writer preserves the old pending task while publishing its latest surface. + merged = saved_draft( + scan.scan_id, surfaces=[surface("upload")], deferred=[current_task, retained_task] + ) + selected = write_checkpoint(output / "checkpoints", merged) + os.utime(selected, ns=(210, 210)) + head = output / "checkpoint-head.json" + head.write_text(json.dumps({"checkpoint": selected.name})) + os.utime(head, ns=(220, 220)) + result = output / "result.json" + result.write_bytes(selected.read_bytes()) + os.utime(result, ns=(230, 230)) + originals = {path: path.read_bytes() for path in output.parent.rglob("*") if path.is_file()} + database = tmp_path / "selected-result.db" + with sqlite3.connect(database) as connection: + workbench_db.backup(connection) + + publications = [] + for hash_seed in (1, 17, 83): + interrupt_in_fresh_python(database, scan, hash_seed) + publications.append(json.loads((scan.scan_dir / "coverage.json").read_text())) + with sqlite3.connect(database) as connection: + connection.row_factory = sqlite3.Row + finish(workbench_api, connection, scan, terminal_reason="capped") + complete(workbench_api, connection, scan) + + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert all(publication == coverage for publication in publications) + surfaces = [row for row in coverage["surfaces"] if row.get("workerId") == worker_id] + assert {row["label"] for row in surfaces} == {"download", "upload"} + upload = next(row for row in surfaces if row["label"] == "upload") + tasks = [row for row in coverage["deferred"] if row.get("workerId") == worker_id] + assert len(tasks) == 2 + current = next(row for row in tasks if row["reason"] == current_task["reason"]) + retained = next(row for row in tasks if row["reason"] == retained_task["reason"]) + assert current["surfaceIds"] == [upload["id"]] + assert retained["surfaceIds"] == ( + [upload["id"]] if changed_task else [row["id"] for row in surfaces] + ) + assert {path: path.read_bytes() for path in originals} == originals + + +@pytest.mark.skipif( + not hasattr(os, "fork"), reason="Crash boundary uses a forked workbench process." +) +@pytest.mark.parametrize( + "selected_result", [False, True], ids=["raw-checkpoint", "selected-result"] +) +def test_deadline_dropped_legacy_surface_does_not_rebind_its_pending_task( + workbench_api, workbench_db, deadline_scan, tmp_path, selected_result +): + scan = deadline_scan + worker_id, output = canceled_discovery_output(workbench_db, scan, attempt=2) + archived = output.parent / "attempts" / "attempt-01" + download_task = pending("download-review", "Download review pending.") + archived_checkpoint = write_checkpoint( + archived / "checkpoints", + saved_draft(scan.scan_id, surfaces=[surface("download")], deferred=[download_task]), + ) + os.utime(archived_checkpoint, ns=(100, 100)) + archived_result = archived / "result.json" + archived_result.write_bytes(archived_checkpoint.read_bytes()) + os.utime(archived_result, ns=(110, 110)) + upload_task = pending("upload-review", "Upload review pending.") + # Historical artifacts can contain labels rejected during finalization. + malformed_surface = surface("\u0085") + raw_checkpoint = write_checkpoint( + output / "checkpoints", + saved_draft(scan.scan_id, surfaces=[malformed_surface], deferred=[upload_task]), + ) + os.utime(raw_checkpoint, ns=(200, 200)) + if selected_result: + selected = write_checkpoint( + output / "checkpoints", + saved_draft( + scan.scan_id, + surfaces=[malformed_surface], + deferred=[upload_task, download_task], + ), + ) + os.utime(selected, ns=(210, 210)) + head = output / "checkpoint-head.json" + head.write_text(json.dumps({"checkpoint": selected.name})) + os.utime(head, ns=(220, 220)) + result = output / "result.json" + result.write_bytes(selected.read_bytes()) + os.utime(result, ns=(230, 230)) + originals = {path: path.read_bytes() for path in output.parent.rglob("*") if path.is_file()} + database = tmp_path / "dropped-surface.db" + with sqlite3.connect(database) as connection: + workbench_db.backup(connection) + + publications = [] + for hash_seed in (1, 17, 83): + interrupt_in_fresh_python(database, scan, hash_seed) + publications.append(json.loads((scan.scan_dir / "coverage.json").read_text())) + with sqlite3.connect(database) as connection: + connection.row_factory = sqlite3.Row + finish(workbench_api, connection, scan, terminal_reason="capped") + completed = complete(workbench_api, connection, scan) + + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert completed["progress"]["status"] == "complete" + assert all(publication == coverage for publication in publications) + surfaces = [row for row in coverage["surfaces"] if row.get("workerId") == worker_id] + assert len(surfaces) == 1 and surfaces[0]["label"] == "download" + tasks = [row for row in coverage["deferred"] if row.get("workerId") == worker_id] + retained_upload = next(row for row in tasks if row["reason"] == upload_task["reason"]) + retained_download = next(row for row in tasks if row["reason"] == download_task["reason"]) + assert retained_upload["surfaceIds"] == [] + assert retained_download["surfaceIds"] == [surfaces[0]["id"]] + assert any( + raw_checkpoint.relative_to(scan.scan_dir).as_posix() in warning + and "coverage.surfaces[0].label: expected a non-empty string" in warning + for warning in completed["warnings"] + ) + assert {path: path.read_bytes() for path in originals} == originals + + +@pytest.mark.skipif( + not hasattr(os, "fork"), reason="Crash boundary uses a forked workbench process." +) +@pytest.mark.parametrize( + "history", + ["parent-collision", "current-checkpoints", "retry-checkpoints", "coalesced-history"], +) +def test_deadline_inherited_surface_links_follow_historical_observations( + workbench_api, workbench_db, deadline_scan, tmp_path, history +): + scan = deadline_scan + worker_id, output = canceled_discovery_output( + workbench_db, scan, attempt=2 if history == "retry-checkpoints" else 1 + ) + archived = output.parent / "attempts" / "attempt-01" + first_output = archived if history == "retry-checkpoints" else output + earlier = write_checkpoint( + first_output / "checkpoints", + saved_draft(scan.scan_id, surfaces=[surface("Earlier observation")]), + ) + os.utime(earlier, ns=(100, 100)) + expected_labels = {"Earlier observation"} + if history != "parent-collision": + newer = write_checkpoint( + output / "checkpoints", + saved_draft(scan.scan_id, surfaces=[surface("Newer observation")]), + ) + os.utime(newer, ns=(200, 200)) + expected_labels.add("Newer observation") + inherited_task = pending("history-review", "Review the inherited surface.") + unknown_task = pending("unknown-review", "Review an unresolved reference.", "not-declared") + deferred_only = write_checkpoint( + output / "checkpoints", + saved_draft(scan.scan_id, deferred=[inherited_task, unknown_task]), + ) + os.utime(deferred_only, ns=(300, 300)) + if history == "coalesced-history": + # The merged checkpoint copies a task whose earlier raw source inherited history. + selected = write_checkpoint( + output / "checkpoints", + saved_draft( + scan.scan_id, + surfaces=[surface("Newer observation")], + deferred=[inherited_task, unknown_task], + ), + ) + os.utime(selected, ns=(400, 400)) + head = output / "checkpoint-head.json" + head.write_text(json.dumps({"checkpoint": selected.name})) + os.utime(head, ns=(410, 410)) + if history == "current-checkpoints": + # An interrupted later write cannot identify an earlier task's target. + future = write_checkpoint( + output / "checkpoints", + saved_draft(scan.scan_id, surfaces=[surface("Later observation")]), + ) + os.utime(future, ns=(400, 400)) + canonical_id = f"{worker_id}/entry" if history == "parent-collision" else "parent-entry" + canonical_surface = surface("Canonical parent", canonical_id) + canonical_task = pending("parent-review", "Parent review remains.", canonical_id) + scan.coverage.update(surfaces=[canonical_surface], deferred=[canonical_task]) + (scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage)) + originals = {path: path.read_bytes() for path in output.parent.rglob("*") if path.is_file()} + database = tmp_path / "historical-links.db" + with sqlite3.connect(database) as connection: + workbench_db.backup(connection) + + publications = [] + for hash_seed in (1, 17, 83): + interrupt_in_fresh_python(database, scan, hash_seed) + publications.append(json.loads((scan.scan_dir / "coverage.json").read_text())) + with sqlite3.connect(database) as connection: + connection.row_factory = sqlite3.Row + finish(workbench_api, connection, scan, terminal_reason="capped") + completed = complete(workbench_api, connection, scan) + + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert completed["progress"]["status"] == "complete" + assert all(publication == coverage for publication in publications) + assert canonical_surface in coverage["surfaces"] + assert canonical_task in coverage["deferred"] + worker_surfaces = [row for row in coverage["surfaces"] if row.get("workerId") == worker_id] + historical = [row for row in worker_surfaces if row["label"] in expected_labels] + assert {row["label"] for row in historical} == expected_labels + assert len(historical) == len(expected_labels) + if history == "current-checkpoints": + assert any(row["label"] == "Later observation" for row in worker_surfaces) + tasks = [row for row in coverage["deferred"] if row.get("workerId") == worker_id] + inherited = next(row for row in tasks if row["reason"] == inherited_task["reason"]) + unknown = next(row for row in tasks if row["reason"] == unknown_task["reason"]) + assert inherited["surfaceIds"] == [row["id"] for row in historical] + assert unknown["surfaceIds"] == [f"{worker_id}/not-declared"] + assert {path: path.read_bytes() for path in originals} == originals + + +@pytest.mark.skipif( + not hasattr(os, "fork"), reason="Crash boundary uses a forked workbench process." +) +@pytest.mark.parametrize( + ("changed_surface_id", "legacy_duplicate_id"), + [(False, False), (True, False), (True, True)], + ids=["same-id", "changed-id", "legacy-duplicate-id"], +) +def test_deadline_candidate_supersession_preserves_independent_surface_links( + workbench_api, workbench_db, deadline_scan, tmp_path, changed_surface_id, legacy_duplicate_id +): + scan = deadline_scan + worker_id, output = canceled_discovery_output(workbench_db, scan) + canonical_id = f"{worker_id}/entry" + canonical_surface = surface("Canonical parent", canonical_id) + canonical_task = pending("parent-review", "Parent review remains.", canonical_id) + scan.coverage.update(surfaces=[canonical_surface], deferred=[canonical_task]) + (scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage)) + independent = pending("independent-review", "Review another caller.") + candidate_surface = {**surface("Candidate review"), "candidateId": "candidate"} + earlier_surfaces = [candidate_surface] + if legacy_duplicate_id: + # Legacy artifacts may reuse an ID for a candidate and an independent observation. + earlier_surfaces.append(surface("Independent observation")) + earlier = write_checkpoint( + output / "checkpoints", + saved_draft(scan.scan_id, surfaces=earlier_surfaces, deferred=[independent]), + ) + os.utime(earlier, ns=(100, 100)) + outcome_id = "outcome" if changed_surface_id else "entry" + outcome = { + **surface("Candidate reviewed", outcome_id, "rejected"), + "candidateId": "candidate", + } + raw_checkpoint = write_checkpoint( + output / "checkpoints", saved_draft(scan.scan_id, complete=True, surfaces=[outcome]) + ) + os.utime(raw_checkpoint, ns=(200, 200)) + # The selected merged checkpoint retains independent work before result publication. + selected = write_checkpoint( + output / "checkpoints", + saved_draft(scan.scan_id, complete=True, surfaces=[outcome], deferred=[independent]), + ) + os.utime(selected, ns=(210, 210)) + head = output / "checkpoint-head.json" + head.write_text(json.dumps({"checkpoint": selected.name})) + os.utime(head, ns=(220, 220)) + originals = {path: path.read_bytes() for path in output.rglob("*") if path.is_file()} + database = tmp_path / "candidate-links.db" + with sqlite3.connect(database) as connection: + workbench_db.backup(connection) + + publications = [] + for hash_seed in (1, 17, 83): + interrupt_in_fresh_python(database, scan, hash_seed) + publications.append(json.loads((scan.scan_dir / "coverage.json").read_text())) + with sqlite3.connect(database) as connection: + connection.row_factory = sqlite3.Row + finish(workbench_api, connection, scan, terminal_reason="capped") + completed = complete(workbench_api, connection, scan) + + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert completed["progress"]["status"] == "complete" + assert all(publication == coverage for publication in publications) + assert canonical_surface in coverage["surfaces"] + assert canonical_task in coverage["deferred"] + retained = [row for row in coverage["surfaces"] if row.get("workerId") == worker_id] + assert len(retained) == (2 if legacy_duplicate_id else 1) + outcome = next(row for row in retained if row["label"] == "Candidate reviewed") + assert outcome["disposition"] == "rejected" + if legacy_duplicate_id: + observation = next(row for row in retained if row["label"] == "Independent observation") + assert observation["disposition"] == "needs_follow_up" + assert all(row["id"] != canonical_id for row in retained) + tasks = [row for row in coverage["deferred"] if row.get("workerId") == worker_id] + assert len(tasks) == 1 and tasks[0]["reason"] == independent["reason"] + assert tasks[0]["surfaceIds"] == [row["id"] for row in retained] + assert {path: path.read_bytes() for path in originals} == originals diff --git a/plugins/codex-security/tests/test_deep_scan_stop_conditions.py b/plugins/codex-security/tests/test_deep_scan_stop_conditions.py index 1c2389e15..6e67fbf2c 100644 --- a/plugins/codex-security/tests/test_deep_scan_stop_conditions.py +++ b/plugins/codex-security/tests/test_deep_scan_stop_conditions.py @@ -1,5 +1,6 @@ from __future__ import annotations +import copy import json import os import sqlite3 @@ -72,6 +73,299 @@ def finish(workbench_api, connection, scan, *, terminal_reason="saturated"): )["deepScan"] +@pytest.fixture +def deadline_scan(saturated_scan, workbench_db): + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_runs SET created_at = '2000-01-01T00:00:00+00:00' WHERE scan_id = ?", + (saturated_scan.scan_id,), + ) + return saturated_scan + + +def canceled_discovery_output(connection, scan, *, attempt=1): + result = add_worker(connection, scan, status="canceled") + worker_id = result.parent.name + output = scan.scan_dir / "artifacts" / "deep_discovery" / "workers" / worker_id / "output" + output.parent.mkdir(parents=True) + result.parent.rename(output) + with connection: + connection.execute( + "UPDATE deep_scan_workers SET artifact_dir = ?, prompt_path = ?, " + "result_manifest_path = NULL, attempt = ? WHERE id = ?", + (str(output), str(output.parent / "prompt.md"), attempt, worker_id), + ) + return worker_id, output + + +@pytest.mark.parametrize("missing_writeup", [False, True], ids=["present", "missing"]) +def test_deadline_preserves_optional_reduced_writeup_behavior( + workbench_api, workbench_db, deadline_scan, missing_writeup +): + scan = deadline_scan + report_path = "findings/reduced/reduced.md" + scan.findings[0]["writeup"] = {"reportPath": report_path} + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": scan.findings})) + if not missing_writeup: + report = scan.scan_dir / report_path + report.parent.mkdir(parents=True) + report.write_text("# Reduced finding\n") + _, output = canceled_discovery_output(workbench_db, scan) + write_checkpoint( + output / "checkpoints", + saved_draft(scan.scan_id, deferred=[{"reason": "Review the remaining upload paths."}]), + ) + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + completed = complete(workbench_api, workbench_db, scan) + + assert completed["progress"]["status"] == "complete" + finding = json.loads((scan.scan_dir / "findings.json").read_text())["findings"][0] + assert finding["title"] == scan.findings[0]["title"] + assert ("writeup" not in finding) == missing_writeup + assert any("Skipped missing writeup" in warning for warning in completed["warnings"]) == ( + missing_writeup + ) + + +def test_deadline_recovers_valid_findings_from_an_invalid_archived_retry( + workbench_api, workbench_db, deadline_scan +): + scan = deadline_scan + worker_id, output = canceled_discovery_output(workbench_db, scan, attempt=2) + archived = output.parent / "attempts" / "attempt-1" + archived.mkdir(parents=True) + valid = copy.deepcopy(scan.findings[0]) + valid["identity"]["anchor"] = "valid-archived-finding" + valid["title"] = "Valid finding saved before retry" + valid["provenance"] = {"source": "local_plugin"} + invalid = copy.deepcopy(valid) + invalid["identity"]["anchor"] = "invalid-archived-finding" + invalid.pop("remediation") + result = archived / "result.json" + result.write_text( + json.dumps(saved_draft(scan.scan_id, complete=True, findings=[valid, invalid])) + ) + original = result.read_bytes() + write_checkpoint( + output / "checkpoints", + saved_draft(scan.scan_id, deferred=[{"reason": "Retry review remains unfinished."}]), + ) + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + completed = complete(workbench_api, workbench_db, scan) + + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert {item["identity"]["anchor"] for item in findings} == { + scan.findings[0]["identity"]["anchor"], + valid["identity"]["anchor"], + } + recovered = next(item for item in findings if item["title"] == valid["title"]) + assert recovered["provenance"]["workerId"] == worker_id + assert any("remediation" in warning for warning in completed["warnings"]) + manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"] + assert manifest["preservedSources"][result.relative_to(scan.scan_dir).as_posix()] == ( + workbench_api["saved_results"]._digest(json.loads(original)) + ) + assert result.read_bytes() == original + + +@pytest.mark.parametrize("invalid", ["finding", "unsafe-writeup"]) +def test_deadline_does_not_recover_invalid_reduced_findings( + workbench_api, workbench_db, deadline_scan, invalid +): + from finalize_scan_contract import ContractError + + scan = deadline_scan + if invalid == "finding": + scan.findings[0].pop("remediation") + else: + scan.findings[0]["writeup"] = {"reportPath": "../invalid.md"} + findings_path = scan.scan_dir / "findings.json" + findings_path.write_text(json.dumps({"findings": scan.findings})) + original = findings_path.read_bytes() + _, output = canceled_discovery_output(workbench_db, scan) + write_checkpoint(output / "checkpoints", saved_draft(scan.scan_id)) + + with pytest.raises(ContractError, match="remediation|reportPath"): + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + + assert findings_path.read_bytes() == original + assert ( + workbench_db.execute( + "SELECT status FROM deep_scan_runs WHERE scan_id = ?", (scan.scan_id,) + ).fetchone()[0] + == "running" + ) + + +def test_deadline_preserves_worker_findings_with_optional_and_shared_identity( + workbench_api, workbench_db, deadline_scan +): + scan = deadline_scan + _, output = canceled_discovery_output(workbench_db, scan) + no_identity = copy.deepcopy(scan.findings[0]) + no_identity.pop("identity") + no_identity["title"] = "Worker finding without authored identity" + no_identity["provenance"] = {"source": "local_plugin"} + first = copy.deepcopy(no_identity) + first["title"] = "Shared semantic anchor at distinct locations" + first["identity"] = {"anchor": "shared-worker-anchor"} + first["locations"] = [{"path": "src/first.py", "startLine": 1}] + second = copy.deepcopy(first) + second["locations"] = [{"path": "src/second.py", "startLine": 1}] + malformed = copy.deepcopy(first) + malformed.pop("remediation") + write_checkpoint( + output / "checkpoints", + saved_draft(scan.scan_id, findings=[no_identity, first, second, malformed]), + ) + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + completed = complete(workbench_api, workbench_db, scan) + + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert len(findings) == 4 + assert any(item["title"] == no_identity["title"] for item in findings) + shared = [item for item in findings if item["identity"]["anchor"] == "shared-worker-anchor"] + assert {item["locations"][0]["path"] for item in shared} == {"src/first.py", "src/second.py"} + assert any("remediation" in warning for warning in completed["warnings"]) + + +def test_deadline_resolves_worker_local_deferred_ids_before_scoping( + workbench_api, workbench_db, deadline_scan +): + scan = deadline_scan + _, output = canceled_discovery_output(workbench_db, scan) + previous = write_checkpoint( + output / "checkpoints", + saved_draft(scan.scan_id, deferred=[{"id": "review", "reason": "Review pending."}]), + ) + os.utime(previous, ns=(100, 100)) + current = saved_draft(scan.scan_id, complete=True) + current["coverage"]["resolvedDeferred"] = [{"id": "review", "reason": "Review completed."}] + latest = write_checkpoint(output / "checkpoints", current) + os.utime(latest, ns=(200, 200)) + (output / "checkpoint-head.json").write_text(json.dumps({"checkpoint": latest.name})) + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + complete(workbench_api, workbench_db, scan) + + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert not any(item["reason"] == "Review pending." for item in coverage["deferred"]) + + +def test_deadline_does_not_turn_invalid_worker_receipts_into_parent_artifacts( + workbench_api, workbench_db, deadline_scan +): + scan = deadline_scan + _, output = canceled_discovery_output(workbench_db, scan) + (output / "notes.txt").write_text("Not a worker receipt under artifacts/.\n") + write_checkpoint( + output / "checkpoints", + saved_draft( + scan.scan_id, + surfaces=[ + { + "id": "entry", + "label": "Unverified review", + "disposition": "no_issue_found", + "receiptRefs": ["notes.txt", "../review.txt", "/artifacts/review.txt"], + } + ], + ), + ) + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + completed = complete(workbench_api, workbench_db, scan) + + surface = json.loads((scan.scan_dir / "coverage.json").read_text())["surfaces"][0] + assert surface["receiptRefs"] == [] + assert surface["disposition"] == "needs_follow_up" + assert ( + len( + [ + warning + for warning in completed["warnings"] + if "malformed coverage receipt" in warning + ] + ) + == 3 + ) + + +@pytest.mark.parametrize("archived", [False, True], ids=["current", "archived"]) +def test_deadline_preserves_worker_receipts_and_surface_links( + workbench_api, workbench_db, deadline_scan, archived +): + scan = deadline_scan + canonical_receipt = scan.scan_dir / "artifacts" / "reduced.txt" + canonical_receipt.parent.mkdir() + canonical_receipt.write_text("Reduced review evidence.\n") + canonical_surface = { + "id": f"{scan.completed_worker_id}/entry", + "workerId": scan.completed_worker_id, + "label": "Reduced review", + "disposition": "no_issue_found", + "receiptRefs": ["artifacts/reduced.txt"], + } + scan.coverage["surfaces"] = [canonical_surface] + (scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage)) + expected = {} + for label in ("upload", "download"): + worker_id, output = canceled_discovery_output( + workbench_db, scan, attempt=2 if archived else 1 + ) + source = output.parent / "attempts" / "attempt-1" if archived else output + receipt = source / "artifacts" / "review.txt" + receipt.parent.mkdir(parents=True) + receipt.write_text(f"Reviewed {label}.\n") + if archived: + current_receipt = output / "artifacts" / "review.txt" + current_receipt.parent.mkdir(parents=True) + current_receipt.write_text("Different current-attempt evidence.\n") + checkpoint = write_checkpoint( + source / "checkpoints", + saved_draft( + scan.scan_id, + surfaces=[ + { + "id": "entry", + "label": label, + "disposition": "no_issue_found", + "receiptRefs": ["artifacts/review.txt"], + } + ], + deferred=[{"id": "follow-up", "reason": label, "surfaceIds": ["entry"]}], + ), + ) + expected[label] = (worker_id, receipt, checkpoint, checkpoint.read_bytes()) + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + complete(workbench_api, workbench_db, scan) + + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert len(coverage["surfaces"]) == 3 + assert canonical_surface in coverage["surfaces"] + sources = json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"][ + "preservedSources" + ] + for label, (worker_id, receipt, checkpoint, original) in expected.items(): + surface = next(item for item in coverage["surfaces"] if item["label"] == label) + deferred = next(item for item in coverage["deferred"] if item["reason"] == label) + assert surface["id"] == f"{worker_id}/entry" + assert deferred["id"] == f"{worker_id}/follow-up" + assert deferred["surfaceIds"] == [surface["id"]] + assert surface["workerId"] == deferred["workerId"] == worker_id + assert surface["disposition"] == "no_issue_found" + assert surface["receiptRefs"] == [receipt.relative_to(scan.scan_dir).as_posix()] + assert (scan.scan_dir / surface["receiptRefs"][0]).read_text() == f"Reviewed {label}.\n" + assert sources[checkpoint.relative_to(scan.scan_dir).as_posix()] == workbench_api[ + "saved_results" + ]._digest(json.loads(original)) + assert checkpoint.read_bytes() == original + + def test_saturated_finish_cancels_unfinished_discovery_without_using_its_drafts( workbench_api, workbench_db, saturated_scan ): @@ -125,6 +419,148 @@ def test_saturated_finish_retains_failed_discovery_diagnostic( assert_published_aggregate(scan) +@pytest.mark.parametrize("stop", ["deadline", "run-limit", "saturated"]) +def test_unfinished_checkpoints_after_a_completed_reduction( + workbench_api, workbench_db, saturated_scan, stop +): + scan = saturated_scan + result = add_worker(workbench_db, scan, status="canceled") + recovered = copy.deepcopy(scan.findings[0]) + recovered["identity"]["anchor"] = "unfinished-upload-review" + recovered["title"] = "Separate upload handler bypasses containment" + recovered["provenance"] = {"source": "local_plugin"} + duplicate = copy.deepcopy(scan.findings[0]) + duplicate["severity"]["level"] = "critical" + duplicate["remediation"] = "Unreduced remediation from an unfinished review." + pending = { + "id": "pending-upload", + "reason": "The remaining upload paths need review.", + "paths": ["subdir/extract.py"], + "surfaceIds": ["upload"], + } + checkpoint = write_checkpoint( + result.parent / "checkpoints", + saved_draft( + scan.scan_id, + findings=[recovered, duplicate], + deferred=[pending], + surfaces=[ + { + "id": "upload", + "label": "Upload handler", + "disposition": "needs_follow_up", + "receiptRefs": [], + } + ], + ), + ) + source_bytes = checkpoint.read_bytes() + # Earlier parent, discovery, and reducer drafts must not undo reduction. + discarded = copy.deepcopy(recovered) + discarded["identity"]["anchor"] = "discarded-during-reduction" + for directory in ( + scan.scan_dir, + scan.scan_dir / "workers" / scan.completed_worker_id, + scan.scan_dir / "workers" / scan.reducer_id, + ): + discarded_checkpoint = write_checkpoint( + directory / "checkpoints", + saved_draft(scan.scan_id, findings=[discarded]), + ) + if directory == scan.scan_dir: + head = directory / "checkpoint-head.json" + head.write_text(json.dumps({"checkpoint": discarded_checkpoint.name})) + observed = ( + max( + (directory / name).stat().st_mtime_ns + for name in ("scan-manifest.json", "findings.json", "coverage.json") + ) + + 1_000_000_000 + ) + for path in (discarded_checkpoint, head): + os.utime(path, ns=(observed, observed)) + with workbench_db: + if stop == "deadline": + workbench_db.execute( + "UPDATE deep_scan_runs SET created_at = '2000-01-01T00:00:00+00:00' " + "WHERE scan_id = ?", + (scan.scan_id,), + ) + elif stop == "run-limit": + workbench_db.execute( + "UPDATE deep_scan_runs SET discovery_runs_dispatched = max_discovery_runs " + "WHERE scan_id = ?", + (scan.scan_id,), + ) + + reason = "saturated" if stop == "saturated" else "capped" + finished = finish(workbench_api, workbench_db, scan, terminal_reason=reason) + + assert finished["status"] == "succeeded" + worker = next(item for item in finished["workers"] if item["id"] == result.parent.name) + assert worker["status"] == "canceled" + assert worker["completionSequence"] is None + row = workbench_db.execute( + "SELECT status, seal_manifest_digest FROM scans WHERE id = ?", (scan.scan_id,) + ).fetchone() + assert tuple(row) == ("running", None) + manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text()) + assert "sealedAt" not in manifest["scan"] + completed = complete(workbench_api, workbench_db, scan) + assert completed["progress"]["status"] == "complete" + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + expected_anchors = {scan.findings[0]["identity"]["anchor"]} + if stop == "deadline": + expected_anchors.add(recovered["identity"]["anchor"]) + retained = next(item for item in findings if item["title"] == recovered["title"]) + assert retained["provenance"]["workerId"] == result.parent.name + assert ( + checkpoint.relative_to(scan.scan_dir).as_posix() in manifest["scan"]["preservedSources"] + ) + assert {item["identity"]["anchor"] for item in findings} == expected_anchors + canonical = next(item for item in findings if item["title"] == scan.findings[0]["title"]) + for field in ("severity", "confidence", "remediation"): + assert canonical[field] == scan.findings[0][field] + assert ( + canonical["provenance"]["sourceFindings"] + == scan.findings[0]["provenance"]["sourceFindings"] + ) + if stop == "deadline": + assert any( + item["remediation"] == duplicate["remediation"] + for item in canonical["provenance"]["previousFindings"] + ) + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert any(item["reason"] == pending["reason"] for item in coverage["deferred"]) == ( + stop == "deadline" + ) + assert any(item["disposition"] == "needs_follow_up" for item in coverage["surfaces"]) == ( + stop == "deadline" + ) + assert checkpoint.read_bytes() == source_bytes + names = ("scan-manifest.json", "findings.json", "coverage.json", "report.md") + sealed = {name: (scan.scan_dir / name).read_bytes() for name in names} + finish(workbench_api, workbench_db, scan, terminal_reason=reason) + complete(workbench_api, workbench_db, scan) + assert {name: (scan.scan_dir / name).read_bytes() for name in names} == sealed + + +def test_deadline_after_all_reviews_complete_keeps_reduced_coverage( + workbench_api, workbench_db, saturated_scan +): + scan = saturated_scan + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_runs SET created_at = '2000-01-01T00:00:00+00:00' WHERE scan_id = ?", + (scan.scan_id,), + ) + + finish(workbench_api, workbench_db, scan, terminal_reason="capped") + complete(workbench_api, workbench_db, scan) + + assert_published_aggregate(scan) + + @pytest.mark.parametrize( ("terminal_reason", "kind"), [("saturated", "setup"), ("saturated", "dedup"), ("capped", "discovery")], @@ -156,14 +592,21 @@ def test_finish_preserves_other_worker_failure_checks( @pytest.mark.parametrize("existing_head", [False, True]) +@pytest.mark.parametrize("with_reduction", [False, True], ids=["zero-accepted", "reduced"]) def test_rejected_deadline_publication_does_not_leave_parent_observations( - workbench_api, workbench_db, publication_scan, existing_head + workbench_api, workbench_db, publication_scan, request, existing_head, with_reduction ): - scan = publication_scan() + scan = request.getfixturevalue("saturated_scan") if with_reduction else publication_scan() canonical_names = ("scan-manifest.json", "findings.json", "coverage.json") for name in canonical_names: os.utime(scan.scan_dir / name, ns=(1_000_000_000, 1_000_000_000)) pending = {"id": "authorization-review", "reason": "Review the remaining authorization path."} + if with_reduction: + result = add_worker(workbench_db, scan, status="canceled") + write_checkpoint( + result.parent / "checkpoints", + saved_draft(scan.scan_id, deferred=[pending]), + ) if existing_head: checkpoint = write_checkpoint( scan.scan_dir / "checkpoints", diff --git a/plugins/codex-security/tests/test_workbench_checkpoint_heads.py b/plugins/codex-security/tests/test_workbench_checkpoint_heads.py index 125dcbd4c..014f294e9 100644 --- a/plugins/codex-security/tests/test_workbench_checkpoint_heads.py +++ b/plugins/codex-security/tests/test_workbench_checkpoint_heads.py @@ -425,14 +425,15 @@ def test_recovery_keeps_generic_work_that_collides_with_candidate_aliases( repeated = replay_saved_results(saved, replay, tmp_path, scan_id, binding, workers) assert first[2] == replay[2] == repeated[2] for result in (first, replay, repeated): - assert [row for row in result[2]["deferred"] if row["id"] != "scan-stopped"] == [generic] - assert ( - sum( - row["label"] == "Generic review" and row["disposition"] == "needs_follow_up" - for row in result[2]["surfaces"] - ) - == 1 - ) + pending_surfaces = [ + row + for row in result[2]["surfaces"] + if row["label"] == "Generic review" and row["disposition"] == "needs_follow_up" + ] + assert len(pending_surfaces) == 1 + assert [row for row in result[2]["deferred"] if row["id"] != "scan-stopped"] == [ + {**generic, "surfaceIds": [pending_surfaces[0]["id"]]} + ] assert any( row.get("candidateId") == "candidate-a" and row["disposition"] == outcome for row in result[2]["surfaces"] diff --git a/plugins/codex-security/tests/test_workbench_recovery_outcome_validation.py b/plugins/codex-security/tests/test_workbench_recovery_outcome_validation.py new file mode 100644 index 000000000..502bf5b9f --- /dev/null +++ b/plugins/codex-security/tests/test_workbench_recovery_outcome_validation.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +import copy +import json +import os +from types import SimpleNamespace + +import pytest +from test_deep_scan_successful_publication import complete +from test_deep_scan_successful_publication import publication_scan as publication_scan +from workbench_test_support import saved_draft, write_checkpoint + + +@pytest.mark.parametrize( + ("label", "error"), + [ + pytest.param(None, None, id="absent-rejection"), + pytest.param("Reviewed candidate", None, id="valid-rejection"), + pytest.param("Invalid \ud800", "well-formed Unicode", id="surrogate-label"), + pytest.param("\u0085", "label: expected a non-empty string", id="nel-label"), + pytest.param(" ", "label: expected a non-empty string", id="blank-label"), + ], +) +def test_stopped_scan_requires_valid_rejection_evidence_before_sealing( + workbench_api, workbench_db, publication_scan, label, error +): + scan = publication_scan(mode="standard") + report_path = scan.scan_dir / "report.md" + report_path.unlink() + finding = copy.deepcopy(scan.findings[0]) + finding["provenance"] = {"source": "local_plugin", "candidateId": "candidate-a"} + checkpoint = write_checkpoint( + scan.scan_dir / "checkpoints", saved_draft(scan.scan_id, findings=[finding]) + ) + os.utime(checkpoint, ns=(100, 100)) + checkpoint_bytes = checkpoint.read_bytes() + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + scan.coverage["surfaces"] = ( + [] + if label is None + else [ + { + "id": "rejection", + "candidateId": "candidate-a", + "label": label, + "disposition": "rejected", + "receiptRefs": [], + } + ] + ) + coverage_path = scan.scan_dir / "coverage.json" + coverage_path.write_text(json.dumps(scan.coverage)) + coverage_bytes = coverage_path.read_bytes() + + workbench_api["saved_results"].fail_scan( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + SimpleNamespace( + scan_id=scan.scan_id, + cost_json=None, + claim_token=None, + message="Synthetic interruption", + ), + ) + + row = workbench_db.execute( + "SELECT status, completion_warnings_json FROM scans WHERE id = ?", (scan.scan_id,) + ).fetchone() + assert row["status"] == "failed" + assert checkpoint.read_bytes() == checkpoint_bytes + manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"] + warnings = json.loads(row["completion_warnings_json"]) + if error: + assert not manifest.get("sealedAt") + assert not report_path.exists() + assert coverage_path.read_bytes() == coverage_bytes + assert any( + "result publication needs follow-up" in warning and error in warning + for warning in warnings + ) + else: + assert manifest["sealedAt"] + assert report_path.is_file() + assert warnings == [] + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert [item["title"] for item in findings] == ([finding["title"]] if label is None else []) + surfaces = json.loads(coverage_path.read_text())["surfaces"] + assert [(item.get("candidateId"), item["disposition"]) for item in surfaces] == ( + [] if label is None else [("candidate-a", "rejected")] + ) + + +@pytest.mark.parametrize( + ("reason", "invalid"), + [ + pytest.param("Review completed.", False, id="valid-closure"), + pytest.param("Invalid \ud800", True, id="surrogate-reason"), + ], +) +def test_standard_completion_requires_valid_closure_evidence_before_sealing( + workbench_api, workbench_db, publication_scan, reason, invalid +): + scan = publication_scan(mode="standard") + report_path = scan.scan_dir / "report.md" + report_path.unlink() + checkpoint = write_checkpoint( + scan.scan_dir / "checkpoints", + saved_draft( + scan.scan_id, + deferred=[{"id": "review", "reason": "Review authorization remains pending."}], + ), + ) + os.utime(checkpoint, ns=(100, 100)) + checkpoint_bytes = checkpoint.read_bytes() + scan.coverage["deferred"] = [] + closure = {"id": "review", "reason": reason} + scan.coverage["resolvedDeferred"] = [closure] + coverage_path = scan.scan_dir / "coverage.json" + coverage_path.write_text(json.dumps(scan.coverage)) + coverage_bytes = coverage_path.read_bytes() + + if invalid: + with pytest.raises(SystemExit, match="well-formed Unicode"): + complete(workbench_api, workbench_db, scan) + else: + completed = complete(workbench_api, workbench_db, scan) + assert completed["warnings"] == [] + + assert checkpoint.read_bytes() == checkpoint_bytes + status = workbench_db.execute( + "SELECT status FROM scans WHERE id = ?", (scan.scan_id,) + ).fetchone()[0] + manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"] + if invalid: + assert status == "running" + assert not manifest.get("sealedAt") + assert not report_path.exists() + assert coverage_path.read_bytes() == coverage_bytes + else: + assert status == "complete" + assert manifest["sealedAt"] + assert report_path.is_file() + coverage = json.loads(coverage_path.read_text()) + assert coverage["deferred"] == [] + assert coverage["resolvedDeferred"] == [closure] diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts index 893257e14..d53c16f1c 100644 --- a/sdk/typescript/src/cli.ts +++ b/sdk/typescript/src/cli.ts @@ -113,6 +113,7 @@ import { } from "./external-findings-publish.js"; import { prepareLocalExternalImport } from "./external-findings-import.js"; import { listLocalExternalFindings } from "./external-findings-local.js"; +import { EXTERNAL_FINDINGS_READERS } from "./external-findings-read.js"; import type { ExternalFindings } from "./external-findings.js"; import { publishPatchReview, @@ -3617,7 +3618,7 @@ export async function main( "Local repository path for --to local; authorized repository ID or URL for --to cloud.", ), provider: z - .enum(["wiz", "snyk", "semgrep", "endor", "github"]) + .enum(EXTERNAL_FINDINGS_READERS) .describe("Provider that produced these findings."), sourceRepositoryUrl: optionValue("--source-repository-url") .optional() diff --git a/sdk/typescript/src/external-findings-import.ts b/sdk/typescript/src/external-findings-import.ts index b06ad5f1c..c350472a3 100644 --- a/sdk/typescript/src/external-findings-import.ts +++ b/sdk/typescript/src/external-findings-import.ts @@ -3,7 +3,10 @@ import { cloudRepositoryUrlKey, repositoryUrlKey, } from "./external-import-contract.js"; -import { readExternalFindings } from "./external-findings-read.js"; +import { + EXTERNAL_FINDINGS_PROVIDERS, + readExternalFindings, +} from "./external-findings-read.js"; import { prepareNormalizedLocalImport, type LocalExternalImportDependencies, @@ -21,8 +24,12 @@ export async function prepareLocalExternalImport( () => readExternalFindings(path, options, "local"), dependencies.signal, ); + const normalizedOptions = { + ...options, + provider: EXTERNAL_FINDINGS_PROVIDERS[options.provider], + }; let sourceRepositories = parsed.sourceRepositories ?? []; - if (options.provider === "semgrep") { + if (normalizedOptions.provider === "semgrep") { // The Cloud reader allows custom GitHub casing. Local provenance must also // agree with the caller's source URL when the checkout has no remote. const source = options.sourceRepositoryUrl!; @@ -47,7 +54,7 @@ export async function prepareLocalExternalImport( })), ]; } - return prepareNormalizedLocalImport(parsed.findings, options, { + return prepareNormalizedLocalImport(parsed.findings, normalizedOptions, { ...dependencies, sourceRepositories: [ ...(dependencies.sourceRepositories ?? []), diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index c617b8234..5f5b83daf 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -25,6 +25,7 @@ import { validatePreparedOutputDir, } from "./runtime.js"; import { + EXTERNAL_FINDINGS_PROVIDERS, readExternalFindings, type ExternalFindingsReaderOptions, } from "./external-findings-read.js"; @@ -61,7 +62,10 @@ export interface NormalizedExternalPublicationOptions { } /** File readers accept the vendor formats installed in this SDK. */ -export interface ExternalPublicationOptions extends NormalizedExternalPublicationOptions { +export interface ExternalPublicationOptions extends Omit< + NormalizedExternalPublicationOptions, + "provider" +> { provider: ExternalFindingsReaderOptions["provider"]; sourceRepositoryUrl?: string; } @@ -347,11 +351,19 @@ export async function prepareExternalPublication( dependencies: ExternalPublicationDependencies = {}, ): Promise { const parsed = await readExternalFindings(path, options); + const normalizedOptions = { + ...options, + provider: EXTERNAL_FINDINGS_PROVIDERS[options.provider], + }; return parsed.publication - ? preparePublicationInput(parsed.publication, options, dependencies) + ? preparePublicationInput( + parsed.publication, + normalizedOptions, + dependencies, + ) : prepareNormalizedExternalPublication( parsed.findings, - options, + normalizedOptions, dependencies, ); } diff --git a/sdk/typescript/src/external-findings-read.ts b/sdk/typescript/src/external-findings-read.ts index a50e6292e..1115783d5 100644 --- a/sdk/typescript/src/external-findings-read.ts +++ b/sdk/typescript/src/external-findings-read.ts @@ -3,6 +3,7 @@ import type { ExternalPublicationInput, ExternalSourceRepository, } from "./external-findings.js"; +import type { Provider } from "./external-import-models.js"; import type { ExternalImportTarget } from "./external-import-contract.js"; import { wizCsvRepositoryUrl } from "./wiz-csv.js"; import { readSnykFindings, snykRepositoryUrls } from "./snyk-findings.js"; @@ -13,9 +14,24 @@ import { isGitHubCloudHost } from "./git-remote.js"; import { isRecord } from "./record.js"; import { readVendorFindingsForPublication } from "./wiz-findings.js"; +/** File reader names mapped to their stored and Cloud source provider. */ +export const EXTERNAL_FINDINGS_PROVIDERS = { + wiz: "wiz", + snyk: "snyk", + semgrep: "semgrep", + endor: "endor", + github: "github", +} as const satisfies Record; + +export type ExternalFindingsReader = keyof typeof EXTERNAL_FINDINGS_PROVIDERS; + +export const EXTERNAL_FINDINGS_READERS = Object.keys( + EXTERNAL_FINDINGS_PROVIDERS, +) as ExternalFindingsReader[]; + /** File readers available in this SDK, shared by local and Cloud imports. */ export interface ExternalFindingsReaderOptions { - provider: "wiz" | "snyk" | "semgrep" | "endor" | "github"; + provider: ExternalFindingsReader; sourceRepositoryUrl?: string; } diff --git a/sdk/typescript/tests-ts/external-findings-import.test.ts b/sdk/typescript/tests-ts/external-findings-import.test.ts index a378a3832..e9a192acd 100644 --- a/sdk/typescript/tests-ts/external-findings-import.test.ts +++ b/sdk/typescript/tests-ts/external-findings-import.test.ts @@ -96,6 +96,10 @@ test("Wiz JSON imports retain parsed evidence through SQLite readback and update dependencies, ); expect(prepared.preview.findings).toEqual(parsed.findings); + expect(prepared.preview.source).toEqual({ + provider: "wiz", + source_key: options.sourceKey, + }); await expect( access(join(fixture.stateDirectory, "workbench.sqlite3")), ).rejects.toThrow(); @@ -107,6 +111,7 @@ test("Wiz JSON imports retain parsed evidence through SQLite readback and update ); expect(stored).toEqual(first.findings); expect(stored[0]).toMatchObject({ + provider: "wiz", sourceFindingId: "package-occurrence", assessment: { state: "not_assessed" }, version: 1, diff --git a/sdk/typescript/tests-ts/external-findings-publish.test.ts b/sdk/typescript/tests-ts/external-findings-publish.test.ts index 63233bbe5..57d278b12 100644 --- a/sdk/typescript/tests-ts/external-findings-publish.test.ts +++ b/sdk/typescript/tests-ts/external-findings-publish.test.ts @@ -3463,9 +3463,12 @@ test("Semgrep publication preserves identity across JSON and SARIF with explicit sourceKey: "organization/project/main", sourceRepositoryUrl: "https://github.com/example/project", }; - const first = await ( - await prepareExternalPublication(f.file, settings, f.deps) - ).publish(); + const prepared = await prepareExternalPublication(f.file, settings, f.deps); + expect(prepared.preview.source).toEqual({ + provider: "semgrep", + source_key: settings.sourceKey, + }); + const first = await prepared.publish(); expect(first.counts.created).toBe(1); expect(first.verified).toBe(1); const original = structuredClone([...f.reports.values()][0]!); diff --git a/sdk/typescript/tests-ts/semgrep-findings-local.test.ts b/sdk/typescript/tests-ts/semgrep-findings-local.test.ts index c3559c9cd..de22cb8a8 100644 --- a/sdk/typescript/tests-ts/semgrep-findings-local.test.ts +++ b/sdk/typescript/tests-ts/semgrep-findings-local.test.ts @@ -196,6 +196,10 @@ test.each(formats)( ); expect(prepared.preview.findings).toHaveLength(1); expect(prepared.preview.excluded).toEqual([]); + expect(prepared.preview.source).toEqual({ + provider: "semgrep", + source_key: options.sourceKey, + }); const imported = await prepared.import(); expect(imported.counts).toEqual({ created: 1, updated: 0, unchanged: 0 });