Add cache-backed numerical model validation

.github/workflows/ci.yml

@@ -10,6 +10,8 @@ env:
   # ONNX Runtime dylib for the reference backend. rustnn pins ort 2.0.0-rc.13
   # with the `api-27` feature, so this must be >= 1.27 (any newer 1.x works).
   ORT_VERSION: 1.29.0
+  RUSTNN_REPO: rustnn/rustnn
+  RUSTNN_REF: main
 
 jobs:
   test:
@@ -69,7 +71,8 @@ jobs:
         id: rustnn
         uses: actions/checkout@v4
         with:
-          repository: rustnn/rustnn
+          repository: ${{ env.RUSTNN_REPO }}
+          ref: ${{ env.RUSTNN_REF }}
           path: rustnn
 
       - name: Install Rust
@@ -150,3 +153,16 @@ jobs:
           O2W_MODEL_TEST_JOBS: ${{ matrix.model_test_jobs || 2 }}
           O2W_MODEL_TEST_SKIP_HEAVY: 1
         run: cargo test --release --locked ${{ matrix.cargo_flags }}
+
+      - name: Validate curated full model
+        if: runner.os != 'Windows'
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          O2W_ONNX_CACHE: ${{ runner.temp }}/onnx2webnn-ci-models
+          O2W_WEBNN_CACHE: ${{ runner.temp }}/onnx2webnn-ci-webnn
+        run: |
+          target/release/onnx2webnn validate-models \
+            --manifest tests/models/ci-validation.json \
+            --selection all \
+            --weights real \
+            --jobs 1

.github/workflows/model-validation.yml

@@ -0,0 +1,119 @@
+name: Full model validation
+
+on:
+  workflow_dispatch:
+
+env:
+  ORT_VERSION: 1.29.0
+  RUSTNN_REPO: rustnn/rustnn
+  RUSTNN_REF: main
+
+jobs:
+  validate:
+    name: Full real-weight sweep
+    runs-on: [self-hosted, linux, x64, onnx2webnn-validation]
+    timeout-minutes: 360
+    defaults:
+      run:
+        shell: bash
+        working-directory: onnx2webnn
+    steps:
+      - name: Checkout onnx2webnn
+        uses: actions/checkout@v4
+        with:
+          path: onnx2webnn
+
+      - name: Checkout rustnn
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ env.RUSTNN_REPO }}
+          ref: ${{ env.RUSTNN_REF }}
+          path: rustnn
+
+      - name: Check runner contract
+        working-directory: ${{ github.workspace }}
+        run: |
+          cache=/var/cache/onnx2webnn
+          test -d "$cache"
+          test -w "$cache"
+          filesystem_bytes=$(df -PB1 "$cache" | awk 'NR == 2 { print $2 }')
+          memory_bytes=$(awk '/MemTotal/ { print $2 * 1024 }' /proc/meminfo)
+          minimum_filesystem=$((100 * 1024 * 1024 * 1024))
+          minimum_memory=$((30 * 1024 * 1024 * 1024))
+          if [ "$filesystem_bytes" -lt "$minimum_filesystem" ]; then
+            echo "Runner filesystem must be at least 100 GiB" >&2
+            exit 1
+          fi
+          if [ "$memory_bytes" -lt "$minimum_memory" ]; then
+            echo "Runner memory must be approximately 32 GiB (30 GiB minimum)" >&2
+            exit 1
+          fi
+
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          components: rustfmt
+
+      - name: Install protobuf compiler
+        uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Cache cargo registry
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+          key: full-validation-cargo-${{ hashFiles('onnx2webnn/Cargo.lock') }}
+          restore-keys: full-validation-cargo-
+
+      - name: Download ONNX Runtime ${{ env.ORT_VERSION }}
+        run: |
+          archive=onnxruntime-linux-x64-${ORT_VERSION}
+          url=https://github.com/microsoft/onnxruntime/releases/download/v${ORT_VERSION}/${archive}.tgz
+          mkdir -p "$RUNNER_TEMP/ort"
+          curl -fsSL "$url" -o "$RUNNER_TEMP/ort/archive.tgz"
+          tar -xzf "$RUNNER_TEMP/ort/archive.tgz" -C "$RUNNER_TEMP/ort"
+          dylib="$RUNNER_TEMP/ort/$archive/lib/libonnxruntime.so"
+          if [ ! -f "$dylib" ]; then
+            dylib=$(find "$RUNNER_TEMP/ort/$archive/lib" -maxdepth 1 -name 'libonnxruntime.so.*' -print -quit)
+          fi
+          test -f "$dylib"
+          echo "ORT_DYLIB_PATH=$dylib" >> "$GITHUB_ENV"
+
+      - name: Build release binary
+        run: cargo build --release --locked
+
+      - name: Run diagnostic full sweep
+        id: validation
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          O2W_ONNX_CACHE: /var/cache/onnx2webnn
+          O2W_WEBNN_CACHE: ${{ runner.temp }}/onnx2webnn-webnn
+        run: |
+          set +e
+          target/release/onnx2webnn validate-models \
+            --manifest tests/models/manifest.json \
+            --selection all \
+            --weights real \
+            --jobs 1 2>&1 | tee "$RUNNER_TEMP/model-validation.log"
+          status=${PIPESTATUS[0]}
+          set -e
+          echo "exit_code=$status" >> "$GITHUB_OUTPUT"
+          if [ "$status" -eq 0 ]; then
+            echo "## Full model validation: passed" >> "$GITHUB_STEP_SUMMARY"
+          else
+            echo "::warning::Full model validation reported failures (exit code $status); see the summary and uploaded log."
+            echo "## Full model validation: diagnostic failures" >> "$GITHUB_STEP_SUMMARY"
+            echo "The validator exited with code $status. This manually dispatched sweep is diagnostic and does not fail the workflow." >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+      - name: Upload validation log
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: full-model-validation-real
+          path: ${{ runner.temp }}/model-validation.log
+          if-no-files-found: warn
+          retention-days: 14

.gitignore

@@ -11,3 +11,7 @@ __pycache__/
 .vscode/
 .idea/
 .DS_Store
+
+# Model caches
+.webnn-cache/
+.onnx-cache/

AGENTS.md

@@ -25,20 +25,23 @@ independent layers:
 |-------|----------------|-------|-----------------|
 | **1. WebNN spec** | Ops the W3C API defines | [webnn spec](https://www.w3.org/TR/webnn/) | ~105 |
 | **2. Name mapping** | ONNX op name ↔ WebNN `MLGraphBuilder` method | `webnn-onnx-utils/src/operation_names.rs` | ~90 |
-| **3. Exporter implementation** | ONNX ops with a Rust lowering handler | `src/onnx/ops/*.rs` + `scripts/webnn_onnx_ops.py` | ~59 |
+| **3. Exporter implementation** | Advertised ONNX names with a lowering or preprocessing path | `src/onnx/ops/*.rs` + `scripts/webnn_onnx_ops.py` | 135 |
 
 An ONNX op can be:
 
-- **No WebNN target** — e.g. `If`, `Loop`, `Scan` (control flow), `StringConcat` (no string
-  tensors), `Compress`, `Einsum`, `Attention`. WebNN graphs are static DAGs; these are permanently
-  rejected. See `docs/operator-conversion-plan.md` Stage 3.
-- **Mapped but not implemented** — e.g. `BatchNormalization`, `InstanceNormalization`, `ArgMax`,
-  `GatherND`, `Resize`. Listed in `operation_names.rs` but no handler in `src/onnx/ops/` yet.
-  Tests expect `UnsupportedOp` until a handler lands.
-- **Implemented** — listed in `scripts/webnn_onnx_ops.py` and handled by `OpRegistry`. Tests
-  expect `Success` (convert + ORT vs rustnn output match).
+- **No current lowering** — for example `Loop`, `Scan`, string operators, and general
+  sequence operators. Some have no portable WebNN representation.
+- **Pattern-only or partial** — for example constant-condition `If`, the narrow
+  `SplitToSequence` + `SequenceAt` export pattern, static `OneHot`, and restricted
+  `Einsum`. The handler rejects unsupported forms.
+- **Direct or decomposed** — advertised in `scripts/webnn_onnx_ops.py`. A decomposed op may be
+  functionally supported while remaining slower or narrower than a native backend kernel.
 
-The test manifest (`webnn_onnx_ops.py`) mirrors layer 3 only. Regenerate tests after changing it.
+The manifest contains 135 accepted names. At opset 26, 127 are standard `ai.onnx` names;
+adding the registry-only `SplitToSequence`/`SequenceAt` pattern gives 129/198 (65.2%) standard
+names with at least one executable lowering. Of those, 96/198 (48.5%) have a direct standardized
+WebNN primitive path. Name presence is not a promise of every ONNX schema variant. See
+`docs/operator-conversion-status.md` for the complete inventory and WebNN-imposed limitations. Regenerate tests after changing the manifest.
 
 ## Build and test
 
@@ -104,9 +107,10 @@ Operators with multiple pre-9 schema revisions skip sub-opset-9 structure bands
 until legacy attribute normalization lands; newer bands use the true latest schema
 revision in range (e.g. `MaxRoiPool` at model opset 22, `Pad` at 9/17/26).
 
-Operator dispatch and the unsupported-op pre-scan key on **`op_type` only** (standard
-`ai.onnx` domain). Custom-domain ops are not supported yet; adding them requires
-domain-aware handler registration and matching pre-scan logic in `convert.rs`.
+Operator dispatch and the unsupported-op pre-scan key on **`op_type` only**, not on the ONNX
+domain. This currently permits selected `com.microsoft` fused operators such as MoE/QMoE, but
+it is not general custom-domain support and cannot distinguish same-named operators across
+domains. Proper domain-aware support requires matching registry and pre-scan changes.
 
 ## Layout (committed)
 
@@ -147,7 +151,8 @@ history and tests the highest buildable opset in each band — e.g. `Pad` at ops
 (attribute `pads` vs input `pads` vs optional `axes` input). Ops with unchanged structure get a
 single test at the newest opset. Unbuildable ops get an `#[ignore]` stub.
 
-See [docs/operator-conversion-plan.md](docs/operator-conversion-plan.md) for the full operator rollout workflow.
+See [docs/operator-conversion-status.md](docs/operator-conversion-status.md) for the complete
+per-operator capability matrix.
 
 ## Dependencies (`Cargo.toml`)