Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions docs/atlas/index.html

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion docs/atlas/pipeline_atlas.svg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
2 changes: 1 addition & 1 deletion docs/atlas/pipeline_atlas_dark.svg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
12 changes: 12 additions & 0 deletions docs/pipeline_catalog.jsonl
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,18 @@
{"task": "ASR", "task_alias": "asr", "language": "ar-sa", "metric": "wer", "pipeline_id": "asr.ar_sa.wer.giga_norm_sau_v1.wenet_wer_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["wer"], "nodes": ["normalization/giga_norm", "scoring/wenet_wer"], "computation_node_ids": ["normalization/giga_norm", "scoring/wenet_wer"], "task_config_path": "src/sure_eval/evaluation/tasks/asr/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/asr/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.asr.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.asr.run", "executor": "sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files", "required_roles": ["hyp", "ref"], "optional_roles": []}
{"task": "ASR", "task_alias": "asr", "language": "ar-sy", "metric": "wer", "pipeline_id": "asr.ar_sy.wer.giga_norm_syr_v1.wenet_wer_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["wer"], "nodes": ["normalization/giga_norm", "scoring/wenet_wer"], "computation_node_ids": ["normalization/giga_norm", "scoring/wenet_wer"], "task_config_path": "src/sure_eval/evaluation/tasks/asr/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/asr/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.asr.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.asr.run", "executor": "sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files", "required_roles": ["hyp", "ref"], "optional_roles": []}
{"task": "ASR", "task_alias": "asr", "language": "ar", "metric": "cer", "pipeline_id": "asr.ar.cer.xnorm_ar_v1.wenet_cer_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["cer"], "nodes": ["normalization/xnorm", "scoring/wenet_cer"], "computation_node_ids": ["normalization/xnorm", "scoring/wenet_cer"], "task_config_path": "src/sure_eval/evaluation/tasks/asr/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/asr/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.asr.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.asr.run", "executor": "sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files", "required_roles": ["hyp", "ref"], "optional_roles": []}
{"task": "ASR", "task_alias": "asr", "language": "ar", "metric": "cer", "pipeline_id": "asr.ar.cer.etc_canonicalization_ar_v1.wenet_cer_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["cer"], "nodes": ["normalization/etc_canonicalization", "scoring/wenet_cer"], "computation_node_ids": ["normalization/etc_canonicalization", "scoring/wenet_cer"], "task_config_path": "src/sure_eval/evaluation/tasks/asr/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/asr/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.asr.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.asr.run", "executor": "sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files", "required_roles": ["hyp", "ref"], "optional_roles": []}
{"task": "ASR", "task_alias": "asr", "language": "bn", "metric": "wer", "pipeline_id": "asr.bn.wer.etc_canonicalization_bn_v1.wenet_wer_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["wer"], "nodes": ["normalization/etc_canonicalization", "scoring/wenet_wer"], "computation_node_ids": ["normalization/etc_canonicalization", "scoring/wenet_wer"], "task_config_path": "src/sure_eval/evaluation/tasks/asr/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/asr/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.asr.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.asr.run", "executor": "sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files", "required_roles": ["hyp", "ref"], "optional_roles": []}
{"task": "ASR", "task_alias": "asr", "language": "de", "metric": "wer", "pipeline_id": "asr.de.wer.etc_canonicalization_de_v1.wenet_wer_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["wer"], "nodes": ["normalization/etc_canonicalization", "scoring/wenet_wer"], "computation_node_ids": ["normalization/etc_canonicalization", "scoring/wenet_wer"], "task_config_path": "src/sure_eval/evaluation/tasks/asr/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/asr/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.asr.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.asr.run", "executor": "sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files", "required_roles": ["hyp", "ref"], "optional_roles": []}
{"task": "ASR", "task_alias": "asr", "language": "en", "metric": "wer", "pipeline_id": "asr.en.wer.etc_canonicalization_en_v1.wenet_wer_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["wer"], "nodes": ["normalization/etc_canonicalization", "scoring/wenet_wer"], "computation_node_ids": ["normalization/etc_canonicalization", "scoring/wenet_wer"], "task_config_path": "src/sure_eval/evaluation/tasks/asr/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/asr/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.asr.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.asr.run", "executor": "sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files", "required_roles": ["hyp", "ref"], "optional_roles": []}
{"task": "ASR", "task_alias": "asr", "language": "fa", "metric": "wer", "pipeline_id": "asr.fa.wer.etc_canonicalization_fa_v1.wenet_wer_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["wer"], "nodes": ["normalization/etc_canonicalization", "scoring/wenet_wer"], "computation_node_ids": ["normalization/etc_canonicalization", "scoring/wenet_wer"], "task_config_path": "src/sure_eval/evaluation/tasks/asr/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/asr/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.asr.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.asr.run", "executor": "sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files", "required_roles": ["hyp", "ref"], "optional_roles": []}
{"task": "ASR", "task_alias": "asr", "language": "fr", "metric": "wer", "pipeline_id": "asr.fr.wer.etc_canonicalization_fr_v1.wenet_wer_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["wer"], "nodes": ["normalization/etc_canonicalization", "scoring/wenet_wer"], "computation_node_ids": ["normalization/etc_canonicalization", "scoring/wenet_wer"], "task_config_path": "src/sure_eval/evaluation/tasks/asr/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/asr/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.asr.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.asr.run", "executor": "sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files", "required_roles": ["hyp", "ref"], "optional_roles": []}
{"task": "ASR", "task_alias": "asr", "language": "hi", "metric": "wer", "pipeline_id": "asr.hi.wer.etc_canonicalization_hi_v1.wenet_wer_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["wer"], "nodes": ["normalization/etc_canonicalization", "scoring/wenet_wer"], "computation_node_ids": ["normalization/etc_canonicalization", "scoring/wenet_wer"], "task_config_path": "src/sure_eval/evaluation/tasks/asr/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/asr/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.asr.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.asr.run", "executor": "sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files", "required_roles": ["hyp", "ref"], "optional_roles": []}
{"task": "ASR", "task_alias": "asr", "language": "id", "metric": "wer", "pipeline_id": "asr.id.wer.etc_canonicalization_id_v1.wenet_wer_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["wer"], "nodes": ["normalization/etc_canonicalization", "scoring/wenet_wer"], "computation_node_ids": ["normalization/etc_canonicalization", "scoring/wenet_wer"], "task_config_path": "src/sure_eval/evaluation/tasks/asr/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/asr/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.asr.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.asr.run", "executor": "sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files", "required_roles": ["hyp", "ref"], "optional_roles": []}
{"task": "ASR", "task_alias": "asr", "language": "ja", "metric": "cer", "pipeline_id": "asr.ja.cer.etc_canonicalization_ja_v1.wenet_cer_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["cer"], "nodes": ["normalization/etc_canonicalization", "scoring/wenet_cer"], "computation_node_ids": ["normalization/etc_canonicalization", "scoring/wenet_cer"], "task_config_path": "src/sure_eval/evaluation/tasks/asr/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/asr/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.asr.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.asr.run", "executor": "sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files", "required_roles": ["hyp", "ref"], "optional_roles": []}
{"task": "ASR", "task_alias": "asr", "language": "ru", "metric": "wer", "pipeline_id": "asr.ru.wer.etc_canonicalization_ru_v1.wenet_wer_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["wer"], "nodes": ["normalization/etc_canonicalization", "scoring/wenet_wer"], "computation_node_ids": ["normalization/etc_canonicalization", "scoring/wenet_wer"], "task_config_path": "src/sure_eval/evaluation/tasks/asr/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/asr/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.asr.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.asr.run", "executor": "sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files", "required_roles": ["hyp", "ref"], "optional_roles": []}
{"task": "ASR", "task_alias": "asr", "language": "vi", "metric": "wer", "pipeline_id": "asr.vi.wer.etc_canonicalization_vi_v1.wenet_wer_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["wer"], "nodes": ["normalization/etc_canonicalization", "scoring/wenet_wer"], "computation_node_ids": ["normalization/etc_canonicalization", "scoring/wenet_wer"], "task_config_path": "src/sure_eval/evaluation/tasks/asr/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/asr/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.asr.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.asr.run", "executor": "sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files", "required_roles": ["hyp", "ref"], "optional_roles": []}
{"task": "ASR", "task_alias": "asr", "language": "zh", "metric": "cer", "pipeline_id": "asr.zh.cer.etc_canonicalization_zh_v1.wenet_cer_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["cer"], "nodes": ["normalization/etc_canonicalization", "scoring/wenet_cer"], "computation_node_ids": ["normalization/etc_canonicalization", "scoring/wenet_cer"], "task_config_path": "src/sure_eval/evaluation/tasks/asr/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/asr/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.asr.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.asr.run", "executor": "sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files", "required_roles": ["hyp", "ref"], "optional_roles": []}
{"task": "ASR", "task_alias": "asr", "language": "de", "metric": "wer", "pipeline_id": "asr.de.wer.xnorm_de_v1.wenet_wer_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["wer"], "nodes": ["normalization/xnorm", "scoring/wenet_wer"], "computation_node_ids": ["normalization/xnorm", "scoring/wenet_wer"], "task_config_path": "src/sure_eval/evaluation/tasks/asr/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/asr/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.asr.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.asr.run", "executor": "sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files", "required_roles": ["hyp", "ref"], "optional_roles": []}
{"task": "ASR", "task_alias": "asr", "language": "en", "metric": "wer", "pipeline_id": "asr.en.wer.xnorm_en_v1.wenet_wer_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["wer"], "nodes": ["normalization/xnorm", "scoring/wenet_wer"], "computation_node_ids": ["normalization/xnorm", "scoring/wenet_wer"], "task_config_path": "src/sure_eval/evaluation/tasks/asr/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/asr/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.asr.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.asr.run", "executor": "sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files", "required_roles": ["hyp", "ref"], "optional_roles": []}
{"task": "ASR", "task_alias": "asr", "language": "es", "metric": "wer", "pipeline_id": "asr.es.wer.xnorm_es_v1.wenet_wer_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["wer"], "nodes": ["normalization/xnorm", "scoring/wenet_wer"], "computation_node_ids": ["normalization/xnorm", "scoring/wenet_wer"], "task_config_path": "src/sure_eval/evaluation/tasks/asr/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/asr/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.asr.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.asr.run", "executor": "sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files", "required_roles": ["hyp", "ref"], "optional_roles": []}
Expand Down
37 changes: 37 additions & 0 deletions docs/tasks/asr.md
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,43 @@ The route config is
- `scoring/sctk_sclite` - optional binary-backed scorer wrapping NIST SCTK
`sclite`; default ASR pipelines continue to use WeNet-compatible scorers.

## ETC-Bench Canonicalization Routes

ETC-Bench is a multilingual benchmark for evaluation-side transcript
canonicalization. It maps speech-equivalent written variants to one canonical
form before comparing a hypothesis with its reference. The same model is
applied independently to both sides; this node does not score the transcript.

This PR adds optional ASR routes using
`normalization/etc_canonicalization` followed by the existing WeNet scorer:

- CER: `ar`, `ja`, `zh`
- WER: `bn`, `de`, `en`, `fa`, `fr`, `hi`, `id`, `ru`, `vi`

For example, the exact English route is
`asr.en.wer.etc_canonicalization_en_v1.wenet_wer_v1`. Prepare the node after
cloning and provide a checkpoint locally:

```bash
sure-eval metric describe asr \
--pipeline-id asr.en.wer.etc_canonicalization_en_v1.wenet_wer_v1 \
--output /tmp/asr_etc.json
sure-eval env setup --pipeline /tmp/asr_etc.json
export SURE_ETC_CANONICALIZATION_MODEL=/path/to/etc-qwen3.5-0.8b
sure-eval metric run --pipeline /tmp/asr_etc.json \
--ref-file ref.txt --hyp-file hyp.txt --output-dir /tmp/asr-etc-run
```

The environment and model checkpoint are user-managed runtime assets and are
not included in the repository.

The model uses a fixed greedy `max_new_tokens=256` generation budget. If a
transcript reaches that budget without an EOS token, the node fails explicitly
instead of scoring a partial canonical transcript. Because this is a learned
canonicalizer, it can change content such as fillers, repetitions, casing,
punctuation, numbers, and symbols; users should inspect the node trace when
comparing runs.

## Canonical ITN Pipelines

Canonical ITN pipelines keep the same reported metrics (`cer`, `wer`, `mer`)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
# ETC canonicalization

## Purpose

This optional node adapts the model canonicalizer released with
[ETC-Bench](https://github.com/zh-qian2004/etc-bench). It independently maps
reference and hypothesis transcripts to a deterministic shared written form so
speech-equivalent transcript variants can be compared fairly. It does not
replace the task scorer or use benchmark pairs as a lookup table.

## Task Scenarios

- `asr/<language>/cer` and `asr/<language>/wer` alternative routes for the 12
ETC-Bench languages: `ar`, `bn`, `de`, `en`, `fa`, `fr`, `hi`, `id`, `ja`,
`ru`, `vi`, and `zh`.
- This is an optional alternative backend; no existing default route changes.

## Input

- `key_text_files` containing reference and hypothesis UTF-8 key-tab-text files.
- Each valid row has an utterance key, a tab, and one transcript. Reference and
hypothesis files are canonicalized independently.

## Output

- `key_text_files` with the original keys preserved and canonical text values.
- The node trace records the model, language, row counts, changed counts,
empty-row counts, and per-row before/after values.

## Versioned Computation

- Node id: `normalization/etc_canonicalization`, version `v1`.
- Internal stages: key-text parsing, ETC model canonicalization, key-text
writing.
- The adapter follows ETC-Bench's evaluation-side transcript canonicalization
contract and uses the published ETC-Qwen3.5 model prompt with greedy
decoding. A model is loaded once per reference or hypothesis file batch.

## Runtime and Assets

- Runtime type: optional node-local `uv` environment, Python 3.11.
- Prepare it after cloning:

```bash
sure-eval env setup --node normalization/etc_canonicalization
```

- Set `SURE_ETC_CANONICALIZATION_MODEL` to a user-downloaded local checkpoint,
or use the published `zhiheng-qian/etc-qwen3.5-0.8b` identifier when it is
already present in the local Hugging Face cache.
- The runner uses `local_files_only=True`; it never downloads weights during
evaluation. Model weights, caches, and virtual environments remain outside
the repository.

## Source and References

- [ETC-Bench](https://github.com/zh-qian2004/etc-bench), upstream revision
`c317d5de3f9462370057df42441a3b7f3ecee728`.
- Published model family: `zhiheng-qian/etc-qwen3.5-0.8b` and
`zhiheng-qian/etc-qwen3.5-4b`.

## Limitations

- Real inference requires a user-prepared checkpoint and the optional node
environment; the base SURE installation does not include either.
- The model may conservatively retain ambiguous homophones or abbreviations.
- Generation uses greedy decoding with `max_new_tokens=256`; output that
reaches the budget without EOS is rejected rather than scored as complete.
- The learned canonicalizer may change fillers, repetitions, casing,
punctuation, numbers, symbols, and other transcript surface content.
- The node currently exposes the ETC-Bench 12-language profile set.
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
"""ETC-Bench-compatible evaluation-side transcript canonicalization."""

from .node import (
NODE_ID,
NODE_VERSION,
SUPPORTED_PROFILES,
normalize_etc_key_text_files,
normalize_etc_text,
)

__all__ = [
"NODE_ID",
"NODE_VERSION",
"SUPPORTED_PROFILES",
"normalize_etc_key_text_files",
"normalize_etc_text",
]
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
"""Dependency-free generation budget checks shared by runtime and tests."""

MAX_NEW_TOKENS = 256


def ensure_generation_completed(generated_count: int, last_token: int, eos_token_id) -> None:
eos_ids = set(eos_token_id if isinstance(eos_token_id, (list, tuple)) else [eos_token_id])
if generated_count >= MAX_NEW_TOKENS and last_token not in eos_ids:
raise RuntimeError(
f"ETC canonicalizer exhausted max_new_tokens={MAX_NEW_TOKENS} without EOS; "
"input requires a shorter transcript or a separately configured runtime budget"
)
Loading
Loading