From 7b6879781baf799bdb21c214d469c08f9f641763 Mon Sep 17 00:00:00 2001 From: Jason040114 Date: Mon, 2 Nov 2026 11:19:54 +0800 Subject: [PATCH 1/7] feat: add ETC-Bench canonicalization node --- docs/pipeline_catalog.jsonl | 12 ++ .../etc_canonicalization/README.md | 21 +++ .../etc_canonicalization/__init__.py | 17 ++ .../etc_canonicalization/manifest.yaml | 31 ++++ .../etc_canonicalization/model_runtime.py | 56 ++++++ .../etc_canonicalization/node.py | 171 ++++++++++++++++++ .../etc_canonicalization/node_env.yaml | 20 ++ .../etc_canonicalization/pyproject.toml | 12 ++ .../etc_canonicalization/runner.py | 33 ++++ src/sure_eval/evaluation/scripts/asr.py | 2 + .../evaluation/tasks/asr/pipeline.py | 39 ++++ .../evaluation/tasks/asr/routes.yaml | 108 +++++++++++ tests/test_etc_canonicalization_node.py | 156 ++++++++++++++++ 13 files changed, 678 insertions(+) create mode 100644 src/sure_eval/evaluation/nodes/normalization/etc_canonicalization/README.md create mode 100644 src/sure_eval/evaluation/nodes/normalization/etc_canonicalization/__init__.py create mode 100644 src/sure_eval/evaluation/nodes/normalization/etc_canonicalization/manifest.yaml create mode 100644 src/sure_eval/evaluation/nodes/normalization/etc_canonicalization/model_runtime.py create mode 100644 src/sure_eval/evaluation/nodes/normalization/etc_canonicalization/node.py create mode 100644 src/sure_eval/evaluation/nodes/normalization/etc_canonicalization/node_env.yaml create mode 100644 src/sure_eval/evaluation/nodes/normalization/etc_canonicalization/pyproject.toml create mode 100644 src/sure_eval/evaluation/nodes/normalization/etc_canonicalization/runner.py create mode 100644 tests/test_etc_canonicalization_node.py diff --git a/docs/pipeline_catalog.jsonl b/docs/pipeline_catalog.jsonl index 67132f08..e898b360 100644 --- a/docs/pipeline_catalog.jsonl +++ b/docs/pipeline_catalog.jsonl @@ -35,6 +35,18 @@ {"task": "ASR", "task_alias": "asr", "language": "ar-sa", "metric": "wer", "pipeline_id": "asr.ar_sa.wer.giga_norm_sau_v1.wenet_wer_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["wer"], "nodes": ["normalization/giga_norm", "scoring/wenet_wer"], "computation_node_ids": ["normalization/giga_norm", "scoring/wenet_wer"], "task_config_path": "src/sure_eval/evaluation/tasks/asr/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/asr/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.asr.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.asr.run", "executor": "sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files", "required_roles": ["hyp", "ref"], "optional_roles": []} {"task": "ASR", "task_alias": "asr", "language": "ar-sy", "metric": "wer", "pipeline_id": "asr.ar_sy.wer.giga_norm_syr_v1.wenet_wer_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["wer"], "nodes": ["normalization/giga_norm", "scoring/wenet_wer"], "computation_node_ids": ["normalization/giga_norm", "scoring/wenet_wer"], "task_config_path": "src/sure_eval/evaluation/tasks/asr/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/asr/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.asr.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.asr.run", "executor": "sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files", "required_roles": ["hyp", "ref"], "optional_roles": []} {"task": "ASR", "task_alias": "asr", "language": "ar", "metric": "cer", "pipeline_id": "asr.ar.cer.xnorm_ar_v1.wenet_cer_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["cer"], "nodes": ["normalization/xnorm", "scoring/wenet_cer"], "computation_node_ids": ["normalization/xnorm", "scoring/wenet_cer"], "task_config_path": "src/sure_eval/evaluation/tasks/asr/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/asr/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.asr.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.asr.run", "executor": "sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files", "required_roles": ["hyp", "ref"], "optional_roles": []} +{"task": "ASR", "task_alias": "asr", "language": "ar", "metric": "cer", "pipeline_id": "asr.ar.cer.etc_canonicalization_ar_v1.wenet_cer_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["cer"], "nodes": ["normalization/etc_canonicalization", "scoring/wenet_cer"], "computation_node_ids": ["normalization/etc_canonicalization", "scoring/wenet_cer"], "task_config_path": "src/sure_eval/evaluation/tasks/asr/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/asr/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.asr.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.asr.run", "executor": "sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files", "required_roles": ["hyp", "ref"], "optional_roles": []} +{"task": "ASR", "task_alias": "asr", "language": "bn", "metric": "wer", "pipeline_id": "asr.bn.wer.etc_canonicalization_bn_v1.wenet_wer_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["wer"], "nodes": ["normalization/etc_canonicalization", "scoring/wenet_wer"], "computation_node_ids": ["normalization/etc_canonicalization", "scoring/wenet_wer"], "task_config_path": "src/sure_eval/evaluation/tasks/asr/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/asr/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.asr.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.asr.run", "executor": "sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files", "required_roles": ["hyp", "ref"], "optional_roles": []} +{"task": "ASR", "task_alias": "asr", "language": "de", "metric": "wer", "pipeline_id": "asr.de.wer.etc_canonicalization_de_v1.wenet_wer_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["wer"], "nodes": ["normalization/etc_canonicalization", "scoring/wenet_wer"], "computation_node_ids": ["normalization/etc_canonicalization", "scoring/wenet_wer"], "task_config_path": "src/sure_eval/evaluation/tasks/asr/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/asr/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.asr.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.asr.run", "executor": "sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files", "required_roles": ["hyp", "ref"], "optional_roles": []} +{"task": "ASR", "task_alias": "asr", "language": "en", "metric": "wer", "pipeline_id": "asr.en.wer.etc_canonicalization_en_v1.wenet_wer_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["wer"], "nodes": ["normalization/etc_canonicalization", "scoring/wenet_wer"], "computation_node_ids": ["normalization/etc_canonicalization", "scoring/wenet_wer"], "task_config_path": "src/sure_eval/evaluation/tasks/asr/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/asr/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.asr.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.asr.run", "executor": "sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files", "required_roles": ["hyp", "ref"], "optional_roles": []} +{"task": "ASR", "task_alias": "asr", "language": "fa", "metric": "wer", "pipeline_id": "asr.fa.wer.etc_canonicalization_fa_v1.wenet_wer_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["wer"], "nodes": ["normalization/etc_canonicalization", "scoring/wenet_wer"], "computation_node_ids": ["normalization/etc_canonicalization", "scoring/wenet_wer"], "task_config_path": "src/sure_eval/evaluation/tasks/asr/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/asr/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.asr.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.asr.run", "executor": "sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files", "required_roles": ["hyp", "ref"], "optional_roles": []} +{"task": "ASR", "task_alias": "asr", "language": "fr", "metric": "wer", "pipeline_id": "asr.fr.wer.etc_canonicalization_fr_v1.wenet_wer_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["wer"], "nodes": ["normalization/etc_canonicalization", "scoring/wenet_wer"], "computation_node_ids": ["normalization/etc_canonicalization", "scoring/wenet_wer"], "task_config_path": "src/sure_eval/evaluation/tasks/asr/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/asr/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.asr.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.asr.run", "executor": "sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files", "required_roles": ["hyp", "ref"], "optional_roles": []} +{"task": "ASR", "task_alias": "asr", "language": "hi", "metric": "wer", "pipeline_id": "asr.hi.wer.etc_canonicalization_hi_v1.wenet_wer_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["wer"], "nodes": ["normalization/etc_canonicalization", "scoring/wenet_wer"], "computation_node_ids": ["normalization/etc_canonicalization", "scoring/wenet_wer"], "task_config_path": "src/sure_eval/evaluation/tasks/asr/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/asr/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.asr.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.asr.run", "executor": "sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files", "required_roles": ["hyp", "ref"], "optional_roles": []} +{"task": "ASR", "task_alias": "asr", "language": "id", "metric": "wer", "pipeline_id": "asr.id.wer.etc_canonicalization_id_v1.wenet_wer_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["wer"], "nodes": ["normalization/etc_canonicalization", "scoring/wenet_wer"], "computation_node_ids": ["normalization/etc_canonicalization", "scoring/wenet_wer"], "task_config_path": "src/sure_eval/evaluation/tasks/asr/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/asr/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.asr.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.asr.run", "executor": "sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files", "required_roles": ["hyp", "ref"], "optional_roles": []} +{"task": "ASR", "task_alias": "asr", "language": "ja", "metric": "cer", "pipeline_id": "asr.ja.cer.etc_canonicalization_ja_v1.wenet_cer_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["cer"], "nodes": ["normalization/etc_canonicalization", "scoring/wenet_cer"], "computation_node_ids": ["normalization/etc_canonicalization", "scoring/wenet_cer"], "task_config_path": "src/sure_eval/evaluation/tasks/asr/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/asr/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.asr.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.asr.run", "executor": "sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files", "required_roles": ["hyp", "ref"], "optional_roles": []} +{"task": "ASR", "task_alias": "asr", "language": "ru", "metric": "wer", "pipeline_id": "asr.ru.wer.etc_canonicalization_ru_v1.wenet_wer_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["wer"], "nodes": ["normalization/etc_canonicalization", "scoring/wenet_wer"], "computation_node_ids": ["normalization/etc_canonicalization", "scoring/wenet_wer"], "task_config_path": "src/sure_eval/evaluation/tasks/asr/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/asr/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.asr.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.asr.run", "executor": "sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files", "required_roles": ["hyp", "ref"], "optional_roles": []} +{"task": "ASR", "task_alias": "asr", "language": "vi", "metric": "wer", "pipeline_id": "asr.vi.wer.etc_canonicalization_vi_v1.wenet_wer_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["wer"], "nodes": ["normalization/etc_canonicalization", "scoring/wenet_wer"], "computation_node_ids": ["normalization/etc_canonicalization", "scoring/wenet_wer"], "task_config_path": "src/sure_eval/evaluation/tasks/asr/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/asr/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.asr.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.asr.run", "executor": "sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files", "required_roles": ["hyp", "ref"], "optional_roles": []} +{"task": "ASR", "task_alias": "asr", "language": "zh", "metric": "cer", "pipeline_id": "asr.zh.cer.etc_canonicalization_zh_v1.wenet_cer_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["cer"], "nodes": ["normalization/etc_canonicalization", "scoring/wenet_cer"], "computation_node_ids": ["normalization/etc_canonicalization", "scoring/wenet_cer"], "task_config_path": "src/sure_eval/evaluation/tasks/asr/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/asr/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.asr.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.asr.run", "executor": "sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files", "required_roles": ["hyp", "ref"], "optional_roles": []} {"task": "ASR", "task_alias": "asr", "language": "de", "metric": "wer", "pipeline_id": "asr.de.wer.xnorm_de_v1.wenet_wer_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["wer"], "nodes": ["normalization/xnorm", "scoring/wenet_wer"], "computation_node_ids": ["normalization/xnorm", "scoring/wenet_wer"], "task_config_path": "src/sure_eval/evaluation/tasks/asr/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/asr/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.asr.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.asr.run", "executor": "sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files", "required_roles": ["hyp", "ref"], "optional_roles": []} {"task": "ASR", "task_alias": "asr", "language": "en", "metric": "wer", "pipeline_id": "asr.en.wer.xnorm_en_v1.wenet_wer_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["wer"], "nodes": ["normalization/xnorm", "scoring/wenet_wer"], "computation_node_ids": ["normalization/xnorm", "scoring/wenet_wer"], "task_config_path": "src/sure_eval/evaluation/tasks/asr/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/asr/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.asr.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.asr.run", "executor": "sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files", "required_roles": ["hyp", "ref"], "optional_roles": []} {"task": "ASR", "task_alias": "asr", "language": "es", "metric": "wer", "pipeline_id": "asr.es.wer.xnorm_es_v1.wenet_wer_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["wer"], "nodes": ["normalization/xnorm", "scoring/wenet_wer"], "computation_node_ids": ["normalization/xnorm", "scoring/wenet_wer"], "task_config_path": "src/sure_eval/evaluation/tasks/asr/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/asr/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.asr.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.asr.run", "executor": "sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files", "required_roles": ["hyp", "ref"], "optional_roles": []} diff --git a/src/sure_eval/evaluation/nodes/normalization/etc_canonicalization/README.md b/src/sure_eval/evaluation/nodes/normalization/etc_canonicalization/README.md new file mode 100644 index 00000000..19ef7acb --- /dev/null +++ b/src/sure_eval/evaluation/nodes/normalization/etc_canonicalization/README.md @@ -0,0 +1,21 @@ +# ETC canonicalization + +This optional node adapts the model canonicalizer interface released with +[ETC-Bench](https://github.com/zh-qian2004/etc-bench). It independently maps +reference and hypothesis transcripts to a deterministic shared written form. + +Prepare its node-local environment with: + +```bash +sure-eval env setup --node normalization/etc_canonicalization +``` + +Download one of the published checkpoints separately, then set +`SURE_ETC_CANONICALIZATION_MODEL` to its local directory. The default model ID +is `zhiheng-qian/etc-qwen3.5-0.8b`; it works only when that checkpoint is +already present in the local Hugging Face cache. The runner uses +`local_files_only=True`, so evaluation never downloads weights implicitly. + +The node environment and model checkpoint are intentionally not included in +SURE-Evaluation or this change. After cloning, install the optional node +environment and configure the checkpoint in your own runtime. diff --git a/src/sure_eval/evaluation/nodes/normalization/etc_canonicalization/__init__.py b/src/sure_eval/evaluation/nodes/normalization/etc_canonicalization/__init__.py new file mode 100644 index 00000000..990f1522 --- /dev/null +++ b/src/sure_eval/evaluation/nodes/normalization/etc_canonicalization/__init__.py @@ -0,0 +1,17 @@ +"""ETC-Bench-compatible evaluation-side transcript canonicalization.""" + +from .node import ( + NODE_ID, + NODE_VERSION, + SUPPORTED_PROFILES, + normalize_etc_key_text_files, + normalize_etc_text, +) + +__all__ = [ + "NODE_ID", + "NODE_VERSION", + "SUPPORTED_PROFILES", + "normalize_etc_key_text_files", + "normalize_etc_text", +] diff --git a/src/sure_eval/evaluation/nodes/normalization/etc_canonicalization/manifest.yaml b/src/sure_eval/evaluation/nodes/normalization/etc_canonicalization/manifest.yaml new file mode 100644 index 00000000..47b579ab --- /dev/null +++ b/src/sure_eval/evaluation/nodes/normalization/etc_canonicalization/manifest.yaml @@ -0,0 +1,31 @@ +id: normalization/etc_canonicalization +version: v1 +stage: normalization +language_sensitive: true +input_schema: key_text_files +output_schema: key_text_files +implementation: sure_eval.evaluation.nodes.normalization.etc_canonicalization.node +runtime: node_local_optional +package: + name: etc-canonicalizer + version: "1.0.0" + source: etc-bench model-canonicalizer adapter + upstream: https://github.com/zh-qian2004/etc-bench + upstream_commit: c317d5de3f9462370057df42441a3b7f3ecee728 +profiles: + ar: {language: ar, canonicalizer: etc-qwen3.5} + bn: {language: bn, canonicalizer: etc-qwen3.5} + de: {language: de, canonicalizer: etc-qwen3.5} + en: {language: en, canonicalizer: etc-qwen3.5} + fa: {language: fa, canonicalizer: etc-qwen3.5} + fr: {language: fr, canonicalizer: etc-qwen3.5} + hi: {language: hi, canonicalizer: etc-qwen3.5} + id: {language: id, canonicalizer: etc-qwen3.5} + ja: {language: ja, canonicalizer: etc-qwen3.5} + ru: {language: ru, canonicalizer: etc-qwen3.5} + vi: {language: vi, canonicalizer: etc-qwen3.5} + zh: {language: zh, canonicalizer: etc-qwen3.5} +notes: + - The model canonicalizes reference and hypothesis independently according to ETC-Bench. + - Model weights are external and must not be committed to the repository. + - The node loads checkpoints only from a user-prepared local directory or local Hugging Face cache. diff --git a/src/sure_eval/evaluation/nodes/normalization/etc_canonicalization/model_runtime.py b/src/sure_eval/evaluation/nodes/normalization/etc_canonicalization/model_runtime.py new file mode 100644 index 00000000..bf654d3d --- /dev/null +++ b/src/sure_eval/evaluation/nodes/normalization/etc_canonicalization/model_runtime.py @@ -0,0 +1,56 @@ +"""Minimal ETC-Qwen inference implementation kept inside the optional runtime.""" + +from __future__ import annotations + +from pathlib import Path + +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer + +SYSTEM_PROMPT = ( + "You are an ASR post-processing normalizer. Rewrite the given transcript into one " + "deterministic shared written form so that reference and hypothesis transcripts of " + "the same speech become identical: restore punctuation and casing, drop fillers, " + "stutters and repetitions, and write numbers and symbols by written convention. " + "Keep all other content unchanged. Output only the normalized text, with no " + "explanation, quotes, JSON or markup." +) + + +def canonicalize_batch(texts: list[str], *, model: str) -> list[str]: + device = "cuda" if torch.cuda.is_available() else "cpu" + dtype = torch.bfloat16 if device != "cpu" else torch.float32 + tokenizer = AutoTokenizer.from_pretrained(model, local_files_only=True) + template = Path(model) / "chat_template.jinja" + if template.exists(): + tokenizer.chat_template = template.read_text(encoding="utf-8") + if tokenizer.pad_token_id is None: + tokenizer.pad_token = tokenizer.eos_token + tokenizer.padding_side = "left" + loaded = AutoModelForCausalLM.from_pretrained( + model, dtype=dtype, local_files_only=True + ).to(device).eval() + loaded.config.use_cache = True + outputs = [] + for text in texts: + prompt = tokenizer.apply_chat_template( + [{"role": "system", "content": SYSTEM_PROMPT}, {"role": "user", "content": text}], + tokenize=False, + add_generation_prompt=True, + enable_thinking=False, + ) + inputs = tokenizer(prompt, return_tensors="pt").to(device) + with torch.no_grad(): + generated = loaded.generate( + **inputs, + max_new_tokens=256, + do_sample=False, + pad_token_id=tokenizer.pad_token_id, + eos_token_id=tokenizer.eos_token_id, + ) + outputs.append( + tokenizer.decode( + generated[0, inputs.input_ids.shape[1] :], skip_special_tokens=True + ).strip() + ) + return outputs diff --git a/src/sure_eval/evaluation/nodes/normalization/etc_canonicalization/node.py b/src/sure_eval/evaluation/nodes/normalization/etc_canonicalization/node.py new file mode 100644 index 00000000..cc47e4a5 --- /dev/null +++ b/src/sure_eval/evaluation/nodes/normalization/etc_canonicalization/node.py @@ -0,0 +1,171 @@ +"""SURE adapter for the ETC-Bench model canonicalizer contract.""" + +from __future__ import annotations + +import json +import os +import subprocess +import tempfile +from pathlib import Path + +from sure_eval.evaluation.core.types import KeyTextFiles, PipelineNodeResult +from sure_eval.evaluation.nodes.common.node_local_python import ( + build_node_local_env, + resolve_node_local_python, +) + +NODE_ID = "normalization/etc_canonicalization" +NODE_VERSION = "v1" +DEFAULT_MODEL = "zhiheng-qian/etc-qwen3.5-0.8b" +ETC_BENCH_VERSION = "2026.10" +NODE_DIR = Path(__file__).resolve().parent +MODULE_NAME = "sure_eval.evaluation.nodes.normalization.etc_canonicalization.runner" +SUPPORTED_PROFILES = ("ar", "bn", "de", "en", "fa", "fr", "hi", "id", "ja", "ru", "vi", "zh") + + +def _validate_language(language: str) -> str: + normalized = str(language).lower().strip() + if normalized not in SUPPORTED_PROFILES: + supported = ", ".join(SUPPORTED_PROFILES) + raise ValueError( + f"Unsupported etc_canonicalization language: {language!r}; supported: {supported}" + ) + return normalized + + +def _model_id(model: str | None = None) -> str: + return str(model or os.environ.get("SURE_ETC_CANONICALIZATION_MODEL") or DEFAULT_MODEL) + + +def canonicalize_batch(texts: list[str], *, language: str, model: str | None = None) -> list[str]: + """Canonicalize a batch with one model load in the node-local runtime.""" + normalized_language = _validate_language(language) + if not texts: + return [] + try: + python_runtime = resolve_node_local_python(NODE_DIR, NODE_ID) + except RuntimeError as exc: + raise RuntimeError( + f"{NODE_ID} requires its node-local environment. Run: sure-eval env setup --node {NODE_ID}" + ) from exc + repo_root = NODE_DIR.parents[5] + env = build_node_local_env( + repo_src=repo_root / "src", + extra_pythonpath=python_runtime.extra_pythonpath, + inherit_pythonpath=python_runtime.inherit_pythonpath, + ) + completed = subprocess.run( + [ + *python_runtime.command_prefix, + "-m", + MODULE_NAME, + "--language", + normalized_language, + "--model", + _model_id(model), + ], + cwd=repo_root, + env=env, + text=True, + capture_output=True, + check=False, + input=json.dumps({"texts": texts}, ensure_ascii=False), + ) + if completed.returncode != 0: + detail = completed.stderr.strip() or completed.stdout.strip() + raise RuntimeError(f"{NODE_ID} canonicalizer failed: {detail}") + try: + payload = json.loads(completed.stdout) + predictions = payload["predictions"] + except (json.JSONDecodeError, KeyError, TypeError) as exc: + raise RuntimeError(f"{NODE_ID} returned invalid runner output") from exc + if not isinstance(predictions, list) or len(predictions) != len(texts) or not all( + isinstance(prediction, str) for prediction in predictions + ): + raise RuntimeError(f"{NODE_ID} returned invalid canonicalization results") + return [prediction.strip() for prediction in predictions] + + +def canonicalize_text(text: str, *, language: str, model: str | None = None) -> str: + return canonicalize_batch([text], language=language, model=model)[0] + + +def normalize_etc_text(text: str, *, language: str, model: str | None = None) -> str: + """Canonicalize one text string after validating its ETC language profile.""" + + normalized_language = _validate_language(language) + return canonicalize_text(text, language=normalized_language, model=model) + + +def normalize_etc_key_text_files( + files: KeyTextFiles, + *, + language: str, + model: str | None = None, +) -> tuple[KeyTextFiles, PipelineNodeResult]: + """Canonicalize ref and hyp files independently while preserving keys.""" + + normalized_language = _validate_language(language) + ref_output = None + try: + ref_output, ref_rows = _normalize_file(files.ref_file, normalized_language, model=model) + hyp_output, hyp_rows = _normalize_file(files.hyp_file, normalized_language, model=model) + except Exception: + if ref_output: + Path(ref_output).unlink(missing_ok=True) + raise + row_counts = {"ref": len(ref_rows), "hyp": len(hyp_rows)} + changed_counts = { + "ref": sum(row["original_text"] != row["normalized_text"] for row in ref_rows), + "hyp": sum(row["original_text"] != row["normalized_text"] for row in hyp_rows), + } + empty_counts = { + "ref": sum(not row["normalized_text"] for row in ref_rows), + "hyp": sum(not row["normalized_text"] for row in hyp_rows), + } + trace = PipelineNodeResult( + stage="normalization", + node_id=NODE_ID, + version=NODE_VERSION, + details={ + "language": normalized_language, + "model": _model_id(model), + "backend": "etc_bench_model_canonicalizer", + "etc_bench_version": ETC_BENCH_VERSION, + "input_schema": "key_text_files", + "output_schema": "key_text_files", + "row_counts": row_counts, + "ref_file": ref_output, + "hyp_file": hyp_output, + "changed_counts": changed_counts, + "num_empty_after_normalization": empty_counts, + "ref_rows": ref_rows, + "hyp_rows": hyp_rows, + }, + internal_stages=("key_text_parse", "etc_model_canonicalize", "key_text_write"), + ) + return KeyTextFiles(ref_file=ref_output, hyp_file=hyp_output), trace + + +def _normalize_file( + input_file: str, language: str, *, model: str | None +) -> tuple[str, list[dict[str, str]]]: + source_rows: list[tuple[str, str]] = [] + with open(input_file, encoding="utf-8") as source: + for line in source: + if "\t" in line: + source_rows.append(tuple(line.rstrip("\n").split("\t", 1))) + predictions = canonicalize_batch( + [original for _, original in source_rows], language=language, model=model + ) + output_handle = tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False, encoding="utf-8") + rows: list[dict[str, str]] = [] + try: + with output_handle as target: + for (key, original_text), normalized_text in zip(source_rows, predictions): + target.write(f"{key}\t{normalized_text}\n") + rows.append({"key": key, "original_text": original_text, "normalized_text": normalized_text}) + except Exception: + Path(output_handle.name).unlink(missing_ok=True) + raise + return output_handle.name, rows diff --git a/src/sure_eval/evaluation/nodes/normalization/etc_canonicalization/node_env.yaml b/src/sure_eval/evaluation/nodes/normalization/etc_canonicalization/node_env.yaml new file mode 100644 index 00000000..a0234086 --- /dev/null +++ b/src/sure_eval/evaluation/nodes/normalization/etc_canonicalization/node_env.yaml @@ -0,0 +1,20 @@ +id: normalization/etc_canonicalization +group: normalization-extra +runtime: + type: uv + python: "3.11" + project: pyproject.toml + frozen: false + optional: true + gpu: true +packages: + - name: torch + version: "==2.10.0" + - name: transformers + version: "==5.14.1" + - name: accelerate + version: "==1.14.0" +verify: + imports: + - torch + - transformers diff --git a/src/sure_eval/evaluation/nodes/normalization/etc_canonicalization/pyproject.toml b/src/sure_eval/evaluation/nodes/normalization/etc_canonicalization/pyproject.toml new file mode 100644 index 00000000..ad586955 --- /dev/null +++ b/src/sure_eval/evaluation/nodes/normalization/etc_canonicalization/pyproject.toml @@ -0,0 +1,12 @@ +[project] +name = "sure-eval-normalization-etc-canonicalization" +version = "0.1.0" +requires-python = ">=3.11" +dependencies = [ + "torch==2.10.0", + "transformers==5.14.1", + "accelerate==1.14.0", +] + +[tool.uv] +package = false diff --git a/src/sure_eval/evaluation/nodes/normalization/etc_canonicalization/runner.py b/src/sure_eval/evaluation/nodes/normalization/etc_canonicalization/runner.py new file mode 100644 index 00000000..36fdd153 --- /dev/null +++ b/src/sure_eval/evaluation/nodes/normalization/etc_canonicalization/runner.py @@ -0,0 +1,33 @@ +"""Node-local inference entrypoint for the released ETC canonicalizer.""" + +from __future__ import annotations + +import argparse +import json +import os +import sys + +os.environ.setdefault("TORCH_DEVICE_BACKEND_AUTOLOAD", "0") + +from .model_runtime import canonicalize_batch + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--language", required=True) + parser.add_argument("--model", required=True) + args = parser.parse_args() + try: + texts = json.load(sys.stdin)["texts"] + if not isinstance(texts, list) or not all(isinstance(text, str) for text in texts): + raise ValueError("texts must be a list of strings") + payload = {"predictions": canonicalize_batch(texts, model=args.model)} + except Exception as exc: + print(f"ETC canonicalizer failed: {exc}", file=sys.stderr) + return 1 + print(json.dumps(payload, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/sure_eval/evaluation/scripts/asr.py b/src/sure_eval/evaluation/scripts/asr.py index abf34f91..4ddeec06 100644 --- a/src/sure_eval/evaluation/scripts/asr.py +++ b/src/sure_eval/evaluation/scripts/asr.py @@ -124,6 +124,8 @@ def _executor_selectors_from_route(route: dict) -> dict[str, str]: selectors["normalizer"] = "nemo:ar_tn" elif node_id == "normalization/xnorm": selectors["normalizer"] = f"xnorm:{route.get('language')}" + elif node_id == "normalization/etc_canonicalization": + selectors["normalizer"] = f"etc:{route.get('language')}" elif node_id in {"scoring/wenet_cer", "scoring/wenet_wer", "scoring/wenet_mer"}: selectors["scorer"] = "wenet" elif node_id == "scoring/token_cer": diff --git a/src/sure_eval/evaluation/tasks/asr/pipeline.py b/src/sure_eval/evaluation/tasks/asr/pipeline.py index 35f3c2a8..f119b37b 100644 --- a/src/sure_eval/evaluation/tasks/asr/pipeline.py +++ b/src/sure_eval/evaluation/tasks/asr/pipeline.py @@ -42,6 +42,10 @@ SUPPORTED_PROFILES as XNORM_PROFILES, normalize_xnorm_key_text_files, ) +from sure_eval.evaluation.nodes.normalization.etc_canonicalization import ( + SUPPORTED_PROFILES as ETC_SUPPORTED_PROFILES, + normalize_etc_key_text_files, +) from sure_eval.evaluation.nodes.scoring.sctk_sclite import ( score_sctk_sclite_cer, score_sctk_sclite_wer, @@ -320,6 +324,18 @@ def _normalize_normalizer(*, language: str, metric: str, normalizer: str | None) profile = normalized.split(":", 1)[1] _validate_xnorm_profile_for_language(language=language, profile=profile) return f"xnorm:{profile}" + if normalized in { + "etc", + "etc_canonicalization", + "etc-canonicalization", + "normalization/etc_canonicalization", + }: + _validate_etc_profile_for_language(language=language, profile=language) + return f"etc:{language}" + if normalized.startswith("etc:"): + profile = normalized.split(":", 1)[1] + _validate_etc_profile_for_language(language=language, profile=profile) + return f"etc:{profile}" raise ValueError(f"Unsupported ASR normalizer: {normalizer}") @@ -402,6 +418,12 @@ def _normalization_node(*, language: str, normalizer: str): lambda files: normalize_xnorm_key_text_files(files, language=profile), f"xnorm_{profile}", ) + if normalizer.startswith("etc:"): + profile = normalizer.split(":", 1)[1] + return ( + lambda files: normalize_etc_key_text_files(files, language=profile), + f"etc_{profile}", + ) raise ValueError(f"Unsupported ASR normalizer: {normalizer}") @@ -457,6 +479,11 @@ def _normalizer_component(*, language: str, normalizer_label: str): "normalization/xnorm", profile=normalizer_label.removeprefix("xnorm_"), ) + if normalizer_label.startswith("etc_"): + return node_component( + "normalization/etc_canonicalization", + profile=normalizer_label.removeprefix("etc_"), + ) raise ValueError(f"Unsupported ASR normalizer label: {normalizer_label}") @@ -522,6 +549,18 @@ def _validate_xnorm_profile_for_language(*, language: str, profile: str) -> None raise ValueError(f"xnorm profile {profile!r} does not match ASR language {language!r}") +def _validate_etc_profile_for_language(*, language: str, profile: str) -> None: + if profile not in ETC_SUPPORTED_PROFILES: + supported = ", ".join(sorted(ETC_SUPPORTED_PROFILES)) + raise ValueError( + f"Unsupported etc_canonicalization profile {profile!r}; supported: {supported}" + ) + if profile != language: + raise ValueError( + f"etc_canonicalization profile {profile!r} does not match ASR language {language!r}" + ) + + def _cleanup_trace_temp_files(trace: tuple[PipelineNodeResult, ...]) -> None: for result in trace: for key in ("ref_file", "hyp_file"): diff --git a/src/sure_eval/evaluation/tasks/asr/routes.yaml b/src/sure_eval/evaluation/tasks/asr/routes.yaml index c2f10b6c..df54f07f 100644 --- a/src/sure_eval/evaluation/tasks/asr/routes.yaml +++ b/src/sure_eval/evaluation/tasks/asr/routes.yaml @@ -341,6 +341,114 @@ routes: - scoring/wenet_cer input_contract: scoring/wenet_cer executor: sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files + - + language: ar + metric: cer + pipeline_id: asr.ar.cer.etc_canonicalization_ar_v1.wenet_cer_v1 + nodes: + - normalization/etc_canonicalization + - scoring/wenet_cer + input_contract: scoring/wenet_cer + executor: sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files + - + language: bn + metric: wer + pipeline_id: asr.bn.wer.etc_canonicalization_bn_v1.wenet_wer_v1 + nodes: + - normalization/etc_canonicalization + - scoring/wenet_wer + input_contract: scoring/wenet_wer + executor: sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files + - + language: de + metric: wer + pipeline_id: asr.de.wer.etc_canonicalization_de_v1.wenet_wer_v1 + nodes: + - normalization/etc_canonicalization + - scoring/wenet_wer + input_contract: scoring/wenet_wer + executor: sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files + - + language: en + metric: wer + pipeline_id: asr.en.wer.etc_canonicalization_en_v1.wenet_wer_v1 + nodes: + - normalization/etc_canonicalization + - scoring/wenet_wer + input_contract: scoring/wenet_wer + executor: sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files + - + language: fa + metric: wer + pipeline_id: asr.fa.wer.etc_canonicalization_fa_v1.wenet_wer_v1 + nodes: + - normalization/etc_canonicalization + - scoring/wenet_wer + input_contract: scoring/wenet_wer + executor: sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files + - + language: fr + metric: wer + pipeline_id: asr.fr.wer.etc_canonicalization_fr_v1.wenet_wer_v1 + nodes: + - normalization/etc_canonicalization + - scoring/wenet_wer + input_contract: scoring/wenet_wer + executor: sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files + - + language: hi + metric: wer + pipeline_id: asr.hi.wer.etc_canonicalization_hi_v1.wenet_wer_v1 + nodes: + - normalization/etc_canonicalization + - scoring/wenet_wer + input_contract: scoring/wenet_wer + executor: sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files + - + language: id + metric: wer + pipeline_id: asr.id.wer.etc_canonicalization_id_v1.wenet_wer_v1 + nodes: + - normalization/etc_canonicalization + - scoring/wenet_wer + input_contract: scoring/wenet_wer + executor: sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files + - + language: ja + metric: cer + pipeline_id: asr.ja.cer.etc_canonicalization_ja_v1.wenet_cer_v1 + nodes: + - normalization/etc_canonicalization + - scoring/wenet_cer + input_contract: scoring/wenet_cer + executor: sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files + - + language: ru + metric: wer + pipeline_id: asr.ru.wer.etc_canonicalization_ru_v1.wenet_wer_v1 + nodes: + - normalization/etc_canonicalization + - scoring/wenet_wer + input_contract: scoring/wenet_wer + executor: sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files + - + language: vi + metric: wer + pipeline_id: asr.vi.wer.etc_canonicalization_vi_v1.wenet_wer_v1 + nodes: + - normalization/etc_canonicalization + - scoring/wenet_wer + input_contract: scoring/wenet_wer + executor: sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files + - + language: zh + metric: cer + pipeline_id: asr.zh.cer.etc_canonicalization_zh_v1.wenet_cer_v1 + nodes: + - normalization/etc_canonicalization + - scoring/wenet_cer + input_contract: scoring/wenet_cer + executor: sure_eval.evaluation.tasks.asr.pipeline.evaluate_asr_files - language: de metric: wer diff --git a/tests/test_etc_canonicalization_node.py b/tests/test_etc_canonicalization_node.py new file mode 100644 index 00000000..8a8bc996 --- /dev/null +++ b/tests/test_etc_canonicalization_node.py @@ -0,0 +1,156 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + + +def test_etc_supported_profiles_and_text_adapter(monkeypatch: pytest.MonkeyPatch) -> None: + from sure_eval.evaluation.nodes.normalization.etc_canonicalization import node + + assert set(node.SUPPORTED_PROFILES) == { + "ar", + "bn", + "de", + "en", + "fa", + "fr", + "hi", + "id", + "ja", + "ru", + "vi", + "zh", + } + monkeypatch.setattr(node, "canonicalize_batch", lambda texts, **_: [text.replace("two", "2") for text in texts]) + assert node.normalize_etc_text("two eggs", language="en") == "2 eggs" + with pytest.raises(ValueError, match="Unsupported etc_canonicalization language"): + node.normalize_etc_text("two", language="xx") + + +def test_etc_key_text_adapter_processes_ref_and_hyp_independently( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from sure_eval.evaluation.core.types import KeyTextFiles + from sure_eval.evaluation.nodes.normalization.etc_canonicalization import node + + monkeypatch.setattr(node, "canonicalize_batch", lambda texts, **_: [text.replace("two", "2") for text in texts]) + ref = tmp_path / "ref.txt" + hyp = tmp_path / "hyp.txt" + ref.write_text("utt-1\tThey need two eggs.\nmalformed\n", encoding="utf-8") + hyp.write_text("utt-1\tThey need 2 eggs.\n", encoding="utf-8") + + normalized, trace = node.normalize_etc_key_text_files( + KeyTextFiles(ref_file=str(ref), hyp_file=str(hyp)), language="en" + ) + + try: + assert Path(normalized.ref_file).read_text(encoding="utf-8") == ( + "utt-1\tThey need 2 eggs.\n" + ) + assert Path(normalized.hyp_file).read_text(encoding="utf-8") == ( + "utt-1\tThey need 2 eggs.\n" + ) + assert trace.node_id == "normalization/etc_canonicalization" + assert trace.details["language"] == "en" + assert trace.details["row_counts"] == {"ref": 1, "hyp": 1} + assert trace.details["changed_counts"] == {"ref": 1, "hyp": 0} + finally: + Path(normalized.ref_file).unlink(missing_ok=True) + Path(normalized.hyp_file).unlink(missing_ok=True) + + +def test_etc_manifest_declares_runtime_and_profiles() -> None: + from sure_eval.evaluation.scripts.contracts import load_node_manifest + + manifest, _ = load_node_manifest("normalization/etc_canonicalization") + assert manifest["runtime"] == "node_local_optional" + assert manifest["package"]["name"] == "etc-canonicalizer" + assert set(manifest["profiles"]) == { + "ar", + "bn", + "de", + "en", + "fa", + "fr", + "hi", + "id", + "ja", + "ru", + "vi", + "zh", + } + + +def test_etc_routes_are_describable_for_every_profile() -> None: + from sure_eval.evaluation.scripts.asr import describe_pipeline + from sure_eval.evaluation.scripts.contracts import load_task_routes + + routes, _ = load_task_routes("asr") + etc_routes = [ + route for route in routes["routes"] + if "normalization/etc_canonicalization" in route["nodes"] + ] + assert {route["language"] for route in etc_routes} == { + "ar", "bn", "de", "en", "fa", "fr", "hi", "id", "ja", "ru", "vi", "zh" + } + for route in etc_routes: + description = describe_pipeline(pipeline_id=route["pipeline_id"]) + assert description.node_ids[0] == "normalization/etc_canonicalization" + + +def test_etc_env_check_requires_user_prepared_runtime() -> None: + from sure_eval.evaluation.env_check import NodeEnvChecker + + result = NodeEnvChecker().check_node("normalization/etc_canonicalization") + assert result.runtime == "node_local_project" + assert result.node_id == "normalization/etc_canonicalization" + if not result.ok: + assert "sure-eval env setup --node normalization/etc_canonicalization" in result.fix + + +def test_etc_normalizer_runs_in_asr_pipeline_with_zero_error( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from sure_eval.evaluation.nodes.normalization.etc_canonicalization import node + from sure_eval.evaluation.tasks.asr.pipeline import evaluate_asr_files + + monkeypatch.setattr(node, "canonicalize_batch", lambda texts, **_: [text.replace("two", "2") for text in texts]) + ref = tmp_path / "ref.txt" + hyp = tmp_path / "hyp.txt" + ref.write_text("utt-1\tThey need two eggs.\n", encoding="utf-8") + hyp.write_text("utt-1\tThey need 2 eggs.\n", encoding="utf-8") + + report = evaluate_asr_files( + str(ref), str(hyp), language="en", metric="wer", normalizer="etc:en" + ) + + assert report.score == 0.0 + assert report.pipeline_trace[0].node_id == "normalization/etc_canonicalization" + assert "etc_canonicalization_en" in report.pipeline_id + + +def test_etc_route_describe_and_run_with_fake_canonicalizer( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from sure_eval.evaluation.nodes.normalization.etc_canonicalization import node + from sure_eval.evaluation.scripts import asr + + monkeypatch.setattr( + node, "canonicalize_batch", lambda texts, **_: [text.replace("two", "2") for text in texts] + ) + pipeline_id = "asr.en.wer.etc_canonicalization_en_v1.wenet_wer_v1" + description = asr.describe_pipeline(pipeline_id=pipeline_id) + assert description.node_ids == ( + "normalization/etc_canonicalization", "scoring/wenet_wer" + ) + + ref = tmp_path / "ref.txt" + hyp = tmp_path / "hyp.txt" + ref.write_text("utt-1\tThey need two eggs.\n", encoding="utf-8") + hyp.write_text("utt-1\tThey need 2 eggs.\n", encoding="utf-8") + result = asr.run( + str(ref), str(hyp), pipeline_id=pipeline_id, output_dir=str(tmp_path / "report") + ) + assert result.pipeline_id == pipeline_id + assert result.score == 0.0 From 2ac190d955fd743d4144bd60a81b807ca3f326a5 Mon Sep 17 00:00:00 2001 From: Jason040114 Date: Mon, 2 Nov 2026 11:48:07 +0800 Subject: [PATCH 2/7] fix: select supported CUDA dtype for ETC inference --- .../normalization/etc_canonicalization/model_runtime.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/sure_eval/evaluation/nodes/normalization/etc_canonicalization/model_runtime.py b/src/sure_eval/evaluation/nodes/normalization/etc_canonicalization/model_runtime.py index bf654d3d..45d05906 100644 --- a/src/sure_eval/evaluation/nodes/normalization/etc_canonicalization/model_runtime.py +++ b/src/sure_eval/evaluation/nodes/normalization/etc_canonicalization/model_runtime.py @@ -19,7 +19,10 @@ def canonicalize_batch(texts: list[str], *, model: str) -> list[str]: device = "cuda" if torch.cuda.is_available() else "cpu" - dtype = torch.bfloat16 if device != "cpu" else torch.float32 + if device == "cuda": + dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16 + else: + dtype = torch.float32 tokenizer = AutoTokenizer.from_pretrained(model, local_files_only=True) template = Path(model) / "chat_template.jinja" if template.exists(): From 379a47f68f545ffbe5b0a225030443a832996cb5 Mon Sep 17 00:00:00 2001 From: Jason040114 Date: Mon, 2 Nov 2026 14:21:35 +0800 Subject: [PATCH 3/7] docs: document ETC node and refresh atlas --- docs/atlas/index.html | 8 +- docs/atlas/pipeline_atlas.svg | 2 +- docs/atlas/pipeline_atlas_dark.svg | 2 +- .../etc_canonicalization/README.md | 74 +++++++++++++++---- 4 files changed, 66 insertions(+), 20 deletions(-) diff --git a/docs/atlas/index.html b/docs/atlas/index.html index 67da6981..5d11152a 100644 --- a/docs/atlas/index.html +++ b/docs/atlas/index.html @@ -103,17 +103,17 @@

SURE Pipeline Atlas

-
FRONTENDTRANSCRIPTIONNORMALIZATIONSCORINGREPORTASR56TTS30TSE16VC16RIR estimation15S2TT10TTA9DOA8SE6VAD5KWS3SA-ASR2SLU2SV2GR1LID1SD1SER1classification1funasr_loader_16k…×4paraformer_zh×4whisper_large_v3×4qwen3_asr_1_7b×2cohere_transcribe…×1xnorm×20giga_norm×17funasr_itn×10whisper_norm×7aispeech_norm×3canonical_itn×3nemo_norm×2wetext_norm×1punctuation_strip…×5doa_jsonl×8vad_timebase×5gstar_norm×1prompt_norm×2lid_label×1wenet_wer×43wenet_cer×20token_mer×2token_cer×1wenet_mer×1dnsmos×9utmos×9wv_mos×9ecapa_tdnn_sim×8eres2net_sim×8wavlm_large_sim×8komos×1si_sdr×3rir_c50_mae×1rir_c80_mae×1rir_doa_mae×1rir_drr_mae×1rir_edc_mae×1rir_iacc_early_mae×1rir_iacc_full_mae×1rir_iacc_late_mae×1rir_iacc_mae×1rir_ild_mae×1rir_ipd_mae×1rir_itd_mae×1rir_npm×1rt30_mae×1rt60_mae×1sacrebleu×6bleurt_20×2xcomet_xl×2audiobox_aestheti…×4audioldm_eval×4clap_score×1doa_localization×8pesq×1stoi×1vad_detection_dur…×4vad_auc_roc×1wekws_det×3meeteval×3classify×6cosine_trial_scor…×2det_eer×1min_dcf_p005×1185 pipelinesreport.json
+
FRONTENDTRANSCRIPTIONNORMALIZATIONSCORINGREPORTASR68TTS30TSE16VC16RIR estimation15S2TT10TTA9DOA8SE6VAD5KWS3SA-ASR2SLU2SV2GR1LID1SD1SER1classification1funasr_loader_16k…×4paraformer_zh×4whisper_large_v3×4qwen3_asr_1_7b×2cohere_transcribe…×1xnorm×20giga_norm×17etc_canonicalizat…×12funasr_itn×10whisper_norm×7aispeech_norm×3canonical_itn×3nemo_norm×2wetext_norm×1punctuation_strip…×5doa_jsonl×8vad_timebase×5gstar_norm×1prompt_norm×2lid_label×1wenet_wer×52wenet_cer×23token_mer×2token_cer×1wenet_mer×1dnsmos×9utmos×9wv_mos×9ecapa_tdnn_sim×8eres2net_sim×8wavlm_large_sim×8komos×1si_sdr×3rir_c50_mae×1rir_c80_mae×1rir_doa_mae×1rir_drr_mae×1rir_edc_mae×1rir_iacc_early_mae×1rir_iacc_full_mae×1rir_iacc_late_mae×1rir_iacc_mae×1rir_ild_mae×1rir_ipd_mae×1rir_itd_mae×1rir_npm×1rt30_mae×1rt60_mae×1sacrebleu×6bleurt_20×2xcomet_xl×2audiobox_aestheti…×4audioldm_eval×4clap_score×1doa_localization×8pesq×1stoi×1vad_detection_dur…×4vad_auc_roc×1wekws_det×3meeteval×3classify×6cosine_trial_scor…×2det_eer×1min_dcf_p005×1197 pipelinesreport.json
-