From 063f3c0cb400dcf5083e6df3d8112ac349288a17 Mon Sep 17 00:00:00 2001 From: Jing Peng <143288982+PigeonDan1@users.noreply.github.com> Date: Tue, 13 Oct 2026 09:28:08 +0800 Subject: [PATCH 01/26] feat: add RIR evaluation task and spatial metrics --- README.md | 1 + README_ZH.md | 1 + docs/atlas/index.html | 12 +- docs/atlas/pipeline_atlas.svg | 20 +- docs/atlas/pipeline_atlas_dark.svg | 20 +- docs/pipeline_catalog.jsonl | 10 + docs/tasks/README.md | 1 + docs/tasks/rir.md | 55 +++++ scripts/generate_pipeline_catalog.py | 2 + src/sure_eval/evaluation/audio_samples.py | 36 ++- src/sure_eval/evaluation/cli.py | 2 +- src/sure_eval/evaluation/cli_adapters.py | 14 +- .../nodes/scoring/rir_c50_mae/README.md | 3 + .../nodes/scoring/rir_c50_mae/__init__.py | 1 + .../nodes/scoring/rir_c50_mae/manifest.yaml | 12 + .../nodes/scoring/rir_c50_mae/node.py | 3 + .../nodes/scoring/rir_c80_mae/README.md | 3 + .../nodes/scoring/rir_c80_mae/__init__.py | 1 + .../nodes/scoring/rir_c80_mae/manifest.yaml | 12 + .../nodes/scoring/rir_c80_mae/node.py | 3 + .../nodes/scoring/rir_corr/README.md | 3 + .../nodes/scoring/rir_corr/__init__.py | 1 + .../nodes/scoring/rir_corr/manifest.yaml | 12 + .../evaluation/nodes/scoring/rir_corr/node.py | 3 + .../nodes/scoring/rir_doa_mae/README.md | 3 + .../nodes/scoring/rir_doa_mae/__init__.py | 1 + .../nodes/scoring/rir_doa_mae/manifest.yaml | 12 + .../nodes/scoring/rir_doa_mae/node.py | 3 + .../nodes/scoring/rir_drr_mae/README.md | 3 + .../nodes/scoring/rir_drr_mae/__init__.py | 1 + .../nodes/scoring/rir_drr_mae/manifest.yaml | 12 + .../nodes/scoring/rir_drr_mae/node.py | 3 + .../nodes/scoring/rir_ild_mae/README.md | 3 + .../nodes/scoring/rir_ild_mae/__init__.py | 1 + .../nodes/scoring/rir_ild_mae/manifest.yaml | 12 + .../nodes/scoring/rir_ild_mae/node.py | 3 + .../nodes/scoring/rir_ipd_mae/README.md | 3 + .../nodes/scoring/rir_ipd_mae/__init__.py | 1 + .../nodes/scoring/rir_ipd_mae/manifest.yaml | 12 + .../nodes/scoring/rir_ipd_mae/node.py | 3 + .../nodes/scoring/rir_nmse/README.md | 3 + .../nodes/scoring/rir_nmse/__init__.py | 1 + .../nodes/scoring/rir_nmse/manifest.yaml | 12 + .../evaluation/nodes/scoring/rir_nmse/node.py | 3 + .../nodes/scoring/rt60_mae/README.md | 3 + .../nodes/scoring/rt60_mae/__init__.py | 1 + .../nodes/scoring/rt60_mae/manifest.yaml | 12 + .../evaluation/nodes/scoring/rt60_mae/node.py | 3 + src/sure_eval/evaluation/scripts/rir.py | 49 +++++ src/sure_eval/evaluation/scripts/run.py | 1 + src/sure_eval/evaluation/tasks/rir/README.md | 12 + .../evaluation/tasks/rir/__init__.py | 5 + .../evaluation/tasks/rir/manifest.yaml | 100 +++++++++ src/sure_eval/evaluation/tasks/rir/metrics.py | 195 +++++++++++++++++ .../evaluation/tasks/rir/pipeline.py | 70 ++++++ .../evaluation/tasks/rir/routes.yaml | 66 ++++++ src/sure_eval/evaluation/tasks/rir/types.py | 23 ++ tests/test_rir_task.py | 206 ++++++++++++++++++ 58 files changed, 1043 insertions(+), 29 deletions(-) create mode 100644 docs/tasks/rir.md create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_c50_mae/README.md create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_c50_mae/__init__.py create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_c50_mae/manifest.yaml create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_c50_mae/node.py create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_c80_mae/README.md create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_c80_mae/__init__.py create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_c80_mae/manifest.yaml create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_c80_mae/node.py create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_corr/README.md create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_corr/__init__.py create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_corr/manifest.yaml create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_corr/node.py create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_doa_mae/README.md create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_doa_mae/__init__.py create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_doa_mae/manifest.yaml create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_doa_mae/node.py create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_drr_mae/README.md create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_drr_mae/__init__.py create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_drr_mae/manifest.yaml create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_drr_mae/node.py create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_ild_mae/README.md create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_ild_mae/__init__.py create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_ild_mae/manifest.yaml create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_ild_mae/node.py create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_ipd_mae/README.md create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_ipd_mae/__init__.py create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_ipd_mae/manifest.yaml create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_ipd_mae/node.py create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_nmse/README.md create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_nmse/__init__.py create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_nmse/manifest.yaml create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_nmse/node.py create mode 100644 src/sure_eval/evaluation/nodes/scoring/rt60_mae/README.md create mode 100644 src/sure_eval/evaluation/nodes/scoring/rt60_mae/__init__.py create mode 100644 src/sure_eval/evaluation/nodes/scoring/rt60_mae/manifest.yaml create mode 100644 src/sure_eval/evaluation/nodes/scoring/rt60_mae/node.py create mode 100644 src/sure_eval/evaluation/scripts/rir.py create mode 100644 src/sure_eval/evaluation/tasks/rir/README.md create mode 100644 src/sure_eval/evaluation/tasks/rir/__init__.py create mode 100644 src/sure_eval/evaluation/tasks/rir/manifest.yaml create mode 100644 src/sure_eval/evaluation/tasks/rir/metrics.py create mode 100644 src/sure_eval/evaluation/tasks/rir/pipeline.py create mode 100644 src/sure_eval/evaluation/tasks/rir/routes.yaml create mode 100644 src/sure_eval/evaluation/tasks/rir/types.py create mode 100644 tests/test_rir_task.py diff --git a/README.md b/README.md index 9dfdfeda..43e1ba54 100644 --- a/README.md +++ b/README.md @@ -203,6 +203,7 @@ packages and Git. | TTS | CER, WER, speaker similarity, DNSMOS, WV-MOS, UTMOS | [TTS](docs/tasks/tts.md) | | VC | CER, WER, speaker similarity, DNSMOS, WV-MOS, UTMOS | [VC](docs/tasks/vc.md) | | SE | SI-SDR, STOI, PESQ, DNSMOS, WV-MOS, UTMOS | [SE](docs/tasks/se.md) | +| RIR | RIR-NMSE, RIR correlation, RT60 MAE, DRR/C50/C80, ILD/IPD/DOA errors | [RIR](docs/tasks/rir.md) | | TSE | SI-SDR, speaker similarity, DNSMOS, WV-MOS, UTMOS, CER, WER | [TSE](docs/tasks/tse.md) | | Classification / SER / GR | Accuracy | [Classification](docs/tasks/classification.md) | | SLU | Accuracy | [SLU](docs/tasks/slu.md) | diff --git a/README_ZH.md b/README_ZH.md index ea04c527..a75dfa1d 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -192,6 +192,7 @@ node-local 虚拟环境只保留在本地,不会进入安装包或 Git。 | TTS | CER、WER、说话人相似度、DNSMOS、WV-MOS、UTMOS | [TTS](docs/tasks/tts.md) | | VC | CER、WER、说话人相似度、DNSMOS、WV-MOS、UTMOS | [VC](docs/tasks/vc.md) | | SE | SI-SDR、STOI、PESQ、DNSMOS、WV-MOS、UTMOS | [SE](docs/tasks/se.md) | +| RIR | RIR-NMSE、RIR 相关系数、RT60 MAE、DRR/C50/C80、ILD/IPD/DOA 误差 | [RIR](docs/tasks/rir.md) | | TSE | SI-SDR、说话人相似度、DNSMOS、WV-MOS、UTMOS、CER、WER | [TSE](docs/tasks/tse.md) | | Classification / SER / GR | Accuracy | [Classification](docs/tasks/classification.md) | | SLU | Accuracy | [SLU](docs/tasks/slu.md) | diff --git a/docs/atlas/index.html b/docs/atlas/index.html index 487bff0c..b54887e2 100644 --- a/docs/atlas/index.html +++ b/docs/atlas/index.html @@ -102,18 +102,18 @@