From ad129afdc987809a3112c12ba84196469712ca9a Mon Sep 17 00:00:00 2001 From: Jing Peng <143288982+PigeonDan1@users.noreply.github.com> Date: Fri, 30 Oct 2026 12:11:22 +0800 Subject: [PATCH 1/3] feat(rir): add ISO 3382 EDT and JND routes --- README.md | 4 +- README_ZH.md | 4 +- docs/pipeline_catalog.jsonl | 8 + docs/tasks/rir.md | 32 +++ pyproject.toml | 1 + scripts/generate_pipeline_catalog.py | 2 + .../nodes/scoring/rir_c80_jnd_mae/README.md | 7 + .../nodes/scoring/rir_c80_jnd_mae/__init__.py | 3 + .../scoring/rir_c80_jnd_mae/manifest.yaml | 12 ++ .../nodes/scoring/rir_c80_jnd_mae/node.py | 3 + .../nodes/scoring/rir_d50_jnd_mae/README.md | 7 + .../nodes/scoring/rir_d50_jnd_mae/__init__.py | 3 + .../scoring/rir_d50_jnd_mae/manifest.yaml | 12 ++ .../nodes/scoring/rir_d50_jnd_mae/node.py | 3 + .../scoring/rir_edt_broadband_mae/README.md | 10 + .../scoring/rir_edt_broadband_mae/__init__.py | 3 + .../rir_edt_broadband_mae/manifest.yaml | 12 ++ .../scoring/rir_edt_broadband_mae/node.py | 3 + .../nodes/scoring/rir_edt_jnd_mae/README.md | 12 ++ .../nodes/scoring/rir_edt_jnd_mae/__init__.py | 3 + .../scoring/rir_edt_jnd_mae/manifest.yaml | 12 ++ .../nodes/scoring/rir_edt_jnd_mae/node.py | 3 + .../nodes/scoring/rir_edt_m_mae/README.md | 13 ++ .../nodes/scoring/rir_edt_m_mae/__init__.py | 3 + .../nodes/scoring/rir_edt_m_mae/manifest.yaml | 12 ++ .../nodes/scoring/rir_edt_m_mae/node.py | 3 + .../nodes/scoring/rir_ts_jnd_mae/README.md | 7 + .../nodes/scoring/rir_ts_jnd_mae/__init__.py | 3 + .../scoring/rir_ts_jnd_mae/manifest.yaml | 12 ++ .../nodes/scoring/rir_ts_jnd_mae/node.py | 3 + src/sure_eval/evaluation/tasks/rir/README.md | 4 +- .../evaluation/tasks/rir/manifest.yaml | 54 +++++ src/sure_eval/evaluation/tasks/rir/metrics.py | 187 ++++++++++++++++++ .../evaluation/tasks/rir/pipeline.py | 21 ++ .../evaluation/tasks/rir/routes.yaml | 48 +++++ tests/test_rir_task.py | 55 +++++- 36 files changed, 578 insertions(+), 6 deletions(-) create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_c80_jnd_mae/README.md create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_c80_jnd_mae/__init__.py create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_c80_jnd_mae/manifest.yaml create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_c80_jnd_mae/node.py create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_d50_jnd_mae/README.md create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_d50_jnd_mae/__init__.py create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_d50_jnd_mae/manifest.yaml create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_d50_jnd_mae/node.py create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_edt_broadband_mae/README.md create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_edt_broadband_mae/__init__.py create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_edt_broadband_mae/manifest.yaml create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_edt_broadband_mae/node.py create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_edt_jnd_mae/README.md create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_edt_jnd_mae/__init__.py create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_edt_jnd_mae/manifest.yaml create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_edt_jnd_mae/node.py create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_edt_m_mae/README.md create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_edt_m_mae/__init__.py create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_edt_m_mae/manifest.yaml create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_edt_m_mae/node.py create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_ts_jnd_mae/README.md create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_ts_jnd_mae/__init__.py create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_ts_jnd_mae/manifest.yaml create mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_ts_jnd_mae/node.py diff --git a/README.md b/README.md index 726f2662..4099c4cd 100644 --- a/README.md +++ b/README.md @@ -203,8 +203,8 @@ packages and Git. | TTS | CER, WER, speaker similarity, DNSMOS, WV-MOS, UTMOS, KOMOS (Korean) | [TTS](docs/tasks/tts.md) | | TTA | FAD, FD, KL, IS, CLAP score, AudioBox Aesthetics CE/CU/PC/PQ | [TTA](docs/tasks/tta.md) | | VC | CER, WER, speaker similarity, DNSMOS, WV-MOS, UTMOS | [VC](docs/tasks/vc.md) | -| SE | SI-SDR, STOI/eSTOI, PESQ, LSD, nSRMR, LPS, DNSMOS P.835/P.808 components, WV-MOS, UTMOS, NISQA DIM, SQUIM objective predictions | [SE](docs/tasks/se.md) | -| RIR estimation | NPM, RT30/RT60 MAE, DRR/C50/C80/EDC, IACC full/early/late, ITD/ILD/IPD/DOA errors | [RIR estimation](docs/tasks/rir.md) | +| SE | SI-SDR, STOI, PESQ, DNSMOS, WV-MOS, UTMOS | [SE](docs/tasks/se.md) | +| RIR estimation | NPM, RT30/RT60 MAE, DRR/C50/C80/EDC, octave-band and broadband EDT, ISO 3382 EDT/C80/D50/Ts JND, IACC full/early/late, ITD/ILD/IPD/DOA errors | [RIR estimation](docs/tasks/rir.md) | | TSE | SI-SDR, speaker similarity, DNSMOS, WV-MOS, UTMOS, CER, WER | [TSE](docs/tasks/tse.md) | | Classification / SER / GR | Accuracy | [Classification](docs/tasks/classification.md) | | SLU | Accuracy | [SLU](docs/tasks/slu.md) | diff --git a/README_ZH.md b/README_ZH.md index f922405c..8611aab3 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -192,8 +192,8 @@ node-local 虚拟环境只保留在本地,不会进入安装包或 Git。 | TTS | CER、WER、说话人相似度、DNSMOS、WV-MOS、UTMOS、KOMOS(韩语) | [TTS](docs/tasks/tts.md) | | TTA | FAD、FD、KL、IS、CLAP score、AudioBox Aesthetics CE/CU/PC/PQ | [TTA](docs/tasks/tta.md) | | VC | CER、WER、说话人相似度、DNSMOS、WV-MOS、UTMOS | [VC](docs/tasks/vc.md) | -| SE | SI-SDR、STOI/eSTOI、PESQ、LSD、nSRMR、LPS、DNSMOS P.835/P.808 分量、WV-MOS、UTMOS、NISQA DIM、SQUIM 预测指标 | [SE](docs/tasks/se.md) | -| RIR estimation | NPM、RT30/RT60 MAE、DRR/C50/C80/EDC、IACC 全窗/早期/晚期、ITD/ILD/IPD/DOA 误差 | [RIR estimation](docs/tasks/rir.md) | +| SE | SI-SDR、STOI、PESQ、DNSMOS、WV-MOS、UTMOS | [SE](docs/tasks/se.md) | +| RIR estimation | NPM、RT30/RT60 MAE、DRR/C50/C80/EDC、倍频带与宽带 EDT、ISO 3382 EDT/C80/D50/Ts JND、IACC 全窗/早期/晚期、ITD/ILD/IPD/DOA 误差 | [RIR estimation](docs/tasks/rir.md) | | TSE | SI-SDR、说话人相似度、DNSMOS、WV-MOS、UTMOS、CER、WER | [TSE](docs/tasks/tse.md) | | Classification / SER / GR | Accuracy | [Classification](docs/tasks/classification.md) | | SLU | Accuracy | [SLU](docs/tasks/slu.md) | diff --git a/docs/pipeline_catalog.jsonl b/docs/pipeline_catalog.jsonl index fc5c7022..3e1bc10e 100644 --- a/docs/pipeline_catalog.jsonl +++ b/docs/pipeline_catalog.jsonl @@ -202,6 +202,12 @@ {"task": "RIR estimation", "task_alias": "rir", "language": "n/a", "metric": "rir_c50_mae", "pipeline_id": "rir.any.rir_c50_mae.rir_c50_mae_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["rir_c50_mae"], "nodes": ["scoring/rir_c50_mae"], "computation_node_ids": ["scoring/rir_c50_mae"], "task_config_path": "src/sure_eval/evaluation/tasks/rir/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/rir/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.rir.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.rir.run", "executor": "sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples", "required_roles": ["prediction_rir", "reference_rir"], "optional_roles": ["metadata"]} {"task": "RIR estimation", "task_alias": "rir", "language": "n/a", "metric": "rir_c80_mae", "pipeline_id": "rir.any.rir_c80_mae.rir_c80_mae_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["rir_c80_mae"], "nodes": ["scoring/rir_c80_mae"], "computation_node_ids": ["scoring/rir_c80_mae"], "task_config_path": "src/sure_eval/evaluation/tasks/rir/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/rir/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.rir.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.rir.run", "executor": "sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples", "required_roles": ["prediction_rir", "reference_rir"], "optional_roles": ["metadata"]} {"task": "RIR estimation", "task_alias": "rir", "language": "n/a", "metric": "rir_edc_mae", "pipeline_id": "rir.any.rir_edc_mae.rir_edc_mae_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["rir_edc_mae"], "nodes": ["scoring/rir_edc_mae"], "computation_node_ids": ["scoring/rir_edc_mae"], "task_config_path": "src/sure_eval/evaluation/tasks/rir/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/rir/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.rir.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.rir.run", "executor": "sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples", "required_roles": ["prediction_rir", "reference_rir"], "optional_roles": ["metadata"]} +{"task": "RIR estimation", "task_alias": "rir", "language": "n/a", "metric": "rir_edt_m_mae", "pipeline_id": "rir.any.rir_edt_m_mae.rir_edt_m_mae_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["rir_edt_m_mae"], "nodes": ["scoring/rir_edt_m_mae"], "computation_node_ids": ["scoring/rir_edt_m_mae"], "task_config_path": "src/sure_eval/evaluation/tasks/rir/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/rir/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.rir.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.rir.run", "executor": "sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples", "required_roles": ["prediction_rir", "reference_rir"], "optional_roles": ["metadata"]} +{"task": "RIR estimation", "task_alias": "rir", "language": "n/a", "metric": "rir_edt_broadband_mae", "pipeline_id": "rir.any.rir_edt_broadband_mae.rir_edt_broadband_mae_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["rir_edt_broadband_mae"], "nodes": ["scoring/rir_edt_broadband_mae"], "computation_node_ids": ["scoring/rir_edt_broadband_mae"], "task_config_path": "src/sure_eval/evaluation/tasks/rir/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/rir/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.rir.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.rir.run", "executor": "sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples", "required_roles": ["prediction_rir", "reference_rir"], "optional_roles": ["metadata"]} +{"task": "RIR estimation", "task_alias": "rir", "language": "n/a", "metric": "rir_edt_jnd_mae", "pipeline_id": "rir.any.rir_edt_jnd_mae.rir_edt_jnd_mae_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["rir_edt_jnd_mae"], "nodes": ["scoring/rir_edt_jnd_mae"], "computation_node_ids": ["scoring/rir_edt_jnd_mae"], "task_config_path": "src/sure_eval/evaluation/tasks/rir/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/rir/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.rir.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.rir.run", "executor": "sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples", "required_roles": ["prediction_rir", "reference_rir"], "optional_roles": ["metadata"]} +{"task": "RIR estimation", "task_alias": "rir", "language": "n/a", "metric": "rir_c80_jnd_mae", "pipeline_id": "rir.any.rir_c80_jnd_mae.rir_c80_jnd_mae_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["rir_c80_jnd_mae"], "nodes": ["scoring/rir_c80_jnd_mae"], "computation_node_ids": ["scoring/rir_c80_jnd_mae"], "task_config_path": "src/sure_eval/evaluation/tasks/rir/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/rir/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.rir.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.rir.run", "executor": "sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples", "required_roles": ["prediction_rir", "reference_rir"], "optional_roles": ["metadata"]} +{"task": "RIR estimation", "task_alias": "rir", "language": "n/a", "metric": "rir_d50_jnd_mae", "pipeline_id": "rir.any.rir_d50_jnd_mae.rir_d50_jnd_mae_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["rir_d50_jnd_mae"], "nodes": ["scoring/rir_d50_jnd_mae"], "computation_node_ids": ["scoring/rir_d50_jnd_mae"], "task_config_path": "src/sure_eval/evaluation/tasks/rir/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/rir/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.rir.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.rir.run", "executor": "sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples", "required_roles": ["prediction_rir", "reference_rir"], "optional_roles": ["metadata"]} +{"task": "RIR estimation", "task_alias": "rir", "language": "n/a", "metric": "rir_ts_jnd_mae", "pipeline_id": "rir.any.rir_ts_jnd_mae.rir_ts_jnd_mae_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["rir_ts_jnd_mae"], "nodes": ["scoring/rir_ts_jnd_mae"], "computation_node_ids": ["scoring/rir_ts_jnd_mae"], "task_config_path": "src/sure_eval/evaluation/tasks/rir/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/rir/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.rir.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.rir.run", "executor": "sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples", "required_roles": ["prediction_rir", "reference_rir"], "optional_roles": ["metadata"]} {"task": "RIR estimation", "task_alias": "rir", "language": "n/a", "metric": "rir_iacc_mae", "pipeline_id": "rir.any.rir_iacc_mae.rir_iacc_mae_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["rir_iacc_mae"], "nodes": ["scoring/rir_iacc_mae"], "computation_node_ids": ["scoring/rir_iacc_mae"], "task_config_path": "src/sure_eval/evaluation/tasks/rir/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/rir/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.rir.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.rir.run", "executor": "sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples", "required_roles": ["prediction_rir", "reference_rir"], "optional_roles": ["metadata"]} {"task": "RIR estimation", "task_alias": "rir", "language": "n/a", "metric": "rir_iacc_full_mae", "pipeline_id": "rir.any.rir_iacc_full_mae.rir_iacc_full_mae_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["rir_iacc_full_mae"], "nodes": ["scoring/rir_iacc_full_mae"], "computation_node_ids": ["scoring/rir_iacc_full_mae"], "task_config_path": "src/sure_eval/evaluation/tasks/rir/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/rir/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.rir.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.rir.run", "executor": "sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples", "required_roles": ["prediction_rir", "reference_rir"], "optional_roles": ["metadata"]} {"task": "RIR estimation", "task_alias": "rir", "language": "n/a", "metric": "rir_iacc_early_mae", "pipeline_id": "rir.any.rir_iacc_early_mae.rir_iacc_early_mae_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["rir_iacc_early_mae"], "nodes": ["scoring/rir_iacc_early_mae"], "computation_node_ids": ["scoring/rir_iacc_early_mae"], "task_config_path": "src/sure_eval/evaluation/tasks/rir/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/rir/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.rir.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.rir.run", "executor": "sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples", "required_roles": ["prediction_rir", "reference_rir"], "optional_roles": ["metadata"]} @@ -221,3 +227,5 @@ {"task": "RIR estimation", "task_alias": "rir", "language": "n/a", "metric": "multi", "pipeline_id": "rir.any.multi.rir_npm.rir_npm_v1__rt30_mae.rt30_mae_v1__rt60_mae.rt60_mae_v1__rir_drr_mae.rir_drr_mae_v1__rir_c50_mae.rir_c50_mae_v1__rir_c80_mae.rir_c80_mae_v1__rir_edc_mae.rir_edc_mae_v1", "pipeline_kind": "bundle", "member_pipeline_ids": ["rir.any.rir_npm.rir_npm_v1", "rir.any.rt30_mae.rt30_mae_v1", "rir.any.rt60_mae.rt60_mae_v1", "rir.any.rir_drr_mae.rir_drr_mae_v1", "rir.any.rir_c50_mae.rir_c50_mae_v1", "rir.any.rir_c80_mae.rir_c80_mae_v1", "rir.any.rir_edc_mae.rir_edc_mae_v1"], "execution_metrics": ["rir_npm", "rt30_mae", "rt60_mae", "rir_drr_mae", "rir_c50_mae", "rir_c80_mae", "rir_edc_mae"], "nodes": ["scoring/rir_npm", "scoring/rt30_mae", "scoring/rt60_mae", "scoring/rir_drr_mae", "scoring/rir_c50_mae", "scoring/rir_c80_mae", "scoring/rir_edc_mae"], "computation_node_ids": ["scoring/rir_npm", "scoring/rt30_mae", "scoring/rt60_mae", "scoring/rir_drr_mae", "scoring/rir_c50_mae", "scoring/rir_c80_mae", "scoring/rir_edc_mae"], "task_config_path": "src/sure_eval/evaluation/tasks/rir/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/rir/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.rir.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.rir.run", "executor": "sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples", "required_roles": ["prediction_rir", "reference_rir"], "optional_roles": ["metadata"]} {"task": "RIR estimation", "task_alias": "rir", "language": "n/a", "metric": "multi", "pipeline_id": "rir.any.multi.rir_npm.rir_npm_v1__rt30_mae.rt30_mae_v1__rt60_mae.rt60_mae_v1__rir_drr_mae.rir_drr_mae_v1__rir_c50_mae.rir_c50_mae_v1__rir_c80_mae.rir_c80_mae_v1__rir_edc_mae.rir_edc_mae_v1__rir_iacc_full_mae.rir_iacc_full_mae_v1__rir_iacc_early_mae.rir_iacc_early_mae_v1__rir_iacc_late_mae.rir_iacc_late_mae_v1__rir_itd_mae.rir_itd_mae_v1__rir_ild_mae.rir_ild_mae_v1__rir_ipd_mae.rir_ipd_mae_v1__rir_doa_mae.rir_doa_mae_v1", "pipeline_kind": "bundle", "member_pipeline_ids": ["rir.any.rir_npm.rir_npm_v1", "rir.any.rt30_mae.rt30_mae_v1", "rir.any.rt60_mae.rt60_mae_v1", "rir.any.rir_drr_mae.rir_drr_mae_v1", "rir.any.rir_c50_mae.rir_c50_mae_v1", "rir.any.rir_c80_mae.rir_c80_mae_v1", "rir.any.rir_edc_mae.rir_edc_mae_v1", "rir.any.rir_iacc_full_mae.rir_iacc_full_mae_v1", "rir.any.rir_iacc_early_mae.rir_iacc_early_mae_v1", "rir.any.rir_iacc_late_mae.rir_iacc_late_mae_v1", "rir.any.rir_itd_mae.rir_itd_mae_v1", "rir.any.rir_ild_mae.rir_ild_mae_v1", "rir.any.rir_ipd_mae.rir_ipd_mae_v1", "rir.any.rir_doa_mae.rir_doa_mae_v1"], "execution_metrics": ["rir_npm", "rt30_mae", "rt60_mae", "rir_drr_mae", "rir_c50_mae", "rir_c80_mae", "rir_edc_mae", "rir_iacc_full_mae", "rir_iacc_early_mae", "rir_iacc_late_mae", "rir_itd_mae", "rir_ild_mae", "rir_ipd_mae", "rir_doa_mae"], "nodes": ["scoring/rir_npm", "scoring/rt30_mae", "scoring/rt60_mae", "scoring/rir_drr_mae", "scoring/rir_c50_mae", "scoring/rir_c80_mae", "scoring/rir_edc_mae", "scoring/rir_iacc_full_mae", "scoring/rir_iacc_early_mae", "scoring/rir_iacc_late_mae", "scoring/rir_itd_mae", "scoring/rir_ild_mae", "scoring/rir_ipd_mae", "scoring/rir_doa_mae"], "computation_node_ids": ["scoring/rir_npm", "scoring/rt30_mae", "scoring/rt60_mae", "scoring/rir_drr_mae", "scoring/rir_c50_mae", "scoring/rir_c80_mae", "scoring/rir_edc_mae", "scoring/rir_iacc_full_mae", "scoring/rir_iacc_early_mae", "scoring/rir_iacc_late_mae", "scoring/rir_itd_mae", "scoring/rir_ild_mae", "scoring/rir_ipd_mae", "scoring/rir_doa_mae"], "task_config_path": "src/sure_eval/evaluation/tasks/rir/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/rir/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.rir.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.rir.run", "executor": "sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples", "required_roles": ["prediction_rir", "reference_rir"], "optional_roles": ["metadata"]} {"task": "RIR estimation", "task_alias": "rir", "language": "n/a", "metric": "multi", "pipeline_id": "rir.any.multi.rir_npm.rir_npm_v1__rt30_mae.rt30_mae_v1__rt60_mae.rt60_mae_v1__rir_drr_mae.rir_drr_mae_v1__rir_c50_mae.rir_c50_mae_v1__rir_c80_mae.rir_c80_mae_v1__rir_edc_mae.rir_edc_mae_v1__rir_itd_mae.rir_itd_mae_v1__rir_ild_mae.rir_ild_mae_v1__rir_ipd_mae.rir_ipd_mae_v1__rir_doa_mae.rir_doa_mae_v1", "pipeline_kind": "bundle", "member_pipeline_ids": ["rir.any.rir_npm.rir_npm_v1", "rir.any.rt30_mae.rt30_mae_v1", "rir.any.rt60_mae.rt60_mae_v1", "rir.any.rir_drr_mae.rir_drr_mae_v1", "rir.any.rir_c50_mae.rir_c50_mae_v1", "rir.any.rir_c80_mae.rir_c80_mae_v1", "rir.any.rir_edc_mae.rir_edc_mae_v1", "rir.any.rir_itd_mae.rir_itd_mae_v1", "rir.any.rir_ild_mae.rir_ild_mae_v1", "rir.any.rir_ipd_mae.rir_ipd_mae_v1", "rir.any.rir_doa_mae.rir_doa_mae_v1"], "execution_metrics": ["rir_npm", "rt30_mae", "rt60_mae", "rir_drr_mae", "rir_c50_mae", "rir_c80_mae", "rir_edc_mae", "rir_itd_mae", "rir_ild_mae", "rir_ipd_mae", "rir_doa_mae"], "nodes": ["scoring/rir_npm", "scoring/rt30_mae", "scoring/rt60_mae", "scoring/rir_drr_mae", "scoring/rir_c50_mae", "scoring/rir_c80_mae", "scoring/rir_edc_mae", "scoring/rir_itd_mae", "scoring/rir_ild_mae", "scoring/rir_ipd_mae", "scoring/rir_doa_mae"], "computation_node_ids": ["scoring/rir_npm", "scoring/rt30_mae", "scoring/rt60_mae", "scoring/rir_drr_mae", "scoring/rir_c50_mae", "scoring/rir_c80_mae", "scoring/rir_edc_mae", "scoring/rir_itd_mae", "scoring/rir_ild_mae", "scoring/rir_ipd_mae", "scoring/rir_doa_mae"], "task_config_path": "src/sure_eval/evaluation/tasks/rir/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/rir/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.rir.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.rir.run", "executor": "sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples", "required_roles": ["prediction_rir", "reference_rir"], "optional_roles": ["metadata"]} +{"task": "RIR estimation", "task_alias": "rir", "language": "n/a", "metric": "multi", "pipeline_id": "rir.any.multi.rir_edt_m_mae.rir_edt_m_mae_v1__rir_edt_broadband_mae.rir_edt_broadband_mae_v1", "pipeline_kind": "bundle", "member_pipeline_ids": ["rir.any.rir_edt_m_mae.rir_edt_m_mae_v1", "rir.any.rir_edt_broadband_mae.rir_edt_broadband_mae_v1"], "execution_metrics": ["rir_edt_m_mae", "rir_edt_broadband_mae"], "nodes": ["scoring/rir_edt_m_mae", "scoring/rir_edt_broadband_mae"], "computation_node_ids": ["scoring/rir_edt_m_mae", "scoring/rir_edt_broadband_mae"], "task_config_path": "src/sure_eval/evaluation/tasks/rir/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/rir/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.rir.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.rir.run", "executor": "sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples", "required_roles": ["prediction_rir", "reference_rir"], "optional_roles": ["metadata"]} +{"task": "RIR estimation", "task_alias": "rir", "language": "n/a", "metric": "multi", "pipeline_id": "rir.any.multi.rir_edt_jnd_mae.rir_edt_jnd_mae_v1__rir_c80_jnd_mae.rir_c80_jnd_mae_v1__rir_d50_jnd_mae.rir_d50_jnd_mae_v1__rir_ts_jnd_mae.rir_ts_jnd_mae_v1", "pipeline_kind": "bundle", "member_pipeline_ids": ["rir.any.rir_edt_jnd_mae.rir_edt_jnd_mae_v1", "rir.any.rir_c80_jnd_mae.rir_c80_jnd_mae_v1", "rir.any.rir_d50_jnd_mae.rir_d50_jnd_mae_v1", "rir.any.rir_ts_jnd_mae.rir_ts_jnd_mae_v1"], "execution_metrics": ["rir_edt_jnd_mae", "rir_c80_jnd_mae", "rir_d50_jnd_mae", "rir_ts_jnd_mae"], "nodes": ["scoring/rir_edt_jnd_mae", "scoring/rir_c80_jnd_mae", "scoring/rir_d50_jnd_mae", "scoring/rir_ts_jnd_mae"], "computation_node_ids": ["scoring/rir_edt_jnd_mae", "scoring/rir_c80_jnd_mae", "scoring/rir_d50_jnd_mae", "scoring/rir_ts_jnd_mae"], "task_config_path": "src/sure_eval/evaluation/tasks/rir/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/rir/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.rir.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.rir.run", "executor": "sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples", "required_roles": ["prediction_rir", "reference_rir"], "optional_roles": ["metadata"]} diff --git a/docs/tasks/rir.md b/docs/tasks/rir.md index 765548be..3af9e995 100644 --- a/docs/tasks/rir.md +++ b/docs/tasks/rir.md @@ -51,6 +51,12 @@ used when it is omitted. | `rir_c50_mae` | C50 error in dB | lower is better | | `rir_c80_mae` | C80 error in dB | lower is better | | `rir_edc_mae` | energy decay curve MAE above -40 dB in dB | lower is better | +| `rir_edt_m_mae` | 500 Hz/1 kHz octave-band EDT MAE in seconds | lower is better | +| `rir_edt_broadband_mae` | broadband EDT MAE in seconds | lower is better | +| `rir_edt_jnd_mae` | EDT error divided by 5% reference EDT (`x_jnd`) | lower is better | +| `rir_c80_jnd_mae` | C80 error divided by 1 dB (`x_jnd`) | lower is better | +| `rir_d50_jnd_mae` | D50 error divided by 0.05 (`x_jnd`) | lower is better | +| `rir_ts_jnd_mae` | centre-time error divided by 10 ms (`x_jnd`) | lower is better | | `rir_iacc_mae` | compatibility alias for full-window IACC error | lower is better | | `rir_iacc_full_mae` | full-window interaural cross-correlation coefficient error in a +/-1 ms window | lower is better | | `rir_iacc_early_mae` | IACC from the ch0 direct peak through 80 ms | lower is better | @@ -69,6 +75,32 @@ rir.any.multi.rir_npm.rir_npm_v1__rt30_mae.rt30_mae_v1__rt60_mae.rt60_mae_v1__ri The bundle's top-level `score` is `rir_npm`; all metric scores are in `details.results`. This is a reporting convention, not a combined score. +## ISO 3382 EDT and JND routes + +`rir_edt_m_mae` follows the leaderboard protocol: detect the first broadband +sample within 20 dB of the peak, filter at 500 Hz and 1 kHz with a zero-phase +Butterworth octave-band filter, fit the initial 0 to -10 dB Schroeder decay, +extrapolate to 60 dB, average the two band values per channel, and report +seconds of absolute error. `rir_edt_broadband_mae` performs the same decay fit +on the full-band signal with an independent channel onset and does not apply a +JND threshold. + +The four `*_jnd_mae` routes use the informative BS EN ISO 3382-1:2009 Annex A +Table A.1 thresholds: EDT 5% of the reference EDT, C80 1 dB, D50 0.05, and Ts +10 ms. Each route reports a dimensionless mean error in `x_jnd`, plus +`absolute_mae`, `mean_jnd`, `within_one_jnd_percent`, `valid_samples`, and +`total_samples` in its result details. All four routes support one or more +channels. Channel errors are averaged; a sample counts as within one JND only +when every channel passes. This channel aggregation and pass-rate definition +are benchmark rules, not ISO-prescribed auditorium reporting. + +The standard's frequency averaging is retained for all four JND parameters: +500 Hz and 1 kHz are averaged per channel. The implementation uses a sixth +order Butterworth prototype and zero-phase filtering; IEC 61260 certification, +measured background levels, source calibration, and auditorium measurement +conditions are not claimed. Existing `rir_edc_mae` remains an energy-curve +error in dB and must not be interpreted as EDT. + For cross-channel base evaluation, the curated bundle is: ```text diff --git a/pyproject.toml b/pyproject.toml index 19a2859d..64f35af1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,7 @@ diarization = [ audio = [ "soundfile>=0.12.0", "librosa>=0.10.0", + "scipy>=1.6.0", ] se = [ "pystoi>=0.4.1", diff --git a/scripts/generate_pipeline_catalog.py b/scripts/generate_pipeline_catalog.py index be1cc36a..c33a7155 100644 --- a/scripts/generate_pipeline_catalog.py +++ b/scripts/generate_pipeline_catalog.py @@ -114,6 +114,8 @@ ) }, ), + ("rir", {"metrics": ("rir_edt_m_mae", "rir_edt_broadband_mae")}), + ("rir", {"metrics": ("rir_edt_jnd_mae", "rir_c80_jnd_mae", "rir_d50_jnd_mae", "rir_ts_jnd_mae")}), ) diff --git a/src/sure_eval/evaluation/nodes/scoring/rir_c80_jnd_mae/README.md b/src/sure_eval/evaluation/nodes/scoring/rir_c80_jnd_mae/README.md new file mode 100644 index 00000000..06029dca --- /dev/null +++ b/src/sure_eval/evaluation/nodes/scoring/rir_c80_jnd_mae/README.md @@ -0,0 +1,7 @@ +# ISO 3382 C80 JND Error + +Computes C80 in the 500 Hz and 1 kHz octave bands, averages per channel, and +normalizes the absolute error by the Table A.1 threshold of 1 dB. The score is +the mean channel error in `x_jnd`; the report retains absolute C80 MAE and +the within-one-JND sample percentage. Mono, binaural, and larger multichannel +inputs are supported. diff --git a/src/sure_eval/evaluation/nodes/scoring/rir_c80_jnd_mae/__init__.py b/src/sure_eval/evaluation/nodes/scoring/rir_c80_jnd_mae/__init__.py new file mode 100644 index 00000000..883fb6d1 --- /dev/null +++ b/src/sure_eval/evaluation/nodes/scoring/rir_c80_jnd_mae/__init__.py @@ -0,0 +1,3 @@ +from sure_eval.evaluation.tasks.rir.metrics import rir_c80_jnd_mae + +__all__ = ["rir_c80_jnd_mae"] diff --git a/src/sure_eval/evaluation/nodes/scoring/rir_c80_jnd_mae/manifest.yaml b/src/sure_eval/evaluation/nodes/scoring/rir_c80_jnd_mae/manifest.yaml new file mode 100644 index 00000000..5aaa4b8e --- /dev/null +++ b/src/sure_eval/evaluation/nodes/scoring/rir_c80_jnd_mae/manifest.yaml @@ -0,0 +1,12 @@ +id: scoring/rir_c80_jnd_mae +version: v1 +stage: scoring +language_sensitive: false +input_schema: rir_samples_jsonl +output_schema: rir_c80_jnd_mae_score +implementation: sure_eval.evaluation.tasks.rir.metrics.rir_c80_jnd_mae +metric: rir_c80_jnd_mae +backend: scipy-rir +internal_stages: [audio_load, direct_onset, octave_band_filter, c80_fit, jnd_normalization, mean_aggregation] +used_by: [RIR ISO 3382 JND] +notes: [Lower is better. Unit is x_jnd. Supports one or more channels. C80 JND is 1 dB.] diff --git a/src/sure_eval/evaluation/nodes/scoring/rir_c80_jnd_mae/node.py b/src/sure_eval/evaluation/nodes/scoring/rir_c80_jnd_mae/node.py new file mode 100644 index 00000000..883fb6d1 --- /dev/null +++ b/src/sure_eval/evaluation/nodes/scoring/rir_c80_jnd_mae/node.py @@ -0,0 +1,3 @@ +from sure_eval.evaluation.tasks.rir.metrics import rir_c80_jnd_mae + +__all__ = ["rir_c80_jnd_mae"] diff --git a/src/sure_eval/evaluation/nodes/scoring/rir_d50_jnd_mae/README.md b/src/sure_eval/evaluation/nodes/scoring/rir_d50_jnd_mae/README.md new file mode 100644 index 00000000..7342b740 --- /dev/null +++ b/src/sure_eval/evaluation/nodes/scoring/rir_d50_jnd_mae/README.md @@ -0,0 +1,7 @@ +# ISO 3382 D50 JND Error + +Computes D50 in the 500 Hz and 1 kHz octave bands, averages per channel, and +normalizes the absolute error by the Table A.1 threshold of 0.05. The score is +the mean channel error in `x_jnd`; the report retains absolute D50 MAE and +the within-one-JND sample percentage. Mono, binaural, and larger multichannel +inputs are supported. diff --git a/src/sure_eval/evaluation/nodes/scoring/rir_d50_jnd_mae/__init__.py b/src/sure_eval/evaluation/nodes/scoring/rir_d50_jnd_mae/__init__.py new file mode 100644 index 00000000..31e59d54 --- /dev/null +++ b/src/sure_eval/evaluation/nodes/scoring/rir_d50_jnd_mae/__init__.py @@ -0,0 +1,3 @@ +from sure_eval.evaluation.tasks.rir.metrics import rir_d50_jnd_mae + +__all__ = ["rir_d50_jnd_mae"] diff --git a/src/sure_eval/evaluation/nodes/scoring/rir_d50_jnd_mae/manifest.yaml b/src/sure_eval/evaluation/nodes/scoring/rir_d50_jnd_mae/manifest.yaml new file mode 100644 index 00000000..2525924e --- /dev/null +++ b/src/sure_eval/evaluation/nodes/scoring/rir_d50_jnd_mae/manifest.yaml @@ -0,0 +1,12 @@ +id: scoring/rir_d50_jnd_mae +version: v1 +stage: scoring +language_sensitive: false +input_schema: rir_samples_jsonl +output_schema: rir_d50_jnd_mae_score +implementation: sure_eval.evaluation.tasks.rir.metrics.rir_d50_jnd_mae +metric: rir_d50_jnd_mae +backend: scipy-rir +internal_stages: [audio_load, direct_onset, octave_band_filter, d50_fit, jnd_normalization, mean_aggregation] +used_by: [RIR ISO 3382 JND] +notes: [Lower is better. Unit is x_jnd. Supports one or more channels. D50 JND is 0.05.] diff --git a/src/sure_eval/evaluation/nodes/scoring/rir_d50_jnd_mae/node.py b/src/sure_eval/evaluation/nodes/scoring/rir_d50_jnd_mae/node.py new file mode 100644 index 00000000..31e59d54 --- /dev/null +++ b/src/sure_eval/evaluation/nodes/scoring/rir_d50_jnd_mae/node.py @@ -0,0 +1,3 @@ +from sure_eval.evaluation.tasks.rir.metrics import rir_d50_jnd_mae + +__all__ = ["rir_d50_jnd_mae"] diff --git a/src/sure_eval/evaluation/nodes/scoring/rir_edt_broadband_mae/README.md b/src/sure_eval/evaluation/nodes/scoring/rir_edt_broadband_mae/README.md new file mode 100644 index 00000000..a8d1294d --- /dev/null +++ b/src/sure_eval/evaluation/nodes/scoring/rir_edt_broadband_mae/README.md @@ -0,0 +1,10 @@ +# Broadband EDT MAE + +Reports the absolute prediction/reference error of broadband EDT in seconds. +Each RIR uses its broadband onset, then each channel fits the full-band 0 to +-10 dB Schroeder decay and extrapolates that slope to 60 dB. Channel errors are +averaged per sample and samples are equally weighted. + +This is distinct from `rir_edt_m_mae`: the latter averages 500 Hz and 1 kHz +octave-band EDT values before computing error. Broadband EDT has no ISO Table +A.1 frequency-band JND applied by this route. diff --git a/src/sure_eval/evaluation/nodes/scoring/rir_edt_broadband_mae/__init__.py b/src/sure_eval/evaluation/nodes/scoring/rir_edt_broadband_mae/__init__.py new file mode 100644 index 00000000..26723355 --- /dev/null +++ b/src/sure_eval/evaluation/nodes/scoring/rir_edt_broadband_mae/__init__.py @@ -0,0 +1,3 @@ +from sure_eval.evaluation.tasks.rir.metrics import rir_edt_broadband_mae + +__all__ = ["rir_edt_broadband_mae"] diff --git a/src/sure_eval/evaluation/nodes/scoring/rir_edt_broadband_mae/manifest.yaml b/src/sure_eval/evaluation/nodes/scoring/rir_edt_broadband_mae/manifest.yaml new file mode 100644 index 00000000..dfd0e724 --- /dev/null +++ b/src/sure_eval/evaluation/nodes/scoring/rir_edt_broadband_mae/manifest.yaml @@ -0,0 +1,12 @@ +id: scoring/rir_edt_broadband_mae +version: v1 +stage: scoring +language_sensitive: false +input_schema: rir_samples_jsonl +output_schema: rir_edt_broadband_mae_score +implementation: sure_eval.evaluation.tasks.rir.metrics.rir_edt_broadband_mae +metric: rir_edt_broadband_mae +backend: numpy-rir +internal_stages: [audio_load, independent_direct_onset, schroeder_decay, edt_fit, mean_aggregation] +used_by: [RIR room acoustics] +notes: [Lower is better. Unit is seconds. Full-band 0 to -10 dB decay fit extrapolated to 60 dB.] diff --git a/src/sure_eval/evaluation/nodes/scoring/rir_edt_broadband_mae/node.py b/src/sure_eval/evaluation/nodes/scoring/rir_edt_broadband_mae/node.py new file mode 100644 index 00000000..26723355 --- /dev/null +++ b/src/sure_eval/evaluation/nodes/scoring/rir_edt_broadband_mae/node.py @@ -0,0 +1,3 @@ +from sure_eval.evaluation.tasks.rir.metrics import rir_edt_broadband_mae + +__all__ = ["rir_edt_broadband_mae"] diff --git a/src/sure_eval/evaluation/nodes/scoring/rir_edt_jnd_mae/README.md b/src/sure_eval/evaluation/nodes/scoring/rir_edt_jnd_mae/README.md new file mode 100644 index 00000000..b765847b --- /dev/null +++ b/src/sure_eval/evaluation/nodes/scoring/rir_edt_jnd_mae/README.md @@ -0,0 +1,12 @@ +# ISO 3382 EDT JND Error + +Computes 500 Hz and 1 kHz EDT, averages those values per channel, and divides +the absolute error by 5% of the reference EDT. The score is the arithmetic mean +of channel JND multiples and is dimensionless (`x_jnd`). Reports also include +absolute MAE in seconds and the percentage of samples for which all channels +are within one JND. Mono, binaural, and larger multichannel inputs are +supported. + +This is the informative ISO 3382-1 Annex A Table A.1 threshold applied as a +benchmark comparison. The standard does not prescribe channel averaging or a +model pass rate. diff --git a/src/sure_eval/evaluation/nodes/scoring/rir_edt_jnd_mae/__init__.py b/src/sure_eval/evaluation/nodes/scoring/rir_edt_jnd_mae/__init__.py new file mode 100644 index 00000000..c89d11b7 --- /dev/null +++ b/src/sure_eval/evaluation/nodes/scoring/rir_edt_jnd_mae/__init__.py @@ -0,0 +1,3 @@ +from sure_eval.evaluation.tasks.rir.metrics import rir_edt_jnd_mae + +__all__ = ["rir_edt_jnd_mae"] diff --git a/src/sure_eval/evaluation/nodes/scoring/rir_edt_jnd_mae/manifest.yaml b/src/sure_eval/evaluation/nodes/scoring/rir_edt_jnd_mae/manifest.yaml new file mode 100644 index 00000000..69d40934 --- /dev/null +++ b/src/sure_eval/evaluation/nodes/scoring/rir_edt_jnd_mae/manifest.yaml @@ -0,0 +1,12 @@ +id: scoring/rir_edt_jnd_mae +version: v1 +stage: scoring +language_sensitive: false +input_schema: rir_samples_jsonl +output_schema: rir_edt_jnd_mae_score +implementation: sure_eval.evaluation.tasks.rir.metrics.rir_edt_jnd_mae +metric: rir_edt_jnd_mae +backend: scipy-rir +internal_stages: [audio_load, direct_onset, octave_band_filter, edt_fit, jnd_normalization, mean_aggregation] +used_by: [RIR ISO 3382 JND] +notes: [Lower is better. Unit is x_jnd. Supports one or more channels. JND is 5 percent of reference EDT.] diff --git a/src/sure_eval/evaluation/nodes/scoring/rir_edt_jnd_mae/node.py b/src/sure_eval/evaluation/nodes/scoring/rir_edt_jnd_mae/node.py new file mode 100644 index 00000000..c89d11b7 --- /dev/null +++ b/src/sure_eval/evaluation/nodes/scoring/rir_edt_jnd_mae/node.py @@ -0,0 +1,3 @@ +from sure_eval.evaluation.tasks.rir.metrics import rir_edt_jnd_mae + +__all__ = ["rir_edt_jnd_mae"] diff --git a/src/sure_eval/evaluation/nodes/scoring/rir_edt_m_mae/README.md b/src/sure_eval/evaluation/nodes/scoring/rir_edt_m_mae/README.md new file mode 100644 index 00000000..51865536 --- /dev/null +++ b/src/sure_eval/evaluation/nodes/scoring/rir_edt_m_mae/README.md @@ -0,0 +1,13 @@ +# RIR Octave-Band EDT MAE + +Computes EDT independently in the 500 Hz and 1 kHz octave bands, averages the +two band values for each channel, and reports the absolute prediction/reference +error in seconds. Channel and sample aggregation are arithmetic means. + +The onset is the first broadband sample within 20 dB of the RIR peak. Filtering +uses a sixth-order Butterworth prototype with zero-phase filtering; the node +does not claim IEC 61260 certification. EDT fits the initial 0 to -10 dB +Schroeder decay and extrapolates the slope to 60 dB. + +The metric runs in process and requires `sure-evaluation[audio]` for SoundFile +and SciPy. No model or checkpoint is used. diff --git a/src/sure_eval/evaluation/nodes/scoring/rir_edt_m_mae/__init__.py b/src/sure_eval/evaluation/nodes/scoring/rir_edt_m_mae/__init__.py new file mode 100644 index 00000000..4cfc571f --- /dev/null +++ b/src/sure_eval/evaluation/nodes/scoring/rir_edt_m_mae/__init__.py @@ -0,0 +1,3 @@ +from sure_eval.evaluation.tasks.rir.metrics import rir_edt_m_mae + +__all__ = ["rir_edt_m_mae"] diff --git a/src/sure_eval/evaluation/nodes/scoring/rir_edt_m_mae/manifest.yaml b/src/sure_eval/evaluation/nodes/scoring/rir_edt_m_mae/manifest.yaml new file mode 100644 index 00000000..9d0873f7 --- /dev/null +++ b/src/sure_eval/evaluation/nodes/scoring/rir_edt_m_mae/manifest.yaml @@ -0,0 +1,12 @@ +id: scoring/rir_edt_m_mae +version: v1 +stage: scoring +language_sensitive: false +input_schema: rir_samples_jsonl +output_schema: rir_edt_m_mae_score +implementation: sure_eval.evaluation.tasks.rir.metrics.rir_edt_m_mae +metric: rir_edt_m_mae +backend: scipy-rir +internal_stages: [audio_load, direct_onset, octave_band_filter, edt_fit, mean_aggregation] +used_by: [RIR room acoustics] +notes: [Lower is better. Unit is seconds. Arithmetic mean of 500 Hz and 1 kHz octave-band EDT per channel.] diff --git a/src/sure_eval/evaluation/nodes/scoring/rir_edt_m_mae/node.py b/src/sure_eval/evaluation/nodes/scoring/rir_edt_m_mae/node.py new file mode 100644 index 00000000..4cfc571f --- /dev/null +++ b/src/sure_eval/evaluation/nodes/scoring/rir_edt_m_mae/node.py @@ -0,0 +1,3 @@ +from sure_eval.evaluation.tasks.rir.metrics import rir_edt_m_mae + +__all__ = ["rir_edt_m_mae"] diff --git a/src/sure_eval/evaluation/nodes/scoring/rir_ts_jnd_mae/README.md b/src/sure_eval/evaluation/nodes/scoring/rir_ts_jnd_mae/README.md new file mode 100644 index 00000000..20f0475b --- /dev/null +++ b/src/sure_eval/evaluation/nodes/scoring/rir_ts_jnd_mae/README.md @@ -0,0 +1,7 @@ +# ISO 3382 Centre-Time JND Error + +Computes energy-weighted centre time (`Ts`) in the 500 Hz and 1 kHz octave +bands, averages per channel, and normalizes the absolute error by the Table A.1 +threshold of 10 ms. The score is the mean channel error in `x_jnd`; the report +retains absolute Ts MAE and the within-one-JND sample percentage. Mono, +binaural, and larger multichannel inputs are supported. diff --git a/src/sure_eval/evaluation/nodes/scoring/rir_ts_jnd_mae/__init__.py b/src/sure_eval/evaluation/nodes/scoring/rir_ts_jnd_mae/__init__.py new file mode 100644 index 00000000..ad364124 --- /dev/null +++ b/src/sure_eval/evaluation/nodes/scoring/rir_ts_jnd_mae/__init__.py @@ -0,0 +1,3 @@ +from sure_eval.evaluation.tasks.rir.metrics import rir_ts_jnd_mae + +__all__ = ["rir_ts_jnd_mae"] diff --git a/src/sure_eval/evaluation/nodes/scoring/rir_ts_jnd_mae/manifest.yaml b/src/sure_eval/evaluation/nodes/scoring/rir_ts_jnd_mae/manifest.yaml new file mode 100644 index 00000000..94bdedbe --- /dev/null +++ b/src/sure_eval/evaluation/nodes/scoring/rir_ts_jnd_mae/manifest.yaml @@ -0,0 +1,12 @@ +id: scoring/rir_ts_jnd_mae +version: v1 +stage: scoring +language_sensitive: false +input_schema: rir_samples_jsonl +output_schema: rir_ts_jnd_mae_score +implementation: sure_eval.evaluation.tasks.rir.metrics.rir_ts_jnd_mae +metric: rir_ts_jnd_mae +backend: scipy-rir +internal_stages: [audio_load, direct_onset, octave_band_filter, centre_time_fit, jnd_normalization, mean_aggregation] +used_by: [RIR ISO 3382 JND] +notes: [Lower is better. Unit is x_jnd. Supports one or more channels. Ts JND is 10 ms.] diff --git a/src/sure_eval/evaluation/nodes/scoring/rir_ts_jnd_mae/node.py b/src/sure_eval/evaluation/nodes/scoring/rir_ts_jnd_mae/node.py new file mode 100644 index 00000000..ad364124 --- /dev/null +++ b/src/sure_eval/evaluation/nodes/scoring/rir_ts_jnd_mae/node.py @@ -0,0 +1,3 @@ +from sure_eval.evaluation.tasks.rir.metrics import rir_ts_jnd_mae + +__all__ = ["rir_ts_jnd_mae"] diff --git a/src/sure_eval/evaluation/tasks/rir/README.md b/src/sure_eval/evaluation/tasks/rir/README.md index 7ae34313..ae5fcc37 100644 --- a/src/sure_eval/evaluation/tasks/rir/README.md +++ b/src/sure_eval/evaluation/tasks/rir/README.md @@ -14,7 +14,9 @@ requires exactly two channels. Its bundle reports full, early, and late windows: full RIR, ch0 direct peak through 80 ms, and the remainder after 80 ms. Each window uses normalized cross-correlation in a +/-1 ms lag window. ITD, ILD, IPD, and DOA use channel 0 as the reference and report every `chN` -pair. ILD and IPD are frequency-domain interaural errors. +pair. ILD and IPD are frequency-domain interaural errors. ISO 3382 EDT, C80, +D50, and Ts routes support one or more channels and average channel errors; +within-one-JND requires every channel in the sample to pass. The default bundle reports `rir_npm` as its top-level score and keeps the individual metric scores in `details.results`. diff --git a/src/sure_eval/evaluation/tasks/rir/manifest.yaml b/src/sure_eval/evaluation/tasks/rir/manifest.yaml index 75b932ce..fd3b0249 100644 --- a/src/sure_eval/evaluation/tasks/rir/manifest.yaml +++ b/src/sure_eval/evaluation/tasks/rir/manifest.yaml @@ -9,6 +9,12 @@ implemented_metrics: - rir_c50_mae - rir_c80_mae - rir_edc_mae + - rir_edt_m_mae + - rir_edt_broadband_mae + - rir_edt_jnd_mae + - rir_c80_jnd_mae + - rir_d50_jnd_mae + - rir_ts_jnd_mae - rir_iacc_mae - rir_iacc_full_mae - rir_iacc_early_mae @@ -92,6 +98,54 @@ input_contracts: alignment_key: sample_id aggregation: mean main_report: true + scoring/rir_edt_m_mae: + purpose: rir_iso3382_octave_band_edt_absolute_error + required_roles: [prediction_rir, reference_rir] + optional_roles: [metadata] + row_format: rir_samples_jsonl + alignment_key: sample_id + aggregation: mean + main_report: true + scoring/rir_edt_broadband_mae: + purpose: rir_broadband_edt_absolute_error + required_roles: [prediction_rir, reference_rir] + optional_roles: [metadata] + row_format: rir_samples_jsonl + alignment_key: sample_id + aggregation: mean + main_report: true + scoring/rir_edt_jnd_mae: + purpose: rir_iso3382_edt_jnd_multiple + required_roles: [prediction_rir, reference_rir] + optional_roles: [metadata] + row_format: rir_samples_jsonl + alignment_key: sample_id + aggregation: mean_over_channel_jnd_errors + main_report: true + scoring/rir_c80_jnd_mae: + purpose: rir_iso3382_c80_jnd_multiple + required_roles: [prediction_rir, reference_rir] + optional_roles: [metadata] + row_format: rir_samples_jsonl + alignment_key: sample_id + aggregation: mean_over_channel_jnd_errors + main_report: true + scoring/rir_d50_jnd_mae: + purpose: rir_iso3382_d50_jnd_multiple + required_roles: [prediction_rir, reference_rir] + optional_roles: [metadata] + row_format: rir_samples_jsonl + alignment_key: sample_id + aggregation: mean_over_channel_jnd_errors + main_report: true + scoring/rir_ts_jnd_mae: + purpose: rir_iso3382_centre_time_jnd_multiple + required_roles: [prediction_rir, reference_rir] + optional_roles: [metadata] + row_format: rir_samples_jsonl + alignment_key: sample_id + aggregation: mean_over_channel_jnd_errors + main_report: true scoring/rir_ild_mae: purpose: multichannel_interchannel_level_difference_error required_roles: [prediction_rir, reference_rir] diff --git a/src/sure_eval/evaluation/tasks/rir/metrics.py b/src/sure_eval/evaluation/tasks/rir/metrics.py index 3e660e1f..ebe1f31c 100644 --- a/src/sure_eval/evaluation/tasks/rir/metrics.py +++ b/src/sure_eval/evaluation/tasks/rir/metrics.py @@ -10,6 +10,13 @@ from sure_eval.evaluation.tasks.rir.types import RIRSample _EPSILON = 1e-12 +_ISO3382_CENTRES_HZ = (500.0, 1000.0) +_ISO3382_JND_THRESHOLDS = { + "edt": "5% relative to reference EDT", + "c80": "1 dB", + "d50": "0.05", + "ts": "10 ms", +} def _load(path: str) -> tuple[np.ndarray, int]: @@ -434,6 +441,180 @@ def rir_edc_mae(samples: list[RIRSample]) -> MetricResult: ) +def _iso3382_filters(sample_rate: int) -> tuple[np.ndarray, ...]: + try: + from scipy.signal import butter + except ImportError as exc: + raise RuntimeError("ISO 3382 RIR metrics require the audio extra: pip install 'sure-evaluation[audio]'") from exc + import math + + return tuple( + butter( + 6, + (centre / math.sqrt(2.0), centre * math.sqrt(2.0)), + btype="bandpass", + fs=sample_rate, + output="sos", + ) + for centre in _ISO3382_CENTRES_HZ + ) + + +def _iso3382_direct_onset(signal: np.ndarray) -> int: + envelope = np.max(np.abs(signal), axis=1) + peak = float(np.max(envelope)) if envelope.size else 0.0 + if peak <= _EPSILON: + raise ValueError("silent RIR") + return int(np.flatnonzero(envelope >= peak * 0.1)[0]) + + +def _iso3382_band_parameters(filtered: np.ndarray, sample_rate: int) -> dict[str, float]: + energy = np.square(np.asarray(filtered, dtype="float64").reshape(-1)) + total = float(np.sum(energy)) + if total <= _EPSILON: + raise ValueError("insufficient band energy") + p50 = min(len(energy), int(round(0.050 * sample_rate))) + p80 = min(len(energy), int(round(0.080 * sample_rate))) + early50 = float(np.sum(energy[:p50])) + early80 = float(np.sum(energy[:p80])) + late80 = total - early80 + c80 = float(10.0 * np.log10(early80 / late80)) if p80 < len(energy) and early80 > _EPSILON and late80 > _EPSILON else float("nan") + curve = np.cumsum(energy[::-1])[::-1] + edc_db = 10.0 * np.log10(np.maximum(curve / total, _EPSILON)) + below = np.flatnonzero(edc_db <= -10.0) + edt = float("nan") + if below.size and int(below[0]) >= 4: + end = int(below[0]) + slope, _ = np.polyfit(np.arange(end + 1, dtype="float64") / sample_rate, edc_db[: end + 1], 1) + if slope < 0.0: + edt = float(-60.0 / slope) + return { + "edt": edt, + "c80": c80, + "d50": float(early50 / total), + "ts": float(1000.0 * np.dot(np.arange(len(energy), dtype="float64") / sample_rate, energy) / total), + } + + +def _iso3382_parameters(signal: np.ndarray, sample_rate: int) -> list[dict[str, float]]: + signal = np.asarray(signal, dtype="float64") + onset = _iso3382_direct_onset(signal) + try: + from scipy.signal import sosfiltfilt + except ImportError as exc: + raise RuntimeError("ISO 3382 RIR metrics require the audio extra: pip install 'sure-evaluation[audio]'") from exc + bands = [sosfiltfilt(sos, signal, axis=0)[onset:] for sos in _iso3382_filters(sample_rate)] + channels = [] + for channel in range(signal.shape[1]): + by_band = [_iso3382_band_parameters(band[:, channel], sample_rate) for band in bands] + channels.append({metric: float(np.mean([item[metric] for item in by_band])) for metric in _ISO3382_JND_THRESHOLDS}) + return channels + + +def _broadband_edt(signal: np.ndarray, sample_rate: int) -> list[float]: + values = [] + for channel in range(signal.shape[1]): + x = signal[:, channel] + energy = np.cumsum((x**2)[::-1])[::-1] + if not energy.size or float(energy[0]) <= _EPSILON: + values.append(float("nan")) + continue + edc_db = 10.0 * np.log10(np.maximum(energy / energy[0], _EPSILON)) + below = np.flatnonzero(edc_db <= -10.0) + if not below.size or int(below[0]) < 4: + values.append(float("nan")) + continue + end = int(below[0]) + slope, _ = np.polyfit(np.arange(end + 1, dtype="float64") / sample_rate, edc_db[: end + 1], 1) + values.append(float(-60.0 / slope) if slope < 0.0 else float("nan")) + return values + + +def _iso3382_edt_mae(samples: list[RIRSample], *, metric_name: str, broadband: bool = False) -> MetricResult: + values = [] + rows = [] + for sample in samples: + prediction, reference, rate = _pair(sample) + if broadband: + prediction_onset = _iso3382_direct_onset(prediction) + reference_onset = _iso3382_direct_onset(reference) + prediction_values = _broadband_edt(prediction[prediction_onset:], rate) + reference_values = _broadband_edt(reference[reference_onset:], rate) + else: + prediction_values = [row["edt"] for row in _iso3382_parameters(prediction, rate)] + reference_values = [row["edt"] for row in _iso3382_parameters(reference, rate)] + errors = [abs(predicted - target) for predicted, target in zip(prediction_values, reference_values) if np.isfinite(predicted) and np.isfinite(target)] + if len(errors) != prediction.shape[1]: + raise ValueError(f"EDT could not be estimated for {sample.sample_id}") + channel_rows = [ + {"channel": channel, "prediction_edt": float(predicted), "reference_edt": float(target), "score": float(error), "unit": "seconds"} + for channel, (predicted, target, error) in enumerate(zip(prediction_values, reference_values, errors)) + ] + score = float(np.mean(errors)) + values.append(score) + rows.append({"sample_id": sample.sample_id, "score": score, "channels": channel_rows, "frequency_bands_hz": [] if broadband else list(_ISO3382_CENTRES_HZ), "unit": "seconds"}) + return MetricResult(metric_name, _mean(values), {"per_sample": rows, "higher_is_better": False, "unit": "seconds", "frequency_bands_hz": [] if broadband else list(_ISO3382_CENTRES_HZ), "frequency_averaging": "arithmetic_mean_per_channel" if not broadband else "broadband"}) + + +def rir_edt_m_mae(samples: list[RIRSample]) -> MetricResult: + return _iso3382_edt_mae(samples, metric_name="rir_edt_m_mae") + + +def rir_edt_broadband_mae(samples: list[RIRSample]) -> MetricResult: + return _iso3382_edt_mae(samples, metric_name="rir_edt_broadband_mae", broadband=True) + + +def _iso3382_jnd_mae(samples: list[RIRSample], *, metric_name: str, parameter: str) -> MetricResult: + values = [] + absolute_values = [] + pass_values = [] + rows = [] + threshold_description = _ISO3382_JND_THRESHOLDS[parameter] + for sample in samples: + prediction, reference, rate = _pair(sample) + prediction_values = _iso3382_parameters(prediction, rate) + reference_values = _iso3382_parameters(reference, rate) + channel_rows = [] + errors = [] + jnd_errors = [] + passes = [] + for channel in range(prediction.shape[1]): + predicted = prediction_values[channel][parameter] + target = reference_values[channel][parameter] + threshold = 0.05 * target if parameter == "edt" else {"c80": 1.0, "d50": 0.05, "ts": 10.0}[parameter] + if not all(np.isfinite(value) for value in (predicted, target, threshold)) or threshold <= _EPSILON: + raise ValueError(f"{parameter.upper()} JND could not be estimated for {sample.sample_id}, channel {channel}") + error = abs(predicted - target) + jnd_error = error / threshold + errors.append(error) + jnd_errors.append(jnd_error) + passes.append(error <= threshold) + channel_rows.append({"channel": channel, "prediction": float(predicted), "reference": float(target), "absolute_error": float(error), "jnd_error": float(jnd_error), "within_one_jnd": bool(error <= threshold), "threshold": float(threshold), "unit": "seconds" if parameter == "edt" else "dB" if parameter == "c80" else "ratio" if parameter == "d50" else "ms"}) + sample_jnd = float(np.mean(jnd_errors)) + sample_absolute = float(np.mean(errors)) + values.append(sample_jnd) + absolute_values.append(sample_absolute) + pass_values.append(bool(all(passes))) + rows.append({"sample_id": sample.sample_id, "score": sample_jnd, "absolute_error": sample_absolute, "jnd_error": sample_jnd, "within_one_jnd": bool(all(passes)), "channels": channel_rows}) + return MetricResult(metric_name, _mean(values), {"per_sample": rows, "higher_is_better": False, "unit": "x_jnd", "jnd_parameter": parameter, "jnd_threshold": threshold_description, "absolute_mae": float(np.mean(absolute_values)), "mean_jnd": _mean(values), "within_one_jnd_percent": 100.0 * float(np.mean(pass_values)), "valid_samples": len(values), "total_samples": len(samples), "channel_aggregation": "arithmetic_mean; all channels required for within-one-JND pass", "frequency_bands_hz": list(_ISO3382_CENTRES_HZ), "frequency_averaging": "arithmetic_mean_per_channel"}) + + +def rir_edt_jnd_mae(samples: list[RIRSample]) -> MetricResult: + return _iso3382_jnd_mae(samples, metric_name="rir_edt_jnd_mae", parameter="edt") + + +def rir_c80_jnd_mae(samples: list[RIRSample]) -> MetricResult: + return _iso3382_jnd_mae(samples, metric_name="rir_c80_jnd_mae", parameter="c80") + + +def rir_d50_jnd_mae(samples: list[RIRSample]) -> MetricResult: + return _iso3382_jnd_mae(samples, metric_name="rir_d50_jnd_mae", parameter="d50") + + +def rir_ts_jnd_mae(samples: list[RIRSample]) -> MetricResult: + return _iso3382_jnd_mae(samples, metric_name="rir_ts_jnd_mae", parameter="ts") + + def _early_late(signal: np.ndarray, rate: int, window_ms: float) -> tuple[float, float]: x = np.ascontiguousarray(np.asarray(signal, dtype="float64").reshape(-1)) peak = int(np.argmax(np.abs(x))) @@ -769,6 +950,12 @@ def rir_doa_mae(samples: list[RIRSample]) -> MetricResult: "rir_doa_mae", "rir_drr_mae", "rir_edc_mae", + "rir_edt_m_mae", + "rir_edt_broadband_mae", + "rir_edt_jnd_mae", + "rir_c80_jnd_mae", + "rir_d50_jnd_mae", + "rir_ts_jnd_mae", "rir_iacc_mae", "rir_ild_mae", "rir_ipd_mae", diff --git a/src/sure_eval/evaluation/tasks/rir/pipeline.py b/src/sure_eval/evaluation/tasks/rir/pipeline.py index d0f5eb09..6546fe73 100644 --- a/src/sure_eval/evaluation/tasks/rir/pipeline.py +++ b/src/sure_eval/evaluation/tasks/rir/pipeline.py @@ -22,6 +22,11 @@ rir_doa_mae, rir_drr_mae, rir_edc_mae, + rir_edt_broadband_mae, + rir_edt_jnd_mae, + rir_edt_m_mae, + rir_c80_jnd_mae, + rir_d50_jnd_mae, rir_iacc_early_mae, rir_iacc_full_mae, rir_iacc_late_mae, @@ -29,6 +34,7 @@ rir_ild_mae, rir_ipd_mae, rir_itd_mae, + rir_ts_jnd_mae, rir_npm, rt30_mae, rt60_mae, @@ -47,6 +53,15 @@ "iacc_late": "rir_iacc_late_mae", "itd": "rir_itd_mae", "rir_itd": "rir_itd_mae", + "edt": "rir_edt_m_mae", + "edt_m": "rir_edt_m_mae", + "rir_edt": "rir_edt_m_mae", + "broadband_edt": "rir_edt_broadband_mae", + "edt_broadband": "rir_edt_broadband_mae", + "edt_jnd": "rir_edt_jnd_mae", + "c80_jnd": "rir_c80_jnd_mae", + "d50_jnd": "rir_d50_jnd_mae", + "ts_jnd": "rir_ts_jnd_mae", } METRICS = { "rir_npm": ("scoring/rir_npm", rir_npm), @@ -56,6 +71,12 @@ "rir_c50_mae": ("scoring/rir_c50_mae", rir_c50_mae), "rir_c80_mae": ("scoring/rir_c80_mae", rir_c80_mae), "rir_edc_mae": ("scoring/rir_edc_mae", rir_edc_mae), + "rir_edt_m_mae": ("scoring/rir_edt_m_mae", rir_edt_m_mae), + "rir_edt_broadband_mae": ("scoring/rir_edt_broadband_mae", rir_edt_broadband_mae), + "rir_edt_jnd_mae": ("scoring/rir_edt_jnd_mae", rir_edt_jnd_mae), + "rir_c80_jnd_mae": ("scoring/rir_c80_jnd_mae", rir_c80_jnd_mae), + "rir_d50_jnd_mae": ("scoring/rir_d50_jnd_mae", rir_d50_jnd_mae), + "rir_ts_jnd_mae": ("scoring/rir_ts_jnd_mae", rir_ts_jnd_mae), "rir_iacc_mae": ("scoring/rir_iacc_mae", rir_iacc_mae), "rir_iacc_full_mae": ("scoring/rir_iacc_full_mae", rir_iacc_full_mae), "rir_iacc_early_mae": ("scoring/rir_iacc_early_mae", rir_iacc_early_mae), diff --git a/src/sure_eval/evaluation/tasks/rir/routes.yaml b/src/sure_eval/evaluation/tasks/rir/routes.yaml index f7e9e258..1f921cfb 100644 --- a/src/sure_eval/evaluation/tasks/rir/routes.yaml +++ b/src/sure_eval/evaluation/tasks/rir/routes.yaml @@ -51,6 +51,54 @@ routes: nodes: [scoring/rir_edc_mae] input_contract: scoring/rir_edc_mae executor: sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples + - + metric: rir_edt_m_mae + aliases: [edt, edt_m, rir_edt] + family: room_acoustics + pipeline_id: rir.any.rir_edt_m_mae.rir_edt_m_mae_v1 + nodes: [scoring/rir_edt_m_mae] + input_contract: scoring/rir_edt_m_mae + executor: sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples + - + metric: rir_edt_broadband_mae + aliases: [broadband_edt, edt_broadband] + family: room_acoustics + pipeline_id: rir.any.rir_edt_broadband_mae.rir_edt_broadband_mae_v1 + nodes: [scoring/rir_edt_broadband_mae] + input_contract: scoring/rir_edt_broadband_mae + executor: sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples + - + metric: rir_edt_jnd_mae + aliases: [edt_jnd] + family: iso3382_jnd + pipeline_id: rir.any.rir_edt_jnd_mae.rir_edt_jnd_mae_v1 + nodes: [scoring/rir_edt_jnd_mae] + input_contract: scoring/rir_edt_jnd_mae + executor: sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples + - + metric: rir_c80_jnd_mae + aliases: [c80_jnd] + family: iso3382_jnd + pipeline_id: rir.any.rir_c80_jnd_mae.rir_c80_jnd_mae_v1 + nodes: [scoring/rir_c80_jnd_mae] + input_contract: scoring/rir_c80_jnd_mae + executor: sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples + - + metric: rir_d50_jnd_mae + aliases: [d50_jnd] + family: iso3382_jnd + pipeline_id: rir.any.rir_d50_jnd_mae.rir_d50_jnd_mae_v1 + nodes: [scoring/rir_d50_jnd_mae] + input_contract: scoring/rir_d50_jnd_mae + executor: sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples + - + metric: rir_ts_jnd_mae + aliases: [ts_jnd, centre_time_jnd] + family: iso3382_jnd + pipeline_id: rir.any.rir_ts_jnd_mae.rir_ts_jnd_mae_v1 + nodes: [scoring/rir_ts_jnd_mae] + input_contract: scoring/rir_ts_jnd_mae + executor: sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples - metric: rir_iacc_mae aliases: [iacc, rir_iacc] diff --git a/tests/test_rir_task.py b/tests/test_rir_task.py index 864ecfee..29db4c18 100644 --- a/tests/test_rir_task.py +++ b/tests/test_rir_task.py @@ -239,6 +239,7 @@ def test_rir_metrics_accept_reference_npy_in_channel_first_layout(tmp_path: Path def test_rir_minimal_examples_cover_every_registered_metric(tmp_path: Path) -> None: acoustic_metrics = ("rir_npm", "rt30_mae", "rt60_mae", "rir_drr_mae", "rir_c50_mae", "rir_c80_mae", "rir_edc_mae") + iso_metrics = ("rir_edt_m_mae", "rir_edt_broadband_mae", "rir_edt_jnd_mae", "rir_c80_jnd_mae", "rir_d50_jnd_mae", "rir_ts_jnd_mae") mono_reference = tmp_path / "mono_reference.wav" mono_prediction = tmp_path / "mono_prediction.wav" mono_reference_stereo = tmp_path / "mono_reference_stereo.wav" @@ -278,7 +279,7 @@ def test_rir_minimal_examples_cover_every_registered_metric(tmp_path: Path) -> N sample_id="minimal-four-channel", sample_rate=16000, channel_layout="multi_channel", metadata={"microphone_spacing_m": 0.08}, ) - mono_report = evaluate_rir_samples([mono], metrics=acoustic_metrics, channel_mode="single") + mono_report = evaluate_rir_samples([mono], metrics=acoustic_metrics + iso_metrics, channel_mode="single") binaural_report = evaluate_rir_samples( [binaural], metrics=acoustic_metrics + ("rir_iacc_mae", "rir_iacc_full_mae", "rir_iacc_early_mae", "rir_iacc_late_mae", "rir_itd_mae", "rir_ild_mae", "rir_ipd_mae", "rir_doa_mae"), channel_mode="binaural" ) @@ -529,6 +530,12 @@ def test_rir_nodes_are_in_process_environment_checks() -> None: "rir_c50_mae", "rir_c80_mae", "rir_edc_mae", + "rir_edt_m_mae", + "rir_edt_broadband_mae", + "rir_edt_jnd_mae", + "rir_c80_jnd_mae", + "rir_d50_jnd_mae", + "rir_ts_jnd_mae", "rir_iacc_mae", "rir_iacc_full_mae", "rir_iacc_early_mae", @@ -544,6 +551,52 @@ def test_rir_nodes_are_in_process_environment_checks() -> None: assert result.required is False +@pytest.mark.parametrize("channels", (1, 2, 4)) +def test_rir_iso3382_metrics_cover_mono_binaural_and_multichannel(tmp_path: Path, channels: int) -> None: + length = 16384 + signal = np.zeros((length, channels), dtype=np.float64) + tail = np.exp(-np.arange(length - 32, dtype=np.float64) / 7000.0) + tone = np.sin(2.0 * np.pi * 500.0 * np.arange(length - 32, dtype=np.float64) / 16000.0) * np.exp(-np.arange(length - 32, dtype=np.float64) / 12000.0) + for channel in range(channels): + signal[32, channel] = 1.0 - channel * 0.05 + signal[32:, channel] += (0.1 + channel * 0.005) * tail + 0.02 * tone + prediction = signal.copy() + prediction[32:, :] *= 0.92 + np.save(tmp_path / "reference.npy", signal) + np.save(tmp_path / "prediction.npy", prediction) + sample = RIRSample( + str(tmp_path / "prediction.npy"), + str(tmp_path / "reference.npy"), + sample_rate=16000, + sample_id=f"iso-{channels}ch", + channel_layout="mono" if channels == 1 else "binaural_lr" if channels == 2 else "multi_channel", + ) + metrics = ( + "rir_edt_m_mae", + "rir_edt_broadband_mae", + "rir_edt_jnd_mae", + "rir_c80_jnd_mae", + "rir_d50_jnd_mae", + "rir_ts_jnd_mae", + ) + results = evaluate_rir_samples([sample], metrics=metrics).details["results"] + assert set(results) == set(metrics) + for metric in ("rir_edt_jnd_mae", "rir_c80_jnd_mae", "rir_d50_jnd_mae", "rir_ts_jnd_mae"): + result = results[metric] + assert result["valid_samples"] == 1 + assert len(result["per_sample"][0]["channels"]) == channels + assert result["within_one_jnd_percent"] == pytest.approx(100.0) + + +def test_rir_iso3382_routes_describe_and_preserve_jnd_details() -> None: + from sure_eval.evaluation.scripts import describe_pipeline + + for metric in ("edt", "broadband_edt", "edt_jnd", "c80_jnd", "d50_jnd", "ts_jnd"): + description = describe_pipeline("rir", metric=metric) + assert description.pipeline_id.startswith("rir.any.") + assert description.computation_node_ids[0].startswith("scoring/rir_") + + @pytest.mark.parametrize("metric", ("rt30_mae", "rt60_mae", "rir_drr_mae", "rir_c50_mae", "rir_c80_mae", "rir_edc_mae")) @pytest.mark.parametrize("invalid_role", ("prediction", "reference")) def test_rir_acoustic_metrics_do_not_drop_unscorable_channels(tmp_path: Path, metric: str, invalid_role: str) -> None: From 2d29205cb74635852dfb1853e9fbd1b50d4b1e92 Mon Sep 17 00:00:00 2001 From: Jing Peng <143288982+PigeonDan1@users.noreply.github.com> Date: Fri, 6 Nov 2026 15:17:48 +0800 Subject: [PATCH 2/3] fix(rir): collapse duplicate IACC route --- docs/pipeline_catalog.jsonl | 2 - docs/tasks/rir.md | 11 +++-- scripts/generate_pipeline_catalog.py | 1 - src/sure_eval/evaluation/cli_adapters.py | 21 ++++++-- .../nodes/scoring/rir_iacc_mae/README.md | 48 ------------------- .../nodes/scoring/rir_iacc_mae/__init__.py | 1 - .../nodes/scoring/rir_iacc_mae/manifest.yaml | 12 ----- .../nodes/scoring/rir_iacc_mae/node.py | 3 -- .../evaluation/tasks/rir/manifest.yaml | 9 ---- src/sure_eval/evaluation/tasks/rir/metrics.py | 9 ++-- .../evaluation/tasks/rir/pipeline.py | 18 ++++--- .../evaluation/tasks/rir/routes.yaml | 10 +--- tests/test_rir_task.py | 34 +++++++++++-- 13 files changed, 70 insertions(+), 109 deletions(-) delete mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_iacc_mae/README.md delete mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_iacc_mae/__init__.py delete mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_iacc_mae/manifest.yaml delete mode 100644 src/sure_eval/evaluation/nodes/scoring/rir_iacc_mae/node.py diff --git a/docs/pipeline_catalog.jsonl b/docs/pipeline_catalog.jsonl index 3e1bc10e..074394f3 100644 --- a/docs/pipeline_catalog.jsonl +++ b/docs/pipeline_catalog.jsonl @@ -208,7 +208,6 @@ {"task": "RIR estimation", "task_alias": "rir", "language": "n/a", "metric": "rir_c80_jnd_mae", "pipeline_id": "rir.any.rir_c80_jnd_mae.rir_c80_jnd_mae_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["rir_c80_jnd_mae"], "nodes": ["scoring/rir_c80_jnd_mae"], "computation_node_ids": ["scoring/rir_c80_jnd_mae"], "task_config_path": "src/sure_eval/evaluation/tasks/rir/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/rir/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.rir.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.rir.run", "executor": "sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples", "required_roles": ["prediction_rir", "reference_rir"], "optional_roles": ["metadata"]} {"task": "RIR estimation", "task_alias": "rir", "language": "n/a", "metric": "rir_d50_jnd_mae", "pipeline_id": "rir.any.rir_d50_jnd_mae.rir_d50_jnd_mae_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["rir_d50_jnd_mae"], "nodes": ["scoring/rir_d50_jnd_mae"], "computation_node_ids": ["scoring/rir_d50_jnd_mae"], "task_config_path": "src/sure_eval/evaluation/tasks/rir/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/rir/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.rir.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.rir.run", "executor": "sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples", "required_roles": ["prediction_rir", "reference_rir"], "optional_roles": ["metadata"]} {"task": "RIR estimation", "task_alias": "rir", "language": "n/a", "metric": "rir_ts_jnd_mae", "pipeline_id": "rir.any.rir_ts_jnd_mae.rir_ts_jnd_mae_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["rir_ts_jnd_mae"], "nodes": ["scoring/rir_ts_jnd_mae"], "computation_node_ids": ["scoring/rir_ts_jnd_mae"], "task_config_path": "src/sure_eval/evaluation/tasks/rir/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/rir/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.rir.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.rir.run", "executor": "sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples", "required_roles": ["prediction_rir", "reference_rir"], "optional_roles": ["metadata"]} -{"task": "RIR estimation", "task_alias": "rir", "language": "n/a", "metric": "rir_iacc_mae", "pipeline_id": "rir.any.rir_iacc_mae.rir_iacc_mae_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["rir_iacc_mae"], "nodes": ["scoring/rir_iacc_mae"], "computation_node_ids": ["scoring/rir_iacc_mae"], "task_config_path": "src/sure_eval/evaluation/tasks/rir/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/rir/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.rir.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.rir.run", "executor": "sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples", "required_roles": ["prediction_rir", "reference_rir"], "optional_roles": ["metadata"]} {"task": "RIR estimation", "task_alias": "rir", "language": "n/a", "metric": "rir_iacc_full_mae", "pipeline_id": "rir.any.rir_iacc_full_mae.rir_iacc_full_mae_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["rir_iacc_full_mae"], "nodes": ["scoring/rir_iacc_full_mae"], "computation_node_ids": ["scoring/rir_iacc_full_mae"], "task_config_path": "src/sure_eval/evaluation/tasks/rir/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/rir/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.rir.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.rir.run", "executor": "sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples", "required_roles": ["prediction_rir", "reference_rir"], "optional_roles": ["metadata"]} {"task": "RIR estimation", "task_alias": "rir", "language": "n/a", "metric": "rir_iacc_early_mae", "pipeline_id": "rir.any.rir_iacc_early_mae.rir_iacc_early_mae_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["rir_iacc_early_mae"], "nodes": ["scoring/rir_iacc_early_mae"], "computation_node_ids": ["scoring/rir_iacc_early_mae"], "task_config_path": "src/sure_eval/evaluation/tasks/rir/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/rir/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.rir.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.rir.run", "executor": "sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples", "required_roles": ["prediction_rir", "reference_rir"], "optional_roles": ["metadata"]} {"task": "RIR estimation", "task_alias": "rir", "language": "n/a", "metric": "rir_iacc_late_mae", "pipeline_id": "rir.any.rir_iacc_late_mae.rir_iacc_late_mae_v1", "pipeline_kind": "atomic", "member_pipeline_ids": [], "execution_metrics": ["rir_iacc_late_mae"], "nodes": ["scoring/rir_iacc_late_mae"], "computation_node_ids": ["scoring/rir_iacc_late_mae"], "task_config_path": "src/sure_eval/evaluation/tasks/rir/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/rir/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.rir.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.rir.run", "executor": "sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples", "required_roles": ["prediction_rir", "reference_rir"], "optional_roles": ["metadata"]} @@ -227,5 +226,4 @@ {"task": "RIR estimation", "task_alias": "rir", "language": "n/a", "metric": "multi", "pipeline_id": "rir.any.multi.rir_npm.rir_npm_v1__rt30_mae.rt30_mae_v1__rt60_mae.rt60_mae_v1__rir_drr_mae.rir_drr_mae_v1__rir_c50_mae.rir_c50_mae_v1__rir_c80_mae.rir_c80_mae_v1__rir_edc_mae.rir_edc_mae_v1", "pipeline_kind": "bundle", "member_pipeline_ids": ["rir.any.rir_npm.rir_npm_v1", "rir.any.rt30_mae.rt30_mae_v1", "rir.any.rt60_mae.rt60_mae_v1", "rir.any.rir_drr_mae.rir_drr_mae_v1", "rir.any.rir_c50_mae.rir_c50_mae_v1", "rir.any.rir_c80_mae.rir_c80_mae_v1", "rir.any.rir_edc_mae.rir_edc_mae_v1"], "execution_metrics": ["rir_npm", "rt30_mae", "rt60_mae", "rir_drr_mae", "rir_c50_mae", "rir_c80_mae", "rir_edc_mae"], "nodes": ["scoring/rir_npm", "scoring/rt30_mae", "scoring/rt60_mae", "scoring/rir_drr_mae", "scoring/rir_c50_mae", "scoring/rir_c80_mae", "scoring/rir_edc_mae"], "computation_node_ids": ["scoring/rir_npm", "scoring/rt30_mae", "scoring/rt60_mae", "scoring/rir_drr_mae", "scoring/rir_c50_mae", "scoring/rir_c80_mae", "scoring/rir_edc_mae"], "task_config_path": "src/sure_eval/evaluation/tasks/rir/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/rir/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.rir.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.rir.run", "executor": "sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples", "required_roles": ["prediction_rir", "reference_rir"], "optional_roles": ["metadata"]} {"task": "RIR estimation", "task_alias": "rir", "language": "n/a", "metric": "multi", "pipeline_id": "rir.any.multi.rir_npm.rir_npm_v1__rt30_mae.rt30_mae_v1__rt60_mae.rt60_mae_v1__rir_drr_mae.rir_drr_mae_v1__rir_c50_mae.rir_c50_mae_v1__rir_c80_mae.rir_c80_mae_v1__rir_edc_mae.rir_edc_mae_v1__rir_iacc_full_mae.rir_iacc_full_mae_v1__rir_iacc_early_mae.rir_iacc_early_mae_v1__rir_iacc_late_mae.rir_iacc_late_mae_v1__rir_itd_mae.rir_itd_mae_v1__rir_ild_mae.rir_ild_mae_v1__rir_ipd_mae.rir_ipd_mae_v1__rir_doa_mae.rir_doa_mae_v1", "pipeline_kind": "bundle", "member_pipeline_ids": ["rir.any.rir_npm.rir_npm_v1", "rir.any.rt30_mae.rt30_mae_v1", "rir.any.rt60_mae.rt60_mae_v1", "rir.any.rir_drr_mae.rir_drr_mae_v1", "rir.any.rir_c50_mae.rir_c50_mae_v1", "rir.any.rir_c80_mae.rir_c80_mae_v1", "rir.any.rir_edc_mae.rir_edc_mae_v1", "rir.any.rir_iacc_full_mae.rir_iacc_full_mae_v1", "rir.any.rir_iacc_early_mae.rir_iacc_early_mae_v1", "rir.any.rir_iacc_late_mae.rir_iacc_late_mae_v1", "rir.any.rir_itd_mae.rir_itd_mae_v1", "rir.any.rir_ild_mae.rir_ild_mae_v1", "rir.any.rir_ipd_mae.rir_ipd_mae_v1", "rir.any.rir_doa_mae.rir_doa_mae_v1"], "execution_metrics": ["rir_npm", "rt30_mae", "rt60_mae", "rir_drr_mae", "rir_c50_mae", "rir_c80_mae", "rir_edc_mae", "rir_iacc_full_mae", "rir_iacc_early_mae", "rir_iacc_late_mae", "rir_itd_mae", "rir_ild_mae", "rir_ipd_mae", "rir_doa_mae"], "nodes": ["scoring/rir_npm", "scoring/rt30_mae", "scoring/rt60_mae", "scoring/rir_drr_mae", "scoring/rir_c50_mae", "scoring/rir_c80_mae", "scoring/rir_edc_mae", "scoring/rir_iacc_full_mae", "scoring/rir_iacc_early_mae", "scoring/rir_iacc_late_mae", "scoring/rir_itd_mae", "scoring/rir_ild_mae", "scoring/rir_ipd_mae", "scoring/rir_doa_mae"], "computation_node_ids": ["scoring/rir_npm", "scoring/rt30_mae", "scoring/rt60_mae", "scoring/rir_drr_mae", "scoring/rir_c50_mae", "scoring/rir_c80_mae", "scoring/rir_edc_mae", "scoring/rir_iacc_full_mae", "scoring/rir_iacc_early_mae", "scoring/rir_iacc_late_mae", "scoring/rir_itd_mae", "scoring/rir_ild_mae", "scoring/rir_ipd_mae", "scoring/rir_doa_mae"], "task_config_path": "src/sure_eval/evaluation/tasks/rir/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/rir/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.rir.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.rir.run", "executor": "sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples", "required_roles": ["prediction_rir", "reference_rir"], "optional_roles": ["metadata"]} {"task": "RIR estimation", "task_alias": "rir", "language": "n/a", "metric": "multi", "pipeline_id": "rir.any.multi.rir_npm.rir_npm_v1__rt30_mae.rt30_mae_v1__rt60_mae.rt60_mae_v1__rir_drr_mae.rir_drr_mae_v1__rir_c50_mae.rir_c50_mae_v1__rir_c80_mae.rir_c80_mae_v1__rir_edc_mae.rir_edc_mae_v1__rir_itd_mae.rir_itd_mae_v1__rir_ild_mae.rir_ild_mae_v1__rir_ipd_mae.rir_ipd_mae_v1__rir_doa_mae.rir_doa_mae_v1", "pipeline_kind": "bundle", "member_pipeline_ids": ["rir.any.rir_npm.rir_npm_v1", "rir.any.rt30_mae.rt30_mae_v1", "rir.any.rt60_mae.rt60_mae_v1", "rir.any.rir_drr_mae.rir_drr_mae_v1", "rir.any.rir_c50_mae.rir_c50_mae_v1", "rir.any.rir_c80_mae.rir_c80_mae_v1", "rir.any.rir_edc_mae.rir_edc_mae_v1", "rir.any.rir_itd_mae.rir_itd_mae_v1", "rir.any.rir_ild_mae.rir_ild_mae_v1", "rir.any.rir_ipd_mae.rir_ipd_mae_v1", "rir.any.rir_doa_mae.rir_doa_mae_v1"], "execution_metrics": ["rir_npm", "rt30_mae", "rt60_mae", "rir_drr_mae", "rir_c50_mae", "rir_c80_mae", "rir_edc_mae", "rir_itd_mae", "rir_ild_mae", "rir_ipd_mae", "rir_doa_mae"], "nodes": ["scoring/rir_npm", "scoring/rt30_mae", "scoring/rt60_mae", "scoring/rir_drr_mae", "scoring/rir_c50_mae", "scoring/rir_c80_mae", "scoring/rir_edc_mae", "scoring/rir_itd_mae", "scoring/rir_ild_mae", "scoring/rir_ipd_mae", "scoring/rir_doa_mae"], "computation_node_ids": ["scoring/rir_npm", "scoring/rt30_mae", "scoring/rt60_mae", "scoring/rir_drr_mae", "scoring/rir_c50_mae", "scoring/rir_c80_mae", "scoring/rir_edc_mae", "scoring/rir_itd_mae", "scoring/rir_ild_mae", "scoring/rir_ipd_mae", "scoring/rir_doa_mae"], "task_config_path": "src/sure_eval/evaluation/tasks/rir/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/rir/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.rir.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.rir.run", "executor": "sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples", "required_roles": ["prediction_rir", "reference_rir"], "optional_roles": ["metadata"]} -{"task": "RIR estimation", "task_alias": "rir", "language": "n/a", "metric": "multi", "pipeline_id": "rir.any.multi.rir_edt_m_mae.rir_edt_m_mae_v1__rir_edt_broadband_mae.rir_edt_broadband_mae_v1", "pipeline_kind": "bundle", "member_pipeline_ids": ["rir.any.rir_edt_m_mae.rir_edt_m_mae_v1", "rir.any.rir_edt_broadband_mae.rir_edt_broadband_mae_v1"], "execution_metrics": ["rir_edt_m_mae", "rir_edt_broadband_mae"], "nodes": ["scoring/rir_edt_m_mae", "scoring/rir_edt_broadband_mae"], "computation_node_ids": ["scoring/rir_edt_m_mae", "scoring/rir_edt_broadband_mae"], "task_config_path": "src/sure_eval/evaluation/tasks/rir/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/rir/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.rir.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.rir.run", "executor": "sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples", "required_roles": ["prediction_rir", "reference_rir"], "optional_roles": ["metadata"]} {"task": "RIR estimation", "task_alias": "rir", "language": "n/a", "metric": "multi", "pipeline_id": "rir.any.multi.rir_edt_jnd_mae.rir_edt_jnd_mae_v1__rir_c80_jnd_mae.rir_c80_jnd_mae_v1__rir_d50_jnd_mae.rir_d50_jnd_mae_v1__rir_ts_jnd_mae.rir_ts_jnd_mae_v1", "pipeline_kind": "bundle", "member_pipeline_ids": ["rir.any.rir_edt_jnd_mae.rir_edt_jnd_mae_v1", "rir.any.rir_c80_jnd_mae.rir_c80_jnd_mae_v1", "rir.any.rir_d50_jnd_mae.rir_d50_jnd_mae_v1", "rir.any.rir_ts_jnd_mae.rir_ts_jnd_mae_v1"], "execution_metrics": ["rir_edt_jnd_mae", "rir_c80_jnd_mae", "rir_d50_jnd_mae", "rir_ts_jnd_mae"], "nodes": ["scoring/rir_edt_jnd_mae", "scoring/rir_c80_jnd_mae", "scoring/rir_d50_jnd_mae", "scoring/rir_ts_jnd_mae"], "computation_node_ids": ["scoring/rir_edt_jnd_mae", "scoring/rir_c80_jnd_mae", "scoring/rir_d50_jnd_mae", "scoring/rir_ts_jnd_mae"], "task_config_path": "src/sure_eval/evaluation/tasks/rir/manifest.yaml", "route_config_path": "src/sure_eval/evaluation/tasks/rir/routes.yaml", "describe_entrypoint": "sure_eval.evaluation.scripts.rir.describe_pipeline", "script_entrypoint": "sure_eval.evaluation.scripts.rir.run", "executor": "sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples", "required_roles": ["prediction_rir", "reference_rir"], "optional_roles": ["metadata"]} diff --git a/docs/tasks/rir.md b/docs/tasks/rir.md index 3af9e995..f09027b9 100644 --- a/docs/tasks/rir.md +++ b/docs/tasks/rir.md @@ -57,8 +57,7 @@ used when it is omitted. | `rir_c80_jnd_mae` | C80 error divided by 1 dB (`x_jnd`) | lower is better | | `rir_d50_jnd_mae` | D50 error divided by 0.05 (`x_jnd`) | lower is better | | `rir_ts_jnd_mae` | centre-time error divided by 10 ms (`x_jnd`) | lower is better | -| `rir_iacc_mae` | compatibility alias for full-window IACC error | lower is better | -| `rir_iacc_full_mae` | full-window interaural cross-correlation coefficient error in a +/-1 ms window | lower is better | +| `rir_iacc_full_mae` | full-window interaural cross-correlation coefficient error in a +/-1 ms window; `iacc`, `rir_iacc`, and `rir_iacc_mae` are compatibility selectors | lower is better | | `rir_iacc_early_mae` | IACC from the ch0 direct peak through 80 ms | lower is better | | `rir_iacc_late_mae` | IACC after the ch0 direct peak plus 80 ms | lower is better | | `rir_itd_mae` | interaural time difference error in ms in a +/-1 ms window | lower is better | @@ -81,9 +80,11 @@ The bundle's top-level `score` is `rir_npm`; all metric scores are in sample within 20 dB of the peak, filter at 500 Hz and 1 kHz with a zero-phase Butterworth octave-band filter, fit the initial 0 to -10 dB Schroeder decay, extrapolate to 60 dB, average the two band values per channel, and report -seconds of absolute error. `rir_edt_broadband_mae` performs the same decay fit -on the full-band signal with an independent channel onset and does not apply a -JND threshold. +seconds of absolute error. `rir_edt_broadband_mae` is an optional strict atomic +route. It performs the same decay fit on the full-band signal with an independent +channel onset and does not apply a JND threshold. It is not part of the +leaderboard or JND bundle because some measured RIRs cross -10 dB within fewer +than four samples, which is insufficient for a meaningful broadband EDT fit. The four `*_jnd_mae` routes use the informative BS EN ISO 3382-1:2009 Annex A Table A.1 thresholds: EDT 5% of the reference EDT, C80 1 dB, D50 0.05, and Ts diff --git a/scripts/generate_pipeline_catalog.py b/scripts/generate_pipeline_catalog.py index c33a7155..d961b22a 100644 --- a/scripts/generate_pipeline_catalog.py +++ b/scripts/generate_pipeline_catalog.py @@ -114,7 +114,6 @@ ) }, ), - ("rir", {"metrics": ("rir_edt_m_mae", "rir_edt_broadband_mae")}), ("rir", {"metrics": ("rir_edt_jnd_mae", "rir_c80_jnd_mae", "rir_d50_jnd_mae", "rir_ts_jnd_mae")}), ) diff --git a/src/sure_eval/evaluation/cli_adapters.py b/src/sure_eval/evaluation/cli_adapters.py index 8f8c9c9a..b98509bc 100644 --- a/src/sure_eval/evaluation/cli_adapters.py +++ b/src/sure_eval/evaluation/cli_adapters.py @@ -136,6 +136,7 @@ def list_metric_routes( routes, _ = load_task_routes(route_task) requested_language = _normalize_language_filter(language) requested_metric = canonical_metric(metric) if metric else None + requested_metric_tokens = _metric_filter_tokens(metric) if metric else set() language_sensitive = bool(routes.get("language_sensitive", False)) if not language_sensitive and requested_language not in {None, "n/a"}: raise ValueError(f"Task {task!r} is language-independent; omit --language or use any") @@ -146,10 +147,7 @@ def list_metric_routes( route = dict(configured_route) if not _route_matches_task_alias(route, requested_task): continue - if ( - requested_metric - and canonical_metric(str(route.get("metric") or "")) != requested_metric - ): + if requested_metric_tokens and not requested_metric_tokens.intersection(_route_metric_tokens(route)): continue route_language = _concrete_route_language( route, @@ -291,6 +289,21 @@ def _route_matches_task_alias(route: dict[str, Any], requested_task: str) -> boo return normalize_task(str(route_alias)) == requested_task +def _metric_filter_tokens(metric: str) -> set[str]: + """Return raw and canonical forms accepted by route selection.""" + + raw = str(metric).strip().lower().replace("-", "_") + return {raw, canonical_metric(raw)} + + +def _route_metric_tokens(route: dict[str, Any]) -> set[str]: + tokens: set[str] = set() + for value in (route.get("metric"), *(route.get("aliases") or ())): + raw = str(value).strip().lower().replace("-", "_") + tokens.update((raw, canonical_metric(raw))) + return tokens + + def _route_matches_language(route: dict[str, Any], requested_language: str | None) -> bool: if requested_language is None: return True diff --git a/src/sure_eval/evaluation/nodes/scoring/rir_iacc_mae/README.md b/src/sure_eval/evaluation/nodes/scoring/rir_iacc_mae/README.md deleted file mode 100644 index 66abbda3..00000000 --- a/src/sure_eval/evaluation/nodes/scoring/rir_iacc_mae/README.md +++ /dev/null @@ -1,48 +0,0 @@ -# RIR IACC MAE - -## Purpose - -Reports interaural cross-correlation coefficient (IACC) error for a binaural -RIR pair. It does not describe arrays with more than two channels. - -## Task Scenarios - -The `RIR estimation` two-channel spatial route selects -`scoring/rir_iacc_mae`, atomically or in the binaural bundle. It is not in -the multichannel bundle for larger arrays. - -## Input - -The `binaural_rir_samples_jsonl` row contains `sample_id`, `prediction_rir`, -and `reference_rir`; alignment is by `sample_id`. Both files must have -exactly two channels and matching sample rates. NumPy arrays require -`sample_rate`. - -## Output - -The `rir_iacc_mae_score` trace reports lower-is-better coefficient error. -Each `per_sample` entry includes predicted IACC, reference IACC, and their -absolute difference. The metric averages sample errors equally. - -## Versioned Computation - -Node `scoring/rir_iacc_mae`, version `v1`, takes the largest absolute -normalized cross-correlation of the two channels within +/-1 ms. It -compares that coefficient between prediction and reference. - -## Runtime and Assets - -The node runs in-process with NumPy and the optional audio loader. It has no -checkpoint, binary, or node-local environment. - -## Source and References - -Local SURE RIR implementation in `tasks/rir/metrics.py`. - -## Limitations - -Undefined correlation (silent or constant channels, or an empty selected -window) fails evaluation rather than receiving zero error. - -Only exactly two channels are accepted. The search is broadband and does -not use microphone spacing or frequency-dependent weighting. diff --git a/src/sure_eval/evaluation/nodes/scoring/rir_iacc_mae/__init__.py b/src/sure_eval/evaluation/nodes/scoring/rir_iacc_mae/__init__.py deleted file mode 100644 index 7390739d..00000000 --- a/src/sure_eval/evaluation/nodes/scoring/rir_iacc_mae/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""RIR IACC scoring node metadata package.""" diff --git a/src/sure_eval/evaluation/nodes/scoring/rir_iacc_mae/manifest.yaml b/src/sure_eval/evaluation/nodes/scoring/rir_iacc_mae/manifest.yaml deleted file mode 100644 index bce726bb..00000000 --- a/src/sure_eval/evaluation/nodes/scoring/rir_iacc_mae/manifest.yaml +++ /dev/null @@ -1,12 +0,0 @@ -id: scoring/rir_iacc_mae -version: v1 -stage: scoring -language_sensitive: false -input_schema: binaural_rir_samples_jsonl -output_schema: rir_iacc_mae_score -implementation: sure_eval.evaluation.tasks.rir.metrics.rir_iacc_mae -metric: rir_iacc_mae -backend: numpy-rir -internal_stages: [audio_load, binaural_channel_check, cross_correlation, mean_aggregation] -used_by: [RIR binaural spatial error] -notes: [Lower is better. Unit is a correlation coefficient error.] diff --git a/src/sure_eval/evaluation/nodes/scoring/rir_iacc_mae/node.py b/src/sure_eval/evaluation/nodes/scoring/rir_iacc_mae/node.py deleted file mode 100644 index 90dfeca5..00000000 --- a/src/sure_eval/evaluation/nodes/scoring/rir_iacc_mae/node.py +++ /dev/null @@ -1,3 +0,0 @@ -from sure_eval.evaluation.tasks.rir.metrics import rir_iacc_mae - -__all__ = ["rir_iacc_mae"] diff --git a/src/sure_eval/evaluation/tasks/rir/manifest.yaml b/src/sure_eval/evaluation/tasks/rir/manifest.yaml index fd3b0249..6d623af5 100644 --- a/src/sure_eval/evaluation/tasks/rir/manifest.yaml +++ b/src/sure_eval/evaluation/tasks/rir/manifest.yaml @@ -15,7 +15,6 @@ implemented_metrics: - rir_c80_jnd_mae - rir_d50_jnd_mae - rir_ts_jnd_mae - - rir_iacc_mae - rir_iacc_full_mae - rir_iacc_early_mae - rir_iacc_late_mae @@ -154,14 +153,6 @@ input_contracts: alignment_key: sample_id aggregation: mean main_report: true - scoring/rir_iacc_mae: - purpose: binaural_interaural_cross_correlation_error - required_roles: [prediction_rir, reference_rir] - optional_roles: [metadata] - row_format: binaural_rir_samples_jsonl - alignment_key: sample_id - aggregation: mean - main_report: true scoring/rir_iacc_full_mae: purpose: rir_binaural_full_window_interaural_cross_correlation_error required_roles: [prediction_rir, reference_rir] diff --git a/src/sure_eval/evaluation/tasks/rir/metrics.py b/src/sure_eval/evaluation/tasks/rir/metrics.py index ebe1f31c..da3cef1c 100644 --- a/src/sure_eval/evaluation/tasks/rir/metrics.py +++ b/src/sure_eval/evaluation/tasks/rir/metrics.py @@ -545,7 +545,7 @@ def _iso3382_edt_mae(samples: list[RIRSample], *, metric_name: str, broadband: b reference_values = [row["edt"] for row in _iso3382_parameters(reference, rate)] errors = [abs(predicted - target) for predicted, target in zip(prediction_values, reference_values) if np.isfinite(predicted) and np.isfinite(target)] if len(errors) != prediction.shape[1]: - raise ValueError(f"EDT could not be estimated for {sample.sample_id}") + raise ValueError(f"{metric_name} could not be estimated for {sample.sample_id}") channel_rows = [ {"channel": channel, "prediction_edt": float(predicted), "reference_edt": float(target), "score": float(error), "unit": "seconds"} for channel, (predicted, target, error) in enumerate(zip(prediction_values, reference_values, errors)) @@ -780,7 +780,7 @@ def rir_iacc_late_mae(samples: list[RIRSample]) -> MetricResult: def rir_iacc_mae(samples: list[RIRSample]) -> MetricResult: - """Compatibility alias for the full-window IACC error.""" + """Deprecated compatibility wrapper for the full-window IACC error.""" result = rir_iacc_full_mae(samples) return MetricResult("rir_iacc_mae", result.score, result.details) @@ -940,7 +940,7 @@ def rir_doa_mae(samples: list[RIRSample]) -> MetricResult: # Short aliases keep direct task-level callers aligned with the public metric # selectors while reports retain the explicit ``*_mae`` names. -iacc = rir_iacc_mae +iacc = rir_iacc_full_mae itd = rir_itd_mae @@ -956,6 +956,9 @@ def rir_doa_mae(samples: list[RIRSample]) -> MetricResult: "rir_c80_jnd_mae", "rir_d50_jnd_mae", "rir_ts_jnd_mae", + "rir_iacc_full_mae", + "rir_iacc_early_mae", + "rir_iacc_late_mae", "rir_iacc_mae", "rir_ild_mae", "rir_ipd_mae", diff --git a/src/sure_eval/evaluation/tasks/rir/pipeline.py b/src/sure_eval/evaluation/tasks/rir/pipeline.py index 6546fe73..aff0cb22 100644 --- a/src/sure_eval/evaluation/tasks/rir/pipeline.py +++ b/src/sure_eval/evaluation/tasks/rir/pipeline.py @@ -30,7 +30,6 @@ rir_iacc_early_mae, rir_iacc_full_mae, rir_iacc_late_mae, - rir_iacc_mae, rir_ild_mae, rir_ipd_mae, rir_itd_mae, @@ -46,8 +45,9 @@ METRIC_ALIASES = { "npm": "rir_npm", "rir_npm": "rir_npm", - "iacc": "rir_iacc_mae", - "rir_iacc": "rir_iacc_mae", + "iacc": "rir_iacc_full_mae", + "rir_iacc": "rir_iacc_full_mae", + "rir_iacc_mae": "rir_iacc_full_mae", "iacc_full": "rir_iacc_full_mae", "iacc_early": "rir_iacc_early_mae", "iacc_late": "rir_iacc_late_mae", @@ -77,7 +77,6 @@ "rir_c80_jnd_mae": ("scoring/rir_c80_jnd_mae", rir_c80_jnd_mae), "rir_d50_jnd_mae": ("scoring/rir_d50_jnd_mae", rir_d50_jnd_mae), "rir_ts_jnd_mae": ("scoring/rir_ts_jnd_mae", rir_ts_jnd_mae), - "rir_iacc_mae": ("scoring/rir_iacc_mae", rir_iacc_mae), "rir_iacc_full_mae": ("scoring/rir_iacc_full_mae", rir_iacc_full_mae), "rir_iacc_early_mae": ("scoring/rir_iacc_early_mae", rir_iacc_early_mae), "rir_iacc_late_mae": ("scoring/rir_iacc_late_mae", rir_iacc_late_mae), @@ -86,9 +85,9 @@ "rir_ipd_mae": ("scoring/rir_ipd_mae", rir_ipd_mae), "rir_doa_mae": ("scoring/rir_doa_mae", rir_doa_mae), } -IACC_METRICS = frozenset({"rir_iacc_mae", "rir_iacc_full_mae", "rir_iacc_early_mae", "rir_iacc_late_mae"}) +IACC_METRICS = frozenset({"rir_iacc_full_mae", "rir_iacc_early_mae", "rir_iacc_late_mae"}) SPATIAL_METRICS = frozenset( - {"rir_iacc_mae", "rir_iacc_full_mae", "rir_iacc_early_mae", "rir_iacc_late_mae", "rir_itd_mae", "rir_ild_mae", "rir_ipd_mae", "rir_doa_mae"} + {"rir_iacc_full_mae", "rir_iacc_early_mae", "rir_iacc_late_mae", "rir_itd_mae", "rir_ild_mae", "rir_ipd_mae", "rir_doa_mae"} ) @@ -100,7 +99,12 @@ def evaluate_rir_samples( ) -> EvaluationReport: if not samples: raise ValueError("at least one RIR sample is required") - requested = tuple(_normalize_metric(metric) for metric in (metrics or DEFAULT_METRICS)) + requested_values = [] + for metric in (metrics or DEFAULT_METRICS): + normalized = _normalize_metric(metric) + if normalized not in requested_values: + requested_values.append(normalized) + requested = tuple(requested_values) if not requested: raise ValueError("at least one RIR metric is required") unsupported = [metric for metric in requested if metric not in METRICS] diff --git a/src/sure_eval/evaluation/tasks/rir/routes.yaml b/src/sure_eval/evaluation/tasks/rir/routes.yaml index 1f921cfb..192a3c8f 100644 --- a/src/sure_eval/evaluation/tasks/rir/routes.yaml +++ b/src/sure_eval/evaluation/tasks/rir/routes.yaml @@ -99,17 +99,9 @@ routes: nodes: [scoring/rir_ts_jnd_mae] input_contract: scoring/rir_ts_jnd_mae executor: sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples - - - metric: rir_iacc_mae - aliases: [iacc, rir_iacc] - family: binaural_spatial_error - pipeline_id: rir.any.rir_iacc_mae.rir_iacc_mae_v1 - nodes: [scoring/rir_iacc_mae] - input_contract: scoring/rir_iacc_mae - executor: sure_eval.evaluation.tasks.rir.pipeline.evaluate_rir_samples - metric: rir_iacc_full_mae - aliases: [iacc_full, rir_iacc_full] + aliases: [iacc, rir_iacc, rir_iacc_mae, iacc_full, rir_iacc_full] family: binaural_spatial_error pipeline_id: rir.any.rir_iacc_full_mae.rir_iacc_full_mae_v1 nodes: [scoring/rir_iacc_full_mae] diff --git a/tests/test_rir_task.py b/tests/test_rir_task.py index 29db4c18..571cb6df 100644 --- a/tests/test_rir_task.py +++ b/tests/test_rir_task.py @@ -72,8 +72,8 @@ def test_rir_pipeline_describes_registered_route() -> None: ) iacc = describe_pipeline("rir", metric="iacc") - assert iacc.pipeline_id == "rir.any.rir_iacc_mae.rir_iacc_mae_v1" - assert iacc.metric == "rir_iacc_mae" + assert iacc.pipeline_id == "rir.any.rir_iacc_full_mae.rir_iacc_full_mae_v1" + assert iacc.metric == "rir_iacc_full_mae" for metric in ("rir_iacc_full_mae", "rir_iacc_early_mae", "rir_iacc_late_mae"): window = describe_pipeline("rir", metric=metric) assert window.metric == metric @@ -137,6 +137,12 @@ def test_rir_cli_routes_describe_and_run_preserve_pipeline_id(tmp_path: Path) -> routes_payload = json.loads(routes.stdout) assert routes_payload["default_pipeline_id"] == "rir.any.rir_npm.rir_npm_v1" + iacc_routes = runner.invoke(app, ["metric", "routes", "rir", "--metric", "iacc", "--json"]) + assert iacc_routes.exit_code == 0, iacc_routes.stdout + assert json.loads(iacc_routes.stdout)["default_pipeline_id"] == ( + "rir.any.rir_iacc_full_mae.rir_iacc_full_mae_v1" + ) + described = runner.invoke( app, [ @@ -281,14 +287,15 @@ def test_rir_minimal_examples_cover_every_registered_metric(tmp_path: Path) -> N ) mono_report = evaluate_rir_samples([mono], metrics=acoustic_metrics + iso_metrics, channel_mode="single") binaural_report = evaluate_rir_samples( - [binaural], metrics=acoustic_metrics + ("rir_iacc_mae", "rir_iacc_full_mae", "rir_iacc_early_mae", "rir_iacc_late_mae", "rir_itd_mae", "rir_ild_mae", "rir_ipd_mae", "rir_doa_mae"), channel_mode="binaural" + [binaural], metrics=acoustic_metrics + ("rir_iacc_full_mae", "rir_iacc_early_mae", "rir_iacc_late_mae", "rir_itd_mae", "rir_ild_mae", "rir_ipd_mae", "rir_doa_mae"), channel_mode="binaural" ) four_report = evaluate_rir_samples( [four], metrics=acoustic_metrics + ("rir_itd_mae", "rir_ild_mae", "rir_ipd_mae", "rir_doa_mae"), channel_mode="multi" ) covered = set(mono_report.details["results"]) | set(binaural_report.details["results"]) | set(four_report.details["results"]) assert covered == set(METRICS) - assert binaural_report.details["results"]["rir_iacc_mae"]["score"] == binaural_report.details["results"]["rir_iacc_full_mae"]["score"] + alias_report = evaluate_rir_samples([binaural], metrics=("iacc", "iacc_full"), channel_mode="binaural") + assert set(alias_report.details["results"]) == {"rir_iacc_full_mae"} assert four_report.details["results"]["rir_doa_mae"]["per_sample"][0]["pair_count"] == 3 @@ -536,7 +543,6 @@ def test_rir_nodes_are_in_process_environment_checks() -> None: "rir_c80_jnd_mae", "rir_d50_jnd_mae", "rir_ts_jnd_mae", - "rir_iacc_mae", "rir_iacc_full_mae", "rir_iacc_early_mae", "rir_iacc_late_mae", @@ -597,6 +603,24 @@ def test_rir_iso3382_routes_describe_and_preserve_jnd_details() -> None: assert description.computation_node_ids[0].startswith("scoring/rir_") +def test_rir_broadband_edt_remains_strict_for_insufficient_decay(tmp_path: Path) -> None: + reference = np.zeros((256, 1), dtype=np.float64) + prediction = np.zeros((256, 1), dtype=np.float64) + reference[8, 0] = 1.0 + prediction[8, 0] = 0.9 + np.save(tmp_path / "reference.npy", reference) + np.save(tmp_path / "prediction.npy", prediction) + sample = RIRSample( + str(tmp_path / "prediction.npy"), + str(tmp_path / "reference.npy"), + sample_rate=16000, + sample_id="broadband-short-decay", + channel_layout="mono", + ) + with pytest.raises(ValueError, match="rir_edt_broadband_mae could not be estimated"): + evaluate_rir_samples([sample], metrics=("rir_edt_broadband_mae",), channel_mode="single") + + @pytest.mark.parametrize("metric", ("rt30_mae", "rt60_mae", "rir_drr_mae", "rir_c50_mae", "rir_c80_mae", "rir_edc_mae")) @pytest.mark.parametrize("invalid_role", ("prediction", "reference")) def test_rir_acoustic_metrics_do_not_drop_unscorable_channels(tmp_path: Path, metric: str, invalid_role: str) -> None: From 7dd2b8a867449e178fa332ab219ef2f38cad9896 Mon Sep 17 00:00:00 2001 From: Jing Peng <143288982+PigeonDan1@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:48:32 +0800 Subject: [PATCH 3/3] fix(rir): align metrics with leaderboard protocol --- README.md | 2 +- README_ZH.md | 2 +- docs/atlas/index.html | 12 +- docs/atlas/pipeline_atlas.svg | 4 +- docs/atlas/pipeline_atlas_dark.svg | 4 +- docs/tasks/rir.md | 24 ++- .../nodes/scoring/rir_c80_jnd_mae/README.md | 55 +++++- .../nodes/scoring/rir_d50_jnd_mae/README.md | 55 +++++- .../nodes/scoring/rir_drr_mae/README.md | 6 +- .../scoring/rir_edt_broadband_mae/README.md | 53 +++++- .../rir_edt_broadband_mae/manifest.yaml | 2 +- .../nodes/scoring/rir_edt_jnd_mae/README.md | 61 +++++-- .../nodes/scoring/rir_edt_m_mae/README.md | 56 +++++- .../nodes/scoring/rir_npm/README.md | 8 +- .../nodes/scoring/rir_ts_jnd_mae/README.md | 55 +++++- .../nodes/scoring/rt30_mae/README.md | 9 +- src/sure_eval/evaluation/tasks/rir/README.md | 15 +- src/sure_eval/evaluation/tasks/rir/metrics.py | 167 ++++++++++-------- tests/test_rir_task.py | 125 ++++++++++++- 19 files changed, 558 insertions(+), 157 deletions(-) diff --git a/README.md b/README.md index 4099c4cd..2ce424b1 100644 --- a/README.md +++ b/README.md @@ -203,7 +203,7 @@ packages and Git. | TTS | CER, WER, speaker similarity, DNSMOS, WV-MOS, UTMOS, KOMOS (Korean) | [TTS](docs/tasks/tts.md) | | TTA | FAD, FD, KL, IS, CLAP score, AudioBox Aesthetics CE/CU/PC/PQ | [TTA](docs/tasks/tta.md) | | VC | CER, WER, speaker similarity, DNSMOS, WV-MOS, UTMOS | [VC](docs/tasks/vc.md) | -| SE | SI-SDR, STOI, PESQ, DNSMOS, WV-MOS, UTMOS | [SE](docs/tasks/se.md) | +| SE | SI-SDR, STOI/eSTOI, PESQ, LSD, nSRMR, LPS, DNSMOS P.835/P.808 components, WV-MOS, UTMOS, NISQA DIM, SQUIM objective predictions | [SE](docs/tasks/se.md) | | RIR estimation | NPM, RT30/RT60 MAE, DRR/C50/C80/EDC, octave-band and broadband EDT, ISO 3382 EDT/C80/D50/Ts JND, IACC full/early/late, ITD/ILD/IPD/DOA errors | [RIR estimation](docs/tasks/rir.md) | | TSE | SI-SDR, speaker similarity, DNSMOS, WV-MOS, UTMOS, CER, WER | [TSE](docs/tasks/tse.md) | | Classification / SER / GR | Accuracy | [Classification](docs/tasks/classification.md) | diff --git a/README_ZH.md b/README_ZH.md index 8611aab3..2ad7a787 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -192,7 +192,7 @@ node-local 虚拟环境只保留在本地,不会进入安装包或 Git。 | TTS | CER、WER、说话人相似度、DNSMOS、WV-MOS、UTMOS、KOMOS(韩语) | [TTS](docs/tasks/tts.md) | | TTA | FAD、FD、KL、IS、CLAP score、AudioBox Aesthetics CE/CU/PC/PQ | [TTA](docs/tasks/tta.md) | | VC | CER、WER、说话人相似度、DNSMOS、WV-MOS、UTMOS | [VC](docs/tasks/vc.md) | -| SE | SI-SDR、STOI、PESQ、DNSMOS、WV-MOS、UTMOS | [SE](docs/tasks/se.md) | +| SE | SI-SDR、STOI/eSTOI、PESQ、LSD、nSRMR、LPS、DNSMOS P.835/P.808 分量、WV-MOS、UTMOS、NISQA DIM、SQUIM 预测指标 | [SE](docs/tasks/se.md) | | RIR estimation | NPM、RT30/RT60 MAE、DRR/C50/C80/EDC、倍频带与宽带 EDT、ISO 3382 EDT/C80/D50/Ts JND、IACC 全窗/早期/晚期、ITD/ILD/IPD/DOA 误差 | [RIR estimation](docs/tasks/rir.md) | | TSE | SI-SDR、说话人相似度、DNSMOS、WV-MOS、UTMOS、CER、WER | [TSE](docs/tasks/tse.md) | | Classification / SER / GR | Accuracy | [Classification](docs/tasks/classification.md) | diff --git a/docs/atlas/index.html b/docs/atlas/index.html index 5d4f6285..b44a227a 100644 --- a/docs/atlas/index.html +++ b/docs/atlas/index.html @@ -102,18 +102,18 @@

SURE Pipeline Atlas

- -
FRONTENDTRANSCRIPTIONNORMALIZATIONSCORINGREPORTASR68TTS30SE21TSE16VC16RIR estimation15S2TT10TTA9DOA8VAD5KWS3SA-ASR2SLU2SV2GR1LID1SD1SER1classification1funasr_loader_16k…×4paraformer_zh×4whisper_large_v3×4qwen3_asr_1_7b×2cohere_transcribe…×1xnorm×20giga_norm×17etc_canonicalizat…×12funasr_itn×10whisper_norm×7aispeech_norm×3canonical_itn×3nemo_norm×2wetext_norm×1punctuation_strip…×5doa_jsonl×8vad_timebase×5gstar_norm×1prompt_norm×2lid_label×1wenet_wer×52wenet_cer×23token_mer×2token_cer×1wenet_mer×1dnsmos×9utmos×9wv_mos×9ecapa_tdnn_sim×8eres2net_sim×8wavlm_large_sim×8komos×1si_sdr×3dnsmos_bak×1dnsmos_p808×1dnsmos_sig×1estoi×1lps×1lsd×1nisqa_col×1nisqa_dis×1nisqa_loud×1nisqa_mos×1nisqa_noi×1nsrmr×1pesq×1squim_pesq×1squim_si_sdr×1squim_stoi×1stoi×1rir_c50_mae×1rir_c80_mae×1rir_doa_mae×1rir_drr_mae×1rir_edc_mae×1rir_iacc_early_mae×1rir_iacc_full_mae×1rir_iacc_late_mae×1rir_iacc_mae×1rir_ild_mae×1rir_ipd_mae×1rir_itd_mae×1rir_npm×1rt30_mae×1rt60_mae×1sacrebleu×6bleurt_20×2xcomet_xl×2audiobox_aestheti…×4audioldm_eval×4clap_score×1doa_localization×8vad_detection_dur…×4vad_auc_roc×1wekws_det×3meeteval×3classify×6cosine_trial_scor…×2det_eer×1min_dcf_p005×1212 pipelinesreport.json
+ +
FRONTENDTRANSCRIPTIONNORMALIZATIONSCORINGREPORTASR68TTS30SE21RIR estimation20TSE16VC16S2TT10TTA9DOA8VAD5KWS3SA-ASR2SLU2SV2GR1LID1SD1SER1classification1funasr_loader_16k…×4paraformer_zh×4whisper_large_v3×4qwen3_asr_1_7b×2cohere_transcribe…×1xnorm×20giga_norm×17etc_canonicalizat…×12funasr_itn×10whisper_norm×7aispeech_norm×3canonical_itn×3nemo_norm×2wetext_norm×1punctuation_strip…×5doa_jsonl×8vad_timebase×5gstar_norm×1prompt_norm×2lid_label×1wenet_wer×52wenet_cer×23token_mer×2token_cer×1wenet_mer×1dnsmos×9utmos×9wv_mos×9ecapa_tdnn_sim×8eres2net_sim×8wavlm_large_sim×8komos×1si_sdr×3dnsmos_bak×1dnsmos_p808×1dnsmos_sig×1estoi×1lps×1lsd×1nisqa_col×1nisqa_dis×1nisqa_loud×1nisqa_mos×1nisqa_noi×1nsrmr×1pesq×1squim_pesq×1squim_si_sdr×1squim_stoi×1stoi×1rir_c50_mae×1rir_c80_jnd_mae×1rir_c80_mae×1rir_d50_jnd_mae×1rir_doa_mae×1rir_drr_mae×1rir_edc_mae×1rir_edt_broadband…×1rir_edt_jnd_mae×1rir_edt_m_mae×1rir_iacc_early_mae×1rir_iacc_full_mae×1rir_iacc_late_mae×1rir_ild_mae×1rir_ipd_mae×1rir_itd_mae×1rir_npm×1rir_ts_jnd_mae×1rt30_mae×1rt60_mae×1sacrebleu×6bleurt_20×2xcomet_xl×2audiobox_aestheti…×4audioldm_eval×4clap_score×1doa_localization×8vad_detection_dur…×4vad_auc_roc×1wekws_det×3meeteval×3classify×6cosine_trial_scor…×2det_eer×1min_dcf_p005×1217 pipelinesreport.json
-