fix(tn&itn): fix 语气词 (#104)

* fix(tn&itn): fix a

* fix(tn&itn): fix a

* fix(tn&itn): add unittest

Author: xingchensong

itn/chinese/inverse_normalizer.py

@@ -22,7 +22,7 @@
 from itn.chinese.rules.money import Money
 from itn.chinese.rules.whitelist import Whitelist
 from itn.chinese.rules.time import Time
-from itn.chinese.rules.preprocessor import PreProcessor
+from itn.chinese.rules.postprocessor import PostProcessor
 
 from pynini.lib.pynutil import add_weight, delete
 from importlib_resources import files
@@ -45,14 +45,13 @@ def build_tagger(self):
                   | add_weight(Whitelist().tagger, 1.01)
                   | add_weight(Fraction().tagger, 1.05)
                   | add_weight(Measure().tagger, 1.05)
-                  | add_weight(Money().tagger, 1.05)
+                  | add_weight(Money().tagger, 1.04)
                   | add_weight(Time().tagger, 1.05)
                   | add_weight(Cardinal(self.convert_number, self.enable_0_to_9).tagger, 1.06)
                   | add_weight(Math().tagger, 1.10)
                   | add_weight(Char().tagger, 100)).optimize()
 
-        processor = PreProcessor(remove_interjections=True).processor
-        tagger = (processor @ tagger).star
+        tagger = tagger.star
         # remove the last space
         self.tagger = tagger @ self.build_rule(delete(' '), '', '[EOS]')
 
@@ -65,6 +64,7 @@ def build_verbalizer(self):
                       | Measure().verbalizer
                       | Money().verbalizer
                       | Time().verbalizer
-                      | Whitelist().verbalizer).optimize().star
+                      | Whitelist().verbalizer).optimize()
+        postprocessor = PostProcessor(remove_interjections=True).processor
 
-        self.verbalizer = verbalizer
+        self.verbalizer = (verbalizer @ postprocessor).star

itn/chinese/rules/measure.py

@@ -31,8 +31,8 @@ def build_tagger(self):
         units_en = string_file('itn/chinese/data/measure/units_en.tsv')
         units_zh = string_file('itn/chinese/data/measure/units_zh.tsv')
         sign = string_file('itn/chinese/data/number/sign.tsv')    # + -
-        units = units_en | ((accep('亿') | accep('兆') | accep('万')).ques
-                            + units_zh)
+        units = add_weight(units_en, -1.0) | \
+            ((accep('亿') | accep('兆') | accep('万')).ques + units_zh)
 
         number = Cardinal().number
         # 百分之三十, 百分三十, 百分之百

itn/chinese/rules/preprocessor.py itn/chinese/rules/postprocessor.pyitn/chinese/rules/preprocessor.py renamed to itn/chinese/rules/postprocessor.py

@@ -1,4 +1,5 @@
 # Copyright (c) 2022 Zhendong Peng (pzd17@tsinghua.org.cn)
+# Copyright (c) 2023 Xingchen Song (sxc19@mails.tsinghua.edu.cn)
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,10 +19,10 @@
 from pynini.lib.pynutil import delete
 
 
-class PreProcessor(Processor):
+class PostProcessor(Processor):
 
     def __init__(self, remove_interjections=True):
-        super().__init__(name='preprocessor')
+        super().__init__(name='postprocessor')
         blacklist = string_file('itn/chinese/data/default/blacklist.tsv')
 
         processor = self.VSIGMA

itn/chinese/test/data/measure.txt

@@ -5,10 +5,13 @@
 一千克 => 1kg
 百分之二 => 2%
 百分三十 => 30%
+百分之三十 => 30%
 百分百 => 100%
 百分之百 => 100%
 百分之二点一五 => 2.15%
 每小时十公里 => 10km/h
+每小时十千米 => 10km/h
+十千米每小时 => 10km/h
 三百二十四点七五克 => 324.75g
 二十点一万 => 20.1万
 一百块钱 => 100块钱

itn/chinese/test/data/normalizer.txt

@@ -1,5 +1,6 @@
 一共有多少人 => 一共有多少人
 呃这个呃啊我不知道 => 这个我不知道
+呃呃啊 => 
 共四百六十五篇,约三百一十五万字 => 共465篇,约315万字
 共计六点四二万人 => 共计6.42万人
 同比升高零点六个百分点 => 同比升高0.6个百分点

runtime/processor/processor.cc

@@ -54,6 +54,9 @@ std::string Processor::Compose(const std::string& input,
 }
 
 std::string Processor::Tag(const std::string& input) {
+  if (input.empty()) {
+    return "";
+  }
   return Compose(input, tagger_.get());
 }
 

tn/chinese/normalizer.py

@@ -26,7 +26,7 @@
 from tn.chinese.rules.time import Time
 from tn.chinese.rules.whitelist import Whitelist
 
-from pynini.lib.pynutil import add_weight, delete, insert
+from pynini.lib.pynutil import add_weight, delete
 from importlib_resources import files
 
 
@@ -52,7 +52,6 @@ def __init__(self,
 
     def build_tagger(self):
         processor = PreProcessor(
-            remove_interjections=self.remove_interjections,
             traditional_to_simple=self.traditional_to_simple).processor
 
         date = add_weight(Date().tagger, 1.02)
@@ -87,7 +86,9 @@ def build_verbalizer(self):
         verbalizer = (cardinal | char | date | fraction | math | measure
                       | money | sport | time | whitelist).optimize()
 
-        processor = PostProcessor(remove_puncts=self.remove_puncts,
-                                  full_to_half=self.full_to_half,
-                                  tag_oov=self.tag_oov).processor
+        processor = PostProcessor(
+            remove_interjections=self.remove_interjections,
+            remove_puncts=self.remove_puncts,
+            full_to_half=self.full_to_half,
+            tag_oov=self.tag_oov).processor
         self.verbalizer = (verbalizer @ processor).star

tn/chinese/rules/postprocessor.py

@@ -21,8 +21,10 @@
 
 class PostProcessor(Processor):
 
-    def __init__(self, remove_puncts=False, full_to_half=True, tag_oov=False):
+    def __init__(self, remove_interjections=True, remove_puncts=False,
+                 full_to_half=True, tag_oov=False):
         super().__init__(name='postprocessor')
+        blacklist = string_file('tn/chinese/data/default/blacklist.tsv')
         puncts = string_file('tn/chinese/data/char/punctuations_zh.tsv')
         full2half = string_file(
             'tn/chinese/data/char/fullwidth_to_halfwidth.tsv')
@@ -32,6 +34,9 @@ def __init__(self, remove_puncts=False, full_to_half=True, tag_oov=False):
             'tn/chinese/data/char/charset_extension.tsv')
 
         processor = self.build_rule('')
+        if remove_interjections:
+            processor @= self.build_rule(delete(blacklist))
+
         if remove_puncts:
             processor @= self.build_rule(delete(puncts | self.PUNCT))
 

tn/chinese/rules/preprocessor.py

@@ -15,22 +15,17 @@
 from tn.processor import Processor
 
 from pynini import string_file
-from pynini.lib.pynutil import delete
 
 
 class PreProcessor(Processor):
 
-    def __init__(self, remove_interjections=True, traditional_to_simple=True):
+    def __init__(self, traditional_to_simple=True):
         super().__init__(name='preprocessor')
-        blacklist = string_file('tn/chinese/data/default/blacklist.tsv')
         traditional2simple = string_file(
             'tn/chinese/data/char/traditional_to_simple.tsv')
 
         processor = self.build_rule('')
         if traditional_to_simple:
             processor @= self.build_rule(traditional2simple)
 
-        if remove_interjections:
-            processor @= self.build_rule(delete(blacklist))
-
         self.processor = processor.optimize()

tn/chinese/test/data/postprocessor.txt

@@ -1,3 +1,5 @@
 好！ => 好!
+好啊 => 好
+啊呃呃 => 
 我们안녕 => 我们<oov>안</oov><oov>녕</oov>
 雪の花 => 雪<oov>の</oov>花