[tn] refine date and time (#65)

* [tn] refine date and time

* [tn] refine measure

* [tn] fix lint

Author: pengzhendong

tn/chinese/normalizer.py

@@ -83,10 +83,6 @@ def build_tagger(self):
         math = add_weight(Math().tagger, 1.08)
         char = add_weight(Char().tagger, 100)
 
-        to = (delete('-') | delete('~')) + insert(' char { value: "到" } ')
-        date = date + (to + date).ques
-        time = time + (to + time).ques
-
         tagger = (date | whitelist | sport | fraction | measure | money | time
                   | cardinal | math | char).optimize()
         tagger = (processor @ tagger).star

tn/chinese/rules/cardinal.py

@@ -23,6 +23,7 @@ class Cardinal(Processor):
     def __init__(self):
         super().__init__('cardinal')
         self.number = None
+        self.digits = None
         self.build_tagger()
         self.build_verbalizer()
 
@@ -36,6 +37,7 @@ def build_tagger(self):
         rmzero = delete('0')
         rmpunct = delete(',').ques
         digits = zero | digit
+        self.digits = digits
 
         # 11 => 十一
         ten = teen + insert('十') + (digit | rmzero)
@@ -69,10 +71,6 @@ def build_tagger(self):
         cardinal = digits.plus + (digits | dot).plus.ques + digits.plus
         # xxxx-xxx-xxx
         cardinal |= digits.plus + (delete('-') + digits.plus).closure(2)
-        # -xxxx年, -xx年, ~xxxx年, ~xx年
-        unit = accep('年') | accep('赛季')
-        rmsign = (delete('-') | delete('~')) + insert('到')
-        cardinal |= rmsign + (digits**2 | digits**4) + unit
 
         tagger = insert('value: "') + cardinal + insert('"')
         self.tagger = self.add_tokens(tagger)

tn/chinese/rules/date.py

@@ -48,7 +48,10 @@ def build_tagger(self):
                 | (year + rmsign + mm)
                 | (mm + rmsign + year)
                 | (mm + rmsign + day))
-        self.tagger = self.add_tokens(date)
+        tagger = self.add_tokens(date)
+
+        to = (delete('-') | delete('~')) + insert(' char { value: "到" } ')
+        self.tagger = tagger + (to + tagger).ques
 
     def build_verbalizer(self):
         year = delete('year: "') + self.SIGMA + delete('" ')

tn/chinese/rules/measure.py

@@ -31,17 +31,31 @@ def build_tagger(self):
         units_zh = string_file('tn/chinese/data/measure/units_zh.tsv')
         units = units_en | units_zh
         rmspace = delete(' ').ques
+        to = cross('-', '到') | cross('~', '到') | accep('到')
 
         number = Cardinal().number
         percent = insert('百分之') + number + delete('%')
 
         number @= self.build_rule(cross('二', '两'), '[BOS]', '[EOS]')
         # 1-11个，1个-11个
-        prefix = (number + (rmspace + units).ques +
-                  (cross('-', '到') | accep('到')))
+        prefix = number + (rmspace + units).ques + to
         measure = prefix.ques + number + rmspace + units
-        measure @= self.build_rule(cross('两两', '二两'), '[BOS]', '')
-        tagger = insert('value: "') + (measure | percent) + insert('"')
+
+        for unit in ['两', '月', '号']:
+            measure @= self.build_rule(cross('两' + unit, '二' + unit),
+                                       l='[BOS]')
+            measure @= self.build_rule(cross('到两' + unit, '到二' + unit),
+                                       r='[EOS]')
+
+        # -xxxx年, -xx年
+        digits = Cardinal().digits
+        cardinal = digits**2 | digits**4
+        unit = accep('年') | accep('年度') | accep('赛季')
+        prefix = cardinal + (rmspace + unit).ques + to
+        annual = prefix.ques + cardinal + unit
+
+        tagger = insert('value: "') + (measure | percent
+                                       | annual) + insert('"')
 
         # 10km/h
         rmsign = rmspace + delete('/') + rmspace

tn/chinese/rules/sport.py

@@ -15,7 +15,7 @@
 from tn.chinese.rules.cardinal import Cardinal
 from tn.processor import Processor
 
-from pynini import string_file
+from pynini import cross, string_file
 from pynini.lib.pynutil import delete, insert
 
 

tn/chinese/rules/time.py

@@ -36,7 +36,10 @@ def build_tagger(self):
                   (delete(':') + insert(' second: "') + s + insert('"')).ques +
                   delete(' ').ques +
                   (insert(' noon: "') + noon + insert('"')).ques)
-        self.tagger = self.add_tokens(tagger)
+        tagger = self.add_tokens(tagger)
+
+        to = (delete('-') | delete('~')) + insert(' char { value: "到" } ')
+        self.tagger = tagger + (to + tagger).ques
 
     def build_verbalizer(self):
         noon = delete('noon: "') + self.SIGMA + delete('" ')