[fix] english tn, fix crash on double quote "" (#224)

xingchensong · web-flow · commit a7d4529f9b15 · 2024-06-04T15:05:05.000+08:00
diff --git a/runtime/processor/wetext_token_parser.cc b/runtime/processor/wetext_token_parser.cc
@@ -116,9 +116,10 @@ std::string TokenParser::ParseValue() {
   std::string value = "";
   while (ch_ != "\"") {
     value += ch_;
-    escape = ch_ == "\\" && !escape;
+    escape = ch_ == "\\";
     Read();
     if (escape) {
+      escape = false;
       value += ch_;
       Read();
     }
diff --git a/tn/english/data/whitelist/tts.tsv b/tn/english/data/whitelist/tts.tsv
@@ -18,8 +18,10 @@ bldg.	building
 Bldg.	Building
 apt.	apartment
 Apt.	Apartment
-World War I	World War one
+World War II.	World War two
 World War II	World War two
+World War I.	World War one
+World War I	World War one
 etc.	etcetera.
 SnO2	tin four oxide
 dept	department
diff --git a/tn/english/rules/date.py b/tn/english/rules/date.py
@@ -223,14 +223,15 @@ def build_tagger(self):
             cardinal_graph=cardinal_graph,
             single_digits_graph=cardinal.single_digits_graph)
         two_digit_year = pynutil.insert(
-            "year: \"") + two_digit_year + self.PUNCT.ques + pynutil.insert(
-                "\"")
+            "year: \"") + two_digit_year + pynini.union(
+                ",", ".").ques + pynutil.insert("\"")
 
-        graph_year = pynutil.insert(" year: \"") + pynutil.delete(
-            " ") + year_graph + self.PUNCT.ques + pynutil.insert("\"")
+        graph_year = pynutil.insert(
+            " year: \"") + pynutil.delete(" ") + year_graph + pynini.union(
+                ",", ".").ques + pynutil.insert("\"")
         graph_year |= (pynutil.insert(" year: \"") + pynini.accep(",") +
                        pynini.closure(pynini.accep(" "), 0, 1) + year_graph +
-                       self.PUNCT.ques + pynutil.insert("\""))
+                       pynini.union(",", ".").ques + pynutil.insert("\""))
         optional_graph_year = pynini.closure(graph_year, 0, 1)
 
         year_graph = pynutil.insert("year: \"") + year_graph + pynutil.insert(
@@ -281,6 +282,12 @@ def build_tagger(self):
 
         final_graph |= graph_fy
 
+        prefix = pynutil.delete(pynini.union("{", "(", "<", "\"",
+                                             "'")).ques + self.DELETE_SPACE
+        suffix = self.DELETE_SPACE + pynutil.delete(
+            pynini.union("}", ")", ">", "\"", "'")).ques
+        final_graph = pynutil.add_weight(
+            prefix, -0.1) + final_graph + pynutil.add_weight(suffix, -0.1)
         self.tagger = self.add_tokens(final_graph)
 
     def build_verbalizer(self):
@@ -309,9 +316,9 @@ def build_verbalizer(self):
         graph_fy = (pynutil.insert("the ") + period + pynutil.insert(" of") +
                     pynini.closure(self.DELETE_EXTRA_SPACE + year, 0, 1))
 
-        # day month year
-        graph_dmy = (pynutil.insert("the ") + day + self.DELETE_EXTRA_SPACE +
-                     pynutil.insert("of ") + month +
+        # day month year, month year
+        graph_dmy = ((pynutil.insert("the ") + day + self.DELETE_EXTRA_SPACE +
+                      pynutil.insert("of ")).ques + month +
                      pynini.closure(self.DELETE_EXTRA_SPACE + year, 0, 1))
 
         final_graph = ((graph_dmy | year | graph_fy) + self.DELETE_SPACE)
diff --git a/tn/english/rules/whitelist.py b/tn/english/rules/whitelist.py
@@ -21,7 +21,6 @@
 
 from tn.processor import Processor
 from tn.utils import get_abs_path, load_labels, augment_labels_with_punct_at_end
-from tn.english.rules.measure import SINGULAR_TO_PLURAL
 from tn.english.rules.roman import get_names
 
 
@@ -93,21 +92,6 @@ def _get_whitelist_graph(input_case,
                       pynini.closure(pynutil.delete(x) + self.UPPER, 2) +
                       pynini.closure(pynutil.delete("."), 0, 1))
 
-        if not self.deterministic:
-            multiple_forms_whitelist_graph = get_formats(
-                get_abs_path(
-                    "english/data/whitelist/alternatives_all_format.tsv"))
-            graph |= multiple_forms_whitelist_graph
-
-            graph_unit = pynini.string_file(
-                get_abs_path("english/data/measure/unit.tsv")
-            ) | pynini.string_file(
-                get_abs_path("english/data/measure/unit_alternatives.tsv"))
-            graph_unit_plural = graph_unit @ SINGULAR_TO_PLURAL
-            units_graph = pynini.compose(self.VCHAR**(3, ...),
-                                         graph_unit | graph_unit_plural)
-            graph |= units_graph
-
         # convert to states only if comma is present before the abbreviation to avoid converting all caps words,
         # e.g. "IN", "OH", "OK"
         # TODO or only exclude above?
@@ -133,7 +117,7 @@ def _get_whitelist_graph(input_case,
 
     def build_verbalizer(self):
         graph = (pynutil.delete("name:") + self.DELETE_SPACE +
-                 pynutil.delete("\"") + pynini.closure(self.VCHAR - " ", 1) +
+                 pynutil.delete("\"") + pynini.closure(self.NOT_QUOTE, 1) +
                  pynutil.delete("\""))
         final_graph = graph.optimize()
         self.verbalizer = self.delete_tokens(final_graph)
diff --git a/tn/english/test/data/normalizer.txt b/tn/english/test/data/normalizer.txt
@@ -1 +1,2 @@
-this is 12th game, number 256, 2024-05-06, 2021-03-07 31.990 billion. ¾ people like chattts, let's eat at 03:43 p.m. run 10 km, give me $12.345 please, call 123-123-5678-1 Mt Hill "HAHAHA" billion 4 March => this is twelfth game, number 256, the sixth of may twenty twenty four , the seventh of march twenty twenty one thirty one point nine nine oh billion. three quarters people like chattts, let's eat at three forty three PM run ten kilometers, give me twelve point three four five dollars please, call one two three, one two three, five six seven eight, one Mount Hill "HAHAHA" billion the fourth of march
+this is 12th game, number 256, 2024-05-06, 2021-03-07 31.990 billion. ¾ people like chattts, let's eat at 03:43 p.m. run 10 km, give me $12.345 please, call 123-123-5678-1 Mt Hill "HAHAHA" billion 4 March => this is twelfth game, number 256, the sixth of may twenty twenty four , the seventh of march twenty twenty one thirty one point nine nine oh billion. three quarters people like chattts, let's eat at three forty three PM run ten kilometers, give me twelve point three four five dollars please, call one two three, one two three, five six seven eight, one Mt Hill "HAHAHA" billion the fourth of march
+The National Map, accessed April 1, 2011" Site Description of Koppers Co. From the quartet's recording" Jefferson Friedman: Quartets,"" String Quartet no, Riots again broke out, Atassi resigned, and Syrian independence was deferred until after World War II. 1988 (1988) ( 1988) ( 1988). Starling, Arthur E.( 1988 ). this is 12th game, number 256, 2024-05-06, 2021-03-07 31.990 billion. 3/4 people like chattts Retrieved December 2011. Information on Album" Thepodule.com"" Biography by Amy Hanson". => The National Map, accessed the first of april , twenty eleven Site Description of Koppers company From the quartet's recording" Jefferson Friedman: Quartets,"" String Quartet no, Riots again broke out, Atassi resigned, and Syrian independence was deferred until after World War two nineteen eighty eight nineteen eighty eight nineteen eighty eight nineteen eighty eight ). Starling, Arthur E.( nineteen eighty eight ). this is twelfth game, number 256, the sixth of may twenty twenty four , the seventh of march twenty twenty one thirty one point nine nine oh billion. three quarters people like chattts Retrieved december twenty eleven. Information on Album" Thepodule.com"" Biography by Amy Hanson".
diff --git a/tn/token_parser.py b/tn/token_parser.py
@@ -122,9 +122,10 @@ def parse_value(self):
         value = ''
         while self.char != '"':
             value += self.char
-            escape = self.char == '\\' and not escape
+            escape = self.char == '\\'
             self.read()
             if escape:
+                escape = False
                 value += self.char
                 self.read()
         return value

Original file line number	Diff line number	Diff line change
`@@ -1 +1,2 @@`
`1`		-this is 12th game, number 256, 2024-05-06, 2021-03-07 31.990 billion. ¾ people like chattts, let's eat at 03:43 p.m. run 10 km, give me $12.345 please, call 123-123-5678-1 Mt Hill "HAHAHA" billion 4 March => this is twelfth game, number 256, the sixth of may twenty twenty four , the seventh of march twenty twenty one thirty one point nine nine oh billion. three quarters people like chattts, let's eat at three forty three PM run ten kilometers, give me twelve point three four five dollars please, call one two three, one two three, five six seven eight, one Mount Hill "HAHAHA" billion the fourth of march
	`1`	+this is 12th game, number 256, 2024-05-06, 2021-03-07 31.990 billion. ¾ people like chattts, let's eat at 03:43 p.m. run 10 km, give me $12.345 please, call 123-123-5678-1 Mt Hill "HAHAHA" billion 4 March => this is twelfth game, number 256, the sixth of may twenty twenty four , the seventh of march twenty twenty one thirty one point nine nine oh billion. three quarters people like chattts, let's eat at three forty three PM run ten kilometers, give me twelve point three four five dollars please, call one two three, one two three, five six seven eight, one Mt Hill "HAHAHA" billion the fourth of march
	`2`	+The National Map, accessed April 1, 2011" Site Description of Koppers Co. From the quartet's recording" Jefferson Friedman: Quartets,"" String Quartet no, Riots again broke out, Atassi resigned, and Syrian independence was deferred until after World War II. 1988 (1988) ( 1988) ( 1988). Starling, Arthur E.( 1988 ). this is 12th game, number 256, 2024-05-06, 2021-03-07 31.990 billion. 3/4 people like chattts Retrieved December 2011. Information on Album" Thepodule.com"" Biography by Amy Hanson". => The National Map, accessed the first of april , twenty eleven Site Description of Koppers company From the quartet's recording" Jefferson Friedman: Quartets,"" String Quartet no, Riots again broke out, Atassi resigned, and Syrian independence was deferred until after World War two nineteen eighty eight nineteen eighty eight nineteen eighty eight nineteen eighty eight ). Starling, Arthur E.( nineteen eighty eight ). this is twelfth game, number 256, the sixth of may twenty twenty four , the seventh of march twenty twenty one thirty one point nine nine oh billion. three quarters people like chattts Retrieved december twenty eleven. Information on Album" Thepodule.com"" Biography by Amy Hanson".