[tn] english tn, support ordinal (#204)

Author: xingchensong

tn/english/data/ordinal/__init__.py

@@ -0,0 +1,9 @@
+first	one
+second	two
+third	three
+fourth	four
+fifth	five
+sixth	sixth
+seventh	seven
+eighth	eight
+ninth	nine

tn/english/data/ordinal/digit.tsv

@@ -0,0 +1 @@
+twelfth	twelve

tn/english/data/ordinal/teen.tsv

@@ -15,6 +15,7 @@
 
 from tn.processor import Processor
 from tn.english.rules.cardinal import Cardinal
+from tn.english.rules.ordinal import Ordinal
 from tn.english.rules.word import Word
 
 from pynini.lib.pynutil import add_weight, delete
@@ -23,23 +24,23 @@
 
 class Normalizer(Processor):
 
-    def __init__(self,
-                 cache_dir=None,
-                 overwrite_cache=False):
+    def __init__(self, cache_dir=None, overwrite_cache=False):
         super().__init__(name='en_normalizer')
         if cache_dir is None:
             cache_dir = files("tn")
         self.build_fst('en_tn', cache_dir, overwrite_cache)
 
     def build_tagger(self):
-        cardinal = add_weight(Cardinal().tagger, 1.06)
+        cardinal = add_weight(Cardinal().tagger, 1.0)
+        ordinal = add_weight(Ordinal().tagger, 1.0)
         word = add_weight(Word().tagger, 100)
-        tagger = (cardinal | word).optimize() + self.DELETE_SPACE
+        tagger = (cardinal | ordinal | word).optimize() + self.DELETE_SPACE
         # delete the last space
         self.tagger = tagger.star @ self.build_rule(delete(' '), r='[EOS]')
 
     def build_verbalizer(self):
         cardinal = Cardinal().verbalizer
+        ordinal = Ordinal().verbalizer
         word = Word().verbalizer
-        verbalizer = (cardinal | word).optimize() + self.INSERT_SPACE
+        verbalizer = (cardinal | ordinal | word).optimize() + self.INSERT_SPACE
         self.verbalizer = verbalizer.star

tn/english/normalizer.py

@@ -0,0 +1,96 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2024, WENET COMMUNITY.  Xingchen Song (sxc19@tsinghua.org.cn).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pynini
+from pynini.lib import pynutil
+
+from tn.processor import Processor
+from tn.utils import get_abs_path
+from tn.english.rules.cardinal import Cardinal
+
+
+class Ordinal(Processor):
+
+    def __init__(self, deterministic: bool = False):
+        """
+        Args:
+            deterministic: if True will provide a single transduction option,
+                for False multiple transduction are generated (used for audio-based normalization)
+        """
+        super().__init__("ordinal", ordertype="tn")
+        self.deterministic = deterministic
+        self.build_tagger()
+        self.build_verbalizer()
+
+    def build_tagger(self):
+        """
+        Finite state transducer for classifying ordinal, e.g.
+            13th -> ordinal { integer: "thirteen" }
+        """
+        cardinal = Cardinal(self.deterministic)
+        cardinal_graph = cardinal.graph
+        cardinal_format = pynini.closure(self.DIGIT | pynini.accep(","))
+        st_format = (pynini.closure(cardinal_format +
+                                    (self.DIGIT - "1"), 0, 1) +
+                     pynini.accep("1") +
+                     pynutil.delete(pynini.union("st", "ST", "ˢᵗ")))
+        nd_format = (pynini.closure(cardinal_format +
+                                    (self.DIGIT - "1"), 0, 1) +
+                     pynini.accep("2") +
+                     pynutil.delete(pynini.union("nd", "ND", "ⁿᵈ")))
+        rd_format = (pynini.closure(cardinal_format +
+                                    (self.DIGIT - "1"), 0, 1) +
+                     pynini.accep("3") +
+                     pynutil.delete(pynini.union("rd", "RD", "ʳᵈ")))
+        th_format = pynini.closure(
+            (self.DIGIT - "1" - "2" - "3")
+            | (cardinal_format + "1" + self.DIGIT)
+            | (cardinal_format + (self.DIGIT - "1") +
+               (self.DIGIT - "1" - "2" - "3")),
+            1,
+        ) + pynutil.delete(pynini.union("th", "TH", "ᵗʰ"))
+        self.graph = (st_format | nd_format | rd_format
+                      | th_format) @ cardinal_graph
+        final_graph = pynutil.insert(
+            "integer: \"") + self.graph + pynutil.insert("\"")
+        final_graph = self.add_tokens(final_graph)
+        self.tagger = final_graph.optimize()
+
+    def build_verbalizer(self):
+        """
+        Finite state transducer for verbalizing ordinal, e.g.
+            ordinal { integer: "thirteen" } } -> thirteenth
+        """
+        graph_digit = pynini.string_file(
+            get_abs_path("english/data/ordinal/digit.tsv")).invert()
+        graph_teens = pynini.string_file(
+            get_abs_path("english/data/ordinal/teen.tsv")).invert()
+
+        graph = (pynutil.delete("integer:") + self.DELETE_SPACE +
+                 pynutil.delete("\"") + pynini.closure(self.NOT_QUOTE, 1) +
+                 pynutil.delete("\""))
+        convert_rest = pynutil.insert("th")
+
+        suffix = pynini.cdrewrite(
+            graph_digit | graph_teens | pynini.cross("ty", "tieth")
+            | convert_rest,
+            "",
+            "[EOS]",
+            pynini.closure(self.VCHAR),
+        ).optimize()
+        self.graph = pynini.compose(graph, suffix)
+        self.suffix = suffix
+        delete_tokens = self.delete_tokens(self.graph)
+        self.verbalizer = delete_tokens.optimize()

tn/english/rules/ordinal.py

@@ -0,0 +1,11 @@
+1st => first
+2nd => second
+3rd => third
+5th => fifth
+11th => eleventh
+13th => thirteenth
+20th => twentieth
+21st => twenty first
+30th => thirtieth
+100th => one hundredth
+1000th => one thousandth

tn/english/test/data/ordinal.txt

@@ -0,0 +1,28 @@
+# Copyright (c) 2024 Xingchen Song (sxc19@tsinghua.org.cn)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+
+from tn.english.rules.ordinal import Ordinal
+from tn.english.test.utils import parse_test_case
+
+
+class TestOrdinal:
+
+    ordinal = Ordinal(deterministic=False)
+    ordinal_cases = parse_test_case('data/ordinal.txt')
+
+    @pytest.mark.parametrize("written, spoken", ordinal_cases)
+    def test_ordinal(self, written, spoken):
+        assert self.ordinal.normalize(written) == spoken