Skip to content

Commit c26d489

Browse files
xingchensongxingchensong
authored
[tn] english tn, support decimal (#207)
1 parent 241b0b0 commit c26d489

6 files changed

Lines changed: 245 additions & 12 deletions

File tree

‎tn/english/normalizer.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
from tn.processor import Processor
1717
from tn.english.rules.cardinal import Cardinal
1818
from tn.english.rules.ordinal import Ordinal
19+
from tn.english.rules.decimal import Decimal
1920
from tn.english.rules.word import Word
2021
from tn.english.rules.date import Date
2122

@@ -34,18 +35,20 @@ def __init__(self, cache_dir=None, overwrite_cache=False):
3435
def build_tagger(self):
3536
cardinal = add_weight(Cardinal().tagger, 1.0)
3637
ordinal = add_weight(Ordinal().tagger, 1.0)
38+
decimal = add_weight(Decimal().tagger, 1.0)
3739
date = add_weight(Date().tagger, 0.99)
3840
word = add_weight(Word().tagger, 100)
3941
tagger = (cardinal | ordinal | word
40-
| date).optimize() + self.DELETE_SPACE
42+
| date | decimal).optimize() + self.DELETE_SPACE
4143
# delete the last space
4244
self.tagger = tagger.star @ self.build_rule(delete(' '), r='[EOS]')
4345

4446
def build_verbalizer(self):
4547
cardinal = Cardinal().verbalizer
4648
ordinal = Ordinal().verbalizer
49+
decimal = Decimal().verbalizer
4750
word = Word().verbalizer
4851
date = Date().verbalizer
4952
verbalizer = (cardinal | ordinal | word
50-
| date).optimize() + self.INSERT_SPACE
53+
| date | decimal).optimize() + self.INSERT_SPACE
5154
self.verbalizer = verbalizer.star

‎tn/english/rules/cardinal.py‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -172,11 +172,15 @@ def build_verbalizer(self):
172172
optional_sign |= pynini.cross("negative: \"true\"", "negative ")
173173
optional_sign |= pynini.cross("negative: \"true\"", "dash ")
174174

175-
optional_sign = pynini.closure(optional_sign + self.DELETE_SPACE, 0, 1)
175+
self.optional_sign = pynini.closure(optional_sign + self.DELETE_SPACE,
176+
0, 1)
176177

177-
integer = pynutil.delete("integer:") + self.DELETE_SPACE + \
178-
pynutil.delete("\"") + pynini.closure(self.NOT_QUOTE) + pynutil.delete("\"")
178+
integer = pynini.closure(self.NOT_QUOTE)
179179

180-
numbers = optional_sign + integer
181-
delete_tokens = self.delete_tokens(numbers)
180+
self.integer = self.DELETE_SPACE + pynutil.delete(
181+
"\"") + integer + pynutil.delete("\"")
182+
integer = pynutil.delete("integer:") + self.integer
183+
184+
self.numbers = self.optional_sign + integer
185+
delete_tokens = self.delete_tokens(self.numbers)
182186
self.verbalizer = delete_tokens.optimize()

‎tn/english/rules/decimal.py‎

Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import pynini
16+
from pynini.lib import pynutil
17+
18+
from tn.processor import Processor
19+
from tn.utils import get_abs_path
20+
from tn.english.rules.cardinal import Cardinal
21+
22+
delete_space = pynutil.delete(" ")
23+
quantities = pynini.string_file(
24+
get_abs_path("english/data/number/thousand.tsv"))
25+
quantities_abbr = pynini.string_file(
26+
get_abs_path("english/data/number/quantity_abbr.tsv"))
27+
quantities_abbr |= Processor("tmp").TO_UPPER @ quantities_abbr
28+
29+
30+
def get_quantity(decimal: 'pynini.FstLike',
31+
cardinal_up_to_hundred: 'pynini.FstLike',
32+
include_abbr: bool) -> 'pynini.FstLike':
33+
"""
34+
Returns FST that transforms either a cardinal or decimal followed by a quantity into a numeral,
35+
e.g. 1 million -> integer_part: "one" quantity: "million"
36+
e.g. 1.5 million -> integer_part: "one" fractional_part: "five" quantity: "million"
37+
38+
Args:
39+
decimal: decimal FST
40+
cardinal_up_to_hundred: cardinal FST
41+
"""
42+
quantity_wo_thousand = pynini.project(quantities, "input") - pynini.union(
43+
"k", "K", "thousand")
44+
if include_abbr:
45+
quantity_wo_thousand |= pynini.project(
46+
quantities_abbr, "input") - pynini.union("k", "K", "thousand")
47+
res = (pynutil.insert("integer_part: \"") + cardinal_up_to_hundred +
48+
pynutil.insert("\"") + pynini.closure(pynutil.delete(" "), 0, 1) +
49+
pynutil.insert(" quantity: \"") +
50+
(quantity_wo_thousand @ (quantities | quantities_abbr)) +
51+
pynutil.insert("\""))
52+
if include_abbr:
53+
quantity = quantities | quantities_abbr
54+
else:
55+
quantity = quantities
56+
res |= (decimal + pynini.closure(pynutil.delete(" "), 0, 1) +
57+
pynutil.insert(" quantity: \"") + quantity + pynutil.insert("\""))
58+
return res
59+
60+
61+
class Decimal(Processor):
62+
63+
def __init__(self, deterministic: bool = False):
64+
"""
65+
Args:
66+
deterministic: if True will provide a single transduction option,
67+
for False multiple transduction are generated (used for audio-based normalization)
68+
"""
69+
super().__init__("decimal", ordertype="en_tn")
70+
self.deterministic = deterministic
71+
self.build_tagger()
72+
self.build_verbalizer()
73+
74+
def build_tagger(self):
75+
"""
76+
Finite state transducer for classifying decimal, e.g.
77+
-12.5006 billion -> decimal { negative: "true" integer_part: "12" fractional_part: "five o o six" quantity: "billion" }
78+
1 billion -> decimal { integer_part: "one" quantity: "billion" }
79+
"""
80+
cardinal = Cardinal(deterministic=self.deterministic)
81+
cardinal_graph = cardinal.graph_with_and
82+
cardinal_graph_hundred_component_at_least_one_none_zero_digit = (
83+
cardinal.graph_hundred_component_at_least_one_none_zero_digit)
84+
85+
self.graph = cardinal.single_digits_graph.optimize()
86+
87+
if not self.deterministic:
88+
self.graph = self.graph | pynutil.add_weight(cardinal_graph, 0.1)
89+
90+
point = pynutil.delete(".")
91+
optional_graph_negative = pynini.closure(
92+
pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0,
93+
1)
94+
95+
self.graph_fractional = pynutil.insert(
96+
"fractional_part: \"") + self.graph + pynutil.insert("\"")
97+
self.graph_integer = pynutil.insert(
98+
"integer_part: \"") + cardinal_graph + pynutil.insert("\"")
99+
final_graph_wo_sign = (
100+
pynini.closure(self.graph_integer + pynutil.insert(" "), 0, 1) +
101+
point + pynutil.insert(" ") + self.graph_fractional)
102+
103+
quantity_w_abbr = get_quantity(
104+
final_graph_wo_sign,
105+
cardinal_graph_hundred_component_at_least_one_none_zero_digit,
106+
include_abbr=True)
107+
quantity_wo_abbr = get_quantity(
108+
final_graph_wo_sign,
109+
cardinal_graph_hundred_component_at_least_one_none_zero_digit,
110+
include_abbr=False)
111+
self.final_graph_wo_negative_w_abbr = final_graph_wo_sign | quantity_w_abbr
112+
self.final_graph_wo_negative = final_graph_wo_sign | quantity_wo_abbr
113+
114+
# reduce options for non_deterministic and allow either "oh" or "zero", but not combination
115+
if not self.deterministic:
116+
no_oh_zero = pynini.difference(
117+
pynini.closure(self.VCHAR),
118+
(pynini.closure(self.VCHAR) + "oh" + pynini.closure(self.VCHAR)
119+
+ "zero" + pynini.closure(self.VCHAR))
120+
| (pynini.closure(self.VCHAR) + "zero" + pynini.closure(
121+
self.VCHAR) + "oh" + pynini.closure(self.VCHAR)),
122+
).optimize()
123+
no_zero_oh = pynini.difference(
124+
pynini.closure(self.VCHAR),
125+
pynini.closure(self.VCHAR) + pynini.accep("zero") +
126+
pynini.closure(self.VCHAR) + pynini.accep("oh") +
127+
pynini.closure(self.VCHAR)).optimize()
128+
129+
self.final_graph_wo_negative |= pynini.compose(
130+
self.final_graph_wo_negative,
131+
pynini.cdrewrite(
132+
pynini.cross("integer_part: \"zero\"",
133+
"integer_part: \"oh\""),
134+
pynini.closure(self.VCHAR), pynini.closure(self.VCHAR),
135+
pynini.closure(self.VCHAR)),
136+
)
137+
self.final_graph_wo_negative = pynini.compose(
138+
self.final_graph_wo_negative, no_oh_zero).optimize()
139+
self.final_graph_wo_negative = pynini.compose(
140+
self.final_graph_wo_negative, no_zero_oh).optimize()
141+
142+
final_graph = optional_graph_negative + self.final_graph_wo_negative
143+
144+
final_graph = self.add_tokens(final_graph)
145+
self.tagger = final_graph.optimize()
146+
147+
def build_verbalizer(self):
148+
"""
149+
Finite state transducer for verbalizing decimal, e.g.
150+
decimal { negative: "true" integer_part: "twelve" fractional_part: "five o o six" quantity: "billion" } -> minus twelve point five o o six billion
151+
"""
152+
cardinal = Cardinal(deterministic=self.deterministic)
153+
self.optional_sign = pynini.cross("negative: \"true\"", "minus ")
154+
if not self.deterministic:
155+
self.optional_sign |= pynutil.add_weight(
156+
pynini.cross("negative: \"true\"", "negative "), 0.1)
157+
self.optional_sign = pynini.closure(
158+
self.optional_sign + self.DELETE_SPACE, 0, 1)
159+
self.integer = pynutil.delete("integer_part:") + cardinal.integer
160+
self.optional_integer = pynini.closure(
161+
self.integer + self.DELETE_SPACE + self.INSERT_SPACE, 0, 1)
162+
self.fractional_default = (pynutil.delete("fractional_part:") +
163+
self.DELETE_SPACE + pynutil.delete("\"") +
164+
pynini.closure(self.NOT_QUOTE, 1) +
165+
pynutil.delete("\""))
166+
167+
self.fractional = pynutil.insert("point ") + self.fractional_default
168+
169+
self.quantity = (self.DELETE_SPACE + self.INSERT_SPACE +
170+
pynutil.delete("quantity:") + self.DELETE_SPACE +
171+
pynutil.delete("\"") +
172+
pynini.closure(self.NOT_QUOTE, 1) +
173+
pynutil.delete("\""))
174+
self.optional_quantity = pynini.closure(self.quantity, 0, 1)
175+
176+
graph = self.optional_sign + (
177+
self.integer
178+
| (self.integer + self.quantity)
179+
|
180+
(self.optional_integer + self.fractional + self.optional_quantity))
181+
182+
self.numbers = graph
183+
delete_tokens = self.delete_tokens(graph)
184+
if not self.deterministic:
185+
delete_tokens |= pynini.compose(
186+
delete_tokens,
187+
pynini.closure(self.VCHAR) +
188+
(pynini.cross(" point five", " and a half")
189+
| pynini.cross("zero point five", "half")
190+
| pynini.cross(" point two five", " and a quarter")
191+
| pynini.cross("zero point two five", "quarter")),
192+
).optimize()
193+
self.verbalizer = delete_tokens.optimize()

‎tn/english/test/data/decimal.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
-12.5006 billion => minus twelve point five oh oh six billion
2+
1 billion => one billion
3+
1.5 million => one point five million

‎tn/english/test/decimal_test.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Copyright (c) 2024 Xingchen Song (sxc19@tsinghua.org.cn)
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import pytest
16+
17+
from tn.english.rules.decimal import Decimal
18+
from tn.english.test.utils import parse_test_case
19+
20+
21+
class TestDecimal:
22+
23+
decimal = Decimal(deterministic=False)
24+
decimal_cases = parse_test_case('data/decimal.txt')
25+
26+
@pytest.mark.parametrize("written, spoken", decimal_cases)
27+
def test_decimal(self, written, spoken):
28+
assert self.decimal.normalize(written) == spoken

‎tn/processor.py‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,8 @@
1717

1818
from tn.token_parser import TokenParser
1919

20-
from pynini import (
21-
cdrewrite, cross, difference, escape,
22-
Fst, shortestpath, union, closure
23-
)
20+
from pynini import (cdrewrite, cross, difference, escape, Fst, shortestpath,
21+
union, closure, invert)
2422
from pynini.lib import byte, utf8
2523
from pynini.lib.pynutil import delete, insert
2624

@@ -45,7 +43,11 @@ def __init__(self, name, ordertype="tn"):
4543
self.DELETE_SPACE = delete(self.SPACE).star
4644
self.DELETE_EXTRA_SPACE = cross(closure(self.SPACE, 1), " ")
4745
self.MIN_NEG_WEIGHT = -0.0001
48-
self.TO_LOWER = union(*[cross(x, y) for x, y in zip(string.ascii_uppercase, string.ascii_lowercase)])
46+
self.TO_LOWER = union(*[
47+
cross(x, y)
48+
for x, y in zip(string.ascii_uppercase, string.ascii_lowercase)
49+
])
50+
self.TO_UPPER = invert(self.TO_LOWER)
4951

5052
self.name = name
5153
self.ordertype = ordertype

0 commit comments

Comments
 (0)