Skip to content

Commit 59dd3f3

Browse files
xingchensongxingchensong
authored
[tn] english, support telephone (#213)
1 parent 28e6dae commit 59dd3f3

8 files changed

Lines changed: 227 additions & 2 deletions

File tree

‎tn/english/data/telephone/__init__.py‎

Whitespace-only changes.

‎tn/english/data/telephone/ip_prompt.tsv‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
IP address is
2+
IP is

‎tn/english/data/telephone/ssn_prompt.tsv‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
ssn is SSN is
2+
ssn is SSN is
3+
SSN is
4+
SSN

‎tn/english/data/telephone/telephone_prompt.tsv‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
call me at
2+
reach at
3+
reached at
4+
my number is
5+
hit me up at

‎tn/english/normalizer.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
from tn.english.rules.time import Time
2424
from tn.english.rules.measure import Measure
2525
from tn.english.rules.money import Money
26+
from tn.english.rules.telephone import Telephone
2627

2728
from pynini.lib.pynutil import add_weight, delete
2829
from importlib_resources import files
@@ -45,10 +46,12 @@ def build_tagger(self):
4546
time = add_weight(Time().tagger, 1.00)
4647
measure = add_weight(Measure().tagger, 1.00)
4748
money = add_weight(Money().tagger, 1.00)
49+
telephone = add_weight(Telephone().tagger, 1.00)
4850
word = add_weight(Word().tagger, 100)
4951
tagger = (cardinal | ordinal | word
5052
| date | decimal | fraction
51-
| time | measure | money).optimize() + self.DELETE_SPACE
53+
| time | measure | money
54+
| telephone).optimize() + self.DELETE_SPACE
5255
# delete the last space
5356
self.tagger = tagger.star @ self.build_rule(delete(' '), r='[EOS]')
5457

@@ -62,8 +65,10 @@ def build_verbalizer(self):
6265
time = Time().verbalizer
6366
measure = Measure().verbalizer
6467
money = Money().verbalizer
68+
telephone = Telephone().verbalizer
6569
verbalizer = (cardinal | ordinal | word
6670
| date | decimal
6771
| fraction | time
68-
| measure | money).optimize() + self.INSERT_SPACE
72+
| measure | money
73+
| telephone).optimize() + self.INSERT_SPACE
6974
self.verbalizer = verbalizer.star

‎tn/english/rules/telephone.py‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import pynini
16+
from pynini.lib import pynutil
17+
from pynini.examples import plurals
18+
19+
from tn.processor import Processor
20+
from tn.utils import get_abs_path
21+
22+
23+
class Telephone(Processor):
24+
25+
def __init__(self, deterministic: bool = True):
26+
"""
27+
Args:
28+
deterministic: if True will provide a single transduction option,
29+
for False multiple transduction are generated (used for audio-based normalization)
30+
"""
31+
super().__init__('telephone', ordertype="en_tn")
32+
self.deterministic = deterministic
33+
self.build_tagger()
34+
self.build_verbalizer()
35+
36+
def build_tagger(self):
37+
"""
38+
Finite state transducer for classifying telephone, and IP, and SSN which includes country code, number part and extension
39+
country code optional: +***
40+
number part: ***-***-****, or (***) ***-****
41+
extension optional: 1-9999
42+
E.g
43+
+1 123-123-5678-1 -> telephone { country_code: "one" number_part: "one two three, one two three, five six seven eight" extension: "one" }
44+
1-800-GO-U-HAUL -> telephone { country_code: "one" number_part: "one, eight hundred GO U HAUL" }
45+
"""
46+
add_separator = pynutil.insert(", ") # between components
47+
zero = pynini.cross("0", "zero")
48+
if not self.deterministic:
49+
zero |= pynini.cross("0", pynini.union("o", "oh"))
50+
digit = pynini.invert(
51+
pynini.string_file(get_abs_path(
52+
"english/data/number/digit.tsv"))).optimize() | zero
53+
54+
telephone_prompts = pynini.string_file(
55+
get_abs_path("english/data/telephone/telephone_prompt.tsv"))
56+
country_code = (
57+
pynini.closure(telephone_prompts + self.DELETE_EXTRA_SPACE, 0, 1) +
58+
pynini.closure(pynini.cross("+", "plus "), 0, 1) +
59+
pynini.closure(digit + self.INSERT_SPACE, 0, 2) + digit +
60+
pynutil.insert(","))
61+
country_code |= telephone_prompts
62+
country_code = pynutil.insert(
63+
"country_code: \"") + country_code + pynutil.insert("\"")
64+
country_code = country_code + pynini.closure(
65+
pynutil.delete("-"), 0, 1) + self.DELETE_SPACE + self.INSERT_SPACE
66+
67+
area_part_default = pynini.closure(digit + self.INSERT_SPACE, 2,
68+
2) + digit
69+
area_part = pynini.cross("800", "eight hundred") | pynini.compose(
70+
pynini.difference(pynini.closure(self.VCHAR), "800"),
71+
area_part_default)
72+
73+
area_part = (
74+
(area_part + (pynutil.delete("-") | pynutil.delete(".")))
75+
|
76+
(pynutil.delete("(") + area_part +
77+
((pynutil.delete(")") + pynini.closure(pynutil.delete(" "), 0, 1))
78+
| pynutil.delete(")-")))) + add_separator
79+
80+
del_separator = pynini.closure(pynini.union("-", " ", "."), 0, 1)
81+
number_length = ((self.DIGIT + del_separator) |
82+
(self.ALPHA + del_separator))**7
83+
number_words = pynini.closure((self.DIGIT @ digit) +
84+
(self.INSERT_SPACE |
85+
(pynini.cross("-", ', ')))
86+
| self.ALPHA
87+
| (self.ALPHA + pynini.cross("-", ' ')))
88+
number_words |= pynini.closure((self.DIGIT @ digit) +
89+
(self.INSERT_SPACE |
90+
(pynini.cross(".", ', ')))
91+
| self.ALPHA
92+
| (self.ALPHA + pynini.cross(".", ' ')))
93+
number_words = pynini.compose(number_length, number_words)
94+
number_part = area_part + number_words
95+
number_part = pynutil.insert(
96+
"number_part: \"") + number_part + pynutil.insert("\"")
97+
extension = (pynutil.insert("extension: \"") +
98+
pynini.closure(digit + self.INSERT_SPACE, 0, 3) + digit +
99+
pynutil.insert("\""))
100+
extension = pynini.closure(self.INSERT_SPACE + extension, 0, 1)
101+
102+
graph = plurals._priority_union(country_code + number_part,
103+
number_part,
104+
pynini.closure(self.VCHAR)).optimize()
105+
graph = plurals._priority_union(country_code + number_part + extension,
106+
graph,
107+
pynini.closure(self.VCHAR)).optimize()
108+
graph = plurals._priority_union(number_part + extension, graph,
109+
pynini.closure(self.VCHAR)).optimize()
110+
111+
# ip
112+
ip_prompts = pynini.string_file(
113+
get_abs_path("english/data/telephone/ip_prompt.tsv"))
114+
digit_to_str_graph = digit + pynini.closure(
115+
pynutil.insert(" ") + digit, 0, 2)
116+
ip_graph = digit_to_str_graph + (pynini.cross(".", " dot ") +
117+
digit_to_str_graph)**3
118+
graph |= (
119+
pynini.closure( # noqa
120+
pynutil.insert("country_code: \"") + ip_prompts + # noqa
121+
pynutil.insert("\"") + self.DELETE_EXTRA_SPACE,
122+
0,
123+
1) # noqa
124+
+ pynutil.insert("number_part: \"") # noqa
125+
+ ip_graph.optimize() + pynutil.insert("\"") # noqa
126+
)
127+
# ssn
128+
ssn_prompts = pynini.string_file(
129+
get_abs_path("english/data/telephone/ssn_prompt.tsv"))
130+
three_digit_part = digit + (pynutil.insert(" ") + digit)**2
131+
two_digit_part = digit + pynutil.insert(" ") + digit
132+
four_digit_part = digit + (pynutil.insert(" ") + digit)**3
133+
ssn_separator = pynini.cross("-", ", ")
134+
ssn_graph = three_digit_part + ssn_separator + two_digit_part + ssn_separator + four_digit_part
135+
136+
graph |= (
137+
pynini.closure( # noqa
138+
pynutil.insert("country_code: \"") + ssn_prompts + # noqa
139+
pynutil.insert("\"") + self.DELETE_EXTRA_SPACE, # noqa
140+
0,
141+
1) + pynutil.insert("number_part: \"") # noqa
142+
+ ssn_graph.optimize() # noqa
143+
+ pynutil.insert("\"") # noqa
144+
)
145+
146+
final_graph = self.add_tokens(graph)
147+
self.tagger = final_graph.optimize()
148+
149+
def build_verbalizer(self):
150+
"""
151+
Finite state transducer for verbalizing telephone numbers, e.g.
152+
telephone { country_code: "one" number_part: "one two three, one two three, five six seven eight" extension: "one" }
153+
-> one, one two three, one two three, five six seven eight, one
154+
"""
155+
optional_country_code = pynini.closure(
156+
pynutil.delete("country_code: \"") +
157+
pynini.closure(self.NOT_QUOTE, 1) + pynutil.delete("\"") +
158+
self.DELETE_SPACE + self.INSERT_SPACE,
159+
0,
160+
1,
161+
)
162+
163+
number_part = (
164+
pynutil.delete("number_part: \"") +
165+
pynini.closure(self.NOT_QUOTE, 1) + pynini.closure(
166+
pynutil.add_weight(pynutil.delete(" "), -0.0001), 0, 1) +
167+
pynutil.delete("\""))
168+
169+
optional_extension = pynini.closure(
170+
self.DELETE_SPACE + self.INSERT_SPACE +
171+
pynutil.delete("extension: \"") +
172+
pynini.closure(self.NOT_QUOTE, 1) + pynutil.delete("\""),
173+
0,
174+
1,
175+
)
176+
177+
graph = optional_country_code + number_part + optional_extension
178+
delete_tokens = self.delete_tokens(graph)
179+
self.verbalizer = delete_tokens.optimize()

‎tn/english/test/data/telephone.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
+1 123-123-5678-1 => plus one, one two three, one two three, five six seven eight, one
2+
1-800-GO-U-HAUL => one, eight hundred, GO U HAUL

‎tn/english/test/telephone_test.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Copyright (c) 2024 Xingchen Song (sxc19@tsinghua.org.cn)
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import pytest
16+
17+
from tn.english.rules.telephone import Telephone
18+
from tn.english.test.utils import parse_test_case
19+
20+
21+
class TestTelephone:
22+
23+
telephone = Telephone(deterministic=False)
24+
telephone_cases = parse_test_case('data/telephone.txt')
25+
26+
@pytest.mark.parametrize("written, spoken", telephone_cases)
27+
def test_telephone(self, written, spoken):
28+
assert self.telephone.normalize(written) == spoken

0 commit comments

Comments
 (0)