[runtime] support english tn (#219)

Author: xingchensong

runtime/processor/wetext_processor.cc

@@ -24,10 +24,12 @@ Processor::Processor(const std::string& tagger_path,
   compiler_ = std::make_shared<StringCompiler<StdArc>>(StringTokenType::BYTE);
   printer_ = std::make_shared<StringPrinter<StdArc>>(StringTokenType::BYTE);
 
-  if (tagger_path.find("_tn_") != tagger_path.npos) {
-    parse_type_ = ParseType::kTN;
-  } else if (tagger_path.find("_itn_") != tagger_path.npos) {
-    parse_type_ = ParseType::kITN;
+  if (tagger_path.find("zh_tn_") != tagger_path.npos) {
+    parse_type_ = ParseType::kZH_TN;
+  } else if (tagger_path.find("zh_itn_") != tagger_path.npos) {
+    parse_type_ = ParseType::kZH_ITN;
+  } else if (tagger_path.find("en_tn_") != tagger_path.npos) {
+    parse_type_ = ParseType::kEN_TN;
   } else {
     LOG(FATAL) << "Invalid fst prefix, prefix should contain"
                << " either \"_tn_\" or \"_itn_\".";

runtime/processor/wetext_token_parser.cc

@@ -26,24 +26,30 @@ const std::set<std::string> ASCII_LETTERS = {
     "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "A", "B",
     "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P",
     "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "_"};
-const std::unordered_map<std::string, std::vector<std::string>> TN_ORDERS = {
+const std::unordered_map<std::string, std::vector<std::string>> ZH_TN_ORDERS = {
     {"date", {"year", "month", "day"}},
     {"fraction", {"denominator", "numerator"}},
     {"measure", {"denominator", "numerator", "value"}},
     {"money", {"value", "currency"}},
     {"time", {"noon", "hour", "minute", "second"}}};
-const std::unordered_map<std::string, std::vector<std::string>> ITN_ORDERS = {
-    {"date", {"year", "month", "day"}},
-    {"fraction", {"sign", "numerator", "denominator"}},
-    {"measure", {"numerator", "denominator", "value"}},
-    {"money", {"currency", "value", "decimal"}},
-    {"time", {"hour", "minute", "second", "noon"}}};
+const std::unordered_map<std::string, std::vector<std::string>> EN_TN_ORDERS = {
+    {"date", {"preserve_order", "text", "day", "month", "year"}}};
+const std::unordered_map<std::string, std::vector<std::string>> ZH_ITN_ORDERS =
+    {{"date", {"year", "month", "day"}},
+     {"fraction", {"sign", "numerator", "denominator"}},
+     {"measure", {"numerator", "denominator", "value"}},
+     {"money", {"currency", "value", "decimal"}},
+     {"time", {"hour", "minute", "second", "noon"}}};
 
 TokenParser::TokenParser(ParseType type) {
-  if (type == ParseType::kTN) {
-    orders_ = TN_ORDERS;
+  if (type == ParseType::kZH_TN) {
+    orders_ = ZH_TN_ORDERS;
+  } else if (type == ParseType::kZH_ITN) {
+    orders_ = ZH_TN_ORDERS;
+  } else if (type == ParseType::kEN_TN) {
+    orders_ = EN_TN_ORDERS;
   } else {
-    orders_ = ITN_ORDERS;
+    orders_ = ZH_ITN_ORDERS;
   }
 }
 

runtime/processor/wetext_token_parser.h

@@ -60,8 +60,9 @@ struct Token {
 };
 
 enum ParseType {
-  kTN = 0x00,  // Text Normalization
-  kITN = 0x01  // Inverse Text Normalization
+  kZH_TN = 0x00,   // Chinese Text Normalization
+  kZH_ITN = 0x01,  // Chinese Inverse Text Normalization
+  kEN_TN = 0x02    // English Text Normalization
 };
 
 class TokenParser {