[runtime] use fst instead of far (#68)

* save fst instead far

* fix runtime

* fix workflow

* update cmakelists

* use shared ptr

Author: pengzhendong

.github/workflows/wheels.yml

@@ -23,8 +23,8 @@ jobs:
       - name: Prepare Graph
         run: |
           mkdir graph
-          cp tn/zh_tn_normalizer.far graph
-          cp itn/zh_itn_normalizer.far graph
+          cp tn/*.fst graph
+          cp itn/*.fst graph
 
       - name: Upload Graph
         uses: actions/upload-artifact@v3

itn/chinese/inverse_normalizer.py

@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
-
 from tn.processor import Processor
 from itn.chinese.rules.cardinal import Cardinal
 from itn.chinese.rules.char import Char
@@ -26,36 +24,19 @@
 from itn.chinese.rules.time import Time
 from itn.chinese.rules.preprocessor import PreProcessor
 
-from pynini import Far
 from pynini.lib.pynutil import add_weight, delete
 from importlib_resources import files
 
 
 class InverseNormalizer(Processor):
 
-    def __init__(self, cache_dir=None, overwrite_cache=False,
+    def __init__(self, cache_dir='itn', overwrite_cache=False,
                  enable_standalone_number=True,
                  enable_0_to_9=True):
         super().__init__(name='inverse_normalizer', ordertype='itn')
-        self.cache_dir = cache_dir
-        self.overwrite_cache = overwrite_cache
         self.convert_number = enable_standalone_number
         self.enable_0_to_9 = enable_0_to_9
-
-        far_file = files('itn').joinpath('zh_itn_normalizer.far')
-        if self.cache_dir:
-            os.makedirs(self.cache_dir, exist_ok=True)
-            far_file = os.path.join(self.cache_dir, 'zh_itn_normalizer.far')
-
-        if far_file and os.path.exists(far_file) and not overwrite_cache:
-            self.tagger = Far(far_file)['tagger']
-            self.verbalizer = Far(far_file)['verbalizer']
-        else:
-            self.build_tagger()
-            self.build_verbalizer()
-
-        if self.cache_dir and self.overwrite_cache:
-            self.export(far_file)
+        self.build_fst('zh_itn', cache_dir, overwrite_cache)
 
     def build_tagger(self):
         tagger = (add_weight(Date().tagger, 1.02)

runtime/CMakeLists.txt

@@ -1,14 +1,12 @@
 cmake_minimum_required(VERSION 3.14 FATAL_ERROR)
 
-project(text_processing VERSION 0.1)
+project(wetextprocessing VERSION 0.1)
 set(CMAKE_CXX_STANDARD 14)
 
 set(CMAKE_VERBOSE_MAKEFILE OFF)
-option(BUILD_TESTING "whether to build unit test" ON)
-option(FST_HAVE_BIN "whether to build fst binaries" OFF)
+option(BUILD_TESTING "whether to build unit test" OFF)
 
 include(FetchContent)
-include(ExternalProject)
 set(FETCHCONTENT_QUIET OFF)
 get_filename_component(fc_base "fc_base" REALPATH BASE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 set(FETCHCONTENT_BASE_DIR ${fc_base})
@@ -21,9 +19,8 @@ if(${CMAKE_SYSTEM_NAME} STREQUAL "Darwin")
 endif()
 
 include(openfst)
-include_directories(${CMAKE_CURRENT_SOURCE_DIR})
+include_directories(${PROJECT_SOURCE_DIR})
 add_subdirectory(utils)
-add_dependencies(utils openfst)
 add_subdirectory(processor)
 add_subdirectory(bin)
 

runtime/bin/processor_main.cc

@@ -17,26 +17,26 @@
 #include <string>
 
 #include "processor/processor.h"
-#include "processor/token_parser.h"
 #include "utils/flags.h"
 
 DEFINE_string(text, "", "input string");
 DEFINE_string(file, "", "input file");
-DEFINE_string(far, "", "FST archives");
+DEFINE_string(tagger, "", "tagger fst path");
+DEFINE_string(verbalizer, "", "verbalizer fst path");
 
 int main(int argc, char* argv[]) {
   gflags::ParseCommandLineFlags(&argc, &argv, false);
   google::InitGoogleLogging(argv[0]);
 
-  if (FLAGS_far.empty()) {
-    LOG(FATAL) << "Please provide the FST archives.";
+  if (FLAGS_tagger.empty() || FLAGS_verbalizer.empty()) {
+    LOG(FATAL) << "Please provide the tagger and verbalizer fst files.";
   }
-  wenet::Processor processor(FLAGS_far);
+  wenet::Processor processor(FLAGS_tagger, FLAGS_verbalizer);
 
   if (!FLAGS_text.empty()) {
     std::string tagged_text = processor.tag(FLAGS_text);
     std::cout << tagged_text << std::endl;
-    std::string normalized_text = processor.normalize(FLAGS_text);
+    std::string normalized_text = processor.verbalize(tagged_text);
     std::cout << normalized_text << std::endl;
   }
 
@@ -46,7 +46,7 @@ int main(int argc, char* argv[]) {
     while (getline(file, line)) {
       std::string tagged_text = processor.tag(line);
       std::cout << tagged_text << std::endl;
-      std::string normalized_text = processor.normalize(line);
+      std::string normalized_text = processor.verbalize(tagged_text);
       std::cout << normalized_text << std::endl;
     }
   }

runtime/cmake/openfst.cmake

@@ -1,28 +1,36 @@
 include(gflags)
+# We can't build glog with gflags, unless gflags is pre-installed.
+# If build glog with pre-installed gflags, there will be conflict.
 set(WITH_GFLAGS OFF CACHE BOOL "whether build glog with gflags" FORCE)
 include(glog)
 
-set(CONFIG_FLAGS "")
-if(NOT FST_HAVE_BIN)
-  set(CONFIG_FLAGS "--disable-bin")
+set(HAVE_BIN OFF CACHE BOOL "Build the fst binaries" FORCE)
+set(HAVE_SCRIPT OFF CACHE BOOL "Build the fstscript" FORCE)
+set(HAVE_COMPACT OFF CACHE BOOL "Build compact" FORCE)
+set(HAVE_CONST OFF CACHE BOOL "Build const" FORCE)
+set(HAVE_GRM OFF CACHE BOOL "Build grm" FORCE)
+set(HAVE_FAR OFF CACHE BOOL "Build far" FORCE)
+set(HAVE_PDT OFF CACHE BOOL "Build pdt" FORCE)
+set(HAVE_MPDT OFF CACHE BOOL "Build mpdt" FORCE)
+set(HAVE_LINEAR OFF CACHE BOOL "Build linear" FORCE)
+set(HAVE_LOOKAHEAD OFF CACHE BOOL "Build lookahead" FORCE)
+set(HAVE_NGRAM OFF CACHE BOOL "Build ngram" FORCE)
+set(HAVE_SPECIAL OFF CACHE BOOL "Build special" FORCE)
+
+if(MSVC)
+  add_compile_options(/W0 /wd4244 /wd4267)
 endif()
 
+# "OpenFST port for Windows" builds openfst with cmake for multiple platforms.
 # Openfst is compiled with glog/gflags to avoid log and flag conflicts with log and flags in wenet/libtorch.
 # To build openfst with gflags and glog, we comment out some vars of {flags, log}.h and flags.cc.
 set(openfst_SOURCE_DIR ${fc_base}/openfst-src CACHE PATH "OpenFST source directory")
-set(openfst_PREFIX_DIR ${fc_base}/openfst-subbuild/openfst-populate-prefix CACHE PATH "OpenFST prefix directory")
-ExternalProject_Add(openfst
-  URL               https://github.com/mjansche/openfst/archive/1.7.2.zip
-  URL_HASH          MD5=96656fee440ee2d71006a4900ef9ac00
-  PREFIX            ${openfst_PREFIX_DIR}
-  SOURCE_DIR        ${openfst_SOURCE_DIR}
-  CONFIGURE_COMMAND ${openfst_SOURCE_DIR}/configure ${CONFIG_FLAGS} --enable-far --prefix=${openfst_PREFIX_DIR}
-                      "CPPFLAGS=-I${gflags_BINARY_DIR}/include -I${glog_SOURCE_DIR}/src -I${glog_BINARY_DIR}"
-                      "LDFLAGS=-L${gflags_BINARY_DIR} -L${glog_BINARY_DIR}"
-                      "LIBS=-lgflags_nothreads -lglog -lpthread"
-  COMMAND           ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/patch/openfst ${openfst_SOURCE_DIR}
-  BUILD_COMMAND     make -j$(nproc)
+  FetchContent_Declare(openfst
+  URL           https://github.com/kkm000/openfst/archive/refs/tags/win/1.7.2.1.tar.gz
+  URL_HASH      SHA256=e04e1dabcecf3a687ace699ccb43a8a27da385777a56e69da6e103344cc66bca
+  PATCH_COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/patch/openfst ${openfst_SOURCE_DIR}
 )
-add_dependencies(openfst gflags glog)
-link_directories(${openfst_PREFIX_DIR}/lib)
+FetchContent_MakeAvailable(openfst)
+add_dependencies(fst gflags glog)
+target_link_libraries(fst PUBLIC gflags_nothreads_static glog)
 include_directories(${openfst_SOURCE_DIR}/src/include)

runtime/patch/openfst/src/CMakeLists.txt

@@ -6,7 +6,10 @@ install(DIRECTORY include/ DESTINATION include/
         FILES_MATCHING PATTERN "*.h")
 
 add_subdirectory(lib)
-add_subdirectory(script)
+
+if(HAVE_SCRIPT)
+  add_subdirectory(script)
+endif(HAVE_SCRIPT)
 
 if(HAVE_BIN)
   add_subdirectory(bin)

runtime/patch/openfst/src/bin/CMakeLists.txt

@@ -10,7 +10,7 @@ if(HAVE_BIN)
     sigma-fst.cc
   )
 
-  set_target_properties(fstspecial-bin PROPERTIE
+  set_target_properties(fstspecial-bin PROPERTIES
     FOLDER special/bin
     OUTPUT_NAME fstspecial
   )

runtime/patch/openfst/src/extensions/special/CMakeLists.txt

@@ -2,4 +2,4 @@ add_library(processor STATIC
   processor.cc
   token_parser.cc
 )
-target_link_libraries(processor PUBLIC fstfar utils)
+target_link_libraries(processor PUBLIC utils)