Implement Multi-tier JIT (#1774)

Implement 2-level Multi-tier JIT engine: tier-up from Fast JIT to LLVM JIT to
get quick cold startup by Fast JIT and better performance by gradually
switching to LLVM JIT when the LLVM JIT functions are compiled by the
backend threads.

Refer to:
#1302

Author: wenyongh

build-scripts/config_common.cmake

@@ -85,6 +85,13 @@ if (NOT WAMR_BUILD_AOT EQUAL 1)
 endif ()
 endif ()
 
+if (WAMR_BUILD_FAST_JIT EQUAL 1)
+  if (NOT WAMR_BUILD_LAZY_JIT EQUAL 0)
+    # Enable Lazy JIT by default
+    set (WAMR_BUILD_LAZY_JIT 1)
+  endif ()
+endif ()
+
 if (WAMR_BUILD_JIT EQUAL 1)
   if (NOT WAMR_BUILD_LAZY_JIT EQUAL 0)
     # Enable Lazy JIT by default
@@ -136,7 +143,12 @@ else ()
   message ("     WAMR AOT disabled")
 endif ()
 if (WAMR_BUILD_FAST_JIT EQUAL 1)
-  message ("     WAMR Fast JIT enabled")
+  if (WAMR_BUILD_LAZY_JIT EQUAL 1)
+    add_definitions("-DWASM_ENABLE_LAZY_JIT=1")
+    message ("     WAMR Fast JIT enabled with Lazy Compilation")
+  else ()
+    message ("     WAMR Fast JIT enabled with Eager Compilation")
+  endif ()
 else ()
   message ("     WAMR Fast JIT disabled")
 endif ()
@@ -151,6 +163,10 @@ if (WAMR_BUILD_JIT EQUAL 1)
 else ()
   message ("     WAMR LLVM ORC JIT disabled")
 endif ()
+if (WAMR_BUILD_FAST_JIT EQUAL 1 AND WAMR_BUILD_JIT EQUAL 1
+    AND WAMR_BUILD_LAZY_JIT EQUAL 1)
+  message ("     Multi-tier JIT enabled")
+endif ()
 if (WAMR_BUILD_LIBC_BUILTIN EQUAL 1)
   message ("     Libc builtin enabled")
 else ()

core/iwasm/common/wasm_runtime_common.c

@@ -2176,6 +2176,9 @@ static const char *exception_msgs[] = {
     "wasm auxiliary stack underflow", /* EXCE_AUX_STACK_UNDERFLOW */
     "out of bounds table access",     /* EXCE_OUT_OF_BOUNDS_TABLE_ACCESS */
     "wasm operand stack overflow",    /* EXCE_OPERAND_STACK_OVERFLOW */
+#if WASM_ENABLE_FAST_JIT != 0
+    "failed to compile fast jit function", /* EXCE_FAILED_TO_COMPILE_FAST_JIT_FUNC */
+#endif
     "",                               /* EXCE_ALREADY_THROWN */
 };
 /* clang-format on */

core/iwasm/compilation/aot.h

@@ -309,6 +309,8 @@ aot_get_imp_tbl_data_slots(const AOTImportTable *tbl, bool is_jit_mode)
 #if WASM_ENABLE_MULTI_MODULE != 0
     if (is_jit_mode)
         return tbl->table_max_size;
+#else
+    (void)is_jit_mode;
 #endif
     return tbl->possible_grow ? tbl->table_max_size : tbl->table_init_size;
 }
@@ -319,6 +321,8 @@ aot_get_tbl_data_slots(const AOTTable *tbl, bool is_jit_mode)
 #if WASM_ENABLE_MULTI_MODULE != 0
     if (is_jit_mode)
         return tbl->table_max_size;
+#else
+    (void)is_jit_mode;
 #endif
     return tbl->possible_grow ? tbl->table_max_size : tbl->table_init_size;
 }

core/iwasm/compilation/aot_emit_function.c

@@ -798,7 +798,53 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
                 func = func_ctx->func;
             }
             else {
-                func = func_ctxes[func_idx - import_func_count]->func;
+                if (!comp_ctx->is_jit_mode) {
+                    func = func_ctxes[func_idx - import_func_count]->func;
+                }
+                else {
+#if !(WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0)
+                    func = func_ctxes[func_idx - import_func_count]->func;
+#else
+                    /* JIT tier-up, load func ptr from func_ptrs[func_idx] */
+                    LLVMValueRef func_ptr, func_idx_const;
+                    LLVMTypeRef func_ptr_type;
+
+                    if (!(func_idx_const = I32_CONST(func_idx))) {
+                        aot_set_last_error("llvm build const failed.");
+                        goto fail;
+                    }
+
+                    if (!(func_ptr = LLVMBuildInBoundsGEP2(
+                              comp_ctx->builder, OPQ_PTR_TYPE,
+                              func_ctx->func_ptrs, &func_idx_const, 1,
+                              "func_ptr_tmp"))) {
+                        aot_set_last_error("llvm build inbounds gep failed.");
+                        goto fail;
+                    }
+
+                    if (!(func_ptr =
+                              LLVMBuildLoad2(comp_ctx->builder, OPQ_PTR_TYPE,
+                                             func_ptr, "func_ptr"))) {
+                        aot_set_last_error("llvm build load failed.");
+                        goto fail;
+                    }
+
+                    if (!(func_ptr_type = LLVMPointerType(
+                              func_ctxes[func_idx - import_func_count]
+                                  ->func_type,
+                              0))) {
+                        aot_set_last_error("construct func ptr type failed.");
+                        goto fail;
+                    }
+
+                    if (!(func = LLVMBuildBitCast(comp_ctx->builder, func_ptr,
+                                                  func_ptr_type,
+                                                  "indirect_func"))) {
+                        aot_set_last_error("llvm build bit cast failed.");
+                        goto fail;
+                    }
+#endif /* end of !(WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0) */
+                }
             }
         }