Add T8 ternary weights: MatMulTernary with NEON, AVX2 and AVX-512 kernels

cmake/onnxruntime_mlas.cmake

@@ -61,6 +61,8 @@ onnxruntime_add_static_library(onnxruntime_mlas
   ${MLAS_SRC_DIR}/qdwconv_kernelsize.cpp
   ${MLAS_SRC_DIR}/qnbitgemm.h
   ${MLAS_SRC_DIR}/qnbitgemm.cpp
+  ${MLAS_SRC_DIR}/t8gemm.cpp
+  ${MLAS_SRC_DIR}/t8gemm_kernel.h
   ${MLAS_SRC_DIR}/qlutgemm.h
   ${MLAS_SRC_DIR}/qlutgemm.cpp
   ${MLAS_SRC_DIR}/sqnbitgemm_q8_block.h
@@ -83,6 +85,7 @@ target_sources(onnxruntime_mlas PRIVATE
   ${MLAS_INC_DIR}/mlas_gemm_postprocessor.h
   ${MLAS_INC_DIR}/mlas_q4.h
   ${MLAS_INC_DIR}/mlas_qnbit.h
+  ${MLAS_INC_DIR}/mlas_t8.h
   ${MLAS_INC_DIR}/mlas_qkv_quant.h
   ${MLAS_INC_DIR}/mlas.h
 )
@@ -259,13 +262,15 @@ function(setup_mlas_source_for_windows)
     set_source_files_properties(${mlas_platform_srcs_avx2} PROPERTIES COMPILE_FLAGS "/arch:AVX2")
 
     set_source_files_properties(${MLAS_SRC_DIR}/layernorm_kernel_avx2.cpp PROPERTIES COMPILE_FLAGS "/arch:AVX2")
+    set_source_files_properties(${MLAS_SRC_DIR}/t8gemm_kernel_avx2.cpp PROPERTIES COMPILE_FLAGS "/arch:AVX2")
 
     set(mlas_platform_srcs_avx512
       ${MLAS_SRC_DIR}/intrinsics/avx512/gelu_avx512f.cpp
       ${MLAS_SRC_DIR}/intrinsics/avx512/silu_avx512f.cpp
       ${MLAS_SRC_DIR}/intrinsics/avx512/quantize_avx512f.cpp
       ${MLAS_SRC_DIR}/intrinsics/avx512/sconv_nchw_depthwise_multiplier_greater_than_1_avx512f.cpp
       ${MLAS_SRC_DIR}/linear_attention_kernel_avx512f.cpp
+      ${MLAS_SRC_DIR}/t8gemm_kernel_avx512f.cpp
       ${MLAS_SRC_DIR}/intrinsics/avx512/reorder_avx512f.cpp
       ${MLAS_SRC_DIR}/intrinsics/avx512/sconv_depthwise_avx512f.cpp
     )
@@ -277,6 +282,7 @@ function(setup_mlas_source_for_windows)
       ${mlas_platform_srcs_avx}
       ${mlas_platform_srcs_avx2}
       ${MLAS_SRC_DIR}/layernorm_kernel_avx2.cpp
+      ${MLAS_SRC_DIR}/t8gemm_kernel_avx2.cpp
       ${MLAS_SRC_DIR}/rotary_embedding_kernel_avx2.h
       ${MLAS_SRC_DIR}/rotary_embedding_kernel_avx2.cpp
       ${MLAS_SRC_DIR}/rotary_embedding_kernel_avx2.cpp
@@ -939,6 +945,7 @@ else()
           ${MLAS_SRC_DIR}/qkv_quant_common.h
           ${MLAS_SRC_DIR}/qkv_quant_kernel_avx2.cpp
           ${MLAS_SRC_DIR}/layernorm_kernel_avx2.cpp
+          ${MLAS_SRC_DIR}/t8gemm_kernel_avx2.cpp
         )
         if(CMAKE_CXX_COMPILER_VERSION GREATER_EQUAL 13.1 AND NOT(APPLE))
           set(mlas_platform_srcs_avx2
@@ -1011,6 +1018,7 @@ else()
           ${MLAS_SRC_DIR}/intrinsics/avx512/quantize_avx512f.cpp
           ${MLAS_SRC_DIR}/intrinsics/avx512/sconv_nchw_depthwise_multiplier_greater_than_1_avx512f.cpp
           ${MLAS_SRC_DIR}/linear_attention_kernel_avx512f.cpp
+          ${MLAS_SRC_DIR}/t8gemm_kernel_avx512f.cpp
           ${MLAS_SRC_DIR}/intrinsics/avx512/reorder_avx512f.cpp
           ${MLAS_SRC_DIR}/intrinsics/avx512/sconv_depthwise_avx512f.cpp
         )

include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h

@@ -20,6 +20,12 @@
 // If the config value is set to "1" then the prepacking is disabled, otherwise prepacking is enabled (default value)
 static const char* const kOrtSessionOptionsConfigDisablePrepacking = "session.disable_prepacking";
 
+// Experimental MatMulTernary CPU packing: "current" (default), "tiled", or "masks".
+// Disk tensors remain T8 v1. Constant weights are packed at session initialization.
+static const char* const kOrtSessionOptionsConfigTernaryPrepack = "session.ternary.prepack";
+// Activation rows sharing decoded masks: "1" (default) or "2"; "current" requires "1".
+static const char* const kOrtSessionOptionsConfigTernaryRowTile = "session.ternary.row_tile";
+
 // Log MoE expert-counter updates at INFO severity. This enables counter updates even when expert counting is disabled.
 // At most 1024 updates are logged per Run; further updates emit one WARNING truncation marker but still count.
 // "0": disable (default); "1": enable.

onnxruntime/contrib_ops/cpu/cpu_contrib_kernels.cc

@@ -75,6 +75,7 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1,
 class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, double, FusedMatMul);
 class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, MatMulNBits);
 class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, MLFloat16, MatMulNBits);
+class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, MatMulTernary);
 class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, MatMulBnb4);
 class ONNX_OPERATOR_TWO_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, int32_t, GatherBlockQuantized);
 class ONNX_OPERATOR_TWO_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, int64_t, GatherBlockQuantized);
@@ -422,6 +423,7 @@ Status RegisterCpuContribKernels(KernelRegistry& kernel_registry) {
       BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, double, FusedMatMul)>,
       BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, MatMulNBits)>,
       BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, MLFloat16, MatMulNBits)>,
+      BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, MatMulTernary)>,
       BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, MatMulBnb4)>,
       BuildKernelCreateInfo<ONNX_OPERATOR_TWO_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, int32_t, GatherBlockQuantized)>,
       BuildKernelCreateInfo<ONNX_OPERATOR_TWO_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, int64_t, GatherBlockQuantized)>,

onnxruntime/contrib_ops/cpu/quantization/matmul_ternary.cc

@@ -0,0 +1,117 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#include "core/common/narrow.h"
+#include "core/common/safeint.h"
+#include "core/framework/op_kernel.h"
+#include "core/mlas/inc/mlas_t8.h"
+#include "core/providers/common.h"
+#include "core/session/onnxruntime_session_options_config_keys.h"
+
+namespace onnxruntime::contrib {
+
+class MatMulTernary final : public OpKernel {
+ public:
+  explicit MatMulTernary(const OpKernelInfo& info) : OpKernel(info) {
+    const int64_t k = info.GetAttr<int64_t>("K"), n = info.GetAttr<int64_t>("N");
+    ORT_ENFORCE(k > 0 && k % 128 == 0 && n > 0, "MatMulTernary requires K > 0 divisible by 128 and N > 0");
+    ORT_ENFORCE(info.GetAttr<int64_t>("format_version") == 1 &&
+                    info.GetAttr<std::string>("layout") == "unit5x128-planes-v1",
+                "Unsupported MatMulTernary T8 format version or layout");
+    k_ = narrow<size_t>(k);
+    n_ = narrow<size_t>(n);
+    const auto packing = info.GetConfigOptions().GetConfigOrDefault(kOrtSessionOptionsConfigTernaryPrepack, "current");
+    ORT_ENFORCE(packing == "current" || packing == "tiled" || packing == "masks",
+                "session.ternary.prepack must be current, tiled, or masks");
+    use_packed_ = packing != "current";
+    layout_ = packing == "masks" ? MLAS_T8_PACKED_LAYOUT::Masks : MLAS_T8_PACKED_LAYOUT::Tiled;
+    const auto row_tile = info.GetConfigOptions().GetConfigOrDefault(kOrtSessionOptionsConfigTernaryRowTile, "1");
+    ORT_ENFORCE(row_tile == "1" || row_tile == "2", "session.ternary.row_tile must be 1 or 2");
+    row_tile_ = row_tile == "2" ? 2 : 1;
+    ORT_ENFORCE(use_packed_ || row_tile_ == 1, "current T8 kernel requires row_tile 1");
+    const Tensor* planes = nullptr;
+    if (info.TryGetConstantInput(1, &planes) && planes != nullptr) {
+      ORT_THROW_IF_ERROR(ValidatePlanes(*planes));
+      constant_planes_ = true;
+    }
+  }
+
+  Status PrePack(const Tensor& tensor, int input_idx, AllocatorPtr allocator,
+                 bool& is_packed, PrePackedWeights*) override {
+    is_packed = false;
+    if (input_idx != 1 || !use_packed_) return Status::OK();
+    ORT_RETURN_IF_ERROR(ValidatePlanes(tensor));
+    packed_ = IAllocator::MakeUniquePtr<uint32_t>(allocator, PackedWords(), true);
+    MlasT8Pack(n_, k_, tensor.Data<uint32_t>(), packed_.get(), layout_);
+    constant_planes_ = true;
+    is_packed = true;
+    return Status::OK();
+  }
+
+  Status Compute(OpKernelContext* context) const override {
+    const Tensor* a = context->Input<Tensor>(0);
+    const Tensor* planes = packed_ ? nullptr : context->Input<Tensor>(1);
+    const Tensor* scales = context->Input<Tensor>(2);
+    const size_t units = n_ / 5 + (n_ % 5 != 0);
+    ORT_RETURN_IF_NOT(a->Shape().NumDimensions() >= 1 && a->Shape().GetDims().back() == static_cast<int64_t>(k_),
+                      "MatMulTernary A must have rank >= 1 and last dimension K");
+    ORT_RETURN_IF_NOT(scales->Shape() == TensorShape({narrow<int64_t>(units), narrow<int64_t>(k_ / 128), 5}),
+                      "MatMulTernary scales must have shape [ceil(N/5), K/128, 5]");
+    if (!constant_planes_) ORT_RETURN_IF_ERROR(ValidatePlanes(*planes));
+    TensorShapeVector shape(a->Shape().GetDims().begin(), a->Shape().GetDims().end());
+    shape.back() = narrow<int64_t>(n_);
+    Tensor* output = context->Output(0, TensorShape(shape));
+    const size_t m = narrow<size_t>(a->Shape().Size() / narrow<int64_t>(k_));
+    if (m == 0) return Status::OK();
+    AllocatorPtr allocator;
+    ORT_RETURN_IF_ERROR(context->GetTempSpaceAllocator(&allocator));
+    auto workspace = IAllocator::MakeUniquePtr<float>(allocator, SafeInt<size_t>(m) * k_, true);
+    if (use_packed_) {
+      IAllocatorUniquePtr<uint32_t> dynamic_packed;
+      const uint32_t* packed = packed_.get();
+      if (packed == nullptr) {
+        dynamic_packed = IAllocator::MakeUniquePtr<uint32_t>(allocator, PackedWords(), true);
+        MlasT8Pack(n_, k_, planes->Data<uint32_t>(), dynamic_packed.get(), layout_);
+        packed = dynamic_packed.get();
+      }
+      MlasT8GemmPacked(m, n_, k_, a->Data<float>(), packed, scales->Data<MLFloat16>(),
+                       output->MutableData<float>(), workspace.get(), context->GetOperatorThreadPool(), layout_, row_tile_);
+    } else {
+      MlasT8Gemm(m, n_, k_, a->Data<float>(), planes->Data<uint32_t>(), scales->Data<MLFloat16>(),
+                 output->MutableData<float>(), workspace.get(), context->GetOperatorThreadPool());
+    }
+    return Status::OK();
+  }
+
+ private:
+  size_t PackedWords() const {
+    return SafeInt<size_t>(n_ / 5 + (n_ % 5 != 0)) * (k_ / 128) *
+           (layout_ == MLAS_T8_PACKED_LAYOUT::Tiled ? 32 : 40);
+  }
+
+  Status ValidatePlanes(const Tensor& planes) const {
+    const size_t units = n_ / 5 + (n_ % 5 != 0);
+    ORT_RETURN_IF_NOT(planes.Shape() == TensorShape({narrow<int64_t>(units), 8, narrow<int64_t>(k_ / 32)}),
+                      "MatMulTernary B must have shape [ceil(N/5), 8, K/32]");
+    ORT_RETURN_IF_NOT(MlasT8Validate(n_, k_, planes.Data<uint32_t>()),
+                      "MatMulTernary B contains an invalid T8 byte (243..255)");
+    return Status::OK();
+  }
+
+  size_t k_{}, n_{};
+  bool constant_planes_{false};
+  bool use_packed_{false};
+  size_t row_tile_{1};
+  MLAS_T8_PACKED_LAYOUT layout_{MLAS_T8_PACKED_LAYOUT::Tiled};
+  IAllocatorUniquePtr<uint32_t> packed_{};
+};
+
+ONNX_OPERATOR_KERNEL_EX(
+    MatMulTernary, kMSDomain, 1, kCpuExecutionProvider,
+    KernelDefBuilder()
+        .TypeConstraint("T", DataTypeImpl::GetTensorType<float>())
+        .TypeConstraint("TPlanes", DataTypeImpl::GetTensorType<uint32_t>())
+        .TypeConstraint("TScale", DataTypeImpl::GetTensorType<MLFloat16>()),
+    MatMulTernary);
+
+}  // namespace onnxruntime::contrib

onnxruntime/core/graph/contrib_ops/contrib_defs.cc

@@ -3855,6 +3855,55 @@ This op functions in much the same was as Dropout-11 and Dropout-13 do, except t
         }
       });
 
+  ONNX_CONTRIB_OPERATOR_SCHEMA(MatMulTernary)
+      .SetDomain(kMSDomain)
+      .SinceVersion(1)
+      .SetDoc(
+          "Matmul with T8 v1 ternary weights: five trits per byte, stored as uint32 bitplanes. "
+          "Trit 0 is zero, 1 is +scale, 2 is -scale. Rows are padded to a multiple of five. "
+          "No bias or zero point is stored. Activations and output are float32; scales are float16.")
+      .Attr("K", "Positive input width, divisible by 128.", AttributeProto::INT)
+      .Attr("N", "Positive output width, excluding padded rows.", AttributeProto::INT)
+      .Attr("format_version", "T8 format version; only 1 is supported.", AttributeProto::INT, int64_t{1})
+      .Attr("layout", "T8 tensor layout.", AttributeProto::STRING, std::string("unit5x128-planes-v1"))
+      .Input(0, "A", "Activations of shape [..., K], rank at least one.", "T")
+      .Input(1, "B", "Bitplanes [ceil(N/5), 8, K/32]; bit l holds lane l of a plane-set.", "TPlanes")
+      .Input(2, "scales", "Per-group scales [ceil(N/5), K/128, 5], padded rows have scale zero.", "TScale")
+      .Output(0, "Y", "Output of shape [..., N].", "T")
+      .TypeConstraint("T", {"tensor(float)"}, "Float32 activations and output.")
+      .TypeConstraint("TPlanes", {"tensor(uint32)"}, "Little-endian uint32 bitplanes.")
+      .TypeConstraint("TScale", {"tensor(float16)"}, "Float16 scales.")
+      .TypeAndShapeInferenceFunction([](ONNX_NAMESPACE::InferenceContext& ctx) {
+        propagateElemTypeFromInputToOutput(ctx, 0, 0);
+        const int64_t k = getAttribute(ctx, "K", -1), n = getAttribute(ctx, "N", -1);
+        if (k <= 0 || k % 128 != 0 || n <= 0) {
+          fail_shape_inference("MatMulTernary requires K > 0 divisible by 128 and N > 0");
+        }
+        const int64_t units = n / 5 + (n % 5 != 0);
+        const int64_t expected[2][3] = {{units, 8, k / 32}, {units, k / 128, 5}};
+        for (size_t input = 1; input <= 2; ++input) {
+          if (!hasInputShape(ctx, input)) continue;
+          const auto& shape = getInputShape(ctx, input);
+          if (shape.dim_size() != 3) fail_shape_inference("MatMulTernary B and scales must have rank 3");
+          for (int d = 0; d < 3; ++d) {
+            if (shape.dim(d).has_dim_value() && shape.dim(d).dim_value() != expected[input - 1][d]) {
+              fail_shape_inference("MatMulTernary invalid packed tensor dimension");
+            }
+          }
+        }
+        if (hasInputShape(ctx, 0)) {
+          auto shape = getInputShape(ctx, 0);
+          const int rank = shape.dim_size();
+          if (rank < 1) fail_shape_inference("MatMulTernary A must have rank >= 1");
+          if (shape.dim(rank - 1).has_dim_value() && shape.dim(rank - 1).dim_value() != k) {
+            fail_shape_inference("MatMulTernary A last dimension must equal K");
+          }
+          shape.mutable_dim(rank - 1)->clear_dim_param();
+          shape.mutable_dim(rank - 1)->set_dim_value(n);
+          updateOutputShape(ctx, 0, shape);
+        }
+      });
+
   static const char* MatMulNBits_ver1_doc = R"DOC(
 MatMulNBits performs a matrix multiplication where the right-hand-side matrix (weights) is quantized to N bits.
 

onnxruntime/core/mlas/inc/mlas_t8.h

@@ -0,0 +1,28 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include "mlas.h"
+
+// T8 v1: uint32 planes [ceil(N/5), 8, K/32], fp16 scales [ceil(N/5), K/128, 5].
+// K is positive and divisible by 128. A and C are contiguous fp32 matrices.
+// Workspace contains M*K floats; no expanded weight matrix is allocated.
+void MLASCALL
+MlasT8Gemm(size_t M, size_t N, size_t K, const float* A, const uint32_t* Planes, const MLAS_FP16* Scales, float* C, float* Workspace, MLAS_THREADPOOL* ThreadPool);
+
+bool MLASCALL
+MlasT8Validate(size_t N, size_t K, const uint32_t* Planes);
+
+enum class MLAS_T8_PACKED_LAYOUT {
+    Tiled,
+    Masks,
+};
+
+// Each 5x128 group contains 32 tiled T8 words or 40 nonzero/sign mask words.
+void MLASCALL
+MlasT8Pack(size_t N, size_t K, const uint32_t* Planes, uint32_t* Packed, MLAS_T8_PACKED_LAYOUT Layout);
+
+// RowTile is 1 or 2. Workspace has the same size as MlasT8Gemm.
+void MLASCALL
+MlasT8GemmPacked(size_t M, size_t N, size_t K, const float* A, const uint32_t* Packed, const MLAS_FP16* Scales, float* C, float* Workspace, MLAS_THREADPOOL* ThreadPool, MLAS_T8_PACKED_LAYOUT Layout, size_t RowTile);

onnxruntime/core/mlas/lib/mlasi.h

@@ -778,6 +778,32 @@ typedef void(MLASCALL MLAS_LAYERNORM_F16_KERNEL)(
     bool Simplified
 );
 
+//
+// T8 ternary GEMM. Weight layouts: T8 v1 bit planes, MlasT8Pack tiled words,
+// or MlasT8Pack decoded nonzero/sign masks. The kernel computes output units
+// [UnitBegin, UnitEnd) (five columns each) for Rows (1 or 2) rows of A, which
+// are contiguous with stride K; C rows have stride N.
+//
+
+enum class MLAS_T8_WEIGHT_LAYOUT {
+    Planes,
+    Tiled,
+    Masks,
+};
+
+typedef void(MLASCALL MLAS_T8_GEMM_KERNEL)(
+    size_t Rows,
+    size_t N,
+    size_t K,
+    size_t UnitBegin,
+    size_t UnitEnd,
+    const float* A,
+    const uint32_t* B,
+    const MLAS_FP16* Scales,
+    float* C,
+    MLAS_T8_WEIGHT_LAYOUT Layout
+);
+
 typedef
 void
 (MLASCALL MLAS_QLINEAR_BINARY_OP_S8_KERNEL)(
@@ -1730,6 +1756,11 @@ extern const MLAS_LINEAR_ATTENTION_DISPATCH MlasLinearAttentionDispatchNeon;
 extern const MLAS_LINEAR_ATTENTION_DISPATCH MlasLinearAttentionDispatchSve;
 extern const MLAS_LINEAR_ATTENTION_DISPATCH MlasLinearAttentionDispatchRvv;
 
+#if defined(MLAS_TARGET_AMD64)
+MLAS_T8_GEMM_KERNEL MlasT8GemmKernelAvx2;
+MLAS_T8_GEMM_KERNEL MlasT8GemmKernelAvx512F;
+#endif
+
 //
 // Quantized depthwise convolution kernels.
 //
@@ -1987,6 +2018,7 @@ MLAS_COMPUTE_TANH_FP16_KERNEL* TanhFP16KernelRoutine = nullptr;
     const MLAS_ELTWISE_DISPATCH* EltwiseDispatch{nullptr};
     const MLAS_KV_QUANT_GEMM_DISPATCH* KVQuantGemmDispatch{nullptr};
     const MLAS_LINEAR_ATTENTION_DISPATCH* LinearAttentionDispatch{nullptr};
+    MLAS_T8_GEMM_KERNEL* T8GemmKernel{nullptr};
 };
 
 inline

onnxruntime/core/mlas/lib/platform.cpp

@@ -582,6 +582,7 @@ Return Value:
                 // TODO(vraspar): check if this really goes here or if there are other platform reqs that we need to fulfill
                 this->LutGenKernel = &MlasLutGenKernelAvx2;
                 this->LayerNormF32Kernel = &MlasLayerNormKernelAvx2;
+                this->T8GemmKernel = MlasT8GemmKernelAvx2;
                 if ((Cpuid1[2] & (1u << 29)) != 0) {
                     this->LayerNormF16Kernel = &MlasLayerNormKernelF16Avx2;
                 }
@@ -637,6 +638,7 @@ Return Value:
                     this->ComputeSumExpF32Kernel = MlasComputeSumExpF32KernelAvx512F;
                     this->ReduceMaximumF32Kernel = MlasReduceMaximumF32KernelAvx512F;
                     this->LinearAttentionDispatch = &MlasLinearAttentionDispatchAvx512F;
+                    this->T8GemmKernel = MlasT8GemmKernelAvx512F;
                     this->QuantizeLinearS8Kernel = MlasQuantizeLinearS8KernelAvx512F;
                     this->QuantizeLinearU8Kernel = MlasQuantizeLinearU8KernelAvx512F;
                     this->NchwcBlockSize = 16;