Repository: microsoft/onnxruntime
File: onnxruntime/contrib_ops/cpu/quantization/dynamic_quantize_matmul.cc (C++)

Code before:
                                       nullptr,
                                       nullptr));

    Tensor* y = ctx->Output(OUT_Y, helper.OutputShape());

    // Bail out early if the output is going to be empty
    if (y->Shape().Size() == 0)
      return Status::OK();

    auto a_data = ctx->Input<Tensor>(IN_A)->Data<float>();
    auto* y_data = y->MutableData<float>();

    // batch gemm
    MLAS_GEMM_DYN_QUANT_SHAPE_PARAMS gemm_shape;
    gemm_shape.M = static_cast<size_t>(helper.M());
    gemm_shape.N = static_cast<size_t>(helper.N());
    gemm_shape.K = static_cast<size_t>(helper.K());

    const size_t num_gemms = helper.OutputOffsets().size();
    std::vector<MLAS_GEMM_DYN_QUANT_DATA_PARAMS> gemm_data_vec(num_gemms);

    for (size_t gemm_idx = 0; gemm_idx < num_gemms; gemm_idx++) {
      auto& params = gemm_data_vec[gemm_idx];
      params.A = a_data + helper.LeftOffsets()[gemm_idx];
      params.lda = gemm_shape.K;
      params.PackedB = packed_b_.get();
      params.C = y_data + helper.OutputOffsets()[gemm_idx];
      params.ldc = gemm_shape.N;
    }

    MlasDynamicQGemmBatch(gemm_shape, gemm_data_vec.data(), num_gemms, ctx->GetOperatorThreadPool());
  }
#endif
  return Status::OK();
}

void MatMulIntegerToFloat::FixupScaleTensor(const Tensor*& a_scale_tensor, const Tensor*& b_scale_tensor) {
  const TensorShape a_scale_shape = a_scale_tensor->Shape();
  const TensorShape b_scale_shape = b_scale_tensor->Shape();
  if (!IsScalarOr1ElementVector(a_scale_tensor)) {
    size_t a_scale_rank = a_scale_shape.NumDimensions();
    if (a_scale_rank == 1 || a_scale_shape[a_scale_rank - 1] != 1) {
      std::swap(a_scale_tensor, b_scale_tensor);
    }
  } else if (!IsScalarOr1ElementVector(b_scale_tensor)) {
    size_t b_scale_rank = b_scale_shape.NumDimensions();
    if (b_scale_rank > 1 && b_scale_shape[b_scale_rank - 2] != 1) {
      std::swap(a_scale_tensor, b_scale_tensor);
    }
  }
}

Review comment (bug):
is this also a bug fix that should go into #25814?
it appears that the pointer arithmetic was incorrect before (adding left offset to a `uint8_t*` instead of a `float*`).

Code after:
                                       b_shape_,  // ctx->Input<Tensor>(IN_B)->Shape(), this is not available now constant data is
                                       // deleted during session init post prepacking
                                       nullptr,
                                       nullptr));

    Tensor* y = ctx->Output(OUT_Y, helper.OutputShape());

    // Bail out early if the output is going to be empty
    if (y->Shape().Size() == 0)
      return Status::OK();

    const float* a_data = ctx->Input<Tensor>(IN_A)->Data<float>();
    auto* y_data = y->MutableData<float>();

    // batch gemm
    MLAS_GEMM_DYN_QUANT_SHAPE_PARAMS gemm_shape;
    gemm_shape.M = static_cast<size_t>(helper.M());
    gemm_shape.N = static_cast<size_t>(helper.N());
    gemm_shape.K = static_cast<size_t>(helper.K());

    const size_t num_gemms = helper.OutputOffsets().size();
    std::vector<MLAS_GEMM_DYN_QUANT_DATA_PARAMS> gemm_data_vec(num_gemms);

    for (size_t gemm_idx = 0; gemm_idx < num_gemms; gemm_idx++) {
      auto& params = gemm_data_vec[gemm_idx];
      params.A = a_data + helper.LeftOffsets()[gemm_idx];
      params.lda = gemm_shape.K;
      params.PackedB = packed_b_.get();
      params.C = y_data + helper.OutputOffsets()[gemm_idx];
      params.ldc = gemm_shape.N;
    }

    MlasDynamicQGemmBatch(gemm_shape, gemm_data_vec.data(), num_gemms, ctx->GetOperatorThreadPool());
    // This evaluates to true if bias data was not provided as constant data for prepacking stage
    if (!dynamic_quant_mlas_bias_data_was_packed_) {
      if (ctx->Input<Tensor>(IN_BIAS) != nullptr) {
        const auto biases = std::vector<float>(&ctx->Input<Tensor>(IN_BIAS)->Data<float>()[0],
                                               &ctx->Input<Tensor>(IN_BIAS)->Data<float>()[gemm_shape.N]);

        // deferred adding of bias
        for (size_t gemm_idx = 0; gemm_idx < num_gemms; gemm_idx++) {
          float* MxN = y_data + helper.OutputOffsets()[gemm_idx];
          for (auto l = gemm_shape.M; l > 0; --l) {
            MlasEltwiseAdd<float>(MxN, biases.data(), MxN, gemm_shape.N);
            MxN += gemm_shape.N;
          }
        }
      }
    }
  }
#endif