Repository: microsoft/onnxruntime
File: onnxruntime/python/tools/transformers/models/whisper/convert_to_onnx.py (Python)

Code before:
    # Quantization options for Whisper
    ###################################

    quant_args.add_argument(
        "--quantize_embedding_layer",
        required=False,
        action="store_true",
        help="Quantize MatMul, GEMM, and Gather.",
    )
    quant_args.set_defaults(quantize_embedding_layer=False)

    quant_args.add_argument(
        "--quantize_per_channel",
        required=False,
        action="store_true",
        help="Quantize weights per each channel.",
    )
    quant_args.set_defaults(quantize_per_channel=False)

    quant_args.add_argument(
        "--quantize_reduce_range",
        required=False,
        action="store_true",
        help="Quantize weights with 7 bits.",
    )
    quant_args.set_defaults(quantize_reduce_range=False)

    args = parser.parse_args(argv)

    # Collect cross QKs if either flag is enabled
    args.collect_cross_qk = args.collect_cross_qk or args.output_cross_qk

    # FP32 CPU can be supported here once the DMMHA CPU kernel bugs are fixed
    args.use_decoder_masked_mha = args.use_decoder_masked_mha and args.provider == "cuda"

    return args


def make_quant_algo_config(precision, quant_method: str, matmul_nodes=None):
    customized_weight_config = {}
    quant_algo_config = None

    # need to use k_quant for int8
    if precision == Precision.INT8:
        for node_name in matmul_nodes:
            customized_weight_config[node_name] = {"bits": 8}
        quant_algo_config = KQuantWeightOnlyQuantConfig(customized_weight_config=customized_weight_config)
    else:
        if quant_method == "rtn":
            quant_algo_config = RTNWeightOnlyQuantConfig()
        elif quant_method == "k_quant":

Review comment (suggestion):
Now that int8 dynamic quantization is being replaced by int4/int8 mixed quantization, we should update the following quantization arguments to work with `MatMulNBitsQuantizer` instead of `quantize_dynamic`.

https://github.com/microsoft/onnxruntime/blob/0431c5687e41c7af8b235d310eece779d60d8f09/onnxruntime/python/tools/transformers/models/whisper/convert_to_onnx.py#L294-L320

Here are some example quantization arguments from [`matmul_nbits_quantizer.py`](https://github.com/microsoft/onnxruntime/blob/main/onnxruntime/python/tools/quantization/matmul_nbits_quantizer.py) we can consider adding.

```py
quant_args.add_argument( 
    "--quant_method", 
    default="k_quant",
    type=str,
    choices=["default", "rtn", "k_quant"],
    help="Quantization algorithm to select", 
)

quant_args.add_argument( 
    "--op_types_to_quantize",
    type=str,
    nargs="+",
    choices=["MatMul", "Gather"],
    help="op_types_to_quantize {MatMul, Gather}. Operators to quantize. Default is MatMul.",
)

quant_args.add_argument(
    "--accuracy_level",
    default=0,  # change to 4 for CPU EP
    required=False,
    type=int,
    help="Accuracy level of the 4-bit quantized MatMul computation.",
)

quant_args.add_argument( 
    "--quantize_symmetric", 
    required=False, 
    action="store_true", 
    help="Quantize weights symmetrically", 
)
quant_args.set_defaults(quantize_symmetric=False)
```

Code after:

    quant_args.add_argument(
        "--accuracy_level",
        default=0,
        required=False,
        type=int,
        help="Accuracy level of the 4-bit quantized MatMul computation.",
    )

    quant_args.add_argument(
        "--quantize_symmetric",
        required=False,
        action="store_true",
        help="Quantize weights symmetrically",
    )
    quant_args.set_defaults(quantize_symmetric=False)

    args = parser.parse_args(argv)

    # Collect cross QKs if either flag is enabled
    args.collect_cross_qk = args.collect_cross_qk or args.output_cross_qk

    # FP32 CPU can be supported here once the DMMHA CPU kernel bugs are fixed
    args.use_decoder_masked_mha = args.use_decoder_masked_mha and args.provider == "cuda"

    return args


# quant_method is reserved for mixed precision in future
def make_quant_algo_config(precision, quant_method: str, matmul_nodes=None):
    customized_weight_config = {}
    quant_algo_config = None

    # need to use k_quant for int8
    if precision == Precision.INT8:
        for node_name in matmul_nodes:
            customized_weight_config[node_name] = {"bits": 8}
        quant_algo_config = KQuantWeightOnlyQuantConfig(customized_weight_config=customized_weight_config)
    else:
        quant_algo_config = KQuantWeightOnlyQuantConfig(customized_weight_config=customized_weight_config)

    return quant_algo_config


def export_onnx_models(
    model_name_or_path,
    model_impl,
    cache_dir,
    output_dir,
    use_gpu,
    use_external_data_format,