Repository: oumi-ai/oumi
File: src/oumi/core/configs/params/peft_params.py (Python)

Code before:

    If True, enables quantization for more memory-efficient fine-tuning.
    """

    q_lora_bits: int = field(
        default=4, metadata={"help": "Quantization (precision) bits."}
    )
    """The number of bits to use for quantization in Q-LoRA.

    This is only used if `q_lora` is True.

    Defaults to 4-bit quantization.
    """

    # FIXME the names below use the bnb short for bits-and-bytes
    # If we consider wrapping more quantization libraries a better
    # naming convention should be applied.
    bnb_4bit_quant_type: str = field(
        default="fp4", metadata={"help": "4-bit quantization type (fp4 or nf4)."}
    )
    """The type of 4-bit quantization to use.

    Can be 'fp4' (float point 4) or 'nf4' (normal float 4).
    """

    use_bnb_nested_quant: bool = field(
        default=False, metadata={"help": "Use nested quantization."}
    )
    """Whether to use nested quantization.

    Nested quantization can provide additional memory savings.
    """

    bnb_4bit_quant_storage: str = field(
        default="uint8",
        metadata={"help": "Storage type to pack the quantized 4-bit params."},
    )
    """The storage type for packing quantized 4-bit parameters.

    Defaults to 'uint8' for efficient storage.
    """

    bnb_4bit_compute_dtype: str = field(
        default="float32",
        metadata={"help": "The compute type of the quantized parameters."},
    )
    """Compute type of the quantized parameters.
    It can be different than the input type, e.g., it can be set to a lower precision
    for improved speed.

    The string will be converted to the corresponding torch.dtype.

Review comment (suggestion):
@stefanwebb since this is a new param could you add it to the docs here `docs/user_guides/train/configuration.md` 

Code after:

    If True, enables quantization for more memory-efficient fine-tuning.
    """

    q_lora_bits: int = field(
        default=4, metadata={"help": "Quantization (precision) bits."}
    )
    """The number of bits to use for quantization in Q-LoRA.

    This is only used if `q_lora` is True.

    Defaults to 4-bit quantization.
    """

    # FIXME the names below use the bnb short for bits-and-bytes
    # If we consider wrapping more quantization libraries a better
    # naming convention should be applied.
    bnb_4bit_quant_type: str = field(
        default="fp4", metadata={"help": "4-bit quantization type (fp4 or nf4)."}
    )
    """The type of 4-bit quantization to use.

    Can be 'fp4' (float point 4) or 'nf4' (normal float 4).
    """

    llm_int8_skip_modules: Optional[list[str]] = field(
        default=None,
        metadata={"help": "Modules that we do not want to convert in 8-bit"},
    )
    """ An explicit list of the modules that we do not want to convert in 8-bit.

    """

    use_bnb_nested_quant: bool = field(
        default=False, metadata={"help": "Use nested quantization."}
    )
    """Whether to use nested quantization.

    Nested quantization can provide additional memory savings.
    """

    bnb_4bit_quant_storage: str = field(
        default="uint8",
        metadata={"help": "Storage type to pack the quantized 4-bit params."},
    )
    """The storage type for packing quantized 4-bit parameters.

    Defaults to 'uint8' for efficient storage.
    """

    bnb_4bit_compute_dtype: str = field(