Repository: unslothai/unsloth
File: unsloth/models/_utils.py (Python)

Code before:
        if not vllm_dynamic_quant_supported(model_name, model_config):
            # Instead use -bnb-4bit variant
            logger.warning_once(
                f"Unsloth: Switching from Unsloth dynamic quant to normal quant since\n"\
                f"we do not yet support fast inference for {model_name}"
            )
            model_name = model_name[:-len("unsloth-bnb-4bit")] + "bnb-4bit"
        pass
    pass
    return fast_inference

def patch_peft_fast_inference(model):
    vllm_engine = getattr(model.model, "vllm_engine", None)
    if vllm_engine is not None:
        model.vllm_engine = model.model.vllm_engine
        model.fast_generate = model.model.fast_generate
        model.fast_generate_batches = model.model.fast_generate_batches

        # Also saving and loading LoRA
        from unsloth_zoo.vllm_utils import save_lora, load_lora
        model.save_lora = functools.partial(save_lora, model)
        model.load_lora = functools.partial(load_lora, model)
    pass

def error_out_no_vllm(*args, **kwargs):
    raise NotImplementedError("vLLM is not yet supported for fast inference for this model! Please use `.generate` instead")


Review comment (suggestion):
```suggestion
    raise NotImplementedError("Unsloth: vLLM is not yet supported for fast inference for this model! Please use `.generate` instead")
```

Code after:
    pass

    if not (type(init_lora_weights) is bool or \
        init_lora_weights == "gaussian" or init_lora_weights == "loftq"):
        raise ValueError(
            'Unsloth: `init_lora_weights` must be either [True, False, "gaussian", "loftq"].'
        )
    pass

    if init_lora_weights == "loftq":

        if not SUPPORTS_LOFTQ:
            import peft
            raise RuntimeError(
                f"Unsloth: Your PEFT version of {peft.__version__} does not support LoftQ init.\n"\
                "Please install PEFT 0.7.2 or higher.\n"\
                "You can also install from source: `pip install git+https://github.com/huggingface/peft.git"
            )
        pass

        if loftq_config == {}:
            from peft import LoftQConfig
            logger.warning_once(
                "Unsloth: init_lora_weights = `loftq` is set, but `loftq_config` is None.\n"\
                "We shall use `loftq_config = LoftQConfig(loftq_bits = 4, loftq_iter = 1)`."
            )
            loftq_config = LoftQConfig(loftq_bits = 4, loftq_iter = 1)
        pass

        if hasattr(model.config, "quantization_config"):
            raise ValueError(
                "Unsloth: You are using `loftq` init, yet `load_in_4bit = True` was set.\n"\
                "Reload your model without any quantization by setting `load_in_4bit = False`."
            )
        pass
    pass

    return loftq_config

def fast_inference_setup(model_name, model_config):
    fast_inference = True
    if not is_vLLM_available():
        logger.warning_once("Unsloth: vLLM is not installed! Will use Unsloth inference!")
        fast_inference = False
    pass
    from unsloth_zoo.vllm_utils import (
        patch_vllm,
        vllm_dynamic_quant_supported,
    )
    patch_vllm()
    if model_name.endswith("unsloth-bnb-4bit"):