Repository: unslothai/unsloth
File: unsloth/models/vision.py (Python)

Code before:

        raise_handler = RaiseUninitialized()
        if not fast_inference:
            model = auto_model.from_pretrained(
                model_name,
                device_map              = device_map,
                torch_dtype             = torch_dtype,
                # quantization_config   = bnb_config,
                token                   = token,
                trust_remote_code       = trust_remote_code,
                # attn_implementation   = attn_implementation,
                **kwargs,
            )
            model.fast_generate = model.generate
            model.fast_generate_batches = error_out_no_vllm
        else:
            from unsloth_zoo.vllm_utils import (
                load_vllm,
                get_vllm_state_dict,
                convert_vllm_to_huggingface,
                generate_batches,
            )
            model_config = AutoConfig.from_pretrained(
                model_name,
                token = token,
                attn_implementation = "sdpa",
            )

            if fast_inference:
                fast_inference = fast_inference_setup(model_name, model_config)

            allowed_args = inspect.getfullargspec(load_vllm).args
            load_vllm_kwargs = dict(
                model_name             = model_name,
                config                 = model_config,
                gpu_memory_utilization = gpu_memory_utilization,
                max_seq_length         = max_seq_length,
                dtype                  = dtype,
                float8_kv_cache        = float8_kv_cache,
                enable_lora            = vllm_enable_lora,
                max_lora_rank          = max_lora_rank,
                disable_log_stats      = disable_log_stats,
                use_bitsandbytes       = load_in_4bit,
                unsloth_vllm_standby   = unsloth_vllm_standby,
                is_vision_model        = is_vlm,
            )
            for allowed_arg in allowed_args:
                if allowed_arg not in load_vllm_kwargs and allowed_arg in kwargs:
                    load_vllm_kwargs[allowed_arg] = kwargs[allowed_arg]
            pass


Review comment (suggestion):
Use `supports_sdpa` --> confirm if this works. It's cause sometimes some models (GPT-OSS) doesn't have SDPA

Code after:
        raise_handler = RaiseUninitialized()
        if not fast_inference:
            model = auto_model.from_pretrained(
                model_name,
                device_map              = device_map,
                # torch_dtype           = torch_dtype, # Transformers removed torch_dtype
                # quantization_config   = bnb_config,
                token                   = token,
                trust_remote_code       = trust_remote_code,
                # attn_implementation   = attn_implementation,
                **kwargs,
            )
            model.fast_generate = model.generate
            model.fast_generate_batches = error_out_no_vllm
        else:
            from unsloth_zoo.vllm_utils import (
                load_vllm,
                get_vllm_state_dict,
                convert_vllm_to_huggingface,
                generate_batches,
            )
            model_config = AutoConfig.from_pretrained(
                model_name,
                token = token,
                attn_implementation = "sdpa" if supports_sdpa else "eager",
            )

            if fast_inference:
                fast_inference, model_name = fast_inference_setup(model_name, model_config)

            allowed_args = inspect.getfullargspec(load_vllm).args
            load_vllm_kwargs = dict(
                model_name             = model_name,
                config                 = model_config,
                gpu_memory_utilization = gpu_memory_utilization,
                max_seq_length         = max_seq_length,
                dtype                  = dtype,
                float8_kv_cache        = float8_kv_cache,
                enable_lora            = vllm_enable_lora,
                max_lora_rank          = max_lora_rank,
                disable_log_stats      = disable_log_stats,
                use_bitsandbytes       = load_in_4bit,
                unsloth_vllm_standby   = unsloth_vllm_standby,
                is_vision_model        = is_vlm,
            )
            for allowed_arg in allowed_args:
                if allowed_arg not in load_vllm_kwargs and allowed_arg in kwargs:
                    load_vllm_kwargs[allowed_arg] = kwargs[allowed_arg]
            pass

            # Load vLLM first