qlora and 4bit check so we are able to merge and unload
Browse files
src/axolotl/utils/models.py
CHANGED
@@ -85,7 +85,7 @@ def load_model(
|
|
85 |
raise e
|
86 |
|
87 |
model_kwargs = {}
|
88 |
-
if cfg.adapter == "qlora":
|
89 |
model_kwargs["quantization_config"] = BitsAndBytesConfig(
|
90 |
load_in_4bit=True,
|
91 |
llm_int8_threshold=6.0,
|
|
|
85 |
raise e
|
86 |
|
87 |
model_kwargs = {}
|
88 |
+
if cfg.adapter == "qlora" and cfg.load_in_4bit:
|
89 |
model_kwargs["quantization_config"] = BitsAndBytesConfig(
|
90 |
load_in_4bit=True,
|
91 |
llm_int8_threshold=6.0,
|