Update README.md

README.md

@@ -35,12 +35,26 @@ from unsloth import FastLlamaModel
 max_seq_length = 2048
 dtype = None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+
 load_in_4bit = True # Use 4bit quantization to reduce memory usage. Can be False.
-model, tokenizer = FastLlamaModel.from_pretrained(
-    model_name = "Finnish-NLP/llama-7b-finnish-instruct-v0.1"
-    max_seq_length = max_seq_length,
-    dtype = dtype,
-    load_in_4bit = load_in_4bit
-)
+
+
+use_unsloth = True
+# use_transformers = True
+
+# LOADING MODEL USIINIG TRANSFORMERS assumes at least 16GB of memory. Tested with this configuration
+# If you have less memory use load_in_4bit or load_in_8_bit as needed
+if use_transformers:
+  major_version, minor_version = torch.cuda.get_device_capability()
+  model = AutoModelForCausalLM.from_pretrained("Finnish-NLP/llama-7b-finnish-instruct-v0.1", device_map='cuda:0', torch_dtype = torch.bfloat16 if major_version >=8 else torch.float16)
+  tokenizer = AutoTokenizer.from_pretrained("Finnish-NLP/llama-7b-finnish-instruct-v0.1")
+
+# USING UNSLOTH, tested with load_in_4bit
+if use_unsloth:
+  model, tokenizer = FastLlamaModel.from_pretrained(
+      model_name = "Finnish-NLP/llama-7b-finnish-instruct-v0.1"
+      max_seq_length = max_seq_length,
+      dtype = dtype,
+      load_in_4bit = load_in_4bit
+  )
 
 alpaca_prompt = """<|alku|> Olet tekoälyavustaja. Seuraavaksi saat kysymyksen tai tehtävän. Kirjoita vastaus parhaasi mukaan siten että se täyttää kysymyksen tai tehtävän vaatimukset.
 <|ihminen|> Kysymys/Tehtävä: