testing lyric generation again

LyricGeneratorModel.py

@@ -1,14 +1,19 @@
 from transformers import AutoTokenizer, AutoModelForCausalLM
 
+from peft import LoraConfig, get_peft_model, PeftModel, PeftConfig
+
 
 class LyricGeneratorModel:
     def __init__(self, repo_id: str):
-        self.model = AutoModelForCausalLM.from_pretrained(repo_id)
-
-        self.tokenizer = AutoTokenizer.from_pretrained(repo_id)
-        self.tokenizer.truncation_side = "right"
-        self.tokenizer.pad_token = self.tokenizer.eos_token
-        self.tokenizer.padding_side = "right"
+        config = PeftConfig.from_pretrained(repo_id)
+        model = AutoModelForCausalLM.from_pretrained(
+            config.base_model_name_or_path,
+            return_dict=True,
+            load_in_8bit=True,
+            device_map="auto",
+        )
+        self.tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)
+        self.model = PeftModel.from_pretrained(model, repo_id)
 
     def generate_lyrics(self, prompt: str, max_length: int):
         input_ids = self.tokenizer(prompt, return_tensors="pt").input_ids

requirements.txt

@@ -2,4 +2,5 @@ transformers==4.40.0
 huggingface_hub==0.22.2
 torch==2.2.2
 numpy==1.26.4
-pandas==2.2.2
+pandas==2.2.2
+peft==0.10.0