Update app.py

app.py

@@ -41,7 +41,10 @@ frame_asr = FrameBatchMultiTaskAED(
 
 amp_dtype = torch.float16
 
-llm_pipeline = transformers.pipeline("text-generation", model="meta-llama/Meta-Llama-3-8B-Instruct")
+
+tokenizer = transformers.AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-8B-Instruct")
+llm_model = transformers.AutoModelForCausalLM.from_pretrained("meta-llama/Meta-Llama-3-8B-Instruct")
+accelerate.disk_offload(model=llm_model, offload_dir="../offload/")
 
 def convert_audio(audio_filepath, tmpdir, utt_id):
 	"""
@@ -224,7 +227,7 @@ def on_src_or_tgt_lang_change(src_lang_value, tgt_lang_value, pnc_value):
 
 def main(audio_filepath, src_lang, tgt_lang, pnc):
     translated = transcribe(audio_filepath, src_lang, tgt_lang, pnc)
-    answer = llm_pipeline(translated)
+    answer = llm_model(translated)
     return answer