Feature(MInference): changing to GPU

app.py

@@ -1,10 +1,10 @@
-import subprocess
-# Install flash attention, skipping CUDA build if necessary
-subprocess.run(
-    "pip install flash-attn --no-build-isolation",
-    env={"FLASH_ATTENTION_SKIP_CUDA_BUILD": "TRUE"},
-    shell=True,
-)
+# import subprocess
+# # Install flash attention, skipping CUDA build if necessary
+# subprocess.run(
+#     "pip install flash-attn --no-build-isolation",
+#     env={"FLASH_ATTENTION_SKIP_CUDA_BUILD": "TRUE"},
+#     shell=True,
+# )
 
 import gradio as gr
 import os
@@ -59,6 +59,10 @@ model_name = "gradientai/Llama-3-8B-Instruct-262k"
 tokenizer = AutoTokenizer.from_pretrained(model_name)
 model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto")  # to("cuda:0") 
 
+from minference import MInference
+minference_patch = MInference("minference", model_name)
+model = minference_patch(model)
+
 terminators = [
     tokenizer.eos_token_id,
     tokenizer.convert_tokens_to_ids("<|eot_id|>")
@@ -80,7 +84,7 @@ def chat_llama3_8b(message: str,
     Returns:
         str: The generated response.
     """
-    global model
+    # global model
     conversation = []
     for user, assistant in history:
         conversation.extend([{"role": "user", "content": user}, {"role": "assistant", "content": assistant}])

requirements.txt

@@ -1,3 +1,5 @@
 triton==2.1.0
 accelerate
-transformers
+transformers
+flash_attn
+pycuda==2023.1