Feature(MInference): add minference

app.py

@@ -81,20 +81,21 @@ def chat_llama3_8b(message: str,
         str: The generated response.
     """
     global model
-    subprocess.run(
-        "pip install pycuda==2023.1",
-        shell=True,
-    )
-    if "has_patch" not in model.__dict__:
-        from minference import MInference
-        minference_patch = MInference("minference", model_name)
-        model = minference_patch(model)
     conversation = []
     for user, assistant in history:
         conversation.extend([{"role": "user", "content": user}, {"role": "assistant", "content": assistant}])
     conversation.append({"role": "user", "content": message})
 
     input_ids = tokenizer.apply_chat_template(conversation, return_tensors="pt").to(model.device)
+    print(model.device)
+    # subprocess.run(
+    #     "pip install pycuda==2023.1",
+    #     shell=True,
+    # )
+    # if "has_patch" not in model.__dict__:
+    #     from minference import MInference
+    #     minference_patch = MInference("minference", model_name)
+    #     model = minference_patch(model)
     
     streamer = TextIteratorStreamer(tokenizer, timeout=10.0, skip_prompt=True, skip_special_tokens=True)