feat: update model

load_models.py

@@ -66,10 +66,10 @@ def load_quantized_model_gguf_ggml(model_id, model_basename, device_type, loggin
             kwargs["n_gpu_layers"] = N_GPU_LAYERS
             kwargs["n_batch"] = MAX_NEW_TOKENS  # set this based on your GPU
 
-        kwargs["stream"] = stream
+        # kwargs["stream"] = stream
 
-        if stream == True:
-            kwargs["callbacks"] = callbacks
+        # if stream == True:
+        #     kwargs["callbacks"] = callbacks
 
         return LlamaCpp(**kwargs)
     except:

main.py

@@ -29,12 +29,14 @@ class Predict(BaseModel):
 class Delete(BaseModel):
     filename: str
 
-if torch.backends.mps.is_available():
-    DEVICE_TYPE = "mps"
-elif torch.cuda.is_available():
-    DEVICE_TYPE = "cuda"
-else:
-    DEVICE_TYPE = "cpu"
+# if torch.backends.mps.is_available():
+#     DEVICE_TYPE = "mps"
+# elif torch.cuda.is_available():
+#     DEVICE_TYPE = "cuda"
+# else:
+#     DEVICE_TYPE = "cpu"
+
+DEVICE_TYPE = "cuda"
 
 EMBEDDINGS = HuggingFaceInstructEmbeddings(model_name=EMBEDDING_MODEL_NAME, model_kwargs={"device": DEVICE_TYPE})
 DB = Chroma(persist_directory=PERSIST_DIRECTORY, embedding_function=EMBEDDINGS, client_settings=CHROMA_SETTINGS)
@@ -97,9 +99,9 @@ def run_ingest_route():
 
         run_langest_commands = ["python", "ingest.py"]
 
-        if DEVICE_TYPE == "cpu":
-            run_langest_commands.append("--device_type")
-            run_langest_commands.append(DEVICE_TYPE)
+        # if DEVICE_TYPE == "cpu":
+        #     run_langest_commands.append("--device_type")
+        #     run_langest_commands.append(DEVICE_TYPE)
 
         result = subprocess.run(run_langest_commands, capture_output=True)