bitsandbytes added, checking with gpu

README.md

@@ -8,9 +8,10 @@ sdk_version: 4.26.0
 app_file: app.py
 pinned: false
 preload_from_hub:
-  - "microsoft/phi-2"
   - "BAAI/bge-small-en-v1.5"
   - "HuggingFaceH4/zephyr-7b-alpha"
   - "meta-llama/Meta-Llama-3-8B"
 ---
 
+- "microsoft/phi-2"
+

app.py

@@ -20,12 +20,12 @@ CHEAPMODE = torch.cuda.is_available()
 # LLM = "HuggingFaceH4/zephyr-7b-alpha" if not CHEAPMODE else "microsoft/phi-2"
 
 config = {
-    # "LLM": "meta-llama/Meta-Llama-3-8B",
-    "LLM": "microsoft/phi-2",
+    "LLM": "meta-llama/Meta-Llama-3-8B",
+    # "LLM": "microsoft/phi-2",
     # "LLM": "HuggingFaceH4/zephyr-7b-alpha",
     "embeddings": "BAAI/bge-small-en-v1.5",
     "similarity_top_k": 2,
-    "context_window": 4048,
+    "context_window": 2048,
     "max_new_tokens": 200,
     "temperature": 0.7,
     "top_k": 5,
@@ -184,7 +184,7 @@ def ask_my_thesis(
         config["temperature"] = temperature
         config["top_p"] = top_p
         config["max_new_tokens"] = max_new_tokens
-        # config["LLM"] = LLM
+        config["LLM"] = LLM
         # config["embeddings"] = embeddings
         config["similarity_top_k"] = similarity_top_k
         config["context_window"] = context_window

requirements.txt

@@ -7,4 +7,6 @@ tokenizers
 llama-index
 llama-index-embeddings-huggingface
 llama-index-llms-huggingface
-einops
+einops
+accelerate
+bitsandbytes