Adding a simple monkey search for Leetcode - Darn LeetMonkey

app.py

@@ -5,7 +5,7 @@ from pinecone_text.sparse import SpladeEncoder
 from sentence_transformers import SentenceTransformer
 import transformers
 transformers.logging.set_verbosity_error()
-from transformers import AutoTokenizer, AutoModelForCausalLM, GPTQConfig
+from transformers import AutoTokenizer, AutoModelForCausalLM
 
 
 import os
@@ -16,11 +16,6 @@ pc = Pinecone(api_key=PINECONE_API_KEY)
 index_name = "leetmonkey-sparse-dense"
 index = pc.Index(index_name)
 
-quantization_config = GPTQConfig(
-    bits=8,
-    disable_exllama=True
-)
-
 
 
 # Initialize models
@@ -29,9 +24,14 @@ splade = SpladeEncoder(device=device)
 dense_model = SentenceTransformer('sentence-transformers/all-Mpnet-base-v2', device=device)
 
 # Load the quantized Llama 2 model and tokenizer
-model_name = "TheBloke/Llama-2-7B-Chat-GPTQ"
+model_name = "TheBloke/Llama-2-7B-Chat-GGML"
 tokenizer = AutoTokenizer.from_pretrained(model_name)
-model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto", quantization_config=quantization_config)
+model = AutoModelForCausalLM.from_pretrained(model_name, low_cpu_mem_usage=True)
+
+
+# Disable Exllama backend if needed
+if hasattr(model, 'quantization_config'):
+    model.quantization_config.use_exllama = False
 
 def search_problems(query, top_k=5):
     dense_query = dense_model.encode([query])[0].tolist()
@@ -77,8 +77,7 @@ def generate_response(user_query, top_k=5):
     with torch.no_grad():
         output = model.generate(
             input_ids,
-            attention_mask=attention_mask,
-            max_new_tokens=250,
+            max_new_tokens=100,  # Reduce this for faster generation
             do_sample=True,
             top_p=0.9,
             temperature=0.7,