Update app.py

app.py

@@ -17,7 +17,7 @@ MODEL_NAME="xu3kev/deepseekcoder-7b-logo-pbe"
 # MODEL_NAME="openlm-research/open_llama_3b" 
 import torch
 from transformers import AutoModelForCausalLM, AutoTokenizer
-hug_model = AutoModelForCausalLM.from_pretrained(MODEL_NAME, torch_dtype=torch.float16, device_map='auto')
+hug_model = AutoModelForCausalLM.from_pretrained(MODEL_NAME, torch_dtype=torch.float16, device_map='auto', load_in_8bit=True)
 hug_tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
 
 INPUT_STRUCTION_TEMPLATE = """Here is a gray scale images representing with integer values 0-9.
@@ -223,7 +223,7 @@ def llm_call(question_prompt, model_name,
     top_p=1, n_samples=64, stop=None):
     if HUGGINGFACE:
         model_inputs = hug_tokenizer([question_prompt], return_tensors="pt").to('cuda')
-        generated_ids = hug_model.generate(**model_inputs, max_length=1400, temperature=1, num_return_sequences=8, do_sample=True)
+        generated_ids = hug_model.generate(**model_inputs, max_length=1400, temperature=1, num_return_sequences=32, do_sample=True)
         responses = hug_tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
         codes = []
         for response in responses: