Qwen/Qwen2-1.5B-Instruct

app.py

@@ -7,13 +7,14 @@ import os
 # model_name = "meta-llama/Meta-Llama-3-8B-Instruct" # uses more than 16GB
 # model_name = "meta-llama/Llama-2-7b-chat-hf" # only part of paid model
 # model_name = "mistralai/Mistral-7B-Instruct-v0.1" # try this with quantisation to reduce memory usage; needs a GPU to run
-model_name = "Qwen/Qwen2-7B-Instruct"
+model_name = "Qwen/Qwen2-1.5B-Instruct"
 # google/gemma-2-9b-it # 18GB
 # meta-llama/Meta-Llama-3-8B # 16GB
 # Qwen/Qwen2-7B-Instruct # 15GB # wouldn't run due to Memory Limit Exceeded
 
 # TODO try the following models:
 # mistralai/Mistral-7B-Instruct-v0.3
+# google/flan-t5
 
 
 
@@ -29,9 +30,6 @@ tokenizer = AutoTokenizer.from_pretrained(model_name, token=token)
 # model = AutoModelForCausalLM.from_pretrained(model_name, quantization_config=quantization_config)
 model = AutoModelForCausalLM.from_pretrained(model_name, token=token)
 
-tokenizer = AutoTokenizer.from_pretrained("5CD-AI/visocial-Sailor-4B-Instruct")
-model = AutoModelForCausalLM.from_pretrained("5CD-AI/visocial-Sailor-4B-Instruct")
-
 def generate_response(prompt):
     inputs = tokenizer(prompt, return_tensors="pt")
     with torch.no_grad():