\5CD-AI/visocial-Sailor-4B-Instruct~

app.py

@@ -3,9 +3,6 @@ from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
 import torch
 import json
 import os
-from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
-from mistral_common.protocol.instruct.messages import UserMessage
-from mistral_common.protocol.instruct.request import ChatCompletionRequest
 
 # model_name = "meta-llama/Meta-Llama-3-8B-Instruct" # uses more than 16GB
 # model_name = "meta-llama/Llama-2-7b-chat-hf" # only part of paid model
@@ -13,7 +10,10 @@ from mistral_common.protocol.instruct.request import ChatCompletionRequest
 model_name = "Qwen/Qwen2-7B-Instruct"
 # google/gemma-2-9b-it # 18GB
 # meta-llama/Meta-Llama-3-8B # 16GB
-# Qwen/Qwen2-7B-Instruct # 15GB
+# Qwen/Qwen2-7B-Instruct # 15GB # wouldn't run due to Memory Limit Exceeded
+
+# TODO try the following models:
+# mistralai/Mistral-7B-Instruct-v0.3
 
 
 
@@ -29,9 +29,8 @@ tokenizer = AutoTokenizer.from_pretrained(model_name, token=token)
 # model = AutoModelForCausalLM.from_pretrained(model_name, quantization_config=quantization_config)
 model = AutoModelForCausalLM.from_pretrained(model_name, token=token)
 
-special_tokens_dict = {'pad_token': tokenizer.eos_token}
-tokenizer.add_special_tokens(special_tokens_dict)
-model.resize_token_embeddings(len(tokenizer))
+tokenizer = AutoTokenizer.from_pretrained("5CD-AI/visocial-Sailor-4B-Instruct")
+model = AutoModelForCausalLM.from_pretrained("5CD-AI/visocial-Sailor-4B-Instruct")
 
 def generate_response(prompt):
     inputs = tokenizer(prompt, return_tensors="pt")