prep to test on machine with gpu

.gitignore

@@ -1,3 +1,4 @@
 
 .env
 .DS_Store
+__pycache__

chat_service.py

@@ -0,0 +1,63 @@
+import os
+import torch
+from transformers import AutoTokenizer, AutoModelForCausalLM
+
+# from huggingface_hub.inference_api import InferenceApi
+
+class ChatService:
+    def __init__(self, api="huggingface", repo_id = "OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5"):
+        self._api = api
+        self._device = "cuda:0" if torch.cuda.is_available() else "cpu"
+
+        if self._api=="huggingface":
+            self._tokenizer = AutoTokenizer.from_pretrained(repo_id)
+            self._model = AutoModelForCausalLM.from_pretrained(repo_id,torch_dtype=torch.float16)
+            # self._model = AutoModelForCausalLM.from_pretrained(repo_id).half()
+            self._model = AutoModelForCausalLM.from_pretrained(repo_id)
+            self._model.eval().to(self._device)
+        else:
+            raise Exception(f"Unknown API: {self._api}")
+
+        self._system_prompt = "Below are a series of dialogues between various people and an AI assistant. The AI tries to be helpful, polite, honest, sophisticated, emotionally aware, and humble-but-knowledgeable. The assistant is happy to help with almost anything, and will do its best to understand exactly what is needed. It also tries to avoid giving false or misleading information, and it caveats when it isn't entirely sure about the right answer. That said, the assistant is practical and really does its best, and doesn't let caution get too much in the way of being useful.\n-----\n"
+        self._user_name = "<|prompter|>"
+        self._agent_name = "<|assistant|>"
+        self.reset()
+
+    def reset(self):
+        self._user_history = []
+        self._agent_history = []
+        self._full_history = self._user_history if self._user_history else ""
+
+
+    def _chat(self, prompt):
+        if self._api=="huggingface":
+            tokens = self._tokenizer.encode(prompt, return_tensors="pt", padding=True)
+            tokens = tokens.to(self._device)
+            outputs = self._model.generate(
+                tokens,
+                early_stopping=True,
+                max_new_tokens=200,
+                do_sample=True,
+                top_k=40,
+                temperature=1.0,
+                pad_token_id=self._tokenizer.eos_token_id,
+            )
+            agent_response = self._tokenizer.decode(outputs[0], truncate_before_pattern=[r"\n\n^#", "^'''", "\n\n\n"])
+
+        else:
+            raise Exception(f"API not implemented: {self._api}")
+        return agent_response
+    
+    def chat(self, prompt):
+        if self._user_name:
+            self._full_history += f"{self._user_name}: {prompt}\n"
+        else:
+            self._full_history += f"{prompt}\n"
+        self._user_history.append(prompt)
+        agent_response = self._chat(self._full_history)
+        if self._agent_name:
+            self._full_history += f"{self._agent_name}: {agent_response}\n"
+        else:
+            self._full_history += f"{agent_response}\n"
+        self._agent_history.append(agent_response)
+        return agent_response

clip_transform.py

@@ -26,7 +26,7 @@ class CLIPTransform:
         self._clip_model="ViT-L-14"
         self._pretrained='datacomp_xl_s13b_b90k'
 
-        self.model, _, self.preprocess = open_clip.create_model_and_transforms(self._clip_model, pretrained=self._pretrained)
+        self.model, _, self.preprocess = open_clip.create_model_and_transforms(self._clip_model, pretrained=self._pretrained,device=self.device)
         self.tokenizer = open_clip.get_tokenizer(self._clip_model)
 
         print ("using device", self.device)        

debug.py

@@ -1,4 +1,25 @@
 from clip_transform import CLIPTransform
+from chat_service import ChatService
+from dotenv import load_dotenv
+
+
+load_dotenv()
+
+print ("Initializing CLIP templates")
 clip_transform = CLIPTransform()
+print ("CLIP success")
+
+print ("Initializing Chat")
+chat_service = ChatService()
+prompts = [
+    "hello, how are you today?",
+    "tell me about your showdow self?",
+    "hmm, interesting, tell me more about that.",
+    "wait, that is so interesting, what else?",
+]
+for prompt in prompts:
+    print (f'prompt: "{prompt}"')
+    response = chat_service.chat(prompt)
+    print (f'response: "{response}"')
 
-print ("Initializing CLIP templates")
+print ("Chat success")