Create models.py

models.py

@@ -0,0 +1,44 @@
+from huggingface_hub import InferenceClient
+
+class LlmBot():
+    def __init__(self, model):
+        self.client = InferenceClient(model)
+
+    def character_prompt(self, dict, max_new_tokens):
+        system_prompt = '<SYSTEM> <'
+        if dict["name"]:
+            system_prompt += f'the person whose name :{dict["name"]}.'
+        if dict["description"]:
+            system_prompt += 'Your description :{dict["description"]}.'
+        if dict["user_name"]:
+            system_prompt += f'users name :{dict["user_name"]}.'
+        system_prompt += 'Do not add the greeting, only at the first request.'
+        system_prompt += 'Be emotional in your responses.'
+        system_prompt += 'Do not include your own name in any responses.'
+        system_prompt += f'ensure responses are shorter than {max_new_tokens} tokens.>'
+        return system_prompt
+
+    def format_prompt(self, prompt, history, system_setting):
+        formatted_prompt = "<history>"
+        for user_prompt, bot_response in history:
+            formatted_prompt += f"[INST] {user_prompt} [/INST] {bot_response} </history> "
+        formatted_prompt += f"[INST] {system_setting}, <user>{prompt}</user> [/INST]"
+        return formatted_prompt
+
+    def call(self, prompt, history, name, description, user_name, max_new_tokens):
+        generate_kwargs = dict(
+            temperature=0.9,
+            max_new_tokens=max_new_tokens,
+            top_p=0.95,
+            repetition_penalty=1.0,
+            do_sample=True,
+            )
+        system_setting = self.character_prompt({"name": name, "description": description, "user_name": user_name}, max_new_tokens)
+        formatted_prompt = self.format_prompt(prompt, history, system_setting)
+        stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
+        output = ""
+        for response in stream:
+            output += response.token.text
+            yield output
+        return output
+