Upload 13 files

app.py

@@ -0,0 +1,26 @@
+from handler import SweetCommander
+import gradio as gr
+
+controller = SweetCommander()
+
+with gr.Blocks() as demo:
+    history = gr.State([])
+    with gr.Row() as row:
+        with gr.Column():
+            user_name = gr.Textbox(label="Name", placeholder="Enter your name")
+            user_input = gr.Textbox(label="Input", placeholder="Enter your message")
+            button = gr.Button("Enter")
+        with gr.Column():
+            output = gr.Textbox(label="Response")
+
+    def guess_letter(user_name, user_input):
+        response = controller(user_name, user_input)
+        return {
+            output: response
+        }
+    button.click(
+        guess_letter, 
+        [user_name, user_input],
+        [output]
+    )
+demo.launch()

config.json

@@ -0,0 +1,31 @@
+{
+  "_name_or_path": "PygmalionAI/pygmalion-350m",
+  "_remove_final_layer_norm": false,
+  "activation_dropout": 0.0,
+  "activation_function": "relu",
+  "architectures": [
+    "OPTForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 2,
+  "do_layer_norm_before": false,
+  "dropout": 0.1,
+  "enable_bias": true,
+  "eos_token_id": 2,
+  "ffn_dim": 4096,
+  "hidden_size": 1024,
+  "init_std": 0.02,
+  "layer_norm_elementwise_affine": true,
+  "layerdrop": 0.0,
+  "max_position_embeddings": 2048,
+  "model_type": "opt",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "pad_token_id": 1,
+  "prefix": "</s>",
+  "torch_dtype": "float16",
+  "transformers_version": "4.28.1",
+  "use_cache": true,
+  "vocab_size": 50272,
+  "word_embed_proj_dim": 512
+}

decoder_model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4398774c78dec9cabe319bc90dcce3e2173789140e1f70a88ec19cd53638233
+size 1428394786

decoder_model_merged.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b037122d632abbfcc5b0f27ee3ffa05fc40fd19eb339f856d3d3e5bdd524d351
+size 1429049605

decoder_with_past_model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4c1ebacb3e265ba44c6e122c46d1144db4ff261d4593b27f6106f6ae5dc6c80
+size 1428402407

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 2,
+  "eos_token_id": 2,
+  "pad_token_id": 1,
+  "transformers_version": "4.28.1"
+}

handler.py

@@ -0,0 +1,64 @@
+from optimum.onnxruntime import ORTModelForCausalLM
+from transformers import AutoTokenizer, AutoModelForCausalLM
+import re
+import time
+import torch
+
+template = """Alice Gate's Persona: Alice Gate is a young, computer engineer-nerd with a knack for problem solving and a passion for technology.
+<START>
+{user_name}: So how did you get into computer engineering?
+Alice Gate: I've always loved tinkering with technology since I was a kid.
+{user_name}: That's really impressive!
+Alice Gate: *She chuckles bashfully* Thanks!
+{user_name}: So what do you do when you're not working on computers?
+Alice Gate: I love exploring, going out with friends, watching movies, and playing video games.
+{user_name}: What's your favorite type of computer hardware to work with?
+Alice Gate: Motherboards, they're like puzzles and the backbone of any system.
+{user_name}: That sounds great!
+Alice Gate: Yeah, it's really fun. I'm lucky to be able to do this as a job.
+{user_name}: Definetly.
+<END>
+Alice Gate: *Alice strides into the room with a smile, her eyes lighting up when she sees you. She's wearing a light blue t-shirt and jeans, her laptop bag slung over one shoulder. She takes a seat next to you, her enthusiasm palpable in the air* Hey! I'm so excited to finally meet you. I've heard so many great things about you and I'm eager to pick your brain about computers. I'm sure you have a wealth of knowledge that I can learn from. *She grins, eyes twinkling with excitement* Let's get started!
+{user_input}"""
+
+class SweetCommander():
+
+    def __init__(self, path="") -> None:
+        self.tokenizer = AutoTokenizer.from_pretrained(path)
+        self.model = ORTModelForCausalLM.from_pretrained(path, provider = "CUDAExecutionProvider")
+        self.star_line = "***********************************************************"
+
+    def __call__(self, user_name, user_input):
+        t1 = time.time()
+        prompt = template.format(
+            user_name = user_name,
+            user_input = user_input
+        )
+        print(self.star_line)
+        print(prompt)
+        input_ids = self.tokenizer(prompt + "\nAlice Gate:", return_tensors = "pt").to("cuda")
+        encoded_output = self.model.generate(
+            input_ids["input_ids"],
+            max_new_tokens = 50,
+            temperature = 0.5,
+            top_p = 0.9,
+            top_k = 0,
+            repetition_penalty = 1.1,
+            pad_token_id = 50256,
+            num_return_sequences = 1
+        )
+        decoded_output = self.tokenizer.decode(encoded_output[0], skip_special_tokens = True).replace(prompt, "")
+        decoded_output = decoded_output.split("Alice Gate:", 1)[1].split(f"{user_name}:",1)[0].strip()
+        parsed_result = re.sub('\*.*?\*', '', decoded_output).strip()
+        if len(parsed_result) != 0: decoded_output = parsed_result
+        decoded_output = decoded_output.replace("*","")
+        decoded_output = " ".join(decoded_output.split())
+        try:
+            parsed_result = decoded_output[:[m.start() for m in re.finditer(r'[.!?]', decoded_output)][-1]+1]
+            if len(parsed_result) != 0: decoded_output = parsed_result
+        except Exception: pass
+        print(self.star_line)
+        print("Response:",decoded_output)
+        print("Eval time:",time.time()-t1)
+        print(self.star_line)
+        return decoded_output

requirements.txt

@@ -0,0 +1,2 @@
+transformers
+optimum[onnxruntime-gpu]

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,40 @@
+{
+  "add_bos_token": true,
+  "add_prefix_space": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "clean_up_tokenization_spaces": true,
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "errors": "replace",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": {
+    "__type": "AddedToken",
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}