Added DeBERTa model from previous semester

Model/DeBERTa/deberta.py

@@ -0,0 +1,233 @@
+import numpy as np
+import tensorflow as tf
+import tensorflow_addons as tfa
+from tensorflow.keras import layers
+import transformers
+import os
+
+MAX_LENGTH = 512  # the maximum number of messages per input
+BATCH_SIZE = 8  # number of messages processed at a time
+
+
+class MeanPool(tf.keras.layers.Layer):
+    def call(self, inputs, mask=None):
+        broadcast_mask = tf.expand_dims(tf.cast(mask, "float32"), -1)
+        embedding_sum = tf.reduce_sum(inputs * broadcast_mask, axis=1)
+        mask_sum = tf.reduce_sum(broadcast_mask, axis=1)
+        mask_sum = tf.math.maximum(mask_sum, tf.constant([1e-9]))
+        return embedding_sum / mask_sum
+
+
+class WeightsSumOne(tf.keras.constraints.Constraint):
+    def __call__(self, w):
+        return tf.nn.softmax(w, axis=0)
+
+
+def deberta_init(
+    pretrained_model_name: str = "microsoft/deberta-v3-large", tokenizer_dir: str = "."
+):
+    """Helper function to quickly initialize the config and tokenizer for a model
+
+    Args:
+        pretrained_model_name (str, optional): The model name. Defaults to "microsoft/deberta-v3-large".
+        tokenizer_dir (str, optional): Directory of the tokenizer. Defaults to ".".
+
+    Returns:
+        The configuration and tokenizer of the model.
+    """
+    tokenizer = transformers.AutoTokenizer.from_pretrained(pretrained_model_name)
+    tokenizer_path = os.path.join(tokenizer_dir, "tokenizer")
+    tokenizer.save_pretrained(tokenizer_path)
+
+    cfg = transformers.AutoConfig.from_pretrained(
+        pretrained_model_name, output_hidden_states=True
+    )
+    cfg.hidden_dropout_prob = 0
+    cfg.attention_probs_dropout_prob = 0
+    cfg.save_pretrained(tokenizer_path)
+    return cfg, tokenizer
+
+
+def get_model(cfg):
+    """Get a DeBERTa model using the specified configuration
+
+    Args:
+        cfg : the configuration of the model (can be generated using deberta_init)
+
+    Returns:
+        The model with respect to the given configuration.
+    """
+    input_ids = tf.keras.layers.Input(
+        shape=(MAX_LENGTH,), dtype=tf.int32, name="input_ids"
+    )
+
+    attention_masks = tf.keras.layers.Input(
+        shape=(MAX_LENGTH,), dtype=tf.int32, name="attention_masks"
+    )
+
+    deberta_model = transformers.TFAutoModel.from_pretrained(
+        "microsoft/deberta-v3-large", config=cfg
+    )
+
+    REINIT_LAYERS = 1
+    normal_initializer = tf.keras.initializers.GlorotUniform()
+    zeros_initializer = tf.keras.initializers.Zeros()
+    ones_initializer = tf.keras.initializers.Ones()
+
+    for encoder_block in deberta_model.deberta.encoder.layer[-REINIT_LAYERS:]:
+        for layer in encoder_block.submodules:
+            if isinstance(layer, tf.keras.layers.Dense):
+                layer.kernel.assign(
+                    normal_initializer(
+                        shape=layer.kernel.shape, dtype=layer.kernel.dtype
+                    )
+                )
+                if layer.bias is not None:
+                    layer.bias.assign(
+                        zeros_initializer(
+                            shape=layer.bias.shape, dtype=layer.bias.dtype
+                        )
+                    )
+
+            elif isinstance(layer, tf.keras.layers.LayerNormalization):
+                layer.beta.assign(
+                    zeros_initializer(shape=layer.beta.shape, dtype=layer.beta.dtype)
+                )
+                layer.gamma.assign(
+                    ones_initializer(shape=layer.gamma.shape, dtype=layer.gamma.dtype)
+                )
+
+    deberta_output = deberta_model.deberta(input_ids, attention_mask=attention_masks)
+    hidden_states = deberta_output.hidden_states
+
+    # WeightedLayerPool + MeanPool of the last 4 hidden states
+    stack_meanpool = tf.stack(
+        [MeanPool()(hidden_s, mask=attention_masks) for hidden_s in hidden_states[-4:]],
+        axis=2,
+    )
+
+    weighted_layer_pool = layers.Dense(
+        1, use_bias=False, kernel_constraint=WeightsSumOne()
+    )(stack_meanpool)
+
+    weighted_layer_pool = tf.squeeze(weighted_layer_pool, axis=-1)
+    output = layers.Dense(15, activation="linear")(weighted_layer_pool)
+
+    model = tf.keras.Model(inputs=[input_ids, attention_masks], outputs=output)
+
+    # Compile model with Layer-wise Learning Rate Decay
+    layer_list = [deberta_model.deberta.embeddings] + list(
+        deberta_model.deberta.encoder.layer
+    )
+    layer_list.reverse()
+
+    INIT_LR = 1e-5
+    LLRDR = 0.9
+    LR_SCH_DECAY_STEPS = 1600
+
+    lr_schedules = [
+        tf.keras.optimizers.schedules.ExponentialDecay(
+            initial_learning_rate=INIT_LR * LLRDR**i,
+            decay_steps=LR_SCH_DECAY_STEPS,
+            decay_rate=0.3,
+        )
+        for i in range(len(layer_list))
+    ]
+    lr_schedule_head = tf.keras.optimizers.schedules.ExponentialDecay(
+        initial_learning_rate=1e-4, decay_steps=LR_SCH_DECAY_STEPS, decay_rate=0.3
+    )
+
+    optimizers = [
+        tf.keras.optimizers.Adam(learning_rate=lr_sch) for lr_sch in lr_schedules
+    ]
+
+    optimizers_and_layers = [
+        (tf.keras.optimizers.Adam(learning_rate=lr_schedule_head), model.layers[-4:])
+    ] + list(zip(optimizers, layer_list))
+
+    optimizer = tfa.optimizers.MultiOptimizer(optimizers_and_layers)
+
+    model.compile(
+        optimizer=optimizer,
+        loss="mse",
+        metrics=[tf.keras.metrics.RootMeanSquaredError()],
+    )
+    return model
+
+
+def deberta_encode(texts: str, tokenizer):
+    """Helper function to tokenize the text using the specified tokenizer"""
+    input_ids = []
+    attention_mask = []
+
+    for text in texts:
+        token = tokenizer(
+            text,
+            add_special_tokens=True,
+            max_length=512,
+            return_attention_mask=True,
+            return_tensors="np",
+            truncation=True,
+            padding="max_length",
+        )
+        input_ids.append(token["input_ids"][0])
+        attention_mask.append(token["attention_mask"][0])
+
+    return np.array(input_ids, dtype="int32"), np.array(attention_mask, dtype="int32")
+
+
+def predict(model, tokenizer, texts):
+    """Predict the labels for each messages in texts
+
+    Args:
+        model: your DeBERTa model
+        tokenizer: a tokenizer (can be generated by deberta_init)
+        texts (_type_): _description_
+
+    Returns:
+        _type_: _description_
+    """
+    prediction = model.predict(deberta_encode(texts, tokenizer))
+    labels = np.argmax(prediction, axis=1)
+    return labels
+
+
+def load_model(cfg, model_dir: str = "."):
+    """Helper function to load a DeBERTa model with pretrained weights
+
+    Args:
+        cfg: configuration for the model (can be generated with deberta_init)
+        model_dir (str, optional): the directory of the pretrained weights. Defaults to ".".
+
+    Returns:
+        A DeBERTa model with pretrained weights.
+    """
+    tf.keras.backend.clear_session()
+    model = get_model(cfg)
+    model_path = os.path.join(model_dir, "best_model_fold2.h5")
+    model.load_weights(model_path)
+    return model
+
+
+# map the integer labels to their original string representation
+DEBERTA_LABEL_MAP = {
+    0: "Greeting",
+    1: "Curiosity",
+    2: "Interest",
+    3: "Obscene",
+    4: "Annoyed",
+    5: "Openness",
+    6: "Anxious",
+    7: "Acceptance",
+    8: "Uninterested",
+    9: "Informative",
+    10: "Accusatory",
+    11: "Denial",
+    12: "Confused",
+    13: "Disapproval",
+    14: "Remorse",
+}
+
+
+def decode_deberta_label(numeric_label):
+    return DEBERTA_LABEL_MAP.get(numeric_label, "Unknown Label")

app.ipynb

@@ -1,457 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 28,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import os\n",
-    "import openai \n",
-    "from openai import OpenAI\n",
-    "from dotenv import load_dotenv, find_dotenv\n",
-    "\n",
-    "import re\n",
-    "import matplotlib.pyplot as plt\n",
-    "import seaborn as sns"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 29,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "sample_input = \\\n",
-    "\"\"\"\n",
-    "Visitor: Heyyy\n",
-    "Visitor: How are you this evening\n",
-    "Agent: better now ;) call me\n",
-    "Visitor: I am at work for now, be off around 10pm\n",
-    "Visitor: Need some company\n",
-    "Visitor: Are you independent honey\n",
-    "Agent: well since you arent available at the moment ill just come out and say-these sites are bad news. \\\n",
-    "    did you know that most of the girls on here are here against their will? \\\n",
-    "    Most of them got dragged into this lifestyle by an abuser, \\\n",
-    "    oftentimes before they were of legal consenting age. isnt that sad?\n",
-    "Agent: we are with some guys who are trying to spread awareness of the realities of this \"industry\".\n",
-    "Agent: https://exoduscry.com/choice/\n",
-    "Visitor: Thanks\n",
-    "Agent: i encourage you to watch this video. it is jarring to think about how bad someone else's options must be to choose to be on these sites\n",
-    "Visitor: Ooohhh\n",
-    "Agent: selling their body to make ends meet or appease a pimp\n",
-    "Visitor: That's really awful\n",
-    "Agent: it is. you seem like the kind of guy who wouldnt wont to proliferate that kind of harmful lifestyle. am i right in thinking that?\n",
-    "Visitor: Well iam just looking for attention\n",
-    "Visitor: My marriage is not going well lol\n",
-    "Agent: i know that it is hard to find ourselves lonely and without much alternative to meet that perceived need but \\\n",
-    "    its humbling to think that our needs can force someone else into such a dark place\n",
-    "Agent: hey, thanks for sharing that my man. i know it can be hard\n",
-    "Agent: marraige is the most humbling of relationships, isnt it?\n",
-    "Visitor: She leaves with her friends n no time for me\n",
-    "Agent: ive been there my guy. i know that it is alot easier to numb that loneliness for sure\n",
-    "Visitor: I want to be faithful\n",
-    "Agent: does your wife know how you feel when she chooses her friends instead of you?\n",
-    "Visitor: I been drinking lately\n",
-    "Visitor: Yes, she takes pills\n",
-    "Agent: if so, i hope you are praying for her to realize the hurt she is causing and to seek change\n",
-    "Visitor: She had surgery 4 yes ago n it's been hard for her n her addiction on pills\n",
-    "Visitor: Yes for now i am looking for a female friend to talk n see what can we do for each other\n",
-    "Agent: that is hard my man. physical pain is a huge obstacle in life for sure so i hear you\n",
-    "Visitor: Well chat later. thanks\n",
-    "Agent: have you considered pursuing other men who can encourage you instead of looking for the easy way out?\n",
-    "Agent: what is your name my friend? i will be praying for you by name if you wouldnt mind sharing it\n",
-    "Agent: well, i gotta run. watch that video i sent and i will definitely be praying for you. \\\n",
-    "    I hope you pray for yourself and for your wife - God can definitely intervene and cause complete change in the situation if He wills it. \\\n",
-    "    He is good and He hears you. You are loved by Him, brother. Good night\n",
-    "\"\"\""
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 30,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "sample_output = \\\n",
-    "\"\"\"\n",
-    "Visitor: Heyyy\n",
-    "[Greeting]\n",
-    "Visitor: How are you this evening\n",
-    "[Greeting]\n",
-    "Agent: better now ;) call me\n",
-    "[Openness]\n",
-    "Visitor: I am at work for now, be off around 10pm\n",
-    "[Interest]\n",
-    "Visitor: Need some company\n",
-    "[Interest]\n",
-    "Visitor: Are you independent honey\n",
-    "[Interest]\n",
-    "Agent: well since you arent available at the moment ill just come out and say-these sites are bad news. \\\n",
-    "    did you know that most of the girls on here are here against their will? \\\n",
-    "    Most of them got dragged into this lifestyle by an abuser, \\\n",
-    "    oftentimes before they were of legal consenting age. isnt that sad?\n",
-    "[Informative]\n",
-    "Agent: we are with some guys who are trying to spread awareness of the realities of this \"industry\".\n",
-    "[Informative]\n",
-    "Agent: https://exoduscry.com/choice/\n",
-    "[Informative]\n",
-    "Visitor: Thanks\n",
-    "[Acceptance]\n",
-    "Agent: i encourage you to watch this video. it is jarring to think about how bad someone else's options must be to choose to be on these sites\n",
-    "[Informative]\n",
-    "Visitor: Ooohhh\n",
-    "[Interest]\n",
-    "Agent: selling their body to make ends meet or appease a pimp\n",
-    "[Informative]\n",
-    "Visitor: That's really awful\n",
-    "[Remorse]\n",
-    "Agent: it is. you seem like the kind of guy who wouldnt wont to proliferate that kind of harmful lifestyle. am i right in thinking that?\n",
-    "[Accusatory]\n",
-    "Visitor: Well iam just looking for attention\n",
-    "[Anxious]\n",
-    "Visitor: My marriage is not going well lol\n",
-    "[Anxious]\n",
-    "Agent: i know that it is hard to find ourselves lonely and without much alternative to meet that perceived need but \\\n",
-    "    its humbling to think that our needs can force someone else into such a dark place\n",
-    "[Informative]\n",
-    "Agent: hey, thanks for sharing that my man. i know it can be hard\n",
-    "[Acceptance]\n",
-    "Agent: marraige is the most humbling of relationships, isnt it?\n",
-    "[Openness]\n",
-    "Visitor: She leaves with her friends n no time for me\n",
-    "[Annoyed]\n",
-    "Agent: ive been there my guy. i know that it is alot easier to numb that loneliness for sure\n",
-    "[Acceptance]\n",
-    "Visitor: I want to be faithful\n",
-    "[Acceptance]\n",
-    "Agent: does your wife know how you feel when she chooses her friends instead of you?\n",
-    "[Curiosity]\n",
-    "Visitor: I been drinking lately\n",
-    "[Anxious]\n",
-    "Visitor: Yes, she takes pills\n",
-    "[Anxious]\n",
-    "Agent: if so, i hope you are praying for her to realize the hurt she is causing and to seek change\n",
-    "[Interest]\n",
-    "Visitor: She had surgery 4 yes ago n it's been hard for her n her addiction on pills\n",
-    "[Anxious]\n",
-    "Visitor: Yes for now i am looking for a female friend to talk n see what can we do for each other\n",
-    "[Informative]\n",
-    "Agent: that is hard my man. physical pain is a huge obstacle in life for sure so i hear you\n",
-    "[Acceptance]\n",
-    "Visitor: Well chat later. thanks\n",
-    "[Openness]\n",
-    "Agent: have you considered pursuing other men who can encourage you instead of looking for the easy way out?\n",
-    "[Informative]\n",
-    "Agent: what is your name my friend? i will be praying for you by name if you wouldnt mind sharing it\n",
-    "[Openness]\n",
-    "Agent: well, i gotta run. watch that video i sent and i will definitely be praying for you. \\\n",
-    "    I hope you pray for yourself and for your wife - God can definitely intervene and cause complete change in the situation if He wills it. \\\n",
-    "    He is good and He hears you. You are loved by Him, brother. Good night\n",
-    "[Openness]\n",
-    "\n",
-    "Sentiment Flow Analysis on the Visitor's side:\n",
-    "\n",
-    "The Visitor begins the conversation with a friendly and casual tone, expressing a desire for company and showing interest in the Agent. \\\n",
-    "However, as the Agent provides information about the harsh realities of the commercial sex industry, the Visitor's sentiment shifts to acceptance of the information \\\n",
-    "and a sense of confusion and remorse about the situation.\n",
-    "\n",
-    "The Visitor then reveals personal issues, indicating anxiety and seeking attention due to marital problems. \\\n",
-    "The sentiment continues to be anxious as the Visitor discusses personal struggles with alcohol and his wife's pill addiction, \\\n",
-    "showing a need for companionship and support.\n",
-    "\n",
-    "Despite the heavy topics, the Visitor expresses a desire to remain faithful and shows interest in finding a friend, albeit with a hint of desperation. \\\n",
-    "The Visitor openly takes the Agent's information and the conversation flows smoothly as both the Visitor and the Agent \\\n",
-    "show openness toward each other.\n",
-    "\"\"\""
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 65,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def get_completion(conversation, model=\"gpt-4-1106-preview\"):\n",
-    "\n",
-    "    prompt = f\"\"\"\n",
-    "    The EPIK Project is about mobilizing male allies \\\n",
-    "    to disrupt the commercial sex market, \\\n",
-    "    equipping them to combat the roots of exploitation \\\n",
-    "    and encouraging them to collaborate effectively \\\n",
-    "    with the wider anti-trafficking movement. \\\n",
-    "    You are an adept expert conversation sentiment analyzer. \\\n",
-    "    Your job is to analyze the conversation and provide a report \\\n",
-    "    based on the sentiment flow of the conversation on the visitor's \\\n",
-    "    perspective. Visitor indicates the potential buyer, and Agent indicates the volunteer from EPIK. \\\n",
-    "    The conversation is going to be given in the format:\n",
-    "    \n",
-    "    Visitor: <Visitor's message here>\n",
-    "    Agent: <Agent's message here>\n",
-    "    \n",
-    "    The actual conversation is delimited by triple backticks\n",
-    "    ```{conversation}```\n",
-    "    \n",
-    "    Here is the list of sentiment labels you should use delimited by square brackets. \\\n",
-    "    [\"Openness\", \"Anxious\", \"Confused\", \"Disapproval\", \"Remorse\", \"Accusatory\", \\\n",
-    "    \"Denial\", \"Obscene\", \"Uninterested\", \"Annoyed\", \"Informative\", \"Greeting\", \\\n",
-    "    \"Interest\", \"Curiosity\", \"Acceptance\"]\n",
-    "    \n",
-    "    Your output should look like:\n",
-    "    ```\n",
-    "    Speaker: <Speaker's message here>\n",
-    "    [sentiment label]\n",
-    "    ...\n",
-    "    Speaker: <Speaker's message here>\n",
-    "    [sentiment label]\n",
-    "    ```\n",
-    "    \n",
-    "    where Speaker can either be Visitor or Agent. Then, you should write your report on the sentiment flow \\\n",
-    "    on the Visitor's side below.\n",
-    "\n",
-    "    Here is a sample input delimited by triple backticks\n",
-    "\n",
-    "    ```{sample_input}```\n",
-    "\n",
-    "    Here is a same output that you should try to aim for delimited by sqaure brackets\n",
-    "\n",
-    "    [{sample_output}]\n",
-    "    \"\"\"\n",
-    "    \n",
-    "    client = OpenAI()\n",
-    "    \n",
-    "    messages = [{\"role\": \"user\", \"content\": prompt}]\n",
-    "    response = client.chat.completions.create(\n",
-    "        model=model,\n",
-    "        messages=messages,\n",
-    "        temperature=0, # this is the degree of randomness of the model's output\n",
-    "    )\n",
-    "\n",
-    "    analysis = response.choices[0].message.content\n",
-    "\n",
-    "    def extract_conv_with_labels(analysis):\n",
-    "        analysis = analysis.replace(\"\\n\", \" \")\n",
-    "        BETWEEN_BACKTICKS = \"\\\\`\\\\`\\\\`(.*?)\\\\`\\\\`\\\\`\"\n",
-    "        match = re.search(BETWEEN_BACKTICKS, analysis)\n",
-    "        if match:\n",
-    "            conv_with_labels = match.group()[4:-4]\n",
-    "        else:\n",
-    "            return \"OUTPUT IS IN WRONG FORMAT\"\n",
-    "        \n",
-    "        # just reformatting it for better format\n",
-    "        conv_with_labels = conv_with_labels.split('] ')\n",
-    "        temp = [utterance + ']' for utterance in conv_with_labels[:-1]]\n",
-    "        conv_with_labels = temp + [conv_with_labels[-1]]\n",
-    "        return conv_with_labels\n",
-    "\n",
-    "    grouped_sentiments = {\n",
-    "        'Acceptance': 3,\n",
-    "        'Openness': 3,\n",
-    "        'Interest': 2,\n",
-    "        'Curiosity': 2,\n",
-    "        'Informative': 1,\n",
-    "        'Greeting': 0,\n",
-    "        'None': 0,\n",
-    "        'Uninterested': -1,\n",
-    "        'Anxious': -2,\n",
-    "        'Confused': -2,\n",
-    "        'Annoyed': -2,\n",
-    "        'Remorse': -2,\n",
-    "        'Disapproval': -3,\n",
-    "        'Accusatory': -3,\n",
-    "        'Denial': -3,\n",
-    "        'Obscene': -3\n",
-    "    }\n",
-    "\n",
-    "\n",
-    "    def sentiment_flow_plot(conv):\n",
-    "        conv_with_labels = extract_conv_with_labels(analysis)\n",
-    "        num_utterances = len(conv_with_labels)\n",
-    "\n",
-    "        visitor_Y = [''] * num_utterances\n",
-    "        agent_Y = [''] * num_utterances\n",
-    "\n",
-    "        for i in range(num_utterances):\n",
-    "            utterance = conv_with_labels[i]\n",
-    "            match = re.search(r'\\[(.*?)\\]$', utterance)\n",
-    "            if match:\n",
-    "                label = match.group(1)\n",
-    "            else:\n",
-    "                print(\"OUTPUT IS IN WRONG FORMAT\")\n",
-    "                break\n",
-    "            \n",
-    "            if utterance.startswith('Visitor'):\n",
-    "                visitor_Y[i] = label\n",
-    "                if i == 0:\n",
-    "                    agent_Y[i] = 'None'\n",
-    "                else:\n",
-    "                    agent_Y[i] = agent_Y[i-1]\n",
-    "            elif utterance.startswith('Agent'):\n",
-    "                agent_Y[i] = label\n",
-    "                if i == 0:\n",
-    "                    visitor_Y[i] = 'None'\n",
-    "                else:\n",
-    "                    visitor_Y[i] = visitor_Y[i-1]\n",
-    "\n",
-    "        X = range(1,num_utterances+1)\n",
-    "        visitor_Y_converted = [grouped_sentiments[visitor_Y[i]] for i in range(num_utterances)]\n",
-    "        agent_Y_converted = [grouped_sentiments[agent_Y[i]] for i in range(num_utterances)]\n",
-    "\n",
-    "\n",
-    "        fig, ax = plt.subplots()\n",
-    "        sns.set(style=\"whitegrid\")\n",
-    "\n",
-    "        ax.plot(X, visitor_Y_converted, label='Visitor', color='blue', marker='o')\n",
-    "        ax.plot(X, agent_Y_converted, label='Agent', color='green', marker='o')\n",
-    "\n",
-    "        plt.legend(loc='upper left', bbox_to_anchor=(1,1))\n",
-    "        plt.subplots_adjust(right=0.8)\n",
-    "\n",
-    "        plt.yticks(ticks=[-3,-2,-1,0,1,2,3])\n",
-    "        \n",
-    "        # y_labels = {-3: 'Disapproval/Accusatory/Denial/Obscene', -2: 'Anxious/Confused\\nAnnoyed/Remorse', -1: 'Uninterested', 0: 'Greeting/None',\n",
-    "        #          1: 'Informative', 2: 'Interest/Curiosity', 3: 'Acceptance/Openness'}\n",
-    "        \n",
-    "        # cell_text = [[label] for label in y_labels.values()]\n",
-    "        # plt.table(cellText=cell_text, rowLabels=list(y_labels.keys()), loc='left')\n",
-    "\n",
-    "        # plt.tick_params(axis='y', labelsize=10)\n",
-    "\n",
-    "        plt.xlabel('Timestamp')\n",
-    "        plt.ylabel('Sentiment Score')\n",
-    "        plt.title('Sentiment Flow Plot')\n",
-    "\n",
-    "        plt.close(fig)\n",
-    "\n",
-    "        return fig\n",
-    "    \n",
-    "    fig = sentiment_flow_plot(analysis)\n",
-    "\n",
-    "    return response.choices[0].message.content, fig\n",
-    "\n",
-    "def set_key(key):\n",
-    "    os.environ['OPENAI_API_KEY'] = key\n",
-    "    load_dotenv()\n",
-    "    return"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 51,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "aligned_markdown_table = \"\"\"\n",
-    "<div style='text-align: right; font-size: small;'>\n",
-    "\n",
-    "| Sentiment Score | Sentiment Label |\n",
-    "|:---------------:|:---------------:|\n",
-    "| 3               | Acceptance, Openness |\n",
-    "| 2               | Interest, Curiosity |\n",
-    "| 1               | Informative  |\n",
-    "| 0               | Greeting  |\n",
-    "| -1              | Uninterested |\n",
-    "| -2              | Anxious, Confused, Annoyed, Remorse |\n",
-    "| -3              | Disapproval, Accusatory, Denial, Obscene |\n",
-    "\n",
-    "</div>\n",
-    "\"\"\""
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 32,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import gradio as gr"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 66,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "with gr.Blocks() as gpt_analysis:\n",
-    "    gr.Markdown(\"## Conversation Sentiment Analysis Report\")\n",
-    "    gr.Markdown(\n",
-    "        \"This is a custom GPT model designed to provide \\\n",
-    "        a report on overall sentiment flow of the conversation on the \\\n",
-    "        volunteer's perspective. It also provies a live plot analysis of sentiments throughout the conversation.<br /><br />Click on them and submit them to the model to see how it works.\")\n",
-    "    api_key = gr.Textbox(label=\"Key\", lines=1)\n",
-    "    btn_key = gr.Button(value=\"Submit Key\")\n",
-    "    btn_key.click(set_key, inputs=api_key)\n",
-    "    with gr.Row():\n",
-    "        with gr.Column():\n",
-    "            conversation = gr.Textbox(label=\"Input\", lines=4)\n",
-    "        with gr.Column():\n",
-    "            output_box = gr.Textbox(value=\"\", label=\"Output\",lines=4)\n",
-    "    btn = gr.Button(value=\"Submit\")\n",
-    "    with gr.Row():\n",
-    "        with gr.Column(scale=2):\n",
-    "            gr.Markdown(aligned_markdown_table)\n",
-    "        with gr.Column(scale=2):\n",
-    "            plot_box = gr.Plot(label=\"Analysis Plot\")\n",
-    "\n",
-    "        \n",
-    "    btn.click(get_completion, inputs=conversation, outputs=[output_box, plot_box])\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 67,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Running on local URL:  http://127.0.0.1:7883\n",
-      "\n",
-      "To create a public link, set `share=True` in `launch()`.\n"
-     ]
-    },
-    {
-     "data": {
-      "text/plain": []
-     },
-     "execution_count": 67,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "gr.TabbedInterface([gpt_analysis], [\"GPT Anlysis\"]).launch(inline=False)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "base",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.9.13"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}

app/__init__.py

@@ -35,3 +35,11 @@ comet = CSKFeatureExtractor(dir=EXTRACTORS_PATH, device="cpu")
 cosmic_args = parse_cosmic_args()
 
 COSMIC_MODEL = load_model(EPIK_MODEL_DIR + "/epik/best_model.pt", cosmic_args)
+
+PATH_TO_DEBERTA = "./Model/DeBERTa"
+sys.path.append(PATH_TO_DEBERTA)
+
+from Model.DeBERTa.deberta import load_model, deberta_init
+
+cfg, tokenizer = deberta_init()
+deberta_model = load_model(cfg, PATH_TO_DEBERTA)

app/deberta_view.py

@@ -0,0 +1,124 @@
+import gradio as gr
+from app.utils import (
+    create_input_instruction,
+    format_prediction_ouptut,
+    display_sentiment_score_table,
+    sentiment_flow_plot,
+    EXAMPLE_CONVERSATIONS,
+)
+
+import sys
+
+sys.path.insert(0, "../")  # neccesary to load modules outside of app
+
+from app import deberta_model, tokenizer
+from preprocessing import preprocess
+from Model.DeBERTa.deberta import predict, decode_deberta_label
+
+
+def deberta_preprocess(input):
+    result = preprocess.process_user_input(input)
+
+    if not result["success"]:
+        raise gr.Error(result["message"])
+
+    data = result["data"]
+    speakers = [item[1] for item in data]
+    messages = [item[2] for item in data]
+    return speakers, messages
+
+
+def deberta_classifier(input):
+    speakers, messages = deberta_preprocess(input)
+
+    predictions = predict(deberta_model, tokenizer, messages)
+
+    # Assuming that there's only one conversation
+    labels = [decode_deberta_label(pred) for pred in predictions]
+    output = format_prediction_ouptut(speakers, messages, labels)
+
+    return output
+
+
+def deberta_ui():
+    with gr.Blocks() as deberta_model:
+        gr.Markdown(
+            """
+            # Deberta
+            Building upon the DeBERTa architecture, the model was customized and
+            retrained on Epik data to classify messages between Visitors and Agents into
+            corresponding sentiment labels. At the time of training by the team prior to
+            the Fall 2023 semester, the model was trained on 15 labels, including
+            Openness, Anxiety, Confusion, Disapproval, Remorse, Accusation, Denial,
+            Obscenity, Disinterest, Annoyance, Information, Greeting, Interest,
+            Curiosity, or Acceptance. 
+            
+            The primary difference between DeBERTa and COSMIC is that while DeBERTa's
+            prediction is solely based on its own context, COSMIC uses the context of
+            the entire conversation (i.e., all messages from the chat history of the
+            conversation).
+            """
+        )
+
+        create_input_instruction()
+        with gr.Row():
+            with gr.Column():
+                example_dropdown = gr.Dropdown(
+                    choices=["-- Not Selected --"] + list(EXAMPLE_CONVERSATIONS.keys()),
+                    value="-- Not Selected --",
+                    label="Select an example",
+                )
+
+                gr.Markdown('<p style="text-align: center;color: gray;">--- OR ---</p>')
+
+                conversation_input = gr.TextArea(
+                    value="",
+                    label="Input you conversation",
+                    placeholder="Plese input your conversation here",
+                    lines=15,
+                    max_lines=15,
+                )
+
+                def on_example_change(input):
+                    if input in EXAMPLE_CONVERSATIONS:
+                        return EXAMPLE_CONVERSATIONS[input]
+
+                    return ""
+
+                example_dropdown.input(
+                    on_example_change,
+                    inputs=example_dropdown,
+                    outputs=conversation_input,
+                )
+
+            with gr.Column():
+                output = gr.Textbox(
+                    value="",
+                    label="Predicted Sentiment Labels",
+                    lines=22,
+                    max_lines=22,
+                    interactive=False,
+                )
+        submit_btn = gr.Button(value="Submit")
+        submit_btn.click(deberta_classifier, conversation_input, output)
+
+        # reset the output whenever a change in the input is detected
+        conversation_input.change(lambda x: "", conversation_input, output)
+
+        gr.Markdown("# Sentiment Flow Plot")
+        with gr.Row():
+            with gr.Column(scale=1):
+                display_sentiment_score_table()
+            with gr.Column(scale=2):
+                plot_box = gr.Plot(label="Analysis Plot")
+
+        plot_btn = gr.Button(value="Plot Sentiment Flow")
+        plot_btn.click(sentiment_flow_plot, inputs=[output], outputs=[plot_box])
+
+        # reset all outputs whenever a change in the input is detected
+        conversation_input.change(
+            lambda x: ("", None),
+            conversation_input,
+            outputs=[output, plot_box],
+        )
+    return deberta_model

app_spring2023.ipynb

@@ -1,483 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import numpy as np\n",
-    "import tensorflow as tf\n",
-    "import tensorflow_addons as tfa\n",
-    "from tensorflow.keras import layers\n",
-    "import transformers\n",
-    "import sentencepiece as spm\n",
-    "#show the version of the package imported with text instructions\\\n",
-    "print(\"Tensorflow version: \", tf.__version__)\n",
-    "print(\"Tensorflow Addons version: \", tfa.__version__)\n",
-    "print(\"Transformers version: \", transformers.__version__)\n",
-    "print(\"Sentencepiece version: \", spm.__version__)\n",
-    "print(\"Numpy version: \", np.__version__)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "class MeanPool(tf.keras.layers.Layer):\n",
-    "    def call(self, inputs, mask=None):\n",
-    "        broadcast_mask = tf.expand_dims(tf.cast(mask, \"float32\"), -1)\n",
-    "        embedding_sum = tf.reduce_sum(inputs * broadcast_mask, axis=1)\n",
-    "        mask_sum = tf.reduce_sum(broadcast_mask, axis=1)\n",
-    "        mask_sum = tf.math.maximum(mask_sum, tf.constant([1e-9]))\n",
-    "        return embedding_sum / mask_sum\n",
-    "class WeightsSumOne(tf.keras.constraints.Constraint):\n",
-    "    def __call__(self, w):\n",
-    "        return tf.nn.softmax(w, axis=0)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "tokenizer = transformers.AutoTokenizer.from_pretrained(\"microsoft/deberta-v3-large\"\n",
-    ")\n",
-    "tokenizer.save_pretrained('./tokenizer/')\n",
-    "\n",
-    "cfg = transformers.AutoConfig.from_pretrained(\"microsoft/deberta-v3-large\", output_hidden_states=True)\n",
-    "cfg.hidden_dropout_prob = 0\n",
-    "cfg.attention_probs_dropout_prob = 0\n",
-    "cfg.save_pretrained('./tokenizer/')"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def deberta_encode(texts, tokenizer=tokenizer):\n",
-    "    input_ids = []\n",
-    "    attention_mask = []\n",
-    "    \n",
-    "    for text in texts:\n",
-    "        token = tokenizer(text, \n",
-    "                          add_special_tokens=True, \n",
-    "                          max_length=512, \n",
-    "                          return_attention_mask=True, \n",
-    "                          return_tensors=\"np\", \n",
-    "                          truncation=True, \n",
-    "                          padding='max_length')\n",
-    "        input_ids.append(token['input_ids'][0])\n",
-    "        attention_mask.append(token['attention_mask'][0])\n",
-    "    \n",
-    "    return np.array(input_ids, dtype=\"int32\"), np.array(attention_mask, dtype=\"int32\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "MAX_LENGTH=512\n",
-    "BATCH_SIZE=8"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def get_model():\n",
-    "    input_ids = tf.keras.layers.Input(\n",
-    "        shape=(MAX_LENGTH,), dtype=tf.int32, name=\"input_ids\"\n",
-    "    )\n",
-    "    \n",
-    "    attention_masks = tf.keras.layers.Input(\n",
-    "        shape=(MAX_LENGTH,), dtype=tf.int32, name=\"attention_masks\"\n",
-    "    )\n",
-    "   \n",
-    "    deberta_model = transformers.TFAutoModel.from_pretrained(\"microsoft/deberta-v3-large\", config=cfg)\n",
-    "    \n",
-    "        \n",
-    "    REINIT_LAYERS = 1\n",
-    "    normal_initializer = tf.keras.initializers.GlorotUniform()\n",
-    "    zeros_initializer = tf.keras.initializers.Zeros()\n",
-    "    ones_initializer = tf.keras.initializers.Ones()\n",
-    "\n",
-    "#     print(f'\\nRe-initializing encoder block:')\n",
-    "    for encoder_block in deberta_model.deberta.encoder.layer[-REINIT_LAYERS:]:\n",
-    "#         print(f'{encoder_block}')\n",
-    "        for layer in encoder_block.submodules:\n",
-    "            if isinstance(layer, tf.keras.layers.Dense):\n",
-    "                layer.kernel.assign(normal_initializer(shape=layer.kernel.shape, dtype=layer.kernel.dtype))\n",
-    "                if layer.bias is not None:\n",
-    "                    layer.bias.assign(zeros_initializer(shape=layer.bias.shape, dtype=layer.bias.dtype))\n",
-    "\n",
-    "            elif isinstance(layer, tf.keras.layers.LayerNormalization):\n",
-    "                layer.beta.assign(zeros_initializer(shape=layer.beta.shape, dtype=layer.beta.dtype))\n",
-    "                layer.gamma.assign(ones_initializer(shape=layer.gamma.shape, dtype=layer.gamma.dtype))\n",
-    "\n",
-    "    deberta_output = deberta_model.deberta(\n",
-    "        input_ids, attention_mask=attention_masks\n",
-    "    )\n",
-    "    hidden_states = deberta_output.hidden_states\n",
-    "    \n",
-    "    #WeightedLayerPool + MeanPool of the last 4 hidden states\n",
-    "    stack_meanpool = tf.stack(\n",
-    "        [MeanPool()(hidden_s, mask=attention_masks) for hidden_s in hidden_states[-4:]], \n",
-    "        axis=2)\n",
-    "    \n",
-    "    weighted_layer_pool = layers.Dense(1,\n",
-    "                                       use_bias=False,\n",
-    "                                       kernel_constraint=WeightsSumOne())(stack_meanpool)\n",
-    "    \n",
-    "    weighted_layer_pool = tf.squeeze(weighted_layer_pool, axis=-1)\n",
-    "    output=layers.Dense(15,activation='linear')(weighted_layer_pool)\n",
-    "    #x = layers.Dense(6, activation='linear')(x)\n",
-    "    \n",
-    "    #output = layers.Rescaling(scale=4.0, offset=1.0)(x)\n",
-    "    model = tf.keras.Model(inputs=[input_ids, attention_masks], outputs=output)\n",
-    "    \n",
-    "    #Compile model with Layer-wise Learning Rate Decay\n",
-    "    layer_list = [deberta_model.deberta.embeddings] + list(deberta_model.deberta.encoder.layer)\n",
-    "    layer_list.reverse()\n",
-    "    \n",
-    "    INIT_LR = 1e-5\n",
-    "    LLRDR = 0.9\n",
-    "    LR_SCH_DECAY_STEPS = 1600\n",
-    "    \n",
-    "    lr_schedules = [tf.keras.optimizers.schedules.ExponentialDecay(\n",
-    "        initial_learning_rate=INIT_LR * LLRDR ** i, \n",
-    "        decay_steps=LR_SCH_DECAY_STEPS, \n",
-    "        decay_rate=0.3) for i in range(len(layer_list))]\n",
-    "    lr_schedule_head = tf.keras.optimizers.schedules.ExponentialDecay(\n",
-    "        initial_learning_rate=1e-4, \n",
-    "        decay_steps=LR_SCH_DECAY_STEPS, \n",
-    "        decay_rate=0.3)\n",
-    "    \n",
-    "    optimizers = [tf.keras.optimizers.Adam(learning_rate=lr_sch) for lr_sch in lr_schedules]\n",
-    "    \n",
-    "    optimizers_and_layers = [(tf.keras.optimizers.Adam(learning_rate=lr_schedule_head), model.layers[-4:])] +\\\n",
-    "        list(zip(optimizers, layer_list))\n",
-    "    \n",
-    "    optimizer = tfa.optimizers.MultiOptimizer(optimizers_and_layers)\n",
-    "    \n",
-    "    model.compile(optimizer=optimizer,\n",
-    "                 loss='mse',\n",
-    "                 metrics=[tf.keras.metrics.RootMeanSquaredError()],\n",
-    "                 )\n",
-    "    return model"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "tf.keras.backend.clear_session()\n",
-    "model = get_model()\n",
-    "model.load_weights('./best_model_fold2.h5')"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# map the integer labels to their original string representation\n",
-    "label_mapping = {\n",
-    "    0: 'Greeting',\n",
-    "    1: 'Curiosity',\n",
-    "    2: 'Interest',\n",
-    "    3: 'Obscene',\n",
-    "    4: 'Annoyed',\n",
-    "    5: 'Openness',\n",
-    "    6: 'Anxious',\n",
-    "    7: 'Acceptance',\n",
-    "    8: 'Uninterested',\n",
-    "    9: 'Informative',\n",
-    "    10: 'Accusatory',\n",
-    "    11: 'Denial',\n",
-    "    12: 'Confused',\n",
-    "    13: 'Disapproval',\n",
-    "    14: 'Remorse'\n",
-    "}\n",
-    "\n",
-    "#label_strings = [label_mapping[label] for label in labels]\n",
-    "\n",
-    "#print(label_strings)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def inference(texts):\n",
-    "    prediction = model.predict(deberta_encode([texts]))\n",
-    "    labels = np.argmax(prediction, axis=1)\n",
-    "    label_strings = [label_mapping[label] for label in labels]\n",
-    "    return label_strings[0]"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# GPT"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import openai\n",
-    "import os\n",
-    "import pandas as pd\n",
-    "import gradio as gr"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "openai.organization = os.environ['org_id']\n",
-    "openai.api_key = os.environ['openai_api']\n",
-    "model_version = \"gpt-3.5-turbo\"\n",
-    "model_token_limit = 10\n",
-    "model_temperature = 0.1\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def generatePrompt () :\n",
-    "    labels = [\"Openness\", \n",
-    "    \"Anxious\",\n",
-    "    \"Confused\",\n",
-    "    \"Disapproval\",\n",
-    "    \"Remorse\",\n",
-    "    \"Uninterested\",\n",
-    "    \"Accusatory\",\n",
-    "    \"Annoyed\",\n",
-    "    \"Interest\",\n",
-    "    \"Curiosity\",\n",
-    "    \"Acceptance\",\n",
-    "    \"Obscene\",\n",
-    "    \"Denial\",\n",
-    "    \"Informative\",\n",
-    "    \"Greeting\"]\n",
-    "\n",
-    "    formatted_labels = ', '.join(labels[:-1]) + ', or ' + labels[-1] + '.'\n",
-    "\n",
-    "    label_set = [\"Openness\", \"Anxious\", \"Confused\", \"Disapproval\", \"Remorse\", \"Accusatory\",\n",
-    "          \"Denial\", \"Obscene\", \"Uninterested\", \"Annoyed\", \"Informative\", \"Greeting\",\n",
-    "          \"Interest\", \"Curiosity\", \"Acceptance\"]\n",
-    "\n",
-    "    formatted_labels = ', '.join(label_set[:-1]) + ', or ' + label_set[-1] + '.\\n'\n",
-    "\n",
-    "    # The basic task to assign GPT (in natural language)\n",
-    "    base_task = \"Classify the following text messages into one of the following categories using one word: \" + formatted_labels\n",
-    "    base_task += \"Provide only a one word response. Use only the labels provided.\\n\"\n",
-    "\n",
-    "    return base_task"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def predict(message):\n",
-    "    \n",
-    "    prompt = [{\"role\": \"user\", \"content\": generatePrompt () + \"Text: \"+ message}]\n",
-    "    \n",
-    "    response = openai.ChatCompletion.create(\n",
-    "                    model=model_version,\n",
-    "                    temperature=model_temperature,\n",
-    "                    max_tokens=model_token_limit,\n",
-    "                    messages=prompt\n",
-    "                )\n",
-    "        \n",
-    "    return response[\"choices\"][0][\"message\"][\"content\"]"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Update"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "model_version = \"gpt-3.5-turbo\"\n",
-    "model_token_limit = 2000\n",
-    "model_temperature = 0.1"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def revision(message):\n",
-    "    base_prompt = \"Here is a conversation between a Caller and a Volunteer. The Volunteer is trying to be as non-accusatory as possible but also wants to get as much information about the caller as possible. What should the volunteer say next in this exchange? Proved 3 possible responses.\"\n",
-    "\n",
-    "    prompt = [{\"role\": \"user\", \"content\": base_prompt + message}]\n",
-    "    \n",
-    "    response = openai.ChatCompletion.create(\n",
-    "                    model=model_version,\n",
-    "                    temperature=model_temperature,\n",
-    "                    max_tokens=model_token_limit,\n",
-    "                    messages=prompt\n",
-    "                )\n",
-    "\n",
-    "    return response[\"choices\"][0][\"message\"][\"content\"]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import gradio as gr\n",
-    "\n",
-    "def combine(a):\n",
-    "    return a + \"hello\"\n",
-    "\n",
-    "\n",
-    "\n",
-    "\n",
-    "with gr.Blocks() as demo:\n",
-    "    gr.Markdown(\"## DeBERTa Sentiment Analysis\")\n",
-    "    gr.Markdown(\"This is a custom DeBERTa model architecture for sentiment analysis with 15 labels: Openness, Anxiety, Confusion, Disapproval, Remorse, Accusation, Denial, Obscenity, Disinterest, Annoyance, Information, Greeting, Interest, Curiosity, or Acceptance.<br />Please enter your sentence(s) in the input box below and click the Submit button. The model will then process the input and provide the sentiment in one of the labels.<br/>The Test Example section below provides some input examples. Click on them and submit them to the model to see how it works.\")\n",
-    "\n",
-    "    txt = gr.Textbox(label=\"Input\", lines=2)\n",
-    "    txt_1 = gr.Textbox(value=\"\", label=\"Output\")\n",
-    "    btn = gr.Button(value=\"Submit\")\n",
-    "    btn.click(inference, inputs=txt, outputs= txt_1)\n",
-    "\n",
-    "    demoExample = [\n",
-    "        \"Hello, how are you?\",\n",
-    "        \"I am so happy to be here!\",\n",
-    "        \"i don't have time for u\"\n",
-    "    ]\n",
-    "\n",
-    "    gr.Markdown(\"## Text Examples\")\n",
-    "    gr.Examples(\n",
-    "        demoExample,\n",
-    "        txt,\n",
-    "        txt_1,\n",
-    "        inference\n",
-    "    )\n",
-    "\n",
-    "with gr.Blocks() as gptdemo:\n",
-    "\n",
-    "    gr.Markdown(\"## GPT Sentiment Analysis\")\n",
-    "    gr.Markdown(\"This a custom GPT model for sentiment analysis with 15 labels: Openness, Anxiety, Confusion, Disapproval, Remorse, Accusation, Denial, Obscenity, Disinterest, Annoyance, Information, Greeting, Interest, Curiosity, or Acceptance.<br />Please enter your sentence(s) in the input box below and click the Submit button. The model will then process the input and provide the sentiment in one of the labels.<br />The Test Example section below provides some input examples. Click on them and submit them to the model to see how it works.Please note that the input may be collected by service providers.\")\n",
-    "    txt = gr.Textbox(label=\"Input\", lines=2)\n",
-    "    txt_1 = gr.Textbox(value=\"\", label=\"Output\")\n",
-    "    btn = gr.Button(value=\"Submit\")\n",
-    "    btn.click(predict, inputs=txt, outputs= txt_1)\n",
-    "\n",
-    "    gptExample = [\n",
-    "        \"Hello, how are you?\",\n",
-    "        \"Are you busy at the moment?\",\n",
-    "        \"I'm doing real good\"\n",
-    "    ]\n",
-    "\n",
-    "    gr.Markdown(\"## Text Examples\")\n",
-    "    gr.Examples(\n",
-    "        gptExample,\n",
-    "        txt,\n",
-    "        txt_1,\n",
-    "        predict\n",
-    "    )\n",
-    "\n",
-    "\n",
-    "with gr.Blocks() as revisiondemo:\n",
-    "    gr.Markdown(\"## Conversation Revision\")\n",
-    "    gr.Markdown(\"This is a custom GPT model designed to generate possible response texts based on previous contexts. You can input a conversation between a caller and a volunteer, and the model will provide three possible responses based on the input. <br />The Test Example section below provides some input examples. Click on them and submit them to the model to see how it works. Please note that the input may be collected by service providers.\")\n",
-    "    txt = gr.Textbox(label=\"Input\", lines=2)\n",
-    "    txt_1 = gr.Textbox(value=\"\", label=\"Output\",lines=4)\n",
-    "    btn = gr.Button(value=\"Submit\")\n",
-    "    btn.click(revision, inputs=txt, outputs= txt_1)\n",
-    "\n",
-    "    revisionExample = [\"Caller: sup\\nVolunteer: Hey, how's it going?\\nCaller: not very well, actually\\nVolunteer: What's the matter?\\nCaller: it's my wife, don't worry about it\"]\n",
-    "\n",
-    "    with gr.Column():\n",
-    "        gr.Markdown(\"## Text Examples\")\n",
-    "        gr.Examples(\n",
-    "            revisionExample,\n",
-    "            [txt],\n",
-    "            txt_1,\n",
-    "            revision\n",
-    "        )\n",
-    "\n",
-    "\n",
-    "\n",
-    "\n",
-    "gr.TabbedInterface([demo, gptdemo,revisiondemo], [\"Model\", \"GPT\",\"Text Revision\"]\n",
-    ").launch(inline=False)"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.9"
-  },
-  "vscode": {
-   "interpreter": {
-    "hash": "76d9096663e4677afe736ff46b3dcdaff586dfdb471519f50b872333a086db78"
-   }
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}

app_spring2023.py

@@ -1,396 +0,0 @@
-#!/usr/bin/env python
-# coding: utf-8
-
-# In[ ]:
-
-
-import numpy as np
-import tensorflow as tf
-import tensorflow_addons as tfa
-from tensorflow.keras import layers
-import transformers
-import sentencepiece as spm
-#show the version of the package imported with text instructions\
-print("Tensorflow version: ", tf.__version__)
-print("Tensorflow Addons version: ", tfa.__version__)
-print("Transformers version: ", transformers.__version__)
-print("Sentencepiece version: ", spm.__version__)
-print("Numpy version: ", np.__version__)
-
-
-# In[ ]:
-
-
-class MeanPool(tf.keras.layers.Layer):
-    def call(self, inputs, mask=None):
-        broadcast_mask = tf.expand_dims(tf.cast(mask, "float32"), -1)
-        embedding_sum = tf.reduce_sum(inputs * broadcast_mask, axis=1)
-        mask_sum = tf.reduce_sum(broadcast_mask, axis=1)
-        mask_sum = tf.math.maximum(mask_sum, tf.constant([1e-9]))
-        return embedding_sum / mask_sum
-class WeightsSumOne(tf.keras.constraints.Constraint):
-    def __call__(self, w):
-        return tf.nn.softmax(w, axis=0)
-
-
-# In[ ]:
-
-
-tokenizer = transformers.AutoTokenizer.from_pretrained("microsoft/deberta-v3-large"
-)
-tokenizer.save_pretrained('./tokenizer/')
-
-cfg = transformers.AutoConfig.from_pretrained("microsoft/deberta-v3-large", output_hidden_states=True)
-cfg.hidden_dropout_prob = 0
-cfg.attention_probs_dropout_prob = 0
-cfg.save_pretrained('./tokenizer/')
-
-
-# In[ ]:
-
-
-def deberta_encode(texts, tokenizer=tokenizer):
-    input_ids = []
-    attention_mask = []
-    
-    for text in texts:
-        token = tokenizer(text, 
-                          add_special_tokens=True, 
-                          max_length=512, 
-                          return_attention_mask=True, 
-                          return_tensors="np", 
-                          truncation=True, 
-                          padding='max_length')
-        input_ids.append(token['input_ids'][0])
-        attention_mask.append(token['attention_mask'][0])
-    
-    return np.array(input_ids, dtype="int32"), np.array(attention_mask, dtype="int32")
-
-
-# In[ ]:
-
-
-MAX_LENGTH=512
-BATCH_SIZE=8
-
-
-# In[ ]:
-
-
-def get_model():
-    input_ids = tf.keras.layers.Input(
-        shape=(MAX_LENGTH,), dtype=tf.int32, name="input_ids"
-    )
-    
-    attention_masks = tf.keras.layers.Input(
-        shape=(MAX_LENGTH,), dtype=tf.int32, name="attention_masks"
-    )
-   
-    deberta_model = transformers.TFAutoModel.from_pretrained("microsoft/deberta-v3-large", config=cfg)
-    
-        
-    REINIT_LAYERS = 1
-    normal_initializer = tf.keras.initializers.GlorotUniform()
-    zeros_initializer = tf.keras.initializers.Zeros()
-    ones_initializer = tf.keras.initializers.Ones()
-
-#     print(f'\nRe-initializing encoder block:')
-    for encoder_block in deberta_model.deberta.encoder.layer[-REINIT_LAYERS:]:
-#         print(f'{encoder_block}')
-        for layer in encoder_block.submodules:
-            if isinstance(layer, tf.keras.layers.Dense):
-                layer.kernel.assign(normal_initializer(shape=layer.kernel.shape, dtype=layer.kernel.dtype))
-                if layer.bias is not None:
-                    layer.bias.assign(zeros_initializer(shape=layer.bias.shape, dtype=layer.bias.dtype))
-
-            elif isinstance(layer, tf.keras.layers.LayerNormalization):
-                layer.beta.assign(zeros_initializer(shape=layer.beta.shape, dtype=layer.beta.dtype))
-                layer.gamma.assign(ones_initializer(shape=layer.gamma.shape, dtype=layer.gamma.dtype))
-
-    deberta_output = deberta_model.deberta(
-        input_ids, attention_mask=attention_masks
-    )
-    hidden_states = deberta_output.hidden_states
-    
-    #WeightedLayerPool + MeanPool of the last 4 hidden states
-    stack_meanpool = tf.stack(
-        [MeanPool()(hidden_s, mask=attention_masks) for hidden_s in hidden_states[-4:]], 
-        axis=2)
-    
-    weighted_layer_pool = layers.Dense(1,
-                                       use_bias=False,
-                                       kernel_constraint=WeightsSumOne())(stack_meanpool)
-    
-    weighted_layer_pool = tf.squeeze(weighted_layer_pool, axis=-1)
-    output=layers.Dense(15,activation='linear')(weighted_layer_pool)
-    #x = layers.Dense(6, activation='linear')(x)
-    
-    #output = layers.Rescaling(scale=4.0, offset=1.0)(x)
-    model = tf.keras.Model(inputs=[input_ids, attention_masks], outputs=output)
-    
-    #Compile model with Layer-wise Learning Rate Decay
-    layer_list = [deberta_model.deberta.embeddings] + list(deberta_model.deberta.encoder.layer)
-    layer_list.reverse()
-    
-    INIT_LR = 1e-5
-    LLRDR = 0.9
-    LR_SCH_DECAY_STEPS = 1600
-    
-    lr_schedules = [tf.keras.optimizers.schedules.ExponentialDecay(
-        initial_learning_rate=INIT_LR * LLRDR ** i, 
-        decay_steps=LR_SCH_DECAY_STEPS, 
-        decay_rate=0.3) for i in range(len(layer_list))]
-    lr_schedule_head = tf.keras.optimizers.schedules.ExponentialDecay(
-        initial_learning_rate=1e-4, 
-        decay_steps=LR_SCH_DECAY_STEPS, 
-        decay_rate=0.3)
-    
-    optimizers = [tf.keras.optimizers.Adam(learning_rate=lr_sch) for lr_sch in lr_schedules]
-    
-    optimizers_and_layers = [(tf.keras.optimizers.Adam(learning_rate=lr_schedule_head), model.layers[-4:])] +\
-        list(zip(optimizers, layer_list))
-    
-    optimizer = tfa.optimizers.MultiOptimizer(optimizers_and_layers)
-    
-    model.compile(optimizer=optimizer,
-                 loss='mse',
-                 metrics=[tf.keras.metrics.RootMeanSquaredError()],
-                 )
-    return model
-
-
-# In[ ]:
-
-
-tf.keras.backend.clear_session()
-model = get_model()
-model.load_weights('./best_model_fold2.h5')
-
-
-# In[ ]:
-
-
-
-
-
-# In[ ]:
-
-
-# map the integer labels to their original string representation
-label_mapping = {
-    0: 'Greeting',
-    1: 'Curiosity',
-    2: 'Interest',
-    3: 'Obscene',
-    4: 'Annoyed',
-    5: 'Openness',
-    6: 'Anxious',
-    7: 'Acceptance',
-    8: 'Uninterested',
-    9: 'Informative',
-    10: 'Accusatory',
-    11: 'Denial',
-    12: 'Confused',
-    13: 'Disapproval',
-    14: 'Remorse'
-}
-
-#label_strings = [label_mapping[label] for label in labels]
-
-#print(label_strings)
-
-
-# In[ ]:
-
-
-def inference(texts):
-    prediction = model.predict(deberta_encode([texts]))
-    labels = np.argmax(prediction, axis=1)
-    label_strings = [label_mapping[label] for label in labels]
-    return label_strings[0]
-
-
-# # GPT
-
-# In[ ]:
-
-
-import openai
-import os
-import pandas as pd
-import gradio as gr
-
-
-# In[ ]:
-
-
-openai.organization = os.environ['org_id']
-openai.api_key = os.environ['openai_api']
-model_version = "gpt-3.5-turbo"
-model_token_limit = 10
-model_temperature = 0.1
-
-
-# In[ ]:
-
-
-def generatePrompt () :
-    labels = ["Openness", 
-    "Anxious",
-    "Confused",
-    "Disapproval",
-    "Remorse",
-    "Uninterested",
-    "Accusatory",
-    "Annoyed",
-    "Interest",
-    "Curiosity",
-    "Acceptance",
-    "Obscene",
-    "Denial",
-    "Informative",
-    "Greeting"]
-
-    formatted_labels = ', '.join(labels[:-1]) + ', or ' + labels[-1] + '.'
-
-    label_set = ["Openness", "Anxious", "Confused", "Disapproval", "Remorse", "Accusatory",
-          "Denial", "Obscene", "Uninterested", "Annoyed", "Informative", "Greeting",
-          "Interest", "Curiosity", "Acceptance"]
-
-    formatted_labels = ', '.join(label_set[:-1]) + ', or ' + label_set[-1] + '.\n'
-
-    # The basic task to assign GPT (in natural language)
-    base_task = "Classify the following text messages into one of the following categories using one word: " + formatted_labels
-    base_task += "Provide only a one word response. Use only the labels provided.\n"
-
-    return base_task
-
-
-# In[ ]:
-
-
-def predict(message):
-    
-    prompt = [{"role": "user", "content": generatePrompt () + "Text: "+ message}]
-    
-    response = openai.ChatCompletion.create(
-                    model=model_version,
-                    temperature=model_temperature,
-                    max_tokens=model_token_limit,
-                    messages=prompt
-                )
-        
-    return response["choices"][0]["message"]["content"]
-
-
-# # Update
-
-# In[ ]:
-
-
-model_version = "gpt-3.5-turbo"
-model_token_limit = 2000
-model_temperature = 0.1
-
-
-# In[ ]:
-
-
-def revision(message):
-    base_prompt = "Here is a conversation between a Caller and a Volunteer. The Volunteer is trying to be as non-accusatory as possible but also wants to get as much information about the caller as possible. What should the volunteer say next in this exchange? Proved 3 possible responses."
-
-    prompt = [{"role": "user", "content": base_prompt + message}]
-    
-    response = openai.ChatCompletion.create(
-                    model=model_version,
-                    temperature=model_temperature,
-                    max_tokens=model_token_limit,
-                    messages=prompt
-                )
-
-    return response["choices"][0]["message"]["content"]
-
-
-# In[ ]:
-
-
-import gradio as gr
-
-def combine(a):
-    return a + "hello"
-
-
-
-
-with gr.Blocks() as demo:
-    gr.Markdown("## DeBERTa Sentiment Analysis")
-    gr.Markdown("This is a custom DeBERTa model architecture for sentiment analysis with 15 labels: Openness, Anxiety, Confusion, Disapproval, Remorse, Accusation, Denial, Obscenity, Disinterest, Annoyance, Information, Greeting, Interest, Curiosity, or Acceptance.<br />Please enter your sentence(s) in the input box below and click the Submit button. The model will then process the input and provide the sentiment in one of the labels.<br/>The Test Example section below provides some input examples. Click on them and submit them to the model to see how it works.")
-
-    txt = gr.Textbox(label="Input", lines=2)
-    txt_1 = gr.Textbox(value="", label="Output")
-    btn = gr.Button(value="Submit")
-    btn.click(inference, inputs=txt, outputs= txt_1)
-
-    demoExample = [
-        "Hello, how are you?",
-        "I am so happy to be here!",
-        "i don't have time for u"
-    ]
-
-    gr.Markdown("## Text Examples")
-    gr.Examples(
-        demoExample,
-        txt,
-        txt_1,
-        inference
-    )
-
-with gr.Blocks() as gptdemo:
-
-    gr.Markdown("## GPT Sentiment Analysis")
-    gr.Markdown("This a custom GPT model for sentiment analysis with 15 labels: Openness, Anxiety, Confusion, Disapproval, Remorse, Accusation, Denial, Obscenity, Disinterest, Annoyance, Information, Greeting, Interest, Curiosity, or Acceptance.<br />Please enter your sentence(s) in the input box below and click the Submit button. The model will then process the input and provide the sentiment in one of the labels.<br />The Test Example section below provides some input examples. Click on them and submit them to the model to see how it works.Please note that the input may be collected by service providers.")
-    txt = gr.Textbox(label="Input", lines=2)
-    txt_1 = gr.Textbox(value="", label="Output")
-    btn = gr.Button(value="Submit")
-    btn.click(predict, inputs=txt, outputs= txt_1)
-
-    gptExample = [
-        "Hello, how are you?",
-        "Are you busy at the moment?",
-        "I'm doing real good"
-    ]
-
-    gr.Markdown("## Text Examples")
-    gr.Examples(
-        gptExample,
-        txt,
-        txt_1,
-        predict
-    )
-
-
-with gr.Blocks() as revisiondemo:
-    gr.Markdown("## Conversation Revision")
-    gr.Markdown("This is a custom GPT model designed to generate possible response texts based on previous contexts. You can input a conversation between a caller and a volunteer, and the model will provide three possible responses based on the input. <br />The Test Example section below provides some input examples. Click on them and submit them to the model to see how it works. Please note that the input may be collected by service providers.")
-    txt = gr.Textbox(label="Input", lines=2)
-    txt_1 = gr.Textbox(value="", label="Output",lines=4)
-    btn = gr.Button(value="Submit")
-    btn.click(revision, inputs=txt, outputs= txt_1)
-
-    revisionExample = ["Caller: sup\nVolunteer: Hey, how's it going?\nCaller: not very well, actually\nVolunteer: What's the matter?\nCaller: it's my wife, don't worry about it"]
-
-    with gr.Column():
-        gr.Markdown("## Text Examples")
-        gr.Examples(
-            revisionExample,
-            [txt],
-            txt_1,
-            revision
-        )
-
-
-
-
-gr.TabbedInterface([demo, gptdemo,revisiondemo], ["Model", "GPT","Text Revision"]
-).launch(inline=False)
-

main.py

@@ -1,14 +1,17 @@
 #
-# to run app, cd to app directory and do:
-# python ./app.py --active-listener --class-weight --residual
+# to run app, in root directory do:
+# python ./main.py
 #
 
 import gradio as gr
-from app import cosmic_view, gpt_view
+from app import cosmic_view, gpt_view, deberta_view
 
-cosmic_model = cosmic_view.cosmic_ui()
-gpt_model = gpt_view.gpt_ui()
-demo = gr.TabbedInterface([cosmic_model, gpt_model], ["COSMIC", "GPT"])
+cosmic_tab = cosmic_view.cosmic_ui()
+gpt_tab = gpt_view.gpt_ui()
+deberta_tab = deberta_view.deberta_ui()
+demo = gr.TabbedInterface(
+    [cosmic_tab, gpt_tab, deberta_tab], ["COSMIC", "GPT", "DeBERTa"]
+)
 
 if __name__ == "__main__":
     demo.launch()

requirements.txt

@@ -1,12 +1,15 @@
+absl-py==2.0.0
 aiofiles==23.2.1
 altair==5.2.0
 annotated-types==0.6.0
 antlr4-python3-runtime==4.8
 anyio==3.7.1
 asttokens==2.4.1
+astunparse==1.6.3
 attrs==23.1.0
 bitarray==2.8.3
 blis==0.7.11
+cachetools==5.3.2
 catalogue==2.0.10
 certifi==2023.11.17
 cffi==1.16.0
@@ -30,12 +33,19 @@ fairseq==0.12.2
 fastapi==0.104.1
 ffmpy==0.3.1
 filelock==3.13.1
+flatbuffers==23.5.26
 fonttools==4.45.1
 fsspec==2023.10.0
 ftfy==6.1.3
+gast==0.4.0
+google-auth==2.25.1
+google-auth-oauthlib==1.0.0
+google-pasta==0.2.0
 gradio==4.7.1
 gradio_client==0.7.0
+grpcio==1.60.0
 h11==0.14.0
+h5py==3.10.0
 httpcore==1.0.2
 httpx==0.25.2
 huggingface-hub==0.19.4
@@ -44,6 +54,7 @@ idna==3.6
 importlib-resources==6.1.1
 ipykernel==6.27.1
 ipython==8.18.1
+jax==0.4.21
 jedi==0.19.1
 Jinja2==3.1.2
 joblib==1.3.2
@@ -51,22 +62,29 @@ jsonschema==4.20.0
 jsonschema-specifications==2023.11.2
 jupyter_client==8.6.0
 jupyter_core==5.5.0
+kaleido==0.2.1
+keras==2.12.0
 kiwisolver==1.4.5
 langcodes==3.3.0
+libclang==16.0.6
 lxml==4.9.3
+Markdown==3.5.1
 markdown-it-py==3.0.0
 MarkupSafe==2.1.3
 matplotlib==3.8.2
 matplotlib-inline==0.1.6
 mdurl==0.1.2
+ml-dtypes==0.2.0
 mpmath==1.3.0
 murmurhash==1.0.10
 nest-asyncio==1.5.8
 networkx==3.2.1
 nltk==3.8.1
-numpy==1.26.2
+numpy==1.23.5
+oauthlib==3.2.2
 omegaconf==2.0.6
 openai==1.3.7
+opt-einsum==3.3.0
 orjson==3.9.10
 packaging==23.2
 pandas==2.1.3
@@ -76,8 +94,11 @@ platformdirs==4.0.0
 portalocker==2.8.2
 preshed==3.0.9
 prompt-toolkit==3.0.41
+protobuf==4.23.4
 psutil==5.9.6
 pure-eval==0.2.2
+pyasn1==0.5.1
+pyasn1-modules==0.3.0
 pycparser==2.21
 pydantic==2.5.2
 pydantic_core==2.14.5
@@ -88,15 +109,22 @@ python-dateutil==2.8.2
 python-dotenv==1.0.0
 python-multipart==0.0.6
 pytz==2023.3.post1
+pywin32==306
 PyYAML==6.0.1
 pyzmq==25.1.1
 referencing==0.31.1
 regex==2023.10.3
 requests==2.31.0
+requests-oauthlib==1.3.1
 rich==13.7.0
 rpds-py==0.13.2
+rsa==4.9
 sacrebleu==2.3.3
+safetensors==0.4.1
+scipy==1.11.4
+seaborn==0.13.0
 semantic-version==2.10.0
+sentencepiece==0.1.99
 shellingham==1.5.4
 six==1.16.0
 smart-open==6.4.0
@@ -109,22 +137,36 @@ stack-data==0.6.3
 starlette==0.27.0
 sympy==1.12
 tabulate==0.9.0
+tensorboard==2.12.3
+tensorboard-data-server==0.7.2
+tensorflow==2.12.0
+tensorflow-addons==0.22.0
+tensorflow-estimator==2.12.0
+tensorflow-intel==2.12.0
+tensorflow-io-gcs-filesystem==0.31.0
+termcolor==2.4.0
+textblob==0.17.1
 thinc==8.2.1
+tokenizers==0.15.0
 tomlkit==0.12.0
 toolz==0.12.0
-torch==2.1.1
+torch==2.1.1+cu121
 torchaudio==2.1.1
-torchvision==0.16.1
+torchvision==0.16.1+cu121
 tornado==6.4
 tqdm==4.66.1
 traitlets==5.14.0
+transformers==4.35.2
+typeguard==2.13.3
 typer==0.9.0
 typing_extensions==4.8.0
 tzdata==2023.3
 urllib3==2.1.0
 uvicorn==0.24.0.post1
+vega-datasets==0.9.0
 wasabi==1.1.2
 wcwidth==0.2.12
 weasel==0.3.4
 websockets==11.0.3
-seaborn==0.13.0
+Werkzeug==3.0.1
+wrapt==1.14.1