Duplicate from enoreyes/call-sentiment-demo

Co-authored-by: Eno Reyes <enoreyes@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,31 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bin.* filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+example_audio.wav filter=lfs diff=lfs merge=lfs -text
+short-take-1.wav filter=lfs diff=lfs merge=lfs -text
+Customer_Support_Call.wav filter=lfs diff=lfs merge=lfs -text

Customer_Support_Call.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db6489658bb04f84503531d628a67028de9d754ee0b18cf229f39deec7828001
+size 31497612

README.md

@@ -0,0 +1,13 @@
+---
+title: Call Sentiment Blocks 2
+emoji: 🐠
+colorFrom: blue
+colorTo: green
+sdk: gradio
+sdk_version: 3.11.0
+app_file: app.py
+pinned: false
+duplicated_from: enoreyes/call-sentiment-demo
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces#reference

app.py

@@ -0,0 +1,117 @@
+import os
+import re
+import functools
+from functools import partial
+
+import requests
+import pandas as pd
+import plotly.express as px
+
+import torch
+import gradio as gr
+from transformers import pipeline, Wav2Vec2ProcessorWithLM
+from pyannote.audio import Pipeline
+import whisperx
+
+from utils import split, create_fig
+from utils import speech_to_text as stt
+
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+device = 0 if torch.cuda.is_available() else -1
+
+# display if the sentiment value is above these thresholds
+thresholds = {"joy": 0.99,"anger": 0.95,"surprise": 0.95,"sadness": 0.98,"fear": 0.95,"love": 0.99,}
+
+color_map = {"joy": "green","anger": "red","surprise": "yellow","sadness": "blue","fear": "orange","love": "purple",}
+
+# Audio components
+whisper_device = "cuda" if torch.cuda.is_available() else "cpu"
+whisper = whisperx.load_model("tiny.en", whisper_device)
+alignment_model, metadata = whisperx.load_align_model(language_code="en", device=whisper_device)
+speaker_segmentation = Pipeline.from_pretrained("pyannote/speaker-diarization@2.1",
+                                    use_auth_token=os.environ['ENO_TOKEN'])
+
+
+# Text components
+emotion_pipeline = pipeline(
+    "text-classification",
+    model="bhadresh-savani/distilbert-base-uncased-emotion",
+    device=device,
+)
+summarization_pipeline = pipeline(
+    "summarization",
+    model="knkarthick/MEETING_SUMMARY",
+    device=device
+)
+
+EXAMPLES = [["Customer_Support_Call.wav"]]
+
+
+speech_to_text = partial(
+    stt, 
+    speaker_segmentation=speaker_segmentation, 
+    whisper=whisper, 
+    alignment_model=alignment_model, 
+    metadata=metadata, 
+    whisper_device=whisper_device
+    )
+
+def summarize(diarized, summarization_pipeline):
+    text = ""
+    for d in diarized:
+        text += f"\n{d[1]}: {d[0]}"
+
+    return summarization_pipeline(text)[0]["summary_text"]
+
+def sentiment(diarized, emotion_pipeline):
+    customer_sentiments = []
+
+    for i in range(0, len(diarized), 2):
+        speaker_speech, speaker_id = diarized[i]
+        sentences = split(speaker_speech)
+
+        if "Customer" in speaker_id:
+            outputs = emotion_pipeline(sentences)
+            for idx, (o, t) in enumerate(zip(outputs, sentences)):
+                if o["score"] > thresholds[o["label"]]:
+                    customer_sentiments.append((t, o["label"]))
+
+    return customer_sentiments
+
+with gr.Blocks() as demo:
+
+    with gr.Row():
+        with gr.Column():
+            audio = gr.Audio(label="Audio file", type="filepath")
+            btn = gr.Button("Transcribe and Diarize")
+
+            gr.Markdown("**Call Transcript:**")
+            diarized = gr.HighlightedText(label="Call Transcript")
+            gr.Markdown("Summarize Speaker")
+            sum_btn = gr.Button("Get Summary")
+            summary = gr.Textbox(lines=4)
+            sentiment_btn = gr.Button("Get Customer Sentiment")
+            analyzed = gr.HighlightedText(color_map=color_map)
+
+        with gr.Column():
+            gr.Markdown("## Example Files")
+            gr.Examples(
+                examples=EXAMPLES,
+                inputs=[audio],
+                outputs=[diarized],
+                fn=speech_to_text,
+                cache_examples=True
+            )
+    # when example button is clicked, convert audio file to text and diarize
+    btn.click(
+        fn=speech_to_text,
+        inputs=audio,
+        outputs=diarized,
+    )
+    # when summarize checkboxes are changed, create summary
+    sum_btn.click(fn=partial(summarize, summarization_pipeline=summarization_pipeline), inputs=[diarized], outputs=summary)
+
+    # when sentiment button clicked, display highlighted text and plot
+    sentiment_btn.click(fn=partial(sentiment, emotion_pipeline=emotion_pipeline), inputs=diarized, outputs=[analyzed])
+
+demo.launch(debug=1)

example_audio.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43166418f743e61807c7681944bf344c4720924adb4e5879dfa954dc7ecc82b2
+size 3202638

packages.txt

@@ -0,0 +1,2 @@
+libsndfile1
+ffmpeg

requirements.txt

@@ -0,0 +1,12 @@
+torch==1.11
+transformers==4.26.1
+torchvision==0.12.0
+torchaudio==0.11.0
+torchtext==0.12.0
+speechbrain==0.5.12
+pyannote.audio
+librosa
+requests
+speechbrain
+plotly
+git+https://github.com/m-bain/whisperx.git

short-take-1.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf15193510fc5a5680fdfdffda6c7cc5b8595bdde3d267b9ef5223e62035a952
+size 20079500

utils.py

@@ -0,0 +1,116 @@
+import re
+import functools
+import requests
+import pandas as pd
+import plotly.express as px
+import torch
+import gradio as gr
+from transformers import pipeline, Wav2Vec2ProcessorWithLM
+from pyannote.audio import Pipeline
+from librosa import load, resample
+import whisperx
+
+import re
+alphabets= "([A-Za-z])"
+prefixes = "(Mr|St|Mrs|Ms|Dr)[.]"
+suffixes = "(Inc|Ltd|Jr|Sr|Co)"
+starters = "(Mr|Mrs|Ms|Dr|He\s|She\s|It\s|They\s|Their\s|Our\s|We\s|But\s|However\s|That\s|This\s|Wherever)"
+acronyms = "([A-Z][.][A-Z][.](?:[A-Z][.])?)"
+websites = "[.](com|net|org|io|gov)"
+
+def split(text):
+    text = " " + text + "  "
+    text = text.replace("\n"," ")
+    text = re.sub(prefixes,"\\1<prd>",text)
+    text = re.sub(websites,"<prd>\\1",text)
+    if "Ph.D" in text: text = text.replace("Ph.D.","Ph<prd>D<prd>")
+    text = re.sub("\s" + alphabets + "[.] "," \\1<prd> ",text)
+    text = re.sub(acronyms+" "+starters,"\\1<stop> \\2",text)
+    text = re.sub(alphabets + "[.]" + alphabets + "[.]" + alphabets + "[.]","\\1<prd>\\2<prd>\\3<prd>",text)
+    text = re.sub(alphabets + "[.]" + alphabets + "[.]","\\1<prd>\\2<prd>",text)
+    text = re.sub(" "+suffixes+"[.] "+starters," \\1<stop> \\2",text)
+    text = re.sub(" "+suffixes+"[.]"," \\1<prd>",text)
+    text = re.sub(" " + alphabets + "[.]"," \\1<prd>",text)
+    if "”" in text: text = text.replace(".”","”.")
+    if "\"" in text: text = text.replace(".\"","\".")
+    if "!" in text: text = text.replace("!\"","\"!")
+    if "?" in text: text = text.replace("?\"","\"?")
+    text = text.replace(".",".<stop>")
+    text = text.replace("?","?<stop>")
+    text = text.replace("!","!<stop>")
+    text = text.replace("<prd>",".")
+    sentences = text.split("<stop>")
+    sentences = sentences[:-1]
+    sentences = [s.strip() for s in sentences]
+    return sentences
+
+def create_fig(x_min, x_max, to_plot, plot_sentences):
+    x, y = list(zip(*to_plot))
+    
+    x_min -= 5
+    x_max += 5
+
+    plot_df = pd.DataFrame(
+        data={
+            "x": x,
+            "y": y,
+            "sentence": plot_sentences,
+        }
+    )
+
+    fig = px.line(
+        plot_df,
+        x="x",
+        y="y",
+        hover_data={
+            "sentence": True,
+            "x": True,
+            "y": False,
+        },
+        labels={"x": "time (seconds)", "y": "sentiment"},
+        title=f"Customer sentiment over time",
+        markers=True,
+    )
+
+    fig = fig.update_yaxes(categoryorder="category ascending")
+    fig = fig.update_layout(
+        font=dict(
+            size=18,
+        ),
+        xaxis_range=[x_min, x_max],
+    )
+    
+    return fig
+
+def speech_to_text(speech_file, speaker_segmentation, whisper, alignment_model, metadata, whisper_device):
+    speaker_output = speaker_segmentation(speech_file)
+    result = whisper.transcribe(speech_file)
+
+    chunks = whisperx.align(result["segments"], alignment_model, metadata, speech_file, whisper_device)["word_segments"]
+
+    diarized_output = []
+    i = 0
+    speaker_counter = 0
+
+    # New iteration every time the speaker changes
+    for turn, _, _ in speaker_output.itertracks(yield_label=True):
+
+        speaker = "Customer" if speaker_counter % 2 == 0 else "Support"
+        diarized = ""
+        while i < len(chunks) and chunks[i]["end"] <= turn.end:
+            diarized += chunks[i]["text"] + " "
+            i += 1
+
+        if diarized != "":
+            # diarized = rpunct.punctuate(re.sub(eng_pattern, "", diarized), lang="en")
+
+            diarized_output.extend(
+                [
+                    (diarized, speaker),
+                    ("from {:.2f}-{:.2f}".format(turn.start, turn.end), None),
+                ]
+            )
+
+            speaker_counter += 1
+
+    return diarized_output