files added

app.py

@@ -0,0 +1,116 @@
+import logging
+import gradio as gr
+import os
+import zipfile
+import pydub
+import datetime
+
+import openai
+import jwt
+
+from summarizer import count_tokens,main_summarizer_action_items,main_summarizer_meet
+from decouple import config
+
+DEBUG = True
+API_KEY = config('API_KEY')
+model_id = 'whisper-1'
+SECRET_KEY = "$§%§$secret"
+
+# Set the summarization parameters
+# Set the maximum chunk size and tokens per chunk
+max_chunk_size = 2000
+max_tokens_per_chunk = 500
+temperature = 0.7
+top_p = 0.5
+frequency_penalty = 0.5
+temp_dir = os.path.join(os.path.dirname(__file__), 'temp')
+
+title = description = article = "Meeting Summariser ⚡️ "
+
+logger = logging.getLogger("Summariser")
+logger.setLevel(logging.INFO)
+ch = logging.StreamHandler()
+ch.setLevel(logging.INFO)
+formatter = logging.Formatter("%(asctime)s;%(levelname)s;%(message)s", "%Y-%m-%d %H:%M:%S")
+ch.setFormatter(formatter)
+logger.addHandler(ch)
+
+def authentication(username, password):
+    if username == "admin" and password == "admin":
+        return True
+
+
+def transcribe_audio(audio_file_path, temp_folder_path):
+    if DEBUG:
+        return "This is a test transcription"
+    
+    max_size_bytes = 20 * 1024 * 1024  # 24 MB
+
+    if os.path.getsize(audio_file_path) <= max_size_bytes:
+        media_file = open(audio_file_path, 'rb')
+        response = openai.Audio.transcribe(
+            api_key=API_KEY,
+            model=model_id,
+            file=media_file
+        )
+        return response['text']
+    else:
+        sound = pydub.AudioSegment.from_file(audio_file_path, format="mp3")
+        chunks = pydub.utils.make_chunks(sound, max_size_bytes)
+        transcriptions = []
+        for i, chunk in enumerate(chunks):
+            print("chunk ", i)
+            chunk_path = os.path.join(temp_folder_path, f"audio_chunk_{i}.mp3")
+            chunk.export(chunk_path, format="mp3")
+            response = openai.Audio.transcribe(api_key=API_KEY,model=model_id,file=open(chunk_path, 'rb'))
+            transcriptions.append(response['text'])
+
+        return ' '.join(transcriptions)
+
+def download_files(transcription: str, summary: str):
+    time_now = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+    # Create transcription file
+    transcript_file_path = os.path.join(temp_dir, f'transcription_{time_now}.txt')
+    with open(transcript_file_path, 'w') as f:
+        f.write(transcription)
+    # Create summary file
+    summary_file_path = os.path.join(temp_dir, f'summary_{time_now}.txt')
+    with open(summary_file_path, 'w') as f:
+        f.write(summary)
+    # Create zip file
+    zip_file_path = os.path.join(temp_dir, 'download.zip')
+    with zipfile.ZipFile(zip_file_path, 'w', zipfile.ZIP_DEFLATED) as zip_file:
+        # Add transcription file to zip
+        zip_file.write(transcript_file_path, 'transcription.txt')
+        # Add summary file to zip
+        zip_file.write(summary_file_path, 'summary.txt')
+    return zip_file_path
+    
+def clean_trancript(text):
+    return text
+
+def main_meet_summarizer(audio_file):
+    
+    summary = ""
+    transcript = ""
+    action_items = ""
+    
+    print("Starting Transcription")
+    transcript = transcribe_audio(audio_file,temp_dir)
+    print(f"Starting Summarization | {count_tokens(transcript)}")
+    cleaned_transcript = clean_trancript(transcript)
+    summary = main_summarizer_meet(cleaned_transcript, debug=DEBUG)
+    action_items = main_summarizer_action_items(cleaned_transcript, debug=DEBUG)
+    print("Finished Summarization")
+    return summary,transcript,download_files(transcription = transcript, summary = (summary + action_items))
+
+
+summarizer_interface = gr.Interface(
+    fn=main_meet_summarizer,
+    inputs=[gr.inputs.Audio(source='upload', type='filepath', label='Audio File')],
+    outputs=[gr.outputs.Textbox(label='Summary'), gr.outputs.Textbox(label='Transcription'),gr.outputs.File(label="Download files here"),],
+    title='Summarizer',
+    description='Transcribe speech in an audio file & summarize it.',
+)
+
+summarizer_interface.launch(debug=True)

requirements.txt

@@ -0,0 +1,7 @@
+gradio==3.28.3
+openai
+openai-async
+asyncio
+nest-asyncio
+transformers==4.26.1
+torch

summarizer.py

@@ -0,0 +1,101 @@
+import openai_async
+import asyncio
+import nest_asyncio
+
+import torch
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("gpt2")
+
+def count_tokens(text):
+    input_ids = torch.tensor(tokenizer.encode(text)).unsqueeze(0)
+    return input_ids.shape[1]
+
+def break_up_file_to_chunks(text, chunk_size=2000, overlap=100):
+    tokens = tokenizer.encode(text)
+    num_tokens = len(tokens)
+    chunks = []
+    for i in range(0, num_tokens, chunk_size - overlap):
+        chunk = tokens[i:i + chunk_size]
+        chunks.append(chunk)
+    
+    return chunks
+
+async def summarize_meeting(prompt, timeout, max_tokens):
+    
+    #timeout = 30
+    temperature = 0.5
+    #max_tokens = 1000
+    top_p = 1
+    frequency_penalty = 0
+    presence_penalty = 0
+    
+    # Call the OpenAI GPT-3 API
+    response = await openai_async.complete(
+        api_key = API_KEY,
+        timeout=timeout,
+        payload={
+            "model": "gpt-3.5-turbo",
+            "prompt": prompt,
+            "temperature": temperature,
+            "max_tokens": max_tokens,
+            "top_p": top_p,
+            "frequency_penalty": frequency_penalty,
+            "presence_penalty": presence_penalty
+        },
+    )
+
+    # Return the generated text
+    return response
+
+def main_summarizer_meet(text, debug=False):
+    if debug:
+        return "This is a test summary function"
+    prompt_response = []
+    prompt_tokens = []
+
+    chunks = break_up_file_to_chunks(text)
+
+    for i, chunk in enumerate(chunks):
+        prompt_request = (
+            f"Summarize this meeting transcript: {tokenizer.decode(chunks[i])}"
+        )
+
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+
+        response = loop.run_until_complete(summarize_meeting(prompt = prompt_request, timeout=30, max_tokens = 1000))
+
+        prompt_response.append(response.json()["choices"][0]["text"].strip())
+        prompt_tokens.append(response.json()["usage"]["total_tokens"])
+
+    prompt_request = f"Consoloidate these meeting summaries: {prompt_response}"
+
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+    response = loop.run_until_complete(summarize_meeting(prompt = prompt_request, timeout=45, max_tokens = 1000))
+    return response.json()["choices"][0]["text"].strip()
+
+# -----------------------------
+
+def main_summarizer_action_items(text, debug=False):
+    
+    if debug:
+        return "This is a test action items function"
+
+    action_response = []
+    action_tokens = []
+
+    chunks = break_up_file_to_chunks(text)
+
+    for i, chunk in enumerate(chunks):
+        prompt_request = f"Provide a list of action items with a due date from the provided meeting transcript text: {tokenizer.decode(chunks[i])}"
+
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        response = loop.run_until_complete(summarize_meeting(prompt = prompt_request, timeout=30, max_tokens = 1000))
+
+        action_response.append(response.json()["choices"][0]["text"].strip())
+        action_tokens.append(response.json()["usage"]["total_tokens"])
+
+    return '\n'.join(action_response)