File size: 10,882 Bytes
dcab4e1
 
 
 
 
 
 
 
 
89524ee
dcab4e1
 
 
 
 
a7d4590
 
dcab4e1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
89524ee
dcab4e1
 
 
 
 
 
d53ce16
dcab4e1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
89524ee
dcab4e1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d53ce16
dcab4e1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
from elevenlabs import VoiceSettings
from elevenlabs.client import ElevenLabs
from transformers import M2M100ForConditionalGeneration, M2M100Tokenizer
import whisper
from ai71  import AI71
from datetime import datetime
import os
import time
from pydub import AudioSegment
# from IPython.display import Audio, display, Video, HTML
# import assemblyai as aai
from base64 import b64encode
import gradio as gr
import concurrent.futures

AI71_API_KEY = os.getenv('AI71_API_KEY')
XI_API_KEY = os.getenv('ELEVEN_LABS_API_KEY')
client = ElevenLabs(api_key=XI_API_KEY)

model = M2M100ForConditionalGeneration.from_pretrained("facebook/m2m100_1.2B")
tokenizer = M2M100Tokenizer.from_pretrained("facebook/m2m100_1.2B")
transcriber = whisper.load_model("turbo")

language_codes = {"English":"en", "Hindi":"hi", "Portuguese":"pt", "Chinese":"zh", "Spanish":"es",
"French":"fr", "German":"de", "Japanese":"ja", "Arabic":"ar", "Russian":"ru",
"Korean":"ko", "Indonesian":"id", "Italian":"it", "Dutch":"nl","Turkish":"tr",
"Polish":"pl", "Swedish":"sv", "Filipino":"fil", "Malay":"ms", "Romanian":"ro",
"Ukrainian":"uk", "Greek":"el", "Czech":"cs", "Danish":"da", "Finnish":"fi",
"Bulgarian":"bg", "Croatian":"hr", "Slovak":"sk"}

meeting_texts = []
n_participants = 4 # This can be adjusted based on the number of people in the call
language_choices = ["English", "Polish", "Hindi", "Arabic"]


def wait_for_dubbing_completion(dubbing_id: str) -> bool:
    """
    Waits for the dubbing process to complete by periodically checking the status.

    Args:
        dubbing_id (str): The dubbing project id.

    Returns:
        bool: True if the dubbing is successful, False otherwise.
    """
    MAX_ATTEMPTS = 120
    CHECK_INTERVAL = 10  # In seconds

    for _ in range(MAX_ATTEMPTS):
        metadata = client.dubbing.get_dubbing_project_metadata(dubbing_id)
        if metadata.status == "dubbed":
            return True
        elif metadata.status == "dubbing":
            print(
                "Dubbing in progress... Will check status again in",
                CHECK_INTERVAL,
                "seconds.",
            )
            time.sleep(CHECK_INTERVAL)
        else:
            print("Dubbing failed:", metadata.error_message)
            return False

    print("Dubbing timed out")
    return False

def download_dubbed_file(dubbing_id: str, language_code: str) -> str:
    """
    Downloads the dubbed file for a given dubbing ID and language code.

    Args:
        dubbing_id: The ID of the dubbing project.
        language_code: The language code for the dubbing.

    Returns:
        The file path to the downloaded dubbed file.
    """
    dir_path = f"data/{dubbing_id}"
    os.makedirs(dir_path, exist_ok=True)

    file_path = f"{dir_path}/{language_code}.mp4"
    with open(file_path, "wb") as file:
        for chunk in client.dubbing.get_dubbed_file(dubbing_id, language_code):
            file.write(chunk)

    return file_path

def create_dub_from_file(
    input_file_path: str,
    file_format: str,
    source_language: str,
    target_language: str,
):
# ) -> Optional[str]:
    """
    Dubs an audio or video file from one language to another and saves the output.

    Args:
        input_file_path (str): The file path of the audio or video to dub.
        file_format (str): The file format of the input file.
        source_language (str): The language of the input file.
        target_language (str): The target language to dub into.

    Returns:
        Optional[str]: The file path of the dubbed file or None if operation failed.
    """
    if not os.path.isfile(input_file_path):
        raise FileNotFoundError(f"The input file does not exist: {input_file_path}")

    with open(input_file_path, "rb") as audio_file:
        response = client.dubbing.dub_a_video_or_an_audio_file(
            file=(os.path.basename(input_file_path), audio_file, file_format), # Optional file
            target_lang=target_language, # The target language to dub the content into. Can be none if dubbing studio editor is enabled and running manual mode
            # mode="automatic", # automatic or manual.
            source_lang=source_language, # Source language
            num_speakers=1, # Number of speakers to use for the dubbing.
            watermark=True,  # Whether to apply watermark to the output video.
        )

    # rest of the code
    dubbing_id = response.dubbing_id
    if wait_for_dubbing_completion(dubbing_id):
        output_file_path = download_dubbed_file(dubbing_id, target_language)
        return output_file_path
    else:
        return None


def summarize(meeting_texts=meeting_texts):
    mt = ', '.join([f"{k}: {v}" for i in meeting_texts for k, v in i.items()])
    meeting_date_time = str(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
    meeting_texts = meeting_date_time + '\n' + mt

    meeting_conversation_processed ='\n'.join(mt)
    # print("M:", session_conversation_processed)

    minutes_of_meeting = ""
    for chunk in AI71(AI71_API_KEY.strip()).chat.completions.create(
        model="tiiuae/falcon-180b-chat",
        messages=[
            {"role": "system", "content": """You are an expereiced Secretary who can summarize meeting discussions into minutes of meeting.
            Summarize the meetings discussions provided as Speakerwise conversation. Ensure to mention the title as 'Minutes of Meeting held on {meeting_date_time} and present the summary with better viewing format and title in bold letters"""},
            {"role": "user", "content": meeting_conversation_processed},
        ],
        stream=True,
    ):
        if chunk.choices[0].delta.content:
            summary = chunk.choices[0].delta.content
            minutes_of_meeting += summary
    minutes_of_meeting = minutes_of_meeting.replace('User:', '').strip()
    print("\n")
    print("minutes_of_meeting:", minutes_of_meeting)
    return minutes_of_meeting


# Placeholder function for speech to text conversion
def speech_to_text(video):
    print('Started transcribing')
    audio = AudioSegment.from_file(video)
    audio.export('temp.mp3', format="mp3")
    transcript= transcriber.transcribe('temp.mp3')['text']
    print('transcript:', transcript)
    return transcript

# Placeholder function for translating text
def translate_text(text, source_language,target_language):
    tokenizer.src_lang = source_language
    encoded_ln = tokenizer(text, return_tensors="pt")
    generated_tokens = model.generate(**encoded_ln, forced_bos_token_id=tokenizer.get_lang_id(target_language))
    translated_text = tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)[0]
    print('translated_text:', translated_text)
    return translated_text

# Placeholder function for dubbing (text-to-speech in another language)
def synthesize_speech(video, source_language,target_language):
    print('Started dubbing')
    dub_video = create_dub_from_file(input_file_path = video,
      file_format = 'audio/mpeg',
      source_language = source_language,
      target_language = target_language)
    return dub_video

# This function handles the processing when any participant speaks
def process_speaker(video, speaker_idx, n_participants, *language_list):
    transcript = speech_to_text(video)

    # Create outputs for each participant
    outputs = []
    global meeting_texts
    def process_translation_dubbing(i):
        if i != speaker_idx:
            participant_language = language_codes[language_list[i]]
            speaker_language = language_codes[language_list[speaker_idx]]
            translated_text = translate_text(transcript, speaker_language, participant_language)
            dubbed_video = synthesize_speech(video, speaker_language, participant_language)
            return translated_text, dubbed_video
        return None, None

    with concurrent.futures.ThreadPoolExecutor() as executor:
        futures = [executor.submit(process_translation_dubbing, i) for i in range(n_participants)]
        results = [f.result() for f in futures]

    for i, (translated_text, dubbed_video) in enumerate(results):
        if i == speaker_idx:
            outputs.insert(0, transcript)
        else:
            outputs.append(translated_text)
            outputs.append(dubbed_video)
    if speaker_idx == 0:
        meeting_texts.append({f"Speaker_{speaker_idx+1}":outputs[0]})
    else:
        meeting_texts.append({f"Speaker_{speaker_idx+1}":outputs[1]})

    print(len(outputs))
    print(outputs)
    print('meeting_texts: ',meeting_texts)
    return outputs

def create_participant_row(i, language_choices):
    """Creates the UI for a single participant."""
    with gr.Row():
        video_input = gr.Video(label=f"Participant {i+1} Video", interactive=True)
        language_dropdown = gr.Dropdown(choices=language_choices, label=f"Participant {i+1} Language", value=language_choices[i])
        transcript_output = gr.Textbox(label=f"Participant {i+1} Transcript")
        translated_text = gr.Textbox(label="Speaker's Translated Text")
        dubbed_video = gr.Video(label="Speaker's Dubbed Video")
        return video_input, language_dropdown, transcript_output, translated_text, dubbed_video

# Main dynamic Gradio interface
def create_gradio_interface(n_participants, language_choices):
    with gr.Blocks() as demo:
        gr.Markdown("# LinguaPolis: Bridging Languages, Uniting Teams Globally - Multilingual Conference Call Simulation")

        video_inputs = []
        language_dropdowns = []
        transcript_outputs = []
        translated_texts = []
        dubbed_videos = []

        # Create a row for each participant
        for i in range(n_participants):
            video_input, language_dropdown, transcript_output, translated_text, dubbed_video = create_participant_row(i, language_choices)
            video_inputs.append(video_input)
            language_dropdowns.append(language_dropdown)
            transcript_outputs.append(transcript_output)
            translated_texts.append(translated_text)
            dubbed_videos.append(dubbed_video)

        # Create dynamic processing buttons for each participant
        for i in range(n_participants):
            gr.Button(f"Submit Speaker {i+1}'s Speech").click(
                process_speaker,
                [video_inputs[i], gr.State(i), gr.State(n_participants)] + [language_dropdowns[j] for j in range(n_participants)],
                [transcript_outputs[i]] + [k for j in zip(translated_texts[:i]+translated_texts[i+1:], dubbed_videos[:i]+dubbed_videos[i+1:]) for k in j]
           )
        minutes = gr.Textbox(label="Minutes of Meeting")
        gr.Button(f"Generate Minutes of meeting").click(summarize, None, minutes)

    # Launch with .queue() to keep it running properly in Jupyter
    demo.queue().launch(debug=True, share=True)


create_gradio_interface(n_participants, language_choices)