PriyankaSatish commited on
Commit
edaa368
1 Parent(s): 5334352

Upload 5 files

Browse files
IntelliStreamLogo.png ADDED
Wiprologo.jpg ADDED
fullapp-demo-2.py ADDED
@@ -0,0 +1,516 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import subprocess
2
+ import streamlink
3
+ import streamlit as st
4
+ import tempfile
5
+ import base64
6
+ import os
7
+ from dotenv import load_dotenv
8
+ from PIL import Image
9
+ from io import BytesIO
10
+ from openai import OpenAI
11
+ import whisper
12
+ from google.cloud import vision
13
+
14
+ # st.set_page_config(layout="wide")
15
+
16
+ load_dotenv()
17
+ OpenAI.api_key = os.getenv("OPENAI_API_KEY")
18
+ if not OpenAI.api_key:
19
+ raise ValueError("The OpenAI API key must be set in the OPENAI_API_KEY environment variable.")
20
+
21
+ whisper.api_key = os.getenv("WHISPER_API_KEY")
22
+ if not whisper.api_key:
23
+ raise ValueError("The WHsiper API Key needs to be set in the env")
24
+ client = OpenAI()
25
+
26
+ # Set Google Cloud credentials in environment
27
+ service_account_path = os.getenv("GOOGLE_APPLICATION_CREDENTIALS")
28
+ os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = 'trueinfolabs-ocr-20c8c095084b.json'
29
+
30
+ # Initialize Google Vision client
31
+ vision_client = vision.ImageAnnotatorClient()
32
+
33
+ wipro_logo_path = "Wiprologo.jpg" # Update this path to where your logo is stored
34
+ wipro_logo = Image.open(wipro_logo_path)
35
+ # Create a layout with columns
36
+ col1, col2 = st.columns([8, 2]) # Adjust the ratio as needed
37
+
38
+ # Display the "Insightly Video" text in the first column (larger space)
39
+
40
+ # Display the logo in the second column (right side, smaller space)
41
+ with col2:
42
+ st.image(wipro_logo, width=200) # Adjust the width as needed
43
+
44
+ # Function to execute FFmpeg command and capture output
45
+ def execute_ffmpeg_command(command):
46
+ try:
47
+ result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
48
+ if result.returncode == 0:
49
+ print("FFmpeg command executed successfully.")
50
+ return result.stdout, result.stderr
51
+ else:
52
+ print("Error executing FFmpeg command:")
53
+ return None, result.stderr
54
+ except Exception as e:
55
+ print("An error occurred during FFmpeg execution:")
56
+ return None, str(e)
57
+
58
+
59
+ # Function to get transcript from audio using OpenAI Whisper
60
+ def get_transcript_from_audio(audio_file_path):
61
+ try:
62
+ # Load the model
63
+ model = whisper.load_model("base") # You can choose another model size if needed
64
+
65
+ # Process the audio file and get the result
66
+ result = model.transcribe(audio_file_path)
67
+
68
+ # Get the transcript text
69
+ transcript_text = result["text"]
70
+ return transcript_text
71
+ except Exception as e:
72
+ print(f"Error submitting transcription job: {e}")
73
+ return None
74
+
75
+ def extract_text_from_base64_frame(base64_frame):
76
+ """Extracts text from a single base64 encoded frame using Google Cloud Vision API."""
77
+ frame_bytes = base64.b64decode(base64_frame) # Decode the base64 string to bytes
78
+ image = vision.Image(content=frame_bytes)
79
+ response = vision_client.text_detection(image=image)
80
+ texts = response.text_annotations
81
+ return texts[0].description.strip() if texts else "No text found."
82
+
83
+ def transcribe_uploaded_mp3(uploaded_mp3):
84
+ try:
85
+ # Save the uploaded MP3 file to a temporary file
86
+ with tempfile.NamedTemporaryFile(delete=False, suffix=".mp3") as tmpfile:
87
+ tmpfile.write(uploaded_mp3.getvalue())
88
+ audio_file_path = tmpfile.name
89
+
90
+ # You might want to process/convert the MP3 file with FFmpeg here if needed
91
+ # For example, to ensure it's in the correct format for Whisper or to extract a specific part
92
+ # This is optional and depends on your requirements
93
+
94
+ # Transcribe the audio file using Whisper
95
+ transcript = get_transcript_from_audio(audio_file_path)
96
+
97
+ if transcript is None:
98
+ return "Transcription failed or no transcript available."
99
+
100
+ return transcript
101
+ except Exception as e:
102
+ return f"Failed to transcribe audio file. Error: {e}"
103
+
104
+ def analyze_image_with_google_vision_api(base64_frame):
105
+ """Analyze image content using Google Cloud Vision API."""
106
+ frame_bytes = base64.b64decode(base64_frame) # Decode the base64 string to bytes
107
+ image = vision.Image(content=frame_bytes)
108
+
109
+ response = vision_client.label_detection(image=image)
110
+ labels = response.label_annotations
111
+
112
+ if labels:
113
+ return ', '.join([label.description for label in labels])
114
+ else:
115
+ return "No labels detected."
116
+
117
+ def analyze_content_with_openai(text, description,labels):
118
+ """Analyze the combined text and image labels to categorize the image using OpenAI."""
119
+ try:
120
+ response = client.chat.completions.create(
121
+ model="gpt-4-vision-preview",
122
+ messages=[
123
+ {"role": "system", "content": "Classify the following image into one or more of these categories based on the extracted text, description of the frames and image labels. Take valuable information from every frame even if available in only one out of many frames. \
124
+ Dont check on the authenticity of the content .Doesn't matter is something from the frame is fake/joke/etc. We dont need context for the categorisation.\
125
+ Categories : Bullying, Nudity & Adult Content, Graphic Violence, Illegal Goods, Child Safety, Sexual Abuse, Profanity, Self Harm/Suicide, Violent Extremism and None. Return the following: Give out the result as Category - {whatever the category(s) is/are} and then GIVE A PROPER JUSTIFICATION of the image categorization for that conclusion WITHOUT any assumption"},
126
+ {"role": "user", "content": f"Text: {text}\nDescription: {description}\n Labels: {labels}"}
127
+ ],
128
+ max_tokens=4096,
129
+ n=1
130
+ )
131
+ if response.choices:
132
+ result_message = response.choices[0].message.content
133
+ return result_message.strip()
134
+ else:
135
+ return "Analysis failed or was inconclusive."
136
+ except Exception as e:
137
+ return f"Failed to analyze content with OpenAI. Error: {e}"
138
+
139
+ def display_categories(analysis_result, categories):
140
+ """Display categories with highlight based on analysis result."""
141
+ # Extracting the 'xyz' from the analysis_result
142
+ try:
143
+ extracted_text = analysis_result.split('Category - ')[1].split('\n')[0].strip()
144
+ category_keywords = [x.strip() for x in extracted_text.split(',')]
145
+ except IndexError:
146
+ # Default to None if parsing fails
147
+ category_keyword = 'None'
148
+
149
+ num_cols = 3
150
+ rows = [categories[i:i + num_cols] for i in range(0, len(categories), num_cols)]
151
+
152
+ matched_style = """
153
+ border: 2px solid #00FF00;
154
+ padding: 10px;
155
+ border-radius: 10px;
156
+ text-align: center;
157
+ background-color: #333333;
158
+ color: #FFFFFF;
159
+ margin: 5px;
160
+ box-shadow: 0 2px 4px 0 rgba(255,255,255,0.2);
161
+ """
162
+ unmatched_style = """
163
+ border: 1px solid #555555;
164
+ padding: 10px;
165
+ border-radius: 10px;
166
+ text-align: center;
167
+ background-color: #222222;
168
+ color: #AAAAAA;
169
+ margin: 5px;
170
+ """
171
+
172
+ # Display categories in a grid layout
173
+ for row in rows:
174
+ cols = st.columns(num_cols)
175
+ for idx, category in enumerate(row):
176
+ with cols[idx]:
177
+ # Check if the category matches any in the list of extracted categories
178
+ if category.lower() in [k.lower() for k in category_keywords]:
179
+ # Highlight matched category
180
+ st.markdown(f"<div style='{matched_style}'><h4 style='margin:0;'>{category}</h4></div>", unsafe_allow_html=True)
181
+ else:
182
+ # Display non-matched category
183
+ st.markdown(f"<div style='{unmatched_style}'><h4 style='margin:0;'>{category}</h4></div>", unsafe_allow_html=True)
184
+
185
+ def execute_fmpeg_command(command):
186
+ try:
187
+ result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True)
188
+ return result.stdout # Return just the stdout part, not a tuple
189
+ except subprocess.CalledProcessError as e:
190
+ print(f"FFmpeg command failed with error: {e.stderr.decode()}")
191
+ return None
192
+
193
+ def search_keyword(keyword, frame_texts):
194
+ return [index for index, text in st.session_state.frame_texts.items() if keyword.lower() in text.lower()]
195
+
196
+
197
+
198
+ # Function to generate description for video frames
199
+ def generate_description(base64_frames):
200
+ try:
201
+ prompt_messages = [
202
+ {
203
+ "role": "user",
204
+ "content": [
205
+ "1. Generate a description for this sequence of video frames in about 90 words. 2.Return the following: i. List of objects in the video ii. Any restrictive content or sensitive content and if so which frame. iii. The frames is supposed to contain news content and we want to detect non-news content such as an advertisement. So analyze specifically for any indications that the content might be promotional or an advertisement.",
206
+ *map(lambda x: {"image": x, "resize": 428}, base64_frames),
207
+ ],
208
+ },
209
+ ]
210
+ response = client.chat.completions.create(
211
+ model="gpt-4-vision-preview",
212
+ messages=prompt_messages,
213
+ max_tokens=3000,
214
+ )
215
+ return response.choices[0].message.content
216
+ except Exception as e:
217
+ print(f"Error in generate_description: {e}")
218
+ return None
219
+
220
+ def generate_overall_description(transcript_text, video_description):
221
+ try:
222
+ combined_input = f"Transcript: {transcript_text}\n\nVideo Description: {video_description}\n\n"
223
+ prompt_message = "Based on the above transcript and video description, generate a very detailed description about the sequence of events in the video and from the transcript within 500 words."
224
+
225
+ prompt_messages = [
226
+ {"role": "user", "content": combined_input + prompt_message}
227
+ ]
228
+
229
+ response = client.chat.completions.create(
230
+ model="gpt-4",
231
+ messages=prompt_messages,
232
+ max_tokens=1000, # Increased from 300 to allow for a more detailed response
233
+ )
234
+
235
+ return response.choices[0].message.content.strip()
236
+ except Exception as e:
237
+ print(f"Error in generate_overall_description: {e}")
238
+ return None
239
+
240
+ with col1:
241
+ is_logo_path = "IntelliStreamLogo.png" # Update this path to where your logo is stored
242
+ is_logo = Image.open(is_logo_path)
243
+ st.image(is_logo, width=200)
244
+
245
+ st.markdown("<h1 style='text-align: left; color: white;'></h1>", unsafe_allow_html=True)
246
+
247
+ # Streamlit UI
248
+
249
+ st.title("Insightly Video")
250
+ stream_url = st.text_input("Enter the live stream URL (YouTube, Twitch, etc.):")
251
+ keyword = st.text_input("Enter a keyword to filter the frames (optional):")
252
+ extract_frames_button = st.button("Extract Frames")
253
+ uploaded_video = st.file_uploader("Or upload a video file (MP4):", type=["mp4"])
254
+
255
+
256
+ # Slider to select the number of seconds for extraction
257
+ seconds = st.slider("Select the number of seconds for extraction:", min_value=1, max_value=60, value=10)
258
+
259
+ uploaded_mp3 = st.file_uploader("Upload an MP3 file for transcription:", type=["mp3"])
260
+
261
+
262
+ # Check if an MP3 file has been uploaded
263
+ if uploaded_mp3 is not None:
264
+ # Call the transcription function with the uploaded MP3 file
265
+ transcript = transcribe_uploaded_mp3(uploaded_mp3)
266
+
267
+ # Display the transcript
268
+ st.text_area("Transcript:", value=transcript, height=300)
269
+ else:
270
+ st.write("Please upload an MP3 file to get started.")
271
+
272
+ if (extract_frames_button and stream_url and keyword) or (extract_frames_button and stream_url):
273
+ # Execute FFmpeg command to extract frames
274
+
275
+ # Check if URL is provided
276
+
277
+ streams = streamlink.streams(stream_url)
278
+ if "best" in streams:
279
+ stream_url = streams["best"].url
280
+
281
+ ffmpeg_command = [
282
+ 'ffmpeg', # Input stream URL
283
+ '-t', str(seconds), # Duration to process the input (selected seconds)
284
+ '-vf', 'fps=1', # Extract one frame per second
285
+ '-f', 'image2pipe', # Output format as image2pipe
286
+ '-c:v', 'mjpeg', # Codec for output video
287
+ '-an', # No audio
288
+ '-'
289
+ ]
290
+
291
+ # Determine the input source for FFmpeg
292
+ input_source = stream_url # Default to stream URL
293
+
294
+ # Insert the input source into the FFmpeg command
295
+ ffmpeg_command.insert(1, input_source)
296
+ ffmpeg_command.insert(1, '-i')
297
+
298
+ # Execute FFmpeg command
299
+ ffmpeg_output, _ = execute_ffmpeg_command(ffmpeg_command)
300
+
301
+ # Modify the section where you display frames to include text extraction and display
302
+ # Modify the section where base64 encoded frames are processed
303
+ # After successfully executing the FFmpeg command to capture frames
304
+ if ffmpeg_output:
305
+ st.write("Frames Extracted:")
306
+ frame_bytes_list = ffmpeg_output.split(b'\xff\xd8')[1:] # Correct splitting for JPEG frames
307
+ n_frames = len(frame_bytes_list)
308
+ base64_frames = [base64.b64encode(b'\xff\xd8' + frame_bytes).decode('utf-8') for frame_bytes in frame_bytes_list]
309
+
310
+ categories_results = []
311
+ frame_texts = {}
312
+
313
+ for idx, frame_base64 in enumerate(base64_frames):
314
+ extracted_text = extract_text_from_base64_frame(frame_base64)
315
+ frame_texts[idx] = extracted_text
316
+
317
+ if not keyword or keyword.lower() in extracted_text.lower():
318
+ col1, col2 = st.columns([3, 2])
319
+ with col1:
320
+ frame_bytes = base64.b64decode(frame_base64)
321
+ st.image(Image.open(BytesIO(frame_bytes)), caption=f'Frame {idx + 1}', use_column_width=True)
322
+ with col2:
323
+ st.write(f"Extracted Text: {extracted_text}")
324
+ if keyword:
325
+ st.write(f"Displaying frames containing the keyword '{keyword}'.")
326
+ else:
327
+ st.write("Displaying all extracted frames.")
328
+ # Use Streamlit columns for side-by-side display (1 column for image, 1 for text)
329
+ # col1, col2 = st.columns([3, 2])
330
+ # with col1:
331
+ # frame_bytes = base64.b64decode(frame_base64)
332
+ # st.image(Image.open(BytesIO(frame_bytes)), caption=f'Frame {idx + 1}', use_column_width=True)
333
+ # with col2:
334
+ # st.write(f"Extracted Text: {extracted_text}")
335
+
336
+ # if 'base64_frames' not in st.session_state:
337
+ # st.session_state.base64_frames = [] # Populate this when frames are first extracted
338
+ # if 'frame_texts' not in st.session_state:
339
+ # st.session_state.frame_texts = {}
340
+
341
+ st.write("Analysis Results for All Frames:")
342
+ # Assuming 'categories' is defined with all possible categories you're interested in
343
+ categories = ["Bullying", "Nudity & Adult Content", "Graphic Violence", "Illegal Goods", "Child Safety", "Sexual Abuse", "Profanity", "Self Harm/Suicide", "Violent Extremism","None"]
344
+ # Here, you might want to process combined_analysis_results to summarize or just display them
345
+
346
+ # display_categories(" ".join(categories_results), categories)
347
+
348
+ # Extract audio
349
+ audio_command = [
350
+ 'ffmpeg',
351
+ '-i', stream_url, # Input stream URL
352
+ '-vn', # Ignore the video for the audio output
353
+ '-acodec', 'libmp3lame', # Set the audio codec to MP3
354
+ '-t', str(seconds), # Duration for the audio extraction (selected seconds)
355
+ '-f', 'mp3', # Output format as MP3
356
+ '-'
357
+ ]
358
+ audio_output, _ = execute_ffmpeg_command(audio_command)
359
+
360
+ st.write("Extracted Audio:")
361
+ audio_tempfile = tempfile.NamedTemporaryFile(delete=False, suffix=".mp3")
362
+ audio_tempfile.write(audio_output)
363
+ audio_tempfile.close()
364
+
365
+ st.audio(audio_output, format='audio/mpeg', start_time=0)
366
+
367
+ # Get the transcript from whisper
368
+ transcript_text = get_transcript_from_audio(audio_tempfile.name)
369
+ if transcript_text:
370
+ st.markdown("**Transcript:**")
371
+ st.write(transcript_text)
372
+ else:
373
+ st.write("Failed to retrieve transcript.")
374
+
375
+
376
+ # Get consolidated description for all frames
377
+ if ffmpeg_output:
378
+ description = generate_description(base64_frames)
379
+ if description:
380
+ st.markdown("**Frame Description:**")
381
+ st.write(description)
382
+ else:
383
+ st.write("Failed to generate description.")
384
+
385
+ image_labels = analyze_image_with_google_vision_api(frame_base64)
386
+ # st.write(image_labels)
387
+ analysis_result = analyze_content_with_openai(extracted_text, description, image_labels)
388
+ st.write(analysis_result)
389
+ display_categories(analysis_result, categories)
390
+ categories_results.append(analysis_result) # Collect results for summary
391
+
392
+ # Get the transcript from whisper
393
+ transcript_text = get_transcript_from_audio(audio_tempfile.name)
394
+ description = generate_description(base64_frames)
395
+ # Generate overall description using transcript and video description
396
+ overall_description = generate_overall_description(transcript_text, description)
397
+ if overall_description:
398
+ st.markdown("**Consolidated Description:**")
399
+ st.write(overall_description)
400
+ else:
401
+ st.write("Failed to generate overall description.")
402
+
403
+ if keyword:
404
+ st.write(f"Displaying frames containing the keyword '{keyword}'.")
405
+ else:
406
+ st.write("Displaying all extracted frames.")
407
+
408
+ elif uploaded_video is not None and extract_frames_button:
409
+ with tempfile.NamedTemporaryFile(delete=False, suffix=".mp4") as tmpfile:
410
+ tmpfile.write(uploaded_video.getvalue())
411
+ video_file_path = tmpfile.name
412
+
413
+ ffmpeg_command = [
414
+ 'ffmpeg', # Input stream URL
415
+ '-i', video_file_path,
416
+ '-t', str(seconds), # Duration to process the input (selected seconds)
417
+ '-vf', 'fps=1', # Extract one frame per second
418
+ '-f', 'image2pipe', # Output format as image2pipe
419
+ '-c:v', 'mjpeg', # Codec for output video
420
+ '-an', # No audio
421
+ '-'
422
+ ]
423
+
424
+ ffmpeg_output = execute_fmpeg_command(ffmpeg_command)
425
+
426
+ if ffmpeg_output:
427
+ st.write("Frames Extracted:")
428
+ frame_bytes_list = ffmpeg_output.split(b'\xff\xd8')[1:] # Correct splitting for JPEG frames
429
+ n_frames = len(frame_bytes_list)
430
+ base64_frames = [base64.b64encode(b'\xff\xd8' + frame_bytes).decode('utf-8') for frame_bytes in frame_bytes_list]
431
+
432
+ categories_results = []
433
+ frame_texts = {}
434
+
435
+ for idx, frame_base64 in enumerate(base64_frames):
436
+ extracted_text = extract_text_from_base64_frame(frame_base64)
437
+ frame_texts[idx] = extracted_text
438
+ # Use Streamlit columns for side-by-side display (1 column for image, 1 for text)
439
+ col1, col2 = st.columns([3, 2])
440
+ with col1:
441
+ frame_bytes = base64.b64decode(frame_base64)
442
+ st.image(Image.open(BytesIO(frame_bytes)), caption=f'Frame {idx + 1}', use_column_width=True)
443
+ with col2:
444
+ st.write(f"Extracted Text: {extracted_text}")
445
+
446
+
447
+ st.write("Analysis Results for All Frames:")
448
+ # Assuming 'categories' is defined with all possible categories you're interested in
449
+ categories = ["Bullying", "Nudity & Adult Content", "Graphic Violence", "Illegal Goods", "Child Safety", "Sexual Abuse", "Profanity", "Self Harm/Suicide", "Violent Extremism", "None"]
450
+ # Here, you might want to process combined_analysis_results to summarize or just display them
451
+
452
+
453
+
454
+
455
+ # Extract audio
456
+ audio_command = [
457
+ 'ffmpeg',
458
+ '-i', video_file_path,
459
+ '-t', str(seconds),
460
+ '-vf', 'fps=1', # Input stream URL
461
+ '-vn', # Ignore the video for the audio output
462
+ '-acodec', 'libmp3lame', # Set the audio codec to MP3 # Duration for the audio extraction (selected seconds)
463
+ '-f', 'mp3', # Output format as MP3
464
+ '-'
465
+ ]
466
+ audio_output, _ = execute_ffmpeg_command(audio_command)
467
+
468
+ st.write("Extracted Audio:")
469
+ audio_tempfile = tempfile.NamedTemporaryFile(delete=False, suffix=".mp3")
470
+ audio_tempfile.write(audio_output)
471
+ audio_tempfile.close()
472
+
473
+ st.audio(audio_output, format='audio/mpeg', start_time=0)
474
+
475
+ # Get the transcript from whisper
476
+ transcript_text = get_transcript_from_audio(audio_tempfile.name)
477
+ if transcript_text:
478
+ st.markdown("**Transcript:**")
479
+ st.write(transcript_text)
480
+ else:
481
+ st.write("Failed to retrieve transcript.")
482
+
483
+ # Get consolidated description for all frames
484
+ if ffmpeg_output:
485
+ description = generate_description(base64_frames)
486
+ if description:
487
+ st.markdown("**Frame Description:**")
488
+ st.write(description)
489
+ else:
490
+ st.write("Failed to generate description.")
491
+
492
+ image_labels = analyze_image_with_google_vision_api(frame_base64)
493
+ # st.write(image_labels)
494
+ analysis_result = analyze_content_with_openai(extracted_text, description, image_labels)
495
+ st.write(analysis_result)
496
+ display_categories(analysis_result, categories)
497
+ categories_results.append(analysis_result) # Collect results for summary
498
+
499
+ # if st.button("Overall Description"):
500
+ # audio_tempfile = tempfile.NamedTemporaryFile(delete=False, suffix=".mp3")
501
+ # audio_tempfile.write(audio_output)
502
+ # audio_tempfile.close()
503
+
504
+ # Get the transcript from whisper
505
+ transcript_text = get_transcript_from_audio(audio_tempfile.name)
506
+ description = generate_description(base64_frames)
507
+ # Generate overall description using transcript and video description
508
+ overall_description = generate_overall_description(transcript_text, description)
509
+ if overall_description:
510
+ st.markdown("**Consolidated Description:**")
511
+ st.write(overall_description)
512
+ else:
513
+ st.write("Failed to generate overall description.")
514
+
515
+ else:
516
+ st.write(" ")
requirements.txt.txt ADDED
@@ -0,0 +1,130 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ aiohttp==3.9.3
2
+ aiosignal==1.3.1
3
+ altair==5.2.0
4
+ annotated-types==0.6.0
5
+ anyio==4.2.0
6
+ assemblyai==0.20.2
7
+ async-timeout==4.0.3
8
+ attrs==23.2.0
9
+ blinker==1.7.0
10
+ cachetools==5.3.2
11
+ certifi==2023.11.17
12
+ cffi==1.16.0
13
+ charset-normalizer==3.3.2
14
+ click==8.1.7
15
+ colorama==0.4.6
16
+ distro==1.9.0
17
+ exceptiongroup==1.2.0
18
+ fastapi==0.109.2
19
+ ffmpeg-python==0.2.0
20
+ filelock==3.13.1
21
+ Flask==3.0.2
22
+ frozenlist==1.4.1
23
+ fsspec==2024.2.0
24
+ future==0.18.3
25
+ gitdb==4.0.11
26
+ GitPython==3.1.41
27
+ google-api-core==2.16.2
28
+ google-api-python-client==2.116.0
29
+ google-auth==2.27.0
30
+ google-auth-httplib2==0.2.0
31
+ google-cloud-vision==3.7.1
32
+ googleapis-common-protos==1.62.0
33
+ grpcio==1.62.0
34
+ grpcio-status==1.62.0
35
+ gunicorn==21.2.0
36
+ h11==0.14.0
37
+ httpcore==1.0.2
38
+ httplib2==0.22.0
39
+ httptools==0.6.1
40
+ httpx==0.26.0
41
+ idna==3.6
42
+ importlib-metadata==7.0.1
43
+ isodate==0.6.1
44
+ itsdangerous==2.1.2
45
+ Jinja2==3.1.3
46
+ jsonschema==4.21.1
47
+ jsonschema-specifications==2023.12.1
48
+ llvmlite==0.42.0
49
+ lxml==5.1.0
50
+ markdown-it-py==3.0.0
51
+ MarkupSafe==2.1.4
52
+ mdurl==0.1.2
53
+ more-itertools==10.2.0
54
+ mpmath==1.3.0
55
+ multidict==6.0.5
56
+ networkx==3.2.1
57
+ numba==0.59.0
58
+ numpy==1.26.3
59
+ openai==1.13.3
60
+ openai-whisper==20231117
61
+ opencv-python==4.9.0.80
62
+ opencv-python-headless==4.9.0.80
63
+ outcome==1.3.0.post0
64
+ packaging==23.2
65
+ pandas==2.2.0
66
+ pillow==10.2.0
67
+ proto-plus==1.23.0
68
+ protobuf==4.25.2
69
+ pyarrow==15.0.0
70
+ pyasn1==0.5.1
71
+ pyasn1-modules==0.3.0
72
+ pycountry==23.12.11
73
+ pycparser==2.21
74
+ pycryptodome==3.20.0
75
+ pydantic==2.6.0
76
+ pydantic_core==2.16.1
77
+ pydeck==0.8.1b0
78
+ pydub==0.25.1
79
+ Pygments==2.17.2
80
+ PyMuPDF==1.23.25
81
+ PyMuPDFb==1.23.22
82
+ pyparsing==3.1.1
83
+ PySocks==1.7.1
84
+ pytesseract==0.3.10
85
+ python-dateutil==2.8.2
86
+ python-dotenv==1.0.1
87
+ python-multipart==0.0.9
88
+ pytz==2023.4
89
+ PyYAML==6.0.1
90
+ referencing==0.33.0
91
+ regex==2023.12.25
92
+ requests==2.31.0
93
+ rich==13.7.0
94
+ rpds-py==0.17.1
95
+ rsa==4.9
96
+ six==1.16.0
97
+ smmap==5.0.1
98
+ sniffio==1.3.0
99
+ sortedcontainers==2.4.0
100
+ SpeechRecognition==3.10.1
101
+ starlette==0.36.3
102
+ streamlink==6.5.1
103
+ streamlit==1.31.0
104
+ sympy==1.12
105
+ tenacity==8.2.3
106
+ tiktoken==0.6.0
107
+ toml==0.10.2
108
+ toolz==0.12.1
109
+ torch==2.2.1
110
+ tornado==6.4
111
+ tqdm==4.66.1
112
+ trio==0.24.0
113
+ trio-websocket==0.11.1
114
+ typing_extensions==4.9.0
115
+ tzdata==2023.4
116
+ tzlocal==5.2
117
+ uritemplate==4.1.1
118
+ urllib3==2.2.0
119
+ uvicorn==0.27.1
120
+ uvloop==0.19.0
121
+ validators==0.22.0
122
+ watchdog==3.0.0
123
+ watchfiles==0.21.0
124
+ websocket-client==1.7.0
125
+ websockets==12.0
126
+ Werkzeug==3.0.1
127
+ whisper==1.1.10
128
+ wsproto==1.2.0
129
+ yarl==1.9.4
130
+ zipp==3.17.0
trueinfolabs-ocr-20c8c095084b.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "type": "service_account",
3
+ "project_id": "trueinfolabs-ocr",
4
+ "private_key_id": "20c8c095084ba31119123d9a801146cfeccbfd75",
5
+ "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCjiK8mipiG6t78\ndFwIGwbouIQ3X94yLaD8bmBqefbjRqgf/NpHuMMX/lZnIo+oAoFQljZLTXQHOG1v\nGdVvHUE5Dar9N8F2Ecrz0AcP8qLG86py2mrds7Q2Ss/vavCimV/VBX8uU0G2/slS\nYrrNIJqaeAXxxYyEzWCwZTY+p+4XLsBp9+aC3dDiDBJdwpNWniYvAmrgtdEYfVOC\n32C+W13qHB9paJUR10bCFU19vpM9JA1MqCeN1s/Ctlbt+/m0KIiXHTV0Ev8c+kov\nbBObLpBEZ2+7ZnX3Cmj/Df/sDaAGqM8884pdFgglOvcQcrnwFP9H7jlDt/XvAxC/\n3lPJLd2zAgMBAAECggEAK+aiKoUTbEi6Iuiz2iMzSB9csybj1fGk4Y9aB8uFKT9L\nHAc7q/xgN106YcaH7TD6+SDSI4X/635M9oHnZ8RhQYk3SXIB2Anvw2MpujDHXQSF\n0f5rqOe4rciIqLu7LNNixCONIGkOX6tnbVv+zNl/V493/Q8s6WvfFIufM9POGV+v\n9mhAgm+5WiA7my6BtX2xoJggsAkCTZIM05DGm9DgXuhcuT0tmBLRgvt1MffiJkD8\nMTQWw0Obc/O0hBk2aK+xSntxnt0Y5ew7u/+0GztmJhJddroV8ev1YnWyAxEZEp7Q\nXeAzZtJ1IpqzLGEW/E2XxcFLwi8rV4E1i1tsrkOAvQKBgQDcIH0Lsy3WY86GBNY2\npn/zkT2itT7/QK8ldw4wg/vvfFOQZVJyrVYgBDBDq9GT88I0ivAGnZit0c8dg0Ph\nHTmjZrzOW/R4Zw9v9fUGkjWH1NywNyw4xgYmllH/pgt73pn0YJhRSPEil6QhRLSz\nXJpGACXFC4feGQn1giQQn79ynwKBgQC+Ly71dVhe+7uZQzCi2iKIOqqjIrOhRWs+\nNd/0d2gMzZ57p5S8e3H1gw8Q2z6TINsv1Vn8PHKBUEY6I9a1/MQDqcNxAZHPAZCz\nU6SUNLZmL+dbOT00dB6DDWeqNXKrBITvYcd4RclMwt6Uy9GIdVYRsYXcpqxg7exJ\nUpgLp6vwbQKBgBPq9KWcXudpPISv51omkqlNWRBh6gNarP5s2WHWb3NqAn2sTVuH\nB6X5+wwEfgAvLEFo2PMgS3Je66i1+exiopaAc1EQzPwgByuQS81+aU2TGyzusReq\nA2B2dZD2S0+4AqI0I1Qnj3ob0oQYHWmlVWhEFybPNkoIZfhLAExox//tAoGAHqLB\nIL8nXh2U6apeIk2bNHCm3iKP5xGkpd/N1LaLZ0yUE/05w2brQ446FhALM24egMUQ\nesZN97Czr1folWZDOQfWuNR05XCuG4UExO781tcuoI5G4rU12QyGv29eqlfPBjSt\nCh3LwHa/nh57AU1NcQo0sweSSwbogiMw5Oe8lb0CgYEAtOZkEYPGHxdhyiPHw3zu\npzLDIBNCP95nFsRC0cVyfQcW2MJ1xTNnB5jC1e/wJBbercQFMsdcmfn8egMpu84V\ndYgH/4cgiOwGsJvv+eT2ngJLcEUM8S2mJ5cAG8z0grf9ChejWMUzbd2frkg2diMJ\nPLlUjwFBd4hY7f2S9xWsvVw=\n-----END PRIVATE KEY-----\n",
6
+ "client_email": "tamil-ocr@trueinfolabs-ocr.iam.gserviceaccount.com",
7
+ "client_id": "102907087999482970510",
8
+ "auth_uri": "https://accounts.google.com/o/oauth2/auth",
9
+ "token_uri": "https://oauth2.googleapis.com/token",
10
+ "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
11
+ "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/tamil-ocr%40trueinfolabs-ocr.iam.gserviceaccount.com",
12
+ "universe_domain": "googleapis.com"
13
+ }