import cv2 import streamlit as st import tempfile import base64 import os from dotenv import load_dotenv from openai import OpenAI import assemblyai as aai from moviepy.editor import * # Load environment variables load_dotenv() aai.settings.api_key = os.getenv("ASSEMBLYAI_API_KEY") OpenAI.api_key = os.getenv("OPENAI_API_KEY") client = OpenAI() def main(): st.title('Insightly Video Content Moderation') # Video upload section uploaded_video = st.file_uploader('Upload a video', type=["mp4", "avi", "mov"]) if uploaded_video is not None: # Save the video to a temp file tfile = tempfile.NamedTemporaryFile(delete=False) tfile.write(uploaded_video.read()) video_file_path = tfile.name tfile.close() transcriber = aai.Transcriber() transcript = transcriber.transcribe(tfile.name) # Process the video and display frames in a grid layout base64_frames = video_to_base64_frames(video_file_path) display_frame_grid(base64_frames[::30]) # Display every 30th frame in a 3-column grid st.write("Actions:") # Header for the actions/buttons section spacer_col1, col1, spacer_col2, col2, spacer_col3 = st.columns([1, 2, 1, 2, 1]) with col1: if st.button("Description"): st.session_state['description'] = generate_description(base64_frames) if 'description' not in st.session_state else st.session_state['description'] with col2: if st.button("Generate Transcript"): transcript = transcriber.transcribe(video_file_path) st.session_state['transcript'] = transcript.text if 'transcript' not in st.session_state else st.session_state['transcript'] # If any value exists in session state then display it if 'description' in st.session_state and st.session_state['description']: st.subheader("Video Description") st.write(st.session_state['description']) if 'transcript' in st.session_state and st.session_state['transcript']: st.subheader("Video Transcript") st.write(st.session_state['transcript']) def video_to_base64_frames(video_file_path): # Logic to extract all frames from the video and convert them to base64 video = cv2.VideoCapture(video_file_path) base64_frames = [] while video.isOpened(): success, frame = video.read() if not success: break _, buffer = cv2.imencode('.jpg', frame) base64_frame = base64.b64encode(buffer).decode('utf-8') base64_frames.append(base64_frame) video.release() return base64_frames ######################################### #Generate Video description def generate_description(base64_frames): prompt_messages = [ { "role": "user", "content": [ "1. Generate a description for this sequence of video frames in about 100 words.\ Return the following : 2. Frame by frame summary of what's happening in the video. 3. List of objects in the video. 4. Any restrictive content or sensitive content and if so which frame 5. What category can this video be tagged to?", *map(lambda x: {"image": x, "resize": 428}, base64_frames[0::30]), ], }, ] response = client.chat.completions.create( model="gpt-4-vision-preview", messages=prompt_messages, max_tokens=3000, ) return response.choices[0].message.content ######################## def display_frame_grid(base64_frames): cols_per_row = 3 n_frames = len(base64_frames) for idx in range(0, n_frames, cols_per_row): cols = st.columns(cols_per_row) for col_index in range(cols_per_row): frame_idx = idx + col_index if frame_idx < n_frames: with cols[col_index]: frame = base64_frames[frame_idx] st.image(base64.b64decode(frame), caption=f'Frame {frame_idx * 30 + 1}', width=200) if __name__ == '__main__': main()