app.py

#THIS IS THE FINAL WORKING VIDEO - TEGNA FINAL TIKTOK.
import subprocess
import streamlit as st
import tempfile
import base64
import os
from dotenv import load_dotenv
from PIL import Image
from io import BytesIO  
from openai import OpenAI
import re

load_dotenv()
OpenAI.api_key = os.getenv("OPENAI_API_KEY")
if not OpenAI.api_key:     
    raise ValueError("The OpenAI API key must be set in the OPENAI_API_KEY environment variable.")
client = OpenAI()


# Create a layout with columns
col1, col2 = st.columns([8, 2])  # Adjust the ratio as needed


def execute_ffmpeg_command(command):
    try:
        result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        if result.returncode == 0:
            print("FFmpeg command executed successfully.")
            return result.stdout, result.stderr
        else:
            print("Error executing FFmpeg command:")
            return None, result.stderr
    except Exception as e:
        print("An error occurred during FFmpeg execution:")
        return None, str(e)


def execute_fmpeg_command(command):
    try:
        result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True)
        return result.stdout  # Return just the stdout part, not a tuple
    except subprocess.CalledProcessError as e:
        print(f"FFmpeg command failed with error: {e.stderr.decode()}")
        return None

def search_keyword(keyword, frame_texts):
    return [index for index, text in st.session_state.frame_texts.items() if keyword.lower() in text.lower()]
frame_numbers = []
# Function to generate description for video frames

def generate_description(base64_frames):
    try:
        prompt_messages = [
            {
                "role": "user",
                "content": [ " Find the most interesting / impactful portions of a video.  The output will be targeted towards social media (like TikTok or Reels) or to news broadcasts. For the provided  frames return the most interesting / impactful frames that will hold the interest of an audience and also describe why you chose it. I am trying to fill these frames for a TikTok video. Hence while selecting the frames keep that in mind. You do not have to give me the script of the Tiktok vfideo. Just return the most interesting frames in a sequence that will come for a tiktok video. List all frame numbers separated by commas at the end like this for eg, Frames : 1,2,4,7,9",
                    *map(lambda x: {"image": x, "resize": 428}, base64_frames),
                ],
            },
        ]
        response = client.chat.completions.create(
            model="gpt-4-vision-preview",
            messages=prompt_messages,
            max_tokens=3000,
        )
        description = response.choices[0].message.content

        # Use regular expression to find frame numbers
        frame_numbers = re.findall(r'Frames\s*:\s*(\d+(?:,\s*\d+)*)', response.choices[0].message.content)

        # Convert the string of numbers into a list of integers
        if frame_numbers:
            frame_numbers = [int(num) for num in frame_numbers[0].split(',')]
        else:
            frame_numbers = []

        print("Frame numbers to extract:", frame_numbers)

        return description, frame_numbers
    
    except Exception as e:
        print(f"Error in generate_description: {e}")
        return None, []

with col2:
    pass

with col1:
    is_logo_path = "IntelliStreamLogo.png"  # Update this path to where your logo is stored
    is_logo = Image.open(is_logo_path)
    st.image(is_logo, width=200)

    st.markdown("<h1 style='text-align: left; color: white;'></h1>", unsafe_allow_html=True)

# Streamlit UI
st.title("Insightly Video")
uploaded_video = st.file_uploader("Or upload a video file (MP4):", type=["mp4"])

extract_frames_button = st.button("Extract Frames")

if uploaded_video is not None and extract_frames_button:
    with tempfile.NamedTemporaryFile(delete=False, suffix=".mp4") as tmpfile:
        tmpfile.write(uploaded_video.getvalue())
        video_file_path = tmpfile.name

        # Command to extract one frame per second for the entire video duration
        ffmpeg_command = [
            'ffmpeg',          
            '-i', video_file_path, 
            '-vf', 'fps=0.25',            
            '-f', 'image2pipe',        
            '-c:v', 'mjpeg',            
            '-an',                      
            '-'
        ]

        ffmpeg_output = execute_fmpeg_command(ffmpeg_command)

        if ffmpeg_output:
            st.write("Frames Extracted:")
            frame_bytes_list = ffmpeg_output.split(b'\xff\xd8')[1:]  # Correct splitting for JPEG frames
            n_frames = len(frame_bytes_list)
            base64_frames = [base64.b64encode(b'\xff\xd8' + frame_bytes).decode('utf-8') for frame_bytes in frame_bytes_list]

            frame_dict = {}

            for idx, frame_base64 in enumerate(base64_frames):
                    col1, col2 = st.columns([3, 2])
                    with col1:
                        frame_bytes = base64.b64decode(frame_base64)
                        frame_dict[idx + 1] = frame_bytes
                        st.image(Image.open(BytesIO(frame_bytes)), caption=f'Frame {idx + 1}', use_column_width=True)
                    with col2:
                        pass

        # Extract audio for the entire video
        audio_command = [
            'ffmpeg',
            '-i', video_file_path,  
            '-vn',                      
            '-acodec', 'libmp3lame',    
            '-f', 'mp3',                
            '-'
        ]
        audio_output, _ = execute_ffmpeg_command(audio_command)

        st.write("Extracted Audio:")
        audio_tempfile = tempfile.NamedTemporaryFile(delete=False, suffix=".mp3")
        audio_tempfile.write(audio_output)
        audio_tempfile.close()

        st.audio(audio_output, format='audio/mpeg', start_time=0)

        # Get consolidated description for all frames
        if ffmpeg_output:
            description, frame_numbers = generate_description(base64_frames)
            if description:
                st.header("Frame Description:")
                st.write(description)
            else:
                st.write("Failed to generate description.")

        if frame_numbers:
            print("Frame numbers to extract:", frame_numbers)  # Check frame numbers

            # Create a mapping from original frame numbers to sequential numbers
            frame_mapping = {}
            new_frame_numbers = []
            for idx, frame_number in enumerate(sorted(frame_numbers)):
                frame_mapping[frame_number] = idx + 1
                new_frame_numbers.append(idx + 1)

            print("New frame numbers:", new_frame_numbers)
            print("Frame mapping:", frame_mapping)

            # Create a temporary directory to store images
            with tempfile.TemporaryDirectory() as temp_dir:
                image_paths = []
                for frame_number in frame_numbers:
                    if frame_number in frame_dict:
                        frame_path = os.path.join(temp_dir, f'frame_{frame_mapping[frame_number]:03}.jpg')  # Updated file naming
                        image_paths.append(frame_path)
                        with open(frame_path, 'wb') as f:
                            f.write(frame_dict[frame_number])

                # Once all selected frames are saved as images, create a video from them using FFmpeg
                video_output_path = os.path.join(temp_dir, 'output.mp4')
                framerate = 1  # Adjust framerate based on the number of frames
                ffmpeg_command = [
                    'ffmpeg',
                    '-framerate', str(framerate),  # Set framerate based on the number of frames
                    '-i', os.path.join(temp_dir, 'frame_%03d.jpg'),  # Input pattern for all frame files
                    '-c:v', 'libx264',
                    '-pix_fmt', 'yuv420p',
                    video_output_path
                ]

                print("FFmpeg command:", ' '.join(ffmpeg_command))  # Debug FFmpeg command

                subprocess.run(ffmpeg_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

                # Display or provide a download link for the created video
                st.header("Final Video")
                st.video(video_output_path)