Added openai and gemini for video QnA.

app.py

@@ -0,0 +1,45 @@
+import streamlit as st
+from lib.services.hf_model import get_transcript
+from lib.services.gemini import gemini
+from lib.services.openai import get_completion
+
+
+@st.cache_resource
+def get_cached_transcript(video_url):
+    return get_transcript(video_url)
+
+
+def main():
+    st.title("VideoClarify")
+
+    # Get video URL from user
+    video_url = st.text_input("Enter Video URL:", key="video_url")
+    selected_model = st.sidebar.selectbox("Select Model", ["Gemini", "OpenAI"])
+    print(selected_model)
+    if video_url:
+        st.video(video_url)
+        # Get transcript from the video
+        transcript = get_cached_transcript(video_url)
+        # Provide an input box for user to ask a question
+        question = st.text_input(
+            label="Ask a question about the video:", key="question")
+
+        if st.button("Get Answer"):
+            if question:
+                if selected_model == "Gemini":
+                    st.info("Using Gemini to answer the question.")
+                    # Use Gemini to summarize and answer the question
+                    response = gemini(transcript, question)
+                if selected_model == "OpenAI":
+                    st.info("Using OpenAI to answer the question.")
+                    # Use OpenAI to summarize and answer the question
+                    response = get_completion(transcript, question)
+                # Display the result to the user
+                st.subheader("Result:")
+                st.write(response)
+            else:
+                st.info("Please ask a question about the video.")
+
+
+if __name__ == "__main__":
+    main()

lib/services/gemini.py

@@ -0,0 +1,40 @@
+import os
+from pathlib import Path
+
+import PIL.Image
+import google.generativeai as genai
+
+def configure_genai(api_key):
+    genai.configure(api_key=api_key)
+
+
+def generate_content(model, prompt, stream=True):
+    response = model.generate_content(prompt, stream=stream)
+    response.resolve()
+    return response.text
+
+
+def gemini(transcript, question):
+    print(transcript, question)
+    configure_genai(os.environ['GOOGLE_API_KEY'])
+
+    # Create GenerativeModel instance
+    model = genai.GenerativeModel('gemini-pro')
+
+    # Generate content using the model and image
+    prompt = [f"""
+Transcript:
+```
+{transcript}
+```
+Provided is a video transcript enclosed within triple backticks. Your task is to respond to questions that are either based on or directly related to the content of the video transcript. If the question does not pertain to or is not in the context of the video transcript, please reply with "Please ask questions related to the video only."
+
+Note:
+- Do not include `video transcript` in your response, refer it as `video`.
+              
+Question: {question}
+"""]
+    response_text = generate_content(model, prompt)
+
+    return response_text
+    # Optionally display as Markdown

lib/services/hf_model.py

@@ -0,0 +1,31 @@
+import os
+import requests
+from settings import HF_API_URL, DATA_DIR
+from pathlib import Path
+from moviepy.editor import VideoFileClip
+
+def convert_video_to_wav(video_path, output_path):
+    """
+    Converts a video file to a WAV audio file.
+
+    Args:
+        video_path (str): The path of the video file to be converted.
+        output_path (str): The desired path for the output WAV audio file.
+
+    Returns:
+        None
+    """
+    video_clip = VideoFileClip(video_path)
+    audio_clip = video_clip.audio
+    audio_clip.write_audiofile(output_path)
+
+def get_transcript(filepath):
+    audio_file = Path(DATA_DIR).joinpath(Path(filepath).stem + ".wav")
+    if not audio_file.exists():
+        convert_video_to_wav(filepath, audio_file)
+    headers = {"Authorization": f"Bearer {os.environ['HF_KEY']}"}
+    with open(audio_file, "rb") as f:
+        data = f.read()
+    response = requests.post(HF_API_URL, headers=headers,
+                             data=data)
+    return response.json()["text"]

lib/services/openai.py

@@ -0,0 +1,35 @@
+import openai
+def get_completion(transcript, question):
+    """
+    Generate a text completion using OpenAI's GPT-3 model.
+
+    Args:
+        prompt (str): The input text prompt for text generation.
+        model (str, optional): The GPT-3 model to use. Default is "gpt-3.5-turbo-16k".
+
+    Returns:
+        str: The generated text based on the prompt.
+    """
+    messages = [
+        {
+            "role": "system", 
+            "content": f"""
+Transcript:
+```
+{transcript}
+```
+Provided is a video transcript enclosed within triple backticks. Your task is to respond to questions that are either based on or directly related to the content of the video transcript. If the question does not pertain to or is not in the context of the video transcript, please reply with "Please ask questions related to the video only."
+
+Note:
+- Do not include `video transcript` in your response, refer it as `video`.
+              
+Question: {question}
+"""
+        }
+    ]
+    response = openai.ChatCompletion.create(
+        model="gpt-3.5-turbo-16k",
+        messages=messages,
+        temperature=0.2,  # This is the degree of randomness of the model's output
+    )
+    return response.choices[0].message["content"]

requirements.txt

@@ -0,0 +1,68 @@
+aiohttp==3.9.1
+aiosignal==1.3.1
+altair==5.2.0
+attrs==23.2.0
+blinker==1.7.0
+cachetools==5.3.2
+certifi==2023.11.17
+charset-normalizer==3.3.2
+click==8.1.7
+decorator==4.4.2
+frozenlist==1.4.1
+gitdb==4.0.11
+GitPython==3.1.41
+google-ai-generativelanguage==0.4.0
+google-api-core==2.15.0
+google-auth==2.26.2
+google-generativeai==0.3.2
+googleapis-common-protos==1.62.0
+grpcio==1.60.0
+grpcio-status==1.60.0
+idna==3.6
+imageio==2.33.1
+imageio-ffmpeg==0.4.9
+importlib-metadata==7.0.1
+Jinja2==3.1.3
+jsonschema==4.21.1
+jsonschema-specifications==2023.12.1
+markdown-it-py==3.0.0
+MarkupSafe==2.1.4
+mdurl==0.1.2
+moviepy==1.0.3
+multidict==6.0.4
+numpy==1.26.3
+openai==0.28.0
+packaging==23.2
+pandas==2.2.0
+pillow==10.2.0
+proglog==0.1.10
+proto-plus==1.23.0
+protobuf==4.25.2
+pyarrow==14.0.2
+pyasn1==0.5.1
+pyasn1-modules==0.3.0
+pydeck==0.8.1b0
+Pygments==2.17.2
+python-dateutil==2.8.2
+pytz==2023.3.post1
+referencing==0.32.1
+requests==2.31.0
+rich==13.7.0
+rpds-py==0.17.1
+rsa==4.9
+six==1.16.0
+smmap==5.0.1
+streamlit==1.30.0
+tenacity==8.2.3
+toml==0.10.2
+toolz==0.12.0
+tornado==6.4
+tqdm==4.66.1
+typing_extensions==4.9.0
+tzdata==2023.4
+tzlocal==5.2
+urllib3==2.1.0
+validators==0.22.0
+watchdog==3.0.0
+yarl==1.9.4
+zipp==3.17.0

settings.py

@@ -0,0 +1,4 @@
+from pathlib import Path
+HF_API_URL = "https://api-inference.huggingface.co/models/openai/whisper-large-v2"
+BASE_DIR = Path(__file__).parent
+DATA_DIR = Path(BASE_DIR).joinpath("data")