1

app.py

@@ -1,13 +1,97 @@
 import gradio as gr
-from IPython.display import HTML, IFrame 
-from IPython.display import YouTubeVideo
+from youtube_transcript_api import YouTubeTranscriptApi
+from transformers import AutoTokenizer
+from transformers import pipeline
+from transformers import AutoModelForQuestionAnswering
+import pandas as pd
+from sentence_transformers import SentenceTransformer, util
+import torch
 
-def display_vid(url):
+#from IPython.display import HTML, IFrame 
+#from IPython.display import YouTubeVideo
+
+#input - video link, output - full transcript
+def get_transcript(link):
+  video_id = link.split("=")[1]
+  print(f"video id extracted is : {video_id}")
+  transcript = YouTubeTranscriptApi.get_transcript(video_id)
+  FinalTranscript = ' '.join([i['text'] for i in transcript])
+  return transcript, video_id
+  
+  
+#input - question and transcript, output - answer timestamp
+def get_answers_timestamp(question, transcript):
+  model_ckpt = "deepset/minilm-uncased-squad2"
+  tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
+  #question = "any funny examples in video??"
+  context = transcript
+  inputs = tokenizer(question, context, return_overflowing_tokens=True, max_length=512, stride = 25)
+
+  #overlaps
+  #getting a list of contexts available after striding
+  contx=[]
+  for window in inputs["input_ids"]:
+      #print(f"{tokenizer.decode(window)} \n")
+      contx.append(tokenizer.decode(window).split('[SEP]')[1].strip())
+  #print(ques)
+  #print(contx)
+
+  model = AutoModelForQuestionAnswering.from_pretrained(model_ckpt)
+  lst=[]
+  pipe = pipeline("question-answering", model=model, tokenizer=tokenizer)
+  for contexts in contx:
+    #print(pipe(question=question, context=contexts))
+    lst.append(pipe(question=question, context=contexts))
+
+  lst_scores = [dicts['score'] for dicts in lst] 
+  #print(lst_scores)
+  #getting highest and second highest scores
+  idxmax = lst_scores.index(max(lst_scores))
+  lst_scores.remove(max(lst_scores))
+  idxmax2 = lst_scores.index(max(lst_scores))
+  #idxmax, idxmax2
+
+  idxcont = lst[idxmax2]['start']
+  answer = FinalTranscript[len(contx[0])-135 + idxcont:]
+  sentence_keyword = answer[:50]
+
+  dftranscript = pd.DataFrame(transcript)
+  #dftranscript.head()
+
+  modelST = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
+  embedding_1= modelST.encode(dftranscript.text, convert_to_tensor=True)
+  embedding_2 = modelST.encode(sentence_keyword, convert_to_tensor=True)
+
+  similarity_tensor = util.pytorch_cos_sim(embedding_1, embedding_2)
+  idx = torch.argmax(similarity_tensor)
+  start_timestamp = dftranscript.iloc[[int(idx)+1]].start.values[0]
+  start_timestamp = round(start_timestamp)
+
+  return start_timestamp
+  
+  
+def display_vid(url, question):
   #https://www.youtube.com/watch?v=smUHQndcmOY&t=425s
   #html = HTML("<iframe width='560' height='315' src='https://www.youtube.com/watch?v=smUHQndcmOY&t=425s' frameborder='0' allowfullscreen></iframe>")
   #html = "<iframe width='560' height='315' src='https://www.youtube.com/embed/smUHQndcmOY' frameborder='0' allowfullscreen></iframe>"
+  #get embedding and youtube link
+  html = "<iframe width='560' height='315' src=" + url + " frameborder='0' allowfullscreen></iframe>"
+  print(html)
+  
+  #get transcript
+  transcript, video_id = get_transcript(html)
+  
+  #get answer timestamp
+  #input - question and transcript, output - answer timestamp
+  ans_timestamp = get_answers_timestamp(question, transcript):
+  
+  #created embedding
+  #sample - smUHQndcmOY?start=234 
+  html_out = "<iframe width='560' height='315' src='https://www.youtube.com/embed/" + video_id + "?start=" + ans_timestamp + " title='YouTube video player' frameborder='0' allow='accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture' allowfullscreen></iframe>"
+  print(f"html output is : {html_out}")
   #vid = YouTubeVideo('smUHQndcmOY&t=425s')
   return html
+  
   #https://youtu.be/smUHQndcmOY
 def fun(url):
   return gr.Video(value=url)
@@ -24,13 +108,13 @@ with demo:
     )
   with gr.Row():
     input_url = gr.Textbox() #gr.HTML(placeholder="Enter a video link here..")
-    #input_vid = gr.Video()
+    input_ques = gr.Textbox()
     output_vid = gr.HTML()
   
   b1 = gr.Button("Publish Video")
   #b2 = gr.Button("Generate Image")
 
-  b1.click(display_vid, input_url, output_vid)
+  b1.click(display_vid, inputs=[input_url,input_ques], outputs=output_vid)
   #b2.click(poem_to_image, poem_txt, output_image)
   #examples=examples