import streamlit as st
import pinecone
from sentence_transformers import SentenceTransformer
import logging
PINECONE_KEY = st.secrets["PINECONE_KEY"] # app.pinecone.io
INDEX_ID = 'youtube-search'
@st.experimental_singleton
def init_pinecone():
pinecone.init(api_key=PINECONE_KEY, environment="us-west1-gcp")
return pinecone.Index(INDEX_ID)
@st.experimental_singleton
def init_retriever():
return SentenceTransformer("multi-qa-mpnet-base-dot-v1")
def make_query(query, retriever, top_k=10, include_values=True, include_metadata=True, filter=None):
xq = retriever.encode([query]).tolist()
logging.info(f"Query: {query}")
attempt = 0
while attempt < 3:
try:
xc = st.session_state.index.query(
xq,
top_k=top_k,
include_values=include_values,
include_metadata=include_metadata,
filter=filter
)
matches = xc['matches']
break
except:
# force reload
pinecone.init(api_key=PINECONE_KEY, environment="us-west1-gcp")
st.session_state.index = pinecone.Index(INDEX_ID)
attempt += 1
matches = []
if len(matches) == 0:
logging.error(f"Query failed")
return matches
st.session_state.index = init_pinecone()
retriever = init_retriever()
def card(thumbnail: str, title: str, urls: list, contexts: list, starts: list, ends: list):
meta = [(e, s, u, c) for e, s, u, c in zip(ends, starts, urls, contexts)]
meta.sort(reverse=False)
text_content = []
current_start = 0
current_end = 0
for end, start, url, context in meta:
# reformat seconds to timestamp
time = start / 60
mins = f"0{int(time)}"[-2:]
secs = f"0{int(round((time - int(mins))*60, 0))}"[-2:]
timestamp = f"{mins}:{secs}"
if start < current_end and start > current_start:
# this means it is a continuation of the previous sentence
text_content[-1][0] = text_content[-1][0].split(context[:10])[0]
text_content.append([f"[{timestamp}] {context.capitalize()}", url])
else:
text_content.append(["xxLINEBREAKxx", ""])
text_content.append([f"[{timestamp}] {context}", url])
current_start = start
current_end = end
html_text = ""
for text, url in text_content:
if text == "xxLINEBREAKxx":
html_text += "
"
else:
html_text += f"{text.strip()}... "
print(text)
html = f"""