Upload 2 files

app.py

@@ -0,0 +1,73 @@
+import validators
+import streamlit as st
+from langchain.prompts import PromptTemplate
+from langchain_groq import ChatGroq
+from langchain.chains.summarize import load_summarize_chain
+from langchain.docstore.document import Document
+from langchain_community.document_loaders import UnstructuredURLLoader
+import yt_dlp
+
+# Streamlit App Configuration
+st.set_page_config(page_title="LangChain: Summarize Text From YT or Website", page_icon="🦜")
+st.title("🦜 LangChain: Summarize Text From YT or Website")
+st.subheader('Summarize URL')
+
+# Hardcoded Groq API Key (use environment variables in production)
+GROQ_API_KEY = "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"
+
+# URL input field
+generic_url = st.text_input("URL", label_visibility="collapsed")
+
+# Gemma Model Using Groq API
+llm = ChatGroq(model="mixtral-8x7b-32768", groq_api_key=GROQ_API_KEY)
+
+prompt_template = """
+Provide a summary of the following content in 300 words:
+Content:{text}
+
+"""
+prompt = PromptTemplate(template=prompt_template, input_variables=["text"])
+
+def load_youtube_data(url):
+    try:
+        ydl_opts = {}
+        with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+            info = ydl.extract_info(url, download=False)
+            title = info.get('title', 'No Title')
+            description = info.get('description', 'No Description')
+            content = f"Title: {title}\n\nDescription: {description}"
+            return [Document(page_content=content, metadata={"title": title})]
+    except Exception as e:
+        raise ValueError(f"Failed to extract YouTube data: {str(e)}")
+
+def load_website_data(url):
+    loader = UnstructuredURLLoader(
+        urls=[url],
+        ssl_verify=False,
+        headers={"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_5_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36"}
+    )
+    docs = loader.load()
+    return [Document(page_content=doc.page_content, metadata=doc.metadata) for doc in docs]
+
+if st.button("Summarize the Content from YT or Website"):
+    # Validate URL input
+    if not generic_url.strip():
+        st.error("Please provide a URL to get started")
+    elif not validators.url(generic_url):
+        st.error("Please enter a valid URL. It can be a YouTube video URL or website URL.")
+    else:
+        try:
+            with st.spinner("Processing..."):
+                # Load data based on URL type
+                if "youtube.com" in generic_url or "youtu.be" in generic_url:
+                    docs = load_youtube_data(generic_url)
+                else:
+                    docs = load_website_data(generic_url)
+
+                # Chain for Summarization
+                chain = load_summarize_chain(llm, chain_type="stuff", prompt=prompt)
+                output_summary = chain.run(docs)
+
+                st.success(output_summary)
+        except Exception as e:
+            st.exception(f"Exception: {e}")

requirements.txt

@@ -0,0 +1,46 @@
+langchain
+langchain_astradb
+python-dotenv
+ipykernel
+langchain-community
+pypdf
+bs4
+arxiv
+pymupdf
+wikipedia
+langchain-text-splitters
+langchain-openai
+chromadb
+sentence_transformers
+langchain_huggingface
+faiss-cpu
+langchain_chroma
+streamlit
+langchain_groq
+fastapi
+uvicorn
+langserve[all]
+sse_starlette
+streamlit
+PyPDF2
+google.generativeai
+arxiv
+wikipedia
+streamlit-pydantic
+validators
+youtube_transcript_api
+pytube
+Unstructured
+yt-dlp
+numexpr
+langchain_huggingface
+huggingface_hub
+duckduckgo-search
+langchain_nvidia_ai_endpoints
+crewai
+crewai_tools
+pinecone
+pinecone-client
+neo4j==5.14
+llama-index
+