Added all files

app.py

@@ -0,0 +1,169 @@
+import os
+from PyPDF2 import PdfReader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.vectorstores import FAISS
+from langchain.prompts import PromptTemplate
+from langchain.chains.question_answering import load_qa_chain
+from langchain_google_genai import GoogleGenerativeAIEmbeddings, ChatGoogleGenerativeAI
+from dotenv import load_dotenv
+import streamlit as st
+import re
+from google.generativeai import configure
+
+# Load environment variables
+load_dotenv()
+os.getenv("GOOGLE_API_KEY")
+
+# Configure Google Generative AI
+configure(api_key=os.getenv("GOOGLE_API_KEY"))
+
+def get_pdf_text(pdf_docs):
+    """Extract text and course details (title, description, and link) from the PDF."""
+    text = ""
+    course_details = []
+    for pdf in pdf_docs:
+        pdf_reader = PdfReader(pdf)
+        for page in pdf_reader.pages:
+            page_text = page.extract_text()
+            text += page_text
+            
+            # Extract the course titles and links (bold titles and underlined links)
+            courses = extract_course_details(page_text)
+            course_details.extend(courses)
+
+    return text, course_details
+
+def extract_course_details(page_text):
+    """Extract course title and link from the page text."""
+    course_details = []
+    # Regex to find bold titles and underlined links
+    title_pattern = r"(\*\*([A-Z\s]+)\*\*)(.*?)(http[s]?://[^\s]+)"
+    matches = re.findall(title_pattern, page_text)
+    
+    for match in matches:
+        title = match[1].strip()
+        description = match[2].strip()
+        link = match[3].strip()
+        
+        # Return tuple of course title, description, and link
+        course_details.append({
+            "title": title,
+            "description": description,
+            "link": link
+        })
+    
+    return course_details
+
+def get_text_chunks(text, course_details):
+    """Split the extracted text into chunks and append course links to the description."""
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=10000, chunk_overlap=1000)
+    
+    chunks = []
+    for course in course_details:
+        course_text = f"**Course Title**: {course['title']}\n"
+        course_text += f"[Course Link]({course['link']})\n"
+        course_text += f"**Description**: {course['description']}\n"
+        
+        text_chunks = text_splitter.split_text(course_text)
+        chunks.extend(text_chunks)
+    
+    return chunks
+
+def get_vector_store(text_chunks):
+    """Generate and store embeddings in a vector store."""
+    embeddings = GoogleGenerativeAIEmbeddings(model="models/embedding-001")
+    vector_store = FAISS.from_texts(text_chunks, embedding=embeddings)
+    vector_store.save_local("embedding")
+    return vector_store
+
+def generate_embeddings_from_pdf(pdf_docs):
+    """Generate and save embeddings from the PDF course file."""
+    # Extract text and course details from the PDF
+    raw_text, course_details = get_pdf_text(pdf_docs)
+    
+    # Get text chunks with course title, link, and description
+    text_chunks = get_text_chunks(raw_text, course_details)
+    
+    # Generate and save the vector store
+    vector_store = get_vector_store(text_chunks)
+    
+    print(f"Embeddings generated and saved successfully.")
+    return vector_store
+
+def load_vector_store():
+    """Load pre-generated embeddings from FAISS."""
+    embeddings = GoogleGenerativeAIEmbeddings(model="models/embedding-001")
+    vector_store = FAISS.load_local("embedding", embeddings, allow_dangerous_deserialization=True)
+    return vector_store
+
+def get_conversational_chain():
+    """Create a structured chain for processing search queries."""
+    system_prompt = """
+    You are an intelligent assistant helping users find the best free courses on data science, machine learning, and related fields.
+    When given a query, recommend courses by analyzing their relevance based on:
+    - Keywords
+    - Topics of interest
+    - User's goals (if provided)
+
+    Format your responses as:
+    - **Course Title**: <Title>
+    - [Course Link](<Link>)
+    - **Description**: <Brief Description>
+    - **Relevance**: <Why it is recommended>
+    Provide concise and actionable recommendations.
+    """
+
+    prompt_template = PromptTemplate(
+        template=system_prompt + "\nContext: {context}\nQuery: {query}",
+        input_variables=["context", "query"]
+    )
+    model = ChatGoogleGenerativeAI(model="gemini-pro", temperature=0.3)
+    chain = load_qa_chain(model, chain_type="stuff", prompt=prompt_template)
+
+    return chain
+
+def user_input(user_query, keywords):
+    """Process user input and search for relevant courses."""
+    vector_store = load_vector_store()
+    chain = get_conversational_chain()
+
+    # Construct search query
+    query_context = f"Keywords: {keywords}. Query: {user_query}."
+    docs = vector_store.similarity_search(query_context)
+
+    # Get recommendations
+    response = chain({
+        "input_documents": docs,
+        "context": "Analytics Vidhya free courses database.",
+        "query": query_context
+    }, return_only_outputs=True)
+
+    return response["output_text"]
+
+def main():
+    # Streamlit app UI
+    st.set_page_config("Smart Course Search", layout="wide")
+    st.title("Smart Course Search Tool")
+    st.write("Search for the most relevant free courses using natural language or keywords.")
+
+    # User inputs
+    user_query = st.text_input("Enter your search query or context (e.g., 'I want to learn deep learning')")
+    keywords = st.text_input("Enter specific keywords (comma-separated, e.g., 'NLP, data visualization')")
+
+    if st.button("Search Courses"):
+        if user_query or keywords:
+            with st.spinner("Searching for the best courses..."):
+                results = user_input(user_query, keywords)
+                st.success("Search Complete!")
+                
+                # Beautify and display the results
+                if results:
+                    formatted_results = results.replace("**", "<b>").replace("**", "</b>").replace("[", "<u>").replace("]", "</u>")
+                    st.markdown(formatted_results, unsafe_allow_html=True)
+                else:
+                    st.warning("No relevant courses found.")
+        else:
+            st.error("Please provide a query or keywords for searching.")
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,169 @@
+aiohttp==3.9.5
+aiosignal==1.3.1
+altair==5.3.0
+annotated-types==0.7.0
+anyio==4.4.0
+asgiref==3.8.1
+asttokens==2.4.1
+async-timeout==4.0.3
+attrs==23.2.0
+backoff==2.2.1
+bcrypt==4.2.0
+blinker==1.8.2
+build==1.2.1
+cachetools==5.4.0
+certifi==2024.7.4
+charset-normalizer==3.3.2
+chroma-hnswlib==0.7.6
+chromadb==0.5.5
+click==8.1.7
+colorama==0.4.6
+coloredlogs==15.0.1
+comm==0.2.2
+dataclasses-json==0.6.7
+debugpy==1.8.2
+decorator==5.1.1
+Deprecated==1.2.14
+dnspython==2.6.1
+email_validator==2.2.0
+exceptiongroup==1.2.2
+executing==2.0.1
+faiss-cpu==1.8.0.post1
+fastapi==0.111.1
+fastapi-cli==0.0.4
+filelock==3.15.4
+flatbuffers==24.3.25
+frozenlist==1.4.1
+fsspec==2024.6.1
+gitdb==4.0.11
+GitPython==3.1.43
+google-ai-generativelanguage==0.6.6
+google-api-core==2.19.1
+google-api-python-client==2.138.0
+google-auth==2.32.0
+google-auth-httplib2==0.2.0
+google-generativeai==0.7.2
+googleapis-common-protos==1.63.2
+greenlet==3.0.3
+grpcio==1.65.1
+h11==0.14.0
+httpcore==1.0.5
+httplib2==0.22.0
+httptools==0.6.1
+httpx==0.27.0
+huggingface-hub==0.24.2
+humanfriendly==10.0
+idna==3.7
+importlib_metadata==8.0.0
+importlib_resources==6.4.0
+ipykernel==6.29.5
+ipython==8.18.1
+jedi==0.19.1
+Jinja2==3.1.4
+jsonpatch==1.33
+jsonpointer==3.0.0
+jsonschema==4.23.0
+jsonschema-specifications==2023.12.1
+jupyter_client==8.6.2
+jupyter_core==5.7.2
+kubernetes==30.1.0
+langchain==0.2.11
+langchain-community==0.2.10
+langchain-core==0.2.24
+langchain-google-genai==1.0.8
+langchain-text-splitters==0.2.2
+langsmith==0.1.93
+markdown-it-py==3.0.0
+MarkupSafe==2.1.5
+marshmallow==3.21.3
+matplotlib-inline==0.1.7
+mdurl==0.1.2
+mmh3==4.1.0
+monotonic==1.6
+mpmath==1.3.0
+multidict==6.0.5
+mypy-extensions==1.0.0
+nest-asyncio==1.6.0
+numpy==1.26.4
+oauthlib==3.2.2
+onnxruntime==1.18.1
+opentelemetry-api==1.26.0
+opentelemetry-exporter-otlp-proto-common==1.26.0
+opentelemetry-exporter-otlp-proto-grpc==1.26.0
+opentelemetry-instrumentation==0.47b0
+opentelemetry-instrumentation-asgi==0.47b0
+opentelemetry-instrumentation-fastapi==0.47b0
+opentelemetry-proto==1.26.0
+opentelemetry-sdk==1.26.0
+opentelemetry-semantic-conventions==0.47b0
+opentelemetry-util-http==0.47b0
+orjson==3.10.6
+overrides==7.7.0
+packaging==24.1
+pandas==2.2.2
+parso==0.8.4
+pillow==10.4.0
+platformdirs==4.2.2
+posthog==3.5.0
+prompt_toolkit==3.0.47
+proto-plus==1.24.0
+protobuf==3.20.2
+psutil==6.0.0
+pure_eval==0.2.3
+pyarrow==17.0.0
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pydantic==2.8.2
+pydantic_core==2.20.1
+pydeck==0.9.1
+Pygments==2.18.0
+pyparsing==3.1.2
+PyPDF2==3.0.1
+PyPika==0.48.9
+pyproject_hooks==1.1.0
+pyreadline3==3.4.1
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+python-multipart==0.0.9
+pytz==2024.1
+pywin32==306
+PyYAML==6.0.1
+pyzmq==26.0.3
+referencing==0.35.1
+requests==2.32.3
+requests-oauthlib==2.0.0
+rich==13.7.1
+rpds-py==0.19.1
+rsa==4.9
+shellingham==1.5.4
+six==1.16.0
+smmap==5.0.1
+sniffio==1.3.1
+SQLAlchemy==2.0.31
+stack-data==0.6.3
+starlette==0.37.2
+streamlit==1.37.0
+sympy==1.13.1
+tenacity==8.5.0
+tokenizers==0.19.1
+toml==0.10.2
+tomli==2.0.1
+toolz==0.12.1
+tornado==6.4.1
+tqdm==4.66.4
+traitlets==5.14.3
+typer==0.12.3
+typing-inspect==0.9.0
+typing_extensions==4.12.2
+tzdata==2024.1
+uritemplate==4.1.1
+urllib3==2.2.2
+uvicorn==0.30.3
+watchdog==4.0.1
+watchfiles==0.22.0
+wcwidth==0.2.13
+websocket-client==1.8.0
+websockets==12.0
+wrapt==1.16.0
+yarl==1.9.4
+zipp==3.19.2