Upload app.py

app.py

@@ -1,124 +1,425 @@
+from streamlit import session_state as ss
+from streamlit_pdf_viewer import pdf_viewer
+import streamlit_pdf_viewer
+
 import streamlit as st
-import os
-from streamlit_chat import message
-import numpy as np
-import pandas as pd
-from io import StringIO
-import io
-import PyPDF2
-import pymupdf
-import tempfile
-import base64
-# from tqdm.auto import tqdm
-import math
-# from transformers import pipeline
 
-from collections import Counter
-import nltk
 
-nltk.download('stopwords')
-from nltk.corpus import stopwords
-import re
+# # Declare variable.
+# if 'pdf_ref' not in ss:
+#     ss.pdf_ref = None
+
+
+# # Access the uploaded ref via a key.
+# st.file_uploader("Upload PDF file", type=('pdf'), key='pdf')
+
+# if ss.pdf:
+#     ss.pdf_ref = ss.pdf  # backup
+
+# # Now you can access "pdf_ref" anywhere in your app.
+# if ss.pdf_ref:
+#     binary_data = ss.pdf_ref.getvalue()
+#     pdf_viewer(input=binary_data, width=700)
+
+# import base64
+
+# def displayPDF(file):
+#     # Opening file from file path
+#     with open(file, "rb") as f:
+#         base64_pdf = base64.b64encode(f.read()).decode('utf-8')
+
+#     # Embedding PDF in HTML
+#     pdf_display = f'<iframe src="data:application/pdf;base64,{base64_pdf}" width="700" height="1000" type="application/pdf"></iframe>'
+
+#     # Displaying File
+#     st.markdown(pdf_display, unsafe_allow_html=True)
+
+# displayPDF("../Transformers/Bhagavad-Gita-As-It-Is.pdf")
+
+
+# import streamlit as st
+# import streamlit_pdf_viewer
 
+# def displayPDF(file):
+#     with open(file, "rb") as f:
+#         pdf_bytes = f.read()
 
-from sentence_transformers import SentenceTransformer
-import torch
-from langchain_community.llms.ollama import Ollama
-from langchain.prompts import ChatPromptTemplate
-from langchain_community.vectorstores import FAISS
+#     streamlit_pdf_viewer.pdf_viewer(pdf_bytes)
 
-from langchain_community.llms import HuggingFaceHub
-# from langchain.vectorstores import faiss
-# from langchain.vectorstores import FAISS
+# displayPDF("../Transformers/Bhagavad-Gita-As-It-Is.pdf")
+# Arial Unicode.ttf
 
-import time
-from time import sleep
-from stqdm import stqdm
-from dotenv import load_dotenv
+# import streamlit as st
+# import fitz  # PyMuPDF library
+# from PIL import Image, ImageDraw, ImageFont
+# import io
+# import numpy as np
 
-# Load environment variables from .env file
-load_dotenv()
+# def display_pdf_with_highlight(file_path, keywords):
+#     # Open the PDF file
+#     with fitz.open(file_path) as doc:
+#         # Create a new PDF file to hold the highlighted pages
+#         highlighted_pdf = fitz.open()
 
-HUGGINGFACEHUB_API_TOKEN = os.getenv("HUGGINGFACEHUB_API_TOKEN")
+#         # Iterate over each page in the PDF
+#         for page_index in range(len(doc)):
+#             page = doc.load_page(page_index)
+#             pix = page.get_pixmap(dpi=300)
+#             img = Image.frombytes("RGB", [pix.width, pix.height], pix.samples)
 
+#             # Create a drawing object and highlight the keywords
+#             draw = ImageDraw.Draw(img)
+#             font = ImageFont.truetype("Arial Unicode.ttf", 14)  # Replace with your desired font
+#             for keyword in keywords:
+#                 areas = page.search_for(keyword)
+#                 for area in areas:
+#                     bbox = area.bbox
+#                     draw.rectangle(bbox, outline="yellow", width=3)
 
-device = 'cuda' if torch.cuda.is_available() else 'cpu'
+#             # Convert the highlighted image to a NumPy array
+#             img_np = np.asarray(img)
 
-# if device != 'cuda':
-#     st.markdown(f"you are using {device}. This is much slower than using "
-#     "a CUDA-enabled GPU. If on colab you can change this by "
-#     "clicking Runtime > change runtime type > GPU.")
+#             # Create a MuPDF-compatible Pixmap from the NumPy array
+#             muimg = fitz.Pixmap(fitz.csRGB, img_np.shape[1], img_np.shape[0])
+#             muimg.set_data(img_np.tobytes())
 
-model = SentenceTransformer("sentence-transformers/all-mpnet-base-v2", device=device)
-def display_title():
-    selected_value = st.session_state["value"]
+#             # Create a new PDF page and insert the highlighted image
+#             new_page = highlighted_pdf.new_page(-1, width=muimg.width, height=muimg.height)
+#             new_page.insert_image(fitz.Rect(0, 0, muimg.width, muimg.height), stream=muimg)
 
-    st.header(f'Vedic Scriptures: {selected_value} :blue[book] :books:')
+#         # Create a BytesIO object to hold the highlighted PDF data
+#         pdf_bytes = io.BytesIO()
+#         highlighted_pdf.write(pdf_bytes)
+#         pdf_bytes.seek(0)
 
-question = "ask anything about scriptures"
-def open_chat():
-    question = st.session_state["faq"]
+#         # Display the highlighted PDF in Streamlit
+#         st.download_button(
+#             label="Download Highlighted PDF",
+#             data=pdf_bytes.getvalue(),
+#             file_name="highlighted_pdf.pdf",
+#             mime="application/pdf",
+#         )
 
+# # Example usage
+# file_path = "../Transformers/Bhagavad-Gita-As-It-Is.pdf"
+# keywords = ["Arjuna", "Krishna"]
+# display_pdf_with_highlight(file_path, keywords)
+
+# import pyperclip
+
+# content = str(pyperclip.paste())
+
+# import streamlit as st
+# import fitz
+
+# def annotate_pdf(file_path, text_to_highlight):
+#     # Open the PDF file
+#     with fitz.open(file_path) as doc:
+#         # Create a new PDF file to hold the annotated pages
+#         annotated_pdf = fitz.open()
+
+#         # Iterate over each page in the PDF
+#         for page_index in range(len(doc)):
+#             page = doc.load_page(page_index)
+
+#             # Search for the text to highlight
+#             areas = page.search_for(text_to_highlight)
+
+#             # Add rectangle annotations for the highlighted areas
+#             for area in areas:
+#                 page.add_rect_annot(area)
+
+#             # Create a new PDF page and insert the annotated page
+#             new_page = annotated_pdf.new_page(-1, width=page.rect.width, height=page.rect.height)
+#             new_page.show_pdf_page(page.rect, doc, page_index)
+
+#         # Create a BytesIO object to hold the annotated PDF data
+#         pdf_bytes = annotated_pdf.write()
+
+#         # Display the annotated PDF in Streamlit
+#         st.download_button(
+#             label="Download Annotated PDF",
+#             data=pdf_bytes,
+#             file_name="annotated_pdf.pdf",
+#             mime="application/pdf",
+#         )
+
+# # Example usage
+# file_path = "../Transformers/Bhagavad-Gita-As-It-Is.pdf"
+# text_to_highlight = "Arjuna"
+# annotate_pdf(file_path, text_to_highlight)
+
+# def displayPDF(file):
+#     # Opening file from file path
+#     with open(file, "rb") as f:
+#         base64_pdf = base64.b64encode(f.read()).decode('utf-8')
+
+#     # Embedding PDF in HTML
+#     pdf_display = f'<iframe src="data:application/pdf;base64,{base64_pdf}" width="700" height="1000" type="application/pdf"></iframe>'
+
+#     # Displaying File
+#     st.markdown(pdf_display, unsafe_allow_html=True)
+
+# displayPDF("../Transformers/Bhagavad-Gita-As-It-Is.pdf")
+
+# import streamlit as st
     
+# import fitz
+
+# def annotate_pdf(file_path, text_to_highlight):
+#     # Open the PDF file
+#     with fitz.open(file_path) as doc:
+#         # Create a new PDF file to hold the annotated pages
+#         annotated_pdf = fitz.open()
+
+#         # Iterate over each page in the PDF
+#         for page_index in range(len(doc)):
+#             page = doc.load_page(page_index)
+
+#             # Search for the text to highlight
+#             areas = page.search_for(text_to_highlight)
+
+#             # Add rectangle annotations for the highlighted areas
+#             for area in areas:
+#                 page.add_rect_annot(area)
+
+#             # Create a new PDF page and insert the annotated page
+#             new_page = annotated_pdf.new_page(-1, width=page.rect.width, height=page.rect.height)
+#             new_page.show_pdf_page(page.rect, doc, page_index)
+
+#         # Create a BytesIO object to hold the annotated PDF data
+#         pdf_bytes = annotated_pdf.write()
+
+#         streamlit_pdf_viewer.pdf_viewer(pdf_bytes)
+
+#         # Display the annotated PDF in Streamlit
+#         st.download_button(
+#             label="Download Annotated PDF",
+#             data=pdf_bytes,
+#             file_name="annotated_pdf.pdf",
+#             mime="application/pdf",
+#         )
+
+# # Example usage
+# file_path = "../Transformers/Bhagavad-Gita-As-It-Is.pdf"
+# text_to_highlight =  "Krishna"
+# annotate_pdf(file_path, text_to_highlight)
+
+
+# import streamlit as st
+# import fitz
+# import io
+
+# def annotate_pdf(uploaded_file, text_to_highlight):
+#     try:
+#         # Open the PDF file from the file-like object
+#         doc = fitz.open(stream=uploaded_file.read(), filetype="pdf")
 
-if "value" not in st.session_state:
-    st.session_state["value"] = None
+#         # Create a new PDF file to hold the annotated pages
+#         annotated_pdf = fitz.open()
 
-if "faq" not in st.session_state:
-    st.session_state["faq"] = None
+#         # Iterate over each page in the PDF
+#         for page_index in range(len(doc)):
+#             page = doc.load_page(page_index)
 
-# st.divider()
+#             # Search for the text to highlight
+#             areas = page.search_for(text_to_highlight)
 
-def upload_file():
-    uploaded_file = st.file_uploader("Upload a file", type=["pdf"])
-    if uploaded_file is not None:
-        st.write(uploaded_file.name)
-        return uploaded_file.name
+#             # Add rectangle annotations for the highlighted areas
+#             for area in areas:
+#                 page.add_rect_annot(area)
 
-def create_pickle_file(filepath):
+#             # Create a new PDF page and insert the annotated page
+#             new_page = annotated_pdf.new_page(-1, width=page.rect.width, height=page.rect.height)
+#             new_page.show_pdf_page(page.rect, doc, page_index)
 
-    from langchain_community.document_loaders import PyMuPDFLoader
-    loader = PyMuPDFLoader(filepath)
-    pages = loader.load()
+#         # Create a BytesIO object to hold the annotated PDF data
+#         pdf_bytes = io.BytesIO(annotated_pdf.write())
+
+#         # Display the annotated PDF in Streamlit
+#         st.download_button(
+#             label="Download Annotated PDF",
+#             data=pdf_bytes.getvalue(),
+#             file_name="annotated_pdf.pdf",
+#             mime="application/pdf",
+#         )
+#     except Exception as e:
+#         st.error(f"An error occurred: {str(e)}")
+
+# # Streamlit app
+# def main():
+#     st.title("PDF Annotation App")
+#     uploaded_file = st.file_uploader("Upload a PDF file", type="pdf")
+#     if uploaded_file is not None:
+#         text_to_highlight = st.text_input("Enter text to highlight")
+#         if text_to_highlight:
+#             annotate_pdf(uploaded_file, text_to_highlight)
+
+# if __name__ == "__main__":
+#     main()
+
+
+    # file_path = "../Transformers/Bhagavad-Gita-As-It-Is.pdf"
+    # text_to_highlight =  "Krishna"
+    # annotate_pdf(file_path, text_to_highlight)
+
+# import fitz
+
+# import base64
+
+# def displayPDF(file):
+
+#     # Open the PDF document
+#     doc = fitz.open("my_pdf.pdf")
+
+#     # Get the first page of the document
+#     page = doc.loadPage(4)
+
+#     # Search for the text string to highlight
+#     text_to_highlight = "Supreme Personality of Godhead"
+
+#     # Create a rectangle around the text to highlight
+#     highlight_rect = fitz.Rect(page.searchFor(text_to_highlight)[0])
+
+#     # Create a highlight annotation
+#     highlight_annot = fitz.Annot(page, highlight_rect, "Highlight", {"color": fitz.utils.getColor("yellow")})
+
+#     # Add the annotation to the page
+#     page.addAnnot(highlight_annot)
+
+#     # Save the document
+#     doc.save("my_pdf_highlighted.pdf")
+
+#     # Opening file from file path
+#     with open(file, "rb") as f:
+#         base64_pdf = base64.b64encode(f.read()).decode('utf-8')
+
+#     # Embedding PDF in HTML
+#     pdf_display = f'<iframe src="data:application/pdf;base64,{base64_pdf}" width="700" height="1000" type="application/pdf"></iframe>'
+
+#     # Displaying File
+#     st.markdown(pdf_display, unsafe_allow_html=True)
+
+# displayPDF("../Transformers/Bhagavad-Gita-As-It-Is.pdf")
+
+
+import streamlit as st
+# import fitz
+import tempfile
+
+# Import the PDF_ANNOT_HIGHLIGHT constant
+# from fitz.PDF_ANNOT import PDF_ANNOT_HIGHLIGHT
+
+import base64
+import io
+
+
+def display_highlighted_pdf(file_path, text_to_highlight):
+    # Open the PDF document
+    doc = fitz.open(file_path)
+
+    # Iterate over each page in the PDF
+    for page_index in range(len(doc)):
+        page = doc.load_page(page_index)
+
+        # Search for the text string to highlight
+        areas = page.search_for(text_to_highlight)
+
+        # Create a highlight annotation for each area
+        for area in areas:
+            highlight_rect = fitz.Rect(area)
+            highlight_annot = page.add_highlight_annot(highlight_rect)  #fitz.Annot(page.parent, highlight_rect, annot_type=fitz.PDF_ANNOT_HIGHLIGHT)
+            highlight_annot.set_colors({"stroke": fitz.utils.getColor("yellow")})
+            highlight_annot.update()
+            # page.add_annot(highlight_annot)
+
+    # Create a BytesIO object to hold the highlighted PDF data
+        # Create a temporary file to save the PDF
+    with tempfile.NamedTemporaryFile(delete=False) as temp_file:
+        temp_file_path = temp_file.name
+        doc.save(temp_file_path)
+
+    # Read the content of the temporary file into a BytesIO object
+    with open(temp_file_path, "rb") as f:
+        pdf_bytes = io.BytesIO(f.read())
+
+    # # Remove the temporary file
+    # st.unlink(temp_file_path)
+
+    # pdf_bytes = io.BytesIO()
+    # doc.write(pdf_bytes)
+    # pdf_bytes.seek(0)
+
+    # Encode the PDF data as base64
+    base64_pdf = base64.b64encode(pdf_bytes.getvalue()).decode('utf-8')
+
+    # Embed the PDF in an HTML iframe
+    pdf_display = f'<iframe src="data:application/pdf;base64,{base64_pdf}" height="1600" width="680" type="application/pdf"></iframe>'
+
+    # Display the PDF in Streamlit
+    st.markdown(pdf_display, unsafe_allow_html=True)
+    
 
-    # Load a pre-trained sentence transformer model
-    model_name = "sentence-transformers/all-mpnet-base-v2"
-    model_kwargs = {'device': 'cpu'}
-    encode_kwargs = {'normalize_embeddings': False}
+# Example usage
+file_path = "../Transformers/Bhagavad-Gita-As-It-Is.pdf"
+text_to_highlight = "Supreme Personality of Godhead"
+# display_highlighted_pdf(file_path, text_to_highlight)
 
-    # Create a HuggingFaceEmbeddings object
-    from langchain_community.embeddings import HuggingFaceEmbeddings
-    embeddings = HuggingFaceEmbeddings(model_name=model_name, model_kwargs=model_kwargs, encode_kwargs=encode_kwargs)
 
-    # from pathlib import Path
 
-    # path = Path(filepath)
+# import streamlit as st
 
-    filename = filepath.split(".")
+# def display_pdf(pdf_path):
+#     # Read the PDF file
+#     with open(pdf_path, "rb") as file:
+#         pdf_bytes = file.read()
 
-    print(filename[0])
+#     # Encode the PDF data as base64
+#     base64_pdf = base64.b64encode(pdf_bytes).decode("utf-8")
 
-    filename = filename[0]
+#     # Embed the PDF in an HTML iframe
+#     pdf_display = f'<iframe src="data:application/pdf;base64,{base64_pdf}" width="700" height="1000" type="application/pdf"></iframe>'
 
-    from datetime import datetime
+#     # Display the PDF in Streamlit
+#     st.markdown(pdf_display, unsafe_allow_html=True)
 
-    # Get current date and time
-    now = datetime.now()
+# # Example usage
+# pdf_path = "../Transformers/Bhagavad-Gita-As-It-Is.pdf"
+# display_pdf(pdf_path)
 
-    # Format as string with milliseconds
-    formatted_datetime = now.strftime("%Y-%m-%d_%H:%M:%S.%f")[:-3]
+# import pymupdf
 
-    print(formatted_datetime)
+# # Open the PDF file
+# doc = pymupdf.open(file_path)
 
-    # Create FAISS index with the HuggingFace embeddings
-    faiss_index = FAISS.from_documents(pages, embeddings)
-    with open(f"./{filename}_{formatted_datetime}.pkl", "wb") as f:
-        pickle.dump(faiss_index, f)
+# # Get the first page of the PDF
+# # page = pdf_file.pa
 
+# for page in doc: # iterate the document pages
+#     text_coordinates = page.search_for("Bhagavad", quads=True)
+#     # Highlight the text
+#     page.add_highlight_annot(text_coordinates)
 
-# uploaded_file_name = upload_file()
-# if uploaded_file_name is not None:
-#     create_pickle_file(uploaded_file_name)
+# # Get the text of the page
+# text = page.get_text()
+
+# # Find the text to highlight
+# text_to_highlight = "Bhagavad"
+
+# # Get the coordinates of the text to highlight
+# text_coordinates = page.search_for(text_to_highlight)
+
+# Highlight the text
+# page.add_highlight_annot(text_coordinates)
+
+# Save the PDF file
+# doc.save("example_highlighted.pdf")
+
+import pymupdf
+import tempfile
+import nltk
+from nltk.corpus import stopwords
+from collections import Counter
 
 def highlight_pdf(file_path, text_to_highlight, page_numbers):
     # Create a temporary file to save the modified PDF
@@ -130,14 +431,13 @@ def highlight_pdf(file_path, text_to_highlight, page_numbers):
     # Open the original PDF
     doc = pymupdf.open(file_path)
 
-    pages_to_display = [doc.load_page(page_number) for page_number in page_numbers]
-
-    print("pages_to_display") 
-    print(pages_to_display)
+    pages_to_display = [doc.load_page(page_number - 1) for page_number in page_numbers]
 
     # Tokenize the text into words
     words = text_to_highlight.split()
 
+   
+
     # Remove stopwords
     stop_words = set(stopwords.words("english"))
     words = [word for word in words if word.lower() not in stop_words]
@@ -176,176 +476,326 @@ def highlight_pdf(file_path, text_to_highlight, page_numbers):
     
     print(temp_pdf_path)
 
-    # new_doc.save("example_highlighted.pdf")
+    new_doc.save("example_highlighted.pdf")
 
     return temp_pdf_path
 
-file_path = "Bhagavad-Gita-As-It-Is.pdf"
-text_to_highlight = ""
-sources = []
+# Example usage
 
 # Function to display PDF in Streamlit
-def display_highlighted_pdf(file_path, text_to_highlight, sources):
-    # pdf_path = "../Transformers/Bhagavad-Gita-As-It-Is.pdf"
-    # sources = [7,8]
-    # response_text = "I offer my respectful obeisances unto the lotus feet of my spiritual master and unto the feet of all Vaiñëavas. I offer my respectful"
+def display_highlighted_pdf():
+    pdf_path = "Bhagavad-Gita-As-It-Is.pdf"
+    sources = [7,8]
+    response_text = "I offer my respectful obeisances unto the lotus feet of my spiritual master and unto the feet of all Vaiñëavas. I offer my respectful"
     
-    pdf_path = highlight_pdf(file_path=file_path, text_to_highlight=text_to_highlight, page_numbers=sources)
+    pdf_path = highlight_pdf(file_path=file_path, text_to_highlight=response_text, page_numbers=sources)
 
     with open(pdf_path, "rb") as file:
         pdf_bytes = file.read()
     base64_pdf = base64.b64encode(pdf_bytes).decode("utf-8")
-    pdf_display = f'<iframe is="x-frame-bypass", src="data:application/pdf;base64,{base64_pdf}" width="700" height="1000" type="application/pdf"></iframe>'
+    pdf_display = f'<iframe src="data:application/pdf;base64,{base64_pdf}" width="700" height="1000" type="application/pdf"></iframe>'
     st.markdown(pdf_display, unsafe_allow_html=True)
 
-# Creating a Index(Pinecone Vector Database)
-import os
-# import pinecone
-
-import pickle
-@st.cache_data
-def get_faiss_semantic_index():
-    try:
-        index_path = "./HuggingFaceEmbeddings.pkl"
-        print(index_path)
-        # Load embeddings from the pickle file
-        for _ in stqdm(range(5)):
-            with open(index_path, "rb") as f:
-                faiss_index = pickle.load(f)
-                sleep(0.1)
-        # st.write("Embeddings loaded successfully.")
-        return faiss_index
-    except Exception as e:
-        st.error(f"Error loading embeddings: {e}")
-        return None
-faiss_index = get_faiss_semantic_index()
-print(faiss_index)
-
-# def promt_engineer(text):
-PROMPT_TEMPLATE = """
-Instructions:
------------------------------------------------------------------------------------------
-Answer the question  only based on the below context:
-- You're a Vedic AI expert in the Hindu Vedic scriptures.
-- Always try to provide Keep it simple answers in nice format without incomplete sentence.
-- Give the answer atleast 5 seperate lines addition to the title info.
-- provide Title: <title> Chapter: <chapter> Text No: <textnumber> Page No: <pagenumber> 
-------------------------------------------------------------------------------------------
-
-{context}
-
-----------------------------------------------------------------------------------
-
-Answer the question based on the above context: {question}
-"""
-    # # Load the summarization pipeline with the specified model
-    # summarizer = pipeline("summarization", model="facebook/bart-large-cnn")
-
-    # # Generate the prompt
-    # prompt = prompt_template.format(text=text)
-
-    # # Generate the summary
-    # summary = summarizer(prompt, max_length=1024, min_length=50)[0]["summary_text"]
-    
-    # with st.sidebar:
-    #     st.divider()
-    #     st.markdown("*:red[Text Summary Generation]* from above Top 5 **:green[similarity search results]**.")
-    #     st.write(summary)
-    #     st.divider()
-
-def chat_actions():
-
-    st.session_state["chat_history"].append(
-        {"role": "user", "content": st.session_state["chat_input"]},
-    )
-
-    # query_embedding = model.encode(st.session_state["chat_input"])
-    query = st.session_state["chat_input"]
-    if faiss_index is not None:
-        docs = faiss_index.similarity_search(query, k=6)
+
+display_highlighted_pdf()
+
+
+# import streamlit as st
+# import streamlit.components.v1 as components
+# path_to_html = "https://vedabase.io/en/library/bg/1/1/" 
+
+# # with open(path_to_html,'r') as f: 
+# #     html_data = f.read()
+
+# # # Show in webpage
+# # st.header("Show an external HTML")
+# # st.components.v1.html(html_data)
+
+# try:
+#     with open(path_to_html, 'r') as f:
+#         # Your file processing code here
+#         html_data = f.read()
+#         st.header("Show an external HTML")
+#         st.components.v1.html(html_data)
+# except FileNotFoundError:
+#     print("File not found. Please check the file path.")
+# except Exception as e:
+#     print(f"An error occurred: {e}")
+
+
+import streamlit as st
+import requests
+from bs4 import BeautifulSoup
+
+def fetch_html(url):
+    # Fetch the webpage content
+    response = requests.get(url)
+    if response.status_code == 200:
+        return response.content
     else:
-        st.error("Failed to load embeddings.")
-    # docs = faiss_index.similarity_search(query, k=2)
+        st.error(f"Failed to fetch webpage. Status code: {response.status_code}")
+
+def scrape_data(html_content):
+    # Parse HTML content
+    soup = BeautifulSoup(html_content, "html.parser")
 
-    for doc in docs:
-        print("\n")
-        print(str(doc.metadata["page"]+1) + ":", doc.page_content)
-    context_text = "\n\n---\n\n".join([doc.page_content for doc in docs])
+    # Scrape data (replace this with your specific scraping logic)
+    data = soup.find_all(class_="container first-container")
 
-    sources = [doc.metadata.get("page", None) for doc in docs]
+    return data
+
+def main(url):
+    st.title("Webpage Scraper")
+
+    # User input for webpage URL
+    url = st.text_input("Enter the URL of the webpage:", value=url)
     
+    # Convert webpage to HTML and scrape data
+    if st.button("Scrape Data"):
+        if url:
+            html_content = fetch_html(url)
+            str_content = """ """
+            if html_content:
+                data = scrape_data(html_content)
+                # st.title("HTML Page Display")
+                # st.components.v1.html(html_content, height=1600, width=800)
+                # Display scraped data in a new tab
+                with st.expander("Scraped Data", expanded=True):
+                    for item in data:
+                        # Convert item to string and display
+                        str_content += str(item)
+                        # st.write(str(item))
+                        # st.title("HTML Page Display")
+                        # st.components.v1.html(data, height=1600, width=800)
+                    # st.title("HTML Page Display")
+                    st.components.v1.html(str_content, height=1600, width=680)
+# main()
+
+
+# import streamlit as st
+
+# html_content = """
+# <div class="container first-container"> <div class="row"> <div class="col-12 breadcrumb"> <a href="/en/library/">Library</a> »
+# <a href="/en/library/bg/">Bhagavad-gītā As It Is</a> » 
+
+#             <a href="/en/library/bg/1/">Chapter One</a>
+
+# </div> </div> <div class="row" id="content" tabindex="-1"> <div class="col-12"> <div class="r r-title r-verse" id="bb181"> <h1>Bg. 1.1</h1> </div> <div class="wrapper-devanagari"> <h2 class="section-title none">Devanagari</h2> <div class="r r-devanagari" id="bb567886">धृतराष्ट्र उवाच<br/>धर्मक्षेत्रे कुरुक्षेत्रे समवेता युयुत्सव: ।<br/>मामका: पाण्डवाश्चैव किमकुर्वत सञ्जय ॥ १ ॥</div> </div> <div class="wrapper-verse-text"> <h2 class="section-title none">Text</h2> <div class="r r-lang-en r-verse-text" id="bb183"><em><em>dhṛtarāṣṭra uvāca</em><br/>dharma-kṣetre kuru-kṣetre<br/>samavetā yuyutsavaḥ<br/>māmakāḥ pāṇḍavāś caiva<br/>kim akurvata sañjaya</em></div> </div> <div class="wrapper-synonyms"> <h2 class="section-title">Synonyms</h2> <div class="r r-lang-en r-synonyms" id="bb184"><p><a href="/en/search/synonyms/?original=dhṛtarāṣṭraḥ"><em>dhṛtarāṣṭraḥ</em></a> <a href="/en/search/synonyms/?original=uvāca"><em>uvāca</em></a> — King Dhṛtarāṣṭra said; <a href="/en/search/synonyms/?original=dharma"><em>dharma</em></a>-<a href="/en/search/synonyms/?original=kṣetre"><em>kṣetre</em></a> — in the place of pilgrimage; <a href="/en/search/synonyms/?original=kuru"><em>kuru</em></a>-<a href="/en/search/synonyms/?original=kṣetre"><em>kṣetre</em></a> — in the place named Kurukṣetra; <a href="/en/search/synonyms/?original=samavetāḥ"><em>samavetāḥ</em></a> — assembled; <a href="/en/search/synonyms/?original=yuyutsavaḥ"><em>yuyutsavaḥ</em></a> — desiring to fight; <a href="/en/search/synonyms/?original=māmakāḥ"><em>māmakāḥ</em></a> — my party (sons); <a href="/en/search/synonyms/?original=pāṇḍavāḥ"><em>pāṇḍavāḥ</em></a> — the sons of Pāṇḍu; <a href="/en/search/synonyms/?original=ca"><em>ca</em></a> — and; <a href="/en/search/synonyms/?original=eva"><em>eva</em></a> — certainly; <a href="/en/search/synonyms/?original=kim"><em>kim</em></a> — what; <a href="/en/search/synonyms/?original=akurvata"><em>akurvata</em></a> — did they do; <a href="/en/search/synonyms/?original=sañjaya"><em>sañjaya</em></a> — O Sañjaya.</p></div> </div> <div class="wrapper-translation"> <h2 class="section-title">Translation</h2> <div class="r r-lang-en r-translation" id="bb185"><p><strong>Dhṛtarāṣṭra said: O Sañjaya, after my sons and the sons of Pāṇḍu assembled in the place of pilgrimage at Kurukṣetra, desiring to fight, what did they do?</strong></p></div> </div> <div class="wrapper-puport"> <h2 class="section-title">Purport</h2> <div class="r r-lang-en r-paragraph" id="bb186"><p><em><a href="/en/library/bg/">Bhagavad-gītā</a></em> is the widely read theistic science summarized in the <em>Gītā-māhātmya</em> (<em>Glorification of the Gītā</em>). There it says that one should read <em><a href="/en/library/bg/">Bhagavad-gītā</a></em> very scrutinizingly with the help of a person who is a devotee of Śrī Kṛṣṇa and try to understand it without personally motivated interpretations. The example of clear understanding is there in the <em><a href="/en/library/bg/">Bhagavad-gītā</a></em> itself, in the way the teaching is understood by Arjuna, who heard the <em>Gītā</em> directly from the Lord. If someone is fortunate enough to understand the <em><a href="/en/library/bg/">Bhagavad-gītā</a></em> in that line of disciplic succession, without motivated interpretation, then he surpasses all studies of Vedic wisdom, and all scriptures of the world. One will find in the <em><a href="/en/library/bg/">Bhagavad-gītā</a></em> all that is contained in other scriptures, but the reader will also find things which are not to be found elsewhere. That is the specific standard of the <em>Gītā.</em> It is the perfect theistic science because it is directly spoken by the Supreme Personality of Godhead, Lord Śrī Kṛṣṇa.</p></div> <div class="r r-lang-en r-paragraph" id="bb187"><p>The topics discussed by Dhṛtarāṣṭra and Sañjaya, as described in the <em>Mahābhārata,</em> form the basic principle for this great philosophy. It is understood that this philosophy evolved on the Battlefield of Kurukṣetra, which is a sacred place of pilgrimage from the immemorial time of the Vedic age. It was spoken by the Lord when He was present personally on this planet for the guidance of mankind.</p></div> <div class="r r-lang-en r-paragraph" id="bb188"><p>The word <em>dharma-kṣetra</em> (a place where religious rituals are performed) is significant because, on the Battlefield of Kurukṣetra, the Supreme Personality of Godhead was present on the side of Arjuna. Dhṛtarāṣṭra, the father of the Kurus, was highly doubtful about the possibility of his sons’ ultimate victory. In his doubt, he inquired from his secretary Sañjaya, “What did they do?” He was confident that both his sons and the sons of his younger brother Pāṇḍu were assembled in that Field of Kurukṣetra for a determined engagement of the war. Still, his inquiry is significant. He did not want a compromise between the cousins and brothers, and he wanted to be sure of the fate of his sons on the battlefield. Because the battle was arranged to be fought at Kurukṣetra, which is mentioned elsewhere in the <em>Vedas</em> as a place of worship – even for the denizens of heaven – Dhṛtarāṣṭra became very fearful about the influence of the holy place on the outcome of the battle. He knew very well that this would influence Arjuna and the sons of Pāṇḍu favorably, because by nature they were all virtuous. Sañjaya was a student of Vyāsa, and therefore, by the mercy of Vyāsa, Sañjaya was able to envision the Battlefield of Kurukṣetra even while he was in the room of Dhṛtarāṣṭra. And so, Dhṛtarāṣṭra asked him about the situation on the battlefield.</p></div> <div class="r r-lang-en r-paragraph" id="bb189"><p>Both the Pāṇḍavas and the sons of Dhṛtarāṣṭra belong to the same family, but Dhṛtarāṣṭra’s mind is disclosed herein. He deliberately claimed only his sons as Kurus, and he separated the sons of Pāṇḍu from the family heritage. One can thus understand the specific position of Dhṛtarāṣṭra in his relationship with his nephews, the sons of Pāṇḍu. As in the paddy field the unnecessary plants are taken out, so it is expected from the very beginning of these topics that in the religious field of Kurukṣetra, where the father of religion, Śrī Kṛṣṇa, was present, the unwanted plants like Dhṛtarāṣṭra’s son Duryodhana and others would be wiped out and the thoroughly religious persons, headed by Yudhiṣṭhira, would be established by the Lord. This is the significance of the words <em>dharma-kṣetre</em> and <em>kuru-kṣetre,</em> apart from their historical and Vedic importance.</p></div> </div> </div> </div> <div class="row d-print-none"> <div class="col-12"> <ul class="mini-pager mt-2 pb-4"> <li class="pager-prev"><a class="btn" href="/en/library/bg/1/"> <i class="fa fa-chevron-left"></i>
+# Previous
+
+#                 </a></li>
+
+# <li class="pager-next"><a class="btn" href="/en/library/bg/1/2/">
+# Next
+
+#                     <i class="fa fa-chevron-right"></i>
+
+# </a></li>
+
+# </ul> </div> </div> <nav class="rich-menu" id="para-menu"> <div class="Panel" data-csrf-token="l7dMxBge1IaZDbFchwWzWmh1CBpo6pWDY9LKjwSlqmvpDKld3RfTLY85AWyycbUS" data-language="en" data-propose-category-url="/categorization/suggest-category/" data-reload-url="/en/paragraph-tool/?page_id=14054&amp;view_slug=index_view&amp;view_args=&amp;next=/en/library/bg/1/1/&amp;page_url=/en/library/bg/1/1/" id="panel"> <h1 class="pt-title">Paragraph Tools <a class="pt-close-button" href="#" onclick="event.preventDefault();_menu.close();return false;">Close <i class="fa fa-times"></i></a></h1> Please select paragraph first. </div> <div class="Panel" id="subpanel"> <p>Suggest this as category.</p> <a class="link-yes btn btn-lg btn-success px-3" href="">Yes</a> <a class="link-cancel btn btn-lg btn-success px-3" href="">Cancel</a> </div> </nav> <div class="paragraph-tool-button no-touch none"> <div class="paragraph-tool-button-nav"> <div class="paragraph-tool-button-trigger" data-target="#para-menu" id="para-menu-button"> <i class="icon fa fa-briefcase fa-2x"></i> <span class="fa-stack fa"> <i class="fa fa-circle fa-stack-1x p-counter"></i> <span class="p-counter fa-stack-1x fa-stack-text file-text fa-inverse"></span> </span> </div> </div> </div> </div>
+# """
+
+# st.title("HTML Page Display")
+# st.components.v1.html(html_content, height=1600, width=800)
+
+
+
+import requests
+from bs4 import BeautifulSoup
+
+# URL of the webpage
+baseurl = "https://vedabase.io/en/library/bg/"
+
+# Fetch the webpage content
+response = requests.get(baseurl)
+if response.status_code == 200:
+    html_content = response.content
 
-    prompt_template = ChatPromptTemplate.from_template(PROMPT_TEMPLATE)
-    prompt = prompt_template.format(context=context_text, question=query)
-    response_text = ""
-    result = ""
-    try:
-        llm = HuggingFaceHub(
-            repo_id="meta-llama/Meta-Llama-3-8B-Instruct", model_kwargs={"temperature": 0.1, "max_new_tokens": 256, "task":"text-generation"}
-        )
-        response_text = llm.invoke(prompt)
-        escaped_query = re.escape(query)
-        result = re.split(f'Answer the question based on the above context: {escaped_query}\n',response_text)[-1]
-        st.write(result)
-    except Exception as e:
-        st.error(f"Error invoke: {e}")
-
-
-    formatted_response = f"Response: {result}\nSources: {sources}"
-    print(formatted_response)
-
-    st.session_state["chat_history"].append(
-        {
-            "role": "assistant",
-            "content": f"{result}",
-        },  # This can be replaced with your chat response logic
-    )
-        # break;
-    # Example usage
-    file_path = "Bhagavad-Gita-As-It-Is.pdf"
-    text_to_highlight = context_text.strip()
-    display_highlighted_pdf(file_path, result, sources)
-
-with st.sidebar:
-    option = st.selectbox(
-    "Select Your Favorite Scriptures",
-    ("Bhagvatgeetha", "Bhagavatham", "Ramayanam"),
-    # index=None,
-    # placeholder="Select scriptures...",
-    key="value",
-    on_change=display_title
-    )
-
-    st.write("You selected:", option)
-
-    faq = st.selectbox(
-    "Select Your Favorite Scriptures",
-    ("what is jeeva and paramathma?",
-     "What are the Krishna told to Arjuna?",
-     "What are the key points from Krishna?",
-     "Why does atheism exist even when all questions are answered in Bhagavad Gita?", 
-     "Why don’t all souls surrender to Lord Krishna, although he has demonstrated that everyone is part and parcel of Him, and all can be liberated from all sufferings by surrendering to Him?",
-     "Why do souls misuse their independence by rebelling against Lord Krishna?",
-     "How do I put an end to my suffering in this world?",
-     "what is the reason behind Krishna decided to go far war?"),
-    # index=None,
-    # placeholder="Select scriptures...",
-    key="faq",
-    on_change=open_chat
-    )
-    st.write("You selected:", faq)
+    # Parse HTML content
+    soup = BeautifulSoup(html_content, "html.parser")
+
+    # Find all direct child div elements with class="r-chapter"
+    direct_child_div_elements = soup.select("div.col-12 > div.r-chapter")
+
+    # List to store the extracted text
+    output = []
+
+    # Iterate over each direct child div element
+    for div in direct_child_div_elements:
+        # Find the <a> tag within the div
+        a_tag = div.find("a")
+        if a_tag:
+            # Extract the text from the <a> tag and append it to the output list
+            output.append(a_tag.text.strip())
+
+    # Print the output list
+    # print(output)
+    # print(len(output))
+
+### Link to all chapters
+import re
+
+# Sample text
+chapter = output[4]
+
+text_to_number = {
+    "One": "1",
+    "Two": "2",
+    "Three": "3",
+    "Four": "4",
+    "Five": "5",
+    "Six": "6",
+    "Seven": "7",
+    "Eight": "8",
+    "Nine": "9",
+    "Ten": "10",
+    # Add more numbers if needed
+}
+
+# Split the text by spaces
+words = chapter.split()
+
+# Find the text representing the number
+number_text = words[1].strip(":")  # Assuming the number text is the second word
+
+# Extract the numeric part
+chapter_number = text_to_number[number_text]
+
+# Print the chapter number
+# print(chapter_number)
+
+url = baseurl + chapter_number
+
+# print(url)
+
+### all Texts in each chapter
+
+response = requests.get(url)
+if response.status_code == 200:
+    html_content = response.content
+
+    # Parse HTML content
+    text = BeautifulSoup(html_content, "html.parser")
+
+    # print(text)
+
+    # Find all direct child div elements with class="r-chapter"
+    direct_child_div_elements = text.select("div.col-12 > dl.r.r-verse")
+
+    # print(direct_child_div_elements)
+
+    # List to store the extracted text
+    text_number = []
+
+    # Iterate over each direct child div element
+    for div in direct_child_div_elements:
+        # Find the <a> tag within the div
+        a_tag = div.find("a")
+        if a_tag:
+            # Extract the text from the <a> tag and append it to the output list
+            text_number.append(a_tag.text.strip())
+
+    # Print the output list
+    # print(text_number)
+    # print(len(text_number))
+
+### link to each Text in each chapter
+text_page = text_number[0]
+
+# Split the text by spaces
+words = text_page.split()
+
+# Find the text representing the number
+text_number = words[1].strip(":")  # Assuming the number text is the second word
+
+# print(f"chapter_number - {chapter_number} : text_number - {text_number}")
+texturl = baseurl + chapter_number + "/" + text_number
+
+# print(texturl)
+
+main(url=texturl)
+
+st.title("Display HTML File in Streamlit")
+
+# Path to the HTML file
+html_file_path = "../Transformers/Bg. 1.1.html"
+
+try:
+    # Read the HTML file
+    with open(html_file_path, "r", encoding="utf-8") as file:
+        html_content = file.read()
     
+    # Display the HTML content using the 'st.components.v1.html' component
+    expanded = st.checkbox("Expand HTML page")
+    if expanded:
+        st.components.v1.html(html_content, height=1600, width=680)
+    else:
+        st.components.v1.html(html_content, height=600)
+except FileNotFoundError:
+    st.error(f"HTML file '{html_file_path}' not found!")
+
+import streamlit as st
+
 
-    if "chat_history" not in st.session_state:
-        st.session_state["chat_history"] = []
+st.title("Streamlit Tabs Example")
 
-    st.chat_input(question, on_submit=chat_actions, key="chat_input")
+# Add tabs to the sidebar
+tabs = st.sidebar.radio("Navigation", ["Home", "About", "Settings"])
 
-    for i in st.session_state["chat_history"]:
-        with st.chat_message(name=i["role"]):
-            st.write(i["content"])
+# Display different content based on the selected tab
+if tabs == "Home":
+    st.header("Home Page")
+    st.write("Welcome to the Home page!")
 
+elif tabs == "About":
+    st.header("About Page")
+    st.write("This is the About page.")
 
+elif tabs == "Settings":
+    st.header("Settings Page")
+    st.write("Here you can configure your settings.")
 
+st.title("Netflix-like Grid Display")
+
+
+
+import streamlit as st
+import os
+import streamlit.components.v1 as components
+
+# Define movie data
+movies = [
+    {"title": "Movie 1", "poster_path": "../Transformers/Bg. 1.1.html"},
+    {"title": "Movie 2", "poster_url": "https://via.placeholder.com/150"},
+    {"title": "Movie 3", "poster_url": "https://via.placeholder.com/150"},
+    {"title": "Movie 4", "poster_url": "https://via.placeholder.com/150"},
+    {"title": "Movie 5", "poster_url": "https://via.placeholder.com/150"},
+    {"title": "Movie 6", "poster_url": "https://via.placeholder.com/150"},
+    {"title": "Movie 7", "poster_url": "https://via.placeholder.com/150"},
+    {"title": "Movie 8", "poster_url": "https://via.placeholder.com/150"},
+]
+
+# Display movies in a grid
+# num_columns = 4
+# col_count = 0
+# cols = st.columns(num_columns)
+# expanded = st.checkbox("Expand HTML page")
+# for movie in movies:
+#     with cols[col_count % num_columns]:
+#         st.markdown(f"<h2>{movie['title']}</h2>", unsafe_allow_html=True)
+#         st.write(f"Placeholder for {movie['title']}")
+#         if 'poster_path' in movie:
+#             # Convert local file path to URL
+#             poster_url = f"file://{os.path.abspath(movie['poster_path'])}"
+#             print(poster_url)
+#             # Display the HTML page using IFrame
+#             if expanded:
+#                 components.iframe(poster_url, width=800, height=600)
+#             else:
+#                 components.iframe(poster_url,width=200)
+#         else:
+#             # Display placeholder image
+#             st.image(movie["poster_url"], width=200)
+#     col_count += 1