feat: interface and model pipline

.gitignore

@@ -1,3 +1,5 @@
+.streamlit/secrets.toml
 .venv
 .idea
 __pycache__
+dev_clear.py

app.py

@@ -0,0 +1,46 @@
+import streamlit as st
+from model import DrakeLM
+from utilis import Processing
+
+initial_page = "pages/upload_url.py"
+
+
+@st.cache_resource()
+def initialize_models():
+    processing = Processing(
+        dataset_path=st.secrets["DEEPLAKE_DB_URL"],
+        embedding_model_name="mixedbread-ai/mxbai-embed-large-v1",
+        chunk_size=1300,
+    )
+    config = {"max_new_tokens": 4096, "context_length": 8192, "temperature": 0.3}
+    drake = DrakeLM(
+        model_path="Mistral/mistral-7b-instruct-v0.2.Q5_K_S.gguf",
+        config=config,
+        db=processing.db,
+    )
+
+    return processing, drake
+
+
+def disable_sidebar():
+    st.set_page_config(page_title="Upload",
+                       page_icon=None,
+                       layout="centered",
+                       initial_sidebar_state="collapsed",
+                       )
+    no_sidebar_style = """
+        <style>
+            div[data-testid="stSidebarNav"] {display: none;}
+        </style>
+    """
+    st.markdown(no_sidebar_style, unsafe_allow_html=True)
+
+
+def main():
+    disable_sidebar()
+    initialize_models()
+    st.switch_page(initial_page)
+
+
+if __name__ == "__main__":
+    main()

examples/chat_examples/example_1.json

@@ -0,0 +1,6 @@
+{
+  "query": "Assess DBSCAN, Dense cells and CLIQUE with appropriate steps. (8 marks)",
+  "context": "DBSACN\nStep 1: Randomly select a point p. Compute distance between P and ail other points '\nStep 2: Find all points]from p with respect to its neighbourhoud and check whether it has minimum number of points m. If 80, it is marked as a core point\nStep 3: If it is a core point, then a new cluster is formed, or existing cluster 1s enlarged.\nStep 4: [fit is a border point, then the algorithm moves to the next point and marks it as visited\nStep 5: If it is a noise point, they are removed.\nStep 6: Merge the clusters if it is mergeable, dist (cc )< ¢.\nStep 7: Repeat the process 3-6 till all Points are processed. \n\nDense Cell\nStep 1: Defining a set of grid points and assigning the given data points on the grid.\nStep 2: Determine the dense and sparse cells. lf the number of points in a cell exceeds the threshold\nvalue t, the cell is categorized as a dense cell. Sparse cells are removed from the list.\nStep 3: Merge the dense cells if they are adjacent.\nStep 4: Form a list of grid cells for every subspace as output.\n\nCLIQUE\nStage 1\nStep 1: Identify the dense cells\nStep 2: Merge dense cells c. and c, if they share the same interval.\nStep 3: Generate Apriori rule to generate (k + 1)\" cell tor higher dimension. Then, check\nwhether the number of points across the threshold This 1s repeated till there are no\ndense cells or a new generation of dense cells\n\nStage 2\nStep 1: Merging of dense cells into a cluster is carried out in each subspace using maximal regions to cover dense cells The maximal region is a hyperrectangle where all cells fall into.\nStep 2; Maximal region tries to cover all dense cells to form clusters.",
+  "rules": "- If the question says answer for X number of marks, you have to provide X number of points.\n - Each point has to be explained in 3-4 sentences.\n - In case the context express a mathematical equation, provide the equation in LaTeX format as shown in the example.\n - In case the user requests for a code snippet, provide the code snippet in the language specified in the example.- If the user requests to summarise or use the previous message as context ignoring the explicit context given in the message.",
+  "answer": "# Dense Cells\nStep 1: Define a set of grid points and assign the given data points on the grid.\nStep 2: Determine the dense and sparse cells. If the number of points in a cell exceeds the threshold value t, the cell is categorized as dense cell. Sparse cells are removed from the list.\nStep 3: Merge the dense cells if they are adjacent.\nStep 4: Form a list of grid cells for every subspace as output.\n\n#CILQUE\n **Stage 1:**\n- Step 1: Identify the dense cells.\n- Step 2: Merge dense cells c₁ and c₂ if they share the same interval.\n- Step 3: Generate a particle rule to generate (k + 1)th cell for higher dimension. Then, check whether the number of points cross the threshold. This is repeated till there are no dense cells or new generation of dense cells.\n\n**Stage 2:**\n- Step 1: Merging of dense cells into a cluster is carried out in each subspace using maximal regions to cover dense cells. The maximal region is an hyperrectangle where all cells fall into.\n- Step 2: Maximal region tries to cover all dense cells to form clusters.\n\n # DBSCAN\n- Step 1: Randomly select a point p. Compute distance between p and all other points.\n- Step 2: Find all points from p with respect to its neighborhood and check whether it has minimum number of points m. If so, it is marked as a core point.\n- Step 3: If it is a core point, then a new cluster is formed, or existing cluster is enlarged.\n- Step 4: If it is a border point, then the algorithm moves to the next point and marks it as visited.\n- Step 5: If it is a noise point, they are removed.\n- Step 6: Merge the clusters if it is mergeable, dist (c, c) < ɛ.\n- Step 7: Repeat the process 3-6 till all points are processed."
+}

examples/chat_examples/example_2.json

@@ -0,0 +1,6 @@
+{
+  "query": "Explain sharding in system design along with its benefits. (10 marks)",
+  "context": "It is a very important concept that helps the system to keep data in different resources\naccording to the sharding process. The word “Shard” means “a small part of a\nwhole“. Hence Sharding means dividing a larger part into smaller parts. In DBMS,\nSharding is a type of DataBase partitioning in which a large database is divided or\n\npartitioned into smaller data and different nodes. These shards are not only smaller,\nbut also faster and hence easily manageable.\nHow does Sharding work?\nIn a sharded system, the data is partitioned into shards based on a predetermined\ncriterion. For example, a sharding scheme may divide the data based on geographic\nlocation, user ID, or time period. Once the data is partitioned, it is distributed across\nmultiple servers or nodes. Each server or node is responsible for storing and processing a\nsubset of the data.\nExample:\n\nTo query data from a sharded database, the system needs to know which shard contains\nthe required data. This is achieved using a shard key, which is a unique identifier that is\nused to map the data to its corresponding shard. When a query is received, the system\nuses the shard key to determine which shard contains the required data and then sends\nthe query to the appropriate server or node.\nFeatures of Sharding:\n\uF0B7 Sharding makes the Database smaller\n\uF0B7 Sharding makes the Database faster\n\uF0B7 Sharding makes the Database much more easily manageable\n\uF0B7 Sharding can be a complex operation sometimes\n\uF0B7 Sharding reduces the transaction cost of the Database\n\uF0B7 Each shard reads and writes its own data.\n\uF0B7 Many NoSQL databases offer auto-sharding.\n\uF0B7 Failure of one shard doesn’t effect the data processing of other shards.\nBenefits of Sharding:\n1. Improved Scalability: Sharding allows the system to scale horizontally by adding more\nservers or nodes as the data grows. This improves the system’s capacity to handle\nlarge volumes of data and requests.\n\n2. Increased Performance: Sharding distributes the data across multiple servers or\nnodes, which improves the system’s performance by reducing the load on each server\nor node. This results in faster response times and better throughput.\n3. Fault Tolerance: Sharding provides a degree of fault tolerance as the system can\ncontinue to function even if one or more servers or nodes fail. This is because the data\nis replicated across multiple servers or nodes, and if one fails, the others can continue\nto serve the requests.\n4. Reduced Costs: Sharding allows the system to scale horizontally, which can be more\ncost-effective than scaling vertically by upgrading hardware. This is because horizontal\nscaling can be done",
+  "rules": "- If the question says answer for X number of marks, you have to provide X number of points.\n - Each point has to be explained in 3-4 sentences.\n - In case the context express a mathematical equation, provide the equation in LaTeX format as shown in the example.\n - In case the user requests for a code snippet, provide the code snippet in the language specified in the example.- If the user requests to summarise or use the previous message as context ignoring the explicit context given in the message.\n",
+  "answer": "Sharding is a technique for dividing a large database into smaller, manageable parts called shards, which are stored across multiple servers or nodes. This process enhances scalability, performance, and fault tolerance by distributing data and processing load. Sharding works by partitioning data based on criteria like geographic location, user ID, or time period, and each shard is responsible for a subset of the data. This method allows for horizontal scaling, improving the system's capacity to handle large volumes of data and requests efficiently.\n\nThe system uses a shard key to identify which shard contains the required data for a query. The shard key is a unique identifier that maps data to its corresponding shard. Upon receiving a query, the system determines the appropriate shard and forwards the query to the correct server or node.\n\n**Features of Sharding:**\n- Sharding makes the database smaller, faster, and more manageable.\n- It can be complex to implement.\n- Sharding reduces transaction costs and allows each shard to read and write its own data.\n- Many NoSQL databases offer auto-sharding.\n- Failure of one shard does not affect the data processing of other shards.\n\n**Benefits of Sharding:**\n1. **Improved Scalability:** Sharding allows horizontal scaling by adding more servers or nodes, enhancing the system's capacity to handle large volumes of data and requests.\n2. **Increased Performance:**By distributing data across multiple servers or nodes, sharding improves performance, resulting in faster response times and better throughput.\n3. **Fault Tolerance:** Sharding provides fault tolerance as the system can continue to function even if one or more servers or nodes fail, thanks to data replication across multiple servers or nodes.\n4. **Reduced Costs:** Horizontal scaling with sharding can be more cost-effective than vertical scaling by upgrading hardware, as it can be done using commodity hardware, which is typically less expensive than high-end servers."
+}

examples/notes_examples/example_1.json

@@ -0,0 +1,5 @@
+{
+  "content_chunk": "The cyclic redundancy check codes are popularly employed in LANs and WANs for error correction. The principle of operation of CRC encoders and decoders can be better explained with the following examples. CRC is the most powerful and easy to implement technique.CRC is based on binary division. In CRC, a sequence of redundant bits, are appended to the end of data unit so that the resulting data unit becomes exactly divisible by a second, predetermined binary number. At the destination, the incoming data unit is divided by the same number. If at this step there is no remainder, the data unit is assumed to be correct and is therefore accepted. A remainder indicates that the data unit has been damaged in transit and therefore must be rejected. The binary number, which is (r+1) bit in length, can also be considered as the coefficients of a polynomial, called Generator Polynomial. PERFORMANCE OF CRC CRC is a very effective error detection technique. If the divisor is chosen according to the previously mentioned rules, its performance can be summarized as follows. CRC can detect all single-bit errors and double bit errors (three 1’s). CRC can detect any odd number of errors (X+1) and it can also detect all burst errors of less than the degree of the polynomial. 1. The Sender follows the given steps:   2.   The block unit is divided into k sections, and each of n bits.   3.   4.   5.   All the k sections are added together by using one's complement to get the sum.   The sum is complemented and it becomes the checksum field.   The original data and checksum field are sent across the network.   Checksum Checker A Checksum is verified at the receiving side. The receiver subdivides the incoming data into equal segments of n bits each, and all these segments are added together, and then this sum is complemented. If the complement of the sum is zero, then the data is accepted otherwise data is rejected. 1. The Receiver follows the given steps:   2.   The block unit is divided into k sections and each of n bits.   3.   4.   5.   All the k sections are added together by using one's complement algorithm to get the sum.   The sum is complemented.   If the result of the sum is zero, then the data is accepted otherwise the data is discarded.   Cyclic Redundancy Check (CRC) CRC is a redundancy error technique used to determine the error. Following are the steps used in CRC for error detection: - In CRC technique, a string of n 0s is appended to the data unit, and this n number is less than the number of bits in a predetermined number, known as division which is n+1 bits. - Secondly, the newly extended data is divided by a divisor using a process is known as binary division. The remainder generated from this division is known as CRC remainder. - Thirdly, the CRC remainder replaces the appended 0s at the end of the original data. This newly generated unit is sent to the receiver. - The receiver receives the data followed by the CRC remainder. The receiver will treat this whole unit as a single unit, and it is divided by the same divisor that was used to find the CRC remainder. If the resultant of this division is zero which means that it has no error, and the data is accepted. If the resultant of this division is not zero which means that the data consists of an error. Therefore, the data is discarded.",
+  "rules": "- Follow the Markdown format for creating notes as shown in the example. \n - The heading of the content should be the title of the markdown file. \n - Create subheadings for each section. \n - Use numbered bullet points for each point.",
+  "notes": "# CRC (Cyclic Redundancy Check) Encoder and Decoder \n- They are popularly employed in LANs and WANs for error correction. \n- Based on binary division - a sequence of redundant bits are appended to the end of the data unit so that the resulting data unit becomes exactly divisible by a predetermined binary number.\n- At destination, if there is no remainder, it is assumed to be correct. A remainder indicates data unit is damaged and therefore rejected. \n- The binary number (r+1 bits by length) can be considered coefficients of a polynomial, called Generator Polynomial. \n- It is very effective, since it can detect all single-bit errors and double bit errors, odd number of errors and also burst errors of lesser degree than the polynomial. \n- It consists of two components, a generator and a checker.\n  1. **CRC Generator**: uses modulo 2 division. n 0s are appended to the end if divisor contains n+1 units. The remainder generated is called CRC remainder which replaces the appended string of 0s with itself and the final string is sent across the network.\n  2. **CRC Checker:** performs modulo 2 division to the number received from generator by the same divisor, and data is accepted if remainder is zero. \n\n- **CRC Algorithm:**\n  1. Append a string of n 0s to the data unit, where n < the predetermined number of bits in the divisor (n+1 bits).\n  2. Perform binary division between the extended data and the divisor to obtain the CRC remainder.\n  3. Replace the appended 0s at the end of the original data with the CRC remainder.\n  4. Transmit the newly generated unit to the receiver.\n  5. Upon reception, treat the data and CRC remainder as a single unit and divide it by the same divisor used earlier to determine the CRC remainder."
+}

examples/notes_examples/example_2.json

@@ -0,0 +1,5 @@
+{
+  "content_chunk": "Meaning/Defination: A content delivery network (CDN) is a group of geographically\ndistributed servers that speed up the delivery of web content by bringing it closer to where\nusers are.\n It is a network of strategically positioned servers aimed at enhancing the speed and\ndependability of delivering content to users in different locations. These servers store cached\ncopies of content, ranging from web pages to videos, guaranteeing that when a user makes\na request, it’s served from the closest server available. This reduces delays and speeds up\nloading times.\nWhen a user requests specific content, CDN architecture comes into play. It directs the\nrequest to the nearest server, taking into account factors like server health and proximity.\nThis approach minimizes data travel distance, resulting in fast and efficient content delivery.\nAnalogy: You could think of a CDN like an ATM. If your money were only available from one\nbank in town, you’d have to make a time-consuming trip and stand in a long line every time\nyou wanted to withdraw cash. However, with a cash machine on practically every corner,\nyou have fast and easy access to your money any time you need it.\n\nWhat is The Use of a Content Distribution Network?\nCDNs are designed to optimize the delivery of web content, and some of the main\nadvantages that they provide to a company and its users include:\n\uF0B7 Faster Load Times: CDNs cache content geographically close to its users,\ndecreasing the distance that requests and responses need to travel.  As a result,\nusers experience faster load times for webpages, which can increase conversions\nand decrease bounce rates. How does a CDN improve page load time?: As\nmentioned earlier, it is a globally distributed network of servers that store (commonly\nreferred to as "cache") and deliver some or all of your website's content. Each of\nthese servers in the CDN's network is called a Point of Presence (PoP) or an edge\nserver.\n\uF0B7 Reduced Bandwidth Costs: Serving all requested content from the origin server\nrequires significant bandwidth and processing power at the origin. CDNs reduce load\nand bandwidth requirements at the bandwidth by caching static content and\nperforming other optimizations. Of course, this helps to greatly reduce costs.\n\uF0B7 Improved Availability and Redundancy: Reliance on centralized infrastructure —\nsuch as serving all content from the origin server — increases the risk of downtime\ndue to hardware failures, network outages, and other events. CDNs distribute content\nand requests across multiple locations, reducing the impact of a localized outage.\nWith a CDN coming into the picture, it does two things. One, a lot of traffic doesn't\neven come to your servers. The edge server of the CDN serves a lot of content from\nits cache. So, you need a slightly fewer number of servers.\nSecond, as long as the content is available in the CDNs cache, even if your actual servers\nare not working, the CDN will keep serving the content. This gives you some buffer time to\nfix issues on your servers while the CDN serves whatever content it can from its cache.\n\n\uF0B7 Enhanced Website Security: In addition to optimizing access to web content, a\nCDN may incorporate security functionality.  By blocking distributed denial-of-service\n(DDoS) attacks, enhancing digital certificate security, and other security controls,\nCDNs can reduce the probability and impact of a cyberattack.\n\uF0B7 Web security: if a CDN can isolate bad traffic from good traffic, it can stop all the\nbad traffic from coming to your servers. Your servers only respond to the "good"\nrequests coming from actual users.",
+  "rules": "- Follow the Markdown format for creating notes as shown in the example. \n - The heading of the content should be the title of the markdown file. \n - Create subheadings for each section. \n - Use numbered bullet points for each point.",
+  "notes": "# CDNs\n- A content delivery network (CDN) is a group of geographically distributed servers that speed up the delivery of web content by bringing it closer to where users are. \n- It is a network of strategically positioned servers aimed at enhancing the speed and dependability of delivering content to users in different locations. \n- These servers store cached copies of content, ranging from web pages to videos, guaranteeing that when a user makes a request, it’s served from the closest server available. This reduces delays and speeds up loading times.\n- When a user requests specific content, CDN architecture comes into play. It directs the request to the nearest server, taking into account factors like server health and proximity. This approach minimizes data travel distance, resulting in fast and efficient content delivery.\n- CDNs are designed to optimize the delivery of web content, and some of the main advantages that they provide to a company and its users include:\n- Faster Load Times: CDNs cache content geographically close to its users, decreasing the distance that requests and responses need to travel.  As a result, users experience faster load times for webpages, which can increase conversions and decrease bounce rates. How does a CDN improve page load time?: As mentioned earlier, it is a globally distributed network of servers that store (commonly referred to as \"cache\") and deliver some or all of your website's content. Each of these servers in the CDN's network is called a Point of Presence (PoP) or an edge server.\n- **Reduced Bandwidth Costs:** Serving all requested content from the origin server requires significant bandwidth and processing power at the origin. CDNs reduce load and bandwidth requirements at the bandwidth by caching static content and performing other optimizations. Of course, this helps to greatly reduce costs.\n- **Improved Availability and Redundancy:** Reliance on centralized infrastructure — such as serving all content from the origin server — increases the risk of downtime due to hardware failures, network outages, and other events. CDNs distribute content and requests across multiple locations, reducing the impact of a localized outage. With a CDN coming into the picture, it does two things. One, a lot of traffic doesn't even come to your servers. The edge server of the CDN serves a lot of content from its cache. So, you need a slightly fewer number of servers.\n- Second, as long as the content is available in the CDNs cache, even if your actual servers are not working, the CDN will keep serving the content. This gives you some buffer time to fix issues on your servers while the CDN serves whatever content it can from its cache.\n- Enhanced Website Security: In addition to optimizing access to web content, a CDN may incorporate security functionality.  By blocking distributed denial-of-service(DDoS) attacks, enhancing digital certificate security, and other security controls, CDNs can reduce the probability and impact of a cyberattack.\n- Web security: if a CDN can isolate bad traffic from good traffic, it can stop all the bad traffic from coming to your servers. Your servers only respond to the \"good\" requests coming from actual users. "
+}

model.py

@@ -0,0 +1,106 @@
+from langchain_community.chat_message_histories.in_memory import ChatMessageHistory
+from langchain_community.llms.ctransformers import CTransformers
+from langchain_community.vectorstores import DeepLake
+from langchain_core.messages import AIMessage
+from langchain_core.prompts import PromptTemplate, load_prompt
+from langchain_google_genai import ChatGoogleGenerativeAI
+from typing import List
+from langchain_core.documents.base import Document
+
+
+class DrakeLM:
+    def __init__(self, model_path: str, db: DeepLake, config: dict, llm_model="gemini-pro"):
+        self.llm_model = llm_model
+
+        if llm_model == "llama":
+            self.llama = CTransformers(
+                model=model_path,
+                model_type="llama",
+                config=config
+            )
+        self.gemini = ChatGoogleGenerativeAI(model="gemini-pro", convert_system_message_to_human=True)
+        self.retriever = db.as_retriever()
+        self.chat_history = ChatMessageHistory()
+        self.chat_history.add_user_message("You are assisting a student to understand topics.")
+        self.notes_prompt = load_prompt("prompt_templates/notes_prompt.yaml")
+        self.chat_prompt = load_prompt("prompt_templates/chat_prompt.yaml")
+
+    def _chat_prompt(self, query: str, context: str):
+        prompt = """You are assisting a student to understand topics. \n\n 
+         You have to answer the below question by utilising the below context to answer the question. \n\n 
+         Note to follow the rules given below \n\n 
+         Question: {query} \n\n 
+         Context: {context} \n\n
+         Rules: {rules} \n\n
+         Answer: 
+         """
+
+        rules = """
+        - If the question says answer for X number of marks, you have to provide X number of points.
+        - Each point has to be explained in 3-4 sentences.
+        - In case the context express a mathematical equation, provide the equation in LaTeX format as shown in the example.
+        - In case the user requests for a code snippet, provide the code snippet in the language specified in the example.
+        - If the user requests to summarise or use the previous message as context ignoring the explicit context given in the message.
+         """
+
+        prompt = prompt.format(query=query, context=context, rules=rules)
+        return PromptTemplate.from_template(prompt), prompt
+
+    def _retrieve(self, query: str, metadata_filter, k=3, distance_metric="cos"):
+        self.retriever.search_kwargs["distance_metric"] = distance_metric
+        self.retriever.search_kwargs["k"] = k
+
+        if metadata_filter:
+            self.retriever.search_kwargs["filter"] = {
+                "metadata": {
+                    "id": metadata_filter["id"]
+                }
+            }
+
+        retrieved_docs = self.retriever.get_relevant_documents(query)
+
+        context = ""
+        for rd in retrieved_docs:
+            context += "\n" + rd.page_content
+
+        return context
+
+    def ask_llm(self, query: str, metadata_filter: dict = None):
+        context = self._retrieve(query, metadata_filter)
+        print("Retrieved context")
+        prompt_template, prompt_string = self._chat_prompt(query, context)
+        self.chat_history.add_user_message(prompt_string)
+        print("Generating response...")
+
+        rules = """
+        - If the question says answer for X number of marks, you have to provide X number of points.
+        - Each point has to be explained in 3-4 sentences.
+        - In case the context express a mathematical equation, provide the equation in LaTeX format as shown in the example.
+        - In case the user requests for a code snippet, provide the code snippet in the language specified in the example.
+        - If the user requests to summarise or use the previous message as context ignoring the explicit context given in the message.
+        """
+
+        prompt_template = self.chat_prompt.format(query=query, context=context, rules=rules)
+
+        if self.llm_model == "llama":
+            self.chat_history.add_ai_message(AIMessage(content=self.llama.invoke(prompt_template)))
+        else:
+            self.chat_history.add_ai_message(AIMessage(content=self.gemini.invoke(prompt_template).content))
+
+        return self.chat_history.messages[-1].content
+
+    def create_notes(self, documents: List[Document]):
+        rules = """
+        - Follow the Markdown format for creating notes as shown in the example.
+        - The heading of the content should be the title of the markdown file.
+        - Create subheadings for each section.
+        - Use numbered bullet points for each point.   
+        """
+
+        notes_chunk = []
+        for doc in documents:
+            prompt = self.notes_prompt.format(content_chunk=doc.page_content, rules=rules)
+            response = self.gemini.invoke(prompt)
+            notes_chunk.append(response.content)
+
+        return '\n'.join(notes_chunk)

pages/chat.py

@@ -0,0 +1,43 @@
+import streamlit as st
+from app import disable_sidebar, initialize_models
+from model import DrakeLM
+from utilis import Processing
+
+disable_sidebar()
+col1, col2 = st.columns([1.2, 0.3])
+
+col1.title('Chat with Drake!')
+if col2.button("Home"):
+    st.switch_page("app.py")
+
+universal_chat = st.toggle("Universal Chat")
+st.caption("Note: Universal Chat uses the complete DB to retrieve context, use it with caution")
+
+st.divider()
+
+if "messages" not in st.session_state:
+    st.session_state.messages = []
+
+# Display chat messages from history on app rerun
+for message in st.session_state.messages:
+    with st.chat_message(message["role"]):
+        st.markdown(message["content"])
+
+if prompt := st.chat_input("Ask Drake your questions"):
+    # Display user message in chat message container
+    with st.chat_message("user"):
+        st.markdown(prompt)
+    # Add user message to chat history
+    st.session_state.messages.append({"role": "user", "content": prompt})
+
+    with st.spinner("Drake is thinking..."):
+        query = f"{prompt}"
+        _, drake = initialize_models()
+        if universal_chat:
+            response = drake.ask_llm(query)
+        else:
+            response = drake.ask_llm(query, metadata_filter=st.session_state["metadata"])
+
+        with st.chat_message("assistant"):
+            st.markdown(response)
+            st.session_state.messages.append({"role": "assistant", "content": response})

pages/upload_file.py

@@ -0,0 +1,69 @@
+import streamlit as st
+import io
+import PyPDF2
+from app import disable_sidebar, initialize_models
+from model import DrakeLM
+from utilis import Processing
+
+disable_sidebar()
+st.title('Drake')
+st.subheader('Learn without the mess of making notes!')
+st.divider()
+
+if st.button("Youtube/Video URL"):
+    st.switch_page("pages/upload_url.py")
+
+st.subheader('Upload the file')
+uploaded_file = st.file_uploader(label="Choose a file", type=['pdf', 'doc'])
+allow_make_notes = st.toggle('Make Complete Notes!')
+
+
+if uploaded_file:
+    if st.button("Upload to DB"):
+
+        # Chunking the file
+        with st.spinner('Please wait, file is chunking ...'):
+            try:
+                pdf_stream = io.BytesIO(uploaded_file.read())
+                pdf_reader = PyPDF2.PdfReader(pdf_stream)
+
+                text = ""
+                for page in pdf_reader.pages:
+                    text += page.extract_text()
+
+                processing, drake = initialize_models()
+                documents, metadata = processing.load_pdf("hello world", text)
+                st.session_state["metadata"] = metadata
+                st.success("Successfully chunked the file")
+
+            except Exception as e:
+                st.error("Error in chunking")
+
+            # Uploading to DB
+            with st.spinner('Please wait, file is uploading ...'):
+                try:
+                    processing.upload_to_db(documents)
+                except Exception as e:
+                    st.error("Error in uploading")
+
+                # Generating Notes
+                if allow_make_notes:
+                    with st.spinner('Please wait, notes are being generated ...'):
+                        try:
+                            config = {"max_new_tokens": 4096, "context_length": 8192, "temperature": 0.3}
+                            notes = drake.create_notes(documents)
+                            encoded_text = notes.encode('utf-8')
+                            st.success("Notes generated successfully")
+                            if st.download_button(
+                                    label="Download data as Markdown",
+                                    data=encoded_text,
+                                    file_name='your_notes.md',
+                                    mime='text/markdown',
+                            ):
+                                st.switch_page("pages/chat.py")
+                        except Exception as e:
+                            print(e)
+                            st.error("Error in generating notes")
+
+                else:
+                    st.switch_page("pages/chat.py")

pages/upload_url.py

@@ -0,0 +1,65 @@
+import streamlit as st
+
+from app import disable_sidebar, initialize_models
+from model import DrakeLM
+from utilis import Processing
+
+# Upload Template
+disable_sidebar()
+processing, drake = initialize_models()
+st.title('Drake')
+st.subheader('Learn without the mess of making notes!')
+st.divider()
+
+if st.button("PDF/Transcript"):
+    st.switch_page("pages/upload_file.py")
+
+st.subheader('Enter the Video URL')
+video_url = st.text_input(label="Enter the URL")
+allow_make_notes = st.toggle('Make Complete Notes!')
+
+
+if video_url:
+    # Upload to DB
+    if st.button("Upload to DB"):
+
+        # Chunking the file
+        with st.spinner('Please wait, file is chunking ...'):
+            try:
+                documents, metadata = processing.load_yt_transcript(video_url)
+                st.session_state["metadata"] = {"id": metadata["id"]}
+                st.success("Successfully chunked the file")
+            except Exception as e:
+                print(e)
+                st.error("Error in chunking")
+
+            # Uploading to DB
+            # with st.spinner('Please wait, documents uploading ...'):
+            #     try:
+            #         processing.upload_to_db(documents)
+            #     except Exception as e:
+            #         st.error("Error in uploading")
+
+        # Generating Notes
+        if allow_make_notes:
+            with st.spinner('Please wait, notes are being generated ...'):
+                try:
+                    config = {"max_new_tokens": 4096, "context_length": 8192, "temperature": 0.3}
+                    notes = drake.create_notes(documents)
+                    encoded_text = notes.encode('utf-8')
+                    st.success("Notes generated successfully")
+                    if st.download_button(
+                        label="Download data as Markdown",
+                        data=encoded_text,
+                        file_name='your_notes.md',
+                        mime='text/markdown',
+                    ):
+                        st.switch_page("pages/chat.py")
+                except Exception as e:
+                    print(e)
+                    st.error("Error in generating notes")
+
+        else:
+            st.switch_page("pages/chat.py")
+
+

prompt_templates/chat_prompt.yaml

@@ -0,0 +1,179 @@
+_type: few_shot
+example_prompt:
+  _type: prompt
+  input_types: {}
+  input_variables:
+  - answer
+  - context
+  - query
+  - rules
+  metadata: null
+  name: null
+  output_parser: null
+  partial_variables: {}
+  tags: null
+  template: "\n<Question> {query}\n \n<Context> {context}\n \n<Rules> {rules}\n \n\
+    <Answer> {answer}\n"
+  template_format: f-string
+  validate_template: false
+example_selector: null
+example_separator: '
+
+
+  '
+examples:
+- answer: "# Dense Cells\nStep 1: Define a set of grid points and assign the given\
+    \ data points on the grid.\nStep 2: Determine the dense and sparse cells. If the\
+    \ number of points in a cell exceeds the threshold value t, the cell is categorized\
+    \ as dense cell. Sparse cells are removed from the list.\nStep 3: Merge the dense\
+    \ cells if they are adjacent.\nStep 4: Form a list of grid cells for every subspace\
+    \ as output.\n\n#CILQUE\n **Stage 1:**\n- Step 1: Identify the dense cells.\n\
+    - Step 2: Merge dense cells c\u2081 and c\u2082 if they share the same interval.\n\
+    - Step 3: Generate a particle rule to generate (k + 1)th cell for higher dimension.\
+    \ Then, check whether the number of points cross the threshold. This is repeated\
+    \ till there are no dense cells or new generation of dense cells.\n\n**Stage 2:**\n\
+    - Step 1: Merging of dense cells into a cluster is carried out in each subspace\
+    \ using maximal regions to cover dense cells. The maximal region is an hyperrectangle\
+    \ where all cells fall into.\n- Step 2: Maximal region tries to cover all dense\
+    \ cells to form clusters.\n\n # DBSCAN\n- Step 1: Randomly select a point p. Compute\
+    \ distance between p and all other points.\n- Step 2: Find all points from p with\
+    \ respect to its neighborhood and check whether it has minimum number of points\
+    \ m. If so, it is marked as a core point.\n- Step 3: If it is a core point, then\
+    \ a new cluster is formed, or existing cluster is enlarged.\n- Step 4: If it is\
+    \ a border point, then the algorithm moves to the next point and marks it as visited.\n\
+    - Step 5: If it is a noise point, they are removed.\n- Step 6: Merge the clusters\
+    \ if it is mergeable, dist (c, c) < \u025B.\n- Step 7: Repeat the process 3-6\
+    \ till all points are processed."
+  context: "DBSACN\nStep 1: Randomly select a point p. Compute distance between P\
+    \ and ail other points '\nStep 2: Find all points]from p with respect to its neighbourhoud\
+    \ and check whether it has minimum number of points m. If 80, it is marked as\
+    \ a core point\nStep 3: If it is a core point, then a new cluster is formed, or\
+    \ existing cluster 1s enlarged.\nStep 4: [fit is a border point, then the algorithm\
+    \ moves to the next point and marks it as visited\nStep 5: If it is a noise point,\
+    \ they are removed.\nStep 6: Merge the clusters if it is mergeable, dist (cc )<\
+    \ \xA2.\nStep 7: Repeat the process 3-6 till all Points are processed. \n\nDense\
+    \ Cell\nStep 1: Defining a set of grid points and assigning the given data points\
+    \ on the grid.\nStep 2: Determine the dense and sparse cells. lf the number of\
+    \ points in a cell exceeds the threshold\nvalue t, the cell is categorized as\
+    \ a dense cell. Sparse cells are removed from the list.\nStep 3: Merge the dense\
+    \ cells if they are adjacent.\nStep 4: Form a list of grid cells for every subspace\
+    \ as output.\n\nCLIQUE\nStage 1\nStep 1: Identify the dense cells\nStep 2: Merge\
+    \ dense cells c. and c, if they share the same interval.\nStep 3: Generate Apriori\
+    \ rule to generate (k + 1)\" cell tor higher dimension. Then, check\nwhether the\
+    \ number of points across the threshold This 1s repeated till there are no\ndense\
+    \ cells or a new generation of dense cells\n\nStage 2\nStep 1: Merging of dense\
+    \ cells into a cluster is carried out in each subspace using maximal regions to\
+    \ cover dense cells The maximal region is a hyperrectangle where all cells fall\
+    \ into.\nStep 2; Maximal region tries to cover all dense cells to form clusters."
+  query: Assess DBSCAN, Dense cells and CLIQUE with appropriate steps. (8 marks)
+  rules: "- If the question says answer for X number of marks, you have to provide\
+    \ X number of points.\n - Each point has to be explained in 3-4 sentences.\n -\
+    \ In case the context express a mathematical equation, provide the equation in\
+    \ LaTeX format as shown in the example.\n - In case the user requests for a code\
+    \ snippet, provide the code snippet in the language specified in the example.-\
+    \ If the user requests to summarise or use the previous message as context ignoring\
+    \ the explicit context given in the message."
+- answer: 'Sharding is a technique for dividing a large database into smaller, manageable
+    parts called shards, which are stored across multiple servers or nodes. This process
+    enhances scalability, performance, and fault tolerance by distributing data and
+    processing load. Sharding works by partitioning data based on criteria like geographic
+    location, user ID, or time period, and each shard is responsible for a subset
+    of the data. This method allows for horizontal scaling, improving the system''s
+    capacity to handle large volumes of data and requests efficiently.
+
+
+    The system uses a shard key to identify which shard contains the required data
+    for a query. The shard key is a unique identifier that maps data to its corresponding
+    shard. Upon receiving a query, the system determines the appropriate shard and
+    forwards the query to the correct server or node.
+
+
+    **Features of Sharding:**
+
+    - Sharding makes the database smaller, faster, and more manageable.
+
+    - It can be complex to implement.
+
+    - Sharding reduces transaction costs and allows each shard to read and write its
+    own data.
+
+    - Many NoSQL databases offer auto-sharding.
+
+    - Failure of one shard does not affect the data processing of other shards.
+
+
+    **Benefits of Sharding:**
+
+    1. **Improved Scalability:** Sharding allows horizontal scaling by adding more
+    servers or nodes, enhancing the system''s capacity to handle large volumes of
+    data and requests.
+
+    2. **Increased Performance:**By distributing data across multiple servers or nodes,
+    sharding improves performance, resulting in faster response times and better throughput.
+
+    3. **Fault Tolerance:** Sharding provides fault tolerance as the system can continue
+    to function even if one or more servers or nodes fail, thanks to data replication
+    across multiple servers or nodes.
+
+    4. **Reduced Costs:** Horizontal scaling with sharding can be more cost-effective
+    than vertical scaling by upgrading hardware, as it can be done using commodity
+    hardware, which is typically less expensive than high-end servers.'
+  context: "It is a very important concept that helps the system to keep data in different\
+    \ resources\naccording to the sharding process. The word \u201CShard\u201D means\
+    \ \u201Ca small part of a\nwhole\u201C. Hence Sharding means dividing a larger\
+    \ part into smaller parts. In DBMS,\nSharding is a type of DataBase partitioning\
+    \ in which a large database is divided or\n\npartitioned into smaller data and\
+    \ different nodes. These shards are not only smaller,\nbut also faster and hence\
+    \ easily manageable.\nHow does Sharding work?\nIn a sharded system, the data is\
+    \ partitioned into shards based on a predetermined\ncriterion. For example, a\
+    \ sharding scheme may divide the data based on geographic\nlocation, user ID,\
+    \ or time period. Once the data is partitioned, it is distributed across\nmultiple\
+    \ servers or nodes. Each server or node is responsible for storing and processing\
+    \ a\nsubset of the data.\nExample:\n\nTo query data from a sharded database, the\
+    \ system needs to know which shard contains\nthe required data. This is achieved\
+    \ using a shard key, which is a unique identifier that is\nused to map the data\
+    \ to its corresponding shard. When a query is received, the system\nuses the shard\
+    \ key to determine which shard contains the required data and then sends\nthe\
+    \ query to the appropriate server or node.\nFeatures of Sharding:\n\uF0B7 Sharding\
+    \ makes the Database smaller\n\uF0B7 Sharding makes the Database faster\n\uF0B7\
+    \ Sharding makes the Database much more easily manageable\n\uF0B7 Sharding can\
+    \ be a complex operation sometimes\n\uF0B7 Sharding reduces the transaction cost\
+    \ of the Database\n\uF0B7 Each shard reads and writes its own data.\n\uF0B7 Many\
+    \ NoSQL databases offer auto-sharding.\n\uF0B7 Failure of one shard doesn\u2019\
+    t effect the data processing of other shards.\nBenefits of Sharding:\n1. Improved\
+    \ Scalability: Sharding allows the system to scale horizontally by adding more\n\
+    servers or nodes as the data grows. This improves the system\u2019s capacity to\
+    \ handle\nlarge volumes of data and requests.\n\n2. Increased Performance: Sharding\
+    \ distributes the data across multiple servers or\nnodes, which improves the system\u2019\
+    s performance by reducing the load on each server\nor node. This results in faster\
+    \ response times and better throughput.\n3. Fault Tolerance: Sharding provides\
+    \ a degree of fault tolerance as the system can\ncontinue to function even if\
+    \ one or more servers or nodes fail. This is because the data\nis replicated across\
+    \ multiple servers or nodes, and if one fails, the others can continue\nto serve\
+    \ the requests.\n4. Reduced Costs: Sharding allows the system to scale horizontally,\
+    \ which can be more\ncost-effective than scaling vertically by upgrading hardware.\
+    \ This is because horizontal\nscaling can be done"
+  query: Explain sharding in system design along with its benefits. (10 marks)
+  rules: "- If the question says answer for X number of marks, you have to provide\
+    \ X number of points.\n - Each point has to be explained in 3-4 sentences.\n -\
+    \ In case the context express a mathematical equation, provide the equation in\
+    \ LaTeX format as shown in the example.\n - In case the user requests for a code\
+    \ snippet, provide the code snippet in the language specified in the example.-\
+    \ If the user requests to summarise or use the previous message as context ignoring\
+    \ the explicit context given in the message.\n"
+input_types: {}
+input_variables:
+- context
+- query
+- rules
+metadata: null
+name: null
+output_parser: null
+partial_variables: {}
+prefix: "\nYou are assisting a student to understand topics.\n \nYou have to answer\
+  \ the below question by utilising the below context to answer the question.\nNote\
+  \ to follow the rules given below.\n"
+suffix: "\n<Question> {query}\n \n<Context> {context}\n \n<Rules> {rules}\n <Answer>"
+tags: null
+template_format: f-string
+validate_template: false

prompt_templates/notes_prompt.yaml

@@ -0,0 +1,208 @@
+_type: few_shot
+example_prompt:
+  _type: prompt
+  input_types: {}
+  input_variables:
+  - content_chunk
+  - notes
+  - rules
+  metadata: null
+  name: null
+  output_parser: null
+  partial_variables: {}
+  tags: null
+  template: "\n<Content Chunk> {content_chunk}\n \n<Rules> {rules}\n \n<Notes> {notes}\n"
+  template_format: f-string
+  validate_template: false
+example_selector: null
+example_separator: '
+
+
+  '
+examples:
+- content_chunk: "The cyclic redundancy check codes are popularly employed in LANs\
+    \ and WANs for error correction. The principle of operation of CRC encoders and\
+    \ decoders can be better explained with the following examples. CRC is the most\
+    \ powerful and easy to implement technique.CRC is based on binary division. In\
+    \ CRC, a sequence of redundant bits, are appended to the end of data unit so that\
+    \ the resulting data unit becomes exactly divisible by a second, predetermined\
+    \ binary number. At the destination, the incoming data unit is divided by the\
+    \ same number. If at this step there is no remainder, the data unit is assumed\
+    \ to be correct and is therefore accepted. A remainder indicates that the data\
+    \ unit has been damaged in transit and therefore must be rejected. The binary\
+    \ number, which is (r+1) bit in length, can also be considered as the coefficients\
+    \ of a polynomial, called Generator Polynomial. PERFORMANCE OF CRC CRC is a very\
+    \ effective error detection technique. If the divisor is chosen according to the\
+    \ previously mentioned rules, its performance can be summarized as follows. CRC\
+    \ can detect all single-bit errors and double bit errors (three 1\u2019s). CRC\
+    \ can detect any odd number of errors (X+1) and it can also detect all burst errors\
+    \ of less than the degree of the polynomial. 1. The Sender follows the given steps:\
+    \   2.   The block unit is divided into k sections, and each of n bits.   3. \
+    \  4.   5.   All the k sections are added together by using one's complement to\
+    \ get the sum.   The sum is complemented and it becomes the checksum field.  \
+    \ The original data and checksum field are sent across the network.   Checksum\
+    \ Checker A Checksum is verified at the receiving side. The receiver subdivides\
+    \ the incoming data into equal segments of n bits each, and all these segments\
+    \ are added together, and then this sum is complemented. If the complement of\
+    \ the sum is zero, then the data is accepted otherwise data is rejected. 1. The\
+    \ Receiver follows the given steps:   2.   The block unit is divided into k sections\
+    \ and each of n bits.   3.   4.   5.   All the k sections are added together by\
+    \ using one's complement algorithm to get the sum.   The sum is complemented.\
+    \   If the result of the sum is zero, then the data is accepted otherwise the\
+    \ data is discarded.   Cyclic Redundancy Check (CRC) CRC is a redundancy error\
+    \ technique used to determine the error. Following are the steps used in CRC for\
+    \ error detection: - In CRC technique, a string of n 0s is appended to the data\
+    \ unit, and this n number is less than the number of bits in a predetermined number,\
+    \ known as division which is n+1 bits. - Secondly, the newly extended data is\
+    \ divided by a divisor using a process is known as binary division. The remainder\
+    \ generated from this division is known as CRC remainder. - Thirdly, the CRC remainder\
+    \ replaces the appended 0s at the end of the original data. This newly generated\
+    \ unit is sent to the receiver. - The receiver receives the data followed by the\
+    \ CRC remainder. The receiver will treat this whole unit as a single unit, and\
+    \ it is divided by the same divisor that was used to find the CRC remainder. If\
+    \ the resultant of this division is zero which means that it has no error, and\
+    \ the data is accepted. If the resultant of this division is not zero which means\
+    \ that the data consists of an error. Therefore, the data is discarded."
+  notes: "# CRC (Cyclic Redundancy Check) Encoder and Decoder \n- They are popularly\
+    \ employed in LANs and WANs for error correction. \n- Based on binary division\
+    \ - a sequence of redundant bits are appended to the end of the data unit so that\
+    \ the resulting data unit becomes exactly divisible by a predetermined binary\
+    \ number.\n- At destination, if there is no remainder, it is assumed to be correct.\
+    \ A remainder indicates data unit is damaged and therefore rejected. \n- The binary\
+    \ number (r+1 bits by length) can be considered coefficients of a polynomial,\
+    \ called Generator Polynomial. \n- It is very effective, since it can detect all\
+    \ single-bit errors and double bit errors, odd number of errors and also burst\
+    \ errors of lesser degree than the polynomial. \n- It consists of two components,\
+    \ a generator and a checker.\n  1. **CRC Generator**: uses modulo 2 division.\
+    \ n 0s are appended to the end if divisor contains n+1 units. The remainder generated\
+    \ is called CRC remainder which replaces the appended string of 0s with itself\
+    \ and the final string is sent across the network.\n  2. **CRC Checker:** performs\
+    \ modulo 2 division to the number received from generator by the same divisor,\
+    \ and data is accepted if remainder is zero. \n\n- **CRC Algorithm:**\n  1. Append\
+    \ a string of n 0s to the data unit, where n < the predetermined number of bits\
+    \ in the divisor (n+1 bits).\n  2. Perform binary division between the extended\
+    \ data and the divisor to obtain the CRC remainder.\n  3. Replace the appended\
+    \ 0s at the end of the original data with the CRC remainder.\n  4. Transmit the\
+    \ newly generated unit to the receiver.\n  5. Upon reception, treat the data and\
+    \ CRC remainder as a single unit and divide it by the same divisor used earlier\
+    \ to determine the CRC remainder."
+  rules: "- Follow the Markdown format for creating notes as shown in the example.\
+    \ \n - The heading of the content should be the title of the markdown file. \n\
+    \ - Create subheadings for each section. \n - Use numbered bullet points for each\
+    \ point."
+- content_chunk: "Meaning/Defination: A content delivery network (CDN) is a group\
+    \ of geographically\ndistributed servers that speed up the delivery of web content\
+    \ by bringing it closer to where\nusers are.\n\_It is a network of strategically\
+    \ positioned servers aimed at enhancing the speed and\ndependability of delivering\
+    \ content to users in different locations. These servers store cached\ncopies\
+    \ of content, ranging from web pages to videos, guaranteeing that when a user\
+    \ makes\na request, it\u2019s served from the closest server available. This reduces\
+    \ delays and speeds up\nloading times.\nWhen a user requests specific content,\
+    \ CDN architecture comes into play. It directs the\nrequest to the nearest server,\
+    \ taking into account factors like server health and proximity.\nThis approach\
+    \ minimizes data travel distance, resulting in fast and efficient content delivery.\n\
+    Analogy: You could think of a CDN like an ATM. If your money were only available\
+    \ from one\nbank in town, you\u2019d have to make a time-consuming trip and stand\
+    \ in a long line every time\nyou wanted to withdraw cash. However, with a cash\
+    \ machine on practically every corner,\nyou have fast and easy access to your\
+    \ money any time you need it.\n\nWhat is The Use of a Content Distribution Network?\n\
+    CDNs are designed to optimize the delivery of web content, and some of the main\n\
+    advantages that they provide to a company and its users include:\n\uF0B7 Faster\
+    \ Load Times:\_CDNs cache content geographically close to its users,\ndecreasing\
+    \ the distance that requests and responses need to travel.\_ As a result,\nusers\
+    \ experience faster load times for webpages, which can increase conversions\n\
+    and decrease bounce rates. How does a CDN improve page load time?: As\nmentioned\
+    \ earlier, it is a globally distributed network of servers that store (commonly\n\
+    referred to as &quot;cache&quot;) and deliver some or all of your website&#39;s\
+    \ content. Each of\nthese servers in the CDN&#39;s network is called a Point of\
+    \ Presence (PoP) or an edge\nserver.\n\uF0B7 Reduced Bandwidth Costs:\_Serving\
+    \ all requested content from the origin server\nrequires significant bandwidth\
+    \ and processing power at the origin. CDNs reduce load\nand bandwidth requirements\
+    \ at the bandwidth by caching static content and\nperforming other optimizations.\
+    \ Of course, this helps to greatly reduce costs.\n\uF0B7 Improved Availability\
+    \ and Redundancy:\_Reliance on centralized infrastructure \u2014\nsuch as serving\
+    \ all content from the origin server \u2014 increases the risk of downtime\ndue\
+    \ to hardware failures, network outages, and other events. CDNs distribute content\n\
+    and requests across multiple locations, reducing the impact of a localized outage.\n\
+    With a CDN coming into the picture, it does two things. One, a lot of traffic\
+    \ doesn&#39;t\neven come to your servers. The edge server of the CDN serves a\
+    \ lot of content from\nits cache. So, you need a slightly fewer number of servers.\n\
+    Second, as long as the content is available in the CDNs cache, even if your actual\
+    \ servers\nare not working, the CDN will keep serving the content. This gives\
+    \ you some buffer time to\nfix issues on your servers while the CDN serves whatever\
+    \ content it can from its cache.\n\n\uF0B7 Enhanced Website Security:\_In addition\
+    \ to optimizing access to web content, a\nCDN may incorporate security functionality.\_\
+    \ By blocking distributed denial-of-service\n(DDoS) attacks, enhancing digital\
+    \ certificate security, and other security controls,\nCDNs can reduce the probability\
+    \ and impact of a cyberattack.\n\uF0B7 Web security: if a CDN can isolate bad\
+    \ traffic from good traffic, it can stop all the\nbad traffic from coming to your\
+    \ servers. Your servers only respond to the &quot;good&quot;\nrequests coming\
+    \ from actual users."
+  notes: "# CDNs\n- A content delivery network (CDN) is a group of geographically\
+    \ distributed servers that speed up the delivery of web content by bringing it\
+    \ closer to where users are. \n- It is a network of strategically positioned servers\
+    \ aimed at enhancing the speed and dependability of delivering content to users\
+    \ in different locations. \n- These servers store cached copies of content, ranging\
+    \ from web pages to videos, guaranteeing that when a user makes a request, it\u2019\
+    s served from the closest server available. This reduces delays and speeds up\
+    \ loading times.\n- When a user requests specific content, CDN architecture comes\
+    \ into play. It directs the request to the nearest server, taking into account\
+    \ factors like server health and proximity. This approach minimizes data travel\
+    \ distance, resulting in fast and efficient content delivery.\n- CDNs are designed\
+    \ to optimize the delivery of web content, and some of the main advantages that\
+    \ they provide to a company and its users include:\n- Faster Load Times:\_CDNs\
+    \ cache content geographically close to its users, decreasing the distance that\
+    \ requests and responses need to travel.\_ As a result, users experience faster\
+    \ load times for webpages, which can increase conversions and decrease bounce\
+    \ rates. How does a CDN improve page load time?: As mentioned earlier, it is a\
+    \ globally distributed network of servers that store (commonly referred to as\
+    \ \"cache\") and deliver some or all of your website's content. Each of these\
+    \ servers in the CDN's network is called a Point of Presence (PoP) or an edge\
+    \ server.\n- **Reduced Bandwidth Costs:**\_Serving all requested content from\
+    \ the origin server requires significant bandwidth and processing power at the\
+    \ origin. CDNs reduce load and bandwidth requirements at the bandwidth by caching\
+    \ static content and performing other optimizations. Of course, this helps to\
+    \ greatly reduce costs.\n- **Improved Availability and Redundancy:**\_Reliance\
+    \ on centralized infrastructure \u2014 such as serving all content from the origin\
+    \ server \u2014 increases the risk of downtime due to hardware failures, network\
+    \ outages, and other events. CDNs distribute content and requests across multiple\
+    \ locations, reducing the impact of a localized outage. With a CDN coming into\
+    \ the picture, it does two things. One, a lot of traffic doesn&#39;t even come\
+    \ to your servers. The edge server of the CDN serves a lot of content from its\
+    \ cache. So, you need a slightly fewer number of servers.\n- Second, as long as\
+    \ the content is available in the CDNs cache, even if your actual servers are\
+    \ not working, the CDN will keep serving the content. This gives you some buffer\
+    \ time to fix issues on your servers while the CDN serves whatever content it\
+    \ can from its cache.\n- Enhanced Website Security:\_In addition to optimizing\
+    \ access to web content, a CDN may incorporate security functionality.\_ By blocking\
+    \ distributed denial-of-service(DDoS) attacks, enhancing digital certificate security,\
+    \ and other security controls, CDNs can reduce the probability and impact of a\
+    \ cyberattack.\n- Web security: if a CDN can isolate bad traffic from good traffic,\
+    \ it can stop all the bad traffic from coming to your servers. Your servers only\
+    \ respond to the \"good\" requests coming from actual users. "
+  rules: "- Follow the Markdown format for creating notes as shown in the example.\
+    \ \n - The heading of the content should be the title of the markdown file. \n\
+    \ - Create subheadings for each section. \n - Use numbered bullet points for each\
+    \ point."
+input_types: {}
+input_variables:
+- content_chunk
+- rules
+metadata: null
+name: null
+output_parser: null
+partial_variables: {}
+prefix: '
+
+  You are assigned a task to create notes for a student.
+
+  You have to create notes based on the below content chunk which is part if a huger
+  document.
+
+  Note to follow the rules given below.
+
+  '
+suffix: "\n<Content Chunk> {content_chunk}\n \n<Rules> {rules}\n \n<Notes>\n"
+tags: null
+template_format: f-string
+validate_template: false

requirements.txt

@@ -0,0 +1,10 @@
+PyPDF2
+streamlit
+langchain
+deeplake
+assemblyai
+sentence-transformers
+youtube-transcript-api
+modal
+ctransformers
+langchain-google-genai

requiremnts.txt

@@ -1 +0,0 @@
-streamlit

utilis.py

@@ -0,0 +1,157 @@
+import os
+import json
+from typing import List
+import assemblyai as aai
+from youtube_transcript_api import YouTubeTranscriptApi
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_core.documents.base import Document
+from langchain_community.vectorstores import DeepLake
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain.prompts.few_shot import FewShotPromptTemplate
+from langchain.prompts.prompt import PromptTemplate
+from typing import Dict
+import uuid
+
+
+
+class Processing:
+    def __init__(self, dataset_path: str, embedding_model_name: str,
+                 device='cpu', chunk_size=500, chunk_overlap=5):
+        """
+        Parameters:
+            dataset_path (str): Path to the dataset in the Vector-DB
+            file_path (str): Path to the file to be processed
+            embedding_model_name (str): Name of the HuggingFace model to be used for embeddings
+            device (str): Device to run the embedding model on
+            chunk_size (int): Size of each chunk to be processed
+            chunk_overlap (int): Overlap between each chunk
+
+        Initialize embedding model, text splitter, transcriber and Vector-DB
+        """
+        self.dataset_path = dataset_path
+        self.text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+        self.transcriber = aai.Transcriber()
+
+        self.embedding_model = HuggingFaceEmbeddings(
+            model_name=embedding_model_name,
+            model_kwargs={'device': device},
+            encode_kwargs={'normalize_embeddings': False}
+        )
+
+        self.db = DeepLake(dataset_path=f"hub://{self.dataset_path}",
+                           embedding=self.embedding_model,
+                           exec_option="compute_engine"
+                           )
+
+    def _add_metadata(self, documents: List[Document], url: str, id: str, source: str, file_type: str, course_tag="") -> (List[
+        Document], Dict[str, str]):
+        """
+        Parameters:
+            documents (List[Document]): List of documents to add metadata to
+            id (str): ID of the documents
+            source (str): Source of the documents
+            file_type (str): Type of the documents
+            course_tag (str): Tag to identify the course the documents belongs to
+
+        Returns:
+            documents (List[Document]): List of documents with metadata added
+
+        Add metadata to the documents
+        """
+        metadata = {
+            "id": id,
+            "source": source,
+            "url": url,
+            "file_type": file_type,
+            "course_tag": course_tag
+        }
+        for doc in documents:
+            doc.metadata = metadata
+        return documents, metadata
+
+    def load_pdf(self, name, text) -> (List[Document], Dict[str, str]):
+        """
+        Returns:
+            pdf_chunk (List[Document]): List of documents with metadata added
+
+        Load PDF file, split into chunks and add metadata
+        """
+        pdf_chunk = self.text_splitter.create_documents([text])
+        print("Created document chunks")
+        return self._add_metadata(pdf_chunk, url="NaN", id=str(uuid.uuid4()), source="document", file_type="pdf")
+
+    def load_transcript(self, url) -> (List[Document], Dict[str, str]):
+        """
+        Returns:
+            transcript_chunk (List[Document]): List of documents with metadata added
+
+        Load transcript, split into chunks and add metadata
+        """
+        transcript = self.transcriber.transcribe(url)
+        print("Transcribed")
+        transcript_chunk = self.text_splitter.create_documents([transcript.text])
+        print("Created transcript chunks")
+        return self._add_metadata(transcript_chunk, url="NaN", id=str(uuid.uuid4()), source="custom_video", file_type="transcript")
+
+    def load_yt_transcript(self, url) -> (List[Document], Dict[str, str]):
+        """
+        Returns:
+            yt_transcript_chunk (List[Document]): List of documents with metadata added
+
+        Load YouTube transcript, split into chunks and add metadata
+        """
+        if url.startswith("https://www.youtube.com/watch?v="):
+            video_id = url.replace("https://www.youtube.com/watch?v=", "")
+        else:
+            video_id = url.replace("https://youtu.be/", "")
+
+        transcript = YouTubeTranscriptApi.get_transcript(video_id)
+        print("Downloaded transcript")
+        transcript = [line['text'] for line in transcript]
+        transcript_text = ' '.join(transcript)
+        yt_transcript_chunk = self.text_splitter.create_documents([transcript_text])
+        print("Created YouTube transcript chunks")
+        return self._add_metadata(yt_transcript_chunk, url=url, id=video_id, source="youtube", file_type="transcript")
+
+    def upload_to_db(self, documents: List[Document]):
+        """
+        Parameters:
+            documents (List[Document]): List of documents to upload to Vector-DB
+
+        Upload documents to Vector-DB
+        """
+        print("Embedding and uploading to Vector-DB...")
+        self.db.add_documents(documents)
+        print("Uploaded to Vector-DB")
+
+
+class PromptCreate:
+    def __init__(self, example_path, save_path):
+        self.examples = []
+        self.example_prompt = None
+        self.few_shot_prompt = None
+        self.example_path = example_path
+        self.file_name = "example_{i}.json"
+        self.save_path = save_path
+
+    def load_examples(self):
+        for i in range(1, len(os.listdir(self.example_path)) + 1):
+            filename = os.path.join(self.example_path, self.file_name.format(i=i))
+            try:
+                with open(filename, "r") as json_file:
+                    self.examples.append(json.load(json_file))
+            except FileNotFoundError:
+                print(f"File {filename} not found.")
+            except json.JSONDecodeError:
+                print(f"Error decoding JSON from file {filename}.")
+
+    def create_prompt_template(self, input_variables, template_string):
+        self.example_prompt = PromptTemplate(input_variables=input_variables, template=template_string)
+
+    def create_few_shot_prompt(self, prefix, suffix):
+        self.few_shot_prompt = FewShotPromptTemplate(
+            examples=self.examples, example_prompt=self.example_prompt, prefix=prefix, suffix=suffix
+        )
+
+    def save_prompt(self):
+        self.few_shot_prompt.save(self.save_path)