First Commit

app.py

@@ -0,0 +1,209 @@
+"""
+This module provides functions for working with PDF files and URLs. It uses the urllib.request library
+to download files from URLs, and the fitz library to extract text from PDF files. And GPT3 modules to generate
+text completions.
+"""
+import urllib.request
+import fitz
+import re
+import numpy as np
+import tensorflow_hub as hub
+import openai
+import gradio as gr
+import os
+from sklearn.neighbors import NearestNeighbors
+from pdfQuestions import dataFromPDF
+
+def download_pdf(url, output_path):
+    urllib.request.urlretrieve(url, output_path)
+
+
+def preprocess(text):
+    text = text.replace('\n', ' ')
+    text = re.sub('\s+', ' ', text)
+    return text
+
+
+def pdf_to_text(path, start_page=1, end_page=None):
+    doc = fitz.open(path)
+    total_pages = doc.page_count
+
+    if end_page is None:
+        end_page = total_pages
+
+    text_list = []
+
+    for i in range(start_page-1, end_page):
+        text = doc.load_page(i).get_text("text")
+        text = preprocess(text)
+        text_list.append(text)
+
+    doc.close()
+    return text_list
+
+
+def text_to_chunks(texts, word_length=150, start_page=1):
+    text_toks = [t.split(' ') for t in texts]
+    page_nums = []
+    chunks = []
+    
+    for idx, words in enumerate(text_toks):
+        for i in range(0, len(words), word_length):
+            chunk = words[i:i+word_length]
+            if (i+word_length) > len(words) and (len(chunk) < word_length) and (
+                len(text_toks) != (idx+1)):
+                text_toks[idx+1] = chunk + text_toks[idx+1]
+                continue
+            chunk = ' '.join(chunk).strip()
+            chunk = f'[{idx+start_page}]' + ' ' + '"' + chunk + '"'
+            chunks.append(chunk)
+    return chunks
+
+
+class SemanticSearch:
+    
+    def __init__(self):
+        self.use = hub.load('https://tfhub.dev/google/universal-sentence-encoder/4')
+        self.fitted = False
+    
+    
+    def fit(self, data, batch=1000, n_neighbors=5):
+        self.data = data
+        self.embeddings = self.get_text_embedding(data, batch=batch)
+        n_neighbors = min(n_neighbors, len(self.embeddings))
+        self.nn = NearestNeighbors(n_neighbors=n_neighbors)
+        self.nn.fit(self.embeddings)
+        self.fitted = True
+    
+    
+    def __call__(self, text, return_data=True):
+        inp_emb = self.use([text])
+        neighbors = self.nn.kneighbors(inp_emb, return_distance=False)[0]
+        
+        if return_data:
+            return [self.data[i] for i in neighbors]
+        else:
+            return neighbors
+    
+    
+    def get_text_embedding(self, texts, batch=1000):
+        embeddings = []
+        for i in range(0, len(texts), batch):
+            text_batch = texts[i:(i+batch)]
+            emb_batch = self.use(text_batch)
+            embeddings.append(emb_batch)
+        embeddings = np.vstack(embeddings)
+        return embeddings
+
+
+
+def load_recommender(path, start_page=1):
+    global recommender
+    texts = pdf_to_text(path, start_page=start_page)
+    chunks = text_to_chunks(texts, start_page=start_page)
+    recommender.fit(chunks)
+    return 'Corpus Loaded.'
+
+def generate_text(openAI_key,prompt, engine="text-davinci-003"):
+    openai.api_key = openAI_key
+    completions = openai.Completion.create(
+        engine=engine,
+        prompt=prompt,
+        max_tokens=512,
+        n=1,
+        stop=None,
+        temperature=0.7,
+    )
+    message = completions.choices[0].text
+    return message
+
+def generate_answer(question,openAI_key):
+    topn_chunks = recommender(question)
+    prompt = ""
+    prompt += 'search results:\n\n'
+    for c in topn_chunks:
+        prompt += c + '\n\n'
+        
+    prompt += "Instructions: Compose a comprehensive reply to the query using the search results given. "\
+              "Cite each reference using [ Page Number] notation (every result has this number at the beginning). "\
+              "Citation should be done at the end of each sentence. If the search results mention multiple subjects "\
+              "with the same name, create separate answers for each. Only include information found in the results and "\
+              "don't add any additional information. Make sure the answer is correct and don't output false content. "\
+              "If the text does not relate to the query, simply state 'Text Not Found in PDF'. Ignore outlier "\
+              "search results which has nothing to do with the question. Only answer what is asked. The "\
+              "answer should be short and concise. Answer step-by-step. \n\nQuery: {question}\nAnswer: "
+    
+    prompt += f"Query: {question}\nAnswer:"
+    answer = generate_text(openAI_key, prompt,"text-davinci-003")
+    return answer
+
+
+def question_answer(file,question,openAI_key):
+    
+    if openAI_key.strip()=='':
+        return '[ERROR]: Please enter you Open AI Key. Get your key here : https://platform.openai.com/account/api-keys'
+
+
+    file_paths = ["./Smart Protect Goal Brochure.pdf", "./Future Wealth Gain Brochure.pdf"]
+    for path in file_paths:
+        if file in path:
+            load_recommender(path)
+            print(path)
+
+    if question.strip() == '':
+        return '[ERROR]: Question field is empty'
+
+    return generate_answer(question,openAI_key)
+
+def allQuestion(file):
+
+    questionOptions1 = ["What are the various options under Life Cover Variant?","What are the Add-on covers options under Variant description Life Cover?","What is the total claim covered under Minor and Major CI?","What is Waiver of Premium Benefit on CI?","What is Annualized Premium under Life Cover Variant?","Does the ROP include GST?","Under what condition is Add-on Covers applicable?","What is the duration period of premiums for CIB & WOPBI?","What is the maximum maturity age with ROP under the variant?","What is the maximum maturity age with Whole Life under the variant?"]
+
+    questionOptions2 = ["What is Future Wealth Gain plan?","If the customer has done a partial withdrawls, is he eligible for the Loyalty Additions/Fund Boosters?","What are the steps to select the plan?","What are the maturity benefits available in the wealth plus variant of this plan?","How can one revive a discontinued policy?",'What are the tax benefit options available under this policy?','What are the features under "Wealth Plus" & "Wealth Plus Care" Variant?',"Can I switch between the funds?"]
+
+    if file  == None:
+        return '[ERROR]: Provide select atleast one option.'
+    if file  == "Smart Protect Goal Brochure":
+        question = questionOptions1
+    if file == "Future Wealth Gain Brochure":
+        question = questionOptions2
+    
+    return gr.Dropdown.update(choices=question)
+
+
+recommender = SemanticSearch()
+
+title = 'Madgical Chatbots - PDF GPT'
+description = """ This chantbot will generate answer from PDF file :"""
+options = ['Smart Protect Goal Brochure', 'Future Wealth Gain Brochure']
+
+newOption = None
+
+with gr.Blocks() as demo:
+
+    gr.Markdown(f'<center><h1>{title}</h1></center>')
+    gr.Markdown(f'<center><h4>{description}</h4></center>')
+    
+
+    with gr.Row():
+        
+        with gr.Group():
+            openAI_key=gr.Textbox(label='Enter your OpenAI API key here')
+            file = gr.Dropdown(options, label="Select PDF file from below options")
+            btn2 = gr.Button(value='Submit')
+            question = gr.Dropdown(newOption,label="Select questions from below options")
+            btn = gr.Button(value='Submit')
+            btn.style(full_width=True)
+            btn2.style(full_width=True)
+
+        with gr.Group():
+            answer = gr.Textbox(label='Medgical Chatbot Answer :')
+        with gr.Group():
+            pdfData = gr.Textbox(label='Answer from PDF :')
+
+        btn.click(question_answer, inputs=[file, question,openAI_key], outputs=[answer])
+        btn.click(dataFromPDF, inputs=[question], outputs=[pdfData])
+        btn2.click(allQuestion, inputs=[file], outputs=[question])
+        
+if __name__ == "__main__":
+    demo.launch() 

pdfQuestions.py

@@ -0,0 +1,79 @@
+def dataFromPDF(question):
+    data = [
+        {
+            'Smart Protect Goal Brochure':{
+                "What are the various options under Life Cover Variant?": 'The following options are\n\
+                \u2022 Whole of Life Life Cover up to age 99 years : This option is only available when limited premium payment option is chosen\n\
+                \u2022 Return of Premiums (ROP) If no claim has been made for the Variant and/or for each Add-on cover(s), the total premiums paid for the Variant and/or each of the Add-on cover(s) opted for, will be returned on their respective maturity dates. If Whole Life is opted, ROP will not be available.',
+
+                "What are the Add-on covers options under Variant description Life Cover?":'The following Add-on covers\n\
+                \u2022 Accidental Death Benefit (ADB): In case of death due to an accident, Sum Assured chosen as ADB is payable.\n\
+                \u2022 Accidental Total Permanent Disability Benefit (ATPDB): In case of occurrence of total permanent disability of the Life Assured due to an accident, Sum Assured chosen as ATPDB is payable.\n\
+                \u2022 Critical Illness Benefit (CIB):\n\
+                    \u2022 In case of diagnosis of any of the listed Critical Illnesses, Sum Assured chosen for Critical Illness benefit is payable.\n\
+                    \u2022 In case of Minor CI other than Angioplasty, 25% of the CIB will be payable.\n\
+                    \u2022 For Angioplasty, lower of 5 lakhs or 25% of CIB will be payable.\n\
+                    \u2022 A maximum of four (4) Minor CI including Angioplasty claims will be payable during the CIB cover period.\n\
+                    \u2022 For Major CI, 100% of CIB will be payable.',
+
+                "What is the total claim covered under Minor and Major CI?":"The total claims paid under Minor and Major CI will not be more than 100% of CIB",
+
+                "What is Waiver of Premium Benefit on CI?":"On occurrence of fourth (4 ) Minor CI or on the date of occurrence of the first (1st) Major CI (incl. ATPD), whichever is earlier, all future premiums due under the policy will be waived and WOPB-I cover will terminate immediately and automatically.",
+
+                "What is Annualized Premium under Life Cover Variant?":"Annualized Premium is the total premium/s payable in a policy year for a LP, RP and the single premium for an SP. The annualized premium is exclusive of extra premium, add-on covers and loadings for modal premiums, if any, and Total Premiums paid shall be equal to (Annualized Premium * number of years for which premiums have been paid)",
+
+                "Does the ROP include GST?":"ROP is excluding GST/any other applicable tax levied, subject to changes in tax laws and any extra premium.",
+                
+                "Under what condition is Add-on Covers applicable?":"Add-on Covers will only be applicable, subject to the conditions, exclusions, waiting period, cooling period &survival period as applicable",
+                "What is the duration period of premiums for CIB & WOPBI?":"The premiums for CIB &amp WOPB are guaranteed for a period of 5 years and reviewable for subsequent cover periods",
+                "What is the maximum maturity age with ROP under the variant?":"The maximum maturity age is 75 years.",
+                "What is the maximum maturity age with Whole Life under the variant?":"The maximum maturity age is 99 years."
+
+            },
+            'Future Wealth Gain Brochure':{
+
+                "What is Future Wealth Gain plan?": 'Bajaj Allianz Life Future Wealth Gain is a non-participating, individual, life, unit-linked regular/limited premium payment endowment plan with two variants, “Wealth Plus” and “Wealth Plus Care”. Bajaj Allianz Life Future Wealth Gain plan offers the dual benefit of protection and growth to fulfil the dreams of your loved ones.',
+
+                "If the customer has done a partial withdrawls, is he eligible for the Loyalty Additions/Fund Boosters?": 'Yes. Policyholder will receive Loyalty Addition/ Fund Boosters, provided the policy is in-force and all premiums have been paid.',
+
+                "What are the steps to select the plan?":'Plan Working\n\
+                                            Step 1: Choose from the two variants Wealth Plus and Wealth Plus Care\n\
+                                            Step 2: Choose the premium you want to pay\n\
+                                            Step 3: Choose the sum assured multiplier to decide your life cover\n\
+                                            Step 4: Choose your policy term and premium payment term\n\
+                                            Step 5: Choose the premium payment frequency\n\
+                                            Step 6: Choose between the two portfolio strategies\n\
+                                            Step 7: Choose the riders (optional and with rider charges applicable)\n\
+                                            Note:\n\
+                                            \u2022 The variant has to be chosen at the inception of the policy and cannot be changed subsequently.\n\
+                                            \u2022 Applicable as per the minimum/maximum sum assured criteria. Please refer to the Eligibility Parameters',
+
+                "What are the maturity benefits available in the wealth plus variant of this plan?":"On the maturity date, you will receive the Regular Premium Fund Value plus Top up Premium Fund Value",
+
+                "How can one revive a discontinued policy?":'Revival\n A policy which has been discontinued due to non-payment of premiums can only be revived subject to following conditions:\n\
+                            1. The Insurance Company receives the request for revival from you within 3 years from the date of first unpaid premium provided the policy is not terminated already\n\
+                            2. You submit such information and documentation as may be requested by the Insurance Company at your own expense.\n\
+                            3. The policy may be revived on the original policy terms & conditions, revised terms & conditions or disallowed revival, based on Board approved underwriting guidelines.\n\
+                            4. On revival of the discontinued policy,\n \u2022 The policy will be revived restoring the risk cover, Loyalty Addition and Fund Booster.\n \u2022 All the due but unpaid premiums will be collected from you without charging any interest or fee.\n \u2022 The Discontinuance Value of the policy together with the amount of discontinuance/ surrender charge (without any interest) as deducted by the Insurance Company on the date of discontinuance of the policy, shall be restored to the chosen fund split into to the applicable Fund/s available as on the date of discontinuance, at their prevailing unit price.\n \u2022 The Premium Allocation Charge and Policy Administration Charge, as applicable, during the discontinuance period shall be deducted as applicable from regular premiums paid or from the fund at the time of revival.\n  \u2022 The Loyalty Additions due-but-not-allotted during the period the Policy was in Discontinuance shall be added to the Regular Premium Fund Value.',
+
+                'What are the tax benefit options available under this policy?':'Tax Benefit \n\ Premium paid, maturity benefit, death benefit and surrender benefit are eligible for tax benefits as per extant Income Tax Act, subject to the provision stated therein and as amended from time to time.\n\ You are requested to consult your tax consultant and obtain independent advice for eligibility and before claiming any benefit under the policy.',
+
+                'What are the features under "Wealth Plus" & "Wealth Plus Care" Variant?':'Bajaj Allianz Life Future Wealth Gain provides you with two unique portfolio strategies, which can be chosen at the inception of your policy or on any subsequent policy anniversary:\n\
+                    a) Investor selectable Portfolio Strategy\n\ b) Wheel of Life Portfolio Strategy',
+                    
+                "Can I switch between the funds?":'Yes, there is an option to switch between funds - only under the Investor Selectable Portfolio Strategy.\n\
+You have the flexibility to switch units between your investment funds according to your risk appetite and investment decisions, by giving written notice to the Insurance Company.\n\
+\u2022 You can make unlimited free switches.\n\
+\u2022 The minimum switching amount is Rs 5,000 or the value of units in the fund to be switched from, whichever is lower.\n\
+\u2022 The Insurance Company shall effect the switch by redeeming units from the fund to be switched from and allocating new units in the fund being switched to at their respective unit price.'
+            }
+        }
+    ]
+
+    for brochure in data:
+        if question in brochure['Smart Protect Goal Brochure']:
+            return brochure['Smart Protect Goal Brochure'][question]
+        elif question in brochure['Future Wealth Gain Brochure']:
+            return brochure['Future Wealth Gain Brochure'][question]
+    
+    return "Answer not found in available data."

requirements.txt

@@ -0,0 +1,107 @@
+absl-py==1.4.0
+aiofiles==23.1.0
+aiohttp==3.8.4
+aiosignal==1.3.1
+altair==4.2.2
+anyio==3.6.2
+astunparse==1.6.3
+async-timeout==4.0.2
+attrs==23.1.0
+cachetools==5.3.0
+certifi==2022.12.7
+charset-normalizer==3.1.0
+click==8.1.3
+colorama==0.4.6
+contourpy==1.0.7
+cycler==0.11.0
+entrypoints==0.4
+et-xmlfile==1.1.0
+fastapi==0.95.1
+ffmpy==0.3.0
+filelock==3.12.0
+flatbuffers==23.3.3
+fonttools==4.39.3
+frozenlist==1.3.3
+fsspec==2023.4.0
+gast==0.4.0
+google-auth==2.17.3
+google-auth-oauthlib==1.0.0
+google-pasta==0.2.0
+gradio==3.27.0
+gradio_client==0.1.3
+grpcio==1.54.0
+h11==0.14.0
+h5py==3.8.0
+httpcore==0.17.0
+httpx==0.24.0
+huggingface-hub==0.14.1
+idna==3.4
+jax==0.4.8
+Jinja2==3.1.2
+joblib==1.2.0
+jsonschema==4.17.3
+keras==2.12.0
+kiwisolver==1.4.4
+libclang==16.0.0
+linkify-it-py==2.0.0
+Markdown==3.4.3
+markdown-it-py==2.2.0
+MarkupSafe==2.1.2
+matplotlib==3.7.1
+mdit-py-plugins==0.3.3
+mdurl==0.1.2
+ml-dtypes==0.1.0
+multidict==6.0.4
+numpy==1.23.5
+oauthlib==3.2.2
+openai==0.10.2
+openpyxl==3.1.2
+opt-einsum==3.3.0
+orjson==3.8.10
+packaging==23.1
+pandas==2.0.1
+pandas-stubs==2.0.0.230412
+Pillow==9.5.0
+protobuf==4.22.3
+pyasn1==0.5.0
+pyasn1-modules==0.3.0
+pydantic==1.10.7
+pydub==0.25.1
+PyMuPDF==1.22.2
+pyparsing==3.0.9
+pyrsistent==0.19.3
+python-dateutil==2.8.2
+python-multipart==0.0.6
+pytz==2023.3
+PyYAML==6.0
+requests==2.29.0
+requests-oauthlib==1.3.1
+rsa==4.9
+scikit-learn==1.2.2
+scipy==1.10.1
+semantic-version==2.10.0
+six==1.16.0
+sniffio==1.3.0
+starlette==0.26.1
+tensorboard==2.12.2
+tensorboard-data-server==0.7.0
+tensorboard-plugin-wit==1.8.1
+tensorflow==2.12.0
+tensorflow-estimator==2.12.0
+tensorflow-hub==0.13.0
+tensorflow-intel==2.12.0
+tensorflow-io-gcs-filesystem==0.31.0
+termcolor==2.3.0
+threadpoolctl==3.1.0
+toolz==0.12.0
+tqdm==4.65.0
+types-pytz==2023.3.0.0
+typing_extensions==4.5.0
+tzdata==2023.3
+uc-micro-py==1.0.1
+urllib3==1.26.15
+uvicorn==0.21.1
+websockets==11.0.2
+Werkzeug==2.3.0
+wrapt==1.14.1
+yarl==1.9.2