3commit

app.py

@@ -17,24 +17,31 @@ COMPLETIONS_MODEL = "gpt-4"
 openai.api_key = "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"
 COMPLETIONS_API_PARAMS = {
     "temperature": 0.0,
-    "max_tokens": 300,
+    "max_tokens": 1000,
     "model": COMPLETIONS_MODEL,
 }
 
 @st.cache_data
 def run_on_chunks(data):
     response = []
-    chunk = data_chunk(data , chunk_size = 1000)
-    for i in chunk:      
+    chunk = data_chunk(data , chunk_size = 2500)
+    num = 0
+    text = st.empty()
+
+    for i in chunk:    
+        num = num + 1  
+        text.write(f"{num}th API request sent out of {len(chunk)}")
         response.append(GPT_4_API(i))
+        text.empty()
+    
     return response
-
+    
 def data_chunk(lst , chunk_size):
     return [lst[i:i+chunk_size] for i in range(0, len(lst), chunk_size)]
-
+    
 def check_file_format(filename):
     return filename.rsplit('.', 1)[1].lower()
-
+    
 def pdf_to_images(pdf_file):
     images = []
     with fitz.open(pdf_file) as doc:
@@ -43,7 +50,7 @@ def pdf_to_images(pdf_file):
             img = Image.frombytes("RGB", [pix.width, pix.height], pix.samples)
             images.append(img)
     return images
-
+    
 def OCR(pdf_file):
     pdf_reader = PyPDF2.PdfReader(pdf_file)
     pdf_writer = PyPDF2.PdfWriter()
@@ -65,7 +72,6 @@ def OCR(pdf_file):
     pdf_file.close()
     return text
 
-
 def txt_extraction(file_path):
     file_contents = file_path.read().decode("utf-8")
     return file_contents
@@ -92,8 +98,7 @@ def download_docx(text):
     )
 
 def GPT_4_API(data):
-    print("request_send")
-    header =  """ create 20 question and answeres from this paragraph, Answer should strictly be exact lines from this paragraph without question answer numbers"."\n\nContext:\n"""    
+    header =  """ create 12 question and answeres from given paragraph dont use numbers to point out questions and answers, Answers should strictly be exact lines from this paragraph"."\n\nContext:\n"""    
     QA = header + "".join(str(list(data)))   
     response = openai.ChatCompletion.create(messages = [{"role": "user", "content": f"{QA}"},],**COMPLETIONS_API_PARAMS)        
     return response["choices"][0]["message"]["content"]
@@ -127,18 +132,18 @@ def set_png_as_page_bg(png_file):
 
 def Extract_pdf_content(pdf_name):
     
-    page_text = []
+    page_text = ""
     pdf_reader = PyPDF2.PdfReader(pdf_name)
     num_pages = len(pdf_reader.pages)
 
     for page in range(num_pages):
         pdf_page = pdf_reader.pages[page]
-        page_text.append(pdf_page.extract_text())
-    
-    return page_text[0]
+        page_text = page_text + pdf_page.extract_text()
+        
+    return page_text
                                      
 def process(uploaded_file):
-    st.write("Filename:", uploaded_file.name)
+
     data = Extract_pdf_content(uploaded_file)
     return data
 
@@ -147,18 +152,22 @@ if __name__=="__main__":
     pytesseract.pytesseract.tesseract_cmd = r'C:\\Program Files\\Tesseract-OCR\\tesseract.exe'
     PAGE_CONFIG = {"page_title":"StColab.io","page_icon":":smiley:","layout":"centered"}
     st.set_page_config(**PAGE_CONFIG) 
-    main_bg = 'bg.png'
+    main_bg = 'bkgnd1.jpg'
     set_png_as_page_bg(main_bg)
     
-    st.title("pdf data extraction web application")	
-    uploaded_file = st.file_uploader("Upload a PDF file", type = ["pdf","docx","txt"])
+    st.title("Advanced Text processing Tool")	
+    uploaded_file = st.file_uploader("Upload a Files here", type = ["pdf","docx","txt"])
     
     if uploaded_file is not None:
         
         if check_file_format(uploaded_file.name) == "pdf":
             data = process(uploaded_file)
+            
+            text = st.empty()
             if data == '':
+                text.write("applying OCR")
                 data = OCR(uploaded_file)
+                text.empty()
 
         elif check_file_format(uploaded_file.name) == "docx":
             data = docx_extraction(uploaded_file)