File size: 19,775 Bytes
e2e8616
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
import json
import PyPDF2
# To analyze the PDF layout and extract text
from pdfminer.high_level import extract_pages, extract_text
from pdfminer.layout import LTTextContainer, LTChar, LTRect, LTFigure
# To extract text from tables in PDF
import pdfplumber
# To extract the images from the PDFs
from PIL import Image
from pdf2image import convert_from_path
# To perform OCR to extract text from images 
import pytesseract 
# To remove the additional created files
import os
import pdfplumber as pdfp

from src.model.paragraph import Paragraph
from src.tools.table_converter import table_converter
from src.tools.reader_pdf_tools import *
import gradio as gr


def get_style_of_line(size : float, fontname : str):
    if fontname == "XFQKGD+Consolas":
        return "code"
    elif (size >= 9 and size < 11.5) or fontname == "CRRYJU+Wingdings-Regular":
        return "content"
    elif size >= 11.5 and size <= 12.7:
        return "title5"
    elif size >= 12.8 and size <= 13.5:
        return "title4"
    elif size > 13.5 and size <= 15.5:
        return "title3"
    elif size > 15.5 and size <= 18.5:
        return "title2"
    elif size > 19 and size < 30:
        return "title1"
    else:
        return "unknown"

class Reader:
    def __init__(self, path,actual_first_page_=0, include_images=True):
        self.path = path
        self.paragraphs = self.pdf_manager(path, actual_first_page_, include_images=include_images)


    def most_occuring_fonts(self, line_formats : list):
        if line_formats != []:
            min_freq = 3
            font_size_freq = {i: line_formats.count(i) for i in set(line_formats) if isinstance(i, float)}
            most_occuring_font_sizes = [size for size, freq in font_size_freq.items() if freq >= min_freq]
            line_formats = [i for i in line_formats if i in most_occuring_font_sizes or isinstance(i, str)]
        return line_formats


    def text_extraction(self,element):
        # Extracting the text from the in line text element
        line_text = element.get_text()
        # Find the formats of the text
        # Initialize the list with all the formats appeared in the line of text
        line_formats = []
        for text_line in element:
            if isinstance(text_line, LTTextContainer):
                # Iterating through each character in the line of text
                for character in text_line:
                    if isinstance(character, LTChar):
                        # Append the font name of the character
                        line_formats.append(character.fontname)
                        # Append the font size of the character
                        line_formats.append(character.size)
        #find the most occuring font size and keep it. If there are more than one, keep all of them.
        line_formats = self.most_occuring_fonts(line_formats)
        # Find the unique font sizes and names in the line and delete the None values
        format_per_line = list(set(line_formats))
        # Return a tuple with the text in each line along with its format
        return (line_text, format_per_line)

    # Extracting tables from the page
    def extract_table(self, pdf_path, page_num, table_num):
        # Open the pdf file
        pdf = pdfplumber.open(pdf_path)
        # Find the examined page
        table_page = pdf.pages[page_num]
        # Extract the appropriate table
        table = table_page.extract_tables()[table_num]
        
        return table

    # Create a function to check if the element is in any tables present in the page
    def is_element_inside_any_table(self, element, page ,tables):
        x0, y0up, x1, y1up = element.bbox
        # Change the cordinates because the pdfminer counts from the botton to top of the page
        y0 = page.bbox[3] - y1up
        y1 = page.bbox[3] - y0up
        for table in tables:
            tx0, ty0, tx1, ty1 = table.bbox
            if tx0 <= x0 <= x1 <= tx1 and ty0 <= y0 <= y1 <= ty1:
                return True
        return False

    # Function to find the table for a given element
    def find_table_for_element(self, element, page ,tables):
        x0, y0up, x1, y1up = element.bbox
        # Change the cordinates because the pdfminer counts from the botton to top of the page
        y0 = page.bbox[3] - y1up
        y1 = page.bbox[3] - y0up
        for i, table in enumerate(tables):
            tx0, ty0, tx1, ty1 = table.bbox
            if tx0 <= x0 <= x1 <= tx1 and ty0 <= y0 <= y1 <= ty1:
                return i  # Return the index of the table
        return None

    # Create a function to crop the image elements from PDFs
    def crop_image(self, element, pageObj):
        # Get the coordinates to crop the image from PDF
        [image_left, image_top, image_right, image_bottom] = [element.x0,element.y0,element.x1,element.y1] 
        # Crop the page using coordinates (left, bottom, right, top)
        pageObj.mediabox.lower_left = (image_left, image_bottom)
        pageObj.mediabox.upper_right = (image_right, image_top)
        # Save the cropped page to a new PDF
        cropped_pdf_writer = PyPDF2.PdfWriter()
        cropped_pdf_writer.add_page(pageObj)
        # Save the cropped PDF to a new file
        with open('cropped_image.pdf', 'wb') as cropped_pdf_file:
            cropped_pdf_writer.write(cropped_pdf_file)

    # Create a function to convert the PDF to images
    def convert_to_images(self, input_file,):
        images = convert_from_path(input_file)
        image = images[0]
        output_file = 'PDF_image.png'
        image.save(output_file, 'PNG')

    # Create a function to read text from images
    def image_to_text(self, image_path):
        # Read the image
        img = Image.open(image_path)
        # Extract the text from the image
        text = pytesseract.image_to_string(img)
        return text

    def pdf_manager(self, pdf_path, actual_first_page=0, include_images=True):
        # create a PDF file object
        pdfFileObj = open(pdf_path, 'rb')
        # create a PDF reader object
        pdfReaded = PyPDF2.PdfReader(pdfFileObj)
        number_of_pages = len(pdfReaded.pages)
        # Create the dictionary to extract text from each image
        text_per_page = {}
        # Create a boolean variable for image detection
        image_flag = False
        actual_first_page = int(actual_first_page)
        if actual_first_page > number_of_pages:
            gr.Warning("The number of pages you want to skip is greater than the number of pages in the document. We will extract all the pages.")
            page_numbers = None
        else:
            page_numbers = [i for i in range(actual_first_page - 1,number_of_pages)]
        # We extract the pages from the PDF
        for pagenum, page in enumerate(extract_pages(pdf_path,page_numbers=page_numbers)):
            # Initialize the page object
            pagenum = page_numbers[pagenum] if page_numbers else pagenum
            pageObj = pdfReaded.pages[pagenum]
            # Initialize the variables needed for the text extraction from the page
            page_text = []
            line_format = []
            text_from_images = []
            text_from_tables = []
            page_content = []
            # Initialize the number of the examined tables
            table_in_page= -1
            # Open the pdf file
            pdf = pdfplumber.open(pdf_path)
            # Find the examined page
            page_tables = pdf.pages[pagenum]
            # Find the number of tables in the page

            tables = page_tables.find_tables()
            if len(tables)!=0:
                table_in_page = 0

            # Extracting the tables of the page
            for table_num in range(len(tables)):
                # Extract the information of the table
                table = self.extract_table(pdf_path, pagenum, table_num)
                # Convert the table information in structured string format
                table_string = table_converter(table)
                # Append the table string into a list
                text_from_tables.append(table_string)

            # Find all the elements
            page_elements = [(element.y1, element) for element in page._objs]
            # Sort all the element as they appear in the page 
            page_elements.sort(key=lambda a: a[0], reverse=True)


            # Find the elements that composed a page
            for i,component in enumerate(page_elements):
                # Extract the element of the page layout
                element = component[1]

                # Check the elements for tables
                if table_in_page == -1:
                    pass
                else:
                    if self.is_element_inside_any_table(element, page ,tables):
                        table_found = self.find_table_for_element(element,page ,tables)
                        if table_found == table_in_page and table_found != None:    
                            page_content.append(text_from_tables[table_in_page])
                            page_text.append('table')
                            line_format.append('table')
                            table_in_page+=1
                        # Pass this iteration because the content of this element was extracted from the tables
                        continue

                if not self.is_element_inside_any_table(element,page,tables):

                    # Check if the element is text element
                    if isinstance(element, LTTextContainer):
                        # Use the function to extract the text and format for each text element
                        (line_text, format_per_line) = self.text_extraction(element)
                        # Append the text of each line to the page text
                        page_text.append(line_text)
                        # Append the format for each line containing text
                        line_format.append(format_per_line)
                        page_content.append(line_text)


                    #Check the elements for images
                    if include_images:
                        if isinstance(element, LTFigure):
                            # Crop the image from PDF
                            self.crop_image(element, pageObj)
                            # Convert the croped pdf to image
                            self.convert_to_images('cropped_image.pdf')
                            # Extract the text from image
                            image_text = self.image_to_text('PDF_image.png')
                            text_from_images.append(image_text)
                            page_content.append(image_text)
                            # Add a placeholder in the text and format lists
                            page_text.append('image')
                            line_format.append('image')
                            # Update the flag for image detection
                            image_flag = True

            # Create the key of the dictionary
            dctkey = 'Page_'+str(pagenum)
            # Add the list of list as value of the page key
            text_per_page[dctkey]= [page_text, line_format, text_from_images, text_from_tables, page_content]


        # Close the pdf file object
        pdfFileObj.close()

        # Create a list of formats for all the pages
        formats = []
        for p in text_per_page.values():
            formats.append(p[1])

        #flatten the list of lists
        formats = flatten(formats)

        #keep only the font sizes in the list
        formats = keep_int_and_floats_in_list(formats)

        #group the formats in lists of similar formats
        grouped_formats = group_formats(formats)

        #create a dictionary with the format as key and the style as value
        styles = create_dict_and_assign_styles_from_format(grouped_formats)

        #display the result on a separate file as a JSON with some indentation for better visualization
        with open(file="styles.txt", mode='a') as fp:
            if fp.tell() == 0:
                fp.write('Document title: ' + pdf_path.split('/')[-1] + '\n') if '/' in pdf_path else fp.write('Document title: ' + pdf_path.split('\\')[-1] + '\n')
            else:
                fp.write('\nDocument title: ' + pdf_path.split('/')[-1] + '\n') if '/' in pdf_path else fp.write('\nDocument title: ' + pdf_path.split('\\')[-1] + '\n')
            json.dump(styles, fp, indent=4)

        # Delete the additional files created if image is detected
        if image_flag:
            os.remove('cropped_image.pdf')
            os.remove('PDF_image.png')

        #beginning of the paragraph extraction
        paragraphs = []
        for index, page in enumerate(text_per_page.values()):
            content_format = page[1]
            j = 0
            while j+1 < len(content_format):
                actual_format = content_format[j]
                n_of_fontsizes = len(list(i for i in actual_format if isinstance(i, int) or isinstance(i, float)))
                if n_of_fontsizes > 1:
                    actual_format = max(keep_int_and_floats_in_list(actual_format))
                    actual_format = find_good_key_in_dict(styles,actual_format)
                elif n_of_fontsizes == 1:
                    actual_format = keep_int_and_floats_in_list(actual_format)[0]
                    actual_format = find_good_key_in_dict(styles,actual_format)
                elif n_of_fontsizes == 0 and actual_format == "table":
                    actual_format = "table"
                else:
                    actual_format = "content"
                #try to find the good format if the current result seems wrong
                #changes depending on the document
                if len(page[4][j]) > 150 and "title" in actual_format:
                    actual_format = "content"
                paragraph = Paragraph(text=page[4][j],font_style=actual_format,id_=j,page_id=index)
                paragraphs.append(paragraph)
                j+=1

        paragraphs = self.concatenate_paragraphs(paragraphs, pdf_path.split('/')[-1]) if '/' in pdf_path else self.concatenate_paragraphs(paragraphs, pdf_path.split('\\')[-1])
        return paragraphs

    def concatenate_paragraphs(self, paragraphs, doc_title):
        concatenated_paragraphs = []
        i = 0
        actual_page_id = paragraphs[0].page_id
        while i < len(paragraphs):
            p = paragraphs[i]
            if p.blank or "REST API Developer Guide 23.3" in p.text or "x! illumio" in p.text:
                i+=1
                continue
            if (p.page_id != actual_page_id) and doc_title == "Illumio_Core_REST_API_Developer_Guide_23.3.pdf" and (not p.font_style == "table" and not "title" in p.font_style):
                i+=2
                actual_page_id = p.page_id
                continue
            if not concatenated_paragraphs:
                concatenated_paragraphs.append(p)
            elif p.font_style != concatenated_paragraphs[-1].font_style:
                if (p.font_style == "table" and concatenated_paragraphs[-1].font_style == "content") \
                    or (p.font_style == "content" and concatenated_paragraphs[-1].font_style == "table"):
                    concatenated_paragraphs[-1].text += '\n' + p.text
                else:
                    concatenated_paragraphs.append(p)
            else:
                if "title" in p.font_style:
                    concatenated_paragraphs[-1].text += ' : ' + p.text
                    concatenated_paragraphs[-1].text = concatenated_paragraphs[-1].text.replace('\n','').replace('\r','')
                else:
                    concatenated_paragraphs[-1].text += '\n' + p.text
            i+=1
        return concatenated_paragraphs
    

class Reader_illumio:
    def __init__(self, path):
        self.path = path
        self.paragraphs = self.get_pdf_paragraphs(path)

    def skip_header(self, dictionary):
        i = 0
        if "Illumio_Core_REST_API_Developer_Guide_23.3" in self.path and not (dictionary[i]["chars"][0]["size"] > 19 and dictionary[i]["chars"][0]["size"] < 30):
            i+=2
        return i


    def get_pdf_paragraphs(self,path):
        pdf_to_read = self.extract_all_lines_from_the_doc(path)
        paragraphs = []
        j = 0
        while j < len(pdf_to_read):
            dictionary = pdf_to_read[j]["content"]
            tables = pdf_to_read[j]["tables"]
            i = self.skip_header(dictionary)
            table_count = 0
            while i < len(dictionary):
                # print(f"{dictionary[i]['chars'][0]}")
                if(dictionary[i]["text"].startswith("RESTAPIDeveloperGuide")):
                    i+=1
                    continue
                if (self.check_if_already_in_table(dictionary[i]['chars'][0],tables) == False):
                    p = Paragraph(dictionary[i]["text"],font_style=get_style_of_line(dictionary[i]["chars"][0]["size"],dictionary[i]["chars"][0]["fontname"]),id_=i,page_id=pdf_to_read[j]["page_number"])
                    if(i != len(dictionary)-1):
                        while((dictionary[i+1]["chars"][0]["size"] == dictionary[i]["chars"][-1]["size"] and dictionary[i+1]["chars"][0]["fontname"] == dictionary[i]["chars"][-1]["fontname"]) and self.check_if_already_in_table(dictionary[i+1]['chars'][0],tables) == False):
                            p.text += " " + dictionary[i+1]["text"]
                            i += 1
                    else:
                        p.text = dictionary[i]["text"]
                    #print(f"{dictionary[i]['chars'][0]} : {dictionary[i]['text']}")
                    i += 1
                    # print(f'{p.page_id} : {p.font_style} ->>>>> {p.text}')
                    paragraphs.append(p)
                else:
                    p = Paragraph(table_converter(tables[table_count].extract()),font_style="table",id_=i,page_id=pdf_to_read[j]["page_number"])
                    paragraphs.append(p)
                    i = self.skip_out_table(dictionary,i,tables[table_count])
                    table_count += 1
            j += 1
        paragraphs = self.rearrange_paragraphs(paragraphs)
        return paragraphs
    
    def rearrange_paragraphs(self, paragraphs : [Paragraph]):
        #associate paragraphs with the same font style
        i = 0
        while i < len(paragraphs):
            paragraphs[i] = paragraphs[i].rearrange_paragraph()
            i+=1
        return paragraphs

    def extract_all_lines_from_the_doc(self,path):
        lines_of_doc = []
        with open(path, 'rb') as f:
            reader = pdfp.PDF(f)
            if "Illumio_Core_REST_API_Developer_Guide_23.3" in path:
                skip_table_of_contents = reader.pages[8:]
                j = 0
                while j < len(skip_table_of_contents):
                    lines_of_doc.append({"page_number": j+9, "content": skip_table_of_contents[j].extract_text_lines(), "tables": skip_table_of_contents[j].find_tables()})
                    j += 1
            else:
                for page in reader.pages:
                    lines_of_doc.append({"page_number": page.page_number, "content": page.extract_text_lines(), "tables": page.find_tables()})
        return lines_of_doc

    def check_if_already_in_table(self,line,tables):
        for table in tables:
            if table.bbox[1] <= line["top"] <= table.bbox[3]:
                return True
        return False

    def skip_out_table(self,dictionary,index,table):
        i = index
        while i < len(dictionary):
            if self.check_if_already_in_table(dictionary[i]['chars'][0],tables=[table]) == True:
                i += 1
            else:
                break
        return i