#**************** IMPORT PACKAGES ******************** import flask from flask import render_template, jsonify, Flask, redirect, url_for, request, flash from flask_cors import CORS, cross_origin from werkzeug.utils import secure_filename import numpy as np import pytesseract as pt import pdf2image from fpdf import FPDF import re import nltk from nltk.tokenize import sent_tokenize from nltk.tokenize import word_tokenize import os import pdfkit import yake from transformers import AutoTokenizer, AutoModelForPreTraining, AutoModel, AutoConfig from summarizer import Summarizer,TransformerSummarizer from transformers import pipelines #nltk.download('punkt') print("lets go") app = flask.Flask(__name__) app.config["DEBUG"] = True UPLOAD_FOLDER = './pdfs' ALLOWED_EXTENSIONS = {'txt', 'pdf'} app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER #***************** FLASK ***************************** CORS(app) def allowed_file(filename): return '.' in filename and \ filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS #model_name = 'laxya007/gpt2_legal' #model_name = 'facebook/bart-large-cnn' model_name = 'nlpaueb/legal-bert-base-uncased' #The setup of huggingface.co print("lets go") custom_config = AutoConfig.from_pretrained(model_name) print("lets go") custom_config.output_hidden_states=True print("lets go") custom_tokenizer = AutoTokenizer.from_pretrained(model_name) print("lets go") #custom_model = AutoModel.from_pretrained(model_name, config=custom_config) print("lets go") bert_legal_model = Summarizer(custom_model=custom_model, custom_tokenizer=custom_tokenizer) print('Using model {}\n'.format(model_name)) # main index page route @app.route('/') @cross_origin() def index(): return render_template('index.html') @cross_origin() @app.route('/results') def results(): return render_template('results.html') @app.route('/predict', methods=['GET', 'POST']) def uploads(): if request.method == 'GET': # Get the file from post request numsent = int(request.args['number']) text = str(request.args['text']) content = text summary_text = "" for i, paragraph in enumerate(content.split("\n\n")): paragraph = paragraph.replace('\n',' ') paragraph = paragraph.replace('\t','') paragraph = ' '.join(paragraph.split()) # count words in the paragraph and exclude if less than 4 words tokens = word_tokenize(paragraph) # only do real words tokens = [word for word in tokens if word.isalpha()] # print("\nTokens: {}\n".format(len(tokens))) # only do sentences with more than 1 words excl. alpha crap if len(tokens) <= 1: continue # Perhaps also ignore paragraphs with no sentence? sentences = sent_tokenize(paragraph) paragraph = ' '.join(tokens) print("\nParagraph:") print(paragraph+"\n") # T5 needs to have 'summarize' in order to work: # text = "summarize:" + paragraph text = paragraph summary = bert_legal_model(text, min_length = 8, ratio = 0.05) # summary = tokenizer_t5.decode(summary_ids[0], skip_special_tokens=True) summary_text += str(summary) + "\n\n" print("Summary:") print(summary) content2 = content.replace('\n',' ') content2 = content2.replace('\t','') summary = bert_legal_model(content2, min_length = 8, num_sentences=25) # write all to file for inspection and storage all_text = "The Summary-- " + str(summary) + "\n\n\n" \ + "The Larger Summary-- " + str(summary_text) all_text2 = all_text.encode('latin-1', 'replace').decode('latin-1') all_text2 = all_text2.replace('?','.') all_text2 = all_text2.replace('\n',' ') all_text2 = all_text2.replace('..','.') all_text2 = all_text2.replace(',.',',') all_text2 = all_text2.replace('-- ','\n\n\n') pdf = FPDF() # Add a page pdf.add_page() pdf.set_font("Times", size = 12) # open the text file in read mode f = all_text2 # insert the texts in pdf pdf.multi_cell(190, 10, txt = f, align = 'C') # save the pdf with name .pdf pdf.output("./static/legal.pdf") all_text return render_template('results.html') return None @app.route('/predictpdf', methods=['GET', 'POST']) def uploads2(): if request.method == 'POST': # Get the file from post request numsent = int(request.args['number']) if 'file' not in request.files: flash('No file part') return redirect(request.url) file = request.files['file'] # if user does not select file, browser also # submit an empty part without filename if file.filename == '': flash('No selected file') return redirect(request.url) if file and allowed_file(file.filename): filename = "legal.pdf" file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename)) f = request.files['file'] f.save(secure_filename(f.filename)) path = os.getcwd() folder_name = 'pdfs' path = os.path.join(path, folder_name) list_of_files = [] for root, dirs, files in os.walk(path): for file in files: if(file.endswith(".pdf")): # print(os.path.join(root,file)) list_of_files.append(os.path.join(root,file)) print("\nProcessing {} files...\n".format(len(list_of_files))) total_pages = 0 for filename in list_of_files: print(filename) file = os.path.splitext(os.path.basename(filename))[0] pages = pdf2image.convert_from_path(pdf_path=filename, dpi=400, size=(1654,2340)) total_pages += len(pages) print("\nProcessing the next {} pages...\n".format(len(pages))) # Then save all pages as images and convert them to text except the last page # TODO: create this as a function content = "" dir_name = 'images/' + file + '/' os.makedirs(dir_name, exist_ok=True) # If folder doesn't exist, then create it. for i in range(len(pages)-1): pages[i].save(dir_name + str(i) + '.jpg') # OCR the image using Google's tesseract content += pt.image_to_string(pages[i]) summary_text = "" for i, paragraph in enumerate(content.split("\n\n")): paragraph = paragraph.replace('\n',' ') paragraph = paragraph.replace('\t','') paragraph = ' '.join(paragraph.split()) # count words in the paragraph and exclude if less than 4 words tokens = word_tokenize(paragraph) # only do real words tokens = [word for word in tokens if word.isalpha()] # print("\nTokens: {}\n".format(len(tokens))) # only do sentences with more than 1 words excl. alpha crap if len(tokens) <= 1: continue # Perhaps also ignore paragraphs with no sentence? sentences = sent_tokenize(paragraph) paragraph = ' '.join(tokens) print("\nParagraph:") print(paragraph+"\n") # T5 needs to have 'summarize' in order to work: # text = "summarize:" + paragraph text = paragraph summary = bert_legal_model(text, min_length = 8, ratio = 0.05) # summary = tokenizer_t5.decode(summary_ids[0], skip_special_tokens=True) summary_text += str(summary) + "\n\n" print("Summary:") print(summary) content2 = content.replace('\n',' ') content2 = content2.replace('\t','') summary = bert_legal_model(content2, min_length = 8, num_sentences=25) # write all to file for inspection and storage all_text = "The Summary-- " + str(summary) + "\n\n\n" \ + "The Larger Summary-- " + str(summary_text) all_text2 = all_text.encode('latin-1', 'replace').decode('latin-1') all_text2 = all_text2.replace('?','.') all_text2 = all_text2.replace('\n',' ') all_text2 = all_text2.replace('..','.') all_text2 = all_text2.replace(',.',',') all_text2 = all_text2.replace('-- ','\n\n\n') pdf = FPDF() # Add a page pdf.add_page() pdf.set_font("Times", size = 12) # open the text file in read mode f = all_text2 # insert the texts in pdf pdf.multi_cell(190, 10, txt = f, align = 'C') # save the pdf with name .pdf pdf.output("./static/legal.pdf") all_text return render_template('results.html') return None if __name__ == "__main__": app.run(host='0.0.0.0', port=8000, debug=True)