Spaces:
Sleeping
Sleeping
File size: 4,746 Bytes
02ae0bf |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 |
import argparse
import json
import requests
from datetime import datetime
from flask import Flask, jsonify, abort, make_response, request, Response
from flask_cors import CORS
import uuid
import os
import subprocess
import threading
import shutil
import hashlib
import base64
from normalization_utils import DocumentNormalizer
import time
import socket
from urllib.parse import urlparse
import re
# Make Flask application
app = Flask(__name__)
CORS(app)
def bytes_to_base64_string(f_bytes):
return base64.b64encode(f_bytes).decode('ASCII')
def base64_string_to_bytes(base64_string):
return base64.b64decode(base64_string)
def get_md5( file_bytes ):
readable_hash = hashlib.md5(file_bytes).hexdigest()
return readable_hash
def adjust_cite_span( cite_span, cite_span_year_matcher ):
non_cite_text_chars = ",; []()"
try:
start = int(cite_span["start"])
end = int(cite_span["end"])
orig_text = cite_span["text"]
text = cite_span_year_matcher.sub( r"Y\1Y", orig_text )
begin_offset = 0
end_offset = 0
for c in text:
if c in non_cite_text_chars:
start += 1
begin_offset += 1
else:
break
for c in text[::-1]:
if c in non_cite_text_chars:
end -= 1
end_offset -= 1
else:
break
assert start < end
new_cite_span = {
"start":str( start ),
"end":str( end ),
"text":orig_text[ begin_offset:len(text) + end_offset ],
"ref_id":cite_span["ref_id"]
}
except:
new_cite_span = cite_span
return new_cite_span
def parse_pdf_base( pdf_bytes ):
root_dir = "root_dir_" + str(uuid.uuid4())
pdf_dir = root_dir + "/pdf/"
temp_dir = root_dir + "/temp_dir/"
output_dir = root_dir + "/output_dir/"
try:
os.makedirs(pdf_dir)
os.makedirs(temp_dir)
os.makedirs(output_dir)
except:
print("warning: folders exist!")
try:
with open( pdf_dir + "pdf.pdf","wb" ) as f:
f.write(pdf_bytes)
pdf_name = [ pdf_dir+fname for fname in os.listdir( pdf_dir )][0]
subprocess.run( list(map( str, [
"python",
PDF2JSON_HOME+"/doc2json/grobid2json/process_pdf.py",
"-i", pdf_name,
"-t", temp_dir,
"-o", output_dir
] ) ) )
print("PDF parsing done!")
json_name = [ output_dir+fname for fname in os.listdir( output_dir )][0]
parsed_data = json.load(open(json_name))
shutil.rmtree(root_dir)
except:
parsed_data = {}
try:
shutil.rmtree(root_dir)
except:
print("warning: removing temporary folder failed!")
return parsed_data
def convert_pdf_to_json( fbytes, count, conversion_results ):
try:
parsed_data = parse_pdf_base( fbytes )
except:
parsed_data = {}
conversion_results[count] = parsed_data
@app.route('/parse-pdf', methods=['POST'])
def parse_pdf():
try:
pdf_bytes = request.files.get('pdf').read()
parsed_data = parse_pdf_base( pdf_bytes )
except:
parsed_data = {}
return {"response":parsed_data}, 201
@app.route('/parse-and-normalize-pdf', methods=['POST'])
def parse_and_normalize_pdf():
global doc_normalizer, cite_span_year_matcher
try:
pdf_bytes = request.files.get('pdf').read()
parsed_data = parse_pdf_base( pdf_bytes )
parsed_data = doc_normalizer.normalize( parsed_data )
""" Clean the citation marker text """
for sec in parsed_data["Content"]["Abstract_Parsed"] + parsed_data["Content"]["Fullbody_Parsed"]:
for para in sec["section_text"]:
for sen in para["paragraph_text"]:
sen["cite_spans"] = [ adjust_cite_span( cite_span, cite_span_year_matcher ) for cite_span in sen["cite_spans"] ]
except:
parsed_data = {}
return {"response":parsed_data}, 201
PDF2JSON_HOME = os.getenv("PDF2JSON_HOME")
doc_normalizer = DocumentNormalizer( "./json_schema.json" )
cite_span_year_matcher = re.compile( "\((\d{4})\)" )
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument( "-flask_port", type = int, default = 8060 )
args = parser.parse_args()
print("\n\nWaiting for requests...")
sem = threading.Semaphore()
app.run(host='0.0.0.0', port=args.flask_port, threaded = True, debug = True) |