Spaces:
Runtime error
Runtime error
Roland Ding
7.7.18.49 updated the application.py and features.py for including existing table extraction and remove anything from article after discussion session. updated the interface to allow unequal heights of components and added original tables in study page.
844a2ba
import json | |
from application import * | |
from pdfminer.high_level import extract_text | |
from pdfminer.pdfparser import PDFParser | |
from pdfminer.pdfdocument import PDFDocument | |
''' | |
following functions are for file manipulation | |
''' | |
keyword_search = lambda kw, text: kw.lower() in text.lower() | |
list_or = lambda l: sum(l)>0 | |
list_and = lambda l: sum(l)==len(l) | |
# read pdf file and return text | |
def read_pdf(file_path): | |
''' | |
this function read the pdf file and return the text | |
Parameters | |
----------''' | |
# open the pdf file | |
if type(file_path) is str: | |
filename = file_path | |
pdfFileObj = open(file_path, 'rb') | |
# elif type(file_path) is tempfile._TemporaryFileWrapper: | |
else: | |
filename = file_path.name | |
pdfFileObj = open(file_path.name, 'rb') | |
# # create a pdf reader object | |
# pdfReader = PyPDF2.PdfReader(pdfFileObj) | |
# # get the number of pages in the pdf file | |
# num_pages = len(pdfReader.pages) | |
# # create an empty string | |
# text = '' | |
# # iterate through all the pages | |
# for page_num in range(num_pages): | |
# page_obj = pdfReader.pages[page_num] | |
# text += page_obj.extract_text () | |
text = extract_text(pdfFileObj) | |
text = remove_symbols(text) | |
# meta = pdfReader.metadata | |
parser = PDFParser(pdfFileObj) | |
doc = PDFDocument(parser) | |
meta = doc.info | |
# close the pdf file object | |
pdfFileObj.close() | |
return text, meta | |
''' | |
following functions are for format standard response | |
''' | |
# format standard response for status code and data | |
def format_response(code,data): | |
return { | |
"statusCode":code, | |
"headers":{ | |
"Access-Control-Allow-Origin": "*", | |
"Content-Type": "application/json" | |
}, | |
"body":json.dumps(data), | |
"isBase64Encoded": False | |
} | |
''' | |
following functions are for string manipulation | |
''' | |
# format text output by removing excessive characters | |
def format_text(text,remove_char_ls = ["\\n--\\n","\\n\\n","\n"]): | |
for c in remove_char_ls: | |
text = text.replace(c,"") | |
return text | |
# function to remove symbols that are not in unicode | |
def remove_symbols(text): | |
# remove symbols that are not in unicode | |
text = text.encode("ascii", "ignore").decode() | |
# remove the break word new line return | |
text = text.replace('-\n', '') | |
return text | |
def str_to_tuple(s): | |
return tuple(s.replace("(","").replace(")","").split(",")) | |
''' | |
following functions are for dynamodb data manipulation | |
''' | |
# convert dynamodb map to python dictionary | |
def db_map_to_py_dict(db_map): | |
py_dict = {} | |
for k,i in db_map.items(): | |
for l,v in i.items(): | |
if l == "M": | |
py_dict[k] = db_map_to_py_dict(v) | |
elif l == "S": | |
py_dict[k] = v | |
elif l == "N": | |
py_dict[k] = int(v) if float(v)%1 ==0 else float(v) | |
elif l == "L": | |
py_dict[k] = db_list_to_py_list(v) | |
elif l == "BS": | |
py_dict[k] = v | |
elif l == "BOOL": | |
py_dict[k] = v | |
elif l =="NULL": | |
py_dict[k] = None | |
else: | |
py_dict[k] = v | |
return py_dict | |
# convert python dictionary to dynamodb map | |
def py_dict_to_db_map(py_dict): | |
db_map = {} | |
for key,value in py_dict.items(): | |
key = str(key) | |
if type(value) is str: | |
db_map[key] = {"S":value} | |
elif type(value) is int or type(value) is float: | |
db_map[key] = {"N":value} | |
elif type(value) is dict: | |
db_map[key] = {"M":py_dict_to_db_map(value)} | |
elif type(value) is list: | |
db_map[key] = {"L":py_list_to_db_list(value)} | |
elif type(value) is bytes: | |
db_map[key] = {"BS":value} | |
elif type(value) is bool: | |
db_map[key] = {"BOOL":value} | |
elif value is None: | |
db_map[key] = {"NULL":True} | |
return db_map | |
# convert dynamodb list to python list | |
def db_list_to_py_list(db_list): | |
py_list = [] | |
for d in db_list: | |
for t,v in d.items(): | |
if t == "M": | |
py_list.append(db_map_to_py_dict(v)) | |
elif t == "L": | |
py_list.append(db_list_to_py_list(v)) | |
elif t =="N" or t =="S" or t =="B" or t =="BOOL" or t =="NULL" or t =="SS" or t =="NS" or t =="BS": | |
py_list.append(v) | |
else: | |
py_list.append(db_map_to_py_dict(v)) | |
return py_list | |
# convert python list to dynamodb list | |
def py_list_to_db_list(py_list): | |
db_list = [] | |
for value in py_list: | |
if type(value) is str: | |
item = {"S":value} | |
elif type(value) is int or type(value) is float: | |
item = {"N":value} | |
elif type(value) is dict: | |
item = {"M":py_dict_to_db_map(value)} | |
# item = py_dict_to_db_map(value) | |
elif type(value) is list: | |
item = {"L":py_list_to_db_list(value)} | |
elif type(value) is tuple: | |
item = {"L":py_list_to_db_list(value)} | |
elif type(value) is bytes: | |
item = {"BS":value} | |
elif type(value) is bool: | |
item = {"BOOL":value} | |
elif value is None: | |
item = {"NULL":True} | |
db_list.append(item) | |
return db_list | |
''' | |
following functions are used for business logic. (to be moved to business logic layer) | |
''' | |
# function to calculate the estimated cost of the translation | |
def est_cost(n_tokens,rate): | |
return round(rate*n_tokens/1000,4) | |
def replace_symbols(s): | |
''' | |
this function replace symbols in the string to comply with file names | |
Parameters | |
---------- | |
s : str | |
string to be replaced | |
Returns | |
------- | |
str | |
replaced string | |
''' | |
s = s.replace(" ","_") | |
s = s.replace(",","") | |
s = s.replace(".","") | |
s = s.replace("-","_") | |
s = s.replace("(","") | |
s = s.replace(")","") | |
s = s.replace("/","_") | |
s = s.replace(":","") | |
s = s.replace(";","") | |
s = s.replace("'","") | |
s = s.replace('"',"") | |
return remove_symbols(s) |