import os
import gradio as gr
import subprocess
import sys
def install(package):
subprocess.check_call([sys.executable, "-m", "pip", "install", package])
install("numpy")
install("torch")
install("transformers")
install("unidecode")
import numpy as np
import torch
from transformers import AutoTokenizer
from transformers import BertForTokenClassification
from collections import Counter
from unidecode import unidecode
import string
import re
auth_token = os.environ.get("AUTH-TOKEN")
tokenizer = AutoTokenizer.from_pretrained("osiria/bert-base-cased-ner-en", use_auth_token=auth_token)
model = BertForTokenClassification.from_pretrained("osiria/bert-base-cased-ner-en", num_labels = 5, use_auth_token=auth_token)
device = torch.device("cpu")
model = model.to(device)
model.eval()
from transformers import pipeline
ner = pipeline('ner', model=model, tokenizer=tokenizer, device=-1)
header = '''--------------------------------------------------------------------------------------------------
D
E
M
O
'''
maps = {"O": "NONE", "PER": "PER", "LOC": "LOC", "ORG": "ORG", "MISC": "MISC", "DATE": "DATE"}
reg_month = "(?:gennaio|febbraio|marzo|aprile|maggio|giugno|luglio|agosto|settembre|ottobre|novembre|dicembre|january|february|march|april|may|june|july|august|september|october|november|december)"
reg_date = "(?:\d{1,2}\°{0,1}|primo|\d{1,2}\º{0,1})" + " " + reg_month + " " + "\d{4}|"
reg_date = reg_date + reg_month + " " + "\d{4}|"
reg_date = reg_date + "\d{1,2}" + " " + reg_month
reg_date = reg_date + "\d{1,2}" + "(?:\/|\.)\d{1,2}(?:\/|\.)" + "\d{4}|"
reg_date = reg_date + "(?<=dal )\d{4}|(?<=al )\d{4}|(?<=nel )\d{4}|(?<=anno )\d{4}|(?<=del )\d{4}|"
reg_date = reg_date + "\d{1,5} a\.c\.|\d{1,5} d\.c\."
map_punct = {"’": "'", "«": '"', "»": '"', "”": '"', "“": '"', "–": "-", "$": ""}
unk_tok = 9005
merge_th_1 = 0.8
merge_th_2 = 0.4
min_th = 0.5
def extract(text):
text = text.strip()
for mp in map_punct:
text = text.replace(mp, map_punct[mp])
text = re.sub("\[\d+\]", "", text)
warn_flag = False
res_total = []
out_text = ""
for p_text in text.split("\n"):
if p_text:
toks = tokenizer.encode(p_text)
if unk_tok in toks:
warn_flag = True
res_orig = ner(p_text, aggregation_strategy = "first")
res_orig = [el for r, el in enumerate(res_orig) if len(el["word"].strip()) > 1]
res = []
for r, ent in enumerate(res_orig):
if r > 0 and ent["score"] < merge_th_1 and ent["start"] <= res[-1]["end"] + 1 and ent["score"] <= res[-1]["score"]:
res[-1]["word"] = res[-1]["word"] + " " + ent["word"]
res[-1]["score"] = merge_th_1*(res[-1]["score"] > merge_th_2)
res[-1]["end"] = ent["end"]
elif r < len(res_orig) - 1 and ent["score"] < merge_th_1 and res_orig[r+1]["start"] <= ent["end"] + 1 and res_orig[r+1]["score"] > ent["score"]:
res_orig[r+1]["word"] = ent["word"] + " " + res_orig[r+1]["word"]
res_orig[r+1]["score"] = merge_th_1*(res_orig[r+1]["score"] > merge_th_2)
res_orig[r+1]["start"] = ent["start"]
else:
res.append(ent)
res = [el for r, el in enumerate(res) if el["score"] >= min_th]
dates = [{"entity_group": "DATE", "score": 1.0, "word": p_text[el.span()[0]:el.span()[1]], "start": el.span()[0], "end": el.span()[1]} for el in re.finditer(reg_date, p_text, flags = re.IGNORECASE)]
res.extend(dates)
res = sorted(res, key = lambda t: t["start"])
res_total.extend(res)
chunks = [("", "", 0, "NONE")]
for el in res:
if maps[el["entity_group"]] != "NONE":
tag = maps[el["entity_group"]]
chunks.append((p_text[el["start"]: el["end"]], p_text[chunks[-1][2]:el["end"]], el["end"], tag))
if chunks[-1][2] < len(p_text):
chunks.append(("END", p_text[chunks[-1][2]:], -1, "NONE"))
chunks = chunks[1:]
n_text = []
for i, chunk in enumerate(chunks):
rep = chunk[0]
if chunk[3] == "PER":
rep = 'ᴘᴇʀ ' + chunk[0] + ''
elif chunk[3] == "LOC":
rep = 'ʟᴏᴄ ' + chunk[0] + ''
elif chunk[3] == "ORG":
rep = 'ᴏʀɢ ' + chunk[0] + ''
elif chunk[3] == "MISC":
rep = 'ᴍɪsᴄ ' + chunk[0] + ''
elif chunk[3] == "DATE":
rep = 'ᴅᴀᴛᴇ ' + chunk[0] + ''
n_text.append(chunk[1].replace(chunk[0], rep))
n_text = "".join(n_text)
if out_text:
out_text = out_text + "
" + n_text
else:
out_text = n_text
tags = [el["word"] for el in res_total if el["entity_group"] not in ['DATE', None]]
cnt = Counter(tags)
tags = sorted(list(set([el for el in tags if cnt[el] > 1])), key = lambda t: cnt[t]*np.exp(-tags.index(t)))[::-1]
tags = [" ".join(re.sub("[^A-Za-z0-9\s]", "", unidecode(tag)).split()) for tag in tags]
tags = ['ᴛᴀɢ ' + el + '' for el in tags]
tags = " ".join(tags)
if tags:
out_text = out_text + "
Tags: " + tags
if warn_flag:
out_text = out_text + "
Warning ⚠️: Unknown tokens detected in text. The model might behave erratically"
return out_text
init_text = '''The American Academy of Arts and Sciences (AAA&S) is one of the oldest learned societies in the United States. It was founded in 1780 during the American Revolution by John Adams, John Hancock, James Bowdoin, Andrew Oliver, and other Founding Fathers of the United States. It is headquartered in Cambridge, Massachusetts.
Membership in the academy is achieved through a thorough petition, review, and election process. The academy's quarterly journal, Dædalus, is published by the MIT Press on behalf of the academy. The academy also conducts multidisciplinary public policy research.
The Academy was established by the Massachusetts legislature on May 4, 1780, charted in order "to cultivate every art and science which may tend to advance the interest, honor, dignity, and happiness of a free, independent, and virtuous people." The sixty-two incorporating fellows represented varying interests and high standing in the political, professional, and commercial sectors of the state. The first class of new members, chosen by the Academy in 1781, included Benjamin Franklin and George Washington as well as several international honorary members.
'''
init_output = extract(init_text)
with gr.Blocks(css="footer {visibility: hidden}", theme=gr.themes.Default(text_size="lg", spacing_size="lg")) as interface:
with gr.Row():
gr.Markdown(header)
with gr.Row():
text = gr.Text(label="Extract entities", lines = 10, value = init_text)
with gr.Row():
with gr.Column():
button = gr.Button("Extract").style(full_width=False)
with gr.Row():
with gr.Column():
entities = gr.Markdown(init_output)
with gr.Row():
with gr.Column():
gr.Markdown("The input examples in this demo are extracted from https://it.wikipedia.org")
button.click(extract, inputs=[text], outputs = [entities])
interface.launch()