import io
import gradio as gr
import spacy
from spacy import displacy
from bib_tokenizers import create_references_tokenizer
nlp = None
nlp = spacy.load("spacy-pipelines/model-best")
# return score for each token:
# with threshold set to zero each suggested span is returned, and span == token,
# because suggester is configured to suggest spans with len(span) == 1:
# [components.spancat.suggester]
# @misc = "spacy.ngram_suggester.v1"
# sizes = [1]
nlp.get_pipe("spancat").cfg["threshold"] = 0.0 # see )
print(nlp.get_pipe("spancat").cfg)
def create_bib_item_start_scorer_for_doc(doc, spanskey="sc"):
span_group = doc.spans[spanskey]
assert not span_group.has_overlap
assert len(span_group) == len(
doc
), "Check suggester config and the spancat threshold to make sure that spangroup contains single token span for each token"
spans_idx = {
offset: span.start
for span in span_group
for offset in range(span.start_char, span.end_char + 1)
}
def scorer(char_offset, fuzzy_in_tokens=(0, 0)):
i = spans_idx[char_offset]
span = span_group[i]
assert i == span.start
# fuzzines might improve fault tolerance if the model made a small mistake,
# e.g., if a number from prev line is classified as "citation number",
# see example at https://www.deeplearningbook.org/contents/bib.html
# if fuzzy == (0,0), it return score for the selected span only
return span, max(
span_group.attrs["scores"][i]
for i in range(i - fuzzy_in_tokens[0], i + fuzzy_in_tokens[1] + 1)
if i >= 0 and i < len(doc.text)
)
return scorer
nlp_blank = spacy.blank("en")
nlp_blank.tokenizer = create_references_tokenizer()(nlp_blank)
def split_up_references(
references: str, is_eol_mode=False, nlp=nlp, nlp_blank=nlp_blank
):
"""
Args:
references - a references section, ideally without a header
nlp - a model that splits up references into separate sentences
nlp_blank - a blank nlp with the same tokenizer/language
"""
normalized_references = references.replace("\n", " ")
# the model trained on 'normalized' references - the ones without '\n'
doc = nlp(normalized_references)
# 'transfer' annotations from doc without '\n' (normalized references) to the target doc created from the original input string
# the problem here is that docs differ in a number of tokens
# however, it should be easy to align on characters level because both '\n' and ' ' are whitespace, so spans have the same boundaries
target_doc = nlp_blank(references)
target_tokens_idx = {
offset: t.i for t in target_doc for offset in range(t.idx, t.idx + len(t))
}
# senter annotations
for i, t in enumerate(target_doc):
t.is_sent_start = i == 0
if is_eol_mode:
# use SpanCat scores to set sentence boundaries on the target doc
char_offset = 0
f = io.StringIO(references)
token_scorer = create_bib_item_start_scorer_for_doc(doc)
threshold = 0.2
lines = [line for line in f]
lines_len_in_tokens = [
_len for _len in map(lambda line: len(nlp_blank.tokenizer(line)), lines)
]
for line_num, line in enumerate(lines):
fuzzy = (
0 if line_num == 0 else lines_len_in_tokens[line_num - 1] // 4,
lines_len_in_tokens[line_num] // 4,
)
span, score = token_scorer(char_offset, fuzzy_in_tokens=fuzzy)
print(span, score)
if score > threshold:
target_doc[target_tokens_idx[char_offset]].is_sent_start = True
char_offset += len(line)
else:
# copy SentenceRecognizer annotations from doc without '\n' to the target doc
for t in doc:
if t.is_sent_start:
target_doc[target_tokens_idx[t.idx]].is_sent_start = True
# copy ner annotations:
target_doc.ents = [
target_doc.char_span(ent.start_char, ent.end_char, ent.label_)
for ent in doc.ents
# remove entities crossing sentence boundaries
if not any([t.is_sent_start for t in ent if t.i != ent.start])
]
return target_doc
def text_analysis(text, is_eol_mode):
html = ""
doc_with_linebreaks = split_up_references(
text, is_eol_mode=is_eol_mode, nlp=nlp, nlp_blank=nlp_blank
)
for i, sent in enumerate(doc_with_linebreaks.sents):
bib_item_doc = sent.as_doc()
bib_item_doc.user_data = {"title": f"***** Bib Item {i+1}: *****"}
html += displacy.render(bib_item_doc, style="ent")
html = (
"
"
+ html
+ "
"
)
return html
demo = gr.Blocks()
with demo:
textbox = gr.components.Textbox(
label="Unparsed Bibliography Section",
placeholder="Enter bibliography here...",
lines=20,
)
is_eol_mode = gr.components.Checkbox(
label="a line does not contain more than one bibitem (Multiline bibitems are supported regardless of this choice)"
)
html = gr.components.HTML(label="Parsed Bib Items")
textbox.change(fn=text_analysis, inputs=[textbox, is_eol_mode], outputs=[html])
is_eol_mode.change(fn=text_analysis, inputs=[textbox, is_eol_mode], outputs=[html])
gr.Examples(
examples=[
[
"""[1] B. Foxman, R. Barlow, H. D'Arcy, B. Gillespie, and J. D. Sobel, "Urinary tract infection: self-reported incidence and associated costs," Ann Epidemiol, vol. 10, pp. 509-515, 2000. [2] B. Foxman, "Epidemiology of urinary tract infections: incidence, morbidity, and economic costs," Am J Med, vol. 113, pp. 5-13, 2002. [3] L. Nicolle, "Urinary tract infections in the elderly," Clin Geriatr Med, vol. 25, pp. 423-436, 2009."""
],
[
"""Barth, Fredrik, ed.
1969 Ethnic groups and boundaries: The social organization of culture difference. Oslo: Scandinavian University Press.
Bondokji, Neven
2016 The Expectation Gap in Humanitarian Operations: Field Perspectives from Jordan. Asian Journal of Peace Building 4(1):1-28.
Bourdieu, Pierre
The forms of capital In Handbook of Theory and Research for the Sociology of Education. J. Richardson, ed. Pp. 241-258. New York: Greenwood Publishesrs.
Carrion, Doris
2015 Are Syrian Refguees a Security Threat to the MIddle East Vol. 2016. London Reuters.
CFR
2016 The Global Humanitarian Regime: Priorities and Prospects for Reform. Council on Foerign Relations, International Institutues and Global Governance Program"""
],
[
"""(2) Hofmann, M.H. et al. Aberrant splicing caused by single nucleotide polymorphism c.516G>T [Q172H], a marker of CYP2B6*6, is responsible for decreased expression and activity of CYP2B6 in liver. J Pharmacol Exp Ther 325, 284-92 (2008).
(3) Zanger, U.M. & Klein, K. Pharmacogenetics of cytochrome P450 2B6 (CYP2B6): advances on polymorphisms, mechanisms, and clinical relevance. Front Genet 4, 24 (2013).
(4) Holzinger, E.R. et al. Genome-wide association study of plasma efavirenz pharmacokinetics in AIDS Clinical Trials Group protocols implicates several CYP2B6 variants. Pharmacogenet Genomics 22, 858-67 (2012).
"""
],
[
"""[Ein05] Albert Einstein. Zur Elektrodynamik bewegter K ̈orper. (German)
[On the electrodynamics of moving bodies]. Annalen der Physik,
322(10):891–921, 1905.
[GMS93] Michel Goossens, Frank Mittelbach, and Alexander Samarin. The LATEX Companion. Addison-Wesley, Reading, Massachusetts, 1993.
[Knu] Donald Knuth. Knuth: Computers and typesetting."""
],
[
"""References
Bartkiewicz, A., Szymczak, M., Cohen, R. J., & Richards, A. M. S. 2005, MN- RAS, 361, 623
Bartkiewicz, A., Szymczak, M., & van Langevelde, H. J. 2016, A&A, 587, A104
Benjamin, R. A., Churchwell, E., Babler, B. L., et al. 2003, PASP, 115, 953
Beuther, H., Mottram, J. C., Ahmadi, A., et al. 2018, A&A, 617, A100
Beuther, H., Walsh, A. J., Thorwirth, S., et al. 2007, A&A, 466, 989
Brogan, C. L., Hunter, T. R., Cyganowski, C. J., et al. 2011, ApJ, 739, L16
Brown, A. T., Little, L. T., MacDonald, G. H., Riley, P. W., & Matheson, D. N.
1981, MNRAS, 195, 607
Brown, R. D. & Cragg, D. M. 1991, ApJ, 378, 445
Carrasco-González, C., Sanna, A., Rodríguez-Kamenetzky, A., et al. 2021, ApJ,
914, L1
Cesaroni, R., Walmsley, C. M., & Churchwell, E. 1992, A&A, 256, 618
Cheung, A. C., Rank, D. M., Townes, C. H., Thornton, D. D., & Welch, W. J.
1968, Phys. Rev. Lett., 21, 1701
Churchwell, E., Babler, B. L., Meade, M. R., et al. 2009, PASP, 121, 213
Cohen, R. J. & Brebner, G. C. 1985, MNRAS, 216, 51P
Comito, C., Schilke, P., Endesfelder, U., Jiménez-Serra, I., & Martín-Pintado, J.
2007, A&A, 469, 207
Curiel, S., Ho, P. T. P., Patel, N. A., et al. 2006, ApJ, 638, 878
Danby, G., Flower, D. R., Valiron, P., Schilke, P., & Walmsley, C. M. 1988,
MNRAS, 235, 229
De Buizer, J. M., Liu, M., Tan, J. C., et al. 2017, ApJ, 843, 33
De Buizer, J. M., Radomski, J. T., Telesco, C. M., & Piña, R. K. 2003, ApJ, 598,
1127
Dzib, S., Loinard, L., Rodríguez, L. F., Mioduszewski, A. J., & Torres, R. M.
2011, ApJ, 733, 71
Flower, D. R., Offer, A., & Schilke, P. 1990, MNRAS, 244, 4P
Galván-Madrid, R., Keto, E., Zhang, Q., et al. 2009, ApJ, 706, 1036"""
],
],
inputs=textbox,
)
demo.launch(share=False, server_name="0.0.0.0", server_port=7080)