postprocessing + new samples

app.py

@@ -1,15 +1,23 @@
 import io
+import logging
+import timeit
+from typing import Optional
 
 import gradio as gr
 import numpy as np
 import spacy
 from spacy import displacy
+from spacy.matcher import Matcher
 from spacy.training import Example
 
 from bib_tokenizers import create_references_tokenizer
 from schema import spankey_sentence_start, tags_ent
 
 
+logging.basicConfig(level=logging.INFO)
+log = logging.getLogger(__name__)
+_LOG_STR_LEN = 16
+
 nlp = spacy.load("en_bib_references_trf")
 # return score for each token:
 # with threshold set to zero each suggested span is returned, and span == token,
@@ -18,7 +26,7 @@ nlp = spacy.load("en_bib_references_trf")
 #     @misc = "spacy.ngram_suggester.v1"
 #     sizes = [1]
 nlp.get_pipe("spancat").cfg["threshold"] = 0.0  #  see )
-print(nlp.get_pipe("spancat").cfg)
+log.info("spancat config: %s", nlp.get_pipe("spancat").cfg)
 
 
 def create_bib_item_start_scorer_for_doc(doc):
@@ -42,7 +50,7 @@ def create_bib_item_start_scorer_for_doc(doc):
         return span, max(
             span_group.attrs["scores"][i]
             for i in range(i - fuzzy_in_tokens[0], i + fuzzy_in_tokens[1] + 1)
-            if i >= 0 and i < len(doc.text)
+            if i >= 0 and i < len(doc)
         )
 
     return scorer
@@ -50,10 +58,36 @@ def create_bib_item_start_scorer_for_doc(doc):
 
 nlp_blank = spacy.blank("en")
 nlp_blank.tokenizer = create_references_tokenizer()(nlp_blank)
+# nlp_blank.tokenizer = nlp.tokenizer
+
+
+def _tokenize_test(nlp):
+    _text = """MNRAS, 216, 51P
+Comito"""
+    tokens = [f"'{t}'" for t in nlp(_text)]
+    log.info("tokens: %s", tokens)
+    return tokens
+
+
+assert len(_tokenize_test(nlp)) == len(
+    _tokenize_test(nlp_blank)
+), "Check that the same tokenizer is used for both: trained model (in its config) and nlp_blank"
+
+
+def _token_index_in_norm_doc(
+    token_index_in_target_doc: int, alignment_data: np.ndarray
+) -> Optional[int]:
+
+    index_in_norm_doc = np.where(alignment_data == token_index_in_target_doc)
+    if type(index_in_norm_doc) == tuple:
+        index_in_norm_doc = index_in_norm_doc[0]  # depends on numpy version...
+
+        if index_in_norm_doc.size > 0:
+            return index_in_norm_doc[0].item()
 
 
 def split_up_references(
-    references: str, is_eol_mode=False, ner=True, nlp=nlp, nlp_blank=nlp_blank
+    references: str, is_eol_mode=True, ner=True, nlp=nlp, nlp_blank=nlp_blank
 ):
     """
     Args:
@@ -62,6 +96,12 @@ def split_up_references(
         nlp_blank - a blank nlp with the same tokenizer/language
     """
 
+    _timeit_start = timeit.default_timer()
+    log.info(
+        "start processing: '%s...'",
+        references[: _LOG_STR_LEN if len(references) > _LOG_STR_LEN else references],
+    )
+
     target_doc = nlp_blank(references)
     target_tokens_idx = {
         offset: t.i for t in target_doc for offset in range(t.idx, t.idx + len(t))
@@ -84,6 +124,12 @@ def split_up_references(
     # extremely useful spacy API for alignment normalized and target(created from non-modified input) docs
     example = Example(target_doc, norm_doc)
 
+    # copy ner annotations:
+    for label in tags_ent:
+        target_doc.vocab[label]
+    target_doc.ents = example.get_aligned_spans_y2x(norm_doc.ents)
+
+    # set senter annotations
     if is_eol_mode:
         alignment_data = example.alignment.y2x.data
 
@@ -92,9 +138,21 @@ def split_up_references(
         for i, t in enumerate(target_doc):
             t.is_sent_start = i == 0
 
-        char_offset = 0
         token_scorer = create_bib_item_start_scorer_for_doc(norm_doc)
+
+        def target_doc_token_scorer(token_index_in_target_doc):
+            index_in_norm_doc = _token_index_in_norm_doc(
+                token_index_in_target_doc, alignment_data
+            )
+            if index_in_norm_doc is not None:
+                span, score = token_scorer(index_in_norm_doc)
+                # print(span, score, index_in_norm_doc)
+                return score
+            return 0.0
+
         threshold = 0.5
+
+        char_offset = 0
         for line_num, line in enumerate(lines):
             if not line.strip():
                 # ignore empty line
@@ -109,39 +167,120 @@ def split_up_references(
             ):
                 token_index_in_target_doc += 1
 
-            index_in_norm_doc = np.where(alignment_data == token_index_in_target_doc)
-            if type(index_in_norm_doc) == tuple:
-                index_in_norm_doc = index_in_norm_doc[0]  # depends on numpy version...
-
-            if index_in_norm_doc.size > 0:
-                index_in_norm_doc = index_in_norm_doc[0].item()
-                span, score = token_scorer(index_in_norm_doc)
-                print(span, score, index_in_norm_doc)
-                if score > threshold:
-                    target_doc[target_tokens_idx[char_offset]].is_sent_start = True
+            score = target_doc_token_scorer(token_index_in_target_doc)
+            if score > threshold:
+                target_doc[target_tokens_idx[char_offset]].is_sent_start = True
 
             char_offset += len(line)
+
+        _level_off_references(target_doc, target_doc_token_scorer)
     else:
         # copy SentenceRecognizer annotations from doc without '\n' to the target doc
         sent_start = example.get_aligned("SENT_START")
         for i, t in enumerate(target_doc):
             target_doc[i].is_sent_start = sent_start[i] == 1
 
-    # copy ner annotations:
-    for label in tags_ent:
-        target_doc.vocab[label]
-    target_doc.ents = example.get_aligned_spans_y2x(norm_doc.ents)
-
+    log.info(
+        "done: '%s...', elapsed: %s",
+        references[: _LOG_STR_LEN if len(references) > _LOG_STR_LEN else references],
+        timeit.default_timer() - _timeit_start,
+    )
     return target_doc
 
 
-def text_analysis(text, is_eol_mode):
+def _level_off_references(doc, token_scorer):
+    """
+    Problem:
+    if a model that predicts the reference boundaries was .99 accurate,
+    the success rate for real papers would be still relative low
+    given that a typical bibliography consists of dozens of references.
+
+    This function attemps to detect references that contain more lines than
+    others and split them somehow... The result will not neccessary be better.
+    """
+
+    lengths = np.array([len(ref.text.strip().split("\n")) for ref in doc.sents])
+    median = np.median(lengths)
+    mean = np.mean(lengths)
+    sigma = np.std(
+        lengths
+    )  # read this: https://stackoverflow.com/questions/27600207/why-does-numpy-std-give-a-different-result-to-matlab-std
+
+    log.info("median: %s, mean: %s, sigma: %s", median, mean, sigma)
+    if sigma == 0.0:
+        return
+
+    sent_starts = []
+    matcher = Matcher(nlp.vocab)
+    pattern = [
+        # {"TEXT": {"REGEX": "^(.*)(\\n)+(.*)$"}, "IS_SPACE": True},
+        {"TEXT": {"REGEX": "^(.*\\n.*)+$"}, "IS_SPACE": True},
+        {"IS_SPACE": True, "OP": "*"},
+        {"IS_SPACE": False},
+    ]
+    matcher.add("line_start", [pattern])
+    for n, ref in enumerate(doc.sents):
+        # print([f"'{t}'" for t in ref])
+        surprising = (lengths[n] - mean) / sigma
+        if surprising > 1.6:
+            log.info("surprising: %s: %s", surprising, ref.text[:_LOG_STR_LEN])
+            scores = [token_scorer(t.i) for t in ref]
+            median_score = np.median(scores)
+            # check each first non-space token on each line
+            start = None  # next reference start is we decided to splip up the ref span
+            for _, eol, token_i_after_eol in matcher(ref):
+                i = token_i_after_eol - 1
+                # using the predicted spancat score
+                log.info(
+                    "line start: token=%s, score=%s, median_score=%s, ahead=%s",
+                    ref[i],
+                    scores[i],
+                    median_score,
+                    len(ref[token_i_after_eol:]),
+                )
+                # TODO: play with softmax temperature: find a way to get activations:
+                # here we have an activated neuron in the softmax input, but corresponding sofmax output is still too low
+                if scores[i] > 10 * median_score and len(ref[token_i_after_eol:]) > 10:
+                    sent_starts.append(ref[i])
+                    start = i
+                    continue
+
+                # using ner output:
+                # an edge case if newx line starts with citation number of namnes and
+                # pref libes already contain names and title
+                before_eol_ents = [
+                    ent.label_ for ent in ref[0 if start is None else start : eol].ents
+                ]
+                # 2 entities after eol, if any
+                after_eol_ents = [ent.label_ for ent in ref[eol:].ents][:2]
+                if (
+                    set(before_eol_ents) & set(["issued", "title", "container-title"])
+                    and set(before_eol_ents) & set(["family", "given"])
+                    and set(after_eol_ents)
+                    & set(
+                        [
+                            "family",
+                            "given",
+                            "citation-number",
+                            "citation-label",
+                        ]
+                    )
+                ):
+                    log.info("splitting up using NER predictions: %s", ref[i])
+                    sent_starts.append(ref[i])
+                    start = i
+
+    for t in sent_starts:
+        t.is_sent_start = True
+
+
+def text_analysis(text: str, more_than_one_ref_per_line: bool):
 
     if not text or not text.strip():
         return "<div style='max-width:100%; overflow:auto; color:grey'><p>Unparsed Bibliography Section is empty</p></div>"
 
     doc_with_linebreaks = split_up_references(
-        text, is_eol_mode=is_eol_mode, nlp=nlp, nlp_blank=nlp_blank
+        text, is_eol_mode=not more_than_one_ref_per_line, nlp=nlp, nlp_blank=nlp_blank
     )
 
     html = ""
@@ -198,15 +337,27 @@ with demo:
         placeholder="Enter bibliography here...",
         lines=20,
     )
-    is_eol_mode = gr.components.Checkbox(
-        label="a line does not contain more than one bibitem (Multiline bibitems are supported regardless of this choice)"
+    more_than_one_ref_per_line = gr.components.Checkbox(
+        value=False,
+        label="My bibliography may contain more than one reference per line - the model will make a prediction for each token: more predictions, more chances to make a mistake",
     )
     html = gr.components.HTML(label="Parsed Bib Items")
-    textbox.change(fn=text_analysis, inputs=[textbox, is_eol_mode], outputs=[html])
-    is_eol_mode.change(fn=text_analysis, inputs=[textbox, is_eol_mode], outputs=[html])
+    textbox.change(
+        fn=text_analysis, inputs=[textbox, more_than_one_ref_per_line], outputs=[html]
+    )
+    more_than_one_ref_per_line.change(
+        fn=text_analysis, inputs=[textbox, more_than_one_ref_per_line], outputs=[html]
+    )
 
     gr.Examples(
         examples=[
+            [
+                """[Ein05] Albert Einstein. Zur Elektrodynamik bewegter K ̈orper. (German)
+[On the electrodynamics of moving bodies]. Annalen der Physik,
+322(10):891–921, 1905. 
+[GMS93] Michel Goossens, Frank Mittelbach, and Alexander Samarin. The LATEX Companion. Addison-Wesley, Reading, Massachusetts, 1993. 
+[Knu] Donald Knuth. Knuth: Computers and typesetting."""
+            ],
             [
                 """[1] B. Foxman, R. Barlow, H. D'Arcy, B. Gillespie, and J. D. Sobel, "Urinary tract infection: self-reported incidence and associated costs," Ann Epidemiol, vol. 10, pp. 509-515, 2000. [2] B. Foxman, "Epidemiology of urinary tract infections: incidence, morbidity, and economic costs," Am J Med, vol. 113, pp. 5-13, 2002. [3] L. Nicolle, "Urinary tract infections in the elderly," Clin Geriatr Med, vol. 25, pp. 423-436, 2009."""
             ],
@@ -228,43 +379,82 @@ CFR
 (4) Holzinger, E.R. et al. Genome-wide association study of plasma efavirenz pharmacokinetics in AIDS Clinical Trials Group protocols implicates several CYP2B6 variants. Pharmacogenet Genomics  22, 858-67 (2012).
 """
             ],
-            [
-                """[Ein05] Albert Einstein. Zur Elektrodynamik bewegter K ̈orper. (German)
-[On the electrodynamics of moving bodies]. Annalen der Physik,
-322(10):891–921, 1905. 
-[GMS93] Michel Goossens, Frank Mittelbach, and Alexander Samarin. The LATEX Companion. Addison-Wesley, Reading, Massachusetts, 1993. 
-[Knu] Donald Knuth. Knuth: Computers and typesetting."""
+            [  # https://arxiv.org/pdf/1910.01108v4.pdf
+                """Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. Bert: Pre-training of deep bidirectional transformers for language understanding. In NAACL-HLT, 2018.
+Alec Radford, Jeffrey Wu, Rewon Child, David Luan, Dario Amodei, and Ilya Sutskever. Language models are unsupervised multitask learners. 2019.
+Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar S. Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke S. Zettlemoyer, and Veselin Stoyanov. Roberta: A robustly optimized bert pretraining approach. ArXiv, abs/1907.11692, 2019.
+Roy Schwartz, Jesse Dodge, Noah A. Smith, and Oren Etzioni. Green ai. ArXiv, abs/1907.10597, 2019. Emma Strubell, Ananya Ganesh, and Andrew McCallum. Energy and policy considerations for deep learning in
+nlp. In ACL, 2019.
+Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser,
+and Illia Polosukhin. Attention is all you need. In NIPS, 2017.
+Thomas Wolf, Lysandre Debut, Victor Sanh, Julien Chaumond, Clement Delangue, Anthony Moi, Pierric Cistac, Tim Rault, Rémi Louf, Morgan Funtowicz, and Jamie Brew. Transformers: State-of-the-art natural language processing, 2019.
+Cristian Bucila, Rich Caruana, and Alexandru Niculescu-Mizil. Model compression. In KDD, 2006.
+Geoffrey E. Hinton, Oriol Vinyals, and Jeffrey Dean. Distilling the knowledge in a neural network. ArXiv,
+abs/1503.02531, 2015.
+Yukun Zhu, Ryan Kiros, Richard S. Zemel, Ruslan Salakhutdinov, Raquel Urtasun, Antonio Torralba, and Sanja Fidler. Aligning books and movies: Towards story-like visual explanations by watching movies and reading books. 2015 IEEE International Conference on Computer Vision (ICCV), pages 19–27, 2015.
+Alex Wang, Amanpreet Singh, Julian Michael, Felix Hill, Omer Levy, and Samuel R. Bowman. Glue: A multi-task benchmark and analysis platform for natural language understanding. In ICLR, 2018.
+Matthew E. Peters, Mark Neumann, Mohit Iyyer, Matt Gardner, Christopher Clark, Kenton Lee, and Luke Zettlemoyer. Deep contextualized word representations. In NAACL, 2018.
+Alex Wang, Ian F. Tenney, Yada Pruksachatkun, Katherin Yu, Jan Hula, Patrick Xia, Raghu Pappagari, Shuning Jin, R. Thomas McCoy, Roma Patel, Yinghui Huang, Jason Phang, Edouard Grave, Najoung Kim, Phu Mon Htut, Thibault F’evry, Berlin Chen, Nikita Nangia, Haokun Liu, Anhad Mohananey, Shikha Bordia, Nicolas Patry, Ellie Pavlick, and Samuel R. Bowman. jiant 1.1: A software toolkit for research on general-purpose text understanding models. http://jiant.info/, 2019.
+Andrew L. Maas, Raymond E. Daly, Peter T. Pham, Dan Huang, Andrew Y. Ng, and Christopher Potts. Learning word vectors for sentiment analysis. In ACL, 2011.
+Pranav Rajpurkar, Jian Zhang, Konstantin Lopyrev, and Percy Liang. Squad: 100, 000+ questions for machine comprehension of text. In EMNLP, 2016."""
             ],
-            [
-                """References.
-Bartkiewicz, A., Szymczak, M., Cohen, R. J., & Richards, A. M. S. 2005, MN- RAS, 361, 623
-Bartkiewicz, A., Szymczak, M., & van Langevelde, H. J. 2016, A&A, 587, A104
-Benjamin, R. A., Churchwell, E., Babler, B. L., et al. 2003, PASP, 115, 953 
-Beuther, H., Mottram, J. C., Ahmadi, A., et al. 2018, A&A, 617, A100
-Beuther, H., Walsh, A. J., Thorwirth, S., et al. 2007, A&A, 466, 989
-Brogan, C. L., Hunter, T. R., Cyganowski, C. J., et al. 2011, ApJ, 739, L16
-Brown, A. T., Little, L. T., MacDonald, G. H., Riley, P. W., & Matheson, D. N.
-1981, MNRAS, 195, 607
-Brown, R. D. & Cragg, D. M. 1991, ApJ, 378, 445
-Carrasco-González, C., Sanna, A., Rodríguez-Kamenetzky, A., et al. 2021, ApJ,
-914, L1
-Cesaroni, R., Walmsley, C. M., & Churchwell, E. 1992, A&A, 256, 618
-Cheung, A. C., Rank, D. M., Townes, C. H., Thornton, D. D., & Welch, W. J.
-1968, Phys. Rev. Lett., 21, 1701
-Churchwell, E., Babler, B. L., Meade, M. R., et al. 2009, PASP, 121, 213
-Cohen, R. J. & Brebner, G. C. 1985, MNRAS, 216, 51P
-Comito, C., Schilke, P., Endesfelder, U., Jiménez-Serra, I., & Martín-Pintado, J.
-2007, A&A, 469, 207
-Curiel, S., Ho, P. T. P., Patel, N. A., et al. 2006, ApJ, 638, 878
-Danby, G., Flower, D. R., Valiron, P., Schilke, P., & Walmsley, C. M. 1988,
-MNRAS, 235, 229
-De Buizer, J. M., Liu, M., Tan, J. C., et al. 2017, ApJ, 843, 33
-De Buizer, J. M., Radomski, J. T., Telesco, C. M., & Piña, R. K. 2003, ApJ, 598,
-1127
-Dzib, S., Loinard, L., Rodríguez, L. F., Mioduszewski, A. J., & Torres, R. M.
-2011, ApJ, 733, 71
-Flower, D. R., Offer, A., & Schilke, P. 1990, MNRAS, 244, 4P
-Galván-Madrid, R., Keto, E., Zhang, Q., et al. 2009, ApJ, 706, 1036"""
+            [  # https://isg.beel.org/blog/2019/12/10/giant-the-1-billion-annotated-synthetic-bibliographic-reference-string-dataset-for-deep-citation-parsing-pre-print/
+                """Crossref, https://www.crossref.org
+A JavaScript implementation of the Citation Style Language (CSL),
+https://github.com/Juris-M/citeproc-js
+Official repository for Citation Style Language (CSL),
+https://github.com/citation-style-language/styles
+Anzaroot, S., McCallum, A.: A New Dataset for fine-Grained Citation field Extraction (2013)
+Councill, I.G., Giles, C.L., Kan, M.Y.: Parscit: an open-source crf reference string parsing package. In: LREC. vol. 8, pp. 661–667 (2008)
+Fedoryszak, M., Tkaczyk, D., Bolikowski, L.: Large scale citation matching using apache hadoop. In: International Conference on Theory and Practice of Digital Libraries. pp. 362–365. Springer (2013)
+Hetzner, E.: A simple method for citation metadata extraction using hidden markov models. In: Proceedings of the 8th ACM/IEEE-CS joint conference on Digital libraries. pp. 280–284. ACM (2008)
+Lample, G., Ballesteros, M., Subramanian, S., Kawakami, K., Dyer, C.: Neural architectures for named entity recognition. arXiv preprint arXiv:1603.01360 (2016)
+Lopez, P.: Grobid: Combining automatic bibliographic data recognition and term extraction for scholarship publications. In: International conference on theory and practice of digital libraries. pp. 473–474. Springer (2009)
+Ma, X., Hovy, E.: End-to-end sequence labeling via bi-directional lstm-cnns-crf. arXiv preprint arXiv:1603.01354 (2016)
+Mikolov, T., Sutskever, I., Chen, K., Corrado, G.S., Dean, J.: Distributed representations of words and phrases and their compositionality. In: Advances in neural information processing systems. pp. 3111–3119 (2013)
+Ojokoh, B., Zhang, M., Tang, J.: A trigram hidden markov model for metadata extraction from heterogeneous references. Information Sciences 181(9), 1538–1551
+(2011)
+Okada, T., Takasu, A., Adachi, J.: Bibliographic component extraction using support vector machines and hidden markov models. In: International Conference on
+Theory and Practice of Digital Libraries. pp. 501–512. Springer (2004)
+Prasad, A., Kaur, M., Kan, M.Y.: Neural parscit: a deep learning-based reference string parser. International Journal on Digital Libraries 19(4), 323–337 (2018)
+Rodrigues Alves, D., Colavizza, G., Kaplan, F.: Deep reference mining from scholarly literature in the arts and humanities. Frontiers in Research Metrics and Analytics 3, 21 (2018)
+Tkaczyk, D., Collins, A., Sheridan, P., Beel, J.: Machine learning vs. rules and out-of-the-box vs. retrained: An evaluation of open-source bibliographic reference and citation parsers. In: Proceedings of the 18th ACM/IEEE on joint conference on digital libraries. pp. 99–108. ACM (2018)
+Tkaczyk, D., Szostek, P., Dendek, P.J., Fedoryszak, M., Bolikowski, L.: Cermine– automatic extraction of metadata and references from scientific literature. In: 2014 11th IAPR International Workshop on Document Analysis Systems. pp. 217–221. IEEE (2014)
+Yin, P., Zhang, M., Deng, Z., Yang, D.: Metadata extraction from bibliographies using bigram hmm. In: International Conference on Asian Digital Libraries. pp.
+310–319. Springer (2004)
+Zhang, X., Zou, J., Le, D.X., Thoma, G.R.: A structural svm approach for reference parsing. BMC bioinformatics 12(3), S7 (2011)"""
+            ],
+            [  # https://arxiv.org/pdf/1706.03762.pdf
+                """[28] Romain Paulus, Caiming Xiong, and Richard Socher. A deep reinforced model for abstractive
+summarization. arXiv preprint arXiv:1705.04304, 2017.
+[29] Slav Petrov, Leon Barrett, Romain Thibaux, and Dan Klein. Learning accurate, compact,
+and interpretable tree annotation. In Proceedings of the 21st International Conference on
+Computational Linguistics and 44th Annual Meeting of the ACL, pages 433–440. ACL, July
+2006.
+[30] Ofir Press and Lior Wolf. Using the output embedding to improve language models. arXiv preprint
+arXiv:1608.05859, 2016.
+[31] Rico Sennrich, Barry Haddow, and Alexandra Birch. Neural machine translation of rare words
+with subword units. arXiv preprint arXiv:1508.07909, 2015.
+[32] Noam Shazeer, Azalia Mirhoseini, Krzysztof Maziarz, Andy Davis, Quoc Le, Geoffrey Hinton,
+and Jeff Dean. Outrageously large neural networks: The sparsely-gated mixture-of-experts
+layer. arXiv preprint arXiv:1701.06538, 2017.
+[33] Nitish Srivastava, Geoffrey E Hinton, Alex Krizhevsky, Ilya Sutskever, and Ruslan Salakhutdi-
+nov. Dropout: a simple way to prevent neural networks from overfitting. Journal of Machine
+Learning Research, 15(1):1929–1958, 2014.
+[34] Sainbayar Sukhbaatar, Arthur Szlam, Jason Weston, and Rob Fergus. End-to-end memory
+networks. In C. Cortes, N. D. Lawrence, D. D. Lee, M. Sugiyama, and R. Garnett, editors,
+Advances in Neural Information Processing Systems 28, pages 2440–2448. Curran Associates,
+Inc., 2015.
+[35] Ilya Sutskever, Oriol Vinyals, and Quoc VV Le. Sequence to sequence learning with neural
+networks. In Advances in Neural Information Processing Systems, pages 3104–3112, 2014.
+[36] Christian Szegedy, Vincent Vanhoucke, Sergey Ioffe, Jonathon Shlens, and Zbigniew Wojna.
+Rethinking the inception architecture for computer vision. CoRR, abs/1512.00567, 2015.
+[37] Vinyals & Kaiser, Koo, Petrov, Sutskever, and Hinton. Grammar as a foreign language. In
+Advances in Neural Information Processing Systems, 2015.
+[38] Yonghui Wu, Mike Schuster, Zhifeng Chen, Quoc V Le, Mohammad Norouzi, Wolfgang
+Macherey, Maxim Krikun, Yuan Cao, Qin Gao, Klaus Macherey, et al. Google’s neural machine
+translation system: Bridging the gap between human and machine translation. arXiv preprint
+arXiv:1609.08144, 2016."""
             ],
         ],
         inputs=textbox,