s2orc-doc2json/temp_dir/N18-3011.tei.xml

<?xml version="1.0" encoding="UTF-8"?>
<TEI xml:space="preserve" xmlns="http://www.tei-c.org/ns/1.0" 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" 
xsi:schemaLocation="http://www.tei-c.org/ns/1.0 /home/scieditor/grobid-0.6.1/grobid-home/schemas/xsd/Grobid.xsd"
 xmlns:xlink="http://www.w3.org/1999/xlink">
	<teiHeader xml:lang="en">
		<fileDesc>
			<titleStmt>
				<title level="a" type="main">Construction of the Literature Graph in Semantic Scholar</title>
			</titleStmt>
			<publicationStmt>
				<publisher/>
				<availability status="unknown"><licence/></availability>
			</publicationStmt>
			<sourceDesc>
				<biblStruct>
					<analytic>
						<author role="corresp">
							<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Waleed</forename><surname>Ammar</surname></persName>
							<email>waleeda@allenai.org</email>
						</author>
						<author>
							<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Dirk</forename><surname>Groeneveld</surname></persName>
						</author>
						<author>
							<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Chandra</forename><surname>Bhagavatula</surname></persName>
						</author>
						<author>
							<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Iz</forename><surname>Beltagy</surname></persName>
						</author>
						<author>
							<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Miles</forename><surname>Crawford</surname></persName>
						</author>
						<author>
							<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Doug</forename><surname>Downey</surname></persName>
						</author>
						<author>
							<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Jason</forename><surname>Dunkelberger</surname></persName>
						</author>
						<author>
							<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Ahmed</forename><surname>Elgohary</surname></persName>
						</author>
						<author>
							<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Sergey</forename><surname>Feldman</surname></persName>
						</author>
						<author>
							<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Vu</forename><surname>Ha</surname></persName>
						</author>
						<author>
							<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Rodney</forename><surname>Kinney</surname></persName>
						</author>
						<author>
							<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Sebastian</forename><surname>Kohlmeier</surname></persName>
						</author>
						<author>
							<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Kyle</forename><surname>Lo</surname></persName>
						</author>
						<author>
							<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Tyler</forename><surname>Murray</surname></persName>
						</author>
						<author>
							<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Hsu-Han</forename><surname>Ooi</surname></persName>
						</author>
						<author>
							<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Matthew</forename><surname>Peters</surname></persName>
						</author>
						<author>
							<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Joanna</forename><surname>Power</surname></persName>
						</author>
						<author>
							<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Sam</forename><surname>Skjonsberg</surname></persName>
						</author>
						<author>
							<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Lucy</forename><forename type="middle">Lu</forename><surname>Wang</surname></persName>
						</author>
						<author>
							<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Chris</forename><surname>Wilhelm</surname></persName>
						</author>
						<author>
							<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Zheng</forename><surname>Yuan</surname></persName>
						</author>
						<author>
							<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Madeleine</forename><surname>Van Zuylen</surname></persName>
						</author>
						<author>
							<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Oren</forename><surname>Etzioni</surname></persName>
						</author>
						<author>
							<affiliation key="aff0">
								<orgName type="institution">Allen Institute for Artificial Intelligence</orgName>
								<address>
									<postCode>98103</postCode>
									<settlement>Seattle</settlement>
									<region>WA</region>
									<country key="US">USA</country>
								</address>
							</affiliation>
						</author>
						<author>
							<affiliation key="aff1">
								<orgName type="institution">Northwestern University</orgName>
								<address>
									<postCode>60208</postCode>
									<settlement>Evanston</settlement>
									<region>IL</region>
									<country key="US">USA</country>
								</address>
							</affiliation>
						</author>
						<author>
							<affiliation key="aff2">
								<orgName type="department">Introduction</orgName>
							</affiliation>
						</author>
						<title level="a" type="main">Construction of the Literature Graph in Semantic Scholar</title>
					</analytic>
					<monogr>
						<imprint>
							<date/>
						</imprint>
					</monogr>
				</biblStruct>
			</sourceDesc>
		</fileDesc>
		<encodingDesc>
			<appInfo>
				<application version="0.6.1" ident="GROBID" when="2022-06-22T18:47+0000">
					<desc>GROBID - A machine learning software for extracting information from scholarly documents</desc>
					<ref target="https://github.com/kermitt2/grobid"/>
				</application>
			</appInfo>
		</encodingDesc>
		<profileDesc>
			<abstract>
				<p>We describe a deployed scalable system for organizing published scientific literature into a heterogeneous graph to facilitate algorithmic manipulation and discovery. The resulting literature graph consists of more than 280M nodes, representing papers, authors, entities and various interactions between them (e.g., authorships, citations, entity mentions). We reduce literature graph construction into familiar NLP tasks (e.g., entity extraction and linking), point out research challenges due to differences from standard formulations of these tasks, and report empirical results for each task. The methods described in this paper are used to enable semantic features in www.semanticscholar.org.</p>
			</abstract>
		</profileDesc>
	</teiHeader>
	<text xml:lang="en">
		<body>
<div xmlns="http://www.tei-c.org/ns/1.0"><head n="1">Introduction</head><p>The goal of this work is to facilitate algorithmic discovery in the scientific literature. Despite notable advances in scientific search engines, data mining and digital libraries (e.g., <ref type="bibr" target="#b25">Wu et al., 2014)</ref>, researchers remain unable to answer simple questions such as:</p><p>What is the percentage of female subjects in depression clinical trials?</p><p>Which of my co-authors published one or more papers on coreference resolution?</p><p>Which papers discuss the effects of Ranibizumab on the Retina?</p><p>In this paper, we focus on the problem of extracting structured data from scientific documents, which can later be used in natural language interfaces (e.g., <ref type="bibr" target="#b12">Iyer et al., 2017)</ref> or to improve ranking of results in academic search (e.g., Xiong et al., <ref type="figure">Figure 1</ref>: Part of the literature graph. 2017). We describe methods used in a scalable deployed production system for extracting structured information from scientific documents into the literature graph (see <ref type="figure">Fig. 1</ref>). The literature graph is a directed property graph which summarizes key information in the literature and can be used to answer the queries mentioned earlier as well as more complex queries. For example, in order to compute the Erdős number of an author X, the graph can be queried to find the number of nodes on the shortest undirected path between author X and Paul Erdős such that all edges on the path are labeled "authored".</p><p>We reduce literature graph construction into familiar NLP tasks such as sequence labeling, entity linking and relation extraction, and address some of the impractical assumptions commonly made in the standard formulations of these tasks. For example, most research on named entity recognition tasks report results on large labeled datasets such as <ref type="bibr">CoNLL-2003</ref><ref type="bibr">and ACE-2005</ref><ref type="bibr">(e.g., Lample et al., 2016</ref>, and assume that entity types in the test set match those labeled in the training set (including work on domain adaptation, e.g., <ref type="bibr" target="#b6">Daumé, 2007)</ref>. These assumptions, while useful for developing and benchmarking new methods, are unrealistic for many domains and applications. The paper also serves as an overview of the approach we adopt at www.semanticscholar.org in a step towards more intelligent academic search engines <ref type="bibr" target="#b8">(Etzioni, 2011)</ref>.</p><p>In the next section, we start by describing our symbolic representation of the literature. Then, we discuss how we extract metadata associated with a paper such as authors and references, then how we extract the entities mentioned in paper text. Before we conclude, we briefly describe other research challenges we are actively working on in order to improve the quality of the literature graph.</p></div>
<div xmlns="http://www.tei-c.org/ns/1.0"><head n="2">Structure of The Literature Graph</head><p>The literature graph is a property graph with directed edges. Unlike Resource Description Framework (RDF) graphs, nodes and edges in property graphs have an internal structure which is more suitable for representing complex data types such as papers and entities. In this section, we describe the attributes associated with nodes and edges of different types in the literature graph.</p></div>
<div xmlns="http://www.tei-c.org/ns/1.0"><head n="2.1">Node Types</head><p>Papers. We obtain metadata and PDF files of papers via partnerships with publishers (e.g., Springer, Nature), catalogs (e.g., DBLP, MED-LINE), pre-publishing services (e.g., arXiv, bioRxive), as well as web-crawling. Paper nodes are associated with a set of attributes such as 'title', 'abstract', 'full text', 'venues' and 'publication year'. While some of the paper sources provide these attributes as metadata, it is often necessary to extract them from the paper PDF (details in §3). We deterministically remove duplicate papers based on string similarity of their metadata, resulting in 37M unique paper nodes. Papers in the literature graph cover a variety of scientific disciplines, including computer science, molecular biology, microbiology and neuroscience.</p><p>Authors. Each node of this type represents a unique author, with attributes such as 'first name' and 'last name'. The literature graph has 12M nodes of this type.</p><p>Entities. Each node of this type represents a unique scientific concept discussed in the literature, with attributes such as 'canonical name', 'aliases' and 'description'. Our literature graph has 0.4M nodes of this type. We describe how we populate entity nodes in §4.3.</p><p>Entity mentions. Each node of this type represents a textual reference of an entity in one of the papers, with attributes such as 'mention text', 'context', and 'confidence'. We describe how we populate the 237M mentions in the literature graph in §4.1.</p></div>
<div xmlns="http://www.tei-c.org/ns/1.0"><head n="2.2">Edge Types</head><p>Citations. We instantiate a directed citation edge from paper nodes p 1 ! p 2 for each p 2 referenced in p 1 . Citation edges have attributes such as 'from paper id', 'to paper id' and 'contexts' (the textual contexts where p 2 is referenced in p 1 ). While some of the paper sources provide these attributes as metadata, it is often necessary to extract them from the paper PDF as detailed in §3.</p><p>Authorship. We instantiate a directed authorship edge between an author node and a paper node a ! p for each author of that paper.</p><p>Entity linking edges. We instantiate a directed edge from an extracted entity mention node to the entity it refers to.</p><p>Mention-mention relations. We instantiate a directed edge between a pair of mentions in the same sentential context if the textual relation extraction model predicts one of a predefined list of relation types between them in a sentential context. <ref type="bibr">1</ref> We encode a symmetric relation between m 1 and m 2 as two directed edges m 1 ! m 2 and m 2 ! m 1 .</p><p>Entity-entity relations. While mentionmention edges represent relations between mentions in a particular context, entity-entity edges represent relations between abstract entities. These relations may be imported from an existing knowledge base (KB) or inferred from other edges in the graph.</p></div>
<div xmlns="http://www.tei-c.org/ns/1.0"><head n="3">Extracting Metadata</head><p>In the previous section, we described the overall structure of the literature graph. Next, we discuss how we populate paper nodes, author nodes, authorship edges, and citation edges.</p><p>Although some publishers provide sufficient metadata about their papers, many papers are provided with incomplete metadata. Also, papers obtained via web-crawling are not associated with any metadata. To fill in this gap, we built the Sci-enceParse system to predict structured data from the raw PDFs using recurrent neural networks (RNNs). 2 For each paper, the system extracts the paper title, list of authors, and list of references; each reference consists of a title, a list of authors, a venue, and a year.</p><p>Preparing the input layer. We split each PDF into individual pages, and feed each page to Apache's PDFBox library 3 to convert it into a sequence of tokens, where each token has features, e.g., 'text', 'font size', 'space width', 'position on the page'.</p><p>We normalize the token-level features before feeding them as inputs to the model. For each of the 'font size' and 'space width' features, we compute three normalized values (with respect to current page, current document, and the whole training corpus), each value ranging between -0.5 to +0.5. The token's 'position on the page' is given in XY coordinate points. We scale the values linearly to range from . 0:5; 0:5/ at the top-left corner of the page to .0:5; 0:5/ at the bottom-right corner.</p><p>In order to capture case information, we add seven numeric features to the input representation of each token: whether the first/second letter is uppercase/lowercase, the fraction of uppercase/lowercase letters and the fraction of digits.</p><p>To help the model make correct predictions for metadata which tend to appear at the beginning (e.g., titles and authors) or at the end of papers (e.g., references), we provide the current page number as two discrete variables (relative to the beginning and end of the PDF file) with values 0, 1 and 2+. These features are repeated for each token on the same page.</p><p>For the k-th token in the sequence, we compute the input representation i k by concatenating the numeric features, an embedding of the 'font size', and the word embedding of the lowercased token. Word embeddings are initialized with GloVe <ref type="bibr" target="#b19">(Pennington et al., 2014)</ref>.</p><p>Model. The input token representations are passed through one fully-connected layer and then </p><formula xml:id="formula_0">g ! k D LSTM.Wi k ; g ! k 1 /; g k D OEg ! k I g k ; h ! k D LSTM.g k ; h ! k 1 /; h k D OEh ! k I g k</formula><p>where W is a weight matrix, g k and h k are defined similarly to g ! k and h ! k but process token sequences in the opposite direction.</p><p>Following Collobert et al. <ref type="formula">2011</ref>, we feed the output of the second layer h k into a dense layer to predict unnormalized label weights for each token and learn label bigram feature weights (often described as a conditional random field layer when used in neural architectures) to account for dependencies between labels.</p><p>Training. The ScienceParse system is trained on a snapshot of the data at PubMed Central. It consists of 1.4M PDFs and their associated metadata, which specify the correct titles, authors, and bibliographies. We use a heuristic labeling process that finds the strings from the metadata in the tokenized PDFs to produce labeled tokens. This labeling process succeeds for 76% of the documents. The remaining documents are not used in the training process. During training, we only use pages which have at least one token with a label that is not "none".</p><p>Decoding. At test time, we use Viterbi decoding to find the most likely global sequence, with no further constraints. To get the title, we use the longest continuous sequence of tokens with the "title" label. Since there can be multiple authors, we use all continuous sequences of tokens with the "author" label as authors, but require that all authors of a paper are mentioned on the same page. If the author labels are predicted in multiple pages, we use the one with the largest number of authors.</p><p>Results. We run our final tests on a held-out set from PubMed Central, consisting of about 54K documents. The results are detailed in <ref type="table">Table 1</ref>. We use a conservative evaluation where an instance is correct if it exactly matches the gold annotation, with no credit for partial matching.</p><p>To give an example for the type of errors our model makes, consider the paper <ref type="bibr" target="#b23">(Wang et al., 2013)</ref> titled "Clinical review: Efficacy of antimicrobial-impregnated catheters in external ventricular drainage -a systematic review and metaanalysis." The title we extract for this paper omits the first part "Clinical review:". This is likely to be a result of the pattern "Foo: Bar Baz" appearing in many training examples with only "Bar Baz" labeled as the title.</p></div>
<div xmlns="http://www.tei-c.org/ns/1.0"><head n="4">Entity Extraction and Linking</head><p>In the previous section, we described how we populate the backbone of the literature graph, i.e., paper nodes, author nodes and citation edges. Next, we discuss how we populate mentions and entities in the literature graph using entity extraction and linking on the paper text. In order to focus on more salient entities in a given paper, we only use the title and abstract.</p></div>
<div xmlns="http://www.tei-c.org/ns/1.0"><head n="4.1">Approaches</head><p>We experiment with three approaches for entity extraction and linking: I. Statistical: uses one or more statistical models for predicting mention spans, then uses another statistical model to link mentions to candidate entities in a KB.</p><p>II. Hybrid: defines a small number of handengineered, deterministic rules for string-based matching of the input text to candidate entities in the KB, then uses a statistical model to disambiguate the mentions. <ref type="bibr">4</ref> III. Off-the-shelf: uses existing libraries, namely (Ferragina and Scaiella, 2010, TagMe) 5 and (Demner-Fushman et al., 2017, MetaMap Lite) 6 , with minimal post-processing to extract and link entities to the KB.  <ref type="table">Table 2</ref>: Document-level evaluation of three approaches in two scientific areas: computer science (CS) and biomedical (Bio).</p><p>We evaluate the performance of each approach in two broad scientific areas: computer science (CS) and biomedical research (Bio). For each unique (paper ID, entity ID) pair predicted by one of the approaches, we ask human annotators to label each mention extracted for this entity in the paper. We use CrowdFlower to manage human annotations and only include instances where three or more annotators agree on the label. If one or more of the entity mentions in that paper is judged to be correct, the pair (paper ID, entity ID) counts as one correct instance. Otherwise, it counts as an incorrect instance. We report 'yield' in lieu of 'recall' due to the difficulty of doing a scalable comprehensive annotation. <ref type="table">Table 2</ref> shows the results based on 500 papers using v1.1.2 of our entity extraction and linking components. In both domains, the statistical approach gives the highest precision and the lowest yield. The hybrid approach consistently gives the highest yield, but sacrifices precision. The TagMe off-the-shelf library used for the CS domain gives surprisingly good results, with precision within 1 point from the statistical models. However, the MetaMap Lite off-the-shelf library we used for the biomedical domain suffered a huge loss in precision. Our error analysis showed that each of the approaches is able to predict entities not predicted by the other approaches so we decided to pool their outputs in our deployed system, which gives significantly higher yield than any individual approach while maintaining reasonably high precision.</p></div>
<div xmlns="http://www.tei-c.org/ns/1.0"><head n="4.2">Entity Extraction Models</head><p>Given the token sequence t 1 ; : : : ; t N in a sentence, we need to identify spans which correspond to entity mentions. We use the BILOU scheme to encode labels at the token level. Unlike most formulations of named entity recognition problems (NER), we do not identify the entity type (e.g., protein, drug, chemical, disease) for each mention since the output mentions are further grounded in a KB with further information about the entity (including its type), using an entity linking module.</p><p>Model. First, we construct the token embedding x k D OEc k I w k for each token t k in the input sequence, where c k is a character-based representation computed using a convolutional neural network (CNN) with filter of size 3 characters, and w k are learned word embeddings initialized with the GloVe embeddings <ref type="bibr" target="#b19">(Pennington et al., 2014)</ref>.</p><p>We also compute context-sensitive word embeddings, denoted as lm k D OElm ! k I lm k , by concatenating the projected outputs of forward and backward recurrent neural network language models (RNN-LM) at position k. The language model (LM) for each direction is trained independently and consists of a single layer long short-term memory (LSTM) network followed by a linear project layer. While training the LM parameters, lm ! k is used to predict t kC1 and lm k is used to predict t k 1 . We fix the LM parameters during training of the entity extraction model. See  and  for more details.</p><p>Given the x k and lm k embeddings for each token k 2 f1; : : : ; N g, we use a two-layer bidirectional LSTM to encode the sequence with x k and lm k feeding into the first and second layer, respectively. That is,</p><formula xml:id="formula_1">g ! k D LSTM.x k ; g ! k 1 /; g k D OEg ! k I g k ; h ! k D LSTM.OEg k I lm k ; h ! k 1 /; h k D OEh ! k I h k ;</formula><p>where g k and h k are defined similarly to g ! k and h ! k but process token sequences in the opposite direction. Similar to the model described in §3, we feed the output of the second LSTM into a dense layer to predict unnormalized label weights for each token and learn label bigram feature weights to account for dependencies between labels.</p><p>Results. We use the standard data splits of the SemEval-2017 Task 10 on entity (and relation) extraction from scientific papers <ref type="bibr" target="#b1">(Augenstein et al., 2017)</ref>. <ref type="table">Table 3</ref> compares three variants of our entity extraction model. The first line omits the LM embeddings lm k , while the second line is the full model (including LM embeddings) showing a large improvement of 4.2 F1 points. The third line shows that creating an ensemble of 15 models further improves the results by 1.1 F1 points.</p><p>Model instances. In the deployed system, we use three instances of the entity extraction model Description F1 Without LM 49.9</p><p>With LM 54.1 Avg. of 15 models with LM 55.2 <ref type="table">Table 3</ref>: Results of the entity extraction model on the development set of SemEval-2017 task 10. with a similar architecture, but trained on different datasets. Two instances are trained on the BC5CDR <ref type="bibr" target="#b16">(Li et al., 2016)</ref> and the CHEMDNER datasets <ref type="bibr" target="#b14">(Krallinger et al., 2015)</ref> to extract key entity mentions in the biomedical domain such as diseases, drugs and chemical compounds. The third instance is trained on mention labels induced from Wikipedia articles in the computer science domain.</p><p>The output of all model instances are pooled together and combined with the rule-based entity extraction module, then fed into the entity linking model (described below).</p></div>
<div xmlns="http://www.tei-c.org/ns/1.0"><head n="4.3">Knowledge Bases</head><p>In this section, we describe the construction of entity nodes and entity-entity edges. Unlike other knowledge extraction systems such as the Never-Ending Language Learner (NELL) 7 and OpenIE 4, 8 we use existing knowledge bases (KBs) of entities to reduce the burden of identifying coherent concepts. Grounding the entity mentions in a manually-curated KB also increases user confidence in automated predictions. We use two KBs: UMLS: The UMLS metathesaurus integrates information about concepts in specialized ontologies in several biomedical domains, and is funded by the U.S. National Library of Medicine. DBpedia: DBpedia provides access to structured information in Wikipedia. Rather than including all Wikipedia pages, we used a short list of Wikipedia categories about CS and included all pages up to depth four in their trees in order to exclude irrelevant entities, e.g., "Lord of the Rings" in DBpedia.</p></div>
<div xmlns="http://www.tei-c.org/ns/1.0"><head n="4.4">Entity Linking Models</head><p>Given a text span s identified by the entity extraction model in §4.2 (or with heuristics) and a reference KB, the goal of the entity linking model is to associate the span with the entity it refers to. A span and its surrounding words are collectively referred to as a mention. We first identify a set of candidate entities that a given mention may refer to. Then, we rank the candidate entities based on a score computed using a neural model trained on labeled data.</p><p>For example, given the string ". . . database of facts, an ILP system will . . . ", the entity extraction model identifies the span "ILP" as a possible entity and the entity linking model associates it with "Inductive_Logic_Programming" as the referent entity (from among other candidates like "Integer_Linear_Programming" or "Instruction-level_Parallelism").</p><p>Datasets. We used two datasets: i) a biomedical dataset formed by combining MSH (Jimeno-Yepes et al., 2011) and BC5CDR <ref type="bibr" target="#b16">(Li et al., 2016)</ref> with UMLS as the reference KB, and ii) a CS dataset we curated using Wikipedia articles about CS concepts with DBpedia as the reference KB.</p><p>Candidate selection. In a preprocessing step, we build an index which maps any token used in a labeled mention or an entity name in the KB to associated entity IDs, along with the frequency this token is associated with that entity. This is similar to the index used in previous entity linking systems (e.g., <ref type="bibr" target="#b3">Bhagavatula et al., 2015)</ref> to estimate the probability that a given mention refers to an entity. At train and test time, we use this index to find candidate entities for a given mention by looking up the tokens in the mention. This method also serves as our baseline in <ref type="table">Table 4</ref> by selecting the entity with the highest frequency for a given mention.</p><p>Scoring candidates. Given a mention (m) and a candidate entity (e), the neural model constructs a vector encoding of the mention and the entity. We encode the mention and entity using the functions f and g, respectively, as follows:</p><formula xml:id="formula_2">f.m/ D OEv m.name I avg.v m.lc ; v m.rc /;</formula><p>g.e/ D OEv e.name I v e.def ; where m.surface, m.lc and m.rc are the mention's surface form, left and right contexts, and e.name and e.def are the candidate entity's name and definition, respectively. v text is a bag-of-words sum encoder for text. We use the same encoder for the mention surface form and the candidate name, and another encoder for the mention contexts and entity definition.</p><p>Additionally, we include numerical features to estimate the confidence of a candidate entity based on the statistics collected in the index described  <ref type="table">Table 4</ref>: The Bag of Concepts F1 score of the baseline and neural model on the two curated datasets.</p><p>earlier. We compute two scores based on the word overlap of (i) mention's context and candidate's definition and (ii) mention's surface span and the candidate entity's name. Finally, we feed the concatenation of the cosine similarity between f.m/ and g.e/ and the intersection-based scores into an affine transformation followed by a sigmoid nonlinearity to compute the final score for the pair (m, e).</p><p>Results. We use the Bag of Concepts F1 metric <ref type="bibr" target="#b17">(Ling et al., 2015)</ref> for comparison. <ref type="table">Table 4</ref> compares the performance of the most-frequent-entity baseline and our neural model described above.</p></div>
<div xmlns="http://www.tei-c.org/ns/1.0"><head n="5">Other Research Problems</head><p>In the previous sections, we discussed how we construct the main components of the literature graph. In this section, we briefly describe several other related challenges we are actively working on.</p><p>Author disambiguation. Despite initiatives to have global author IDs ORCID and ResearcherID, most publishers provide author information as names (e.g., arXiv). However, author names cannot be used as a unique identifier since several people often share the same name. Moreover, different venues and sources use different conventions in reporting the author names, e.g., "first initial, last name" vs. "last name, first name". Inspired by <ref type="bibr" target="#b5">Culotta et al. (2007)</ref>, we train a supervised binary classifier for merging pairs of author instances and use it to incrementally create author clusters. We only consider merging two author instances if they have the same last name and share the first initial. If the first name is spelled out (rather than abbreviated) in both author instances, we also require that the first name matches.</p><p>Ontology matching. Popular concepts are often represented in multiple KBs. For example, the concept of "artificial neural networks" is represented as entity ID D016571 in the MESH ontology, and represented as page ID '21523' in DBpedia. Ontology matching is the problem of identifying semantically-equivalent entities across KBs or ontologies. <ref type="bibr">9</ref> Limited KB coverage. The convenience of grounding entities in a hand-curated KB comes at the cost of limited coverage. Introduction of new concepts and relations in the scientific literature occurs at a faster pace than KB curation, resulting in a large gap in KB coverage of scientific concepts. In order to close this gap, we need to develop models which can predict textual relations as well as detailed concept descriptions in scientific papers. For the same reasons, we also need to augment the relations imported from the KB with relations extracted from text. Our approach to address both entity and relation coverage is based on distant supervision <ref type="bibr" target="#b18">(Mintz et al., 2009)</ref>. In short, we train two models for identifying entity definitions and relations expressed in natural language in scientific documents, and automatically generate labeled data for training these models using known definitions and relations in the KB.</p><p>We note that the literature graph currently lacks coverage for important entity types (e.g., affiliations) and domains (e.g., physics). Covering affiliations requires small modifications to the metadata extraction model followed by an algorithm for matching author names with their affiliations. In order to cover additional scientific domains, more agreements need to be signed with publishers.</p><p>Figure and table extraction. Non-textual components such as charts, diagrams and tables provide key information in many scientific documents, but the lack of large labeled datasets has impeded the development of data-driven methods for scientific figure extraction. In <ref type="bibr" target="#b21">Siegel et al. (2018)</ref>, we induced high-quality training labels for the task of figure extraction in a large number of scientific documents, with no human intervention. To accomplish this we leveraged the auxiliary data provided in two large web collections of scientific documents (arXiv and PubMed) to locate figures and their associated captions in the rasterized PDF. We use the resulting dataset to train a deep neural network for end-to-end figure detection, yielding a model that can be more easily extended to new domains compared to previous work.</p><p>Understanding and predicting citations. The citation edges in the literature graph provide a wealth of information (e.g., at what rate a paper is being cited and whether it is accelerating), and opens the door for further research to better understand and predict citations. For example, in order to allow users to better understand what impact a paper had and effectively navigate its citations, we experimented with methods for classifying a citation as important or incidental, as well as more finegrained classes <ref type="bibr" target="#b22">(Valenzuela et al., 2015)</ref>. The citation information also enables us to develop models for estimating the potential of a paper or an author. In Weihs and Etzioni (2017), we predict citationbased metrics such as an author's h-index and the citation rate of a paper in the future. Also related is the problem of predicting which papers should be cited in a given draft <ref type="bibr" target="#b2">(Bhagavatula et al., 2018)</ref>, which can help improve the quality of a paper draft before it is submitted for peer review, or used to supplement the list of references after a paper is published.</p></div>
<div xmlns="http://www.tei-c.org/ns/1.0"><head n="6">Conclusion and Future Work</head><p>In this paper, we discuss the construction of a graph, providing a symbolic representation of the scientific literature. We describe deployed models for identifying authors, references and entities in the paper text, and provide experimental results to evaluate the performance of each model. Three research directions follow from this work and other similar projects, e.g., <ref type="bibr" target="#b10">Hahn-Powell et al. (2017)</ref>; <ref type="bibr" target="#b25">Wu et al. (2014)</ref>: i) improving quality and enriching content of the literature graph (e.g., ontology matching and knowledge base population). ii) aggregating domain-specific extractions across many papers to enable a better understanding of the literature as a whole (e.g., identifying demographic biases in clinical trial participants and summarizing empirical results on important tasks). iii) exploring the literature via natural language interfaces.</p><p>In order to help future research efforts, we make the following resources publicly available: metadata for over 20 million papers, 10 meaningful citations dataset, 11 models for figure and table extraction, 12 models for predicting citations in a paper draft 13 and models for extracting paper metadata, 14 among other resources. <ref type="bibr">15</ref> </p></div>			<note xmlns="http://www.tei-c.org/ns/1.0" place="foot" n="1">Due to space constraints, we opted not to discuss our relation extraction models in this draft.</note>
			<note xmlns="http://www.tei-c.org/ns/1.0" place="foot" n="2">The ScienceParse libraries can be found at http:// allenai.org/software/.3 https://pdfbox.apache.org</note>
			<note xmlns="http://www.tei-c.org/ns/1.0" place="foot" n="4">We also experimented with a "pure" rules-based approach which disambiguates deterministically but the hybrid approach consistently gave better results.5 The TagMe APIs are described at https://sobigdata. d4science.org/web/tagme/tagme-help6  We use v3.4 (L0) of MetaMap Lite, available at https: //metamap.nlm.nih.gov/MetaMapLite.shtml</note>
			<note xmlns="http://www.tei-c.org/ns/1.0" place="foot" n="7">http://rtw.ml.cmu.edu/rtw/ 8 https://github.com/allenai/ openie-standalone</note>
			<note xmlns="http://www.tei-c.org/ns/1.0" place="foot" n="9">Variants of this problem are also known as deduplication or record linkage.</note>
		</body>
		<back>
			<div type="references">

				<listBibl>

<biblStruct xml:id="b0">
	<analytic>
		<title level="a" type="main">The ai2 system at semeval-2017 task 10 (scienceie): semi-supervised end-to-end entity and relation extraction</title>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Waleed</forename><surname>Ammar</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Matthew</forename><forename type="middle">E</forename><surname>Peters</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Chandra</forename><surname>Bhagavatula</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Russell</forename><surname>Power</surname></persName>
		</author>
	</analytic>
	<monogr>
		<title level="m">ACL workshop (SemEval)</title>
		<imprint>
			<date type="published" when="2017" />
		</imprint>
	</monogr>
	<note type="raw_reference">Waleed Ammar, Matthew E. Peters, Chandra Bhagavat- ula, and Russell Power. 2017. The ai2 system at semeval-2017 task 10 (scienceie): semi-supervised end-to-end entity and relation extraction. In ACL workshop (SemEval).</note>
</biblStruct>

<biblStruct xml:id="b1">
	<monogr>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Isabelle</forename><surname>Augenstein</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Mrinal</forename><surname>Das</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Sebastian</forename><surname>Riedel</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Lakshmi</forename><surname>Vikraman</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Andrew</forename><forename type="middle">D</forename><surname>Mccallum</surname></persName>
		</author>
		<title level="m">Semeval 2017 task 10 (scienceie): Extracting keyphrases and relations from scientific publications</title>
		<imprint>
			<date type="published" when="2017" />
		</imprint>
	</monogr>
	<note>ACL workshop (SemEval)</note>
	<note type="raw_reference">Isabelle Augenstein, Mrinal Das, Sebastian Riedel, Lakshmi Vikraman, and Andrew D. McCallum. 2017. Semeval 2017 task 10 (scienceie): Extracting keyphrases and relations from scientific publications. In ACL workshop (SemEval).</note>
</biblStruct>

<biblStruct xml:id="b2">
	<analytic>
		<title level="a" type="main">Content-based citation recommendation</title>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Chandra</forename><surname>Bhagavatula</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Sergey</forename><surname>Feldman</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Russell</forename><surname>Power</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Waleed</forename><surname>Ammar</surname></persName>
		</author>
	</analytic>
	<monogr>
		<title level="m">NAACL</title>
		<imprint>
			<date type="published" when="2018" />
		</imprint>
	</monogr>
	<note type="raw_reference">Chandra Bhagavatula, Sergey Feldman, Russell Power, and Waleed Ammar. 2018. Content-based citation recommendation. In NAACL.</note>
</biblStruct>

<biblStruct xml:id="b3">
	<monogr>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Chandra</forename><surname>Bhagavatula</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Thanapon</forename><surname>Noraset</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Doug</forename><surname>Downey</surname></persName>
		</author>
		<title level="m">TabEL: entity linking in web tables. In ISWC</title>
		<imprint>
			<date type="published" when="2015" />
		</imprint>
	</monogr>
	<note type="raw_reference">Chandra Bhagavatula, Thanapon Noraset, and Doug Downey. 2015. TabEL: entity linking in web tables. In ISWC.</note>
</biblStruct>

<biblStruct xml:id="b4">
	<analytic>
		<title level="a" type="main">Natural language processing (almost) from scratch</title>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Ronan</forename><surname>Collobert</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Jason</forename><surname>Weston</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Léon</forename><surname>Bottou</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Michael</forename><surname>Karlen</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Koray</forename><surname>Kavukcuoglu</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Pavel</forename><forename type="middle">P</forename><surname>Kuksa</surname></persName>
		</author>
	</analytic>
	<monogr>
		<title level="m">JMLR</title>
		<imprint>
			<date type="published" when="2011" />
		</imprint>
	</monogr>
	<note type="raw_reference">Ronan Collobert, Jason Weston, Léon Bottou, Michael Karlen, Koray Kavukcuoglu, and Pavel P. Kuksa. 2011. Natural language processing (almost) from scratch. In JMLR.</note>
</biblStruct>

<biblStruct xml:id="b5">
	<analytic>
		<title level="a" type="main">Author disambiguation using error-driven machine learning with a ranking loss function</title>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Aron</forename><surname>Culotta</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Pallika</forename><surname>Kanani</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Robert</forename><surname>Hall</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Michael</forename><surname>Wick</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Andrew</forename><forename type="middle">D</forename><surname>Mccallum</surname></persName>
		</author>
	</analytic>
	<monogr>
		<title level="m">IIWeb Workshop</title>
		<imprint>
			<date type="published" when="2007" />
		</imprint>
	</monogr>
	<note type="raw_reference">Aron Culotta, Pallika Kanani, Robert Hall, Michael Wick, and Andrew D. McCallum. 2007. Author disambiguation using error-driven machine learning with a ranking loss function. In IIWeb Workshop.</note>
</biblStruct>

<biblStruct xml:id="b6">
	<analytic>
		<title level="a" type="main">Frustratingly easy domain adaptation</title>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Hal</forename><surname>Daumé</surname></persName>
		</author>
	</analytic>
	<monogr>
		<title level="m">ACL</title>
		<imprint>
			<date type="published" when="2007" />
		</imprint>
	</monogr>
	<note type="raw_reference">Hal Daumé. 2007. Frustratingly easy domain adapta- tion. In ACL.</note>
</biblStruct>

<biblStruct xml:id="b7">
	<analytic>
		<title level="a" type="main">MetaMap Lite: an evaluation of a new Java implementation of MetaMap</title>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Dina</forename><surname>Demner-Fushman</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Willie</forename><forename type="middle">J</forename><surname>Rogers</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Alan</forename><forename type="middle">R</forename><surname>Aronson</surname></persName>
		</author>
	</analytic>
	<monogr>
		<title level="m">JAMIA</title>
		<imprint>
			<date type="published" when="2017" />
		</imprint>
	</monogr>
	<note type="raw_reference">Dina Demner-Fushman, Willie J. Rogers, and Alan R. Aronson. 2017. MetaMap Lite: an evaluation of a new Java implementation of MetaMap. In JAMIA.</note>
</biblStruct>

<biblStruct xml:id="b8">
	<analytic>
		<title level="a" type="main">Search needs a shake-up</title>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Oren</forename><forename type="middle">Etzioni</forename></persName>
		</author>
	</analytic>
	<monogr>
		<title level="j">Nature</title>
		<imprint>
			<biblScope unit="volume">476</biblScope>
			<biblScope unit="page" from="25" to="31" />
			<date type="published" when="2011" />
		</imprint>
	</monogr>
	<note type="raw_reference">Oren Etzioni. 2011. Search needs a shake-up. Nature 476 7358:25-6.</note>
</biblStruct>

<biblStruct xml:id="b9">
	<analytic>
		<title level="a" type="main">TAGME: on-the-fly annotation of short text fragments (by wikipedia entities)</title>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Paolo</forename><surname>Ferragina</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Ugo</forename><surname>Scaiella</surname></persName>
		</author>
	</analytic>
	<monogr>
		<title level="m">CIKM</title>
		<imprint>
			<date type="published" when="2010" />
		</imprint>
	</monogr>
	<note type="raw_reference">Paolo Ferragina and Ugo Scaiella. 2010. TAGME: on-the-fly annotation of short text fragments (by wikipedia entities). In CIKM.</note>
</biblStruct>

<biblStruct xml:id="b10">
	<analytic>
		<title level="a" type="main">Swanson linking revisited: Accelerating literature-based discovery across domains using a conceptual influence graph</title>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Gus</forename><surname>Hahn-Powell</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Marco</forename><forename type="middle">Antonio</forename><surname>Valenzuela-Escarcega</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Mihai</forename><surname>Surdeanu</surname></persName>
		</author>
	</analytic>
	<monogr>
		<title level="m">ACL</title>
		<imprint>
			<date type="published" when="2017" />
		</imprint>
	</monogr>
	<note type="raw_reference">Gus Hahn-Powell, Marco Antonio Valenzuela- Escarcega, and Mihai Surdeanu. 2017. Swanson linking revisited: Accelerating literature-based dis- covery across domains using a conceptual influence graph. In ACL.</note>
</biblStruct>

<biblStruct xml:id="b11">
	<analytic>
		<title level="a" type="main">Long short-term memory</title>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Sepp</forename><surname>Hochreiter</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Jürgen</forename><surname>Schmidhuber</surname></persName>
		</author>
	</analytic>
	<monogr>
		<title level="j">Neural computation</title>
		<imprint>
			<date type="published" when="1997" />
		</imprint>
	</monogr>
	<note type="raw_reference">Sepp Hochreiter and Jürgen Schmidhuber. 1997. Long short-term memory. Neural computation .</note>
</biblStruct>

<biblStruct xml:id="b12">
	<analytic>
		<title level="a" type="main">Learning a neural semantic parser from user feedback</title>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Srinivasan</forename><surname>Iyer</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Ioannis</forename><surname>Konstas</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Alvin</forename><surname>Cheung</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Jayant</forename><surname>Krishnamurthy</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Luke</forename><forename type="middle">S</forename><surname>Zettlemoyer</surname></persName>
		</author>
	</analytic>
	<monogr>
		<title level="m">ACL</title>
		<imprint>
			<date type="published" when="2017" />
		</imprint>
	</monogr>
	<note type="raw_reference">Srinivasan Iyer, Ioannis Konstas, Alvin Cheung, Jayant Krishnamurthy, and Luke S. Zettlemoyer. 2017. Learning a neural semantic parser from user feed- back. In ACL.</note>
</biblStruct>

<biblStruct xml:id="b13">
	<analytic>
		<title level="a" type="main">Exploiting mesh indexing in medline to generate a data set for word sense disambiguation</title>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">J</forename><surname>Antonio</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Bridget</forename><forename type="middle">T</forename><surname>Jimeno-Yepes</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Alan</forename><forename type="middle">R</forename><surname>Mcinnes</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><surname>Aronson</surname></persName>
		</author>
	</analytic>
	<monogr>
		<title level="j">BMC bioinformatics</title>
		<imprint>
			<biblScope unit="volume">12</biblScope>
			<biblScope unit="issue">1</biblScope>
			<biblScope unit="page">223</biblScope>
			<date type="published" when="2011" />
		</imprint>
	</monogr>
	<note type="raw_reference">Antonio J. Jimeno-Yepes, Bridget T. McInnes, and Alan R. Aronson. 2011. Exploiting mesh indexing in medline to generate a data set for word sense dis- ambiguation. BMC bioinformatics 12(1):223.</note>
</biblStruct>

<biblStruct xml:id="b14">
	<analytic>
		<title level="a" type="main">CHEMDNER: The drugs and chemical names extraction challenge</title>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Martin</forename><surname>Krallinger</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Florian</forename><surname>Leitner</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Obdulia</forename><surname>Rabal</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Miguel</forename><surname>Vazquez</surname></persName>
		</author>
	</analytic>
	<monogr>
		<title level="j">In J. Cheminformatics</title>
		<imprint>
			<date type="published" when="2015" />
		</imprint>
	</monogr>
	<note>Julen Oyarzabal, and Alfonso Valencia</note>
	<note type="raw_reference">Martin Krallinger, Florian Leitner, Obdulia Rabal, Miguel Vazquez, Julen Oyarzabal, and Alfonso Va- lencia. 2015. CHEMDNER: The drugs and chemi- cal names extraction challenge. In J. Cheminformat- ics.</note>
</biblStruct>

<biblStruct xml:id="b15">
	<analytic>
		<title level="a" type="main">Neural architectures for named entity recognition</title>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Guillaume</forename><surname>Lample</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Miguel</forename><surname>Ballesteros</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">K</forename><surname>Sandeep</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Kazuya</forename><surname>Subramanian</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Chris</forename><surname>Kawakami</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><surname>Dyer</surname></persName>
		</author>
	</analytic>
	<monogr>
		<title level="m">HLT-NAACL</title>
		<imprint>
			<date type="published" when="2016" />
		</imprint>
	</monogr>
	<note type="raw_reference">Guillaume Lample, Miguel Ballesteros, Sandeep K Subramanian, Kazuya Kawakami, and Chris Dyer. 2016. Neural architectures for named entity recog- nition. In HLT-NAACL.</note>
</biblStruct>

<biblStruct xml:id="b16">
	<monogr>
		<title level="m" type="main">Biocreative v cdr task corpus: a resource for chemical disease relation extraction. Database : the journal of biological databases and curation</title>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Jiao</forename><surname>Li</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Yueping</forename><surname>Sun</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Robin</forename><forename type="middle">J</forename><surname>Johnson</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Daniela</forename><surname>Sciaky</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Chih-Hsuan</forename><surname>Wei</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Robert</forename><surname>Leaman</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Allan</forename><forename type="middle">Peter</forename><surname>Davis</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Carolyn</forename><forename type="middle">J</forename><surname>Mattingly</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Thomas</forename><forename type="middle">C</forename><surname>Wiegers</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Zhiyong</forename><surname>Lu</surname></persName>
		</author>
		<imprint>
			<date type="published" when="2016" />
		</imprint>
	</monogr>
	<note type="raw_reference">Jiao Li, Yueping Sun, Robin J. Johnson, Daniela Sci- aky, Chih-Hsuan Wei, Robert Leaman, Allan Peter Davis, Carolyn J. Mattingly, Thomas C. Wiegers, and Zhiyong Lu. 2016. Biocreative v cdr task cor- pus: a resource for chemical disease relation extrac- tion. Database : the journal of biological databases and curation 2016.</note>
</biblStruct>

<biblStruct xml:id="b17">
	<analytic>
		<title level="a" type="main">Design challenges for entity linking</title>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Xiao</forename><surname>Ling</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Sameer</forename><surname>Singh</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Daniel</forename><forename type="middle">S</forename><surname>Weld</surname></persName>
		</author>
	</analytic>
	<monogr>
		<title level="j">Transactions of the Association for Computational Linguistics</title>
		<imprint>
			<biblScope unit="volume">3</biblScope>
			<biblScope unit="page" from="315" to="328" />
			<date type="published" when="2015" />
		</imprint>
	</monogr>
	<note type="raw_reference">Xiao Ling, Sameer Singh, and Daniel S. Weld. 2015. Design challenges for entity linking. Transactions of the Association for Computational Linguistics 3:315-328.</note>
</biblStruct>

<biblStruct xml:id="b18">
	<analytic>
		<title level="a" type="main">Distant supervision for relation extraction without labeled data</title>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Mike</forename><surname>Mintz</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Steven</forename><surname>Bills</surname></persName>
		</author>
	</analytic>
	<monogr>
		<title level="m">ACL</title>
		<imprint>
			<date type="published" when="2009" />
		</imprint>
	</monogr>
	<note>Rion Snow, and Daniel Jurafsky</note>
	<note type="raw_reference">Mike Mintz, Steven Bills, Rion Snow, and Daniel Ju- rafsky. 2009. Distant supervision for relation extrac- tion without labeled data. In ACL.</note>
</biblStruct>

<biblStruct xml:id="b19">
	<analytic>
		<title level="a" type="main">GloVe: Global vectors for word representation</title>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Jeffrey</forename><surname>Pennington</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Richard</forename><surname>Socher</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Christopher</forename><forename type="middle">D</forename><surname>Manning</surname></persName>
		</author>
	</analytic>
	<monogr>
		<title level="m">EMNLP</title>
		<imprint>
			<date type="published" when="2014" />
		</imprint>
	</monogr>
	<note type="raw_reference">Jeffrey Pennington, Richard Socher, and Christopher D. Manning. 2014. GloVe: Global vectors for word rep- resentation. In EMNLP.</note>
</biblStruct>

<biblStruct xml:id="b20">
	<analytic>
		<title level="a" type="main">Semi-supervised sequence tagging with bidirectional language models</title>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Matthew</forename><forename type="middle">E</forename><surname>Peters</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Waleed</forename><surname>Ammar</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Chandra</forename><surname>Bhagavatula</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Russell</forename><surname>Power</surname></persName>
		</author>
	</analytic>
	<monogr>
		<title level="m">ACL</title>
		<imprint>
			<date type="published" when="2017" />
		</imprint>
	</monogr>
	<note type="raw_reference">Matthew E. Peters, Waleed Ammar, Chandra Bhagavat- ula, and Russell Power. 2017. Semi-supervised se- quence tagging with bidirectional language models. In ACL.</note>
</biblStruct>

<biblStruct xml:id="b21">
	<monogr>
		<title level="m" type="main">Extracting scientific figures with distantly supervised neural networks</title>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Noah</forename><surname>Siegel</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Nicholas</forename><surname>Lourie</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Russell</forename><surname>Power</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Waleed</forename><surname>Ammar</surname></persName>
		</author>
		<imprint>
			<date type="published" when="2018" />
		</imprint>
	</monogr>
	<note>In JCDL</note>
	<note type="raw_reference">Noah Siegel, Nicholas Lourie, Russell Power, and Waleed Ammar. 2018. Extracting scientific figures with distantly supervised neural networks. In JCDL.</note>
</biblStruct>

<biblStruct xml:id="b22">
	<analytic>
		<title level="a" type="main">Identifying meaningful citations</title>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Marco</forename><surname>Valenzuela</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Vu</forename><surname>Ha</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Oren</forename><surname>Etzioni</surname></persName>
		</author>
	</analytic>
	<monogr>
		<title level="m">AAAI Workshop (Scholarly Big Data)</title>
		<imprint>
			<date type="published" when="2015" />
		</imprint>
	</monogr>
	<note type="raw_reference">Marco Valenzuela, Vu Ha, and Oren Etzioni. 2015. Identifying meaningful citations. In AAAI Workshop (Scholarly Big Data).</note>
</biblStruct>

<biblStruct xml:id="b23">
	<analytic>
		<title level="a" type="main">Clinical review: Efficacy of antimicrobial-impregnated catheters in external ventricular drainage -a systematic review and meta-analysis</title>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Xiang</forename><surname>Wang</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Yan</forename><surname>Dong</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Yi-Ming</forename><surname>Xiang Qian Qi</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Cheng-Guang</forename><surname>Li</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Lijun</forename><surname>Huang</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><surname>Hou</surname></persName>
		</author>
	</analytic>
	<monogr>
		<title level="m">Critical care</title>
		<imprint>
			<date type="published" when="2013" />
		</imprint>
	</monogr>
	<note type="raw_reference">Xiang Wang, Yan Dong, Xiang qian Qi, Yi-Ming Li, Cheng-Guang Huang, and Lijun Hou. 2013. Clin- ical review: Efficacy of antimicrobial-impregnated catheters in external ventricular drainage -a system- atic review and meta-analysis. In Critical care.</note>
</biblStruct>

<biblStruct xml:id="b24">
	<analytic>
		<title level="a" type="main">Learning to predict citation-based impact measures</title>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Luca</forename><surname>Weihs</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Oren</forename><surname>Etzioni</surname></persName>
		</author>
	</analytic>
	<monogr>
		<title level="m">JCDL</title>
		<imprint>
			<date type="published" when="2017" />
		</imprint>
	</monogr>
	<note type="raw_reference">Luca Weihs and Oren Etzioni. 2017. Learning to pre- dict citation-based impact measures. In JCDL.</note>
</biblStruct>

<biblStruct xml:id="b25">
	<analytic>
		<title level="a" type="main">CiteSeerX: AI in a digital library search engine</title>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Jian</forename><surname>Wu</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Kyle</forename><surname>Williams</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Hung-Hsuan</forename><surname>Chen</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Madian</forename><surname>Khabsa</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Cornelia</forename><surname>Caragea</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Alexander</forename><surname>Ororbia</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Douglas</forename><surname>Jordan</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">C. Lee</forename><surname>Giles</surname></persName>
		</author>
	</analytic>
	<monogr>
		<title level="m">AAAI</title>
		<imprint>
			<date type="published" when="2014" />
		</imprint>
	</monogr>
	<note type="raw_reference">Jian Wu, Kyle Williams, Hung-Hsuan Chen, Madian Khabsa, Cornelia Caragea, Alexander Ororbia, Dou- glas Jordan, and C. Lee Giles. 2014. CiteSeerX: AI in a digital library search engine. In AAAI.</note>
</biblStruct>

<biblStruct xml:id="b26">
	<monogr>
		<title level="m" type="main">Explicit semantic ranking for academic search via knowledge graph embedding</title>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Chenyan</forename><surname>Xiong</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Russell</forename><surname>Power</surname></persName>
		</author>
		<author>
			<persName xmlns="http://www.tei-c.org/ns/1.0"><forename type="first">Jamie</forename><surname>Callan</surname></persName>
		</author>
		<imprint>
			<date type="published" when="2017" />
			<publisher>WWW</publisher>
		</imprint>
	</monogr>
	<note type="raw_reference">Chenyan Xiong, Russell Power, and Jamie Callan. 2017. Explicit semantic ranking for academic search via knowledge graph embedding. In WWW.</note>
</biblStruct>

				</listBibl>
			</div>
		</back>
	</text>
</TEI>