# https://github.com/arXiv/arxiv-base@32e6ad0 """ Copyright 2017 Cornell University Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ """Parse Authors lines to extract author and affiliation data.""" import re import os import gzip import json from itertools import dropwhile from typing import Dict, Iterator, List, Tuple from multiprocessing import Pool, cpu_count from arxiv_public_data.tex2utf import tex2utf from arxiv_public_data.config import LOGGER, DIR_OUTPUT logger = LOGGER.getChild('authorsplit') PREFIX_MATCH = 'van|der|de|la|von|del|della|da|mac|ter|dem|di|vaziri' """ Takes data from an Author: line in the current arXiv abstract file and returns a structured set of data: author_list_ptr = [ [ author1_keyname, author1_firstnames, author1_suffix, affil1, affil2 ] , [ author2_keyname, author2_firstnames, author1_suffix, affil1 ] , [ author3_keyname, author3_firstnames, author1_suffix ] ] Abstracted from Dienst software for OAI1 and other uses. This routine should just go away when a better metadata structure is adopted that deals with names and affiliations properly. Must remember that there is at least one person one the archive who has only one name, this should clearly be considered the key name. Code originally written by Christina Scovel, Simeon Warner Dec99/Jan00 2000-10-16 - separated. 2000-12-07 - added support for suffix 2003-02-14 - get surname prefixes from arXiv::Filters::Index [Simeon] 2007-10-01 - created test script, some tidying [Simeon] 2018-05-25 - Translated from Perl to Python [Brian C.] """ def parse_author_affil(authors: str) -> List[List[str]]: """ Parse author line and returns an list of author and affiliation data. The list for each author will have at least three elements for keyname, firstname(s) and suffix. The keyname will always have content but the other strings might be empty strings if there is no firstname or suffix. Any additional elements after the first three are affiliations, there may be zero or more. Handling of prefix "XX collaboration" etc. is duplicated here and in arXiv::HTML::AuthorLink -- it shouldn't be. Likely should just be here. This routine is just a wrapper around the two parts that first split the authors line into parts, and then back propagate the affiliations. The first part is to be used along for display where we do not want to back propagate affiliation information. :param authors: string of authors from abs file or similar :return: Returns a structured set of data: author_list_ptr = [ [ author1_keyname, author1_firstnames, author1_suffix, affil1, affil2 ], [ author2_keyname, author2_firstnames, author1_suffix, affil1 ] , [ author3_keyname, author3_firstnames, author1_suffix ] ] """ return _parse_author_affil_back_propagate( **_parse_author_affil_split(authors)) def _parse_author_affil_split(author_line: str) -> Dict: """ Split author line into author and affiliation data. Take author line, tidy spacing and punctuation, and then split up into individual author an affiliation data. Has special cases to avoid splitting an initial collaboration name and records in $back_propagate_affiliation_to the fact that affiliations should not be back propagated to collaboration names. Does not handle multiple collaboration names. """ if not author_line: return {'author_list': [], 'back_prop': 0} names: List[str] = split_authors(author_line) if not names: return {'author_list': [], 'back_prop': 0} names = _remove_double_commas(names) # get rid of commas at back namesIter: Iterator[str] = reversed( list(dropwhile(lambda x: x == ',', reversed(names)))) # get rid of commas at front names = list(dropwhile(lambda x: x == ',', namesIter)) # Extract all names (all parts not starting with comma or paren) names = list(map(_tidy_name, filter( lambda x: re.match('^[^](,]', x), names))) names = list(filter(lambda n: not re.match( r'^\s*et\.?\s+al\.?\s*', n, flags=re.IGNORECASE), names)) (names, author_list, back_propagate_affiliations_to) = _collaboration_at_start(names) (enumaffils) = _enum_collaboration_at_end(author_line) # Split name into keyname and firstnames/initials. # Deal with different patterns in turn: prefixes, suffixes, plain # and single name. patterns = [('double-prefix', r'^(.*)\s+(' + PREFIX_MATCH + r')\s(' + PREFIX_MATCH + r')\s(\S+)$'), ('name-prefix-name', r'^(.*)\s+(' + PREFIX_MATCH + r')\s(\S+)$'), ('name-name-prefix', r'^(.*)\s+(\S+)\s(I|II|III|IV|V|Sr|Jr|Sr\.|Jr\.)$'), ('name-name', r'^(.*)\s+(\S+)$'), ] # Now go through names in turn and try to get affiliations # to go with them for name in names: pattern_matches = ((mtype, re.match(m, name, flags=re.IGNORECASE)) for (mtype, m) in patterns) (mtype, match) = next(((mtype, m) for (mtype, m) in pattern_matches if m is not None), ('default', None)) if match is None: author_entry = [name, '', ''] elif mtype == 'double-prefix': s = '{} {} {}'.format(match.group( 2), match.group(3), match.group(4)) author_entry = [s, match.group(1), ''] elif mtype == 'name-prefix-name': s = '{} {}'.format(match.group(2), match.group(3)) author_entry = [s, match.group(1), ''] elif mtype == 'name-name-prefix': author_entry = [match.group(2), match.group(1), match.group(3)] elif mtype == 'name-name': author_entry = [match.group(2), match.group(1), ''] else: author_entry = [name, '', ''] # search back in author_line for affiliation author_entry = _add_affiliation( author_line, enumaffils, author_entry, name) author_list.append(author_entry) return {'author_list': author_list, 'back_prop': back_propagate_affiliations_to} def parse_author_affil_utf(authors: str) -> List: """ Call parse_author_affil() and do TeX to UTF conversion. Output structure is the same but should be in UTF and not TeX. """ if not authors: return [] return list(map(lambda author: list(map(tex2utf, author)), parse_author_affil(authors))) def _remove_double_commas(items: List[str]) -> List[str]: parts: List[str] = [] last = '' for pt in items: if pt == ',' and last == ',': continue else: parts.append(pt) last = pt return parts def _tidy_name(name: str) -> str: name = re.sub(r'\s\s+', ' ', name) # also gets rid of CR # add space after dot (except in TeX) name = re.sub(r'(?', name) return name def _collaboration_at_start(names: List[str]) \ -> Tuple[List[str], List[List[str]], int]: """Perform special handling of collaboration at start.""" author_list = [] back_propagate_affiliations_to = 0 while len(names) > 0: m = re.search(r'([a-z0-9\s]+\s+(collaboration|group|team))', names[0], flags=re.IGNORECASE) if not m: break # Add to author list author_list.append([m.group(1), '', '']) back_propagate_affiliations_to += 1 # Remove from names names.pop(0) # Also swallow and following comma or colon if names and (names[0] == ',' or names[0] == ':'): names.pop(0) return names, author_list, back_propagate_affiliations_to def _enum_collaboration_at_end(author_line: str)->Dict: """Get separate set of enumerated affiliations from end of author_line.""" # Now see if we have a separate set of enumerated affiliations # This is indicated by finding '(\s*(' line_m = re.search(r'\(\s*\((.*)$', author_line) if not line_m: return {} enumaffils = {} affils = re.sub(r'\s*\)\s*$', '', line_m.group(1)) # Now expect to have '1) affil1 (2) affil2 (3) affil3' for affil in affils.split('('): # Now expect `1) affil1 ', discard if no match m = re.match(r'^(\d+)\)\s*(\S.*\S)\s*$', affil) if m: enumaffils[m.group(1)] = re.sub(r'[\.,\s]*$', '', m.group(2)) return enumaffils def _add_affiliation(author_line: str, enumaffils: Dict, author_entry: List[str], name: str) -> List: """ Add author affiliation to author_entry if one is found in author_line. This should deal with these cases Smith B(labX) Smith B(1) Smith B(1, 2) Smith B(1 & 2) Smith B(1 and 2) """ en = re.escape(name) namerex = r'{}\s*\(([^\(\)]+)'.format(en.replace(' ', 's*')) m = re.search(namerex, author_line, flags=re.IGNORECASE) if not m: return author_entry # Now see if we have enumerated references (just commas, digits, &, and) affils = m.group(1).rstrip().lstrip() affils = re.sub(r'(&|and)/,', ',', affils, flags=re.IGNORECASE) if re.match(r'^[\d,\s]+$', affils): for affil in affils.split(','): if affil in enumaffils: author_entry.append(enumaffils[affil]) else: author_entry.append(affils) return author_entry def _parse_author_affil_back_propagate(author_list: List[List[str]], back_prop: int) -> List[List[str]]: """Back propagate author affiliation. Take the author list structure generated by parse_author_affil_split(..) and propagate affiliation information backwards to preceeding author entries where none was give. Stop before entry $back_prop to avoid adding affiliation information to collaboration names. given, eg: a.b.first, c.d.second (affil) implies a.b.first (affil), c.d.second (affil) and in more complex cases: a.b.first, c.d.second (1), e.f.third, g.h.forth (2,3) implies a.b.first (1), c.d.second (1), e.f.third (2,3), g.h.forth (2,3) """ last_affil: List[str] = [] for x in range(len(author_list) - 1, max(back_prop - 1, -1), -1): author_entry = author_list[x] if len(author_entry) > 3: # author has affiliation,store last_affil = author_entry elif last_affil: # author doesn't have affil but later one did => copy author_entry.extend(last_affil[3:]) return author_list def split_authors(authors: str) -> List: """ Split author string into authors entity lists. Take an author line as a string and return a reference to a list of the different name and affiliation blocks. While this does normalize spacing and 'and', it is a key feature that the set of strings returned can be concatenated to reproduce the original authors line. This code thus provides a very graceful degredation for badly formatted authors lines, as the text at least shows up. """ # split authors field into blocks with boundaries of ( and ) if not authors: return [] aus = re.split(r'(\(|\))', authors) aus = list(filter(lambda x: x != '', aus)) blocks = [] if len(aus) == 1: blocks.append(authors) else: c = '' depth = 0 for bit in aus: if bit == '': continue if bit == '(': # track open parentheses depth += 1 if depth == 1: blocks.append(c) c = '(' else: c = c + bit elif bit == ')': # track close parentheses depth -= 1 c = c + bit if depth == 0: blocks.append(c) c = '' else: # haven't closed, so keep accumulating continue else: c = c + bit if c: blocks.append(c) listx = [] for block in blocks: block = re.sub(r'\s+', ' ', block) if re.match(r'^\(', block): # it is a comment listx.append(block) else: # it is a name block = re.sub(r',?\s+(and|\&)\s', ',', block) names = re.split(r'(,|:)\s*', block) for name in names: if not name: continue name = name.rstrip().lstrip() if name: listx.append(name) # Recombine suffixes that were separated with a comma parts: List[str] = [] for p in listx: if re.match(r'^(Jr\.?|Sr\.?\[IV]{2,})$', p) \ and len(parts) >= 2 \ and parts[-1] == ',' \ and not re.match(r'\)$', parts[-2]): separator = parts.pop() last = parts.pop() recomb = "{}{} {}".format(last, separator, p) parts.append(recomb) else: parts.append(p) return parts def parse_authorline(authors: str) -> str: """ The external facing function from this module. Converts a complex authorline into a simple one with only UTF-8. Parameters ---------- authors : string The raw author line from the metadata Returns ------- clean_authors : string String represeting cleaned author line Examples -------- >>> parse_authorline('A. Losev, S. Shadrin, I. Shneiberg') 'Losev, A.; Shadrin, S.; Shneiberg, I.' >>> parse_authorline("C. Bal\\'azs, E. L. Berger, P. M. Nadolsky, C.-P. Yuan") 'Balázs, C.; Berger, E. L.; Nadolsky, P. M.; Yuan, C. -P.' >>> parse_authorline('Stephen C. Power (Lancaster University), Baruch Solel (Technion)') 'Power, Stephen C.; Solel, Baruch' >>> parse_authorline("L. Scheck (1), H.-Th. Janka (1), T. Foglizzo (2), and K. Kifonidis (1)\n ((1) MPI for Astrophysics, Garching; (2) Service d'Astrophysique, CEA-Saclay)") 'Scheck, L.; Janka, H. -Th.; Foglizzo, T.; Kifonidis, K.' """ names = parse_author_affil_utf(authors) return '; '.join([', '.join([q for q in n[:2] if q]) for n in names]) def _parse_article_authors(article_author): try: return [article_author[0], parse_author_affil_utf(article_author[1])] except Exception as e: msg = "Author split failed for article {}".format(article_author[0]) logger.error(msg) logger.exception(e) return [article_author[0], ''] def parse_authorline_parallel(article_authors, n_processes=None): """ Parallelize `parse_authorline` Parameters ---------- article_authors : list list of tuples (arXiv id, author strings from metadata) (optional) n_processes : int number of processes Returns ------- authorsplit : list list of author strings in standardized format [ [ author1_keyname, author1_firstnames, author1_suffix, affil1, affil2 ] , [ author2_keyname, author2_firstnames, author1_suffix, affil1 ] , [ author3_keyname, author3_firstnames, author1_suffix ] ] """ logger.info( 'Parsing author lines for {} articles...'.format(len(article_authors)) ) pool = Pool(n_processes) parsed = pool.map(_parse_article_authors, article_authors) outdict = {aid: auth for aid, auth in parsed} filename = os.path.join(DIR_OUTPUT, 'authors-parsed.json.gz') logger.info('Saving to {}'.format(filename)) with gzip.open(filename, 'wb') as fout: fout.write(json.dumps(outdict).encode('utf-8'))