Repository: acl-org/acl-anthology

Issue:
No 2022 on the start page
Someone^tm should add 2022 to the years on the start page.


Discussion:

Resolving patch:
diff --git a/bin/ingest.py b/bin/ingest.py
--- a/bin/ingest.py
+++ b/bin/ingest.py
@@ -100,7 +100,7 @@ def maybe_copy(source_path, dest_path):
     if not os.path.exists(dest_path) or compute_hash_from_file(
         source_path
     ) != compute_hash_from_file(dest_path):
-        log(f"Copying {source_path} -> {dest_path}", args.dry_run)
+        log(f"Copying {source_path} -> {dest_path}")
         shutil.copyfile(source_path, dest_path)
 
 
@@ -235,9 +235,9 @@ def disambiguate_name(node, anth_id):
 
     # Build list of volumes, confirm uniqueness
     unseen_venues = []
+
     for proceedings in args.proceedings:
         meta = read_meta(os.path.join(proceedings, "meta"))
-
         venue_abbrev = meta["abbrev"]
         venue_slug = venue_index.get_slug(venue_abbrev)
 
diff --git a/bin/ingest_aclpub2.py b/bin/ingest_aclpub2.py
new file mode 100644
--- /dev/null
+++ b/bin/ingest_aclpub2.py
@@ -0,0 +1,625 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright 2021, 2022 Xinru Yan <xinru1414@gmail.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Usage:
+#   python bin/ingest_aclpub2.py
+#
+#
+#
+from collections import defaultdict
+import click
+import yaml
+import re
+import sys
+import os
+import glob
+import PyPDF2
+from pathlib import Path
+from datetime import datetime
+from collections import defaultdict
+from itertools import chain
+import lxml.etree as etree
+from typing import Dict, List, Tuple, Any, Optional
+from ingest import maybe_copy
+
+from normalize_anth import normalize
+from anthology.index import AnthologyIndex
+from anthology.venues import VenueIndex
+from anthology.people import PersonName
+from anthology.utils import (
+    make_simple_element,
+    indent,
+    compute_hash_from_file,
+)
+
+
+def disambiguate_name(node, anth_id, people):
+    name = PersonName.from_element(node)
+    ids = people.get_ids(name)
+    choice = -1
+    if len(ids) > 1:
+        while choice < 0 or choice >= len(ids):
+            print(
+                f'({anth_id}): ambiguous author {name}; Please choose from the following:'
+            )
+            for i, id_ in enumerate(ids):
+                print(f'[{i}] {id_} ({people.get_comment(id_)})')
+            choice = int(input("--> "))
+
+    return ids[choice], choice
+
+
+def correct_caps(name):
+    '''
+    Many people submit their names in "ALL CAPS" or "all lowercase".
+    Correct this with heuristics.
+    '''
+    if name.islower() or name.isupper():
+        # capitalize all parts
+        corrected = " ".join(list(map(lambda x: x.capitalize(), name.split())))
+        print(
+            f"-> Correcting capitalization of '{name}' to '{corrected}'",
+            file=sys.stderr,
+        )
+        name = corrected
+    return name
+
+
+def parse_conf_yaml(ingestion_dir: str) -> Dict[str, Any]:
+    '''
+    poss meta keys = [
+                 'book_title',
+                 'event_name',
+                 'cover_subtitle',
+                 'anthology_venue_id',
+                 'volume',
+                 'start_date',
+                 'end_date',
+                 'isbn',
+                 'location',
+                 'editors',
+                 'publisher'
+                 ]
+    must meta keys = [
+                 'book_title',
+                 'anthology_venue_id',
+                 'volume',
+                 'month',
+                 'year',
+                 'location',
+                 'editors',
+                 'publisher'
+                ]
+
+    anthology_venue_id == abbrev
+    event_name == title
+    cover_subtitle == shortbooktitle
+    '''
+    if os.path.exists(Path(ingestion_dir + 'inputs/conference_details.yml')):
+        meta = yaml.safe_load(
+            Path(ingestion_dir + 'inputs/conference_details.yml').read_text()
+        )
+    else:
+        meta = yaml.safe_load(
+            Path(ingestion_dir + 'input/conference_details.yml').read_text()
+        )
+    meta['month'] = meta['start_date'].strftime('%B')
+    meta['year'] = str(meta['start_date'].year)
+
+    must_have_keys = [
+        'book_title',
+        'anthology_venue_id',
+        'volume',
+        'month',
+        'year',
+        'location',
+        'editors',
+        'publisher',
+        'event_name',
+    ]
+    for key in must_have_keys:
+        assert key in meta.keys(), f'{key} is missing in the conference_details.yml file'
+
+    if re.match(rf'^[a-z0-9]+$', meta['volume']) is None:
+        raise Exception(
+            f"Invalid volume key '{meta['volume']}' in {ingestion_dir + 'inputs/conference_details.yml'}"
+        )
+
+    return meta
+
+
+def parse_paper_yaml(ingestion_dir: str) -> List[Dict[str, str]]:
+    if os.path.exists(Path(ingestion_dir + 'inputs/conference_details.yml')):
+        papers = yaml.safe_load(Path(ingestion_dir + 'inputs/papers.yml').read_text())
+    else:
+        papers = yaml.safe_load(Path(ingestion_dir + 'input/papers.yml').read_text())
+    return papers
+
+
+def add_paper_nums_in_paper_yaml(
+    papers: List[Dict[str, str]], ingestion_dir: str
+) -> List[Dict[str, str]]:
+    start, end = 1, 0
+    for paper in papers:
+        paper_id = str(paper['id'])
+        paper_name = paper['file']
+        if os.path.exists(f'{ingestion_dir}inputs/papers/{paper_id}.pdf'):
+            paper_need_read_path = f'{ingestion_dir}inputs/papers/{paper_id}.pdf'
+        elif os.path.exists(f'{ingestion_dir}input/papers/{paper_id}.pdf'):
+            paper_need_read_path = f'{ingestion_dir}input/papers/{paper_id}.pdf'
+        elif os.path.exists(f'{ingestion_dir}inputs/papers/{paper_name}'):
+            paper_need_read_path = f'{ingestion_dir}inputs/papers/{paper_name}'
+        elif os.path.exists(f'{ingestion_dir}input/papers/{paper_name}'):
+            paper_need_read_path = f'{ingestion_dir}input/papers/{paper_name}'
+        else:
+            paper_need_read_path = None
+        assert paper_need_read_path, f'{paper_id} path is None'
+        pdf = open(paper_need_read_path, 'rb')
+        pdf_reader = PyPDF2.PdfFileReader(pdf)
+        start = end + 1
+        end = start + pdf_reader.numPages - 1
+        paper['pages'] = f'{start}-{end}'
+    return papers
+
+
+def create_des_path(org_dir_name: str, venue_name: str) -> str:
+    dest_dir = os.path.join(org_dir_name, venue_name)
+    if not os.path.exists(dest_dir):
+        os.makedirs(dest_dir)
+    return dest_dir
+
+
+def find_paper_attachment(paper_name: str, attachments_dir: str) -> Optional[str]:
+    '''
+    files in the attachments folder need to be named filename.zip
+    '''
+    attachment_path = None
+    for filename in glob.glob(attachments_dir + '/*'):
+        if os.path.splitext(os.path.split(filename)[1])[0] == paper_name:
+            attachment_path = filename
+            break
+    return attachment_path
+
+
+def proceeding2xml(anthology_id: str, meta: Dict[str, Any]):
+    fields = [
+        'editor',
+        'booktitle',
+        'month',
+        'year',
+        'url',
+    ]
+    paper = make_simple_element('paper', attrib={'id': '0'})
+    for field in fields:
+        if field == 'editor':
+            authors = meta['editors']
+            for author in authors:
+                name_node = make_simple_element(field, parent=paper)
+                make_simple_element('first', author['first_name'], parent=name_node)
+                make_simple_element('last', author['last_name'], parent=name_node)
+        else:
+            if field == 'url':
+                value = f'{anthology_id}'
+            elif field == 'booktitle':
+                value = meta['book_title']
+            elif field == 'month':
+                value = meta['month']
+            elif field == 'year':
+                value = meta['year']
+
+            try:
+                make_simple_element(field, text=value, parent=paper)
+            except:
+                print(f"Couldn't process {paper} for {anthology_id}", file=sys.stderr)
+                sys.exit(2)
+    return paper
+
+
+def paper2xml(
+    paper_item: Dict[str, str], paper_num: int, anthology_id: str, meta: Dict[str, Any]
+):
+    '''
+    paper keys = ['abstract',
+                  'attachments',
+                  'attributes',
+                  'authors',
+                  'decision',
+                  'file',
+                  'id',
+                  'openreview_id',
+                  'pdf_file',
+                  'title']
+    author keys = ['emails',
+                   'first_name',
+                   'google_scholar_id',
+                   'homepage',
+                   'last_name',
+                   'name',
+                   'semantic_scholar_id',
+                   'username']
+    '''
+    fields = [
+        'title',
+        'author',
+        'pages',
+        'abstract',
+        'url',
+        'doi',
+        'language',
+    ]
+    paper = make_simple_element('paper', attrib={'id': str(paper_num)})
+    for field in fields:
+        if field == 'author':
+            authors = paper_item['authors']
+            for author in authors:
+                name_node = make_simple_element(field, parent=paper)
+                make_simple_element('first', author['first_name'], parent=name_node)
+                make_simple_element('last', author['last_name'], parent=name_node)
+        else:
+            if field == 'url':
+                value = f'{anthology_id}'
+            elif field == 'abstract':
+                value = paper_item['abstract'].replace('\n', '')
+            elif field == 'title':
+                value = paper_item[field]
+            elif field == 'pages':
+                value = paper_item[field]
+            else:
+                continue
+
+            try:
+                make_simple_element(field, text=value, parent=paper)
+            except:
+                print(f"Couldn't process {paper} for {anthology_id}", file=sys.stderr)
+                sys.exit(2)
+    return paper
+
+
+def process_procedding(
+    ingestion_dir: str,
+    anthology_datadir: str,
+    venue_index: VenueIndex,
+    venue_keys: List[str],
+) -> Tuple[str, Dict[str, Any]]:
+    meta = parse_conf_yaml(ingestion_dir)
+    venue_abbrev = meta["anthology_venue_id"]
+    venue_slug = venue_index.get_slug(venue_abbrev)
+
+    if str(datetime.now().year) in venue_abbrev:
+        print(f"Fatal: Venue assembler put year in acronym: '{venue_abbrev}'")
+        sys.exit(1)
+
+    if re.match(r".*\d$", venue_abbrev) is not None:
+        print(
+            f"WARNING: Venue {venue_abbrev} ends in a number, this is probably a mistake"
+        )
+
+    if venue_slug not in venue_keys:
+        event_name = meta['event_name']
+        assert (
+            re.match(r'(.)* [Ww]orkshop', event_name) is None
+        ), f"event name should start with Workshop or The Workshop, instead it started with {re.match(r'(.)* [Ww]orkshop', event_name)[0]}"
+        venue_index.add_venue(venue_abbrev, meta['event_name'])
+        print(f"Creating venue '{venue_abbrev}' ({event_name})")
+        venue_index.dump(directory=anthology_datadir)
+
+    meta["path"] = ingestion_dir
+    meta["collection_id"] = collection_id = meta["year"] + "." + venue_slug
+    volume_name = meta["volume"].lower()
+    volume_full_id = f"{collection_id}-{volume_name}"
+
+    # if "sig" in meta:
+    #     print(
+    #         f"Add this line to {anthology_datadir}/sigs/{meta['sig'].lower()}.yaml:"
+    #     )
+    #     print(f"  - {meta['year']}:")
+    #     print(f"    - {volume_full_id} # {meta['booktitle']}")
+
+    # print(f'volume_full_id {volume_full_id} meta {meta}')
+    return volume_full_id, meta
+
+
+def copy_pdf_and_attachment(
+    meta: Dict[str, Any],
+    pdfs_dir: str,
+    attachments_dir: str,
+    papers: List[Dict[str, str]],
+    dry_run: bool,
+) -> Tuple[Dict[str, Dict[str, str]], str, str, str]:
+    volume = dict()
+    collection_id = meta['collection_id']
+    venue_name = meta['anthology_venue_id'].lower()
+    volume_name = meta['volume'].lower()
+
+    pdfs_dest_dir = create_des_path(pdfs_dir, venue_name)
+
+    pdfs_src_dir = os.path.join(meta['path'], 'watermarked_pdfs')
+
+    # copy proceedings.pdf
+    proceedings_pdf_src_path = os.path.join(meta['path'], 'proceedings.pdf')
+    assert os.path.exists(proceedings_pdf_src_path), f'proceedings.pdf was not found'
+    proceedings_pdf_dest_path = (
+        os.path.join(pdfs_dest_dir, f"{collection_id}-{volume_name}") + ".pdf"
+    )
+    if dry_run:
+        print(
+            f'would\'ve moved {proceedings_pdf_src_path} to {proceedings_pdf_dest_path}'
+        )
+    if not dry_run:
+        maybe_copy(proceedings_pdf_src_path, proceedings_pdf_dest_path)
+
+    paper_id_full = f'{collection_id}-{volume_name}.0'
+    volume[0] = {
+        'anthology_id': paper_id_full,
+        'pdf': proceedings_pdf_dest_path,
+        'attachments': [],
+    }
+
+    for i, paper in enumerate(papers):
+        # copy pdf
+        paper_name = paper['file']
+        paper_id = str(paper['id'])
+        paper_num = i + 1
+        paper_id_full = f'{collection_id}-{volume_name}.{paper_num}'
+
+        if os.path.exists(os.path.join(pdfs_src_dir, paper_name)):
+            pdf_src_path = os.path.join(pdfs_src_dir, paper_name)
+        elif os.path.exists(os.path.join(pdfs_src_dir, f'{paper_id}.pdf')):
+            pdf_src_path = os.path.join(pdfs_src_dir, f'{paper_id}.pdf')
+        else:
+            pdf_src_path = None
+        assert pdf_src_path, f'{paper_name} path is None'
+        pdf_dest_path = os.path.join(
+            pdfs_dest_dir, f"{collection_id}-{volume_name}.{paper_num}.pdf"
+        )
+        if dry_run:
+            print(f'would\'ve moved {pdf_src_path} to {pdf_dest_path}')
+        if not dry_run:
+
+            maybe_copy(pdf_src_path, pdf_dest_path)
+
+        volume[paper_num] = {
+            'anthology_id': paper_id_full,
+            'pdf': pdf_dest_path,
+            'attachments': [],
+        }
+        # copy attachments
+        if 'attachments' in paper.keys():
+            attchs_dest_dir = create_des_path(attachments_dir, venue_name)
+            attchs_src_dir = os.path.join(meta['path'], 'attachments')
+            assert os.path.exists(
+                attchs_src_dir
+            ), f'paper {i, paper_name} contains attachments but attachments folder was not found'
+            attch_src_path = find_paper_attachment(
+                str(os.path.splitext(paper_name)[0]), attchs_src_dir
+            )
+            print(f'{attch_src_path}')
+            assert attch_src_path, f'{paper_name} attachment path is None'
+            _, attch_src_extension = os.path.splitext(attch_src_path)
+            type_ = paper['attachments'][0]['type']
+            file_name = (
+                f'{collection_id}-{volume_name}.{paper_num}.{type_}{attch_src_extension}'
+            )
+            attch_dest_path = os.path.join(attchs_dest_dir, file_name)
+
+            if dry_run:
+                print(f'would\'ve moved {attch_src_path} to {attch_dest_path}')
+            if not dry_run:
+                maybe_copy(attch_src_path, attch_dest_path)
+            volume[paper_num]['attachments'].append((attch_dest_path, type_))
+    return volume, collection_id, volume_name, proceedings_pdf_dest_path
+
+
+def create_xml(
+    volume: Dict[str, Dict[str, str]],
+    anthology_dir: str,
+    ingest_date: str,
+    collection_id: str,
+    volume_name: str,
+    meta: Dict[str, Any],
+    proceedings_pdf_dest_path: str,
+    people,
+    papers: List[Dict[str, str]],
+) -> None:
+    collection_file = os.path.join(anthology_dir, 'data', 'xml', f'{collection_id}.xml')
+    if os.path.exists(collection_file):
+        root_node = etree.parse(collection_file).getroot()
+    else:
+        root_node = make_simple_element('collection', attrib={'id': collection_id})
+
+    volume_node = make_simple_element(
+        'volume',
+        attrib={'id': volume_name, 'ingest-date': ingest_date},
+    )
+    # Replace the existing one if present
+    existing_volume_node = root_node.find(f"./volume[@id='{volume_name}']")
+    for i, child in enumerate(root_node):
+        if child.attrib['id'] == volume_name:
+            root_node[i] = volume_node
+            break
+    else:
+        root_node.append(volume_node)
+
+    meta_node = None
+
+    for paper_num, paper in sorted(volume.items()):
+        paper_id_full = paper['anthology_id']
+        # print(f'creating xml for paper name {paper}, in papers {papers[paper_num-1]}')
+        if paper_num == 0:
+            paper_node = proceeding2xml(paper_id_full, meta)
+        else:
+            paper_node = paper2xml(papers[paper_num - 1], paper_num, paper_id_full, meta)
+
+        if paper_node.attrib['id'] == '0':
+            # create metadata subtree
+            meta_node = make_simple_element('meta', parent=volume_node)
+            title_node = paper_node.find('booktitle')
+            meta_node.append(title_node)
+            for editor in paper_node.findall('./editor'):
+                disamb_name, name_choice = disambiguate_name(
+                    editor, paper_id_full, people
+                )
+                if name_choice != -1:
+                    editor.attrib['id'] = disamb_name
+                person = PersonName.from_element(editor)
+                for name_part in editor:
+                    name_part.text = correct_caps(name_part.text)
+                meta_node.append(editor)
+
+            # Get the publisher from the meta file
+            publisher_node = make_simple_element('publisher', meta['publisher'])
+            meta_node.append(publisher_node)
+
+            # Get the address from the meta file
+            address_node = make_simple_element("address", meta['location'])
+            meta_node.append(address_node)
+
+            meta_node.append(paper_node.find('month'))
+            meta_node.append(paper_node.find('year'))
+
+            make_simple_element(
+                'url',
+                text=f"{collection_id}-{volume_name}",
+                attrib={'hash': compute_hash_from_file(proceedings_pdf_dest_path)},
+                parent=meta_node,
+            )
+
+            # modify frontmatter tag
+            paper_node.tag = 'frontmatter'
+            del paper_node.attrib['id']
+
+        url = paper_node.find('./url')
+        # if url is not None:
+        url.attrib['hash'] = compute_hash_from_file(paper['pdf'])
+
+        for path, type_ in paper['attachments']:
+            make_simple_element(
+                'attachment',
+                text=os.path.basename(path),
+                attrib={
+                    'type': type_,
+                    'hash': compute_hash_from_file(path),
+                },
+                parent=paper_node,
+            )
+
+        if len(paper_node) > 0:
+            volume_node.append(paper_node)
+
+        # Normalize
+        for oldnode in paper_node:
+            normalize(oldnode, informat='latex')
+
+        # Adjust the language tag
+        # language_node = paper_node.find('./language')
+        # if language_node is not None:
+        #     try:
+        #         lang = iso639.languages.get(name=language_node.text)
+        #     except KeyError:
+        #         raise Exception(f"Can't find language '{language_node.text}'")
+        #     language_node.text = lang.part3
+
+        # Fix author names
+        for name_node in paper_node.findall('./author'):
+            disamb_name, name_choice = disambiguate_name(name_node, paper_id_full, people)
+            if name_choice != -1:
+                name_node.attrib['id'] = disamb_name
+            person = PersonName.from_element(name_node)
+            for name_part in name_node:
+                name_part.text = correct_caps(name_part.text)
+
+    # Other data from the meta file
+    # if 'isbn' in meta:
+    #     make_simple_element('isbn', meta['isbn'], parent=meta_node)
+
+    indent(root_node)
+    tree = etree.ElementTree(root_node)
+    tree.write(collection_file, encoding='UTF-8', xml_declaration=True, with_tail=True)
+
+
+@click.command()
+@click.option(
+    '-i',
+    '--ingestion_dir',
+    help='Directory contains proceedings need to be ingested',
+)
+@click.option(
+    '-p',
+    '--pdfs_dir',
+    default=os.path.join(os.environ['HOME'], 'anthology-files', 'pdf'),
+    help='Root path for placement of PDF files',
+)
+@click.option(
+    '-a',
+    '--attachments_dir',
+    default=os.path.join(os.environ['HOME'], 'anthology-files', 'attachments'),
+    help='Root path for placement of attachment files',
+)
+@click.option(
+    '-n',
+    '--dry_run',
+    default=False,
+    help='Do not actually copy anything',
+)
+@click.option(
+    '-r',
+    '--anthology_dir',
+    default=os.path.join(os.path.dirname(sys.argv[0]), ".."),
+    help='Root path of ACL Anthology Github repo.',
+)
+@click.option(
+    '-d',
+    '--ingest_date',
+    default=f'{datetime.now().year}-{datetime.now().month:02d}-{datetime.now().day:02d}',
+    help='Ingestion date',
+)
+def main(ingestion_dir, pdfs_dir, attachments_dir, dry_run, anthology_dir, ingest_date):
+    anthology_datadir = os.path.join(os.path.dirname(sys.argv[0]), "..", "data")
+    venue_index = VenueIndex(srcdir=anthology_datadir)
+    venue_keys = [venue["slug"].lower() for _, venue in venue_index.items()]
+
+    people = AnthologyIndex(srcdir=anthology_datadir)
+    # people.bibkeys = load_bibkeys(anthology_datadir)
+
+    volume_full_id, meta = process_procedding(
+        ingestion_dir, anthology_datadir, venue_index, venue_keys
+    )
+    papers = parse_paper_yaml(ingestion_dir)
+    # print(f'original paper {papers[0]}')
+    papers = add_paper_nums_in_paper_yaml(papers, ingestion_dir)
+    # print(f'updated paper {papers[0]}')
+    (
+        volume,
+        collection_id,
+        volume_name,
+        proceedings_pdf_dest_path,
+    ) = copy_pdf_and_attachment(meta, pdfs_dir, attachments_dir, papers, dry_run)
+    create_xml(
+        volume=volume,
+        anthology_dir=anthology_dir,
+        ingest_date=ingest_date,
+        collection_id=collection_id,
+        volume_name=volume_name,
+        meta=meta,
+        proceedings_pdf_dest_path=proceedings_pdf_dest_path,
+        people=people,
+        papers=papers,
+    )
+
+
+if __name__ == '__main__':
+    main()


Test patch: