Repository: acl-org/acl-anthology

Issue:
Wrong title for 2020.lrec-1.648
[2020.lrec-1.648](https://www.aclweb.org/anthology/2020.lrec-1.648/) is listed with an old title that contains "GUMBY", but the title should be the one used in the pdf which contains AMALGUM, i.e. `AMALGUM – A Free, Balanced, Multilayer English Web Corpus`


Discussion:

Resolving patch:
diff --git a/bin/anthology/index.py b/bin/anthology/index.py
--- a/bin/anthology/index.py
+++ b/bin/anthology/index.py
@@ -316,6 +316,15 @@ def get_ids(self, name: PersonName) -> List[str]:
 
         return sorted(self.name_to_ids[name])
 
+    def get_comment(self, id_: str) -> str:
+        """
+        Returns the comment associated with the name ID.
+
+        :param id_: The name ID (e.g., "fei-liu-ftdallas")
+        :return: The comment (e.g., "UT Dallas, Bosch, CMU, University of Central Florida")
+        """
+        return self.comments.get(id_, None)
+
     def resolve_name(self, name, id_=None):
         """Find person named 'name' and return a dict with fields
         'first', 'last', 'id'"""
diff --git a/bin/create_hugo_yaml.py b/bin/create_hugo_yaml.py
--- a/bin/create_hugo_yaml.py
+++ b/bin/create_hugo_yaml.py
@@ -133,7 +133,7 @@ class SortedVolume:
         and the Anthology ID of the current volume. For example, LREC 2020
         has the following joint events, which get sorted in the following manner:
 
-        ['2020.lrec-1', '2020.aespen -1', '2020.ai4hi-1',
+        ['2020.lrec-1', '2020.aespen-1', '2020.ai4hi-1',
         '2020.bucc-1', '2020.calcs-1', '2020.cllrd-1', '2020.clssts-1',
         '2020.cmlc-1', '2020.computerm-1', '2020.framenet-1', '2020.gamnlp-1',
         '2020.globalex-1', '2020.isa-1', '2020.iwltp-1',
@@ -160,13 +160,13 @@ def __str__(self):
             return self.anth_id
 
         def __eq__(self, other):
-            return self.anth_id == other.anth_id
+            """We define equivalence at the venue (not volume) level in order
+            to preserve the sort order found in the XML"""
+            return self.venue == other.venue
 
         def __lt__(self, other):
             """First parent volumes, then sort by venue name"""
             if self.is_parent_venue == other.is_parent_venue:
-                if self.venue == other.venue:
-                    return self.volume_id < other.volume_id
                 return self.venue < other.venue
             return self.is_parent_venue and not other.is_parent_venue
 
diff --git a/bin/ingest.py b/bin/ingest.py
--- a/bin/ingest.py
+++ b/bin/ingest.py
@@ -33,6 +33,7 @@
 import argparse
 import os
 import re
+import readline
 import shutil
 import sys
 
@@ -52,6 +53,7 @@
 from anthology.index import AnthologyIndex
 from anthology.people import PersonName
 from anthology.bibtex import read_bibtex
+from anthology.venues import VenueIndex
 
 from itertools import chain
 from typing import Dict, Any
@@ -94,7 +96,6 @@ def bib2xml(bibfilename, anthology_id):
         'doi',
     ]
 
-    # log(f"Reading {bibfilename}")
     collection_id, volume_name, paper_no = deconstruct_anthology_id(anthology_id)
     if paper_no == '':
         return  # skip the master bib file; we only process the individual files
@@ -104,7 +105,6 @@ def bib2xml(bibfilename, anthology_id):
         log(f"more than one entry in {bibfilename}")
     bibkey, bibentry = bibdata.entries.items()[0]
     if len(bibentry.fields) == 0:
-        # print(paper_no, bibentry.fields)
         log(f"parsing bib of paper {paper_no} failed")
         sys.exit(1)
 
@@ -137,8 +137,6 @@ def bib2xml(bibfilename, anthology_id):
         else:
             if field == 'url':
                 value = f"{anthology_id}"
-                if 'url' in bibentry.fields and bibentry.fields['url'] != value:
-                    log(f"rewriting url {bibentry.fields['url']} -> {value}")
             elif field in bibentry.fields:
                 value = bibentry.fields[field]
             elif field == 'bibtype':
@@ -157,9 +155,21 @@ def main(args):
     collections = defaultdict(OrderedDict)
     volumes = {}
 
+    anthology_datadir = os.path.join(os.path.dirname(sys.argv[0]), "..", "data")
+    venue_keys = [
+        venue["slug"].lower() for _, venue in VenueIndex(srcdir=anthology_datadir).items()
+    ]
+
     # Build list of volumes, confirm uniqueness
+    unseen_venues = []
     for proceedings in args.proceedings:
         meta = read_meta(os.path.join(proceedings, "meta"))
+
+        venue_name = meta["abbrev"].lower()
+
+        if venue_name not in venue_keys:
+            unseen_venues.append(meta["abbrev"])
+
         meta["path"] = proceedings
 
         meta["collection_id"] = collection_id = (
@@ -174,21 +184,28 @@ def main(args):
         collections[collection_id][volume_name] = {}
         volumes[volume_full_id] = meta
 
+    # Make sure all venues exist
+    if len(unseen_venues) > 0:
+        print("FATAL: The following venue(s) don't exist in venues.yaml")
+        for venue in unseen_venues:
+            print(f"- {venue}")
+        print("Please create entries for them and re-ingest.")
+        sys.exit(1)
+
     # Copy over the PDFs and attachments
     for volume, meta in volumes.items():
         root_path = os.path.join(meta["path"], "cdrom")
         collection_id = meta["collection_id"]
         venue_name = meta["abbrev"].lower()
         volume_name = meta["volume"]
+        year = meta["year"]
 
         pdfs_dest_dir = os.path.join(args.pdfs_dir, venue_name)
         if not os.path.exists(pdfs_dest_dir):
             os.makedirs(pdfs_dest_dir)
 
-        print(f"VOLUME: {volume}")
-
         # copy the book
-        book_src_filename = meta["abbrev"] + "-" + meta["year"]
+        book_src_filename = meta["abbrev"] + "-" + year
         book_src_path = os.path.join(root_path, book_src_filename) + ".pdf"
         book_dest_path = None
         if os.path.exists(book_src_path):
@@ -196,7 +213,7 @@ def main(args):
                 os.path.join(pdfs_dest_dir, f"{collection_id}-{volume_name}") + ".pdf"
             )
 
-            log(f"Copying {book_src_path} -> {book_dest_path}", args.dry_run)
+            # log(f"Copying {book_src_path} -> {book_dest_path}", args.dry_run)
             if not args.dry_run:
                 shutil.copyfile(book_src_path, book_dest_path)
 
@@ -222,10 +239,6 @@ def main(args):
                 pdf_dest_path = os.path.join(
                     pdfs_dest_dir, f"{collection_id}-{volume_name}.{paper_num}.pdf"
                 )
-                log(
-                    f"Copying [{paper_id_full}] from {pdf_src_path} -> {pdf_dest_path}",
-                    args.dry_run,
-                )
                 if not args.dry_run:
                     shutil.copyfile(pdf_src_path, pdf_dest_path)
 
@@ -242,35 +255,89 @@ def main(args):
             if not os.path.exists(attachments_dest_dir):
                 os.makedirs(attachments_dest_dir)
             for attachment_file in os.listdir(os.path.join(root_path, "additional")):
-                match = re.match(rf"{abbrev}(\d+)_(\w+)\.(\w+)")
-                if match is not None:
-                    paper_num, type_, ext = match.groups()
-                    paper_num = int(paper_num)
-
-                    file_name = f"{collection_id}-{volume_name}.{paper_num}.{type_}.{ext}"
-                    dest_path = os.path.join(attachments_dest_dir, file_name)
-                    log(f"Copying {attachment_file} -> {dest_path}", args.dry_run)
-                    if not args.dry_run:
-                        shutil.copyfile(attachment_file, dest_path)
-
-                    collections[collection_id][volume_name][paper_num][
-                        "attachments"
-                    ].append(dest_path)
-
-    people = AnthologyIndex(
-        None, srcdir=os.path.join(os.path.dirname(sys.argv[0]), "..", "data")
-    )
+                attachment_file_path = os.path.join(
+                    root_path, "additional", attachment_file
+                )
+                match = re.match(
+                    rf"{year}\.{venue_name}-\w+\.(\d+)_?(\w+)\.(\w+)$", attachment_file
+                )
+                if match is None:
+                    print(
+                        f"* Warning: no attachment match for {attachment_file}",
+                        file=sys.stderr,
+                    )
+                    sys.exit(2)
+
+                paper_num, type_, ext = match.groups()
+                paper_num = int(paper_num)
+
+                file_name = f"{collection_id}-{volume_name}.{paper_num}.{type_}.{ext}"
+                dest_path = os.path.join(attachments_dest_dir, file_name)
+                # log(f"Copying {attachment_file} -> {dest_path}", args.dry_run)
+                if not args.dry_run:
+                    shutil.copyfile(attachment_file_path, dest_path)
+
+                collections[collection_id][volume_name][paper_num]["attachments"].append(
+                    (dest_path, type_)
+                )
+
+    people = AnthologyIndex(None, srcdir=anthology_datadir)
+
+    def disambiguate_name(node):
+        name = PersonName.from_element(node)
+        ids = people.get_ids(name)
+
+        if node.tag == "editor":
+            # meta block, get volume
+            anth_id = build_anthology_id(
+                collection_id, node.getparent().getparent().attrib["id"]
+            )
+        elif node.tag == "author":
+            # paper, get full ID
+            anth_id = build_anthology_id(
+                collection_id,
+                node.getparent().getparent().attrib["id"],
+                node.getparent().attrib["id"],
+            )
+
+        if len(ids) > 1:
+            choice = -1
+            while choice < 0 or choice >= len(ids):
+                print(
+                    f"({anth_id}): ambiguous author {name}; Please choose from the following:"
+                )
+                for i, id_ in enumerate(ids):
+                    print(f"[{i}] {id_} ({people.get_comment(id_)})")
+                choice = int(input("--> "))
+
+            node.attrib["id"] = ids[choice]
 
     for collection_id, collection in collections.items():
+        # Newly added volumes, so we can normalize and name-disambig later
+        newly_added_volumes = []
+
         collection_file = os.path.join(
             args.anthology_dir, "data", "xml", f"{collection_id}.xml"
         )
-        root_node = make_simple_element("collection", attrib={"id": collection_id})
+        if os.path.exists(collection_file):
+            root_node = etree.parse(collection_file).getroot()
+        else:
+            root_node = make_simple_element("collection", attrib={"id": collection_id})
 
         for volume_id, volume in collection.items():
             volume_node = make_simple_element(
-                "volume", attrib={"id": volume_id}, parent=root_node
+                "volume", attrib={"id": volume_id, "ingest-date": args.ingest_date},
             )
+
+            # Replace the existing one if present
+            existing_volume_node = root_node.find(f"./volume[@id='{volume_id}']")
+            for i, child in enumerate(root_node):
+                if child.attrib["id"] == volume_id:
+                    root_node[i] = volume_node
+                    break
+            else:
+                root_node.append(volume_node)
+
             meta_node = None
 
             for paper_num, paper in sorted(volume.items()):
@@ -285,8 +352,7 @@ def main(args):
                     title_node = paper_node.find("title")
                     title_node.tag = "booktitle"
                     meta_node.append(title_node)
-                    for editor in paper_node.findall("author"):
-                        editor.tag = "editor"
+                    for editor in paper_node.findall("editor"):
                         meta_node.append(editor)
                     meta_node.append(paper_node.find("publisher"))
                     meta_node.append(paper_node.find("address"))
@@ -320,46 +386,30 @@ def main(args):
                 if url is not None:
                     url.attrib["hash"] = compute_hash_from_file(paper["pdf"])
 
-                for attachment in paper["attachments"]:
+                for path, type_ in paper["attachments"]:
                     make_simple_element(
                         "attachment",
-                        text=attachment.path,
-                        attrib={
-                            "type": attachment.type,
-                            "hash": compute_hash_from_file(attachment.path),
-                        },
+                        text=os.path.basename(path),
+                        attrib={"type": type_, "hash": compute_hash_from_file(path),},
                         parent=paper_node,
                     )
 
                 if len(paper_node) > 0:
                     volume_node.append(paper_node)
 
+                # Normalize
+                for oldnode in paper_node:
+                    normalize(oldnode, informat="latex")
+
+                for name_node in chain(
+                    paper_node.findall("./author"), paper_node.findall("./editor")
+                ):
+                    disambiguate_name(name_node)
+
         # Other data from the meta file
         if "isbn" in meta:
             make_simple_element("isbn", meta["isbn"], parent=meta_node)
 
-        # Normalize
-        for paper in root_node.findall(".//paper"):
-            for oldnode in paper:
-                normalize(oldnode, informat="latex")
-
-            # Ensure names are properly identified
-            ambiguous = {}
-            anth_id = build_anthology_id(
-                collection_id, paper.getparent().attrib["id"], paper.attrib["id"]
-            )
-
-            for node in chain(paper.findall("author"), paper.findall("editor")):
-                name = PersonName.from_element(node)
-                ids = people.get_ids(name)
-                if len(ids) > 1:
-                    print(
-                        f"WARNING ({anth_id}): ambiguous author {name}, defaulting to first of {ids}"
-                    )
-                    ambiguous[anth_id] = (name, ids)
-
-                    node.attrib["id"] = ids[0]
-
         indent(root_node)
         tree = etree.ElementTree(root_node)
         tree.write(


Test patch: