Repository: acl-org/acl-anthology

Issue:
merge yaml venues files
For the ID transition scheduled for January 1, 2020 (#291), we will need the following for every venue:

- A primary key, its code, e.g., `acl` or `mtsummit`
- Its long name
- Whether it's an ACL event or not
- Its short letter under the old system, e.g., `P`

I propose we merge `venues.yaml` and `venues_letters.yaml` to consolidate all of this information, and to use a structure that can grow, should other information arise.

Note that this could also be used with [acleanbib](https://github.com/Olamyy/acleanbib) for producing more concise citations in a references section.


Discussion:
For supporting the new ID format in #732, I've created a new YAML file that currently looks as follows:

```yaml
test4nlp:
  venue: WS
  joint:
    2020.test4nlp-1: ACL
```

This file maps *venue primary keys* (like `test4nlp`) to *top-level venues* (here: WS), which correspond to the short letters under the old system and are currently used to group venues on the front page.

The `joint` key can assign additional venues to individual volumes. I haven't done so yet, but we could easily add keys here for a venue's short name, long name, and status as an ACL event.

Note that this file currently co-exists with the old files `venues.yaml`, `venues_letters.yaml`, and `venues_joint_map.yaml`. It's not so trivial to merge all of these into this new format, as the old-style volumes only encode their *top-level* venue in their IDs (by way of the short letter), while looking up volumes in this new YAML file requires knowing the *specific venue key*. Importantly though, if we also gather short/long names and ACL event status in the new YAML file, it should not be necessary to modify the old files in any way when adding new volumes from 2020 onwards.
This looks good to me.

Couldn't this file at least subsume `venues.yaml`, however, if we folded that file into it? 
(In fact, if so, we could just update `venues.yaml` to this new format, instead of creating a new file)

Resolving patch:
diff --git a/bin/anthology/venues.py b/bin/anthology/venues.py
--- a/bin/anthology/venues.py
+++ b/bin/anthology/venues.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright 2019 Marcel Bollmann <marcel@bollmann.me>
+# Copyright 2019-2020 Marcel Bollmann <marcel@bollmann.me>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,6 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from collections import defaultdict
 from slugify import slugify
 import logging as log
 import yaml
@@ -28,46 +29,62 @@
 
 class VenueIndex:
     def __init__(self, srcdir=None):
-        self.venues, self.letters, self.joint_map = {}, {}, {}
-        self.mapping = {}  # map new-style IDs to venues
+        self.venues, self.letters, self.joint_map = {}, {}, defaultdict(list)
+        self.acronyms_by_key = {}
         if srcdir is not None:
             self.load_from_dir(srcdir)
 
     def load_from_dir(self, directory):
         with open("{}/yaml/venues.yaml".format(directory), "r") as f:
             venue_dict = yaml.load(f, Loader=Loader)
-            for acronym, name_str in venue_dict.items():
-                name, venue_type = name_str.split(":")
-                self.venues[acronym] = {
-                    "name": name,
-                    "is_acl": (venue_type == "ACL"),
-                    "is_toplevel": False,
-                    "slug": slugify(acronym),
-                    "type": venue_type,
-                    "years": set(),
-                    "volumes": [],
-                }
-        with open("{}/yaml/venues_letters.yaml".format(directory), "r") as f:
-            self.letters = yaml.load(f, Loader=Loader)
-            for letter, acronym in self.letters.items():
-                self.venues[acronym]["is_toplevel"] = True
-                self.venues[acronym]["main_letter"] = letter
-        with open("{}/yaml/venues_joint_map.yaml".format(directory), "r") as f:
-            map_dict = yaml.load(f, Loader=Loader)
-            for id_, joint in map_dict.items():
-                if isinstance(joint, str):
-                    joint = [joint]
-                self.joint_map[id_] = joint
-        with open("{}/yaml/venues_mapping.yaml".format(directory), "r") as f:
+            for key, val in venue_dict.items():
+                if "acronym" not in val:
+                    log.critical(f"Venues must have 'acronym' - none defined for '{key}'")
+                if "name" not in val:
+                    log.error(f"Venues must have 'name' - none defined for '{key}'")
+                if val["acronym"] in self.venues:
+                    log.critical(
+                        f"Venue acronyms must be unique - '{val['acronym']}' used"
+                        f" for '{self.venues[val['acronym']]['slug']}' and '{key}'"
+                    )
+
+                if "is_toplevel" not in val:  # defaults to False
+                    val["is_toplevel"] = False
+                if "is_acl" not in val:  # defaults to False
+                    val["is_acl"] = False
+                if "joint" in val:
+                    if isinstance(val["joint"], str):
+                        val["joint"] = [val["joint"]]
+                val["years"] = set()
+                val["volumes"] = list()
+                val["slug"] = key
+                # for legacy reasons, this dict is still indexed by acronym
+                # rather than key (that's what I get for not using proper
+                # encapsulation --MB)
+                self.venues[val["acronym"]] = val
+                self.acronyms_by_key[key] = val["acronym"]
+
+                if "oldstyle_letter" in val:
+                    if not val["is_toplevel"]:
+                        log.error(
+                            f"Venues with old-style letter must be top-level - '{key}' is not"
+                        )
+                    self.letters[val["oldstyle_letter"]] = val["acronym"]
+
+        with open("{}/yaml/joint.yaml".format(directory), "r") as f:
             map_dict = yaml.load(f, Loader=Loader)
-            for id_, data in map_dict.items():
-                if data["venue"] not in self.venues:
-                    log.critical(f"Venue '{data['venue']}' not defined in venues.yaml")
-                self.mapping[id_] = data["venue"]
-                for volume_id, joint in data.get("joint", {}).items():
-                    if isinstance(joint, str):
-                        joint = [joint]
-                    self.joint_map[volume_id] = joint
+            for venue, data in map_dict.items():
+                acronym = self.acronyms_by_key[venue]
+                if isinstance(data, dict):
+                    idlist = [id_ for ids in data.values() for id_ in ids]
+                elif isinstance(data, list):
+                    idlist = data
+                else:
+                    log.exception(
+                        f"Values in joint.yaml must be dict or list, found: {type(data)}"
+                    )
+                for id_ in idlist:
+                    self.joint_map[id_].append(acronym)
 
     def get_by_letter(self, letter):
         """Get a venue acronym by first letter (e.g., Q -> TACL)."""
@@ -75,21 +92,21 @@ def get_by_letter(self, letter):
             return self.letters[letter]
         except KeyError:
             log.critical("Unknown venue letter: {}".format(letter))
-            log.critical(
-                "Maybe '{}' needs to be defined in venues_letters.yaml?".format(letter)
-            )
 
     def get_main_venue(self, anthology_id):
         """Get a venue acronym by anthology ID (e.g., acl -> ACL)."""
         collection_id, *_ = deconstruct_anthology_id(anthology_id)
         if is_newstyle_id(collection_id):
-            return self.mapping[collection_id.split(".")[-1]]
+            return self.acronyms_by_key[collection_id.split(".")[-1]]
         else:
             return self.get_by_letter(collection_id[0])
 
     def get_associated_venues(self, anthology_id):
         """Get a list of all venue acronyms for a given (volume) anthology ID."""
-        venues = [self.get_main_venue(anthology_id)]
+        main_venue = self.get_main_venue(anthology_id)
+        venues = [main_venue]
+        if "joint" in self.venues[main_venue]:
+            venues += self.venues[main_venue]["joint"]
         if anthology_id in self.joint_map:
             venues += self.joint_map[anthology_id]
         return sorted(set(venues))
diff --git a/bin/create_hugo_pages.py b/bin/create_hugo_pages.py
--- a/bin/create_hugo_pages.py
+++ b/bin/create_hugo_pages.py
@@ -167,9 +167,6 @@ def create_venues_and_events(srcdir, clean=False):
         with open("{}/content/venues/{}.md".format(srcdir, venue_str), "w") as f:
             print("---", file=f)
             yaml_data = {"venue": venue, "title": venue_data["name"]}
-            if venue_data["is_toplevel"]:
-                main_letter = venue_data["main_letter"]
-                yaml_data["aliases"] = ["/papers/{}/".format(main_letter)]
             yaml.dump(yaml_data, default_flow_style=False, stream=f)
             print("---", file=f)
 
@@ -189,12 +186,6 @@ def create_venues_and_events(srcdir, clean=False):
                     "year": year,
                     "title": "{} ({})".format(venue_data["name"], year),
                 }
-                if venue_data["is_toplevel"]:
-                    main_letter = venue_data["main_letter"]
-                    main_prefix = main_letter + year[-2:]  # e.g., P05
-                    yaml_data["aliases"] = [
-                        "/papers/{}/{}/".format(main_letter, main_prefix)
-                    ]
                 yaml.dump(yaml_data, default_flow_style=False, stream=f)
                 print("---", file=f)
 


Test patch: