Repository: texworld/betterbib (language: python)

Issue:
Better handle contradictory journal abbreviations from Crossref
Thanks for this useful tool!

Journal abbreviations from Crossref can be a bit inconsistent.  For example, the following entries are both from PNAS: https://api.crossref.org/v1/works/10.1073/pnas.1719906115 https://api.crossref.org/v1/works/10.1073/pnas.1215175109 but the first one has a short-container-title of "Proc Natl Acad Sci USA" whereas the second one has "Proceedings of the National Academy of Sciences" which later gets abbreviated by betterbib into "Proc. Natl. Acad. Sci."

Perhaps betterbib could check whether there are cases where two different entries with the same journal name end up with two different abbreviations, and do something (prefer the Crossref abbreviation, or prefer its own autogenerated abbreviation), perhaps togglable, about it?


Discussion / hints:

Fix patch:
diff --git a/setup.cfg b/setup.cfg
index 2888eb7..0fa0a4c 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,6 +1,6 @@
 [metadata]
 name = betterbib
-version = 4.1.1
+version = 4.1.2
 author = Nico Schlömer
 author_email = nico.schloemer@gmail.com
 description = Better BibTeX data
diff --git a/src/betterbib/crossref.py b/src/betterbib/crossref.py
index 4d0d0c6..ffd9dab 100644
--- a/src/betterbib/crossref.py
+++ b/src/betterbib/crossref.py
@@ -69,13 +69,10 @@ class Crossref:
     <https://github.com/Crossref/rest-api-doc/blob/master/rest_api.md>.
     """
 
-    def __init__(self, prefer_long_journal_name=False):
+    def __init__(self):
         self.api_url = "https://api.crossref.org/works"
-        self.prefer_long_journal_name = prefer_long_journal_name
-
         requests_cache.install_cache("betterbib_cache", expire_after=3600)
         # requests_cache.remove_expired_responses()
-        return
 
     def _crossref_to_bibtex_type(self, entry):
         # The difficulty here: Translating book-chapter. If the book is a monograph (all
@@ -280,10 +277,13 @@ def _crossref_to_pybtex(self, data):
                 pass
 
         container_title = None
-        keys = ["short-container-title", "container-title"]
-        if self.prefer_long_journal_name:
-            keys = reversed(keys)
-        for key in keys:
+        # If possible, get the long journal name; it's later abbreviated if
+        # requested. See <https://github.com/nschloe/betterbib/issues/233> for
+        # an example of how the CrossRef-native short journal names are off.
+        for key in [
+            "container-title",
+            "short-container-title",
+        ]:
             if key in data and data[key]:
                 container_title = data[key][0]
                 break
diff --git a/src/betterbib/journal_abbrev.py b/src/betterbib/journal_abbrev.py
index 17653ab..6173338 100644
--- a/src/betterbib/journal_abbrev.py
+++ b/src/betterbib/journal_abbrev.py
@@ -1,35 +1,37 @@
+from __future__ import annotations
+
 import json
 from pathlib import Path
 
+from pybtex.database import Entry
+
 
-def journal_abbrev(d, long_journal_names: bool = False, custom_abbrev=None):
+def journal_abbrev(
+    d: dict[str, Entry],
+    long_journal_names: bool = False,
+    custom_abbrev: str | None = None,
+) -> dict[str, Entry]:
     this_dir = Path(__file__).resolve().parent
     with open(this_dir / "data/journals.json") as f:
         table = json.load(f)
 
     if custom_abbrev is not None:
         with open(custom_abbrev) as f:
-            custom_table = json.load(f)
-    else:
-        custom_table = {}
+            table.update(json.load(f))
 
     if long_journal_names:
-        custom_table = {v: k for k, v in custom_table.items()}
         table = {v: k for k, v in table.items()}
 
-    table.update(custom_table)
-
     # fallback option
     table_keys_lower = {k.lower(): v for k, v in table.items()}
 
     for value in d.values():
+        journal = value.fields["journal"]
         try:
-            value.fields["journal"] = table[value.fields["journal"]]
+            value.fields["journal"] = table[journal]
         except KeyError:
             try:
-                value.fields["journal"] = table_keys_lower[
-                    value.fields["journal"].lower()
-                ]
+                value.fields["journal"] = table_keys_lower[journal.lower()]
             except KeyError:
                 pass
 
diff --git a/src/betterbib/sync.py b/src/betterbib/sync.py
index b7602a2..38c29e5 100644
--- a/src/betterbib/sync.py
+++ b/src/betterbib/sync.py
@@ -2,27 +2,27 @@
 
 from concurrent.futures import ThreadPoolExecutor, as_completed
 
-import pybtex
-import pybtex.database
+from pybtex.database import Entry
 from rich.progress import track
 
 from . import crossref, dblp, errors, tools
+from .journal_abbrev import journal_abbrev
 
 
 def sync(
-    d: dict[str, pybtex.database.Entry],
+    d: dict[str, Entry],
     source: str,
-    long_journal_name: bool,
+    long_journal_names: bool,
     max_workers: int,
     verbose: bool,
-) -> dict[str, pybtex.database.Entry]:
+) -> dict[str, Entry]:
     """
     Sync a bibtex dict with an external source
 
         Parameters:
             d (dict): bibtex dict
             source (str): data source to sync against
-            long_journal_name (bool): use the long journal name instead of short
+            long_journal_names (bool): use the long journal name instead of short
             max_workers (int): number of concurrent workers to use
             verbose (bool): print additional information to stdout
 
@@ -31,7 +31,7 @@ def sync(
     """
 
     if source == "crossref":
-        src = crossref.Crossref(long_journal_name)
+        src = crossref.Crossref()
     else:
         assert source == "dblp", "Illegal source."
         src = dblp.Dblp()
@@ -59,6 +59,9 @@ def sync(
                 num_success += 1
                 d[bib_id] = tools.merge(entry, data)
 
+    # (un)abbreviate journal names
+    d = journal_abbrev(d, long_journal_names)
+
     if verbose:
         print(f"\n\nTotal number of entries: {len(d)}")
         print(f"Found: {num_success}")


Test patch:
diff --git a/tests/test_crossref.py b/tests/test_crossref.py
index b2d0bed..681e77e 100644
--- a/tests/test_crossref.py
+++ b/tests/test_crossref.py
@@ -30,7 +30,7 @@ def test_crossref_article0():
             "publisher": "Society for Industrial & Applied Mathematics (SIAM)",
             "title": "A Framework for Deflated and Augmented Krylov Subspace Methods",
             "url": "http://dx.doi.org/10.1137/110820713",
-            "journal": "SIAM J. Matrix Anal. & Appl.",
+            "journal": "SIAM Journal on Matrix Analysis and Applications",
             "number": "2",
             "month": 1,
             "volume": "34",
@@ -41,9 +41,9 @@ def test_crossref_article0():
         persons=pybtex.database.OrderedCaseInsensitiveDict(
             {
                 "author": [
-                    pybtex.database.Person("Gaul, Andr\xe9"),
+                    pybtex.database.Person("Gaul, André"),
                     pybtex.database.Person("Gutknecht, Martin H."),
-                    pybtex.database.Person("Liesen, J\xf6rg"),
+                    pybtex.database.Person("Liesen, Jörg"),
                     pybtex.database.Person("Nabben, Reinhard"),
                 ]
             }
@@ -152,7 +152,7 @@ def test_crossref_book1():
             {
                 "author": [
                     pybtex.database.Person("Golub, Gene H."),
-                    pybtex.database.Person("Meurant, G\xe9rard"),
+                    pybtex.database.Person("Meurant, Gérard"),
                 ]
             }
         ),
@@ -229,12 +229,12 @@ def test_crossref_incollection0():
         persons=pybtex.database.OrderedCaseInsensitiveDict(
             {
                 "author": [
-                    pybtex.database.Person("Beyn, Wolf-J\xfcrgen"),
+                    pybtex.database.Person("Beyn, Wolf-Jürgen"),
                     pybtex.database.Person("Champneys, Alan"),
                     pybtex.database.Person("Doedel, Eusebius"),
                     pybtex.database.Person("Govaerts, Willy"),
                     pybtex.database.Person("Kuznetsov, Yuri A."),
-                    pybtex.database.Person("Sandstede, Bj\xf6rn"),
+                    pybtex.database.Person("Sandstede, Björn"),
                 ]
             }
         ),
@@ -344,7 +344,7 @@ def test_crossref_proceedings0():
 
 
 def test_doi_only():
-    source = betterbib.Crossref(prefer_long_journal_name=True)
+    source = betterbib.Crossref()
 
     bt = source.get_by_doi("10.1137/110820713")
     reference = pybtex.database.Entry(
@@ -410,7 +410,7 @@ def test_crossref_all_capitals():
             "publisher": "World Scientific Pub Co Pte Lt",
             "title": "ONTOLOGICAL COGNITIVE MAP",
             "url": "http://dx.doi.org/10.1142/s0218213009000366",
-            "journal": "Int. J. Artif. Intell. Tools",
+            "journal": "International Journal on Artificial Intelligence Tools",
             "number": "05",
             "month": 10,
             "volume": "18",
@@ -464,7 +464,7 @@ def test_subtitle():
             "title": "TUBE",
             "subtitle": "time-dependent pricing for mobile data",
             "url": "http://dx.doi.org/10.1145/2377677.2377723",
-            "journal": "SIGCOMM Comput. Commun. Rev.",
+            "journal": "ACM SIGCOMM Computer Communication Review",
             "issn": "0146-4833",
             "number": "4",
             "month": 9,