Repository: texworld/betterbib (language: python)

Issue:
ambersand in journal names
When running `betterbib` with a journal name that has `&` the output is not converted to `\&` in bib resulting in a compilation error when using biblatex.

For example:

```
@article{nouy,
 author = {Nouy},
 title = {M\'ethode de construction de bases spectrales g\'en\'eralis\'ees pour l'approximation de probl\`emes stochastiques},
 journal = {M\'ecanique \& Industries},
}
```

Results in an output with 
```
 journal = {M\'ecanique & Industries},
```

which gives a `Misplaced alignment tab character &.` when used in biblatex.



Discussion / hints:

Fix patch:
diff --git a/README.md b/README.md
index 70e423b..585b9f6 100644
--- a/README.md
+++ b/README.md
@@ -67,7 +67,7 @@ is converted to
 Use `-i`/`--in-place` to modify the input file in place. Use `-h`/`--help` to see all
 options.
 
-<!--pytest-codeblocks:skipif(sys.version_info >= (3, 10))-->
+<!--pytest-codeblocks:skipif(sys.version_info < (3, 10))-->
 
 ```sh
 betterbib up -h
@@ -77,15 +77,15 @@ betterbib up -h
 
 ```
 usage: betterbib update [-h] [-i] [-b] [-t] [-d {braces,quotes}]
-                        [-u {unchanged,new,short}] [-p PAGE_RANGE_SEPARATOR]
-                        [-s {crossref,dblp}] [-l]
-                        [--extra-abbrev-file EXTRA_ABBREV_FILE] [-c N] [-a]
+                        [--doi-url-type {unchanged,new,short}]
+                        [-p PAGE_RANGE_SEPARATOR] [-s {crossref,dblp}] [-l]
+                        [--extra-abbrev-file EXTRA_ABBREV_FILE] [-c N] [-u]
                         infiles [infiles ...]
 
 positional arguments:
   infiles               input BibTeX files (default: stdin)
 
-optional arguments:
+options:
   -h, --help            show this help message and exit
   -i, --in-place        modify infile in place
   -s {crossref,dblp}, --source {crossref,dblp}
@@ -96,7 +96,7 @@ optional arguments:
                         custom journal abbreviations, as JSON file
   -c N, --num-concurrent-requests N
                         number of concurrent HTTPS requests (default: 10)
-  -a, --latex-output    force LaTeX output (default: unicode)
+  -u, --unicode-output  unicode output (default: escape special characters)
 
 Formatting:
   -b, --sort-by-bibkey  sort entries by BibTeX key (default: false)
@@ -104,7 +104,7 @@ Formatting:
   -d {braces,quotes}, --delimiter-type {braces,quotes}
                         which delimiters to use in the output file (default:
                         braces {...})
-  -u {unchanged,new,short}, --doi-url-type {unchanged,new,short}
+  --doi-url-type {unchanged,new,short}
                         DOI URL (new: https://doi.org/<DOI> (default), short:
                         https://doi.org/abcde)
   -p PAGE_RANGE_SEPARATOR, --page-range-separator PAGE_RANGE_SEPARATOR
diff --git a/setup.cfg b/setup.cfg
index 6e3bc30..9845dc5 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,6 +1,6 @@
 [metadata]
 name = betterbib
-version = 4.1.4
+version = 4.2.0
 author = Nico Schlömer
 author_email = nico.schloemer@gmail.com
 description = Better BibTeX data
diff --git a/src/betterbib/adapt_doi_urls.py b/src/betterbib/adapt_doi_urls.py
index 015cfd4..5a880a3 100644
--- a/src/betterbib/adapt_doi_urls.py
+++ b/src/betterbib/adapt_doi_urls.py
@@ -1,32 +1,37 @@
+from __future__ import annotations
+
 from typing import Callable
 
+from pybtex.database import Entry
+
 from . import tools
 
 
-def adapt_doi_urls(d, doi_url_type: str):
+def adapt_doi_urls(d: dict[str, Entry], doi_url_type: str) -> None:
     if doi_url_type == "new":
-        d = _update_doi_url(d, lambda doi: "https://doi.org/" + doi)
+        _update_doi_url(d, lambda doi: f"https://doi.org/{doi}")
+
     elif doi_url_type == "short":
 
-        def update_to_short_doi(doi):
+        def update_to_short_doi(doi: str) -> str | None:
             short_doi = tools.get_short_doi(doi)
             if short_doi:
-                return "https://doi.org/" + short_doi
+                return f"https://doi.org/{short_doi}"
             return None
 
-        d = _update_doi_url(d, update_to_short_doi)
+        _update_doi_url(d, update_to_short_doi)
+
     else:
         assert doi_url_type == "unchanged"
 
-    return d
 
+def _update_doi_url(d: dict[str, Entry], url_from_doi: Callable[[str], str]) -> None:
+    for bib_id, value in d.items():
+        if "url" not in value.fields:
+            continue
 
-def _update_doi_url(d, url_from_doi: Callable[[str], str]):
-    for bib_id in d:
-        if "url" in d[bib_id].fields:
-            doi = tools.doi_from_url(d[bib_id].fields["url"])
-            if doi:
-                new_url = url_from_doi(doi)
-                if new_url:
-                    d[bib_id].fields["url"] = new_url
-    return d
+        doi = tools.doi_from_url(value.fields["url"])
+        if doi:
+            new_url = url_from_doi(doi)
+            if new_url:
+                d[bib_id].fields["url"] = new_url
diff --git a/src/betterbib/cli/_doi_to_bibtex.py b/src/betterbib/cli/_doi_to_bibtex.py
index 8880d66..c0e8ee0 100644
--- a/src/betterbib/cli/_doi_to_bibtex.py
+++ b/src/betterbib/cli/_doi_to_bibtex.py
@@ -1,7 +1,7 @@
 from unidecode import unidecode
 
 from .. import crossref
-from ..tools import dict_to_string, write
+from ..tools import dict_to_string, set_page_range_separator, write
 from .helpers import add_formatting_parser_arguments
 
 
@@ -11,12 +11,10 @@ def run(args):
 
     bibtex_citekey = _create_citekey_for_entry(entry)
 
-    string = dict_to_string(
-        {bibtex_citekey: entry},
-        args.delimiter_type,
-        tab_indent=args.tab_indent,
-        page_range_separator="--",
-    )
+    d = {bibtex_citekey: entry}
+    set_page_range_separator(d, "--")
+
+    string = dict_to_string(d, args.delimiter_type, tab_indent=args.tab_indent)
 
     write(string)
 
diff --git a/src/betterbib/cli/_format.py b/src/betterbib/cli/_format.py
index c491617..25b38af 100644
--- a/src/betterbib/cli/_format.py
+++ b/src/betterbib/cli/_format.py
@@ -1,5 +1,11 @@
 from ..adapt_doi_urls import adapt_doi_urls
-from ..tools import bibtex_parser, dict_to_string, filter_fields, write
+from ..tools import (
+    bibtex_parser,
+    dict_to_string,
+    filter_fields,
+    set_page_range_separator,
+    write,
+)
 from .helpers import add_file_parser_arguments, add_formatting_parser_arguments
 
 
@@ -17,12 +23,13 @@ def run(args):
             tuples = sorted(data.entries.items())
 
         d = dict(tuples)
-        d = adapt_doi_urls(d, args.doi_url_type)
+        adapt_doi_urls(d, args.doi_url_type)
+        set_page_range_separator(d, "--")
+
         string = dict_to_string(
             d,
             args.delimiter_type,
             tab_indent=args.tab_indent,
-            page_range_separator="--",
             # FIXME: use public argument when it becomes possible
             preamble=data._preamble,
         )
diff --git a/src/betterbib/cli/_journal_abbrev.py b/src/betterbib/cli/_journal_abbrev.py
index 1e090b7..89f4664 100644
--- a/src/betterbib/cli/_journal_abbrev.py
+++ b/src/betterbib/cli/_journal_abbrev.py
@@ -1,5 +1,5 @@
 from ..journal_abbrev import journal_abbrev
-from ..tools import bibtex_parser, dict_to_string, write
+from ..tools import bibtex_parser, dict_to_string, set_page_range_separator, write
 from .helpers import add_file_parser_arguments, add_formatting_parser_arguments
 
 
@@ -8,11 +8,10 @@ def run(args):
         data = bibtex_parser(infile)
         d = dict(data.entries.items())
 
-        d = journal_abbrev(d, args.long_journal_names, args.extra_abbrev_file)
+        journal_abbrev(d, args.long_journal_names, args.extra_abbrev_file)
+        set_page_range_separator(d, "--")
 
-        string = dict_to_string(
-            d, "braces", tab_indent=False, page_range_separator="--"
-        )
+        string = dict_to_string(d, "braces", tab_indent=False)
 
         write(string, infile if args.in_place else None)
 
diff --git a/src/betterbib/cli/_update.py b/src/betterbib/cli/_update.py
index c213e02..d1dbaa3 100644
--- a/src/betterbib/cli/_update.py
+++ b/src/betterbib/cli/_update.py
@@ -1,7 +1,14 @@
 from ..adapt_doi_urls import adapt_doi_urls
 from ..journal_abbrev import journal_abbrev
 from ..sync import sync
-from ..tools import bibtex_parser, dict_to_string, sanitize_title, write
+from ..tools import (
+    bibtex_parser,
+    dict_to_string,
+    preserve_title_capitalization,
+    remove_multiple_spaces,
+    set_page_range_separator,
+    write,
+)
 from .helpers import add_file_parser_arguments, add_formatting_parser_arguments
 
 
@@ -23,16 +30,17 @@ def run(args):
             args.num_concurrent_requests,
             args.in_place,
         )
-        d = adapt_doi_urls(d, args.doi_url_type)
-        sanitize_title(d)
-        d = journal_abbrev(d, args.long_journal_names, args.extra_abbrev_file)
+        adapt_doi_urls(d, args.doi_url_type)
+        preserve_title_capitalization(d)
+        set_page_range_separator(d, "--")
+        remove_multiple_spaces(d)
+        journal_abbrev(d, args.long_journal_names, args.extra_abbrev_file)
 
         string = dict_to_string(
             d,
             args.delimiter_type,
             tab_indent=args.tab_indent,
-            unicode=not args.latex_output,
-            page_range_separator="--",
+            unicode=args.unicode_output,
         )
 
         write(string, infile if args.in_place else None)
@@ -70,10 +78,10 @@ def add_args(parser):
         help="number of concurrent HTTPS requests (default: 10)",
     )
     parser.add_argument(
-        "-a",
-        "--latex-output",
+        "-u",
+        "--unicode-output",
         action="store_true",
         default=False,
-        help="force LaTeX output (default: unicode)",
+        help="unicode output (default: escape special characters)",
     )
     return parser
diff --git a/src/betterbib/cli/helpers.py b/src/betterbib/cli/helpers.py
index 995ce3d..6ca1889 100644
--- a/src/betterbib/cli/helpers.py
+++ b/src/betterbib/cli/helpers.py
@@ -55,7 +55,6 @@ def add_formatting_parser_arguments(parser):
         help=("which delimiters to use in the output file " "(default: braces {...})"),
     )
     formatting_group.add_argument(
-        "-u",
         "--doi-url-type",
         choices=["unchanged", "new", "short"],
         default="new",
diff --git a/src/betterbib/journal_abbrev.py b/src/betterbib/journal_abbrev.py
index 1202b02..eb05715 100644
--- a/src/betterbib/journal_abbrev.py
+++ b/src/betterbib/journal_abbrev.py
@@ -10,7 +10,7 @@ def journal_abbrev(
     d: dict[str, Entry],
     long_journal_names: bool = False,
     custom_abbrev: str | None = None,
-) -> dict[str, Entry]:
+) -> None:
     this_dir = Path(__file__).resolve().parent
     with open(this_dir / "data/journals.json") as f:
         table = json.load(f)
@@ -37,5 +37,3 @@ def journal_abbrev(
                 value.fields["journal"] = table_keys_lower[journal.lower()]
             except KeyError:
                 pass
-
-    return d
diff --git a/src/betterbib/sync.py b/src/betterbib/sync.py
index 38c29e5..45125ae 100644
--- a/src/betterbib/sync.py
+++ b/src/betterbib/sync.py
@@ -60,7 +60,7 @@ def sync(
                 d[bib_id] = tools.merge(entry, data)
 
     # (un)abbreviate journal names
-    d = journal_abbrev(d, long_journal_names)
+    journal_abbrev(d, long_journal_names)
 
     if verbose:
         print(f"\n\nTotal number of entries: {len(d)}")
diff --git a/src/betterbib/tools.py b/src/betterbib/tools.py
index 2b07604..b815355 100644
--- a/src/betterbib/tools.py
+++ b/src/betterbib/tools.py
@@ -214,7 +214,7 @@ def _translate_title(val, dictionary=create_dict()):
     return " ".join(words)
 
 
-def sanitize_title(d) -> None:
+def preserve_title_capitalization(d: dict[str, Entry]) -> None:
     for _, entry in d.items():
         try:
             title = entry.fields["title"]
@@ -223,11 +223,48 @@ def sanitize_title(d) -> None:
             warn(f"'entry' {entry} has no title")
 
 
+def set_page_range_separator(d: dict[str, Entry], string: str) -> None:
+    # Replace any number of dashes (hypen, en dash, em dash etc.) by
+    # page_range_separator; see
+    # <https://tex.stackexchange.com/a/58671/13262>
+    # <https://jkorpela.fi/dashes.html>
+    chars = "".join(
+        [
+            "-",
+            "\N{HYPHEN}",
+            "\N{NON-BREAKING HYPHEN}",
+            "\N{FIGURE DASH}",
+            "\N{EN DASH}",
+            "\N{EM DASH}",
+            "\N{HORIZONTAL BAR}",
+        ]
+    )
+    for _, entry in d.items():
+        if "pages" not in entry.fields:
+            continue
+        entry.fields["pages"] = re.sub(f"[{chars}]+", string, entry.fields["pages"])
+
+
+def remove_multiple_spaces(d: dict[str, Entry]) -> None:
+    for _, entry in d.items():
+        for key, value in entry.fields.items():
+            if key in ["url", "doi"]:
+                continue
+            try:
+                value = re.sub(" +", " ", value)
+                # Remove trailing spaces
+                value = value.rstrip()
+            except TypeError:
+                # expected unicode for encode input, but got int instead
+                pass
+            else:
+                entry.fields[key] = value
+
+
 def pybtex_to_bibtex_string(
     entry: Entry,
     bibtex_key: str,
     delimiters: tuple[str, str] = ("{", "}"),
-    page_range_separator: str = "--",
     indent: str = " ",
     sort: bool = False,
     unicode: bool = True,
@@ -255,43 +292,25 @@ def pybtex_to_bibtex_string(
     for key in keys:
         value = entry.fields[key]
 
-        try:
-            value = value.replace("\N{REPLACEMENT CHARACTER}", "?")
-        except AttributeError:
-            pass
-
-        try:
-            if key not in ["url", "doi"]:
-                # Parse the original value to get a unified version
-                # value = translator.latex_to_text(value)
-                # # back to latex to escape "&" etc.
-                # value = unicode_to_latex(value)
-                # Replace multiple spaces by one
-                value = re.sub(" +", " ", value)
-                # Remove trailing spaces
-                value = value.rstrip()
-        except TypeError:
-            # expected unicode for encode input, but got int instead
-            pass
-
         # Always make keys lowercase
         key = key.lower()
 
-        if key == "pages":
-            # Replace any number of dashes (hypen, en dash, em dash etc.) by
-            # page_range_separator; see
-            # <https://tex.stackexchange.com/a/58671/13262>
-            # <https://jkorpela.fi/dashes.html>
-            value = re.sub(
-                "[-\u2010\u2011\u2012\u2013\u2014\u2015]+", page_range_separator, value
-            )
-
         if key == "month":
             month_string = translate_month(value)
             if month_string:
                 content.append(f"{key} = {month_string}")
             continue
 
+        try:
+            value = value.replace("\N{REPLACEMENT CHARACTER}", "?")
+        except AttributeError:
+            pass
+
+        # skip title; otherwise, the "protective" braces (e.g., "{Krylov}") get
+        # escaped as well
+        if not unicode and key != "title":
+            value = unicode_to_latex(value)
+
         if value is not None:
             content.append(f"{key} = {left}{value}{right}")
 
@@ -438,7 +457,6 @@ def heuristic_unique_result(results, d):
 def dict_to_string(
     od: dict[str, Entry],
     delimiter_type: str,
-    page_range_separator: str,
     tab_indent: bool,
     preamble: list | None = None,
     unicode: bool = True,
@@ -468,19 +486,17 @@ def dict_to_string(
         )
 
     # Add segments for each bibtex entry in order
-    segments.extend(
-        [
-            pybtex_to_bibtex_string(
-                d,
-                bib_id,
-                delimiters=delimiters,
-                page_range_separator=page_range_separator,
-                indent="\t" if tab_indent else " ",
-                unicode=unicode,
-            )
-            for bib_id, d in od.items()
-        ]
-    )
+    segments += [
+        pybtex_to_bibtex_string(
+            d,
+            bib_id,
+            delimiters=delimiters,
+            indent="\t" if tab_indent else " ",
+            unicode=unicode,
+        )
+        for bib_id, d in od.items()
+    ]
+
     return "\n\n".join(segments) + "\n"
 
 


Test patch:
diff --git a/tests/test_journal_abbrev.py b/tests/test_journal_abbrev.py
index ee6a36b..a29b7c9 100644
--- a/tests/test_journal_abbrev.py
+++ b/tests/test_journal_abbrev.py
@@ -30,8 +30,8 @@ def test_standard_abbrev():
     abbrevs = [PNAS_ABBREV]
     for journal, abbrev in zip(journals, abbrevs):
         entries = make_fake_entry(journal)
-        abbreviated = betterbib.journal_abbrev(entries)
-        assert abbreviated["foo"].fields["journal"] == abbrev
+        betterbib.journal_abbrev(entries)
+        assert entries["foo"].fields["journal"] == abbrev
 
 
 @pytest.mark.skipif(not data_file_exists, reason="Data file missing")
@@ -42,16 +42,16 @@ def test_custom_abbrev():
         f.write(super_short)
 
     entries = make_fake_entry("PNAS")
-    abbreviated = betterbib.journal_abbrev(entries, custom_abbrev=infile)
-    assert abbreviated["foo"].fields["journal"] == PNAS_ABBREV
+    betterbib.journal_abbrev(entries, custom_abbrev=infile)
+    assert entries["foo"].fields["journal"] == PNAS_ABBREV
 
     super_short = '{"' + FULL_PNAS + '": "PNAS"}'
     with open(infile, "w") as f:
         f.write(super_short)
 
     entries = make_fake_entry(FULL_PNAS)
-    abbreviated = betterbib.journal_abbrev(entries, custom_abbrev=infile)
-    assert abbreviated["foo"].fields["journal"] == "PNAS"
+    betterbib.journal_abbrev(entries, custom_abbrev=infile)
+    assert entries["foo"].fields["journal"] == "PNAS"
 
 
 @pytest.mark.skipif(not data_file_exists, reason="Data file missing")
@@ -60,8 +60,8 @@ def test_standard_abbrev_long():
     abbrevs = [PNAS_ABBREV]
     for journal, abbrev in zip(journals, abbrevs):
         entries = make_fake_entry(abbrev)
-        abbreviated = betterbib.journal_abbrev(entries, long_journal_names=True)
-        assert abbreviated["foo"].fields["journal"] == journal
+        betterbib.journal_abbrev(entries, long_journal_names=True)
+        assert entries["foo"].fields["journal"] == journal
 
 
 @pytest.mark.skipif(not data_file_exists, reason="Data file missing")
@@ -72,5 +72,5 @@ def test_custom_abbrev_long():
         f.write(super_short)
 
     entries = make_fake_entry(PNAS_ABBREV)
-    abbreviated = betterbib.journal_abbrev(entries, long_journal_names=True)
-    assert abbreviated["foo"].fields["journal"] == FULL_PNAS
+    betterbib.journal_abbrev(entries, long_journal_names=True)
+    assert entries["foo"].fields["journal"] == FULL_PNAS
diff --git a/tests/test_tools.py b/tests/test_tools.py
index 827bace..c4f9cad 100644
--- a/tests/test_tools.py
+++ b/tests/test_tools.py
@@ -45,14 +45,16 @@ def test_journal_name():
         fields={"journal": "SIAM Journal on Matrix Analysis and Applications"},
     )
 
-    tmp = betterbib.journal_abbrev({"key": lng})
+    tmp = {"key": lng}
+    betterbib.journal_abbrev(tmp)
     assert tmp["key"].fields["journal"] == shrt.fields["journal"]
 
     lng = pybtex.database.Entry(
         "article",
         fields={"journal": "SIAM Journal on Matrix Analysis and Applications"},
     )
-    tmp = betterbib.journal_abbrev({"key": shrt}, long_journal_names=True)
+    tmp = {"key": shrt}
+    betterbib.journal_abbrev(tmp, long_journal_names=True)
     assert tmp["key"].fields["journal"] == lng.fields["journal"]