Repository: texworld/betterbib (language: python)

Issue:
Using the correct dash for page ranges
In the last version of betterbib, I noticed that the page range are using `-` or sometimes a unicode character for en-dash (I guess depending on the original CrossRef entry). Can these always be replaced by `--`?


Discussion / hints:

Fix patch:
diff --git a/README.md b/README.md
index 19825ba..70e423b 100644
--- a/README.md
+++ b/README.md
@@ -21,11 +21,15 @@ comparing them with online sources and correcting whatever entries are faulty.
 
 Install with
 
+<!--pytest-codeblocks:skip-->
+
 ```sh
 pip install betterbib
 ```
 
-and run simply run
+and simply run
+
+<!--pytest-codeblocks:skip-->
 
 ```sh
 betterbib update in.bib   # or short `betterbib up`
@@ -63,19 +67,61 @@ is converted to
 Use `-i`/`--in-place` to modify the input file in place. Use `-h`/`--help` to see all
 options.
 
-#### Sync
+<!--pytest-codeblocks:skipif(sys.version_info >= (3, 10))-->
+
+```sh
+betterbib up -h
+```
+
+<!--pytest-codeblocks:expected-output-->
+
+```
+usage: betterbib update [-h] [-i] [-b] [-t] [-d {braces,quotes}]
+                        [-u {unchanged,new,short}] [-p PAGE_RANGE_SEPARATOR]
+                        [-s {crossref,dblp}] [-l]
+                        [--extra-abbrev-file EXTRA_ABBREV_FILE] [-c N] [-a]
+                        infiles [infiles ...]
+
+positional arguments:
+  infiles               input BibTeX files (default: stdin)
+
+optional arguments:
+  -h, --help            show this help message and exit
+  -i, --in-place        modify infile in place
+  -s {crossref,dblp}, --source {crossref,dblp}
+                        data source (default: crossref)
+  -l, --long-journal-names
+                        prefer long journal names (default: false)
+  --extra-abbrev-file EXTRA_ABBREV_FILE
+                        custom journal abbreviations, as JSON file
+  -c N, --num-concurrent-requests N
+                        number of concurrent HTTPS requests (default: 10)
+  -a, --latex-output    force LaTeX output (default: unicode)
+
+Formatting:
+  -b, --sort-by-bibkey  sort entries by BibTeX key (default: false)
+  -t, --tab-indent      use tabs for indentation (default: false)
+  -d {braces,quotes}, --delimiter-type {braces,quotes}
+                        which delimiters to use in the output file (default:
+                        braces {...})
+  -u {unchanged,new,short}, --doi-url-type {unchanged,new,short}
+                        DOI URL (new: https://doi.org/<DOI> (default), short:
+                        https://doi.org/abcde)
+  -p PAGE_RANGE_SEPARATOR, --page-range-separator PAGE_RANGE_SEPARATOR
+                        page range separator (default: --)
+```
 
 betterbib fetches data from
 
 - [Crossref](http://www.crossref.org/) (default) or
 - [DBLP](http://dblp.uni-trier.de/) (`--source dblp`).
 
-All betterbib-sync command-line options are explained in `betterbib-sync -h`.
-
 #### Format
 
 The tool
 
+<!--pytest-codeblocks:skip-->
+
 ```sh
 betterbib format in.bib
 ```
diff --git a/setup.cfg b/setup.cfg
index 309797b..2888eb7 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,6 +1,6 @@
 [metadata]
 name = betterbib
-version = 4.1.0
+version = 4.1.1
 author = Nico Schlömer
 author_email = nico.schloemer@gmail.com
 description = Better BibTeX data
diff --git a/src/betterbib/__init__.py b/src/betterbib/__init__.py
index e258bad..03090ad 100644
--- a/src/betterbib/__init__.py
+++ b/src/betterbib/__init__.py
@@ -8,10 +8,10 @@
 from .tools import (
     create_dict,
     decode,
+    dict_to_string,
     merge,
     pybtex_to_bibtex_string,
     pybtex_to_dict,
-    to_string,
     translate_month,
 )
 
@@ -23,7 +23,7 @@
     "errors",
     "pybtex_to_dict",
     "pybtex_to_bibtex_string",
-    "to_string",
+    "dict_to_string",
     "merge",
     "translate_month",
     "Crossref",
diff --git a/src/betterbib/cli/_doi_to_bibtex.py b/src/betterbib/cli/_doi_to_bibtex.py
index 8bb3c56..8880d66 100644
--- a/src/betterbib/cli/_doi_to_bibtex.py
+++ b/src/betterbib/cli/_doi_to_bibtex.py
@@ -1,7 +1,7 @@
 from unidecode import unidecode
 
 from .. import crossref
-from ..tools import to_string, write
+from ..tools import dict_to_string, write
 from .helpers import add_formatting_parser_arguments
 
 
@@ -11,8 +11,11 @@ def run(args):
 
     bibtex_citekey = _create_citekey_for_entry(entry)
 
-    string = to_string(
-        {bibtex_citekey: entry}, args.delimiter_type, tab_indent=args.tab_indent
+    string = dict_to_string(
+        {bibtex_citekey: entry},
+        args.delimiter_type,
+        tab_indent=args.tab_indent,
+        page_range_separator="--",
     )
 
     write(string)
diff --git a/src/betterbib/cli/_format.py b/src/betterbib/cli/_format.py
index 54937d6..c491617 100644
--- a/src/betterbib/cli/_format.py
+++ b/src/betterbib/cli/_format.py
@@ -1,5 +1,5 @@
 from ..adapt_doi_urls import adapt_doi_urls
-from ..tools import bibtex_parser, filter_fields, to_string, write
+from ..tools import bibtex_parser, dict_to_string, filter_fields, write
 from .helpers import add_file_parser_arguments, add_formatting_parser_arguments
 
 
@@ -18,10 +18,11 @@ def run(args):
 
         d = dict(tuples)
         d = adapt_doi_urls(d, args.doi_url_type)
-        string = to_string(
+        string = dict_to_string(
             d,
             args.delimiter_type,
             tab_indent=args.tab_indent,
+            page_range_separator="--",
             # FIXME: use public argument when it becomes possible
             preamble=data._preamble,
         )
diff --git a/src/betterbib/cli/_journal_abbrev.py b/src/betterbib/cli/_journal_abbrev.py
index a29d340..1e090b7 100644
--- a/src/betterbib/cli/_journal_abbrev.py
+++ b/src/betterbib/cli/_journal_abbrev.py
@@ -1,5 +1,5 @@
 from ..journal_abbrev import journal_abbrev
-from ..tools import bibtex_parser, to_string, write
+from ..tools import bibtex_parser, dict_to_string, write
 from .helpers import add_file_parser_arguments, add_formatting_parser_arguments
 
 
@@ -10,7 +10,9 @@ def run(args):
 
         d = journal_abbrev(d, args.long_journal_names, args.extra_abbrev_file)
 
-        string = to_string(d, "braces", tab_indent=False)
+        string = dict_to_string(
+            d, "braces", tab_indent=False, page_range_separator="--"
+        )
 
         write(string, infile if args.in_place else None)
 
diff --git a/src/betterbib/cli/_update.py b/src/betterbib/cli/_update.py
index 64b59ac..c213e02 100644
--- a/src/betterbib/cli/_update.py
+++ b/src/betterbib/cli/_update.py
@@ -1,7 +1,7 @@
 from ..adapt_doi_urls import adapt_doi_urls
 from ..journal_abbrev import journal_abbrev
 from ..sync import sync
-from ..tools import bibtex_parser, sanitize_title, to_string, write
+from ..tools import bibtex_parser, dict_to_string, sanitize_title, write
 from .helpers import add_file_parser_arguments, add_formatting_parser_arguments
 
 
@@ -27,11 +27,12 @@ def run(args):
         sanitize_title(d)
         d = journal_abbrev(d, args.long_journal_names, args.extra_abbrev_file)
 
-        string = to_string(
+        string = dict_to_string(
             d,
             args.delimiter_type,
             tab_indent=args.tab_indent,
             unicode=not args.latex_output,
+            page_range_separator="--",
         )
 
         write(string, infile if args.in_place else None)
diff --git a/src/betterbib/cli/helpers.py b/src/betterbib/cli/helpers.py
index 1d05ebd..995ce3d 100644
--- a/src/betterbib/cli/helpers.py
+++ b/src/betterbib/cli/helpers.py
@@ -64,3 +64,10 @@ def add_formatting_parser_arguments(parser):
             "short: https://doi.org/abcde)"
         ),
     )
+    formatting_group.add_argument(
+        "-p",
+        "--page-range-separator",
+        type=str,
+        default="--",
+        help="page range separator (default: --)",
+    )
diff --git a/src/betterbib/crossref.py b/src/betterbib/crossref.py
index 9363687..4d0d0c6 100644
--- a/src/betterbib/crossref.py
+++ b/src/betterbib/crossref.py
@@ -143,7 +143,6 @@ def find_unique(self, entry: pybtex.database.Entry) -> pybtex.database.Entry:
             "title",
             "journal",
             "doi",
-            "pages",
             "year",
             "volume",
             "number",
@@ -270,7 +269,6 @@ def _crossref_to_pybtex(self, data):
         pairs = {
             "doi": "DOI",
             "number": "issue",
-            "pages": "page",
             "source": "source",
             "url": "URL",
             "volume": "volume",
@@ -376,6 +374,12 @@ def _crossref_to_pybtex(self, data):
         except KeyError:
             pass
 
+        try:
+            # unify page ranges to xx-xx (instead of the double dash)
+            fields_dict["pages"] = data["page"]
+        except KeyError:
+            pass
+
         try:
             year = data["issued"]["date-parts"][0][0]
             if year is not None:
diff --git a/src/betterbib/dblp.py b/src/betterbib/dblp.py
index 58211e1..2827710 100644
--- a/src/betterbib/dblp.py
+++ b/src/betterbib/dblp.py
@@ -61,6 +61,12 @@ def _dblp_to_pybtex(data):
         except KeyError:
             pass
 
+    try:
+        # unify page ranges to xx-xx (instead of the double dash)
+        fields_dict["pages"] = data["pages"].replace("--", "-")
+    except KeyError:
+        pass
+
     try:
         fields_dict["source"] = data["source"]
     except KeyError:
diff --git a/src/betterbib/tools.py b/src/betterbib/tools.py
index b575433..4a1ac94 100644
--- a/src/betterbib/tools.py
+++ b/src/betterbib/tools.py
@@ -227,6 +227,7 @@ def pybtex_to_bibtex_string(
     entry: Entry,
     bibtex_key: str,
     delimiters: tuple[str, str] = ("{", "}"),
+    page_range_separator: str = "--",
     indent: str = " ",
     sort: bool = False,
     unicode: bool = True,
@@ -276,13 +277,18 @@ def pybtex_to_bibtex_string(
         # Always make keys lowercase
         key = key.lower()
 
+        if key == "pages":
+            # force double https://tex.stackexchange.com/a/58671/13262
+            value = value.replace("-", page_range_separator)
+
         if key == "month":
             month_string = translate_month(value)
             if month_string:
                 content.append(f"{key} = {month_string}")
-        else:
-            if value is not None:
-                content.append(f"{key} = {left}{value}{right}")
+            continue
+
+        if value is not None:
+            content.append(f"{key} = {left}{value}{right}")
 
     # Make sure that every line ends with a comma
     out += indent.join([line + ",\n" for line in content])
@@ -424,9 +430,10 @@ def heuristic_unique_result(results, d):
 
 # This used to be a write() function, but beware of exceptions! Files would get
 # unintentionally overridden, see <https://github.com/nschloe/betterbib/issues/184>
-def to_string(
+def dict_to_string(
     od: dict[str, Entry],
     delimiter_type: str,
+    page_range_separator: str,
     tab_indent: bool,
     preamble: list | None = None,
     unicode: bool = True,
@@ -462,6 +469,7 @@ def to_string(
                 d,
                 bib_id,
                 delimiters=delimiters,
+                page_range_separator=page_range_separator,
                 indent="\t" if tab_indent else " ",
                 unicode=unicode,
             )
diff --git a/tox.ini b/tox.ini
index 6374079..59d3515 100644
--- a/tox.ini
+++ b/tox.ini
@@ -5,6 +5,7 @@ isolated_build = True
 [testenv]
 deps =
     pytest
+    pytest-codeblocks >= 0.12.2
     pytest-cov
 commands =
-    pytest {posargs}
+    pytest {posargs} --codeblocks


Test patch:
diff --git a/tests/cli/test_doi_to_bibtex_cli.py b/tests/cli/test_doi_to_bibtex_cli.py
index faea01f..17b2490 100644
--- a/tests/cli/test_doi_to_bibtex_cli.py
+++ b/tests/cli/test_doi_to_bibtex_cli.py
@@ -11,7 +11,6 @@ def test_cli_doi_to_bibtex(capsys):
         "@article{stockman2011,\n"
         " author = {Stockman, J.A.},\n"
         " doi = {10.1016/s0084-3954(09)79550-8},\n"
-        " pages = {193-194},\n"
         " source = {Crossref},\n"
         " url = {http://dx.doi.org/10.1016/s0084-3954(09)79550-8},\n"
         " volume = {2011},\n"
@@ -19,6 +18,7 @@ def test_cli_doi_to_bibtex(capsys):
         " publisher = {Elsevier BV},\n"
         " title = {A New Equation to Estimate Glomerular Filtration Rate},\n"
         " issn = {0084-3954},\n"
+        " pages = {193--194},\n"
         " year = {2011},\n"
         " month = jan,\n"
         "}\n"
diff --git a/tests/cli/test_update_cli.py b/tests/cli/test_update_cli.py
index 8ba6046..a5d1f8c 100644
--- a/tests/cli/test_update_cli.py
+++ b/tests/cli/test_update_cli.py
@@ -25,13 +25,13 @@ def test_cli_update(capsys):
         " author = {Stockman, J.A.},\n"
         " title = {A New Equation to Estimate Glomerular Filtration Rate},\n"
         " doi = {10.1016/s0084-3954(09)79550-8},\n"
-        " pages = {193-194},\n"
         " source = {Crossref},\n"
         " url = {https://doi.org/10.1016/s0084-3954(09)79550-8},\n"
         " volume = {2011},\n"
         " journal = {Yearbook of Pediatrics},\n"
         " publisher = {Elsevier BV},\n"
         " issn = {0084-3954},\n"
+        " pages = {193--194},\n"
         " year = {2011},\n"
         " month = jan,\n"
         "}\n"
diff --git a/tests/test_crossref.py b/tests/test_crossref.py
index d8f5a2a..b2d0bed 100644
--- a/tests/test_crossref.py
+++ b/tests/test_crossref.py
@@ -28,8 +28,7 @@ def test_crossref_article0():
             "doi": "10.1137/110820713",
             "issn": "0895-4798, 1095-7162",
             "publisher": "Society for Industrial & Applied Mathematics (SIAM)",
-            "title": "A Framework for Deflated and Augmented "
-            + "Krylov Subspace Methods",
+            "title": "A Framework for Deflated and Augmented Krylov Subspace Methods",
             "url": "http://dx.doi.org/10.1137/110820713",
             "journal": "SIAM J. Matrix Anal. & Appl.",
             "number": "2",
@@ -182,7 +181,8 @@ def test_crossref_inbook0():
             "publisher": "John Wiley & Sons, Ltd",
             "chapter": "Differential and Difference Equations",
             "url": "http://dx.doi.org/10.1002/0470868279.ch1",
-            "booktitle": "Numerical Methods " + "for Ordinary Differential Equations",
+            "booktitle": "Numerical Methods for Ordinary Differential Equations",
+            "isbn": "9780470868270, 9780471967583",
             "month": 1,
             "source": "Crossref",
             "year": 2005,
@@ -200,9 +200,7 @@ def test_crossref_incollection0():
 
     test_entry = pybtex.database.Entry(
         "incollection",
-        fields={
-            "title": "Numerical continuation, " + "and computation of normal forms"
-        },
+        fields={"title": "Numerical continuation, and computation of normal forms"},
         persons=pybtex.database.OrderedCaseInsensitiveDict(
             {
                 "author": [
@@ -331,7 +329,7 @@ def test_crossref_proceedings0():
     reference = pybtex.database.Entry(
         "proceedings",
         fields={
-            "publisher": "Wydawnictwo Uniwersytetu Ekonomicznego " + "we Wrocławiu",
+            "publisher": "Wydawnictwo Uniwersytetu Ekonomicznego we Wrocławiu",
             "doi": "10.15611/amse.2014.17",
             "title": "International Scientific Conference",
             "url": "http://dx.doi.org/10.15611/amse.2014.17",
@@ -355,8 +353,7 @@ def test_doi_only():
             "doi": "10.1137/110820713",
             "issn": "0895-4798, 1095-7162",
             "publisher": "Society for Industrial & Applied Mathematics (SIAM)",
-            "title": "A Framework for Deflated and Augmented "
-            + "Krylov Subspace Methods",
+            "title": "A Framework for Deflated and Augmented Krylov Subspace Methods",
             "url": "http://dx.doi.org/10.1137/110820713",
             "journal": "SIAM Journal on Matrix Analysis and Applications",
             "number": "2",
@@ -369,9 +366,9 @@ def test_doi_only():
         persons=pybtex.database.OrderedCaseInsensitiveDict(
             {
                 "author": [
-                    pybtex.database.Person("Gaul, Andr\xe9"),
+                    pybtex.database.Person("Gaul, André"),
                     pybtex.database.Person("Gutknecht, Martin H."),
-                    pybtex.database.Person("Liesen, J\xf6rg"),
+                    pybtex.database.Person("Liesen, Jörg"),
                     pybtex.database.Person("Nabben, Reinhard"),
                 ]
             }