Repository: manubot/manubot

Issue:
shortDOI citation support
Supporting citations of shortDOIs could be a nice feature. I see the major benefits as saving characters for excessively long DOIs and enabling a way to directly cite DOIs with forbidden characters such as `10.1016/S0933-3657(96)00367-3`. At least one user has attempted this workaround in https://github.com/Benjamin-Lee/deep-rules/pull/117/commits/c76ee4d20fa3f45935194b43b495b8159cbd30b4 / https://github.com/Benjamin-Lee/deep-rules/pull/117#issuecomment-456227977.

It seems like we could support a few different syntaxes:

1. `@doi:10/b6vnmd`
2. `@doi:b6vnmd`
3. `@shortdoi:b6vnmd`
4. `@shortdoi:10/b6vnmd`

I didn't see much shortDOI documentation online, but there is some provided when viewing a [shortened DOI result](http://web.archive.org/web/20190123160439/http://shortdoi.org/10.1016/S0933-3657(96)00367-3)

```
Your request was processed. The previously-created shortcut for 10.1016/S0933-3657(96)00367-3 is the handle:
10/b6vnmd
The shortcut HTTP URI is:
http://doi.org/b6vnmd
This shortcut will return the same results as http://dx.doi.org/10.1016/S0933-3657(96)00367-3, and doi:10/b6vnmd can be used in place of doi:10.1016/S0933-3657(96)00367-3.
```

Given the documentation, it seems that option 1 is the most canonical method. However, method 3 & 4 could help avoid user confusion.

***

Note that DOI content negotiation for crossref DOIs appears to work:

```
curl --location --header "Accept: application/vnd.citationstyles.csl+json" https://doi.org/b6vnmd
```

Comments:

Patch:
diff --git a/manubot/cite/doi.py b/manubot/cite/doi.py
index 928b849..0a7d0f2 100644
--- a/manubot/cite/doi.py
+++ b/manubot/cite/doi.py
@@ -1,3 +1,4 @@
+import json
 import logging
 import urllib.request
 
@@ -7,6 +8,38 @@ from manubot.cite.pubmed import get_pubmed_ids_for_doi
 from manubot.util import get_manubot_user_agent
 
 
+def expand_short_doi(short_doi):
+    """
+    Convert a shortDOI to a regular DOI.
+    """
+    if not short_doi.startswith('10/'):
+        raise ValueError(f'shortDOIs start with `10/`, but expand_short_doi received: {short_doi}')
+    url = f'https://doi.org/api/handles/{short_doi.lower()}'
+    params = {
+        "type": "HS_ALIAS",
+    }
+    response = requests.get(url, params=params)
+    # response documentation at https://www.handle.net/proxy_servlet.html
+    results = response.json()
+    response_code = results.get('responseCode')  # Handle protocol response code
+    if response_code == 100:
+        raise ValueError(f'Handle not found. Double check short_doi: {short_doi}')
+    if response_code == 200:
+        raise ValueError(f'HS_ALIAS values not found. Double check short_doi: {short_doi}')
+    if response_code != 1:
+        raise ValueError(f'Error response code of {response_code} returned by {response.url}')
+    values = results.get('values', [])
+    for value in values:
+        if value.get('type') == 'HS_ALIAS':
+            doi = value['data']['value']
+            return doi.lower()
+    raise RuntimeError(
+        f'HS_ALIAS value not found by expand_short_doi("{short_doi}")\n'
+        f'The following JSON was retrieved from {response.url}:\n'
+        + json.dumps(results, indent=2)
+    )
+
+
 def get_short_doi_url(doi):
     """
     Get the shortDOI URL for a DOI.
@@ -19,8 +52,7 @@ def get_short_doi_url(doi):
     try:
         response = requests.get(url, headers=headers).json()
         short_doi = response['ShortDOI']
-        short_doi = short_doi[3:]  # Remove "10/" prefix
-        short_url = 'https://doi.org/' + short_doi
+        short_url = 'https://doi.org/' + short_doi[3:]  # Remove "10/" prefix
         return short_url
     except Exception:
         logging.warning(f'shortDOI lookup failed for {doi}', exc_info=True)
diff --git a/manubot/cite/util.py b/manubot/cite/util.py
index 755d5f0..8c2005a 100644
--- a/manubot/cite/util.py
+++ b/manubot/cite/util.py
@@ -1,3 +1,4 @@
+import functools
 import logging
 import re
 
@@ -34,16 +35,33 @@ citation_pattern = re.compile(
     r'(?<!\w)@[a-zA-Z0-9][\w:.#$%&\-+?<>~/]*[a-zA-Z0-9/]')
 
 
+@functools.lru_cache(maxsize=5_000)
 def standardize_citation(citation):
     """
-    Standardize citation idenfiers based on their source
+    Standardize citation identifiers based on their source
     """
     source, identifier = citation.split(':', 1)
+
     if source == 'doi':
+        if identifier.startswith('10/'):
+            from manubot.cite.doi import expand_short_doi
+            try:
+                identifier = expand_short_doi(identifier)
+            except Exception as error:
+                # If DOI shortening fails, return the unshortened DOI.
+                # DOI metadata lookup will eventually fail somewhere with
+                # appropriate error handling, as opposed to here.
+                logging.error(
+                    f'Error in expand_short_doi for {identifier} '
+                    f'due to a {error.__class__.__name__}:\n{error}'
+                )
+                logging.info(error, exc_info=True)
         identifier = identifier.lower()
+
     if source == 'isbn':
         from isbnlib import to_isbn13
         identifier = to_isbn13(identifier)
+
     return f'{source}:{identifier}'
 
 
@@ -51,6 +69,7 @@ regexes = {
     'pmid': re.compile(r'[1-9][0-9]{0,7}'),
     'pmcid': re.compile(r'PMC[0-9]+'),
     'doi': re.compile(r'10\.[0-9]{4,9}/\S+'),
+    'shortdoi': re.compile(r'10/[a-zA-Z0-9]+'),
     'wikidata': re.compile(r'Q[0-9]+'),
 }
 
@@ -83,15 +102,23 @@ def inspect_citation_identifier(citation):
             )
 
     if source == 'doi':
-        # https://www.crossref.org/blog/dois-and-matching-regular-expressions/
-        if not identifier.startswith('10.'):
-            return (
-                'DOIs must start with `10.`.'
-            )
-        elif not regexes['doi'].fullmatch(identifier):
+        if identifier.startswith('10.'):
+            # https://www.crossref.org/blog/dois-and-matching-regular-expressions/
+            if not regexes['doi'].fullmatch(identifier):
+                return (
+                    'Identifier does not conform to the DOI regex. '
+                    'Double check the DOI.'
+                )
+        elif identifier.startswith('10/'):
+            # shortDOI, see http://shortdoi.org
+            if not regexes['shortdoi'].fullmatch(identifier):
+                return (
+                    'Identifier does not conform to the shortDOI regex. '
+                    'Double check the shortDOI.'
+                )
+        else:
             return (
-                'Identifier does not conform to the DOI regex. '
-                'Double check the DOI.'
+                'DOIs must start with `10.` (or `10/` for shortDOIs).'
             )
 
     if source == 'isbn':


Test patch:
diff --git a/manubot/cite/tests/test_doi.py b/manubot/cite/tests/test_doi.py
new file mode 100644
index 0000000..de7781c
--- /dev/null
+++ b/manubot/cite/tests/test_doi.py
@@ -0,0 +1,20 @@
+import pytest
+
+from manubot.cite.doi import (
+    expand_short_doi,
+)
+
+
+def test_expand_short_doi():
+    doi = expand_short_doi('10/b6vnmd')
+    assert doi == "10.1016/s0933-3657(96)00367-3"
+
+
+def test_expand_short_doi_invalid():
+    with pytest.raises(ValueError, match='Handle not found. Double check short_doi'):
+        expand_short_doi('10/b6vnmdxxxxxx')
+
+
+def test_expand_short_doi_not_short():
+    with pytest.raises(ValueError, match='shortDOIs start with `10/`'):
+        expand_short_doi('10.1016/S0933-3657(96)00367-3')
diff --git a/manubot/cite/tests/test_util.py b/manubot/cite/tests/test_util.py
index 700fb44..ab1da27 100644
--- a/manubot/cite/tests/test_util.py
+++ b/manubot/cite/tests/test_util.py
@@ -50,6 +50,9 @@ def test_get_citation_id(standard_citation, expected):
 @pytest.mark.parametrize("citation,expected", [
     ('doi:10.5061/DRYAD.q447c/1', 'doi:10.5061/dryad.q447c/1'),
     ('doi:10.5061/dryad.q447c/1', 'doi:10.5061/dryad.q447c/1'),
+    ('doi:10/b6vnmd', 'doi:10.1016/s0933-3657(96)00367-3'),
+    ('doi:10/B6VNMD', 'doi:10.1016/s0933-3657(96)00367-3'),
+    ('doi:10/xxxxxxxxxxxxxYY', 'doi:10/xxxxxxxxxxxxxyy'),  # passthrough non-existent shortDOI
     ('pmid:24159271', 'pmid:24159271'),
     ('isbn:1339919885', 'isbn:9781339919881'),
     ('isbn:1-339-91988-5', 'isbn:9781339919881'),
@@ -60,7 +63,7 @@ def test_get_citation_id(standard_citation, expected):
 ])
 def test_standardize_citation(citation, expected):
     """
-    Standardize idenfiers based on their source
+    Standardize identifiers based on their source
     """
     output = standardize_citation(citation)
     assert output == expected
@@ -68,6 +71,7 @@ def test_standardize_citation(citation, expected):
 
 @pytest.mark.parametrize('citation', [
     'doi:10.7717/peerj.705',
+    'doi:10/b6vnmd',
     'pmcid:PMC4304851',
     'pmid:25648772',
     'arxiv:1407.3561',
@@ -86,7 +90,9 @@ def test_inspect_citation_identifier_passes(citation):
 
 @pytest.mark.parametrize(['citation', 'contains'], [
     ('doi:10.771/peerj.705', 'Double check the DOI'),
+    ('doi:10/b6v_nmd', 'Double check the shortDOI'),
     ('doi:7717/peerj.705', 'must start with `10.`'),
+    ('doi:b6vnmd', 'must start with `10.`'),
     ('pmcid:25648772', 'must start with `PMC`'),
     ('pmid:PMC4304851', 'Should pmid:PMC4304851 switch the citation source to `pmcid`?'),
     ('isbn:1-339-91988-X', 'identifier violates the ISBN syntax'),
diff --git a/manubot/process/tests/manuscripts/example/content/02.body.md b/manubot/process/tests/manuscripts/example/content/02.body.md
index e179734..7372fc4 100644
--- a/manubot/process/tests/manuscripts/example/content/02.body.md
+++ b/manubot/process/tests/manuscripts/example/content/02.body.md
@@ -10,6 +10,7 @@ The website, started in 2011, is run by Alexandra Elbakyan, a graduate student a
 Elbakyan describes herself as motivated to provide universal access to knowledge [@url:https://engineuring.wordpress.com/2016/03/11/sci-hub-is-a-goal-changing-the-system-is-a-method/; @url:https://www.courtlistener.com/docket/4355308/50/elsevier-inc-v-sci-hub/; @url:http://www.leafscience.org/alexandra-elbakyan/].
 
 Cite the same paper in three ways [@pmid:25648772; @pmcid:PMC4304851; @doi:10.7717/peerj.705].
+Here's a citation of the same paper using its shortDOI [@doi:10/98p], which should be grouped with the other DOI reference.
 
 Cite using a `raw` citation @raw:model-free-data.
 Let's cite the same citation but using a `tag`, i.e. @tag:model-free-data.