Repository: internetarchive/openlibrary

Issue:
Incorrectly placed books on some shelves in the new Explorer
I see some incorrectly placed books on some shelves in the new Explorer.

Please see this screenshot:
![image](https://user-images.githubusercontent.com/381556/102857583-a8d64d80-4463-11eb-8a65-e7e836aa6511.png)

e.g.
- 00X sub-shelf:
  - [823.914](https://openlibrary.org/works/OL82537W)
  - [621.3916](https://openlibrary.org/works/OL65487W)
- 01X sub-shelf:
  - [364](https://openlibrary.org/works/OL20262W)
  - [002](https://openlibrary.org/works/OL4119177W)
etc.


Discussion:
Actually, I think I just realized what happened. Some books have multiple codes for some reason, even though some of those codes don't seem to make sense:
![image](https://user-images.githubusercontent.com/381556/102858009-7d079780-4464-11eb-9a34-df9dce69d4f9.png)

Assuming the codes are indeed correct, I would suggest displaying the code that got the book on a particular shelf instead of the first code in the list, to reduce confusion.
Looks like this is related to #4314.
https://github.com/internetarchive/openlibrary/issues/4322#issuecomment-749380583

Thank you @ksze! Good insights, agreed
Also seems related: #4323

Resolving patch:
diff --git a/openlibrary/solr/update_work.py b/openlibrary/solr/update_work.py
--- a/openlibrary/solr/update_work.py
+++ b/openlibrary/solr/update_work.py
@@ -27,9 +27,9 @@
 from openlibrary.core import ia
 from openlibrary.plugins.upstream.utils import url_quote
 from openlibrary.solr.data_provider import get_data_provider, DataProvider
-from openlibrary.utils.ddc import normalize_ddc
+from openlibrary.utils.ddc import normalize_ddc, choose_sorting_ddc
 from openlibrary.utils.isbn import opposite_isbn
-from openlibrary.utils.lcc import short_lcc_to_sortable_lcc
+from openlibrary.utils.lcc import short_lcc_to_sortable_lcc, choose_sorting_lcc
 
 logger = logging.getLogger("openlibrary.solr")
 
@@ -566,7 +566,7 @@ def add_list(name, values):
         if lccs:
             add_list("lcc", lccs)
             # Choose the... idk, longest for sorting?
-            add("lcc_sort", sorted(lccs, key=len, reverse=True)[0])
+            add("lcc_sort", choose_sorting_lcc(lccs))
 
         def get_edition_ddcs(ed: dict):
             ddcs = ed.get('dewey_decimal_class', [])  # type: List[str]
@@ -579,10 +579,9 @@ def get_edition_ddcs(ed: dict):
                 #   https://openlibrary.org/books/OL3029363M.json
                 # * [ "813/.54", "B", "92" ]
                 #   https://openlibrary.org/books/OL2401343M.json
-                ddcs = [
-                    ddcs[0],
-                    *[ddc for ddc in ddcs[1:] if ddc not in ('92', '920')]
-                ]
+                # * [ "092", "823.914" ]
+                # https://openlibrary.org/books/OL24767417M
+                ddcs = [ddc for ddc in ddcs if ddc not in ('92', '920', '092')]
             return ddcs
 
         raw_ddcs = set(ddc
@@ -593,8 +592,7 @@ def get_edition_ddcs(ed: dict):
                    for ddc in normalize_ddc(raw_ddc))
         if ddcs:
             add_list("ddc", ddcs)
-            # Choose longest, since has most precision?
-            add("ddc_sort", sorted(ddcs, key=len, reverse=True)[0])
+            add("ddc_sort", choose_sorting_ddc(ddcs))
 
         add_list("isbn", self.get_isbns(editions))
         add("last_modified_i", self.get_last_modified(w, editions))
diff --git a/openlibrary/utils/ddc.py b/openlibrary/utils/ddc.py
--- a/openlibrary/utils/ddc.py
+++ b/openlibrary/utils/ddc.py
@@ -7,6 +7,8 @@
 https://www.oclc.org/bibformats/en/0xx/082.html
 """
 import re
+from string import printable
+from typing import Iterable, List
 
 MULTIPLE_SPACES_RE = re.compile(r'\s+')
 DDC_RE = re.compile(r'''
@@ -18,31 +20,33 @@
         C?  # Canadian CIP
 
         # The number
-        (?P<number>\d{1,3}(\.\d+)?)
+        (?P<number>\d{1,3}(\.+\s?\d+)?)
 
         # Suffix
         (?P<poststar>\*?)
         (?P
\s?s)?  # Series suffix
         (?P<B>\s?\[?B\]?)?  # Biographical
-        (?P<ninetwo>\s920?)?  # No clue; shouldn't be its own DDC though
+        (?P<ninetwo>\s(092|920|92))?  # No clue; shouldn't be its own DDC though
     )
     |
     (\[?(?P<fic>Fic|E)\.?\]?)
 ''', re.IGNORECASE | re.X)
 
 
-def collapse_multiple_space(s):
+def collapse_multiple_space(s: str) -> str:
     return MULTIPLE_SPACES_RE.sub(' ', s)
 
 
-def normalize_ddc(ddc):
-    """
-    :param str ddc:
-    :rtype: list of str
-    """
-    ddc = collapse_multiple_space(ddc.strip()).replace('/', '').replace("'", '')
+VALID_CHARS = set(printable) - set("/'′’,")
+
+
+def normalize_ddc(ddc: str) -> List[str]:
+    ddc = ''.join(
+        char
+        for char in collapse_multiple_space(ddc.strip())
+        if char in VALID_CHARS)
 
-    results = []
+    results: List[str] = []
     for match in DDC_RE.finditer(ddc):
         parts = match.groupdict()
         prefix = ''
@@ -54,7 +58,7 @@ def normalize_ddc(ddc):
             continue
         # And end at them
         end = match.end()
-        if end < (len(ddc) - 1) and re.search(r'\b', ddc[end]):
+        if end < len(ddc) and re.search(r'\b', ddc[end]):
             continue
 
         # Some old standard which isn't used anymore; might need to filter these
@@ -90,15 +94,15 @@ def normalize_ddc(ddc):
             integer = number_parts[0]
 
             # Copy decimal without losing precision
-            decimal = '.' + number_parts[1] if len(number_parts) > 1 else ''
+            decimal = '.' + number_parts[-1].strip() if len(number_parts) > 1 else ''
 
             number = '%03d%s' % (int(integer), decimal)
 
             # Discard catalog edition number
             # At least one classification number available
             if len(results) > 0:
-                # Check if number like '0*' without decimal component
-                if re.search(r'(^0\d*$)', parts['number']):
+                # Check if number is without decimal component
+                if re.search(r'(^0?\d{1,2}$)', parts['number']):
                     continue
 
         # Handle [Fic] or [E]
@@ -109,6 +113,10 @@ def normalize_ddc(ddc):
 
         results.append(prefix + number + suffix)
 
+        # Include the non-j-prefixed form as well for correct sorting
+        if prefix == 'j':
+            results.append(number + suffix)
+
     return results
 
 
@@ -136,11 +144,9 @@ def normalize_ddc_range(start, end):
     return ddc_range_norm
 
 
-def normalize_ddc_prefix(prefix):
+def normalize_ddc_prefix(prefix: str) -> str:
     """
     Normalizes a DDC prefix to be used in searching. Integer prefixes are not modified
-    :param str prefix:
-    :rtype: str
 
     >>> normalize_ddc_prefix('1')
     '1'
@@ -151,10 +157,16 @@ def normalize_ddc_prefix(prefix):
     # 23.45* should become 023.45*
     if '.' in prefix:
         normed = normalize_ddc(prefix)
-        if normed:
-            return normed[0]
+        return normed[0] if normed else prefix
     # 0* should stay as is
     # 23* should stay as is
     # j* should stay as is
     else:
         return prefix
+
+
+def choose_sorting_ddc(normalized_ddcs: Iterable[str]) -> str:
+    # Prefer unprefixed DDCs (so they sort correctly)
+    preferred_ddcs = [ddc for ddc in normalized_ddcs if ddc[0] in '0123456789']
+    # Choose longest; theoretically most precise?
+    return sorted(preferred_ddcs or normalized_ddcs, key=len, reverse=True)[0]
diff --git a/openlibrary/utils/lcc.py b/openlibrary/utils/lcc.py
--- a/openlibrary/utils/lcc.py
+++ b/openlibrary/utils/lcc.py
@@ -85,6 +85,7 @@ class number fixed-width. For example:
 [2]: https://ejournals.bc.edu/index.php/ital/article/download/11585/9839/
 """
 import re
+from typing import Iterable
 
 from openlibrary.utils.ddc import collapse_multiple_space
 
@@ -117,6 +118,13 @@ def short_lcc_to_sortable_lcc(lcc):
     parts['cutter1'] = '.' + parts['cutter1'].lstrip(' .') if parts['cutter1'] else ''
     parts['rest'] = ' ' + parts['rest'].strip() if parts['rest'] else ''
 
+    # There will often be a CPB Box No (whatever that is) in the LCC field;
+    # E.g. "CPB Box no. 1516 vol. 17"
+    # Although this might be useful to search by, it's not really an LCC,
+    # so considering it invalid here.
+    if parts['letters'] == 'CPB':
+        return None
+
     return '%(letters)s%(number)013.8f%(cutter1)s%(rest)s' % parts
 
 
@@ -183,3 +191,11 @@ def normalize_lcc_range(start, end):
         lcc if lcc == '*' else short_lcc_to_sortable_lcc(lcc)
         for lcc in (start, end)
     ]
+
+
+def choose_sorting_lcc(sortable_lccs: Iterable[str]) -> str:
+    # Choose longest; theoretically most precise?
+    # Note we go to short-form first, so eg 'A123' beats 'A'
+    def short_len(lcc: str) -> int:
+        return len(sortable_lcc_to_short_lcc(lcc))
+    return sorted(sortable_lccs, key=short_len, reverse=True)[0]


Test patch:
diff --git a/openlibrary/tests/solr/test_update_work.py b/openlibrary/tests/solr/test_update_work.py
--- a/openlibrary/tests/solr/test_update_work.py
+++ b/openlibrary/tests/solr/test_update_work.py
@@ -400,7 +400,7 @@ def test_lccs(self, doc_lccs, solr_lccs, sort_lcc_index):
         'Handles none': ([], None, None),
         'Handles empty string': ([''], None, None),
         'Stores multiple': (['05', '123.5'], ['005', '123.5'], 1),
-        'Handles full DDC': (['j132.452939 [B]'], ['j132.452939 B'], 0),
+        'Handles full DDC': (['j132.452939 [B]'], ['132.452939 B', 'j132.452939 B'], 0),
         'Handles alternate DDCs': (
             ['132.52 153.6'], ['132.52', '153.6'], 0),
         'Stores longest for sorting': (
@@ -412,8 +412,9 @@ def test_lccs(self, doc_lccs, solr_lccs, sort_lcc_index):
         'Ignores superfluous 920s': (['123.5', '920'], ['123.5'], 0),
         'Ignores superfluous 92s': (['123.5', '92'], ['123.5'], 0),
         'Ignores superfluous 92s (2)': (['123.5', 'B', '92'], ['123.5'], 0),
-        'Does not skip 920s': (['920', '123.5'], ['123.5', '920'], 0),
-        'Does not skip 92s': (['92', '123.5'], ['092', '123.5'], 1),
+        'Skips 920s': (['920', '123.5'], ['123.5'], 0),
+        'Skips 92s': (['92', '123.5'], ['123.5'], 0),
+        'Skips 092s': (['092', '123.5'], ['123.5'], 0),
     }
 
     @pytest.mark.parametrize("doc_ddcs,solr_ddcs,sort_ddc_index",
diff --git a/openlibrary/utils/tests/test_ddc.py b/openlibrary/utils/tests/test_ddc.py
--- a/openlibrary/utils/tests/test_ddc.py
+++ b/openlibrary/utils/tests/test_ddc.py
@@ -1,6 +1,7 @@
 import pytest
 
 from openlibrary.utils.ddc import (
+    choose_sorting_ddc,
     normalize_ddc,
     normalize_ddc_prefix,
     normalize_ddc_range,
@@ -10,7 +11,7 @@
 TESTS_FROM_OCLC = [
     ("370.19'342", ['370.19342'], "Segmentation (prime) marks"),
     ("370.19/342", ['370.19342'], "Segmentation (prime) marks"),
-    ("j574", ['j574'], "Juvenile works."),
+    ("j574", ['j574', '574'], "Juvenile works."),
     ("[E]", ['[E]'], "Juvenile works with [E]"),
     ("[Fic]", ['[Fic]'], "Juvenile works with [Fic]."),
     ("658.404 92", ['658.404 92'], "Dewey numbers followed by 92 or 920."),
@@ -43,13 +44,30 @@
     ('1', ['001'], 'whole number padding'),
     ('hello world!', [], 'junk'),
     ('978123412341', [], 'junk'),
-    ('338.9/009171/7 019', ['338.90091717'], 
+    ('338.9/009171/7 019', ['338.90091717'],
      'Dewey number with segmentation and edition number'),
-    ('332.6 021', ['332.6'], 'Dewey number and edition number'),
-    ('[E] 021', ['[E]'], 'Juvenile work and edition number'),
+    ('332.6 021', ['332.6'], 'Dewey number and DDC edition number'),
+    ('[E] 021', ['[E]'], 'Juvenile work and DDC edition number'),
     ('015', ['015'], 'Single Dewey number with edition number pattern'),
-    ('(015.73) 015.791 021', ['015.73 s', '015.791'], 
+    ('(015.73) 015.791 021', ['015.73 s', '015.791'],
      'Two Dewey numbers and one edition number'),
+    ('813. 54', ['813.54'], 'Space after decimal'),
+    ('813.′54', ['813.54'], 'Curly quote separator (real world)'),
+    ('813’.54', ['813.54'], 'Other kind of curly quote (real world)'),
+    ('813. 54 (ddc21)', ['813.54'], 'catalog number with ddc prefix (real world)'),
+    ('823/.92 22', ['823.92'], 'catalog number without leading 0 (real world)'),
+    ("813.' 54", ['813.54'], 'Space and quote separate (real world)'),
+    ("726. 6' 0945' 51 (ddc21)", ['726.6'], 'Random spaces (real world)'),
+    ('813./​54', ['813.54'], 'Random non-printable chars (real world)'),
+    ('868 G216m', ['868'], 'Cutter number (real world)'),
+    ('863.,64', ['863.64'], 'Random comma (real world)'),
+    ('616..8/3', ['616.83'], 'Double dot (real world)'),
+    ('813.54 P38 1995', ['813.54'], 'Cutter/year (real world)'),
+    ('21ddc', [], 'DDCs must end at word boundaries'),
+    ('123; 216;', ['123', '216'], 'DDCs ending at word boundaries are ok'),
+    ('[Fic] 2 21', ['[Fic]'], 'Ignores single digits after Fic'),
+    ('[Fic] 813', ['[Fic]', '813'], 'Does not ignore tridigits after Fic'),
+    ('813/.52/.52', ['813.52'], 'Too much decimal'),
 ]
 
 
@@ -71,7 +89,8 @@ def test_normalize_ddc_with_oclc_spec(raw_ddc, expected, name):
     ('12.3', '012.3', 'Decimal'),
     ('12.300', '012.300', 'Trailing decimal zeros'),
     ('100', '100', 'Trailing zeros'),
-    ('noise', 'noise', 'Noise')
+    ('noise', 'noise', 'Noise'),
+    ('j100', 'j100', 'Limited juvenile'),
 ]
 
 
@@ -92,3 +111,17 @@ def test_normalize_ddc_prefix(prefix, normed, name):
                          ids=[t[-1] for t in RANGE_TESTS])
 def test_normalize_ddc_range(raw, normed, name):
     assert normalize_ddc_range(*raw) == normed
+
+
+SORTING_DDC_TEST = [
+    (['123', '123.554'], '123.554', 'Chooses longest'),
+    (['j123', '123'], '123', 'Prefer without j'),
+    (['-222.14', '927.5'], '927.5', 'Prefer without -'),
+    (['-222.14'], '-222.14', 'Begrudgingly uses prefixed if only option')
+]
+
+
+@pytest.mark.parametrize("ddcs,outpt,name", SORTING_DDC_TEST,
+                         ids=[t[-1] for t in SORTING_DDC_TEST])
+def test_choose_sorting_ddc(ddcs, outpt, name):
+    assert choose_sorting_ddc(ddcs) == outpt
diff --git a/openlibrary/utils/tests/test_lcc.py b/openlibrary/utils/tests/test_lcc.py
--- a/openlibrary/utils/tests/test_lcc.py
+++ b/openlibrary/utils/tests/test_lcc.py
@@ -1,6 +1,7 @@
 import pytest
 
 from openlibrary.utils.lcc import (
+    choose_sorting_lcc,
     normalize_lcc_prefix,
     normalize_lcc_range,
     short_lcc_to_sortable_lcc,
@@ -45,6 +46,7 @@ def test_to_short_lcc(sortable_lcc, raw_lcc, short_lcc, name):
     ('Microfiche 92/80965 (G)', 'real ol data'),
     ('MLCSN+', 'real ol data'),
     ('UNCLASSIFIED 809 (S)', 'real ol data'),
+    ('CPB Box no. 1516 vol. 17', 'CPB box number'),
 ]
 
 
@@ -117,3 +119,15 @@ def test_normalize_lcc_prefix(prefix, normed, name):
                          ids=[t[-1] for t in RANGE_TESTS])
 def test_normalize_lcc_range(raw, normed, name):
     assert normalize_lcc_range(*raw) == normed
+
+
+SORTING_TESTS = [
+    (['A--0001.00000000', 'B--0001.13000000.C89'], 1, 'Chooses longest'),
+    (['A--0001.00000000', 'A--0001.13000000'], 1, 'Chooses most precise'),
+]
+
+
+@pytest.mark.parametrize("lccs,result,name", SORTING_TESTS,
+                         ids=[t[-1] for t in SORTING_TESTS])
+def test_choose_sorting_lcc(lccs, result, name):
+    assert choose_sorting_lcc(lccs) == lccs[result]