Repository: internetarchive/openlibrary

Issue:
Remove unused deprecated functions from merge/merge.py
<!-- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] -->
[`openlibrary/catalog/merge/merge.py`](https://github.com/internetarchive/openlibrary/blob/master/openlibrary/catalog/merge/merge.py) has a few functions marked as `@deprecated`, and it looks as if they may be unused.

Perhaps they can be removed.

### Proposal & Constraints
<!-- What is the proposed solution / implementation? Is there a precedent of this approach succeeding elsewhere? -->
See if the `@deprecated` functions in `merge/merge.py` can be removed.

<!-- Which suggestions or requirements should be considered for how feature needs to appear or be implemented? -->

### Additional context
<!-- Add any other context or screenshots about the feature request here. -->

### Stakeholders
<!-- @ tag stakeholders of this bug -->
@cclauss
@hornc 




Discussion:

Resolving patch:
diff --git a/openlibrary/catalog/merge/merge.py b/openlibrary/catalog/merge/merge.py
deleted file mode 100644
--- a/openlibrary/catalog/merge/merge.py
+++ /dev/null
@@ -1,267 +0,0 @@
-from deprecated import deprecated
-import re
-
-from openlibrary.catalog.merge.names import match_name
-from openlibrary.catalog.merge.normalize import normalize
-
-
-re_amazon_title_paren = re.compile(r'^(.*) \([^)]+?\)$')
-re_and_of_space = re.compile(' and | of | ')
-
-isbn_match = 85
-
-
-def set_isbn_match(score):
-    isbn_match = score
-
-
-def build_titles(title):
-    normalized_title = normalize(title).lower()
-    titles = [title, normalized_title]
-    if title.find(' & ') != -1:
-        t = title.replace(" & ", " and ")
-        titles.append(t)
-        titles.append(normalize(t))
-    t2 = []
-    for t in titles:
-        if t.lower().startswith('the '):
-            t2.append(t[4:])
-        elif t.lower().startswith('a '):
-            t2.append(t[2:])
-    titles += t2
-
-    if re_amazon_title_paren.match(title):
-        t2 = []
-        for t in titles:
-            m = re_amazon_title_paren.match(t)
-            if m:
-                t2.append(m.group(1))
-                t2.append(normalize(m.group(1)))
-        titles += t2
-
-    return {
-        'full_title': title,
-        'normalized_title': normalized_title,
-        'titles': titles,
-        'short_title': normalized_title[:25],
-    }
-
-
-def within(a, b, distance):
-    return abs(a - b) <= distance
-
-
-def compare_date(e1, e2):
-    if 'publish_date' not in e1 or 'publish_date' not in e2:
-        return ('date', 'value missing', 0)
-    if e1['publish_date'] == e2['publish_date']:
-        return ('date', 'exact match', 200)
-    try:
-        e1_pub = int(e1['publish_date'])
-        e2_pub = int(e2['publish_date'])
-        if within(e1_pub, e2_pub, 1):
-            return ('date', 'within 1 year', 100)
-        elif within(e1_pub, e2_pub, 2):
-            return ('date', '+/-2 years', -25)
-        else:
-            return ('date', 'mismatch', -250)
-    except ValueError as TypeError:
-        return ('date', 'mismatch', -250)
-
-
-@deprecated('Use openlibrary.catalog.merge.merge_marc.compare_isbn10() instead.')
-def compare_isbn10(e1, e2):
-    if len(e1['isbn']) == 0 or len(e2['isbn']) == 0:
-        return ('ISBN', 'missing', 0)
-    for i in e1['isbn']:
-        for j in e2['isbn']:
-            if i == j:
-                return ('ISBN', 'match', isbn_match)
-
-    return ('ISBN', 'mismatch', -225)
-
-
-@deprecated('Use openlibrary.catalog.merge.merge_marc.level1_merge() instead.')
-def level1_merge(e1, e2):
-    score = []
-    if e1['short_title'] == e2['short_title']:
-        score.append(('short-title', 'match', 450))
-    else:
-        score.append(('short-title', 'mismatch', 0))
-
-    score.append(compare_date(e1, e2))
-    score.append(compare_isbn10(e1, e2))
-    return score
-
-
-@deprecated('Use openlibrary.catalog.merge.merge_marc.compare_authors() instead.')
-def compare_authors(amazon, marc):
-    if len(amazon['authors']) == 0 and 'authors' not in marc:
-        return ('main', 'no authors', 75)
-    if len(amazon['authors']) == 0:
-        return ('main', 'field missing from one record', -25)
-
-    for name in amazon['authors']:
-        if 'authors' in marc and match_name(name, marc['authors'][0]['name']):
-            return ('main', 'exact match', 125)
-        if 'by_statement' in marc and marc['by_statement'].find(name) != -1:
-            return ('main', 'exact match', 125)
-    if 'authors' not in marc:
-        return ('main', 'field missing from one record', -25)
-
-    max_score = 0
-    for a in amazon['authors']:
-        percent, ordered = keyword_match(a[0], marc['authors'][0]['name'])
-        if percent > 0.50:
-            score = percent * 80
-            if ordered:
-                score += 10
-            if score > max_score:
-                max_score = score
-    if max_score:
-        return ('main', 'keyword match', max_score)
-    else:
-        return ('main', 'mismatch', -200)
-
-
-def title_replace_amp(amazon):
-    return normalize(amazon['full-title'].replace(" & ", " and ")).lower()
-
-
-def substr_match(a, b):
-    return a.find(b) != -1 or b.find(a) != -1
-
-
-def keyword_match(in1, in2):
-    s1, s2 = (i.split() for i in (in1, in2))
-    s1_set = set(s1)
-    s2_set = set(s2)
-    match = s1_set & s2_set
-    if len(s1) == 0 and len(s2) == 0:
-        return 0, True
-    ordered = [x for x in s1 if x in match] == [x for x in s2 if x in match]
-    return float(len(match)) / max(len(s1), len(s2)), ordered
-
-
-def strip_and_compare(t1, t2):
-    t1 = re_and_of_space.sub('', t1).lower()
-    t2 = re_and_of_space.sub('', t2).lower()
-    return t1 == t2
-
-
-def compare_title(amazon, marc):
-    amazon_title = amazon['normalized_title'].lower()
-    marc_title = normalize(marc['full_title']).lower()
-    short = False
-    if len(amazon_title) < 9 or len(marc_title) < 9:
-        short = True
-
-    if not short:
-        for a in amazon['titles']:
-            for m in marc['titles']:
-                if a.lower() == m.lower():
-                    return ('full-title', 'exact match', 600)
-                if strip_and_compare(a, m):
-                    return ('full-title', 'exact match', 600)
-
-        for a in amazon['titles']:
-            for m in marc['titles']:
-                if substr_match(a.lower(), m.lower()):
-                    return ('full-title', 'contained within other title', 350)
-
-    max_score = 0
-    for a in amazon['titles']:
-        for m in marc['titles']:
-            percent, ordered = keyword_match(a, m)
-            score = percent * 450
-            if ordered:
-                score += 50
-            if score and score > max_score:
-                max_score = score
-    if max_score:
-        return ('full-title', 'keyword match', max_score)
-    elif short:
-        return ('full-title', 'shorter than 9 characters', 0)
-    else:
-        return ('full-title', 'mismatch', -600)
-
-
-def compare_number_of_pages(amazon, marc):
-    if 'number_of_pages' not in amazon or 'number_of_pages' not in marc:
-        return
-    amazon_pages = amazon['number_of_pages']
-    marc_pages = marc['number_of_pages']
-    if amazon_pages == marc_pages:
-        if amazon_pages > 10:
-            return ('pagination', 'match exactly and > 10', 100)
-        else:
-            return ('pagination', 'match exactly and < 10', 50)
-    elif within(amazon_pages, marc_pages, 10):
-        if amazon_pages > 10 and marc_pages > 10:
-            return ('pagination', 'match within 10 and both are > 10', 50)
-        else:
-            return ('pagination', 'match within 10 and either are < 10', 20)
-    else:
-        return ('pagination', 'non-match (by more than 10)', -225)
-
-
-@deprecated
-def short_part_publisher_match(p1, p2):
-    pub1 = p1.split()
-    pub2 = p2.split()
-    if len(pub1) == 1 or len(pub2) == 1:
-        return False
-    return all(substr_match(i, j) for i, j in zip(pub1, pub2))
-
-
-@deprecated('Use openlibrary.catalog.merge.merge_marc.compare_publisher() instead.')
-def compare_publisher(amazon, marc):
-    if 'publisher' in amazon and 'publishers' in marc:
-        amazon_pub = amazon['publisher']
-        norm_amazon = normalize(amazon_pub)
-        for marc_pub in marc['publishers']:
-            norm_marc = normalize(marc_pub)
-            if norm_amazon == norm_marc:
-                return ('publisher', 'match', 100)
-            elif substr_match(norm_amazon, norm_marc) or substr_match(
-                norm_amazon.replace(' ', ''), norm_marc.replace(' ', '')
-            ):
-                return ('publisher', 'occur within the other', 100)
-            elif short_part_publisher_match(norm_amazon, norm_marc):
-                return ('publisher', 'match', 100)
-        return ('publisher', 'mismatch', -25)
-
-    if 'publisher' not in amazon or 'publishers' not in marc:
-        return ('publisher', 'either missing', 0)
-
-
-@deprecated('Use openlibrary.catalog.merge.merge_marc.level2_merge() instead.')
-def level2_merge(amazon, marc):
-    score = []
-    score.append(compare_date(amazon, marc))
-    score.append(compare_isbn10(amazon, marc))
-    score.append(compare_title(amazon, marc))
-    if page_score := compare_number_of_pages(amazon, marc):
-        score.append(page_score)
-
-    score.append(compare_publisher(amazon, marc))
-    score.append(compare_authors(amazon, marc))
-    return score
-
-
-def full_title(edition):
-    title = edition['title']
-    if 'subtitle' in edition:
-        title += ' ' + edition['subtitle']
-    return title
-
-
-@deprecated('Use openlibrary.catalog.merge.merge_marc.attempt_merge() instead.')
-def attempt_merge(amazon, marc, threshold):
-    l1 = level1_merge(amazon, marc)
-    total = sum(i[2] for i in l1)
-    if total >= threshold:
-        return True
-    l2 = level2_merge(amazon, marc)
-    total = sum(i[2] for i in l2)
-    return total >= threshold


Test patch:
diff --git a/openlibrary/catalog/merge/tests/test_merge.py b/openlibrary/catalog/merge/tests/test_merge.py
deleted file mode 100644
--- a/openlibrary/catalog/merge/tests/test_merge.py
+++ /dev/null
@@ -1,333 +0,0 @@
-import pytest
-
-from openlibrary.catalog.merge.merge import (
-    attempt_merge,
-    full_title,
-    build_titles,
-    compare_title,
-)
-
-""" These tests seem to be duplicates of those in test_amazon.py,
-    which in turn are duplicates of methods that are actually used
-    by Open Library in openlibrary.catalog.merge.merge_marc
-    Investigate and clean up!
-    """
-
-
-def test_full_title():
-    assert full_title({'title': "Hamlet"}) == "Hamlet"
-    edition = {
-        'title': 'Flatland',
-        'subtitle': 'A Romance of Many Dimensions',
-    }
-    assert full_title(edition) == "Flatland A Romance of Many Dimensions"
-
-
-def test_merge_titles():
-    marc = {
-        'title_with_subtitles': (
-            'Spytime : the undoing of James Jesus Angleton : a novel'
-        ),
-        'title': 'Spytime',
-        'full_title': 'Spytime : the undoing of James Jesus Angleton : a novel',
-    }
-    amazon = {
-        'subtitle': 'The Undoing oF James Jesus Angleton',
-        'title': 'Spytime',
-    }
-
-    amazon = build_titles(full_title(amazon))
-    marc = build_titles(marc['title_with_subtitles'])
-    assert amazon['short_title'] == marc['short_title']
-    assert compare_title(amazon, marc) == (
-        'full-title',
-        'contained within other title',
-        350,
-    )
-
-
-def test_merge_titles2():
-    amazon = {'title': 'Sea Birds Britain Ireland'}
-    marc = {
-        'title_with_subtitles': 'seabirds of Britain and Ireland',
-        'title': 'seabirds of Britain and Ireland',
-        'full_title': 'The seabirds of Britain and Ireland',
-    }
-    amazon = build_titles(full_title(amazon))
-    marc = build_titles(marc['title_with_subtitles'])
-    assert compare_title(amazon, marc) == ('full-title', 'exact match', 600)
-
-
-@pytest.mark.skip(reason="Did not pass when rescuing tests. Check thresholds.")
-def test_merge():
-    amazon = {
-        'publisher': 'Collins',
-        'isbn_10': ['0002167360'],
-        'number_of_pages': 120,
-        'short_title': 'souvenirs',
-        'normalized_title': 'souvenirs',
-        'full_title': 'Souvenirs',
-        'titles': ['Souvenirs', 'souvenirs'],
-        'publish_date': '1975',
-        'authors': [('David Hamilton', 'Photographer')],
-    }
-    marc = {
-        'publisher': ['Collins'],
-        'isbn_10': ['0002167360'],
-        'short_title': 'souvenirs',
-        'normalized_title': 'souvenirs',
-        'full_title': 'Souvenirs',
-        'titles': ['Souvenirs', 'souvenirs'],
-        'publish_date': '1978',
-        'authors': [
-            {
-                'birth_date': '1933',
-                'db_name': 'Hamilton, David 1933-',
-                'entity_type': 'person',
-                'name': 'Hamilton, David',
-                'personal_name': 'Hamilton, David',
-            }
-        ],
-        'source_record_loc': 'marc_records_scriblio_net/part11.dat:155728070:617',
-        'number_of_pages': 120,
-    }
-
-    threshold = 735
-    assert attempt_merge(amazon, marc, threshold)
-
-
-@pytest.mark.skip(reason="Should be tested on openlibrary.catalog.merge.merge_marc.")
-def test_merge2():
-    amazon = {
-        'publisher': 'Collins',
-        'isbn_10': ['0002167530'],
-        'number_of_pages': 287,
-        'short_title': 'sea birds britain ireland',
-        'normalized_title': 'sea birds britain ireland',
-        'full_title': 'Sea Birds Britain Ireland',
-        'titles': ['Sea Birds Britain Ireland', 'sea birds britain ireland'],
-        'publish_date': '1975',
-        'authors': [('Stanley Cramp', 'Author')],
-    }
-    marc = {
-        'publisher': ['Collins'],
-        'isbn_10': ['0002167530'],
-        'short_title': 'seabirds of britain and i',
-        'normalized_title': 'seabirds of britain and ireland',
-        'full_title': 'seabirds of Britain and Ireland',
-        'titles': [
-            'seabirds of Britain and Ireland',
-            'seabirds of britain and ireland',
-        ],
-        'publish_date': '1974',
-        'authors': [
-            {
-                'db_name': 'Cramp, Stanley.',
-                'entity_type': 'person',
-                'name': 'Cramp, Stanley.',
-                'personal_name': 'Cramp, Stanley.',
-            }
-        ],
-        'source_record_loc': 'marc_records_scriblio_net/part08.dat:61449973:855',
-    }
-
-    threshold = 735
-    assert attempt_merge(amazon, marc, threshold)
-
-
-@pytest.mark.skip(reason="Did not pass when rescuing tests. Check thresholds.")
-def test_merge3():
-    amazon = {
-        'publisher': 'Intl Specialized Book Service Inc',
-        'isbn_10': ['0002169770'],
-        'number_of_pages': 207,
-        'short_title': 'women of the north',
-        'normalized_title': 'women of the north',
-        'full_title': 'Women of the North',
-        'titles': ['Women of the North', 'women of the north'],
-        'publish_date': '1985',
-        'authors': [('Jane Wordsworth', 'Author')],
-    }
-    marc = {
-        'publisher': ['Collins', 'Exclusive distributor ISBS'],
-        'isbn_10': ['0002169770'],
-        'short_title': 'women of the north',
-        'normalized_title': 'women of the north',
-        'full_title': 'Women of the North',
-        'titles': ['Women of the North', 'women of the north'],
-        'publish_date': '1981',
-        'number_of_pages': 207,
-        'authors': [
-            {
-                'db_name': 'Wordsworth, Jane.',
-                'entity_type': 'person',
-                'name': 'Wordsworth, Jane.',
-                'personal_name': 'Wordsworth, Jane.',
-            }
-        ],
-        'source_record_loc': 'marc_records_scriblio_net/part17.dat:110989084:798',
-    }
-
-    threshold = 735
-    assert attempt_merge(amazon, marc, threshold)
-
-
-@pytest.mark.skip(reason="Did not pass when rescuing tests. Check thresholds.")
-def test_merge4():
-    amazon = {
-        'publisher': 'HarperCollins Publishers Ltd',
-        'isbn_10': ['0002173433'],
-        'number_of_pages': 128,
-        'short_title': 'd day to victory',
-        'normalized_title': 'd day to victory',
-        'full_title': 'D-Day to Victory',
-        'titles': ['D-Day to Victory', 'd day to victory'],
-        'publish_date': '1984',
-        'authors': [('Wynfod Vaughan-Thomas', 'Editor, Introduction')],
-    }
-    marc = {
-        'publisher': ['Collins'],
-        'isbn_10': ['0002173433'],
-        'short_title': 'great front pages  d day ',
-        'normalized_title': 'great front pages  d day to victory 1944 1945',
-        'full_title': 'Great front pages : D-Day to victory 1944-1945',
-        'titles': [
-            'Great front pages : D-Day to victory 1944-1945',
-            'great front pages  dday to victory 1944 1945',
-        ],
-        'publish_date': '1984',
-        'number_of_pages': 128,
-        'by_statement': 'introduced by Wynford Vaughan-Thomas.',
-        'source_record_loc': 'marc_records_scriblio_net/part17.dat:102360356:983',
-    }
-
-    threshold = 735
-    assert attempt_merge(amazon, marc, threshold)
-
-
-@pytest.mark.skip(reason="Should be tested on openlibrary.catalog.merge.merge_marc.")
-def test_merge5():
-    amazon = {
-        'publisher': 'HarperCollins Publishers (Australia) Pty Ltd',
-        'isbn_10': ['0002174049'],
-        'number_of_pages': 120,
-        'short_title': 'netherlandish and german ',
-        'normalized_title': (
-            'netherlandish and german paintings national gallery schools of painting'
-        ),
-        'full_title': (
-            'Netherlandish and German Paintings (National Gallery Schools of Painting)'
-        ),
-        'titles': [
-            'Netherlandish and German Paintings (National Gallery Schools of Painting)',
-            'netherlandish and german paintings national gallery schools of painting',
-            'Netherlandish and German Paintings',
-            'netherlandish and german paintings',
-        ],
-        'publish_date': '1985',
-        'authors': [('Alistair Smith', 'Author')],
-    }
-    marc = {
-        'publisher': ['National Gallery in association with W. Collins'],
-        'isbn_10': ['0002174049'],
-        'short_title': 'early netherlandish and g',
-        'normalized_title': 'early netherlandish and german paintings',
-        'full_title': 'Early Netherlandish and German paintings',
-        'titles': [
-            'Early Netherlandish and German paintings',
-            'early netherlandish and german paintings',
-        ],
-        'publish_date': '1985',
-        'authors': [
-            {
-                'db_name': 'National Gallery (Great Britain)',
-                'name': 'National Gallery (Great Britain)',
-                'entity_type': 'org',
-            }
-        ],
-        'number_of_pages': 116,
-        'by_statement': 'Alistair Smith.',
-        'source_record_loc': 'marc_records_scriblio_net/part17.dat:170029527:1210',
-    }
-    threshold = 735
-    assert attempt_merge(amazon, marc, threshold)
-
-
-@pytest.mark.skip(reason="Did not pass when rescuing tests. Check thresholds.")
-def test_merge6():
-    amazon = {
-        'publisher': 'Fount',
-        'isbn_10': ['0002176157'],
-        'number_of_pages': 224,
-        'short_title': 'basil hume',
-        'normalized_title': 'basil hume',
-        'full_title': 'Basil Hume',
-        'titles': ['Basil Hume', 'basil hume'],
-        'publish_date': '1986',
-        'authors': [('Tony Castle', 'Editor')],
-    }
-    marc = {
-        'publisher': ['Collins'],
-        'isbn_10': ['0002176157'],
-        'short_title': 'basil hume  a portrait',
-        'normalized_title': 'basil hume  a portrait',
-        'full_title': 'Basil Hume : a portrait',
-        'titles': ['Basil Hume : a portrait', 'basil hume  a portrait'],
-        'number_of_pages': 158,
-        'publish_date': '1986',
-        'by_statement': 'edited by Tony Castle.',
-        'source_record_loc': 'marc_records_scriblio_net/part19.dat:39883132:951',
-    }
-    threshold = 735
-    assert attempt_merge(amazon, marc, threshold)
-
-
-@pytest.mark.skip(reason="Should be tested on openlibrary.catalog.merge.merge_marc.")
-def test_merge7():
-    amazon = {
-        'publisher': 'HarperCollins Publishers Ltd',
-        'isbn_10': ['0002176319'],
-        'number_of_pages': 256,
-        'short_title': 'pucklers progress',
-        'normalized_title': 'pucklers progress',
-        'full_title': "Puckler's Progress",
-        'titles': ["Puckler's Progress", 'pucklers progress'],
-        'publish_date': '1987',
-        'authors': [('Flora Brennan', 'Editor')],
-    }
-    marc = {
-        'publisher': ['Collins'],
-        'isbn_10': ['0002176319'],
-        'short_title': 'pucklers progress  the ad',
-        'normalized_title': (
-            'pucklers progress  the adventures of prince puckler muskau in england '
-            'wales and ireland as told in letters to his former wife 1826 9'
-        ),
-        'full_title': (
-            "Puckler's progress : the adventures of Prince Pu\u0308ckler-Muskau in "
-            "England, Wales, and Ireland as told in letters to his former wife, 1826-9"
-        ),
-        'titles': [
-            "Puckler's progress : the adventures of Prince Pu\u0308ckler-Muskau in "
-            "England, Wales, and Ireland as told in letters to his former wife, 1826-9",
-            'pucklers progress  the adventures of prince puckler muskau in england '
-            'wales and ireland as told in letters to his former wife 1826 9',
-        ],
-        'publish_date': '1987',
-        'authors': [
-            {
-                'name': 'Pu\u0308ckler-Muskau, Hermann Furst von',
-                'title': 'Furst von',
-                'death_date': '1871.',
-                'db_name': 'Pu\u0308ckler-Muskau, Hermann Furst von 1785-1871.',
-                'birth_date': '1785',
-                'personal_name': 'Pu\u0308ckler-Muskau, Hermann',
-                'entity_type': 'person',
-            }
-        ],
-        'number_of_pages': 254,
-        'by_statement': 'translated by Flora Brennan.',
-        'source_record_loc': 'marc_records_scriblio_net/part19.dat:148554594:1050',
-    }
-    threshold = 735
-    assert attempt_merge(amazon, marc, threshold)