Repository: pcubillos/bibmanager

Issue:
Bug: Regression from #90 with False duplicates detected for Incollection book entries 
Issue #89 resolved by PR #90 dealt with false duplicate detections, which was merged into version 1.3.4. I am running version 1.4.5 installed via conda-forge and I am running into the same issue.  

Here is a false double detection. The ISBNs are the same, but the DOIs are different.

What's expected here is that bibmanager does not complain during a merge with these two entries, or other similar entries.

Note: This is possibly a DOI parsing issue as these DOIs have some Latex escape sequences in them. 


```DATABASE:
@incollection{OConnor2017,
    author = "O’Connor, Evan",
    editor = "Alsabti, A. W. and Murdin, P",
    title = "{The Core-Collapse Supernova-Black Hole Connection}",
    year = "2016",
    booktitle = "Handbook of Supernovae",
    pages = "1--18",
    publisher = "Springer",
    url = "https://doi.org/10.1007/978-3-319-20794-0\_129-1 http://link.springer.com/10.1007/978-3-319-20794-0\_129-1",
    address = "Cham",
    isbn = "9783319218465",
    doi = "10.1007/978-3-319-20794-0{\\_}129-1"
}

NEW:
@incollection{Alsabti2016,
    title = {{Supernovae and Supernova Remnants: The Big Picture in Low Resolution}},
    year = {2017},
    booktitle = {Handbook of Supernovae},
    author = {Alsabti, Athem W. and Murdin, Paul},
    editor = {Alsabti, A.~W. and Murdin, P},
    pages = {3--28},
    publisher = {Springer, Cham},
    isbn = {9783319218465},
    doi = {10.1007/978-3-319-21846-5{\_}1},
    keywords = {Physics}
```

Discussion:

Patch:
diff --git a/CHANGELOG.txt b/CHANGELOG.txt
index f4cd8cb..e4a7740 100644
--- a/CHANGELOG.txt
+++ b/CHANGELOG.txt
@@ -2671,3 +2671,15 @@ Bumped bibmanager version to 1.4.5.
 
 Updated bibtex entry in docs and README.
 
+
+*****  Sun Oct 23 13:06:26 CEST 2022  *****
+
+Updated bm.filter_field() to detect when entries with same isbn
+have different dois (and allow that case).
+Updated tests with new edge cases.
+
+*****
+
+Bumped bibmanager to version 1.4.6.
+When merged, this resolves #108
+
diff --git a/bibmanager/bib_manager/bib_manager.py b/bibmanager/bib_manager/bib_manager.py
index 76d5035..86ed497 100644
--- a/bibmanager/bib_manager/bib_manager.py
+++ b/bibmanager/bib_manager/bib_manager.py
@@ -52,8 +52,10 @@ from ..__init__ import __version__
 # Some constant definitions:
 lexer = prompt_toolkit.lexers.PygmentsLexer(BibTeXLexer)
 
-months  = {"jan":1, "feb":2, "mar":3, "apr": 4, "may": 5, "jun":6,
-           "jul":7, "aug":8, "sep":9, "oct":10, "nov":11, "dec":12}
+months  = {
+    "jan":1, "feb":2, "mar":3, "apr": 4, "may": 5, "jun":6,
+    "jul":7, "aug":8, "sep":9, "oct":10, "nov":11, "dec":12,
+}
 
 
 class Bib(object):
@@ -184,7 +186,8 @@ class Bib(object):
           last, first, von, jr = None, None, None, None
 
       self.sort_author = u.Sort_author(
-          last, first, von, jr, self.year, self.month)
+          last, first, von, jr, self.year, self.month,
+      )
 
   def update_content(self, other):
       """Update the bibtex content of self with that of other."""
@@ -336,7 +339,8 @@ class Bib(object):
           s.von == o.von and
           s.jr == o.jr and
           s.year == o.year and
-          s.month == o.month)
+          s.month == o.month
+      )
 
   def __le__(self, other):
       return self.__lt__(other) or self.__eq__(other)
@@ -502,7 +506,6 @@ def display_list(bibs, verb=-1):
             output=create_output(sys.stdout))
 
 
-
 def remove_duplicates(bibs, field):
     """
     Look for duplicates (within a same list of entries) by field and
@@ -556,13 +559,15 @@ def remove_duplicates(bibs, field):
         if field == 'isbn':
             dois = [
                 bibs[idx].doi if bibs[idx].doi is not None else ""
-                for idx in indices]
-            u_doi, doi_inv, doi_counts = np.unique(
-                dois, return_inverse=True, return_counts=True)
+                for idx in indices
+            ]
+            u_doi, doi_counts = np.unique(dois, return_counts=True)
             single_dois = u_doi[doi_counts==1]
             indices = [
-                idx for idx,doi in zip(indices,dois)
-                if doi not in single_dois]
+                idx
+                for idx,doi in zip(indices,dois)
+                if doi not in single_dois
+            ]
             nbibs = len(indices)
             if nbibs <= 1:
                 continue
@@ -611,6 +616,20 @@ def filter_field(bibs, new, field, take):
         if getattr(bib,field) is None or getattr(bib,field) not in fields:
             continue
         idx = fields.index(getattr(bib,field))
+        # There could be entries with same ISBN but different DOI:
+        if field == 'isbn':
+            new_doi = '' if bib.doi is None else bib.doi
+            dois = [
+                '' if bib.doi is None else bib.doi
+                for bib in bibs
+            ]
+            really_isbn_duplicates = [
+                isbn == bib.isbn and doi == new_doi
+                for isbn,doi in zip(fields,dois)
+            ]
+            if not np.any(really_isbn_duplicates):
+                continue
+            idx = np.where(really_isbn_duplicates)[0][0]
         # Replace if duplicated and new has newer bibcode:
         if bib.published() > bibs[idx].published() or take == 'new':
             bibs[idx].update_content(bib)
diff --git a/bibmanager/version.py b/bibmanager/version.py
index 3bedc93..3837de7 100644
--- a/bibmanager/version.py
+++ b/bibmanager/version.py
@@ -2,4 +2,4 @@
 # bibmanager is open-source software under the MIT license (see LICENSE).
 
 # bibmanager Version:
-__version__ = '1.4.5'
+__version__ = '1.4.6'


Test patch:
diff --git a/tests/test_bib_manager.py b/tests/test_bib_manager.py
index b2aaf5c..1c86a71 100644
--- a/tests/test_bib_manager.py
+++ b/tests/test_bib_manager.py
@@ -320,8 +320,7 @@ def test_Bib_warning_year():
             "Bad year format value '200X' for entry 'JonesEtal2001scipy'"
 
 
-@pytest.mark.parametrize('month',
-    ['15', 'tuesday',])
+@pytest.mark.parametrize('month', ['15', 'tuesday',])
 def test_Bib_warning_month(month):
     e = '''@Misc{JonesEtal2001scipy,
        author = {Eric Jones},
@@ -949,6 +948,72 @@ def test_merge_duplicate_title_add(bibs, mock_init_sample, mock_input):
     assert bibs['slipher_guy'] in loaded_bibs
 
 
+# bm.filter_field() is actually getting the job done here
+@pytest.mark.parametrize('doi', ['', 'doi = {10.1007/978-3-319-21846-5}'])
+def test_merge_duplicate_isbn_same_doi(doi):
+    base_bibs = [bm.Bib(
+    """@incollection{OConnor2017,
+        title = "{The Core-Collapse Supernova-Black Hole Connection}",
+        author = "O'Connor, Evan",
+        year = "2016",
+        booktitle = "Handbook of Supernovae",
+        isbn = "9783319218465",
+        DOI
+    }""".replace('DOI', doi)
+    )]
+    new_bibs = [bm.Bib(
+    """@incollection{Alsabti2016,
+        title = {{Supernovae and Supernova Remnants: The Big Picture in Low Resolution}},
+        author = {Alsabti, Athem W. and Murdin, Paul},
+        year = {2017},
+        booktitle = {Handbook of Supernovae},
+        isbn = {9783319218465},
+        DOI
+    }""".replace('DOI', doi)
+    )]
+    merged = bm.merge(base=base_bibs, new=new_bibs)
+
+    assert len(merged) == 1
+    assert merged[0] == base_bibs[0]
+
+
+# bm.filter_field() is actually getting the job done here
+@pytest.mark.parametrize(
+    'dois',
+    [
+        ('doi = "10.1007/978-3-319-20794-0"', 'doi = {10.1007/978-3-319-21846-5}'),
+        ('doi = "10.1007/978-3-319-20794-0"', ''),
+        ('', 'doi = {10.1007/978-3-319-21846-5}')
+    ]
+)
+def test_merge_duplicate_isbn_different_doi(dois):
+    base_bibs = [bm.Bib(
+    """@incollection{OConnor2017,
+        title = "{The Core-Collapse Supernova-Black Hole Connection}",
+        author = "O'Connor, Evan",
+        year = "2016",
+        booktitle = "Handbook of Supernovae",
+        isbn = "9783319218465",
+        DOI
+    }""".replace('DOI', dois[0])
+    )]
+    new_bibs = [bm.Bib(
+    """@incollection{Alsabti2016,
+        title = {{Supernovae and Supernova Remnants: The Big Picture in Low Resolution}},
+        author = {Alsabti, Athem W. and Murdin, Paul},
+        year = {2017},
+        booktitle = {Handbook of Supernovae},
+        isbn = {9783319218465},
+        DOI
+    }""".replace('DOI', dois[1])
+    )]
+    merged = bm.merge(base=base_bibs, new=new_bibs)
+
+    assert len(merged) == 2
+    assert merged[0] == new_bibs[0]
+    assert merged[1] == base_bibs[0]
+
+
 def test_duplicate_isbn_different_doi(capfd, entries):
     text = entries['isbn_doi1'] + entries['isbn_doi2']
     bibs = bm.read_file(text=text)