Repository: dag-hammarskjold-library/dlx

Issue:
Search history using marc query documents
For finding deleted records and searching within revisions

Discussion:
@jbukhari: Required for https://github.com/dag-hammarskjold-library/dlx-rest/issues/1457

Solution patch:
diff --git a/dlx/marc/__init__.py b/dlx/marc/__init__.py
index 13505bbd..58440ab0 100644
--- a/dlx/marc/__init__.py
+++ b/dlx/marc/__init__.py
@@ -1,13 +1,13 @@
 """dlx.marc"""
 
-import time, re, json, threading, copy
+import time, re, json, threading, copy, typing
 from collections import Counter
-from datetime import datetime
+from datetime import datetime, timezone
 from warnings import warn
 from xml.etree import ElementTree
 import jsonschema
 from bson import SON, Regex
-from pymongo import ReturnDocument, UpdateOne, DeleteOne
+from pymongo import ReturnDocument, UpdateOne, DeleteOne, CursorType
 from pymongo.collation import Collation
 from dlx.config import Config
 from dlx.db import DB
@@ -273,9 +273,7 @@ def __init__(self):
 
     @property
     def count(self):
-        import types
-        
-        if isinstance(self.records, (map, types.GeneratorType)):
+        if isinstance(self.records, (map, typing.Generator)):
             args, kwargs = self.query_params
 
             if isinstance(args[0], list):
@@ -726,7 +724,7 @@ def set_008(self):
         date_tag, date_code = Config.date_field
         pub_date = self.get_value(date_tag, date_code)
         pub_year = pub_date[0:4].ljust(4, '|')
-        cat_date = datetime.utcnow().strftime('%y%m%d')
+        cat_date = datetime.now(timezone.utc).strftime('%y%m%d')
 
         self.set('008', None, cat_date + text[6] + pub_year + text[11:])
 
@@ -770,7 +768,7 @@ def commit(self, user='admin', auth_check=True, update_attached=True):
         self.id = type(self)._increment_ids() if new_record else self.id
         self.validate()
         data = self.to_bson()
-        self.updated = data['updated'] = datetime.utcnow()
+        self.updated = data['updated'] = datetime.now(timezone.utc)
         self.user = data['user'] = user
         previous_state = (DB.bibs if self.record_type == 'bib' else DB.auths).find_one({'_id': self.id})
         
@@ -977,7 +975,7 @@ def save_history():
                     record_history['_id'] = self.id
 
                     if new_record:
-                        record_history['created'] = SON({'user': user, 'time': datetime.utcnow()})
+                        record_history['created'] = SON({'user': user, 'time': datetime.now(timezone.utc)})
 
                     # capture previous state if record originated in another db
                     if previous_state:
@@ -1124,7 +1122,7 @@ def update_browse_collections():
             record_history = SON()
             record_history['history'] = [self.to_bson()]
 
-        record_history['deleted'] = SON({'user': user, 'time': datetime.utcnow()})
+        record_history['deleted'] = SON({'user': user, 'time': datetime.now(timezone.utc)})
         history_collection.replace_one({'_id': self.id}, record_history, upsert=True)    
 
         return type(self).handle().delete_one({'_id': self.id})
@@ -1445,7 +1443,7 @@ def from_mrc(cls, string):
             directory = directory[12:]
 
     @classmethod
-    def from_mrk(cls, string: str, auth_control=True):
+    def from_mrk(cls, string: str, auth_control=True, delete_subfield_zero=True):
         self = cls()
 
         for line in filter(None, string.split('\n')):
@@ -1458,7 +1456,6 @@ def from_mrk(cls, string: str, auth_control=True):
             else:
                 ind1, ind2 = [x.replace('\\', ' ') for x in rest[:2]]
                 field = Datafield(record_type=cls.record_type, tag=tag, ind1=ind1, ind2=ind2)
-                fallback = {}
                 ambiguous = []
                 
                 # capture the xref from subfield $0, if exists
@@ -1472,35 +1469,30 @@ def from_mrk(cls, string: str, auth_control=True):
                     code, value = chunk[0], chunk[1:]
 
                     if Config.is_authority_controlled(self.record_type, tag, code):
-                        value = xref
+                        value = xref if xref else value
 
                     try:
                         field.set(code, value, place='+', auth_control=auth_control)
                     except(AmbiguousAuthValue):
-                        fallback[code] = value
                         ambiguous.append(Literal(code, value))
-
-                # attempt to use multiple subfields to resolve ambiguity. may 
-                # be deprecated in the future
-                if fallback and len(fallback) > 1:
-                    xrefs = Auth.xlookup_multi(tag, ambiguous, record_type=cls.record_type)
-                    
-                    if xrefs:
-                        if len(xrefs) == 1:
-                            for code in fallback.keys():
-                                field.set(code, xrefs[0], place='+')
-                    elif len(xrefs) > 1:
-                        raise AmbiguousAuthValue('bib', field.tag, '*', str(fallback))       
+                
+                # attempt to use multiple subfields to resolve ambiguity
+                if ambiguous:
+                    if xref := Auth.resolve_ambiguous(tag, ambiguous, record_type=self.record_type):
+                        field.set(code, xref, place='+', auth_control=auth_control)
+                    else:
+                        raise AmbiguousAuthValue('bib', field.tag, '*', str([x.to_str() for x in ambiguous]))
             
                 # remove subfield $0
-                field.subfields = list(filter(lambda x: x.code != '0', field.subfields))
+                if delete_subfield_zero:
+                    field.subfields = list(filter(lambda x: x.code != '0', field.subfields))
 
             self.fields.append(field)
 
         return self
     
     @classmethod
-    def from_xml_raw(cls, root, *, auth_control=True):
+    def from_xml_raw(cls, root: ElementTree.Element, *, auth_control=True, delete_subfield_zero=True):
         if DB.database_name == 'testing':
             Auth({'_id': 1}).set('150', 'a', 'Header').commit()
 
@@ -1511,7 +1503,8 @@ def from_xml_raw(cls, root, *, auth_control=True):
             self.set(node.attrib['tag'], None, node.text)
 
         for field_node in filter(lambda x: re.search('datafield$', x.tag), root):
-            field = Datafield(record_type=cls.record_type, tag=field_node.attrib['tag'], ind1=field_node.attrib['ind1'], ind2=field_node.attrib['ind2'])
+            tag = field_node.attrib['tag']
+            field = Datafield(record_type=cls.record_type, tag=tag, ind1=field_node.attrib['ind1'], ind2=field_node.attrib['ind2'])
             tag_nodes = filter(lambda x: re.search('subfield$', x.tag), field_node)
 
             # capture the xref if any
@@ -1519,19 +1512,34 @@ def from_xml_raw(cls, root, *, auth_control=True):
 
             for subfield_node in copy.deepcopy(tag_nodes): # use a copy to prevent the iterating orginal
                 if subfield_node.attrib['code'] == '0':
-                    xref = int(subfield_node.text)
+                    xref = int(''.join([x for x in subfield_node.text if ord(x) >= 48 and ord(x) <= 57]))
 
             # iterate though nodes and set the marc values
+            ambiguous = []
+
             for subfield_node in tag_nodes:
-                if Config.is_authority_controlled(self.record_type, field.tag, subfield_node.attrib['code']):
+                code = subfield_node.attrib['code']
+
+                if auth_control and Config.is_authority_controlled(self.record_type, field.tag, subfield_node.attrib['code']):
                     value = xref if xref else subfield_node.text
                 else:
                     value = str(subfield_node.text)
 
-                # .set handles auth control
-                field.set(subfield_node.attrib['code'], value, auth_control=auth_control, place='+')
+                # .set handles auth control. 
+                try:
+                    field.set(code, value, auth_control=auth_control, place='+')
+                except(AmbiguousAuthValue):
+                    ambiguous.append(Literal(code, value))
+
+                # attempt to use multiple subfields to resolve ambiguity
+                if ambiguous:
+                    if xref := Auth.resolve_ambiguous(tag, ambiguous, record_type=self.record_type):
+                        field.set(code, xref, auth_control=auth_control, place='+')
+                    else:
+                        raise AmbiguousAuthValue('bib', tag, '*', str([x.to_str() for x in ambiguous]))
             
-            field.subfields = list(filter(lambda x: x.code != '0', field.subfields))
+            if delete_subfield_zero:
+                field.subfields = list(filter(lambda x: x.code != '0', field.subfields))
                 
             self.fields.append(field)
             
@@ -1690,7 +1698,7 @@ def xlookup_multi(cls, tag, subfields, *, record_type):
         return xrefs
 
     @classmethod
-    def resolve_ambiguous(cls, *, tag: str, subfields: list, record_type: str) -> int:
+    def resolve_ambiguous(cls, *, tag: str, subfields: list['Subfield'], record_type: str) -> int:
         '''Determines if there is an exact authority match for specific subfields'''
 
         subfields_str = str([(x.code, x.value) for x in subfields])
@@ -1710,7 +1718,7 @@ def resolve_ambiguous(cls, *, tag: str, subfields: list, record_type: str) -> in
                     auth_subfields = cls.from_id(xref).heading_field.subfields
                     auth_subfields = [(x.code, x.value) for x in auth_subfields]
 
-                    if [(x.code, x.value) for x in subfields] ==  auth_subfields:
+                    if [(x.code, x.value) for x in subfields] == auth_subfields:
                         exact_matches.append(xref)
 
                     if exact_matches:
@@ -2006,6 +2014,56 @@ def __init__(self, a: Marc, b: Marc):
         self.different = True if self.a or self.b or self.d or self.e else False
         self.same = not self.different
 
+class History():
+    def __init__(self):
+        pass
+
+    @classmethod
+    def from_query(cls, query: Query, **kwargs) -> typing.Generator[None, CursorType, Marc]:
+        '''Yields history reords that mtch the query as Marc objects'''
+
+        self = cls()
+        handle = DB.handle[self.record_type + '_history']
+
+
+        for doc in handle.find({'history': {'$elemMatch': query.compile()}}, **kwargs):
+            for version in doc['history']:
+                yield self.record_class(version)
+
+    @classmethod
+    def find_deleted(cls, query: Query, **kwargs) -> typing.Generator[None, CursorType, int]:
+        '''Yields the ids of deleted records matching the query'''
+
+        self = cls()
+        handle = DB.handle[self.record_type + '_history']
+
+        for doc in handle.find({'history': {'$elemMatch': query.compile()}}, **kwargs):
+            if doc.get('deleted'):
+                yield doc['_id']
+
+    @classmethod
+    def deleted_by_date(cls, date_from: datetime, date_to: datetime = datetime.now(timezone.utc)) -> typing.Generator[None, CursorType, Marc]:
+        '''Yields the ids of records delete between the given dates'''
+
+        self = cls()
+        handle = DB.handle[self.record_type + '_history']
+
+        for doc in handle.find({'deleted.time': {'$gte': date_from, '$lt': date_to}}):
+            if d:= doc.get('deleted'):
+                yield doc['_id']
+
+class BibHistory(History):
+    def __init__(self):
+        self.record_class = Bib
+        self.record_type = 'bib'
+        super().__init__()
+
+class AuthHistory(History):
+    def __init(self):
+        self.record_class = Auth
+        self.record_type = 'auth'
+        super().__init__()
+
 ### Field classes
 
 class Field():


Test patch:
diff --git a/tests/test_marc.py b/tests/test_marc.py
index d0914c07..630857a9 100644
--- a/tests/test_marc.py
+++ b/tests/test_marc.py
@@ -962,4 +962,22 @@ def test_resolve_ambiguous(db):
 
     assert xref == auth.id
 
+def test_history(db):
+    from datetime import datetime
+    from dlx.marc import Bib, Auth, BibHistory, AuthHistory, Query
+
+    bib = Bib().set('245', 'a', 'new record').commit()
+    query = Query.from_string("245__a:'new record'")
+    assert next(BibHistory.from_query(query), None).id == bib.id
 
+    bib.delete()
+    hist = next(BibHistory.find_deleted(query), None)
+    assert hist == bib.id
+
+    assert next(
+        BibHistory.deleted_by_date(
+            datetime.fromisoformat('1999-12-31'), 
+            datetime.fromisoformat('2999-12-31') # set in the future because the delete is too fast here
+        ), 
+        None
+    ) == bib.id
\ No newline at end of file