Repository: dag-hammarskjold-library/dlx

Issue:
Add a "user" field to file data
Create a field in the file data to record a username of the file uploader

Discussion:

Solution patch:
diff --git a/dlx/config.py b/dlx/config.py
index cb37c561..ba81b44f 100644
--- a/dlx/config.py
+++ b/dlx/config.py
@@ -288,6 +288,9 @@ class Config():
     # the collation used in the MDB CE indexes on the bibs and auths collections
     marc_index_default_collation = {'locale': 'en', 'strength': 1, 'numericOrdering': True}
 
+    # control subthreading of database actions on Marc.commit() and Auth.merge() for debugging
+    threading = True
+
     # utility functions
     @staticmethod
     def is_authority_controlled(record_type, tag, code):
diff --git a/dlx/file/__init__.py b/dlx/file/__init__.py
index 7d404bd3..298b4e08 100644
--- a/dlx/file/__init__.py
+++ b/dlx/file/__init__.py
@@ -1,4 +1,5 @@
 import os, requests, hashlib
+from warnings import warn 
 from datetime import datetime, timezone
 from io import BytesIO
 from json import dumps
@@ -69,7 +70,7 @@ def to_str(self):
 
 class File(object):
     @classmethod    
-    def import_from_path(cls, path, *, identifiers, languages, mimetype, source, filename=None, overwrite=False):
+    def import_from_path(cls, path, *, identifiers, languages, mimetype, source, filename=None, overwrite=False, user=None):
         fh = open(path, 'rb')
         
         return cls.import_from_handle(
@@ -79,11 +80,12 @@ def import_from_path(cls, path, *, identifiers, languages, mimetype, source, fil
             languages=languages, 
             mimetype=mimetype, 
             source=source,
-            overwrite=overwrite
+            overwrite=overwrite,
+            user=None
         )
 
     @classmethod
-    def import_from_url(cls, url, *, identifiers, languages, mimetype, source, filename=None, overwrite=False):
+    def import_from_url(cls, url, *, identifiers, languages, mimetype, source, filename=None, overwrite=False, user=None):
         hasher = hashlib.md5()
         fh = TemporaryFile('wb+')
         response = requests.get(url, stream=True)
@@ -101,11 +103,12 @@ def import_from_url(cls, url, *, identifiers, languages, mimetype, source, filen
             languages=languages, 
             mimetype=mimetype, 
             source=source,
-            overwrite=overwrite
+            overwrite=overwrite,
+            user=None
         )
         
     @classmethod
-    def import_from_binary(cls, data, *, identifiers, languages, mimetype, source, filename=None, overwrite=False):
+    def import_from_binary(cls, data, *, identifiers, languages, mimetype, source, filename=None, overwrite=False, user=None):
         return cls.import_from_handle(
             BytesIO(data), 
             filename=filename, 
@@ -113,11 +116,12 @@ def import_from_binary(cls, data, *, identifiers, languages, mimetype, source, f
             languages=languages, 
             mimetype=mimetype, 
             source=source,
-            overwrite=overwrite
+            overwrite=overwrite,
+            user=user
     )
         
     @classmethod
-    def import_from_handle(cls, handle, *, identifiers, languages, mimetype, source, filename=None, overwrite=False):
+    def import_from_handle(cls, handle, *, identifiers, languages, mimetype, source, filename=None, overwrite=False, user=None):
         '''Import a file using a file-like object (handle). The file is uploaded to 
         the s3 bucket specified in `dlx.Config`. The metadata is stored in the
         database
@@ -159,7 +163,8 @@ def import_from_handle(cls, handle, *, identifiers, languages, mimetype, source,
             system but with different languages
         '''
         
-        ###
+        if user is None:
+            warn('Uploading a file without the `user` param is deprecated', DeprecationWarning  )
         
         if len(identifiers) == 0 or len(languages) == 0:
             raise ValueError('Params `identifiers` and `languages` cannot be an empty list')
@@ -210,6 +215,7 @@ def import_from_handle(cls, handle, *, identifiers, languages, mimetype, source,
                     'source': source,
                     'timestamp': datetime.now(timezone.utc),
                     'uri': '{}.s3.amazonaws.com/{}'.format(S3.bucket, checksum),
+                    'user': user
                 }
             )
             
@@ -341,6 +347,7 @@ def __init__(self, doc):
         self.timestamp = doc['timestamp']
         self.uri = doc['uri']
         self.updated = doc.get('updated')
+        self.user = doc.get('user')
         
     @property
     def checksum(self):
@@ -366,7 +373,8 @@ def to_bson(self):
                 ('size', self.size),
                 ('source', self.source),
                 ('timestamp', self.timestamp),
-                ('uri', self.uri)
+                ('uri', self.uri),
+                ('user', self.user)
             ]
         )
         
diff --git a/dlx/marc/__init__.py b/dlx/marc/__init__.py
index a119d434..20add2a5 100644
--- a/dlx/marc/__init__.py
+++ b/dlx/marc/__init__.py
@@ -142,9 +142,9 @@ def sort_table_header(cls, header: list) -> list:
         return sorted(
             header, 
             key=lambda x: ( 
-                (re.match('\d+\.(\w+)', x)).group(1), # sort by tag
-                int(re.match('(\d+)\.', x).group(1)), # sort by prefix group
-                (re.match('\d\.\d{3}\$?(\w)?', x)).group(1) # sort by subfield code
+                (re.match(r'\d+\.(\w+)', x)).group(1), # sort by tag
+                int(re.match(r'(\d+)\.', x).group(1)), # sort by prefix group
+                (re.match(r'\d\.\d{3}\$?(\w)?', x)).group(1) # sort by subfield code
             )
         )
 
@@ -445,7 +445,7 @@ def match_id(cls, idx):
         Deprecated
         """
 
-        warn('dlx.marc.Marc.match_id() is deprecated. Use dlx.marc.Marc.from_id() instead')
+        warn('dlx.marc.Marc.match_id() is deprecated. Use dlx.marc.Marc.from_id() instead', DeprecationWarning)
 
         return cls.find_one(filter={'_id' : idx})
 
@@ -459,7 +459,7 @@ def match_ids(cls, *ids, **kwargs):
         Deprecated
         """
 
-        warn('dlx.marc.Marc.match_ids() is deprecated. Use dlx.marc.MarcSet.from_ids() instead')
+        warn('dlx.marc.Marc.match_ids() is deprecated. Use dlx.marc.MarcSet.from_ids() instead', DeprecationWarning)
 
         return cls.find(filter={'_id' : {'$in' : [*ids]}})
 
@@ -469,7 +469,7 @@ def match(cls, *matchers, **kwargs):
         Deprecated
         """
 
-        warn('dlx.marc.Marc.match() is deprecated. Use dlx.marc.MarcSet.from_query() instead')
+        warn('dlx.marc.Marc.match() is deprecated. Use dlx.marc.MarcSet.from_query() instead', DeprecationWarning)
 
         pymongo_kwargs = {}
 
@@ -502,7 +502,7 @@ def find(cls, *args, **kwargs):
         Deprecated
         """
 
-        warn('dlx.marc.Marc.find() is deprecated. Use dlx.marc.MarcSet.from_query() instead')
+        warn('dlx.marc.Marc.find() is deprecated. Use dlx.marc.MarcSet.from_query() instead', DeprecationWarning)
 
         cursor = cls.handle().find(*args, **kwargs)
 
@@ -515,7 +515,7 @@ def find_one(cls, *args, **kwargs):
         Deprecated
         """
 
-        warn('dlx.marc.Marc.find() is deprecated. Use dlx.marc.Marc.from_query() instead')
+        warn('dlx.marc.Marc.find_one() is deprecated. Use dlx.marc.Marc.from_query() instead', DeprecationWarning)
 
         found = cls.handle().find_one(*args, **kwargs)
 
@@ -525,7 +525,7 @@ def find_one(cls, *args, **kwargs):
     @classmethod
     def from_query(cls, *args, **kwargs):
         if 'collation' not in kwargs and Config.marc_index_default_collation:
-            warn('Collation not set. Using default collation set in config')
+            #warn('Collation not set. Using default collation set in config')
             kwargs['collation'] = Config.marc_index_default_collation
 
         return next(cls.set_class.from_query(*args, **kwargs), None)
@@ -536,7 +536,7 @@ def count_documents(cls, *args, **kwargs):
         Deprecated
         """
 
-        warn('dlx.marc.Marc.count_documents() is deprecated. Use dlx.marc.MarcSet.count instead')
+        warn('dlx.marc.Marc.count_documents() is deprecated. Use dlx.marc.MarcSet.count instead', DeprecationWarning)
 
         return cls.handle().count_documents(*args, **kwargs)
 
@@ -563,7 +563,7 @@ def datafields(self):
         return list(filter(lambda x: x.tag[:2] != '00', sorted(self.fields, key=lambda x: x.tag)))
 
     def parse(self, doc, *, auth_control=False):
-        for tag in filter(lambda x: re.match('^(\d{3}|[A-Z]{3})', x), doc.keys()):
+        for tag in filter(lambda x: re.match(r'^(\d{3}|[A-Z]{3})', x), doc.keys()):
             if tag == '000':
                 self.leader = doc['000'][0]
 
@@ -869,11 +869,10 @@ def index_field_text(*, threaded=True):
         #count = Counter(data['words'])
         #self.word_count = [{'stem': k, 'count': v} for k, v in count.items()]
 
-        if DB.database_name == 'testing':
+        if DB.database_name == 'testing' or Config.threading == False:
             index_field_text()
         else:
             thread1 = threading.Thread(target=index_field_text, args=[])
-            thread1.setDaemon(False) # stop the thread after complete
             thread1.start()
         
         # add logical fields
@@ -893,14 +892,24 @@ def index_logical_fields():
                         updates = []
 
                         for value in old_values:
-                            if value in new_values: continue    
-
-                            found_bibs = list(DB.bibs.find({logical_field: value}, limit=2, collation=Config.marc_index_default_collation))
-                            bibcount = len(found_bibs)
-                            if bibcount > 1: continue
-                            found_auths = list(DB.auths.find({logical_field: value}, limit=2, collation=Config.marc_index_default_collation))
-                            authcount = len(found_auths)
-                            if authcount > 1: continue
+                            if value in new_values: continue
+
+                            bibcount, authcount = 0, 0 
+                            
+                            if logical_field in Config.bib_logical_fields:
+                                found_bibs = list(DB.bibs.find({logical_field: value}, limit=2, collation=Config.marc_index_default_collation))
+                                bibcount = len(found_bibs)
+                                
+                                if bibcount > 1: 
+                                    continue
+
+                            if logical_field in Config.bib_logical_fields:
+                            
+                                found_auths = list(DB.auths.find({logical_field: value}, limit=2, collation=Config.marc_index_default_collation))
+                                authcount = len(found_auths)
+                                
+                                if authcount > 1:
+                                    continue
 
                             if bibcount + authcount == 0:
                                 # no records exist with this value
@@ -955,11 +964,10 @@ def index_logical_fields():
             if field not in self.logical_fields():
                 self.data.pop(field, None)
 
-        if DB.database_name == 'testing':
+        if DB.database_name == 'testing' or Config.threading == False:
             index_logical_fields()
         else:
             thread2 = threading.Thread(target=index_logical_fields, args=[])
-            thread2.setDaemon(False) # stop the thread after complete
             thread2.start()
 
         # history
@@ -997,11 +1005,10 @@ def save_history():
                 LOGGER.exception(err)
                 raise err
 
-        if DB.database_name == 'testing':
+        if DB.database_name == 'testing' or Config.threading == False:
             save_history()
         else:
             thread3 = threading.Thread(target=save_history, args=[])
-            thread3.setDaemon(False) # stop the thread after complete
             thread3.start()
 
         # commit
@@ -1052,14 +1059,13 @@ def do_update():
                             raise err
 
                         # for debugging
-                        DB.handle['auth_linked_update_log'].insert_one({'record_type': record.record_type, 'record_id': record.id, 'action': 'updated', 'triggered_by': auth.id, 'time': datetime.now()})
+                        DB.handle['auth_linked_update_log'].insert_one({'record_type': record.record_type, 'record_id': record.id, 'action': 'updated', 'triggered_by': auth.id, 'time': datetime.now(timezone.utc)})
 
                     if 1: #DB.database_name == 'testing':
                         do_update()
                     else:
                         # subthreading here doesn't work ?
                         subthread = threading.Thread(target=do_update, args=[])
-                        subthread.setDaemon(False) # stop the thread after complete
                         subthread.start()
             except Exception as err:
                     LOGGER.exception(err)
@@ -1076,11 +1082,10 @@ def do_update():
 
                     if heading_serialized != prev_serialized or self.heading_field.tag != previous.heading_field.tag:
                         # the heading has changed
-                        if DB.database_name == 'testing': 
+                        if DB.database_name == 'testing' or Config.threading == False: 
                             update_attached_records(self)
                         else:
                             thread4 = threading.Thread(target=update_attached_records, args=[self])
-                            thread4.setDaemon(False) # stop the thread after complete
                             thread4.start()
         
         return self
@@ -1103,10 +1108,15 @@ def update_browse_collections():
                     updates = []
 
                     for val in values:
-                        bibcount = len(list(DB.bibs.find({field: val}, limit=2, collation=Config.marc_index_default_collation)))
-                        authcount = len(list(DB.auths.find({field: val}, limit=2, collation=Config.marc_index_default_collation)))
+                        count = 0
+
+                        if field in Config.bib_logical_fields:
+                            count += len(list(DB.bibs.find({field: val}, limit=2, collation=Config.marc_index_default_collation)))
+                        
+                        if field in Config.auth_logical_fields:
+                            count += len(list(DB.auths.find({field: val}, limit=2, collation=Config.marc_index_default_collation)))
 
-                        if bibcount + authcount in [0, 1]:
+                        if count in [0, 1]:
                             # this record is the only instance of the value
                             updates.append(DeleteOne({'_id': val}))
 
@@ -1116,11 +1126,10 @@ def update_browse_collections():
                 LOGGER.exception(err)
                 raise err
 
-        if DB.database_name == 'testing': 
+        if DB.database_name == 'testing' or Config.threading == False: 
             update_browse_collections()
         else:
             thread = threading.Thread(target=update_browse_collections, args=[])
-            thread.setDaemon(False) # stop the thread after complete
             thread.start()
         
         history_collection = DB.handle[self.record_type + '_history']
@@ -1467,7 +1476,7 @@ def from_mrk(cls, string: str, auth_control=True, delete_subfield_zero=True):
                 ambiguous = []
                 
                 # capture the xref from subfield $0, if exists
-                if match := re.search('\$0(\d+)', rest[2:]):
+                if match := re.search(r'\$0(\d+)', rest[2:]):
                     xref = int(match.group(1))
                 else:
                     xref = None
@@ -1501,7 +1510,7 @@ def from_mrk(cls, string: str, auth_control=True, delete_subfield_zero=True):
     
     @classmethod
     def from_xml_raw(cls, root: ElementTree.Element, *, auth_control=True, delete_subfield_zero=True):
-        if DB.database_name == 'testing':
+        if DB.database_name == 'testing' or Config.threading == False:
             Auth({'_id': 1}).set('150', 'a', 'Header').commit()
 
         assert isinstance(root, ElementTree.Element)
@@ -1889,8 +1898,8 @@ def merge(self, *, user, losing_record):
             raise Exception("Losing record must be of type Auth")
 
         # for debugging
-        DB.handle['merge_log'].insert_one({'record_type': 'auth', 'record_id': losing_record.id, 'action': 'losing', 'time': datetime.now()})
-        DB.handle['merge_log'].insert_one({'record_type': 'auth', 'record_id': self.id, 'action': 'gaining', 'time': datetime.now()})
+        DB.handle['merge_log'].insert_one({'record_type': 'auth', 'record_id': losing_record.id, 'action': 'losing', 'time': datetime.now(timezone.utc), 'user': user})
+        DB.handle['merge_log'].insert_one({'record_type': 'auth', 'record_id': self.id, 'action': 'gaining', 'time': datetime.now(timezone.utc), 'user': user})
 
         def update_records(record_type, gaining, losing):
             authmap = getattr(Config, f'{record_type}_authority_controlled')
@@ -1899,9 +1908,10 @@ def update_records(record_type, gaining, losing):
             conditions = []
                  
             for ref_tag, d in authmap.items():
-                for subfield_code, auth_tag in d.items():
-                    if auth_tag == losing.heading_field.tag:
-                        conditions.append(Raw({f'{ref_tag}.subfields.xref': losing.id}, record_type=record_type))
+                #for subfield_code, auth_tag in d.items():
+                #    if auth_tag == losing.heading_field.tag:
+                
+                conditions.append(Raw({f'{ref_tag}.subfields.xref': losing.id}, record_type=record_type))
 
             if len(conditions) == 0:
                 return 0
@@ -1938,11 +1948,15 @@ def do_commit():
                             raise err
 
                         # for debugging
-                        DB.handle['merge_log'].insert_one({'record_type': record_type, 'record_id': record.id, 'action': 'updated', 'time': datetime.now()})
+                        DB.handle['merge_log'].insert_one({'record_type': record_type, 'record_id': record.id, 'action': 'updated', 'time': datetime.now(timezone.utc), 'user': user})
 
-                    t = threading.Thread(target=do_commit, args=[])
-                    t.setDaemon(False) # stop the thread after complete
-                    t.start()
+                    Config.threading = True if not hasattr(Config, 'threading') else Config.threading
+                    
+                    if Config.threading == False: # or DB.database_name == 'testing':
+                        do_commit()
+                    else:
+                        t = threading.Thread(target=do_commit, args=[])
+                        t.start()
                 
                 changed += 1
 
@@ -1951,11 +1965,11 @@ def do_commit():
         changed = 0
         
         for record_type in ('bib', 'auth'):
-            changed += update_records(record_type, self, losing_record)    
+            changed += update_records(record_type, self, losing_record)
         
         i = 0
 
-        while losing_record.in_use(usage_type='bib') or losing_record.in_use(usage_type='auth'):
+        while changed > 0 and losing_record.in_use(usage_type='bib') or losing_record.in_use(usage_type='auth'):
             # wait for all the links to be updated, otherwise the delete fails
             i += 1
 
@@ -1967,11 +1981,11 @@ def do_commit():
         losing_record.delete(user)
 
         # add to history
-        DB.handle['auth_history'].update_one({'_id': losing_record.id}, {'$set': {'merged': {'into': self.id, 'time': datetime.now()}}})
+        DB.handle['auth_history'].update_one({'_id': losing_record.id}, {'$set': {'merged': {'into': self.id, 'time': datetime.now(timezone.utc), user: user}}})
 
         # for debugging
-        DB.handle['merge_log'].insert_one({'record_type': 'auth', 'record_id': losing_record.id, 'action': 'deleted', 'time': datetime.now()})
-        DB.handle['merge_log'].insert_one({'record_type': 'auth', 'record_id': self.id, 'action': 'merge complete', 'time': datetime.now()})
+        DB.handle['merge_log'].insert_one({'record_type': 'auth', 'record_id': losing_record.id, 'action': 'deleted', 'time': datetime.now(timezone.utc), 'user': user})
+        DB.handle['merge_log'].insert_one({'record_type': 'auth', 'record_id': self.id, 'action': 'merge complete', 'time': datetime.now(timezone.utc), 'user': user})
 
 class Diff():
     """Compare two Marc objects.
@@ -2390,7 +2404,7 @@ class Matcher(Condition):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
-        warn('dlx.marc.Matcher is deprecated. Use dlx.marc.Condition instead')
+        warn('dlx.marc.Matcher is deprecated. Use dlx.marc.Condition instead', DeprecationWarning)
 
 class OrMatch(Or):
     # for backwards compatibility
@@ -2399,6 +2413,6 @@ def __init__(self, *matchers):
         super().__init__(*matchers)
         self.matchers = matchers
 
-        warn('dlx.marc.OrMatch is deprecated. Use dlx.marc.query.Or instead')
+        warn('dlx.marc.OrMatch is deprecated. Use dlx.marc.query.Or instead', DeprecationWarning)
 
 # end
diff --git a/dlx/marc/query.py b/dlx/marc/query.py
index 8185af93..fc0ab175 100644
--- a/dlx/marc/query.py
+++ b/dlx/marc/query.py
@@ -194,11 +194,11 @@ def parse(token, modifier=None):
                 if sys.getsizeof(matched_subfield_values) > 1e6: # 1 MB
                     if isinstance(value, Regex):
                         # fall back to normal regex
-                        return Condition(tag, {code: value}, modifier=modifier, record_type=record_type)
+                        return Condition(tag, {code: value}, modifier=modifier)
                     
                     raise InvalidQueryString(f'Text search "{value}" has too many hits on field "{tag}". Try narrowing the search')
                 elif len(matched_subfield_values) == 0:
-                    return Raw({'_id': 0}, record_type=record_type) # query that matches no documents
+                    return Raw({'_id': 0}) # query that matches no documents
                 
                 if modifier == 'not':
                     q = {f'{tag}': {'$not': {'$elemMatch': {'subfields': {'$elemMatch': {'code': code, 'value': {'$in': matched_subfield_values}}}}}}}
@@ -244,7 +244,7 @@ def parse(token, modifier=None):
                     except ValueError:
                         raise InvalidQueryString(f'ID must be a number')
                 elif tag[:2] == '00':
-                    return Raw({tag: value}, record_type=record_type)
+                    return Raw({tag: value})
 
                 # exists
                 if value == '*':
@@ -253,7 +253,7 @@ def parse(token, modifier=None):
                 # exact match
                 if not isinstance(value, Regex):
                     if value[0] == '\'' and value[-1] == '\'':
-                        return TagOnly(tag, value[1:-1], modifier=modifier)
+                        return TagOnly(tag, value[1:-1], modifier=modifier, record_type=record_type)
                     elif value[0] == '\'' and value[-1] != '\'':
                         raise InvalidQueryString(f'Invalid exact match using single quote: "{token}"')
 
@@ -320,7 +320,7 @@ def parse(token, modifier=None):
 
                     raise InvalidQueryString(f'Text search "{value}" has too many hits on field "{tag}". Try narrowing the search')
                 elif len(matched_subfield_values) == 0:
-                    return Raw({'_id': 0}, record_type=record_type) # query that matches no documents
+                    return Raw({'_id': 0}) # query that matches no documents
                 
                 if modifier == 'not':
                     q = {f'{tag}': {'$not': {'$elemMatch': {'subfields': {'$elemMatch': {'value': {'$in': matched_subfield_values}}}}}}}
@@ -360,7 +360,7 @@ def parse(token, modifier=None):
                 value = match.group(1)
                 
                 try:
-                    return Raw({'_id': int(value)}, record_type=record_type)
+                    return Raw({'_id': int(value)})
                 except ValueError:
                     raise InvalidQueryString(f'ID must be a number')
 
@@ -428,7 +428,7 @@ def parse(token, modifier=None):
 
                     # exact match
                     if value[0] == '\'' and value[-1] == '\'':
-                        return Raw({field: value[1:-1] }, record_type=record_type)
+                        return Raw({field: value[1:-1] })
                     elif value[0] == '\'' and value[-1] != '\'':
                         raise InvalidQueryString(f'Invalid exact match using single quote: "{token}"')
 
@@ -465,16 +465,16 @@ def parse(token, modifier=None):
                     if sys.getsizeof(values) > 1e6: # 1 MB
                         if isinstance(value, Regex):
                             # fall back to normal regex
-                            return Raw({field: value}, modifier=modifier, record_type=record_type)
+                            return Raw({field: value}, modifier=modifier)
                 
                         raise InvalidQueryString(f'Text search "{value}" has too many hits on field "{field}". Try narrowing the search')
                     elif len(values) == 0:
-                        return Raw({'_id': 0}, record_type=record_type) # query that matches no documents
+                        return Raw({'_id': 0}) # query that matches no documents
 
                     if modifier == 'not':
-                        return Raw({field: {'$not': {'$in': values}}}, record_type=record_type)
+                        return Raw({field: {'$not': {'$in': values}}})
                     else:
-                        return Raw({field: {'$in': values}}, record_type=record_type)
+                        return Raw({field: {'$in': values}})
                 else:
                     raise InvalidQueryString(f'Unrecognized query field "{field}"')
 
@@ -573,7 +573,7 @@ def to_string(self):
 
 class QueryDocument(Query):
     def __init__(self, *args, **kwargs):
-        warn('dlx.marc.QueryDocument is deprecated. Use dlx.marc.Query instead')
+        warn('dlx.marc.QueryDocument is deprecated. Use dlx.marc.Query instead', DeprecationWarning)
         
         super().__init__(*args, **kwargs)
         
@@ -650,10 +650,16 @@ def __init__(self, tag=None, *subs, record_type=None, **kwargs):
                 raise Exception('Invalid modifier: "{}"'.format(mod))
 
     def compile(self):
+        # if self.tag in [x for x in Config.bib_authority_controlled.keys()] + [x for x in Config.auth_authority_controlled.keys()]:
+        
         if not self.record_type:
-            warn('Record type is not set for query condition. Defaulting to bib')
-            self.record_type = 'bib'
+            if self.tag in [x for x in Config.bib_authority_controlled.keys()] + [x for x in Config.auth_authority_controlled.keys()]:
+                warn('Record type is not set for query condition. Defaulting to bib')
+
+                print(self.tag)
             
+            self.record_type = 'bib'    
+        
         tag = self.tag
         subconditions = []
 
@@ -665,7 +671,8 @@ def compile(self):
                 subconditions.append(
                     SON({'$elemMatch': {'code': code, 'value': val}})
                 )
-            else:     
+            else:
+                  
                 if isinstance(val, int):
                     xrefs = [val]
                 else:
@@ -736,9 +743,9 @@ def compile(self):
             data.update({'words': {'$all': words}})
 
         if len(quoted) > 1:
-            data['$and'] = [{'text': Regex(f'\s{Tokenizer.scrub(x)}\s')} for x in quoted]
+            data['$and'] = [{'text': Regex(rf'\s{Tokenizer.scrub(x)}\s')} for x in quoted]
         elif len(quoted) == 1:
-            data['text'] = Regex(f'\s{Tokenizer.scrub(quoted[0])}\s')
+            data['text'] = Regex(rf'\s{Tokenizer.scrub(quoted[0])}\s')
 
         # use the text index for these cases
         text_searches = []
@@ -784,7 +791,9 @@ def __init__(self, tag, value, *, record_type=None, modifier=None):
         if record_type:
             self.record_type = record_type
         else:
-            warn('Record type is not set for query condition. Defaulting to bib')
+            if tag in list(Config.bib_authority_controlled.keys()) + list(Config.auth_authority_controlled.keys()):
+                warn('Record type is not set for query condition. Defaulting to bib')
+            
             self.record_type = 'bib'
             
         auth_ctrl = Config.bib_authority_controlled if self.record_type == 'bib' else Config.auth_authority_controlled
diff --git a/requirements.txt b/requirements.txt
index e16afc3d..a6d9c48a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,11 +1,11 @@
-boto3==1.35.69
+boto3==1.35.76
 certifi==2024.8.30
 jsonschema==4.23.0
 mongomock==4.3.0
-moto==5.0.21
+moto==5.0.22
 nltk==3.9.1
 pymongo==4.10.1
-pytest==8.3.3
+pytest==8.3.4
 requests>=2.32.2
 responses==0.25.3
 xlrd==2.0.1


Test patch:
diff --git a/tests/test_file.py b/tests/test_file.py
index 3c1e1021..5df1725a 100644
--- a/tests/test_file.py
+++ b/tests/test_file.py
@@ -36,7 +36,15 @@ def test_import_from_handle(db, s3, tempfile):
     handle = TemporaryFile()
     handle.write(b'some data')
     handle.seek(0)
-    File.import_from_handle(handle, identifiers=[Identifier('isbn', '1')], filename='fn.ext', languages=['EN'], mimetype='application/dlx', source='test')
+    File.import_from_handle(
+        handle, 
+        identifiers=[Identifier('isbn', '1')], 
+        filename='fn.ext', 
+        languages=['EN'], 
+        mimetype='application/dlx', 
+        source='test',
+        user='test'
+    )
     
     results = list(DB.files.find({'identifiers': {'type': 'isbn', 'value': '1'}}))
     assert(len(results)) == 1
@@ -45,6 +53,7 @@ def test_import_from_handle(db, s3, tempfile):
     assert results[0]['mimetype'] == 'application/dlx'
     assert results[0]['source'] == 'test'
     assert results[0]['uri'] == '{}.s3.amazonaws.com/{}'.format(S3.bucket, results[0]['_id'])
+    assert results[0]['user'] == 'test'
     
     with TemporaryFile() as fh:
         S3.client.download_fileobj(S3.bucket, results[0]['_id'], fh)
diff --git a/tests/test_marc.py b/tests/test_marc.py
index 630857a9..ef93d5bd 100644
--- a/tests/test_marc.py
+++ b/tests/test_marc.py
@@ -183,39 +183,39 @@ def test_delete(db):
 def test_find_one(db, bibs, auths):
     from dlx.marc import Bib, Auth
     
-    bib = Bib.find_one({'_id': 1})
+    bib = Bib.from_query({'_id': 1})
     assert bib.id == 1
     assert isinstance(bib, Bib)
     
-    auth = Auth.find_one({'_id': 1})
+    auth = Auth.from_query({'_id': 1})
     assert auth.id == 1
     assert isinstance(auth, Auth)
         
 @pytest.mark.skip(reason='deprecated')
 def test_find(db):
-    from dlx.marc import Bib, Auth
+    from dlx.marc import Bib, BibSet, Auth, AuthSet
         
-    bibs = Bib.find({})
+    bibs = BibSet.from_query({})
     assert inspect.isgenerator(bibs)
     assert len(list(bibs)) == 2
     
-    for bib in Bib.find({}):
+    for bib in BibSet.from_query({}):
         assert isinstance(bib, Bib)
     
-    auths = Auth.find({})
+    auths = AuthSet.from_query({})
     assert inspect.isgenerator(auths)
     assert len(list(auths)) == 2
     
-    for auth in Auth.find({}):
+    for auth in AuthSet.from_query({}):
         assert isinstance(auth, Auth)
         
-    auths = Auth.find({'_id': 1})
+    auths = AuthSet.from_query({'_id': 1})
     assert len(list(auths)) == 1
         
-    auths = Auth.find({}, limit=1)
+    auths = AuthSet.from_query({}, limit=1)
     assert len(list(auths)) == 1
     
-    auths = Auth.find({}, limit=0, skip=1, projection={})
+    auths = AuthSet.from_query({}, limit=0, skip=1, projection={})
     assert len(list(auths)) == 1
         
 def test_from_id(db, bibs, auths):
@@ -224,13 +224,18 @@ def test_from_id(db, bibs, auths):
     assert Bib.from_id(2).id == 2
     
 def test_querydocument(db):
-    from dlx.marc import Bib, Auth
+    from dlx.marc import Bib, BibSet, Auth, AuthSet
     from dlx.marc.query import Query, Condition, Or, TagOnly
     from bson import SON
     from json import loads
     import re
+
+    class BibCondition(Condition):
+        def __init__(self, *args, **kwargs):
+            super().__init__(*args, **kwargs)
+            self.record_type = 'bib'
     
-    query = Query(Condition(tag='245', subfields={'a': 'This'}))
+    query = Query(Condition(tag='245', subfields={'a': 'This'}, record_type='bib'))
     assert isinstance(query.compile(), SON)
     
     qjson = query.to_json()
@@ -240,30 +245,30 @@ def test_querydocument(db):
     
     query = Query(
         Condition(tag='245', subfields={'a': re.compile(r'(This|Another)'), 'b': 'is the', 'c': 'title'}),
-        Condition(tag='650', modifier='exists'),
+        BibCondition(tag='650', modifier='exists'),
         Or(
-            Condition(tag='710', modifier='exists'),
+            BibCondition(tag='710', modifier='exists'),
             Condition(tag='520', modifier='not_exists')
         )
     )
-    assert len(list(Bib.find(query.compile()))) == 2
+    assert len(list(BibSet.from_query(query.compile()))) == 2
     
     query = Query(
         Condition(tag='110', subfields={'a': 'Another header'}, record_type='auth'),
     )
 
-    assert len(list(Auth.find(query.compile()))) == 1
-    assert Auth.find_one(query.compile()).id == 2
+    assert len(list(AuthSet.from_query(query.compile()))) == 1
+    assert Auth.from_query(query.compile()).id == 2
 
     query = Query(
-        TagOnly('245', 'This')
+        TagOnly('245', 'This', record_type='bib')
     )
-    assert len(list(Bib.find(query.compile()))) == 1
+    assert len(list(BibSet.from_query(query.compile()))) == 1
 
     query = Query(
-        TagOnly('245', 'This', modifier='not')
+        TagOnly('245', 'This', modifier='not', record_type='bib')
     )
-    assert len(list(Bib.find(query.compile()))) == 1
+    assert len(list(BibSet.from_query(query.compile()))) == 1
 
 def test_from_query(db):
     from dlx.marc import Bib, Auth, Query, Condition
@@ -272,7 +277,7 @@ def test_from_query(db):
     assert bib.id == 2
 
 def test_querystring(db):
-    from datetime import datetime
+    from datetime import datetime, timezone
     from bson import Regex
     from dlx.marc import BibSet, Bib, Auth, Query
 
@@ -356,7 +361,7 @@ def test_querystring(db):
     query = Query.from_string('updated>1900-01-01')
     assert len(list(BibSet.from_query(query.compile()))) == 3
 
-    query = Query.from_string(f'updated:{(datetime.utcnow()).strftime("%Y-%m-%d")}')
+    query = Query.from_string(f'updated:{(datetime.now(timezone.utc)).strftime("%Y-%m-%d")}')
     assert len(list(BibSet.from_query(query.compile()))) == 3
     
     # xref
@@ -606,7 +611,7 @@ def test_set_008(bibs):
 def test_delete_field(bibs):
     from dlx.marc import Bib
     
-    bib = Bib.find_one({'_id': 1})
+    bib = Bib.from_query({'_id': 1})
     bib.delete_field('008')
     assert list(bib.get_fields('008')) == []
     bib.delete_field('500')
@@ -630,13 +635,13 @@ def test_auth_lookup(db):
     with pytest.raises(KeyError):
         Auth._cache[1]
     
-    bib = Bib.find_one({'_id': 1})
+    bib = Bib.from_query({'_id': 1})
     assert bib.get_xref('650', 'a') == 1
     assert bib.get_value('650', 'a') == 'Header'
     
     assert Auth._cache[1]['a'] == 'Header'
    
-    auth = Auth.find_one({'_id': 1})
+    auth = Auth.from_query({'_id': 1})
     auth.set('150', 'a', 'Changed').commit()
     assert bib.get_value('650', 'a') == 'Changed'
     
@@ -672,7 +677,7 @@ def test_to_xml(db):
     from xmldiff import main
     
     control = '<record><controlfield tag="000">leader</controlfield><controlfield tag="001">1</controlfield><controlfield tag="008">controlfield</controlfield><datafield ind1=" " ind2=" " tag="245"><subfield code="a">This</subfield><subfield code="b">is the</subfield><subfield code="c">title</subfield></datafield><datafield ind1=" " ind2=" " tag="520"><subfield code="a">Description</subfield></datafield><datafield ind1=" " ind2=" " tag="520"><subfield code="a">Another description</subfield><subfield code="a">Repeated subfield</subfield></datafield><datafield ind1=" " ind2=" " tag="650"><subfield code="a">Header</subfield><subfield code="0">1</subfield></datafield><datafield ind1=" " ind2=" " tag="710"><subfield code="a">Another header</subfield><subfield code="0">2</subfield></datafield></record>'
-    bib = Bib.find_one({'_id': 1})
+    bib = Bib.from_query({'_id': 1})
     assert main.diff_texts(bib.to_xml(), control) == []
     
 def test_xml_encoding():
@@ -688,12 +693,12 @@ def test_to_mrc(db):
     
     control = '00238r|||a2200109|||4500001000200000008001300002245002400015520001600039520004300055650001100098710001900109\x1e1\x1econtrolfield\x1e  \x1faThis\x1fbis the\x1fctitle\x1e  \x1faDescription\x1e  \x1faAnother description\x1faRepeated subfield\x1e  \x1faHeader\x1e  \x1faAnother header\x1e\x1d'
 
-    bib = Bib.find_one({'_id': 1})
+    bib = Bib.from_query({'_id': 1})
     assert bib.to_mrc() == control
     
     control = '00049||||a2200037|||4500150001100000\x1e  \x1faHeader\x1e\x1d'
    
-    auth = Auth.find_one({'_id': 1})
+    auth = Auth.from_query({'_id': 1})
     assert auth.to_mrc(write_id=False) == control
     
     auth.set('994', 'a', 'Titulo').commit()
@@ -704,7 +709,7 @@ def test_to_mrk(bibs):
     
     control = '=000  leader\n=001  1\n=008  controlfield\n=245  \\\\$aThis$bis the$ctitle\n=520  \\\\$aDescription\n=520  \\\\$aAnother description$aRepeated subfield\n=650  \\\\$aHeader$01\n=710  \\\\$aAnother header$02\n'
 
-    bib = Bib.find_one({'_id': 1})
+    bib = Bib.from_query({'_id': 1})
     assert bib.to_mrk() == control
 
 def test_from_mrk(db):
diff --git a/tests/test_marcset.py b/tests/test_marcset.py
index 12775a4d..1d8e4c82 100644
--- a/tests/test_marcset.py
+++ b/tests/test_marcset.py
@@ -40,7 +40,7 @@ def test_iterate(db):
         assert isinstance(auth, Auth)
 
 def test_from_query(db):
-    from dlx.marc import MarcSet, BibSet, AuthSet, QueryDocument, Condition
+    from dlx.marc import MarcSet, BibSet, AuthSet, Query, Condition
     
     bibset = BibSet.from_query({'_id': {'$in': [1, 2]}})
     assert isinstance(bibset, (MarcSet, BibSet))
@@ -67,7 +67,7 @@ def test_from_query(db):
     authset.cache()
     assert isinstance(authset.records, list)
     
-    query = QueryDocument(
+    query = Query(
         Condition('245', modifier='exists')
         
     )