Repository: dag-hammarskjold-library/dlx

Issue:
Allow truncated words in whole record text search
https://github.com/dag-hammarskjold-library/dlx-rest/issues/578

Discussion:

Solution patch:
diff --git a/dlx/marc/__init__.py b/dlx/marc/__init__.py
index b9c4741..ca9ed36 100644
--- a/dlx/marc/__init__.py
+++ b/dlx/marc/__init__.py
@@ -40,9 +40,13 @@ def __init__(self, rtype, tag, code):
         super().__init__(f'{tag}, {code} is an authority-controlled field')
 
 class AuthInUse(Exception):
-     def __init__(self):
+    def __init__(self):
         super().__init__('Can\'t delete Auth record because it is in use by other records')
 
+class InvalidRecordString(Exception):
+    def __init__(self, string, message='Can\'t parse record from string:'):
+        super().__init__(f'{message} {string}')
+
 ### Decorators
 
 class Decorators():
@@ -1390,6 +1394,9 @@ def to_mrk(self, *tags, language=None, write_id=True):
 
         if write_id and record.id is not None:
             record.set('001', None, str(record.id))
+
+        if not record.get_value('000'):
+            record.set('000', 'None', '****')
         
         return '\n'.join([field.to_mrk(language=language) for field in record.get_fields()]) + '\n'
 
@@ -1507,12 +1514,19 @@ def from_mrc(cls, string):
     @classmethod
     def from_mrk(cls, string: str, auth_control=True, delete_subfield_zero=True):
         self = cls()
+        last_tag = 0
 
         for line in filter(None, string.split('\n')):
             match = re.match(r'=(\w{3})  (.*)', line)            
             tag, rest = match.group(1), match.group(2)
             if tag == 'LDR': tag = '000'
 
+            if int(tag) < int(last_tag):
+                # this can happen if a newline is missing between records
+                raise InvalidRecordString(string, 'Tag order does not look valid')
+            else:
+                last_tag = tag
+
             if tag[:2] == '00':
                 field = Controlfield(tag, rest)
                 
diff --git a/dlx/marc/query.py b/dlx/marc/query.py
index c542a53..6d1a069 100644
--- a/dlx/marc/query.py
+++ b/dlx/marc/query.py
@@ -707,11 +707,14 @@ def compile(self):
         quoted = re.findall(r'(".+?")', self.string)
         # capture words starting with hyphen (denotes "not" search)
         negated = [x[1] for x in re.findall(r'(^|\s)(\-\w+)', self.string)]
+        # capture words that contain stars
+        starred = [x for x in re.split('\s+', self.string) if '*' in x]
         copied_string = self.string
 
-        for _ in negated:
+        for _ in negated + starred:
             copied_string = copied_string.replace(_, '')
-
+        
+        #starred = [x.replace('*', '.*') for x in starred]
         exclude = ('the', 'of', 'to', 'at', 'and', 'in', 'on', 'by', 'at', 'it', 'its')
         words = Tokenizer.tokenize(copied_string)
         words = list(filter(lambda x: x not in exclude, words))
@@ -729,10 +732,12 @@ def compile(self):
         elif words:
             data.update({'words': {'$all': words}})
 
-        if len(quoted) > 1:
-            data['$and'] = [{'text': Regex(rf'\s{Tokenizer.scrub(x)}\s')} for x in quoted]
+        if len(quoted + starred) > 1:
+            data['$and'] = [{'text': Regex(rf'\s{Tokenizer.scrub(x)}\s')} for x in quoted] + [{'text': Regex(f' {x.replace("*", ".*")}')} for x in starred]
         elif len(quoted) == 1:
             data['text'] = Regex(rf'\s{Tokenizer.scrub(quoted[0])}\s')
+        elif len(starred) == 1:
+            data['text'] = Regex(f' {starred[0].replace("*", ".*")}')
 
         # use the text index for these cases
         text_searches = []
diff --git a/requirements.txt b/requirements.txt
index 7fe0960..aa1dbbf 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,8 +1,8 @@
-boto3==1.37.24
+boto3==1.37.34
 certifi==2025.1.31
 jsonschema==4.23.0
 mongomock==4.3.0
-moto==5.1.1
+moto==5.1.3
 nltk==3.9.1
 pymongo==4.10.1
 pytest==8.3.5


Test patch:
diff --git a/tests/test_marc.py b/tests/test_marc.py
index e67a4f4..97e9152 100644
--- a/tests/test_marc.py
+++ b/tests/test_marc.py
@@ -373,6 +373,12 @@ def test_querystring(db):
     query = Query.from_string('Another -header')
     assert query.compile() == {'$and': [{'words': {'$all': ['anoth']}}, {'words': {'$nin': ['header']}}]}
 
+    # starred words (truncation)
+    query = Query.from_string('anoth*')
+    assert query.compile() == {'text': Regex(' anoth.*')}
+    query = Query.from_string('anoth* h*der')
+    assert query.compile() == {'$and': [{'text': Regex(' anoth.*')}, {'text': Regex(' h.*der')}]}
+
     # tag no subfield
     query = Query.from_string('245:\'is the\'')
     results = list(BibSet.from_query(query.compile()))
@@ -778,8 +784,12 @@ def test_to_mrk(bibs):
     control = '\n'.join([line for line in control.split("\n") if line[:4] != '=001'])
     assert bib.to_mrk(write_id=False) == control
 
+    # add 000 if not exists
+    bib = Bib().set('245', 'a', 'title')
+    assert bib.to_mrk() == '=000  ****\n=245  \\\\$atitle\n'
+
 def test_from_mrk(db):
-    from dlx.marc import Bib
+    from dlx.marc import Bib, InvalidRecordString
     
     control = '=000  leader\n=001  1\n=008  controlfield\n=245  \\\\$aThis$bis the$ctitle\n=520  \\\\$aDescription\n=520  \\\\$aAnother description$aRepeated subfield\n=650  \\\\$aHeader$01\n=710  \\\\$aAnother header$02\n'
 
@@ -788,6 +798,10 @@ def test_from_mrk(db):
     assert bib.to_mrk() == control
     assert bib.commit(auth_check=True)
 
+    # invalid tag order
+    with pytest.raises(InvalidRecordString):
+        Bib.from_mrk('=001  99999\n=000  leader')
+
 def test_from_csv(db, bibs):
     from dlx.marc import Bib