Repository: dag-hammarskjold-library/dlx

Issue:
Error when parsing indicators in .from_table if field doesn't exist
If a record in the table does not contain a field, there is an error because the indicators are trying to be set to a null value

Discussion:

Solution patch:
diff --git a/dlx/marc/__init__.py b/dlx/marc/__init__.py
index daf400a..6eddfbd 100644
--- a/dlx/marc/__init__.py
+++ b/dlx/marc/__init__.py
@@ -196,7 +196,7 @@ def from_table(cls, table, auth_control=True, auth_flag=False, field_check=None,
                     exceptions.append(Exception(f'Column header {instance}.{tag}{code} is repeated'))
                     continue
 
-                if tag == '001':
+                if tag == '001' and value:
                     record.id = int(value)
 
                 # make a first pass at parsing the data field by field without doing auth control or other checks.
@@ -205,16 +205,22 @@ def from_table(cls, table, auth_control=True, auth_flag=False, field_check=None,
                 address = [instance] if field else ['+']
                 
                 if is_indicator_col:
-                    if len(value) != 2:
+                    if not value:
+                        pass
+                    elif len(value) == 2:
+                        record.set(tag, None, None, ind1=value[0], ind2=value[1])
+                    else:
                         # inds must be two chars
                         exceptions.append(Exception(f'Invalid indicators: {value}'))
-
-                    record.set(tag, None, None, ind1=value[0], ind2=value[1])
                 else:
                     record.set(tag, code, value or '__null__', address=address, auth_control=False) # set a placeholder value if there is no data in that cell in the table
 
             # go back through the record and validate auth controlled values and do checks
             for field in record.datafields:
+                if all([x.value == '__null__' for x in field.subfields]):
+                    record.delete_field(field)
+                    continue
+
                 for i, subfield in enumerate(field.subfields):
                     if subfield.value == '__null__':
                         subfield.value = ''
diff --git a/requirements.txt b/requirements.txt
index 62971e3..4fa6e7e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
-boto3==1.38.9
+boto3==1.38.14
 certifi==2025.4.26
 jsonschema==4.23.0
 mongomock==4.3.0


Test patch:
diff --git a/tests/test_marcset.py b/tests/test_marcset.py
index fdd9df5..5803826 100644
--- a/tests/test_marcset.py
+++ b/tests/test_marcset.py
@@ -95,19 +95,22 @@ def test_from_table(db):
     table = Table([
         ['1.001', '1.246$a',  '1.246$b',  '1.269$c', '2.269$c', '1.650$a', '1.650$0', ''],
         ['98', 'title', 'subtitle', '1999-12-31','repeated', 'x', 1, '', ''],
-        ['99', 'title2','subtitle2','2000-01-01','repeated', 'x', 1],
+        ['99', 'title2','subtitle2', '2000-01-01','repeated', 'x', 1],
+        ['', 'title2', 'subtitle2', '', '', 'x', 1]
     ])
     
     bibset = BibSet.from_table(table)
     assert bibset.records[0].id == 98
     assert bibset.records[1].id == 99
 
-    for bib in bibset.records:
+    for bib in bibset.records[:2]:
         assert bib.get_value('246','b')[:8] == 'subtitle'
         assert bib.get_values('269','c')[1] == 'repeated'
         assert bib.get_value('650', 'a') == 'Header'
         assert not bib.get_value('650', '0')
 
+    assert bibset.records[2].get_field('269') is None
+
     bibset = BibSet.from_table(table, delete_subfield_zero=False)
     for bib in bibset.records:
         assert bib.get_value('650', '0')
@@ -176,7 +179,8 @@ def test_from_table(db):
     # indicators
     table = Table([
         ['1.245$a', '1.245__', '1.269__', '1.269$a'],
-        ['title', '98', '34', 'date']
+        ['title', '98', '34', 'date'],
+        ['title', '98', '', '']
     ])
 
     bibs = BibSet.from_table(table)