Repository: dag-hammarskjold-library/dlx

Issue:
Handle repeated subfields in CSV/TSV export
Currently, in cases of fields with repeated subfield codes, only one of the subfields makes it into the serialization. This is due to the tabular nature of the serialization.

- [ ] Concatenate the values of repeated subfields into one string joined by a separator i.e. ";" or "||"

Discussion:

Solution patch:
diff --git a/dlx/scripts/init_indexes.py b/dlx/scripts/init_indexes.py
index 7d993572..ad89d0e3 100644
--- a/dlx/scripts/init_indexes.py
+++ b/dlx/scripts/init_indexes.py
@@ -56,7 +56,7 @@ def run():
                 collation=Config.marc_index_default_collation
             )
         )
-        indexes.append(col.create_index('_record_type', collation=Config.marc_index_default_collation))
+        indexes.append(col.create_index('_record_type_collated', collation=Config.marc_index_default_collation))
 
         print(f'creating text indexes...')
         indexes.append(col.create_index('words', collation=Config.marc_index_default_collation))
diff --git a/dlx/util.py b/dlx/util.py
index 37de9789..4d2c016a 100644
--- a/dlx/util.py
+++ b/dlx/util.py
@@ -61,7 +61,12 @@ def __init__(self, list_of_lists=None, **kwargs):
     def set(self, rowx, field_name, value):
         self.index.setdefault(rowx, {})
         self.index[rowx].setdefault(field_name, {})
-        self.index[rowx][field_name] = value
+
+        if existing := self.index[rowx].get(field_name):
+            # if this is a repeated subfield, append the next value 
+            self.index[rowx][field_name] += f'||{value}'
+        else:
+            self.index[rowx][field_name] = value
 
         if field_name not in self.header:
             self.header.append(field_name)


Test patch:
diff --git a/tests/test_marcset.py b/tests/test_marcset.py
index 21bac695..d7ff4b84 100644
--- a/tests/test_marcset.py
+++ b/tests/test_marcset.py
@@ -168,7 +168,7 @@ def test_to_csv(db):
     bibset = BibSet.from_query({})
     bibset.records = list(bibset.records)
     #[bib.set('001', None, str(bib.id)) for bib in bibset.records]
-    assert bibset.to_csv(write_id=True) == '1.001,1.245$a,1.245$b,1.245$c,1.520$a,2.520$a,1.650$a,1.710$a\n1,This,is the,title,Description,Repeated subfield,Header,Another header\n2,Another,is the,title,,,Header,'
+    assert bibset.to_csv(write_id=True) == '1.001,1.245$a,1.245$b,1.245$c,1.520$a,2.520$a,1.650$a,1.710$a\n1,This,is the,title,Description,Another description||Repeated subfield,Header,Another header\n2,Another,is the,title,,,Header,'
 
     # comma and quote handling
     bibs = BibSet()