Repository: monarch-initiative/koza

Issue:
Add integration tests
To start, run a few variations of the String load and confirm output, maybe including kgx validation?

Discussion:

Patch:
diff --git a/examples/string/protein-links-detailed.py b/examples/string/protein-links-detailed.py
index 2a28f87..e30dbc9 100644
--- a/examples/string/protein-links-detailed.py
+++ b/examples/string/protein-links-detailed.py
@@ -2,6 +2,8 @@ import re
 import uuid
 
 from biolink_model_pydantic.model import Protein, PairwiseGeneToGeneInteraction, Predicate
+from koza.manager.data_collector import collect
+
 from koza.koza_runner import get_koza_app
 
 koza = get_koza_app()
@@ -22,4 +24,4 @@ for row in koza.file_registry[source_name]:
         relation='IAO:0002436'
     )
 
-    koza.write(source_name, [protein_a, protein_b, pairwise_gene_to_gene_interaction])
+    collect(source_name, protein_a, protein_b, pairwise_gene_to_gene_interaction)
diff --git a/examples/string/protein-links-detailed.yaml b/examples/string/protein-links-detailed.yaml
index a6103b2..f1dc080 100644
--- a/examples/string/protein-links-detailed.yaml
+++ b/examples/string/protein-links-detailed.yaml
@@ -4,7 +4,6 @@ delimiter: ' '
 
 files:
   - './examples/data/string.tsv'
-  - './examples/data/string.tsv'
 
 columns:
   - 'protein1'
diff --git a/koza/app.py b/koza/app.py
index f4d7370..90eaa6f 100644
--- a/koza/app.py
+++ b/koza/app.py
@@ -137,13 +137,19 @@ class KozaApp:
                     except StopIteration:
                         break
             elif source_file.config.transform_mode == 'loop':
-                importlib.import_module(transform_code)
+                if transform_code not in sys.modules.keys():
+                    importlib.import_module(transform_code)
+                else:
+                    importlib.reload(importlib.import_module(transform_code))
             else:
                 raise NotImplementedError
 
             # close the writer when the source is done processing
             self.writer_registry[source_file.config.name].finalize()
 
+            # remove directory from sys.path to prevent name clashes
+            sys.path.remove(str(parent_path))
+
     def load_map(self, map_file_config: MapFileConfig):
         source_file = SourceFile(map_file_config)
 
diff --git a/koza/io/writer/jsonl_writer.py b/koza/io/writer/jsonl_writer.py
index 89272c5..d3074ac 100644
--- a/koza/io/writer/jsonl_writer.py
+++ b/koza/io/writer/jsonl_writer.py
@@ -1,4 +1,5 @@
 import json
+import os
 from dataclasses import asdict
 from typing import List
 
@@ -12,6 +13,8 @@ class JSONLWriter(KozaWriter):
         self.output_dir = output_dir
         self.source_name = source_name
 
+        os.makedirs(output_dir, exist_ok=True)
+
         self.nodes_file = open(f"{output_dir}/{source_name}_nodes.jsonl", "w")
         self.edges_file = open(f"{output_dir}/{source_name}_edges.jsonl", "w")
 


Test patch:
diff --git a/tests/integration/test_examples.py b/tests/integration/test_examples.py
new file mode 100644
index 0000000..91bed5d
--- /dev/null
+++ b/tests/integration/test_examples.py
@@ -0,0 +1,69 @@
+"""
+End to end test of String load from examples/string
+
+"""
+
+import os
+
+import pytest
+
+from koza.koza_runner import transform_source
+from koza.model.config.source_config import OutputFormat
+
+
+@pytest.mark.parametrize(
+    "ingest, output_names, output_format",
+    [
+        (
+            "string",
+            [
+                "stringdb.protein-links-detailed",
+            ],
+            OutputFormat.tsv,
+        ),
+        (
+            "string",
+            [
+                "stringdb.protein-links-detailed",
+            ],
+            OutputFormat.jsonl,
+        ),
+        (
+            "string-declarative",
+            [
+                "stringdb.protein-links-detailed",
+            ],
+            OutputFormat.tsv,
+        ),
+        (
+            "string-declarative",
+            [
+                "stringdb.protein-links-detailed",
+            ],
+            OutputFormat.jsonl,
+        ),
+    ],
+)
+def test_examples(ingest, output_names, output_format):
+
+    source = f"examples/{ingest}/metadata.yaml"
+    output_suffix = str(output_format).split('.')[1]
+    output_dir = f"./test-output/{ingest}-{output_suffix}"
+
+    output_files = [
+        output_file
+        for file in output_names
+        for output_file in [
+            f"{output_dir}/{file}_nodes.{output_suffix}",
+            f"{output_dir}/{file}_edges.{output_suffix}",
+        ]
+    ]
+
+    transform_source(source, output_dir, output_format, None, None)
+
+    for file in output_files:
+        assert os.path.exists(file)
+        assert os.path.getsize(file) > 0
+
+    # TODO: at some point, these assertions could get more rigorous, but knowing if we have errors/exceptions is a start
+    # TODO: kgx validation could also be added back in, especially if something programatic is done with the output