Repository: monarch-initiative/koza

Issue:
Add option to specify translation tables in source config
To simplify executing transforms, we should add the option to specify global and local translation tables as yaml imports in source_file_config.


Discussion:

Patch:
diff --git a/docs/ingest_configuration.md b/docs/ingest_configuration.md
index 1283d59..0c137b0 100644
--- a/docs/ingest_configuration.md
+++ b/docs/ingest_configuration.md
@@ -26,6 +26,14 @@ metadata:
   description: 'SomethingBase: A Website With Some Data'
   rights: 'https://somethingbase.org/rights.html'
 
+# The local and global tables can be specified either in the command line or the config
+global_table: './path_to/translation_table.yaml'
+local_table: './somethingbase/something-translation.yaml'
+
+# in addition to specifying yaml files, it's also possible to define the tables inline
+# local_table: 
+#   "around here somewhere": "RO:9999999"
+
 
 # in a JSON ingest, this will be the path to the array to be iterated over as the input collection
 json_path:
diff --git a/examples/string-declarative/protein-links-detailed.yaml b/examples/string-declarative/protein-links-detailed.yaml
index 1e11e84..6a0293f 100644
--- a/examples/string-declarative/protein-links-detailed.yaml
+++ b/examples/string-declarative/protein-links-detailed.yaml
@@ -12,6 +12,8 @@ metadata:
   description: 'STRING: functional protein association networks'
   rights: 'https://string-db.org/cgi/access.pl?footer_active_subpage=licensing'
 
+global_table: './examples/translation_table.yaml'
+
 columns:
   - 'protein1'
   - 'protein2'
diff --git a/koza/__init__.py b/koza/__init__.py
index 7948824..04f67b9 100644
--- a/koza/__init__.py
+++ b/koza/__init__.py
@@ -1,2 +1,2 @@
 """Koza, an ETL framework for LinkML data models"""
-__version__ = '0.1.5'
+__version__ = '0.1.6'
diff --git a/koza/cli_runner.py b/koza/cli_runner.py
index 7e73146..d90b80d 100644
--- a/koza/cli_runner.py
+++ b/koza/cli_runner.py
@@ -4,7 +4,7 @@ Module for managing koza runs through the CLI
 
 import logging
 from pathlib import Path
-from typing import Optional
+from typing import Optional, Union, Dict
 
 import yaml
 
@@ -92,7 +92,7 @@ def validate_file(
             pass
 
 
-def get_translation_table(global_table: str = None, local_table: str = None) -> TranslationTable:
+def get_translation_table(global_table: Union[str, Dict] = None, local_table: Union[str, Dict] = None) -> TranslationTable:
     """
     Create a translation table object from two file paths
     :param global_table: str, path to global table yaml
@@ -110,12 +110,19 @@ def get_translation_table(global_table: str = None, local_table: str = None) ->
             LOG.info("No global table used for transform")
     else:
 
-        with open(global_table, 'r') as global_tt_fh:
-            global_tt = yaml.safe_load(global_tt_fh)
+        if isinstance(global_table, str):
+            with open(global_table, 'r') as global_tt_fh:
+                global_tt = yaml.safe_load(global_tt_fh)
+        elif isinstance(global_table, Dict):
+            global_tt = global_table
 
         if local_table:
-            with open(local_table, 'r') as local_tt_fh:
-                local_tt = yaml.safe_load(local_tt_fh)
+            if isinstance(local_table, str):
+                with open(local_table, 'r') as local_tt_fh:
+                    local_tt = yaml.safe_load(local_tt_fh)
+            elif isinstance(local_table, Dict):
+                local_tt = local_table
+
         else:
             LOG.info("No local table used for transform")
 
@@ -130,8 +137,6 @@ def transform_source(
     local_table: str = None,
 ):
 
-    translation_table = get_translation_table(global_table, local_table)
-
     with open(source, 'r') as source_fh:
         source_config = PrimaryFileConfig(**yaml.load(source_fh, Loader=UniqueIncludeLoader))
         if not source_config.name:
@@ -143,6 +148,9 @@ def transform_source(
 
         koza_source = Source(source_config)
 
+        translation_table = get_translation_table(global_table if global_table else source_config.global_table,
+                                                  local_table if local_table else source_config.local_table)
+
         koza_app = set_koza_app(koza_source, translation_table, output_dir, output_format)
         koza_app.process_maps()
         koza_app.process_sources()
diff --git a/koza/model/config/source_config.py b/koza/model/config/source_config.py
index 75105ce..c4a4d91 100644
--- a/koza/model/config/source_config.py
+++ b/koza/model/config/source_config.py
@@ -186,6 +186,9 @@ class SourceConfig:
     json_path: List[Union[StrictStr, StrictInt]] = None
     transform_code: str = None
     transform_mode: TransformMode = TransformMode.flat
+    global_table: Union[str, Dict] = None
+    local_table: Union[str, Dict] = None
+
 
     def __post_init_post_parse__(self):
         """


Test patch:
diff --git a/tests/unit/resources/primary-source.yaml b/tests/unit/resources/primary-source.yaml
index d520837..af1c5a2 100644
--- a/tests/unit/resources/primary-source.yaml
+++ b/tests/unit/resources/primary-source.yaml
@@ -17,6 +17,9 @@ files:
   - '9606.protein.links.detailed.v11.0.txt.gz'
   - '10090.protein.links.detailed.v11.0.txt.gz'
 
+local_table:
+  "is just a little to the left of": "RO:123"
+
 columns:
   - 'protein1'
   - 'protein2'
diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py
index 9928952..8419380 100644
--- a/tests/unit/test_config.py
+++ b/tests/unit/test_config.py
@@ -16,6 +16,10 @@ def test_source_primary_config():
     with open(base_config, 'r') as config:
         PrimaryFileConfig(**yaml.safe_load(config))
 
+def test_inline_local_table():
+    with open(base_config, 'r') as config:
+        config = PrimaryFileConfig(**yaml.safe_load(config))
+    assert config.local_table["is just a little to the left of"] == "RO:123"
 
 @pytest.mark.parametrize(
     "inclusion, column, filter_code, value",