Repository: iossifovlab/gpf

Issue:
[ENH] Add internal in-memory genotype storage
**Is your feature request related to a problem? Please describe.**

Currently, each GPF instance has genotype storages that are described in the `genotype_storages` configuration section. 

We would like to have a default `genotype_storages` configuration that configures internal in-memory genotype storage. This internal genotype storage should be available in all cases.

**Describe the solution you'd like**
Change the GPF instance to create and register the `internal` genotype storage.


Discussion:

Patch:
diff --git a/dae/dae/gpf_instance/gpf_instance.py b/dae/dae/gpf_instance/gpf_instance.py
index 4e950071a..3fc166e7c 100644
--- a/dae/dae/gpf_instance/gpf_instance.py
+++ b/dae/dae/gpf_instance/gpf_instance.py
@@ -83,6 +83,7 @@ class GPFInstance:
 
         self.dae_config = dae_config
         self.dae_dir = str(dae_dir)
+        assert self.dae_dir is not None
 
         self._grr = kwargs.get("grr")
         self._reference_genome = kwargs.get("reference_genome")
@@ -99,7 +100,7 @@ class GPFInstance:
         self._variants_db
         self.denovo_gene_sets_db
         self.genomic_scores_db
-        self.genotype_storage_db
+        self.genotype_storages
         self._background_facade
 
     @cached_property
@@ -202,13 +203,19 @@ class GPFInstance:
         return GenomicScoresDb(self.grr, scores)
 
     @cached_property
-    def genotype_storage_db(self):
+    def genotype_storages(self):
         """Construct and return genotype storage registry."""
         # pylint: disable=import-outside-toplevel
         from dae.genotype_storage.genotype_storage_registry import \
             GenotypeStorageRegistry
-
         registry = GenotypeStorageRegistry()
+        internal_storage = registry.register_storage_config({
+            "id": "internal",
+            "storage_type": "inmemory",
+            "dir": os.path.join(self.dae_dir, "internal_storage"),
+        })
+        registry.register_default_storage(internal_storage)
+
         if self.dae_config.genotype_storage:
             registry.register_storages_configs(
                 self.dae_config.genotype_storage)
@@ -220,7 +227,7 @@ class GPFInstance:
             self.dae_config,
             self.reference_genome,
             self.gene_models,
-            self.genotype_storage_db,
+            self.genotype_storages,
         )
 
     @cached_property
diff --git a/dae/dae/impala_storage/schema1/import_commons.py b/dae/dae/impala_storage/schema1/import_commons.py
index eeb30f918..dcd5824f1 100644
--- a/dae/dae/impala_storage/schema1/import_commons.py
+++ b/dae/dae/impala_storage/schema1/import_commons.py
@@ -843,7 +843,7 @@ class BatchImporter:
         else:
             genotype_storage_id = argv.genotype_storage
 
-        genotype_storage = self.gpf_instance.genotype_storage_db \
+        genotype_storage = self.gpf_instance.genotype_storages \
             .get_genotype_storage(
                 genotype_storage_id
             )
@@ -1551,7 +1551,7 @@ class DatasetHelpers:
             return None
         gpf_instance = self.gpf_instance
         genotype_storage = gpf_instance \
-            .genotype_storage_db \
+            .genotype_storages \
             .get_genotype_storage(
                 config.genotype_storage.id)
         return genotype_storage
diff --git a/dae/dae/import_tools/import_tools.py b/dae/dae/import_tools/import_tools.py
index 9d8c2543a..1cfa5f44f 100644
--- a/dae/dae/import_tools/import_tools.py
+++ b/dae/dae/import_tools/import_tools.py
@@ -289,12 +289,12 @@ class ImportProject():
         )
         if not explicit_config:
             gpf_instance = self.get_gpf_instance()
-            genotype_storage_db = gpf_instance.genotype_storage_db
+            genotype_storages = gpf_instance.genotype_storages
             storage_id = self.import_config.get("destination", {})\
                 .get("storage_id", None)
             if storage_id is not None:
-                return genotype_storage_db.get_genotype_storage(storage_id)
-            return genotype_storage_db.get_default_genotype_storage()
+                return genotype_storages.get_genotype_storage(storage_id)
+            return genotype_storages.get_default_genotype_storage()
         # explicit storage config
         registry = GenotypeStorageRegistry()
         # FIXME: switch to using new storage configuration
@@ -336,7 +336,7 @@ class ImportProject():
             # get default storage schema from GPF instance
             gpf_instance = self.get_gpf_instance()
             storage: GenotypeStorage = gpf_instance\
-                .genotype_storage_db.get_default_genotype_storage()
+                .genotype_storages.get_default_genotype_storage()
             return storage.get_storage_type()
 
         destination = self.import_config["destination"]
@@ -344,7 +344,7 @@ class ImportProject():
             storage_id = destination["storage_id"]
             gpf_instance = self.get_gpf_instance()
             storage = gpf_instance\
-                .genotype_storage_db\
+                .genotype_storages\
                 .get_genotype_storage(storage_id)
             return storage.get_storage_type()
 
diff --git a/dae/dae/tools/hdfs_parquet_loader.py b/dae/dae/tools/hdfs_parquet_loader.py
index fee5501fe..3739c2919 100755
--- a/dae/dae/tools/hdfs_parquet_loader.py
+++ b/dae/dae/tools/hdfs_parquet_loader.py
@@ -80,8 +80,8 @@ def main(argv=None, gpf_instance=None):
 
     logging.getLogger("impala").setLevel(logging.WARNING)
 
-    genotype_storage_db = gpf_instance.genotype_storage_db
-    genotype_storage = genotype_storage_db.get_genotype_storage(
+    genotype_storages = gpf_instance.genotype_storages
+    genotype_storage = genotype_storages.get_genotype_storage(
         argv.genotype_storage
     )
     if not genotype_storage or \
diff --git a/dae/dae/tools/impala_parquet_loader.py b/dae/dae/tools/impala_parquet_loader.py
index d9c3f92a4..9ce27571f 100755
--- a/dae/dae/tools/impala_parquet_loader.py
+++ b/dae/dae/tools/impala_parquet_loader.py
@@ -82,8 +82,8 @@ def main(argv=None, gpf_instance=None):
 
     argv = parse_cli_arguments(argv or sys.argv[1:], gpf_instance)
 
-    genotype_storage_db = gpf_instance.genotype_storage_db
-    genotype_storage = genotype_storage_db.get_genotype_storage(
+    genotype_storages = gpf_instance.genotype_storages
+    genotype_storage = genotype_storages.get_genotype_storage(
         argv.genotype_storage
     )
     if not genotype_storage or \
diff --git a/dae/dae/tools/impala_tables_loader.py b/dae/dae/tools/impala_tables_loader.py
index 978da1fdd..6a8704a88 100755
--- a/dae/dae/tools/impala_tables_loader.py
+++ b/dae/dae/tools/impala_tables_loader.py
@@ -109,8 +109,8 @@ def main(argv=None, gpf_instance=None):
 
     logging.getLogger("impala").setLevel(logging.WARNING)
 
-    genotype_storage_db = gpf_instance.genotype_storage_db
-    genotype_storage = genotype_storage_db.get_genotype_storage(
+    genotype_storages = gpf_instance.genotype_storages
+    genotype_storage = genotype_storages.get_genotype_storage(
         argv.genotype_storage)
 
     if not genotype_storage or \


Test patch:
diff --git a/dae/dae/genomic_resources/tests/test_cli_browse.py b/dae/dae/genomic_resources/tests/test_cli_browse.py
index c2ba9d0b1..7e1787b9e 100644
--- a/dae/dae/genomic_resources/tests/test_cli_browse.py
+++ b/dae/dae/genomic_resources/tests/test_cli_browse.py
@@ -76,7 +76,7 @@ def test_cli_browse_with_env_variable(repo_fixture, repo_def, capsys, mocker):
 def test_cli_browse_default_defintion(repo_fixture, repo_def, capsys, mocker):
 
     mocker.patch.dict(os.environ, {"HOME": str(repo_def.parent)})
-    os.environ.pop("GRR_DEFINITION_FILE")
+    os.environ.pop("GRR_DEFINITION_FILE", None)
 
     cli_browse([])
     out, err = capsys.readouterr()
diff --git a/dae/dae/genotype_storage/tests/test_genotype_storage_registry.py b/dae/dae/genotype_storage/tests/test_genotype_storage_registry.py
index 7f13608c3..31f2b8bdd 100644
--- a/dae/dae/genotype_storage/tests/test_genotype_storage_registry.py
+++ b/dae/dae/genotype_storage/tests/test_genotype_storage_registry.py
@@ -9,7 +9,7 @@ from dae.inmemory_storage.inmemory_genotype_storage import \
 
 @pytest.fixture(scope="session")
 def genotype_storage_registry(fixtures_gpf_instance):
-    return fixtures_gpf_instance.genotype_storage_db
+    return fixtures_gpf_instance.genotype_storages
 
 
 @pytest.fixture(scope="session")
@@ -21,8 +21,9 @@ def test_get_genotype_storage_ids(genotype_storage_registry):
     genotype_storage_ids = \
         genotype_storage_registry.get_all_genotype_storage_ids()
 
-    assert len(genotype_storage_ids) == 7
+    assert len(genotype_storage_ids) == 8
     assert genotype_storage_ids == [
+        "internal",
         "genotype_impala",
         "genotype_impala_2",
         "genotype_impala_backends",
diff --git a/dae/dae/gpf_instance/tests/test_internal_storage.py b/dae/dae/gpf_instance/tests/test_internal_storage.py
new file mode 100644
index 000000000..a905439fa
--- /dev/null
+++ b/dae/dae/gpf_instance/tests/test_internal_storage.py
@@ -0,0 +1,77 @@
+# pylint: disable=W0621,C0114,C0116,W0212,W0613
+import textwrap
+
+from dae.testing import setup_gpf_instance, setup_genome, \
+    setup_empty_gene_models, setup_directories
+
+
+def test_internal_genotype_storage(tmp_path_factory):
+    # Given
+    root_path = tmp_path_factory.mktemp("internal_storage_test")
+
+    genome = setup_genome(
+        root_path / "alla_gpf" / "genome" / "allChr.fa",
+        f"""
+        >chrA
+        {100 * "A"}
+        """
+    )
+    empty_gene_models = setup_empty_gene_models(
+        root_path / "alla_gpf" / "empty_gene_models" / "empty_genes.txt")
+    gpf = setup_gpf_instance(
+        root_path / "gpf_instance",
+        reference_genome=genome,
+        gene_models=empty_gene_models,
+    )
+
+    # When
+    internal_storage = gpf.genotype_storages.get_genotype_storage("internal")
+
+    # Then
+    assert internal_storage.get_storage_type() == "inmemory"
+    assert internal_storage.storage_config["dir"] == \
+        str(root_path / "gpf_instance" / "internal_storage")
+    assert gpf.genotype_storages.get_default_genotype_storage() == \
+        internal_storage
+
+
+def test_internal_genotype_storage_with_other_storages(tmp_path_factory):
+    # Given
+    root_path = tmp_path_factory.mktemp("internal_storage_test")
+
+    genome = setup_genome(
+        root_path / "alla_gpf" / "genome" / "allChr.fa",
+        f"""
+        >chrA
+        {100 * "A"}
+        """
+    )
+    empty_gene_models = setup_empty_gene_models(
+        root_path / "alla_gpf" / "empty_gene_models" / "empty_genes.txt")
+
+    setup_directories(root_path / "gpf_instance", {
+        "gpf_instance.yaml": textwrap.dedent("""
+        genotype_storage:
+            default: alabala
+            storages:
+            - id: alabala
+              storage_type: inmemory
+              dir: "%(wd)s/alabala_storage"
+        """)
+    })
+
+    gpf = setup_gpf_instance(
+        root_path / "gpf_instance",
+        reference_genome=genome,
+        gene_models=empty_gene_models,
+    )
+
+    # When
+    internal_storage = gpf.genotype_storages.get_genotype_storage("internal")
+
+    # Then
+    assert internal_storage.get_storage_type() == "inmemory"
+    assert internal_storage.storage_config["dir"] == \
+        str(root_path / "gpf_instance" / "internal_storage")
+    assert gpf.genotype_storages.get_default_genotype_storage() != \
+        internal_storage
diff --git a/dae/dae/import_tools/tests/test_import_configs.py b/dae/dae/import_tools/tests/test_import_configs.py
index 8f6768a21..0a8b24c1b 100644
--- a/dae/dae/import_tools/tests/test_import_configs.py
+++ b/dae/dae/import_tools/tests/test_import_configs.py
@@ -226,7 +226,7 @@ def test_get_genotype_storage_no_explicit_config():
     assert genotype_storage is not None
     assert (
         genotype_storage.storage_id
-        == project.get_gpf_instance().genotype_storage_db
+        == project.get_gpf_instance().genotype_storages
         .get_default_genotype_storage().storage_id
     )
 
diff --git a/dae/dae/parquet/schema1/tests/test_serializer.py b/dae/dae/parquet/schema1/tests/test_serializer.py
index 3a501601b..3ec32bcb3 100644
--- a/dae/dae/parquet/schema1/tests/test_serializer.py
+++ b/dae/dae/parquet/schema1/tests/test_serializer.py
@@ -21,7 +21,7 @@ def extra_attrs_impala(
     variants_filename = fixture_dirname("backends/iossifov_extra_attrs.tsv")
     root_path = tmp_path_factory.mktemp(study_id)
     gpf_instance_2013\
-        .genotype_storage_db\
+        .genotype_storages\
         .register_genotype_storage(impala_genotype_storage)
 
     project = {
diff --git a/dae/dae/studies/tests/conftest.py b/dae/dae/studies/tests/conftest.py
index b55e3bcaa..f985fb089 100644
--- a/dae/dae/studies/tests/conftest.py
+++ b/dae/dae/studies/tests/conftest.py
@@ -45,7 +45,7 @@ def gene_scores_db(local_gpf_instance):
 
 @pytest.fixture(scope="session")
 def genotype_storage_factory(local_gpf_instance):
-    return local_gpf_instance.genotype_storage_db
+    return local_gpf_instance.genotype_storages
 
 
 @pytest.fixture(scope="session")
diff --git a/dae/dae/studies/tests/test_study_without_variants.py b/dae/dae/studies/tests/test_study_without_variants.py
index 811a6380f..57fca0262 100644
--- a/dae/dae/studies/tests/test_study_without_variants.py
+++ b/dae/dae/studies/tests/test_study_without_variants.py
@@ -48,7 +48,7 @@ def no_variants_study(gpf_instance_2013, tmp_path):
         content, study_config_schema, default_config, tmp_path)
 
     genotype_storage = \
-        gpf_instance_2013.genotype_storage_db.get_genotype_storage(
+        gpf_instance_2013.genotype_storages.get_genotype_storage(
             "genotype_impala"
         )
 
diff --git a/dae/dae/testing/import_helpers.py b/dae/dae/testing/import_helpers.py
index 8c082b04e..e81ca043a 100644
--- a/dae/dae/testing/import_helpers.py
+++ b/dae/dae/testing/import_helpers.py
@@ -22,7 +22,7 @@ def data_import(root_path: pathlib.Path, study: StudyLayout, gpf_instance):
     params = asdict(study)
     params["work_dir"] = str(root_path / "work_dir")
     params["storage_id"] = gpf_instance\
-        .genotype_storage_db\
+        .genotype_storages\
         .get_default_genotype_storage()\
         .storage_id
 
diff --git a/dae/tests/alla_import.py b/dae/tests/alla_import.py
index e19c3fad4..db68571c0 100644
--- a/dae/tests/alla_import.py
+++ b/dae/tests/alla_import.py
@@ -21,6 +21,6 @@ def alla_gpf(root_path, storage=None):
 
     if storage:
         gpf_instance\
-            .genotype_storage_db\
+            .genotype_storages\
             .register_default_storage(storage)
     return gpf_instance
diff --git a/dae/tests/foobar_import.py b/dae/tests/foobar_import.py
index 1cb31b5bc..ade7a4559 100644
--- a/dae/tests/foobar_import.py
+++ b/dae/tests/foobar_import.py
@@ -37,6 +37,6 @@ def foobar_gpf(root_path, storage=None):
 
     if storage:
         gpf_instance\
-            .genotype_storage_db\
+            .genotype_storages\
             .register_default_storage(storage)
     return gpf_instance
diff --git a/dae/tests/integration/schema1/test_cnv_variants.py b/dae/tests/integration/schema1/test_cnv_variants.py
index f2aa99206..23e2bc8d5 100644
--- a/dae/tests/integration/schema1/test_cnv_variants.py
+++ b/dae/tests/integration/schema1/test_cnv_variants.py
@@ -89,7 +89,7 @@ def cnv_impala(
         impala_hosts=[impala_host], impala_port=21050)
 
     study_id = f"cnv_test_{impala_genotype_storage.storage_id}"
-    gpf_instance_2013.genotype_storage_db.register_genotype_storage(
+    gpf_instance_2013.genotype_storages.register_genotype_storage(
         impala_genotype_storage)
     (variant_table, pedigree_table) = \
         impala_genotype_storage.study_tables(
diff --git a/dae_conftests/dae_conftests/dae_conftests.py b/dae_conftests/dae_conftests/dae_conftests.py
index 818ff49bb..67b75b626 100644
--- a/dae_conftests/dae_conftests/dae_conftests.py
+++ b/dae_conftests/dae_conftests/dae_conftests.py
@@ -175,7 +175,7 @@ def gpf_instance_2013(
     grr = GenomicResourceGroupRepo(repositories)
     gpf_instance = GPFInstance2013(
         dae_config=default_dae_config,
-        dae_dir=default_dae_config.config_dir,
+        dae_dir=global_dae_fixtures_dir,
         grr=grr)
 
     return gpf_instance
@@ -843,7 +843,7 @@ def data_import(
 
     impala_helpers = ImpalaHelpers(
         impala_hosts=[impala_host], impala_port=21050)
-    gpf_instance_2013.genotype_storage_db.register_genotype_storage(
+    gpf_instance_2013.genotype_storages.register_genotype_storage(
         impala_genotype_storage)
 
     def build(dirname):
@@ -922,7 +922,7 @@ def iossifov2014_import(
                 impala_test_dbname(), pedigree_table):
         return
 
-    gpf_instance_2013.genotype_storage_db.register_genotype_storage(
+    gpf_instance_2013.genotype_storages.register_genotype_storage(
         impala_genotype_storage)
     config = from_prefix_denovo(
         fixture_dirname("dae_iossifov2014/iossifov2014"))