Repository: iossifovlab/gpf

Issue:
[BUG] Legacy snakemake based import system is broken
**What happened**:

The `gpf_validation_data` build is broken. The snakemake could not run on snakemake files produced by the GPF batch import command.

**What you expected to happen**:

I expect the `gpf_validation_data` build to pass.

**Minimal Complete Verifiable Example**:

Any `gpf_validation_data` import project.

**Environment**:

- GPF version: development version 3.7dev from the master


Discussion:

Patch:
diff --git a/dae/dae/genomic_resources/genomic_scores.py b/dae/dae/genomic_resources/genomic_scores.py
index 47c96e10a..7322e11b6 100644
--- a/dae/dae/genomic_resources/genomic_scores.py
+++ b/dae/dae/genomic_resources/genomic_scores.py
@@ -419,7 +419,7 @@ class GenomicScore(GenomicResourceImplementation):
     def get_schema():
         return {
             **get_base_resource_schema(),
-            "table": {"type": "dict", "schema": {
+            "table": {"type": "dict", "allow_unknown": True, "schema": {
                 "filename": {"type": "string"},
                 "format": {"type": "string"},
                 "header_mode": {"type": "string"},
diff --git a/dae/dae/impala_storage/schema1/import_commons.py b/dae/dae/impala_storage/schema1/import_commons.py
index dcd5824f1..35803ed75 100644
--- a/dae/dae/impala_storage/schema1/import_commons.py
+++ b/dae/dae/impala_storage/schema1/import_commons.py
@@ -116,9 +116,7 @@ class MakefilePartitionHelper:
     def __init__(
             self,
             partition_descriptor,
-            genome,
-            add_chrom_prefix=None,
-            del_chrom_prefix=None):
+            genome):
 
         self.genome = genome
         self.partition_descriptor = partition_descriptor
@@ -126,26 +124,6 @@ class MakefilePartitionHelper:
             self.genome.get_all_chrom_lengths()
         )
 
-        self._build_adjust_chrom(add_chrom_prefix, del_chrom_prefix)
-
-    def _build_adjust_chrom(self, add_chrom_prefix, del_chrom_prefix):
-        self._chrom_prefix = None
-
-        def same_chrom(chrom):
-            return chrom
-
-        self._adjust_chrom = same_chrom
-        self._unadjust_chrom = same_chrom
-
-        if add_chrom_prefix is not None:
-            self._chrom_prefix = add_chrom_prefix
-            self._adjust_chrom = self._prepend_chrom_prefix
-            self._unadjust_chrom = self._remove_chrom_prefix
-        elif del_chrom_prefix is not None:
-            self._chrom_prefix = del_chrom_prefix
-            self._adjust_chrom = self._remove_chrom_prefix
-            self._unadjust_chrom = self._prepend_chrom_prefix
-
     def region_bins_count(self, chrom):
         result = ceil(
             self.chromosome_lengths[chrom]
@@ -153,22 +131,9 @@ class MakefilePartitionHelper:
         )
         return result
 
-    def _remove_chrom_prefix(self, chrom):
-        assert self._chrom_prefix
-        if chrom.startswith(self._chrom_prefix):
-            # fmt: off
-            return chrom[len(self._chrom_prefix):]
-            # fmt: on
-        return chrom
-
-    def _prepend_chrom_prefix(self, chrom):
-        assert self._chrom_prefix
-        if not chrom.startswith(self._chrom_prefix):
-            return f"{self._chrom_prefix}{chrom}"
-        return chrom
-
-    def build_target_chromosomes(self, target_chromosomes):
-        return [self._adjust_chrom(tg) for tg in target_chromosomes]
+    @staticmethod
+    def build_target_chromosomes(target_chromosomes):
+        return target_chromosomes[:]
 
     def generate_chrom_targets(self, target_chrom):
         """Generate variant targets based on partition descriptor."""
@@ -177,7 +142,7 @@ class MakefilePartitionHelper:
             target = "other"
         region_bins_count = self.region_bins_count(target_chrom)
 
-        chrom = self._unadjust_chrom(target_chrom)
+        chrom = target_chrom
 
         if region_bins_count == 1:
             return [(f"{target}_0", chrom)]
@@ -210,10 +175,7 @@ class MakefilePartitionHelper:
         if len(self.partition_descriptor.chromosomes) == 0:
             return {"none": [self.partition_descriptor.output]}
 
-        generated_target_chromosomes = [
-            self._adjust_chrom(tg) for tg in target_chromosomes[:]
-        ]
-
+        generated_target_chromosomes = target_chromosomes[:]
         if mode == "single_bucket":
             targets = {"all": [None]}
             return targets
@@ -822,14 +784,9 @@ class BatchImporter:
         else:
             partition_description = NoPartitionDescriptor(argv.output)
 
-        add_chrom_prefix = argv.add_chrom_prefix
-        del_chrom_prefix = argv.del_chrom_prefix
-
         self.partition_helper = MakefilePartitionHelper(
             partition_description,
             self.gpf_instance.reference_genome,
-            add_chrom_prefix=add_chrom_prefix,
-            del_chrom_prefix=del_chrom_prefix,
         )
 
         return self
@@ -1376,7 +1333,7 @@ class Variants2ParquetTool:
 
         partition_description = cls._build_partition_description(argv)
         generator = cls._build_partition_helper(
-            argv, gpf_instance, partition_description
+            gpf_instance, partition_description
         )
 
         target_chromosomes = cls._collect_target_chromosomes(
@@ -1477,16 +1434,11 @@ class Variants2ParquetTool:
         return partition_description
 
     @staticmethod
-    def _build_partition_helper(argv, gpf_instance, partition_description):
-
-        add_chrom_prefix = argv.add_chrom_prefix
-        del_chrom_prefix = argv.del_chrom_prefix
+    def _build_partition_helper(gpf_instance, partition_description):
 
         generator = MakefilePartitionHelper(
             partition_description,
             gpf_instance.reference_genome,
-            add_chrom_prefix=add_chrom_prefix,
-            del_chrom_prefix=del_chrom_prefix,
         )
         return generator
 
diff --git a/dae/dae/import_tools/import_tools.py b/dae/dae/import_tools/import_tools.py
index ad7a941cc..adc6c29df 100644
--- a/dae/dae/import_tools/import_tools.py
+++ b/dae/dae/import_tools/import_tools.py
@@ -146,7 +146,7 @@ class ImportProject():
         for loader_type in ["denovo", "vcf", "cnv", "dae"]:
             config = self.import_config["input"].get(loader_type, None)
             if config is not None:
-                for bucket in self._loader_region_bins(config, loader_type):
+                for bucket in self._loader_region_bins(loader_type):
                     buckets.append(bucket)
         return buckets
 
@@ -387,7 +387,7 @@ class ImportProject():
             res[key] = val
         return res
 
-    def _loader_region_bins(self, loader_args, loader_type):
+    def _loader_region_bins(self, loader_type):
         # pylint: disable=too-many-locals
         reference_genome = self.get_gpf_instance().reference_genome
 
@@ -408,8 +408,6 @@ class ImportProject():
         partition_helper = MakefilePartitionHelper(
             partition_description,
             reference_genome,
-            add_chrom_prefix=loader_args.get("add_chrom_prefix", None),
-            del_chrom_prefix=loader_args.get("del_chrom_prefix", None),
         )
 
         processing_config = self._get_loader_processing_config(loader_type)
diff --git a/dae/dae/variants_loaders/dae/loader.py b/dae/dae/variants_loaders/dae/loader.py
index 7f29854f7..0a762c4c7 100644
--- a/dae/dae/variants_loaders/dae/loader.py
+++ b/dae/dae/variants_loaders/dae/loader.py
@@ -434,9 +434,8 @@ class DenovoLoader(VariantsGenotypesLoader):
 
         return argv.denovo_file, params
 
-    @classmethod
     def _flexible_denovo_load_internal(
-            cls,
+            self,
             filepath: str,
             genome: ReferenceGenome,
             families: FamiliesData,
@@ -496,7 +495,7 @@ class DenovoLoader(VariantsGenotypesLoader):
 
         if denovo_location:
             chrom_col, pos_col = zip(
-                *map(cls.split_location, raw_df[denovo_location])
+                *map(self.split_location, raw_df[denovo_location])
             )
         else:
             assert denovo_chrom is not None
@@ -508,7 +507,8 @@ class DenovoLoader(VariantsGenotypesLoader):
             variant_col = raw_df.loc[:, denovo_variant]
             ref_alt_tuples = [
                 dae2vcf_variant(
-                    variant_tuple[0], variant_tuple[1], variant_tuple[2],
+                    self._adjust_chrom_prefix(variant_tuple[0]),
+                    variant_tuple[1], variant_tuple[2],
                     genome
                 ) for variant_tuple in zip(chrom_col, pos_col, variant_col)
             ]
@@ -565,7 +565,7 @@ class DenovoLoader(VariantsGenotypesLoader):
                         "reference": variant[2],
                         "alternative": variant[3],
                         "family_id": family_id,
-                        "genotype": cls.produce_genotype(
+                        "genotype": self.produce_genotype(
                             variant[0],
                             variant[1],
                             genome,
@@ -630,9 +630,8 @@ class DenovoLoader(VariantsGenotypesLoader):
 
         return (denovo_df, extra_attributes_cols.tolist())
 
-    @classmethod
     def flexible_denovo_load(
-            cls,
+            self,
             filepath: str,
             genome: ReferenceGenome,
             families: FamiliesData,
@@ -702,7 +701,7 @@ class DenovoLoader(VariantsGenotypesLoader):
         """
         # FIXME too many arguments
         # pylint: disable=too-many-arguments
-        denovo_df, _ = cls._flexible_denovo_load_internal(
+        denovo_df, _ = self._flexible_denovo_load_internal(
             filepath,
             genome,
             families,
diff --git a/environment.yml b/environment.yml
index 57b648e7e..e3bac01c1 100644
--- a/environment.yml
+++ b/environment.yml
@@ -45,7 +45,7 @@ dependencies:
   - toml=0.10.2
   - cerberus=1.3.4
   - jinja2=3.1.2
-  - snakemake-minimal=7.12.1
+  - snakemake-minimal=7.18.2
   - ijson=3.1.4
   - dask=2022.7.1
   - dask-kubernetes=2022.7.0


Test patch:
diff --git a/dae/dae/impala_storage/schema1/tests/test_makefile_partition_helper.py b/dae/dae/impala_storage/schema1/tests/test_makefile_partition_helper.py
index d39a47656..b8f20a211 100644
--- a/dae/dae/impala_storage/schema1/tests/test_makefile_partition_helper.py
+++ b/dae/dae/impala_storage/schema1/tests/test_makefile_partition_helper.py
@@ -179,18 +179,17 @@ def test_target_generator_chrom_prefix_target_other(
     generator = MakefilePartitionHelper(
         partition_descriptor,
         gpf_instance_2013.reference_genome,
-        add_chrom_prefix="chr",
     )
     print(generator.chromosome_lengths)
     assert len(generator.chromosome_lengths) == 4
 
-    result = generator.generate_variants_targets(["3", "4"])
+    result = generator.generate_variants_targets(["chr3", "chr4"])
     print(result)
     assert set(result.keys()) == targets
     for regions in result.values():
         for region in regions:
             print(region)
-            assert "chr" not in region
+            assert "chr" in region
 
 
 @pytest.mark.parametrize(
@@ -223,18 +222,17 @@ def test_target_generator_add_chrom_prefix_target_chrom(
     generator = MakefilePartitionHelper(
         partition_descriptor,
         gpf_instance_2013.reference_genome,
-        add_chrom_prefix="chr",
     )
     print(generator.chromosome_lengths)
     assert len(generator.chromosome_lengths) == 4
 
-    result = generator.generate_variants_targets(["1", "2"])
+    result = generator.generate_variants_targets(["chr1", "chr2"])
     print(result)
     assert set(result.keys()) == targets
     for regions in result.values():
         for region in regions:
             print(region)
-            assert "chr" not in region
+            assert "chr" in region
 
 
 @pytest.mark.parametrize(
@@ -267,7 +265,7 @@ def test_target_generator_del_chrom_prefix_target_chrom(
     generator = MakefilePartitionHelper(
         partition_descriptor,
         gpf_instance_2013.reference_genome,
-        del_chrom_prefix="chr",
+        # del_chrom_prefix="chr",
     )
     print(generator.chromosome_lengths)
     assert len(generator.chromosome_lengths) == 4
@@ -331,7 +329,6 @@ def test_makefile_generator_bucket_numbering(
     generator = MakefilePartitionHelper(
         partition_descriptor,
         gpf_instance_2013.reference_genome,
-        add_chrom_prefix="chr",
     )
     print(generator.chromosome_lengths)
     assert len(generator.chromosome_lengths) == 4
@@ -408,23 +405,23 @@ def test_makefile_generator_regions(
         (
             3_000_000_000,
             [
-                ("1_0", ["chr1"]),
-                ("2_0", ["chr2"]),
-                ("other_0", ["chr3", "chr4"]),
+                ("1_0", ["1"]),
+                ("2_0", ["2"]),
+                ("other_0", ["3", "4"]),
             ],
         ),
         (
             150_000_000,
             [
-                ("1_0", ["chr1"]),
-                ("2_0", ["chr2:1-150000000"]),
-                ("2_1", ["chr2:150000001-200000000"]),
-                ("other_0", ["chr3:1-150000000", "chr4:1-150000000"]),
+                ("1_0", ["1"]),
+                ("2_0", ["2:1-150000000"]),
+                ("2_1", ["2:150000001-200000000"]),
+                ("other_0", ["3:1-150000000", "4:1-150000000"]),
                 (
                     "other_1",
-                    ["chr3:150000001-300000000", "chr4:150000001-300000000"],
+                    ["3:150000001-300000000", "4:150000001-300000000"],
                 ),
-                ("other_2", ["chr4:300000001-400000000"]),
+                ("other_2", ["4:300000001-400000000"]),
             ],
         ),
     ],
@@ -451,14 +448,14 @@ def test_makefile_generator_regions_del_chrom_prefix(
     generator = MakefilePartitionHelper(
         partition_descriptor,
         gpf_instance_2013.reference_genome,
-        del_chrom_prefix="chr",
+        # del_chrom_prefix="chr",
     )
 
     print(generator.chromosome_lengths)
     assert len(generator.chromosome_lengths) == 4
 
     variants_targets = generator.generate_variants_targets(
-        ["chr1", "chr2", "chr3", "chr4"]
+        ["1", "2", "3", "4"]
     )
 
     for (region_bin, regions) in targets:
@@ -471,14 +468,14 @@ def test_makefile_generator_regions_del_chrom_prefix(
     [
         (
             3_000_000_000,
-            [("chr1_0", ["1"]), ("chr2_0", ["2"]), ("other_0", ["3", "4"])],
+            [("1_0", ["1"]), ("2_0", ["2"]), ("other_0", ["3", "4"])],
         ),
         (
             150_000_000,
             [
-                ("chr1_0", ["1"]),
-                ("chr2_0", ["2:1-150000000"]),
-                ("chr2_1", ["2:150000001-200000000"]),
+                ("1_0", ["1"]),
+                ("2_0", ["2:1-150000000"]),
+                ("2_1", ["2:150000001-200000000"]),
                 ("other_0", ["3:1-150000000", "4:1-150000000"]),
                 (
                     "other_1",
@@ -497,21 +494,20 @@ def test_makefile_generator_regions_add_chrom_prefix(
         ReferenceGenome,
         "get_all_chrom_lengths",
         return_value=[
-            ("chr1", 100_000_000),
-            ("chr2", 200_000_000),
-            ("chr3", 300_000_000),
-            ("chr4", 400_000_000),
+            ("1", 100_000_000),
+            ("2", 200_000_000),
+            ("3", 300_000_000),
+            ("4", 400_000_000),
         ],
     )
 
     partition_descriptor = ParquetPartitionDescriptor(
-        ["chr1", "chr2"], region_length
+        ["1", "2"], region_length
     )
 
     generator = MakefilePartitionHelper(
         partition_descriptor,
         gpf_instance_2013.reference_genome,
-        add_chrom_prefix="chr",
     )
 
     print(generator.chromosome_lengths)
diff --git a/dae/dae/import_tools/tests/test_custom_region_length.py b/dae/dae/import_tools/tests/test_custom_region_length.py
index 58f28b9de..10958f1fb 100644
--- a/dae/dae/import_tools/tests/test_custom_region_length.py
+++ b/dae/dae/import_tools/tests/test_custom_region_length.py
@@ -112,7 +112,7 @@ def test_bucket_generation(gpf_instance_2019, mocker):
     mocker.patch.object(import_tools.ImportProject, "get_gpf_instance",
                         return_value=gpf_instance_2019)
     project = import_tools.ImportProject.build_from_config(import_config)
-    buckets = list(project._loader_region_bins({}, "denovo"))
+    buckets = list(project._loader_region_bins("denovo"))
     assert len(buckets) == 4
     assert buckets[0].regions == ["1:1-70000000"]
     assert buckets[1].regions == ["1:70000001-140000000"]
@@ -204,7 +204,7 @@ def test_single_bucket_generation(add_chrom_prefix, gpf_instance_2019, mocker):
     mocker.patch.object(import_tools.ImportProject, "get_gpf_instance",
                         return_value=gpf_instance_2019)
     project = import_tools.ImportProject.build_from_config(import_config)
-    buckets = list(project._loader_region_bins({}, "denovo"))
+    buckets = list(project._loader_region_bins("denovo"))
     assert len(buckets) == 1
     # assert buckets[0].regions == [f"{prefix}1", f"{prefix}2", f"{prefix}3",
     #                               f"{prefix}4", f"{prefix}5"]
@@ -219,7 +219,7 @@ def test_single_bucket_is_default_when_missing_processing_config(
     mocker.patch.object(import_tools.ImportProject, "get_gpf_instance",
                         return_value=gpf_instance_2019)
     project = import_tools.ImportProject.build_from_config(import_config)
-    buckets = list(project._loader_region_bins({}, "denovo"))
+    buckets = list(project._loader_region_bins("denovo"))
     assert len(buckets) == 1
     assert buckets[0].regions == [None]
 
@@ -238,7 +238,7 @@ def test_chromosome_bucket_generation(add_chrom_prefix, gpf_instance_2019,
     mocker.patch.object(import_tools.ImportProject, "get_gpf_instance",
                         return_value=gpf_instance_2019)
     project = import_tools.ImportProject.build_from_config(import_config)
-    buckets = list(project._loader_region_bins({}, "denovo"))
+    buckets = list(project._loader_region_bins("denovo"))
     assert len(buckets) == 5
     assert buckets[0].regions == [f"{prefix}1"]
     assert buckets[1].regions == [f"{prefix}2"]
@@ -256,7 +256,7 @@ def test_chromosome_list_bucket_generation(gpf_instance_2019, mocker):
     mocker.patch.object(import_tools.ImportProject, "get_gpf_instance",
                         return_value=gpf_instance_2019)
     project = import_tools.ImportProject.build_from_config(import_config)
-    buckets = list(project._loader_region_bins({}, "denovo"))
+    buckets = list(project._loader_region_bins("denovo"))
     assert len(buckets) == 4
     assert buckets[0].regions == ["1"]
     assert buckets[1].regions == ["2"]
diff --git a/dae/dae/import_tools/tests/test_import_configs.py b/dae/dae/import_tools/tests/test_import_configs.py
index 3ce571857..17b4a95b4 100644
--- a/dae/dae/import_tools/tests/test_import_configs.py
+++ b/dae/dae/import_tools/tests/test_import_configs.py
@@ -218,10 +218,13 @@ def test_project_input_dir(input_dir):
     assert project.input_dir == os.path.join("/dir", input_dir)
 
 
-def test_get_genotype_storage_no_explicit_config():
+def test_get_genotype_storage_no_explicit_config(fixture_dirname):
     config = dict(
         id="test_import",
         input={},
+        gpf_instance={
+            "path": fixture_dirname("")
+        }
     )
     project = import_tools.ImportProject.build_from_config(config)
     genotype_storage = project.get_genotype_storage()
@@ -276,14 +279,17 @@ def test_input_in_external_file(resources_dir):
     assert project.input_dir == f"{resources_dir}/external_input/files/"
 
 
-def test_embedded_annotation_pipeline():
+def test_embedded_annotation_pipeline(fixture_dirname):
     import_config = dict(
         input={},
         annotation=[
             dict(np_score=dict(
                 resource_id="hg19/scores/CADD",
             ))
-        ]
+        ],
+        gpf_instance={
+            "path": fixture_dirname("")
+        }
     )
     project = import_tools.ImportProject.build_from_config(import_config)
     pipeline = project._build_annotation_pipeline(project.get_gpf_instance())
@@ -293,7 +299,7 @@ def test_embedded_annotation_pipeline():
     ]
 
 
-def test_annotation_file(tmpdir):
+def test_annotation_file(tmpdir, fixture_dirname):
     annotation = [
         dict(np_score=dict(
             resource_id="hg19/scores/CADD",
@@ -307,7 +313,10 @@ def test_annotation_file(tmpdir):
         input={},
         annotation=dict(
             file="annotation.yaml"
-        )
+        ),
+        gpf_instance={
+            "path": fixture_dirname("")
+        }
     )
     config_fn = str(tmpdir / "import_config.yaml")
     with open(config_fn, "wt") as out_file:
@@ -321,7 +330,7 @@ def test_annotation_file(tmpdir):
     ]
 
 
-def test_annotation_file_and_external_input_config(tmpdir):
+def test_annotation_file_and_external_input_config(tmpdir, fixture_dirname):
     annotation = [
         dict(np_score=dict(
             resource_id="hg19/scores/CADD",
@@ -340,7 +349,10 @@ def test_annotation_file_and_external_input_config(tmpdir):
         ),
         annotation=dict(
             file="annotation.yaml"
-        )
+        ),
+        gpf_instance={
+            "path": fixture_dirname("")
+        }
     )
     config_fn = str(tmpdir / "import_config.yaml")
     with open(config_fn, "wt") as out_file:
diff --git a/dae/dae/import_tools/tests/test_import_project.py b/dae/dae/import_tools/tests/test_import_project.py
index b9bb5c309..1f124cedc 100644
--- a/dae/dae/import_tools/tests/test_import_project.py
+++ b/dae/dae/import_tools/tests/test_import_project.py
@@ -3,9 +3,12 @@ import cloudpickle  # type: ignore
 from dae.import_tools.import_tools import ImportProject
 
 
-def test_import_project_is_cpickle_serializable():
+def test_import_project_is_cpickle_serializable(fixture_dirname):
     import_config = dict(
         input={},
+        gpf_instance={
+            "path": fixture_dirname("")
+        }
     )
     project = ImportProject.build_from_config(import_config)
     _ = cloudpickle.dumps(project)
diff --git a/dae/dae/testing/__init__.py b/dae/dae/testing/__init__.py
index 3bc9678ee..80f6b9c51 100644
--- a/dae/dae/testing/__init__.py
+++ b/dae/dae/testing/__init__.py
@@ -4,6 +4,9 @@ from .setup_helpers import convert_to_tab_separated, setup_directories, \
     setup_gpf_instance
 from .import_helpers import vcf_import, vcf_study, \
     denovo_import, denovo_study, setup_dataset
+from .acgt_import import acgt_gpf
+from .alla_import import alla_gpf
+from .foobar_import import foobar_gpf
 
 
 __all__ = [
@@ -15,4 +18,8 @@ __all__ = [
     "vcf_import", "vcf_study",
     "denovo_import", "denovo_study",
     "setup_dataset",
+
+    "acgt_gpf",
+    "alla_gpf",
+    "foobar_gpf",
 ]
diff --git a/dae/dae/testing/acgt_import.py b/dae/dae/testing/acgt_import.py
new file mode 100644
index 000000000..09e280980
--- /dev/null
+++ b/dae/dae/testing/acgt_import.py
@@ -0,0 +1,28 @@
+# pylint: disable=W0621,C0114,C0116,W0212,W0613
+from dae.testing import \
+    setup_genome, setup_empty_gene_models, setup_gpf_instance
+
+
+def acgt_gpf(root_path, storage=None):
+    genome = setup_genome(
+        root_path / "acgt_gpf" / "genome" / "allChr.fa",
+        f"""
+        >chr1
+        {25 * "ACGT"}
+        >chr2
+        {25 * "ACGT"}
+        """
+    )
+
+    empty_gene_models = setup_empty_gene_models(
+        root_path / "acgt_gpf" / "empty_gene_models" / "empty_genes.txt")
+    gpf_instance = setup_gpf_instance(
+        root_path / "gpf_instance",
+        reference_genome=genome,
+        gene_models=empty_gene_models)
+
+    if storage:
+        gpf_instance\
+            .genotype_storages\
+            .register_default_storage(storage)
+    return gpf_instance
diff --git a/dae/dae/variants_loaders/dae/tests/test_add_chrom_denovo_loader.py b/dae/dae/variants_loaders/dae/tests/test_add_chrom_denovo_loader.py
new file mode 100644
index 000000000..3fc0b209f
--- /dev/null
+++ b/dae/dae/variants_loaders/dae/tests/test_add_chrom_denovo_loader.py
@@ -0,0 +1,71 @@
+# pylint: disable=W0621,C0114,C0116,W0212,W0613
+import pytest
+
+from dae.testing import setup_pedigree, setup_denovo, acgt_gpf
+from dae.pedigrees.loader import FamiliesLoader
+from dae.variants_loaders.dae.loader import DenovoLoader
+
+
+@pytest.fixture
+def trio_families(tmp_path_factory):
+    root_path = tmp_path_factory.mktemp(
+        "denovo_add_chrom_trio_families")
+    ped_path = setup_pedigree(
+        root_path / "trio_data" / "in.ped",
+        """
+        familyId personId dadId	 momId	sex status role
+        f1       m1       0      0      2   1      mom
+        f1       d1       0      0      1   1      dad
+        f1       p1       d1     m1     1   2      prb
+        """)
+    loader = FamiliesLoader(ped_path)
+    return loader.load()
+
+
+@pytest.fixture
+def trio_gpf(tmp_path_factory):
+    root_path = tmp_path_factory.mktemp(
+        "denovo_acgt_gpf_instance")
+    gpf_instance = acgt_gpf(root_path)
+    return gpf_instance
+
+
+@pytest.fixture
+def trio_denovo(tmp_path_factory):
+    root_path = tmp_path_factory.mktemp(
+        "denovo_add_chrom_trio")
+    denovo_path = setup_denovo(
+        root_path / "trio_data" / "in.tsv",
+        """
+          familyId  location  variant    bestState
+          f1        1:2       del(2)     2||2||1/0||0||1
+          f1        2:2       ins(AA)    2||2||1/0||0||1
+        """
+    )
+    return denovo_path
+
+
+@pytest.mark.parametrize(
+    "variant_index,chrom,pos,ref,alt",
+    [
+        (0, "chr1", 1, "ACG", "A"),
+        (1, "chr2", 1, "A", "AAA"),
+    ]
+)
+def test_add_chrom_denovo_loader(
+        trio_gpf, trio_families, trio_denovo,
+        variant_index, chrom, pos, ref, alt):
+
+    loader = DenovoLoader(
+        trio_families, str(trio_denovo), trio_gpf.reference_genome,
+        params={
+            "add_chrom_prefix": "chr",
+        })
+    alt_alleles = [
+        sv.alt_alleles[0] for sv, _ in loader.full_variants_iterator()]
+    aa = alt_alleles[variant_index]
+
+    assert aa.chrom == chrom
+    assert aa.position == pos
+    assert aa.reference == ref
+    assert aa.alternative == alt
diff --git a/dae/dae/variants_loaders/dae/tests/test_read_variants_from_dsv.py b/dae/dae/variants_loaders/dae/tests/test_read_variants_from_dsv.py
index f0e2a7d1d..f91f0443c 100644
--- a/dae/dae/variants_loaders/dae/tests/test_read_variants_from_dsv.py
+++ b/dae/dae/variants_loaders/dae/tests/test_read_variants_from_dsv.py
@@ -61,10 +61,14 @@ def test_produce_genotype_no_people_with_variants(
     assert output.dtype == GENOTYPE_TYPE
 
 
-def test_families_instance_type_assertion():
+def test_families_instance_type_assertion(
+        fixture_dirname, fake_families, gpf_instance_2013):
     error_message = "families must be an instance of FamiliesData!"
+    filename = fixture_dirname("denovo_import/variants_DAE_style.tsv")
+    loader = DenovoLoader(
+        fake_families, filename, gpf_instance_2013.reference_genome)
     with pytest.raises(AssertionError) as excinfo:
-        DenovoLoader.flexible_denovo_load(
+        loader.flexible_denovo_load(
             None,
             None,
             denovo_location="foo",
@@ -78,7 +82,9 @@ def test_families_instance_type_assertion():
 def test_read_variants_dae_style(
         gpf_instance_2013, fixture_dirname, fake_families):
     filename = fixture_dirname("denovo_import/variants_DAE_style.tsv")
-    res_df = DenovoLoader.flexible_denovo_load(
+    loader = DenovoLoader(
+        fake_families, filename, gpf_instance_2013.reference_genome)
+    res_df = loader.flexible_denovo_load(
         filename,
         gpf_instance_2013.reference_genome,
         families=fake_families,
@@ -113,7 +119,17 @@ def test_read_variants_a_la_vcf_style(
     gpf_instance_2013, fixture_dirname, fake_families
 ):
     filename = fixture_dirname("denovo_import/variants_VCF_style.tsv")
-    res_df = DenovoLoader.flexible_denovo_load(
+    loader = DenovoLoader(
+        fake_families, filename, gpf_instance_2013.reference_genome,
+        params={
+            "denovo_chrom": "chrom",
+            "denovo_pos": "pos",
+            "denovo_ref": "ref",
+            "denovo_alt": "alt",
+            "denovo_family_id": "familyId",
+            "denovo_best_state": "bestState",
+        })
+    res_df = loader.flexible_denovo_load(
         filename,
         gpf_instance_2013.reference_genome,
         families=fake_families,
@@ -149,7 +165,16 @@ def test_read_variants_a_la_vcf_style(
 def test_read_variants_mixed_a(
         gpf_instance_2013, fixture_dirname, fake_families):
     filename = fixture_dirname("denovo_import/variants_mixed_style_A.tsv")
-    res_df = DenovoLoader.flexible_denovo_load(
+    loader = DenovoLoader(
+        fake_families, filename, gpf_instance_2013.reference_genome,
+        params={
+            "denovo_location": "location",
+            "denovo_ref": "ref",
+            "denovo_alt": "alt",
+            "denovo_family_id": "familyId",
+            "denovo_best_state": "bestState",
+        })
+    res_df = loader.flexible_denovo_load(
         filename,
         gpf_instance_2013.reference_genome,
         families=fake_families,
@@ -184,7 +209,16 @@ def test_read_variants_mixed_a(
 def test_read_variants_mixed_b(
         gpf_instance_2013, fixture_dirname, fake_families):
     filename = fixture_dirname("denovo_import/variants_mixed_style_B.tsv")
-    res_df = DenovoLoader.flexible_denovo_load(
+    loader = DenovoLoader(
+        fake_families, filename, gpf_instance_2013.reference_genome,
+        params={
+            "denovo_chrom": "chrom",
+            "denovo_pos": "pos",
+            "denovo_variant": "variant",
+            "denovo_family_id": "familyId",
+            "denovo_best_state": "bestState",
+        })
+    res_df = loader.flexible_denovo_load(
         filename,
         gpf_instance_2013.reference_genome,
         families=fake_families,
@@ -227,7 +261,16 @@ def test_read_variants_person_ids(
     gpf_instance_2013, filename, fake_families, fixture_dirname
 ):
     filename = fixture_dirname(filename)
-    res_df = DenovoLoader.flexible_denovo_load(
+    loader = DenovoLoader(
+        fake_families, filename, gpf_instance_2013.reference_genome,
+        params={
+            "denovo_chrom": "chrom",
+            "denovo_pos": "pos",
+            "denovo_ref": "ref",
+            "denovo_alt": "alt",
+            "denovo_person_id": "personId",
+        })
+    res_df = loader.flexible_denovo_load(
         filename,
         gpf_instance_2013.reference_genome,
         families=fake_families,
@@ -276,7 +319,18 @@ def test_read_variants_different_separator(
     filename = fixture_dirname(
         "denovo_import/variants_different_separator.dsv"
     )
-    res_df = DenovoLoader.flexible_denovo_load(
+    loader = DenovoLoader(
+        fake_families, filename, gpf_instance_2013.reference_genome,
+        params={
+            "denovo_sep": "$",
+            "denovo_chrom": "chrom",
+            "denovo_pos": "pos",
+            "denovo_ref": "ref",
+            "denovo_alt": "alt",
+            "denovo_family_id": "familyId",
+            "denovo_best_state": "bestState",
+        })
+    res_df = loader.flexible_denovo_load(
         filename,
         gpf_instance_2013.reference_genome,
         families=fake_families,
@@ -316,7 +370,17 @@ def test_read_variants_with_genotype(
     filename = fixture_dirname(
         "denovo_import/variants_VCF_genotype.tsv"
     )
-    res_df = DenovoLoader.flexible_denovo_load(
+    loader = DenovoLoader(
+        fake_families, filename, gpf_instance_2013.reference_genome,
+        params={
+            "denovo_chrom": "chrom",
+            "denovo_pos": "pos",
+            "denovo_ref": "ref",
+            "denovo_alt": "alt",
+            "denovo_family_id": "familyId",
+            "denovo_genotype": "genotype",
+        })
+    res_df = loader.flexible_denovo_load(
         filename,
         gpf_instance_2013.reference_genome,
         families=fake_families,
@@ -355,11 +419,14 @@ def test_read_variants_with_genotype(
     assert compare_variant_dfs(res_df, expected_df)
 
 
-def test_read_variants_genome_assertion(fixture_dirname, fake_families):
+def test_read_variants_genome_assertion(
+        fixture_dirname, fake_families, gpf_instance_2013):
     filename = fixture_dirname("denovo_import/variants_DAE_style.tsv")
 
     with pytest.raises(AssertionError) as excinfo:
-        DenovoLoader.flexible_denovo_load(
+        loader = DenovoLoader(
+            fake_families, filename, gpf_instance_2013.reference_genome)
+        loader.flexible_denovo_load(
             filename,
             None,
             families=fake_families,
diff --git a/dae/dae/variants_loaders/vcf/tests/test_add_chrom_vcf_loader.py b/dae/dae/variants_loaders/vcf/tests/test_add_chrom_vcf_loader.py
new file mode 100644
index 000000000..f3eb54ab3
--- /dev/null
+++ b/dae/dae/variants_loaders/vcf/tests/test_add_chrom_vcf_loader.py
@@ -0,0 +1,83 @@
+# pylint: disable=W0621,C0114,C0116,W0212,W0613
+import pytest
+
+from dae.testing import setup_pedigree, setup_vcf, acgt_gpf
+from dae.pedigrees.loader import FamiliesLoader
+from dae.variants_loaders.vcf.loader import VcfLoader
+
+
+@pytest.fixture
+def trio_families(tmp_path_factory):
+    root_path = tmp_path_factory.mktemp(
+        "vcf_add_chrom_trio_families")
+    ped_path = setup_pedigree(
+        root_path / "trio_data" / "in.ped",
+        """
+        familyId personId dadId	 momId	sex status role
+        f1       m1       0      0      2   1      mom
+        f1       d1       0      0      1   1      dad
+        f1       p1       d1     m1     1   2      prb
+        """)
+    loader = FamiliesLoader(ped_path)
+    return loader.load()
+
+
+@pytest.fixture
+def trio_gpf(tmp_path_factory):
+    root_path = tmp_path_factory.mktemp(
+        "vcf_acgt_gpf_instance")
+    gpf_instance = acgt_gpf(root_path)
+    return gpf_instance
+
+
+@pytest.fixture
+def trio_vcf(tmp_path_factory):
+    root_path = tmp_path_factory.mktemp(
+        "vcf_add_chrom_trio")
+    vcf_path = setup_vcf(
+        root_path / "trio_data" / "in.vcf.gz",
+        """
+        ##fileformat=VCFv4.2
+        ##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
+        ##contig=<ID=1>
+        ##contig=<ID=2>
+        #CHROM POS ID REF ALT  QUAL FILTER INFO FORMAT m1  d1  p1
+        1      1   .  A   T    .    .      .    GT     0/1 0/0 0/1
+        1      2   .  C   T    .    .      .    GT     0/1 0/0 0/1
+        1      3   .  G   T    .    .      .    GT     0/1 0/0 0/1
+        1      5   .  A   T    .    .      .    GT     0/1 0/0 0/1
+        1      6   .  C   T    .    .      .    GT     0/1 0/0 0/1
+        1      7   .  G   T    .    .      .    GT     0/1 0/0 0/1
+        2      1   .  A   T    .    .      .    GT     0/1 0/0 0/1
+        2      2   .  C   T    .    .      .    GT     0/1 0/0 0/1
+        2      3   .  G   T    .    .      .    GT     0/1 0/0 0/1
+        2      5   .  A   T    .    .      .    GT     0/1 0/0 0/1
+        2      6   .  C   T    .    .      .    GT     0/1 0/0 0/1
+        2      7   .  G   T    .    .      .    GT     0/1 0/0 0/1
+        """)
+
+    return vcf_path
+
+
+@pytest.mark.parametrize(
+    "region,count",
+    [
+        ("chr1:5-8", 3),
+        ("chr2:1-4", 3),
+        ("chr2:5-8", 3),
+    ]
+)
+def test_add_chrom_reset_region(
+        trio_families, trio_gpf, trio_vcf,
+        region, count):
+
+    loader = VcfLoader(
+        trio_families, [str(trio_vcf)], trio_gpf.reference_genome,
+        params={
+            "add_chrom_prefix": "chr",
+        })
+    loader.reset_regions(region)
+    alt_alleles = [
+        sv.alt_alleles[0] for sv, _ in loader.full_variants_iterator()]
+
+    assert len(alt_alleles) == count