Repository: bihealth/biomedsheets

Issue:
Cryptic error message if sample sheet inconsistent
**Issue**
Inconsistent in sample sheet leads to cryptic error message while running `variant_calling`:
```
Error in rule variant_calling_write_pedigree_run:
    jobid: 0
    output: work/write_pedigree.16_0933_WGS-N1-DNA1-WGS1/out/16_0933_WGS-N1-DNA1-WGS1.ped

RuleException:
KeyError in line 67 of /path/to/snappy-pipeline/snappy_pipeline/workflows/variant_calling/Snakefile:
'9'
  File "/path/to/snappy-pipeline/snappy_pipeline/workflows/variant_calling/Snakefile", line 67, in __rule_variant_calling_write_pedigree_run
  File "/path/to/snappy-pipeline/snappy_pipeline/workflows/abstract/__init__.py", line 732, in substep_dispatch
  File "/path/to/snappy-pipeline/snappy_pipeline/workflows/abstract/__init__.py", line 203, in run
  File "/path/to/miniconda/lib/python3.8/site-packages/biomedsheets/shortcuts/germline.py", line 198, in write_pedigree_to_ped
  File "/path/to/miniconda/lib/python3.8/site-packages/biomedsheets/shortcuts/germline.py", line 171, in _append_pedigree_to_ped
  File "/path/to/miniconda/lib/python3.8/concurrent/futures/thread.py", line 57, in run
```

**To Reproduce**
Steps to reproduce the behavior:
- Assign pedigree based on a custom sample sheet column (_e.g._, 'familyId') in the config.
```
...
data_sets:
  test_cohort:
    sodar_uuid: ...
    file: 2022_test_cohort.tsv
    type: germline_variants
    pedigree_field: familyId
...
```
- Include conflicts in pedigree:
```
...
[Data]
familyId        patientName     fatherName      motherName      sex     isAffected      libraryType     folderName      batchNo hpoTerms        projectUuid     seqPlatform     libraryKit
FAM_P001 P001     P003     P002     F       Y       WGS     99999   5       .       ...    ILLUMINA        Illumina TruSeq PCR-free
FAM_P002 P002     0       0       F       N       WGS     99998  5       .       ...    ILLUMINA        Illumina TruSeq PCR-free
FAM_P003 P003     0       0       M       N       WGS     99997  5       .       ...    ILLUMINA        Illumina TruSeq PCR-free
```

**Possible Solution**
Implement consistency check with informative error message. Workflow shouldn't start.



Comments:

Patch:
diff --git a/biomedsheets/io_tsv/germline.py b/biomedsheets/io_tsv/germline.py
index 2214d46..9aa02c9 100644
--- a/biomedsheets/io_tsv/germline.py
+++ b/biomedsheets/io_tsv/germline.py
@@ -104,17 +104,7 @@ class GermlineTSVReader(BaseTSVReader):
     optional_body_header_columns = ('seqPlatform', 'bioSample', 'testSample')
 
     def check_tsv_line(self, mapping, lineno):
-        """Cancer sample sheet--specific valiation"""
-        # Check for hyphen in patient or sample name
-        if '-' in mapping['patientName']:
-            raise GermlineTSVSheetException(
-                'Hyphen not allowed in patientName column')  # pragma: no cover
-        if mapping['fatherName'] and '-' in mapping['fatherName']:
-            raise GermlineTSVSheetException(
-                'Hyphen not allowed in fatherName column')  # pragma: no cover
-        if mapping['motherName'] and '-' in mapping['motherName']:
-            raise GermlineTSVSheetException(
-                'Hyphen not allowed in motherName column')  # pragma: no cover
+        """Germline sample sheet--specific validation"""
         # Check "libraryType" field
         if mapping['libraryType'] and (
                 mapping['libraryType'] not in LIBRARY_TYPES):
diff --git a/biomedsheets/shortcuts/germline.py b/biomedsheets/shortcuts/germline.py
index 5ae32ed..67c5367 100644
--- a/biomedsheets/shortcuts/germline.py
+++ b/biomedsheets/shortcuts/germline.py
@@ -43,6 +43,18 @@ class UndefinedFieldException(Exception):
     defined in extra_infos."""
 
 
+class InconsistentPedigreeException(Exception):
+    """Raised if pedigree information from custom field is inconsistent with row definition.
+
+    Example for field 'familyId':
+        [Data]
+        familyId | patientName | fatherName | motherName | ...
+        family1  |   index1    |   father1  |   mother1  | ...
+        family2  |   father1   |     0      |     0      | ...
+        family3  |   mother1   |     0      |     0      | ...
+    """
+
+
 class Pedigree:
     """Class for accessing information in a pedigree
 
@@ -325,13 +337,27 @@ class CohortBuilder:
         """Return :py:class:`Cohort` object with :py:class:`Pedigree` sub
         structure
         """
+        error_msg = (
+            "Found inconsistent in input sample sheet. For index '{id_}' pedigree description from "
+            "row is not the same as the one found using custom join field '{join_by_field}'."
+        )
         cohort = Cohort(self._yield_pedigrees())
         for pedigree in cohort.pedigrees:
             for donor in pedigree.donors:
                 if donor.father_pk:
                     donor._father = cohort.pk_to_donor[int(donor.father_pk)]
+                    # Consistent check - it shouldn't be 'None' if pedigree correctly joint.
+                    if not pedigree.pk_to_donor.get(donor.father_pk, None):
+                        raise InconsistentPedigreeException(error_msg.format(
+                            id_=donor.bio_entity.secondary_id, join_by_field=self.join_by_field)
+                        )
                 if donor.mother_pk:
                     donor._mother = cohort.pk_to_donor[int(donor.mother_pk)]
+                    # Consistent check - it shouldn't be 'None' if pedigree correctly joint
+                    if not pedigree.pk_to_donor.get(donor.father_pk, None):
+                        raise InconsistentPedigreeException(error_msg.format(
+                            id_=donor.bio_entity.secondary_id, join_by_field=self.join_by_field)
+                        )
         return cohort
 
     def _yield_pedigrees(self):


Test patch:
diff --git a/tests/test_io_tsv_germline.py b/tests/test_io_tsv_germline.py
index 40e48c6..ace0084 100644
--- a/tests/test_io_tsv_germline.py
+++ b/tests/test_io_tsv_germline.py
@@ -45,6 +45,18 @@ def tsv_sheet_germline_no_header():
     return f
 
 
+@pytest.fixture
+def tsv_sheet_germline_no_header_hyphened_identifier():
+    """Germline TSV sheet without header"""
+    return io.StringIO(textwrap.dedent("""
+    patientName\tfatherName\tmotherName\tsex\tisAffected\tlibraryType\tfolderName\thpoTerms
+    12-345\t12-346\t12-347\tM\tY\tWGS\t12-345\tHP:0009946,HP:0009899
+    12-348\t12-346\t12-347\tM\tN\tWGS\t12-348\t.
+    12-346\t.\t.\tM\tN\t.\t.\t.
+    12-347\t.\t.\tF\tN\tWGS\t12-347\t.
+    """.lstrip()))
+
+
 @pytest.fixture
 def tsv_sheet_germline_platform_name():
     """Germline TSV sheet with seqPlatform name"""
@@ -510,6 +522,232 @@ EXPECTED_GERMLINE_SHEET_JSON_NO_HEADER = r"""
     }
 }""".lstrip()
 
+# Expected value for the germline sheet JSON without header and hyphen in identifiers
+EXPECTED_GERMLINE_SHEET_JSON_NO_HEADER_HYPHEN = r"""
+{
+    "identifier": "file://<unknown>",
+    "title": "Germline Sample Sheet",
+    "description": "Sample Sheet constructed from germline compact TSV file",
+    "extraInfoDefs": {
+        "bioEntity": {
+            "ncbiTaxon": {
+                "docs": "Reference to NCBI taxonomy",
+                "key": "taxon",
+                "type": "string",
+                "pattern": "^NCBITaxon_[1-9][0-9]*$"
+            },
+            "fatherPk": {
+                "docs": "Primary key of mother",
+                "key": "fatherPk",
+                "type": "string"
+            },
+            "motherPk": {
+                "docs": "Primary key of mother",
+                "key": "motherPk",
+                "type": "string"
+            },
+            "fatherName": {
+                "docs": "secondary_id of father, used for construction only",
+                "key": "fatherName",
+                "type": "string"
+            },
+            "motherName": {
+                "key": "motherName",
+                "docs": "secondary_id of mother, used for construction only",
+                "type": "string"
+            },
+            "sex": {
+                "docs": "Biological sex of individual",
+                "key": "sex",
+                "type": "enum",
+                "choices": [
+                    "male",
+                    "female",
+                    "unknown"
+                ]
+            },
+            "isAffected": {
+                "docs": "Flag for marking individiual as (un-)affected",
+                "key": "isAffected",
+                "type": "enum",
+                "choices": [
+                    "affected",
+                    "unaffected",
+                    "unknown"
+                ]
+            },
+            "hpoTerms": {
+                "docs": "HPO terms for individual",
+                "key": "hpoTerms",
+                "type": "array",
+                "entry": "string",
+                "pattern": "^HPO:[0-9]+$"
+            }
+        },
+        "bioSample": {},
+        "testSample": {
+            "extractionType": {
+                "docs": "Describes extracted",
+                "key": "extractionType",
+                "type": "enum",
+                "choices": [
+                    "DNA",
+                    "RNA",
+                    "other"
+                ]
+            }
+        },
+        "ngsLibrary": {
+            "seqPlatform": {
+                "docs": "Sequencing platform used",
+                "key": "kitName",
+                "type": "enum",
+                "choices": [
+                    "Illumina",
+                    "PacBio",
+                    "other"
+                ]
+            },
+            "libraryType": {
+                "docs": "Rough classificiation of the library type",
+                "key": "libraryType",
+                "type": "enum",
+                "choices": [
+                    "Panel-seq",
+                    "WES",
+                    "WGS",
+                    "mRNA-seq",
+                    "tRNA-seq",
+                    "other"
+                ]
+            },
+            "folderName": {
+                "docs": "Name of folder with FASTQ files",
+                "key": "folderName",
+                "type": "string"
+            }
+        }
+    },
+    "bioEntities": {
+        "12-345": {
+            "pk": 1,
+            "extraInfo": {
+                "fatherName": "12-346",
+                "motherName": "12-347",
+                "sex": "male",
+                "isAffected": "affected",
+                "hpoTerms": [
+                    "HP:0009946",
+                    "HP:0009899"
+                ],
+                "ncbiTaxon": "NCBITaxon_9606",
+                "fatherPk": 9,
+                "motherPk": 10
+            },
+            "bioSamples": {
+                "N1": {
+                    "pk": 2,
+                    "extraInfo": {},
+                    "testSamples": {
+                        "DNA1": {
+                            "pk": 3,
+                            "extraInfo": {
+                                "extractionType": "DNA"
+                            },
+                            "ngsLibraries": {
+                                "WGS1": {
+                                    "pk": 4,
+                                    "extraInfo": {
+                                        "seqPlatform": "Illumina",
+                                        "folderName": "12-345",
+                                        "libraryType": "WGS"
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        },
+        "12-348": {
+            "pk": 5,
+            "extraInfo": {
+                "fatherName": "12-346",
+                "motherName": "12-347",
+                "sex": "male",
+                "isAffected": "unaffected",
+                "ncbiTaxon": "NCBITaxon_9606",
+                "fatherPk": 9,
+                "motherPk": 10
+            },
+            "bioSamples": {
+                "N1": {
+                    "pk": 6,
+                    "extraInfo": {},
+                    "testSamples": {
+                        "DNA1": {
+                            "pk": 7,
+                            "extraInfo": {
+                                "extractionType": "DNA"
+                            },
+                            "ngsLibraries": {
+                                "WGS1": {
+                                    "pk": 8,
+                                    "extraInfo": {
+                                        "seqPlatform": "Illumina",
+                                        "folderName": "12-348",
+                                        "libraryType": "WGS"
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        },
+        "12-346": {
+            "pk": 9,
+            "extraInfo": {
+                "sex": "male",
+                "isAffected": "unaffected",
+                "ncbiTaxon": "NCBITaxon_9606"
+            },
+            "bioSamples": {}
+        },
+        "12-347": {
+            "pk": 10,
+            "extraInfo": {
+                "sex": "female",
+                "isAffected": "unaffected",
+                "ncbiTaxon": "NCBITaxon_9606"
+            },
+            "bioSamples": {
+                "N1": {
+                    "pk": 11,
+                    "extraInfo": {},
+                    "testSamples": {
+                        "DNA1": {
+                            "pk": 12,
+                            "extraInfo": {
+                                "extractionType": "DNA"
+                            },
+                            "ngsLibraries": {
+                                "WGS1": {
+                                    "pk": 13,
+                                    "extraInfo": {
+                                        "seqPlatform": "Illumina",
+                                        "folderName": "12-347",
+                                        "libraryType": "WGS"
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}""".lstrip()
 
 # Expected value when platform name is given
 EXPECTED_GERMLINE_SHEET_JSON_PLATFORM_NAME = r"""
@@ -673,6 +911,13 @@ def test_read_germline_sheet_no_header(tsv_sheet_germline_no_header):
     assert EXPECTED_GERMLINE_SHEET_JSON_NO_HEADER == json.dumps(
         sheet.json_data, indent='    ')
 
+def test_read_germline_sheet_no_header_hyphened_identifiers(
+    tsv_sheet_germline_no_header_hyphened_identifier
+):
+    sheet = io_tsv.read_germline_tsv_sheet(tsv_sheet_germline_no_header_hyphened_identifier)
+    assert EXPECTED_GERMLINE_SHEET_JSON_NO_HEADER_HYPHEN == json.dumps(
+        sheet.json_data, indent='    ')
+
 
 def test_read_germline_sheet_platform_name(tsv_sheet_germline_platform_name):
     sheet = io_tsv.read_germline_tsv_sheet(tsv_sheet_germline_platform_name)
diff --git a/tests/test_shortcuts_germline.py b/tests/test_shortcuts_germline.py
index 67510a6..ecc30f9 100644
--- a/tests/test_shortcuts_germline.py
+++ b/tests/test_shortcuts_germline.py
@@ -7,7 +7,7 @@ import textwrap
 import pytest
 
 from biomedsheets import io_tsv, naming, shortcuts
-from biomedsheets.shortcuts.germline import UndefinedFieldException
+from biomedsheets.shortcuts.germline import InconsistentPedigreeException, UndefinedFieldException
 
 __author__ = 'Manuel Holtgrewe <manuel.holtgrewe@bihealth.de>'
 
@@ -66,6 +66,31 @@ def tsv_sheet_germline_trio_plus():
     """.lstrip()))
     return f
 
+@pytest.fixture
+def tsv_sheet_germline_inconsistent_pedigree():
+    """Example TSV germline sheet with inconsistent pedigree definition.
+
+    :return: Returns StringIO with sample sheet for with inconsistent pedigree definition:
+    family identifier and row information don't agree.
+    """
+    return io.StringIO(textwrap.dedent("""
+    [Metadata]
+    schema\tgermline_variants
+    schema_version\tv1
+    title\tExample germline study
+    description\tSimple study with one trio plus
+
+    [Custom Fields]
+    key\tannotatedEntity\tdocs\ttype\tminimum\tmaximum\tunit\tchoices\tpattern
+    familyId\tbioEntity\tFamily\tstring\t.\t.\t.\t.\t.
+
+    [Data]
+    familyId\tpatientName\tfatherName\tmotherName\tsex\tisAffected\tlibraryType\tfolderName\thpoTerms
+    family1\tindex1\tfather1\tmother1\tM\tY\tWES\tindex1\t.
+    family2\tfather1\t0\t0\tM\tN\tWES\tfather1\t.
+    family3\tmother1\t0\t0\tM\tN\tWES\tmother1\t.
+    """.lstrip()))
+
 
 @pytest.fixture
 def sheet_germline_trio_plus(tsv_sheet_germline_trio_plus):
@@ -247,7 +272,7 @@ def test_undefined_field_exception():
 
 def test_sheet_germline_trio_plus_exception(tsv_sheet_germline_trio_plus):
     """Tests UndefinedFieldException raise while creating GermlineCaseSheet"""
-    with pytest.raises(Exception):
+    with pytest.raises(UndefinedFieldException):
         shortcuts.GermlineCaseSheet(
             sheet=io_tsv.read_germline_tsv_sheet(tsv_sheet_germline_trio_plus),
             join_by_field='undefined_field'
@@ -388,3 +413,21 @@ def test_cohorts(sheet_germline):
     assert set(cohort.secondary_id_to_donor) == {'index1', 'father1', 'mother2', 'index2', 'mother1', 'father2'}
     assert cohort.member_count == 6
     assert cohort.pedigree_count == 2
+
+
+def test_sheet_germline_inconsistent_pedigree(
+    tsv_sheet_germline_inconsistent_pedigree,
+    tsv_sheet_germline_trio_plus,
+):
+    """Tests Germline sheet for sheet with conflict information for joint field and row."""
+    # Sanity check
+    shortcuts.GermlineCaseSheet(
+        sheet=io_tsv.read_germline_tsv_sheet(tsv_sheet_germline_trio_plus),
+        join_by_field='familyId'
+    )
+    # Expect error as each member of the pedigree has its own `familyId` instead of a common one
+    with pytest.raises(InconsistentPedigreeException):
+        shortcuts.GermlineCaseSheet(
+            sheet=io_tsv.read_germline_tsv_sheet(tsv_sheet_germline_inconsistent_pedigree),
+            join_by_field='familyId'
+        )