Repository: Sage-Bionetworks/Genie

Issue:
Strip white space from Reference_Allele and TSA2.
Duplicated variants masked due to extra white space in the end

Discussion:

Patch:
diff --git a/genie_registry/maf.py b/genie_registry/maf.py
index e90e4eb..512c998 100644
--- a/genie_registry/maf.py
+++ b/genie_registry/maf.py
@@ -27,7 +27,7 @@ def _check_tsa1_tsa2(df):
         )
         if not (tsa1_eq_ref or tsa1_eq_tsa2):
             error = (
-                "Mutation File: Contains both "
+                "maf: Contains both "
                 "TUMOR_SEQ_ALLELE1 and TUMOR_SEQ_ALLELE2 columns. "
                 "The values in TUMOR_SEQ_ALLELE1 must be the same as "
                 "all the values in REFERENCE_ALELLE OR TUMOR_SEQ_ALLELE2."
@@ -55,7 +55,7 @@ def _check_allele_col(df, col):
         # CHECK: The value "NA" can't be used as a placeholder
         if sum(df[col].fillna('') == "NA") > 0:
             warning = (
-                "Mutation File: "
+                "maf: "
                 f"{col} column contains 'NA' values, "
                 "which cannot be placeholders for blank values.  "
                 "Please put in empty strings for blank values.\n"
@@ -63,7 +63,7 @@ def _check_allele_col(df, col):
         # CHECK: There can't be any null values
         if sum(df[col].isnull()) > 0:
             error = (
-                f"Mutation File: {col} can't have any blank or null values.\n"
+                f"maf: {col} can't have any blank or null values.\n"
             )
 
     return error, warning
@@ -128,8 +128,7 @@ class maf(FileTypeFormat):
         if not all([process_functions.checkColExist(mutationDF, i)
                     for i in correct_column_headers]):
             total_error += (
-                "Mutation File: "
-                "Must at least have these headers: {}. "
+                "maf: Must at least have these headers: {}. "
                 "If you are writing your maf file with R, please make"
                 "sure to specify the 'quote=FALSE' parameter.\n".format(
                     ",".join([i for i in correct_column_headers
@@ -137,23 +136,36 @@ class maf(FileTypeFormat):
         else:
             # CHECK: First column must be in the first_header list
             if mutationDF.columns[0] not in first_header:
-                total_error += ("Mutation File: First column header must be "
+                total_error += ("maf: First column header must be "
                                 "one of these: {}.\n".format(
                                     ", ".join(first_header)))
             # No duplicated values
             primary_cols = ['CHROMOSOME', 'START_POSITION',
                             'REFERENCE_ALLELE', 'TUMOR_SAMPLE_BARCODE',
                             'TUMOR_SEQ_ALLELE2']
-            if mutationDF.duplicated(primary_cols).any():
-                total_error += ("Mutation File: "
-                                "Should not have duplicate rows\n")
+            # Strip white space if string column
+            for col in primary_cols:
+                if mutationDF[col].dtype == object:
+                    mutationDF[col] = mutationDF[col].str.strip()
+            duplicated_idx = mutationDF.duplicated(primary_cols)
+            # Find samples with duplicated variants
+            duplicated_variants = mutationDF['TUMOR_SAMPLE_BARCODE'][
+                duplicated_idx
+            ].unique().tolist()
+
+            if duplicated_idx.any():
+                total_error += (
+                    "maf: Must not have duplicated variants. "
+                    "Samples with duplicated variants: "
+                    f"{', '.join(duplicated_variants)}\n"
+                )
 
         check_col = process_functions.checkColExist(mutationDF, "T_DEPTH")
         if not check_col and not SP:
             if not process_functions.checkColExist(mutationDF, "T_REF_COUNT"):
                 total_error += (
-                    "Mutation File: "
-                    "If you are missing T_DEPTH, you must have T_REF_COUNT!\n")
+                    "maf: If missing T_DEPTH, must have T_REF_COUNT!\n"
+                )
 
         # CHECK: Must have TUMOR_SEQ_ALLELE2
         error, warn = _check_allele_col(mutationDF, "TUMOR_SEQ_ALLELE2")
@@ -164,9 +176,8 @@ class maf(FileTypeFormat):
         if not all([process_functions.checkColExist(mutationDF, i)
                     for i in optional_headers]) and not SP:
             warning += (
-                "Mutation File: "
-                "Does not have the column headers that can give extra "
-                "information to the processed mutation file: {}.\n".format(
+                "maf: Does not have the column headers that can give extra "
+                "information to the processed maf: {}.\n".format(
                     ", ".join([
                         i for i in optional_headers
                         if i not in mutationDF.columns.values])))
@@ -184,8 +195,7 @@ class maf(FileTypeFormat):
                 for i in mutationDF['CHROMOSOME']]
             if sum(invalidValues) > 0:
                 total_error += (
-                    "Mutation File: "
-                    "CHROMOSOME column cannot have any values that "
+                    "maf: CHROMOSOME column cannot have any values that "
                     "start with 'chr' or any 'WT' values.\n")
 
         error = _check_tsa1_tsa2(mutationDF)
diff --git a/genie_registry/vcf.py b/genie_registry/vcf.py
index 65129fa..2c54853 100644
--- a/genie_registry/vcf.py
+++ b/genie_registry/vcf.py
@@ -65,21 +65,23 @@ class vcf(FileTypeFormat):
         total_error = ""
         warning = ""
         if not all(required_headers.isin(vcfdf.columns)):
-            total_error += ("Your vcf file must have these headers: "
+            total_error += ("vcf: Must have these headers: "
                             "CHROM, POS, ID, REF, ALT, QUAL, FILTER, INFO.\n")
         else:
             # No duplicated values
             primary_cols = ["#CHROM", "POS", "REF", "ALT"]
             if vcfdf.duplicated(primary_cols).any():
-                total_error += "Your vcf file should not have duplicate rows\n"
+                total_error += (
+                    "vcf: Must not have duplicate variants.\n"
+                )
 
             if vcfdf[['#CHROM', 'POS']].isnull().values.any():
-                total_error += ("Your vcf file may contain rows that are "
+                total_error += ("vcf: May contain rows that are "
                                 "space delimited instead of tab delimited.\n")
 
         if len(vcfdf.columns) > 8:
             if "FORMAT" not in vcfdf.columns:
-                total_error += ("Your vcf file must have FORMAT header "
+                total_error += ("vcf: Must have FORMAT header "
                                 "if genotype columns exist.\n")
 
         # Require that they report variants mapped to
@@ -89,15 +91,15 @@ class vcf(FileTypeFormat):
         if have_column:
             nochr = ["chr" in i for i in vcfdf['#CHROM'] if isinstance(i, str)]
             if sum(nochr) > 0:
-                warning += ("Your vcf file should not have the chr prefix "
+                warning += ("vcf: Should not have the chr prefix "
                             "in front of chromosomes.\n")
             if sum(vcfdf['#CHROM'].isin(["chrM"])) > 0:
-                total_error += "Your vcf file must not have variants on chrM.\n"
+                total_error += "vcf: Must not have variants on chrM.\n"
 
         # No white spaces
         white_space = vcfdf.apply(lambda x: contains_whitespace(x), axis=1)
         if sum(white_space) > 0:
-            warning += ("Your vcf file should not have any "
+            warning += ("vcf: Should not have any "
                         "white spaces in any of the columns.\n")
 
         # I can also recommend a `bcftools query` command that


Test patch:
diff --git a/tests/test_maf.py b/tests/test_maf.py
index f1a2cf1..3b41f68 100644
--- a/tests/test_maf.py
+++ b/tests/test_maf.py
@@ -66,7 +66,7 @@ def test_firstcolumn_validation():
              'T_REF_COUNT', 'N_DEPTH', 'N_REF_COUNT', 'N_ALT_COUNT',
              'TUMOR_SEQ_ALLELE2']
     error, warning = maf_class._validate(mafDf[order])
-    expectedErrors = ("Mutation File: First column header must be "
+    expectedErrors = ("maf: First column header must be "
                       "one of these: CHROMOSOME, HUGO_SYMBOL, "
                       "TUMOR_SAMPLE_BARCODE.\n")
     assert error == expectedErrors
@@ -77,17 +77,16 @@ def test_missingcols_validation():
     """Tests missing columns"""
     emptydf = pd.DataFrame()
     error, warning = maf_class._validate(emptydf)
-    expected_errors = ("Mutation File: Must at least have these headers: "
+    expected_errors = ("maf: Must at least have these headers: "
                        "CHROMOSOME,START_POSITION,REFERENCE_ALLELE,"
                        "TUMOR_SAMPLE_BARCODE,T_ALT_COUNT,"
                        "TUMOR_SEQ_ALLELE2. "
                        "If you are writing your maf file with R, please make"
                        "sure to specify the 'quote=FALSE' parameter.\n"
-                       "Mutation File: If you are missing T_DEPTH, "
-                       "you must have T_REF_COUNT!\n")
-    expected_warnings = ("Mutation File: Does not have the column headers "
+                       "maf: If missing T_DEPTH, must have T_REF_COUNT!\n")
+    expected_warnings = ("maf: Does not have the column headers "
                          "that can give extra information to the processed "
-                         "mutation file: T_REF_COUNT, N_DEPTH, N_REF_COUNT, "
+                         "maf: T_REF_COUNT, N_DEPTH, N_REF_COUNT, "
                          "N_ALT_COUNT.\n")
     assert error == expected_errors
     assert warning == expected_warnings
@@ -108,17 +107,17 @@ def test_errors_validation():
     error, warning = maf_class._validate(mafDf)
 
     expectedErrors = (
-        "Mutation File: "
+        "maf: "
         "REFERENCE_ALLELE can't have any blank or null values.\n"
-        "Mutation File: "
+        "maf: "
         "CHROMOSOME column cannot have any values that start "
         "with 'chr' or any 'WT' values.\n"
     )
-    expectedWarnings = ("Mutation File: "
+    expectedWarnings = ("maf: "
                         "Does not have the column headers that can give "
-                        "extra information to the processed mutation file: "
+                        "extra information to the processed maf: "
                         "T_REF_COUNT, N_DEPTH.\n"
-                        "Mutation File: "
+                        "maf: "
                         "REFERENCE_ALLELE column contains 'NA' values, "
                         "which cannot be placeholders for blank values.  "
                         "Please put in empty strings for blank values.\n")
@@ -132,30 +131,32 @@ def test_invalid_validation():
         CHROMOSOME=[1, 2, 3, 4, 2, 4],
         T_ALT_COUNT=[1, 2, 3, 4, 3, 4],
         START_POSITION=[1, 2, 3, 4, 2, 4],
-        REFERENCE_ALLELE=["A", "A", "A", "A", "A", "A"],
-        TUMOR_SAMPLE_BARCODE=["ID1-1", "ID1-1", "ID1-1", "ID1-1", "ID1-1", "ID1-1"],
+        REFERENCE_ALLELE=["A ", "A ", "A", "A ", "A ", " A"],
+        TUMOR_SAMPLE_BARCODE=["ID1-1", "ID1-1", "ID1-1",
+                              "ID1-1", "ID1-1", "ID1-1"],
         N_DEPTH=[1, 2, 3, 4, 3, 4],
         N_REF_COUNT=[1, 2, 3, 4, 3, 4],
         N_ALT_COUNT=[1, 2, 3, 4, 3, 4],
-        TUMOR_SEQ_ALLELE2=["NA", float('nan'), "A", "A", "A", "A"]))
+        TUMOR_SEQ_ALLELE2=["NA", float('nan'), " A", "A ", " A", " A"]))
 
     with patch.object(genie_registry.maf, "_check_tsa1_tsa2",
                       return_value="") as check_tsa1_tsa2:
         error, warning = maf_class._validate(mafDf)
         check_tsa1_tsa2.assert_called_once_with(mafDf)
     expectedErrors = (
-        "Mutation File: Should not have duplicate rows\n"
-        "Mutation File: "
-        "If you are missing T_DEPTH, you must have T_REF_COUNT!\n"
-        "Mutation File: "
+        "maf: Must not have duplicated variants. "
+        "Samples with duplicated variants: ID1-1\n"
+        "maf: "
+        "If missing T_DEPTH, must have T_REF_COUNT!\n"
+        "maf: "
         "TUMOR_SEQ_ALLELE2 can't have any blank or null values.\n"
     )
     expectedWarnings = (
-        "Mutation File: TUMOR_SEQ_ALLELE2 column contains 'NA' values, "
+        "maf: TUMOR_SEQ_ALLELE2 column contains 'NA' values, "
         "which cannot be placeholders for blank values.  "
         "Please put in empty strings for blank values.\n"
-        "Mutation File: Does not have the column headers that can give "
-        "extra information to the processed mutation file: T_REF_COUNT.\n")
+        "maf: Does not have the column headers that can give "
+        "extra information to the processed maf: T_REF_COUNT.\n")
     assert error == expectedErrors
     assert warning == expectedWarnings
 
@@ -179,7 +180,7 @@ def test_warning__check_allele_col():
     error, warning = genie_registry.maf._check_allele_col(df, "TEMP")
     assert error == ""
     assert warning == (
-        "Mutation File: "
+        "maf: "
         "TEMP column contains 'NA' values, "
         "which cannot be placeholders for blank values.  "
         "Please put in empty strings for blank values.\n"
@@ -193,7 +194,7 @@ def test_error__check_allele_col():
     ))
     error, warning = genie_registry.maf._check_allele_col(df, "TEMP")
     assert error == (
-        "Mutation File: TEMP can't have any blank or null values.\n"
+        "maf: TEMP can't have any blank or null values.\n"
     )
     assert warning == ""
 
@@ -207,7 +208,7 @@ def test_invalid__check_tsa1_tsa2():
     ))
     error = genie_registry.maf._check_tsa1_tsa2(df)
     assert error == (
-        "Mutation File: Contains both "
+        "maf: Contains both "
         "TUMOR_SEQ_ALLELE1 and TUMOR_SEQ_ALLELE2 columns. "
         "The values in TUMOR_SEQ_ALLELE1 must be the same as "
         "all the values in REFERENCE_ALELLE OR TUMOR_SEQ_ALLELE2."
diff --git a/tests/test_vcf.py b/tests/test_vcf.py
index 1743994..94fdc37 100644
--- a/tests/test_vcf.py
+++ b/tests/test_vcf.py
@@ -51,11 +51,11 @@ def test_validation():
 
     error, warning = vcfClass._validate(vcfDf)
     expectedError = (
-        "Your vcf file must have these headers: CHROM, POS, ID, REF, "
+        "vcf: Must have these headers: CHROM, POS, ID, REF, "
         "ALT, QUAL, FILTER, INFO.\n"
-        "Your vcf file must have FORMAT header if genotype columns exist.\n")
+        "vcf: Must have FORMAT header if genotype columns exist.\n")
     expectedWarning = (
-        "Your vcf file should not have any white spaces "
+        "vcf: Should not have any white spaces "
         "in any of the columns.\n")
     assert error == expectedError
     assert warning == expectedWarning
@@ -70,11 +70,11 @@ def test_validation():
                           "INFO": ['AAK1', 'AAED1', 'AAAS', 'AAK1']})
 
     error, warning = vcfClass._validate(vcfDf)
-    expectedError = ("Your vcf file should not have duplicate rows\n"
-                     "Your vcf file may contain rows that are "
+    expectedError = ("vcf: Must not have duplicate variants.\n"
+                     "vcf: May contain rows that are "
                      "space delimited instead of tab delimited.\n"
-                     "Your vcf file must not have variants on chrM.\n")
-    expectedWarning = ("Your vcf file should not have the chr prefix "
+                     "vcf: Must not have variants on chrM.\n")
+    expectedWarning = ("vcf: Should not have the chr prefix "
                        "in front of chromosomes.\n")
     assert error == expectedError
     assert warning == expectedWarning