Repository: kids-first/kf-lib-data-ingest

Issue:
Optionally exclude tests in validation report
I would like a way to exclude relevant test sections from the validation report (e.g. only display relationship tests and count tests).

The Data Tracker is using the ingest library's data validator and reporting functionality for validating uploaded files and displaying results to the user. However, only the relationship test results are useful to display right now since the attribute tests rely on a set of hardcoded rules tailored to the Kids First program. In the future, we will be able to build the attribute test rules (e.g. Gender must be one of "Female, Male, Other") from the organization's template definitions and pass those into the validator. But this will require some refactoring in the data validator and some time to implement in Data Tracker.

It would be great if in the interim we can simply turn off the display of certain test sections in the report.

Discussion:

Patch:
diff --git a/kf_lib_data_ingest/validation/reporting/markdown.py b/kf_lib_data_ingest/validation/reporting/markdown.py
index 3ed8e7f..d7a3d43 100644
--- a/kf_lib_data_ingest/validation/reporting/markdown.py
+++ b/kf_lib_data_ingest/validation/reporting/markdown.py
@@ -53,7 +53,7 @@ class MarkdownReportBuilder(AbstractReportBuilder):
         """
         super().__init__(output_dir=output_dir, setup_logger=setup_logger)
 
-    def _build(self, results, title=DEFAULT_REPORT_TITLE):
+    def _build(self, results, title=DEFAULT_REPORT_TITLE, exclude_tests=None):
         """
         Build markdown content for validation report
 
@@ -62,6 +62,7 @@ class MarkdownReportBuilder(AbstractReportBuilder):
         :param title: Report title
         :type title: str
         """
+        exclude_tests = exclude_tests or []
         self.counts = results["counts"]
         self.files_validated = results["files_validated"]
         self.results = results["validation"]
@@ -72,14 +73,13 @@ class MarkdownReportBuilder(AbstractReportBuilder):
         output.append(self._files_md(results))
         output.append("\n## #️⃣ Counts")
         output.append(self._counts_md(results))
-        output.append("\n## 🚦 Relationship Tests")
-        output.append(self._tests_section_md(results, REL_TEST))
-        output.append("\n## 🚦 Gap Tests")
-        output.append(self._tests_section_md(results, GAP_TEST))
-        output.append("\n## 🚦 Attribute Value Tests")
-        output.append(self._tests_section_md(results, ATTR_TEST))
-        output.append("\n## 🚦 Count Tests")
-        output.append(self._tests_section_md(results, "count"))
+
+        tests = [REL_TEST, GAP_TEST, ATTR_TEST, COUNT_TEST]
+        for test_type in tests:
+            if test_type in exclude_tests:
+                continue
+            output.append(f"\n## 🚦 {test_type.title()} Tests")
+            output.append(self._tests_section_md(results, test_type))
         output = "\n".join(output)
 
         return output


Test patch:
diff --git a/tests/test_validation.py b/tests/test_validation.py
index 3739ae4..17272e2 100644
--- a/tests/test_validation.py
+++ b/tests/test_validation.py
@@ -15,6 +15,14 @@ from kf_lib_data_ingest.validation.data_validator import (
     NA,
     Validator as DataValidator,
 )
+from kf_lib_data_ingest.validation.reporting.markdown import (
+    MarkdownReportBuilder,
+    REL_TEST,
+    GAP_TEST,
+    ATTR_TEST,
+    COUNT_TEST,
+)
+
 from kf_lib_data_ingest.validation.default_hierarchy import DH
 from kf_lib_data_ingest.validation.validation import Validator
 from kf_lib_data_ingest.validation.reporting import sample_validation_results
@@ -142,13 +150,15 @@ def test_build_reports(tmpdir, info_caplog):
     """
     Test validation report building in kf_lib_data_ingest.validation.reporting
     """
+    results = sample_validation_results.results
+
     # Test format of validation results
     _validate_result_schema(sample_validation_results.results)
 
     # Test normal case with sample validation results
     path = tmpdir.mkdir("validation_results")
     report_paths = Validator(output_dir=path)._build_report(
-        sample_validation_results.results, formats={"tsv", "md", "foobar"}
+        results, formats={"tsv", "md", "foobar"}
     )
     # Unsupported report format
     assert "`foobar` not supported" in info_caplog.text
@@ -168,6 +178,19 @@ def test_build_reports(tmpdir, info_caplog):
                 assert False
             assert (df is not None) and (not df.empty)
 
+    # Test exclude_tests kwarg
+    # Default - all tests
+    b = MarkdownReportBuilder()
+    report = b._build(results)
+    tests = [REL_TEST, GAP_TEST, ATTR_TEST, COUNT_TEST]
+    for t in tests:
+        assert t.title() + " Tests" in report
+
+    # Exclude some tests
+    report = b._build(results, exclude_tests=tests[1:])
+    for t in tests[1:]:
+        assert t.title() + " Tests" not in report
+
 
 def _validate_result_schema(results):
     """