Repository: ministryofjustice/find-moj-data
Pull request #1469: fmd-1414: Add Quality Assertions and logic to Parse into models:
Author: login: hjribeiro-moj
Created: 2025-03-10T10:44:57Z | Merged: 2025-03-20T09:56:09Z

  - Update grapqhql dataset query to get assertions from datahub
  - Create ColumnAssertions object and updated ColumnObject
  - Updated Parser to parse assertions and add it to columns


Diff:
diff --git a/datahub_client/entities.py b/datahub_client/entities.py
index 1b383c18c..0a31318ac 100644
--- a/datahub_client/entities.py
+++ b/datahub_client/entities.py
@@ -115,6 +115,41 @@ class FindMoJdataEntityMapper:
 ]
 
 
+class ColumnAssertionType(Enum):
+    COMPLETENESS = "completeness"
+    CONSISTENCY = "consistency"
+    VALIDITY = "validity"
+    ACCURACY = "accuracy"
+    UNIQUENESS = "uniqueness"
+
+
+class ColumnAssertionLevel(Enum):
+    RED = "poor"
+    AMBER = "acceptable"
+    GREEN = "good"
+    NA = "na"
+
+
+class ColumnAssertionResult(Enum):
+    SUCCESS = "SUCCESS"
+    FAILURE = "FAILURE"
+    NA = "N/A"
+
+
+class ColumnAssertion(BaseModel):
+    type: ColumnAssertionType
+    level: ColumnAssertionLevel
+    result: ColumnAssertionResult
+
+
+class ColumnQualityMetrics(BaseModel):
+    completeness: str = Field(default="na", description="Completeness level")
+    consistency: str = Field(default="na", description="Consistency level")
+    accuracy: str = Field(default="na", description="Accuracy level")
+    uniqueness: str = Field(default="na", description="Uniqueness level")
+    validity: str = Field(default="na", description="Validity level")
+
+
 class SecurityClassification(Enum):
     """Enumeration representing the security classification of the data.
     Attributes:
@@ -211,6 +246,9 @@ class Column(BaseModel):
     foreign_keys: list[ColumnRef] = Field(
         description="References to columns in other tables", default_factory=list
     )
+    quality_metrics: ColumnQualityMetrics = Field(
+        description="List of Assertions associated with this column",
+    )
 
 
 class OwnerRef(BaseModel):
@@ -666,7 +704,8 @@ class Table(Entity):
                     description="unique ID for custody",
                     nullable=False,
                     is_primary_key=True,
-                ),
+                    quality_metrics=ColumnQualityMetrics()
+                )
             ]
         ],
     )
diff --git a/datahub_client/graphql/getDatasetDetails.graphql b/datahub_client/graphql/getDatasetDetails.graphql
index 5029d0b9c..3255f7797 100644
--- a/datahub_client/graphql/getDatasetDetails.graphql
+++ b/datahub_client/graphql/getDatasetDetails.graphql
@@ -108,6 +108,37 @@ query getDatasetDetails($urn: String!) {
     glossaryTerms {
       ...glossaryTermFields
     }
+
+    # Adding quality test results from Assertions
+    assertions {
+      total
+      assertions {
+        urn
+        type
+        info {
+          description
+          datasetAssertion{
+            nativeType
+            nativeParameters{
+              key
+              value
+            }
+          }
+        }
+        runEvents {
+          runEvents{
+            timestampMillis
+            lastObservedMillis
+              result{
+                type
+              }
+          }
+
+
+        }
+      }
+    }
+
     lastIngested
     schemaMetadata {
       fields {
diff --git a/datahub_client/parsers.py b/datahub_client/parsers.py
index d3ff5db73..0c0e543dc 100644
--- a/datahub_client/parsers.py
+++ b/datahub_client/parsers.py
@@ -8,6 +8,8 @@
     Chart,
     ChartEntityMapping,
     Column,
+    ColumnAssertion,
+    ColumnAssertionType,
     ColumnRef,
     CustomEntityProperties,
     Dashboard,
@@ -35,7 +37,7 @@
     Table,
     TableEntityMapping,
     TagRef,
-    UsageRestrictions,
+    UsageRestrictions, ColumnQualityMetrics,
 )
 from datahub_client.search.search_types import SearchResult
 
@@ -47,6 +49,39 @@
 DATA_CUSTODIAN = "urn:li:ownershipType:data_custodian"
 
 
+def parse_assertions(assertions: dict) -> dict[str, ColumnAssertion]:
+    assertions_map = {}
+    if assertions.get("total", 0) > 0:
+        for assertion in assertions.get("assertions", []):
+            display_name = assertion["info"]["datasetAssertion"]["nativeType"]
+            if display_name.startswith("column_completeness_"):
+                assertion_type = ColumnAssertionType.COMPLETENESS
+                assertion_level = assertion["info"]["datasetAssertion"]["nativeType"].split("column_completeness_")[1]
+                assertion_level = assertion_level.split("_property")[0]
+            elif display_name.startswith("consistency"):
+                assertion_type = ColumnAssertionType.CONSISTENCY
+                assertion_level = assertion["info"]["datasetAssertion"]["nativeType"].split("consistency_")[1]
+                assertion_level = assertion_level.split("_property")[0]
+            else:
+                continue
+            try:
+                result = assertion["runEvents"]["runEvents"][0]["result"]["type"]
+            except KeyError as ke:
+                logger.info(f"Skipping assertion, with KeyError {ke=}")
+                continue
+            except IndexError as ie:
+                logger.info(f"Skipping assertion, with IndexError {ie=}")
+                logger.info(assertion)
+                continue
+
+            for parameter in assertion["info"]["datasetAssertion"]["nativeParameters"]:
+                if parameter["key"] == "column_name":
+                    column_name = parameter["value"]
+                    assertions_map.setdefault(column_name, {}).setdefault(assertion_type, {})[assertion_level] = result
+
+    return assertions_map
+
+
 class EntityParser:
 
     def parse(self, search_response) -> SearchResult:
@@ -353,15 +388,14 @@ def parse_columns(self, entity: dict[str, Any]) -> list[Column]:
                 )
             )
 
+        all_column_assertions = parse_assertions(entity.get("assertions", {}))
         for field in schema_metadata.get("fields", ()):
             foreign_keys_for_field = foreign_keys[field["fieldPath"]]
 
-            # Work out if the field is primary.
-            # This is an oversimplification: in the case of a composite
-            # primary key, we report that each component field is primary.
             is_primary_key = field["fieldPath"] in primary_keys
             field_path = field["fieldPath"]
             display_name = field_path.split(".")[-1]
+            quality_metrics = self.form_column_quality_metrics(all_column_assertions, display_name)
 
             result.append(
                 Column(
@@ -372,12 +406,28 @@ def parse_columns(self, entity: dict[str, Any]) -> list[Column]:
                     nullable=field["nullable"],
                     is_primary_key=is_primary_key,
                     foreign_keys=foreign_keys_for_field,
+                    quality_metrics=quality_metrics,
                 )
             )
 
         # Sort primary keys first, then sort alphabetically
         return sorted(result, key=lambda c: (0 if c.is_primary_key else 1, c.name))
 
+    def form_column_quality_metrics(self, all_column_assertions, display_name):
+        column_assertions = all_column_assertions.get(display_name, {})
+        PRIORITY_ORDER = ["green", "amber", "red"]
+        level_to_description_map = {"green": "good", "amber": "acceptable", "red": "poor"}
+        quality_dict = {metric.value: "na" for metric in ColumnAssertionType}
+        # Iterate through assertions and update dictionary
+        for metric, levels in column_assertions.items():
+            quality_dict[metric.value] = "poor" if levels else "na"
+            for level in PRIORITY_ORDER:
+                if level in levels and levels[level] == "SUCCESS":
+                    quality_dict[metric.value] = level_to_description_map[level]
+                    break
+        quality_metrics = ColumnQualityMetrics(**quality_dict)
+        return quality_metrics
+
     def _parse_owners_by_type(
         self,
         entity: dict[str, Any],
diff --git a/scss/base.scss b/scss/base.scss
index 50f6250f9..5e0121a37 100644
--- a/scss/base.scss
+++ b/scss/base.scss
@@ -96,3 +96,33 @@ code {
   @extend .govuk-list;
   @extend .govuk-list--number;
 }
+
+/* Custom GOV.UK style status tags */
+.govuk-tag-status {
+  display: inline-block;
+  padding: 5px 10px;
+  font-weight: bold;
+  border-radius: 3px;
+  background-color: #e0e0e0; /* Default light gray */
+  color: #333; /* Default dark text */
+}
+
+.govuk-tag-status.good {
+  background-color: #d8ebe6; /* Light green */
+  color: #00703c; /* Dark green */
+}
+
+.govuk-tag-status.acceptable {
+  background-color: #fff1d2; /* Light amber */
+  color: #ffbf47; /* Dark amber */
+}
+
+.govuk-tag-status.poor {
+  background-color: #f3d3d3; /* Light red */
+  color: #d4351c; /* Dark red */
+}
+
+.govuk-tag-status.na {
+  background-color: #f3f2f1; /* Light gray */
+  color: #505a5f; /* Dark gray */
+}
\ No newline at end of file
diff --git a/templates/details_table.html b/templates/details_table.html
index 957517f94..11e6eca6e 100644
--- a/templates/details_table.html
+++ b/templates/details_table.html
@@ -9,6 +9,7 @@
       {% if entity.custom_properties.security_classification == "Official-Sensitive" %}
         {% include "partial/access_ap_data.html" with access_requirements=entity.custom_properties.access_information.dc_access_requirements is_access_requirements_a_url=is_access_requirements_a_url entity_name=entity.name %}
       {% endif %}
+      
       {% if entity.column_details %}
         <div class="govuk-heading-m">Table schema</div>
         {% include "partial/download_button.html" with entity_type='table' title="table descriptions" %}
@@ -22,12 +23,19 @@
               {% switch 'show_is_nullable_in_table_details_column' %}
                 <th scope="col" class="govuk-table__header">Is nullable</th>
               {% endswitch %}
+              {% if entity.name == "all_assets" %}
+                <th scope="col" class="govuk-table__header">Completeness</th>
+                <th scope="col" class="govuk-table__header">Accuracy</th>
+                <th scope="col" class="govuk-table__header">Uniqueness</th>
+                <th scope="col" class="govuk-table__header">Consistency</th>
+                <th scope="col" class="govuk-table__header">Validity</th>
+              {% endif %}
             </tr>
           </thead>
           <tbody class="govuk-table__body app-table-responsive__body">
             {% for column in entity.column_details %}
               <tr class="govuk-table__row app-table-responsive__row">
-                <td class="govuk-table__cell"><span class="app-table-responsive__heading">Column name:</span>{{column.display_name}}</td>
+                <td class="govuk-table__cell"><span class="app-table-responsive__heading">Column name:</span>{{ column.display_name }}</td>
                 <td class="govuk-table__cell column-description"><span class="app-table-responsive__heading">Description:</span>
                 {% if column.description %}
                 <div>
@@ -41,6 +49,7 @@
                 {% switch 'show_is_nullable_in_table_details_column' %}
                   <td class="govuk-table__cell"><span class="app-table-responsive__heading">Is nullable:</span>{{column.nullable|yesno:"Yes,No,"}}</td>
                 {% endswitch %}
+                {% include "partial/_specific_table_metrics.html" %}
               </tr>
             {% endfor %}
           </tbody>
@@ -50,6 +59,7 @@
         <h2 class="govuk-heading-m">Table schema</h2>
         <p class="govuk-body">The schema for this table is not available.</p>
       {% endif %}
+      
       {% if has_lineage %}
         <h2 class="govuk-heading-m">Lineage</h2>
         <div class="govuk-body-m" >
@@ -61,5 +71,7 @@ <h2 class="govuk-heading-m">Lineage</h2>
           </a>
         </div>
       {% endif %}
+      
     </div>
+  </div>
 {% endblock extra_details %}
diff --git a/templates/partial/_specific_table_metrics.html b/templates/partial/_specific_table_metrics.html
new file mode 100644
index 000000000..71441c2db
--- /dev/null
+++ b/templates/partial/_specific_table_metrics.html
@@ -0,0 +1,28 @@
+{% if entity.name == "all_assets" %}
+      <td class="govuk-table__cell">
+        <span class="govuk-tag-status {{ column.quality_metrics.completeness|default:'na' }}">
+          {{ column.quality_metrics.completeness|default:"na" }}
+        </span>
+      </td>
+      <td class="govuk-table__cell">
+        <span class="govuk-tag-status {{ column.quality_metrics.accuracy|default:'na' }}">
+          {{ column.quality_metrics.accuracy|default:"na" }}
+        </span>
+      </td>
+      <td class="govuk-table__cell">
+        <span class="govuk-tag-status {{ column.quality_metrics.uniqueness|default:'na' }}">
+          {{ column.quality_metrics.uniqueness|default:"na" }}
+        </span>
+      </td>
+      <td class="govuk-table__cell">
+        <span class="govuk-tag-status {{ column.quality_metrics.consistency|default:'na' }}">
+          {{ column.quality_metrics.consistency|default:"na" }}
+        </span>
+      </td>
+      <td class="govuk-table__cell">
+        <span class="govuk-tag-status {{ column.quality_metrics.validity|default:'na' }}">
+          {{ column.quality_metrics.validity|default:"na" }}
+        </span>
+      </td>
+    </tr>
+{% endif %}
diff --git a/tests/conftest.py b/tests/conftest.py
index 1a2370d10..5fa849f6f 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -38,7 +38,7 @@
     RelationshipType,
     Table,
     TableEntityMapping,
-    TagRef,
+    TagRef, ColumnQualityMetrics,
 )
 from datahub_client.search.search_types import (
     SearchResponse,
@@ -439,6 +439,7 @@ def generate_table_metadata(
                 description=column_description,
                 nullable=False,
                 is_primary_key=True,
+                quality_metrics=ColumnQualityMetrics(),
                 foreign_keys=[
                     ColumnRef(
                         name="urn",
diff --git a/tests/datahub_client/test_datahub_client.py b/tests/datahub_client/test_datahub_client.py
index f1e42e824..0672ee133 100644
--- a/tests/datahub_client/test_datahub_client.py
+++ b/tests/datahub_client/test_datahub_client.py
@@ -22,7 +22,7 @@
     SecurityClassification,
     Table,
     TagRef,
-    UsageRestrictions,
+    UsageRestrictions, ColumnQualityMetrics,
 )
 from datahub_client.exceptions import EntityDoesNotExist
 
@@ -393,6 +393,7 @@ def test_get_dataset(
                     description="The primary identifier for the dataset entity.",
                     nullable=False,
                     is_primary_key=True,
+                    quality_metrics=ColumnQualityMetrics(),
                     foreign_keys=[
                         ColumnRef(
                             name="urn",
diff --git a/tests/datahub_client/test_parsers.py b/tests/datahub_client/test_parsers.py
index 51b5983f9..d8e83ef80 100644
--- a/tests/datahub_client/test_parsers.py
+++ b/tests/datahub_client/test_parsers.py
@@ -22,7 +22,7 @@
     SecurityClassification,
     Table,
     TagRef,
-    UsageRestrictions,
+    UsageRestrictions, ColumnAssertionType, ColumnQualityMetrics,
 )
 from datahub_client.parsers import (
     DATA_CUSTODIAN,
@@ -34,7 +34,7 @@
     EntityParserFactory,
     PublicationCollectionParser,
     PublicationDatasetParser,
-    TableParser,
+    TableParser, parse_assertions,
 )
 from datahub_client.search.search_types import SearchResult
 
@@ -151,11 +151,114 @@ def test_parse_to_entity_object(
         assert table.urn == urn
 
 
+
+
 class TestEntityParser:
     @pytest.fixture
     def parser(self):
         return EntityParser()
 
+    def test_parse_assertions(self):
+        assertions_input = {
+            "total": 2,
+            "assertions": [
+                {
+                    "info": {
+                        "datasetAssertion": {
+                            "nativeType": "column_completeness_green_property",
+                            "nativeParameters": [{"key": "column_name", "value": "col1"}]
+                        }
+                    },
+                    "runEvents": {
+                        "runEvents": [{"result": {"type": "SUCCESS"}}]
+                    }
+                },
+                {
+                    "info": {
+                        "datasetAssertion": {
+                            "nativeType": "consistency_amber_property",
+                            "nativeParameters": [{"key": "column_name", "value": "col2"}]
+                        }
+                    },
+                    "runEvents": {
+                        "runEvents": [{"result": {"type": "FAILED"}}]
+                    }
+                }
+            ]
+        }
+
+        parsed_assertions = parse_assertions(assertions_input)
+        assert "col1" in parsed_assertions
+        assert "col2" in parsed_assertions
+        assert parsed_assertions["col1"][ColumnAssertionType.COMPLETENESS]["green"] == "SUCCESS"
+        assert parsed_assertions["col2"][ColumnAssertionType.CONSISTENCY]["amber"] == "FAILED"
+
+    def test_parse_columns_with_assertions(self, parser):
+        entity = {
+            "schemaMetadata": {
+                "fields": [
+                    {
+                        "fieldPath": "col1",
+                        "label": None,
+                        "nullable": False,
+                        "description": "Test column 1",
+                        "type": "STRING",
+                        "nativeDataType": "string",
+                    },
+                    {
+                        "fieldPath": "col2",
+                        "label": None,
+                        "nullable": True,
+                        "description": "Test column 2",
+                        "type": "INT",
+                        "nativeDataType": "integer",
+                    },
+                ],
+                "primaryKeys": ["col1"],
+                "foreignKeys": [],
+            },
+            "assertions": {
+                "total": 2,
+                "assertions": [
+                    {
+                        "info": {
+                            "datasetAssertion": {
+                                "nativeType": "column_completeness_green_property",
+                                "nativeParameters": [
+                                    {"key": "column_name", "value": "col1"}
+                                ],
+                            }
+                        },
+                        "runEvents": {
+                            "runEvents": [{"result": {"type": "SUCCESS"}}]
+                        },
+                    },
+                    {
+                        "info": {
+                            "datasetAssertion": {
+                                "nativeType": "consistency_amber_property",
+                                "nativeParameters": [
+                                    {"key": "column_name", "value": "col2"}
+                                ],
+                            }
+                        },
+                        "runEvents": {
+                            "runEvents": [{"result": {"type": "FAILED"}}]
+                        },
+                    },
+                ],
+            },
+        }
+
+        columns = parser.parse_columns(entity)
+
+
+        assert len(columns) == 2
+        assert columns[0].name == "col1"
+        assert columns[0].quality_metrics.completeness == "good"
+        assert columns[1].name == "col2"
+        assert columns[1].quality_metrics.consistency == "poor"
+
     def test_parse_columns_with_primary_key_and_foreign_key(self, parser):
         entity = {
             "schemaMetadata": {
@@ -200,6 +303,7 @@ def test_parse_columns_with_primary_key_and_foreign_key(self, parser):
                 description="The primary identifier for the dataset entity.",
                 nullable=False,
                 is_primary_key=True,
+                quality_metrics=ColumnQualityMetrics(),
                 foreign_keys=[],
             ),
             Column(
@@ -209,6 +313,7 @@ def test_parse_columns_with_primary_key_and_foreign_key(self, parser):
                 description="Upstream lineage of a dataset",
                 nullable=False,
                 is_primary_key=False,
+                quality_metrics=ColumnQualityMetrics(),
                 foreign_keys=[
                     ColumnRef(
                         name="urn",
@@ -256,6 +361,7 @@ def test_parse_columns_with_no_keys(self, parser):
                 description="Upstream lineage of a dataset",
                 nullable=False,
                 is_primary_key=False,
+                quality_metrics=ColumnQualityMetrics(),
                 foreign_keys=[],
             ),
             Column(
@@ -265,6 +371,7 @@ def test_parse_columns_with_no_keys(self, parser):
                 description="The primary identifier for the dataset entity.",
                 nullable=False,
                 is_primary_key=False,
+                quality_metrics=ColumnQualityMetrics(),
                 foreign_keys=[],
             ),
         ]
@@ -295,6 +402,7 @@ def test_parse_columns_with_null_descriptions(self, parser):
                 description="",
                 nullable=False,
                 is_primary_key=False,
+                quality_metrics=ColumnQualityMetrics(),
                 foreign_keys=[],
             )
         ]
@@ -542,6 +650,7 @@ def test_parse_columns_with_unrecognized_type(self, parser):
                 description="An unknown field type",
                 nullable=True,
                 is_primary_key=False,
+                quality_metrics=ColumnQualityMetrics(),
                 foreign_keys=[],
             )
         ]
diff --git a/tests/home/service/test_details_csv.py b/tests/home/service/test_details_csv.py
index 5d53694b0..c71c838d9 100644
--- a/tests/home/service/test_details_csv.py
+++ b/tests/home/service/test_details_csv.py
@@ -1,6 +1,6 @@
 from unittest.mock import MagicMock
 
-from datahub_client.entities import Column, EntityRef, EntitySummary
+from datahub_client.entities import Column, EntityRef, EntitySummary, ColumnQualityMetrics
 from home.service.details import (
     DashboardDetailsService,
     DatabaseDetailsService,
@@ -23,6 +23,7 @@ def test_dataset_details_csv_formatter(example_table):
             description="an example",
             nullable=False,
             is_primary_key=True,
+            quality_metrics=ColumnQualityMetrics(),
         ),
         Column(
             name="bar",
@@ -31,6 +32,7 @@ def test_dataset_details_csv_formatter(example_table):
             description="another example",
             nullable=True,
             is_primary_key=False,
+            quality_metrics=ColumnQualityMetrics(),
         ),
     ]
     details_service.table_metadata = example_table


Reviews:
user: login: murdo-moj
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-19T13:26:36Z
user: login: murdo-moj
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-19T13:30:46Z
user: login: murdo-moj
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-19T13:32:46Z
user: login: hjribeiro-moj
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-19T18:50:41Z
user: login: hjribeiro-moj
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-19T19:12:03Z
user: login: murdo-moj
body: 💯 
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-20T09:55:12Z

Review comments:
diff_hunk: @@ -115,6 +115,42 @@ class FindMoJdataEntityMapper:
 ]
 
 
+class ColumnAssertionType(Enum):
+    COMPLETENESS = "completeness"
+    CONSISTENCY = "consistency"
+    VALIDITY = "validity"
+    ACCURACY = "accuracy"
+    UNIQUENESS = "uniqueness"
+
+
+class ColumnAssertionLevel(Enum):
+    RED = "poor"
+    AMBER = "acceptable"
+    GREEN = "good"
+    NA = "na"
+
+
+class ColumnAssertionResult(Enum):
+    SUCCESS = "SUCCESS"
+    FAILURE = "FAILURE"
+    NA = "N/A"
+
+
+class ColumnAssertion(BaseModel):
+    type: ColumnAssertionType
+    level: ColumnAssertionLevel
+    # last_run: Optional[datetime]
path: datahub_client/entities.py
user: login: murdo-moj
body: ```suggestion
```
created_at: 2025-03-19T13:26:35Z
author_association: CONTRIBUTOR
diff_hunk: @@ -353,15 +388,28 @@ def parse_columns(self, entity: dict[str, Any]) -> list[Column]:
                 )
             )
 
+        all_column_assertions = parse_assertions(entity.get("assertions", {}))
         for field in schema_metadata.get("fields", ()):
             foreign_keys_for_field = foreign_keys[field["fieldPath"]]
 
-            # Work out if the field is primary.
-            # This is an oversimplification: in the case of a composite
-            # primary key, we report that each component field is primary.
             is_primary_key = field["fieldPath"] in primary_keys
             field_path = field["fieldPath"]
             display_name = field_path.split(".")[-1]
+            column_assertions = all_column_assertions.get(display_name, {})
path: datahub_client/parsers.py
user: login: murdo-moj
body: what do you think about turning this block into a function/method `form_quality_metrics` or including the logic in `parse_assertions`?

so the output of `parse_assertions`/`form_quality_metrics` would be type `ColumnQualityMetrics`
created_at: 2025-03-19T13:30:46Z
author_association: CONTRIBUTOR
diff_hunk: @@ -61,5 +71,39 @@ <h2 class="govuk-heading-m">Lineage</h2>
           </a>
         </div>
       {% endif %}
+      
     </div>
+  </div>
+
+  <style>
+    /* GOV.UK style status colors */
path: templates/details_table.html
user: login: murdo-moj
body: Can you move the styling to the custom CSS file we have?
created_at: 2025-03-19T13:32:45Z
author_association: CONTRIBUTOR
diff_hunk: @@ -353,15 +388,28 @@ def parse_columns(self, entity: dict[str, Any]) -> list[Column]:
                 )
             )
 
+        all_column_assertions = parse_assertions(entity.get("assertions", {}))
         for field in schema_metadata.get("fields", ()):
             foreign_keys_for_field = foreign_keys[field["fieldPath"]]
 
-            # Work out if the field is primary.
-            # This is an oversimplification: in the case of a composite
-            # primary key, we report that each component field is primary.
             is_primary_key = field["fieldPath"] in primary_keys
             field_path = field["fieldPath"]
             display_name = field_path.split(".")[-1]
+            column_assertions = all_column_assertions.get(display_name, {})
path: datahub_client/parsers.py
user: login: hjribeiro-moj
body: the problem is that we have 2 separate blocks: assertions and schema (columns). For us to do it all-in-one, it would be per column and we'd be iterating possibly N! times, where N is the number of columns. I have refactored the code you highlighted into a function for readability.
created_at: 2025-03-19T18:50:40Z
author_association: CONTRIBUTOR
diff_hunk: @@ -61,5 +71,39 @@ <h2 class="govuk-heading-m">Lineage</h2>
           </a>
         </div>
       {% endif %}
+      
     </div>
+  </div>
+
+  <style>
+    /* GOV.UK style status colors */
path: templates/details_table.html
user: login: hjribeiro-moj
body: done
created_at: 2025-03-19T19:12:03Z
author_association: CONTRIBUTOR

PR comments:
user: login: murdo-moj
created_at: 2025-03-11T14:03:23Z
body: Can you add a test case for `parse_assertions`?
author_association: CONTRIBUTOR
user: login: murdo-moj
created_at: 2025-03-12T10:07:32Z
body: I can see the DQ columns and they look good - the columns should only show for the one table we have assertions for. Currently they show for every table in the service, all of which are N/A
author_association: CONTRIBUTOR