Repository: HXLStandard/libhxl-python

Issue:
Double counting of errors p-code adm name combination consistency errors
When I put the below into data check, I get 2 of every cell eg. F3,F3,F4,F4,F5,F5...
https://data.humdata.org/dataset/77c97850-4004-4285-94db-0b390a962d6e/resource/d6c0dbac-683d-42d7-82b4-a6379bd4f48e/download/mrt_population_statistics_ons_rgph_2013_2017.xlsx

Discussion:

Patch:
diff --git a/hxl/validation.py b/hxl/validation.py
index 3eb1eba..922bfaf 100644
--- a/hxl/validation.py
+++ b/hxl/validation.py
@@ -1508,11 +1508,6 @@ def validate(data, schema=None):
 
     issue_map = dict()
 
-    def make_rule_hash(rule):
-        """Make a good-enough hash for a rule."""
-        s = "\r".join([str(rule.severity), str(rule.description), str(rule.tag_pattern)])
-        return base64.urlsafe_b64encode(hashlib.md5(s.encode('utf-8')).digest())[:8].decode('ascii')
-
     def add_issue(issue):
         hash = make_rule_hash(issue.rule)
         issue_map.setdefault(hash, []).append(issue)
@@ -1562,9 +1557,10 @@ def make_json_report(status, issue_map, schema_url=None, data_url=None):
 
     # add the issue objects
     for rule_id, locations in issue_map.items():
-        json_report['stats']['total'] += len(locations)
-        json_report['stats'][locations[0].rule.severity] += len(locations)
-        json_report['issues'].append(make_json_issue(rule_id, locations))
+        json_issue = make_json_issue(rule_id, locations)
+        json_report['stats']['total'] += len(json_issue['locations'])
+        json_report['stats'][locations[0].rule.severity] += len(json_issue['locations'])
+        json_report['issues'].append(json_issue)
 
     return json_report
 
@@ -1581,6 +1577,15 @@ def make_json_issue(rule_id, locations):
     if not description:
         description = model.message
 
+    # get all unique locations
+    location_keys = set()
+    json_locations = []
+    for location in locations:
+        location_key = (location.row.row_number, location.column.column_number, location.value, location.suggested_value,)
+        if not location_key in location_keys:
+            json_locations.append(make_json_location(location))
+            location_keys.add(location_key)
+
     # make the issue
     json_issue = {
         "rule_id": rule_id,
@@ -1589,7 +1594,7 @@ def make_json_issue(rule_id, locations):
         "severity": model.rule.severity,
         "location_count": len(locations),
         "scope": model.scope,
-        "locations": [make_json_location(location) for location in locations]
+        "locations": json_locations
     }
 
     return json_issue
@@ -1622,4 +1627,10 @@ def make_json_location(location):
 
     return json_location
 
+
+def make_rule_hash(rule):
+    """Make a good-enough hash for a rule."""
+    s = "\r".join([str(rule.severity), str(rule.description), str(rule.tag_pattern)])
+    return base64.urlsafe_b64encode(hashlib.md5(s.encode('utf-8')).digest())[:8].decode('ascii')
+
 # end


Test patch:
diff --git a/tests/test_validation.py b/tests/test_validation.py
index 43ab00b..7e8b4f3 100644
--- a/tests/test_validation.py
+++ b/tests/test_validation.py
@@ -655,8 +655,8 @@ class TestValidateDataset(unittest.TestCase):
     def test_double_correlation(self):
         """Test correlation when more than one column has same tagspec"""
         SCHEMA = [
-            ['#valid_tag', '#valid_correlation'],
-            ['#adm1+code', '#adm1+name']
+            ['#valid_tag', '#description', '#valid_correlation', '#valid_value+list'],
+            ['#adm1+code', 'xxxxx', '#adm1+name', 'X001|X002']
         ]
         DATASET = [
             ['#adm1+name', '#adm1+code', '#adm1+code'],