Repository: open-contracting/ocds-merge

Issue:
Crashs if no 'date' key in data.
https://github.com/open-contracting/ocds-merge/blob/master/ocdsmerge/merge.py#L316

We are currently seeing crashes in Kingfisher process where there is data with no 'date' key.

In fairness, I'm not sure what the merge library can actually do in this case 😄 ... but maybe we want to think about good ways to report this upwards.

Comments:

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8c9d549..b732eb9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,10 @@
 
 ## 0.5.2
 
+### Changed
+
+* If there is more than one release, but the `date` field is missing or null, the `MissingDateKeyError` and `NullDateValueError` exceptions are raised, respestively, instead of the generic `KeyError` and `TypeError`.
+
 ### Fixed
 
 * If a field's value is set to `null`, it is omitted from the compiled release.
diff --git a/ocdsmerge/merge.py b/ocdsmerge/merge.py
index 7f6c1ef..842cb66 100644
--- a/ocdsmerge/merge.py
+++ b/ocdsmerge/merge.py
@@ -31,6 +31,18 @@ class IdDict(OrderedDict):
         self._identifier = identifier
 
 
+class OCDSMergeError(Exception):
+    """Base class for exceptions from within this package"""
+
+
+class MissingDateKeyError(OCDSMergeError):
+    """Raised when a release is missing a 'date' key"""
+
+
+class NullDateValueError(OCDSMergeError):
+    """Raised when a release has a null 'date' value"""
+
+
 @lru_cache()
 def get_tags():
     """
@@ -274,6 +286,20 @@ def process_flattened(flattened):
     return processed
 
 
+def sorted_releases(releases):
+    """
+    Sorts a list of releases by date.
+    """
+    if len(releases) == 1:
+        return releases
+    try:
+        return sorted(releases, key=lambda release: release['date'])
+    except KeyError:
+        raise MissingDateKeyError
+    except TypeError:
+        raise NullDateValueError
+
+
 def merge(releases, schema=None, merge_rules=None):
     """
     Merges a list of releases into a compiledRelease.
@@ -282,11 +308,12 @@ def merge(releases, schema=None, merge_rules=None):
         merge_rules = get_merge_rules(schema)
 
     merged = OrderedDict({('tag',): ['compiled']})
-    for release in sorted(releases, key=lambda release: release['date']):
+    for release in sorted_releases(releases):
         release = release.copy()
 
-        ocid = release['ocid']
-        date = release['date']
+        # `ocid` and `date` are required fields, but the data can be invalid.
+        ocid = release.get('ocid')
+        date = release.get('date')
         # Prior to OCDS 1.1.4, `tag` didn't set "omitWhenMerged": true.
         release.pop('tag', None)  # becomes ["compiled"]
 
@@ -313,15 +340,16 @@ def merge_versioned(releases, schema=None, merge_rules=None):
         merge_rules = get_merge_rules(schema)
 
     merged = OrderedDict()
-    for release in sorted(releases, key=lambda release: release['date']):
+    for release in sorted_releases(releases):
         release = release.copy()
 
         # Don't version the OCID.
-        ocid = release.pop('ocid')
+        ocid = release.pop('ocid', None)
         merged[('ocid',)] = ocid
 
-        releaseID = release['id']
-        date = release['date']
+        # `id` and `date` are required fields, but the data can be invalid.
+        releaseID = release.get('id')
+        date = release.get('date')
         # Prior to OCDS 1.1.4, `tag` didn't set "omitWhenMerged": true.
         tag = release.pop('tag', None)
 


Test patch:
diff --git a/tests/test_merge.py b/tests/test_merge.py
index 1b25c32..7b4df8e 100644
--- a/tests/test_merge.py
+++ b/tests/test_merge.py
@@ -24,7 +24,8 @@ from jsonschema import FormatChecker
 from jsonschema.validators import Draft4Validator as validator
 
 from ocdsmerge import merge, merge_versioned, get_merge_rules
-from ocdsmerge.merge import get_tags, get_release_schema_url, flatten, process_flattened
+from ocdsmerge.merge import (get_tags, get_release_schema_url, flatten, process_flattened, MissingDateKeyError,
+                             NullDateValueError)
 
 tags = {
     '1.0': '1__0__3',
@@ -64,6 +65,30 @@ def custom_warning_formatter(message, category, filename, lineno, line=None):
 warnings.formatwarning = custom_warning_formatter
 
 
+@pytest.mark.parametrize('error, data', [(MissingDateKeyError, {}), (NullDateValueError, {'date': None})])
+def test_date_errors(error, data):
+    for method in (merge, merge_versioned):
+        with pytest.raises(error):
+            method([{'date': '2010-01-01'}, data])
+
+    release = deepcopy(data)
+    assert merge([release]) == {
+        'id': 'None-None',
+        'tag': ['compiled'],
+    }
+
+    release = deepcopy(data)
+    release['initiationType'] = 'tender'
+    assert merge_versioned([release]) == {
+        'initiationType': [{
+            'releaseID': None,
+            'releaseDate': None,
+            'releaseTag': None,
+            'value': 'tender',
+        }],
+    }
+
+
 @pytest.mark.parametrize('filename,schema', test_merge_argvalues)
 def test_merge(filename, schema):
     if filename.endswith('-compiled.json'):