Repository: oasis-open/cti-python-stix2

Issue:
Many datastores and helpers assume created and modified fields exist in objects
As an example, `CompositeDataSource`s attempt to access the `modified` key directly as well as making use of the `deduplicate()` helper which attempts to access `modified` and then falls back to `created` if that's missing. The following snippet shows a handful of breakages which present as you squash each one.

```python
import traceback

import pytest
import stix2.datastore
import stix2.datastore.memory
import stix2.v21

m1 = stix2.datastore.memory.MemoryStore()
m2 = stix2.datastore.memory.MemoryStore()
c = stix2.datastore.CompositeDataSource()
c.add_data_sources((m1.source, m2.source))

o = stix2.v21.Software(name="foo")
m1.add(o)
m2.add(o)

with pytest.raises(KeyError) as caught:
    c.get(o.id)
print(caught.getrepr())
```

The table in section 3.2 shows the exceptions, but even that's not reliable since custom properties could be implemented per section 11.1 to implement versioning per section 3.6 (e.g. for SCOs).

This grep line shows all accesses outside the tests, but I suspect only `utils.py`, `versioning.py` and the datastores would need to be changed.
```sh
$ grep -Er "\[[\"'](created|modified|revoked)" stix2/*.py stix2/canonicalization/ stix2/confidence/ stix2/datastore/ stix2/markings/ stix2/v20/ stix2/v21/
...
```

Most of these accesses seem to be related to sorting versions so it seems like it might be reasonable to instead implement a single helper in `versioning.py` which accepts a series of objects, sorts and (optionally?) deduplicates them. I'm happy to work on a patchset for this but wanted to get some input on an approach.

Discussion:

Patch:
diff --git a/setup.py b/setup.py
index 2fc5d70..3a5ef8d 100644
--- a/setup.py
+++ b/setup.py
@@ -52,7 +52,7 @@ setup(
         'requests',
         'simplejson',
         'six>=1.13.0',
-        'stix2-patterns>=1.2.0',
+        'stix2-patterns',
     ],
     project_urls={
         'Documentation': 'https://stix2.readthedocs.io/',
@@ -60,7 +60,7 @@ setup(
         'Bug Tracker': 'https://github.com/oasis-open/cti-python-stix2/issues/',
     },
     extras_require={
-        'taxii': ['taxii2-client>=2.2.1'],
+        'taxii': ['taxii2-client'],
         'semantic': ['haversine', 'rapidfuzz'],
     },
 )
diff --git a/stix2/datastore/__init__.py b/stix2/datastore/__init__.py
index 57cb513..1ff0769 100644
--- a/stix2/datastore/__init__.py
+++ b/stix2/datastore/__init__.py
@@ -481,14 +481,14 @@ class CompositeDataSource(DataSource):
             if data:
                 all_data.append(data)
 
-        # remove duplicate versions
-        if len(all_data) > 0:
-            all_data = deduplicate(all_data)
-        else:
-            return None
-
-        # reduce to most recent version
-        stix_obj = sorted(all_data, key=lambda k: k['modified'], reverse=True)[0]
+        # Search for latest version
+        stix_obj = latest_ver = None
+        for obj in all_data:
+            ver = obj.get("modified") or obj.get("created")
+
+            if stix_obj is None or ver is None or ver > latest_ver:
+                stix_obj = obj
+                latest_ver = ver
 
         return stix_obj
 
diff --git a/stix2/utils.py b/stix2/utils.py
index f741581..1b88f72 100644
--- a/stix2/utils.py
+++ b/stix2/utils.py
@@ -132,11 +132,12 @@ def deduplicate(stix_obj_list):
     unique_objs = {}
 
     for obj in stix_obj_list:
-        try:
-            unique_objs[(obj['id'], obj['modified'])] = obj
-        except KeyError:
-            # Handle objects with no `modified` property, e.g. marking-definition
-            unique_objs[(obj['id'], obj['created'])] = obj
+        ver = obj.get("modified") or obj.get("created")
+
+        if ver is None:
+            unique_objs[obj["id"]] = obj
+        else:
+            unique_objs[(obj['id'], ver)] = obj
 
     return list(unique_objs.values())
 


Test patch:
diff --git a/stix2/test/v20/test_utils.py b/stix2/test/v20/test_utils.py
index 9372bbb..a66f3e8 100644
--- a/stix2/test/v20/test_utils.py
+++ b/stix2/test/v20/test_utils.py
@@ -114,7 +114,7 @@ def test_deduplicate(stix_objs1):
     mods = [obj['modified'] for obj in unique]
 
     assert "indicator--00000000-0000-4000-8000-000000000001" in ids
-    assert "indicator--00000000-0000-4000-8000-000000000001" in ids
+    assert "indicator--00000000-0000-4000-8000-000000000002" in ids
     assert "2017-01-27T13:49:53.935Z" in mods
     assert "2017-01-27T13:49:53.936Z" in mods
 
diff --git a/stix2/test/v21/conftest.py b/stix2/test/v21/conftest.py
index d602f42..6efcf39 100644
--- a/stix2/test/v21/conftest.py
+++ b/stix2/test/v21/conftest.py
@@ -132,7 +132,13 @@ def stix_objs1():
         "type": "indicator",
         "valid_from": "2017-01-27T13:49:53.935382Z",
     }
-    return [ind1, ind2, ind3, ind4, ind5]
+    sco = {
+        "type": "url",
+        "spec_version": "2.1",
+        "id": "url--cc1deced-d99b-4d72-9268-8182420cb2fd",
+        "value": "http://example.com/",
+    }
+    return [ind1, ind2, ind3, ind4, ind5, sco]
 
 
 @pytest.fixture
diff --git a/stix2/test/v21/test_datastore_composite.py b/stix2/test/v21/test_datastore_composite.py
index 76119c3..c6128e5 100644
--- a/stix2/test/v21/test_datastore_composite.py
+++ b/stix2/test/v21/test_datastore_composite.py
@@ -59,6 +59,17 @@ def test_composite_datasource_operations(stix_objs1, stix_objs2):
     assert indicator["modified"] == parse_into_datetime("2017-01-31T13:49:53.935Z")
     assert indicator["type"] == "indicator"
 
+    sco = cds1.get("url--cc1deced-d99b-4d72-9268-8182420cb2fd")
+    assert sco["id"] == "url--cc1deced-d99b-4d72-9268-8182420cb2fd"
+
+    scos = cds1.all_versions("url--cc1deced-d99b-4d72-9268-8182420cb2fd")
+    assert len(scos) == 1
+    assert scos[0]["id"] == "url--cc1deced-d99b-4d72-9268-8182420cb2fd"
+
+    scos = cds1.query([Filter("value", "=", "http://example.com/")])
+    assert len(scos) == 1
+    assert scos[0]["id"] == "url--cc1deced-d99b-4d72-9268-8182420cb2fd"
+
     query1 = [
         Filter("type", "=", "indicator"),
     ]
diff --git a/stix2/test/v21/test_utils.py b/stix2/test/v21/test_utils.py
index 03477aa..f64cec2 100644
--- a/stix2/test/v21/test_utils.py
+++ b/stix2/test/v21/test_utils.py
@@ -104,17 +104,18 @@ def test_get_type_from_id(stix_id, type):
 def test_deduplicate(stix_objs1):
     unique = stix2.utils.deduplicate(stix_objs1)
 
-    # Only 3 objects are unique
-    # 2 id's vary
+    # Only 4 objects are unique
+    # 3 id's vary
     # 2 modified times vary for a particular id
 
-    assert len(unique) == 3
+    assert len(unique) == 4
 
     ids = [obj['id'] for obj in unique]
-    mods = [obj['modified'] for obj in unique]
+    mods = [obj.get('modified') for obj in unique]
 
     assert "indicator--00000000-0000-4000-8000-000000000001" in ids
-    assert "indicator--00000000-0000-4000-8000-000000000001" in ids
+    assert "indicator--00000000-0000-4000-8000-000000000002" in ids
+    assert "url--cc1deced-d99b-4d72-9268-8182420cb2fd" in ids
     assert "2017-01-27T13:49:53.935Z" in mods
     assert "2017-01-27T13:49:53.936Z" in mods