Repository: oasis-open/cti-python-stix2
Pull request #147 (2018-03-22 18:41:29)

119 taxii filters
Covers:

- #119 - Filters that are destined for TAXII server are extracted and sent to taxii2client as keywords. The STIX objects returned from TAXII are then evaluated against the rest of the filters in the query. Filters that are sent to TAXII are not re-evaluated locally against the STIX objects again - doing so causes error as TAXII filter properties are outside the scope of STIX2 common properties and thus would cause STIX objects to be discarded; specifically for filters with properties 'version', 'added_after'. This code was tested against local Medallion server.

- #136 - I put a check for this when Filters are initialized. Its a little incongruous within the context of the other more generic checks. If dont want the check to occur in the Filter initialization, then would have to be when the Filter is applied in _check_filter(). 


Patch:
diff --git a/stix2/datastore/filesystem.py b/stix2/datastore/filesystem.py
index a6f31cf..26d0c58 100644
--- a/stix2/datastore/filesystem.py
+++ b/stix2/datastore/filesystem.py
@@ -301,26 +301,11 @@ class FileSystemSource(DataSource):
         for path in include_paths:
             for root, dirs, files in os.walk(path):
                 for file_ in files:
-                    if not file_.endswith(".json"):
-                        # skip non '.json' files as more likely to be random non-STIX files
-                        continue
-
                     if not id_ or id_ == file_.split(".")[0]:
                         # have to load into memory regardless to evaluate other filters
-                        try:
-                            stix_obj = json.load(open(os.path.join(root, file_)))
-
-                            if stix_obj["type"] == "bundle":
-                                stix_obj = stix_obj["objects"][0]
-
-                            # naive STIX type checking
-                            stix_obj["type"]
-                            stix_obj["id"]
-
-                        except (ValueError, KeyError):  # likely not a JSON file
-                            raise TypeError("STIX JSON object at '{0}' could either not be parsed to "
-                                            "JSON or was not valid STIX JSON".format(os.path.join(root, file_)))
-
+                        stix_obj = json.load(open(os.path.join(root, file_)))
+                        if stix_obj.get('type', '') == 'bundle':
+                            stix_obj = stix_obj['objects'][0]
                         # check against other filters, add if match
                         all_data.extend(apply_common_filters([stix_obj], query))
 
diff --git a/stix2/datastore/filters.py b/stix2/datastore/filters.py
index 5af48cd..9065b61 100644
--- a/stix2/datastore/filters.py
+++ b/stix2/datastore/filters.py
@@ -35,7 +35,11 @@ def _check_filter_components(prop, op, value):
 
     if type(value) not in FILTER_VALUE_TYPES:
         # check filter value type is supported
-        raise TypeError("Filter value type '%s' is not supported. The type must be a Python immutable type or dictionary" % type(value))
+        raise TypeError("Filter value of '%s' is not supported. The type must be a Python immutable type or dictionary" % type(value))
+
+    if prop == "type" and "_" in value:
+        # check filter where the property is type, value (type name) cannot have underscores
+        raise ValueError("Filter for property 'type' cannot have its value '%s' include underscores" % value)
 
     return True
 
diff --git a/stix2/datastore/taxii.py b/stix2/datastore/taxii.py
index cb136d9..0a58763 100644
--- a/stix2/datastore/taxii.py
+++ b/stix2/datastore/taxii.py
@@ -224,18 +224,21 @@ class TAXIICollectionSource(DataSource):
         if _composite_filters:
             query.update(_composite_filters)
 
-        # separate taxii query terms (can be done remotely)
+        # parse taxii query params (that can be applied remotely)
         taxii_filters = self._parse_taxii_filters(query)
 
+        # taxii2client requires query params as keywords
+        taxii_filters_dict = dict((f.property, f.value) for f in taxii_filters)
+
         # query TAXII collection
         try:
-            all_data = self.collection.get_objects(filters=taxii_filters)["objects"]
+            all_data = self.collection.get_objects(**taxii_filters_dict)["objects"]
 
             # deduplicate data (before filtering as reduces wasted filtering)
             all_data = deduplicate(all_data)
 
-            # apply local (CompositeDataSource, TAXIICollectionSource and query filters)
-            all_data = list(apply_common_filters(all_data, query))
+            # apply local (CompositeDataSource, TAXIICollectionSource and query) filters
+            all_data = list(apply_common_filters(all_data, (query - taxii_filters)))
 
         except HTTPError:
             # if resources not found or access is denied from TAXII server, return empty list
@@ -247,30 +250,35 @@ class TAXIICollectionSource(DataSource):
         return stix_objs
 
     def _parse_taxii_filters(self, query):
-        """Parse out TAXII filters that the TAXII server can filter on.
+        """Parse out TAXII filters that the TAXII server can filter on
+
+        Does not put in TAXII spec format as the TAXII2Client (that we use)
+        does this for us.
+
+        NOTE:
+            Currently, the TAXII2Client can handle TAXII filters where the
+            filter value is list, as both a comma-seperated string or python list
 
-        Note:
-            For instance - "?match[type]=indicator,sighting" should be in a
-            query dict as follows:
+            For instance - "?match[type]=indicator,sighting" can be in a
+            filter in any of these formats:
+
+            Filter("type", "<any op>", "indicator,sighting")
+
+            Filter("type", "<any op>", ["indicator", "sighting"])
 
-            Filter("type", "=", "indicator,sighting")
 
         Args:
-            query (list): list of filters to extract which ones are TAXII
+            query (set): set of filters to extract which ones are TAXII
                 specific.
 
         Returns:
-            params (dict): dict of the TAXII filters but in format required
-                for 'requests.get()'.
+            taxii_filters (set): set of the TAXII filters
 
         """
-        params = {}
+        taxii_filters = set()
 
         for filter_ in query:
             if filter_.property in TAXII_FILTERS:
-                if filter_.property == "added_after":
-                    params[filter_.property] = filter_.value
-                else:
-                    taxii_field = "match[%s]" % filter_.property
-                    params[taxii_field] = filter_.value
-        return params
+                taxii_filters.add(filter_)
+
+        return taxii_filters


Test patch:
diff --git a/stix2/test/test_datastore.py b/stix2/test/test_datastore.py
index 3490d6a..e80e8d8 100644
--- a/stix2/test/test_datastore.py
+++ b/stix2/test/test_datastore.py
@@ -148,18 +148,18 @@ def test_parse_taxii_filters():
         Filter("created_by_ref", "=", "Bane"),
     ]
 
-    expected_params = {
-        "added_after": "2016-02-01T00:00:01.000Z",
-        "match[id]": "taxii stix object ID",
-        "match[type]": "taxii stix object ID",
-        "match[version]": "first"
-    }
+    taxii_filters_expected = set([
+        Filter("added_after", "=", "2016-02-01T00:00:01.000Z"),
+        Filter("id", "=", "taxii stix object ID"),
+        Filter("type", "=", "taxii stix object ID"),
+        Filter("version", "=", "first")
+    ])
 
     ds = taxii.TAXIICollectionSource(collection)
 
     taxii_filters = ds._parse_taxii_filters(query)
 
-    assert taxii_filters == expected_params
+    assert taxii_filters == taxii_filters_expected
 
 
 def test_add_get_remove_filter():
@@ -172,22 +172,6 @@ def test_add_get_remove_filter():
         Filter('labels', 'in', ["heartbleed", "malicious-activity"]),
     ]
 
-    # Invalid filters - wont pass creation
-    # these filters will not be allowed to be created
-    # check proper errors are raised when trying to create them
-
-    with pytest.raises(ValueError) as excinfo:
-        # create Filter that has an operator that is not allowed
-        Filter('modified', '*', 'not supported operator - just place holder')
-    assert str(excinfo.value) == "Filter operator '*' not supported for specified property: 'modified'"
-
-    with pytest.raises(TypeError) as excinfo:
-        # create Filter that has a value type that is not allowed
-        Filter('created', '=', object())
-    # On Python 2, the type of object() is `<type 'object'>` On Python 3, it's `<class 'object'>`.
-    assert str(excinfo.value).startswith("Filter value type")
-    assert str(excinfo.value).endswith("is not supported. The type must be a Python immutable type or dictionary")
-
     assert len(ds.filters) == 0
 
     ds.filters.add(valid_filters[0])
@@ -212,6 +196,46 @@ def test_add_get_remove_filter():
     ds.filters.update(valid_filters)
 
 
+def test_filter_ops_check():
+    # invalid filters - non supported operators
+
+    with pytest.raises(ValueError) as excinfo:
+        # create Filter that has an operator that is not allowed
+        Filter('modified', '*', 'not supported operator')
+    assert str(excinfo.value) == "Filter operator '*' not supported for specified property: 'modified'"
+
+    with pytest.raises(ValueError) as excinfo:
+        Filter("type", "%", "4")
+    assert "Filter operator '%' not supported for specified property" in str(excinfo.value)
+
+
+def test_filter_value_type_check():
+    # invalid filters - non supported value types
+
+    with pytest.raises(TypeError) as excinfo:
+        Filter('created', '=', object())
+    # On Python 2, the type of object() is `<type 'object'>` On Python 3, it's `<class 'object'>`.
+    assert any([s in str(excinfo.value) for s in ["<type 'object'>", "'<class 'object'>'"]])
+    assert "is not supported. The type must be a Python immutable type or dictionary" in str(excinfo.value)
+
+    with pytest.raises(TypeError) as excinfo:
+        Filter("type", "=", complex(2, -1))
+    assert any([s in str(excinfo.value) for s in ["<type 'complex'>", "'<class 'complex'>'"]])
+    assert "is not supported. The type must be a Python immutable type or dictionary" in str(excinfo.value)
+
+    with pytest.raises(TypeError) as excinfo:
+        Filter("type", "=", set([16, 23]))
+    assert any([s in str(excinfo.value) for s in ["<type 'set'>", "'<class 'set'>'"]])
+    assert "is not supported. The type must be a Python immutable type or dictionary" in str(excinfo.value)
+
+
+def test_filter_type_underscore_check():
+    # check that Filters where property="type", value (name) doesnt have underscores
+    with pytest.raises(ValueError) as excinfo:
+        Filter("type", "=", "oh_underscore")
+    assert "Filter for property 'type' cannot have its value 'oh_underscore'" in str(excinfo.value)
+
+
 def test_apply_common_filters():
     stix_objs = [
         {
diff --git a/stix2/test/test_filesystem.py b/stix2/test/test_filesystem.py
index f59136e..020fee5 100644
--- a/stix2/test/test_filesystem.py
+++ b/stix2/test/test_filesystem.py
@@ -1,4 +1,3 @@
-import json
 import os
 import shutil
 
@@ -46,41 +45,6 @@ def fs_sink():
     shutil.rmtree(os.path.join(FS_PATH, "campaign"), True)
 
 
-@pytest.fixture
-def bad_json_files():
-    # create erroneous JSON files for tests to make sure handled gracefully
-
-    with open(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-non-json.txt"), "w+") as f:
-        f.write("Im not a JSON file")
-
-    with open(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-bad-json.json"), "w+") as f:
-        f.write("Im not a JSON formatted file")
-
-    yield True  # dummy yield so can have teardown
-
-    os.remove(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-non-json.txt"))
-    os.remove(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-bad-json.json"))
-
-
-@pytest.fixture
-def bad_stix_files():
-    # create erroneous STIX JSON files for tests to make sure handled correctly
-
-    # bad STIX object
-    stix_obj = {
-        "id": "intrusion-set--test-bad-stix",
-        "spec_version": "2.0"
-        # no "type" field
-    }
-
-    with open(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-non-stix.json"), "w+") as f:
-        f.write(json.dumps(stix_obj))
-
-    yield True  # dummy yield so can have teardown
-
-    os.remove(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-non-stix.json"))
-
-
 @pytest.fixture(scope='module')
 def rel_fs_store():
     cam = Campaign(id=CAMPAIGN_ID, **CAMPAIGN_KWARGS)
@@ -112,26 +76,6 @@ def test_filesystem_sink_nonexistent_folder():
     assert "for STIX data does not exist" in str(excinfo)
 
 
-def test_filesystem_source_bad_json_file(fs_source, bad_json_files):
-    # this tests the handling of two bad json files
-    #  - one file should just be skipped (silently) as its a ".txt" extension
-    #  - one file should be parsed and raise Exception bc its not JSON
-    try:
-        fs_source.get("intrusion-set--test-bad-json")
-    except TypeError as e:
-        assert "intrusion-set--test-bad-json" in str(e)
-        assert "could either not be parsed to JSON or was not valid STIX JSON" in str(e)
-
-
-def test_filesystem_source_bad_stix_file(fs_source, bad_stix_files):
-    # this tests handling of bad STIX json object
-    try:
-        fs_source.get("intrusion-set--test-non-stix")
-    except TypeError as e:
-        assert "intrusion-set--test-non-stix" in str(e)
-        assert "could either not be parsed to JSON or was not valid STIX JSON" in str(e)
-
-
 def test_filesytem_source_get_object(fs_source):
     # get object
     mal = fs_source.get("malware--6b616fc1-1505-48e3-8b2c-0d19337bff38")