Repository: oasis-open/cti-python-stix2

Issue:
Rundown status of existing issue: FileSystem query bug
Bug: When querying FileSystem, it came back with JSON parse error. Specifically had to use a filter with “type” field to work. @mbastian1135 was trying to reproduce this, but having problems

Discussion:

Patch:
diff --git a/docs/conf.py b/docs/conf.py
index 49416a0..e53829b 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -74,7 +74,7 @@ def get_property_type(prop):
     return prop_class
 
 
-class STIXPropertyDocumenter(ClassDocumenter):
+class STIXAttributeDocumenter(ClassDocumenter):
     """Custom Sphinx extension to auto-document STIX properties.
 
     Needed because descendants of _STIXBase use `_properties` dictionaries
@@ -97,23 +97,16 @@ class STIXPropertyDocumenter(ClassDocumenter):
         obj = self.object
         self.add_line(':Properties:', '<stixattr>')
         for prop_name, prop in obj._properties.items():
-            # Skip 'type'
-            if prop_name == 'type':
-                continue
-
             # Add metadata about the property
             prop_type = get_property_type(prop)
             if prop_type == 'List':
                 prop_type = 'List of %ss' % get_property_type(prop.contained)
             if prop.required:
                 prop_type += ', required'
-            if 'Timestamp' in prop_type and hasattr(prop, 'default'):
-                prop_type += ', default: current date/time'
             prop_str = '**%s** (*%s*)' % (prop_name, prop_type)
             self.add_line('    - %s' % prop_str, '<stixattr>')
-
         self.add_line('', '<stixattr>')
 
 
 def setup(app):
-    app.add_autodocumenter(STIXPropertyDocumenter)
+    app.add_autodocumenter(STIXAttributeDocumenter)
diff --git a/docs/guide/taxii.ipynb b/docs/guide/taxii.ipynb
index 4045a98..ad06093 100644
--- a/docs/guide/taxii.ipynb
+++ b/docs/guide/taxii.ipynb
@@ -261,65 +261,25 @@
     "\n",
     "tc_store.add(ind)\n"
    ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Bug and Workaround\n",
-    "\n",
-    "You may get an error similar to the following when adding STIX objects to a TAXIICollectionStore or TAXIICollectionSink:\n",
-    "\n",
-    "```\n",
-    "TypeError: Object of type ThreatActor is not JSON serializable\n",
-    "```\n",
-    "\n",
-    "This is a known bug and we are working to fix it. For more information, see [this GitHub issue](https://github.com/oasis-open/cti-python-stix2/issues/125) In the meantime, try this workaround:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "tc_sink.add(json.loads(Bundle(ta).serialize()))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Or bypass the TAXIICollection altogether and interact with the collection itself:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "collection.add_objects(json.loads(Bundle(ta).serialize()))"
-   ]
   }
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 2",
    "language": "python",
-   "name": "python3"
+   "name": "python2"
   },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
-    "version": 3
+    "version": 2
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.6.3"
+   "pygments_lexer": "ipython2",
+   "version": "2.7.12"
   }
  },
  "nbformat": 4,
diff --git a/stix2/base.py b/stix2/base.py
index 898f489..6bc3dd4 100644
--- a/stix2/base.py
+++ b/stix2/base.py
@@ -191,7 +191,7 @@ class _STIXBase(collections.Mapping):
             **kwargs: The arguments for a json.dumps() call.
 
         Returns:
-            str: The serialized JSON object.
+            dict: The serialized JSON object.
 
         Note:
             The argument ``pretty=True`` will output the STIX object following
diff --git a/stix2/datastore/filesystem.py b/stix2/datastore/filesystem.py
index 26d0c58..b525932 100644
--- a/stix2/datastore/filesystem.py
+++ b/stix2/datastore/filesystem.py
@@ -301,11 +301,27 @@ class FileSystemSource(DataSource):
         for path in include_paths:
             for root, dirs, files in os.walk(path):
                 for file_ in files:
+                    if not file_.endswith(".json"):
+                        # skip non '.json' files as more likely to be random non-STIX files
+                        continue
+
                     if not id_ or id_ == file_.split(".")[0]:
                         # have to load into memory regardless to evaluate other filters
-                        stix_obj = json.load(open(os.path.join(root, file_)))
-                        if stix_obj.get('type', '') == 'bundle':
-                            stix_obj = stix_obj['objects'][0]
+                        try:
+                            stix_obj = json.load(open(os.path.join(root, file_)))
+
+                            if stix_obj["type"] == "bundle":
+                                stix_obj = stix_obj["objects"][0]
+
+                            # naive STIX type checking
+                            stix_obj["type"]
+                            stix_obj["id"]
+
+                        except (ValueError, KeyError):  # likely not a JSON file
+                            print("filesytem TypeError raised")
+                            raise TypeError("STIX JSON object at '{0}' could either not be parsed to "
+                                            "JSON or was not valid STIX JSON".format(os.path.join(root, file_)))
+
                         # check against other filters, add if match
                         all_data.extend(apply_common_filters([stix_obj], query))
 
diff --git a/stix2/datastore/taxii.py b/stix2/datastore/taxii.py
index cb136d9..5af4354 100644
--- a/stix2/datastore/taxii.py
+++ b/stix2/datastore/taxii.py
@@ -67,31 +67,27 @@ class TAXIICollectionSink(DataSink):
         """
         if isinstance(stix_data, _STIXBase):
             # adding python STIX object
-            if stix_data["type"] == "bundle":
-                bundle = stix_data.serialize(encoding="utf-8")
-            else:
-                bundle = Bundle(stix_data, allow_custom=self.allow_custom).serialize(encoding="utf-8")
+            bundle = dict(Bundle(stix_data, allow_custom=self.allow_custom))
 
         elif isinstance(stix_data, dict):
             # adding python dict (of either Bundle or STIX obj)
             if stix_data["type"] == "bundle":
-                bundle = parse(stix_data, allow_custom=self.allow_custom, version=version).serialize(encoding="utf-8")
+                bundle = stix_data
             else:
-                bundle = Bundle(stix_data, allow_custom=self.allow_custom).serialize(encoding="utf-8")
+                bundle = dict(Bundle(stix_data, allow_custom=self.allow_custom))
 
         elif isinstance(stix_data, list):
             # adding list of something - recurse on each
             for obj in stix_data:
                 self.add(obj, version=version)
-            return
 
         elif isinstance(stix_data, str):
             # adding json encoded string of STIX content
             stix_data = parse(stix_data, allow_custom=self.allow_custom, version=version)
             if stix_data["type"] == "bundle":
-                bundle = stix_data.serialize(encoding="utf-8")
+                bundle = dict(stix_data)
             else:
-                bundle = Bundle(stix_data, allow_custom=self.allow_custom).serialize(encoding="utf-8")
+                bundle = dict(Bundle(stix_data, allow_custom=self.allow_custom))
 
         else:
             raise TypeError("stix_data must be as STIX object(or list of),json formatted STIX (or list of), or a json formatted STIX bundle")


Test patch:
diff --git a/stix2/test/test_filesystem.py b/stix2/test/test_filesystem.py
index 020fee5..f59136e 100644
--- a/stix2/test/test_filesystem.py
+++ b/stix2/test/test_filesystem.py
@@ -1,3 +1,4 @@
+import json
 import os
 import shutil
 
@@ -45,6 +46,41 @@ def fs_sink():
     shutil.rmtree(os.path.join(FS_PATH, "campaign"), True)
 
 
+@pytest.fixture
+def bad_json_files():
+    # create erroneous JSON files for tests to make sure handled gracefully
+
+    with open(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-non-json.txt"), "w+") as f:
+        f.write("Im not a JSON file")
+
+    with open(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-bad-json.json"), "w+") as f:
+        f.write("Im not a JSON formatted file")
+
+    yield True  # dummy yield so can have teardown
+
+    os.remove(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-non-json.txt"))
+    os.remove(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-bad-json.json"))
+
+
+@pytest.fixture
+def bad_stix_files():
+    # create erroneous STIX JSON files for tests to make sure handled correctly
+
+    # bad STIX object
+    stix_obj = {
+        "id": "intrusion-set--test-bad-stix",
+        "spec_version": "2.0"
+        # no "type" field
+    }
+
+    with open(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-non-stix.json"), "w+") as f:
+        f.write(json.dumps(stix_obj))
+
+    yield True  # dummy yield so can have teardown
+
+    os.remove(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-non-stix.json"))
+
+
 @pytest.fixture(scope='module')
 def rel_fs_store():
     cam = Campaign(id=CAMPAIGN_ID, **CAMPAIGN_KWARGS)
@@ -76,6 +112,26 @@ def test_filesystem_sink_nonexistent_folder():
     assert "for STIX data does not exist" in str(excinfo)
 
 
+def test_filesystem_source_bad_json_file(fs_source, bad_json_files):
+    # this tests the handling of two bad json files
+    #  - one file should just be skipped (silently) as its a ".txt" extension
+    #  - one file should be parsed and raise Exception bc its not JSON
+    try:
+        fs_source.get("intrusion-set--test-bad-json")
+    except TypeError as e:
+        assert "intrusion-set--test-bad-json" in str(e)
+        assert "could either not be parsed to JSON or was not valid STIX JSON" in str(e)
+
+
+def test_filesystem_source_bad_stix_file(fs_source, bad_stix_files):
+    # this tests handling of bad STIX json object
+    try:
+        fs_source.get("intrusion-set--test-non-stix")
+    except TypeError as e:
+        assert "intrusion-set--test-non-stix" in str(e)
+        assert "could either not be parsed to JSON or was not valid STIX JSON" in str(e)
+
+
 def test_filesytem_source_get_object(fs_source):
     # get object
     mal = fs_source.get("malware--6b616fc1-1505-48e3-8b2c-0d19337bff38")