Repository: oasis-open/cti-python-stix2
Pull request #166 (2018-04-13 18:44:26)

139 dict filter value
Primary purpose of the branch is to allow stix2.Filter to take Python dictionary as a value. Could not be done previously as we would internally use Python sets to pass around filters, and dicts can not be added to sets. Main component changes for this:
- FilterSet class ( uses a list to maintain filters but acts like a set , for internal use only)
- removal of _assemble_filters() as FilterSet can handle same functionality

Additionally, made some changes to FIlters to allow for more seamless comparison of datetimes. Now a Filter created with a datetime as its value will store the datetime in str format, and any datatime object being compared (to the filter) will have its str format used. So all datetime comparisons are done str to str. Component changes:
- Filter class
- added tests to check for this

 Along the way, found a bug in the parsing of observables, essentially the original dictionaries being passed to parse were having their nested observable properties converted to STIX objects.  This required some changes (deep-copying) of supplied user data before altering and further parsing. Also made the utility get_dict() private as part of the reason for the oversight. Main component changes for this:
- changing to _get_dict()
- deepcopy'ing being done in ObservableProperty and ExtensionProperty
- added tests for this

#139 


Patch:
diff --git a/docs/guide/datastore.ipynb b/docs/guide/datastore.ipynb
index 7d40930..c3980a0 100644
--- a/docs/guide/datastore.ipynb
+++ b/docs/guide/datastore.ipynb
@@ -4,6 +4,7 @@
    "cell_type": "code",
    "execution_count": 1,
    "metadata": {
+    "collapsed": true,
     "nbsphinx": "hidden"
    },
    "outputs": [],
@@ -24,6 +25,7 @@
    "cell_type": "code",
    "execution_count": 2,
    "metadata": {
+    "collapsed": true,
     "nbsphinx": "hidden"
    },
    "outputs": [],
@@ -383,8 +385,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
-   "metadata": {},
+   "execution_count": 3,
+   "metadata": {
+    "collapsed": true
+   },
    "outputs": [],
    "source": [
     "import sys\n",
@@ -415,7 +419,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -428,11 +432,11 @@
     "fs.source.filters.add(f)\n",
     "\n",
     "# attach multiple filters to FileSystemStore\n",
-    "fs.source.filters.update([f1,f2])\n",
+    "fs.source.filters.add([f1,f2])\n",
     "\n",
     "# can also attach filters to a Source\n",
     "# attach multiple filters to FileSystemSource\n",
-    "fs_source.filters.update([f3, f4])\n",
+    "fs_source.filters.add([f3, f4])\n",
     "\n",
     "\n",
     "mem = MemoryStore()\n",
@@ -442,7 +446,7 @@
     "mem.source.filters.add(f)\n",
     "\n",
     "# attach multiple filters to a MemoryStore\n",
-    "mem.source.filters.update([f1,f2])"
+    "mem.source.filters.add([f1,f2])"
    ]
   },
   {
@@ -457,7 +461,9 @@
   {
    "cell_type": "code",
    "execution_count": 10,
-   "metadata": {},
+   "metadata": {
+    "collapsed": true
+   },
    "outputs": [],
    "source": [
     "from stix2 import Campaign, Identity, Indicator, Malware, Relationship\n",
@@ -719,21 +725,21 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "cti-python-stix2",
    "language": "python",
-   "name": "python3"
+   "name": "cti-python-stix2"
   },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
-    "version": 3
+    "version": 2
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.6.3"
+   "pygments_lexer": "ipython2",
+   "version": "2.7.12"
   }
  },
  "nbformat": 4,
diff --git a/stix2/__init__.py b/stix2/__init__.py
index 17f2277..449be68 100644
--- a/stix2/__init__.py
+++ b/stix2/__init__.py
@@ -50,7 +50,7 @@ from .patterns import (AndBooleanExpression, AndObservationExpression,
                        ReferenceObjectPathComponent, RepeatQualifier,
                        StartStopQualifier, StringConstant, TimestampConstant,
                        WithinQualifier)
-from .utils import get_dict, new_version, revoke
+from .utils import new_version, revoke
 from .v20 import *  # This import will always be the latest STIX 2.X version
 from .version import __version__
 
diff --git a/stix2/core.py b/stix2/core.py
index 7de7984..0b222a9 100644
--- a/stix2/core.py
+++ b/stix2/core.py
@@ -9,7 +9,7 @@ import stix2
 from . import exceptions
 from .base import _STIXBase
 from .properties import IDProperty, ListProperty, Property, TypeProperty
-from .utils import get_class_hierarchy_names, get_dict
+from .utils import _get_dict, get_class_hierarchy_names
 
 
 class STIXObjectProperty(Property):
@@ -25,7 +25,7 @@ class STIXObjectProperty(Property):
                for x in get_class_hierarchy_names(value)):
             return value
         try:
-            dictified = get_dict(value)
+            dictified = _get_dict(value)
         except ValueError:
             raise ValueError("This property may only contain a dictionary or object")
         if dictified == {}:
@@ -95,7 +95,7 @@ def parse(data, allow_custom=False, version=None):
 
     """
     # convert STIX object to dict, if not already
-    obj = get_dict(data)
+    obj = _get_dict(data)
 
     # convert dict to full python-stix2 obj
     obj = dict_to_stix2(obj, allow_custom, version)
diff --git a/stix2/datastore/__init__.py b/stix2/datastore/__init__.py
index 4ca9d5f..7fdf515 100644
--- a/stix2/datastore/__init__.py
+++ b/stix2/datastore/__init__.py
@@ -16,7 +16,7 @@ import uuid
 
 from six import with_metaclass
 
-from stix2.datastore.filters import Filter, _assemble_filters
+from stix2.datastore.filters import Filter, FilterSet
 from stix2.utils import deduplicate
 
 
@@ -222,13 +222,13 @@ class DataSource(with_metaclass(ABCMeta)):
 
     Attributes:
         id (str): A unique UUIDv4 to identify this DataSource.
-        filters (set): A collection of filters attached to this DataSource.
+        filters (FilterSet): A collection of filters attached to this DataSource.
 
     """
     def __init__(self):
         super(DataSource, self).__init__()
         self.id = make_id()
-        self.filters = set()
+        self.filters = FilterSet()
 
     @abstractmethod
     def get(self, stix_id):
@@ -379,10 +379,10 @@ class DataSource(with_metaclass(ABCMeta)):
         ids.discard(obj_id)
 
         # Assemble filters
-        filter_list = _assemble_filters(filters)
+        filter_list = FilterSet(filters)
 
         for i in ids:
-            results.extend(self.query(filter_list + [Filter('id', '=', i)]))
+            results.extend(self.query([f for f in filter_list] + [Filter('id', '=', i)]))
 
         return results
 
@@ -427,7 +427,7 @@ class CompositeDataSource(DataSource):
 
         Args:
             stix_id (str): the id of the STIX object to retrieve.
-            _composite_filters (list): a list of filters passed from a
+            _composite_filters (FilterSet): a collection of filters passed from a
                 CompositeDataSource (i.e. if this CompositeDataSource is attached
                 to another parent CompositeDataSource), not user supplied.
 
@@ -439,11 +439,12 @@ class CompositeDataSource(DataSource):
             raise AttributeError('CompositeDataSource has no data sources')
 
         all_data = []
-        all_filters = set()
-        all_filters.update(self.filters)
+        all_filters = FilterSet()
+
+        all_filters.add(self.filters)
 
         if _composite_filters:
-            all_filters.update(_composite_filters)
+            all_filters.add(_composite_filters)
 
         # for every configured Data Source, call its retrieve handler
         for ds in self.data_sources:
@@ -473,7 +474,7 @@ class CompositeDataSource(DataSource):
 
         Args:
             stix_id (str): id of the STIX objects to retrieve.
-            _composite_filters (list): a list of filters passed from a
+            _composite_filters (FilterSet): a collection of filters passed from a
                 CompositeDataSource (i.e. if this CompositeDataSource is
                 attached to a parent CompositeDataSource), not user supplied.
 
@@ -485,12 +486,12 @@ class CompositeDataSource(DataSource):
             raise AttributeError('CompositeDataSource has no data sources')
 
         all_data = []
-        all_filters = set()
+        all_filters = FilterSet()
 
-        all_filters.update(self.filters)
+        all_filters.add(self.filters)
 
         if _composite_filters:
-            all_filters.update(_composite_filters)
+            all_filters.add(_composite_filters)
 
         # retrieve STIX objects from all configured data sources
         for ds in self.data_sources:
@@ -512,7 +513,7 @@ class CompositeDataSource(DataSource):
 
         Args:
             query (list): list of filters to search on.
-            _composite_filters (list): a list of filters passed from a
+            _composite_filters (FilterSet): a collection of filters passed from a
                 CompositeDataSource (i.e. if this CompositeDataSource is
                 attached to a parent CompositeDataSource), not user supplied.
 
@@ -524,17 +525,17 @@ class CompositeDataSource(DataSource):
             raise AttributeError('CompositeDataSource has no data sources')
 
         if not query:
-            # don't mess with the query (i.e. convert to a set, as that's done
+            # don't mess with the query (i.e. deduplicate, as that's done
             # within the specific DataSources that are called)
             query = []
 
         all_data = []
+        all_filters = FilterSet()
 
-        all_filters = set()
-        all_filters.update(self.filters)
+        all_filters.add(self.filters)
 
         if _composite_filters:
-            all_filters.update(_composite_filters)
+            all_filters.add(_composite_filters)
 
         # federate query to all attached data sources,
         # pass composite filters to id
diff --git a/stix2/datastore/filesystem.py b/stix2/datastore/filesystem.py
index a6f31cf..c13b02c 100644
--- a/stix2/datastore/filesystem.py
+++ b/stix2/datastore/filesystem.py
@@ -8,7 +8,7 @@ import os
 
 from stix2.core import Bundle, parse
 from stix2.datastore import DataSink, DataSource, DataStoreMixin
-from stix2.datastore.filters import Filter, apply_common_filters
+from stix2.datastore.filters import Filter, FilterSet, apply_common_filters
 from stix2.utils import deduplicate, get_class_hierarchy_names
 
 
@@ -165,7 +165,7 @@ class FileSystemSource(DataSource):
 
         Args:
             stix_id (str): The STIX ID of the STIX object to be retrieved.
-            _composite_filters (set): set of filters passed from the parent
+            _composite_filters (FilterSet): collection of filters passed from the parent
                 CompositeDataSource, not user supplied
             version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
                 None, use latest version.
@@ -195,7 +195,7 @@ class FileSystemSource(DataSource):
 
         Args:
             stix_id (str): The STIX ID of the STIX objects to be retrieved.
-            _composite_filters (set): set of filters passed from the parent
+            _composite_filters (FilterSet): collection of filters passed from the parent
                 CompositeDataSource, not user supplied
             version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
                 None, use latest version.
@@ -217,7 +217,7 @@ class FileSystemSource(DataSource):
 
         Args:
             query (list): list of filters to search on
-            _composite_filters (set): set of filters passed from the
+            _composite_filters (FilterSet): collection of filters passed from the
                 CompositeDataSource, not user supplied
             version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
                 None, use latest version.
@@ -231,20 +231,13 @@ class FileSystemSource(DataSource):
 
         all_data = []
 
-        if query is None:
-            query = set()
-        else:
-            if not isinstance(query, list):
-                # make sure dont make set from a Filter object,
-                # need to make a set from a list of Filter objects (even if just one Filter)
-                query = [query]
-            query = set(query)
+        query = FilterSet(query)
 
         # combine all query filters
         if self.filters:
-            query.update(self.filters)
+            query.add(self.filters)
         if _composite_filters:
-            query.update(_composite_filters)
+            query.add(_composite_filters)
 
         # extract any filters that are for "type" or "id" , as we can then do
         # filtering before reading in the STIX objects. A STIX 'type' filter
@@ -343,8 +336,8 @@ class FileSystemSource(DataSource):
         search space of a FileSystemStore (or FileSystemSink).
 
         """
-        file_filters = set()
+        file_filters = []
         for filter_ in query:
             if filter_.property == "id" or filter_.property == "type":
-                file_filters.add(filter_)
+                file_filters.append(filter_)
         return file_filters
diff --git a/stix2/datastore/filters.py b/stix2/datastore/filters.py
index 10bbeee..ab94e89 100644
--- a/stix2/datastore/filters.py
+++ b/stix2/datastore/filters.py
@@ -4,6 +4,9 @@ Filters for Python STIX 2.0 DataSources, DataSinks, DataStores
 """
 
 import collections
+from datetime import datetime
+
+from stix2.utils import format_datetime
 
 """Supported filter operations"""
 FILTER_OPS = ['=', '!=', 'in', '>', '<', '>=', '<=']
@@ -44,37 +47,6 @@ def _check_filter_components(prop, op, value):
     return True
 
 
-def _assemble_filters(filters1=None, filters2=None):
-    """Assemble a list of filters.
-
-    This can be used to allow certain functions to work correctly no matter if
-    the user provides a single filter or a list of them.
-
-    Args:
-        filters1 (Filter or list, optional): The single Filter or list of Filters to
-            coerce into a list of Filters.
-        filters2 (Filter or list, optional): The single Filter or list of Filters to
-            append to the list of Filters.
-
-    Returns:
-        List of Filters.
-
-    """
-    if filters1 is None:
-        filter_list = []
-    elif not isinstance(filters1, list):
-        filter_list = [filters1]
-    else:
-        filter_list = filters1
-
-    if isinstance(filters2, list):
-        filter_list.extend(filters2)
-    elif filters2 is not None:
-        filter_list.append(filters2)
-
-    return filter_list
-
-
 class Filter(collections.namedtuple("Filter", ['property', 'op', 'value'])):
     """STIX 2 filters that support the querying functionality of STIX 2
     DataStores and DataSources.
@@ -97,6 +69,10 @@ class Filter(collections.namedtuple("Filter", ['property', 'op', 'value'])):
         if isinstance(value, list):
             value = tuple(value)
 
+        if isinstance(value, datetime):
+            # if value is a datetime obj, convert to str
+            value = format_datetime(value)
+
         _check_filter_components(prop, op, value)
 
         self = super(Filter, cls).__new__(cls, prop, op, value)
@@ -112,6 +88,12 @@ class Filter(collections.namedtuple("Filter", ['property', 'op', 'value'])):
             True if property matches the filter,
             False otherwise.
         """
+        if isinstance(stix_obj_property, datetime):
+            # if a datetime obj, convert to str format before comparison
+            # NOTE: this check seems like it should be done upstream
+            # but will put here for now
+            stix_obj_property = format_datetime(stix_obj_property)
+
         if self.op == "=":
             return stix_obj_property == self.value
         elif self.op == "!=":
@@ -183,19 +165,94 @@ def _check_filter(filter_, stix_obj):
         # Check embedded properties, from e.g. granular_markings or external_references
         sub_property = filter_.property.split(".", 1)[1]
         sub_filter = filter_._replace(property=sub_property)
+
         if isinstance(stix_obj[prop], list):
             for elem in stix_obj[prop]:
                 if _check_filter(sub_filter, elem) is True:
                     return True
             return False
+
         else:
             return _check_filter(sub_filter, stix_obj[prop])
+
     elif isinstance(stix_obj[prop], list):
         # Check each item in list property to see if it matches
         for elem in stix_obj[prop]:
             if filter_._check_property(elem) is True:
                 return True
         return False
+
     else:
         # Check if property matches
         return filter_._check_property(stix_obj[prop])
+
+
+class FilterSet(object):
+    """Internal STIX2 class to facilitate the grouping of Filters
+    into sets. The primary motivation for this class came from the problem
+    that Filters that had a dict as a value could not be added to a Python
+    set as dicts are not hashable. Thus this class provides set functionality
+    but internally stores filters in a list.
+     """
+
+    def __init__(self, filters=None):
+        """
+        Args:
+            filters: see FilterSet.add()
+        """
+        self._filters = []
+        if filters:
+            self.add(filters)
+
+    def __iter__(self):
+        """provide iteration functionality of FilterSet"""
+        for f in self._filters:
+            yield f
+
+    def __len__(self):
+        """provide built-in len() utility of FilterSet"""
+        return len(self._filters)
+
+    def add(self, filters=None):
+        """add a Filter, FilterSet, or list of Filters to the FilterSet
+
+        Operates like set, only adding unique stix2.Filters to the FilterSet
+
+        NOTE: method designed to be very accomodating (i.e. even accepting filters=None)
+        as it allows for blind calls (very useful in DataStore)
+
+        Args:
+            filters: stix2.Filter OR list of stix2.Filter OR stix2.FilterSet
+
+        """
+        if not filters:
+            # so add() can be called blindly, useful for
+            # DataStore/Environment usage of filter operations
+            return
+
+        if not isinstance(filters, (FilterSet, list)):
+            filters = [filters]
+
+        for f in filters:
+            if f not in self._filters:
+                self._filters.append(f)
+
+    def remove(self, filters=None):
+        """remove a Filter, list of Filters, or FilterSet from the FilterSet
+
+        NOTE: method designed to be very accomodating (i.e. even accepting filters=None)
+        as it allows for blind calls (very useful in DataStore)
+
+        Args:
+            filters: stix2.Filter OR list of stix2.Filter or stix2.FilterSet
+        """
+        if not filters:
+            # so remove() can be called blindly, useful for
+            # DataStore/Environemnt usage of filter ops
+            return
+
+        if not isinstance(filters, (FilterSet, list)):
+            filters = [filters]
+
+        for f in filters:
+            self._filters.remove(f)
diff --git a/stix2/datastore/memory.py b/stix2/datastore/memory.py
index 1c4ff6c..c43da8b 100644
--- a/stix2/datastore/memory.py
+++ b/stix2/datastore/memory.py
@@ -18,7 +18,7 @@ import os
 from stix2.base import _STIXBase
 from stix2.core import Bundle, parse
 from stix2.datastore import DataSink, DataSource, DataStoreMixin
-from stix2.datastore.filters import Filter, apply_common_filters
+from stix2.datastore.filters import Filter, FilterSet, apply_common_filters
 
 
 def _add(store, stix_data=None, version=None):
@@ -197,7 +197,7 @@ class MemorySource(DataSource):
 
         Args:
             stix_id (str): The STIX ID of the STIX object to be retrieved.
-            _composite_filters (set): set of filters passed from the parent
+            _composite_filters (FilterSet): collection of filters passed from the parent
                 CompositeDataSource, not user supplied
 
         Returns:
@@ -236,7 +236,7 @@ class MemorySource(DataSource):
 
         Args:
             stix_id (str): The STIX ID of the STIX 2 object to retrieve.
-            _composite_filters (set): set of filters passed from the parent
+            _composite_filters (FilterSet): collection of filters passed from the parent
                 CompositeDataSource, not user supplied
 
         Returns:
@@ -258,7 +258,7 @@ class MemorySource(DataSource):
 
         Args:
             query (list): list of filters to search on
-            _composite_filters (set): set of filters passed from the
+            _composite_filters (FilterSet): collection of filters passed from the
                 CompositeDataSource, not user supplied
 
         Returns:
@@ -269,19 +269,15 @@ class MemorySource(DataSource):
 
         """
         if query is None:
-            query = set()
+            query = FilterSet()
         else:
-            if not isinstance(query, list):
-                # make sure don't make set from a Filter object,
-                # need to make a set from a list of Filter objects (even if just one Filter)
-                query = [query]
-            query = set(query)
+            query = FilterSet(query)
 
         # combine all query filters
         if self.filters:
-            query.update(self.filters)
+            query.add(self.filters)
         if _composite_filters:
-            query.update(_composite_filters)
+            query.add(_composite_filters)
 
         # Apply STIX common property filters.
         all_data = list(apply_common_filters(self._data.values(), query))
diff --git a/stix2/datastore/taxii.py b/stix2/datastore/taxii.py
index 0a58763..faa2669 100644
--- a/stix2/datastore/taxii.py
+++ b/stix2/datastore/taxii.py
@@ -6,7 +6,7 @@ from requests.exceptions import HTTPError
 from stix2.base import _STIXBase
 from stix2.core import Bundle, parse
 from stix2.datastore import DataSink, DataSource, DataStoreMixin
-from stix2.datastore.filters import Filter, apply_common_filters
+from stix2.datastore.filters import Filter, FilterSet, apply_common_filters
 from stix2.utils import deduplicate
 
 TAXII_FILTERS = ['added_after', 'id', 'type', 'version']
@@ -120,7 +120,7 @@ class TAXIICollectionSource(DataSource):
 
         Args:
             stix_id (str): The STIX ID of the STIX object to be retrieved.
-            _composite_filters (set): set of filters passed from the parent
+            _composite_filters (FilterSet): collection of filters passed from the parent
                 CompositeDataSource, not user supplied
             version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
                 None, use latest version.
@@ -132,11 +132,12 @@ class TAXIICollectionSource(DataSource):
 
         """
         # combine all query filters
-        query = set()
+        query = FilterSet()
+
         if self.filters:
-            query.update(self.filters)
+            query.add(self.filters)
         if _composite_filters:
-            query.update(_composite_filters)
+            query.add(_composite_filters)
 
         # dont extract TAXII filters from query (to send to TAXII endpoint)
         # as directly retrieveing a STIX object by ID
@@ -164,7 +165,7 @@ class TAXIICollectionSource(DataSource):
 
         Args:
             stix_id (str): The STIX ID of the STIX objects to be retrieved.
-            _composite_filters (set): set of filters passed from the parent
+            _composite_filters (FilterSet): collection of filters passed from the parent
                 CompositeDataSource, not user supplied
             version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
                 None, use latest version.
@@ -198,7 +199,7 @@ class TAXIICollectionSource(DataSource):
 
         Args:
             query (list): list of filters to search on
-            _composite_filters (set): set of filters passed from the
+            _composite_filters (FilterSet): collection of filters passed from the
                 CompositeDataSource, not user supplied
             version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
                 None, use latest version.
@@ -209,20 +210,13 @@ class TAXIICollectionSource(DataSource):
                 parsed into python STIX objects and then returned.
 
         """
-        if query is None:
-            query = set()
-        else:
-            if not isinstance(query, list):
-                # make sure dont make set from a Filter object,
-                # need to make a set from a list of Filter objects (even if just one Filter)
-                query = [query]
-            query = set(query)
+        query = FilterSet(query)
 
         # combine all query filters
         if self.filters:
-            query.update(self.filters)
+            query.add(self.filters)
         if _composite_filters:
-            query.update(_composite_filters)
+            query.add(_composite_filters)
 
         # parse taxii query params (that can be applied remotely)
         taxii_filters = self._parse_taxii_filters(query)
@@ -268,17 +262,16 @@ class TAXIICollectionSource(DataSource):
 
 
         Args:
-            query (set): set of filters to extract which ones are TAXII
+            query (list): list of filters to extract which ones are TAXII
                 specific.
 
-        Returns:
-            taxii_filters (set): set of the TAXII filters
+        Returns: a list of the TAXII filters
 
         """
-        taxii_filters = set()
+        taxii_filters = []
 
         for filter_ in query:
             if filter_.property in TAXII_FILTERS:
-                taxii_filters.add(filter_)
+                taxii_filters.append(filter_)
 
         return taxii_filters
diff --git a/stix2/environment.py b/stix2/environment.py
index a456e95..cc589ae 100644
--- a/stix2/environment.py
+++ b/stix2/environment.py
@@ -158,7 +158,7 @@ class Environment(DataStoreMixin):
     set_default_object_marking_refs.__doc__ = ObjectFactory.set_default_object_marking_refs.__doc__
 
     def add_filters(self, *args, **kwargs):
-        return self.source.filters.update(*args, **kwargs)
+        return self.source.filters.add(*args, **kwargs)
 
     def add_filter(self, *args, **kwargs):
         return self.source.filters.add(*args, **kwargs)
diff --git a/stix2/properties.py b/stix2/properties.py
index 41841b6..a650d55 100644
--- a/stix2/properties.py
+++ b/stix2/properties.py
@@ -12,7 +12,7 @@ from stix2patterns.validator import run_validator
 
 from .base import _STIXBase
 from .exceptions import DictionaryKeyError
-from .utils import get_dict, parse_into_datetime
+from .utils import _get_dict, parse_into_datetime
 
 
 class Property(object):
@@ -129,8 +129,6 @@ class ListProperty(Property):
                 # constructor again
                 result.append(valid)
                 continue
-            elif type(self.contained) is DictionaryProperty:
-                obj_type = dict
             else:
                 obj_type = self.contained
 
@@ -234,7 +232,7 @@ class DictionaryProperty(Property):
 
     def clean(self, value):
         try:
-            dictified = get_dict(value)
+            dictified = _get_dict(value)
         except ValueError:
             raise ValueError("The dictionary property must contain a dictionary")
         if dictified == {}:
diff --git a/stix2/utils.py b/stix2/utils.py
index 4ef3d23..fdb7679 100644
--- a/stix2/utils.py
+++ b/stix2/utils.py
@@ -140,7 +140,7 @@ def parse_into_datetime(value, precision=None):
     return STIXdatetime(ts, precision=precision)
 
 
-def get_dict(data):
+def _get_dict(data):
     """Return data as a dictionary.
 
     Input can be a dictionary, string, or file-like object.
@@ -166,7 +166,7 @@ def get_dict(data):
 def find_property_index(obj, properties, tuple_to_find):
     """Recursively find the property in the object model, return the index
     according to the _properties OrderedDict. If it's a list look for
-    individual objects. Returns and integer indicating its location
+    individual objects.
     """
     from .base import _STIXBase
     try:
@@ -183,11 +183,6 @@ def find_property_index(obj, properties, tuple_to_find):
                                                   tuple_to_find)
                         if val is not None:
                             return val
-                    elif isinstance(item, dict):
-                        for idx, val in enumerate(sorted(item)):
-                            if (tuple_to_find[0] == val and
-                                    item.get(val) == tuple_to_find[1]):
-                                return idx
             elif isinstance(pv, dict):
                 if pv.get(tuple_to_find[0]) is not None:
                     try:
diff --git a/stix2/v20/common.py b/stix2/v20/common.py
index e915df6..0bba3b1 100644
--- a/stix2/v20/common.py
+++ b/stix2/v20/common.py
@@ -7,7 +7,7 @@ from ..markings import _MarkingsMixin
 from ..properties import (HashesProperty, IDProperty, ListProperty, Property,
                           ReferenceProperty, SelectorProperty, StringProperty,
                           TimestampProperty, TypeProperty)
-from ..utils import NOW, get_dict
+from ..utils import NOW, _get_dict
 
 
 class ExternalReference(_STIXBase):
@@ -125,7 +125,7 @@ class MarkingDefinition(_STIXBase, _MarkingsMixin):
                 raise ValueError("definition_type must be a valid marking type")
 
             if not isinstance(kwargs['definition'], marking_type):
-                defn = get_dict(kwargs['definition'])
+                defn = _get_dict(kwargs['definition'])
                 kwargs['definition'] = marking_type(**defn)
 
         super(MarkingDefinition, self).__init__(**kwargs)
diff --git a/stix2/v20/observables.py b/stix2/v20/observables.py
index f6bac2b..254498b 100644
--- a/stix2/v20/observables.py
+++ b/stix2/v20/observables.py
@@ -6,6 +6,7 @@ Observable and do not have a ``_type`` attribute.
 """
 
 from collections import OrderedDict
+import copy
 
 from ..base import _Extension, _Observable, _STIXBase
 from ..exceptions import (AtLeastOnePropertyError, CustomContentError,
@@ -15,7 +16,7 @@ from ..properties import (BinaryProperty, BooleanProperty, DictionaryProperty,
                           HashesProperty, HexProperty, IntegerProperty,
                           ListProperty, ObjectReferenceProperty, Property,
                           StringProperty, TimestampProperty, TypeProperty)
-from ..utils import get_dict
+from ..utils import _get_dict
 
 
 class ObservableProperty(Property):
@@ -24,7 +25,11 @@ class ObservableProperty(Property):
 
     def clean(self, value):
         try:
-            dictified = get_dict(value)
+            dictified = _get_dict(value)
+            # get deep copy since we are going modify the dict and might
+            # modify the original dict as _get_dict() does not return new
+            # dict when passed a dict
+            dictified = copy.deepcopy(dictified)
         except ValueError:
             raise ValueError("The observable property must contain a dictionary")
         if dictified == {}:
@@ -49,7 +54,11 @@ class ExtensionsProperty(DictionaryProperty):
 
     def clean(self, value):
         try:
-            dictified = get_dict(value)
+            dictified = _get_dict(value)
+            # get deep copy since we are going modify the dict and might
+            # modify the original dict as _get_dict() does not return new
+            # dict when passed a dict
+            dictified = copy.deepcopy(dictified)
         except ValueError:
             raise ValueError("The extensions property must contain a dictionary")
         if dictified == {}:
@@ -915,7 +924,12 @@ def parse_observable(data, _valid_refs=None, allow_custom=False):
         An instantiated Python STIX Cyber Observable object.
     """
 
-    obj = get_dict(data)
+    obj = _get_dict(data)
+    # get deep copy since we are going modify the dict and might
+    # modify the original dict as _get_dict() does not return new
+    # dict when passed a dict
+    obj = copy.deepcopy(obj)
+
     obj['_valid_refs'] = _valid_refs or []
 
     if 'type' not in obj:
diff --git a/stix2/workbench.py b/stix2/workbench.py
index b47968e..3697c63 100644
--- a/stix2/workbench.py
+++ b/stix2/workbench.py
@@ -48,7 +48,7 @@ from . import (AlternateDataStream, ArchiveExt, Artifact, AutonomousSystem,  # n
                WindowsPEOptionalHeaderType, WindowsPESection,
                WindowsProcessExt, WindowsRegistryKey, WindowsRegistryValueType,
                WindowsServiceExt, X509Certificate, X509V3ExtenstionsType)
-from .datastore.filters import _assemble_filters
+from .datastore.filters import FilterSet
 
 # Use an implicit MemoryStore
 _environ = Environment(store=MemoryStore())
@@ -156,7 +156,8 @@ def attack_patterns(filters=None):
             the query.
 
     """
-    filter_list = _assemble_filters(filters, [Filter('type', '=', 'attack-pattern')])
+    filter_list = FilterSet(filters)
+    filter_list.add(Filter('type', '=', 'attack-pattern'))
     return query(filter_list)
 
 
@@ -168,7 +169,8 @@ def campaigns(filters=None):
             the query.
 
     """
-    filter_list = _assemble_filters(filters, [Filter('type', '=', 'campaign')])
+    filter_list = FilterSet(filters)
+    filter_list.add(Filter('type', '=', 'campaign'))
     return query(filter_list)
 
 
@@ -180,7 +182,8 @@ def courses_of_action(filters=None):
             the query.
 
     """
-    filter_list = _assemble_filters(filters, [Filter('type', '=', 'course-of-action')])
+    filter_list = FilterSet(filters)
+    filter_list.add(Filter('type', '=', 'course-of-action'))
     return query(filter_list)
 
 
@@ -192,7 +195,8 @@ def identities(filters=None):
             the query.
 
     """
-    filter_list = _assemble_filters(filters, [Filter('type', '=', 'identity')])
+    filter_list = FilterSet(filters)
+    filter_list.add(Filter('type', '=', 'identity'))
     return query(filter_list)
 
 
@@ -204,7 +208,8 @@ def indicators(filters=None):
             the query.
 
     """
-    filter_list = _assemble_filters(filters, [Filter('type', '=', 'indicator')])
+    filter_list = FilterSet(filters)
+    filter_list.add(Filter('type', '=', 'indicator'))
     return query(filter_list)
 
 
@@ -216,7 +221,8 @@ def intrusion_sets(filters=None):
             the query.
 
     """
-    filter_list = _assemble_filters(filters, [Filter('type', '=', 'intrusion-set')])
+    filter_list = FilterSet(filters)
+    filter_list.add(Filter('type', '=', 'intrusion-set'))
     return query(filter_list)
 
 
@@ -228,7 +234,8 @@ def malware(filters=None):
             the query.
 
     """
-    filter_list = _assemble_filters(filters, [Filter('type', '=', 'malware')])
+    filter_list = FilterSet(filters)
+    filter_list.add(Filter('type', '=', 'malware'))
     return query(filter_list)
 
 
@@ -240,7 +247,8 @@ def observed_data(filters=None):
             the query.
 
     """
-    filter_list = _assemble_filters(filters, [Filter('type', '=', 'observed-data')])
+    filter_list = FilterSet(filters)
+    filter_list.add(Filter('type', '=', 'observed-data'))
     return query(filter_list)
 
 
@@ -252,7 +260,8 @@ def reports(filters=None):
             the query.
 
     """
-    filter_list = _assemble_filters(filters, [Filter('type', '=', 'report')])
+    filter_list = FilterSet(filters)
+    filter_list.add(Filter('type', '=', 'report'))
     return query(filter_list)
 
 
@@ -264,7 +273,8 @@ def threat_actors(filters=None):
             the query.
 
     """
-    filter_list = _assemble_filters(filters, [Filter('type', '=', 'threat-actor')])
+    filter_list = FilterSet(filters)
+    filter_list.add(Filter('type', '=', 'threat-actor'))
     return query(filter_list)
 
 
@@ -276,7 +286,8 @@ def tools(filters=None):
             the query.
 
     """
-    filter_list = _assemble_filters(filters, [Filter('type', '=', 'tool')])
+    filter_list = FilterSet(filters)
+    filter_list.add(Filter('type', '=', 'tool'))
     return query(filter_list)
 
 
@@ -288,5 +299,6 @@ def vulnerabilities(filters=None):
             the query.
 
     """
-    filter_list = _assemble_filters(filters, [Filter('type', '=', 'vulnerability')])
+    filter_list = FilterSet(filters)
+    filter_list.add(Filter('type', '=', 'vulnerability'))
     return query(filter_list)


Test patch:
diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py
index 56e578f..f9bb875 100644
--- a/stix2/test/test_custom.py
+++ b/stix2/test/test_custom.py
@@ -516,27 +516,6 @@ def test_custom_extension_wrong_observable_type():
     assert 'Cannot determine extension type' in excinfo.value.reason
 
 
-@pytest.mark.parametrize("data", [
-    """{
-    "keys": [
-        {
-            "test123": 123,
-            "test345": "aaaa"
-        }
-    ]
-}""",
-])
-def test_custom_extension_with_list_and_dict_properties_observable_type(data):
-    @stix2.observables.CustomExtension(stix2.UserAccount, 'some-extension', [
-        ('keys', stix2.properties.ListProperty(stix2.properties.DictionaryProperty, required=True))
-    ])
-    class SomeCustomExtension:
-        pass
-
-    example = SomeCustomExtension(keys=[{'test123': 123, 'test345': 'aaaa'}])
-    assert data == str(example)
-
-
 def test_custom_extension_invalid_observable():
     # These extensions are being applied to improperly-created Observables.
     # The Observable classes should have been created with the CustomObservable decorator.
diff --git a/stix2/test/test_datastore.py b/stix2/test/test_datastore.py
index 8f40401..1c7fa43 100644
--- a/stix2/test/test_datastore.py
+++ b/stix2/test/test_datastore.py
@@ -2,10 +2,11 @@ import pytest
 from taxii2client import Collection
 
 from stix2 import Filter, MemorySink, MemorySource
+from stix2.core import parse
 from stix2.datastore import (CompositeDataSource, DataSink, DataSource,
                              make_id, taxii)
-from stix2.datastore.filters import _assemble_filters, apply_common_filters
-from stix2.utils import deduplicate
+from stix2.datastore.filters import apply_common_filters
+from stix2.utils import STIXdatetime, deduplicate, parse_into_datetime
 
 COLLECTION_URL = 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/'
 
@@ -20,6 +21,108 @@ def collection():
     return Collection(COLLECTION_URL, MockTAXIIClient())
 
 
+stix_objs = [
+    {
+        "created": "2017-01-27T13:49:53.997Z",
+        "description": "\n\nTITLE:\n\tPoison Ivy",
+        "id": "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111",
+        "labels": [
+            "remote-access-trojan"
+        ],
+        "modified": "2017-01-27T13:49:53.997Z",
+        "name": "Poison Ivy",
+        "type": "malware"
+    },
+    {
+        "created": "2014-05-08T09:00:00.000Z",
+        "id": "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade",
+        "labels": [
+            "file-hash-watchlist"
+        ],
+        "modified": "2014-05-08T09:00:00.000Z",
+        "name": "File hash for Poison Ivy variant",
+        "pattern": "[file:hashes.'SHA-256' = 'ef537f25c895bfa782526529a9b63d97aa631564d5d789c2b765448c8635fb6c']",
+        "type": "indicator",
+        "valid_from": "2014-05-08T09:00:00.000000Z"
+    },
+    {
+        "created": "2014-05-08T09:00:00.000Z",
+        "granular_markings": [
+            {
+                "marking_ref": "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed",
+                "selectors": [
+                    "relationship_type"
+                ]
+            }
+        ],
+        "id": "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463",
+        "modified": "2014-05-08T09:00:00.000Z",
+        "object_marking_refs": [
+            "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9"
+        ],
+        "relationship_type": "indicates",
+        "revoked": True,
+        "source_ref": "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade",
+        "target_ref": "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111",
+        "type": "relationship"
+    },
+    {
+        "id": "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef",
+        "created": "2016-02-14T00:00:00.000Z",
+        "created_by_ref": "identity--00000000-0000-0000-0000-b8e91df99dc9",
+        "modified": "2016-02-14T00:00:00.000Z",
+        "type": "vulnerability",
+        "name": "CVE-2014-0160",
+        "description": "The (1) TLS...",
+        "external_references": [
+            {
+                "source_name": "cve",
+                "external_id": "CVE-2014-0160"
+            }
+        ],
+        "labels": ["heartbleed", "has-logo"]
+    },
+    {
+        "type": "observed-data",
+        "id": "observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf",
+        "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff",
+        "created": "2016-04-06T19:58:16.000Z",
+        "modified": "2016-04-06T19:58:16.000Z",
+        "first_observed": "2015-12-21T19:00:00Z",
+        "last_observed": "2015-12-21T19:00:00Z",
+        "number_observed": 1,
+        "objects": {
+            "0": {
+                "type": "file",
+                "name": "HAL 9000.exe"
+            }
+        }
+
+    }
+]
+
+# same as above objects but converted to real Python STIX2 objects
+# to test filters against true Python STIX2 objects
+real_stix_objs = [parse(stix_obj) for stix_obj in stix_objs]
+
+filters = [
+    Filter("type", "!=", "relationship"),
+    Filter("id", "=", "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463"),
+    Filter("labels", "in", "remote-access-trojan"),
+    Filter("created", ">", "2015-01-01T01:00:00.000Z"),
+    Filter("revoked", "=", True),
+    Filter("revoked", "!=", True),
+    Filter("object_marking_refs", "=", "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9"),
+    Filter("granular_markings.selectors", "in", "relationship_type"),
+    Filter("granular_markings.marking_ref", "=", "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed"),
+    Filter("external_references.external_id", "in", "CVE-2014-0160,CVE-2017-6608"),
+    Filter("created_by_ref", "=", "identity--00000000-0000-0000-0000-b8e91df99dc9"),
+    Filter("object_marking_refs", "=", "marking-definition--613f2e26-0000-0000-0000-b8e91df99dc9"),
+    Filter("granular_markings.selectors", "in", "description"),
+    Filter("external_references.source_name", "=", "CVE"),
+    Filter("objects", "=", {"0": {"type": "file", "name": "HAL 9000.exe"}})
+]
+
 IND1 = {
     "created": "2017-01-27T13:49:53.935Z",
     "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f",
@@ -120,6 +223,9 @@ IND8 = {
 STIX_OBJS2 = [IND6, IND7, IND8]
 STIX_OBJS1 = [IND1, IND2, IND3, IND4, IND5]
 
+REAL_STIX_OBJS2 = [parse(IND6), parse(IND7), parse(IND8)]
+REAL_STIX_OBJS1 = [parse(IND1), parse(IND2), parse(IND3), parse(IND4), parse(IND5)]
+
 
 def test_ds_abstract_class_smoke():
     with pytest.raises(TypeError):
@@ -148,12 +254,12 @@ def test_parse_taxii_filters():
         Filter("created_by_ref", "=", "Bane"),
     ]
 
-    taxii_filters_expected = set([
+    taxii_filters_expected = [
         Filter("added_after", "=", "2016-02-01T00:00:01.000Z"),
         Filter("id", "=", "taxii stix object ID"),
         Filter("type", "=", "taxii stix object ID"),
         Filter("version", "=", "first")
-    ])
+    ]
 
     ds = taxii.TAXIICollectionSource(collection)
 
@@ -177,7 +283,7 @@ def test_add_get_remove_filter():
     ds.filters.add(valid_filters[0])
     assert len(ds.filters) == 1
 
-    # Addin the same filter again will have no effect since `filters` uses a set
+    # Addin the same filter again will have no effect since `filters` acts like a set
     ds.filters.add(valid_filters[0])
     assert len(ds.filters) == 1
 
@@ -186,14 +292,14 @@ def test_add_get_remove_filter():
     ds.filters.add(valid_filters[2])
     assert len(ds.filters) == 3
 
-    assert set(valid_filters) == ds.filters
+    assert valid_filters == [f for f in ds.filters]
 
     # remove
     ds.filters.remove(valid_filters[0])
 
     assert len(ds.filters) == 2
 
-    ds.filters.update(valid_filters)
+    ds.filters.add(valid_filters)
 
 
 def test_filter_ops_check():
@@ -236,153 +342,199 @@ def test_filter_type_underscore_check():
     assert "Filter for property 'type' cannot have its value 'oh_underscore'" in str(excinfo.value)
 
 
-def test_apply_common_filters():
-    stix_objs = [
-        {
-            "created": "2017-01-27T13:49:53.997Z",
-            "description": "\n\nTITLE:\n\tPoison Ivy",
-            "id": "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111",
-            "labels": [
-                "remote-access-trojan"
-            ],
-            "modified": "2017-01-27T13:49:53.997Z",
-            "name": "Poison Ivy",
-            "type": "malware"
-        },
-        {
-            "created": "2014-05-08T09:00:00.000Z",
-            "id": "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade",
-            "labels": [
-                "file-hash-watchlist"
-            ],
-            "modified": "2014-05-08T09:00:00.000Z",
-            "name": "File hash for Poison Ivy variant",
-            "pattern": "[file:hashes.'SHA-256' = 'ef537f25c895bfa782526529a9b63d97aa631564d5d789c2b765448c8635fb6c']",
-            "type": "indicator",
-            "valid_from": "2014-05-08T09:00:00.000000Z"
-        },
-        {
-            "created": "2014-05-08T09:00:00.000Z",
-            "granular_markings": [
-                {
-                    "marking_ref": "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed",
-                    "selectors": [
-                        "relationship_type"
-                    ]
-                }
-            ],
-            "id": "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463",
-            "modified": "2014-05-08T09:00:00.000Z",
-            "object_marking_refs": [
-                "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9"
-            ],
-            "relationship_type": "indicates",
-            "revoked": True,
-            "source_ref": "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade",
-            "target_ref": "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111",
-            "type": "relationship"
-        },
-        {
-            "id": "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef",
-            "created": "2016-02-14T00:00:00.000Z",
-            "created_by_ref": "identity--00000000-0000-0000-0000-b8e91df99dc9",
-            "modified": "2016-02-14T00:00:00.000Z",
-            "type": "vulnerability",
-            "name": "CVE-2014-0160",
-            "description": "The (1) TLS...",
-            "external_references": [
-                {
-                    "source_name": "cve",
-                    "external_id": "CVE-2014-0160"
-                }
-            ],
-            "labels": ["heartbleed", "has-logo"]
-        }
-    ]
-
-    filters = [
-        Filter("type", "!=", "relationship"),
-        Filter("id", "=", "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463"),
-        Filter("labels", "in", "remote-access-trojan"),
-        Filter("created", ">", "2015-01-01T01:00:00.000Z"),
-        Filter("revoked", "=", True),
-        Filter("revoked", "!=", True),
-        Filter("object_marking_refs", "=", "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9"),
-        Filter("granular_markings.selectors", "in", "relationship_type"),
-        Filter("granular_markings.marking_ref", "=", "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed"),
-        Filter("external_references.external_id", "in", "CVE-2014-0160,CVE-2017-6608"),
-        Filter("created_by_ref", "=", "identity--00000000-0000-0000-0000-b8e91df99dc9"),
-        Filter("object_marking_refs", "=", "marking-definition--613f2e26-0000-0000-0000-b8e91df99dc9"),
-        Filter("granular_markings.selectors", "in", "description"),
-        Filter("external_references.source_name", "=", "CVE"),
-    ]
-
+def test_apply_common_filters0():
     # "Return any object whose type is not relationship"
     resp = list(apply_common_filters(stix_objs, [filters[0]]))
     ids = [r['id'] for r in resp]
     assert stix_objs[0]['id'] in ids
     assert stix_objs[1]['id'] in ids
     assert stix_objs[3]['id'] in ids
-    assert len(ids) == 3
+    assert len(ids) == 4
 
+    resp = list(apply_common_filters(real_stix_objs, [filters[0]]))
+    ids = [r.id for r in resp]
+    assert real_stix_objs[0].id in ids
+    assert real_stix_objs[1].id in ids
+    assert real_stix_objs[3].id in ids
+    assert len(ids) == 4
+
+
+def test_apply_common_filters1():
     # "Return any object that matched id relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463"
     resp = list(apply_common_filters(stix_objs, [filters[1]]))
     assert resp[0]['id'] == stix_objs[2]['id']
     assert len(resp) == 1
 
+    resp = list(apply_common_filters(real_stix_objs, [filters[1]]))
+    assert resp[0].id == real_stix_objs[2].id
+    assert len(resp) == 1
+
+
+def test_apply_common_filters2():
     # "Return any object that contains remote-access-trojan in labels"
     resp = list(apply_common_filters(stix_objs, [filters[2]]))
     assert resp[0]['id'] == stix_objs[0]['id']
     assert len(resp) == 1
 
+    resp = list(apply_common_filters(real_stix_objs, [filters[2]]))
+    assert resp[0].id == real_stix_objs[0].id
+    assert len(resp) == 1
+
+
+def test_apply_common_filters3():
     # "Return any object created after 2015-01-01T01:00:00.000Z"
     resp = list(apply_common_filters(stix_objs, [filters[3]]))
     assert resp[0]['id'] == stix_objs[0]['id']
-    assert len(resp) == 2
+    assert len(resp) == 3
+
+    resp = list(apply_common_filters(real_stix_objs, [filters[3]]))
+    assert resp[0].id == real_stix_objs[0].id
+    assert len(resp) == 3
 
+
+def test_apply_common_filters4():
     # "Return any revoked object"
     resp = list(apply_common_filters(stix_objs, [filters[4]]))
     assert resp[0]['id'] == stix_objs[2]['id']
     assert len(resp) == 1
 
+    resp = list(apply_common_filters(real_stix_objs, [filters[4]]))
+    assert resp[0].id == real_stix_objs[2].id
+    assert len(resp) == 1
+
+
+def test_apply_common_filters5():
     # "Return any object whose not revoked"
     # Note that if 'revoked' property is not present in object.
     # Currently we can't use such an expression to filter for... :(
     resp = list(apply_common_filters(stix_objs, [filters[5]]))
     assert len(resp) == 0
 
+    resp = list(apply_common_filters(real_stix_objs, [filters[5]]))
+    assert len(resp) == 0
+
+
+def test_apply_common_filters6():
     # "Return any object that matches marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9 in object_marking_refs"
     resp = list(apply_common_filters(stix_objs, [filters[6]]))
     assert resp[0]['id'] == stix_objs[2]['id']
     assert len(resp) == 1
 
+    resp = list(apply_common_filters(real_stix_objs, [filters[6]]))
+    assert resp[0].id == real_stix_objs[2].id
+    assert len(resp) == 1
+
+
+def test_apply_common_filters7():
     # "Return any object that contains relationship_type in their selectors AND
     # also has marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed in marking_ref"
     resp = list(apply_common_filters(stix_objs, [filters[7], filters[8]]))
     assert resp[0]['id'] == stix_objs[2]['id']
     assert len(resp) == 1
 
+    resp = list(apply_common_filters(real_stix_objs, [filters[7], filters[8]]))
+    assert resp[0].id == real_stix_objs[2].id
+    assert len(resp) == 1
+
+
+def test_apply_common_filters8():
     # "Return any object that contains CVE-2014-0160,CVE-2017-6608 in their external_id"
     resp = list(apply_common_filters(stix_objs, [filters[9]]))
     assert resp[0]['id'] == stix_objs[3]['id']
     assert len(resp) == 1
 
+    resp = list(apply_common_filters(real_stix_objs, [filters[9]]))
+    assert resp[0].id == real_stix_objs[3].id
+    assert len(resp) == 1
+
+
+def test_apply_common_filters9():
     # "Return any object that matches created_by_ref identity--00000000-0000-0000-0000-b8e91df99dc9"
     resp = list(apply_common_filters(stix_objs, [filters[10]]))
     assert len(resp) == 1
 
+    resp = list(apply_common_filters(real_stix_objs, [filters[10]]))
+    assert len(resp) == 1
+
+
+def test_apply_common_filters10():
     # "Return any object that matches marking-definition--613f2e26-0000-0000-0000-b8e91df99dc9 in object_marking_refs" (None)
     resp = list(apply_common_filters(stix_objs, [filters[11]]))
     assert len(resp) == 0
 
+    resp = list(apply_common_filters(real_stix_objs, [filters[11]]))
+    assert len(resp) == 0
+
+
+def test_apply_common_filters11():
     # "Return any object that contains description in its selectors" (None)
     resp = list(apply_common_filters(stix_objs, [filters[12]]))
     assert len(resp) == 0
 
-    # "Return any object that object that matches CVE in source_name" (None, case sensitive)
+    resp = list(apply_common_filters(real_stix_objs, [filters[12]]))
+    assert len(resp) == 0
+
+
+def test_apply_common_filters12():
+    # "Return any object that matches CVE in source_name" (None, case sensitive)
     resp = list(apply_common_filters(stix_objs, [filters[13]]))
     assert len(resp) == 0
 
+    resp = list(apply_common_filters(real_stix_objs, [filters[13]]))
+    assert len(resp) == 0
+
+
+def test_apply_common_filters13():
+    # Return any object that matches file object in "objects"
+    resp = list(apply_common_filters(stix_objs, [filters[14]]))
+    assert resp[0]["id"] == stix_objs[4]["id"]
+    assert len(resp) == 1
+    # important additional check to make sure original File dict was
+    # not converted to File object. (this was a deep bug found)
+    assert isinstance(resp[0]["objects"]["0"], dict)
+
+    resp = list(apply_common_filters(real_stix_objs, [filters[14]]))
+    assert resp[0].id == real_stix_objs[4].id
+    assert len(resp) == 1
+
+
+def test_datetime_filter_behavior():
+    """if a filter is initialized with its value being a datetime object
+    OR the STIX object property being filtered on is a datetime object, all
+    resulting comparisons executed are done on the string representations
+    of the datetime objects, as the Filter functionality will convert
+    all datetime objects to there string forms using format_datetim()
+
+    This test makes sure all datetime comparisons are carried out correctly
+    """
+    filter_with_dt_obj = Filter("created", "=", parse_into_datetime("2016-02-14T00:00:00.000Z", "millisecond"))
+    filter_with_str = Filter("created", "=", "2016-02-14T00:00:00.000Z")
+
+    # check that filter value is converted from datetime to str
+    assert isinstance(filter_with_dt_obj.value, str)
+
+    # compare datetime string to filter w/ datetime obj
+    resp = list(apply_common_filters(stix_objs, [filter_with_dt_obj]))
+    assert len(resp) == 1
+    assert resp[0]["id"] == "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef"
+
+    # compare datetime obj to filter w/ datetime obj
+    resp = list(apply_common_filters(real_stix_objs, [filter_with_dt_obj]))
+    assert len(resp) == 1
+    assert resp[0]["id"] == "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef"
+    assert isinstance(resp[0].created, STIXdatetime)  # make sure original object not altered
+
+    # compare datetime string to filter w/ str
+    resp = list(apply_common_filters(stix_objs, [filter_with_str]))
+    assert len(resp) == 1
+    assert resp[0]["id"] == "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef"
+
+    # compare datetime obj to filter w/ str
+    resp = list(apply_common_filters(real_stix_objs, [filter_with_str]))
+    assert len(resp) == 1
+    assert resp[0]["id"] == "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef"
+    assert isinstance(resp[0].created, STIXdatetime)  # make sure original object not altered
+
 
 def test_filters0():
     # "Return any object modified before 2017-01-28T13:49:53.935Z"
@@ -390,6 +542,10 @@ def test_filters0():
     assert resp[0]['id'] == STIX_OBJS2[1]['id']
     assert len(resp) == 2
 
+    resp = list(apply_common_filters(REAL_STIX_OBJS2, [Filter("modified", "<", parse_into_datetime("2017-01-28T13:49:53.935Z"))]))
+    assert resp[0].id == REAL_STIX_OBJS2[1].id
+    assert len(resp) == 2
+
 
 def test_filters1():
     # "Return any object modified after 2017-01-28T13:49:53.935Z"
@@ -397,6 +553,10 @@ def test_filters1():
     assert resp[0]['id'] == STIX_OBJS2[0]['id']
     assert len(resp) == 1
 
+    resp = list(apply_common_filters(REAL_STIX_OBJS2, [Filter("modified", ">", parse_into_datetime("2017-01-28T13:49:53.935Z"))]))
+    assert resp[0].id == REAL_STIX_OBJS2[0].id
+    assert len(resp) == 1
+
 
 def test_filters2():
     # "Return any object modified after or on 2017-01-28T13:49:53.935Z"
@@ -404,6 +564,10 @@ def test_filters2():
     assert resp[0]['id'] == STIX_OBJS2[0]['id']
     assert len(resp) == 3
 
+    resp = list(apply_common_filters(REAL_STIX_OBJS2, [Filter("modified", ">=", parse_into_datetime("2017-01-27T13:49:53.935Z"))]))
+    assert resp[0].id == REAL_STIX_OBJS2[0].id
+    assert len(resp) == 3
+
 
 def test_filters3():
     # "Return any object modified before or on 2017-01-28T13:49:53.935Z"
@@ -411,6 +575,12 @@ def test_filters3():
     assert resp[0]['id'] == STIX_OBJS2[1]['id']
     assert len(resp) == 2
 
+    # "Return any object modified before or on 2017-01-28T13:49:53.935Z"
+    fv = Filter("modified", "<=", parse_into_datetime("2017-01-27T13:49:53.935Z"))
+    resp = list(apply_common_filters(REAL_STIX_OBJS2, [fv]))
+    assert resp[0].id == REAL_STIX_OBJS2[1].id
+    assert len(resp) == 2
+
 
 def test_filters4():
     # Assert invalid Filter cannot be created
@@ -426,6 +596,10 @@ def test_filters5():
     assert resp[0]['id'] == STIX_OBJS2[0]['id']
     assert len(resp) == 1
 
+    resp = list(apply_common_filters(REAL_STIX_OBJS2, [Filter("id", "!=", "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f")]))
+    assert resp[0].id == REAL_STIX_OBJS2[0].id
+    assert len(resp) == 1
+
 
 def test_filters6():
     # Test filtering on non-common property
@@ -433,10 +607,14 @@ def test_filters6():
     assert resp[0]['id'] == STIX_OBJS2[0]['id']
     assert len(resp) == 3
 
+    resp = list(apply_common_filters(REAL_STIX_OBJS2, [Filter("name", "=", "Malicious site hosting downloader")]))
+    assert resp[0].id == REAL_STIX_OBJS2[0].id
+    assert len(resp) == 3
+
 
 def test_filters7():
     # Test filtering on embedded property
-    stix_objects = list(STIX_OBJS2) + [{
+    obsvd_data_obj = {
         "type": "observed-data",
         "id": "observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf",
         "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff",
@@ -467,19 +645,18 @@ def test_filters7():
                 }
             }
         }
-    }]
+    }
+
+    stix_objects = list(STIX_OBJS2) + [obsvd_data_obj]
+    real_stix_objects = list(REAL_STIX_OBJS2) + [parse(obsvd_data_obj)]
+
     resp = list(apply_common_filters(stix_objects, [Filter("objects.0.extensions.pdf-ext.version", ">", "1.2")]))
     assert resp[0]['id'] == stix_objects[3]['id']
     assert len(resp) == 1
 
-
-def test_assemble_filters():
-    filter1 = Filter("name", "=", "Malicious site hosting downloader")
-    filter2 = Filter("modified", ">", "2017-01-28T13:49:53.935Z")
-    result = _assemble_filters(filter1, filter2)
-    assert len(result) == 2
-    assert result[0].property == 'name'
-    assert result[1].property == 'modified'
+    resp = list(apply_common_filters(real_stix_objects, [Filter("objects.0.extensions.pdf-ext.version", ">", "1.2")]))
+    assert resp[0].id == real_stix_objects[3].id
+    assert len(resp) == 1
 
 
 def test_deduplicate():
@@ -557,7 +734,7 @@ def test_composite_datasource_operations():
         Filter("valid_from", "=", "2017-01-27T13:49:53.935382Z")
     ]
 
-    cds1.filters.update(query2)
+    cds1.filters.add(query2)
 
     results = cds1.query(query1)
 
diff --git a/stix2/test/test_properties.py b/stix2/test/test_properties.py
index 16ff06a..34edc96 100644
--- a/stix2/test/test_properties.py
+++ b/stix2/test/test_properties.py
@@ -1,6 +1,6 @@
 import pytest
 
-from stix2 import CustomObject, EmailMIMEComponent, ExtensionsProperty, TCPExt
+from stix2 import EmailMIMEComponent, ExtensionsProperty, TCPExt
 from stix2.exceptions import AtLeastOnePropertyError, DictionaryKeyError
 from stix2.properties import (BinaryProperty, BooleanProperty,
                               DictionaryProperty, EmbeddedObjectProperty,
@@ -266,17 +266,6 @@ def test_dictionary_property_invalid(d):
     assert str(excinfo.value) == d[1]
 
 
-def test_property_list_of_dictionary():
-    @CustomObject('x-new-obj', [
-        ('property1', ListProperty(DictionaryProperty(), required=True)),
-    ])
-    class NewObj():
-        pass
-
-    test_obj = NewObj(property1=[{'foo': 'bar'}])
-    assert test_obj.property1[0]['foo'] == 'bar'
-
-
 @pytest.mark.parametrize("value", [
     {"sha256": "6db12788c37247f2316052e142f42f4b259d6561751e5f401a1ae2a6df9c674b"},
     [('MD5', '2dfb1bcc980200c6706feee399d41b3f'), ('RIPEMD-160', 'b3a8cd8a27c90af79b3c81754f267780f443dfef')],
diff --git a/stix2/test/test_utils.py b/stix2/test/test_utils.py
index cbe5b0f..84b2476 100644
--- a/stix2/test/test_utils.py
+++ b/stix2/test/test_utils.py
@@ -62,7 +62,7 @@ def test_parse_datetime_invalid(ts):
     [("a", 1,)],
 ])
 def test_get_dict(data):
-    assert stix2.utils.get_dict(data)
+    assert stix2.utils._get_dict(data)
 
 
 @pytest.mark.parametrize('data', [
@@ -73,7 +73,7 @@ def test_get_dict(data):
 ])
 def test_get_dict_invalid(data):
     with pytest.raises(ValueError):
-        stix2.utils.get_dict(data)
+        stix2.utils._get_dict(data)
 
 
 @pytest.mark.parametrize('stix_id, typ', [