Repository: oasis-open/cti-python-stix2

Issue:
Create "Environment" class
This is an abstraction that will encompass a data source, data sink, and object factory, and potentially other state, in a unified interface for interacting with external STIX repositories. 

Discussion:

Patch:
diff --git a/stix2/__init__.py b/stix2/__init__.py
index c2aae2e..35b65b0 100644
--- a/stix2/__init__.py
+++ b/stix2/__init__.py
@@ -7,7 +7,7 @@ from .common import (TLP_AMBER, TLP_GREEN, TLP_RED, TLP_WHITE, CustomMarking,
                      ExternalReference, GranularMarking, KillChainPhase,
                      MarkingDefinition, StatementMarking, TLPMarking)
 from .core import Bundle, _register_type, parse
-from .environment import ObjectFactory
+from .environment import Environment, ObjectFactory
 from .observables import (URL, AlternateDataStream, ArchiveExt, Artifact,
                           AutonomousSystem, CustomObservable, Directory,
                           DomainName, EmailAddress, EmailMessage,
@@ -41,6 +41,13 @@ from .patterns import (AndBooleanExpression, AndObservationExpression,
 from .sdo import (AttackPattern, Campaign, CourseOfAction, CustomObject,
                   Identity, Indicator, IntrusionSet, Malware, ObservedData,
                   Report, ThreatActor, Tool, Vulnerability)
+from .sources import CompositeDataSource
+from .sources.filesystem import (FileSystemSink, FileSystemSource,
+                                 FileSystemStore)
+from .sources.filters import Filter
+from .sources.memory import MemorySink, MemorySource, MemoryStore
+from .sources.taxii import (TAXIICollectionSink, TAXIICollectionSource,
+                            TAXIICollectionStore)
 from .sro import Relationship, Sighting
 from .utils import get_dict, new_version, revoke
 from .version import __version__
diff --git a/stix2/core.py b/stix2/core.py
index be2a53d..0271e34 100644
--- a/stix2/core.py
+++ b/stix2/core.py
@@ -75,13 +75,13 @@ def parse(data, allow_custom=False):
     """Deserialize a string or file-like object into a STIX object.
 
     Args:
-        data: The STIX 2 string to be parsed.
+        data (str, dict, file-like object): The STIX 2 content to be parsed.
         allow_custom (bool): Whether to allow custom properties or not. Default: False.
 
     Returns:
         An instantiated Python STIX object.
-    """
 
+    """
     obj = get_dict(data)
 
     if 'type' not in obj:
@@ -96,6 +96,6 @@ def parse(data, allow_custom=False):
 
 def _register_type(new_type):
     """Register a custom STIX Object type.
-    """
 
+    """
     OBJ_MAP[new_type._type] = new_type
diff --git a/stix2/environment.py b/stix2/environment.py
index f855755..8e24c9b 100644
--- a/stix2/environment.py
+++ b/stix2/environment.py
@@ -1,22 +1,22 @@
 import copy
 
+from .core import parse as _parse
+from .sources import CompositeDataSource, DataSource, DataStore
 
-class ObjectFactory(object):
-    """Object Factory
 
-    Used to easily create STIX objects with default values for certain
-    properties.
+class ObjectFactory(object):
+    """Easily create STIX objects with default values for certain properties.
 
     Args:
-        created_by_ref: Default created_by_ref value to apply to all
+        created_by_ref (optional): Default created_by_ref value to apply to all
             objects created by this factory.
-        created: Default created value to apply to all
+        created (optional): Default created value to apply to all
             objects created by this factory.
-        external_references: Default `external_references` value to apply
+        external_references (optional): Default `external_references` value to apply
             to all objects created by this factory.
-        object_marking_refs: Default `object_marking_refs` value to apply
+        object_marking_refs (optional): Default `object_marking_refs` value to apply
             to all objects created by this factory.
-        list_append: When a default is set for a list property like
+        list_append (bool, optional): When a default is set for a list property like
             `external_references` or `object_marking_refs` and a value for
             that property is passed into `create()`, if this is set to True,
             that value will be added to the list alongside the default. If
@@ -44,6 +44,13 @@ class ObjectFactory(object):
         self._list_properties = ['external_references', 'object_marking_refs']
 
     def create(self, cls, **kwargs):
+        """Create a STIX object using object factory defaults.
+
+        Args:
+            cls: the python-stix2 class of the object to be created (eg. Indicator)
+            **kwargs: The property/value pairs of the STIX object to be created
+        """
+
         # Use self.defaults as the base, but update with any explicit args
         # provided by the user.
         properties = copy.deepcopy(self._defaults)
@@ -66,3 +73,84 @@ class ObjectFactory(object):
             properties.update(**kwargs)
 
         return cls(**properties)
+
+
+class Environment(object):
+    """
+
+    Args:
+        factory (ObjectFactory, optional): Factory for creating objects with common
+            defaults for certain properties.
+        store (DataStore, optional): Data store providing the source and sink for the
+            environment.
+        source (DataSource, optional): Source for retrieving STIX objects.
+        sink (DataSink, optional): Destination for saving STIX objects.
+            Invalid if `store` is also provided.
+    """
+
+    def __init__(self, factory=ObjectFactory(), store=None, source=None, sink=None):
+        self.factory = factory
+        self.source = CompositeDataSource()
+        if store:
+            self.source.add_data_source(store.source)
+            self.sink = store.sink
+        if source:
+            self.source.add_data_source(source)
+        if sink:
+            if store:
+                raise ValueError("Data store already provided! Environment may only have one data sink.")
+            self.sink = sink
+
+    def create(self, *args, **kwargs):
+        return self.factory.create(*args, **kwargs)
+    create.__doc__ = ObjectFactory.create.__doc__
+
+    def get(self, *args, **kwargs):
+        try:
+            return self.source.get(*args, **kwargs)
+        except AttributeError:
+            raise AttributeError('Environment has no data source to query')
+    get.__doc__ = DataStore.get.__doc__
+
+    def all_versions(self, *args, **kwargs):
+        """Retrieve all versions of a single STIX object by ID.
+        """
+        try:
+            return self.source.all_versions(*args, **kwargs)
+        except AttributeError:
+            raise AttributeError('Environment has no data source to query')
+    all_versions.__doc__ = DataStore.all_versions.__doc__
+
+    def query(self, *args, **kwargs):
+        """Retrieve STIX objects matching a set of filters.
+        """
+        try:
+            return self.source.query(*args, **kwargs)
+        except AttributeError:
+            raise AttributeError('Environment has no data source to query')
+    query.__doc__ = DataStore.query.__doc__
+
+    def add_filters(self, *args, **kwargs):
+        try:
+            return self.source.add_filters(*args, **kwargs)
+        except AttributeError:
+            raise AttributeError('Environment has no data source')
+    add_filters.__doc__ = DataSource.add_filters.__doc__
+
+    def add_filter(self, *args, **kwargs):
+        try:
+            return self.source.add_filter(*args, **kwargs)
+        except AttributeError:
+            raise AttributeError('Environment has no data source')
+    add_filter.__doc__ = DataSource.add_filter.__doc__
+
+    def add(self, *args, **kwargs):
+        try:
+            return self.sink.add(*args, **kwargs)
+        except AttributeError:
+            raise AttributeError('Environment has no data sink to put objects in')
+    add.__doc__ = DataStore.add.__doc__
+
+    def parse(self, *args, **kwargs):
+        return _parse(*args, **kwargs)
+    parse.__doc__ = _parse.__doc__
diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py
index b50fd1d..cb6e5b5 100644
--- a/stix2/sources/__init__.py
+++ b/stix2/sources/__init__.py
@@ -5,7 +5,7 @@ Classes:
     DataStore
     DataSink
     DataSource
-    STIXCommonPropertyFilters
+    CompositeDataSource
 
 TODO:Test everything
 
@@ -45,30 +45,29 @@ class DataStore(object):
         self.sink = sink
 
     def get(self, stix_id):
-        """
+        """Retrieve the most recent version of a single STIX object by ID.
+
         Notes:
             Translate API get() call to the appropriate DataSource call.
 
         Args:
-            stix_id (str): the id of the STIX 2.0 object to retrieve. Should
-                return a single object, the most recent version of the object
-                specified by the "id".
+            stix_id (str): the id of the STIX 2.0 object to retrieve.
 
         Returns:
-            stix_obj (dictionary): the STIX object to be returned
+            stix_obj (dictionary): the single most recent version of the STIX
+                object specified by the "id".
 
         """
         return self.source.get(stix_id)
 
     def all_versions(self, stix_id):
-        """
+        """Retrieve all versions of a single STIX object by ID.
+
         Implement:
             Translate all_versions() call to the appropriate DataSource call
 
         Args:
-            stix_id (str): the id of the STIX 2.0 object to retrieve. Should
-                return a single object, the most recent version of the object
-                specified by the "id".
+            stix_id (str): the id of the STIX 2.0 object to retrieve.
 
         Returns:
             stix_objs (list): a list of STIX objects (where each object is a
@@ -78,7 +77,8 @@ class DataStore(object):
         return self.source.all_versions(stix_id)
 
     def query(self, query):
-        """
+        """Retrieve STIX objects matching a set of filters.
+
         Notes:
             Implement the specific data source API calls, processing,
             functionality required for retrieving query from the data source.
@@ -95,10 +95,15 @@ class DataStore(object):
         return self.source.query(query=query)
 
     def add(self, stix_objs):
-        """
+        """Store STIX objects.
+
         Notes:
             Translate add() to the appropriate DataSink call().
 
+        Args:
+            stix_objs (list): a list of STIX objects (where each object is a
+                STIX object)
+
         """
         return self.sink.add(stix_objs)
 
@@ -116,11 +121,16 @@ class DataSink(object):
         self.id = make_id()
 
     def add(self, stix_objs):
-        """
+        """Store STIX objects.
+
         Notes:
             Implement the specific data sink API calls, processing,
             functionality required for adding data to the sink
 
+        Args:
+            stix_objs (list): a list of STIX objects (where each object is a
+                STIX object)
+
         """
         raise NotImplementedError()
 
@@ -201,16 +211,22 @@ class DataSource(object):
         raise NotImplementedError()
 
     def add_filters(self, filters):
-        """Add multiple filters to the DataSource.
+        """Add multiple filters to be applied to all queries for STIX objects.
 
         Args:
             filters (list): list of filters (dict) to add to the Data Source.
+
         """
         for filter in filters:
             self.add_filter(filter)
 
     def add_filter(self, filter):
-        """Add a filter."""
+        """Add a filter to be applied to all queries for STIX objects.
+
+        Args:
+            filter: filter to add to the Data Source.
+
+        """
         # check filter field is a supported STIX 2.0 common field
         if filter.field not in STIX_COMMON_FIELDS:
             raise ValueError("Filter 'field' is not a STIX 2.0 common property. Currently only STIX object common properties supported")
@@ -226,7 +242,7 @@ class DataSource(object):
         self.filters.add(filter)
 
     def apply_common_filters(self, stix_objs, query):
-        """Evaluates filters against a set of STIX 2.0 objects
+        """Evaluate filters against a set of STIX 2.0 objects.
 
         Supports only STIX 2.0 common property fields
 
@@ -300,11 +316,10 @@ class DataSource(object):
 
 
 class CompositeDataSource(DataSource):
-    """Composite Data Source
+    """Controller for all the defined/configured STIX Data Sources.
 
-    Acts as a controller for all the defined/configured STIX Data Sources
-    e.g. a user can define n Data Sources - creating Data Source (objects)
-    for each. There is only one instance of this for any python STIX 2.0
+    E.g. a user can define n Data Sources - creating Data Source (objects)
+    for each. There is only one instance of this for any Python STIX 2.0
     application.
 
     Attributes:
@@ -314,8 +329,7 @@ class CompositeDataSource(DataSource):
 
     """
     def __init__(self):
-        """
-        Creates a new STIX Data Source.
+        """Create a new STIX Data Source.
 
         Args:
             name (str): A string containing the name to attach in the
@@ -348,6 +362,9 @@ class CompositeDataSource(DataSource):
             stix_obj (dict): the STIX object to be returned.
 
         """
+        if not self.get_all_data_sources():
+            raise AttributeError('CompositeDataSource has no data sources')
+
         all_data = []
 
         # for every configured Data Source, call its retrieve handler
@@ -384,6 +401,9 @@ class CompositeDataSource(DataSource):
             all_data (list): list of STIX objects that have the specified id
 
         """
+        if not self.get_all_data_sources():
+            raise AttributeError('CompositeDataSource has no data sources')
+
         all_data = []
         all_filters = self.filters
 
@@ -403,9 +423,7 @@ class CompositeDataSource(DataSource):
         return all_data
 
     def query(self, query=None, _composite_filters=None):
-        """Composite data source query
-
-        Federate the query to all Data Sources attached to the
+        """Federate the query to all Data Sources attached to the
         Composite Data Source.
 
         Args:
@@ -418,6 +436,9 @@ class CompositeDataSource(DataSource):
             all_data (list): list of STIX objects to be returned
 
         """
+        if not self.get_all_data_sources():
+            raise AttributeError('CompositeDataSource has no data sources')
+
         if not query:
             query = []
 
@@ -448,6 +469,8 @@ class CompositeDataSource(DataSource):
                 to the Composite Data Source
 
         """
+        if not isinstance(data_sources, list):
+            data_sources = [data_sources]
         for ds in data_sources:
             if issubclass(ds.__class__, DataSource):
                 if ds.id in self.data_sources:
diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py
index 9eca969..95d053c 100644
--- a/stix2/sources/memory.py
+++ b/stix2/sources/memory.py
@@ -22,33 +22,22 @@ import collections
 import json
 import os
 
-from stix2validator import validate_instance
-
 from stix2 import Bundle
 from stix2.sources import DataSink, DataSource, DataStore
 from stix2.sources.filters import Filter
 
 
-def _add(store, stix_data):
+def _add(store, stix_data=None):
     """Adds stix objects to MemoryStore/Source/Sink."""
     if isinstance(stix_data, collections.Mapping):
         # stix objects are in a bundle
-        # verify STIX json data
-        r = validate_instance(stix_data)
         # make dictionary of the objects for easy lookup
-        if r.is_valid:
-            for stix_obj in stix_data["objects"]:
-                store.data[stix_obj["id"]] = stix_obj
-        else:
-            raise ValueError("Error: data passed was found to not be valid by the STIX 2 Validator: \n%s", r.as_dict())
+        for stix_obj in stix_data["objects"]:
+            store.data[stix_obj["id"]] = stix_obj
     elif isinstance(stix_data, list):
         # stix objects are in a list
         for stix_obj in stix_data:
-            r = validate_instance(stix_obj)
-            if r.is_valid:
-                store.data[stix_obj["id"]] = stix_obj
-            else:
-                raise ValueError("Error: STIX object %s is not valid under STIX 2 validator.\n%s", stix_obj["id"], r)
+            store.data[stix_obj["id"]] = stix_obj
     else:
         raise ValueError("stix_data must be in bundle format or raw list")
 
@@ -56,7 +45,7 @@ def _add(store, stix_data):
 class MemoryStore(DataStore):
     """
     """
-    def __init__(self, stix_data):
+    def __init__(self, stix_data=None):
         """
         Notes:
             It doesn't make sense to create a MemoryStore by passing
@@ -83,7 +72,7 @@ class MemoryStore(DataStore):
 class MemorySink(DataSink):
     """
     """
-    def __init__(self, stix_data, _store=False):
+    def __init__(self, stix_data=None, _store=False):
         """
         Args:
             stix_data (dictionary OR list): valid STIX 2.0 content in
@@ -114,7 +103,7 @@ class MemorySink(DataSink):
 
 class MemorySource(DataSource):
 
-    def __init__(self, stix_data, _store=False):
+    def __init__(self, stix_data=None, _store=False):
         """
         Args:
             stix_data (dictionary OR list): valid STIX 2.0 content in
@@ -193,10 +182,5 @@ class MemorySource(DataSource):
         file_path = os.path.abspath(file_path)
         stix_data = json.load(open(file_path, "r"))
 
-        r = validate_instance(stix_data)
-
-        if r.is_valid:
-            for stix_obj in stix_data["objects"]:
-                    self.data[stix_obj["id"]] = stix_obj
-
-        raise ValueError("Error: STIX data loaded from file (%s) was found to not be validated by STIX 2 Validator.\n%s", file_path, r)
+        for stix_obj in stix_data["objects"]:
+            self.data[stix_obj["id"]] = stix_obj


Test patch:
diff --git a/stix2/test/test_environment.py b/stix2/test/test_environment.py
index 9be8101..0871bb5 100644
--- a/stix2/test/test_environment.py
+++ b/stix2/test/test_environment.py
@@ -1,7 +1,9 @@
+import pytest
+
 import stix2
 
-from .constants import (FAKE_TIME, IDENTITY_ID, IDENTITY_KWARGS,
-                        INDICATOR_KWARGS)
+from .constants import (FAKE_TIME, IDENTITY_ID, IDENTITY_KWARGS, INDICATOR_ID,
+                        INDICATOR_KWARGS, MALWARE_ID)
 
 
 def test_object_factory_created_by_ref_str():
@@ -81,3 +83,106 @@ def test_object_factory_list_replace():
     ind = factory.create(stix2.Indicator, external_references=ext_ref2, **INDICATOR_KWARGS)
     assert len(ind.external_references) == 1
     assert ind.external_references[0].source_name == "Yet Another Threat Report"
+
+
+def test_environment_functions():
+    env = stix2.Environment(stix2.ObjectFactory(created_by_ref=IDENTITY_ID),
+                            stix2.MemoryStore())
+
+    # Create a STIX object
+    ind = env.create(stix2.Indicator, id=INDICATOR_ID, **INDICATOR_KWARGS)
+    assert ind.created_by_ref == IDENTITY_ID
+
+    # Add objects to datastore
+    ind2 = ind.new_version(labels=['benign'])
+    env.add([ind, ind2])
+
+    # Get both versions of the object
+    resp = env.all_versions(INDICATOR_ID)
+    assert len(resp) == 1  # should be 2, but MemoryStore only keeps 1 version of objects
+
+    # Get just the most recent version of the object
+    resp = env.get(INDICATOR_ID)
+    assert resp['labels'][0] == 'benign'
+
+    # Search on something other than id
+    query = [stix2.Filter('type', '=', 'vulnerability')]
+    resp = env.query(query)
+    assert len(resp) == 0
+
+    # See different results after adding filters to the environment
+    env.add_filters([stix2.Filter('type', '=', 'indicator'),
+                    stix2.Filter('created_by_ref', '=', IDENTITY_ID)])
+    env.add_filter(stix2.Filter('labels', '=', 'benign'))  # should be 'malicious-activity'
+    resp = env.get(INDICATOR_ID)
+    assert resp['labels'][0] == 'benign'  # should be 'malicious-activity'
+
+
+def test_environment_source_and_sink():
+    ind = stix2.Indicator(id=INDICATOR_ID, **INDICATOR_KWARGS)
+    env = stix2.Environment(source=stix2.MemorySource([ind]), sink=stix2.MemorySink([ind]))
+    assert env.get(INDICATOR_ID).labels[0] == 'malicious-activity'
+
+
+def test_environment_datastore_and_sink():
+    with pytest.raises(ValueError) as excinfo:
+        stix2.Environment(factory=stix2.ObjectFactory(),
+                          store=stix2.MemoryStore(), sink=stix2.MemorySink)
+    assert 'Data store already provided' in str(excinfo.value)
+
+
+def test_environment_no_datastore():
+    env = stix2.Environment(factory=stix2.ObjectFactory())
+
+    with pytest.raises(AttributeError) as excinfo:
+        env.add(stix2.Indicator(**INDICATOR_KWARGS))
+    assert 'Environment has no data sink to put objects in' in str(excinfo.value)
+
+    with pytest.raises(AttributeError) as excinfo:
+        env.get(INDICATOR_ID)
+    assert 'Environment has no data source' in str(excinfo.value)
+
+    with pytest.raises(AttributeError) as excinfo:
+        env.all_versions(INDICATOR_ID)
+    assert 'Environment has no data source' in str(excinfo.value)
+
+    with pytest.raises(AttributeError) as excinfo:
+        env.query(INDICATOR_ID)
+    assert 'Environment has no data source' in str(excinfo.value)
+
+    with pytest.raises(AttributeError) as excinfo:
+        env.add_filters(INDICATOR_ID)
+    assert 'Environment has no data source' in str(excinfo.value)
+
+    with pytest.raises(AttributeError) as excinfo:
+        env.add_filter(INDICATOR_ID)
+    assert 'Environment has no data source' in str(excinfo.value)
+
+
+def test_environment_datastore_and_no_object_factory():
+    # Uses a default object factory
+    env = stix2.Environment(store=stix2.MemoryStore())
+    ind = env.create(stix2.Indicator, id=INDICATOR_ID, **INDICATOR_KWARGS)
+    assert ind.id == INDICATOR_ID
+
+
+def test_parse_malware():
+    env = stix2.Environment()
+    data = """{
+        "type": "malware",
+        "id": "malware--fedcba98-7654-3210-fedc-ba9876543210",
+        "created": "2017-01-01T12:34:56.000Z",
+        "modified": "2017-01-01T12:34:56.000Z",
+        "name": "Cryptolocker",
+        "labels": [
+            "ransomware"
+        ]
+    }"""
+    mal = env.parse(data)
+
+    assert mal.type == 'malware'
+    assert mal.id == MALWARE_ID
+    assert mal.created == FAKE_TIME
+    assert mal.modified == FAKE_TIME
+    assert mal.labels == ['ransomware']
+    assert mal.name == "Cryptolocker"