Repository: oasis-open/cti-python-stix2

Issue:
Create an Extension with Dict annidate inside List
Hi,
I'm trying to create a CyberObservable Extension for UserAccount which have to contain a DictionaryProperty() inside a ListProperty(). It is possible? Because when I try to create an extension like this one

```
@CustomExtension(UserAccount, 'ssh_keys', {
   keys': ListProperty(DictionaryProperty(), required=True)
 })
 class SSHKeysExtension:
     pass
```

and use it with example = SSHKeysExtension(keys=[{'test123':123, 'test345','aaaa'}])
I obtain a lot of strange errors (the library seems to interpreter the dict as parameters for __init__()


Comments:

Patch:
diff --git a/stix2/base.py b/stix2/base.py
index 898f489..3219007 100644
--- a/stix2/base.py
+++ b/stix2/base.py
@@ -6,9 +6,9 @@ import datetime as dt
 
 import simplejson as json
 
-from .exceptions import (AtLeastOnePropertyError, DependentPropertiesError,
-                         ExtraPropertiesError, ImmutableError,
-                         InvalidObjRefError, InvalidValueError,
+from .exceptions import (AtLeastOnePropertyError, CustomContentError,
+                         DependentPropertiesError, ExtraPropertiesError,
+                         ImmutableError, InvalidObjRefError, InvalidValueError,
                          MissingPropertiesError,
                          MutuallyExclusivePropertiesError)
 from .markings.utils import validate
@@ -61,6 +61,8 @@ class _STIXBase(collections.Mapping):
             try:
                 kwargs[prop_name] = prop.clean(kwargs[prop_name])
             except ValueError as exc:
+                if self.__allow_custom and isinstance(exc, CustomContentError):
+                    return
                 raise InvalidValueError(self.__class__, prop_name, reason=str(exc))
 
     # interproperty constraint methods
@@ -97,6 +99,7 @@ class _STIXBase(collections.Mapping):
 
     def __init__(self, allow_custom=False, **kwargs):
         cls = self.__class__
+        self.__allow_custom = allow_custom
 
         # Use the same timestamp for any auto-generated datetimes
         self.__now = get_timestamp()
diff --git a/stix2/exceptions.py b/stix2/exceptions.py
index 841a8e9..79c5a81 100644
--- a/stix2/exceptions.py
+++ b/stix2/exceptions.py
@@ -163,6 +163,13 @@ class ParseError(STIXError, ValueError):
         super(ParseError, self).__init__(msg)
 
 
+class CustomContentError(STIXError, ValueError):
+    """Custom STIX Content (SDO, Observable, Extension, etc.) detected."""
+
+    def __init__(self, msg):
+        super(CustomContentError, self).__init__(msg)
+
+
 class InvalidSelectorError(STIXError, AssertionError):
     """Granular Marking selector violation. The selector must resolve into an existing STIX object property."""
 
diff --git a/stix2/properties.py b/stix2/properties.py
index ca7f04c..41841b6 100644
--- a/stix2/properties.py
+++ b/stix2/properties.py
@@ -129,6 +129,8 @@ class ListProperty(Property):
                 # constructor again
                 result.append(valid)
                 continue
+            elif type(self.contained) is DictionaryProperty:
+                obj_type = dict
             else:
                 obj_type = self.contained
 
diff --git a/stix2/utils.py b/stix2/utils.py
index 9febd78..4ef3d23 100644
--- a/stix2/utils.py
+++ b/stix2/utils.py
@@ -166,7 +166,7 @@ def get_dict(data):
 def find_property_index(obj, properties, tuple_to_find):
     """Recursively find the property in the object model, return the index
     according to the _properties OrderedDict. If it's a list look for
-    individual objects.
+    individual objects. Returns and integer indicating its location
     """
     from .base import _STIXBase
     try:
@@ -183,6 +183,11 @@ def find_property_index(obj, properties, tuple_to_find):
                                                   tuple_to_find)
                         if val is not None:
                             return val
+                    elif isinstance(item, dict):
+                        for idx, val in enumerate(sorted(item)):
+                            if (tuple_to_find[0] == val and
+                                    item.get(val) == tuple_to_find[1]):
+                                return idx
             elif isinstance(pv, dict):
                 if pv.get(tuple_to_find[0]) is not None:
                     try:
diff --git a/stix2/v20/observables.py b/stix2/v20/observables.py
index 39a8f19..f6bac2b 100644
--- a/stix2/v20/observables.py
+++ b/stix2/v20/observables.py
@@ -8,8 +8,8 @@ Observable and do not have a ``_type`` attribute.
 from collections import OrderedDict
 
 from ..base import _Extension, _Observable, _STIXBase
-from ..exceptions import (AtLeastOnePropertyError, DependentPropertiesError,
-                          ParseError)
+from ..exceptions import (AtLeastOnePropertyError, CustomContentError,
+                          DependentPropertiesError, ParseError)
 from ..properties import (BinaryProperty, BooleanProperty, DictionaryProperty,
                           EmbeddedObjectProperty, EnumProperty, FloatProperty,
                           HashesProperty, HexProperty, IntegerProperty,
@@ -67,7 +67,7 @@ class ExtensionsProperty(DictionaryProperty):
                     else:
                         raise ValueError("Cannot determine extension type.")
                 else:
-                    raise ValueError("The key used in the extensions dictionary is not an extension type name")
+                    raise CustomContentError("Can't parse unknown extension type: {}".format(key))
         else:
             raise ValueError("The enclosing type '%s' has no extensions defined" % self.enclosing_type)
         return dictified
@@ -923,15 +923,23 @@ def parse_observable(data, _valid_refs=None, allow_custom=False):
     try:
         obj_class = OBJ_MAP_OBSERVABLE[obj['type']]
     except KeyError:
-        raise ParseError("Can't parse unknown observable type '%s'! For custom observables, "
-                         "use the CustomObservable decorator." % obj['type'])
+        if allow_custom:
+            # flag allows for unknown custom objects too, but will not
+            # be parsed into STIX observable object, just returned as is
+            return obj
+        raise CustomContentError("Can't parse unknown observable type '%s'! For custom observables, "
+                                 "use the CustomObservable decorator." % obj['type'])
 
     if 'extensions' in obj and obj['type'] in EXT_MAP:
         for name, ext in obj['extensions'].items():
-            if name not in EXT_MAP[obj['type']]:
-                raise ParseError("Can't parse Unknown extension type '%s' for observable type '%s'!" % (name, obj['type']))
-            ext_class = EXT_MAP[obj['type']][name]
-            obj['extensions'][name] = ext_class(allow_custom=allow_custom, **obj['extensions'][name])
+            try:
+                ext_class = EXT_MAP[obj['type']][name]
+            except KeyError:
+                if not allow_custom:
+                    raise CustomContentError("Can't parse unknown extension type '%s'"
+                                             "for observable type '%s'!" % (name, obj['type']))
+            else:  # extension was found
+                obj['extensions'][name] = ext_class(allow_custom=allow_custom, **obj['extensions'][name])
 
     return obj_class(allow_custom=allow_custom, **obj)
 


Test patch:
diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py
index a14503f..56e578f 100644
--- a/stix2/test/test_custom.py
+++ b/stix2/test/test_custom.py
@@ -363,6 +363,7 @@ def test_parse_custom_observable_object():
     }"""
 
     nt = stix2.parse_observable(nt_string, [])
+    assert isinstance(nt, stix2.core._STIXBase)
     assert nt.property1 == 'something'
 
 
@@ -372,10 +373,46 @@ def test_parse_unregistered_custom_observable_object():
         "property1": "something"
     }"""
 
-    with pytest.raises(stix2.exceptions.ParseError) as excinfo:
+    with pytest.raises(stix2.exceptions.CustomContentError) as excinfo:
         stix2.parse_observable(nt_string)
     assert "Can't parse unknown observable type" in str(excinfo.value)
 
+    parsed_custom = stix2.parse_observable(nt_string, allow_custom=True)
+    assert parsed_custom['property1'] == 'something'
+    with pytest.raises(AttributeError) as excinfo:
+        assert parsed_custom.property1 == 'something'
+    assert not isinstance(parsed_custom, stix2.core._STIXBase)
+
+
+def test_parse_unregistered_custom_observable_object_with_no_type():
+    nt_string = """{
+        "property1": "something"
+    }"""
+
+    with pytest.raises(stix2.exceptions.ParseError) as excinfo:
+        stix2.parse_observable(nt_string, allow_custom=True)
+    assert "Can't parse observable with no 'type' property" in str(excinfo.value)
+
+
+def test_parse_observed_data_with_custom_observable():
+    input_str = """{
+        "type": "observed-data",
+        "id": "observed-data--dc20c4ca-a2a3-4090-a5d5-9558c3af4758",
+        "created": "2016-04-06T19:58:16.000Z",
+        "modified": "2016-04-06T19:58:16.000Z",
+        "first_observed": "2015-12-21T19:00:00Z",
+        "last_observed": "2015-12-21T19:00:00Z",
+        "number_observed": 1,
+        "objects": {
+            "0": {
+                "type": "x-foobar-observable",
+                "property1": "something"
+            }
+        }
+    }"""
+    parsed = stix2.parse(input_str, allow_custom=True)
+    assert parsed.objects['0']['property1'] == 'something'
+
 
 def test_parse_invalid_custom_observable_object():
     nt_string = """{
@@ -479,6 +516,27 @@ def test_custom_extension_wrong_observable_type():
     assert 'Cannot determine extension type' in excinfo.value.reason
 
 
+@pytest.mark.parametrize("data", [
+    """{
+    "keys": [
+        {
+            "test123": 123,
+            "test345": "aaaa"
+        }
+    ]
+}""",
+])
+def test_custom_extension_with_list_and_dict_properties_observable_type(data):
+    @stix2.observables.CustomExtension(stix2.UserAccount, 'some-extension', [
+        ('keys', stix2.properties.ListProperty(stix2.properties.DictionaryProperty, required=True))
+    ])
+    class SomeCustomExtension:
+        pass
+
+    example = SomeCustomExtension(keys=[{'test123': 123, 'test345': 'aaaa'}])
+    assert data == str(example)
+
+
 def test_custom_extension_invalid_observable():
     # These extensions are being applied to improperly-created Observables.
     # The Observable classes should have been created with the CustomObservable decorator.
@@ -591,7 +649,11 @@ def test_parse_observable_with_unregistered_custom_extension():
 
     with pytest.raises(ValueError) as excinfo:
         stix2.parse_observable(input_str)
-    assert "Can't parse Unknown extension type" in str(excinfo.value)
+    assert "Can't parse unknown extension type" in str(excinfo.value)
+
+    parsed_ob = stix2.parse_observable(input_str, allow_custom=True)
+    assert parsed_ob['extensions']['x-foobar-ext']['property1'] == 'foo'
+    assert not isinstance(parsed_ob['extensions']['x-foobar-ext'], stix2.core._STIXBase)
 
 
 def test_register_custom_object():
diff --git a/stix2/test/test_properties.py b/stix2/test/test_properties.py
index 34edc96..16ff06a 100644
--- a/stix2/test/test_properties.py
+++ b/stix2/test/test_properties.py
@@ -1,6 +1,6 @@
 import pytest
 
-from stix2 import EmailMIMEComponent, ExtensionsProperty, TCPExt
+from stix2 import CustomObject, EmailMIMEComponent, ExtensionsProperty, TCPExt
 from stix2.exceptions import AtLeastOnePropertyError, DictionaryKeyError
 from stix2.properties import (BinaryProperty, BooleanProperty,
                               DictionaryProperty, EmbeddedObjectProperty,
@@ -266,6 +266,17 @@ def test_dictionary_property_invalid(d):
     assert str(excinfo.value) == d[1]
 
 
+def test_property_list_of_dictionary():
+    @CustomObject('x-new-obj', [
+        ('property1', ListProperty(DictionaryProperty(), required=True)),
+    ])
+    class NewObj():
+        pass
+
+    test_obj = NewObj(property1=[{'foo': 'bar'}])
+    assert test_obj.property1[0]['foo'] == 'bar'
+
+
 @pytest.mark.parametrize("value", [
     {"sha256": "6db12788c37247f2316052e142f42f4b259d6561751e5f401a1ae2a6df9c674b"},
     [('MD5', '2dfb1bcc980200c6706feee399d41b3f'), ('RIPEMD-160', 'b3a8cd8a27c90af79b3c81754f267780f443dfef')],