Repository: oasis-open/cti-python-stix2

Issue:
Incorrect Pattern Error Propagation
When using `stix2/pattern_visitor.py/create_pattern_object()` with a pattern that does not include matching single quotes (e.g. "[ file: name = 'weirdname]" ), the stix2 library returns an unhelpful and unclear error. This is in comparison to the more useful error message provided by the pattern validator if the pattern was supplied to it directly. Thus, there should be a way for the error(s) generated by the pattern validator to be propagated back through stix2. See below stacktrace for more details.

(Based off of https://github.com/oasis-open/cti-python-stix2/issues/303)

![stacktrace](https://user-images.githubusercontent.com/43383927/73215354-f2b69680-4121-11ea-89ef-8d949202e2ed.png)


Comments:

Patch:
diff --git a/stix2/base.py b/stix2/base.py
index a283902..4248075 100644
--- a/stix2/base.py
+++ b/stix2/base.py
@@ -388,14 +388,12 @@ class _Observable(_STIXBase):
                         temp_deep_copy = copy.deepcopy(dict(kwargs[key]))
                         _recursive_stix_to_dict(temp_deep_copy)
                         streamlined_obj_vals.append(temp_deep_copy)
-                    elif isinstance(kwargs[key], list) and isinstance(kwargs[key][0], _STIXBase):
-                        for obj in kwargs[key]:
-                            temp_deep_copy = copy.deepcopy(dict(obj))
-                            _recursive_stix_to_dict(temp_deep_copy)
-                            streamlined_obj_vals.append(temp_deep_copy)
+                    elif isinstance(kwargs[key], list):
+                        temp_deep_copy = copy.deepcopy(kwargs[key])
+                        _recursive_stix_list_to_dict(temp_deep_copy)
+                        streamlined_obj_vals.append(temp_deep_copy)
                     else:
                         streamlined_obj_vals.append(kwargs[key])
-
             if streamlined_obj_vals:
                 data = canonicalize(streamlined_obj_vals, utf8=False)
 
@@ -448,5 +446,20 @@ def _recursive_stix_to_dict(input_dict):
 
             # There may stil be nested _STIXBase objects
             _recursive_stix_to_dict(input_dict[key])
+        elif isinstance(input_dict[key], list):
+            _recursive_stix_list_to_dict(input_dict[key])
         else:
-            return
+            pass
+
+
+def _recursive_stix_list_to_dict(input_list):
+    for i in range(len(input_list)):
+        if isinstance(input_list[i], _STIXBase):
+            input_list[i] = dict(input_list[i])
+        elif isinstance(input_list[i], dict):
+            pass
+        elif isinstance(input_list[i], list):
+            _recursive_stix_list_to_dict(input_list[i])
+        else:
+            continue
+        _recursive_stix_to_dict(input_list[i])
diff --git a/stix2/pattern_visitor.py b/stix2/pattern_visitor.py
index b2d7a53..6ac3e98 100644
--- a/stix2/pattern_visitor.py
+++ b/stix2/pattern_visitor.py
@@ -1,16 +1,12 @@
 import importlib
 import inspect
 
-from antlr4 import CommonTokenStream, InputStream
-from antlr4.tree.Trees import Trees
-import six
 from stix2patterns.exceptions import ParseException
-from stix2patterns.grammars.STIXPatternLexer import STIXPatternLexer
 from stix2patterns.grammars.STIXPatternParser import (
     STIXPatternParser, TerminalNode,
 )
 from stix2patterns.grammars.STIXPatternVisitor import STIXPatternVisitor
-from stix2patterns.validator import STIXPatternErrorListener
+from stix2patterns.v20.pattern import Pattern
 
 from .patterns import *
 from .patterns import _BooleanExpression
@@ -328,41 +324,9 @@ class STIXPatternVisitorForSTIX2(STIXPatternVisitor):
 
 def create_pattern_object(pattern, module_suffix="", module_name=""):
     """
-    Validates a pattern against the STIX Pattern grammar.  Error messages are
-    returned in a list.  The test passed if the returned list is empty.
+    Create a STIX pattern AST from a pattern string.
     """
 
-    start = ''
-    if isinstance(pattern, six.string_types):
-        start = pattern[:2]
-        pattern = InputStream(pattern)
-
-    if not start:
-        start = pattern.readline()[:2]
-        pattern.seek(0)
-
-    parseErrListener = STIXPatternErrorListener()
-
-    lexer = STIXPatternLexer(pattern)
-    # it always adds a console listener by default... remove it.
-    lexer.removeErrorListeners()
-
-    stream = CommonTokenStream(lexer)
-
-    parser = STIXPatternParser(stream)
-
-    parser.buildParseTrees = True
-    # it always adds a console listener by default... remove it.
-    parser.removeErrorListeners()
-    parser.addErrorListener(parseErrListener)
-
-    # To improve error messages, replace "<INVALID>" in the literal
-    # names with symbolic names.  This is a hack, but seemed like
-    # the simplest workaround.
-    for i, lit_name in enumerate(parser.literalNames):
-        if lit_name == u"<INVALID>":
-            parser.literalNames[i] = parser.symbolicNames[i]
-
-    tree = parser.pattern()
+    pattern_obj = Pattern(pattern)
     builder = STIXPatternVisitorForSTIX2(module_suffix, module_name)
-    return builder.visit(tree)
+    return pattern_obj.visit(builder)
diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py
index 1703263..ed560a6 100644
--- a/stix2/v21/observables.py
+++ b/stix2/v21/observables.py
@@ -7,13 +7,10 @@ Observable and do not have a ``_type`` attribute.
 
 from collections import OrderedDict
 import itertools
-import warnings
 
 from ..base import _Extension, _Observable, _STIXBase
 from ..custom import _custom_extension_builder, _custom_observable_builder
-from ..exceptions import (
-    AtLeastOnePropertyError, DependentPropertiesError, STIXDeprecationWarning,
-)
+from ..exceptions import AtLeastOnePropertyError, DependentPropertiesError
 from ..properties import (
     BinaryProperty, BooleanProperty, DictionaryProperty,
     EmbeddedObjectProperty, EnumProperty, ExtensionsProperty, FloatProperty,
@@ -122,14 +119,6 @@ class DomainName(_Observable):
     ])
     _id_contributing_properties = ["value"]
 
-    def _check_object_constraints(self):
-        if self.get('resolves_to_refs'):
-            warnings.warn(
-                "The 'resolves_to_refs' property of domain-name is deprecated in "
-                "STIX 2.1. Use the 'resolves-to' relationship type instead",
-                STIXDeprecationWarning,
-            )
-
 
 class EmailAddress(_Observable):
     # TODO: Add link
@@ -421,21 +410,6 @@ class IPv4Address(_Observable):
     ])
     _id_contributing_properties = ["value"]
 
-    def _check_object_constraints(self):
-        if self.get('resolves_to_refs'):
-            warnings.warn(
-                "The 'resolves_to_refs' property of ipv4-addr is deprecated in "
-                "STIX 2.1. Use the 'resolves-to' relationship type instead",
-                STIXDeprecationWarning,
-            )
-
-        if self.get('belongs_to_refs'):
-            warnings.warn(
-                "The 'belongs_to_refs' property of ipv4-addr is deprecated in "
-                "STIX 2.1. Use the 'belongs-to' relationship type instead",
-                STIXDeprecationWarning,
-            )
-
 
 class IPv6Address(_Observable):
     # TODO: Add link
@@ -458,21 +432,6 @@ class IPv6Address(_Observable):
     ])
     _id_contributing_properties = ["value"]
 
-    def _check_object_constraints(self):
-        if self.get('resolves_to_refs'):
-            warnings.warn(
-                "The 'resolves_to_refs' property of ipv6-addr is deprecated in "
-                "STIX 2.1. Use the 'resolves-to' relationship type instead",
-                STIXDeprecationWarning,
-            )
-
-        if self.get('belongs_to_refs'):
-            warnings.warn(
-                "The 'belongs_to_refs' property of ipv6-addr is deprecated in "
-                "STIX 2.1. Use the 'belongs-to' relationship type instead",
-                STIXDeprecationWarning,
-            )
-
 
 class MACAddress(_Observable):
     # TODO: Add link


Test patch:
diff --git a/stix2/test/v21/test_indicator.py b/stix2/test/v21/test_indicator.py
index 152f253..23bad29 100644
--- a/stix2/test/v21/test_indicator.py
+++ b/stix2/test/v21/test_indicator.py
@@ -271,7 +271,7 @@ def test_indicator_stix20_invalid_pattern():
         )
 
     assert excinfo.value.cls == stix2.v21.Indicator
-    assert "FAIL: The same qualifier is used more than once" in str(excinfo.value)
+    assert "FAIL: Duplicate qualifier type encountered" in str(excinfo.value)
 
     ind = stix2.v21.Indicator(
         type="indicator",
diff --git a/stix2/test/v21/test_observed_data.py b/stix2/test/v21/test_observed_data.py
index 371018c..71bad46 100644
--- a/stix2/test/v21/test_observed_data.py
+++ b/stix2/test/v21/test_observed_data.py
@@ -1540,37 +1540,45 @@ def test_deterministic_id_no_contributing_props():
     assert uuid_obj_2.version == 4
 
 
-def test_ipv4_resolves_to_refs_deprecation():
-    with pytest.warns(stix2.exceptions.STIXDeprecationWarning):
-
-        stix2.v21.IPv4Address(
-            value="26.09.19.70",
-            resolves_to_refs=["mac-addr--08900593-0265-52fc-93c0-5b4a942f5887"],
-        )
-
-
-def test_ipv4_belongs_to_refs_deprecation():
-    with pytest.warns(stix2.exceptions.STIXDeprecationWarning):
-
-        stix2.v21.IPv4Address(
-            value="21.12.19.64",
-            belongs_to_refs=["autonomous-system--52e0a49d-d683-5801-a7b8-145765a1e116"],
-        )
-
-
-def test_ipv6_resolves_to_refs_deprecation():
-    with pytest.warns(stix2.exceptions.STIXDeprecationWarning):
+def test_id_gen_recursive_dict_conversion_1():
+    file_observable = stix2.v21.File(
+        name="example.exe",
+        size=68 * 1000,
+        magic_number_hex="50000000",
+        hashes={
+            "SHA-256": "841a8921140aba50671ebb0770fecc4ee308c4952cfeff8de154ab14eeef4649",
+        },
+        extensions={
+            "windows-pebinary-ext": stix2.v21.WindowsPEBinaryExt(
+                pe_type="exe",
+                machine_hex="014c",
+                sections=[
+                    stix2.v21.WindowsPESection(
+                        name=".data",
+                        size=4096,
+                        entropy=7.980693,
+                        hashes={"SHA-256": "6e3b6f3978e5cd96ba7abee35c24e867b7e64072e2ecb22d0ee7a6e6af6894d0"},
+                    ),
+                ],
+            ),
+        },
+    )
 
-        stix2.v21.IPv6Address(
-            value="2001:0db8:85a3:0000:0000:8a2e:0370:7334",
-            resolves_to_refs=["mac-addr--08900593-0265-52fc-93c0-5b4a942f5887"],
-        )
+    assert file_observable.id == "file--5219d93d-13c1-5f1f-896b-039f10ec67ea"
 
 
-def test_ipv6_belongs_to_refs_deprecation():
-    with pytest.warns(stix2.exceptions.STIXDeprecationWarning):
+def test_id_gen_recursive_dict_conversion_2():
+    wrko = stix2.v21.WindowsRegistryKey(
+        values=[
+            stix2.v21.WindowsRegistryValueType(
+                name="Foo",
+                data="qwerty",
+            ),
+            stix2.v21.WindowsRegistryValueType(
+                name="Bar",
+                data="42",
+            ),
+        ],
+    )
 
-        stix2.v21.IPv6Address(
-            value="2001:0db8:85a3:0000:0000:8a2e:0370:7334",
-            belongs_to_refs=["autonomous-system--52e0a49d-d683-5801-a7b8-145765a1e116"],
-        )
+    assert wrko.id == "windows-registry-key--c087d9fe-a03e-5922-a1cd-da116e5b8a7b"
diff --git a/stix2/test/v21/test_pattern_expressions.py b/stix2/test/v21/test_pattern_expressions.py
index 76880be..0c298f8 100644
--- a/stix2/test/v21/test_pattern_expressions.py
+++ b/stix2/test/v21/test_pattern_expressions.py
@@ -1,6 +1,7 @@
 import datetime
 
 import pytest
+from stix2patterns.exceptions import ParseException
 
 import stix2
 from stix2.pattern_visitor import create_pattern_object
@@ -515,3 +516,8 @@ def test_list_constant():
 def test_parsing_multiple_slashes_quotes():
     patt_obj = create_pattern_object("[ file:name = 'weird_name\\'' ]")
     assert str(patt_obj) == "[file:name = 'weird_name\\'']"
+
+
+def test_parse_error():
+    with pytest.raises(ParseException):
+        create_pattern_object("[ file: name = 'weirdname]")