Repository: oasis-open/cti-stix-validator
Pull request #28 (2017-07-07 17:16:37)

Warn on multiple UUIDs, Allow list properties in patterns


Patch:
diff --git a/README.rst b/README.rst
index 034549a..32d97f8 100644
--- a/README.rst
+++ b/README.rst
@@ -132,6 +132,9 @@ Some checks access Internet resources to determine valid values for certain prop
 |  202   | relationship-types          | relationships are among those defined  |
 |        |                             | in the specification                   |
 +--------+-----------------------------+----------------------------------------+
+|  203   | duplicate-ids               | objects in a bundle with duplicate IDs |
+|        |                             | have different `modified` timestamps   |
++--------+-----------------------------+----------------------------------------+
 |  210   | all-vocabs                  | all of the following open vocabulary   |
 |        |                             | checks are run                         |
 +--------+-----------------------------+----------------------------------------+
diff --git a/stix2validator/musts.py b/stix2validator/musts.py
index 201654e..4f04150 100644
--- a/stix2validator/musts.py
+++ b/stix2validator/musts.py
@@ -382,7 +382,9 @@ def patterns(instance, options):
         expression_list = inspection[objtype]
         for exp in expression_list:
             path = exp[0]
-            prop = path.split('.', 1)[0]
+            # Get the property name without list index, dictionary key, or referenced object property
+            prop_matches = re.match('([^\.\[]+)(\[\.+\])?(\..+)?', path)
+            prop = prop_matches.group(1)
             if objtype in enums.OBSERVABLE_PROPERTIES and prop in enums.OBSERVABLE_PROPERTIES[objtype]:
                 continue
             elif options.strict_types:
diff --git a/stix2validator/scripts/stix2_validator.py b/stix2validator/scripts/stix2_validator.py
index 58b2cc5..fb33164 100644
--- a/stix2validator/scripts/stix2_validator.py
+++ b/stix2validator/scripts/stix2_validator.py
@@ -46,6 +46,8 @@ the validator performs, along with the code to use with the --enable or
 |      |                             | definition_type                        |
 | 202  | relationship-types          | relationships are among those defined  |
 |      |                             | in the specification                   |
+| 203  | duplicate-ids               | objects in a bundle with duplicate IDs |
+|      |                             | have different `modified` timestamps   |
 | 210  | all-vocabs                  | all of the following open vocabulary   |
 |      |                             | checks are run                         |
 | 211  | attack-motivation           | certain property values are from the   |
diff --git a/stix2validator/shoulds.py b/stix2validator/shoulds.py
index 9658d93..5e624cf 100644
--- a/stix2validator/shoulds.py
+++ b/stix2validator/shoulds.py
@@ -58,7 +58,7 @@ def custom_object_prefix_strict(instance):
                         "followed by a source unique identifier (like a "
                         "domain name with dots replaced by hyphens), a hyphen "
                         "and then the name." % instance['type'],
-                        instance['id'], 'custom-object-prefix')
+                        instance['id'], 'custom-prefix')
 
 
 def custom_object_prefix_lax(instance):
@@ -71,7 +71,7 @@ def custom_object_prefix_lax(instance):
         yield JSONError("Custom object type '%s' should start with 'x-' in "
                         "order to be compatible with future versions of the "
                         "STIX 2 specification." % instance['type'],
-                        instance['id'], 'custom-object-prefix')
+                        instance['id'], 'custom-prefix-lax')
 
 
 def custom_property_prefix_strict(instance):
@@ -90,7 +90,7 @@ def custom_property_prefix_strict(instance):
                             "identifier (like a domain name with dots "
                             "replaced by hyphen), a hyphen and then the name."
                             % prop_name, instance['id'],
-                            'custom-property-prefix')
+                            'custom-prefix')
 
 
 def custom_property_prefix_lax(instance):
@@ -109,7 +109,7 @@ def custom_property_prefix_lax(instance):
                             "starts with 'x_' in order to be compatible with "
                             "future versions of the STIX 2 specification." %
                             prop_name, instance['id'],
-                            'custom-property-prefix')
+                            'custom-prefix-lax')
 
 
 def open_vocab_values(instance):
@@ -523,7 +523,7 @@ def custom_observable_object_prefix_strict(instance):
                             "(like a domain name with dots replaced by "
                             "hyphens), a hyphen and then the name."
                             % obj['type'], instance['id'],
-                            'custom-observable-object-prefix')
+                            'custom-prefix')
 
 
 @cyber_observable_check
@@ -537,7 +537,7 @@ def custom_observable_object_prefix_lax(instance):
             yield JSONError("Custom Observable Object type '%s' should start "
                             "with 'x-'."
                             % obj['type'], instance['id'],
-                            'custom-observable-object-prefix')
+                            'custom-prefix-lax')
 
 
 @cyber_observable_check
@@ -557,7 +557,7 @@ def custom_object_extension_prefix_strict(instance):
                                 "unique identifier (like a domain name with dots "
                                 "replaced by hyphens), a hyphen and then the name."
                                 % ext_key, instance['id'],
-                                'custom-object-extension-prefix')
+                                'custom-prefix')
 
 
 @cyber_observable_check
@@ -575,7 +575,7 @@ def custom_object_extension_prefix_lax(instance):
                 yield JSONError("Custom Cyber Observable Object extension type"
                                 " '%s' should start with 'x-'."
                                 % ext_key, instance['id'],
-                                'custom-object-extension-prefix')
+                                'custom-prefix-lax')
 
 
 @cyber_observable_check
@@ -599,7 +599,7 @@ def custom_observable_properties_prefix_strict(instance):
                                 "dots replaced by hyphens), a hyphen and then the"
                                 " name."
                                 % prop, instance['id'],
-                                'custom-observable-properties-prefix')
+                                'custom-prefix')
             # Check properties of embedded cyber observable types
             if (type_ in enums.OBSERVABLE_EMBEDDED_PROPERTIES and
                     prop in enums.OBSERVABLE_EMBEDDED_PROPERTIES[type_]):
@@ -616,7 +616,7 @@ def custom_observable_properties_prefix_strict(instance):
                                                 "dots replaced by hyphens), a hyphen and "
                                                 "then the name."
                                                 % (embedded, prop, type_), instance['id'],
-                                                'custom-observable-properties-prefix')
+                                                'custom-prefix')
                     elif (embed_prop not in enums.OBSERVABLE_EMBEDDED_PROPERTIES[type_][prop] and
                             not CUSTOM_PROPERTY_PREFIX_RE.match(embed_prop)):
                         yield JSONError("Cyber Observable Object custom "
@@ -627,7 +627,7 @@ def custom_observable_properties_prefix_strict(instance):
                                         "dots replaced by hyphens), a hyphen and "
                                         "then the name."
                                         % (embed_prop, prop, type_), instance['id'],
-                                        'custom-observable-properties-prefix')
+                                        'custom-prefix')
 
         # Check object extensions' properties
         if (type_ in enums.OBSERVABLE_EXTENSIONS and 'extensions' in obj):
@@ -644,7 +644,7 @@ def custom_observable_properties_prefix_strict(instance):
                                             "domain name with dots replaced by "
                                             "hyphens), a hyphen and then the name."
                                             % (ext_prop, ext_key), instance['id'],
-                                            'custom-observable-properties-prefix')
+                                            'custom-prefix')
 
                 if ext_key in enums.OBSERVABLE_EXTENSIONS[type_]:
                     for ext_prop in obj['extensions'][ext_key]:
@@ -664,7 +664,7 @@ def custom_observable_properties_prefix_strict(instance):
                                                         " with dots replaced by hyphens), a "
                                                         "hyphen and then the name."
                                                         % (p, ext_prop, ext_key), instance['id'],
-                                                        'custom-observable-properties-prefix')
+                                                        'custom-prefix')
 
 
 @cyber_observable_check
@@ -685,7 +685,7 @@ def custom_observable_properties_prefix_lax(instance):
                 yield JSONError("Cyber Observable Object custom property '%s' "
                                 "should start with 'x_'."
                                 % prop, instance['id'],
-                                'custom-observable-properties-prefix')
+                                'custom-prefix-lax')
             # Check properties of embedded cyber observable types
             if (type_ in enums.OBSERVABLE_EMBEDDED_PROPERTIES and
                     prop in enums.OBSERVABLE_EMBEDDED_PROPERTIES[type_]):
@@ -698,14 +698,14 @@ def custom_observable_properties_prefix_lax(instance):
                                                 "property '%s' in the %s property of "
                                                 "%s object should start with 'x_'."
                                                 % (embedded, prop, type_), instance['id'],
-                                                'custom-observable-properties-prefix')
+                                                'custom-prefix-lax')
                     elif (embed_prop not in enums.OBSERVABLE_EMBEDDED_PROPERTIES[type_][prop] and
                             not CUSTOM_PROPERTY_LAX_PREFIX_RE.match(embed_prop)):
                         yield JSONError("Cyber Observable Object custom "
                                         "property '%s' in the %s property of "
                                         "%s object should start with 'x_'."
                                         % (embed_prop, prop, type_), instance['id'],
-                                        'custom-observable-properties-prefix')
+                                        'custom-prefix-lax')
 
         # Check object extensions' properties
         if (type_ in enums.OBSERVABLE_EXTENSIONS and 'extensions' in obj):
@@ -719,7 +719,7 @@ def custom_observable_properties_prefix_lax(instance):
                                             "property '%s' in the %s extension "
                                             "should start with 'x_'."
                                             % (ext_prop, ext_key), instance['id'],
-                                            'custom-observable-properties-prefix')
+                                            'custom-prefix-lax')
 
                 if ext_key in enums.OBSERVABLE_EXTENSIONS[type_]:
                     for ext_prop in obj['extensions'][ext_key]:
@@ -736,7 +736,7 @@ def custom_observable_properties_prefix_lax(instance):
                                                         "property of the %s extension should "
                                                         "start with 'x_'."
                                                         % (p, ext_prop, ext_key), instance['id'],
-                                                        'custom-observable-properties-prefix')
+                                                        'custom-prefix-lax')
 
 
 @cyber_observable_check
@@ -1024,6 +1024,25 @@ def extref_hashes(instance):
                                  % (src), instance['id'], 'extref-hashes')
 
 
+def duplicate_ids(instance):
+    """Ensure objects with duplicate IDs have different `modified` timestamps.
+    """
+    if instance['type'] != 'bundle' or 'objects' not in instance:
+        return
+
+    unique_ids = {}
+    for obj in instance['objects']:
+        if 'id' not in obj or 'modified' not in obj:
+            continue
+        elif obj['id'] not in unique_ids:
+            unique_ids[obj['id']] = obj['modified']
+        elif obj['modified'] == unique_ids[obj['id']]:
+            yield JSONError("Duplicate ID '%s' has identical `modified` timestamp."
+                            " If they are different versions of the same object, "
+                            "they should have different `modified` properties."
+                            % obj['id'], instance['id'], 'duplicate-ids')
+
+
 # Mapping of check names to the functions which perform the checks
 CHECKS = {
     'all': [
@@ -1063,6 +1082,7 @@ CHECKS = {
         pdf_doc_info,
         network_traffic_ports,
         extref_hashes,
+        duplicate_ids,
     ],
     'format-checks': [
         custom_object_prefix_strict,
@@ -1088,6 +1108,7 @@ CHECKS = {
     'approved-values': [
         vocab_marking_definition,
         relationships_strict,
+        duplicate_ids,
         vocab_attack_motivation,
         vocab_attack_resource_level,
         vocab_identity_class,
@@ -1112,6 +1133,7 @@ CHECKS = {
     ],
     'marking-definition-type': vocab_marking_definition,
     'relationship-types': relationships_strict,
+    'duplicate-ids': duplicate_ids,
     'all-vocabs': [
         vocab_attack_motivation,
         vocab_attack_resource_level,
@@ -1200,6 +1222,8 @@ def list_shoulds(options):
                     validator_list.append(CHECKS['marking-definition-type'])
                 if 'relationship-types' not in options.disabled:
                     validator_list.append(CHECKS['relationship-types'])
+                if 'duplicate-ids' not in options.disabled:
+                    validator_list.append(CHECKS['duplicate-ids'])
                 if 'all-vocabs' not in options.disabled:
                     if 'attack-motivation' not in options.disabled:
                         validator_list.append(CHECKS['attack-motivation'])
diff --git a/stix2validator/util.py b/stix2validator/util.py
index 75e5387..34121c8 100644
--- a/stix2validator/util.py
+++ b/stix2validator/util.py
@@ -93,25 +93,18 @@ class ValidationOptions(object):
 # Mapping of check code numbers to names
 CHECK_CODES = {
     '1': 'format-checks',
-    '101': 'custom-object-prefix',
-    '102': 'custom-object-prefix-lax',
-    '103': 'custom-property-prefix',
-    '104': 'custom-property-prefix-lax',
+    '101': 'custom-prefix',
+    '102': 'custom-prefix-lax',
     '111': 'open-vocab-format',
     '121': 'kill-chain-names',
     '141': 'observable-object-keys',
     '142': 'observable-dictionary-keys',
-    '143': 'custom-observable-object-prefix',
-    '144': 'custom-observable-object-prefix-lax',
-    '145': 'custom-object-extension-prefix',
-    '146': 'custom-object-extension-prefix-lax',
-    '147': 'custom-observable-properties-prefix',
-    '148': 'custom-observable-properties-prefix-lax',
     '149': 'windows-process-priority-format',
     '150': 'hash-length',
     '2': 'approved-values',
     '201': 'marking-definition-type',
     '202': 'relationship-types',
+    '203': 'duplicate-ids',
     '210': 'all-vocabs',
     '211': 'attack-motivation',
     '212': 'attack-resource-level',


Test patch:
diff --git a/stix2validator/test/bundle_tests.py b/stix2validator/test/bundle_tests.py
index b6f7f59..8185e05 100644
--- a/stix2validator/test/bundle_tests.py
+++ b/stix2validator/test/bundle_tests.py
@@ -47,6 +47,14 @@ class BundleTestCases(ValidatorTest):
         bundle = json.dumps(bundle)
         self.assertFalseWithOptions(bundle)
 
+    def test_bundle_duplicate_ids(self):
+        bundle = copy.deepcopy(self.valid_bundle)
+        bundle['objects'].append(bundle['objects'][0].copy())
+        self.assertFalseWithOptions(json.dumps(bundle))
+
+        bundle['objects'][1]['modified'] = "2017-06-22T14:09:00.123Z"
+        self.assertTrueWithOptions(json.dumps(bundle))
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/stix2validator/test/indicator_tests.py b/stix2validator/test/indicator_tests.py
index 87b1a28..f3f2131 100644
--- a/stix2validator/test/indicator_tests.py
+++ b/stix2validator/test/indicator_tests.py
@@ -165,6 +165,11 @@ class IndicatorTestCases(ValidatorTest):
         indicator['pattern'] = """[x-foo":x_name = 'something']"""
         self.check_ignore(json.dumps(indicator), 'custom-prefix')
 
+    def test_pattern_list_object_property(self):
+        indicator = copy.deepcopy(self.valid_indicator)
+        indicator['pattern'] = """[windows-registry-key:values[*].data='badstuff']"""
+        self.assertTrueWithOptions(json.dumps(indicator))
+
 
 if __name__ == "__main__":
     unittest.main()