Repository: elastic/ecs

Issue:
Allow regex pattern to be specified in schema to improve data consistency
<!--
Please first search existing issues for the changes you are requesting; it may already exist as an open issue.

Substantial schema changes or additions should follow the RFC process: https://github.com/elastic/ecs/blob/main/rfcs/README.md

Please fill in the following sections describing your proposed changes: -->

**Summary**

Add information to the schema to support automated data validation. I propose allowing a regular expression field that tools can use to validate a value.

For example `source.mac` would give a pattern that checks for RFC 7042 format in uppercase.

```yaml
- name: source.mac
  pattern: '^[A-F0-9]{2}(-[A-F0-9]{2}){5,}$'
```

[JSON schema](https://json-schema.org/draft/2020-12/json-schema-validation.html#rfc.section.7.3.8) has a similar concept and supports many more validation types, but I think have a regular expression would be a good place to start.

**Motivation**:

https://github.com/elastic/elastic-package/issues/615

I'm trying to improve the consistency of values contained in ECS fields through automated tests.

**Detailed Design**:

It would be good to go through the schema and see how many places this could be applied.



Discussion:
I agree this would be useful. Additionally, some sort of `enum` would also be helpful for fields with "allowed values."

Would `elastic-package` be looking for this value in the [generated ECS YAML files](https://github.com/elastic/ecs/tree/main/generated/ecs)?
That correct, it would reading from the generated files. Here's where it looks: 

https://github.com/elastic/elastic-package/blob/8d9732d0c4364d28ba1c34f2ad7fe91dfc7d2785/internal/fields/dependency_manager.go#L28-L29
The allowed_values appear to [already be](https://github.com/elastic/elastic-package/blob/8d9732d0c4364d28ba1c34f2ad7fe91dfc7d2785/internal/fields/dependency_manager.go#L28-L29) machine readable. It just takes a bit more work to interpret the data than a straight `enum` string array.


Another use would be for validating fields that say they should be lowercase like network.protocol and network.transport.

Resolving patch:
diff --git a/scripts/generators/beats.py b/scripts/generators/beats.py
--- a/scripts/generators/beats.py
+++ b/scripts/generators/beats.py
@@ -75,7 +75,7 @@ def fieldset_field_array(source_fields, fieldset_prefix):
                     'ignore_above', 'multi_fields', 'format', 'input_format',
                     'output_format', 'output_precision', 'description',
                     'example', 'enabled', 'index', 'doc_values', 'path',
-                    'scaling_factor']
+                    'scaling_factor', 'pattern']
     multi_fields_allowed_keys = ['name', 'type', 'norms', 'default_field', 'normalizer', 'ignore_above']
 
     fields = []
diff --git a/scripts/schema/cleaner.py b/scripts/schema/cleaner.py
--- a/scripts/schema/cleaner.py
+++ b/scripts/schema/cleaner.py
@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import copy
+import re
 
 from generators import ecs_helpers
 from schema import visitor
@@ -202,9 +202,11 @@ def field_assertions_and_warnings(field):
     if not ecs_helpers.is_intermediate(field):
         # check short description length if in strict mode
         single_line_short_description(field, strict=strict_mode)
-        check_example_value(field, strict=strict_mode)
         if 'beta' in field['field_details']:
             single_line_beta_description(field, strict=strict_mode)
+        if 'pattern' in field['field_details']:
+            validate_pattern_regex(field['field_details'], strict=strict_mode)
+        check_example_value(field, strict=strict_mode)
         if field['field_details']['level'] not in ACCEPTABLE_FIELD_LEVELS:
             msg = "Invalid level for field '{}'.\nValue: {}\nAcceptable values: {}".format(
                 field['field_details']['name'], field['field_details']['level'],
@@ -229,13 +231,18 @@ def single_line_short_check(short_to_check, short_name):
     return None
 
 
+def strict_warning_handler(message, strict):
+    """Handles warnings based on --strict mode"""
+    if strict:
+        raise ValueError(message)
+    else:
+        ecs_helpers.strict_warning(message)
+
+
 def single_line_short_description(schema_or_field, strict=True):
     error = single_line_short_check(schema_or_field['field_details']['short'], schema_or_field['field_details']['name'])
     if error:
-        if strict:
-            raise ValueError(error)
-        else:
-            ecs_helpers.strict_warning(error)
+        strict_warning_handler(error, strict)
 
 
 def single_line_short_override_description(schema_or_field, strict=True):
@@ -244,10 +251,7 @@ def single_line_short_override_description(schema_or_field, strict=True):
             continue
         error = single_line_short_check(field['short_override'], field['full'])
         if error:
-            if strict:
-                raise ValueError(error)
-            else:
-                ecs_helpers.strict_warning(error)
+            strict_warning_handler(error, strict)
 
 
 def check_example_value(field, strict=True):
@@ -256,20 +260,34 @@ def check_example_value(field, strict=True):
     Fails or warns (depending on strict mode) if so.
     """
     example_value = field['field_details'].get('example', None)
+    pattern = field['field_details'].get('pattern', None)
+    name = field['field_details']['name']
+
     if isinstance(example_value, (list, dict)):
-        name = field['field_details']['name']
         msg = f"Example value for field `{name}` contains an object or array which must be quoted to avoid YAML interpretation."
-        if strict:
-            raise ValueError(msg)
-        else:
-            ecs_helpers.strict_warning(msg)
+        strict_warning_handler(msg, strict)
+
+    if pattern:
+        match = re.match(pattern, example_value)
+        if not match:
+            msg = f"Example value for field `{name}` does not match the regex defined in the pattern attribute: `{pattern}`."
+            strict_warning_handler(msg, strict)
 
 
 def single_line_beta_description(schema_or_field, strict=True):
     if "\n" in schema_or_field['field_details']['beta']:
         msg = "Beta descriptions must be single line.\n"
         msg += f"Offending field or field set: {schema_or_field['field_details']['name']}"
-        if strict:
-            raise ValueError(msg)
-        else:
-            ecs_helpers.strict_warning(msg)
+        strict_warning_handler(msg, strict)
+
+
+def validate_pattern_regex(field, strict=True):
+    """
+    Validates if field['pattern'] contains a valid regular expression.
+    """
+    try:
+        re.compile(field['pattern'])
+    except re.error:
+        msg = "Pattern value must be a valid regular expression.\n"
+        msg += f"Offending field name: {field['name']}"
+        strict_warning_handler(msg, strict)


Test patch:
diff --git a/scripts/tests/unit/test_beats_generator.py b/scripts/tests/unit/test_beats_generator.py
--- a/scripts/tests/unit/test_beats_generator.py
+++ b/scripts/tests/unit/test_beats_generator.py
@@ -90,6 +90,38 @@ def test_fieldset_field_array_expected_values_keyword_index_false(self):
         self.assertFalse(field_entry['index'])
         self.assertFalse(field_entry['doc_values'])
 
+    def test_fieldset_field_array_allowed_keys(self):
+        fields = {
+            'id': {
+                'dashed_name': 'agent-id',
+                'description': 'description',
+                'flat_name': 'agent.id',
+                'level': 'extended',
+                'name': 'agent.id',
+                'normalize': [],
+                'short': 'short',
+                'type': 'keyword',
+                'index': False,
+                'doc_values': False,
+                'pattern': '[.*]',
+                'foo': 'bar'
+            }
+        }
+
+        beats_fields = beats.fieldset_field_array(fields, '')
+        field_entry = beats_fields[0]
+        self.assertIn('description', field_entry)
+        self.assertIn('level', field_entry)
+        self.assertIn('name', field_entry)
+        self.assertIn('type', field_entry)
+        self.assertIn('index', field_entry)
+        self.assertIn('doc_values', field_entry)
+        self.assertNotIn('foo', field_entry)
+        self.assertNotIn('dashed_name', field_entry)
+        self.assertNotIn('flat_name', field_entry)
+        self.assertNotIn('normalize', field_entry)
+        self.assertNotIn('short', field_entry)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/scripts/tests/unit/test_schema_cleaner.py b/scripts/tests/unit/test_schema_cleaner.py
--- a/scripts/tests/unit/test_schema_cleaner.py
+++ b/scripts/tests/unit/test_schema_cleaner.py
@@ -329,6 +329,32 @@ def test_multiline_short_description_warns_strict_disabled(self):
         except Exception:
             self.fail("cleaner.single_line_short_description() raised Exception unexpectedly.")
 
+    def test_field_pattern_regex_raises_if_invalid(self):
+        field = {
+            'name': 'test',
+            'pattern': '[.*'
+        }
+        with self.assertRaisesRegex(ValueError, 'Pattern value must be a valid regular expression'):
+            cleaner.validate_pattern_regex(field, strict=True)
+
+    def test_field_pattern_regex_warns_strict_disabled(self):
+        field = {
+            'name': 'test',
+            'pattern': '[.*'
+        }
+        try:
+            with self.assertWarnsRegex(UserWarning, 'valid regular expression'):
+                cleaner.validate_pattern_regex(field, strict=False)
+        except Exception:
+            self.fail("cleaner.validate_pattern_regex() raised Exception unexpectedly.")
+
+    def test_field_pattern_regex_success(self):
+        field = {
+            'name': 'test',
+            'pattern': '[.*]'
+        }
+        self.assertIsNone(cleaner.validate_pattern_regex(field))
+
     def test_field_example_value_is_object_raises(self):
         field = {
             'field_details': {
@@ -387,6 +413,31 @@ def test_example_field_value_is_array_warns_strict_disabled(self):
         except Exception:
             self.fail("cleaner.check_example_value() raised Exception unexpectedly.")
 
+    def test_example_value_mismatch_with_pattern(self):
+        field = {
+            'field_details': {
+                'name': 'test',
+                'example': 'AA',
+                'pattern': 'A{3}'
+            }
+        }
+        with self.assertRaisesRegex(ValueError, 'does not match the regex defined in the pattern'):
+            cleaner.check_example_value(field)
+
+    def test_example_value_mismatch_with_pattern_strict_disabled(self):
+        field = {
+            'field_details': {
+                'name': 'test',
+                'example': 'AA',
+                'pattern': 'A{3}'
+            }
+        }
+        try:
+            with self.assertWarnsRegex(UserWarning, 'does not match the regex defined in the pattern'):
+                cleaner.check_example_value(field, strict=False)
+        except Exception:
+            self.fail("clean.check_example_value() raised Exception unexpectedly.")
+
     def test_very_long_short_override_description_raises(self):
         schema = {
             'schema_details': {