Repository: elastic/ecs

Issue:
Mark which field set reuses expect arrays
### Purpose

Indicate in the field definitions when a field set reuse expects an array.

### Background

Reuses allowing arrays may benefit specific limited use. However, there's currently no way to declare this reuse type in the ECS tooling.



```json
{
  "process": {
    "previous" : [
      {
        "name": "abc.exe"
      },
      {
        "name": "xyz.exe"
      }
    ]
  }
}
```

The above basic example presents an ECS mapped event using field set array reuse. The `previous` object is `process.*` field set reused at `process.previous`.

The `normalize: [ 'array' ]` setting controls when a particular field expects an array of values. There's no dedicated `array` type in Elasticsearch, and any field can contain zero or more values of the same type. So when the `array` attribute is specified, a note is added by the generator script to that field's description in the ECS docs. 

Field def setting:

```yaml
    - name: example
      normalize:
        - array
```

Results in this text added to field's description:

<img width="390" alt="Screen Shot 2022-01-20 at 1 06 36 PM" src="https://user-images.githubusercontent.com/7226265/150405432-baefc222-0990-4717-9b84-47f24261ee7f.png">

### Proposed design

Implement a similar mechanism as leaf fields:

* specify an attribute on a field reuse
* have the asciidoc generator handle including that content in the field reuse description

### Open question(s)

* Do we reuse the existing `normalize` attribute convention? Or a simple `bool` parameter (e.g., `is_array`)?



Discussion:

Resolving patch:
diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py
--- a/scripts/generators/asciidoc_fields.py
+++ b/scripts/generators/asciidoc_fields.py
@@ -61,7 +61,8 @@ def render_nestings_reuse_section(fieldset):
             'flat_nesting': "{}.*".format(reused_here_entry['full']),
             'name': reused_here_entry['schema_name'],
             'short': reused_here_entry['short'],
-            'beta': reused_here_entry.get('beta', '')
+            'beta': reused_here_entry.get('beta', ''),
+            'normalize': reused_here_entry.get('normalize')
         })
 
     return sorted(rows, key=lambda x: x['flat_nesting'])
diff --git a/scripts/schema/finalizer.py b/scripts/schema/finalizer.py
--- a/scripts/schema/finalizer.py
+++ b/scripts/schema/finalizer.py
@@ -150,6 +150,9 @@ def append_reused_here(reused_schema, reuse_entry, destination_schema):
         # Check for a short override, if not present, fall back to the top-level fieldset's short
         'short': reuse_entry['short_override'] if 'short_override' in reuse_entry else reused_schema['field_details']['short']
     }
+    # If it exists, bring through the normalization
+    if 'normalize' in reuse_entry:
+        reused_here_entry['normalize'] = reuse_entry['normalize']
     # Check for beta attribute
     if 'beta' in reuse_entry:
         reused_here_entry['beta'] = reuse_entry['beta']


Test patch:
diff --git a/scripts/tests/test_asciidoc_fields.py b/scripts/tests/test_asciidoc_fields.py
--- a/scripts/tests/test_asciidoc_fields.py
+++ b/scripts/tests/test_asciidoc_fields.py
@@ -103,7 +103,13 @@ def dummy_fieldset(self):
                     'full': 'foo.as',
                     'schema_name': 'as',
                     'short': 'Fields describing an AS'
-                }
+                },
+                {
+                    'full': 'foo.file',
+                    'schema_name': 'file',
+                    'short': 'Fields describing files',
+                    'normalize': ['array']
+                },
             ],
             'group': 2,
             'name': 'foo',
@@ -215,6 +221,11 @@ def test_rendering_fieldset_nesting(self):
         self.assertEqual('as', foo_nesting_fields[0]['name'])
         self.assertEqual('Fields describing an AS', foo_nesting_fields[0]['short'])
 
+        self.assertEqual('foo.file.*', foo_nesting_fields[1]['flat_nesting'])
+        self.assertEqual('file', foo_nesting_fields[1]['name'])
+        self.assertEqual('Fields describing files', foo_nesting_fields[1]['short'])
+        self.assertEqual(['array'], foo_nesting_fields[1]['normalize'])
+
     def test_check_for_usage_doc_true(self):
         usage_files = ["foo.asciidoc"]
         foo_name = self.foo_fieldset.get('name')
diff --git a/scripts/tests/unit/test_schema_finalizer.py b/scripts/tests/unit/test_schema_finalizer.py
--- a/scripts/tests/unit/test_schema_finalizer.py
+++ b/scripts/tests/unit/test_schema_finalizer.py
@@ -65,6 +65,7 @@ def schema_process(self):
                         'expected': [
                             {'full': 'process.parent', 'at': 'process', 'as': 'parent',
                                 'short_override': 'short override desc'},
+                            {'full': 'process.previous', 'at': 'process', 'as': 'previous', 'normalize': ['array']},
                             {'full': 'reuse.process', 'at': 'reuse', 'as': 'process'},
                             {'full': 'reuse.process.parent', 'at': 'reuse.process', 'as': 'parent'},
                             {'full': 'reuse.process.target', 'at': 'reuse.process', 'as': 'target'},
@@ -201,6 +202,7 @@ def test_perform_reuse_with_foreign_reuse_and_self_reuse(self):
         process_target_reuse_fields = fields['reuse']['fields']['process']['fields']['target']['fields']
         # Expected reuse
         self.assertIn('parent', process_fields)
+        self.assertIn('previous', process_fields)
         self.assertIn('user', server_fields)
         self.assertIn('target', user_fields)
         self.assertIn('effective', user_fields)
@@ -226,6 +228,7 @@ def test_perform_reuse_with_foreign_reuse_and_self_reuse(self):
         self.assertNotIn('target', server_fields['user']['fields'])
         # Legacy list of nestings, added to destination schema
         self.assertIn('process.parent', fields['process']['schema_details']['nestings'])
+        self.assertIn('process.previous', fields['process']['schema_details']['nestings'])
         self.assertIn('user.effective', fields['user']['schema_details']['nestings'])
         self.assertIn('user.target', fields['user']['schema_details']['nestings'])
         self.assertIn('server.user', fields['server']['schema_details']['nestings'])
@@ -234,6 +237,8 @@ def test_perform_reuse_with_foreign_reuse_and_self_reuse(self):
         # Attribute 'reused_here' lists nestings inside a destination schema
         self.assertIn({'full': 'process.parent', 'schema_name': 'process', 'short': 'short override desc'},
                       fields['process']['schema_details']['reused_here'])
+        self.assertIn({'full': 'process.previous', 'schema_name': 'process', 'short': 'short desc', 'normalize': ['array']},
+                      fields['process']['schema_details']['reused_here'])
         self.assertIn({'full': 'user.effective', 'schema_name': 'user', 'short': 'short desc'},
                       fields['user']['schema_details']['reused_here'])
         self.assertIn({'full': 'user.target', 'schema_name': 'user', 'short': 'short desc', 'beta': 'Some beta notice'},