Repository: elastic/ecs

Issue:
Multi-level self-nesting support
### Problem

In ECS a self-nesting is when one field set is nested inside itself using a different name:

Two examples:

* `process.parent.*` self-nests the `process` field set renamed as `parent`
* `user.target.*` self-nests the `user.*` field set renamed as `target` (same applies to `user.changes` and `user.effective`)

In a _very_ specific case, you may want to self nest the same field set twice:

1. Reuse `process.*` as `process.target.*`
2. Reuse `process` again but underneath `process.target.*` to create `process.target.parent.*`

The current ECS tooling allows specifying this in the `reusable.expected` section, but it will not produce the desired result:

```json
- name: process
  reusable:
    expected:
      - at: process
        as: target
      # collect the parent of the target process at process.target.parent
      - at: process.target
        as: parent
```

### Solution

Implement logic into the [self-nesting phase](https://github.com/elastic/ecs/blob/master/scripts/schema/finalizer.py#L78-L98) of `.finalize` to account for when the `at` attribute is not the top-level field set but an existing self-nesting.


Discussion:
Addressing will make the proposed reuse of `process.target.*` and `process.target.parent.*` for [RFC 0016](https://github.com/elastic/ecs/blob/master/rfcs/text/0016-target-process.md) possible.

cc @rw-access 

Resolving patch:
diff --git a/scripts/schema/finalizer.py b/scripts/schema/finalizer.py
--- a/scripts/schema/finalizer.py
+++ b/scripts/schema/finalizer.py
@@ -89,7 +89,11 @@ def perform_reuse(fields):
             new_field_details['name'] = nest_as
             new_field_details['original_fieldset'] = schema_name
             new_field_details['intermediate'] = True
-            destination_fields = schema['fields']
+            # to handle multi-level self-nesting
+            if reuse_entry['at'] != schema_name:
+                destination_fields = field_group_at_path(reuse_entry['at'], fields)
+            else:
+                destination_fields = schema['fields']
             destination_fields[nest_as] = {
                 'field_details': new_field_details,
                 # Make a new copy of the pristine copy


Test patch:
diff --git a/scripts/tests/unit/test_schema_finalizer.py b/scripts/tests/unit/test_schema_finalizer.py
--- a/scripts/tests/unit/test_schema_finalizer.py
+++ b/scripts/tests/unit/test_schema_finalizer.py
@@ -50,6 +50,8 @@ def schema_process(self):
                                 'short_override': 'short override desc'},
                             {'full': 'reuse.process', 'at': 'reuse', 'as': 'process'},
                             {'full': 'reuse.process.parent', 'at': 'reuse.process', 'as': 'parent'},
+                            {'full': 'reuse.process.target', 'at': 'reuse.process', 'as': 'target'},
+                            {'full': 'reuse.process.target.parent', 'at': 'reuse.process.target', 'as': 'parent'}
                         ]
                     }
                 },
@@ -179,24 +181,28 @@ def test_perform_reuse_with_foreign_reuse_and_self_reuse(self):
         server_fields = fields['server']['fields']
         user_fields = fields['user']['fields']
         process_reuse_fields = fields['reuse']['fields']['process']['fields']
+        process_target_reuse_fields = fields['reuse']['fields']['process']['fields']['target']['fields']
         # Expected reuse
         self.assertIn('parent', process_fields)
         self.assertIn('user', server_fields)
         self.assertIn('target', user_fields)
         self.assertIn('effective', user_fields)
         self.assertIn('parent', process_reuse_fields)
+        self.assertIn('parent', process_target_reuse_fields)
         # Sanity check for presence of leaf fields, after performing reuse
         self.assertIn('name', user_fields['target']['fields'])
         self.assertIn('name', user_fields['effective']['fields'])
         self.assertIn('name', server_fields['user']['fields'])
         self.assertIn('pid', process_fields['parent']['fields'])
         self.assertIn('pid', process_reuse_fields['parent']['fields'])
+        self.assertIn('pid', process_target_reuse_fields['parent']['fields'])
         # Ensure the parent field of reused fields is marked as intermediate
         self.assertTrue(server_fields['user']['field_details']['intermediate'])
         self.assertTrue(process_fields['parent']['field_details']['intermediate'])
         self.assertTrue(user_fields['target']['field_details']['intermediate'])
         self.assertTrue(user_fields['effective']['field_details']['intermediate'])
         self.assertTrue(process_reuse_fields['parent']['field_details']['intermediate'])
+        self.assertTrue(process_target_reuse_fields['parent']['field_details']['intermediate'])
         # No unexpected cross-nesting
         self.assertNotIn('target', user_fields['target']['fields'])
         self.assertNotIn('target', user_fields['effective']['fields'])
@@ -207,6 +213,7 @@ def test_perform_reuse_with_foreign_reuse_and_self_reuse(self):
         self.assertIn('user.target', fields['user']['schema_details']['nestings'])
         self.assertIn('server.user', fields['server']['schema_details']['nestings'])
         self.assertIn('reuse.process.parent', fields['reuse']['schema_details']['nestings'])
+        self.assertIn('reuse.process.target.parent', fields['reuse']['schema_details']['nestings'])
         # Attribute 'reused_here' lists nestings inside a destination schema
         self.assertIn({'full': 'process.parent', 'schema_name': 'process', 'short': 'short override desc'},
                       fields['process']['schema_details']['reused_here'])
@@ -218,6 +225,8 @@ def test_perform_reuse_with_foreign_reuse_and_self_reuse(self):
                       fields['server']['schema_details']['reused_here'])
         self.assertIn({'full': 'reuse.process.parent', 'schema_name': 'process', 'short': 'short desc'},
                       fields['reuse']['schema_details']['reused_here'])
+        self.assertIn({'full': 'reuse.process.target.parent', 'schema_name': 'process', 'short': 'short desc'},
+                      fields['reuse']['schema_details']['reused_here'])
         # Reused fields have an indication they're reused
         self.assertEqual(process_fields['parent']['field_details']['original_fieldset'], 'process',
                          "The parent field of reused fields should have 'original_fieldset' populated")
@@ -228,6 +237,8 @@ def test_perform_reuse_with_foreign_reuse_and_self_reuse(self):
         self.assertEqual(server_fields['user']['fields']['name']['field_details']['original_fieldset'], 'user')
         self.assertEqual(process_reuse_fields['parent']['field_details']['original_fieldset'], 'process',
                          "The parent field of reused fields should have 'original_fieldset' populated")
+        self.assertEqual(process_target_reuse_fields['parent']['field_details']['original_fieldset'], 'process',
+                         "The parent field of reused fields should have 'original_fieldset' populated")
         # Original fieldset's fields must not be marked with 'original_fieldset='
         self.assertNotIn('original_fieldset', user_fields['name']['field_details'])
         self.assertNotIn('original_fieldset', process_fields['pid']['field_details'])