Repository: elastic/ecs

Issue:
Incorrect generated/beats/fields.ecs.yml, not accounting on top_level: false 
**Description of the issue:**

We are using the following file as a source of ECS mappings for osquery_manager integration:
https://github.com/elastic/ecs/blob/main/generated/beats/fields.ecs.yml

It does have `as` field at the the top level there, which causes our integration validation to fail for 3.0.0 format spec.

As far as I understand generated file should not include `as` field at the top level because it is marked with top_level: false attribute.
https://github.com/elastic/ecs/blob/969aeba5018d92331eda7e4104c4c53225c32fd7/generated/ecs/ecs_nested.yml#L157

Currently it has

<img width="678" alt="Screenshot 2023-09-20 at 10 21 06 AM" src="https://github.com/elastic/ecs/assets/872351/820a9444-d655-42cb-afa2-3f4ebc080a62">

and not additional information for us of the field is a top level or not
<img width="817" alt="Screenshot 2023-09-20 at 10 22 00 AM" src="https://github.com/elastic/ecs/assets/872351/39465fb1-b91a-4e76-a013-7ad08511ea8f">





Discussion:
Same issue with other fields, here is the list
```
	"as",
	"code_signature",
	"elf",
	"macho",
	"pe",
	"risk",
	"x509",
```

Resolving patch:
diff --git a/scripts/generators/beats.py b/scripts/generators/beats.py
--- a/scripts/generators/beats.py
+++ b/scripts/generators/beats.py
@@ -36,6 +36,7 @@ def generate(
     out_dir: str
 ) -> None:
     # base first
+    ecs_nested = ecs_helpers.remove_top_level_reusable_false(ecs_nested)
     beats_fields: List[OrderedDict] = fieldset_field_array(ecs_nested['base']['fields'], ecs_nested['base']['prefix'])
 
     allowed_fieldset_keys: List[str] = ['name', 'title', 'group', 'description', 'footnote', 'type']
diff --git a/scripts/generators/ecs_helpers.py b/scripts/generators/ecs_helpers.py
--- a/scripts/generators/ecs_helpers.py
+++ b/scripts/generators/ecs_helpers.py
@@ -254,6 +254,17 @@ def is_intermediate(field: FieldEntry) -> bool:
     return ('intermediate' in field['field_details'] and field['field_details']['intermediate'])
 
 
+def remove_top_level_reusable_false(ecs_nested: Dict[str, FieldNestedEntry]) -> Dict[str, FieldNestedEntry]:
+    """Returns same structure as ecs_nested, but skips all field sets with reusable.top_level: False"""
+    components: Dict[str, FieldNestedEntry] = {}
+    for (fieldset_name, fieldset) in ecs_nested.items():
+        if fieldset.get('reusable', None):
+            if not fieldset['reusable']['top_level']:
+                continue
+        components[fieldset_name] = fieldset
+    return components
+
+
 # Warning helper
 
 
diff --git a/scripts/generators/es_template.py b/scripts/generators/es_template.py
--- a/scripts/generators/es_template.py
+++ b/scripts/generators/es_template.py
@@ -65,7 +65,7 @@ def all_component_templates(
     component_dir: str = join(out_dir, 'elasticsearch/composable/component')
     ecs_helpers.make_dirs(component_dir)
 
-    for (fieldset_name, fieldset) in candidate_components(ecs_nested).items():
+    for (fieldset_name, fieldset) in ecs_helpers.remove_top_level_reusable_false(ecs_nested).items():
         field_mappings = {}
         for (flat_name, field) in fieldset['fields'].items():
             name_parts = flat_name.split('.')
@@ -104,22 +104,11 @@ def component_name_convention(
 ) -> List[str]:
     version: str = ecs_version.replace('+', '-')
     names: List[str] = []
-    for (fieldset_name, fieldset) in candidate_components(ecs_nested).items():
+    for (fieldset_name, fieldset) in ecs_helpers.remove_top_level_reusable_false(ecs_nested).items():
         names.append("ecs_{}_{}".format(version, fieldset_name.lower()))
     return names
 
 
-def candidate_components(ecs_nested: Dict[str, FieldNestedEntry]) -> Dict[str, FieldNestedEntry]:
-    """Returns same structure as ecs_nested, but skips all field sets with reusable.top_level: False"""
-    components: Dict[str, FieldNestedEntry] = {}
-    for (fieldset_name, fieldset) in ecs_nested.items():
-        if fieldset.get('reusable', None):
-            if not fieldset['reusable']['top_level']:
-                continue
-        components[fieldset_name] = fieldset
-    return components
-
-
 # Legacy template
 
 


Test patch:
diff --git a/scripts/tests/test_ecs_helpers.py b/scripts/tests/test_ecs_helpers.py
--- a/scripts/tests/test_ecs_helpers.py
+++ b/scripts/tests/test_ecs_helpers.py
@@ -157,6 +157,18 @@ def test_glob_yaml_files(self):
             ecs_helpers.glob_yaml_files('schemas/*.yml'))) >= min_schema_count)
         self.assertEqual(len(ecs_helpers.glob_yaml_files(ecs_helpers.glob_yaml_files('schemas/*.yaml'))), 0)
 
+    # Remove top_level:false field sets helper
+
+    def test_remove_top_level_false_field_sets(self):
+        nested_schema_original = {
+            'as': {'group': 2, 'name': 'as', 'reusable': {'top_level': False}},
+            'agent': {'group': 2, 'name': 'agent'},
+        }
+        nested_schema_expected = {
+            'agent': {'group': 2, 'name': 'agent'}
+        }
+        self.assertEqual(ecs_helpers.remove_top_level_reusable_false(nested_schema_original), nested_schema_expected)
+
 
 if __name__ == '__main__':
     unittest.main()