Repository: elastic/ecs

Issue:
`doc_values` parameter not set in Beats artifact 
Certain fields have `index: false` and `doc_values: false` in their ECS definition, like `event.original`:

https://github.com/elastic/ecs/blob/master/schemas/event.yml#L577-L599

When `doc_values: false` is defined in the field definition, it's not being added to the maintained Beats fields YAML artifact:

https://github.com/elastic/ecs/blob/master/generated/beats/fields.ecs.yml#L1737-L1750


Discussion:

Resolving patch:
diff --git a/scripts/generators/beats.py b/scripts/generators/beats.py
--- a/scripts/generators/beats.py
+++ b/scripts/generators/beats.py
@@ -39,7 +39,8 @@ def fieldset_field_array(source_fields, df_allowlist, fieldset_prefix):
     allowed_keys = ['name', 'level', 'required', 'type', 'object_type',
                     'ignore_above', 'multi_fields', 'format', 'input_format',
                     'output_format', 'output_precision', 'description',
-                    'example', 'enabled', 'index', 'path', 'scaling_factor']
+                    'example', 'enabled', 'index', 'doc_values', 'path',
+                    'scaling_factor']
     multi_fields_allowed_keys = ['name', 'type', 'norms', 'default_field', 'normalizer', 'ignore_above']
 
     fields = []


Test patch:
diff --git a/scripts/tests/unit/test_beats_generator.py b/scripts/tests/unit/test_beats_generator.py
new file mode 100644
--- /dev/null
+++ b/scripts/tests/unit/test_beats_generator.py
@@ -0,0 +1,78 @@
+import os
+import sys
+from typing import OrderedDict
+import unittest
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '../..'))
+
+from scripts.generators import beats
+
+
+class TestGeneratorsBeatsFields(unittest.TestCase):
+    def setUp(self):
+        self.df_allowlist = {"@timestamp"}
+
+    def test_fieldset_field_array_expected_structure(self):
+        fields = {
+            'id': {
+                'dashed_name': 'agent-id',
+                'description': 'description',
+                'flat_name': 'agent.id',
+                'ignore_above': 1024,
+                'level': 'extended',
+                'name': 'agent.id',
+                'normalize': [],
+                'short': 'short',
+                'type': 'keyword'
+            }
+        }
+        beats_fields = beats.fieldset_field_array(fields, self.df_allowlist, 'prefix')
+        self.assertIsInstance(beats_fields, list)
+        self.assertIsInstance(beats_fields[0], OrderedDict)
+
+    def test_fieldset_field_array_expected_values_keyword(self):
+        fields = {
+            'id': {
+                'dashed_name': 'agent-id',
+                'description': 'description',
+                'flat_name': 'agent.id',
+                'ignore_above': 1024,
+                'level': 'extended',
+                'name': 'agent.id',
+                'normalize': [],
+                'short': 'short',
+                'type': 'keyword'
+            }
+        }
+
+        beats_fields = beats.fieldset_field_array(fields, self.df_allowlist, '')
+        field_entry = beats_fields[0]
+        self.assertEqual(field_entry['type'], 'keyword')
+        self.assertEqual(field_entry['ignore_above'], 1024)
+        self.assertEqual(field_entry['name'], 'id')
+
+    def test_fieldset_field_array_expected_values_keyword_index_false(self):
+        fields = {
+            'id': {
+                'dashed_name': 'agent-id',
+                'description': 'description',
+                'flat_name': 'agent.id',
+                'level': 'extended',
+                'name': 'agent.id',
+                'normalize': [],
+                'short': 'short',
+                'type': 'keyword',
+                'index': False,
+                'doc_values': False
+            }
+        }
+
+        beats_fields = beats.fieldset_field_array(fields, self.df_allowlist, '')
+        field_entry = beats_fields[0]
+        self.assertEqual(field_entry['type'], 'keyword')
+        self.assertFalse(field_entry['index'])
+        self.assertFalse(field_entry['doc_values'])
+
+
+if __name__ == '__main__':
+    unittest.main()