Repository: elastic/ecs

Issue:
Documentation-only field descriptions
### Overview

Support declaring field descriptions only appearing in the documentation.

### Background

Currently, only fields defined in a top-level field set have their fields described in the [field reference](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html).

This proposed feature adds a field's description in the docs. However, no other artifacts (index mappings, Beats, `ecs.*`, etc.) include the field. The field is removed from other artifacts to reduce incorrect usage.

### Examples

* `same_as_process` exists at `process.entry_leader`, `process.session_leader`, and `process.group_leader` but not `process.*`
* `entry_meta.type` exists `process.entry_leader`, `process.session_leader`, and `process.group_leader` but also not at `process.*`

Fields using this option will state their intended usage in the fields' descriptions. For example, `This field is only populated on the process.entry_leader, process.session_leader, and process.group_leader reuses.`


Discussion:

Resolving patch:
diff --git a/scripts/generator.py b/scripts/generator.py
--- a/scripts/generator.py
+++ b/scripts/generator.py
@@ -68,7 +68,7 @@ def main() -> None:
     fields: dict[str, FieldEntry] = loader.load_schemas(ref=args.ref, included_files=args.include)
     cleaner.clean(fields, strict=args.strict)
     finalizer.finalize(fields)
-    fields = subset_filter.filter(fields, args.subset, out_dir)
+    fields, docs_only_fields = subset_filter.filter(fields, args.subset, out_dir)
     fields = exclude_filter.exclude(fields, args.exclude)
     nested, flat = intermediate_files.generate(fields, os.path.join(out_dir, 'ecs'), default_dirs)
 
@@ -84,7 +84,8 @@ def main() -> None:
         exit()
 
     ecs_helpers.make_dirs(docs_dir)
-    asciidoc_fields.generate(nested, ecs_generated_version, docs_dir)
+    docs_only_nested = intermediate_files.generate_nested_fields(docs_only_fields)
+    asciidoc_fields.generate(nested, docs_only_nested, ecs_generated_version, docs_dir)
 
 
 def argument_parser() -> argparse.Namespace:
diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py
--- a/scripts/generators/asciidoc_fields.py
+++ b/scripts/generators/asciidoc_fields.py
@@ -23,13 +23,13 @@
 from generators import ecs_helpers
 
 
-def generate(nested, ecs_generated_version, out_dir):
+def generate(nested, docs_only_nested, ecs_generated_version, out_dir):
     # fields docs now have a dedicated docs subdir: docs/fields
     fields_docs_dir = out_dir + '/fields'
 
     save_asciidoc(path.join(out_dir, 'index.asciidoc'), page_index(ecs_generated_version))
     save_asciidoc(path.join(fields_docs_dir, 'fields.asciidoc'), page_field_index(nested, ecs_generated_version))
-    save_asciidoc(path.join(fields_docs_dir, 'field-details.asciidoc'), page_field_details(nested))
+    save_asciidoc(path.join(fields_docs_dir, 'field-details.asciidoc'), page_field_details(nested, docs_only_nested))
     save_asciidoc(path.join(fields_docs_dir, 'field-values.asciidoc'), page_field_values(nested))
 
 # Helpers
@@ -141,7 +141,8 @@ def save_asciidoc(f, text):
 # jinja2 setup
 
 
-TEMPLATE_DIR = path.join(path.dirname(path.abspath(__file__)), '../templates')
+local_dir = path.dirname(path.abspath(__file__))
+TEMPLATE_DIR = path.join(local_dir, '../templates')
 template_loader = jinja2.FileSystemLoader(searchpath=TEMPLATE_DIR)
 template_env = jinja2.Environment(loader=template_loader, keep_trailing_newline=True)
 
@@ -166,7 +167,10 @@ def page_field_index(nested, ecs_generated_version):
 
 # Field Details Page
 
-def page_field_details(nested):
+def page_field_details(nested, docs_only_nested):
+    if docs_only_nested:
+        for fieldset_name, fieldset in docs_only_nested.items():
+            nested[fieldset_name]['fields'].update(fieldset['fields'])
     fieldsets = ecs_helpers.dict_sorted_by_keys(nested, ['group', 'name'])
     results = (generate_field_details_page(fieldset) for fieldset in fieldsets)
     return ''.join(results)
diff --git a/scripts/schema/subset_filter.py b/scripts/schema/subset_filter.py
--- a/scripts/schema/subset_filter.py
+++ b/scripts/schema/subset_filter.py
@@ -15,11 +15,13 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import copy
 import os
 from typing import (
     Any,
     Dict,
     List,
+    Optional,
 )
 
 from generators import intermediate_files
@@ -48,6 +50,77 @@ def filter(
     merged_subset: Dict[str, Any] = combine_all_subsets(subsets)
     if merged_subset:
         fields = extract_matching_fields(fields, merged_subset)
+
+    # Looks for the `docs_only` attribute, which generates a second field subset
+    # to pass to the ascii_doc generator
+    # After second subset is generated, `docs_only: True` fields are removed
+    # from the `fields` subset
+    docs_only_field_paths = generate_docs_only_paths(merged_subset)
+    if docs_only_field_paths:
+        docs_only_subset = generate_docs_only_subset(docs_only_field_paths)
+        docs_only_fields = extract_matching_fields(fields, docs_only_subset)
+        fields = remove_docs_only_entries(docs_only_field_paths, fields)
+    else:
+        docs_only_fields = {}
+    return fields, docs_only_fields
+
+
+def generate_docs_only_subset(paths: List[str]) -> Dict[str, Any]:
+    """
+    Takes paths list of `docs_only` fields and generates a subset
+    """
+    docs_only_subset = {}
+    for path in paths:
+        # split and reverse
+        split_path = path.split('.')[::-1]
+        current_obj = docs_only_subset
+        while len(split_path) > 1:
+            temp_path = split_path.pop()
+            if not current_obj.get(temp_path):
+                current_obj[temp_path] = {'fields': {}}
+            current_obj = current_obj[temp_path]['fields']
+        current_obj[split_path[-1]] = {}
+    return docs_only_subset
+
+
+def generate_docs_only_paths(
+    subset: Dict[str, Any],
+    filtered: Optional[Dict[str, Any]] = {},
+    parent: Optional[str] = '',
+    path: Optional[str] = '',
+    paths: Optional[List[str]] = [],
+) -> List[str]:
+    """
+    Returns a list of field paths: ['process.same_as_process'] for subset fields
+    marked as `docs_only: True`
+    """
+    for current in subset:
+        if subset[current].get('docs_only'):
+            path += f'.{current}'
+            paths.append(path)
+        if 'fields' in subset[current] and isinstance(subset[current]['fields'], dict):
+            if not parent:
+                path_name = current
+            else:
+                path_name = f'{parent}.{current}'
+            generate_docs_only_paths(subset[current]['fields'],
+                                     filtered=filtered,
+                                     parent=current,
+                                     path=path_name,
+                                     paths=paths
+                                     )
+    return paths
+
+
+def remove_docs_only_entries(paths: List[str], fields: Dict[str, FieldEntry]) -> Dict[str, FieldEntry]:
+    """
+    Removed each path in paths list from the fields object
+    """
+    for path in paths:
+        split_path = path.split('.')
+        field_set = split_path[0]
+        field = split_path[1]
+        del(fields[field_set]['fields'][field])
     return fields
 
 
@@ -69,7 +142,7 @@ def load_subset_definitions(file_globs: List[str]) -> List[Dict[str, Any]]:
     return subsets
 
 
-ecs_options: List[str] = ['fields', 'enabled', 'index']
+ecs_options: List[str] = ['fields', 'enabled', 'index', 'docs_only']
 
 
 def strip_non_ecs_options(subset: Dict[str, Any]) -> None:


Test patch:
diff --git a/scripts/tests/test_asciidoc_fields.py b/scripts/tests/test_asciidoc_fields.py
--- a/scripts/tests/test_asciidoc_fields.py
+++ b/scripts/tests/test_asciidoc_fields.py
@@ -22,10 +22,6 @@
 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
 
 from scripts.generators import asciidoc_fields
-from scripts.generators import intermediate_files
-from scripts.schema import cleaner
-from scripts.schema import loader
-from scripts.schema import finalizer
 
 
 class TestGeneratorsAsciiFields(unittest.TestCase):
diff --git a/scripts/tests/unit/test_schema_subset_filter.py b/scripts/tests/unit/test_schema_subset_filter.py
--- a/scripts/tests/unit/test_schema_subset_filter.py
+++ b/scripts/tests/unit/test_schema_subset_filter.py
@@ -17,7 +17,6 @@
 
 import mock
 import os
-import pprint
 import sys
 import unittest
 
@@ -280,3 +279,92 @@ def test_extract_field_with_options(self):
             }
         }
         self.assertEqual(filtered_fields, expected_fields)
+
+    def test_generate_docs_only_paths_no_entries(self):
+        subset = {
+            'process': {
+                'fields': {
+                    'same_as_process': {},
+                    'meta_entry': {
+                        'fields': {
+                            'type': {}
+                        }
+                    }
+                }
+            }
+        }
+
+        docs_only_paths_empty = subset_filter.generate_docs_only_paths(subset)
+        self.assertEqual(docs_only_paths_empty, [])
+
+    def test_generate_docs_only_paths_with_entries(self):
+        subset = {
+            'process': {
+                'fields': {
+                    'same_as_process': {
+                        'docs_only': True
+                    },
+                    'meta_entry': {
+                        'fields': {
+                            'type': {
+                                'docs_only': True
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        expected_list = ['process.same_as_process', 'process.meta_entry.type']
+        docs_only_paths = subset_filter.generate_docs_only_paths(subset)
+
+        self.assertEqual(docs_only_paths, expected_list)
+
+    def test_generate_docs_only_subset(self):
+        paths = ['process.same_as_process', 'process.meta_entry.type']
+        expected_subset = {
+            'process': {
+                'fields': {
+                    'same_as_process': {},
+                    'meta_entry': {
+                        'fields': {
+                            'type': {}
+                        }
+                    }
+                }
+            }
+        }
+
+        subset = subset_filter.generate_docs_only_subset(paths)
+        self.assertEqual(expected_subset, subset)
+
+    def test_remove_docs_only_entries(self):
+        paths = ['process.same_as_process', 'process.meta_entry.type']
+        orig_subset = {
+            'process': {
+                'fields': {
+                    'same_as_process': {
+                        'docs_only': True
+                    },
+                    'meta_entry': {
+                        'fields': {
+                            'type': {
+                                'docs_only': True
+                            }
+                        }
+                    },
+                    'name': {}
+                }
+            }
+        }
+
+        expected_subset = {
+            'process': {
+                'fields': {
+                    'name': {}
+                }
+            }
+        }
+
+        result = subset_filter.remove_docs_only_entries(paths, orig_subset)
+        self.assertEqual(result, expected_subset)