Repository: elastic/ecs

Issue:
Consolidate `field-details` template components
### Summary

Continuing on from the refactor started in #865, consolidate the various [`*.j2` files](https://github.com/elastic/ecs/tree/master/scripts/templates/field_details) that generate the  [field-details](https://github.com/elastic/ecs/blob/master/docs/field-details.asciidoc) documentation.

### Motivation

Have a single file `field-details` template matching the other [templates](https://github.com/elastic/ecs/tree/master/scripts/templates). End goal is to have one single template for each generated page; any asciidoc updates can be done without modifying the underlying generator code or schema definitions.


Discussion:

Resolving patch:
diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py
--- a/scripts/generators/asciidoc_fields.py
+++ b/scripts/generators/asciidoc_fields.py
@@ -5,11 +5,6 @@
 
 from generators import ecs_helpers
 
-# jinja2 setup
-TEMPLATE_DIR = path.join(path.dirname(path.abspath(__file__)), '../templates')
-template_loader = jinja2.FileSystemLoader(searchpath=TEMPLATE_DIR)
-template_env = jinja2.Environment(loader=template_loader)
-
 
 def generate(nested, ecs_version, out_dir):
     save_asciidoc(path.join(out_dir, 'fields.asciidoc'), page_field_index(nested, ecs_version))
@@ -19,6 +14,64 @@ def generate(nested, ecs_version, out_dir):
 # Helpers
 
 
+def render_fieldset_reuse_text(fieldset):
+    """Renders the expected nesting locations
+       if the the `reusable` object is present.
+
+    :param fieldset: The fieldset to evaluate
+    """
+    if not fieldset.get('reusable'):
+        return None
+    reusable_fields = fieldset['reusable']['expected']
+    sorted_fields = sorted(reusable_fields, key=lambda k: k['full'])
+    return map(lambda f: f['full'], sorted_fields)
+
+
+def render_nestings_reuse_section(fieldset):
+    """Renders the reuse section entries.
+
+    :param fieldset: The target fieldset
+    """
+    if not fieldset.get('reused_here'):
+        return None
+    rows = []
+    for reused_here_entry in fieldset['reused_here']:
+        rows.append({
+            'flat_nesting': "{}.*".format(reused_here_entry['full']),
+            'name': reused_here_entry['schema_name'],
+            'short': reused_here_entry['short']
+        })
+
+    return sorted(rows, key=lambda x: x['flat_nesting'])
+
+
+def extract_allowed_values_key_names(field):
+    """Extracts the `name` keys from the field's
+       allowed_values if present in the field
+       object.
+
+    :param field: The target field
+    """
+    if not field.get('allowed_values'):
+        return []
+    return ecs_helpers.list_extract_keys(field['allowed_values'], 'name')
+
+
+def sort_fields(fieldset):
+    """Prepares a fieldset's fields for being
+    passed into the j2 template for rendering. This
+    includes sorting them into a list of objects and
+    adding a field for the names of any allowed values
+    for the field, if present.
+
+    :param fieldset: The target fieldset
+    """
+    fields_list = list(fieldset['fields'].values())
+    for field in fields_list:
+        field['allowed_value_names'] = extract_allowed_values_key_names(field)
+    return sorted(fields_list, key=lambda field: field['name'])
+
+
 def templated(template_name):
     """Decorator function to simplify rendering a template.
 
@@ -53,12 +106,19 @@ def save_asciidoc(f, text):
     with open(f, "w") as outfile:
         outfile.write(text)
 
+# jinja2 setup
+
+
+TEMPLATE_DIR = path.join(path.dirname(path.abspath(__file__)), '../templates')
+template_loader = jinja2.FileSystemLoader(searchpath=TEMPLATE_DIR)
+template_env = jinja2.Environment(loader=template_loader)
 
 # Rendering schemas
 
 # Field Index
 
-@templated('fields_template.j2')
+
+@templated('fields.j2')
 def page_field_index(nested, ecs_version):
     fieldsets = ecs_helpers.dict_sorted_by_keys(nested, ['group', 'name'])
     return dict(ecs_version=ecs_version, fieldsets=fieldsets)
@@ -66,197 +126,27 @@ def page_field_index(nested, ecs_version):
 
 # Field Details Page
 
-
 def page_field_details(nested):
-    page_text = ''
-    for fieldset in ecs_helpers.dict_sorted_by_keys(nested, ['group', 'name']):
-        page_text += render_fieldset(fieldset, nested)
-    return page_text
-
-
-def render_fieldset(fieldset, nested):
-    text = field_details_table_header(
-        title=fieldset['title'],
-        name=fieldset['name'],
-        description=fieldset['description']
-    )
-
-    text += render_fields(fieldset['fields'])
-
-    text += table_footer()
-
-    text += render_fieldset_reuse_section(fieldset, nested)
-
-    return text
-
-
-def render_fields(fields):
-    text = ''
-    for _, field in sorted(fields.items()):
-        # Skip fields nested in this field set
-        if 'original_fieldset' not in field:
-            text += render_field_details_row(field)
-    return text
-
-
-def render_field_allowed_values(field):
-    if not 'allowed_values' in field:
-        return ''
-    allowed_values = ', '.join(ecs_helpers.list_extract_keys(field['allowed_values'], 'name'))
-
-    return field_acceptable_value_names(
-        allowed_values=allowed_values,
-        flat_name=field['flat_name'],
-        dashed_name=field['dashed_name']
-    )
-
-
-def render_field_details_row(field):
-    example = ''
-    if 'allowed_values' in field:
-        example = render_field_allowed_values(field)
-    elif 'example' in field:
-        example = "example: `{}`".format(str(field['example']))
-
-    field_type_with_mf = field['type']
-    if 'multi_fields' in field:
-        field_type_with_mf += "\n\nMulti-fields:\n\n"
-        for mf in field['multi_fields']:
-            field_type_with_mf += "* {} (type: {})\n\n".format(mf['flat_name'], mf['type'])
-
-    field_normalization = ''
-    if 'array' in field['normalize']:
-        field_normalization = "\nNote: this field should contain an array of values.\n\n"
-
-    text = field_details_row(
-        flat_name=field['flat_name'],
-        description=field['description'],
-        field_type=field_type_with_mf,
-        example=example,
-        normalization=field_normalization,
-        level=field['level']
-    )
-
-    return text
-
-
-def render_fieldset_reuse_section(fieldset, nested):
-    '''Render the section on where field set can be nested, and which field sets can be nested here'''
-    if not ('nestings' in fieldset or 'reusable' in fieldset):
-        return ''
-
-    text = field_reuse_section(
-        reuse_of_fieldset=render_fieldset_reuses_text(fieldset)
-    )
-
-    if 'nestings' in fieldset:
-        text += nestings_table_header(
-            name=fieldset['name'],
-            title=fieldset['title']
-        )
-        rows = []
-        for reused_here_entry in fieldset['reused_here']:
-            rows.append({
-                'flat_nesting': "{}.*".format(reused_here_entry['full']),
-                'name': reused_here_entry['schema_name'],
-                'short': reused_here_entry['short']
-            })
-
-        for row in sorted(rows, key=lambda x: x['flat_nesting']):
-            text += nestings_row(
-                nesting_name=row['name'],
-                flat_nesting=row['flat_nesting'],
-                nesting_short=row['short']
-            )
-
-        text += table_footer()
-    return text
-
-
-def render_fieldset_reuses_text(fieldset):
-    '''Render where a given field set is expected to be reused'''
-    if 'reusable' not in fieldset:
-        return ''
-
-    section_name = fieldset['name']
-    sorted_fields = sorted(fieldset['reusable']['expected'], key=lambda k: k['full'])
-    rendered_fields = map(lambda f: "`{}`".format(f['full']), sorted_fields)
-    text = "The `{}` fields are expected to be nested at: {}.\n\n".format(
-        section_name, ', '.join(rendered_fields))
-
-    if 'top_level' in fieldset['reusable'] and fieldset['reusable']['top_level']:
-        template = "Note also that the `{}` fields may be used directly at the root of the events.\n\n"
-    else:
-        template = "Note also that the `{}` fields are not expected to " + \
-            "be used directly at the root of the events.\n\n"
-    text += template.format(section_name)
-    return text
-
-
-# Templates
-
-def table_footer():
-    return '''
-|=====
-'''
-
-# Field Details Page
-
-# Main Fields Table
-
-
-@templated('field_details/table_header.j2')
-def field_details_table_header(title, name, description):
-    return dict(name=name, title=title, description=description)
-
-
-@templated('field_details/row.j2')
-def field_details_row(flat_name, description, field_type, normalization, example, level):
-    return dict(
-        flat_name=flat_name,
-        description=description,
-        field_type=field_type,
-        normalization=normalization,
-        example=example,
-        level=level
-    )
-
-
-@templated('field_details/acceptable_value_names.j2')
-def field_acceptable_value_names(allowed_values, dashed_name, flat_name):
-    return dict(
-        allowed_values=allowed_values,
-        dashed_name=dashed_name,
-        flat_name=flat_name
-    )
-
-
-# Field reuse
-
-@templated('field_details/field_reuse_section.j2')
-def field_reuse_section(reuse_of_fieldset):
-    return dict(reuse_of_fieldset=reuse_of_fieldset)
-
-
-# Nestings table
-
-@templated('field_details/nestings_table_header.j2')
-def nestings_table_header(name, title):
-    return dict(name=name, title=title)
+    fieldsets = ecs_helpers.dict_sorted_by_keys(nested, ['group', 'name'])
+    results = (generate_field_details_page(fieldset) for fieldset in fieldsets)
+    return ''.join(results)
 
 
-@templated('field_details/nestings_row.j2')
-def nestings_row(nesting_name, flat_nesting, nesting_short):
-    return dict(
-        nesting_name=nesting_name,
-        flat_nesting=flat_nesting,
-        nesting_short=nesting_short
-    )
+@templated('field_details.j2')
+def generate_field_details_page(fieldset):
+    # render field reuse text section
+    sorted_reuse_fields = render_fieldset_reuse_text(fieldset)
+    render_nestings_reuse_fields = render_nestings_reuse_section(fieldset)
+    sorted_fields = sort_fields(fieldset)
+    return dict(fieldset=fieldset,
+                sorted_reuse_fields=sorted_reuse_fields,
+                render_nestings_reuse_section=render_nestings_reuse_fields,
+                sorted_fields=sorted_fields)
 
 
 # Allowed values section
 
-@templated('field_values_template.j2')
+@templated('field_values.j2')
 def page_field_values(nested, template_name='field_values_template.j2'):
     category_fields = ['event.kind', 'event.category', 'event.type', 'event.outcome']
     nested_fields = []


Test patch:
diff --git a/scripts/tests/test_asciidoc_fields.py b/scripts/tests/test_asciidoc_fields.py
new file mode 100644
--- /dev/null
+++ b/scripts/tests/test_asciidoc_fields.py
@@ -0,0 +1,132 @@
+import os
+import sys
+import unittest
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+
+from scripts.generators import asciidoc_fields
+from scripts.generators import intermediate_files
+from scripts.schema import cleaner
+from scripts.schema import loader
+from scripts.schema import finalizer
+
+
+class TestGeneratorsAsciiFields(unittest.TestCase):
+
+    def setUp(self):
+        self.foo_fieldset = self.dummy_fieldset()
+
+    def dummy_fieldset(self):
+        return {
+            'description': 'foo',
+            'fields': {
+                'foo.type': {
+                    'dashed_name': 'foo-type',
+                    'description': 'describes the foo',
+                    'example': '2016-05-23T08:05:34.853Z',
+                    'flat_name': 'foo.type',
+                    'level': 'core',
+                    'name': 'type',
+                    'normalize': ['array'],
+                    'short': 'describes the foo',
+                    'ignore_above': 1024,
+                    'type': 'keyword',
+                    'allowed_values': [{
+                        'description': 'fluffy foo',
+                        'name': 'fluffy',
+                    },
+                        {
+                        'description': 'coarse foo',
+                        'name': 'coarse',
+                    }
+                    ]
+                },
+                'foo.id': {
+                    'dashed_name': 'foo-id',
+                    'description': 'Unique ID of the foo.',
+                    'example': 'foo123',
+                    'flat_name': 'foo.id',
+                    'ignore_above': 1024,
+                    'level': 'core',
+                    'name': 'id',
+                    'normalize': [],
+                    'short': 'Unique ID of the foo.',
+                    'type': 'keyword'
+                }
+            },
+            'reusable': {
+                'expected': [
+                    {
+                        'as': 'foo',
+                        'at': 'server',
+                        'full': 'server.foo'
+                    },
+                    {
+                        'as': 'foo',
+                        'at': 'source',
+                        'full': 'source.foo'
+                    },
+                    {
+                        'as': 'foo',
+                        'at': 'client',
+                        'full': 'client.foo',
+                    },
+                    {
+                        'as': 'foo',
+                        'at': 'destination',
+                        'full': 'destination.foo'
+                    }
+                ],
+                'top_level': False,
+            },
+            'reused_here': [
+                {
+                    'full': 'foo.as',
+                    'schema_name': 'as',
+                    'short': 'Fields describing an AS'
+                }
+            ],
+            'group': 2,
+            'name': 'foo',
+            'prefix': 'foo.',
+            'short': 'Foo fields',
+            'title': 'Foo',
+            'type': 'group'
+        }
+
+    def test_validate_sort_fieldset(self):
+        sorted_foo_fields = asciidoc_fields.sort_fields(self.foo_fieldset)
+        #import pdb;pdb.set_trace()
+        self.assertIsInstance(sorted_foo_fields, list)
+
+        # `allowed_value_names` always present
+        for field in sorted_foo_fields:
+            self.assertIsInstance(field.get('allowed_value_names'), list)
+
+        self.assertFalse(sorted_foo_fields[0]['allowed_value_names'])
+        self.assertEqual('id', sorted_foo_fields[0]['name'])
+        self.assertEqual('type', sorted_foo_fields[1]['name'])
+        self.assertIn('fluffy', sorted_foo_fields[1]['allowed_value_names'])
+        self.assertIn('coarse', sorted_foo_fields[1]['allowed_value_names'])
+
+    def test_rendering_fieldset_reuse(self):
+        foo_reuse_fields = asciidoc_fields.render_fieldset_reuse_text(self.foo_fieldset)
+        expected_sorted_reuse_fields = (
+            'client.foo',
+            'destination.foo',
+            'server.foo',
+            'source.foo'
+        )
+
+        self.assertEqual(expected_sorted_reuse_fields, tuple(foo_reuse_fields))
+
+    def test_rendering_fieldset_nesting(self):
+        foo_nesting_fields = asciidoc_fields.render_nestings_reuse_section(self.foo_fieldset)
+        self.assertIsInstance(foo_nesting_fields, list)
+        self.assertEqual('foo.as.*', foo_nesting_fields[0]['flat_nesting'])
+        self.assertEqual('as', foo_nesting_fields[0]['name'])
+        self.assertEqual('Fields describing an AS', foo_nesting_fields[0]['short'])
+
+
+if __name__ == '__main__':
+    unittest.main()