Repository: apache/airflow

Issue:
Automate docs for `values.yaml` via pre-commit config and break them into logical groups
Automate docs for values.yaml via pre-commit config and break them into logical groups like https://github.com/bitnami/charts/tree/master/bitnami/airflow


Discussion:

Resolving patch:
diff --git a/docs/conf.py b/docs/conf.py
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -32,6 +32,7 @@
 # serve to show the default.
 """Configuration of Airflow Docs"""
 import glob
+import json
 import os
 import sys
 from collections import defaultdict
@@ -77,6 +78,9 @@
         raise Exception(f"Could not find provider.yaml file for package: {PACKAGE_NAME}")
     PACKAGE_DIR = CURRENT_PROVIDER['package-dir']
     PACKAGE_VERSION = 'devel'
+elif PACKAGE_NAME == 'helm-chart':
+    PACKAGE_DIR = os.path.join(ROOT_DIR, 'chart')
+    PACKAGE_VERSION = 'devel'  # TODO do we care? probably
 else:
     PACKAGE_DIR = None
     PACKAGE_VERSION = 'devel'
@@ -144,6 +148,7 @@
             "sphinx_script_update",
         ]
     )
+
 if PACKAGE_NAME == "apache-airflow-providers":
     extensions.extend(
         [
@@ -151,7 +156,9 @@
             'providers_packages_ref',
         ]
     )
-elif PACKAGE_NAME in ("helm-chart", "docker-stack"):
+elif PACKAGE_NAME == "helm-chart":
+    extensions.append("sphinxcontrib.jinja")
+elif PACKAGE_NAME == "docker-stack":
     # No extra extensions
     pass
 else:
@@ -347,13 +354,88 @@ def _load_config():
         if not os.path.exists(file_path):
             return {}
 
-        with open(file_path) as config_file:
-            return yaml.load(config_file, SafeLoader)
+        with open(file_path) as f:
+            return yaml.load(f, SafeLoader)
 
     config = _load_config()
     if config:
         jinja_contexts = {'config_ctx': {"configs": config}}
         extensions.append('sphinxcontrib.jinja')
+elif PACKAGE_NAME == 'helm-chart':
+
+    def _format_default(value: Any) -> str:
+        if value == "":
+            return '""'
+        if value is None:
+            return '~'
+        return str(value)
+
+    def _format_examples(param_name: str, schema: dict) -> Optional[str]:
+        if not schema.get("examples"):
+            return None
+
+        # Nicer to have the parameter name shown as well
+        out = ""
+        for ex in schema["examples"]:
+            if schema["type"] == "array":
+                ex = [ex]
+            out += yaml.dump({param_name: ex})
+        return out
+
+    def _get_params(root_schema: dict, prefix: str = "", default_section: str = "") -> List[dict]:
+        """
+        Given an jsonschema objects properties dict, return a flattened list of all parameters
+        from that object and any nested objects
+        """
+        # TODO: handle arrays? probably missing more cases too
+        out = []
+        for param_name, schema in root_schema.items():
+            prefixed_name = f"{prefix}.{param_name}" if prefix else param_name
+            section_name = schema["docs_section"] if "docs_section" in schema else default_section
+            if section_name and schema["description"] and "default" in schema:
+                out.append(
+                    {
+                        "section": section_name,
+                        "name": prefixed_name,
+                        "description": schema["description"],
+                        "default": _format_default(schema["default"]),
+                        "examples": _format_examples(param_name, schema),
+                    }
+                )
+            if schema.get("properties"):
+                out += _get_params(schema["properties"], prefixed_name, section_name)
+        return out
+
+    schema_file = os.path.join(PACKAGE_DIR, "values.schema.json")  # type: ignore
+    with open(schema_file) as config_file:
+        chart_schema = json.load(config_file)
+
+    params = _get_params(chart_schema["properties"])
+
+    # Now, split into sections
+    sections: Dict[str, List[Dict[str, str]]] = {}
+    for param in params:
+        if param["section"] not in sections:
+            sections[param["section"]] = []
+
+        sections[param["section"]].append(param)
+
+    # and order each section
+    for section in sections.values():  # type: ignore
+        section.sort(key=lambda i: i["name"])  # type: ignore
+
+    # and finally order the sections!
+    ordered_sections = []
+    for name in chart_schema["docs_section_order"]:
+        if name not in sections:
+            raise ValueError(f"Unable to find any parameters for section: {name}")
+        ordered_sections.append({"name": name, "params": sections.pop(name)})
+
+    if sections:
+        raise ValueError(f"Found section(s) which were not in `section_order`: {list(sections.keys())}")
+
+    jinja_contexts = {"params_ctx": {"sections": ordered_sections}}
+
 
 # -- Options for sphinx.ext.autodoc --------------------------------------------
 # See: https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html
diff --git a/scripts/ci/pre_commit/pre_commit_json_schema.py b/scripts/ci/pre_commit/pre_commit_json_schema.py
--- a/scripts/ci/pre_commit/pre_commit_json_schema.py
+++ b/scripts/ci/pre_commit/pre_commit_json_schema.py
@@ -26,7 +26,8 @@
 
 import requests
 import yaml
-from jsonschema.validators import validator_for
+from jsonschema.exceptions import ValidationError
+from jsonschema.validators import extend, validator_for
 
 if __name__ != "__main__":
     raise Exception(
@@ -107,6 +108,9 @@ def load_file(file_path: str):
 
 def _get_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(description='Validates the file using JSON Schema specifications')
+    parser.add_argument(
+        '--enforce-defaults', action='store_true', help="Values must match the default in the schema"
+    )
 
     group = parser.add_mutually_exclusive_group(required=True)
     group.add_argument('--spec-file', help="The path to specification")
@@ -128,12 +132,20 @@ def _process_files(validator, file_paths: List[str]):
     return exit_code
 
 
-def _create_validator(schema):
+def _create_validator(schema, enforce_defaults: bool):
     cls = validator_for(schema)
     cls.check_schema(schema)
+    if enforce_defaults:
+        cls = extend(cls, {"default": _default_validator})
     return cls(schema)
 
 
+def _default_validator(validator, default, instance, schema):
+    # We will also accept a "See values.yaml" default
+    if default != instance and default != "See values.yaml":
+        yield ValidationError(f"{instance} is not equal to the default of {default}")
+
+
 def _load_spec(spec_file: Optional[str], spec_url: Optional[str]):
     if spec_url:
         spec_file = fetch_and_cache(url=spec_url, output_filename=re.sub(r"[^a-zA-Z0-9]", "-", spec_url))
@@ -150,10 +162,11 @@ def main() -> int:
     args = parser.parse_args()
     spec_url = args.spec_url
     spec_file = args.spec_file
+    enforce_defaults = args.enforce_defaults
 
     schema = _load_spec(spec_file, spec_url)
 
-    validator = _create_validator(schema)
+    validator = _create_validator(schema, enforce_defaults)
 
     file_paths = args.file
     exit_code = _process_files(validator, file_paths)


Test patch: