Repository: canonical/cloud-init

Issue:
tmpl files not rendered by `render-cloudcfg`
Our Makefile has some rules that depend on `render-cloudcfg` being able to render generic `.tmpl` files. In this particular case, it is trying to render `systemd/cloud-init-generator.tmpl`:
```
render-template:
	$(PYTHON) ./tools/render-cloudcfg --variant=$(VARIANT) $(FILE) $(subst .tmpl,,$(FILE))
...
benchmark-generator: FILE=$(GENERATOR_F).tmpl
benchmark-generator: export ITER=$(NUM_ITER)
benchmark-generator: render-template
        $(BENCHMARK) $(GENERATOR_F)
```

However, the `render-cloudcfg` script eventually [attempts to load yaml](https://github.com/canonical/cloud-init/blob/main/cloudinit/templater.py/#L168). This works fine for rendered cloud-config, but it breaks for generic jinja templated scripts like `cloud-init-generator.tmpl`. 

The actual rendering code used on package build is [in setup.py](https://github.com/canonical/cloud-init/blob/main/setup.py/#L53). It looks like it may just be ignoring the error since the rendering portion works before hitting the error, and the script is exiting 0. We should create a new `render-template` script that can be used generically and fix the load yaml error.


Discussion:
I assume this is the issue I'm seeing when I'm packaging up git main (but not seeing with 23.3): https://tpaste.us/papW

This appears to be due to parsing of systemd and FreeBSD init.d tmpl files.

What is strange is that I run: "python3 setup.py install --prefix=/usr --root="$pkgdir" --init-system=sysvinit_openrc"
 so shouldn't the "--init-system=sysvinit_openrc" option avoid **any** actions with Systemd and FreeBSD init.d files?

Yes, that is strange. I wouldn't expect you to see systemd failures there...
> Yes, that is strange. I wouldn't expect you to see systemd failures there...

Yeah I've run both 23.3 and git main packaging on the same environment (so same python version etc) and only see this output with git main.

I've diff'ed 23.3 and git main source and can't see anything that explains this.

BTW I'm still seeing that "strange" output with git master when building/packaging for Alpine...

Resolving patch:
diff --git a/cloudinit/templater.py b/cloudinit/templater.py
--- a/cloudinit/templater.py
+++ b/cloudinit/templater.py
@@ -160,12 +160,17 @@ def render_string(content, params):
     return renderer(content, params)
 
 
-def render_cloudcfg(variant, template, output, prefix=None):
+def render_template(variant, template, output, is_yaml, prefix=None):
     with open(template, "r") as fh:
         contents = fh.read()
     tpl_params = {"variant": variant, "prefix": prefix}
     contents = (render_string(contents, tpl_params)).rstrip() + "\n"
-    util.load_yaml(contents)
+    if is_yaml:
+        out = util.load_yaml(contents, default=True)
+        if not out:
+            raise RuntimeError(
+                "Cannot render template file %s - invalid yaml." % template
+            )
     if output == "-":
         sys.stdout.write(contents)
     else:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -50,7 +50,7 @@
 PREFIX = None
 
 
-def render_tmpl(template, mode=None):
+def render_tmpl(template, mode=None, is_yaml=False):
     """render template into a tmpdir under same dir as setup.py
 
     This is rendered to a temporary directory under the top level
@@ -85,13 +85,11 @@ def render_tmpl(template, mode=None):
     subprocess.run(
         [
             sys.executable,
-            "./tools/render-cloudcfg",
-        ]
-        + cmd_prefix
-        + cmd_variant
-        + [
-            template,
-            fpath,
+            "./tools/render-template",
+            *(["--is-yaml"] if is_yaml else []),
+            *cmd_prefix,
+            *cmd_variant,
+            *[template, fpath],
         ],
         check=True,
     )
@@ -123,20 +121,22 @@ def render_tmpl(template, mode=None):
             PREFIX = sys.argv[idx + 1]
 
 INITSYS_FILES = {
-    "sysvinit": [f for f in glob("sysvinit/redhat/*") if is_f(f)],
-    "sysvinit_freebsd": [
+    "sysvinit": lambda: [f for f in glob("sysvinit/redhat/*") if is_f(f)],
+    "sysvinit_freebsd": lambda: [
         render_tmpl(f, mode=0o755)
         for f in glob("sysvinit/freebsd/*")
         if is_f(f)
     ],
-    "sysvinit_netbsd": [
+    "sysvinit_netbsd": lambda: [
         render_tmpl(f, mode=0o755)
         for f in glob("sysvinit/netbsd/*")
         if is_f(f)
     ],
-    "sysvinit_deb": [f for f in glob("sysvinit/debian/*") if is_f(f)],
-    "sysvinit_openrc": [f for f in glob("sysvinit/gentoo/*") if is_f(f)],
-    "systemd": [
+    "sysvinit_deb": lambda: [f for f in glob("sysvinit/debian/*") if is_f(f)],
+    "sysvinit_openrc": lambda: [
+        f for f in glob("sysvinit/gentoo/*") if is_f(f)
+    ],
+    "systemd": lambda: [
         render_tmpl(f)
         for f in (
             glob("systemd/*.tmpl")
@@ -146,7 +146,7 @@ def render_tmpl(template, mode=None):
         )
         if (is_f(f) and not is_generator(f))
     ],
-    "systemd.generators": [
+    "systemd.generators": lambda: [
         render_tmpl(f, mode=0o755)
         for f in glob("systemd/*")
         if is_f(f) and is_generator(f)
@@ -250,11 +250,10 @@ def finalize_options(self):
                 k for k in INITSYS_ROOTS if k.partition(".")[0] == system
             ]
             for k in datakeys:
-                if not INITSYS_FILES[k]:
+                files = INITSYS_FILES[k]()
+                if not files:
                     continue
-                self.distribution.data_files.append(
-                    (INITSYS_ROOTS[k], INITSYS_FILES[k])
-                )
+                self.distribution.data_files.append((INITSYS_ROOTS[k], files))
         # Force that command to reinitialize (with new file list)
         self.distribution.reinitialize_command("install_data", True)
 
@@ -268,7 +267,7 @@ def finalize_options(self):
         INITSYS_ROOTS[k] = "/" + INITSYS_ROOTS[k]
 
 data_files = [
-    (ETC + "/cloud", [render_tmpl("config/cloud.cfg.tmpl")]),
+    (ETC + "/cloud", [render_tmpl("config/cloud.cfg.tmpl", is_yaml=True)]),
     (ETC + "/cloud/clean.d", glob("config/clean.d/*")),
     (ETC + "/cloud/cloud.cfg.d", glob("config/cloud.cfg.d/*")),
     (ETC + "/cloud/templates", glob("templates/*")),


Test patch:
diff --git a/tests/unittests/test_render_cloudcfg.py b/tests/unittests/test_render_template.py
similarity index 89%
rename from tests/unittests/test_render_cloudcfg.py
rename to tests/unittests/test_render_template.py
--- a/tests/unittests/test_render_cloudcfg.py
+++ b/tests/unittests/test_render_template.py
@@ -1,4 +1,4 @@
-"""Tests for tools/render-cloudcfg"""
+"""Tests for tools/render-template"""
 
 import sys
 
@@ -7,7 +7,7 @@
 from cloudinit import subp, templater, util
 from tests.unittests.helpers import cloud_init_project_dir
 
-# TODO(Look to align with tools.render-cloudcfg or cloudinit.distos.OSFAMILIES)
+# TODO(Look to align with tools.render-template or cloudinit.distos.OSFAMILIES)
 DISTRO_VARIANTS = [
     "amazon",
     "arch",
@@ -30,7 +30,7 @@
 
 @pytest.mark.allow_subp_for(sys.executable)
 class TestRenderCloudCfg:
-    cmd = [sys.executable, cloud_init_project_dir("tools/render-cloudcfg")]
+    cmd = [sys.executable, cloud_init_project_dir("tools/render-template")]
     tmpl_path = cloud_init_project_dir("config/cloud.cfg.tmpl")
     init_path = cloud_init_project_dir("sysvinit/freebsd/dsidentify.tmpl")
 
@@ -67,7 +67,9 @@ def test_variant_sets_distro_in_cloud_cfg(self, variant, tmpdir):
         """
         outfile = tmpdir.join("outcfg").strpath
 
-        templater.render_cloudcfg(variant, self.tmpl_path, outfile)
+        templater.render_template(
+            variant, self.tmpl_path, outfile, is_yaml=True
+        )
         with open(outfile) as stream:
             system_cfg = util.load_yaml(stream.read())
         if variant == "unknown":
@@ -80,7 +82,9 @@ def test_variant_sets_default_user_in_cloud_cfg(self, variant, tmpdir):
         call versus calling as subp
         """
         outfile = tmpdir.join("outcfg").strpath
-        templater.render_cloudcfg(variant, self.tmpl_path, outfile)
+        templater.render_template(
+            variant, self.tmpl_path, outfile, is_yaml=True
+        )
         with open(outfile) as stream:
             system_cfg = util.load_yaml(stream.read())
 
@@ -109,7 +113,9 @@ def test_variant_sets_network_renderer_priority_in_cloud_cfg(
         call versus calling as subp
         """
         outfile = tmpdir.join("outcfg").strpath
-        templater.render_cloudcfg(variant, self.tmpl_path, outfile)
+        templater.render_template(
+            variant, self.tmpl_path, outfile, is_yaml=True
+        )
         with open(outfile) as stream:
             system_cfg = util.load_yaml(stream.read())