Repository: osbuild/osbuild
Pull request #57 (2019-08-02 00:07:19)

Pipeline: drop the "base" key
A pipeline must now be fully self-contained, without references to the id of others. The tool will automaticaly find existing trees that can be used to skip the initial part of a a pipeline, in case it exists already. The output of the final stage of a pipeline is (as before) saved to the object store.

The only difference from before is that instead of specifying a base id, you'd have to explicitly prepend the desired pipeline.

Tests still need updating, and there are still bugs.

packit-as-a-service[bot]: Congratulations! The build [has finished](https://copr-be.cloud.fedoraproject.org/results/packit/osbuild-osbuild-57) successfully. :champagne:

You can install the built RPMs by following these steps:

* `sudo yum install -y dnf-plugins-core` on RHEL 8
* `sudo dnf install -y dnf-plugins-core` on Fedora
* `dnf copr enable packit/osbuild-osbuild-57`
* And now you can install the packages.

Please note that the RPMs should be used only in a testing environment.
msehnout: Before we merge this, I'd expect the object store to cache results from each stage. More specifically if I run the same pipeline twice, it should not perform the same stages again, is that correct?
teg: > Before we merge this, I'd expect the object store to cache results from each stage. More specifically if I run the same pipeline twice, it should not perform the same stages again, is that correct?

Correct. As before, the output of the final stage is saved to the object store, and a tree is not regenerated if it exists in the object store. I will update the description of the PR to clarify.
packit-as-a-service[bot]: Congratulations! The build [has finished](https://copr-be.cloud.fedoraproject.org/results/packit/osbuild-osbuild-57) successfully. :champagne:

You can install the built RPMs by following these steps:

* `sudo yum install -y dnf-plugins-core` on RHEL 8
* `sudo dnf install -y dnf-plugins-core` on Fedora
* `dnf copr enable packit/osbuild-osbuild-57`
* And now you can install the packages.

Please note that the RPMs should be used only in a testing environment.
packit-as-a-service[bot]: Congratulations! The build [has finished](https://copr-be.cloud.fedoraproject.org/results/packit/osbuild-osbuild-57) successfully. :champagne:

You can install the built RPMs by following these steps:

* `sudo yum install -y dnf-plugins-core` on RHEL 8
* `sudo dnf install -y dnf-plugins-core` on Fedora
* `dnf copr enable packit/osbuild-osbuild-57`
* And now you can install the packages.

Please note that the RPMs should be used only in a testing environment.
packit-as-a-service[bot]: Congratulations! The build [has finished](https://copr-be.cloud.fedoraproject.org/results/packit/osbuild-osbuild-57) successfully. :champagne:

You can install the built RPMs by following these steps:

* `sudo yum install -y dnf-plugins-core` on RHEL 8
* `sudo dnf install -y dnf-plugins-core` on Fedora
* `dnf copr enable packit/osbuild-osbuild-57`
* And now you can install the packages.

Please note that the RPMs should be used only in a testing environment.
packit-as-a-service[bot]: Congratulations! The build [has finished](https://copr-be.cloud.fedoraproject.org/results/packit/osbuild-osbuild-57) successfully. :champagne:

You can install the built RPMs by following these steps:

* `sudo yum install -y dnf-plugins-core` on RHEL 8
* `sudo dnf install -y dnf-plugins-core` on Fedora
* `dnf copr enable packit/osbuild-osbuild-57`
* And now you can install the packages.

Please note that the RPMs should be used only in a testing environment.

Patch:
diff --git a/osbuild/__init__.py b/osbuild/__init__.py
index 3a31d842..a9c30d2c 100644
--- a/osbuild/__init__.py
+++ b/osbuild/__init__.py
@@ -276,7 +276,7 @@ def print_header(title, options):
 
 
 class Stage:
-    def __init__(self, name, base, build, options):
+    def __init__(self, name, build, base, options):
         m = hashlib.sha256()
         m.update(json.dumps(name, sort_keys=True).encode())
         m.update(json.dumps(build, sort_keys=True).encode())
@@ -374,19 +374,13 @@ class Assembler:
 
 
 class Pipeline:
-    def __init__(self, base=None):
-        self.base = base
-        self.build = None
+    def __init__(self, build=None):
+        self.build = build
         self.stages = []
         self.assembler = None
 
     def get_id(self):
-        return self.stages[-1].id if self.stages else self.base
-
-    def set_build(self, pipeline):
-        if self.stages:
-            raise ValueError("Must set build before stages.")
-        self.build = pipeline
+        return self.stages[-1].id if self.stages else None
 
     def add_stage(self, name, options=None):
         build = self.build.get_id() if self.build else None
@@ -398,8 +392,6 @@ class Pipeline:
 
     def description(self):
         description = {}
-        if self.base:
-            description["base"] = self.base
         if self.build:
             description["build"] = self.build.description()
         if self.stages:
@@ -423,8 +415,6 @@ class Pipeline:
 
     def run(self, output_dir, store, interactive=False, check=True, libdir=None):
         os.makedirs("/run/osbuild", exist_ok=True)
-        if self.base and not store:
-            raise ValueError("'store' argument must be given when pipeline has a 'base'")
         object_store = ObjectStore(store)
         results = {
             "stages": []
@@ -439,13 +429,22 @@ class Pipeline:
         with self.get_buildtree(object_store) as build_tree:
             if self.stages:
                 if not object_store.has_tree(self.get_id()):
+                    # Find the last stage that already exists in the object store, and use
+                    # that as the base.
+                    base = None
+                    base_idx = -1
+                    for i in range(len(self.stages) - 1, 0, -1):
+                        if object_store.has_tree(self.stages[i].id):
+                            base = self.stages[i].id
+                            base_idx = i
+                            break
                     # The tree does not exist. Create it and save it to the object store. If
                     # two run() calls race each-other, two trees may be generated, and it
                     # is nondeterministic which of them will end up referenced by the tree_id
                     # in the content store. However, we guarantee that all tree_id's and all
                     # generated trees remain valid.
-                    with object_store.new_tree(self.get_id(), base_id=self.base) as tree:
-                        for stage in self.stages:
+                    with object_store.new_tree(self.get_id(), base_id=base) as tree:
+                        for stage in self.stages[base_idx + 1:]:
                             r = stage.run(tree,
                                           build_tree,
                                           interactive=interactive,
@@ -474,11 +473,12 @@ class Pipeline:
 
 
 def load(description):
-    pipeline = Pipeline(description.get("base"))
-
-    b = description.get("build")
-    if b:
-        pipeline.set_build(load(b))
+    build_description = description.get("build")
+    if build_description:
+        build = load(build_description)
+    else:
+        build = None
+    pipeline = Pipeline(build)
 
     for s in description.get("stages", []):
         pipeline.add_stage(s["name"], s.get("options", {}))
diff --git a/samples/base-qcow2.json b/samples/base-qcow2.json
index 20d5f476..9c97d7b1 100644
--- a/samples/base-qcow2.json
+++ b/samples/base-qcow2.json
@@ -1,6 +1,43 @@
 {
   "name": "base-qcow2",
-  "base": "b317426a4dcbef722cf6cccb117deedc0e636dcc24e5d3283e7e6fc2076f1e62",
+  "stages": [
+    {
+      "name": "org.osbuild.dnf",
+      "options": {
+        "releasever": "30",
+        "repos": {
+          "fedora": {
+            "name": "Fedora",
+            "metalink": "https://mirrors.fedoraproject.org/metalink?repo=fedora-$releasever&arch=$basearch",
+            "gpgkey": "file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch"
+          }
+        },
+        "packages": [
+          "@Core",
+          "selinux-policy-targeted",
+          "grub2-pc"
+        ]
+      }
+    },
+    {
+      "name": "org.osbuild.locale",
+      "options": {
+        "language": "en_US"
+      }
+    },
+    {
+      "name": "org.osbuild.selinux",
+      "options": {
+        "file_contexts": "etc/selinux/targeted/contexts/files/file_contexts"
+      }
+    },
+    {
+      "name": "org.osbuild.grub2",
+      "options": {
+        "root_fs_uuid": "76a22bf4-f153-4541-b6c7-0332c0dfaeac"
+      }
+    }
+  ],
   "assembler":
     {
       "name": "org.osbuild.qcow2",
diff --git a/samples/base-with-grub2.json b/samples/base-with-grub2.json
deleted file mode 100644
index 283fbf3e..00000000
--- a/samples/base-with-grub2.json
+++ /dev/null
@@ -1,12 +0,0 @@
-{
-  "name": "base-with-grub2",
-  "base": "cf184c3183ada9edd0b7a728dedc337ac7f4b3df0393b4628a361d8d1fcfa9c0",
-  "stages": [
-    {
-      "name": "org.osbuild.grub2",
-      "options": {
-        "root_fs_uuid": "76a22bf4-f153-4541-b6c7-0332c0dfaeac"
-      }
-    }
-  ]
-}
diff --git a/samples/base-with-locale.json b/samples/base-with-locale.json
deleted file mode 100644
index d5678e49..00000000
--- a/samples/base-with-locale.json
+++ /dev/null
@@ -1,12 +0,0 @@
-{
-  "name": "base-with-locale",
-  "base": "ba22ac2afc2db87df992d8937ee6664f53808465821670b0d636b3ebfac4ce9c",
-  "stages": [
-    {
-      "name": "org.osbuild.locale",
-      "options": {
-        "language": "en_US"
-      }
-    }
-  ]
-}
diff --git a/samples/base-with-selinux.json b/samples/base-with-selinux.json
deleted file mode 100644
index f18c8414..00000000
--- a/samples/base-with-selinux.json
+++ /dev/null
@@ -1,12 +0,0 @@
-{
-  "name": "base-with-selinux",
-  "base": "a26f5fd7cee0b606f35887e1df9c7449af13ab0f62c01b81ecd1bf9e623449ed",
-  "stages": [
-    {
-      "name": "org.osbuild.selinux",
-      "options": {
-        "file_contexts": "etc/selinux/targeted/contexts/files/file_contexts"
-      }
-    }
-  ]
-}


Test patch:
diff --git a/test/test_osbuild.py b/test/test_osbuild.py
index 1142bb28..63d2024c 100644
--- a/test/test_osbuild.py
+++ b/test/test_osbuild.py
@@ -47,8 +47,7 @@ class TestDescriptions(unittest.TestCase):
         build = osbuild.Pipeline()
         build.add_stage("org.osbuild.test", { "one": 1 })
 
-        pipeline = osbuild.Pipeline()
-        pipeline.set_build(build)
+        pipeline = osbuild.Pipeline(build)
         pipeline.add_stage("org.osbuild.test", { "one": 2 })
         pipeline.set_assembler("org.osbuild.test")