Repository: osbuild/osbuild

Issue:
Best practices for new runners?
In the [AutoSD](https://sigs.centos.org/automotive/) project, we've added a `centos-release-automotive` RPM package with a customized `os-release` file, so that it's easy to distinguish AutoSD from a regular CentOS stream system.

This is however causing some issues with osbuild which complains about the lack of autosd runner: https://gitlab.com/CentOS/automotive/sample-images/-/issues/12

So what would be the best-practice for this situation? Should we just submit a pull-request creating an autosd running symlinked to the centos one?

Thanks

Discussion:
pypingou: Many thanks!
I wasn't sure if you were ok with this or had a different idea in mind :)

Patch:
diff --git a/osbuild/monitor.py b/osbuild/monitor.py
index f8f1f279..b5b499e7 100644
--- a/osbuild/monitor.py
+++ b/osbuild/monitor.py
@@ -9,6 +9,7 @@ are called on the monitor object at certain events. Consult the
 """
 
 import abc
+import datetime
 import json
 import os
 import sys
@@ -108,6 +109,10 @@ class LogMonitor(BaseMonitor):
         else:
             self.out.write("<host>")
         self.out.write(f"\n  runner: {pipeline.runner.name} ({pipeline.runner.exec})")
+        source_epoch = pipeline.source_epoch
+        if source_epoch is not None:
+            timepoint = datetime.datetime.fromtimestamp(source_epoch).strftime('%c')
+            self.out.write(f"\n  source-epoch: {timepoint} [{source_epoch}]")
         self.out.write("\n")
 
     def stage(self, stage):
diff --git a/osbuild/objectstore.py b/osbuild/objectstore.py
index 606e79e3..9f1dcef4 100644
--- a/osbuild/objectstore.py
+++ b/osbuild/objectstore.py
@@ -4,11 +4,13 @@ import json
 import os
 import subprocess
 import tempfile
+import time
 from typing import Any, Optional, Set, Union
 
 from osbuild.util import jsoncomm
 from osbuild.util.fscache import FsCache, FsCacheInfo
 from osbuild.util.mnt import mount, umount
+from osbuild.util.path import clamp_mtime
 from osbuild.util.types import PathLike
 
 from . import api
@@ -112,6 +114,7 @@ class Object:
         self._path = None
         self._meta: Optional[Object.Metadata] = None
         self._stack: Optional[contextlib.ExitStack] = None
+        self.source_epoch = None  # see finalize()
 
     def _open_for_reading(self):
         name = self._stack.enter_context(
@@ -140,6 +143,11 @@ class Object:
         wrapped = PathAdapter(self, "_path")
         self._meta = self.Metadata(wrapped, folder="meta")
 
+        if self.mode == Object.Mode.WRITE:
+            self.meta.set("info", {
+                "created": int(time.time()),
+            })
+
         return self
 
     def __exit__(self, exc_type, exc_value, exc_tb):
@@ -177,10 +185,38 @@ class Object:
         assert self._meta
         return self._meta
 
+    @property
+    def created(self) -> int:
+        """When was the object created
+
+        It is stored as `created` in the `info` metadata entry,
+        and thus will also get overwritten if the metadata gets
+        overwritten via `init()`.
+        NB: only valid to access when the object is active.
+        """
+        info = self.meta.get("info")
+        assert info, "info metadata missing"
+        return info["created"]
+
+    def clamp_mtime(self):
+        """Clamp mtime of files and dirs to source_epoch
+
+        If a source epoch is specified we clamp all files that
+        are newer then our own creation timestap to the given
+        source epoch. As a result all files created during the
+        build should receive the source epoch modification time
+        """
+        if self.source_epoch is None:
+            return
+
+        clamp_mtime(self.tree, self.created, self.source_epoch)
+
     def finalize(self):
         if self.mode != Object.Mode.WRITE:
             return
 
+        self.clamp_mtime()
+
         # put the object into the READER state
         self._mode = Object.Mode.READ
 
@@ -387,6 +423,10 @@ class ObjectStore(contextlib.AbstractContextManager):
 
         with self.cache.store(object_id) as name:
             path = os.path.join(self.cache, name)
+            # we clamp the mtime of `obj` itself so that it
+            # resuming a snapshop and building with a snapshot
+            # goes through the same code path
+            obj.clamp_mtime()
             obj.clone(path)
 
     def cleanup(self):
diff --git a/osbuild/pipeline.py b/osbuild/pipeline.py
index 5495244d..59b2f020 100644
--- a/osbuild/pipeline.py
+++ b/osbuild/pipeline.py
@@ -319,6 +319,7 @@ class Pipeline:
         # an intermediate checkpoint exists: Find the last stage that
         # already exists in the store and use that as the base.
         tree = object_store.new(self.id)
+        tree.source_epoch = self.source_epoch
 
         todo = collections.deque()
         for stage in reversed(self.stages):
diff --git a/osbuild/util/path.py b/osbuild/util/path.py
index 0988b5a0..9f5ef246 100644
--- a/osbuild/util/path.py
+++ b/osbuild/util/path.py
@@ -1,5 +1,36 @@
 """Path handling utility functions"""
+import errno
 import os
+import os.path
+from typing import Optional
+
+from .ctx import suppress_oserror
+
+
+def clamp_mtime(path: str, start: int, to: int):
+    """Clamp all modification times of 'path'
+
+    Set the mtime of 'path' to 'to' if it is greater or equal to 'start'.
+    If 'to' is None, the mtime is set to the current time.
+    """
+
+    times = (to, to)
+
+    def fix_utime(path, dfd: Optional[int] = None):
+        sb = os.stat(path, dir_fd=dfd, follow_symlinks=False)
+        if sb.st_mtime < start:
+            return
+
+        # We might get a permission error when the immutable flag is set;
+        # since there is nothing much we can do, we just ignore it
+        with suppress_oserror(errno.EPERM):
+            os.utime(path, times, dir_fd=dfd, follow_symlinks=False)
+
+    fix_utime(path)
+
+    for _, dirs, files, dfd in os.fwalk(path):
+        for f in dirs + files:
+            fix_utime(f, dfd)
 
 
 def in_tree(path: str, tree: str, must_exist: bool = False) -> bool:
diff --git a/runners/org.osbuild.AutoSD9 b/runners/org.osbuild.AutoSD9
new file mode 120000
index 00000000..ffc70521
--- /dev/null
+++ b/runners/org.osbuild.AutoSD9
@@ -0,0 +1,1 @@
+org.osbuild.centos9
\ No newline at end of file


Test patch:
diff --git a/test/data/manifests/fedora-build-v2.json b/test/data/manifests/fedora-build-v2.json
index a89a9ff7..4e77b353 100644
--- a/test/data/manifests/fedora-build-v2.json
+++ b/test/data/manifests/fedora-build-v2.json
@@ -3,6 +3,7 @@
   "pipelines": [
     {
       "name": "build",
+      "source-epoch": 1659397331,
       "stages": [
         {
           "type": "org.osbuild.rpm",
diff --git a/test/data/manifests/fedora-build-v2.mpp.json b/test/data/manifests/fedora-build-v2.mpp.json
index 6ea15aeb..d61c818a 100644
--- a/test/data/manifests/fedora-build-v2.mpp.json
+++ b/test/data/manifests/fedora-build-v2.mpp.json
@@ -10,6 +10,7 @@
   "pipelines": [
     {
       "name": "build",
+      "source-epoch": 1659397331,
       "stages": [
         {
           "type": "org.osbuild.rpm",
diff --git a/test/data/manifests/fedora-ostree-commit.json b/test/data/manifests/fedora-ostree-commit.json
index 230a53d4..ccf757a7 100644
--- a/test/data/manifests/fedora-ostree-commit.json
+++ b/test/data/manifests/fedora-ostree-commit.json
@@ -4,6 +4,7 @@
     {
       "runner": "org.osbuild.fedora36",
       "name": "build",
+      "source-epoch": 1659397331,
       "stages": [
         {
           "type": "org.osbuild.rpm",
@@ -379,6 +380,7 @@
     {
       "name": "ostree-tree",
       "build": "name:build",
+      "source-epoch": 1659397331,
       "stages": [
         {
           "type": "org.osbuild.rpm.macros",
diff --git a/test/data/manifests/fedora-ostree-commit.mpp.json b/test/data/manifests/fedora-ostree-commit.mpp.json
index bb388f8a..24706293 100644
--- a/test/data/manifests/fedora-ostree-commit.mpp.json
+++ b/test/data/manifests/fedora-ostree-commit.mpp.json
@@ -20,6 +20,7 @@
     {
       "name": "ostree-tree",
       "build": "name:build",
+      "source-epoch": 1659397331,
       "stages": [
         {
           "type": "org.osbuild.rpm.macros",
diff --git a/test/data/manifests/fedora-ostree-image.json b/test/data/manifests/fedora-ostree-image.json
index e0978302..d80ebea8 100644
--- a/test/data/manifests/fedora-ostree-image.json
+++ b/test/data/manifests/fedora-ostree-image.json
@@ -4,6 +4,7 @@
     {
       "runner": "org.osbuild.fedora36",
       "name": "build",
+      "source-epoch": 1659397331,
       "stages": [
         {
           "type": "org.osbuild.rpm",
@@ -379,6 +380,7 @@
     {
       "name": "ostree-tree",
       "build": "name:build",
+      "source-epoch": 1659397331,
       "stages": [
         {
           "type": "org.osbuild.rpm.macros",
@@ -960,6 +962,7 @@
     {
       "name": "image-tree",
       "build": "name:build",
+      "source-epoch": 1659397331,
       "stages": [
         {
           "type": "org.osbuild.ostree.init-fs"
diff --git a/test/data/manifests/fedora-ostree-image.mpp.json b/test/data/manifests/fedora-ostree-image.mpp.json
index 541a9a27..b14b1b23 100644
--- a/test/data/manifests/fedora-ostree-image.mpp.json
+++ b/test/data/manifests/fedora-ostree-image.mpp.json
@@ -63,6 +63,7 @@
     {
       "name": "image-tree",
       "build": "name:build",
+      "source-epoch": 1659397331,
       "stages": [
         {
           "type": "org.osbuild.ostree.init-fs"
diff --git a/test/data/manifests/fedora-ostree-native-container.json b/test/data/manifests/fedora-ostree-native-container.json
index a262dc7b..406b8bc9 100644
--- a/test/data/manifests/fedora-ostree-native-container.json
+++ b/test/data/manifests/fedora-ostree-native-container.json
@@ -4,6 +4,7 @@
     {
       "runner": "org.osbuild.fedora36",
       "name": "build",
+      "source-epoch": 1659397331,
       "stages": [
         {
           "type": "org.osbuild.rpm",
@@ -379,6 +380,7 @@
     {
       "name": "ostree-tree",
       "build": "name:build",
+      "source-epoch": 1659397331,
       "stages": [
         {
           "type": "org.osbuild.rpm.macros",
diff --git a/test/data/manifests/fedora-uki.json b/test/data/manifests/fedora-uki.json
index 4b661f49..2628fa1d 100644
--- a/test/data/manifests/fedora-uki.json
+++ b/test/data/manifests/fedora-uki.json
@@ -4,6 +4,7 @@
     {
       "runner": "org.osbuild.fedora36",
       "name": "build",
+      "source-epoch": 1659397331,
       "stages": [
         {
           "type": "org.osbuild.rpm",
diff --git a/test/mod/test_objectstore.py b/test/mod/test_objectstore.py
index 0cbedd2f..01e13d52 100644
--- a/test/mod/test_objectstore.py
+++ b/test/mod/test_objectstore.py
@@ -201,7 +201,60 @@ def test_host_tree(tmpdir):
         _ = host.tree
 
 
+def test_source_epoch(object_store):
+    tree = object_store.new("a")
+    tree.source_epoch = 946688461
+
+    t = Path(tree)
+
+    a = Path(tree, "A")
+    a.touch()
+
+    d = Path(tree, "d")
+    d.mkdir()
+
+    l = Path(tree, "l")
+    l.symlink_to(d, target_is_directory=True)
+
+    for i in (a, d, l, t):
+        si = i.stat(follow_symlinks=False)
+        before = si.st_mtime
+        assert before != tree.source_epoch
+
+    # FINALIZING
+    tree.finalize()
+
+    for i in (a, d, l, t):
+        si = i.stat(follow_symlinks=False)
+        after = si.st_mtime
+
+        assert after != before, f"mtime not changed for {i}"
+        assert after == tree.source_epoch
+
+    baum = object_store.new("b")
+    baum.init(tree)
+
+    assert baum.created == tree.created
+
+    b = Path(tree, "B")
+    b.touch()
+
+    si = b.stat(follow_symlinks=False)
+    before = si.st_mtime
+
+    assert before != tree.source_epoch
+
+    # FINALIZING
+    baum.finalize()
+    si = a.stat(follow_symlinks=False)
+    after = si.st_mtime
+
+    assert after != before
+    assert after == tree.source_epoch
+
 # pylint: disable=too-many-statements
+
+
 @pytest.mark.skipif(not test.TestBase.can_bind_mount(), reason="Need root for bind mount")
 def test_store_server(tmpdir):
     with contextlib.ExitStack() as stack:
diff --git a/test/mod/test_util_path.py b/test/mod/test_util_path.py
index c9efcb6a..1efff247 100644
--- a/test/mod/test_util_path.py
+++ b/test/mod/test_util_path.py
@@ -2,10 +2,54 @@
 # Tests for the 'osbuild.util.path' module
 #
 import os
+import time
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+import pytest
 
 from osbuild.util import path
 
 
+@pytest.fixture(name="tempdir")
+def tempdir_fixture():
+    with TemporaryDirectory(prefix="path-") as tmp:
+        yield tmp
+
+
+def test_clamp_mtime(tempdir):
+    start = int(time.time())
+
+    tree = Path(tempdir, "tree")
+    tree.mkdir()
+    os.makedirs(os.path.join(tree, "a", "bunch", "of", "directories"))
+
+    file = Path(tree, "file")
+    file.touch()
+
+    folder = Path(tree, "folder")
+    folder.mkdir()
+
+    link = Path(tree, "link")
+    link.symlink_to(folder, target_is_directory=True)
+
+    timepoint = 1644758228
+    path.clamp_mtime(tree, start, timepoint)
+
+    def ensure_mtime(target, dfd):
+        stat = os.stat(target, dir_fd=dfd, follow_symlinks=False)
+        assert stat.st_mtime <= timepoint
+
+    for _, dirs, files, dfd in os.fwalk(tree):
+        ensure_mtime(".", dfd)
+
+        for d in dirs:
+            ensure_mtime(d, dfd)
+
+        for f in files:
+            ensure_mtime(f, dfd)
+
+
 def test_in_tree():
     cases = {
         ("/tmp/file", "/tmp", False): True,  # Simple, non-existent