Repository: osbuild/osbuild
Pull request #640 (2021-04-29 11:31:38)

Introduce a new `org.osbuild.inline` source
This new `org.osbuild.inline` source can be used to transport binary file data within the source itself. Individual items will be decoded to the store as `org.osbuild.files` and can thus be used in all inputs that can handle files.

Additionally, this PR moves the `pylint` configuration from `.pylintrc` to `setup.cfg`; the latter can be used by more tools, e.g. `autopep8`.

@bcl had done something very similar in #512 before

gicmo: > Also a question: What's the intended use case for this?

The main use-case I envision is in testing, i.e. to test the `tree` and `files` input but not having to fetch arbitrary files from somewhere via the `curl` stage, but having it self-contained in the test manifest.


Patch:
diff --git a/osbuild/util/checksum.py b/osbuild/util/checksum.py
new file mode 100644
index 00000000..6350a809
--- /dev/null
+++ b/osbuild/util/checksum.py
@@ -0,0 +1,50 @@
+"""Checksum Utilities
+
+Small convenience functions to work with checksums.
+"""
+import hashlib
+import os
+
+from .types import PathLike
+
+
+# How many bytes to read in one go. Taken from coreutils/gnulib
+BLOCKSIZE = 32768
+
+
+def hexdigest_file(path: PathLike, algorithm: str) -> str:
+    """Return the hexdigest of the file at `path` using `algorithm`
+
+    Will stream the contents of file to the hash `algorithm` and
+    return the hexdigest. If the specified `algorithm` is not
+    supported a `ValueError` will be raised.
+    """
+    hasher = hashlib.new(algorithm)
+
+    with open(path, "rb") as f:
+
+        os.posix_fadvise(f.fileno(), 0, 0, os.POSIX_FADV_SEQUENTIAL)
+
+        while True:
+            data = f.read(BLOCKSIZE)
+            if not data:
+                break
+
+            hasher.update(data)
+
+    return hasher.hexdigest()
+
+
+def verify_file(path: PathLike, checksum: str) -> bool:
+    """Hash the file and return if the specified `checksum` matches
+
+    Uses `hexdigest_file` to hash the contents of the file at
+    `path` and return if the hexdigest matches the one specified
+    in `checksum`, where `checksum` consist of the algorithm used
+    and the digest joined via `:`, e.g. `sha256:abcd...`.
+    """
+    algorithm, want = checksum.split(":", 1)
+
+    have = hexdigest_file(path, algorithm)
+
+    return have == want
diff --git a/.pylintrc b/setup.cfg
similarity index 71%
rename from .pylintrc
rename to setup.cfg
index 1a7c8a30..c75bf968 100644
--- a/.pylintrc
+++ b/setup.cfg
@@ -1,9 +1,11 @@
-[MASTER]
+[pylint.MASTER]
 disable=missing-docstring,too-few-public-methods,invalid-name,duplicate-code,superfluous-parens,too-many-locals,attribute-defined-outside-init,too-many-arguments
-max-line-length=120
 
-[TYPECHECK]
+[pylint.TYPECHECK]
 ignored-classes=osbuild.loop.LoopInfo
 
-[DESIGN]
+[pylint.DESIGN]
 max-attributes=10
+
+[pycodestyle]
+max-line-length = 120
diff --git a/sources/org.osbuild.curl b/sources/org.osbuild.curl
index 7be50e8e..28928448 100755
--- a/sources/org.osbuild.curl
+++ b/sources/org.osbuild.curl
@@ -26,6 +26,8 @@ import time
 
 from typing import Dict
 
+from osbuild.util.checksum import verify_file
+
 
 SCHEMA = """
 "additionalProperties": false,
@@ -35,7 +37,7 @@ SCHEMA = """
     "type": "object",
     "additionalProperties": false,
     "patternProperties": {
-      "(md5|sha1|sha256|sha384|sha512):[0-9a-f]{5,64}": {
+      "(md5|sha1|sha256|sha384|sha512):[0-9a-f]{32,128}": {
         "oneOf": [
           {
             "type": "string",
@@ -84,22 +86,6 @@ SCHEMA = """
 """
 
 
-def verify_checksum(filename, checksum):
-    algorithm, checksum = checksum.split(":", 1)
-    if algorithm not in ("md5", "sha1", "sha256", "sha384", "sha512"):
-        raise RuntimeError(f"unsupported checksum algorithm: {algorithm}")
-
-    ret = subprocess.run(
-        [f"{algorithm}sum", "-c"],
-        input=f"{checksum} {filename}",
-        stdout=subprocess.DEVNULL,
-        encoding="utf-8",
-        check=False
-    )
-
-    return ret.returncode == 0
-
-
 def fetch(url, checksum, directory):
     # Invariant: all files in @directory must be named after their (verified) checksum.
     if os.path.isfile(f"{directory}/{checksum}"):
@@ -144,7 +130,7 @@ def fetch(url, checksum, directory):
         else:
             raise RuntimeError(f"curl: error downloading {url}: error code {return_code}")
 
-        if not verify_checksum(f"{tmpdir}/{checksum}", checksum):
+        if not verify_file(f"{tmpdir}/{checksum}", checksum):
             raise RuntimeError(f"checksum mismatch: {checksum} {url}")
 
         # The checksum has been verified, move the file into place. in case we race
diff --git a/sources/org.osbuild.inline b/sources/org.osbuild.inline
new file mode 100755
index 00000000..b942ab5c
--- /dev/null
+++ b/sources/org.osbuild.inline
@@ -0,0 +1,102 @@
+#!/usr/bin/python3
+"""Source for binary data encoded inline in the manifest
+
+This source can be used to transport data in the source
+section of the manifest. Each resource is ascii-encoded
+in the `data` property, where the encoding is specified
+in the `encoding` property. The resources is content
+addressed via the hash value of the raw data before the
+ascii encoding. This hash value is verified after the
+resource is decoded and written to the store.
+"""
+
+
+import base64
+import contextlib
+import json
+import os
+import sys
+import tempfile
+
+from typing import Dict
+
+from osbuild.util.checksum import verify_file
+
+
+SCHEMA = """
+"additionalProperties": false,
+"required": ["items"],
+"properties": {
+  "item": {
+    "description": "Inline data indexed by their checksum",
+    "type": "object",
+    "additionalProperties": false,
+    "patternProperties": {
+      "(md5|sha1|sha256|sha384|sha512):[0-9a-f]{32,128}": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": ["method", "data"],
+        "properties": {
+          "encoding": {
+            "description": "The specific encoding of `data`",
+            "enum": ["base64"]
+          },
+          "data": {
+            "description": "The ascii encoded raw data",
+            "type": "string"
+          }
+        }
+      }
+    }
+  }
+}
+"""
+
+
+def process(items: Dict, cache: str, tmpdir):
+    for checksum, item in items.items():
+        target = os.path.join(cache, checksum)
+        floating = os.path.join(tmpdir, checksum)
+
+        if os.path.isfile(target):
+            return
+
+        data = base64.b64decode(item["data"])
+
+        # Write the bits to disk and then verify the checksum
+        # This ensures that 1) the data is ok and that 2) we
+        # wrote them correctly as well
+        with open(floating, "wb") as f:
+            f.write(data)
+
+        if not verify_file(floating, checksum):
+            json.dump({"error": f"checksum mismatch: {checksum}"}, sys.stdout)
+            sys.exit(1)
+
+        with contextlib.suppress(FileExistsError):
+            os.rename(floating, target)
+
+
+def main(items: Dict, base: str):
+    cache = os.path.join(base, "org.osbuild.files")
+
+    if not items:
+        json.dump({}, sys.stdout)
+        return 0
+
+    try:
+        os.makedirs(cache, exist_ok=True)
+        with tempfile.TemporaryDirectory(prefix=".unverified-", dir=base) as tmpdir:
+            process(items, cache, tmpdir)
+    except Exception as e:  # pylint: disable=broad-except
+        json.dump({"error": str(e)}, sys.stdout)
+        return 0
+
+    json.dump({}, sys.stdout)
+    return 0
+
+
+if __name__ == '__main__':
+    source_args = json.load(sys.stdin)
+    r = main(source_args["items"], source_args["cache"])
+    sys.exit(r)
diff --git a/tools/inline-source.py b/tools/inline-source.py
new file mode 100755
index 00000000..5a28b4e9
--- /dev/null
+++ b/tools/inline-source.py
@@ -0,0 +1,42 @@
+#!/usr/bin/python3
+"""
+Encode binary file data within the manifest by using
+the org.osbuild.inline source.
+"""
+
+import argparse
+import binascii
+import hashlib
+import json
+import sys
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("FILE", help="The file to encode")
+    args = parser.parse_args()
+
+    with open(args.FILE, "rb") as f:
+        raw = f.read()
+
+    m = hashlib.sha256()
+    m.update(raw)
+    checksum = "sha256:" + m.hexdigest()
+    data = binascii.b2a_base64(raw, newline=False).decode("ascii")
+
+    source = {
+        "org.osbuild.inline": {
+            "items": {
+                checksum: {
+                    "encoding": "base64",
+                    "data": data
+                }
+            }
+        }
+    }
+
+    json.dump(source, sys.stdout, indent=2)
+
+
+if __name__ == "__main__":
+    main()


Test patch:
diff --git a/test/data/sources/org.osbuild.inline/cases/empty.json b/test/data/sources/org.osbuild.inline/cases/empty.json
new file mode 100644
index 00000000..9a315340
--- /dev/null
+++ b/test/data/sources/org.osbuild.inline/cases/empty.json
@@ -0,0 +1,6 @@
+{
+  "expects": "success",
+  "org.osbuild.inline": {
+    "items": {}
+  }
+}
diff --git a/test/data/sources/org.osbuild.inline/cases/invalid_checksum.json b/test/data/sources/org.osbuild.inline/cases/invalid_checksum.json
new file mode 100644
index 00000000..3031d8c3
--- /dev/null
+++ b/test/data/sources/org.osbuild.inline/cases/invalid_checksum.json
@@ -0,0 +1,11 @@
+{
+  "expects": "error",
+  "org.osbuild.inline": {
+    "items": {
+      "sha256:d2a84f4b8b650937ec8f73cd8be2c74add5a911ba64df27458ed8229da804a27": {
+        "encoding": "base64",
+        "data": "SGVsbG8gV29ybGQK"
+      }
+    }
+  }
+}
diff --git a/test/data/sources/org.osbuild.inline/cases/success.json b/test/data/sources/org.osbuild.inline/cases/success.json
new file mode 100644
index 00000000..430b886d
--- /dev/null
+++ b/test/data/sources/org.osbuild.inline/cases/success.json
@@ -0,0 +1,11 @@
+{
+  "expects": "success",
+  "org.osbuild.inline": {
+    "items": {
+      "sha256:9a605b69e4f60eceaa3d2c6d8c4c541f48c3ec1b238d3dfd517e7182bef46872": {
+        "encoding": "base64",
+        "data": "IyEvdXNyL2Jpbi9weXRob24zCgppbXBvcnQgYXJncGFyc2UKaW1wb3J0IGJpbmFzY2lpCmltcG9ydCBoYXNobGliCmltcG9ydCBqc29uCmltcG9ydCBvcwppbXBvcnQgc3lzCmltcG9ydCBzdWJwcm9jZXNzCmltcG9ydCB0ZW1wZmlsZQoKZnJvbSB0eXBpbmcgaW1wb3J0IERpY3QKCgpkZWYgbWFpbigpOgogICAgcGFyc2VyID0gYXJncGFyc2UuQXJndW1lbnRQYXJzZXIoKQogICAgcGFyc2VyLmFkZF9hcmd1bWVudCgiRklMRSIsIGhlbHA9IlRoZSB0ZW1wbGF0ZSB0byBwcm9jZXNzIikKICAgIGFyZ3MgPSBwYXJzZXIucGFyc2VfYXJncygpCgogICAgd2l0aCBvcGVuKGFyZ3MuRklMRSwgInJiIikgYXMgZjoKICAgICAgICByYXcgPSBmLnJlYWQoKQoKICAgIG0gPSBoYXNobGliLnNoYTI1NigpCiAgICBtLnVwZGF0ZShyYXcpCiAgICBjaGVja3N1bSA9ICJzaGEyNTY6IiArIG0uaGV4ZGlnZXN0KCkKICAgIGRhdGEgPSBiaW5hc2NpaS5iMmFfYmFzZTY0KHJhdywgbmV3bGluZT1GYWxzZSkuZGVjb2RlKCJhc2NpaSIpCgogICAgc291cmNlID0gewogICAgICAgICJvcmcub3NidWlsZC5pbmxpbmUiOiB7CiAgICAgICAgICAgICJpdGVtcyI6IHsKICAgICAgICAgICAgICAgIGNoZWNrc3VtOiB7CiAgICAgICAgICAgICAgICAgICAgImVuY29kaW5nIjogImJhc2U2NCIsCiAgICAgICAgICAgICAgICAgICAgImRhdGEiOiBkYXRhCiAgICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICB9CiAgICB9CgogICAganNvbi5kdW1wKHNvdXJjZSwgc3lzLnN0ZG91dCwgaW5kZW50PTIpCgoKaWYgX19uYW1lX18gPT0gIl9fbWFpbl9fIjoKICAgIG1haW4oKQo="
+      }
+    }
+  }
+}
diff --git a/test/mod/test_util_checksum.py b/test/mod/test_util_checksum.py
new file mode 100644
index 00000000..557a7ea2
--- /dev/null
+++ b/test/mod/test_util_checksum.py
@@ -0,0 +1,34 @@
+#
+# Test for the util.checksum module
+#
+from tempfile import NamedTemporaryFile
+import pytest
+
+from osbuild.util import checksum
+
+
+# pylint: disable=line-too-long
+TEST_STRING = "I have of late, but wherefore I know not, lost all my bytes\n"
+TEST_RESULT = {
+    "md5": "537f2c2e965f5ce9c524704b58cb7660",
+    "sha1": "897b9791e74e937fccc885e093897a071c2a54fb",
+    "sha256": "16bf29978fc76774eef051739e1e9e9983bcb29285250ffddfc1955f8a3f95fb",
+    "sha384": "7f7c72cabd9af1872e22713efe02deb6234475affcc6b8c9816ce683fee2f63285b466880171f462d49fbefb0dd35ee6",
+    "sha512": "f0874cb102032ca78070c07fd4a894c3eb752b903ab758a54aa8937b2a77830bac40f1faf8ae0e206ff56c4446699a2ae4941f4a6d6556cab4a4ac072216bacc"
+}
+
+
+@pytest.fixture(name="tempfile")
+def tempfile_fixture():
+    with NamedTemporaryFile(prefix="verify-file-", mode="w") as f:
+        yield f
+
+
+@pytest.mark.parametrize("algorithm", TEST_RESULT.keys())
+def test_verify_file(algorithm, tempfile):
+    tempfile.write(TEST_STRING)
+    tempfile.flush()
+
+    digest = TEST_RESULT[algorithm]
+    full_digest = f"{algorithm}:{digest}"
+    assert checksum.verify_file(tempfile.name, full_digest), "checksums mismatch"