Repository: osbuild/osbuild
Pull request #575 (2021-02-08 20:44:47)

Introduce format meta information and detection code
Add a `FormatInfo` that provides meta information about formats, together with `Index.list_formats` and `Index.get_format_info` to list and obtain said `FormatInfo`. Additionally a `Index.detect_format_info` can be used to infer format information from manifest descriptions.


Patch:
diff --git a/osbuild/formats/v1.py b/osbuild/formats/v1.py
index 422f336c..d4e73931 100644
--- a/osbuild/formats/v1.py
+++ b/osbuild/formats/v1.py
@@ -1,10 +1,20 @@
-# Version 1 of the manifest description
-
+""" Version 1 of the manifest description
+
+This is the first version of the osbuild manifest description,
+that has a "main" pipeline that consists of zero or more stages
+to create a tree and optionally one assembler that assembles
+the created tree into an artefact. The pipeline can have any
+number of nested build pipelines. A sources section is used
+to fetch resources.
+"""
 from typing import Dict, Optional, Tuple
 from osbuild.meta import Index, ValidationResult
 from ..pipeline import Manifest, Pipeline, detect_host_runner
 
 
+VERSION = "1"
+
+
 def describe(manifest: Manifest, *, with_id=False) -> Dict:
     """Create the manifest description for the pipeline"""
     def describe_stage(stage):
diff --git a/osbuild/main_cli.py b/osbuild/main_cli.py
index c1b913de..5cc67741 100644
--- a/osbuild/main_cli.py
+++ b/osbuild/main_cli.py
@@ -15,7 +15,6 @@ import osbuild
 import osbuild.meta
 import osbuild.monitor
 from osbuild.objectstore import ObjectStore
-from osbuild.formats import v1 as fmt
 
 
 RESET = "\033[0m"
@@ -74,13 +73,21 @@ def parse_arguments(sys_argv):
     return parser.parse_args(sys_argv[1:])
 
 
-# pylint: disable=too-many-branches
+# pylint: disable=too-many-branches,too-many-return-statements
 def osbuild_cli():
     args = parse_arguments(sys.argv)
     desc = parse_manifest(args.manifest_path)
 
-    # first thing after parsing is validation of the input
     index = osbuild.meta.Index(args.libdir)
+
+    # detect the format from the manifest description
+    info = index.detect_format_info(desc)
+    if not info:
+        print("Unsupported manifest format")
+        return 2
+    fmt = info.module
+
+    # first thing is validation of the manifest
     res = fmt.validate(desc, index)
     if not res:
         if args.json or args.inspect:
diff --git a/osbuild/meta.py b/osbuild/meta.py
index ea1718ef..36d9f3eb 100644
--- a/osbuild/meta.py
+++ b/osbuild/meta.py
@@ -12,18 +12,22 @@ information. For the former a `ModuleInfo` class is returned via
 the individual stages. Schemata, obtained via `Index.get_schema`
 is represented via a `Schema` class that can in turn be used
 to validate the individual components.
-
-The high level `validate` function can be used to check a given
-manifest (parsed form JSON input in dictionary form) against all
-available schemata. The result is a `ValidationResult` which
-contains a single `ValidationError` for each error detected in
-the manifest. See the individual documentation for details.
+Additionally, the `Index` also provides meta information about
+the different formats and version that are supported to read
+manifest descriptions and write output data. Fir this a class
+called `FormatInfo` together with `Index.get_format_inf` and
+`Index.list_formats` is provided. A `FormatInfo` can also be
+inferred for a specific manifest description via a helper
+method called `detect_format_info`
 """
 import ast
 import contextlib
 import copy
+import importlib.util
 import os
+import pkgutil
 import json
+import sys
 from collections import deque
 from typing import Iterable, List, Optional
 
@@ -368,18 +372,76 @@ class ModuleInfo:
         return mapping.get(klass)
 
 
+class FormatInfo:
+    """Meta information about a format
+
+    Class the can be used to get meta information about
+    the the different formats in which osbuild accepts
+    manifest descriptions and writes results.
+    """
+
+    def __init__(self, module):
+        self.module = module
+        self.version = getattr(module, "VERSION")
+        docs = getattr(module, "__doc__")
+        info, desc = docs.split("\n", 1)
+        self.info = info.strip()
+        self.desc = desc.strip()
+
+    @classmethod
+    def load(cls, name):
+        mod = sys.modules.get(name)
+        if not mod:
+            mod = importlib.import_module(name)
+        if not mod:
+            raise ValueError(f"Could not load module {name}")
+        return cls(mod)
+
+
 class Index:
-    """Index of stages and assemblers
+    """Index of modules and formats
 
     Class that can be used to get the meta information about
-    osbuild stages and assemblers as well as JSON schemata.
+    osbuild modules as well as JSON schemata.
     """
 
     def __init__(self, path: str):
         self.path = path
         self._module_info = {}
+        self._format_info = {}
         self._schemata = {}
 
+    @staticmethod
+    def list_formats() -> List[str]:
+        """List all known formats for manifest descriptions"""
+        base = "osbuild.formats"
+        spec = importlib.util.find_spec(base)
+        locations = spec.submodule_search_locations
+        modinfo = [
+            mod for mod in pkgutil.walk_packages(locations)
+            if not mod.ispkg
+        ]
+
+        return [base + "." + m.name for m in modinfo]
+
+    def get_format_info(self, name) -> FormatInfo:
+        """Get the `FormatInfo` for the format called `name`"""
+        info = self._format_info.get(name)
+        if not info:
+            info = FormatInfo.load(name)
+            self._format_info[name] = info
+        return info
+
+    def detect_format_info(self, data) -> Optional[FormatInfo]:
+        """Obtain a `FormatInfo` for the format that can handle `data`"""
+        formats = self.list_formats()
+        version = data.get("version", "1")
+        for fmt in formats:
+            info = self.get_format_info(fmt)
+            if info.version == version:
+                return info
+        return None
+
     def list_modules_for_class(self, klass: str) -> List[str]:
         """List all available modules for the given `klass`"""
         module_path = ModuleInfo.module_class_to_directory(klass)


Test patch:
diff --git a/test/mod/test_fmt_v1.py b/test/mod/test_fmt_v1.py
index 771dd654..5e39b1d4 100644
--- a/test/mod/test_fmt_v1.py
+++ b/test/mod/test_fmt_v1.py
@@ -171,6 +171,22 @@ class TestFormatV1(unittest.TestCase):
 
         self.assertEqual(fmt.describe(manifest), BASIC_PIPELINE)
 
+    def test_format_info(self):
+        index = osbuild.meta.Index(os.curdir)
+
+        lst = index.list_formats()
+        self.assertIn("osbuild.formats.v1", lst)
+
+        # an empty manifest is format "1"
+        info = index.detect_format_info({})
+        self.assertEqual(info.version, "1")
+        self.assertEqual(info.module, fmt)
+
+        # the basic test manifest
+        info = index.detect_format_info(BASIC_PIPELINE)
+        self.assertEqual(info.version, "1")
+        self.assertEqual(info.module, fmt)
+
     @unittest.skipUnless(test.TestBase.can_bind_mount(), "root-only")
     def test_format_output(self):
         """Test that output formatting is as expected"""