Repository: easybuilders/easybuild-framework

Issue:
Add support for Sylabs Library as singularity source
Add
- https://cloud.sylabs.io/home

at:
- https://github.com/easybuilders/easybuild-framework/blob/master/easybuild/tools/containers/singularity.py#L44 

There is a client at:
- https://github.com/sylabs/scs-library-client


Discussion:

Resolving patch:
diff --git a/easybuild/tools/containers/singularity.py b/easybuild/tools/containers/singularity.py
--- a/easybuild/tools/containers/singularity.py
+++ b/easybuild/tools/containers/singularity.py
@@ -41,15 +41,37 @@
 from easybuild.tools.containers.base import ContainerGenerator
 
 
-DOCKER = 'docker'
-LOCALIMAGE = 'localimage'
-SHUB = 'shub'
-SINGULARITY_BOOTSTRAP_TYPES = [DOCKER, LOCALIMAGE, SHUB]
-
+ARCH = 'arch'  # Arch Linux
+BUSYBOX = 'busybox'  # BusyBox Linux
+DEBOOTSTRAP = 'debootstrap'  # apt-based systems like Ubuntu/Debian
+DOCKER = 'docker'  # image hosted on Docker Hub
+LIBRARY = 'library'  # Sylabs Container Library
+LOCALIMAGE = 'localimage'  # local image file
+SHUB = 'shub'  # image hosted on Singularity Hub
+YUM = 'yum'  # yum-based systems like CentOS
+ZYPPER = 'zypper'  # zypper-based systems like openSUSE
+
+# valid bootstrap agents for 'bootstrap' keyword in --container-base-config
+SINGULARITY_BOOTSTRAP_AGENTS = [ARCH, BUSYBOX, DEBOOTSTRAP, DOCKER, LIBRARY, LOCALIMAGE, SHUB, YUM, ZYPPER]
+
+# valid bootstrap agents for --container-base-image value
+SINGULARITY_BOOTSTRAP_AGENTS_IMAGE = [DOCKER, LIBRARY, LOCALIMAGE, SHUB]
+
+SINGULARITY_INCLUDE_DEFAULTS = {
+    YUM: 'yum',
+    ZYPPER: 'zypper',
+}
+
+SINGULARITY_MIRRORURL_DEFAULTS = {
+    BUSYBOX: 'https://www.busybox.net/downloads/binaries/%{OSVERSION}/busybox-x86_64',
+    DEBOOTSTRAP: 'http://us.archive.ubuntu.com/ubuntu/',
+    YUM: 'http://mirror.centos.org/centos-%{OSVERSION}/%{OSVERSION}/os/x86_64/',
+    ZYPPER: 'http://download.opensuse.org/distribution/leap/%{OSVERSION}/repo/oss/',
+}
 
 SINGULARITY_TEMPLATE = """
 Bootstrap: %(bootstrap)s
-From: %(from)s
+%(bootstrap_config)s
 
 %%post
 %(install_os_deps)s
@@ -115,26 +137,63 @@ def resolve_template_data_base_config(self):
 
         template_data = {}
 
-        base_config_keys = ['bootstrap', 'from']
+        base_config_known_keys = [
+            # bootstrap agent to use
+            # see https://www.sylabs.io/guides/latest/user-guide/definition_files.html#header
+            'bootstrap',
+            # argument for bootstrap agents; only valid for: docker, library, localimage, shub
+            'from',
+            # list of additional OS packages to include; only valid with debootstrap, yum, zypper
+            'include',
+            # URI to use to download OS; only valid with busybox, debootstrap, yum, zypper
+            'mirrorurl',
+            # OS 'version' to use; only valid with busybox, debootstrap, yum, zypper
+            # only required if value for %(mirrorurl)s contains %{OSVERSION}s
+            'osversion',
+        ]
 
         # configuration for base container is assumed to have <key>=<value>[,<key>=<value>] format
         config_items = self.container_base_config.split(',')
         for item in config_items:
             key, value = item.split('=', 1)
-            if key in base_config_keys:
+            if key in base_config_known_keys:
                 template_data[key] = value
             else:
                 raise EasyBuildError("Unknown key for base container configuration: %s", key)
 
-        if sorted(base_config_keys) != sorted(template_data.keys()):
-            raise EasyBuildError("Not all keys for base configuration were specified! Found %s, expected %s",
-                                 ', '.join(sorted(base_config_keys)), ', '.join(sorted(template_data.keys())))
+        # make sure correct bootstrap agent is specified
+        bootstrap = template_data.get('bootstrap')
+        if bootstrap:
+            if bootstrap not in SINGULARITY_BOOTSTRAP_AGENTS:
+                raise EasyBuildError("Unknown value specified for 'bootstrap' keyword: %s (known: %s)",
+                                     bootstrap, ', '.join(SINGULARITY_BOOTSTRAP_AGENTS))
+        else:
+            raise EasyBuildError("Keyword 'bootstrap' is required in container base config")
+
+        # make sure 'from' is specified when required
+        if bootstrap in SINGULARITY_BOOTSTRAP_AGENTS_IMAGE and template_data.get('from') is None:
+            raise EasyBuildError("Keyword 'from' is required in container base config when using bootstrap agent '%s'",
+                                 bootstrap)
+
+        # use default value for mirror URI if none was specified
+        if bootstrap in SINGULARITY_MIRRORURL_DEFAULTS and template_data.get('mirrorurl') is None:
+            template_data['mirrorurl'] = SINGULARITY_MIRRORURL_DEFAULTS[bootstrap]
+
+        # check whether OS version is specified if required
+        mirrorurl = template_data.get('mirrorurl')
+        if mirrorurl and '%{OSVERSION}' in mirrorurl and template_data.get('osversion') is None:
+            raise EasyBuildError("Keyword 'osversion' is required in container base config when '%%{OSVERSION}' "
+                                 "is used in mirror URI: %s", mirrorurl)
+
+        # use default value for list of included OS packages if nothing else was specified
+        if bootstrap in SINGULARITY_INCLUDE_DEFAULTS and template_data.get('include') is None:
+            template_data['include'] = SINGULARITY_INCLUDE_DEFAULTS[bootstrap]
 
         return template_data
 
     def resolve_template_data_base_image(self):
         """Return template data for container recipe based on what is passed to --container-base-image."""
-        base_specs = parse_container_base(self.container_base_image)
+        base_specs = parse_container_base_image(self.container_base_image)
 
         # extracting application name,version, version suffix, toolchain name, toolchain version from
         # easyconfig class
@@ -157,8 +216,8 @@ def resolve_template_data_base_image(self):
             else:
                 raise EasyBuildError("Singularity base image at specified path does not exist: %s", base_image)
 
-        # otherwise, bootstrap agent is 'docker' or 'shub'
-        # format --container-base-image {docker|shub}:<image>:<tag>
+        # otherwise, bootstrap agent is 'docker', 'library' or 'shub'
+        # format --container-base-image {docker|library|shub}:<image>:<tag>
         else:
             base_image = base_specs['arg1']
             # image tag is optional
@@ -190,6 +249,13 @@ def resolve_template_data(self):
         else:
             raise EasyBuildError("Either --container-base-config or --container-base-image must be specified!")
 
+        # puzzle together specs for bootstrap agent
+        bootstrap_config_lines = []
+        for key in ['From', 'OSVersion', 'MirrorURL', 'Include']:
+            if key.lower() in template_data:
+                bootstrap_config_lines.append('%s: %s' % (key, template_data[key.lower()]))
+        template_data['bootstrap_config'] = '\n'.join(bootstrap_config_lines)
+
         # if there is osdependencies in easyconfig then add them to Singularity recipe
         install_os_deps = ''
         for ec in self.easyconfigs:
@@ -213,6 +279,7 @@ def resolve_template_data(self):
         return template_data
 
     def build_image(self, recipe_path):
+        """Build container image by calling out to 'sudo singularity build'."""
 
         cont_path = container_path()
         def_file = os.path.basename(recipe_path)
@@ -276,7 +343,7 @@ def build_image(self, recipe_path):
         print_msg("Singularity image created at %s" % img_path, log=self.log)
 
 
-def parse_container_base(base):
+def parse_container_base_image(base):
     """Parse value passed to --container-base-image option."""
     if base:
         base_specs = base.split(':')
@@ -284,21 +351,22 @@ def parse_container_base(base):
             error_msg = '\n'.join([
                 "Invalid format for --container-base-image, must be one of the following:",
                 '',
+                "--container-base-image library:<entity>/<collection>/<container>:<tag>",
                 "--container-base-image localimage:/path/to/image",
-                "--container-base-image shub:<image>:<tag>",
-                "--container-base-image docker:<image>:<tag>",
+                "--container-base-image shub:<registry>/<username>/<container-name>:<tag>@digest",
+                "--container-base-image docker:<registry>/<namespace>/<container>:<tag>@<digest>",
             ])
             raise EasyBuildError(error_msg)
     else:
         raise EasyBuildError("--container-base-image must be specified")
 
-    # first argument to --container-base-image is the Singularity bootstrap agent (localimage, shub, docker)
+    # first argument to --container-base-image is the Singularity bootstrap agent (library, localimage, shub, docker)
     bootstrap_agent = base_specs[0]
 
-    # check bootstrap type value and ensure it is localimage, shub, docker
-    if bootstrap_agent not in SINGULARITY_BOOTSTRAP_TYPES:
-        known_bootstrap_agents = ', '.join(SINGULARITY_BOOTSTRAP_TYPES)
-        raise EasyBuildError("Bootstrap agent in container base spec must be one of: %s" % known_bootstrap_agents)
+    # check bootstrap type value and ensure it is library, localimage, shub, docker
+    if bootstrap_agent not in SINGULARITY_BOOTSTRAP_AGENTS_IMAGE:
+        known_bootstrap_agents = ', '.join(SINGULARITY_BOOTSTRAP_AGENTS_IMAGE)
+        raise EasyBuildError("Bootstrap agent in container base image spec must be one of: %s" % known_bootstrap_agents)
 
     res = {'bootstrap_agent': bootstrap_agent}
 


Test patch:
diff --git a/test/framework/containers.py b/test/framework/containers.py
--- a/test/framework/containers.py
+++ b/test/framework/containers.py
@@ -36,7 +36,7 @@
 
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.filetools import adjust_permissions, mkdir, read_file, remove_file, which, write_file
-from easybuild.tools.containers.singularity import parse_container_base
+from easybuild.tools.containers.singularity import parse_container_base_image
 
 
 MOCKED_SINGULARITY = """#!/bin/bash
@@ -73,32 +73,32 @@
 class ContainersTest(EnhancedTestCase):
     """Tests for containers support"""
 
-    def test_parse_container_base(self):
-        """Test parse_container_basefunction."""
+    def test_parse_container_base_image(self):
+        """Test parse_container_base_image function."""
 
         for base_spec in [None, '']:
             error_pattern = "--container-base-image must be specified"
-            self.assertErrorRegex(EasyBuildError, error_pattern, parse_container_base, base_spec)
+            self.assertErrorRegex(EasyBuildError, error_pattern, parse_container_base_image, base_spec)
 
         # format of base spec must be correct: <bootstrap_agent>:<arg> or <bootstrap_agent>:<arg1>:<arg2>
         error_regex = "Invalid format for --container-base-image"
         for base_spec in ['foo', 'foo:bar:baz:sjee']:
-            self.assertErrorRegex(EasyBuildError, error_regex, parse_container_base, base_spec)
+            self.assertErrorRegex(EasyBuildError, error_regex, parse_container_base_image, base_spec)
 
         # bootstrap agent must be known
-        error_regex = "Bootstrap agent in container base spec must be one of: docker, localimage, shub"
-        self.assertErrorRegex(EasyBuildError, error_regex, parse_container_base, 'foo:bar')
+        error_regex = "Bootstrap agent in container base image spec must be one of: docker, library, localimage, shub"
+        self.assertErrorRegex(EasyBuildError, error_regex, parse_container_base_image, 'foo:bar')
 
         # check parsing of 'localimage' base spec
         expected = {'bootstrap_agent': 'localimage', 'arg1': '/path/to/base.img'}
-        self.assertEqual(parse_container_base('localimage:/path/to/base.img'), expected)
+        self.assertEqual(parse_container_base_image('localimage:/path/to/base.img'), expected)
 
         # check parsing of 'docker' and 'shub' base spec (2nd argument, image tag, is optional)
         for agent in ['docker', 'shub']:
             expected = {'bootstrap_agent': agent, 'arg1': 'foo'}
-            self.assertEqual(parse_container_base('%s:foo' % agent), expected)
+            self.assertEqual(parse_container_base_image('%s:foo' % agent), expected)
             expected.update({'arg2': 'bar'})
-            self.assertEqual(parse_container_base('%s:foo:bar' % agent), expected)
+            self.assertEqual(parse_container_base_image('%s:foo:bar' % agent), expected)
 
     def run_main(self, args, raise_error=True):
         """Helper function to run main with arguments specified in 'args' and return stdout/stderr."""
@@ -118,8 +118,8 @@ def check_regexs(self, regexs, stdout):
             regex = re.compile(regex, re.M)
             self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout))
 
-    def test_end2end_singularity_recipe(self):
-        """End-to-end test for --containerize (recipe only)."""
+    def test_end2end_singularity_recipe_base_image(self):
+        """End-to-end test for --containerize (recipe only), using --container-base-image."""
         test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
         toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb')
 
@@ -134,7 +134,7 @@ def test_end2end_singularity_recipe(self):
             '--experimental',
         ]
 
-        error_pattern = "--container-base-image must be specified"
+        error_pattern = "--container-base-config or --container-base-image must be specified"
         self.assertErrorRegex(EasyBuildError, error_pattern, self.run_main, args, raise_error=True)
 
         # generating Singularity definition file with 'docker' or 'shub' bootstrap agents always works,
@@ -203,6 +203,105 @@ def test_end2end_singularity_recipe(self):
             error_pattern = "Invalid image extension '.*' must be \.img or \.simg"
             self.assertErrorRegex(EasyBuildError, error_pattern, self.run_main, args, raise_error=True)
 
+    def test_end2end_singularity_recipe_base_config(self):
+        """End-to-end test for --containerize (recipe only), using --container-base-config."""
+        test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
+        toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb')
+
+        containerpath = os.path.join(self.test_prefix, 'containers')
+        os.environ['EASYBUILD_CONTAINERPATH'] = containerpath
+        # --containerpath must be an existing directory (this is done to avoid misconfiguration)
+        mkdir(containerpath)
+
+        test_container_recipe = os.path.join(self.test_prefix, 'containers', 'Singularity.toy-0.0')
+
+        args = [
+            toy_ec,
+            '--containerize',
+            '--experimental',
+        ]
+
+        args.extend(['--container-base-config', 'osversion=7.6.1810'])
+        error_pattern = r"Keyword 'bootstrap' is required in container base config"
+        self.assertErrorRegex(EasyBuildError, error_pattern, self.run_main, args, raise_error=True)
+
+        args.extend(['--container-base-config', 'bootstrap=foobar'])
+        error_pattern = r"Unknown value specified for 'bootstrap' keyword: foobar \(known: arch, busybox, debootstrap, "
+        self.assertErrorRegex(EasyBuildError, error_pattern, self.run_main, args, raise_error=True)
+
+        # default mirror URL for yum bootstrap agent uses ${OSVERSION}, so 'osversion' must be specified too
+        args.extend(['--container-base-config', 'bootstrap=yum'])
+        error_pattern = "Keyword 'osversion' is required in container base config when '%{OSVERSION}' is used"
+        self.assertErrorRegex(EasyBuildError, error_pattern, self.run_main, args, raise_error=True)
+
+        args[-1] = 'bootstrap=yum,osversion=7.6.1810'
+        stdout, stderr = self.run_main(args, raise_error=True)
+
+        txt = read_file(test_container_recipe)
+        expected = '\n'.join([
+            "Bootstrap: yum",
+            "OSVersion: 7.6.1810",
+            "MirrorURL: http://mirror.centos.org/centos-%{OSVERSION}/%{OSVERSION}/os/x86_64/",
+            "Include: yum",
+            '\n',
+        ])
+        self.assertTrue(txt.startswith(expected), "Container recipe starts with '%s':\n\n%s" % (expected, txt))
+
+        remove_file(test_container_recipe)
+
+        # can also specify a custom mirror URL
+        args[-1] += ',mirrorurl=https://example.com'
+        stdout, stderr = self.run_main(args, raise_error=True)
+
+        txt = read_file(test_container_recipe)
+        expected = '\n'.join([
+            "Bootstrap: yum",
+            "OSVersion: 7.6.1810",
+            "MirrorURL: https://example.com",
+            "Include: yum",
+            '\n',
+        ])
+        self.assertTrue(txt.startswith(expected), "Container recipe starts with '%s':\n\n%s" % (expected, txt))
+
+        remove_file(test_container_recipe)
+
+        # osversion is not required when %{OSVERSION} is nost used in mirror URL
+        args[-1] = 'bootstrap=yum,mirrorurl=https://example.com,include=test123'
+        stdout, stderr = self.run_main(args, raise_error=True)
+
+        txt = read_file(test_container_recipe)
+        expected = '\n'.join([
+            "Bootstrap: yum",
+            "MirrorURL: https://example.com",
+            "Include: test123",
+            '\n',
+        ])
+        self.assertTrue(txt.startswith(expected), "Container recipe starts with '%s':\n\n%s" % (expected, txt))
+
+        # also check with image-based bootstrap agent, which requires 'from'
+        test_cases = [
+            ('docker', 'test'),
+            ('localimage', 'test.simg'),
+            ('library', 'sylabsed/examples/lolcow:latest'),
+            ('shub', 'test'),
+        ]
+        error_pattern = "Keyword 'from' is required in container base config when using bootstrap agent"
+        for (bootstrap, from_spec) in test_cases:
+            args[-1] = 'bootstrap=%s' % bootstrap
+            self.assertErrorRegex(EasyBuildError, error_pattern, self.run_main, args, raise_error=True)
+
+            args[-1] += ',from=%s' % from_spec
+            remove_file(test_container_recipe)
+            stdout, stderr = self.run_main(args, raise_error=True)
+
+            txt = read_file(test_container_recipe)
+            expected = '\n'.join([
+                "Bootstrap: %s" % bootstrap,
+                "From: %s" % from_spec,
+                '',
+            ])
+            self.assertTrue(txt.startswith(expected), "Container recipe starts with '%s':\n\n%s" % (expected, txt))
+
     def test_end2end_singularity_image(self):
         """End-to-end test for --containerize (recipe + image)."""
         topdir = os.path.dirname(os.path.abspath(__file__))
@@ -435,7 +534,7 @@ def test_container_base_config_template_recipe(self):
         error_pattern = "Either --container-base-config or --container-base-image must be specified!"
         self.assertErrorRegex(EasyBuildError, error_pattern, self.run_main, args)
 
-        args.extend(['--container-base-config', 'bootstrap=test,from=foobar'])
+        args.extend(['--container-base-config', 'bootstrap=localimage,from=foobar'])
         stdout, stderr = self.run_main(args)
 
         self.assertFalse(stderr)
@@ -444,7 +543,7 @@ def test_container_base_config_template_recipe(self):
 
         expected = '\n'.join([
             "# this is just a test",
-            "bootstrap: test",
+            "bootstrap: localimage",
             "from: foobar",
             "",
             "%post",