Repository: easybuilders/easybuild-framework

Issue:
Add ability to archive easyblocks
Based on a question that came up at the workshop in Birmingham 2018-10-22.


Discussion:
This should go into the `easybuild/reprod` dir then that we already have now? Thoughts @ocaisa?

We should probably also have some dedicated documentation on how to reproduce installations done with EasyBuild?

Or maybe even drop a script into `easybuild/reprod` that specifies everything that's needed (`--include-easyblocks=...` for example)?
Definitely should go in the `easybuild/reprod` dir. Since the inheritance implies there will typically be a few, I'd suggest we put all the necessary easyblocks in a subdir there so that someone can just use `--include-easyblocks=/.../easybuild/reprod/reprod_easyblocks/*`
Hmm, I can't find any way of doing this fully. It's straightforward to get the source file for the easyblock (using `inspect.getsourcefile()` and `get_easyblock_class()`) but that is incomplete unless you follow the inheritance all the way back to the base `easyblock` class.
Ok, I found out how to do it:
```
        app_class = get_easyblock_class(easyblock, name=name)
        for easyblock_class in inspect.getmro(app_class):
            easyblock_path = inspect.getsourcefile(easyblock_class)
            if os.path.split(easyblock_path)[0].endswith(os.path.join('easybuild-framework', 'easybuild', 'framework')):
                break
            else:
                print easyblock_path
```
PR is in the works.

Resolving patch:
diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -2824,12 +2824,14 @@ def build_and_install_one(ecdict, init_env):
 
     # load easyblock
     easyblock = build_option('easyblock')
-    if not easyblock:
+    if easyblock:
+        # set the value in the dict so this is included in the reproducability dump of the easyconfig
+        ecdict['ec']['easyblock'] = easyblock
+    else:
         easyblock = fetch_parameters_from_easyconfig(rawtxt, ['easyblock'])[0]
 
     try:
         app_class = get_easyblock_class(easyblock, name=name)
-
         app = app_class(ecdict['ec'])
         _log.info("Obtained application instance of for %s (easyblock: %s)" % (name, easyblock))
     except EasyBuildError, err:
@@ -2859,7 +2861,6 @@ def build_and_install_one(ecdict, init_env):
         errormsg = "build failed (first %d chars): %s" % (first_n, err.msg[:first_n])
         _log.warning(errormsg)
         result = False
-    app.close_log()
 
     ended = 'ended'
 
@@ -2893,12 +2894,8 @@ def build_and_install_one(ecdict, init_env):
 
             # for reproducability we dump out the fully processed easyconfig since the contents can be affected
             # by subtoolchain resolution (and related options) and/or hooks
-            reprod_spec = os.path.join(new_log_dir, 'reprod', ec_filename)
-            try:
-                app.cfg.dump(reprod_spec)
-                _log.info("Dumped fully processed easyconfig to %s", reprod_spec)
-            except NotImplementedError as err:
-                _log.warn("Unable to dumped fully processed easyconfig to %s: %s", reprod_spec, err)
+            reprod_dir = reproduce_build(app, new_log_dir)
+            _log.info("Wrote files for reproducability to %s", reprod_dir)
 
             try:
                 # upload easyconfig (and patch files) to central repository
@@ -2916,6 +2913,7 @@ def build_and_install_one(ecdict, init_env):
                 _log.warn("Unable to commit easyconfig to repository: %s", err)
 
         # cleanup logs
+        app.close_log()
         log_fn = os.path.basename(get_log_filename(app.name, app.version))
         application_log = os.path.join(new_log_dir, log_fn)
         move_logs(app.logfile, application_log)
@@ -2982,6 +2980,41 @@ def build_and_install_one(ecdict, init_env):
     return (success, application_log, errormsg)
 
 
+def reproduce_build(app, reprod_dir_root):
+    """
+    Create reproducability files (processed easyconfig and easyblocks used) from class instance
+
+    :param app: easyblock class instance
+    :param reprod_dir_root: root directory in which to create the 'reprod' directory
+
+    :return reprod_dir: directory containing reproducability files
+    """
+
+    ec_filename = '%s-%s.eb' % (app.name, det_full_ec_version(app.cfg))
+
+    reprod_dir = os.path.join(reprod_dir_root, 'reprod')
+    reprod_spec = os.path.join(reprod_dir, ec_filename)
+    try:
+        app.cfg.dump(reprod_spec)
+        _log.info("Dumped easyconfig instance to %s", reprod_spec)
+    except NotImplementedError as err:
+        _log.warn("Unable to dump easyconfig instance to %s: %s", reprod_spec, err)
+
+    # also archive the relevant easyblocks
+    reprod_easyblock_dir = os.path.join(reprod_dir, 'easyblocks')
+    for easyblock_class in inspect.getmro(type(app)):
+        easyblock_path = inspect.getsourcefile(easyblock_class)
+        easyblock_basedir, easyblock_filename = os.path.split(easyblock_path)
+        # if we reach EasyBlock or ExtensionEasyBlock class, we are done
+        # (ExtensionEasyblock is hardcoded to avoid a cyclical import)
+        if easyblock_class.__name__ in [EasyBlock.__name__, 'ExtensionEasyBlock']:
+            break
+        else:
+            copy_file(easyblock_path, os.path.join(reprod_easyblock_dir, easyblock_filename))
+            _log.info("Dumped easyblock %s required for reproduction to %s", easyblock_filename, reprod_easyblock_dir)
+
+    return reprod_dir
+
 def get_easyblock_instance(ecdict):
     """
     Get an instance for this easyconfig


Test patch:
diff --git a/test/framework/options.py b/test/framework/options.py
--- a/test/framework/options.py
+++ b/test/framework/options.py
@@ -582,6 +582,7 @@ def test_list_easyblocks(self):
                 r'\|-- ExtensionEasyBlock',
                 r'\|   \|-- DummyExtension',
                 r'\|   \|-- EB_toy',
+                r'\|   \|   \|-- EB_toytoy',
                 r'\|   \|-- Toy_Extension',
                 r'\|-- ModuleRC',
                 r'\|-- Toolchain',
@@ -589,6 +590,7 @@ def test_list_easyblocks(self):
                 r'\|-- ExtensionEasyBlock',
                 r'\|   \|-- DummyExtension',
                 r'\|   \|-- EB_toy',
+                r'\|   \|   \|-- EB_toytoy',
                 r'\|   \|-- Toy_Extension',
             ])
             regex = re.compile(expected, re.M)
diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toytoy.py b/test/framework/sandbox/easybuild/easyblocks/t/toytoy.py
new file mode 100644
--- /dev/null
+++ b/test/framework/sandbox/easybuild/easyblocks/t/toytoy.py
@@ -0,0 +1,44 @@
+##
+# Copyright 2009-2018 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
+##
+"""
+EasyBuild support for building and installing foofoo, implemented as an easyblock
+
+@author: Alan O'Cais (Juelich Supercomputing Centre)
+"""
+
+from easybuild.easyblocks.toy import EB_toy
+from easybuild.framework.easyconfig import CUSTOM
+
+
+class EB_toytoy(EB_toy):
+    """Support for building/installing toy."""
+
+    @staticmethod
+    def extra_options():
+        """Custom easyconfig parameters for toytoy."""
+        extra_vars = {
+            'toytoy_extra1': [None, "first toytoy-specific easyconfig parameter", CUSTOM],
+         }
+        return EB_toy.extra_options(extra_vars)
diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py
--- a/test/framework/toy_build.py
+++ b/test/framework/toy_build.py
@@ -1532,10 +1532,32 @@ def test_minimal_toolchains(self):
         # this test doesn't check for anything specific to using minimal toolchains, only side-effects
         self.test_toy_build(extra_args=['--minimal-toolchains'])
 
+    def test_reproducability(self):
+        """Test toy build produces expected reproducability files"""
+        # use the easyblock with inheritance to fully test
+        self.test_toy_build(extra_args=['--minimal-toolchains', '--easyblock=EB_toytoy'])
         # also check whether easyconfig is dumped to reprod/ subdir
-        reprod_ec = os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'easybuild', 'reprod', 'toy-0.0.eb')
+        reprod_dir = os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'easybuild', 'reprod')
+        reprod_ec = os.path.join(reprod_dir, 'toy-0.0.eb')
+
         self.assertTrue(os.path.exists(reprod_ec))
 
+        # Check that the toytoy easyblock is recorded in the reprod easyconfig
+        ec = EasyConfig(reprod_ec)
+        self.assertEqual(ec.parser.get_config_dict()['easyblock'], 'EB_toytoy')
+
+        # Check for easyblock existence
+        child_easyblock = os.path.join(reprod_dir, 'easyblocks', 'toytoy.py')
+        self.assertTrue(os.path.exists(child_easyblock))
+        # Check for parent easyblock existence
+        parent_easyblock = os.path.join(reprod_dir, 'easyblocks', 'toy.py')
+        self.assertTrue(os.path.exists(parent_easyblock))
+
+        # Make sure framework easyblock modules are not included
+        for framework_easyblock in ['easyblock.py', 'extensioneasyblock.py']:
+            path = os.path.join(reprod_dir, 'easyblocks', framework_easyblock)
+            self.assertFalse(os.path.exists(path))
+
     def test_toy_toy(self):
         """Test building two easyconfigs in a single go, with one depending on the other."""
         topdir = os.path.dirname(os.path.abspath(__file__))