Repository: easybuilders/easybuild-framework

Issue:
pyver and pyshortver template names don't work with cray-python
I am not sure if this a bug or a feature request. As far as I understood the pyver and pyshortver templates should work with external module `cray-python` as long as there is an entry in the external-modules-metadata file.

I have for example this easyconfig:
```
easyblock = 'PythonPackage'

name = 'parameterized'
version = '0.6.1'

py_maj_ver = '3'
py_min_ver = '6'
py_rev_ver = '5.7'

versionsuffix = '-python%s' % py_maj_ver

homepage = 'https://pypi.python.org/pypi/%s' % name
description = """Parameterized testing with any Python test framework."""

toolchain = {'name': 'CrayGNU', 'version': '19.10'}

source_urls = [PYPI_SOURCE]
sources = [SOURCE_TAR_GZ]

dependencies = [
     ('cray-python/3.6.5.7', EXTERNAL_MODULE),
]

sanity_check_paths = {
    'files': [],
    'dirs': ['lib/python%(pyshortver)s/site-packages'],
}

exts_filter = ('python -c "import parameterized"', "")

moduleclass = 'tools'
```

which fails with the error:
```
.../parameterized-0.6.1-CrayGNU-19.10-python3.eb failed (err: "build failed (first 300 chars): Sanity check failed: no (non-empty) directory found at 'lib/python%(pyshortver)s/site-packages' in /apps/dom/UES/eirinik/easybuild/software/parameterized/0.6.1-CrayGNU-19.10-python3")
```

If I replace this line:
```
'dirs': ['lib/python%(pyshortver)s/site-packages'],
```
with
```
'dirs': ['lib/python3.6/site-packages'],
```
the installation is completed successfully.

I checked the metadata file (https://github.com/eth-cscs/production/blob/master/easybuild/cray_external_modules_metadata-19.10.cfg) and it has this entry
```
[cray-python/3.6.5.7]
name = Python
version = 3.6.5.7
prefix = /opt/python/3.6.5.7
```

Am I missing something in the easyconfig or should there be some change in the framework to be able to use these template values?


Discussion:
@ekouts Do you remember if this ever worked with the `cray-python` external module as dependency?

It's probably not very difficult to fix this, regardless whether it worked or not, it's possible we're simply not taking into account external modules when defining the templates right now...
That's great, as far as I know it never worked with `cray-python`.

Resolving patch:
diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py
--- a/easybuild/framework/easyconfig/templates.py
+++ b/easybuild/framework/easyconfig/templates.py
@@ -222,12 +222,24 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None)
         for dep in deps:
             if isinstance(dep, dict):
                 dep_name, dep_version = dep['name'], dep['version']
+
+                # take into account dependencies marked as external modules,
+                # where name/version may have to be harvested from metadata available for that external module
+                if dep.get('external_module', False):
+                    metadata = dep.get('external_module_metadata', {})
+                    if dep_name is None:
+                        # name is a list in metadata, just take first value (if any)
+                        dep_name = metadata.get('name', [None])[0]
+                    if dep_version is None:
+                        # version is a list in metadata, just take first value (if any)
+                        dep_version = metadata.get('version', [None])[0]
+
             elif isinstance(dep, (list, tuple)):
                 dep_name, dep_version = dep[0], dep[1]
             else:
                 raise EasyBuildError("Unexpected type for dependency: %s", dep)
 
-            if isinstance(dep_name, string_type) and dep_name.lower() == name.lower():
+            if isinstance(dep_name, string_type) and dep_name.lower() == name.lower() and dep_version:
                 dep_version = pick_dep_version(dep_version)
                 template_values['%sver' % pref] = dep_version
                 dep_version_parts = dep_version.split('.')


Test patch:
diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py
--- a/test/framework/easyconfig.py
+++ b/test/framework/easyconfig.py
@@ -1622,6 +1622,57 @@ def test_external_dependencies(self):
         self.assertEqual(os.environ.get('EBVERSIONPI'), '3.14')
         self.assertEqual(os.environ.get('EBVERSIONTEST'), '9.7.5')
 
+    def test_external_dependencies_templates(self):
+        """Test use of templates for dependencies marked as external modules."""
+
+        topdir = os.path.dirname(os.path.abspath(__file__))
+        toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
+        toy_ectxt = read_file(toy_ec)
+
+        extra_ectxt = '\n'.join([
+            "versionsuffix = '-Python-%(pyver)s-Perl-%(perlshortver)s'",
+            '',
+            "dependencies = [",
+            "    ('cray-python/3.6.5.7', EXTERNAL_MODULE),",
+            "    ('perl/5.30.0-1', EXTERNAL_MODULE),",
+            "]",
+        ])
+        test_ec = os.path.join(self.test_prefix, 'test.eb')
+        write_file(test_ec, toy_ectxt + '\n' + extra_ectxt)
+
+        # put metadata in place so templates can be defined
+        metadata = os.path.join(self.test_prefix, 'external_modules_metadata.cfg')
+        metadatatxt = '\n'.join([
+            '[cray-python]',
+            'name = Python',
+            '[cray-python/3.6.5.7]',
+            'version = 3.6.5',
+            '[perl/5.30.0-1]',
+            'name = Perl',
+            'version = 5.30.0',
+        ])
+        write_file(metadata, metadatatxt)
+        build_options = {
+            'external_modules_metadata': parse_external_modules_metadata([metadata]),
+            'valid_module_classes': module_classes(),
+        }
+        init_config(build_options=build_options)
+
+        ec = EasyConfig(test_ec)
+
+        expected_template_values = {
+            'perlmajver': '5',
+            'perlshortver': '5.30',
+            'perlver': '5.30.0',
+            'pymajver': '3',
+            'pyshortver': '3.6',
+            'pyver': '3.6.5',
+        }
+        for key in expected_template_values:
+            self.assertEqual(ec.template_values[key], expected_template_values[key])
+
+        self.assertEqual(ec['versionsuffix'], '-Python-3.6.5-Perl-5.30')
+
     def test_update(self):
         """Test use of update() method for EasyConfig instances."""
         topdir = os.path.abspath(os.path.dirname(__file__))