Repository: easybuilders/easybuild-easyblocks

Issue:
TensorFlow ignores system dependencies
TensorFlow has some bundled stuff that is built alongside the main build. We should use system libs where possible. There is a list of stuff that we can replace: https://github.com/tensorflow/tensorflow/blob/f64e839687b6129dfe2a9e21ef965593c7f3a3c2/third_party/systemlibs/syslibs_configure.bzl#L11-L46

I did a grep for compilations done during TF 2.1.0, grouped and sorted them (should be the number of files compiled basically as a metric how expensive a dependency is):

      2 @org_sqlite
      3 @farmhash_archive
      4 @lmdb
      6 @com_github_nanopb_nanopb
      6 @fft2d
      6 @highwayhash
      6 @jsoncpp_git
      6 @kissfft
      9 @snappy
     21 @pcre
     24 @double_conversion
     24 @gif
     30 @png
     44 @flatbuffers
     45 @zlib_archive
     48 @swig
     51 @nsync
     54 @hwloc
     63 @com_googlesource_code_re2
     78 @nasm
    150 @libjpeg_turbo
    178 @curl
    223 @com_google_absl
    252 @local_config_mlir
    290 @mkl_dnn
    347 @com_google_protobuf
    382 @icu
    552 @boringssl
    676 @aws
    831 @grpc
    840 @llvm

So judging from that protobuf is the obvious #1 candidate: We already include it as a dependency. 

llvm would be great to move out but not sure if this is worth.

aws is strange. We set TF_NEED_S3 and TF_NEED_AWS to 0 but it is still built. From the source I judge it is not supported (anymore). But there is `--config=noaws` that we can pass to configure

Curl is another candidate as we already have ECs for that. Same for double-conversion.

Generally: We should comb through the list and use explicit dependencies where possible.


Discussion:
I tried to use at least protobuf with TF but failed, there are so many issues. What I got is the following just for reference in case future TF versions are not as broken as the current 2.1.0:

```
    def get_sys_libs_options(self):
        """Get list of options to append to the build command for "system" (or EB installed) libraries to use"""
        # See third_party/systemlibs/syslibs_configure.bzl for available libraries

        result = []
        libs = []
        dependencies = set(dep['name'] for dep in self.cfg.dependencies())

        # (Proper) support for external protobuf added in 1.15
        if self.version >= LooseVersion("1.15.0") and 'protobuf' in dependencies:
            libs.append('com_google_protobuf')
            # Required to make it find the headers, see https://github.com/tensorflow/tensorflow/issues/37835
            result.append('--define INCLUDEDIR=%s' % os.path.join(get_software_root('protobuf'), 'include'))
            result.append('--action_env CPATH=%s' % os.path.join(get_software_root('protobuf'), 'include'))

        if libs:
            result.append('--action_env TF_SYSTEM_LIBS=%s' % ','.join(libs))
        return result
```

Bottom line: We would need to pass pretty much all required folders through `action_env` down to the compiler (or use a wrapper as e.g. done with Intel compiler) and maybe some juggling with defines (see INCLUDE_DIR). Finally the protobuf.bzl file was not updated for a while so it kinda works in TF 2.0.0 with Bazel 0.26.1 but 2.1 fails with Bazel 0.29.1 with 
> Found tool(s) 'bazel-out/k8-opt/bin/external/grpc/grpc_python_plugin' in inputs. A tool is an input with executable=True set. All tools should be passed using the 'tools' argument instead of 'inputs' in order to make their runfiles availa
ble to the action. This safety check will not be performed once the action is modified to take a 'tools' argument. To temporarily disable this check, set --incompatible_no_support_tools_in_action_inputs=false.


And at this point I gave up. Let them fix their stuff first -.-

Resolving patch:
diff --git a/easybuild/easyblocks/t/tensorflow.py b/easybuild/easyblocks/t/tensorflow.py
--- a/easybuild/easyblocks/t/tensorflow.py
+++ b/easybuild/easyblocks/t/tensorflow.py
@@ -34,6 +34,7 @@
 import re
 import stat
 import tempfile
+import json
 from distutils.version import LooseVersion
 
 import easybuild.tools.environment as env
@@ -41,13 +42,15 @@
 from easybuild.easyblocks.generic.pythonpackage import PythonPackage, det_python_version
 from easybuild.easyblocks.python import EXTS_FILTER_PYTHON_PACKAGES
 from easybuild.framework.easyconfig import CUSTOM
+from easybuild.tools import run
 from easybuild.tools.build_log import EasyBuildError, print_warning
 from easybuild.tools.config import build_option
 from easybuild.tools.filetools import adjust_permissions, apply_regex_substitutions, copy_file, mkdir, resolve_path
 from easybuild.tools.filetools import is_readable, read_file, which, write_file, remove_file
-from easybuild.tools.modules import get_software_root, get_software_version
+from easybuild.tools.modules import get_software_root, get_software_version, get_software_libdir
 from easybuild.tools.run import run_cmd
 from easybuild.tools.systemtools import X86_64, get_cpu_architecture, get_os_name, get_os_version
+from easybuild.tools.py2vs3 import subprocess_popen_text
 
 
 # Wrapper for Intel(MPI) compilers, where required environment variables
@@ -73,6 +76,130 @@
 """
 
 
+def split_tf_libs_txt(valid_libs_txt):
+    """Split the VALID_LIBS entry from the TF file into single names"""
+    entries = valid_libs_txt.split(',')
+    # Remove double quotes and whitespace
+    result = [entry.strip().strip('"') for entry in entries]
+    # Remove potentially trailing empty element due to trailing comma in the txt
+    if not result[-1]:
+        result.pop()
+    return result
+
+
+def get_system_libs_from_tf(source_dir):
+    """Return the valid values for TF_SYSTEM_LIBS from the TensorFlow source directory"""
+    syslibs_path = os.path.join(source_dir, 'third_party', 'systemlibs', 'syslibs_configure.bzl')
+    result = []
+    if os.path.exists(syslibs_path):
+        txt = read_file(syslibs_path)
+        valid_libs_match = re.search(r'VALID_LIBS\s*=\s*\[(.*?)\]', txt, re.DOTALL)
+        if not valid_libs_match:
+            raise EasyBuildError('VALID_LIBS definition not found in %s', syslibs_path)
+        result = split_tf_libs_txt(valid_libs_match.group(1))
+    return result
+
+
+def get_system_libs_for_version(tf_version, as_valid_libs=False):
+    """
+    Determine valid values for $TF_SYSTEM_LIBS for the given TF version
+
+    If as_valid_libs=False (default) then returns 2 dictioniaries:
+        1: Mapping of <EB name> to <TF name>
+        2: Mapping of <package name> to <TF name> (for python extensions)
+    else returns a string formated like the VALID_LIBS variable in third_party/systemlibs/syslibs_configure.bzl
+        Those can be used to check/diff against third_party/systemlibs/syslibs_configure.bzl by running:
+            python -c 'from easybuild.easyblocks.tensorflow import get_system_libs_for_version; \
+                        print(get_system_libs_for_version("2.1.0", as_valid_libs=True))'
+    """
+    tf_version = LooseVersion(tf_version)
+
+    def is_version_ok(version_range):
+        """Return True if the TF version to be installed matches the version_range"""
+        min_version, max_version = version_range.split(':')
+        result = True
+        if min_version and tf_version < LooseVersion(min_version):
+            result = False
+        if max_version and tf_version >= LooseVersion(max_version):
+            result = False
+        return result
+
+    # For these lists check third_party/systemlibs/syslibs_configure.bzl --> VALID_LIBS
+    # Also verify third_party/systemlibs/<name>.BUILD or third_party/systemlibs/<name>/BUILD.system
+    # if it does something "strange" (e.g. link hardcoded headers)
+
+    # Software which is added as a dependency in the EC
+    available_system_libs = {
+        # Format: (<EB name>, <version range>): <TF name>
+        #         <version range> is '<min version>:<exclusive max version>'
+        ('cURL', '2.1.0:'): 'curl',
+        ('double-conversion', '2.1.0:'): 'double_conversion',
+        ('flatbuffers', '2.1.0:'): 'flatbuffers',
+        ('giflib', '2.1.0:'): 'gif',
+        ('hwloc', '2.1.0:'): 'hwloc',
+        ('ICU', '2.1.0:'): 'icu',
+        ('JsonCpp', '2.1.0:'): 'jsoncpp_git',
+        ('libjpeg-turbo', '2.1.0:2.2.0'): 'jpeg',
+        ('libjpeg-turbo', '2.2.0:'): 'libjpeg_turbo',
+        ('libpng', '2.1.0:'): 'png',
+        ('LMDB', '2.1.0:'): 'lmdb',
+        ('NASM', '2.1.0:'): 'nasm',
+        ('nsync', '2.1.0:'): 'nsync',
+        ('PCRE', '2.1.0:'): 'pcre',
+        ('protobuf-python', '2.1.0:'): 'com_google_protobuf',
+        ('pybind11', '2.2.0:'): 'pybind11',
+        ('snappy', '2.1.0:'): 'snappy',
+        ('SQLite', '2.1.0:'): 'org_sqlite',
+        ('SWIG', '2.1.0:'): 'swig',
+        ('zlib', '2.1.0:2.2.0'): 'zlib_archive',
+        ('zlib', '2.2.0:'): 'zlib',
+    }
+    # Software recognized by TF but which is always disabled (usually because no EC is known)
+    # Format: <TF name>: <version range>
+    unused_system_libs = {
+        'boringssl': '2.1.0:',
+        'com_github_googleapis_googleapis': '2.1.0:',
+        'com_github_googlecloudplatform_google_cloud_cpp': '2.1.0:',  # Not used due to $TF_NEED_GCP=0
+        'com_github_grpc_grpc': '2.2.0:',
+        'com_googlesource_code_re2': '2.1.0:',
+        'grpc': '2.1.0:2.2.0',
+    }
+    # Python packages installed as extensions or in the Python module
+    # Will be checked for availabilitly
+    # Format: (<package name>, <version range>): <TF name>
+    python_system_libs = {
+        ('absl', '2.1.0:'): 'absl_py',
+        ('astor', '2.1.0:'): 'astor_archive',
+        ('astunparse', '2.2.0:'): 'astunparse_archive',
+        ('cython', '2.1.0:'): 'cython',  # Part of Python EC
+        ('enum', '2.1.0:'): 'enum34_archive',  # Part of Python3
+        ('functools', '2.1.0:'): 'functools32_archive',  # Part of Python3
+        ('gast', '2.1.0:'): 'gast_archive',
+        ('keras_applications', '2.1.0:2.2.0'): 'keras_applications_archive',
+        ('opt_einsum', '2.1.0:'): 'opt_einsum_archive',
+        ('pasta', '2.1.0:'): 'pasta',
+        ('six', '2.1.0:'): 'six_archive',  # Part of Python EC
+        ('termcolor', '2.1.0:'): 'termcolor_archive',
+        ('wrapt', '2.1.0:'): 'wrapt',
+    }
+    dependency_mapping = dict((dep_name, tf_name)
+                              for (dep_name, version_range), tf_name in available_system_libs.items()
+                              if is_version_ok(version_range))
+    python_mapping = dict((pkg_name, tf_name)
+                          for (pkg_name, version_range), tf_name in python_system_libs.items()
+                          if is_version_ok(version_range))
+
+    if as_valid_libs:
+        tf_names = [tf_name for tf_name, version_range in unused_system_libs.items()
+                    if is_version_ok(version_range)]
+        tf_names.extend(dependency_mapping.values())
+        tf_names.extend(python_mapping.values())
+        result = '\n'.join(['    "%s",' % name for name in sorted(tf_names)])
+    else:
+        result = dependency_mapping, python_mapping
+    return result
+
+
 class EB_TensorFlow(PythonPackage):
     """Support for building/installing TensorFlow."""
 
@@ -102,6 +229,7 @@ def __init__(self, *args, **kwargs):
             'use_pip': True,
         }
         self.cfg['exts_filter'] = EXTS_FILTER_PYTHON_PACKAGES
+        self.system_libs_info = None
 
         self.test_script = None
 
@@ -152,6 +280,107 @@ def write_wrapper(self, wrapper_dir, compiler, i_mpi_root):
             adjust_permissions(wrapper, stat.S_IXUSR)
             self.log.info("Using wrapper script for '%s': %s", compiler, which(compiler))
 
+    def verify_system_libs_info(self):
+        """Verifies that the stored info about $TF_SYSTEM_LIBS is complete"""
+        available_libs_src = set(get_system_libs_from_tf(self.start_dir))
+        available_libs_eb = set(split_tf_libs_txt(get_system_libs_for_version(self.version, as_valid_libs=True)))
+        # If available_libs_eb is empty it is not an error e.g. it is not worth trying to make all old ECs work
+        # So we just log it so it can be verified manually if required
+        if not available_libs_eb:
+            self.log.warning('TensorFlow EasyBlock does not have any information for $TF_SYSTEM_LIBS stored. ' +
+                             'This means most dependencies will be downloaded at build time by TensorFlow.\n' +
+                             'Available $TF_SYSTEM_LIBS according to the TensorFlow sources: %s',
+                             sorted(available_libs_src))
+            return
+        # Those 2 sets should be equal. We determine the differences here to report better errors
+        missing_libs = available_libs_src - available_libs_eb
+        unknown_libs = available_libs_eb - available_libs_src
+        if missing_libs or unknown_libs:
+            if not available_libs_src:
+                msg = 'Failed to determine available $TF_SYSTEM_LIBS from the source'
+            else:
+                msg = 'Values for $TF_SYSTEM_LIBS in the TensorFlow EasyBlock are incomplete.\n'
+                if missing_libs:
+                    # Libs available according to TF sources but not listed in this EasyBlock
+                    msg += 'Missing entries for $TF_SYSTEM_LIBS: %s\n' % missing_libs
+                if unknown_libs:
+                    # Libs listed in this EasyBlock but not present in the TF sources -> Removed?
+                    msg += 'Unrecognized entries for $TF_SYSTEM_LIBS: %s\n' % unknown_libs
+                msg += 'The EasyBlock needs to be updated to fully work with TensorFlow version %s' % self.version
+            if build_option('strict') == run.ERROR:
+                raise EasyBuildError(msg)
+            else:
+                print_warning(msg)
+
+    def get_system_libs(self):
+        """
+        Get list of dependencies for $TF_SYSTEM_LIBS
+
+        Returns a tuple of lists: $TF_SYSTEM_LIBS names, include paths, library paths
+        """
+        dependency_mapping, python_mapping = get_system_libs_for_version(self.version)
+
+        system_libs = []
+        cpaths = []
+        libpaths = []
+        ignored_system_deps = []
+
+        # Check direct dependencies
+        dep_names = set(dep['name'] for dep in self.cfg.dependencies())
+        for dep_name, tf_name in sorted(dependency_mapping.items()):
+            if dep_name in dep_names:
+                system_libs.append(tf_name)
+                # For protobuf we need protobuf and protobuf-python where the latter depends on the former
+                # For includes etc. we need to get the values from protobuf
+                if dep_name == 'protobuf-python':
+                    dep_name = 'protobuf'
+                sw_root = get_software_root(dep_name)
+                # Dependency might be filtered via --filter-deps. In that case assume globally installed version
+                if not sw_root:
+                    continue
+                incpath = os.path.join(sw_root, 'include')
+                if dep_name == 'JsonCpp':
+                    # Need to use the install prefix instead: https://github.com/tensorflow/tensorflow/issues/42303
+                    incpath = sw_root
+                if os.path.exists(incpath):
+                    cpaths.append(incpath)
+                    if dep_name == 'protobuf':
+                        # Need to set INCLUDEDIR as TF wants to symlink headers from there:
+                        # https://github.com/tensorflow/tensorflow/issues/37835
+                        env.setvar('INCLUDEDIR', incpath)
+                libpath = get_software_libdir(dep_name)
+                if libpath:
+                    libpaths.append(os.path.join(sw_root, libpath))
+            else:
+                ignored_system_deps.append('%s (%s)' % (tf_name, dep_name))
+
+        # Check installed python packages but only check stdout, not stderr which might contain user facing warnings
+        cmd_list = [self.python_cmd, '-m', 'pip', 'list', '--isolated', '--disable-pip-version-check',
+                    '--format', 'json']
+        full_cmd = ' '.join(cmd_list)
+        self.log.info("Running command '%s'" % full_cmd)
+        proc = subprocess_popen_text(cmd_list, env=os.environ)
+        (stdout, stderr) = proc.communicate()
+        ec = proc.returncode
+        self.log.info("Command '%s' returned with %s: stdout: %s; stderr: %s" % (full_cmd, ec, stdout, stderr))
+        if ec:
+            raise EasyBuildError('Failed to determine installed python packages: %s', stderr)
+
+        installed_package_names = [pkg['name'] for pkg in json.loads(stdout.strip())]
+
+        for pkg_name, tf_name in sorted(python_mapping.items()):
+            if pkg_name in installed_package_names:
+                system_libs.append(tf_name)
+            else:
+                ignored_system_deps.append('%s (%s)' % (tf_name, dep_name))
+
+        if ignored_system_deps:
+            self.log.warning('For the following dependencies TensorFlow will download a copy as an EB dependency ' +
+                             'was not found: \n%s',
+                             ', '.join(ignored_system_deps))
+
+        return system_libs, cpaths, libpaths
+
     def configure_step(self):
         """Custom configuration procedure for TensorFlow."""
 
@@ -205,9 +434,11 @@ def configure_step(self):
             env.setvar('PATH', os.pathsep.join([wrapper_dir, os.getenv('PATH')]))
 
         self.prepare_python()
-
         self.handle_jemalloc()
 
+        self.verify_system_libs_info()
+        self.system_libs_info = self.get_system_libs()
+
         cuda_root = get_software_root('CUDA')
         cudnn_root = get_software_root('cuDNN')
         opencl_root = get_software_root('OpenCL')
@@ -237,6 +468,7 @@ def configure_step(self):
             'TF_NEED_KAFKA': '0',  # Amazon Kafka Platform
             'TF_SET_ANDROID_WORKSPACE': '0',
             'TF_DOWNLOAD_CLANG': '0',  # Still experimental in TF 2.1.0
+            'TF_SYSTEM_LIBS': ','.join(self.system_libs_info[0]),
         }
         if cuda_root:
             cuda_version = get_software_version('CUDA')
@@ -471,6 +703,12 @@ def build_step(self):
         pythonpath = os.getenv('PYTHONPATH', '')
         env.setvar('PYTHONPATH', os.pathsep.join([os.path.join(self.installdir, self.pylibdir), pythonpath]))
 
+        # Make TF find our modules. LD_LIBRARY_PATH gets automatically added by configure.py
+        cpaths, libpaths = self.system_libs_info[1:]
+        if cpaths:
+            cmd.append("--action_env=CPATH='%s'" % ':'.join(cpaths))
+        if libpaths:
+            cmd.append("--action_env=LIBRARY_PATH='%s'" % ':'.join(libpaths))
         cmd.append('--action_env=PYTHONPATH')
         # Also export $EBPYTHONPREFIXES to handle the multi-deps python setup
         # See https://github.com/easybuilders/easybuild-easyblocks/pull/1664
@@ -588,10 +826,6 @@ def sanity_check_step(self):
         ]
         res = super(EB_TensorFlow, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands)
 
-        # determine top-level directory
-        # start_dir is not set when TensorFlow is installed as an extension, then fall back to ext_dir
-        topdir = self.start_dir or self.ext_dir
-
         # test installation using MNIST tutorial examples
         if self.cfg['runtest']:
             pythonpath = os.getenv('PYTHONPATH', '')
@@ -609,7 +843,7 @@ def sanity_check_step(self):
             for mnist_py in mnist_pys:
                 datadir = tempfile.mkdtemp(suffix='-tf-%s-data' % os.path.splitext(mnist_py)[0])
                 logdir = tempfile.mkdtemp(suffix='-tf-%s-logs' % os.path.splitext(mnist_py)[0])
-                mnist_py = os.path.join(topdir, 'tensorflow', 'examples', 'tutorials', 'mnist', mnist_py)
+                mnist_py = os.path.join(self.start_dir, 'tensorflow', 'examples', 'tutorials', 'mnist', mnist_py)
                 cmd = "%s %s --data_dir %s --log_dir %s" % (self.python_cmd, mnist_py, datadir, logdir)
                 run_cmd(cmd, log_all=True, simple=True, log_ok=True)
 


Test patch: