Repository: easybuilders/easybuild-framework

Issue:
EB 3.9.3 regression for --tmp-logdir
With EB 3.9.2 this works in an empty directory:
```
eb ASE-3.15.0-intel-2017b-Python-2.7.14.eb --robot --tmp-logdir=`pwd`/tmplogs
```

But with 3.9.3 it crashes:
```
eb ASE-3.15.0-intel-2017b-Python-2.7.14.eb --robot --tmp-logdir=`pwd`/tmplogs 
Traceback (most recent call last):
  File "/usr/lib/python2.7/runpy.py", line 174, in _run_module_as_main
    "__main__", fname, loader, pkg_name)
  File "/usr/lib/python2.7/runpy.py", line 72, in _run_code
    exec code in run_globals
  File "/cvmfs/ebsw.hpc2n.umu.se/amd64_ubuntu1604_common/software/Core/EasyBuild/3.9.3/lib/python2.7/site-packages/easybuild_framework-3.9.3-py2.7.egg/easybuild/main.py", line 457, in <module>
    main()
  File "/cvmfs/ebsw.hpc2n.umu.se/amd64_ubuntu1604_common/software/Core/EasyBuild/3.9.3/lib/python2.7/site-packages/easybuild_framework-3.9.3-py2.7.egg/easybuild/main.py", line 193, in main
    eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing)
  File "/cvmfs/ebsw.hpc2n.umu.se/amd64_ubuntu1604_common/software/Core/EasyBuild/3.9.3/lib/python2.7/site-packages/easybuild_framework-3.9.3-py2.7.egg/easybuild/tools/options.py", line 1296, in set_up_configuration
    colorize=options.color, tmp_logdir=options.tmp_logdir)
  File "/cvmfs/ebsw.hpc2n.umu.se/amd64_ubuntu1604_common/software/Core/EasyBuild/3.9.3/lib/python2.7/site-packages/easybuild_framework-3.9.3-py2.7.egg/easybuild/tools/build_log.py", line 201, in init_logging
    fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-', dir=tmp_logdir)
  File "/usr/lib/python2.7/tempfile.py", line 314, in mkstemp
    return _mkstemp_inner(dir, prefix, suffix, flags)
  File "/usr/lib/python2.7/tempfile.py", line 244, in _mkstemp_inner
    fd = _os.open(file, flags, 0600)
OSError: [Errno 2] No such file or directory: '/import/ebswbuild/home/easybuild/q/tmplogs/easybuild-sD5Vcp.log'
```



Discussion:
reproduced, I'll look into it...

Resolving patch:
diff --git a/easybuild/tools/build_log.py b/easybuild/tools/build_log.py
--- a/easybuild/tools/build_log.py
+++ b/easybuild/tools/build_log.py
@@ -197,6 +197,13 @@ def init_logging(logfile, logtostdout=False, silent=False, colorize=fancylogger.
         fancylogger.logToScreen(enable=True, stdout=True, colorize=colorize)
     else:
         if logfile is None:
+            # if logdir is specified but doesn't exist yet, create it first
+            if tmp_logdir and not os.path.exists(tmp_logdir):
+                try:
+                    os.makedirs(tmp_logdir)
+                except (IOError, OSError) as err:
+                    raise EasyBuildError("Failed to create temporary log directory %s: %s", tmp_logdir, err)
+
             # mkstemp returns (fd,filename), fd is from os.open, not regular open!
             fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-', dir=tmp_logdir)
             os.close(fd)


Test patch:
diff --git a/test/framework/build_log.py b/test/framework/build_log.py
--- a/test/framework/build_log.py
+++ b/test/framework/build_log.py
@@ -36,8 +36,8 @@
 from unittest import TextTestRunner
 from vsc.utils.fancylogger import getLogger, getRootLoggerName, logToFile, setLogFormat
 
-from easybuild.tools.build_log import LOGGING_FORMAT, EasyBuildError, dry_run_msg, dry_run_warning
-from easybuild.tools.build_log import print_error, print_msg, print_warning, time_str_since
+from easybuild.tools.build_log import LOGGING_FORMAT, EasyBuildError, EasyBuildLog, dry_run_msg, dry_run_warning
+from easybuild.tools.build_log import init_logging, print_error, print_msg, print_warning, stop_logging, time_str_since
 from easybuild.tools.filetools import read_file, write_file
 
 
@@ -372,6 +372,60 @@ def run_check(msg, args, expected_stdout='', **kwargs):
 
         self.assertErrorRegex(EasyBuildError, "Unknown named arguments", dry_run_warning, 'foo', unknown_arg='bar')
 
+    def test_init_logging(self):
+        """Test init_logging function."""
+        # first, make very sure $TMPDIR is a subdir of self.test_prefix
+        tmpdir = os.getenv('TMPDIR')
+        self.assertTrue(tmpdir.startswith(self.test_prefix))
+
+        # use provided path for log file
+        tmp_logfile = os.path.join(self.test_prefix, 'test.log')
+        log, logfile = init_logging(tmp_logfile, silent=True)
+        self.assertEqual(logfile, tmp_logfile)
+        self.assertTrue(os.path.exists(logfile))
+        self.assertTrue(isinstance(log, EasyBuildLog))
+
+        stop_logging(logfile)
+
+        # no log provided, so create one (should be file in $TMPDIR)
+        log, logfile = init_logging(None, silent=True)
+        self.assertTrue(os.path.exists(logfile))
+        self.assertEqual(os.path.dirname(logfile), tmpdir)
+        self.assertTrue(isinstance(log, EasyBuildLog))
+
+        stop_logging(logfile)
+
+        # no problem with specifying a different directory to put log file in (even if it doesn't exist yet)
+        tmp_logdir = os.path.join(self.test_prefix, 'tmp_logs')
+        self.assertFalse(os.path.exists(tmp_logdir))
+
+        log, logfile = init_logging(None, silent=True, tmp_logdir=tmp_logdir)
+        self.assertEqual(os.path.dirname(logfile), tmp_logdir)
+        self.assertTrue(isinstance(log, EasyBuildLog))
+
+        stop_logging(logfile)
+
+        # by default, path to tmp log file is printed
+        self.mock_stdout(True)
+        log, logfile = init_logging(None)
+        stdout = self.get_stdout()
+        self.mock_stdout(False)
+        self.assertTrue(os.path.exists(logfile))
+        self.assertEqual(os.path.dirname(logfile), tmpdir)
+        self.assertTrue(isinstance(log, EasyBuildLog))
+        self.assertTrue(stdout.startswith("== temporary log file in case of crash"))
+
+        stop_logging(logfile)
+
+        # logging to stdout implies no log file
+        self.mock_stdout(True)
+        log, logfile = init_logging(None, logtostdout=True)
+        self.mock_stdout(False)
+        self.assertEqual(logfile, None)
+        self.assertTrue(isinstance(log, EasyBuildLog))
+
+        stop_logging(logfile, logtostdout=True)
+
 
 def suite():
     """ returns all the testcases in this module """
diff --git a/test/framework/options.py b/test/framework/options.py
--- a/test/framework/options.py
+++ b/test/framework/options.py
@@ -4074,6 +4074,37 @@ def test_show_system_info(self):
             regex = re.compile(pattern, re.M)
             self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))
 
+    def test_tmp_logdir(self):
+        """Test use of --tmp-logdir."""
+
+        topdir = os.path.abspath(os.path.dirname(__file__))
+        toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
+
+        # purposely use a non-existing directory as log directory
+        tmp_logdir = os.path.join(self.test_prefix, 'tmp-logs')
+        self.assertFalse(os.path.exists(tmp_logdir))
+
+        # force passing logfile=None to main in eb_main
+        self.logfile = None
+
+        # check log message with --skip for existing module
+        args = [
+            toy_ec,
+            '--sourcepath=%s' % self.test_sourcepath,
+            '--buildpath=%s' % self.test_buildpath,
+            '--installpath=%s' % self.test_installpath,
+            '--force',
+            '--debug',
+            '--tmp-logdir=%s' % tmp_logdir,
+        ]
+        self.eb_main(args, do_build=True, raise_error=True)
+
+        tmp_logs = os.listdir(tmp_logdir)
+        self.assertEqual(len(tmp_logs), 1)
+
+        logtxt = read_file(os.path.join(tmp_logdir, tmp_logs[0]))
+        self.assertTrue("COMPLETED: Installation ended successfully" in logtxt)
+
 
 def suite():
     """ returns all the testcases in this module """