Repository: easybuilders/easybuild-framework

Issue:
Endless job submission when job=True in config file
An endless number of jobs are submitted via Slurm when the following option is enabled in the `.cfg` file:

```
# Submit the build as a job (default: False)
job=True
```
No build process is actually started; it seems each job just spawns another Slurm job that is then added to the queue.

For example, issuing `eb -r zlib-1.2.8.eb` will yield the following sample output from any Slurm job:

```
== temporary log file in case of crash ~/easybuild/tmp/eb-viHTks/easybuild-2KFvjK.log
== List of submitted jobs (1): zlib-1.2.8 (Core/zlib/1.2.8): 632849
== Submitted parallel build jobs, exiting now
== Temporary log file(s) ~/easybuild/tmp/eb-viHTks/easybuild-2KFvjK.log* have been removed.
== Temporary directory ~/easybuild/tmp/eb-viHTks has been removed.
```


Discussion:

Resolving patch:
diff --git a/easybuild/tools/parallelbuild.py b/easybuild/tools/parallelbuild.py
--- a/easybuild/tools/parallelbuild.py
+++ b/easybuild/tools/parallelbuild.py
@@ -127,11 +127,17 @@ def submit_jobs(ordered_ecs, cmd_line_opts, testing=False, prepare_first=True):
     curdir = os.getcwd()
 
     # regex pattern for options to ignore (help options can't reach here)
-    ignore_opts = re.compile('^--robot$|^--job$|^--try-.*$')
+    ignore_opts = re.compile('^--robot$|^--job|^--try-.*$')
 
     # generate_cmd_line returns the options in form --longopt=value
     opts = [o for o in cmd_line_opts if not ignore_opts.match(o.split('=')[0])]
 
+    # add --disable-job to make sure the submitted job doesn't submit a job itself,
+    # resulting in an infinite cycle of jobs;
+    # this can happen if job submission is enabled via a configuration file or via $EASYBUILD_JOB,
+    # cfr. https://github.com/easybuilders/easybuild-framework/issues/3307
+    opts.append('--disable-job')
+
     # compose string with command line options, properly quoted and with '%' characters escaped
     opts_str = ' '.join(opts).replace('%', '%%')
 


Test patch:
diff --git a/test/framework/parallelbuild.py b/test/framework/parallelbuild.py
--- a/test/framework/parallelbuild.py
+++ b/test/framework/parallelbuild.py
@@ -173,8 +173,10 @@ def test_build_easyconfigs_in_parallel_pbs_python(self):
         # dependencies for gzip/1.4-GCC-4.6.3: GCC/4.6.3 (toolchain) + toy/.0.0-deps
         self.assertTrue('gzip-1.4-GCC-4.6.3.eb' in jobs[3].script)
         self.assertEqual(len(jobs[3].deps), 2)
-        regex = re.compile('toy-0.0-deps.eb\s* --hidden')
-        self.assertTrue(regex.search(jobs[3].deps[0].script))
+        regex = re.compile(r'toy-0.0-deps\.eb.* --hidden')
+        script_txt = jobs[3].deps[0].script
+        fail_msg = "Pattern '%s' should be found in: %s" % (regex.pattern, script_txt)
+        self.assertTrue(regex.search(script_txt), fail_msg)
         self.assertTrue('GCC-4.6.3.eb' in jobs[3].deps[1].script)
 
         # also test use of --pre-create-installdir
@@ -290,6 +292,7 @@ def test_submit_jobs(self):
             '--try-toolchain=intel,2016a',  # should be excluded in job script
             '--robot', self.test_prefix,  # should be excluded in job script
             '--job',  # should be excluded in job script
+            '--job-cores=3',
         ]
         eb_go = parse_options(args=args)
         cmd = submit_jobs([toy_ec], eb_go.generate_cmd_line(), testing=True)
@@ -306,6 +309,7 @@ def test_submit_jobs(self):
             ' eb %\(spec\)s ',
             ' %\(add_opts\)s ',
             ' --testoutput=%\(output_dir\)s',
+            ' --disable-job ',
         ]
         for regex in regexs:
             regex = re.compile(regex)
@@ -313,9 +317,9 @@ def test_submit_jobs(self):
 
         # these patterns should NOT be found, these options get filtered out
         # (self.test_prefix was argument to --robot)
-        for regex in ['--job', '--try-toolchain', '--robot=[ =]', self.test_prefix + ' ']:
+        for regex in ['--job', '--job-cores', '--try-toolchain', '--robot=[ =]', self.test_prefix + ' ']:
             regex = re.compile(regex)
-            self.assertFalse(regex.search(cmd), "Pattern '%s' *not* found in: %s" % (regex.pattern, cmd))
+            self.assertFalse(regex.search(cmd), "Pattern '%s' should *not* be found in: %s" % (regex.pattern, cmd))
 
     def test_build_easyconfigs_in_parallel_slurm(self):
         """Test build_easyconfigs_in_parallel(), using (mocked) Slurm as backend for --job."""