Repository: reframe-hpc/reframe

Issue:
`--distribute` option is too slow
```
time ./bin/reframe -C config/cscs.py -c cscstests/microbenchmarks/gpu/gpu_burn/gpu_burn_test.py -S num_tasks=1 -J reservation=foo -l
...
real	0m0.734s
user	0m0.509s
sys	0m0.105s
```

```
time ./bin/reframe -C config/cscs.py -c cscstests/microbenchmarks/gpu/gpu_burn/gpu_burn_test.py -S num_tasks=1 -J reservation=foo --distribute=all -l
...
real	0m4.629s
user	0m3.077s
sys	0m0.443s
```

I think that the slowdown is due to the `scontrol`s done behind the scenes, but we need to investigate further and either improve it or add a note in the documentation.

Comments:

Patch:
diff --git a/reframe/frontend/executors/__init__.py b/reframe/frontend/executors/__init__.py
index 795d2869..aa2b47e2 100644
--- a/reframe/frontend/executors/__init__.py
+++ b/reframe/frontend/executors/__init__.py
@@ -36,11 +36,11 @@ class TestCase:
 
     def __init__(self, check, partition, environ):
         self._check_orig = check
-        self._check = copy.deepcopy(check)
-        self._partition = copy.deepcopy(partition)
-        self._environ = copy.deepcopy(environ)
-        self._check._case = weakref.ref(self)
+        self._check = check
+        self._partition = partition
+        self._environ = environ
         self._deps = []
+        self._is_ready = False
 
         # Incoming dependencies
         self.in_degree = 0
@@ -72,6 +72,15 @@ class TestCase:
         e = self.environ.name if self.environ else None
         return f'({c!r}, {p!r}, {e!r})'
 
+    def prepare(self):
+        '''Prepare test case for sending down the test pipeline'''
+        if self._is_ready:
+            return
+
+        self._check = copy.deepcopy(self._check)
+        self._check._case = weakref.ref(self)
+        self._is_ready = True
+
     @property
     def check(self):
         return self._check
@@ -97,8 +106,14 @@ class TestCase:
         return TestCase(self._check_orig, self._partition, self._environ)
 
 
-def generate_testcases(checks):
-    '''Generate concrete test cases from checks.'''
+def generate_testcases(checks, prepare=False):
+    '''Generate concrete test cases from checks.
+
+    If `prepare` is true then each of the cases will also be prepared for
+    being sent to the test pipeline. Note that setting this to true may slow down
+    the test case generation.
+
+    '''
 
     rt = runtime.runtime()
     cases = []
@@ -107,7 +122,11 @@ def generate_testcases(checks):
                                                c.valid_prog_environs)
         for part, environs in valid_comb.items():
             for env in environs:
-                cases.append(TestCase(c, part, env))
+                case = TestCase(c, part, env)
+                if prepare:
+                    case.prepare()
+
+                cases.append(case)
 
     return cases
 
@@ -295,6 +314,7 @@ class RegressionTask:
             raise TaskExit from e
 
     def setup(self, *args, **kwargs):
+        self.testcase.prepare()
         self._safe_call(self.check.setup, *args, **kwargs)
         self._notify_listeners('on_task_setup')
 
diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json
index 472b0d8a..46df7c9f 100644
--- a/reframe/schemas/config.json
+++ b/reframe/schemas/config.json
@@ -514,7 +514,7 @@
         "environments/features": [],
         "environments/target_systems": ["*"],
         "general/dump_pipeline_progress": false,
-        "general/pipeline_timeout": null,
+        "general/pipeline_timeout": 3,
         "general/check_search_path": ["${RFM_INSTALL_PREFIX}/checks/"],
         "general/check_search_recursive": false,
         "general/clean_stagedir": true,


Test patch:
diff --git a/unittests/test_dependencies.py b/unittests/test_dependencies.py
index 3cf15f3e..29066810 100644
--- a/unittests/test_dependencies.py
+++ b/unittests/test_dependencies.py
@@ -375,7 +375,11 @@ def test_build_deps_deprecated_syntax(loader, default_exec_ctx):
 
 def test_build_deps(loader, default_exec_ctx):
     checks = loader.load_all(force=True)
-    cases = executors.generate_testcases(checks)
+
+    # We need to prepare the test cases as if we were about to run them,
+    # because we want to test `getdep()` as well, which normally gets resolved
+    # during the `setup` phase of the pipeline
+    cases = executors.generate_testcases(checks, prepare=True)
 
     # Test calling getdep() before having built the graph
     t = find_check('Test1_fully', checks)
diff --git a/unittests/test_pipeline.py b/unittests/test_pipeline.py
index f3ba74ed..8ef7818b 100644
--- a/unittests/test_pipeline.py
+++ b/unittests/test_pipeline.py
@@ -1210,7 +1210,7 @@ def test_require_deps(HelloTest, local_exec_ctx):
         def setz(self, T0):
             self.z = T0().x + 2
 
-    cases = executors.generate_testcases([T0(), T1()])
+    cases = executors.generate_testcases([T0(), T1()], prepare=True)
     deps, _ = dependencies.build_deps(cases)
     for c in dependencies.toposort(deps):
         _run(*c)