Repository: Biometria-se/grizzly (language: python)

Issue:
grizzly: E2E distributed tests takes too long time
probably the image has to be built for every testcase.
Created a runnable example project.
While doing this, a couple of bugs and a feature was also implemented, to actually be able to run the example.


Discussion / hints:

Fix patch:
diff --git a/.gitignore b/.gitignore
index d08a6f08..4a9a261c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,6 +16,7 @@ coverage.xml
 .venv/
 **/tmp*
 *.log
+*.hprof
 
 # automagically generated
 docs/licenses/*
diff --git a/pyproject.toml b/pyproject.toml
index b382999c..28e8a4a5 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -83,12 +83,12 @@ dev = [
     "types-PyYAML <6.0.0,>=5.3.0",
     "types-requests >=2.27.0",
     "types-Jinja2 >=2.0.0",
-    "types-backports >=0.1.3"
+    "types-backports >=0.1.3",
+    "snakeviz"
 ]
 ci = [
     "build >=0.7.0",
-    "twine >=3.8.0,<4.0.0",
-    "pip-tools >=6.5.0"
+    "twine >=3.8.0,<4.0.0"
 ]
 
 [tool.setuptools_scm]
@@ -265,5 +265,6 @@ addopts = [
 ]
 timeout = 10
 filterwarnings = [
-    "ignore:setDaemon\\(\\) is deprecated.*:DeprecationWarning"
+    "ignore:setDaemon\\(\\) is deprecated.*:DeprecationWarning",
+    "ignore:.*pkg_resources.*:DeprecationWarning"
 ]
diff --git a/requirements-docs.txt b/requirements-docs.txt
index 2386c2f5..827e11ed 100644
--- a/requirements-docs.txt
+++ b/requirements-docs.txt
@@ -4,7 +4,7 @@ pydoc-markdown[novella]>=4.6.1
 docspec==2.1.2
 docspec-python==2.1.2
 pytablewriter>=0.64.1
-pip-licenses>=3.5.3
+pip-licenses>=3.5.3,<4.2.0
 requests>=2.27.1
 mkdocs>=1.3.0
 mkdocs-material>=8.3.2


Test patch:
diff --git a/tests/e2e/steps/background/test_setup.py b/tests/e2e/steps/background/test_setup.py
index 37660c9a..e88e9f03 100644
--- a/tests/e2e/steps/background/test_setup.py
+++ b/tests/e2e/steps/background/test_setup.py
@@ -1,19 +1,20 @@
-from tempfile import NamedTemporaryFile
+import inspect
+
 from typing import cast, Dict, Any, List
 
 import pytest
-import yaml
 
 from grizzly.types.behave import Context
 from grizzly.context import GrizzlyContext
 
 from tests.fixtures import End2EndFixture
+from tests.helpers import message_callback
 
 
 @pytest.mark.parametrize('url', [
-    'influxdb://grizzly:password@localhost/grizzly-statistics',
+    'influxdb://grizzly:password@localhost/grizzly-statistics?Testplan=grizzly-statistics',
     'insights://localhost/?Testplan=grizzly-statistics&InstrumentationKey=asdfasdf=',
-    'influxdb://$conf::statistics.username$:$conf::statistics.password$@localhost/$conf::statistics.database$',
+    'influxdb://$conf::statistics.username$:$conf::statistics.password$@localhost/$conf::statistics.database$?Testplan=grizzly-statistics',
 ])
 def test_e2e_step_setup_save_statistics(e2e_fixture: End2EndFixture, url: str) -> None:
     env_conf: Dict[str, Any] = {
@@ -57,13 +58,9 @@ def validator(context: Context) -> None:
         identifier=url,
     )
 
-    with NamedTemporaryFile(delete=True, suffix='.yaml', dir=e2e_fixture.test_tmp_dir) as env_conf_file:
-        env_conf_file.write(yaml.dump(env_conf, Dumper=yaml.Dumper).encode())
-        env_conf_file.flush()
-
-        rc, _ = e2e_fixture.execute(feature_file, env_conf_file=env_conf_file.name)
+    rc, _ = e2e_fixture.execute(feature_file, env_conf=env_conf)
 
-        assert rc == 0
+    assert rc == 0
 
 
 @pytest.mark.parametrize('level', [
@@ -192,7 +189,7 @@ def validator(context: Context) -> None:
         direction = MessageDirection.from_string(data['direction'])
         message_type = data['message_type']
 
-        from tests.helpers import message_callback
+        from steps.helpers import message_callback  # type: ignore  # pylint: disable=import-error
 
         assert grizzly.setup.locust.messages == {
             direction: {
@@ -202,6 +199,13 @@ def validator(context: Context) -> None:
 
         grizzly.setup.locust.messages.clear()
 
+    with open(e2e_fixture.root / 'features' / 'steps' / 'helpers.py', 'w+') as fd:
+        source = inspect.getsource(message_callback)
+        fd.write(f'''from grizzly.types.locust import Message, Environment
+
+{source}
+''')
+
     table: List[Dict[str, str]] = [{
         'direction': f'{from_node}_{to_node}',
         'message_type': message_type,
@@ -211,7 +215,7 @@ def validator(context: Context) -> None:
 
     feature_file = e2e_fixture.test_steps(
         background=[
-            f'And add callback "tests.helpers.message_callback" for message type "{message_type}" from {from_node} to {to_node}',
+            f'And add callback "steps.helpers.message_callback" for message type "{message_type}" from {from_node} to {to_node}',
         ],
         identifier=message_type,
     )
diff --git a/tests/e2e/steps/scenario/test_tasks.py b/tests/e2e/steps/scenario/test_tasks.py
index 4e6b8b1d..9f958b50 100644
--- a/tests/e2e/steps/scenario/test_tasks.py
+++ b/tests/e2e/steps/scenario/test_tasks.py
@@ -1,10 +1,8 @@
 from json import dumps as jsondumps
-from tempfile import NamedTemporaryFile
 from typing import cast, List, Dict
 from textwrap import dedent
 
 import pytest
-import yaml
 
 from grizzly.context import GrizzlyContext
 from grizzly.types.behave import Context, Feature
@@ -588,20 +586,17 @@ def validate_client_task_until(context: Context) -> None:
         ]
     )
 
-    with NamedTemporaryFile(delete=True, suffix='.yaml', dir=e2e_fixture.test_tmp_dir) as env_conf_file:
-        env_conf: Dict[str, Dict[str, Dict[str, str]]] = {
-            'configuration': {
-                'test': {
-                    'host': f'http://{e2e_fixture.host}',
-                }
+    env_conf: Dict[str, Dict[str, Dict[str, str]]] = {
+        'configuration': {
+            'test': {
+                'host': f'http://{e2e_fixture.host}',
             }
         }
-        env_conf_file.write(yaml.dump(env_conf, Dumper=yaml.Dumper).encode())
-        env_conf_file.flush()
+    }
 
-        rc, _ = e2e_fixture.execute(feature_file, env_conf_file=env_conf_file.name)
+    rc, _ = e2e_fixture.execute(feature_file, env_conf=env_conf)
 
-        assert rc == 0
+    assert rc == 0
 
 
 def test_e2e_step_task_client_put_endpoint_file_destination(e2e_fixture: End2EndFixture) -> None:
diff --git a/tests/e2e/test_example.py b/tests/e2e/test_example.py
index a2220801..3c0b2db8 100644
--- a/tests/e2e/test_example.py
+++ b/tests/e2e/test_example.py
@@ -1,12 +1,10 @@
-from tempfile import NamedTemporaryFile
 from typing import Optional
 from pathlib import Path
-from getpass import getuser
+from shutil import copytree
 
 import yaml
 
 from tests.fixtures import End2EndFixture
-from tests.helpers import run_command
 
 
 def test_e2e_example(e2e_fixture: End2EndFixture) -> None:
@@ -20,20 +18,20 @@ def test_e2e_example(e2e_fixture: End2EndFixture) -> None:
                 env_conf['configuration']['facts'][name]['host'] = f'http://{e2e_fixture.host}'
 
         if e2e_fixture._distributed:
-            example_root = str((e2e_fixture.root / '..' / 'test-example').resolve())
-            command = ['grizzly-cli', 'dist', '--project-name', 'test-example', 'build', '--no-cache', '--local-install']
-            code, output = run_command(
-                command,
-                cwd=str(e2e_fixture.mode_root),
-                env=e2e_fixture._env,
-            )
+            # copy examples
+            source = (Path(__file__) / '..' / '..' / '..' / 'example').resolve()
+            example_root = e2e_fixture.root.parent / 'test-example'
+            copytree(source, example_root, dirs_exist_ok=True)
 
-            try:
-                assert code == 0
-            except AssertionError:
-                print(''.join(output))
+            with open(Path(example_root) / 'features' / 'steps' / 'steps.py', 'a') as fd:
+                fd.write(
+                    e2e_fixture.step_start_webserver.format('/srv/grizzly')
+                )
 
-                raise
+            # create steps/webserver.py
+            webserver_source = e2e_fixture.test_tmp_dir.parent / 'tests' / 'webserver.py'
+            webserver_destination = example_root / 'features' / 'steps' / 'webserver.py'
+            webserver_destination.write_text(webserver_source.read_text())
 
             root = (Path(__file__) / '..' / '..' / '..').resolve()
             feature_file_root = str(example_root).replace(f'{root}/', '')
@@ -47,90 +45,59 @@ def test_e2e_example(e2e_fixture: End2EndFixture) -> None:
             with open(feature_file, 'w') as fd:
                 fd.truncate(0)
                 fd.write('\n'.join(feature_file_contents))
-
-            exec_root = str(e2e_fixture.mode_root)
         else:
-            example_root = exec_root = str(Path.cwd() / 'example')
-            feature_file = 'features/example.feature'
-
-        with NamedTemporaryFile(delete=True, suffix='.yaml', dir=f'{example_root}/environments') as env_conf_file:
-            env_conf_file.write(yaml.dump(env_conf, Dumper=yaml.Dumper).encode())
-            env_conf_file.flush()
-
-            command = [
-                'grizzly-cli',
-                e2e_fixture.mode, 'run',
-                '--yes',
-                '--verbose',
-                '-e', env_conf_file.name,
-                feature_file,
-            ]
-
-            if e2e_fixture._distributed:
-                command = command[:2] + ['--project-name', 'test-example'] + command[2:]
-
-            code, output = run_command(
-                command,
-                env=e2e_fixture._env,
-                cwd=exec_root,
-            )
-
-            result = ''.join(output)
-
-            if e2e_fixture._distributed:
-                result = ''
-                for container in ['master', 'worker']:
-                    command = ['docker', 'container', 'logs', f'test-example-{getuser()}_{container}_1']
-                    _, output = run_command(
-                        command,
-                        cwd=exec_root,
-                        env=e2e_fixture._env,
-                    )
-
-                    result += ''.join(output)
-
-                if code != 0:
-                    print(''.join(output))
-
-            assert code == 0
-            assert 'ERROR' not in result
-            assert 'WARNING' not in result
-            assert '1 feature passed, 0 failed, 0 skipped' in result
-            assert '3 scenarios passed, 0 failed, 0 skipped' in result
-            # Then start webserver... added when running distributed
-            step_count = 26 if e2e_fixture._distributed else 25
-            assert f'{step_count} steps passed, 0 failed, 0 skipped, 0 undefined' in result
-
-            assert 'ident   iter  status   description' in result
-            assert '001      2/2  passed   dog facts api' in result
-            assert '002      1/1  passed   cat facts api' in result
-            assert '003      1/1  passed   book api' in result
-            assert '------|-----|--------|---------------|' in result
-
-            assert 'executing custom.User.request for get-cat-facts and /facts?limit=' in result
-
-            assert 'sending "client_server" from CLIENT' in result
-            assert "received from CLIENT" in result
-            assert "AtomicCustomVariable.foobar='foobar'" in result
-
-            # check debugging and that task index -> step expression is correct
-            assert 'executing task 1 of 3: iterator' in result
-            assert (
-                'executing task 2 of 3: Then get request with name "get-dog-facts" from endpoint '
-                '"/api/v1/resources/dogs?number={{ AtomicRandomInteger.dog_facts_count }}'
-            ) in result
-            assert 'executing task 3 of 3: pace' in result
-
-            assert 'executing task 1 of 4: iterator' in result
-            assert 'executing task 2 of 4: Then get request with name "get-cat-facts" from endpoint "/facts?limit={{ AtomicRandomInteger.cat_facts_count }}"' in result
-            assert 'executing task 3 of 4: And send message "{\'client\': \'server\'}"' in result
-            assert 'executing task 4 of 4: pace' in result
-
-            assert 'executing task 1 of 5: iterator' in result
-            assert 'executing task 2 of 5: Then get request with name "1-get-book" from endpoint "/books/{{ AtomicCsvReader.books.book }}.json | content_type=json"' in result
-            assert 'executing task 3 of 5: Then get request with name "2-get-author" from endpoint "{{ author_endpoint }}.json | content_type=json"' in result
-            assert 'executing task 4 of 5: Then log message "AtomicCustomVariable.foobar=\'{{ steps.custom.AtomicCustomVariable.foobar }}\'"' in result
-            assert 'executing task 5 of 5: pace' in result
+            example_root = Path.cwd() / 'example'
+
+        feature_file = 'features/example.feature'
+
+        original_root = e2e_fixture._root
+        e2e_fixture._root = example_root
+
+        code, output = e2e_fixture.execute(feature_file, env_conf=env_conf)
+
+        result = ''.join(output)
+
+        e2e_fixture._root = original_root
+
+        assert code == 0
+        assert 'ERROR' not in result
+        assert 'WARNING' not in result
+        assert '1 feature passed, 0 failed, 0 skipped' in result
+        assert '3 scenarios passed, 0 failed, 0 skipped' in result
+        # Then start webserver... added when running distributed
+        step_count = 26 if e2e_fixture._distributed else 25
+        assert f'{step_count} steps passed, 0 failed, 0 skipped, 0 undefined' in result
+
+        assert 'ident   iter  status   description' in result
+        assert '001      2/2  passed   dog facts api' in result
+        assert '002      1/1  passed   cat facts api' in result
+        assert '003      1/1  passed   book api' in result
+        assert '------|-----|--------|---------------|' in result
+
+        assert 'executing custom.User.request for get-cat-facts and /facts?limit=' in result
+
+        assert 'sending "client_server" from CLIENT' in result
+        assert "received from CLIENT" in result
+        assert "AtomicCustomVariable.foobar='foobar'" in result
+
+        # check debugging and that task index -> step expression is correct
+        assert 'executing task 1 of 3: iterator' in result
+        assert (
+            'executing task 2 of 3: Then get request with name "get-dog-facts" from endpoint '
+            '"/api/v1/resources/dogs?number={{ AtomicRandomInteger.dog_facts_count }}'
+        ) in result
+        assert 'executing task 3 of 3: pace' in result
+
+        assert 'executing task 1 of 4: iterator' in result
+        assert 'executing task 2 of 4: Then get request with name "get-cat-facts" from endpoint "/facts?limit={{ AtomicRandomInteger.cat_facts_count }}"' in result
+        assert 'executing task 3 of 4: And send message "{\'client\': \'server\'}"' in result
+        assert 'executing task 4 of 4: pace' in result
+
+        assert 'executing task 1 of 5: iterator' in result
+        assert 'executing task 2 of 5: Then get request with name "1-get-book" from endpoint "/books/{{ AtomicCsvReader.books.book }}.json | content_type=json"' in result
+        assert 'executing task 3 of 5: Then get request with name "2-get-author" from endpoint "{{ author_endpoint }}.json | content_type=json"' in result
+        assert 'executing task 4 of 5: Then log message "AtomicCustomVariable.foobar=\'{{ steps.custom.AtomicCustomVariable.foobar }}\'"' in result
+        assert 'executing task 5 of 5: pace' in result
     except:
         if result is not None:
             print(result)
diff --git a/tests/e2e/test_iteration_pace.py b/tests/e2e/test_iteration_pace.py
index e8a630a5..60b3bb55 100644
--- a/tests/e2e/test_iteration_pace.py
+++ b/tests/e2e/test_iteration_pace.py
@@ -1,8 +1,5 @@
 from typing import Dict, cast
 from textwrap import dedent
-from tempfile import NamedTemporaryFile
-
-import yaml
 
 from grizzly.context import GrizzlyContext
 from grizzly.types.behave import Context, Feature
@@ -70,10 +67,6 @@ def after_feature(context: Context, feature: Feature) -> None:
         Then end condition
     '''))  # noqa: E501
 
-    with NamedTemporaryFile(delete=True, suffix='.yaml', dir=e2e_fixture.test_tmp_dir) as env_conf_file:
-        env_conf_file.write(yaml.dump(env_conf, Dumper=yaml.Dumper).encode())
-        env_conf_file.flush()
-
-        rc, _ = e2e_fixture.execute(feature_file, env_conf_file=env_conf_file.name)
+    rc, _ = e2e_fixture.execute(feature_file, env_conf=env_conf)
 
-        assert rc == 0
+    assert rc == 0
diff --git a/tests/e2e/test_until.py b/tests/e2e/test_until.py
index b58fe34a..6b79da3a 100644
--- a/tests/e2e/test_until.py
+++ b/tests/e2e/test_until.py
@@ -1,8 +1,5 @@
 from typing import Dict, cast
 from textwrap import dedent
-from tempfile import NamedTemporaryFile
-
-import yaml
 
 from grizzly.types.behave import Context, Feature
 from grizzly.context import GrizzlyContext
@@ -72,10 +69,6 @@ def after_feature(context: Context, feature: Feature) -> None:
         Then get "http://$conf::test.host$/api/until/foofoo?nth=2&wrong=foobar&right=world&as_array=true | content_type=json" with name "http-client-task" until "$.`this`[?foofoo="world"] | retries=3, expected_matches=1"
     '''))  # noqa: E501
 
-    with NamedTemporaryFile(delete=True, suffix='.yaml', dir=e2e_fixture.test_tmp_dir) as env_conf_file:
-        env_conf_file.write(yaml.dump(env_conf, Dumper=yaml.Dumper).encode())
-        env_conf_file.flush()
-
-        rc, _ = e2e_fixture.execute(feature_file, env_conf_file=env_conf_file.name)
+    rc, _ = e2e_fixture.execute(feature_file, env_conf=env_conf)
 
-        assert rc == 0
+    assert rc == 0
diff --git a/tests/fixtures.py b/tests/fixtures.py
index b3c85f31..bec3ecf8 100644
--- a/tests/fixtures.py
+++ b/tests/fixtures.py
@@ -7,13 +7,18 @@
 from unittest.mock import MagicMock
 from urllib.parse import urlparse
 from mypy_extensions import VarArg, KwArg
-from os import environ, getcwd, path
-from shutil import rmtree, copytree
+from os import environ, path
+from shutil import rmtree
 from json import dumps as jsondumps
 from pathlib import Path
 from textwrap import dedent, indent
 from hashlib import sha1
 from getpass import getuser
+from cProfile import Profile
+from contextlib import nullcontext
+from tempfile import NamedTemporaryFile
+
+import yaml
 
 from locust.clients import ResponseContextManager
 from locust.contrib.fasthttp import FastResponse, FastRequest
@@ -623,6 +628,25 @@ def {self.name}_{self.implementation.__name__}_wrapper(context: Context) -> None
 
 
 class End2EndFixture:
+    step_start_webserver = '''
+
+@then(u'start webserver on master port "{{port:d}}"')
+def step_start_webserver(context: Context, port: int) -> None:
+    from grizzly.locust import on_master
+    if not on_master(context):
+        return
+
+    from importlib.machinery import SourceFileLoader
+
+    w = SourceFileLoader(
+        'steps.webserver',
+        '{}/features/steps/webserver.py',
+    ).load_module('steps.webserver')
+
+    webserver = w.Webserver(port)
+    webserver.start()
+'''
+
     _tmp_path_factory: TempPathFactory
     _env: Dict[str, str]
     _validators: Dict[Optional[str], List[End2EndValidator]]
@@ -639,6 +663,7 @@ class End2EndFixture:
     test_tmp_dir: Path
     _tmp_path_factory_basetemp: Optional[Path]
     webserver: Webserver
+    profile: Optional[Profile]
 
     def __init__(self, tmp_path_factory: TempPathFactory, webserver: Webserver, distributed: bool) -> None:
         self.test_tmp_dir = (Path(__file__) / '..' / '..' / '.pytest_tmp').resolve()
@@ -647,7 +672,7 @@ def __init__(self, tmp_path_factory: TempPathFactory, webserver: Webserver, dist
         tmp_path_factory._basetemp = self.test_tmp_dir
 
         self._tmp_path_factory = tmp_path_factory
-        self.cwd = Path(getcwd())
+        self.cwd = Path.cwd()
         self._env = {}
         self._validators = {}
         self._root = None
@@ -655,16 +680,7 @@ def __init__(self, tmp_path_factory: TempPathFactory, webserver: Webserver, dist
         self._before_features = {}
         self._distributed = distributed
         self._has_pymqi = None
-
-    @property
-    def mode_root(self) -> Path:
-        if self._root is None:
-            raise AttributeError('root is not set')
-
-        if self._distributed:
-            return self.cwd
-        else:
-            return self._root
+        self.profile = None
 
     @property
     def root(self) -> Path:
@@ -695,6 +711,10 @@ def has_pymqi(self) -> bool:
         return self._has_pymqi
 
     def __enter__(self) -> 'End2EndFixture':
+        if environ.get('PROFILE', None) is not None:
+            self.profile = Profile()
+            self.profile.enable()
+
         project_name = 'test-project'
         test_context = self._tmp_path_factory.mktemp('test_context')
 
@@ -758,27 +778,8 @@ def __enter__(self) -> 'End2EndFixture':
 
         (self._root / 'features' / f'{project_name}.feature').unlink()
 
-        step_start_webserver = '''
-
-@then(u'start webserver on master port "{{port:d}}"')
-def step_start_webserver(context: Context, port: int) -> None:
-    from grizzly.locust import on_master
-    if not on_master(context):
-        return
-
-    from importlib.machinery import SourceFileLoader
-
-    w = SourceFileLoader(
-        'steps.webserver',
-        '{}/features/steps/webserver.py',
-    ).load_module('steps.webserver')
-
-    webserver = w.Webserver(port)
-    webserver.start()
-'''
-
         # create base test-project ... steps.py
-        with open(self._root / 'features' / 'steps' / 'steps.py', 'w') as fd:
+        with open(self.root / 'features' / 'steps' / 'steps.py', 'w') as fd:
             fd.write('from importlib import import_module\n')
             fd.write('from typing import cast, Callable, Any\n\n')
             fd.write('from grizzly.types.behave import Context, then\n')
@@ -789,35 +790,28 @@ def step_start_webserver(context: Context, port: int) -> None:
             fd.write('from grizzly.steps import *\n')
 
         if self._distributed:
-            # copy examples
-            source = (Path(__file__) / '..' / '..' / 'example').resolve()
-            destination = test_context / 'test-example'
-            copytree(source, destination)
-
             # rewrite test requirements.txt to point to local code
-            for root in [self._root, destination]:
-                with open(f'{root}/requirements.txt', 'r+') as fd:
-                    fd.truncate(0)
-                    fd.flush()
-                    fd.write('grizzly-loadtester\n')
-
-                with open(root / 'features' / 'steps' / 'steps.py', 'a') as fd:
-                    fd.write(
-                        step_start_webserver.format(
-                            str(root).replace(f'{Path.cwd()}', '/srv/grizzly'),
-                        )
+            with open(f'{self.root}/requirements.txt', 'r+') as fd:
+                fd.truncate(0)
+                fd.flush()
+                fd.write('grizzly-loadtester\n')
+
+            with open(self.root / 'features' / 'steps' / 'steps.py', 'a') as fd:
+                fd.write(
+                    self.step_start_webserver.format(
+                        str(self.root).replace(str(self.root), '/srv/grizzly'),
                     )
+                )
 
-                # create steps/webserver.py
-                webserver_source = self.test_tmp_dir.parent / 'tests' / 'webserver.py'
-                webserver_destination = root / 'features' / 'steps' / 'webserver.py'
-
-                webserver_destination.write_text(webserver_source.read_text())
+            # create steps/webserver.py
+            webserver_source = self.test_tmp_dir.parent / 'tests' / 'webserver.py'
+            webserver_destination = self.root / 'features' / 'steps' / 'webserver.py'
+            webserver_destination.write_text(webserver_source.read_text())
 
-            command = ['grizzly-cli', 'dist', '--project-name', self._root.name, 'build', '--no-cache', '--local-install']
+            command = ['grizzly-cli', 'dist', '--project-name', self.root.name, 'build', '--no-cache', '--local-install']
             rc, output = run_command(
                 command,
-                cwd=str(self.mode_root),
+                cwd=str(self.cwd),
                 env=self._env,
             )
             try:
@@ -845,6 +839,10 @@ def step_start_webserver(context: Context, port: int) -> None:
 
         return self
 
+    @property
+    def keep_files(self) -> bool:
+        return environ.get('KEEP_FILES', None) is not None
+
     def __exit__(
         self,
         exc_type: Optional[Type[BaseException]],
@@ -854,18 +852,22 @@ def __exit__(
         # reset fixture basetemp
         self._tmp_path_factory._basetemp = self._tmp_path_factory_basetemp
 
+        if self.profile is not None:
+            self.profile.disable()
+            self.profile.dump_stats('grizzly-e2e-tests.hprof')
+
         if exc is None:
-            if self._distributed:
+            if self._distributed and not self.keep_files:
                 rc, output = run_command(
                     ['grizzly-cli', 'dist', '--project-name', self.root.name, 'clean'],
-                    cwd=str(self.mode_root),
+                    cwd=str(self.root),
                     env=self._env,
                 )
 
                 if rc != 0:
                     print(''.join(output))
 
-            if environ.get('KEEP_FILES', None) is None:
+            if not self.keep_files:
                 try:
                     rmtree(self.root.parent, onerror=onerror)
                 except AttributeError:
@@ -1065,48 +1067,54 @@ def create_feature(self, contents: str, name: Optional[str] = None, identifier:
 
         return feature_file_name
 
-    def execute(self, feature_file: str, env_conf_file: Optional[str] = None, testdata: Optional[Dict[str, str]] = None) -> Tuple[int, List[str]]:
-        if self._distributed:
-            root = (Path(__file__) / '..' / '..').resolve()
-            feature_file_root = str(self.root).replace(f'{root}/', '')
-            feature_file = f'{feature_file_root}/{feature_file}'
-
-        command = [
-            'grizzly-cli',
-            self.mode,
-            'run',
-            '--yes',
-            '--verbose',
-            feature_file,
-        ]
+    def execute(self, feature_file: str, env_conf: Optional[Dict[str, Any]] = None, testdata: Optional[Dict[str, str]] = None) -> Tuple[int, List[str]]:
+        env_conf_fd: Any
+        if env_conf is not None:
+            prefix = path.basename(feature_file).replace('.feature', '')
+            env_conf_fd = NamedTemporaryFile(delete=not self.keep_files, prefix=prefix, suffix='.yaml', dir=(self.root / 'environments'))
+        else:
+            env_conf_fd = nullcontext()
+
+        with env_conf_fd as env_conf_file:
+            command = [
+                'grizzly-cli',
+                self.mode,
+                'run',
+                '--yes',
+                '--verbose',
+                feature_file,
+            ]
 
-        if self._distributed:
-            command = command[:2] + ['--project-name', self.root.name] + command[2:]
+            if self._distributed:
+                command = command[:2] + ['--project-name', self.root.name] + command[2:]
 
-        if env_conf_file is not None:
-            command += ['-e', env_conf_file]
+            if env_conf is not None:
+                env_conf_file.write(yaml.dump(env_conf, Dumper=yaml.Dumper).encode())
+                env_conf_file.flush()
+                env_conf_path = str(env_conf_file.name).replace(f'{self.root}/', '')
+                command += ['-e', env_conf_path]
 
-        if testdata is not None:
-            for key, value in testdata.items():
-                command += ['-T', f'{key}={value}']
+            if testdata is not None:
+                for key, value in testdata.items():
+                    command += ['-T', f'{key}={value}']
 
-        rc, output = run_command(
-            command,
-            cwd=str(self.mode_root),
-            env=self._env,
-        )
+            rc, output = run_command(
+                command,
+                cwd=str(self.root),
+                env=self._env,
+            )
 
-        if rc != 0:
-            print(''.join(output))
+            if rc != 0:
+                print(''.join(output))
 
-            for container in ['master', 'worker'] if self._distributed else []:
-                command = ['docker', 'container', 'logs', f'{self.root.name}-{getuser()}_{container}_1']
-                _, output = run_command(
-                    command,
-                    cwd=str(self.mode_root),
-                    env=self._env,
-                )
+                for container in ['master', 'worker'] if self._distributed else []:
+                    command = ['docker', 'container', 'logs', f'{self.root.name}-{getuser()}_{container}_1']
+                    _, output = run_command(
+                        command,
+                        cwd=str(self.root),
+                        env=self._env,
+                    )
 
-                print(''.join(output))
+                    print(''.join(output))
 
-        return rc, output
+            return rc, output
diff --git a/tests/webserver.py b/tests/webserver.py
index ae2c5cc9..ba085f59 100644
--- a/tests/webserver.py
+++ b/tests/webserver.py
@@ -21,11 +21,9 @@
 
 # ugly hack to get correct path when webserver.py is injected for running distributed
 root_dir = (Path(__file__).parent / '..').resolve()
-project_name = 'example'
 
-if '.pytest_tmp' in __file__:
-    root_dir = (root_dir / '..' / '..').resolve()
-    project_name = 'test-example'
+if '/srv/grizzly' not in str(root_dir):
+    root_dir = root_dir / 'example' / 'features'
 
 
 @app.route('/api/v1/resources/dogs')
@@ -46,9 +44,8 @@ def app_get_cat_fact() -> FlaskResponse:
 
 @app.route('/books/<book>.json')
 def app_get_book(book: str) -> FlaskResponse:
-    logger.debug(f'/books/{book}.json called')
-    logger.debug(f'{root_dir=}, {project_name=}')
-    with open(f'{root_dir}/{project_name}/features/requests/books/books.csv', 'r') as fd:
+    logger.debug(f'/books/{book}.json called, {root_dir=}')
+    with open(f'{root_dir}/requests/books/books.csv', 'r') as fd:
         reader = csv.DictReader(fd)
         for row in reader:
             if row['book'] == book: