Repository: Biometria-se/grizzly (language: python)

Issue:
variables only used in template statements is not detected
**Describe the bug**
testdata variables only used in template statements (`{% .. %}`) is not detected by `grizzly.testdata.utils.initialize_testdata` (or `get_template_variables`), which means that the testdata producer will not include them in the served testdata.

**To Reproduce**
Steps to reproduce the behavior:
...

**Expected behavior**
Also parse and find variables used in statements, without having to workaround it by first using them in a expression (`{{ .. }}`).




Discussion / hints:

Fix patch:
diff --git a/.gitignore b/.gitignore
index 52ecde75..7e09fd4b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
 **/__pycache__
 **/.mypy_cache
 **/.pytest_cache
+**/.ruff_cache
 *.egg-info/
 build/
 dist/


Test patch:
diff --git a/grizzly/testdata/ast.py b/grizzly/testdata/ast.py
index 1e7fdd20..9510125f 100644
--- a/grizzly/testdata/ast.py
+++ b/grizzly/testdata/ast.py
@@ -5,7 +5,6 @@
 from typing import TYPE_CHECKING, Dict, Generator, List, Optional, Set
 
 from jinja2 import Environment as Jinja2Environment
-from jinja2 import FileSystemLoader as Jinja2FileSystemLoader
 from jinja2 import nodes as j2
 
 if TYPE_CHECKING:  # pragma: no cover
@@ -31,7 +30,7 @@ def get_template_variables(grizzly: GrizzlyContext) -> Dict[str, Set[str]]:
         if len(templates[scenario]) == 0:
             del templates[scenario]
 
-    return _parse_templates(templates)
+    return _parse_templates(templates, env=grizzly.state.jinja2)
 
 
 def walk_attr(node: j2.Getattr) -> List[str]:
@@ -60,7 +59,7 @@ def _walk_attr(parent: j2.Getattr) -> List[str]:
     return attributes
 
 
-def _parse_templates(templates: Dict[GrizzlyContextScenario, Set[str]]) -> Dict[str, Set[str]]:  # noqa: C901, PLR0915
+def _parse_templates(templates: Dict[GrizzlyContextScenario, Set[str]], *, env: Jinja2Environment) -> Dict[str, Set[str]]:  # noqa: C901, PLR0915
     variables: Dict[str, Set[str]] = {}
 
     def _getattr(node: j2.Node) -> Generator[List[str], None, None]:  # noqa: C901, PLR0912, PLR0915
@@ -140,19 +139,22 @@ def _getattr(node: j2.Node) -> Generator[List[str], None, None]:  # noqa: C901,
 
         # can raise TemplateError which should be handled else where
         for template in template_sources:
-            j2env = Jinja2Environment(
-                autoescape=False,
-                loader=Jinja2FileSystemLoader('.'),
-            )
-
             # json.dumps escapes quote (") causing it to be \\", which inturn causes problems for jinja
             template_normalized = template.replace('\\"', "'")
 
-            parsed = j2env.parse(template_normalized)
+            parsed = env.parse(template_normalized)
 
             for body in getattr(parsed, 'body', []):
-                for node in getattr(body, 'nodes', []):
+                if isinstance(body, j2.Assign):  #  {% .. %} expressions
+                    node = getattr(body, 'node', None)
+                    if node is None:
+                        continue
+
                     for attributes in _getattr(node):
                         variables[scenario_name].add('.'.join(attributes))
+                else:
+                    for node in getattr(body, 'nodes', []):
+                        for attributes in _getattr(node):
+                            variables[scenario_name].add('.'.join(attributes))
 
     return variables
diff --git a/tests/unit/test_grizzly/testdata/test_ast.py b/tests/unit/test_grizzly/testdata/test_ast.py
index 112bf9be..7c9c0037 100644
--- a/tests/unit/test_grizzly/testdata/test_ast.py
+++ b/tests/unit/test_grizzly/testdata/test_ast.py
@@ -35,7 +35,7 @@ def test__parse_template(request_task: RequestTaskFixture) -> None:
     scenario.tasks.add(request)
 
     templates = {scenario: set(request.get_templates())}
-    variables = _parse_templates(templates)
+    variables = _parse_templates(templates, env=request_task.behave_fixture.grizzly.state.jinja2)
 
     assert variables == {
         'TestScenario_001': {
@@ -67,7 +67,7 @@ def test__parse_template_nested_pipe(request_task: RequestTaskFixture) -> None:
 
     templates = {scenario: set(request.get_templates())}
 
-    variables = _parse_templates(templates)
+    variables = _parse_templates(templates, env=request_task.behave_fixture.grizzly.state.jinja2)
 
     assert variables == {
         'TestScenario_001': {
@@ -83,7 +83,7 @@ def test__parse_template_nested_pipe(request_task: RequestTaskFixture) -> None:
 
     templates = {scenario: set(request.get_templates())}
 
-    variables = _parse_templates(templates)
+    variables = _parse_templates(templates, env=request_task.behave_fixture.grizzly.state.jinja2)
 
     assert variables == {
         'TestScenario_001': {
@@ -98,7 +98,7 @@ def test__parse_template_nested_pipe(request_task: RequestTaskFixture) -> None:
 
     templates = {scenario: set(request.get_templates())}
 
-    variables = _parse_templates(templates)
+    variables = _parse_templates(templates, env=request_task.behave_fixture.grizzly.state.jinja2)
 
     assert variables == {
         'TestScenario_001': {
@@ -131,7 +131,7 @@ def test__parse_template_nested_pipe(request_task: RequestTaskFixture) -> None:
     scenario.tasks.add(request)
     templates = {scenario: set(request.get_templates())}
 
-    variables = _parse_templates(templates)
+    variables = _parse_templates(templates, env=request_task.behave_fixture.grizzly.state.jinja2)
 
     assert variables == {
         'TestScenario_001': {
@@ -149,6 +149,79 @@ def test__parse_template_nested_pipe(request_task: RequestTaskFixture) -> None:
         },
     }
 
+    request.source = '{%- set hello = world -%} {{ foobar }}'
+    scenario.tasks.clear()
+    scenario.tasks.add(request)
+    templates = {scenario: set(request.get_templates())}
+
+    variables = _parse_templates(templates, env=request_task.behave_fixture.grizzly.state.jinja2)
+
+    assert variables == {
+        'TestScenario_001': {
+            'foobar',
+            'world',
+        },
+    }
+
+    request.source = """
+{%- set t1l = AtomicCsvReader.input.value1 -%}
+{%- set first_parts = [] -%}
+{%- for p in t1l.split(';') -%}
+	{%- set parts = p.split('|') -%}
+	{%- set _ = first_parts.append({'p': parts[0], 'c': parts[1]}) -%}
+{%- endfor -%}
+{%- set t2l = AtomicCsvReader.input.value2 -%}
+{%- set second_parts = [] -%}
+{%- for p in t2l.split(';') -%}
+	{%- set parts = p.split('|') -%}
+	{%- set _ = second_parts.append({'p': parts[0], 'c': parts[1]}) -%}
+{%- endfor -%}
+{
+    "id": ["{{ id }}"],
+    "list": [{
+            "subId": "{{ first_subid }}",
+            "isIncluded": true,
+            "concurrencyTag": {{ first_concurrencytag | tojson }},
+            "length": {{ AtomicCsvReader.input.first_length | int }},
+            "width": {{ AtomicCsvReader.input.first_width | int }},
+            "height": {{ AtomicCsvReader.input.first_height | int }},
+            "volume": {{ ((AtomicCsvReader.input.first_width | int) / 100) * ((AtomicCsvReader.input.first_height | int) / 100) * ((AtomicCsvReader.input.first_length | int) / 100) }},
+        }, {
+            "subId": "{{ second_subid }}",
+            "isIncluded": true,
+            "concurrencyTag": {{ second_concurrencytag | tojson }},
+            "length": {{ AtomicCsvReader.input.second_length | int }},
+            "width": {{ AtomicCsvReader.input.second_width | int }},
+            "height": {{ AtomicCsvReader.input.second_height | int }},
+            "volume": {{ ((AtomicCsvReader.input.second_width | int) / 100) * ((AtomicCsvReader.input.second_height | int) / 100) * ((AtomicCsvReader.input.second_length | int) / 100) }},
+        }
+    ]
+}
+"""  # noqa: E501
+    scenario.tasks.clear()
+    scenario.tasks.add(request)
+    templates = {scenario: set(request.get_templates())}
+
+    variables = _parse_templates(templates, env=request_task.behave_fixture.grizzly.state.jinja2)
+
+    assert variables == {
+        'TestScenario_001': {
+            'AtomicCsvReader.input.value1',
+            'AtomicCsvReader.input.value2',
+            'id',
+            'first_subid',
+            'first_concurrencytag',
+            'AtomicCsvReader.input.first_length',
+            'AtomicCsvReader.input.first_width',
+            'AtomicCsvReader.input.first_height',
+            'second_subid',
+            'second_concurrencytag',
+            'AtomicCsvReader.input.second_length',
+            'AtomicCsvReader.input.second_width',
+            'AtomicCsvReader.input.second_height',
+        },
+    }
+
 
 def test_get_template_variables(behave_fixture: BehaveFixture) -> None:
     behave = behave_fixture.context
@@ -170,7 +243,7 @@ def test_get_template_variables(behave_fixture: BehaveFixture) -> None:
         RequestTask(RequestMethod.GET, name='{{ env }} GET request', endpoint='/api/{{ env }}/get'),
     )
     task = cast(RequestTask, grizzly.scenario.tasks()[-1])
-    task.source = '{{ AtomicIntegerIncrementer.test }}'
+    task.source = '{{ AtomicIntegerIncrementer.test }} {%- set hello = world -%}'
 
     grizzly.scenario.tasks.add(
         LogMessageTask(message='{{ foo }}'),
@@ -190,5 +263,6 @@ def test_get_template_variables(behave_fixture: BehaveFixture) -> None:
             'foo',
             'env',
             'foobar',
+            'world',
         },
     }