Repository: reanahub/reana-client

Issue:
cli: Implement test command
Add fleshed out implementation of the test command, using the gherkin parser module in reana-commons.

Discussion:
ajclyall: The tests are failing because gherkin-parser doesn't exist in reana-commons yet.

Patch:
diff --git a/AUTHORS.md b/AUTHORS.md
index c8a3060..29c3f24 100644
--- a/AUTHORS.md
+++ b/AUTHORS.md
@@ -3,6 +3,7 @@
 The list of contributors in alphabetical order:
 
 - [Adelina Lintuluoto](https://orcid.org/0000-0002-0726-1452)
+- [Alastair Lyall](https://orcid.org/0009-0000-4955-8935)
 - [Anton Khodak](https://orcid.org/0000-0003-3263-4553)
 - [Audrius Mecionis](https://orcid.org/0000-0002-3759-1663)
 - [Camila Diaz](https://orcid.org/0000-0001-5543-797X)
diff --git a/docs/cmd_list.txt b/docs/cmd_list.txt
index 0b9a7d3..2131cd7 100644
--- a/docs/cmd_list.txt
+++ b/docs/cmd_list.txt
@@ -55,3 +55,6 @@ Secret management commands:
   secrets-add     Add secrets from literal string or from file.
   secrets-delete  Delete user secrets by name.
   secrets-list    List user secrets.
+
+Workflow run test commands:
+  test  Test workflow execution, based on a given Gherkin file.
diff --git a/reana_client/cli/__init__.py b/reana_client/cli/__init__.py
index 2b46cf0..6a940e7 100644
--- a/reana_client/cli/__init__.py
+++ b/reana_client/cli/__init__.py
@@ -11,7 +11,15 @@ import os
 import sys
 
 import click
-from reana_client.cli import files, ping, quotas, retention_rules, secrets, workflow
+from reana_client.cli import (
+    files,
+    ping,
+    quotas,
+    retention_rules,
+    secrets,
+    test,
+    workflow,
+)
 from reana_client.utils import get_api_url
 from urllib3 import disable_warnings
 
@@ -45,6 +53,7 @@ class ReanaCLI(click.Group):
         files.files_group,
         retention_rules.retention_rules_group,
         secrets.secrets_group,
+        test.test_group,
     ]
 
     def __init__(self, name=None, commands=None, **attrs):
diff --git a/reana_client/cli/files.py b/reana_client/cli/files.py
index 50ff6e5..3acc0a8 100644
--- a/reana_client/cli/files.py
+++ b/reana_client/cli/files.py
@@ -326,10 +326,11 @@ def upload_files(  # noqa: C901
                 msg_type="error",
             )
             sys.exit(1)
-
+        filenames = []
+        if reana_spec.get("tests"):
+            for f in reana_spec["tests"].get("files") or []:
+                filenames.append(os.path.join(os.getcwd(), f))
         if reana_spec.get("inputs"):
-            filenames = []
-
             # collect all files in input.files
             for f in reana_spec["inputs"].get("files") or []:
                 # check for directories in files
@@ -340,7 +341,6 @@ def upload_files(  # noqa: C901
                     )
                     sys.exit(1)
                 filenames.append(os.path.join(os.getcwd(), f))
-
             # collect all files in input.directories
             files_from_directories = []
             directories = reana_spec["inputs"].get("directories") or []


Test patch:
diff --git a/reana_client/cli/test.py b/reana_client/cli/test.py
new file mode 100644
index 0000000..e6676a0
--- /dev/null
+++ b/reana_client/cli/test.py
@@ -0,0 +1,169 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of REANA.
+# Copyright (C) 2024 CERN.
+#
+# REANA is free software; you can redistribute it and/or modify it
+# under the terms of the MIT License; see LICENSE file for more details.
+"""REANA client test commands."""
+
+import sys
+import click
+import logging
+import traceback
+import time
+from reana_client.cli.utils import add_access_token_options, check_connection
+from reana_commons.gherkin_parser.parser import (
+    parse_and_run_tests,
+    AnalysisTestStatus,
+)
+from reana_commons.gherkin_parser.data_fetcher import DataFetcherBase
+from reana_commons.gherkin_parser.errors import FeatureFileError
+from reana_client.printer import display_message
+from reana_client.api.client import (
+    list_files,
+    get_workflow_disk_usage,
+    get_workflow_logs,
+    get_workflow_status,
+    get_workflow_specification,
+    download_file,
+)
+from reana_client.cli.utils import add_workflow_option
+
+
+class DataFetcherClient(DataFetcherBase):
+    """Implementation of the DataFetcherBase using reana_client.api.client methods."""
+
+    def __init__(self, access_token):
+        """Initialize DataFetcherClient with access token."""
+        self.access_token = access_token
+
+    def list_files(self, workflow, file_name=None, page=None, size=None, search=None):
+        """Return the list of files for a given workflow workspace."""
+        return list_files(workflow, self.access_token, file_name, page, size, search)
+
+    def get_workflow_disk_usage(self, workflow, parameters):
+        """Display disk usage workflow."""
+        return get_workflow_disk_usage(workflow, parameters, self.access_token)
+
+    def get_workflow_logs(self, workflow, steps=None, page=None, size=None):
+        """Get logs from a workflow engine, use existing API function."""
+        return get_workflow_logs(workflow, self.access_token, steps, page, size)
+
+    def get_workflow_status(self, workflow):
+        """Get of a previously created workflow."""
+        return get_workflow_status(workflow, self.access_token)
+
+    def get_workflow_specification(self, workflow):
+        """Get specification of previously created workflow."""
+        return get_workflow_specification(workflow, self.access_token)
+
+    def download_file(self, workflow, file_path):
+        """Download the requested file if it exists."""
+        return download_file(workflow, file_path, self.access_token)
+
+
+@click.group(help="Workflow run test commands")
+def test_group():
+    """Workflow run test commands."""
+
+
+@test_group.command("test")
+@click.option(
+    "-n",
+    "--test-files",
+    multiple=True,
+    default=None,
+    help="Gherkin file for testing properties of a workflow execution. Overrides files in reana.yaml if provided.",
+)
+@click.pass_context
+@add_access_token_options
+@check_connection
+@add_workflow_option
+def test(ctx, workflow, test_files, access_token):
+    r"""
+    Test workflow execution, based on a given Gherkin file.
+
+    Gherkin files can be specified in the reana specification file (reana.yaml),
+    or by using the ``-n`` option.
+
+    The ``test`` command allows for testing of a workflow execution,
+    by assessing whether it meets certain properties specified in a
+    chosen gherkin file.
+
+    Example:
+        $ reana-client test -w myanalysis -n test_analysis.feature
+        $ reana-client test -w myanalysis
+        $ reana-client test -w myanalysis -n test1.feature -n test2.feature
+    """
+    start_time = time.time()
+    try:
+        workflow_status = get_workflow_status(
+            workflow=workflow, access_token=access_token
+        )
+        status = workflow_status["status"]
+        workflow_name = workflow_status["name"]
+    except Exception as e:
+        logging.debug(traceback.format_exc())
+        logging.debug(str(e))
+        display_message(f"Could not find workflow ``{workflow}``.", msg_type="error")
+        sys.exit(1)
+
+    if status != "finished":
+        display_message(
+            f"``{workflow}`` is {status}. It must be finished to run tests.",
+            msg_type="error",
+        )
+        sys.exit(1)
+
+    if not test_files:
+        reana_specification = get_workflow_specification(workflow, access_token)
+        try:
+            test_files = reana_specification["specification"]["tests"]["files"]
+        except KeyError as e:
+            logging.debug(traceback.format_exc())
+            logging.debug(str(e))
+            display_message(
+                "No test files specified in reana.yaml and no -n option provided.",
+                msg_type="error",
+            )
+            sys.exit(1)
+
+    passed = 0
+    failed = 0
+    data_fetcher = DataFetcherClient(access_token)
+    for test_file in test_files:
+        click.echo("\n", nl=False)
+        display_message(f'Testing file "{test_file}"...', msg_type="info")
+        try:
+            results = parse_and_run_tests(test_file, workflow_name, data_fetcher)
+        except FileNotFoundError as e:
+            logging.debug(traceback.format_exc())
+            logging.debug(str(e))
+            display_message(f"Test file {test_file} not found.", msg_type="error")
+            sys.exit(1)
+        except FeatureFileError as e:
+            logging.debug(traceback.format_exc())
+            logging.debug(str(e))
+            display_message(
+                f"Error parsing feature file {test_file}: {e}", msg_type="error"
+            )
+            sys.exit(1)
+
+        for scenario in results[1]:
+            if scenario.result == AnalysisTestStatus.failed:
+                display_message(
+                    f'Scenario "{scenario.scenario}"', msg_type="error", indented=True
+                )
+                failed += 1
+            else:
+                display_message(
+                    f'Scenario "{scenario.scenario}"', msg_type="success", indented=True
+                )
+                passed += 1
+
+    end_time = time.time()
+    duration = round(end_time - start_time)
+    click.echo(f"\n{passed} passed, {failed} failed in {duration}s")
+    if failed > 0:
+        sys.exit(1)
diff --git a/tests/test_cli_files.py b/tests/test_cli_files.py
index 8e22e8a..fbb182a 100644
--- a/tests/test_cli_files.py
+++ b/tests/test_cli_files.py
@@ -240,6 +240,35 @@ def test_upload_file(create_yaml_workflow_schema):
                 assert message in result.output
 
 
+def test_upload_file_with_test_files_from_spec(
+    get_workflow_specification_with_directory,
+):
+    """Test upload file with test files from the specification, not from the command line."""
+    reana_token = "000000"
+    file = "upload-this-test.feature"
+    env = {"REANA_SERVER_URL": "http://localhost"}
+    runner = CliRunner(env=env)
+
+    with patch(
+        "reana_client.api.client.get_workflow_specification"
+    ) as mock_specification, patch("reana_client.api.client.requests.post"):
+        with runner.isolated_filesystem():
+            with open(file, "w") as f:
+                f.write("Scenario: Test scenario")
+
+            get_workflow_specification_with_directory["specification"]["tests"] = {
+                "files": [file]
+            }
+            mock_specification.return_value = get_workflow_specification_with_directory
+            result = runner.invoke(
+                cli, ["upload", "-t", reana_token, "--workflow", "test-workflow.1"]
+            )
+            assert result.exit_code == 0
+            assert (
+                "upload-this-test.feature was successfully uploaded." in result.output
+            )
+
+
 def test_upload_file_respect_gitignore(
     get_workflow_specification_with_directory,
 ):
diff --git a/tests/test_cli_test.py b/tests/test_cli_test.py
new file mode 100644
index 0000000..cd62351
--- /dev/null
+++ b/tests/test_cli_test.py
@@ -0,0 +1,391 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of REANA.
+# Copyright (C) 2024 CERN.
+#
+# REANA is free software; you can redistribute it and/or modify it
+# under the terms of the MIT License; see LICENSE file for more details.
+
+"""REANA client test tests."""
+
+from click.testing import CliRunner
+from reana_client.cli import cli
+from unittest.mock import patch
+
+from reana_commons.gherkin_parser.parser import AnalysisTestStatus, TestResult
+from reana_commons.gherkin_parser.errors import FeatureFileError
+from dataclasses import replace
+
+passed_test = TestResult(
+    scenario="scenario1",
+    error_log=None,
+    result=AnalysisTestStatus.passed,
+    failed_testcase=None,
+    feature="Run Duration",
+    checked_at="2024-01-01T00:00:00.000000",
+)
+
+failed_test = TestResult(
+    scenario="scenario2",
+    error_log="Test designed to fail",
+    result=AnalysisTestStatus.failed,
+    failed_testcase="Scenario to fail",
+    feature="Run Duration",
+    checked_at="2024-01-01T00:00:00.000000",
+)
+
+
+def test_test_workflow_not_found():
+    """Test test command when workflow is not found."""
+    env = {"REANA_SERVER_URL": "localhost"}
+    runner = CliRunner(env=env)
+    with runner.isolated_filesystem():
+        result = runner.invoke(
+            cli,
+            [
+                "test",
+                "-w",
+                "myanalysis",
+                "-n",
+                "test_analysis.feature",
+                "-t",
+                "000000",
+            ],
+        )
+    assert "Could not find workflow ``myanalysis``." in result.output
+    assert result.exit_code == 1
+
+
+@patch(
+    "reana_client.cli.test.get_workflow_status",
+    return_value={"status": "running", "name": "myanalysis"},
+)
+def test_test_workflow_not_finished(mock_get_workflow_status):
+    """Test test command when workflow is not finished."""
+    env = {"REANA_SERVER_URL": "localhost"}
+    runner = CliRunner(env=env)
+    with runner.isolation():
+        result = runner.invoke(
+            cli,
+            [
+                "test",
+                "-w",
+                "myanalysis",
+                "-n",
+                "test_analysis.feature",
+                "-t",
+                "000000",
+            ],
+        )
+        assert (
+            "``myanalysis`` is running. It must be finished to run tests."
+            in result.output
+        )
+        assert result.exit_code == 1
+
+
+@patch(
+    "reana_client.cli.test.get_workflow_status",
+    return_value={"status": "deleted", "name": "myanalysis"},
+)
+def test_test_workflow_deleted(mock_get_workflow_status):
+    """Test test command when workflow is deleted."""
+    env = {"REANA_SERVER_URL": "localhost"}
+    runner = CliRunner(env=env)
+    with runner.isolation():
+        result = runner.invoke(
+            cli,
+            ["test", "-w", "myanalysis", "-n", "test_analysis.feature", "-t", "000000"],
+        )
+        assert (
+            "``myanalysis`` is deleted. It must be finished to run tests."
+            in result.output
+        )
+        assert result.exit_code == 1
+
+
+@patch(
+    "reana_client.cli.test.get_workflow_status",
+    return_value={"status": "finished", "name": "myanalysis"},
+)
+@patch(
+    "reana_client.cli.test.get_workflow_specification",
+    return_value={"specification": {"inputs": {"directories": ["data"]}}},
+)
+def test_test_no_test_files(mock_get_workflow_status, mock_get_workflow_specification):
+    """Test test command when no test files are specified."""
+    env = {"REANA_SERVER_URL": "localhost"}
+    runner = CliRunner(env=env)
+    with runner.isolation():
+        result = runner.invoke(cli, ["test", "-w", "myanalysis", "-t", "000000"])
+        assert (
+            "No test files specified in reana.yaml and no -n option provided."
+            in result.output
+        )
+        assert result.exit_code == 1
+
+
+@patch(
+    "reana_client.cli.test.get_workflow_status",
+    return_value={"status": "finished", "name": "myanalysis"},
+)
+@patch(
+    "reana_client.cli.test.get_workflow_specification",
+    return_value={"specification": {"inputs": {"directories": ["data"]}}},
+)
+def test_test_no_test_files_with_test_file_option(
+    mock_get_workflow_status, mock_get_workflow_specification
+):
+    """Test test command when no test files are specified in reana.yml and when the test file option is provided."""
+    env = {"REANA_SERVER_URL": "localhost"}
+    runner = CliRunner(env=env)
+    with runner.isolation():
+        result = runner.invoke(
+            cli,
+            [
+                "test",
+                "-w",
+                "myanalysis",
+                "-t",
+                "000000",
+                "-n",
+                "test_analysis.feature",
+            ],
+        )
+        assert 'Testing file "test_analysis.feature"' in result.output
+
+
+@patch(
+    "reana_client.cli.test.get_workflow_status",
+    return_value={"status": "finished", "name": "myanalysis"},
+)
+@patch(
+    "reana_client.cli.test.get_workflow_specification",
+    return_value={
+        "specification": {
+            "tests": {"files": ["test_analysis.feature", "test_analysis2.feature"]}
+        }
+    },
+)
+def test_test_multiple_test_files_with_test_file_option(
+    mock_get_workflow_status, mock_get_workflow_specification
+):
+    """Test test command when multiple test files are specified in reana.yml and test file option is provided.
+    In this case, the test-file option should be used instead of the test files specified in reana.yml.
+    """
+    env = {"REANA_SERVER_URL": "localhost"}
+    runner = CliRunner(env=env)
+    with runner.isolation():
+        result = runner.invoke(
+            cli,
+            [
+                "test",
+                "-w",
+                "myanalysis",
+                "-t",
+                "000000",
+                "-n",
+                "use_this.feature",
+            ],
+        )
+        assert 'Testing file "use_this.feature"' in result.output
+        assert 'Testing file "test_analysis.feature"' not in result.output
+        assert 'Testing file "test_analysis2.feature"' not in result.output
+
+
+@patch(
+    "reana_client.cli.test.get_workflow_status",
+    return_value={"status": "finished", "name": "myanalysis"},
+)
+@patch(
+    "reana_client.cli.test.get_workflow_specification",
+    return_value={
+        "specification": {"tests": {"files": ["use-me.feature", "me-too.feature"]}}
+    },
+)
+@patch(
+    "reana_client.cli.test.parse_and_run_tests",
+    return_value=(
+        "myanalysis",
+        [passed_test],
+    ),
+)
+def test_test_files_from_spec(
+    mock_get_workflow_status, get_workflow_specification, mock_parse_and_run_tests
+):
+    """Test test command when test files are specified in reana.yml."""
+    env = {"REANA_SERVER_URL": "localhost"}
+    runner = CliRunner(env=env)
+    with runner.isolation():
+        result = runner.invoke(
+            cli,
+            ["test", "-w", "myanalysis", "-t", "000000"],
+        )
+        assert 'Testing file "use-me.feature"' in result.output
+        assert 'Testing file "me-too.feature"' in result.output
+
+
+@patch(
+    "reana_client.cli.test.get_workflow_status",
+    return_value={"status": "finished", "name": "myanalysis"},
+)
+@patch(
+    "reana_client.cli.test.parse_and_run_tests",
+    side_effect=FeatureFileError,
+)
+def test_test_parser_error(mock_get_workflow_status, mock_parse_and_run_tests):
+    """Test test command when parser error occurs."""
+    env = {"REANA_SERVER_URL": "localhost"}
+    runner = CliRunner(env=env)
+    with runner.isolation():
+        result = runner.invoke(
+            cli,
+            [
+                "test",
+                "-w",
+                "myanalysis",
+                "-t",
+                "000000",
+                "-n",
+                "test_analysis.feature",
+            ],
+        )
+        assert "Error parsing feature file test_analysis.feature" in result.output
+        assert result.exit_code == 1
+
+
+@patch(
+    "reana_client.cli.test.get_workflow_status",
+    return_value={"status": "finished", "name": "myanalysis"},
+)
+@patch(
+    "reana_client.cli.test.parse_and_run_tests",
+    side_effect=FileNotFoundError,
+)
+def test_test_feature_file_not_found(
+    mock_get_workflow_status, mock_parse_and_run_tests
+):
+    """Test test command when a feature file is not found."""
+    env = {"REANA_SERVER_URL": "localhost"}
+    runner = CliRunner(env=env)
+    with runner.isolation():
+        result = runner.invoke(
+            cli,
+            [
+                "test",
+                "-w",
+                "myanalysis",
+                "-t",
+                "000000",
+                "-n",
+                "test_analysis.feature",
+            ],
+        )
+        assert "Test file test_analysis.feature not found." in result.output
+        assert result.exit_code == 1
+
+
+@patch(
+    "reana_client.cli.test.get_workflow_status",
+    return_value={"status": "finished", "name": "myanalysis"},
+)
+@patch(
+    "reana_client.cli.test.parse_and_run_tests",
+    return_value=(
+        "myanalysis",
+        [passed_test],
+    ),
+)
+def test_test_multiple_test_files(mock_workflow_status, mock_parse_and_run_tests):
+    """Test test command when multiple test files are specified."""
+    env = {"REANA_SERVER_URL": "localhost"}
+    runner = CliRunner(env=env)
+    with runner.isolation():
+        result = runner.invoke(
+            cli,
+            [
+                "test",
+                "-w",
+                "myanalysis",
+                "-t",
+                "000000",
+                "-n",
+                "test_analysis.feature",
+                "-n",
+                "test_analysis2.feature",
+            ],
+        )
+        assert 'Testing file "test_analysis.feature"' in result.output
+        assert 'Testing file "test_analysis2.feature"' in result.output
+
+
+@patch(
+    "reana_client.cli.test.get_workflow_status",
+    return_value={"status": "finished", "name": "myanalysis"},
+)
+@patch(
+    "reana_client.cli.test.parse_and_run_tests",
+    return_value=(
+        "myanalysis",
+        [passed_test, replace(passed_test, scenario="scenario2")],
+    ),
+)
+def test_test_all_scenarios_pass(mock_workflow_status, mock_parse_and_run_tests):
+    """Test test command when tests pass."""
+    env = {"REANA_SERVER_URL": "localhost"}
+    runner = CliRunner(env=env)
+    with runner.isolated_filesystem():
+        result = runner.invoke(
+            cli,
+            ["test", "-w", "myanalysis", "-n", "test_analysis.feature", "-t", "000000"],
+        )
+        assert "SUCCESS" in result.output and "ERROR" not in result.output
+        assert result.exit_code == 0
+
+
+@patch(
+    "reana_client.cli.test.get_workflow_status",
+    return_value={"status": "finished", "name": "myanalysis"},
+)
+@patch(
+    "reana_client.cli.test.parse_and_run_tests",
+    return_value=(
+        "myanalysis",
+        [replace(failed_test, scenario="scenario1"), failed_test],
+    ),
+)
+def test_test_all_scenarios_fail(mock_workflow_status, mock_parse_and_run_tests):
+    """Test test command when tests fail."""
+    env = {"REANA_SERVER_URL": "localhost"}
+    runner = CliRunner(env=env)
+    with runner.isolation():
+        result = runner.invoke(
+            cli,
+            ["test", "-w", "myanalysis", "-n", "test_analysis.feature", "-t", "000000"],
+        )
+        assert "ERROR" in result.output and "SUCCESS" not in result.output
+        assert result.exit_code == 1
+
+
+@patch(
+    "reana_client.cli.test.get_workflow_status",
+    return_value={"status": "finished", "name": "myanalysis"},
+)
+@patch(
+    "reana_client.cli.test.parse_and_run_tests",
+    return_value=(
+        "myanalysis",
+        [passed_test, failed_test],
+    ),
+)
+def test_test_some_scenarios_pass(mock_workflow_status, mock_parse_and_run_tests):
+    """Test test command when some tests pass and some fail."""
+    env = {"REANA_SERVER_URL": "localhost"}
+    runner = CliRunner(env=env)
+    with runner.isolated_filesystem():
+        result = runner.invoke(
+            cli,
+            ["test", "-w", "myanalysis", "-n", "test_analysis.feature", "-t", "000000"],
+        )
+        assert "SUCCESS" in result.output and "ERROR" in result.output
+        assert result.exit_code == 1