Repository: opendatahub-io/opendatahub-tests
Pull request #175: [model server] add deployment mode markers for upgrade tests (#173)
Author: login: rnetser
Labels: name: Verified
description: Verified pr in Jenkins
name: size/m
Created: 2025-03-12T15:20:04Z | Merged: 2025-03-17T18:06:50Z

cherry-pick upgrade markers


Diff:
diff --git a/conftest.py b/conftest.py
index 5f5a96c79..3b0272c3c 100644
--- a/conftest.py
+++ b/conftest.py
@@ -6,7 +6,15 @@
 import shutil
 
 import shortuuid
-from pytest import Parser, Session, FixtureRequest, FixtureDef, Item, Config, CollectReport
+from pytest import (
+    Parser,
+    Session,
+    FixtureRequest,
+    FixtureDef,
+    Item,
+    Config,
+    CollectReport,
+)
 from _pytest.terminal import TerminalReporter
 from typing import Optional, Any
 from pytest_testconfig import config as py_config
@@ -39,18 +47,26 @@ def pytest_addoption(parser: Parser) -> None:
 
     # Buckets options
     buckets_group.addoption(
-        "--ci-s3-bucket-name", default=os.environ.get("CI_S3_BUCKET_NAME"), help="Ci S3 bucket name"
+        "--ci-s3-bucket-name",
+        default=os.environ.get("CI_S3_BUCKET_NAME"),
+        help="Ci S3 bucket name",
     )
     buckets_group.addoption(
-        "--ci-s3-bucket-region", default=os.environ.get("CI_S3_BUCKET_REGION"), help="Ci S3 bucket region"
+        "--ci-s3-bucket-region",
+        default=os.environ.get("CI_S3_BUCKET_REGION"),
+        help="Ci S3 bucket region",
     )
 
     buckets_group.addoption(
-        "--ci-s3-bucket-endpoint", default=os.environ.get("CI_S3_BUCKET_ENDPOINT"), help="Ci S3 bucket endpoint"
+        "--ci-s3-bucket-endpoint",
+        default=os.environ.get("CI_S3_BUCKET_ENDPOINT"),
+        help="Ci S3 bucket endpoint",
     )
 
     buckets_group.addoption(
-        "--models-s3-bucket-name", default=os.environ.get("MODELS_S3_BUCKET_NAME"), help="Models S3 bucket name"
+        "--models-s3-bucket-name",
+        default=os.environ.get("MODELS_S3_BUCKET_NAME"),
+        help="Models S3 bucket name",
     )
     buckets_group.addoption(
         "--models-s3-bucket-region",
@@ -91,6 +107,11 @@ def pytest_addoption(parser: Parser) -> None:
         action="store_true",
         help="Delete pre-upgrade resources; useful when debugging pre-upgrade tests",
     )
+    upgrade_group.addoption(
+        "--upgrade-deployment-modes",
+        help="Coma-separated str; specify inference service deployment modes tests to run in upgrade tests. "
+        "If not set, all will be tested.",
+    )
 
 
 def pytest_cmdline_main(config: Any) -> None:
@@ -102,19 +123,45 @@ def pytest_collection_modifyitems(session: Session, config: Config, items: list[
     Pytest fixture to filter or re-order the items in-place.
 
     Filters upgrade tests based on '--pre-upgrade' / '--post-upgrade' option and marker.
+    If `--upgrade-deployment-modes` option is set, only tests with the specified deployment modes will be added.
     """
+
+    def _add_upgrade_test(_item: Item, _upgrade_deployment_modes: list[str]) -> bool:
+        """
+        Add upgrade test to the list of tests to run.
+
+        Args:
+            _item (Item): The test item.
+            _upgrade_deployment_modes (list[str]): The deployment modes to test.
+
+        Returns:
+            True if the test should be added, False otherwise.
+
+        """
+        if not _upgrade_deployment_modes:
+            return True
+
+        return any([keyword for keyword in _item.keywords if keyword in _upgrade_deployment_modes])
+
     pre_upgrade_tests: list[Item] = []
     post_upgrade_tests: list[Item] = []
     non_upgrade_tests: list[Item] = []
+    upgrade_deployment_modes: list[str] = []
 
     run_pre_upgrade_tests: str | None = config.getoption(name="pre_upgrade")
     run_post_upgrade_tests: str | None = config.getoption(name="post_upgrade")
+    if config_upgrade_deployment_modes := config.getoption(name="upgrade_deployment_modes"):
+        upgrade_deployment_modes = config_upgrade_deployment_modes.split(",")
 
     for item in items:
-        if "pre_upgrade" in item.keywords:
+        if "pre_upgrade" in item.keywords and _add_upgrade_test(
+            _item=item, _upgrade_deployment_modes=upgrade_deployment_modes
+        ):
             pre_upgrade_tests.append(item)
 
-        elif "post_upgrade" in item.keywords:
+        elif "post_upgrade" in item.keywords and _add_upgrade_test(
+            _item=item, _upgrade_deployment_modes=upgrade_deployment_modes
+        ):
             post_upgrade_tests.append(item)
 
         else:
diff --git a/tests/model_serving/model_server/upgrade/README.md b/tests/model_serving/model_server/upgrade/README.md
index 4d1c5199e..8afb28e34 100644
--- a/tests/model_serving/model_server/upgrade/README.md
+++ b/tests/model_serving/model_server/upgrade/README.md
@@ -30,3 +30,13 @@ uv run pytest --post-upgrade
 ```bash
 uv run pytest --pre-upgrade --post-upgrade
 ```
+
+## To run only specific deployment tests, pass --upgrade-deployment-modes with requested mode(s), for example:
+
+```bash
+uv run pytest --pre-upgrade --post-upgrade --upgrade-deployment-modes=servelerss
+```
+
+```bash
+uv run pytest --pre-upgrade --post-upgrade --upgrade-deployment-modes=servelerss,rawdeployment
+```
diff --git a/tests/model_serving/model_server/upgrade/test_upgrade.py b/tests/model_serving/model_server/upgrade/test_upgrade.py
index 6273370c1..74f3a0b64 100644
--- a/tests/model_serving/model_server/upgrade/test_upgrade.py
+++ b/tests/model_serving/model_server/upgrade/test_upgrade.py
@@ -7,8 +7,6 @@
 from utilities.manifests.onnx import ONNX_INFERENCE_CONFIG
 from utilities.manifests.openvino import OPENVINO_INFERENCE_CONFIG
 
-pytestmark = [pytest.mark.serverless, pytest.mark.rawdeployment, pytest.mark.modelmesh]
-
 
 # TODO: add auth, external route and grpc tests
 
@@ -16,6 +14,7 @@
 @pytest.mark.usefixtures("valid_aws_config", "skipped_teardown_resources")
 class TestPreUpgradeModelServer:
     @pytest.mark.pre_upgrade
+    @pytest.mark.serverless
     def test_serverless_onnx_pre_upgrade_inference(self, ovms_serverless_inference_service_scope_session):
         """Verify that kserve Serverless ONNX model can be queried using REST before upgrade"""
         verify_inference_response(
@@ -27,6 +26,7 @@ def test_serverless_onnx_pre_upgrade_inference(self, ovms_serverless_inference_s
         )
 
     @pytest.mark.pre_upgrade
+    @pytest.mark.rawdeployment
     def test_raw_caikit_bge_pre_upgrade_inference(self, caikit_raw_inference_service_scope_session):
         """Test Caikit bge-large-en embedding model inference using internal route before upgrade"""
         verify_inference_response(
@@ -39,6 +39,7 @@ def test_raw_caikit_bge_pre_upgrade_inference(self, caikit_raw_inference_service
         )
 
     @pytest.mark.pre_upgrade
+    @pytest.mark.modelmesh
     def test_model_mesh_openvino_pre_upgrade_inference(self, openvino_model_mesh_inference_service_scope_session):
         """Test OpenVINO ModelMesh inference with internal route before upgrade"""
         verify_inference_response(
@@ -53,6 +54,7 @@ def test_model_mesh_openvino_pre_upgrade_inference(self, openvino_model_mesh_inf
 @pytest.mark.usefixtures("reused_resources")
 class TestPostUpgradeModelServer:
     @pytest.mark.post_upgrade
+    @pytest.mark.serverless
     @pytest.mark.dependency(name="test_serverless_onnx_post_upgrade_inference_service_exists")
     def test_serverless_onnx_post_upgrade_inference_service_exists(
         self, ovms_serverless_inference_service_scope_session
@@ -61,6 +63,7 @@ def test_serverless_onnx_post_upgrade_inference_service_exists(
         assert ovms_serverless_inference_service_scope_session.exists
 
     @pytest.mark.post_upgrade
+    @pytest.mark.serverless
     @pytest.mark.dependency(depends=["test_serverless_onnx_post_upgrade_inference_service_exists"])
     def test_serverless_onnx_post_upgrade_inference(self, ovms_serverless_inference_service_scope_session):
         """Verify that kserve Serverless ONNX model can be queried using REST after upgrade"""
@@ -73,12 +76,14 @@ def test_serverless_onnx_post_upgrade_inference(self, ovms_serverless_inference_
         )
 
     @pytest.mark.post_upgrade
+    @pytest.mark.rawdeployment
     @pytest.mark.dependency(name="test_raw_caikit_bge_post_upgrade_inference_exists")
     def test_raw_caikit_bge_post_upgrade_inference_exists(self, caikit_raw_inference_service_scope_session):
         """Test that raw deployment inference service exists after upgrade"""
         assert caikit_raw_inference_service_scope_session.exists
 
     @pytest.mark.post_upgrade
+    @pytest.mark.rawdeployment
     @pytest.mark.dependency(depends=["test_raw_caikit_bge_post_upgrade_inference_exists"])
     def test_raw_caikit_bge_post_upgrade_inference(self, caikit_raw_inference_service_scope_session):
         """Test Caikit bge-large-en embedding model inference using internal route after upgrade"""
@@ -92,6 +97,7 @@ def test_raw_caikit_bge_post_upgrade_inference(self, caikit_raw_inference_servic
         )
 
     @pytest.mark.post_upgrade
+    @pytest.mark.modelmesh
     @pytest.mark.dependency(name="test_model_mesh_openvino_post_upgrade_inference_exists")
     def test_model_mesh_openvino_post_upgrade_inference_exists(
         self, openvino_model_mesh_inference_service_scope_session
@@ -100,6 +106,7 @@ def test_model_mesh_openvino_post_upgrade_inference_exists(
         assert openvino_model_mesh_inference_service_scope_session.exists
 
     @pytest.mark.post_upgrade
+    @pytest.mark.modelmesh
     @pytest.mark.dependency(depends=["test_model_mesh_openvino_post_upgrade_inference_exists"])
     def test_model_mesh_openvino_post_upgrade_inference(self, openvino_model_mesh_inference_service_scope_session):
         """Test OpenVINO ModelMesh inference with internal route after upgrade"""


Reviews:
user: login: lugi0
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T11:22:20Z
user: login: adolfo-ab
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T13:17:09Z
user: login: dbasunag
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-03-13T19:36:04Z
user: login: rnetser
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-17T18:03:07Z
user: login: rnetser
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-17T18:04:41Z

Review comments:
diff_hunk: @@ -91,6 +107,11 @@ def pytest_addoption(parser: Parser) -> None:
         action="store_true",
         help="Delete pre-upgrade resources; useful when debugging pre-upgrade tests",
     )
+    upgrade_group.addoption(
+        "--upgrade-deployment-modes",
+        help="Coma-separated str; specify inference service deployment modes tests to run in upgrade tests. "
path: conftest.py
user: login: dbasunag
body: You have a typo here. Since this is a cherry-pick, wanted to draw your attention to it.
created_at: 2025-03-13T19:34:08Z
author_association: COLLABORATOR
diff_hunk: @@ -102,19 +123,45 @@ def pytest_collection_modifyitems(session: Session, config: Config, items: list[
     Pytest fixture to filter or re-order the items in-place.
 
     Filters upgrade tests based on '--pre-upgrade' / '--post-upgrade' option and marker.
+    If `--upgrade-deployment-modes` option is set, only tests with the specified deployment modes will be added.
     """
+
+    def _add_upgrade_test(_item: Item, _upgrade_deployment_modes: list[str]) -> bool:
+        """
+        Add upgrade test to the list of tests to run.
+
+        Args:
+            _item (Item): The test item.
+            _upgrade_deployment_modes (list[str]): The deployment modes to test.
+
+        Returns:
+            True if the test should be added, False otherwise.
+
+        """
+        if not _upgrade_deployment_modes:
+            return True
path: conftest.py
user: login: dbasunag
body: Just drawing attention only, as it is a cherry-pick:
```
if _upgrade_deployment_modes:
      return False
```
is easy to read.
created_at: 2025-03-13T19:35:15Z
author_association: COLLABORATOR
diff_hunk: @@ -91,6 +107,11 @@ def pytest_addoption(parser: Parser) -> None:
         action="store_true",
         help="Delete pre-upgrade resources; useful when debugging pre-upgrade tests",
     )
+    upgrade_group.addoption(
+        "--upgrade-deployment-modes",
+        help="Coma-separated str; specify inference service deployment modes tests to run in upgrade tests. "
path: conftest.py
user: login: rnetser
body: will fix in another pr
created_at: 2025-03-17T18:03:07Z
author_association: CONTRIBUTOR
diff_hunk: @@ -102,19 +123,45 @@ def pytest_collection_modifyitems(session: Session, config: Config, items: list[
     Pytest fixture to filter or re-order the items in-place.
 
     Filters upgrade tests based on '--pre-upgrade' / '--post-upgrade' option and marker.
+    If `--upgrade-deployment-modes` option is set, only tests with the specified deployment modes will be added.
     """
+
+    def _add_upgrade_test(_item: Item, _upgrade_deployment_modes: list[str]) -> bool:
+        """
+        Add upgrade test to the list of tests to run.
+
+        Args:
+            _item (Item): The test item.
+            _upgrade_deployment_modes (list[str]): The deployment modes to test.
+
+        Returns:
+            True if the test should be added, False otherwise.
+
+        """
+        if not _upgrade_deployment_modes:
+            return True
path: conftest.py
user: login: rnetser
body: `if _upgrade_deployment_modes` -> need to return `any([keyword for keyword in _item.keywords if keyword in _upgrade_deployment_modes])`
created_at: 2025-03-17T18:04:41Z
author_association: CONTRIBUTOR

PR comments:
user: login: github-actions[bot]
created_at: 2025-03-12T15:20:22Z
body: 
The following are automatically added/executed:
 * PR size label.
 * Run [pre-commit](https://pre-commit.ci/)
 * Run [tox](https://tox.wiki/)

Available user actions:
 * To mark a PR as `WIP`, add `/wip` in a comment. To remove it from the PR comment `/wip cancel` to the PR.
 * To block merging of a PR, add `/hold` in a comment. To un-block merging of PR comment `/hold cancel`.
 * To mark a PR as approved, add `/lgtm` in a comment. To remove, add `/lgtm cancel`.
        `lgtm` label removed on each new commit push.
 * To mark PR as verified comment `/verified` to the PR, to un-verify comment `/verified cancel` to the PR.
        `verified` label removed on each new commit push.

<details>
<summary>Supported labels</summary>

{'/hold', '/lgtm', '/wip', '/verified'}
</details>
    
author_association: NONE
user: login: rnetser
created_at: 2025-03-12T15:20:27Z
body: /verified
author_association: CONTRIBUTOR