Repository: microsoft/Olive (language: python)

Issue:
✂️ Separate docker test case
## Describe your changes
Separate docker test case to avoid hanging on CI pipeline.


## Checklist before requesting a review
- [ ] Add unit tests for this change.
- [ ] Make sure all tests can pass.
- [ ] Update documents if necessary.
- [ ] Format your code by running `pre-commit run --all-files`
- [ ] Is this a user-facing change? If yes, give a description of this change to be included in the release notes.

## (Optional) Issue link



Discussion / hints:

Fix patch:
diff --git a/.azure_pipelines/olive-ci.yaml b/.azure_pipelines/olive-ci.yaml
index 0a9e111c2..f24fa4cb0 100644
--- a/.azure_pipelines/olive-ci.yaml
+++ b/.azure_pipelines/olive-ci.yaml
@@ -51,6 +51,9 @@ jobs:
       bert_ptq_cpu_aml:
         exampleFolder: bert
         exampleName: bert_ptq_cpu_aml
+      bert_ptq_cpu_docker:
+        exampleFolder: bert
+        exampleName: bert_ptq_cpu_docker
 
 - template: job_templates/olive-build-template.yaml
   parameters:


Test patch:
diff --git a/examples/test/test_bert_ptq_cpu.py b/examples/test/test_bert_ptq_cpu.py
index 2482620c9..e9b094742 100644
--- a/examples/test/test_bert_ptq_cpu.py
+++ b/examples/test/test_bert_ptq_cpu.py
@@ -3,7 +3,6 @@
 # Licensed under the MIT License.
 # --------------------------------------------------------------------------
 import os
-import platform
 from pathlib import Path
 
 import pytest
@@ -22,12 +21,9 @@ def setup():
 
 @pytest.mark.parametrize("search_algorithm", ["tpe"])
 @pytest.mark.parametrize("execution_order", ["joint"])
-@pytest.mark.parametrize("system", ["local_system", "aml_system", "docker_system"])
+@pytest.mark.parametrize("system", ["local_system", "aml_system"])
 @pytest.mark.parametrize("olive_json", ["bert_ptq_cpu.json"])
 def test_bert(search_algorithm, execution_order, system, olive_json):
-    # TODO: add gpu e2e test
-    if system == "docker_system" and platform.system() == "Windows":
-        pytest.skip("Skip Linux containers on Windows host test case.")
 
     from olive.workflows import run as olive_run
 
diff --git a/examples/test/test_bert_ptq_cpu_docker.py b/examples/test/test_bert_ptq_cpu_docker.py
new file mode 100644
index 000000000..22a422c59
--- /dev/null
+++ b/examples/test/test_bert_ptq_cpu_docker.py
@@ -0,0 +1,36 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+# --------------------------------------------------------------------------
+import os
+import platform
+from pathlib import Path
+
+import pytest
+from utils import check_search_output, patch_config
+
+
+@pytest.fixture(scope="module", autouse=True)
+def setup():
+    """setup any state specific to the execution of the given module."""
+    cur_dir = Path(__file__).resolve().parent.parent
+    example_dir = cur_dir / "bert"
+    os.chdir(example_dir)
+    yield
+    os.chdir(cur_dir)
+
+
+@pytest.mark.parametrize("search_algorithm", ["tpe"])
+@pytest.mark.parametrize("execution_order", ["joint"])
+@pytest.mark.parametrize("system", ["docker_system"])
+@pytest.mark.parametrize("olive_json", ["bert_ptq_cpu.json"])
+def test_bert(search_algorithm, execution_order, system, olive_json):
+    if system == "docker_system" and platform.system() == "Windows":
+        pytest.skip("Skip Linux containers on Windows host test case.")
+
+    from olive.workflows import run as olive_run
+
+    olive_config = patch_config(olive_json, search_algorithm, execution_order, system)
+
+    footprint = olive_run(olive_config)
+    check_search_output(footprint)
diff --git a/examples/test/utils.py b/examples/test/utils.py
index 7736d7db1..d852f592c 100644
--- a/examples/test/utils.py
+++ b/examples/test/utils.py
@@ -51,6 +51,13 @@ def patch_config(config_json_path: str, search_algorithm: str, execution_order:
         # set docker_system
         set_docker_system(olive_config)
         olive_config["engine"]["target"] = system
+        # reduce agent size for docker system
+
+        # as our docker image is big, we need to reduce the agent size to avoid timeout
+        # for the docker system test, we skip to search for transformers optimization as
+        # it is tested in other olive system tests
+        olive_config["passes"]["transformers_optimization"]["disable_search"] = True
+        olive_config["engine"]["search_strategy"]["search_algorithm_config"]["num_samples"] = 2
 
     return olive_config