Repository: StackStorm/st2 (language: python)

Issue:
pants ci: add integration tests jobs
This adds 2 GHA jobs to the pants+pytest CI workflow:
- `integration-tests`
- `integration-st2cluster-tests`

Sadly, GHA is very repetitive, so a lot of the steps for these jobs are just copies of steps in other jobs. So, this should be fairly easy to review despite the line count.  Go through each commit to see why I made a few changes to some steps.

Currently, we run Makefile-based integration tests in CI and Orquesta CI workflows. For the pants+pytest CI, I opted to do 2 jobs instead of separate workflows. Also, I found that the only integration tests that require a running st2cluster (started with `tools/launchdev.sh`) are the orquesta tests. Since st2cluster is not required by most integration tests, I was able to simplify the setup and avoid starting it. This also means there will be far fewer logs to go through when looking up failures about one of these integration tests. So, I used that requirement (needs a running st2cluster) as the dividing line between the 2 sets of integration tests. By using "st2cluster" instead of "orquesta" in the job name, I created an (hopefully) obvious home for future non-orquesta tests that need to use st2cluster.

These PRs were split out of this PR to keep each PR focused and easier to review: 
- #6275
- #6276
- #6277
- #6279
- #6278
- #6282
- #6283


Discussion / hints:

Fix patch:
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index abd62f9cc8..613f01a3a8 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -69,7 +69,7 @@ Added
   working on StackStorm, improve our security posture, and improve CI reliability thanks in part
   to pants' use of PEX lockfiles. This is not a user-facing addition.
   #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 #6240 #6241 #6244 #6251 #6253
-  #6254 #6258 #6259 #6260 #6269
+  #6254 #6258 #6259 #6260 #6269 #6275
   Contributed by @cognifloyd
 * Build of ST2 EL9 packages #6153
   Contributed by @amanda11
diff --git a/pants.ci.toml b/pants.ci.toml
index 428d3b8de3..158b7b3c8a 100644
--- a/pants.ci.toml
+++ b/pants.ci.toml
@@ -11,6 +11,13 @@ remote_provider = "experimental-github-actions-cache"
 remote_cache_read = true
 remote_cache_write = true
 
+# https://www.pantsbuild.org/stable/reference/global-options#ignore_warnings
+ignore_warnings = [
+    # remote cache errors caused by GitHub rate-limits are not helpful
+    "Failed to read from remote cache",
+    "Failed to write to remote cache",
+]
+
 [stats]
 # "print metrics of your cache's performance at the end of the run,
 # including the number of cache hits and the total time saved thanks
diff --git a/pants.toml b/pants.toml
index 72e5fcb6c8..6dc6bba87f 100644
--- a/pants.toml
+++ b/pants.toml
@@ -244,6 +244,8 @@ extra_env_vars = [
   "ST2TESTS_REDIS_HOST",
   "ST2TESTS_REDIS_PORT",
 ]
+# 10 min should be more than enough even for integration tests.
+timeout_default = 600 # seconds
 
 [twine]
 install_from_resolve = "twine"


Test patch:
diff --git a/contrib/runners/local_runner/tests/integration/test_localrunner.py b/contrib/runners/local_runner/tests/integration/test_localrunner.py
index 1ca4f4275f..f165a72ff3 100644
--- a/contrib/runners/local_runner/tests/integration/test_localrunner.py
+++ b/contrib/runners/local_runner/tests/integration/test_localrunner.py
@@ -15,6 +15,7 @@
 
 from __future__ import absolute_import
 import os
+import unittest
 import uuid
 
 import mock
@@ -45,6 +46,8 @@
 
 __all__ = ["LocalShellCommandRunnerTestCase", "LocalShellScriptRunnerTestCase"]
 
+ST2_CI = os.environ.get("ST2_CI", "false").lower() == "true"
+
 MOCK_EXECUTION = mock.Mock()
 MOCK_EXECUTION.id = "598dbf0c0640fd54bffc688b"
 
@@ -94,6 +97,11 @@ def test_shell_command_action_basic(self):
         self.assertEqual(output_dbs[0].output_type, "stdout")
         self.assertEqual(output_dbs[0].data, "10\n")
 
+    # This test depends on passwordless sudo. Don't require that for local development.
+    @unittest.skipIf(
+        not ST2_CI,
+        'Skipping tests because ST2_CI environment variable is not set to "true"',
+    )
     def test_timeout(self):
         models = self.fixtures_loader.load_models(
             fixtures_pack=GENERIC_PACK, fixtures_dict={"actions": ["local.yaml"]}
@@ -141,8 +149,13 @@ def test_common_st2_env_vars_are_available_to_the_action(self):
         self.assertEqual(status, action_constants.LIVEACTION_STATUS_SUCCEEDED)
         self.assertEqual(result["stdout"].strip(), "mock-token")
 
+    # This test depends on passwordless sudo. Don't require that for local development.
+    @unittest.skipIf(
+        not ST2_CI,
+        'Skipping tests because ST2_CI environment variable is not set to "true"',
+    )
     def test_sudo_and_env_variable_preservation(self):
-        # Verify that the environment environment are correctly preserved when running as a
+        # Verify that the environment vars are correctly preserved when running as a
         # root / non-system user
         # Note: This test will fail if SETENV option is not present in the sudoers file
         models = self.fixtures_loader.load_models(
@@ -297,6 +310,11 @@ def test_action_stdout_and_stderr_is_stored_in_the_db_short_running_action(
             self.assertEqual(output_dbs[db_index_1].data, mock_stderr[0])
             self.assertEqual(output_dbs[db_index_2].data, mock_stderr[1])
 
+    # This test depends on passwordless sudo. Don't require that for local development.
+    @unittest.skipIf(
+        not ST2_CI,
+        'Skipping tests because ST2_CI environment variable is not set to "true"',
+    )
     def test_shell_command_sudo_password_is_passed_to_sudo_binary(self):
         # Verify that sudo password is correctly passed to sudo binary via stdin
         models = self.fixtures_loader.load_models(
diff --git a/contrib/runners/orquesta_runner/tests/integration/BUILD b/contrib/runners/orquesta_runner/tests/integration/BUILD
index 54651017ff..8d33e316f4 100644
--- a/contrib/runners/orquesta_runner/tests/integration/BUILD
+++ b/contrib/runners/orquesta_runner/tests/integration/BUILD
@@ -5,5 +5,5 @@ __defaults__(
 
 python_tests(
     name="tests",
-    uses=["redis"],
+    uses=["mongo", "rabbitmq", "redis", "system_user"],
 )
diff --git a/st2actions/tests/integration/BUILD b/st2actions/tests/integration/BUILD
index 2d782aaea0..51c5b56d14 100644
--- a/st2actions/tests/integration/BUILD
+++ b/st2actions/tests/integration/BUILD
@@ -5,4 +5,8 @@ __defaults__(
 
 python_tests(
     name="tests",
+    uses=["rabbitmq"],
+    stevedore_namespaces=[
+        "st2common.metrics.driver",
+    ],
 )
diff --git a/st2api/tests/integration/BUILD b/st2api/tests/integration/BUILD
index e1fda4d067..5302bad1b6 100644
--- a/st2api/tests/integration/BUILD
+++ b/st2api/tests/integration/BUILD
@@ -7,5 +7,13 @@ python_tests(
     name="tests",
     dependencies=[
         "conf/st2.tests.conf:st2_tests_conf",
+        "st2api/st2api/wsgi.py",
+        "st2auth/st2auth/wsgi.py",
     ],
+    stevedore_namespaces=[
+        "st2auth.sso.backends",
+        "st2common.metrics.driver",
+        "st2common.rbac.backend",
+    ],
+    uses=["mongo", "rabbitmq", "redis"],
 )
diff --git a/st2common/tests/fixtures/BUILD b/st2common/tests/fixtures/BUILD
index 3ba6abf422..01dbde214a 100644
--- a/st2common/tests/fixtures/BUILD
+++ b/st2common/tests/fixtures/BUILD
@@ -1,5 +1,6 @@
 python_sources()
 
-shell_sources(
+st2_shell_sources_and_resources(
     name="shell",
+    sources=["*.sh"],
 )
diff --git a/st2common/tests/integration/BUILD b/st2common/tests/integration/BUILD
index 6587940ceb..275336dae7 100644
--- a/st2common/tests/integration/BUILD
+++ b/st2common/tests/integration/BUILD
@@ -5,19 +5,46 @@ __defaults__(
 
 python_tests(
     name="tests",
-    dependencies=[
-        # used by test_register_content_script
-        "conf/st2.tests.conf:st2_tests_conf",
-        "conf/st2.tests1.conf:st2_tests_conf",
-    ],
     stevedore_namespaces=[
         "orquesta.expressions.functions",
         "st2common.runners.runner",
         "st2common.rbac.backend",
         "st2common.metrics.driver",
     ],
+    uses=["mongo", "rabbitmq", "redis"],
+    overrides={
+        "test_logging.py": dict(
+            dependencies=[
+                "./log_unicode_data.py",
+            ],
+        ),
+        "test_register_content_script.py": dict(
+            dependencies=[
+                "conf/st2.tests.conf:st2_tests_conf",
+                "conf/st2.tests1.conf:st2_tests_conf",
+                "st2common/bin/st2-register-content",
+            ],
+        ),
+        "test_service_setup_log_level_filtering.py": dict(
+            dependencies=[
+                "st2api/bin/st2api",
+            ],
+        ),
+        "test_util_green.py": dict(
+            dependencies=[
+                "st2common/tests/fixtures/print_to_stdout_stderr_sleep.sh:shell_resources",
+            ],
+        ),
+    },
 )
 
 python_test_utils(
     sources=["*.py", "!test_*.py"],
+    overrides={
+        "log_unicode_data.py": dict(
+            dependencies=[
+                "st2tests/st2tests/fixtures/conf:st2.tests.conf",
+            ],
+        ),
+    },
 )
diff --git a/st2reactor/tests/integration/BUILD b/st2reactor/tests/integration/BUILD
index 3fea4cca37..147337ac47 100644
--- a/st2reactor/tests/integration/BUILD
+++ b/st2reactor/tests/integration/BUILD
@@ -3,10 +3,50 @@ __defaults__(
     extend=True,
 )
 
+_conf_deps = [
+    "conf/st2.tests.conf:st2_tests_conf",
+    "conf/st2.tests2.conf:st2_tests_conf",
+]
+
 python_tests(
     name="tests",
-    dependencies=[
-        "conf/st2.tests.conf:st2_tests_conf",
-        "conf/st2.tests2.conf:st2_tests_conf",
-    ],
+    dependencies=_conf_deps,
+    uses=["mongo", "rabbitmq", "redis"],
+    overrides={
+        "test_garbage_collector.py": dict(
+            dependencies=[
+                *_conf_deps,
+                "st2reactor/bin/st2garbagecollector",
+            ],
+            stevedore_namespaces=[
+                "st2common.metrics.driver",
+            ],
+            entry_point_dependencies={
+                "contrib/runners/inquirer_runner": ["st2common.runners.runner"],
+            },
+        ),
+        "test_rules_engine.py": dict(
+            dependencies=[
+                *_conf_deps,
+                "st2reactor/bin/st2timersengine",
+            ],
+            stevedore_namespaces=[
+                "st2common.metrics.driver",
+            ],
+        ),
+        "test_sensor_container.py": dict(
+            dependencies=[
+                *_conf_deps,
+                "st2reactor/bin/st2sensorcontainer",
+                "contrib/examples/sensors",
+                "contrib/examples:metadata",
+            ],
+            stevedore_namespaces=[
+                "st2common.metrics.driver",
+            ],
+        ),
+        "test_sensor_watcher.py": dict(
+            uses=["rabbitmq"],
+        ),
+    },
 )
diff --git a/st2tests/integration/orquesta/BUILD b/st2tests/integration/orquesta/BUILD
index 5b6c97ec1f..8073d3f7a5 100644
--- a/st2tests/integration/orquesta/BUILD
+++ b/st2tests/integration/orquesta/BUILD
@@ -1,6 +1,6 @@
 python_tests(
     name="tests",
-    uses=["redis"],
+    uses=["mongo", "rabbitmq", "redis", "system_user"],
 )
 
 python_test_utils(