Repository: StackStorm/st2 (language: python)

Issue:
Python 3.8 to 3.11 mongo 7.0
python 3.12 works.  mongodb 7.0 works.
I also fixed some unit tests.
This merges in @FileMagic  and @nzlosh work on mongo 7.0.

Make unit tests use redis
The graceful_shutdown tests have been very flaky using the `NoOpDriver`. Switching from `NoOpDriver` to `RedisDriver` will hopefully make our CI more stable.

This was discussed in a [TSC meeting](https://github.com/StackStorm/community/issues/141) and implemented by @FileMagic (#6223) and @guzzijones (#6236). I cherry-picked the redis-related parts of their excellent work. After stumbling through the cherry-picks (I missed a few things, and had to rebase/cherry-pick a few times), I refactored a few things:
- GHA: use the `redis` container defined in `services`, dropping the tasks that managed it manually.
- use `ST2TESTS_REDIS_*` as the format for new env vars instead of `ST2_OVERRIDE_COORDINATOR_REDIS_*`.
- fix a regression I added in #6241 that only became apparent once we stopped using NoOpDriver. It's surprising and scary how many of our tests rely on import-time side-effects.
- refactored tests to ensure each test got clean config. Several tests were relying on the config changes previous tests made, making them somewhat brittle. This will hopefully make that better.

:clap: Thank you @FileMagic and @guzzijones for getting this figured out! I have high hopes for more stable CI thanks to your excellent work! :tada:


Discussion / hints:

Fix patch:
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 35187cc7de..13c500d354 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -120,7 +120,7 @@ jobs:
             python-version: '3.8.14'
     services:
       mongo:
-        image: mongo:4.4
+        image: mongo:7.0
         ports:
           - 27017:27017
 
@@ -288,7 +288,7 @@ jobs:
           #  nosetests_node_ index: 0
     services:
       mongo:
-        image: mongo:4.4
+        image: mongo:7.0
         ports:
           - 27017:27017
       redis:
@@ -453,7 +453,7 @@ jobs:
             python-version: '3.9.14'
     services:
       mongo:
-        image: mongo:4.4
+        image: mongo:7.0
         ports:
           - 27017:27017
 
diff --git a/.github/workflows/microbenchmarks.yaml b/.github/workflows/microbenchmarks.yaml
index 667a252a94..2c05fadb74 100644
--- a/.github/workflows/microbenchmarks.yaml
+++ b/.github/workflows/microbenchmarks.yaml
@@ -48,7 +48,7 @@ jobs:
             python-version: '3.9.14'
     services:
       mongo:
-        image: mongo:4.4
+        image: mongo:7.0
         ports:
           - 27017:27017
 
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 9c7667727d..61477d5073 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -6,6 +6,8 @@ in development
 
 Python 3.6 is no longer supported; Stackstorm requires at least Python 3.8.
 
+Newer MongoDB versions are now supported. CI uses MongoDB 7.0.
+
 Several st2.conf database options have been renamed or deprecated. Most of the options will continue to work using their old name.
 However, if you use `[database].ssl_keyfile` and/or `[database].ssl_certfile`, you MUST migrate to `[database].tls_certificate_key_file`.
 This new option expects the key and certificate in the same file. Use something like the following to create that file from your old files:
@@ -54,6 +56,9 @@ Changed
 * Update mongoengine to 0.29 and pymongo to 4.6.3. The pymongo bump (from 3.x to 4.x) is a major update. #6252
   Contributed by @cognifloyd
 
+* Update CI from testing with mongo 4.4 to testing with MongoDB 7.0. #6246
+  Contributed by @guzzijones
+
 Added
 ~~~~~
 * Continue introducing `pants <https://www.pantsbuild.org/docs>`_ to improve DX (Developer Experience)
diff --git a/Makefile b/Makefile
index 653e3bad92..b06a9dac78 100644
--- a/Makefile
+++ b/Makefile
@@ -821,7 +821,7 @@ unit-tests: requirements .unit-tests
 	@echo "==================== tests ===================="
 	@echo
 	@echo "----- Dropping st2-test db -----"
-	@mongo st2-test --eval "db.dropDatabase();"
+	@mongosh st2-test --eval "db.dropDatabase();"
 	@failed=0; \
 	for component in $(COMPONENTS_TEST); do\
 		echo "==========================================================="; \
@@ -847,7 +847,7 @@ endif
 	@echo "==================== unit tests with coverage  ===================="
 	@echo
 	@echo "----- Dropping st2-test db -----"
-	@mongo st2-test --eval "db.dropDatabase();"
+	@mongosh st2-test --eval "db.dropDatabase();"
 	failed=0; \
 	for component in $(COMPONENTS_TEST); do\
 		echo "==========================================================="; \
@@ -908,7 +908,7 @@ itests: requirements .itests
 	@echo "==================== integration tests ===================="
 	@echo
 	@echo "----- Dropping st2-test db -----"
-	@mongo st2-test --eval "db.dropDatabase();"
+	@mongosh st2-test --eval "db.dropDatabase();"
 	@failed=0; \
 	for component in $(COMPONENTS_TEST); do\
 		echo "==========================================================="; \
@@ -932,7 +932,7 @@ endif
 	@echo "================ integration tests with coverage ================"
 	@echo
 	@echo "----- Dropping st2-test db -----"
-	@mongo st2-test --eval "db.dropDatabase();"
+	@mongosh st2-test --eval "db.dropDatabase();"
 	@failed=0; \
 	for component in $(COMPONENTS_TEST); do\
 		echo "==========================================================="; \
@@ -1073,7 +1073,7 @@ runners-tests: requirements .runners-tests
 	@echo "==================== runners-tests ===================="
 	@echo
 	@echo "----- Dropping st2-test db -----"
-	@mongo st2-test --eval "db.dropDatabase();"
+	@mongosh st2-test --eval "db.dropDatabase();"
 	@failed=0; \
 	for component in $(COMPONENTS_RUNNERS); do\
 		echo "==========================================================="; \


Test patch:
diff --git a/.github/workflows/orquesta-integration-tests.yaml b/.github/workflows/orquesta-integration-tests.yaml
index fe3e855fc3..9bde1b2488 100644
--- a/.github/workflows/orquesta-integration-tests.yaml
+++ b/.github/workflows/orquesta-integration-tests.yaml
@@ -69,7 +69,7 @@ jobs:
             python-version: '3.9.14'
     services:
       mongo:
-        image: mongo:4.4
+        image: mongo:7.0
         ports:
           - 27017:27017
 
@@ -82,21 +82,18 @@ jobs:
           - 5672:5672/tcp   # AMQP standard port
           - 15672:15672/tcp # Management: HTTP, CLI
 
-      # Used for the coordination backend for integration tests
-      # NOTE: To speed things up, we only start redis for integration tests
-      # where it's needed
-      # redis:
-      #   # Docker Hub image
-      #   image: redis
-      #   # Set health checks to wait until redis has started
-      #   options: >-
-      #     --name "redis"
-      #     --health-cmd "redis-cli ping"
-      #     --health-interval 10s
-      #     --health-timeout 5s
-      #     --health-retries 5
-      #   ports:
-      #     - 6379:6379/tcp
+      redis:
+        # Docker Hub image
+        image: redis
+        # Set health checks to wait until redis has started
+        options: >-
+          --name "redis"
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 6379:6379/tcp
 
     env:
       TASK: '${{ matrix.task }}'
@@ -147,11 +144,6 @@ jobs:
           cp conf/st2.dev.conf "${ST2_CONF}" ; sed -i -e "s/stanley/${ST2_CI_USER}/" "${ST2_CONF}"
 
           sudo -E ./scripts/ci/add-itest-user-key.sh
-      - name: Run Redis Service Container
-        timeout-minutes: 2
-        run: |
-          docker run --rm --detach -p 127.0.0.1:6379:6379/tcp --name redis redis:latest
-          until [ "$(docker inspect -f {{.State.Running}} redis)" == "true" ]; do sleep 0.1; done
       - name: Permissions Workaround
         run: |
           echo "$ST2_CI_REPO_PATH"
@@ -194,9 +186,6 @@ jobs:
           name: logs-py${{ matrix.python-version }}
           path: logs.tar.gz
           retention-days: 7
-      - name: Stop Redis Service Container
-        if: "${{ always() }}"
-        run: docker rm --force redis || true
 
   slack-notification:
     name: Slack notification for failed master builds
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index 4ec93d2f56..5bfb2b3815 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -41,7 +41,7 @@ jobs:
 
     services:
       mongo:
-        image: mongo:4.4
+        image: mongo:7.0
         ports:
           - 27017:27017