Repository: risingwavelabs/risingwave (language: python)

Issue:
feat(risedev): support configuring meta store by url from env var
I hereby agree to the terms of the [RisingWave Labs, Inc. Contributor License Agreement](https://gist.github.com/TennyZhuang/f00be7f16996ea48effb049aa7be4d66#file-rw_cla).

## What's changed and what's your intention?

See ideas in #19527.

When specifying `meta-backend: env` for the meta-node service in `risedev dev`, RiseDev will now pass it verbatim as `--sql-endpoint` to the meta node.

Additionally, if the cluster is started with `ci-start`, RiseDev will help to recreate the database that will be used.

## Checklist

- [x] I have written necessary rustdoc comments
- [x] I have added necessary unit tests and integration tests
- [x] All checks passed in `./risedev check` (or alias, `./risedev c`)

## Documentation

- [ ] My PR needs documentation updates. (Please use the **Release note** section below to summarize the impact on users)

## Release note

If this PR includes changes that directly affect users or other significant modifications relevant to the community, kindly draft a release note to provide a concise summary of these changes. Please prioritize highlighting the impact these changes will have on users.


<!--
Please create a release note for your changes.

Discuss technical details in the "What's changed" section, and
focus on the impact on users in the release note.

You should also mention the environment or conditions where the impact may occur.
-->

</details>



Discussion / hints:
yaml not updated? :eyes:
> yaml not updated? 👀

Won't be in this PR. 🥰

Fix patch:
diff --git a/ci/workflows/main-cron.yml b/ci/workflows/main-cron.yml
index 9a1cf0b3b5b9..3a371afa4641 100644
--- a/ci/workflows/main-cron.yml
+++ b/ci/workflows/main-cron.yml
@@ -4,6 +4,37 @@ auto-retry: &auto-retry
     - signal_reason: agent_stop
       limit: 3
 
+sql-backend: &sql-backend
+  matrix:
+    setup:
+      backend: [""]
+      endpoint: [""]
+    adjustments:
+      - with:
+          backend: ""
+          endpoint: ""
+        skip: true # hack
+      - with:
+          backend: "sqlite"
+          # sqlite3 /tmp/rwmeta.db
+          endpoint: "sqlite:///tmp/rwmeta.db?mode=rwc"
+      - with:
+          backend: "postgres"
+          # PGPASSWORD=postgres psql -h db -p 5432 -U postgres -d rwmeta
+          endpoint: "postgres://postgres:postgres@db:5432/rwmeta"
+      - with:
+          backend: "mysql"
+          # mysql -h mysql -P 3306 -u root -p123456 -D rwmeta
+          endpoint: "mysql://root:123456@mysql:3306/rwmeta"
+  env:
+    RISEDEV_SQL_ENDPOINT: "{{matrix.endpoint}}"
+
+docker-compose-common: &docker-compose-common
+    config: ci/docker-compose.yml
+    mount-buildkite-agent: true
+    propagate-environment: true
+    run: rw-build-env
+
 steps:
   - label: "build"
     command: "ci/scripts/build.sh -p ci-release"
@@ -62,8 +93,9 @@ steps:
     timeout_in_minutes: 10
     retry: *auto-retry
 
-  - label: "end-to-end test (release)"
+  - label: "end-to-end test (release, {{matrix.backend}} backend)"
     key: "e2e-test-release"
+    <<: *sql-backend
     command: "ci/scripts/cron-e2e-test.sh -p ci-release -m ci-3streaming-2serving-3fe"
     if: |
       !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
@@ -75,11 +107,10 @@ steps:
       - "docslt"
     plugins:
       - docker-compose#v5.1.0:
-          run: rw-build-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
+          <<: *docker-compose-common
+          run: pg-mysql-backend-test-env
       - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 25
+    timeout_in_minutes: 30
     retry: *auto-retry
 
   - label: "slow end-to-end test (release)"
@@ -716,55 +747,15 @@ steps:
       - "build"
       - "build-other"
       - "docslt"
-    plugins:
-      - docker-compose#v4.9.0:
-          run: rw-build-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-      - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 25
-    retry: *auto-retry
-
-  - label: "end-to-end test (postgres backend)"
-    key: e2e-test-postgres-backend
-    command: "ci/scripts/e2e-test.sh -p ci-release -m ci-3streaming-2serving-3fe-pg-backend"
-    if: |
-      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-e2e-test-other-backends"
-      || build.env("CI_STEPS") =~ /(^|,)e2e-test-other-backends?(,|$$)/
-    depends_on:
-      - "build"
-      - "build-other"
-      - "docslt"
     plugins:
       - docker-compose#v5.1.0:
-          run: pg-mysql-backend-test-env
+          run: rw-build-env
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
       - ./ci/plugins/upload-failure-logs
     timeout_in_minutes: 25
     retry: *auto-retry
 
-  - label: "end-to-end test (mysql backend)"
-    key: e2e-test-mysql-backend
-    command: "ci/scripts/e2e-test.sh -p ci-release -m ci-3streaming-2serving-3fe-mysql-backend"
-    if: |
-      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-e2e-test-other-backends"
-      || build.env("CI_STEPS") =~ /(^|,)e2e-test-other-backends?(,|$$)/
-    depends_on:
-      - "build"
-      - "build-other"
-      - "docslt"
-    plugins:
-      - docker-compose#v5.1.0:
-          run: pg-mysql-backend-test-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-      - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 30
-    retry: *auto-retry
-
   - label: "end-to-end test for opendal (parallel)"
     key: "e2e-test-opendal-parallel"
     command: "ci/scripts/e2e-test-parallel-for-opendal.sh -p ci-release"
diff --git a/ci/workflows/pull-request.yml b/ci/workflows/pull-request.yml
index 45502ab3037c..e1c7afa7986e 100644
--- a/ci/workflows/pull-request.yml
+++ b/ci/workflows/pull-request.yml
@@ -13,12 +13,39 @@ cargo-cache: &cargo-cache
   backend: s3
   s3:
     bucket: rw-ci-cache-bucket
-    args: '--no-progress'
+    args: "--no-progress"
   paths:
     - ".cargo/registry/index"
     - ".cargo/registry/cache"
     - ".cargo/git"
 
+other-sql-backend: &other-sql-backend
+  matrix:
+    setup:
+      label: [""]
+      endpoint: [""]
+    adjustments:
+      - with:
+          label: ""
+          endpoint: ""
+        skip: true # hack
+      - with:
+          label: "postgres"
+          # PGPASSWORD=postgres psql -h db -p 5432 -U postgres -d rwmeta
+          endpoint: "postgres://postgres:postgres@db:5432/rwmeta"
+      - with:
+          label: "mysql"
+          # mysql -h mysql -P 3306 -u root -p123456 -D rwmeta
+          endpoint: "mysql://root:123456@mysql:3306/rwmeta"
+  env:
+    RISEDEV_SQL_ENDPOINT: "{{matrix.endpoint}}"
+
+docker-compose-common: &docker-compose-common
+    config: ci/docker-compose.yml
+    mount-buildkite-agent: true
+    propagate-environment: true
+    run: rw-build-env
+
 steps:
   - label: "check ci image rebuild"
     plugins:
@@ -789,24 +816,9 @@ steps:
     timeout_in_minutes: 30
     retry: *auto-retry
 
-  - label: "end-to-end test (postgres backend)"
-    command: "ci/scripts/e2e-test.sh -p ci-dev -m ci-3streaming-2serving-3fe-pg-backend"
-    if: build.pull_request.labels includes "ci/run-e2e-test-other-backends" || build.env("CI_STEPS") =~ /(^|,)e2e-test-other-backends?(,|$$)/
-    depends_on:
-      - "build"
-      - "build-other"
-      - "docslt"
-    plugins:
-      - docker-compose#v5.1.0:
-          run: pg-mysql-backend-test-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-      - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 30
-    retry: *auto-retry
-
-  - label: "end-to-end test (mysql backend)"
-    command: "ci/scripts/e2e-test.sh -p ci-dev -m ci-3streaming-2serving-3fe-mysql-backend"
+  - label: "end-to-end test ({{matrix.label}} backend)"
+    <<: *other-sql-backend
+    command: "ci/scripts/e2e-test.sh -p ci-dev -m ci-3streaming-2serving-3fe"
     if: build.pull_request.labels includes "ci/run-e2e-test-other-backends" || build.env("CI_STEPS") =~ /(^|,)e2e-test-other-backends?(,|$$)/
     depends_on:
       - "build"
@@ -814,9 +826,8 @@ steps:
       - "docslt"
     plugins:
       - docker-compose#v5.1.0:
+          <<: *docker-compose-common
           run: pg-mysql-backend-test-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
       - ./ci/plugins/upload-failure-logs
     timeout_in_minutes: 32
     retry: *auto-retry
diff --git a/risedev.yml b/risedev.yml
index 5f7e43d1b75d..b51b0c768d33 100644
--- a/risedev.yml
+++ b/risedev.yml
@@ -777,125 +777,8 @@ profile:
     config-path: src/config/ci.toml
     steps:
       - use: minio
-      - use: sqlite
-      - use: meta-node
-        meta-backend: sqlite
-      - use: compute-node
-        port: 5687
-        exporter-port: 1222
-        enable-tiered-cache: true
-        role: streaming
-        parallelism: 4
-      - use: compute-node
-        port: 5688
-        exporter-port: 1223
-        enable-tiered-cache: true
-        role: streaming
-        parallelism: 4
-      - use: compute-node
-        port: 5689
-        exporter-port: 1224
-        enable-tiered-cache: true
-        role: streaming
-        parallelism: 4
-      - use: compute-node
-        port: 5685
-        exporter-port: 1225
-        enable-tiered-cache: true
-        role: serving
-        parallelism: 4
-      - use: compute-node
-        port: 5686
-        exporter-port: 1226
-        enable-tiered-cache: true
-        role: serving
-        parallelism: 8
-      - use: frontend
-        port: 4565
-        exporter-port: 2222
-        health-check-port: 6786
-      - use: frontend
-        port: 4566
-        exporter-port: 2223
-        health-check-port: 6787
-      - use: frontend
-        port: 4567
-        exporter-port: 2224
-        health-check-port: 6788
-      - use: compactor
-
-  ci-3streaming-2serving-3fe-pg-backend:
-    config-path: src/config/ci.toml
-    steps:
-      - use: minio
-      - use: postgres
-        port: 5432
-        address: db
-        database: metadata
-        user: postgres
-        password: postgres
-        user-managed: true
-        application: metastore
       - use: meta-node
-        meta-backend: postgres
-      - use: compute-node
-        port: 5687
-        exporter-port: 1222
-        enable-tiered-cache: true
-        role: streaming
-        parallelism: 4
-      - use: compute-node
-        port: 5688
-        exporter-port: 1223
-        enable-tiered-cache: true
-        role: streaming
-        parallelism: 4
-      - use: compute-node
-        port: 5689
-        exporter-port: 1224
-        enable-tiered-cache: true
-        role: streaming
-        parallelism: 4
-      - use: compute-node
-        port: 5685
-        exporter-port: 1225
-        enable-tiered-cache: true
-        role: serving
-        parallelism: 4
-      - use: compute-node
-        port: 5686
-        exporter-port: 1226
-        enable-tiered-cache: true
-        role: serving
-        parallelism: 8
-      - use: frontend
-        port: 4565
-        exporter-port: 2222
-        health-check-port: 6786
-      - use: frontend
-        port: 4566
-        exporter-port: 2223
-        health-check-port: 6787
-      - use: frontend
-        port: 4567
-        exporter-port: 2224
-        health-check-port: 6788
-      - use: compactor
-
-  ci-3streaming-2serving-3fe-mysql-backend:
-    config-path: src/config/ci.toml
-    steps:
-      - use: minio
-      - use: mysql
-        port: 3306
-        address: mysql
-        database: metadata
-        user: root
-        password: 123456
-        user-managed: true
-        application: metastore
-      - use: meta-node
-        meta-backend: mysql
+        meta-backend: env
       - use: compute-node
         port: 5687
         exporter-port: 1222
diff --git a/src/risedevtool/src/bin/risedev-dev.rs b/src/risedevtool/src/bin/risedev-dev.rs
index 8bf9aa52fe8f..41e3c3811125 100644
--- a/src/risedevtool/src/bin/risedev-dev.rs
+++ b/src/risedevtool/src/bin/risedev-dev.rs
@@ -384,7 +384,7 @@ fn main() -> Result<()> {
     tracing_subscriber::fmt()
         .with_env_filter(
             EnvFilter::builder()
-                .with_default_directive(LevelFilter::WARN.into())
+                .with_default_directive(LevelFilter::INFO.into())
                 .with_env_var("RISEDEV_RUST_LOG")
                 .from_env_lossy(),
         )
diff --git a/src/risedevtool/src/task/meta_node_service.rs b/src/risedevtool/src/task/meta_node_service.rs
index 752dd87cd0cd..721f33573fa7 100644
--- a/src/risedevtool/src/task/meta_node_service.rs
+++ b/src/risedevtool/src/task/meta_node_service.rs
@@ -42,6 +42,10 @@ use crate::{
 fn sql_endpoint_from_env() -> String {
     static SQL_ENDPOINT: LazyLock<String> = LazyLock::new(|| {
         if let Ok(endpoint) = env::var("RISEDEV_SQL_ENDPOINT") {
+            tracing::info!(
+                "sql endpoint from env RISEDEV_SQL_ENDPOINT resolved to `{}`",
+                endpoint
+            );
             endpoint
         } else {
             let temp_path = NamedTempFile::with_suffix(".db").unwrap().into_temp_path();


Test patch:
diff --git a/ci/scripts/run-e2e-test.sh b/ci/scripts/run-e2e-test.sh
index e84ead4a81df..5a450f20b6ed 100755
--- a/ci/scripts/run-e2e-test.sh
+++ b/ci/scripts/run-e2e-test.sh
@@ -48,13 +48,6 @@ cluster_start() {
     # Give it a while to make sure the single-node is ready.
     sleep 10
   else
-    # Initialize backends.
-    if [[ $mode == *"mysql-backend" ]]; then
-      mysql -h mysql -P 3306 -u root -p123456 -e "DROP DATABASE IF EXISTS metadata; CREATE DATABASE metadata;"
-    elif [[ $mode == *"pg-backend" ]]; then
-      PGPASSWORD=postgres psql -h db -p 5432 -U postgres -c "DROP DATABASE IF EXISTS metadata;" -c "CREATE DATABASE metadata;"
-    fi
-
     risedev ci-start "$mode"
   fi
 }