Repository: aicers/crusher
Pull request #192: Fix connection behavior after manager restart
Author: login: danbi2990
Created: 2025-04-17T06:17:30Z | Merged: 2025-04-28T01:46:44Z

Update the `connect` method to properly check whether the existing connection has been closed before attempting to reuse it.

Closes #190

Linked issue #190: Crusher fails to reconnect when the manager is restarted
After the manager is restarted, Crusher shows the following message and does not attempt to reconnect:

> Failed to accept bidirectional stream: ApplicationClosed(ApplicationClose { error_code: 0, reason: b"" })

Diff:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4f3ad1e..0784e57 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,11 @@ Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
 
+### Fixed
+
+- Fixed bugs where the connections to the manager and data-store were not
+  re-established after either application was restarted.
+
 ### Changed
 
 - Replaced the `log_dir` configuration option with `log_path`. The `log_path`
diff --git a/src/request.rs b/src/request.rs
index bf0f797..9619aac 100644
--- a/src/request.rs
+++ b/src/request.rs
@@ -112,29 +112,33 @@ impl Client {
     }
 
     async fn connect(&mut self) -> Result<&Connection> {
-        if let Some(ref conn) = self.connection {
-            return Ok(conn);
+        let needs_reconnect = self
+            .connection
+            .as_ref()
+            .is_none_or(|conn| conn.close_reason().is_some());
+
+        if needs_reconnect {
+            let mut conn_builder = ConnectionBuilder::new(
+                &self.server_name,
+                self.server_address,
+                env!("CARGO_PKG_NAME"),
+                env!("CARGO_PKG_VERSION"),
+                REQUIRED_MANAGER_VERSION,
+                self.status,
+                &self.cert,
+                &self.key,
+            )?;
+            conn_builder.root_certs(&self.ca_certs)?;
+            self.connection = Some(conn_builder.connect().await?);
+            info!(
+                "Connection established to the manager server {}",
+                self.server_address
+            );
         }
 
-        let mut conn_builder = ConnectionBuilder::new(
-            &self.server_name,
-            self.server_address,
-            env!("CARGO_PKG_NAME"),
-            env!("CARGO_PKG_VERSION"),
-            REQUIRED_MANAGER_VERSION,
-            self.status,
-            &self.cert,
-            &self.key,
-        )?;
-        conn_builder.root_certs(&self.ca_certs)?;
-        self.connection = Some(conn_builder.connect().await?);
-        info!(
-            "Connection established to the manager server {}",
-            self.server_address
-        );
         self.connection
             .as_ref()
-            .context("Failed to access to the connection")
+            .context("Failed to access the connection")
     }
 
     pub(crate) async fn get_config(&mut self) -> Result<String> {
diff --git a/src/subscribe.rs b/src/subscribe.rs
index 70cc3f2..a252a3b 100644
--- a/src/subscribe.rs
+++ b/src/subscribe.rs
@@ -150,6 +150,7 @@ impl Client {
         shutdown: Arc<Notify>,
     ) -> Result<()> {
         let (sender, receiver) = async_channel::bounded::<TimeSeries>(TIME_SERIES_CHANNEL_SIZE);
+        let connection_notify = Arc::new(Notify::new());
         tokio::select! {
             Err(e) = async {tokio::try_join!(
                 ingest_connection_control(
@@ -159,6 +160,7 @@ impl Client {
                     &self.endpoint,
                     &self.last_series_time_path,
                     REQUIRED_GIGANTO_VERSION,
+                    connection_notify.clone(),
                 ),
                 publish_connection_control(
                     sender,
@@ -170,6 +172,7 @@ impl Client {
                     active_policy_list,
                     delete_policy_ids,
                     &self.last_series_time_path,
+                    connection_notify.clone(),
                 )
             )} => {
                 self.endpoint.close(0u32.into(), &[]);
@@ -192,9 +195,10 @@ async fn ingest_connection_control(
     endpoint: &Endpoint,
     last_series_time_path: &Path,
     version: &str,
+    connection_notify: Arc<Notify>,
 ) -> Result<()> {
     'connection: loop {
-        let connection_notify = Arc::new(Notify::new());
+        let connection_notify = connection_notify.clone();
         match ingest_connect(endpoint, server_addr, server_name, version).await {
             Ok(conn) => {
                 let arc_conn = Arc::new(conn);
@@ -270,9 +274,10 @@ async fn publish_connection_control(
     active_policy_list: Arc<RwLock<HashMap<u32, SamplingPolicy>>>,
     delete_policy_ids: Arc<RwLock<Vec<u32>>>,
     last_series_time_path: &Path,
+    connection_notify: Arc<Notify>,
 ) -> Result<()> {
     'connection: loop {
-        let connection_notify = Arc::new(Notify::new());
+        let connection_notify = connection_notify.clone();
         match publish_connect(endpoint, server_addr, server_name, version).await {
             Ok((conn, send)) => {
                 let req_send = Arc::new(Mutex::new(send));
@@ -322,6 +327,15 @@ async fn publish_connection_control(
                             sleep(Duration::from_secs(SERVER_RETRY_INTERVAL)).await;
                             continue 'connection;
                         }
+                        err = conn.closed() => {
+                            error!(
+                                "Stream channel closed: {:?}. Retry connection to {} after {} seconds.",
+                                err, server_addr, SERVER_RETRY_INTERVAL,
+                            );
+                            connection_notify.notify_waiters();
+                            sleep(Duration::from_secs(SERVER_RETRY_INTERVAL)).await;
+                            continue 'connection;
+                        }
                         Ok(policy) = request_recv.recv() => {
                             if let Err(e) = process_network_stream(
                                 conn.clone(),
@@ -458,7 +472,7 @@ async fn receiver(
 ) -> Result<()> {
     if let Ok(mut recv) = conn.accept_uni().await {
         let Ok(id) = receive_time_series_generator_stream_start_message(&mut recv).await else {
-            connection_notify.notify_one();
+            connection_notify.notify_waiters();
             warn!("Failed to receive stream id value");
             bail!("Failed to receive stream id value");
         };
@@ -496,7 +510,7 @@ async fn receiver(
                     error!("Failed to generate time series: {}", e);
                 }
             } else {
-                connection_notify.notify_one();
+                connection_notify.notify_waiters();
                 break;
             }
         }
@@ -579,7 +593,7 @@ async fn receive_time_series_timestamp(
                 break;
             }
             Err(e) => {
-                connection_notify.notify_one();
+                connection_notify.notify_waiters();
                 bail!("Last series times error: {}", e);
             }
         }


Reviews:
user: login: sehkone
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T23:39:07Z
user: login: danbi2990
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-18T00:12:11Z
user: login: kimhanbeom
body: LGTM
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-23T01:43:45Z

Review comments:
diff_hunk: @@ -112,29 +112,33 @@ impl Client {
     }
 
     async fn connect(&mut self) -> Result<&Connection> {
-        if let Some(ref conn) = self.connection {
-            return Ok(conn);
+        let needs_reconnect = match self.connection.as_ref() {
+            Some(conn) => conn.close_reason().is_some(),
+            None => true,
+        };
path: src/request.rs
user: login: sehkone
body: I think this can be refactored using method chaining. I believe that would make the code simpler and more idiomatic in Rust.
created_at: 2025-04-17T23:39:07Z
author_association: CONTRIBUTOR
diff_hunk: @@ -112,29 +112,33 @@ impl Client {
     }
 
     async fn connect(&mut self) -> Result<&Connection> {
-        if let Some(ref conn) = self.connection {
-            return Ok(conn);
+        let needs_reconnect = match self.connection.as_ref() {
+            Some(conn) => conn.close_reason().is_some(),
+            None => true,
+        };
path: src/request.rs
user: login: danbi2990
body: Applied the suggested adjustment.
created_at: 2025-04-18T00:12:11Z
author_association: CONTRIBUTOR

PR comments:
user: login: kimhanbeom
created_at: 2025-04-21T10:32:12Z
body: I've tested with the current branch and it doesn't reconnect when I restart the data_store. Could you please also check this part?


author_association: CONTRIBUTOR
user: login: danbi2990
created_at: 2025-04-22T06:10:53Z
body: > I've tested with the current branch and it doesn't reconnect when I restart the data_store. Could you please also check this part?

The bug has been fixed. Could you confirm whether the following reproduction steps are correct?

Bug reproduction steps:

1. Run the manager, data-store and Crusher
2. Insert a sampling policy into the manager
3. Restart the data-store
author_association: CONTRIBUTOR
user: login: kimhanbeom
created_at: 2025-04-22T08:11:02Z
body: if we disconnect while only maintaining a connection (i.e., no policy insert), it will not reconnect.

Bug reproduction steps:
- Run the manager, data-store and Crusher
- Restart the data-store

author_association: CONTRIBUTOR
user: login: danbi2990
created_at: 2025-04-23T01:29:38Z
body: > Bug reproduction steps:
> 
> * Run the manager, data-store and Crusher
> * Restart the data-store

The issue has been fixed.

While addressing it, I discovered the `Connection::closed()` method, which can be used to detect connection termination. This could potentially eliminate the need to pass around `connection_notify`.
If you're okay with it, I'll open a new issue to track this refactoring.
author_association: CONTRIBUTOR