Repository: nervosnetwork/ckb (language: rust)

Issue:
CKB Node fails to shutdown properly.
## Bug Report

There seems to be an intermittent issue where the CKB node will not shutdown properly. The attached log demonstrates this on v0.112.1, but I believe I also saw this on v0.113.0.

In this particular log, I pressed ctrl+c, and it appeared to shutdown but never completed. I waited quite some time and pressed ctrl+c multiple times.

### Environment

- **CKB version**: v0.112.1
- **Chain**: testnet
- **Operating system**: Ubuntu 20.04
- **Arch**: Linux dev5 5.15.108-1-pve #1 SMP PVE 5.15.108-2 (2023-07-20T10:06Z) x86_64 x86_64 x86_64 GNU/Linux
- **Installation**: GitHub Release

### Additional context/Screenshots

[testnet-v0.112.1.log.txt](https://github.com/nervosnetwork/ckb/files/13997084/testnet-v0.112.1.log.txt)



Discussion / hints:

Fix patch:
diff --git a/Cargo.lock b/Cargo.lock
index 1f913ffcf9..bf42e1e1e6 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -907,6 +907,7 @@ version = "0.114.0-pre"
 dependencies = [
  "ckb-app-config",
  "ckb-async-runtime",
+ "ckb-channel",
  "ckb-db-schema",
  "ckb-jsonrpc-types",
  "ckb-logger",
diff --git a/util/indexer/Cargo.toml b/util/indexer/Cargo.toml
index ea01bb6ffe..294921a99b 100644
--- a/util/indexer/Cargo.toml
+++ b/util/indexer/Cargo.toml
@@ -22,6 +22,7 @@ ckb-notify = { path = "../../notify", version = "= 0.114.0-pre" }
 ckb-store = { path = "../../store", version = "= 0.114.0-pre" }
 ckb-stop-handler = { path = "../stop-handler", version = "= 0.114.0-pre" }
 ckb-async-runtime = { path = "../runtime", version = "= 0.114.0-pre" }
+ckb-channel = { path = "../channel", version = "= 0.114.0-pre" }
 rhai = { version = "1.10.0", features = ["no_function", "no_float", "no_module", "sync"]}
 serde_json = "1.0"
 numext-fixed-uint = "0.1"
diff --git a/util/indexer/src/service.rs b/util/indexer/src/service.rs
index fc3aed3c83..6a8d059162 100644
--- a/util/indexer/src/service.rs
+++ b/util/indexer/src/service.rs
@@ -18,7 +18,7 @@ use ckb_jsonrpc_types::{
 };
 use ckb_logger::{error, info};
 use ckb_notify::NotifyController;
-use ckb_stop_handler::{new_tokio_exit_rx, CancellationToken};
+use ckb_stop_handler::{new_crossbeam_exit_rx, new_tokio_exit_rx, CancellationToken};
 use ckb_store::ChainStore;
 use ckb_types::{
     core::{self, BlockNumber},
@@ -153,7 +153,16 @@ impl IndexerService {
                     }
                 }
             }
+            let stop_rx = new_crossbeam_exit_rx();
             loop {
+                ckb_channel::select! {
+                    recv(stop_rx) -> _ =>{
+                        info!("apply_init_tip received exit signal, exit now");
+                        break;
+                    },
+                    default() => {},
+                }
+
                 if let Err(e) = self.secondary_db.try_catch_up_with_primary() {
                     error!("secondary_db try_catch_up_with_primary error {}", e);
                 }
@@ -228,6 +237,16 @@ impl IndexerService {
         let poll_service = self.clone();
         self.async_handle.spawn(async move {
             let _initial_finished = initial_syncing.await;
+            info!("initial_syncing finished");
+
+            tokio::select! {
+                _ = stop.cancelled() => {
+                    info!("Indexer received exit signal, cancel new_block_watcher task, exit now");
+                    return;
+                },
+                else => {},
+            }
+
             let mut new_block_watcher = notify_controller
                 .watch_new_block(SUBSCRIBER_NAME.to_string())
                 .await;


Test patch:
diff --git a/util/app-config/src/tests/graceful_shutdown.bats b/util/app-config/src/tests/graceful_shutdown.bats
index 17ac2661b3..6d6aa5e82e 100644
--- a/util/app-config/src/tests/graceful_shutdown.bats
+++ b/util/app-config/src/tests/graceful_shutdown.bats
@@ -26,7 +26,7 @@ function ckb_graceful_shutdown { #@test
   assert_output --regexp "INFO ckb_tx_pool::chunk_process  TxPool chunk_command service received exit signal, exit now"
   assert_output --regexp "INFO ckb_tx_pool::service  TxPool is saving, please wait..."
   assert_output --regexp "INFO ckb_tx_pool::service  TxPool reorg process service received exit signal, exit now"
-  assert_output --regexp "INFO ckb_indexer::service  Indexer received exit signal, exit now"
+  assert_output --regexp "INFO ckb_indexer::service  Indexer received exit signal, cancel new_block_watcher task, exit now"
   assert_output --regexp "INFO ckb_notify  NotifyService received exit signal, exit now"
   assert_output --regexp "INFO ckb_block_filter::filter  BlockFilter received exit signal, exit now"
   assert_output --regexp "INFO ckb_sync::types::header_map  HeaderMap limit_memory received exit signal, exit now"