Repository: GreptimeTeam/greptimedb (Rust)

Issue:
Observable write stalls on high load
### What type of bug is this?

Performance issue

### What subsystems are affected?

Datanode

### Minimal reproduce step

You can simply reproduce this in TSBS suite.

### What did you expect to see?

There should be no peaks and valleys in requests handled per second.

### What did you see instead?

When performs benchmark on disks with average performance (like AWS gp2/3 with no extra IOPS budget), we can observe noticeable write stall according to the metric `mito_write_rows_total`.

![image](https://github.com/GreptimeTeam/greptimedb/assets/6406592/c061d038-defc-4e0e-ab2a-077fa59354c2)


The stalls have a correlation in terms of time with fsync operations in WAL. Everytime the WAL rotates, it will allocate a new log file and fsync the previous log file to ensure durability. This will cause high IO util. 

Some methods to mitigate these stalls:
- We may break the large fsync into frequent smaller fsyncs to amortize the cost
- Enable log recycle to reuse obsolate log files.

### What operating system did you use?

NA




Fix (patch):
diff --git a/Cargo.lock b/Cargo.lock
index a5e188f70c17..aa0bbd3e1126 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4509,6 +4509,8 @@ dependencies = [
  "common-runtime",
  "common-telemetry",
  "common-test-util",
+ "common-time",
+ "dashmap",
  "futures",
  "futures-util",
  "itertools 0.10.5",
diff --git a/config/standalone.example.toml b/config/standalone.example.toml
index 24412443cb75..c9fd320b077d 100644
--- a/config/standalone.example.toml
+++ b/config/standalone.example.toml
@@ -138,6 +138,12 @@ purge_interval = "10m"
 read_batch_size = 128
 # Whether to sync log file after every write.
 sync_write = false
+# Whether to reuse logically truncated log files.
+enable_log_recycle = true
+# Whether to pre-create log files on start up
+prefill_log_files = false
+# Duration for fsyncing log files.
+sync_period = "1000ms"
 
 # Metadata storage options.
 [metadata_store]
diff --git a/src/common/config/src/wal/raft_engine.rs b/src/common/config/src/wal/raft_engine.rs
index ac9ff3047783..9ce8ebec406c 100644
--- a/src/common/config/src/wal/raft_engine.rs
+++ b/src/common/config/src/wal/raft_engine.rs
@@ -34,6 +34,13 @@ pub struct RaftEngineConfig {
     pub read_batch_size: usize,
     // whether to sync log file after every write
     pub sync_write: bool,
+    // whether to reuse logically truncated log files.
+    pub enable_log_recycle: bool,
+    // whether to pre-create log files on start up
+    pub prefill_log_files: bool,
+    // duration for fsyncing log files.
+    #[serde(with = "humantime_serde")]
+    pub sync_period: Option<Duration>,
 }
 
 impl Default for RaftEngineConfig {
@@ -45,6 +52,9 @@ impl Default for RaftEngineConfig {
             purge_interval: Duration::from_secs(600),
             read_batch_size: 128,
             sync_write: false,
+            enable_log_recycle: true,
+            prefill_log_files: false,
+            sync_period: None,
         }
     }
 }
diff --git a/src/log-store/Cargo.toml b/src/log-store/Cargo.toml
index 9f2c8b9bc728..370fbc8696b1 100644
--- a/src/log-store/Cargo.toml
+++ b/src/log-store/Cargo.toml
@@ -22,6 +22,8 @@ common-macro.workspace = true
 common-meta.workspace = true
 common-runtime.workspace = true
 common-telemetry.workspace = true
+common-time.workspace = true
+dashmap.workspace = true
 futures-util.workspace = true
 futures.workspace = true
 protobuf = { version = "2", features = ["bytes"] }
diff --git a/src/log-store/src/raft_engine/log_store.rs b/src/log-store/src/raft_engine/log_store.rs
index 33d1d6247687..15c81d1e49d6 100644
--- a/src/log-store/src/raft_engine/log_store.rs
+++ b/src/log-store/src/raft_engine/log_store.rs
@@ -14,6 +14,7 @@
 
 use std::collections::HashMap;
 use std::fmt::{Debug, Formatter};
+use std::sync::atomic::{AtomicI64, Ordering};
 use std::sync::Arc;
 
 use async_stream::stream;
@@ -41,6 +42,7 @@ pub struct RaftEngineLogStore {
     config: RaftEngineConfig,
     engine: Arc<Engine>,
     gc_task: RepeatedTask<Error>,
+    last_sync_time: AtomicI64,
 }
 
 pub struct PurgeExpiredFilesFunction {
@@ -80,6 +82,8 @@ impl RaftEngineLogStore {
             recovery_mode: RecoveryMode::TolerateTailCorruption,
             batch_compression_threshold: ReadableSize::kb(8),
             target_file_size: ReadableSize(config.file_size.0),
+            enable_log_recycle: config.enable_log_recycle,
+            prefill_for_recycle: config.prefill_log_files,
             ..Default::default()
         };
         let engine = Arc::new(Engine::open(raft_engine_config).context(RaftEngineSnafu)?);
@@ -94,6 +98,7 @@ impl RaftEngineLogStore {
             config,
             engine,
             gc_task,
+            last_sync_time: AtomicI64::new(0),
         };
         log_store.start()?;
         Ok(log_store)
@@ -210,9 +215,19 @@ impl LogStore for RaftEngineLogStore {
                 .context(AddEntryLogBatchSnafu)?;
         }
 
+        let mut sync = self.config.sync_write;
+
+        if let Some(sync_period) = &self.config.sync_period {
+            let now = common_time::util::current_time_millis();
+            if now - self.last_sync_time.load(Ordering::Relaxed) >= sync_period.as_millis() as i64 {
+                self.last_sync_time.store(now, Ordering::Relaxed);
+                sync = true;
+            }
+        }
+
         let _ = self
             .engine
-            .write(&mut batch, self.config.sync_write)
+            .write(&mut batch, sync)
             .context(RaftEngineSnafu)?;
 
         Ok(AppendBatchResponse { last_entry_ids })