Repository: fast/logforth
Pull request #111: feat: blocking appender for rolling file and syslog
Author: login: tisonkun
Created: 2025-03-24T04:17:12Z | Merged: 2025-03-24T07:24:45Z

This closes #96 

Linked issue #96: Implement blocking appender for RollingFile and Syslog
Sometiems, using `NonBlocking` and send the formatted record to another threads for logging can be even slower.

Use a mutex to lock the file/stream and log can be fast.

This issue can require a benchmark suite in front.

Diff:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index fae69cf..4ea00f5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,12 @@
 
 All notable changes to this project will be documented in this file.
 
+## Unreleased
+
+### New features
+
+* Add `BlockingRollingFile` and `BlockingSyslog` appenders. They would lock the inner appender and log synchronously. In some lower rate logging scenarios, this would be more efficient than the non-blocking version.
+
 ## [0.23.1] 2025-03-23
 
 ### Improvements
diff --git a/src/append/mod.rs b/src/append/mod.rs
index 5b064aa..3a66526 100644
--- a/src/append/mod.rs
+++ b/src/append/mod.rs
@@ -37,10 +37,14 @@ pub use self::journald::Journald;
 #[cfg(feature = "opentelemetry")]
 pub use self::opentelemetry::OpentelemetryLog;
 #[cfg(feature = "rolling-file")]
+pub use self::rolling_file::BlockingRollingFile;
+#[cfg(feature = "rolling-file")]
 pub use self::rolling_file::RollingFile;
 pub use self::stdio::Stderr;
 pub use self::stdio::Stdout;
 #[cfg(feature = "syslog")]
+pub use self::syslog::BlockingSyslog;
+#[cfg(feature = "syslog")]
 pub use self::syslog::Syslog;
 
 /// A trait representing an appender that can process log records.
diff --git a/src/append/rolling_file/append.rs b/src/append/rolling_file/append.rs
index b3b7d40..e7939f5 100644
--- a/src/append/rolling_file/append.rs
+++ b/src/append/rolling_file/append.rs
@@ -12,6 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::io::Write;
+use std::sync::Mutex;
+
 use log::Record;
 
 use crate::append::rolling_file::RollingFileWriter;
@@ -21,6 +24,48 @@ use crate::non_blocking::NonBlocking;
 use crate::Diagnostic;
 use crate::Layout;
 
+/// An appender that blocking writes log records to rolling files.
+#[derive(Debug)]
+pub struct BlockingRollingFile {
+    layout: Box<dyn Layout>,
+    writer: Mutex<RollingFileWriter>,
+}
+
+impl BlockingRollingFile {
+    /// Creates a new [`BlockingRollingFile`] appender.
+    ///
+    /// This appender by default uses [`TextLayout`] to format log records.
+    pub fn new(writer: RollingFileWriter) -> Self {
+        Self {
+            layout: Box::new(TextLayout::default().no_color()),
+            writer: Mutex::new(writer),
+        }
+    }
+
+    /// Sets the layout used to format log records.
+    pub fn with_layout(mut self, layout: impl Into<Box<dyn Layout>>) -> Self {
+        self.layout = layout.into();
+        self
+    }
+}
+
+impl Append for BlockingRollingFile {
+    fn append(&self, record: &Record, diagnostics: &[Diagnostic]) -> anyhow::Result<()> {
+        let mut bytes = self.layout.format(record, diagnostics)?;
+        bytes.push(b'\n');
+        let mut writer = self.writer.lock().unwrap_or_else(|e| e.into_inner());
+        Write::write_all(&mut *writer, bytes.as_slice())?;
+        Ok(())
+    }
+
+    fn flush(&self) {
+        let mut writer = self.writer.lock().unwrap_or_else(|e| e.into_inner());
+        if let Err(err) = Write::flush(&mut *writer) {
+            eprintln!("failed to flush writer: {err}");
+        }
+    }
+}
+
 /// An appender that writes log records to rolling files.
 #[derive(Debug)]
 pub struct RollingFile {
diff --git a/src/append/rolling_file/mod.rs b/src/append/rolling_file/mod.rs
index c07b4ca..01850ad 100644
--- a/src/append/rolling_file/mod.rs
+++ b/src/append/rolling_file/mod.rs
@@ -41,6 +41,7 @@
 //! log::info!("This log will be written to a rolling file.");
 //! ```
 
+pub use append::BlockingRollingFile;
 pub use append::RollingFile;
 pub use rolling::RollingFileWriter;
 pub use rolling::RollingFileWriterBuilder;
diff --git a/src/append/syslog.rs b/src/append/syslog.rs
index 0020e7e..9073925 100644
--- a/src/append/syslog.rs
+++ b/src/append/syslog.rs
@@ -35,6 +35,7 @@
 //! ```
 
 use std::io;
+use std::sync::Mutex;
 
 use fasyslog::format::SyslogContext;
 use fasyslog::sender::SyslogSender;
@@ -67,9 +68,7 @@ pub enum SyslogFormat {
 #[derive(Debug)]
 pub struct Syslog {
     writer: NonBlocking<SyslogWriter>,
-    format: SyslogFormat,
-    context: SyslogContext,
-    layout: Option<Box<dyn Layout>>,
+    formatter: SyslogFormatter,
 }
 
 impl Syslog {
@@ -77,21 +76,72 @@ impl Syslog {
     pub fn new(writer: NonBlocking<SyslogWriter>) -> Self {
         Self {
             writer,
-            format: SyslogFormat::RFC3164,
-            context: SyslogContext::default(),
-            layout: None,
+            formatter: SyslogFormatter {
+                format: SyslogFormat::RFC3164,
+                context: SyslogContext::default(),
+                layout: None,
+            },
+        }
+    }
+
+    /// Set the format of the [`Syslog`] appender.
+    pub fn with_format(mut self, format: SyslogFormat) -> Self {
+        self.formatter.format = format;
+        self
+    }
+
+    /// Set the context of the [`Syslog`] appender.
+    pub fn with_context(mut self, context: SyslogContext) -> Self {
+        self.formatter.context = context;
+        self
+    }
+
+    /// Set the layout of the [`Syslog`] appender.
+    ///
+    /// Default to `None`, only the args will be logged.
+    pub fn with_layout(mut self, layout: impl Into<Box<dyn Layout>>) -> Self {
+        self.formatter.layout = Some(layout.into());
+        self
+    }
+}
+
+impl Append for Syslog {
+    fn append(&self, record: &Record, diagnostics: &[Diagnostic]) -> anyhow::Result<()> {
+        let message = self.formatter.format_message(record, diagnostics)?;
+        self.writer.send(message)?;
+        Ok(())
+    }
+}
+
+/// An appender that blocking writes log records to syslog.
+#[derive(Debug)]
+pub struct BlockingSyslog {
+    writer: Mutex<SyslogWriter>,
+    formatter: SyslogFormatter,
+}
+
+impl BlockingSyslog {
+    /// Creates a new [`Syslog`] appender.
+    pub fn new(writer: SyslogWriter) -> Self {
+        Self {
+            writer: Mutex::new(writer),
+            formatter: SyslogFormatter {
+                format: SyslogFormat::RFC3164,
+                context: SyslogContext::default(),
+                layout: None,
+            },
         }
     }
 
     /// Set the format of the [`Syslog`] appender.
     pub fn with_format(mut self, format: SyslogFormat) -> Self {
-        self.format = format;
+        self.formatter.format = format;
         self
     }
 
     /// Set the context of the [`Syslog`] appender.
     pub fn with_context(mut self, context: SyslogContext) -> Self {
-        self.context = context;
+        self.formatter.context = context;
         self
     }
 
@@ -99,11 +149,34 @@ impl Syslog {
     ///
     /// Default to `None`, only the args will be logged.
     pub fn with_layout(mut self, layout: impl Into<Box<dyn Layout>>) -> Self {
-        self.layout = Some(layout.into());
+        self.formatter.layout = Some(layout.into());
         self
     }
 }
 
+impl Append for BlockingSyslog {
+    fn append(&self, record: &Record, diagnostics: &[Diagnostic]) -> anyhow::Result<()> {
+        let message = self.formatter.format_message(record, diagnostics)?;
+        let mut writer = self.writer.lock().unwrap_or_else(|e| e.into_inner());
+        writer.write_all(message.as_slice())?;
+        Ok(())
+    }
+
+    fn flush(&self) {
+        let mut writer = self.writer.lock().unwrap_or_else(|e| e.into_inner());
+        if let Err(err) = writer.flush() {
+            eprintln!("failed to flush writer: {err}");
+        }
+    }
+}
+
+#[derive(Debug)]
+struct SyslogFormatter {
+    format: SyslogFormat,
+    context: SyslogContext,
+    layout: Option<Box<dyn Layout>>,
+}
+
 fn log_level_to_otel_severity(level: log::Level) -> fasyslog::Severity {
     match level {
         log::Level::Error => fasyslog::Severity::ERROR,
@@ -114,9 +187,14 @@ fn log_level_to_otel_severity(level: log::Level) -> fasyslog::Severity {
     }
 }
 
-impl Append for Syslog {
-    fn append(&self, record: &Record, diagnostics: &[Diagnostic]) -> anyhow::Result<()> {
+impl SyslogFormatter {
+    fn format_message(
+        &self,
+        record: &Record,
+        diagnostics: &[Diagnostic],
+    ) -> anyhow::Result<Vec<u8>> {
         let severity = log_level_to_otel_severity(record.level());
+
         let message = match self.format {
             SyslogFormat::RFC3164 => match self.layout {
                 None => format!(
@@ -159,8 +237,8 @@ impl Append for Syslog {
                 }
             }
         };
-        self.writer.send(message.into_bytes())?;
-        Ok(())
+
+        Ok(message.into_bytes())
     }
 }
 


Reviews:
user: login: tisonkun
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T04:18:32Z
user: login: andylokandy
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T11:32:16Z
user: login: tisonkun
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T11:45:03Z

Review comments:
diff_hunk: @@ -21,6 +23,47 @@ use crate::non_blocking::NonBlocking;
 use crate::Diagnostic;
 use crate::Layout;
 
+#[derive(Debug)]
+pub struct BlockingRollingFile {
+    layout: Box<dyn Layout>,
+    writer: Mutex<RollingFileWriter>,
+}
+
+impl BlockingRollingFile {
+    /// Creates a new [`BlockingRollingFile`] appender.
+    ///
+    /// This appender by default uses [`TextLayout`] to format log records.
+    pub fn new(writer: RollingFileWriter) -> Self {
+        Self {
+            layout: Box::new(TextLayout::default().no_color()),
+            writer: Mutex::new(writer),
+        }
+    }
+
+    /// Sets the layout used to format log records.
+    pub fn with_layout(mut self, layout: impl Into<Box<dyn Layout>>) -> Self {
+        self.layout = layout.into();
+        self
+    }
+}
+
+impl Append for BlockingRollingFile {
+    fn append(&self, record: &Record, diagnostics: &[Diagnostic]) -> anyhow::Result<()> {
+        let mut writer = self.writer.lock().unwrap_or_else(|e| e.into_inner());
+        let mut bytes = self.layout.format(record, diagnostics)?;
+        bytes.push(b'\n');
+        Write::write_all(&mut *writer, bytes.as_slice())?;
+        Ok(())
+    }
+
+    fn flush(&self) {
path: src/append/rolling_file/append.rs
user: login: tisonkun
body: @andylokandy maybe flush should return `anyhow::Result<()>` and we handle error all at one level up:

```rust
    fn log(&self, record: &Record) {
        for dispatch in &self.dispatches {
            if let Err(err) = dispatch.log(record) {
                handle_error(record, err);
            }
        }
    }

    fn flush(&self) {
        for dispatch in &self.dispatches {
            if let Err(err) = dispatch.flush() {
                handle_flush_error(err);
            }
        }
    }
```

also maybe add a ErrorSink trait for custom error listener.
created_at: 2025-03-24T04:18:32Z
author_association: CONTRIBUTOR
diff_hunk: @@ -21,6 +23,47 @@ use crate::non_blocking::NonBlocking;
 use crate::Diagnostic;
 use crate::Layout;
 
+#[derive(Debug)]
+pub struct BlockingRollingFile {
+    layout: Box<dyn Layout>,
+    writer: Mutex<RollingFileWriter>,
+}
+
+impl BlockingRollingFile {
+    /// Creates a new [`BlockingRollingFile`] appender.
+    ///
+    /// This appender by default uses [`TextLayout`] to format log records.
+    pub fn new(writer: RollingFileWriter) -> Self {
+        Self {
+            layout: Box::new(TextLayout::default().no_color()),
+            writer: Mutex::new(writer),
+        }
+    }
+
+    /// Sets the layout used to format log records.
+    pub fn with_layout(mut self, layout: impl Into<Box<dyn Layout>>) -> Self {
+        self.layout = layout.into();
+        self
+    }
+}
+
+impl Append for BlockingRollingFile {
+    fn append(&self, record: &Record, diagnostics: &[Diagnostic]) -> anyhow::Result<()> {
+        let mut writer = self.writer.lock().unwrap_or_else(|e| e.into_inner());
+        let mut bytes = self.layout.format(record, diagnostics)?;
+        bytes.push(b'\n');
+        Write::write_all(&mut *writer, bytes.as_slice())?;
+        Ok(())
+    }
+
+    fn flush(&self) {
path: src/append/rolling_file/append.rs
user: login: andylokandy
body: I think we can keep the behavior in sync with non-blocking its variant. is the non blocking one return result on flush?
created_at: 2025-03-24T11:32:16Z
author_association: CONTRIBUTOR
diff_hunk: @@ -21,6 +23,47 @@ use crate::non_blocking::NonBlocking;
 use crate::Diagnostic;
 use crate::Layout;
 
+#[derive(Debug)]
+pub struct BlockingRollingFile {
+    layout: Box<dyn Layout>,
+    writer: Mutex<RollingFileWriter>,
+}
+
+impl BlockingRollingFile {
+    /// Creates a new [`BlockingRollingFile`] appender.
+    ///
+    /// This appender by default uses [`TextLayout`] to format log records.
+    pub fn new(writer: RollingFileWriter) -> Self {
+        Self {
+            layout: Box::new(TextLayout::default().no_color()),
+            writer: Mutex::new(writer),
+        }
+    }
+
+    /// Sets the layout used to format log records.
+    pub fn with_layout(mut self, layout: impl Into<Box<dyn Layout>>) -> Self {
+        self.layout = layout.into();
+        self
+    }
+}
+
+impl Append for BlockingRollingFile {
+    fn append(&self, record: &Record, diagnostics: &[Diagnostic]) -> anyhow::Result<()> {
+        let mut writer = self.writer.lock().unwrap_or_else(|e| e.into_inner());
+        let mut bytes = self.layout.format(record, diagnostics)?;
+        bytes.push(b'\n');
+        Write::write_all(&mut *writer, bytes.as_slice())?;
+        Ok(())
+    }
+
+    fn flush(&self) {
path: src/append/rolling_file/append.rs
user: login: tisonkun
body: The non-blocking one do flush in the worker thread where the error is handled as `eprintln!("failed to write log: {err}");`.
created_at: 2025-03-24T11:45:03Z
author_association: CONTRIBUTOR

PR comments:
user: login: andylokandy
created_at: 2025-03-24T11:34:01Z
body: I'm curious why `BlockingRollingFile` is necessary since there `RollingFile`. It should be intuitive that `RollingFile` is blocking if `non_blocking` is not used.
author_association: CONTRIBUTOR
user: login: tisonkun
created_at: 2025-03-24T11:40:03Z
body: >  It should be intuitive that RollingFile is blocking if non_blocking is not used.

`RollingFile` accepts always NonBlocking. Do we have some way to paramizied this?
author_association: CONTRIBUTOR
user: login: andylokandy
created_at: 2025-03-24T12:12:10Z
body: > RollingFile accepts always NonBlocking. Do we have some way to paramizied this?

Sound reasonable. I'll take some time to play with it.
author_association: CONTRIBUTOR