Repository: kscalelabs/kinfer
Pull request #35: Async thread that logs to NDJSON
Author: login: alik-git
Created: 2025-05-24T00:23:42Z | Merged: 2025-05-27T07:58:05Z

https://linear.app/k-scale/issue/K-818/kinfer-logging-to-ndjson

- Async worker thread for logging to NDJSON
- No overhead at all of logging is disabled 
- SIGINT is handled, logger flushes before exiting 

I use the env var `KINFER_LOG_PATH` to set the dir to place the logs in. It will create a timestamped file in that dir. The user has to set this manually 
```
# IMPORTANT: set this 
export KINFER_LOG_PATH=~/.kscale/kinfer/logs 
```

see kinfer sim PR https://github.com/kscalelabs/kinfer-sim/pull/12

logs look like this `/Users/ali/.kscale/kinfer/logs/20250523-171847.ndjson`
```json
{"step_id":0,"t_us":1748045928288479,"joint_angles":[-7.195355e-20,-1.5258905e-19,-1.1094459e-20,-3.983125e-19,-1.8847016e-20,8.621992e-20,1.0991208e-20,1.22313275e-20,7.120704e-19,3.186125e-20,2.3034907e-20,1.4873449e-20,-1.0761669e-20,2.1096014e-19,2.8970603e-19,6.666445e-20,4.5469876e-21,-1.3498924e-20,-3.9390712e-20,-2.8386181e-19],"joint_vels":[-4.82947e-18,-1.1990409e-17,-1.2323475e-18,-2.7000624e-17,-8.5077904e-19,5.173639e-18,4.2188476e-19,7.105427e-19,5.2224892e-17,2.3750012e-18,3.619327e-18,1.0158541e-18,-6.286638e-19,1.767475e-17,1.936229e-17,1.6653345e-18,2.0781987e-19,-8.798518e-19,-1.05693234e-17,-1.4921397e-17],"projected_g":[9.80998,-0.000022824233,2.6844074e-7],"accel":[-1.7671129e-15,5.183823e-17,3.7401782e-17],"gyro":[-4.3937075e-18,1.1102767e-20,2.6367797e-18],"command":null,"output":[0.15737313,-0.31186575,0.0007334482,0.23677662,-0.019751938,0.06333378,0.1536933,0.010943025,-0.19811615,0.045603845,-0.016002268,-0.1548894,-0.057537023,-0.9540473,0.25265682,0.118926376,0.12770893,-0.0030555967,0.75394654,-0.080420285]}
{"step_id":1,"t_us":1748045928490683,"joint_angles":[0.023508964,-0.02721244,-0.00092442456,0.09897869,-0.011084773,0.0027521343,0.012742579,0.0030571364,-0.06953044,0.029861242,-0.024421258,-0.007308449,-0.020156845,-0.17326254,0.16416366,0.024633361,0.0032470066,-0.0003922049,0.14042354,-0.07206235],"joint_vels":[1.6107906,-2.2679906,-0.009096729,5.048211,-0.39128497,0.32814133,1.0816278,0.19167915,-3.9251735,1.00954,-0.9267016,-0.6407124,-1.3072103,-12.02908,6.886463,1.0247277,0.3407547,-0.008730757,9.685398,-2.2892327],"projected_g":[9.809852,0.050058465,-0.005300933],"accel":[4.254864,13.010618,0.03308813],"gyro":[-0.50897986,-0.032684065,-0.7830548],"command":null,"output":[0.17523,-0.3241016,0.008352118,0.26177353,-0.010173204,0.16744977,0.012884244,-0.01222945,-0.117710784,0.033091433,-0.20588528,-0.24511875,0.14515194,-0.7058203,0.7490652,0.054662645,0.078237705,-0.17055407,0.6050309,-0.09090328]}
```

here is what the plots look like from the logged data when run on the testbench
![image](https://github.com/user-attachments/assets/36eb7375-a085-439a-9c5c-9babd15b7ee2)


Diff:
diff --git a/Cargo.toml b/Cargo.toml
index 879be58..97f5559 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -8,7 +8,7 @@ resolver = "2"
 
 [workspace.package]
 
-version = "0.5.0"
+version = "0.5.1"
 authors = ["Wesley Maa <wesley@kscale.dev>", "Benjamin Bolte <ben@kscale.dev>"]
 edition = "2021"
 description = "K-Scale Inference Library"
diff --git a/kinfer/requirements.txt b/kinfer/requirements.txt
index 5236354..0910a77 100644
--- a/kinfer/requirements.txt
+++ b/kinfer/requirements.txt
@@ -6,3 +6,8 @@ onnxruntime==1.20.0
 
 # Serialization
 pydantic
+
+# Plotting
+matplotlib
+numpy
+pathlib
diff --git a/kinfer/rust/Cargo.toml b/kinfer/rust/Cargo.toml
index aefd3aa..031d43e 100644
--- a/kinfer/rust/Cargo.toml
+++ b/kinfer/rust/Cargo.toml
@@ -17,8 +17,11 @@ crate-type = ["cdylib", "rlib"]
 [dependencies]
 
 async-trait = "0.1"
+chrono = "0.4.41"
+crossbeam-channel = "0.5.15"
 flate2 = "1.0"
 futures-util = "0.3.30"
+log = "0.4"
 ndarray = "0.16.1"
 ort = { version = "2.0.0-rc.9", features = [ "load-dynamic" ] }
 serde = { version = "1.0", features = ["derive"] }
diff --git a/kinfer/rust/src/lib.rs b/kinfer/rust/src/lib.rs
index 420c7f1..49d4ddb 100644
--- a/kinfer/rust/src/lib.rs
+++ b/kinfer/rust/src/lib.rs
@@ -1,3 +1,4 @@
+pub mod logger;
 pub mod model;
 pub mod runtime;
 pub mod types;
diff --git a/kinfer/rust/src/logger.rs b/kinfer/rust/src/logger.rs
new file mode 100644
index 0000000..18a0507
--- /dev/null
+++ b/kinfer/rust/src/logger.rs
@@ -0,0 +1,135 @@
+use std::{
+    fs::OpenOptions,
+    io::{BufWriter, Write},
+    path::{Path, PathBuf},
+    sync::atomic::{AtomicU64, Ordering},
+    thread,
+};
+
+use crossbeam_channel::{bounded, Sender};
+use log::{info, warn};
+use serde::Serialize;
+
+#[derive(Serialize)]
+struct NdjsonStep<'a> {
+    step_id: u64,
+    t_us: u64,
+    joint_angles: Option<&'a [f32]>,
+    joint_vels: Option<&'a [f32]>,
+    projected_g: Option<&'a [f32]>,
+    accel: Option<&'a [f32]>,
+    gyro: Option<&'a [f32]>,
+    command: Option<&'a [f32]>,
+    output: Option<&'a [f32]>,
+}
+
+// Channel capacity for non-blocking logging.
+// ~1000 entires at 50Hz is 20 seconds of buffering.
+// Warns if messages are dropped due to full buffer.
+const CHANNEL_CAP: usize = 1024;
+
+// Flush buffered writes every 100 log entries.
+// At 50Hz control frequency, this flushes every 2 seconds.
+const FLUSH_EVERY: u64 = 100;
+
+pub struct StepLogger {
+    tx: Option<Sender<Vec<u8>>>,
+    worker: Option<thread::JoinHandle<()>>,
+    next_id: AtomicU64,
+}
+
+impl StepLogger {
+    pub fn new(path: impl AsRef<Path>) -> std::io::Result<Self> {
+        let path: PathBuf = path.as_ref().into();
+        if let Some(parent) = path.parent() {
+            std::fs::create_dir_all(parent)?;
+        }
+        info!("kinfer: logging to NDJSON: {}", path.display());
+
+        // I/O objects created here, but moved into the worker thread.
+        let file = OpenOptions::new().create(true).append(true).open(&path)?;
+        let mut bw = BufWriter::new(file);
+
+        // Bounded channel -> back-pressure capped at CHANNEL_CAP lines
+        let (tx, rx) = bounded::<Vec<u8>>(CHANNEL_CAP);
+
+        let worker = thread::spawn(move || {
+            let mut line_ctr: u64 = 0;
+            for msg in rx {
+                // drains until all senders dropped
+                let _ = bw.write_all(&msg);
+                line_ctr += 1;
+                if line_ctr % FLUSH_EVERY == 0 {
+                    let _ = bw.flush();
+                }
+            }
+            // Final flush on graceful shutdown
+            let _ = bw.flush();
+        });
+
+        Ok(Self {
+            tx: Some(tx),
+            worker: Some(worker),
+            next_id: AtomicU64::new(0),
+        })
+    }
+
+    #[inline]
+    fn now_us() -> u128 {
+        std::time::SystemTime::now()
+            .duration_since(std::time::UNIX_EPOCH)
+            .unwrap()
+            .as_micros()
+    }
+
+    /// Non-blocking; drops a line if the channel is full.
+    #[allow(clippy::too_many_arguments)]
+    pub fn log_step(
+        &self,
+        joint_angles: Option<&[f32]>,
+        joint_vels: Option<&[f32]>,
+        projected_g: Option<&[f32]>,
+        accel: Option<&[f32]>,
+        gyro: Option<&[f32]>,
+        command: Option<&[f32]>,
+        output: Option<&[f32]>,
+    ) {
+        let record = NdjsonStep {
+            step_id: self.next_id.fetch_add(1, Ordering::Relaxed),
+            t_us: Self::now_us() as u64,
+            joint_angles,
+            joint_vels,
+            projected_g,
+            accel,
+            gyro,
+            command,
+            output,
+        };
+
+        // Serialise directly into a Vec<u8>; then push newline and send.
+        if let Ok(mut line) = serde_json::to_vec(&record) {
+            line.push(b'\n');
+            if let Some(tx) = &self.tx {
+                if tx.try_send(line).is_err() {
+                    warn!(
+                        "kinfer: logging buffer full, dropped message (step_id: {})",
+                        record.step_id
+                    );
+                }
+            }
+        }
+    }
+}
+
+/// Ensure the worker drains and flushes before program exit.
+impl Drop for StepLogger {
+    fn drop(&mut self) {
+        if let Some(tx) = self.tx.take() {
+            drop(tx); // Drop sender to close channel
+        }
+        // Wait for worker to finish
+        if let Some(worker) = self.worker.take() {
+            let _ = worker.join();
+        }
+    }
+}
diff --git a/kinfer/rust/src/model.rs b/kinfer/rust/src/model.rs
index 4aa734a..35907f5 100644
--- a/kinfer/rust/src/model.rs
+++ b/kinfer/rust/src/model.rs
@@ -1,4 +1,7 @@
+use crate::logger::StepLogger;
+use crate::types::{InputType, ModelMetadata};
 use async_trait::async_trait;
+use chrono;
 use flate2::read::GzDecoder;
 use ndarray::{Array, IxDyn};
 use ort::session::Session;
@@ -11,8 +14,6 @@ use tar::Archive;
 use tokio::fs::File;
 use tokio::io::AsyncReadExt;
 
-use crate::types::{InputType, ModelMetadata};
-
 #[derive(Debug, thiserror::Error)]
 pub enum ModelError {
     #[error("IO error: {0}")]
@@ -41,6 +42,7 @@ pub struct ModelRunner {
     step_session: Session,
     metadata: ModelMetadata,
     provider: Arc<dyn ModelProvider>,
+    logger: Option<Arc<StepLogger>>,
 }
 
 impl ModelRunner {
@@ -119,11 +121,29 @@ impl ModelRunner {
         // Validate step_fn inputs and outputs
         Self::validate_step_fn(&step_session, &metadata, &carry_shape)?;
 
+        let logger = if let Ok(log_dir) = std::env::var("KINFER_LOG_PATH") {
+            let log_dir_path = std::path::Path::new(&log_dir);
+
+            // Create the directory if it doesn't exist
+            if !log_dir_path.exists() {
+                std::fs::create_dir_all(log_dir_path)?;
+            }
+
+            // Generate a timestamped filename
+            let timestamp = chrono::Utc::now().format("%Y-%m-%d_%H-%M-%S").to_string();
+            let log_file_path = log_dir_path.join(format!("{}.ndjson", timestamp));
+
+            Some(StepLogger::new(log_file_path).map(Arc::new)?)
+        } else {
+            None
+        };
+
         Ok(Self {
             init_session,
             step_session,
             metadata,
             provider: input_provider,
+            logger,
         })
     }
 
@@ -268,6 +288,39 @@ impl ModelRunner {
         let output_tensor = outputs[0].try_extract_tensor::<f32>()?;
         let carry_tensor = outputs[1].try_extract_tensor::<f32>()?;
 
+        // Log the step if needed
+        if let Some(lg) = &self.logger {
+            let joint_angles_opt = inputs
+                .get(&InputType::JointAngles)
+                .map(|a| a.as_slice().unwrap());
+            let joint_vels_opt = inputs
+                .get(&InputType::JointAngularVelocities)
+                .map(|a| a.as_slice().unwrap());
+            let projected_g_opt = inputs
+                .get(&InputType::ProjectedGravity)
+                .map(|a| a.as_slice().unwrap());
+            let accel_opt = inputs
+                .get(&InputType::Accelerometer)
+                .map(|a| a.as_slice().unwrap());
+            let gyro_opt = inputs
+                .get(&InputType::Gyroscope)
+                .map(|a| a.as_slice().unwrap());
+            let command_opt = inputs
+                .get(&InputType::Command)
+                .map(|a| a.as_slice().unwrap());
+            let output_opt = Some(output_tensor.as_slice().unwrap());
+
+            lg.log_step(
+                joint_angles_opt,
+                joint_vels_opt,
+                projected_g_opt,
+                accel_opt,
+                gyro_opt,
+                command_opt,
+                output_opt,
+            );
+        }
+
         Ok((
             output_tensor.view().to_owned(),
             carry_tensor.view().to_owned(),
diff --git a/kinfer/scripts/plot_ndjson.py b/kinfer/scripts/plot_ndjson.py
new file mode 100644
index 0000000..01aafc6
--- /dev/null
+++ b/kinfer/scripts/plot_ndjson.py
@@ -0,0 +1,177 @@
+"""Plot NDJSON logs saved by kinfer."""
+
+import argparse
+import json
+import logging
+from pathlib import Path
+from typing import Optional, Union
+
+import matplotlib.pyplot as plt
+import numpy as np
+
+# Set up logger
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+
+
+def read_ndjson(filepath: str) -> list[dict]:
+    """Read NDJSON file and return list of parsed objects."""
+    data = []
+    with open(filepath, "r") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                data.append(json.loads(line))
+    return data
+
+
+def skip_initial_data(data: list[dict], skip_seconds: float) -> list[dict]:
+    """Skip the first n seconds of data based on timestamps."""
+    if not data or skip_seconds <= 0.0:
+        return data
+
+    # Extract timestamps and convert to seconds relative to first timestamp
+    timestamps = [d["t_us"] for d in data]
+    t_start = timestamps[0]
+    times = [(t - t_start) / 1e6 for t in timestamps]  # Convert to seconds
+
+    # Find indices where time >= skip_seconds
+    skip_indices = [i for i, t in enumerate(times) if t >= skip_seconds]
+    if not skip_indices:
+        logger.info("All data points are within the skip period (%.2f seconds). No data to plot.", skip_seconds)
+        return []
+
+    # Filter data
+    start_idx = skip_indices[0]
+    filtered_data = data[start_idx:]
+    logger.info("Skipped first %.2f seconds (%d data points)", skip_seconds, start_idx)
+
+    return filtered_data
+
+
+def plot_data(data: list[dict], save_path: Optional[Union[str, Path]] = None) -> None:
+    """Plot all data fields from the NDJSON."""
+    if not data:
+        logger.info("No data to plot")
+        return
+
+    # Extract timestamps and convert to seconds relative to first timestamp
+    timestamps = [d["t_us"] for d in data]
+    t_start = timestamps[0]
+    times = [(t - t_start) / 1e6 for t in timestamps]  # Convert to seconds
+
+    # Extract data arrays
+    joint_angles = np.array([d["joint_angles"] for d in data if d["joint_angles"] is not None])
+    joint_vels = np.array([d["joint_vels"] for d in data if d["joint_vels"] is not None])
+    projected_g = np.array([d["projected_g"] for d in data if d["projected_g"] is not None])
+    accel = np.array([d["accel"] for d in data if d["accel"] is not None])
+    command = np.array([d["command"] for d in data if d["command"] is not None])
+    output = np.array([d["output"] for d in data if d["output"] is not None])
+
+    # Create subplots
+    fig, axes = plt.subplots(3, 2, figsize=(15, 12))
+    fig.suptitle("Robot Data Over Time", fontsize=16)
+
+    if len(joint_angles) > 0:
+        ax = axes[0, 0]
+        for i in range(joint_angles.shape[1]):
+            ax.plot(times[: len(joint_angles)], joint_angles[:, i], alpha=0.7, linewidth=0.8)
+        ax.set_title("Joint Angles")
+        ax.set_xlabel("Time (s)")
+        ax.set_ylabel("Angle (rad)")
+        ax.grid(True, alpha=0.3)
+
+    if len(joint_vels) > 0:
+        ax = axes[0, 1]
+        for i in range(joint_vels.shape[1]):
+            ax.plot(times[: len(joint_vels)], joint_vels[:, i], alpha=0.7, linewidth=0.8)
+        ax.set_title("Joint Velocities")
+        ax.set_xlabel("Time (s)")
+        ax.set_ylabel("Velocity (rad/s)")
+        ax.grid(True, alpha=0.3)
+
+    if len(projected_g) > 0:
+        ax = axes[1, 0]
+        labels = ["X", "Y", "Z"]
+        for i in range(projected_g.shape[1]):
+            ax.plot(times[: len(projected_g)], projected_g[:, i], label=labels[i], linewidth=1.5)
+        ax.set_title("Projected Gravity")
+        ax.set_xlabel("Time (s)")
+        ax.set_ylabel("Acceleration (m/s²)")
+        ax.legend()
+        ax.grid(True, alpha=0.3)
+
+    if len(accel) > 0:
+        ax = axes[1, 1]
+        labels = ["X", "Y", "Z"]
+        for i in range(accel.shape[1]):
+            ax.plot(times[: len(accel)], accel[:, i], label=labels[i], linewidth=1.5)
+        ax.set_title("Acceleration")
+        ax.set_xlabel("Time (s)")
+        ax.set_ylabel("Acceleration (m/s²)")
+        ax.legend()
+        ax.grid(True, alpha=0.3)
+
+    if len(command) > 0:
+        ax = axes[2, 0]
+        for i in range(command.shape[1]):
+            ax.plot(times[: len(command)], command[:, i], label=f"Cmd {i}", linewidth=1.2)
+        ax.set_title("Command")
+        ax.set_xlabel("Time (s)")
+        ax.set_ylabel("Command Value")
+        ax.legend()
+        ax.grid(True, alpha=0.3)
+
+    if len(output) > 0:
+        ax = axes[2, 1]
+        for i in range(output.shape[1]):
+            ax.plot(times[: len(output)], output[:, i], alpha=0.7, linewidth=0.8)
+        ax.set_title("Output")
+        ax.set_xlabel("Time (s)")
+        ax.set_ylabel("Output Value")
+        ax.grid(True, alpha=0.3)
+
+    plt.tight_layout()
+
+    if save_path:
+        plt.savefig(save_path, dpi=300, bbox_inches="tight")
+        logger.info("Plot saved to: %s", save_path)
+        plt.close()
+    else:
+        plt.show()
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Plot NDJSON logs saved by kinfer")
+    parser.add_argument("filepath", help="Path to the NDJSON file to plot")
+    parser.add_argument("--skip", type=float, default=0.0, help="Skip the first n seconds of data")
+    parser.add_argument("--save", action="store_true", help="Save the plot to a PNG file in a plots folder")
+    args = parser.parse_args()
+
+    filepath = args.filepath
+    if not Path(filepath).exists():
+        logger.info("File not found: %s", filepath)
+        return
+
+    logger.info("Reading data from %s...", filepath)
+    data = read_ndjson(filepath)
+    logger.info("Loaded %d data points", len(data))
+
+    filtered_data = skip_initial_data(data, args.skip)
+
+    save_path = None
+    if args.save:
+        # Create save path in plots folder with same name but .png extension
+        input_path = Path(filepath)
+        plots_dir = input_path.parent / "plots"
+        plots_dir.mkdir(exist_ok=True)
+
+        # Change extension from .ndjson to .png
+        filename = input_path.stem + ".png"
+        save_path = str(plots_dir / filename)
+
+    plot_data(filtered_data, save_path)
+
+
+if __name__ == "__main__":
+    main()


Reviews:
user: login: gitfish7
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-24T08:15:19Z
user: login: gitfish7
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-24T08:26:47Z
user: login: gitfish7
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-24T08:40:42Z
user: login: copilot-pull-request-reviewer[bot]
body: ## Pull Request Overview

This PR introduces an optional, asynchronous NDJSON logger that writes step data to a file on a background thread, with zero overhead when logging is disabled and ensures a flush on exit.  

- Adds `StepLogger` (with crossbeam channel and background thread) in `logger.rs`.  
- Hooks logger into `ModelRunner`: reads `KINFER_LOG_PATH`, stores an optional logger, and emits logs each step.  
- Exposes the `logger` module and updates `Cargo.toml` for new dependencies.

### Reviewed Changes

Copilot reviewed 4 out of 4 changed files in this pull request and generated 2 comments.

| File                         | Description                                              |
|------------------------------|----------------------------------------------------------|
| kinfer/rust/src/model.rs     | Added optional `StepLogger` field, init from env, and per-step logging calls. |
| kinfer/rust/src/logger.rs    | New `StepLogger` implementation with background thread, channel, and NDJSON serialization. |
| kinfer/rust/src/lib.rs       | Exported `logger` module.                                |
| kinfer/rust/Cargo.toml       | Added `crossbeam-channel` and `log` dependencies (missing `serde_json`). |


<details>
<summary>Comments suppressed due to low confidence (1)</summary>

**kinfer/rust/Cargo.toml:23**
* The code uses `serde_json::to_vec`, but `serde_json` is not listed in `Cargo.toml`. Add `serde_json = "1.0"` (or appropriate version) under `[dependencies]`.
```
log = "0.4"
```
</details>


state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-25T06:18:02Z
user: login: jingxiangmo
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-26T18:29:35Z
user: login: jingxiangmo
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-26T18:30:12Z
user: login: gitfish7
body: approving for later refactor as this is needed urgently
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-26T19:21:51Z
user: login: codekansas
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-26T21:09:03Z
user: login: alik-git
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-26T21:58:08Z
user: login: alik-git
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-26T21:58:17Z
user: login: alik-git
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-06-02T21:56:55Z

Review comments:
diff_hunk: @@ -0,0 +1,119 @@
+use std::{
+    fs::OpenOptions,
+    io::{BufWriter, Write},
+    path::{Path, PathBuf},
+    thread,
+};
+
+use crossbeam_channel::{bounded, Sender};
+use log::info;
+use serde::Serialize;
+
+#[derive(Serialize)]
+struct NdjsonStep<'a> {
+    step_id: u64,
+    t_us: u128,
path: kinfer/rust/src/logger.rs
user: login: gitfish7
body: ```suggestion
    t_us: u64,
```

(2^64) microseconds =
584 554.531 years

we should be good! :laughing: 
created_at: 2025-05-24T08:15:19Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,119 @@
+use std::{
+    fs::OpenOptions,
+    io::{BufWriter, Write},
+    path::{Path, PathBuf},
+    thread,
+};
+
+use crossbeam_channel::{bounded, Sender};
+use log::info;
+use serde::Serialize;
+
+#[derive(Serialize)]
+struct NdjsonStep<'a> {
+    step_id: u64,
+    t_us: u128,
+    joint_angles: Option<&'a [f32]>,
+    joint_vels: Option<&'a [f32]>,
+    projected_g: Option<&'a [f32]>,
+    accel: Option<&'a [f32]>,
+    gyro: Option<&'a [f32]>,
+    command: Option<&'a [f32]>,
+    output: Option<&'a [f32]>,
+}
+
+const CHANNEL_CAP: usize = 1024;
+const FLUSH_EVERY: u64 = 100;
+
+pub struct StepLogger {
+    tx: Sender<Vec<u8>>,
+    worker: Option<thread::JoinHandle<()>>,
path: kinfer/rust/src/logger.rs
user: login: gitfish7
body: let's see if option is not needed (seems like it's not but try changing and verifing)
created_at: 2025-05-24T08:26:46Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,119 @@
+use std::{
+    fs::OpenOptions,
+    io::{BufWriter, Write},
+    path::{Path, PathBuf},
+    thread,
+};
+
+use crossbeam_channel::{bounded, Sender};
+use log::info;
+use serde::Serialize;
+
+#[derive(Serialize)]
+struct NdjsonStep<'a> {
+    step_id: u64,
+    t_us: u128,
+    joint_angles: Option<&'a [f32]>,
+    joint_vels: Option<&'a [f32]>,
+    projected_g: Option<&'a [f32]>,
+    accel: Option<&'a [f32]>,
+    gyro: Option<&'a [f32]>,
+    command: Option<&'a [f32]>,
+    output: Option<&'a [f32]>,
path: kinfer/rust/src/logger.rs
user: login: gitfish7
body: we should review if we need optional's here. is the idea that data is indeed "optional"? want to verify that if data is not present we should not panic and log a None on the json.
created_at: 2025-05-24T08:40:42Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,119 @@
+use std::{
+    fs::OpenOptions,
+    io::{BufWriter, Write},
+    path::{Path, PathBuf},
+    thread,
+};
+
+use crossbeam_channel::{bounded, Sender};
+use log::info;
+use serde::Serialize;
+
+#[derive(Serialize)]
+struct NdjsonStep<'a> {
+    step_id: u64,
+    t_us: u64,
+    joint_angles: Option<&'a [f32]>,
+    joint_vels: Option<&'a [f32]>,
+    projected_g: Option<&'a [f32]>,
+    accel: Option<&'a [f32]>,
+    gyro: Option<&'a [f32]>,
+    command: Option<&'a [f32]>,
+    output: Option<&'a [f32]>,
+}
+
+const CHANNEL_CAP: usize = 1024;
+const FLUSH_EVERY: u64 = 100;
+
+pub struct StepLogger {
+    tx: Sender<Vec<u8>>,
+    worker: Option<thread::JoinHandle<()>>,
+    next_id: std::sync::atomic::AtomicU64,
+}
+
+impl StepLogger {
+    pub fn new(path: impl AsRef<Path>) -> std::io::Result<Self> {
+        let path: PathBuf = path.as_ref().into();
+        if let Some(parent) = path.parent() {
+            std::fs::create_dir_all(parent)?;
+        }
+        info!("kinfer: logging to NDJSON: {}", path.display());
+
+        // I/O objects created here, but moved into the worker thread.
+        let file = OpenOptions::new().create(true).append(true).open(&path)?;
+        let mut bw = BufWriter::new(file);
+
+        // Bounded channel -> back-pressure capped at CHANNEL_CAP lines
+        let (tx, rx) = bounded::<Vec<u8>>(CHANNEL_CAP);
+
+        let worker = thread::spawn(move || {
+            let mut line_ctr: u64 = 0;
+            for msg in rx {
+                // drains until all senders dropped
+                let _ = bw.write_all(&msg);
+                line_ctr += 1;
+                if line_ctr % FLUSH_EVERY == 0 {
+                    let _ = bw.flush();
+                }
+            }
+            // Final flush on graceful shutdown
+            let _ = bw.flush();
+        });
+
+        Ok(Self {
+            tx,
+            worker: Some(worker),
+            next_id: std::sync::atomic::AtomicU64::new(0),
+        })
+    }
+
+    #[inline]
+    fn now_us() -> u128 {
+        std::time::SystemTime::now()
+            .duration_since(std::time::UNIX_EPOCH)
+            .unwrap()
+            .as_micros()
+    }
+
+    /// Non-blocking; drops a line if the channel is full.
+    pub fn log_step(
+        &self,
+        joint_angles: Option<&[f32]>,
+        joint_vels: Option<&[f32]>,
+        projected_g: Option<&[f32]>,
+        accel: Option<&[f32]>,
+        gyro: Option<&[f32]>,
+        command: Option<&[f32]>,
+        output: Option<&[f32]>,
+    ) {
+        let record = NdjsonStep {
+            step_id: self
+                .next_id
+                .fetch_add(1, std::sync::atomic::Ordering::Relaxed),
+            t_us: Self::now_us(),
path: kinfer/rust/src/logger.rs
user: login: Copilot
body: The `now_us()` helper returns a `u128` but `t_us` is declared as `u64`, causing a type mismatch. Cast the value (e.g., `t_us: Self::now_us() as u64`) or change `t_us` to `u128`.
```suggestion
            t_us: Self::now_us() as u64,
```
created_at: 2025-05-25T06:18:02Z
author_association: CONTRIBUTOR
diff_hunk: @@ -305,6 +311,27 @@ impl ModelRunner {
         let output_tensor = outputs[0].try_extract_tensor::<f32>()?;
         let carry_tensor = outputs[1].try_extract_tensor::<f32>()?;
 
+        // Log the step if needed
+        if let Some(lg) = &self.logger {
+            let joint_angles_opt = inputs.get("joint_angles").map(|a| a.as_slice().unwrap());
+            let joint_vels_opt = inputs.get("joint_angular_velocities").map(|a| a.as_slice().unwrap());
+            let projected_g_opt = inputs.get("projected_gravity").map(|a| a.as_slice().unwrap());
+            let accel_opt = inputs.get("accelerometer").map(|a| a.as_slice().unwrap());
+            let gyro_opt = inputs.get("gyroscope").map(|a| a.as_slice().unwrap());
+            let command_opt = inputs.get("command").map(|a| a.as_slice().unwrap());
+            let output_opt = Some(output_tensor.as_slice().unwrap());
path: kinfer/rust/src/model.rs
user: login: Copilot
body: Calling `unwrap()` on `as_slice()` can panic if the tensor isn't contiguous. Consider handling the `Option<&[f32]>` safely (e.g., propagate an error or skip logging that field).
```suggestion
            let joint_angles_opt = inputs.get("joint_angles").and_then(|a| a.as_slice());
            let joint_vels_opt = inputs.get("joint_angular_velocities").and_then(|a| a.as_slice());
            let projected_g_opt = inputs.get("projected_gravity").and_then(|a| a.as_slice());
            let accel_opt = inputs.get("accelerometer").and_then(|a| a.as_slice());
            let gyro_opt = inputs.get("gyroscope").and_then(|a| a.as_slice());
            let command_opt = inputs.get("command").and_then(|a| a.as_slice());
            let output_opt = output_tensor.as_slice();
```
created_at: 2025-05-25T06:18:02Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,119 @@
+use std::{
+    fs::OpenOptions,
+    io::{BufWriter, Write},
+    path::{Path, PathBuf},
+    thread,
+};
+
+use crossbeam_channel::{bounded, Sender};
+use log::info;
+use serde::Serialize;
+
+#[derive(Serialize)]
+struct NdjsonStep<'a> {
+    step_id: u64,
+    t_us: u128,
+    joint_angles: Option<&'a [f32]>,
+    joint_vels: Option<&'a [f32]>,
+    projected_g: Option<&'a [f32]>,
+    accel: Option<&'a [f32]>,
+    gyro: Option<&'a [f32]>,
+    command: Option<&'a [f32]>,
+    output: Option<&'a [f32]>,
+}
+
+const CHANNEL_CAP: usize = 1024;
+const FLUSH_EVERY: u64 = 100;
+
+pub struct StepLogger {
+    tx: Sender<Vec<u8>>,
+    worker: Option<thread::JoinHandle<()>>,
path: kinfer/rust/src/logger.rs
user: login: jingxiangmo
body: we would need option here right? as drop is implemented later:

```
/// Ensure the worker drains and flushes before program exit.
impl Drop for StepLogger {
    fn drop(&mut self) {
        // drop Sender -> channel closes
        if let Some(worker) = self.worker.take() {
            let _ = worker.join();
        }
    }
}
```

What would you recommend instead?
created_at: 2025-05-26T18:29:35Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,119 @@
+use std::{
+    fs::OpenOptions,
+    io::{BufWriter, Write},
+    path::{Path, PathBuf},
+    thread,
+};
+
+use crossbeam_channel::{bounded, Sender};
+use log::info;
+use serde::Serialize;
+
+#[derive(Serialize)]
+struct NdjsonStep<'a> {
+    step_id: u64,
+    t_us: u128,
+    joint_angles: Option<&'a [f32]>,
+    joint_vels: Option<&'a [f32]>,
+    projected_g: Option<&'a [f32]>,
+    accel: Option<&'a [f32]>,
+    gyro: Option<&'a [f32]>,
+    command: Option<&'a [f32]>,
+    output: Option<&'a [f32]>,
path: kinfer/rust/src/logger.rs
user: login: jingxiangmo
body: Hmmmm, suppose we're not logging data. What would you recommend the error handling be?
created_at: 2025-05-26T18:30:12Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,124 @@
+use std::{
+    fs::OpenOptions,
+    io::{BufWriter, Write},
+    path::{Path, PathBuf},
+    thread,
+};
+
+use crossbeam_channel::{bounded, Sender};
+use log::info;
+use serde::Serialize;
+
+#[derive(Serialize)]
+struct NdjsonStep<'a> {
+    step_id: u64,
+    t_us: u64,
+    joint_angles: Option<&'a [f32]>,
+    joint_vels: Option<&'a [f32]>,
+    projected_g: Option<&'a [f32]>,
+    accel: Option<&'a [f32]>,
+    gyro: Option<&'a [f32]>,
+    command: Option<&'a [f32]>,
+    output: Option<&'a [f32]>,
+}
+
+const CHANNEL_CAP: usize = 1024;
+const FLUSH_EVERY: u64 = 100;
path: kinfer/rust/src/logger.rs
user: login: codekansas
body: can you put comments about why these constants are the values they are?
created_at: 2025-05-26T21:04:56Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,124 @@
+use std::{
+    fs::OpenOptions,
+    io::{BufWriter, Write},
+    path::{Path, PathBuf},
+    thread,
+};
+
+use crossbeam_channel::{bounded, Sender};
+use log::info;
+use serde::Serialize;
+
+#[derive(Serialize)]
+struct NdjsonStep<'a> {
+    step_id: u64,
+    t_us: u64,
+    joint_angles: Option<&'a [f32]>,
+    joint_vels: Option<&'a [f32]>,
+    projected_g: Option<&'a [f32]>,
+    accel: Option<&'a [f32]>,
+    gyro: Option<&'a [f32]>,
+    command: Option<&'a [f32]>,
+    output: Option<&'a [f32]>,
+}
+
+const CHANNEL_CAP: usize = 1024;
+const FLUSH_EVERY: u64 = 100;
+
+pub struct StepLogger {
+    tx: Option<Sender<Vec<u8>>>,
+    worker: Option<thread::JoinHandle<()>>,
+    next_id: std::sync::atomic::AtomicU64,
+}
+
+impl StepLogger {
+    pub fn new(path: impl AsRef<Path>) -> std::io::Result<Self> {
+        let path: PathBuf = path.as_ref().into();
+        if let Some(parent) = path.parent() {
+            std::fs::create_dir_all(parent)?;
+        }
+        info!("kinfer: logging to NDJSON: {}", path.display());
+
+        // I/O objects created here, but moved into the worker thread.
+        let file = OpenOptions::new().create(true).append(true).open(&path)?;
+        let mut bw = BufWriter::new(file);
+
+        // Bounded channel -> back-pressure capped at CHANNEL_CAP lines
+        let (tx, rx) = bounded::<Vec<u8>>(CHANNEL_CAP);
+
+        let worker = thread::spawn(move || {
+            let mut line_ctr: u64 = 0;
+            for msg in rx {
+                // drains until all senders dropped
+                let _ = bw.write_all(&msg);
+                line_ctr += 1;
+                if line_ctr % FLUSH_EVERY == 0 {
+                    let _ = bw.flush();
+                }
+            }
+            // Final flush on graceful shutdown
+            let _ = bw.flush();
+        });
path: kinfer/rust/src/logger.rs
user: login: codekansas
body: question for @gitfish7 maybe - why use `std::io::BufWriter` instead of `tokio::io::BufWriter`?
created_at: 2025-05-26T21:06:27Z
author_association: MEMBER
diff_hunk: @@ -61,6 +62,7 @@ pub struct ModelRunner {
     step_session: Session,
     metadata: ModelMetadata,
     provider: Arc<dyn ModelProvider>,
+    logger: Option<Arc<crate::logger::StepLogger>>,
path: kinfer/rust/src/model.rs
user: login: codekansas
body: nit, put

```
use logger::StepLogger
```

in import instead of this
created_at: 2025-05-26T21:08:02Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,124 @@
+use std::{
+    fs::OpenOptions,
+    io::{BufWriter, Write},
+    path::{Path, PathBuf},
+    thread,
+};
+
+use crossbeam_channel::{bounded, Sender};
+use log::info;
+use serde::Serialize;
+
+#[derive(Serialize)]
+struct NdjsonStep<'a> {
+    step_id: u64,
+    t_us: u64,
+    joint_angles: Option<&'a [f32]>,
+    joint_vels: Option<&'a [f32]>,
+    projected_g: Option<&'a [f32]>,
+    accel: Option<&'a [f32]>,
+    gyro: Option<&'a [f32]>,
+    command: Option<&'a [f32]>,
+    output: Option<&'a [f32]>,
+}
+
+const CHANNEL_CAP: usize = 1024;
+const FLUSH_EVERY: u64 = 100;
path: kinfer/rust/src/logger.rs
user: login: alik-git
body: Added comments, also added warnings if messages are ever dropped.
created_at: 2025-05-26T21:58:08Z
author_association: CONTRIBUTOR
diff_hunk: @@ -61,6 +62,7 @@ pub struct ModelRunner {
     step_session: Session,
     metadata: ModelMetadata,
     provider: Arc<dyn ModelProvider>,
+    logger: Option<Arc<crate::logger::StepLogger>>,
path: kinfer/rust/src/model.rs
user: login: alik-git
body: done
created_at: 2025-05-26T21:58:17Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,135 @@
+use std::{
+    fs::OpenOptions,
+    io::{BufWriter, Write},
+    path::{Path, PathBuf},
+    sync::atomic::{AtomicU64, Ordering},
+    thread,
+};
+
+use crossbeam_channel::{bounded, Sender};
+use log::{info, warn};
+use serde::Serialize;
+
+#[derive(Serialize)]
+struct NdjsonStep<'a> {
+    step_id: u64,
+    t_us: u64,
+    joint_angles: Option<&'a [f32]>,
+    joint_vels: Option<&'a [f32]>,
+    projected_g: Option<&'a [f32]>,
+    accel: Option<&'a [f32]>,
+    gyro: Option<&'a [f32]>,
+    command: Option<&'a [f32]>,
+    output: Option<&'a [f32]>,
+}
+
+// Channel capacity for non-blocking logging.
+// ~1000 entires at 50Hz is 20 seconds of buffering.
+// Warns if messages are dropped due to full buffer.
+const CHANNEL_CAP: usize = 1024;
+
+// Flush buffered writes every 100 log entries.
+// At 50Hz control frequency, this flushes every 2 seconds.
+const FLUSH_EVERY: u64 = 100;
+
+pub struct StepLogger {
+    tx: Option<Sender<Vec<u8>>>,
+    worker: Option<thread::JoinHandle<()>>,
+    next_id: AtomicU64,
+}
+
+impl StepLogger {
+    pub fn new(path: impl AsRef<Path>) -> std::io::Result<Self> {
+        let path: PathBuf = path.as_ref().into();
+        if let Some(parent) = path.parent() {
+            std::fs::create_dir_all(parent)?;
+        }
+        info!("kinfer: logging to NDJSON: {}", path.display());
+
+        // I/O objects created here, but moved into the worker thread.
+        let file = OpenOptions::new().create(true).append(true).open(&path)?;
+        let mut bw = BufWriter::new(file);
+
+        // Bounded channel -> back-pressure capped at CHANNEL_CAP lines
+        let (tx, rx) = bounded::<Vec<u8>>(CHANNEL_CAP);
+
+        let worker = thread::spawn(move || {
+            let mut line_ctr: u64 = 0;
+            for msg in rx {
+                // drains until all senders dropped
+                let _ = bw.write_all(&msg);
+                line_ctr += 1;
+                if line_ctr % FLUSH_EVERY == 0 {
+                    let _ = bw.flush();
+                }
+            }
+            // Final flush on graceful shutdown
+            let _ = bw.flush();
+        });
+
+        Ok(Self {
+            tx: Some(tx),
+            worker: Some(worker),
+            next_id: AtomicU64::new(0),
+        })
+    }
+
+    #[inline]
+    fn now_us() -> u128 {
path: kinfer/rust/src/logger.rs
user: login: alik-git
body: @AJ-Williams1 here
created_at: 2025-06-02T21:56:54Z
author_association: CONTRIBUTOR

PR comments:
user: login: gitfish7
created_at: 2025-05-24T08:41:33Z
body: Thanks for the validation results! If this is something that is helpful, we should merge and we can cleanup later.
author_association: CONTRIBUTOR
user: login: jingxiangmo
created_at: 2025-05-26T19:20:50Z
body: Will need refactoring for the Options, but should not be blocking
author_association: CONTRIBUTOR
user: login: nfreq
created_at: 2025-05-27T00:02:28Z
body: don't forget to bump the package version
author_association: CONTRIBUTOR
user: login: codekansas
created_at: 2025-05-27T01:32:33Z
body: random idea, can we change from json to bjson? we're probably io-bound

edit: seems like serde prefers cbor to bjson - this would make the most sense to me to use
author_association: MEMBER
user: login: nfreq
created_at: 2025-05-27T01:51:53Z
body: small improvement to plot script, --save and --skip n seconds (skips first section of data)
author_association: CONTRIBUTOR