Repository: nathanbabcock/ffmpeg-sidecar (language: rust)

Issue:
Would it be possible to include the frame timestamp in `OutputVideoFrame`?
First, I want to thank you for making this crate. ffmpeg is a powerful tool but it can be a major pain to work with, especially via the rust bindings available in many crates. Dealing with the CLI is slightly better but still far from ergonomic. This crate makes it so much better!

I was wondering if ffmpeg emits the frame timestamp when outputting `rawvideo`, and if so if this data can be captured and included in `OutputVideoFrame`. I'm using your library to overlay images on top off the video frames and for aligning the images and the video frame it be a lot easier if I had access to each frame's timestamp. The `Progress(FfmpegProgress)` event does not get emitted for every decoded frame so it's not usable for this purpose.


Discussion / hints:
Hey @avsaase. Great suggestion. I'll make a PR for this feature later this week. After that you can let me know if it works for your use case.

Fix patch:
diff --git a/src/event.rs b/src/event.rs
index 8e7e05c..0cfab6a 100644
--- a/src/event.rs
+++ b/src/event.rs
@@ -47,6 +47,8 @@ pub struct AVStream {
   pub width: u32,
   /// Height in pixels
   pub height: u32,
+  /// Framerate in frames per second
+  pub fps: f32,
   /// The index of the input or output that this stream belongs to
   pub parent_index: usize,
   /// The stderr line that this stream was parsed from
@@ -97,11 +99,23 @@ pub struct FfmpegProgress {
 
 #[derive(Clone, PartialEq)]
 pub struct OutputVideoFrame {
+  /// The width of this video frame in pixels
   pub width: u32,
+  /// The height of this video frame in pixels
   pub height: u32,
+  /// The pixel format of the video frame, corresponding to the chosen
+  /// `-pix_fmt` FFmpeg parameter.
   pub pix_fmt: String,
+  /// The index of the FFmpeg output stream that emitted this frame.
+  /// In a typical case, there is only one output stream and this will be 0.
   pub output_index: u32,
+  /// Raw image frame data. The layout of the pixels in memory depends on
+  /// `width`, `height`, and `pix_fmt`.
   pub data: Vec<u8>,
+  /// Index of current frame, starting at 0 and monotonically increasing by 1
+  pub frame_num: u32,
+  /// Output frame timestamp in seconds
+  pub timestamp: f32,
 }
 
 impl std::fmt::Debug for OutputVideoFrame {
diff --git a/src/iter.rs b/src/iter.rs
index 0f27b2b..60c70c7 100644
--- a/src/iter.rs
+++ b/src/iter.rs
@@ -196,12 +196,17 @@ pub fn spawn_stdout_thread(
     }
 
     // Read into buffers
+    let num_buffers = buffers.len();
     let mut buffer_index = (0..buffers.len()).cycle();
     let mut reader = BufReader::new(stdout);
+    let mut frame_num = 0;
     loop {
       let i = buffer_index.next().unwrap();
       let stream = &output_streams[i];
       let buffer = &mut buffers[i];
+      let output_frame_num = frame_num / num_buffers;
+      let timestamp = output_frame_num as f32 / stream.fps;
+      frame_num += 1;
 
       // Handle two scenarios:
       match stream.format.as_str() {
@@ -214,6 +219,8 @@ pub fn spawn_stdout_thread(
               pix_fmt: stream.pix_fmt.clone(),
               output_index: i as u32,
               data: buffer.clone(),
+              frame_num: output_frame_num as u32,
+              timestamp,
             }))
             .ok(),
           Err(e) => match e.kind() {
diff --git a/src/log_parser.rs b/src/log_parser.rs
index 328ba51..26cfb41 100644
--- a/src/log_parser.rs
+++ b/src/log_parser.rs
@@ -232,6 +232,7 @@ pub fn try_parse_output(mut string: &str) -> Option<FfmpegOutput> {
 /// assert!(stream.pix_fmt == "rgb24");
 /// assert!(stream.width == 320);
 /// assert!(stream.height == 240);
+/// assert!(stream.fps == 25.0);
 /// assert!(stream.parent_index == 0);
 /// ```
 ///
@@ -245,6 +246,7 @@ pub fn try_parse_output(mut string: &str) -> Option<FfmpegOutput> {
 /// assert!(stream.pix_fmt == "yuv444p");
 /// assert!(stream.width == 320);
 /// assert!(stream.height == 240);
+/// assert!(stream.fps == 25.0);
 /// assert!(stream.parent_index == 1);
 /// ```
 pub fn try_parse_stream(mut string: &str) -> Option<AVStream> {
@@ -278,17 +280,28 @@ pub fn try_parse_stream(mut string: &str) -> Option<AVStream> {
     .split(&[' ', '(']) // trim trailing junk like "(tv, progressive)"
     .next()?
     .to_string();
+
   let dims = comma_iter.next()?.split_whitespace().next()?;
   let mut dims_iter = dims.split('x');
   let width = dims_iter.next()?.parse::<u32>().ok()?;
   let height = dims_iter.next()?.parse::<u32>().ok()?;
 
+  let fps = string
+    .split("fps,")
+    .next()?
+    .trim()
+    .split_whitespace()
+    .last()?
+    .parse()
+    .ok()?;
+
   Some(AVStream {
     parent_index,
     format,
     pix_fmt,
     width,
     height,
+    fps,
     raw_log_message,
   })
 }


Test patch:
diff --git a/src/test.rs b/src/test.rs
index 48af6c3..80d56a8 100644
--- a/src/test.rs
+++ b/src/test.rs
@@ -1,5 +1,9 @@
 use crate::{command::FfmpegCommand, event::FfmpegEvent};
 
+fn approx_eq(a: f32, b: f32, error: f32) -> bool {
+  (a - b).abs() < error
+}
+
 #[test]
 fn test_frame_count() {
   let fps = 1;
@@ -176,3 +180,25 @@ fn test_quit() {
   let count = child.iter().unwrap().filter_progress().count();
   assert!(count <= 1);
 }
+
+#[test]
+fn test_frame_timestamp() {
+  let mut last_timestamp: Option<f32> = None;
+  FfmpegCommand::new()
+    .format("lavfi")
+    .input("testsrc=duration=1:rate=10")
+    .rawvideo()
+    .spawn()
+    .unwrap()
+    .iter()
+    .unwrap()
+    .filter_frames()
+    .for_each(|frame| {
+      match last_timestamp {
+        None => assert!(frame.timestamp == 0.0),
+        Some(last_timestamp) => assert!(approx_eq(frame.timestamp, last_timestamp + 0.1, 0.001)),
+      }
+      last_timestamp = Some(frame.timestamp);
+    });
+  assert!(approx_eq(last_timestamp.unwrap(), 0.9, 0.001));
+}