Repository: Kitware/kwiver (language: python)

Issue:
Add audio stream copying
This PR adds the capability to copy any number of audio streams from an FFmpeg video input to output. No interpretation of the audio data is available, but the information will not be lost during transcoding.

This functionality has been manually tested on a couple random video clips (`.ts` and `.mp4`). None of our existing test videos have audio; not sure to what extent we should put effort into constructing a short test clip to put in the repo for unit testing, or if this is a niche enough feature that it's not worth the time / additional file size.


Discussion / hints:
ghostflow-check-master check fails, needs to be fixed first. A short test video with multiple audio streams recommended.

Fix patch:
diff --git a/arrows/ffmpeg/ffmpeg_video_output.cxx b/arrows/ffmpeg/ffmpeg_video_output.cxx
index a417cf06cf..35954f947e 100644
--- a/arrows/ffmpeg/ffmpeg_video_output.cxx
+++ b/arrows/ffmpeg/ffmpeg_video_output.cxx
@@ -511,8 +511,6 @@ ::open_video_state(
   }
   output_format = format_context->oformat;
 
-  // Set timestamp value to start at
-  format_context->output_ts_offset = settings.start_timestamp;
   format_context->flags |= AVFMT_FLAG_AUTO_BSF;
   format_context->flags |= AVFMT_FLAG_GENPTS;
 
@@ -965,15 +963,6 @@ ::add_uninterpreted_data( vital::video_uninterpreted_data const& misc_data )
       av_packet_rescale_ts(
         tmp_packet.get(), stream.settings.time_base, stream.stream->time_base );
 
-      // Adjust for any global timestamp offset
-      auto const counter_offset =
-        av_rescale_q(
-          format_context->output_ts_offset,
-          AVRational{ 1, AV_TIME_BASE },
-          stream.stream->time_base );
-      tmp_packet->dts -= counter_offset;
-      tmp_packet->pts -= counter_offset;
-
       // Write the packet
       throw_error_code(
         av_interleaved_write_frame( format_context.get(), tmp_packet.get() ),
@@ -1002,6 +991,18 @@ ::write_next_packet()
   }
   throw_error_code( err, "Could not get next packet from encoder" );
 
+  // Adjust for any global timestamp offset
+  if( video_settings.start_timestamp != AV_NOPTS_VALUE )
+  {
+    auto const offset =
+      av_rescale_q(
+        video_settings.start_timestamp,
+        AVRational{ 1, AV_TIME_BASE },
+        video_stream->time_base );
+    packet->dts += offset;
+    packet->pts += offset;
+  }
+
   // Succeeded; write to file
   throw_error_code(
     av_interleaved_write_frame( format_context.get(), packet.get() ),
diff --git a/arrows/ffmpeg/ffmpeg_video_settings.h b/arrows/ffmpeg/ffmpeg_video_settings.h
index f16c7cb2ee..06075bebb8 100644
--- a/arrows/ffmpeg/ffmpeg_video_settings.h
+++ b/arrows/ffmpeg/ffmpeg_video_settings.h
@@ -71,9 +71,9 @@ struct KWIVER_ALGO_FFMPEG_EXPORT ffmpeg_video_settings
   /// guaranteed to determine the time base in the output video.
   AVRational time_base;
 
-  /// Desired PTS of the first video frame, in AV_TIME_BASE units
-  /// (microseconds). For some formats, the actual first PTS may exceed this
-  /// value to ensure non-negative PTS or DTS.
+  /// Start time of the input video, in AV_TIME_BASE units (microseconds).
+  /// This information is necessary for copied and newly-encoded packets to sync
+  /// correctly.
   int64_t start_timestamp;
 
   /// FFmpeg-defined string options passed to the video codec.


Test patch:
diff --git a/arrows/ffmpeg/tests/test_video_output_ffmpeg.cxx b/arrows/ffmpeg/tests/test_video_output_ffmpeg.cxx
index 3ccc4ebc12..42c5b8eeb2 100644
--- a/arrows/ffmpeg/tests/test_video_output_ffmpeg.cxx
+++ b/arrows/ffmpeg/tests/test_video_output_ffmpeg.cxx
@@ -364,7 +364,7 @@ TEST_F ( ffmpeg_video_output, round_trip_audio )
 // Similar to round_trip_direct, but for a test video with an audio stream.
 TEST_F ( ffmpeg_video_output, round_trip_audio_direct )
 {
-  auto const src_path = data_dir + "/" + short_video_name;
+  auto const src_path = data_dir + "/" + audio_video_name;
   auto const tmp_path =
     kwiver::testing::temp_file_name( "test-ffmpeg-output-", ".ts" );