Repository: tvheadend/tvheadend
Pull request #1792: update vaapi transcoding as recommended by ffmpeg 6.1.1/doc/examples/…
Author: login: uknunknown
Labels: name: IMPROVEMENT
name: CODE
Created: 2024-12-30T03:03:57Z | Merged: 2025-05-19T13:21:52Z

- update vaapi transcoding as recommended by ffmpeg 6.1.1/doc/examples/vaapi_*.c
- src/transcoding/transcode/hwaccels/vaapi was completely redesigned


Diff:
diff --git a/src/libav.h b/src/libav.h
index c73b61089e..902b305d85 100644
--- a/src/libav.h
+++ b/src/libav.h
@@ -54,6 +54,13 @@ This list must be updated every time we use a new AV_CODEC_ID
 #define AV_CODEC_ID_DVB_TELETEXT CODEC_ID_DVB_TELETEXT
 #endif
 
+// Enable new transcoding starting ffmpeg 4
+#if LIBAVCODEC_VERSION_MAJOR < 58
+#define ENABLE_FFMPEG4_TRANSCODING 0
+#else
+#define ENABLE_FFMPEG4_TRANSCODING 1
+#endif
+
 enum AVCodecID streaming_component_type2codec_id(streaming_component_type_t type);
 streaming_component_type_t codec_id2streaming_component_type(enum AVCodecID id);
 void libav_set_loglevel(void);
diff --git a/src/transcoding/transcode/hwaccels/hwaccels.c b/src/transcoding/transcode/hwaccels/hwaccels.c
index 892bbbc2f6..57524c44e2 100644
--- a/src/transcoding/transcode/hwaccels/hwaccels.c
+++ b/src/transcoding/transcode/hwaccels/hwaccels.c
@@ -213,13 +213,43 @@ hwaccels_get_sharpness_filter(AVCodecContext *avctx, int value, char *filter, si
 
 /* encoding ================================================================= */
 
+#if ENABLE_FFMPEG4_TRANSCODING
 int
+hwaccels_initialize_encoder_from_decoder(const AVCodecContext *iavctx, AVCodecContext *oavctx)
+{
+    switch (iavctx->pix_fmt) {
+        case AV_PIX_FMT_VAAPI:
+            /* we need to ref hw_frames_ctx of decoder to initialize encoder's codec.
+            Only after we get a decoded frame, can we obtain its hw_frames_ctx */
+            oavctx->hw_frames_ctx = av_buffer_ref(iavctx->hw_frames_ctx);
+            if (!oavctx->hw_frames_ctx) {
+                return AVERROR(ENOMEM);
+            }
+            return 0;
+        case AV_PIX_FMT_YUV420P:
+            break;
+        default:
+            break;
+    }
+    return 0;
+}
+#endif
+
+int
+#if ENABLE_FFMPEG4_TRANSCODING
+hwaccels_encode_setup_context(AVCodecContext *avctx)
+#else
 hwaccels_encode_setup_context(AVCodecContext *avctx, int low_power)
+#endif
 {
     switch (avctx->pix_fmt) {
 #if ENABLE_VAAPI
         case AV_PIX_FMT_VAAPI:
+#if ENABLE_FFMPEG4_TRANSCODING
+            return vaapi_encode_setup_context(avctx);
+#else
             return vaapi_encode_setup_context(avctx, low_power);
+#endif
 #endif
         default:
             break;
diff --git a/src/transcoding/transcode/hwaccels/hwaccels.h b/src/transcoding/transcode/hwaccels/hwaccels.h
index ca129ef883..ed04f67485 100644
--- a/src/transcoding/transcode/hwaccels/hwaccels.h
+++ b/src/transcoding/transcode/hwaccels/hwaccels.h
@@ -23,7 +23,7 @@
 
 
 #include "tvheadend.h"
-
+#include "../internals.h"
 #include <libavcodec/avcodec.h>
 
 
@@ -51,9 +51,17 @@ hwaccels_get_sharpness_filter(AVCodecContext *avctx, int value, char *filter, si
 
 
 /* encoding ================================================================= */
+#if ENABLE_FFMPEG4_TRANSCODING
+int
+hwaccels_initialize_encoder_from_decoder(const AVCodecContext *iavctx, AVCodecContext *oavctx);
+#endif
 
 int
+#if ENABLE_FFMPEG4_TRANSCODING
+hwaccels_encode_setup_context(AVCodecContext *avctx);
+#else
 hwaccels_encode_setup_context(AVCodecContext *avctx, int low_power);
+#endif
 
 void
 hwaccels_encode_close_context(AVCodecContext *avctx);
diff --git a/src/transcoding/transcode/hwaccels/vaapi.c b/src/transcoding/transcode/hwaccels/vaapi.c
index f3971f3cde..53bc2f3791 100644
--- a/src/transcoding/transcode/hwaccels/vaapi.c
+++ b/src/transcoding/transcode/hwaccels/vaapi.c
@@ -27,7 +27,41 @@
 
 #include <va/va.h>
 
+#if ENABLE_FFMPEG4_TRANSCODING
+typedef struct tvh_vaapi_context_t {
+    int width;
+    int height;
+    AVBufferRef *hw_device_ref;
+    AVBufferRef *hw_frame_ref;
+} TVHVAContext;
+
+
+static void
+tvhva_done()
+{
+    /* nothing to do */
+}
+
 
+/* TVHVAContext ============================================================= */
+
+static void
+tvhva_context_destroy(TVHVAContext *self)
+{
+    if (self) {
+        if (self->hw_device_ref) {
+            av_buffer_unref(&self->hw_device_ref);
+            self->hw_device_ref = NULL;
+        }
+        if (self->hw_frame_ref) {
+            av_buffer_unref(&self->hw_frame_ref);
+            self->hw_frame_ref = NULL;
+        }
+        free(self);
+        self = NULL;
+    }
+}
+#else
 typedef struct tvh_vaapi_device {
     char *hw_device_name;
     AVBufferRef *hw_device_ref;
@@ -569,6 +603,7 @@ tvhva_context_create(const char *logpref,
     }
     return self;
 }
+#endif
 
 
 /* decoding ================================================================= */
@@ -588,6 +623,34 @@ vaapi_decode_setup_context(AVCodecContext *avctx)
 {
     TVHContext *ctx = avctx->opaque;
 
+#if ENABLE_FFMPEG4_TRANSCODING
+    TVHVAContext *self = NULL;
+    int ret = -1;
+
+    if (!(self = calloc(1, sizeof(TVHVAContext)))) {
+        tvherror(LS_VAAPI, "Decode: Failed to allocate VAAPI context (TVHVAContext)");
+        return AVERROR(ENOMEM);
+    }
+
+    /* Open VAAPI device and create an AVHWDeviceContext for it*/
+    if ((ret = av_hwdevice_ctx_create(&self->hw_device_ref, AV_HWDEVICE_TYPE_VAAPI, ctx->hw_accel_device, NULL, 0)) < 0) {
+        tvherror(LS_VAAPI, "Decode: Failed to Open VAAPI device and create an AVHWDeviceContext for device: "
+                            "%s with error code: %s", 
+                            ctx->hw_accel_device, av_err2str(ret));
+        return ret;
+    }
+
+    /* set hw_frames_ctx for decoder's AVCodecContext */
+    avctx->hw_device_ctx = av_buffer_ref(self->hw_device_ref);
+    if (!avctx->hw_device_ctx) {
+        tvherror(LS_VAAPI, "Decode: Failed to create a hardware device reference for device: %s.", 
+                        ctx->hw_accel_device);
+        return AVERROR(ENOMEM);
+    }
+    ctx->hw_accel_ictx = self;
+    avctx->get_buffer2 = vaapi_get_buffer2;
+    avctx->pix_fmt = AV_PIX_FMT_VAAPI;
+#else
     if (!(ctx->hw_accel_ictx =
           tvhva_context_create("decode", avctx, VAEntrypointVLD))) {
         return -1;
@@ -597,6 +660,7 @@ vaapi_decode_setup_context(AVCodecContext *avctx)
 #if LIBAVCODEC_VERSION_MAJOR < 60
     avctx->thread_safe_callbacks = 0;
 #endif
+#endif 
 
     return 0;
 }
@@ -607,6 +671,12 @@ vaapi_decode_close_context(AVCodecContext *avctx)
 {
     TVHContext *ctx = avctx->opaque;
 
+#if ENABLE_FFMPEG4_TRANSCODING
+    if (avctx->hw_device_ctx) {
+        av_buffer_unref(&avctx->hw_device_ctx);
+        avctx->hw_device_ctx = NULL;
+    }
+#endif
     tvhva_context_destroy(ctx->hw_accel_ictx);
 }
 
@@ -645,7 +715,69 @@ vaapi_get_sharpness_filter(AVCodecContext *avctx, int value, char *filter, size_
 
 /* encoding ================================================================= */
 
+#if ENABLE_FFMPEG4_TRANSCODING
+// lifted from ffmpeg-6.1.1/doc/examples/vaapi_encode.c line 41
+static int set_hwframe_ctx(AVCodecContext *ctx, AVBufferRef *hw_device_ctx)
+{
+    AVBufferRef *hw_frames_ref;
+    AVHWFramesContext *frames_ctx = NULL;
+    int err = 0;
+
+    if (!(hw_frames_ref = av_hwframe_ctx_alloc(hw_device_ctx))) {
+        tvherror(LS_VAAPI, "Encode: Failed to create VAAPI frame context.");
+        return AVERROR(ENOMEM);
+    }
+    frames_ctx = (AVHWFramesContext *)(hw_frames_ref->data);
+    frames_ctx->format    = AV_PIX_FMT_VAAPI;
+    frames_ctx->sw_format = AV_PIX_FMT_NV12;
+    frames_ctx->width     = ctx->width;
+    frames_ctx->height    = ctx->height;
+    frames_ctx->initial_pool_size = 20;
+    if ((err = av_hwframe_ctx_init(hw_frames_ref)) < 0) {
+        tvherror(LS_VAAPI, "Encode: Failed to initialize VAAPI frame context."
+                "Error code: %s",av_err2str(err));
+        av_buffer_unref(&hw_frames_ref);
+        return err;
+    }
+    ctx->hw_frames_ctx = av_buffer_ref(hw_frames_ref);
+    if (!ctx->hw_frames_ctx) {
+        err = AVERROR(ENOMEM);
+        tvherror(LS_VAAPI, "Encode: Failed to create a hardware device reference."
+                "Error code: %s",av_err2str(err));
+    }
+    av_buffer_unref(&hw_frames_ref);
+    return err;
+}
+#endif
+
 int
+#if ENABLE_FFMPEG4_TRANSCODING
+vaapi_encode_setup_context(AVCodecContext *avctx)
+{
+    TVHContext *ctx = avctx->opaque;
+    TVHVAContext *self = NULL;
+    int ret = 0;
+
+    if (!(self = calloc(1, sizeof(TVHVAContext)))) {
+        tvherror(LS_VAAPI, "Encode: Failed to allocate VAAPI context (TVHVAContext)");
+        return AVERROR(ENOMEM);
+    }
+
+    /* Open VAAPI device and create an AVHWDeviceContext for it*/
+    if ((ret = av_hwdevice_ctx_create(&self->hw_frame_ref, AV_HWDEVICE_TYPE_VAAPI, NULL, NULL, 0)) < 0) {
+        tvherror(LS_VAAPI, "Encode: Failed to open VAAPI device and create an AVHWDeviceContext for it."
+                "Error code: %s",av_err2str(ret));
+        return ret;
+    }
+
+    /* set hw_frames_ctx for encoder's AVCodecContext */
+    if ((ret = set_hwframe_ctx(avctx, self->hw_frame_ref)) < 0) {
+        tvherror(LS_VAAPI, "Encode: Failed to set hwframe context."
+                "Error code: %s",av_err2str(ret));
+        return ret;
+    }
+    ctx->hw_device_octx = av_buffer_ref(self->hw_frame_ref);
+#else
 vaapi_encode_setup_context(AVCodecContext *avctx, int low_power)
 {
     TVHContext *ctx = avctx->opaque;
@@ -668,6 +800,7 @@ vaapi_encode_setup_context(AVCodecContext *avctx, int low_power)
         return AVERROR(ENOMEM);
     }
     tvhva_context_destroy(hwaccel_context);
+#endif
     return 0;
 }
 
@@ -678,6 +811,16 @@ vaapi_encode_close_context(AVCodecContext *avctx)
     TVHContext *ctx = avctx->opaque;
     av_buffer_unref(&ctx->hw_device_octx);
     ctx->hw_device_octx = NULL;
+#if ENABLE_FFMPEG4_TRANSCODING
+    if (avctx->hw_device_ctx) {
+        av_buffer_unref(&avctx->hw_device_ctx);
+        avctx->hw_device_ctx = NULL;
+    }
+    if (avctx->hw_frames_ctx) {
+        av_buffer_unref(&avctx->hw_frames_ctx);
+        avctx->hw_frames_ctx = NULL;
+    }
+#endif
 }
 
 
diff --git a/src/transcoding/transcode/hwaccels/vaapi.h b/src/transcoding/transcode/hwaccels/vaapi.h
index 72a2c4babe..57417b0527 100644
--- a/src/transcoding/transcode/hwaccels/vaapi.h
+++ b/src/transcoding/transcode/hwaccels/vaapi.h
@@ -51,7 +51,11 @@ vaapi_get_sharpness_filter(AVCodecContext *avctx, int value, char *filter, size_
 /* encoding ================================================================= */
 
 int
+#if ENABLE_FFMPEG4_TRANSCODING
+vaapi_encode_setup_context(AVCodecContext *avctx);
+#else
 vaapi_encode_setup_context(AVCodecContext *avctx, int low_power);
+#endif
 
 void
 vaapi_encode_close_context(AVCodecContext *avctx);
diff --git a/src/transcoding/transcode/video.c b/src/transcoding/transcode/video.c
index b75e71dc1c..5069fb5e41 100644
--- a/src/transcoding/transcode/video.c
+++ b/src/transcoding/transcode/video.c
@@ -226,10 +226,37 @@ tvh_video_context_open_encoder(TVHContext *self, AVDictionary **opts)
 #if ENABLE_HWACCELS
     self->oavctx->coded_width = self->oavctx->width;
     self->oavctx->coded_height = self->oavctx->height;
+#if ENABLE_FFMPEG4_TRANSCODING
+    // hwaccel is the user input for Hardware acceleration from Codec parameteres
+    int hwaccel = -1;
+    if ((hwaccel = tvh_codec_profile_video_get_hwaccel(self->profile)) < 0) {
+        return -1;
+    }
+    if (_video_filters_hw_pix_fmt(self->oavctx->pix_fmt)){
+        // encoder is hw accelerated
+        if (hwaccel) {
+            // decoder is hw accelerated 
+            // --> we initialize encoder from decoder as recomanded in: 
+            // ffmpeg-6.1.1/doc/examples/vaapi_transcode.c line 169
+            if (hwaccels_initialize_encoder_from_decoder(self->iavctx, self->oavctx)) {
+                return -1;
+            }
+        }
+        else {
+            // decoder is sw
+            // --> we initialize as recommended in:
+            // ffmpeg-6.1.1/doc/examples/vaapi_encode.c line 145
+            if (hwaccels_encode_setup_context(self->oavctx)) {
+                return -1;
+            }
+        }
+    }
+#else
     if (hwaccels_encode_setup_context(self->oavctx, self->profile->low_power)) {
         return -1;
     }
-#endif
+#endif // from ENABLE_FFMPEG4_TRANSCODING
+#endif // from ENABLE_HWACCELS
 
     // XXX: is this a safe assumption?
     if (!self->iavctx->framerate.num) {


Reviews:
user: login: chewitt
state: COMMENTED
author_association: MEMBER
submitted_at: 2024-12-30T09:28:49Z
user: login: uknunknown
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2024-12-30T18:03:05Z
user: login: Flole998
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-01-01T03:42:39Z
user: login: Flole998
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-01-01T03:45:13Z
user: login: uknunknown
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-01-03T02:46:35Z
user: login: uknunknown
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-01-03T02:58:01Z
user: login: uknunknown
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-01-04T03:59:00Z
user: login: uknunknown
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-01-05T05:22:48Z
user: login: uknunknown
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-01-12T03:52:42Z
user: login: uknunknown
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-02-22T19:00:09Z
user: login: uknunknown
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-02-22T19:00:51Z
user: login: Flole998
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-18T20:49:03Z
user: login: uknunknown
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-19T00:31:16Z
user: login: Flole998
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-19T12:52:25Z

Review comments:
diff_hunk: @@ -54,6 +54,13 @@ This list must be updated every time we use a new AV_CODEC_ID
 #define AV_CODEC_ID_DVB_TELETEXT CODEC_ID_DVB_TELETEXT
 #endif
 
+// Enable new transcoding starting ffmpeg 6
+#if LIBAVCODEC_VERSION_MAJOR < 60
+#define ENABLE_NEW_TRANSCODING 0
path: src/libav.h
user: login: chewitt
body: Can we not use the `#if LIBAVCODEC_VERSION_MAJOR < 60` test everywhere instead of adding a new variable? I'm generally against calling things "new" in code because the new method is inevitably superseded and becomes another old method at some point requiring a future "new2" or "nextgen" naming silliness.
created_at: 2024-12-30T09:28:49Z
author_association: MEMBER
diff_hunk: @@ -54,6 +54,13 @@ This list must be updated every time we use a new AV_CODEC_ID
 #define AV_CODEC_ID_DVB_TELETEXT CODEC_ID_DVB_TELETEXT
 #endif
 
+// Enable new transcoding starting ffmpeg 6
+#if LIBAVCODEC_VERSION_MAJOR < 60
+#define ENABLE_NEW_TRANSCODING 0
path: src/libav.h
user: login: uknunknown
body: @chewitt let me clarify why I choose this solution (maybe not the best): I don't know how far back this new methodology of transcoding works ... so in order to easily test and update later on, is better to update a single line that will allow users to test functionality. Maybe the name was not the best option ... frankly I am open to any naming convention. ENABLE_TRANSCODING_V2 ? this is a little mode future proof. 
More importantly: @Flole998 do you think the failures are real failures? 
created_at: 2024-12-30T18:03:05Z
author_association: CONTRIBUTOR
diff_hunk: @@ -164,7 +164,11 @@ struct tvh_context {
     char *hw_accel_device;
     AVBufferRef *hw_device_ref;
     void *hw_accel_ictx;
+#if ENABLE_NEW_TRANSCODING
+    AVBufferRef *hw_frame_octx;
path: src/transcoding/transcode/internals.h
user: login: Flole998
body: You are basically just renaming this and introduce a bunch of preprocessor-if-directives for this? That's about the worst thing you could do from a maintainability point of view.
created_at: 2025-01-01T03:42:39Z
author_association: MEMBER
diff_hunk: @@ -255,13 +282,21 @@ tvh_video_context_open_filters(TVHContext *self, AVDictionary **opts)
     char source_args[128];
     char *filters = NULL;
 
+#if ENABLE_HWACCELS && ENABLE_NEW_TRANSCODING
+    enum AVPixelFormat pix_fmt = hwaccels_get_pixfmt_format_for_filter(self->iavctx);
+#endif
+
     // source args
     memset(source_args, 0, sizeof(source_args));
     if (str_snprintf(source_args, sizeof(source_args),
             "video_size=%dx%d:pix_fmt=%s:time_base=%d/%d:pixel_aspect=%d/%d",
             self->iavctx->width,
             self->iavctx->height,
+#if ENABLE_HWACCELS && ENABLE_NEW_TRANSCODING
+            av_get_pix_fmt_name(pix_fmt),
path: src/transcoding/transcode/video.c
user: login: Flole998
body: Why is the new way of getting the pix_fmt_name better than the old way?
created_at: 2025-01-01T03:45:13Z
author_association: MEMBER
diff_hunk: @@ -164,7 +164,11 @@ struct tvh_context {
     char *hw_accel_device;
     AVBufferRef *hw_device_ref;
     void *hw_accel_ictx;
+#if ENABLE_NEW_TRANSCODING
+    AVBufferRef *hw_frame_octx;
path: src/transcoding/transcode/internals.h
user: login: uknunknown
body: ENABLE_NEW_TRANSCODING was not introduced only to 'mask' the new variable change ... also changes a lot of other things. Please check src\transcoding\transcode\hwaccels\vaapi.c
There is a difference between TVH variable names and ffmpeg (example) variable names. In order for me to bring TVH naming convention to ffmpeg naming conventions I decided to rename that variable only for new transcoding (v.2).
This can be resolved in one of the 3 options:
1. option 1: Rename that variable in transcoding v.1 in TVH to match ffmpeg.
2. option 2: Keep the TVH naming and rename the variable in ffmpeg code imported in TVH. This is not recommended because will fork TVH away from ffmpeg.
3. option 3: Leave v.1 with old name from TVH and v.2 with new name from ffmpeg --> and transition to new names only for new ffmpeg. 

I selected option 3 because my expectation was that we will deprecate at some point transcoding v.1 and that code will be removed (at some later point). After this deprecation TVH will be aligned to ffmpeg.
Before you tell me what option do you prefer ... please think about the question: how far back should we prove the transcoding v.2 work in order for you to accept the transition directly to v.2 (and remove the v.1 code completely)? if it works with ffmpeg 4 are you OK to remove v.1? 
created_at: 2025-01-03T02:46:35Z
author_association: CONTRIBUTOR
diff_hunk: @@ -255,13 +282,21 @@ tvh_video_context_open_filters(TVHContext *self, AVDictionary **opts)
     char source_args[128];
     char *filters = NULL;
 
+#if ENABLE_HWACCELS && ENABLE_NEW_TRANSCODING
+    enum AVPixelFormat pix_fmt = hwaccels_get_pixfmt_format_for_filter(self->iavctx);
+#endif
+
     // source args
     memset(source_args, 0, sizeof(source_args));
     if (str_snprintf(source_args, sizeof(source_args),
             "video_size=%dx%d:pix_fmt=%s:time_base=%d/%d:pixel_aspect=%d/%d",
             self->iavctx->width,
             self->iavctx->height,
+#if ENABLE_HWACCELS && ENABLE_NEW_TRANSCODING
+            av_get_pix_fmt_name(pix_fmt),
path: src/transcoding/transcode/video.c
user: login: uknunknown
body: This change is not required for vaapi implementation .... but will be required for QSV implementation: please check https://github.com/tvheadend/tvheadend/pull/1781/files#diff-8ecb3c20d3cb77afadf97cb19260496ae8a05ac2e03486107abe5d15e30e6fceR315
If you want me to remove that part until we implement QSV ... please let me know.
created_at: 2025-01-03T02:58:01Z
author_association: CONTRIBUTOR
diff_hunk: @@ -255,13 +282,21 @@ tvh_video_context_open_filters(TVHContext *self, AVDictionary **opts)
     char source_args[128];
     char *filters = NULL;
 
+#if ENABLE_HWACCELS && ENABLE_NEW_TRANSCODING
+    enum AVPixelFormat pix_fmt = hwaccels_get_pixfmt_format_for_filter(self->iavctx);
+#endif
+
     // source args
     memset(source_args, 0, sizeof(source_args));
     if (str_snprintf(source_args, sizeof(source_args),
             "video_size=%dx%d:pix_fmt=%s:time_base=%d/%d:pixel_aspect=%d/%d",
             self->iavctx->width,
             self->iavctx->height,
+#if ENABLE_HWACCELS && ENABLE_NEW_TRANSCODING
+            av_get_pix_fmt_name(pix_fmt),
path: src/transcoding/transcode/video.c
user: login: uknunknown
body: removed
created_at: 2025-01-04T03:59:00Z
author_association: CONTRIBUTOR
diff_hunk: @@ -164,7 +164,11 @@ struct tvh_context {
     char *hw_accel_device;
     AVBufferRef *hw_device_ref;
     void *hw_accel_ictx;
+#if ENABLE_NEW_TRANSCODING
+    AVBufferRef *hw_frame_octx;
path: src/transcoding/transcode/internals.h
user: login: uknunknown
body: Experiment 1: I changed makefile.ffmpeg to compile with ffmpeg 5.1.2 and enabled transcoding v.2 for ffmpeg 5 --> all my tests have worked as expected.
Experiment 2: I changed makefile.ffmpeg to compile with ffmpeg 4.4 --> unfortunately I can't compile due to some errors. But the error is not related to the new transcoding (/tvheadend_new_transcoding/build.linux/ffmpeg/build/ffmpeg/lib/libavcodec.a] Error 2)
According to the date when ffmpeg has added the example (used for transcoding implementation) https://github.com/FFmpeg/FFmpeg/commit/23db3a1ae6d1be3438aec73c4dc91185d7958300#diff-860cbca3d54a06c6224b8b88bba80bbb62f1d7d8f34d3062b1dad5cc08ec2984 ffmpeg 4.4 should work also.
created_at: 2025-01-05T05:22:48Z
author_association: CONTRIBUTOR
diff_hunk: @@ -164,7 +164,11 @@ struct tvh_context {
     char *hw_accel_device;
     AVBufferRef *hw_device_ref;
     void *hw_accel_ictx;
+#if ENABLE_NEW_TRANSCODING
+    AVBufferRef *hw_frame_octx;
path: src/transcoding/transcode/internals.h
user: login: uknunknown
body: Experiment 3: I generated a new branch from: commit https://github.com/tvheadend/tvheadend/commit/470f02fb3f00d3f88e61303cd5db7ec303d0145d + new transcoding changes -->results is available on branch (if you want to test it yourself) https://github.com/uknunknown/tvheadend/tree/dev_transc_ffmpg4 compiled with ffmpeg 4.4.3
HEVC VAAPI was not working properly even before I change anything, but I tested H264 before and after that change and was working the same.
@Flole998 taking into account that VAAPI transcoding is only made with x86 platforms, I don;t think that any OS is using older than ffmpeg 4.4, is this far enough back to accept the transcoding change without keeping the old implementation? If you want to go further back I can try the same change on some older build (how old should that be)?
created_at: 2025-01-12T03:52:42Z
author_association: CONTRIBUTOR
diff_hunk: @@ -54,6 +54,13 @@ This list must be updated every time we use a new AV_CODEC_ID
 #define AV_CODEC_ID_DVB_TELETEXT CODEC_ID_DVB_TELETEXT
 #endif
 
+// Enable new transcoding starting ffmpeg 6
+#if LIBAVCODEC_VERSION_MAJOR < 60
+#define ENABLE_NEW_TRANSCODING 0
path: src/libav.h
user: login: uknunknown
body: change the name to be connected with ffmpeg
created_at: 2025-02-22T19:00:09Z
author_association: CONTRIBUTOR
diff_hunk: @@ -164,7 +164,11 @@ struct tvh_context {
     char *hw_accel_device;
     AVBufferRef *hw_device_ref;
     void *hw_accel_ictx;
+#if ENABLE_NEW_TRANSCODING
+    AVBufferRef *hw_frame_octx;
path: src/transcoding/transcode/internals.h
user: login: uknunknown
body: remove the rename ... simplified as much as possible the changes.
created_at: 2025-02-22T19:00:51Z
author_association: CONTRIBUTOR
diff_hunk: @@ -226,10 +226,37 @@ tvh_video_context_open_encoder(TVHContext *self, AVDictionary **opts)
 #if ENABLE_HWACCELS
     self->oavctx->coded_width = self->oavctx->width;
     self->oavctx->coded_height = self->oavctx->height;
-    if (hwaccels_encode_setup_context(self->oavctx, self->profile->low_power)) {
+    #if ENABLE_FFMPEG4_TRANSCODING
+    // hwaccel is the user input for Hardware acceleration from Codec parameteres
+    int hwaccel = -1;
+    if ((hwaccel = tvh_codec_profile_video_get_hwaccel(self->profile)) < 0) {
         return -1;
     }
+    if (_video_filters_hw_pix_fmt(self->oavctx->pix_fmt)){
+        // encoder is hw accelerated
+        if (hwaccel) {
+            // decoder is hw accelerated 
+            // --> we initialize encoder from decoder as recomanded in: 
+            // ffmpeg-6.1.1/doc/examples/vaapi_transcode.c line 169
+            if (hwaccels_initialize_encoder_from_decoder(self->iavctx, self->oavctx)) {
+                return -1;
+            }
+        }
+        else {
+            // decoder is sw
+            // --> we initialize as recommended in:
+            // ffmpeg-6.1.1/doc/examples/vaapi_encode.c line 145
+            if (hwaccels_encode_setup_context(self->oavctx)) {
+#else
+            if (hwaccels_encode_setup_context(self->oavctx, self->profile->low_power)) {
+#endif
+                return -1;
+            }
+#if ENABLE_FFMPEG4_TRANSCODING
path: src/transcoding/transcode/video.c
user: login: Flole998
body: This is wild... If you need an extra #ifdef just to close some } then your code is most likely hard to understand. Just copy the `return -1`, overall readability will be improved even though it's there twice then.
created_at: 2025-05-18T20:49:03Z
author_association: MEMBER
diff_hunk: @@ -226,10 +226,37 @@ tvh_video_context_open_encoder(TVHContext *self, AVDictionary **opts)
 #if ENABLE_HWACCELS
     self->oavctx->coded_width = self->oavctx->width;
     self->oavctx->coded_height = self->oavctx->height;
-    if (hwaccels_encode_setup_context(self->oavctx, self->profile->low_power)) {
+    #if ENABLE_FFMPEG4_TRANSCODING
+    // hwaccel is the user input for Hardware acceleration from Codec parameteres
+    int hwaccel = -1;
+    if ((hwaccel = tvh_codec_profile_video_get_hwaccel(self->profile)) < 0) {
         return -1;
     }
+    if (_video_filters_hw_pix_fmt(self->oavctx->pix_fmt)){
+        // encoder is hw accelerated
+        if (hwaccel) {
+            // decoder is hw accelerated 
+            // --> we initialize encoder from decoder as recomanded in: 
+            // ffmpeg-6.1.1/doc/examples/vaapi_transcode.c line 169
+            if (hwaccels_initialize_encoder_from_decoder(self->iavctx, self->oavctx)) {
+                return -1;
+            }
+        }
+        else {
+            // decoder is sw
+            // --> we initialize as recommended in:
+            // ffmpeg-6.1.1/doc/examples/vaapi_encode.c line 145
+            if (hwaccels_encode_setup_context(self->oavctx)) {
+#else
+            if (hwaccels_encode_setup_context(self->oavctx, self->profile->low_power)) {
+#endif
+                return -1;
+            }
+#if ENABLE_FFMPEG4_TRANSCODING
path: src/transcoding/transcode/video.c
user: login: uknunknown
body: I am sorry Flole ... I am not sure what was in my head. Thank you for catching that. Fixed.
Lets squash after you finish the review.
created_at: 2025-05-19T00:31:16Z
author_association: CONTRIBUTOR
diff_hunk: @@ -226,10 +226,37 @@ tvh_video_context_open_encoder(TVHContext *self, AVDictionary **opts)
 #if ENABLE_HWACCELS
     self->oavctx->coded_width = self->oavctx->width;
     self->oavctx->coded_height = self->oavctx->height;
-    if (hwaccels_encode_setup_context(self->oavctx, self->profile->low_power)) {
+    #if ENABLE_FFMPEG4_TRANSCODING
+    // hwaccel is the user input for Hardware acceleration from Codec parameteres
+    int hwaccel = -1;
+    if ((hwaccel = tvh_codec_profile_video_get_hwaccel(self->profile)) < 0) {
         return -1;
     }
+    if (_video_filters_hw_pix_fmt(self->oavctx->pix_fmt)){
+        // encoder is hw accelerated
+        if (hwaccel) {
+            // decoder is hw accelerated 
+            // --> we initialize encoder from decoder as recomanded in: 
+            // ffmpeg-6.1.1/doc/examples/vaapi_transcode.c line 169
+            if (hwaccels_initialize_encoder_from_decoder(self->iavctx, self->oavctx)) {
+                return -1;
+            }
+        }
+        else {
+            // decoder is sw
+            // --> we initialize as recommended in:
+            // ffmpeg-6.1.1/doc/examples/vaapi_encode.c line 145
+            if (hwaccels_encode_setup_context(self->oavctx)) {
+#else
+            if (hwaccels_encode_setup_context(self->oavctx, self->profile->low_power)) {
+#endif
+                return -1;
+            }
+#if ENABLE_FFMPEG4_TRANSCODING
path: src/transcoding/transcode/video.c
user: login: Flole998
body: Much better, thanks!
created_at: 2025-05-19T12:52:25Z
author_association: MEMBER

PR comments:
user: login: sonarqubecloud[bot]
created_at: 2025-01-04T04:05:49Z
body: ## [![Quality Gate Passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/qg-passed-20px.png 'Quality Gate Passed')](https://sonarcloud.io/dashboard?id=tvheadend_tvheadend&pullRequest=1792) **Quality Gate passed**  
Issues  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0 New issues](https://sonarcloud.io/project/issues?id=tvheadend_tvheadend&pullRequest=1792&issueStatuses=OPEN,CONFIRMED&sinceLeakPeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/accepted-16px.png '') [0 Accepted issues](https://sonarcloud.io/project/issues?id=tvheadend_tvheadend&pullRequest=1792&issueStatuses=ACCEPTED)

Measures  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=tvheadend_tvheadend&pullRequest=1792&issueStatuses=OPEN,CONFIRMED&sinceLeakPeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0.0% Coverage on New Code](https://sonarcloud.io/component_measures?id=tvheadend_tvheadend&pullRequest=1792&metric=new_coverage&view=list)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0.0% Duplication on New Code](https://sonarcloud.io/component_measures?id=tvheadend_tvheadend&pullRequest=1792&metric=new_duplicated_lines_density&view=list)  
  
[See analysis details on SonarQube Cloud](https://sonarcloud.io/dashboard?id=tvheadend_tvheadend&pullRequest=1792)


author_association: NONE
user: login: sonarqubecloud[bot]
created_at: 2025-02-22T19:07:24Z
body: ## [![Quality Gate Passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/qg-passed-20px.png 'Quality Gate Passed')](https://sonarcloud.io/dashboard?id=tvheadend_tvheadend&pullRequest=1792) **Quality Gate passed**  
Issues  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0 New issues](https://sonarcloud.io/project/issues?id=tvheadend_tvheadend&pullRequest=1792&issueStatuses=OPEN,CONFIRMED&sinceLeakPeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/accepted-16px.png '') [0 Accepted issues](https://sonarcloud.io/project/issues?id=tvheadend_tvheadend&pullRequest=1792&issueStatuses=ACCEPTED)

Measures  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=tvheadend_tvheadend&pullRequest=1792&issueStatuses=OPEN,CONFIRMED&sinceLeakPeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0.0% Coverage on New Code](https://sonarcloud.io/component_measures?id=tvheadend_tvheadend&pullRequest=1792&metric=new_coverage&view=list)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0.0% Duplication on New Code](https://sonarcloud.io/component_measures?id=tvheadend_tvheadend&pullRequest=1792&metric=new_duplicated_lines_density&view=list)  
  
[See analysis details on SonarQube Cloud](https://sonarcloud.io/dashboard?id=tvheadend_tvheadend&pullRequest=1792)


author_association: NONE
user: login: uknunknown
created_at: 2025-02-23T18:16:43Z
body: @Flole998 my proposal is to use this switch (ENABLE_FFMPEG4_TRANSCODING) for one year or so; if we have any complains we can ask users to switch back and retest. If we don't find an issue, we can remove the old implementation and we keep the new implementation (some concept like deprecation).
author_association: CONTRIBUTOR
user: login: jahutchi
created_at: 2025-03-19T11:57:35Z
body: I had been having all sorts of off issues with TVH Built-in VAAPI transcoding on my Intel NUC Coffee Lake. Ended up using TS-Spawn but that was far from perfect and could never get it working reliably.

I came across this pull request so thought I would give it a go. My VAAPI transcoding is now working perfectly - so hope this gets merged one day. Great Job 👍
author_association: CONTRIBUTOR
user: login: chewitt
created_at: 2025-03-19T12:18:33Z
body: I'm still not understanding why the extra abstraction and ifdef for `ENABLE_FFMPEG4_TRANSCODING 0` is required when testing for `LIBAVCODEC_VERSION_MAJOR < 58` does the same thing, no? NB: I am not a coding developer (any time I fiddle with code and it still compiles I'm surprised) and thus might be missing something obvious. That's also the reason I should not be a formal reviewer for the PR!
author_association: MEMBER
user: login: uknunknown
created_at: 2025-03-23T17:01:12Z
body: If you replace `ENABLE_FFMPEG4_TRANSCODING 0` with `LIBAVCODEC_VERSION_MAJOR < 58` will be very hard for user to go back and forth ... because in order to go back to old implementation is forced to change ffmpeg4 in `makefile.ffmpeg`. Practically master will not compile now with ffmpeg4 ... as it is (if you enable transcoding). You need to make some changes, beside going back to some very old version of master (I made this exercise in https://github.com/uknunknown/tvheadend/tree/dev_transc_ffmpg4).
**In conclusion**: is to make it more accessible for users to go back and forth between implementations.
author_association: CONTRIBUTOR
user: login: sonarqubecloud[bot]
created_at: 2025-05-19T00:33:59Z
body: ## [![Quality Gate Passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/qg-passed-20px.png 'Quality Gate Passed')](https://sonarcloud.io/dashboard?id=tvheadend_tvheadend&pullRequest=1792) **Quality Gate passed**  
Issues  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0 New issues](https://sonarcloud.io/project/issues?id=tvheadend_tvheadend&pullRequest=1792&issueStatuses=OPEN,CONFIRMED&sinceLeakPeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/accepted-16px.png '') [0 Accepted issues](https://sonarcloud.io/project/issues?id=tvheadend_tvheadend&pullRequest=1792&issueStatuses=ACCEPTED)

Measures  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=tvheadend_tvheadend&pullRequest=1792&issueStatuses=OPEN,CONFIRMED&sinceLeakPeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0.0% Coverage on New Code](https://sonarcloud.io/component_measures?id=tvheadend_tvheadend&pullRequest=1792&metric=new_coverage&view=list)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0.0% Duplication on New Code](https://sonarcloud.io/component_measures?id=tvheadend_tvheadend&pullRequest=1792&metric=new_duplicated_lines_density&view=list)  
  
[See analysis details on SonarQube Cloud](https://sonarcloud.io/dashboard?id=tvheadend_tvheadend&pullRequest=1792)


author_association: NONE
user: login: chewitt
created_at: 2025-05-19T12:46:39Z
body: @uknunknown I shouldn't be reviewing code (except for spelling checks) .. I leave that to @Flole998 
author_association: MEMBER
user: login: Flole998
created_at: 2025-05-19T13:00:35Z
body: > I'm still not understanding why the extra abstraction and ifdef for `ENABLE_FFMPEG4_TRANSCODING 0` is required when testing for `LIBAVCODEC_VERSION_MAJOR < 58` does the same thing, no?

It could be done both ways, the advantage here is to allow using the "legacy method" on newer versions without checking which `LIBAVCODEC_VERSION_MAJOR ` checks need to be modified. At some point it can be removed if nobody reports issues, but at this point its much easier to tell someone to "change xyz" instead of "change this, that, oh, don't forget another thing over there". It's just a "safer" approach and in this case it is rather simple so the price (making code more difficult to understand) is not that high. In this case it might even be easier to understand as there isn't just a comparison but also a "description" of what its for
author_association: MEMBER
user: login: Flole998
created_at: 2025-05-19T13:01:07Z
body: Lets squash and merge
author_association: MEMBER