[PR] avcodec/nvenc: accept P012/P212/YUV444P12MSB input (truncated to 10-bit) (PR #23592)

PR #23592 opened by Diego de Souza (ddesouza)
URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23592
Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23592.patch

NVDEC and CUVID now output AV_PIX_FMT_P012 (12-bit 4:2:0),
AV_PIX_FMT_P212 (12-bit 4:2:2) and AV_PIX_FMT_YUV444P12MSB (12-bit 4:4:4)
for high-bit-depth content, but nvenc rejected them: nvenc_map_buffer_format()
returned NV_ENC_BUFFER_FORMAT_UNDEFINED, so a pipeline such as
"-hwaccel cuda -i 12bit.mp4 -c:v hevc_nvenc" failed with EINVAL.

NVENC has no native 12-bit encode format. As is already done for the
16-bit inputs (P016, YUV444P16, GBRP16), accept the 12-bit formats and
map them to the 10-bit NVENC buffer formats, truncating the two least
significant bits: P012 -> YUV420_10BIT, P212 -> P210, YUV444P12MSB ->
YUV444_10BIT. They are added to the supported input format list and to
the IS_10BIT / IS_YUV444 / IS_YUV422 helpers accordingly.

Signed-off-by: Diego de Souza <[email protected]>


>From dac3bdfe53c5c92b804c756be000d362d996fd6c Mon Sep 17 00:00:00 2001
From: Diego de Souza <[email protected]>
Date: Wed, 24 Jun 2026 22:53:58 +0200
Subject: [PATCH] avcodec/nvenc: accept P012/P212/YUV444P12MSB input (truncated
 to 10-bit)

NVDEC and CUVID now output AV_PIX_FMT_P012 (12-bit 4:2:0),
AV_PIX_FMT_P212 (12-bit 4:2:2) and AV_PIX_FMT_YUV444P12MSB (12-bit 4:4:4)
for high-bit-depth content, but nvenc rejected them: nvenc_map_buffer_format()
returned NV_ENC_BUFFER_FORMAT_UNDEFINED, so a pipeline such as
"-hwaccel cuda -i 12bit.mp4 -c:v hevc_nvenc" failed with EINVAL.

NVENC has no native 12-bit encode format. As is already done for the
16-bit inputs (P016, YUV444P16, GBRP16), accept the 12-bit formats and
map them to the 10-bit NVENC buffer formats, truncating the two least
significant bits: P012 -> YUV420_10BIT, P212 -> P210, YUV444P12MSB ->
YUV444_10BIT. They are added to the supported input format list and to
the IS_10BIT / IS_YUV444 / IS_YUV422 helpers accordingly.

Signed-off-by: Diego de Souza <[email protected]>
---
 libavcodec/nvenc.c | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/libavcodec/nvenc.c b/libavcodec/nvenc.c
index ff2b26c3bc..1c931d43af 100644
--- a/libavcodec/nvenc.c
+++ b/libavcodec/nvenc.c
@@ -55,13 +55,16 @@ const enum AVPixelFormat ff_nvenc_pix_fmts[] = {
     AV_PIX_FMT_NV12,
     AV_PIX_FMT_P010,
     AV_PIX_FMT_YUV444P,
+    AV_PIX_FMT_P012,      // Truncated to 10bits
     AV_PIX_FMT_P016,      // Truncated to 10bits
 #ifdef NVENC_HAVE_422_SUPPORT
     AV_PIX_FMT_NV16,
     AV_PIX_FMT_P210,
+    AV_PIX_FMT_P212,      // Truncated to 10bits
     AV_PIX_FMT_P216,
 #endif
     AV_PIX_FMT_YUV444P10MSB,
+    AV_PIX_FMT_YUV444P12MSB, // Truncated to 10bits
     AV_PIX_FMT_YUV444P16, // Truncated to 10bits
     AV_PIX_FMT_0RGB32,
     AV_PIX_FMT_RGB32,
@@ -90,10 +93,13 @@ const AVCodecHWConfigInternal *const ff_nvenc_hw_configs[] = {
 };
 
 #define IS_10BIT(pix_fmt)  (pix_fmt == AV_PIX_FMT_P010         || \
+                            pix_fmt == AV_PIX_FMT_P012         || \
                             pix_fmt == AV_PIX_FMT_P016         || \
                             pix_fmt == AV_PIX_FMT_P210         || \
+                            pix_fmt == AV_PIX_FMT_P212         || \
                             pix_fmt == AV_PIX_FMT_P216         || \
                             pix_fmt == AV_PIX_FMT_YUV444P10MSB || \
+                            pix_fmt == AV_PIX_FMT_YUV444P12MSB || \
                             pix_fmt == AV_PIX_FMT_YUV444P16    || \
                             pix_fmt == AV_PIX_FMT_X2RGB10      || \
                             pix_fmt == AV_PIX_FMT_X2BGR10      || \
@@ -109,6 +115,7 @@ const AVCodecHWConfigInternal *const ff_nvenc_hw_configs[] = {
 
 #define IS_YUV444(pix_fmt) (pix_fmt == AV_PIX_FMT_YUV444P      || \
                             pix_fmt == AV_PIX_FMT_YUV444P10MSB || \
+                            pix_fmt == AV_PIX_FMT_YUV444P12MSB || \
                             pix_fmt == AV_PIX_FMT_YUV444P16    || \
                             pix_fmt == AV_PIX_FMT_GBRP         || \
                             pix_fmt == AV_PIX_FMT_GBRP10MSB    || \
@@ -117,6 +124,7 @@ const AVCodecHWConfigInternal *const ff_nvenc_hw_configs[] = {
 
 #define IS_YUV422(pix_fmt) (pix_fmt == AV_PIX_FMT_NV16 || \
                             pix_fmt == AV_PIX_FMT_P210 || \
+                            pix_fmt == AV_PIX_FMT_P212 || \
                             pix_fmt == AV_PIX_FMT_P216)
 
 #define IS_GBRP(pix_fmt) (pix_fmt == AV_PIX_FMT_GBRP      || \
@@ -1829,6 +1837,7 @@ static NV_ENC_BUFFER_FORMAT nvenc_map_buffer_format(enum AVPixelFormat pix_fmt)
     case AV_PIX_FMT_NV12:
         return NV_ENC_BUFFER_FORMAT_NV12;
     case AV_PIX_FMT_P010:
+    case AV_PIX_FMT_P012:
     case AV_PIX_FMT_P016:
         return NV_ENC_BUFFER_FORMAT_YUV420_10BIT;
     case AV_PIX_FMT_GBRP:
@@ -1838,6 +1847,7 @@ static NV_ENC_BUFFER_FORMAT nvenc_map_buffer_format(enum AVPixelFormat pix_fmt)
     case AV_PIX_FMT_GBRP10MSB:
     case AV_PIX_FMT_YUV444P16:
     case AV_PIX_FMT_YUV444P10MSB:
+    case AV_PIX_FMT_YUV444P12MSB:
         return NV_ENC_BUFFER_FORMAT_YUV444_10BIT;
     case AV_PIX_FMT_0RGB32:
     case AV_PIX_FMT_RGB32:
@@ -1853,6 +1863,7 @@ static NV_ENC_BUFFER_FORMAT nvenc_map_buffer_format(enum AVPixelFormat pix_fmt)
     case AV_PIX_FMT_NV16:
         return NV_ENC_BUFFER_FORMAT_NV16;
     case AV_PIX_FMT_P210:
+    case AV_PIX_FMT_P212:
     case AV_PIX_FMT_P216:
         return NV_ENC_BUFFER_FORMAT_P210;
 #endif
-- 
2.52.0

_______________________________________________
ffmpeg-devel mailing list -- [email protected]
To unsubscribe send an email to [email protected]