From 1e9d7a6a236939228efa22e0f0ace0a7f7328639 Mon Sep 17 00:00:00 2001 From: Guillaume Desmottes Date: Thu, 5 Jul 2018 15:13:47 +0200 Subject: [PATCH] omxvideoenc: include vertical padding in nFilledLen when copying According to the OMX spec (3.1.3.7.1) nFilledLen is meant to include any padding. We use to include the horizontal one (stride) but not the vertical one if nSliceHeight is bigger than the actual height. The calculated nFilledLen was wrong as it didn't include the padding between planes. https://bugzilla.gnome.org/show_bug.cgi?id=796749 --- omx/gstomxvideoenc.c | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/omx/gstomxvideoenc.c b/omx/gstomxvideoenc.c index 1a630c41ee..c296b95f00 100644 --- a/omx/gstomxvideoenc.c +++ b/omx/gstomxvideoenc.c @@ -2366,10 +2366,18 @@ gst_omx_video_enc_semi_planar_manual_copy (GstOMXVideoEnc * self, for (j = 0; j < height; j++) { memcpy (dest, src, width); - outbuf->omx_buf->nFilledLen += dest_stride; src += src_stride; dest += dest_stride; } + + /* nFilledLen should include the vertical padding in each slice (spec 3.1.3.7.1) */ + if (i == 0) + outbuf->omx_buf->nFilledLen += + port_def->format.video.nSliceHeight * port_def->format.video.nStride; + else + outbuf->omx_buf->nFilledLen += + (port_def->format.video.nSliceHeight / 2) * + port_def->format.video.nStride; } gst_video_frame_unmap (&frame); @@ -2501,10 +2509,19 @@ gst_omx_video_enc_fill_buffer (GstOMXVideoEnc * self, GstBuffer * inbuf, for (j = 0; j < height; j++) { memcpy (dest, src, width); - outbuf->omx_buf->nFilledLen += dest_stride; src += src_stride; dest += dest_stride; } + + /* nFilledLen should include the vertical padding in each slice (spec 3.1.3.7.1) */ + if (i == 0) + outbuf->omx_buf->nFilledLen += + port_def->format.video.nSliceHeight * + port_def->format.video.nStride; + else + outbuf->omx_buf->nFilledLen += + (port_def->format.video.nSliceHeight / 2) * + (port_def->format.video.nStride / 2); } gst_video_frame_unmap (&frame); ret = TRUE;