From 5f5df3f723a6ea4314e539c41e22c82da70b53e4 Mon Sep 17 00:00:00 2001 From: Gwenole Beauchesne Date: Wed, 15 Jan 2014 15:10:48 +0100 Subject: [PATCH] encoder: h264: really fix frame cropping rectangle calculation. Make frame cropping rectangle calculation future proof, i.e. exactly follow the specification (7-18) to (7-21), and subsampling definitions from Table 6-1. https://bugzilla.gnome.org/show_bug.cgi?id=722089 https://bugzilla.gnome.org/show_bug.cgi?id=722238 --- gst-libs/gst/vaapi/gstvaapiencoder_h264.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/gst-libs/gst/vaapi/gstvaapiencoder_h264.c b/gst-libs/gst/vaapi/gstvaapiencoder_h264.c index 63385226ca..d4aafa1eaf 100644 --- a/gst-libs/gst/vaapi/gstvaapiencoder_h264.c +++ b/gst-libs/gst/vaapi/gstvaapiencoder_h264.c @@ -995,14 +995,23 @@ fill_va_sequence_param (GstVaapiEncoderH264 * encoder, /* frame_cropping_flag */ if ((GST_VAAPI_ENCODER_WIDTH (encoder) & 15) || (GST_VAAPI_ENCODER_HEIGHT (encoder) & 15)) { + static const guint SubWidthC[] = { 1, 2, 2, 1 }; + static const guint SubHeightC[] = { 1, 2, 1, 1 }; + const guint CropUnitX = + SubWidthC[seq_param->seq_fields.bits.chroma_format_idc]; + const guint CropUnitY = + SubHeightC[seq_param->seq_fields.bits.chroma_format_idc] * + (2 - seq_param->seq_fields.bits.frame_mbs_only_flag); + seq_param->frame_cropping_flag = 1; seq_param->frame_crop_left_offset = 0; seq_param->frame_crop_right_offset = - 16 * encoder->mb_width - GST_VAAPI_ENCODER_WIDTH (encoder); + (16 * encoder->mb_width - + GST_VAAPI_ENCODER_WIDTH (encoder)) / CropUnitX; seq_param->frame_crop_top_offset = 0; seq_param->frame_crop_bottom_offset = - (16 * encoder->mb_height - GST_VAAPI_ENCODER_HEIGHT (encoder)) / - (2 - seq_param->seq_fields.bits.frame_mbs_only_flag); + (16 * encoder->mb_height - + GST_VAAPI_ENCODER_HEIGHT (encoder)) / CropUnitY; } /* vui not set */