libs: encoder: VP9: fix > 4k encode fail issue

The VP9 spec defines the MAX_TILE_WIDTH_B64(64), which is the maximum
width of a tile in units of superblocks. So the max width of one tile
should not be larger than 64x64=4096. When the width exceeds 4k, we
need to split it into multiple tiles in columns. The current vp9 encoder
does not handle this correctly.
The command such as:
  gst-launch-1.0 videotestsrc ! video/x-raw,width=7680,height=4320 ! \
    vaapivp9enc ! fakesink
will crash.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer-vaapi/-/merge_requests/424>
This commit is contained in:
Zhang yuankun 2021-04-23 11:02:05 +08:00
parent 0193751ce8
commit 722d6bb625

View file

@ -33,6 +33,8 @@
#define DEBUG 1
#include "gstvaapidebug.h"
#define MAX_TILE_WIDTH_B64 64
/* Define default rate control mode ("constant-qp") */
#define DEFAULT_RATECONTROL GST_VAAPI_RATECONTROL_CQP
@ -356,6 +358,7 @@ fill_picture (GstVaapiEncoderVP9 * encoder,
VAEncPictureParameterBufferVP9 *const pic_param = picture->param;
guint i, last_idx = 0, gf_idx = 0, arf_idx = 0;
guint8 refresh_frame_flags = 0;
gint sb_cols = 0, min_log2_tile_columns = 0;
memset (pic_param, 0, sizeof (VAEncPictureParameterBufferVP9));
@ -399,6 +402,14 @@ fill_picture (GstVaapiEncoderVP9 * encoder,
pic_param->refresh_frame_flags = refresh_frame_flags;
}
/* Maximum width of a tile in units of superblocks is MAX_TILE_WIDTH_B64(64).
* When the width is enough to partition more than MAX_TILE_WIDTH_B64(64) superblocks,
* we need multi tiles to handle it.*/
sb_cols = (pic_param->frame_width_src + 63) / 64;
while ((MAX_TILE_WIDTH_B64 << min_log2_tile_columns) < sb_cols)
++min_log2_tile_columns;
pic_param->log2_tile_columns = min_log2_tile_columns;
pic_param->luma_ac_qindex = encoder->yac_qi;
pic_param->luma_dc_qindex_delta = 1;
pic_param->chroma_ac_qindex_delta = 1;