From f2b0d02168e73feaae0023ccf864cea4805a0b88 Mon Sep 17 00:00:00 2001 From: Gwenole Beauchesne Date: Tue, 22 Apr 2014 19:53:50 +0200 Subject: [PATCH] vp8: fix per-segment deblocking filter level in relative mode. Fix possible bug when a per-segment deblocking filter level value needs to be set in non-absolute mode, i.e. when the loop filter update value is negative in delta mode. Also clamp the resulting filter level value to 0..63 range. --- gst-libs/gst/vaapi/gstvaapidecoder_vp8.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/gst-libs/gst/vaapi/gstvaapidecoder_vp8.c b/gst-libs/gst/vaapi/gstvaapidecoder_vp8.c index 7b828548f5..c7d22858a0 100644 --- a/gst-libs/gst/vaapi/gstvaapidecoder_vp8.c +++ b/gst-libs/gst/vaapi/gstvaapidecoder_vp8.c @@ -315,12 +315,14 @@ fill_picture (GstVaapiDecoderVp8 * decoder, GstVaapiPicture * picture) pic_param->mb_segment_tree_probs[i] = seg->segment_prob[i]; for (i = 0; i < 4; i++) { + gint8 level; if (seg->segmentation_enabled) { - pic_param->loop_filter_level[i] = seg->lf_update_value[i]; - if (!seg->segment_feature_mode) - pic_param->loop_filter_level[i] += frame_hdr->loop_filter_level; + level = seg->lf_update_value[i]; + if (!seg->segment_feature_mode) // 0 means delta update + level += frame_hdr->loop_filter_level; } else - pic_param->loop_filter_level[i] = frame_hdr->loop_filter_level; + level = frame_hdr->loop_filter_level; + pic_param->loop_filter_level[i] = CLAMP (level, 0, 63); pic_param->loop_filter_deltas_ref_frame[i] = parser->mb_lf_adjust.ref_frame_delta[i];