From ca84fd211a94ebd0bbca8adae6f083689a9d9511 Mon Sep 17 00:00:00 2001
From: Hyunjun Ko <zzoon@igalia.com>
Date: Tue, 27 Jun 2017 16:03:37 +0900
Subject: [PATCH] libs: encoder: h264: set profile via capsfilter

Until now, the encoder ignored the profile in src caps and chose one
according with the given parameters. But the encoder must honor the
profile specifed in src caps.

This patch do that, and if the encoder needs to choose the profile,
it will do it by following these rules:

1\ If given parameters are not compatible with given profile, the
   encoder will bail out with an error.
2\ The encoder will choose the higher profile indicated in the
   src caps.

https://bugzilla.gnome.org/show_bug.cgi?id=757941
---
 gst-libs/gst/vaapi/gstvaapiencoder_h264.c | 34 +++++++++++++++--------
 1 file changed, 22 insertions(+), 12 deletions(-)

diff --git a/gst-libs/gst/vaapi/gstvaapiencoder_h264.c b/gst-libs/gst/vaapi/gstvaapiencoder_h264.c
index 42a05db3a5..d1265c17d1 100644
--- a/gst-libs/gst/vaapi/gstvaapiencoder_h264.c
+++ b/gst-libs/gst/vaapi/gstvaapiencoder_h264.c
@@ -1043,24 +1043,34 @@ ensure_profile_limits (GstVaapiEncoderH264 * encoder)
   GstVaapiProfile profile;
 
   if (!encoder->max_profile_idc
-      || encoder->profile_idc <= encoder->max_profile_idc)
+      || encoder->profile_idc == encoder->max_profile_idc)
     return TRUE;
 
-  GST_WARNING ("lowering coding tools to meet target decoder constraints");
+  /* Give an error if the given parameters are invalid for requested
+   * profile rather than lowering profile.
+   */
+  if (encoder->profile_idc > encoder->max_profile_idc) {
+    GST_WARNING ("Invalid parameter for maximum profile");
+    return FALSE;
+  }
 
   profile = GST_VAAPI_PROFILE_UNKNOWN;
 
-  /* Try Main profile coding tools */
-  if (encoder->max_profile_idc < GST_H264_PROFILE_HIGH) {
-    encoder->use_dct8x8 = FALSE;
-    profile = GST_VAAPI_PROFILE_H264_MAIN;
-  }
+  if (encoder->profile_idc < encoder->max_profile_idc) {
+    /* Let profile be higher to fit in the maximum profile
+     * without changing parameters */
+    if (encoder->max_profile_idc > GST_H264_PROFILE_BASELINE)
+      profile = GST_VAAPI_PROFILE_H264_MAIN;
 
-  /* Try Constrained Baseline profile coding tools */
-  if (encoder->max_profile_idc < GST_H264_PROFILE_MAIN) {
-    encoder->num_bframes = 0;
-    encoder->use_cabac = FALSE;
-    profile = GST_VAAPI_PROFILE_H264_CONSTRAINED_BASELINE;
+    if (encoder->max_profile_idc > GST_H264_PROFILE_MAIN)
+      profile = GST_VAAPI_PROFILE_H264_HIGH;
+
+    if (encoder->max_profile_idc > GST_H264_PROFILE_HIGH) {
+      if (encoder->num_views > 2)
+        profile = GST_VAAPI_PROFILE_H264_MULTIVIEW_HIGH;
+      else if (encoder->num_views == 2)
+        profile = GST_VAAPI_PROFILE_H264_STEREO_HIGH;
+    }
   }
 
   if (profile) {