x264: Use profile and level from caps

Enforces the profile and level from the downstream caps, also
sets them on the fixated caps

https://bugzilla.gnome.org/show_bug.cgi?id=644233

ext/x264/Makefile.am

@@ -6,7 +6,9 @@ libgstx264_la_CFLAGS = \
 	$(GST_CFLAGS) \
 	$(X264_CFLAGS)
 libgstx264_la_LIBADD = \
-	$(GST_PLUGINS_BASE_LIBS) -lgstvideo-$(GST_MAJORMINOR) \
+	$(GST_PLUGINS_BASE_LIBS) \
+	-lgstvideo-$(GST_MAJORMINOR) \
+	-lgstpbutils-$(GST_MAJORMINOR) \
 	$(GST_LIBS) \
 	$(X264_LIBS)
 libgstx264_la_LDFLAGS = $(GST_PLUGIN_LDFLAGS)

ext/x264/gstx264enc.c

@@ -37,9 +37,11 @@
  * If #GstX264Enc:dct8x8 is enabled, then High profile is used.
  * Otherwise, if #GstX264Enc:cabac entropy coding is enabled or #GstX264Enc:bframes
  * are allowed, then Main Profile is in effect, and otherwise Baseline profile
- * applies.  No profile is imposed by default, which is fine for most software
+ * applies.  The main profile is imposed by default,
- * players and settings, but in some cases (e.g. hardware platforms) a more
+ * which is fine for most software players and settings,
- * restricted profile/level may be necessary.
+ * but in some cases (e.g. hardware platforms) a more restricted profile/level
+ * may be necessary. The recommended way to set a profile is to set it in the
+ * downstream caps.
  *
  * If a preset/tuning are specified then these will define the default values and
  * the property defaults will be ignored. After this the option-string property is
@@ -71,7 +73,7 @@
  * ]| This example pipeline will encode a test video source to H264 using fixed
  * quantization, and muxes it in a Matroska container.
  * |[
- * gst-launch -v videotestsrc num-buffers=1000 ! x264enc pass=5 quantizer=25 speed-preset=6 profile=1 ! \
+ * gst-launch -v videotestsrc num-buffers=1000 ! x264enc pass=5 quantizer=25 speed-preset=6 ! video/x-h264, profile=baseline ! \
  *   qtmux ! filesink location=videotestsrc.mov
  * ]| This example pipeline will encode a test video source to H264 using
  * constant quality at around Q25 using the 'medium' speed/quality preset and
@@ -94,6 +96,8 @@
 
 #include "gstx264enc.h"
 
+#include <gst/pbutils/pbutils.h>
+
 #if X264_BUILD >= 71
 #define X264_DELAYED_FRAMES_API
 #endif
@@ -461,7 +465,9 @@ static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src",
         "framerate = (fraction) [0/1, MAX], "
         "width = (int) [ 1, MAX ], " "height = (int) [ 1, MAX ], "
         "stream-format = (string) { byte-stream, avc }, "
-        "alignment = (string) { au }")
+        "alignment = (string) { au }, "
+        "profile = (string) { high-10, high, main, constrained-baseline, "
+        "high-10-intra }")
     );
 
 static void gst_x264_enc_finalize (GObject * object);
@@ -604,7 +610,8 @@ gst_x264_enc_class_init (GstX264EncClass * klass)
   g_object_class_install_property (gobject_class, ARG_PROFILE,
       g_param_spec_enum ("profile", "H.264 profile",
           "Apply restrictions to meet H.264 Profile constraints. This will "
-          "override other properties if necessary.",
+          "override other properties if necessary. This will only be used "
+          "if downstream elements do not specify a profile in their caps (DEPRECATED)",
           GST_X264_ENC_PROFILE_TYPE, ARG_PROFILE_DEFAULT,
           G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
 #endif /* X264_PRESETS */
@@ -1150,8 +1157,8 @@ gst_x264_enc_init_encoder (GstX264Enc * encoder)
       encoder->x264param.rc.i_rc_method = X264_RC_ABR;
       encoder->x264param.rc.i_bitrate = encoder->bitrate;
       encoder->x264param.rc.i_vbv_max_bitrate = encoder->bitrate;
-      encoder->x264param.rc.i_vbv_buffer_size
+      encoder->x264param.rc.i_vbv_buffer_size =
-          = encoder->x264param.rc.i_vbv_max_bitrate
+          encoder->x264param.rc.i_vbv_max_bitrate
           * encoder->vbv_buf_capacity / 1000;
       pass = encoder->pass & 0xF;
       break;
@@ -1207,14 +1214,43 @@ gst_x264_enc_init_encoder (GstX264Enc * encoder)
 #endif
 
 #ifdef X264_PRESETS
-  if (encoder->profile
+  if (encoder->peer_profile) {
-      && x264_param_apply_profile (&encoder->x264param,
+    if (x264_param_apply_profile (&encoder->x264param, encoder->peer_profile))
-          x264_profile_names[encoder->profile - 1])) {
+      GST_WARNING_OBJECT (encoder, "Bad downstream profile name: %s",
-    GST_WARNING_OBJECT (encoder, "Bad profile name: %s",
+          encoder->peer_profile);
-        x264_profile_names[encoder->profile - 1]);
+  } else if (encoder->profile) {
+    if (x264_param_apply_profile (&encoder->x264param,
+            x264_profile_names[encoder->profile - 1]))
+      GST_WARNING_OBJECT (encoder, "Bad profile name: %s",
+          x264_profile_names[encoder->profile - 1]);
   }
 #endif /* X264_PRESETS */
 
+  /* If using an intra profile, all frames are intra frames */
+  if (encoder->peer_intra_profile)
+    encoder->x264param.i_keyint_max = encoder->x264param.i_keyint_min = 1;
+
+  /* Enforce level limits if they were in the caps */
+  if (encoder->peer_level) {
+    encoder->x264param.i_level_idc = encoder->peer_level->level_idc;
+
+    encoder->x264param.rc.i_bitrate = MIN (encoder->x264param.rc.i_bitrate,
+        encoder->peer_level->bitrate);
+    encoder->x264param.rc.i_vbv_max_bitrate =
+        MIN (encoder->x264param.rc.i_vbv_max_bitrate,
+        encoder->peer_level->bitrate);
+    encoder->x264param.rc.i_vbv_buffer_size =
+        MIN (encoder->x264param.rc.i_vbv_buffer_size, encoder->peer_level->cpb);
+    encoder->x264param.analyse.i_mv_range =
+        MIN (encoder->x264param.analyse.i_mv_range,
+        encoder->peer_level->mv_range);
+
+    if (encoder->peer_level->frame_only) {
+      encoder->x264param.b_interlaced = FALSE;
+      encoder->x264param.b_fake_interlaced = FALSE;
+    }
+  }
+
   encoder->reconfig = FALSE;
 
   GST_OBJECT_UNLOCK (encoder);
@@ -1247,6 +1283,42 @@ gst_x264_enc_close_encoder (GstX264Enc * encoder)
   }
 }
 
+static gboolean
+gst_x264_enc_set_profile_and_level (GstX264Enc * encoder, GstCaps * caps)
+{
+  x264_nal_t *nal;
+  int i_nal;
+  int header_return;
+  gint sps_ni = 0;
+  guint8 *sps;
+
+
+  header_return = x264_encoder_headers (encoder->x264enc, &nal, &i_nal);
+  if (header_return < 0) {
+    GST_ELEMENT_ERROR (encoder, STREAM, ENCODE, ("Encode x264 header failed."),
+        ("x264_encoder_headers return code=%d", header_return));
+    return FALSE;
+  }
+
+  /* old x264 returns SEI, SPS and PPS, newer one has SEI last */
+  if (i_nal == 3 && nal[sps_ni].i_type != 7)
+    sps_ni = 1;
+
+  /* old style API: nal's are not encapsulated, and have no sync/size prefix,
+   * new style API: nal's are encapsulated, and have 4-byte size prefix */
+#ifndef X264_ENC_NALS
+  sps = nal[sps_ni].p_payload;
+#else
+  sps = nal[sps_ni].p_payload + 4;
+  /* skip NAL unit type */
+  sps++;
+#endif
+
+  gst_codec_utils_h264_caps_set_level_and_profile (caps, sps, 3);
+
+  return TRUE;
+}
+
 /*
  * Returns: Buffer with the stream headers.
  */
@@ -1386,6 +1458,11 @@ gst_x264_enc_set_src_caps (GstX264Enc * encoder, GstPad * pad, GstCaps * caps)
   }
   gst_structure_set (structure, "alignment", G_TYPE_STRING, "au", NULL);
 
+  if (!gst_x264_enc_set_profile_and_level (encoder, outcaps)) {
+    gst_caps_unref (outcaps);
+    return FALSE;
+  }
+
   res = gst_pad_set_caps (pad, outcaps);
   gst_caps_unref (outcaps);
 
@@ -1401,6 +1478,10 @@ gst_x264_enc_sink_set_caps (GstPad * pad, GstCaps * caps)
   gint fps_num, fps_den;
   gint par_num, par_den;
   gint i;
+  GstCaps *peer_caps;
+  const GstCaps *template_caps;
+  GstCaps *allowed_caps = NULL;
+  gboolean level_ok = TRUE;
 
   /* get info from caps */
   if (!gst_video_format_parse_caps (caps, &format, &width, &height))
@@ -1447,6 +1528,115 @@ gst_x264_enc_sink_set_caps (GstPad * pad, GstCaps * caps)
         i, width);
   }
 
+  encoder->peer_profile = NULL;
+  encoder->peer_intra_profile = FALSE;
+  encoder->peer_level = NULL;
+
+  /* FIXME: Remove THIS bit in 0.11 when the profile property is removed */
+  peer_caps = gst_pad_peer_get_caps_reffed (encoder->srcpad);
+  if (peer_caps) {
+    gint i;
+    gboolean has_profile_or_level = FALSE;
+
+    for (i = 0; i < gst_caps_get_size (peer_caps); i++) {
+      GstStructure *s = gst_caps_get_structure (peer_caps, i);
+
+      if (gst_structure_has_name (s, "video/x-h264") &&
+          (gst_structure_has_field (s, "profile") ||
+              gst_structure_has_field (s, "level"))) {
+        has_profile_or_level = TRUE;
+        break;
+      }
+    }
+
+    if (has_profile_or_level) {
+      template_caps = gst_pad_get_pad_template_caps (encoder->srcpad);
+
+      allowed_caps = gst_caps_intersect (peer_caps, template_caps);
+    }
+
+    gst_caps_unref (peer_caps);
+  }
+
+  /* Replace the bit since FIXME with this
+   * allowed_caps = gst_pad_get_allowed_caps (encoder->srcpad);
+   */
+
+  if (allowed_caps) {
+    GstStructure *s;
+    const gchar *profile;
+    const gchar *level;
+
+    if (gst_caps_is_empty (allowed_caps)) {
+      gst_caps_unref (allowed_caps);
+      return FALSE;
+    }
+
+    allowed_caps = gst_caps_make_writable (allowed_caps);
+    gst_pad_fixate_caps (encoder->srcpad, allowed_caps);
+    s = gst_caps_get_structure (allowed_caps, 0);
+
+    profile = gst_structure_get_string (s, "profile");
+    if (profile) {
+      if (!strcmp (profile, "constrained-baseline")) {
+        encoder->peer_profile = "baseline";
+      } else if (!strcmp (profile, "high-10-intra")) {
+        encoder->peer_intra_profile = TRUE;
+        encoder->peer_profile = "high10";
+      } else if (!strcmp (profile, "high-10")) {
+        encoder->peer_profile = "high10";
+      } else if (!strcmp (profile, "high")) {
+        encoder->peer_profile = "high";
+      } else if (!strcmp (profile, "main")) {
+        encoder->peer_profile = "main";
+      } else {
+        g_assert_not_reached ();
+      }
+    }
+
+    level = gst_structure_get_string (s, "level");
+    if (level) {
+      int level_idc = gst_codec_utils_h264_get_level_idc (level);
+
+      if (level_idc) {
+        gint i;
+
+        for (i = 0; x264_levels[i].level_idc; i++) {
+          if (level_idc == x264_levels[i].level_idc) {
+            int mb_width = (width + 15) / 16;
+            int mb_height = (height + 15) / 16;
+            int mbs = mb_width * mb_height;
+
+            if (x264_levels[i].frame_size < mbs ||
+                x264_levels[i].frame_size * 8 < mb_width * mb_width ||
+                x264_levels[i].frame_size * 8 < mb_height * mb_height) {
+              GST_WARNING_OBJECT (encoder,
+                  "Frame size larger than level %s allows", level);
+              level_ok = FALSE;
+              break;
+            }
+
+            if (fps_den &&
+                x264_levels[i].mbps < (gint64) mbs * fps_num / fps_den) {
+              GST_WARNING_OBJECT (encoder,
+                  "Macroblock rate higher than level %s allows", level);
+              level_ok = FALSE;
+              break;
+            }
+
+            encoder->peer_level = &x264_levels[i];
+            break;
+          }
+        }
+      }
+    }
+
+    gst_caps_unref (allowed_caps);
+  }
+
+  if (!level_ok)
+    return FALSE;
+
   if (!gst_x264_enc_init_encoder (encoder))
     return FALSE;
 

ext/x264/gstx264enc.h

@@ -115,6 +115,11 @@ struct _GstX264Enc
 
   /* configuration changed  while playing */
   gboolean reconfig;
+
+  /* from the downstream caps */
+  const gchar *peer_profile;
+  gboolean peer_intra_profile;
+  const x264_level_t *peer_level;
 };
 
 struct _GstX264EncClass