basetransform: handle invalid subclass implementation for fixate_caps

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/575>

libs/gst/base/gstbasetransform.c

@@ -1179,7 +1179,12 @@ gst_base_transform_find_transform (GstBaseTransform * trans, GstPad * pad,
      * function, it needs to truncate itself */
     othercaps =
         klass->fixate_caps (trans, GST_PAD_DIRECTION (pad), caps, othercaps);
-    is_fixed = gst_caps_is_fixed (othercaps);
+
+    if (!othercaps) {
+      g_critical ("basetransform: second attempt to fixate caps returned "
+          "invalid (NULL) caps on pad %s:%s", GST_DEBUG_PAD_NAME (pad));
+    }
+    is_fixed = othercaps && gst_caps_is_fixed (othercaps);
     GST_DEBUG_OBJECT (trans, "after fixating %" GST_PTR_FORMAT, othercaps);
   }
 

tests/check/libs/test_transform.c

@@ -72,6 +72,8 @@ static GstFlowReturn (*klass_transform_ip) (GstBaseTransform * trans,
     GstBuffer * buf) = NULL;
 static gboolean (*klass_set_caps) (GstBaseTransform * trans, GstCaps * incaps,
     GstCaps * outcaps) = NULL;
+static GstCaps *(*klass_fixate_caps) (GstBaseTransform * trans,
+    GstPadDirection direction, GstCaps * caps, GstCaps * othercaps) = NULL;
 static GstCaps *(*klass_transform_caps) (GstBaseTransform * trans,
     GstPadDirection direction, GstCaps * caps, GstCaps * filter) = NULL;
 static gboolean (*klass_transform_size) (GstBaseTransform * trans,
@@ -114,6 +116,8 @@ gst_test_trans_class_init (GstTestTransClass * klass)
     trans_class->transform_size = klass_transform_size;
   if (klass_set_caps != NULL)
     trans_class->set_caps = klass_set_caps;
+  if (klass_fixate_caps != NULL)
+    trans_class->fixate_caps = klass_fixate_caps;
   if (klass_submit_input_buffer != NULL)
     trans_class->submit_input_buffer = klass_submit_input_buffer;
   if (klass_generate_output)

tests/check/libs/transform1.c

@@ -887,6 +887,60 @@ GST_START_TEST (basetransform_chain_ct3)
 
 GST_END_TEST;
 
+static gboolean transform_fixate_caps_invalid_called = FALSE;
+
+static GstCaps *
+transform_fixate_caps_invalid (G_GNUC_UNUSED GstBaseTransform * trans,
+    G_GNUC_UNUSED GstPadDirection direction, G_GNUC_UNUSED GstCaps * caps,
+    GstCaps * othercaps)
+{
+  transform_fixate_caps_invalid_called = TRUE;
+  gst_caps_unref (othercaps);
+  return NULL;
+}
+
+GST_START_TEST (basetransform_invalid_fixatecaps_impl)
+{
+  TestTransData *trans;
+  GstBuffer *buffer;
+  GstFlowReturn res;
+  GstCaps *caps;
+
+  klass_fixate_caps = transform_fixate_caps_invalid;
+
+  trans = gst_test_trans_new ();
+
+  gst_test_trans_push_segment (trans);
+
+  /* passthrough without caps */
+  buffer = gst_buffer_new_and_alloc (20);
+  res = gst_test_trans_push (trans, buffer);
+  fail_unless (res == GST_FLOW_OK);
+
+  buffer = gst_test_trans_pop (trans);
+  fail_unless (buffer != NULL);
+  fail_unless (gst_buffer_get_size (buffer) == 20);
+
+  /* try to set caps prior to push data
+   * all following attempts to fixate_caps would fail producing a g_critical */
+  gst_pad_push_event (trans->srcpad, gst_event_new_flush_start ());
+  gst_pad_push_event (trans->srcpad, gst_event_new_flush_stop (TRUE));
+
+  caps = gst_caps_from_string ("foo/x-bar");
+  ASSERT_CRITICAL (gst_test_trans_setcaps (trans, caps));
+  gst_caps_unref (caps);
+
+  ASSERT_CRITICAL (gst_test_trans_push_segment (trans));
+
+  ASSERT_CRITICAL (res = gst_test_trans_push (trans, buffer));
+  fail_unless (transform_fixate_caps_invalid_called == TRUE);
+  fail_unless (res == GST_FLOW_NOT_NEGOTIATED);
+
+  gst_test_trans_free (trans);
+}
+
+GST_END_TEST;
+
 static void
 transform1_setup (void)
 {
@@ -903,6 +957,7 @@ transform1_teardown (void)
   klass_transform_caps = NULL;
   klass_transform_size = NULL;
   klass_set_caps = NULL;
+  klass_fixate_caps = NULL;
   klass_submit_input_buffer = NULL;
   klass_generate_output = NULL;
 }
@@ -927,6 +982,8 @@ gst_basetransform_suite (void)
   tcase_add_test (tc, basetransform_chain_ct2);
   tcase_add_test (tc, basetransform_chain_ct3);
 
+  tcase_add_test (tc, basetransform_invalid_fixatecaps_impl);
+
   return s;
 }