From dbc04a27ecd75bc8bdb52af9324fafa6099e9ff8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Wed, 8 Jun 2011 11:11:05 +0100 Subject: [PATCH 1/6] docs: add some text about parser/decoder autoplugging issues --- docs/design/design-decodebin.txt | 88 +++++++++++++++++++++++++++++++- 1 file changed, 86 insertions(+), 2 deletions(-) diff --git a/docs/design/design-decodebin.txt b/docs/design/design-decodebin.txt index 34919455df..52b343a0f5 100644 --- a/docs/design/design-decodebin.txt +++ b/docs/design/design-decodebin.txt @@ -13,9 +13,9 @@ Description: _ a GstTypeFindElement connected to the single sink pad - _ optionnaly a demuxer/parser + _ optionally a demuxer/parser - _ optionnaly one or more DecodeGroup + _ optionally one or more DecodeGroup * Autoplugging @@ -203,3 +203,87 @@ differences: controlled by the element. This means that a buffer cannot be pushed to a non-linked pad any sooner than buffers in any other stream which were received before it. + + +===================================== + Parsers, decoders and auto-plugging +===================================== + +This section has DRAFT status. + +Some media formats come in different "flavours" or "stream formats". These +formats differ in the way the setup data and media data is signalled and/or +packaged. An example for this is H.264 video, where there is a bytestream +format (with codec setup data signalled inline and units prefixed by a sync +code and packet length information) and a "raw" format where codec setup +data is signalled out of band (via the caps) and the chunking is implicit +in the way the buffers were muxed into a container, to mention just two of +the possible variants. + +Especially on embedded platforms it is common that decoders can only +handle one particular stream format, and not all of them. + +Where there are multiple stream formats, parsers are usually expected +to be able to convert between the different formats. This will, if +implemented correctly, work as expected in a static pipeline such as + + ... ! parser ! decoder ! sink + +where the parser can query the decoder's capabilities even before +processing the first piece of data, and configure itself to convert +accordingly, if conversion is needed at all. + +In an auto-plugging context this is not so straight-forward though, +because elements are plugged incrementally and not before the previous +element has processes some data and decided what it will output exactly +(unless the template caps are completely fixed, then it can continue +right away, this is not always the case here though, see below). A +parser will thus have to decide on *some* output format so auto-plugging +can continue. It doesn't know anything about the available decoders and +their capabilities though, so it's possible that it will choose a format +that is not supported by any of the available decoders, or by the preferred +decoder. + +If the parser had sufficiently concise but fixed source pad template caps, +decodebin could continue to plug a decoder right away, allowing the +parser to configure itself in the same way as it would with a static +pipeline. This is not an option, unfortunately, because often the +parser needs to process some data to determine e.g. the format's profile or +other stream properties (resolution, sample rate, channel configuration, etc.), +and there may be different decoders for different profiles (e.g. DSP codec +for baseline profile, and software fallback for main/high profile; or a DSP +codec only supporting certain resolutions, with a software fallback for +unusual resolutions). So if decodebin just plugged the most highest-ranking +decoder, that decoder might not be be able to handle the actual stream later +on, which would yield in an error (this is a data flow error then which would +be hard to intercept and avoid in decodebin). In other words, we can't solve +this issue by plugging a decoder right away with the parser. + +So decodebin need to communicate to the parser the set of available decoder +caps (which would contain the relevant capabilities/restrictions such as +supported profiles, resolutions, etc.), after the usual "autoplug-*" signal +filtering/sorting of course. + +This could be done in multiple ways, e.g. + + - plug a capsfilter element right after the parser, and construct + a set of filter caps from the list of available decoders (one + could append at the end just the name(s) of the caps structures + from the parser pad template caps to function as an 'ANY other' + caps equivalent). This would let the parser negotiate to a + supported stream format in the same way as with the static + pipeline mentioned above, but of course incur some overhead + through the additional capsfilter element. + + - one could add a filter-caps equivalent property to the parsers + (and/or GstBaseParse class) (e.g. "prefered-caps" or so). + + - one could add some kind of "fixate-caps" or "fixate-format" + signal to such parsers + +Alternatively, one could simply make all decoders incorporate parsers, so +that always all formats are supported. This is problematic for other reasons +though (e.g. we would not be able to detect the profile in all cases then +before plugging a decoder, which would make it hard to just play the audio +part of a stream and not the video if a suitable decoder was missing, for +example). From a9b979d45f38c592a46106a86b37d1c6c83c0be4 Mon Sep 17 00:00:00 2001 From: Christophe Fergeau Date: Wed, 8 Jun 2011 11:33:07 +0200 Subject: [PATCH 2/6] examples: don't link testchannels example with system libgstaudio The testchannels audio test program is using -lgstaudio-0.10 to link with libgstaudio which won't use the gstaudio library that was just built but the one from the system. This is an issue since it means we won't be testing the code from the current source tree, and it also breaks the build when building on a system which don't have a libgstaudio yet. https://bugzilla.gnome.org/show_bug.cgi?id=652100 --- tests/examples/audio/Makefile.am | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/examples/audio/Makefile.am b/tests/examples/audio/Makefile.am index f4a844abe6..e0cc5ef0d0 100644 --- a/tests/examples/audio/Makefile.am +++ b/tests/examples/audio/Makefile.am @@ -2,5 +2,6 @@ noinst_PROGRAMS = testchannels testchannels_SOURCES = testchannels.c testchannels_CFLAGS = $(GST_PLUGINS_BASE_CFLAGS) $(GST_CFLAGS) -testchannels_LDADD = $(GST_PLUGINS_BASE_LIBS) -lgstaudio-$(GST_MAJORMINOR) $(GST_LIBS) +testchannels_LDADD = $(top_builddir)/gst-libs/gst/audio/libgstaudio-$(GST_MAJORMINOR).la \ + $(GST_LIBS) From c692191c33195870ff4f5bd5dd6d4bce926a3512 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Wed, 8 Jun 2011 12:21:43 +0100 Subject: [PATCH 3/6] GST_PLUGINS_BASE_LIBS is not defined in -base. --- ext/pango/Makefile.am | 1 - gst/audioresample/Makefile.am | 1 - tests/check/Makefile.am | 1 - tests/examples/v4l/Makefile.am | 1 - 4 files changed, 4 deletions(-) diff --git a/ext/pango/Makefile.am b/ext/pango/Makefile.am index 2ffea22685..55aa3a0e24 100644 --- a/ext/pango/Makefile.am +++ b/ext/pango/Makefile.am @@ -19,7 +19,6 @@ libgstpango_la_CFLAGS = \ $(GST_CFLAGS) \ $(PANGO_CFLAGS) libgstpango_la_LIBADD = \ - $(GST_PLUGINS_BASE_LIBS) \ $(top_builddir)/gst-libs/gst/video/libgstvideo-$(GST_MAJORMINOR).la \ $(GST_BASE_LIBS) \ $(GST_CONTROLLER_LIBS) \ diff --git a/gst/audioresample/Makefile.am b/gst/audioresample/Makefile.am index 3464a8270d..f2be5c702a 100644 --- a/gst/audioresample/Makefile.am +++ b/gst/audioresample/Makefile.am @@ -20,7 +20,6 @@ libgstaudioresample_la_CFLAGS = \ $(ORC_CFLAGS) libgstaudioresample_la_LIBADD = \ - $(GST_PLUGINS_BASE_LIBS) \ $(GST_BASE_LIBS) \ $(GST_LIBS) \ $(ORC_LIBS) $(ORC_TEST_LIBS) \ diff --git a/tests/check/Makefile.am b/tests/check/Makefile.am index 6fc9351065..cae871467c 100644 --- a/tests/check/Makefile.am +++ b/tests/check/Makefile.am @@ -409,7 +409,6 @@ elements_textoverlay_CFLAGS = $(GST_BASE_CFLAGS) $(AM_CFLAGS) elements_volume_LDADD = \ $(top_builddir)/gst-libs/gst/interfaces/libgstinterfaces-@GST_MAJORMINOR@.la \ - $(GST_PLUGINS_BASE_LIBS) \ $(GST_CONTROLLER_LIBS) \ $(GST_BASE_LIBS) \ $(LDADD) diff --git a/tests/examples/v4l/Makefile.am b/tests/examples/v4l/Makefile.am index 6132cdc063..14d6d7dead 100644 --- a/tests/examples/v4l/Makefile.am +++ b/tests/examples/v4l/Makefile.am @@ -5,6 +5,5 @@ probe_CFLAGS = $(GST_PLUGINS_BASE_CFLAGS) \ $(GST_BASE_CFLAGS) $(GST_CFLAGS) probe_LDFLAGS = \ $(top_builddir)/gst-libs/gst/interfaces/libgstinterfaces-@GST_MAJORMINOR@.la \ - $(GST_PLUGINS_BASE_LIBS) \ $(GST_BASE_LIBS) $(GST_LIBS) From 9e6d3214f848217f323b8e35134efb085da0c1d0 Mon Sep 17 00:00:00 2001 From: Thiago Santos Date: Wed, 8 Jun 2011 14:21:40 -0300 Subject: [PATCH 4/6] tag: xmp: Add room for extra namespace definitions Adds an extra field to the namespace definitions of the schemas so they can add the namespace of any array/struct fields they might use internally. --- gst-libs/gst/tag/gstxmptag.c | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/gst-libs/gst/tag/gstxmptag.c b/gst-libs/gst/tag/gstxmptag.c index 920007ebbc..89973e9960 100644 --- a/gst-libs/gst/tag/gstxmptag.c +++ b/gst-libs/gst/tag/gstxmptag.c @@ -1041,17 +1041,26 @@ struct _GstXmpNamespaceMatch { const gchar *ns_prefix; const gchar *ns_uri; + + /* + * Stores extra namespaces for array tags + * The namespaces should be writen in the form: + * + * xmlns:XpTo="http://some.org/your/ns/name/ (next ones)" + */ + const gchar *extra_ns; }; static const GstXmpNamespaceMatch ns_match[] = { - {"dc", "http://purl.org/dc/elements/1.1/"}, - {"exif", "http://ns.adobe.com/exif/1.0/"}, - {"tiff", "http://ns.adobe.com/tiff/1.0/"}, - {"xap", "http://ns.adobe.com/xap/1.0/"}, - {"photoshop", "http://ns.adobe.com/photoshop/1.0/"}, - {"Iptc4xmpCore", "http://iptc.org/std/Iptc4xmpCore/1.0/xmlns/"}, - {"Iptc4xmpExt", "http://iptc.org/std/Iptc4xmpExt/2008-02-29/"}, - {NULL, NULL} + {"dc", "http://purl.org/dc/elements/1.1/", NULL}, + {"exif", "http://ns.adobe.com/exif/1.0/", NULL}, + {"tiff", "http://ns.adobe.com/tiff/1.0/", NULL}, + {"xap", "http://ns.adobe.com/xap/1.0/", NULL}, + {"photoshop", "http://ns.adobe.com/photoshop/1.0/", NULL}, + {"Iptc4xmpCore", "http://iptc.org/std/Iptc4xmpCore/1.0/xmlns/", NULL}, + {"Iptc4xmpExt", "http://iptc.org/std/Iptc4xmpExt/2008-02-29/", + "xmlns:LocationDetails=\"http://iptc.org/std/Iptc4xmpExt/2008-02-29/LocationDetails/\""}, + {NULL, NULL, NULL} }; typedef struct _GstXmpNamespaceMap GstXmpNamespaceMap; @@ -1762,9 +1771,13 @@ gst_tag_list_to_xmp_buffer_full (const GstTagList * list, gboolean read_only, i = 0; while (ns_match[i].ns_prefix) { if (xmp_serialization_data_use_schema (&serialization_data, - ns_match[i].ns_prefix)) + ns_match[i].ns_prefix)) { g_string_append_printf (data, " xmlns:%s=\"%s\"", ns_match[i].ns_prefix, ns_match[i].ns_uri); + if (ns_match[i].extra_ns) { + g_string_append_printf (data, " %s", ns_match[i].extra_ns); + } + } i++; } g_string_append (data, ">\n"); From 02580714de9245d3af2e20e8dadddcb081877b01 Mon Sep 17 00:00:00 2001 From: Thiago Santos Date: Thu, 9 Jun 2011 00:02:07 -0300 Subject: [PATCH 5/6] tag: xmp: Fix LocationShown syntax According to the specification, the LocationShown requires its struct fields to be inside a Bag type. --- gst-libs/gst/tag/gstxmptag.c | 39 ++++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/gst-libs/gst/tag/gstxmptag.c b/gst-libs/gst/tag/gstxmptag.c index 89973e9960..dd20fd3b51 100644 --- a/gst-libs/gst/tag/gstxmptag.c +++ b/gst-libs/gst/tag/gstxmptag.c @@ -115,7 +115,8 @@ xmp_serialization_data_use_schema (XmpSerializationData * serdata, typedef enum { - GstXmpTagTypeSimple = 0, + GstXmpTagTypeNone = 0, + GstXmpTagTypeSimple, GstXmpTagTypeBag, GstXmpTagTypeSeq, GstXmpTagTypeStruct, @@ -136,6 +137,13 @@ struct _XmpTag const gchar *tag_name; GstXmpTagType type; + /* some tags must be inside a Bag even + * if they are a single entry. Set it here so we know */ + GstXmpTagType supertype; + + /* For tags that need a rdf:parseType attribute */ + const gchar *parse_type; + /* Used for struct and compound types */ GSList *children; @@ -157,9 +165,9 @@ xmp_tag_get_merge_mode (XmpTag * xmptag) } static const gchar * -xmp_tag_get_type_name (XmpTag * xmptag) +xmp_tag_type_get_name (GstXmpTagType tagtype) { - switch (xmptag->type) { + switch (tagtype) { case GstXmpTagTypeSeq: return "rdf:Seq"; case GstXmpTagTypeBag: @@ -251,6 +259,8 @@ gst_xmp_tag_create (const gchar * gst_tag, const gchar * xmp_tag, xmpinfo->gst_tag = gst_tag; xmpinfo->tag_name = xmp_tag; xmpinfo->type = xmp_type; + xmpinfo->supertype = GstXmpTagTypeNone; + xmpinfo->parse_type = NULL; xmpinfo->serialize = serialization_func; xmpinfo->deserialize = deserialization_func; xmpinfo->children = NULL; @@ -1012,6 +1022,8 @@ _init_xmp_tag_map (gpointer user_data) schema = gst_xmp_schema_new (); xmpinfo = gst_xmp_tag_create (NULL, "Iptc4xmpExt:LocationShown", GstXmpTagTypeStruct, NULL, NULL); + xmpinfo->supertype = GstXmpTagTypeBag; + xmpinfo->parse_type = "Resource"; xmpinfo->children = g_slist_prepend (xmpinfo->children, gst_xmp_tag_create (GST_TAG_GEO_LOCATION_SUBLOCATION, "LocationDetails:Sublocation", GstXmpTagTypeSimple, NULL, NULL)); @@ -1667,10 +1679,29 @@ write_one_tag (const GstTagList * list, XmpTag * xmp_tag, gpointer user_data) if (use_it) { if (xmp_tag->tag_name) string_open_tag (data, xmp_tag->tag_name); + + if (xmp_tag->supertype) { + string_open_tag (data, xmp_tag_type_get_name (xmp_tag->supertype)); + if (xmp_tag->parse_type) { + g_string_append (data, "parse_type); + g_string_append_c (data, '"'); + g_string_append_c (data, '>'); + } else { + string_open_tag (data, "rdf:li"); + } + } + /* now write it */ for (iter = xmp_tag->children; iter; iter = g_slist_next (iter)) { write_one_tag (list, iter->data, user_data); } + + if (xmp_tag->supertype) { + string_close_tag (data, "rdf:li"); + string_close_tag (data, xmp_tag_type_get_name (xmp_tag->supertype)); + } + if (xmp_tag->tag_name) string_close_tag (data, xmp_tag->tag_name); } @@ -1703,7 +1734,7 @@ write_one_tag (const GstTagList * list, XmpTag * xmp_tag, gpointer user_data) } else { const gchar *typename; - typename = xmp_tag_get_type_name (xmp_tag); + typename = xmp_tag_type_get_name (xmp_tag->type); string_open_tag (data, typename); for (i = 0; i < ct; i++) { From efdd32580e887aeb68108d9578c0388bbc373375 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Thu, 9 Jun 2011 18:30:33 +0100 Subject: [PATCH 6/6] libs: replace G_CONST_RETURN with 'const' G_CONST_RETURN will be deprecated soon. https://bugzilla.gnome.org/show_bug.cgi?id=652211 --- gst-libs/gst/tag/gstid3tag.c | 8 ++++---- gst-libs/gst/tag/gstvorbistag.c | 4 ++-- gst-libs/gst/tag/tag.h | 12 ++++++------ 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/gst-libs/gst/tag/gstid3tag.c b/gst-libs/gst/tag/gstid3tag.c index 2c3c66147d..1613be7f4a 100644 --- a/gst-libs/gst/tag/gstid3tag.c +++ b/gst-libs/gst/tag/gstid3tag.c @@ -225,7 +225,7 @@ static const GstTagEntryMatch tag_matches[] = { * * Returns: The corresponding GStreamer tag or NULL if none exists. */ -G_CONST_RETURN gchar * +const gchar * gst_tag_from_id3_tag (const gchar * id3_tag) { int i = 0; @@ -285,7 +285,7 @@ static const GstTagEntryMatch user_tag_matches[] = { * * Returns: The corresponding GStreamer tag or NULL if none exists. */ -G_CONST_RETURN gchar * +const gchar * gst_tag_from_id3_user_tag (const gchar * type, const gchar * id3_user_tag) { int i = 0; @@ -317,7 +317,7 @@ gst_tag_from_id3_user_tag (const gchar * type, const gchar * id3_user_tag) * * Returns: The corresponding ID3v2 tag or NULL if none exists. */ -G_CONST_RETURN gchar * +const gchar * gst_tag_to_id3_tag (const gchar * gst_tag) { int i = 0; @@ -423,7 +423,7 @@ gst_tag_id3_genre_count (void) * * Returns: the genre or NULL if no genre is associated with that ID. */ -G_CONST_RETURN gchar * +const gchar * gst_tag_id3_genre_get (const guint id) { if (id >= G_N_ELEMENTS (genres)) diff --git a/gst-libs/gst/tag/gstvorbistag.c b/gst-libs/gst/tag/gstvorbistag.c index 400118bb1b..da1a1f326c 100644 --- a/gst-libs/gst/tag/gstvorbistag.c +++ b/gst-libs/gst/tag/gstvorbistag.c @@ -109,7 +109,7 @@ static const GstTagEntryMatch tag_matches[] = { * * Returns: The corresponding GStreamer tag or NULL if none exists. */ -G_CONST_RETURN gchar * +const gchar * gst_tag_from_vorbis_tag (const gchar * vorbis_tag) { int i = 0; @@ -138,7 +138,7 @@ gst_tag_from_vorbis_tag (const gchar * vorbis_tag) * * Returns: The corresponding vorbiscomment tag or NULL if none exists. */ -G_CONST_RETURN gchar * +const gchar * gst_tag_to_vorbis_tag (const gchar * gst_tag) { int i = 0; diff --git a/gst-libs/gst/tag/tag.h b/gst-libs/gst/tag/tag.h index 219fe85aa5..8eea4111a3 100644 --- a/gst-libs/gst/tag/tag.h +++ b/gst-libs/gst/tag/tag.h @@ -444,8 +444,8 @@ GType gst_tag_image_type_get_type (void); /* functions for vorbis comment manipulation */ -G_CONST_RETURN gchar * gst_tag_from_vorbis_tag (const gchar * vorbis_tag); -G_CONST_RETURN gchar * gst_tag_to_vorbis_tag (const gchar * gst_tag); +const gchar * gst_tag_from_vorbis_tag (const gchar * vorbis_tag); +const gchar * gst_tag_to_vorbis_tag (const gchar * gst_tag); void gst_vorbis_tag_add (GstTagList * list, const gchar * tag, const gchar * value); @@ -466,13 +466,13 @@ GstBuffer * gst_tag_list_to_vorbiscomment_buffer (const GstTagLis /* functions for ID3 tag manipulation */ guint gst_tag_id3_genre_count (void); -G_CONST_RETURN gchar * gst_tag_id3_genre_get (const guint id); +const gchar * gst_tag_id3_genre_get (const guint id); GstTagList * gst_tag_list_new_from_id3v1 (const guint8 * data); -G_CONST_RETURN gchar * gst_tag_from_id3_tag (const gchar * id3_tag); -G_CONST_RETURN gchar * gst_tag_from_id3_user_tag (const gchar * type, +const gchar * gst_tag_from_id3_tag (const gchar * id3_tag); +const gchar * gst_tag_from_id3_user_tag (const gchar * type, const gchar * id3_user_tag); -G_CONST_RETURN gchar * gst_tag_to_id3_tag (const gchar * gst_tag); +const gchar * gst_tag_to_id3_tag (const gchar * gst_tag); gboolean gst_tag_list_add_id3_image (GstTagList * tag_list, const guint8 * image_data,