/* GStreamer * * jpegparse: a parser for JPEG streams * * Copyright (C) <2009> Arnout Vandecappelle (Essensium/Mind) * <2022> Víctor Manuel Jáquez Leal * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, * Boston, MA 02110-1301, USA. */ /** * SECTION:element-jpegparse * @title: jpegparse * @short_description: JPEG parser * * Parses a JPEG stream into JPEG images. It looks for EOI boundaries to * split a continuous stream into single-frame buffers. Also reads the * image header searching for image properties such as width and height * among others. Jpegparse can also extract metadata (e.g. xmp). * * ## Example launch line * |[ * gst-launch-1.0 -v souphttpsrc location=... ! jpegparse ! matroskamux ! filesink location=... * ]| * The above pipeline fetches a motion JPEG stream from an IP camera over * HTTP and stores it in a matroska file. * */ /* FIXME: output plain JFIF APP marker only. This provides best code reuse. * JPEG decoders would not need to handle this part anymore. Also when remuxing * (... ! jpegparse ! ... ! jifmux ! ...) metadata consolidation would be * easier. */ /* TODO: * + APP2 -- ICC color profile * + APP3 -- meta (same as exif) * + APP12 -- Photoshop Save for Web: Ducky / Picture info * + APP13 -- Adobe IRB * + check for interlaced mjpeg */ #ifdef HAVE_CONFIG_H #include #endif #include #include #include #include #include "gstjpegparse.h" enum ParserState { GST_JPEG_PARSER_STATE_GOT_SOI = 1 << 0, GST_JPEG_PARSER_STATE_GOT_SOF = 1 << 1, GST_JPEG_PARSER_STATE_GOT_SOS = 1 << 2, GST_JPEG_PARSER_STATE_GOT_JFIF = 1 << 3, GST_JPEG_PARSER_STATE_GOT_ADOBE = 1 << 4, GST_JPEG_PARSER_STATE_VALID_PICTURE = (GST_JPEG_PARSER_STATE_GOT_SOI | GST_JPEG_PARSER_STATE_GOT_SOF | GST_JPEG_PARSER_STATE_GOT_SOS), }; static GstStaticPadTemplate gst_jpeg_parse_src_pad_template = GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_SRC, GST_PAD_ALWAYS, GST_STATIC_CAPS ("image/jpeg, " "framerate = (fraction) [ 0/1, MAX ], " "parsed = (boolean) true") ); static GstStaticPadTemplate gst_jpeg_parse_sink_pad_template = GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS ("image/jpeg") ); GST_DEBUG_CATEGORY_STATIC (jpeg_parse_debug); #define GST_CAT_DEFAULT jpeg_parse_debug static GstFlowReturn gst_jpeg_parse_handle_frame (GstBaseParse * bparse, GstBaseParseFrame * frame, gint * skipsize); static gboolean gst_jpeg_parse_set_sink_caps (GstBaseParse * parse, GstCaps * caps); static gboolean gst_jpeg_parse_sink_event (GstBaseParse * parse, GstEvent * event); static gboolean gst_jpeg_parse_start (GstBaseParse * parse); static gboolean gst_jpeg_parse_stop (GstBaseParse * parse); #define gst_jpeg_parse_parent_class parent_class G_DEFINE_TYPE (GstJpegParse, gst_jpeg_parse, GST_TYPE_BASE_PARSE); GST_ELEMENT_REGISTER_DEFINE (jpegparse, "jpegparse", GST_RANK_NONE, GST_TYPE_JPEG_PARSE); enum GstJPEGColorspace { GST_JPEG_COLORSPACE_NONE, GST_JPEG_COLORSPACE_RGB, GST_JPEG_COLORSPACE_YUV, GST_JPEG_COLORSPACE_GRAY, GST_JPEG_COLORSPACE_CMYK, GST_JPEG_COLORSPACE_YCCK, }; static const gchar *gst_jpeg_colorspace_strings[] = { [GST_JPEG_COLORSPACE_NONE] = NULL, [GST_JPEG_COLORSPACE_RGB] = "sRGB", [GST_JPEG_COLORSPACE_YUV] = "sYUV", [GST_JPEG_COLORSPACE_GRAY] = "GRAY", [GST_JPEG_COLORSPACE_CMYK] = "CMYK", [GST_JPEG_COLORSPACE_YCCK] = "YCCK", }; enum GstJPEGSampling { GST_JPEG_SAMPLING_NONE, GST_JPEG_SAMPLING_RGB, GST_JPEG_SAMPLING_BGR, GST_JPEG_SAMPLING_YBR444, GST_JPEG_SAMPLING_YBR422, GST_JPEG_SAMPLING_YBR420, GST_JPEG_SAMPLING_YBR440, GST_JPEG_SAMPLING_YBR410, GST_JPEG_SAMPLING_YBR411, GST_JPEG_SAMPLING_GRAYSCALE, }; static const gchar *gst_jpeg_sampling_strings[] = { [GST_JPEG_SAMPLING_NONE] = NULL, [GST_JPEG_SAMPLING_RGB] = "RGB", [GST_JPEG_SAMPLING_BGR] = "BGR", [GST_JPEG_SAMPLING_YBR444] = "YCbCr-4:4:4", [GST_JPEG_SAMPLING_YBR422] = "YCbCr-4:2:2", [GST_JPEG_SAMPLING_YBR420] = "YCbCr-4:2:0", [GST_JPEG_SAMPLING_YBR440] = "YCbCr-4:4:0", [GST_JPEG_SAMPLING_YBR410] = "YCbCr-4:1:0", [GST_JPEG_SAMPLING_YBR411] = "YCbCr-4:1:1", [GST_JPEG_SAMPLING_GRAYSCALE] = "GRAYSCALE", }; static void gst_jpeg_parse_class_init (GstJpegParseClass * klass) { GstBaseParseClass *gstbaseparse_class; GstElementClass *gstelement_class; gstbaseparse_class = (GstBaseParseClass *) klass; gstelement_class = (GstElementClass *) klass; gstbaseparse_class->start = gst_jpeg_parse_start; gstbaseparse_class->stop = gst_jpeg_parse_stop; gstbaseparse_class->set_sink_caps = gst_jpeg_parse_set_sink_caps; gstbaseparse_class->sink_event = gst_jpeg_parse_sink_event; gstbaseparse_class->handle_frame = gst_jpeg_parse_handle_frame; gst_element_class_add_static_pad_template (gstelement_class, &gst_jpeg_parse_src_pad_template); gst_element_class_add_static_pad_template (gstelement_class, &gst_jpeg_parse_sink_pad_template); gst_element_class_set_static_metadata (gstelement_class, "JPEG stream parser", "Codec/Parser/Image", "Parse JPEG images into single-frame buffers", "Víctor Jáquez "); GST_DEBUG_CATEGORY_INIT (jpeg_parse_debug, "jpegparse", 0, "JPEG parser"); } static void gst_jpeg_parse_init (GstJpegParse * parse) { parse->sof = -1; } static void parse_avid (GstJpegParse * parse, const guint8 * data, guint16 len) { parse->avid = 1; if (len > 14 && data[12] == 1) /* 1 - NTSC */ parse->field_order = GST_VIDEO_FIELD_ORDER_BOTTOM_FIELD_FIRST; if (len > 14 && data[12] == 2) /* 2 - PAL */ parse->field_order = GST_VIDEO_FIELD_ORDER_TOP_FIELD_FIRST; GST_INFO_OBJECT (parse, "AVID: %s", gst_video_field_order_to_string (parse->field_order)); } static gboolean gst_jpeg_parse_set_sink_caps (GstBaseParse * bparse, GstCaps * caps) { GstJpegParse *parse = GST_JPEG_PARSE_CAST (bparse); GstStructure *s = gst_caps_get_structure (caps, 0); const GValue *codec_data; const char *interlace_mode, *field_order; GST_DEBUG_OBJECT (parse, "get sink caps %" GST_PTR_FORMAT, caps); gst_structure_get_fraction (s, "framerate", &parse->framerate_numerator, &parse->framerate_denominator); gst_structure_get_int (s, "height", &parse->orig_height); gst_structure_get_int (s, "width", &parse->orig_width); codec_data = gst_structure_get_value (s, "codec_data"); if (codec_data && G_VALUE_TYPE (codec_data) == GST_TYPE_BUFFER) { GstMapInfo map; gst_clear_buffer (&parse->codec_data); parse->codec_data = GST_BUFFER (g_value_dup_boxed (codec_data)); if (gst_buffer_map (parse->codec_data, &map, GST_MAP_READ)) { if (map.size > 8 && map.data[0] == 0x2c && map.data[4] == 0x18) parse_avid (parse, map.data, map.size); gst_buffer_unmap (parse->codec_data, &map); } } interlace_mode = gst_structure_get_string (s, "interlace-mode"); if (interlace_mode) { parse->interlace_mode = gst_video_interlace_mode_from_string (interlace_mode); } if (parse->interlace_mode != GST_VIDEO_INTERLACE_MODE_PROGRESSIVE) { field_order = gst_structure_get_string (s, "field-order"); if (field_order) parse->field_order = gst_video_field_order_from_string (field_order); } g_clear_pointer (&parse->colorimetry, g_free); parse->colorimetry = g_strdup (gst_structure_get_string (s, "colorimetry")); return TRUE; } static inline gboolean valid_state (guint state, guint ref_state) { return (state & ref_state) == ref_state; } /* https://zpl.fi/chroma-subsampling-and-jpeg-sampling-factors/ */ /* *INDENT-OFF* */ static const struct { gint h[3]; gint v[3]; enum GstJPEGSampling sampling; } subsampling_map[] = { {{1, 1, 1}, {1, 1, 1}, GST_JPEG_SAMPLING_YBR444}, {{2, 2, 2}, {1, 1, 1}, GST_JPEG_SAMPLING_YBR444}, {{3, 3, 3}, {1, 1, 1}, GST_JPEG_SAMPLING_YBR444}, {{1, 1, 1}, {2, 2, 2}, GST_JPEG_SAMPLING_YBR444}, {{1, 1, 1}, {3, 3, 3}, GST_JPEG_SAMPLING_YBR444}, {{1, 1, 1}, {2, 1, 1}, GST_JPEG_SAMPLING_YBR440}, {{2, 2, 2}, {2, 1, 1}, GST_JPEG_SAMPLING_YBR440}, {{1, 1, 1}, {4, 2, 2}, GST_JPEG_SAMPLING_YBR440}, {{2, 1, 1}, {1, 1, 1}, GST_JPEG_SAMPLING_YBR422}, {{2, 1, 1}, {2, 2, 2}, GST_JPEG_SAMPLING_YBR422}, {{4, 2, 2}, {1, 1, 1}, GST_JPEG_SAMPLING_YBR422}, {{2, 1, 1}, {2, 1, 1}, GST_JPEG_SAMPLING_YBR420}, {{4, 1, 1}, {1, 1, 1}, GST_JPEG_SAMPLING_YBR411}, {{4, 1, 1}, {2, 1, 1}, GST_JPEG_SAMPLING_YBR410}, }; /* *INDENT-ON* */ static guint16 yuv_sampling (GstJpegFrameHdr * frame_hdr) { int i, h0, h1, h2, v0, v1, v2; g_return_val_if_fail (frame_hdr->num_components == 3, GST_JPEG_SAMPLING_NONE); h0 = frame_hdr->components[0].horizontal_factor; h1 = frame_hdr->components[1].horizontal_factor; h2 = frame_hdr->components[2].horizontal_factor; v0 = frame_hdr->components[0].vertical_factor; v1 = frame_hdr->components[1].vertical_factor; v2 = frame_hdr->components[2].vertical_factor; for (i = 0; i < G_N_ELEMENTS (subsampling_map); i++) { if (subsampling_map[i].h[0] == h0 && subsampling_map[i].h[1] == h1 && subsampling_map[i].h[2] == h2 && subsampling_map[i].v[0] == v0 && subsampling_map[i].v[1] == v1 && subsampling_map[i].v[2] == v2) return subsampling_map[i].sampling; } return GST_JPEG_SAMPLING_NONE; } static const gchar * colorspace_to_string (enum GstJPEGColorspace colorspace) { return gst_jpeg_colorspace_strings[colorspace]; } /* https://entropymine.wordpress.com/2018/10/22/how-is-a-jpeg-images-color-type-determined/ */ /* T-REC-T.872-201206 6.1 Colour encodings and associated values to define white and black */ static gboolean gst_jpeg_parse_sof (GstJpegParse * parse, GstJpegSegment * seg) { GstJpegFrameHdr hdr = { 0, }; guint colorspace; guint sampling; if (!gst_jpeg_segment_parse_frame_header (seg, &hdr)) { return FALSE; } colorspace = GST_JPEG_COLORSPACE_NONE; sampling = GST_JPEG_SAMPLING_NONE; switch (hdr.num_components) { case 1: colorspace = GST_JPEG_COLORSPACE_GRAY; sampling = GST_JPEG_SAMPLING_GRAYSCALE; break; case 3: if (valid_state (parse->state, GST_JPEG_PARSER_STATE_GOT_JFIF)) { colorspace = GST_JPEG_COLORSPACE_YUV; sampling = yuv_sampling (&hdr); } else { if (valid_state (parse->state, GST_JPEG_PARSER_STATE_GOT_ADOBE)) { if (parse->adobe_transform == 0) { colorspace = GST_JPEG_COLORSPACE_RGB; sampling = GST_JPEG_SAMPLING_RGB; } else if (parse->adobe_transform == 1) { colorspace = GST_JPEG_COLORSPACE_YUV;; sampling = yuv_sampling (&hdr); } else { GST_DEBUG_OBJECT (parse, "Unknown Adobe color transform code"); colorspace = GST_JPEG_COLORSPACE_YUV;; sampling = yuv_sampling (&hdr); } } else { int cid0, cid1, cid2; cid0 = hdr.components[0].identifier; cid1 = hdr.components[1].identifier; cid2 = hdr.components[2].identifier; if (cid0 == 1 && cid1 == 2 && cid2 == 3) { colorspace = GST_JPEG_COLORSPACE_YUV; sampling = yuv_sampling (&hdr); } else if (cid0 == 'R' && cid1 == 'G' && cid2 == 'B') { colorspace = GST_JPEG_COLORSPACE_RGB; sampling = GST_JPEG_SAMPLING_RGB; } else { GST_DEBUG_OBJECT (parse, "Unrecognized component IDs"); colorspace = GST_JPEG_COLORSPACE_YUV; sampling = yuv_sampling (&hdr); } } } break; case 4: if (valid_state (parse->state, GST_JPEG_PARSER_STATE_GOT_ADOBE)) { if (parse->adobe_transform == 0) { colorspace = GST_JPEG_COLORSPACE_CMYK; } else if (parse->adobe_transform == 2) { colorspace = GST_JPEG_COLORSPACE_YCCK; } else { GST_DEBUG_OBJECT (parse, "Unknown Adobe color transform code"); colorspace = GST_JPEG_COLORSPACE_YCCK; } } else { colorspace = GST_JPEG_COLORSPACE_CMYK; } break; default: GST_WARNING_OBJECT (parse, "Unknown color space"); break; } if (hdr.width != parse->width || hdr.height != parse->height || colorspace != parse->colorspace || sampling != parse->sampling) { parse->width = hdr.width; parse->height = hdr.height; parse->colorspace = colorspace; parse->sampling = sampling; if (parse->first_picture && !parse->multiscope) { if (parse->orig_height > 0 && parse->height < ((parse->orig_height * 3) / 4)) { parse->interlace_mode = GST_VIDEO_INTERLACE_MODE_INTERLEAVED; } else if (parse->avid) { if (parse->orig_height == 0) parse->orig_height = 2 * hdr.height; parse->interlace_mode = GST_VIDEO_INTERLACE_MODE_INTERLEAVED; } } parse->first_picture = FALSE; parse->renegotiate = TRUE; } GST_INFO_OBJECT (parse, "SOF [%dx%d] %d comp - %s", parse->width, parse->height, hdr.num_components, GST_STR_NULL (colorspace_to_string (parse->colorspace))); return TRUE; } static inline GstTagList * get_tag_list (GstJpegParse * parse) { if (!parse->tags) parse->tags = gst_tag_list_new_empty (); return parse->tags; } static gboolean gst_jpeg_parse_app0 (GstJpegParse * parse, GstJpegSegment * seg) { GstByteReader reader; const gchar *id_str; guint16 xd, yd; guint8 unit, xt, yt; if (seg->size < 6) /* less than 6 means no id string */ return FALSE; gst_byte_reader_init (&reader, seg->data + seg->offset, seg->size); gst_byte_reader_skip_unchecked (&reader, 2); if (!gst_byte_reader_get_string_utf8 (&reader, &id_str)) return FALSE; if (!valid_state (parse->state, GST_JPEG_PARSER_STATE_GOT_JFIF) && g_strcmp0 (id_str, "JFIF") == 0) { parse->state |= GST_JPEG_PARSER_STATE_GOT_JFIF; /* version */ gst_byte_reader_skip_unchecked (&reader, 2); /* units */ if (!gst_byte_reader_get_uint8 (&reader, &unit)) return FALSE; /* x density */ if (!gst_byte_reader_get_uint16_be (&reader, &xd)) return FALSE; /* y density */ if (!gst_byte_reader_get_uint16_be (&reader, &yd)) return FALSE; /* x thumbnail */ if (!gst_byte_reader_get_uint8 (&reader, &xt)) return FALSE; /* y thumbnail */ if (!gst_byte_reader_get_uint8 (&reader, &yt)) return FALSE; if (unit == 0) { /* no units, X and Y specify the pixel aspect ratio */ if (parse->x_density != xd || parse->y_density != yd) { parse->renegotiate = TRUE; parse->x_density = xd; parse->y_density = yd; } } else if (unit == 1 || unit == 2) { /* tag pixel per inches */ double hppi = xd, vppi = yd; /* cm to in */ if (unit == 2) { hppi *= 2.54; vppi *= 2.54; } gst_tag_register_musicbrainz_tags (); gst_tag_list_add (get_tag_list (parse), GST_TAG_MERGE_REPLACE, GST_TAG_IMAGE_HORIZONTAL_PPI, hppi, GST_TAG_IMAGE_VERTICAL_PPI, vppi, NULL); } if (xt > 0 && yt > 0) GST_FIXME_OBJECT (parse, "embedded thumbnail ignored"); return TRUE; } /* JFIF Extension */ if (g_strcmp0 (id_str, "JFXX") == 0) { if (!valid_state (parse->state, GST_JPEG_PARSER_STATE_GOT_JFIF)) return FALSE; return TRUE; } /* https://exiftool.org/TagNames/JPEG.html#AVI1 */ if (g_strcmp0 (id_str, "AVI1") == 0) { /* polarity */ if (!gst_byte_reader_get_uint8 (&reader, &unit)) return FALSE; parse->avid = TRUE; parse->field = unit == 1 ? 0 : 1; /* TODO: update caps for interlaced MJPEG */ GST_DEBUG_OBJECT (parse, "MJPEG interleaved field: %d", unit); return TRUE; } GST_MEMDUMP_OBJECT (parse, "Unhandled app0", seg->data + seg->offset, seg->size); return TRUE; } /* *INDENT-OFF* */ static const struct { const gchar *suffix; GstTagList *(*tag_func) (GstBuffer * buff); } TagMap[] = { {"Exif", gst_tag_list_from_exif_buffer_with_tiff_header}, {"http://ns.adobe.com/xap/1.0/", gst_tag_list_from_xmp_buffer}, }; /* *INDENT-ON* */ static gboolean gst_jpeg_parse_app1 (GstJpegParse * parse, GstJpegSegment * seg) { GstByteReader reader; GstBuffer *buf; guint16 size = 0; const gchar *id_str; const guint8 *data; gint i; if (seg->size < 6) /* less than 6 means no id string */ return FALSE; gst_byte_reader_init (&reader, seg->data + seg->offset, seg->size); gst_byte_reader_skip_unchecked (&reader, 2); if (!gst_byte_reader_get_string_utf8 (&reader, &id_str)) return FALSE; for (i = 0; i < G_N_ELEMENTS (TagMap); i++) { if (!g_str_has_suffix (id_str, TagMap[i].suffix)) continue; /* skip NUL only for Exif */ if (i == 0) { if (!gst_byte_reader_skip (&reader, 1)) return FALSE; } size = gst_byte_reader_get_remaining (&reader); if (!gst_byte_reader_get_data (&reader, size, &data)) return FALSE; buf = gst_buffer_new_wrapped_full (GST_MEMORY_FLAG_READONLY, (gpointer) data, size, 0, size, NULL, NULL); if (buf) { GstTagList *tags; tags = TagMap[i].tag_func (buf); gst_buffer_unref (buf); if (tags) { GST_LOG_OBJECT (parse, "parsed marker %x: '%s' %" GST_PTR_FORMAT, GST_JPEG_MARKER_APP1, id_str, tags); gst_tag_list_insert (get_tag_list (parse), tags, GST_TAG_MERGE_REPLACE); gst_tag_list_unref (tags); } else { GST_INFO_OBJECT (parse, "failed to parse %s: %s", id_str, data); return FALSE; } } return TRUE; } GST_MEMDUMP_OBJECT (parse, "Unhandled app1", seg->data + seg->offset, seg->size); return TRUE; } static gboolean gst_jpeg_parse_app14 (GstJpegParse * parse, GstJpegSegment * seg) { GstByteReader reader; const gchar *id_str; guint8 transform; if (seg->size < 6) /* less than 6 means no id string */ return FALSE; gst_byte_reader_init (&reader, seg->data + seg->offset, seg->size); gst_byte_reader_skip_unchecked (&reader, 2); if (!gst_byte_reader_get_string_utf8 (&reader, &id_str)) return FALSE; if (!g_str_has_prefix (id_str, "Adobe")) { GST_DEBUG_OBJECT (parse, "Unhandled app14: %s", id_str); return TRUE; } /* skip version and flags */ if (!gst_byte_reader_skip (&reader, 6)) return FALSE; parse->state |= GST_JPEG_PARSER_STATE_GOT_ADOBE; /* transform bit might not exist */ if (!gst_byte_reader_get_uint8 (&reader, &transform)) return TRUE; parse->adobe_transform = transform; return TRUE; } static inline gchar * get_utf8_from_data (const guint8 * data, guint16 size) { const gchar *env_vars[] = { "GST_JPEG_TAG_ENCODING", "GST_TAG_ENCODING", NULL }; const char *str = (gchar *) data; char *ret; ret = gst_tag_freeform_string_to_utf8 (str, size, env_vars); if (!ret) GST_MEMDUMP ("non-parsed marker data", data, size); return ret; } /* read comment and post as tag */ static inline gboolean gst_jpeg_parse_com (GstJpegParse * parse, GstJpegSegment * seg) { GstByteReader reader; const guint8 *data = NULL; guint16 size; const gchar *buf; gst_byte_reader_init (&reader, seg->data + seg->offset, seg->size); gst_byte_reader_skip_unchecked (&reader, 2); size = gst_byte_reader_get_remaining (&reader); if (!gst_byte_reader_get_data (&reader, size, &data)) return FALSE; buf = (const gchar *) data; /* buggy avid, it puts EOI only at every 10th frame */ if (g_str_has_prefix (buf, "AVID")) { parse_avid (parse, data, size); } else if (g_str_has_prefix (buf, "MULTISCOPE II")) { parse->x_density = 1; parse->y_density = 2; parse->multiscope = TRUE; } else { gchar *comment; comment = get_utf8_from_data (data, size); if (!comment) return FALSE; GST_INFO_OBJECT (parse, "comment found: %s", comment); gst_tag_list_add (get_tag_list (parse), GST_TAG_MERGE_REPLACE, GST_TAG_COMMENT, comment, NULL); g_free (comment); } return TRUE; } /* reset per image */ static void gst_jpeg_parse_reset (GstJpegParse * parse) { parse->last_offset = 0; parse->state = 0; parse->adobe_transform = 0; parse->field = 0; if (parse->tags) { gst_tag_list_unref (parse->tags); parse->tags = NULL; } } static const gchar * sampling_to_string (enum GstJPEGSampling sampling) { return gst_jpeg_sampling_strings[sampling]; } static gboolean gst_jpeg_parse_set_new_caps (GstJpegParse * parse) { GstCaps *caps; GstEvent *event; gboolean res; if (!parse->renegotiate) return TRUE; caps = gst_caps_new_simple ("image/jpeg", "parsed", G_TYPE_BOOLEAN, TRUE, NULL); if (parse->width > 0) gst_caps_set_simple (caps, "width", G_TYPE_INT, parse->width, NULL); if (parse->orig_height > 0 && parse->orig_height > parse->height) gst_caps_set_simple (caps, "height", G_TYPE_INT, parse->orig_height, NULL); else if (parse->height > 0) gst_caps_set_simple (caps, "height", G_TYPE_INT, parse->height, NULL); if (parse->sof >= 0) gst_caps_set_simple (caps, "sof-marker", G_TYPE_INT, parse->sof, NULL); if (parse->colorspace != GST_JPEG_COLORSPACE_NONE) { gst_caps_set_simple (caps, "colorspace", G_TYPE_STRING, colorspace_to_string (parse->colorspace), NULL); } if (parse->sampling != GST_JPEG_SAMPLING_NONE) { gst_caps_set_simple (caps, "sampling", G_TYPE_STRING, sampling_to_string (parse->sampling), NULL); } if (parse->colorimetry) { gst_caps_set_simple (caps, "colorimetry", G_TYPE_STRING, parse->colorimetry, NULL); } gst_caps_set_simple (caps, "interlace-mode", G_TYPE_STRING, gst_video_interlace_mode_to_string (parse->interlace_mode), NULL); if (parse->interlace_mode == GST_VIDEO_INTERLACE_MODE_INTERLEAVED) { gst_caps_set_simple (caps, "field-order", G_TYPE_STRING, gst_video_field_order_to_string (parse->field_order), NULL); } gst_caps_set_simple (caps, "framerate", GST_TYPE_FRACTION, parse->framerate_numerator, parse->framerate_denominator, NULL); if (parse->x_density > 0 && parse->y_density > 0) { gst_caps_set_simple (caps, "pixel-aspect-ratio", GST_TYPE_FRACTION, parse->x_density, parse->y_density, NULL); } if (parse->codec_data) { gst_caps_set_simple (caps, "codec_data", GST_TYPE_BUFFER, parse->codec_data, NULL); } parse->renegotiate = FALSE; GST_DEBUG_OBJECT (parse, "setting downstream caps on %s:%s to %" GST_PTR_FORMAT, GST_DEBUG_PAD_NAME (GST_BASE_PARSE_SRC_PAD (parse)), caps); event = gst_event_new_caps (caps); res = gst_pad_push_event (GST_BASE_PARSE_SRC_PAD (parse), event); gst_caps_unref (caps); return res; } static GstFlowReturn gst_jpeg_parse_finish_frame (GstJpegParse * parse, GstBaseParseFrame * frame, gint size) { GstBaseParse *bparse = GST_BASE_PARSE (parse); GstFlowReturn ret; if (parse->tags) gst_base_parse_merge_tags (bparse, parse->tags, GST_TAG_MERGE_REPLACE); if (!gst_jpeg_parse_set_new_caps (parse)) return GST_FLOW_ERROR; if (!valid_state (parse->state, GST_JPEG_PARSER_STATE_VALID_PICTURE)) { /* this validation breaks unit tests */ /* frame->flags |= GST_BASE_PARSE_FRAME_FLAG_DROP; */ GST_WARNING_OBJECT (parse, "Potentially invalid picture"); } ret = gst_base_parse_finish_frame (bparse, frame, size); gst_jpeg_parse_reset (parse); return ret; } static GstFlowReturn gst_jpeg_parse_handle_frame (GstBaseParse * bparse, GstBaseParseFrame * frame, gint * skipsize) { GstJpegParse *parse = GST_JPEG_PARSE_CAST (bparse); GstMapInfo mapinfo; GstJpegMarker marker; GstJpegSegment seg; guint offset; GST_TRACE_OBJECT (parse, "frame %" GST_PTR_FORMAT, frame->buffer); if (!gst_buffer_map (frame->buffer, &mapinfo, GST_MAP_READ)) return GST_FLOW_ERROR; offset = parse->last_offset; if (offset > 0) offset -= 1; /* it migth be in the middle marker */ while (offset < mapinfo.size) { if (!gst_jpeg_parse (&seg, mapinfo.data, mapinfo.size, offset)) { if (!valid_state (parse->state, GST_JPEG_PARSER_STATE_GOT_SOI)) { /* Skip any garbage until SOI */ *skipsize = mapinfo.size; GST_INFO_OBJECT (parse, "skipping %d bytes", *skipsize); } else { /* Accept anything after SOI */ parse->last_offset = mapinfo.size; } goto beach; } offset = seg.offset; marker = seg.marker; if (!valid_state (parse->state, GST_JPEG_PARSER_STATE_GOT_SOI) && marker != GST_JPEG_MARKER_SOI) continue; /* check if the whole segment is available */ if (offset + seg.size > mapinfo.size) { GST_DEBUG_OBJECT (parse, "incomplete segment: %x [offset %d]", marker, offset); parse->last_offset = offset - 2; goto beach; } offset += seg.size; GST_LOG_OBJECT (parse, "marker found: %x [offset %d / size %" G_GSSIZE_FORMAT "]", marker, seg.offset, seg.size); switch (marker) { case GST_JPEG_MARKER_SOI: /* This means that new SOI comes without an previous EOI. */ if (offset > 2 && (parse->interlace_mode == GST_VIDEO_INTERLACE_MODE_PROGRESSIVE || parse->field == 0)) { /* If already some data segment parsed, push it as a frame. */ if (valid_state (parse->state, GST_JPEG_PARSER_STATE_GOT_SOS)) { gst_buffer_unmap (frame->buffer, &mapinfo); frame->out_buffer = gst_buffer_copy_region (frame->buffer, GST_BUFFER_COPY_ALL, 0, seg.offset - 2); GST_MINI_OBJECT_FLAGS (frame->out_buffer) |= GST_BUFFER_FLAG_CORRUPTED; GST_WARNING_OBJECT (parse, "Push a frame without EOI, size %d", seg.offset - 2); return gst_jpeg_parse_finish_frame (parse, frame, seg.offset - 2); } gst_jpeg_parse_reset (parse); parse->state |= GST_JPEG_PARSER_STATE_GOT_SOI; /* unset tags */ gst_base_parse_merge_tags (bparse, NULL, GST_TAG_MERGE_UNDEFINED); *skipsize = offset - 2; GST_DEBUG_OBJECT (parse, "skipping %d bytes before SOI", *skipsize); parse->last_offset = 2; goto beach; } /* unset tags */ gst_base_parse_merge_tags (bparse, NULL, GST_TAG_MERGE_UNDEFINED); parse->state |= GST_JPEG_PARSER_STATE_GOT_SOI; break; case GST_JPEG_MARKER_EOI: if (parse->interlace_mode == GST_VIDEO_INTERLACE_MODE_PROGRESSIVE || parse->field == 1) { gst_buffer_unmap (frame->buffer, &mapinfo); return gst_jpeg_parse_finish_frame (parse, frame, seg.offset); } else if (parse->interlace_mode == GST_VIDEO_INTERLACE_MODE_INTERLEAVED && parse->field == 0) { parse->field = 1; parse->state = 0; } break; case GST_JPEG_MARKER_SOS: if (!valid_state (parse->state, GST_JPEG_PARSER_STATE_GOT_SOF)) GST_WARNING_OBJECT (parse, "SOS marker without SOF one"); parse->state |= GST_JPEG_PARSER_STATE_GOT_SOS; break; case GST_JPEG_MARKER_COM: if (!gst_jpeg_parse_com (parse, &seg)) GST_WARNING_OBJECT (parse, "Failed to parse com segment"); break; case GST_JPEG_MARKER_APP0: if (!gst_jpeg_parse_app0 (parse, &seg)) { GST_ELEMENT_WARNING (parse, STREAM, FORMAT, ("Invalid data"), ("Failed to parse app0 segment")); } break; case GST_JPEG_MARKER_APP1: if (!gst_jpeg_parse_app1 (parse, &seg)) { GST_ELEMENT_WARNING (parse, STREAM, FORMAT, ("Invalid data"), ("Failed to parse app1 segment")); } break; case GST_JPEG_MARKER_APP14: if (!gst_jpeg_parse_app14 (parse, &seg)) { GST_ELEMENT_WARNING (parse, STREAM, FORMAT, ("Invalid data"), ("Failed to parse app14 segment")); } break; case GST_JPEG_MARKER_DHT: case GST_JPEG_MARKER_DAC: /* to avoid break the below SOF interval */ break; default: /* SOFn segments */ if (marker >= GST_JPEG_MARKER_SOF_MIN && marker <= GST_JPEG_MARKER_SOF_MAX) { if (!valid_state (parse->state, GST_JPEG_PARSER_STATE_GOT_SOF) && gst_jpeg_parse_sof (parse, &seg)) { gint8 sof; parse->state |= GST_JPEG_PARSER_STATE_GOT_SOF; sof = marker - 0xc0; if (parse->sof != sof) { parse->sof = sof; parse->renegotiate = TRUE; } } else { GST_ELEMENT_ERROR (parse, STREAM, FORMAT, ("Invalid data"), ("Duplicated or bad SOF marker")); gst_buffer_unmap (frame->buffer, &mapinfo); gst_jpeg_parse_reset (parse); return GST_FLOW_ERROR; } } break; } } parse->last_offset = offset; beach: gst_buffer_unmap (frame->buffer, &mapinfo); return GST_FLOW_OK; } static gboolean gst_jpeg_parse_sink_event (GstBaseParse * bparse, GstEvent * event) { GstJpegParse *parse = GST_JPEG_PARSE_CAST (bparse); GST_DEBUG_OBJECT (parse, "event : %s", GST_EVENT_TYPE_NAME (event)); switch (GST_EVENT_TYPE (event)) { case GST_EVENT_FLUSH_STOP: gst_jpeg_parse_reset (parse); break; default: break; } return GST_BASE_PARSE_CLASS (parent_class)->sink_event (bparse, event); } static gboolean gst_jpeg_parse_start (GstBaseParse * bparse) { GstJpegParse *parse = GST_JPEG_PARSE_CAST (bparse); parse->framerate_numerator = 0; parse->framerate_denominator = 1; parse->first_picture = TRUE; parse->renegotiate = TRUE; parse->interlace_mode = GST_VIDEO_INTERLACE_MODE_PROGRESSIVE; parse->field_order = GST_VIDEO_FIELD_ORDER_TOP_FIELD_FIRST; gst_jpeg_parse_reset (parse); gst_base_parse_set_min_frame_size (bparse, 2); return TRUE; } static gboolean gst_jpeg_parse_stop (GstBaseParse * bparse) { GstJpegParse *parse = GST_JPEG_PARSE_CAST (bparse); if (parse->tags) { gst_tag_list_unref (parse->tags); parse->tags = NULL; } gst_clear_buffer (&parse->codec_data); gst_clear_caps (&parse->prev_caps); g_clear_pointer (&parse->colorimetry, g_free); return TRUE; }