v4l2: add alternate interlace mode

When using this mode each frame is split in two fields, each one being
transferred using its own buffer.
This is implemented with the V4L2_FIELD_ALTERNATE field format in v4l2.

This mode is enabled using a caps filter such as
"v4l2src ! video/x-raw\(format:Interlaced\)"

Here are the main changes related to this feature:

- use the INTERLACED caps feature with this mode.

- in this mode both fields of a given frame have the same sequence/offset
so adjust the algorithm checking for lost field/frame accordingly.

- double pool's min number of buffers as each frame requires 2 buffers.

Fix #504
Co-authored-by: Zeeshan Ali <zeenix@collabora.co.uk>
This commit is contained in:
Guillaume Desmottes 2018-08-08 09:27:19 +02:00 committed by Guillaume Desmottes
parent 0834ef15ce
commit d1501f0952
5 changed files with 184 additions and 47 deletions

View file

@ -1149,22 +1149,12 @@ gst_v4l2_buffer_pool_qbuf (GstV4l2BufferPool * pool, GstBuffer * buf,
if (V4L2_TYPE_IS_OUTPUT (obj->type)) {
enum v4l2_field field;
/* Except when field is set to alternate, buffer field is the same as
* the one defined in format */
/* Buffer field is the same as the one defined in format */
if (V4L2_TYPE_IS_MULTIPLANAR (obj->type))
field = obj->format.fmt.pix_mp.field;
else
field = obj->format.fmt.pix.field;
/* NB: At this moment, we can't have alternate mode because it not handled
* yet */
if (field == V4L2_FIELD_ALTERNATE) {
if (GST_BUFFER_FLAG_IS_SET (buf, GST_VIDEO_FRAME_FLAG_TFF))
field = V4L2_FIELD_TOP;
else
field = V4L2_FIELD_BOTTOM;
}
group->buffer.field = field;
}
@ -1303,6 +1293,14 @@ gst_v4l2_buffer_pool_dqbuf (GstV4l2BufferPool * pool, GstBuffer ** buffer,
GST_BUFFER_FLAG_UNSET (outbuf, GST_VIDEO_BUFFER_FLAG_INTERLACED);
GST_BUFFER_FLAG_UNSET (outbuf, GST_VIDEO_BUFFER_FLAG_TFF);
break;
case V4L2_FIELD_TOP:
GST_BUFFER_FLAG_SET (outbuf, GST_VIDEO_BUFFER_FLAG_INTERLACED);
GST_BUFFER_FLAG_SET (outbuf, GST_VIDEO_BUFFER_FLAG_TOP_FIELD);
break;
case V4L2_FIELD_BOTTOM:
GST_BUFFER_FLAG_SET (outbuf, GST_VIDEO_BUFFER_FLAG_INTERLACED);
GST_BUFFER_FLAG_SET (outbuf, GST_VIDEO_BUFFER_FLAG_BOTTOM_FIELD);
break;
case V4L2_FIELD_INTERLACED_TB:
GST_BUFFER_FLAG_SET (outbuf, GST_VIDEO_BUFFER_FLAG_INTERLACED);
GST_BUFFER_FLAG_SET (outbuf, GST_VIDEO_BUFFER_FLAG_TFF);

View file

@ -1617,6 +1617,25 @@ done:
return template;
}
/* Add an 'alternate' variant of the caps with the feature */
static void
add_alternate_variant (GstV4l2Object * v4l2object, GstCaps * caps,
GstStructure * structure)
{
GstStructure *alt_s;
if (v4l2object && v4l2object->never_interlaced)
return;
if (!gst_structure_has_name (structure, "video/x-raw"))
return;
alt_s = gst_structure_copy (structure);
gst_structure_set (alt_s, "interlace-mode", G_TYPE_STRING, "alternate", NULL);
gst_caps_append_structure_full (caps, alt_s,
gst_caps_features_new (GST_CAPS_FEATURE_FORMAT_INTERLACED, NULL));
}
static GstCaps *
gst_v4l2_object_get_caps_helper (GstV4L2FormatFlags flags)
@ -1660,6 +1679,8 @@ gst_v4l2_object_get_caps_helper (GstV4L2FormatFlags flags)
if (alt_s)
gst_caps_append_structure (caps, alt_s);
add_alternate_variant (NULL, caps, structure);
}
}
@ -1984,6 +2005,9 @@ gst_v4l2_object_get_interlace_mode (enum v4l2_field field,
case V4L2_FIELD_INTERLACED_BT:
*interlace_mode = GST_VIDEO_INTERLACE_MODE_INTERLEAVED;
return TRUE;
case V4L2_FIELD_ALTERNATE:
*interlace_mode = GST_VIDEO_INTERLACE_MODE_ALTERNATE;
return TRUE;
default:
GST_ERROR ("Unknown enum v4l2_field %d", field);
return FALSE;
@ -2214,7 +2238,9 @@ gst_v4l2_object_add_interlace_mode (GstV4l2Object * v4l2object,
{
struct v4l2_format fmt;
GValue interlace_formats = { 0, };
enum v4l2_field formats[] = { V4L2_FIELD_NONE, V4L2_FIELD_INTERLACED };
enum v4l2_field formats[] = { V4L2_FIELD_NONE,
V4L2_FIELD_INTERLACED, V4L2_FIELD_ALTERNATE
};
gsize i;
GstVideoInterlaceMode interlace_mode, prev = -1;
@ -2228,7 +2254,7 @@ gst_v4l2_object_add_interlace_mode (GstV4l2Object * v4l2object,
g_value_init (&interlace_formats, GST_TYPE_LIST);
/* Try twice - once for NONE, once for INTERLACED. */
/* Try thrice - once for NONE, once for INTERLACED and once for ALTERNATE. */
for (i = 0; i < G_N_ELEMENTS (formats); i++) {
memset (&fmt, 0, sizeof (fmt));
fmt.type = v4l2object->type;
@ -2237,8 +2263,19 @@ gst_v4l2_object_add_interlace_mode (GstV4l2Object * v4l2object,
fmt.fmt.pix.pixelformat = pixelformat;
fmt.fmt.pix.field = formats[i];
if (gst_v4l2_object_try_fmt (v4l2object, &fmt) == 0 &&
gst_v4l2_object_get_interlace_mode (fmt.fmt.pix.field, &interlace_mode)
if (fmt.fmt.pix.field == V4L2_FIELD_ALTERNATE)
fmt.fmt.pix.height /= 2;
/* if skip_try_fmt_probes is set it's up to the caller to filter out the
* formats from the formats requested by peer.
* For this negotiation to work with 'alternate' we need the caps to contain
* the feature so we have an intersection with downstream caps.
*/
if (!v4l2object->skip_try_fmt_probes
&& gst_v4l2_object_try_fmt (v4l2object, &fmt) != 0)
continue;
if (gst_v4l2_object_get_interlace_mode (fmt.fmt.pix.field, &interlace_mode)
&& prev != interlace_mode) {
GValue interlace_enum = { 0, };
const gchar *mode_string;
@ -2574,6 +2611,54 @@ sort_by_frame_size (GstStructure * s1, GstStructure * s2)
return ((w2 * h2) - (w1 * h1));
}
static void
check_alternate_and_append_struct (GstCaps * caps, GstStructure * s)
{
const GValue *mode;
mode = gst_structure_get_value (s, "interlace-mode");
if (!mode)
goto done;
if (G_VALUE_HOLDS_STRING (mode)) {
/* Add the INTERLACED feature if the mode is alternate */
if (!g_strcmp0 (gst_structure_get_string (s, "interlace-mode"),
"alternate")) {
GstCapsFeatures *feat;
feat = gst_caps_features_new (GST_CAPS_FEATURE_FORMAT_INTERLACED, NULL);
gst_caps_set_features (caps, gst_caps_get_size (caps) - 1, feat);
}
} else if (GST_VALUE_HOLDS_LIST (mode)) {
/* If the mode is a list containing alternate, remove it from the list and add a
* variant with interlace-mode=alternate and the INTERLACED feature. */
GValue alter = G_VALUE_INIT;
GValue inter = G_VALUE_INIT;
g_value_init (&alter, G_TYPE_STRING);
g_value_set_string (&alter, "alternate");
/* Cannot use gst_value_can_intersect() as it requires args to have the
* same type. */
if (gst_value_intersect (&inter, mode, &alter)) {
GValue minus_alter = G_VALUE_INIT;
GstStructure *copy;
gst_value_subtract (&minus_alter, mode, &alter);
gst_structure_take_value (s, "interlace-mode", &minus_alter);
copy = gst_structure_copy (s);
gst_structure_take_value (copy, "interlace-mode", &inter);
gst_caps_append_structure_full (caps, copy,
gst_caps_features_new (GST_CAPS_FEATURE_FORMAT_INTERLACED, NULL));
}
g_value_unset (&alter);
}
done:
gst_caps_append_structure (caps, s);
}
static void
gst_v4l2_object_update_and_append (GstV4l2Object * v4l2object,
guint32 format, GstCaps * caps, GstStructure * s)
@ -2612,10 +2697,11 @@ gst_v4l2_object_update_and_append (GstV4l2Object * v4l2object,
}
}
gst_caps_append_structure (caps, s);
check_alternate_and_append_struct (caps, s);
if (alt_s)
gst_caps_append_structure (caps, alt_s);
if (alt_s) {
check_alternate_and_append_struct (caps, alt_s);
}
}
static GstCaps *
@ -2850,10 +2936,11 @@ default_frame_sizes:
gst_v4l2_object_add_aspect_ratio (v4l2object, tmp);
/* We could consider setting interlace mode from min and max. */
gst_v4l2_object_add_interlace_mode (v4l2object, tmp, max_w, max_h,
pixelformat);
if (!v4l2object->skip_try_fmt_probes) {
/* We could consider setting interlace mode from min and max. */
gst_v4l2_object_add_interlace_mode (v4l2object, tmp, max_w, max_h,
pixelformat);
/* We could consider to check colorspace for min too, in case it depends on
* the size. But in this case, min and max could not be enough */
gst_v4l2_object_add_colorspace (v4l2object, tmp, max_w, max_h,
@ -3220,6 +3307,9 @@ store_info:
if (info->fps_n > 0 && info->fps_d > 0) {
v4l2object->duration = gst_util_uint64_scale_int (GST_SECOND, info->fps_d,
info->fps_n);
if (GST_VIDEO_INFO_INTERLACE_MODE (info) ==
GST_VIDEO_INTERLACE_MODE_ALTERNATE)
v4l2object->duration /= 2;
} else {
v4l2object->duration = GST_CLOCK_TIME_NONE;
}
@ -3249,6 +3339,27 @@ gst_v4l2_object_extrapolate_stride (const GstVideoFormatInfo * finfo,
return estride;
}
static enum v4l2_field
get_v4l2_field_for_info (GstVideoInfo * info)
{
if (!GST_VIDEO_INFO_IS_INTERLACED (info))
return V4L2_FIELD_NONE;
if (GST_VIDEO_INFO_INTERLACE_MODE (info) ==
GST_VIDEO_INTERLACE_MODE_ALTERNATE)
return V4L2_FIELD_ALTERNATE;
switch (GST_VIDEO_INFO_FIELD_ORDER (info)) {
case GST_VIDEO_FIELD_ORDER_TOP_FIELD_FIRST:
return V4L2_FIELD_INTERLACED_TB;
case GST_VIDEO_FIELD_ORDER_BOTTOM_FIELD_FIRST:
return V4L2_FIELD_INTERLACED_BT;
case GST_VIDEO_FIELD_ORDER_UNKNOWN:
default:
return V4L2_FIELD_INTERLACED;
}
}
static gboolean
gst_v4l2_video_colorimetry_matches (const GstVideoColorimetry * cinfo,
const gchar * color)
@ -3356,16 +3467,11 @@ gst_v4l2_object_set_format_full (GstV4l2Object * v4l2object, GstCaps * caps,
if (!n_v4l_planes || !v4l2object->prefered_non_contiguous)
n_v4l_planes = 1;
if (GST_VIDEO_INFO_IS_INTERLACED (&info)) {
GST_DEBUG_OBJECT (v4l2object->dbg_obj, "interlaced video");
/* ideally we would differentiate between types of interlaced video
* but there is not sufficient information in the caps..
*/
field = V4L2_FIELD_INTERLACED;
} else {
GST_DEBUG_OBJECT (v4l2object->dbg_obj, "progressive video");
field = V4L2_FIELD_NONE;
}
field = get_v4l2_field_for_info (&info);
if (field != V4L2_FIELD_NONE)
GST_DEBUG_OBJECT (v4l2object->element, "interlaced video");
else
GST_DEBUG_OBJECT (v4l2object->element, "progressive video");
/* We first pick the main colorspace from the primaries */
switch (info.colorimetry.primaries) {
@ -3675,17 +3781,13 @@ gst_v4l2_object_set_format_full (GstV4l2Object * v4l2object, GstCaps * caps,
}
/* In case we have skipped the try_fmt probes, we'll need to set the
* colorimetry and interlace-mode back into the caps. */
* colorimetry back into the caps. */
if (v4l2object->skip_try_fmt_probes) {
if (!disable_colorimetry && !gst_structure_has_field (s, "colorimetry")) {
gchar *str = gst_video_colorimetry_to_string (&info.colorimetry);
gst_structure_set (s, "colorimetry", G_TYPE_STRING, str, NULL);
g_free (str);
}
if (!gst_structure_has_field (s, "interlace-mode"))
gst_structure_set (s, "interlace-mode", G_TYPE_STRING,
gst_video_interlace_mode_to_string (info.interlace_mode), NULL);
}
if (try_only) /* good enough for trying only */
@ -3989,6 +4091,7 @@ gst_v4l2_object_acquire_format (GstV4l2Object * v4l2object, GstVideoInfo * info)
GstVideoFormat format;
guint width, height;
GstVideoAlignment align;
GstVideoInterlaceMode interlace_mode;
gst_video_info_init (info);
gst_video_alignment_reset (&align);
@ -4038,22 +4141,26 @@ gst_v4l2_object_acquire_format (GstV4l2Object * v4l2object, GstVideoInfo * info)
height = r->height;
}
gst_video_info_set_format (info, format, width, height);
switch (fmt.fmt.pix.field) {
case V4L2_FIELD_ANY:
case V4L2_FIELD_NONE:
info->interlace_mode = GST_VIDEO_INTERLACE_MODE_PROGRESSIVE;
interlace_mode = GST_VIDEO_INTERLACE_MODE_PROGRESSIVE;
break;
case V4L2_FIELD_INTERLACED:
case V4L2_FIELD_INTERLACED_TB:
case V4L2_FIELD_INTERLACED_BT:
info->interlace_mode = GST_VIDEO_INTERLACE_MODE_INTERLEAVED;
interlace_mode = GST_VIDEO_INTERLACE_MODE_INTERLEAVED;
break;
case V4L2_FIELD_ALTERNATE:
interlace_mode = GST_VIDEO_INTERLACE_MODE_ALTERNATE;
break;
default:
goto unsupported_field;
}
gst_video_info_set_interlaced_format (info, format, interlace_mode, width,
height);
gst_v4l2_object_get_colorspace (&fmt, &info->colorimetry);
gst_v4l2_object_save_format (v4l2object, fmtdesc, &fmt, info, &align);
@ -4322,9 +4429,14 @@ gst_v4l2_object_probe_caps (GstV4l2Object * v4l2object, GstCaps * filter)
tmp = gst_v4l2_object_probe_caps_for_format (v4l2object,
format->pixelformat, template);
if (tmp)
if (tmp) {
gst_caps_append (ret, tmp);
/* Add a variant of the caps with the Interlaced feature so we can negotiate it if needed */
add_alternate_variant (v4l2object, ret, gst_caps_get_structure (ret,
gst_caps_get_size (ret) - 1));
}
gst_structure_free (template);
}

View file

@ -42,8 +42,12 @@ typedef struct _GstV4l2ObjectClassHelper GstV4l2ObjectClassHelper;
#include <gstv4l2bufferpool.h>
/* size of v4l2 buffer pool in streaming case */
#define GST_V4L2_MIN_BUFFERS(obj) 2
/* size of v4l2 buffer pool in streaming case, obj->info needs to be valid */
#define GST_V4L2_MIN_BUFFERS(obj) \
((GST_VIDEO_INFO_INTERLACE_MODE (&obj->info) == \
GST_VIDEO_INTERLACE_MODE_ALTERNATE) ? \
/* 2x buffers needed with each field in its own buffer */ \
4 : 2)
/* max frame width/height */
#define GST_V4L2_MAX_SIZE (1<<15) /* 2^15 == 32768 */

View file

@ -696,8 +696,11 @@ gst_v4l2src_query (GstBaseSrc * bsrc, GstQuery * query)
goto done;
}
/* min latency is the time to capture one frame */
/* min latency is the time to capture one frame/field */
min_latency = gst_util_uint64_scale_int (GST_SECOND, fps_d, fps_n);
if (GST_VIDEO_INFO_INTERLACE_MODE (&obj->info) ==
GST_VIDEO_INTERLACE_MODE_ALTERNATE)
min_latency /= 2;
/* max latency is total duration of the frame buffer */
if (obj->pool != NULL)
@ -739,6 +742,7 @@ gst_v4l2src_start (GstBaseSrc * src)
GstV4l2Src *v4l2src = GST_V4L2SRC (src);
v4l2src->offset = 0;
v4l2src->next_offset_same = FALSE;
v4l2src->renegotiation_adjust = 0;
/* activate settings for first frame */
@ -828,6 +832,7 @@ gst_v4l2src_create (GstPushSrc * src, GstBuffer ** buf)
GstClockTime abs_time, base_time, timestamp, duration;
GstClockTime delay;
GstMessage *qos_msg;
gboolean half_frame;
do {
ret = GST_BASE_SRC_CLASS (parent_class)->alloc (GST_BASE_SRC (src), 0,
@ -920,7 +925,7 @@ retry:
" delay %" GST_TIME_FORMAT, GST_TIME_ARGS (timestamp),
GST_TIME_ARGS (gstnow), GST_TIME_ARGS (delay));
} else {
/* we assume 1 frame latency otherwise */
/* we assume 1 frame/field latency otherwise */
if (GST_CLOCK_TIME_IS_VALID (duration))
delay = duration;
else
@ -956,12 +961,28 @@ retry:
GST_LOG_OBJECT (src, "sync to %" GST_TIME_FORMAT " out ts %" GST_TIME_FORMAT,
GST_TIME_ARGS (v4l2src->ctrl_time), GST_TIME_ARGS (timestamp));
if (v4l2src->next_offset_same &&
GST_BUFFER_OFFSET_IS_VALID (*buf) &&
GST_BUFFER_OFFSET (*buf) != v4l2src->offset) {
/* Probably had a lost field then, best to forget about last field. */
GST_WARNING_OBJECT (v4l2src,
"lost field detected - ts: %" GST_TIME_FORMAT,
GST_TIME_ARGS (timestamp));
v4l2src->next_offset_same = FALSE;
}
half_frame = (GST_BUFFER_FLAG_IS_SET (*buf, GST_VIDEO_BUFFER_FLAG_ONEFIELD));
if (half_frame)
v4l2src->next_offset_same = !v4l2src->next_offset_same;
/* use generated offset values only if there are not already valid ones
* set by the v4l2 device */
if (!GST_BUFFER_OFFSET_IS_VALID (*buf)
|| !GST_BUFFER_OFFSET_END_IS_VALID (*buf)) {
GST_BUFFER_OFFSET (*buf) = v4l2src->offset++;
GST_BUFFER_OFFSET_END (*buf) = v4l2src->offset;
GST_BUFFER_OFFSET (*buf) = v4l2src->offset;
GST_BUFFER_OFFSET_END (*buf) = v4l2src->offset + 1;
if (!half_frame || !v4l2src->next_offset_same)
v4l2src->offset++;
} else {
/* adjust raw v4l2 device sequence, will restart at null in case of renegotiation
* (streamoff/streamon) */
@ -969,6 +990,7 @@ retry:
GST_BUFFER_OFFSET_END (*buf) += v4l2src->renegotiation_adjust;
/* check for frame loss with given (from v4l2 device) buffer offset */
if ((v4l2src->offset != 0)
&& (!half_frame || v4l2src->next_offset_same)
&& (GST_BUFFER_OFFSET (*buf) != (v4l2src->offset + 1))) {
guint64 lost_frame_count = GST_BUFFER_OFFSET (*buf) - v4l2src->offset - 1;
GST_WARNING_OBJECT (v4l2src,

View file

@ -56,6 +56,7 @@ struct _GstV4l2Src
GstV4l2Object * v4l2object;
guint64 offset;
gboolean next_offset_same;
/* offset adjust after renegotiation */
guint64 renegotiation_adjust;