multifile: add splitfilesrc element

Add new splitfilesrc element that presents multiple files
(selectable via a location regex) as one single contiguous
file.
This commit is contained in:
Tim-Philipp Müller 2011-07-28 20:19:56 +01:00
parent 1e6bd5ad57
commit 2dd67036e5
5 changed files with 675 additions and 4 deletions

View file

@ -208,6 +208,7 @@ dnl *** checks for dependency libraries ***
dnl GLib is required
AG_GST_GLIB_CHECK([2.20])
PKG_CHECK_MODULES(GIO, [ gio-2.0 >= 2.20 ], , AC_MSG_ERROR([gio is required]))
dnl Orc
ORC_CHECK([0.4.11])

View file

@ -1,13 +1,17 @@
plugin_LTLIBRARIES = libgstmultifile.la
libgstmultifile_la_SOURCES = gstmultifilesink.c gstmultifilesrc.c gstmultifile.c
libgstmultifile_la_CFLAGS = $(GST_BASE_CFLAGS) $(GST_CFLAGS)
libgstmultifile_la_LIBADD = $(GST_BASE_LIBS) $(GST_LIBS)
libgstmultifile_la_SOURCES = \
gstmultifilesink.c \
gstmultifilesrc.c \
gstmultifile.c \
gstsplitfilesrc.c
libgstmultifile_la_CFLAGS = $(GST_BASE_CFLAGS) $(GST_CFLAGS) $(GIO_CFLAGS)
libgstmultifile_la_LIBADD = $(GST_BASE_LIBS) $(GST_LIBS) $(GIO_LIBS)
libgstmultifile_la_LDFLAGS = $(GST_PLUGIN_LDFLAGS)
libgstmultifile_la_LIBTOOLFLAGS = --tag=disable-static
noinst_HEADERS = gstmultifilesrc.h gstmultifilesink.h
noinst_HEADERS = gstmultifilesrc.h gstmultifilesink.h gstsplitfilesrc.h
Android.mk: Makefile.am $(BUILT_SOURCES)

View file

@ -30,6 +30,7 @@
#include "gstmultifilesink.h"
#include "gstmultifilesrc.h"
#include "gstsplitfilesrc.h"
static gboolean
plugin_init (GstPlugin * plugin)
@ -38,6 +39,8 @@ plugin_init (GstPlugin * plugin)
gst_multi_file_src_get_type ());
gst_element_register (plugin, "multifilesink", GST_RANK_NONE,
gst_multi_file_sink_get_type ());
gst_element_register (plugin, "splitfilesrc", GST_RANK_NONE,
gst_split_file_src_get_type ());
return TRUE;
}

View file

@ -0,0 +1,589 @@
/* GStreamer Split File Source
* Copyright (C) 2011 Collabora Ltd. <tim.muller@collabora.co.uk>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 59 Temple Place - Suite 330,
* Boston, MA 02111-1307, USA.
*/
/**
* SECTION:element-splitfilesrc
* @see_also: #GstFileSrc, #GstMultiFileSrc
*
* Reads data from multiple files, presenting those files as one continuous
* file to downstream elements. This is useful for reading a large file that
* had to be split into multiple parts due to filesystem file size limitations,
* for example.
*
* The files to select are chosen via the location property, which takes a
* regular expression (note: shell-style wildcards will not work). If the
* location is an absolute path or contains directory components, only the
* base file name part will be considered a regular expression. The results
* will be sorted. The location may include directory components, but the
* regular expression to select the files can only be in the filename part.
*
* <refsect2>
* <title>Example launch line</title>
* |[
* gst-launch splitfilesrc location="/path/to/part-.*.mpg" ! decodebin ! ... \
* ]| Plays the different parts as if they were one single MPEG file.
* </refsect2>
*
* Since: 0.10.31
*/
/* TODO:
* - implement splitfile:// URI handler?
*/
#ifdef HAVE_CONFIG_H
# include "config.h"
#endif
#include "gstsplitfilesrc.h"
#include <string.h>
enum
{
PROP_LOCATION = 1
};
#define DEFAULT_LOCATION NULL
static void gst_split_file_src_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec);
static void gst_split_file_src_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec);
static void gst_split_file_src_finalize (GObject * obj);
static gboolean gst_split_file_src_start (GstBaseSrc * basesrc);
static gboolean gst_split_file_src_stop (GstBaseSrc * basesrc);
static gboolean gst_split_file_src_can_seek (GstBaseSrc * basesrc);
static gboolean gst_split_file_src_check_get_range (GstBaseSrc * basesrc);
static gboolean gst_split_file_src_get_size (GstBaseSrc * basesrc, guint64 * s);
static gboolean gst_split_file_src_unlock (GstBaseSrc * basesrc);
static GstFlowReturn gst_split_file_src_create (GstBaseSrc * basesrc,
guint64 offset, guint size, GstBuffer ** buffer);
static GstStaticPadTemplate gst_split_file_src_pad_template =
GST_STATIC_PAD_TEMPLATE ("src",
GST_PAD_SRC,
GST_PAD_ALWAYS,
GST_STATIC_CAPS_ANY);
GST_DEBUG_CATEGORY_STATIC (splitfilesrc_debug);
#define GST_CAT_DEFAULT splitfilesrc_debug
GST_BOILERPLATE (GstSplitFileSrc, gst_split_file_src, GstBaseSrc,
GST_TYPE_BASE_SRC);
static void
gst_split_file_src_base_init (gpointer g_class)
{
GstElementClass *gstelement_class = GST_ELEMENT_CLASS (g_class);
GST_DEBUG_CATEGORY_INIT (splitfilesrc_debug, "splitfilesrc", 0,
"splitfilesrc element");
gst_element_class_add_pad_template (gstelement_class,
gst_static_pad_template_get (&gst_split_file_src_pad_template));
gst_element_class_set_details_simple (gstelement_class, "Split-File Source",
"Source/File",
"Read a sequentially named set of files as if it was one large file",
"Tim-Philipp Müller <tim.muller@collabora.co.uk>");
}
static void
gst_split_file_src_class_init (GstSplitFileSrcClass * klass)
{
GstBaseSrcClass *gstbasesrc_class = GST_BASE_SRC_CLASS (klass);
GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
gobject_class->set_property = gst_split_file_src_set_property;
gobject_class->get_property = gst_split_file_src_get_property;
gobject_class->finalize = gst_split_file_src_finalize;
/* We're using a regular expression here instead of wildcards, because
* GPatternSpec can only handle UTF-8 and filenames on unix tend to be
* just bytes and are often ISO-8859-X, and we don't feel like
* re-inventing GPatternSpec */
g_object_class_install_property (gobject_class, PROP_LOCATION,
g_param_spec_string ("location", "File Location",
"Regular expression to create file names of the input files. If "
"the location is an absolute path or contains directory components, "
"only the base file name part will be considered a regular "
"expression. The results will be sorted.",
DEFAULT_LOCATION, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
gstbasesrc_class->start = GST_DEBUG_FUNCPTR (gst_split_file_src_start);
gstbasesrc_class->stop = GST_DEBUG_FUNCPTR (gst_split_file_src_stop);
gstbasesrc_class->create = GST_DEBUG_FUNCPTR (gst_split_file_src_create);
gstbasesrc_class->get_size = GST_DEBUG_FUNCPTR (gst_split_file_src_get_size);
gstbasesrc_class->unlock = GST_DEBUG_FUNCPTR (gst_split_file_src_unlock);
gstbasesrc_class->is_seekable =
GST_DEBUG_FUNCPTR (gst_split_file_src_can_seek);
gstbasesrc_class->check_get_range =
GST_DEBUG_FUNCPTR (gst_split_file_src_check_get_range);
}
static void
gst_split_file_src_init (GstSplitFileSrc * splitfilesrc,
GstSplitFileSrcClass * g_class)
{
}
static void
gst_split_file_src_finalize (GObject * obj)
{
GstSplitFileSrc *src = GST_SPLIT_FILE_SRC (obj);
g_free (src->location);
src->location = NULL;
G_OBJECT_CLASS (parent_class)->finalize (obj);
}
static gboolean
gst_split_file_src_can_seek (GstBaseSrc * basesrc)
{
return TRUE;
}
static gboolean
gst_split_file_src_check_get_range (GstBaseSrc * basesrc)
{
return TRUE;
}
static gboolean
gst_split_file_src_unlock (GstBaseSrc * basesrc)
{
/* This is not actually that useful, since all normal file
* operations are fully blocking anyway */
#if 0
GstSplitFileSrc *src = GST_SPLIT_FILE_SRC (basesrc);
GST_DEBUG_OBJECT (src, "cancelling pending I/O operation if there is one");
/* g_cancellable_cancel (src->cancellable); */
GST_DEBUG_OBJECT (src, "done");
#endif
return TRUE;
}
static gboolean
gst_split_file_src_get_size (GstBaseSrc * basesrc, guint64 * size)
{
GstSplitFileSrc *src = GST_SPLIT_FILE_SRC (basesrc);
*size = src->parts[src->num_parts - 1].stop + 1;
return TRUE;
}
static void
gst_split_file_src_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec)
{
GstSplitFileSrc *src = GST_SPLIT_FILE_SRC (object);
switch (prop_id) {
case PROP_LOCATION:
GST_OBJECT_LOCK (src);
g_free (src->location);
src->location = g_value_dup_string (value);
GST_OBJECT_UNLOCK (src);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
static void
gst_split_file_src_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec)
{
GstSplitFileSrc *src = GST_SPLIT_FILE_SRC (object);
switch (prop_id) {
case PROP_LOCATION:
GST_OBJECT_LOCK (src);
g_value_set_string (value, src->location);
GST_OBJECT_UNLOCK (src);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
static int
gst_split_file_src_array_sortfunc (gchar ** a, gchar ** b)
{
return strcmp (*a, *b);
}
static gchar **
gst_split_file_src_find_files (GstSplitFileSrc * src, const gchar * dirname,
const gchar * basename, GError ** err)
{
GPtrArray *files;
GRegex *regex;
const gchar *name;
gchar *regex_string;
GDir *dir;
if (dirname == NULL || basename == NULL)
goto invalid_location;
GST_INFO_OBJECT (src, "checking in directory '%s' for pattern '%s'",
dirname, basename);
dir = g_dir_open (dirname, 0, err);
if (dir == NULL)
return NULL;
/* we want the filename to be the whole filename, not just some match
* in the middle of the filename */
if (g_str_has_suffix (basename, "$"))
regex_string = g_strdup (basename);
else
regex_string = g_strconcat (basename, "$", NULL);
regex = g_regex_new (regex_string, G_REGEX_RAW, (GRegexMatchFlags) 0, err);
g_free (regex_string);
if (regex == NULL)
goto regex_fail;
files = g_ptr_array_new ();
while ((name = g_dir_read_name (dir))) {
GST_TRACE_OBJECT (src, "check: %s", name);
if (g_regex_match (regex, name, (GRegexMatchFlags) 0, NULL)) {
GST_LOG_OBJECT (src, "match: %s", name);
g_ptr_array_add (files, g_build_filename (dirname, name, NULL));
}
}
if (files->len == 0)
goto no_matches;
g_ptr_array_sort (files, (GCompareFunc) gst_split_file_src_array_sortfunc);
g_ptr_array_add (files, NULL);
g_regex_unref (regex);
g_dir_close (dir);
return (gchar **) g_ptr_array_free (files, FALSE);
/* ERRORS */
invalid_location:
{
g_set_error_literal (err, G_FILE_ERROR, G_FILE_ERROR_INVAL,
"No filename specified.");
return NULL;
}
regex_fail:
{
GST_WARNING_OBJECT (src, "g_regex_new() failed: %s", (*err)->message);
g_dir_close (dir);
return NULL;
}
no_matches:
{
g_regex_unref (regex);
g_dir_close (dir);
g_set_error_literal (err, G_FILE_ERROR, G_FILE_ERROR_NOENT,
"Found no files matching the pattern.");
return NULL;
}
}
static gboolean
gst_split_file_src_start (GstBaseSrc * basesrc)
{
GstSplitFileSrc *src = GST_SPLIT_FILE_SRC (basesrc);
GCancellable *cancel;
gboolean ret = FALSE;
guint64 offset;
GError *err = NULL;
gchar *basename = NULL;
gchar *dirname = NULL;
gchar **files;
guint i;
GST_OBJECT_LOCK (src);
if (src->location != NULL && src->location[0] != '\0') {
basename = g_path_get_basename (src->location);
dirname = g_path_get_dirname (src->location);
}
GST_OBJECT_UNLOCK (src);
files = gst_split_file_src_find_files (src, dirname, basename, &err);
if (files == NULL || *files == NULL)
goto no_files;
src->num_parts = g_strv_length (files);
src->parts = g_new0 (GstFilePart, src->num_parts);
cancel = src->cancellable;
offset = 0;
for (i = 0; i < src->num_parts; ++i) {
GFileInputStream *stream;
GFileInfo *info;
goffset size;
GFile *file;
file = g_file_new_for_path (files[i]);
stream = g_file_read (file, cancel, &err);
g_object_unref (file);
if (err != NULL)
goto open_read_error;
info = g_file_input_stream_query_info (stream, "standard::*", NULL, &err);
if (err != NULL) {
g_object_unref (stream);
goto query_info_error;
}
size = g_file_info_get_size (info);
g_object_unref (info);
src->parts[i].stream = stream;
src->parts[i].path = g_strdup (files[i]);
src->parts[i].start = offset;
src->parts[i].stop = offset + size - 1;
GST_DEBUG ("[%010" G_GUINT64_FORMAT "-%010" G_GUINT64_FORMAT "] %s",
src->parts[i].start, src->parts[i].stop, src->parts[i].path);
offset += size;
}
GST_INFO ("Successfully opened %u file parts for reading", src->num_parts);
src->cur_part = 0;
src->cancellable = g_cancellable_new ();
ret = TRUE;
done:
if (err != NULL)
g_error_free (err);
g_strfreev (files);
g_free (basename);
g_free (dirname);
return ret;
/* ERRORS */
no_files:
{
if (err->code == G_IO_ERROR_CANCELLED)
goto cancelled;
GST_ELEMENT_ERROR (src, RESOURCE, OPEN_READ, ("%s", err->message),
("Failed to find files in '%s' for pattern '%s'",
GST_STR_NULL (dirname), GST_STR_NULL (basename)));
goto done;
}
open_read_error:
{
if (err->code == G_IO_ERROR_CANCELLED)
goto cancelled;
GST_ELEMENT_ERROR (src, RESOURCE, OPEN_READ, ("%s", err->message),
("Failed to open file '%s' for reading", files[i]));
goto done;
}
query_info_error:
{
if (err->code == G_IO_ERROR_CANCELLED)
goto cancelled;
GST_ELEMENT_ERROR (src, RESOURCE, OPEN_READ, ("%s", err->message),
("Failed to query info for file '%s'", files[i]));
goto done;
}
cancelled:
{
GST_DEBUG_OBJECT (src, "I/O operation cancelled from another thread");
goto done;
}
}
static gboolean
gst_split_file_src_stop (GstBaseSrc * basesrc)
{
GstSplitFileSrc *src = GST_SPLIT_FILE_SRC (basesrc);
guint i;
for (i = 0; i < src->num_parts; ++i) {
if (src->parts[i].stream != NULL)
g_object_unref (src->parts[i].stream);
g_free (src->parts[i].path);
}
g_free (src->parts);
src->parts = NULL;
src->num_parts = 0;
g_object_unref (src->cancellable);
src->cancellable = NULL;
return TRUE;
}
static gboolean
gst_split_file_src_find_part_for_offset (GstSplitFileSrc * src, guint64 offset,
guint * part_number)
{
GstFilePart *part;
guint i;
/* TODO: could use gst_util_array_binary_search() here */
part = src->parts;
for (i = 0; i < src->num_parts; ++i) {
if (offset >= part->start && offset <= part->stop) {
*part_number = i;
return TRUE;
}
++part;
}
return FALSE;
}
static GstFlowReturn
gst_split_file_src_create (GstBaseSrc * basesrc, guint64 offset, guint size,
GstBuffer ** buffer)
{
GstSplitFileSrc *src = GST_SPLIT_FILE_SRC (basesrc);
GstFilePart cur_part;
GInputStream *stream;
GCancellable *cancel;
GSeekable *seekable;
GstBuffer *buf;
GError *err = NULL;
guint64 read_offset;
guint8 *data;
guint to_read;
cur_part = src->parts[src->cur_part];
if (offset < cur_part.start || offset > cur_part.stop) {
if (!gst_split_file_src_find_part_for_offset (src, offset, &src->cur_part))
return GST_FLOW_UNEXPECTED;
cur_part = src->parts[src->cur_part];
}
GST_LOG_OBJECT (src, "current part: %u (%" G_GUINT64_FORMAT " - "
"%" G_GUINT64_FORMAT ", %s)", src->cur_part, cur_part.start,
cur_part.stop, cur_part.path);
buf = gst_buffer_new_and_alloc (size);
data = GST_BUFFER_DATA (buf);
cancel = src->cancellable;
while (size > 0) {
guint64 bytes_to_end_of_part;
/* we want the offset into the file part */
read_offset = offset - cur_part.start;
GST_LOG ("Reading part %03u from offset %" G_GUINT64_FORMAT " (%s)",
src->cur_part, read_offset, cur_part.path);
/* FIXME: only seek when needed (hopefully gio is smart) */
seekable = G_SEEKABLE (cur_part.stream);
if (!g_seekable_seek (seekable, read_offset, G_SEEK_SET, cancel, &err))
goto seek_failed;
GST_LOG_OBJECT (src, "now: %" G_GUINT64_FORMAT, g_seekable_tell (seekable));
bytes_to_end_of_part = (cur_part.stop - cur_part.start) + 1 - read_offset;
to_read = MIN (size, bytes_to_end_of_part);
GST_LOG_OBJECT (src, "reading %u bytes from part %u (bytes to end of "
"part: %u)", to_read, src->cur_part, (guint) bytes_to_end_of_part);
stream = G_INPUT_STREAM (cur_part.stream);
/* NB: we won't try to read beyond EOF */
if (!g_input_stream_read_all (stream, data, to_read, NULL, cancel, &err))
goto read_failed;
GST_LOG_OBJECT (src, "read %u bytes", to_read);
data += to_read;
size -= to_read;
offset += to_read;
/* are we done? */
if (size == 0)
break;
GST_LOG_OBJECT (src, "%u bytes left to read for this chunk", size);
if (src->cur_part == src->num_parts - 1) {
/* FIXME: at end, need to truncate buffer */
break;
}
++src->cur_part;
cur_part = src->parts[src->cur_part];
}
*buffer = buf;
GST_LOG_OBJECT (src, "read %u bytes into buf %p", GST_BUFFER_SIZE (buf), buf);
return GST_FLOW_OK;
/* ERRORS */
seek_failed:
{
if (err->code == G_IO_ERROR_CANCELLED)
goto cancelled;
GST_ELEMENT_ERROR (src, RESOURCE, SEEK, (NULL),
("Seek to %" G_GUINT64_FORMAT " in %s failed", read_offset,
cur_part.path));
g_error_free (err);
gst_buffer_unref (buf);
return GST_FLOW_ERROR;
}
read_failed:
{
if (err->code == G_IO_ERROR_CANCELLED)
goto cancelled;
GST_ELEMENT_ERROR (src, RESOURCE, READ, ("%s", err->message),
("Read from %" G_GUINT64_FORMAT " in %s failed", read_offset,
cur_part.path));
g_error_free (err);
gst_buffer_unref (buf);
return GST_FLOW_ERROR;
}
cancelled:
{
GST_DEBUG_OBJECT (src, "I/O operation cancelled from another thread");
g_error_free (err);
gst_buffer_unref (buf);
return GST_FLOW_WRONG_STATE;
}
}

View file

@ -0,0 +1,74 @@
/* GStreamer Split File Source
* Copyright (C) 2011 Collabora Ltd. <tim.muller@collabora.co.uk>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 59 Temple Place - Suite 330,
* Boston, MA 02111-1307, USA.
*/
#ifndef __GST_SPLIT_FILE_SRC_H__
#define __GST_SPLIT_FILE_SRC_H__
#include <gst/gst.h>
#include <gst/base/gstbasesrc.h>
#include <gio/gio.h>
G_BEGIN_DECLS
#define GST_TYPE_SPLIT_FILE_SRC \
(gst_split_file_src_get_type())
#define GST_SPLIT_FILE_SRC(obj) \
(G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_SPLIT_FILE_SRC,GstSplitFileSrc))
#define GST_SPLIT_FILE_SRC_CLASS(klass) \
(G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_SPLIT_FILE_SRC,GstSplitFileSrcClass))
#define GST_IS_SPLIT_FILE_SRC(obj) \
(G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_SPLIT_FILE_SRC))
#define GST_IS_SPLIT_FILE_SRC_CLASS(klass) \
(G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_SPLIT_FILE_SRC))
typedef struct _GstFilePart GstFilePart;
typedef struct _GstSplitFileSrc GstSplitFileSrc;
typedef struct _GstSplitFileSrcClass GstSplitFileSrcClass;
struct _GstFilePart
{
GFileInputStream *stream;
gchar *path;
guint64 start; /* inclusive */
guint64 stop; /* inclusive */
};
struct _GstSplitFileSrc
{
GstBaseSrc parent;
gchar *location; /* OBJECT_LOCK */
GstFilePart *parts;
guint num_parts;
guint cur_part; /* part used last (likely also to be used next) */
GCancellable *cancellable; /* so we can interrupt blocking operations */
};
struct _GstSplitFileSrcClass
{
GstBaseSrcClass parent_class;
};
GType gst_split_file_src_get_type (void);
G_END_DECLS
#endif /* __GST_SPLIT_FILE_SRC_H__ */