gstreamer/gst/indexers/gstfileindex.c

/* GStreamer
 * Copyright (C) 2003 Erik Walthinsen <omega@cse.ogi.edu>
 *               2003 Joshua N Pritikin <jpritikin@pobox.com>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with this library; if not, write to the
 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 * Boston, MA 02111-1307, USA.
 */

#include <gst/gst.h>

#include <unistd.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <errno.h>
#include <fcntl.h>
#include <string.h>

#define GST_TYPE_FILE_INDEX		\
  (gst_file_index_get_type ())
#define GST_FILE_INDEX(obj)		\
  (G_TYPE_CHECK_INSTANCE_CAST ((obj), GST_TYPE_FILE_INDEX, GstFileIndex))
#define GST_FILE_INDEX_CLASS(klass)	\
  (G_TYPE_CHECK_CLASS_CAST ((klass), GST_TYPE_FILE_INDEX, GstFileIndexClass))
#define GST_IS_FILE_INDEX(obj)		\
  (G_TYPE_CHECK_INSTANCE_TYPE ((obj), GST_TYPE_FILE_INDEX))
#define GST_IS_FILE_INDEX_CLASS(obj)	\
  (GST_TYPE_CHECK_CLASS_TYPE ((klass), GST_TYPE_FILE_INDEX))

#ifndef __FUNCTION__
#define __FUNCTION__ "Unavailable"
#endif

/*
 * Object model:
 *
 * We build an index to each entry for each id.
 * 
 *
 *  fileindex
 *    -----------------------------...
 *    !                  !         
 *   id1                 id2        
 *    !
 *   GArray
 *
 * The fileindex creates a FileIndexId object for each writer id, a
 * Hashtable is kept to map the id to the FileIndexId
 *
 * The FileIndexId also keeps all the values in a sorted GArray.
 *
 * Finding a value for an id/format requires locating the correct GArray,
 * then do a binary search to get the required value.
 *
 * Unlike gstmemindex:  All formats are assumed to sort to the
 * same order.  All formats are assumed to be available from
 * any entry.
 */

/*
 * Each array element is (32bits flags, nformats * 64bits)
 */
typedef struct
{
  gint id;
  gchar *id_desc;
  gint nformats;
  GstFormat *format;
  GArray *array;
} GstFileIndexId;

typedef struct _GstFileIndex GstFileIndex;
typedef struct _GstFileIndexClass GstFileIndexClass;

#define ARRAY_ROW_SIZE(_ii) \
  (sizeof (gint32) + (_ii)->nformats * sizeof (gint64))
#define ARRAY_TOTAL_SIZE(_ii) \
  (_ii->array->len * ARRAY_ROW_SIZE(_ii))

/* don't forget to convert to/from BE byte-order */
#define ARRAY_ROW_FLAGS(_row) \
  (*((gint32*) (_row)))
#define ARRAY_ROW_VALUE(_row,_vx) \
  (*(gint64*) (((gchar*)(_row)) + sizeof (gint32) + (_vx) * sizeof (gint64)))

GST_DEBUG_CATEGORY_STATIC (DC);

struct _GstFileIndex
{
  GstIndex parent;

  gchar *location;
  gboolean is_loaded;
  GSList *unresolved;
  gint next_id;
  GHashTable *id_index;

  GstIndexEntry *ret_entry;	/* hack to avoid leaking memory */
};

struct _GstFileIndexClass
{
  GstIndexClass parent_class;
};

enum
{
  ARG_0,
  ARG_LOCATION,
};

static void gst_file_index_class_init (GstFileIndexClass * klass);
static void gst_file_index_init (GstFileIndex * index);
static void gst_file_index_dispose (GObject * object);

static void
gst_file_index_set_property (GObject * object,
    guint prop_id, const GValue * value, GParamSpec * pspec);
static void
gst_file_index_get_property (GObject * object,
    guint prop_id, GValue * value, GParamSpec * pspec);

static gboolean
gst_file_index_get_writer_id (GstIndex * _index, gint * id,
    gchar * writer_string);

static void gst_file_index_commit (GstIndex * index, gint writer_id);
static void gst_file_index_add_entry (GstIndex * index, GstIndexEntry * entry);
static GstIndexEntry *gst_file_index_get_assoc_entry (GstIndex * index, gint id,
    GstIndexLookupMethod method,
    GstAssocFlags flags,
    GstFormat format, gint64 value, GCompareDataFunc func, gpointer user_data);

#define CLASS(file_index)  GST_FILE_INDEX_CLASS (G_OBJECT_GET_CLASS (file_index))

static GstIndex *parent_class = NULL;

GType
gst_file_index_get_type (void)
{
  static GType file_index_type = 0;

  if (!file_index_type) {
    static const GTypeInfo file_index_info = {
      sizeof (GstFileIndexClass),
      NULL,
      NULL,
      (GClassInitFunc) gst_file_index_class_init,
      NULL,
      NULL,
      sizeof (GstFileIndex),
      1,
      (GInstanceInitFunc) gst_file_index_init,
      NULL
    };
    file_index_type =
	g_type_register_static (GST_TYPE_INDEX, "GstFileIndex",
	&file_index_info, 0);
  }
  return file_index_type;
}

static void
gst_file_index_class_init (GstFileIndexClass * klass)
{
  GObjectClass *gobject_class;
  GstIndexClass *gstindex_class;

  gobject_class = (GObjectClass *) klass;
  gstindex_class = (GstIndexClass *) klass;

  parent_class = g_type_class_ref (GST_TYPE_INDEX);

  gobject_class->dispose = gst_file_index_dispose;
  gobject_class->set_property = gst_file_index_set_property;
  gobject_class->get_property = gst_file_index_get_property;

  gstindex_class->add_entry = gst_file_index_add_entry;
  gstindex_class->get_assoc_entry = gst_file_index_get_assoc_entry;
  gstindex_class->commit = gst_file_index_commit;
  gstindex_class->get_writer_id = gst_file_index_get_writer_id;

  g_object_class_install_property (gobject_class, ARG_LOCATION,
      g_param_spec_string ("location", "File Location",
	  "Location of the index file", NULL, G_PARAM_READWRITE));
}

static void
gst_file_index_init (GstFileIndex * index)
{
  GST_DEBUG ("created new file index");

  index->id_index = g_hash_table_new (g_int_hash, g_int_equal);
}

static void
_file_index_id_free (GstFileIndexId * index_id, gboolean is_mmapped)
{
  if (index_id->id_desc)
    g_free (index_id->id_desc);
  if (index_id->format)
    g_free (index_id->format);
  if (index_id->array) {
    if (is_mmapped)
      munmap (index_id->array->data, ARRAY_TOTAL_SIZE (index_id));
    g_array_free (index_id->array, !is_mmapped);
  }
  g_free (index_id);
}

static gboolean
_id_index_free_helper (gpointer _key, GstFileIndexId * index_id,
    GstFileIndex * index)
{
  _file_index_id_free (index_id, index->is_loaded);
  return TRUE;
}

static void
gst_file_index_dispose (GObject * object)
{
  GstFileIndex *index = GST_FILE_INDEX (object);

  if (index->location) {
    g_free (index->location);
    index->location = NULL;
  }

  {
    GSList *elem;

    for (elem = index->unresolved; elem; elem = g_slist_next (elem))
      _file_index_id_free (elem->data, index->is_loaded);
    g_slist_free (index->unresolved);
    index->unresolved = NULL;
  }

  g_hash_table_foreach_steal (index->id_index,
      (GHRFunc) _id_index_free_helper, index);
  g_hash_table_destroy (index->id_index);
  index->id_index = NULL;

  gst_index_entry_free (index->ret_entry);	/* hack */

  G_OBJECT_CLASS (parent_class)->dispose (object);
}

struct fi_find_writer_context
{
  const gchar *writer_string;
  GstFileIndexId *ii;
};

void
_fi_find_writer (gpointer key, gpointer val, gpointer data)
{
  struct fi_find_writer_context *cx = data;
  GstFileIndexId *ii = val;

  if (strcmp (ii->id_desc, cx->writer_string) == 0)
    cx->ii = ii;
}

static gboolean
gst_file_index_get_writer_id (GstIndex * _index,
    gint * id, gchar * writer_string)
{
  GstFileIndex *index = GST_FILE_INDEX (_index);
  GSList *pending = index->unresolved;
  gboolean match = FALSE;
  GSList *elem;

  if (!index->is_loaded)
    return FALSE;

  g_return_val_if_fail (id, FALSE);
  g_return_val_if_fail (writer_string, FALSE);

  index->unresolved = NULL;

  for (elem = pending; elem; elem = g_slist_next (elem)) {
    GstFileIndexId *ii = elem->data;

    if (strcmp (ii->id_desc, writer_string) != 0) {
      index->unresolved = g_slist_prepend (index->unresolved, ii);
      continue;
    }

    if (match) {
      GST_CAT_WARNING_OBJECT (DC, index, "Duplicate matches for writer '%s'",
	  writer_string);
      continue;
    }

    ii->id = *id = ++index->next_id;
    g_hash_table_insert (index->id_index, &ii->id, ii);
    match = TRUE;
  }

  g_slist_free (pending);

  if (!match) {
    struct fi_find_writer_context cx;

    cx.writer_string = writer_string;
    cx.ii = NULL;
    g_hash_table_foreach (index->id_index, _fi_find_writer, &cx);

    if (cx.ii) {
      match = TRUE;
      GST_CAT_DEBUG_OBJECT (DC, index, "Resolved writer '%s' again",
	  writer_string);
    } else
      GST_CAT_WARNING_OBJECT (DC, index, "Can't resolve writer '%s'",
	  writer_string);
  }

  return match;
}

static void
_fc_alloc_array (GstFileIndexId * id_index)
{
  g_assert (!id_index->array);
  id_index->array =
      g_array_sized_new (FALSE, FALSE, ARRAY_ROW_SIZE (id_index), 0);
}

static void
gst_file_index_load (GstFileIndex * index)
{
  xmlDocPtr doc;
  xmlNodePtr root, part;
  xmlChar *val;

  g_assert (index->location);
  g_return_if_fail (!index->is_loaded);

  {
    gchar *path = g_strdup_printf ("%s/gstindex.xml", index->location);
    GError *err = NULL;
    gchar *buf;
    gsize len;

    g_file_get_contents (path, &buf, &len, &err);
    g_free (path);
    if (err) {
      GST_CAT_ERROR_OBJECT (DC, index, "%s", err->message);
      return;
    }

    doc = xmlParseMemory (buf, len);
    g_free (buf);
  }

  //xmlDocFormatDump (stderr, doc, TRUE);

  root = doc->xmlRootNode;
  if (strcmp (root->name, "gstfileindex") != 0) {
    GST_CAT_ERROR_OBJECT (DC, index, "root node isn't a gstfileindex");
    return;
  }

  val = xmlGetProp (root, "version");
  if (!val || atoi (val) != 1) {
    GST_CAT_ERROR_OBJECT (DC, index, "version != 1");
    return;
  }
  free (val);

  for (part = root->children; part; part = part->next) {
    if (strcmp (part->name, "writers") == 0) {
      xmlNodePtr writer;

      for (writer = part->children; writer; writer = writer->next) {
	xmlChar *datafile = xmlGetProp (writer, "datafile");
	gchar *path = g_strdup_printf ("%s/%s", index->location, datafile);
	int fd;
	GstFileIndexId *id_index;
	xmlNodePtr wpart;
	xmlChar *entries_str;
	gpointer array_data;

	free (datafile);

	fd = open (path, O_RDONLY);
	g_free (path);
	if (fd < 0) {
	  GST_CAT_ERROR_OBJECT (DC, index,
	      "Can't open '%s': %s", path, strerror (errno));
	  continue;
	}

	id_index = g_new0 (GstFileIndexId, 1);
	id_index->id_desc = xmlGetProp (writer, "id");

	for (wpart = writer->children; wpart; wpart = wpart->next) {
	  if (strcmp (wpart->name, "formats") == 0) {
	    xmlChar *count_str = xmlGetProp (wpart, "count");
	    gint fx = 0;
	    xmlNodePtr format;

	    id_index->nformats = atoi (count_str);
	    free (count_str);

	    id_index->format = g_new (GstFormat, id_index->nformats);

	    for (format = wpart->children; format; format = format->next) {
	      xmlChar *nick = xmlGetProp (format, "nick");
	      GstFormat fmt = gst_format_get_by_nick (nick);

	      if (fmt == GST_FORMAT_UNDEFINED)
		GST_CAT_ERROR_OBJECT (DC, index, "format '%s' undefined", nick);
	      g_assert (fx < id_index->nformats);
	      id_index->format[fx++] = fmt;
	      free (nick);
	    }
	  } else
	    GST_CAT_INFO_OBJECT (DC, index, "unknown wpart '%s'", wpart->name);
	}

	g_assert (id_index->nformats > 0);
	_fc_alloc_array (id_index);
	g_assert (id_index->array->data == NULL);	/* little bit risky */

	entries_str = xmlGetProp (writer, "entries");
	id_index->array->len = atoi (entries_str);
	free (entries_str);

	array_data =
	    mmap (NULL, ARRAY_TOTAL_SIZE (id_index), PROT_READ, MAP_SHARED, fd,
	    0);
	close (fd);
	if (array_data == MAP_FAILED) {
	  GST_CAT_ERROR_OBJECT (DC, index,
	      "mmap %s failed: %s", path, strerror (errno));
	  continue;
	}

	id_index->array->data = array_data;

	index->unresolved = g_slist_prepend (index->unresolved, id_index);
      }
    } else
      GST_CAT_INFO_OBJECT (DC, index, "unknown part '%s'", part->name);
  }

  xmlFreeDoc (doc);

  GST_FLAG_UNSET (index, GST_INDEX_WRITABLE);
  index->is_loaded = TRUE;
  GST_CAT_LOG_OBJECT (DC, index, "index %s loaded OK", index->location);
}

static void
gst_file_index_set_property (GObject * object,
    guint prop_id, const GValue * value, GParamSpec * pspec)
{
  GstFileIndex *index = GST_FILE_INDEX (object);

  switch (prop_id) {
    case ARG_LOCATION:
      if (index->location)
	g_free (index->location);
      index->location = g_value_dup_string (value);

      if (index->location && !g_hash_table_size (index->id_index))
	gst_file_index_load (index);
      break;
  }
}

static void
gst_file_index_get_property (GObject * object,
    guint prop_id, GValue * value, GParamSpec * pspec)
{
  GstFileIndex *index = GST_FILE_INDEX (object);

  switch (prop_id) {
    case ARG_LOCATION:
      g_value_set_string (value, index->location);
      break;
  }
}

static void
_file_index_id_save_xml (gpointer _key, GstFileIndexId * ii, xmlNodePtr writers)
{
  const gint bufsize = 16;
  gchar buf[bufsize];
  xmlNodePtr writer;
  xmlNodePtr formats;
  gint xx;

  if (!ii->array) {
    GST_INFO ("Index for %s is empty", ii->id_desc);
    return;
  }

  writer = xmlNewChild (writers, NULL, "writer", NULL);
  xmlSetProp (writer, "id", ii->id_desc);
  g_snprintf (buf, bufsize, "%d", ii->array->len);
  xmlSetProp (writer, "entries", buf);
  g_snprintf (buf, bufsize, "%d", ii->id);	/* any unique number is OK */
  xmlSetProp (writer, "datafile", buf);

  formats = xmlNewChild (writer, NULL, "formats", NULL);
  g_snprintf (buf, bufsize, "%d", ii->nformats);
  xmlSetProp (formats, "count", buf);

  for (xx = 0; xx < ii->nformats; xx++) {
    xmlNodePtr format = xmlNewChild (formats, NULL, "format", NULL);
    const GstFormatDefinition *def = gst_format_get_details (ii->format[xx]);

    xmlSetProp (format, "nick", def->nick);
  }
}

/*
  We must save the binary data in separate files because
  mmap wants getpagesize() alignment.  If we append all
  the data to one file then we don't know the appropriate
  padding since the page size isn't fixed.
*/
static void
_file_index_id_save_entries (gpointer * _key,
    GstFileIndexId * ii, gchar * prefix)
{
  GError *err;
  gchar *path;
  GIOChannel *chan;

  if (!ii->array)
    return;

  err = NULL;
  path = g_strdup_printf ("%s/%d", prefix, ii->id);
  chan = g_io_channel_new_file (path, "w", &err);
  g_free (path);
  if (err)
    goto fail;

  g_io_channel_set_encoding (chan, NULL, &err);
  if (err)
    goto fail;

  g_io_channel_write_chars (chan,
      ii->array->data, ARRAY_TOTAL_SIZE (ii), NULL, &err);
  if (err)
    goto fail;

  g_io_channel_shutdown (chan, TRUE, &err);
  if (err)
    goto fail;

  g_io_channel_unref (chan);
  return;

fail:
  GST_CAT_ERROR (DC, "%s", err->message);
}

/*
  We have to save the whole set of indexes into a single file
  so it doesn't make sense to commit only a single writer.

  i suggest:

  gst_index_commit (index, -1);
*/
static void
gst_file_index_commit (GstIndex * _index, gint _writer_id)
{
  GstFileIndex *index = GST_FILE_INDEX (_index);
  xmlDocPtr doc;
  xmlNodePtr writers;
  GError *err = NULL;
  gchar *path;
  GIOChannel *tocfile;

  g_return_if_fail (index->location);
  g_return_if_fail (!index->is_loaded);

  GST_FLAG_UNSET (index, GST_INDEX_WRITABLE);

  doc = xmlNewDoc ("1.0");
  doc->xmlRootNode = xmlNewDocNode (doc, NULL, "gstfileindex", NULL);
  xmlSetProp (doc->xmlRootNode, "version", "1");

  writers = xmlNewChild (doc->xmlRootNode, NULL, "writers", NULL);
  g_hash_table_foreach (index->id_index,
      (GHFunc) _file_index_id_save_xml, writers);

  if (mkdir (index->location, 0777) && errno != EEXIST) {
    GST_CAT_ERROR_OBJECT (DC, index,
	"mkdir %s: %s", index->location, strerror (errno));
    return;
  }

  path = g_strdup_printf ("%s/gstindex.xml", index->location);
  tocfile = g_io_channel_new_file (path, "w", &err);
  g_free (path);
  if (err) {
    GST_CAT_ERROR_OBJECT (DC, index, "%s", err->message);
    return;
  }

  g_io_channel_set_encoding (tocfile, NULL, &err);
  if (err) {
    GST_CAT_ERROR_OBJECT (DC, index, "%s", err->message);
    return;
  }

  {
    xmlChar *xmlmem;
    int xmlsize;

    xmlDocDumpMemory (doc, &xmlmem, &xmlsize);
    g_io_channel_write_chars (tocfile, xmlmem, xmlsize, NULL, &err);
    if (err) {
      GST_CAT_ERROR_OBJECT (DC, index, "%s", err->message);
      return;
    }
    xmlFreeDoc (doc);
    free (xmlmem);
  }

  g_io_channel_shutdown (tocfile, TRUE, &err);
  if (err) {
    GST_CAT_ERROR_OBJECT (DC, index, "%s", err->message);
    return;
  }

  g_io_channel_unref (tocfile);

  g_hash_table_foreach (index->id_index,
      (GHFunc) _file_index_id_save_entries, index->location);
}

static void
gst_file_index_add_id (GstIndex * index, GstIndexEntry * entry)
{
  GstFileIndex *fileindex = GST_FILE_INDEX (index);
  GstFileIndexId *id_index;

  id_index = g_hash_table_lookup (fileindex->id_index, &entry->id);

  if (!id_index) {
    id_index = g_new0 (GstFileIndexId, 1);

    id_index->id = entry->id;
    id_index->id_desc = g_strdup (entry->data.id.description);

    /* It would be useful to know the GType of the writer so
       we can try to cope with changes in the id_desc path. */

    g_hash_table_insert (fileindex->id_index, &id_index->id, id_index);
  }
}

/* This algorithm differs from libc bsearch in the handling
   of non-exact matches. */

static gboolean
_fc_bsearch (GArray * ary,
    gint stride,
    gint * ret,
    GCompareDataFunc compare, gconstpointer sample, gpointer user_data)
{
  gint first, last;
  gint mid;
  gint midsize;
  gint cmp;
  gint tx;

  g_return_val_if_fail (compare, FALSE);

  if (!ary->len) {
    if (ret)
      *ret = 0;
    return FALSE;
  }

  first = 0;
  last = ary->len - 1;

  midsize = last - first;

  while (midsize > 1) {
    mid = first + midsize / 2;

    cmp = (*compare) (sample, ary->data + mid * stride, user_data);

    if (cmp == 0) {
      /* if there are multiple matches then scan for the first match */
      while (mid > 0 &&
	  (*compare) (sample, ary->data + (mid - 1) * stride, user_data) == 0)
	--mid;

      if (ret)
	*ret = mid;
      return TRUE;
    }

    if (cmp < 0)
      last = mid - 1;
    else
      first = mid + 1;

    midsize = last - first;
  }

  for (tx = first; tx <= last; tx++) {
    cmp = (*compare) (sample, ary->data + tx * stride, user_data);

    if (cmp < 0) {
      if (ret)
	*ret = tx;
      return FALSE;
    }
    if (cmp == 0) {
      if (ret)
	*ret = tx;
      return TRUE;
    }
  }

  if (ret)
    *ret = last + 1;
  return FALSE;
}

static gint
file_index_compare (gconstpointer sample, gconstpointer row, gpointer user_data)
{
  //GstFileIndexId *id_index = user_data;
  const GstIndexAssociation *ca = sample;
  gint64 val1 = ca->value;
  gint64 val2_be = ARRAY_ROW_VALUE (row, ca->format);
  gint64 val2 = GINT64_FROM_BE (val2_be);
  gint64 diff = val2 - val1;

  return (diff == 0 ? 0 : (diff < 0 ? 1 : -1));
}

static void
gst_file_index_add_association (GstIndex * index, GstIndexEntry * entry)
{
  GstFileIndex *fileindex = GST_FILE_INDEX (index);
  GstFileIndexId *id_index;
  gint mx;
  GstIndexAssociation sample;
  gboolean exact;

  id_index = g_hash_table_lookup (fileindex->id_index, &entry->id);
  if (!id_index)
    return;

  if (!id_index->nformats) {
    gint fx;

    id_index->nformats = GST_INDEX_NASSOCS (entry);
    GST_CAT_LOG_OBJECT (DC, fileindex, "creating %d formats for %d",
	id_index->nformats, entry->id);
    id_index->format = g_new (GstFormat, id_index->nformats);
    for (fx = 0; fx < id_index->nformats; fx++)
      id_index->format[fx] = GST_INDEX_ASSOC_FORMAT (entry, fx);
    _fc_alloc_array (id_index);
  } else {
    /* only sanity checking */
    if (id_index->nformats != GST_INDEX_NASSOCS (entry))
      GST_CAT_WARNING_OBJECT (DC, fileindex, "arity change %d -> %d",
	  id_index->nformats, GST_INDEX_NASSOCS (entry));
    else {
      gint fx;

      for (fx = 0; fx < id_index->nformats; fx++)
	if (id_index->format[fx] != GST_INDEX_ASSOC_FORMAT (entry, fx))
	  GST_CAT_WARNING_OBJECT (DC, fileindex, "format[%d] changed %d -> %d",
	      fx, id_index->format[fx], GST_INDEX_ASSOC_FORMAT (entry, fx));
    }
  }

  /* this is a hack, we should use a private structure instead */
  sample.format = 0;
  sample.value = GST_INDEX_ASSOC_VALUE (entry, 0);

  exact =
      _fc_bsearch (id_index->array, ARRAY_ROW_SIZE (id_index),
      &mx, file_index_compare, &sample, id_index);

  if (exact) {
    /* maybe overwrite instead? */
    GST_CAT_DEBUG_OBJECT (DC, index,
	"Ignoring duplicate index association at %lld",
	GST_INDEX_ASSOC_VALUE (entry, 0));
    return;
  }

  {
    gchar row_data[ARRAY_ROW_SIZE (id_index)];
    gint fx;

    gint32 flags_host = GST_INDEX_ASSOC_FLAGS (entry);

    ARRAY_ROW_FLAGS (row_data) = GINT32_TO_BE (flags_host);

    for (fx = 0; fx < id_index->nformats; fx++) {
      gint64 val_host = GST_INDEX_ASSOC_VALUE (entry, fx);

      ARRAY_ROW_VALUE (row_data, fx) = GINT64_TO_BE (val_host);
    }

    g_array_insert_vals (id_index->array, mx, row_data, 1);
  }
}

/*
static void
show_entry (GstIndexEntry *entry)
{
  switch (entry->type) {
    case GST_INDEX_ENTRY_ID:
      g_print ("id %d describes writer %s\n", entry->id, 
		      GST_INDEX_ID_DESCRIPTION (entry));
      break;
    case GST_INDEX_ENTRY_FORMAT:
      g_print ("%d: registered format %d for %s\n", entry->id, 
		      GST_INDEX_FORMAT_FORMAT (entry),
		      GST_INDEX_FORMAT_KEY (entry));
      break;
    case GST_INDEX_ENTRY_ASSOCIATION:
    {
      gint i;

      g_print ("%d: %08x ", entry->id, GST_INDEX_ASSOC_FLAGS (entry));
      for (i = 0; i < GST_INDEX_NASSOCS (entry); i++) {
	g_print ("%d %lld ", GST_INDEX_ASSOC_FORMAT (entry, i), 
			     GST_INDEX_ASSOC_VALUE (entry, i));
      }
      g_print ("\n");
      break;
    }
    default:
      break;
  }
}
*/

static void
gst_file_index_add_entry (GstIndex * index, GstIndexEntry * entry)
{
  GST_LOG_OBJECT (index, "adding this entry");

  switch (entry->type) {
    case GST_INDEX_ENTRY_ID:
      gst_file_index_add_id (index, entry);
      break;
    case GST_INDEX_ENTRY_ASSOCIATION:
      gst_file_index_add_association (index, entry);
      break;
    case GST_INDEX_ENTRY_OBJECT:
      GST_CAT_ERROR_OBJECT (DC, index,
	  "gst_file_index_add_object not implemented");
      break;
    case GST_INDEX_ENTRY_FORMAT:
      /*
         We infer the formats from the entry itself so this type of
         GST_INDEX_ENTRY_* can probably go away.
       */
      GST_CAT_DEBUG_OBJECT (DC, index,
	  "gst_file_index_add_format not implemented");
      break;
    default:
      break;
  }
}

static GstIndexEntry *
gst_file_index_get_assoc_entry (GstIndex * index,
    gint id,
    GstIndexLookupMethod method,
    GstAssocFlags flags,
    GstFormat format,
    gint64 value, GCompareDataFunc _ignore_func, gpointer _ignore_user_data)
{
  GstFileIndex *fileindex = GST_FILE_INDEX (index);
  GstFileIndexId *id_index;
  gint formatx = -1;
  gint fx;
  GstIndexAssociation sample;
  gint mx;
  gboolean exact;
  gpointer row_data;
  GstIndexEntry *entry;
  gint xx;

  g_return_val_if_fail (id > 0, NULL);

  id_index = g_hash_table_lookup (fileindex->id_index, &id);
  if (!id_index) {
    GST_CAT_WARNING_OBJECT (DC, fileindex, "writer %d unavailable", id);
    return NULL;
  }

  for (fx = 0; fx < id_index->nformats; fx++)
    if (id_index->format[fx] == format) {
      formatx = fx;
      break;
    }

  if (formatx == -1) {
    GST_CAT_WARNING_OBJECT (DC, fileindex,
	"%s, format %d not available", __FUNCTION__, format);
    return NULL;
  }

  /* this is a hack, we should use a private structure instead */
  sample.format = formatx;
  sample.value = value;

  exact = _fc_bsearch (id_index->array, ARRAY_ROW_SIZE (id_index),
      &mx, file_index_compare, &sample, id_index);

  if (!exact) {
    if (method == GST_INDEX_LOOKUP_EXACT)
      return NULL;
    else if (method == GST_INDEX_LOOKUP_BEFORE) {
      if (mx == 0)
	return NULL;
      mx -= 1;
    } else if (method == GST_INDEX_LOOKUP_AFTER) {
      if (mx == id_index->array->len)
	return NULL;
    }
  }

  row_data = id_index->array->data + mx * ARRAY_ROW_SIZE (id_index);

  /* if exact then ignore flags (?) */
  if (method != GST_INDEX_LOOKUP_EXACT)
    while ((GINT32_FROM_BE (ARRAY_ROW_FLAGS (row_data)) & flags) != flags) {
      if (method == GST_INDEX_LOOKUP_BEFORE)
	mx -= 1;
      else if (method == GST_INDEX_LOOKUP_AFTER)
	mx += 1;
      if (mx < 0 || mx >= id_index->array->len)
	return NULL;
      row_data = id_index->array->data + mx * ARRAY_ROW_SIZE (id_index);
    }

  /* entry memory management needs improvement FIXME */
  if (!fileindex->ret_entry)
    fileindex->ret_entry = g_new0 (GstIndexEntry, 1);
  entry = fileindex->ret_entry;
  if (entry->data.assoc.assocs) {
    g_free (entry->data.assoc.assocs);
    entry->data.assoc.assocs = NULL;
  }

  entry->type = GST_INDEX_ENTRY_ASSOCIATION;

  GST_INDEX_NASSOCS (entry) = id_index->nformats;
  entry->data.assoc.assocs = g_new (GstIndexAssociation, id_index->nformats);

  {
    gint32 flags_be = ARRAY_ROW_FLAGS (row_data);

    GST_INDEX_ASSOC_FLAGS (entry) = GINT32_FROM_BE (flags_be);

    for (xx = 0; xx < id_index->nformats; xx++) {
      gint64 val_be = ARRAY_ROW_VALUE (row_data, xx);

      GST_INDEX_ASSOC_FORMAT (entry, xx) = id_index->format[xx];
      GST_INDEX_ASSOC_VALUE (entry, xx) = GINT64_FROM_BE (val_be);
    }
  }

  return entry;
}

gboolean
gst_file_index_plugin_init (GstPlugin * plugin)
{
  GstIndexFactory *factory;

  GST_DEBUG_CATEGORY_INIT (DC, "GST_FILEINDEX", 0, NULL);

  factory = gst_index_factory_new ("fileindex",
      "A index that stores entries in file", gst_file_index_get_type ());

  if (factory != NULL) {
    gst_plugin_add_feature (plugin, GST_PLUGIN_FEATURE (factory));
  } else {
    GST_CAT_ERROR (DC, "could not register fileindex");
  }
  return TRUE;
}