gst/bayer/gstbayer2rgb.c: Significant improvements. Fixes #521392.

Original commit message from CVS:
Patch by: William M. Brack
* gst/bayer/gstbayer2rgb.c: Significant improvements.  Fixes #521392.
This commit is contained in:
William M. Brack 2008-03-14 17:38:10 +00:00 committed by David Schleef
parent 665be109b1
commit 80b032ef17
2 changed files with 399 additions and 116 deletions

View file

@ -1,3 +1,9 @@
2008-03-14 David Schleef <ds@schleef.org>
Patch by: William M. Brack
* gst/bayer/gstbayer2rgb.c: Significant improvements. Fixes #521392.
2008-03-14 Wim Taymans <wim.taymans@collabora.co.uk>
* gst/selector/gstinputselector.c: (gst_selector_pad_event),

View file

@ -16,12 +16,59 @@
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 59 Temple Place - Suite 330,
* Boston, MA 02111-1307, USA.
*
* March 2008
* Logic enhanced by William Brack <wbrack@mmm.com.hk>
*/
/*
* SECTION:element-bayer2rgb
*
* Decodes raw camera sensor images.
* Decodes raw camera bayer (fourcc BA81) to RGB.
*/
/*
* In order to guard against my advancing maturity, some extra detailed
* information about the logic of the decode is included here. Much of
* this was inspired by a technical paper from siliconimaging.com, which
* in turn was based upon an article from IEEE,
* T. Sakamoto, C. Nakanishi and T. Hase,
* Software pixel interpolation for digital still cameras suitable for
* a 32-bit MCU,
* IEEE Trans. Consumer Electronics, vol. 44, no. 4, November 1998.
*
* The code assumes a Bayer matrix of the type produced by the fourcc
* BA81 (v4l2 format SBGGR8) of width w and height h which looks like:
* 0 1 2 3 w-2 w-1
*
* 0 B G B G ....B G
* 1 G R G R ....G R
* 2 B G B G ....B G
* ...............
* h-2 B G B G ....B G
* h-1 G R G R ....G R
*
* We expand this matrix, producing a separate {r, g, b} triple for each
* of the individual elements. The algorithm for doing this expansion is
* as follows.
*
* We are designing for speed of transformation, at a slight expense of code.
* First, we calculate the appropriate triples for the four corners, the
* remainder of the top and bottom rows, and the left and right columns.
* The reason for this is that those elements are transformed slightly
* differently than all of the remainder of the matrix. Finally, we transform
* all of the remainder.
*
* The transformation into the "appropriate triples" is based upon the
* "nearest neighbor" principal, with some additional complexity for the
* calculation of the "green" element, where an "adaptive" pairing is used.
*
* For purposes of documentation and indentification, each element of the
* original array can be put into one of four classes:
* R A red element
* B A blue element
* GR A green element which is followed by a red one
* GB A green element which is followed by a blue one
*/
#ifdef HAVE_CONFIG_H
@ -56,8 +103,10 @@ struct _GstBayer2RGB
int width;
int height;
int stride;
uint8_t *tmpdata;
int pixsize; /* bytes per pixel */
int r_off; /* offset for red */
int g_off; /* offset for green */
int b_off; /* offset for blue */
};
struct _GstBayer2RGBClass
@ -66,12 +115,24 @@ struct _GstBayer2RGBClass
};
static const GstElementDetails element_details =
GST_ELEMENT_DETAILS ("RAW Camera sensor decoder",
"Filter/Effect",
"FIXME example filter",
"FIXME <fixme@fixme.com>");
GST_ELEMENT_DETAILS ("Bayer to RGB decoder for cameras",
"Filter/Converter/Video",
"Converts video/x-raw-bayer to video/x-raw-rgb",
"William Brack <wbrack@mmm.com.hk>");
//#define SRC_CAPS GST_VIDEO_CAPS_RGBx
#define SRC_CAPS \
GST_VIDEO_CAPS_RGBx ";" \
GST_VIDEO_CAPS_xRGB ";" \
GST_VIDEO_CAPS_BGRx ";" \
GST_VIDEO_CAPS_xBGR ";" \
GST_VIDEO_CAPS_RGBA ";" \
GST_VIDEO_CAPS_ARGB ";" \
GST_VIDEO_CAPS_BGRA ";" \
GST_VIDEO_CAPS_ABGR ";" \
GST_VIDEO_CAPS_RGB ";" \
GST_VIDEO_CAPS_BGR
#define SRC_CAPS GST_VIDEO_CAPS_ARGB
#define SINK_CAPS "video/x-raw-bayer,width=(int)[1,MAX],height=(int)[1,MAX]"
enum
@ -142,11 +203,11 @@ gst_bayer2rgb_init (GstBayer2RGB * filter, GstBayer2RGBClass * klass)
gst_base_transform_set_in_place (GST_BASE_TRANSFORM (filter), TRUE);
}
/* No properties are implemented, so only a warning is produced */
static void
gst_bayer2rgb_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec)
{
//GstBayer2RGB *filter = GST_BAYER2RGB (object);
switch (prop_id) {
default:
@ -159,7 +220,6 @@ static void
gst_bayer2rgb_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec)
{
//GstBayer2RGB *filter = GST_BAYER2RGB (object);
switch (prop_id) {
default:
@ -168,14 +228,34 @@ gst_bayer2rgb_get_property (GObject * object, guint prop_id,
}
}
/* Routine to convert colormask value into relative byte offset */
static int
get_pix_offset (int offset)
{
switch (offset) {
case 255:
return 3;
case 65280:
return 2;
case 16711680:
return 1;
case -16777216:
return 0;
default:
GST_ERROR ("Invalid color mask 0x%08x", offset);
return -1;
}
}
static gboolean
gst_bayer2rgb_set_caps (GstBaseTransform * base, GstCaps * incaps,
GstCaps * outcaps)
{
GstBayer2RGB *filter = GST_BAYER2RGB (base);
GstStructure *structure;
int val;
GST_ERROR ("in caps %" GST_PTR_FORMAT " out caps %" GST_PTR_FORMAT, incaps,
GST_DEBUG ("in caps %" GST_PTR_FORMAT " out caps %" GST_PTR_FORMAT, incaps,
outcaps);
structure = gst_caps_get_structure (incaps, 0);
@ -184,10 +264,16 @@ gst_bayer2rgb_set_caps (GstBaseTransform * base, GstCaps * incaps,
gst_structure_get_int (structure, "height", &filter->height);
filter->stride = GST_ROUND_UP_4 (filter->width);
if (filter->tmpdata) {
g_free (filter->tmpdata);
}
filter->tmpdata = g_malloc (filter->stride * (4 * 3 + 1));
/* To cater for different RGB formats, we need to set params for later */
structure = gst_caps_get_structure (outcaps, 0);
gst_structure_get_int (structure, "bpp", &val);
filter->pixsize = val / 8;
gst_structure_get_int (structure, "red_mask", &val);
filter->r_off = get_pix_offset (val);
gst_structure_get_int (structure, "green_mask", &val);
filter->g_off = get_pix_offset (val);
gst_structure_get_int (structure, "blue_mask", &val);
filter->b_off = get_pix_offset (val);
return TRUE;
}
@ -198,10 +284,10 @@ gst_bayer2rgb_reset (GstBayer2RGB * filter)
filter->width = 0;
filter->height = 0;
filter->stride = 0;
if (filter->tmpdata) {
g_free (filter->tmpdata);
filter->tmpdata = NULL;
}
filter->pixsize = 0;
filter->r_off = 0;
filter->g_off = 0;
filter->b_off = 0;
}
static GstCaps *
@ -212,7 +298,7 @@ gst_bayer2rgb_transform_caps (GstBaseTransform * base,
GstCaps *newcaps;
GstStructure *newstruct;
GST_ERROR ("transforming caps %" GST_PTR_FORMAT, caps);
GST_DEBUG_OBJECT (caps, "transforming caps (from)");
structure = gst_caps_get_structure (caps, 0);
@ -229,10 +315,8 @@ gst_bayer2rgb_transform_caps (GstBaseTransform * base,
gst_structure_get_value (structure, "height"));
gst_structure_set_value (newstruct, "framerate",
gst_structure_get_value (structure, "framerate"));
gst_structure_set_value (newstruct, "pixel-aspect-ratio",
gst_structure_get_value (structure, "pixel-aspect-ratio"));
GST_ERROR ("into %" GST_PTR_FORMAT, newcaps);
GST_DEBUG_OBJECT (newcaps, "transforming caps (into)");
return newcaps;
}
@ -244,72 +328,306 @@ gst_bayer2rgb_get_unit_size (GstBaseTransform * base, GstCaps * caps,
GstStructure *structure;
int width;
int height;
int pixsize;
const char *name;
structure = gst_caps_get_structure (caps, 0);
gst_structure_get_int (structure, "width", &width);
gst_structure_get_int (structure, "height", &height);
name = gst_structure_get_name (structure);
if (strcmp (name, "video/x-raw-rgb")) {
*size = GST_ROUND_UP_4 (width) * height;
} else {
*size = 4 * width * height;
if (gst_structure_get_int (structure, "width", &width) &&
gst_structure_get_int (structure, "height", &height)) {
name = gst_structure_get_name (structure);
/* Our name must be either video/x-raw-bayer video/x-raw-rgb */
if (strcmp (name, "video/x-raw-rgb")) {
/* For bayer, we handle only BA81 (BGGR), which is BPP=24 */
*size = GST_ROUND_UP_4 (width) * height;
return TRUE;
} else {
/* For output, calculate according to format */
if (gst_structure_get_int (structure, "bpp", &pixsize)) {
*size = width * height * (pixsize / 8);
return TRUE;
}
}
}
return TRUE;
GST_ELEMENT_ERROR (base, CORE, NEGOTIATION, (NULL),
("Incomplete caps, some required field missing"));
return FALSE;
}
#define ARGB(a,r,g,b) ((b)<<24 | (g)<<16 | (r)<<8 | a)
/*
* We define values for the colors, just to make the code more readable.
*/
#define RED 0 /* Pure red element */
#define GREENB 1 /* Green element which is on a blue line */
#define BLUE 2 /* Pure blue element */
#define GREENR 3 /* Green element which is on a red line */
/* Routine to generate the top and bottom edges (not including corners) */
static void
upsample_even (uint8_t * dest, uint8_t * src, int width)
hborder (uint8_t * input, uint8_t * output, int bot_top,
int typ, GstBayer2RGB * filter)
{
int i;
uint8_t *op; /* output pointer */
uint8_t *ip; /* input pointer */
uint8_t *nx; /* next line pointer */
int ix; /* loop index */
for (i = 0; i < width - 2; i += 2) {
dest[i] = src[i];
dest[i + 1] = (src[i] + src[i + 2] + 1) / 2;
}
dest[i] = src[i];
if (i + 1 < width) {
dest[i + 1] = src[i];
op = output + (bot_top * filter->width * (filter->height - 1) + 1) *
filter->pixsize;
ip = input + bot_top * filter->stride * (filter->height - 1);
/* calculate minus or plus one line, depending upon bot_top flag */
nx = ip + (1 - 2 * bot_top) * filter->stride;
/* Stepping horizontally */
for (ix = 1; ix < filter->width - 1; ix++, op += filter->pixsize) {
switch (typ) {
case RED:
op[filter->r_off] = ip[ix];
op[filter->g_off] = (ip[ix + 1] + ip[ix - 1] + nx[ix] + 1) / 3;
op[filter->b_off] = (nx[ix + 1] + nx[ix - 1] + 1) / 2;
typ = GREENR;
break;
case GREENR:
op[filter->r_off] = (ip[ix + 1] + ip[ix - 1] + 1) / 2;
op[filter->g_off] = ip[ix];
op[filter->b_off] = nx[ix];
typ = RED;
break;
case GREENB:
op[filter->r_off] = nx[ix];
op[filter->g_off] = ip[ix];
op[filter->b_off] = (ip[ix + 1] + ip[ix - 1] + 1) / 2;
typ = BLUE;
break;
case BLUE:
op[filter->r_off] = (nx[ix + 1] + nx[ix - 1] + 1) / 2;
op[filter->g_off] = (ip[ix + 1] + ip[ix - 1] + nx[ix] + 1) / 3;
op[filter->b_off] = ip[ix];
typ = GREENB;
break;
}
}
}
/* Routine to generate the left and right edges, not including corners */
static void
upsample_odd (uint8_t * dest, uint8_t * src, int width)
vborder (uint8_t * input, uint8_t * output, int right_left,
int typ, GstBayer2RGB * filter)
{
int i;
uint8_t *op; /* output pointer */
uint8_t *ip; /* input pointer */
uint8_t *la; /* line above pointer */
uint8_t *lb; /* line below pointer */
int ix; /* loop index */
int lr; /* 'left-right' flag - +1 is right, -1 is left */
dest[0] = src[1];
for (i = 1; i < width - 2; i += 2) {
dest[i] = src[i];
dest[i + 1] = (src[i] + src[i + 2] + 1) / 2;
}
dest[i] = src[i];
if (i + 1 < width) {
dest[i + 1] = src[i];
lr = (1 - 2 * right_left);
/* stepping vertically */
for (ix = 1; ix < filter->height - 1; ix++) {
ip = input + right_left * (filter->width - 1) + ix * filter->stride;
op = output + (right_left * (filter->width - 1) + ix * filter->width) *
filter->pixsize;
la = ip + filter->stride;
lb = ip - filter->stride;
switch (typ) {
case RED:
op[filter->r_off] = ip[0];
op[filter->g_off] = (la[0] + ip[lr] + lb[0] + 1) / 3;
op[filter->b_off] = (la[lr] + lb[lr] + 1) / 2;
typ = GREENB;
break;
case GREENR:
op[filter->r_off] = ip[lr];
op[filter->g_off] = ip[0];
op[filter->b_off] = (la[lr] + lb[lr] + 1) / 2;
typ = BLUE;
break;
case GREENB:
op[filter->r_off] = (la[lr] + lb[lr] + 1) / 2;
op[filter->g_off] = ip[0];
op[filter->b_off] = ip[lr];
typ = RED;
break;
case BLUE:
op[filter->r_off] = (la[lr] + lb[lr] + 1) / 2;
op[filter->g_off] = (la[0] + ip[lr] + lb[0] + 1) / 3;
op[filter->b_off] = ip[0];
typ = GREENR;
break;
}
}
}
/* Produce the four (top, bottom, left, right) edges */
static void
interpolate (uint8_t * dest, uint8_t * src1, uint8_t * src2, int width)
do_row0_col0 (uint8_t * input, uint8_t * output, GstBayer2RGB * filter)
{
int i;
int type;
for (i = 0; i < width; i++) {
dest[i] = (src1[i] + src2[i] + 1) / 2;
}
/* Horizontal edges */
hborder (input, output, 0, GREENB, filter);
if (filter->height & 1)
type = GREENB; /* odd # rows, "bottom" edge same as top */
else
type = RED; /* even #, bottom side different */
hborder (input, output, 1, type, filter);
/* Vertical edges */
vborder (input, output, 0, GREENR, filter);
if (filter->width & 1)
type = GREENR; /* odd # cols, "right" edge same as left */
else
type = RED; /* even #, right side different */
vborder (input, output, 1, type, filter);
}
static void
merge (uint32_t * dest, uint8_t * r, uint8_t * g, uint8_t * b, int width)
corner (uint8_t * input, uint8_t * output, int x, int y,
int xd, int yd, int typ, GstBayer2RGB * filter)
{
int i;
uint8_t *ip; /* input pointer */
uint8_t *op; /* output pointer */
uint8_t *nx; /* adjacent line */
for (i = 0; i < width; i++) {
dest[i] = ARGB (0xff, r[i], g[i], b[i]);
op = output + y * filter->width * filter->pixsize + x * filter->pixsize;
ip = input + y * filter->stride + x;
nx = ip + yd * filter->stride;
switch (typ) {
case RED:
op[filter->r_off] = ip[0];
op[filter->g_off] = (nx[0] + ip[xd] + 1) / 2;
op[filter->b_off] = nx[xd];
break;
case GREENR:
op[filter->r_off] = ip[xd];
op[filter->g_off] = ip[0];
op[filter->b_off] = nx[0];
break;
case GREENB:
op[filter->r_off] = nx[0];
op[filter->g_off] = ip[0];
op[filter->b_off] = ip[xd];
break;
case BLUE:
op[filter->r_off] = nx[xd];
op[filter->g_off] = (nx[0] + ip[xd] + 1) / 2;
op[filter->b_off] = ip[0];
break;
}
}
static void
do_corners (uint8_t * input, uint8_t * output, GstBayer2RGB * filter)
{
int typ;
/* Top left */
corner (input, output, 0, 0, 1, 1, BLUE, filter);
/* Bottom left */
corner (input, output, 0, filter->height - 1, 1, -1,
(filter->height & 1) ? BLUE : GREENR, filter);
/* Top right */
corner (input, output, filter->width - 1, 0, -1, 0,
(filter->width & 1) ? BLUE : GREENB, filter);
/* Bottom right */
if (filter->width & 1) /* if odd # cols, B or GB */
typ = BLUE;
else
typ = GREENB; /* if even # cols, B or GR */
typ |= (filter->height & 1); /* if odd # rows, GB or GR */
corner (input, output, filter->width - 1, filter->height - 1, -1, -1,
typ, filter);
}
static void
do_body (uint8_t * input, uint8_t * output, GstBayer2RGB * filter)
{
int ip, op; /* input and output pointers */
int w, h; /* loop indices */
int type; /* calculated colour of current element */
int a1, a2;
int v1, v2, h1, h2;
/*
* We are processing row (line) by row, starting with the second
* row and continuing through the next to last. Each row is processed
* column by column, starting with the second and continuing through
* to the next to last.
*/
for (h = 1; h < filter->height - 1; h++) {
/*
* Remember we are processing "row by row". For each row, we need
* to set the type of the first element to be processed. Since we
* have already processed the edges, the "first element" will be
* the pixel at position (1,1). Assuming BG format, this should
* be RED for odd-numbered rows and GREENB for even rows.
*/
if (h & 1)
type = RED;
else
type = GREENB;
/* Calculate the starting position for the row */
op = h * filter->width * filter->pixsize; /* output (converted) pos */
ip = h * filter->stride; /* input (bayer data) pos */
for (w = 1; w < filter->width - 1; w++) {
op += filter->pixsize; /* we are processing "horizontally" */
ip++;
switch (type) {
case RED:
output[op + filter->r_off] = input[ip];
output[op + filter->b_off] = (input[ip - filter->stride - 1] +
input[ip - filter->stride + 1] +
input[ip + filter->stride - 1] +
input[ip + filter->stride + 1] + 2) / 4;
v1 = input[ip + filter->stride];
v2 = input[ip - filter->stride];
h1 = input[ip + 1];
h2 = input[ip - 1];
a1 = abs (v1 - v2);
a2 = abs (h1 - h2);
if (a1 < a2)
output[op + filter->g_off] = (v1 + v2 + 1) / 2;
else if (a1 > a2)
output[op + filter->g_off] = (h1 + h2 + 1) / 2;
else
output[op + filter->g_off] = (v1 + h1 + v2 + h2 + 2) / 4;
type = GREENR;
break;
case GREENR:
output[op + filter->r_off] = (input[ip + 1] + input[ip - 1] + 1) / 2;
output[op + filter->g_off] = input[ip];
output[op + filter->b_off] = (input[ip - filter->stride] +
input[ip + filter->stride] + 1) / 2;
type = RED;
break;
case GREENB:
output[op + filter->r_off] = (input[ip - filter->stride] +
input[ip + filter->stride] + 1) / 2;
output[op + filter->g_off] = input[ip];
output[op + filter->b_off] = (input[ip + 1] + input[ip - 1] + 1) / 2;
type = BLUE;
break;
case BLUE:
output[op + filter->r_off] = (input[ip - filter->stride - 1] +
input[ip - filter->stride + 1] +
input[ip + filter->stride - 1] +
input[ip + filter->stride + 1] + 2) / 4;
output[op + filter->b_off] = input[ip];
v1 = input[ip + filter->stride];
v2 = input[ip - filter->stride];
h1 = input[ip + 1];
h2 = input[ip - 1];
a1 = abs (v1 - v2);
a2 = abs (h1 - h2);
if (a1 < a2)
output[op + filter->g_off] = (v1 + v2 + 1) / 2;
else if (a1 > a2)
output[op + filter->g_off] = (h1 + h2 + 1) / 2;
else
output[op + filter->g_off] = (v1 + h1 + v2 + h2 + 2) / 4;
type = GREENB;
break;
}
}
}
}
@ -318,63 +636,22 @@ gst_bayer2rgb_transform (GstBaseTransform * base, GstBuffer * inbuf,
GstBuffer * outbuf)
{
GstBayer2RGB *filter = GST_BAYER2RGB (base);
uint8_t *tmpdata;
int j;
uint8_t *input, *output;
GST_DEBUG ("got here");
tmpdata = filter->tmpdata;
/*
* We need to lock our filter params to prevent changing
* caps in the middle of a transformation (nice way to get
* segfaults)
*/
GST_OBJECT_LOCK (filter);
/* This is a pretty lousy algorithm. In particular, most higher
* quality algorithms will apply some non-linear weighting factors
* in red/blue interpolation based on the green components. This
* just does a linear interpolation between surrounding pixels.
* For green, we only interpolate horizontally. */
for (j = 0; j < filter->height + 1; j++) {
if (j < filter->height) {
/* upsample horizontally */
if ((j & 1) == 0) {
upsample_even (tmpdata + (1 * 4 + (j & 3)) * filter->stride,
(uint8_t *) GST_BUFFER_DATA (inbuf) + filter->stride * j,
filter->width);
upsample_odd (tmpdata + (0 * 4 + (j & 3)) * filter->stride,
(uint8_t *) GST_BUFFER_DATA (inbuf) + filter->stride * j,
filter->width);
} else {
upsample_even (tmpdata + (2 * 4 + (j & 3)) * filter->stride,
(uint8_t *) GST_BUFFER_DATA (inbuf) + filter->stride * j,
filter->width);
upsample_odd (tmpdata + (1 * 4 + (j & 3)) * filter->stride,
(uint8_t *) GST_BUFFER_DATA (inbuf) + filter->stride * j,
filter->width);
}
}
if (j - 1 >= 0 && j - 1 < filter->height) {
int comp, j1, j2;
if (((j - 1) & 1) == 0) {
comp = 2;
} else {
comp = 0;
}
j1 = j - 2;
if (j1 < 0)
j1 += 2;
j2 = j;
if (j2 > filter->height - 1)
j2 -= 2;
interpolate (tmpdata + (comp * 4 + ((j - 1) & 3)) * filter->stride,
tmpdata + (comp * 4 + (j1 & 3)) * filter->stride,
tmpdata + (comp * 4 + (j2 & 3)) * filter->stride, filter->width);
merge (
(uint32_t *) ((uint8_t *) GST_BUFFER_DATA (outbuf) +
4 * filter->width * (j - 1)),
tmpdata + (0 * 4 + ((j - 1) & 3)) * filter->stride,
tmpdata + (1 * 4 + ((j - 1) & 3)) * filter->stride,
tmpdata + (2 * 4 + ((j - 1) & 3)) * filter->stride, filter->width);
}
}
GST_DEBUG ("transforming buffer");
input = (uint8_t *) GST_BUFFER_DATA (inbuf);
output = (uint8_t *) GST_BUFFER_DATA (outbuf);
do_corners (input, output, filter);
do_row0_col0 (input, output, filter);
do_body (input, output, filter);
GST_OBJECT_UNLOCK (filter);
return GST_FLOW_OK;
}