diff --git a/sys/nvcodec/gstcudabasefilter.c b/sys/nvcodec/gstcudabasefilter.c new file mode 100644 index 0000000000..e9bff466f3 --- /dev/null +++ b/sys/nvcodec/gstcudabasefilter.c @@ -0,0 +1,313 @@ +/* GStreamer + * Copyright (C) <1999> Erik Walthinsen + * Copyright (C) 2005-2012 David Schleef + * Copyright (C) <2019> Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +# include +#endif + +#include "gstcudabasefilter.h" +#include "gstcudautils.h" +#include + +GST_DEBUG_CATEGORY_STATIC (gst_cuda_base_filter_debug); +#define GST_CAT_DEFAULT gst_cuda_base_filter_debug + +static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_CUDA_MEMORY, GST_CUDA_CONVERTER_FORMATS)) + ); + +static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_CUDA_MEMORY, GST_CUDA_CONVERTER_FORMATS)) + ); + +#define gst_cuda_base_filter_parent_class parent_class +G_DEFINE_ABSTRACT_TYPE (GstCudaBaseFilter, + gst_cuda_base_filter, GST_TYPE_CUDA_BASE_TRANSFORM); + +static void gst_cuda_base_filter_dispose (GObject * object); +static GstFlowReturn +gst_cuda_base_filter_transform_frame (GstCudaBaseTransform * btrans, + GstVideoFrame * in_frame, GstCudaMemory * in_cuda_mem, + GstVideoFrame * out_frame, GstCudaMemory * out_cuda_mem); +static gboolean gst_cuda_base_filter_set_info (GstCudaBaseTransform * btrans, + GstCaps * incaps, GstVideoInfo * in_info, GstCaps * outcaps, + GstVideoInfo * out_info); + +static void +gst_cuda_base_filter_class_init (GstCudaBaseFilterClass * klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + GstElementClass *element_class = GST_ELEMENT_CLASS (klass); + GstBaseTransformClass *trans_class = GST_BASE_TRANSFORM_CLASS (klass); + GstCudaBaseTransformClass *btrans_class = + GST_CUDA_BASE_TRANSFORM_CLASS (klass); + + gobject_class->dispose = gst_cuda_base_filter_dispose; + + gst_element_class_add_static_pad_template (element_class, &sink_template); + gst_element_class_add_static_pad_template (element_class, &src_template); + + trans_class->passthrough_on_same_caps = TRUE; + + btrans_class->set_info = GST_DEBUG_FUNCPTR (gst_cuda_base_filter_set_info); + btrans_class->transform_frame = + GST_DEBUG_FUNCPTR (gst_cuda_base_filter_transform_frame); + + GST_DEBUG_CATEGORY_INIT (gst_cuda_base_filter_debug, + "cudabasefilter", 0, "CUDA Base Filter"); +} + +static void +gst_cuda_base_filter_init (GstCudaBaseFilter * convert) +{ +} + +static void +gst_cuda_base_filter_dispose (GObject * object) +{ + GstCudaBaseFilter *filter = GST_CUDA_BASE_FILTER (object); + + if (filter->converter) { + gst_cuda_converter_free (filter->converter); + filter->converter = NULL; + } + + if (filter->in_fallback) { + gst_memory_unref (GST_MEMORY_CAST (filter->in_fallback)); + filter->in_fallback = NULL; + } + + if (filter->out_fallback) { + gst_memory_unref (GST_MEMORY_CAST (filter->out_fallback)); + filter->out_fallback = NULL; + } + + gst_clear_object (&filter->allocator); + + G_OBJECT_CLASS (parent_class)->dispose (object); +} + +static gboolean +gst_cuda_base_filter_configure (GstCudaBaseFilter * filter, + GstVideoInfo * in_info, GstVideoInfo * out_info) +{ + GstCudaBaseTransform *btrans = GST_CUDA_BASE_TRANSFORM (filter); + + /* cleanup internal pool */ + if (filter->in_fallback) { + gst_memory_unref (GST_MEMORY_CAST (filter->in_fallback)); + filter->in_fallback = NULL; + } + + if (filter->out_fallback) { + gst_memory_unref (GST_MEMORY_CAST (filter->out_fallback)); + filter->out_fallback = NULL; + } + + if (!filter->allocator) + filter->allocator = gst_cuda_allocator_new (btrans->context); + + if (!filter->allocator) { + GST_ERROR_OBJECT (filter, "Failed to create CUDA allocator"); + return FALSE; + } + + return TRUE; +} + +static gboolean +gst_cuda_base_filter_set_info (GstCudaBaseTransform * btrans, GstCaps * incaps, + GstVideoInfo * in_info, GstCaps * outcaps, GstVideoInfo * out_info) +{ + GstCudaBaseFilter *filter = GST_CUDA_BASE_FILTER (btrans); + + if (!gst_cuda_base_filter_configure (filter, in_info, out_info)) { + return FALSE; + } + + if (filter->converter) + gst_cuda_converter_free (filter->converter); + + filter->converter = + gst_cuda_converter_new (in_info, out_info, btrans->context); + + if (filter->converter == NULL) + goto no_converter; + + GST_DEBUG_OBJECT (filter, "reconfigured %d %d", + GST_VIDEO_INFO_FORMAT (in_info), GST_VIDEO_INFO_FORMAT (out_info)); + + return TRUE; + +no_converter: + { + GST_ERROR_OBJECT (filter, "could not create converter"); + return FALSE; + } +} + +static GstFlowReturn +gst_cuda_base_filter_transform_frame (GstCudaBaseTransform * btrans, + GstVideoFrame * in_frame, GstCudaMemory * in_cuda_mem, + GstVideoFrame * out_frame, GstCudaMemory * out_cuda_mem) +{ + GstCudaBaseFilter *filter = GST_CUDA_BASE_FILTER (btrans); + gboolean conv_ret; + GstCudaMemory *in_mem; + GstCudaMemory *out_mem; + gint i; + + if (in_cuda_mem) { + in_mem = in_cuda_mem; + } else { + if (!filter->in_fallback) { + GstCudaAllocationParams params; + + memset (¶ms, 0, sizeof (GstCudaAllocationParams)); + params.info = btrans->in_info; + + filter->in_fallback = + (GstCudaMemory *) gst_cuda_allocator_alloc (filter->allocator, + GST_VIDEO_INFO_SIZE (¶ms.info), ¶ms); + } + + if (!filter->in_fallback) { + GST_ERROR_OBJECT (filter, "Couldn't allocate fallback memory"); + return GST_FLOW_ERROR; + } + + GST_TRACE_OBJECT (filter, "use CUDA fallback memory input"); + + if (!gst_cuda_context_push (btrans->context)) { + GST_ELEMENT_ERROR (filter, LIBRARY, FAILED, (NULL), + ("Cannot push CUDA context")); + return FALSE; + } + + /* upload frame to device memory */ + for (i = 0; i < GST_VIDEO_FRAME_N_PLANES (in_frame); i++) { + CUDA_MEMCPY2D param = { 0, }; + guint width, height; + + width = GST_VIDEO_FRAME_COMP_WIDTH (in_frame, i) * + GST_VIDEO_FRAME_COMP_PSTRIDE (in_frame, i); + height = GST_VIDEO_FRAME_COMP_HEIGHT (in_frame, i); + + param.srcMemoryType = CU_MEMORYTYPE_HOST; + param.srcPitch = GST_VIDEO_FRAME_PLANE_STRIDE (in_frame, i); + param.srcHost = GST_VIDEO_FRAME_PLANE_DATA (in_frame, i); + param.dstMemoryType = CU_MEMORYTYPE_DEVICE; + param.dstPitch = filter->in_fallback->stride; + param.dstDevice = + filter->in_fallback->data + filter->in_fallback->offset[i]; + param.WidthInBytes = width; + param.Height = height; + + if (!gst_cuda_result (CuMemcpy2DAsync (¶m, btrans->cuda_stream))) { + gst_cuda_context_pop (NULL); + GST_ELEMENT_ERROR (filter, LIBRARY, FAILED, (NULL), + ("Cannot upload input video frame")); + return GST_FLOW_ERROR; + } + } + + gst_cuda_result (CuStreamSynchronize (btrans->cuda_stream)); + gst_cuda_context_pop (NULL); + + in_mem = filter->in_fallback; + } + + if (out_cuda_mem) { + out_mem = out_cuda_mem; + } else { + if (!filter->out_fallback) { + GstCudaAllocationParams params; + + memset (¶ms, 0, sizeof (GstCudaAllocationParams)); + params.info = btrans->out_info; + + filter->out_fallback = + (GstCudaMemory *) gst_cuda_allocator_alloc (filter->allocator, + GST_VIDEO_INFO_SIZE (¶ms.info), ¶ms); + } + + if (!filter->out_fallback) { + GST_ERROR_OBJECT (filter, "Couldn't allocate fallback memory"); + return GST_FLOW_ERROR; + } + + out_mem = filter->out_fallback; + } + + conv_ret = + gst_cuda_converter_frame (filter->converter, in_mem, &btrans->in_info, + out_mem, &btrans->out_info, btrans->cuda_stream); + + if (!conv_ret) { + GST_ERROR_OBJECT (filter, "Failed to convert frame"); + return GST_FLOW_ERROR; + } + + if (!out_cuda_mem) { + if (!gst_cuda_context_push (btrans->context)) { + GST_ELEMENT_ERROR (filter, LIBRARY, FAILED, (NULL), + ("Cannot push CUDA context")); + return FALSE; + } + + for (i = 0; i < GST_VIDEO_FRAME_N_PLANES (out_frame); i++) { + CUDA_MEMCPY2D param = { 0, }; + guint width, height; + + width = GST_VIDEO_FRAME_COMP_WIDTH (out_frame, i) * + GST_VIDEO_FRAME_COMP_PSTRIDE (out_frame, i); + height = GST_VIDEO_FRAME_COMP_HEIGHT (out_frame, i); + + param.srcMemoryType = CU_MEMORYTYPE_DEVICE; + param.srcPitch = out_mem->stride; + param.srcDevice = + filter->out_fallback->data + filter->out_fallback->offset[i]; + param.dstMemoryType = CU_MEMORYTYPE_HOST; + param.dstPitch = GST_VIDEO_FRAME_PLANE_STRIDE (out_frame, i); + param.dstHost = GST_VIDEO_FRAME_PLANE_DATA (out_frame, i); + param.WidthInBytes = width; + param.Height = height; + + if (!gst_cuda_result (CuMemcpy2DAsync (¶m, btrans->cuda_stream))) { + gst_cuda_context_pop (NULL); + GST_ELEMENT_ERROR (filter, LIBRARY, FAILED, (NULL), + ("Cannot upload input video frame")); + return GST_FLOW_ERROR; + } + } + + gst_cuda_result (CuStreamSynchronize (btrans->cuda_stream)); + gst_cuda_context_pop (NULL); + } + + return GST_FLOW_OK; +} diff --git a/sys/nvcodec/gstcudabasefilter.h b/sys/nvcodec/gstcudabasefilter.h new file mode 100644 index 0000000000..abec8a53d8 --- /dev/null +++ b/sys/nvcodec/gstcudabasefilter.h @@ -0,0 +1,61 @@ +/* GStreamer + * Copyright (C) <2019> Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_CUDA_BASE_FILTER_H__ +#define __GST_CUDA_BASE_FILTER_H__ + +#include + +#include "gstcudabasetransform.h" +#include "cuda-converter.h" + +G_BEGIN_DECLS + +#define GST_TYPE_CUDA_BASE_FILTER (gst_cuda_base_filter_get_type()) +#define GST_CUDA_BASE_FILTER(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_CUDA_BASE_FILTER,GstCudaBaseFilter)) +#define GST_CUDA_BASE_FILTER_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass), GST_TYPE_CUDA_BASE_FILTER,GstCudaBaseFilterClass)) +#define GST_CUDA_BASE_FILTER_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS((obj), GST_TYPE_CUDA_BASE_FILTER,GstCudaBaseFilterClass)) +#define GST_IS_CUDA_BASE_FILTER(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_CUDA_BASE_FILTER)) +#define GST_IS_CUDA_BASE_FILTER_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass), GST_TYPE_CUDA_BASE_FILTER)) + +typedef struct _GstCudaBaseFilter GstCudaBaseFilter; +typedef struct _GstCudaBaseFilterClass GstCudaBaseFilterClass; + +struct _GstCudaBaseFilter +{ + GstCudaBaseTransform parent; + + GstCudaConverter *converter; + + /* fallback CUDA memory */ + GstAllocator *allocator; + GstCudaMemory *in_fallback; + GstCudaMemory *out_fallback; +}; + +struct _GstCudaBaseFilterClass +{ + GstCudaBaseTransformClass parent_class; +}; + +GType gst_cuda_base_filter_get_type (void); + +G_END_DECLS + +#endif /* __GST_CUDA_BASE_FILTER_H__ */ diff --git a/sys/nvcodec/gstcudaconvert.c b/sys/nvcodec/gstcudaconvert.c new file mode 100644 index 0000000000..76de2a8d6f --- /dev/null +++ b/sys/nvcodec/gstcudaconvert.c @@ -0,0 +1,414 @@ +/* GStreamer + * Copyright (C) <1999> Erik Walthinsen + * Copyright (C) 2005-2012 David Schleef + * Copyright (C) <2019> Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/** + * SECTION:element-cudaconvert + * @title: cudaconvert + * + * Convert video frames between supported video formats. + * + * ## Example launch line + * |[ + * gst-launch-1.0 -v videotestsrc ! video/x-raw,format=Y444_16LE ! cudaupload ! cudaconvert ! cudadownload ! autovideosink + * ]| + * This will output a test video (generated in Y444_16LE format) in a video + * window. If the video sink selected does not support Y444_16LE + * cudaconvert will automatically convert the video to a format understood + * by the video sink. + * + */ + +#ifdef HAVE_CONFIG_H +# include +#endif + +#include "gstcudaconvert.h" +#include "gstcudautils.h" + +GST_DEBUG_CATEGORY_STATIC (gst_cuda_convert_debug); +#define GST_CAT_DEFAULT gst_cuda_convert_debug + +#define gst_cuda_convert_parent_class parent_class +G_DEFINE_TYPE (GstCudaConvert, gst_cuda_convert, GST_TYPE_CUDA_BASE_FILTER); + +static GstCaps *gst_cuda_convert_transform_caps (GstBaseTransform * trans, + GstPadDirection direction, GstCaps * caps, GstCaps * filter); +static GstCaps *gst_cuda_convert_fixate_caps (GstBaseTransform * base, + GstPadDirection direction, GstCaps * caps, GstCaps * othercaps); +static gboolean gst_cuda_convert_filter_meta (GstBaseTransform * trans, + GstQuery * query, GType api, const GstStructure * params); +static gboolean +gst_cuda_convert_set_info (GstCudaBaseTransform * btrans, GstCaps * incaps, + GstVideoInfo * in_info, GstCaps * outcaps, GstVideoInfo * out_info); + +/* copies the given caps */ +static GstCaps * +gst_cuda_convert_caps_remove_format_info (GstCaps * caps) +{ + GstStructure *st; + GstCapsFeatures *f; + gint i, n; + GstCaps *res; + GstCapsFeatures *feature = + gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_CUDA_MEMORY); + + res = gst_caps_new_empty (); + + n = gst_caps_get_size (caps); + for (i = 0; i < n; i++) { + st = gst_caps_get_structure (caps, i); + f = gst_caps_get_features (caps, i); + + /* If this is already expressed by the existing caps + * skip this structure */ + if (i > 0 && gst_caps_is_subset_structure_full (res, st, f)) + continue; + + st = gst_structure_copy (st); + /* Only remove format info for the cases when we can actually convert */ + if (!gst_caps_features_is_any (f) + && gst_caps_features_is_equal (f, feature)) + gst_structure_remove_fields (st, "format", "colorimetry", "chroma-site", + NULL); + + gst_caps_append_structure_full (res, st, gst_caps_features_copy (f)); + } + gst_caps_features_free (feature); + + return res; +} + +/* + * This is an incomplete matrix of in formats and a score for the prefered output + * format. + * + * out: RGB24 RGB16 ARGB AYUV YUV444 YUV422 YUV420 YUV411 YUV410 PAL GRAY + * in + * RGB24 0 2 1 2 2 3 4 5 6 7 8 + * RGB16 1 0 1 2 2 3 4 5 6 7 8 + * ARGB 2 3 0 1 4 5 6 7 8 9 10 + * AYUV 3 4 1 0 2 5 6 7 8 9 10 + * YUV444 2 4 3 1 0 5 6 7 8 9 10 + * YUV422 3 5 4 2 1 0 6 7 8 9 10 + * YUV420 4 6 5 3 2 1 0 7 8 9 10 + * YUV411 4 6 5 3 2 1 7 0 8 9 10 + * YUV410 6 8 7 5 4 3 2 1 0 9 10 + * PAL 1 3 2 6 4 6 7 8 9 0 10 + * GRAY 1 4 3 2 1 5 6 7 8 9 0 + * + * PAL or GRAY are never prefered, if we can we would convert to PAL instead + * of GRAY, though + * less subsampling is prefered and if any, preferably horizontal + * We would like to keep the alpha, even if we would need to to colorspace conversion + * or lose depth. + */ +#define SCORE_FORMAT_CHANGE 1 +#define SCORE_DEPTH_CHANGE 1 +#define SCORE_ALPHA_CHANGE 1 +#define SCORE_CHROMA_W_CHANGE 1 +#define SCORE_CHROMA_H_CHANGE 1 +#define SCORE_PALETTE_CHANGE 1 + +#define SCORE_COLORSPACE_LOSS 2 /* RGB <-> YUV */ +#define SCORE_DEPTH_LOSS 4 /* change bit depth */ +#define SCORE_ALPHA_LOSS 8 /* lose the alpha channel */ +#define SCORE_CHROMA_W_LOSS 16 /* vertical subsample */ +#define SCORE_CHROMA_H_LOSS 32 /* horizontal subsample */ +#define SCORE_PALETTE_LOSS 64 /* convert to palette format */ +#define SCORE_COLOR_LOSS 128 /* convert to GRAY */ + +#define COLORSPACE_MASK (GST_VIDEO_FORMAT_FLAG_YUV | \ + GST_VIDEO_FORMAT_FLAG_RGB | GST_VIDEO_FORMAT_FLAG_GRAY) +#define ALPHA_MASK (GST_VIDEO_FORMAT_FLAG_ALPHA) +#define PALETTE_MASK (GST_VIDEO_FORMAT_FLAG_PALETTE) + +/* calculate how much loss a conversion would be */ +static void +score_value (GstBaseTransform * base, const GstVideoFormatInfo * in_info, + const GValue * val, gint * min_loss, const GstVideoFormatInfo ** out_info) +{ + const gchar *fname; + const GstVideoFormatInfo *t_info; + GstVideoFormatFlags in_flags, t_flags; + gint loss; + + fname = g_value_get_string (val); + t_info = gst_video_format_get_info (gst_video_format_from_string (fname)); + if (!t_info) + return; + + /* accept input format immediately without loss */ + if (in_info == t_info) { + *min_loss = 0; + *out_info = t_info; + return; + } + + loss = SCORE_FORMAT_CHANGE; + + in_flags = GST_VIDEO_FORMAT_INFO_FLAGS (in_info); + in_flags &= ~GST_VIDEO_FORMAT_FLAG_LE; + in_flags &= ~GST_VIDEO_FORMAT_FLAG_COMPLEX; + in_flags &= ~GST_VIDEO_FORMAT_FLAG_UNPACK; + + t_flags = GST_VIDEO_FORMAT_INFO_FLAGS (t_info); + t_flags &= ~GST_VIDEO_FORMAT_FLAG_LE; + t_flags &= ~GST_VIDEO_FORMAT_FLAG_COMPLEX; + t_flags &= ~GST_VIDEO_FORMAT_FLAG_UNPACK; + + if ((t_flags & PALETTE_MASK) != (in_flags & PALETTE_MASK)) { + loss += SCORE_PALETTE_CHANGE; + if (t_flags & PALETTE_MASK) + loss += SCORE_PALETTE_LOSS; + } + + if ((t_flags & COLORSPACE_MASK) != (in_flags & COLORSPACE_MASK)) { + loss += SCORE_COLORSPACE_LOSS; + if (t_flags & GST_VIDEO_FORMAT_FLAG_GRAY) + loss += SCORE_COLOR_LOSS; + } + + if ((t_flags & ALPHA_MASK) != (in_flags & ALPHA_MASK)) { + loss += SCORE_ALPHA_CHANGE; + if (in_flags & ALPHA_MASK) + loss += SCORE_ALPHA_LOSS; + } + + if ((in_info->h_sub[1]) != (t_info->h_sub[1])) { + loss += SCORE_CHROMA_H_CHANGE; + if ((in_info->h_sub[1]) < (t_info->h_sub[1])) + loss += SCORE_CHROMA_H_LOSS; + } + if ((in_info->w_sub[1]) != (t_info->w_sub[1])) { + loss += SCORE_CHROMA_W_CHANGE; + if ((in_info->w_sub[1]) < (t_info->w_sub[1])) + loss += SCORE_CHROMA_W_LOSS; + } + + if ((in_info->bits) != (t_info->bits)) { + loss += SCORE_DEPTH_CHANGE; + if ((in_info->bits) > (t_info->bits)) + loss += SCORE_DEPTH_LOSS; + } + + GST_DEBUG_OBJECT (base, "score %s -> %s = %d", + GST_VIDEO_FORMAT_INFO_NAME (in_info), + GST_VIDEO_FORMAT_INFO_NAME (t_info), loss); + + if (loss < *min_loss) { + GST_DEBUG_OBJECT (base, "found new best %d", loss); + *out_info = t_info; + *min_loss = loss; + } +} + +static void +gst_cuda_convert_class_init (GstCudaConvertClass * klass) +{ + GstElementClass *element_class = GST_ELEMENT_CLASS (klass); + GstBaseTransformClass *trans_class = GST_BASE_TRANSFORM_CLASS (klass); + GstCudaBaseTransformClass *btrans_class = + GST_CUDA_BASE_TRANSFORM_CLASS (klass); + + gst_element_class_set_static_metadata (element_class, + "CUDA Colorspace converter", + "Filter/Converter/Video/Hardware", + "Converts video from one colorspace to another using CUDA", + "Seungha Yang "); + + trans_class->passthrough_on_same_caps = TRUE; + + trans_class->transform_caps = + GST_DEBUG_FUNCPTR (gst_cuda_convert_transform_caps); + trans_class->fixate_caps = GST_DEBUG_FUNCPTR (gst_cuda_convert_fixate_caps); + trans_class->filter_meta = GST_DEBUG_FUNCPTR (gst_cuda_convert_filter_meta); + + btrans_class->set_info = GST_DEBUG_FUNCPTR (gst_cuda_convert_set_info); + + GST_DEBUG_CATEGORY_INIT (gst_cuda_convert_debug, + "cudaconvert", 0, "Video ColorSpace convert using CUDA"); +} + +static void +gst_cuda_convert_init (GstCudaConvert * convert) +{ +} + +static GstCaps * +gst_cuda_convert_transform_caps (GstBaseTransform * trans, + GstPadDirection direction, GstCaps * caps, GstCaps * filter) +{ + GstCaps *tmp, *tmp2; + GstCaps *result; + + /* Get all possible caps that we can transform to */ + tmp = gst_cuda_convert_caps_remove_format_info (caps); + + if (filter) { + tmp2 = gst_caps_intersect_full (filter, tmp, GST_CAPS_INTERSECT_FIRST); + gst_caps_unref (tmp); + tmp = tmp2; + } + + result = tmp; + + GST_DEBUG_OBJECT (trans, "transformed %" GST_PTR_FORMAT " into %" + GST_PTR_FORMAT, caps, result); + + return result; +} + +/* fork of gstvideoconvert */ +static void +gst_cuda_convert_fixate_format (GstBaseTransform * base, GstCaps * caps, + GstCaps * result) +{ + GstStructure *ins, *outs; + const gchar *in_format; + const GstVideoFormatInfo *in_info, *out_info = NULL; + gint min_loss = G_MAXINT; + guint i, capslen; + + ins = gst_caps_get_structure (caps, 0); + in_format = gst_structure_get_string (ins, "format"); + if (!in_format) + return; + + GST_DEBUG_OBJECT (base, "source format %s", in_format); + + in_info = + gst_video_format_get_info (gst_video_format_from_string (in_format)); + if (!in_info) + return; + + outs = gst_caps_get_structure (result, 0); + + capslen = gst_caps_get_size (result); + GST_DEBUG_OBJECT (base, "iterate %d structures", capslen); + for (i = 0; i < capslen; i++) { + GstStructure *tests; + const GValue *format; + + tests = gst_caps_get_structure (result, i); + format = gst_structure_get_value (tests, "format"); + /* should not happen */ + if (format == NULL) + continue; + + if (GST_VALUE_HOLDS_LIST (format)) { + gint j, len; + + len = gst_value_list_get_size (format); + GST_DEBUG_OBJECT (base, "have %d formats", len); + for (j = 0; j < len; j++) { + const GValue *val; + + val = gst_value_list_get_value (format, j); + if (G_VALUE_HOLDS_STRING (val)) { + score_value (base, in_info, val, &min_loss, &out_info); + if (min_loss == 0) + break; + } + } + } else if (G_VALUE_HOLDS_STRING (format)) { + score_value (base, in_info, format, &min_loss, &out_info); + } + } + if (out_info) + gst_structure_set (outs, "format", G_TYPE_STRING, + GST_VIDEO_FORMAT_INFO_NAME (out_info), NULL); +} + +static GstCaps * +gst_cuda_convert_fixate_caps (GstBaseTransform * trans, + GstPadDirection direction, GstCaps * caps, GstCaps * othercaps) +{ + GstCaps *result; + + GST_DEBUG_OBJECT (trans, "trying to fixate othercaps %" GST_PTR_FORMAT + " based on caps %" GST_PTR_FORMAT, othercaps, caps); + + result = gst_caps_intersect (othercaps, caps); + if (gst_caps_is_empty (result)) { + gst_caps_unref (result); + result = othercaps; + } else { + gst_caps_unref (othercaps); + } + + GST_DEBUG_OBJECT (trans, "now fixating %" GST_PTR_FORMAT, result); + + result = gst_caps_make_writable (result); + gst_cuda_convert_fixate_format (trans, caps, result); + + /* fixate remaining fields */ + result = gst_caps_fixate (result); + + if (direction == GST_PAD_SINK) { + if (gst_caps_is_subset (caps, result)) { + gst_caps_replace (&result, caps); + } + } + + return result; +} + +static gboolean +gst_cuda_convert_filter_meta (GstBaseTransform * trans, GstQuery * query, + GType api, const GstStructure * params) +{ + /* This element cannot passthrough the crop meta, because it would convert the + * wrong sub-region of the image, and worst, our output image may not be large + * enough for the crop to be applied later */ + if (api == GST_VIDEO_CROP_META_API_TYPE) + return FALSE; + + /* propose all other metadata upstream */ + return TRUE; +} + +static gboolean +gst_cuda_convert_set_info (GstCudaBaseTransform * btrans, GstCaps * incaps, + GstVideoInfo * in_info, GstCaps * outcaps, GstVideoInfo * out_info) +{ + /* these must match */ + if (in_info->width != out_info->width || in_info->height != out_info->height + || in_info->fps_n != out_info->fps_n || in_info->fps_d != out_info->fps_d) + goto format_mismatch; + + /* if present, these must match too */ + if (in_info->par_n != out_info->par_n || in_info->par_d != out_info->par_d) + goto format_mismatch; + + /* if present, these must match too */ + if (in_info->interlace_mode != out_info->interlace_mode) + goto format_mismatch; + + return GST_CUDA_BASE_TRANSFORM_CLASS (parent_class)->set_info (btrans, incaps, + in_info, outcaps, out_info); + + /* ERRORS */ +format_mismatch: + { + GST_ERROR_OBJECT (btrans, "input and output formats do not match"); + return FALSE; + } +} diff --git a/sys/nvcodec/gstcudaconvert.h b/sys/nvcodec/gstcudaconvert.h new file mode 100644 index 0000000000..f8c2f50d0d --- /dev/null +++ b/sys/nvcodec/gstcudaconvert.h @@ -0,0 +1,53 @@ +/* GStreamer + * Copyright (C) <2019> Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_CUDA_CONVERT_H__ +#define __GST_CUDA_CONVERT_H__ + +#include + +#include "gstcudabasefilter.h" + +G_BEGIN_DECLS + +#define GST_TYPE_CUDA_CONVERT (gst_cuda_convert_get_type()) +#define GST_CUDA_CONVERT(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_CUDA_CONVERT,GstCudaConvert)) +#define GST_CUDA_CONVERT_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass), GST_TYPE_CUDA_CONVERT,GstCudaConvertClass)) +#define GST_CUDA_CONVERT_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS((obj), GST_TYPE_CUDA_CONVERT,GstCudaConvertClass)) +#define GST_IS_CUDA_CONVERT(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_CUDA_CONVERT)) +#define GST_IS_CUDA_CONVERT_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass), GST_TYPE_CUDA_CONVERT)) + +typedef struct _GstCudaConvert GstCudaConvert; +typedef struct _GstCudaConvertClass GstCudaConvertClass; + +struct _GstCudaConvert +{ + GstCudaBaseFilter parent; +}; + +struct _GstCudaConvertClass +{ + GstCudaBaseFilterClass parent_class; +}; + +GType gst_cuda_convert_get_type (void); + +G_END_DECLS + +#endif /* __GST_CUDA_CONVERT_H__ */ diff --git a/sys/nvcodec/gstcudafilter.c b/sys/nvcodec/gstcudafilter.c new file mode 100644 index 0000000000..b338df72be --- /dev/null +++ b/sys/nvcodec/gstcudafilter.c @@ -0,0 +1,53 @@ +/* GStreamer + * Copyright (C) 2019 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstcudafilter.h" +#include "gstcudaloader.h" +#include "gstnvrtcloader.h" +#include "gstcudanvrtc.h" +#include "gstcudaconvert.h" + +/* *INDENT-OFF* */ +const gchar *nvrtc_test_source = + "__global__ void\n" + "my_kernel (void) {}"; +/* *INDENT-ON* */ + +void +gst_cuda_filter_plugin_init (GstPlugin * plugin) +{ + gchar *test_ptx = NULL; + + if (!gst_nvrtc_load_library ()) + return; + + test_ptx = gst_cuda_nvrtc_compile (nvrtc_test_source); + + if (!test_ptx) { + return; + } + g_free (test_ptx); + + gst_element_register (plugin, "cudaconvert", GST_RANK_NONE, + GST_TYPE_CUDA_CONVERT); +} diff --git a/sys/nvcodec/gstcudafilter.h b/sys/nvcodec/gstcudafilter.h new file mode 100644 index 0000000000..174ecd6390 --- /dev/null +++ b/sys/nvcodec/gstcudafilter.h @@ -0,0 +1,32 @@ +/* GStreamer + * Copyright (C) 2019 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_CUDA_FILTER_H__ +#define __GST_CUDA_FILTER_H__ + +#include + +G_BEGIN_DECLS + +G_GNUC_INTERNAL +void gst_cuda_filter_plugin_init (GstPlugin * plugin); + +G_END_DECLS + +#endif /* __GST_CUDA_FILTER_H__ */ diff --git a/sys/nvcodec/meson.build b/sys/nvcodec/meson.build index f35c0c0d76..092d373f25 100644 --- a/sys/nvcodec/meson.build +++ b/sys/nvcodec/meson.build @@ -20,6 +20,9 @@ nvcodec_sources = [ 'gstcudanvrtc.c', 'gstnvrtcloader.c', 'cuda-converter.c', + 'gstcudafilter.c', + 'gstcudabasefilter.c', + 'gstcudaconvert.c', ] if get_option('nvcodec').disabled() diff --git a/sys/nvcodec/plugin.c b/sys/nvcodec/plugin.c index abb653442f..828b8775e4 100644 --- a/sys/nvcodec/plugin.c +++ b/sys/nvcodec/plugin.c @@ -36,6 +36,7 @@ #include "gstnvdecoder.h" #include "gstcudadownload.h" #include "gstcudaupload.h" +#include "gstcudafilter.h" GST_DEBUG_CATEGORY (gst_nvcodec_debug); GST_DEBUG_CATEGORY (gst_nvdec_debug); @@ -201,6 +202,8 @@ plugin_init (GstPlugin * plugin) gst_element_register (plugin, "cudaupload", GST_RANK_NONE, GST_TYPE_CUDA_UPLOAD); + gst_cuda_filter_plugin_init (plugin); + return TRUE; }