From 4449881b0819035ee15cefd06f257f968dbd9bcc Mon Sep 17 00:00:00 2001 From: Seungha Yang Date: Sun, 30 Apr 2023 20:42:56 +0900 Subject: [PATCH] d3d11: Add support for IPC Adding d3d11ipcsrc and d3d11ipcsink elements so that Direct3D11 textures can be shared between processes Part-of: --- .../gst-plugins-bad/sys/d3d11/gstd3d11ipc.cpp | 419 +++++++ .../gst-plugins-bad/sys/d3d11/gstd3d11ipc.h | 161 +++ .../sys/d3d11/gstd3d11ipcclient.cpp | 1098 +++++++++++++++++ .../sys/d3d11/gstd3d11ipcclient.h | 58 + .../sys/d3d11/gstd3d11ipcserver.cpp | 825 +++++++++++++ .../sys/d3d11/gstd3d11ipcserver.h | 45 + .../sys/d3d11/gstd3d11ipcsink.cpp | 812 ++++++++++++ .../sys/d3d11/gstd3d11ipcsink.h | 33 + .../sys/d3d11/gstd3d11ipcsrc.cpp | 525 ++++++++ .../sys/d3d11/gstd3d11ipcsrc.h | 33 + .../gst-plugins-bad/sys/d3d11/meson.build | 5 + .../gst-plugins-bad/sys/d3d11/plugin.cpp | 6 + 12 files changed, 4020 insertions(+) create mode 100644 subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipc.cpp create mode 100644 subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipc.h create mode 100644 subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipcclient.cpp create mode 100644 subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipcclient.h create mode 100644 subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipcserver.cpp create mode 100644 subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipcserver.h create mode 100644 subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipcsink.cpp create mode 100644 subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipcsink.h create mode 100644 subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipcsrc.cpp create mode 100644 subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipcsrc.h diff --git a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipc.cpp b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipc.cpp new file mode 100644 index 0000000000..ea4305aa42 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipc.cpp @@ -0,0 +1,419 @@ +/* GStreamer + * Copyright (C) 2023 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstd3d11ipc.h" +#include +#include +#include +#include +#include +#include + +#define GST_D3D11_IPC_MAGIC_NUMBER 0xD3D1110C + +bool +gst_d3d11_ipc_pkt_identify (std::vector < guint8 > &buf, + GstD3D11IpcPacketHeader & header) +{ + g_return_val_if_fail (buf.size () >= GST_D3D11_IPC_PKT_HEADER_SIZE, false); + + memcpy (&header, &buf[0], GST_D3D11_IPC_PKT_HEADER_SIZE); + + if (header.magic != GST_D3D11_IPC_MAGIC_NUMBER) + return false; + + buf.resize (GST_D3D11_IPC_PKT_HEADER_SIZE + header.payload_size); + + return true; +} + +bool +gst_d3d11_ipc_pkt_build_config (std::vector < guint8 > &buf, + gint64 adapter_luid, GstCaps * caps) +{ + GstD3D11IpcPacketHeader header; + guint8 *ptr; + gchar *caps_str = nullptr; + guint caps_size = 0; + + g_return_val_if_fail (GST_IS_CAPS (caps), false); + + caps_str = gst_caps_serialize (caps, GST_SERIALIZE_FLAG_NONE); + if (!caps_str) + return false; + + caps_size = strlen (caps_str) + 1; + + header.type = GstD3D11IpcPktType::CONFIG; + header.magic = GST_D3D11_IPC_MAGIC_NUMBER; + header.payload_size = sizeof (gint64) + caps_size; + + buf.resize (GST_D3D11_IPC_PKT_HEADER_SIZE + header.payload_size); + + ptr = &buf[0]; + + memcpy (ptr, &header, GST_D3D11_IPC_PKT_HEADER_SIZE); + ptr += GST_D3D11_IPC_PKT_HEADER_SIZE; + + memcpy (ptr, &adapter_luid, sizeof (gint64)); + ptr += sizeof (gint64); + + strcpy ((char *) ptr, caps_str); + g_free (caps_str); + + return true; +} + +bool +gst_d3d11_ipc_pkt_parse_config (std::vector < guint8 > &buf, + gint64 & adapter_luid, GstCaps ** caps) +{ + GstD3D11IpcPacketHeader header; + const guint8 *ptr; + std::string str; + + g_return_val_if_fail (buf.size () > + GST_D3D11_IPC_PKT_HEADER_SIZE + sizeof (gint64), false); + g_return_val_if_fail (caps, false); + + ptr = &buf[0]; + memcpy (&header, ptr, GST_D3D11_IPC_PKT_HEADER_SIZE); + + if (header.type != GstD3D11IpcPktType::CONFIG || + header.magic != GST_D3D11_IPC_MAGIC_NUMBER || + header.payload_size <= sizeof (gint64)) { + return false; + } + + ptr += GST_D3D11_IPC_PKT_HEADER_SIZE; + + memcpy (&adapter_luid, ptr, sizeof (gint64)); + ptr += sizeof (gint64); + + *caps = gst_caps_from_string ((const gchar *) ptr); + if (*caps == nullptr) + return false; + + return true; +} + +void +gst_d3d11_ipc_pkt_build_need_data (std::vector < guint8 > &buf) +{ + GstD3D11IpcPacketHeader header; + + header.type = GstD3D11IpcPktType::NEED_DATA; + header.magic = GST_D3D11_IPC_MAGIC_NUMBER; + header.payload_size = 0; + + buf.resize (GST_D3D11_IPC_PKT_HEADER_SIZE); + + memcpy (&buf[0], &header, GST_D3D11_IPC_PKT_HEADER_SIZE); +} + +/* *INDENT-OFF* */ +bool +gst_d3d11_ipc_pkt_build_have_data (std::vector < guint8 > &buf, + GstClockTime pts, const GstD3D11IpcMemLayout & layout, + const std::wstring & name, GstCaps * caps) +{ + GstD3D11IpcPacketHeader header; + guint8 *ptr; + guint name_size; + gchar *caps_str = nullptr; + guint caps_size = 1; + + name_size = (name.length () + 1) * sizeof (wchar_t); + + if (caps) { + caps_str = gst_caps_serialize (caps, GST_SERIALIZE_FLAG_NONE); + if (!caps_str) + return false; + + caps_size = strlen (caps_str) + 1; + } + + header.type = GstD3D11IpcPktType::HAVE_DATA; + header.magic = GST_D3D11_IPC_MAGIC_NUMBER; + header.payload_size = sizeof (GstClockTime) + sizeof (GstD3D11IpcMemLayout) + + name_size + caps_size; + + buf.resize (GST_D3D11_IPC_PKT_HEADER_SIZE + header.payload_size); + + ptr = &buf[0]; + memcpy (ptr, &header, GST_D3D11_IPC_PKT_HEADER_SIZE); + ptr += GST_D3D11_IPC_PKT_HEADER_SIZE; + + memcpy (ptr, &pts, sizeof (GstClockTime)); + ptr += sizeof (GstClockTime); + + memcpy (ptr, &layout, sizeof (GstD3D11IpcMemLayout)); + ptr += sizeof (GstD3D11IpcMemLayout); + + wcscpy ((wchar_t *) ptr, name.c_str ()); + ptr += name_size; + + if (caps) { + *ptr = 1; + ptr++; + + strcpy ((char *) ptr, caps_str); + } else { + *ptr = 0; + } + + g_free (caps_str); + + return true; +} +/* *INDENT-ON* */ + +bool +gst_d3d11_ipc_pkt_parse_have_data (const std::vector < guint8 > &buf, + GstClockTime & pts, GstD3D11IpcMemLayout & layout, + std::wstring & name, GstCaps ** caps) +{ + GstD3D11IpcPacketHeader header; + const guint8 *ptr; + std::string str; + + g_return_val_if_fail (buf.size () > + GST_D3D11_IPC_PKT_HEADER_SIZE + sizeof (GstClockTime) + + sizeof (GstD3D11IpcMemLayout), false); + g_return_val_if_fail (caps, false); + + ptr = &buf[0]; + memcpy (&header, ptr, GST_D3D11_IPC_PKT_HEADER_SIZE); + + if (header.type != GstD3D11IpcPktType::HAVE_DATA || + header.magic != GST_D3D11_IPC_MAGIC_NUMBER || + header.payload_size <= sizeof (GstClockTime) + + sizeof (GstD3D11IpcMemLayout)) { + return false; + } + ptr += GST_D3D11_IPC_PKT_HEADER_SIZE; + + memcpy (&pts, ptr, sizeof (GstClockTime)); + ptr += sizeof (GstClockTime); + + memcpy (&layout, ptr, sizeof (GstD3D11IpcMemLayout)); + ptr += sizeof (GstD3D11IpcMemLayout); + + name = (wchar_t *) ptr; + + ptr += (name.size () + 1) * sizeof (wchar_t); + + if (*ptr) { + ptr++; + + *caps = gst_caps_from_string ((const gchar *) ptr); + if (*caps == nullptr) + return false; + } + + return true; +} + +void +gst_d3d11_ipc_pkt_build_read_done (std::vector < guint8 > &buf) +{ + GstD3D11IpcPacketHeader header; + + header.type = GstD3D11IpcPktType::READ_DONE; + header.magic = GST_D3D11_IPC_MAGIC_NUMBER; + header.payload_size = 0; + + buf.resize (GST_D3D11_IPC_PKT_HEADER_SIZE); + + memcpy (&buf[0], &header, GST_D3D11_IPC_PKT_HEADER_SIZE); +} + +void +gst_d3d11_ipc_pkt_build_release_data (std::vector < guint8 > &buf, + const std::wstring & name) +{ + GstD3D11IpcPacketHeader header; + guint8 *ptr; + + header.type = GstD3D11IpcPktType::RELEASE_DATA; + header.magic = GST_D3D11_IPC_MAGIC_NUMBER; + header.payload_size = (name.size () + 1) * sizeof (wchar_t); + + buf.resize (GST_D3D11_IPC_PKT_HEADER_SIZE + header.payload_size); + + ptr = &buf[0]; + memcpy (ptr, &header, GST_D3D11_IPC_PKT_HEADER_SIZE); + ptr += GST_D3D11_IPC_PKT_HEADER_SIZE; + + wcscpy ((wchar_t *) ptr, name.c_str ()); +} + +bool +gst_d3d11_ipc_pkt_parse_release_data (std::vector < guint8 > &buf, + std::wstring & name) +{ + g_return_val_if_fail (buf.size () > GST_D3D11_IPC_PKT_HEADER_SIZE, false); + + name = (wchar_t *) (&buf[0] + GST_D3D11_IPC_PKT_HEADER_SIZE); + + return true; +} + +void +gst_d3d11_ipc_pkt_build_eos (std::vector < guint8 > &buf) +{ + GstD3D11IpcPacketHeader header; + + header.type = GstD3D11IpcPktType::EOS; + header.magic = GST_D3D11_IPC_MAGIC_NUMBER; + header.payload_size = 0; + + buf.resize (GST_D3D11_IPC_PKT_HEADER_SIZE); + + memcpy (&buf[0], &header, GST_D3D11_IPC_PKT_HEADER_SIZE); +} + +void +gst_d3d11_ipc_pkt_build_fin (std::vector < guint8 > &buf) +{ + GstD3D11IpcPacketHeader header; + + header.type = GstD3D11IpcPktType::FIN; + header.magic = GST_D3D11_IPC_MAGIC_NUMBER; + header.payload_size = 0; + + buf.resize (GST_D3D11_IPC_PKT_HEADER_SIZE); + + memcpy (&buf[0], &header, GST_D3D11_IPC_PKT_HEADER_SIZE); +} + +bool +gst_d3d11_ipc_clock_is_system (GstClock * clock) +{ + GstClockType clock_type = GST_CLOCK_TYPE_MONOTONIC; + GstClock *mclock; + + if (G_OBJECT_TYPE (clock) != GST_TYPE_SYSTEM_CLOCK) + return false; + + g_object_get (clock, "clock-type", &clock_type, nullptr); + if (clock_type != GST_CLOCK_TYPE_MONOTONIC) + return false; + + mclock = gst_clock_get_master (clock); + if (!mclock) + return true; + + gst_object_unref (mclock); + return false; +} + +GstClockTime +gst_d3d11_ipc_clock_now (void) +{ + static LARGE_INTEGER freq; + LARGE_INTEGER cur_time; + + GST_D3D11_CALL_ONCE_BEGIN { + QueryPerformanceFrequency (&freq); + } GST_D3D11_CALL_ONCE_END; + + QueryPerformanceCounter (&cur_time); + return gst_util_uint64_scale (cur_time.QuadPart, GST_SECOND, freq.QuadPart); +} + +std::string +gst_d3d11_ipc_wstring_to_string (const std::wstring & str) +{ + std::wstring_convert < std::codecvt_utf8 < wchar_t >>conv; + return conv.to_bytes (str); +} + +std::wstring +gst_d3d11_ipc_string_to_wstring (const std::string & str) +{ + std::wstring_convert < std::codecvt_utf8 < wchar_t >>conv; + return conv.from_bytes (str); +} + +/* *INDENT-OFF* */ +static inline void rtrim(std::string &s) { + s.erase (std::find_if (s.rbegin(), s.rend(), + [](unsigned char ch) { + return !std::isspace (ch); + }).base (), s.end ()); +} +/* *INDENT-ON* */ + +std::string +gst_d3d11_ipc_win32_error_to_string (guint err) +{ + wchar_t buffer[1024]; + + if (!FormatMessageW (FORMAT_MESSAGE_IGNORE_INSERTS | + FORMAT_MESSAGE_FROM_SYSTEM, nullptr, err, 0, buffer, 1024, nullptr)) { + return std::string (""); + } + + std::string ret = gst_d3d11_ipc_wstring_to_string (buffer); + rtrim (ret); + + return ret; +} + +gint64 +gst_d3d11_ipc_get_shared_resource_token (void) +{ + static gint64 token = 0; + + GST_D3D11_CALL_ONCE_BEGIN { + token = gst_d3d11_create_user_token (); + } GST_D3D11_CALL_ONCE_END; + + return token; +} + +static DWORD +gst_d3d11_ipc_get_pid (void) +{ + static DWORD pid = 0; + + GST_D3D11_CALL_ONCE_BEGIN { + pid = GetCurrentProcessId (); + } GST_D3D11_CALL_ONCE_END; + + return pid; +} + +std::wstring +gst_d3d11_ipc_get_resource_prefix (void) +{ + static ULONG global_index = 0; + + std::wstring prefix = std::wstring (L"Local\\gst.d3d11.ipc.") + + std::to_wstring (gst_d3d11_ipc_get_pid ()) + std::wstring (L".") + + std::to_wstring (InterlockedIncrement (&global_index)) + + std::wstring (L"."); + + return prefix; +} diff --git a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipc.h b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipc.h new file mode 100644 index 0000000000..ca72017da3 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipc.h @@ -0,0 +1,161 @@ +/* GStreamer + * Copyright (C) 2023 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +/* + * Communication Sequence + * + * +--------+ +--------+ + * | client | | server | + * +--------+ +--------+ + * | | + * | | + * |<---------- CONFIG ------------+ + * | | + * +--------- NEED-DATA ---------->| + * | +-------+ + * | | Export + * | | D3D11 memory + * | |<------+ + * |<-------- HAVE-DATA -----------+ + * +--------+ | + * Import | | + * D3D11 memory | | + * +------->+ | + * +--------- READ-DONE ---------->| + * +--------+ | + * Release | | + * D3D11 memory | | + * +------->| | + * +-------- RELEASE-DATA -------->| + * | | + * +--------- NEED-DATA ---------->| + * | | + * |<----------- EOS --------------+ + * +--------+ | + * Cleanup all | | + * shared resources | | + * +------->| | + * +------------ FIN ------------->| + */ + +enum class GstD3D11IpcPktType : guint8 +{ + UNKNOWN, + CONFIG, + NEED_DATA, + HAVE_DATA, + READ_DONE, + RELEASE_DATA, + EOS, + FIN, +}; + +#pragma pack(push, 1) +struct GstD3D11IpcPacketHeader +{ + GstD3D11IpcPktType type; + guint32 payload_size; + guint32 magic; +}; + +struct GstD3D11IpcMemLayout +{ + guint32 size; + guint32 pitch; + guint32 offset[4]; +}; +#pragma pack(pop) + +struct GstD3D11IpcHandleData +{ + ~GstD3D11IpcHandleData () + { + if (handle) + CloseHandle (handle); + } + + HANDLE handle = nullptr; + std::wstring name; +}; + +constexpr guint GST_D3D11_IPC_PKT_HEADER_SIZE = sizeof (GstD3D11IpcPacketHeader); + +#define GST_D3D11_IPC_FORMATS \ + "{ RGBA64_LE, RGB10A2_LE, BGRA, RGBA, BGRx, RGBx, VUYA, NV12, NV21, " \ + "P010_10LE, P012_LE, P016_LE }" + +bool gst_d3d11_ipc_pkt_identify (std::vector & buf, + GstD3D11IpcPacketHeader & header); + +bool gst_d3d11_ipc_pkt_build_config (std::vector & buf, + gint64 adapter_luid, + GstCaps * caps); + +bool gst_d3d11_ipc_pkt_parse_config (std::vector & buf, + gint64 & adapter_luid, + GstCaps ** caps); + +void gst_d3d11_ipc_pkt_build_need_data (std::vector & buf); + +bool gst_d3d11_ipc_pkt_build_have_data (std::vector & buf, + GstClockTime pts, + const GstD3D11IpcMemLayout & layout, + const std::wstring & name, + GstCaps * caps); + +bool gst_d3d11_ipc_pkt_parse_have_data (const std::vector & buf, + GstClockTime & pts, + GstD3D11IpcMemLayout & layout, + std::wstring & name, + GstCaps ** caps); + +void gst_d3d11_ipc_pkt_build_read_done (std::vector & buf); + +void gst_d3d11_ipc_pkt_build_release_data (std::vector & buf, + const std::wstring & name); + +bool gst_d3d11_ipc_pkt_parse_release_data (std::vector & buf, + std::wstring & name); + +void gst_d3d11_ipc_pkt_build_eos (std::vector & buf); + +void gst_d3d11_ipc_pkt_build_fin (std::vector & buf); + +bool gst_d3d11_ipc_clock_is_system (GstClock * clock); + +GstClockTime gst_d3d11_ipc_clock_now (void); + +std::string gst_d3d11_ipc_wstring_to_string (const std::wstring & str); + +std::wstring gst_d3d11_ipc_string_to_wstring (const std::string & str); + +std::string gst_d3d11_ipc_win32_error_to_string (guint err); + +gint64 gst_d3d11_ipc_get_shared_resource_token (void); + +std::wstring gst_d3d11_ipc_get_resource_prefix (void); diff --git a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipcclient.cpp b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipcclient.cpp new file mode 100644 index 0000000000..646476346d --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipcclient.cpp @@ -0,0 +1,1098 @@ +/* GStreamer + * Copyright (C) 2023 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstd3d11ipcclient.h" +#include +#include +#include +#include +#include +#include +#include + +/* *INDENT-OFF* */ +using namespace Microsoft::WRL; +/* *INDENT-ON* */ + +GST_DEBUG_CATEGORY_STATIC (gst_d3d11_ipc_client_debug); +#define GST_CAT_DEFAULT gst_d3d11_ipc_client_debug + +GType +gst_d3d11_ipc_io_mode_get_type (void) +{ + static GType type = 0; + static const GEnumValue io_modes[] = { + {GST_D3D11_IPC_IO_COPY, "Copy remote memory", "copy"}, + {GST_D3D11_IPC_IO_IMPORT, "Import remote memory", "import"}, + {0, nullptr, nullptr} + }; + + GST_D3D11_CALL_ONCE_BEGIN { + type = g_enum_register_static ("GstD3D11IpcIOMode", io_modes); + } GST_D3D11_CALL_ONCE_END; + + return type; +} + +/* *INDENT-OFF* */ +struct GstD3D11IpcClientConn : public OVERLAPPED +{ + GstD3D11IpcClientConn (GstD3D11IpcClient * client, HANDLE pipe_handle) + : client (client), pipe (pipe_handle) + { + OVERLAPPED *parent = static_cast (this); + parent->Internal = 0; + parent->InternalHigh = 0; + parent->Offset = 0; + parent->OffsetHigh = 0; + + client_msg.resize (GST_D3D11_IPC_PKT_HEADER_SIZE); + server_msg.resize (GST_D3D11_IPC_PKT_HEADER_SIZE); + } + + ~GstD3D11IpcClientConn () + { + if (pipe != INVALID_HANDLE_VALUE) { + CancelIo (pipe); + CloseHandle (pipe); + } + } + + GstD3D11IpcClient *client; + + HANDLE pipe = INVALID_HANDLE_VALUE; + + GstD3D11IpcPktType type; + std::vector client_msg; + std::vector server_msg; +}; + +struct GstD3D11IpcImportData +{ + ~GstD3D11IpcImportData () + { + auto dump = gst_d3d11_ipc_wstring_to_string (name); + GST_LOG_OBJECT (client, "Release handle \"%s\"", dump.c_str ()); + gst_object_unref (client); + } + + GstD3D11IpcClient *client; + ComPtr texture; + ComPtr mutex; + GstD3D11IpcMemLayout layout; + std::wstring name; +}; + +struct GstD3D11IpcReleaseData +{ + GstD3D11IpcClient *self; + std::shared_ptr imported; +}; + +struct GstD3D11IpcClientPrivate +{ + GstD3D11IpcClientPrivate () + { + wakeup_event = CreateEvent (nullptr, FALSE, FALSE, nullptr); + cancellable = CreateEvent (nullptr, TRUE, FALSE, nullptr); + + shutdown = false; + io_pending = true; + } + + ~GstD3D11IpcClientPrivate () + { + gst_clear_caps (&caps); + if (pool) { + gst_buffer_pool_set_active (pool, FALSE); + gst_object_unref (pool); + } + + gst_clear_object (&device); + + CloseHandle (wakeup_event); + CloseHandle (cancellable); + } + + std::string address; + GstD3D11IpcIOMode io_mode = GST_D3D11_IPC_IO_COPY; + GstClockTime timeout; + HANDLE wakeup_event; + HANDLE cancellable; + std::mutex lock; + std::condition_variable cond; + GstD3D11Device *device = nullptr; + GstCaps *caps = nullptr; + GstBufferPool *pool = nullptr; + GstVideoInfo info; + bool server_eos = false; + bool flushing = false; + bool aborted = false; + bool sent_fin = false; + std::atomic shutdown; + std::atomic io_pending; + GThread *loop_thread = nullptr; + std::queue samples; + std::shared_ptr conn; + std::queue unused_data; + std::vector> imported; +}; +/* *INDENT-ON* */ + +struct _GstD3D11IpcClient +{ + GstObject parent; + + GstD3D11IpcClientPrivate *priv; +}; + +static void gst_d3d11_ipc_client_dispose (GObject * object); +static void gst_d3d11_ipc_client_finalize (GObject * object); +static void gst_d3d11_ipc_client_continue (GstD3D11IpcClient * self); +static void gst_d3d11_ipc_client_send_msg (GstD3D11IpcClient * self); + +#define gst_d3d11_ipc_client_parent_class parent_class +G_DEFINE_TYPE (GstD3D11IpcClient, gst_d3d11_ipc_client, GST_TYPE_OBJECT); + +static void +gst_d3d11_ipc_client_class_init (GstD3D11IpcClientClass * klass) +{ + GObjectClass *object_class = G_OBJECT_CLASS (klass); + + object_class->dispose = gst_d3d11_ipc_client_dispose; + object_class->finalize = gst_d3d11_ipc_client_finalize; + + GST_DEBUG_CATEGORY_INIT (gst_d3d11_ipc_client_debug, "d3d11ipcclient", + 0, "d3d11ipcclient"); +} + +static void +gst_d3d11_ipc_client_init (GstD3D11IpcClient * self) +{ + self->priv = new GstD3D11IpcClientPrivate (); +} + +static void +gst_d3d11_ipc_client_dispose (GObject * object) +{ + GstD3D11IpcClient *self = GST_D3D11_IPC_CLIENT (object); + GstD3D11IpcClientPrivate *priv = self->priv; + + GST_DEBUG_OBJECT (self, "dispose"); + + SetEvent (priv->cancellable); + + g_clear_pointer (&priv->loop_thread, g_thread_join); + + G_OBJECT_CLASS (parent_class)->dispose (object); +} + +static void +gst_d3d11_ipc_client_finalize (GObject * object) +{ + GstD3D11IpcClient *self = GST_D3D11_IPC_CLIENT (object); + + GST_DEBUG_OBJECT (self, "finalize"); + + delete self->priv; + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static void +gst_d3d11_ipc_client_abort (GstD3D11IpcClient * self) +{ + GstD3D11IpcClientPrivate *priv = self->priv; + std::lock_guard < std::mutex > lk (priv->lock); + priv->aborted = true; + priv->cond.notify_all (); +} + +static bool +gst_d3d11_client_update_caps (GstD3D11IpcClient * self, GstCaps * caps) +{ + GstD3D11IpcClientPrivate *priv = self->priv; + + if (!caps) + return true; + + gst_clear_caps (&priv->caps); + priv->caps = caps; + + if (priv->pool) { + gst_buffer_pool_set_active (priv->pool, FALSE); + gst_clear_object (&priv->pool); + } + + if (!gst_video_info_from_caps (&priv->info, caps)) { + GST_ERROR_OBJECT (self, "Invalid caps"); + return false; + } + + if (priv->io_mode == GST_D3D11_IPC_IO_COPY) { + GstStructure *config; + GstD3D11AllocationParams *params; + guint bind_flags = 0; + GstD3D11Format device_format; + + if (!gst_d3d11_device_get_format (priv->device, + GST_VIDEO_INFO_FORMAT (&priv->info), &device_format)) { + GST_ERROR_OBJECT (self, "Couldn't get device format"); + return false; + } + + if ((device_format.format_support[0] & + (guint) D3D11_FORMAT_SUPPORT_SHADER_SAMPLE) != 0) { + bind_flags |= D3D11_BIND_SHADER_RESOURCE; + } + + if ((device_format.format_support[0] & + (guint) D3D11_FORMAT_SUPPORT_RENDER_TARGET) != 0) { + bind_flags |= D3D11_BIND_RENDER_TARGET; + } + + priv->pool = gst_d3d11_buffer_pool_new (priv->device); + config = gst_buffer_pool_get_config (priv->pool); + + gst_buffer_pool_config_add_option (config, + GST_BUFFER_POOL_OPTION_VIDEO_META); + + gst_buffer_pool_config_set_params (config, priv->caps, + GST_VIDEO_INFO_SIZE (&priv->info), 0, 0); + + params = gst_d3d11_allocation_params_new (priv->device, &priv->info, + GST_D3D11_ALLOCATION_FLAG_DEFAULT, bind_flags, 0); + + gst_buffer_pool_config_set_d3d11_allocation_params (config, params); + gst_d3d11_allocation_params_free (params); + + if (!gst_buffer_pool_set_config (priv->pool, config)) { + GST_ERROR_OBJECT (self, "Couldn't set pool config"); + gst_clear_object (&priv->pool); + return false; + } + + if (!gst_buffer_pool_set_active (priv->pool, TRUE)) { + GST_ERROR_OBJECT (self, "Couldn't active pool"); + gst_clear_object (&priv->pool); + return false; + } + } + + return true; +} + +static bool +gst_d3d11_ipc_client_config_data (GstD3D11IpcClient * self) +{ + GstD3D11IpcClientPrivate *priv = self->priv; + gint64 prev_luid, luid; + GstCaps *caps = nullptr; + auto conn = priv->conn; + std::lock_guard < std::mutex > lk (priv->lock); + + g_object_get (priv->device, "adapter-luid", &prev_luid, nullptr); + + if (!gst_d3d11_ipc_pkt_parse_config (conn->server_msg, luid, &caps)) { + GST_ERROR_OBJECT (self, "Couldn't parse CONFIG-DATA"); + return false; + } + + if (prev_luid != luid) { + GstD3D11Device *device = gst_d3d11_device_new_for_adapter_luid (luid, + D3D11_CREATE_DEVICE_BGRA_SUPPORT); + if (!device) { + GST_ERROR_OBJECT (self, "Couldn't create device"); + return false; + } + + gst_object_unref (priv->device); + priv->device = device; + } + + if (!gst_d3d11_client_update_caps (self, caps)) + return false; + + priv->cond.notify_all (); + + return true; +} + +static void +gst_d3d11_ipc_client_release_imported_data (GstD3D11IpcReleaseData * data) +{ + GstD3D11IpcClient *self = data->self; + GstD3D11IpcClientPrivate *priv = self->priv; + auto name = data->imported->name; + auto handle_dump = gst_d3d11_ipc_wstring_to_string (name); + + GST_LOG_OBJECT (self, "Releasing data \"%s\"", handle_dump.c_str ()); + + data->imported = nullptr; + + priv->lock.lock (); + priv->unused_data.push (name); + priv->lock.unlock (); + + SetEvent (priv->wakeup_event); + + gst_object_unref (data->self); + + delete data; +} + +static bool +gst_d3d11_ipc_client_have_data (GstD3D11IpcClient * self) +{ + GstD3D11IpcClientPrivate *priv = self->priv; + GstBuffer *buffer; + GstMemory *mem; + GstSample *sample; + GstClockTime pts; + GstCaps *caps = nullptr; + GstD3D11IpcMemLayout layout; + std::shared_ptr < GstD3D11IpcImportData > import_data; + std::unique_lock < std::mutex > lk (priv->lock); + std::wstring name; + auto conn = priv->conn; + ComPtr < ID3D11Texture2D > texture; + HRESULT hr; + + if (!gst_d3d11_ipc_pkt_parse_have_data (conn->server_msg, + pts, layout, name, &caps)) { + GST_ERROR_OBJECT (self, "Couldn't parse HAVE-DATA packet"); + return false; + } + + if (!gst_d3d11_client_update_caps (self, caps)) + return false; + + auto handle_dump = gst_d3d11_ipc_wstring_to_string (name); + GST_LOG_OBJECT (self, "Importing handle \"%s\"", handle_dump.c_str ()); + + /* Check if this memory handle was imported already */ + /* *INDENT-OFF* */ + if (priv->io_mode == GST_D3D11_IPC_IO_IMPORT && !priv->imported.empty ()) { + GST_LOG_OBJECT (self, "Checking already imported handles, size %" + G_GSIZE_FORMAT, priv->imported.size ()); + for (auto it = priv->imported.begin (); it != priv->imported.end (); ) { + auto data = it->lock (); + if (!data) { + it = priv->imported.erase (it); + } else { + if (data->name == name) { + GST_DEBUG_OBJECT (self, "Already imported handle"); + import_data = data; + break; + } + + it++; + } + } + } + /* *INDENT-ON* */ + + if (!import_data) { + ID3D11Device *device = gst_d3d11_device_get_device_handle (priv->device); + ComPtr < ID3D11Device1 > device1; + ComPtr < IDXGIKeyedMutex > mutex; + + hr = device->QueryInterface (IID_PPV_ARGS (&device1)); + if (!gst_d3d11_result (hr, priv->device)) { + GST_ERROR_OBJECT (self, "ID3D11Device1 interface is not available"); + return false; + } + + hr = device1->OpenSharedResourceByName (name.c_str (), + DXGI_SHARED_RESOURCE_READ, IID_PPV_ARGS (&texture)); + if (!gst_d3d11_result (hr, priv->device)) { + GST_ERROR_OBJECT (self, "Couldn't open resource"); + return false; + } + + hr = texture->QueryInterface (IID_PPV_ARGS (&mutex)); + if (!gst_d3d11_result (hr, priv->device)) { + GST_ERROR_OBJECT (self, "couldn't get keyed mutex interface"); + return false; + } + + import_data = std::make_shared < GstD3D11IpcImportData > (); + import_data->client = (GstD3D11IpcClient *) gst_object_ref (self); + import_data->texture = texture; + import_data->mutex = mutex; + import_data->layout = layout; + import_data->name = name; + + if (priv->io_mode == GST_D3D11_IPC_IO_IMPORT) + priv->imported.push_back (import_data); + } else { + texture = import_data->texture; + } + + if (priv->io_mode == GST_D3D11_IPC_IO_COPY) { + ID3D11DeviceContext *context = + gst_d3d11_device_get_device_context_handle (priv->device); + ID3D11Texture2D *dst_texture; + D3D11_BOX src_box = { 0, }; + D3D11_TEXTURE2D_DESC dst_desc, src_desc; + GstMapInfo info; + + hr = import_data->mutex->AcquireSync (0, INFINITE); + if (hr != S_OK) { + GST_ERROR_OBJECT (self, "Couldn't acquire sync"); + return false; + } + + gst_buffer_pool_acquire_buffer (priv->pool, &buffer, nullptr); + mem = gst_buffer_peek_memory (buffer, 0); + + gst_memory_map (mem, &info, (GstMapFlags) (GST_MAP_WRITE | GST_MAP_D3D11)); + + dst_texture = (ID3D11Texture2D *) info.data; + dst_texture->GetDesc (&dst_desc); + + texture->GetDesc (&src_desc); + + src_box.left = 0; + src_box.top = 0; + src_box.front = 0; + src_box.back = 1; + src_box.right = MIN (src_desc.Width, dst_desc.Width); + src_box.bottom = MIN (src_desc.Height, dst_desc.Height); + + context->CopySubresourceRegion (dst_texture, 0, 0, 0, 0, + texture.Get (), 0, &src_box); + + import_data->mutex->ReleaseSync (0); + + gst_memory_unmap (mem, &info); + + priv->unused_data.push (name); + } else { + GstMemory *mem; + gint stride[GST_VIDEO_MAX_PLANES]; + gsize offset[GST_VIDEO_MAX_PLANES]; + + for (guint i = 0; i < GST_VIDEO_MAX_PLANES; i++) { + stride[i] = import_data->layout.pitch; + offset[i] = import_data->layout.offset[i]; + } + + auto data = new GstD3D11IpcReleaseData (); + data->self = (GstD3D11IpcClient *) gst_object_ref (self); + data->imported = import_data; + + mem = gst_d3d11_allocator_alloc_wrapped (nullptr, priv->device, + texture.Get (), import_data->layout.size, data, + (GDestroyNotify) gst_d3d11_ipc_client_release_imported_data); + GST_MINI_OBJECT_FLAG_SET (mem, GST_MEMORY_FLAG_READONLY); + + buffer = gst_buffer_new (); + gst_buffer_append_memory (buffer, mem); + + gst_buffer_add_video_meta_full (buffer, GST_VIDEO_FRAME_FLAG_NONE, + GST_VIDEO_INFO_FORMAT (&priv->info), GST_VIDEO_INFO_WIDTH (&priv->info), + GST_VIDEO_INFO_HEIGHT (&priv->info), + GST_VIDEO_INFO_N_PLANES (&priv->info), offset, stride); + } + + GST_BUFFER_PTS (buffer) = pts; + GST_BUFFER_DTS (buffer) = GST_CLOCK_TIME_NONE; + GST_BUFFER_DURATION (buffer) = GST_CLOCK_TIME_NONE; + + sample = gst_sample_new (buffer, priv->caps, nullptr, nullptr); + gst_buffer_unref (buffer); + + /* Drops too old samples */ + std::queue < GstSample * >drop_queue; + while (priv->samples.size () > 2) { + drop_queue.push (priv->samples.front ()); + priv->samples.pop (); + } + + priv->samples.push (sample); + priv->cond.notify_all (); + lk.unlock (); + + import_data = nullptr; + while (!drop_queue.empty ()) { + auto old = drop_queue.front (); + gst_sample_unref (old); + drop_queue.pop (); + } + + return true; +} + +static void +gst_d3d11_ipc_client_wait_msg_finish (GstD3D11IpcClient * client) +{ + GstD3D11IpcClientPrivate *priv = client->priv; + GstD3D11IpcPacketHeader header; + auto conn = priv->conn; + + if (!gst_d3d11_ipc_pkt_identify (conn->server_msg, header)) { + GST_ERROR_OBJECT (client, "Broken header"); + gst_d3d11_ipc_client_abort (client); + return; + } + + switch (header.type) { + case GstD3D11IpcPktType::CONFIG: + GST_LOG_OBJECT (client, "Got CONFIG"); + if (!gst_d3d11_ipc_client_config_data (client)) { + gst_d3d11_ipc_client_abort (client); + return; + } + + gst_d3d11_ipc_client_continue (client); + break; + case GstD3D11IpcPktType::HAVE_DATA: + GST_LOG_OBJECT (client, "Got HAVE-DATA"); + gst_d3d11_device_lock (priv->device); + if (!gst_d3d11_ipc_client_have_data (client)) { + gst_d3d11_device_unlock (priv->device); + gst_d3d11_ipc_client_abort (client); + return; + } + + gst_d3d11_device_unlock (priv->device); + + GST_LOG_OBJECT (client, "Sending READ-DONE"); + gst_d3d11_ipc_pkt_build_read_done (conn->client_msg); + conn->type = GstD3D11IpcPktType::READ_DONE; + gst_d3d11_ipc_client_send_msg (client); + break; + case GstD3D11IpcPktType::EOS: + GST_DEBUG_OBJECT (client, "Got EOS"); + priv->server_eos = true; + priv->lock.lock (); + priv->cond.notify_all (); + priv->lock.unlock (); + gst_d3d11_ipc_client_continue (client); + break; + default: + GST_WARNING_OBJECT (client, "Unexpected packet type"); + gst_d3d11_ipc_client_abort (client); + break; + } +} + +static void WINAPI +gst_d3d11_ipc_client_payload_finish (DWORD error_code, DWORD size, + OVERLAPPED * overlap) +{ + GstD3D11IpcClientConn *conn = + static_cast < GstD3D11IpcClientConn * >(overlap); + GstD3D11IpcClient *self = conn->client; + + if (error_code != ERROR_SUCCESS) { + auto err = gst_d3d11_ipc_win32_error_to_string (error_code); + GST_WARNING_OBJECT (self, "ReadFileEx callback failed with 0x%x (%s)", + (guint) error_code, err.c_str ()); + gst_d3d11_ipc_client_abort (self); + } + + gst_d3d11_ipc_client_wait_msg_finish (self); +} + +static void WINAPI +gst_d3d11_ipc_client_win32_wait_header_finish (DWORD error_code, DWORD size, + OVERLAPPED * overlap) +{ + GstD3D11IpcClientConn *conn = + static_cast < GstD3D11IpcClientConn * >(overlap); + GstD3D11IpcClient *self = conn->client; + GstD3D11IpcPacketHeader header; + + if (error_code != ERROR_SUCCESS) { + auto err = gst_d3d11_ipc_win32_error_to_string (error_code); + GST_WARNING_OBJECT (self, "ReadFileEx callback failed with 0x%x (%s)", + (guint) error_code, err.c_str ()); + gst_d3d11_ipc_client_abort (self); + return; + } + + if (!gst_d3d11_ipc_pkt_identify (conn->server_msg, header)) { + GST_ERROR_OBJECT (self, "Broken header"); + gst_d3d11_ipc_client_abort (self); + return; + } + + if (header.payload_size == 0) { + gst_d3d11_ipc_client_wait_msg_finish (self); + return; + } + + GST_LOG_OBJECT (self, "Reading payload"); + + if (!ReadFileEx (conn->pipe, &conn->server_msg[0] + + GST_D3D11_IPC_PKT_HEADER_SIZE, header.payload_size, conn, + gst_d3d11_ipc_client_payload_finish)) { + guint last_err = GetLastError (); + auto err = gst_d3d11_ipc_win32_error_to_string (last_err); + GST_WARNING_OBJECT (self, "ReadFileEx failed with 0x%x (%s)", + last_err, err.c_str ()); + gst_d3d11_ipc_client_abort (self); + } +} + +static void +gst_d3d11_ipc_client_wait_msg (GstD3D11IpcClient * self) +{ + GstD3D11IpcClientPrivate *priv = self->priv; + auto conn = priv->conn; + priv->io_pending = true; + + if (!ReadFileEx (conn->pipe, &conn->server_msg[0], + GST_D3D11_IPC_PKT_HEADER_SIZE, conn.get (), + gst_d3d11_ipc_client_win32_wait_header_finish)) { + guint last_err = GetLastError (); + auto err = gst_d3d11_ipc_win32_error_to_string (last_err); + GST_WARNING_OBJECT (self, "ReadFileEx failed with 0x%x (%s)", + last_err, err.c_str ()); + gst_d3d11_ipc_client_abort (self); + } +} + +static void WINAPI +gst_cuda_ipc_client_win32_send_msg_finish (DWORD error_code, DWORD size, + OVERLAPPED * overlap) +{ + GstD3D11IpcClientConn *conn = + static_cast < GstD3D11IpcClientConn * >(overlap); + GstD3D11IpcClient *self = conn->client; + + if (error_code != ERROR_SUCCESS) { + auto err = gst_d3d11_ipc_win32_error_to_string (error_code); + GST_WARNING_OBJECT (self, "WriteFileEx callback failed with 0x%x (%s)", + (guint) error_code, err.c_str ()); + gst_d3d11_ipc_client_abort (self); + return; + } + + switch (conn->type) { + case GstD3D11IpcPktType::NEED_DATA: + GST_LOG_OBJECT (self, "Sent NEED-DATA"); + gst_d3d11_ipc_client_wait_msg (self); + break; + case GstD3D11IpcPktType::READ_DONE: + GST_LOG_OBJECT (self, "Sent READ-DONE"); + gst_d3d11_ipc_client_continue (self); + break; + case GstD3D11IpcPktType::RELEASE_DATA: + GST_LOG_OBJECT (self, "Sent RELEASE-DATA"); + gst_d3d11_ipc_client_continue (self); + break; + case GstD3D11IpcPktType::FIN: + GST_DEBUG_OBJECT (self, "Sent FIN"); + gst_d3d11_ipc_client_abort (self); + break; + default: + GST_ERROR_OBJECT (self, "Unexpected msg type"); + gst_d3d11_ipc_client_abort (self); + break; + } +} + +static void +gst_d3d11_ipc_client_send_msg (GstD3D11IpcClient * self) +{ + GstD3D11IpcClientPrivate *priv = self->priv; + + auto conn = priv->conn; + priv->io_pending = true; + + if (!WriteFileEx (conn->pipe, &conn->client_msg[0], + conn->client_msg.size (), conn.get (), + gst_cuda_ipc_client_win32_send_msg_finish)) { + guint last_err = GetLastError (); + auto err = gst_d3d11_ipc_win32_error_to_string (last_err); + GST_WARNING_OBJECT (self, "WriteFileEx failed with 0x%x (%s)", + last_err, err.c_str ()); + gst_d3d11_ipc_client_abort (self); + } +} + +static void +gst_d3d11_ipc_client_run_gc (GstD3D11IpcClient * self) +{ + GstD3D11IpcClientPrivate *priv = self->priv; + + for (auto it = priv->imported.begin (); it != priv->imported.end ();) { + auto data = it->lock (); + if (!data) { + it = priv->imported.erase (it); + } else { + it++; + } + } +} + +static void +gst_d3d11_ipc_client_continue (GstD3D11IpcClient * self) +{ + GstD3D11IpcClientPrivate *priv = self->priv; + std::unique_lock < std::mutex > lk (priv->lock); + auto conn = priv->conn; + + if (!conn) { + GST_WARNING_OBJECT (self, "No connection was made"); + priv->aborted = true; + priv->cond.notify_all (); + return; + } + + if (priv->aborted) { + priv->cond.notify_all (); + GST_DEBUG_OBJECT (self, "Operation was aborted"); + return; + } + + if (!priv->unused_data.empty ()) { + auto name = priv->unused_data.front (); + priv->unused_data.pop (); + + auto handle_dump = gst_d3d11_ipc_wstring_to_string (name); + GST_LOG_OBJECT (self, "Sending RELEASE-DATA \"%s\"", handle_dump.c_str ()); + + gst_d3d11_ipc_pkt_build_release_data (conn->client_msg, name); + conn->type = GstD3D11IpcPktType::RELEASE_DATA; + lk.unlock (); + + gst_d3d11_ipc_client_send_msg (self); + return; + } + + if (priv->shutdown) { + auto drop_queue = priv->samples; + while (!priv->samples.empty ()) + priv->samples.pop (); + lk.unlock (); + + while (!drop_queue.empty ()) { + auto sample = drop_queue.front (); + gst_sample_unref (sample); + drop_queue.pop (); + } + lk.lock (); + } + + if (priv->server_eos || priv->shutdown) { + gst_d3d11_ipc_client_run_gc (self); + + GST_DEBUG_OBJECT (self, "Remaining imported memory %" G_GSIZE_FORMAT, + priv->imported.size ()); + + if (priv->imported.empty ()) { + GST_DEBUG_OBJECT (self, "Drained"); + if (priv->sent_fin) { + priv->aborted = true; + priv->cond.notify_all (); + } else { + lk.unlock (); + + priv->sent_fin = true; + gst_d3d11_ipc_pkt_build_fin (conn->client_msg); + conn->type = GstD3D11IpcPktType::FIN; + + GST_DEBUG_OBJECT (self, "Sending FIN"); + gst_d3d11_ipc_client_send_msg (self); + return; + } + } else { + priv->io_pending = false; + } + return; + } + + lk.unlock (); + + gst_d3d11_ipc_pkt_build_need_data (conn->client_msg); + conn->type = GstD3D11IpcPktType::NEED_DATA; + + GST_LOG_OBJECT (self, "Sending NEED-DATA"); + gst_d3d11_ipc_client_send_msg (self); +} + +static gpointer +gst_d3d11_ipc_client_loop_thread_func (GstD3D11IpcClient * self) +{ + GstD3D11IpcClientPrivate *priv = self->priv; + DWORD mode = PIPE_READMODE_MESSAGE; + guint wait_ret; + HANDLE pipe = INVALID_HANDLE_VALUE; + GstClockTime start_time = gst_d3d11_ipc_clock_now (); + HANDLE waitables[] = { priv->cancellable, priv->wakeup_event }; + std::wstring address = gst_d3d11_ipc_string_to_wstring (priv->address); + +#if (_WIN32_WINNT >= _WIN32_WINNT_WIN8) + CREATEFILE2_EXTENDED_PARAMETERS params; + memset (¶ms, 0, sizeof (CREATEFILE2_EXTENDED_PARAMETERS)); + params.dwSize = sizeof (CREATEFILE2_EXTENDED_PARAMETERS); + params.dwFileAttributes = 0; + params.dwFileFlags = FILE_FLAG_OVERLAPPED; + params.dwSecurityQosFlags = SECURITY_IMPERSONATION; +#endif + + GST_DEBUG_OBJECT (self, "Starting loop thread"); + + std::unique_lock < std::mutex > lk (priv->lock); + do { + GstClockTime diff; + + if (priv->flushing) { + GST_DEBUG_OBJECT (self, "We are flushing"); + priv->aborted = true; + priv->cond.notify_all (); + goto out; + } +#if (_WIN32_WINNT >= _WIN32_WINNT_WIN8) + pipe = CreateFile2 (address.c_str (), GENERIC_READ | GENERIC_WRITE, 0, + OPEN_EXISTING, ¶ms); +#else + pipe = CreateFileW (address.c_str (), + GENERIC_READ | GENERIC_WRITE, 0, nullptr, OPEN_EXISTING, + FILE_FLAG_OVERLAPPED, nullptr); +#endif + + if (pipe != INVALID_HANDLE_VALUE) + break; + + if (priv->timeout > 0) { + diff = gst_d3d11_ipc_clock_now () - start_time; + if (diff > priv->timeout) { + GST_WARNING_OBJECT (self, "Timeout"); + priv->aborted = true; + priv->cond.notify_all (); + goto out; + } + } + + /* Retry per 100ms */ + GST_DEBUG_OBJECT (self, "Sleep for next retry"); + priv->cond.wait_for (lk, std::chrono::milliseconds (100)); + } while (true); + + if (!SetNamedPipeHandleState (pipe, &mode, nullptr, nullptr)) { + guint last_err = GetLastError (); + auto err = gst_d3d11_ipc_win32_error_to_string (last_err); + GST_WARNING_OBJECT (self, "SetNamedPipeHandleState failed with 0x%x (%s)", + last_err, err.c_str ()); + + CloseHandle (pipe); + priv->aborted = true; + priv->cond.notify_all (); + goto out; + } + + priv->conn = std::make_shared < GstD3D11IpcClientConn > (self, pipe); + priv->cond.notify_all (); + lk.unlock (); + + gst_d3d11_ipc_client_wait_msg (self); + + do { + /* Enters alertable thread state and wait for I/O completion event + * or cancellable event */ + wait_ret = WaitForMultipleObjectsEx (G_N_ELEMENTS (waitables), waitables, + FALSE, INFINITE, TRUE); + if (wait_ret == WAIT_OBJECT_0) { + GST_DEBUG ("Operation cancelled"); + goto out; + } + + switch (wait_ret) { + case WAIT_IO_COMPLETION: + break; + case WAIT_OBJECT_0 + 1: + if (!priv->io_pending) + gst_d3d11_ipc_client_continue (self); + break; + default: + GST_WARNING ("Unexpected wait return 0x%x", wait_ret); + gst_d3d11_ipc_client_abort (self); + goto out; + } + } while (true); + +out: + while (!priv->samples.empty ()) { + auto sample = priv->samples.front (); + gst_sample_unref (sample); + priv->samples.pop (); + } + + priv->conn = nullptr; + + GST_DEBUG_OBJECT (self, "Exit loop thread"); + + return nullptr; +} + +GstFlowReturn +gst_d3d11_ipc_client_run (GstD3D11IpcClient * client) +{ + GstD3D11IpcClientPrivate *priv; + + g_return_val_if_fail (GST_IS_D3D11_IPC_CLIENT (client), GST_FLOW_ERROR); + + priv = client->priv; + std::unique_lock < std::mutex > lk (priv->lock); + if (!priv->loop_thread) { + priv->loop_thread = g_thread_new ("d3d11-ipc-client", + (GThreadFunc) gst_d3d11_ipc_client_loop_thread_func, client); + + while (!priv->caps && !priv->aborted && !priv->flushing) + priv->cond.wait (lk); + } + + if (priv->flushing) { + GST_DEBUG_OBJECT (client, "We are flushing"); + return GST_FLOW_FLUSHING; + } else if (priv->aborted || !priv->caps) { + GST_DEBUG_OBJECT (client, "Aborted"); + return GST_FLOW_ERROR; + } + + return GST_FLOW_OK; +} + +GstCaps * +gst_d3d11_ipc_client_get_caps (GstD3D11IpcClient * client) +{ + GstCaps *caps = nullptr; + GstD3D11IpcClientPrivate *priv; + + g_return_val_if_fail (GST_IS_D3D11_IPC_CLIENT (client), nullptr); + + priv = client->priv; + + if (gst_d3d11_ipc_client_run (client) != GST_FLOW_OK) + return nullptr; + + std::lock_guard < std::mutex > lk (priv->lock); + if (priv->caps) + caps = gst_caps_ref (priv->caps); + + return caps; +} + +void +gst_d3d11_ipc_client_stop (GstD3D11IpcClient * client) +{ + GstD3D11IpcClientPrivate *priv; + + g_return_if_fail (GST_IS_D3D11_IPC_CLIENT (client)); + + priv = client->priv; + + GST_DEBUG_OBJECT (client, "Stopping"); + priv->shutdown = true; + SetEvent (priv->wakeup_event); + + std::unique_lock < std::mutex > lk (priv->lock); + while (!priv->aborted) + priv->cond.wait (lk); + lk.unlock (); + + GST_DEBUG_OBJECT (client, "Terminating"); + + SetEvent (priv->cancellable); + + g_clear_pointer (&priv->loop_thread, g_thread_join); + + GST_DEBUG_OBJECT (client, "Stopped"); +} + +void +gst_d3d11_ipc_client_set_flushing (GstD3D11IpcClient * client, bool flushing) +{ + GstD3D11IpcClientPrivate *priv; + + g_return_if_fail (GST_IS_D3D11_IPC_CLIENT (client)); + + priv = client->priv; + + std::lock_guard < std::mutex > lk (priv->lock); + priv->flushing = flushing; + priv->cond.notify_all (); +} + +GstFlowReturn +gst_d3d11_ipc_client_get_sample (GstD3D11IpcClient * client, + GstSample ** sample) +{ + GstD3D11IpcClientPrivate *priv; + + g_return_val_if_fail (GST_IS_D3D11_IPC_CLIENT (client), GST_FLOW_ERROR); + g_return_val_if_fail (sample, GST_FLOW_ERROR); + + priv = client->priv; + + GST_LOG_OBJECT (client, "Waiting for sample"); + std::unique_lock < std::mutex > lk (priv->lock); + while (!priv->flushing && !priv->aborted && !priv->server_eos && + priv->samples.empty ()) { + priv->cond.wait (lk); + } + + if (!priv->samples.empty ()) { + *sample = priv->samples.front (); + priv->samples.pop (); + + GST_LOG_OBJECT (client, "Have sample"); + return GST_FLOW_OK; + } + + if (priv->flushing) { + GST_DEBUG_OBJECT (client, "Flushing"); + return GST_FLOW_FLUSHING; + } + + GST_DEBUG_OBJECT (client, "EOS"); + + return GST_FLOW_EOS; +} + +GstD3D11IpcClient * +gst_d3d11_ipc_client_new (const std::string & address, GstD3D11Device * device, + GstD3D11IpcIOMode io_mode, guint timeout) +{ + GstD3D11IpcClient *self; + GstD3D11IpcClientPrivate *priv; + + g_return_val_if_fail (GST_IS_D3D11_DEVICE (device), nullptr); + + self = (GstD3D11IpcClient *) + g_object_new (GST_TYPE_D3D11_IPC_CLIENT, nullptr); + gst_object_ref_sink (self); + + priv = self->priv; + priv->address = address; + priv->timeout = timeout * GST_SECOND; + priv->io_mode = io_mode; + priv->device = (GstD3D11Device *) gst_object_ref (device); + + return self; +} diff --git a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipcclient.h b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipcclient.h new file mode 100644 index 0000000000..79c2274b6f --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipcclient.h @@ -0,0 +1,58 @@ +/* GStreamer + * Copyright (C) 2023 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#include +#include +#include "gstd3d11ipc.h" + +G_BEGIN_DECLS + +enum GstD3D11IpcIOMode +{ + GST_D3D11_IPC_IO_COPY, + GST_D3D11_IPC_IO_IMPORT, +}; + +#define GST_TYPE_D3D11_IPC_IO_MODE (gst_d3d11_ipc_io_mode_get_type ()) +GType gst_d3d11_ipc_io_mode_get_type (void); + +#define GST_TYPE_D3D11_IPC_CLIENT (gst_d3d11_ipc_client_get_type()) +G_DECLARE_FINAL_TYPE (GstD3D11IpcClient, gst_d3d11_ipc_client, + GST, D3D11_IPC_CLIENT, GstObject); + +GstD3D11IpcClient * gst_d3d11_ipc_client_new (const std::string & address, + GstD3D11Device * device, + GstD3D11IpcIOMode io_mode, + guint timeout); + +GstFlowReturn gst_d3d11_ipc_client_get_sample (GstD3D11IpcClient * client, + GstSample ** sample); + +void gst_d3d11_ipc_client_set_flushing (GstD3D11IpcClient * client, + bool flushing); + +GstCaps * gst_d3d11_ipc_client_get_caps (GstD3D11IpcClient * client); + +GstFlowReturn gst_d3d11_ipc_client_run (GstD3D11IpcClient * client); + +void gst_d3d11_ipc_client_stop (GstD3D11IpcClient * client); + +G_END_DECLS diff --git a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipcserver.cpp b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipcserver.cpp new file mode 100644 index 0000000000..9e19a20570 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipcserver.cpp @@ -0,0 +1,825 @@ +/* GStreamer + * Copyright (C) 2023 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstd3d11ipcserver.h" +#include +#include +#include +#include +#include + +GST_DEBUG_CATEGORY_STATIC (gst_d3d11_ipc_server_debug); +#define GST_CAT_DEFAULT gst_d3d11_ipc_server_debug + +/* *INDENT-OFF* */ +struct GstD3D11IpcServerData +{ + ~GstD3D11IpcServerData () + { + if (sample) + gst_sample_unref (sample); + } + + GstSample *sample = nullptr; + std::wstring name; + GstD3D11IpcMemLayout layout; + GstClockTime pts; + guint64 seq_num; +}; + +struct GstD3D11IpcServerConn : public OVERLAPPED +{ + GstD3D11IpcServerConn (HANDLE pipe_handle) : pipe (pipe_handle) + { + OVERLAPPED *parent = static_cast (this); + parent->Internal = 0; + parent->InternalHigh = 0; + parent->Offset = 0; + parent->OffsetHigh = 0; + + client_msg.resize (GST_D3D11_IPC_PKT_HEADER_SIZE); + server_msg.resize (GST_D3D11_IPC_PKT_HEADER_SIZE); + } + + ~GstD3D11IpcServerConn() + { + if (pipe != INVALID_HANDLE_VALUE) { + CancelIo (pipe); + DisconnectNamedPipe (pipe); + CloseHandle (pipe); + } + + gst_clear_caps (&caps); + } + + GstD3D11IpcServer *server; + + HANDLE pipe; + + GstD3D11IpcPktType type; + std::vector client_msg; + std::vector server_msg; + std::shared_ptr data; + std::vector> peer_handles; + GstCaps *caps = nullptr; + guint64 seq_num = 0; + guint id; + bool eos = false; + bool pending_have_data = false; + bool configured = false; +}; + +struct GstD3D11IpcServerPrivate +{ + GstD3D11IpcServerPrivate () + { + cancellable = CreateEvent (nullptr, TRUE, FALSE, nullptr); + wakeup_event = CreateEvent (nullptr, FALSE, FALSE, nullptr); + + shutdown = false; + aborted = false; + } + + ~GstD3D11IpcServerPrivate () + { + CloseHandle (cancellable); + CloseHandle (wakeup_event); + gst_clear_object (&device); + } + + GstD3D11Device *device = nullptr; + gint64 adapter_luid = 0; + std::mutex lock; + guint64 seq_num = 0; + guint next_conn_id = 0; + std::unordered_map> conn_map; + GThread *loop_thread = nullptr; + std::atomicshutdown; + std::atomicaborted; + std::shared_ptr data; + std::string address; + HANDLE cancellable; + HANDLE wakeup_event; +}; +/* *INDENT-ON* */ + +struct _GstD3D11IpcServer +{ + GstObject parent; + + GstD3D11IpcServerPrivate *priv; +}; + +static void gst_d3d11_ipc_server_dispose (GObject * object); +static void gst_d3d11_ipc_server_finalize (GObject * object); +static void gst_d3d11_ipc_server_on_idle (GstD3D11IpcServer * self); +static void gst_d3d11_ipc_server_send_msg (GstD3D11IpcServer * self, + GstD3D11IpcServerConn * conn); +static void gst_d3d11_ipc_server_wait_msg (GstD3D11IpcServer * self, + GstD3D11IpcServerConn * conn); + +#define gst_d3d11_ipc_server_parent_class parent_class +G_DEFINE_TYPE (GstD3D11IpcServer, gst_d3d11_ipc_server, GST_TYPE_OBJECT); + +static void +gst_d3d11_ipc_server_class_init (GstD3D11IpcServerClass * klass) +{ + GObjectClass *object_class = G_OBJECT_CLASS (klass); + + object_class->finalize = gst_d3d11_ipc_server_finalize; + object_class->dispose = gst_d3d11_ipc_server_dispose; + + GST_DEBUG_CATEGORY_INIT (gst_d3d11_ipc_server_debug, "d3d11ipcserver", + 0, "d3d11ipcserver"); +} + +static void +gst_d3d11_ipc_server_init (GstD3D11IpcServer * self) +{ + self->priv = new GstD3D11IpcServerPrivate (); +} + +static void +gst_d3d11_ipc_server_dispose (GObject * object) +{ + GstD3D11IpcServer *self = GST_D3D11_IPC_SERVER (object); + GstD3D11IpcServerPrivate *priv = self->priv; + + GST_DEBUG_OBJECT (self, "dispose"); + + SetEvent (priv->cancellable); + + g_clear_pointer (&priv->loop_thread, g_thread_join); + + G_OBJECT_CLASS (parent_class)->dispose (object); +} + +static void +gst_d3d11_ipc_server_finalize (GObject * object) +{ + GstD3D11IpcServer *self = GST_D3D11_IPC_SERVER (object); + + GST_DEBUG_OBJECT (self, "finalize"); + + delete self->priv; + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static HANDLE +gst_cuda_ipc_server_win32_create_pipe (GstD3D11IpcServer * self, + OVERLAPPED * overlap, bool &io_pending) +{ + GstD3D11IpcServerPrivate *priv = self->priv; + HANDLE pipe = CreateNamedPipeA (priv->address.c_str (), + PIPE_ACCESS_DUPLEX | FILE_FLAG_OVERLAPPED, + PIPE_TYPE_MESSAGE | PIPE_READMODE_MESSAGE | PIPE_WAIT, + PIPE_UNLIMITED_INSTANCES, 1024, 1024, 5000, nullptr); + + if (pipe == INVALID_HANDLE_VALUE) { + guint last_err = GetLastError (); + auto err = gst_d3d11_ipc_win32_error_to_string (last_err); + GST_ERROR_OBJECT (self, "CreateNamedPipeA failed with 0x%x (%s)", + last_err, err.c_str ()); + return INVALID_HANDLE_VALUE; + } + + if (ConnectNamedPipe (pipe, overlap)) { + guint last_err = GetLastError (); + auto err = gst_d3d11_ipc_win32_error_to_string (last_err); + GST_ERROR_OBJECT (self, "ConnectNamedPipe failed with 0x%x (%s)", + last_err, err.c_str ()); + return INVALID_HANDLE_VALUE; + } + + io_pending = false; + guint last_err = GetLastError (); + + switch (last_err) { + case ERROR_IO_PENDING: + io_pending = true; + break; + case ERROR_PIPE_CONNECTED: + SetEvent (overlap->hEvent); + break; + default: + { + auto err = gst_d3d11_ipc_win32_error_to_string (last_err); + GST_ERROR_OBJECT (self, "ConnectNamedPipe failed with 0x%x (%s)", + last_err, err.c_str ()); + CloseHandle (pipe); + return INVALID_HANDLE_VALUE; + } + } + + return pipe; +} + +static void +gst_d3d11_ipc_server_close_connection (GstD3D11IpcServer * self, + GstD3D11IpcServerConn * conn) +{ + GstD3D11IpcServerPrivate *priv = self->priv; + + GST_DEBUG_OBJECT (self, "Closing conn-id %u", conn->id); + + priv->conn_map.erase (conn->id); + + if (priv->shutdown && priv->conn_map.empty ()) { + GST_DEBUG_OBJECT (self, "All connection were closed"); + SetEvent (priv->cancellable); + } +} + +static void +gst_d3d11_ipc_server_have_data (GstD3D11IpcServer * self, + GstD3D11IpcServerConn * conn) +{ + GstCaps *caps; + + if (!conn->data) { + GST_ERROR_OBJECT (self, "Have no data to send, conn-id: %u", conn->id); + gst_d3d11_ipc_server_close_connection (self, conn); + return; + } + + conn->pending_have_data = false; + conn->seq_num = conn->data->seq_num + 1; + + caps = gst_sample_get_caps (conn->data->sample); + if (!conn->caps || !gst_caps_is_equal (conn->caps, caps)) { + GST_DEBUG_OBJECT (self, "Sending caps %" GST_PTR_FORMAT " to conn-id %u", + caps, conn->id); + gst_caps_replace (&conn->caps, caps); + } else { + caps = nullptr; + } + + auto handle_dump = gst_d3d11_ipc_wstring_to_string (conn->data->name); + GST_LOG_OBJECT (self, "Sending HAVE-DATA with handle \"%s\", conn-id :%u", + handle_dump.c_str (), conn->id); + + if (!gst_d3d11_ipc_pkt_build_have_data (conn->server_msg, conn->data->pts, + conn->data->layout, conn->data->name, caps)) { + GST_ERROR_OBJECT (self, "Couldn't build HAVE-DATA pkt, conn-id: %u", + conn->id); + gst_d3d11_ipc_server_close_connection (self, conn); + return; + } + + conn->type = GstD3D11IpcPktType::HAVE_DATA; + gst_d3d11_ipc_server_send_msg (self, conn); +} + +static bool +gst_d3d11_ipc_server_on_release_data (GstD3D11IpcServer * self, + GstD3D11IpcServerConn * conn) +{ + std::wstring name; + bool found = false; + + if (!gst_d3d11_ipc_pkt_parse_release_data (conn->client_msg, name)) { + GST_ERROR_OBJECT (self, "Couldn't parse RELEASE-DATA, conn-id: %u", + conn->id); + return false; + } + + auto handle_dump = gst_d3d11_ipc_wstring_to_string (name); + GST_LOG_OBJECT (self, "RELEASE-DATA \"%s\", conn-id: %u", + handle_dump.c_str (), conn->id); + + for (auto it = conn->peer_handles.begin (); it != conn->peer_handles.end (); + it++) { + auto other = (*it)->name; + if (name == other) { + found = true; + conn->peer_handles.erase (it); + break; + } + } + + if (!found) { + GST_WARNING_OBJECT (self, + "Unexpected name to remove, conn-id: %u", conn->id); + return false; + } + + GST_LOG_OBJECT (self, "Client is holding %" G_GSIZE_FORMAT " handles", + conn->peer_handles.size ()); + + return true; +} + +static void +gst_d3d11_ipc_server_wait_msg_finish (GstD3D11IpcServer * server, + GstD3D11IpcServerConn * conn) +{ + GstD3D11IpcPacketHeader header; + + if (!gst_d3d11_ipc_pkt_identify (conn->client_msg, header)) { + GST_ERROR_OBJECT (server, "Broken header, conn-id: %u", conn->id); + gst_d3d11_ipc_server_close_connection (server, conn); + return; + } + + switch (header.type) { + case GstD3D11IpcPktType::NEED_DATA: + GST_LOG_OBJECT (server, "NEED-DATA, conn-id: %u", conn->id); + if (!conn->data) { + GST_LOG_OBJECT (server, "Wait for available data, conn-id: %u", + conn->id); + conn->pending_have_data = true; + gst_d3d11_ipc_server_on_idle (server); + return; + } + gst_d3d11_ipc_server_have_data (server, conn); + break; + case GstD3D11IpcPktType::READ_DONE: + GST_LOG_OBJECT (server, "READ-DONE, conn-id: %u", conn->id); + + if (!conn->data) { + GST_ERROR_OBJECT (server, "Unexpected READ-DATA, conn-id: %u", + conn->id); + gst_d3d11_ipc_server_close_connection (server, conn); + return; + } + + conn->peer_handles.push_back (conn->data); + conn->data = nullptr; + gst_d3d11_ipc_server_wait_msg (server, conn); + break; + case GstD3D11IpcPktType::RELEASE_DATA: + GST_LOG_OBJECT (server, "RELEASE-DATA, conn-id: %u", conn->id); + if (!gst_d3d11_ipc_server_on_release_data (server, conn)) + gst_d3d11_ipc_server_close_connection (server, conn); + else + gst_d3d11_ipc_server_wait_msg (server, conn); + break; + case GstD3D11IpcPktType::FIN: + GST_DEBUG_OBJECT (server, "FIN, conn-id %u", conn->id); + gst_d3d11_ipc_server_close_connection (server, conn); + break; + default: + GST_ERROR_OBJECT (server, "Unexpected packet, conn-id: %u", conn->id); + gst_d3d11_ipc_server_close_connection (server, conn); + break; + } +} + +static void WINAPI +gst_d3d11_ipc_server_payload_finish (DWORD error_code, DWORD size, + OVERLAPPED * overlap) +{ + GstD3D11IpcServerConn *conn = + static_cast < GstD3D11IpcServerConn * >(overlap); + GstD3D11IpcServer *self = conn->server; + + if (error_code != ERROR_SUCCESS) { + auto err = gst_d3d11_ipc_win32_error_to_string (error_code); + GST_WARNING_OBJECT (self, "ReadFileEx callback failed with 0x%x (%s)", + (guint) error_code, err.c_str ()); + gst_d3d11_ipc_server_close_connection (self, conn); + return; + } + + gst_d3d11_ipc_server_wait_msg_finish (self, conn); +} + +static void WINAPI +gst_d3d11_ipc_server_wait_msg_header_finish (DWORD error_code, DWORD size, + OVERLAPPED * overlap) +{ + GstD3D11IpcServerConn *conn = + static_cast < GstD3D11IpcServerConn * >(overlap); + GstD3D11IpcServer *self = conn->server; + GstD3D11IpcPacketHeader header; + + if (error_code != ERROR_SUCCESS) { + auto err = gst_d3d11_ipc_win32_error_to_string (error_code); + GST_WARNING_OBJECT (self, "ReadFileEx callback failed with 0x%x (%s)", + (guint) error_code, err.c_str ()); + gst_d3d11_ipc_server_close_connection (self, conn); + return; + } + + if (!gst_d3d11_ipc_pkt_identify (conn->client_msg, header)) { + GST_ERROR_OBJECT (self, "Broken header"); + gst_d3d11_ipc_server_close_connection (self, conn); + return; + } + + if (header.payload_size == 0) { + gst_d3d11_ipc_server_wait_msg_finish (conn->server, conn); + return; + } + + GST_LOG_OBJECT (self, "Reading payload"); + + if (!ReadFileEx (conn->pipe, &conn->client_msg[0] + + GST_D3D11_IPC_PKT_HEADER_SIZE, header.payload_size, conn, + gst_d3d11_ipc_server_payload_finish)) { + guint last_err = GetLastError (); + auto err = gst_d3d11_ipc_win32_error_to_string (last_err); + GST_WARNING_OBJECT (self, "ReadFileEx failed with 0x%x (%s)", + last_err, err.c_str ()); + gst_d3d11_ipc_server_close_connection (self, conn); + } +} + +static void +gst_d3d11_ipc_server_wait_msg (GstD3D11IpcServer * self, + GstD3D11IpcServerConn * conn) +{ + if (!ReadFileEx (conn->pipe, &conn->client_msg[0], + GST_D3D11_IPC_PKT_HEADER_SIZE, conn, + gst_d3d11_ipc_server_wait_msg_header_finish)) { + guint last_err = GetLastError (); + auto err = gst_d3d11_ipc_win32_error_to_string (last_err); + GST_WARNING_OBJECT (self, "ReadFileEx failed with 0x%x (%s)", + last_err, err.c_str ()); + gst_d3d11_ipc_server_close_connection (self, conn); + } +} + +static void +gst_d3d11_ipc_server_eos (GstD3D11IpcServer * self, + GstD3D11IpcServerConn * conn) +{ + gst_d3d11_ipc_pkt_build_eos (conn->server_msg); + conn->eos = true; + conn->type = GstD3D11IpcPktType::EOS; + + gst_d3d11_ipc_server_send_msg (self, conn); +} + +static void +gst_d3d11_ipc_server_config_data (GstD3D11IpcServer * self, + GstD3D11IpcServerConn * conn) +{ + GstD3D11IpcServerPrivate *priv = self->priv; + GstCaps *caps = gst_sample_get_caps (conn->data->sample); + + gst_caps_replace (&conn->caps, caps); + + gst_d3d11_ipc_pkt_build_config (conn->server_msg, priv->adapter_luid, + conn->caps); + conn->type = GstD3D11IpcPktType::CONFIG; + + GST_LOG_OBJECT (self, "Sending CONFIG, conn-id %u", conn->id); + gst_d3d11_ipc_server_send_msg (self, conn); +} + +static void +gst_d3d11_ipc_server_on_idle (GstD3D11IpcServer * self) +{ + GstD3D11IpcServerPrivate *priv = self->priv; + + GST_LOG_OBJECT (self, "idle"); + + if (priv->shutdown) { + GST_DEBUG_OBJECT (self, "We are stopping"); + + if (priv->conn_map.empty ()) { + GST_DEBUG_OBJECT (self, "All connections were closed"); + SetEvent (priv->cancellable); + return; + } + + std::vector < std::shared_ptr < GstD3D11IpcServerConn >> to_send_eos; + /* *INDENT-OFF* */ + for (auto it : priv->conn_map) { + auto conn = it.second; + if (conn->eos || !conn->pending_have_data) + continue; + + to_send_eos.push_back (conn); + } + + for (auto it : to_send_eos) { + GST_DEBUG_OBJECT (self, "Sending EOS to conn-id: %u", it->id); + gst_d3d11_ipc_server_eos (self, it.get ()); + } + + GST_DEBUG_OBJECT (self, "Have %" G_GSIZE_FORMAT " alive connections", + priv->conn_map.size()); + for (auto it : priv->conn_map) { + auto conn = it.second; + GST_DEBUG_OBJECT (self, "conn-id %u" + " peer handle size %" G_GSIZE_FORMAT, conn->id, + conn->peer_handles.size ()); + } + /* *INDENT-ON* */ + + return; + } + + if (priv->conn_map.empty ()) { + GST_LOG_OBJECT (self, "Have no connection"); + return; + } + + std::unique_lock < std::mutex > lk (priv->lock); + if (!priv->data) + return; + + /* *INDENT-OFF* */ + std::vector < std::shared_ptr < GstD3D11IpcServerConn >> to_config_data; + std::vector < std::shared_ptr < GstD3D11IpcServerConn >> to_send_have_data; + for (auto it : priv->conn_map) { + auto conn = it.second; + if (!conn->configured) { + conn->configured = true; + conn->data = priv->data; + to_config_data.push_back (conn); + } else if (conn->pending_have_data && conn->seq_num <= priv->data->seq_num) { + conn->data = priv->data; + to_send_have_data.push_back (conn); + } + } + lk.unlock (); + + for (auto it: to_config_data) + gst_d3d11_ipc_server_config_data (self, it.get ()); + + for (auto it: to_send_have_data) + gst_d3d11_ipc_server_have_data (self, it.get ()); + /* *INDENT-ON* */ +} + +static void WINAPI +gst_d3d11_ipc_server_send_msg_finish (DWORD error_code, DWORD size, + OVERLAPPED * overlap) +{ + GstD3D11IpcServerConn *conn = + static_cast < GstD3D11IpcServerConn * >(overlap); + GstD3D11IpcServer *self = conn->server; + + if (error_code != ERROR_SUCCESS) { + auto err = gst_d3d11_ipc_win32_error_to_string (error_code); + GST_WARNING_OBJECT (self, "ReadFileEx callback failed with 0x%x (%s)", + (guint) error_code, err.c_str ()); + gst_d3d11_ipc_server_close_connection (self, conn); + return; + } + + GST_LOG_OBJECT (self, "Sent message"); + + switch (conn->type) { + case GstD3D11IpcPktType::CONFIG: + GST_DEBUG_OBJECT (self, "Sent CONFIG-DATA, conn-id %u", conn->id); + gst_d3d11_ipc_server_wait_msg (self, conn); + break; + case GstD3D11IpcPktType::HAVE_DATA: + GST_LOG_OBJECT (self, "Sent HAVE-DATA, conn-id %u", conn->id); + gst_d3d11_ipc_server_wait_msg (self, conn); + break; + case GstD3D11IpcPktType::EOS: + GST_DEBUG_OBJECT (self, "Sent EOS, conn-id %u", conn->id); + gst_d3d11_ipc_server_wait_msg (self, conn); + break; + default: + GST_ERROR_OBJECT (self, "Unexpected msg type"); + gst_d3d11_ipc_server_close_connection (self, conn); + break; + } +} + +static void +gst_d3d11_ipc_server_send_msg (GstD3D11IpcServer * self, + GstD3D11IpcServerConn * conn) +{ + GST_LOG_OBJECT (self, "Sending message"); + + if (!WriteFileEx (conn->pipe, &conn->server_msg[0], + conn->server_msg.size (), conn, + gst_d3d11_ipc_server_send_msg_finish)) { + guint last_err = GetLastError (); + auto err = gst_d3d11_ipc_win32_error_to_string (last_err); + GST_WARNING_OBJECT (self, "WriteFileEx failed with 0x%x (%s)", + last_err, err.c_str ()); + gst_d3d11_ipc_server_close_connection (self, conn); + } +} + +static void +gst_d3d11_ipc_server_on_incoming_connection (GstD3D11IpcServer * self, + std::shared_ptr < GstD3D11IpcServerConn > conn) +{ + GstD3D11IpcServerPrivate *priv = self->priv; + + priv->lock.lock (); + conn->server = self; + conn->id = priv->next_conn_id; + conn->data = priv->data; + priv->next_conn_id++; + priv->lock.unlock (); + + GST_DEBUG_OBJECT (self, "New connection, conn-id: %u", conn->id); + + /* *INDENT-OFF* */ + priv->conn_map.insert ({conn->id, conn}); + /* *INDENT-ON* */ + + if (conn->data) { + conn->configured = true; + gst_d3d11_ipc_server_config_data (self, conn.get ()); + } else { + GST_DEBUG_OBJECT (self, "Have no config data yet, waiting for data"); + } +} + +static gpointer +gst_d3d11_ipc_server_loop_thread_func (GstD3D11IpcServer * self) +{ + GstD3D11IpcServerPrivate *priv = self->priv; + bool io_pending = false; + guint wait_ret; + HANDLE pipe; + OVERLAPPED overlap; + HANDLE waitables[3]; + + GST_DEBUG_OBJECT (self, "Entering loop"); + + memset (&overlap, 0, sizeof (OVERLAPPED)); + + overlap.hEvent = CreateEvent (nullptr, TRUE, TRUE, nullptr); + pipe = gst_cuda_ipc_server_win32_create_pipe (self, &overlap, io_pending); + if (pipe == INVALID_HANDLE_VALUE) { + CloseHandle (overlap.hEvent); + priv->aborted = true; + goto out; + } + + waitables[0] = overlap.hEvent; + waitables[1] = priv->wakeup_event; + waitables[2] = priv->cancellable; + + do { + wait_ret = WaitForMultipleObjectsEx (G_N_ELEMENTS (waitables), waitables, + FALSE, INFINITE, TRUE); + + if (wait_ret == WAIT_OBJECT_0 + 2) { + GST_DEBUG_OBJECT (self, "Operation cancelled"); + goto out; + } + + switch (wait_ret) { + case WAIT_OBJECT_0: + { + DWORD n_bytes; + + if (io_pending + && !GetOverlappedResult (pipe, &overlap, &n_bytes, FALSE)) { + guint last_err = GetLastError (); + auto err = gst_d3d11_ipc_win32_error_to_string (last_err); + GST_WARNING_OBJECT (self, "GetOverlappedResult failed with 0x%x (%s)", + last_err, err.c_str ()); + CloseHandle (pipe); + pipe = INVALID_HANDLE_VALUE; + break; + } + + auto conn = std::make_shared < GstD3D11IpcServerConn > (pipe); + conn->server = self; + pipe = INVALID_HANDLE_VALUE; + gst_d3d11_ipc_server_on_incoming_connection (self, conn); + + pipe = gst_cuda_ipc_server_win32_create_pipe (self, + &overlap, io_pending); + break; + } + case WAIT_IO_COMPLETION: + break; + case WAIT_OBJECT_0 + 1: + gst_d3d11_ipc_server_on_idle (self); + break; + default: + { + guint last_err = GetLastError (); + auto err = gst_d3d11_ipc_win32_error_to_string (last_err); + GST_ERROR_OBJECT (self, + "WaitForMultipleObjectsEx return 0x%x, last error 0x%x (%s)", + wait_ret, last_err, err.c_str ()); + priv->aborted = true; + goto out; + } + } + } while (true); + +out: + if (pipe != INVALID_HANDLE_VALUE) { + CancelIo (pipe); + DisconnectNamedPipe (pipe); + CloseHandle (pipe); + } + + CloseHandle (overlap.hEvent); + + priv->conn_map.clear (); + + GST_DEBUG_OBJECT (self, "Exit loop thread"); + + return nullptr; +} + +GstFlowReturn +gst_d3d11_ipc_server_send_data (GstD3D11IpcServer * server, GstSample * sample, + const GstD3D11IpcMemLayout & layout, const std::wstring & name, + GstClockTime pts) +{ + GstD3D11IpcServerPrivate *priv; + + g_return_val_if_fail (GST_IS_D3D11_IPC_SERVER (server), GST_FLOW_ERROR); + g_return_val_if_fail (GST_IS_SAMPLE (sample), GST_FLOW_ERROR); + + priv = server->priv; + + GST_LOG_OBJECT (server, "Sending data"); + + std::unique_lock < std::mutex > lk (priv->lock); + if (priv->aborted) { + GST_DEBUG_OBJECT (server, "Was aborted"); + return GST_FLOW_ERROR; + } + + auto data = std::make_shared < GstD3D11IpcServerData > (); + data->sample = gst_sample_ref (sample); + data->name = name; + data->layout = layout; + data->pts = pts; + data->seq_num = priv->seq_num; + + priv->seq_num++; + priv->data = data; + lk.unlock (); + + SetEvent (priv->wakeup_event); + + return GST_FLOW_OK; +} + +void +gst_d3d11_ipc_server_stop (GstD3D11IpcServer * server) +{ + GstD3D11IpcServerPrivate *priv; + + g_return_if_fail (GST_IS_D3D11_IPC_SERVER (server)); + + priv = server->priv; + + GST_DEBUG_OBJECT (server, "Stopping"); + priv->shutdown = true; + SetEvent (priv->wakeup_event); + + g_clear_pointer (&priv->loop_thread, g_thread_join); + + GST_DEBUG_OBJECT (server, "Stopped"); +} + +GstD3D11IpcServer * +gst_d3d11_ipc_server_new (const std::string & address, gint64 adapter_luid) +{ + GstD3D11IpcServer *self; + GstD3D11IpcServerPrivate *priv; + + self = (GstD3D11IpcServer *) + g_object_new (GST_TYPE_D3D11_IPC_SERVER, nullptr); + gst_object_ref_sink (self); + + priv = self->priv; + priv->address = address; + priv->adapter_luid = adapter_luid; + + priv->loop_thread = g_thread_new ("d3d11-ipc-server", + (GThreadFunc) gst_d3d11_ipc_server_loop_thread_func, self); + + return self; +} + +gint64 +gst_d3d11_ipc_server_get_adapter_luid (GstD3D11IpcServer * server) +{ + g_return_val_if_fail (GST_IS_D3D11_IPC_SERVER (server), 0); + + return server->priv->adapter_luid; +} diff --git a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipcserver.h b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipcserver.h new file mode 100644 index 0000000000..152b92d725 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipcserver.h @@ -0,0 +1,45 @@ +/* GStreamer + * Copyright (C) 2023 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#include +#include +#include "gstd3d11ipc.h" + +G_BEGIN_DECLS + +#define GST_TYPE_D3D11_IPC_SERVER (gst_d3d11_ipc_server_get_type()) +G_DECLARE_FINAL_TYPE (GstD3D11IpcServer, gst_d3d11_ipc_server, + GST, D3D11_IPC_SERVER, GstObject); + +GstD3D11IpcServer * gst_d3d11_ipc_server_new (const std::string & address, + gint64 adapter_luid); + +GstFlowReturn gst_d3d11_ipc_server_send_data (GstD3D11IpcServer * server, + GstSample * sample, + const GstD3D11IpcMemLayout & layout, + const std::wstring & name, + GstClockTime pts); + +void gst_d3d11_ipc_server_stop (GstD3D11IpcServer * server); + +gint64 gst_d3d11_ipc_server_get_adapter_luid (GstD3D11IpcServer * server); + +G_END_DECLS diff --git a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipcsink.cpp b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipcsink.cpp new file mode 100644 index 0000000000..ef71da9a1e --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipcsink.cpp @@ -0,0 +1,812 @@ +/* GStreamer + * Copyright (C) 2023 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/** + * SECTION:element-d3d11ipcsink + * @title: d3d11ipcsink + * @short_description: Direct3D11 Inter Process Communication (IPC) sink + * + * d3d11ipcsink exports Direct3D11 texture for connected d3d11ipcsrc elements + * to be able to import it + * + * ## Example launch line + * ``` + * gst-launch-1.0 videotestsrc ! d3d11upload ! d3d11ipcsink + * ``` + * + * Since: 1.24 + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstd3d11ipcsink.h" +#include "gstd3d11ipcserver.h" +#include "gstd3d11pluginutils.h" +#include +#include +#include +#include + +/* *INDENT-OFF* */ +using namespace Microsoft::WRL; +/* *INDENT-ON* */ + +GST_DEBUG_CATEGORY_STATIC (gst_d3d11_ipc_sink_debug); +#define GST_CAT_DEFAULT gst_d3d11_ipc_sink_debug + +static GstStaticCaps pad_template_caps = + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY, GST_D3D11_IPC_FORMATS) "; " + GST_VIDEO_CAPS_MAKE (GST_D3D11_IPC_FORMATS)); + +enum +{ + PROP_0, + PROP_ADAPTER, + PROP_PIPE_NAME, + PROP_MIN_BUFFER_SIZE, +}; + +#define DEFAULT_ADAPTER -1 +#define DEFAULT_PIPE_NAME "\\\\.\\pipe\\gst.d3d11.ipc" +#define DEFAULT_MIN_BUFFER_SIZE 0 + +/* *INDENT-OFF* */ +struct GstD3D11IpcSinkResource +{ + ~GstD3D11IpcSinkResource () + { + if (handle) + CloseHandle (handle); + } + + HANDLE handle = nullptr; + std::wstring name; +}; + +struct GstD3D11IpcSinkPrivate +{ + GstD3D11IpcSinkPrivate () + { + prefix = gst_d3d11_ipc_get_resource_prefix (); + } + + GstD3D11Device *device = nullptr; + + GstBufferPool *fallback_pool = nullptr; + GstVideoInfo info; + + GstD3D11IpcServer *server = nullptr; + GstCaps *caps = nullptr; + GstSample *prepared_sample = nullptr; + GstD3D11IpcMemLayout layout; + std::wstring resource_name; + std::wstring prefix; + guint64 seq_num = 0; + + std::mutex lock; + + /* properties */ + gint adapter = DEFAULT_ADAPTER; + std::string pipe_name = DEFAULT_PIPE_NAME; + guint buffer_size = DEFAULT_MIN_BUFFER_SIZE; +}; +/* *INDENT-ON* */ + +struct _GstD3D11IpcSink +{ + GstBaseSink parent; + + GstD3D11IpcSinkPrivate *priv; +}; + +static void gst_d3d11_ipc_sink_finalize (GObject * object); +static void gst_d3d11_ipc_sink_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec); +static void gst_win32_video_sink_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec); + +static GstClock *gst_d3d11_ipc_sink_provide_clock (GstElement * elem); +static void gst_d3d11_ipc_sink_set_context (GstElement * elem, + GstContext * context); + +static gboolean gst_d3d11_ipc_sink_start (GstBaseSink * sink); +static gboolean gst_d3d11_ipc_sink_stop (GstBaseSink * sink); +static gboolean gst_d3d11_ipc_sink_set_caps (GstBaseSink * sink, + GstCaps * caps); +static void gst_d3d11_ipc_sink_get_time (GstBaseSink * sink, GstBuffer * buf, + GstClockTime * start, GstClockTime * end); +static gboolean gst_d3d11_ipc_sink_propose_allocation (GstBaseSink * sink, + GstQuery * query); +static gboolean gst_d3d11_ipc_sink_query (GstBaseSink * sink, GstQuery * query); +static GstFlowReturn gst_d3d11_ipc_sink_prepare (GstBaseSink * sink, + GstBuffer * buf); +static GstFlowReturn gst_d3d11_ipc_sink_render (GstBaseSink * sink, + GstBuffer * buf); + +#define gst_d3d11_ipc_sink_parent_class parent_class +G_DEFINE_TYPE (GstD3D11IpcSink, gst_d3d11_ipc_sink, GST_TYPE_BASE_SINK); + +static void +gst_d3d11_ipc_sink_class_init (GstD3D11IpcSinkClass * klass) +{ + GObjectClass *object_class = G_OBJECT_CLASS (klass); + GstElementClass *element_class = GST_ELEMENT_CLASS (klass); + GstBaseSinkClass *sink_class = GST_BASE_SINK_CLASS (klass); + GstCaps *caps; + + object_class->finalize = gst_d3d11_ipc_sink_finalize; + object_class->set_property = gst_d3d11_ipc_sink_set_property; + object_class->get_property = gst_win32_video_sink_get_property; + + g_object_class_install_property (object_class, PROP_ADAPTER, + g_param_spec_int ("adapter", "Adapter", + "DXGI adapter index (-1 for default)", + -1, G_MAXINT32, DEFAULT_ADAPTER, + (GParamFlags) (G_PARAM_READWRITE | GST_PARAM_MUTABLE_READY | + G_PARAM_STATIC_STRINGS))); + + g_object_class_install_property (object_class, PROP_PIPE_NAME, + g_param_spec_string ("pipe-name", "Pipe Name", + "The name of Win32 named pipe to communicate with clients. " + "Validation of the pipe name is caller's responsibility", + DEFAULT_PIPE_NAME, (GParamFlags) (G_PARAM_READWRITE | + G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_READY))); + + g_object_class_install_property (object_class, PROP_MIN_BUFFER_SIZE, + g_param_spec_uint ("min-buffer-size", "Min Buffer Size", + "Minumum number of buffers", 0, G_MAXUINT, DEFAULT_MIN_BUFFER_SIZE, + (GParamFlags) (G_PARAM_READWRITE | GST_PARAM_MUTABLE_READY | + G_PARAM_STATIC_STRINGS))); + + gst_element_class_set_static_metadata (element_class, + "D3D11 IPC Sink", "Sink/Video", + "Send D3D11 memory to peer d3d11ipcsrc elements", + "Seungha Yang "); + + caps = gst_d3d11_get_updated_template_caps (&pad_template_caps); + gst_element_class_add_pad_template (element_class, + gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, caps)); + gst_caps_unref (caps); + + element_class->provide_clock = + GST_DEBUG_FUNCPTR (gst_d3d11_ipc_sink_provide_clock); + element_class->set_context = + GST_DEBUG_FUNCPTR (gst_d3d11_ipc_sink_set_context); + + sink_class->start = GST_DEBUG_FUNCPTR (gst_d3d11_ipc_sink_start); + sink_class->stop = GST_DEBUG_FUNCPTR (gst_d3d11_ipc_sink_stop); + sink_class->set_caps = GST_DEBUG_FUNCPTR (gst_d3d11_ipc_sink_set_caps); + sink_class->propose_allocation = + GST_DEBUG_FUNCPTR (gst_d3d11_ipc_sink_propose_allocation); + sink_class->query = GST_DEBUG_FUNCPTR (gst_d3d11_ipc_sink_query); + sink_class->get_times = GST_DEBUG_FUNCPTR (gst_d3d11_ipc_sink_get_time); + sink_class->prepare = GST_DEBUG_FUNCPTR (gst_d3d11_ipc_sink_prepare); + sink_class->render = GST_DEBUG_FUNCPTR (gst_d3d11_ipc_sink_render); + + GST_DEBUG_CATEGORY_INIT (gst_d3d11_ipc_sink_debug, "d3d11ipcsink", + 0, "d3d11ipcsink"); +} + +static void +gst_d3d11_ipc_sink_init (GstD3D11IpcSink * self) +{ + self->priv = new GstD3D11IpcSinkPrivate (); + + GST_OBJECT_FLAG_SET (self, GST_ELEMENT_FLAG_PROVIDE_CLOCK); + GST_OBJECT_FLAG_SET (self, GST_ELEMENT_FLAG_REQUIRE_CLOCK); +} + +static void +gst_d3d11_ipc_sink_finalize (GObject * object) +{ + GstD3D11IpcSink *self = GST_D3D11_IPC_SINK (object); + + delete self->priv; + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static void +gst_d3d11_ipc_sink_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstD3D11IpcSink *self = GST_D3D11_IPC_SINK (object); + GstD3D11IpcSinkPrivate *priv = self->priv; + std::lock_guard < std::mutex > lk (priv->lock); + + switch (prop_id) { + case PROP_ADAPTER: + priv->adapter = g_value_get_int (value); + break; + case PROP_PIPE_NAME: + { + const gchar *pipe_name = g_value_get_string (value); + priv->pipe_name.clear (); + + if (pipe_name) + priv->pipe_name = pipe_name; + else + priv->pipe_name = DEFAULT_PIPE_NAME; + break; + } + case PROP_MIN_BUFFER_SIZE: + priv->buffer_size = g_value_get_uint (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_win32_video_sink_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstD3D11IpcSink *self = GST_D3D11_IPC_SINK (object); + GstD3D11IpcSinkPrivate *priv = self->priv; + std::lock_guard < std::mutex > lk (priv->lock); + + switch (prop_id) { + case PROP_ADAPTER: + g_value_set_int (value, priv->adapter); + break; + case PROP_PIPE_NAME: + g_value_set_string (value, priv->pipe_name.c_str ()); + break; + case PROP_MIN_BUFFER_SIZE: + g_value_set_uint (value, priv->buffer_size); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static GstClock * +gst_d3d11_ipc_sink_provide_clock (GstElement * elem) +{ + return gst_system_clock_obtain (); +} + +static void +gst_d3d11_ipc_sink_set_context (GstElement * elem, GstContext * context) +{ + GstD3D11IpcSink *self = GST_D3D11_IPC_SINK (elem); + GstD3D11IpcSinkPrivate *priv = self->priv; + + gst_d3d11_handle_set_context (elem, context, priv->adapter, &priv->device); + + GST_ELEMENT_CLASS (parent_class)->set_context (elem, context); +} + +static gboolean +gst_d3d11_ipc_sink_start (GstBaseSink * sink) +{ + GstD3D11IpcSink *self = GST_D3D11_IPC_SINK (sink); + GstD3D11IpcSinkPrivate *priv = self->priv; + + GST_DEBUG_OBJECT (self, "Start"); + + if (!gst_d3d11_ensure_element_data (GST_ELEMENT_CAST (self), priv->adapter, + &priv->device)) { + GST_ERROR_OBJECT (sink, "Cannot create d3d11device"); + return FALSE; + } + + return TRUE; +} + +static gboolean +gst_d3d11_ipc_sink_stop (GstBaseSink * sink) +{ + GstD3D11IpcSink *self = GST_D3D11_IPC_SINK (sink); + GstD3D11IpcSinkPrivate *priv = self->priv; + + GST_DEBUG_OBJECT (self, "Stop"); + + if (priv->server) + gst_d3d11_ipc_server_stop (priv->server); + gst_clear_object (&priv->server); + + GST_DEBUG_OBJECT (self, "Server cleared"); + + if (priv->fallback_pool) { + gst_buffer_pool_set_active (priv->fallback_pool, FALSE); + gst_clear_object (&priv->fallback_pool); + } + + gst_clear_sample (&priv->prepared_sample); + gst_clear_object (&priv->device); + + return TRUE; +} + +static void +gst_d3d11_ipc_sink_get_time (GstBaseSink * sink, GstBuffer * buf, + GstClockTime * start, GstClockTime * end) +{ + GstD3D11IpcSink *self = GST_D3D11_IPC_SINK (sink); + GstD3D11IpcSinkPrivate *priv = self->priv; + GstClockTime timestamp; + + timestamp = GST_BUFFER_PTS (buf); + if (!GST_CLOCK_TIME_IS_VALID (timestamp)) + timestamp = GST_BUFFER_DTS (buf); + + if (GST_CLOCK_TIME_IS_VALID (timestamp)) { + *start = timestamp; + if (GST_BUFFER_DURATION_IS_VALID (buf)) { + *end = timestamp + GST_BUFFER_DURATION (buf); + } else if (priv->info.fps_n > 0) { + *end = timestamp + + gst_util_uint64_scale_int (GST_SECOND, priv->info.fps_d, + priv->info.fps_n); + } else if (sink->segment.rate < 0) { + *end = timestamp; + } + } +} + +static GstBufferPool * +gst_d3d11_ipc_sink_create_pool (GstD3D11IpcSink * self, + const GstVideoInfo * info, GstCaps * caps, guint min_buffers) +{ + GstD3D11IpcSinkPrivate *priv = self->priv; + GstStructure *config; + GstD3D11AllocationParams *params; + guint bind_flags = 0; + GstD3D11Format device_format; + GstBufferPool *pool; + + if (!gst_d3d11_device_get_format (priv->device, + GST_VIDEO_INFO_FORMAT (info), &device_format)) { + GST_ERROR_OBJECT (self, "Couldn't get device format"); + return nullptr; + } + + if ((device_format.format_support[0] & + (guint) D3D11_FORMAT_SUPPORT_SHADER_SAMPLE) != 0) { + bind_flags |= D3D11_BIND_SHADER_RESOURCE; + } + + if ((device_format.format_support[0] & + (guint) D3D11_FORMAT_SUPPORT_RENDER_TARGET) != 0) { + bind_flags |= D3D11_BIND_RENDER_TARGET; + } + + pool = gst_d3d11_buffer_pool_new (priv->device); + config = gst_buffer_pool_get_config (pool); + + gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META); + gst_buffer_pool_config_set_params (config, priv->caps, + GST_VIDEO_INFO_SIZE (info), min_buffers, 0); + + params = gst_d3d11_allocation_params_new (priv->device, &priv->info, + GST_D3D11_ALLOCATION_FLAG_DEFAULT, bind_flags, + D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX | + D3D11_RESOURCE_MISC_SHARED_NTHANDLE); + + gst_buffer_pool_config_set_d3d11_allocation_params (config, params); + gst_d3d11_allocation_params_free (params); + + if (!gst_buffer_pool_set_config (pool, config)) { + GST_ERROR_OBJECT (self, "Couldn't set pool config"); + gst_object_unref (pool); + return nullptr; + } + + return pool; +} + +static gboolean +gst_d3d11_ipc_sink_set_caps (GstBaseSink * sink, GstCaps * caps) +{ + GstD3D11IpcSink *self = GST_D3D11_IPC_SINK (sink); + GstD3D11IpcSinkPrivate *priv = self->priv; + GstCaps *new_caps; + GstStructure *s; + const gchar *str; + + GST_DEBUG_OBJECT (self, "New caps %" GST_PTR_FORMAT, caps); + + if (!gst_video_info_from_caps (&priv->info, caps)) { + GST_ERROR_OBJECT (self, "Invalid caps %" GST_PTR_FORMAT, caps); + return FALSE; + } + + if (priv->fallback_pool) { + gst_buffer_pool_set_active (priv->fallback_pool, FALSE); + gst_clear_object (&priv->fallback_pool); + } + + s = gst_caps_get_structure (caps, 0); + + /* Takes values we know it's always deserializable */ + new_caps = gst_caps_new_empty_simple ("video/x-raw"); + gst_caps_set_simple (new_caps, "format", G_TYPE_STRING, + gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (&priv->info)), + "width", G_TYPE_INT, priv->info.width, + "height", G_TYPE_INT, priv->info.height, + "framerate", GST_TYPE_FRACTION, priv->info.fps_n, priv->info.fps_d, + "pixel-aspect-ratio", GST_TYPE_FRACTION, priv->info.par_n, + priv->info.par_d, nullptr); + + str = gst_structure_get_string (s, "colorimetry"); + if (str) + gst_caps_set_simple (new_caps, "colorimetry", G_TYPE_STRING, str, nullptr); + + str = gst_structure_get_string (s, "mastering-display-info"); + if (str) { + gst_caps_set_simple (new_caps, "mastering-display-info", G_TYPE_STRING, + str, nullptr); + } + + str = gst_structure_get_string (s, "content-light-level"); + if (str) { + gst_caps_set_simple (new_caps, "content-light-level", G_TYPE_STRING, + str, nullptr); + } + + gst_caps_set_features_simple (new_caps, + gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY, nullptr)); + + gst_clear_caps (&priv->caps); + priv->caps = new_caps; + + return TRUE; +} + +static gboolean +gst_d3d11_ipc_sink_propose_allocation (GstBaseSink * sink, GstQuery * query) +{ + GstD3D11IpcSink *self = GST_D3D11_IPC_SINK (sink); + GstD3D11IpcSinkPrivate *priv = self->priv; + GstCaps *caps; + GstBufferPool *pool = nullptr; + GstVideoInfo info; + guint size; + gboolean need_pool; + + gst_query_parse_allocation (query, &caps, &need_pool); + if (!caps) { + GST_WARNING_OBJECT (sink, "No caps specified"); + return FALSE; + } + + if (!gst_video_info_from_caps (&info, caps)) { + GST_WARNING_OBJECT (sink, "Invalid caps %" GST_PTR_FORMAT, caps); + return FALSE; + } + + /* the normal size of a frame */ + size = info.size; + if (need_pool) { + GstCapsFeatures *features; + GstStructure *config; + + features = gst_caps_get_features (caps, 0); + if (features + && gst_caps_features_contains (features, + GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY)) { + GST_DEBUG_OBJECT (self, "upstream support d3d11 memory"); + pool = gst_d3d11_ipc_sink_create_pool (self, &info, caps, + priv->buffer_size); + if (!pool) { + GST_ERROR_OBJECT (self, "Couldn't create pool"); + return FALSE; + } + + config = gst_buffer_pool_get_config (pool); + gst_buffer_pool_config_get_params (config, nullptr, &size, nullptr, + nullptr); + gst_structure_free (config); + } else { + pool = gst_video_buffer_pool_new (); + config = gst_buffer_pool_get_config (pool); + gst_buffer_pool_config_add_option (config, + GST_BUFFER_POOL_OPTION_VIDEO_META); + gst_buffer_pool_config_add_option (config, + GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT); + + gst_buffer_pool_config_set_params (config, + caps, size, priv->buffer_size, 0); + + if (!gst_buffer_pool_set_config (pool, config)) { + GST_ERROR_OBJECT (pool, "Couldn't set config"); + gst_object_unref (pool); + + return FALSE; + } + } + } + + gst_query_add_allocation_pool (query, pool, size, priv->buffer_size, 0); + gst_clear_object (&pool); + + gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL); + + return TRUE; +} + +static gboolean +gst_d3d11_ipc_sink_query (GstBaseSink * sink, GstQuery * query) +{ + GstD3D11IpcSink *self = GST_D3D11_IPC_SINK (sink); + GstD3D11IpcSinkPrivate *priv = self->priv; + + switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_CONTEXT: + if (gst_d3d11_handle_context_query (GST_ELEMENT (self), query, + priv->device)) { + return TRUE; + } + break; + default: + break; + } + + return GST_BASE_SINK_CLASS (parent_class)->query (sink, query); +} + +static GstBuffer * +gst_d3d11_ipc_upload (GstD3D11IpcSink * self, GstBuffer * buf) +{ + GstD3D11IpcSinkPrivate *priv = self->priv; + GstBuffer *uploaded = nullptr; + GstFlowReturn ret; + GstMemory *mem; + const guint misc_flags = D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX | + D3D11_RESOURCE_MISC_SHARED_NTHANDLE; + + mem = gst_buffer_peek_memory (buf, 0); + if (gst_is_d3d11_memory (mem)) { + GstD3D11Memory *dmem = GST_D3D11_MEMORY_CAST (mem); + bool is_same_device = false; + + if (dmem->device == priv->device) { + is_same_device = true; + } else { + gint64 adapter_luid; + + g_object_get (dmem->device, "adapter-luid", &adapter_luid, nullptr); + if (adapter_luid == gst_d3d11_ipc_server_get_adapter_luid (priv->server)) + is_same_device = true; + } + + if (is_same_device) { + D3D11_TEXTURE2D_DESC desc; + + gst_d3d11_memory_get_texture_desc (dmem, &desc); + if ((desc.MiscFlags & misc_flags) == misc_flags) + return gst_buffer_ref (buf); + } + } + + if (!priv->fallback_pool) { + priv->fallback_pool = gst_d3d11_ipc_sink_create_pool (self, &priv->info, + priv->caps, 0); + if (!priv->fallback_pool) { + GST_ERROR_OBJECT (self, "Couldn't create fallback pool"); + return nullptr; + } + + if (!gst_buffer_pool_set_active (priv->fallback_pool, TRUE)) { + GST_ERROR_OBJECT (self, "Couldn't active pool"); + gst_clear_object (&priv->fallback_pool); + return nullptr; + } + } + + ret = gst_buffer_pool_acquire_buffer (priv->fallback_pool, + &uploaded, nullptr); + if (ret != GST_FLOW_OK) { + GST_ERROR_OBJECT (self, "Couldn't acquire fallback buffer"); + return nullptr; + } + + if (!gst_d3d11_buffer_copy_into (uploaded, buf, &priv->info)) { + GST_ERROR_OBJECT (self, "Couldn't copy memory"); + gst_buffer_unref (uploaded); + return nullptr; + } + + return uploaded; +} + +static gboolean +gst_d3d11_ipc_sink_ensure_server (GstD3D11IpcSink * self, GstBuffer * buffer) +{ + GstD3D11IpcSinkPrivate *priv = self->priv; + GstMemory *mem; + gint64 adapter_luid; + + if (priv->server) + return TRUE; + + g_object_get (priv->device, "adapter-luid", &adapter_luid, nullptr); + + mem = gst_buffer_peek_memory (buffer, 0); + if (gst_is_d3d11_memory (mem)) { + GstD3D11Memory *dmem = GST_D3D11_MEMORY_CAST (mem); + if (dmem->device != priv->device) { + g_object_get (dmem->device, "adapter-luid", &adapter_luid, nullptr); + gst_object_unref (priv->device); + priv->device = (GstD3D11Device *) gst_object_ref (dmem->device); + } + } + + priv->server = gst_d3d11_ipc_server_new (priv->pipe_name, adapter_luid); + if (!priv->server) { + GST_ERROR_OBJECT (self, "Couldn't create server"); + return FALSE; + } + + return TRUE; +} + +static GstFlowReturn +gst_d3d11_ipc_sink_prepare (GstBaseSink * sink, GstBuffer * buf) +{ + GstD3D11IpcSink *self = GST_D3D11_IPC_SINK (sink); + GstD3D11IpcSinkPrivate *priv = self->priv; + GstBuffer *uploaded; + GstD3D11Memory *dmem; + std::wstring name; + GstVideoFrame frame; + GstD3D11IpcSinkResource *resource; + gint64 token = gst_d3d11_ipc_get_shared_resource_token (); + + gst_clear_sample (&priv->prepared_sample); + + if (!gst_d3d11_ipc_sink_ensure_server (self, buf)) + return GST_FLOW_ERROR; + + uploaded = gst_d3d11_ipc_upload (self, buf); + if (!uploaded) { + GST_ERROR_OBJECT (self, "Couldn't upload buffer"); + return GST_FLOW_ERROR; + } + + dmem = (GstD3D11Memory *) gst_buffer_peek_memory (uploaded, 0); + + /* Upload staging to device memory */ + if (!gst_video_frame_map (&frame, &priv->info, uploaded, + (GstMapFlags) (GST_MAP_READ | GST_MAP_D3D11))) { + GST_ERROR_OBJECT (self, "Couldn't upload memory"); + gst_buffer_unref (uploaded); + return GST_FLOW_ERROR; + } + + priv->layout.size = dmem->mem.size; + priv->layout.pitch = frame.info.stride[0]; + for (guint i = 0; i < GST_VIDEO_FRAME_N_PLANES (&frame); i++) + priv->layout.offset[i] = frame.info.offset[i]; + + gst_video_frame_unmap (&frame); + + gst_d3d11_device_lock (dmem->device); + resource = (GstD3D11IpcSinkResource *) + gst_d3d11_memory_get_token_data (dmem, token); + if (!resource) { + ID3D11Resource *d3d11_resource = + gst_d3d11_memory_get_resource_handle (dmem); + HRESULT hr; + ComPtr < IDXGIResource1 > dxgi_resource; + std::wstring name = priv->prefix + std::to_wstring (priv->seq_num); + HANDLE handle; + + priv->seq_num++; + + hr = d3d11_resource->QueryInterface (IID_PPV_ARGS (&dxgi_resource)); + if (!gst_d3d11_result (hr, dmem->device)) { + GST_ERROR_OBJECT (self, "Couldn't get IDXGIResource1 interface"); + gst_d3d11_device_unlock (dmem->device); + gst_buffer_unref (uploaded); + return GST_FLOW_ERROR; + } + + hr = dxgi_resource->CreateSharedHandle (nullptr, + DXGI_SHARED_RESOURCE_READ, name.c_str (), &handle); + if (!gst_d3d11_result (hr, dmem->device)) { + GST_ERROR_OBJECT (self, "Couldn't create shared handle"); + gst_d3d11_device_unlock (dmem->device); + gst_buffer_unref (uploaded); + return GST_FLOW_ERROR; + } + + resource = new GstD3D11IpcSinkResource (); + resource->handle = handle; + resource->name = name; + /* *INDENT-OFF* */ + gst_d3d11_memory_set_token_data (dmem, token, resource, + [] (gpointer data) -> void { + delete (GstD3D11IpcSinkResource *) data; + }); + /* *INDENT-ON* */ + } + gst_d3d11_device_unlock (dmem->device); + + priv->prepared_sample = gst_sample_new (uploaded, + priv->caps, nullptr, nullptr); + priv->resource_name = resource->name; + + gst_buffer_unref (uploaded); + + return GST_FLOW_OK; +} + +static GstFlowReturn +gst_d3d11_ipc_sink_render (GstBaseSink * sink, GstBuffer * buf) +{ + GstD3D11IpcSink *self = GST_D3D11_IPC_SINK (sink); + GstD3D11IpcSinkPrivate *priv = self->priv; + GstClockTime pts; + GstClockTime now_system; + GstClockTime buf_pts; + GstClockTime buffer_clock = GST_CLOCK_TIME_NONE; + GstFlowReturn ret; + + if (!priv->prepared_sample) { + GST_ERROR_OBJECT (self, "Have no prepared sample"); + return GST_FLOW_ERROR; + } + + pts = now_system = gst_d3d11_ipc_clock_now (); + buf_pts = GST_BUFFER_PTS (buf); + if (!GST_CLOCK_TIME_IS_VALID (buf_pts)) + buf_pts = GST_BUFFER_DTS (buf); + + if (GST_CLOCK_TIME_IS_VALID (buf_pts)) { + buffer_clock = gst_segment_to_running_time (&sink->segment, + GST_FORMAT_TIME, buf_pts) + + GST_ELEMENT_CAST (sink)->base_time + gst_base_sink_get_latency (sink); + } + + if (GST_CLOCK_TIME_IS_VALID (buffer_clock)) { + GstClock *clock = gst_element_get_clock (GST_ELEMENT_CAST (sink)); + if (!gst_d3d11_ipc_clock_is_system (clock)) { + GstClockTime now_gst = gst_clock_get_time (clock); + GstClockTimeDiff converted = buffer_clock; + + converted -= now_gst; + converted += now_system; + + if (converted < 0) { + /* Shouldn't happen */ + GST_WARNING_OBJECT (self, "Negative buffer clock"); + pts = 0; + } else { + pts = converted; + } + } else { + /* buffer clock is already system time */ + pts = buffer_clock; + } + gst_object_unref (clock); + } + + ret = gst_d3d11_ipc_server_send_data (priv->server, priv->prepared_sample, + priv->layout, priv->resource_name, pts); + gst_clear_sample (&priv->prepared_sample); + + return ret; +} diff --git a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipcsink.h b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipcsink.h new file mode 100644 index 0000000000..f02a938147 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipcsink.h @@ -0,0 +1,33 @@ +/* GStreamer + * Copyright (C) 2023 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#include +#include +#include +#include + +G_BEGIN_DECLS + +#define GST_TYPE_D3D11_IPC_SINK (gst_d3d11_ipc_sink_get_type()) +G_DECLARE_FINAL_TYPE (GstD3D11IpcSink, gst_d3d11_ipc_sink, + GST, D3D11_IPC_SINK, GstBaseSink); + +G_END_DECLS diff --git a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipcsrc.cpp b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipcsrc.cpp new file mode 100644 index 0000000000..4042183cfa --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipcsrc.cpp @@ -0,0 +1,525 @@ +/* GStreamer + * Copyright (C) 2023 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/** + * SECTION:element-d3d11ipcsrc + * @title: d3d11ipcsrc + * @short_description: Direct3D11 Inter Process Communication (IPC) src + * + * d3d11ipcsrc imports Direct3D11 texture exported by peer d3d11ipcsrc element + * + * ## Example launch line + * ``` + * gst-launch-1.0 d3d11ipcsrc ! queue ! d3d11videosink + * ``` + * + * Since: 1.24 + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstd3d11ipcsrc.h" +#include "gstd3d11ipcclient.h" +#include "gstd3d11pluginutils.h" +#include +#include + +GST_DEBUG_CATEGORY_STATIC (gst_d3d11_ipc_src_debug); +#define GST_CAT_DEFAULT gst_d3d11_ipc_src_debug + +static GstStaticCaps pad_template_caps = +GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY, GST_D3D11_IPC_FORMATS)); + +enum +{ + PROP_0, + PROP_PIPE_NAME, + PROP_PROCESSING_DEADLINE, + PROP_IO_MODE, + PROP_CONN_TIMEOUT, +}; + +#define DEFAULT_PIPE_NAME "\\\\.\\pipe\\gst.d3d11.ipc" +#define DEFAULT_PROCESSING_DEADLINE (20 * GST_MSECOND) +#define DEFAULT_IO_MODE GST_D3D11_IPC_IO_COPY +#define DEFAULT_CONN_TIMEOUT 5 + +/* *INDENT-OFF* */ +struct GstD3D11IpcSrcPrivate +{ + GstD3D11Device *device = nullptr; + + GstD3D11IpcClient *client = nullptr; + GstCaps *caps = nullptr; + + GstVideoInfo info; + std::mutex lock; + bool flushing = false; + + /* properties */ + std::string pipe_name = DEFAULT_PIPE_NAME; + GstClockTime processing_deadline = DEFAULT_PROCESSING_DEADLINE; + GstD3D11IpcIOMode io_mode = DEFAULT_IO_MODE; + guint conn_timeout = DEFAULT_CONN_TIMEOUT; +}; +/* *INDENT-ON* */ + +struct _GstD3D11IpcSrc +{ + GstBaseSrc parent; + + GstD3D11IpcSrcPrivate *priv; +}; + +static void gst_d3d11_ipc_src_finalize (GObject * object); +static void gst_d3d11_ipc_src_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec); +static void gst_win32_video_src_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec); + +static GstClock *gst_d3d11_ipc_src_provide_clock (GstElement * elem); +static void gst_d3d11_ipc_src_set_context (GstElement * elem, + GstContext * context); + +static gboolean gst_d3d11_ipc_src_start (GstBaseSrc * src); +static gboolean gst_d3d11_ipc_src_stop (GstBaseSrc * src); +static gboolean gst_d3d11_ipc_src_unlock (GstBaseSrc * src); +static gboolean gst_d3d11_ipc_src_unlock_stop (GstBaseSrc * src); +static gboolean gst_d3d11_ipc_src_query (GstBaseSrc * src, GstQuery * query); +static GstCaps *gst_d3d11_ipc_src_get_caps (GstBaseSrc * src, GstCaps * filter); +static GstCaps *gst_d3d11_ipc_src_fixate (GstBaseSrc * src, GstCaps * caps); +static GstFlowReturn gst_d3d11_ipc_src_create (GstBaseSrc * src, guint64 offset, + guint size, GstBuffer ** buf); + +#define gst_d3d11_ipc_src_parent_class parent_class +G_DEFINE_TYPE (GstD3D11IpcSrc, gst_d3d11_ipc_src, GST_TYPE_BASE_SRC); + +static void +gst_d3d11_ipc_src_class_init (GstD3D11IpcSrcClass * klass) +{ + GObjectClass *object_class = G_OBJECT_CLASS (klass); + GstElementClass *element_class = GST_ELEMENT_CLASS (klass); + GstBaseSrcClass *src_class = GST_BASE_SRC_CLASS (klass); + GstCaps *caps; + + object_class->finalize = gst_d3d11_ipc_src_finalize; + object_class->set_property = gst_d3d11_ipc_src_set_property; + object_class->get_property = gst_win32_video_src_get_property; + + g_object_class_install_property (object_class, PROP_PIPE_NAME, + g_param_spec_string ("pipe-name", "Pipe Name", + "The name of Win32 named pipe to communicate with clients. " + "Validation of the pipe name is caller's responsibility", + DEFAULT_PIPE_NAME, (GParamFlags) (G_PARAM_READWRITE | + G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_READY))); + + g_object_class_install_property (object_class, PROP_PROCESSING_DEADLINE, + g_param_spec_uint64 ("processing-deadline", "Processing deadline", + "Maximum processing time for a buffer in nanoseconds", 0, G_MAXUINT64, + DEFAULT_PROCESSING_DEADLINE, (GParamFlags) (G_PARAM_READWRITE | + G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + + g_object_class_install_property (object_class, PROP_IO_MODE, + g_param_spec_enum ("io-mode", "IO Mode", "Memory I/O mode to use", + GST_TYPE_D3D11_IPC_IO_MODE, DEFAULT_IO_MODE, + (GParamFlags) (G_PARAM_READWRITE | GST_PARAM_MUTABLE_READY | + G_PARAM_STATIC_STRINGS))); + + g_object_class_install_property (object_class, PROP_CONN_TIMEOUT, + g_param_spec_uint ("connection-timeout", "Connection Timeout", + "Connection timeout in seconds (0 = never timeout)", 0, G_MAXINT, + DEFAULT_CONN_TIMEOUT, + (GParamFlags) (G_PARAM_READWRITE | GST_PARAM_MUTABLE_READY | + G_PARAM_STATIC_STRINGS))); + + gst_element_class_set_static_metadata (element_class, + "D3D11 IPC Src", "Source/Video", + "Receive D3D11 memory from the d3d11ipcsrc element", + "Seungha Yang "); + + caps = gst_d3d11_get_updated_template_caps (&pad_template_caps); + gst_element_class_add_pad_template (element_class, + gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS, caps)); + gst_caps_unref (caps); + + element_class->provide_clock = + GST_DEBUG_FUNCPTR (gst_d3d11_ipc_src_provide_clock); + element_class->set_context = + GST_DEBUG_FUNCPTR (gst_d3d11_ipc_src_set_context); + + src_class->start = GST_DEBUG_FUNCPTR (gst_d3d11_ipc_src_start); + src_class->stop = GST_DEBUG_FUNCPTR (gst_d3d11_ipc_src_stop); + src_class->unlock = GST_DEBUG_FUNCPTR (gst_d3d11_ipc_src_unlock); + src_class->unlock_stop = GST_DEBUG_FUNCPTR (gst_d3d11_ipc_src_unlock_stop); + src_class->query = GST_DEBUG_FUNCPTR (gst_d3d11_ipc_src_query); + src_class->get_caps = GST_DEBUG_FUNCPTR (gst_d3d11_ipc_src_get_caps); + src_class->fixate = GST_DEBUG_FUNCPTR (gst_d3d11_ipc_src_fixate); + src_class->create = GST_DEBUG_FUNCPTR (gst_d3d11_ipc_src_create); + + GST_DEBUG_CATEGORY_INIT (gst_d3d11_ipc_src_debug, "d3d11ipcsrc", + 0, "d3d11ipcsrc"); + + gst_type_mark_as_plugin_api (GST_TYPE_D3D11_IPC_IO_MODE, + (GstPluginAPIFlags) 0); +} + +static void +gst_d3d11_ipc_src_init (GstD3D11IpcSrc * self) +{ + gst_base_src_set_format (GST_BASE_SRC (self), GST_FORMAT_TIME); + gst_base_src_set_live (GST_BASE_SRC (self), TRUE); + + self->priv = new GstD3D11IpcSrcPrivate (); + + GST_OBJECT_FLAG_SET (self, GST_ELEMENT_FLAG_PROVIDE_CLOCK); + GST_OBJECT_FLAG_SET (self, GST_ELEMENT_FLAG_REQUIRE_CLOCK); +} + +static void +gst_d3d11_ipc_src_finalize (GObject * object) +{ + GstD3D11IpcSrc *self = GST_D3D11_IPC_SRC (object); + + delete self->priv; + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static void +gst_d3d11_ipc_src_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstD3D11IpcSrc *self = GST_D3D11_IPC_SRC (object); + GstD3D11IpcSrcPrivate *priv = self->priv; + std::unique_lock < std::mutex > lk (priv->lock); + + switch (prop_id) { + case PROP_PIPE_NAME: + { + const gchar *pipe_name = g_value_get_string (value); + priv->pipe_name.clear (); + + if (pipe_name) + priv->pipe_name = pipe_name; + else + priv->pipe_name = DEFAULT_PIPE_NAME; + break; + } + case PROP_PROCESSING_DEADLINE: + { + GstClockTime prev_val, new_val; + prev_val = priv->processing_deadline; + new_val = g_value_get_uint64 (value); + priv->processing_deadline = new_val; + + if (prev_val != new_val) { + lk.unlock (); + GST_DEBUG_OBJECT (self, "Posting latency message"); + gst_element_post_message (GST_ELEMENT_CAST (self), + gst_message_new_latency (GST_OBJECT_CAST (self))); + } + break; + } + case PROP_IO_MODE: + priv->io_mode = (GstD3D11IpcIOMode) g_value_get_enum (value); + break; + case PROP_CONN_TIMEOUT: + priv->conn_timeout = g_value_get_uint (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_win32_video_src_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstD3D11IpcSrc *self = GST_D3D11_IPC_SRC (object); + GstD3D11IpcSrcPrivate *priv = self->priv; + std::lock_guard < std::mutex > lk (priv->lock); + + switch (prop_id) { + case PROP_PIPE_NAME: + g_value_set_string (value, priv->pipe_name.c_str ()); + break; + case PROP_PROCESSING_DEADLINE: + g_value_set_uint64 (value, priv->processing_deadline); + break; + case PROP_IO_MODE: + g_value_set_enum (value, priv->io_mode); + break; + case PROP_CONN_TIMEOUT: + g_value_set_uint (value, priv->conn_timeout); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static GstClock * +gst_d3d11_ipc_src_provide_clock (GstElement * elem) +{ + return gst_system_clock_obtain (); +} + +static void +gst_d3d11_ipc_src_set_context (GstElement * elem, GstContext * context) +{ + GstD3D11IpcSrc *self = GST_D3D11_IPC_SRC (elem); + GstD3D11IpcSrcPrivate *priv = self->priv; + + gst_d3d11_handle_set_context (elem, context, -1, &priv->device); + + GST_ELEMENT_CLASS (parent_class)->set_context (elem, context); +} + +static gboolean +gst_d3d11_ipc_src_start (GstBaseSrc * src) +{ + GstD3D11IpcSrc *self = GST_D3D11_IPC_SRC (src); + GstD3D11IpcSrcPrivate *priv = self->priv; + + GST_DEBUG_OBJECT (self, "Start"); + + if (!gst_d3d11_ensure_element_data (GST_ELEMENT_CAST (self), + -1, &priv->device)) { + GST_ERROR_OBJECT (self, "Couldn't get D3D11 context"); + return FALSE; + } + + std::lock_guard < std::mutex > lk (priv->lock); + priv->client = gst_d3d11_ipc_client_new (priv->pipe_name, priv->device, + priv->io_mode, priv->conn_timeout); + + return TRUE; +} + +static gboolean +gst_d3d11_ipc_src_stop (GstBaseSrc * src) +{ + GstD3D11IpcSrc *self = GST_D3D11_IPC_SRC (src); + GstD3D11IpcSrcPrivate *priv = self->priv; + + std::lock_guard < std::mutex > lk (priv->lock); + + GST_DEBUG_OBJECT (self, "Stop"); + + if (priv->client) + gst_d3d11_ipc_client_stop (priv->client); + + gst_clear_object (&priv->client); + gst_clear_object (&priv->device); + gst_clear_caps (&priv->caps); + + return TRUE; +} + +static gboolean +gst_d3d11_ipc_src_unlock (GstBaseSrc * src) +{ + GstD3D11IpcSrc *self = GST_D3D11_IPC_SRC (src); + GstD3D11IpcSrcPrivate *priv = self->priv; + + GST_DEBUG_OBJECT (self, "Unlock"); + + std::lock_guard < std::mutex > lk (priv->lock); + priv->flushing = true; + if (priv->client) + gst_d3d11_ipc_client_set_flushing (priv->client, true); + + return TRUE; +} + +static gboolean +gst_d3d11_ipc_src_unlock_stop (GstBaseSrc * src) +{ + GstD3D11IpcSrc *self = GST_D3D11_IPC_SRC (src); + GstD3D11IpcSrcPrivate *priv = self->priv; + + GST_DEBUG_OBJECT (self, "Unlock stop"); + + std::lock_guard < std::mutex > lk (priv->lock); + priv->flushing = false; + if (priv->client) + gst_d3d11_ipc_client_set_flushing (priv->client, false); + + return TRUE; +} + +static gboolean +gst_d3d11_ipc_src_query (GstBaseSrc * src, GstQuery * query) +{ + GstD3D11IpcSrc *self = GST_D3D11_IPC_SRC (src); + GstD3D11IpcSrcPrivate *priv = self->priv; + + switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_LATENCY: + { + std::lock_guard < std::mutex > lk (priv->lock); + if (GST_CLOCK_TIME_IS_VALID (priv->processing_deadline)) { + gst_query_set_latency (query, TRUE, priv->processing_deadline, + GST_CLOCK_TIME_NONE); + } else { + gst_query_set_latency (query, TRUE, 0, 0); + } + return TRUE; + } + case GST_QUERY_CONTEXT: + if (gst_d3d11_handle_context_query (GST_ELEMENT (self), query, + priv->device)) { + return TRUE; + } + break; + default: + break; + } + + return GST_BASE_SRC_CLASS (parent_class)->query (src, query); +} + +static GstCaps * +gst_d3d11_ipc_src_get_caps (GstBaseSrc * src, GstCaps * filter) +{ + GstD3D11IpcSrc *self = GST_D3D11_IPC_SRC (src); + GstD3D11IpcSrcPrivate *priv = self->priv; + GstD3D11IpcClient *client = nullptr; + GstCaps *caps = nullptr; + + GST_DEBUG_OBJECT (self, "Get caps"); + + priv->lock.lock (); + if (priv->caps) + caps = gst_caps_ref (priv->caps); + else if (priv->client) + client = (GstD3D11IpcClient *) gst_object_ref (priv->client); + priv->lock.unlock (); + + if (!caps && client) + caps = gst_d3d11_ipc_client_get_caps (priv->client); + + if (!caps) + caps = gst_pad_get_pad_template_caps (GST_BASE_SRC_PAD (src)); + + if (filter) { + GstCaps *tmp = gst_caps_intersect_full (filter, + caps, GST_CAPS_INTERSECT_FIRST); + gst_caps_unref (caps); + caps = tmp; + } + + gst_clear_object (&client); + GST_DEBUG_OBJECT (self, "Returning caps %" GST_PTR_FORMAT, caps); + + return caps; +} + +static GstCaps * +gst_d3d11_ipc_src_fixate (GstBaseSrc * src, GstCaps * caps) +{ + /* We don't negotiate with server. In here, we do fixate resolution to + * 320 x 240 (same as default of videotestsrc) which makes a little more + * sense than 1x1 */ + caps = gst_caps_make_writable (caps); + + for (guint i = 0; i < gst_caps_get_size (caps); i++) { + GstStructure *s = gst_caps_get_structure (caps, i); + + gst_structure_fixate_field_nearest_int (s, "width", 320); + gst_structure_fixate_field_nearest_int (s, "height", 240); + } + + return gst_caps_fixate (caps); +} + +static GstFlowReturn +gst_d3d11_ipc_src_create (GstBaseSrc * src, guint64 offset, guint size, + GstBuffer ** buf) +{ + GstD3D11IpcSrc *self = GST_D3D11_IPC_SRC (src); + GstD3D11IpcSrcPrivate *priv = self->priv; + GstFlowReturn ret; + GstSample *sample = nullptr; + GstCaps *caps; + GstClock *clock; + bool is_system_clock = true; + GstClockTime pts; + GstClockTime base_time; + GstClockTime now_system; + GstClockTime now_gst; + GstClockTime remote_pts; + GstBuffer *buffer; + + ret = gst_d3d11_ipc_client_run (priv->client); + if (ret != GST_FLOW_OK) + return ret; + + ret = gst_d3d11_ipc_client_get_sample (priv->client, &sample); + if (ret != GST_FLOW_OK) + return ret; + + now_system = gst_d3d11_ipc_clock_now (); + clock = gst_element_get_clock (GST_ELEMENT_CAST (self)); + now_gst = gst_clock_get_time (clock); + base_time = GST_ELEMENT_CAST (self)->base_time; + is_system_clock = gst_d3d11_ipc_clock_is_system (clock); + gst_object_unref (clock); + + buffer = gst_sample_get_buffer (sample); + remote_pts = GST_BUFFER_PTS (buffer); + + if (!is_system_clock) { + GstClockTimeDiff now_pts = now_gst - base_time + remote_pts - now_system; + + if (now_pts >= 0) + pts = now_pts; + else + pts = 0; + } else { + if (remote_pts >= base_time) { + pts = remote_pts - base_time; + } else { + GST_WARNING_OBJECT (self, + "Remote clock is smaller than our base time, remote %" + GST_TIME_FORMAT ", base_time %" GST_TIME_FORMAT, + GST_TIME_ARGS (remote_pts), GST_TIME_ARGS (base_time)); + pts = 0; + } + } + + GST_BUFFER_PTS (buffer) = pts; + + std::unique_lock < std::mutex > lk (priv->lock); + caps = gst_sample_get_caps (sample); + if (!priv->caps || !gst_caps_is_equal (priv->caps, caps)) { + gst_caps_replace (&priv->caps, caps); + lk.unlock (); + gst_base_src_set_caps (src, priv->caps); + } + + *buf = gst_buffer_ref (buffer); + gst_sample_unref (sample); + + return GST_FLOW_OK; +} diff --git a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipcsrc.h b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipcsrc.h new file mode 100644 index 0000000000..24c82527d3 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11ipcsrc.h @@ -0,0 +1,33 @@ +/* GStreamer + * Copyright (C) 2023 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#include +#include +#include +#include + +G_BEGIN_DECLS + +#define GST_TYPE_D3D11_IPC_SRC (gst_d3d11_ipc_src_get_type()) +G_DECLARE_FINAL_TYPE (GstD3D11IpcSrc, gst_d3d11_ipc_src, + GST, D3D11_IPC_SRC, GstBaseSrc); + +G_END_DECLS diff --git a/subprojects/gst-plugins-bad/sys/d3d11/meson.build b/subprojects/gst-plugins-bad/sys/d3d11/meson.build index bc3ea06959..bfea59c8ad 100644 --- a/subprojects/gst-plugins-bad/sys/d3d11/meson.build +++ b/subprojects/gst-plugins-bad/sys/d3d11/meson.build @@ -8,6 +8,11 @@ d3d11_sources = [ 'gstd3d11download.cpp', 'gstd3d11h264dec.cpp', 'gstd3d11h265dec.cpp', + 'gstd3d11ipc.cpp', + 'gstd3d11ipcclient.cpp', + 'gstd3d11ipcserver.cpp', + 'gstd3d11ipcsink.cpp', + 'gstd3d11ipcsrc.cpp', 'gstd3d11mpeg2dec.cpp', 'gstd3d11overlay.cpp', 'gstd3d11overlaycompositor.cpp', diff --git a/subprojects/gst-plugins-bad/sys/d3d11/plugin.cpp b/subprojects/gst-plugins-bad/sys/d3d11/plugin.cpp index 9d9963b32b..89fe8a08e4 100644 --- a/subprojects/gst-plugins-bad/sys/d3d11/plugin.cpp +++ b/subprojects/gst-plugins-bad/sys/d3d11/plugin.cpp @@ -76,6 +76,8 @@ #include "gstd3d11deinterlace.h" #include "gstd3d11testsrc.h" #include "gstd3d11overlay.h" +#include "gstd3d11ipcsink.h" +#include "gstd3d11ipcsrc.h" #if !GST_D3D11_WINAPI_ONLY_APP #include "gstd3d11screencapturesrc.h" @@ -238,6 +240,10 @@ plugin_init (GstPlugin * plugin) "d3d11testsrc", GST_RANK_NONE, GST_TYPE_D3D11_TEST_SRC); gst_element_register (plugin, "d3d11overlay", GST_RANK_NONE, GST_TYPE_D3D11_OVERLAY); + gst_element_register (plugin, + "d3d11ipcsink", GST_RANK_NONE, GST_TYPE_D3D11_IPC_SINK); + gst_element_register (plugin, + "d3d11ipcsrc", GST_RANK_NONE, GST_TYPE_D3D11_IPC_SRC); #if !GST_D3D11_WINAPI_ONLY_APP if (gst_d3d11_is_windows_8_or_greater ()) {