gstreamer-rs/examples/src/bin/gtkvideooverlay.rs

// This example demonstrates another type of combination of gtk and gstreamer,
// in comparison to the gtksink example.
// This example uses regions that are managed by the window system, and uses
// the window system's api to insert a videostream into these regions.
// So essentially, the window system of the system overlays our gui with
// the video frames - within the region that we tell it to use.
// Disadvantage of this method is, that it's highly platform specific, since
// the big platforms all have their own window system. Thus, this example
// has special code to handle differences between platforms.
// Windows could theoretically be supported by this example, but is not yet implemented.
// One of the very few (if not the single one) platform, that can not provide the API
// needed for this are Linux desktops using Wayland.
// TODO: Add Windows support
// In this case, a testvideo is displayed within our gui, using the
// following pipeline:

// {videotestsrc} - {xvimagesink(on linux)}
// {videotestsrc} - {glimagesink(on mac)}

use std::{cell::RefCell, os::raw::c_void, process};

use gio::prelude::*;
use gst_video::prelude::*;
use gtk::prelude::*;

#[cfg(all(target_os = "linux", feature = "gtkvideooverlay-x11"))]
fn create_video_sink() -> gst::Element {
    // When we are on linux with the Xorg display server, we use the
    // X11 protocol's XV extension, which allows to overlay regions
    // with video streams. For this, we use the xvimagesink element.
    gst::ElementFactory::make("xvimagesink").build().unwrap()
}
#[cfg(all(target_os = "linux", feature = "gtkvideooverlay-x11"))]
fn set_window_handle(video_overlay: &gst_video::VideoOverlay, gdk_window: &gdk::Window) {
    let display_type_name = gdk_window.display().type_().name();

    // Check if we're using X11 or ...
    if display_type_name == "GdkX11Display" {
        extern "C" {
            pub fn gdk_x11_window_get_xid(window: *mut glib::gobject_ffi::GObject) -> *mut c_void;
        }

        // This is unsafe because the "window handle" we pass here is basically like a raw pointer.
        // If a wrong value were to be passed here (and you can pass any integer), then the window
        // system will most likely cause the application to crash.
        #[allow(clippy::cast_ptr_alignment)]
        unsafe {
            // Here we ask gdk what native window handle we got assigned for
            // our video region from the window system, and then we will
            // pass this unique identifier to the overlay provided by our
            // sink - so the sink can then arrange the overlay.
            let xid = gdk_x11_window_get_xid(gdk_window.as_ptr() as *mut _);
            video_overlay.set_window_handle(xid as usize);
        }
    } else {
        println!("Add support for display type '{display_type_name}'");
        process::exit(-1);
    }
}

#[cfg(all(target_os = "macos", feature = "gtkvideooverlay-quartz"))]
fn create_video_sink() -> gst::Element {
    // On Mac, this is done by overlaying a window region with an
    // OpenGL-texture, using the glimagesink element.
    gst::ElementFactory::make("glimagesink").build().unwrap()
}

#[cfg(all(target_os = "macos", feature = "gtkvideooverlay-quartz"))]
fn set_window_handle(video_overlay: &gst_video::VideoOverlay, gdk_window: &gdk::Window) {
    let display_type_name = gdk_window.display().type_().name();

    if display_type_name == "GdkQuartzDisplay" {
        extern "C" {
            pub fn gdk_quartz_window_get_nsview(
                window: *mut glib::gobject_ffi::GObject,
            ) -> *mut c_void;
        }

        // This is unsafe because the "window handle" we pass here is basically like a raw pointer.
        // If a wrong value were to be passed here (and you can pass any integer), then the window
        // system will most likely cause the application to crash.
        #[allow(clippy::cast_ptr_alignment)]
        unsafe {
            // Here we ask gdk what native window handle we got assigned for
            // our video region from the windowing system, and then we will
            // pass this unique identifier to the overlay provided by our
            // sink - so the sink can then arrange the overlay.
            let window = gdk_quartz_window_get_nsview(gdk_window.as_ptr() as *mut _);
            video_overlay.set_window_handle(window as usize);
        }
    } else {
        println!("Unsupported display type '{}", display_type_name);
        process::exit(-1);
    }
}

fn create_ui(app: &gtk::Application) {
    let pipeline = gst::Pipeline::default();
    let src = gst::ElementFactory::make("videotestsrc").build().unwrap();

    // Since using the window system to overlay our gui window is making
    // direct contact with the windowing system, this is highly platform-
    // specific. This example supports Linux and Mac (using X11 and Quartz).
    let sink = create_video_sink();

    pipeline.add_many([&src, &sink]).unwrap();
    src.link(&sink).unwrap();

    // First, we create our gtk window - which will contain a region where
    // our overlaid video will be displayed in.
    let window = gtk::Window::new(gtk::WindowType::Toplevel);
    window.set_default_size(320, 240);

    let vbox = gtk::Box::new(gtk::Orientation::Vertical, 0);

    // This creates the widget we will display our overlay in.
    // Later, we will try to tell our window system about this region, so
    // it can overlay it with our video stream.
    let video_window = gtk::DrawingArea::new();
    video_window.set_size_request(320, 240);

    // Use the platform-specific sink to create our overlay.
    // Since we only use the video_overlay in the closure below, we need a weak reference.
    // !!ATTENTION!!:
    // It might seem appealing to use .clone() here, because that greatly
    // simplifies the code within the callback. What this actually does, however, is creating
    // a memory leak.
    let video_overlay = sink
        .dynamic_cast::<gst_video::VideoOverlay>()
        .unwrap()
        .downgrade();
    // Connect to this widget's realize signal, which will be emitted
    // after its display has been initialized. This is necessary, because
    // the window system doesn't know about our region until it was initialized.
    video_window.connect_realize(move |video_window| {
        // Here we temporarily retrieve a strong reference on the video-overlay from the
        // weak reference that we moved into the closure.
        let video_overlay = match video_overlay.upgrade() {
            Some(video_overlay) => video_overlay,
            None => return,
        };

        // Gtk uses gdk under the hood, to handle its drawing. Drawing regions are
        // called gdk windows. We request this underlying drawing region from the
        // widget we will overlay with our video.
        let gdk_window = video_window.window().unwrap();

        // This is where we tell our window system about the drawing-region we
        // want it to overlay. Most often, the window system would only know
        // about our most outer region (or: our window).
        if !gdk_window.ensure_native() {
            println!("Can't create native window for widget");
            process::exit(-1);
        }

        set_window_handle(&video_overlay, &gdk_window);
    });

    vbox.pack_start(&video_window, true, true, 0);

    let label = gtk::Label::new(Some("Position: 00:00:00"));
    vbox.pack_start(&label, true, true, 5);
    window.add(&vbox);

    window.show_all();

    app.add_window(&window);

    // Need to move a new reference into the closure.
    // !!ATTENTION!!:
    // It might seem appealing to use pipeline.clone() here, because that greatly
    // simplifies the code within the callback. What this actually does, however, is creating
    // a memory leak. The clone of a pipeline is a new strong reference on the pipeline.
    // Storing this strong reference of the pipeline within the callback (we are moving it in!),
    // which is in turn stored in another strong reference on the pipeline is creating a
    // reference cycle.
    // DO NOT USE pipeline.clone() TO USE THE PIPELINE WITHIN A CALLBACK
    let pipeline_weak = pipeline.downgrade();
    // Add a timeout to the main loop that will periodically (every 500ms) be
    // executed. This will query the current position within the stream from
    // the underlying pipeline, and display it in our gui.
    // Since this closure is called by the mainloop thread, we are allowed
    // to modify the gui widgets here.
    let timeout_id = glib::timeout_add_local(std::time::Duration::from_millis(500), move || {
        // Here we temporarily retrieve a strong reference on the pipeline from the weak one
        // we moved into this callback.
        let pipeline = match pipeline_weak.upgrade() {
            Some(pipeline) => pipeline,
            None => return glib::Continue(false),
        };

        // Query the current playing position from the underlying pipeline.
        let position = pipeline.query_position::<gst::ClockTime>();
        // Display the playing position in the gui.
        label.set_text(&format!("Position: {:.0}", position.display()));
        // Tell the timeout to continue calling this callback.
        glib::Continue(true)
    });

    let bus = pipeline.bus().unwrap();

    pipeline
        .set_state(gst::State::Playing)
        .expect("Unable to set the pipeline to the `Playing` state");

    let app_weak = app.downgrade();
    let _bus_watch = bus
        .add_watch_local(move |_, msg| {
            use gst::MessageView;

            let app = match app_weak.upgrade() {
                Some(app) => app,
                None => return glib::Continue(false),
            };

            match msg.view() {
                MessageView::Eos(..) => app.quit(),
                MessageView::Error(err) => {
                    println!(
                        "Error from {:?}: {} ({:?})",
                        err.src().map(|s| s.path_string()),
                        err.error(),
                        err.debug()
                    );
                    app.quit();
                }
                _ => (),
            };

            glib::Continue(true)
        })
        .expect("Failed to add bus watch");

    // Pipeline reference is owned by the closure below, so will be
    // destroyed once the app is destroyed
    let timeout_id = RefCell::new(Some(timeout_id));
    let pipeline = RefCell::new(Some(pipeline));
    app.connect_shutdown(move |_| {
        // Optional, by manually destroying the window here we ensure that
        // the gst element is destroyed when shutting down instead of having to wait
        // for the process to terminate, allowing us to use the leaks tracer.
        unsafe {
            window.destroy();
        }

        // GTK will keep the Application alive for the whole process lifetime.
        // Wrapping the pipeline in a RefCell<Option<_>> and removing it from it here
        // ensures the pipeline is actually destroyed when shutting down, allowing us
        // to use the leaks tracer for example.
        if let Some(pipeline) = pipeline.borrow_mut().take() {
            pipeline
                .set_state(gst::State::Null)
                .expect("Unable to set the pipeline to the `Null` state");
        }

        if let Some(timeout_id) = timeout_id.borrow_mut().take() {
            timeout_id.remove();
        }
    });
}

fn main() -> glib::ExitCode {
    #[cfg(not(unix))]
    {
        println!("Add support for target platform");
        process::exit(-1);
    }

    // Initialize gstreamer and the gtk widget toolkit libraries.
    gst::init().unwrap();
    gtk::init().unwrap();

    let res = {
        let app = gtk::Application::new(None, gio::ApplicationFlags::FLAGS_NONE);

        app.connect_activate(create_ui);
        app.run()
    };

    // Optional, can be used to detect leaks using the leaks tracer
    unsafe {
        gst::deinit();
    }

    res
}