gstreamer-rs/examples/src/glupload.rs

// This example demonstrates how to output GL textures, within an
// EGL/X11 context provided by the application, and render those
// textures in the GL application.

// {videotestsrc} - { glsinkbin }

use gst::element_error;

use gst_gl::prelude::*;

use std::ffi::CStr;
use std::mem;
use std::ptr;
use std::sync;

use anyhow::Error;
use derive_more::{Display, Error};

#[derive(Debug, Display, Error)]
#[display(fmt = "Missing element {}", _0)]
struct MissingElement(#[error(not(source))] &'static str);

#[derive(Debug, Display, Error)]
#[display(fmt = "Received error from {}: {} (debug: {:?})", src, error, debug)]
struct ErrorMessage {
    src: String,
    error: String,
    debug: Option<String>,
    source: glib::Error,
}

#[rustfmt::skip]
static VERTICES: [f32; 20] = [
     1.0,  1.0, 0.0, 1.0, 0.0,
    -1.0,  1.0, 0.0, 0.0, 0.0,
    -1.0, -1.0, 0.0, 0.0, 1.0,
     1.0, -1.0, 0.0, 1.0, 1.0,
];

static INDICES: [u16; 6] = [0, 1, 2, 0, 2, 3];

#[rustfmt::skip]
static IDENTITY: [f32; 16] = [
    1.0, 0.0, 0.0, 0.0,
    0.0, 1.0, 0.0, 0.0,
    0.0, 0.0, 1.0, 0.0,
    0.0, 0.0, 0.0, 1.0,
];

const VS_SRC: &[u8] = b"
uniform mat4 u_transformation;
attribute vec4 a_position;
attribute vec2 a_texcoord;
varying vec2 v_texcoord;

void main() {
    gl_Position = u_transformation * a_position;
    v_texcoord = a_texcoord;
}
\0";

const FS_SRC: &[u8] = b"
#ifdef GL_ES
precision mediump float;
#endif
varying vec2 v_texcoord;
uniform sampler2D tex;

void main() {
    gl_FragColor = texture2D(tex, v_texcoord);
}
\0";

#[allow(clippy::unreadable_literal)]
#[allow(clippy::unused_unit)]
#[allow(clippy::too_many_arguments)]
#[allow(clippy::manual_non_exhaustive)]
#[allow(clippy::upper_case_acronyms)]
pub(crate) mod gl {
    pub use self::Gles2 as Gl;
    include!(concat!(env!("OUT_DIR"), "/test_gl_bindings.rs"));
}

struct Gl {
    gl: gl::Gl,
    program: gl::types::GLuint,
    attr_position: gl::types::GLint,
    attr_texture: gl::types::GLint,
    vao: Option<gl::types::GLuint>,
    vertex_buffer: gl::types::GLuint,
    vbo_indices: gl::types::GLuint,
}

impl Gl {
    fn draw_frame(&self, texture_id: gl::types::GLuint) {
        unsafe {
            // render
            self.gl.ClearColor(0.0, 0.0, 0.0, 1.0);
            self.gl.Clear(gl::COLOR_BUFFER_BIT);

            self.gl.BlendColor(0.0, 0.0, 0.0, 1.0);
            if self.gl.BlendFuncSeparate.is_loaded() {
                self.gl.BlendFuncSeparate(
                    gl::SRC_ALPHA,
                    gl::CONSTANT_COLOR,
                    gl::ONE,
                    gl::ONE_MINUS_SRC_ALPHA,
                );
            } else {
                self.gl.BlendFunc(gl::SRC_ALPHA, gl::CONSTANT_COLOR);
            }
            self.gl.BlendEquation(gl::FUNC_ADD);
            self.gl.Enable(gl::BLEND);

            self.gl.UseProgram(self.program);

            if self.gl.BindVertexArray.is_loaded() {
                self.gl.BindVertexArray(self.vao.unwrap());
            }

            {
                self.gl
                    .BindBuffer(gl::ELEMENT_ARRAY_BUFFER, self.vbo_indices);
                self.gl.BindBuffer(gl::ARRAY_BUFFER, self.vertex_buffer);

                // Load the vertex position
                self.gl.VertexAttribPointer(
                    self.attr_position as gl::types::GLuint,
                    3,
                    gl::FLOAT,
                    gl::FALSE,
                    (5 * mem::size_of::<f32>()) as gl::types::GLsizei,
                    ptr::null(),
                );

                // Load the texture coordinate
                self.gl.VertexAttribPointer(
                    self.attr_texture as gl::types::GLuint,
                    2,
                    gl::FLOAT,
                    gl::FALSE,
                    (5 * mem::size_of::<f32>()) as gl::types::GLsizei,
                    (3 * mem::size_of::<f32>()) as *const () as *const _,
                );

                self.gl.EnableVertexAttribArray(self.attr_position as _);
                self.gl.EnableVertexAttribArray(self.attr_texture as _);
            }

            self.gl.ActiveTexture(gl::TEXTURE0);
            self.gl.BindTexture(gl::TEXTURE_2D, texture_id);

            let location = self
                .gl
                .GetUniformLocation(self.program, b"tex\0".as_ptr() as *const _);
            self.gl.Uniform1i(location, 0);

            let location = self
                .gl
                .GetUniformLocation(self.program, b"u_transformation\0".as_ptr() as *const _);
            self.gl
                .UniformMatrix4fv(location, 1, gl::FALSE, IDENTITY.as_ptr() as *const _);

            self.gl
                .DrawElements(gl::TRIANGLES, 6, gl::UNSIGNED_SHORT, ptr::null());

            self.gl.BindTexture(gl::TEXTURE_2D, 0);
            self.gl.UseProgram(0);

            if self.gl.BindVertexArray.is_loaded() {
                self.gl.BindVertexArray(0);
            }

            {
                self.gl.BindBuffer(gl::ELEMENT_ARRAY_BUFFER, 0);
                self.gl.BindBuffer(gl::ARRAY_BUFFER, 0);

                self.gl.DisableVertexAttribArray(self.attr_position as _);
                self.gl.DisableVertexAttribArray(self.attr_texture as _);
            }
        }
    }

    fn resize(&self, size: glutin::dpi::PhysicalSize<u32>) {
        unsafe {
            self.gl
                .Viewport(0, 0, size.width as i32, size.height as i32);
        }
    }
}

fn load(gl_context: &glutin::WindowedContext<glutin::PossiblyCurrent>) -> Gl {
    let gl = gl::Gl::load_with(|ptr| gl_context.get_proc_address(ptr) as *const _);

    let version = unsafe {
        let data = CStr::from_ptr(gl.GetString(gl::VERSION) as *const _)
            .to_bytes()
            .to_vec();
        String::from_utf8(data).unwrap()
    };

    println!("OpenGL version {}", version);

    let (program, attr_position, attr_texture, vao, vertex_buffer, vbo_indices) = unsafe {
        let vs = gl.CreateShader(gl::VERTEX_SHADER);
        gl.ShaderSource(vs, 1, [VS_SRC.as_ptr() as *const _].as_ptr(), ptr::null());
        gl.CompileShader(vs);

        let fs = gl.CreateShader(gl::FRAGMENT_SHADER);
        gl.ShaderSource(fs, 1, [FS_SRC.as_ptr() as *const _].as_ptr(), ptr::null());
        gl.CompileShader(fs);

        let program = gl.CreateProgram();
        gl.AttachShader(program, vs);
        gl.AttachShader(program, fs);
        gl.LinkProgram(program);

        {
            let mut success: gl::types::GLint = 1;
            gl.GetProgramiv(fs, gl::LINK_STATUS, &mut success);
            assert!(success != 0);
        }

        let attr_position = gl.GetAttribLocation(program, b"a_position\0".as_ptr() as *const _);
        let attr_texture = gl.GetAttribLocation(program, b"a_texcoord\0".as_ptr() as *const _);

        let vao = if gl.BindVertexArray.is_loaded() {
            let mut vao = mem::MaybeUninit::uninit();
            gl.GenVertexArrays(1, vao.as_mut_ptr());
            let vao = vao.assume_init();
            gl.BindVertexArray(vao);
            Some(vao)
        } else {
            None
        };

        let mut vertex_buffer = mem::MaybeUninit::uninit();
        gl.GenBuffers(1, vertex_buffer.as_mut_ptr());
        let vertex_buffer = vertex_buffer.assume_init();
        gl.BindBuffer(gl::ARRAY_BUFFER, vertex_buffer);
        gl.BufferData(
            gl::ARRAY_BUFFER,
            (VERTICES.len() * mem::size_of::<f32>()) as gl::types::GLsizeiptr,
            VERTICES.as_ptr() as *const _,
            gl::STATIC_DRAW,
        );

        let mut vbo_indices = mem::MaybeUninit::uninit();
        gl.GenBuffers(1, vbo_indices.as_mut_ptr());
        let vbo_indices = vbo_indices.assume_init();
        gl.BindBuffer(gl::ELEMENT_ARRAY_BUFFER, vbo_indices);
        gl.BufferData(
            gl::ELEMENT_ARRAY_BUFFER,
            (INDICES.len() * mem::size_of::<u16>()) as gl::types::GLsizeiptr,
            INDICES.as_ptr() as *const _,
            gl::STATIC_DRAW,
        );

        if gl.BindVertexArray.is_loaded() {
            gl.BindBuffer(gl::ELEMENT_ARRAY_BUFFER, vbo_indices);
            gl.BindBuffer(gl::ARRAY_BUFFER, vertex_buffer);

            // Load the vertex position
            gl.VertexAttribPointer(
                attr_position as gl::types::GLuint,
                3,
                gl::FLOAT,
                gl::FALSE,
                (5 * mem::size_of::<f32>()) as gl::types::GLsizei,
                ptr::null(),
            );

            // Load the texture coordinate
            gl.VertexAttribPointer(
                attr_texture as gl::types::GLuint,
                2,
                gl::FLOAT,
                gl::FALSE,
                (5 * mem::size_of::<f32>()) as gl::types::GLsizei,
                (3 * mem::size_of::<f32>()) as *const () as *const _,
            );

            gl.EnableVertexAttribArray(attr_position as _);
            gl.EnableVertexAttribArray(attr_texture as _);

            gl.BindVertexArray(0);
        }

        gl.BindBuffer(gl::ELEMENT_ARRAY_BUFFER, 0);
        gl.BindBuffer(gl::ARRAY_BUFFER, 0);

        (
            program,
            attr_position,
            attr_texture,
            vao,
            vertex_buffer,
            vbo_indices,
        )
    };

    Gl {
        gl,
        program,
        attr_position,
        attr_texture,
        vao,
        vertex_buffer,
        vbo_indices,
    }
}

#[derive(Debug)]
enum Message {
    Sample(gst::Sample),
    BusEvent,
}

pub(crate) struct App {
    pipeline: gst::Pipeline,
    appsink: gst_app::AppSink,
    glupload: gst::Element,
    bus: gst::Bus,
    event_loop: glutin::event_loop::EventLoop<Message>,
    windowed_context: glutin::WindowedContext<glutin::PossiblyCurrent>,
    shared_context: gst_gl::GLContext,
}

impl App {
    pub(crate) fn new(gl_element: Option<&gst::Element>) -> Result<App, Error> {
        gst::init()?;

        let (pipeline, appsink, glupload) = App::create_pipeline(gl_element)?;
        let bus = pipeline
            .bus()
            .expect("Pipeline without bus. Shouldn't happen!");

        let event_loop = glutin::event_loop::EventLoopBuilder::with_user_event().build();
        let window = glutin::window::WindowBuilder::new().with_title("GL rendering");
        let windowed_context = glutin::ContextBuilder::new()
            .with_vsync(true)
            .build_windowed(window, &event_loop)?;

        let windowed_context = unsafe { windowed_context.make_current().map_err(|(_, err)| err)? };

        #[cfg(any(feature = "gst-gl-x11", feature = "gst-gl-wayland"))]
        let inner_window = windowed_context.window();

        let shared_context: gst_gl::GLContext;
        if cfg!(target_os = "linux") {
            use glutin::platform::unix::RawHandle;
            #[cfg(any(feature = "gst-gl-x11", feature = "gst-gl-wayland"))]
            use glutin::platform::unix::WindowExtUnix;
            use glutin::platform::ContextTraitExt;

            let api = App::map_gl_api(windowed_context.get_api());

            let (gl_context, gl_display, platform) = match unsafe { windowed_context.raw_handle() }
            {
                #[cfg(any(feature = "gst-gl-egl", feature = "gst-gl-wayland"))]
                RawHandle::Egl(egl_context) => {
                    let mut gl_display = None;

                    #[cfg(feature = "gst-gl-egl")]
                    if let Some(display) = unsafe { windowed_context.get_egl_display() } {
                        gl_display = Some(
                            unsafe { gst_gl_egl::GLDisplayEGL::with_egl_display(display as usize) }
                                .unwrap()
                                .upcast::<gst_gl::GLDisplay>(),
                        )
                    };

                    #[cfg(feature = "gst-gl-wayland")]
                    if let Some(display) = inner_window.wayland_display() {
                        gl_display = Some(
                            unsafe {
                                gst_gl_wayland::GLDisplayWayland::with_display(display as usize)
                            }
                            .unwrap()
                            .upcast::<gst_gl::GLDisplay>(),
                        )
                    };

                    (
                        egl_context as usize,
                        gl_display.expect("Could not retrieve GLDisplay through EGL context and/or Wayland display"),
                        gst_gl::GLPlatform::EGL,
                    )
                }
                #[cfg(feature = "gst-gl-x11")]
                RawHandle::Glx(glx_context) => {
                    let gl_display = if let Some(display) = inner_window.xlib_display() {
                        unsafe { gst_gl_x11::GLDisplayX11::with_display(display as usize) }.unwrap()
                    } else {
                        panic!("X11 window without X Display");
                    };

                    (
                        glx_context as usize,
                        gl_display.upcast::<gst_gl::GLDisplay>(),
                        gst_gl::GLPlatform::GLX,
                    )
                }
                #[allow(unreachable_patterns)]
                handler => panic!("Unsupported platform: {:?}.", handler),
            };

            shared_context =
                unsafe { gst_gl::GLContext::new_wrapped(&gl_display, gl_context, platform, api) }
                    .unwrap();

            shared_context
                .activate(true)
                .expect("Couldn't activate wrapped GL context");

            shared_context.fill_info()?;

            let gl_context = shared_context.clone();
            let event_proxy = sync::Mutex::new(event_loop.create_proxy());

            #[allow(clippy::single_match)]
            bus.set_sync_handler(move |_, msg| {
                match msg.view() {
                    gst::MessageView::NeedContext(ctxt) => {
                        let context_type = ctxt.context_type();
                        if context_type == *gst_gl::GL_DISPLAY_CONTEXT_TYPE {
                            if let Some(el) =
                                msg.src().map(|s| s.downcast::<gst::Element>().unwrap())
                            {
                                let context = gst::Context::new(context_type, true);
                                context.set_gl_display(&gl_display);
                                el.set_context(&context);
                            }
                        }
                        if context_type == "gst.gl.app_context" {
                            if let Some(el) =
                                msg.src().map(|s| s.downcast::<gst::Element>().unwrap())
                            {
                                let mut context = gst::Context::new(context_type, true);
                                {
                                    let context = context.get_mut().unwrap();
                                    let s = context.structure_mut();
                                    s.set("context", &gl_context);
                                }
                                el.set_context(&context);
                            }
                        }
                    }
                    _ => (),
                }

                if let Err(e) = event_proxy.lock().unwrap().send_event(Message::BusEvent) {
                    eprintln!("Failed to send BusEvent to event proxy: {}", e)
                }

                gst::BusSyncReply::Pass
            });
        } else {
            panic!("This example only has Linux support");
        }

        Ok(App {
            pipeline,
            appsink,
            glupload,
            bus,
            event_loop,
            windowed_context,
            shared_context,
        })
    }

    fn setup(&self, event_loop: &glutin::event_loop::EventLoop<Message>) -> Result<(), Error> {
        let event_proxy = event_loop.create_proxy();
        self.appsink.set_callbacks(
            gst_app::AppSinkCallbacks::builder()
                .new_sample(move |appsink| {
                    let sample = appsink.pull_sample().map_err(|_| gst::FlowError::Eos)?;

                    {
                        let _buffer = sample.buffer().ok_or_else(|| {
                            element_error!(
                                appsink,
                                gst::ResourceError::Failed,
                                ("Failed to get buffer from appsink")
                            );

                            gst::FlowError::Error
                        })?;

                        let _info = sample
                            .caps()
                            .and_then(|caps| gst_video::VideoInfo::from_caps(caps).ok())
                            .ok_or_else(|| {
                                element_error!(
                                    appsink,
                                    gst::ResourceError::Failed,
                                    ("Failed to get video info from sample")
                                );

                                gst::FlowError::Error
                            })?;
                    }

                    event_proxy
                        .send_event(Message::Sample(sample))
                        .map(|()| gst::FlowSuccess::Ok)
                        .map_err(|e| {
                            element_error!(
                                appsink,
                                gst::ResourceError::Failed,
                                ("Failed to send sample to event loop: {}", e)
                            );

                            gst::FlowError::Error
                        })
                })
                .build(),
        );

        self.pipeline.set_state(gst::State::Playing)?;

        Ok(())
    }

    fn map_gl_api(api: glutin::Api) -> gst_gl::GLAPI {
        match api {
            glutin::Api::OpenGl => gst_gl::GLAPI::OPENGL3,
            glutin::Api::OpenGlEs => gst_gl::GLAPI::GLES2,
            _ => gst_gl::GLAPI::empty(),
        }
    }

    fn create_pipeline(
        gl_element: Option<&gst::Element>,
    ) -> Result<(gst::Pipeline, gst_app::AppSink, gst::Element), Error> {
        let pipeline = gst::Pipeline::new(None);
        let src = gst::ElementFactory::make("videotestsrc", None)
            .map_err(|_| MissingElement("videotestsrc"))?;

        let appsink = gst::ElementFactory::make("appsink", None)
            .map_err(|_| MissingElement("appsink"))?
            .dynamic_cast::<gst_app::AppSink>()
            .expect("Sink element is expected to be an appsink!");

        appsink.set_property("enable-last-sample", false);
        appsink.set_property("emit-signals", false);
        appsink.set_property("max-buffers", 1u32);

        let caps = gst_video::VideoCapsBuilder::new()
            .features(&[&gst_gl::CAPS_FEATURE_MEMORY_GL_MEMORY])
            .format(gst_video::VideoFormat::Rgba)
            .field("texture-target", "2D")
            .build();
        appsink.set_caps(Some(&caps));

        if let Some(gl_element) = gl_element {
            let glupload = gst::ElementFactory::make("glupload", None)
                .map_err(|_| MissingElement("glupload"))?;

            pipeline.add_many(&[&src, &glupload])?;
            pipeline.add(gl_element)?;
            pipeline.add(&appsink)?;

            src.link(&glupload)?;
            glupload.link(gl_element)?;
            gl_element.link(&appsink)?;

            Ok((pipeline, appsink, glupload))
        } else {
            let sink = gst::ElementFactory::make("glsinkbin", None)
                .map_err(|_| MissingElement("glsinkbin"))?;

            sink.set_property("sink", &appsink);

            pipeline.add_many(&[&src, &sink])?;
            src.link(&sink)?;

            // get the glupload element to extract later the used context in it
            let mut iter = sink.dynamic_cast::<gst::Bin>().unwrap().iterate_elements();
            let glupload = loop {
                match iter.next() {
                    Ok(Some(element)) => {
                        if "glupload" == element.factory().unwrap().name() {
                            break Some(element);
                        }
                    }
                    Err(gst::IteratorError::Resync) => iter.resync(),
                    _ => break None,
                }
            };

            Ok((pipeline, appsink, glupload.unwrap()))
        }
    }

    fn handle_messages(bus: &gst::Bus) -> Result<(), Error> {
        use gst::MessageView;

        for msg in bus.iter() {
            match msg.view() {
                MessageView::Eos(..) => break,
                MessageView::Error(err) => {
                    return Err(ErrorMessage {
                        src: msg
                            .src()
                            .map(|s| String::from(s.path_string()))
                            .unwrap_or_else(|| String::from("None")),
                        error: err.error().to_string(),
                        debug: err.debug(),
                        source: err.error(),
                    }
                    .into());
                }
                _ => (),
            }
        }

        Ok(())
    }
}

pub(crate) fn main_loop(app: App) -> Result<(), Error> {
    app.setup(&app.event_loop)?;

    println!(
        "Pixel format of the window's GL context {:?}",
        app.windowed_context.get_pixel_format()
    );

    let gl = load(&app.windowed_context);

    let mut curr_frame: Option<gst_video::VideoFrame<gst_video::video_frame::Readable>> = None;
    let mut gst_gl_context: Option<gst_gl::GLContext> = None;

    let App {
        bus,
        event_loop,
        glupload,
        pipeline,
        shared_context,
        windowed_context,
        ..
    } = app;

    event_loop.run(move |event, _, cf| {
        *cf = glutin::event_loop::ControlFlow::Wait;

        let mut needs_redraw = false;
        match event {
            glutin::event::Event::LoopDestroyed => {
                pipeline.send_event(gst::event::Eos::new());
                pipeline.set_state(gst::State::Null).unwrap();
            }
            glutin::event::Event::WindowEvent { event, .. } => match event {
                glutin::event::WindowEvent::CloseRequested
                | glutin::event::WindowEvent::KeyboardInput {
                    input:
                        glutin::event::KeyboardInput {
                            state: glutin::event::ElementState::Released,
                            virtual_keycode: Some(glutin::event::VirtualKeyCode::Escape),
                            ..
                        },
                    ..
                } => *cf = glutin::event_loop::ControlFlow::Exit,
                glutin::event::WindowEvent::Resized(physical_size) => {
                    windowed_context.resize(physical_size);
                    gl.resize(physical_size);
                }
                _ => (),
            },
            glutin::event::Event::RedrawRequested(_) => needs_redraw = true,
            // Receive a frame
            glutin::event::Event::UserEvent(Message::Sample(sample)) => {
                let buffer = sample.buffer_owned().unwrap();
                let info = sample
                    .caps()
                    .and_then(|caps| gst_video::VideoInfo::from_caps(caps).ok())
                    .unwrap();

                {
                    if gst_gl_context.is_none() {
                        gst_gl_context = glupload.property::<Option<gst_gl::GLContext>>("context");
                    }

                    let sync_meta = buffer.meta::<gst_gl::GLSyncMeta>().unwrap();
                    sync_meta.set_sync_point(gst_gl_context.as_ref().unwrap());
                }

                if let Ok(frame) = gst_video::VideoFrame::from_buffer_readable_gl(buffer, &info) {
                    curr_frame = Some(frame);
                    needs_redraw = true;
                }
            }
            // Handle all pending messages when we are awaken by set_sync_handler
            glutin::event::Event::UserEvent(Message::BusEvent) => {
                App::handle_messages(&bus).unwrap();
            }
            _ => (),
        }

        if needs_redraw {
            if let Some(frame) = curr_frame.as_ref() {
                let sync_meta = frame.buffer().meta::<gst_gl::GLSyncMeta>().unwrap();
                sync_meta.wait(&shared_context);
                if let Some(texture) = frame.texture_id(0) {
                    gl.draw_frame(texture as gl::types::GLuint);
                }
            }
            windowed_context.swap_buffers().unwrap();
        }
    })
}