diff --git a/ext/wayland/gstwaylandsink.c b/ext/wayland/gstwaylandsink.c
index 9730ccc113..db3f406b0f 100644
--- a/ext/wayland/gstwaylandsink.c
+++ b/ext/wayland/gstwaylandsink.c
@@ -597,7 +597,6 @@ render_last_buffer (GstWaylandSink * sink)
   GstWlMeta *meta;
   struct wl_surface *surface;
   struct wl_callback *callback;
-  GstVideoRectangle src, dst, res;
 
   meta = gst_buffer_get_wl_meta (sink->last_buffer);
   surface = gst_wl_window_get_wl_surface (sink->window);
@@ -612,19 +611,9 @@ render_last_buffer (GstWaylandSink * sink)
    * releases it. The release is handled internally in the pool */
   gst_wayland_compositor_acquire_buffer (meta->pool, sink->last_buffer);
 
-  src.w = sink->video_width;
-  src.h = sink->video_height;
-  dst.w = sink->window->width;
-  dst.h = sink->window->height;
-  gst_video_sink_center_rect (src, dst, &res, TRUE);
-
-  if (sink->window->subsurface)
-    wl_subsurface_set_position (sink->window->subsurface,
-        sink->window->x + res.x, sink->window->y + res.y);
-  wl_viewport_set_destination (sink->window->viewport, res.w, res.h);
-
   wl_surface_attach (surface, meta->wbuffer, 0, 0);
-  wl_surface_damage (surface, 0, 0, dst.w, dst.h);
+  wl_surface_damage (surface, 0, 0, sink->window->surface_width,
+      sink->window->surface_height);
 
   wl_surface_commit (surface);
   wl_display_flush (sink->display->display);
@@ -647,13 +636,6 @@ gst_wayland_sink_render (GstBaseSink * bsink, GstBuffer * buffer)
 
   GST_LOG_OBJECT (sink, "render buffer %p", buffer);
 
-  /* if we were not provided a window, create one ourselves */
-  if (!sink->window)
-    sink->window = gst_wl_window_new_toplevel (sink->display, sink->video_width,
-        sink->video_height);
-  else if (sink->window->width == 0 || sink->window->height == 0)
-    goto no_window_size;
-
   /* surface is resizing - drop buffers until finished */
   if (sink->drawing_frozen)
     goto done;
@@ -662,6 +644,17 @@ gst_wayland_sink_render (GstBaseSink * bsink, GstBuffer * buffer)
   if (g_atomic_int_get (&sink->redraw_pending) == TRUE)
     goto done;
 
+  /* if we were not provided a window, create one ourselves */
+  if (!sink->window) {
+    sink->window = gst_wl_window_new_toplevel (sink->display, sink->video_width,
+        sink->video_height);
+  } else {
+    gst_wl_window_set_video_size (sink->window, sink->video_width,
+        sink->video_height);
+    if (sink->window->surface_width == 0 || sink->window->surface_height == 0)
+      goto no_window_size;
+  }
+
   meta = gst_buffer_get_wl_meta (buffer);
 
   if (meta && meta->pool->display == sink->display) {
@@ -793,7 +786,8 @@ gst_wayland_sink_set_render_rectangle (GstVideoOverlay * overlay,
 
   GST_DEBUG_OBJECT (sink, "window geometry changed to (%d, %d) %d x %d",
       x, y, w, h);
-  gst_wl_window_set_size (sink->window, x, y, w, h);
+  gst_wl_window_set_render_rectangle (sink->window, x, y, w, h);
+
   GST_OBJECT_UNLOCK (sink);
 }
 
diff --git a/ext/wayland/wlwindow.c b/ext/wayland/wlwindow.c
index 6d4588539a..a75e2faf6a 100644
--- a/ext/wayland/wlwindow.c
+++ b/ext/wayland/wlwindow.c
@@ -116,14 +116,16 @@ gst_wl_window_new_internal (GstWlDisplay * display, struct wl_surface *surface)
 }
 
 GstWlWindow *
-gst_wl_window_new_toplevel (GstWlDisplay * display, gint width, gint height)
+gst_wl_window_new_toplevel (GstWlDisplay * display, gint video_width,
+    gint video_height)
 {
   GstWlWindow *window;
 
   window = gst_wl_window_new_internal (display,
       wl_compositor_create_surface (display->compositor));
 
-  gst_wl_window_set_size (window, 0, 0, width, height);
+  gst_wl_window_set_video_size (window, video_width, video_height);
+  gst_wl_window_set_render_rectangle (window, 0, 0, video_width, video_height);
 
   window->shell_surface = wl_shell_get_shell_surface (display->shell,
       window->surface);
@@ -182,13 +184,53 @@ gst_wl_window_is_toplevel (GstWlWindow * window)
   return (window->shell_surface != NULL);
 }
 
+static void
+gst_wl_window_resize_internal (GstWlWindow * window)
+{
+  GstVideoRectangle src, res;
+
+  src.w = window->video_width;
+  src.h = window->video_height;
+  gst_video_sink_center_rect (src, window->render_rectangle, &res, TRUE);
+
+  if (window->subsurface)
+    wl_subsurface_set_position (window->subsurface,
+        window->render_rectangle.x + res.x, window->render_rectangle.y + res.y);
+  wl_viewport_set_destination (window->viewport, res.w, res.h);
+
+  wl_surface_damage (window->surface, 0, 0, res.w, res.h);
+  wl_surface_commit (window->surface);
+
+  /* this is saved for use in wl_surface_damage */
+  window->surface_width = res.w;
+  window->surface_height = res.h;
+}
+
 void
-gst_wl_window_set_size (GstWlWindow * window, gint x, gint y, gint w, gint h)
+gst_wl_window_set_video_size (GstWlWindow * window, gint w, gint h)
 {
   g_return_if_fail (window != NULL);
 
-  window->x = x;
-  window->y = y;
-  window->width = w;
-  window->height = h;
+  if (w != window->video_width || h != window->video_height) {
+    window->video_width = w;
+    window->video_height = h;
+
+    if (window->render_rectangle.w != 0)
+      gst_wl_window_resize_internal (window);
+  }
+}
+
+void
+gst_wl_window_set_render_rectangle (GstWlWindow * window, gint x, gint y,
+    gint w, gint h)
+{
+  g_return_if_fail (window != NULL);
+
+  window->render_rectangle.x = x;
+  window->render_rectangle.y = y;
+  window->render_rectangle.w = w;
+  window->render_rectangle.h = h;
+
+  if (window->video_width != 0)
+    gst_wl_window_resize_internal (window);
 }
diff --git a/ext/wayland/wlwindow.h b/ext/wayland/wlwindow.h
index 51e3d793c1..21ecdce34c 100644
--- a/ext/wayland/wlwindow.h
+++ b/ext/wayland/wlwindow.h
@@ -22,6 +22,7 @@
 #define __GST_WL_WINDOW_H__
 
 #include "wldisplay.h"
+#include <gst/video/gstvideosink.h>
 
 G_BEGIN_DECLS
 
@@ -44,7 +45,13 @@ struct _GstWlWindow
   struct wl_subsurface *subsurface;
   struct wl_viewport *viewport;
   struct wl_shell_surface *shell_surface;
-  gint x, y, width, height;
+
+  /* the size of the destination area where we are overlaying our subsurface */
+  GstVideoRectangle render_rectangle;
+  /* the size of the video in the buffers */
+  gint video_width, video_height;
+  /* the size of the (sub)surface */
+  gint surface_width, surface_height;
 };
 
 struct _GstWlWindowClass
@@ -55,7 +62,7 @@ struct _GstWlWindowClass
 GType gst_wl_window_get_type (void);
 
 GstWlWindow *gst_wl_window_new_toplevel (GstWlDisplay * display,
-        gint width, gint height);
+        gint video_width, gint video_height);
 GstWlWindow *gst_wl_window_new_in_surface (GstWlDisplay * display,
         struct wl_surface * parent);
 
@@ -63,8 +70,10 @@ GstWlDisplay *gst_wl_window_get_display (GstWlWindow * window);
 struct wl_surface *gst_wl_window_get_wl_surface (GstWlWindow * window);
 gboolean gst_wl_window_is_toplevel (GstWlWindow *window);
 
-void gst_wl_window_set_size (GstWlWindow * window, gint x, gint y, gint w,
-        gint h);
+/* functions to manipulate the size on non-toplevel windows */
+void gst_wl_window_set_video_size (GstWlWindow * window, gint w, gint h);
+void gst_wl_window_set_render_rectangle (GstWlWindow * window, gint x, gint y,
+        gint w, gint h);
 
 G_END_DECLS