diff --git a/ci/docker/debian/deps.txt b/ci/docker/debian/deps.txt
index 73a91cea86..6cd8e4b5a5 100644
--- a/ci/docker/debian/deps.txt
+++ b/ci/docker/debian/deps.txt
@@ -168,7 +168,6 @@ libqt5x11extras5-dev
 libraw1394-dev
 librsvg2-dev
 librtmp-dev
-librust-wayland-protocols-dev
 libsbc-dev
 libsdl2-dev
 libshaderc1
diff --git a/ci/docker/debian/install-deps.sh b/ci/docker/debian/install-deps.sh
index 20a29c512f..24c6571a53 100644
--- a/ci/docker/debian/install-deps.sh
+++ b/ci/docker/debian/install-deps.sh
@@ -5,6 +5,8 @@ set -eux
 apt update -y && apt full-upgrade -y
 apt install -y $(<./ci/docker/debian/deps.txt)
 
+apt remove -y rustc cargo
+
 pip3 install --break-system-packages meson hotdoc python-gitlab tomli junitparser
 
 apt clean all
diff --git a/ci/docker/fedora/install-deps.sh b/ci/docker/fedora/install-deps.sh
index 17e1f00370..85b5d88e22 100644
--- a/ci/docker/fedora/install-deps.sh
+++ b/ci/docker/fedora/install-deps.sh
@@ -39,6 +39,9 @@ pip3 install meson hotdoc python-gitlab tomli junitparser
 # Remove gst-devel packages installed by builddep above
 dnf remove -y "gstreamer1*devel"
 
+# We use rustup to install the rust toolchain, we don't need fedora's
+dnf remove -y rust cargo
+
 dnf install -y glib2-doc gdk-pixbuf2-devel gtk3-devel-docs gtk4-devel-docs libsoup-doc
 
 # Install most debug symbols, except the big ones from things we use