diff --git a/utils/livesync/src/livesync/imp.rs b/utils/livesync/src/livesync/imp.rs
index ce7a0364..268eb1d7 100644
--- a/utils/livesync/src/livesync/imp.rs
+++ b/utils/livesync/src/livesync/imp.rs
@@ -1302,36 +1302,43 @@ impl LiveSync {
         let buffer = out_buffer.make_mut();
 
         if !duplicate {
-            let duration = state.out_duration.map_or(DEFAULT_DURATION, |dur| {
-                dur.clamp(MINIMUM_DURATION, MAXIMUM_DURATION)
-            });
+            let duration_is_valid =
+                (MINIMUM_DURATION..=MAXIMUM_DURATION).contains(&buffer.duration().unwrap());
 
-            if let Some(audio_info) = &state.out_audio_info {
-                let Some(size) = audio_info
-                    .convert::<Option<gst::format::Bytes>>(duration)
-                    .flatten()
-                    .and_then(|bytes| usize::try_from(bytes).ok())
-                else {
-                    gst::error!(CAT, imp: self, "Failed to calculate size of repeat buffer");
-                    return Err(gst::FlowError::Error);
-                };
+            if state.out_duration.is_some() || !duration_is_valid {
+                // Resize the buffer if caps gave us a duration
+                // or the current duration is unreasonable
 
-                let mut mapped_memory = gst::Memory::with_size(size)
-                    .into_mapped_memory_writable()
-                    .map_err(|_| {
-                        gst::error!(CAT, imp: self, "Failed to map memory");
-                        gst::FlowError::Error
-                    })?;
+                let duration = state.out_duration.map_or(DEFAULT_DURATION, |dur| {
+                    dur.clamp(MINIMUM_DURATION, MAXIMUM_DURATION)
+                });
 
-                audio_info
-                    .format_info()
-                    .fill_silence(mapped_memory.as_mut_slice());
+                if let Some(audio_info) = &state.out_audio_info {
+                    let Some(size) = audio_info
+                        .convert::<Option<gst::format::Bytes>>(duration)
+                        .flatten()
+                        .and_then(|bytes| usize::try_from(bytes).ok())
+                    else {
+                        gst::error!(CAT, imp: self, "Failed to calculate size of repeat buffer");
+                        return Err(gst::FlowError::Error);
+                    };
 
-                buffer.replace_all_memory(mapped_memory.into_memory());
+                    buffer.replace_all_memory(gst::Memory::with_size(size));
+                }
+
+                buffer.set_duration(duration);
+                gst::debug!(CAT, imp: self, "Patched output buffer duration to {duration}");
             }
 
-            buffer.set_duration(duration);
-            gst::debug!(CAT, imp: self, "Patched output buffer duration to {duration}");
+            if let Some(audio_info) = &state.out_audio_info {
+                let mut map_info = buffer.map_writable().map_err(|e| {
+                    gst::error!(CAT, imp: self, "Failed to map buffer: {}", e);
+                    gst::FlowError::Error
+                })?;
+                audio_info
+                    .format_info()
+                    .fill_silence(map_info.as_mut_slice());
+            }
         }
 
         buffer.set_dts(dts);