aesdec: Fix padding removal for per-buffer-padding=FALSE

When per-buffer-padding is FALSE, the OpenSSL context needs
to be told to remove any padding at the end of the ciphertext

Fixes https://gitlab.freedesktop.org/gstreamer/gstreamer/-/issues/1243

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/3406>
This commit is contained in:
Jan Schmidt 2022-11-15 00:11:15 +11:00
parent 9e74c728cb
commit 452890093d

View file

@ -583,12 +583,11 @@ gst_aes_dec_init_cipher (GstAesDec * filter)
GST_ERROR_OBJECT (filter, "Could not initialize openssl cipher");
return FALSE;
}
if (filter->per_buffer_padding) {
if (!EVP_CIPHER_CTX_set_padding (filter->evp_ctx, 0)) {
if (!EVP_CIPHER_CTX_set_padding (filter->evp_ctx,
filter->per_buffer_padding ? 0 : 1)) {
GST_ERROR_OBJECT (filter, "Could not set padding");
return FALSE;
}
}
return TRUE;
}