Re: In-place decryption error

IMO this is a documentation issue. The in-place decryption works only
with stream ciphers or block ciphers with padding switched off. 

That the inplace decryption won't work is actually indicated by outlen
being 0 at the first EVP_CipherUpdate() call in your test program. That
means the next EVP_CipherUpdate() is already not doing true inplace
decryption but instead decrypting from the buffer.

If the decryption was done into a memory buffer instead of reusing a
single block buffer memory (which is not particularly useful as you can
easily have a separate buffers for input and output), the decryption
would work. Please try the attached modified test with any data up to
the size of the buffer (4096 bytes).

Tomas Mraz, OpenSSL Foundation


On Tue, 2025-10-21 at 20:00 +0000, 'Sands, Daniel N.' via openssl-users
wrote:
> It looks like either there need to be some restrictions on in-place
> decryption, or there's a bug to fix.  Here's the summary:
> 
> 1) Applies to block ciphers
> 2) Does not apply to first call to EVP_DecryptUpdate, but to all
> subsequent calls
> 3) First block(s) of data will be decrypted properly.  After that,
> it's
> all gibberish.  EVP_DecryptFinal_ex will fail.
> 4) Applies to all Openssl 3 versions, as well as the current MASTER
> head.
> 
> 
> A more detailed analysis of the inner workings shows that on the
> second
> EVP_DecryptUpdate where inbuf==outbuf, the next block of data will be
> written to outbuf, then the next ciphertext block will be read from
> inbuf.  Problem is, it just overwrote that ciphertext.  It seems that
> decryption may require another intermediary buffer for the
> ciphertext.
> Perhaps a rotation of 2 buffers where it reads the inbuf ciphertext
> into the standby context buffer, decrypts the active context buffer
> into the outbuf, and finally rotates the two context buffers.
> 
> 
> 
> I used a modification of the EVP_Cipher API example to set this test
> up.  I compile it to an executable called 'test'.  Then I run these
> commands:
> 
> $ echo "This text is 6 words long" > test.txt
> $ ./test test.txt test.enc e
> $ openssl aes-128-cbc -d -K 30313233343536373839616263646546 -iv
> 31323334353637383837363534333231 -in test.enc
> This text is 6 words long
> $ ./test test.enc test.dec d
> Error
> $ cat test.dec
> This text is 6 w$
> 
> 
> Here's the test.c source:
> 
> #include <openssl/evp.h>
> #include <netinet/in.h>
> #include <unistd.h>
> #include <string.h>
> 
> int do_crypt(FILE *in, FILE *out, int do_encrypt)
> {
>     /* Allow enough space in output buffer for additional block */
>     unsigned char buf[16];
>     int inlen, outlen;
>     EVP_CIPHER_CTX *ctx;
>     /*
>      * Bogus key and IV: we'd normally set these from
>      * another source.
>      */
>     unsigned char key[] = "0123456789abcdeF";
>     unsigned char iv[] = "1234567887654321";
> 
>     /* Don't set key or IV right away; we want to check lengths */
>     ctx = EVP_CIPHER_CTX_new();
>     if (!EVP_CipherInit_ex2(ctx, EVP_aes_128_cbc(), NULL, NULL,
>                             do_encrypt, NULL)) {
>         /* Error */
>         EVP_CIPHER_CTX_free(ctx);
>         fprintf(stderr,"Error\n");
>         return 0;
>     }
>     OPENSSL_assert(EVP_CIPHER_CTX_get_key_length(ctx) == 16);
>     OPENSSL_assert(EVP_CIPHER_CTX_get_iv_length(ctx) == 16);
> 
>     /* Now we can set key and IV */
>     if (!EVP_CipherInit_ex2(ctx, NULL, key, iv, do_encrypt, NULL)) {
>         /* Error */
>         EVP_CIPHER_CTX_free(ctx);
>         return 0;
>     }
> 
>     for (;;) {
>         inlen = fread(buf, 1, sizeof(buf), in);
>         if (inlen <= 0)
>             break;
>         if (!EVP_CipherUpdate(ctx, buf, &outlen, buf, inlen)) {
>             /* Error */
>             EVP_CIPHER_CTX_free(ctx);
>             fprintf(stderr,"Error\n");
>             return 0;
>         }
>         fwrite(buf, 1, outlen, out);
>     }
>     if (!EVP_CipherFinal_ex(ctx, buf, &outlen)) {
>         /* Error */
>         EVP_CIPHER_CTX_free(ctx);
>         fprintf(stderr,"Error\n");
>         return 0;
>     }
>     fwrite(buf, 1, outlen, out);
> 
>     EVP_CIPHER_CTX_free(ctx);
>     return 1;
> }
> 
> int main(int argc, char *argv[]) {
>    FILE *f1 = fopen(argv[1], "rb");
>    FILE *f2 = fopen(argv[2], "wb");
>    do_crypt(f1, f2, argv[3][0] == 'e');
> }
> 
> 
> And the relevant docs:
> 
> =item EVP_EncryptUpdate()
> 
> Encrypts I<inl> bytes from the buffer I<in> and writes the encrypted
> version to
> I<out>. The pointers I<out> and I<in> may point to the same location,
> in which
> case the encryption will be done in-place. However, in-place
> encryption
> is
> guaranteed to work only if the encryption context (I<ctx>) has
> processed data in
> multiples of the block size. If the context contains an incomplete
> data
> block
> from previous operations, in-place encryption will fail. I<ctx>
> B<MUST
> NOT> be NULL.
> 
> ...
> 
> =item EVP_DecryptInit_ex2(), EVP_DecryptInit_ex(),
> EVP_DecryptUpdate()
> and EVP_DecryptFinal_ex()
> 
> These functions are the corresponding decryption operations.
> EVP_DecryptFinal() will return an error code if padding is enabled
> and
> the
> final block is not correctly formatted. The parameters and
> restrictions
> are
> identical to the encryption operations. I<ctx> B<MUST NOT> be NULL.
> 

-- 
Tomáš Mráz, Public Support and Security Manager, OpenSSL Foundation
Join the Code Protectors or support us on Github Sponsors
https://openssl-foundation.org/donate/

-- 
You received this message because you are subscribed to the Google Groups "openssl-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to openssl-users+unsubscribe-MCmKBN63+Bmbup2nOX2J7Q@public.gmane.org
To view this discussion visit https://groups.google.com/a/openssl.org/d/msgid/openssl-users/c3b3afc295064f11a93c66df321cd7ac02d09e55.camel%40openssl.org.

#include <openssl/evp.h>
#include <netinet/in.h>
#include <unistd.h>
#include <string.h>

int do_crypt(FILE *in, FILE *out, int do_encrypt)
{
    /* Allow enough space in output buffer for additional block */
    unsigned char buf[4096];
    unsigned char *iptr = buf, *optr = buf;
    int inlen, outlen;
    EVP_CIPHER_CTX *ctx;
    /*
     * Bogus key and IV: we'd normally set these from
     * another source.
     */
    unsigned char key[] = "0123456789abcdeF";
    unsigned char iv[] = "1234567887654321";

    /* Don't set key or IV right away; we want to check lengths */
    ctx = EVP_CIPHER_CTX_new();
    if (!EVP_CipherInit_ex2(ctx, EVP_aes_128_cbc(), NULL, NULL,
                            do_encrypt, NULL)) {
        /* Error */
        EVP_CIPHER_CTX_free(ctx);
        fprintf(stderr,"Error\n");
        return 0;
    }
    OPENSSL_assert(EVP_CIPHER_CTX_get_key_length(ctx) == 16);
    OPENSSL_assert(EVP_CIPHER_CTX_get_iv_length(ctx) == 16);

    /* Now we can set key and IV */
    if (!EVP_CipherInit_ex2(ctx, NULL, key, iv, do_encrypt, NULL)) {
        /* Error */
        EVP_CIPHER_CTX_free(ctx);
        return 0;
    }

    for (;;) {
        inlen = fread(iptr, 1, 48, in);
        if (inlen <= 0)
            break;
        outlen = inlen;
        fprintf(stderr, "inlen = %d\n", inlen);
        if (!EVP_CipherUpdate(ctx, optr, &outlen, iptr, inlen)) {
            /* Error */
            EVP_CIPHER_CTX_free(ctx);
            fprintf(stderr,"Error\n");
            return 0;
        }
        fprintf(stderr, "outlen = %d\n", outlen);
        fwrite(optr, 1, outlen, out);
        iptr += inlen;
        optr += outlen;
    }
    if (!EVP_CipherFinal_ex(ctx, optr, &outlen)) {
        /* Error */
        EVP_CIPHER_CTX_free(ctx);
        fprintf(stderr,"Error\n");
        return 0;
    }
    fwrite(optr, 1, outlen, out);

    EVP_CIPHER_CTX_free(ctx);
    return 1;
}

int main(int argc, char *argv[]) {
   FILE *f1 = fopen(argv[1], "rb");
   FILE *f2 = fopen(argv[2], "wb");
   do_crypt(f1, f2, argv[3][0] == 'e');
}