[PATCH v2] kdf: Fix memory cost overflow in Argon2 KDF

* cipher/kdf.c (argon2_init) Fix memory cost overflow in Argon2 KDF.
--

Argon2 memory cost is set kilobytes, so for memory allocation
it must be converted to bytes (multiplied by 1024).

This patch fixes integer overflow for 64bit platforms, so more than
4GB of memory can be allocated.

For 32bit platforms (where size_t is 32bit) it cannot allocate
such amount of memory, so the code tries to detect the issue early
before overflow.

The issue can be easily reproduced with Argon2 mcost set to:
  m_cost = 4194304 (= 4 * 1024 * 1024, set in param[2])

Originally reported for cryptsetup with libgcrypt backend
https://gitlab.com/cryptsetup/cryptsetup/-/issues/922

Signed-off-by: Milan Broz <[email protected]>
---
 cipher/kdf.c | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/cipher/kdf.c b/cipher/kdf.c
index 52e6a9ba..1eae2b90 100644
--- a/cipher/kdf.c
+++ b/cipher/kdf.c
@@ -494,6 +494,7 @@ argon2_init (argon2_ctx_t a, unsigned int parallelism,
 {
   gpg_err_code_t ec = 0;
   unsigned int memory_blocks;
+  size_t memory_bytes;
   unsigned int segment_length;
   void *block;
   struct argon2_thread_data *thread_data;
@@ -514,13 +515,17 @@ argon2_init (argon2_ctx_t a, unsigned int parallelism,
   a->block = NULL;
   a->thread_data = NULL;
 
-  block = xtrymalloc (1024 * memory_blocks);
+  if (U64_C(1024) * memory_blocks > SIZE_MAX)
+    return GPG_ERR_INV_VALUE;
+
+  memory_bytes = 1024 * (size_t)memory_blocks;
+  block = xtrymalloc (memory_bytes);
   if (!block)
     {
       ec = gpg_err_code_from_errno (errno);
       return ec;
     }
-  memset (block, 0, 1024 * memory_blocks);
+  memset (block, 0, memory_bytes);
 
   thread_data = xtrymalloc (a->lanes * sizeof (struct argon2_thread_data));
   if (!thread_data)
@@ -873,6 +878,9 @@ argon2_open (gcry_kdf_hd_t *hd, int subalgo,
   if (parallelism == 0)
     return GPG_ERR_INV_VALUE;
 
+  if (U64_C(1024) * m_cost > SIZE_MAX)
+    return GPG_ERR_INV_VALUE;
+
   n = offsetof (struct argon2_context, out) + taglen;
   a = xtrymalloc (n);
   if (!a)
-- 
2.45.2