commit 6fee37f08391415cdf8d8610c501516c3d3ed29c

Author: Ingo Franzki <ifranzki@linux.ibm.com>

Date:   Wed Jun 30 13:41:57 2021 +0200

    COMMON: mech_md5: Remove deprecated OpenSSL functions

    All low level MD5 functions are deprecated in OpenSSL 3.0.

    Update the code to not use any of those.

    Signed-off-by: Ingo Franzki <ifranzki@linux.ibm.com>

diff --git a/usr/lib/common/h_extern.h b/usr/lib/common/h_extern.h

index 47b96ba0..314613a5 100644

--- a/usr/lib/common/h_extern.h

+++ b/usr/lib/common/h_extern.h

@@ -1667,7 +1667,7 @@ CK_RV md5_hmac_verify(STDLL_TokData_t *tokdata,

                       CK_ULONG in_data_len,

                       CK_BYTE *signature, CK_ULONG sig_len);

-void sw_md5_init(DIGEST_CONTEXT *ctx);

+CK_RV sw_md5_init(DIGEST_CONTEXT *ctx);

 CK_RV sw_md5_hash(DIGEST_CONTEXT *ctx, CK_BYTE *in_data,

                   CK_ULONG in_data_len, CK_BYTE *out_data,

diff --git a/usr/lib/common/mech_md5.c b/usr/lib/common/mech_md5.c

index 320e2549..65c11def 100644

--- a/usr/lib/common/mech_md5.c

+++ b/usr/lib/common/mech_md5.c

@@ -20,30 +20,50 @@

 #include "tok_spec_struct.h"

 #include "trace.h"

-#include <openssl/md5.h>

+#include <openssl/evp.h>

 #include <openssl/crypto.h>

 //

 // Software MD5 implementation (OpenSSL based)

 //

-void sw_md5_init(DIGEST_CONTEXT *ctx)

+static void sw_md5_free(STDLL_TokData_t *tokdata, SESSION *sess,

+                        CK_BYTE *context, CK_ULONG context_len)

 {

-    ctx->context_len = sizeof(MD5_CTX);

-    ctx->context = (CK_BYTE *) malloc(sizeof(MD5_CTX));

+    UNUSED(tokdata);

+    UNUSED(sess);

+    UNUSED(context_len);

+

+    EVP_MD_CTX_free((EVP_MD_CTX *)context);

+}

+

+CK_RV sw_md5_init(DIGEST_CONTEXT *ctx)

+{

+    ctx->context_len = 1;

+    ctx->context = (CK_BYTE *)EVP_MD_CTX_new();

     if (ctx->context == NULL) {

         TRACE_ERROR("%s\n", ock_err(ERR_HOST_MEMORY));

-        // TODO: propagate error up?

-        return;

+        return CKR_HOST_MEMORY;

+    }

+

+    if (!EVP_DigestInit_ex((EVP_MD_CTX *)ctx->context, EVP_md5(), NULL)) {

+        TRACE_ERROR("%s\n", ock_err(ERR_FUNCTION_FAILED));

+        EVP_MD_CTX_free((EVP_MD_CTX *)ctx->context);

+        return CKR_FUNCTION_FAILED;

     }

-    MD5_Init((MD5_CTX *)ctx->context);

+    ctx->state_unsaveable = CK_TRUE;

+    ctx->context_free_func = sw_md5_free;

+

+    return CKR_OK;

 }

 CK_RV sw_md5_hash(DIGEST_CONTEXT *ctx, CK_BYTE *in_data,

                   CK_ULONG in_data_len, CK_BYTE *out_data,

                   CK_ULONG *out_data_len)

 {

+    unsigned int len;

+

     if (!ctx || !out_data_len) {

         TRACE_ERROR("%s received bad argument(s)\n", __func__);

         return CKR_FUNCTION_FAILED;

@@ -57,43 +77,60 @@ CK_RV sw_md5_hash(DIGEST_CONTEXT *ctx, CK_BYTE *in_data,

     if (ctx->context == NULL)

         return CKR_OPERATION_NOT_INITIALIZED;

-    MD5_Update((MD5_CTX *)ctx->context, in_data, in_data_len);

-    MD5_Final(out_data, (MD5_CTX *)ctx->context);

-    *out_data_len = MD5_HASH_SIZE;

+    len = *out_data_len;

+    if (!EVP_DigestUpdate((EVP_MD_CTX *)ctx->context, in_data, in_data_len) ||

+        !EVP_DigestFinal((EVP_MD_CTX *)ctx->context, out_data, &len)) {

+        TRACE_ERROR("%s\n", ock_err(ERR_FUNCTION_FAILED));

+        return CKR_FUNCTION_FAILED;

+    }

-    if (ctx->context_free_func != NULL)

-        ctx->context_free_func(ctx->context, ctx->context_len);

-    else

-        free(ctx->context);

+    *out_data_len = len;

+

+    EVP_MD_CTX_free((EVP_MD_CTX *)ctx->context);

     ctx->context = NULL;

+    ctx->context_free_func = NULL;

     return CKR_OK;

 }

-CK_RV sw_MD5_Update(DIGEST_CONTEXT *ctx, CK_BYTE *in_data,

-                     CK_ULONG in_data_len)

+static CK_RV sw_md5_update(DIGEST_CONTEXT *ctx, CK_BYTE *in_data,

+                           CK_ULONG in_data_len)

 {

     if (ctx->context == NULL)

         return CKR_OPERATION_NOT_INITIALIZED;

-    MD5_Update((MD5_CTX *)ctx->context, in_data, in_data_len);

+    if (!EVP_DigestUpdate((EVP_MD_CTX *)ctx->context, in_data, in_data_len)) {

+        TRACE_ERROR("%s\n", ock_err(ERR_FUNCTION_FAILED));

+        return CKR_FUNCTION_FAILED;

+    }

+

     return CKR_OK;

 }

-CK_RV sw_MD5_Final(DIGEST_CONTEXT *ctx, CK_BYTE *out_data,

-                    CK_ULONG *out_data_len)

+static CK_RV sw_md5_final(DIGEST_CONTEXT *ctx, CK_BYTE *out_data,

+                          CK_ULONG *out_data_len)

 {

+    unsigned int len;

+

     if (ctx->context == NULL)

         return CKR_OPERATION_NOT_INITIALIZED;

-    MD5_Final(out_data, (MD5_CTX *)ctx->context);

-    *out_data_len = MD5_HASH_SIZE;

+    if (*out_data_len < MD5_HASH_SIZE) {

+        TRACE_ERROR("%s\n", ock_err(ERR_BUFFER_TOO_SMALL));

+        return CKR_BUFFER_TOO_SMALL;

+    }

-    if (ctx->context_free_func != NULL)

-        ctx->context_free_func(ctx->context, ctx->context_len);

-    else

-        free(ctx->context);

+    len = *out_data_len;

+    if (!EVP_DigestFinal((EVP_MD_CTX *)ctx->context, out_data, &len)) {

+        TRACE_ERROR("%s\n", ock_err(ERR_FUNCTION_FAILED));

+        return CKR_FUNCTION_FAILED;

+    }

+

+    *out_data_len = len;

+

+    EVP_MD_CTX_free((EVP_MD_CTX *)ctx->context);

     ctx->context = NULL;

+    ctx->context_free_func = NULL;

     return CKR_OK;

 }

@@ -105,8 +142,7 @@ CK_RV md5_init(STDLL_TokData_t *tokdata, SESSION *sess, DIGEST_CONTEXT *ctx,

     UNUSED(sess);

     if (mech->mechanism == CKM_MD5) {

-        sw_md5_init(ctx);

-        return CKR_OK;

+        return sw_md5_init(ctx);

     } else {

         return CKR_MECHANISM_INVALID;

     }

@@ -159,7 +195,7 @@ CK_RV md5_hash_update(STDLL_TokData_t *tokdata, SESSION *sess,

         return CKR_OK;

     if (ctx->mech.mechanism == CKM_MD5)

-        return sw_MD5_Update(ctx, in_data, in_data_len);

+        return sw_md5_update(ctx, in_data, in_data_len);

     else

         return CKR_MECHANISM_INVALID;

 }

@@ -188,7 +224,7 @@ CK_RV md5_hash_final(STDLL_TokData_t *tokdata, SESSION *sess,

     }

     if (ctx->mech.mechanism == CKM_MD5)

-        return sw_MD5_Final(ctx, out_data, out_data_len);

+        return sw_md5_final(ctx, out_data, out_data_len);

     else

         return CKR_MECHANISM_INVALID;

 }