Make MySQL compile with openssl 3.x without FIPS properly implemented

This change takes some pieces from MariaDB, including compat_ssl.h and

changes in my_md5.cc.

MySQL utilizes FIPS_mode() and FIPS_mode_set() functions that are not

available in OpenSSL 3.x any more. This patch only mocks the call of

those functions, returning 0 every time, which effectively makes usage

of those functions non working. For making the MySQL build with

OpenSSL 3.x this seems to be enough though.

Resolves: #1952951

diff -rup mysql-8.0.22-orig/cmake/ssl.cmake mysql-8.0.22/cmake/ssl.cmake

--- mysql-8.0.22-orig/cmake/ssl.cmake	2021-05-19 21:36:33.161996422 +0200

+++ mysql-8.0.22/cmake/ssl.cmake	2021-05-19 23:06:54.211877057 +0200

@@ -227,8 +227,7 @@ MACRO (MYSQL_CHECK_SSL)

     ENDIF()

     IF(OPENSSL_INCLUDE_DIR AND

        OPENSSL_LIBRARY   AND

-       CRYPTO_LIBRARY      AND

-       OPENSSL_MAJOR_VERSION STREQUAL "1"

+       CRYPTO_LIBRARY

       )

       SET(OPENSSL_FOUND TRUE)

       FIND_PROGRAM(OPENSSL_EXECUTABLE openssl

diff -rup mysql-8.0.22-orig/include/ssl_compat.h mysql-8.0.22/include/ssl_compat.h

--- mysql-8.0.22-orig/include/ssl_compat.h	2021-05-19 23:19:36.152956356 +0200

+++ mysql-8.0.22/include/ssl_compat.h	2021-05-19 23:06:55.048885933 +0200

@@ -0,0 +1,105 @@

+/*

+ Copyright (c) 2016, 2021, MariaDB Corporation.

+

+ This program is free software; you can redistribute it and/or modify

+ it under the terms of the GNU General Public License as published by

+ the Free Software Foundation; version 2 of the License.

+

+ This program is distributed in the hope that it will be useful,

+ but WITHOUT ANY WARRANTY; without even the implied warranty of

+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

+ GNU General Public License for more details.

+

+ You should have received a copy of the GNU General Public License

+ along with this program; if not, write to the Free Software

+ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA */

+

+#include <openssl/opensslv.h>

+

+/* OpenSSL version specific definitions */

+#if defined(OPENSSL_VERSION_NUMBER)

+

+#if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER)

+#define HAVE_OPENSSL11 1

+#define SSL_LIBRARY OpenSSL_version(OPENSSL_VERSION)

+#define ERR_remove_state(X) ERR_clear_error()

+#define EVP_CIPHER_CTX_SIZE 176

+#define EVP_MD_CTX_SIZE 48

+#undef EVP_MD_CTX_init

+#define EVP_MD_CTX_init(X) do { memset((X), 0, EVP_MD_CTX_SIZE); EVP_MD_CTX_reset(X); } while(0)

+#undef EVP_CIPHER_CTX_init

+#define EVP_CIPHER_CTX_init(X) do { memset((X), 0, EVP_CIPHER_CTX_SIZE); EVP_CIPHER_CTX_reset(X); } while(0)

+

+/*

+  Macros below are deprecated. OpenSSL 1.1 may define them or not,

+  depending on how it was built.

+*/

+#undef ERR_free_strings

+#define ERR_free_strings()

+#undef EVP_cleanup

+#define EVP_cleanup()

+#undef CRYPTO_cleanup_all_ex_data

+#define CRYPTO_cleanup_all_ex_data()

+#undef SSL_load_error_strings

+#define SSL_load_error_strings()

+

+#else

+#define HAVE_OPENSSL10 1

+#ifdef HAVE_WOLFSSL

+#define SSL_LIBRARY "WolfSSL " WOLFSSL_VERSION

+#else

+#define SSL_LIBRARY SSLeay_version(SSLEAY_VERSION)

+#endif

+

+#ifdef HAVE_WOLFSSL

+#undef ERR_remove_state

+#define ERR_remove_state(x) do {} while(0)

+#elif defined (HAVE_ERR_remove_thread_state)

+#define ERR_remove_state(X) ERR_remove_thread_state(NULL)

+#endif /* HAVE_ERR_remove_thread_state */

+

+#endif /* HAVE_OPENSSL11 */

+#endif

+

+#ifdef HAVE_WOLFSSL

+#define EVP_MD_CTX_SIZE                 sizeof(wc_Md5)

+#endif

+

+#ifndef HAVE_OPENSSL11

+#ifndef ASN1_STRING_get0_data

+#define ASN1_STRING_get0_data(X)        ASN1_STRING_data(X)

+#endif

+#ifndef EVP_MD_CTX_SIZE

+#define EVP_MD_CTX_SIZE                 sizeof(EVP_MD_CTX)

+#endif

+

+#define DH_set0_pqg(D,P,Q,G)            ((D)->p= (P), (D)->g= (G))

+#define EVP_CIPHER_CTX_buf_noconst(ctx) ((ctx)->buf)

+#define EVP_CIPHER_CTX_encrypting(ctx)  ((ctx)->encrypt)

+#define EVP_CIPHER_CTX_SIZE             sizeof(EVP_CIPHER_CTX)

+

+#ifndef HAVE_WOLFSSL

+#define OPENSSL_init_ssl(X,Y)           SSL_library_init()

+#define EVP_MD_CTX_reset(X) EVP_MD_CTX_cleanup(X)

+#define EVP_CIPHER_CTX_reset(X) EVP_CIPHER_CTX_cleanup(X)

+#define X509_get0_notBefore(X) X509_get_notBefore(X)

+#define X509_get0_notAfter(X) X509_get_notAfter(X)

+#endif

+#endif

+

+#ifndef TLS1_3_VERSION

+//#define SSL_CTX_set_ciphersuites(X,Y) 0

+#endif

+

+#ifdef	__cplusplus

+extern "C" {

+#endif /* __cplusplus */

+

+int check_openssl_compatibility();

+

+#define FIPS_mode_set(X) 0

+#define FIPS_mode() 0

+

+#ifdef	__cplusplus

+}

+#endif

diff -rup mysql-8.0.22-orig/mysys/my_md5.cc mysql-8.0.22/mysys/my_md5.cc

--- mysql-8.0.22-orig/mysys/my_md5.cc	2021-05-19 21:36:31.738980913 +0200

+++ mysql-8.0.22/mysys/my_md5.cc	2021-05-19 23:13:41.380194493 +0200

@@ -34,13 +34,12 @@

 #include <openssl/crypto.h>

 #include <openssl/md5.h>

+#include <openssl/evp.h>

+#include <ssl_compat.h>

 static void my_md5_hash(unsigned char *digest, unsigned const char *buf,

                         int len) {

-  MD5_CTX ctx;

-  MD5_Init(&ctx;;

-  MD5_Update(&ctx, buf, len);

-  MD5_Final(digest, &ctx;;

+  MD5(buf, len, digest);

 }

 /**

diff -rup mysql-8.0.22-orig/plugin/group_replication/libmysqlgcs/src/bindings/xcom/xcom/xcom_ssl_transport.cc mysql-8.0.22/plugin/group_replication/libmysqlgcs/src/bindings/xcom/xcom/xcom_ssl_transport.cc

--- mysql-8.0.22-orig/plugin/group_replication/libmysqlgcs/src/bindings/xcom/xcom/xcom_ssl_transport.cc	2021-05-19 21:36:14.531793376 +0200

+++ mysql-8.0.22/plugin/group_replication/libmysqlgcs/src/bindings/xcom/xcom/xcom_ssl_transport.cc	2021-05-19 23:06:55.049885943 +0200

@@ -25,6 +25,7 @@

 #include <assert.h>

 #include <stdlib.h>

+#include <openssl/crypto.h>

 #include <openssl/dh.h>

 #include <openssl/opensslv.h>

 #include <openssl/x509v3.h>

@@ -35,6 +36,7 @@

 #endif

 #include "openssl/engine.h"

+#include <ssl_compat.h>

 #include "xcom/task_debug.h"

 #include "xcom/x_platform.h"

diff -rup mysql-8.0.22-orig/plugin/x/client/xconnection_impl.cc mysql-8.0.22/plugin/x/client/xconnection_impl.cc

--- mysql-8.0.22-orig/plugin/x/client/xconnection_impl.cc	2021-05-19 21:36:14.388791818 +0200

+++ mysql-8.0.22/plugin/x/client/xconnection_impl.cc	2021-05-19 23:06:55.049885943 +0200

@@ -31,6 +31,7 @@

 #ifdef HAVE_NETINET_IN_H

 #include <netinet/in.h>

 #endif  // HAVE_NETINET_IN_H

+#include <openssl/crypto.h>

 #include <openssl/x509v3.h>

 #include <cassert>

 #include <chrono>  // NOLINT(build/c++11)

@@ -38,6 +39,7 @@

 #include <limits>

 #include <sstream>

 #include <string>

+#include <ssl_compat.h>

 #include "errmsg.h"       // NOLINT(build/include_subdir)

 #include "my_config.h"    // NOLINT(build/include_subdir)

diff -rup mysql-8.0.22-orig/vio/viosslfactories.cc mysql-8.0.22/vio/viosslfactories.cc

--- mysql-8.0.22-orig/vio/viosslfactories.cc	2021-05-19 21:36:33.310998046 +0200

+++ mysql-8.0.22/vio/viosslfactories.cc	2021-05-19 23:06:55.049885943 +0200

@@ -39,7 +39,9 @@

 #include "mysys_err.h"

 #include "vio/vio_priv.h"

+#include <openssl/crypto.h>

 #include <openssl/dh.h>

+#include <ssl_compat.h>

 #if OPENSSL_VERSION_NUMBER < 0x10002000L

 #include <openssl/ec.h>