From 31068e063b8795ae11f3a59d4080db1fe111cfaf Mon Sep 17 00:00:00 2001

From: George Bosilca <bosilca@icl.utk.edu>

Date: Mon, 28 Dec 2020 15:36:05 -0500

Subject: [PATCH 1/3] Major update to the AVX* detection and support

1. Consistent march flag order between configure and make.

2. op/avx: give the option to skip some tests

it is possible to skip some intrinsic tests by setting some environment variables to "no" before invoking configure:

 - ompi_cv_op_avx_check_avx512

 - ompi_cv_op_avx_check_avx2

 - ompi_cv_op_avx_check_avx

 - ompi_cv_op_avx_check_sse41

 - ompi_cv_op_avx_check_sse3

3. op/avx: update AVX512 flags

try

-mavx512f -mavx512bw -mavx512vl -mavx512dq

instead of

-march=skylake-avx512

since the former is less likely to conflict with user provided CFLAGS

(e.g. -march=...)

Thanks Bart Oldeman for pointing this.

4. op/avx: have the op/avx library depend on libmpi.so

Refs. open-mpi/ompi#8323

Signed-off-by: Gilles Gouaillardet <gilles@rist.or.jp>

Signed-off-by: George Bosilca <bosilca@icl.utk.edu>

---

 ompi/mca/op/avx/Makefile.am  |   4 +-

 ompi/mca/op/avx/configure.m4 | 325 ++++++++++++++++++-----------------

 2 files changed, 174 insertions(+), 155 deletions(-)

diff --git a/ompi/mca/op/avx/Makefile.am b/ompi/mca/op/avx/Makefile.am

index 41dcf2e1834..b1d84d90b33 100644

--- a/ompi/mca/op/avx/Makefile.am

+++ b/ompi/mca/op/avx/Makefile.am

@@ -2,7 +2,7 @@

 # Copyright (c) 2019-2020 The University of Tennessee and The University

 #                         of Tennessee Research Foundation.  All rights

 #                         reserved.

-# Copyright (c) 2020      Research Organization for Information Science

+# Copyright (c) 2020-2021 Research Organization for Information Science

 #                         and Technology (RIST).  All rights reserved.

 # $COPYRIGHT$

 #

@@ -86,7 +86,7 @@ mcacomponentdir = $(ompilibdir)

 mcacomponent_LTLIBRARIES = $(component_install)

 mca_op_avx_la_SOURCES = $(sources)

 mca_op_avx_la_LIBADD = $(specialized_op_libs)

-mca_op_avx_la_LDFLAGS = -module -avoid-version

+mca_op_avx_la_LDFLAGS = -module -avoid-version $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la

 # Specific information for static builds.

diff --git a/ompi/mca/op/avx/configure.m4 b/ompi/mca/op/avx/configure.m4

index 09d8b374c8e..f61b7100ef4 100644

--- a/ompi/mca/op/avx/configure.m4

+++ b/ompi/mca/op/avx/configure.m4

@@ -29,6 +29,13 @@ AC_DEFUN([MCA_ompi_op_avx_CONFIG],[

     op_avx_support=0

     op_avx2_support=0

     op_avx512_support=0

+

+    AS_VAR_PUSHDEF([op_avx_check_sse3], [ompi_cv_op_avx_check_sse3])

+    AS_VAR_PUSHDEF([op_avx_check_sse41], [ompi_cv_op_avx_check_sse41])

+    AS_VAR_PUSHDEF([op_avx_check_avx], [ompi_cv_op_avx_check_avx])

+    AS_VAR_PUSHDEF([op_avx_check_avx2], [ompi_cv_op_avx_check_avx2])

+    AS_VAR_PUSHDEF([op_avx_check_avx512], [ompi_cv_op_avx_check_avx512])

+

     OPAL_VAR_SCOPE_PUSH([op_avx_cflags_save])

     AS_IF([test "$opal_cv_asm_arch" = "X86_64"],

@@ -37,21 +44,9 @@ AC_DEFUN([MCA_ompi_op_avx_CONFIG],[

            #

            # Check for AVX512 support

            #

-           AC_MSG_CHECKING([for AVX512 support (no additional flags)])

-           AC_LINK_IFELSE(

-               [AC_LANG_PROGRAM([[#include <immintrin.h>]],

-                                [[

-    __m512 vA, vB;

-    _mm512_add_ps(vA, vB)

-                                ]])],

-               [op_avx512_support=1

-                AC_MSG_RESULT([yes])],

-               [AC_MSG_RESULT([no])])

-

-           AS_IF([test $op_avx512_support -eq 0],

-                 [AC_MSG_CHECKING([for AVX512 support (with -march=skylake-avx512)])

-                  op_avx_cflags_save="$CFLAGS"

-                  CFLAGS="$CFLAGS -march=skylake-avx512"

+           AC_CACHE_CHECK([if we are checking for AVX512 support], op_avx_check_avx512, AS_VAR_SET(op_avx_check_avx512, yes))

+           AS_IF([test "$op_avx_check_avx512" = "yes"],

+                 [AC_MSG_CHECKING([for AVX512 support (no additional flags)])

                   AC_LINK_IFELSE(

                       [AC_LANG_PROGRAM([[#include <immintrin.h>]],

                                        [[

@@ -59,99 +54,115 @@ AC_DEFUN([MCA_ompi_op_avx_CONFIG],[

     _mm512_add_ps(vA, vB)

                                        ]])],

                       [op_avx512_support=1

-                       MCA_BUILD_OP_AVX512_FLAGS="-march=skylake-avx512"

                        AC_MSG_RESULT([yes])],

                       [AC_MSG_RESULT([no])])

-                  CFLAGS="$op_avx_cflags_save"

-                 ])

-           #

-           # Some combination of gcc and older as would not correctly build the code generated by

-           # _mm256_loadu_si256. Screen them out.

-           #

-           AS_IF([test $op_avx512_support -eq 1],

-                 [AC_MSG_CHECKING([if _mm512_loadu_si512 generates code that can be compiled])

-                  op_avx_cflags_save="$CFLAGS"

-                  CFLAGS="$CFLAGS_WITHOUT_OPTFLAGS -O0 $MCA_BUILD_OP_AVX512_FLAGS"

-                  AC_LINK_IFELSE(

-                      [AC_LANG_PROGRAM([[#include <immintrin.h>]],

-                               [[

+

+                  AS_IF([test $op_avx512_support -eq 0],

+                        [AC_MSG_CHECKING([for AVX512 support (with -mavx512f -mavx512bw -mavx512vl -mavx512dq)])

+                         op_avx_cflags_save="$CFLAGS"

+                         CFLAGS="-mavx512f -mavx512bw -mavx512vl -mavx512dq $CFLAGS"

+                         AC_LINK_IFELSE(

+                             [AC_LANG_PROGRAM([[#include <immintrin.h>]],

+                                              [[

+    __m512 vA, vB;

+    _mm512_add_ps(vA, vB)

+                                       ]])],

+                             [op_avx512_support=1

+                              MCA_BUILD_OP_AVX512_FLAGS="-mavx512f -mavx512bw -mavx512vl -mavx512dq"

+                              AC_MSG_RESULT([yes])],

+                             [AC_MSG_RESULT([no])])

+                         CFLAGS="$op_avx_cflags_save"

+                        ])

+                  #

+                  # Some combination of gcc and older as would not correctly build the code generated by

+                  # _mm256_loadu_si256. Screen them out.

+                  #

+                  AS_IF([test $op_avx512_support -eq 1],

+                        [AC_MSG_CHECKING([if _mm512_loadu_si512 generates code that can be compiled])

+                         op_avx_cflags_save="$CFLAGS"

+                         CFLAGS="$CFLAGS_WITHOUT_OPTFLAGS -O0 $MCA_BUILD_OP_AVX512_FLAGS"

+                         AC_LINK_IFELSE(

+                             [AC_LANG_PROGRAM([[#include <immintrin.h>]],

+                                      [[

     int A[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};

     __m512i vA = _mm512_loadu_si512((__m512i*)&(A[1]))

-                               ]])],

-                      [AC_MSG_RESULT([yes])],

-                      [op_avx512_support=0

-                       MCA_BUILD_OP_AVX512_FLAGS=""

-                       AC_MSG_RESULT([no])])

-                  CFLAGS="$op_avx_cflags_save"

-                 ])

-           #

-           # Some PGI compilers do not define _mm512_mullo_epi64. Screen them out.

-           #

-           AS_IF([test $op_avx512_support -eq 1],

-                 [AC_MSG_CHECKING([if _mm512_mullo_epi64 generates code that can be compiled])

-                  op_avx_cflags_save="$CFLAGS"

-                  CFLAGS="$CFLAGS_WITHOUT_OPTFLAGS -O0 $MCA_BUILD_OP_AVX512_FLAGS"

-                  AC_LINK_IFELSE(

-                      [AC_LANG_PROGRAM([[#include <immintrin.h>]],

-                               [[

+                                      ]])],

+                             [AC_MSG_RESULT([yes])],

+                             [op_avx512_support=0

+                              MCA_BUILD_OP_AVX512_FLAGS=""

+                              AC_MSG_RESULT([no])])

+                         CFLAGS="$op_avx_cflags_save"

+                        ])

+                  #

+                  # Some PGI compilers do not define _mm512_mullo_epi64. Screen them out.

+                  #

+                  AS_IF([test $op_avx512_support -eq 1],

+                        [AC_MSG_CHECKING([if _mm512_mullo_epi64 generates code that can be compiled])

+                         op_avx_cflags_save="$CFLAGS"

+                         CFLAGS="$CFLAGS_WITHOUT_OPTFLAGS -O0 $MCA_BUILD_OP_AVX512_FLAGS"

+                         AC_LINK_IFELSE(

+                             [AC_LANG_PROGRAM([[#include <immintrin.h>]],

+                                      [[

     __m512i vA, vB;

     _mm512_mullo_epi64(vA, vB)

-                               ]])],

-                      [AC_MSG_RESULT([yes])],

-                      [op_avx512_support=0

-                       MCA_BUILD_OP_AVX512_FLAGS=""

-                       AC_MSG_RESULT([no])])

-                  CFLAGS="$op_avx_cflags_save"

-                 ])

+                                      ]])],

+                             [AC_MSG_RESULT([yes])],

+                             [op_avx512_support=0

+                              MCA_BUILD_OP_AVX512_FLAGS=""

+                              AC_MSG_RESULT([no])])

+                         CFLAGS="$op_avx_cflags_save"

+                        ])])

            #

            # Check support for AVX2

            #

-           AC_MSG_CHECKING([for AVX2 support (no additional flags)])

-           AC_LINK_IFELSE(

-               [AC_LANG_PROGRAM([[#include <immintrin.h>]],

-                       [[

+           AC_CACHE_CHECK([if we are checking for AVX2 support], op_avx_check_avx2, AS_VAR_SET(op_avx_check_avx2, yes))

+           AS_IF([test "$op_avx_check_avx2" = "yes"],

+                 [AC_MSG_CHECKING([for AVX2 support (no additional flags)])

+                  AC_LINK_IFELSE(

+                      [AC_LANG_PROGRAM([[#include <immintrin.h>]],

+                              [[

     __m256 vA, vB;

     _mm256_add_ps(vA, vB)

-                       ]])],

-               [op_avx2_support=1

-                AC_MSG_RESULT([yes])],

-               [AC_MSG_RESULT([no])])

-           AS_IF([test $op_avx2_support -eq 0],

-               [AC_MSG_CHECKING([for AVX2 support (with -mavx2)])

-                op_avx_cflags_save="$CFLAGS"

-                CFLAGS="$CFLAGS -mavx2"

-                AC_LINK_IFELSE(

-                    [AC_LANG_PROGRAM([[#include <immintrin.h>]],

-                            [[

+                              ]])],

+                      [op_avx2_support=1

+                       AC_MSG_RESULT([yes])],

+                      [AC_MSG_RESULT([no])])

+                  AS_IF([test $op_avx2_support -eq 0],

+                      [AC_MSG_CHECKING([for AVX2 support (with -mavx2)])

+                       op_avx_cflags_save="$CFLAGS"

+                       CFLAGS="-mavx2 $CFLAGS"

+                       AC_LINK_IFELSE(

+                           [AC_LANG_PROGRAM([[#include <immintrin.h>]],

+                                   [[

     __m256 vA, vB;

     _mm256_add_ps(vA, vB)

-                            ]])],

-                    [op_avx2_support=1

-                     MCA_BUILD_OP_AVX2_FLAGS="-mavx2"

-                     AC_MSG_RESULT([yes])],

-                    [AC_MSG_RESULT([no])])

-                CFLAGS="$op_avx_cflags_save"

-                ])

-           #

-           # Some combination of gcc and older as would not correctly build the code generated by

-           # _mm256_loadu_si256. Screen them out.

-           #

-           AS_IF([test $op_avx2_support -eq 1],

-                 [AC_MSG_CHECKING([if _mm256_loadu_si256 generates code that can be compiled])

-                  op_avx_cflags_save="$CFLAGS"

-                  CFLAGS="$CFLAGS_WITHOUT_OPTFLAGS -O0 $MCA_BUILD_OP_AVX2_FLAGS"

-                  AC_LINK_IFELSE(

-                      [AC_LANG_PROGRAM([[#include <immintrin.h>]],

-                               [[

+                                   ]])],

+                           [op_avx2_support=1

+                            MCA_BUILD_OP_AVX2_FLAGS="-mavx2"

+                            AC_MSG_RESULT([yes])],

+                           [AC_MSG_RESULT([no])])

+                       CFLAGS="$op_avx_cflags_save"

+                       ])

+                  #

+                  # Some combination of gcc and older as would not correctly build the code generated by

+                  # _mm256_loadu_si256. Screen them out.

+                  #

+                  AS_IF([test $op_avx2_support -eq 1],

+                        [AC_MSG_CHECKING([if _mm256_loadu_si256 generates code that can be compiled])

+                         op_avx_cflags_save="$CFLAGS"

+                         CFLAGS="$CFLAGS_WITHOUT_OPTFLAGS -O0 $MCA_BUILD_OP_AVX2_FLAGS"

+                         AC_LINK_IFELSE(

+                             [AC_LANG_PROGRAM([[#include <immintrin.h>]],

+                                      [[

     int A[8] = {0, 1, 2, 3, 4, 5, 6, 7};

     __m256i vA = _mm256_loadu_si256((__m256i*)&A)

-                               ]])],

-                      [AC_MSG_RESULT([yes])],

-                      [op_avx2_support=0

-                       MCA_BUILD_OP_AVX2_FLAGS=""

-                       AC_MSG_RESULT([no])])

-                  CFLAGS="$op_avx_cflags_save"

-                 ])

+                                      ]])],

+                             [AC_MSG_RESULT([yes])],

+                             [op_avx2_support=0

+                              MCA_BUILD_OP_AVX2_FLAGS=""

+                              AC_MSG_RESULT([no])])

+                         CFLAGS="$op_avx_cflags_save"

+                        ])])

            #

            # What about early AVX support. The rest of the logic is slightly different as

            # we need to include some of the SSE4.1 and SSE3 instructions. So, we first check

@@ -160,90 +171,92 @@ AC_DEFUN([MCA_ompi_op_avx_CONFIG],[

            # the AVX flag, and then recheck if we have support for the SSE4.1 and SSE3

            # instructions.

            #

-           AC_MSG_CHECKING([for AVX support (no additional flags)])

-           AC_LINK_IFELSE(

-               [AC_LANG_PROGRAM([[#include <immintrin.h>]],

-                       [[

+           AC_CACHE_CHECK([if we are checking for AVX support], op_avx_check_avx, AS_VAR_SET(op_avx_check_avx, yes))

+           AS_IF([test "$op_avx_check_avx" = "yes"],

+                 [AC_MSG_CHECKING([for AVX support (no additional flags)])

+                  AC_LINK_IFELSE(

+                      [AC_LANG_PROGRAM([[#include <immintrin.h>]],

+                              [[

     __m128 vA, vB;

     _mm_add_ps(vA, vB)

-                       ]])],

-               [op_avx_support=1

-                AC_MSG_RESULT([yes])],

-               [AC_MSG_RESULT([no])])

+                              ]])],

+                      [op_avx_support=1

+                       AC_MSG_RESULT([yes])],

+                      [AC_MSG_RESULT([no])])])

            #

            # Check for SSE4.1 support

            #

-           AS_IF([test $op_avx_support -eq 1],

-               [AC_MSG_CHECKING([for SSE4.1 support])

-                AC_LINK_IFELSE(

-                    [AC_LANG_PROGRAM([[#include <immintrin.h>]],

-                            [[

+           AC_CACHE_CHECK([if we are checking for SSE4.1 support], op_avx_check_sse41, AS_VAR_SET(op_avx_check_sse41, yes))

+           AS_IF([test $op_avx_support -eq 1 && test "$op_avx_check_sse41" = "yes"],

+                 [AC_MSG_CHECKING([for SSE4.1 support])

+                  AC_LINK_IFELSE(

+                      [AC_LANG_PROGRAM([[#include <immintrin.h>]],

+                              [[

     __m128i vA, vB;

     (void)_mm_max_epi8(vA, vB)

-                            ]])],

-                    [op_sse41_support=1

-                     AC_MSG_RESULT([yes])],

-                    [AC_MSG_RESULT([no])])

-                ])

+                              ]])],

+                      [op_sse41_support=1

+                       AC_MSG_RESULT([yes])],

+                      [AC_MSG_RESULT([no])])

+                  ])

            #

            # Check for SSE3 support

            #

-           AS_IF([test $op_avx_support -eq 1],

-               [AC_MSG_CHECKING([for SSE3 support])

-                AC_LINK_IFELSE(

-                    [AC_LANG_PROGRAM([[#include <immintrin.h>]],

-                            [[

+           AC_CACHE_CHECK([if we are checking for SSE3 support], op_avx_check_sse3, AS_VAR_SET(op_avx_check_sse3, yes))

+           AS_IF([test $op_avx_support -eq 1 && test "$op_avx_check_sse3" = "yes"],

+                 [AC_MSG_CHECKING([for SSE3 support])

+                  AC_LINK_IFELSE(

+                      [AC_LANG_PROGRAM([[#include <immintrin.h>]],

+                              [[

     int A[4] = {0, 1, 2, 3};

     __m128i vA = _mm_lddqu_si128((__m128i*)&A)

-                            ]])],

-                    [op_sse3_support=1

-                     AC_MSG_RESULT([yes])],

-                    [AC_MSG_RESULT([no])])

-                ])

+                              ]])],

+                      [op_sse3_support=1

+                       AC_MSG_RESULT([yes])],

+                      [AC_MSG_RESULT([no])])

+                  ])

            # Second pass, do we need to add the AVX flag ?

            AS_IF([test $op_avx_support -eq 0 || test $op_sse41_support -eq 0 || test $op_sse3_support -eq 0],

-               [AC_MSG_CHECKING([for AVX support (with -mavx)])

-                op_avx_cflags_save="$CFLAGS"

-                CFLAGS="$CFLAGS -mavx"

-                AC_LINK_IFELSE(

-                    [AC_LANG_PROGRAM([[#include <immintrin.h>]],

-                            [[

+                 [AS_IF([test "$op_avx_check_avx" = "yes"],

+                        [AC_MSG_CHECKING([for AVX support (with -mavx)])

+                         op_avx_cflags_save="$CFLAGS"

+                         CFLAGS="-mavx $CFLAGS"

+                         AC_LINK_IFELSE(

+                             [AC_LANG_PROGRAM([[#include <immintrin.h>]],

+                                   [[

     __m128 vA, vB;

     _mm_add_ps(vA, vB)

                             ]])],

-                    [op_avx_support=1

-                     MCA_BUILD_OP_AVX_FLAGS="-mavx"

-                     op_sse41_support=0

-                     op_sse3_support=0

-                     AC_MSG_RESULT([yes])],

-                    [AC_MSG_RESULT([no])])

+                             [op_avx_support=1

+                              MCA_BUILD_OP_AVX_FLAGS="-mavx"

+                              op_sse41_support=0

+                              op_sse3_support=0

+                              AC_MSG_RESULT([yes])],

+                             [AC_MSG_RESULT([no])])])

-                AS_IF([test $op_sse41_support -eq 0],

-                    [AC_MSG_CHECKING([for SSE4.1 support])

-                     AC_LINK_IFELSE(

-                         [AC_LANG_PROGRAM([[#include <immintrin.h>]],

-                                 [[

+                  AS_IF([test "$op_avx_check_sse41" = "yes" && test $op_sse41_support -eq 0],

+                        [AC_MSG_CHECKING([for SSE4.1 support])

+                         AC_LINK_IFELSE(

+                             [AC_LANG_PROGRAM([[#include <immintrin.h>]],

+                                     [[

     __m128i vA, vB;

     (void)_mm_max_epi8(vA, vB)

-                                 ]])],

-                         [op_sse41_support=1

-                          AC_MSG_RESULT([yes])],

-                         [AC_MSG_RESULT([no])])

-                     ])

-                AS_IF([test $op_sse3_support -eq 0],

-                    [AC_MSG_CHECKING([for SSE3 support])

-                     AC_LINK_IFELSE(

-                         [AC_LANG_PROGRAM([[#include <immintrin.h>]],

+                                     ]])],

+                             [op_sse41_support=1

+                              AC_MSG_RESULT([yes])],

+                             [AC_MSG_RESULT([no])])])

+                  AS_IF([test "$op_avx_check_sse3" = "yes" && test $op_sse3_support -eq 0],

+                        [AC_MSG_CHECKING([for SSE3 support])

+                         AC_LINK_IFELSE(

+                             [AC_LANG_PROGRAM([[#include <immintrin.h>]],

                                  [[

     int A[4] = {0, 1, 2, 3};

     __m128i vA = _mm_lddqu_si128((__m128i*)&A)

                                  ]])],

-                         [op_sse3_support=1

-                          AC_MSG_RESULT([yes])],

-                         [AC_MSG_RESULT([no])])

-                     ])

-                CFLAGS="$op_avx_cflags_save"

-               ])

+                             [op_sse3_support=1

+                              AC_MSG_RESULT([yes])],

+                             [AC_MSG_RESULT([no])])])

+                  CFLAGS="$op_avx_cflags_save"])

            AC_LANG_POP([C])

           ])

@@ -276,6 +289,12 @@ AC_DEFUN([MCA_ompi_op_avx_CONFIG],[

     AC_SUBST(MCA_BUILD_OP_AVX2_FLAGS)

     AC_SUBST(MCA_BUILD_OP_AVX_FLAGS)

+    AS_VAR_POPDEF([op_avx_check_avx512])

+    AS_VAR_POPDEF([op_avx_check_avx2])

+    AS_VAR_POPDEF([op_avx_check_avx])

+    AS_VAR_POPDEF([op_avx_check_sse41])

+    AS_VAR_POPDEF([op_avx_check_sse3])

+

     OPAL_VAR_SCOPE_POP

     # Enable this component iff we have at least the most basic form of support

     # for vectorial ISA

From fcf2766a03e3c2a1001679013878209bcddd50ae Mon Sep 17 00:00:00 2001

From: George Bosilca <bosilca@icl.utk.edu>

Date: Mon, 28 Dec 2020 12:18:07 -0500

Subject: [PATCH 2/3] AVX code generation improvements

1. Allow fallback to a lesser AVX support during make

Due to the fact that some distro restrict the compiule architecture

during make (while not setting any restrictions during configure) we

need to detect the target architecture also during make in order to

restrict the code we generate.

2. Add comments and better protect the arch specific code.

Identify all the vectorial functions used and clasify them according to

the neccesary hardware capabilities.

Use these requirements to protect the code for load and stores (the rest

of the code being automatically generated it is more difficult to

protect).

3. Correctly check for AVX* support.

Signed-off-by: George Bosilca <bosilca@icl.utk.edu>

---

 ompi/mca/op/avx/configure.m4       |  28 +--

 ompi/mca/op/avx/op_avx_functions.c | 322 ++++++++++++++++++++++++-----

 2 files changed, 288 insertions(+), 62 deletions(-)

diff --git a/ompi/mca/op/avx/configure.m4 b/ompi/mca/op/avx/configure.m4

index f61b7100ef4..f3651f09d43 100644

--- a/ompi/mca/op/avx/configure.m4

+++ b/ompi/mca/op/avx/configure.m4

@@ -44,7 +44,7 @@ AC_DEFUN([MCA_ompi_op_avx_CONFIG],[

            #

            # Check for AVX512 support

            #

-           AC_CACHE_CHECK([if we are checking for AVX512 support], op_avx_check_avx512, AS_VAR_SET(op_avx_check_avx512, yes))

+           AC_CACHE_CHECK([for AVX512 support], op_avx_check_avx512, AS_VAR_SET(op_avx_check_avx512, yes))

            AS_IF([test "$op_avx_check_avx512" = "yes"],

                  [AC_MSG_CHECKING([for AVX512 support (no additional flags)])

                   AC_LINK_IFELSE(

@@ -115,14 +115,14 @@ AC_DEFUN([MCA_ompi_op_avx_CONFIG],[

            #

            # Check support for AVX2

            #

-           AC_CACHE_CHECK([if we are checking for AVX2 support], op_avx_check_avx2, AS_VAR_SET(op_avx_check_avx2, yes))

+           AC_CACHE_CHECK([for AVX2 support], op_avx_check_avx2, AS_VAR_SET(op_avx_check_avx2, yes))

            AS_IF([test "$op_avx_check_avx2" = "yes"],

                  [AC_MSG_CHECKING([for AVX2 support (no additional flags)])

                   AC_LINK_IFELSE(

                       [AC_LANG_PROGRAM([[#include <immintrin.h>]],

                               [[

-    __m256 vA, vB;

-    _mm256_add_ps(vA, vB)

+    __m256i vA, vB, vC;

+    vC = _mm256_and_si256(vA, vB)

                               ]])],

                       [op_avx2_support=1

                        AC_MSG_RESULT([yes])],

@@ -134,8 +134,8 @@ AC_DEFUN([MCA_ompi_op_avx_CONFIG],[

                        AC_LINK_IFELSE(

                            [AC_LANG_PROGRAM([[#include <immintrin.h>]],

                                    [[

-    __m256 vA, vB;

-    _mm256_add_ps(vA, vB)

+    __m256i vA, vB, vC;

+    vC = _mm256_and_si256(vA, vB)

                                    ]])],

                            [op_avx2_support=1

                             MCA_BUILD_OP_AVX2_FLAGS="-mavx2"

@@ -164,21 +164,21 @@ AC_DEFUN([MCA_ompi_op_avx_CONFIG],[

                          CFLAGS="$op_avx_cflags_save"

                         ])])

            #

-           # What about early AVX support. The rest of the logic is slightly different as

+           # What about early AVX support? The rest of the logic is slightly different as

            # we need to include some of the SSE4.1 and SSE3 instructions. So, we first check

            # if we can compile AVX code without a flag, then we validate that we have support

            # for the SSE4.1 and SSE3 instructions we need. If not, we check for the usage of

            # the AVX flag, and then recheck if we have support for the SSE4.1 and SSE3

            # instructions.

            #

-           AC_CACHE_CHECK([if we are checking for AVX support], op_avx_check_avx, AS_VAR_SET(op_avx_check_avx, yes))

+           AC_CACHE_CHECK([for AVX support], op_avx_check_avx, AS_VAR_SET(op_avx_check_avx, yes))

            AS_IF([test "$op_avx_check_avx" = "yes"],

                  [AC_MSG_CHECKING([for AVX support (no additional flags)])

                   AC_LINK_IFELSE(

                       [AC_LANG_PROGRAM([[#include <immintrin.h>]],

                               [[

-    __m128 vA, vB;

-    _mm_add_ps(vA, vB)

+    __m256 vA, vB, vC;

+    vC = _mm256_add_ps(vA, vB)

                               ]])],

                       [op_avx_support=1

                        AC_MSG_RESULT([yes])],

@@ -186,7 +186,7 @@ AC_DEFUN([MCA_ompi_op_avx_CONFIG],[

            #

            # Check for SSE4.1 support

            #

-           AC_CACHE_CHECK([if we are checking for SSE4.1 support], op_avx_check_sse41, AS_VAR_SET(op_avx_check_sse41, yes))

+           AC_CACHE_CHECK([for SSE4.1 support], op_avx_check_sse41, AS_VAR_SET(op_avx_check_sse41, yes))

            AS_IF([test $op_avx_support -eq 1 && test "$op_avx_check_sse41" = "yes"],

                  [AC_MSG_CHECKING([for SSE4.1 support])

                   AC_LINK_IFELSE(

@@ -202,7 +202,7 @@ AC_DEFUN([MCA_ompi_op_avx_CONFIG],[

            #

            # Check for SSE3 support

            #

-           AC_CACHE_CHECK([if we are checking for SSE3 support], op_avx_check_sse3, AS_VAR_SET(op_avx_check_sse3, yes))

+           AC_CACHE_CHECK([for SSE3 support], op_avx_check_sse3, AS_VAR_SET(op_avx_check_sse3, yes))

            AS_IF([test $op_avx_support -eq 1 && test "$op_avx_check_sse3" = "yes"],

                  [AC_MSG_CHECKING([for SSE3 support])

                   AC_LINK_IFELSE(

@@ -224,8 +224,8 @@ AC_DEFUN([MCA_ompi_op_avx_CONFIG],[

                          AC_LINK_IFELSE(

                              [AC_LANG_PROGRAM([[#include <immintrin.h>]],

                                    [[

-    __m128 vA, vB;

-    _mm_add_ps(vA, vB)

+    __m256 vA, vB, vC;

+    vC = _mm256_add_ps(vA, vB)

                             ]])],

                              [op_avx_support=1

                               MCA_BUILD_OP_AVX_FLAGS="-mavx"

diff --git a/ompi/mca/op/avx/op_avx_functions.c b/ompi/mca/op/avx/op_avx_functions.c

index 95a9c9ab84e..ef3f0932906 100644

--- a/ompi/mca/op/avx/op_avx_functions.c

+++ b/ompi/mca/op/avx/op_avx_functions.c

@@ -1,5 +1,5 @@

 /*

- * Copyright (c) 2019-2020 The University of Tennessee and The University

+ * Copyright (c) 2019-2021 The University of Tennessee and The University

  *                         of Tennessee Research Foundation.  All rights

  *                         reserved.

  * Copyright (c) 2020      Research Organization for Information Science

@@ -24,16 +24,42 @@

 #include "ompi/mca/op/avx/op_avx.h"

 #include <immintrin.h>

-

+/**

+ * The following logic is necessary to cope with distro maintainer's desire to change the compilation

+ * flags after the configure step, leading to inconsistencies between what OMPI has detected and what

+ * code can be generated during make. If we detect that the current code generation architecture has

+ * been changed from our own setting and cannot generate the code we need (AVX512, AVX2) we fall back

+ * to a lesser support (AVX512 -> AVX2, AVX2 -> AVX, AVX -> error out).

+ */

 #if defined(GENERATE_AVX512_CODE)

-#define PREPEND _avx512

-#elif defined(GENERATE_AVX2_CODE)

-#define PREPEND _avx2

-#elif defined(GENERATE_AVX_CODE)

-#define PREPEND _avx

-#else

-#error This file should not be compiled in this conditions

-#endif

+#  if defined(__AVX512BW__) && defined(__AVX512F__) && defined(__AVX512VL__)

+#    define PREPEND _avx512

+#  else

+#    undef GENERATE_AVX512_CODE

+#  endif  /* defined(__AVX512BW__) && defined(__AVX512F__) && defined(__AVX512VL__) */

+#endif  /* defined(GENERATE_AVX512_CODE) */

+

+#if !defined(PREPEND) && defined(GENERATE_AVX2_CODE)

+#  if defined(__AVX2__)

+#    define PREPEND _avx2

+#  else

+#    undef GENERATE_AVX2_CODE

+#  endif  /* defined(__AVX2__) */

+#endif  /* !defined(PREPEND) && defined(GENERATE_AVX2_CODE) */

+

+#if !defined(PREPEND) && defined(GENERATE_AVX_CODE)

+#  if defined(__AVX__)

+#    define PREPEND _avx

+#  endif

+#endif  /* !defined(PREPEND) && defined(GENERATE_AVX_CODE) */

+

+#if !defined(PREPEND)

+#  if OMPI_MCA_OP_HAVE_AVX512 || OMPI_MCA_OP_HAVE_AVX2

+#    error The configure step has detected possible support for AVX512 and/or AVX2 but the compiler flags during make are too restrictive. Please disable the AVX component by adding --enable-mca-no-build=op-avx to your configure step.

+#  else

+#    error This file should not be compiled in this conditions. Please provide the config.log file to the OMPI developers.

+#  endif  /* OMPI_MCA_OP_HAVE_AVX512 || OMPI_MCA_OP_HAVE_AVX2 */

+#endif  /* !defined(PREPEND) */

 /*

  * Concatenate preprocessor tokens A and B without expanding macro definitions

@@ -46,6 +72,102 @@

  */

 #define OP_CONCAT(A, B) OP_CONCAT_NX(A, B)

+/*

+ * grep -e "_mm[125][251][862]_.*(" avx512.c -o | sed 's/(//g' | sort | uniq

+ *

+ * https://software.intel.com/sites/landingpage/IntrinsicsGuide

+ *

+ * _mm_add_epi[8,16,32,64]         SSE2

+ * _mm_add_pd                      SSE2

+ * _mm_add_ps                      SSE

+ * _mm_adds_epi[8,16]              SSE2

+ * _mm_adds_epu[8,16]              SSE2

+ * _mm_and_si128                   SSE2

+ * _mm_lddqu_si128                 SSE3

+ * _mm_loadu_pd                    SSE2

+ * _mm_loadu_ps                    SSE

+ * _mm_max_epi8                    SSE4.1

+ * _mm_max_epi16                   SSE2

+ * _mm_max_epi32                   SSE4.1

+ * _mm_max_epi64                   AVX512VL + AVX512F

+ * _mm_max_epu8                    SSE2

+ * _mm_max_epu[16,32]              SSE4.1

+ * _mm_max_epu64                   AVX512VL + AVX512F

+ * _mm_max_pd                      SSE2

+ * _mm_max_ps                      SSE

+ * _mm_min_epi8                    SSE4.1

+ * _mm_min_epi16                   SSE2

+ * _mm_min_epi32                   SSE4.1

+ * _mm_min_epi64                   AVX512VL + AVX512F

+ * _mm_min_epu8                    SSE2

+ * _mm_min_epu[16,32]              SSE4.1

+ * _mm_min_epu64                   AVX512VL + AVX512F

+ * _mm_min_pd                      SSE2

+ * _mm_min_ps                      SSE

+ * _mm_mul_pd                      SSE2

+ * _mm_mul_ps                      SSE

+ * _mm_mullo_epi16                 SSE2

+ * _mm_mullo_epi32                 SSE4.1

+ * _mm_mullo_epi64                 AVX512VL + AVX512DQ

+ * _mm_or_si128                    SSE2

+ * _mm_storeu_pd                   SSE2

+ * _mm_storeu_ps                   SSE

+ * _mm_storeu_si128                SSE2

+ * _mm_xor_si128                   SSE2

+ * _mm256_add_epi[8,16,32,64]      AVX2

+ * _mm256_add_p[s,d]               AVX

+ * _mm256_adds_epi[8,16]           AVX2

+ * _mm256_adds_epu[8,16]           AVX2

+ * _mm256_and_si256                AVX2

+ * _mm256_loadu_p[s,d]             AVX

+ * _mm256_loadu_si256              AVX

+ * _mm256_max_epi[8,16,32]         AVX2

+ * _mm256_max_epi64                AVX512VL + AVX512F

+ * _mm256_max_epu[8,16,32]         AVX2

+ * _mm256_max_epu64                AVX512VL + AVX512F

+ * _mm256_max_p[s,d]               AVX

+ * _mm256_min_epi[8,16,32]         AVX2

+ * _mm256_min_epi64                AVX512VL + AVX512F

+ * _mm256_min_epu[8,16,32]         AVX2

+ * _mm256_min_epu64                AVX512VL + AVX512F

+ * _mm256_min_p[s,d]               AVX

+ * _mm256_mul_p[s,d]               AVX

+ * _mm256_mullo_epi[16,32]         AVX2

+ * _mm256_mullo_epi64              AVX512VL + AVX512DQ

+ * _mm256_or_si256                 AVX2

+ * _mm256_storeu_p[s,d]            AVX

+ * _mm256_storeu_si256             AVX

+ * _mm256_xor_si256                AVX2

+ * _mm512_add_epi[8,16]            AVX512BW

+ * _mm512_add_epi[32,64]           AVX512F

+ * _mm512_add_p[s,d]               AVX512F

+ * _mm512_adds_epi[8,16]           AVX512BW

+ * _mm512_adds_epu[8,16]           AVX512BW

+ * _mm512_and_si512                AVX512F

+ * _mm512_cvtepi16_epi8            AVX512BW

+ * _mm512_cvtepi8_epi16            AVX512BW

+ * _mm512_loadu_p[s,d]             AVX512F

+ * _mm512_loadu_si512              AVX512F

+ * _mm512_max_epi[8,16]            AVX512BW

+ * _mm512_max_epi[32,64]           AVX512F

+ * _mm512_max_epu[8,16]            AVX512BW

+ * _mm512_max_epu[32,64]           AVX512F

+ * _mm512_max_p[s,d]               AVX512F

+ * _mm512_min_epi[8,16]            AVX512BW

+ * _mm512_min_epi[32,64]           AVX512F

+ * _mm512_min_epu[8,16]            AVX512BW

+ * _mm512_min_epu[32,64]           AVX512F

+ * _mm512_min_p[s,d]               AVX512F

+ * _mm512_mul_p[s,d]               AVX512F

+ * _mm512_mullo_epi16              AVX512BW

+ * _mm512_mullo_epi32              AVX512F

+ * _mm512_mullo_epi64              AVX512DQ

+ * _mm512_or_si512                 AVX512F

+ * _mm512_storeu_p[s,d]            AVX512F

+ * _mm512_storeu_si512             AVX512F

+ * _mm512_xor_si512                AVX512F

+ */

+

 /*

  * Since all the functions in this file are essentially identical, we

  * use a macro to substitute in names and types.  The core operation

@@ -62,13 +184,14 @@

   (((_flag) & mca_op_avx_component.flags) == (_flag))

 #if defined(GENERATE_AVX512_CODE) && defined(OMPI_MCA_OP_HAVE_AVX512) && (1 == OMPI_MCA_OP_HAVE_AVX512)

+#if __AVX512F__

 #define OP_AVX_AVX512_FUNC(name, type_sign, type_size, type, op)               \

     if( OMPI_OP_AVX_HAS_FLAGS(OMPI_OP_AVX_HAS_AVX512F_FLAG|OMPI_OP_AVX_HAS_AVX512BW_FLAG) ) { \

         int types_per_step = (512 / 8) / sizeof(type);                         \

         for( ; left_over >= types_per_step; left_over -= types_per_step ) {    \

-            __m512i vecA =  _mm512_loadu_si512((__m512*)in);                   \

+            __m512i vecA = _mm512_loadu_si512((__m512*)in);                    \

             in += types_per_step;                                              \

-            __m512i vecB =  _mm512_loadu_si512((__m512*)out);                  \

+            __m512i vecB = _mm512_loadu_si512((__m512*)out);                   \

             __m512i res = _mm512_##op##_ep##type_sign##type_size(vecA, vecB);  \

             _mm512_storeu_si512((__m512*)out, res);                            \

             out += types_per_step;                                             \

@@ -76,10 +199,14 @@

         if( 0 == left_over ) return;                                           \

     }

 #else

+#error Target architecture lacks AVX512F support needed for _mm512_loadu_si512 and _mm512_storeu_si512

+#endif  /* __AVX512F__ */

+#else

 #define OP_AVX_AVX512_FUNC(name, type_sign, type_size, type, op) {}

 #endif  /* defined(OMPI_MCA_OP_HAVE_AVX512) && (1 == OMPI_MCA_OP_HAVE_AVX512) */

 #if defined(GENERATE_AVX2_CODE) && defined(OMPI_MCA_OP_HAVE_AVX2) && (1 == OMPI_MCA_OP_HAVE_AVX2)

+#if __AVX__

 #define OP_AVX_AVX2_FUNC(name, type_sign, type_size, type, op)                 \

     if( OMPI_OP_AVX_HAS_FLAGS(OMPI_OP_AVX_HAS_AVX2_FLAG | OMPI_OP_AVX_HAS_AVX_FLAG) ) {  \

         int types_per_step = (256 / 8) / sizeof(type);  /* AVX2 */             \

@@ -87,30 +214,37 @@

             __m256i vecA = _mm256_loadu_si256((__m256i*)in);                   \

             in += types_per_step;                                              \

             __m256i vecB = _mm256_loadu_si256((__m256i*)out);                  \

-            __m256i res =  _mm256_##op##_ep##type_sign##type_size(vecA, vecB); \

+            __m256i res = _mm256_##op##_ep##type_sign##type_size(vecA, vecB);  \

             _mm256_storeu_si256((__m256i*)out, res);                           \

             out += types_per_step;                                             \

         }                                                                      \

         if( 0 == left_over ) return;                                           \

     }

 #else

+#error Target architecture lacks AVX support needed for _mm256_loadu_si256 and _mm256_storeu_si256

+#endif  /* __AVX__ */

+#else

 #define OP_AVX_AVX2_FUNC(name, type_sign, type_size, type, op) {}

 #endif  /* defined(OMPI_MCA_OP_HAVE_AVX2) && (1 == OMPI_MCA_OP_HAVE_AVX2) */

 #if defined(GENERATE_SSE3_CODE) && defined(OMPI_MCA_OP_HAVE_AVX) && (1 == OMPI_MCA_OP_HAVE_AVX)

+#if __SSE3__

 #define OP_AVX_SSE4_1_FUNC(name, type_sign, type_size, type, op)               \

     if( OMPI_OP_AVX_HAS_FLAGS(OMPI_OP_AVX_HAS_SSE3_FLAG | OMPI_OP_AVX_HAS_SSE4_1_FLAG) ) { \

-        int types_per_step = (128 / 8) / sizeof(type);  /* AVX */              \

+        int types_per_step = (128 / 8) / sizeof(type);                         \

         for( ; left_over >= types_per_step; left_over -= types_per_step ) {    \

             __m128i vecA = _mm_lddqu_si128((__m128i*)in);                      \

             in += types_per_step;                                              \

             __m128i vecB = _mm_lddqu_si128((__m128i*)out);                     \

-            __m128i res =  _mm_##op##_ep##type_sign##type_size(vecA, vecB);    \

+            __m128i res = _mm_##op##_ep##type_sign##type_size(vecA, vecB);     \

             _mm_storeu_si128((__m128i*)out, res);                              \

             out += types_per_step;                                             \

         }                                                                      \

     }

 #else

+#error Target architecture lacks SSE3 support needed for _mm_lddqu_si128 and _mm_storeu_si128

+#endif  /* __SSE3__ */

+#else

 #define OP_AVX_SSE4_1_FUNC(name, type_sign, type_size, type, op) {}

 #endif  /* defined(OMPI_MCA_OP_HAVE_AVX) && (1 == OMPI_MCA_OP_HAVE_AVX) */

@@ -143,12 +277,13 @@ static void OP_CONCAT(ompi_op_avx_2buff_##name##_##type,PREPEND)(const void *_in

 }

 #if defined(GENERATE_AVX512_CODE) && defined(OMPI_MCA_OP_HAVE_AVX512) && (1 == OMPI_MCA_OP_HAVE_AVX512)

+#if __AVX512BW__ && __AVX__

 #define OP_AVX_AVX512_MUL(name, type_sign, type_size, type, op)         \

     if( OMPI_OP_AVX_HAS_FLAGS(OMPI_OP_AVX_HAS_AVX512F_FLAG | OMPI_OP_AVX_HAS_AVX512BW_FLAG) ) {  \

         int types_per_step = (256 / 8) / sizeof(type);                  \

         for (; left_over >= types_per_step; left_over -= types_per_step) { \

-            __m256i vecA_tmp =  _mm256_loadu_si256((__m256i*)in);       \

-            __m256i vecB_tmp =  _mm256_loadu_si256((__m256i*)out);      \

+            __m256i vecA_tmp = _mm256_loadu_si256((__m256i*)in);        \

+            __m256i vecB_tmp = _mm256_loadu_si256((__m256i*)out);       \

             in += types_per_step;                                       \

             __m512i vecA = _mm512_cvtepi8_epi16(vecA_tmp);              \

             __m512i vecB = _mm512_cvtepi8_epi16(vecB_tmp);              \

@@ -160,6 +295,9 @@ static void OP_CONCAT(ompi_op_avx_2buff_##name##_##type,PREPEND)(const void *_in

         if( 0 == left_over ) return;                                    \

     }

 #else

+#error Target architecture lacks AVX512BW and AVX support needed for _mm256_loadu_si256, _mm256_storeu_si256 and _mm512_cvtepi8_epi16

+#endif  /* __AVX512BW__ && __AVX__ */

+#else

 #define OP_AVX_AVX512_MUL(name, type_sign, type_size, type, op) {}

 #endif  /* defined(OMPI_MCA_OP_HAVE_AVX512) && (1 == OMPI_MCA_OP_HAVE_AVX512) */

 /**

@@ -201,13 +339,14 @@ static void OP_CONCAT( ompi_op_avx_2buff_##name##_##type, PREPEND)(const void *_

  *

  */

 #if defined(GENERATE_AVX512_CODE) && defined(OMPI_MCA_OP_HAVE_AVX512) && (1 == OMPI_MCA_OP_HAVE_AVX512)

+#if __AVX512F__

 #define OP_AVX_AVX512_BIT_FUNC(name, type_size, type, op)               \

     if( OMPI_OP_AVX_HAS_FLAGS( OMPI_OP_AVX_HAS_AVX512F_FLAG) ) {        \

         types_per_step = (512 / 8) / sizeof(type);                      \

         for (; left_over >= types_per_step; left_over -= types_per_step) { \

-            __m512i vecA =  _mm512_loadu_si512((__m512i*)in);           \

+            __m512i vecA = _mm512_loadu_si512((__m512i*)in);            \

             in += types_per_step;                                       \

-            __m512i vecB =  _mm512_loadu_si512((__m512i*)out);          \

+            __m512i vecB = _mm512_loadu_si512((__m512i*)out);           \

             __m512i res = _mm512_##op##_si512(vecA, vecB);              \

             _mm512_storeu_si512((__m512i*)out, res);                    \

             out += types_per_step;                                      \

@@ -215,10 +354,14 @@ static void OP_CONCAT( ompi_op_avx_2buff_##name##_##type, PREPEND)(const void *_

         if( 0 == left_over ) return;                                    \

     }

 #else

+#error Target architecture lacks AVX512F support needed for _mm512_loadu_si512 and _mm512_storeu_si512

+#endif  /* __AVX512F__ */

+#else