076f82
commit 82a707aeb74f23bb1783af0f9e93790a2038ff7e
076f82
Author: Raghuveer Devulapalli <raghuveer.devulapalli@intel.com>
076f82
Date:   Mon Jun 6 12:17:43 2022 -0700
076f82
076f82
    x86_64: Add strstr function with 512-bit EVEX
076f82
    
076f82
    Adding a 512-bit EVEX version of strstr. The algorithm works as follows:
076f82
    
076f82
    (1) We spend a few cycles at the begining to peek into the needle. We
076f82
    locate an edge in the needle (first occurance of 2 consequent distinct
076f82
    characters) and also store the first 64-bytes into a zmm register.
076f82
    
076f82
    (2) We search for the edge in the haystack by looking into one cache
076f82
    line of the haystack at a time. This avoids having to read past a page
076f82
    boundary which can cause a seg fault.
076f82
    
076f82
    (3) If an edge is found in the haystack we first compare the first
076f82
    64-bytes of the needle (already stored in a zmm register) before we
076f82
    proceed with a full string compare performed byte by byte.
076f82
    
076f82
    Benchmarking results: (old = strstr_sse2_unaligned, new = strstr_avx512)
076f82
    
076f82
    Geometric mean of all benchmarks: new / old =  0.66
076f82
    
076f82
    Difficult skiptable(0) : new / old =  0.02
076f82
    Difficult skiptable(1) : new / old =  0.01
076f82
    Difficult 2-way : new / old =  0.25
076f82
    Difficult testing first 2 : new / old =  1.26
076f82
    Difficult skiptable(0) : new / old =  0.05
076f82
    Difficult skiptable(1) : new / old =  0.06
076f82
    Difficult 2-way : new / old =  0.26
076f82
    Difficult testing first 2 : new / old =  1.05
076f82
    Difficult skiptable(0) : new / old =  0.42
076f82
    Difficult skiptable(1) : new / old =  0.24
076f82
    Difficult 2-way : new / old =  0.21
076f82
    Difficult testing first 2 : new / old =  1.04
076f82
    Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
076f82
    
076f82
    (cherry picked from commit 5082a287d5e9a1f9cb98b7c982a708a3684f1d5c)
076f82
    
076f82
    x86: Remove __mmask intrinsics in strstr-avx512.c
076f82
    
076f82
    The intrinsics are not available before GCC7 and using standard
076f82
    operators generates code of equivalent or better quality.
076f82
    
076f82
    Removed:
076f82
        _cvtmask64_u64
076f82
        _kshiftri_mask64
076f82
        _kand_mask64
076f82
    
076f82
    Geometric Mean of 5 Runs of Full Benchmark Suite New / Old: 0.958
076f82
    
076f82
    (cherry picked from commit f2698954ff9c2f9626d4bcb5a30eb5729714e0b0)
076f82
076f82
diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile
076f82
index 4d4ad2a3686b5bc3..0e39e63ef6be6a86 100644
076f82
--- a/sysdeps/x86_64/multiarch/Makefile
076f82
+++ b/sysdeps/x86_64/multiarch/Makefile
076f82
@@ -126,6 +126,7 @@ sysdep_routines += \
076f82
   strrchr-sse2 \
076f82
   strspn-c \
076f82
   strspn-sse2 \
076f82
+  strstr-avx512 \
076f82
   strstr-sse2-unaligned \
076f82
   varshift \
076f82
 # sysdep_routines
076f82
@@ -133,6 +134,7 @@ CFLAGS-varshift.c += -msse4
076f82
 CFLAGS-strcspn-c.c += -msse4
076f82
 CFLAGS-strpbrk-c.c += -msse4
076f82
 CFLAGS-strspn-c.c += -msse4
076f82
+CFLAGS-strstr-avx512.c += -mavx512f -mavx512vl -mavx512dq -mavx512bw -mbmi -mbmi2 -O3
076f82
 endif
076f82
 
076f82
 ifeq ($(subdir),wcsmbs)
076f82
diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
076f82
index 6b75a7106e174bce..043821278fdb6d8f 100644
076f82
--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c
076f82
+++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
076f82
@@ -633,6 +633,12 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
076f82
 
076f82
   /* Support sysdeps/x86_64/multiarch/strstr.c.  */
076f82
   IFUNC_IMPL (i, name, strstr,
076f82
+              IFUNC_IMPL_ADD (array, i, strstr,
076f82
+                              (CPU_FEATURE_USABLE (AVX512VL)
076f82
+                               && CPU_FEATURE_USABLE (AVX512BW)
076f82
+                               && CPU_FEATURE_USABLE (AVX512DQ)
076f82
+                               && CPU_FEATURE_USABLE (BMI2)),
076f82
+                              __strstr_avx512)
076f82
 	      IFUNC_IMPL_ADD (array, i, strstr, 1, __strstr_sse2_unaligned)
076f82
 	      IFUNC_IMPL_ADD (array, i, strstr, 1, __strstr_sse2))
076f82
 
076f82
diff --git a/sysdeps/x86_64/multiarch/strstr-avx512.c b/sysdeps/x86_64/multiarch/strstr-avx512.c
076f82
new file mode 100644
076f82
index 0000000000000000..e44c1a05dc0007e5
076f82
--- /dev/null
076f82
+++ b/sysdeps/x86_64/multiarch/strstr-avx512.c
076f82
@@ -0,0 +1,218 @@
076f82
+/* strstr optimized with 512-bit AVX-512 instructions
076f82
+   Copyright (C) 2022 Free Software Foundation, Inc.
076f82
+   This file is part of the GNU C Library.
076f82
+
076f82
+   The GNU C Library is free software; you can redistribute it and/or
076f82
+   modify it under the terms of the GNU Lesser General Public
076f82
+   License as published by the Free Software Foundation; either
076f82
+   version 2.1 of the License, or (at your option) any later version.
076f82
+
076f82
+   The GNU C Library is distributed in the hope that it will be useful,
076f82
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
076f82
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
076f82
+   Lesser General Public License for more details.
076f82
+
076f82
+   You should have received a copy of the GNU Lesser General Public
076f82
+   License along with the GNU C Library; if not, see
076f82
+   <https://www.gnu.org/licenses/>.  */
076f82
+
076f82
+#include <immintrin.h>
076f82
+#include <inttypes.h>
076f82
+#include <stdbool.h>
076f82
+#include <string.h>
076f82
+
076f82
+#define FULL_MMASK64 0xffffffffffffffff
076f82
+#define ONE_64BIT 0x1ull
076f82
+#define ZMM_SIZE_IN_BYTES 64
076f82
+#define PAGESIZE 4096
076f82
+
076f82
+#define cvtmask64_u64(...) (uint64_t) (__VA_ARGS__)
076f82
+#define kshiftri_mask64(x, y) ((x) >> (y))
076f82
+#define kand_mask64(x, y) ((x) & (y))
076f82
+
076f82
+/*
076f82
+ Returns the index of the first edge within the needle, returns 0 if no edge
076f82
+ is found. Example: 'ab' is the first edge in 'aaaaaaaaaabaarddg'
076f82
+ */
076f82
+static inline size_t
076f82
+find_edge_in_needle (const char *ned)
076f82
+{
076f82
+  size_t ind = 0;
076f82
+  while (ned[ind + 1] != '\0')
076f82
+    {
076f82
+      if (ned[ind] != ned[ind + 1])
076f82
+        return ind;
076f82
+      else
076f82
+        ind = ind + 1;
076f82
+    }
076f82
+  return 0;
076f82
+}
076f82
+
076f82
+/*
076f82
+ Compare needle with haystack byte by byte at specified location
076f82
+ */
076f82
+static inline bool
076f82
+verify_string_match (const char *hay, const size_t hay_index, const char *ned,
076f82
+                     size_t ind)
076f82
+{
076f82
+  while (ned[ind] != '\0')
076f82
+    {
076f82
+      if (ned[ind] != hay[hay_index + ind])
076f82
+        return false;
076f82
+      ind = ind + 1;
076f82
+    }
076f82
+  return true;
076f82
+}
076f82
+
076f82
+/*
076f82
+ Compare needle with haystack at specified location. The first 64 bytes are
076f82
+ compared using a ZMM register.
076f82
+ */
076f82
+static inline bool
076f82
+verify_string_match_avx512 (const char *hay, const size_t hay_index,
076f82
+                            const char *ned, const __mmask64 ned_mask,
076f82
+                            const __m512i ned_zmm)
076f82
+{
076f82
+  /* check first 64 bytes using zmm and then scalar */
076f82
+  __m512i hay_zmm = _mm512_loadu_si512 (hay + hay_index); // safe to do so
076f82
+  __mmask64 match = _mm512_mask_cmpneq_epi8_mask (ned_mask, hay_zmm, ned_zmm);
076f82
+  if (match != 0x0) // failed the first few chars
076f82
+    return false;
076f82
+  else if (ned_mask == FULL_MMASK64)
076f82
+    return verify_string_match (hay, hay_index, ned, ZMM_SIZE_IN_BYTES);
076f82
+  return true;
076f82
+}
076f82
+
076f82
+char *
076f82
+__strstr_avx512 (const char *haystack, const char *ned)
076f82
+{
076f82
+  char first = ned[0];
076f82
+  if (first == '\0')
076f82
+    return (char *)haystack;
076f82
+  if (ned[1] == '\0')
076f82
+    return (char *)strchr (haystack, ned[0]);
076f82
+
076f82
+  size_t edge = find_edge_in_needle (ned);
076f82
+
076f82
+  /* ensure haystack is as long as the pos of edge in needle */
076f82
+  for (int ii = 0; ii < edge; ++ii)
076f82
+    {
076f82
+      if (haystack[ii] == '\0')
076f82
+        return NULL;
076f82
+    }
076f82
+
076f82
+  /*
076f82
+   Load 64 bytes of the needle and save it to a zmm register
076f82
+   Read one cache line at a time to avoid loading across a page boundary
076f82
+   */
076f82
+  __mmask64 ned_load_mask = _bzhi_u64 (
076f82
+      FULL_MMASK64, 64 - ((uintptr_t) (ned) & 63));
076f82
+  __m512i ned_zmm = _mm512_maskz_loadu_epi8 (ned_load_mask, ned);
076f82
+  __mmask64 ned_nullmask
076f82
+      = _mm512_mask_testn_epi8_mask (ned_load_mask, ned_zmm, ned_zmm);
076f82
+
076f82
+  if (__glibc_unlikely (ned_nullmask == 0x0))
076f82
+    {
076f82
+      ned_zmm = _mm512_loadu_si512 (ned);
076f82
+      ned_nullmask = _mm512_testn_epi8_mask (ned_zmm, ned_zmm);
076f82
+      ned_load_mask = ned_nullmask ^ (ned_nullmask - ONE_64BIT);
076f82
+      if (ned_nullmask != 0x0)
076f82
+        ned_load_mask = ned_load_mask >> 1;
076f82
+    }
076f82
+  else
076f82
+    {
076f82
+      ned_load_mask = ned_nullmask ^ (ned_nullmask - ONE_64BIT);
076f82
+      ned_load_mask = ned_load_mask >> 1;
076f82
+    }
076f82
+  const __m512i ned0 = _mm512_set1_epi8 (ned[edge]);
076f82
+  const __m512i ned1 = _mm512_set1_epi8 (ned[edge + 1]);
076f82
+
076f82
+  /*
076f82
+   Read the bytes of haystack in the current cache line
076f82
+   */
076f82
+  size_t hay_index = edge;
076f82
+  __mmask64 loadmask = _bzhi_u64 (
076f82
+      FULL_MMASK64, 64 - ((uintptr_t) (haystack + hay_index) & 63));
076f82
+  /* First load is a partial cache line */
076f82
+  __m512i hay0 = _mm512_maskz_loadu_epi8 (loadmask, haystack + hay_index);
076f82
+  /* Search for NULL and compare only till null char */
076f82
+  uint64_t nullmask
076f82
+      = cvtmask64_u64 (_mm512_mask_testn_epi8_mask (loadmask, hay0, hay0));
076f82
+  uint64_t cmpmask = nullmask ^ (nullmask - ONE_64BIT);
076f82
+  cmpmask = cmpmask & cvtmask64_u64 (loadmask);
076f82
+  /* Search for the 2 charaters of needle */
076f82
+  __mmask64 k0 = _mm512_cmpeq_epi8_mask (hay0, ned0);
076f82
+  __mmask64 k1 = _mm512_cmpeq_epi8_mask (hay0, ned1);
076f82
+  k1 = kshiftri_mask64 (k1, 1);
076f82
+  /* k2 masks tell us if both chars from needle match */
076f82
+  uint64_t k2 = cvtmask64_u64 (kand_mask64 (k0, k1)) & cmpmask;
076f82
+  /* For every match, search for the entire needle for a full match */
076f82
+  while (k2)
076f82
+    {
076f82
+      uint64_t bitcount = _tzcnt_u64 (k2);
076f82
+      k2 = _blsr_u64 (k2);
076f82
+      size_t match_pos = hay_index + bitcount - edge;
076f82
+      if (((uintptr_t) (haystack + match_pos) & (PAGESIZE - 1))
076f82
+          < PAGESIZE - 1 - ZMM_SIZE_IN_BYTES)
076f82
+        {
076f82
+          /*
076f82
+           * Use vector compare as long as you are not crossing a page
076f82
+           */
076f82
+          if (verify_string_match_avx512 (haystack, match_pos, ned,
076f82
+                                          ned_load_mask, ned_zmm))
076f82
+            return (char *)haystack + match_pos;
076f82
+        }
076f82
+      else
076f82
+        {
076f82
+          if (verify_string_match (haystack, match_pos, ned, 0))
076f82
+            return (char *)haystack + match_pos;
076f82
+        }
076f82
+    }
076f82
+  /* We haven't checked for potential match at the last char yet */
076f82
+  haystack = (const char *)(((uintptr_t) (haystack + hay_index) | 63));
076f82
+  hay_index = 0;
076f82
+
076f82
+  /*
076f82
+   Loop over one cache line at a time to prevent reading over page
076f82
+   boundary
076f82
+   */
076f82
+  __m512i hay1;
076f82
+  while (nullmask == 0)
076f82
+    {
076f82
+      hay0 = _mm512_loadu_si512 (haystack + hay_index);
076f82
+      hay1 = _mm512_load_si512 (haystack + hay_index
076f82
+                                + 1); // Always 64 byte aligned
076f82
+      nullmask = cvtmask64_u64 (_mm512_testn_epi8_mask (hay1, hay1));
076f82
+      /* Compare only till null char */
076f82
+      cmpmask = nullmask ^ (nullmask - ONE_64BIT);
076f82
+      k0 = _mm512_cmpeq_epi8_mask (hay0, ned0);
076f82
+      k1 = _mm512_cmpeq_epi8_mask (hay1, ned1);
076f82
+      /* k2 masks tell us if both chars from needle match */
076f82
+      k2 = cvtmask64_u64 (kand_mask64 (k0, k1)) & cmpmask;
076f82
+      /* For every match, compare full strings for potential match */
076f82
+      while (k2)
076f82
+        {
076f82
+          uint64_t bitcount = _tzcnt_u64 (k2);
076f82
+          k2 = _blsr_u64 (k2);
076f82
+          size_t match_pos = hay_index + bitcount - edge;
076f82
+          if (((uintptr_t) (haystack + match_pos) & (PAGESIZE - 1))
076f82
+              < PAGESIZE - 1 - ZMM_SIZE_IN_BYTES)
076f82
+            {
076f82
+              /*
076f82
+               * Use vector compare as long as you are not crossing a page
076f82
+               */
076f82
+              if (verify_string_match_avx512 (haystack, match_pos, ned,
076f82
+                                              ned_load_mask, ned_zmm))
076f82
+                return (char *)haystack + match_pos;
076f82
+            }
076f82
+          else
076f82
+            {
076f82
+              /* Compare byte by byte */
076f82
+              if (verify_string_match (haystack, match_pos, ned, 0))
076f82
+                return (char *)haystack + match_pos;
076f82
+            }
076f82
+        }
076f82
+      hay_index += ZMM_SIZE_IN_BYTES;
076f82
+    }
076f82
+  return NULL;
076f82
+}
076f82
diff --git a/sysdeps/x86_64/multiarch/strstr.c b/sysdeps/x86_64/multiarch/strstr.c
076f82
index 848601bde7583ca3..9474d6234e9b62d3 100644
076f82
--- a/sysdeps/x86_64/multiarch/strstr.c
076f82
+++ b/sysdeps/x86_64/multiarch/strstr.c
076f82
@@ -35,16 +35,32 @@
076f82
 
076f82
 extern __typeof (__redirect_strstr) __strstr_sse2_unaligned attribute_hidden;
076f82
 extern __typeof (__redirect_strstr) __strstr_sse2 attribute_hidden;
076f82
+extern __typeof (__redirect_strstr) __strstr_avx512 attribute_hidden;
076f82
 
076f82
 #include "init-arch.h"
076f82
 
076f82
 /* Avoid DWARF definition DIE on ifunc symbol so that GDB can handle
076f82
    ifunc symbol properly.  */
076f82
 extern __typeof (__redirect_strstr) __libc_strstr;
076f82
-libc_ifunc (__libc_strstr,
076f82
-	    HAS_ARCH_FEATURE (Fast_Unaligned_Load)
076f82
-	    ? __strstr_sse2_unaligned
076f82
-	    : __strstr_sse2)
076f82
 
076f82
+static inline void *
076f82
+IFUNC_SELECTOR (void)
076f82
+{
076f82
+  const struct cpu_features *cpu_features = __get_cpu_features ();
076f82
+
076f82
+  if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_AVX512)
076f82
+      && CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)
076f82
+      && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)
076f82
+      && CPU_FEATURE_USABLE_P (cpu_features, AVX512DQ)
076f82
+      && CPU_FEATURE_USABLE_P (cpu_features, BMI2))
076f82
+    return __strstr_avx512;
076f82
+
076f82
+  if (CPU_FEATURES_ARCH_P (cpu_features, Fast_Unaligned_Load))
076f82
+    return __strstr_sse2_unaligned;
076f82
+
076f82
+  return __strstr_sse2;
076f82
+}
076f82
+
076f82
+libc_ifunc_redirected (__redirect_strstr, __libc_strstr, IFUNC_SELECTOR ());
076f82
 #undef strstr
076f82
 strong_alias (__libc_strstr, strstr)