|
 |
08c3a6 |
commit 3051cf3e745015a9106cf71be7f7adbb2f83fcac
|
|
|
08c3a6 |
Author: Noah Goldstein <goldstein.w.n@gmail.com>
|
|
|
08c3a6 |
Date: Thu Mar 24 18:56:12 2022 -0500
|
|
|
08c3a6 |
|
|
|
08c3a6 |
x86: Add AVX2 optimized str{n}casecmp
|
|
|
08c3a6 |
|
|
|
08c3a6 |
geometric_mean(N=40) of all benchmarks AVX2 / SSE42: .702
|
|
|
08c3a6 |
|
|
|
08c3a6 |
All string/memory tests pass.
|
|
|
08c3a6 |
Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
|
|
|
08c3a6 |
|
|
|
08c3a6 |
(cherry picked from commit bbf81222343fed5cd704001a2ae0d86c71544151)
|
|
|
08c3a6 |
|
|
|
08c3a6 |
diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile
|
|
|
08c3a6 |
index 8c9e7812c6af10b8..711ecf2ee45d61b9 100644
|
|
|
08c3a6 |
--- a/sysdeps/x86_64/multiarch/Makefile
|
|
|
08c3a6 |
+++ b/sysdeps/x86_64/multiarch/Makefile
|
|
|
08c3a6 |
@@ -51,6 +51,8 @@ sysdep_routines += \
|
|
|
08c3a6 |
stpncpy-sse2-unaligned \
|
|
|
08c3a6 |
stpncpy-ssse3 \
|
|
|
08c3a6 |
strcasecmp_l-avx \
|
|
|
08c3a6 |
+ strcasecmp_l-avx2 \
|
|
|
08c3a6 |
+ strcasecmp_l-avx2-rtm \
|
|
|
08c3a6 |
strcasecmp_l-sse2 \
|
|
|
08c3a6 |
strcasecmp_l-sse4_2 \
|
|
|
08c3a6 |
strcasecmp_l-ssse3 \
|
|
|
08c3a6 |
@@ -89,6 +91,8 @@ sysdep_routines += \
|
|
|
08c3a6 |
strlen-evex \
|
|
|
08c3a6 |
strlen-sse2 \
|
|
|
08c3a6 |
strncase_l-avx \
|
|
|
08c3a6 |
+ strncase_l-avx2 \
|
|
|
08c3a6 |
+ strncase_l-avx2-rtm \
|
|
|
08c3a6 |
strncase_l-sse2 \
|
|
|
08c3a6 |
strncase_l-sse4_2 \
|
|
|
08c3a6 |
strncase_l-ssse3 \
|
|
|
08c3a6 |
diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
|
|
|
08c3a6 |
index 4992d7bd3206a7c0..a687b387c91aa9ae 100644
|
|
|
08c3a6 |
--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c
|
|
|
08c3a6 |
+++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
|
|
|
08c3a6 |
@@ -418,6 +418,13 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
|
|
|
08c3a6 |
|
|
|
08c3a6 |
/* Support sysdeps/x86_64/multiarch/strcasecmp_l.c. */
|
|
|
08c3a6 |
IFUNC_IMPL (i, name, strcasecmp,
|
|
|
08c3a6 |
+ IFUNC_IMPL_ADD (array, i, strcasecmp,
|
|
|
08c3a6 |
+ CPU_FEATURE_USABLE (AVX2),
|
|
|
08c3a6 |
+ __strcasecmp_avx2)
|
|
|
08c3a6 |
+ IFUNC_IMPL_ADD (array, i, strcasecmp,
|
|
|
08c3a6 |
+ (CPU_FEATURE_USABLE (AVX2)
|
|
|
08c3a6 |
+ && CPU_FEATURE_USABLE (RTM)),
|
|
|
08c3a6 |
+ __strcasecmp_avx2_rtm)
|
|
|
08c3a6 |
IFUNC_IMPL_ADD (array, i, strcasecmp,
|
|
|
08c3a6 |
CPU_FEATURE_USABLE (AVX),
|
|
|
08c3a6 |
__strcasecmp_avx)
|
|
|
08c3a6 |
@@ -431,6 +438,13 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
|
|
|
08c3a6 |
|
|
|
08c3a6 |
/* Support sysdeps/x86_64/multiarch/strcasecmp_l.c. */
|
|
|
08c3a6 |
IFUNC_IMPL (i, name, strcasecmp_l,
|
|
|
08c3a6 |
+ IFUNC_IMPL_ADD (array, i, strcasecmp,
|
|
|
08c3a6 |
+ CPU_FEATURE_USABLE (AVX2),
|
|
|
08c3a6 |
+ __strcasecmp_l_avx2)
|
|
|
08c3a6 |
+ IFUNC_IMPL_ADD (array, i, strcasecmp,
|
|
|
08c3a6 |
+ (CPU_FEATURE_USABLE (AVX2)
|
|
|
08c3a6 |
+ && CPU_FEATURE_USABLE (RTM)),
|
|
|
08c3a6 |
+ __strcasecmp_l_avx2_rtm)
|
|
|
08c3a6 |
IFUNC_IMPL_ADD (array, i, strcasecmp_l,
|
|
|
08c3a6 |
CPU_FEATURE_USABLE (AVX),
|
|
|
08c3a6 |
__strcasecmp_l_avx)
|
|
|
08c3a6 |
@@ -558,6 +572,13 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
|
|
|
08c3a6 |
|
|
|
08c3a6 |
/* Support sysdeps/x86_64/multiarch/strncase_l.c. */
|
|
|
08c3a6 |
IFUNC_IMPL (i, name, strncasecmp,
|
|
|
08c3a6 |
+ IFUNC_IMPL_ADD (array, i, strncasecmp,
|
|
|
08c3a6 |
+ CPU_FEATURE_USABLE (AVX2),
|
|
|
08c3a6 |
+ __strncasecmp_avx2)
|
|
|
08c3a6 |
+ IFUNC_IMPL_ADD (array, i, strncasecmp,
|
|
|
08c3a6 |
+ (CPU_FEATURE_USABLE (AVX2)
|
|
|
08c3a6 |
+ && CPU_FEATURE_USABLE (RTM)),
|
|
|
08c3a6 |
+ __strncasecmp_avx2_rtm)
|
|
|
08c3a6 |
IFUNC_IMPL_ADD (array, i, strncasecmp,
|
|
|
08c3a6 |
CPU_FEATURE_USABLE (AVX),
|
|
|
08c3a6 |
__strncasecmp_avx)
|
|
|
08c3a6 |
@@ -572,6 +593,13 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
|
|
|
08c3a6 |
|
|
|
08c3a6 |
/* Support sysdeps/x86_64/multiarch/strncase_l.c. */
|
|
|
08c3a6 |
IFUNC_IMPL (i, name, strncasecmp_l,
|
|
|
08c3a6 |
+ IFUNC_IMPL_ADD (array, i, strncasecmp,
|
|
|
08c3a6 |
+ CPU_FEATURE_USABLE (AVX2),
|
|
|
08c3a6 |
+ __strncasecmp_l_avx2)
|
|
|
08c3a6 |
+ IFUNC_IMPL_ADD (array, i, strncasecmp,
|
|
|
08c3a6 |
+ (CPU_FEATURE_USABLE (AVX2)
|
|
|
08c3a6 |
+ && CPU_FEATURE_USABLE (RTM)),
|
|
|
08c3a6 |
+ __strncasecmp_l_avx2_rtm)
|
|
|
08c3a6 |
IFUNC_IMPL_ADD (array, i, strncasecmp_l,
|
|
|
08c3a6 |
CPU_FEATURE_USABLE (AVX),
|
|
|
08c3a6 |
__strncasecmp_l_avx)
|
|
|
08c3a6 |
diff --git a/sysdeps/x86_64/multiarch/ifunc-strcasecmp.h b/sysdeps/x86_64/multiarch/ifunc-strcasecmp.h
|
|
|
08c3a6 |
index 931770e079fcc69f..64d0cd6ef25f73c0 100644
|
|
|
08c3a6 |
--- a/sysdeps/x86_64/multiarch/ifunc-strcasecmp.h
|
|
|
08c3a6 |
+++ b/sysdeps/x86_64/multiarch/ifunc-strcasecmp.h
|
|
|
08c3a6 |
@@ -23,12 +23,24 @@ extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden;
|
|
|
08c3a6 |
extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden;
|
|
|
08c3a6 |
extern __typeof (REDIRECT_NAME) OPTIMIZE (sse42) attribute_hidden;
|
|
|
08c3a6 |
extern __typeof (REDIRECT_NAME) OPTIMIZE (avx) attribute_hidden;
|
|
|
08c3a6 |
+extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden;
|
|
|
08c3a6 |
+extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden;
|
|
|
08c3a6 |
|
|
|
08c3a6 |
static inline void *
|
|
|
08c3a6 |
IFUNC_SELECTOR (void)
|
|
|
08c3a6 |
{
|
|
|
08c3a6 |
const struct cpu_features* cpu_features = __get_cpu_features ();
|
|
|
08c3a6 |
|
|
|
08c3a6 |
+ if (CPU_FEATURE_USABLE_P (cpu_features, AVX2)
|
|
|
08c3a6 |
+ && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load))
|
|
|
08c3a6 |
+ {
|
|
|
08c3a6 |
+ if (CPU_FEATURE_USABLE_P (cpu_features, RTM))
|
|
|
08c3a6 |
+ return OPTIMIZE (avx2_rtm);
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+ if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER))
|
|
|
08c3a6 |
+ return OPTIMIZE (avx2);
|
|
|
08c3a6 |
+ }
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
if (CPU_FEATURE_USABLE_P (cpu_features, AVX))
|
|
|
08c3a6 |
return OPTIMIZE (avx);
|
|
|
08c3a6 |
|
|
|
08c3a6 |
diff --git a/sysdeps/x86_64/multiarch/strcasecmp_l-avx2-rtm.S b/sysdeps/x86_64/multiarch/strcasecmp_l-avx2-rtm.S
|
|
|
08c3a6 |
new file mode 100644
|
|
|
08c3a6 |
index 0000000000000000..09957fc3c543b40c
|
|
|
08c3a6 |
--- /dev/null
|
|
|
08c3a6 |
+++ b/sysdeps/x86_64/multiarch/strcasecmp_l-avx2-rtm.S
|
|
|
08c3a6 |
@@ -0,0 +1,15 @@
|
|
|
08c3a6 |
+#ifndef STRCMP
|
|
|
08c3a6 |
+# define STRCMP __strcasecmp_l_avx2_rtm
|
|
|
08c3a6 |
+#endif
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+#define _GLABEL(x) x ## _rtm
|
|
|
08c3a6 |
+#define GLABEL(x) _GLABEL(x)
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+#define ZERO_UPPER_VEC_REGISTERS_RETURN \
|
|
|
08c3a6 |
+ ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+#define VZEROUPPER_RETURN jmp L(return_vzeroupper)
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+#define SECTION(p) p##.avx.rtm
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+#include "strcasecmp_l-avx2.S"
|
|
|
08c3a6 |
diff --git a/sysdeps/x86_64/multiarch/strcasecmp_l-avx2.S b/sysdeps/x86_64/multiarch/strcasecmp_l-avx2.S
|
|
|
08c3a6 |
new file mode 100644
|
|
|
08c3a6 |
index 0000000000000000..e2762f2a222b2a65
|
|
|
08c3a6 |
--- /dev/null
|
|
|
08c3a6 |
+++ b/sysdeps/x86_64/multiarch/strcasecmp_l-avx2.S
|
|
|
08c3a6 |
@@ -0,0 +1,23 @@
|
|
|
08c3a6 |
+/* strcasecmp_l optimized with AVX2.
|
|
|
08c3a6 |
+ Copyright (C) 2017-2022 Free Software Foundation, Inc.
|
|
|
08c3a6 |
+ This file is part of the GNU C Library.
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+ The GNU C Library is free software; you can redistribute it and/or
|
|
|
08c3a6 |
+ modify it under the terms of the GNU Lesser General Public
|
|
|
08c3a6 |
+ License as published by the Free Software Foundation; either
|
|
|
08c3a6 |
+ version 2.1 of the License, or (at your option) any later version.
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+ The GNU C Library is distributed in the hope that it will be useful,
|
|
|
08c3a6 |
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
08c3a6 |
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
08c3a6 |
+ Lesser General Public License for more details.
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+ You should have received a copy of the GNU Lesser General Public
|
|
|
08c3a6 |
+ License along with the GNU C Library; if not, see
|
|
|
08c3a6 |
+ <https://www.gnu.org/licenses/>. */
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+#ifndef STRCMP
|
|
|
08c3a6 |
+# define STRCMP __strcasecmp_l_avx2
|
|
|
08c3a6 |
+#endif
|
|
|
08c3a6 |
+#define USE_AS_STRCASECMP_L
|
|
|
08c3a6 |
+#include "strcmp-avx2.S"
|
|
|
08c3a6 |
diff --git a/sysdeps/x86_64/multiarch/strcmp-avx2.S b/sysdeps/x86_64/multiarch/strcmp-avx2.S
|
|
|
08c3a6 |
index 09a73942086f9c9f..aa91f6e48a0e1ce5 100644
|
|
|
08c3a6 |
--- a/sysdeps/x86_64/multiarch/strcmp-avx2.S
|
|
|
08c3a6 |
+++ b/sysdeps/x86_64/multiarch/strcmp-avx2.S
|
|
|
08c3a6 |
@@ -20,6 +20,10 @@
|
|
|
08c3a6 |
|
|
|
08c3a6 |
# include <sysdep.h>
|
|
|
08c3a6 |
|
|
|
08c3a6 |
+# if defined USE_AS_STRCASECMP_L
|
|
|
08c3a6 |
+# include "locale-defines.h"
|
|
|
08c3a6 |
+# endif
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
# ifndef STRCMP
|
|
|
08c3a6 |
# define STRCMP __strcmp_avx2
|
|
|
08c3a6 |
# endif
|
|
|
08c3a6 |
@@ -74,13 +78,88 @@
|
|
|
08c3a6 |
# define VEC_OFFSET (-VEC_SIZE)
|
|
|
08c3a6 |
# endif
|
|
|
08c3a6 |
|
|
|
08c3a6 |
+# ifdef USE_AS_STRCASECMP_L
|
|
|
08c3a6 |
+# define BYTE_LOOP_REG OFFSET_REG
|
|
|
08c3a6 |
+# else
|
|
|
08c3a6 |
+# define BYTE_LOOP_REG ecx
|
|
|
08c3a6 |
+# endif
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+# ifdef USE_AS_STRCASECMP_L
|
|
|
08c3a6 |
+# ifdef USE_AS_STRNCMP
|
|
|
08c3a6 |
+# define STRCASECMP __strncasecmp_avx2
|
|
|
08c3a6 |
+# define LOCALE_REG rcx
|
|
|
08c3a6 |
+# define LOCALE_REG_LP RCX_LP
|
|
|
08c3a6 |
+# define STRCASECMP_NONASCII __strncasecmp_l_nonascii
|
|
|
08c3a6 |
+# else
|
|
|
08c3a6 |
+# define STRCASECMP __strcasecmp_avx2
|
|
|
08c3a6 |
+# define LOCALE_REG rdx
|
|
|
08c3a6 |
+# define LOCALE_REG_LP RDX_LP
|
|
|
08c3a6 |
+# define STRCASECMP_NONASCII __strcasecmp_l_nonascii
|
|
|
08c3a6 |
+# endif
|
|
|
08c3a6 |
+# endif
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
# define xmmZERO xmm15
|
|
|
08c3a6 |
# define ymmZERO ymm15
|
|
|
08c3a6 |
|
|
|
08c3a6 |
+# define LCASE_MIN_ymm %ymm10
|
|
|
08c3a6 |
+# define LCASE_MAX_ymm %ymm11
|
|
|
08c3a6 |
+# define CASE_ADD_ymm %ymm12
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+# define LCASE_MIN_xmm %xmm10
|
|
|
08c3a6 |
+# define LCASE_MAX_xmm %xmm11
|
|
|
08c3a6 |
+# define CASE_ADD_xmm %xmm12
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+ /* r11 is never use elsewhere so this is safe to maintain. */
|
|
|
08c3a6 |
+# define TOLOWER_BASE %r11
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
# ifndef SECTION
|
|
|
08c3a6 |
# define SECTION(p) p##.avx
|
|
|
08c3a6 |
# endif
|
|
|
08c3a6 |
|
|
|
08c3a6 |
+# ifdef USE_AS_STRCASECMP_L
|
|
|
08c3a6 |
+# define REG(x, y) x ## y
|
|
|
08c3a6 |
+# define TOLOWER(reg1_in, reg1_out, reg2_in, reg2_out, ext) \
|
|
|
08c3a6 |
+ vpaddb REG(LCASE_MIN_, ext), reg1_in, REG(%ext, 8); \
|
|
|
08c3a6 |
+ vpaddb REG(LCASE_MIN_, ext), reg2_in, REG(%ext, 9); \
|
|
|
08c3a6 |
+ vpcmpgtb REG(LCASE_MAX_, ext), REG(%ext, 8), REG(%ext, 8); \
|
|
|
08c3a6 |
+ vpcmpgtb REG(LCASE_MAX_, ext), REG(%ext, 9), REG(%ext, 9); \
|
|
|
08c3a6 |
+ vpandn REG(CASE_ADD_, ext), REG(%ext, 8), REG(%ext, 8); \
|
|
|
08c3a6 |
+ vpandn REG(CASE_ADD_, ext), REG(%ext, 9), REG(%ext, 9); \
|
|
|
08c3a6 |
+ vpaddb REG(%ext, 8), reg1_in, reg1_out; \
|
|
|
08c3a6 |
+ vpaddb REG(%ext, 9), reg2_in, reg2_out
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+# define TOLOWER_gpr(src, dst) movl (TOLOWER_BASE, src, 4), dst
|
|
|
08c3a6 |
+# define TOLOWER_ymm(...) TOLOWER(__VA_ARGS__, ymm)
|
|
|
08c3a6 |
+# define TOLOWER_xmm(...) TOLOWER(__VA_ARGS__, xmm)
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+# define CMP_R1_R2(s1_reg, s2_reg, scratch_reg, reg_out, ext) \
|
|
|
08c3a6 |
+ TOLOWER (s1_reg, scratch_reg, s2_reg, s2_reg, ext); \
|
|
|
08c3a6 |
+ VPCMPEQ scratch_reg, s2_reg, reg_out
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+# define CMP_R1_S2(s1_reg, s2_mem, scratch_reg, reg_out, ext) \
|
|
|
08c3a6 |
+ VMOVU s2_mem, reg_out; \
|
|
|
08c3a6 |
+ CMP_R1_R2(s1_reg, reg_out, scratch_reg, reg_out, ext)
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+# define CMP_R1_R2_ymm(...) CMP_R1_R2(__VA_ARGS__, ymm)
|
|
|
08c3a6 |
+# define CMP_R1_R2_xmm(...) CMP_R1_R2(__VA_ARGS__, xmm)
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+# define CMP_R1_S2_ymm(...) CMP_R1_S2(__VA_ARGS__, ymm)
|
|
|
08c3a6 |
+# define CMP_R1_S2_xmm(...) CMP_R1_S2(__VA_ARGS__, xmm)
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+# else
|
|
|
08c3a6 |
+# define TOLOWER_gpr(...)
|
|
|
08c3a6 |
+# define TOLOWER_ymm(...)
|
|
|
08c3a6 |
+# define TOLOWER_xmm(...)
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+# define CMP_R1_R2_ymm(s1_reg, s2_reg, scratch_reg, reg_out) \
|
|
|
08c3a6 |
+ VPCMPEQ s2_reg, s1_reg, reg_out
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+# define CMP_R1_R2_xmm(...) CMP_R1_R2_ymm(__VA_ARGS__)
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+# define CMP_R1_S2_ymm(...) CMP_R1_R2_ymm(__VA_ARGS__)
|
|
|
08c3a6 |
+# define CMP_R1_S2_xmm(...) CMP_R1_R2_xmm(__VA_ARGS__)
|
|
|
08c3a6 |
+# endif
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
/* Warning!
|
|
|
08c3a6 |
wcscmp/wcsncmp have to use SIGNED comparison for elements.
|
|
|
08c3a6 |
strcmp/strncmp have to use UNSIGNED comparison for elements.
|
|
|
08c3a6 |
@@ -102,8 +181,49 @@
|
|
|
08c3a6 |
returned. */
|
|
|
08c3a6 |
|
|
|
08c3a6 |
.section SECTION(.text), "ax", @progbits
|
|
|
08c3a6 |
-ENTRY(STRCMP)
|
|
|
08c3a6 |
+ .align 16
|
|
|
08c3a6 |
+ .type STRCMP, @function
|
|
|
08c3a6 |
+ .globl STRCMP
|
|
|
08c3a6 |
+ .hidden STRCMP
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+# ifndef GLABEL
|
|
|
08c3a6 |
+# define GLABEL(...) __VA_ARGS__
|
|
|
08c3a6 |
+# endif
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+# ifdef USE_AS_STRCASECMP_L
|
|
|
08c3a6 |
+ENTRY (GLABEL(STRCASECMP))
|
|
|
08c3a6 |
+ movq __libc_tsd_LOCALE@gottpoff(%rip), %rax
|
|
|
08c3a6 |
+ mov %fs:(%rax), %LOCALE_REG_LP
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+ /* Either 1 or 5 bytes (dependeing if CET is enabled). */
|
|
|
08c3a6 |
+ .p2align 4
|
|
|
08c3a6 |
+END (GLABEL(STRCASECMP))
|
|
|
08c3a6 |
+ /* FALLTHROUGH to strcasecmp/strncasecmp_l. */
|
|
|
08c3a6 |
+# endif
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+ .p2align 4
|
|
|
08c3a6 |
+STRCMP:
|
|
|
08c3a6 |
+ cfi_startproc
|
|
|
08c3a6 |
+ _CET_ENDBR
|
|
|
08c3a6 |
+ CALL_MCOUNT
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+# if defined USE_AS_STRCASECMP_L
|
|
|
08c3a6 |
+ /* We have to fall back on the C implementation for locales with
|
|
|
08c3a6 |
+ encodings not matching ASCII for single bytes. */
|
|
|
08c3a6 |
+# if LOCALE_T___LOCALES != 0 || LC_CTYPE != 0
|
|
|
08c3a6 |
+ mov LOCALE_T___LOCALES + LC_CTYPE * LP_SIZE(%LOCALE_REG), %RAX_LP
|
|
|
08c3a6 |
+# else
|
|
|
08c3a6 |
+ mov (%LOCALE_REG), %RAX_LP
|
|
|
08c3a6 |
+# endif
|
|
|
08c3a6 |
+ testl $1, LOCALE_DATA_VALUES + _NL_CTYPE_NONASCII_CASE * SIZEOF_VALUES(%rax)
|
|
|
08c3a6 |
+ jne STRCASECMP_NONASCII
|
|
|
08c3a6 |
+ leaq _nl_C_LC_CTYPE_tolower + 128 * 4(%rip), TOLOWER_BASE
|
|
|
08c3a6 |
+# endif
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
# ifdef USE_AS_STRNCMP
|
|
|
08c3a6 |
+ /* Don't overwrite LOCALE_REG (rcx) until we have pass
|
|
|
08c3a6 |
+ L(one_or_less). Otherwise we might use the wrong locale in
|
|
|
08c3a6 |
+ the OVERFLOW_STRCMP (strcasecmp_l). */
|
|
|
08c3a6 |
# ifdef __ILP32__
|
|
|
08c3a6 |
/* Clear the upper 32 bits. */
|
|
|
08c3a6 |
movl %edx, %edx
|
|
|
08c3a6 |
@@ -128,6 +248,30 @@ ENTRY(STRCMP)
|
|
|
08c3a6 |
# endif
|
|
|
08c3a6 |
# endif
|
|
|
08c3a6 |
vpxor %xmmZERO, %xmmZERO, %xmmZERO
|
|
|
08c3a6 |
+# if defined USE_AS_STRCASECMP_L
|
|
|
08c3a6 |
+ .section .rodata.cst32, "aM", @progbits, 32
|
|
|
08c3a6 |
+ .align 32
|
|
|
08c3a6 |
+L(lcase_min):
|
|
|
08c3a6 |
+ .quad 0x3f3f3f3f3f3f3f3f
|
|
|
08c3a6 |
+ .quad 0x3f3f3f3f3f3f3f3f
|
|
|
08c3a6 |
+ .quad 0x3f3f3f3f3f3f3f3f
|
|
|
08c3a6 |
+ .quad 0x3f3f3f3f3f3f3f3f
|
|
|
08c3a6 |
+L(lcase_max):
|
|
|
08c3a6 |
+ .quad 0x9999999999999999
|
|
|
08c3a6 |
+ .quad 0x9999999999999999
|
|
|
08c3a6 |
+ .quad 0x9999999999999999
|
|
|
08c3a6 |
+ .quad 0x9999999999999999
|
|
|
08c3a6 |
+L(case_add):
|
|
|
08c3a6 |
+ .quad 0x2020202020202020
|
|
|
08c3a6 |
+ .quad 0x2020202020202020
|
|
|
08c3a6 |
+ .quad 0x2020202020202020
|
|
|
08c3a6 |
+ .quad 0x2020202020202020
|
|
|
08c3a6 |
+ .previous
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+ vmovdqa L(lcase_min)(%rip), LCASE_MIN_ymm
|
|
|
08c3a6 |
+ vmovdqa L(lcase_max)(%rip), LCASE_MAX_ymm
|
|
|
08c3a6 |
+ vmovdqa L(case_add)(%rip), CASE_ADD_ymm
|
|
|
08c3a6 |
+# endif
|
|
|
08c3a6 |
movl %edi, %eax
|
|
|
08c3a6 |
orl %esi, %eax
|
|
|
08c3a6 |
sall $20, %eax
|
|
|
08c3a6 |
@@ -138,8 +282,10 @@ ENTRY(STRCMP)
|
|
|
08c3a6 |
L(no_page_cross):
|
|
|
08c3a6 |
/* Safe to compare 4x vectors. */
|
|
|
08c3a6 |
VMOVU (%rdi), %ymm0
|
|
|
08c3a6 |
- /* 1s where s1 and s2 equal. */
|
|
|
08c3a6 |
- VPCMPEQ (%rsi), %ymm0, %ymm1
|
|
|
08c3a6 |
+ /* 1s where s1 and s2 equal. Just VPCMPEQ if its not strcasecmp.
|
|
|
08c3a6 |
+ Otherwise converts ymm0 and load from rsi to lower. ymm2 is
|
|
|
08c3a6 |
+ scratch and ymm1 is the return. */
|
|
|
08c3a6 |
+ CMP_R1_S2_ymm (%ymm0, (%rsi), %ymm2, %ymm1)
|
|
|
08c3a6 |
/* 1s at null CHAR. */
|
|
|
08c3a6 |
VPCMPEQ %ymm0, %ymmZERO, %ymm2
|
|
|
08c3a6 |
/* 1s where s1 and s2 equal AND not null CHAR. */
|
|
|
08c3a6 |
@@ -172,6 +318,8 @@ L(return_vec_0):
|
|
|
08c3a6 |
# else
|
|
|
08c3a6 |
movzbl (%rdi, %rcx), %eax
|
|
|
08c3a6 |
movzbl (%rsi, %rcx), %ecx
|
|
|
08c3a6 |
+ TOLOWER_gpr (%rax, %eax)
|
|
|
08c3a6 |
+ TOLOWER_gpr (%rcx, %ecx)
|
|
|
08c3a6 |
subl %ecx, %eax
|
|
|
08c3a6 |
# endif
|
|
|
08c3a6 |
L(ret0):
|
|
|
08c3a6 |
@@ -192,6 +340,10 @@ L(ret_zero):
|
|
|
08c3a6 |
|
|
|
08c3a6 |
.p2align 4,, 5
|
|
|
08c3a6 |
L(one_or_less):
|
|
|
08c3a6 |
+# ifdef USE_AS_STRCASECMP_L
|
|
|
08c3a6 |
+ /* Set locale argument for strcasecmp. */
|
|
|
08c3a6 |
+ movq %LOCALE_REG, %rdx
|
|
|
08c3a6 |
+# endif
|
|
|
08c3a6 |
jb L(ret_zero)
|
|
|
08c3a6 |
# ifdef USE_AS_WCSCMP
|
|
|
08c3a6 |
/* 'nbe' covers the case where length is negative (large
|
|
|
08c3a6 |
@@ -211,6 +363,8 @@ L(one_or_less):
|
|
|
08c3a6 |
jnbe __strcmp_avx2
|
|
|
08c3a6 |
movzbl (%rdi), %eax
|
|
|
08c3a6 |
movzbl (%rsi), %ecx
|
|
|
08c3a6 |
+ TOLOWER_gpr (%rax, %eax)
|
|
|
08c3a6 |
+ TOLOWER_gpr (%rcx, %ecx)
|
|
|
08c3a6 |
subl %ecx, %eax
|
|
|
08c3a6 |
# endif
|
|
|
08c3a6 |
L(ret1):
|
|
|
08c3a6 |
@@ -238,6 +392,8 @@ L(return_vec_1):
|
|
|
08c3a6 |
# else
|
|
|
08c3a6 |
movzbl VEC_SIZE(%rdi, %rcx), %eax
|
|
|
08c3a6 |
movzbl VEC_SIZE(%rsi, %rcx), %ecx
|
|
|
08c3a6 |
+ TOLOWER_gpr (%rax, %eax)
|
|
|
08c3a6 |
+ TOLOWER_gpr (%rcx, %ecx)
|
|
|
08c3a6 |
subl %ecx, %eax
|
|
|
08c3a6 |
# endif
|
|
|
08c3a6 |
L(ret2):
|
|
|
08c3a6 |
@@ -269,6 +425,8 @@ L(return_vec_2):
|
|
|
08c3a6 |
# else
|
|
|
08c3a6 |
movzbl (VEC_SIZE * 2)(%rdi, %rcx), %eax
|
|
|
08c3a6 |
movzbl (VEC_SIZE * 2)(%rsi, %rcx), %ecx
|
|
|
08c3a6 |
+ TOLOWER_gpr (%rax, %eax)
|
|
|
08c3a6 |
+ TOLOWER_gpr (%rcx, %ecx)
|
|
|
08c3a6 |
subl %ecx, %eax
|
|
|
08c3a6 |
# endif
|
|
|
08c3a6 |
L(ret3):
|
|
|
08c3a6 |
@@ -289,6 +447,8 @@ L(return_vec_3):
|
|
|
08c3a6 |
# else
|
|
|
08c3a6 |
movzbl (VEC_SIZE * 3)(%rdi, %rcx), %eax
|
|
|
08c3a6 |
movzbl (VEC_SIZE * 3)(%rsi, %rcx), %ecx
|
|
|
08c3a6 |
+ TOLOWER_gpr (%rax, %eax)
|
|
|
08c3a6 |
+ TOLOWER_gpr (%rcx, %ecx)
|
|
|
08c3a6 |
subl %ecx, %eax
|
|
|
08c3a6 |
# endif
|
|
|
08c3a6 |
L(ret4):
|
|
|
08c3a6 |
@@ -299,7 +459,7 @@ L(ret4):
|
|
|
08c3a6 |
L(more_3x_vec):
|
|
|
08c3a6 |
/* Safe to compare 4x vectors. */
|
|
|
08c3a6 |
VMOVU VEC_SIZE(%rdi), %ymm0
|
|
|
08c3a6 |
- VPCMPEQ VEC_SIZE(%rsi), %ymm0, %ymm1
|
|
|
08c3a6 |
+ CMP_R1_S2_ymm (%ymm0, VEC_SIZE(%rsi), %ymm2, %ymm1)
|
|
|
08c3a6 |
VPCMPEQ %ymm0, %ymmZERO, %ymm2
|
|
|
08c3a6 |
vpandn %ymm1, %ymm2, %ymm1
|
|
|
08c3a6 |
vpmovmskb %ymm1, %ecx
|
|
|
08c3a6 |
@@ -312,7 +472,7 @@ L(more_3x_vec):
|
|
|
08c3a6 |
# endif
|
|
|
08c3a6 |
|
|
|
08c3a6 |
VMOVU (VEC_SIZE * 2)(%rdi), %ymm0
|
|
|
08c3a6 |
- VPCMPEQ (VEC_SIZE * 2)(%rsi), %ymm0, %ymm1
|
|
|
08c3a6 |
+ CMP_R1_S2_ymm (%ymm0, (VEC_SIZE * 2)(%rsi), %ymm2, %ymm1)
|
|
|
08c3a6 |
VPCMPEQ %ymm0, %ymmZERO, %ymm2
|
|
|
08c3a6 |
vpandn %ymm1, %ymm2, %ymm1
|
|
|
08c3a6 |
vpmovmskb %ymm1, %ecx
|
|
|
08c3a6 |
@@ -320,7 +480,7 @@ L(more_3x_vec):
|
|
|
08c3a6 |
jnz L(return_vec_2)
|
|
|
08c3a6 |
|
|
|
08c3a6 |
VMOVU (VEC_SIZE * 3)(%rdi), %ymm0
|
|
|
08c3a6 |
- VPCMPEQ (VEC_SIZE * 3)(%rsi), %ymm0, %ymm1
|
|
|
08c3a6 |
+ CMP_R1_S2_ymm (%ymm0, (VEC_SIZE * 3)(%rsi), %ymm2, %ymm1)
|
|
|
08c3a6 |
VPCMPEQ %ymm0, %ymmZERO, %ymm2
|
|
|
08c3a6 |
vpandn %ymm1, %ymm2, %ymm1
|
|
|
08c3a6 |
vpmovmskb %ymm1, %ecx
|
|
|
08c3a6 |
@@ -395,12 +555,10 @@ L(loop_skip_page_cross_check):
|
|
|
08c3a6 |
VMOVA (VEC_SIZE * 3)(%rdi), %ymm6
|
|
|
08c3a6 |
|
|
|
08c3a6 |
/* ymm1 all 1s where s1 and s2 equal. All 0s otherwise. */
|
|
|
08c3a6 |
- VPCMPEQ (VEC_SIZE * 0)(%rsi), %ymm0, %ymm1
|
|
|
08c3a6 |
-
|
|
|
08c3a6 |
- VPCMPEQ (VEC_SIZE * 1)(%rsi), %ymm2, %ymm3
|
|
|
08c3a6 |
- VPCMPEQ (VEC_SIZE * 2)(%rsi), %ymm4, %ymm5
|
|
|
08c3a6 |
- VPCMPEQ (VEC_SIZE * 3)(%rsi), %ymm6, %ymm7
|
|
|
08c3a6 |
-
|
|
|
08c3a6 |
+ CMP_R1_S2_ymm (%ymm0, (VEC_SIZE * 0)(%rsi), %ymm3, %ymm1)
|
|
|
08c3a6 |
+ CMP_R1_S2_ymm (%ymm2, (VEC_SIZE * 1)(%rsi), %ymm5, %ymm3)
|
|
|
08c3a6 |
+ CMP_R1_S2_ymm (%ymm4, (VEC_SIZE * 2)(%rsi), %ymm7, %ymm5)
|
|
|
08c3a6 |
+ CMP_R1_S2_ymm (%ymm6, (VEC_SIZE * 3)(%rsi), %ymm13, %ymm7)
|
|
|
08c3a6 |
|
|
|
08c3a6 |
/* If any mismatches or null CHAR then 0 CHAR, otherwise non-
|
|
|
08c3a6 |
zero. */
|
|
|
08c3a6 |
@@ -469,6 +627,8 @@ L(return_vec_2_3_end):
|
|
|
08c3a6 |
# else
|
|
|
08c3a6 |
movzbl (VEC_SIZE * 2 - VEC_OFFSET)(%rdi, %LOOP_REG64), %eax
|
|
|
08c3a6 |
movzbl (VEC_SIZE * 2 - VEC_OFFSET)(%rsi, %LOOP_REG64), %ecx
|
|
|
08c3a6 |
+ TOLOWER_gpr (%rax, %eax)
|
|
|
08c3a6 |
+ TOLOWER_gpr (%rcx, %ecx)
|
|
|
08c3a6 |
subl %ecx, %eax
|
|
|
08c3a6 |
xorl %r8d, %eax
|
|
|
08c3a6 |
subl %r8d, %eax
|
|
|
08c3a6 |
@@ -512,6 +672,8 @@ L(return_vec_0_end):
|
|
|
08c3a6 |
# else
|
|
|
08c3a6 |
movzbl (%rdi, %rcx), %eax
|
|
|
08c3a6 |
movzbl (%rsi, %rcx), %ecx
|
|
|
08c3a6 |
+ TOLOWER_gpr (%rax, %eax)
|
|
|
08c3a6 |
+ TOLOWER_gpr (%rcx, %ecx)
|
|
|
08c3a6 |
subl %ecx, %eax
|
|
|
08c3a6 |
xorl %r8d, %eax
|
|
|
08c3a6 |
subl %r8d, %eax
|
|
|
08c3a6 |
@@ -534,6 +696,8 @@ L(return_vec_1_end):
|
|
|
08c3a6 |
# else
|
|
|
08c3a6 |
movzbl VEC_SIZE(%rdi, %rcx), %eax
|
|
|
08c3a6 |
movzbl VEC_SIZE(%rsi, %rcx), %ecx
|
|
|
08c3a6 |
+ TOLOWER_gpr (%rax, %eax)
|
|
|
08c3a6 |
+ TOLOWER_gpr (%rcx, %ecx)
|
|
|
08c3a6 |
subl %ecx, %eax
|
|
|
08c3a6 |
xorl %r8d, %eax
|
|
|
08c3a6 |
subl %r8d, %eax
|
|
|
08c3a6 |
@@ -560,6 +724,8 @@ L(return_vec_2_end):
|
|
|
08c3a6 |
# else
|
|
|
08c3a6 |
movzbl (VEC_SIZE * 2)(%rdi, %rcx), %eax
|
|
|
08c3a6 |
movzbl (VEC_SIZE * 2)(%rsi, %rcx), %ecx
|
|
|
08c3a6 |
+ TOLOWER_gpr (%rax, %eax)
|
|
|
08c3a6 |
+ TOLOWER_gpr (%rcx, %ecx)
|
|
|
08c3a6 |
subl %ecx, %eax
|
|
|
08c3a6 |
xorl %r8d, %eax
|
|
|
08c3a6 |
subl %r8d, %eax
|
|
|
08c3a6 |
@@ -587,7 +753,7 @@ L(page_cross_during_loop):
|
|
|
08c3a6 |
jle L(less_1x_vec_till_page_cross)
|
|
|
08c3a6 |
|
|
|
08c3a6 |
VMOVA (%rdi), %ymm0
|
|
|
08c3a6 |
- VPCMPEQ (%rsi), %ymm0, %ymm1
|
|
|
08c3a6 |
+ CMP_R1_S2_ymm (%ymm0, (%rsi), %ymm2, %ymm1)
|
|
|
08c3a6 |
VPCMPEQ %ymm0, %ymmZERO, %ymm2
|
|
|
08c3a6 |
vpandn %ymm1, %ymm2, %ymm1
|
|
|
08c3a6 |
vpmovmskb %ymm1, %ecx
|
|
|
08c3a6 |
@@ -609,7 +775,7 @@ L(less_1x_vec_till_page_cross):
|
|
|
08c3a6 |
here, it means the previous page (rdi - VEC_SIZE) has already
|
|
|
08c3a6 |
been loaded earlier so must be valid. */
|
|
|
08c3a6 |
VMOVU -VEC_SIZE(%rdi, %rax), %ymm0
|
|
|
08c3a6 |
- VPCMPEQ -VEC_SIZE(%rsi, %rax), %ymm0, %ymm1
|
|
|
08c3a6 |
+ CMP_R1_S2_ymm (%ymm0, -VEC_SIZE(%rsi, %rax), %ymm2, %ymm1)
|
|
|
08c3a6 |
VPCMPEQ %ymm0, %ymmZERO, %ymm2
|
|
|
08c3a6 |
vpandn %ymm1, %ymm2, %ymm1
|
|
|
08c3a6 |
vpmovmskb %ymm1, %ecx
|
|
|
08c3a6 |
@@ -651,6 +817,8 @@ L(return_page_cross_cmp_mem):
|
|
|
08c3a6 |
# else
|
|
|
08c3a6 |
movzbl VEC_OFFSET(%rdi, %rcx), %eax
|
|
|
08c3a6 |
movzbl VEC_OFFSET(%rsi, %rcx), %ecx
|
|
|
08c3a6 |
+ TOLOWER_gpr (%rax, %eax)
|
|
|
08c3a6 |
+ TOLOWER_gpr (%rcx, %ecx)
|
|
|
08c3a6 |
subl %ecx, %eax
|
|
|
08c3a6 |
xorl %r8d, %eax
|
|
|
08c3a6 |
subl %r8d, %eax
|
|
|
08c3a6 |
@@ -677,7 +845,7 @@ L(more_2x_vec_till_page_cross):
|
|
|
08c3a6 |
iteration here. */
|
|
|
08c3a6 |
|
|
|
08c3a6 |
VMOVU VEC_SIZE(%rdi), %ymm0
|
|
|
08c3a6 |
- VPCMPEQ VEC_SIZE(%rsi), %ymm0, %ymm1
|
|
|
08c3a6 |
+ CMP_R1_S2_ymm (%ymm0, VEC_SIZE(%rsi), %ymm2, %ymm1)
|
|
|
08c3a6 |
VPCMPEQ %ymm0, %ymmZERO, %ymm2
|
|
|
08c3a6 |
vpandn %ymm1, %ymm2, %ymm1
|
|
|
08c3a6 |
vpmovmskb %ymm1, %ecx
|
|
|
08c3a6 |
@@ -693,7 +861,7 @@ L(more_2x_vec_till_page_cross):
|
|
|
08c3a6 |
|
|
|
08c3a6 |
/* Safe to include comparisons from lower bytes. */
|
|
|
08c3a6 |
VMOVU -(VEC_SIZE * 2)(%rdi, %rax), %ymm0
|
|
|
08c3a6 |
- VPCMPEQ -(VEC_SIZE * 2)(%rsi, %rax), %ymm0, %ymm1
|
|
|
08c3a6 |
+ CMP_R1_S2_ymm (%ymm0, -(VEC_SIZE * 2)(%rsi, %rax), %ymm2, %ymm1)
|
|
|
08c3a6 |
VPCMPEQ %ymm0, %ymmZERO, %ymm2
|
|
|
08c3a6 |
vpandn %ymm1, %ymm2, %ymm1
|
|
|
08c3a6 |
vpmovmskb %ymm1, %ecx
|
|
|
08c3a6 |
@@ -701,7 +869,7 @@ L(more_2x_vec_till_page_cross):
|
|
|
08c3a6 |
jnz L(return_vec_page_cross_0)
|
|
|
08c3a6 |
|
|
|
08c3a6 |
VMOVU -(VEC_SIZE * 1)(%rdi, %rax), %ymm0
|
|
|
08c3a6 |
- VPCMPEQ -(VEC_SIZE * 1)(%rsi, %rax), %ymm0, %ymm1
|
|
|
08c3a6 |
+ CMP_R1_S2_ymm (%ymm0, -(VEC_SIZE * 1)(%rsi, %rax), %ymm2, %ymm1)
|
|
|
08c3a6 |
VPCMPEQ %ymm0, %ymmZERO, %ymm2
|
|
|
08c3a6 |
vpandn %ymm1, %ymm2, %ymm1
|
|
|
08c3a6 |
vpmovmskb %ymm1, %ecx
|
|
|
08c3a6 |
@@ -719,8 +887,8 @@ L(more_2x_vec_till_page_cross):
|
|
|
08c3a6 |
VMOVA (VEC_SIZE * 2)(%rdi), %ymm4
|
|
|
08c3a6 |
VMOVA (VEC_SIZE * 3)(%rdi), %ymm6
|
|
|
08c3a6 |
|
|
|
08c3a6 |
- VPCMPEQ (VEC_SIZE * 2)(%rsi), %ymm4, %ymm5
|
|
|
08c3a6 |
- VPCMPEQ (VEC_SIZE * 3)(%rsi), %ymm6, %ymm7
|
|
|
08c3a6 |
+ CMP_R1_S2_ymm (%ymm4, (VEC_SIZE * 2)(%rsi), %ymm7, %ymm5)
|
|
|
08c3a6 |
+ CMP_R1_S2_ymm (%ymm6, (VEC_SIZE * 3)(%rsi), %ymm13, %ymm7)
|
|
|
08c3a6 |
vpand %ymm4, %ymm5, %ymm5
|
|
|
08c3a6 |
vpand %ymm6, %ymm7, %ymm7
|
|
|
08c3a6 |
VPMINU %ymm5, %ymm7, %ymm7
|
|
|
08c3a6 |
@@ -771,6 +939,8 @@ L(return_vec_page_cross_1):
|
|
|
08c3a6 |
# else
|
|
|
08c3a6 |
movzbl VEC_OFFSET(%rdi, %rcx), %eax
|
|
|
08c3a6 |
movzbl VEC_OFFSET(%rsi, %rcx), %ecx
|
|
|
08c3a6 |
+ TOLOWER_gpr (%rax, %eax)
|
|
|
08c3a6 |
+ TOLOWER_gpr (%rcx, %ecx)
|
|
|
08c3a6 |
subl %ecx, %eax
|
|
|
08c3a6 |
xorl %r8d, %eax
|
|
|
08c3a6 |
subl %r8d, %eax
|
|
|
08c3a6 |
@@ -826,7 +996,7 @@ L(page_cross):
|
|
|
08c3a6 |
L(page_cross_loop):
|
|
|
08c3a6 |
|
|
|
08c3a6 |
VMOVU (%rdi, %OFFSET_REG64), %ymm0
|
|
|
08c3a6 |
- VPCMPEQ (%rsi, %OFFSET_REG64), %ymm0, %ymm1
|
|
|
08c3a6 |
+ CMP_R1_S2_ymm (%ymm0, (%rsi, %OFFSET_REG64), %ymm2, %ymm1)
|
|
|
08c3a6 |
VPCMPEQ %ymm0, %ymmZERO, %ymm2
|
|
|
08c3a6 |
vpandn %ymm1, %ymm2, %ymm1
|
|
|
08c3a6 |
vpmovmskb %ymm1, %ecx
|
|
|
08c3a6 |
@@ -844,11 +1014,11 @@ L(page_cross_loop):
|
|
|
08c3a6 |
subl %eax, %OFFSET_REG
|
|
|
08c3a6 |
/* OFFSET_REG has distance to page cross - VEC_SIZE. Guranteed
|
|
|
08c3a6 |
to not cross page so is safe to load. Since we have already
|
|
|
08c3a6 |
- loaded at least 1 VEC from rsi it is also guranteed to be safe.
|
|
|
08c3a6 |
- */
|
|
|
08c3a6 |
+ loaded at least 1 VEC from rsi it is also guranteed to be
|
|
|
08c3a6 |
+ safe. */
|
|
|
08c3a6 |
|
|
|
08c3a6 |
VMOVU (%rdi, %OFFSET_REG64), %ymm0
|
|
|
08c3a6 |
- VPCMPEQ (%rsi, %OFFSET_REG64), %ymm0, %ymm1
|
|
|
08c3a6 |
+ CMP_R1_S2_ymm (%ymm0, (%rsi, %OFFSET_REG64), %ymm2, %ymm1)
|
|
|
08c3a6 |
VPCMPEQ %ymm0, %ymmZERO, %ymm2
|
|
|
08c3a6 |
vpandn %ymm1, %ymm2, %ymm1
|
|
|
08c3a6 |
vpmovmskb %ymm1, %ecx
|
|
|
08c3a6 |
@@ -881,6 +1051,8 @@ L(ret_vec_page_cross_cont):
|
|
|
08c3a6 |
# else
|
|
|
08c3a6 |
movzbl (%rdi, %rcx), %eax
|
|
|
08c3a6 |
movzbl (%rsi, %rcx), %ecx
|
|
|
08c3a6 |
+ TOLOWER_gpr (%rax, %eax)
|
|
|
08c3a6 |
+ TOLOWER_gpr (%rcx, %ecx)
|
|
|
08c3a6 |
subl %ecx, %eax
|
|
|
08c3a6 |
xorl %r8d, %eax
|
|
|
08c3a6 |
subl %r8d, %eax
|
|
|
08c3a6 |
@@ -934,7 +1106,7 @@ L(less_1x_vec_till_page):
|
|
|
08c3a6 |
ja L(less_16_till_page)
|
|
|
08c3a6 |
|
|
|
08c3a6 |
VMOVU (%rdi), %xmm0
|
|
|
08c3a6 |
- VPCMPEQ (%rsi), %xmm0, %xmm1
|
|
|
08c3a6 |
+ CMP_R1_S2_xmm (%xmm0, (%rsi), %xmm2, %xmm1)
|
|
|
08c3a6 |
VPCMPEQ %xmm0, %xmmZERO, %xmm2
|
|
|
08c3a6 |
vpandn %xmm1, %xmm2, %xmm1
|
|
|
08c3a6 |
vpmovmskb %ymm1, %ecx
|
|
|
08c3a6 |
@@ -952,7 +1124,7 @@ L(less_1x_vec_till_page):
|
|
|
08c3a6 |
# endif
|
|
|
08c3a6 |
|
|
|
08c3a6 |
VMOVU (%rdi, %OFFSET_REG64), %xmm0
|
|
|
08c3a6 |
- VPCMPEQ (%rsi, %OFFSET_REG64), %xmm0, %xmm1
|
|
|
08c3a6 |
+ CMP_R1_S2_xmm (%xmm0, (%rsi, %OFFSET_REG64), %xmm2, %xmm1)
|
|
|
08c3a6 |
VPCMPEQ %xmm0, %xmmZERO, %xmm2
|
|
|
08c3a6 |
vpandn %xmm1, %xmm2, %xmm1
|
|
|
08c3a6 |
vpmovmskb %ymm1, %ecx
|
|
|
08c3a6 |
@@ -990,7 +1162,7 @@ L(less_16_till_page):
|
|
|
08c3a6 |
vmovq (%rdi), %xmm0
|
|
|
08c3a6 |
vmovq (%rsi), %xmm1
|
|
|
08c3a6 |
VPCMPEQ %xmm0, %xmmZERO, %xmm2
|
|
|
08c3a6 |
- VPCMPEQ %xmm1, %xmm0, %xmm1
|
|
|
08c3a6 |
+ CMP_R1_R2_xmm (%xmm0, %xmm1, %xmm3, %xmm1)
|
|
|
08c3a6 |
vpandn %xmm1, %xmm2, %xmm1
|
|
|
08c3a6 |
vpmovmskb %ymm1, %ecx
|
|
|
08c3a6 |
incb %cl
|
|
|
08c3a6 |
@@ -1010,7 +1182,7 @@ L(less_16_till_page):
|
|
|
08c3a6 |
vmovq (%rdi, %OFFSET_REG64), %xmm0
|
|
|
08c3a6 |
vmovq (%rsi, %OFFSET_REG64), %xmm1
|
|
|
08c3a6 |
VPCMPEQ %xmm0, %xmmZERO, %xmm2
|
|
|
08c3a6 |
- VPCMPEQ %xmm1, %xmm0, %xmm1
|
|
|
08c3a6 |
+ CMP_R1_R2_xmm (%xmm0, %xmm1, %xmm3, %xmm1)
|
|
|
08c3a6 |
vpandn %xmm1, %xmm2, %xmm1
|
|
|
08c3a6 |
vpmovmskb %ymm1, %ecx
|
|
|
08c3a6 |
incb %cl
|
|
|
08c3a6 |
@@ -1066,7 +1238,7 @@ L(ret_less_8_wcs):
|
|
|
08c3a6 |
vmovd (%rdi), %xmm0
|
|
|
08c3a6 |
vmovd (%rsi), %xmm1
|
|
|
08c3a6 |
VPCMPEQ %xmm0, %xmmZERO, %xmm2
|
|
|
08c3a6 |
- VPCMPEQ %xmm1, %xmm0, %xmm1
|
|
|
08c3a6 |
+ CMP_R1_R2_xmm (%xmm0, %xmm1, %xmm3, %xmm1)
|
|
|
08c3a6 |
vpandn %xmm1, %xmm2, %xmm1
|
|
|
08c3a6 |
vpmovmskb %ymm1, %ecx
|
|
|
08c3a6 |
subl $0xf, %ecx
|
|
|
08c3a6 |
@@ -1085,7 +1257,7 @@ L(ret_less_8_wcs):
|
|
|
08c3a6 |
vmovd (%rdi, %OFFSET_REG64), %xmm0
|
|
|
08c3a6 |
vmovd (%rsi, %OFFSET_REG64), %xmm1
|
|
|
08c3a6 |
VPCMPEQ %xmm0, %xmmZERO, %xmm2
|
|
|
08c3a6 |
- VPCMPEQ %xmm1, %xmm0, %xmm1
|
|
|
08c3a6 |
+ CMP_R1_R2_xmm (%xmm0, %xmm1, %xmm3, %xmm1)
|
|
|
08c3a6 |
vpandn %xmm1, %xmm2, %xmm1
|
|
|
08c3a6 |
vpmovmskb %ymm1, %ecx
|
|
|
08c3a6 |
subl $0xf, %ecx
|
|
|
08c3a6 |
@@ -1119,7 +1291,9 @@ L(less_4_till_page):
|
|
|
08c3a6 |
L(less_4_loop):
|
|
|
08c3a6 |
movzbl (%rdi), %eax
|
|
|
08c3a6 |
movzbl (%rsi, %rdi), %ecx
|
|
|
08c3a6 |
- subl %ecx, %eax
|
|
|
08c3a6 |
+ TOLOWER_gpr (%rax, %eax)
|
|
|
08c3a6 |
+ TOLOWER_gpr (%rcx, %BYTE_LOOP_REG)
|
|
|
08c3a6 |
+ subl %BYTE_LOOP_REG, %eax
|
|
|
08c3a6 |
jnz L(ret_less_4_loop)
|
|
|
08c3a6 |
testl %ecx, %ecx
|
|
|
08c3a6 |
jz L(ret_zero_4_loop)
|
|
|
08c3a6 |
@@ -1146,5 +1320,6 @@ L(ret_less_4_loop):
|
|
|
08c3a6 |
subl %r8d, %eax
|
|
|
08c3a6 |
ret
|
|
|
08c3a6 |
# endif
|
|
|
08c3a6 |
-END(STRCMP)
|
|
|
08c3a6 |
+ cfi_endproc
|
|
|
08c3a6 |
+ .size STRCMP, .-STRCMP
|
|
|
08c3a6 |
#endif
|
|
|
08c3a6 |
diff --git a/sysdeps/x86_64/multiarch/strncase_l-avx2-rtm.S b/sysdeps/x86_64/multiarch/strncase_l-avx2-rtm.S
|
|
|
08c3a6 |
new file mode 100644
|
|
|
08c3a6 |
index 0000000000000000..58c05dcfb8643791
|
|
|
08c3a6 |
--- /dev/null
|
|
|
08c3a6 |
+++ b/sysdeps/x86_64/multiarch/strncase_l-avx2-rtm.S
|
|
|
08c3a6 |
@@ -0,0 +1,16 @@
|
|
|
08c3a6 |
+#ifndef STRCMP
|
|
|
08c3a6 |
+# define STRCMP __strncasecmp_l_avx2_rtm
|
|
|
08c3a6 |
+#endif
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+#define _GLABEL(x) x ## _rtm
|
|
|
08c3a6 |
+#define GLABEL(x) _GLABEL(x)
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+#define ZERO_UPPER_VEC_REGISTERS_RETURN \
|
|
|
08c3a6 |
+ ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+#define VZEROUPPER_RETURN jmp L(return_vzeroupper)
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+#define SECTION(p) p##.avx.rtm
|
|
|
08c3a6 |
+#define OVERFLOW_STRCMP __strcasecmp_l_avx2_rtm
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+#include "strncase_l-avx2.S"
|
|
|
08c3a6 |
diff --git a/sysdeps/x86_64/multiarch/strncase_l-avx2.S b/sysdeps/x86_64/multiarch/strncase_l-avx2.S
|
|
|
08c3a6 |
new file mode 100644
|
|
|
08c3a6 |
index 0000000000000000..48c0aa21f84ad32c
|
|
|
08c3a6 |
--- /dev/null
|
|
|
08c3a6 |
+++ b/sysdeps/x86_64/multiarch/strncase_l-avx2.S
|
|
|
08c3a6 |
@@ -0,0 +1,27 @@
|
|
|
08c3a6 |
+/* strncasecmp_l optimized with AVX2.
|
|
|
08c3a6 |
+ Copyright (C) 2017-2022 Free Software Foundation, Inc.
|
|
|
08c3a6 |
+ This file is part of the GNU C Library.
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+ The GNU C Library is free software; you can redistribute it and/or
|
|
|
08c3a6 |
+ modify it under the terms of the GNU Lesser General Public
|
|
|
08c3a6 |
+ License as published by the Free Software Foundation; either
|
|
|
08c3a6 |
+ version 2.1 of the License, or (at your option) any later version.
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+ The GNU C Library is distributed in the hope that it will be useful,
|
|
|
08c3a6 |
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
08c3a6 |
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
08c3a6 |
+ Lesser General Public License for more details.
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+ You should have received a copy of the GNU Lesser General Public
|
|
|
08c3a6 |
+ License along with the GNU C Library; if not, see
|
|
|
08c3a6 |
+ <https://www.gnu.org/licenses/>. */
|
|
|
08c3a6 |
+
|
|
|
08c3a6 |
+#ifndef STRCMP
|
|
|
08c3a6 |
+# define STRCMP __strncasecmp_l_avx2
|
|
|
08c3a6 |
+#endif
|
|
|
08c3a6 |
+#define USE_AS_STRCASECMP_L
|
|
|
08c3a6 |
+#define USE_AS_STRNCMP
|
|
|
08c3a6 |
+#ifndef OVERFLOW_STRCMP
|
|
|
08c3a6 |
+# define OVERFLOW_STRCMP __strcasecmp_l_avx2
|
|
|
08c3a6 |
+#endif
|
|
|
08c3a6 |
+#include "strcmp-avx2.S"
|