From 1908853c722701099c5890628d9911ca83994b44 Mon Sep 17 00:00:00 2001 From: aekoroglu Date: May 20 2022 12:43:59 +0000 Subject: H.J's backports and RHEL sync patches --- diff --git a/SOURCES/glibc-rh1961109.patch b/SOURCES/glibc-rh1961109.patch new file mode 100644 index 0000000..e47c4d7 --- /dev/null +++ b/SOURCES/glibc-rh1961109.patch @@ -0,0 +1,165 @@ +commit f17164bd51db31f47fbbdae826c63b6d78184c45 +Author: Florian Weimer +Date: Tue May 18 07:21:33 2021 +0200 + + localedata: Use U+00AF MACRON in more EBCDIC charsets [BZ #27882] + + This updates IBM256, IBM277, IBM278, IBM280, IBM284, IBM297, IBM424 + in the same way that IBM273 was updated for bug 23290. + + IBM256 and IBM424 still have holes after this change, so HAS_HOLES + is not updated. + + Reviewed-by: Siddhesh Poyarekar + +diff --git a/iconvdata/ibm277.c b/iconvdata/ibm277.c +index f93ca2acb8718dd5..0e337dbbdc06a02f 100644 +--- a/iconvdata/ibm277.c ++++ b/iconvdata/ibm277.c +@@ -23,6 +23,6 @@ + #define TABLES + + #define CHARSET_NAME "IBM277//" +-#define HAS_HOLES 1 /* Not all 256 character are defined. */ ++#define HAS_HOLES 0 + + #include <8bit-gap.c> +diff --git a/iconvdata/ibm278.c b/iconvdata/ibm278.c +index 4263000760472913..7450fb8e5b846101 100644 +--- a/iconvdata/ibm278.c ++++ b/iconvdata/ibm278.c +@@ -23,6 +23,6 @@ + #define TABLES + + #define CHARSET_NAME "IBM278//" +-#define HAS_HOLES 1 /* Not all 256 character are defined. */ ++#define HAS_HOLES 0 + + #include <8bit-gap.c> +diff --git a/iconvdata/ibm280.c b/iconvdata/ibm280.c +index 3efddd7dec2728d9..2ea5478e4e0d7007 100644 +--- a/iconvdata/ibm280.c ++++ b/iconvdata/ibm280.c +@@ -23,6 +23,6 @@ + #define TABLES + + #define CHARSET_NAME "IBM280//" +-#define HAS_HOLES 1 /* Not all 256 character are defined. */ ++#define HAS_HOLES 0 + + #include <8bit-gap.c> +diff --git a/iconvdata/ibm284.c b/iconvdata/ibm284.c +index 57dab27d0cec4a33..8dbbc6344d18528f 100644 +--- a/iconvdata/ibm284.c ++++ b/iconvdata/ibm284.c +@@ -23,6 +23,6 @@ + #define TABLES + + #define CHARSET_NAME "IBM284//" +-#define HAS_HOLES 1 /* Not all 256 character are defined. */ ++#define HAS_HOLES 0 + + #include <8bit-gap.c> +diff --git a/iconvdata/ibm297.c b/iconvdata/ibm297.c +index f355659afd4b4502..81e63ba1f28f1548 100644 +--- a/iconvdata/ibm297.c ++++ b/iconvdata/ibm297.c +@@ -23,6 +23,6 @@ + #define TABLES + + #define CHARSET_NAME "IBM297//" +-#define HAS_HOLES 1 /* Not all 256 character are defined. */ ++#define HAS_HOLES 0 + + #include <8bit-gap.c> +diff --git a/localedata/charmaps/IBM256 b/localedata/charmaps/IBM256 +index 5cfd2db5f436cd07..bdc1abf0ade3bfc4 100644 +--- a/localedata/charmaps/IBM256 ++++ b/localedata/charmaps/IBM256 +@@ -194,7 +194,7 @@ CHARMAP + /xb9 VULGAR FRACTION THREE QUARTERS + /xba NOT SIGN + /xbb VERTICAL LINE +- /xbc OVERLINE ++ /xbc MACRON + /xbd DIAERESIS + /xbe ACUTE ACCENT + /xbf DOUBLE LOW LINE +diff --git a/localedata/charmaps/IBM277 b/localedata/charmaps/IBM277 +index 1c0b5cb9fb659364..2f6e3992109a2b33 100644 +--- a/localedata/charmaps/IBM277 ++++ b/localedata/charmaps/IBM277 +@@ -195,7 +195,7 @@ CHARMAP + /xb9 VULGAR FRACTION THREE QUARTERS + /xba NOT SIGN + /xbb VERTICAL LINE +- /xbc OVERLINE ++ /xbc MACRON + /xbd DIAERESIS + /xbe ACUTE ACCENT + /xbf MULTIPLICATION SIGN +diff --git a/localedata/charmaps/IBM278 b/localedata/charmaps/IBM278 +index 646961501c74c4df..bdfae7621028f003 100644 +--- a/localedata/charmaps/IBM278 ++++ b/localedata/charmaps/IBM278 +@@ -196,7 +196,7 @@ CHARMAP + /xb9 VULGAR FRACTION THREE QUARTERS + /xba NOT SIGN + /xbb VERTICAL LINE +- /xbc OVERLINE ++ /xbc MACRON + /xbd DIAERESIS + /xbe ACUTE ACCENT + /xbf MULTIPLICATION SIGN +diff --git a/localedata/charmaps/IBM280 b/localedata/charmaps/IBM280 +index 5de3b3e7b96796c0..4c31242806b0ac19 100644 +--- a/localedata/charmaps/IBM280 ++++ b/localedata/charmaps/IBM280 +@@ -195,7 +195,7 @@ CHARMAP + /xb9 VULGAR FRACTION THREE QUARTERS + /xba NOT SIGN + /xbb VERTICAL LINE +- /xbc OVERLINE ++ /xbc MACRON + /xbd DIAERESIS + /xbe ACUTE ACCENT + /xbf MULTIPLICATION SIGN +diff --git a/localedata/charmaps/IBM284 b/localedata/charmaps/IBM284 +index c64b2a65ab748540..46a8737a715e4e56 100644 +--- a/localedata/charmaps/IBM284 ++++ b/localedata/charmaps/IBM284 +@@ -195,7 +195,7 @@ CHARMAP + /xb9 VULGAR FRACTION THREE QUARTERS + /xba CIRCUMFLEX ACCENT + /xbb EXCLAMATION MARK +- /xbc OVERLINE ++ /xbc MACRON + /xbd TILDE + /xbe ACUTE ACCENT + /xbf MULTIPLICATION SIGN +diff --git a/localedata/charmaps/IBM297 b/localedata/charmaps/IBM297 +index 33b74eee437241aa..14361ad418cf1bc7 100644 +--- a/localedata/charmaps/IBM297 ++++ b/localedata/charmaps/IBM297 +@@ -195,7 +195,7 @@ CHARMAP + /xb9 VULGAR FRACTION THREE QUARTERS + /xba NOT SIGN + /xbb VERTICAL LINE +- /xbc OVERLINE ++ /xbc MACRON + /xbd TILDE + /xbe ACUTE ACCENT + /xbf MULTIPLICATION SIGN +diff --git a/localedata/charmaps/IBM424 b/localedata/charmaps/IBM424 +index 883e43b8ae04ee4c..deca11e1b18ec0a6 100644 +--- a/localedata/charmaps/IBM424 ++++ b/localedata/charmaps/IBM424 +@@ -175,7 +175,7 @@ CHARMAP + /xb9 VULGAR FRACTION THREE QUARTERS + /xba LEFT SQUARE BRACKET + /xbb RIGHT SQUARE BRACKET +- /xbc OVERLINE ++ /xbc MACRON + /xbd DIAERESIS + /xbe ACUTE ACCENT + /xbf MULTIPLICATION SIGN diff --git a/SOURCES/glibc-rh1982608.patch b/SOURCES/glibc-rh1982608.patch new file mode 100644 index 0000000..6f67ed6 --- /dev/null +++ b/SOURCES/glibc-rh1982608.patch @@ -0,0 +1,2216 @@ +This is a rebase of posix/glob.c from upstream (gnulib->glibc->rhel). + +Relevent upstream commits: + +7c477b57a31487eda516db02b9e04f22d1a6e6af posix/glob.c: update from gnulib + (This is the master commit to which we're syncing) + +gnulib commit 98f034a0c2ba8917c96f363de1a8d66244e411da + (This is the gnulib commit to which glibc upstream sync'd) + +Additional glibc upstream commits of note: +84f7ce84474c1648ce96884f1c91ca7b97ca3fc2 posix: Add glob64 with 64-bit time_t support + (just posix/glob.c and sysdeps/gnu/glob64-lstat-compat.c) +9a7ab0769b295cbf5232140401742a8f34bda3de hurd: Fix glob lstat compatibility +4883360415f1ed772ba44decc501d59deb17bdf0 posix: Sync glob code with gnulib +04986243d1af37ac0177ed2f9db0a066ebd2b212 Remove internal usage of extensible stat functions +ddc650e9b3dc916eab417ce9f79e67337b05035c Fix use-after-free in glob when expanding ~user (bug 25414) + + +diff -rup a/posix/glob-lstat-compat.c b/posix/glob-lstat-compat.c +--- a/posix/glob-lstat-compat.c 2018-08-01 01:10:47.000000000 -0400 ++++ b/posix/glob-lstat-compat.c 2022-05-02 22:49:06.504676711 -0400 +@@ -28,7 +28,8 @@ + # define GLOB_ATTRIBUTE attribute_compat_text_section + + /* Avoid calling gl_lstat with GLOB_ALTDIRFUNC. */ +-# define GLOB_NO_LSTAT ++# define GLOB_LSTAT gl_stat ++# define GLOB_LSTAT64 __stat64 + + # include + +diff -rup a/posix/glob.c b/posix/glob.c +--- a/posix/glob.c 2022-05-03 14:37:52.959042051 -0400 ++++ b/posix/glob.c 2022-05-02 22:49:18.655134696 -0400 +@@ -1,4 +1,4 @@ +-/* Copyright (C) 1991-2018 Free Software Foundation, Inc. ++/* Copyright (C) 1991-2022 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or +@@ -13,11 +13,22 @@ + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see +- . */ ++ . */ ++ ++#ifndef _LIBC ++ ++/* Don't use __attribute__ __nonnull__ in this compilation unit. Otherwise gcc ++ optimizes away the pattern == NULL test below. */ ++# define _GL_ARG_NONNULL(params) ++ ++# include ++ ++#endif + + #include + + #include ++#include + #include + #include + #include +@@ -26,7 +37,7 @@ + #include + #include + +-#if (defined _WIN32 || defined __WIN32__) && ! defined __CYGWIN__ ++#if defined _WIN32 && ! defined __CYGWIN__ + # define WINDOWS32 + #endif + +@@ -46,30 +57,38 @@ + # define sysconf(id) __sysconf (id) + # define closedir(dir) __closedir (dir) + # define opendir(name) __opendir (name) ++# undef dirfd ++# define dirfd(str) __dirfd (str) + # define readdir(str) __readdir64 (str) + # define getpwnam_r(name, bufp, buf, len, res) \ + __getpwnam_r (name, bufp, buf, len, res) +-# ifndef __lstat64 +-# define __lstat64(fname, buf) __lxstat64 (_STAT_VER, fname, buf) ++# define FLEXIBLE_ARRAY_MEMBER ++# ifndef struct_stat ++# define struct_stat struct stat + # endif +-# ifndef __stat64 +-# define __stat64(fname, buf) __xstat64 (_STAT_VER, fname, buf) ++# ifndef struct_stat64 ++# define struct_stat64 struct stat64 ++# endif ++# ifndef GLOB_LSTAT ++# define GLOB_LSTAT gl_lstat ++# endif ++# ifndef GLOB_FSTATAT64 ++# define GLOB_FSTATAT64 __fstatat64 + # endif +-# define struct_stat64 struct stat64 +-# define FLEXIBLE_ARRAY_MEMBER + # include + #else /* !_LIBC */ + # define __glob glob + # define __getlogin_r(buf, len) getlogin_r (buf, len) +-# define __lstat64(fname, buf) lstat (fname, buf) +-# define __stat64(fname, buf) stat (fname, buf) + # define __fxstatat64(_, d, f, st, flag) fstatat (d, f, st, flag) +-# define struct_stat64 struct stat + # ifndef __MVS__ + # define __alloca alloca + # endif + # define __readdir readdir + # define COMPILE_GLOB64 ++# define struct_stat struct stat ++# define struct_stat64 struct stat ++# define GLOB_LSTAT gl_lstat ++# define GLOB_FSTATAT64 fstatat + #endif /* _LIBC */ + + #include +@@ -80,7 +99,9 @@ + + static const char *next_brace_sub (const char *begin, int flags) __THROWNL; + +-typedef uint_fast8_t dirent_type; ++/* The type of ((struct dirent *) 0)->d_type is 'unsigned char' on most ++ platforms, but 'unsigned int' in the mingw from mingw.org. */ ++typedef uint_fast32_t dirent_type; + + #if !defined _LIBC && !defined HAVE_STRUCT_DIRENT_D_TYPE + /* Any distinct values will do here. +@@ -119,9 +140,9 @@ readdir_result_type (struct readdir_resu + /* Construct an initializer for a struct readdir_result object from a + struct dirent *. No copy of the name is made. */ + #define READDIR_RESULT_INITIALIZER(source) \ +- { \ +- source->d_name, \ +- D_TYPE_TO_RESULT (source) \ ++ { \ ++ source->d_name, \ ++ D_TYPE_TO_RESULT (source) \ + } + + /* Call gl_readdir on STREAM. This macro can be overridden to reduce +@@ -186,22 +207,15 @@ glob_lstat (glob_t *pglob, int flags, co + { + /* Use on glob-lstat-compat.c to provide a compat symbol which does not + use lstat / gl_lstat. */ +-#ifdef GLOB_NO_LSTAT +-# define GL_LSTAT gl_stat +-# define LSTAT64 __stat64 +-#else +-# define GL_LSTAT gl_lstat +-# define LSTAT64 __lstat64 +-#endif +- + union + { +- struct stat st; ++ struct_stat st; + struct_stat64 st64; + } ust; + return (__glibc_unlikely (flags & GLOB_ALTDIRFUNC) +- ? pglob->GL_LSTAT (fullname, &ust.st) +- : LSTAT64 (fullname, &ust.st64)); ++ ? pglob->GLOB_LSTAT (fullname, &ust.st) ++ : GLOB_FSTATAT64 (AT_FDCWD, fullname, &ust.st64, ++ AT_SYMLINK_NOFOLLOW)); + } + + /* Set *R = A + B. Return true if the answer is mathematically +@@ -211,7 +225,7 @@ glob_lstat (glob_t *pglob, int flags, co + static bool + size_add_wrapv (size_t a, size_t b, size_t *r) + { +-#if 5 <= __GNUC__ && !defined __ICC ++#if 7 <= __GNUC__ && !defined __ICC + return __builtin_add_overflow (a, b, r); + #else + *r = a + b; +@@ -228,8 +242,8 @@ glob_use_alloca (size_t alloca_used, siz + } + + static int glob_in_dir (const char *pattern, const char *directory, +- int flags, int (*errfunc) (const char *, int), +- glob_t *pglob, size_t alloca_used); ++ int flags, int (*errfunc) (const char *, int), ++ glob_t *pglob, size_t alloca_used); + static int prefix_array (const char *prefix, char **array, size_t n) __THROWNL; + static int collated_compare (const void *, const void *) __THROWNL; + +@@ -239,11 +253,12 @@ static int collated_compare (const void + static bool + is_dir (char const *filename, int flags, glob_t const *pglob) + { +- struct stat st; ++ struct_stat st; + struct_stat64 st64; + return (__glibc_unlikely (flags & GLOB_ALTDIRFUNC) + ? pglob->gl_stat (filename, &st) == 0 && S_ISDIR (st.st_mode) +- : __stat64 (filename, &st64) == 0 && S_ISDIR (st64.st_mode)); ++ : (GLOB_FSTATAT64 (AT_FDCWD, filename, &st64, 0) == 0 ++ && S_ISDIR (st64.st_mode))); + } + + /* Find the end of the sub-pattern in a brace expression. */ +@@ -254,17 +269,17 @@ next_brace_sub (const char *cp, int flag + while (*cp != '\0') + if ((flags & GLOB_NOESCAPE) == 0 && *cp == '\\') + { +- if (*++cp == '\0') +- break; +- ++cp; ++ if (*++cp == '\0') ++ break; ++ ++cp; + } + else + { +- if ((*cp == '}' && depth-- == 0) || (*cp == ',' && depth == 0)) +- break; ++ if ((*cp == '}' && depth-- == 0) || (*cp == ',' && depth == 0)) ++ break; + +- if (*cp++ == '{') +- depth++; ++ if (*cp++ == '{') ++ depth++; + } + + return *cp != '\0' ? cp : NULL; +@@ -285,7 +300,7 @@ next_brace_sub (const char *cp, int flag + int + GLOB_ATTRIBUTE + __glob (const char *pattern, int flags, int (*errfunc) (const char *, int), +- glob_t *pglob) ++ glob_t *pglob) + { + const char *filename; + char *dirname = NULL; +@@ -319,22 +334,22 @@ __glob (const char *pattern, int flags, + { + pglob->gl_pathc = 0; + if (!(flags & GLOB_DOOFFS)) +- pglob->gl_pathv = NULL; ++ pglob->gl_pathv = NULL; + else +- { +- size_t i; ++ { ++ size_t i; + +- if (pglob->gl_offs >= ~((size_t) 0) / sizeof (char *)) +- return GLOB_NOSPACE; ++ if (pglob->gl_offs >= ~((size_t) 0) / sizeof (char *)) ++ return GLOB_NOSPACE; + +- pglob->gl_pathv = (char **) malloc ((pglob->gl_offs + 1) +- * sizeof (char *)); +- if (pglob->gl_pathv == NULL) +- return GLOB_NOSPACE; +- +- for (i = 0; i <= pglob->gl_offs; ++i) +- pglob->gl_pathv[i] = NULL; +- } ++ pglob->gl_pathv = (char **) malloc ((pglob->gl_offs + 1) ++ * sizeof (char *)); ++ if (pglob->gl_pathv == NULL) ++ return GLOB_NOSPACE; ++ ++ for (i = 0; i <= pglob->gl_offs; ++i) ++ pglob->gl_pathv[i] = NULL; ++ } + } + + if (flags & GLOB_BRACE) +@@ -342,129 +357,129 @@ __glob (const char *pattern, int flags, + const char *begin; + + if (flags & GLOB_NOESCAPE) +- begin = strchr (pattern, '{'); ++ begin = strchr (pattern, '{'); + else +- { +- begin = pattern; +- while (1) +- { +- if (*begin == '\0') +- { +- begin = NULL; +- break; +- } +- +- if (*begin == '\\' && begin[1] != '\0') +- ++begin; +- else if (*begin == '{') +- break; +- +- ++begin; +- } +- } ++ { ++ begin = pattern; ++ while (1) ++ { ++ if (*begin == '\0') ++ { ++ begin = NULL; ++ break; ++ } ++ ++ if (*begin == '\\' && begin[1] != '\0') ++ ++begin; ++ else if (*begin == '{') ++ break; ++ ++ ++begin; ++ } ++ } + + if (begin != NULL) +- { +- /* Allocate working buffer large enough for our work. Note that +- we have at least an opening and closing brace. */ +- size_t firstc; +- char *alt_start; +- const char *p; +- const char *next; +- const char *rest; +- size_t rest_len; +- char *onealt; +- size_t pattern_len = strlen (pattern) - 1; +- int alloca_onealt = glob_use_alloca (alloca_used, pattern_len); +- if (alloca_onealt) +- onealt = alloca_account (pattern_len, alloca_used); +- else +- { +- onealt = malloc (pattern_len); +- if (onealt == NULL) +- return GLOB_NOSPACE; +- } +- +- /* We know the prefix for all sub-patterns. */ +- alt_start = mempcpy (onealt, pattern, begin - pattern); +- +- /* Find the first sub-pattern and at the same time find the +- rest after the closing brace. */ +- next = next_brace_sub (begin + 1, flags); +- if (next == NULL) +- { +- /* It is an invalid expression. */ +- illegal_brace: +- if (__glibc_unlikely (!alloca_onealt)) +- free (onealt); +- flags &= ~GLOB_BRACE; +- goto no_brace; +- } +- +- /* Now find the end of the whole brace expression. */ +- rest = next; +- while (*rest != '}') +- { +- rest = next_brace_sub (rest + 1, flags); +- if (rest == NULL) +- /* It is an illegal expression. */ +- goto illegal_brace; +- } +- /* Please note that we now can be sure the brace expression +- is well-formed. */ +- rest_len = strlen (++rest) + 1; +- +- /* We have a brace expression. BEGIN points to the opening {, +- NEXT points past the terminator of the first element, and END +- points past the final }. We will accumulate result names from +- recursive runs for each brace alternative in the buffer using +- GLOB_APPEND. */ +- firstc = pglob->gl_pathc; +- +- p = begin + 1; +- while (1) +- { +- int result; +- +- /* Construct the new glob expression. */ +- mempcpy (mempcpy (alt_start, p, next - p), rest, rest_len); +- +- result = __glob (onealt, +- ((flags & ~(GLOB_NOCHECK | GLOB_NOMAGIC)) +- | GLOB_APPEND), +- errfunc, pglob); +- +- /* If we got an error, return it. */ +- if (result && result != GLOB_NOMATCH) +- { +- if (__glibc_unlikely (!alloca_onealt)) +- free (onealt); +- if (!(flags & GLOB_APPEND)) +- { +- globfree (pglob); +- pglob->gl_pathc = 0; +- } +- return result; +- } +- +- if (*next == '}') +- /* We saw the last entry. */ +- break; +- +- p = next + 1; +- next = next_brace_sub (p, flags); +- assert (next != NULL); +- } +- +- if (__glibc_unlikely (!alloca_onealt)) +- free (onealt); +- +- if (pglob->gl_pathc != firstc) +- /* We found some entries. */ +- return 0; +- else if (!(flags & (GLOB_NOCHECK|GLOB_NOMAGIC))) +- return GLOB_NOMATCH; +- } ++ { ++ /* Allocate working buffer large enough for our work. Note that ++ we have at least an opening and closing brace. */ ++ size_t firstc; ++ char *alt_start; ++ const char *p; ++ const char *next; ++ const char *rest; ++ size_t rest_len; ++ char *onealt; ++ size_t pattern_len = strlen (pattern) - 1; ++ int alloca_onealt = glob_use_alloca (alloca_used, pattern_len); ++ if (alloca_onealt) ++ onealt = alloca_account (pattern_len, alloca_used); ++ else ++ { ++ onealt = malloc (pattern_len); ++ if (onealt == NULL) ++ return GLOB_NOSPACE; ++ } ++ ++ /* We know the prefix for all sub-patterns. */ ++ alt_start = mempcpy (onealt, pattern, begin - pattern); ++ ++ /* Find the first sub-pattern and at the same time find the ++ rest after the closing brace. */ ++ next = next_brace_sub (begin + 1, flags); ++ if (next == NULL) ++ { ++ /* It is an invalid expression. */ ++ illegal_brace: ++ if (__glibc_unlikely (!alloca_onealt)) ++ free (onealt); ++ flags &= ~GLOB_BRACE; ++ goto no_brace; ++ } ++ ++ /* Now find the end of the whole brace expression. */ ++ rest = next; ++ while (*rest != '}') ++ { ++ rest = next_brace_sub (rest + 1, flags); ++ if (rest == NULL) ++ /* It is an illegal expression. */ ++ goto illegal_brace; ++ } ++ /* Please note that we now can be sure the brace expression ++ is well-formed. */ ++ rest_len = strlen (++rest) + 1; ++ ++ /* We have a brace expression. BEGIN points to the opening {, ++ NEXT points past the terminator of the first element, and END ++ points past the final }. We will accumulate result names from ++ recursive runs for each brace alternative in the buffer using ++ GLOB_APPEND. */ ++ firstc = pglob->gl_pathc; ++ ++ p = begin + 1; ++ while (1) ++ { ++ int result; ++ ++ /* Construct the new glob expression. */ ++ mempcpy (mempcpy (alt_start, p, next - p), rest, rest_len); ++ ++ result = __glob (onealt, ++ ((flags & ~(GLOB_NOCHECK | GLOB_NOMAGIC)) ++ | GLOB_APPEND), ++ errfunc, pglob); ++ ++ /* If we got an error, return it. */ ++ if (result && result != GLOB_NOMATCH) ++ { ++ if (__glibc_unlikely (!alloca_onealt)) ++ free (onealt); ++ if (!(flags & GLOB_APPEND)) ++ { ++ globfree (pglob); ++ pglob->gl_pathc = 0; ++ } ++ return result; ++ } ++ ++ if (*next == '}') ++ /* We saw the last entry. */ ++ break; ++ ++ p = next + 1; ++ next = next_brace_sub (p, flags); ++ assert (next != NULL); ++ } ++ ++ if (__glibc_unlikely (!alloca_onealt)) ++ free (onealt); ++ ++ if (pglob->gl_pathc != firstc) ++ /* We found some entries. */ ++ return 0; ++ else if (!(flags & (GLOB_NOCHECK|GLOB_NOMAGIC))) ++ return GLOB_NOMATCH; ++ } + } + + no_brace: +@@ -486,33 +501,33 @@ __glob (const char *pattern, int flags, + if (filename == NULL) + { + /* This can mean two things: a simple name or "~name". The latter +- case is nothing but a notation for a directory. */ ++ case is nothing but a notation for a directory. */ + if ((flags & (GLOB_TILDE|GLOB_TILDE_CHECK)) && pattern[0] == '~') +- { +- dirname = (char *) pattern; +- dirlen = strlen (pattern); +- +- /* Set FILENAME to NULL as a special flag. This is ugly but +- other solutions would require much more code. We test for +- this special case below. */ +- filename = NULL; +- } ++ { ++ dirname = (char *) pattern; ++ dirlen = strlen (pattern); ++ ++ /* Set FILENAME to NULL as a special flag. This is ugly but ++ other solutions would require much more code. We test for ++ this special case below. */ ++ filename = NULL; ++ } + else +- { +- if (__glibc_unlikely (pattern[0] == '\0')) +- { +- dirs.gl_pathv = NULL; +- goto no_matches; +- } +- +- filename = pattern; +- dirname = (char *) "."; +- dirlen = 0; +- } ++ { ++ if (__glibc_unlikely (pattern[0] == '\0')) ++ { ++ dirs.gl_pathv = NULL; ++ goto no_matches; ++ } ++ ++ filename = pattern; ++ dirname = (char *) "."; ++ dirlen = 0; ++ } + } + else if (filename == pattern +- || (filename == pattern + 1 && pattern[0] == '\\' +- && (flags & GLOB_NOESCAPE) == 0)) ++ || (filename == pattern + 1 && pattern[0] == '\\' ++ && (flags & GLOB_NOESCAPE) == 0)) + { + /* "/pattern" or "\\/pattern". */ + dirname = (char *) "/"; +@@ -525,32 +540,32 @@ __glob (const char *pattern, int flags, + dirlen = filename - pattern; + #if defined __MSDOS__ || defined WINDOWS32 + if (*filename == ':' +- || (filename > pattern + 1 && filename[-1] == ':')) +- { +- char *drive_spec; +- +- ++dirlen; +- drive_spec = __alloca (dirlen + 1); +- *((char *) mempcpy (drive_spec, pattern, dirlen)) = '\0'; +- /* For now, disallow wildcards in the drive spec, to +- prevent infinite recursion in glob. */ +- if (__glob_pattern_p (drive_spec, !(flags & GLOB_NOESCAPE))) +- return GLOB_NOMATCH; +- /* If this is "d:pattern", we need to copy ':' to DIRNAME +- as well. If it's "d:/pattern", don't remove the slash +- from "d:/", since "d:" and "d:/" are not the same.*/ +- } ++ || (filename > pattern + 1 && filename[-1] == ':')) ++ { ++ char *drive_spec; ++ ++ ++dirlen; ++ drive_spec = __alloca (dirlen + 1); ++ *((char *) mempcpy (drive_spec, pattern, dirlen)) = '\0'; ++ /* For now, disallow wildcards in the drive spec, to ++ prevent infinite recursion in glob. */ ++ if (__glob_pattern_p (drive_spec, !(flags & GLOB_NOESCAPE))) ++ return GLOB_NOMATCH; ++ /* If this is "d:pattern", we need to copy ':' to DIRNAME ++ as well. If it's "d:/pattern", don't remove the slash ++ from "d:/", since "d:" and "d:/" are not the same.*/ ++ } + #endif + + if (glob_use_alloca (alloca_used, dirlen + 1)) +- newp = alloca_account (dirlen + 1, alloca_used); ++ newp = alloca_account (dirlen + 1, alloca_used); + else +- { +- newp = malloc (dirlen + 1); +- if (newp == NULL) +- return GLOB_NOSPACE; +- malloc_dirname = 1; +- } ++ { ++ newp = malloc (dirlen + 1); ++ if (newp == NULL) ++ return GLOB_NOSPACE; ++ malloc_dirname = 1; ++ } + *((char *) mempcpy (newp, pattern, dirlen)) = '\0'; + dirname = newp; + ++filename; +@@ -566,363 +581,383 @@ __glob (const char *pattern, int flags, + + if (filename[0] == '\0' && dirlen > 1 && !drive_root) + /* "pattern/". Expand "pattern", appending slashes. */ +- { +- int orig_flags = flags; +- if (!(flags & GLOB_NOESCAPE) && dirname[dirlen - 1] == '\\') +- { +- /* "pattern\\/". Remove the final backslash if it hasn't +- been quoted. */ +- char *p = (char *) &dirname[dirlen - 1]; +- +- while (p > dirname && p[-1] == '\\') --p; +- if ((&dirname[dirlen] - p) & 1) +- { +- *(char *) &dirname[--dirlen] = '\0'; +- flags &= ~(GLOB_NOCHECK | GLOB_NOMAGIC); +- } +- } +- int val = __glob (dirname, flags | GLOB_MARK, errfunc, pglob); +- if (val == 0) +- pglob->gl_flags = ((pglob->gl_flags & ~GLOB_MARK) +- | (flags & GLOB_MARK)); +- else if (val == GLOB_NOMATCH && flags != orig_flags) +- { +- /* Make sure globfree (&dirs); is a nop. */ +- dirs.gl_pathv = NULL; +- flags = orig_flags; +- oldcount = pglob->gl_pathc + pglob->gl_offs; +- goto no_matches; +- } +- retval = val; +- goto out; +- } ++ { ++ int orig_flags = flags; ++ if (!(flags & GLOB_NOESCAPE) && dirname[dirlen - 1] == '\\') ++ { ++ /* "pattern\\/". Remove the final backslash if it hasn't ++ been quoted. */ ++ char *p = (char *) &dirname[dirlen - 1]; ++ ++ while (p > dirname && p[-1] == '\\') --p; ++ if ((&dirname[dirlen] - p) & 1) ++ { ++ *(char *) &dirname[--dirlen] = '\0'; ++ flags &= ~(GLOB_NOCHECK | GLOB_NOMAGIC); ++ } ++ } ++ int val = __glob (dirname, flags | GLOB_MARK, errfunc, pglob); ++ if (val == 0) ++ pglob->gl_flags = ((pglob->gl_flags & ~GLOB_MARK) ++ | (flags & GLOB_MARK)); ++ else if (val == GLOB_NOMATCH && flags != orig_flags) ++ { ++ /* Make sure globfree (&dirs); is a nop. */ ++ dirs.gl_pathv = NULL; ++ flags = orig_flags; ++ oldcount = pglob->gl_pathc + pglob->gl_offs; ++ goto no_matches; ++ } ++ retval = val; ++ goto out; ++ } + } + + if ((flags & (GLOB_TILDE|GLOB_TILDE_CHECK)) && dirname[0] == '~') + { + if (dirname[1] == '\0' || dirname[1] == '/' +- || (!(flags & GLOB_NOESCAPE) && dirname[1] == '\\' +- && (dirname[2] == '\0' || dirname[2] == '/'))) +- { +- /* Look up home directory. */ +- char *home_dir = getenv ("HOME"); +- int malloc_home_dir = 0; +- if (home_dir == NULL || home_dir[0] == '\0') +- { ++ || (!(flags & GLOB_NOESCAPE) && dirname[1] == '\\' ++ && (dirname[2] == '\0' || dirname[2] == '/'))) ++ { ++ /* Look up home directory. */ ++ char *home_dir = getenv ("HOME"); ++ int malloc_home_dir = 0; ++ if (home_dir == NULL || home_dir[0] == '\0') ++ { + #ifdef WINDOWS32 +- /* Windows NT defines HOMEDRIVE and HOMEPATH. But give +- preference to HOME, because the user can change HOME. */ +- const char *home_drive = getenv ("HOMEDRIVE"); +- const char *home_path = getenv ("HOMEPATH"); +- +- if (home_drive != NULL && home_path != NULL) +- { +- size_t home_drive_len = strlen (home_drive); +- size_t home_path_len = strlen (home_path); +- char *mem = alloca (home_drive_len + home_path_len + 1); +- +- memcpy (mem, home_drive, home_drive_len); +- memcpy (mem + home_drive_len, home_path, home_path_len + 1); +- home_dir = mem; +- } +- else +- home_dir = "c:/users/default"; /* poor default */ ++ /* Windows NT defines HOMEDRIVE and HOMEPATH. But give ++ preference to HOME, because the user can change HOME. */ ++ const char *home_drive = getenv ("HOMEDRIVE"); ++ const char *home_path = getenv ("HOMEPATH"); ++ ++ if (home_drive != NULL && home_path != NULL) ++ { ++ size_t home_drive_len = strlen (home_drive); ++ size_t home_path_len = strlen (home_path); ++ char *mem = alloca (home_drive_len + home_path_len + 1); ++ ++ memcpy (mem, home_drive, home_drive_len); ++ memcpy (mem + home_drive_len, home_path, home_path_len + 1); ++ home_dir = mem; ++ } ++ else ++ home_dir = "c:/users/default"; /* poor default */ + #else +- int err; +- struct passwd *p; +- struct passwd pwbuf; +- struct scratch_buffer s; +- scratch_buffer_init (&s); +- while (true) +- { +- p = NULL; +- err = __getlogin_r (s.data, s.length); +- if (err == 0) +- { ++ int err; ++ struct passwd *p; ++ struct passwd pwbuf; ++ struct scratch_buffer s; ++ scratch_buffer_init (&s); ++ while (true) ++ { ++ p = NULL; ++ err = __getlogin_r (s.data, s.length); ++ if (err == 0) ++ { + # if defined HAVE_GETPWNAM_R || defined _LIBC +- size_t ssize = strlen (s.data) + 1; +- char *sdata = s.data; +- err = getpwnam_r (sdata, &pwbuf, sdata + ssize, +- s.length - ssize, &p); ++ size_t ssize = strlen (s.data) + 1; ++ char *sdata = s.data; ++ err = getpwnam_r (sdata, &pwbuf, sdata + ssize, ++ s.length - ssize, &p); + # else +- p = getpwnam (s.data); +- if (p == NULL) +- err = errno; ++ p = getpwnam (s.data); ++ if (p == NULL) ++ err = errno; + # endif +- } +- if (err != ERANGE) +- break; +- if (!scratch_buffer_grow (&s)) +- { +- retval = GLOB_NOSPACE; +- goto out; +- } +- } +- if (err == 0) +- { +- home_dir = strdup (p->pw_dir); +- malloc_home_dir = 1; +- } +- scratch_buffer_free (&s); +- if (err == 0 && home_dir == NULL) +- { +- retval = GLOB_NOSPACE; +- goto out; +- } ++ } ++ if (err != ERANGE) ++ break; ++ if (!scratch_buffer_grow (&s)) ++ { ++ retval = GLOB_NOSPACE; ++ goto out; ++ } ++ } ++ if (err == 0) ++ { ++ home_dir = strdup (p->pw_dir); ++ malloc_home_dir = 1; ++ } ++ scratch_buffer_free (&s); ++ if (err == 0 && home_dir == NULL) ++ { ++ retval = GLOB_NOSPACE; ++ goto out; ++ } + #endif /* WINDOWS32 */ +- } +- if (home_dir == NULL || home_dir[0] == '\0') +- { +- if (__glibc_unlikely (malloc_home_dir)) +- free (home_dir); +- if (flags & GLOB_TILDE_CHECK) +- { +- retval = GLOB_NOMATCH; +- goto out; +- } +- else +- { +- home_dir = (char *) "~"; /* No luck. */ +- malloc_home_dir = 0; +- } +- } +- /* Now construct the full directory. */ +- if (dirname[1] == '\0') +- { +- if (__glibc_unlikely (malloc_dirname)) +- free (dirname); +- +- dirname = home_dir; +- dirlen = strlen (dirname); +- malloc_dirname = malloc_home_dir; +- } +- else +- { +- char *newp; +- size_t home_len = strlen (home_dir); +- int use_alloca = glob_use_alloca (alloca_used, home_len + dirlen); +- if (use_alloca) +- newp = alloca_account (home_len + dirlen, alloca_used); +- else +- { +- newp = malloc (home_len + dirlen); +- if (newp == NULL) +- { +- if (__glibc_unlikely (malloc_home_dir)) +- free (home_dir); +- retval = GLOB_NOSPACE; +- goto out; +- } +- } +- +- mempcpy (mempcpy (newp, home_dir, home_len), +- &dirname[1], dirlen); +- +- if (__glibc_unlikely (malloc_dirname)) +- free (dirname); +- +- dirname = newp; +- dirlen += home_len - 1; +- malloc_dirname = !use_alloca; +- +- if (__glibc_unlikely (malloc_home_dir)) +- free (home_dir); +- } +- dirname_modified = 1; +- } ++ } ++ if (home_dir == NULL || home_dir[0] == '\0') ++ { ++ if (__glibc_unlikely (malloc_home_dir)) ++ free (home_dir); ++ if (flags & GLOB_TILDE_CHECK) ++ { ++ retval = GLOB_NOMATCH; ++ goto out; ++ } ++ else ++ { ++ home_dir = (char *) "~"; /* No luck. */ ++ malloc_home_dir = 0; ++ } ++ } ++ /* Now construct the full directory. */ ++ if (dirname[1] == '\0') ++ { ++ if (__glibc_unlikely (malloc_dirname)) ++ free (dirname); ++ ++ dirname = home_dir; ++ dirlen = strlen (dirname); ++ malloc_dirname = malloc_home_dir; ++ } ++ else ++ { ++ char *newp; ++ size_t home_len = strlen (home_dir); ++ int use_alloca = glob_use_alloca (alloca_used, home_len + dirlen); ++ if (use_alloca) ++ newp = alloca_account (home_len + dirlen, alloca_used); ++ else ++ { ++ newp = malloc (home_len + dirlen); ++ if (newp == NULL) ++ { ++ if (__glibc_unlikely (malloc_home_dir)) ++ free (home_dir); ++ retval = GLOB_NOSPACE; ++ goto out; ++ } ++ } ++ ++ mempcpy (mempcpy (newp, home_dir, home_len), ++ &dirname[1], dirlen); ++ ++ if (__glibc_unlikely (malloc_dirname)) ++ free (dirname); ++ ++ dirname = newp; ++ dirlen += home_len - 1; ++ malloc_dirname = !use_alloca; ++ ++ if (__glibc_unlikely (malloc_home_dir)) ++ free (home_dir); ++ } ++ dirname_modified = 1; ++ } + else +- { ++ { + #ifndef WINDOWS32 +- char *end_name = strchr (dirname, '/'); +- char *user_name; +- int malloc_user_name = 0; +- char *unescape = NULL; +- +- if (!(flags & GLOB_NOESCAPE)) +- { +- if (end_name == NULL) +- { +- unescape = strchr (dirname, '\\'); +- if (unescape) +- end_name = strchr (unescape, '\0'); +- } +- else +- unescape = memchr (dirname, '\\', end_name - dirname); +- } +- if (end_name == NULL) +- user_name = dirname + 1; +- else +- { +- char *newp; +- if (glob_use_alloca (alloca_used, end_name - dirname)) +- newp = alloca_account (end_name - dirname, alloca_used); +- else +- { +- newp = malloc (end_name - dirname); +- if (newp == NULL) +- { +- retval = GLOB_NOSPACE; +- goto out; +- } +- malloc_user_name = 1; +- } +- if (unescape != NULL) +- { +- char *p = mempcpy (newp, dirname + 1, +- unescape - dirname - 1); +- char *q = unescape; +- while (q != end_name) +- { +- if (*q == '\\') +- { +- if (q + 1 == end_name) +- { +- /* "~fo\\o\\" unescape to user_name "foo\\", +- but "~fo\\o\\/" unescape to user_name +- "foo". */ +- if (filename == NULL) +- *p++ = '\\'; +- break; +- } +- ++q; +- } +- *p++ = *q++; +- } +- *p = '\0'; +- } +- else +- *((char *) mempcpy (newp, dirname + 1, end_name - dirname - 1)) +- = '\0'; +- user_name = newp; +- } +- +- /* Look up specific user's home directory. */ +- { +- struct passwd *p; +- struct scratch_buffer pwtmpbuf; +- scratch_buffer_init (&pwtmpbuf); ++ /* Recognize ~user as a shorthand for the specified user's home ++ directory. */ ++ char *end_name = strchr (dirname, '/'); ++ char *user_name; ++ int malloc_user_name = 0; ++ char *unescape = NULL; ++ ++ if (!(flags & GLOB_NOESCAPE)) ++ { ++ if (end_name == NULL) ++ { ++ unescape = strchr (dirname, '\\'); ++ if (unescape) ++ end_name = strchr (unescape, '\0'); ++ } ++ else ++ unescape = memchr (dirname, '\\', end_name - dirname); ++ } ++ if (end_name == NULL) ++ user_name = dirname + 1; ++ else ++ { ++ char *newp; ++ if (glob_use_alloca (alloca_used, end_name - dirname)) ++ newp = alloca_account (end_name - dirname, alloca_used); ++ else ++ { ++ newp = malloc (end_name - dirname); ++ if (newp == NULL) ++ { ++ retval = GLOB_NOSPACE; ++ goto out; ++ } ++ malloc_user_name = 1; ++ } ++ if (unescape != NULL) ++ { ++ char *p = mempcpy (newp, dirname + 1, ++ unescape - dirname - 1); ++ char *q = unescape; ++ while (q != end_name) ++ { ++ if (*q == '\\') ++ { ++ if (q + 1 == end_name) ++ { ++ /* "~fo\\o\\" unescape to user_name "foo\\", ++ but "~fo\\o\\/" unescape to user_name ++ "foo". */ ++ if (filename == NULL) ++ *p++ = '\\'; ++ break; ++ } ++ ++q; ++ } ++ *p++ = *q++; ++ } ++ *p = '\0'; ++ } ++ else ++ *((char *) mempcpy (newp, dirname + 1, end_name - dirname - 1)) ++ = '\0'; ++ user_name = newp; ++ } ++ ++ /* Look up specific user's home directory. */ ++ { ++ struct passwd *p; ++ struct scratch_buffer pwtmpbuf; ++ scratch_buffer_init (&pwtmpbuf); + + # if defined HAVE_GETPWNAM_R || defined _LIBC +- struct passwd pwbuf; ++ struct passwd pwbuf; + +- while (getpwnam_r (user_name, &pwbuf, +- pwtmpbuf.data, pwtmpbuf.length, &p) +- == ERANGE) +- { +- if (!scratch_buffer_grow (&pwtmpbuf)) +- { +- retval = GLOB_NOSPACE; +- goto out; +- } +- } ++ while (getpwnam_r (user_name, &pwbuf, ++ pwtmpbuf.data, pwtmpbuf.length, &p) ++ == ERANGE) ++ { ++ if (!scratch_buffer_grow (&pwtmpbuf)) ++ { ++ retval = GLOB_NOSPACE; ++ goto out; ++ } ++ } + # else +- p = getpwnam (user_name); ++ p = getpwnam (user_name); + # endif + +- if (__glibc_unlikely (malloc_user_name)) +- free (user_name); ++ if (__glibc_unlikely (malloc_user_name)) ++ free (user_name); + +- /* If we found a home directory use this. */ +- if (p != NULL) +- { +- size_t home_len = strlen (p->pw_dir); +- size_t rest_len = end_name == NULL ? 0 : strlen (end_name); +- char *d, *newp; +- bool use_alloca = glob_use_alloca (alloca_used, +- home_len + rest_len + 1); +- +- if (use_alloca) +- newp = alloca_account (home_len + rest_len + 1, alloca_used); +- else +- { +- newp = malloc (home_len + rest_len + 1); +- if (newp == NULL) +- { +- scratch_buffer_free (&pwtmpbuf); +- retval = GLOB_NOSPACE; +- goto out; +- } +- } +- d = mempcpy (newp, p->pw_dir, home_len); +- if (end_name != NULL) +- d = mempcpy (d, end_name, rest_len); +- *d = '\0'; +- +- if (__glibc_unlikely (malloc_dirname)) +- free (dirname); +- dirname = newp; +- malloc_dirname = !use_alloca; +- +- dirlen = home_len + rest_len; +- dirname_modified = 1; +- } +- else +- { +- if (flags & GLOB_TILDE_CHECK) +- { +- /* We have to regard it as an error if we cannot find the +- home directory. */ +- retval = GLOB_NOMATCH; +- goto out; +- } +- } +- scratch_buffer_free (&pwtmpbuf); +- } +-#endif /* !WINDOWS32 */ +- } ++ /* If we found a home directory use this. */ ++ if (p != NULL) ++ { ++ size_t home_len = strlen (p->pw_dir); ++ size_t rest_len = end_name == NULL ? 0 : strlen (end_name); ++ /* dirname contains end_name; we can't free it now. */ ++ char *prev_dirname = ++ (__glibc_unlikely (malloc_dirname) ? dirname : NULL); ++ char *d; ++ ++ malloc_dirname = 0; ++ ++ if (glob_use_alloca (alloca_used, home_len + rest_len + 1)) ++ dirname = alloca_account (home_len + rest_len + 1, ++ alloca_used); ++ else ++ { ++ dirname = malloc (home_len + rest_len + 1); ++ if (dirname == NULL) ++ { ++ free (prev_dirname); ++ scratch_buffer_free (&pwtmpbuf); ++ retval = GLOB_NOSPACE; ++ goto out; ++ } ++ malloc_dirname = 1; ++ } ++ d = mempcpy (dirname, p->pw_dir, home_len); ++ if (end_name != NULL) ++ d = mempcpy (d, end_name, rest_len); ++ *d = '\0'; ++ ++ free (prev_dirname); ++ ++ dirlen = home_len + rest_len; ++ dirname_modified = 1; ++ } ++ else ++ { ++ if (flags & GLOB_TILDE_CHECK) ++ { ++ /* We have to regard it as an error if we cannot find the ++ home directory. */ ++ retval = GLOB_NOMATCH; ++ goto out; ++ } ++ } ++ scratch_buffer_free (&pwtmpbuf); ++ } ++#else /* WINDOWS32 */ ++ /* On native Windows, access to a user's home directory ++ (via GetUserProfileDirectory) or to a user's environment ++ variables (via ExpandEnvironmentStringsForUser) requires ++ the credentials of the user. Therefore we cannot support ++ the ~user syntax on this platform. ++ Handling ~user specially (and treat it like plain ~) if ++ user is getenv ("USERNAME") would not be a good idea, ++ since it would make people think that ~user is supported ++ in general. */ ++ if (flags & GLOB_TILDE_CHECK) ++ { ++ retval = GLOB_NOMATCH; ++ goto out; ++ } ++#endif /* WINDOWS32 */ ++ } + } + + /* Now test whether we looked for "~" or "~NAME". In this case we + can give the answer now. */ + if (filename == NULL) + { +- size_t newcount = pglob->gl_pathc + pglob->gl_offs; +- char **new_gl_pathv; ++ size_t newcount = pglob->gl_pathc + pglob->gl_offs; ++ char **new_gl_pathv; ++ ++ if (newcount > SIZE_MAX / sizeof (char *) - 2) ++ { ++ nospace: ++ free (pglob->gl_pathv); ++ pglob->gl_pathv = NULL; ++ pglob->gl_pathc = 0; ++ retval = GLOB_NOSPACE; ++ goto out; ++ } + +- if (newcount > SIZE_MAX / sizeof (char *) - 2) +- { +- nospace: +- free (pglob->gl_pathv); +- pglob->gl_pathv = NULL; +- pglob->gl_pathc = 0; +- retval = GLOB_NOSPACE; +- goto out; +- } +- +- new_gl_pathv = realloc (pglob->gl_pathv, +- (newcount + 2) * sizeof (char *)); +- if (new_gl_pathv == NULL) +- goto nospace; +- pglob->gl_pathv = new_gl_pathv; +- +- if (flags & GLOB_MARK && is_dir (dirname, flags, pglob)) +- { +- char *p; +- pglob->gl_pathv[newcount] = malloc (dirlen + 2); +- if (pglob->gl_pathv[newcount] == NULL) +- goto nospace; +- p = mempcpy (pglob->gl_pathv[newcount], dirname, dirlen); +- p[0] = '/'; +- p[1] = '\0'; +- if (__glibc_unlikely (malloc_dirname)) +- free (dirname); +- } +- else +- { +- if (__glibc_unlikely (malloc_dirname)) +- pglob->gl_pathv[newcount] = dirname; +- else +- { +- pglob->gl_pathv[newcount] = strdup (dirname); +- if (pglob->gl_pathv[newcount] == NULL) +- goto nospace; +- } +- } +- pglob->gl_pathv[++newcount] = NULL; +- ++pglob->gl_pathc; +- pglob->gl_flags = flags; ++ new_gl_pathv = realloc (pglob->gl_pathv, ++ (newcount + 2) * sizeof (char *)); ++ if (new_gl_pathv == NULL) ++ goto nospace; ++ pglob->gl_pathv = new_gl_pathv; ++ ++ if (flags & GLOB_MARK && is_dir (dirname, flags, pglob)) ++ { ++ char *p; ++ pglob->gl_pathv[newcount] = malloc (dirlen + 2); ++ if (pglob->gl_pathv[newcount] == NULL) ++ goto nospace; ++ p = mempcpy (pglob->gl_pathv[newcount], dirname, dirlen); ++ p[0] = '/'; ++ p[1] = '\0'; ++ if (__glibc_unlikely (malloc_dirname)) ++ free (dirname); ++ } ++ else ++ { ++ if (__glibc_unlikely (malloc_dirname)) ++ pglob->gl_pathv[newcount] = dirname; ++ else ++ { ++ pglob->gl_pathv[newcount] = strdup (dirname); ++ if (pglob->gl_pathv[newcount] == NULL) ++ goto nospace; ++ } ++ } ++ pglob->gl_pathv[++newcount] = NULL; ++ ++pglob->gl_pathc; ++ pglob->gl_flags = flags; + +- return 0; ++ return 0; + } + + meta = __glob_pattern_type (dirname, !(flags & GLOB_NOESCAPE)); +@@ -934,135 +969,135 @@ __glob (const char *pattern, int flags, + if (meta & (GLOBPAT_SPECIAL | GLOBPAT_BRACKET)) + { + /* The directory name contains metacharacters, so we +- have to glob for the directory, and then glob for +- the pattern in each directory found. */ ++ have to glob for the directory, and then glob for ++ the pattern in each directory found. */ + size_t i; + + if (!(flags & GLOB_NOESCAPE) && dirlen > 0 && dirname[dirlen - 1] == '\\') +- { +- /* "foo\\/bar". Remove the final backslash from dirname +- if it has not been quoted. */ +- char *p = (char *) &dirname[dirlen - 1]; +- +- while (p > dirname && p[-1] == '\\') --p; +- if ((&dirname[dirlen] - p) & 1) +- *(char *) &dirname[--dirlen] = '\0'; +- } ++ { ++ /* "foo\\/bar". Remove the final backslash from dirname ++ if it has not been quoted. */ ++ char *p = (char *) &dirname[dirlen - 1]; ++ ++ while (p > dirname && p[-1] == '\\') --p; ++ if ((&dirname[dirlen] - p) & 1) ++ *(char *) &dirname[--dirlen] = '\0'; ++ } + + if (__glibc_unlikely ((flags & GLOB_ALTDIRFUNC) != 0)) +- { +- /* Use the alternative access functions also in the recursive +- call. */ +- dirs.gl_opendir = pglob->gl_opendir; +- dirs.gl_readdir = pglob->gl_readdir; +- dirs.gl_closedir = pglob->gl_closedir; +- dirs.gl_stat = pglob->gl_stat; +- dirs.gl_lstat = pglob->gl_lstat; +- } ++ { ++ /* Use the alternative access functions also in the recursive ++ call. */ ++ dirs.gl_opendir = pglob->gl_opendir; ++ dirs.gl_readdir = pglob->gl_readdir; ++ dirs.gl_closedir = pglob->gl_closedir; ++ dirs.gl_stat = pglob->gl_stat; ++ dirs.gl_lstat = pglob->gl_lstat; ++ } + + status = __glob (dirname, +- ((flags & (GLOB_ERR | GLOB_NOESCAPE | GLOB_ALTDIRFUNC)) +- | GLOB_NOSORT | GLOB_ONLYDIR), +- errfunc, &dirs); ++ ((flags & (GLOB_ERR | GLOB_NOESCAPE | GLOB_ALTDIRFUNC)) ++ | GLOB_NOSORT | GLOB_ONLYDIR), ++ errfunc, &dirs); + if (status != 0) +- { +- if ((flags & GLOB_NOCHECK) == 0 || status != GLOB_NOMATCH) +- { +- retval = status; +- goto out; +- } +- goto no_matches; +- } ++ { ++ if ((flags & GLOB_NOCHECK) == 0 || status != GLOB_NOMATCH) ++ { ++ retval = status; ++ goto out; ++ } ++ goto no_matches; ++ } + + /* We have successfully globbed the preceding directory name. +- For each name we found, call glob_in_dir on it and FILENAME, +- appending the results to PGLOB. */ ++ For each name we found, call glob_in_dir on it and FILENAME, ++ appending the results to PGLOB. */ + for (i = 0; i < dirs.gl_pathc; ++i) +- { +- size_t old_pathc; ++ { ++ size_t old_pathc; + +- old_pathc = pglob->gl_pathc; +- status = glob_in_dir (filename, dirs.gl_pathv[i], +- ((flags | GLOB_APPEND) +- & ~(GLOB_NOCHECK | GLOB_NOMAGIC)), +- errfunc, pglob, alloca_used); +- if (status == GLOB_NOMATCH) +- /* No matches in this directory. Try the next. */ +- continue; +- +- if (status != 0) +- { +- globfree (&dirs); +- globfree (pglob); +- pglob->gl_pathc = 0; +- retval = status; +- goto out; +- } +- +- /* Stick the directory on the front of each name. */ +- if (prefix_array (dirs.gl_pathv[i], +- &pglob->gl_pathv[old_pathc + pglob->gl_offs], +- pglob->gl_pathc - old_pathc)) +- { +- globfree (&dirs); +- globfree (pglob); +- pglob->gl_pathc = 0; +- retval = GLOB_NOSPACE; +- goto out; +- } +- } ++ old_pathc = pglob->gl_pathc; ++ status = glob_in_dir (filename, dirs.gl_pathv[i], ++ ((flags | GLOB_APPEND) ++ & ~(GLOB_NOCHECK | GLOB_NOMAGIC)), ++ errfunc, pglob, alloca_used); ++ if (status == GLOB_NOMATCH) ++ /* No matches in this directory. Try the next. */ ++ continue; ++ ++ if (status != 0) ++ { ++ globfree (&dirs); ++ globfree (pglob); ++ pglob->gl_pathc = 0; ++ retval = status; ++ goto out; ++ } ++ ++ /* Stick the directory on the front of each name. */ ++ if (prefix_array (dirs.gl_pathv[i], ++ &pglob->gl_pathv[old_pathc + pglob->gl_offs], ++ pglob->gl_pathc - old_pathc)) ++ { ++ globfree (&dirs); ++ globfree (pglob); ++ pglob->gl_pathc = 0; ++ retval = GLOB_NOSPACE; ++ goto out; ++ } ++ } + + flags |= GLOB_MAGCHAR; + + /* We have ignored the GLOB_NOCHECK flag in the 'glob_in_dir' calls. +- But if we have not found any matching entry and the GLOB_NOCHECK +- flag was set we must return the input pattern itself. */ ++ But if we have not found any matching entry and the GLOB_NOCHECK ++ flag was set we must return the input pattern itself. */ + if (pglob->gl_pathc + pglob->gl_offs == oldcount) +- { +- no_matches: +- /* No matches. */ +- if (flags & GLOB_NOCHECK) +- { +- size_t newcount = pglob->gl_pathc + pglob->gl_offs; +- char **new_gl_pathv; +- +- if (newcount > SIZE_MAX / sizeof (char *) - 2) +- { +- nospace2: +- globfree (&dirs); +- retval = GLOB_NOSPACE; +- goto out; +- } +- +- new_gl_pathv = realloc (pglob->gl_pathv, +- (newcount + 2) * sizeof (char *)); +- if (new_gl_pathv == NULL) +- goto nospace2; +- pglob->gl_pathv = new_gl_pathv; +- +- pglob->gl_pathv[newcount] = strdup (pattern); +- if (pglob->gl_pathv[newcount] == NULL) +- { +- globfree (&dirs); +- globfree (pglob); +- pglob->gl_pathc = 0; +- retval = GLOB_NOSPACE; +- goto out; +- } +- +- ++pglob->gl_pathc; +- ++newcount; +- +- pglob->gl_pathv[newcount] = NULL; +- pglob->gl_flags = flags; +- } +- else +- { +- globfree (&dirs); +- retval = GLOB_NOMATCH; +- goto out; +- } +- } ++ { ++ no_matches: ++ /* No matches. */ ++ if (flags & GLOB_NOCHECK) ++ { ++ size_t newcount = pglob->gl_pathc + pglob->gl_offs; ++ char **new_gl_pathv; ++ ++ if (newcount > SIZE_MAX / sizeof (char *) - 2) ++ { ++ nospace2: ++ globfree (&dirs); ++ retval = GLOB_NOSPACE; ++ goto out; ++ } ++ ++ new_gl_pathv = realloc (pglob->gl_pathv, ++ (newcount + 2) * sizeof (char *)); ++ if (new_gl_pathv == NULL) ++ goto nospace2; ++ pglob->gl_pathv = new_gl_pathv; ++ ++ pglob->gl_pathv[newcount] = strdup (pattern); ++ if (pglob->gl_pathv[newcount] == NULL) ++ { ++ globfree (&dirs); ++ globfree (pglob); ++ pglob->gl_pathc = 0; ++ retval = GLOB_NOSPACE; ++ goto out; ++ } ++ ++ ++pglob->gl_pathc; ++ ++newcount; ++ ++ pglob->gl_pathv[newcount] = NULL; ++ pglob->gl_flags = flags; ++ } ++ else ++ { ++ globfree (&dirs); ++ retval = GLOB_NOMATCH; ++ goto out; ++ } ++ } + + globfree (&dirs); + } +@@ -1072,57 +1107,57 @@ __glob (const char *pattern, int flags, + int orig_flags = flags; + + if (meta & GLOBPAT_BACKSLASH) +- { +- char *p = strchr (dirname, '\\'), *q; +- /* We need to unescape the dirname string. It is certainly +- allocated by alloca, as otherwise filename would be NULL +- or dirname wouldn't contain backslashes. */ +- q = p; +- do +- { +- if (*p == '\\') +- { +- *q = *++p; +- --dirlen; +- } +- else +- *q = *p; +- ++q; +- } +- while (*p++ != '\0'); +- dirname_modified = 1; +- } ++ { ++ char *p = strchr (dirname, '\\'), *q; ++ /* We need to unescape the dirname string. It is certainly ++ allocated by alloca, as otherwise filename would be NULL ++ or dirname wouldn't contain backslashes. */ ++ q = p; ++ do ++ { ++ if (*p == '\\') ++ { ++ *q = *++p; ++ --dirlen; ++ } ++ else ++ *q = *p; ++ ++q; ++ } ++ while (*p++ != '\0'); ++ dirname_modified = 1; ++ } + if (dirname_modified) +- flags &= ~(GLOB_NOCHECK | GLOB_NOMAGIC); ++ flags &= ~(GLOB_NOCHECK | GLOB_NOMAGIC); + status = glob_in_dir (filename, dirname, flags, errfunc, pglob, +- alloca_used); ++ alloca_used); + if (status != 0) +- { +- if (status == GLOB_NOMATCH && flags != orig_flags +- && pglob->gl_pathc + pglob->gl_offs == oldcount) +- { +- /* Make sure globfree (&dirs); is a nop. */ +- dirs.gl_pathv = NULL; +- flags = orig_flags; +- goto no_matches; +- } +- retval = status; +- goto out; +- } ++ { ++ if (status == GLOB_NOMATCH && flags != orig_flags ++ && pglob->gl_pathc + pglob->gl_offs == oldcount) ++ { ++ /* Make sure globfree (&dirs); is a nop. */ ++ dirs.gl_pathv = NULL; ++ flags = orig_flags; ++ goto no_matches; ++ } ++ retval = status; ++ goto out; ++ } + + if (dirlen > 0) +- { +- /* Stick the directory on the front of each name. */ +- if (prefix_array (dirname, +- &pglob->gl_pathv[old_pathc + pglob->gl_offs], +- pglob->gl_pathc - old_pathc)) +- { +- globfree (pglob); +- pglob->gl_pathc = 0; +- retval = GLOB_NOSPACE; +- goto out; +- } +- } ++ { ++ /* Stick the directory on the front of each name. */ ++ if (prefix_array (dirname, ++ &pglob->gl_pathv[old_pathc + pglob->gl_offs], ++ pglob->gl_pathc - old_pathc)) ++ { ++ globfree (pglob); ++ pglob->gl_pathc = 0; ++ retval = GLOB_NOSPACE; ++ goto out; ++ } ++ } + } + + if (flags & GLOB_MARK) +@@ -1131,28 +1166,28 @@ __glob (const char *pattern, int flags, + size_t i; + + for (i = oldcount; i < pglob->gl_pathc + pglob->gl_offs; ++i) +- if (is_dir (pglob->gl_pathv[i], flags, pglob)) +- { +- size_t len = strlen (pglob->gl_pathv[i]) + 2; +- char *new = realloc (pglob->gl_pathv[i], len); +- if (new == NULL) +- { +- globfree (pglob); +- pglob->gl_pathc = 0; +- retval = GLOB_NOSPACE; +- goto out; +- } +- strcpy (&new[len - 2], "/"); +- pglob->gl_pathv[i] = new; +- } ++ if (is_dir (pglob->gl_pathv[i], flags, pglob)) ++ { ++ size_t len = strlen (pglob->gl_pathv[i]) + 2; ++ char *new = realloc (pglob->gl_pathv[i], len); ++ if (new == NULL) ++ { ++ globfree (pglob); ++ pglob->gl_pathc = 0; ++ retval = GLOB_NOSPACE; ++ goto out; ++ } ++ strcpy (&new[len - 2], "/"); ++ pglob->gl_pathv[i] = new; ++ } + } + + if (!(flags & GLOB_NOSORT)) + { + /* Sort the vector. */ + qsort (&pglob->gl_pathv[oldcount], +- pglob->gl_pathc + pglob->gl_offs - oldcount, +- sizeof (char *), collated_compare); ++ pglob->gl_pathc + pglob->gl_offs - oldcount, ++ sizeof (char *), collated_compare); + } + + out: +@@ -1204,14 +1239,14 @@ prefix_array (const char *dirname, char + if (dirlen > 1) + { + if (dirname[dirlen - 1] == '/' && dirname[dirlen - 2] == ':') +- /* DIRNAME is "d:/". Don't prepend the slash from DIRNAME. */ +- --dirlen; ++ /* DIRNAME is "d:/". Don't prepend the slash from DIRNAME. */ ++ --dirlen; + else if (dirname[dirlen - 1] == ':') +- { +- /* DIRNAME is "d:". Use ':' instead of '/'. */ +- --dirlen; +- dirsep_char = ':'; +- } ++ { ++ /* DIRNAME is "d:". Use ':' instead of '/'. */ ++ --dirlen; ++ dirsep_char = ':'; ++ } + } + #endif + +@@ -1220,16 +1255,16 @@ prefix_array (const char *dirname, char + size_t eltlen = strlen (array[i]) + 1; + char *new = malloc (dirlen + 1 + eltlen); + if (new == NULL) +- { +- while (i > 0) +- free (array[--i]); +- return 1; +- } ++ { ++ while (i > 0) ++ free (array[--i]); ++ return 1; ++ } + + { +- char *endp = mempcpy (new, dirname, dirlen); +- *endp++ = dirsep_char; +- mempcpy (endp, array[i], eltlen); ++ char *endp = mempcpy (new, dirname, dirlen); ++ *endp++ = dirsep_char; ++ mempcpy (endp, array[i], eltlen); + } + free (array[i]); + array[i] = new; +@@ -1244,11 +1279,13 @@ prefix_array (const char *dirname, char + The GLOB_APPEND flag is assumed to be set (always appends). */ + static int + glob_in_dir (const char *pattern, const char *directory, int flags, +- int (*errfunc) (const char *, int), +- glob_t *pglob, size_t alloca_used) ++ int (*errfunc) (const char *, int), ++ glob_t *pglob, size_t alloca_used) + { + size_t dirlen = strlen (directory); + void *stream = NULL; ++ struct scratch_buffer s; ++ scratch_buffer_init (&s); + # define GLOBNAMES_MEMBERS(nnames) \ + struct globnames *next; size_t count; char *name[nnames]; + struct globnames { GLOBNAMES_MEMBERS (FLEXIBLE_ARRAY_MEMBER) }; +@@ -1273,8 +1310,8 @@ glob_in_dir (const char *pattern, const + if (meta == GLOBPAT_NONE && (flags & (GLOB_NOCHECK|GLOB_NOMAGIC))) + { + /* We need not do any tests. The PATTERN contains no meta +- characters and we must not return an error therefore the +- result will always contain exactly one name. */ ++ characters and we must not return an error therefore the ++ result will always contain exactly one name. */ + flags |= GLOB_NOCHECK; + } + else if (meta == GLOBPAT_NONE) +@@ -1288,102 +1325,127 @@ glob_in_dir (const char *pattern, const + if (alloca_fullname) + fullname = alloca_account (fullsize, alloca_used); + else +- { +- fullname = malloc (fullsize); +- if (fullname == NULL) +- return GLOB_NOSPACE; +- } ++ { ++ fullname = malloc (fullsize); ++ if (fullname == NULL) ++ return GLOB_NOSPACE; ++ } + + mempcpy (mempcpy (mempcpy (fullname, directory, dirlen), +- "/", 1), +- pattern, patlen + 1); ++ "/", 1), ++ pattern, patlen + 1); + if (glob_lstat (pglob, flags, fullname) == 0 +- || errno == EOVERFLOW) +- /* We found this file to be existing. Now tell the rest +- of the function to copy this name into the result. */ +- flags |= GLOB_NOCHECK; ++ || errno == EOVERFLOW) ++ /* We found this file to be existing. Now tell the rest ++ of the function to copy this name into the result. */ ++ flags |= GLOB_NOCHECK; + + if (__glibc_unlikely (!alloca_fullname)) +- free (fullname); ++ free (fullname); + } + else + { + stream = (__builtin_expect (flags & GLOB_ALTDIRFUNC, 0) +- ? (*pglob->gl_opendir) (directory) +- : opendir (directory)); ++ ? (*pglob->gl_opendir) (directory) ++ : opendir (directory)); + if (stream == NULL) +- { +- if (errno != ENOTDIR +- && ((errfunc != NULL && (*errfunc) (directory, errno)) +- || (flags & GLOB_ERR))) +- return GLOB_ABORTED; +- } ++ { ++ if (errno != ENOTDIR ++ && ((errfunc != NULL && (*errfunc) (directory, errno)) ++ || (flags & GLOB_ERR))) ++ return GLOB_ABORTED; ++ } + else +- { +- int fnm_flags = ((!(flags & GLOB_PERIOD) ? FNM_PERIOD : 0) +- | ((flags & GLOB_NOESCAPE) ? FNM_NOESCAPE : 0)); +- flags |= GLOB_MAGCHAR; +- +- while (1) +- { +- struct readdir_result d; +- { +- if (__builtin_expect (flags & GLOB_ALTDIRFUNC, 0)) +- d = convert_dirent (GL_READDIR (pglob, stream)); +- else +- { ++ { ++ int dfd = dirfd (stream); ++ int fnm_flags = ((!(flags & GLOB_PERIOD) ? FNM_PERIOD : 0) ++ | ((flags & GLOB_NOESCAPE) ? FNM_NOESCAPE : 0)); ++ flags |= GLOB_MAGCHAR; ++ ++ while (1) ++ { ++ struct readdir_result d; ++ { ++ if (__builtin_expect (flags & GLOB_ALTDIRFUNC, 0)) ++ d = convert_dirent (GL_READDIR (pglob, stream)); ++ else ++ { + #ifdef COMPILE_GLOB64 +- d = convert_dirent (__readdir (stream)); ++ d = convert_dirent (__readdir (stream)); + #else +- d = convert_dirent64 (__readdir64 (stream)); ++ d = convert_dirent64 (__readdir64 (stream)); + #endif +- } +- } +- if (d.name == NULL) +- break; +- +- /* If we shall match only directories use the information +- provided by the dirent call if possible. */ +- if (flags & GLOB_ONLYDIR) +- switch (readdir_result_type (d)) +- { +- case DT_DIR: case DT_LNK: case DT_UNKNOWN: break; +- default: continue; +- } +- +- if (fnmatch (pattern, d.name, fnm_flags) == 0) +- { +- if (cur == names->count) +- { +- struct globnames *newnames; +- size_t count = names->count * 2; +- size_t nameoff = offsetof (struct globnames, name); +- size_t size = FLEXSIZEOF (struct globnames, name, +- count * sizeof (char *)); +- if ((SIZE_MAX - nameoff) / 2 / sizeof (char *) +- < names->count) +- goto memory_error; +- if (glob_use_alloca (alloca_used, size)) +- newnames = names_alloca +- = alloca_account (size, alloca_used); +- else if ((newnames = malloc (size)) +- == NULL) +- goto memory_error; +- newnames->count = count; +- newnames->next = names; +- names = newnames; +- cur = 0; +- } +- names->name[cur] = strdup (d.name); +- if (names->name[cur] == NULL) +- goto memory_error; +- ++cur; +- ++nfound; +- if (SIZE_MAX - pglob->gl_offs <= nfound) +- goto memory_error; +- } +- } +- } ++ } ++ } ++ if (d.name == NULL) ++ break; ++ ++ /* If we shall match only directories use the information ++ provided by the dirent call if possible. */ ++ if (flags & GLOB_ONLYDIR) ++ switch (readdir_result_type (d)) ++ { ++ default: continue; ++ case DT_DIR: break; ++ case DT_LNK: case DT_UNKNOWN: ++ /* The filesystem was too lazy to give us a hint, ++ so we have to do it the hard way. */ ++ if (__glibc_unlikely (dfd < 0 || flags & GLOB_ALTDIRFUNC)) ++ { ++ size_t namelen = strlen (d.name); ++ size_t need = dirlen + 1 + namelen + 1; ++ if (s.length < need ++ && !scratch_buffer_set_array_size (&s, need, 1)) ++ goto memory_error; ++ char *p = mempcpy (s.data, directory, dirlen); ++ *p = '/'; ++ p += p[-1] != '/'; ++ memcpy (p, d.name, namelen + 1); ++ if (! is_dir (s.data, flags, pglob)) ++ continue; ++ } ++ else ++ { ++ struct_stat64 st64; ++ if (! (GLOB_FSTATAT64 (dfd, d.name, &st64, 0) == 0 ++ && S_ISDIR (st64.st_mode))) ++ continue; ++ } ++ } ++ ++ if (fnmatch (pattern, d.name, fnm_flags) == 0) ++ { ++ if (cur == names->count) ++ { ++ struct globnames *newnames; ++ size_t count = names->count * 2; ++ size_t nameoff = offsetof (struct globnames, name); ++ size_t size = FLEXSIZEOF (struct globnames, name, ++ count * sizeof (char *)); ++ if ((SIZE_MAX - nameoff) / 2 / sizeof (char *) ++ < names->count) ++ goto memory_error; ++ if (glob_use_alloca (alloca_used, size)) ++ newnames = names_alloca ++ = alloca_account (size, alloca_used); ++ else if ((newnames = malloc (size)) ++ == NULL) ++ goto memory_error; ++ newnames->count = count; ++ newnames->next = names; ++ names = newnames; ++ cur = 0; ++ } ++ names->name[cur] = strdup (d.name); ++ if (names->name[cur] == NULL) ++ goto memory_error; ++ ++cur; ++ ++nfound; ++ if (SIZE_MAX - pglob->gl_offs <= nfound) ++ goto memory_error; ++ } ++ } ++ } + } + + if (nfound == 0 && (flags & GLOB_NOCHECK)) +@@ -1392,7 +1454,7 @@ glob_in_dir (const char *pattern, const + nfound = 1; + names->name[cur] = malloc (len + 1); + if (names->name[cur] == NULL) +- goto memory_error; ++ goto memory_error; + *((char *) mempcpy (names->name[cur++], pattern, len)) = '\0'; + } + +@@ -1403,82 +1465,83 @@ glob_in_dir (const char *pattern, const + result = 0; + + if (SIZE_MAX / sizeof (char *) - pglob->gl_pathc +- < pglob->gl_offs + nfound + 1) +- goto memory_error; ++ < pglob->gl_offs + nfound + 1) ++ goto memory_error; + + new_gl_pathv +- = realloc (pglob->gl_pathv, +- (pglob->gl_pathc + pglob->gl_offs + nfound + 1) +- * sizeof (char *)); ++ = realloc (pglob->gl_pathv, ++ (pglob->gl_pathc + pglob->gl_offs + nfound + 1) ++ * sizeof (char *)); + + if (new_gl_pathv == NULL) +- { +- memory_error: +- while (1) +- { +- struct globnames *old = names; +- for (size_t i = 0; i < cur; ++i) +- free (names->name[i]); +- names = names->next; +- /* NB: we will not leak memory here if we exit without +- freeing the current block assigned to OLD. At least +- the very first block is always allocated on the stack +- and this is the block assigned to OLD here. */ +- if (names == NULL) +- { +- assert (old == init_names); +- break; +- } +- cur = names->count; +- if (old == names_alloca) +- names_alloca = names; +- else +- free (old); +- } +- result = GLOB_NOSPACE; +- } ++ { ++ memory_error: ++ while (1) ++ { ++ struct globnames *old = names; ++ for (size_t i = 0; i < cur; ++i) ++ free (names->name[i]); ++ names = names->next; ++ /* NB: we will not leak memory here if we exit without ++ freeing the current block assigned to OLD. At least ++ the very first block is always allocated on the stack ++ and this is the block assigned to OLD here. */ ++ if (names == NULL) ++ { ++ assert (old == init_names); ++ break; ++ } ++ cur = names->count; ++ if (old == names_alloca) ++ names_alloca = names; ++ else ++ free (old); ++ } ++ result = GLOB_NOSPACE; ++ } + else +- { +- while (1) +- { +- struct globnames *old = names; +- for (size_t i = 0; i < cur; ++i) +- new_gl_pathv[pglob->gl_offs + pglob->gl_pathc++] +- = names->name[i]; +- names = names->next; +- /* NB: we will not leak memory here if we exit without +- freeing the current block assigned to OLD. At least +- the very first block is always allocated on the stack +- and this is the block assigned to OLD here. */ +- if (names == NULL) +- { +- assert (old == init_names); +- break; +- } +- cur = names->count; +- if (old == names_alloca) +- names_alloca = names; +- else +- free (old); +- } ++ { ++ while (1) ++ { ++ struct globnames *old = names; ++ for (size_t i = 0; i < cur; ++i) ++ new_gl_pathv[pglob->gl_offs + pglob->gl_pathc++] ++ = names->name[i]; ++ names = names->next; ++ /* NB: we will not leak memory here if we exit without ++ freeing the current block assigned to OLD. At least ++ the very first block is always allocated on the stack ++ and this is the block assigned to OLD here. */ ++ if (names == NULL) ++ { ++ assert (old == init_names); ++ break; ++ } ++ cur = names->count; ++ if (old == names_alloca) ++ names_alloca = names; ++ else ++ free (old); ++ } + +- pglob->gl_pathv = new_gl_pathv; ++ pglob->gl_pathv = new_gl_pathv; + +- pglob->gl_pathv[pglob->gl_offs + pglob->gl_pathc] = NULL; ++ pglob->gl_pathv[pglob->gl_offs + pglob->gl_pathc] = NULL; + +- pglob->gl_flags = flags; +- } ++ pglob->gl_flags = flags; ++ } + } + + if (stream != NULL) + { + save = errno; + if (__glibc_unlikely (flags & GLOB_ALTDIRFUNC)) +- (*pglob->gl_closedir) (stream); ++ (*pglob->gl_closedir) (stream); + else +- closedir (stream); ++ closedir (stream); + __set_errno (save); + } + ++ scratch_buffer_free (&s); + return result; + } +diff -rup a/sysdeps/gnu/glob-lstat-compat.c b/sysdeps/gnu/glob-lstat-compat.c +--- a/sysdeps/gnu/glob-lstat-compat.c 2018-08-01 01:10:47.000000000 -0400 ++++ b/sysdeps/gnu/glob-lstat-compat.c 2022-05-02 17:51:04.167557574 -0400 +@@ -29,7 +29,8 @@ + #define GLOB_ATTRIBUTE attribute_compat_text_section + + /* Avoid calling gl_lstat with GLOB_ALTDIRFUNC. */ +-#define GLOB_NO_LSTAT ++#define GLOB_LSTAT gl_stat ++#define GLOB_LSTAT64 __stat64 + + #include + +diff -rup a/sysdeps/unix/sysv/linux/glob-lstat-compat.c b/sysdeps/unix/sysv/linux/glob-lstat-compat.c +--- a/sysdeps/unix/sysv/linux/glob-lstat-compat.c 2018-08-01 01:10:47.000000000 -0400 ++++ b/sysdeps/unix/sysv/linux/glob-lstat-compat.c 2022-05-02 23:05:45.197297341 -0400 +@@ -30,7 +30,12 @@ + #define GLOB_ATTRIBUTE attribute_compat_text_section + + /* Avoid calling gl_lstat with GLOB_ALTDIRFUNC. */ +-#define GLOB_NO_LSTAT ++# define COMPILE_GLOB64 1 ++# define struct_stat struct stat ++# define struct_stat64 struct stat64 ++# define GLOB_LSTAT gl_stat ++# define GLOB_STAT64 __stat64 ++# define GLOB_LSTAT64 __stat64 + + #include + diff --git a/SOURCES/glibc-rh2077835.patch b/SOURCES/glibc-rh2077835.patch new file mode 100644 index 0000000..7323d49 --- /dev/null +++ b/SOURCES/glibc-rh2077835.patch @@ -0,0 +1,211 @@ +commit 2376944b9e5c0364b9fb473e4d8dabca31b57167 +Author: Stefan Liebler +Date: Wed Apr 13 14:36:09 2022 +0200 + + S390: Add new s390 platform z16. + + The new IBM z16 is added to platform string array. + The macro _DL_PLATFORMS_COUNT is incremented. + + _dl_hwcaps_subdir is extended by "z16" if HWCAP_S390_VXRS_PDE2 + is set. HWCAP_S390_NNPA is not tested in _dl_hwcaps_subdirs_active + as those instructions may be replaced or removed in future. + + tst-glibc-hwcaps.c is extended in order to test z16 via new marker5. + + A fatal glibc error is dumped if glibc was build with architecture + level set for z16, but run on an older machine. (See dl-hwcap-check.h) + +Reworked for RHEL 8.7.0 + +diff -Nrup a/elf/Makefile b/elf/Makefile +--- a/elf/Makefile 2022-05-16 21:48:11.267916411 -0400 ++++ b/elf/Makefile 2022-05-16 21:48:56.106095151 -0400 +@@ -347,7 +347,8 @@ modules-names = testobj1 testobj2 testob + libmarkermod2-1 libmarkermod2-2 \ + libmarkermod3-1 libmarkermod3-2 libmarkermod3-3 \ + libmarkermod4-1 libmarkermod4-2 libmarkermod4-3 libmarkermod4-4 \ +- tst-tls20mod-bad tst-tls21mod \ ++ libmarkermod5-1 libmarkermod5-2 libmarkermod5-3 libmarkermod5-4 \ ++ libmarkermod5-5 tst-tls20mod-bad tst-tls21mod \ + + # Most modules build with _ISOMAC defined, but those filtered out + # depend on internal headers. +@@ -1782,6 +1783,7 @@ LDFLAGS-libmarkermod1-1.so += -Wl,-sonam + LDFLAGS-libmarkermod2-1.so += -Wl,-soname,libmarkermod2.so + LDFLAGS-libmarkermod3-1.so += -Wl,-soname,libmarkermod3.so + LDFLAGS-libmarkermod4-1.so += -Wl,-soname,libmarkermod4.so ++LDFLAGS-libmarkermod5-1.so += -Wl,-soname,libmarkermod5.so + $(objpfx)libmarkermod%.os : markermodMARKER-VALUE.c + $(compile-command.c) \ + -DMARKER=marker$(firstword $(subst -, ,$*)) \ +@@ -1794,6 +1796,8 @@ $(objpfx)libmarkermod3.so: $(objpfx)libm + cp $< $@ + $(objpfx)libmarkermod4.so: $(objpfx)libmarkermod4-1.so + cp $< $@ ++$(objpfx)libmarkermod5.so: $(objpfx)libmarkermod5-1.so ++ cp $< $@ + + # tst-glibc-hwcaps-prepend checks that --glibc-hwcaps-prepend is + # preferred over auto-detected subdirectories. +diff -Nrup a/elf/tst-glibc-hwcaps-cache.script b/elf/tst-glibc-hwcaps-cache.script +--- a/elf/tst-glibc-hwcaps-cache.script 2022-05-16 21:48:11.053915558 -0400 ++++ b/elf/tst-glibc-hwcaps-cache.script 2022-05-16 21:48:56.107095155 -0400 +@@ -4,6 +4,7 @@ + cp $B/elf/libmarkermod2-1.so $L/libmarkermod2.so + cp $B/elf/libmarkermod3-1.so $L/libmarkermod3.so + cp $B/elf/libmarkermod4-1.so $L/libmarkermod4.so ++cp $B/elf/libmarkermod5-1.so $L/libmarkermod5.so + + mkdirp 0770 $L/glibc-hwcaps/power9 + cp $B/elf/libmarkermod2-2.so $L/glibc-hwcaps/power9/libmarkermod2.so +@@ -20,6 +21,11 @@ mkdirp 0770 $L/glibc-hwcaps/z15 + cp $B/elf/libmarkermod4-2.so $L/glibc-hwcaps/z13/libmarkermod4.so + cp $B/elf/libmarkermod4-3.so $L/glibc-hwcaps/z14/libmarkermod4.so + cp $B/elf/libmarkermod4-4.so $L/glibc-hwcaps/z15/libmarkermod4.so ++mkdirp 0770 $L/glibc-hwcaps/z16 ++cp $B/elf/libmarkermod5-2.so $L/glibc-hwcaps/z13/libmarkermod5.so ++cp $B/elf/libmarkermod5-3.so $L/glibc-hwcaps/z14/libmarkermod5.so ++cp $B/elf/libmarkermod5-4.so $L/glibc-hwcaps/z15/libmarkermod5.so ++cp $B/elf/libmarkermod5-5.so $L/glibc-hwcaps/z16/libmarkermod5.so + + mkdirp 0770 $L/glibc-hwcaps/x86-64-v2 + cp $B/elf/libmarkermod2-2.so $L/glibc-hwcaps/x86-64-v2/libmarkermod2.so +diff -Nrup a/sysdeps/s390/dl-procinfo.c b/sysdeps/s390/dl-procinfo.c +--- a/sysdeps/s390/dl-procinfo.c 2022-05-16 21:48:11.250916343 -0400 ++++ b/sysdeps/s390/dl-procinfo.c 2022-05-16 21:48:56.107095155 -0400 +@@ -64,11 +64,12 @@ PROCINFO_CLASS const char _dl_s390_cap_f + #if !defined PROCINFO_DECL && defined SHARED + ._dl_s390_platforms + #else +-PROCINFO_CLASS const char _dl_s390_platforms[10][7] ++PROCINFO_CLASS const char _dl_s390_platforms[11][7] + #endif + #ifndef PROCINFO_DECL + = { +- "g5", "z900", "z990", "z9-109", "z10", "z196", "zEC12", "z13", "z14", "z15" ++ "g5", "z900", "z990", "z9-109", "z10", "z196", "zEC12", "z13", "z14", "z15", ++ "z16" + } + #endif + #if !defined SHARED || defined PROCINFO_DECL +diff -Nrup a/sysdeps/s390/dl-procinfo.h b/sysdeps/s390/dl-procinfo.h +--- a/sysdeps/s390/dl-procinfo.h 2022-05-16 21:48:11.250916343 -0400 ++++ b/sysdeps/s390/dl-procinfo.h 2022-05-16 21:48:56.107095155 -0400 +@@ -23,7 +23,7 @@ + + #define _DL_HWCAP_COUNT 23 + +-#define _DL_PLATFORMS_COUNT 10 ++#define _DL_PLATFORMS_COUNT 11 + + /* The kernel provides up to 32 capability bits with elf_hwcap. */ + #define _DL_FIRST_PLATFORM 32 +diff -Nrup a/sysdeps/s390/s390-64/dl-hwcaps-subdirs.c b/sysdeps/s390/s390-64/dl-hwcaps-subdirs.c +--- a/sysdeps/s390/s390-64/dl-hwcaps-subdirs.c 2022-05-16 21:48:11.053915558 -0400 ++++ b/sysdeps/s390/s390-64/dl-hwcaps-subdirs.c 2022-05-16 21:58:02.840301911 -0400 +@@ -19,8 +19,8 @@ + #include + #include + +-const char _dl_hwcaps_subdirs[] = "z15:z14:z13"; +-enum { subdirs_count = 3 }; /* Number of components in _dl_hwcaps_subdirs. */ ++const char _dl_hwcaps_subdirs[] = "z16:z15:z14:z13"; ++enum { subdirs_count = 4 }; /* Number of components in _dl_hwcaps_subdirs. */ + + uint32_t + _dl_hwcaps_subdirs_active (void) +@@ -50,5 +50,12 @@ _dl_hwcaps_subdirs_active (void) + return _dl_hwcaps_subdirs_build_bitmask (subdirs_count, active); + ++active; + ++ /* z16. ++ Note: We do not list HWCAP_S390_NNPA here as, according to the Principles of ++ Operation, those instructions may be replaced or removed in future. */ ++ if (!(GLRO (dl_hwcap) & HWCAP_S390_VXRS_PDE2)) ++ return _dl_hwcaps_subdirs_build_bitmask (subdirs_count, active); ++ ++active; ++ + return _dl_hwcaps_subdirs_build_bitmask (subdirs_count, active); + } +diff -Nrup a/sysdeps/s390/s390-64/Makefile b/sysdeps/s390/s390-64/Makefile +--- a/sysdeps/s390/s390-64/Makefile 2022-05-16 21:48:11.053915558 -0400 ++++ b/sysdeps/s390/s390-64/Makefile 2022-05-16 21:54:08.832355745 -0400 +@@ -7,8 +7,11 @@ CFLAGS-rtld.c += -Wno-uninitialized -Wno + CFLAGS-dl-load.c += -Wno-unused + CFLAGS-dl-reloc.c += -Wno-unused + +-$(objpfx)tst-glibc-hwcaps: $(objpfx)libmarkermod2-1.so \ +- $(objpfx)libmarkermod3-1.so $(objpfx)libmarkermod4-1.so ++$(objpfx)tst-glibc-hwcaps: \ ++ $(objpfx)libmarkermod2-1.so \ ++ $(objpfx)libmarkermod3-1.so \ ++ $(objpfx)libmarkermod4-1.so \ ++ $(objpfx)libmarkermod5-1.so + $(objpfx)tst-glibc-hwcaps.out: \ + $(objpfx)libmarkermod2.so \ + $(objpfx)glibc-hwcaps/z13/libmarkermod2.so \ +@@ -19,6 +22,11 @@ $(objpfx)tst-glibc-hwcaps.out: \ + $(objpfx)glibc-hwcaps/z13/libmarkermod4.so \ + $(objpfx)glibc-hwcaps/z14/libmarkermod4.so \ + $(objpfx)glibc-hwcaps/z15/libmarkermod4.so \ ++ $(objpfx)libmarkermod5.so \ ++ $(objpfx)glibc-hwcaps/z13/libmarkermod5.so \ ++ $(objpfx)glibc-hwcaps/z14/libmarkermod5.so \ ++ $(objpfx)glibc-hwcaps/z15/libmarkermod5.so \ ++ $(objpfx)glibc-hwcaps/z16/libmarkermod5.so + + $(objpfx)glibc-hwcaps/z13/libmarkermod2.so: $(objpfx)libmarkermod2-2.so + $(make-target-directory) +@@ -38,6 +46,18 @@ $(objpfx)glibc-hwcaps/z14/libmarkermod4. + $(objpfx)glibc-hwcaps/z15/libmarkermod4.so: $(objpfx)libmarkermod4-4.so + $(make-target-directory) + cp $< $@ ++$(objpfx)glibc-hwcaps/z13/libmarkermod5.so: $(objpfx)libmarkermod5-2.so ++ $(make-target-directory) ++ cp $< $@ ++$(objpfx)glibc-hwcaps/z14/libmarkermod5.so: $(objpfx)libmarkermod5-3.so ++ $(make-target-directory) ++ cp $< $@ ++$(objpfx)glibc-hwcaps/z15/libmarkermod5.so: $(objpfx)libmarkermod5-4.so ++ $(make-target-directory) ++ cp $< $@ ++$(objpfx)glibc-hwcaps/z16/libmarkermod5.so: $(objpfx)libmarkermod5-5.so ++ $(make-target-directory) ++ cp $< $@ + + ifeq (no,$(build-hardcoded-path-in-tests)) + # This is an ld.so.cache test, and RPATH/RUNPATH in the executable +diff -Nrup a/sysdeps/s390/s390-64/tst-glibc-hwcaps.c b/sysdeps/s390/s390-64/tst-glibc-hwcaps.c +--- a/sysdeps/s390/s390-64/tst-glibc-hwcaps.c 2022-05-16 21:48:11.053915558 -0400 ++++ b/sysdeps/s390/s390-64/tst-glibc-hwcaps.c 2022-05-16 21:48:56.107095155 -0400 +@@ -25,6 +25,7 @@ + extern int marker2 (void); + extern int marker3 (void); + extern int marker4 (void); ++extern int marker5 (void); + + /* Return the arch level, 10 for the baseline libmarkermod*.so's. */ + static int +@@ -63,9 +64,13 @@ compute_level (void) + return 12; + if (strcmp (platform, "z15") == 0) + return 13; ++ if (strcmp (platform, "z16") == 0) ++ return 14; + printf ("warning: unrecognized AT_PLATFORM value: %s\n", platform); + /* Assume that the new platform supports z15. */ + return 13; ++ /* Assume that the new platform supports z16. */ ++ return 14; + } + + static int +@@ -76,6 +81,7 @@ do_test (void) + TEST_COMPARE (marker2 (), MIN (level - 9, 2)); + TEST_COMPARE (marker3 (), MIN (level - 9, 3)); + TEST_COMPARE (marker4 (), MIN (level - 9, 4)); ++ TEST_COMPARE (marker5 (), MIN (level - 9, 5)); + return 0; + } + diff --git a/SOURCES/glibc-rh2086853.patch b/SOURCES/glibc-rh2086853.patch new file mode 100644 index 0000000..d11e4cb --- /dev/null +++ b/SOURCES/glibc-rh2086853.patch @@ -0,0 +1,30 @@ +commit 61a87530108ec9181e1b18a9b727ec3cc3ba7532 +Author: Siddhesh Poyarekar +Date: Fri May 13 10:01:47 2022 +0530 + + fortify: Ensure that __glibc_fortify condition is a constant [BZ #29141] + + The fix c8ee1c85 introduced a -1 check for object size without also + checking that object size is a constant. Because of this, the tree + optimizer passes in gcc fail to fold away one of the branches in + __glibc_fortify and trips on a spurious Wstringop-overflow. The warning + itself is incorrect and the branch does go away eventually in DCE in the + rtl passes in gcc, but the constant check is a helpful hint to simplify + code early, so add it in. + + Resolves: BZ #29141 + Signed-off-by: Siddhesh Poyarekar + +diff --git a/misc/sys/cdefs.h b/misc/sys/cdefs.h +index 404496c7d6da4fb3..f3d7efdd2a9320f7 100644 +--- a/misc/sys/cdefs.h ++++ b/misc/sys/cdefs.h +@@ -145,7 +145,7 @@ + /* Length is known to be safe at compile time if the __L * __S <= __OBJSZ + condition can be folded to a constant and if it is true, or unknown (-1) */ + #define __glibc_safe_or_unknown_len(__l, __s, __osz) \ +- ((__osz) == (__SIZE_TYPE__) -1 \ ++ ((__builtin_constant_p (__osz) && (__osz) == (__SIZE_TYPE__) -1) \ + || (__glibc_unsigned_or_positive (__l) \ + && __builtin_constant_p (__glibc_safe_len_cond ((__SIZE_TYPE__) (__l), \ + (__s), (__osz))) \ diff --git a/SOURCES/glibc-sw24097-1.patch b/SOURCES/glibc-sw24097-1.patch new file mode 100644 index 0000000..0894a24 --- /dev/null +++ b/SOURCES/glibc-sw24097-1.patch @@ -0,0 +1,254 @@ +From ea31ff23cc0f5577d6947a717a2d733d7963f5c6 Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Fri, 1 Feb 2019 12:17:09 -0800 +Subject: [PATCH] x86-64 memchr/wmemchr: Properly handle the length parameter + [BZ #24097] + +On x32, the size_t parameter may be passed in the lower 32 bits of a +64-bit register with the non-zero upper 32 bits. The string/memory +functions written in assembly can only use the lower 32 bits of a +64-bit register as length or must clear the upper 32 bits before using +the full 64-bit register for length. + +This pach fixes memchr/wmemchr for x32. Tested on x86-64 and x32. On +x86-64, libc.so is the same with and withou the fix. + + [BZ #24097] + CVE-2019-6488 + * sysdeps/x86_64/memchr.S: Use RDX_LP for length. Clear the + upper 32 bits of RDX register. + * sysdeps/x86_64/multiarch/memchr-avx2.S: Likewise. + * sysdeps/x86_64/x32/Makefile (tests): Add tst-size_t-memchr and + tst-size_t-wmemchr. + * sysdeps/x86_64/x32/test-size_t.h: New file. + * sysdeps/x86_64/x32/tst-size_t-memchr.c: Likewise. + * sysdeps/x86_64/x32/tst-size_t-wmemchr.c: Likewise. + +(cherry picked from commit 97700a34f36721b11a754cf37a1cc40695ece1fd) +--- + sysdeps/x86_64/memchr.S | 10 ++-- + sysdeps/x86_64/multiarch/memchr-avx2.S | 8 ++- + sysdeps/x86_64/x32/Makefile | 8 +++ + sysdeps/x86_64/x32/test-size_t.h | 35 ++++++++++++ + sysdeps/x86_64/x32/tst-size_t-memchr.c | 72 +++++++++++++++++++++++++ + sysdeps/x86_64/x32/tst-size_t-wmemchr.c | 20 +++++++ + 6 files changed, 148 insertions(+), 5 deletions(-) + create mode 100644 sysdeps/x86_64/x32/test-size_t.h + create mode 100644 sysdeps/x86_64/x32/tst-size_t-memchr.c + create mode 100644 sysdeps/x86_64/x32/tst-size_t-wmemchr.c + +diff --git a/sysdeps/x86_64/memchr.S b/sysdeps/x86_64/memchr.S +index feef5d4f..cb320257 100644 +--- a/sysdeps/x86_64/memchr.S ++++ b/sysdeps/x86_64/memchr.S +@@ -34,12 +34,16 @@ ENTRY(MEMCHR) + mov %edi, %ecx + + #ifdef USE_AS_WMEMCHR +- test %rdx, %rdx ++ test %RDX_LP, %RDX_LP + jz L(return_null) +- shl $2, %rdx ++ shl $2, %RDX_LP + #else ++# ifdef __ILP32__ ++ /* Clear the upper 32 bits. */ ++ movl %edx, %edx ++# endif + punpcklbw %xmm1, %xmm1 +- test %rdx, %rdx ++ test %RDX_LP, %RDX_LP + jz L(return_null) + punpcklbw %xmm1, %xmm1 + #endif +diff --git a/sysdeps/x86_64/multiarch/memchr-avx2.S b/sysdeps/x86_64/multiarch/memchr-avx2.S +index 5f5e7725..c81da19b 100644 +--- a/sysdeps/x86_64/multiarch/memchr-avx2.S ++++ b/sysdeps/x86_64/multiarch/memchr-avx2.S +@@ -40,16 +40,20 @@ + ENTRY (MEMCHR) + # ifndef USE_AS_RAWMEMCHR + /* Check for zero length. */ +- testq %rdx, %rdx ++ test %RDX_LP, %RDX_LP + jz L(null) + # endif + movl %edi, %ecx + /* Broadcast CHAR to YMM0. */ + vmovd %esi, %xmm0 + # ifdef USE_AS_WMEMCHR +- shl $2, %rdx ++ shl $2, %RDX_LP + vpbroadcastd %xmm0, %ymm0 + # else ++# ifdef __ILP32__ ++ /* Clear the upper 32 bits. */ ++ movl %edx, %edx ++# endif + vpbroadcastb %xmm0, %ymm0 + # endif + /* Check if we may cross page boundary with one vector load. */ +diff --git a/sysdeps/x86_64/x32/Makefile b/sysdeps/x86_64/x32/Makefile +index f2ebc24f..7d528889 100644 +--- a/sysdeps/x86_64/x32/Makefile ++++ b/sysdeps/x86_64/x32/Makefile +@@ -4,3 +4,11 @@ ifeq ($(subdir),math) + # 64-bit llround. Add -fno-builtin-lround to silence the compiler. + CFLAGS-s_llround.c += -fno-builtin-lround + endif ++ ++ifeq ($(subdir),string) ++tests += tst-size_t-memchr ++endif ++ ++ifeq ($(subdir),wcsmbs) ++tests += tst-size_t-wmemchr ++endif +diff --git a/sysdeps/x86_64/x32/test-size_t.h b/sysdeps/x86_64/x32/test-size_t.h +new file mode 100644 +index 00000000..78a94086 +--- /dev/null ++++ b/sysdeps/x86_64/x32/test-size_t.h +@@ -0,0 +1,35 @@ ++/* Test string/memory functions with size_t in the lower 32 bits of ++ 64-bit register. ++ Copyright (C) 2019 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#define TEST_MAIN ++#include ++ ++/* On x32, parameter_t may be passed in a 64-bit register with the LEN ++ field in the lower 32 bits. When the LEN field of 64-bit register ++ is passed to string/memory function as the size_t parameter, only ++ the lower 32 bits can be used. */ ++typedef struct ++{ ++ union ++ { ++ size_t len; ++ void (*fn) (void); ++ }; ++ void *p; ++} parameter_t; +diff --git a/sysdeps/x86_64/x32/tst-size_t-memchr.c b/sysdeps/x86_64/x32/tst-size_t-memchr.c +new file mode 100644 +index 00000000..29a3daf1 +--- /dev/null ++++ b/sysdeps/x86_64/x32/tst-size_t-memchr.c +@@ -0,0 +1,72 @@ ++/* Test memchr with size_t in the lower 32 bits of 64-bit register. ++ Copyright (C) 2019 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#ifndef WIDE ++# define TEST_NAME "memchr" ++#else ++# define TEST_NAME "wmemchr" ++#endif /* WIDE */ ++#include "test-size_t.h" ++ ++#ifndef WIDE ++# define MEMCHR memchr ++# define CHAR char ++# define UCHAR unsigned char ++#else ++# include ++# define MEMCHR wmemchr ++# define CHAR wchar_t ++# define UCHAR wchar_t ++#endif /* WIDE */ ++ ++IMPL (MEMCHR, 1) ++ ++typedef CHAR * (*proto_t) (const CHAR*, int, size_t); ++ ++static CHAR * ++__attribute__ ((noinline, noclone)) ++do_memchr (parameter_t a, parameter_t b) ++{ ++ return CALL (&b, a.p, (uintptr_t) b.p, a.len); ++} ++ ++static int ++test_main (void) ++{ ++ test_init (); ++ ++ parameter_t src = { { page_size / sizeof (CHAR) }, buf2 }; ++ parameter_t c = { { 0 }, (void *) (uintptr_t) 0x12 }; ++ ++ int ret = 0; ++ FOR_EACH_IMPL (impl, 0) ++ { ++ c.fn = impl->fn; ++ CHAR *res = do_memchr (src, c); ++ if (res) ++ { ++ error (0, 0, "Wrong result in function %s: %p != NULL", ++ impl->name, res); ++ ret = 1; ++ } ++ } ++ ++ return ret ? EXIT_FAILURE : EXIT_SUCCESS; ++} ++ ++#include +diff --git a/sysdeps/x86_64/x32/tst-size_t-wmemchr.c b/sysdeps/x86_64/x32/tst-size_t-wmemchr.c +new file mode 100644 +index 00000000..877801d6 +--- /dev/null ++++ b/sysdeps/x86_64/x32/tst-size_t-wmemchr.c +@@ -0,0 +1,20 @@ ++/* Test wmemchr with size_t in the lower 32 bits of 64-bit register. ++ Copyright (C) 2019 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#define WIDE 1 ++#include "tst-size_t-memchr.c" +-- +GitLab + diff --git a/SOURCES/glibc-sw24097-2.patch b/SOURCES/glibc-sw24097-2.patch new file mode 100644 index 0000000..937d393 --- /dev/null +++ b/SOURCES/glibc-sw24097-2.patch @@ -0,0 +1,227 @@ +From a107bd201f836b1d82e1a9086a2a48625ac1c0dd Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Wed, 2 Mar 2022 13:52:42 -0800 +Subject: [PATCH] x86-64 memcmp/wmemcmp: Properly handle the length parameter + [BZ #24097] + +On x32, the size_t parameter may be passed in the lower 32 bits of a +64-bit register with the non-zero upper 32 bits. The string/memory +functions written in assembly can only use the lower 32 bits of a +64-bit register as length or must clear the upper 32 bits before using +the full 64-bit register for length. + +This pach fixes memcmp/wmemcmp for x32. Tested on x86-64 and x32. On +x86-64, libc.so is the same with and withou the fix. + + [BZ #24097] + CVE-2019-6488 + * sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S: Use RDX_LP for + length. Clear the upper 32 bits of RDX register. + * sysdeps/x86_64/multiarch/memcmp-sse4.S: Likewise. + * sysdeps/x86_64/multiarch/memcmp-ssse3.S: Likewise. + * sysdeps/x86_64/x32/Makefile (tests): Add tst-size_t-memcmp and + tst-size_t-wmemcmp. + * sysdeps/x86_64/x32/tst-size_t-memcmp.c: New file. + * sysdeps/x86_64/x32/tst-size_t-wmemcmp.c: Likewise. + +(cherry picked from commit b304fc201d2f6baf52ea790df8643e99772243cd) +--- + sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S | 7 +- + sysdeps/x86_64/multiarch/memcmp-sse4.S | 9 ++- + sysdeps/x86_64/multiarch/memcmp-ssse3.S | 7 +- + sysdeps/x86_64/x32/Makefile | 4 +- + sysdeps/x86_64/x32/tst-size_t-memcmp.c | 76 ++++++++++++++++++++ + sysdeps/x86_64/x32/tst-size_t-wmemcmp.c | 20 ++++++ + 6 files changed, 114 insertions(+), 9 deletions(-) + create mode 100644 sysdeps/x86_64/x32/tst-size_t-memcmp.c + create mode 100644 sysdeps/x86_64/x32/tst-size_t-wmemcmp.c + +diff --git a/sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S b/sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S +index 30f764c3..e3a35b89 100644 +--- a/sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S ++++ b/sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S +@@ -58,9 +58,12 @@ + .section .text.avx,"ax",@progbits + ENTRY (MEMCMP) + # ifdef USE_AS_WMEMCMP +- shl $2, %rdx ++ shl $2, %RDX_LP ++# elif defined __ILP32__ ++ /* Clear the upper 32 bits. */ ++ movl %edx, %edx + # endif +- cmpq $VEC_SIZE, %rdx ++ cmp $VEC_SIZE, %RDX_LP + jb L(less_vec) + + /* From VEC to 2 * VEC. No branch when size == VEC_SIZE. */ +diff --git a/sysdeps/x86_64/multiarch/memcmp-sse4.S b/sysdeps/x86_64/multiarch/memcmp-sse4.S +index 8e164f2c..302900f5 100644 +--- a/sysdeps/x86_64/multiarch/memcmp-sse4.S ++++ b/sysdeps/x86_64/multiarch/memcmp-sse4.S +@@ -42,13 +42,16 @@ + .section .text.sse4.1,"ax",@progbits + ENTRY (MEMCMP) + # ifdef USE_AS_WMEMCMP +- shl $2, %rdx ++ shl $2, %RDX_LP ++# elif defined __ILP32__ ++ /* Clear the upper 32 bits. */ ++ mov %edx, %edx + # endif + pxor %xmm0, %xmm0 +- cmp $79, %rdx ++ cmp $79, %RDX_LP + ja L(79bytesormore) + # ifndef USE_AS_WMEMCMP +- cmp $1, %rdx ++ cmp $1, %RDX_LP + je L(firstbyte) + # endif + add %rdx, %rsi +diff --git a/sysdeps/x86_64/multiarch/memcmp-ssse3.S b/sysdeps/x86_64/multiarch/memcmp-ssse3.S +index 6f76c641..69d030fc 100644 +--- a/sysdeps/x86_64/multiarch/memcmp-ssse3.S ++++ b/sysdeps/x86_64/multiarch/memcmp-ssse3.S +@@ -33,9 +33,12 @@ + atom_text_section + ENTRY (MEMCMP) + # ifdef USE_AS_WMEMCMP +- shl $2, %rdx +- test %rdx, %rdx ++ shl $2, %RDX_LP ++ test %RDX_LP, %RDX_LP + jz L(equal) ++# elif defined __ILP32__ ++ /* Clear the upper 32 bits. */ ++ mov %edx, %edx + # endif + mov %rdx, %rcx + mov %rdi, %rdx +diff --git a/sysdeps/x86_64/x32/Makefile b/sysdeps/x86_64/x32/Makefile +index 7d528889..ddec7f04 100644 +--- a/sysdeps/x86_64/x32/Makefile ++++ b/sysdeps/x86_64/x32/Makefile +@@ -6,9 +6,9 @@ CFLAGS-s_llround.c += -fno-builtin-lround + endif + + ifeq ($(subdir),string) +-tests += tst-size_t-memchr ++tests += tst-size_t-memchr tst-size_t-memcmp + endif + + ifeq ($(subdir),wcsmbs) +-tests += tst-size_t-wmemchr ++tests += tst-size_t-wmemchr tst-size_t-wmemcmp + endif +diff --git a/sysdeps/x86_64/x32/tst-size_t-memcmp.c b/sysdeps/x86_64/x32/tst-size_t-memcmp.c +new file mode 100644 +index 00000000..9bd6fdb4 +--- /dev/null ++++ b/sysdeps/x86_64/x32/tst-size_t-memcmp.c +@@ -0,0 +1,76 @@ ++/* Test memcmp with size_t in the lower 32 bits of 64-bit register. ++ Copyright (C) 2019 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#define TEST_MAIN ++#ifdef WIDE ++# define TEST_NAME "wmemcmp" ++#else ++# define TEST_NAME "memcmp" ++#endif ++ ++#include "test-size_t.h" ++ ++#ifdef WIDE ++# include ++# include ++ ++# define MEMCMP wmemcmp ++# define CHAR wchar_t ++#else ++# define MEMCMP memcmp ++# define CHAR char ++#endif ++ ++IMPL (MEMCMP, 1) ++ ++typedef int (*proto_t) (const CHAR *, const CHAR *, size_t); ++ ++static int ++__attribute__ ((noinline, noclone)) ++do_memcmp (parameter_t a, parameter_t b) ++{ ++ return CALL (&b, a.p, b.p, a.len); ++} ++ ++static int ++test_main (void) ++{ ++ test_init (); ++ ++ parameter_t dest = { { page_size / sizeof (CHAR) }, buf1 }; ++ parameter_t src = { { 0 }, buf2 }; ++ ++ memcpy (buf1, buf2, page_size); ++ ++ int ret = 0; ++ FOR_EACH_IMPL (impl, 0) ++ { ++ src.fn = impl->fn; ++ int res = do_memcmp (dest, src); ++ if (res) ++ { ++ error (0, 0, "Wrong result in function %s: %i != 0", ++ impl->name, res); ++ ret = 1; ++ } ++ } ++ ++ return ret ? EXIT_FAILURE : EXIT_SUCCESS; ++} ++ ++#include +diff --git a/sysdeps/x86_64/x32/tst-size_t-wmemcmp.c b/sysdeps/x86_64/x32/tst-size_t-wmemcmp.c +new file mode 100644 +index 00000000..e8b5ffd0 +--- /dev/null ++++ b/sysdeps/x86_64/x32/tst-size_t-wmemcmp.c +@@ -0,0 +1,20 @@ ++/* Test wmemcmp with size_t in the lower 32 bits of 64-bit register. ++ Copyright (C) 2019 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#define WIDE 1 ++#include "tst-size_t-memcmp.c" +-- +GitLab + diff --git a/SOURCES/glibc-sw24097-3.patch b/SOURCES/glibc-sw24097-3.patch new file mode 100644 index 0000000..45c3c04 --- /dev/null +++ b/SOURCES/glibc-sw24097-3.patch @@ -0,0 +1,393 @@ +From e6e209eb05545a36283335b07a1db9471cbccd1a Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Wed, 2 Mar 2022 13:56:00 -0800 +Subject: [PATCH] x86-64 memcpy: Properly handle the length parameter [BZ + #24097] + +On x32, the size_t parameter may be passed in the lower 32 bits of a +64-bit register with the non-zero upper 32 bits. The string/memory +functions written in assembly can only use the lower 32 bits of a +64-bit register as length or must clear the upper 32 bits before using +the full 64-bit register for length. + +This pach fixes memcpy for x32. Tested on x86-64 and x32. On x86-64, +libc.so is the same with and withou the fix. + + [BZ #24097] + CVE-2019-6488 + * sysdeps/x86_64/multiarch/memcpy-ssse3-back.S: Use RDX_LP for + length. Clear the upper 32 bits of RDX register. + * sysdeps/x86_64/multiarch/memcpy-ssse3.S: Likewise. + * sysdeps/x86_64/multiarch/memmove-avx512-no-vzeroupper.S: + Likewise. + * sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S: + Likewise. + * sysdeps/x86_64/x32/Makefile (tests): Add tst-size_t-memcpy. + tst-size_t-wmemchr. + * sysdeps/x86_64/x32/tst-size_t-memcpy.c: New file. + +(cherry picked from commit 231c56760c1e2ded21ad96bbb860b1f08c556c7a) +--- + sysdeps/x86_64/multiarch/memcpy-ssse3-back.S | 17 ++++-- + sysdeps/x86_64/multiarch/memcpy-ssse3.S | 17 ++++-- + .../multiarch/memmove-avx512-no-vzeroupper.S | 16 +++-- + .../multiarch/memmove-vec-unaligned-erms.S | 54 +++++++++-------- + sysdeps/x86_64/x32/Makefile | 2 +- + sysdeps/x86_64/x32/tst-size_t-memcpy.c | 58 +++++++++++++++++++ + 6 files changed, 122 insertions(+), 42 deletions(-) + create mode 100644 sysdeps/x86_64/x32/tst-size_t-memcpy.c + +diff --git a/sysdeps/x86_64/multiarch/memcpy-ssse3-back.S b/sysdeps/x86_64/multiarch/memcpy-ssse3-back.S +index 3cd11233..568eebd3 100644 +--- a/sysdeps/x86_64/multiarch/memcpy-ssse3-back.S ++++ b/sysdeps/x86_64/multiarch/memcpy-ssse3-back.S +@@ -45,28 +45,33 @@ + .section .text.ssse3,"ax",@progbits + #if !defined USE_AS_MEMPCPY && !defined USE_AS_MEMMOVE + ENTRY (MEMPCPY_CHK) +- cmpq %rdx, %rcx ++ cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) + END (MEMPCPY_CHK) + + ENTRY (MEMPCPY) +- movq %rdi, %rax +- addq %rdx, %rax ++ mov %RDI_LP, %RAX_LP ++ add %RDX_LP, %RAX_LP + jmp L(start) + END (MEMPCPY) + #endif + + #if !defined USE_AS_BCOPY + ENTRY (MEMCPY_CHK) +- cmpq %rdx, %rcx ++ cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) + END (MEMCPY_CHK) + #endif + + ENTRY (MEMCPY) +- mov %rdi, %rax ++ mov %RDI_LP, %RAX_LP + #ifdef USE_AS_MEMPCPY +- add %rdx, %rax ++ add %RDX_LP, %RAX_LP ++#endif ++ ++#ifdef __ILP32__ ++ /* Clear the upper 32 bits. */ ++ mov %edx, %edx + #endif + + #ifdef USE_AS_MEMMOVE +diff --git a/sysdeps/x86_64/multiarch/memcpy-ssse3.S b/sysdeps/x86_64/multiarch/memcpy-ssse3.S +index 0240bfa3..0bd5ee99 100644 +--- a/sysdeps/x86_64/multiarch/memcpy-ssse3.S ++++ b/sysdeps/x86_64/multiarch/memcpy-ssse3.S +@@ -45,28 +45,33 @@ + .section .text.ssse3,"ax",@progbits + #if !defined USE_AS_MEMPCPY && !defined USE_AS_MEMMOVE + ENTRY (MEMPCPY_CHK) +- cmpq %rdx, %rcx ++ cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) + END (MEMPCPY_CHK) + + ENTRY (MEMPCPY) +- movq %rdi, %rax +- addq %rdx, %rax ++ mov %RDI_LP, %RAX_LP ++ add %RDX_LP, %RAX_LP + jmp L(start) + END (MEMPCPY) + #endif + + #if !defined USE_AS_BCOPY + ENTRY (MEMCPY_CHK) +- cmpq %rdx, %rcx ++ cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) + END (MEMCPY_CHK) + #endif + + ENTRY (MEMCPY) +- mov %rdi, %rax ++ mov %RDI_LP, %RAX_LP + #ifdef USE_AS_MEMPCPY +- add %rdx, %rax ++ add %RDX_LP, %RAX_LP ++#endif ++ ++#ifdef __ILP32__ ++ /* Clear the upper 32 bits. */ ++ mov %edx, %edx + #endif + + #ifdef USE_AS_MEMMOVE +diff --git a/sysdeps/x86_64/multiarch/memmove-avx512-no-vzeroupper.S b/sysdeps/x86_64/multiarch/memmove-avx512-no-vzeroupper.S +index effc3ac2..6ca2bbc9 100644 +--- a/sysdeps/x86_64/multiarch/memmove-avx512-no-vzeroupper.S ++++ b/sysdeps/x86_64/multiarch/memmove-avx512-no-vzeroupper.S +@@ -24,27 +24,31 @@ + + .section .text.avx512,"ax",@progbits + ENTRY (__mempcpy_chk_avx512_no_vzeroupper) +- cmpq %rdx, %rcx ++ cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) + END (__mempcpy_chk_avx512_no_vzeroupper) + + ENTRY (__mempcpy_avx512_no_vzeroupper) +- movq %rdi, %rax +- addq %rdx, %rax ++ mov %RDI_LP, %RAX_LP ++ add %RDX_LP, %RAX_LP + jmp L(start) + END (__mempcpy_avx512_no_vzeroupper) + + ENTRY (__memmove_chk_avx512_no_vzeroupper) +- cmpq %rdx, %rcx ++ cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) + END (__memmove_chk_avx512_no_vzeroupper) + + ENTRY (__memmove_avx512_no_vzeroupper) +- mov %rdi, %rax ++ mov %RDI_LP, %RAX_LP + # ifdef USE_AS_MEMPCPY +- add %rdx, %rax ++ add %RDX_LP, %RAX_LP + # endif + L(start): ++# ifdef __ILP32__ ++ /* Clear the upper 32 bits. */ ++ mov %edx, %edx ++# endif + lea (%rsi, %rdx), %rcx + lea (%rdi, %rdx), %r9 + cmp $512, %rdx +diff --git a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S +index c952576c..274aa1c7 100644 +--- a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S ++++ b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S +@@ -95,20 +95,20 @@ + .section SECTION(.text),"ax",@progbits + #if defined SHARED && IS_IN (libc) + ENTRY (MEMMOVE_CHK_SYMBOL (__mempcpy_chk, unaligned)) +- cmpq %rdx, %rcx ++ cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) + END (MEMMOVE_CHK_SYMBOL (__mempcpy_chk, unaligned)) + #endif + + ENTRY (MEMPCPY_SYMBOL (__mempcpy, unaligned)) +- movq %rdi, %rax +- addq %rdx, %rax ++ mov %RDI_LP, %RAX_LP ++ add %RDX_LP, %RAX_LP + jmp L(start) + END (MEMPCPY_SYMBOL (__mempcpy, unaligned)) + + #if defined SHARED && IS_IN (libc) + ENTRY (MEMMOVE_CHK_SYMBOL (__memmove_chk, unaligned)) +- cmpq %rdx, %rcx ++ cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) + END (MEMMOVE_CHK_SYMBOL (__memmove_chk, unaligned)) + #endif +@@ -116,9 +116,13 @@ END (MEMMOVE_CHK_SYMBOL (__memmove_chk, unaligned)) + ENTRY (MEMMOVE_SYMBOL (__memmove, unaligned)) + movq %rdi, %rax + L(start): +- cmpq $VEC_SIZE, %rdx ++# ifdef __ILP32__ ++ /* Clear the upper 32 bits. */ ++ movl %edx, %edx ++# endif ++ cmp $VEC_SIZE, %RDX_LP + jb L(less_vec) +- cmpq $(VEC_SIZE * 2), %rdx ++ cmp $(VEC_SIZE * 2), %RDX_LP + ja L(more_2x_vec) + #if !defined USE_MULTIARCH || !IS_IN (libc) + L(last_2x_vec): +@@ -138,38 +142,38 @@ END (MEMMOVE_SYMBOL (__memmove, unaligned)) + + # if VEC_SIZE == 16 + ENTRY (__mempcpy_chk_erms) +- cmpq %rdx, %rcx ++ cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) + END (__mempcpy_chk_erms) + + /* Only used to measure performance of REP MOVSB. */ + ENTRY (__mempcpy_erms) +- movq %rdi, %rax ++ mov %RDI_LP, %RAX_LP + /* Skip zero length. */ +- testq %rdx, %rdx ++ test %RDX_LP, %RDX_LP + jz 2f +- addq %rdx, %rax ++ add %RDX_LP, %RAX_LP + jmp L(start_movsb) + END (__mempcpy_erms) + + ENTRY (__memmove_chk_erms) +- cmpq %rdx, %rcx ++ cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) + END (__memmove_chk_erms) + + ENTRY (__memmove_erms) + movq %rdi, %rax + /* Skip zero length. */ +- testq %rdx, %rdx ++ test %RDX_LP, %RDX_LP + jz 2f + L(start_movsb): +- movq %rdx, %rcx +- cmpq %rsi, %rdi ++ mov %RDX_LP, %RCX_LP ++ cmp %RSI_LP, %RDI_LP + jb 1f + /* Source == destination is less common. */ + je 2f +- leaq (%rsi,%rcx), %rdx +- cmpq %rdx, %rdi ++ lea (%rsi,%rcx), %RDX_LP ++ cmp %RDX_LP, %RDI_LP + jb L(movsb_backward) + 1: + rep movsb +@@ -189,20 +193,20 @@ strong_alias (__memmove_chk_erms, __memcpy_chk_erms) + + # ifdef SHARED + ENTRY (MEMMOVE_CHK_SYMBOL (__mempcpy_chk, unaligned_erms)) +- cmpq %rdx, %rcx ++ cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) + END (MEMMOVE_CHK_SYMBOL (__mempcpy_chk, unaligned_erms)) + # endif + + ENTRY (MEMMOVE_SYMBOL (__mempcpy, unaligned_erms)) +- movq %rdi, %rax +- addq %rdx, %rax ++ mov %RDI_LP, %RAX_LP ++ add %RDX_LP, %RAX_LP + jmp L(start_erms) + END (MEMMOVE_SYMBOL (__mempcpy, unaligned_erms)) + + # ifdef SHARED + ENTRY (MEMMOVE_CHK_SYMBOL (__memmove_chk, unaligned_erms)) +- cmpq %rdx, %rcx ++ cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) + END (MEMMOVE_CHK_SYMBOL (__memmove_chk, unaligned_erms)) + # endif +@@ -210,9 +214,13 @@ END (MEMMOVE_CHK_SYMBOL (__memmove_chk, unaligned_erms)) + ENTRY (MEMMOVE_SYMBOL (__memmove, unaligned_erms)) + movq %rdi, %rax + L(start_erms): +- cmpq $VEC_SIZE, %rdx ++# ifdef __ILP32__ ++ /* Clear the upper 32 bits. */ ++ movl %edx, %edx ++# endif ++ cmp $VEC_SIZE, %RDX_LP + jb L(less_vec) +- cmpq $(VEC_SIZE * 2), %rdx ++ cmp $(VEC_SIZE * 2), %RDX_LP + ja L(movsb_more_2x_vec) + L(last_2x_vec): + /* From VEC and to 2 * VEC. No branch when size == VEC_SIZE. */ +@@ -236,7 +244,7 @@ L(movsb): + /* Avoid slow backward REP MOVSB. */ + jb L(more_8x_vec_backward) + 1: +- movq %rdx, %rcx ++ mov %RDX_LP, %RCX_LP + rep movsb + L(nop): + ret +diff --git a/sysdeps/x86_64/x32/Makefile b/sysdeps/x86_64/x32/Makefile +index ddec7f04..2fe1e5ac 100644 +--- a/sysdeps/x86_64/x32/Makefile ++++ b/sysdeps/x86_64/x32/Makefile +@@ -6,7 +6,7 @@ CFLAGS-s_llround.c += -fno-builtin-lround + endif + + ifeq ($(subdir),string) +-tests += tst-size_t-memchr tst-size_t-memcmp ++tests += tst-size_t-memchr tst-size_t-memcmp tst-size_t-memcpy + endif + + ifeq ($(subdir),wcsmbs) +diff --git a/sysdeps/x86_64/x32/tst-size_t-memcpy.c b/sysdeps/x86_64/x32/tst-size_t-memcpy.c +new file mode 100644 +index 00000000..66b71e17 +--- /dev/null ++++ b/sysdeps/x86_64/x32/tst-size_t-memcpy.c +@@ -0,0 +1,58 @@ ++/* Test memcpy with size_t in the lower 32 bits of 64-bit register. ++ Copyright (C) 2019 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#define TEST_NAME "memcpy" ++#include "test-size_t.h" ++ ++IMPL (memcpy, 1) ++ ++typedef void *(*proto_t) (void *, const void *, size_t); ++ ++static void * ++__attribute__ ((noinline, noclone)) ++do_memcpy (parameter_t a, parameter_t b) ++{ ++ return CALL (&b, a.p, b.p, a.len); ++} ++ ++static int ++test_main (void) ++{ ++ test_init (); ++ ++ parameter_t dest = { { page_size }, buf1 }; ++ parameter_t src = { { 0 }, buf2 }; ++ ++ int ret = 0; ++ FOR_EACH_IMPL (impl, 0) ++ { ++ src.fn = impl->fn; ++ do_memcpy (dest, src); ++ int res = memcmp (dest.p, src.p, dest.len); ++ if (res) ++ { ++ error (0, 0, "Wrong result in function %s: %i != 0", ++ impl->name, res); ++ ret = 1; ++ } ++ } ++ ++ return ret ? EXIT_FAILURE : EXIT_SUCCESS; ++} ++ ++#include +-- +GitLab + diff --git a/SOURCES/glibc-sw24097-4.patch b/SOURCES/glibc-sw24097-4.patch new file mode 100644 index 0000000..ed6ef9e --- /dev/null +++ b/SOURCES/glibc-sw24097-4.patch @@ -0,0 +1,148 @@ +From c62e1cf68a70de8ac4cc8aebae760784009777c5 Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Fri, 1 Feb 2019 12:20:54 -0800 +Subject: [PATCH] x86-64 memrchr: Properly handle the length parameter [BZ + #24097] + +On x32, the size_t parameter may be passed in the lower 32 bits of a +64-bit register with the non-zero upper 32 bits. The string/memory +functions written in assembly can only use the lower 32 bits of a +64-bit register as length or must clear the upper 32 bits before using +the full 64-bit register for length. + +This pach fixes memrchr for x32. Tested on x86-64 and x32. On x86-64, +libc.so is the same with and withou the fix. + + [BZ #24097] + CVE-2019-6488 + * sysdeps/x86_64/memrchr.S: Use RDX_LP for length. + * sysdeps/x86_64/multiarch/memrchr-avx2.S: Likewise. + * sysdeps/x86_64/x32/Makefile (tests): Add tst-size_t-memrchr. + * sysdeps/x86_64/x32/tst-size_t-memrchr.c: New file. + +(cherry picked from commit ecd8b842cf37ea112e59cd9085ff1f1b6e208ae0) +--- + sysdeps/x86_64/memrchr.S | 4 +- + sysdeps/x86_64/multiarch/memrchr-avx2.S | 4 +- + sysdeps/x86_64/x32/Makefile | 3 +- + sysdeps/x86_64/x32/tst-size_t-memrchr.c | 57 +++++++++++++++++++++++++ + 4 files changed, 63 insertions(+), 5 deletions(-) + create mode 100644 sysdeps/x86_64/x32/tst-size_t-memrchr.c + +diff --git a/sysdeps/x86_64/memrchr.S b/sysdeps/x86_64/memrchr.S +index b8e3fa1d..dc82f8f7 100644 +--- a/sysdeps/x86_64/memrchr.S ++++ b/sysdeps/x86_64/memrchr.S +@@ -24,13 +24,13 @@ + ENTRY (__memrchr) + movd %esi, %xmm1 + +- sub $16, %rdx ++ sub $16, %RDX_LP + jbe L(length_less16) + + punpcklbw %xmm1, %xmm1 + punpcklbw %xmm1, %xmm1 + +- add %rdx, %rdi ++ add %RDX_LP, %RDI_LP + pshufd $0, %xmm1, %xmm1 + + movdqu (%rdi), %xmm0 +diff --git a/sysdeps/x86_64/multiarch/memrchr-avx2.S b/sysdeps/x86_64/multiarch/memrchr-avx2.S +index b41a58bc..ce488dd9 100644 +--- a/sysdeps/x86_64/multiarch/memrchr-avx2.S ++++ b/sysdeps/x86_64/multiarch/memrchr-avx2.S +@@ -32,10 +32,10 @@ ENTRY (__memrchr_avx2) + vmovd %esi, %xmm0 + vpbroadcastb %xmm0, %ymm0 + +- subq $VEC_SIZE, %rdx ++ sub $VEC_SIZE, %RDX_LP + jbe L(last_vec_or_less) + +- addq %rdx, %rdi ++ add %RDX_LP, %RDI_LP + + /* Check the last VEC_SIZE bytes. */ + vpcmpeqb (%rdi), %ymm0, %ymm1 +diff --git a/sysdeps/x86_64/x32/Makefile b/sysdeps/x86_64/x32/Makefile +index 2fe1e5ac..e99dbd7c 100644 +--- a/sysdeps/x86_64/x32/Makefile ++++ b/sysdeps/x86_64/x32/Makefile +@@ -6,7 +6,8 @@ CFLAGS-s_llround.c += -fno-builtin-lround + endif + + ifeq ($(subdir),string) +-tests += tst-size_t-memchr tst-size_t-memcmp tst-size_t-memcpy ++tests += tst-size_t-memchr tst-size_t-memcmp tst-size_t-memcpy \ ++ tst-size_t-memrchr + endif + + ifeq ($(subdir),wcsmbs) +diff --git a/sysdeps/x86_64/x32/tst-size_t-memrchr.c b/sysdeps/x86_64/x32/tst-size_t-memrchr.c +new file mode 100644 +index 00000000..c83699c0 +--- /dev/null ++++ b/sysdeps/x86_64/x32/tst-size_t-memrchr.c +@@ -0,0 +1,57 @@ ++/* Test memrchr with size_t in the lower 32 bits of 64-bit register. ++ Copyright (C) 2019 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#define TEST_NAME "memrchr" ++#include "test-size_t.h" ++ ++IMPL (memchr, 1) ++ ++typedef void * (*proto_t) (const void *, int, size_t); ++ ++static void * ++__attribute__ ((noinline, noclone)) ++do_memrchr (parameter_t a, parameter_t b) ++{ ++ return CALL (&b, a.p, (uintptr_t) b.p, a.len); ++} ++ ++static int ++test_main (void) ++{ ++ test_init (); ++ ++ parameter_t src = { { page_size }, buf2 }; ++ parameter_t c = { { 0 }, (void *) (uintptr_t) 0x12 }; ++ ++ int ret = 0; ++ FOR_EACH_IMPL (impl, 0) ++ { ++ c.fn = impl->fn; ++ void * res = do_memrchr (src, c); ++ if (res) ++ { ++ error (0, 0, "Wrong result in function %s: %p != NULL", ++ impl->name, res); ++ ret = 1; ++ } ++ } ++ ++ return ret ? EXIT_FAILURE : EXIT_SUCCESS; ++} ++ ++#include +-- +GitLab + diff --git a/SOURCES/glibc-sw24097-5.patch b/SOURCES/glibc-sw24097-5.patch new file mode 100644 index 0000000..8c51514 --- /dev/null +++ b/SOURCES/glibc-sw24097-5.patch @@ -0,0 +1,287 @@ +From bd6061191c63fb24025f3bb4cfc88519024f2b7a Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Wed, 2 Mar 2022 14:03:51 -0800 +Subject: [PATCH] x86-64 memset/wmemset: Properly handle the length parameter + [BZ #24097] + +On x32, the size_t parameter may be passed in the lower 32 bits of a +64-bit register with the non-zero upper 32 bits. The string/memory +functions written in assembly can only use the lower 32 bits of a +64-bit register as length or must clear the upper 32 bits before using +the full 64-bit register for length. + +This pach fixes memset/wmemset for x32. Tested on x86-64 and x32. On +x86-64, libc.so is the same with and withou the fix. + + [BZ #24097] + CVE-2019-6488 + * sysdeps/x86_64/multiarch/memset-avx512-no-vzeroupper.S: Use + RDX_LP for length. Clear the upper 32 bits of RDX register. + * sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S: Likewise. + * sysdeps/x86_64/x32/Makefile (tests): Add tst-size_t-wmemset. + * sysdeps/x86_64/x32/tst-size_t-memset.c: New file. + * sysdeps/x86_64/x32/tst-size_t-wmemset.c: Likewise. + +(cherry picked from commit 82d0b4a4d76db554eb6757acb790fcea30b19965) +--- + .../multiarch/memset-avx512-no-vzeroupper.S | 6 +- + .../multiarch/memset-vec-unaligned-erms.S | 34 +++++---- + sysdeps/x86_64/x32/Makefile | 4 +- + sysdeps/x86_64/x32/tst-size_t-memset.c | 73 +++++++++++++++++++ + sysdeps/x86_64/x32/tst-size_t-wmemset.c | 20 +++++ + 5 files changed, 121 insertions(+), 16 deletions(-) + create mode 100644 sysdeps/x86_64/x32/tst-size_t-memset.c + create mode 100644 sysdeps/x86_64/x32/tst-size_t-wmemset.c + +diff --git a/sysdeps/x86_64/multiarch/memset-avx512-no-vzeroupper.S b/sysdeps/x86_64/multiarch/memset-avx512-no-vzeroupper.S +index 689cc119..99e25519 100644 +--- a/sysdeps/x86_64/multiarch/memset-avx512-no-vzeroupper.S ++++ b/sysdeps/x86_64/multiarch/memset-avx512-no-vzeroupper.S +@@ -29,12 +29,16 @@ + .section .text.avx512,"ax",@progbits + #if defined PIC + ENTRY (MEMSET_CHK) +- cmpq %rdx, %rcx ++ cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) + END (MEMSET_CHK) + #endif + + ENTRY (MEMSET) ++# ifdef __ILP32__ ++ /* Clear the upper 32 bits. */ ++ mov %edx, %edx ++# endif + vpxor %xmm0, %xmm0, %xmm0 + vmovd %esi, %xmm1 + lea (%rdi, %rdx), %rsi +diff --git a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S +index 270a1d49..9a0fd818 100644 +--- a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S ++++ b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S +@@ -65,8 +65,8 @@ + .section SECTION(.text),"ax",@progbits + #if VEC_SIZE == 16 && IS_IN (libc) + ENTRY (__bzero) +- movq %rdi, %rax /* Set return value. */ +- movq %rsi, %rdx /* Set n. */ ++ mov %RDI_LP, %RAX_LP /* Set return value. */ ++ mov %RSI_LP, %RDX_LP /* Set n. */ + pxor %xmm0, %xmm0 + jmp L(entry_from_bzero) + END (__bzero) +@@ -76,13 +76,13 @@ weak_alias (__bzero, bzero) + #if IS_IN (libc) + # if defined SHARED + ENTRY_CHK (WMEMSET_CHK_SYMBOL (__wmemset_chk, unaligned)) +- cmpq %rdx, %rcx ++ cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) + END_CHK (WMEMSET_CHK_SYMBOL (__wmemset_chk, unaligned)) + # endif + + ENTRY (WMEMSET_SYMBOL (__wmemset, unaligned)) +- shlq $2, %rdx ++ shl $2, %RDX_LP + WMEMSET_VDUP_TO_VEC0_AND_SET_RETURN (%esi, %rdi) + jmp L(entry_from_bzero) + END (WMEMSET_SYMBOL (__wmemset, unaligned)) +@@ -90,13 +90,17 @@ END (WMEMSET_SYMBOL (__wmemset, unaligned)) + + #if defined SHARED && IS_IN (libc) + ENTRY_CHK (MEMSET_CHK_SYMBOL (__memset_chk, unaligned)) +- cmpq %rdx, %rcx ++ cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) + END_CHK (MEMSET_CHK_SYMBOL (__memset_chk, unaligned)) + #endif + + ENTRY (MEMSET_SYMBOL (__memset, unaligned)) + MEMSET_VDUP_TO_VEC0_AND_SET_RETURN (%esi, %rdi) ++# ifdef __ILP32__ ++ /* Clear the upper 32 bits. */ ++ mov %edx, %edx ++# endif + L(entry_from_bzero): + cmpq $VEC_SIZE, %rdx + jb L(less_vec) +@@ -112,14 +116,14 @@ END (MEMSET_SYMBOL (__memset, unaligned)) + + # if VEC_SIZE == 16 + ENTRY (__memset_chk_erms) +- cmpq %rdx, %rcx ++ cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) + END (__memset_chk_erms) + + /* Only used to measure performance of REP STOSB. */ + ENTRY (__memset_erms) + /* Skip zero length. */ +- testq %rdx, %rdx ++ test %RDX_LP, %RDX_LP + jnz L(stosb) + movq %rdi, %rax + ret +@@ -131,11 +135,11 @@ ENTRY (MEMSET_SYMBOL (__memset, erms)) + L(stosb): + /* Issue vzeroupper before rep stosb. */ + VZEROUPPER +- movq %rdx, %rcx ++ mov %RDX_LP, %RCX_LP + movzbl %sil, %eax +- movq %rdi, %rdx ++ mov %RDI_LP, %RDX_LP + rep stosb +- movq %rdx, %rax ++ mov %RDX_LP, %RAX_LP + ret + # if VEC_SIZE == 16 + END (__memset_erms) +@@ -145,16 +149,20 @@ END (MEMSET_SYMBOL (__memset, erms)) + + # if defined SHARED && IS_IN (libc) + ENTRY_CHK (MEMSET_CHK_SYMBOL (__memset_chk, unaligned_erms)) +- cmpq %rdx, %rcx ++ cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) + END_CHK (MEMSET_CHK_SYMBOL (__memset_chk, unaligned_erms)) + # endif + + ENTRY (MEMSET_SYMBOL (__memset, unaligned_erms)) + MEMSET_VDUP_TO_VEC0_AND_SET_RETURN (%esi, %rdi) +- cmpq $VEC_SIZE, %rdx ++# ifdef __ILP32__ ++ /* Clear the upper 32 bits. */ ++ mov %edx, %edx ++# endif ++ cmp $VEC_SIZE, %RDX_LP + jb L(less_vec) +- cmpq $(VEC_SIZE * 2), %rdx ++ cmp $(VEC_SIZE * 2), %RDX_LP + ja L(stosb_more_2x_vec) + /* From VEC and to 2 * VEC. No branch when size == VEC_SIZE. */ + VMOVU %VEC(0), -VEC_SIZE(%rdi,%rdx) +diff --git a/sysdeps/x86_64/x32/Makefile b/sysdeps/x86_64/x32/Makefile +index e99dbd7c..98bd9ae9 100644 +--- a/sysdeps/x86_64/x32/Makefile ++++ b/sysdeps/x86_64/x32/Makefile +@@ -7,9 +7,9 @@ endif + + ifeq ($(subdir),string) + tests += tst-size_t-memchr tst-size_t-memcmp tst-size_t-memcpy \ +- tst-size_t-memrchr ++ tst-size_t-memrchr tst-size_t-memset + endif + + ifeq ($(subdir),wcsmbs) +-tests += tst-size_t-wmemchr tst-size_t-wmemcmp ++tests += tst-size_t-wmemchr tst-size_t-wmemcmp tst-size_t-wmemset + endif +diff --git a/sysdeps/x86_64/x32/tst-size_t-memset.c b/sysdeps/x86_64/x32/tst-size_t-memset.c +new file mode 100644 +index 00000000..2c367af6 +--- /dev/null ++++ b/sysdeps/x86_64/x32/tst-size_t-memset.c +@@ -0,0 +1,73 @@ ++/* Test memset with size_t in the lower 32 bits of 64-bit register. ++ Copyright (C) 2019 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#ifdef WIDE ++# define TEST_NAME "wmemset" ++#else ++# define TEST_NAME "memset" ++#endif /* WIDE */ ++ ++#include "test-size_t.h" ++ ++#ifdef WIDE ++# include ++# define MEMSET wmemset ++# define CHAR wchar_t ++#else ++# define MEMSET memset ++# define CHAR char ++#endif /* WIDE */ ++ ++IMPL (MEMSET, 1) ++ ++typedef CHAR *(*proto_t) (CHAR *, int, size_t); ++ ++static void * ++__attribute__ ((noinline, noclone)) ++do_memset (parameter_t a, parameter_t b) ++{ ++ return CALL (&b, a.p, (uintptr_t) b.p, a.len); ++} ++ ++static int ++test_main (void) ++{ ++ test_init (); ++ ++ CHAR ch = 0x23; ++ parameter_t src = { { page_size / sizeof (CHAR) }, buf2 }; ++ parameter_t c = { { 0 }, (void *) (uintptr_t) ch }; ++ ++ int ret = 0; ++ FOR_EACH_IMPL (impl, 0) ++ { ++ c.fn = impl->fn; ++ CHAR *p = (CHAR *) do_memset (src, c); ++ size_t i; ++ for (i = 0; i < src.len; i++) ++ if (p[i] != ch) ++ { ++ error (0, 0, "Wrong result in function %s", impl->name); ++ ret = 1; ++ } ++ } ++ ++ return ret ? EXIT_FAILURE : EXIT_SUCCESS; ++} ++ ++#include +diff --git a/sysdeps/x86_64/x32/tst-size_t-wmemset.c b/sysdeps/x86_64/x32/tst-size_t-wmemset.c +new file mode 100644 +index 00000000..955eb488 +--- /dev/null ++++ b/sysdeps/x86_64/x32/tst-size_t-wmemset.c +@@ -0,0 +1,20 @@ ++/* Test wmemset with size_t in the lower 32 bits of 64-bit register. ++ Copyright (C) 2019 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#define WIDE 1 ++#include "tst-size_t-memset.c" +-- +GitLab + diff --git a/SOURCES/glibc-sw24097-6.patch b/SOURCES/glibc-sw24097-6.patch new file mode 100644 index 0000000..6884642 --- /dev/null +++ b/SOURCES/glibc-sw24097-6.patch @@ -0,0 +1,297 @@ +From 323c698927f67a153e278cc99870998a9c7c87af Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Fri, 1 Feb 2019 12:22:33 -0800 +Subject: [PATCH] x86-64 strncmp family: Properly handle the length parameter + [BZ #24097] + +On x32, the size_t parameter may be passed in the lower 32 bits of a +64-bit register with the non-zero upper 32 bits. The string/memory +functions written in assembly can only use the lower 32 bits of a +64-bit register as length or must clear the upper 32 bits before using +the full 64-bit register for length. + +This pach fixes the strncmp family for x32. Tested on x86-64 and x32. +On x86-64, libc.so is the same with and withou the fix. + + [BZ #24097] + CVE-2019-6488 + * sysdeps/x86_64/multiarch/strcmp-avx2.S: Use RDX_LP for length. + * sysdeps/x86_64/multiarch/strcmp-sse42.S: Likewise. + * sysdeps/x86_64/strcmp.S: Likewise. + * sysdeps/x86_64/x32/Makefile (tests): Add tst-size_t-strncasecmp, + tst-size_t-strncmp and tst-size_t-wcsncmp. + * sysdeps/x86_64/x32/tst-size_t-strncasecmp.c: New file. + * sysdeps/x86_64/x32/tst-size_t-strncmp.c: Likewise. + * sysdeps/x86_64/x32/tst-size_t-wcsncmp.c: Likewise. + +(cherry picked from commit ee915088a0231cd421054dbd8abab7aadf331153) +--- + sysdeps/x86_64/multiarch/strcmp-avx2.S | 6 +- + sysdeps/x86_64/multiarch/strcmp-sse42.S | 6 +- + sysdeps/x86_64/strcmp.S | 6 +- + sysdeps/x86_64/x32/Makefile | 6 +- + sysdeps/x86_64/x32/tst-size_t-strncasecmp.c | 59 ++++++++++++++++ + sysdeps/x86_64/x32/tst-size_t-strncmp.c | 78 +++++++++++++++++++++ + sysdeps/x86_64/x32/tst-size_t-wcsncmp.c | 20 ++++++ + 7 files changed, 170 insertions(+), 11 deletions(-) + create mode 100644 sysdeps/x86_64/x32/tst-size_t-strncasecmp.c + create mode 100644 sysdeps/x86_64/x32/tst-size_t-strncmp.c + create mode 100644 sysdeps/x86_64/x32/tst-size_t-wcsncmp.c + +diff --git a/sysdeps/x86_64/multiarch/strcmp-avx2.S b/sysdeps/x86_64/multiarch/strcmp-avx2.S +index 327e3d87..156c1949 100644 +--- a/sysdeps/x86_64/multiarch/strcmp-avx2.S ++++ b/sysdeps/x86_64/multiarch/strcmp-avx2.S +@@ -79,15 +79,15 @@ + ENTRY (STRCMP) + # ifdef USE_AS_STRNCMP + /* Check for simple cases (0 or 1) in offset. */ +- cmp $1, %rdx ++ cmp $1, %RDX_LP + je L(char0) + jb L(zero) + # ifdef USE_AS_WCSCMP + /* Convert units: from wide to byte char. */ +- shl $2, %rdx ++ shl $2, %RDX_LP + # endif + /* Register %r11 tracks the maximum offset. */ +- movq %rdx, %r11 ++ mov %RDX_LP, %R11_LP + # endif + movl %edi, %eax + xorl %edx, %edx +diff --git a/sysdeps/x86_64/multiarch/strcmp-sse42.S b/sysdeps/x86_64/multiarch/strcmp-sse42.S +index d3c07bd2..a1ebea46 100644 +--- a/sysdeps/x86_64/multiarch/strcmp-sse42.S ++++ b/sysdeps/x86_64/multiarch/strcmp-sse42.S +@@ -156,11 +156,11 @@ STRCMP_SSE42: + #endif + + #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L +- test %rdx, %rdx ++ test %RDX_LP, %RDX_LP + je LABEL(strcmp_exitz) +- cmp $1, %rdx ++ cmp $1, %RDX_LP + je LABEL(Byte0) +- mov %rdx, %r11 ++ mov %RDX_LP, %R11_LP + #endif + mov %esi, %ecx + mov %edi, %eax +diff --git a/sysdeps/x86_64/strcmp.S b/sysdeps/x86_64/strcmp.S +index e16945b9..f47c8ad4 100644 +--- a/sysdeps/x86_64/strcmp.S ++++ b/sysdeps/x86_64/strcmp.S +@@ -135,11 +135,11 @@ ENTRY (STRCMP) + * This implementation uses SSE to compare up to 16 bytes at a time. + */ + #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L +- test %rdx, %rdx ++ test %RDX_LP, %RDX_LP + je LABEL(strcmp_exitz) +- cmp $1, %rdx ++ cmp $1, %RDX_LP + je LABEL(Byte0) +- mov %rdx, %r11 ++ mov %RDX_LP, %R11_LP + #endif + mov %esi, %ecx + mov %edi, %eax +diff --git a/sysdeps/x86_64/x32/Makefile b/sysdeps/x86_64/x32/Makefile +index 98bd9ae9..db302839 100644 +--- a/sysdeps/x86_64/x32/Makefile ++++ b/sysdeps/x86_64/x32/Makefile +@@ -7,9 +7,11 @@ endif + + ifeq ($(subdir),string) + tests += tst-size_t-memchr tst-size_t-memcmp tst-size_t-memcpy \ +- tst-size_t-memrchr tst-size_t-memset ++ tst-size_t-memrchr tst-size_t-memset tst-size_t-strncasecmp \ ++ tst-size_t-strncmp + endif + + ifeq ($(subdir),wcsmbs) +-tests += tst-size_t-wmemchr tst-size_t-wmemcmp tst-size_t-wmemset ++tests += tst-size_t-wmemchr tst-size_t-wmemcmp tst-size_t-wmemset \ ++ tst-size_t-wcsncmp + endif +diff --git a/sysdeps/x86_64/x32/tst-size_t-strncasecmp.c b/sysdeps/x86_64/x32/tst-size_t-strncasecmp.c +new file mode 100644 +index 00000000..86233593 +--- /dev/null ++++ b/sysdeps/x86_64/x32/tst-size_t-strncasecmp.c +@@ -0,0 +1,59 @@ ++/* Test strncaecmp with size_t in the lower 32 bits of 64-bit register. ++ Copyright (C) 2019 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#define TEST_NAME "strncasecmp" ++#include "test-size_t.h" ++ ++IMPL (strncasecmp, 1) ++ ++typedef int (*proto_t) (const char *, const char *, size_t); ++ ++static int ++__attribute__ ((noinline, noclone)) ++do_strncasecmp (parameter_t a, parameter_t b) ++{ ++ return CALL (&b, a.p, b.p, a.len); ++} ++ ++static int ++test_main (void) ++{ ++ test_init (); ++ ++ parameter_t dest = { { page_size }, buf1 }; ++ parameter_t src = { { 0 }, buf2 }; ++ ++ strncpy ((char *) buf1, (const char *) buf2, page_size); ++ ++ int ret = 0; ++ FOR_EACH_IMPL (impl, 0) ++ { ++ src.fn = impl->fn; ++ int res = do_strncasecmp (dest, src); ++ if (res) ++ { ++ error (0, 0, "Wrong result in function %s: %i != 0", ++ impl->name, res); ++ ret = 1; ++ } ++ } ++ ++ return ret ? EXIT_FAILURE : EXIT_SUCCESS; ++} ++ ++#include +diff --git a/sysdeps/x86_64/x32/tst-size_t-strncmp.c b/sysdeps/x86_64/x32/tst-size_t-strncmp.c +new file mode 100644 +index 00000000..54e6bd83 +--- /dev/null ++++ b/sysdeps/x86_64/x32/tst-size_t-strncmp.c +@@ -0,0 +1,78 @@ ++/* Test strncmp with size_t in the lower 32 bits of 64-bit register. ++ Copyright (C) 2019 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#ifdef WIDE ++# define TEST_NAME "wcsncmp" ++#else ++# define TEST_NAME "strncmp" ++#endif ++ ++#include "test-size_t.h" ++ ++#ifdef WIDE ++# include ++ ++# define STRNCMP wcsncmp ++# define STRNCPY wcsncpy ++# define CHAR wchar_t ++#else ++# define STRNCMP strncmp ++# define STRNCPY strncpy ++# define CHAR char ++#endif ++ ++IMPL (STRNCMP, 1) ++ ++typedef int (*proto_t) (const CHAR *, const CHAR *, size_t); ++ ++ ++static int ++__attribute__ ((noinline, noclone)) ++do_strncmp (parameter_t a, parameter_t b) ++{ ++ return CALL (&b, a.p, b.p, a.len); ++} ++ ++static int ++test_main (void) ++{ ++ test_init (); ++ ++ size_t size = page_size / sizeof (CHAR); ++ parameter_t dest = { { size }, buf1 }; ++ parameter_t src = { { 0 }, buf2 }; ++ ++ STRNCPY ((CHAR *) buf1, (const CHAR *) buf2, size); ++ ++ int ret = 0; ++ FOR_EACH_IMPL (impl, 0) ++ { ++ src.fn = impl->fn; ++ int res = do_strncmp (dest, src); ++ if (res) ++ { ++ error (0, 0, "Wrong result in function %s: %i != 0", ++ impl->name, res); ++ ret = 1; ++ } ++ } ++ ++ return ret ? EXIT_FAILURE : EXIT_SUCCESS; ++} ++ ++#include +diff --git a/sysdeps/x86_64/x32/tst-size_t-wcsncmp.c b/sysdeps/x86_64/x32/tst-size_t-wcsncmp.c +new file mode 100644 +index 00000000..4829647c +--- /dev/null ++++ b/sysdeps/x86_64/x32/tst-size_t-wcsncmp.c +@@ -0,0 +1,20 @@ ++/* Test wcsncmp with size_t in the lower 32 bits of 64-bit register. ++ Copyright (C) 2019 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#define WIDE 1 ++#include "tst-size_t-strncmp.c" +-- +GitLab + diff --git a/SOURCES/glibc-sw24097-7.patch b/SOURCES/glibc-sw24097-7.patch new file mode 100644 index 0000000..2cff09e --- /dev/null +++ b/SOURCES/glibc-sw24097-7.patch @@ -0,0 +1,148 @@ +From 4f97e8708512462e87d2f1f1bdeb8cf55ce10e5a Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Fri, 1 Feb 2019 12:23:23 -0800 +Subject: [PATCH] x86-64 strncpy: Properly handle the length parameter [BZ + #24097] + +On x32, the size_t parameter may be passed in the lower 32 bits of a +64-bit register with the non-zero upper 32 bits. The string/memory +functions written in assembly can only use the lower 32 bits of a +64-bit register as length or must clear the upper 32 bits before using +the full 64-bit register for length. + +This pach fixes strncpy for x32. Tested on x86-64 and x32. On x86-64, +libc.so is the same with and withou the fix. + + [BZ #24097] + CVE-2019-6488 + * sysdeps/x86_64/multiarch/strcpy-sse2-unaligned.S: Use RDX_LP + for length. + * sysdeps/x86_64/multiarch/strcpy-ssse3.S: Likewise. + * sysdeps/x86_64/x32/Makefile (tests): Add tst-size_t-strncpy. + * sysdeps/x86_64/x32/tst-size_t-strncpy.c: New file. + +(cherry picked from commit c7c54f65b080affb87a1513dee449c8ad6143c8b) +--- + .../x86_64/multiarch/strcpy-sse2-unaligned.S | 4 +- + sysdeps/x86_64/multiarch/strcpy-ssse3.S | 6 +- + sysdeps/x86_64/x32/Makefile | 2 +- + sysdeps/x86_64/x32/tst-size_t-strncpy.c | 58 +++++++++++++++++++ + 4 files changed, 64 insertions(+), 6 deletions(-) + create mode 100644 sysdeps/x86_64/x32/tst-size_t-strncpy.c + +diff --git a/sysdeps/x86_64/multiarch/strcpy-sse2-unaligned.S b/sysdeps/x86_64/multiarch/strcpy-sse2-unaligned.S +index 72bf7e85..50aca22d 100644 +--- a/sysdeps/x86_64/multiarch/strcpy-sse2-unaligned.S ++++ b/sysdeps/x86_64/multiarch/strcpy-sse2-unaligned.S +@@ -40,8 +40,8 @@ + .text + ENTRY (STRCPY) + # ifdef USE_AS_STRNCPY +- mov %rdx, %r8 +- test %r8, %r8 ++ mov %RDX_LP, %R8_LP ++ test %R8_LP, %R8_LP + jz L(ExitZero) + # endif + mov %rsi, %rcx +diff --git a/sysdeps/x86_64/multiarch/strcpy-ssse3.S b/sysdeps/x86_64/multiarch/strcpy-ssse3.S +index 9858d0c4..0a62814a 100644 +--- a/sysdeps/x86_64/multiarch/strcpy-ssse3.S ++++ b/sysdeps/x86_64/multiarch/strcpy-ssse3.S +@@ -31,13 +31,13 @@ ENTRY (STRCPY) + + mov %rsi, %rcx + # ifdef USE_AS_STRNCPY +- mov %rdx, %r8 ++ mov %RDX_LP, %R8_LP + # endif + mov %rdi, %rdx + # ifdef USE_AS_STRNCPY +- test %r8, %r8 ++ test %R8_LP, %R8_LP + jz L(Exit0) +- cmp $8, %r8 ++ cmp $8, %R8_LP + jbe L(StrncpyExit8Bytes) + # endif + cmpb $0, (%rcx) +diff --git a/sysdeps/x86_64/x32/Makefile b/sysdeps/x86_64/x32/Makefile +index db302839..2a9e20a9 100644 +--- a/sysdeps/x86_64/x32/Makefile ++++ b/sysdeps/x86_64/x32/Makefile +@@ -8,7 +8,7 @@ endif + ifeq ($(subdir),string) + tests += tst-size_t-memchr tst-size_t-memcmp tst-size_t-memcpy \ + tst-size_t-memrchr tst-size_t-memset tst-size_t-strncasecmp \ +- tst-size_t-strncmp ++ tst-size_t-strncmp tst-size_t-strncpy + endif + + ifeq ($(subdir),wcsmbs) +diff --git a/sysdeps/x86_64/x32/tst-size_t-strncpy.c b/sysdeps/x86_64/x32/tst-size_t-strncpy.c +new file mode 100644 +index 00000000..4dec71e6 +--- /dev/null ++++ b/sysdeps/x86_64/x32/tst-size_t-strncpy.c +@@ -0,0 +1,58 @@ ++/* Test strncpy with size_t in the lower 32 bits of 64-bit register. ++ Copyright (C) 2019 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#define TEST_NAME "strncpy" ++#include "test-size_t.h" ++ ++IMPL (strncpy, 1) ++ ++typedef char *(*proto_t) (char *, const char*, size_t); ++ ++static void * ++__attribute__ ((noinline, noclone)) ++do_strncpy (parameter_t a, parameter_t b) ++{ ++ return CALL (&b, a.p, b.p, a.len); ++} ++ ++static int ++test_main (void) ++{ ++ test_init (); ++ ++ parameter_t dest = { { page_size }, buf1 }; ++ parameter_t src = { { 0 }, buf2 }; ++ ++ int ret = 0; ++ FOR_EACH_IMPL (impl, 0) ++ { ++ src.fn = impl->fn; ++ do_strncpy (dest, src); ++ int res = strncmp (dest.p, src.p, dest.len); ++ if (res) ++ { ++ error (0, 0, "Wrong result in function %s: %i != 0", ++ impl->name, res); ++ ret = 1; ++ } ++ } ++ ++ return ret ? EXIT_FAILURE : EXIT_SUCCESS; ++} ++ ++#include +-- +GitLab + diff --git a/SOURCES/glibc-sw24097-8.patch b/SOURCES/glibc-sw24097-8.patch new file mode 100644 index 0000000..49f7327 --- /dev/null +++ b/SOURCES/glibc-sw24097-8.patch @@ -0,0 +1,215 @@ +From 3e2556a4407e551a9c4086e67f46ef8d235cfe0d Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Fri, 1 Feb 2019 12:24:08 -0800 +Subject: [PATCH] x86-64 strnlen/wcsnlen: Properly handle the length parameter + [BZ #24097] + +On x32, the size_t parameter may be passed in the lower 32 bits of a +64-bit register with the non-zero upper 32 bits. The string/memory +functions written in assembly can only use the lower 32 bits of a +64-bit register as length or must clear the upper 32 bits before using +the full 64-bit register for length. + +This pach fixes strnlen/wcsnlen for x32. Tested on x86-64 and x32. On +x86-64, libc.so is the same with and withou the fix. + + [BZ #24097] + CVE-2019-6488 + * sysdeps/x86_64/multiarch/strlen-avx2.S: Use RSI_LP for length. + Clear the upper 32 bits of RSI register. + * sysdeps/x86_64/strlen.S: Use RSI_LP for length. + * sysdeps/x86_64/x32/Makefile (tests): Add tst-size_t-strnlen + and tst-size_t-wcsnlen. + * sysdeps/x86_64/x32/tst-size_t-strnlen.c: New file. + * sysdeps/x86_64/x32/tst-size_t-wcsnlen.c: Likewise. + +(cherry picked from commit 5165de69c0908e28a380cbd4bb054e55ea4abc95) +--- + sysdeps/x86_64/multiarch/strlen-avx2.S | 9 ++-- + sysdeps/x86_64/strlen.S | 12 ++--- + sysdeps/x86_64/x32/Makefile | 4 +- + sysdeps/x86_64/x32/tst-size_t-strnlen.c | 72 +++++++++++++++++++++++++ + sysdeps/x86_64/x32/tst-size_t-wcsnlen.c | 20 +++++++ + 5 files changed, 106 insertions(+), 11 deletions(-) + create mode 100644 sysdeps/x86_64/x32/tst-size_t-strnlen.c + create mode 100644 sysdeps/x86_64/x32/tst-size_t-wcsnlen.c + +diff --git a/sysdeps/x86_64/multiarch/strlen-avx2.S b/sysdeps/x86_64/multiarch/strlen-avx2.S +index fb2418cd..645e0446 100644 +--- a/sysdeps/x86_64/multiarch/strlen-avx2.S ++++ b/sysdeps/x86_64/multiarch/strlen-avx2.S +@@ -42,12 +42,15 @@ + ENTRY (STRLEN) + # ifdef USE_AS_STRNLEN + /* Check for zero length. */ +- testq %rsi, %rsi ++ test %RSI_LP, %RSI_LP + jz L(zero) + # ifdef USE_AS_WCSLEN +- shl $2, %rsi ++ shl $2, %RSI_LP ++# elif defined __ILP32__ ++ /* Clear the upper 32 bits. */ ++ movl %esi, %esi + # endif +- movq %rsi, %r8 ++ mov %RSI_LP, %R8_LP + # endif + movl %edi, %ecx + movq %rdi, %rdx +diff --git a/sysdeps/x86_64/strlen.S b/sysdeps/x86_64/strlen.S +index 01cb5fa8..f845f3d4 100644 +--- a/sysdeps/x86_64/strlen.S ++++ b/sysdeps/x86_64/strlen.S +@@ -59,21 +59,21 @@ ENTRY(strlen) + + #ifdef AS_STRNLEN + /* Do not read anything when n==0. */ +- test %rsi, %rsi ++ test %RSI_LP, %RSI_LP + jne L(n_nonzero) + xor %rax, %rax + ret + L(n_nonzero): + # ifdef AS_WCSLEN +- shlq $2, %rsi ++ shl $2, %RSI_LP + # endif + + /* Initialize long lived registers. */ + +- add %rdi, %rsi +- mov %rsi, %r10 +- and $-64, %r10 +- mov %rsi, %r11 ++ add %RDI_LP, %RSI_LP ++ mov %RSI_LP, %R10_LP ++ and $-64, %R10_LP ++ mov %RSI_LP, %R11_LP + #endif + + pxor %xmm0, %xmm0 +diff --git a/sysdeps/x86_64/x32/Makefile b/sysdeps/x86_64/x32/Makefile +index 2a9e20a9..1557724b 100644 +--- a/sysdeps/x86_64/x32/Makefile ++++ b/sysdeps/x86_64/x32/Makefile +@@ -8,10 +8,10 @@ endif + ifeq ($(subdir),string) + tests += tst-size_t-memchr tst-size_t-memcmp tst-size_t-memcpy \ + tst-size_t-memrchr tst-size_t-memset tst-size_t-strncasecmp \ +- tst-size_t-strncmp tst-size_t-strncpy ++ tst-size_t-strncmp tst-size_t-strncpy tst-size_t-strnlen + endif + + ifeq ($(subdir),wcsmbs) + tests += tst-size_t-wmemchr tst-size_t-wmemcmp tst-size_t-wmemset \ +- tst-size_t-wcsncmp ++ tst-size_t-wcsncmp tst-size_t-wcsnlen + endif +diff --git a/sysdeps/x86_64/x32/tst-size_t-strnlen.c b/sysdeps/x86_64/x32/tst-size_t-strnlen.c +new file mode 100644 +index 00000000..690a4a8a +--- /dev/null ++++ b/sysdeps/x86_64/x32/tst-size_t-strnlen.c +@@ -0,0 +1,72 @@ ++/* Test strnlen with size_t in the lower 32 bits of 64-bit register. ++ Copyright (C) 2019 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#ifdef WIDE ++# define TEST_NAME "wcsnlen" ++#else ++# define TEST_NAME "strnlen" ++#endif /* WIDE */ ++ ++#include "test-size_t.h" ++ ++#ifdef WIDE ++# include ++# define STRNLEN wcsnlen ++# define CHAR wchar_t ++#else ++# define STRNLEN strnlen ++# define CHAR char ++#endif /* WIDE */ ++ ++IMPL (STRNLEN, 1) ++ ++typedef size_t (*proto_t) (const CHAR *, size_t); ++ ++static size_t ++__attribute__ ((noinline, noclone)) ++do_strnlen (parameter_t a, parameter_t b) ++{ ++ return CALL (&a, a.p, b.len); ++} ++ ++static int ++test_main (void) ++{ ++ test_init (); ++ ++ size_t size = page_size / sizeof (CHAR); ++ parameter_t src = { { 0 }, buf2 }; ++ parameter_t c = { { size }, (void *) (uintptr_t) 'a' }; ++ ++ int ret = 0; ++ FOR_EACH_IMPL (impl, 0) ++ { ++ src.fn = impl->fn; ++ size_t res = do_strnlen (src, c); ++ if (res != size) ++ { ++ error (0, 0, "Wrong result in function %s: 0x%x != 0x%x", ++ impl->name, res, size); ++ ret = 1; ++ } ++ } ++ ++ return ret ? EXIT_FAILURE : EXIT_SUCCESS; ++} ++ ++#include +diff --git a/sysdeps/x86_64/x32/tst-size_t-wcsnlen.c b/sysdeps/x86_64/x32/tst-size_t-wcsnlen.c +new file mode 100644 +index 00000000..093b4bbe +--- /dev/null ++++ b/sysdeps/x86_64/x32/tst-size_t-wcsnlen.c +@@ -0,0 +1,20 @@ ++/* Test wcsnlen with size_t in the lower 32 bits of 64-bit register. ++ Copyright (C) 2019 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#define WIDE 1 ++#include "tst-size_t-strnlen.c" +-- +GitLab + diff --git a/SOURCES/glibc-sw24155.patch b/SOURCES/glibc-sw24155.patch new file mode 100644 index 0000000..49005d6 --- /dev/null +++ b/SOURCES/glibc-sw24155.patch @@ -0,0 +1,201 @@ +From 3cef261bb97c00473cb41ae4d7401a46e3d96ce7 Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Mon, 4 Feb 2019 08:55:52 -0800 +Subject: [PATCH] x86-64 memcmp: Use unsigned Jcc instructions on size [BZ + #24155] + +Since the size argument is unsigned. we should use unsigned Jcc +instructions, instead of signed, to check size. + +Tested on x86-64 and x32, with and without --disable-multi-arch. + + [BZ #24155] + CVE-2019-7309 + * NEWS: Updated for CVE-2019-7309. + * sysdeps/x86_64/memcmp.S: Use RDX_LP for size. Clear the + upper 32 bits of RDX register for x32. Use unsigned Jcc + instructions, instead of signed. + * sysdeps/x86_64/x32/Makefile (tests): Add tst-size_t-memcmp-2. + * sysdeps/x86_64/x32/tst-size_t-memcmp-2.c: New test. + +(cherry picked from commit 3f635fb43389b54f682fc9ed2acc0b2aaf4a923d) +--- + sysdeps/x86_64/memcmp.S | 20 +++--- + sysdeps/x86_64/x32/Makefile | 3 +- + sysdeps/x86_64/x32/tst-size_t-memcmp-2.c | 79 ++++++++++++++++++++++++ + 3 files changed, 93 insertions(+), 9 deletions(-) + create mode 100644 sysdeps/x86_64/x32/tst-size_t-memcmp-2.c + +diff --git a/sysdeps/x86_64/memcmp.S b/sysdeps/x86_64/memcmp.S +index bcb4a2e8..45918d37 100644 +--- a/sysdeps/x86_64/memcmp.S ++++ b/sysdeps/x86_64/memcmp.S +@@ -21,14 +21,18 @@ + + .text + ENTRY (memcmp) +- test %rdx, %rdx ++#ifdef __ILP32__ ++ /* Clear the upper 32 bits. */ ++ movl %edx, %edx ++#endif ++ test %RDX_LP, %RDX_LP + jz L(finz) + cmpq $1, %rdx +- jle L(finr1b) ++ jbe L(finr1b) + subq %rdi, %rsi + movq %rdx, %r10 + cmpq $32, %r10 +- jge L(gt32) ++ jae L(gt32) + /* Handle small chunks and last block of less than 32 bytes. */ + L(small): + testq $1, %r10 +@@ -156,7 +160,7 @@ L(A32): + movq %r11, %r10 + andq $-32, %r10 + cmpq %r10, %rdi +- jge L(mt16) ++ jae L(mt16) + /* Pre-unroll to be ready for unrolled 64B loop. */ + testq $32, %rdi + jz L(A64) +@@ -178,7 +182,7 @@ L(A64): + movq %r11, %r10 + andq $-64, %r10 + cmpq %r10, %rdi +- jge L(mt32) ++ jae L(mt32) + + L(A64main): + movdqu (%rdi,%rsi), %xmm0 +@@ -216,7 +220,7 @@ L(mt32): + movq %r11, %r10 + andq $-32, %r10 + cmpq %r10, %rdi +- jge L(mt16) ++ jae L(mt16) + + L(A32main): + movdqu (%rdi,%rsi), %xmm0 +@@ -254,7 +258,7 @@ L(ATR): + movq %r11, %r10 + andq $-32, %r10 + cmpq %r10, %rdi +- jge L(mt16) ++ jae L(mt16) + testq $16, %rdi + jz L(ATR32) + +@@ -325,7 +329,7 @@ L(ATR64main): + movq %r11, %r10 + andq $-32, %r10 + cmpq %r10, %rdi +- jge L(mt16) ++ jae L(mt16) + + L(ATR32res): + movdqa (%rdi,%rsi), %xmm0 +diff --git a/sysdeps/x86_64/x32/Makefile b/sysdeps/x86_64/x32/Makefile +index 1557724b..87489565 100644 +--- a/sysdeps/x86_64/x32/Makefile ++++ b/sysdeps/x86_64/x32/Makefile +@@ -8,7 +8,8 @@ endif + ifeq ($(subdir),string) + tests += tst-size_t-memchr tst-size_t-memcmp tst-size_t-memcpy \ + tst-size_t-memrchr tst-size_t-memset tst-size_t-strncasecmp \ +- tst-size_t-strncmp tst-size_t-strncpy tst-size_t-strnlen ++ tst-size_t-strncmp tst-size_t-strncpy tst-size_t-strnlen \ ++ tst-size_t-memcmp-2 + endif + + ifeq ($(subdir),wcsmbs) +diff --git a/sysdeps/x86_64/x32/tst-size_t-memcmp-2.c b/sysdeps/x86_64/x32/tst-size_t-memcmp-2.c +new file mode 100644 +index 00000000..d8ae1a08 +--- /dev/null ++++ b/sysdeps/x86_64/x32/tst-size_t-memcmp-2.c +@@ -0,0 +1,79 @@ ++/* Test memcmp with size_t in the lower 32 bits of 64-bit register. ++ Copyright (C) 2019 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#define TEST_MAIN ++#ifdef WIDE ++# define TEST_NAME "wmemcmp" ++#else ++# define TEST_NAME "memcmp" ++#endif ++ ++#include "test-size_t.h" ++ ++#ifdef WIDE ++# include ++# include ++ ++# define MEMCMP wmemcmp ++# define CHAR wchar_t ++#else ++# define MEMCMP memcmp ++# define CHAR char ++#endif ++ ++IMPL (MEMCMP, 1) ++ ++typedef int (*proto_t) (const CHAR *, const CHAR *, size_t); ++ ++static int ++__attribute__ ((noinline, noclone)) ++do_memcmp (parameter_t a, parameter_t b) ++{ ++ return CALL (&b, a.p, b.p, a.len); ++} ++ ++static int ++test_main (void) ++{ ++ test_init (); ++ ++ parameter_t dest = { { page_size / sizeof (CHAR) }, buf1 }; ++ parameter_t src = { { 0 }, buf2 }; ++ ++ memcpy (buf1, buf2, page_size); ++ ++ CHAR *p = (CHAR *) buf1; ++ p[page_size / sizeof (CHAR) - 1] = (CHAR) 1; ++ ++ int ret = 0; ++ FOR_EACH_IMPL (impl, 0) ++ { ++ src.fn = impl->fn; ++ int res = do_memcmp (dest, src); ++ if (res >= 0) ++ { ++ error (0, 0, "Wrong result in function %s: %i >= 0", ++ impl->name, res); ++ ret = 1; ++ } ++ } ++ ++ return ret ? EXIT_FAILURE : EXIT_SUCCESS; ++} ++ ++#include +-- +GitLab + diff --git a/SOURCES/glibc-sw25966.patch b/SOURCES/glibc-sw25966.patch new file mode 100644 index 0000000..11c9df8 --- /dev/null +++ b/SOURCES/glibc-sw25966.patch @@ -0,0 +1,52 @@ +From c640bf1ea99d9598f8e3afc922567d8b3623d23e Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Wed, 2 Mar 2022 14:57:54 -0800 +Subject: [PATCH] x86-64: Use RDX_LP on __x86_shared_non_temporal_threshold [BZ + #25966] + +Since __x86_shared_non_temporal_threshold is defined as + +long int __x86_shared_non_temporal_threshold; + +and long int is 4 bytes for x32, use RDX_LP to compare against +__x86_shared_non_temporal_threshold in assembly code. + +(cherry picked from commit 55c7bcc71b84123d5d4bd2814366a6b05fcf8ebd) +--- + sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S +index 71f5954d..673b73aa 100644 +--- a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S ++++ b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S +@@ -245,7 +245,7 @@ L(return): + #endif + + L(movsb): +- cmpq __x86_shared_non_temporal_threshold(%rip), %rdx ++ cmp __x86_shared_non_temporal_threshold(%rip), %RDX_LP + jae L(more_8x_vec) + cmpq %rsi, %rdi + jb 1f +@@ -397,7 +397,7 @@ L(more_8x_vec): + addq %r8, %rdx + #if (defined USE_MULTIARCH || VEC_SIZE == 16) && IS_IN (libc) + /* Check non-temporal store threshold. */ +- cmpq __x86_shared_non_temporal_threshold(%rip), %rdx ++ cmp __x86_shared_non_temporal_threshold(%rip), %RDX_LP + ja L(large_forward) + #endif + L(loop_4x_vec_forward): +@@ -448,7 +448,7 @@ L(more_8x_vec_backward): + subq %r8, %rdx + #if (defined USE_MULTIARCH || VEC_SIZE == 16) && IS_IN (libc) + /* Check non-temporal store threshold. */ +- cmpq __x86_shared_non_temporal_threshold(%rip), %rdx ++ cmp __x86_shared_non_temporal_threshold(%rip), %RDX_LP + ja L(large_backward) + #endif + L(loop_4x_vec_backward): +-- +GitLab + diff --git a/SOURCES/glibc-sw27130.patch b/SOURCES/glibc-sw27130.patch new file mode 100644 index 0000000..9065525 --- /dev/null +++ b/SOURCES/glibc-sw27130.patch @@ -0,0 +1,68 @@ +From b4cf48b235d64d0eab5739e6d496717c0023a32e Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Wed, 2 Mar 2022 15:17:29 -0800 +Subject: [PATCH] x86-64: Avoid rep movsb with short distance [BZ #27130] + +When copying with "rep movsb", if the distance between source and +destination is N*4GB + [1..63] with N >= 0, performance may be very +slow. This patch updates memmove-vec-unaligned-erms.S for AVX and +AVX512 versions with the distance in RCX: + + cmpl $63, %ecx + // Don't use "rep movsb" if ECX <= 63 + jbe L(Don't use rep movsb") + Use "rep movsb" + +Benchtests data with bench-memcpy, bench-memcpy-large, bench-memcpy-random +and bench-memcpy-walk on Skylake, Ice Lake and Tiger Lake show that its +performance impact is within noise range as "rep movsb" is only used for +data size >= 4KB. + +(cherry picked from commit 3ec5d83d2a237d39e7fd6ef7a0bc8ac4c171a4a5) +--- + .../multiarch/memmove-vec-unaligned-erms.S | 21 +++++++++++++++++++ + 1 file changed, 21 insertions(+) + +diff --git a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S +index 673b73aa..c475fed4 100644 +--- a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S ++++ b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S +@@ -64,6 +64,13 @@ + # endif + #endif + ++/* Avoid short distance rep movsb only with non-SSE vector. */ ++#ifndef AVOID_SHORT_DISTANCE_REP_MOVSB ++# define AVOID_SHORT_DISTANCE_REP_MOVSB (VEC_SIZE > 16) ++#else ++# define AVOID_SHORT_DISTANCE_REP_MOVSB 0 ++#endif ++ + #ifndef PREFETCH + # define PREFETCH(addr) prefetcht0 addr + #endif +@@ -255,7 +262,21 @@ L(movsb): + cmpq %r9, %rdi + /* Avoid slow backward REP MOVSB. */ + jb L(more_8x_vec_backward) ++# if AVOID_SHORT_DISTANCE_REP_MOVSB ++ movq %rdi, %rcx ++ subq %rsi, %rcx ++ jmp 2f ++# endif + 1: ++# if AVOID_SHORT_DISTANCE_REP_MOVSB ++ movq %rsi, %rcx ++ subq %rdi, %rcx ++2: ++/* Avoid "rep movsb" if RCX, the distance between source and destination, ++ is N*4GB + [1..63] with N >= 0. */ ++ cmpl $63, %ecx ++ jbe L(more_2x_vec) /* Avoid "rep movsb" if ECX <= 63. */ ++# endif + mov %RDX_LP, %RCX_LP + rep movsb + L(nop): +-- +GitLab + diff --git a/SOURCES/glibc-sw27974-1.patch b/SOURCES/glibc-sw27974-1.patch new file mode 100644 index 0000000..d584422 --- /dev/null +++ b/SOURCES/glibc-sw27974-1.patch @@ -0,0 +1,388 @@ +From d8d5c44ed7636fdd2b736e152f8207ca063da386 Mon Sep 17 00:00:00 2001 +From: Noah Goldstein +Date: Wed, 9 Jun 2021 16:25:32 -0400 +Subject: [PATCH] x86: Fix overflow bug with wmemchr-sse2 and wmemchr-avx2 [BZ + #27974] + +This commit fixes the bug mentioned in the previous commit. + +The previous implementations of wmemchr in these files relied +on n * sizeof(wchar_t) which was not guranteed by the standard. + +The new overflow tests added in the previous commit now +pass (As well as all the other tests). + +Signed-off-by: Noah Goldstein +Reviewed-by: H.J. Lu +(cherry picked from commit 645a158978f9520e74074e8c14047503be4db0f0) +--- + sysdeps/x86_64/memchr.S | 77 +++++++++++++++++++------- + sysdeps/x86_64/multiarch/memchr-avx2.S | 58 +++++++++++++------ + 2 files changed, 98 insertions(+), 37 deletions(-) + +diff --git a/sysdeps/x86_64/memchr.S b/sysdeps/x86_64/memchr.S +index cb320257..24f9a0c5 100644 +--- a/sysdeps/x86_64/memchr.S ++++ b/sysdeps/x86_64/memchr.S +@@ -21,9 +21,11 @@ + #ifdef USE_AS_WMEMCHR + # define MEMCHR wmemchr + # define PCMPEQ pcmpeqd ++# define CHAR_PER_VEC 4 + #else + # define MEMCHR memchr + # define PCMPEQ pcmpeqb ++# define CHAR_PER_VEC 16 + #endif + + /* fast SSE2 version with using pmaxub and 64 byte loop */ +@@ -33,15 +35,14 @@ ENTRY(MEMCHR) + movd %esi, %xmm1 + mov %edi, %ecx + ++#ifdef __ILP32__ ++ /* Clear the upper 32 bits. */ ++ movl %edx, %edx ++#endif + #ifdef USE_AS_WMEMCHR + test %RDX_LP, %RDX_LP + jz L(return_null) +- shl $2, %RDX_LP + #else +-# ifdef __ILP32__ +- /* Clear the upper 32 bits. */ +- movl %edx, %edx +-# endif + punpcklbw %xmm1, %xmm1 + test %RDX_LP, %RDX_LP + jz L(return_null) +@@ -60,13 +61,16 @@ ENTRY(MEMCHR) + test %eax, %eax + + jnz L(matches_1) +- sub $16, %rdx ++ sub $CHAR_PER_VEC, %rdx + jbe L(return_null) + add $16, %rdi + and $15, %ecx + and $-16, %rdi ++#ifdef USE_AS_WMEMCHR ++ shr $2, %ecx ++#endif + add %rcx, %rdx +- sub $64, %rdx ++ sub $(CHAR_PER_VEC * 4), %rdx + jbe L(exit_loop) + jmp L(loop_prolog) + +@@ -77,16 +81,21 @@ L(crosscache): + movdqa (%rdi), %xmm0 + + PCMPEQ %xmm1, %xmm0 +-/* Check if there is a match. */ ++ /* Check if there is a match. */ + pmovmskb %xmm0, %eax +-/* Remove the leading bytes. */ ++ /* Remove the leading bytes. */ + sar %cl, %eax + test %eax, %eax + je L(unaligned_no_match) +-/* Check which byte is a match. */ ++ /* Check which byte is a match. */ + bsf %eax, %eax +- ++#ifdef USE_AS_WMEMCHR ++ mov %eax, %esi ++ shr $2, %esi ++ sub %rsi, %rdx ++#else + sub %rax, %rdx ++#endif + jbe L(return_null) + add %rdi, %rax + add %rcx, %rax +@@ -94,15 +103,18 @@ L(crosscache): + + .p2align 4 + L(unaligned_no_match): +- /* "rcx" is less than 16. Calculate "rdx + rcx - 16" by using ++ /* "rcx" is less than 16. Calculate "rdx + rcx - 16" by using + "rdx - (16 - rcx)" instead of "(rdx + rcx) - 16" to void + possible addition overflow. */ + neg %rcx + add $16, %rcx ++#ifdef USE_AS_WMEMCHR ++ shr $2, %ecx ++#endif + sub %rcx, %rdx + jbe L(return_null) + add $16, %rdi +- sub $64, %rdx ++ sub $(CHAR_PER_VEC * 4), %rdx + jbe L(exit_loop) + + .p2align 4 +@@ -135,7 +147,7 @@ L(loop_prolog): + test $0x3f, %rdi + jz L(align64_loop) + +- sub $64, %rdx ++ sub $(CHAR_PER_VEC * 4), %rdx + jbe L(exit_loop) + + movdqa (%rdi), %xmm0 +@@ -167,11 +179,14 @@ L(loop_prolog): + mov %rdi, %rcx + and $-64, %rdi + and $63, %ecx ++#ifdef USE_AS_WMEMCHR ++ shr $2, %ecx ++#endif + add %rcx, %rdx + + .p2align 4 + L(align64_loop): +- sub $64, %rdx ++ sub $(CHAR_PER_VEC * 4), %rdx + jbe L(exit_loop) + movdqa (%rdi), %xmm0 + movdqa 16(%rdi), %xmm2 +@@ -218,7 +233,7 @@ L(align64_loop): + + .p2align 4 + L(exit_loop): +- add $32, %edx ++ add $(CHAR_PER_VEC * 2), %edx + jle L(exit_loop_32) + + movdqa (%rdi), %xmm0 +@@ -238,7 +253,7 @@ L(exit_loop): + pmovmskb %xmm3, %eax + test %eax, %eax + jnz L(matches32_1) +- sub $16, %edx ++ sub $CHAR_PER_VEC, %edx + jle L(return_null) + + PCMPEQ 48(%rdi), %xmm1 +@@ -250,13 +265,13 @@ L(exit_loop): + + .p2align 4 + L(exit_loop_32): +- add $32, %edx ++ add $(CHAR_PER_VEC * 2), %edx + movdqa (%rdi), %xmm0 + PCMPEQ %xmm1, %xmm0 + pmovmskb %xmm0, %eax + test %eax, %eax + jnz L(matches_1) +- sub $16, %edx ++ sub $CHAR_PER_VEC, %edx + jbe L(return_null) + + PCMPEQ 16(%rdi), %xmm1 +@@ -293,7 +308,13 @@ L(matches32): + .p2align 4 + L(matches_1): + bsf %eax, %eax ++#ifdef USE_AS_WMEMCHR ++ mov %eax, %esi ++ shr $2, %esi ++ sub %rsi, %rdx ++#else + sub %rax, %rdx ++#endif + jbe L(return_null) + add %rdi, %rax + ret +@@ -301,7 +322,13 @@ L(matches_1): + .p2align 4 + L(matches16_1): + bsf %eax, %eax ++#ifdef USE_AS_WMEMCHR ++ mov %eax, %esi ++ shr $2, %esi ++ sub %rsi, %rdx ++#else + sub %rax, %rdx ++#endif + jbe L(return_null) + lea 16(%rdi, %rax), %rax + ret +@@ -309,7 +336,13 @@ L(matches16_1): + .p2align 4 + L(matches32_1): + bsf %eax, %eax ++#ifdef USE_AS_WMEMCHR ++ mov %eax, %esi ++ shr $2, %esi ++ sub %rsi, %rdx ++#else + sub %rax, %rdx ++#endif + jbe L(return_null) + lea 32(%rdi, %rax), %rax + ret +@@ -317,7 +350,13 @@ L(matches32_1): + .p2align 4 + L(matches48_1): + bsf %eax, %eax ++#ifdef USE_AS_WMEMCHR ++ mov %eax, %esi ++ shr $2, %esi ++ sub %rsi, %rdx ++#else + sub %rax, %rdx ++#endif + jbe L(return_null) + lea 48(%rdi, %rax), %rax + ret +diff --git a/sysdeps/x86_64/multiarch/memchr-avx2.S b/sysdeps/x86_64/multiarch/memchr-avx2.S +index b377f22e..16027abb 100644 +--- a/sysdeps/x86_64/multiarch/memchr-avx2.S ++++ b/sysdeps/x86_64/multiarch/memchr-avx2.S +@@ -54,21 +54,19 @@ + + # define VEC_SIZE 32 + # define PAGE_SIZE 4096 ++# define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE) + + .section SECTION(.text),"ax",@progbits + ENTRY (MEMCHR) + # ifndef USE_AS_RAWMEMCHR + /* Check for zero length. */ +- test %RDX_LP, %RDX_LP +- jz L(null) +-# endif +-# ifdef USE_AS_WMEMCHR +- shl $2, %RDX_LP +-# else + # ifdef __ILP32__ +- /* Clear the upper 32 bits. */ +- movl %edx, %edx ++ /* Clear upper bits. */ ++ and %RDX_LP, %RDX_LP ++# else ++ test %RDX_LP, %RDX_LP + # endif ++ jz L(null) + # endif + /* Broadcast CHAR to YMMMATCH. */ + vmovd %esi, %xmm0 +@@ -84,7 +82,7 @@ ENTRY (MEMCHR) + vpmovmskb %ymm1, %eax + # ifndef USE_AS_RAWMEMCHR + /* If length < CHAR_PER_VEC handle special. */ +- cmpq $VEC_SIZE, %rdx ++ cmpq $CHAR_PER_VEC, %rdx + jbe L(first_vec_x0) + # endif + testl %eax, %eax +@@ -98,6 +96,10 @@ ENTRY (MEMCHR) + L(first_vec_x0): + /* Check if first match was before length. */ + tzcntl %eax, %eax ++# ifdef USE_AS_WMEMCHR ++ /* NB: Multiply length by 4 to get byte count. */ ++ sall $2, %edx ++# endif + xorl %ecx, %ecx + cmpl %eax, %edx + leaq (%rdi, %rax), %rax +@@ -110,12 +112,12 @@ L(null): + # endif + .p2align 4 + L(cross_page_boundary): +- /* Save pointer before aligning as its original value is necessary +- for computer return address if byte is found or adjusting length +- if it is not and this is memchr. */ ++ /* Save pointer before aligning as its original value is ++ necessary for computer return address if byte is found or ++ adjusting length if it is not and this is memchr. */ + movq %rdi, %rcx +- /* Align data to VEC_SIZE - 1. ALGN_PTR_REG is rcx for memchr and +- rdi for rawmemchr. */ ++ /* Align data to VEC_SIZE - 1. ALGN_PTR_REG is rcx for memchr ++ and rdi for rawmemchr. */ + orq $(VEC_SIZE - 1), %ALGN_PTR_REG + VPCMPEQ -(VEC_SIZE - 1)(%ALGN_PTR_REG), %ymm0, %ymm1 + vpmovmskb %ymm1, %eax +@@ -124,6 +126,10 @@ L(cross_page_boundary): + match). */ + leaq 1(%ALGN_PTR_REG), %rsi + subq %RRAW_PTR_REG, %rsi ++# ifdef USE_AS_WMEMCHR ++ /* NB: Divide bytes by 4 to get wchar_t count. */ ++ shrl $2, %esi ++# endif + # endif + /* Remove the leading bytes. */ + sarxl %ERAW_PTR_REG, %eax, %eax +@@ -181,6 +187,10 @@ L(cross_page_continue): + orq $(VEC_SIZE - 1), %rdi + /* esi is for adjusting length to see if near the end. */ + leal (VEC_SIZE * 4 + 1)(%rdi, %rcx), %esi ++# ifdef USE_AS_WMEMCHR ++ /* NB: Divide bytes by 4 to get the wchar_t count. */ ++ sarl $2, %esi ++# endif + # else + orq $(VEC_SIZE - 1), %rdi + L(cross_page_continue): +@@ -213,7 +223,7 @@ L(cross_page_continue): + + # ifndef USE_AS_RAWMEMCHR + /* Check if at last VEC_SIZE * 4 length. */ +- subq $(VEC_SIZE * 4), %rdx ++ subq $(CHAR_PER_VEC * 4), %rdx + jbe L(last_4x_vec_or_less_cmpeq) + /* Align data to VEC_SIZE * 4 - 1 for the loop and readjust + length. */ +@@ -221,6 +231,10 @@ L(cross_page_continue): + movl %edi, %ecx + orq $(VEC_SIZE * 4 - 1), %rdi + andl $(VEC_SIZE * 4 - 1), %ecx ++# ifdef USE_AS_WMEMCHR ++ /* NB: Divide bytes by 4 to get the wchar_t count. */ ++ sarl $2, %ecx ++# endif + addq %rcx, %rdx + # else + /* Align data to VEC_SIZE * 4 - 1 for loop. */ +@@ -250,15 +264,19 @@ L(loop_4x_vec): + + subq $-(VEC_SIZE * 4), %rdi + +- subq $(VEC_SIZE * 4), %rdx ++ subq $(CHAR_PER_VEC * 4), %rdx + ja L(loop_4x_vec) + +- /* Fall through into less than 4 remaining vectors of length case. +- */ ++ /* Fall through into less than 4 remaining vectors of length ++ case. */ + VPCMPEQ (VEC_SIZE * 0 + 1)(%rdi), %ymm0, %ymm1 + vpmovmskb %ymm1, %eax + .p2align 4 + L(last_4x_vec_or_less): ++# ifdef USE_AS_WMEMCHR ++ /* NB: Multiply length by 4 to get byte count. */ ++ sall $2, %edx ++# endif + /* Check if first VEC contained match. */ + testl %eax, %eax + jnz L(first_vec_x1_check) +@@ -355,6 +373,10 @@ L(last_vec_x2_return): + L(last_4x_vec_or_less_cmpeq): + VPCMPEQ (VEC_SIZE * 4 + 1)(%rdi), %ymm0, %ymm1 + vpmovmskb %ymm1, %eax ++# ifdef USE_AS_WMEMCHR ++ /* NB: Multiply length by 4 to get byte count. */ ++ sall $2, %edx ++# endif + subq $-(VEC_SIZE * 4), %rdi + /* Check first VEC regardless. */ + testl %eax, %eax +-- +GitLab + diff --git a/SOURCES/glibc-sw27974-2.patch b/SOURCES/glibc-sw27974-2.patch new file mode 100644 index 0000000..96edc19 --- /dev/null +++ b/SOURCES/glibc-sw27974-2.patch @@ -0,0 +1,497 @@ +From 78ff769ceac455cb6749f64effe77d178216f0b0 Mon Sep 17 00:00:00 2001 +From: Noah Goldstein +Date: Wed, 23 Jun 2021 01:56:29 -0400 +Subject: [PATCH] x86: Fix overflow bug in wcsnlen-sse4_1 and wcsnlen-avx2 [BZ + #27974] + +This commit fixes the bug mentioned in the previous commit. + +The previous implementations of wmemchr in these files relied +on maxlen * sizeof(wchar_t) which was not guranteed by the standard. + +The new overflow tests added in the previous commit now +pass (As well as all the other tests). + +Signed-off-by: Noah Goldstein +Reviewed-by: H.J. Lu +(cherry picked from commit a775a7a3eb1e85b54af0b4ee5ff4dcf66772a1fb) +--- + sysdeps/x86_64/multiarch/strlen-avx2.S | 130 ++++++++++++++++++------- + sysdeps/x86_64/multiarch/strlen-vec.S | 15 ++- + 2 files changed, 107 insertions(+), 38 deletions(-) + +diff --git a/sysdeps/x86_64/multiarch/strlen-avx2.S b/sysdeps/x86_64/multiarch/strlen-avx2.S +index be8a5db5..37688966 100644 +--- a/sysdeps/x86_64/multiarch/strlen-avx2.S ++++ b/sysdeps/x86_64/multiarch/strlen-avx2.S +@@ -44,21 +44,21 @@ + + # define VEC_SIZE 32 + # define PAGE_SIZE 4096 ++# define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE) + + .section SECTION(.text),"ax",@progbits + ENTRY (STRLEN) + # ifdef USE_AS_STRNLEN + /* Check zero length. */ ++# ifdef __ILP32__ ++ /* Clear upper bits. */ ++ and %RSI_LP, %RSI_LP ++# else + test %RSI_LP, %RSI_LP ++# endif + jz L(zero) + /* Store max len in R8_LP before adjusting if using WCSLEN. */ + mov %RSI_LP, %R8_LP +-# ifdef USE_AS_WCSLEN +- shl $2, %RSI_LP +-# elif defined __ILP32__ +- /* Clear the upper 32 bits. */ +- movl %esi, %esi +-# endif + # endif + movl %edi, %eax + movq %rdi, %rdx +@@ -72,10 +72,10 @@ ENTRY (STRLEN) + + /* Check the first VEC_SIZE bytes. */ + VPCMPEQ (%rdi), %ymm0, %ymm1 +- vpmovmskb %ymm1, %eax ++ vpmovmskb %ymm1, %eax + # ifdef USE_AS_STRNLEN + /* If length < VEC_SIZE handle special. */ +- cmpq $VEC_SIZE, %rsi ++ cmpq $CHAR_PER_VEC, %rsi + jbe L(first_vec_x0) + # endif + /* If empty continue to aligned_more. Otherwise return bit +@@ -84,6 +84,7 @@ ENTRY (STRLEN) + jz L(aligned_more) + tzcntl %eax, %eax + # ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get wchar_t count. */ + shrl $2, %eax + # endif + VZEROUPPER_RETURN +@@ -97,9 +98,14 @@ L(zero): + L(first_vec_x0): + /* Set bit for max len so that tzcnt will return min of max len + and position of first match. */ ++# ifdef USE_AS_WCSLEN ++ /* NB: Multiply length by 4 to get byte count. */ ++ sall $2, %esi ++# endif + btsq %rsi, %rax + tzcntl %eax, %eax + # ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get wchar_t count. */ + shrl $2, %eax + # endif + VZEROUPPER_RETURN +@@ -113,14 +119,19 @@ L(first_vec_x1): + # ifdef USE_AS_STRNLEN + /* Use ecx which was computed earlier to compute correct value. + */ ++# ifdef USE_AS_WCSLEN ++ leal -(VEC_SIZE * 4 + 1)(%rax, %rcx, 4), %eax ++# else + subl $(VEC_SIZE * 4 + 1), %ecx + addl %ecx, %eax ++# endif + # else + subl %edx, %edi + incl %edi + addl %edi, %eax + # endif + # ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get wchar_t count. */ + shrl $2, %eax + # endif + VZEROUPPER_RETURN +@@ -133,14 +144,19 @@ L(first_vec_x2): + # ifdef USE_AS_STRNLEN + /* Use ecx which was computed earlier to compute correct value. + */ ++# ifdef USE_AS_WCSLEN ++ leal -(VEC_SIZE * 3 + 1)(%rax, %rcx, 4), %eax ++# else + subl $(VEC_SIZE * 3 + 1), %ecx + addl %ecx, %eax ++# endif + # else + subl %edx, %edi + addl $(VEC_SIZE + 1), %edi + addl %edi, %eax + # endif + # ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get wchar_t count. */ + shrl $2, %eax + # endif + VZEROUPPER_RETURN +@@ -153,14 +169,19 @@ L(first_vec_x3): + # ifdef USE_AS_STRNLEN + /* Use ecx which was computed earlier to compute correct value. + */ ++# ifdef USE_AS_WCSLEN ++ leal -(VEC_SIZE * 2 + 1)(%rax, %rcx, 4), %eax ++# else + subl $(VEC_SIZE * 2 + 1), %ecx + addl %ecx, %eax ++# endif + # else + subl %edx, %edi + addl $(VEC_SIZE * 2 + 1), %edi + addl %edi, %eax + # endif + # ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get wchar_t count. */ + shrl $2, %eax + # endif + VZEROUPPER_RETURN +@@ -173,14 +194,19 @@ L(first_vec_x4): + # ifdef USE_AS_STRNLEN + /* Use ecx which was computed earlier to compute correct value. + */ ++# ifdef USE_AS_WCSLEN ++ leal -(VEC_SIZE * 1 + 1)(%rax, %rcx, 4), %eax ++# else + subl $(VEC_SIZE + 1), %ecx + addl %ecx, %eax ++# endif + # else + subl %edx, %edi + addl $(VEC_SIZE * 3 + 1), %edi + addl %edi, %eax + # endif + # ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get wchar_t count. */ + shrl $2, %eax + # endif + VZEROUPPER_RETURN +@@ -195,10 +221,14 @@ L(cross_page_continue): + /* Check the first 4 * VEC_SIZE. Only one VEC_SIZE at a time + since data is only aligned to VEC_SIZE. */ + # ifdef USE_AS_STRNLEN +- /* + 1 because rdi is aligned to VEC_SIZE - 1. + CHAR_SIZE because +- it simplies the logic in last_4x_vec_or_less. */ ++ /* + 1 because rdi is aligned to VEC_SIZE - 1. + CHAR_SIZE ++ because it simplies the logic in last_4x_vec_or_less. */ + leaq (VEC_SIZE * 4 + CHAR_SIZE + 1)(%rdi), %rcx + subq %rdx, %rcx ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get the wchar_t count. */ ++ sarl $2, %ecx ++# endif + # endif + /* Load first VEC regardless. */ + VPCMPEQ 1(%rdi), %ymm0, %ymm1 +@@ -207,34 +237,38 @@ L(cross_page_continue): + subq %rcx, %rsi + jb L(last_4x_vec_or_less) + # endif +- vpmovmskb %ymm1, %eax ++ vpmovmskb %ymm1, %eax + testl %eax, %eax + jnz L(first_vec_x1) + + VPCMPEQ (VEC_SIZE + 1)(%rdi), %ymm0, %ymm1 +- vpmovmskb %ymm1, %eax ++ vpmovmskb %ymm1, %eax + testl %eax, %eax + jnz L(first_vec_x2) + + VPCMPEQ (VEC_SIZE * 2 + 1)(%rdi), %ymm0, %ymm1 +- vpmovmskb %ymm1, %eax ++ vpmovmskb %ymm1, %eax + testl %eax, %eax + jnz L(first_vec_x3) + + VPCMPEQ (VEC_SIZE * 3 + 1)(%rdi), %ymm0, %ymm1 +- vpmovmskb %ymm1, %eax ++ vpmovmskb %ymm1, %eax + testl %eax, %eax + jnz L(first_vec_x4) + + /* Align data to VEC_SIZE * 4 - 1. */ + # ifdef USE_AS_STRNLEN + /* Before adjusting length check if at last VEC_SIZE * 4. */ +- cmpq $(VEC_SIZE * 4 - 1), %rsi ++ cmpq $(CHAR_PER_VEC * 4 - 1), %rsi + jbe L(last_4x_vec_or_less_load) + incq %rdi + movl %edi, %ecx + orq $(VEC_SIZE * 4 - 1), %rdi + andl $(VEC_SIZE * 4 - 1), %ecx ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get the wchar_t count. */ ++ sarl $2, %ecx ++# endif + /* Readjust length. */ + addq %rcx, %rsi + # else +@@ -246,13 +280,13 @@ L(cross_page_continue): + L(loop_4x_vec): + # ifdef USE_AS_STRNLEN + /* Break if at end of length. */ +- subq $(VEC_SIZE * 4), %rsi ++ subq $(CHAR_PER_VEC * 4), %rsi + jb L(last_4x_vec_or_less_cmpeq) + # endif +- /* Save some code size by microfusing VPMINU with the load. Since +- the matches in ymm2/ymm4 can only be returned if there where no +- matches in ymm1/ymm3 respectively there is no issue with overlap. +- */ ++ /* Save some code size by microfusing VPMINU with the load. ++ Since the matches in ymm2/ymm4 can only be returned if there ++ where no matches in ymm1/ymm3 respectively there is no issue ++ with overlap. */ + vmovdqa 1(%rdi), %ymm1 + VPMINU (VEC_SIZE + 1)(%rdi), %ymm1, %ymm2 + vmovdqa (VEC_SIZE * 2 + 1)(%rdi), %ymm3 +@@ -260,7 +294,7 @@ L(loop_4x_vec): + + VPMINU %ymm2, %ymm4, %ymm5 + VPCMPEQ %ymm5, %ymm0, %ymm5 +- vpmovmskb %ymm5, %ecx ++ vpmovmskb %ymm5, %ecx + + subq $-(VEC_SIZE * 4), %rdi + testl %ecx, %ecx +@@ -268,27 +302,28 @@ L(loop_4x_vec): + + + VPCMPEQ %ymm1, %ymm0, %ymm1 +- vpmovmskb %ymm1, %eax ++ vpmovmskb %ymm1, %eax + subq %rdx, %rdi + testl %eax, %eax + jnz L(last_vec_return_x0) + + VPCMPEQ %ymm2, %ymm0, %ymm2 +- vpmovmskb %ymm2, %eax ++ vpmovmskb %ymm2, %eax + testl %eax, %eax + jnz L(last_vec_return_x1) + + /* Combine last 2 VEC. */ + VPCMPEQ %ymm3, %ymm0, %ymm3 +- vpmovmskb %ymm3, %eax +- /* rcx has combined result from all 4 VEC. It will only be used if +- the first 3 other VEC all did not contain a match. */ ++ vpmovmskb %ymm3, %eax ++ /* rcx has combined result from all 4 VEC. It will only be used ++ if the first 3 other VEC all did not contain a match. */ + salq $32, %rcx + orq %rcx, %rax + tzcntq %rax, %rax + subq $(VEC_SIZE * 2 - 1), %rdi + addq %rdi, %rax + # ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get wchar_t count. */ + shrq $2, %rax + # endif + VZEROUPPER_RETURN +@@ -297,15 +332,19 @@ L(loop_4x_vec): + # ifdef USE_AS_STRNLEN + .p2align 4 + L(last_4x_vec_or_less_load): +- /* Depending on entry adjust rdi / prepare first VEC in ymm1. */ ++ /* Depending on entry adjust rdi / prepare first VEC in ymm1. ++ */ + subq $-(VEC_SIZE * 4), %rdi + L(last_4x_vec_or_less_cmpeq): + VPCMPEQ 1(%rdi), %ymm0, %ymm1 + L(last_4x_vec_or_less): +- +- vpmovmskb %ymm1, %eax +- /* If remaining length > VEC_SIZE * 2. This works if esi is off by +- VEC_SIZE * 4. */ ++# ifdef USE_AS_WCSLEN ++ /* NB: Multiply length by 4 to get byte count. */ ++ sall $2, %esi ++# endif ++ vpmovmskb %ymm1, %eax ++ /* If remaining length > VEC_SIZE * 2. This works if esi is off ++ by VEC_SIZE * 4. */ + testl $(VEC_SIZE * 2), %esi + jnz L(last_4x_vec) + +@@ -320,7 +359,7 @@ L(last_4x_vec_or_less): + jb L(max) + + VPCMPEQ (VEC_SIZE + 1)(%rdi), %ymm0, %ymm1 +- vpmovmskb %ymm1, %eax ++ vpmovmskb %ymm1, %eax + tzcntl %eax, %eax + /* Check the end of data. */ + cmpl %eax, %esi +@@ -329,6 +368,7 @@ L(last_4x_vec_or_less): + addl $(VEC_SIZE + 1), %eax + addq %rdi, %rax + # ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get wchar_t count. */ + shrq $2, %rax + # endif + VZEROUPPER_RETURN +@@ -340,6 +380,7 @@ L(last_vec_return_x0): + subq $(VEC_SIZE * 4 - 1), %rdi + addq %rdi, %rax + # ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get wchar_t count. */ + shrq $2, %rax + # endif + VZEROUPPER_RETURN +@@ -350,6 +391,7 @@ L(last_vec_return_x1): + subq $(VEC_SIZE * 3 - 1), %rdi + addq %rdi, %rax + # ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get wchar_t count. */ + shrq $2, %rax + # endif + VZEROUPPER_RETURN +@@ -366,6 +408,7 @@ L(last_vec_x1_check): + incl %eax + addq %rdi, %rax + # ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get wchar_t count. */ + shrq $2, %rax + # endif + VZEROUPPER_RETURN +@@ -381,14 +424,14 @@ L(last_4x_vec): + jnz L(last_vec_x1) + + VPCMPEQ (VEC_SIZE + 1)(%rdi), %ymm0, %ymm1 +- vpmovmskb %ymm1, %eax ++ vpmovmskb %ymm1, %eax + testl %eax, %eax + jnz L(last_vec_x2) + + /* Normalize length. */ + andl $(VEC_SIZE * 4 - 1), %esi + VPCMPEQ (VEC_SIZE * 2 + 1)(%rdi), %ymm0, %ymm1 +- vpmovmskb %ymm1, %eax ++ vpmovmskb %ymm1, %eax + testl %eax, %eax + jnz L(last_vec_x3) + +@@ -396,7 +439,7 @@ L(last_4x_vec): + jb L(max) + + VPCMPEQ (VEC_SIZE * 3 + 1)(%rdi), %ymm0, %ymm1 +- vpmovmskb %ymm1, %eax ++ vpmovmskb %ymm1, %eax + tzcntl %eax, %eax + /* Check the end of data. */ + cmpl %eax, %esi +@@ -405,6 +448,7 @@ L(last_4x_vec): + addl $(VEC_SIZE * 3 + 1), %eax + addq %rdi, %rax + # ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get wchar_t count. */ + shrq $2, %rax + # endif + VZEROUPPER_RETURN +@@ -419,6 +463,7 @@ L(last_vec_x1): + incl %eax + addq %rdi, %rax + # ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get wchar_t count. */ + shrq $2, %rax + # endif + VZEROUPPER_RETURN +@@ -432,6 +477,7 @@ L(last_vec_x2): + addl $(VEC_SIZE + 1), %eax + addq %rdi, %rax + # ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get wchar_t count. */ + shrq $2, %rax + # endif + VZEROUPPER_RETURN +@@ -447,6 +493,7 @@ L(last_vec_x3): + addl $(VEC_SIZE * 2 + 1), %eax + addq %rdi, %rax + # ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get wchar_t count. */ + shrq $2, %rax + # endif + VZEROUPPER_RETURN +@@ -455,13 +502,13 @@ L(max_end): + VZEROUPPER_RETURN + # endif + +- /* Cold case for crossing page with first load. */ ++ /* Cold case for crossing page with first load. */ + .p2align 4 + L(cross_page_boundary): + /* Align data to VEC_SIZE - 1. */ + orq $(VEC_SIZE - 1), %rdi + VPCMPEQ -(VEC_SIZE - 1)(%rdi), %ymm0, %ymm1 +- vpmovmskb %ymm1, %eax ++ vpmovmskb %ymm1, %eax + /* Remove the leading bytes. sarxl only uses bits [5:0] of COUNT + so no need to manually mod rdx. */ + sarxl %edx, %eax, %eax +@@ -470,6 +517,10 @@ L(cross_page_boundary): + jnz L(cross_page_less_vec) + leaq 1(%rdi), %rcx + subq %rdx, %rcx ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get wchar_t count. */ ++ shrl $2, %ecx ++# endif + /* Check length. */ + cmpq %rsi, %rcx + jb L(cross_page_continue) +@@ -479,6 +530,7 @@ L(cross_page_boundary): + jz L(cross_page_continue) + tzcntl %eax, %eax + # ifdef USE_AS_WCSLEN ++ /* NB: Divide length by 4 to get wchar_t count. */ + shrl $2, %eax + # endif + # endif +@@ -489,6 +541,10 @@ L(return_vzeroupper): + .p2align 4 + L(cross_page_less_vec): + tzcntl %eax, %eax ++# ifdef USE_AS_WCSLEN ++ /* NB: Multiply length by 4 to get byte count. */ ++ sall $2, %esi ++# endif + cmpq %rax, %rsi + cmovb %esi, %eax + # ifdef USE_AS_WCSLEN +diff --git a/sysdeps/x86_64/multiarch/strlen-vec.S b/sysdeps/x86_64/multiarch/strlen-vec.S +index 8f660bb9..439e486a 100644 +--- a/sysdeps/x86_64/multiarch/strlen-vec.S ++++ b/sysdeps/x86_64/multiarch/strlen-vec.S +@@ -65,12 +65,25 @@ ENTRY(strlen) + ret + L(n_nonzero): + # ifdef AS_WCSLEN +- shl $2, %RSI_LP ++/* Check for overflow from maxlen * sizeof(wchar_t). If it would ++ overflow the only way this program doesn't have undefined behavior ++ is if there is a null terminator in valid memory so wcslen will ++ suffice. */ ++ mov %RSI_LP, %R10_LP ++ sar $62, %R10_LP ++ test %R10_LP, %R10_LP ++ jnz __wcslen_sse4_1 ++ sal $2, %RSI_LP + # endif + ++ + /* Initialize long lived registers. */ + + add %RDI_LP, %RSI_LP ++# ifdef AS_WCSLEN ++/* Check for overflow again from s + maxlen * sizeof(wchar_t). */ ++ jbe __wcslen_sse4_1 ++# endif + mov %RSI_LP, %R10_LP + and $-64, %R10_LP + mov %RSI_LP, %R11_LP +-- +GitLab + diff --git a/SOURCES/glibc-sw28033.patch b/SOURCES/glibc-sw28033.patch new file mode 100644 index 0000000..d6b677c --- /dev/null +++ b/SOURCES/glibc-sw28033.patch @@ -0,0 +1,153 @@ +From 81174145d5f7b76744c8f635165e88735a944fca Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Wed, 2 Mar 2022 14:45:02 -0800 +Subject: [PATCH] x86: Check RTM_ALWAYS_ABORT for RTM [BZ #28033] + +From + +https://www.intel.com/content/www/us/en/support/articles/000059422/processors.html + +* Intel TSX will be disabled by default. +* The processor will force abort all Restricted Transactional Memory (RTM) + transactions by default. +* A new CPUID bit CPUID.07H.0H.EDX[11](RTM_ALWAYS_ABORT) will be enumerated, + which is set to indicate to updated software that the loaded microcode is + forcing RTM abort. +* On processors that enumerate support for RTM, the CPUID enumeration bits + for Intel TSX (CPUID.07H.0H.EBX[11] and CPUID.07H.0H.EBX[4]) continue to + be set by default after microcode update. +* Workloads that were benefited from Intel TSX might experience a change + in performance. +* System software may use a new bit in Model-Specific Register (MSR) 0x10F + TSX_FORCE_ABORT[TSX_CPUID_CLEAR] functionality to clear the Hardware Lock + Elision (HLE) and RTM bits to indicate to software that Intel TSX is + disabled. + +1. Add RTM_ALWAYS_ABORT to CPUID features. +2. Set RTM usable only if RTM_ALWAYS_ABORT isn't set. This skips the +string/tst-memchr-rtm etc. testcases on the affected processors, which +always fail after a microcde update. +3. Check RTM feature, instead of usability, against /proc/cpuinfo. + +This fixes BZ #28033. + +(cherry picked from commit ea8e465a6b8d0f26c72bcbe453a854de3abf68ec) +--- + manual/platform.texi | 3 +++ + sysdeps/x86/cpu-features.c | 5 ++++- + sysdeps/x86/sys/platform/x86.h | 6 +++--- + sysdeps/x86/tst-cpu-features-supports.c | 2 +- + sysdeps/x86/tst-get-cpu-features.c | 2 ++ + 5 files changed, 13 insertions(+), 5 deletions(-) + +diff --git a/manual/platform.texi b/manual/platform.texi +index 8fec2933..b7e8aef7 100644 +--- a/manual/platform.texi ++++ b/manual/platform.texi +@@ -510,6 +510,9 @@ capability. + @item + @code{RTM} -- RTM instruction extensions. + ++@item ++@code{RTM_ALWAYS_ABORT} -- Transactions always abort, making RTM unusable. ++ + @item + @code{SDBG} -- IA32_DEBUG_INTERFACE MSR for silicon debug. + +diff --git a/sysdeps/x86/cpu-features.c b/sysdeps/x86/cpu-features.c +index 3610ee5c..4889f062 100644 +--- a/sysdeps/x86/cpu-features.c ++++ b/sysdeps/x86/cpu-features.c +@@ -74,7 +74,6 @@ update_usable (struct cpu_features *cpu_features) + CPU_FEATURE_SET_USABLE (cpu_features, HLE); + CPU_FEATURE_SET_USABLE (cpu_features, BMI2); + CPU_FEATURE_SET_USABLE (cpu_features, ERMS); +- CPU_FEATURE_SET_USABLE (cpu_features, RTM); + CPU_FEATURE_SET_USABLE (cpu_features, RDSEED); + CPU_FEATURE_SET_USABLE (cpu_features, ADX); + CPU_FEATURE_SET_USABLE (cpu_features, CLFLUSHOPT); +@@ -90,6 +89,7 @@ update_usable (struct cpu_features *cpu_features) + CPU_FEATURE_SET_USABLE (cpu_features, MOVDIRI); + CPU_FEATURE_SET_USABLE (cpu_features, MOVDIR64B); + CPU_FEATURE_SET_USABLE (cpu_features, FSRM); ++ CPU_FEATURE_SET_USABLE (cpu_features, RTM_ALWAYS_ABORT); + CPU_FEATURE_SET_USABLE (cpu_features, SERIALIZE); + CPU_FEATURE_SET_USABLE (cpu_features, TSXLDTRK); + CPU_FEATURE_SET_USABLE (cpu_features, LAHF64_SAHF64); +@@ -779,6 +779,9 @@ no_cpuid: + GLRO(dl_platform) = "i586"; + #endif + ++ if (!CPU_FEATURES_CPU_P (cpu_features, RTM_ALWAYS_ABORT)) ++ CPU_FEATURE_SET_USABLE (cpu_features, RTM); ++ + #if CET_ENABLED + # if HAVE_TUNABLES + TUNABLE_GET (x86_ibt, tunable_val_t *, +diff --git a/sysdeps/x86/sys/platform/x86.h b/sysdeps/x86/sys/platform/x86.h +index e5cc7c68..7a434926 100644 +--- a/sysdeps/x86/sys/platform/x86.h ++++ b/sysdeps/x86/sys/platform/x86.h +@@ -247,7 +247,7 @@ extern const struct cpu_features *__x86_get_cpu_features (unsigned int) + #define bit_cpu_AVX512_VP2INTERSECT (1u << 8) + #define bit_cpu_INDEX_7_EDX_9 (1u << 9) + #define bit_cpu_MD_CLEAR (1u << 10) +-#define bit_cpu_INDEX_7_EDX_11 (1u << 11) ++#define bit_cpu_RTM_ALWAYS_ABORT (1u << 11) + #define bit_cpu_INDEX_7_EDX_12 (1u << 12) + #define bit_cpu_INDEX_7_EDX_13 (1u << 13) + #define bit_cpu_SERIALIZE (1u << 14) +@@ -471,7 +471,7 @@ extern const struct cpu_features *__x86_get_cpu_features (unsigned int) + #define index_cpu_AVX512_VP2INTERSECT COMMON_CPUID_INDEX_7 + #define index_cpu_INDEX_7_EDX_9 COMMON_CPUID_INDEX_7 + #define index_cpu_MD_CLEAR COMMON_CPUID_INDEX_7 +-#define index_cpu_INDEX_7_EDX_11 COMMON_CPUID_INDEX_7 ++#define index_cpu_RTM_ALWAYS_ABORT COMMON_CPUID_INDEX_7 + #define index_cpu_INDEX_7_EDX_12 COMMON_CPUID_INDEX_7 + #define index_cpu_INDEX_7_EDX_13 COMMON_CPUID_INDEX_7 + #define index_cpu_SERIALIZE COMMON_CPUID_INDEX_7 +@@ -695,7 +695,7 @@ extern const struct cpu_features *__x86_get_cpu_features (unsigned int) + #define reg_AVX512_VP2INTERSECT edx + #define reg_INDEX_7_EDX_9 edx + #define reg_MD_CLEAR edx +-#define reg_INDEX_7_EDX_11 edx ++#define reg_RTM_ALWAYS_ABORT edx + #define reg_INDEX_7_EDX_12 edx + #define reg_INDEX_7_EDX_13 edx + #define reg_SERIALIZE edx +diff --git a/sysdeps/x86/tst-cpu-features-supports.c b/sysdeps/x86/tst-cpu-features-supports.c +index 287cf01f..8100a319 100644 +--- a/sysdeps/x86/tst-cpu-features-supports.c ++++ b/sysdeps/x86/tst-cpu-features-supports.c +@@ -152,7 +152,7 @@ do_test (int argc, char **argv) + fails += CHECK_SUPPORTS (rdpid, RDPID); + fails += CHECK_SUPPORTS (rdrnd, RDRAND); + fails += CHECK_SUPPORTS (rdseed, RDSEED); +- fails += CHECK_SUPPORTS (rtm, RTM); ++ fails += CHECK_CPU_SUPPORTS (rtm, RTM); + fails += CHECK_SUPPORTS (serialize, SERIALIZE); + fails += CHECK_SUPPORTS (sha, SHA); + fails += CHECK_CPU_SUPPORTS (shstk, SHSTK); +diff --git a/sysdeps/x86/tst-get-cpu-features.c b/sysdeps/x86/tst-get-cpu-features.c +index 2763deb6..0717e5d8 100644 +--- a/sysdeps/x86/tst-get-cpu-features.c ++++ b/sysdeps/x86/tst-get-cpu-features.c +@@ -183,6 +183,7 @@ do_test (void) + CHECK_CPU_FEATURE (UINTR); + CHECK_CPU_FEATURE (AVX512_VP2INTERSECT); + CHECK_CPU_FEATURE (MD_CLEAR); ++ CHECK_CPU_FEATURE (RTM_ALWAYS_ABORT); + CHECK_CPU_FEATURE (SERIALIZE); + CHECK_CPU_FEATURE (HYBRID); + CHECK_CPU_FEATURE (TSXLDTRK); +@@ -344,6 +345,7 @@ do_test (void) + CHECK_CPU_FEATURE_USABLE (FSRM); + CHECK_CPU_FEATURE_USABLE (AVX512_VP2INTERSECT); + CHECK_CPU_FEATURE_USABLE (MD_CLEAR); ++ CHECK_CPU_FEATURE_USABLE (RTM_ALWAYS_ABORT); + CHECK_CPU_FEATURE_USABLE (SERIALIZE); + CHECK_CPU_FEATURE_USABLE (HYBRID); + CHECK_CPU_FEATURE_USABLE (TSXLDTRK); +-- +GitLab + diff --git a/SOURCES/glibc-sw28064.patch b/SOURCES/glibc-sw28064.patch new file mode 100644 index 0000000..edc3b0d --- /dev/null +++ b/SOURCES/glibc-sw28064.patch @@ -0,0 +1,51 @@ +From 88405b75deedca35e9fbccc8a39368fcf0e58783 Mon Sep 17 00:00:00 2001 +From: Noah Goldstein +Date: Thu, 8 Jul 2021 16:13:19 -0400 +Subject: [PATCH] x86: Remove wcsnlen-sse4_1 from wcslen ifunc-impl-list [BZ + #28064] + +The following commit + +commit 6f573a27b6c8b4236445810a44660612323f5a73 +Author: Noah Goldstein +Date: Wed Jun 23 01:19:34 2021 -0400 + + x86-64: Add wcslen optimize for sse4.1 + +Added wcsnlen-sse4.1 to the wcslen ifunc implementation list and did +not add wcslen-sse4.1 to wcslen ifunc implementation list. This commit +fixes that by removing wcsnlen-sse4.1 from the wcslen ifunc +implementation list and adding wcslen-sse4.1 to the ifunc +implementation list. + +Testing: +test-wcslen.c, test-rsi-wcslen.c, and test-rsi-strlen.c are passing as +well as all other tests in wcsmbs and string. + +Signed-off-by: Noah Goldstein +Reviewed-by: H.J. Lu +Reviewed-by: H.J. Lu +(cherry picked from commit 0679442defedf7e52a94264975880ab8674736b2) +--- + sysdeps/x86_64/multiarch/ifunc-impl-list.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c +index 580913ca..695cdba6 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c ++++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c +@@ -657,9 +657,9 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + && CPU_FEATURE_USABLE (AVX512BW) + && CPU_FEATURE_USABLE (BMI2)), + __wcslen_evex) +- IFUNC_IMPL_ADD (array, i, wcsnlen, ++ IFUNC_IMPL_ADD (array, i, wcslen, + CPU_FEATURE_USABLE (SSE4_1), +- __wcsnlen_sse4_1) ++ __wcslen_sse4_1) + IFUNC_IMPL_ADD (array, i, wcslen, 1, __wcslen_sse2)) + + /* Support sysdeps/x86_64/multiarch/wcsnlen.c. */ +-- +GitLab + diff --git a/SOURCES/glibc-sw28252.patch b/SOURCES/glibc-sw28252.patch new file mode 100644 index 0000000..0632632 --- /dev/null +++ b/SOURCES/glibc-sw28252.patch @@ -0,0 +1,269 @@ +From c166f44e4488af4f4af035645775fe44b12bab13 Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Fri, 20 Aug 2021 06:42:24 -0700 +Subject: [PATCH] x86-64: Optimize load of all bits set into ZMM register [BZ + #28252] + +Optimize loads of all bits set into ZMM register in AVX512 SVML codes +by replacing + + vpbroadcastq .L_2il0floatpacket.16(%rip), %zmmX + +and + + vmovups .L_2il0floatpacket.13(%rip), %zmmX + +with + vpternlogd $0xff, %zmmX, %zmmX, %zmmX + +This fixes BZ #28252. + +(cherry picked from commit 78c9ec9000f873abe7a15a91b87080a2e4308260) +--- + .../x86_64/fpu/multiarch/svml_d_cos8_core_avx512.S | 7 +------ + .../x86_64/fpu/multiarch/svml_d_log8_core_avx512.S | 7 +------ + .../x86_64/fpu/multiarch/svml_d_sin8_core_avx512.S | 7 +------ + .../fpu/multiarch/svml_d_sincos8_core_avx512.S | 7 +------ + .../x86_64/fpu/multiarch/svml_s_cosf16_core_avx512.S | 7 +------ + .../x86_64/fpu/multiarch/svml_s_expf16_core_avx512.S | 7 +------ + .../x86_64/fpu/multiarch/svml_s_logf16_core_avx512.S | 7 +------ + .../x86_64/fpu/multiarch/svml_s_powf16_core_avx512.S | 12 ++---------- + .../fpu/multiarch/svml_s_sincosf16_core_avx512.S | 7 +------ + .../x86_64/fpu/multiarch/svml_s_sinf16_core_avx512.S | 7 +------ + 10 files changed, 11 insertions(+), 64 deletions(-) + +diff --git a/sysdeps/x86_64/fpu/multiarch/svml_d_cos8_core_avx512.S b/sysdeps/x86_64/fpu/multiarch/svml_d_cos8_core_avx512.S +index 24e3b363..07dfed85 100644 +--- a/sysdeps/x86_64/fpu/multiarch/svml_d_cos8_core_avx512.S ++++ b/sysdeps/x86_64/fpu/multiarch/svml_d_cos8_core_avx512.S +@@ -265,7 +265,7 @@ WRAPPER_IMPL_AVX512 _ZGVdN4v_cos + vmovaps %zmm0, %zmm8 + + /* Check for large arguments path */ +- vpbroadcastq .L_2il0floatpacket.16(%rip), %zmm2 ++ vpternlogd $0xff, %zmm2, %zmm2, %zmm2 + + /* + ARGUMENT RANGE REDUCTION: +@@ -456,8 +456,3 @@ WRAPPER_IMPL_AVX512 _ZGVdN4v_cos + jmp .LBL_2_7 + #endif + END (_ZGVeN8v_cos_skx) +- +- .section .rodata, "a" +-.L_2il0floatpacket.16: +- .long 0xffffffff,0xffffffff +- .type .L_2il0floatpacket.16,@object +diff --git a/sysdeps/x86_64/fpu/multiarch/svml_d_log8_core_avx512.S b/sysdeps/x86_64/fpu/multiarch/svml_d_log8_core_avx512.S +index ae8af8d8..ddb60e5b 100644 +--- a/sysdeps/x86_64/fpu/multiarch/svml_d_log8_core_avx512.S ++++ b/sysdeps/x86_64/fpu/multiarch/svml_d_log8_core_avx512.S +@@ -274,7 +274,7 @@ WRAPPER_IMPL_AVX512 _ZGVdN4v_log + + /* preserve mantissa, set input exponent to 2^(-10) */ + vpternlogq $248, _ExpMask(%rax), %zmm3, %zmm2 +- vpbroadcastq .L_2il0floatpacket.12(%rip), %zmm1 ++ vpternlogd $0xff, %zmm1, %zmm1, %zmm1 + vpsrlq $32, %zmm4, %zmm6 + + /* reciprocal approximation good to at least 11 bits */ +@@ -461,8 +461,3 @@ WRAPPER_IMPL_AVX512 _ZGVdN4v_log + jmp .LBL_2_7 + #endif + END (_ZGVeN8v_log_skx) +- +- .section .rodata, "a" +-.L_2il0floatpacket.12: +- .long 0xffffffff,0xffffffff +- .type .L_2il0floatpacket.12,@object +diff --git a/sysdeps/x86_64/fpu/multiarch/svml_d_sin8_core_avx512.S b/sysdeps/x86_64/fpu/multiarch/svml_d_sin8_core_avx512.S +index 2d4b14fd..529c454a 100644 +--- a/sysdeps/x86_64/fpu/multiarch/svml_d_sin8_core_avx512.S ++++ b/sysdeps/x86_64/fpu/multiarch/svml_d_sin8_core_avx512.S +@@ -261,7 +261,7 @@ WRAPPER_IMPL_AVX512 _ZGVdN4v_sin + andq $-64, %rsp + subq $1280, %rsp + movq __svml_d_trig_data@GOTPCREL(%rip), %rax +- vpbroadcastq .L_2il0floatpacket.14(%rip), %zmm14 ++ vpternlogd $0xff, %zmm1, %zmm1, %zmm14 + vmovups __dAbsMask(%rax), %zmm7 + vmovups __dInvPI(%rax), %zmm2 + vmovups __dRShifter(%rax), %zmm1 +@@ -458,8 +458,3 @@ WRAPPER_IMPL_AVX512 _ZGVdN4v_sin + jmp .LBL_2_7 + #endif + END (_ZGVeN8v_sin_skx) +- +- .section .rodata, "a" +-.L_2il0floatpacket.14: +- .long 0xffffffff,0xffffffff +- .type .L_2il0floatpacket.14,@object +diff --git a/sysdeps/x86_64/fpu/multiarch/svml_d_sincos8_core_avx512.S b/sysdeps/x86_64/fpu/multiarch/svml_d_sincos8_core_avx512.S +index 2df626c0..e501a53a 100644 +--- a/sysdeps/x86_64/fpu/multiarch/svml_d_sincos8_core_avx512.S ++++ b/sysdeps/x86_64/fpu/multiarch/svml_d_sincos8_core_avx512.S +@@ -430,7 +430,7 @@ WRAPPER_IMPL_AVX512_fFF _ZGVdN4vl8l8_sincos + + /* SinPoly = SinR*SinPoly */ + vfmadd213pd %zmm5, %zmm5, %zmm4 +- vpbroadcastq .L_2il0floatpacket.15(%rip), %zmm3 ++ vpternlogd $0xff, %zmm3, %zmm3, %zmm3 + + /* Update Cos result's sign */ + vxorpd %zmm2, %zmm1, %zmm1 +@@ -741,8 +741,3 @@ END (_ZGVeN8vvv_sincos_knl) + ENTRY (_ZGVeN8vvv_sincos_skx) + WRAPPER_AVX512_vvv_vl8l8 _ZGVeN8vl8l8_sincos_skx + END (_ZGVeN8vvv_sincos_skx) +- +- .section .rodata, "a" +-.L_2il0floatpacket.15: +- .long 0xffffffff,0xffffffff +- .type .L_2il0floatpacket.15,@object +diff --git a/sysdeps/x86_64/fpu/multiarch/svml_s_cosf16_core_avx512.S b/sysdeps/x86_64/fpu/multiarch/svml_s_cosf16_core_avx512.S +index 6ea1137b..377af394 100644 +--- a/sysdeps/x86_64/fpu/multiarch/svml_s_cosf16_core_avx512.S ++++ b/sysdeps/x86_64/fpu/multiarch/svml_s_cosf16_core_avx512.S +@@ -278,7 +278,7 @@ WRAPPER_IMPL_AVX512 _ZGVdN8v_cosf + X = X - Y*PI1 - Y*PI2 - Y*PI3 + */ + vmovaps %zmm0, %zmm6 +- vmovups .L_2il0floatpacket.13(%rip), %zmm12 ++ vpternlogd $0xff, %zmm12, %zmm12, %zmm12 + vmovups __sRShifter(%rax), %zmm3 + vmovups __sPI1_FMA(%rax), %zmm5 + vmovups __sA9_FMA(%rax), %zmm9 +@@ -453,8 +453,3 @@ WRAPPER_IMPL_AVX512 _ZGVdN8v_cosf + jmp .LBL_2_7 + #endif + END (_ZGVeN16v_cosf_skx) +- +- .section .rodata, "a" +-.L_2il0floatpacket.13: +- .long 0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff +- .type .L_2il0floatpacket.13,@object +diff --git a/sysdeps/x86_64/fpu/multiarch/svml_s_expf16_core_avx512.S b/sysdeps/x86_64/fpu/multiarch/svml_s_expf16_core_avx512.S +index 89ba0df2..46f33d46 100644 +--- a/sysdeps/x86_64/fpu/multiarch/svml_s_expf16_core_avx512.S ++++ b/sysdeps/x86_64/fpu/multiarch/svml_s_expf16_core_avx512.S +@@ -264,7 +264,7 @@ WRAPPER_IMPL_AVX512 _ZGVdN8v_expf + vmovaps %zmm0, %zmm7 + + /* compare against threshold */ +- vmovups .L_2il0floatpacket.13(%rip), %zmm3 ++ vpternlogd $0xff, %zmm3, %zmm3, %zmm3 + vmovups __sInvLn2(%rax), %zmm4 + vmovups __sShifter(%rax), %zmm1 + vmovups __sLn2hi(%rax), %zmm6 +@@ -440,8 +440,3 @@ WRAPPER_IMPL_AVX512 _ZGVdN8v_expf + + #endif + END (_ZGVeN16v_expf_skx) +- +- .section .rodata, "a" +-.L_2il0floatpacket.13: +- .long 0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff +- .type .L_2il0floatpacket.13,@object +diff --git a/sysdeps/x86_64/fpu/multiarch/svml_s_logf16_core_avx512.S b/sysdeps/x86_64/fpu/multiarch/svml_s_logf16_core_avx512.S +index 4cf0a96f..9e254956 100644 +--- a/sysdeps/x86_64/fpu/multiarch/svml_s_logf16_core_avx512.S ++++ b/sysdeps/x86_64/fpu/multiarch/svml_s_logf16_core_avx512.S +@@ -235,7 +235,7 @@ WRAPPER_IMPL_AVX512 _ZGVdN8v_logf + andq $-64, %rsp + subq $1280, %rsp + movq __svml_slog_data@GOTPCREL(%rip), %rax +- vmovups .L_2il0floatpacket.7(%rip), %zmm6 ++ vpternlogd $0xff, %zmm6, %zmm6, %zmm6 + vmovups _iBrkValue(%rax), %zmm4 + vmovups _sPoly_7(%rax), %zmm8 + +@@ -409,8 +409,3 @@ WRAPPER_IMPL_AVX512 _ZGVdN8v_logf + + #endif + END (_ZGVeN16v_logf_skx) +- +- .section .rodata, "a" +-.L_2il0floatpacket.7: +- .long 0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff +- .type .L_2il0floatpacket.7,@object +diff --git a/sysdeps/x86_64/fpu/multiarch/svml_s_powf16_core_avx512.S b/sysdeps/x86_64/fpu/multiarch/svml_s_powf16_core_avx512.S +index bdcd50af..e8331ba1 100644 +--- a/sysdeps/x86_64/fpu/multiarch/svml_s_powf16_core_avx512.S ++++ b/sysdeps/x86_64/fpu/multiarch/svml_s_powf16_core_avx512.S +@@ -385,7 +385,7 @@ WRAPPER_IMPL_AVX512_ff _ZGVdN8vv_powf + vpsrlq $32, %zmm3, %zmm2 + vpmovqd %zmm2, %ymm11 + vcvtps2pd %ymm14, %zmm13 +- vmovups .L_2il0floatpacket.23(%rip), %zmm14 ++ vpternlogd $0xff, %zmm14, %zmm14, %zmm14 + vmovaps %zmm14, %zmm26 + vpandd _ABSMASK(%rax), %zmm1, %zmm8 + vpcmpd $1, _INF(%rax), %zmm8, %k2 +@@ -427,7 +427,7 @@ WRAPPER_IMPL_AVX512_ff _ZGVdN8vv_powf + vpmovqd %zmm11, %ymm5 + vpxord %zmm10, %zmm10, %zmm10 + vgatherdpd _Log2Rcp_lookup(%rax,%ymm4), %zmm10{%k3} +- vpbroadcastq .L_2il0floatpacket.24(%rip), %zmm4 ++ vpternlogd $0xff, %zmm4, %zmm4, %zmm4 + vpxord %zmm11, %zmm11, %zmm11 + vcvtdq2pd %ymm7, %zmm7 + vgatherdpd _Log2Rcp_lookup(%rax,%ymm5), %zmm11{%k1} +@@ -643,11 +643,3 @@ WRAPPER_IMPL_AVX512_ff _ZGVdN8vv_powf + jmp .LBL_2_7 + #endif + END (_ZGVeN16vv_powf_skx) +- +- .section .rodata, "a" +-.L_2il0floatpacket.23: +- .long 0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff +- .type .L_2il0floatpacket.23,@object +-.L_2il0floatpacket.24: +- .long 0xffffffff,0xffffffff +- .type .L_2il0floatpacket.24,@object +diff --git a/sysdeps/x86_64/fpu/multiarch/svml_s_sincosf16_core_avx512.S b/sysdeps/x86_64/fpu/multiarch/svml_s_sincosf16_core_avx512.S +index 5fa4bc41..1f46f334 100644 +--- a/sysdeps/x86_64/fpu/multiarch/svml_s_sincosf16_core_avx512.S ++++ b/sysdeps/x86_64/fpu/multiarch/svml_s_sincosf16_core_avx512.S +@@ -317,7 +317,7 @@ WRAPPER_IMPL_AVX512_fFF _ZGVdN8vvv_sincosf + + /* Result sign calculations */ + vpternlogd $150, %zmm0, %zmm14, %zmm1 +- vmovups .L_2il0floatpacket.13(%rip), %zmm14 ++ vpternlogd $0xff, %zmm14, %zmm14, %zmm14 + + /* Add correction term 0.5 for cos() part */ + vaddps %zmm8, %zmm5, %zmm15 +@@ -748,8 +748,3 @@ END (_ZGVeN16vvv_sincosf_knl) + ENTRY (_ZGVeN16vvv_sincosf_skx) + WRAPPER_AVX512_vvv_vl4l4 _ZGVeN16vl4l4_sincosf_skx + END (_ZGVeN16vvv_sincosf_skx) +- +- .section .rodata, "a" +-.L_2il0floatpacket.13: +- .long 0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff +- .type .L_2il0floatpacket.13,@object +diff --git a/sysdeps/x86_64/fpu/multiarch/svml_s_sinf16_core_avx512.S b/sysdeps/x86_64/fpu/multiarch/svml_s_sinf16_core_avx512.S +index 141f747e..1fc9308a 100644 +--- a/sysdeps/x86_64/fpu/multiarch/svml_s_sinf16_core_avx512.S ++++ b/sysdeps/x86_64/fpu/multiarch/svml_s_sinf16_core_avx512.S +@@ -280,7 +280,7 @@ WRAPPER_IMPL_AVX512 _ZGVdN8v_sinf + movq __svml_s_trig_data@GOTPCREL(%rip), %rax + + /* Check for large and special values */ +- vmovups .L_2il0floatpacket.11(%rip), %zmm14 ++ vpternlogd $0xff, %zmm14, %zmm14, %zmm14 + vmovups __sAbsMask(%rax), %zmm5 + vmovups __sInvPI(%rax), %zmm1 + vmovups __sRShifter(%rax), %zmm2 +@@ -472,8 +472,3 @@ WRAPPER_IMPL_AVX512 _ZGVdN8v_sinf + jmp .LBL_2_7 + #endif + END (_ZGVeN16v_sinf_skx) +- +- .section .rodata, "a" +-.L_2il0floatpacket.11: +- .long 0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff +- .type .L_2il0floatpacket.11,@object +-- +GitLab + diff --git a/SOURCES/glibc-sw28537-1.patch b/SOURCES/glibc-sw28537-1.patch new file mode 100644 index 0000000..31fdf34 --- /dev/null +++ b/SOURCES/glibc-sw28537-1.patch @@ -0,0 +1,39 @@ +From ca37f880abe1c8e3464fd646fd8b89171d8fca0c Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Thu, 11 Nov 2021 06:31:51 -0800 +Subject: [PATCH] Avoid extra load with CAS in __pthread_mutex_lock_full [BZ + #28537] + +Replace boolean CAS with value CAS to avoid the extra load. + +Reviewed-by: Szabolcs Nagy +(cherry picked from commit 0b82747dc48d5bf0871bdc6da8cb6eec1256355f) +--- + nptl/pthread_mutex_lock.c | 10 +++++----- + 1 file changed, 5 insertions(+), 5 deletions(-) + +diff --git a/nptl/pthread_mutex_lock.c b/nptl/pthread_mutex_lock.c +index 29cc143e..60ada70d 100644 +--- a/nptl/pthread_mutex_lock.c ++++ b/nptl/pthread_mutex_lock.c +@@ -292,12 +292,12 @@ __pthread_mutex_lock_full (pthread_mutex_t *mutex) + meantime. */ + if ((oldval & FUTEX_WAITERS) == 0) + { +- if (atomic_compare_and_exchange_bool_acq (&mutex->__data.__lock, +- oldval | FUTEX_WAITERS, +- oldval) +- != 0) ++ int val; ++ if ((val = atomic_compare_and_exchange_val_acq ++ (&mutex->__data.__lock, oldval | FUTEX_WAITERS, ++ oldval)) != oldval) + { +- oldval = mutex->__data.__lock; ++ oldval = val; + continue; + } + oldval |= FUTEX_WAITERS; +-- +GitLab + diff --git a/SOURCES/glibc-sw28537-2.patch b/SOURCES/glibc-sw28537-2.patch new file mode 100644 index 0000000..1e477b9 --- /dev/null +++ b/SOURCES/glibc-sw28537-2.patch @@ -0,0 +1,39 @@ +From 56212a35fe258f45f242f4bb80866e4980e4ec0e Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Thu, 11 Nov 2021 06:54:01 -0800 +Subject: [PATCH] Avoid extra load with CAS in __pthread_mutex_clocklock_common + [BZ #28537] + +Replace boolean CAS with value CAS to avoid the extra load. + +Reviewed-by: Szabolcs Nagy +(cherry picked from commit 49302b8fdf9103b6fc0a398678668a22fa19574c) +--- + nptl/pthread_mutex_timedlock.c | 10 +++++----- + 1 file changed, 5 insertions(+), 5 deletions(-) + +diff --git a/nptl/pthread_mutex_timedlock.c b/nptl/pthread_mutex_timedlock.c +index 888c12fe..c4627ef6 100644 +--- a/nptl/pthread_mutex_timedlock.c ++++ b/nptl/pthread_mutex_timedlock.c +@@ -269,12 +269,12 @@ __pthread_mutex_timedlock (pthread_mutex_t *mutex, + meantime. */ + if ((oldval & FUTEX_WAITERS) == 0) + { +- if (atomic_compare_and_exchange_bool_acq (&mutex->__data.__lock, +- oldval | FUTEX_WAITERS, +- oldval) +- != 0) ++ int val; ++ if ((val = atomic_compare_and_exchange_val_acq ++ (&mutex->__data.__lock, oldval | FUTEX_WAITERS, ++ oldval)) != oldval) + { +- oldval = mutex->__data.__lock; ++ oldval = val; + continue; + } + oldval |= FUTEX_WAITERS; +-- +GitLab + diff --git a/SOURCES/glibc-sw28537-3.patch b/SOURCES/glibc-sw28537-3.patch new file mode 100644 index 0000000..422dcc9 --- /dev/null +++ b/SOURCES/glibc-sw28537-3.patch @@ -0,0 +1,51 @@ +From 2fa2ea233bec906b682fc82376649a1a6e18e9df Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Tue, 2 Nov 2021 18:33:07 -0700 +Subject: [PATCH] Add LLL_MUTEX_READ_LOCK [BZ #28537] + +CAS instruction is expensive. From the x86 CPU's point of view, getting +a cache line for writing is more expensive than reading. See Appendix +A.2 Spinlock in: + +https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/xeon-lock-scaling-analysis-paper.pdf + +The full compare and swap will grab the cache line exclusive and cause +excessive cache line bouncing. + +Add LLL_MUTEX_READ_LOCK to do an atomic load and skip CAS in spinlock +loop if compare may fail to reduce cache line bouncing on contended locks. + +Reviewed-by: Szabolcs Nagy +(cherry picked from commit d672a98a1af106bd68deb15576710cd61363f7a6) +--- + nptl/pthread_mutex_lock.c | 7 +++++++ + 1 file changed, 7 insertions(+) + +diff --git a/nptl/pthread_mutex_lock.c b/nptl/pthread_mutex_lock.c +index 60ada70d..eb4d8baa 100644 +--- a/nptl/pthread_mutex_lock.c ++++ b/nptl/pthread_mutex_lock.c +@@ -56,6 +56,11 @@ + #define FORCE_ELISION(m, s) + #endif + ++#ifndef LLL_MUTEX_READ_LOCK ++# define LLL_MUTEX_READ_LOCK(mutex) \ ++ atomic_load_relaxed (&(mutex)->__data.__lock) ++#endif ++ + static int __pthread_mutex_lock_full (pthread_mutex_t *mutex) + __attribute_noinline__; + +@@ -136,6 +141,8 @@ __pthread_mutex_lock (pthread_mutex_t *mutex) + break; + } + atomic_spin_nop (); ++ if (LLL_MUTEX_READ_LOCK (mutex) != 0) ++ continue; + } + while (LLL_MUTEX_TRYLOCK (mutex) != 0); + +-- +GitLab + diff --git a/SOURCES/glibc-sw28537-4.patch b/SOURCES/glibc-sw28537-4.patch new file mode 100644 index 0000000..08457cb --- /dev/null +++ b/SOURCES/glibc-sw28537-4.patch @@ -0,0 +1,43 @@ +From bae6e60ec32acdbc5e61d94d6e222e456b796054 Mon Sep 17 00:00:00 2001 +From: Jangwoong Kim <6812skiii@gmail.com> +Date: Tue, 14 Dec 2021 21:30:51 +0900 +Subject: [PATCH] nptl: Effectively skip CAS in spinlock loop + +The commit: +"Add LLL_MUTEX_READ_LOCK [BZ #28537]" +SHA1: d672a98a1af106bd68deb15576710cd61363f7a6 + +introduced LLL_MUTEX_READ_LOCK, to skip CAS in spinlock loop +if atomic load fails. But, "continue" inside of do-while loop +does not skip the evaluation of escape expression, thus CAS +is not skipped. + +Replace do-while with while and skip LLL_MUTEX_TRYLOCK if +LLL_MUTEX_READ_LOCK fails. + +Reviewed-by: H.J. Lu +(cherry picked from commit 6b8dbbd03ac88f169b65b5c7d7278576a11d2e44) +--- + nptl/pthread_mutex_lock.c | 5 ++--- + 1 file changed, 2 insertions(+), 3 deletions(-) + +diff --git a/nptl/pthread_mutex_lock.c b/nptl/pthread_mutex_lock.c +index a633d95e..d96a9933 100644 +--- a/nptl/pthread_mutex_lock.c ++++ b/nptl/pthread_mutex_lock.c +@@ -141,10 +141,9 @@ __pthread_mutex_lock (pthread_mutex_t *mutex) + break; + } + atomic_spin_nop (); +- if (LLL_MUTEX_READ_LOCK (mutex) != 0) +- continue; + } +- while (LLL_MUTEX_TRYLOCK (mutex) != 0); ++ while (LLL_MUTEX_READ_LOCK (mutex) != 0 ++ || LLL_MUTEX_TRYLOCK (mutex) != 0); + + mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8; + } +-- +GitLab + diff --git a/SOURCES/glibc-sw28646.patch b/SOURCES/glibc-sw28646.patch new file mode 100644 index 0000000..c0e2f6f --- /dev/null +++ b/SOURCES/glibc-sw28646.patch @@ -0,0 +1,56 @@ +From 3c819146751bedbca46f5f089a786d7b5595baa8 Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Wed, 2 Mar 2022 16:30:33 -0800 +Subject: [PATCH] x86-64: Use notl in EVEX strcmp [BZ #28646] + +Must use notl %edi here as lower bits are for CHAR comparisons +potentially out of range thus can be 0 without indicating mismatch. +This fixes BZ #28646. + +Co-Authored-By: H.J. Lu +(cherry picked from commit 4df1fa6ddc8925a75f3da644d5da3bb16eb33f02) +--- + sysdeps/x86_64/multiarch/strcmp-evex.S | 14 ++++++++------ + 1 file changed, 8 insertions(+), 6 deletions(-) + +diff --git a/sysdeps/x86_64/multiarch/strcmp-evex.S b/sysdeps/x86_64/multiarch/strcmp-evex.S +index 82f12ac8..6f5c4bf9 100644 +--- a/sysdeps/x86_64/multiarch/strcmp-evex.S ++++ b/sysdeps/x86_64/multiarch/strcmp-evex.S +@@ -656,12 +656,13 @@ L(loop_cross_page): + in YMM3 and 32 bytes at VEC_SIZE(%rdx, %r10). */ + VPCMP $0, VEC_SIZE(%rdx, %r10), %YMM3, %k3{%k4} + kmovd %k3, %edi ++ /* Must use notl %edi here as lower bits are for CHAR ++ comparisons potentially out of range thus can be 0 without ++ indicating mismatch. */ ++ notl %edi + # ifdef USE_AS_WCSCMP + /* Don't use subl since it is the upper 8 bits of EDI below. */ +- notl %edi + andl $0xff, %edi +-# else +- incl %edi + # endif + + # ifdef USE_AS_WCSCMP +@@ -743,12 +744,13 @@ L(loop_cross_page_2_vec): + in YMM1 and 32 bytes at (VEC_SIZE * 3)(%rdx, %r10). */ + VPCMP $0, (VEC_SIZE * 3)(%rdx, %r10), %YMM1, %k3{%k4} + kmovd %k3, %edi ++ /* Must use notl %edi here as lower bits are for CHAR ++ comparisons potentially out of range thus can be 0 without ++ indicating mismatch. */ ++ notl %edi + # ifdef USE_AS_WCSCMP + /* Don't use subl since it is the upper 8 bits of EDI below. */ +- notl %edi + andl $0xff, %edi +-# else +- incl %edi + # endif + + # ifdef USE_AS_WCSCMP +-- +GitLab + diff --git a/SOURCES/glibc-sw28755-1.patch b/SOURCES/glibc-sw28755-1.patch new file mode 100644 index 0000000..5b9e539 --- /dev/null +++ b/SOURCES/glibc-sw28755-1.patch @@ -0,0 +1,41 @@ +From 62bbd92e0ffb25a1e31b3d6b0f03a307109320c1 Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Wed, 2 Mar 2022 12:42:18 -0800 +Subject: [PATCH] x86: Fix __wcsncmp_avx2 in strcmp-avx2.S [BZ# 28755] + +Fixes [BZ# 28755] for wcsncmp by redirecting length >= 2^56 to +__wcscmp_avx2. For x86_64 this covers the entire address range so any +length larger could not possibly be used to bound `s1` or `s2`. + +test-strcmp, test-strncmp, test-wcscmp, and test-wcsncmp all pass. + +Signed-off-by: Noah Goldstein +(cherry picked from commit ddf0992cf57a93200e0c782e2a94d0733a5a0b87) +--- + sysdeps/x86_64/multiarch/strcmp-avx2.S | 10 ++++++++++ + 1 file changed, 10 insertions(+) + +diff --git a/sysdeps/x86_64/multiarch/strcmp-avx2.S b/sysdeps/x86_64/multiarch/strcmp-avx2.S +index 156c1949..8fb8eedc 100644 +--- a/sysdeps/x86_64/multiarch/strcmp-avx2.S ++++ b/sysdeps/x86_64/multiarch/strcmp-avx2.S +@@ -83,6 +83,16 @@ ENTRY (STRCMP) + je L(char0) + jb L(zero) + # ifdef USE_AS_WCSCMP ++# ifndef __ILP32__ ++ movq %rdx, %rcx ++ /* Check if length could overflow when multiplied by ++ sizeof(wchar_t). Checking top 8 bits will cover all potential ++ overflow cases as well as redirect cases where its impossible to ++ length to bound a valid memory region. In these cases just use ++ 'wcscmp'. */ ++ shrq $56, %rcx ++ jnz __wcscmp_avx2 ++# endif + /* Convert units: from wide to byte char. */ + shl $2, %RDX_LP + # endif +-- +GitLab + diff --git a/SOURCES/glibc-sw28755-2.patch b/SOURCES/glibc-sw28755-2.patch new file mode 100644 index 0000000..4b87cb6 --- /dev/null +++ b/SOURCES/glibc-sw28755-2.patch @@ -0,0 +1,41 @@ +From edb82a1c8cb1e29dae2fa897013aa5f74edc537e Mon Sep 17 00:00:00 2001 +From: Noah Goldstein +Date: Sun, 9 Jan 2022 16:02:28 -0600 +Subject: [PATCH] x86: Fix __wcsncmp_evex in strcmp-evex.S [BZ# 28755] + +Fixes [BZ# 28755] for wcsncmp by redirecting length >= 2^56 to +__wcscmp_evex. For x86_64 this covers the entire address range so any +length larger could not possibly be used to bound `s1` or `s2`. + +test-strcmp, test-strncmp, test-wcscmp, and test-wcsncmp all pass. + +Signed-off-by: Noah Goldstein +(cherry picked from commit 7e08db3359c86c94918feb33a1182cd0ff3bb10b) +--- + sysdeps/x86_64/multiarch/strcmp-evex.S | 10 ++++++++++ + 1 file changed, 10 insertions(+) + +diff --git a/sysdeps/x86_64/multiarch/strcmp-evex.S b/sysdeps/x86_64/multiarch/strcmp-evex.S +index 459eeed0..d5aa6daa 100644 +--- a/sysdeps/x86_64/multiarch/strcmp-evex.S ++++ b/sysdeps/x86_64/multiarch/strcmp-evex.S +@@ -97,6 +97,16 @@ ENTRY (STRCMP) + je L(char0) + jb L(zero) + # ifdef USE_AS_WCSCMP ++# ifndef __ILP32__ ++ movq %rdx, %rcx ++ /* Check if length could overflow when multiplied by ++ sizeof(wchar_t). Checking top 8 bits will cover all potential ++ overflow cases as well as redirect cases where its impossible to ++ length to bound a valid memory region. In these cases just use ++ 'wcscmp'. */ ++ shrq $56, %rcx ++ jnz __wcscmp_evex ++# endif + /* Convert units: from wide to byte char. */ + shl $2, %RDX_LP + # endif +-- +GitLab + diff --git a/SOURCES/glibc-sw28896.patch b/SOURCES/glibc-sw28896.patch new file mode 100644 index 0000000..051c7a9 --- /dev/null +++ b/SOURCES/glibc-sw28896.patch @@ -0,0 +1,132 @@ +From 8f2267e67ea924c949ca4a70a0d2c484d81a8fc9 Mon Sep 17 00:00:00 2001 +From: Noah Goldstein +Date: Tue, 15 Feb 2022 08:18:15 -0600 +Subject: [PATCH] x86: Fallback {str|wcs}cmp RTM in the ncmp overflow case [BZ + #28896] + +In the overflow fallback strncmp-avx2-rtm and wcsncmp-avx2-rtm would +call strcmp-avx2 and wcscmp-avx2 respectively. This would have +not checks around vzeroupper and would trigger spurious +aborts. This commit fixes that. + +test-strcmp, test-strncmp, test-wcscmp, and test-wcsncmp all pass on +AVX2 machines with and without RTM. + +Co-authored-by: H.J. Lu + +(cherry picked from commit c6272098323153db373f2986c67786ea8c85f1cf) +--- + sysdeps/x86/Makefile | 2 +- + sysdeps/x86/tst-strncmp-rtm.c | 17 ++++++++++++++++- + sysdeps/x86_64/multiarch/strcmp-avx2.S | 2 +- + sysdeps/x86_64/multiarch/strncmp-avx2-rtm.S | 1 + + sysdeps/x86_64/multiarch/strncmp-avx2.S | 1 + + sysdeps/x86_64/multiarch/wcsncmp-avx2-rtm.S | 2 +- + sysdeps/x86_64/multiarch/wcsncmp-avx2.S | 2 +- + 7 files changed, 22 insertions(+), 5 deletions(-) + +diff --git a/sysdeps/x86/Makefile b/sysdeps/x86/Makefile +index 5be71ada..2d814915 100644 +--- a/sysdeps/x86/Makefile ++++ b/sysdeps/x86/Makefile +@@ -38,7 +38,7 @@ CFLAGS-tst-memset-rtm.c += -mrtm + CFLAGS-tst-strchr-rtm.c += -mrtm + CFLAGS-tst-strcpy-rtm.c += -mrtm + CFLAGS-tst-strlen-rtm.c += -mrtm +-CFLAGS-tst-strncmp-rtm.c += -mrtm ++CFLAGS-tst-strncmp-rtm.c += -mrtm -Wno-error + CFLAGS-tst-strrchr-rtm.c += -mrtm + endif + +diff --git a/sysdeps/x86/tst-strncmp-rtm.c b/sysdeps/x86/tst-strncmp-rtm.c +index 236ad951..4d0004b5 100644 +--- a/sysdeps/x86/tst-strncmp-rtm.c ++++ b/sysdeps/x86/tst-strncmp-rtm.c +@@ -16,6 +16,7 @@ + License along with the GNU C Library; if not, see + . */ + ++#include + #include + + #define LOOP 3000 +@@ -45,8 +46,22 @@ function (void) + return 1; + } + ++__attribute__ ((noinline, noclone)) ++static int ++function_overflow (void) ++{ ++ if (strncmp (string1, string2, SIZE_MAX) == 0) ++ return 0; ++ else ++ return 1; ++} ++ + static int + do_test (void) + { +- return do_test_1 ("strncmp", LOOP, prepare, function); ++ int status = do_test_1 ("strncmp", LOOP, prepare, function); ++ if (status != EXIT_SUCCESS) ++ return status; ++ status = do_test_1 ("strncmp", LOOP, prepare, function_overflow); ++ return status; + } +diff --git a/sysdeps/x86_64/multiarch/strcmp-avx2.S b/sysdeps/x86_64/multiarch/strcmp-avx2.S +index 5d1c9d90..433ae047 100644 +--- a/sysdeps/x86_64/multiarch/strcmp-avx2.S ++++ b/sysdeps/x86_64/multiarch/strcmp-avx2.S +@@ -95,7 +95,7 @@ ENTRY (STRCMP) + length to bound a valid memory region. In these cases just use + 'wcscmp'. */ + shrq $56, %rcx +- jnz __wcscmp_avx2 ++ jnz OVERFLOW_STRCMP + # endif + /* Convert units: from wide to byte char. */ + shl $2, %RDX_LP +diff --git a/sysdeps/x86_64/multiarch/strncmp-avx2-rtm.S b/sysdeps/x86_64/multiarch/strncmp-avx2-rtm.S +index 37d1224b..68bad365 100644 +--- a/sysdeps/x86_64/multiarch/strncmp-avx2-rtm.S ++++ b/sysdeps/x86_64/multiarch/strncmp-avx2-rtm.S +@@ -1,3 +1,4 @@ + #define STRCMP __strncmp_avx2_rtm + #define USE_AS_STRNCMP 1 ++#define OVERFLOW_STRCMP __strcmp_avx2_rtm + #include "strcmp-avx2-rtm.S" +diff --git a/sysdeps/x86_64/multiarch/strncmp-avx2.S b/sysdeps/x86_64/multiarch/strncmp-avx2.S +index 1678bcc2..f138e9f1 100644 +--- a/sysdeps/x86_64/multiarch/strncmp-avx2.S ++++ b/sysdeps/x86_64/multiarch/strncmp-avx2.S +@@ -1,3 +1,4 @@ + #define STRCMP __strncmp_avx2 + #define USE_AS_STRNCMP 1 ++#define OVERFLOW_STRCMP __strcmp_avx2 + #include "strcmp-avx2.S" +diff --git a/sysdeps/x86_64/multiarch/wcsncmp-avx2-rtm.S b/sysdeps/x86_64/multiarch/wcsncmp-avx2-rtm.S +index 4e88c70c..f467582c 100644 +--- a/sysdeps/x86_64/multiarch/wcsncmp-avx2-rtm.S ++++ b/sysdeps/x86_64/multiarch/wcsncmp-avx2-rtm.S +@@ -1,5 +1,5 @@ + #define STRCMP __wcsncmp_avx2_rtm + #define USE_AS_STRNCMP 1 + #define USE_AS_WCSCMP 1 +- ++#define OVERFLOW_STRCMP __wcscmp_avx2_rtm + #include "strcmp-avx2-rtm.S" +diff --git a/sysdeps/x86_64/multiarch/wcsncmp-avx2.S b/sysdeps/x86_64/multiarch/wcsncmp-avx2.S +index 4fa1de4d..e9ede522 100644 +--- a/sysdeps/x86_64/multiarch/wcsncmp-avx2.S ++++ b/sysdeps/x86_64/multiarch/wcsncmp-avx2.S +@@ -1,5 +1,5 @@ + #define STRCMP __wcsncmp_avx2 + #define USE_AS_STRNCMP 1 + #define USE_AS_WCSCMP 1 +- ++#define OVERFLOW_STRCMP __wcscmp_avx2 + #include "strcmp-avx2.S" +-- +GitLab + diff --git a/SOURCES/ia-avoid_short_distance_rep_movsb.patch b/SOURCES/ia-avoid_short_distance_rep_movsb.patch new file mode 100644 index 0000000..a7d087b --- /dev/null +++ b/SOURCES/ia-avoid_short_distance_rep_movsb.patch @@ -0,0 +1,136 @@ +From 869a7106bfc301aa021a77a9bcede85eddd17da1 Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Wed, 2 Mar 2022 15:33:52 -0800 +Subject: [PATCH] x86-64: Add Avoid_Short_Distance_REP_MOVSB + +commit 3ec5d83d2a237d39e7fd6ef7a0bc8ac4c171a4a5 +Author: H.J. Lu +Date: Sat Jan 25 14:19:40 2020 -0800 + + x86-64: Avoid rep movsb with short distance [BZ #27130] + +introduced some regressions on Intel processors without Fast Short REP +MOV (FSRM). Add Avoid_Short_Distance_REP_MOVSB to avoid rep movsb with +short distance only on Intel processors with FSRM. bench-memmove-large +on Skylake server shows that cycles of __memmove_evex_unaligned_erms +improves for the following data size: + + before after Improvement +length=4127, align1=3, align2=0: 479.38 349.25 27% +length=4223, align1=9, align2=5: 405.62 333.25 18% +length=8223, align1=3, align2=0: 786.12 496.38 37% +length=8319, align1=9, align2=5: 727.50 501.38 31% +length=16415, align1=3, align2=0: 1436.88 840.00 41% +length=16511, align1=9, align2=5: 1375.50 836.38 39% +length=32799, align1=3, align2=0: 2890.00 1860.12 36% +length=32895, align1=9, align2=5: 2891.38 1931.88 33% + +(cherry picked from commit 91cc803d27bda34919717b496b53cf279e44a922) +--- + sysdeps/x86/cacheinfo.h | 9 +++++++++ + sysdeps/x86/cpu-features.c | 5 +++++ + .../include/cpu-features-preferred_feature_index_1.def | 1 + + sysdeps/x86/sysdep.h | 5 +++++ + sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S | 5 +++++ + 5 files changed, 25 insertions(+) + +diff --git a/sysdeps/x86/cacheinfo.h b/sysdeps/x86/cacheinfo.h +index b982982f..f72f634a 100644 +--- a/sysdeps/x86/cacheinfo.h ++++ b/sysdeps/x86/cacheinfo.h +@@ -48,6 +48,11 @@ long int __x86_rep_stosb_threshold attribute_hidden = 2048; + /* Threshold to stop using Enhanced REP MOVSB. */ + long int __x86_rep_movsb_stop_threshold attribute_hidden; + ++/* A bit-wise OR of string/memory requirements for optimal performance ++ e.g. X86_STRING_CONTROL_AVOID_SHORT_DISTANCE_REP_MOVSB. These bits ++ are used at runtime to tune implementation behavior. */ ++int __x86_string_control attribute_hidden; ++ + static void + get_common_cache_info (long int *shared_ptr, unsigned int *threads_ptr, + long int core) +@@ -435,6 +440,10 @@ init_cacheinfo (void) + if (cpu_features->basic.kind != arch_kind_amd) + __x86_rep_movsb_stop_threshold = __x86_shared_non_temporal_threshold; + ++ if (CPU_FEATURES_ARCH_P (cpu_features, Avoid_Short_Distance_REP_MOVSB)) ++ __x86_string_control ++ |= X86_STRING_CONTROL_AVOID_SHORT_DISTANCE_REP_MOVSB; ++ + # if HAVE_TUNABLES + __x86_rep_stosb_threshold = cpu_features->rep_stosb_threshold; + # endif +diff --git a/sysdeps/x86/cpu-features.c b/sysdeps/x86/cpu-features.c +index 4889f062..8885b48e 100644 +--- a/sysdeps/x86/cpu-features.c ++++ b/sysdeps/x86/cpu-features.c +@@ -580,6 +580,11 @@ init_cpu_features (struct cpu_features *cpu_features) + &= ~bit_arch_AVX_Fast_Unaligned_Load; + } + } ++ ++ /* Avoid avoid short distance REP MOVSB on processor with FSRM. */ ++ if (CPU_FEATURES_CPU_P (cpu_features, FSRM)) ++ cpu_features->preferred[index_arch_Avoid_Short_Distance_REP_MOVSB] ++ |= bit_arch_Avoid_Short_Distance_REP_MOVSB; + } + /* This spells out "CentaurHauls" or " Shanghai ". */ + else if ((ebx == 0x746e6543 && ecx == 0x736c7561 && edx == 0x48727561) +diff --git a/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def b/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def +index 4ca70b40..f2340624 100644 +--- a/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def ++++ b/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def +@@ -33,3 +33,4 @@ BIT (Prefer_FSRM) + BIT (Prefer_No_AVX512) + BIT (MathVec_Prefer_No_AVX512) + BIT (Prefer_AVX2_STRCMP) ++BIT (Avoid_Short_Distance_REP_MOVSB) +diff --git a/sysdeps/x86/sysdep.h b/sysdeps/x86/sysdep.h +index f41f4ebd..01bac0f6 100644 +--- a/sysdeps/x86/sysdep.h ++++ b/sysdeps/x86/sysdep.h +@@ -57,6 +57,11 @@ enum cf_protection_level + #define STATE_SAVE_MASK \ + ((1 << 1) | (1 << 2) | (1 << 3) | (1 << 5) | (1 << 6) | (1 << 7)) + ++/* Constants for bits in __x86_string_control: */ ++ ++/* Avoid short distance REP MOVSB. */ ++#define X86_STRING_CONTROL_AVOID_SHORT_DISTANCE_REP_MOVSB (1 << 0) ++ + #ifdef __ASSEMBLER__ + + /* Syntactic details of assembler. */ +diff --git a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S +index 620ce3a8..0469bf99 100644 +--- a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S ++++ b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S +@@ -325,12 +325,16 @@ L(movsb): + /* Avoid slow backward REP MOVSB. */ + jb L(more_8x_vec_backward) + # if AVOID_SHORT_DISTANCE_REP_MOVSB ++ andl $X86_STRING_CONTROL_AVOID_SHORT_DISTANCE_REP_MOVSB, __x86_string_control(%rip) ++ jz 3f + movq %rdi, %rcx + subq %rsi, %rcx + jmp 2f + # endif + 1: + # if AVOID_SHORT_DISTANCE_REP_MOVSB ++ andl $X86_STRING_CONTROL_AVOID_SHORT_DISTANCE_REP_MOVSB, __x86_string_control(%rip) ++ jz 3f + movq %rsi, %rcx + subq %rdi, %rcx + 2: +@@ -338,6 +342,7 @@ L(movsb): + is N*4GB + [1..63] with N >= 0. */ + cmpl $63, %ecx + jbe L(more_2x_vec) /* Avoid "rep movsb" if ECX <= 63. */ ++3: + # endif + mov %RDX_LP, %RCX_LP + rep movsb +-- +GitLab + diff --git a/SOURCES/ia-avx-opt-funct-rtm.patch b/SOURCES/ia-avx-opt-funct-rtm.patch new file mode 100644 index 0000000..5101b4c --- /dev/null +++ b/SOURCES/ia-avx-opt-funct-rtm.patch @@ -0,0 +1,2564 @@ +From 041fc69e69905aa9193178c49b44dce7bb8b5d6d Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Wed, 2 Mar 2022 14:25:33 -0800 +Subject: [PATCH] x86-64: Add AVX optimized string/memory functions for RTM + +Since VZEROUPPER triggers RTM abort while VZEROALL won't, select AVX +optimized string/memory functions with + + xtest + jz 1f + vzeroall + ret +1: + vzeroupper + ret + +at function exit on processors with usable RTM, but without 256-bit EVEX +instructions to avoid VZEROUPPER inside a transactionally executing RTM +region. + +(cherry picked from commit 7ebba91361badf7531d4e75050627a88d424872f) +--- + sysdeps/x86_64/multiarch/Makefile | 27 +++ + sysdeps/x86_64/multiarch/ifunc-avx2.h | 4 + + sysdeps/x86_64/multiarch/ifunc-impl-list.c | 170 ++++++++++++++++++ + sysdeps/x86_64/multiarch/ifunc-memcmp.h | 4 + + sysdeps/x86_64/multiarch/ifunc-memmove.h | 12 ++ + sysdeps/x86_64/multiarch/ifunc-memset.h | 12 ++ + sysdeps/x86_64/multiarch/ifunc-strcpy.h | 4 + + sysdeps/x86_64/multiarch/ifunc-wmemset.h | 5 + + sysdeps/x86_64/multiarch/memchr-avx2-rtm.S | 12 ++ + sysdeps/x86_64/multiarch/memchr-avx2.S | 45 +++-- + .../x86_64/multiarch/memcmp-avx2-movbe-rtm.S | 12 ++ + sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S | 28 ++- + .../memmove-avx-unaligned-erms-rtm.S | 17 ++ + .../multiarch/memmove-vec-unaligned-erms.S | 33 ++-- + sysdeps/x86_64/multiarch/memrchr-avx2-rtm.S | 12 ++ + sysdeps/x86_64/multiarch/memrchr-avx2.S | 53 +++--- + .../memset-avx2-unaligned-erms-rtm.S | 10 ++ + .../multiarch/memset-avx2-unaligned-erms.S | 12 +- + .../multiarch/memset-vec-unaligned-erms.S | 41 ++--- + sysdeps/x86_64/multiarch/rawmemchr-avx2-rtm.S | 4 + + sysdeps/x86_64/multiarch/stpcpy-avx2-rtm.S | 3 + + sysdeps/x86_64/multiarch/stpncpy-avx2-rtm.S | 4 + + sysdeps/x86_64/multiarch/strcat-avx2-rtm.S | 12 ++ + sysdeps/x86_64/multiarch/strcat-avx2.S | 6 +- + sysdeps/x86_64/multiarch/strchr-avx2-rtm.S | 12 ++ + sysdeps/x86_64/multiarch/strchr-avx2.S | 22 +-- + sysdeps/x86_64/multiarch/strchr.c | 4 + + sysdeps/x86_64/multiarch/strchrnul-avx2-rtm.S | 3 + + sysdeps/x86_64/multiarch/strcmp-avx2-rtm.S | 12 ++ + sysdeps/x86_64/multiarch/strcmp-avx2.S | 55 +++--- + sysdeps/x86_64/multiarch/strcmp.c | 4 + + sysdeps/x86_64/multiarch/strcpy-avx2-rtm.S | 12 ++ + sysdeps/x86_64/multiarch/strcpy-avx2.S | 85 ++++----- + sysdeps/x86_64/multiarch/strlen-avx2-rtm.S | 12 ++ + sysdeps/x86_64/multiarch/strlen-avx2.S | 43 ++--- + sysdeps/x86_64/multiarch/strncat-avx2-rtm.S | 3 + + sysdeps/x86_64/multiarch/strncmp-avx2-rtm.S | 3 + + sysdeps/x86_64/multiarch/strncmp.c | 4 + + sysdeps/x86_64/multiarch/strncpy-avx2-rtm.S | 3 + + sysdeps/x86_64/multiarch/strnlen-avx2-rtm.S | 4 + + sysdeps/x86_64/multiarch/strrchr-avx2-rtm.S | 12 ++ + sysdeps/x86_64/multiarch/strrchr-avx2.S | 19 +- + sysdeps/x86_64/multiarch/wcschr-avx2-rtm.S | 3 + + sysdeps/x86_64/multiarch/wcscmp-avx2-rtm.S | 4 + + sysdeps/x86_64/multiarch/wcslen-avx2-rtm.S | 4 + + sysdeps/x86_64/multiarch/wcsncmp-avx2-rtm.S | 5 + + sysdeps/x86_64/multiarch/wcsnlen-avx2-rtm.S | 5 + + sysdeps/x86_64/multiarch/wcsnlen.c | 4 + + sysdeps/x86_64/multiarch/wcsrchr-avx2-rtm.S | 3 + + sysdeps/x86_64/multiarch/wmemchr-avx2-rtm.S | 4 + + .../x86_64/multiarch/wmemcmp-avx2-movbe-rtm.S | 4 + + sysdeps/x86_64/sysdep.h | 22 +++ + 52 files changed, 668 insertions(+), 244 deletions(-) + create mode 100644 sysdeps/x86_64/multiarch/memchr-avx2-rtm.S + create mode 100644 sysdeps/x86_64/multiarch/memcmp-avx2-movbe-rtm.S + create mode 100644 sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-rtm.S + create mode 100644 sysdeps/x86_64/multiarch/memrchr-avx2-rtm.S + create mode 100644 sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms-rtm.S + create mode 100644 sysdeps/x86_64/multiarch/rawmemchr-avx2-rtm.S + create mode 100644 sysdeps/x86_64/multiarch/stpcpy-avx2-rtm.S + create mode 100644 sysdeps/x86_64/multiarch/stpncpy-avx2-rtm.S + create mode 100644 sysdeps/x86_64/multiarch/strcat-avx2-rtm.S + create mode 100644 sysdeps/x86_64/multiarch/strchr-avx2-rtm.S + create mode 100644 sysdeps/x86_64/multiarch/strchrnul-avx2-rtm.S + create mode 100644 sysdeps/x86_64/multiarch/strcmp-avx2-rtm.S + create mode 100644 sysdeps/x86_64/multiarch/strcpy-avx2-rtm.S + create mode 100644 sysdeps/x86_64/multiarch/strlen-avx2-rtm.S + create mode 100644 sysdeps/x86_64/multiarch/strncat-avx2-rtm.S + create mode 100644 sysdeps/x86_64/multiarch/strncmp-avx2-rtm.S + create mode 100644 sysdeps/x86_64/multiarch/strncpy-avx2-rtm.S + create mode 100644 sysdeps/x86_64/multiarch/strnlen-avx2-rtm.S + create mode 100644 sysdeps/x86_64/multiarch/strrchr-avx2-rtm.S + create mode 100644 sysdeps/x86_64/multiarch/wcschr-avx2-rtm.S + create mode 100644 sysdeps/x86_64/multiarch/wcscmp-avx2-rtm.S + create mode 100644 sysdeps/x86_64/multiarch/wcslen-avx2-rtm.S + create mode 100644 sysdeps/x86_64/multiarch/wcsncmp-avx2-rtm.S + create mode 100644 sysdeps/x86_64/multiarch/wcsnlen-avx2-rtm.S + create mode 100644 sysdeps/x86_64/multiarch/wcsrchr-avx2-rtm.S + create mode 100644 sysdeps/x86_64/multiarch/wmemchr-avx2-rtm.S + create mode 100644 sysdeps/x86_64/multiarch/wmemcmp-avx2-movbe-rtm.S + +diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile +index 9d79b138..491c7698 100644 +--- a/sysdeps/x86_64/multiarch/Makefile ++++ b/sysdeps/x86_64/multiarch/Makefile +@@ -40,6 +40,25 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c \ + memset-sse2-unaligned-erms \ + memset-avx2-unaligned-erms \ + memset-avx512-unaligned-erms \ ++ memchr-avx2-rtm \ ++ memcmp-avx2-movbe-rtm \ ++ memmove-avx-unaligned-erms-rtm \ ++ memrchr-avx2-rtm \ ++ memset-avx2-unaligned-erms-rtm \ ++ rawmemchr-avx2-rtm \ ++ strchr-avx2-rtm \ ++ strcmp-avx2-rtm \ ++ strchrnul-avx2-rtm \ ++ stpcpy-avx2-rtm \ ++ stpncpy-avx2-rtm \ ++ strcat-avx2-rtm \ ++ strcpy-avx2-rtm \ ++ strlen-avx2-rtm \ ++ strncat-avx2-rtm \ ++ strncmp-avx2-rtm \ ++ strncpy-avx2-rtm \ ++ strnlen-avx2-rtm \ ++ strrchr-avx2-rtm \ + memchr-evex \ + memcmp-evex-movbe \ + memmove-evex-unaligned-erms \ +@@ -76,6 +95,14 @@ sysdep_routines += wmemcmp-sse4 wmemcmp-ssse3 wmemcmp-c \ + wcsrchr-sse2 wcsrchr-avx2 \ + wcsnlen-sse4_1 wcsnlen-c \ + wcslen-sse2 wcslen-avx2 wcsnlen-avx2 \ ++ wcschr-avx2-rtm \ ++ wcscmp-avx2-rtm \ ++ wcslen-avx2-rtm \ ++ wcsncmp-avx2-rtm \ ++ wcsnlen-avx2-rtm \ ++ wcsrchr-avx2-rtm \ ++ wmemchr-avx2-rtm \ ++ wmemcmp-avx2-movbe-rtm \ + wcschr-evex \ + wcscmp-evex \ + wcslen-evex \ +diff --git a/sysdeps/x86_64/multiarch/ifunc-avx2.h b/sysdeps/x86_64/multiarch/ifunc-avx2.h +index 7081b0c9..e0f30e61 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-avx2.h ++++ b/sysdeps/x86_64/multiarch/ifunc-avx2.h +@@ -21,6 +21,7 @@ + + extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden; + + static inline void * +@@ -36,6 +37,9 @@ IFUNC_SELECTOR (void) + && CPU_FEATURE_USABLE_P (cpu_features, BMI2)) + return OPTIMIZE (evex); + ++ if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) ++ return OPTIMIZE (avx2_rtm); ++ + if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) + return OPTIMIZE (avx2); + } +diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c +index c8da910e..c1efeec0 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c ++++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c +@@ -43,6 +43,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, memchr, + CPU_FEATURE_USABLE (AVX2), + __memchr_avx2) ++ IFUNC_IMPL_ADD (array, i, memchr, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __memchr_avx2_rtm) + IFUNC_IMPL_ADD (array, i, memchr, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW) +@@ -56,6 +60,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + (CPU_FEATURE_USABLE (AVX2) + && CPU_FEATURE_USABLE (MOVBE)), + __memcmp_avx2_movbe) ++ IFUNC_IMPL_ADD (array, i, memcmp, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (MOVBE) ++ && CPU_FEATURE_USABLE (RTM)), ++ __memcmp_avx2_movbe_rtm) + IFUNC_IMPL_ADD (array, i, memcmp, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW) +@@ -85,6 +94,14 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, __memmove_chk, + CPU_FEATURE_USABLE (AVX), + __memmove_chk_avx_unaligned_erms) ++ IFUNC_IMPL_ADD (array, i, __memmove_chk, ++ (CPU_FEATURE_USABLE (AVX) ++ && CPU_FEATURE_USABLE (RTM)), ++ __memmove_chk_avx_unaligned_rtm) ++ IFUNC_IMPL_ADD (array, i, __memmove_chk, ++ (CPU_FEATURE_USABLE (AVX) ++ && CPU_FEATURE_USABLE (RTM)), ++ __memmove_chk_avx_unaligned_erms_rtm) + IFUNC_IMPL_ADD (array, i, __memmove_chk, + CPU_FEATURE_USABLE (AVX512VL), + __memmove_chk_evex_unaligned) +@@ -113,6 +130,14 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, memmove, + CPU_FEATURE_USABLE (AVX), + __memmove_avx_unaligned_erms) ++ IFUNC_IMPL_ADD (array, i, memmove, ++ (CPU_FEATURE_USABLE (AVX) ++ && CPU_FEATURE_USABLE (RTM)), ++ __memmove_avx_unaligned_rtm) ++ IFUNC_IMPL_ADD (array, i, memmove, ++ (CPU_FEATURE_USABLE (AVX) ++ && CPU_FEATURE_USABLE (RTM)), ++ __memmove_avx_unaligned_erms_rtm) + IFUNC_IMPL_ADD (array, i, memmove, + CPU_FEATURE_USABLE (AVX512VL), + __memmove_evex_unaligned) +@@ -143,6 +168,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, memrchr, + CPU_FEATURE_USABLE (AVX2), + __memrchr_avx2) ++ IFUNC_IMPL_ADD (array, i, memrchr, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __memrchr_avx2_rtm) + IFUNC_IMPL_ADD (array, i, memrchr, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW)), +@@ -165,6 +194,14 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, __memset_chk, + CPU_FEATURE_USABLE (AVX2), + __memset_chk_avx2_unaligned_erms) ++ IFUNC_IMPL_ADD (array, i, __memset_chk, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __memset_chk_avx2_unaligned_rtm) ++ IFUNC_IMPL_ADD (array, i, __memset_chk, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __memset_chk_avx2_unaligned_erms_rtm) + IFUNC_IMPL_ADD (array, i, __memset_chk, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW)), +@@ -198,6 +235,14 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, memset, + CPU_FEATURE_USABLE (AVX2), + __memset_avx2_unaligned_erms) ++ IFUNC_IMPL_ADD (array, i, memset, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __memset_avx2_unaligned_rtm) ++ IFUNC_IMPL_ADD (array, i, memset, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __memset_avx2_unaligned_erms_rtm) + IFUNC_IMPL_ADD (array, i, memset, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW)), +@@ -222,6 +267,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, rawmemchr, + CPU_FEATURE_USABLE (AVX2), + __rawmemchr_avx2) ++ IFUNC_IMPL_ADD (array, i, rawmemchr, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __rawmemchr_avx2_rtm) + IFUNC_IMPL_ADD (array, i, rawmemchr, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW) +@@ -234,6 +283,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, strlen, + CPU_FEATURE_USABLE (AVX2), + __strlen_avx2) ++ IFUNC_IMPL_ADD (array, i, strlen, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __strlen_avx2_rtm) + IFUNC_IMPL_ADD (array, i, strlen, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW)), +@@ -245,6 +298,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, strnlen, + CPU_FEATURE_USABLE (AVX2), + __strnlen_avx2) ++ IFUNC_IMPL_ADD (array, i, strnlen, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __strnlen_avx2_rtm) + IFUNC_IMPL_ADD (array, i, strnlen, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW)), +@@ -257,6 +314,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + __stpncpy_ssse3) + IFUNC_IMPL_ADD (array, i, stpncpy, CPU_FEATURE_USABLE (AVX2), + __stpncpy_avx2) ++ IFUNC_IMPL_ADD (array, i, stpncpy, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __stpncpy_avx2_rtm) + IFUNC_IMPL_ADD (array, i, stpncpy, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW)), +@@ -271,6 +332,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + __stpcpy_ssse3) + IFUNC_IMPL_ADD (array, i, stpcpy, CPU_FEATURE_USABLE (AVX2), + __stpcpy_avx2) ++ IFUNC_IMPL_ADD (array, i, stpcpy, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __stpcpy_avx2_rtm) + IFUNC_IMPL_ADD (array, i, stpcpy, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW)), +@@ -309,6 +374,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL (i, name, strcat, + IFUNC_IMPL_ADD (array, i, strcat, CPU_FEATURE_USABLE (AVX2), + __strcat_avx2) ++ IFUNC_IMPL_ADD (array, i, strcat, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __strcat_avx2_rtm) + IFUNC_IMPL_ADD (array, i, strcat, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW)), +@@ -323,6 +392,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, strchr, + CPU_FEATURE_USABLE (AVX2), + __strchr_avx2) ++ IFUNC_IMPL_ADD (array, i, strchr, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __strchr_avx2_rtm) + IFUNC_IMPL_ADD (array, i, strchr, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW) +@@ -336,6 +409,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, strchrnul, + CPU_FEATURE_USABLE (AVX2), + __strchrnul_avx2) ++ IFUNC_IMPL_ADD (array, i, strchrnul, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __strchrnul_avx2_rtm) + IFUNC_IMPL_ADD (array, i, strchrnul, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW) +@@ -348,6 +425,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, strrchr, + CPU_FEATURE_USABLE (AVX2), + __strrchr_avx2) ++ IFUNC_IMPL_ADD (array, i, strrchr, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __strrchr_avx2_rtm) + IFUNC_IMPL_ADD (array, i, strrchr, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW)), +@@ -359,6 +440,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, strcmp, + CPU_FEATURE_USABLE (AVX2), + __strcmp_avx2) ++ IFUNC_IMPL_ADD (array, i, strcmp, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __strcmp_avx2_rtm) + IFUNC_IMPL_ADD (array, i, strcmp, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW) +@@ -375,6 +460,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL (i, name, strcpy, + IFUNC_IMPL_ADD (array, i, strcpy, CPU_FEATURE_USABLE (AVX2), + __strcpy_avx2) ++ IFUNC_IMPL_ADD (array, i, strcpy, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __strcpy_avx2_rtm) + IFUNC_IMPL_ADD (array, i, strcpy, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW)), +@@ -422,6 +511,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL (i, name, strncat, + IFUNC_IMPL_ADD (array, i, strncat, CPU_FEATURE_USABLE (AVX2), + __strncat_avx2) ++ IFUNC_IMPL_ADD (array, i, strncat, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __strncat_avx2_rtm) + IFUNC_IMPL_ADD (array, i, strncat, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW)), +@@ -436,6 +529,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL (i, name, strncpy, + IFUNC_IMPL_ADD (array, i, strncpy, CPU_FEATURE_USABLE (AVX2), + __strncpy_avx2) ++ IFUNC_IMPL_ADD (array, i, strncpy, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __strncpy_avx2_rtm) + IFUNC_IMPL_ADD (array, i, strncpy, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW)), +@@ -469,6 +566,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, wcschr, + CPU_FEATURE_USABLE (AVX2), + __wcschr_avx2) ++ IFUNC_IMPL_ADD (array, i, wcschr, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __wcschr_avx2_rtm) + IFUNC_IMPL_ADD (array, i, wcschr, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW) +@@ -481,6 +582,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, wcsrchr, + CPU_FEATURE_USABLE (AVX2), + __wcsrchr_avx2) ++ IFUNC_IMPL_ADD (array, i, wcsrchr, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __wcsrchr_avx2_rtm) + IFUNC_IMPL_ADD (array, i, wcsrchr, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW) +@@ -493,6 +598,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, wcscmp, + CPU_FEATURE_USABLE (AVX2), + __wcscmp_avx2) ++ IFUNC_IMPL_ADD (array, i, wcscmp, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __wcscmp_avx2_rtm) + IFUNC_IMPL_ADD (array, i, wcscmp, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW) +@@ -505,6 +614,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, wcsncmp, + CPU_FEATURE_USABLE (AVX2), + __wcsncmp_avx2) ++ IFUNC_IMPL_ADD (array, i, wcsncmp, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __wcsncmp_avx2_rtm) + IFUNC_IMPL_ADD (array, i, wcsncmp, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW) +@@ -523,6 +636,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, wcslen, + CPU_FEATURE_USABLE (AVX2), + __wcslen_avx2) ++ IFUNC_IMPL_ADD (array, i, wcslen, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __wcslen_avx2_rtm) + IFUNC_IMPL_ADD (array, i, wcslen, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW) +@@ -535,6 +652,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, wcsnlen, + CPU_FEATURE_USABLE (AVX2), + __wcsnlen_avx2) ++ IFUNC_IMPL_ADD (array, i, wcsnlen, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __wcsnlen_avx2_rtm) + IFUNC_IMPL_ADD (array, i, wcsnlen, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW) +@@ -550,6 +671,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, wmemchr, + CPU_FEATURE_USABLE (AVX2), + __wmemchr_avx2) ++ IFUNC_IMPL_ADD (array, i, wmemchr, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __wmemchr_avx2_rtm) + IFUNC_IMPL_ADD (array, i, wmemchr, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW) +@@ -563,6 +688,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + (CPU_FEATURE_USABLE (AVX2) + && CPU_FEATURE_USABLE (MOVBE)), + __wmemcmp_avx2_movbe) ++ IFUNC_IMPL_ADD (array, i, wmemcmp, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (MOVBE) ++ && CPU_FEATURE_USABLE (RTM)), ++ __wmemcmp_avx2_movbe_rtm) + IFUNC_IMPL_ADD (array, i, wmemcmp, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW) +@@ -581,6 +711,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, wmemset, + CPU_FEATURE_USABLE (AVX2), + __wmemset_avx2_unaligned) ++ IFUNC_IMPL_ADD (array, i, wmemset, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __wmemset_avx2_unaligned_rtm) + IFUNC_IMPL_ADD (array, i, wmemset, + CPU_FEATURE_USABLE (AVX512VL), + __wmemset_evex_unaligned) +@@ -606,6 +740,14 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, __memcpy_chk, + CPU_FEATURE_USABLE (AVX), + __memcpy_chk_avx_unaligned_erms) ++ IFUNC_IMPL_ADD (array, i, __memcpy_chk, ++ (CPU_FEATURE_USABLE (AVX) ++ && CPU_FEATURE_USABLE (RTM)), ++ __memcpy_chk_avx_unaligned_rtm) ++ IFUNC_IMPL_ADD (array, i, __memcpy_chk, ++ (CPU_FEATURE_USABLE (AVX) ++ && CPU_FEATURE_USABLE (RTM)), ++ __memcpy_chk_avx_unaligned_erms_rtm) + IFUNC_IMPL_ADD (array, i, __memcpy_chk, + CPU_FEATURE_USABLE (AVX512VL), + __memcpy_chk_evex_unaligned) +@@ -634,6 +776,14 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, memcpy, + CPU_FEATURE_USABLE (AVX), + __memcpy_avx_unaligned_erms) ++ IFUNC_IMPL_ADD (array, i, memcpy, ++ (CPU_FEATURE_USABLE (AVX) ++ && CPU_FEATURE_USABLE (RTM)), ++ __memcpy_avx_unaligned_rtm) ++ IFUNC_IMPL_ADD (array, i, memcpy, ++ (CPU_FEATURE_USABLE (AVX) ++ && CPU_FEATURE_USABLE (RTM)), ++ __memcpy_avx_unaligned_erms_rtm) + IFUNC_IMPL_ADD (array, i, memcpy, + CPU_FEATURE_USABLE (AVX512VL), + __memcpy_evex_unaligned) +@@ -676,6 +826,14 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, __mempcpy_chk, + CPU_FEATURE_USABLE (AVX), + __mempcpy_chk_avx_unaligned_erms) ++ IFUNC_IMPL_ADD (array, i, __mempcpy_chk, ++ (CPU_FEATURE_USABLE (AVX) ++ && CPU_FEATURE_USABLE (RTM)), ++ __mempcpy_chk_avx_unaligned_rtm) ++ IFUNC_IMPL_ADD (array, i, __mempcpy_chk, ++ (CPU_FEATURE_USABLE (AVX) ++ && CPU_FEATURE_USABLE (RTM)), ++ __mempcpy_chk_avx_unaligned_erms_rtm) + IFUNC_IMPL_ADD (array, i, __mempcpy_chk, + CPU_FEATURE_USABLE (AVX512VL), + __mempcpy_chk_evex_unaligned) +@@ -713,6 +871,14 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, mempcpy, + CPU_FEATURE_USABLE (AVX), + __mempcpy_avx_unaligned_erms) ++ IFUNC_IMPL_ADD (array, i, mempcpy, ++ (CPU_FEATURE_USABLE (AVX) ++ && CPU_FEATURE_USABLE (RTM)), ++ __mempcpy_avx_unaligned_rtm) ++ IFUNC_IMPL_ADD (array, i, mempcpy, ++ (CPU_FEATURE_USABLE (AVX) ++ && CPU_FEATURE_USABLE (RTM)), ++ __mempcpy_avx_unaligned_erms_rtm) + IFUNC_IMPL_ADD (array, i, mempcpy, + CPU_FEATURE_USABLE (AVX512VL), + __mempcpy_evex_unaligned) +@@ -734,6 +900,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, strncmp, + CPU_FEATURE_USABLE (AVX2), + __strncmp_avx2) ++ IFUNC_IMPL_ADD (array, i, strncmp, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __strncmp_avx2_rtm) + IFUNC_IMPL_ADD (array, i, strncmp, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW)), +diff --git a/sysdeps/x86_64/multiarch/ifunc-memcmp.h b/sysdeps/x86_64/multiarch/ifunc-memcmp.h +index 3ca1f0a6..8043c635 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-memcmp.h ++++ b/sysdeps/x86_64/multiarch/ifunc-memcmp.h +@@ -23,6 +23,7 @@ extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (sse4_1) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_movbe) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_movbe_rtm) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_movbe) attribute_hidden; + + static inline void * +@@ -38,6 +39,9 @@ IFUNC_SELECTOR (void) + && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)) + return OPTIMIZE (evex_movbe); + ++ if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) ++ return OPTIMIZE (avx2_movbe_rtm); ++ + if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) + return OPTIMIZE (avx2_movbe); + } +diff --git a/sysdeps/x86_64/multiarch/ifunc-memmove.h b/sysdeps/x86_64/multiarch/ifunc-memmove.h +index 6f8bce5f..fa09b9fb 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-memmove.h ++++ b/sysdeps/x86_64/multiarch/ifunc-memmove.h +@@ -29,6 +29,10 @@ extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3_back) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned_erms) + attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned_rtm) ++ attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned_erms_rtm) ++ attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_unaligned) + attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_unaligned_erms) +@@ -71,6 +75,14 @@ IFUNC_SELECTOR (void) + return OPTIMIZE (evex_unaligned); + } + ++ if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) ++ { ++ if (CPU_FEATURE_USABLE_P (cpu_features, ERMS)) ++ return OPTIMIZE (avx_unaligned_erms_rtm); ++ ++ return OPTIMIZE (avx_unaligned_rtm); ++ } ++ + if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) + { + if (CPU_FEATURE_USABLE_P (cpu_features, ERMS)) +diff --git a/sysdeps/x86_64/multiarch/ifunc-memset.h b/sysdeps/x86_64/multiarch/ifunc-memset.h +index 6f31f4dc..6f3375cc 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-memset.h ++++ b/sysdeps/x86_64/multiarch/ifunc-memset.h +@@ -27,6 +27,10 @@ extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned_erms) + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_unaligned) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_unaligned_erms) + attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_unaligned_rtm) ++ attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_unaligned_erms_rtm) ++ attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_unaligned) + attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_unaligned_erms) +@@ -69,6 +73,14 @@ IFUNC_SELECTOR (void) + return OPTIMIZE (evex_unaligned); + } + ++ if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) ++ { ++ if (CPU_FEATURE_USABLE_P (cpu_features, ERMS)) ++ return OPTIMIZE (avx2_unaligned_erms_rtm); ++ ++ return OPTIMIZE (avx2_unaligned_rtm); ++ } ++ + if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) + { + if (CPU_FEATURE_USABLE_P (cpu_features, ERMS)) +diff --git a/sysdeps/x86_64/multiarch/ifunc-strcpy.h b/sysdeps/x86_64/multiarch/ifunc-strcpy.h +index deae6348..a924762e 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-strcpy.h ++++ b/sysdeps/x86_64/multiarch/ifunc-strcpy.h +@@ -25,6 +25,7 @@ extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned) + attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden; + + static inline void * +@@ -39,6 +40,9 @@ IFUNC_SELECTOR (void) + && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)) + return OPTIMIZE (evex); + ++ if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) ++ return OPTIMIZE (avx2_rtm); ++ + if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) + return OPTIMIZE (avx2); + } +diff --git a/sysdeps/x86_64/multiarch/ifunc-wmemset.h b/sysdeps/x86_64/multiarch/ifunc-wmemset.h +index 9290c4bf..bdc94c6c 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-wmemset.h ++++ b/sysdeps/x86_64/multiarch/ifunc-wmemset.h +@@ -20,6 +20,8 @@ + + extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_unaligned) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_unaligned_rtm) ++ attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_unaligned) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_unaligned) attribute_hidden; + +@@ -39,6 +41,9 @@ IFUNC_SELECTOR (void) + if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)) + return OPTIMIZE (evex_unaligned); + ++ if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) ++ return OPTIMIZE (avx2_unaligned_rtm); ++ + if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) + return OPTIMIZE (avx2_unaligned); + } +diff --git a/sysdeps/x86_64/multiarch/memchr-avx2-rtm.S b/sysdeps/x86_64/multiarch/memchr-avx2-rtm.S +new file mode 100644 +index 00000000..87b076c7 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/memchr-avx2-rtm.S +@@ -0,0 +1,12 @@ ++#ifndef MEMCHR ++# define MEMCHR __memchr_avx2_rtm ++#endif ++ ++#define ZERO_UPPER_VEC_REGISTERS_RETURN \ ++ ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST ++ ++#define VZEROUPPER_RETURN jmp L(return_vzeroupper) ++ ++#define SECTION(p) p##.avx.rtm ++ ++#include "memchr-avx2.S" +diff --git a/sysdeps/x86_64/multiarch/memchr-avx2.S b/sysdeps/x86_64/multiarch/memchr-avx2.S +index c81da19b..cf893e77 100644 +--- a/sysdeps/x86_64/multiarch/memchr-avx2.S ++++ b/sysdeps/x86_64/multiarch/memchr-avx2.S +@@ -34,9 +34,13 @@ + # define VZEROUPPER vzeroupper + # endif + ++# ifndef SECTION ++# define SECTION(p) p##.avx ++# endif ++ + # define VEC_SIZE 32 + +- .section .text.avx,"ax",@progbits ++ .section SECTION(.text),"ax",@progbits + ENTRY (MEMCHR) + # ifndef USE_AS_RAWMEMCHR + /* Check for zero length. */ +@@ -107,8 +111,8 @@ L(cros_page_boundary): + # endif + addq %rdi, %rax + addq %rcx, %rax +- VZEROUPPER +- ret ++L(return_vzeroupper): ++ ZERO_UPPER_VEC_REGISTERS_RETURN + + .p2align 4 + L(aligned_more): +@@ -224,8 +228,7 @@ L(last_4x_vec_or_less): + + jnz L(first_vec_x3_check) + xorl %eax, %eax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(last_2x_vec): +@@ -243,8 +246,7 @@ L(last_2x_vec): + testl %eax, %eax + jnz L(first_vec_x1_check) + xorl %eax, %eax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(first_vec_x0_check): +@@ -253,8 +255,7 @@ L(first_vec_x0_check): + cmpq %rax, %rdx + jbe L(zero) + addq %rdi, %rax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(first_vec_x1_check): +@@ -264,8 +265,7 @@ L(first_vec_x1_check): + jbe L(zero) + addq $VEC_SIZE, %rax + addq %rdi, %rax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(first_vec_x2_check): +@@ -275,8 +275,7 @@ L(first_vec_x2_check): + jbe L(zero) + addq $(VEC_SIZE * 2), %rax + addq %rdi, %rax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(first_vec_x3_check): +@@ -286,12 +285,14 @@ L(first_vec_x3_check): + jbe L(zero) + addq $(VEC_SIZE * 3), %rax + addq %rdi, %rax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(zero): +- VZEROUPPER ++ xorl %eax, %eax ++ jmp L(return_vzeroupper) ++ ++ .p2align 4 + L(null): + xorl %eax, %eax + ret +@@ -301,24 +302,21 @@ L(null): + L(first_vec_x0): + tzcntl %eax, %eax + addq %rdi, %rax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(first_vec_x1): + tzcntl %eax, %eax + addq $VEC_SIZE, %rax + addq %rdi, %rax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(first_vec_x2): + tzcntl %eax, %eax + addq $(VEC_SIZE * 2), %rax + addq %rdi, %rax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(4x_vec_end): +@@ -337,8 +335,7 @@ L(first_vec_x3): + tzcntl %eax, %eax + addq $(VEC_SIZE * 3), %rax + addq %rdi, %rax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + END (MEMCHR) + #endif +diff --git a/sysdeps/x86_64/multiarch/memcmp-avx2-movbe-rtm.S b/sysdeps/x86_64/multiarch/memcmp-avx2-movbe-rtm.S +new file mode 100644 +index 00000000..cf4eff5d +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/memcmp-avx2-movbe-rtm.S +@@ -0,0 +1,12 @@ ++#ifndef MEMCMP ++# define MEMCMP __memcmp_avx2_movbe_rtm ++#endif ++ ++#define ZERO_UPPER_VEC_REGISTERS_RETURN \ ++ ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST ++ ++#define VZEROUPPER_RETURN jmp L(return_vzeroupper) ++ ++#define SECTION(p) p##.avx.rtm ++ ++#include "memcmp-avx2-movbe.S" +diff --git a/sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S b/sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S +index e3a35b89..9d5c9c72 100644 +--- a/sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S ++++ b/sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S +@@ -47,6 +47,10 @@ + # define VZEROUPPER vzeroupper + # endif + ++# ifndef SECTION ++# define SECTION(p) p##.avx ++# endif ++ + # define VEC_SIZE 32 + # define VEC_MASK ((1 << VEC_SIZE) - 1) + +@@ -55,7 +59,7 @@ + memcmp has to use UNSIGNED comparison for elemnts. + */ + +- .section .text.avx,"ax",@progbits ++ .section SECTION(.text),"ax",@progbits + ENTRY (MEMCMP) + # ifdef USE_AS_WMEMCMP + shl $2, %RDX_LP +@@ -123,8 +127,8 @@ ENTRY (MEMCMP) + vptest %ymm0, %ymm5 + jnc L(4x_vec_end) + xorl %eax, %eax +- VZEROUPPER +- ret ++L(return_vzeroupper): ++ ZERO_UPPER_VEC_REGISTERS_RETURN + + .p2align 4 + L(last_2x_vec): +@@ -144,8 +148,7 @@ L(last_vec): + vpmovmskb %ymm2, %eax + subl $VEC_MASK, %eax + jnz L(first_vec) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(first_vec): +@@ -164,8 +167,7 @@ L(wmemcmp_return): + movzbl (%rsi, %rcx), %edx + sub %edx, %eax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + # ifdef USE_AS_WMEMCMP + .p2align 4 +@@ -367,8 +369,7 @@ L(last_4x_vec): + vpmovmskb %ymm2, %eax + subl $VEC_MASK, %eax + jnz L(first_vec) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(4x_vec_end): +@@ -394,8 +395,7 @@ L(4x_vec_end): + movzbl (VEC_SIZE * 3)(%rsi, %rcx), %edx + sub %edx, %eax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(first_vec_x1): +@@ -410,8 +410,7 @@ L(first_vec_x1): + movzbl VEC_SIZE(%rsi, %rcx), %edx + sub %edx, %eax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(first_vec_x2): +@@ -426,7 +425,6 @@ L(first_vec_x2): + movzbl (VEC_SIZE * 2)(%rsi, %rcx), %edx + sub %edx, %eax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + END (MEMCMP) + #endif +diff --git a/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-rtm.S b/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-rtm.S +new file mode 100644 +index 00000000..1ec1962e +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-rtm.S +@@ -0,0 +1,17 @@ ++#if IS_IN (libc) ++# define VEC_SIZE 32 ++# define VEC(i) ymm##i ++# define VMOVNT vmovntdq ++# define VMOVU vmovdqu ++# define VMOVA vmovdqa ++ ++# define ZERO_UPPER_VEC_REGISTERS_RETURN \ ++ ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST ++ ++# define VZEROUPPER_RETURN jmp L(return) ++ ++# define SECTION(p) p##.avx.rtm ++# define MEMMOVE_SYMBOL(p,s) p##_avx_##s##_rtm ++ ++# include "memmove-vec-unaligned-erms.S" ++#endif +diff --git a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S +index 08e21692..71f5954d 100644 +--- a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S ++++ b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S +@@ -140,11 +140,12 @@ L(last_2x_vec): + VMOVU -VEC_SIZE(%rsi,%rdx), %VEC(1) + VMOVU %VEC(0), (%rdi) + VMOVU %VEC(1), -VEC_SIZE(%rdi,%rdx) +- VZEROUPPER + #if !defined USE_MULTIARCH || !IS_IN (libc) + L(nop): +-#endif + ret ++#else ++ VZEROUPPER_RETURN ++#endif + #if defined USE_MULTIARCH && IS_IN (libc) + END (MEMMOVE_SYMBOL (__memmove, unaligned)) + +@@ -237,8 +238,11 @@ L(last_2x_vec): + VMOVU %VEC(0), (%rdi) + VMOVU %VEC(1), -VEC_SIZE(%rdi,%rdx) + L(return): +- VZEROUPPER ++#if VEC_SIZE > 16 ++ ZERO_UPPER_VEC_REGISTERS_RETURN ++#else + ret ++#endif + + L(movsb): + cmpq __x86_shared_non_temporal_threshold(%rip), %rdx +@@ -289,8 +293,7 @@ L(between_32_63): + VMOVU -32(%rsi,%rdx), %YMM1 + VMOVU %YMM0, (%rdi) + VMOVU %YMM1, -32(%rdi,%rdx) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + #endif + #if VEC_SIZE > 16 + /* From 16 to 31. No branch when size == 16. */ +@@ -299,7 +302,7 @@ L(between_16_31): + VMOVU -16(%rsi,%rdx), %XMM1 + VMOVU %XMM0, (%rdi) + VMOVU %XMM1, -16(%rdi,%rdx) +- ret ++ VZEROUPPER_RETURN + #endif + L(between_8_15): + /* From 8 to 15. No branch when size == 8. */ +@@ -352,8 +355,7 @@ L(more_2x_vec): + VMOVU %VEC(5), -(VEC_SIZE * 2)(%rdi,%rdx) + VMOVU %VEC(6), -(VEC_SIZE * 3)(%rdi,%rdx) + VMOVU %VEC(7), -(VEC_SIZE * 4)(%rdi,%rdx) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + L(last_4x_vec): + /* Copy from 2 * VEC to 4 * VEC. */ + VMOVU (%rsi), %VEC(0) +@@ -364,8 +366,7 @@ L(last_4x_vec): + VMOVU %VEC(1), VEC_SIZE(%rdi) + VMOVU %VEC(2), -VEC_SIZE(%rdi,%rdx) + VMOVU %VEC(3), -(VEC_SIZE * 2)(%rdi,%rdx) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + L(more_8x_vec): + cmpq %rsi, %rdi +@@ -421,8 +422,7 @@ L(loop_4x_vec_forward): + VMOVU %VEC(8), -(VEC_SIZE * 3)(%rcx) + /* Store the first VEC. */ + VMOVU %VEC(4), (%r11) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + L(more_8x_vec_backward): + /* Load the first 4 * VEC and last VEC to support overlapping +@@ -473,8 +473,7 @@ L(loop_4x_vec_backward): + VMOVU %VEC(7), (VEC_SIZE * 3)(%rdi) + /* Store the last VEC. */ + VMOVU %VEC(8), (%r11) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + #if (defined USE_MULTIARCH || VEC_SIZE == 16) && IS_IN (libc) + L(large_forward): +@@ -509,8 +508,7 @@ L(loop_large_forward): + VMOVU %VEC(8), -(VEC_SIZE * 3)(%rcx) + /* Store the first VEC. */ + VMOVU %VEC(4), (%r11) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + L(large_backward): + /* Don't use non-temporal store if there is overlap between +@@ -544,8 +542,7 @@ L(loop_large_backward): + VMOVU %VEC(7), (VEC_SIZE * 3)(%rdi) + /* Store the last VEC. */ + VMOVU %VEC(8), (%r11) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + #endif + END (MEMMOVE_SYMBOL (__memmove, unaligned_erms)) + +diff --git a/sysdeps/x86_64/multiarch/memrchr-avx2-rtm.S b/sysdeps/x86_64/multiarch/memrchr-avx2-rtm.S +new file mode 100644 +index 00000000..cea2d2a7 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/memrchr-avx2-rtm.S +@@ -0,0 +1,12 @@ ++#ifndef MEMRCHR ++# define MEMRCHR __memrchr_avx2_rtm ++#endif ++ ++#define ZERO_UPPER_VEC_REGISTERS_RETURN \ ++ ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST ++ ++#define VZEROUPPER_RETURN jmp L(return_vzeroupper) ++ ++#define SECTION(p) p##.avx.rtm ++ ++#include "memrchr-avx2.S" +diff --git a/sysdeps/x86_64/multiarch/memrchr-avx2.S b/sysdeps/x86_64/multiarch/memrchr-avx2.S +index ce488dd9..20efe7ac 100644 +--- a/sysdeps/x86_64/multiarch/memrchr-avx2.S ++++ b/sysdeps/x86_64/multiarch/memrchr-avx2.S +@@ -20,14 +20,22 @@ + + # include + ++# ifndef MEMRCHR ++# define MEMRCHR __memrchr_avx2 ++# endif ++ + # ifndef VZEROUPPER + # define VZEROUPPER vzeroupper + # endif + ++# ifndef SECTION ++# define SECTION(p) p##.avx ++# endif ++ + # define VEC_SIZE 32 + +- .section .text.avx,"ax",@progbits +-ENTRY (__memrchr_avx2) ++ .section SECTION(.text),"ax",@progbits ++ENTRY (MEMRCHR) + /* Broadcast CHAR to YMM0. */ + vmovd %esi, %xmm0 + vpbroadcastb %xmm0, %ymm0 +@@ -134,8 +142,8 @@ L(loop_4x_vec): + vpmovmskb %ymm1, %eax + bsrl %eax, %eax + addq %rdi, %rax +- VZEROUPPER +- ret ++L(return_vzeroupper): ++ ZERO_UPPER_VEC_REGISTERS_RETURN + + .p2align 4 + L(last_4x_vec_or_less): +@@ -169,8 +177,7 @@ L(last_4x_vec_or_less): + addq %rax, %rdx + jl L(zero) + addq %rdi, %rax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(last_2x_vec): +@@ -191,31 +198,27 @@ L(last_2x_vec): + jl L(zero) + addl $(VEC_SIZE * 2), %eax + addq %rdi, %rax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(last_vec_x0): + bsrl %eax, %eax + addq %rdi, %rax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(last_vec_x1): + bsrl %eax, %eax + addl $VEC_SIZE, %eax + addq %rdi, %rax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(last_vec_x2): + bsrl %eax, %eax + addl $(VEC_SIZE * 2), %eax + addq %rdi, %rax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(last_vec_x3): +@@ -232,8 +235,7 @@ L(last_vec_x1_check): + jl L(zero) + addl $VEC_SIZE, %eax + addq %rdi, %rax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(last_vec_x3_check): +@@ -243,12 +245,14 @@ L(last_vec_x3_check): + jl L(zero) + addl $(VEC_SIZE * 3), %eax + addq %rdi, %rax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(zero): +- VZEROUPPER ++ xorl %eax, %eax ++ VZEROUPPER_RETURN ++ ++ .p2align 4 + L(null): + xorl %eax, %eax + ret +@@ -273,8 +277,7 @@ L(last_vec_or_less_aligned): + + bsrl %eax, %eax + addq %rdi, %rax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(last_vec_or_less): +@@ -315,8 +318,7 @@ L(last_vec_or_less): + bsrl %eax, %eax + addq %rdi, %rax + addq %r8, %rax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(last_vec_2x_aligned): +@@ -353,7 +355,6 @@ L(last_vec_2x_aligned): + bsrl %eax, %eax + addq %rdi, %rax + addq %r8, %rax +- VZEROUPPER +- ret +-END (__memrchr_avx2) ++ VZEROUPPER_RETURN ++END (MEMRCHR) + #endif +diff --git a/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms-rtm.S b/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms-rtm.S +new file mode 100644 +index 00000000..8ac3e479 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms-rtm.S +@@ -0,0 +1,10 @@ ++#define ZERO_UPPER_VEC_REGISTERS_RETURN \ ++ ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST ++ ++#define VZEROUPPER_RETURN jmp L(return) ++ ++#define SECTION(p) p##.avx.rtm ++#define MEMSET_SYMBOL(p,s) p##_avx2_##s##_rtm ++#define WMEMSET_SYMBOL(p,s) p##_avx2_##s##_rtm ++ ++#include "memset-avx2-unaligned-erms.S" +diff --git a/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S +index 7ab3d898..ae0860f3 100644 +--- a/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S ++++ b/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S +@@ -14,9 +14,15 @@ + movq r, %rax; \ + vpbroadcastd %xmm0, %ymm0 + +-# define SECTION(p) p##.avx +-# define MEMSET_SYMBOL(p,s) p##_avx2_##s +-# define WMEMSET_SYMBOL(p,s) p##_avx2_##s ++# ifndef SECTION ++# define SECTION(p) p##.avx ++# endif ++# ifndef MEMSET_SYMBOL ++# define MEMSET_SYMBOL(p,s) p##_avx2_##s ++# endif ++# ifndef WMEMSET_SYMBOL ++# define WMEMSET_SYMBOL(p,s) p##_avx2_##s ++# endif + + # include "memset-vec-unaligned-erms.S" + #endif +diff --git a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S +index 71e91a8f..bae5cba4 100644 +--- a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S ++++ b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S +@@ -45,17 +45,14 @@ + #ifndef VZEROUPPER + # if VEC_SIZE > 16 + # define VZEROUPPER vzeroupper ++# define VZEROUPPER_SHORT_RETURN vzeroupper; ret + # else + # define VZEROUPPER + # endif + #endif + + #ifndef VZEROUPPER_SHORT_RETURN +-# if VEC_SIZE > 16 +-# define VZEROUPPER_SHORT_RETURN vzeroupper +-# else +-# define VZEROUPPER_SHORT_RETURN rep +-# endif ++# define VZEROUPPER_SHORT_RETURN rep; ret + #endif + + #ifndef MOVQ +@@ -117,8 +114,7 @@ L(entry_from_bzero): + /* From VEC and to 2 * VEC. No branch when size == VEC_SIZE. */ + VMOVU %VEC(0), -VEC_SIZE(%rdi,%rdx) + VMOVU %VEC(0), (%rdi) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + #if defined USE_MULTIARCH && IS_IN (libc) + END (MEMSET_SYMBOL (__memset, unaligned)) + +@@ -141,14 +137,12 @@ ENTRY (__memset_erms) + ENTRY (MEMSET_SYMBOL (__memset, erms)) + # endif + L(stosb): +- /* Issue vzeroupper before rep stosb. */ +- VZEROUPPER + mov %RDX_LP, %RCX_LP + movzbl %sil, %eax + mov %RDI_LP, %RDX_LP + rep stosb + mov %RDX_LP, %RAX_LP +- ret ++ VZEROUPPER_RETURN + # if VEC_SIZE == 16 + END (__memset_erms) + # else +@@ -175,8 +169,7 @@ ENTRY (MEMSET_SYMBOL (__memset, unaligned_erms)) + /* From VEC and to 2 * VEC. No branch when size == VEC_SIZE. */ + VMOVU %VEC(0), -VEC_SIZE(%rdi,%rdx) + VMOVU %VEC(0), (%rdi) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + L(stosb_more_2x_vec): + cmp __x86_rep_stosb_threshold(%rip), %RDX_LP +@@ -190,8 +183,11 @@ L(more_2x_vec): + VMOVU %VEC(0), -VEC_SIZE(%rdi,%rdx) + VMOVU %VEC(0), -(VEC_SIZE * 2)(%rdi,%rdx) + L(return): +- VZEROUPPER ++#if VEC_SIZE > 16 ++ ZERO_UPPER_VEC_REGISTERS_RETURN ++#else + ret ++#endif + + L(loop_start): + leaq (VEC_SIZE * 4)(%rdi), %rcx +@@ -217,7 +213,6 @@ L(loop): + cmpq %rcx, %rdx + jne L(loop) + VZEROUPPER_SHORT_RETURN +- ret + L(less_vec): + /* Less than 1 VEC. */ + # if VEC_SIZE != 16 && VEC_SIZE != 32 && VEC_SIZE != 64 +@@ -241,40 +236,34 @@ L(less_vec): + jb 1f + movb %cl, (%rdi) + 1: +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + # if VEC_SIZE > 32 + /* From 32 to 63. No branch when size == 32. */ + L(between_32_63): + VMOVU %YMM0, -32(%rdi,%rdx) + VMOVU %YMM0, (%rdi) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + # endif + # if VEC_SIZE > 16 + /* From 16 to 31. No branch when size == 16. */ + L(between_16_31): + VMOVU %XMM0, -16(%rdi,%rdx) + VMOVU %XMM0, (%rdi) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + # endif + /* From 8 to 15. No branch when size == 8. */ + L(between_8_15): + movq %rcx, -8(%rdi,%rdx) + movq %rcx, (%rdi) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + L(between_4_7): + /* From 4 to 7. No branch when size == 4. */ + movl %ecx, -4(%rdi,%rdx) + movl %ecx, (%rdi) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + L(between_2_3): + /* From 2 to 3. No branch when size == 2. */ + movw %cx, -2(%rdi,%rdx) + movw %cx, (%rdi) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + END (MEMSET_SYMBOL (__memset, unaligned_erms)) +diff --git a/sysdeps/x86_64/multiarch/rawmemchr-avx2-rtm.S b/sysdeps/x86_64/multiarch/rawmemchr-avx2-rtm.S +new file mode 100644 +index 00000000..acc5f6e2 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/rawmemchr-avx2-rtm.S +@@ -0,0 +1,4 @@ ++#define MEMCHR __rawmemchr_avx2_rtm ++#define USE_AS_RAWMEMCHR 1 ++ ++#include "memchr-avx2-rtm.S" +diff --git a/sysdeps/x86_64/multiarch/stpcpy-avx2-rtm.S b/sysdeps/x86_64/multiarch/stpcpy-avx2-rtm.S +new file mode 100644 +index 00000000..2b9c07a5 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/stpcpy-avx2-rtm.S +@@ -0,0 +1,3 @@ ++#define USE_AS_STPCPY ++#define STRCPY __stpcpy_avx2_rtm ++#include "strcpy-avx2-rtm.S" +diff --git a/sysdeps/x86_64/multiarch/stpncpy-avx2-rtm.S b/sysdeps/x86_64/multiarch/stpncpy-avx2-rtm.S +new file mode 100644 +index 00000000..60a2ccfe +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/stpncpy-avx2-rtm.S +@@ -0,0 +1,4 @@ ++#define USE_AS_STPCPY ++#define USE_AS_STRNCPY ++#define STRCPY __stpncpy_avx2_rtm ++#include "strcpy-avx2-rtm.S" +diff --git a/sysdeps/x86_64/multiarch/strcat-avx2-rtm.S b/sysdeps/x86_64/multiarch/strcat-avx2-rtm.S +new file mode 100644 +index 00000000..637fb557 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strcat-avx2-rtm.S +@@ -0,0 +1,12 @@ ++#ifndef STRCAT ++# define STRCAT __strcat_avx2_rtm ++#endif ++ ++#define ZERO_UPPER_VEC_REGISTERS_RETURN \ ++ ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST ++ ++#define VZEROUPPER_RETURN jmp L(return_vzeroupper) ++ ++#define SECTION(p) p##.avx.rtm ++ ++#include "strcat-avx2.S" +diff --git a/sysdeps/x86_64/multiarch/strcat-avx2.S b/sysdeps/x86_64/multiarch/strcat-avx2.S +index b0623564..aa48c058 100644 +--- a/sysdeps/x86_64/multiarch/strcat-avx2.S ++++ b/sysdeps/x86_64/multiarch/strcat-avx2.S +@@ -30,7 +30,11 @@ + /* Number of bytes in a vector register */ + # define VEC_SIZE 32 + +- .section .text.avx,"ax",@progbits ++# ifndef SECTION ++# define SECTION(p) p##.avx ++# endif ++ ++ .section SECTION(.text),"ax",@progbits + ENTRY (STRCAT) + mov %rdi, %r9 + # ifdef USE_AS_STRNCAT +diff --git a/sysdeps/x86_64/multiarch/strchr-avx2-rtm.S b/sysdeps/x86_64/multiarch/strchr-avx2-rtm.S +new file mode 100644 +index 00000000..81f20d1d +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strchr-avx2-rtm.S +@@ -0,0 +1,12 @@ ++#ifndef STRCHR ++# define STRCHR __strchr_avx2_rtm ++#endif ++ ++#define ZERO_UPPER_VEC_REGISTERS_RETURN \ ++ ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST ++ ++#define VZEROUPPER_RETURN jmp L(return_vzeroupper) ++ ++#define SECTION(p) p##.avx.rtm ++ ++#include "strchr-avx2.S" +diff --git a/sysdeps/x86_64/multiarch/strchr-avx2.S b/sysdeps/x86_64/multiarch/strchr-avx2.S +index 47bc3c99..da7d2620 100644 +--- a/sysdeps/x86_64/multiarch/strchr-avx2.S ++++ b/sysdeps/x86_64/multiarch/strchr-avx2.S +@@ -38,9 +38,13 @@ + # define VZEROUPPER vzeroupper + # endif + ++# ifndef SECTION ++# define SECTION(p) p##.avx ++# endif ++ + # define VEC_SIZE 32 + +- .section .text.avx,"ax",@progbits ++ .section SECTION(.text),"ax",@progbits + ENTRY (STRCHR) + movl %edi, %ecx + /* Broadcast CHAR to YMM0. */ +@@ -93,8 +97,8 @@ L(cros_page_boundary): + cmp (%rax), %CHAR_REG + cmovne %rdx, %rax + # endif +- VZEROUPPER +- ret ++L(return_vzeroupper): ++ ZERO_UPPER_VEC_REGISTERS_RETURN + + .p2align 4 + L(aligned_more): +@@ -190,8 +194,7 @@ L(first_vec_x0): + cmp (%rax), %CHAR_REG + cmovne %rdx, %rax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(first_vec_x1): +@@ -205,8 +208,7 @@ L(first_vec_x1): + cmp (%rax), %CHAR_REG + cmovne %rdx, %rax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(first_vec_x2): +@@ -220,8 +222,7 @@ L(first_vec_x2): + cmp (%rax), %CHAR_REG + cmovne %rdx, %rax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(4x_vec_end): +@@ -247,8 +248,7 @@ L(first_vec_x3): + cmp (%rax), %CHAR_REG + cmovne %rdx, %rax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + END (STRCHR) + #endif +diff --git a/sysdeps/x86_64/multiarch/strchr.c b/sysdeps/x86_64/multiarch/strchr.c +index be05e197..7e582f02 100644 +--- a/sysdeps/x86_64/multiarch/strchr.c ++++ b/sysdeps/x86_64/multiarch/strchr.c +@@ -29,6 +29,7 @@ + extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_no_bsf) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden; + + static inline void * +@@ -44,6 +45,9 @@ IFUNC_SELECTOR (void) + && CPU_FEATURE_USABLE_P (cpu_features, BMI2)) + return OPTIMIZE (evex); + ++ if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) ++ return OPTIMIZE (avx2_rtm); ++ + if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) + return OPTIMIZE (avx2); + } +diff --git a/sysdeps/x86_64/multiarch/strchrnul-avx2-rtm.S b/sysdeps/x86_64/multiarch/strchrnul-avx2-rtm.S +new file mode 100644 +index 00000000..cdcf818b +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strchrnul-avx2-rtm.S +@@ -0,0 +1,3 @@ ++#define STRCHR __strchrnul_avx2_rtm ++#define USE_AS_STRCHRNUL 1 ++#include "strchr-avx2-rtm.S" +diff --git a/sysdeps/x86_64/multiarch/strcmp-avx2-rtm.S b/sysdeps/x86_64/multiarch/strcmp-avx2-rtm.S +new file mode 100644 +index 00000000..aecd30d9 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strcmp-avx2-rtm.S +@@ -0,0 +1,12 @@ ++#ifndef STRCMP ++# define STRCMP __strcmp_avx2_rtm ++#endif ++ ++#define ZERO_UPPER_VEC_REGISTERS_RETURN \ ++ ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST ++ ++#define VZEROUPPER_RETURN jmp L(return_vzeroupper) ++ ++#define SECTION(p) p##.avx.rtm ++ ++#include "strcmp-avx2.S" +diff --git a/sysdeps/x86_64/multiarch/strcmp-avx2.S b/sysdeps/x86_64/multiarch/strcmp-avx2.S +index 8fb8eedc..5d1c9d90 100644 +--- a/sysdeps/x86_64/multiarch/strcmp-avx2.S ++++ b/sysdeps/x86_64/multiarch/strcmp-avx2.S +@@ -55,6 +55,10 @@ + # define VZEROUPPER vzeroupper + # endif + ++# ifndef SECTION ++# define SECTION(p) p##.avx ++# endif ++ + /* Warning! + wcscmp/wcsncmp have to use SIGNED comparison for elements. + strcmp/strncmp have to use UNSIGNED comparison for elements. +@@ -75,7 +79,7 @@ + the maximum offset is reached before a difference is found, zero is + returned. */ + +- .section .text.avx,"ax",@progbits ++ .section SECTION(.text),"ax",@progbits + ENTRY (STRCMP) + # ifdef USE_AS_STRNCMP + /* Check for simple cases (0 or 1) in offset. */ +@@ -137,8 +141,8 @@ L(return): + movzbl (%rsi, %rdx), %edx + subl %edx, %eax + # endif +- VZEROUPPER +- ret ++L(return_vzeroupper): ++ ZERO_UPPER_VEC_REGISTERS_RETURN + + .p2align 4 + L(return_vec_size): +@@ -171,8 +175,7 @@ L(return_vec_size): + subl %edx, %eax + # endif + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(return_2_vec_size): +@@ -205,8 +208,7 @@ L(return_2_vec_size): + subl %edx, %eax + # endif + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(return_3_vec_size): +@@ -239,8 +241,7 @@ L(return_3_vec_size): + subl %edx, %eax + # endif + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(next_3_vectors): +@@ -366,8 +367,7 @@ L(back_to_loop): + subl %edx, %eax + # endif + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(test_vec): +@@ -410,8 +410,7 @@ L(test_vec): + subl %edx, %eax + # endif + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(test_2_vec): +@@ -454,8 +453,7 @@ L(test_2_vec): + subl %edx, %eax + # endif + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(test_3_vec): +@@ -496,8 +494,7 @@ L(test_3_vec): + subl %edx, %eax + # endif + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(loop_cross_page): +@@ -566,8 +563,7 @@ L(loop_cross_page): + subl %edx, %eax + # endif + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(loop_cross_page_2_vec): +@@ -641,8 +637,7 @@ L(loop_cross_page_2_vec): + subl %edx, %eax + # endif + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + # ifdef USE_AS_STRNCMP + L(string_nbyte_offset_check): +@@ -684,8 +679,7 @@ L(cross_page_loop): + # ifndef USE_AS_WCSCMP + L(different): + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + # ifdef USE_AS_WCSCMP + .p2align 4 +@@ -695,16 +689,14 @@ L(different): + setl %al + negl %eax + orl $1, %eax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + # endif + + # ifdef USE_AS_STRNCMP + .p2align 4 + L(zero): + xorl %eax, %eax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(char0): +@@ -718,8 +710,7 @@ L(char0): + movzbl (%rdi), %eax + subl %ecx, %eax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + # endif + + .p2align 4 +@@ -744,8 +735,7 @@ L(last_vector): + movzbl (%rsi, %rdx), %edx + subl %edx, %eax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + /* Comparing on page boundary region requires special treatment: + It must done one vector at the time, starting with the wider +@@ -866,7 +856,6 @@ L(cross_page_4bytes): + testl %eax, %eax + jne L(cross_page_loop) + subl %ecx, %eax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + END (STRCMP) + #endif +diff --git a/sysdeps/x86_64/multiarch/strcmp.c b/sysdeps/x86_64/multiarch/strcmp.c +index c5f38510..11bbea2b 100644 +--- a/sysdeps/x86_64/multiarch/strcmp.c ++++ b/sysdeps/x86_64/multiarch/strcmp.c +@@ -30,6 +30,7 @@ extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden; + + static inline void * +@@ -46,6 +47,9 @@ IFUNC_SELECTOR (void) + && !CPU_FEATURES_ARCH_P (cpu_features, Prefer_AVX2_STRCMP)) + return OPTIMIZE (evex); + ++ if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) ++ return OPTIMIZE (avx2_rtm); ++ + if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) + return OPTIMIZE (avx2); + } +diff --git a/sysdeps/x86_64/multiarch/strcpy-avx2-rtm.S b/sysdeps/x86_64/multiarch/strcpy-avx2-rtm.S +new file mode 100644 +index 00000000..c2c581ec +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strcpy-avx2-rtm.S +@@ -0,0 +1,12 @@ ++#ifndef STRCPY ++# define STRCPY __strcpy_avx2_rtm ++#endif ++ ++#define ZERO_UPPER_VEC_REGISTERS_RETURN \ ++ ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST ++ ++#define VZEROUPPER_RETURN jmp L(return_vzeroupper) ++ ++#define SECTION(p) p##.avx.rtm ++ ++#include "strcpy-avx2.S" +diff --git a/sysdeps/x86_64/multiarch/strcpy-avx2.S b/sysdeps/x86_64/multiarch/strcpy-avx2.S +index 81677f90..613c59aa 100644 +--- a/sysdeps/x86_64/multiarch/strcpy-avx2.S ++++ b/sysdeps/x86_64/multiarch/strcpy-avx2.S +@@ -37,6 +37,10 @@ + # define VZEROUPPER vzeroupper + # endif + ++# ifndef SECTION ++# define SECTION(p) p##.avx ++# endif ++ + /* zero register */ + #define xmmZ xmm0 + #define ymmZ ymm0 +@@ -46,7 +50,7 @@ + + # ifndef USE_AS_STRCAT + +- .section .text.avx,"ax",@progbits ++ .section SECTION(.text),"ax",@progbits + ENTRY (STRCPY) + # ifdef USE_AS_STRNCPY + mov %rdx, %r8 +@@ -369,8 +373,8 @@ L(CopyVecSizeExit): + lea 1(%rdi), %rdi + jnz L(StrncpyFillTailWithZero) + # endif +- VZEROUPPER +- ret ++L(return_vzeroupper): ++ ZERO_UPPER_VEC_REGISTERS_RETURN + + .p2align 4 + L(CopyTwoVecSize1): +@@ -553,8 +557,7 @@ L(Exit1): + lea 2(%rdi), %rdi + jnz L(StrncpyFillTailWithZero) + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(Exit2): +@@ -569,8 +572,7 @@ L(Exit2): + lea 3(%rdi), %rdi + jnz L(StrncpyFillTailWithZero) + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(Exit3): +@@ -584,8 +586,7 @@ L(Exit3): + lea 4(%rdi), %rdi + jnz L(StrncpyFillTailWithZero) + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(Exit4_7): +@@ -602,8 +603,7 @@ L(Exit4_7): + lea 1(%rdi, %rdx), %rdi + jnz L(StrncpyFillTailWithZero) + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(Exit8_15): +@@ -620,8 +620,7 @@ L(Exit8_15): + lea 1(%rdi, %rdx), %rdi + jnz L(StrncpyFillTailWithZero) + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(Exit16_31): +@@ -638,8 +637,7 @@ L(Exit16_31): + lea 1(%rdi, %rdx), %rdi + jnz L(StrncpyFillTailWithZero) + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(Exit32_63): +@@ -656,8 +654,7 @@ L(Exit32_63): + lea 1(%rdi, %rdx), %rdi + jnz L(StrncpyFillTailWithZero) + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + # ifdef USE_AS_STRNCPY + +@@ -671,8 +668,7 @@ L(StrncpyExit1): + # ifdef USE_AS_STRCAT + movb $0, 1(%rdi) + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(StrncpyExit2): +@@ -684,8 +680,7 @@ L(StrncpyExit2): + # ifdef USE_AS_STRCAT + movb $0, 2(%rdi) + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(StrncpyExit3_4): +@@ -699,8 +694,7 @@ L(StrncpyExit3_4): + # ifdef USE_AS_STRCAT + movb $0, (%rdi, %r8) + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(StrncpyExit5_8): +@@ -714,8 +708,7 @@ L(StrncpyExit5_8): + # ifdef USE_AS_STRCAT + movb $0, (%rdi, %r8) + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(StrncpyExit9_16): +@@ -729,8 +722,7 @@ L(StrncpyExit9_16): + # ifdef USE_AS_STRCAT + movb $0, (%rdi, %r8) + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(StrncpyExit17_32): +@@ -744,8 +736,7 @@ L(StrncpyExit17_32): + # ifdef USE_AS_STRCAT + movb $0, (%rdi, %r8) + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(StrncpyExit33_64): +@@ -760,8 +751,7 @@ L(StrncpyExit33_64): + # ifdef USE_AS_STRCAT + movb $0, (%rdi, %r8) + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(StrncpyExit65): +@@ -778,50 +768,43 @@ L(StrncpyExit65): + # ifdef USE_AS_STRCAT + movb $0, 65(%rdi) + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + # ifndef USE_AS_STRCAT + + .p2align 4 + L(Fill1): + mov %dl, (%rdi) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(Fill2): + mov %dx, (%rdi) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(Fill3_4): + mov %dx, (%rdi) + mov %dx, -2(%rdi, %r8) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(Fill5_8): + mov %edx, (%rdi) + mov %edx, -4(%rdi, %r8) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(Fill9_16): + mov %rdx, (%rdi) + mov %rdx, -8(%rdi, %r8) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(Fill17_32): + vmovdqu %xmmZ, (%rdi) + vmovdqu %xmmZ, -16(%rdi, %r8) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(CopyVecSizeUnalignedVec2): +@@ -898,8 +881,7 @@ L(Fill): + cmp $1, %r8d + ja L(Fill2) + je L(Fill1) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + /* end of ifndef USE_AS_STRCAT */ + # endif +@@ -929,8 +911,7 @@ L(UnalignedFourVecSizeLeaveCase3): + # ifdef USE_AS_STRCAT + movb $0, (VEC_SIZE * 4)(%rdi) + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(UnalignedFourVecSizeLeaveCase2): +@@ -1001,16 +982,14 @@ L(StrncpyExit): + # ifdef USE_AS_STRCAT + movb $0, (%rdi) + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(ExitZero): + # ifndef USE_AS_STRCAT + mov %rdi, %rax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + # endif + +diff --git a/sysdeps/x86_64/multiarch/strlen-avx2-rtm.S b/sysdeps/x86_64/multiarch/strlen-avx2-rtm.S +new file mode 100644 +index 00000000..75b4b761 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strlen-avx2-rtm.S +@@ -0,0 +1,12 @@ ++#ifndef STRLEN ++# define STRLEN __strlen_avx2_rtm ++#endif ++ ++#define ZERO_UPPER_VEC_REGISTERS_RETURN \ ++ ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST ++ ++#define VZEROUPPER_RETURN jmp L(return_vzeroupper) ++ ++#define SECTION(p) p##.avx.rtm ++ ++#include "strlen-avx2.S" +diff --git a/sysdeps/x86_64/multiarch/strlen-avx2.S b/sysdeps/x86_64/multiarch/strlen-avx2.S +index 645e0446..82826e10 100644 +--- a/sysdeps/x86_64/multiarch/strlen-avx2.S ++++ b/sysdeps/x86_64/multiarch/strlen-avx2.S +@@ -36,9 +36,13 @@ + # define VZEROUPPER vzeroupper + # endif + ++# ifndef SECTION ++# define SECTION(p) p##.avx ++# endif ++ + # define VEC_SIZE 32 + +- .section .text.avx,"ax",@progbits ++ .section SECTION(.text),"ax",@progbits + ENTRY (STRLEN) + # ifdef USE_AS_STRNLEN + /* Check for zero length. */ +@@ -111,8 +115,8 @@ L(cros_page_boundary): + # ifdef USE_AS_WCSLEN + shrq $2, %rax + # endif +- VZEROUPPER +- ret ++L(return_vzeroupper): ++ ZERO_UPPER_VEC_REGISTERS_RETURN + + .p2align 4 + L(aligned_more): +@@ -231,8 +235,7 @@ L(last_4x_vec_or_less): + # ifdef USE_AS_WCSLEN + shrq $2, %rax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(last_2x_vec): +@@ -253,8 +256,7 @@ L(last_2x_vec): + # ifdef USE_AS_WCSLEN + shrq $2, %rax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(first_vec_x0_check): +@@ -267,8 +269,7 @@ L(first_vec_x0_check): + # ifdef USE_AS_WCSLEN + shrq $2, %rax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(first_vec_x1_check): +@@ -282,8 +283,7 @@ L(first_vec_x1_check): + # ifdef USE_AS_WCSLEN + shrq $2, %rax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(first_vec_x2_check): +@@ -297,8 +297,7 @@ L(first_vec_x2_check): + # ifdef USE_AS_WCSLEN + shrq $2, %rax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(first_vec_x3_check): +@@ -312,8 +311,7 @@ L(first_vec_x3_check): + # ifdef USE_AS_WCSLEN + shrq $2, %rax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(max): +@@ -321,8 +319,7 @@ L(max): + # ifdef USE_AS_WCSLEN + shrq $2, %rax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(zero): +@@ -338,8 +335,7 @@ L(first_vec_x0): + # ifdef USE_AS_WCSLEN + shrq $2, %rax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(first_vec_x1): +@@ -350,8 +346,7 @@ L(first_vec_x1): + # ifdef USE_AS_WCSLEN + shrq $2, %rax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(first_vec_x2): +@@ -362,8 +357,7 @@ L(first_vec_x2): + # ifdef USE_AS_WCSLEN + shrq $2, %rax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(4x_vec_end): +@@ -389,8 +383,7 @@ L(first_vec_x3): + # ifdef USE_AS_WCSLEN + shrq $2, %rax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + END (STRLEN) + #endif +diff --git a/sysdeps/x86_64/multiarch/strncat-avx2-rtm.S b/sysdeps/x86_64/multiarch/strncat-avx2-rtm.S +new file mode 100644 +index 00000000..0dcea18d +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strncat-avx2-rtm.S +@@ -0,0 +1,3 @@ ++#define USE_AS_STRNCAT ++#define STRCAT __strncat_avx2_rtm ++#include "strcat-avx2-rtm.S" +diff --git a/sysdeps/x86_64/multiarch/strncmp-avx2-rtm.S b/sysdeps/x86_64/multiarch/strncmp-avx2-rtm.S +new file mode 100644 +index 00000000..37d1224b +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strncmp-avx2-rtm.S +@@ -0,0 +1,3 @@ ++#define STRCMP __strncmp_avx2_rtm ++#define USE_AS_STRNCMP 1 ++#include "strcmp-avx2-rtm.S" +diff --git a/sysdeps/x86_64/multiarch/strncmp.c b/sysdeps/x86_64/multiarch/strncmp.c +index 4c15542f..44c85116 100644 +--- a/sysdeps/x86_64/multiarch/strncmp.c ++++ b/sysdeps/x86_64/multiarch/strncmp.c +@@ -30,6 +30,7 @@ extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (sse42) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden; + + static inline void * +@@ -46,6 +47,9 @@ IFUNC_SELECTOR (void) + && !CPU_FEATURES_ARCH_P (cpu_features, Prefer_AVX2_STRCMP)) + return OPTIMIZE (evex); + ++ if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) ++ return OPTIMIZE (avx2_rtm); ++ + if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) + return OPTIMIZE (avx2); + } +diff --git a/sysdeps/x86_64/multiarch/strncpy-avx2-rtm.S b/sysdeps/x86_64/multiarch/strncpy-avx2-rtm.S +new file mode 100644 +index 00000000..79e70832 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strncpy-avx2-rtm.S +@@ -0,0 +1,3 @@ ++#define USE_AS_STRNCPY ++#define STRCPY __strncpy_avx2_rtm ++#include "strcpy-avx2-rtm.S" +diff --git a/sysdeps/x86_64/multiarch/strnlen-avx2-rtm.S b/sysdeps/x86_64/multiarch/strnlen-avx2-rtm.S +new file mode 100644 +index 00000000..04f1626a +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strnlen-avx2-rtm.S +@@ -0,0 +1,4 @@ ++#define STRLEN __strnlen_avx2_rtm ++#define USE_AS_STRNLEN 1 ++ ++#include "strlen-avx2-rtm.S" +diff --git a/sysdeps/x86_64/multiarch/strrchr-avx2-rtm.S b/sysdeps/x86_64/multiarch/strrchr-avx2-rtm.S +new file mode 100644 +index 00000000..5def14ec +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strrchr-avx2-rtm.S +@@ -0,0 +1,12 @@ ++#ifndef STRRCHR ++# define STRRCHR __strrchr_avx2_rtm ++#endif ++ ++#define ZERO_UPPER_VEC_REGISTERS_RETURN \ ++ ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST ++ ++#define VZEROUPPER_RETURN jmp L(return_vzeroupper) ++ ++#define SECTION(p) p##.avx.rtm ++ ++#include "strrchr-avx2.S" +diff --git a/sysdeps/x86_64/multiarch/strrchr-avx2.S b/sysdeps/x86_64/multiarch/strrchr-avx2.S +index 4381e6ab..9f22a15e 100644 +--- a/sysdeps/x86_64/multiarch/strrchr-avx2.S ++++ b/sysdeps/x86_64/multiarch/strrchr-avx2.S +@@ -36,9 +36,13 @@ + # define VZEROUPPER vzeroupper + # endif + ++# ifndef SECTION ++# define SECTION(p) p##.avx ++# endif ++ + # define VEC_SIZE 32 + +- .section .text.avx,"ax",@progbits ++ .section SECTION(.text),"ax",@progbits + ENTRY (STRRCHR) + movd %esi, %xmm4 + movl %edi, %ecx +@@ -166,8 +170,8 @@ L(return_value): + # endif + bsrl %eax, %eax + leaq -VEC_SIZE(%rdi, %rax), %rax +- VZEROUPPER +- ret ++L(return_vzeroupper): ++ ZERO_UPPER_VEC_REGISTERS_RETURN + + .p2align 4 + L(match): +@@ -198,8 +202,7 @@ L(find_nul): + jz L(return_value) + bsrl %eax, %eax + leaq -VEC_SIZE(%rdi, %rax), %rax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(char_and_nul): +@@ -222,14 +225,12 @@ L(char_and_nul_in_first_vec): + jz L(return_null) + bsrl %eax, %eax + leaq -VEC_SIZE(%rdi, %rax), %rax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(return_null): + xorl %eax, %eax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + END (STRRCHR) + #endif +diff --git a/sysdeps/x86_64/multiarch/wcschr-avx2-rtm.S b/sysdeps/x86_64/multiarch/wcschr-avx2-rtm.S +new file mode 100644 +index 00000000..d49dbbf0 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/wcschr-avx2-rtm.S +@@ -0,0 +1,3 @@ ++#define STRCHR __wcschr_avx2_rtm ++#define USE_AS_WCSCHR 1 ++#include "strchr-avx2-rtm.S" +diff --git a/sysdeps/x86_64/multiarch/wcscmp-avx2-rtm.S b/sysdeps/x86_64/multiarch/wcscmp-avx2-rtm.S +new file mode 100644 +index 00000000..d6ca2b80 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/wcscmp-avx2-rtm.S +@@ -0,0 +1,4 @@ ++#define STRCMP __wcscmp_avx2_rtm ++#define USE_AS_WCSCMP 1 ++ ++#include "strcmp-avx2-rtm.S" +diff --git a/sysdeps/x86_64/multiarch/wcslen-avx2-rtm.S b/sysdeps/x86_64/multiarch/wcslen-avx2-rtm.S +new file mode 100644 +index 00000000..35658d73 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/wcslen-avx2-rtm.S +@@ -0,0 +1,4 @@ ++#define STRLEN __wcslen_avx2_rtm ++#define USE_AS_WCSLEN 1 ++ ++#include "strlen-avx2-rtm.S" +diff --git a/sysdeps/x86_64/multiarch/wcsncmp-avx2-rtm.S b/sysdeps/x86_64/multiarch/wcsncmp-avx2-rtm.S +new file mode 100644 +index 00000000..4e88c70c +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/wcsncmp-avx2-rtm.S +@@ -0,0 +1,5 @@ ++#define STRCMP __wcsncmp_avx2_rtm ++#define USE_AS_STRNCMP 1 ++#define USE_AS_WCSCMP 1 ++ ++#include "strcmp-avx2-rtm.S" +diff --git a/sysdeps/x86_64/multiarch/wcsnlen-avx2-rtm.S b/sysdeps/x86_64/multiarch/wcsnlen-avx2-rtm.S +new file mode 100644 +index 00000000..7437ebee +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/wcsnlen-avx2-rtm.S +@@ -0,0 +1,5 @@ ++#define STRLEN __wcsnlen_avx2_rtm ++#define USE_AS_WCSLEN 1 ++#define USE_AS_STRNLEN 1 ++ ++#include "strlen-avx2-rtm.S" +diff --git a/sysdeps/x86_64/multiarch/wcsnlen.c b/sysdeps/x86_64/multiarch/wcsnlen.c +index 84254b83..20b731ae 100644 +--- a/sysdeps/x86_64/multiarch/wcsnlen.c ++++ b/sysdeps/x86_64/multiarch/wcsnlen.c +@@ -29,6 +29,7 @@ + extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (sse4_1) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden; + + static inline void * +@@ -44,6 +45,9 @@ IFUNC_SELECTOR (void) + && CPU_FEATURE_USABLE_P (cpu_features, BMI2)) + return OPTIMIZE (evex); + ++ if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) ++ return OPTIMIZE (avx2_rtm); ++ + if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) + return OPTIMIZE (avx2); + } +diff --git a/sysdeps/x86_64/multiarch/wcsrchr-avx2-rtm.S b/sysdeps/x86_64/multiarch/wcsrchr-avx2-rtm.S +new file mode 100644 +index 00000000..9bf76083 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/wcsrchr-avx2-rtm.S +@@ -0,0 +1,3 @@ ++#define STRRCHR __wcsrchr_avx2_rtm ++#define USE_AS_WCSRCHR 1 ++#include "strrchr-avx2-rtm.S" +diff --git a/sysdeps/x86_64/multiarch/wmemchr-avx2-rtm.S b/sysdeps/x86_64/multiarch/wmemchr-avx2-rtm.S +new file mode 100644 +index 00000000..58ed21db +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/wmemchr-avx2-rtm.S +@@ -0,0 +1,4 @@ ++#define MEMCHR __wmemchr_avx2_rtm ++#define USE_AS_WMEMCHR 1 ++ ++#include "memchr-avx2-rtm.S" +diff --git a/sysdeps/x86_64/multiarch/wmemcmp-avx2-movbe-rtm.S b/sysdeps/x86_64/multiarch/wmemcmp-avx2-movbe-rtm.S +new file mode 100644 +index 00000000..31104d12 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/wmemcmp-avx2-movbe-rtm.S +@@ -0,0 +1,4 @@ ++#define MEMCMP __wmemcmp_avx2_movbe_rtm ++#define USE_AS_WMEMCMP 1 ++ ++#include "memcmp-avx2-movbe-rtm.S" +diff --git a/sysdeps/x86_64/sysdep.h b/sysdeps/x86_64/sysdep.h +index 1738d7f9..223f1a59 100644 +--- a/sysdeps/x86_64/sysdep.h ++++ b/sysdeps/x86_64/sysdep.h +@@ -95,6 +95,28 @@ lose: \ + #define R14_LP r14 + #define R15_LP r15 + ++/* Zero upper vector registers and return with xtest. NB: Use VZEROALL ++ to avoid RTM abort triggered by VZEROUPPER inside transactionally. */ ++#define ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST \ ++ xtest; \ ++ jz 1f; \ ++ vzeroall; \ ++ ret; \ ++1: \ ++ vzeroupper; \ ++ ret ++ ++/* Zero upper vector registers and return. */ ++#ifndef ZERO_UPPER_VEC_REGISTERS_RETURN ++# define ZERO_UPPER_VEC_REGISTERS_RETURN \ ++ VZEROUPPER; \ ++ ret ++#endif ++ ++#ifndef VZEROUPPER_RETURN ++# define VZEROUPPER_RETURN VZEROUPPER; ret ++#endif ++ + #else /* __ASSEMBLER__ */ + + /* Long and pointer size in bytes. */ +-- +GitLab + diff --git a/SOURCES/ia-bmi2-req-strchr-avx2.patch b/SOURCES/ia-bmi2-req-strchr-avx2.patch new file mode 100644 index 0000000..a4316f6 --- /dev/null +++ b/SOURCES/ia-bmi2-req-strchr-avx2.patch @@ -0,0 +1,93 @@ +From 6b2150da7cac8cccdcf1de189b3ab7cbd6519535 Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Mon, 19 Apr 2021 10:45:07 -0700 +Subject: [PATCH] x86-64: Require BMI2 for strchr-avx2.S + +Since strchr-avx2.S updated by + +commit 1f745ecc2109890886b161d4791e1406fdfc29b8 +Author: noah +Date: Wed Feb 3 00:38:59 2021 -0500 + + x86-64: Refactor and improve performance of strchr-avx2.S + +uses sarx: + +c4 e2 72 f7 c0 sarx %ecx,%eax,%eax + +for strchr-avx2 family functions, require BMI2 in ifunc-impl-list.c and +ifunc-avx2.h. + +(cherry picked from commit 83c5b368226c34a2f0a5287df40fc290b2b34359) +--- + sysdeps/x86_64/multiarch/ifunc-avx2.h | 4 ++-- + sysdeps/x86_64/multiarch/ifunc-impl-list.c | 12 +++++++++--- + 2 files changed, 11 insertions(+), 5 deletions(-) + +diff --git a/sysdeps/x86_64/multiarch/ifunc-avx2.h b/sysdeps/x86_64/multiarch/ifunc-avx2.h +index e0f30e61..ef72b73f 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-avx2.h ++++ b/sysdeps/x86_64/multiarch/ifunc-avx2.h +@@ -30,11 +30,11 @@ IFUNC_SELECTOR (void) + const struct cpu_features* cpu_features = __get_cpu_features (); + + if (CPU_FEATURE_USABLE_P (cpu_features, AVX2) ++ && CPU_FEATURE_USABLE_P (cpu_features, BMI2) + && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) + { + if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) +- && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW) +- && CPU_FEATURE_USABLE_P (cpu_features, BMI2)) ++ && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)) + return OPTIMIZE (evex); + + if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) +diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c +index 695cdba6..85b8863a 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c ++++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c +@@ -400,10 +400,12 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + /* Support sysdeps/x86_64/multiarch/strchr.c. */ + IFUNC_IMPL (i, name, strchr, + IFUNC_IMPL_ADD (array, i, strchr, +- CPU_FEATURE_USABLE (AVX2), ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (BMI2)), + __strchr_avx2) + IFUNC_IMPL_ADD (array, i, strchr, + (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (BMI2) + && CPU_FEATURE_USABLE (RTM)), + __strchr_avx2_rtm) + IFUNC_IMPL_ADD (array, i, strchr, +@@ -417,10 +419,12 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + /* Support sysdeps/x86_64/multiarch/strchrnul.c. */ + IFUNC_IMPL (i, name, strchrnul, + IFUNC_IMPL_ADD (array, i, strchrnul, +- CPU_FEATURE_USABLE (AVX2), ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (BMI2)), + __strchrnul_avx2) + IFUNC_IMPL_ADD (array, i, strchrnul, + (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (BMI2) + && CPU_FEATURE_USABLE (RTM)), + __strchrnul_avx2_rtm) + IFUNC_IMPL_ADD (array, i, strchrnul, +@@ -574,10 +578,12 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + /* Support sysdeps/x86_64/multiarch/wcschr.c. */ + IFUNC_IMPL (i, name, wcschr, + IFUNC_IMPL_ADD (array, i, wcschr, +- CPU_FEATURE_USABLE (AVX2), ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (BMI2)), + __wcschr_avx2) + IFUNC_IMPL_ADD (array, i, wcschr, + (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (BMI2) + && CPU_FEATURE_USABLE (RTM)), + __wcschr_avx2_rtm) + IFUNC_IMPL_ADD (array, i, wcschr, +-- +GitLab + diff --git a/SOURCES/ia-bmi2-req-strlen-strnlen.patch b/SOURCES/ia-bmi2-req-strlen-strnlen.patch new file mode 100644 index 0000000..7533cdd --- /dev/null +++ b/SOURCES/ia-bmi2-req-strlen-strnlen.patch @@ -0,0 +1,52 @@ +From 6518570fa9ff5c4d0a7e0c4b6788fb0f8e9ebb22 Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Mon, 19 Apr 2021 07:07:21 -0700 +Subject: [PATCH] x86-64: Require BMI2 for __strlen_evex and __strnlen_evex + +Since __strlen_evex and __strnlen_evex added by + +commit 1fd8c163a83d96ace1ff78fa6bac7aee084f6f77 +Author: H.J. Lu +Date: Fri Mar 5 06:24:52 2021 -0800 + + x86-64: Add ifunc-avx2.h functions with 256-bit EVEX + +use sarx: + +c4 e2 6a f7 c0 sarx %edx,%eax,%eax + +require BMI2 for __strlen_evex and __strnlen_evex in ifunc-impl-list.c. +ifunc-avx2.h already requires BMI2 for EVEX implementation. + +(cherry picked from commit 55bf411b451c13f0fb7ff3d3bf9a820020b45df1) +--- + sysdeps/x86_64/multiarch/ifunc-impl-list.c | 6 ++++-- + 1 file changed, 4 insertions(+), 2 deletions(-) + +diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c +index fec384f6..cbfc1a5d 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c ++++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c +@@ -293,7 +293,8 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + __strlen_avx2_rtm) + IFUNC_IMPL_ADD (array, i, strlen, + (CPU_FEATURE_USABLE (AVX512VL) +- && CPU_FEATURE_USABLE (AVX512BW)), ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), + __strlen_evex) + IFUNC_IMPL_ADD (array, i, strlen, 1, __strlen_sse2)) + +@@ -308,7 +309,8 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + __strnlen_avx2_rtm) + IFUNC_IMPL_ADD (array, i, strnlen, + (CPU_FEATURE_USABLE (AVX512VL) +- && CPU_FEATURE_USABLE (AVX512BW)), ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), + __strnlen_evex) + IFUNC_IMPL_ADD (array, i, strnlen, 1, __strnlen_sse2)) + +-- +GitLab + diff --git a/SOURCES/ia-double-rep_movsb_threshold-erms.patch b/SOURCES/ia-double-rep_movsb_threshold-erms.patch new file mode 100644 index 0000000..2c8534c --- /dev/null +++ b/SOURCES/ia-double-rep_movsb_threshold-erms.patch @@ -0,0 +1,136 @@ +From aa1f037077a41b36dd477e6ca754e207b37d661a Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Wed, 2 Mar 2022 16:27:24 -0800 +Subject: [PATCH] x86: Double size of ERMS rep_movsb_threshold in + dl-cacheinfo.h + +No bug. + +This patch doubles the rep_movsb_threshold when using ERMS. Based on +benchmarks the vector copy loop, especially now that it handles 4k +aliasing, is better for these medium ranged. + +On Skylake with ERMS: + +Size, Align1, Align2, dst>src,(rep movsb) / (vec copy) +4096, 0, 0, 0, 0.975 +4096, 0, 0, 1, 0.953 +4096, 12, 0, 0, 0.969 +4096, 12, 0, 1, 0.872 +4096, 44, 0, 0, 0.979 +4096, 44, 0, 1, 0.83 +4096, 0, 12, 0, 1.006 +4096, 0, 12, 1, 0.989 +4096, 0, 44, 0, 0.739 +4096, 0, 44, 1, 0.942 +4096, 12, 12, 0, 1.009 +4096, 12, 12, 1, 0.973 +4096, 44, 44, 0, 0.791 +4096, 44, 44, 1, 0.961 +4096, 2048, 0, 0, 0.978 +4096, 2048, 0, 1, 0.951 +4096, 2060, 0, 0, 0.986 +4096, 2060, 0, 1, 0.963 +4096, 2048, 12, 0, 0.971 +4096, 2048, 12, 1, 0.941 +4096, 2060, 12, 0, 0.977 +4096, 2060, 12, 1, 0.949 +8192, 0, 0, 0, 0.85 +8192, 0, 0, 1, 0.845 +8192, 13, 0, 0, 0.937 +8192, 13, 0, 1, 0.939 +8192, 45, 0, 0, 0.932 +8192, 45, 0, 1, 0.927 +8192, 0, 13, 0, 0.621 +8192, 0, 13, 1, 0.62 +8192, 0, 45, 0, 0.53 +8192, 0, 45, 1, 0.516 +8192, 13, 13, 0, 0.664 +8192, 13, 13, 1, 0.659 +8192, 45, 45, 0, 0.593 +8192, 45, 45, 1, 0.575 +8192, 2048, 0, 0, 0.854 +8192, 2048, 0, 1, 0.834 +8192, 2061, 0, 0, 0.863 +8192, 2061, 0, 1, 0.857 +8192, 2048, 13, 0, 0.63 +8192, 2048, 13, 1, 0.629 +8192, 2061, 13, 0, 0.627 +8192, 2061, 13, 1, 0.62 + +Signed-off-by: Noah Goldstein +Reviewed-by: H.J. Lu +(cherry picked from commit 475b63702ef38b69558fc3d31a0b66776a70f1d3) +--- + sysdeps/x86/cacheinfo.h | 8 +++++--- + sysdeps/x86/dl-tunables.list | 26 +++++++++++++++----------- + 2 files changed, 20 insertions(+), 14 deletions(-) + +diff --git a/sysdeps/x86/cacheinfo.h b/sysdeps/x86/cacheinfo.h +index cc3941d3..ac025e08 100644 +--- a/sysdeps/x86/cacheinfo.h ++++ b/sysdeps/x86/cacheinfo.h +@@ -411,18 +411,20 @@ init_cacheinfo (void) + + /* NB: The REP MOVSB threshold must be greater than VEC_SIZE * 8. */ + unsigned int minimum_rep_movsb_threshold; +- /* NB: The default REP MOVSB threshold is 2048 * (VEC_SIZE / 16). */ ++ /* NB: The default REP MOVSB threshold is 4096 * (VEC_SIZE / 16) for ++ VEC_SIZE == 64 or 32. For VEC_SIZE == 16, the default REP MOVSB ++ threshold is 2048 * (VEC_SIZE / 16). */ + unsigned int rep_movsb_threshold; + if (CPU_FEATURE_USABLE_P (cpu_features, AVX512F) + && !CPU_FEATURE_PREFERRED_P (cpu_features, Prefer_No_AVX512)) + { +- rep_movsb_threshold = 2048 * (64 / 16); ++ rep_movsb_threshold = 4096 * (64 / 16); + minimum_rep_movsb_threshold = 64 * 8; + } + else if (CPU_FEATURE_PREFERRED_P (cpu_features, + AVX_Fast_Unaligned_Load)) + { +- rep_movsb_threshold = 2048 * (32 / 16); ++ rep_movsb_threshold = 4096 * (32 / 16); + minimum_rep_movsb_threshold = 32 * 8; + } + else +diff --git a/sysdeps/x86/dl-tunables.list b/sysdeps/x86/dl-tunables.list +index 89bf2966..56c6834a 100644 +--- a/sysdeps/x86/dl-tunables.list ++++ b/sysdeps/x86/dl-tunables.list +@@ -32,17 +32,21 @@ glibc { + } + x86_rep_movsb_threshold { + type: SIZE_T +- # Since there is overhead to set up REP MOVSB operation, REP MOVSB +- # isn't faster on short data. The memcpy micro benchmark in glibc +- # shows that 2KB is the approximate value above which REP MOVSB +- # becomes faster than SSE2 optimization on processors with Enhanced +- # REP MOVSB. Since larger register size can move more data with a +- # single load and store, the threshold is higher with larger register +- # size. Note: Since the REP MOVSB threshold must be greater than 8 +- # times of vector size and the default value is 2048 * (vector size +- # / 16), the default value and the minimum value must be updated at +- # run-time. NB: Don't set the default value since we can't tell if +- # the tunable value is set by user or not [BZ #27069]. ++ # Since there is overhead to set up REP MOVSB operation, REP ++ # MOVSB isn't faster on short data. The memcpy micro benchmark ++ # in glibc shows that 2KB is the approximate value above which ++ # REP MOVSB becomes faster than SSE2 optimization on processors ++ # with Enhanced REP MOVSB. Since larger register size can move ++ # more data with a single load and store, the threshold is ++ # higher with larger register size. Micro benchmarks show AVX ++ # REP MOVSB becomes faster apprximately at 8KB. The AVX512 ++ # threshold is extrapolated to 16KB. For machines with FSRM the ++ # threshold is universally set at 2112 bytes. Note: Since the ++ # REP MOVSB threshold must be greater than 8 times of vector ++ # size and the default value is 4096 * (vector size / 16), the ++ # default value and the minimum value must be updated at ++ # run-time. NB: Don't set the default value since we can't tell ++ # if the tunable value is set by user or not [BZ #27069]. + minval: 1 + } + x86_rep_stosb_threshold { +-- +GitLab + diff --git a/SOURCES/ia-ifdef-indt-strlen-evex.patch b/SOURCES/ia-ifdef-indt-strlen-evex.patch new file mode 100644 index 0000000..d0f63fd --- /dev/null +++ b/SOURCES/ia-ifdef-indt-strlen-evex.patch @@ -0,0 +1,72 @@ +From b1ecc9d86746b8c0285935eae99c31ede19a7d9e Mon Sep 17 00:00:00 2001 +From: Sunil K Pandey +Date: Thu, 1 Apr 2021 15:47:04 -0700 +Subject: [PATCH] x86-64: Fix ifdef indentation in strlen-evex.S + +Fix some indentations of ifdef in file strlen-evex.S which are off by 1 +and confusing to read. + +(cherry picked from commit 595c22ecd8e87a27fd19270ed30fdbae9ad25426) +--- + sysdeps/x86_64/multiarch/strlen-evex.S | 16 ++++++++-------- + 1 file changed, 8 insertions(+), 8 deletions(-) + +diff --git a/sysdeps/x86_64/multiarch/strlen-evex.S b/sysdeps/x86_64/multiarch/strlen-evex.S +index cd022509..05838190 100644 +--- a/sysdeps/x86_64/multiarch/strlen-evex.S ++++ b/sysdeps/x86_64/multiarch/strlen-evex.S +@@ -276,10 +276,10 @@ L(last_2x_vec): + .p2align 4 + L(first_vec_x0_check): + tzcntl %eax, %eax +-# ifdef USE_AS_WCSLEN ++# ifdef USE_AS_WCSLEN + /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ + sall $2, %eax +-# endif ++# endif + /* Check the end of data. */ + cmpq %rax, %rsi + jbe L(max) +@@ -293,10 +293,10 @@ L(first_vec_x0_check): + .p2align 4 + L(first_vec_x1_check): + tzcntl %eax, %eax +-# ifdef USE_AS_WCSLEN ++# ifdef USE_AS_WCSLEN + /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ + sall $2, %eax +-# endif ++# endif + /* Check the end of data. */ + cmpq %rax, %rsi + jbe L(max) +@@ -311,10 +311,10 @@ L(first_vec_x1_check): + .p2align 4 + L(first_vec_x2_check): + tzcntl %eax, %eax +-# ifdef USE_AS_WCSLEN ++# ifdef USE_AS_WCSLEN + /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ + sall $2, %eax +-# endif ++# endif + /* Check the end of data. */ + cmpq %rax, %rsi + jbe L(max) +@@ -329,10 +329,10 @@ L(first_vec_x2_check): + .p2align 4 + L(first_vec_x3_check): + tzcntl %eax, %eax +-# ifdef USE_AS_WCSLEN ++# ifdef USE_AS_WCSLEN + /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ + sall $2, %eax +-# endif ++# endif + /* Check the end of data. */ + cmpq %rax, %rsi + jbe L(max) +-- +GitLab + diff --git a/SOURCES/ia-imp-strcmp-evex.patch b/SOURCES/ia-imp-strcmp-evex.patch new file mode 100644 index 0000000..52229e9 --- /dev/null +++ b/SOURCES/ia-imp-strcmp-evex.patch @@ -0,0 +1,695 @@ +From 4c0828b81a6d4fd6ca49a4fcef45bb6b479d0d67 Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Fri, 29 Oct 2021 12:40:20 -0700 +Subject: [PATCH] x86-64: Improve EVEX strcmp with masked load + +In strcmp-evex.S, to compare 2 32-byte strings, replace + + VMOVU (%rdi, %rdx), %YMM0 + VMOVU (%rsi, %rdx), %YMM1 + /* Each bit in K0 represents a mismatch in YMM0 and YMM1. */ + VPCMP $4, %YMM0, %YMM1, %k0 + VPCMP $0, %YMMZERO, %YMM0, %k1 + VPCMP $0, %YMMZERO, %YMM1, %k2 + /* Each bit in K1 represents a NULL in YMM0 or YMM1. */ + kord %k1, %k2, %k1 + /* Each bit in K1 represents a NULL or a mismatch. */ + kord %k0, %k1, %k1 + kmovd %k1, %ecx + testl %ecx, %ecx + jne L(last_vector) + +with + + VMOVU (%rdi, %rdx), %YMM0 + VPTESTM %YMM0, %YMM0, %k2 + /* Each bit cleared in K1 represents a mismatch or a null CHAR + in YMM0 and 32 bytes at (%rsi, %rdx). */ + VPCMP $0, (%rsi, %rdx), %YMM0, %k1{%k2} + kmovd %k1, %ecx + incl %ecx + jne L(last_vector) + +It makes EVEX strcmp faster than AVX2 strcmp by up to 40% on Tiger Lake +and Ice Lake. + +Co-Authored-By: Noah Goldstein +(cherry picked from commit c46e9afb2df5fc9e39ff4d13777e4b4c26e04e55) +--- + sysdeps/x86_64/multiarch/strcmp-evex.S | 461 +++++++++++++------------ + 1 file changed, 243 insertions(+), 218 deletions(-) + +diff --git a/sysdeps/x86_64/multiarch/strcmp-evex.S b/sysdeps/x86_64/multiarch/strcmp-evex.S +index d5aa6daa..82f12ac8 100644 +--- a/sysdeps/x86_64/multiarch/strcmp-evex.S ++++ b/sysdeps/x86_64/multiarch/strcmp-evex.S +@@ -41,6 +41,8 @@ + # ifdef USE_AS_WCSCMP + /* Compare packed dwords. */ + # define VPCMP vpcmpd ++# define VPMINU vpminud ++# define VPTESTM vptestmd + # define SHIFT_REG32 r8d + # define SHIFT_REG64 r8 + /* 1 dword char == 4 bytes. */ +@@ -48,6 +50,8 @@ + # else + /* Compare packed bytes. */ + # define VPCMP vpcmpb ++# define VPMINU vpminub ++# define VPTESTM vptestmb + # define SHIFT_REG32 ecx + # define SHIFT_REG64 rcx + /* 1 byte char == 1 byte. */ +@@ -67,6 +71,9 @@ + # define YMM5 ymm22 + # define YMM6 ymm23 + # define YMM7 ymm24 ++# define YMM8 ymm25 ++# define YMM9 ymm26 ++# define YMM10 ymm27 + + /* Warning! + wcscmp/wcsncmp have to use SIGNED comparison for elements. +@@ -76,7 +83,7 @@ + /* The main idea of the string comparison (byte or dword) using 256-bit + EVEX instructions consists of comparing (VPCMP) two ymm vectors. The + latter can be on either packed bytes or dwords depending on +- USE_AS_WCSCMP. In order to check the null char, algorithm keeps the ++ USE_AS_WCSCMP. In order to check the null CHAR, algorithm keeps the + matched bytes/dwords, requiring 5 EVEX instructions (3 VPCMP and 2 + KORD). In general, the costs of comparing VEC_SIZE bytes (32-bytes) + are 3 VPCMP and 2 KORD instructions, together with VMOVU and ktestd +@@ -123,27 +130,21 @@ ENTRY (STRCMP) + jg L(cross_page) + /* Start comparing 4 vectors. */ + VMOVU (%rdi), %YMM0 +- VMOVU (%rsi), %YMM1 + +- /* Each bit in K0 represents a mismatch in YMM0 and YMM1. */ +- VPCMP $4, %YMM0, %YMM1, %k0 ++ /* Each bit set in K2 represents a non-null CHAR in YMM0. */ ++ VPTESTM %YMM0, %YMM0, %k2 + +- /* Check for NULL in YMM0. */ +- VPCMP $0, %YMMZERO, %YMM0, %k1 +- /* Check for NULL in YMM1. */ +- VPCMP $0, %YMMZERO, %YMM1, %k2 +- /* Each bit in K1 represents a NULL in YMM0 or YMM1. */ +- kord %k1, %k2, %k1 ++ /* Each bit cleared in K1 represents a mismatch or a null CHAR ++ in YMM0 and 32 bytes at (%rsi). */ ++ VPCMP $0, (%rsi), %YMM0, %k1{%k2} + +- /* Each bit in K1 represents: +- 1. A mismatch in YMM0 and YMM1. Or +- 2. A NULL in YMM0 or YMM1. +- */ +- kord %k0, %k1, %k1 +- +- ktestd %k1, %k1 +- je L(next_3_vectors) + kmovd %k1, %ecx ++# ifdef USE_AS_WCSCMP ++ subl $0xff, %ecx ++# else ++ incl %ecx ++# endif ++ je L(next_3_vectors) + tzcntl %ecx, %edx + # ifdef USE_AS_WCSCMP + /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ +@@ -172,9 +173,7 @@ L(return): + # endif + ret + +- .p2align 4 + L(return_vec_size): +- kmovd %k1, %ecx + tzcntl %ecx, %edx + # ifdef USE_AS_WCSCMP + /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ +@@ -210,9 +209,7 @@ L(return_vec_size): + # endif + ret + +- .p2align 4 + L(return_2_vec_size): +- kmovd %k1, %ecx + tzcntl %ecx, %edx + # ifdef USE_AS_WCSCMP + /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ +@@ -248,9 +245,7 @@ L(return_2_vec_size): + # endif + ret + +- .p2align 4 + L(return_3_vec_size): +- kmovd %k1, %ecx + tzcntl %ecx, %edx + # ifdef USE_AS_WCSCMP + /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ +@@ -289,43 +284,45 @@ L(return_3_vec_size): + .p2align 4 + L(next_3_vectors): + VMOVU VEC_SIZE(%rdi), %YMM0 +- VMOVU VEC_SIZE(%rsi), %YMM1 +- /* Each bit in K0 represents a mismatch in YMM0 and YMM1. */ +- VPCMP $4, %YMM0, %YMM1, %k0 +- VPCMP $0, %YMMZERO, %YMM0, %k1 +- VPCMP $0, %YMMZERO, %YMM1, %k2 +- /* Each bit in K1 represents a NULL in YMM0 or YMM1. */ +- kord %k1, %k2, %k1 +- /* Each bit in K1 represents a NULL or a mismatch. */ +- kord %k0, %k1, %k1 +- ktestd %k1, %k1 ++ /* Each bit set in K2 represents a non-null CHAR in YMM0. */ ++ VPTESTM %YMM0, %YMM0, %k2 ++ /* Each bit cleared in K1 represents a mismatch or a null CHAR ++ in YMM0 and 32 bytes at VEC_SIZE(%rsi). */ ++ VPCMP $0, VEC_SIZE(%rsi), %YMM0, %k1{%k2} ++ kmovd %k1, %ecx ++# ifdef USE_AS_WCSCMP ++ subl $0xff, %ecx ++# else ++ incl %ecx ++# endif + jne L(return_vec_size) + +- VMOVU (VEC_SIZE * 2)(%rdi), %YMM2 +- VMOVU (VEC_SIZE * 3)(%rdi), %YMM3 +- VMOVU (VEC_SIZE * 2)(%rsi), %YMM4 +- VMOVU (VEC_SIZE * 3)(%rsi), %YMM5 +- +- /* Each bit in K0 represents a mismatch in YMM2 and YMM4. */ +- VPCMP $4, %YMM2, %YMM4, %k0 +- VPCMP $0, %YMMZERO, %YMM2, %k1 +- VPCMP $0, %YMMZERO, %YMM4, %k2 +- /* Each bit in K1 represents a NULL in YMM2 or YMM4. */ +- kord %k1, %k2, %k1 +- /* Each bit in K1 represents a NULL or a mismatch. */ +- kord %k0, %k1, %k1 +- ktestd %k1, %k1 ++ VMOVU (VEC_SIZE * 2)(%rdi), %YMM0 ++ /* Each bit set in K2 represents a non-null CHAR in YMM0. */ ++ VPTESTM %YMM0, %YMM0, %k2 ++ /* Each bit cleared in K1 represents a mismatch or a null CHAR ++ in YMM0 and 32 bytes at (VEC_SIZE * 2)(%rsi). */ ++ VPCMP $0, (VEC_SIZE * 2)(%rsi), %YMM0, %k1{%k2} ++ kmovd %k1, %ecx ++# ifdef USE_AS_WCSCMP ++ subl $0xff, %ecx ++# else ++ incl %ecx ++# endif + jne L(return_2_vec_size) + +- /* Each bit in K0 represents a mismatch in YMM3 and YMM5. */ +- VPCMP $4, %YMM3, %YMM5, %k0 +- VPCMP $0, %YMMZERO, %YMM3, %k1 +- VPCMP $0, %YMMZERO, %YMM5, %k2 +- /* Each bit in K1 represents a NULL in YMM3 or YMM5. */ +- kord %k1, %k2, %k1 +- /* Each bit in K1 represents a NULL or a mismatch. */ +- kord %k0, %k1, %k1 +- ktestd %k1, %k1 ++ VMOVU (VEC_SIZE * 3)(%rdi), %YMM0 ++ /* Each bit set in K2 represents a non-null CHAR in YMM0. */ ++ VPTESTM %YMM0, %YMM0, %k2 ++ /* Each bit cleared in K1 represents a mismatch or a null CHAR ++ in YMM0 and 32 bytes at (VEC_SIZE * 2)(%rsi). */ ++ VPCMP $0, (VEC_SIZE * 3)(%rsi), %YMM0, %k1{%k2} ++ kmovd %k1, %ecx ++# ifdef USE_AS_WCSCMP ++ subl $0xff, %ecx ++# else ++ incl %ecx ++# endif + jne L(return_3_vec_size) + L(main_loop_header): + leaq (VEC_SIZE * 4)(%rdi), %rdx +@@ -375,56 +372,51 @@ L(back_to_loop): + VMOVA VEC_SIZE(%rax), %YMM2 + VMOVA (VEC_SIZE * 2)(%rax), %YMM4 + VMOVA (VEC_SIZE * 3)(%rax), %YMM6 +- VMOVU (%rdx), %YMM1 +- VMOVU VEC_SIZE(%rdx), %YMM3 +- VMOVU (VEC_SIZE * 2)(%rdx), %YMM5 +- VMOVU (VEC_SIZE * 3)(%rdx), %YMM7 +- +- VPCMP $4, %YMM0, %YMM1, %k0 +- VPCMP $0, %YMMZERO, %YMM0, %k1 +- VPCMP $0, %YMMZERO, %YMM1, %k2 +- kord %k1, %k2, %k1 +- /* Each bit in K4 represents a NULL or a mismatch in YMM0 and +- YMM1. */ +- kord %k0, %k1, %k4 +- +- VPCMP $4, %YMM2, %YMM3, %k0 +- VPCMP $0, %YMMZERO, %YMM2, %k1 +- VPCMP $0, %YMMZERO, %YMM3, %k2 +- kord %k1, %k2, %k1 +- /* Each bit in K5 represents a NULL or a mismatch in YMM2 and +- YMM3. */ +- kord %k0, %k1, %k5 +- +- VPCMP $4, %YMM4, %YMM5, %k0 +- VPCMP $0, %YMMZERO, %YMM4, %k1 +- VPCMP $0, %YMMZERO, %YMM5, %k2 +- kord %k1, %k2, %k1 +- /* Each bit in K6 represents a NULL or a mismatch in YMM4 and +- YMM5. */ +- kord %k0, %k1, %k6 +- +- VPCMP $4, %YMM6, %YMM7, %k0 +- VPCMP $0, %YMMZERO, %YMM6, %k1 +- VPCMP $0, %YMMZERO, %YMM7, %k2 +- kord %k1, %k2, %k1 +- /* Each bit in K7 represents a NULL or a mismatch in YMM6 and +- YMM7. */ +- kord %k0, %k1, %k7 +- +- kord %k4, %k5, %k0 +- kord %k6, %k7, %k1 +- +- /* Test each mask (32 bits) individually because for VEC_SIZE +- == 32 is not possible to OR the four masks and keep all bits +- in a 64-bit integer register, differing from SSE2 strcmp +- where ORing is possible. */ +- kortestd %k0, %k1 +- je L(loop) +- ktestd %k4, %k4 ++ ++ VPMINU %YMM0, %YMM2, %YMM8 ++ VPMINU %YMM4, %YMM6, %YMM9 ++ ++ /* A zero CHAR in YMM8 means that there is a null CHAR. */ ++ VPMINU %YMM8, %YMM9, %YMM8 ++ ++ /* Each bit set in K1 represents a non-null CHAR in YMM8. */ ++ VPTESTM %YMM8, %YMM8, %k1 ++ ++ /* (YMM ^ YMM): A non-zero CHAR represents a mismatch. */ ++ vpxorq (%rdx), %YMM0, %YMM1 ++ vpxorq VEC_SIZE(%rdx), %YMM2, %YMM3 ++ vpxorq (VEC_SIZE * 2)(%rdx), %YMM4, %YMM5 ++ vpxorq (VEC_SIZE * 3)(%rdx), %YMM6, %YMM7 ++ ++ vporq %YMM1, %YMM3, %YMM9 ++ vporq %YMM5, %YMM7, %YMM10 ++ ++ /* A non-zero CHAR in YMM9 represents a mismatch. */ ++ vporq %YMM9, %YMM10, %YMM9 ++ ++ /* Each bit cleared in K0 represents a mismatch or a null CHAR. */ ++ VPCMP $0, %YMMZERO, %YMM9, %k0{%k1} ++ kmovd %k0, %ecx ++# ifdef USE_AS_WCSCMP ++ subl $0xff, %ecx ++# else ++ incl %ecx ++# endif ++ je L(loop) ++ ++ /* Each bit set in K1 represents a non-null CHAR in YMM0. */ ++ VPTESTM %YMM0, %YMM0, %k1 ++ /* Each bit cleared in K0 represents a mismatch or a null CHAR ++ in YMM0 and (%rdx). */ ++ VPCMP $0, %YMMZERO, %YMM1, %k0{%k1} ++ kmovd %k0, %ecx ++# ifdef USE_AS_WCSCMP ++ subl $0xff, %ecx ++# else ++ incl %ecx ++# endif + je L(test_vec) +- kmovd %k4, %edi +- tzcntl %edi, %ecx ++ tzcntl %ecx, %ecx + # ifdef USE_AS_WCSCMP + /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ + sall $2, %ecx +@@ -466,9 +458,18 @@ L(test_vec): + cmpq $VEC_SIZE, %r11 + jbe L(zero) + # endif +- ktestd %k5, %k5 ++ /* Each bit set in K1 represents a non-null CHAR in YMM2. */ ++ VPTESTM %YMM2, %YMM2, %k1 ++ /* Each bit cleared in K0 represents a mismatch or a null CHAR ++ in YMM2 and VEC_SIZE(%rdx). */ ++ VPCMP $0, %YMMZERO, %YMM3, %k0{%k1} ++ kmovd %k0, %ecx ++# ifdef USE_AS_WCSCMP ++ subl $0xff, %ecx ++# else ++ incl %ecx ++# endif + je L(test_2_vec) +- kmovd %k5, %ecx + tzcntl %ecx, %edi + # ifdef USE_AS_WCSCMP + /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ +@@ -512,9 +513,18 @@ L(test_2_vec): + cmpq $(VEC_SIZE * 2), %r11 + jbe L(zero) + # endif +- ktestd %k6, %k6 ++ /* Each bit set in K1 represents a non-null CHAR in YMM4. */ ++ VPTESTM %YMM4, %YMM4, %k1 ++ /* Each bit cleared in K0 represents a mismatch or a null CHAR ++ in YMM4 and (VEC_SIZE * 2)(%rdx). */ ++ VPCMP $0, %YMMZERO, %YMM5, %k0{%k1} ++ kmovd %k0, %ecx ++# ifdef USE_AS_WCSCMP ++ subl $0xff, %ecx ++# else ++ incl %ecx ++# endif + je L(test_3_vec) +- kmovd %k6, %ecx + tzcntl %ecx, %edi + # ifdef USE_AS_WCSCMP + /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ +@@ -558,8 +568,18 @@ L(test_3_vec): + cmpq $(VEC_SIZE * 3), %r11 + jbe L(zero) + # endif +- kmovd %k7, %esi +- tzcntl %esi, %ecx ++ /* Each bit set in K1 represents a non-null CHAR in YMM6. */ ++ VPTESTM %YMM6, %YMM6, %k1 ++ /* Each bit cleared in K0 represents a mismatch or a null CHAR ++ in YMM6 and (VEC_SIZE * 3)(%rdx). */ ++ VPCMP $0, %YMMZERO, %YMM7, %k0{%k1} ++ kmovd %k0, %ecx ++# ifdef USE_AS_WCSCMP ++ subl $0xff, %ecx ++# else ++ incl %ecx ++# endif ++ tzcntl %ecx, %ecx + # ifdef USE_AS_WCSCMP + /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ + sall $2, %ecx +@@ -615,39 +635,51 @@ L(loop_cross_page): + + VMOVU (%rax, %r10), %YMM2 + VMOVU VEC_SIZE(%rax, %r10), %YMM3 +- VMOVU (%rdx, %r10), %YMM4 +- VMOVU VEC_SIZE(%rdx, %r10), %YMM5 +- +- VPCMP $4, %YMM4, %YMM2, %k0 +- VPCMP $0, %YMMZERO, %YMM2, %k1 +- VPCMP $0, %YMMZERO, %YMM4, %k2 +- kord %k1, %k2, %k1 +- /* Each bit in K1 represents a NULL or a mismatch in YMM2 and +- YMM4. */ +- kord %k0, %k1, %k1 +- +- VPCMP $4, %YMM5, %YMM3, %k3 +- VPCMP $0, %YMMZERO, %YMM3, %k4 +- VPCMP $0, %YMMZERO, %YMM5, %k5 +- kord %k4, %k5, %k4 +- /* Each bit in K3 represents a NULL or a mismatch in YMM3 and +- YMM5. */ +- kord %k3, %k4, %k3 ++ ++ /* Each bit set in K2 represents a non-null CHAR in YMM2. */ ++ VPTESTM %YMM2, %YMM2, %k2 ++ /* Each bit cleared in K1 represents a mismatch or a null CHAR ++ in YMM2 and 32 bytes at (%rdx, %r10). */ ++ VPCMP $0, (%rdx, %r10), %YMM2, %k1{%k2} ++ kmovd %k1, %r9d ++ /* Don't use subl since it is the lower 16/32 bits of RDI ++ below. */ ++ notl %r9d ++# ifdef USE_AS_WCSCMP ++ /* Only last 8 bits are valid. */ ++ andl $0xff, %r9d ++# endif ++ ++ /* Each bit set in K4 represents a non-null CHAR in YMM3. */ ++ VPTESTM %YMM3, %YMM3, %k4 ++ /* Each bit cleared in K3 represents a mismatch or a null CHAR ++ in YMM3 and 32 bytes at VEC_SIZE(%rdx, %r10). */ ++ VPCMP $0, VEC_SIZE(%rdx, %r10), %YMM3, %k3{%k4} ++ kmovd %k3, %edi ++# ifdef USE_AS_WCSCMP ++ /* Don't use subl since it is the upper 8 bits of EDI below. */ ++ notl %edi ++ andl $0xff, %edi ++# else ++ incl %edi ++# endif + + # ifdef USE_AS_WCSCMP +- /* NB: Each bit in K1/K3 represents 4-byte element. */ +- kshiftlw $8, %k3, %k2 ++ /* NB: Each bit in EDI/R9D represents 4-byte element. */ ++ sall $8, %edi + /* NB: Divide shift count by 4 since each bit in K1 represent 4 + bytes. */ + movl %ecx, %SHIFT_REG32 + sarl $2, %SHIFT_REG32 ++ ++ /* Each bit in EDI represents a null CHAR or a mismatch. */ ++ orl %r9d, %edi + # else +- kshiftlq $32, %k3, %k2 +-# endif ++ salq $32, %rdi + +- /* Each bit in K1 represents a NULL or a mismatch. */ +- korq %k1, %k2, %k1 +- kmovq %k1, %rdi ++ /* Each bit in RDI represents a null CHAR or a mismatch. */ ++ orq %r9, %rdi ++# endif + + /* Since ECX < VEC_SIZE * 2, simply skip the first ECX bytes. */ + shrxq %SHIFT_REG64, %rdi, %rdi +@@ -692,35 +724,45 @@ L(loop_cross_page_2_vec): + /* The first VEC_SIZE * 2 bytes match or are ignored. */ + VMOVU (VEC_SIZE * 2)(%rax, %r10), %YMM0 + VMOVU (VEC_SIZE * 3)(%rax, %r10), %YMM1 +- VMOVU (VEC_SIZE * 2)(%rdx, %r10), %YMM2 +- VMOVU (VEC_SIZE * 3)(%rdx, %r10), %YMM3 +- +- VPCMP $4, %YMM0, %YMM2, %k0 +- VPCMP $0, %YMMZERO, %YMM0, %k1 +- VPCMP $0, %YMMZERO, %YMM2, %k2 +- kord %k1, %k2, %k1 +- /* Each bit in K1 represents a NULL or a mismatch in YMM0 and +- YMM2. */ +- kord %k0, %k1, %k1 +- +- VPCMP $4, %YMM1, %YMM3, %k3 +- VPCMP $0, %YMMZERO, %YMM1, %k4 +- VPCMP $0, %YMMZERO, %YMM3, %k5 +- kord %k4, %k5, %k4 +- /* Each bit in K3 represents a NULL or a mismatch in YMM1 and +- YMM3. */ +- kord %k3, %k4, %k3 + ++ VPTESTM %YMM0, %YMM0, %k2 ++ /* Each bit cleared in K1 represents a mismatch or a null CHAR ++ in YMM0 and 32 bytes at (VEC_SIZE * 2)(%rdx, %r10). */ ++ VPCMP $0, (VEC_SIZE * 2)(%rdx, %r10), %YMM0, %k1{%k2} ++ kmovd %k1, %r9d ++ /* Don't use subl since it is the lower 16/32 bits of RDI ++ below. */ ++ notl %r9d + # ifdef USE_AS_WCSCMP +- /* NB: Each bit in K1/K3 represents 4-byte element. */ +- kshiftlw $8, %k3, %k2 ++ /* Only last 8 bits are valid. */ ++ andl $0xff, %r9d ++# endif ++ ++ VPTESTM %YMM1, %YMM1, %k4 ++ /* Each bit cleared in K3 represents a mismatch or a null CHAR ++ in YMM1 and 32 bytes at (VEC_SIZE * 3)(%rdx, %r10). */ ++ VPCMP $0, (VEC_SIZE * 3)(%rdx, %r10), %YMM1, %k3{%k4} ++ kmovd %k3, %edi ++# ifdef USE_AS_WCSCMP ++ /* Don't use subl since it is the upper 8 bits of EDI below. */ ++ notl %edi ++ andl $0xff, %edi + # else +- kshiftlq $32, %k3, %k2 ++ incl %edi + # endif + +- /* Each bit in K1 represents a NULL or a mismatch. */ +- korq %k1, %k2, %k1 +- kmovq %k1, %rdi ++# ifdef USE_AS_WCSCMP ++ /* NB: Each bit in EDI/R9D represents 4-byte element. */ ++ sall $8, %edi ++ ++ /* Each bit in EDI represents a null CHAR or a mismatch. */ ++ orl %r9d, %edi ++# else ++ salq $32, %rdi ++ ++ /* Each bit in RDI represents a null CHAR or a mismatch. */ ++ orq %r9, %rdi ++# endif + + xorl %r8d, %r8d + /* If ECX > VEC_SIZE * 2, skip ECX - (VEC_SIZE * 2) bytes. */ +@@ -729,12 +771,15 @@ L(loop_cross_page_2_vec): + /* R8 has number of bytes skipped. */ + movl %ecx, %r8d + # ifdef USE_AS_WCSCMP +- /* NB: Divide shift count by 4 since each bit in K1 represent 4 ++ /* NB: Divide shift count by 4 since each bit in RDI represent 4 + bytes. */ + sarl $2, %ecx +-# endif ++ /* Skip ECX bytes. */ ++ shrl %cl, %edi ++# else + /* Skip ECX bytes. */ + shrq %cl, %rdi ++# endif + 1: + /* Before jumping back to the loop, set ESI to the number of + VEC_SIZE * 4 blocks before page crossing. */ +@@ -818,7 +863,7 @@ L(cross_page_loop): + movzbl (%rdi, %rdx), %eax + movzbl (%rsi, %rdx), %ecx + # endif +- /* Check null char. */ ++ /* Check null CHAR. */ + testl %eax, %eax + jne L(cross_page_loop) + /* Since %eax == 0, subtract is OK for both SIGNED and UNSIGNED +@@ -901,18 +946,17 @@ L(cross_page): + jg L(cross_page_1_vector) + L(loop_1_vector): + VMOVU (%rdi, %rdx), %YMM0 +- VMOVU (%rsi, %rdx), %YMM1 +- +- /* Each bit in K0 represents a mismatch in YMM0 and YMM1. */ +- VPCMP $4, %YMM0, %YMM1, %k0 +- VPCMP $0, %YMMZERO, %YMM0, %k1 +- VPCMP $0, %YMMZERO, %YMM1, %k2 +- /* Each bit in K1 represents a NULL in YMM0 or YMM1. */ +- kord %k1, %k2, %k1 +- /* Each bit in K1 represents a NULL or a mismatch. */ +- kord %k0, %k1, %k1 ++ ++ VPTESTM %YMM0, %YMM0, %k2 ++ /* Each bit cleared in K1 represents a mismatch or a null CHAR ++ in YMM0 and 32 bytes at (%rsi, %rdx). */ ++ VPCMP $0, (%rsi, %rdx), %YMM0, %k1{%k2} + kmovd %k1, %ecx +- testl %ecx, %ecx ++# ifdef USE_AS_WCSCMP ++ subl $0xff, %ecx ++# else ++ incl %ecx ++# endif + jne L(last_vector) + + addl $VEC_SIZE, %edx +@@ -931,18 +975,17 @@ L(cross_page_1_vector): + cmpl $(PAGE_SIZE - 16), %eax + jg L(cross_page_1_xmm) + VMOVU (%rdi, %rdx), %XMM0 +- VMOVU (%rsi, %rdx), %XMM1 +- +- /* Each bit in K0 represents a mismatch in XMM0 and XMM1. */ +- VPCMP $4, %XMM0, %XMM1, %k0 +- VPCMP $0, %XMMZERO, %XMM0, %k1 +- VPCMP $0, %XMMZERO, %XMM1, %k2 +- /* Each bit in K1 represents a NULL in XMM0 or XMM1. */ +- korw %k1, %k2, %k1 +- /* Each bit in K1 represents a NULL or a mismatch. */ +- korw %k0, %k1, %k1 +- kmovw %k1, %ecx +- testl %ecx, %ecx ++ ++ VPTESTM %YMM0, %YMM0, %k2 ++ /* Each bit cleared in K1 represents a mismatch or a null CHAR ++ in XMM0 and 16 bytes at (%rsi, %rdx). */ ++ VPCMP $0, (%rsi, %rdx), %XMM0, %k1{%k2} ++ kmovd %k1, %ecx ++# ifdef USE_AS_WCSCMP ++ subl $0xf, %ecx ++# else ++ subl $0xffff, %ecx ++# endif + jne L(last_vector) + + addl $16, %edx +@@ -965,25 +1008,16 @@ L(cross_page_1_xmm): + vmovq (%rdi, %rdx), %XMM0 + vmovq (%rsi, %rdx), %XMM1 + +- /* Each bit in K0 represents a mismatch in XMM0 and XMM1. */ +- VPCMP $4, %XMM0, %XMM1, %k0 +- VPCMP $0, %XMMZERO, %XMM0, %k1 +- VPCMP $0, %XMMZERO, %XMM1, %k2 +- /* Each bit in K1 represents a NULL in XMM0 or XMM1. */ +- kord %k1, %k2, %k1 +- /* Each bit in K1 represents a NULL or a mismatch. */ +- kord %k0, %k1, %k1 +- kmovd %k1, %ecx +- ++ VPTESTM %YMM0, %YMM0, %k2 ++ /* Each bit cleared in K1 represents a mismatch or a null CHAR ++ in XMM0 and XMM1. */ ++ VPCMP $0, %XMM1, %XMM0, %k1{%k2} ++ kmovb %k1, %ecx + # ifdef USE_AS_WCSCMP +- /* Only last 2 bits are valid. */ +- andl $0x3, %ecx ++ subl $0x3, %ecx + # else +- /* Only last 8 bits are valid. */ +- andl $0xff, %ecx ++ subl $0xff, %ecx + # endif +- +- testl %ecx, %ecx + jne L(last_vector) + + addl $8, %edx +@@ -1002,25 +1036,16 @@ L(cross_page_8bytes): + vmovd (%rdi, %rdx), %XMM0 + vmovd (%rsi, %rdx), %XMM1 + +- /* Each bit in K0 represents a mismatch in XMM0 and XMM1. */ +- VPCMP $4, %XMM0, %XMM1, %k0 +- VPCMP $0, %XMMZERO, %XMM0, %k1 +- VPCMP $0, %XMMZERO, %XMM1, %k2 +- /* Each bit in K1 represents a NULL in XMM0 or XMM1. */ +- kord %k1, %k2, %k1 +- /* Each bit in K1 represents a NULL or a mismatch. */ +- kord %k0, %k1, %k1 ++ VPTESTM %YMM0, %YMM0, %k2 ++ /* Each bit cleared in K1 represents a mismatch or a null CHAR ++ in XMM0 and XMM1. */ ++ VPCMP $0, %XMM1, %XMM0, %k1{%k2} + kmovd %k1, %ecx +- + # ifdef USE_AS_WCSCMP +- /* Only the last bit is valid. */ +- andl $0x1, %ecx ++ subl $0x1, %ecx + # else +- /* Only last 4 bits are valid. */ +- andl $0xf, %ecx ++ subl $0xf, %ecx + # endif +- +- testl %ecx, %ecx + jne L(last_vector) + + addl $4, %edx +-- +GitLab + diff --git a/SOURCES/ia-impr-memmove-vec-unaligned-erms.patch b/SOURCES/ia-impr-memmove-vec-unaligned-erms.patch new file mode 100644 index 0000000..15fd4cd --- /dev/null +++ b/SOURCES/ia-impr-memmove-vec-unaligned-erms.patch @@ -0,0 +1,84 @@ +From 29dfcb178f889d1bd91e9eb9d98d7e74fddcde51 Mon Sep 17 00:00:00 2001 +From: Noah Goldstein +Date: Sun, 23 May 2021 19:43:24 -0400 +Subject: [PATCH] x86: Improve memmove-vec-unaligned-erms.S + +This patch changes the condition for copy 4x VEC so that if length is +exactly equal to 4 * VEC_SIZE it will use the 4x VEC case instead of +8x VEC case. + +Results For Skylake memcpy-avx2-erms +size, al1 , al2 , Cur T , New T , Win , New / Cur +128 , 0 , 0 , 9.137 , 6.873 , New , 75.22 +128 , 7 , 0 , 12.933 , 7.732 , New , 59.79 +128 , 0 , 7 , 11.852 , 6.76 , New , 57.04 +128 , 7 , 7 , 12.587 , 6.808 , New , 54.09 + +Results For Icelake memcpy-evex-erms +size, al1 , al2 , Cur T , New T , Win , New / Cur +128 , 0 , 0 , 9.963 , 5.416 , New , 54.36 +128 , 7 , 0 , 16.467 , 8.061 , New , 48.95 +128 , 0 , 7 , 14.388 , 7.644 , New , 53.13 +128 , 7 , 7 , 14.546 , 7.642 , New , 52.54 + +Results For Tigerlake memcpy-evex-erms +size, al1 , al2 , Cur T , New T , Win , New / Cur +128 , 0 , 0 , 8.979 , 4.95 , New , 55.13 +128 , 7 , 0 , 14.245 , 7.122 , New , 50.0 +128 , 0 , 7 , 12.668 , 6.675 , New , 52.69 +128 , 7 , 7 , 13.042 , 6.802 , New , 52.15 + +Results For Skylake memmove-avx2-erms +size, al1 , al2 , Cur T , New T , Win , New / Cur +128 , 0 , 32 , 6.181 , 5.691 , New , 92.07 +128 , 32 , 0 , 6.165 , 5.752 , New , 93.3 +128 , 0 , 7 , 13.923 , 9.37 , New , 67.3 +128 , 7 , 0 , 12.049 , 10.182 , New , 84.5 + +Results For Icelake memmove-evex-erms +size, al1 , al2 , Cur T , New T , Win , New / Cur +128 , 0 , 32 , 5.479 , 4.889 , New , 89.23 +128 , 32 , 0 , 5.127 , 4.911 , New , 95.79 +128 , 0 , 7 , 18.885 , 13.547 , New , 71.73 +128 , 7 , 0 , 15.565 , 14.436 , New , 92.75 + +Results For Tigerlake memmove-evex-erms +size, al1 , al2 , Cur T , New T , Win , New / Cur +128 , 0 , 32 , 5.275 , 4.815 , New , 91.28 +128 , 32 , 0 , 5.376 , 4.565 , New , 84.91 +128 , 0 , 7 , 19.426 , 14.273 , New , 73.47 +128 , 7 , 0 , 15.924 , 14.951 , New , 93.89 + +Signed-off-by: Noah Goldstein +(cherry picked from commit 1b992204f68af851e905c16016756fd4421e1934) +--- + sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S +index 3e2dd6bc..572cef04 100644 +--- a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S ++++ b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S +@@ -417,8 +417,8 @@ L(more_2x_vec): + cmpq $(VEC_SIZE * 8), %rdx + ja L(more_8x_vec) + cmpq $(VEC_SIZE * 4), %rdx +- jb L(last_4x_vec) +- /* Copy from 4 * VEC to 8 * VEC, inclusively. */ ++ jbe L(last_4x_vec) ++ /* Copy from 4 * VEC + 1 to 8 * VEC, inclusively. */ + VMOVU (%rsi), %VEC(0) + VMOVU VEC_SIZE(%rsi), %VEC(1) + VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2) +@@ -437,7 +437,7 @@ L(more_2x_vec): + VMOVU %VEC(7), -(VEC_SIZE * 4)(%rdi,%rdx) + VZEROUPPER_RETURN + L(last_4x_vec): +- /* Copy from 2 * VEC to 4 * VEC. */ ++ /* Copy from 2 * VEC + 1 to 4 * VEC, inclusively. */ + VMOVU (%rsi), %VEC(0) + VMOVU VEC_SIZE(%rsi), %VEC(1) + VMOVU -VEC_SIZE(%rsi,%rdx), %VEC(2) +-- +GitLab + diff --git a/SOURCES/ia-impr-memset-vec-unaligned-erms.patch b/SOURCES/ia-impr-memset-vec-unaligned-erms.patch new file mode 100644 index 0000000..2256ac6 --- /dev/null +++ b/SOURCES/ia-impr-memset-vec-unaligned-erms.patch @@ -0,0 +1,104 @@ +From 91272636c23028e55554be4e677bf40ac22b1adc Mon Sep 17 00:00:00 2001 +From: Noah Goldstein +Date: Thu, 20 May 2021 13:13:51 -0400 +Subject: [PATCH] x86: Improve memset-vec-unaligned-erms.S + +No bug. This commit makes a few small improvements to +memset-vec-unaligned-erms.S. The changes are 1) only aligning to 64 +instead of 128. Either alignment will perform equally well in a loop +and 128 just increases the odds of having to do an extra iteration +which can be significant overhead for small values. 2) Align some +targets and the loop. 3) Remove an ALU from the alignment process. 4) +Reorder the last 4x VEC so that they are stored after the loop. 5) +Move the condition for leq 8x VEC to before the alignment +process. test-memset and test-wmemset are both passing. + +Signed-off-by: Noah Goldstein +Reviewed-by: H.J. Lu +(cherry picked from commit 6abf27980a947f9b6e514d6b33b83059d39566ae) +--- + .../multiarch/memset-vec-unaligned-erms.S | 50 +++++++++++-------- + 1 file changed, 28 insertions(+), 22 deletions(-) + +diff --git a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S +index f877ac9d..909c33f6 100644 +--- a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S ++++ b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S +@@ -173,17 +173,22 @@ ENTRY (MEMSET_SYMBOL (__memset, unaligned_erms)) + VMOVU %VEC(0), (%rdi) + VZEROUPPER_RETURN + ++ .p2align 4 + L(stosb_more_2x_vec): + cmp __x86_rep_stosb_threshold(%rip), %RDX_LP + ja L(stosb) ++#else ++ .p2align 4 + #endif + L(more_2x_vec): +- cmpq $(VEC_SIZE * 4), %rdx +- ja L(loop_start) ++ /* Stores to first 2x VEC before cmp as any path forward will ++ require it. */ + VMOVU %VEC(0), (%rdi) + VMOVU %VEC(0), VEC_SIZE(%rdi) +- VMOVU %VEC(0), -VEC_SIZE(%rdi,%rdx) ++ cmpq $(VEC_SIZE * 4), %rdx ++ ja L(loop_start) + VMOVU %VEC(0), -(VEC_SIZE * 2)(%rdi,%rdx) ++ VMOVU %VEC(0), -VEC_SIZE(%rdi,%rdx) + L(return): + #if VEC_SIZE > 16 + ZERO_UPPER_VEC_REGISTERS_RETURN +@@ -192,28 +197,29 @@ L(return): + #endif + + L(loop_start): +- leaq (VEC_SIZE * 4)(%rdi), %rcx +- VMOVU %VEC(0), (%rdi) +- andq $-(VEC_SIZE * 4), %rcx +- VMOVU %VEC(0), -VEC_SIZE(%rdi,%rdx) +- VMOVU %VEC(0), VEC_SIZE(%rdi) +- VMOVU %VEC(0), -(VEC_SIZE * 2)(%rdi,%rdx) + VMOVU %VEC(0), (VEC_SIZE * 2)(%rdi) +- VMOVU %VEC(0), -(VEC_SIZE * 3)(%rdi,%rdx) + VMOVU %VEC(0), (VEC_SIZE * 3)(%rdi) +- VMOVU %VEC(0), -(VEC_SIZE * 4)(%rdi,%rdx) +- addq %rdi, %rdx +- andq $-(VEC_SIZE * 4), %rdx +- cmpq %rdx, %rcx +- je L(return) ++ cmpq $(VEC_SIZE * 8), %rdx ++ jbe L(loop_end) ++ andq $-(VEC_SIZE * 2), %rdi ++ subq $-(VEC_SIZE * 4), %rdi ++ leaq -(VEC_SIZE * 4)(%rax, %rdx), %rcx ++ .p2align 4 + L(loop): +- VMOVA %VEC(0), (%rcx) +- VMOVA %VEC(0), VEC_SIZE(%rcx) +- VMOVA %VEC(0), (VEC_SIZE * 2)(%rcx) +- VMOVA %VEC(0), (VEC_SIZE * 3)(%rcx) +- addq $(VEC_SIZE * 4), %rcx +- cmpq %rcx, %rdx +- jne L(loop) ++ VMOVA %VEC(0), (%rdi) ++ VMOVA %VEC(0), VEC_SIZE(%rdi) ++ VMOVA %VEC(0), (VEC_SIZE * 2)(%rdi) ++ VMOVA %VEC(0), (VEC_SIZE * 3)(%rdi) ++ subq $-(VEC_SIZE * 4), %rdi ++ cmpq %rcx, %rdi ++ jb L(loop) ++L(loop_end): ++ /* NB: rax is set as ptr in MEMSET_VDUP_TO_VEC0_AND_SET_RETURN. ++ rdx as length is also unchanged. */ ++ VMOVU %VEC(0), -(VEC_SIZE * 4)(%rax, %rdx) ++ VMOVU %VEC(0), -(VEC_SIZE * 3)(%rax, %rdx) ++ VMOVU %VEC(0), -(VEC_SIZE * 2)(%rax, %rdx) ++ VMOVU %VEC(0), -VEC_SIZE(%rax, %rdx) + VZEROUPPER_SHORT_RETURN + + .p2align 4 +-- +GitLab + diff --git a/SOURCES/ia-memchr-opt-avx2.patch b/SOURCES/ia-memchr-opt-avx2.patch new file mode 100644 index 0000000..9dc6487 --- /dev/null +++ b/SOURCES/ia-memchr-opt-avx2.patch @@ -0,0 +1,584 @@ +From acd575144cc6340edfbf0a0e0580e38344ab623a Mon Sep 17 00:00:00 2001 +From: Noah Goldstein +Date: Mon, 3 May 2021 03:01:58 -0400 +Subject: [PATCH] x86: Optimize memchr-avx2.S + +No bug. This commit optimizes memchr-avx2.S. The optimizations include +replacing some branches with cmovcc, avoiding some branches entirely +in the less_4x_vec case, making the page cross logic less strict, +asaving a few instructions the in loop return loop. test-memchr, +test-rawmemchr, and test-wmemchr are all passing. + +Signed-off-by: Noah Goldstein +Reviewed-by: H.J. Lu +(cherry picked from commit acfd088a1963ba51cd83c78f95c0ab25ead79e04) +--- + sysdeps/x86_64/multiarch/memchr-avx2.S | 425 ++++++++++++++----------- + 1 file changed, 247 insertions(+), 178 deletions(-) + +diff --git a/sysdeps/x86_64/multiarch/memchr-avx2.S b/sysdeps/x86_64/multiarch/memchr-avx2.S +index cf893e77..b377f22e 100644 +--- a/sysdeps/x86_64/multiarch/memchr-avx2.S ++++ b/sysdeps/x86_64/multiarch/memchr-avx2.S +@@ -26,8 +26,22 @@ + + # ifdef USE_AS_WMEMCHR + # define VPCMPEQ vpcmpeqd ++# define VPBROADCAST vpbroadcastd ++# define CHAR_SIZE 4 + # else + # define VPCMPEQ vpcmpeqb ++# define VPBROADCAST vpbroadcastb ++# define CHAR_SIZE 1 ++# endif ++ ++# ifdef USE_AS_RAWMEMCHR ++# define ERAW_PTR_REG ecx ++# define RRAW_PTR_REG rcx ++# define ALGN_PTR_REG rdi ++# else ++# define ERAW_PTR_REG edi ++# define RRAW_PTR_REG rdi ++# define ALGN_PTR_REG rcx + # endif + + # ifndef VZEROUPPER +@@ -39,6 +53,7 @@ + # endif + + # define VEC_SIZE 32 ++# define PAGE_SIZE 4096 + + .section SECTION(.text),"ax",@progbits + ENTRY (MEMCHR) +@@ -47,295 +62,349 @@ ENTRY (MEMCHR) + test %RDX_LP, %RDX_LP + jz L(null) + # endif +- movl %edi, %ecx +- /* Broadcast CHAR to YMM0. */ +- vmovd %esi, %xmm0 + # ifdef USE_AS_WMEMCHR + shl $2, %RDX_LP +- vpbroadcastd %xmm0, %ymm0 + # else + # ifdef __ILP32__ + /* Clear the upper 32 bits. */ + movl %edx, %edx + # endif +- vpbroadcastb %xmm0, %ymm0 + # endif ++ /* Broadcast CHAR to YMMMATCH. */ ++ vmovd %esi, %xmm0 ++ VPBROADCAST %xmm0, %ymm0 + /* Check if we may cross page boundary with one vector load. */ +- andl $(2 * VEC_SIZE - 1), %ecx +- cmpl $VEC_SIZE, %ecx +- ja L(cros_page_boundary) ++ movl %edi, %eax ++ andl $(PAGE_SIZE - 1), %eax ++ cmpl $(PAGE_SIZE - VEC_SIZE), %eax ++ ja L(cross_page_boundary) + + /* Check the first VEC_SIZE bytes. */ +- VPCMPEQ (%rdi), %ymm0, %ymm1 ++ VPCMPEQ (%rdi), %ymm0, %ymm1 + vpmovmskb %ymm1, %eax +- testl %eax, %eax +- + # ifndef USE_AS_RAWMEMCHR +- jnz L(first_vec_x0_check) +- /* Adjust length and check the end of data. */ +- subq $VEC_SIZE, %rdx +- jbe L(zero) +-# else +- jnz L(first_vec_x0) ++ /* If length < CHAR_PER_VEC handle special. */ ++ cmpq $VEC_SIZE, %rdx ++ jbe L(first_vec_x0) + # endif +- +- /* Align data for aligned loads in the loop. */ +- addq $VEC_SIZE, %rdi +- andl $(VEC_SIZE - 1), %ecx +- andq $-VEC_SIZE, %rdi ++ testl %eax, %eax ++ jz L(aligned_more) ++ tzcntl %eax, %eax ++ addq %rdi, %rax ++ VZEROUPPER_RETURN + + # ifndef USE_AS_RAWMEMCHR +- /* Adjust length. */ +- addq %rcx, %rdx ++ .p2align 5 ++L(first_vec_x0): ++ /* Check if first match was before length. */ ++ tzcntl %eax, %eax ++ xorl %ecx, %ecx ++ cmpl %eax, %edx ++ leaq (%rdi, %rax), %rax ++ cmovle %rcx, %rax ++ VZEROUPPER_RETURN + +- subq $(VEC_SIZE * 4), %rdx +- jbe L(last_4x_vec_or_less) ++L(null): ++ xorl %eax, %eax ++ ret + # endif +- jmp L(more_4x_vec) +- + .p2align 4 +-L(cros_page_boundary): +- andl $(VEC_SIZE - 1), %ecx +- andq $-VEC_SIZE, %rdi +- VPCMPEQ (%rdi), %ymm0, %ymm1 ++L(cross_page_boundary): ++ /* Save pointer before aligning as its original value is necessary ++ for computer return address if byte is found or adjusting length ++ if it is not and this is memchr. */ ++ movq %rdi, %rcx ++ /* Align data to VEC_SIZE - 1. ALGN_PTR_REG is rcx for memchr and ++ rdi for rawmemchr. */ ++ orq $(VEC_SIZE - 1), %ALGN_PTR_REG ++ VPCMPEQ -(VEC_SIZE - 1)(%ALGN_PTR_REG), %ymm0, %ymm1 + vpmovmskb %ymm1, %eax ++# ifndef USE_AS_RAWMEMCHR ++ /* Calculate length until end of page (length checked for a ++ match). */ ++ leaq 1(%ALGN_PTR_REG), %rsi ++ subq %RRAW_PTR_REG, %rsi ++# endif + /* Remove the leading bytes. */ +- sarl %cl, %eax +- testl %eax, %eax +- jz L(aligned_more) +- tzcntl %eax, %eax ++ sarxl %ERAW_PTR_REG, %eax, %eax + # ifndef USE_AS_RAWMEMCHR + /* Check the end of data. */ +- cmpq %rax, %rdx +- jbe L(zero) ++ cmpq %rsi, %rdx ++ jbe L(first_vec_x0) + # endif +- addq %rdi, %rax +- addq %rcx, %rax ++ testl %eax, %eax ++ jz L(cross_page_continue) ++ tzcntl %eax, %eax ++ addq %RRAW_PTR_REG, %rax + L(return_vzeroupper): + ZERO_UPPER_VEC_REGISTERS_RETURN + + .p2align 4 +-L(aligned_more): +-# ifndef USE_AS_RAWMEMCHR +- /* Calculate "rdx + rcx - VEC_SIZE" with "rdx - (VEC_SIZE - rcx)" +- instead of "(rdx + rcx) - VEC_SIZE" to void possible addition +- overflow. */ +- negq %rcx +- addq $VEC_SIZE, %rcx ++L(first_vec_x1): ++ tzcntl %eax, %eax ++ incq %rdi ++ addq %rdi, %rax ++ VZEROUPPER_RETURN + +- /* Check the end of data. */ +- subq %rcx, %rdx +- jbe L(zero) +-# endif ++ .p2align 4 ++L(first_vec_x2): ++ tzcntl %eax, %eax ++ addq $(VEC_SIZE + 1), %rdi ++ addq %rdi, %rax ++ VZEROUPPER_RETURN ++ ++ .p2align 4 ++L(first_vec_x3): ++ tzcntl %eax, %eax ++ addq $(VEC_SIZE * 2 + 1), %rdi ++ addq %rdi, %rax ++ VZEROUPPER_RETURN + +- addq $VEC_SIZE, %rdi + +-# ifndef USE_AS_RAWMEMCHR +- subq $(VEC_SIZE * 4), %rdx +- jbe L(last_4x_vec_or_less) +-# endif ++ .p2align 4 ++L(first_vec_x4): ++ tzcntl %eax, %eax ++ addq $(VEC_SIZE * 3 + 1), %rdi ++ addq %rdi, %rax ++ VZEROUPPER_RETURN + +-L(more_4x_vec): ++ .p2align 4 ++L(aligned_more): + /* Check the first 4 * VEC_SIZE. Only one VEC_SIZE at a time + since data is only aligned to VEC_SIZE. */ +- VPCMPEQ (%rdi), %ymm0, %ymm1 +- vpmovmskb %ymm1, %eax +- testl %eax, %eax +- jnz L(first_vec_x0) + +- VPCMPEQ VEC_SIZE(%rdi), %ymm0, %ymm1 ++# ifndef USE_AS_RAWMEMCHR ++L(cross_page_continue): ++ /* Align data to VEC_SIZE - 1. */ ++ xorl %ecx, %ecx ++ subl %edi, %ecx ++ orq $(VEC_SIZE - 1), %rdi ++ /* esi is for adjusting length to see if near the end. */ ++ leal (VEC_SIZE * 4 + 1)(%rdi, %rcx), %esi ++# else ++ orq $(VEC_SIZE - 1), %rdi ++L(cross_page_continue): ++# endif ++ /* Load first VEC regardless. */ ++ VPCMPEQ 1(%rdi), %ymm0, %ymm1 + vpmovmskb %ymm1, %eax ++# ifndef USE_AS_RAWMEMCHR ++ /* Adjust length. If near end handle specially. */ ++ subq %rsi, %rdx ++ jbe L(last_4x_vec_or_less) ++# endif + testl %eax, %eax + jnz L(first_vec_x1) + +- VPCMPEQ (VEC_SIZE * 2)(%rdi), %ymm0, %ymm1 ++ VPCMPEQ (VEC_SIZE + 1)(%rdi), %ymm0, %ymm1 + vpmovmskb %ymm1, %eax + testl %eax, %eax + jnz L(first_vec_x2) + +- VPCMPEQ (VEC_SIZE * 3)(%rdi), %ymm0, %ymm1 ++ VPCMPEQ (VEC_SIZE * 2 + 1)(%rdi), %ymm0, %ymm1 + vpmovmskb %ymm1, %eax + testl %eax, %eax + jnz L(first_vec_x3) + +- addq $(VEC_SIZE * 4), %rdi ++ VPCMPEQ (VEC_SIZE * 3 + 1)(%rdi), %ymm0, %ymm1 ++ vpmovmskb %ymm1, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x4) + + # ifndef USE_AS_RAWMEMCHR ++ /* Check if at last VEC_SIZE * 4 length. */ + subq $(VEC_SIZE * 4), %rdx +- jbe L(last_4x_vec_or_less) +-# endif +- +- /* Align data to 4 * VEC_SIZE. */ +- movq %rdi, %rcx +- andl $(4 * VEC_SIZE - 1), %ecx +- andq $-(4 * VEC_SIZE), %rdi +- +-# ifndef USE_AS_RAWMEMCHR +- /* Adjust length. */ ++ jbe L(last_4x_vec_or_less_cmpeq) ++ /* Align data to VEC_SIZE * 4 - 1 for the loop and readjust ++ length. */ ++ incq %rdi ++ movl %edi, %ecx ++ orq $(VEC_SIZE * 4 - 1), %rdi ++ andl $(VEC_SIZE * 4 - 1), %ecx + addq %rcx, %rdx ++# else ++ /* Align data to VEC_SIZE * 4 - 1 for loop. */ ++ incq %rdi ++ orq $(VEC_SIZE * 4 - 1), %rdi + # endif + ++ /* Compare 4 * VEC at a time forward. */ + .p2align 4 + L(loop_4x_vec): +- /* Compare 4 * VEC at a time forward. */ +- VPCMPEQ (%rdi), %ymm0, %ymm1 +- VPCMPEQ VEC_SIZE(%rdi), %ymm0, %ymm2 +- VPCMPEQ (VEC_SIZE * 2)(%rdi), %ymm0, %ymm3 +- VPCMPEQ (VEC_SIZE * 3)(%rdi), %ymm0, %ymm4 +- ++ VPCMPEQ 1(%rdi), %ymm0, %ymm1 ++ VPCMPEQ (VEC_SIZE + 1)(%rdi), %ymm0, %ymm2 ++ VPCMPEQ (VEC_SIZE * 2 + 1)(%rdi), %ymm0, %ymm3 ++ VPCMPEQ (VEC_SIZE * 3 + 1)(%rdi), %ymm0, %ymm4 + vpor %ymm1, %ymm2, %ymm5 + vpor %ymm3, %ymm4, %ymm6 + vpor %ymm5, %ymm6, %ymm5 + +- vpmovmskb %ymm5, %eax +- testl %eax, %eax +- jnz L(4x_vec_end) +- +- addq $(VEC_SIZE * 4), %rdi +- ++ vpmovmskb %ymm5, %ecx + # ifdef USE_AS_RAWMEMCHR +- jmp L(loop_4x_vec) ++ subq $-(VEC_SIZE * 4), %rdi ++ testl %ecx, %ecx ++ jz L(loop_4x_vec) + # else +- subq $(VEC_SIZE * 4), %rdx +- ja L(loop_4x_vec) ++ testl %ecx, %ecx ++ jnz L(loop_4x_vec_end) + +-L(last_4x_vec_or_less): +- /* Less than 4 * VEC and aligned to VEC_SIZE. */ +- addl $(VEC_SIZE * 2), %edx +- jle L(last_2x_vec) ++ subq $-(VEC_SIZE * 4), %rdi + +- VPCMPEQ (%rdi), %ymm0, %ymm1 +- vpmovmskb %ymm1, %eax +- testl %eax, %eax +- jnz L(first_vec_x0) ++ subq $(VEC_SIZE * 4), %rdx ++ ja L(loop_4x_vec) + +- VPCMPEQ VEC_SIZE(%rdi), %ymm0, %ymm1 ++ /* Fall through into less than 4 remaining vectors of length case. ++ */ ++ VPCMPEQ (VEC_SIZE * 0 + 1)(%rdi), %ymm0, %ymm1 + vpmovmskb %ymm1, %eax ++ .p2align 4 ++L(last_4x_vec_or_less): ++ /* Check if first VEC contained match. */ + testl %eax, %eax +- jnz L(first_vec_x1) ++ jnz L(first_vec_x1_check) + +- VPCMPEQ (VEC_SIZE * 2)(%rdi), %ymm0, %ymm1 +- vpmovmskb %ymm1, %eax +- testl %eax, %eax ++ /* If remaining length > VEC_SIZE * 2. */ ++ addl $(VEC_SIZE * 2), %edx ++ jg L(last_4x_vec) + +- jnz L(first_vec_x2_check) +- subl $VEC_SIZE, %edx +- jle L(zero) ++L(last_2x_vec): ++ /* If remaining length < VEC_SIZE. */ ++ addl $VEC_SIZE, %edx ++ jle L(zero_end) + +- VPCMPEQ (VEC_SIZE * 3)(%rdi), %ymm0, %ymm1 ++ /* Check VEC2 and compare any match with remaining length. */ ++ VPCMPEQ (VEC_SIZE + 1)(%rdi), %ymm0, %ymm1 + vpmovmskb %ymm1, %eax +- testl %eax, %eax +- +- jnz L(first_vec_x3_check) +- xorl %eax, %eax ++ tzcntl %eax, %eax ++ cmpl %eax, %edx ++ jbe L(set_zero_end) ++ addq $(VEC_SIZE + 1), %rdi ++ addq %rdi, %rax ++L(zero_end): + VZEROUPPER_RETURN + + .p2align 4 +-L(last_2x_vec): +- addl $(VEC_SIZE * 2), %edx +- VPCMPEQ (%rdi), %ymm0, %ymm1 ++L(loop_4x_vec_end): ++# endif ++ /* rawmemchr will fall through into this if match was found in ++ loop. */ ++ + vpmovmskb %ymm1, %eax + testl %eax, %eax ++ jnz L(last_vec_x1_return) + +- jnz L(first_vec_x0_check) +- subl $VEC_SIZE, %edx +- jle L(zero) +- +- VPCMPEQ VEC_SIZE(%rdi), %ymm0, %ymm1 +- vpmovmskb %ymm1, %eax ++ vpmovmskb %ymm2, %eax + testl %eax, %eax +- jnz L(first_vec_x1_check) +- xorl %eax, %eax +- VZEROUPPER_RETURN ++ jnz L(last_vec_x2_return) + +- .p2align 4 +-L(first_vec_x0_check): +- tzcntl %eax, %eax +- /* Check the end of data. */ +- cmpq %rax, %rdx +- jbe L(zero) ++ vpmovmskb %ymm3, %eax ++ /* Combine VEC3 matches (eax) with VEC4 matches (ecx). */ ++ salq $32, %rcx ++ orq %rcx, %rax ++ tzcntq %rax, %rax ++# ifdef USE_AS_RAWMEMCHR ++ subq $(VEC_SIZE * 2 - 1), %rdi ++# else ++ subq $-(VEC_SIZE * 2 + 1), %rdi ++# endif + addq %rdi, %rax + VZEROUPPER_RETURN ++# ifndef USE_AS_RAWMEMCHR + + .p2align 4 + L(first_vec_x1_check): + tzcntl %eax, %eax +- /* Check the end of data. */ +- cmpq %rax, %rdx +- jbe L(zero) +- addq $VEC_SIZE, %rax ++ /* Adjust length. */ ++ subl $-(VEC_SIZE * 4), %edx ++ /* Check if match within remaining length. */ ++ cmpl %eax, %edx ++ jbe L(set_zero_end) ++ incq %rdi + addq %rdi, %rax + VZEROUPPER_RETURN ++ .p2align 4 ++L(set_zero_end): ++ xorl %eax, %eax ++ VZEROUPPER_RETURN ++# endif + + .p2align 4 +-L(first_vec_x2_check): ++L(last_vec_x1_return): + tzcntl %eax, %eax +- /* Check the end of data. */ +- cmpq %rax, %rdx +- jbe L(zero) +- addq $(VEC_SIZE * 2), %rax ++# ifdef USE_AS_RAWMEMCHR ++ subq $(VEC_SIZE * 4 - 1), %rdi ++# else ++ incq %rdi ++# endif + addq %rdi, %rax + VZEROUPPER_RETURN + + .p2align 4 +-L(first_vec_x3_check): ++L(last_vec_x2_return): + tzcntl %eax, %eax +- /* Check the end of data. */ +- cmpq %rax, %rdx +- jbe L(zero) +- addq $(VEC_SIZE * 3), %rax ++# ifdef USE_AS_RAWMEMCHR ++ subq $(VEC_SIZE * 3 - 1), %rdi ++# else ++ subq $-(VEC_SIZE + 1), %rdi ++# endif + addq %rdi, %rax + VZEROUPPER_RETURN + ++# ifndef USE_AS_RAWMEMCHR + .p2align 4 +-L(zero): +- xorl %eax, %eax +- jmp L(return_vzeroupper) ++L(last_4x_vec_or_less_cmpeq): ++ VPCMPEQ (VEC_SIZE * 4 + 1)(%rdi), %ymm0, %ymm1 ++ vpmovmskb %ymm1, %eax ++ subq $-(VEC_SIZE * 4), %rdi ++ /* Check first VEC regardless. */ ++ testl %eax, %eax ++ jnz L(first_vec_x1_check) + ++ /* If remaining length <= CHAR_PER_VEC * 2. */ ++ addl $(VEC_SIZE * 2), %edx ++ jle L(last_2x_vec) + .p2align 4 +-L(null): +- xorl %eax, %eax +- ret +-# endif ++L(last_4x_vec): ++ VPCMPEQ (VEC_SIZE + 1)(%rdi), %ymm0, %ymm1 ++ vpmovmskb %ymm1, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x2_return) + +- .p2align 4 +-L(first_vec_x0): +- tzcntl %eax, %eax +- addq %rdi, %rax +- VZEROUPPER_RETURN ++ VPCMPEQ (VEC_SIZE * 2 + 1)(%rdi), %ymm0, %ymm1 ++ vpmovmskb %ymm1, %eax + +- .p2align 4 +-L(first_vec_x1): +- tzcntl %eax, %eax +- addq $VEC_SIZE, %rax +- addq %rdi, %rax +- VZEROUPPER_RETURN ++ /* Create mask for possible matches within remaining length. */ ++ movq $-1, %rcx ++ bzhiq %rdx, %rcx, %rcx + +- .p2align 4 +-L(first_vec_x2): ++ /* Test matches in data against length match. */ ++ andl %ecx, %eax ++ jnz L(last_vec_x3) ++ ++ /* if remaining length <= VEC_SIZE * 3 (Note this is after ++ remaining length was found to be > VEC_SIZE * 2. */ ++ subl $VEC_SIZE, %edx ++ jbe L(zero_end2) ++ ++ VPCMPEQ (VEC_SIZE * 3 + 1)(%rdi), %ymm0, %ymm1 ++ vpmovmskb %ymm1, %eax ++ /* Shift remaining length mask for last VEC. */ ++ shrq $32, %rcx ++ andl %ecx, %eax ++ jz L(zero_end2) + tzcntl %eax, %eax +- addq $(VEC_SIZE * 2), %rax ++ addq $(VEC_SIZE * 3 + 1), %rdi + addq %rdi, %rax ++L(zero_end2): + VZEROUPPER_RETURN + + .p2align 4 +-L(4x_vec_end): +- vpmovmskb %ymm1, %eax +- testl %eax, %eax +- jnz L(first_vec_x0) +- vpmovmskb %ymm2, %eax +- testl %eax, %eax +- jnz L(first_vec_x1) +- vpmovmskb %ymm3, %eax +- testl %eax, %eax +- jnz L(first_vec_x2) +- vpmovmskb %ymm4, %eax +- testl %eax, %eax +-L(first_vec_x3): ++L(last_vec_x3): + tzcntl %eax, %eax +- addq $(VEC_SIZE * 3), %rax ++ subq $-(VEC_SIZE * 2 + 1), %rdi + addq %rdi, %rax + VZEROUPPER_RETURN ++# endif + + END (MEMCHR) + #endif +-- +GitLab + diff --git a/SOURCES/ia-move-strlen-multiarch.patch b/SOURCES/ia-move-strlen-multiarch.patch new file mode 100644 index 0000000..2b7084c --- /dev/null +++ b/SOURCES/ia-move-strlen-multiarch.patch @@ -0,0 +1,563 @@ +From 3848cd2cab96c673c98ea339aeefd5a27837f587 Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Tue, 22 Jun 2021 20:42:10 -0700 +Subject: [PATCH] x86-64: Move strlen.S to multiarch/strlen-vec.S + +Since strlen.S contains SSE2 version of strlen/strnlen and SSE4.1 +version of wcslen/wcsnlen, move strlen.S to multiarch/strlen-vec.S +and include multiarch/strlen-vec.S from SSE2 and SSE4.1 variants. +This also removes the unused symbols, __GI___strlen_sse2 and +__GI___wcsnlen_sse4_1. + +(cherry picked from commit a0db678071c60b6c47c468d231dd0b3694ba7a98) +--- + sysdeps/x86_64/multiarch/strlen-sse2.S | 2 +- + sysdeps/x86_64/multiarch/strlen-vec.S | 257 ++++++++++++++++++++++ + sysdeps/x86_64/multiarch/wcsnlen-sse4_1.S | 2 +- + sysdeps/x86_64/strlen.S | 243 +------------------- + 4 files changed, 262 insertions(+), 242 deletions(-) + create mode 100644 sysdeps/x86_64/multiarch/strlen-vec.S + +diff --git a/sysdeps/x86_64/multiarch/strlen-sse2.S b/sysdeps/x86_64/multiarch/strlen-sse2.S +index 7bc57b8d..449c8a7f 100644 +--- a/sysdeps/x86_64/multiarch/strlen-sse2.S ++++ b/sysdeps/x86_64/multiarch/strlen-sse2.S +@@ -20,4 +20,4 @@ + # define strlen __strlen_sse2 + #endif + +-#include "../strlen.S" ++#include "strlen-vec.S" +diff --git a/sysdeps/x86_64/multiarch/strlen-vec.S b/sysdeps/x86_64/multiarch/strlen-vec.S +new file mode 100644 +index 00000000..8f660bb9 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strlen-vec.S +@@ -0,0 +1,257 @@ ++/* SSE2 version of strlen and SSE4.1 version of wcslen. ++ Copyright (C) 2012-2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#include ++ ++#ifdef AS_WCSLEN ++# define PMINU pminud ++# define PCMPEQ pcmpeqd ++# define SHIFT_RETURN shrq $2, %rax ++#else ++# define PMINU pminub ++# define PCMPEQ pcmpeqb ++# define SHIFT_RETURN ++#endif ++ ++/* Long lived register in strlen(s), strnlen(s, n) are: ++ ++ %xmm3 - zero ++ %rdi - s ++ %r10 (s+n) & (~(64-1)) ++ %r11 s+n ++*/ ++ ++ ++.text ++ENTRY(strlen) ++ ++/* Test 64 bytes from %rax for zero. Save result as bitmask in %rdx. */ ++#define FIND_ZERO \ ++ PCMPEQ (%rax), %xmm0; \ ++ PCMPEQ 16(%rax), %xmm1; \ ++ PCMPEQ 32(%rax), %xmm2; \ ++ PCMPEQ 48(%rax), %xmm3; \ ++ pmovmskb %xmm0, %esi; \ ++ pmovmskb %xmm1, %edx; \ ++ pmovmskb %xmm2, %r8d; \ ++ pmovmskb %xmm3, %ecx; \ ++ salq $16, %rdx; \ ++ salq $16, %rcx; \ ++ orq %rsi, %rdx; \ ++ orq %r8, %rcx; \ ++ salq $32, %rcx; \ ++ orq %rcx, %rdx; ++ ++#ifdef AS_STRNLEN ++/* Do not read anything when n==0. */ ++ test %RSI_LP, %RSI_LP ++ jne L(n_nonzero) ++ xor %rax, %rax ++ ret ++L(n_nonzero): ++# ifdef AS_WCSLEN ++ shl $2, %RSI_LP ++# endif ++ ++/* Initialize long lived registers. */ ++ ++ add %RDI_LP, %RSI_LP ++ mov %RSI_LP, %R10_LP ++ and $-64, %R10_LP ++ mov %RSI_LP, %R11_LP ++#endif ++ ++ pxor %xmm0, %xmm0 ++ pxor %xmm1, %xmm1 ++ pxor %xmm2, %xmm2 ++ pxor %xmm3, %xmm3 ++ movq %rdi, %rax ++ movq %rdi, %rcx ++ andq $4095, %rcx ++/* Offsets 4032-4047 will be aligned into 4032 thus fit into page. */ ++ cmpq $4047, %rcx ++/* We cannot unify this branching as it would be ~6 cycles slower. */ ++ ja L(cross_page) ++ ++#ifdef AS_STRNLEN ++/* Test if end is among first 64 bytes. */ ++# define STRNLEN_PROLOG \ ++ mov %r11, %rsi; \ ++ subq %rax, %rsi; \ ++ andq $-64, %rax; \ ++ testq $-64, %rsi; \ ++ je L(strnlen_ret) ++#else ++# define STRNLEN_PROLOG andq $-64, %rax; ++#endif ++ ++/* Ignore bits in mask that come before start of string. */ ++#define PROLOG(lab) \ ++ movq %rdi, %rcx; \ ++ xorq %rax, %rcx; \ ++ STRNLEN_PROLOG; \ ++ sarq %cl, %rdx; \ ++ test %rdx, %rdx; \ ++ je L(lab); \ ++ bsfq %rdx, %rax; \ ++ SHIFT_RETURN; \ ++ ret ++ ++#ifdef AS_STRNLEN ++ andq $-16, %rax ++ FIND_ZERO ++#else ++ /* Test first 16 bytes unaligned. */ ++ movdqu (%rax), %xmm4 ++ PCMPEQ %xmm0, %xmm4 ++ pmovmskb %xmm4, %edx ++ test %edx, %edx ++ je L(next48_bytes) ++ bsf %edx, %eax /* If eax is zeroed 16bit bsf can be used. */ ++ SHIFT_RETURN ++ ret ++ ++L(next48_bytes): ++/* Same as FIND_ZERO except we do not check first 16 bytes. */ ++ andq $-16, %rax ++ PCMPEQ 16(%rax), %xmm1 ++ PCMPEQ 32(%rax), %xmm2 ++ PCMPEQ 48(%rax), %xmm3 ++ pmovmskb %xmm1, %edx ++ pmovmskb %xmm2, %r8d ++ pmovmskb %xmm3, %ecx ++ salq $16, %rdx ++ salq $16, %rcx ++ orq %r8, %rcx ++ salq $32, %rcx ++ orq %rcx, %rdx ++#endif ++ ++ /* When no zero byte is found xmm1-3 are zero so we do not have to ++ zero them. */ ++ PROLOG(loop) ++ ++ .p2align 4 ++L(cross_page): ++ andq $-64, %rax ++ FIND_ZERO ++ PROLOG(loop_init) ++ ++#ifdef AS_STRNLEN ++/* We must do this check to correctly handle strnlen (s, -1). */ ++L(strnlen_ret): ++ bts %rsi, %rdx ++ sarq %cl, %rdx ++ test %rdx, %rdx ++ je L(loop_init) ++ bsfq %rdx, %rax ++ SHIFT_RETURN ++ ret ++#endif ++ .p2align 4 ++L(loop_init): ++ pxor %xmm1, %xmm1 ++ pxor %xmm2, %xmm2 ++ pxor %xmm3, %xmm3 ++#ifdef AS_STRNLEN ++ .p2align 4 ++L(loop): ++ ++ addq $64, %rax ++ cmpq %rax, %r10 ++ je L(exit_end) ++ ++ movdqa (%rax), %xmm0 ++ PMINU 16(%rax), %xmm0 ++ PMINU 32(%rax), %xmm0 ++ PMINU 48(%rax), %xmm0 ++ PCMPEQ %xmm3, %xmm0 ++ pmovmskb %xmm0, %edx ++ testl %edx, %edx ++ jne L(exit) ++ jmp L(loop) ++ ++ .p2align 4 ++L(exit_end): ++ cmp %rax, %r11 ++ je L(first) /* Do not read when end is at page boundary. */ ++ pxor %xmm0, %xmm0 ++ FIND_ZERO ++ ++L(first): ++ bts %r11, %rdx ++ bsfq %rdx, %rdx ++ addq %rdx, %rax ++ subq %rdi, %rax ++ SHIFT_RETURN ++ ret ++ ++ .p2align 4 ++L(exit): ++ pxor %xmm0, %xmm0 ++ FIND_ZERO ++ ++ bsfq %rdx, %rdx ++ addq %rdx, %rax ++ subq %rdi, %rax ++ SHIFT_RETURN ++ ret ++ ++#else ++ ++ /* Main loop. Unrolled twice to improve L2 cache performance on core2. */ ++ .p2align 4 ++L(loop): ++ ++ movdqa 64(%rax), %xmm0 ++ PMINU 80(%rax), %xmm0 ++ PMINU 96(%rax), %xmm0 ++ PMINU 112(%rax), %xmm0 ++ PCMPEQ %xmm3, %xmm0 ++ pmovmskb %xmm0, %edx ++ testl %edx, %edx ++ jne L(exit64) ++ ++ subq $-128, %rax ++ ++ movdqa (%rax), %xmm0 ++ PMINU 16(%rax), %xmm0 ++ PMINU 32(%rax), %xmm0 ++ PMINU 48(%rax), %xmm0 ++ PCMPEQ %xmm3, %xmm0 ++ pmovmskb %xmm0, %edx ++ testl %edx, %edx ++ jne L(exit0) ++ jmp L(loop) ++ ++ .p2align 4 ++L(exit64): ++ addq $64, %rax ++L(exit0): ++ pxor %xmm0, %xmm0 ++ FIND_ZERO ++ ++ bsfq %rdx, %rdx ++ addq %rdx, %rax ++ subq %rdi, %rax ++ SHIFT_RETURN ++ ret ++ ++#endif ++ ++END(strlen) +diff --git a/sysdeps/x86_64/multiarch/wcsnlen-sse4_1.S b/sysdeps/x86_64/multiarch/wcsnlen-sse4_1.S +index a8cab0cb..5fa51fe0 100644 +--- a/sysdeps/x86_64/multiarch/wcsnlen-sse4_1.S ++++ b/sysdeps/x86_64/multiarch/wcsnlen-sse4_1.S +@@ -2,4 +2,4 @@ + #define AS_STRNLEN + #define strlen __wcsnlen_sse4_1 + +-#include "../strlen.S" ++#include "strlen-vec.S" +diff --git a/sysdeps/x86_64/strlen.S b/sysdeps/x86_64/strlen.S +index f845f3d4..ad047d84 100644 +--- a/sysdeps/x86_64/strlen.S ++++ b/sysdeps/x86_64/strlen.S +@@ -1,5 +1,5 @@ +-/* SSE2 version of strlen/wcslen. +- Copyright (C) 2012-2018 Free Software Foundation, Inc. ++/* SSE2 version of strlen. ++ Copyright (C) 2021 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or +@@ -16,243 +16,6 @@ + License along with the GNU C Library; if not, see + . */ + +-#include ++#include "multiarch/strlen-vec.S" + +-#ifdef AS_WCSLEN +-# define PMINU pminud +-# define PCMPEQ pcmpeqd +-# define SHIFT_RETURN shrq $2, %rax +-#else +-# define PMINU pminub +-# define PCMPEQ pcmpeqb +-# define SHIFT_RETURN +-#endif +- +-/* Long lived register in strlen(s), strnlen(s, n) are: +- +- %xmm3 - zero +- %rdi - s +- %r10 (s+n) & (~(64-1)) +- %r11 s+n +-*/ +- +- +-.text +-ENTRY(strlen) +- +-/* Test 64 bytes from %rax for zero. Save result as bitmask in %rdx. */ +-#define FIND_ZERO \ +- PCMPEQ (%rax), %xmm0; \ +- PCMPEQ 16(%rax), %xmm1; \ +- PCMPEQ 32(%rax), %xmm2; \ +- PCMPEQ 48(%rax), %xmm3; \ +- pmovmskb %xmm0, %esi; \ +- pmovmskb %xmm1, %edx; \ +- pmovmskb %xmm2, %r8d; \ +- pmovmskb %xmm3, %ecx; \ +- salq $16, %rdx; \ +- salq $16, %rcx; \ +- orq %rsi, %rdx; \ +- orq %r8, %rcx; \ +- salq $32, %rcx; \ +- orq %rcx, %rdx; +- +-#ifdef AS_STRNLEN +-/* Do not read anything when n==0. */ +- test %RSI_LP, %RSI_LP +- jne L(n_nonzero) +- xor %rax, %rax +- ret +-L(n_nonzero): +-# ifdef AS_WCSLEN +- shl $2, %RSI_LP +-# endif +- +-/* Initialize long lived registers. */ +- +- add %RDI_LP, %RSI_LP +- mov %RSI_LP, %R10_LP +- and $-64, %R10_LP +- mov %RSI_LP, %R11_LP +-#endif +- +- pxor %xmm0, %xmm0 +- pxor %xmm1, %xmm1 +- pxor %xmm2, %xmm2 +- pxor %xmm3, %xmm3 +- movq %rdi, %rax +- movq %rdi, %rcx +- andq $4095, %rcx +-/* Offsets 4032-4047 will be aligned into 4032 thus fit into page. */ +- cmpq $4047, %rcx +-/* We cannot unify this branching as it would be ~6 cycles slower. */ +- ja L(cross_page) +- +-#ifdef AS_STRNLEN +-/* Test if end is among first 64 bytes. */ +-# define STRNLEN_PROLOG \ +- mov %r11, %rsi; \ +- subq %rax, %rsi; \ +- andq $-64, %rax; \ +- testq $-64, %rsi; \ +- je L(strnlen_ret) +-#else +-# define STRNLEN_PROLOG andq $-64, %rax; +-#endif +- +-/* Ignore bits in mask that come before start of string. */ +-#define PROLOG(lab) \ +- movq %rdi, %rcx; \ +- xorq %rax, %rcx; \ +- STRNLEN_PROLOG; \ +- sarq %cl, %rdx; \ +- test %rdx, %rdx; \ +- je L(lab); \ +- bsfq %rdx, %rax; \ +- SHIFT_RETURN; \ +- ret +- +-#ifdef AS_STRNLEN +- andq $-16, %rax +- FIND_ZERO +-#else +- /* Test first 16 bytes unaligned. */ +- movdqu (%rax), %xmm4 +- PCMPEQ %xmm0, %xmm4 +- pmovmskb %xmm4, %edx +- test %edx, %edx +- je L(next48_bytes) +- bsf %edx, %eax /* If eax is zeroed 16bit bsf can be used. */ +- SHIFT_RETURN +- ret +- +-L(next48_bytes): +-/* Same as FIND_ZERO except we do not check first 16 bytes. */ +- andq $-16, %rax +- PCMPEQ 16(%rax), %xmm1 +- PCMPEQ 32(%rax), %xmm2 +- PCMPEQ 48(%rax), %xmm3 +- pmovmskb %xmm1, %edx +- pmovmskb %xmm2, %r8d +- pmovmskb %xmm3, %ecx +- salq $16, %rdx +- salq $16, %rcx +- orq %r8, %rcx +- salq $32, %rcx +- orq %rcx, %rdx +-#endif +- +- /* When no zero byte is found xmm1-3 are zero so we do not have to +- zero them. */ +- PROLOG(loop) +- +- .p2align 4 +-L(cross_page): +- andq $-64, %rax +- FIND_ZERO +- PROLOG(loop_init) +- +-#ifdef AS_STRNLEN +-/* We must do this check to correctly handle strnlen (s, -1). */ +-L(strnlen_ret): +- bts %rsi, %rdx +- sarq %cl, %rdx +- test %rdx, %rdx +- je L(loop_init) +- bsfq %rdx, %rax +- SHIFT_RETURN +- ret +-#endif +- .p2align 4 +-L(loop_init): +- pxor %xmm1, %xmm1 +- pxor %xmm2, %xmm2 +- pxor %xmm3, %xmm3 +-#ifdef AS_STRNLEN +- .p2align 4 +-L(loop): +- +- addq $64, %rax +- cmpq %rax, %r10 +- je L(exit_end) +- +- movdqa (%rax), %xmm0 +- PMINU 16(%rax), %xmm0 +- PMINU 32(%rax), %xmm0 +- PMINU 48(%rax), %xmm0 +- PCMPEQ %xmm3, %xmm0 +- pmovmskb %xmm0, %edx +- testl %edx, %edx +- jne L(exit) +- jmp L(loop) +- +- .p2align 4 +-L(exit_end): +- cmp %rax, %r11 +- je L(first) /* Do not read when end is at page boundary. */ +- pxor %xmm0, %xmm0 +- FIND_ZERO +- +-L(first): +- bts %r11, %rdx +- bsfq %rdx, %rdx +- addq %rdx, %rax +- subq %rdi, %rax +- SHIFT_RETURN +- ret +- +- .p2align 4 +-L(exit): +- pxor %xmm0, %xmm0 +- FIND_ZERO +- +- bsfq %rdx, %rdx +- addq %rdx, %rax +- subq %rdi, %rax +- SHIFT_RETURN +- ret +- +-#else +- +- /* Main loop. Unrolled twice to improve L2 cache performance on core2. */ +- .p2align 4 +-L(loop): +- +- movdqa 64(%rax), %xmm0 +- PMINU 80(%rax), %xmm0 +- PMINU 96(%rax), %xmm0 +- PMINU 112(%rax), %xmm0 +- PCMPEQ %xmm3, %xmm0 +- pmovmskb %xmm0, %edx +- testl %edx, %edx +- jne L(exit64) +- +- subq $-128, %rax +- +- movdqa (%rax), %xmm0 +- PMINU 16(%rax), %xmm0 +- PMINU 32(%rax), %xmm0 +- PMINU 48(%rax), %xmm0 +- PCMPEQ %xmm3, %xmm0 +- pmovmskb %xmm0, %edx +- testl %edx, %edx +- jne L(exit0) +- jmp L(loop) +- +- .p2align 4 +-L(exit64): +- addq $64, %rax +-L(exit0): +- pxor %xmm0, %xmm0 +- FIND_ZERO +- +- bsfq %rdx, %rdx +- addq %rdx, %rax +- subq %rdi, %rax +- SHIFT_RETURN +- ret +- +-#endif +- +-END(strlen) + libc_hidden_builtin_def (strlen) +-- +GitLab + diff --git a/SOURCES/ia-new-macro-entry_p2align.patch b/SOURCES/ia-new-macro-entry_p2align.patch new file mode 100644 index 0000000..3cdf3ec --- /dev/null +++ b/SOURCES/ia-new-macro-entry_p2align.patch @@ -0,0 +1,48 @@ +From ed5f8e29601a35c955183ed218c78438596ed824 Mon Sep 17 00:00:00 2001 +From: Noah Goldstein +Date: Tue, 21 Sep 2021 18:31:49 -0500 +Subject: [PATCH] x86: Modify ENTRY in sysdep.h so that p2align can be + specified + +No bug. + +This change adds a new macro ENTRY_P2ALIGN which takes a second +argument, log2 of the desired function alignment. + +The old ENTRY(name) macro is just ENTRY_P2ALIGN(name, 4) so this +doesn't affect any existing functionality. + +Signed-off-by: Noah Goldstein +(cherry picked from commit fc5bd179ef3a953dff8d1655bd530d0e230ffe71) +--- + sysdeps/x86/sysdep.h | 7 +++++-- + 1 file changed, 5 insertions(+), 2 deletions(-) + +diff --git a/sysdeps/x86/sysdep.h b/sysdeps/x86/sysdep.h +index 01bac0f6..a70bb3a2 100644 +--- a/sysdeps/x86/sysdep.h ++++ b/sysdeps/x86/sysdep.h +@@ -78,15 +78,18 @@ enum cf_protection_level + #define ASM_SIZE_DIRECTIVE(name) .size name,.-name; + + /* Define an entry point visible from C. */ +-#define ENTRY(name) \ ++#define ENTRY_P2ALIGN(name, alignment) \ + .globl C_SYMBOL_NAME(name); \ + .type C_SYMBOL_NAME(name),@function; \ +- .align ALIGNARG(4); \ ++ .align ALIGNARG(alignment); \ + C_LABEL(name) \ + cfi_startproc; \ + _CET_ENDBR; \ + CALL_MCOUNT + ++/* Common entry 16 byte aligns. */ ++#define ENTRY(name) ENTRY_P2ALIGN (name, 4) ++ + #undef END + #define END(name) \ + cfi_endproc; \ +-- +GitLab + diff --git a/SOURCES/ia-no-index_arch_prefer_no_avx512-avx-vnni.patch b/SOURCES/ia-no-index_arch_prefer_no_avx512-avx-vnni.patch new file mode 100644 index 0000000..b094272 --- /dev/null +++ b/SOURCES/ia-no-index_arch_prefer_no_avx512-avx-vnni.patch @@ -0,0 +1,36 @@ +From f64058914af3c3a429e5947a8a90220240f7d51b Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Mon, 6 Dec 2021 07:14:12 -0800 +Subject: [PATCH] x86: Don't set Prefer_No_AVX512 for processors with AVX512 + and AVX-VNNI + +Don't set Prefer_No_AVX512 on processors with AVX512 and AVX-VNNI since +they won't lower CPU frequency when ZMM load and store instructions are +used. + +(cherry picked from commit ceeffe968c01b1202e482f4855cb6baf5c6cb713) +--- + sysdeps/x86/cpu-features.c | 7 +++++-- + 1 file changed, 5 insertions(+), 2 deletions(-) + +diff --git a/sysdeps/x86/cpu-features.c b/sysdeps/x86/cpu-features.c +index 956bfb4f..5ff2baa0 100644 +--- a/sysdeps/x86/cpu-features.c ++++ b/sysdeps/x86/cpu-features.c +@@ -525,8 +525,11 @@ init_cpu_features (struct cpu_features *cpu_features) + |= bit_arch_Prefer_No_VZEROUPPER; + else + { +- cpu_features->preferred[index_arch_Prefer_No_AVX512] +- |= bit_arch_Prefer_No_AVX512; ++ /* Processors with AVX512 and AVX-VNNI won't lower CPU frequency ++ when ZMM load and store instructions are used. */ ++ if (!CPU_FEATURES_CPU_P (cpu_features, AVX_VNNI)) ++ cpu_features->preferred[index_arch_Prefer_No_AVX512] ++ |= bit_arch_Prefer_No_AVX512; + + /* Avoid RTM abort triggered by VZEROUPPER inside a + transactionally executing RTM region. */ +-- +GitLab + diff --git a/SOURCES/ia-opt-less_vec-memcmp-evex-movb.patch b/SOURCES/ia-opt-less_vec-memcmp-evex-movb.patch new file mode 100644 index 0000000..0f0e9c1 --- /dev/null +++ b/SOURCES/ia-opt-less_vec-memcmp-evex-movb.patch @@ -0,0 +1,389 @@ +From 6c6a4eae9d8131531c9231f9f177d2db4130df01 Mon Sep 17 00:00:00 2001 +From: Noah Goldstein +Date: Fri, 24 Dec 2021 18:54:41 -0600 +Subject: [PATCH] x86: Optimize L(less_vec) case in memcmp-evex-movbe.S + +No bug. +Optimizations are twofold. + +1) Replace page cross and 0/1 checks with masked load instructions in + L(less_vec). In applications this reduces branch-misses in the + hot [0, 32] case. +2) Change controlflow so that L(less_vec) case gets the fall through. + +Change 2) helps copies in the [0, 32] size range but comes at the cost +of copies in the [33, 64] size range. From profiles of GCC and +Python3, 94%+ and 99%+ of calls are in the [0, 32] range so this +appears to the the right tradeoff. + +Signed-off-by: Noah Goldstein +Reviewed-by: H.J. Lu +(cherry picked from commit abddd61de090ae84e380aff68a98bd94ef704667) +--- + sysdeps/x86_64/multiarch/memcmp-evex-movbe.S | 249 +++++-------------- + 1 file changed, 56 insertions(+), 193 deletions(-) + +diff --git a/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S b/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S +index 640f6757..d2899e7c 100644 +--- a/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S ++++ b/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S +@@ -62,15 +62,18 @@ Latency: + # define VMOVU vmovdqu64 + + # ifdef USE_AS_WMEMCMP ++# define VMOVU_MASK vmovdqu32 + # define CHAR_SIZE 4 + # define VPCMP vpcmpd + # define VPTEST vptestmd + # else ++# define VMOVU_MASK vmovdqu8 + # define CHAR_SIZE 1 + # define VPCMP vpcmpub + # define VPTEST vptestmb + # endif + ++ + # define VEC_SIZE 32 + # define PAGE_SIZE 4096 + # define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE) +@@ -102,12 +105,48 @@ ENTRY_P2ALIGN (MEMCMP, 6) + movl %edx, %edx + # endif + cmp $CHAR_PER_VEC, %RDX_LP +- jb L(less_vec) ++ /* Fall through for [0, VEC_SIZE] as its the hottest. */ ++ ja L(more_1x_vec) ++ ++ /* Create mask for CHAR's we want to compare. This allows us to ++ avoid having to include page cross logic. */ ++ movl $-1, %ecx ++ bzhil %edx, %ecx, %ecx ++ kmovd %ecx, %k2 ++ ++ /* Safe to load full ymm with mask. */ ++ VMOVU_MASK (%rsi), %YMM2{%k2} ++ VPCMP $4,(%rdi), %YMM2, %k1{%k2} ++ kmovd %k1, %eax ++ testl %eax, %eax ++ jnz L(return_vec_0) ++ ret + ++ .p2align 4 ++L(return_vec_0): ++ tzcntl %eax, %eax ++# ifdef USE_AS_WMEMCMP ++ movl (%rdi, %rax, CHAR_SIZE), %ecx ++ xorl %edx, %edx ++ cmpl (%rsi, %rax, CHAR_SIZE), %ecx ++ /* NB: no partial register stall here because xorl zero idiom ++ above. */ ++ setg %dl ++ leal -1(%rdx, %rdx), %eax ++# else ++ movzbl (%rsi, %rax), %ecx ++ movzbl (%rdi, %rax), %eax ++ subl %ecx, %eax ++# endif ++ ret ++ ++ ++ .p2align 4 ++L(more_1x_vec): + /* From VEC to 2 * VEC. No branch when size == VEC_SIZE. */ + VMOVU (%rsi), %YMM1 + /* Use compare not equals to directly check for mismatch. */ +- VPCMP $4, (%rdi), %YMM1, %k1 ++ VPCMP $4,(%rdi), %YMM1, %k1 + kmovd %k1, %eax + /* NB: eax must be destination register if going to + L(return_vec_[0,2]). For L(return_vec_3) destination register +@@ -131,13 +170,13 @@ ENTRY_P2ALIGN (MEMCMP, 6) + + /* Check third and fourth VEC no matter what. */ + VMOVU (VEC_SIZE * 2)(%rsi), %YMM3 +- VPCMP $4, (VEC_SIZE * 2)(%rdi), %YMM3, %k1 ++ VPCMP $4,(VEC_SIZE * 2)(%rdi), %YMM3, %k1 + kmovd %k1, %eax + testl %eax, %eax + jnz L(return_vec_2) + + VMOVU (VEC_SIZE * 3)(%rsi), %YMM4 +- VPCMP $4, (VEC_SIZE * 3)(%rdi), %YMM4, %k1 ++ VPCMP $4,(VEC_SIZE * 3)(%rdi), %YMM4, %k1 + kmovd %k1, %ecx + testl %ecx, %ecx + jnz L(return_vec_3) +@@ -169,7 +208,7 @@ ENTRY_P2ALIGN (MEMCMP, 6) + VMOVU (VEC_SIZE * 3)(%rsi), %YMM4 + /* Ternary logic to xor (VEC_SIZE * 3)(%rdi) with YMM4 while + oring with YMM1. Result is stored in YMM4. */ +- vpternlogd $0xde, (VEC_SIZE * 3)(%rdi), %YMM1, %YMM4 ++ vpternlogd $0xde,(VEC_SIZE * 3)(%rdi), %YMM1, %YMM4 + + /* Or together YMM2, YMM3, and YMM4 into YMM4. */ + vpternlogd $0xfe, %YMM2, %YMM3, %YMM4 +@@ -184,7 +223,8 @@ ENTRY_P2ALIGN (MEMCMP, 6) + /* NB: eax must be zero to reach here. */ + ret + +- .p2align 4 ++ ++ .p2align 4,, 8 + L(8x_end_return_vec_0_1_2_3): + movq %rdx, %rdi + L(8x_return_vec_0_1_2_3): +@@ -222,23 +262,6 @@ L(return_vec_3): + # endif + ret + +- .p2align 4 +-L(return_vec_0): +- tzcntl %eax, %eax +-# ifdef USE_AS_WMEMCMP +- movl (%rdi, %rax, CHAR_SIZE), %ecx +- xorl %edx, %edx +- cmpl (%rsi, %rax, CHAR_SIZE), %ecx +- /* NB: no partial register stall here because xorl zero idiom +- above. */ +- setg %dl +- leal -1(%rdx, %rdx), %eax +-# else +- movzbl (%rsi, %rax), %ecx +- movzbl (%rdi, %rax), %eax +- subl %ecx, %eax +-# endif +- ret + + .p2align 4 + L(return_vec_1): +@@ -297,7 +320,7 @@ L(loop_4x_vec): + VMOVU (VEC_SIZE * 2)(%rsi, %rdi), %YMM3 + vpxorq (VEC_SIZE * 2)(%rdi), %YMM3, %YMM3 + VMOVU (VEC_SIZE * 3)(%rsi, %rdi), %YMM4 +- vpternlogd $0xde, (VEC_SIZE * 3)(%rdi), %YMM1, %YMM4 ++ vpternlogd $0xde,(VEC_SIZE * 3)(%rdi), %YMM1, %YMM4 + vpternlogd $0xfe, %YMM2, %YMM3, %YMM4 + VPTEST %YMM4, %YMM4, %k1 + kmovd %k1, %ecx +@@ -324,7 +347,7 @@ L(loop_4x_vec): + VMOVU VEC_SIZE(%rsi, %rdx), %YMM2 + vpxorq VEC_SIZE(%rdx), %YMM2, %YMM2 + VMOVU (VEC_SIZE * 3)(%rsi, %rdx), %YMM4 +- vpternlogd $0xde, (VEC_SIZE * 3)(%rdx), %YMM1, %YMM4 ++ vpternlogd $0xde,(VEC_SIZE * 3)(%rdx), %YMM1, %YMM4 + vpternlogd $0xfe, %YMM2, %YMM3, %YMM4 + VPTEST %YMM4, %YMM4, %k1 + kmovd %k1, %ecx +@@ -336,14 +359,14 @@ L(loop_4x_vec): + /* Only entry is from L(more_8x_vec). */ + .p2align 4,, 10 + L(8x_last_2x_vec): +- VPCMP $4, (VEC_SIZE * 2)(%rdx), %YMM3, %k1 ++ VPCMP $4,(VEC_SIZE * 2)(%rdx), %YMM3, %k1 + kmovd %k1, %eax + testl %eax, %eax + jnz L(8x_return_vec_2) + /* Naturally aligned to 16 bytes. */ + L(8x_last_1x_vec): + VMOVU (VEC_SIZE * 3)(%rsi, %rdx), %YMM1 +- VPCMP $4, (VEC_SIZE * 3)(%rdx), %YMM1, %k1 ++ VPCMP $4,(VEC_SIZE * 3)(%rdx), %YMM1, %k1 + kmovd %k1, %eax + testl %eax, %eax + jnz L(8x_return_vec_3) +@@ -392,7 +415,9 @@ L(last_1x_vec): + jnz L(return_vec_0_end) + ret + +- .p2align 4,, 10 ++ ++ /* Don't align. Takes 2-fetch blocks either way and aligning ++ will cause code to spill into another cacheline. */ + L(return_vec_1_end): + /* Use bsf to save code size. This is necessary to have + L(one_or_less) fit in aligning bytes between. */ +@@ -411,31 +436,8 @@ L(return_vec_1_end): + # endif + ret + +- /* NB: L(one_or_less) fits in alignment padding between +- L(return_vec_1_end) and L(return_vec_0_end). */ +-# ifdef USE_AS_WMEMCMP +-L(one_or_less): +- jb L(zero) +- movl (%rdi), %ecx +- xorl %edx, %edx +- cmpl (%rsi), %ecx +- je L(zero) +- setg %dl +- leal -1(%rdx, %rdx), %eax +- ret +-# else +-L(one_or_less): +- jb L(zero) +- movzbl (%rsi), %ecx +- movzbl (%rdi), %eax +- subl %ecx, %eax +- ret +-# endif +-L(zero): +- xorl %eax, %eax +- ret +- +- .p2align 4 ++ /* Don't align. Takes 2-fetch blocks either way and aligning ++ will cause code to spill into another cacheline. */ + L(return_vec_0_end): + tzcntl %eax, %eax + addl %edx, %eax +@@ -451,146 +453,7 @@ L(return_vec_0_end): + subl %ecx, %eax + # endif + ret ++ /* 1-byte until next cache line. */ + +- .p2align 4 +-L(less_vec): +- /* Check if one or less CHAR. This is necessary for size == 0 +- but is also faster for size == CHAR_SIZE. */ +- cmpl $1, %edx +- jbe L(one_or_less) +- +- /* Check if loading one VEC from either s1 or s2 could cause a +- page cross. This can have false positives but is by far the +- fastest method. */ +- movl %edi, %eax +- orl %esi, %eax +- andl $(PAGE_SIZE - 1), %eax +- cmpl $(PAGE_SIZE - VEC_SIZE), %eax +- jg L(page_cross_less_vec) +- +- /* No page cross possible. */ +- VMOVU (%rsi), %YMM2 +- VPCMP $4, (%rdi), %YMM2, %k1 +- kmovd %k1, %eax +- /* Check if any matches where in bounds. Intentionally not +- storing result in eax to limit dependency chain if it goes to +- L(return_vec_0_lv). */ +- bzhil %edx, %eax, %edx +- jnz L(return_vec_0_lv) +- xorl %eax, %eax +- ret +- +- /* Essentially duplicate of L(return_vec_0). Ends up not costing +- any code as shrinks L(less_vec) by allowing 2-byte encoding of +- the jump and ends up fitting in aligning bytes. As well fits on +- same cache line as L(less_vec) so also saves a line from having +- to be fetched on cold calls to memcmp. */ +- .p2align 4,, 4 +-L(return_vec_0_lv): +- tzcntl %eax, %eax +-# ifdef USE_AS_WMEMCMP +- movl (%rdi, %rax, CHAR_SIZE), %ecx +- xorl %edx, %edx +- cmpl (%rsi, %rax, CHAR_SIZE), %ecx +- /* NB: no partial register stall here because xorl zero idiom +- above. */ +- setg %dl +- leal -1(%rdx, %rdx), %eax +-# else +- movzbl (%rsi, %rax), %ecx +- movzbl (%rdi, %rax), %eax +- subl %ecx, %eax +-# endif +- ret +- +- .p2align 4 +-L(page_cross_less_vec): +- /* if USE_AS_WMEMCMP it can only be 0, 4, 8, 12, 16, 20, 24, 28 +- bytes. */ +- cmpl $(16 / CHAR_SIZE), %edx +- jae L(between_16_31) +-# ifndef USE_AS_WMEMCMP +- cmpl $8, %edx +- jae L(between_8_15) +- cmpl $4, %edx +- jb L(between_2_3) +- +- /* Load as big endian with overlapping movbe to avoid branches. +- */ +- movbe (%rdi), %eax +- movbe (%rsi), %ecx +- shlq $32, %rax +- shlq $32, %rcx +- movbe -4(%rdi, %rdx), %edi +- movbe -4(%rsi, %rdx), %esi +- orq %rdi, %rax +- orq %rsi, %rcx +- subq %rcx, %rax +- /* edx is guranteed to be positive int32 in range [4, 7]. */ +- cmovne %edx, %eax +- /* ecx is -1 if rcx > rax. Otherwise 0. */ +- sbbl %ecx, %ecx +- /* If rcx > rax, then ecx is 0 and eax is positive. If rcx == +- rax then eax and ecx are zero. If rax < rax then ecx is -1 so +- eax doesn't matter. */ +- orl %ecx, %eax +- ret +- +- .p2align 4,, 8 +-L(between_8_15): +-# endif +- /* If USE_AS_WMEMCMP fall through into 8-15 byte case. */ +- vmovq (%rdi), %xmm1 +- vmovq (%rsi), %xmm2 +- VPCMP $4, %xmm1, %xmm2, %k1 +- kmovd %k1, %eax +- testl %eax, %eax +- jnz L(return_vec_0_lv) +- /* Use overlapping loads to avoid branches. */ +- vmovq -8(%rdi, %rdx, CHAR_SIZE), %xmm1 +- vmovq -8(%rsi, %rdx, CHAR_SIZE), %xmm2 +- VPCMP $4, %xmm1, %xmm2, %k1 +- addl $(CHAR_PER_VEC - (8 / CHAR_SIZE)), %edx +- kmovd %k1, %eax +- testl %eax, %eax +- jnz L(return_vec_0_end) +- ret +- +- .p2align 4,, 8 +-L(between_16_31): +- /* From 16 to 31 bytes. No branch when size == 16. */ +- +- /* Use movups to save code size. */ +- vmovdqu (%rsi), %xmm2 +- VPCMP $4, (%rdi), %xmm2, %k1 +- kmovd %k1, %eax +- testl %eax, %eax +- jnz L(return_vec_0_lv) +- /* Use overlapping loads to avoid branches. */ +- vmovdqu -16(%rsi, %rdx, CHAR_SIZE), %xmm2 +- VPCMP $4, -16(%rdi, %rdx, CHAR_SIZE), %xmm2, %k1 +- addl $(CHAR_PER_VEC - (16 / CHAR_SIZE)), %edx +- kmovd %k1, %eax +- testl %eax, %eax +- jnz L(return_vec_0_end) +- ret +- +-# ifndef USE_AS_WMEMCMP +-L(between_2_3): +- /* Load as big endian to avoid branches. */ +- movzwl (%rdi), %eax +- movzwl (%rsi), %ecx +- shll $8, %eax +- shll $8, %ecx +- bswap %eax +- bswap %ecx +- movzbl -1(%rdi, %rdx), %edi +- movzbl -1(%rsi, %rdx), %esi +- orl %edi, %eax +- orl %esi, %ecx +- /* Subtraction is okay because the upper 8 bits are zero. */ +- subl %ecx, %eax +- ret +-# endif + END (MEMCMP) + #endif +-- +GitLab + diff --git a/SOURCES/ia-opt-less_vec-memset-avx512.patch b/SOURCES/ia-opt-less_vec-memset-avx512.patch new file mode 100644 index 0000000..0dac8a4 --- /dev/null +++ b/SOURCES/ia-opt-less_vec-memset-avx512.patch @@ -0,0 +1,265 @@ +From 1e65cc3c2278019406125f13d48494ece9f47e95 Mon Sep 17 00:00:00 2001 +From: Noah Goldstein +Date: Mon, 19 Apr 2021 17:48:10 -0400 +Subject: [PATCH] x86: Optimize less_vec evex and avx512 + memset-vec-unaligned-erms.S + +No bug. This commit adds optimized cased for less_vec memset case that +uses the avx512vl/avx512bw mask store avoiding the excessive +branches. test-memset and test-wmemset are passing. + +Signed-off-by: Noah Goldstein +(cherry picked from commit f53790272ce7bdc5ecd14b45f65d0464d2a61a3a) +--- + sysdeps/x86_64/multiarch/ifunc-impl-list.c | 40 ++++++++++----- + sysdeps/x86_64/multiarch/ifunc-memset.h | 6 ++- + .../multiarch/memset-avx512-unaligned-erms.S | 2 +- + .../multiarch/memset-evex-unaligned-erms.S | 2 +- + .../multiarch/memset-vec-unaligned-erms.S | 51 +++++++++++++++---- + 5 files changed, 74 insertions(+), 27 deletions(-) + +diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c +index 85b8863a..d59d65f8 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c ++++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c +@@ -204,19 +204,23 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + __memset_chk_avx2_unaligned_erms_rtm) + IFUNC_IMPL_ADD (array, i, __memset_chk, + (CPU_FEATURE_USABLE (AVX512VL) +- && CPU_FEATURE_USABLE (AVX512BW)), ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), + __memset_chk_evex_unaligned) + IFUNC_IMPL_ADD (array, i, __memset_chk, + (CPU_FEATURE_USABLE (AVX512VL) +- && CPU_FEATURE_USABLE (AVX512BW)), ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), + __memset_chk_evex_unaligned_erms) + IFUNC_IMPL_ADD (array, i, __memset_chk, + (CPU_FEATURE_USABLE (AVX512VL) +- && CPU_FEATURE_USABLE (AVX512BW)), ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), + __memset_chk_avx512_unaligned_erms) + IFUNC_IMPL_ADD (array, i, __memset_chk, + (CPU_FEATURE_USABLE (AVX512VL) +- && CPU_FEATURE_USABLE (AVX512BW)), ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), + __memset_chk_avx512_unaligned) + IFUNC_IMPL_ADD (array, i, __memset_chk, + CPU_FEATURE_USABLE (AVX512F), +@@ -247,19 +251,23 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + __memset_avx2_unaligned_erms_rtm) + IFUNC_IMPL_ADD (array, i, memset, + (CPU_FEATURE_USABLE (AVX512VL) +- && CPU_FEATURE_USABLE (AVX512BW)), ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), + __memset_evex_unaligned) + IFUNC_IMPL_ADD (array, i, memset, + (CPU_FEATURE_USABLE (AVX512VL) +- && CPU_FEATURE_USABLE (AVX512BW)), ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), + __memset_evex_unaligned_erms) + IFUNC_IMPL_ADD (array, i, memset, + (CPU_FEATURE_USABLE (AVX512VL) +- && CPU_FEATURE_USABLE (AVX512BW)), ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), + __memset_avx512_unaligned_erms) + IFUNC_IMPL_ADD (array, i, memset, + (CPU_FEATURE_USABLE (AVX512VL) +- && CPU_FEATURE_USABLE (AVX512BW)), ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), + __memset_avx512_unaligned) + IFUNC_IMPL_ADD (array, i, memset, + CPU_FEATURE_USABLE (AVX512F), +@@ -739,10 +747,14 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + && CPU_FEATURE_USABLE (RTM)), + __wmemset_avx2_unaligned_rtm) + IFUNC_IMPL_ADD (array, i, wmemset, +- CPU_FEATURE_USABLE (AVX512VL), ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), + __wmemset_evex_unaligned) + IFUNC_IMPL_ADD (array, i, wmemset, +- CPU_FEATURE_USABLE (AVX512VL), ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), + __wmemset_avx512_unaligned)) + + #ifdef SHARED +@@ -946,10 +958,14 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + CPU_FEATURE_USABLE (AVX2), + __wmemset_chk_avx2_unaligned) + IFUNC_IMPL_ADD (array, i, __wmemset_chk, +- CPU_FEATURE_USABLE (AVX512VL), ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), + __wmemset_chk_evex_unaligned) + IFUNC_IMPL_ADD (array, i, __wmemset_chk, +- CPU_FEATURE_USABLE (AVX512F), ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), + __wmemset_chk_avx512_unaligned)) + #endif + +diff --git a/sysdeps/x86_64/multiarch/ifunc-memset.h b/sysdeps/x86_64/multiarch/ifunc-memset.h +index 19795938..100e3707 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-memset.h ++++ b/sysdeps/x86_64/multiarch/ifunc-memset.h +@@ -54,7 +54,8 @@ IFUNC_SELECTOR (void) + && !CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_AVX512)) + { + if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) +- && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)) ++ && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW) ++ && CPU_FEATURE_USABLE_P (cpu_features, BMI2)) + { + if (CPU_FEATURE_USABLE_P (cpu_features, ERMS)) + return OPTIMIZE (avx512_unaligned_erms); +@@ -68,7 +69,8 @@ IFUNC_SELECTOR (void) + if (CPU_FEATURE_USABLE_P (cpu_features, AVX2)) + { + if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) +- && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)) ++ && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW) ++ && CPU_FEATURE_USABLE_P (cpu_features, BMI2)) + { + if (CPU_FEATURE_USABLE_P (cpu_features, ERMS)) + return OPTIMIZE (evex_unaligned_erms); +diff --git a/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S +index 22e7b187..8ad842fc 100644 +--- a/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S ++++ b/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S +@@ -19,6 +19,6 @@ + # define SECTION(p) p##.evex512 + # define MEMSET_SYMBOL(p,s) p##_avx512_##s + # define WMEMSET_SYMBOL(p,s) p##_avx512_##s +- ++# define USE_LESS_VEC_MASK_STORE 1 + # include "memset-vec-unaligned-erms.S" + #endif +diff --git a/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S +index ae0a4d6e..640f0929 100644 +--- a/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S ++++ b/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S +@@ -19,6 +19,6 @@ + # define SECTION(p) p##.evex + # define MEMSET_SYMBOL(p,s) p##_evex_##s + # define WMEMSET_SYMBOL(p,s) p##_evex_##s +- ++# define USE_LESS_VEC_MASK_STORE 1 + # include "memset-vec-unaligned-erms.S" + #endif +diff --git a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S +index bae5cba4..f877ac9d 100644 +--- a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S ++++ b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S +@@ -63,6 +63,8 @@ + # endif + #endif + ++#define PAGE_SIZE 4096 ++ + #ifndef SECTION + # error SECTION is not defined! + #endif +@@ -213,11 +215,38 @@ L(loop): + cmpq %rcx, %rdx + jne L(loop) + VZEROUPPER_SHORT_RETURN ++ ++ .p2align 4 + L(less_vec): + /* Less than 1 VEC. */ + # if VEC_SIZE != 16 && VEC_SIZE != 32 && VEC_SIZE != 64 + # error Unsupported VEC_SIZE! + # endif ++# ifdef USE_LESS_VEC_MASK_STORE ++ /* Clear high bits from edi. Only keeping bits relevant to page ++ cross check. Note that we are using rax which is set in ++ MEMSET_VDUP_TO_VEC0_AND_SET_RETURN as ptr from here on out. ++ */ ++ andl $(PAGE_SIZE - 1), %edi ++ /* Check if VEC_SIZE store cross page. Mask stores suffer serious ++ performance degradation when it has to fault supress. */ ++ cmpl $(PAGE_SIZE - VEC_SIZE), %edi ++ ja L(cross_page) ++# if VEC_SIZE > 32 ++ movq $-1, %rcx ++ bzhiq %rdx, %rcx, %rcx ++ kmovq %rcx, %k1 ++# else ++ movl $-1, %ecx ++ bzhil %edx, %ecx, %ecx ++ kmovd %ecx, %k1 ++# endif ++ vmovdqu8 %VEC(0), (%rax) {%k1} ++ VZEROUPPER_RETURN ++ ++ .p2align 4 ++L(cross_page): ++# endif + # if VEC_SIZE > 32 + cmpb $32, %dl + jae L(between_32_63) +@@ -234,36 +263,36 @@ L(less_vec): + cmpb $1, %dl + ja L(between_2_3) + jb 1f +- movb %cl, (%rdi) ++ movb %cl, (%rax) + 1: + VZEROUPPER_RETURN + # if VEC_SIZE > 32 + /* From 32 to 63. No branch when size == 32. */ + L(between_32_63): +- VMOVU %YMM0, -32(%rdi,%rdx) +- VMOVU %YMM0, (%rdi) ++ VMOVU %YMM0, -32(%rax,%rdx) ++ VMOVU %YMM0, (%rax) + VZEROUPPER_RETURN + # endif + # if VEC_SIZE > 16 + /* From 16 to 31. No branch when size == 16. */ + L(between_16_31): +- VMOVU %XMM0, -16(%rdi,%rdx) +- VMOVU %XMM0, (%rdi) ++ VMOVU %XMM0, -16(%rax,%rdx) ++ VMOVU %XMM0, (%rax) + VZEROUPPER_RETURN + # endif + /* From 8 to 15. No branch when size == 8. */ + L(between_8_15): +- movq %rcx, -8(%rdi,%rdx) +- movq %rcx, (%rdi) ++ movq %rcx, -8(%rax,%rdx) ++ movq %rcx, (%rax) + VZEROUPPER_RETURN + L(between_4_7): + /* From 4 to 7. No branch when size == 4. */ +- movl %ecx, -4(%rdi,%rdx) +- movl %ecx, (%rdi) ++ movl %ecx, -4(%rax,%rdx) ++ movl %ecx, (%rax) + VZEROUPPER_RETURN + L(between_2_3): + /* From 2 to 3. No branch when size == 2. */ +- movw %cx, -2(%rdi,%rdx) +- movw %cx, (%rdi) ++ movw %cx, -2(%rax,%rdx) ++ movw %cx, (%rax) + VZEROUPPER_RETURN + END (MEMSET_SYMBOL (__memset, unaligned_erms)) +-- +GitLab + diff --git a/SOURCES/ia-opt-memchr-evex-rtm.patch b/SOURCES/ia-opt-memchr-evex-rtm.patch new file mode 100644 index 0000000..83f3d2b --- /dev/null +++ b/SOURCES/ia-opt-memchr-evex-rtm.patch @@ -0,0 +1,536 @@ +From adf509678af2cf861f3e6d34aba8d062cfc27bf5 Mon Sep 17 00:00:00 2001 +From: Noah Goldstein +Date: Tue, 4 May 2021 19:02:40 -0400 +Subject: [PATCH] x86: Add EVEX optimized memchr family not safe for RTM + +No bug. + +This commit adds a new implementation for EVEX memchr that is not safe +for RTM because it uses vzeroupper. The benefit is that by using +ymm0-ymm15 it can use vpcmpeq and vpternlogd in the 4x loop which is +faster than the RTM safe version which cannot use vpcmpeq because +there is no EVEX encoding for the instruction. All parts of the +implementation aside from the 4x loop are the same for the two +versions and the optimization is only relevant for large sizes. + +Tigerlake: +size , algn , Pos , Cur T , New T , Win , Dif +512 , 6 , 192 , 9.2 , 9.04 , no-RTM , 0.16 +512 , 7 , 224 , 9.19 , 8.98 , no-RTM , 0.21 +2048 , 0 , 256 , 10.74 , 10.54 , no-RTM , 0.2 +2048 , 0 , 512 , 14.81 , 14.87 , RTM , 0.06 +2048 , 0 , 1024 , 22.97 , 22.57 , no-RTM , 0.4 +2048 , 0 , 2048 , 37.49 , 34.51 , no-RTM , 2.98 <-- + +Icelake: +size , algn , Pos , Cur T , New T , Win , Dif +512 , 6 , 192 , 7.6 , 7.3 , no-RTM , 0.3 +512 , 7 , 224 , 7.63 , 7.27 , no-RTM , 0.36 +2048 , 0 , 256 , 8.48 , 8.38 , no-RTM , 0.1 +2048 , 0 , 512 , 11.57 , 11.42 , no-RTM , 0.15 +2048 , 0 , 1024 , 17.92 , 17.38 , no-RTM , 0.54 +2048 , 0 , 2048 , 30.37 , 27.34 , no-RTM , 3.03 <-- + +test-memchr, test-wmemchr, and test-rawmemchr are all passing. + +Signed-off-by: Noah Goldstein +Reviewed-by: H.J. Lu +(cherry picked from commit 104c7b1967c3e78435c6f7eab5e225a7eddf9c6e) +--- + sysdeps/x86_64/multiarch/Makefile | 7 +- + sysdeps/x86_64/multiarch/ifunc-evex.h | 55 ++++++ + sysdeps/x86_64/multiarch/ifunc-impl-list.c | 15 ++ + sysdeps/x86_64/multiarch/memchr-evex-rtm.S | 8 + + sysdeps/x86_64/multiarch/memchr-evex.S | 161 ++++++++++++++---- + sysdeps/x86_64/multiarch/memchr.c | 2 +- + sysdeps/x86_64/multiarch/rawmemchr-evex-rtm.S | 3 + + sysdeps/x86_64/multiarch/rawmemchr.c | 2 +- + sysdeps/x86_64/multiarch/wmemchr-evex-rtm.S | 3 + + sysdeps/x86_64/multiarch/wmemchr.c | 2 +- + 10 files changed, 217 insertions(+), 41 deletions(-) + create mode 100644 sysdeps/x86_64/multiarch/ifunc-evex.h + create mode 100644 sysdeps/x86_64/multiarch/memchr-evex-rtm.S + create mode 100644 sysdeps/x86_64/multiarch/rawmemchr-evex-rtm.S + create mode 100644 sysdeps/x86_64/multiarch/wmemchr-evex-rtm.S + +diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile +index 65fde4eb..26be4095 100644 +--- a/sysdeps/x86_64/multiarch/Makefile ++++ b/sysdeps/x86_64/multiarch/Makefile +@@ -77,7 +77,9 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c \ + strncmp-evex \ + strncpy-evex \ + strnlen-evex \ +- strrchr-evex ++ strrchr-evex \ ++ memchr-evex-rtm \ ++ rawmemchr-evex-rtm + CFLAGS-varshift.c += -msse4 + CFLAGS-strcspn-c.c += -msse4 + CFLAGS-strpbrk-c.c += -msse4 +@@ -110,7 +112,8 @@ sysdep_routines += wmemcmp-sse4 wmemcmp-ssse3 wmemcmp-c \ + wcsnlen-evex \ + wcsrchr-evex \ + wmemchr-evex \ +- wmemcmp-evex-movbe ++ wmemcmp-evex-movbe \ ++ wmemchr-evex-rtm + endif + + ifeq ($(subdir),debug) +diff --git a/sysdeps/x86_64/multiarch/ifunc-evex.h b/sysdeps/x86_64/multiarch/ifunc-evex.h +new file mode 100644 +index 00000000..fc391edb +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/ifunc-evex.h +@@ -0,0 +1,55 @@ ++/* Common definition for ifunc selection optimized with EVEX. ++ All versions must be listed in ifunc-impl-list.c. ++ Copyright (C) 2017-2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#include ++ ++extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_rtm) attribute_hidden; ++ ++ ++static inline void * ++IFUNC_SELECTOR (void) ++{ ++ const struct cpu_features* cpu_features = __get_cpu_features (); ++ ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX2) ++ && CPU_FEATURE_USABLE_P (cpu_features, BMI2) ++ && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) ++ { ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) ++ && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)) ++ { ++ if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) ++ return OPTIMIZE (evex_rtm); ++ ++ return OPTIMIZE (evex); ++ } ++ ++ if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) ++ return OPTIMIZE (avx2_rtm); ++ ++ if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) ++ return OPTIMIZE (avx2); ++ } ++ ++ return OPTIMIZE (sse2); ++} +diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c +index d59d65f8..ac097e8d 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c ++++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c +@@ -52,6 +52,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + && CPU_FEATURE_USABLE (AVX512BW) + && CPU_FEATURE_USABLE (BMI2)), + __memchr_evex) ++ IFUNC_IMPL_ADD (array, i, memchr, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), ++ __memchr_evex_rtm) + IFUNC_IMPL_ADD (array, i, memchr, 1, __memchr_sse2)) + + /* Support sysdeps/x86_64/multiarch/memcmp.c. */ +@@ -288,6 +293,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + && CPU_FEATURE_USABLE (AVX512BW) + && CPU_FEATURE_USABLE (BMI2)), + __rawmemchr_evex) ++ IFUNC_IMPL_ADD (array, i, rawmemchr, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), ++ __rawmemchr_evex_rtm) + IFUNC_IMPL_ADD (array, i, rawmemchr, 1, __rawmemchr_sse2)) + + /* Support sysdeps/x86_64/multiarch/strlen.c. */ +@@ -711,6 +721,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + && CPU_FEATURE_USABLE (AVX512BW) + && CPU_FEATURE_USABLE (BMI2)), + __wmemchr_evex) ++ IFUNC_IMPL_ADD (array, i, wmemchr, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), ++ __wmemchr_evex_rtm) + IFUNC_IMPL_ADD (array, i, wmemchr, 1, __wmemchr_sse2)) + + /* Support sysdeps/x86_64/multiarch/wmemcmp.c. */ +diff --git a/sysdeps/x86_64/multiarch/memchr-evex-rtm.S b/sysdeps/x86_64/multiarch/memchr-evex-rtm.S +new file mode 100644 +index 00000000..19871882 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/memchr-evex-rtm.S +@@ -0,0 +1,8 @@ ++#ifndef MEMCHR ++# define MEMCHR __memchr_evex_rtm ++#endif ++ ++#define USE_IN_RTM 1 ++#define SECTION(p) p##.evex.rtm ++ ++#include "memchr-evex.S" +diff --git a/sysdeps/x86_64/multiarch/memchr-evex.S b/sysdeps/x86_64/multiarch/memchr-evex.S +index f3fdad4f..4d0ed6d1 100644 +--- a/sysdeps/x86_64/multiarch/memchr-evex.S ++++ b/sysdeps/x86_64/multiarch/memchr-evex.S +@@ -38,10 +38,32 @@ + # define CHAR_SIZE 1 + # endif + ++ /* In the 4x loop the RTM and non-RTM versions have data pointer ++ off by VEC_SIZE * 4 with RTM version being VEC_SIZE * 4 greater. ++ This is represented by BASE_OFFSET. As well because the RTM ++ version uses vpcmp which stores a bit per element compared where ++ the non-RTM version uses vpcmpeq which stores a bit per byte ++ compared RET_SCALE of CHAR_SIZE is only relevant for the RTM ++ version. */ ++# ifdef USE_IN_RTM ++# define VZEROUPPER ++# define BASE_OFFSET (VEC_SIZE * 4) ++# define RET_SCALE CHAR_SIZE ++# else ++# define VZEROUPPER vzeroupper ++# define BASE_OFFSET 0 ++# define RET_SCALE 1 ++# endif ++ ++ /* In the return from 4x loop memchr and rawmemchr versions have ++ data pointers off by VEC_SIZE * 4 with memchr version being ++ VEC_SIZE * 4 greater. */ + # ifdef USE_AS_RAWMEMCHR ++# define RET_OFFSET (BASE_OFFSET - (VEC_SIZE * 4)) + # define RAW_PTR_REG rcx + # define ALGN_PTR_REG rdi + # else ++# define RET_OFFSET BASE_OFFSET + # define RAW_PTR_REG rdi + # define ALGN_PTR_REG rcx + # endif +@@ -57,11 +79,15 @@ + # define YMM5 ymm21 + # define YMM6 ymm22 + ++# ifndef SECTION ++# define SECTION(p) p##.evex ++# endif ++ + # define VEC_SIZE 32 + # define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE) + # define PAGE_SIZE 4096 + +- .section .text.evex,"ax",@progbits ++ .section SECTION(.text),"ax",@progbits + ENTRY (MEMCHR) + # ifndef USE_AS_RAWMEMCHR + /* Check for zero length. */ +@@ -237,14 +263,15 @@ L(cross_page_continue): + /* Check if at last CHAR_PER_VEC * 4 length. */ + subq $(CHAR_PER_VEC * 4), %rdx + jbe L(last_4x_vec_or_less_cmpeq) +- addq $VEC_SIZE, %rdi ++ /* +VEC_SIZE if USE_IN_RTM otherwise +VEC_SIZE * 5. */ ++ addq $(VEC_SIZE + (VEC_SIZE * 4 - BASE_OFFSET)), %rdi + + /* Align data to VEC_SIZE * 4 for the loop and readjust length. + */ + # ifdef USE_AS_WMEMCHR + movl %edi, %ecx + andq $-(4 * VEC_SIZE), %rdi +- andl $(VEC_SIZE * 4 - 1), %ecx ++ subl %edi, %ecx + /* NB: Divide bytes by 4 to get the wchar_t count. */ + sarl $2, %ecx + addq %rcx, %rdx +@@ -254,15 +281,28 @@ L(cross_page_continue): + subq %rdi, %rdx + # endif + # else +- addq $VEC_SIZE, %rdi ++ addq $(VEC_SIZE + (VEC_SIZE * 4 - BASE_OFFSET)), %rdi + andq $-(4 * VEC_SIZE), %rdi + # endif +- ++# ifdef USE_IN_RTM + vpxorq %XMMZERO, %XMMZERO, %XMMZERO ++# else ++ /* copy ymmmatch to ymm0 so we can use vpcmpeq which is not ++ encodable with EVEX registers (ymm16-ymm31). */ ++ vmovdqa64 %YMMMATCH, %ymm0 ++# endif + + /* Compare 4 * VEC at a time forward. */ + .p2align 4 + L(loop_4x_vec): ++ /* Two versions of the loop. One that does not require ++ vzeroupper by not using ymm0-ymm15 and another does that require ++ vzeroupper because it uses ymm0-ymm15. The reason why ymm0-ymm15 ++ is used at all is because there is no EVEX encoding vpcmpeq and ++ with vpcmpeq this loop can be performed more efficiently. The ++ non-vzeroupper version is safe for RTM while the vzeroupper ++ version should be prefered if RTM are not supported. */ ++# ifdef USE_IN_RTM + /* It would be possible to save some instructions using 4x VPCMP + but bottleneck on port 5 makes it not woth it. */ + VPCMP $4, (VEC_SIZE * 4)(%rdi), %YMMMATCH, %k1 +@@ -273,12 +313,55 @@ L(loop_4x_vec): + /* Reduce VEC2 / VEC3 with min and VEC1 with zero mask. */ + VPMINU %YMM2, %YMM3, %YMM3{%k1}{z} + VPCMP $0, %YMM3, %YMMZERO, %k2 ++# else ++ /* Since vptern can only take 3x vectors fastest to do 1 vec ++ seperately with EVEX vpcmp. */ ++# ifdef USE_AS_WMEMCHR ++ /* vptern can only accept masks for epi32/epi64 so can only save ++ instruction using not equals mask on vptern with wmemchr. */ ++ VPCMP $4, (%rdi), %YMMMATCH, %k1 ++# else ++ VPCMP $0, (%rdi), %YMMMATCH, %k1 ++# endif ++ /* Compare 3x with vpcmpeq and or them all together with vptern. ++ */ ++ VPCMPEQ VEC_SIZE(%rdi), %ymm0, %ymm2 ++ VPCMPEQ (VEC_SIZE * 2)(%rdi), %ymm0, %ymm3 ++ VPCMPEQ (VEC_SIZE * 3)(%rdi), %ymm0, %ymm4 ++# ifdef USE_AS_WMEMCHR ++ /* This takes the not of or between ymm2, ymm3, ymm4 as well as ++ combines result from VEC0 with zero mask. */ ++ vpternlogd $1, %ymm2, %ymm3, %ymm4{%k1}{z} ++ vpmovmskb %ymm4, %ecx ++# else ++ /* 254 is mask for oring ymm2, ymm3, ymm4 into ymm4. */ ++ vpternlogd $254, %ymm2, %ymm3, %ymm4 ++ vpmovmskb %ymm4, %ecx ++ kmovd %k1, %eax ++# endif ++# endif ++ + # ifdef USE_AS_RAWMEMCHR + subq $-(VEC_SIZE * 4), %rdi ++# endif ++# ifdef USE_IN_RTM + kortestd %k2, %k3 ++# else ++# ifdef USE_AS_WMEMCHR ++ /* ecx contains not of matches. All 1s means no matches. incl will ++ overflow and set zeroflag if that is the case. */ ++ incl %ecx ++# else ++ /* If either VEC1 (eax) or VEC2-VEC4 (ecx) are not zero. Adding ++ to ecx is not an issue because if eax is non-zero it will be ++ used for returning the match. If it is zero the add does ++ nothing. */ ++ addq %rax, %rcx ++# endif ++# endif ++# ifdef USE_AS_RAWMEMCHR + jz L(loop_4x_vec) + # else +- kortestd %k2, %k3 + jnz L(loop_4x_vec_end) + + subq $-(VEC_SIZE * 4), %rdi +@@ -288,10 +371,11 @@ L(loop_4x_vec): + + /* Fall through into less than 4 remaining vectors of length case. + */ +- VPCMP $0, (VEC_SIZE * 4)(%rdi), %YMMMATCH, %k0 ++ VPCMP $0, BASE_OFFSET(%rdi), %YMMMATCH, %k0 ++ addq $(BASE_OFFSET - VEC_SIZE), %rdi + kmovd %k0, %eax +- addq $(VEC_SIZE * 3), %rdi +- .p2align 4 ++ VZEROUPPER ++ + L(last_4x_vec_or_less): + /* Check if first VEC contained match. */ + testl %eax, %eax +@@ -338,73 +422,78 @@ L(loop_4x_vec_end): + /* rawmemchr will fall through into this if match was found in + loop. */ + ++# if defined USE_IN_RTM || defined USE_AS_WMEMCHR + /* k1 has not of matches with VEC1. */ + kmovd %k1, %eax +-# ifdef USE_AS_WMEMCHR ++# ifdef USE_AS_WMEMCHR + subl $((1 << CHAR_PER_VEC) - 1), %eax +-# else ++# else + incl %eax ++# endif ++# else ++ /* eax already has matches for VEC1. */ ++ testl %eax, %eax + # endif + jnz L(last_vec_x1_return) + ++# ifdef USE_IN_RTM + VPCMP $0, %YMM2, %YMMZERO, %k0 + kmovd %k0, %eax ++# else ++ vpmovmskb %ymm2, %eax ++# endif + testl %eax, %eax + jnz L(last_vec_x2_return) + ++# ifdef USE_IN_RTM + kmovd %k2, %eax + testl %eax, %eax + jnz L(last_vec_x3_return) + + kmovd %k3, %eax + tzcntl %eax, %eax +-# ifdef USE_AS_RAWMEMCHR +- leaq (VEC_SIZE * 3)(%rdi, %rax, CHAR_SIZE), %rax ++ leaq (VEC_SIZE * 3 + RET_OFFSET)(%rdi, %rax, CHAR_SIZE), %rax + # else +- leaq (VEC_SIZE * 7)(%rdi, %rax, CHAR_SIZE), %rax ++ vpmovmskb %ymm3, %eax ++ /* Combine matches in VEC3 (eax) with matches in VEC4 (ecx). */ ++ salq $VEC_SIZE, %rcx ++ orq %rcx, %rax ++ tzcntq %rax, %rax ++ leaq (VEC_SIZE * 2 + RET_OFFSET)(%rdi, %rax), %rax ++ VZEROUPPER + # endif + ret + + .p2align 4 + L(last_vec_x1_return): + tzcntl %eax, %eax +-# ifdef USE_AS_RAWMEMCHR +-# ifdef USE_AS_WMEMCHR ++# if defined USE_AS_WMEMCHR || RET_OFFSET != 0 + /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */ +- leaq (%rdi, %rax, CHAR_SIZE), %rax +-# else +- addq %rdi, %rax +-# endif ++ leaq RET_OFFSET(%rdi, %rax, CHAR_SIZE), %rax + # else +- /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */ +- leaq (VEC_SIZE * 4)(%rdi, %rax, CHAR_SIZE), %rax ++ addq %rdi, %rax + # endif ++ VZEROUPPER + ret + + .p2align 4 + L(last_vec_x2_return): + tzcntl %eax, %eax +-# ifdef USE_AS_RAWMEMCHR +- /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */ +- leaq VEC_SIZE(%rdi, %rax, CHAR_SIZE), %rax +-# else +- /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */ +- leaq (VEC_SIZE * 5)(%rdi, %rax, CHAR_SIZE), %rax +-# endif ++ /* NB: Multiply bytes by RET_SCALE to get the wchar_t count ++ if relevant (RET_SCALE = CHAR_SIZE if USE_AS_WMEMCHAR and ++ USE_IN_RTM are both defined. Otherwise RET_SCALE = 1. */ ++ leaq (VEC_SIZE + RET_OFFSET)(%rdi, %rax, RET_SCALE), %rax ++ VZEROUPPER + ret + ++# ifdef USE_IN_RTM + .p2align 4 + L(last_vec_x3_return): + tzcntl %eax, %eax +-# ifdef USE_AS_RAWMEMCHR +- /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */ +- leaq (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %rax +-# else + /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */ +- leaq (VEC_SIZE * 6)(%rdi, %rax, CHAR_SIZE), %rax +-# endif ++ leaq (VEC_SIZE * 2 + RET_OFFSET)(%rdi, %rax, CHAR_SIZE), %rax + ret +- ++# endif + + # ifndef USE_AS_RAWMEMCHR + L(last_4x_vec_or_less_cmpeq): +diff --git a/sysdeps/x86_64/multiarch/memchr.c b/sysdeps/x86_64/multiarch/memchr.c +index 016f5784..f28aea77 100644 +--- a/sysdeps/x86_64/multiarch/memchr.c ++++ b/sysdeps/x86_64/multiarch/memchr.c +@@ -24,7 +24,7 @@ + # undef memchr + + # define SYMBOL_NAME memchr +-# include "ifunc-avx2.h" ++# include "ifunc-evex.h" + + libc_ifunc_redirected (__redirect_memchr, memchr, IFUNC_SELECTOR ()); + strong_alias (memchr, __memchr) +diff --git a/sysdeps/x86_64/multiarch/rawmemchr-evex-rtm.S b/sysdeps/x86_64/multiarch/rawmemchr-evex-rtm.S +new file mode 100644 +index 00000000..deda1ca3 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/rawmemchr-evex-rtm.S +@@ -0,0 +1,3 @@ ++#define MEMCHR __rawmemchr_evex_rtm ++#define USE_AS_RAWMEMCHR 1 ++#include "memchr-evex-rtm.S" +diff --git a/sysdeps/x86_64/multiarch/rawmemchr.c b/sysdeps/x86_64/multiarch/rawmemchr.c +index 8a0bc313..1f764f35 100644 +--- a/sysdeps/x86_64/multiarch/rawmemchr.c ++++ b/sysdeps/x86_64/multiarch/rawmemchr.c +@@ -26,7 +26,7 @@ + # undef __rawmemchr + + # define SYMBOL_NAME rawmemchr +-# include "ifunc-avx2.h" ++# include "ifunc-evex.h" + + libc_ifunc_redirected (__redirect_rawmemchr, __rawmemchr, + IFUNC_SELECTOR ()); +diff --git a/sysdeps/x86_64/multiarch/wmemchr-evex-rtm.S b/sysdeps/x86_64/multiarch/wmemchr-evex-rtm.S +new file mode 100644 +index 00000000..a346cd35 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/wmemchr-evex-rtm.S +@@ -0,0 +1,3 @@ ++#define MEMCHR __wmemchr_evex_rtm ++#define USE_AS_WMEMCHR 1 ++#include "memchr-evex-rtm.S" +diff --git a/sysdeps/x86_64/multiarch/wmemchr.c b/sysdeps/x86_64/multiarch/wmemchr.c +index 6d833702..f9c91915 100644 +--- a/sysdeps/x86_64/multiarch/wmemchr.c ++++ b/sysdeps/x86_64/multiarch/wmemchr.c +@@ -26,7 +26,7 @@ + # undef __wmemchr + + # define SYMBOL_NAME wmemchr +-# include "ifunc-avx2.h" ++# include "ifunc-evex.h" + + libc_ifunc_redirected (__redirect_wmemchr, __wmemchr, IFUNC_SELECTOR ()); + weak_alias (__wmemchr, wmemchr) +-- +GitLab + diff --git a/SOURCES/ia-opt-memchr-evex.patch b/SOURCES/ia-opt-memchr-evex.patch new file mode 100644 index 0000000..905e400 --- /dev/null +++ b/SOURCES/ia-opt-memchr-evex.patch @@ -0,0 +1,701 @@ +From 8bb82fd62bdb2d8741998a5a1be38388890452da Mon Sep 17 00:00:00 2001 +From: Noah Goldstein +Date: Mon, 3 May 2021 03:03:19 -0400 +Subject: [PATCH] x86: Optimize memchr-evex.S + +No bug. This commit optimizes memchr-evex.S. The optimizations include +replacing some branches with cmovcc, avoiding some branches entirely +in the less_4x_vec case, making the page cross logic less strict, +saving some ALU in the alignment process, and most importantly +increasing ILP in the 4x loop. test-memchr, test-rawmemchr, and +test-wmemchr are all passing. + +Signed-off-by: Noah Goldstein +Reviewed-by: H.J. Lu +(cherry picked from commit 2a76821c3081d2c0231ecd2618f52662cb48fccd) +--- + sysdeps/x86_64/multiarch/memchr-evex.S | 547 +++++++++++++++---------- + 1 file changed, 322 insertions(+), 225 deletions(-) + +diff --git a/sysdeps/x86_64/multiarch/memchr-evex.S b/sysdeps/x86_64/multiarch/memchr-evex.S +index 6dd5d67b..81d5cd64 100644 +--- a/sysdeps/x86_64/multiarch/memchr-evex.S ++++ b/sysdeps/x86_64/multiarch/memchr-evex.S +@@ -26,14 +26,28 @@ + + # ifdef USE_AS_WMEMCHR + # define VPBROADCAST vpbroadcastd +-# define VPCMP vpcmpd +-# define SHIFT_REG r8d ++# define VPMINU vpminud ++# define VPCMP vpcmpd ++# define VPCMPEQ vpcmpeqd ++# define CHAR_SIZE 4 + # else + # define VPBROADCAST vpbroadcastb +-# define VPCMP vpcmpb +-# define SHIFT_REG ecx ++# define VPMINU vpminub ++# define VPCMP vpcmpb ++# define VPCMPEQ vpcmpeqb ++# define CHAR_SIZE 1 + # endif + ++# ifdef USE_AS_RAWMEMCHR ++# define RAW_PTR_REG rcx ++# define ALGN_PTR_REG rdi ++# else ++# define RAW_PTR_REG rdi ++# define ALGN_PTR_REG rcx ++# endif ++ ++# define XMMZERO xmm23 ++# define YMMZERO ymm23 + # define XMMMATCH xmm16 + # define YMMMATCH ymm16 + # define YMM1 ymm17 +@@ -44,6 +58,8 @@ + # define YMM6 ymm22 + + # define VEC_SIZE 32 ++# define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE) ++# define PAGE_SIZE 4096 + + .section .text.evex,"ax",@progbits + ENTRY (MEMCHR) +@@ -51,11 +67,7 @@ ENTRY (MEMCHR) + /* Check for zero length. */ + test %RDX_LP, %RDX_LP + jz L(zero) +-# endif +- movl %edi, %ecx +-# ifdef USE_AS_WMEMCHR +- shl $2, %RDX_LP +-# else ++ + # ifdef __ILP32__ + /* Clear the upper 32 bits. */ + movl %edx, %edx +@@ -64,318 +76,403 @@ ENTRY (MEMCHR) + /* Broadcast CHAR to YMMMATCH. */ + VPBROADCAST %esi, %YMMMATCH + /* Check if we may cross page boundary with one vector load. */ +- andl $(2 * VEC_SIZE - 1), %ecx +- cmpl $VEC_SIZE, %ecx +- ja L(cros_page_boundary) ++ movl %edi, %eax ++ andl $(PAGE_SIZE - 1), %eax ++ cmpl $(PAGE_SIZE - VEC_SIZE), %eax ++ ja L(cross_page_boundary) + + /* Check the first VEC_SIZE bytes. */ +- VPCMP $0, (%rdi), %YMMMATCH, %k1 +- kmovd %k1, %eax +- testl %eax, %eax +- ++ VPCMP $0, (%rdi), %YMMMATCH, %k0 ++ kmovd %k0, %eax + # ifndef USE_AS_RAWMEMCHR +- jnz L(first_vec_x0_check) +- /* Adjust length and check the end of data. */ +- subq $VEC_SIZE, %rdx +- jbe L(zero) ++ /* If length < CHAR_PER_VEC handle special. */ ++ cmpq $CHAR_PER_VEC, %rdx ++ jbe L(first_vec_x0) ++# endif ++ testl %eax, %eax ++ jz L(aligned_more) ++ tzcntl %eax, %eax ++# ifdef USE_AS_WMEMCHR ++ /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */ ++ leaq (%rdi, %rax, CHAR_SIZE), %rax + # else +- jnz L(first_vec_x0) ++ addq %rdi, %rax + # endif +- +- /* Align data for aligned loads in the loop. */ +- addq $VEC_SIZE, %rdi +- andl $(VEC_SIZE - 1), %ecx +- andq $-VEC_SIZE, %rdi ++ ret + + # ifndef USE_AS_RAWMEMCHR +- /* Adjust length. */ +- addq %rcx, %rdx +- +- subq $(VEC_SIZE * 4), %rdx +- jbe L(last_4x_vec_or_less) +-# endif +- jmp L(more_4x_vec) ++L(zero): ++ xorl %eax, %eax ++ ret + ++ .p2align 5 ++L(first_vec_x0): ++ /* Check if first match was before length. */ ++ tzcntl %eax, %eax ++ xorl %ecx, %ecx ++ cmpl %eax, %edx ++ leaq (%rdi, %rax, CHAR_SIZE), %rax ++ cmovle %rcx, %rax ++ ret ++# else ++ /* NB: first_vec_x0 is 17 bytes which will leave ++ cross_page_boundary (which is relatively cold) close enough ++ to ideal alignment. So only realign L(cross_page_boundary) if ++ rawmemchr. */ + .p2align 4 +-L(cros_page_boundary): +- andl $(VEC_SIZE - 1), %ecx ++# endif ++L(cross_page_boundary): ++ /* Save pointer before aligning as its original value is ++ necessary for computer return address if byte is found or ++ adjusting length if it is not and this is memchr. */ ++ movq %rdi, %rcx ++ /* Align data to VEC_SIZE. ALGN_PTR_REG is rcx for memchr and rdi ++ for rawmemchr. */ ++ andq $-VEC_SIZE, %ALGN_PTR_REG ++ VPCMP $0, (%ALGN_PTR_REG), %YMMMATCH, %k0 ++ kmovd %k0, %r8d + # ifdef USE_AS_WMEMCHR +- /* NB: Divide shift count by 4 since each bit in K1 represent 4 ++ /* NB: Divide shift count by 4 since each bit in K0 represent 4 + bytes. */ +- movl %ecx, %SHIFT_REG +- sarl $2, %SHIFT_REG ++ sarl $2, %eax ++# endif ++# ifndef USE_AS_RAWMEMCHR ++ movl $(PAGE_SIZE / CHAR_SIZE), %esi ++ subl %eax, %esi + # endif +- andq $-VEC_SIZE, %rdi +- VPCMP $0, (%rdi), %YMMMATCH, %k1 +- kmovd %k1, %eax +- /* Remove the leading bytes. */ +- sarxl %SHIFT_REG, %eax, %eax +- testl %eax, %eax +- jz L(aligned_more) +- tzcntl %eax, %eax + # ifdef USE_AS_WMEMCHR +- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ +- sall $2, %eax ++ andl $(CHAR_PER_VEC - 1), %eax + # endif ++ /* Remove the leading bytes. */ ++ sarxl %eax, %r8d, %eax + # ifndef USE_AS_RAWMEMCHR + /* Check the end of data. */ +- cmpq %rax, %rdx +- jbe L(zero) ++ cmpq %rsi, %rdx ++ jbe L(first_vec_x0) ++# endif ++ testl %eax, %eax ++ jz L(cross_page_continue) ++ tzcntl %eax, %eax ++# ifdef USE_AS_WMEMCHR ++ /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */ ++ leaq (%RAW_PTR_REG, %rax, CHAR_SIZE), %rax ++# else ++ addq %RAW_PTR_REG, %rax + # endif +- addq %rdi, %rax +- addq %rcx, %rax + ret + + .p2align 4 +-L(aligned_more): +-# ifndef USE_AS_RAWMEMCHR +- /* Calculate "rdx + rcx - VEC_SIZE" with "rdx - (VEC_SIZE - rcx)" +- instead of "(rdx + rcx) - VEC_SIZE" to void possible addition +- overflow. */ +- negq %rcx +- addq $VEC_SIZE, %rcx ++L(first_vec_x1): ++ tzcntl %eax, %eax ++ leaq VEC_SIZE(%rdi, %rax, CHAR_SIZE), %rax ++ ret + +- /* Check the end of data. */ +- subq %rcx, %rdx +- jbe L(zero) +-# endif ++ .p2align 4 ++L(first_vec_x2): ++ tzcntl %eax, %eax ++ leaq (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %rax ++ ret + +- addq $VEC_SIZE, %rdi ++ .p2align 4 ++L(first_vec_x3): ++ tzcntl %eax, %eax ++ leaq (VEC_SIZE * 3)(%rdi, %rax, CHAR_SIZE), %rax ++ ret + +-# ifndef USE_AS_RAWMEMCHR +- subq $(VEC_SIZE * 4), %rdx +- jbe L(last_4x_vec_or_less) +-# endif ++ .p2align 4 ++L(first_vec_x4): ++ tzcntl %eax, %eax ++ leaq (VEC_SIZE * 4)(%rdi, %rax, CHAR_SIZE), %rax ++ ret + +-L(more_4x_vec): ++ .p2align 5 ++L(aligned_more): + /* Check the first 4 * VEC_SIZE. Only one VEC_SIZE at a time + since data is only aligned to VEC_SIZE. */ +- VPCMP $0, (%rdi), %YMMMATCH, %k1 +- kmovd %k1, %eax +- testl %eax, %eax +- jnz L(first_vec_x0) + +- VPCMP $0, VEC_SIZE(%rdi), %YMMMATCH, %k1 +- kmovd %k1, %eax ++# ifndef USE_AS_RAWMEMCHR ++ /* Align data to VEC_SIZE. */ ++L(cross_page_continue): ++ xorl %ecx, %ecx ++ subl %edi, %ecx ++ andq $-VEC_SIZE, %rdi ++ /* esi is for adjusting length to see if near the end. */ ++ leal (VEC_SIZE * 5)(%rdi, %rcx), %esi ++# ifdef USE_AS_WMEMCHR ++ /* NB: Divide bytes by 4 to get the wchar_t count. */ ++ sarl $2, %esi ++# endif ++# else ++ andq $-VEC_SIZE, %rdi ++L(cross_page_continue): ++# endif ++ /* Load first VEC regardless. */ ++ VPCMP $0, (VEC_SIZE)(%rdi), %YMMMATCH, %k0 ++ kmovd %k0, %eax ++# ifndef USE_AS_RAWMEMCHR ++ /* Adjust length. If near end handle specially. */ ++ subq %rsi, %rdx ++ jbe L(last_4x_vec_or_less) ++# endif + testl %eax, %eax + jnz L(first_vec_x1) + +- VPCMP $0, (VEC_SIZE * 2)(%rdi), %YMMMATCH, %k1 +- kmovd %k1, %eax ++ VPCMP $0, (VEC_SIZE * 2)(%rdi), %YMMMATCH, %k0 ++ kmovd %k0, %eax + testl %eax, %eax + jnz L(first_vec_x2) + +- VPCMP $0, (VEC_SIZE * 3)(%rdi), %YMMMATCH, %k1 +- kmovd %k1, %eax ++ VPCMP $0, (VEC_SIZE * 3)(%rdi), %YMMMATCH, %k0 ++ kmovd %k0, %eax + testl %eax, %eax + jnz L(first_vec_x3) + +- addq $(VEC_SIZE * 4), %rdi ++ VPCMP $0, (VEC_SIZE * 4)(%rdi), %YMMMATCH, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x4) ++ + + # ifndef USE_AS_RAWMEMCHR +- subq $(VEC_SIZE * 4), %rdx +- jbe L(last_4x_vec_or_less) +-# endif ++ /* Check if at last CHAR_PER_VEC * 4 length. */ ++ subq $(CHAR_PER_VEC * 4), %rdx ++ jbe L(last_4x_vec_or_less_cmpeq) ++ addq $VEC_SIZE, %rdi + +- /* Align data to 4 * VEC_SIZE. */ +- movq %rdi, %rcx +- andl $(4 * VEC_SIZE - 1), %ecx ++ /* Align data to VEC_SIZE * 4 for the loop and readjust length. ++ */ ++# ifdef USE_AS_WMEMCHR ++ movl %edi, %ecx + andq $-(4 * VEC_SIZE), %rdi +- +-# ifndef USE_AS_RAWMEMCHR +- /* Adjust length. */ ++ andl $(VEC_SIZE * 4 - 1), %ecx ++ /* NB: Divide bytes by 4 to get the wchar_t count. */ ++ sarl $2, %ecx + addq %rcx, %rdx ++# else ++ addq %rdi, %rdx ++ andq $-(4 * VEC_SIZE), %rdi ++ subq %rdi, %rdx ++# endif ++# else ++ addq $VEC_SIZE, %rdi ++ andq $-(4 * VEC_SIZE), %rdi + # endif + ++ vpxorq %XMMZERO, %XMMZERO, %XMMZERO ++ ++ /* Compare 4 * VEC at a time forward. */ + .p2align 4 + L(loop_4x_vec): +- /* Compare 4 * VEC at a time forward. */ +- VPCMP $0, (%rdi), %YMMMATCH, %k1 +- VPCMP $0, VEC_SIZE(%rdi), %YMMMATCH, %k2 +- kord %k1, %k2, %k5 +- VPCMP $0, (VEC_SIZE * 2)(%rdi), %YMMMATCH, %k3 +- VPCMP $0, (VEC_SIZE * 3)(%rdi), %YMMMATCH, %k4 +- +- kord %k3, %k4, %k6 +- kortestd %k5, %k6 +- jnz L(4x_vec_end) +- +- addq $(VEC_SIZE * 4), %rdi +- ++ /* It would be possible to save some instructions using 4x VPCMP ++ but bottleneck on port 5 makes it not woth it. */ ++ VPCMP $4, (VEC_SIZE * 4)(%rdi), %YMMMATCH, %k1 ++ /* xor will set bytes match esi to zero. */ ++ vpxorq (VEC_SIZE * 5)(%rdi), %YMMMATCH, %YMM2 ++ vpxorq (VEC_SIZE * 6)(%rdi), %YMMMATCH, %YMM3 ++ VPCMP $0, (VEC_SIZE * 7)(%rdi), %YMMMATCH, %k3 ++ /* Reduce VEC2 / VEC3 with min and VEC1 with zero mask. */ ++ VPMINU %YMM2, %YMM3, %YMM3 {%k1} {z} ++ VPCMP $0, %YMM3, %YMMZERO, %k2 + # ifdef USE_AS_RAWMEMCHR +- jmp L(loop_4x_vec) ++ subq $-(VEC_SIZE * 4), %rdi ++ kortestd %k2, %k3 ++ jz L(loop_4x_vec) + # else +- subq $(VEC_SIZE * 4), %rdx ++ kortestd %k2, %k3 ++ jnz L(loop_4x_vec_end) ++ ++ subq $-(VEC_SIZE * 4), %rdi ++ ++ subq $(CHAR_PER_VEC * 4), %rdx + ja L(loop_4x_vec) + ++ /* Fall through into less than 4 remaining vectors of length case. ++ */ ++ VPCMP $0, (VEC_SIZE * 4)(%rdi), %YMMMATCH, %k0 ++ kmovd %k0, %eax ++ addq $(VEC_SIZE * 3), %rdi ++ .p2align 4 + L(last_4x_vec_or_less): +- /* Less than 4 * VEC and aligned to VEC_SIZE. */ +- addl $(VEC_SIZE * 2), %edx +- jle L(last_2x_vec) +- +- VPCMP $0, (%rdi), %YMMMATCH, %k1 +- kmovd %k1, %eax ++ /* Check if first VEC contained match. */ + testl %eax, %eax +- jnz L(first_vec_x0) ++ jnz L(first_vec_x1_check) + +- VPCMP $0, VEC_SIZE(%rdi), %YMMMATCH, %k1 +- kmovd %k1, %eax +- testl %eax, %eax +- jnz L(first_vec_x1) ++ /* If remaining length > CHAR_PER_VEC * 2. */ ++ addl $(CHAR_PER_VEC * 2), %edx ++ jg L(last_4x_vec) + +- VPCMP $0, (VEC_SIZE * 2)(%rdi), %YMMMATCH, %k1 +- kmovd %k1, %eax +- testl %eax, %eax ++L(last_2x_vec): ++ /* If remaining length < CHAR_PER_VEC. */ ++ addl $CHAR_PER_VEC, %edx ++ jle L(zero_end) + +- jnz L(first_vec_x2_check) +- subl $VEC_SIZE, %edx +- jle L(zero) ++ /* Check VEC2 and compare any match with remaining length. */ ++ VPCMP $0, (VEC_SIZE * 2)(%rdi), %YMMMATCH, %k0 ++ kmovd %k0, %eax ++ tzcntl %eax, %eax ++ cmpl %eax, %edx ++ jbe L(set_zero_end) ++ leaq (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %rax ++L(zero_end): ++ ret + +- VPCMP $0, (VEC_SIZE * 3)(%rdi), %YMMMATCH, %k1 +- kmovd %k1, %eax +- testl %eax, %eax + +- jnz L(first_vec_x3_check) ++ .p2align 4 ++L(first_vec_x1_check): ++ tzcntl %eax, %eax ++ /* Adjust length. */ ++ subl $-(CHAR_PER_VEC * 4), %edx ++ /* Check if match within remaining length. */ ++ cmpl %eax, %edx ++ jbe L(set_zero_end) ++ /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */ ++ leaq VEC_SIZE(%rdi, %rax, CHAR_SIZE), %rax ++ ret ++L(set_zero_end): + xorl %eax, %eax + ret + + .p2align 4 +-L(last_2x_vec): +- addl $(VEC_SIZE * 2), %edx +- VPCMP $0, (%rdi), %YMMMATCH, %k1 ++L(loop_4x_vec_end): ++# endif ++ /* rawmemchr will fall through into this if match was found in ++ loop. */ ++ ++ /* k1 has not of matches with VEC1. */ + kmovd %k1, %eax +- testl %eax, %eax ++# ifdef USE_AS_WMEMCHR ++ subl $((1 << CHAR_PER_VEC) - 1), %eax ++# else ++ incl %eax ++# endif ++ jnz L(last_vec_x1_return) + +- jnz L(first_vec_x0_check) +- subl $VEC_SIZE, %edx +- jle L(zero) ++ VPCMP $0, %YMM2, %YMMZERO, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x2_return) + +- VPCMP $0, VEC_SIZE(%rdi), %YMMMATCH, %k1 +- kmovd %k1, %eax ++ kmovd %k2, %eax + testl %eax, %eax +- jnz L(first_vec_x1_check) +- xorl %eax, %eax +- ret ++ jnz L(last_vec_x3_return) + +- .p2align 4 +-L(first_vec_x0_check): ++ kmovd %k3, %eax + tzcntl %eax, %eax +-# ifdef USE_AS_WMEMCHR +- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ +- sall $2, %eax ++# ifdef USE_AS_RAWMEMCHR ++ leaq (VEC_SIZE * 3)(%rdi, %rax, CHAR_SIZE), %rax ++# else ++ leaq (VEC_SIZE * 7)(%rdi, %rax, CHAR_SIZE), %rax + # endif +- /* Check the end of data. */ +- cmpq %rax, %rdx +- jbe L(zero) +- addq %rdi, %rax + ret + + .p2align 4 +-L(first_vec_x1_check): ++L(last_vec_x1_return): + tzcntl %eax, %eax +-# ifdef USE_AS_WMEMCHR +- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ +- sall $2, %eax +-# endif +- /* Check the end of data. */ +- cmpq %rax, %rdx +- jbe L(zero) +- addq $VEC_SIZE, %rax ++# ifdef USE_AS_RAWMEMCHR ++# ifdef USE_AS_WMEMCHR ++ /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */ ++ leaq (%rdi, %rax, CHAR_SIZE), %rax ++# else + addq %rdi, %rax +- ret +- +- .p2align 4 +-L(first_vec_x2_check): +- tzcntl %eax, %eax +-# ifdef USE_AS_WMEMCHR +- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ +- sall $2, %eax ++# endif ++# else ++ /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */ ++ leaq (VEC_SIZE * 4)(%rdi, %rax, CHAR_SIZE), %rax + # endif +- /* Check the end of data. */ +- cmpq %rax, %rdx +- jbe L(zero) +- addq $(VEC_SIZE * 2), %rax +- addq %rdi, %rax + ret + + .p2align 4 +-L(first_vec_x3_check): ++L(last_vec_x2_return): + tzcntl %eax, %eax +-# ifdef USE_AS_WMEMCHR +- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ +- sall $2, %eax ++# ifdef USE_AS_RAWMEMCHR ++ /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */ ++ leaq VEC_SIZE(%rdi, %rax, CHAR_SIZE), %rax ++# else ++ /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */ ++ leaq (VEC_SIZE * 5)(%rdi, %rax, CHAR_SIZE), %rax + # endif +- /* Check the end of data. */ +- cmpq %rax, %rdx +- jbe L(zero) +- addq $(VEC_SIZE * 3), %rax +- addq %rdi, %rax + ret + + .p2align 4 +-L(zero): +- xorl %eax, %eax +- ret +-# endif +- +- .p2align 4 +-L(first_vec_x0): ++L(last_vec_x3_return): + tzcntl %eax, %eax +-# ifdef USE_AS_WMEMCHR +- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ +- leaq (%rdi, %rax, 4), %rax ++# ifdef USE_AS_RAWMEMCHR ++ /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */ ++ leaq (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %rax + # else +- addq %rdi, %rax ++ /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */ ++ leaq (VEC_SIZE * 6)(%rdi, %rax, CHAR_SIZE), %rax + # endif + ret + ++ ++# ifndef USE_AS_RAWMEMCHR ++L(last_4x_vec_or_less_cmpeq): ++ VPCMP $0, (VEC_SIZE * 5)(%rdi), %YMMMATCH, %k0 ++ kmovd %k0, %eax ++ subq $-(VEC_SIZE * 4), %rdi ++ /* Check first VEC regardless. */ ++ testl %eax, %eax ++ jnz L(first_vec_x1_check) ++ ++ /* If remaining length <= CHAR_PER_VEC * 2. */ ++ addl $(CHAR_PER_VEC * 2), %edx ++ jle L(last_2x_vec) ++ + .p2align 4 +-L(first_vec_x1): ++L(last_4x_vec): ++ VPCMP $0, (VEC_SIZE * 2)(%rdi), %YMMMATCH, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x2) ++ ++ ++ VPCMP $0, (VEC_SIZE * 3)(%rdi), %YMMMATCH, %k0 ++ kmovd %k0, %eax ++ /* Create mask for possible matches within remaining length. */ ++# ifdef USE_AS_WMEMCHR ++ movl $((1 << (CHAR_PER_VEC * 2)) - 1), %ecx ++ bzhil %edx, %ecx, %ecx ++# else ++ movq $-1, %rcx ++ bzhiq %rdx, %rcx, %rcx ++# endif ++ /* Test matches in data against length match. */ ++ andl %ecx, %eax ++ jnz L(last_vec_x3) ++ ++ /* if remaining length <= CHAR_PER_VEC * 3 (Note this is after ++ remaining length was found to be > CHAR_PER_VEC * 2. */ ++ subl $CHAR_PER_VEC, %edx ++ jbe L(zero_end2) ++ ++ ++ VPCMP $0, (VEC_SIZE * 4)(%rdi), %YMMMATCH, %k0 ++ kmovd %k0, %eax ++ /* Shift remaining length mask for last VEC. */ ++# ifdef USE_AS_WMEMCHR ++ shrl $CHAR_PER_VEC, %ecx ++# else ++ shrq $CHAR_PER_VEC, %rcx ++# endif ++ andl %ecx, %eax ++ jz L(zero_end2) + tzcntl %eax, %eax +-# ifdef USE_AS_WMEMCHR +- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ +- leaq VEC_SIZE(%rdi, %rax, 4), %rax +-# else +- addq $VEC_SIZE, %rax +- addq %rdi, %rax +-# endif ++ leaq (VEC_SIZE * 4)(%rdi, %rax, CHAR_SIZE), %rax ++L(zero_end2): + ret + +- .p2align 4 +-L(first_vec_x2): ++L(last_vec_x2): + tzcntl %eax, %eax +-# ifdef USE_AS_WMEMCHR +- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ +- leaq (VEC_SIZE * 2)(%rdi, %rax, 4), %rax +-# else +- addq $(VEC_SIZE * 2), %rax +- addq %rdi, %rax +-# endif ++ leaq (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %rax + ret + + .p2align 4 +-L(4x_vec_end): +- kmovd %k1, %eax +- testl %eax, %eax +- jnz L(first_vec_x0) +- kmovd %k2, %eax +- testl %eax, %eax +- jnz L(first_vec_x1) +- kmovd %k3, %eax +- testl %eax, %eax +- jnz L(first_vec_x2) +- kmovd %k4, %eax +- testl %eax, %eax +-L(first_vec_x3): ++L(last_vec_x3): + tzcntl %eax, %eax +-# ifdef USE_AS_WMEMCHR +- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ +- leaq (VEC_SIZE * 3)(%rdi, %rax, 4), %rax +-# else +- addq $(VEC_SIZE * 3), %rax +- addq %rdi, %rax +-# endif ++ leaq (VEC_SIZE * 3)(%rdi, %rax, CHAR_SIZE), %rax + ret ++# endif + + END (MEMCHR) + #endif +-- +GitLab + diff --git a/SOURCES/ia-opt-memcmp-avx2-movbe.patch b/SOURCES/ia-opt-memcmp-avx2-movbe.patch new file mode 100644 index 0000000..d55adb7 --- /dev/null +++ b/SOURCES/ia-opt-memcmp-avx2-movbe.patch @@ -0,0 +1,873 @@ +From 65438851072f6131049a0ae471dcab90870e51f3 Mon Sep 17 00:00:00 2001 +From: Noah Goldstein +Date: Mon, 17 May 2021 13:56:52 -0400 +Subject: [PATCH] x86: Optimize memcmp-avx2-movbe.S + +No bug. This commit optimizes memcmp-avx2.S. The optimizations include +adding a new vec compare path for small sizes, reorganizing the entry +control flow, and removing some unnecissary ALU instructions from the +main loop. test-memcmp and test-wmemcmp are both passing. + +Signed-off-by: Noah Goldstein +Reviewed-by: H.J. Lu +(cherry picked from commit 16d12015c57701b08d7bbed6ec536641bcafb428) +--- + sysdeps/x86_64/multiarch/ifunc-impl-list.c | 6 + + sysdeps/x86_64/multiarch/ifunc-memcmp.h | 1 + + sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S | 676 +++++++++++-------- + 3 files changed, 402 insertions(+), 281 deletions(-) + +diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c +index ac097e8d..8be0d78a 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c ++++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c +@@ -63,16 +63,19 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL (i, name, memcmp, + IFUNC_IMPL_ADD (array, i, memcmp, + (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (BMI2) + && CPU_FEATURE_USABLE (MOVBE)), + __memcmp_avx2_movbe) + IFUNC_IMPL_ADD (array, i, memcmp, + (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (BMI2) + && CPU_FEATURE_USABLE (MOVBE) + && CPU_FEATURE_USABLE (RTM)), + __memcmp_avx2_movbe_rtm) + IFUNC_IMPL_ADD (array, i, memcmp, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2) + && CPU_FEATURE_USABLE (MOVBE)), + __memcmp_evex_movbe) + IFUNC_IMPL_ADD (array, i, memcmp, CPU_FEATURE_USABLE (SSE4_1), +@@ -732,16 +735,19 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL (i, name, wmemcmp, + IFUNC_IMPL_ADD (array, i, wmemcmp, + (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (BMI2) + && CPU_FEATURE_USABLE (MOVBE)), + __wmemcmp_avx2_movbe) + IFUNC_IMPL_ADD (array, i, wmemcmp, + (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (BMI2) + && CPU_FEATURE_USABLE (MOVBE) + && CPU_FEATURE_USABLE (RTM)), + __wmemcmp_avx2_movbe_rtm) + IFUNC_IMPL_ADD (array, i, wmemcmp, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2) + && CPU_FEATURE_USABLE (MOVBE)), + __wmemcmp_evex_movbe) + IFUNC_IMPL_ADD (array, i, wmemcmp, CPU_FEATURE_USABLE (SSE4_1), +diff --git a/sysdeps/x86_64/multiarch/ifunc-memcmp.h b/sysdeps/x86_64/multiarch/ifunc-memcmp.h +index 8043c635..690dffe8 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-memcmp.h ++++ b/sysdeps/x86_64/multiarch/ifunc-memcmp.h +@@ -33,6 +33,7 @@ IFUNC_SELECTOR (void) + + if (CPU_FEATURE_USABLE_P (cpu_features, AVX2) + && CPU_FEATURE_USABLE_P (cpu_features, MOVBE) ++ && CPU_FEATURE_USABLE_P (cpu_features, BMI2) + && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) + { + if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) +diff --git a/sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S b/sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S +index 9d5c9c72..16fc673e 100644 +--- a/sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S ++++ b/sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S +@@ -19,17 +19,23 @@ + #if IS_IN (libc) + + /* memcmp/wmemcmp is implemented as: +- 1. For size from 2 to 7 bytes, load as big endian with movbe and bswap +- to avoid branches. +- 2. Use overlapping compare to avoid branch. +- 3. Use vector compare when size >= 4 bytes for memcmp or size >= 8 +- bytes for wmemcmp. +- 4. If size is 8 * VEC_SIZE or less, unroll the loop. +- 5. Compare 4 * VEC_SIZE at a time with the aligned first memory ++ 1. Use ymm vector compares when possible. The only case where ++ vector compares is not possible for when size < VEC_SIZE ++ and loading from either s1 or s2 would cause a page cross. ++ 2. For size from 2 to 7 bytes on page cross, load as big endian ++ with movbe and bswap to avoid branches. ++ 3. Use xmm vector compare when size >= 4 bytes for memcmp or ++ size >= 8 bytes for wmemcmp. ++ 4. Optimistically compare up to first 4 * VEC_SIZE one at a ++ to check for early mismatches. Only do this if its guranteed the ++ work is not wasted. ++ 5. If size is 8 * VEC_SIZE or less, unroll the loop. ++ 6. Compare 4 * VEC_SIZE at a time with the aligned first memory + area. +- 6. Use 2 vector compares when size is 2 * VEC_SIZE or less. +- 7. Use 4 vector compares when size is 4 * VEC_SIZE or less. +- 8. Use 8 vector compares when size is 8 * VEC_SIZE or less. */ ++ 7. Use 2 vector compares when size is 2 * VEC_SIZE or less. ++ 8. Use 4 vector compares when size is 4 * VEC_SIZE or less. ++ 9. Use 8 vector compares when size is 8 * VEC_SIZE or less. */ ++ + + # include + +@@ -38,8 +44,10 @@ + # endif + + # ifdef USE_AS_WMEMCMP ++# define CHAR_SIZE 4 + # define VPCMPEQ vpcmpeqd + # else ++# define CHAR_SIZE 1 + # define VPCMPEQ vpcmpeqb + # endif + +@@ -52,7 +60,7 @@ + # endif + + # define VEC_SIZE 32 +-# define VEC_MASK ((1 << VEC_SIZE) - 1) ++# define PAGE_SIZE 4096 + + /* Warning! + wmemcmp has to use SIGNED comparison for elements. +@@ -71,136 +79,359 @@ ENTRY (MEMCMP) + jb L(less_vec) + + /* From VEC to 2 * VEC. No branch when size == VEC_SIZE. */ +- vmovdqu (%rsi), %ymm2 +- VPCMPEQ (%rdi), %ymm2, %ymm2 +- vpmovmskb %ymm2, %eax +- subl $VEC_MASK, %eax +- jnz L(first_vec) ++ vmovdqu (%rsi), %ymm1 ++ VPCMPEQ (%rdi), %ymm1, %ymm1 ++ vpmovmskb %ymm1, %eax ++ /* NB: eax must be destination register if going to ++ L(return_vec_[0,2]). For L(return_vec_3 destination register ++ must be ecx. */ ++ incl %eax ++ jnz L(return_vec_0) + + cmpq $(VEC_SIZE * 2), %rdx +- jbe L(last_vec) +- +- VPCMPEQ %ymm0, %ymm0, %ymm0 +- /* More than 2 * VEC. */ +- cmpq $(VEC_SIZE * 8), %rdx +- ja L(more_8x_vec) +- cmpq $(VEC_SIZE * 4), %rdx +- jb L(last_4x_vec) +- +- /* From 4 * VEC to 8 * VEC, inclusively. */ +- vmovdqu (%rsi), %ymm1 +- VPCMPEQ (%rdi), %ymm1, %ymm1 ++ jbe L(last_1x_vec) + ++ /* Check second VEC no matter what. */ + vmovdqu VEC_SIZE(%rsi), %ymm2 +- VPCMPEQ VEC_SIZE(%rdi), %ymm2, %ymm2 ++ VPCMPEQ VEC_SIZE(%rdi), %ymm2, %ymm2 ++ vpmovmskb %ymm2, %eax ++ /* If all 4 VEC where equal eax will be all 1s so incl will ++ overflow and set zero flag. */ ++ incl %eax ++ jnz L(return_vec_1) + +- vmovdqu (VEC_SIZE * 2)(%rsi), %ymm3 +- VPCMPEQ (VEC_SIZE * 2)(%rdi), %ymm3, %ymm3 ++ /* Less than 4 * VEC. */ ++ cmpq $(VEC_SIZE * 4), %rdx ++ jbe L(last_2x_vec) + ++ /* Check third and fourth VEC no matter what. */ ++ vmovdqu (VEC_SIZE * 2)(%rsi), %ymm3 ++ VPCMPEQ (VEC_SIZE * 2)(%rdi), %ymm3, %ymm3 ++ vpmovmskb %ymm3, %eax ++ incl %eax ++ jnz L(return_vec_2) + vmovdqu (VEC_SIZE * 3)(%rsi), %ymm4 +- VPCMPEQ (VEC_SIZE * 3)(%rdi), %ymm4, %ymm4 ++ VPCMPEQ (VEC_SIZE * 3)(%rdi), %ymm4, %ymm4 ++ vpmovmskb %ymm4, %ecx ++ incl %ecx ++ jnz L(return_vec_3) + +- vpand %ymm1, %ymm2, %ymm5 +- vpand %ymm3, %ymm4, %ymm6 +- vpand %ymm5, %ymm6, %ymm5 ++ /* Go to 4x VEC loop. */ ++ cmpq $(VEC_SIZE * 8), %rdx ++ ja L(more_8x_vec) + +- vptest %ymm0, %ymm5 +- jnc L(4x_vec_end) ++ /* Handle remainder of size = 4 * VEC + 1 to 8 * VEC without any ++ branches. */ + ++ /* Load first two VEC from s2 before adjusting addresses. */ ++ vmovdqu -(VEC_SIZE * 4)(%rsi, %rdx), %ymm1 ++ vmovdqu -(VEC_SIZE * 3)(%rsi, %rdx), %ymm2 + leaq -(4 * VEC_SIZE)(%rdi, %rdx), %rdi + leaq -(4 * VEC_SIZE)(%rsi, %rdx), %rsi +- vmovdqu (%rsi), %ymm1 +- VPCMPEQ (%rdi), %ymm1, %ymm1 + +- vmovdqu VEC_SIZE(%rsi), %ymm2 +- VPCMPEQ VEC_SIZE(%rdi), %ymm2, %ymm2 +- vpand %ymm2, %ymm1, %ymm5 ++ /* Wait to load from s1 until addressed adjust due to ++ unlamination of microfusion with complex address mode. */ ++ VPCMPEQ (%rdi), %ymm1, %ymm1 ++ VPCMPEQ (VEC_SIZE)(%rdi), %ymm2, %ymm2 + + vmovdqu (VEC_SIZE * 2)(%rsi), %ymm3 +- VPCMPEQ (VEC_SIZE * 2)(%rdi), %ymm3, %ymm3 +- vpand %ymm3, %ymm5, %ymm5 +- ++ VPCMPEQ (VEC_SIZE * 2)(%rdi), %ymm3, %ymm3 + vmovdqu (VEC_SIZE * 3)(%rsi), %ymm4 +- VPCMPEQ (VEC_SIZE * 3)(%rdi), %ymm4, %ymm4 +- vpand %ymm4, %ymm5, %ymm5 ++ VPCMPEQ (VEC_SIZE * 3)(%rdi), %ymm4, %ymm4 + +- vptest %ymm0, %ymm5 +- jnc L(4x_vec_end) +- xorl %eax, %eax ++ /* Reduce VEC0 - VEC4. */ ++ vpand %ymm1, %ymm2, %ymm5 ++ vpand %ymm3, %ymm4, %ymm6 ++ vpand %ymm5, %ymm6, %ymm7 ++ vpmovmskb %ymm7, %ecx ++ incl %ecx ++ jnz L(return_vec_0_1_2_3) ++ /* NB: eax must be zero to reach here. */ ++ VZEROUPPER_RETURN ++ ++ .p2align 4 ++L(return_vec_0): ++ tzcntl %eax, %eax ++# ifdef USE_AS_WMEMCMP ++ movl (%rdi, %rax), %ecx ++ xorl %edx, %edx ++ cmpl (%rsi, %rax), %ecx ++ /* NB: no partial register stall here because xorl zero idiom ++ above. */ ++ setg %dl ++ leal -1(%rdx, %rdx), %eax ++# else ++ movzbl (%rsi, %rax), %ecx ++ movzbl (%rdi, %rax), %eax ++ subl %ecx, %eax ++# endif + L(return_vzeroupper): + ZERO_UPPER_VEC_REGISTERS_RETURN + + .p2align 4 +-L(last_2x_vec): +- /* From VEC to 2 * VEC. No branch when size == VEC_SIZE. */ +- vmovdqu (%rsi), %ymm2 +- VPCMPEQ (%rdi), %ymm2, %ymm2 +- vpmovmskb %ymm2, %eax +- subl $VEC_MASK, %eax +- jnz L(first_vec) ++L(return_vec_1): ++ tzcntl %eax, %eax ++# ifdef USE_AS_WMEMCMP ++ movl VEC_SIZE(%rdi, %rax), %ecx ++ xorl %edx, %edx ++ cmpl VEC_SIZE(%rsi, %rax), %ecx ++ setg %dl ++ leal -1(%rdx, %rdx), %eax ++# else ++ movzbl VEC_SIZE(%rsi, %rax), %ecx ++ movzbl VEC_SIZE(%rdi, %rax), %eax ++ subl %ecx, %eax ++# endif ++ VZEROUPPER_RETURN ++ ++ .p2align 4 ++L(return_vec_2): ++ tzcntl %eax, %eax ++# ifdef USE_AS_WMEMCMP ++ movl (VEC_SIZE * 2)(%rdi, %rax), %ecx ++ xorl %edx, %edx ++ cmpl (VEC_SIZE * 2)(%rsi, %rax), %ecx ++ setg %dl ++ leal -1(%rdx, %rdx), %eax ++# else ++ movzbl (VEC_SIZE * 2)(%rsi, %rax), %ecx ++ movzbl (VEC_SIZE * 2)(%rdi, %rax), %eax ++ subl %ecx, %eax ++# endif ++ VZEROUPPER_RETURN ++ ++ /* NB: p2align 5 here to ensure 4x loop is 32 byte aligned. */ ++ .p2align 5 ++L(8x_return_vec_0_1_2_3): ++ /* Returning from L(more_8x_vec) requires restoring rsi. */ ++ addq %rdi, %rsi ++L(return_vec_0_1_2_3): ++ vpmovmskb %ymm1, %eax ++ incl %eax ++ jnz L(return_vec_0) + +-L(last_vec): +- /* Use overlapping loads to avoid branches. */ +- leaq -VEC_SIZE(%rdi, %rdx), %rdi +- leaq -VEC_SIZE(%rsi, %rdx), %rsi +- vmovdqu (%rsi), %ymm2 +- VPCMPEQ (%rdi), %ymm2, %ymm2 + vpmovmskb %ymm2, %eax +- subl $VEC_MASK, %eax +- jnz L(first_vec) ++ incl %eax ++ jnz L(return_vec_1) ++ ++ vpmovmskb %ymm3, %eax ++ incl %eax ++ jnz L(return_vec_2) ++L(return_vec_3): ++ tzcntl %ecx, %ecx ++# ifdef USE_AS_WMEMCMP ++ movl (VEC_SIZE * 3)(%rdi, %rcx), %eax ++ xorl %edx, %edx ++ cmpl (VEC_SIZE * 3)(%rsi, %rcx), %eax ++ setg %dl ++ leal -1(%rdx, %rdx), %eax ++# else ++ movzbl (VEC_SIZE * 3)(%rdi, %rcx), %eax ++ movzbl (VEC_SIZE * 3)(%rsi, %rcx), %ecx ++ subl %ecx, %eax ++# endif ++ VZEROUPPER_RETURN ++ ++ .p2align 4 ++L(more_8x_vec): ++ /* Set end of s1 in rdx. */ ++ leaq -(VEC_SIZE * 4)(%rdi, %rdx), %rdx ++ /* rsi stores s2 - s1. This allows loop to only update one ++ pointer. */ ++ subq %rdi, %rsi ++ /* Align s1 pointer. */ ++ andq $-VEC_SIZE, %rdi ++ /* Adjust because first 4x vec where check already. */ ++ subq $-(VEC_SIZE * 4), %rdi ++ .p2align 4 ++L(loop_4x_vec): ++ /* rsi has s2 - s1 so get correct address by adding s1 (in rdi). ++ */ ++ vmovdqu (%rsi, %rdi), %ymm1 ++ VPCMPEQ (%rdi), %ymm1, %ymm1 ++ ++ vmovdqu VEC_SIZE(%rsi, %rdi), %ymm2 ++ VPCMPEQ VEC_SIZE(%rdi), %ymm2, %ymm2 ++ ++ vmovdqu (VEC_SIZE * 2)(%rsi, %rdi), %ymm3 ++ VPCMPEQ (VEC_SIZE * 2)(%rdi), %ymm3, %ymm3 ++ ++ vmovdqu (VEC_SIZE * 3)(%rsi, %rdi), %ymm4 ++ VPCMPEQ (VEC_SIZE * 3)(%rdi), %ymm4, %ymm4 ++ ++ vpand %ymm1, %ymm2, %ymm5 ++ vpand %ymm3, %ymm4, %ymm6 ++ vpand %ymm5, %ymm6, %ymm7 ++ vpmovmskb %ymm7, %ecx ++ incl %ecx ++ jnz L(8x_return_vec_0_1_2_3) ++ subq $-(VEC_SIZE * 4), %rdi ++ /* Check if s1 pointer at end. */ ++ cmpq %rdx, %rdi ++ jb L(loop_4x_vec) ++ ++ subq %rdx, %rdi ++ /* rdi has 4 * VEC_SIZE - remaining length. */ ++ cmpl $(VEC_SIZE * 3), %edi ++ jae L(8x_last_1x_vec) ++ /* Load regardless of branch. */ ++ vmovdqu (VEC_SIZE * 2)(%rsi, %rdx), %ymm3 ++ cmpl $(VEC_SIZE * 2), %edi ++ jae L(8x_last_2x_vec) ++ ++ /* Check last 4 VEC. */ ++ vmovdqu (%rsi, %rdx), %ymm1 ++ VPCMPEQ (%rdx), %ymm1, %ymm1 ++ ++ vmovdqu VEC_SIZE(%rsi, %rdx), %ymm2 ++ VPCMPEQ VEC_SIZE(%rdx), %ymm2, %ymm2 ++ ++ VPCMPEQ (VEC_SIZE * 2)(%rdx), %ymm3, %ymm3 ++ ++ vmovdqu (VEC_SIZE * 3)(%rsi, %rdx), %ymm4 ++ VPCMPEQ (VEC_SIZE * 3)(%rdx), %ymm4, %ymm4 ++ ++ vpand %ymm1, %ymm2, %ymm5 ++ vpand %ymm3, %ymm4, %ymm6 ++ vpand %ymm5, %ymm6, %ymm7 ++ vpmovmskb %ymm7, %ecx ++ /* Restore s1 pointer to rdi. */ ++ movq %rdx, %rdi ++ incl %ecx ++ jnz L(8x_return_vec_0_1_2_3) ++ /* NB: eax must be zero to reach here. */ ++ VZEROUPPER_RETURN ++ ++ /* Only entry is from L(more_8x_vec). */ ++ .p2align 4 ++L(8x_last_2x_vec): ++ /* Check second to last VEC. rdx store end pointer of s1 and ++ ymm3 has already been loaded with second to last VEC from s2. ++ */ ++ VPCMPEQ (VEC_SIZE * 2)(%rdx), %ymm3, %ymm3 ++ vpmovmskb %ymm3, %eax ++ incl %eax ++ jnz L(8x_return_vec_2) ++ /* Check last VEC. */ ++ .p2align 4 ++L(8x_last_1x_vec): ++ vmovdqu (VEC_SIZE * 3)(%rsi, %rdx), %ymm4 ++ VPCMPEQ (VEC_SIZE * 3)(%rdx), %ymm4, %ymm4 ++ vpmovmskb %ymm4, %eax ++ incl %eax ++ jnz L(8x_return_vec_3) + VZEROUPPER_RETURN + + .p2align 4 +-L(first_vec): +- /* A byte or int32 is different within 16 or 32 bytes. */ +- tzcntl %eax, %ecx ++L(last_2x_vec): ++ /* Check second to last VEC. */ ++ vmovdqu -(VEC_SIZE * 2)(%rsi, %rdx), %ymm1 ++ VPCMPEQ -(VEC_SIZE * 2)(%rdi, %rdx), %ymm1, %ymm1 ++ vpmovmskb %ymm1, %eax ++ incl %eax ++ jnz L(return_vec_1_end) ++ /* Check last VEC. */ ++L(last_1x_vec): ++ vmovdqu -(VEC_SIZE * 1)(%rsi, %rdx), %ymm1 ++ VPCMPEQ -(VEC_SIZE * 1)(%rdi, %rdx), %ymm1, %ymm1 ++ vpmovmskb %ymm1, %eax ++ incl %eax ++ jnz L(return_vec_0_end) ++ VZEROUPPER_RETURN ++ ++ .p2align 4 ++L(8x_return_vec_2): ++ subq $VEC_SIZE, %rdx ++L(8x_return_vec_3): ++ tzcntl %eax, %eax ++ addq %rdx, %rax + # ifdef USE_AS_WMEMCMP +- xorl %eax, %eax +- movl (%rdi, %rcx), %edx +- cmpl (%rsi, %rcx), %edx +-L(wmemcmp_return): +- setl %al +- negl %eax +- orl $1, %eax ++ movl (VEC_SIZE * 3)(%rax), %ecx ++ xorl %edx, %edx ++ cmpl (VEC_SIZE * 3)(%rsi, %rax), %ecx ++ setg %dl ++ leal -1(%rdx, %rdx), %eax + # else +- movzbl (%rdi, %rcx), %eax +- movzbl (%rsi, %rcx), %edx +- sub %edx, %eax ++ movzbl (VEC_SIZE * 3)(%rsi, %rax), %ecx ++ movzbl (VEC_SIZE * 3)(%rax), %eax ++ subl %ecx, %eax + # endif + VZEROUPPER_RETURN + +-# ifdef USE_AS_WMEMCMP + .p2align 4 +-L(4): +- xorl %eax, %eax +- movl (%rdi), %edx +- cmpl (%rsi), %edx +- jne L(wmemcmp_return) +- ret ++L(return_vec_1_end): ++ tzcntl %eax, %eax ++ addl %edx, %eax ++# ifdef USE_AS_WMEMCMP ++ movl -(VEC_SIZE * 2)(%rdi, %rax), %ecx ++ xorl %edx, %edx ++ cmpl -(VEC_SIZE * 2)(%rsi, %rax), %ecx ++ setg %dl ++ leal -1(%rdx, %rdx), %eax + # else ++ movzbl -(VEC_SIZE * 2)(%rsi, %rax), %ecx ++ movzbl -(VEC_SIZE * 2)(%rdi, %rax), %eax ++ subl %ecx, %eax ++# endif ++ VZEROUPPER_RETURN ++ + .p2align 4 +-L(between_4_7): +- /* Load as big endian with overlapping movbe to avoid branches. */ +- movbe (%rdi), %eax +- movbe (%rsi), %ecx +- shlq $32, %rax +- shlq $32, %rcx +- movbe -4(%rdi, %rdx), %edi +- movbe -4(%rsi, %rdx), %esi +- orq %rdi, %rax +- orq %rsi, %rcx +- subq %rcx, %rax +- je L(exit) +- sbbl %eax, %eax +- orl $1, %eax +- ret ++L(return_vec_0_end): ++ tzcntl %eax, %eax ++ addl %edx, %eax ++# ifdef USE_AS_WMEMCMP ++ movl -VEC_SIZE(%rdi, %rax), %ecx ++ xorl %edx, %edx ++ cmpl -VEC_SIZE(%rsi, %rax), %ecx ++ setg %dl ++ leal -1(%rdx, %rdx), %eax ++# else ++ movzbl -VEC_SIZE(%rsi, %rax), %ecx ++ movzbl -VEC_SIZE(%rdi, %rax), %eax ++ subl %ecx, %eax ++# endif ++ VZEROUPPER_RETURN + + .p2align 4 +-L(exit): +- ret ++L(less_vec): ++ /* Check if one or less CHAR. This is necessary for size = 0 but ++ is also faster for size = CHAR_SIZE. */ ++ cmpl $CHAR_SIZE, %edx ++ jbe L(one_or_less) ++ ++ /* Check if loading one VEC from either s1 or s2 could cause a ++ page cross. This can have false positives but is by far the ++ fastest method. */ ++ movl %edi, %eax ++ orl %esi, %eax ++ andl $(PAGE_SIZE - 1), %eax ++ cmpl $(PAGE_SIZE - VEC_SIZE), %eax ++ jg L(page_cross_less_vec) ++ ++ /* No page cross possible. */ ++ vmovdqu (%rsi), %ymm2 ++ VPCMPEQ (%rdi), %ymm2, %ymm2 ++ vpmovmskb %ymm2, %eax ++ incl %eax ++ /* Result will be zero if s1 and s2 match. Otherwise first set ++ bit will be first mismatch. */ ++ bzhil %edx, %eax, %edx ++ jnz L(return_vec_0) ++ xorl %eax, %eax ++ VZEROUPPER_RETURN + + .p2align 4 +-L(between_2_3): ++L(page_cross_less_vec): ++ /* if USE_AS_WMEMCMP it can only be 0, 4, 8, 12, 16, 20, 24, 28 ++ bytes. */ ++ cmpl $16, %edx ++ jae L(between_16_31) ++# ifndef USE_AS_WMEMCMP ++ cmpl $8, %edx ++ jae L(between_8_15) ++ cmpl $4, %edx ++ jae L(between_4_7) ++ + /* Load as big endian to avoid branches. */ + movzwl (%rdi), %eax + movzwl (%rsi), %ecx +@@ -208,223 +439,106 @@ L(between_2_3): + shll $8, %ecx + bswap %eax + bswap %ecx +- movb -1(%rdi, %rdx), %al +- movb -1(%rsi, %rdx), %cl ++ movzbl -1(%rdi, %rdx), %edi ++ movzbl -1(%rsi, %rdx), %esi ++ orl %edi, %eax ++ orl %esi, %ecx + /* Subtraction is okay because the upper 8 bits are zero. */ + subl %ecx, %eax ++ /* No ymm register was touched. */ + ret + + .p2align 4 +-L(1): +- movzbl (%rdi), %eax ++L(one_or_less): ++ jb L(zero) + movzbl (%rsi), %ecx ++ movzbl (%rdi), %eax + subl %ecx, %eax +- ret +-# endif +- +- .p2align 4 +-L(zero): +- xorl %eax, %eax ++ /* No ymm register was touched. */ + ret + + .p2align 4 +-L(less_vec): +-# ifdef USE_AS_WMEMCMP +- /* It can only be 0, 4, 8, 12, 16, 20, 24, 28 bytes. */ +- cmpb $4, %dl +- je L(4) +- jb L(zero) +-# else +- cmpb $1, %dl +- je L(1) +- jb L(zero) +- cmpb $4, %dl +- jb L(between_2_3) +- cmpb $8, %dl +- jb L(between_4_7) ++L(between_8_15): + # endif +- cmpb $16, %dl +- jae L(between_16_31) +- /* It is between 8 and 15 bytes. */ ++ /* If USE_AS_WMEMCMP fall through into 8-15 byte case. */ + vmovq (%rdi), %xmm1 + vmovq (%rsi), %xmm2 +- VPCMPEQ %xmm1, %xmm2, %xmm2 ++ VPCMPEQ %xmm1, %xmm2, %xmm2 + vpmovmskb %xmm2, %eax +- subl $0xffff, %eax +- jnz L(first_vec) ++ subl $0xffff, %eax ++ jnz L(return_vec_0) + /* Use overlapping loads to avoid branches. */ + leaq -8(%rdi, %rdx), %rdi + leaq -8(%rsi, %rdx), %rsi + vmovq (%rdi), %xmm1 + vmovq (%rsi), %xmm2 +- VPCMPEQ %xmm1, %xmm2, %xmm2 ++ VPCMPEQ %xmm1, %xmm2, %xmm2 + vpmovmskb %xmm2, %eax +- subl $0xffff, %eax +- jnz L(first_vec) ++ subl $0xffff, %eax ++ jnz L(return_vec_0) ++ /* No ymm register was touched. */ ++ ret ++ ++ .p2align 4 ++L(zero): ++ xorl %eax, %eax + ret + + .p2align 4 + L(between_16_31): + /* From 16 to 31 bytes. No branch when size == 16. */ + vmovdqu (%rsi), %xmm2 +- VPCMPEQ (%rdi), %xmm2, %xmm2 ++ VPCMPEQ (%rdi), %xmm2, %xmm2 + vpmovmskb %xmm2, %eax +- subl $0xffff, %eax +- jnz L(first_vec) ++ subl $0xffff, %eax ++ jnz L(return_vec_0) + + /* Use overlapping loads to avoid branches. */ ++ ++ vmovdqu -16(%rsi, %rdx), %xmm2 + leaq -16(%rdi, %rdx), %rdi + leaq -16(%rsi, %rdx), %rsi +- vmovdqu (%rsi), %xmm2 +- VPCMPEQ (%rdi), %xmm2, %xmm2 ++ VPCMPEQ (%rdi), %xmm2, %xmm2 + vpmovmskb %xmm2, %eax +- subl $0xffff, %eax +- jnz L(first_vec) ++ subl $0xffff, %eax ++ jnz L(return_vec_0) ++ /* No ymm register was touched. */ + ret + +- .p2align 4 +-L(more_8x_vec): +- /* More than 8 * VEC. Check the first VEC. */ +- vmovdqu (%rsi), %ymm2 +- VPCMPEQ (%rdi), %ymm2, %ymm2 +- vpmovmskb %ymm2, %eax +- subl $VEC_MASK, %eax +- jnz L(first_vec) +- +- /* Align the first memory area for aligned loads in the loop. +- Compute how much the first memory area is misaligned. */ +- movq %rdi, %rcx +- andl $(VEC_SIZE - 1), %ecx +- /* Get the negative of offset for alignment. */ +- subq $VEC_SIZE, %rcx +- /* Adjust the second memory area. */ +- subq %rcx, %rsi +- /* Adjust the first memory area which should be aligned now. */ +- subq %rcx, %rdi +- /* Adjust length. */ +- addq %rcx, %rdx +- +-L(loop_4x_vec): +- /* Compare 4 * VEC at a time forward. */ +- vmovdqu (%rsi), %ymm1 +- VPCMPEQ (%rdi), %ymm1, %ymm1 +- +- vmovdqu VEC_SIZE(%rsi), %ymm2 +- VPCMPEQ VEC_SIZE(%rdi), %ymm2, %ymm2 +- vpand %ymm2, %ymm1, %ymm5 +- +- vmovdqu (VEC_SIZE * 2)(%rsi), %ymm3 +- VPCMPEQ (VEC_SIZE * 2)(%rdi), %ymm3, %ymm3 +- vpand %ymm3, %ymm5, %ymm5 +- +- vmovdqu (VEC_SIZE * 3)(%rsi), %ymm4 +- VPCMPEQ (VEC_SIZE * 3)(%rdi), %ymm4, %ymm4 +- vpand %ymm4, %ymm5, %ymm5 +- +- vptest %ymm0, %ymm5 +- jnc L(4x_vec_end) +- +- addq $(VEC_SIZE * 4), %rdi +- addq $(VEC_SIZE * 4), %rsi +- +- subq $(VEC_SIZE * 4), %rdx +- cmpq $(VEC_SIZE * 4), %rdx +- jae L(loop_4x_vec) +- +- /* Less than 4 * VEC. */ +- cmpq $VEC_SIZE, %rdx +- jbe L(last_vec) +- cmpq $(VEC_SIZE * 2), %rdx +- jbe L(last_2x_vec) +- +-L(last_4x_vec): +- /* From 2 * VEC to 4 * VEC. */ +- vmovdqu (%rsi), %ymm2 +- VPCMPEQ (%rdi), %ymm2, %ymm2 +- vpmovmskb %ymm2, %eax +- subl $VEC_MASK, %eax +- jnz L(first_vec) +- +- addq $VEC_SIZE, %rdi +- addq $VEC_SIZE, %rsi +- vmovdqu (%rsi), %ymm2 +- VPCMPEQ (%rdi), %ymm2, %ymm2 +- vpmovmskb %ymm2, %eax +- subl $VEC_MASK, %eax +- jnz L(first_vec) +- +- /* Use overlapping loads to avoid branches. */ +- leaq -(3 * VEC_SIZE)(%rdi, %rdx), %rdi +- leaq -(3 * VEC_SIZE)(%rsi, %rdx), %rsi +- vmovdqu (%rsi), %ymm2 +- VPCMPEQ (%rdi), %ymm2, %ymm2 +- vpmovmskb %ymm2, %eax +- subl $VEC_MASK, %eax +- jnz L(first_vec) +- +- addq $VEC_SIZE, %rdi +- addq $VEC_SIZE, %rsi +- vmovdqu (%rsi), %ymm2 +- VPCMPEQ (%rdi), %ymm2, %ymm2 +- vpmovmskb %ymm2, %eax +- subl $VEC_MASK, %eax +- jnz L(first_vec) +- VZEROUPPER_RETURN +- +- .p2align 4 +-L(4x_vec_end): +- vpmovmskb %ymm1, %eax +- subl $VEC_MASK, %eax +- jnz L(first_vec) +- vpmovmskb %ymm2, %eax +- subl $VEC_MASK, %eax +- jnz L(first_vec_x1) +- vpmovmskb %ymm3, %eax +- subl $VEC_MASK, %eax +- jnz L(first_vec_x2) +- vpmovmskb %ymm4, %eax +- subl $VEC_MASK, %eax +- tzcntl %eax, %ecx + # ifdef USE_AS_WMEMCMP +- xorl %eax, %eax +- movl (VEC_SIZE * 3)(%rdi, %rcx), %edx +- cmpl (VEC_SIZE * 3)(%rsi, %rcx), %edx +- jmp L(wmemcmp_return) +-# else +- movzbl (VEC_SIZE * 3)(%rdi, %rcx), %eax +- movzbl (VEC_SIZE * 3)(%rsi, %rcx), %edx +- sub %edx, %eax +-# endif +- VZEROUPPER_RETURN +- + .p2align 4 +-L(first_vec_x1): +- tzcntl %eax, %ecx +-# ifdef USE_AS_WMEMCMP +- xorl %eax, %eax +- movl VEC_SIZE(%rdi, %rcx), %edx +- cmpl VEC_SIZE(%rsi, %rcx), %edx +- jmp L(wmemcmp_return) ++L(one_or_less): ++ jb L(zero) ++ movl (%rdi), %ecx ++ xorl %edx, %edx ++ cmpl (%rsi), %ecx ++ je L(zero) ++ setg %dl ++ leal -1(%rdx, %rdx), %eax ++ /* No ymm register was touched. */ ++ ret + # else +- movzbl VEC_SIZE(%rdi, %rcx), %eax +- movzbl VEC_SIZE(%rsi, %rcx), %edx +- sub %edx, %eax +-# endif +- VZEROUPPER_RETURN + + .p2align 4 +-L(first_vec_x2): +- tzcntl %eax, %ecx +-# ifdef USE_AS_WMEMCMP +- xorl %eax, %eax +- movl (VEC_SIZE * 2)(%rdi, %rcx), %edx +- cmpl (VEC_SIZE * 2)(%rsi, %rcx), %edx +- jmp L(wmemcmp_return) +-# else +- movzbl (VEC_SIZE * 2)(%rdi, %rcx), %eax +- movzbl (VEC_SIZE * 2)(%rsi, %rcx), %edx +- sub %edx, %eax ++L(between_4_7): ++ /* Load as big endian with overlapping movbe to avoid branches. ++ */ ++ movbe (%rdi), %eax ++ movbe (%rsi), %ecx ++ shlq $32, %rax ++ shlq $32, %rcx ++ movbe -4(%rdi, %rdx), %edi ++ movbe -4(%rsi, %rdx), %esi ++ orq %rdi, %rax ++ orq %rsi, %rcx ++ subq %rcx, %rax ++ jz L(zero_4_7) ++ sbbl %eax, %eax ++ orl $1, %eax ++L(zero_4_7): ++ /* No ymm register was touched. */ ++ ret + # endif +- VZEROUPPER_RETURN ++ + END (MEMCMP) + #endif +-- +GitLab + diff --git a/SOURCES/ia-opt-memcmp-evex-movbe-1.patch b/SOURCES/ia-opt-memcmp-evex-movbe-1.patch new file mode 100644 index 0000000..0216ed0 --- /dev/null +++ b/SOURCES/ia-opt-memcmp-evex-movbe-1.patch @@ -0,0 +1,851 @@ +From 2d8eaea7ad74328d806d3f1a377f1168eaa2f348 Mon Sep 17 00:00:00 2001 +From: Noah Goldstein +Date: Mon, 17 May 2021 13:57:24 -0400 +Subject: [PATCH] x86: Optimize memcmp-evex-movbe.S + +No bug. This commit optimizes memcmp-evex.S. The optimizations include +adding a new vec compare path for small sizes, reorganizing the entry +control flow, removing some unnecissary ALU instructions from the main +loop, and most importantly replacing the heavy use of vpcmp + kand +logic with vpxor + vptern. test-memcmp and test-wmemcmp are both +passing. + +Signed-off-by: Noah Goldstein +Reviewed-by: H.J. Lu +(cherry picked from commit 4ad473e97acdc5f6d811755b67c09f2128a644ce) +--- + sysdeps/x86_64/multiarch/memcmp-evex-movbe.S | 710 +++++++++++-------- + 1 file changed, 408 insertions(+), 302 deletions(-) + +diff --git a/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S b/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S +index 9c093972..654dc7ac 100644 +--- a/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S ++++ b/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S +@@ -19,17 +19,22 @@ + #if IS_IN (libc) + + /* memcmp/wmemcmp is implemented as: +- 1. For size from 2 to 7 bytes, load as big endian with movbe and bswap +- to avoid branches. +- 2. Use overlapping compare to avoid branch. +- 3. Use vector compare when size >= 4 bytes for memcmp or size >= 8 +- bytes for wmemcmp. +- 4. If size is 8 * VEC_SIZE or less, unroll the loop. +- 5. Compare 4 * VEC_SIZE at a time with the aligned first memory ++ 1. Use ymm vector compares when possible. The only case where ++ vector compares is not possible for when size < CHAR_PER_VEC ++ and loading from either s1 or s2 would cause a page cross. ++ 2. For size from 2 to 7 bytes on page cross, load as big endian ++ with movbe and bswap to avoid branches. ++ 3. Use xmm vector compare when size >= 4 bytes for memcmp or ++ size >= 8 bytes for wmemcmp. ++ 4. Optimistically compare up to first 4 * CHAR_PER_VEC one at a ++ to check for early mismatches. Only do this if its guranteed the ++ work is not wasted. ++ 5. If size is 8 * VEC_SIZE or less, unroll the loop. ++ 6. Compare 4 * VEC_SIZE at a time with the aligned first memory + area. +- 6. Use 2 vector compares when size is 2 * VEC_SIZE or less. +- 7. Use 4 vector compares when size is 4 * VEC_SIZE or less. +- 8. Use 8 vector compares when size is 8 * VEC_SIZE or less. */ ++ 7. Use 2 vector compares when size is 2 * CHAR_PER_VEC or less. ++ 8. Use 4 vector compares when size is 4 * CHAR_PER_VEC or less. ++ 9. Use 8 vector compares when size is 8 * CHAR_PER_VEC or less. */ + + # include + +@@ -40,11 +45,21 @@ + # define VMOVU vmovdqu64 + + # ifdef USE_AS_WMEMCMP +-# define VPCMPEQ vpcmpeqd ++# define CHAR_SIZE 4 ++# define VPCMP vpcmpd + # else +-# define VPCMPEQ vpcmpeqb ++# define CHAR_SIZE 1 ++# define VPCMP vpcmpub + # endif + ++# define VEC_SIZE 32 ++# define PAGE_SIZE 4096 ++# define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE) ++ ++# define XMM0 xmm16 ++# define XMM1 xmm17 ++# define XMM2 xmm18 ++# define YMM0 ymm16 + # define XMM1 xmm17 + # define XMM2 xmm18 + # define YMM1 ymm17 +@@ -54,15 +69,6 @@ + # define YMM5 ymm21 + # define YMM6 ymm22 + +-# define VEC_SIZE 32 +-# ifdef USE_AS_WMEMCMP +-# define VEC_MASK 0xff +-# define XMM_MASK 0xf +-# else +-# define VEC_MASK 0xffffffff +-# define XMM_MASK 0xffff +-# endif +- + /* Warning! + wmemcmp has to use SIGNED comparison for elements. + memcmp has to use UNSIGNED comparison for elemnts. +@@ -70,145 +76,370 @@ + + .section .text.evex,"ax",@progbits + ENTRY (MEMCMP) +-# ifdef USE_AS_WMEMCMP +- shl $2, %RDX_LP +-# elif defined __ILP32__ ++# ifdef __ILP32__ + /* Clear the upper 32 bits. */ + movl %edx, %edx + # endif +- cmp $VEC_SIZE, %RDX_LP ++ cmp $CHAR_PER_VEC, %RDX_LP + jb L(less_vec) + + /* From VEC to 2 * VEC. No branch when size == VEC_SIZE. */ +- VMOVU (%rsi), %YMM2 +- VPCMPEQ (%rdi), %YMM2, %k1 ++ VMOVU (%rsi), %YMM1 ++ /* Use compare not equals to directly check for mismatch. */ ++ VPCMP $4, (%rdi), %YMM1, %k1 + kmovd %k1, %eax +- subl $VEC_MASK, %eax +- jnz L(first_vec) +- +- cmpq $(VEC_SIZE * 2), %rdx +- jbe L(last_vec) +- +- /* More than 2 * VEC. */ +- cmpq $(VEC_SIZE * 8), %rdx +- ja L(more_8x_vec) +- cmpq $(VEC_SIZE * 4), %rdx +- jb L(last_4x_vec) ++ /* NB: eax must be destination register if going to ++ L(return_vec_[0,2]). For L(return_vec_3 destination register ++ must be ecx. */ ++ testl %eax, %eax ++ jnz L(return_vec_0) + +- /* From 4 * VEC to 8 * VEC, inclusively. */ +- VMOVU (%rsi), %YMM1 +- VPCMPEQ (%rdi), %YMM1, %k1 ++ cmpq $(CHAR_PER_VEC * 2), %rdx ++ jbe L(last_1x_vec) + ++ /* Check second VEC no matter what. */ + VMOVU VEC_SIZE(%rsi), %YMM2 +- VPCMPEQ VEC_SIZE(%rdi), %YMM2, %k2 ++ VPCMP $4, VEC_SIZE(%rdi), %YMM2, %k1 ++ kmovd %k1, %eax ++ testl %eax, %eax ++ jnz L(return_vec_1) ++ ++ /* Less than 4 * VEC. */ ++ cmpq $(CHAR_PER_VEC * 4), %rdx ++ jbe L(last_2x_vec) + ++ /* Check third and fourth VEC no matter what. */ + VMOVU (VEC_SIZE * 2)(%rsi), %YMM3 +- VPCMPEQ (VEC_SIZE * 2)(%rdi), %YMM3, %k3 ++ VPCMP $4, (VEC_SIZE * 2)(%rdi), %YMM3, %k1 ++ kmovd %k1, %eax ++ testl %eax, %eax ++ jnz L(return_vec_2) + + VMOVU (VEC_SIZE * 3)(%rsi), %YMM4 +- VPCMPEQ (VEC_SIZE * 3)(%rdi), %YMM4, %k4 ++ VPCMP $4, (VEC_SIZE * 3)(%rdi), %YMM4, %k1 ++ kmovd %k1, %ecx ++ testl %ecx, %ecx ++ jnz L(return_vec_3) + +- kandd %k1, %k2, %k5 +- kandd %k3, %k4, %k6 +- kandd %k5, %k6, %k6 ++ /* Zero YMM0. 4x VEC reduction is done with vpxor + vtern so ++ compare with zero to get a mask is needed. */ ++ vpxorq %XMM0, %XMM0, %XMM0 + +- kmovd %k6, %eax +- cmpl $VEC_MASK, %eax +- jne L(4x_vec_end) ++ /* Go to 4x VEC loop. */ ++ cmpq $(CHAR_PER_VEC * 8), %rdx ++ ja L(more_8x_vec) + +- leaq -(4 * VEC_SIZE)(%rdi, %rdx), %rdi +- leaq -(4 * VEC_SIZE)(%rsi, %rdx), %rsi +- VMOVU (%rsi), %YMM1 +- VPCMPEQ (%rdi), %YMM1, %k1 ++ /* Handle remainder of size = 4 * VEC + 1 to 8 * VEC without any ++ branches. */ + +- VMOVU VEC_SIZE(%rsi), %YMM2 +- VPCMPEQ VEC_SIZE(%rdi), %YMM2, %k2 +- kandd %k1, %k2, %k5 ++ /* Load first two VEC from s2 before adjusting addresses. */ ++ VMOVU -(VEC_SIZE * 4)(%rsi, %rdx, CHAR_SIZE), %YMM1 ++ VMOVU -(VEC_SIZE * 3)(%rsi, %rdx, CHAR_SIZE), %YMM2 ++ leaq -(4 * VEC_SIZE)(%rdi, %rdx, CHAR_SIZE), %rdi ++ leaq -(4 * VEC_SIZE)(%rsi, %rdx, CHAR_SIZE), %rsi ++ ++ /* Wait to load from s1 until addressed adjust due to ++ unlamination of microfusion with complex address mode. */ ++ ++ /* vpxor will be all 0s if s1 and s2 are equal. Otherwise it ++ will have some 1s. */ ++ vpxorq (%rdi), %YMM1, %YMM1 ++ vpxorq (VEC_SIZE)(%rdi), %YMM2, %YMM2 + + VMOVU (VEC_SIZE * 2)(%rsi), %YMM3 +- VPCMPEQ (VEC_SIZE * 2)(%rdi), %YMM3, %k3 +- kandd %k3, %k5, %k5 ++ vpxorq (VEC_SIZE * 2)(%rdi), %YMM3, %YMM3 ++ /* Or together YMM1, YMM2, and YMM3 into YMM3. */ ++ vpternlogd $0xfe, %YMM1, %YMM2, %YMM3 + + VMOVU (VEC_SIZE * 3)(%rsi), %YMM4 +- VPCMPEQ (VEC_SIZE * 3)(%rdi), %YMM4, %k4 +- kandd %k4, %k5, %k5 ++ /* Ternary logic to xor (VEC_SIZE * 3)(%rdi) with YMM4 while ++ oring with YMM3. Result is stored in YMM4. */ ++ vpternlogd $0xde, (VEC_SIZE * 3)(%rdi), %YMM3, %YMM4 ++ /* Compare YMM4 with 0. If any 1s s1 and s2 don't match. */ ++ VPCMP $4, %YMM4, %YMM0, %k1 ++ kmovd %k1, %ecx ++ testl %ecx, %ecx ++ jnz L(return_vec_0_1_2_3) ++ /* NB: eax must be zero to reach here. */ ++ ret + +- kmovd %k5, %eax +- cmpl $VEC_MASK, %eax +- jne L(4x_vec_end) +- xorl %eax, %eax ++ /* NB: aligning 32 here allows for the rest of the jump targets ++ to be tuned for 32 byte alignment. Most important this ensures ++ the L(more_8x_vec) loop is 32 byte aligned. */ ++ .p2align 5 ++L(less_vec): ++ /* Check if one or less CHAR. This is necessary for size = 0 but ++ is also faster for size = CHAR_SIZE. */ ++ cmpl $1, %edx ++ jbe L(one_or_less) ++ ++ /* Check if loading one VEC from either s1 or s2 could cause a ++ page cross. This can have false positives but is by far the ++ fastest method. */ ++ movl %edi, %eax ++ orl %esi, %eax ++ andl $(PAGE_SIZE - 1), %eax ++ cmpl $(PAGE_SIZE - VEC_SIZE), %eax ++ jg L(page_cross_less_vec) ++ ++ /* No page cross possible. */ ++ VMOVU (%rsi), %YMM2 ++ VPCMP $4, (%rdi), %YMM2, %k1 ++ kmovd %k1, %eax ++ /* Create mask in ecx for potentially in bound matches. */ ++ bzhil %edx, %eax, %eax ++ jnz L(return_vec_0) + ret + + .p2align 4 +-L(last_2x_vec): +- /* From VEC to 2 * VEC. No branch when size == VEC_SIZE. */ +- VMOVU (%rsi), %YMM2 +- VPCMPEQ (%rdi), %YMM2, %k2 +- kmovd %k2, %eax +- subl $VEC_MASK, %eax +- jnz L(first_vec) ++L(return_vec_0): ++ tzcntl %eax, %eax ++# ifdef USE_AS_WMEMCMP ++ movl (%rdi, %rax, CHAR_SIZE), %ecx ++ xorl %edx, %edx ++ cmpl (%rsi, %rax, CHAR_SIZE), %ecx ++ /* NB: no partial register stall here because xorl zero idiom ++ above. */ ++ setg %dl ++ leal -1(%rdx, %rdx), %eax ++# else ++ movzbl (%rsi, %rax), %ecx ++ movzbl (%rdi, %rax), %eax ++ subl %ecx, %eax ++# endif ++ ret + +-L(last_vec): +- /* Use overlapping loads to avoid branches. */ +- leaq -VEC_SIZE(%rdi, %rdx), %rdi +- leaq -VEC_SIZE(%rsi, %rdx), %rsi +- VMOVU (%rsi), %YMM2 +- VPCMPEQ (%rdi), %YMM2, %k2 +- kmovd %k2, %eax +- subl $VEC_MASK, %eax +- jnz L(first_vec) ++ /* NB: No p2align necessary. Alignment % 16 is naturally 1 ++ which is good enough for a target not in a loop. */ ++L(return_vec_1): ++ tzcntl %eax, %eax ++# ifdef USE_AS_WMEMCMP ++ movl VEC_SIZE(%rdi, %rax, CHAR_SIZE), %ecx ++ xorl %edx, %edx ++ cmpl VEC_SIZE(%rsi, %rax, CHAR_SIZE), %ecx ++ setg %dl ++ leal -1(%rdx, %rdx), %eax ++# else ++ movzbl VEC_SIZE(%rsi, %rax), %ecx ++ movzbl VEC_SIZE(%rdi, %rax), %eax ++ subl %ecx, %eax ++# endif + ret + +- .p2align 4 +-L(first_vec): +- /* A byte or int32 is different within 16 or 32 bytes. */ +- tzcntl %eax, %ecx ++ /* NB: No p2align necessary. Alignment % 16 is naturally 2 ++ which is good enough for a target not in a loop. */ ++L(return_vec_2): ++ tzcntl %eax, %eax + # ifdef USE_AS_WMEMCMP +- xorl %eax, %eax +- movl (%rdi, %rcx, 4), %edx +- cmpl (%rsi, %rcx, 4), %edx +-L(wmemcmp_return): +- setl %al +- negl %eax +- orl $1, %eax ++ movl (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %ecx ++ xorl %edx, %edx ++ cmpl (VEC_SIZE * 2)(%rsi, %rax, CHAR_SIZE), %ecx ++ setg %dl ++ leal -1(%rdx, %rdx), %eax + # else +- movzbl (%rdi, %rcx), %eax +- movzbl (%rsi, %rcx), %edx +- sub %edx, %eax ++ movzbl (VEC_SIZE * 2)(%rsi, %rax), %ecx ++ movzbl (VEC_SIZE * 2)(%rdi, %rax), %eax ++ subl %ecx, %eax + # endif + ret + ++ .p2align 4 ++L(8x_return_vec_0_1_2_3): ++ /* Returning from L(more_8x_vec) requires restoring rsi. */ ++ addq %rdi, %rsi ++L(return_vec_0_1_2_3): ++ VPCMP $4, %YMM1, %YMM0, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(return_vec_0) ++ ++ VPCMP $4, %YMM2, %YMM0, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(return_vec_1) ++ ++ VPCMP $4, %YMM3, %YMM0, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(return_vec_2) ++L(return_vec_3): ++ tzcntl %ecx, %ecx + # ifdef USE_AS_WMEMCMP ++ movl (VEC_SIZE * 3)(%rdi, %rcx, CHAR_SIZE), %eax ++ xorl %edx, %edx ++ cmpl (VEC_SIZE * 3)(%rsi, %rcx, CHAR_SIZE), %eax ++ setg %dl ++ leal -1(%rdx, %rdx), %eax ++# else ++ movzbl (VEC_SIZE * 3)(%rdi, %rcx), %eax ++ movzbl (VEC_SIZE * 3)(%rsi, %rcx), %ecx ++ subl %ecx, %eax ++# endif ++ ret ++ + .p2align 4 +-L(4): +- xorl %eax, %eax +- movl (%rdi), %edx +- cmpl (%rsi), %edx +- jne L(wmemcmp_return) ++L(more_8x_vec): ++ /* Set end of s1 in rdx. */ ++ leaq -(VEC_SIZE * 4)(%rdi, %rdx, CHAR_SIZE), %rdx ++ /* rsi stores s2 - s1. This allows loop to only update one ++ pointer. */ ++ subq %rdi, %rsi ++ /* Align s1 pointer. */ ++ andq $-VEC_SIZE, %rdi ++ /* Adjust because first 4x vec where check already. */ ++ subq $-(VEC_SIZE * 4), %rdi ++ .p2align 4 ++L(loop_4x_vec): ++ VMOVU (%rsi, %rdi), %YMM1 ++ vpxorq (%rdi), %YMM1, %YMM1 ++ ++ VMOVU VEC_SIZE(%rsi, %rdi), %YMM2 ++ vpxorq VEC_SIZE(%rdi), %YMM2, %YMM2 ++ ++ VMOVU (VEC_SIZE * 2)(%rsi, %rdi), %YMM3 ++ vpxorq (VEC_SIZE * 2)(%rdi), %YMM3, %YMM3 ++ vpternlogd $0xfe, %YMM1, %YMM2, %YMM3 ++ ++ VMOVU (VEC_SIZE * 3)(%rsi, %rdi), %YMM4 ++ vpternlogd $0xde, (VEC_SIZE * 3)(%rdi), %YMM3, %YMM4 ++ VPCMP $4, %YMM4, %YMM0, %k1 ++ kmovd %k1, %ecx ++ testl %ecx, %ecx ++ jnz L(8x_return_vec_0_1_2_3) ++ subq $-(VEC_SIZE * 4), %rdi ++ cmpq %rdx, %rdi ++ jb L(loop_4x_vec) ++ ++ subq %rdx, %rdi ++ /* rdi has 4 * VEC_SIZE - remaining length. */ ++ cmpl $(VEC_SIZE * 3), %edi ++ jae L(8x_last_1x_vec) ++ /* Load regardless of branch. */ ++ VMOVU (VEC_SIZE * 2)(%rsi, %rdx), %YMM3 ++ cmpl $(VEC_SIZE * 2), %edi ++ jae L(8x_last_2x_vec) ++ ++ VMOVU (%rsi, %rdx), %YMM1 ++ vpxorq (%rdx), %YMM1, %YMM1 ++ ++ VMOVU VEC_SIZE(%rsi, %rdx), %YMM2 ++ vpxorq VEC_SIZE(%rdx), %YMM2, %YMM2 ++ ++ vpxorq (VEC_SIZE * 2)(%rdx), %YMM3, %YMM3 ++ vpternlogd $0xfe, %YMM1, %YMM2, %YMM3 ++ ++ VMOVU (VEC_SIZE * 3)(%rsi, %rdx), %YMM4 ++ vpternlogd $0xde, (VEC_SIZE * 3)(%rdx), %YMM3, %YMM4 ++ VPCMP $4, %YMM4, %YMM0, %k1 ++ kmovd %k1, %ecx ++ /* Restore s1 pointer to rdi. */ ++ movq %rdx, %rdi ++ testl %ecx, %ecx ++ jnz L(8x_return_vec_0_1_2_3) ++ /* NB: eax must be zero to reach here. */ ++ ret ++ ++ /* Only entry is from L(more_8x_vec). */ ++ .p2align 4 ++L(8x_last_2x_vec): ++ VPCMP $4, (VEC_SIZE * 2)(%rdx), %YMM3, %k1 ++ kmovd %k1, %eax ++ testl %eax, %eax ++ jnz L(8x_return_vec_2) ++ /* Naturally aligned to 16 bytes. */ ++L(8x_last_1x_vec): ++ VMOVU (VEC_SIZE * 3)(%rsi, %rdx), %YMM1 ++ VPCMP $4, (VEC_SIZE * 3)(%rdx), %YMM1, %k1 ++ kmovd %k1, %eax ++ testl %eax, %eax ++ jnz L(8x_return_vec_3) ++ ret ++ ++ .p2align 4 ++L(last_2x_vec): ++ /* Check second to last VEC. */ ++ VMOVU -(VEC_SIZE * 2)(%rsi, %rdx, CHAR_SIZE), %YMM1 ++ VPCMP $4, -(VEC_SIZE * 2)(%rdi, %rdx, CHAR_SIZE), %YMM1, %k1 ++ kmovd %k1, %eax ++ testl %eax, %eax ++ jnz L(return_vec_1_end) ++ ++ /* Check last VEC. */ ++ .p2align 4 ++L(last_1x_vec): ++ VMOVU -(VEC_SIZE * 1)(%rsi, %rdx, CHAR_SIZE), %YMM1 ++ VPCMP $4, -(VEC_SIZE * 1)(%rdi, %rdx, CHAR_SIZE), %YMM1, %k1 ++ kmovd %k1, %eax ++ testl %eax, %eax ++ jnz L(return_vec_0_end) + ret ++ ++ .p2align 4 ++L(8x_return_vec_2): ++ subq $VEC_SIZE, %rdx ++L(8x_return_vec_3): ++ tzcntl %eax, %eax ++# ifdef USE_AS_WMEMCMP ++ leaq (%rdx, %rax, CHAR_SIZE), %rax ++ movl (VEC_SIZE * 3)(%rax), %ecx ++ xorl %edx, %edx ++ cmpl (VEC_SIZE * 3)(%rsi, %rax), %ecx ++ setg %dl ++ leal -1(%rdx, %rdx), %eax + # else ++ addq %rdx, %rax ++ movzbl (VEC_SIZE * 3)(%rsi, %rax), %ecx ++ movzbl (VEC_SIZE * 3)(%rax), %eax ++ subl %ecx, %eax ++# endif ++ ret ++ + .p2align 4 +-L(between_4_7): +- /* Load as big endian with overlapping movbe to avoid branches. */ +- movbe (%rdi), %eax +- movbe (%rsi), %ecx +- shlq $32, %rax +- shlq $32, %rcx +- movbe -4(%rdi, %rdx), %edi +- movbe -4(%rsi, %rdx), %esi +- orq %rdi, %rax +- orq %rsi, %rcx +- subq %rcx, %rax +- je L(exit) +- sbbl %eax, %eax +- orl $1, %eax ++L(return_vec_0_end): ++ tzcntl %eax, %eax ++ addl %edx, %eax ++# ifdef USE_AS_WMEMCMP ++ movl -VEC_SIZE(%rdi, %rax, CHAR_SIZE), %ecx ++ xorl %edx, %edx ++ cmpl -VEC_SIZE(%rsi, %rax, CHAR_SIZE), %ecx ++ setg %dl ++ leal -1(%rdx, %rdx), %eax ++# else ++ movzbl -VEC_SIZE(%rsi, %rax), %ecx ++ movzbl -VEC_SIZE(%rdi, %rax), %eax ++ subl %ecx, %eax ++# endif + ret + + .p2align 4 +-L(exit): ++L(return_vec_1_end): ++ tzcntl %eax, %eax ++ addl %edx, %eax ++# ifdef USE_AS_WMEMCMP ++ movl -(VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %ecx ++ xorl %edx, %edx ++ cmpl -(VEC_SIZE * 2)(%rsi, %rax, CHAR_SIZE), %ecx ++ setg %dl ++ leal -1(%rdx, %rdx), %eax ++# else ++ movzbl -(VEC_SIZE * 2)(%rsi, %rax), %ecx ++ movzbl -(VEC_SIZE * 2)(%rdi, %rax), %eax ++ subl %ecx, %eax ++# endif + ret + ++ + .p2align 4 ++L(page_cross_less_vec): ++ /* if USE_AS_WMEMCMP it can only be 0, 4, 8, 12, 16, 20, 24, 28 ++ bytes. */ ++ cmpl $(16 / CHAR_SIZE), %edx ++ jae L(between_16_31) ++# ifndef USE_AS_WMEMCMP ++ cmpl $8, %edx ++ jae L(between_8_15) ++ cmpl $4, %edx ++ jae L(between_4_7) + L(between_2_3): + /* Load as big endian to avoid branches. */ + movzwl (%rdi), %eax +@@ -217,224 +448,99 @@ L(between_2_3): + shll $8, %ecx + bswap %eax + bswap %ecx +- movb -1(%rdi, %rdx), %al +- movb -1(%rsi, %rdx), %cl ++ movzbl -1(%rdi, %rdx), %edi ++ movzbl -1(%rsi, %rdx), %esi ++ orl %edi, %eax ++ orl %esi, %ecx + /* Subtraction is okay because the upper 8 bits are zero. */ + subl %ecx, %eax + ret +- + .p2align 4 +-L(1): +- movzbl (%rdi), %eax ++L(one_or_less): ++ jb L(zero) + movzbl (%rsi), %ecx ++ movzbl (%rdi), %eax + subl %ecx, %eax + ret +-# endif +- +- .p2align 4 +-L(zero): +- xorl %eax, %eax +- ret + + .p2align 4 +-L(less_vec): +-# ifdef USE_AS_WMEMCMP +- /* It can only be 0, 4, 8, 12, 16, 20, 24, 28 bytes. */ +- cmpb $4, %dl +- je L(4) +- jb L(zero) +-# else +- cmpb $1, %dl +- je L(1) +- jb L(zero) +- cmpb $4, %dl +- jb L(between_2_3) +- cmpb $8, %dl +- jb L(between_4_7) ++L(between_8_15): + # endif +- cmpb $16, %dl +- jae L(between_16_31) +- /* It is between 8 and 15 bytes. */ ++ /* If USE_AS_WMEMCMP fall through into 8-15 byte case. */ + vmovq (%rdi), %XMM1 + vmovq (%rsi), %XMM2 +- VPCMPEQ %XMM1, %XMM2, %k2 +- kmovw %k2, %eax +- subl $XMM_MASK, %eax +- jnz L(first_vec) ++ VPCMP $4, %XMM1, %XMM2, %k1 ++ kmovd %k1, %eax ++ testl %eax, %eax ++ jnz L(return_vec_0) + /* Use overlapping loads to avoid branches. */ +- leaq -8(%rdi, %rdx), %rdi +- leaq -8(%rsi, %rdx), %rsi ++ leaq -8(%rdi, %rdx, CHAR_SIZE), %rdi ++ leaq -8(%rsi, %rdx, CHAR_SIZE), %rsi + vmovq (%rdi), %XMM1 + vmovq (%rsi), %XMM2 +- VPCMPEQ %XMM1, %XMM2, %k2 +- kmovw %k2, %eax +- subl $XMM_MASK, %eax +- jnz L(first_vec) ++ VPCMP $4, %XMM1, %XMM2, %k1 ++ kmovd %k1, %eax ++ testl %eax, %eax ++ jnz L(return_vec_0) + ret + + .p2align 4 +-L(between_16_31): +- /* From 16 to 31 bytes. No branch when size == 16. */ +- VMOVU (%rsi), %XMM2 +- VPCMPEQ (%rdi), %XMM2, %k2 +- kmovw %k2, %eax +- subl $XMM_MASK, %eax +- jnz L(first_vec) +- +- /* Use overlapping loads to avoid branches. */ +- leaq -16(%rdi, %rdx), %rdi +- leaq -16(%rsi, %rdx), %rsi +- VMOVU (%rsi), %XMM2 +- VPCMPEQ (%rdi), %XMM2, %k2 +- kmovw %k2, %eax +- subl $XMM_MASK, %eax +- jnz L(first_vec) ++L(zero): ++ xorl %eax, %eax + ret + + .p2align 4 +-L(more_8x_vec): +- /* More than 8 * VEC. Check the first VEC. */ +- VMOVU (%rsi), %YMM2 +- VPCMPEQ (%rdi), %YMM2, %k2 +- kmovd %k2, %eax +- subl $VEC_MASK, %eax +- jnz L(first_vec) +- +- /* Align the first memory area for aligned loads in the loop. +- Compute how much the first memory area is misaligned. */ +- movq %rdi, %rcx +- andl $(VEC_SIZE - 1), %ecx +- /* Get the negative of offset for alignment. */ +- subq $VEC_SIZE, %rcx +- /* Adjust the second memory area. */ +- subq %rcx, %rsi +- /* Adjust the first memory area which should be aligned now. */ +- subq %rcx, %rdi +- /* Adjust length. */ +- addq %rcx, %rdx +- +-L(loop_4x_vec): +- /* Compare 4 * VEC at a time forward. */ +- VMOVU (%rsi), %YMM1 +- VPCMPEQ (%rdi), %YMM1, %k1 +- +- VMOVU VEC_SIZE(%rsi), %YMM2 +- VPCMPEQ VEC_SIZE(%rdi), %YMM2, %k2 +- kandd %k2, %k1, %k5 +- +- VMOVU (VEC_SIZE * 2)(%rsi), %YMM3 +- VPCMPEQ (VEC_SIZE * 2)(%rdi), %YMM3, %k3 +- kandd %k3, %k5, %k5 +- +- VMOVU (VEC_SIZE * 3)(%rsi), %YMM4 +- VPCMPEQ (VEC_SIZE * 3)(%rdi), %YMM4, %k4 +- kandd %k4, %k5, %k5 +- +- kmovd %k5, %eax +- cmpl $VEC_MASK, %eax +- jne L(4x_vec_end) +- +- addq $(VEC_SIZE * 4), %rdi +- addq $(VEC_SIZE * 4), %rsi +- +- subq $(VEC_SIZE * 4), %rdx +- cmpq $(VEC_SIZE * 4), %rdx +- jae L(loop_4x_vec) +- +- /* Less than 4 * VEC. */ +- cmpq $VEC_SIZE, %rdx +- jbe L(last_vec) +- cmpq $(VEC_SIZE * 2), %rdx +- jbe L(last_2x_vec) +- +-L(last_4x_vec): +- /* From 2 * VEC to 4 * VEC. */ +- VMOVU (%rsi), %YMM2 +- VPCMPEQ (%rdi), %YMM2, %k2 +- kmovd %k2, %eax +- subl $VEC_MASK, %eax +- jnz L(first_vec) +- +- addq $VEC_SIZE, %rdi +- addq $VEC_SIZE, %rsi +- VMOVU (%rsi), %YMM2 +- VPCMPEQ (%rdi), %YMM2, %k2 +- kmovd %k2, %eax +- subl $VEC_MASK, %eax +- jnz L(first_vec) ++L(between_16_31): ++ /* From 16 to 31 bytes. No branch when size == 16. */ ++ VMOVU (%rsi), %XMM2 ++ VPCMP $4, (%rdi), %XMM2, %k1 ++ kmovd %k1, %eax ++ testl %eax, %eax ++ jnz L(return_vec_0) + + /* Use overlapping loads to avoid branches. */ +- leaq -(3 * VEC_SIZE)(%rdi, %rdx), %rdi +- leaq -(3 * VEC_SIZE)(%rsi, %rdx), %rsi +- VMOVU (%rsi), %YMM2 +- VPCMPEQ (%rdi), %YMM2, %k2 +- kmovd %k2, %eax +- subl $VEC_MASK, %eax +- jnz L(first_vec) + +- addq $VEC_SIZE, %rdi +- addq $VEC_SIZE, %rsi +- VMOVU (%rsi), %YMM2 +- VPCMPEQ (%rdi), %YMM2, %k2 +- kmovd %k2, %eax +- subl $VEC_MASK, %eax +- jnz L(first_vec) +- ret +- +- .p2align 4 +-L(4x_vec_end): ++ VMOVU -16(%rsi, %rdx, CHAR_SIZE), %XMM2 ++ leaq -16(%rdi, %rdx, CHAR_SIZE), %rdi ++ leaq -16(%rsi, %rdx, CHAR_SIZE), %rsi ++ VPCMP $4, (%rdi), %XMM2, %k1 + kmovd %k1, %eax +- subl $VEC_MASK, %eax +- jnz L(first_vec) +- kmovd %k2, %eax +- subl $VEC_MASK, %eax +- jnz L(first_vec_x1) +- kmovd %k3, %eax +- subl $VEC_MASK, %eax +- jnz L(first_vec_x2) +- kmovd %k4, %eax +- subl $VEC_MASK, %eax +- tzcntl %eax, %ecx +-# ifdef USE_AS_WMEMCMP +- xorl %eax, %eax +- movl (VEC_SIZE * 3)(%rdi, %rcx, 4), %edx +- cmpl (VEC_SIZE * 3)(%rsi, %rcx, 4), %edx +- jmp L(wmemcmp_return) +-# else +- movzbl (VEC_SIZE * 3)(%rdi, %rcx), %eax +- movzbl (VEC_SIZE * 3)(%rsi, %rcx), %edx +- sub %edx, %eax +-# endif ++ testl %eax, %eax ++ jnz L(return_vec_0) + ret + +- .p2align 4 +-L(first_vec_x1): +- tzcntl %eax, %ecx + # ifdef USE_AS_WMEMCMP +- xorl %eax, %eax +- movl VEC_SIZE(%rdi, %rcx, 4), %edx +- cmpl VEC_SIZE(%rsi, %rcx, 4), %edx +- jmp L(wmemcmp_return) +-# else +- movzbl VEC_SIZE(%rdi, %rcx), %eax +- movzbl VEC_SIZE(%rsi, %rcx), %edx +- sub %edx, %eax +-# endif ++ .p2align 4 ++L(one_or_less): ++ jb L(zero) ++ movl (%rdi), %ecx ++ xorl %edx, %edx ++ cmpl (%rsi), %ecx ++ je L(zero) ++ setg %dl ++ leal -1(%rdx, %rdx), %eax + ret ++# else + + .p2align 4 +-L(first_vec_x2): +- tzcntl %eax, %ecx +-# ifdef USE_AS_WMEMCMP +- xorl %eax, %eax +- movl (VEC_SIZE * 2)(%rdi, %rcx, 4), %edx +- cmpl (VEC_SIZE * 2)(%rsi, %rcx, 4), %edx +- jmp L(wmemcmp_return) +-# else +- movzbl (VEC_SIZE * 2)(%rdi, %rcx), %eax +- movzbl (VEC_SIZE * 2)(%rsi, %rcx), %edx +- sub %edx, %eax +-# endif ++L(between_4_7): ++ /* Load as big endian with overlapping movbe to avoid branches. ++ */ ++ movbe (%rdi), %eax ++ movbe (%rsi), %ecx ++ shlq $32, %rax ++ shlq $32, %rcx ++ movbe -4(%rdi, %rdx), %edi ++ movbe -4(%rsi, %rdx), %esi ++ orq %rdi, %rax ++ orq %rsi, %rcx ++ subq %rcx, %rax ++ jz L(zero_4_7) ++ sbbl %eax, %eax ++ orl $1, %eax ++L(zero_4_7): + ret ++# endif ++ + END (MEMCMP) + #endif +-- +GitLab + diff --git a/SOURCES/ia-opt-memcmp-evex-movbe-2.patch b/SOURCES/ia-opt-memcmp-evex-movbe-2.patch new file mode 100644 index 0000000..360a4f9 --- /dev/null +++ b/SOURCES/ia-opt-memcmp-evex-movbe-2.patch @@ -0,0 +1,659 @@ +From 851ab0499680a3369da724d3d6d2ba71652d530d Mon Sep 17 00:00:00 2001 +From: Noah Goldstein +Date: Tue, 21 Sep 2021 18:45:03 -0500 +Subject: [PATCH] x86: Optimize memcmp-evex-movbe.S for frontend behavior and + size + +No bug. + +The frontend optimizations are to: +1. Reorganize logically connected basic blocks so they are either in + the same cache line or adjacent cache lines. +2. Avoid cases when basic blocks unnecissarily cross cache lines. +3. Try and 32 byte align any basic blocks possible without sacrificing + code size. Smaller / Less hot basic blocks are used for this. + +Overall code size shrunk by 168 bytes. This should make up for any +extra costs due to aligning to 64 bytes. + +In general performance before deviated a great deal dependending on +whether entry alignment % 64 was 0, 16, 32, or 48. These changes +essentially make it so that the current implementation is at least +equal to the best alignment of the original for any arguments. + +The only additional optimization is in the page cross case. Branch on +equals case was removed from the size == [4, 7] case. As well the [4, +7] and [2, 3] case where swapped as [4, 7] is likely a more hot +argument size. + +test-memcmp and test-wmemcmp are both passing. + +(cherry picked from commit 1bd8b8d58fc9967cc073d2c13bfb6befefca2faa) +--- + sysdeps/x86_64/multiarch/memcmp-evex-movbe.S | 434 +++++++++++-------- + 1 file changed, 242 insertions(+), 192 deletions(-) + +diff --git a/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S b/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S +index 654dc7ac..2761b54f 100644 +--- a/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S ++++ b/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S +@@ -34,7 +34,24 @@ + area. + 7. Use 2 vector compares when size is 2 * CHAR_PER_VEC or less. + 8. Use 4 vector compares when size is 4 * CHAR_PER_VEC or less. +- 9. Use 8 vector compares when size is 8 * CHAR_PER_VEC or less. */ ++ 9. Use 8 vector compares when size is 8 * CHAR_PER_VEC or less. ++ ++When possible the implementation tries to optimize for frontend in the ++following ways: ++Throughput: ++ 1. All code sections that fit are able to run optimally out of the ++ LSD. ++ 2. All code sections that fit are able to run optimally out of the ++ DSB ++ 3. Basic blocks are contained in minimum number of fetch blocks ++ necessary. ++ ++Latency: ++ 1. Logically connected basic blocks are put in the same ++ cache-line. ++ 2. Logically connected basic blocks that do not fit in the same ++ cache-line are put in adjacent lines. This can get beneficial ++ L2 spatial prefetching and L1 next-line prefetching. */ + + # include + +@@ -47,9 +64,11 @@ + # ifdef USE_AS_WMEMCMP + # define CHAR_SIZE 4 + # define VPCMP vpcmpd ++# define VPTEST vptestmd + # else + # define CHAR_SIZE 1 + # define VPCMP vpcmpub ++# define VPTEST vptestmb + # endif + + # define VEC_SIZE 32 +@@ -75,7 +94,9 @@ + */ + + .section .text.evex,"ax",@progbits +-ENTRY (MEMCMP) ++/* Cache align memcmp entry. This allows for much more thorough ++ frontend optimization. */ ++ENTRY_P2ALIGN (MEMCMP, 6) + # ifdef __ILP32__ + /* Clear the upper 32 bits. */ + movl %edx, %edx +@@ -89,7 +110,7 @@ ENTRY (MEMCMP) + VPCMP $4, (%rdi), %YMM1, %k1 + kmovd %k1, %eax + /* NB: eax must be destination register if going to +- L(return_vec_[0,2]). For L(return_vec_3 destination register ++ L(return_vec_[0,2]). For L(return_vec_3) destination register + must be ecx. */ + testl %eax, %eax + jnz L(return_vec_0) +@@ -121,10 +142,6 @@ ENTRY (MEMCMP) + testl %ecx, %ecx + jnz L(return_vec_3) + +- /* Zero YMM0. 4x VEC reduction is done with vpxor + vtern so +- compare with zero to get a mask is needed. */ +- vpxorq %XMM0, %XMM0, %XMM0 +- + /* Go to 4x VEC loop. */ + cmpq $(CHAR_PER_VEC * 8), %rdx + ja L(more_8x_vec) +@@ -148,47 +165,61 @@ ENTRY (MEMCMP) + + VMOVU (VEC_SIZE * 2)(%rsi), %YMM3 + vpxorq (VEC_SIZE * 2)(%rdi), %YMM3, %YMM3 +- /* Or together YMM1, YMM2, and YMM3 into YMM3. */ +- vpternlogd $0xfe, %YMM1, %YMM2, %YMM3 + + VMOVU (VEC_SIZE * 3)(%rsi), %YMM4 + /* Ternary logic to xor (VEC_SIZE * 3)(%rdi) with YMM4 while +- oring with YMM3. Result is stored in YMM4. */ +- vpternlogd $0xde, (VEC_SIZE * 3)(%rdi), %YMM3, %YMM4 +- /* Compare YMM4 with 0. If any 1s s1 and s2 don't match. */ +- VPCMP $4, %YMM4, %YMM0, %k1 ++ oring with YMM1. Result is stored in YMM4. */ ++ vpternlogd $0xde, (VEC_SIZE * 3)(%rdi), %YMM1, %YMM4 ++ ++ /* Or together YMM2, YMM3, and YMM4 into YMM4. */ ++ vpternlogd $0xfe, %YMM2, %YMM3, %YMM4 ++ ++ /* Test YMM4 against itself. Store any CHAR mismatches in k1. ++ */ ++ VPTEST %YMM4, %YMM4, %k1 ++ /* k1 must go to ecx for L(return_vec_0_1_2_3). */ + kmovd %k1, %ecx + testl %ecx, %ecx + jnz L(return_vec_0_1_2_3) + /* NB: eax must be zero to reach here. */ + ret + +- /* NB: aligning 32 here allows for the rest of the jump targets +- to be tuned for 32 byte alignment. Most important this ensures +- the L(more_8x_vec) loop is 32 byte aligned. */ +- .p2align 5 +-L(less_vec): +- /* Check if one or less CHAR. This is necessary for size = 0 but +- is also faster for size = CHAR_SIZE. */ +- cmpl $1, %edx +- jbe L(one_or_less) ++ .p2align 4 ++L(8x_end_return_vec_0_1_2_3): ++ movq %rdx, %rdi ++L(8x_return_vec_0_1_2_3): ++ addq %rdi, %rsi ++L(return_vec_0_1_2_3): ++ VPTEST %YMM1, %YMM1, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(return_vec_0) + +- /* Check if loading one VEC from either s1 or s2 could cause a +- page cross. This can have false positives but is by far the +- fastest method. */ +- movl %edi, %eax +- orl %esi, %eax +- andl $(PAGE_SIZE - 1), %eax +- cmpl $(PAGE_SIZE - VEC_SIZE), %eax +- jg L(page_cross_less_vec) ++ VPTEST %YMM2, %YMM2, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(return_vec_1) + +- /* No page cross possible. */ +- VMOVU (%rsi), %YMM2 +- VPCMP $4, (%rdi), %YMM2, %k1 +- kmovd %k1, %eax +- /* Create mask in ecx for potentially in bound matches. */ +- bzhil %edx, %eax, %eax +- jnz L(return_vec_0) ++ VPTEST %YMM3, %YMM3, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(return_vec_2) ++L(return_vec_3): ++ /* bsf saves 1 byte from tzcnt. This keep L(return_vec_3) in one ++ fetch block and the entire L(*return_vec_0_1_2_3) in 1 cache ++ line. */ ++ bsfl %ecx, %ecx ++# ifdef USE_AS_WMEMCMP ++ movl (VEC_SIZE * 3)(%rdi, %rcx, CHAR_SIZE), %eax ++ xorl %edx, %edx ++ cmpl (VEC_SIZE * 3)(%rsi, %rcx, CHAR_SIZE), %eax ++ setg %dl ++ leal -1(%rdx, %rdx), %eax ++# else ++ movzbl (VEC_SIZE * 3)(%rdi, %rcx), %eax ++ movzbl (VEC_SIZE * 3)(%rsi, %rcx), %ecx ++ subl %ecx, %eax ++# endif + ret + + .p2align 4 +@@ -209,10 +240,11 @@ L(return_vec_0): + # endif + ret + +- /* NB: No p2align necessary. Alignment % 16 is naturally 1 +- which is good enough for a target not in a loop. */ ++ .p2align 4 + L(return_vec_1): +- tzcntl %eax, %eax ++ /* bsf saves 1 byte over tzcnt and keeps L(return_vec_1) in one ++ fetch block. */ ++ bsfl %eax, %eax + # ifdef USE_AS_WMEMCMP + movl VEC_SIZE(%rdi, %rax, CHAR_SIZE), %ecx + xorl %edx, %edx +@@ -226,10 +258,11 @@ L(return_vec_1): + # endif + ret + +- /* NB: No p2align necessary. Alignment % 16 is naturally 2 +- which is good enough for a target not in a loop. */ ++ .p2align 4,, 10 + L(return_vec_2): +- tzcntl %eax, %eax ++ /* bsf saves 1 byte over tzcnt and keeps L(return_vec_2) in one ++ fetch block. */ ++ bsfl %eax, %eax + # ifdef USE_AS_WMEMCMP + movl (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %ecx + xorl %edx, %edx +@@ -243,40 +276,6 @@ L(return_vec_2): + # endif + ret + +- .p2align 4 +-L(8x_return_vec_0_1_2_3): +- /* Returning from L(more_8x_vec) requires restoring rsi. */ +- addq %rdi, %rsi +-L(return_vec_0_1_2_3): +- VPCMP $4, %YMM1, %YMM0, %k0 +- kmovd %k0, %eax +- testl %eax, %eax +- jnz L(return_vec_0) +- +- VPCMP $4, %YMM2, %YMM0, %k0 +- kmovd %k0, %eax +- testl %eax, %eax +- jnz L(return_vec_1) +- +- VPCMP $4, %YMM3, %YMM0, %k0 +- kmovd %k0, %eax +- testl %eax, %eax +- jnz L(return_vec_2) +-L(return_vec_3): +- tzcntl %ecx, %ecx +-# ifdef USE_AS_WMEMCMP +- movl (VEC_SIZE * 3)(%rdi, %rcx, CHAR_SIZE), %eax +- xorl %edx, %edx +- cmpl (VEC_SIZE * 3)(%rsi, %rcx, CHAR_SIZE), %eax +- setg %dl +- leal -1(%rdx, %rdx), %eax +-# else +- movzbl (VEC_SIZE * 3)(%rdi, %rcx), %eax +- movzbl (VEC_SIZE * 3)(%rsi, %rcx), %ecx +- subl %ecx, %eax +-# endif +- ret +- + .p2align 4 + L(more_8x_vec): + /* Set end of s1 in rdx. */ +@@ -288,21 +287,19 @@ L(more_8x_vec): + andq $-VEC_SIZE, %rdi + /* Adjust because first 4x vec where check already. */ + subq $-(VEC_SIZE * 4), %rdi ++ + .p2align 4 + L(loop_4x_vec): + VMOVU (%rsi, %rdi), %YMM1 + vpxorq (%rdi), %YMM1, %YMM1 +- + VMOVU VEC_SIZE(%rsi, %rdi), %YMM2 + vpxorq VEC_SIZE(%rdi), %YMM2, %YMM2 +- + VMOVU (VEC_SIZE * 2)(%rsi, %rdi), %YMM3 + vpxorq (VEC_SIZE * 2)(%rdi), %YMM3, %YMM3 +- vpternlogd $0xfe, %YMM1, %YMM2, %YMM3 +- + VMOVU (VEC_SIZE * 3)(%rsi, %rdi), %YMM4 +- vpternlogd $0xde, (VEC_SIZE * 3)(%rdi), %YMM3, %YMM4 +- VPCMP $4, %YMM4, %YMM0, %k1 ++ vpternlogd $0xde, (VEC_SIZE * 3)(%rdi), %YMM1, %YMM4 ++ vpternlogd $0xfe, %YMM2, %YMM3, %YMM4 ++ VPTEST %YMM4, %YMM4, %k1 + kmovd %k1, %ecx + testl %ecx, %ecx + jnz L(8x_return_vec_0_1_2_3) +@@ -319,28 +316,25 @@ L(loop_4x_vec): + cmpl $(VEC_SIZE * 2), %edi + jae L(8x_last_2x_vec) + ++ vpxorq (VEC_SIZE * 2)(%rdx), %YMM3, %YMM3 ++ + VMOVU (%rsi, %rdx), %YMM1 + vpxorq (%rdx), %YMM1, %YMM1 + + VMOVU VEC_SIZE(%rsi, %rdx), %YMM2 + vpxorq VEC_SIZE(%rdx), %YMM2, %YMM2 +- +- vpxorq (VEC_SIZE * 2)(%rdx), %YMM3, %YMM3 +- vpternlogd $0xfe, %YMM1, %YMM2, %YMM3 +- + VMOVU (VEC_SIZE * 3)(%rsi, %rdx), %YMM4 +- vpternlogd $0xde, (VEC_SIZE * 3)(%rdx), %YMM3, %YMM4 +- VPCMP $4, %YMM4, %YMM0, %k1 ++ vpternlogd $0xde, (VEC_SIZE * 3)(%rdx), %YMM1, %YMM4 ++ vpternlogd $0xfe, %YMM2, %YMM3, %YMM4 ++ VPTEST %YMM4, %YMM4, %k1 + kmovd %k1, %ecx +- /* Restore s1 pointer to rdi. */ +- movq %rdx, %rdi + testl %ecx, %ecx +- jnz L(8x_return_vec_0_1_2_3) ++ jnz L(8x_end_return_vec_0_1_2_3) + /* NB: eax must be zero to reach here. */ + ret + + /* Only entry is from L(more_8x_vec). */ +- .p2align 4 ++ .p2align 4,, 10 + L(8x_last_2x_vec): + VPCMP $4, (VEC_SIZE * 2)(%rdx), %YMM3, %k1 + kmovd %k1, %eax +@@ -355,7 +349,31 @@ L(8x_last_1x_vec): + jnz L(8x_return_vec_3) + ret + +- .p2align 4 ++ /* Not ideally aligned (at offset +9 bytes in fetch block) but ++ not aligning keeps it in the same cache line as ++ L(8x_last_1x/2x_vec) so likely worth it. As well, saves code ++ size. */ ++ .p2align 4,, 4 ++L(8x_return_vec_2): ++ subq $VEC_SIZE, %rdx ++L(8x_return_vec_3): ++ bsfl %eax, %eax ++# ifdef USE_AS_WMEMCMP ++ leaq (%rdx, %rax, CHAR_SIZE), %rax ++ movl (VEC_SIZE * 3)(%rax), %ecx ++ xorl %edx, %edx ++ cmpl (VEC_SIZE * 3)(%rsi, %rax), %ecx ++ setg %dl ++ leal -1(%rdx, %rdx), %eax ++# else ++ addq %rdx, %rax ++ movzbl (VEC_SIZE * 3)(%rsi, %rax), %ecx ++ movzbl (VEC_SIZE * 3)(%rax), %eax ++ subl %ecx, %eax ++# endif ++ ret ++ ++ .p2align 4,, 10 + L(last_2x_vec): + /* Check second to last VEC. */ + VMOVU -(VEC_SIZE * 2)(%rsi, %rdx, CHAR_SIZE), %YMM1 +@@ -374,26 +392,49 @@ L(last_1x_vec): + jnz L(return_vec_0_end) + ret + +- .p2align 4 +-L(8x_return_vec_2): +- subq $VEC_SIZE, %rdx +-L(8x_return_vec_3): +- tzcntl %eax, %eax ++ .p2align 4,, 10 ++L(return_vec_1_end): ++ /* Use bsf to save code size. This is necessary to have ++ L(one_or_less) fit in aligning bytes between. */ ++ bsfl %eax, %eax ++ addl %edx, %eax + # ifdef USE_AS_WMEMCMP +- leaq (%rdx, %rax, CHAR_SIZE), %rax +- movl (VEC_SIZE * 3)(%rax), %ecx ++ movl -(VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %ecx + xorl %edx, %edx +- cmpl (VEC_SIZE * 3)(%rsi, %rax), %ecx ++ cmpl -(VEC_SIZE * 2)(%rsi, %rax, CHAR_SIZE), %ecx + setg %dl + leal -1(%rdx, %rdx), %eax + # else +- addq %rdx, %rax +- movzbl (VEC_SIZE * 3)(%rsi, %rax), %ecx +- movzbl (VEC_SIZE * 3)(%rax), %eax ++ movzbl -(VEC_SIZE * 2)(%rsi, %rax), %ecx ++ movzbl -(VEC_SIZE * 2)(%rdi, %rax), %eax + subl %ecx, %eax + # endif + ret + ++ /* NB: L(one_or_less) fits in alignment padding between ++ L(return_vec_1_end) and L(return_vec_0_end). */ ++# ifdef USE_AS_WMEMCMP ++L(one_or_less): ++ jb L(zero) ++ movl (%rdi), %ecx ++ xorl %edx, %edx ++ cmpl (%rsi), %ecx ++ je L(zero) ++ setg %dl ++ leal -1(%rdx, %rdx), %eax ++ ret ++# else ++L(one_or_less): ++ jb L(zero) ++ movzbl (%rsi), %ecx ++ movzbl (%rdi), %eax ++ subl %ecx, %eax ++ ret ++# endif ++L(zero): ++ xorl %eax, %eax ++ ret ++ + .p2align 4 + L(return_vec_0_end): + tzcntl %eax, %eax +@@ -412,23 +453,56 @@ L(return_vec_0_end): + ret + + .p2align 4 +-L(return_vec_1_end): ++L(less_vec): ++ /* Check if one or less CHAR. This is necessary for size == 0 ++ but is also faster for size == CHAR_SIZE. */ ++ cmpl $1, %edx ++ jbe L(one_or_less) ++ ++ /* Check if loading one VEC from either s1 or s2 could cause a ++ page cross. This can have false positives but is by far the ++ fastest method. */ ++ movl %edi, %eax ++ orl %esi, %eax ++ andl $(PAGE_SIZE - 1), %eax ++ cmpl $(PAGE_SIZE - VEC_SIZE), %eax ++ jg L(page_cross_less_vec) ++ ++ /* No page cross possible. */ ++ VMOVU (%rsi), %YMM2 ++ VPCMP $4, (%rdi), %YMM2, %k1 ++ kmovd %k1, %eax ++ /* Check if any matches where in bounds. Intentionally not ++ storing result in eax to limit dependency chain if it goes to ++ L(return_vec_0_lv). */ ++ bzhil %edx, %eax, %edx ++ jnz L(return_vec_0_lv) ++ xorl %eax, %eax ++ ret ++ ++ /* Essentially duplicate of L(return_vec_0). Ends up not costing ++ any code as shrinks L(less_vec) by allowing 2-byte encoding of ++ the jump and ends up fitting in aligning bytes. As well fits on ++ same cache line as L(less_vec) so also saves a line from having ++ to be fetched on cold calls to memcmp. */ ++ .p2align 4,, 4 ++L(return_vec_0_lv): + tzcntl %eax, %eax +- addl %edx, %eax + # ifdef USE_AS_WMEMCMP +- movl -(VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %ecx ++ movl (%rdi, %rax, CHAR_SIZE), %ecx + xorl %edx, %edx +- cmpl -(VEC_SIZE * 2)(%rsi, %rax, CHAR_SIZE), %ecx ++ cmpl (%rsi, %rax, CHAR_SIZE), %ecx ++ /* NB: no partial register stall here because xorl zero idiom ++ above. */ + setg %dl + leal -1(%rdx, %rdx), %eax + # else +- movzbl -(VEC_SIZE * 2)(%rsi, %rax), %ecx +- movzbl -(VEC_SIZE * 2)(%rdi, %rax), %eax ++ movzbl (%rsi, %rax), %ecx ++ movzbl (%rdi, %rax), %eax + subl %ecx, %eax + # endif + ret + +- + .p2align 4 + L(page_cross_less_vec): + /* if USE_AS_WMEMCMP it can only be 0, 4, 8, 12, 16, 20, 24, 28 +@@ -439,108 +513,84 @@ L(page_cross_less_vec): + cmpl $8, %edx + jae L(between_8_15) + cmpl $4, %edx +- jae L(between_4_7) +-L(between_2_3): +- /* Load as big endian to avoid branches. */ +- movzwl (%rdi), %eax +- movzwl (%rsi), %ecx +- shll $8, %eax +- shll $8, %ecx +- bswap %eax +- bswap %ecx +- movzbl -1(%rdi, %rdx), %edi +- movzbl -1(%rsi, %rdx), %esi +- orl %edi, %eax +- orl %esi, %ecx +- /* Subtraction is okay because the upper 8 bits are zero. */ +- subl %ecx, %eax +- ret +- .p2align 4 +-L(one_or_less): +- jb L(zero) +- movzbl (%rsi), %ecx +- movzbl (%rdi), %eax +- subl %ecx, %eax ++ jb L(between_2_3) ++ ++ /* Load as big endian with overlapping movbe to avoid branches. ++ */ ++ movbe (%rdi), %eax ++ movbe (%rsi), %ecx ++ shlq $32, %rax ++ shlq $32, %rcx ++ movbe -4(%rdi, %rdx), %edi ++ movbe -4(%rsi, %rdx), %esi ++ orq %rdi, %rax ++ orq %rsi, %rcx ++ subq %rcx, %rax ++ /* edx is guranteed to be positive int32 in range [4, 7]. */ ++ cmovne %edx, %eax ++ /* ecx is -1 if rcx > rax. Otherwise 0. */ ++ sbbl %ecx, %ecx ++ /* If rcx > rax, then ecx is 0 and eax is positive. If rcx == ++ rax then eax and ecx are zero. If rax < rax then ecx is -1 so ++ eax doesn't matter. */ ++ orl %ecx, %eax + ret + +- .p2align 4 ++ .p2align 4,, 8 + L(between_8_15): + # endif + /* If USE_AS_WMEMCMP fall through into 8-15 byte case. */ +- vmovq (%rdi), %XMM1 +- vmovq (%rsi), %XMM2 +- VPCMP $4, %XMM1, %XMM2, %k1 ++ vmovq (%rdi), %xmm1 ++ vmovq (%rsi), %xmm2 ++ VPCMP $4, %xmm1, %xmm2, %k1 + kmovd %k1, %eax + testl %eax, %eax +- jnz L(return_vec_0) ++ jnz L(return_vec_0_lv) + /* Use overlapping loads to avoid branches. */ +- leaq -8(%rdi, %rdx, CHAR_SIZE), %rdi +- leaq -8(%rsi, %rdx, CHAR_SIZE), %rsi +- vmovq (%rdi), %XMM1 +- vmovq (%rsi), %XMM2 +- VPCMP $4, %XMM1, %XMM2, %k1 ++ vmovq -8(%rdi, %rdx, CHAR_SIZE), %xmm1 ++ vmovq -8(%rsi, %rdx, CHAR_SIZE), %xmm2 ++ VPCMP $4, %xmm1, %xmm2, %k1 ++ addl $(CHAR_PER_VEC - (8 / CHAR_SIZE)), %edx + kmovd %k1, %eax + testl %eax, %eax +- jnz L(return_vec_0) +- ret +- +- .p2align 4 +-L(zero): +- xorl %eax, %eax ++ jnz L(return_vec_0_end) + ret + +- .p2align 4 ++ .p2align 4,, 8 + L(between_16_31): + /* From 16 to 31 bytes. No branch when size == 16. */ +- VMOVU (%rsi), %XMM2 +- VPCMP $4, (%rdi), %XMM2, %k1 ++ ++ /* Use movups to save code size. */ ++ movups (%rsi), %xmm2 ++ VPCMP $4, (%rdi), %xmm2, %k1 + kmovd %k1, %eax + testl %eax, %eax +- jnz L(return_vec_0) +- ++ jnz L(return_vec_0_lv) + /* Use overlapping loads to avoid branches. */ +- +- VMOVU -16(%rsi, %rdx, CHAR_SIZE), %XMM2 +- leaq -16(%rdi, %rdx, CHAR_SIZE), %rdi +- leaq -16(%rsi, %rdx, CHAR_SIZE), %rsi +- VPCMP $4, (%rdi), %XMM2, %k1 ++ movups -16(%rsi, %rdx, CHAR_SIZE), %xmm2 ++ VPCMP $4, -16(%rdi, %rdx, CHAR_SIZE), %xmm2, %k1 ++ addl $(CHAR_PER_VEC - (16 / CHAR_SIZE)), %edx + kmovd %k1, %eax + testl %eax, %eax +- jnz L(return_vec_0) +- ret +- +-# ifdef USE_AS_WMEMCMP +- .p2align 4 +-L(one_or_less): +- jb L(zero) +- movl (%rdi), %ecx +- xorl %edx, %edx +- cmpl (%rsi), %ecx +- je L(zero) +- setg %dl +- leal -1(%rdx, %rdx), %eax ++ jnz L(return_vec_0_end) + ret +-# else + +- .p2align 4 +-L(between_4_7): +- /* Load as big endian with overlapping movbe to avoid branches. +- */ +- movbe (%rdi), %eax +- movbe (%rsi), %ecx +- shlq $32, %rax +- shlq $32, %rcx +- movbe -4(%rdi, %rdx), %edi +- movbe -4(%rsi, %rdx), %esi +- orq %rdi, %rax +- orq %rsi, %rcx +- subq %rcx, %rax +- jz L(zero_4_7) +- sbbl %eax, %eax +- orl $1, %eax +-L(zero_4_7): ++# ifndef USE_AS_WMEMCMP ++L(between_2_3): ++ /* Load as big endian to avoid branches. */ ++ movzwl (%rdi), %eax ++ movzwl (%rsi), %ecx ++ shll $8, %eax ++ shll $8, %ecx ++ bswap %eax ++ bswap %ecx ++ movzbl -1(%rdi, %rdx), %edi ++ movzbl -1(%rsi, %rdx), %esi ++ orl %edi, %eax ++ orl %esi, %ecx ++ /* Subtraction is okay because the upper 8 bits are zero. */ ++ subl %ecx, %eax + ret + # endif +- + END (MEMCMP) + #endif +-- +GitLab + diff --git a/SOURCES/ia-opt-memmove-vec-unaligned-erms.patch b/SOURCES/ia-opt-memmove-vec-unaligned-erms.patch new file mode 100644 index 0000000..697c32f --- /dev/null +++ b/SOURCES/ia-opt-memmove-vec-unaligned-erms.patch @@ -0,0 +1,853 @@ +From b27eed69c1aa2e0fcdcda8b34249ee5b50b913d6 Mon Sep 17 00:00:00 2001 +From: Noah Goldstein +Date: Mon, 1 Nov 2021 00:49:51 -0500 +Subject: [PATCH] x86: Optimize memmove-vec-unaligned-erms.S + +No bug. + +The optimizations are as follows: + +1) Always align entry to 64 bytes. This makes behavior more + predictable and makes other frontend optimizations easier. + +2) Make the L(more_8x_vec) cases 4k aliasing aware. This can have + significant benefits in the case that: + 0 < (dst - src) < [256, 512] + +3) Align before `rep movsb`. For ERMS this is roughly a [0, 30%] + improvement and for FSRM [-10%, 25%]. + +In addition to these primary changes there is general cleanup +throughout to optimize the aligning routines and control flow logic. + +Signed-off-by: Noah Goldstein +Reviewed-by: H.J. Lu +(cherry picked from commit a6b7502ec0c2da89a7437f43171f160d713e39c6) +--- + sysdeps/x86_64/memmove.S | 2 +- + .../memmove-avx-unaligned-erms-rtm.S | 2 +- + .../multiarch/memmove-avx-unaligned-erms.S | 2 +- + .../multiarch/memmove-avx512-unaligned-erms.S | 2 +- + .../multiarch/memmove-evex-unaligned-erms.S | 2 +- + .../multiarch/memmove-vec-unaligned-erms.S | 595 +++++++++++------- + 6 files changed, 381 insertions(+), 224 deletions(-) + +diff --git a/sysdeps/x86_64/memmove.S b/sysdeps/x86_64/memmove.S +index 9cc92ff9..990fa6c5 100644 +--- a/sysdeps/x86_64/memmove.S ++++ b/sysdeps/x86_64/memmove.S +@@ -25,7 +25,7 @@ + /* Use movups and movaps for smaller code sizes. */ + #define VMOVU movups + #define VMOVA movaps +- ++#define MOV_SIZE 3 + #define SECTION(p) p + + #ifdef USE_MULTIARCH +diff --git a/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-rtm.S b/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-rtm.S +index 1ec1962e..67a55f0c 100644 +--- a/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-rtm.S ++++ b/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-rtm.S +@@ -4,7 +4,7 @@ + # define VMOVNT vmovntdq + # define VMOVU vmovdqu + # define VMOVA vmovdqa +- ++# define MOV_SIZE 4 + # define ZERO_UPPER_VEC_REGISTERS_RETURN \ + ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST + +diff --git a/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms.S +index e195e93f..975ae6c0 100644 +--- a/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms.S ++++ b/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms.S +@@ -4,7 +4,7 @@ + # define VMOVNT vmovntdq + # define VMOVU vmovdqu + # define VMOVA vmovdqa +- ++# define MOV_SIZE 4 + # define SECTION(p) p##.avx + # define MEMMOVE_SYMBOL(p,s) p##_avx_##s + +diff --git a/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S +index 848848ab..0fa71268 100644 +--- a/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S ++++ b/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S +@@ -25,7 +25,7 @@ + # define VMOVU vmovdqu64 + # define VMOVA vmovdqa64 + # define VZEROUPPER +- ++# define MOV_SIZE 6 + # define SECTION(p) p##.evex512 + # define MEMMOVE_SYMBOL(p,s) p##_avx512_##s + +diff --git a/sysdeps/x86_64/multiarch/memmove-evex-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-evex-unaligned-erms.S +index 0cbce8f9..88715441 100644 +--- a/sysdeps/x86_64/multiarch/memmove-evex-unaligned-erms.S ++++ b/sysdeps/x86_64/multiarch/memmove-evex-unaligned-erms.S +@@ -25,7 +25,7 @@ + # define VMOVU vmovdqu64 + # define VMOVA vmovdqa64 + # define VZEROUPPER +- ++# define MOV_SIZE 6 + # define SECTION(p) p##.evex + # define MEMMOVE_SYMBOL(p,s) p##_evex_##s + +diff --git a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S +index c0809b1b..e5495286 100644 +--- a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S ++++ b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S +@@ -76,6 +76,25 @@ + # endif + #endif + ++/* Whether to align before movsb. Ultimately we want 64 byte ++ align and not worth it to load 4x VEC for VEC_SIZE == 16. */ ++#define ALIGN_MOVSB (VEC_SIZE > 16) ++/* Number of bytes to align movsb to. */ ++#define MOVSB_ALIGN_TO 64 ++ ++#define SMALL_MOV_SIZE (MOV_SIZE <= 4) ++#define LARGE_MOV_SIZE (MOV_SIZE > 4) ++ ++#if SMALL_MOV_SIZE + LARGE_MOV_SIZE != 1 ++# error MOV_SIZE Unknown ++#endif ++ ++#if LARGE_MOV_SIZE ++# define SMALL_SIZE_OFFSET (4) ++#else ++# define SMALL_SIZE_OFFSET (0) ++#endif ++ + #ifndef PAGE_SIZE + # define PAGE_SIZE 4096 + #endif +@@ -199,25 +218,21 @@ L(start): + # endif + cmp $VEC_SIZE, %RDX_LP + jb L(less_vec) ++ /* Load regardless. */ ++ VMOVU (%rsi), %VEC(0) + cmp $(VEC_SIZE * 2), %RDX_LP + ja L(more_2x_vec) +-#if !defined USE_MULTIARCH || !IS_IN (libc) +-L(last_2x_vec): +-#endif + /* From VEC and to 2 * VEC. No branch when size == VEC_SIZE. */ +- VMOVU (%rsi), %VEC(0) + VMOVU -VEC_SIZE(%rsi,%rdx), %VEC(1) + VMOVU %VEC(0), (%rdi) + VMOVU %VEC(1), -VEC_SIZE(%rdi,%rdx) +-#if !defined USE_MULTIARCH || !IS_IN (libc) +-L(nop): +- ret ++#if !(defined USE_MULTIARCH && IS_IN (libc)) ++ ZERO_UPPER_VEC_REGISTERS_RETURN + #else + VZEROUPPER_RETURN + #endif + #if defined USE_MULTIARCH && IS_IN (libc) + END (MEMMOVE_SYMBOL (__memmove, unaligned)) +- + # if VEC_SIZE == 16 + ENTRY (__mempcpy_chk_erms) + cmp %RDX_LP, %RCX_LP +@@ -289,7 +304,7 @@ ENTRY (MEMMOVE_CHK_SYMBOL (__memmove_chk, unaligned_erms)) + END (MEMMOVE_CHK_SYMBOL (__memmove_chk, unaligned_erms)) + # endif + +-ENTRY (MEMMOVE_SYMBOL (__memmove, unaligned_erms)) ++ENTRY_P2ALIGN (MEMMOVE_SYMBOL (__memmove, unaligned_erms), 6) + movq %rdi, %rax + L(start_erms): + # ifdef __ILP32__ +@@ -298,310 +313,448 @@ L(start_erms): + # endif + cmp $VEC_SIZE, %RDX_LP + jb L(less_vec) ++ /* Load regardless. */ ++ VMOVU (%rsi), %VEC(0) + cmp $(VEC_SIZE * 2), %RDX_LP + ja L(movsb_more_2x_vec) +-L(last_2x_vec): +- /* From VEC and to 2 * VEC. No branch when size == VEC_SIZE. */ +- VMOVU (%rsi), %VEC(0) +- VMOVU -VEC_SIZE(%rsi,%rdx), %VEC(1) ++ /* From VEC and to 2 * VEC. No branch when size == VEC_SIZE. ++ */ ++ VMOVU -VEC_SIZE(%rsi, %rdx), %VEC(1) + VMOVU %VEC(0), (%rdi) +- VMOVU %VEC(1), -VEC_SIZE(%rdi,%rdx) ++ VMOVU %VEC(1), -VEC_SIZE(%rdi, %rdx) + L(return): +-#if VEC_SIZE > 16 ++# if VEC_SIZE > 16 + ZERO_UPPER_VEC_REGISTERS_RETURN +-#else ++# else + ret ++# endif + #endif + +-L(movsb): +- cmp __x86_rep_movsb_stop_threshold(%rip), %RDX_LP +- jae L(more_8x_vec) +- cmpq %rsi, %rdi +- jb 1f +- /* Source == destination is less common. */ +- je L(nop) +- leaq (%rsi,%rdx), %r9 +- cmpq %r9, %rdi +- /* Avoid slow backward REP MOVSB. */ +- jb L(more_8x_vec_backward) +-# if AVOID_SHORT_DISTANCE_REP_MOVSB +- testl $X86_STRING_CONTROL_AVOID_SHORT_DISTANCE_REP_MOVSB, __x86_string_control(%rip) +- jz 3f +- movq %rdi, %rcx +- subq %rsi, %rcx +- jmp 2f +-# endif +-1: +-# if AVOID_SHORT_DISTANCE_REP_MOVSB +- testl $X86_STRING_CONTROL_AVOID_SHORT_DISTANCE_REP_MOVSB, __x86_string_control(%rip) +- jz 3f +- movq %rsi, %rcx +- subq %rdi, %rcx +-2: +-/* Avoid "rep movsb" if RCX, the distance between source and destination, +- is N*4GB + [1..63] with N >= 0. */ +- cmpl $63, %ecx +- jbe L(more_2x_vec) /* Avoid "rep movsb" if ECX <= 63. */ +-3: +-# endif +- mov %RDX_LP, %RCX_LP +- rep movsb +-L(nop): ++#if LARGE_MOV_SIZE ++ /* If LARGE_MOV_SIZE this fits in the aligning bytes between the ++ ENTRY block and L(less_vec). */ ++ .p2align 4,, 8 ++L(between_4_7): ++ /* From 4 to 7. No branch when size == 4. */ ++ movl (%rsi), %ecx ++ movl (%rsi, %rdx), %esi ++ movl %ecx, (%rdi) ++ movl %esi, (%rdi, %rdx) + ret + #endif + ++ .p2align 4 + L(less_vec): + /* Less than 1 VEC. */ + #if VEC_SIZE != 16 && VEC_SIZE != 32 && VEC_SIZE != 64 + # error Unsupported VEC_SIZE! + #endif + #if VEC_SIZE > 32 +- cmpb $32, %dl ++ cmpl $32, %edx + jae L(between_32_63) + #endif + #if VEC_SIZE > 16 +- cmpb $16, %dl ++ cmpl $16, %edx + jae L(between_16_31) + #endif +- cmpb $8, %dl ++ cmpl $8, %edx + jae L(between_8_15) +- cmpb $4, %dl ++#if SMALL_MOV_SIZE ++ cmpl $4, %edx ++#else ++ subq $4, %rdx ++#endif + jae L(between_4_7) +- cmpb $1, %dl +- ja L(between_2_3) +- jb 1f +- movzbl (%rsi), %ecx ++ cmpl $(1 - SMALL_SIZE_OFFSET), %edx ++ jl L(copy_0) ++ movb (%rsi), %cl ++ je L(copy_1) ++ movzwl (-2 + SMALL_SIZE_OFFSET)(%rsi, %rdx), %esi ++ movw %si, (-2 + SMALL_SIZE_OFFSET)(%rdi, %rdx) ++L(copy_1): + movb %cl, (%rdi) +-1: ++L(copy_0): + ret ++ ++#if SMALL_MOV_SIZE ++ .p2align 4,, 8 ++L(between_4_7): ++ /* From 4 to 7. No branch when size == 4. */ ++ movl -4(%rsi, %rdx), %ecx ++ movl (%rsi), %esi ++ movl %ecx, -4(%rdi, %rdx) ++ movl %esi, (%rdi) ++ ret ++#endif ++ ++#if VEC_SIZE > 16 ++ /* From 16 to 31. No branch when size == 16. */ ++ .p2align 4,, 8 ++L(between_16_31): ++ vmovdqu (%rsi), %xmm0 ++ vmovdqu -16(%rsi, %rdx), %xmm1 ++ vmovdqu %xmm0, (%rdi) ++ vmovdqu %xmm1, -16(%rdi, %rdx) ++ /* No ymm registers have been touched. */ ++ ret ++#endif ++ + #if VEC_SIZE > 32 ++ .p2align 4,, 10 + L(between_32_63): + /* From 32 to 63. No branch when size == 32. */ + VMOVU (%rsi), %YMM0 +- VMOVU -32(%rsi,%rdx), %YMM1 ++ VMOVU -32(%rsi, %rdx), %YMM1 + VMOVU %YMM0, (%rdi) +- VMOVU %YMM1, -32(%rdi,%rdx) +- VZEROUPPER_RETURN +-#endif +-#if VEC_SIZE > 16 +- /* From 16 to 31. No branch when size == 16. */ +-L(between_16_31): +- VMOVU (%rsi), %XMM0 +- VMOVU -16(%rsi,%rdx), %XMM1 +- VMOVU %XMM0, (%rdi) +- VMOVU %XMM1, -16(%rdi,%rdx) ++ VMOVU %YMM1, -32(%rdi, %rdx) + VZEROUPPER_RETURN + #endif ++ ++ .p2align 4,, 10 + L(between_8_15): + /* From 8 to 15. No branch when size == 8. */ +- movq -8(%rsi,%rdx), %rcx ++ movq -8(%rsi, %rdx), %rcx + movq (%rsi), %rsi +- movq %rcx, -8(%rdi,%rdx) + movq %rsi, (%rdi) ++ movq %rcx, -8(%rdi, %rdx) + ret +-L(between_4_7): +- /* From 4 to 7. No branch when size == 4. */ +- movl -4(%rsi,%rdx), %ecx +- movl (%rsi), %esi +- movl %ecx, -4(%rdi,%rdx) +- movl %esi, (%rdi) +- ret +-L(between_2_3): +- /* From 2 to 3. No branch when size == 2. */ +- movzwl -2(%rsi,%rdx), %ecx +- movzwl (%rsi), %esi +- movw %cx, -2(%rdi,%rdx) +- movw %si, (%rdi) +- ret + ++ .p2align 4,, 10 ++L(last_4x_vec): ++ /* Copy from 2 * VEC + 1 to 4 * VEC, inclusively. */ ++ ++ /* VEC(0) and VEC(1) have already been loaded. */ ++ VMOVU -VEC_SIZE(%rsi, %rdx), %VEC(2) ++ VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VEC(3) ++ VMOVU %VEC(0), (%rdi) ++ VMOVU %VEC(1), VEC_SIZE(%rdi) ++ VMOVU %VEC(2), -VEC_SIZE(%rdi, %rdx) ++ VMOVU %VEC(3), -(VEC_SIZE * 2)(%rdi, %rdx) ++ VZEROUPPER_RETURN ++ ++ .p2align 4 + #if defined USE_MULTIARCH && IS_IN (libc) + L(movsb_more_2x_vec): + cmp __x86_rep_movsb_threshold(%rip), %RDX_LP + ja L(movsb) + #endif + L(more_2x_vec): +- /* More than 2 * VEC and there may be overlap between destination +- and source. */ ++ /* More than 2 * VEC and there may be overlap between ++ destination and source. */ + cmpq $(VEC_SIZE * 8), %rdx + ja L(more_8x_vec) ++ /* Load VEC(1) regardless. VEC(0) has already been loaded. */ ++ VMOVU VEC_SIZE(%rsi), %VEC(1) + cmpq $(VEC_SIZE * 4), %rdx + jbe L(last_4x_vec) +- /* Copy from 4 * VEC + 1 to 8 * VEC, inclusively. */ +- VMOVU (%rsi), %VEC(0) +- VMOVU VEC_SIZE(%rsi), %VEC(1) ++ /* Copy from 4 * VEC + 1 to 8 * VEC, inclusively. */ + VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2) + VMOVU (VEC_SIZE * 3)(%rsi), %VEC(3) +- VMOVU -VEC_SIZE(%rsi,%rdx), %VEC(4) +- VMOVU -(VEC_SIZE * 2)(%rsi,%rdx), %VEC(5) +- VMOVU -(VEC_SIZE * 3)(%rsi,%rdx), %VEC(6) +- VMOVU -(VEC_SIZE * 4)(%rsi,%rdx), %VEC(7) ++ VMOVU -VEC_SIZE(%rsi, %rdx), %VEC(4) ++ VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VEC(5) ++ VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VEC(6) ++ VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VEC(7) + VMOVU %VEC(0), (%rdi) + VMOVU %VEC(1), VEC_SIZE(%rdi) + VMOVU %VEC(2), (VEC_SIZE * 2)(%rdi) + VMOVU %VEC(3), (VEC_SIZE * 3)(%rdi) +- VMOVU %VEC(4), -VEC_SIZE(%rdi,%rdx) +- VMOVU %VEC(5), -(VEC_SIZE * 2)(%rdi,%rdx) +- VMOVU %VEC(6), -(VEC_SIZE * 3)(%rdi,%rdx) +- VMOVU %VEC(7), -(VEC_SIZE * 4)(%rdi,%rdx) +- VZEROUPPER_RETURN +-L(last_4x_vec): +- /* Copy from 2 * VEC + 1 to 4 * VEC, inclusively. */ +- VMOVU (%rsi), %VEC(0) +- VMOVU VEC_SIZE(%rsi), %VEC(1) +- VMOVU -VEC_SIZE(%rsi,%rdx), %VEC(2) +- VMOVU -(VEC_SIZE * 2)(%rsi,%rdx), %VEC(3) +- VMOVU %VEC(0), (%rdi) +- VMOVU %VEC(1), VEC_SIZE(%rdi) +- VMOVU %VEC(2), -VEC_SIZE(%rdi,%rdx) +- VMOVU %VEC(3), -(VEC_SIZE * 2)(%rdi,%rdx) ++ VMOVU %VEC(4), -VEC_SIZE(%rdi, %rdx) ++ VMOVU %VEC(5), -(VEC_SIZE * 2)(%rdi, %rdx) ++ VMOVU %VEC(6), -(VEC_SIZE * 3)(%rdi, %rdx) ++ VMOVU %VEC(7), -(VEC_SIZE * 4)(%rdi, %rdx) + VZEROUPPER_RETURN + ++ .p2align 4,, 4 + L(more_8x_vec): ++ movq %rdi, %rcx ++ subq %rsi, %rcx ++ /* Go to backwards temporal copy if overlap no matter what as ++ backward REP MOVSB is slow and we don't want to use NT stores if ++ there is overlap. */ ++ cmpq %rdx, %rcx ++ /* L(more_8x_vec_backward_check_nop) checks for src == dst. */ ++ jb L(more_8x_vec_backward_check_nop) + /* Check if non-temporal move candidate. */ + #if (defined USE_MULTIARCH || VEC_SIZE == 16) && IS_IN (libc) + /* Check non-temporal store threshold. */ +- cmp __x86_shared_non_temporal_threshold(%rip), %RDX_LP ++ cmp __x86_shared_non_temporal_threshold(%rip), %RDX_LP + ja L(large_memcpy_2x) + #endif +- /* Entry if rdx is greater than non-temporal threshold but there +- is overlap. */ ++ /* To reach this point there cannot be overlap and dst > src. So ++ check for overlap and src > dst in which case correctness ++ requires forward copy. Otherwise decide between backward/forward ++ copy depending on address aliasing. */ ++ ++ /* Entry if rdx is greater than __x86_rep_movsb_stop_threshold ++ but less than __x86_shared_non_temporal_threshold. */ + L(more_8x_vec_check): +- cmpq %rsi, %rdi +- ja L(more_8x_vec_backward) +- /* Source == destination is less common. */ +- je L(nop) +- /* Load the first VEC and last 4 * VEC to support overlapping +- addresses. */ +- VMOVU (%rsi), %VEC(4) ++ /* rcx contains dst - src. Add back length (rdx). */ ++ leaq (%rcx, %rdx), %r8 ++ /* If r8 has different sign than rcx then there is overlap so we ++ must do forward copy. */ ++ xorq %rcx, %r8 ++ /* Isolate just sign bit of r8. */ ++ shrq $63, %r8 ++ /* Get 4k difference dst - src. */ ++ andl $(PAGE_SIZE - 256), %ecx ++ /* If r8 is non-zero must do foward for correctness. Otherwise ++ if ecx is non-zero there is 4k False Alaising so do backward ++ copy. */ ++ addl %r8d, %ecx ++ jz L(more_8x_vec_backward) ++ ++ /* if rdx is greater than __x86_shared_non_temporal_threshold ++ but there is overlap, or from short distance movsb. */ ++L(more_8x_vec_forward): ++ /* Load first and last 4 * VEC to support overlapping addresses. ++ */ ++ ++ /* First vec was already loaded into VEC(0). */ + VMOVU -VEC_SIZE(%rsi, %rdx), %VEC(5) + VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VEC(6) ++ /* Save begining of dst. */ ++ movq %rdi, %rcx ++ /* Align dst to VEC_SIZE - 1. */ ++ orq $(VEC_SIZE - 1), %rdi + VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VEC(7) + VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VEC(8) +- /* Save start and stop of the destination buffer. */ +- movq %rdi, %r11 +- leaq -VEC_SIZE(%rdi, %rdx), %rcx +- /* Align destination for aligned stores in the loop. Compute +- how much destination is misaligned. */ +- movq %rdi, %r8 +- andq $(VEC_SIZE - 1), %r8 +- /* Get the negative of offset for alignment. */ +- subq $VEC_SIZE, %r8 +- /* Adjust source. */ +- subq %r8, %rsi +- /* Adjust destination which should be aligned now. */ +- subq %r8, %rdi +- /* Adjust length. */ +- addq %r8, %rdx + +- .p2align 4 ++ /* Subtract dst from src. Add back after dst aligned. */ ++ subq %rcx, %rsi ++ /* Finish aligning dst. */ ++ incq %rdi ++ /* Restore src adjusted with new value for aligned dst. */ ++ addq %rdi, %rsi ++ /* Store end of buffer minus tail in rdx. */ ++ leaq (VEC_SIZE * -4)(%rcx, %rdx), %rdx ++ ++ /* Dont use multi-byte nop to align. */ ++ .p2align 4,, 11 + L(loop_4x_vec_forward): + /* Copy 4 * VEC a time forward. */ +- VMOVU (%rsi), %VEC(0) +- VMOVU VEC_SIZE(%rsi), %VEC(1) +- VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2) +- VMOVU (VEC_SIZE * 3)(%rsi), %VEC(3) ++ VMOVU (%rsi), %VEC(1) ++ VMOVU VEC_SIZE(%rsi), %VEC(2) ++ VMOVU (VEC_SIZE * 2)(%rsi), %VEC(3) ++ VMOVU (VEC_SIZE * 3)(%rsi), %VEC(4) + subq $-(VEC_SIZE * 4), %rsi +- addq $-(VEC_SIZE * 4), %rdx +- VMOVA %VEC(0), (%rdi) +- VMOVA %VEC(1), VEC_SIZE(%rdi) +- VMOVA %VEC(2), (VEC_SIZE * 2)(%rdi) +- VMOVA %VEC(3), (VEC_SIZE * 3)(%rdi) ++ VMOVA %VEC(1), (%rdi) ++ VMOVA %VEC(2), VEC_SIZE(%rdi) ++ VMOVA %VEC(3), (VEC_SIZE * 2)(%rdi) ++ VMOVA %VEC(4), (VEC_SIZE * 3)(%rdi) + subq $-(VEC_SIZE * 4), %rdi +- cmpq $(VEC_SIZE * 4), %rdx ++ cmpq %rdi, %rdx + ja L(loop_4x_vec_forward) + /* Store the last 4 * VEC. */ +- VMOVU %VEC(5), (%rcx) +- VMOVU %VEC(6), -VEC_SIZE(%rcx) +- VMOVU %VEC(7), -(VEC_SIZE * 2)(%rcx) +- VMOVU %VEC(8), -(VEC_SIZE * 3)(%rcx) ++ VMOVU %VEC(5), (VEC_SIZE * 3)(%rdx) ++ VMOVU %VEC(6), (VEC_SIZE * 2)(%rdx) ++ VMOVU %VEC(7), VEC_SIZE(%rdx) ++ VMOVU %VEC(8), (%rdx) + /* Store the first VEC. */ +- VMOVU %VEC(4), (%r11) ++ VMOVU %VEC(0), (%rcx) ++ /* Keep L(nop_backward) target close to jmp for 2-byte encoding. ++ */ ++L(nop_backward): + VZEROUPPER_RETURN + ++ .p2align 4,, 8 ++L(more_8x_vec_backward_check_nop): ++ /* rcx contains dst - src. Test for dst == src to skip all of ++ memmove. */ ++ testq %rcx, %rcx ++ jz L(nop_backward) + L(more_8x_vec_backward): + /* Load the first 4 * VEC and last VEC to support overlapping + addresses. */ +- VMOVU (%rsi), %VEC(4) ++ ++ /* First vec was also loaded into VEC(0). */ + VMOVU VEC_SIZE(%rsi), %VEC(5) + VMOVU (VEC_SIZE * 2)(%rsi), %VEC(6) ++ /* Begining of region for 4x backward copy stored in rcx. */ ++ leaq (VEC_SIZE * -4 + -1)(%rdi, %rdx), %rcx + VMOVU (VEC_SIZE * 3)(%rsi), %VEC(7) +- VMOVU -VEC_SIZE(%rsi,%rdx), %VEC(8) +- /* Save stop of the destination buffer. */ +- leaq -VEC_SIZE(%rdi, %rdx), %r11 +- /* Align destination end for aligned stores in the loop. Compute +- how much destination end is misaligned. */ +- leaq -VEC_SIZE(%rsi, %rdx), %rcx +- movq %r11, %r9 +- movq %r11, %r8 +- andq $(VEC_SIZE - 1), %r8 +- /* Adjust source. */ +- subq %r8, %rcx +- /* Adjust the end of destination which should be aligned now. */ +- subq %r8, %r9 +- /* Adjust length. */ +- subq %r8, %rdx +- +- .p2align 4 ++ VMOVU -VEC_SIZE(%rsi, %rdx), %VEC(8) ++ /* Subtract dst from src. Add back after dst aligned. */ ++ subq %rdi, %rsi ++ /* Align dst. */ ++ andq $-(VEC_SIZE), %rcx ++ /* Restore src. */ ++ addq %rcx, %rsi ++ ++ /* Don't use multi-byte nop to align. */ ++ .p2align 4,, 11 + L(loop_4x_vec_backward): + /* Copy 4 * VEC a time backward. */ +- VMOVU (%rcx), %VEC(0) +- VMOVU -VEC_SIZE(%rcx), %VEC(1) +- VMOVU -(VEC_SIZE * 2)(%rcx), %VEC(2) +- VMOVU -(VEC_SIZE * 3)(%rcx), %VEC(3) +- addq $-(VEC_SIZE * 4), %rcx +- addq $-(VEC_SIZE * 4), %rdx +- VMOVA %VEC(0), (%r9) +- VMOVA %VEC(1), -VEC_SIZE(%r9) +- VMOVA %VEC(2), -(VEC_SIZE * 2)(%r9) +- VMOVA %VEC(3), -(VEC_SIZE * 3)(%r9) +- addq $-(VEC_SIZE * 4), %r9 +- cmpq $(VEC_SIZE * 4), %rdx +- ja L(loop_4x_vec_backward) ++ VMOVU (VEC_SIZE * 3)(%rsi), %VEC(1) ++ VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2) ++ VMOVU (VEC_SIZE * 1)(%rsi), %VEC(3) ++ VMOVU (VEC_SIZE * 0)(%rsi), %VEC(4) ++ addq $(VEC_SIZE * -4), %rsi ++ VMOVA %VEC(1), (VEC_SIZE * 3)(%rcx) ++ VMOVA %VEC(2), (VEC_SIZE * 2)(%rcx) ++ VMOVA %VEC(3), (VEC_SIZE * 1)(%rcx) ++ VMOVA %VEC(4), (VEC_SIZE * 0)(%rcx) ++ addq $(VEC_SIZE * -4), %rcx ++ cmpq %rcx, %rdi ++ jb L(loop_4x_vec_backward) + /* Store the first 4 * VEC. */ +- VMOVU %VEC(4), (%rdi) ++ VMOVU %VEC(0), (%rdi) + VMOVU %VEC(5), VEC_SIZE(%rdi) + VMOVU %VEC(6), (VEC_SIZE * 2)(%rdi) + VMOVU %VEC(7), (VEC_SIZE * 3)(%rdi) + /* Store the last VEC. */ +- VMOVU %VEC(8), (%r11) ++ VMOVU %VEC(8), -VEC_SIZE(%rdx, %rdi) ++ VZEROUPPER_RETURN ++ ++#if defined USE_MULTIARCH && IS_IN (libc) ++ /* L(skip_short_movsb_check) is only used with ERMS. Not for ++ FSRM. */ ++ .p2align 5,, 16 ++# if ALIGN_MOVSB ++L(skip_short_movsb_check): ++# if MOVSB_ALIGN_TO > VEC_SIZE ++ VMOVU VEC_SIZE(%rsi), %VEC(1) ++# endif ++# if MOVSB_ALIGN_TO > (VEC_SIZE * 2) ++# error Unsupported MOVSB_ALIGN_TO ++# endif ++ /* If CPU does not have FSRM two options for aligning. Align src ++ if dst and src 4k alias. Otherwise align dst. */ ++ testl $(PAGE_SIZE - 512), %ecx ++ jnz L(movsb_align_dst) ++ /* Fall through. dst and src 4k alias. It's better to align src ++ here because the bottleneck will be loads dues to the false ++ dependency on dst. */ ++ ++ /* rcx already has dst - src. */ ++ movq %rcx, %r9 ++ /* Add src to len. Subtract back after src aligned. -1 because ++ src is initially aligned to MOVSB_ALIGN_TO - 1. */ ++ leaq -1(%rsi, %rdx), %rcx ++ /* Inclusively align src to MOVSB_ALIGN_TO - 1. */ ++ orq $(MOVSB_ALIGN_TO - 1), %rsi ++ /* Restore dst and len adjusted with new values for aligned dst. ++ */ ++ leaq 1(%rsi, %r9), %rdi ++ subq %rsi, %rcx ++ /* Finish aligning src. */ ++ incq %rsi ++ ++ rep movsb ++ ++ VMOVU %VEC(0), (%r8) ++# if MOVSB_ALIGN_TO > VEC_SIZE ++ VMOVU %VEC(1), VEC_SIZE(%r8) ++# endif + VZEROUPPER_RETURN ++# endif ++ ++ .p2align 4,, 12 ++L(movsb): ++ movq %rdi, %rcx ++ subq %rsi, %rcx ++ /* Go to backwards temporal copy if overlap no matter what as ++ backward REP MOVSB is slow and we don't want to use NT stores if ++ there is overlap. */ ++ cmpq %rdx, %rcx ++ /* L(more_8x_vec_backward_check_nop) checks for src == dst. */ ++ jb L(more_8x_vec_backward_check_nop) ++# if ALIGN_MOVSB ++ /* Save dest for storing aligning VECs later. */ ++ movq %rdi, %r8 ++# endif ++ /* If above __x86_rep_movsb_stop_threshold most likely is ++ candidate for NT moves aswell. */ ++ cmp __x86_rep_movsb_stop_threshold(%rip), %RDX_LP ++ jae L(large_memcpy_2x_check) ++# if AVOID_SHORT_DISTANCE_REP_MOVSB || ALIGN_MOVSB ++ /* Only avoid short movsb if CPU has FSRM. */ ++ testl $X86_STRING_CONTROL_AVOID_SHORT_DISTANCE_REP_MOVSB, __x86_string_control(%rip) ++ jz L(skip_short_movsb_check) ++# if AVOID_SHORT_DISTANCE_REP_MOVSB ++ /* Avoid "rep movsb" if RCX, the distance between source and ++ destination, is N*4GB + [1..63] with N >= 0. */ ++ ++ /* ecx contains dst - src. Early check for backward copy ++ conditions means only case of slow movsb with src = dst + [0, ++ 63] is ecx in [-63, 0]. Use unsigned comparison with -64 check ++ for that case. */ ++ cmpl $-64, %ecx ++ ja L(more_8x_vec_forward) ++# endif ++# endif ++# if ALIGN_MOVSB ++# if MOVSB_ALIGN_TO > VEC_SIZE ++ VMOVU VEC_SIZE(%rsi), %VEC(1) ++# endif ++# if MOVSB_ALIGN_TO > (VEC_SIZE * 2) ++# error Unsupported MOVSB_ALIGN_TO ++# endif ++ /* Fall through means cpu has FSRM. In that case exclusively ++ align destination. */ ++L(movsb_align_dst): ++ /* Subtract dst from src. Add back after dst aligned. */ ++ subq %rdi, %rsi ++ /* Exclusively align dst to MOVSB_ALIGN_TO (64). */ ++ addq $(MOVSB_ALIGN_TO - 1), %rdi ++ /* Add dst to len. Subtract back after dst aligned. */ ++ leaq (%r8, %rdx), %rcx ++ /* Finish aligning dst. */ ++ andq $-(MOVSB_ALIGN_TO), %rdi ++ /* Restore src and len adjusted with new values for aligned dst. ++ */ ++ addq %rdi, %rsi ++ subq %rdi, %rcx ++ ++ rep movsb ++ ++ /* Store VECs loaded for aligning. */ ++ VMOVU %VEC(0), (%r8) ++# if MOVSB_ALIGN_TO > VEC_SIZE ++ VMOVU %VEC(1), VEC_SIZE(%r8) ++# endif ++ VZEROUPPER_RETURN ++# else /* !ALIGN_MOVSB. */ ++L(skip_short_movsb_check): ++ mov %RDX_LP, %RCX_LP ++ rep movsb ++ ret ++# endif ++#endif + ++ .p2align 4,, 10 + #if (defined USE_MULTIARCH || VEC_SIZE == 16) && IS_IN (libc) +- .p2align 4 ++L(large_memcpy_2x_check): ++ cmp __x86_rep_movsb_threshold(%rip), %RDX_LP ++ jb L(more_8x_vec_check) + L(large_memcpy_2x): +- /* Compute absolute value of difference between source and +- destination. */ +- movq %rdi, %r9 +- subq %rsi, %r9 +- movq %r9, %r8 +- leaq -1(%r9), %rcx +- sarq $63, %r8 +- xorq %r8, %r9 +- subq %r8, %r9 +- /* Don't use non-temporal store if there is overlap between +- destination and source since destination may be in cache when +- source is loaded. */ +- cmpq %r9, %rdx +- ja L(more_8x_vec_check) ++ /* To reach this point it is impossible for dst > src and ++ overlap. Remaining to check is src > dst and overlap. rcx ++ already contains dst - src. Negate rcx to get src - dst. If ++ length > rcx then there is overlap and forward copy is best. */ ++ negq %rcx ++ cmpq %rcx, %rdx ++ ja L(more_8x_vec_forward) + + /* Cache align destination. First store the first 64 bytes then + adjust alignments. */ +- VMOVU (%rsi), %VEC(8) +-#if VEC_SIZE < 64 +- VMOVU VEC_SIZE(%rsi), %VEC(9) +-#if VEC_SIZE < 32 +- VMOVU (VEC_SIZE * 2)(%rsi), %VEC(10) +- VMOVU (VEC_SIZE * 3)(%rsi), %VEC(11) +-#endif +-#endif +- VMOVU %VEC(8), (%rdi) +-#if VEC_SIZE < 64 +- VMOVU %VEC(9), VEC_SIZE(%rdi) +-#if VEC_SIZE < 32 +- VMOVU %VEC(10), (VEC_SIZE * 2)(%rdi) +- VMOVU %VEC(11), (VEC_SIZE * 3)(%rdi) +-#endif +-#endif ++ ++ /* First vec was also loaded into VEC(0). */ ++# if VEC_SIZE < 64 ++ VMOVU VEC_SIZE(%rsi), %VEC(1) ++# if VEC_SIZE < 32 ++ VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2) ++ VMOVU (VEC_SIZE * 3)(%rsi), %VEC(3) ++# endif ++# endif ++ VMOVU %VEC(0), (%rdi) ++# if VEC_SIZE < 64 ++ VMOVU %VEC(1), VEC_SIZE(%rdi) ++# if VEC_SIZE < 32 ++ VMOVU %VEC(2), (VEC_SIZE * 2)(%rdi) ++ VMOVU %VEC(3), (VEC_SIZE * 3)(%rdi) ++# endif ++# endif ++ + /* Adjust source, destination, and size. */ + movq %rdi, %r8 + andq $63, %r8 +@@ -614,9 +767,13 @@ L(large_memcpy_2x): + /* Adjust length. */ + addq %r8, %rdx + +- /* Test if source and destination addresses will alias. If they do +- the larger pipeline in large_memcpy_4x alleviated the ++ /* Test if source and destination addresses will alias. If they ++ do the larger pipeline in large_memcpy_4x alleviated the + performance drop. */ ++ ++ /* ecx contains -(dst - src). not ecx will return dst - src - 1 ++ which works for testing aliasing. */ ++ notl %ecx + testl $(PAGE_SIZE - VEC_SIZE * 8), %ecx + jz L(large_memcpy_4x) + +@@ -704,8 +861,8 @@ L(loop_large_memcpy_4x_outer): + /* ecx stores inner loop counter. */ + movl $(PAGE_SIZE / LARGE_LOAD_SIZE), %ecx + L(loop_large_memcpy_4x_inner): +- /* Only one prefetch set per page as doing 4 pages give more time +- for prefetcher to keep up. */ ++ /* Only one prefetch set per page as doing 4 pages give more ++ time for prefetcher to keep up. */ + PREFETCH_ONE_SET(1, (%rsi), PREFETCHED_LOAD_SIZE) + PREFETCH_ONE_SET(1, (%rsi), PAGE_SIZE + PREFETCHED_LOAD_SIZE) + PREFETCH_ONE_SET(1, (%rsi), PAGE_SIZE * 2 + PREFETCHED_LOAD_SIZE) +-- +GitLab + diff --git a/SOURCES/ia-opt-memset-vec-unaligned-erms.patch b/SOURCES/ia-opt-memset-vec-unaligned-erms.patch new file mode 100644 index 0000000..63f57dd --- /dev/null +++ b/SOURCES/ia-opt-memset-vec-unaligned-erms.patch @@ -0,0 +1,506 @@ +From 5deda2b73383bf16788cc83c8ea6262d89608263 Mon Sep 17 00:00:00 2001 +From: Noah Goldstein +Date: Mon, 20 Sep 2021 16:20:15 -0500 +Subject: [PATCH] x86: Optimize memset-vec-unaligned-erms.S + +No bug. + +Optimization are + +1. change control flow for L(more_2x_vec) to fall through to loop and + jump for L(less_4x_vec) and L(less_8x_vec). This uses less code + size and saves jumps for length > 4x VEC_SIZE. + +2. For EVEX/AVX512 move L(less_vec) closer to entry. + +3. Avoid complex address mode for length > 2x VEC_SIZE + +4. Slightly better aligning code for the loop from the perspective of + code size and uops. + +5. Align targets so they make full use of their fetch block and if + possible cache line. + +6. Try and reduce total number of icache lines that will need to be + pulled in for a given length. + +7. Include "local" version of stosb target. For AVX2/EVEX/AVX512 + jumping to the stosb target in the sse2 code section will almost + certainly be to a new page. The new version does increase code size + marginally by duplicating the target but should get better iTLB + behavior as a result. + +test-memset, test-wmemset, and test-bzero are all passing. + +Signed-off-by: Noah Goldstein +Reviewed-by: H.J. Lu +(cherry picked from commit e59ced238482fd71f3e493717f14f6507346741e) +--- + sysdeps/x86_64/memset.S | 10 +- + .../multiarch/memset-avx2-unaligned-erms.S | 10 +- + .../multiarch/memset-avx512-unaligned-erms.S | 11 +- + .../multiarch/memset-evex-unaligned-erms.S | 11 +- + .../multiarch/memset-vec-unaligned-erms.S | 285 ++++++++++++------ + 5 files changed, 232 insertions(+), 95 deletions(-) + +diff --git a/sysdeps/x86_64/memset.S b/sysdeps/x86_64/memset.S +index b3426795..8672b030 100644 +--- a/sysdeps/x86_64/memset.S ++++ b/sysdeps/x86_64/memset.S +@@ -18,13 +18,15 @@ + . */ + + #include ++#define USE_WITH_SSE2 1 + + #define VEC_SIZE 16 ++#define MOV_SIZE 3 ++#define RET_SIZE 1 ++ + #define VEC(i) xmm##i +-/* Don't use movups and movaps since it will get larger nop paddings for +- alignment. */ +-#define VMOVU movdqu +-#define VMOVA movdqa ++#define VMOVU movups ++#define VMOVA movaps + + #define MEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ + movd d, %xmm0; \ +diff --git a/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S +index ae0860f3..1af668af 100644 +--- a/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S ++++ b/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S +@@ -1,8 +1,14 @@ + #if IS_IN (libc) ++# define USE_WITH_AVX2 1 ++ + # define VEC_SIZE 32 ++# define MOV_SIZE 4 ++# define RET_SIZE 4 ++ + # define VEC(i) ymm##i +-# define VMOVU vmovdqu +-# define VMOVA vmovdqa ++ ++# define VMOVU vmovdqu ++# define VMOVA vmovdqa + + # define MEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ + vmovd d, %xmm0; \ +diff --git a/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S +index 8ad842fc..f14d6f84 100644 +--- a/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S ++++ b/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S +@@ -1,11 +1,18 @@ + #if IS_IN (libc) ++# define USE_WITH_AVX512 1 ++ + # define VEC_SIZE 64 ++# define MOV_SIZE 6 ++# define RET_SIZE 1 ++ + # define XMM0 xmm16 + # define YMM0 ymm16 + # define VEC0 zmm16 + # define VEC(i) VEC##i +-# define VMOVU vmovdqu64 +-# define VMOVA vmovdqa64 ++ ++# define VMOVU vmovdqu64 ++# define VMOVA vmovdqa64 ++ + # define VZEROUPPER + + # define MEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ +diff --git a/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S +index 640f0929..64b09e77 100644 +--- a/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S ++++ b/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S +@@ -1,11 +1,18 @@ + #if IS_IN (libc) ++# define USE_WITH_EVEX 1 ++ + # define VEC_SIZE 32 ++# define MOV_SIZE 6 ++# define RET_SIZE 1 ++ + # define XMM0 xmm16 + # define YMM0 ymm16 + # define VEC0 ymm16 + # define VEC(i) VEC##i +-# define VMOVU vmovdqu64 +-# define VMOVA vmovdqa64 ++ ++# define VMOVU vmovdqu64 ++# define VMOVA vmovdqa64 ++ + # define VZEROUPPER + + # define MEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ +diff --git a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S +index 909c33f6..f08b7323 100644 +--- a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S ++++ b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S +@@ -63,8 +63,27 @@ + # endif + #endif + ++#if VEC_SIZE == 64 ++# define LOOP_4X_OFFSET (VEC_SIZE * 4) ++#else ++# define LOOP_4X_OFFSET (0) ++#endif ++ ++#if defined USE_WITH_EVEX || defined USE_WITH_AVX512 ++# define END_REG rcx ++# define LOOP_REG rdi ++#else ++# define END_REG rdi ++# define LOOP_REG rdx ++#endif ++ + #define PAGE_SIZE 4096 + ++/* Macro to calculate size of small memset block for aligning ++ purposes. */ ++#define SMALL_MEMSET_ALIGN(mov_sz, ret_sz) (2 * (mov_sz) + (ret_sz) + 1) ++ ++ + #ifndef SECTION + # error SECTION is not defined! + #endif +@@ -74,6 +93,7 @@ + ENTRY (__bzero) + mov %RDI_LP, %RAX_LP /* Set return value. */ + mov %RSI_LP, %RDX_LP /* Set n. */ ++ xorl %esi, %esi + pxor %XMM0, %XMM0 + jmp L(entry_from_bzero) + END (__bzero) +@@ -158,7 +178,7 @@ ENTRY_CHK (MEMSET_CHK_SYMBOL (__memset_chk, unaligned_erms)) + END_CHK (MEMSET_CHK_SYMBOL (__memset_chk, unaligned_erms)) + # endif + +-ENTRY (MEMSET_SYMBOL (__memset, unaligned_erms)) ++ENTRY_P2ALIGN (MEMSET_SYMBOL (__memset, unaligned_erms), 6) + MEMSET_VDUP_TO_VEC0_AND_SET_RETURN (%esi, %rdi) + # ifdef __ILP32__ + /* Clear the upper 32 bits. */ +@@ -168,75 +188,43 @@ ENTRY (MEMSET_SYMBOL (__memset, unaligned_erms)) + jb L(less_vec) + cmp $(VEC_SIZE * 2), %RDX_LP + ja L(stosb_more_2x_vec) +- /* From VEC and to 2 * VEC. No branch when size == VEC_SIZE. */ +- VMOVU %VEC(0), -VEC_SIZE(%rdi,%rdx) +- VMOVU %VEC(0), (%rdi) ++ /* From VEC and to 2 * VEC. No branch when size == VEC_SIZE. ++ */ ++ VMOVU %VEC(0), (%rax) ++ VMOVU %VEC(0), -VEC_SIZE(%rax, %rdx) + VZEROUPPER_RETURN +- +- .p2align 4 +-L(stosb_more_2x_vec): +- cmp __x86_rep_stosb_threshold(%rip), %RDX_LP +- ja L(stosb) +-#else +- .p2align 4 + #endif +-L(more_2x_vec): +- /* Stores to first 2x VEC before cmp as any path forward will +- require it. */ +- VMOVU %VEC(0), (%rdi) +- VMOVU %VEC(0), VEC_SIZE(%rdi) +- cmpq $(VEC_SIZE * 4), %rdx +- ja L(loop_start) +- VMOVU %VEC(0), -(VEC_SIZE * 2)(%rdi,%rdx) +- VMOVU %VEC(0), -VEC_SIZE(%rdi,%rdx) +-L(return): +-#if VEC_SIZE > 16 +- ZERO_UPPER_VEC_REGISTERS_RETURN ++ ++ .p2align 4,, 10 ++L(last_2x_vec): ++#ifdef USE_LESS_VEC_MASK_STORE ++ VMOVU %VEC(0), (VEC_SIZE * 2 + LOOP_4X_OFFSET)(%rcx) ++ VMOVU %VEC(0), (VEC_SIZE * 3 + LOOP_4X_OFFSET)(%rcx) + #else +- ret ++ VMOVU %VEC(0), (VEC_SIZE * -2)(%rdi) ++ VMOVU %VEC(0), (VEC_SIZE * -1)(%rdi) + #endif ++ VZEROUPPER_RETURN + +-L(loop_start): +- VMOVU %VEC(0), (VEC_SIZE * 2)(%rdi) +- VMOVU %VEC(0), (VEC_SIZE * 3)(%rdi) +- cmpq $(VEC_SIZE * 8), %rdx +- jbe L(loop_end) +- andq $-(VEC_SIZE * 2), %rdi +- subq $-(VEC_SIZE * 4), %rdi +- leaq -(VEC_SIZE * 4)(%rax, %rdx), %rcx +- .p2align 4 +-L(loop): +- VMOVA %VEC(0), (%rdi) +- VMOVA %VEC(0), VEC_SIZE(%rdi) +- VMOVA %VEC(0), (VEC_SIZE * 2)(%rdi) +- VMOVA %VEC(0), (VEC_SIZE * 3)(%rdi) +- subq $-(VEC_SIZE * 4), %rdi +- cmpq %rcx, %rdi +- jb L(loop) +-L(loop_end): +- /* NB: rax is set as ptr in MEMSET_VDUP_TO_VEC0_AND_SET_RETURN. +- rdx as length is also unchanged. */ +- VMOVU %VEC(0), -(VEC_SIZE * 4)(%rax, %rdx) +- VMOVU %VEC(0), -(VEC_SIZE * 3)(%rax, %rdx) +- VMOVU %VEC(0), -(VEC_SIZE * 2)(%rax, %rdx) +- VMOVU %VEC(0), -VEC_SIZE(%rax, %rdx) +- VZEROUPPER_SHORT_RETURN +- +- .p2align 4 ++ /* If have AVX512 mask instructions put L(less_vec) close to ++ entry as it doesn't take much space and is likely a hot target. ++ */ ++#ifdef USE_LESS_VEC_MASK_STORE ++ .p2align 4,, 10 + L(less_vec): + /* Less than 1 VEC. */ + # if VEC_SIZE != 16 && VEC_SIZE != 32 && VEC_SIZE != 64 + # error Unsupported VEC_SIZE! + # endif +-# ifdef USE_LESS_VEC_MASK_STORE + /* Clear high bits from edi. Only keeping bits relevant to page + cross check. Note that we are using rax which is set in +- MEMSET_VDUP_TO_VEC0_AND_SET_RETURN as ptr from here on out. +- */ ++ MEMSET_VDUP_TO_VEC0_AND_SET_RETURN as ptr from here on out. */ + andl $(PAGE_SIZE - 1), %edi +- /* Check if VEC_SIZE store cross page. Mask stores suffer serious +- performance degradation when it has to fault supress. */ ++ /* Check if VEC_SIZE store cross page. Mask stores suffer ++ serious performance degradation when it has to fault supress. ++ */ + cmpl $(PAGE_SIZE - VEC_SIZE), %edi ++ /* This is generally considered a cold target. */ + ja L(cross_page) + # if VEC_SIZE > 32 + movq $-1, %rcx +@@ -247,58 +235,185 @@ L(less_vec): + bzhil %edx, %ecx, %ecx + kmovd %ecx, %k1 + # endif +- vmovdqu8 %VEC(0), (%rax) {%k1} ++ vmovdqu8 %VEC(0), (%rax){%k1} + VZEROUPPER_RETURN + ++# if defined USE_MULTIARCH && IS_IN (libc) ++ /* Include L(stosb_local) here if including L(less_vec) between ++ L(stosb_more_2x_vec) and ENTRY. This is to cache align the ++ L(stosb_more_2x_vec) target. */ ++ .p2align 4,, 10 ++L(stosb_local): ++ movzbl %sil, %eax ++ mov %RDX_LP, %RCX_LP ++ mov %RDI_LP, %RDX_LP ++ rep stosb ++ mov %RDX_LP, %RAX_LP ++ VZEROUPPER_RETURN ++# endif ++#endif ++ ++#if defined USE_MULTIARCH && IS_IN (libc) + .p2align 4 +-L(cross_page): ++L(stosb_more_2x_vec): ++ cmp __x86_rep_stosb_threshold(%rip), %RDX_LP ++ ja L(stosb_local) ++#endif ++ /* Fallthrough goes to L(loop_4x_vec). Tests for memset (2x, 4x] ++ and (4x, 8x] jump to target. */ ++L(more_2x_vec): ++ ++ /* Two different methods of setting up pointers / compare. The ++ two methods are based on the fact that EVEX/AVX512 mov ++ instructions take more bytes then AVX2/SSE2 mov instructions. As ++ well that EVEX/AVX512 machines also have fast LEA_BID. Both ++ setup and END_REG to avoid complex address mode. For EVEX/AVX512 ++ this saves code size and keeps a few targets in one fetch block. ++ For AVX2/SSE2 this helps prevent AGU bottlenecks. */ ++#if defined USE_WITH_EVEX || defined USE_WITH_AVX512 ++ /* If EVEX/AVX512 compute END_REG - (VEC_SIZE * 4 + ++ LOOP_4X_OFFSET) with LEA_BID. */ ++ ++ /* END_REG is rcx for EVEX/AVX512. */ ++ leaq -(VEC_SIZE * 4 + LOOP_4X_OFFSET)(%rdi, %rdx), %END_REG ++#endif ++ ++ /* Stores to first 2x VEC before cmp as any path forward will ++ require it. */ ++ VMOVU %VEC(0), (%rax) ++ VMOVU %VEC(0), VEC_SIZE(%rax) ++ ++ ++#if !(defined USE_WITH_EVEX || defined USE_WITH_AVX512) ++ /* If AVX2/SSE2 compute END_REG (rdi) with ALU. */ ++ addq %rdx, %END_REG ++#endif ++ ++ cmpq $(VEC_SIZE * 4), %rdx ++ jbe L(last_2x_vec) ++ ++ /* Store next 2x vec regardless. */ ++ VMOVU %VEC(0), (VEC_SIZE * 2)(%rax) ++ VMOVU %VEC(0), (VEC_SIZE * 3)(%rax) ++ ++ ++#if defined USE_WITH_EVEX || defined USE_WITH_AVX512 ++ /* If LOOP_4X_OFFSET don't readjust LOOP_REG (rdi), just add ++ extra offset to addresses in loop. Used for AVX512 to save space ++ as no way to get (VEC_SIZE * 4) in imm8. */ ++# if LOOP_4X_OFFSET == 0 ++ subq $-(VEC_SIZE * 4), %LOOP_REG + # endif +-# if VEC_SIZE > 32 +- cmpb $32, %dl +- jae L(between_32_63) ++ /* Avoid imm32 compare here to save code size. */ ++ cmpq %rdi, %rcx ++#else ++ addq $-(VEC_SIZE * 4), %END_REG ++ cmpq $(VEC_SIZE * 8), %rdx ++#endif ++ jbe L(last_4x_vec) ++#if !(defined USE_WITH_EVEX || defined USE_WITH_AVX512) ++ /* Set LOOP_REG (rdx). */ ++ leaq (VEC_SIZE * 4)(%rax), %LOOP_REG ++#endif ++ /* Align dst for loop. */ ++ andq $(VEC_SIZE * -2), %LOOP_REG ++ .p2align 4 ++L(loop): ++ VMOVA %VEC(0), LOOP_4X_OFFSET(%LOOP_REG) ++ VMOVA %VEC(0), (VEC_SIZE + LOOP_4X_OFFSET)(%LOOP_REG) ++ VMOVA %VEC(0), (VEC_SIZE * 2 + LOOP_4X_OFFSET)(%LOOP_REG) ++ VMOVA %VEC(0), (VEC_SIZE * 3 + LOOP_4X_OFFSET)(%LOOP_REG) ++ subq $-(VEC_SIZE * 4), %LOOP_REG ++ cmpq %END_REG, %LOOP_REG ++ jb L(loop) ++ .p2align 4,, MOV_SIZE ++L(last_4x_vec): ++ VMOVU %VEC(0), LOOP_4X_OFFSET(%END_REG) ++ VMOVU %VEC(0), (VEC_SIZE + LOOP_4X_OFFSET)(%END_REG) ++ VMOVU %VEC(0), (VEC_SIZE * 2 + LOOP_4X_OFFSET)(%END_REG) ++ VMOVU %VEC(0), (VEC_SIZE * 3 + LOOP_4X_OFFSET)(%END_REG) ++L(return): ++#if VEC_SIZE > 16 ++ ZERO_UPPER_VEC_REGISTERS_RETURN ++#else ++ ret ++#endif ++ ++ .p2align 4,, 10 ++#ifndef USE_LESS_VEC_MASK_STORE ++# if defined USE_MULTIARCH && IS_IN (libc) ++ /* If no USE_LESS_VEC_MASK put L(stosb_local) here. Will be in ++ range for 2-byte jump encoding. */ ++L(stosb_local): ++ movzbl %sil, %eax ++ mov %RDX_LP, %RCX_LP ++ mov %RDI_LP, %RDX_LP ++ rep stosb ++ mov %RDX_LP, %RAX_LP ++ VZEROUPPER_RETURN + # endif +-# if VEC_SIZE > 16 +- cmpb $16, %dl ++ /* Define L(less_vec) only if not otherwise defined. */ ++ .p2align 4 ++L(less_vec): ++#endif ++L(cross_page): ++#if VEC_SIZE > 32 ++ cmpl $32, %edx ++ jae L(between_32_63) ++#endif ++#if VEC_SIZE > 16 ++ cmpl $16, %edx + jae L(between_16_31) +-# endif +- MOVQ %XMM0, %rcx +- cmpb $8, %dl ++#endif ++ MOVQ %XMM0, %rdi ++ cmpl $8, %edx + jae L(between_8_15) +- cmpb $4, %dl ++ cmpl $4, %edx + jae L(between_4_7) +- cmpb $1, %dl ++ cmpl $1, %edx + ja L(between_2_3) +- jb 1f +- movb %cl, (%rax) +-1: ++ jb L(return) ++ movb %sil, (%rax) + VZEROUPPER_RETURN +-# if VEC_SIZE > 32 ++ ++ /* Align small targets only if not doing so would cross a fetch ++ line. */ ++#if VEC_SIZE > 32 ++ .p2align 4,, SMALL_MEMSET_ALIGN(MOV_SIZE, RET_SIZE) + /* From 32 to 63. No branch when size == 32. */ + L(between_32_63): +- VMOVU %YMM0, -32(%rax,%rdx) + VMOVU %YMM0, (%rax) ++ VMOVU %YMM0, -32(%rax, %rdx) + VZEROUPPER_RETURN +-# endif +-# if VEC_SIZE > 16 +- /* From 16 to 31. No branch when size == 16. */ ++#endif ++ ++#if VEC_SIZE >= 32 ++ .p2align 4,, SMALL_MEMSET_ALIGN(MOV_SIZE, RET_SIZE) + L(between_16_31): +- VMOVU %XMM0, -16(%rax,%rdx) ++ /* From 16 to 31. No branch when size == 16. */ + VMOVU %XMM0, (%rax) ++ VMOVU %XMM0, -16(%rax, %rdx) + VZEROUPPER_RETURN +-# endif +- /* From 8 to 15. No branch when size == 8. */ ++#endif ++ ++ .p2align 4,, SMALL_MEMSET_ALIGN(3, RET_SIZE) + L(between_8_15): +- movq %rcx, -8(%rax,%rdx) +- movq %rcx, (%rax) ++ /* From 8 to 15. No branch when size == 8. */ ++ movq %rdi, (%rax) ++ movq %rdi, -8(%rax, %rdx) + VZEROUPPER_RETURN ++ ++ .p2align 4,, SMALL_MEMSET_ALIGN(2, RET_SIZE) + L(between_4_7): + /* From 4 to 7. No branch when size == 4. */ +- movl %ecx, -4(%rax,%rdx) +- movl %ecx, (%rax) ++ movl %edi, (%rax) ++ movl %edi, -4(%rax, %rdx) + VZEROUPPER_RETURN ++ ++ .p2align 4,, SMALL_MEMSET_ALIGN(3, RET_SIZE) + L(between_2_3): + /* From 2 to 3. No branch when size == 2. */ +- movw %cx, -2(%rax,%rdx) +- movw %cx, (%rax) ++ movw %di, (%rax) ++ movb %dil, -1(%rax, %rdx) + VZEROUPPER_RETURN + END (MEMSET_SYMBOL (__memset, unaligned_erms)) +-- +GitLab + diff --git a/SOURCES/ia-opt-strchr-avx2.patch b/SOURCES/ia-opt-strchr-avx2.patch new file mode 100644 index 0000000..20978ce --- /dev/null +++ b/SOURCES/ia-opt-strchr-avx2.patch @@ -0,0 +1,392 @@ +From 93b1c47bd092f8e1444a10b5d6ec20e44d66459a Mon Sep 17 00:00:00 2001 +From: Noah Goldstein +Date: Fri, 23 Apr 2021 15:56:24 -0400 +Subject: [PATCH] x86: Optimize strchr-avx2.S + +No bug. This commit optimizes strchr-avx2.S. The optimizations are all +small things such as save an ALU in the alignment process, saving a +few instructions in the loop return, saving some bytes in the main +loop, and increasing the ILP in the return cases. test-strchr, +test-strchrnul, test-wcschr, and test-wcschrnul are all passing. + +Signed-off-by: Noah Goldstein +(cherry picked from commit ccabe7971f508709d034b63b8672f6f751a3d356) +--- + sysdeps/x86_64/multiarch/strchr-avx2.S | 290 +++++++++++++++---------- + 1 file changed, 170 insertions(+), 120 deletions(-) + +diff --git a/sysdeps/x86_64/multiarch/strchr-avx2.S b/sysdeps/x86_64/multiarch/strchr-avx2.S +index 919d256c..5884726b 100644 +--- a/sysdeps/x86_64/multiarch/strchr-avx2.S ++++ b/sysdeps/x86_64/multiarch/strchr-avx2.S +@@ -49,133 +49,144 @@ + + .section SECTION(.text),"ax",@progbits + ENTRY (STRCHR) +- movl %edi, %ecx +-# ifndef USE_AS_STRCHRNUL +- xorl %edx, %edx +-# endif +- + /* Broadcast CHAR to YMM0. */ + vmovd %esi, %xmm0 ++ movl %edi, %eax ++ andl $(PAGE_SIZE - 1), %eax ++ VPBROADCAST %xmm0, %ymm0 + vpxor %xmm9, %xmm9, %xmm9 +- VPBROADCAST %xmm0, %ymm0 + + /* Check if we cross page boundary with one vector load. */ +- andl $(PAGE_SIZE - 1), %ecx +- cmpl $(PAGE_SIZE - VEC_SIZE), %ecx +- ja L(cross_page_boundary) ++ cmpl $(PAGE_SIZE - VEC_SIZE), %eax ++ ja L(cross_page_boundary) + + /* Check the first VEC_SIZE bytes. Search for both CHAR and the + null byte. */ + vmovdqu (%rdi), %ymm8 +- VPCMPEQ %ymm8, %ymm0, %ymm1 +- VPCMPEQ %ymm8, %ymm9, %ymm2 ++ VPCMPEQ %ymm8, %ymm0, %ymm1 ++ VPCMPEQ %ymm8, %ymm9, %ymm2 + vpor %ymm1, %ymm2, %ymm1 + vpmovmskb %ymm1, %eax + testl %eax, %eax +- jz L(more_vecs) ++ jz L(aligned_more) + tzcntl %eax, %eax ++# ifndef USE_AS_STRCHRNUL + /* Found CHAR or the null byte. */ ++ cmp (%rdi, %rax), %CHAR_REG ++ jne L(zero) ++# endif + addq %rdi, %rax ++ VZEROUPPER_RETURN ++ ++ /* .p2align 5 helps keep performance more consistent if ENTRY() ++ alignment % 32 was either 16 or 0. As well this makes the ++ alignment % 32 of the loop_4x_vec fixed which makes tuning it ++ easier. */ ++ .p2align 5 ++L(first_vec_x4): ++ tzcntl %eax, %eax ++ addq $(VEC_SIZE * 3 + 1), %rdi + # ifndef USE_AS_STRCHRNUL +- cmp (%rax), %CHAR_REG +- cmovne %rdx, %rax ++ /* Found CHAR or the null byte. */ ++ cmp (%rdi, %rax), %CHAR_REG ++ jne L(zero) + # endif +-L(return_vzeroupper): +- ZERO_UPPER_VEC_REGISTERS_RETURN +- +- .p2align 4 +-L(more_vecs): +- /* Align data for aligned loads in the loop. */ +- andq $-VEC_SIZE, %rdi +-L(aligned_more): +- +- /* Check the next 4 * VEC_SIZE. Only one VEC_SIZE at a time +- since data is only aligned to VEC_SIZE. */ +- vmovdqa VEC_SIZE(%rdi), %ymm8 +- addq $VEC_SIZE, %rdi +- VPCMPEQ %ymm8, %ymm0, %ymm1 +- VPCMPEQ %ymm8, %ymm9, %ymm2 +- vpor %ymm1, %ymm2, %ymm1 +- vpmovmskb %ymm1, %eax +- testl %eax, %eax +- jnz L(first_vec_x0) +- +- vmovdqa VEC_SIZE(%rdi), %ymm8 +- VPCMPEQ %ymm8, %ymm0, %ymm1 +- VPCMPEQ %ymm8, %ymm9, %ymm2 +- vpor %ymm1, %ymm2, %ymm1 +- vpmovmskb %ymm1, %eax +- testl %eax, %eax +- jnz L(first_vec_x1) +- +- vmovdqa (VEC_SIZE * 2)(%rdi), %ymm8 +- VPCMPEQ %ymm8, %ymm0, %ymm1 +- VPCMPEQ %ymm8, %ymm9, %ymm2 +- vpor %ymm1, %ymm2, %ymm1 +- vpmovmskb %ymm1, %eax +- testl %eax, %eax +- jnz L(first_vec_x2) +- +- vmovdqa (VEC_SIZE * 3)(%rdi), %ymm8 +- VPCMPEQ %ymm8, %ymm0, %ymm1 +- VPCMPEQ %ymm8, %ymm9, %ymm2 +- vpor %ymm1, %ymm2, %ymm1 +- vpmovmskb %ymm1, %eax +- testl %eax, %eax +- jz L(prep_loop_4x) ++ addq %rdi, %rax ++ VZEROUPPER_RETURN + +- tzcntl %eax, %eax +- leaq (VEC_SIZE * 3)(%rdi, %rax), %rax + # ifndef USE_AS_STRCHRNUL +- cmp (%rax), %CHAR_REG +- cmovne %rdx, %rax ++L(zero): ++ xorl %eax, %eax ++ VZEROUPPER_RETURN + # endif +- VZEROUPPER +- ret ++ + + .p2align 4 +-L(first_vec_x0): ++L(first_vec_x1): + tzcntl %eax, %eax +- /* Found CHAR or the null byte. */ +- addq %rdi, %rax ++ incq %rdi + # ifndef USE_AS_STRCHRNUL +- cmp (%rax), %CHAR_REG +- cmovne %rdx, %rax ++ /* Found CHAR or the null byte. */ ++ cmp (%rdi, %rax), %CHAR_REG ++ jne L(zero) + # endif ++ addq %rdi, %rax + VZEROUPPER_RETURN + + .p2align 4 +-L(first_vec_x1): ++L(first_vec_x2): + tzcntl %eax, %eax +- leaq VEC_SIZE(%rdi, %rax), %rax ++ addq $(VEC_SIZE + 1), %rdi + # ifndef USE_AS_STRCHRNUL +- cmp (%rax), %CHAR_REG +- cmovne %rdx, %rax ++ /* Found CHAR or the null byte. */ ++ cmp (%rdi, %rax), %CHAR_REG ++ jne L(zero) + # endif ++ addq %rdi, %rax + VZEROUPPER_RETURN + + .p2align 4 +-L(first_vec_x2): ++L(first_vec_x3): + tzcntl %eax, %eax +- /* Found CHAR or the null byte. */ +- leaq (VEC_SIZE * 2)(%rdi, %rax), %rax ++ addq $(VEC_SIZE * 2 + 1), %rdi + # ifndef USE_AS_STRCHRNUL +- cmp (%rax), %CHAR_REG +- cmovne %rdx, %rax ++ /* Found CHAR or the null byte. */ ++ cmp (%rdi, %rax), %CHAR_REG ++ jne L(zero) + # endif ++ addq %rdi, %rax + VZEROUPPER_RETURN + +-L(prep_loop_4x): +- /* Align data to 4 * VEC_SIZE. */ +- andq $-(VEC_SIZE * 4), %rdi ++ .p2align 4 ++L(aligned_more): ++ /* Align data to VEC_SIZE - 1. This is the same number of ++ instructions as using andq -VEC_SIZE but saves 4 bytes of code ++ on x4 check. */ ++ orq $(VEC_SIZE - 1), %rdi ++L(cross_page_continue): ++ /* Check the next 4 * VEC_SIZE. Only one VEC_SIZE at a time ++ since data is only aligned to VEC_SIZE. */ ++ vmovdqa 1(%rdi), %ymm8 ++ VPCMPEQ %ymm8, %ymm0, %ymm1 ++ VPCMPEQ %ymm8, %ymm9, %ymm2 ++ vpor %ymm1, %ymm2, %ymm1 ++ vpmovmskb %ymm1, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x1) ++ ++ vmovdqa (VEC_SIZE + 1)(%rdi), %ymm8 ++ VPCMPEQ %ymm8, %ymm0, %ymm1 ++ VPCMPEQ %ymm8, %ymm9, %ymm2 ++ vpor %ymm1, %ymm2, %ymm1 ++ vpmovmskb %ymm1, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x2) ++ ++ vmovdqa (VEC_SIZE * 2 + 1)(%rdi), %ymm8 ++ VPCMPEQ %ymm8, %ymm0, %ymm1 ++ VPCMPEQ %ymm8, %ymm9, %ymm2 ++ vpor %ymm1, %ymm2, %ymm1 ++ vpmovmskb %ymm1, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x3) + ++ vmovdqa (VEC_SIZE * 3 + 1)(%rdi), %ymm8 ++ VPCMPEQ %ymm8, %ymm0, %ymm1 ++ VPCMPEQ %ymm8, %ymm9, %ymm2 ++ vpor %ymm1, %ymm2, %ymm1 ++ vpmovmskb %ymm1, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x4) ++ /* Align data to VEC_SIZE * 4 - 1. */ ++ addq $(VEC_SIZE * 4 + 1), %rdi ++ andq $-(VEC_SIZE * 4), %rdi + .p2align 4 + L(loop_4x_vec): + /* Compare 4 * VEC at a time forward. */ +- vmovdqa (VEC_SIZE * 4)(%rdi), %ymm5 +- vmovdqa (VEC_SIZE * 5)(%rdi), %ymm6 +- vmovdqa (VEC_SIZE * 6)(%rdi), %ymm7 +- vmovdqa (VEC_SIZE * 7)(%rdi), %ymm8 ++ vmovdqa (%rdi), %ymm5 ++ vmovdqa (VEC_SIZE)(%rdi), %ymm6 ++ vmovdqa (VEC_SIZE * 2)(%rdi), %ymm7 ++ vmovdqa (VEC_SIZE * 3)(%rdi), %ymm8 + + /* Leaves only CHARS matching esi as 0. */ + vpxor %ymm5, %ymm0, %ymm1 +@@ -191,63 +202,102 @@ L(loop_4x_vec): + VPMINU %ymm1, %ymm2, %ymm5 + VPMINU %ymm3, %ymm4, %ymm6 + +- VPMINU %ymm5, %ymm6, %ymm5 ++ VPMINU %ymm5, %ymm6, %ymm6 + +- VPCMPEQ %ymm5, %ymm9, %ymm5 +- vpmovmskb %ymm5, %eax ++ VPCMPEQ %ymm6, %ymm9, %ymm6 ++ vpmovmskb %ymm6, %ecx ++ subq $-(VEC_SIZE * 4), %rdi ++ testl %ecx, %ecx ++ jz L(loop_4x_vec) + +- addq $(VEC_SIZE * 4), %rdi +- testl %eax, %eax +- jz L(loop_4x_vec) + +- VPCMPEQ %ymm1, %ymm9, %ymm1 ++ VPCMPEQ %ymm1, %ymm9, %ymm1 + vpmovmskb %ymm1, %eax + testl %eax, %eax +- jnz L(first_vec_x0) ++ jnz L(last_vec_x0) ++ + +- VPCMPEQ %ymm2, %ymm9, %ymm2 ++ VPCMPEQ %ymm5, %ymm9, %ymm2 + vpmovmskb %ymm2, %eax + testl %eax, %eax +- jnz L(first_vec_x1) ++ jnz L(last_vec_x1) ++ ++ VPCMPEQ %ymm3, %ymm9, %ymm3 ++ vpmovmskb %ymm3, %eax ++ /* rcx has combined result from all 4 VEC. It will only be used ++ if the first 3 other VEC all did not contain a match. */ ++ salq $32, %rcx ++ orq %rcx, %rax ++ tzcntq %rax, %rax ++ subq $(VEC_SIZE * 2), %rdi ++# ifndef USE_AS_STRCHRNUL ++ /* Found CHAR or the null byte. */ ++ cmp (%rdi, %rax), %CHAR_REG ++ jne L(zero_end) ++# endif ++ addq %rdi, %rax ++ VZEROUPPER_RETURN ++ ++ ++ .p2align 4 ++L(last_vec_x0): ++ tzcntl %eax, %eax ++ addq $-(VEC_SIZE * 4), %rdi ++# ifndef USE_AS_STRCHRNUL ++ /* Found CHAR or the null byte. */ ++ cmp (%rdi, %rax), %CHAR_REG ++ jne L(zero_end) ++# endif ++ addq %rdi, %rax ++ VZEROUPPER_RETURN + +- VPCMPEQ %ymm3, %ymm9, %ymm3 +- VPCMPEQ %ymm4, %ymm9, %ymm4 +- vpmovmskb %ymm3, %ecx +- vpmovmskb %ymm4, %eax +- salq $32, %rax +- orq %rcx, %rax +- tzcntq %rax, %rax +- leaq (VEC_SIZE * 2)(%rdi, %rax), %rax + # ifndef USE_AS_STRCHRNUL +- cmp (%rax), %CHAR_REG +- cmovne %rdx, %rax ++L(zero_end): ++ xorl %eax, %eax ++ VZEROUPPER_RETURN + # endif +- VZEROUPPER +- ret ++ ++ .p2align 4 ++L(last_vec_x1): ++ tzcntl %eax, %eax ++ subq $(VEC_SIZE * 3), %rdi ++# ifndef USE_AS_STRCHRNUL ++ /* Found CHAR or the null byte. */ ++ cmp (%rdi, %rax), %CHAR_REG ++ jne L(zero_end) ++# endif ++ addq %rdi, %rax ++ VZEROUPPER_RETURN ++ + + /* Cold case for crossing page with first load. */ + .p2align 4 + L(cross_page_boundary): +- andq $-VEC_SIZE, %rdi +- andl $(VEC_SIZE - 1), %ecx +- +- vmovdqa (%rdi), %ymm8 +- VPCMPEQ %ymm8, %ymm0, %ymm1 +- VPCMPEQ %ymm8, %ymm9, %ymm2 ++ movq %rdi, %rdx ++ /* Align rdi to VEC_SIZE - 1. */ ++ orq $(VEC_SIZE - 1), %rdi ++ vmovdqa -(VEC_SIZE - 1)(%rdi), %ymm8 ++ VPCMPEQ %ymm8, %ymm0, %ymm1 ++ VPCMPEQ %ymm8, %ymm9, %ymm2 + vpor %ymm1, %ymm2, %ymm1 + vpmovmskb %ymm1, %eax +- /* Remove the leading bits. */ +- sarxl %ecx, %eax, %eax ++ /* Remove the leading bytes. sarxl only uses bits [5:0] of COUNT ++ so no need to manually mod edx. */ ++ sarxl %edx, %eax, %eax + testl %eax, %eax +- jz L(aligned_more) ++ jz L(cross_page_continue) + tzcntl %eax, %eax +- addq %rcx, %rdi +- addq %rdi, %rax + # ifndef USE_AS_STRCHRNUL +- cmp (%rax), %CHAR_REG +- cmovne %rdx, %rax ++ xorl %ecx, %ecx ++ /* Found CHAR or the null byte. */ ++ cmp (%rdx, %rax), %CHAR_REG ++ leaq (%rdx, %rax), %rax ++ cmovne %rcx, %rax ++# else ++ addq %rdx, %rax + # endif +- VZEROUPPER_RETURN ++L(return_vzeroupper): ++ ZERO_UPPER_VEC_REGISTERS_RETURN + + END (STRCHR) + # endif +-- +GitLab + diff --git a/SOURCES/ia-opt-strchr-evex.patch b/SOURCES/ia-opt-strchr-evex.patch new file mode 100644 index 0000000..faf7223 --- /dev/null +++ b/SOURCES/ia-opt-strchr-evex.patch @@ -0,0 +1,532 @@ +From c17aa053d5b26520fddad8bfb590b521cb027280 Mon Sep 17 00:00:00 2001 +From: Noah Goldstein +Date: Fri, 23 Apr 2021 15:56:25 -0400 +Subject: [PATCH] x86: Optimize strchr-evex.S + +No bug. This commit optimizes strchr-evex.S. The optimizations are +mostly small things such as save an ALU in the alignment process, +saving a few instructions in the loop return. The one significant +change is saving 2 instructions in the 4x loop. test-strchr, +test-strchrnul, test-wcschr, and test-wcschrnul are all passing. + +Signed-off-by: Noah Goldstein +(cherry picked from commit 7f3e7c262cab4e2401e4331a6ef29c428de02044) +--- + sysdeps/x86_64/multiarch/strchr-evex.S | 392 ++++++++++++++----------- + 1 file changed, 218 insertions(+), 174 deletions(-) + +diff --git a/sysdeps/x86_64/multiarch/strchr-evex.S b/sysdeps/x86_64/multiarch/strchr-evex.S +index ddc86a70..7f9d4ee4 100644 +--- a/sysdeps/x86_64/multiarch/strchr-evex.S ++++ b/sysdeps/x86_64/multiarch/strchr-evex.S +@@ -32,13 +32,15 @@ + # define VPCMP vpcmpd + # define VPMINU vpminud + # define CHAR_REG esi +-# define SHIFT_REG r8d ++# define SHIFT_REG ecx ++# define CHAR_SIZE 4 + # else + # define VPBROADCAST vpbroadcastb + # define VPCMP vpcmpb + # define VPMINU vpminub + # define CHAR_REG sil +-# define SHIFT_REG ecx ++# define SHIFT_REG edx ++# define CHAR_SIZE 1 + # endif + + # define XMMZERO xmm16 +@@ -56,23 +58,20 @@ + + # define VEC_SIZE 32 + # define PAGE_SIZE 4096 ++# define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE) + + .section .text.evex,"ax",@progbits + ENTRY (STRCHR) +- movl %edi, %ecx +-# ifndef USE_AS_STRCHRNUL +- xorl %edx, %edx +-# endif +- + /* Broadcast CHAR to YMM0. */ +- VPBROADCAST %esi, %YMM0 +- ++ VPBROADCAST %esi, %YMM0 ++ movl %edi, %eax ++ andl $(PAGE_SIZE - 1), %eax + vpxorq %XMMZERO, %XMMZERO, %XMMZERO + +- /* Check if we cross page boundary with one vector load. */ +- andl $(PAGE_SIZE - 1), %ecx +- cmpl $(PAGE_SIZE - VEC_SIZE), %ecx +- ja L(cross_page_boundary) ++ /* Check if we cross page boundary with one vector load. ++ Otherwise it is safe to use an unaligned load. */ ++ cmpl $(PAGE_SIZE - VEC_SIZE), %eax ++ ja L(cross_page_boundary) + + /* Check the first VEC_SIZE bytes. Search for both CHAR and the + null bytes. */ +@@ -83,251 +82,296 @@ ENTRY (STRCHR) + VPMINU %YMM2, %YMM1, %YMM2 + /* Each bit in K0 represents a CHAR or a null byte in YMM1. */ + VPCMP $0, %YMMZERO, %YMM2, %k0 +- ktestd %k0, %k0 +- jz L(more_vecs) + kmovd %k0, %eax ++ testl %eax, %eax ++ jz L(aligned_more) + tzcntl %eax, %eax +- /* Found CHAR or the null byte. */ + # ifdef USE_AS_WCSCHR +- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ +- leaq (%rdi, %rax, 4), %rax ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. ++ */ ++ leaq (%rdi, %rax, CHAR_SIZE), %rax + # else + addq %rdi, %rax + # endif + # ifndef USE_AS_STRCHRNUL +- cmp (%rax), %CHAR_REG +- cmovne %rdx, %rax ++ /* Found CHAR or the null byte. */ ++ cmp (%rax), %CHAR_REG ++ jne L(zero) + # endif + ret + +- .p2align 4 +-L(more_vecs): +- /* Align data for aligned loads in the loop. */ +- andq $-VEC_SIZE, %rdi +-L(aligned_more): +- +- /* Check the next 4 * VEC_SIZE. Only one VEC_SIZE at a time +- since data is only aligned to VEC_SIZE. */ +- VMOVA VEC_SIZE(%rdi), %YMM1 +- addq $VEC_SIZE, %rdi +- +- /* Leaves only CHARS matching esi as 0. */ +- vpxorq %YMM1, %YMM0, %YMM2 +- VPMINU %YMM2, %YMM1, %YMM2 +- /* Each bit in K0 represents a CHAR or a null byte in YMM1. */ +- VPCMP $0, %YMMZERO, %YMM2, %k0 +- kmovd %k0, %eax +- testl %eax, %eax +- jnz L(first_vec_x0) +- +- VMOVA VEC_SIZE(%rdi), %YMM1 +- /* Leaves only CHARS matching esi as 0. */ +- vpxorq %YMM1, %YMM0, %YMM2 +- VPMINU %YMM2, %YMM1, %YMM2 +- /* Each bit in K0 represents a CHAR or a null byte in YMM1. */ +- VPCMP $0, %YMMZERO, %YMM2, %k0 +- kmovd %k0, %eax +- testl %eax, %eax +- jnz L(first_vec_x1) +- +- VMOVA (VEC_SIZE * 2)(%rdi), %YMM1 +- /* Leaves only CHARS matching esi as 0. */ +- vpxorq %YMM1, %YMM0, %YMM2 +- VPMINU %YMM2, %YMM1, %YMM2 +- /* Each bit in K0 represents a CHAR or a null byte in YMM1. */ +- VPCMP $0, %YMMZERO, %YMM2, %k0 +- kmovd %k0, %eax +- testl %eax, %eax +- jnz L(first_vec_x2) +- +- VMOVA (VEC_SIZE * 3)(%rdi), %YMM1 +- /* Leaves only CHARS matching esi as 0. */ +- vpxorq %YMM1, %YMM0, %YMM2 +- VPMINU %YMM2, %YMM1, %YMM2 +- /* Each bit in K0 represents a CHAR or a null byte in YMM1. */ +- VPCMP $0, %YMMZERO, %YMM2, %k0 +- ktestd %k0, %k0 +- jz L(prep_loop_4x) +- +- kmovd %k0, %eax ++ /* .p2align 5 helps keep performance more consistent if ENTRY() ++ alignment % 32 was either 16 or 0. As well this makes the ++ alignment % 32 of the loop_4x_vec fixed which makes tuning it ++ easier. */ ++ .p2align 5 ++L(first_vec_x3): + tzcntl %eax, %eax ++# ifndef USE_AS_STRCHRNUL + /* Found CHAR or the null byte. */ +-# ifdef USE_AS_WCSCHR +- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ +- leaq (VEC_SIZE * 3)(%rdi, %rax, 4), %rax +-# else +- leaq (VEC_SIZE * 3)(%rdi, %rax), %rax ++ cmp (VEC_SIZE * 3)(%rdi, %rax, CHAR_SIZE), %CHAR_REG ++ jne L(zero) + # endif ++ /* NB: Multiply sizeof char type (1 or 4) to get the number of ++ bytes. */ ++ leaq (VEC_SIZE * 3)(%rdi, %rax, CHAR_SIZE), %rax ++ ret ++ + # ifndef USE_AS_STRCHRNUL +- cmp (%rax), %CHAR_REG +- cmovne %rdx, %rax +-# endif ++L(zero): ++ xorl %eax, %eax + ret ++# endif + + .p2align 4 +-L(first_vec_x0): ++L(first_vec_x4): ++# ifndef USE_AS_STRCHRNUL ++ /* Check to see if first match was CHAR (k0) or null (k1). */ ++ kmovd %k0, %eax + tzcntl %eax, %eax +- /* Found CHAR or the null byte. */ +-# ifdef USE_AS_WCSCHR +- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ +- leaq (%rdi, %rax, 4), %rax ++ kmovd %k1, %ecx ++ /* bzhil will not be 0 if first match was null. */ ++ bzhil %eax, %ecx, %ecx ++ jne L(zero) + # else +- addq %rdi, %rax +-# endif +-# ifndef USE_AS_STRCHRNUL +- cmp (%rax), %CHAR_REG +- cmovne %rdx, %rax ++ /* Combine CHAR and null matches. */ ++ kord %k0, %k1, %k0 ++ kmovd %k0, %eax ++ tzcntl %eax, %eax + # endif ++ /* NB: Multiply sizeof char type (1 or 4) to get the number of ++ bytes. */ ++ leaq (VEC_SIZE * 4)(%rdi, %rax, CHAR_SIZE), %rax + ret + + .p2align 4 + L(first_vec_x1): + tzcntl %eax, %eax +- /* Found CHAR or the null byte. */ +-# ifdef USE_AS_WCSCHR +- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ +- leaq VEC_SIZE(%rdi, %rax, 4), %rax +-# else +- leaq VEC_SIZE(%rdi, %rax), %rax +-# endif + # ifndef USE_AS_STRCHRNUL +- cmp (%rax), %CHAR_REG +- cmovne %rdx, %rax ++ /* Found CHAR or the null byte. */ ++ cmp (VEC_SIZE)(%rdi, %rax, CHAR_SIZE), %CHAR_REG ++ jne L(zero) ++ + # endif ++ /* NB: Multiply sizeof char type (1 or 4) to get the number of ++ bytes. */ ++ leaq (VEC_SIZE)(%rdi, %rax, CHAR_SIZE), %rax + ret + + .p2align 4 + L(first_vec_x2): ++# ifndef USE_AS_STRCHRNUL ++ /* Check to see if first match was CHAR (k0) or null (k1). */ ++ kmovd %k0, %eax + tzcntl %eax, %eax +- /* Found CHAR or the null byte. */ +-# ifdef USE_AS_WCSCHR +- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ +- leaq (VEC_SIZE * 2)(%rdi, %rax, 4), %rax ++ kmovd %k1, %ecx ++ /* bzhil will not be 0 if first match was null. */ ++ bzhil %eax, %ecx, %ecx ++ jne L(zero) + # else +- leaq (VEC_SIZE * 2)(%rdi, %rax), %rax +-# endif +-# ifndef USE_AS_STRCHRNUL +- cmp (%rax), %CHAR_REG +- cmovne %rdx, %rax ++ /* Combine CHAR and null matches. */ ++ kord %k0, %k1, %k0 ++ kmovd %k0, %eax ++ tzcntl %eax, %eax + # endif ++ /* NB: Multiply sizeof char type (1 or 4) to get the number of ++ bytes. */ ++ leaq (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %rax + ret + +-L(prep_loop_4x): +- /* Align data to 4 * VEC_SIZE. */ ++ .p2align 4 ++L(aligned_more): ++ /* Align data to VEC_SIZE. */ ++ andq $-VEC_SIZE, %rdi ++L(cross_page_continue): ++ /* Check the next 4 * VEC_SIZE. Only one VEC_SIZE at a time since ++ data is only aligned to VEC_SIZE. Use two alternating methods ++ for checking VEC to balance latency and port contention. */ ++ ++ /* This method has higher latency but has better port ++ distribution. */ ++ VMOVA (VEC_SIZE)(%rdi), %YMM1 ++ /* Leaves only CHARS matching esi as 0. */ ++ vpxorq %YMM1, %YMM0, %YMM2 ++ VPMINU %YMM2, %YMM1, %YMM2 ++ /* Each bit in K0 represents a CHAR or a null byte in YMM1. */ ++ VPCMP $0, %YMMZERO, %YMM2, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x1) ++ ++ /* This method has higher latency but has better port ++ distribution. */ ++ VMOVA (VEC_SIZE * 2)(%rdi), %YMM1 ++ /* Each bit in K0 represents a CHAR in YMM1. */ ++ VPCMP $0, %YMM1, %YMM0, %k0 ++ /* Each bit in K1 represents a CHAR in YMM1. */ ++ VPCMP $0, %YMM1, %YMMZERO, %k1 ++ kortestd %k0, %k1 ++ jnz L(first_vec_x2) ++ ++ VMOVA (VEC_SIZE * 3)(%rdi), %YMM1 ++ /* Leaves only CHARS matching esi as 0. */ ++ vpxorq %YMM1, %YMM0, %YMM2 ++ VPMINU %YMM2, %YMM1, %YMM2 ++ /* Each bit in K0 represents a CHAR or a null byte in YMM1. */ ++ VPCMP $0, %YMMZERO, %YMM2, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x3) ++ ++ VMOVA (VEC_SIZE * 4)(%rdi), %YMM1 ++ /* Each bit in K0 represents a CHAR in YMM1. */ ++ VPCMP $0, %YMM1, %YMM0, %k0 ++ /* Each bit in K1 represents a CHAR in YMM1. */ ++ VPCMP $0, %YMM1, %YMMZERO, %k1 ++ kortestd %k0, %k1 ++ jnz L(first_vec_x4) ++ ++ /* Align data to VEC_SIZE * 4 for the loop. */ ++ addq $VEC_SIZE, %rdi + andq $-(VEC_SIZE * 4), %rdi + + .p2align 4 + L(loop_4x_vec): +- /* Compare 4 * VEC at a time forward. */ ++ /* Check 4x VEC at a time. No penalty to imm32 offset with evex ++ encoding. */ + VMOVA (VEC_SIZE * 4)(%rdi), %YMM1 + VMOVA (VEC_SIZE * 5)(%rdi), %YMM2 + VMOVA (VEC_SIZE * 6)(%rdi), %YMM3 + VMOVA (VEC_SIZE * 7)(%rdi), %YMM4 + +- /* Leaves only CHARS matching esi as 0. */ ++ /* For YMM1 and YMM3 use xor to set the CHARs matching esi to ++ zero. */ + vpxorq %YMM1, %YMM0, %YMM5 +- vpxorq %YMM2, %YMM0, %YMM6 ++ /* For YMM2 and YMM4 cmp not equals to CHAR and store result in ++ k register. Its possible to save either 1 or 2 instructions ++ using cmp no equals method for either YMM1 or YMM1 and YMM3 ++ respectively but bottleneck on p5 makes it not worth it. */ ++ VPCMP $4, %YMM0, %YMM2, %k2 + vpxorq %YMM3, %YMM0, %YMM7 +- vpxorq %YMM4, %YMM0, %YMM8 +- +- VPMINU %YMM5, %YMM1, %YMM5 +- VPMINU %YMM6, %YMM2, %YMM6 +- VPMINU %YMM7, %YMM3, %YMM7 +- VPMINU %YMM8, %YMM4, %YMM8 +- +- VPMINU %YMM5, %YMM6, %YMM1 +- VPMINU %YMM7, %YMM8, %YMM2 +- +- VPMINU %YMM1, %YMM2, %YMM1 +- +- /* Each bit in K0 represents a CHAR or a null byte. */ +- VPCMP $0, %YMMZERO, %YMM1, %k0 +- +- addq $(VEC_SIZE * 4), %rdi +- +- ktestd %k0, %k0 ++ VPCMP $4, %YMM0, %YMM4, %k4 ++ ++ /* Use min to select all zeros from either xor or end of string). ++ */ ++ VPMINU %YMM1, %YMM5, %YMM1 ++ VPMINU %YMM3, %YMM7, %YMM3 ++ ++ /* Use min + zeromask to select for zeros. Since k2 and k4 will ++ have 0 as positions that matched with CHAR which will set ++ zero in the corresponding destination bytes in YMM2 / YMM4. ++ */ ++ VPMINU %YMM1, %YMM2, %YMM2{%k2}{z} ++ VPMINU %YMM3, %YMM4, %YMM4 ++ VPMINU %YMM2, %YMM4, %YMM4{%k4}{z} ++ ++ VPCMP $0, %YMMZERO, %YMM4, %k1 ++ kmovd %k1, %ecx ++ subq $-(VEC_SIZE * 4), %rdi ++ testl %ecx, %ecx + jz L(loop_4x_vec) + +- /* Each bit in K0 represents a CHAR or a null byte in YMM1. */ +- VPCMP $0, %YMMZERO, %YMM5, %k0 ++ VPCMP $0, %YMMZERO, %YMM1, %k0 + kmovd %k0, %eax + testl %eax, %eax +- jnz L(first_vec_x0) ++ jnz L(last_vec_x1) + +- /* Each bit in K1 represents a CHAR or a null byte in YMM2. */ +- VPCMP $0, %YMMZERO, %YMM6, %k1 +- kmovd %k1, %eax ++ VPCMP $0, %YMMZERO, %YMM2, %k0 ++ kmovd %k0, %eax + testl %eax, %eax +- jnz L(first_vec_x1) +- +- /* Each bit in K2 represents a CHAR or a null byte in YMM3. */ +- VPCMP $0, %YMMZERO, %YMM7, %k2 +- /* Each bit in K3 represents a CHAR or a null byte in YMM4. */ +- VPCMP $0, %YMMZERO, %YMM8, %k3 ++ jnz L(last_vec_x2) + ++ VPCMP $0, %YMMZERO, %YMM3, %k0 ++ kmovd %k0, %eax ++ /* Combine YMM3 matches (eax) with YMM4 matches (ecx). */ + # ifdef USE_AS_WCSCHR +- /* NB: Each bit in K2/K3 represents 4-byte element. */ +- kshiftlw $8, %k3, %k1 ++ sall $8, %ecx ++ orl %ecx, %eax ++ tzcntl %eax, %eax + # else +- kshiftlq $32, %k3, %k1 ++ salq $32, %rcx ++ orq %rcx, %rax ++ tzcntq %rax, %rax + # endif ++# ifndef USE_AS_STRCHRNUL ++ /* Check if match was CHAR or null. */ ++ cmp (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %CHAR_REG ++ jne L(zero_end) ++# endif ++ /* NB: Multiply sizeof char type (1 or 4) to get the number of ++ bytes. */ ++ leaq (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %rax ++ ret + +- /* Each bit in K1 represents a NULL or a mismatch. */ +- korq %k1, %k2, %k1 +- kmovq %k1, %rax ++# ifndef USE_AS_STRCHRNUL ++L(zero_end): ++ xorl %eax, %eax ++ ret ++# endif + +- tzcntq %rax, %rax +-# ifdef USE_AS_WCSCHR +- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ +- leaq (VEC_SIZE * 2)(%rdi, %rax, 4), %rax +-# else +- leaq (VEC_SIZE * 2)(%rdi, %rax), %rax ++ .p2align 4 ++L(last_vec_x1): ++ tzcntl %eax, %eax ++# ifndef USE_AS_STRCHRNUL ++ /* Check if match was null. */ ++ cmp (%rdi, %rax, CHAR_SIZE), %CHAR_REG ++ jne L(zero_end) + # endif ++ /* NB: Multiply sizeof char type (1 or 4) to get the number of ++ bytes. */ ++ leaq (%rdi, %rax, CHAR_SIZE), %rax ++ ret ++ ++ .p2align 4 ++L(last_vec_x2): ++ tzcntl %eax, %eax + # ifndef USE_AS_STRCHRNUL +- cmp (%rax), %CHAR_REG +- cmovne %rdx, %rax ++ /* Check if match was null. */ ++ cmp (VEC_SIZE)(%rdi, %rax, CHAR_SIZE), %CHAR_REG ++ jne L(zero_end) + # endif ++ /* NB: Multiply sizeof char type (1 or 4) to get the number of ++ bytes. */ ++ leaq (VEC_SIZE)(%rdi, %rax, CHAR_SIZE), %rax + ret + + /* Cold case for crossing page with first load. */ + .p2align 4 + L(cross_page_boundary): ++ movq %rdi, %rdx ++ /* Align rdi. */ + andq $-VEC_SIZE, %rdi +- andl $(VEC_SIZE - 1), %ecx +- + VMOVA (%rdi), %YMM1 +- + /* Leaves only CHARS matching esi as 0. */ + vpxorq %YMM1, %YMM0, %YMM2 + VPMINU %YMM2, %YMM1, %YMM2 + /* Each bit in K0 represents a CHAR or a null byte in YMM1. */ + VPCMP $0, %YMMZERO, %YMM2, %k0 + kmovd %k0, %eax +- testl %eax, %eax +- ++ /* Remove the leading bits. */ + # ifdef USE_AS_WCSCHR ++ movl %edx, %SHIFT_REG + /* NB: Divide shift count by 4 since each bit in K1 represent 4 + bytes. */ +- movl %ecx, %SHIFT_REG +- sarl $2, %SHIFT_REG ++ sarl $2, %SHIFT_REG ++ andl $(CHAR_PER_VEC - 1), %SHIFT_REG + # endif +- +- /* Remove the leading bits. */ + sarxl %SHIFT_REG, %eax, %eax ++ /* If eax is zero continue. */ + testl %eax, %eax +- +- jz L(aligned_more) ++ jz L(cross_page_continue) + tzcntl %eax, %eax +- addq %rcx, %rdi ++# ifndef USE_AS_STRCHRNUL ++ /* Check to see if match was CHAR or null. */ ++ cmp (%rdx, %rax, CHAR_SIZE), %CHAR_REG ++ jne L(zero_end) ++# endif + # ifdef USE_AS_WCSCHR +- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ +- leaq (%rdi, %rax, 4), %rax ++ /* NB: Multiply wchar_t count by 4 to get the number of ++ bytes. */ ++ leaq (%rdx, %rax, CHAR_SIZE), %rax + # else +- addq %rdi, %rax +-# endif +-# ifndef USE_AS_STRCHRNUL +- cmp (%rax), %CHAR_REG +- cmovne %rdx, %rax ++ addq %rdx, %rax + # endif + ret + +-- +GitLab + diff --git a/SOURCES/ia-opt-strlen-evex.patch b/SOURCES/ia-opt-strlen-evex.patch new file mode 100644 index 0000000..ce1a1af --- /dev/null +++ b/SOURCES/ia-opt-strlen-evex.patch @@ -0,0 +1,745 @@ +From 417f10b43cd3a0bc5c67b0b5151e92b722bdd8d7 Mon Sep 17 00:00:00 2001 +From: Noah Goldstein +Date: Mon, 19 Apr 2021 19:36:06 -0400 +Subject: [PATCH] x86: Optimize strlen-evex.S + +No bug. This commit optimizes strlen-evex.S. The +optimizations are mostly small things but they add up to roughly +10-30% performance improvement for strlen. The results for strnlen are +bit more ambiguous. test-strlen, test-strnlen, test-wcslen, and +test-wcsnlen are all passing. + +Signed-off-by: Noah Goldstein +(cherry picked from commit 4ba65586847751372520a36757c17f114588794e) +--- + sysdeps/x86_64/multiarch/strlen-evex.S | 581 ++++++++++++++----------- + 1 file changed, 317 insertions(+), 264 deletions(-) + +diff --git a/sysdeps/x86_64/multiarch/strlen-evex.S b/sysdeps/x86_64/multiarch/strlen-evex.S +index 05838190..4bf6874b 100644 +--- a/sysdeps/x86_64/multiarch/strlen-evex.S ++++ b/sysdeps/x86_64/multiarch/strlen-evex.S +@@ -29,11 +29,13 @@ + # ifdef USE_AS_WCSLEN + # define VPCMP vpcmpd + # define VPMINU vpminud +-# define SHIFT_REG r9d ++# define SHIFT_REG ecx ++# define CHAR_SIZE 4 + # else + # define VPCMP vpcmpb + # define VPMINU vpminub +-# define SHIFT_REG ecx ++# define SHIFT_REG edx ++# define CHAR_SIZE 1 + # endif + + # define XMMZERO xmm16 +@@ -46,132 +48,165 @@ + # define YMM6 ymm22 + + # define VEC_SIZE 32 ++# define PAGE_SIZE 4096 ++# define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE) + + .section .text.evex,"ax",@progbits + ENTRY (STRLEN) + # ifdef USE_AS_STRNLEN +- /* Check for zero length. */ ++ /* Check zero length. */ + test %RSI_LP, %RSI_LP + jz L(zero) +-# ifdef USE_AS_WCSLEN +- shl $2, %RSI_LP +-# elif defined __ILP32__ ++# ifdef __ILP32__ + /* Clear the upper 32 bits. */ + movl %esi, %esi + # endif + mov %RSI_LP, %R8_LP + # endif +- movl %edi, %ecx +- movq %rdi, %rdx ++ movl %edi, %eax + vpxorq %XMMZERO, %XMMZERO, %XMMZERO +- ++ /* Clear high bits from edi. Only keeping bits relevant to page ++ cross check. */ ++ andl $(PAGE_SIZE - 1), %eax + /* Check if we may cross page boundary with one vector load. */ +- andl $(2 * VEC_SIZE - 1), %ecx +- cmpl $VEC_SIZE, %ecx +- ja L(cros_page_boundary) ++ cmpl $(PAGE_SIZE - VEC_SIZE), %eax ++ ja L(cross_page_boundary) + + /* Check the first VEC_SIZE bytes. Each bit in K0 represents a + null byte. */ + VPCMP $0, (%rdi), %YMMZERO, %k0 + kmovd %k0, %eax +- testl %eax, %eax +- + # ifdef USE_AS_STRNLEN +- jnz L(first_vec_x0_check) +- /* Adjust length and check the end of data. */ +- subq $VEC_SIZE, %rsi +- jbe L(max) +-# else +- jnz L(first_vec_x0) ++ /* If length < CHAR_PER_VEC handle special. */ ++ cmpq $CHAR_PER_VEC, %rsi ++ jbe L(first_vec_x0) + # endif +- +- /* Align data for aligned loads in the loop. */ +- addq $VEC_SIZE, %rdi +- andl $(VEC_SIZE - 1), %ecx +- andq $-VEC_SIZE, %rdi +- ++ testl %eax, %eax ++ jz L(aligned_more) ++ tzcntl %eax, %eax ++ ret + # ifdef USE_AS_STRNLEN +- /* Adjust length. */ +- addq %rcx, %rsi ++L(zero): ++ xorl %eax, %eax ++ ret + +- subq $(VEC_SIZE * 4), %rsi +- jbe L(last_4x_vec_or_less) ++ .p2align 4 ++L(first_vec_x0): ++ /* Set bit for max len so that tzcnt will return min of max len ++ and position of first match. */ ++ btsq %rsi, %rax ++ tzcntl %eax, %eax ++ ret + # endif +- jmp L(more_4x_vec) + + .p2align 4 +-L(cros_page_boundary): +- andl $(VEC_SIZE - 1), %ecx +- andq $-VEC_SIZE, %rdi +- +-# ifdef USE_AS_WCSLEN +- /* NB: Divide shift count by 4 since each bit in K0 represent 4 +- bytes. */ +- movl %ecx, %SHIFT_REG +- sarl $2, %SHIFT_REG ++L(first_vec_x1): ++ tzcntl %eax, %eax ++ /* Safe to use 32 bit instructions as these are only called for ++ size = [1, 159]. */ ++# ifdef USE_AS_STRNLEN ++ /* Use ecx which was computed earlier to compute correct value. ++ */ ++ leal -(CHAR_PER_VEC * 4 + 1)(%rcx, %rax), %eax ++# else ++ subl %edx, %edi ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get the wchar_t count. */ ++ sarl $2, %edi ++# endif ++ leal CHAR_PER_VEC(%rdi, %rax), %eax + # endif +- VPCMP $0, (%rdi), %YMMZERO, %k0 +- kmovd %k0, %eax ++ ret + +- /* Remove the leading bytes. */ +- sarxl %SHIFT_REG, %eax, %eax +- testl %eax, %eax +- jz L(aligned_more) ++ .p2align 4 ++L(first_vec_x2): + tzcntl %eax, %eax +-# ifdef USE_AS_WCSLEN +- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ +- sall $2, %eax +-# endif ++ /* Safe to use 32 bit instructions as these are only called for ++ size = [1, 159]. */ + # ifdef USE_AS_STRNLEN +- /* Check the end of data. */ +- cmpq %rax, %rsi +- jbe L(max) +-# endif +- addq %rdi, %rax +- addq %rcx, %rax +- subq %rdx, %rax +-# ifdef USE_AS_WCSLEN +- shrq $2, %rax ++ /* Use ecx which was computed earlier to compute correct value. ++ */ ++ leal -(CHAR_PER_VEC * 3 + 1)(%rcx, %rax), %eax ++# else ++ subl %edx, %edi ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get the wchar_t count. */ ++ sarl $2, %edi ++# endif ++ leal (CHAR_PER_VEC * 2)(%rdi, %rax), %eax + # endif + ret + + .p2align 4 +-L(aligned_more): ++L(first_vec_x3): ++ tzcntl %eax, %eax ++ /* Safe to use 32 bit instructions as these are only called for ++ size = [1, 159]. */ + # ifdef USE_AS_STRNLEN +- /* "rcx" is less than VEC_SIZE. Calculate "rdx + rcx - VEC_SIZE" +- with "rdx - (VEC_SIZE - rcx)" instead of "(rdx + rcx) - VEC_SIZE" +- to void possible addition overflow. */ +- negq %rcx +- addq $VEC_SIZE, %rcx +- +- /* Check the end of data. */ +- subq %rcx, %rsi +- jbe L(max) ++ /* Use ecx which was computed earlier to compute correct value. ++ */ ++ leal -(CHAR_PER_VEC * 2 + 1)(%rcx, %rax), %eax ++# else ++ subl %edx, %edi ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get the wchar_t count. */ ++ sarl $2, %edi ++# endif ++ leal (CHAR_PER_VEC * 3)(%rdi, %rax), %eax + # endif ++ ret + +- addq $VEC_SIZE, %rdi +- ++ .p2align 4 ++L(first_vec_x4): ++ tzcntl %eax, %eax ++ /* Safe to use 32 bit instructions as these are only called for ++ size = [1, 159]. */ + # ifdef USE_AS_STRNLEN +- subq $(VEC_SIZE * 4), %rsi +- jbe L(last_4x_vec_or_less) ++ /* Use ecx which was computed earlier to compute correct value. ++ */ ++ leal -(CHAR_PER_VEC + 1)(%rcx, %rax), %eax ++# else ++ subl %edx, %edi ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get the wchar_t count. */ ++ sarl $2, %edi ++# endif ++ leal (CHAR_PER_VEC * 4)(%rdi, %rax), %eax + # endif ++ ret + +-L(more_4x_vec): ++ .p2align 5 ++L(aligned_more): ++ movq %rdi, %rdx ++ /* Align data to VEC_SIZE. */ ++ andq $-(VEC_SIZE), %rdi ++L(cross_page_continue): + /* Check the first 4 * VEC_SIZE. Only one VEC_SIZE at a time + since data is only aligned to VEC_SIZE. */ +- VPCMP $0, (%rdi), %YMMZERO, %k0 +- kmovd %k0, %eax +- testl %eax, %eax +- jnz L(first_vec_x0) +- ++# ifdef USE_AS_STRNLEN ++ /* + CHAR_SIZE because it simplies the logic in ++ last_4x_vec_or_less. */ ++ leaq (VEC_SIZE * 5 + CHAR_SIZE)(%rdi), %rcx ++ subq %rdx, %rcx ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get the wchar_t count. */ ++ sarl $2, %ecx ++# endif ++# endif ++ /* Load first VEC regardless. */ + VPCMP $0, VEC_SIZE(%rdi), %YMMZERO, %k0 ++# ifdef USE_AS_STRNLEN ++ /* Adjust length. If near end handle specially. */ ++ subq %rcx, %rsi ++ jb L(last_4x_vec_or_less) ++# endif + kmovd %k0, %eax + testl %eax, %eax + jnz L(first_vec_x1) + + VPCMP $0, (VEC_SIZE * 2)(%rdi), %YMMZERO, %k0 + kmovd %k0, %eax +- testl %eax, %eax ++ test %eax, %eax + jnz L(first_vec_x2) + + VPCMP $0, (VEC_SIZE * 3)(%rdi), %YMMZERO, %k0 +@@ -179,258 +214,276 @@ L(more_4x_vec): + testl %eax, %eax + jnz L(first_vec_x3) + +- addq $(VEC_SIZE * 4), %rdi +- +-# ifdef USE_AS_STRNLEN +- subq $(VEC_SIZE * 4), %rsi +- jbe L(last_4x_vec_or_less) +-# endif +- +- /* Align data to 4 * VEC_SIZE. */ +- movq %rdi, %rcx +- andl $(4 * VEC_SIZE - 1), %ecx +- andq $-(4 * VEC_SIZE), %rdi ++ VPCMP $0, (VEC_SIZE * 4)(%rdi), %YMMZERO, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x4) + ++ addq $VEC_SIZE, %rdi + # ifdef USE_AS_STRNLEN +- /* Adjust length. */ ++ /* Check if at last VEC_SIZE * 4 length. */ ++ cmpq $(CHAR_PER_VEC * 4 - 1), %rsi ++ jbe L(last_4x_vec_or_less_load) ++ movl %edi, %ecx ++ andl $(VEC_SIZE * 4 - 1), %ecx ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get the wchar_t count. */ ++ sarl $2, %ecx ++# endif ++ /* Readjust length. */ + addq %rcx, %rsi + # endif ++ /* Align data to VEC_SIZE * 4. */ ++ andq $-(VEC_SIZE * 4), %rdi + ++ /* Compare 4 * VEC at a time forward. */ + .p2align 4 + L(loop_4x_vec): +- /* Compare 4 * VEC at a time forward. */ +- VMOVA (%rdi), %YMM1 +- VMOVA VEC_SIZE(%rdi), %YMM2 +- VMOVA (VEC_SIZE * 2)(%rdi), %YMM3 +- VMOVA (VEC_SIZE * 3)(%rdi), %YMM4 +- +- VPMINU %YMM1, %YMM2, %YMM5 +- VPMINU %YMM3, %YMM4, %YMM6 ++ /* Load first VEC regardless. */ ++ VMOVA (VEC_SIZE * 4)(%rdi), %YMM1 ++# ifdef USE_AS_STRNLEN ++ /* Break if at end of length. */ ++ subq $(CHAR_PER_VEC * 4), %rsi ++ jb L(last_4x_vec_or_less_cmpeq) ++# endif ++ /* Save some code size by microfusing VPMINU with the load. Since ++ the matches in ymm2/ymm4 can only be returned if there where no ++ matches in ymm1/ymm3 respectively there is no issue with overlap. ++ */ ++ VPMINU (VEC_SIZE * 5)(%rdi), %YMM1, %YMM2 ++ VMOVA (VEC_SIZE * 6)(%rdi), %YMM3 ++ VPMINU (VEC_SIZE * 7)(%rdi), %YMM3, %YMM4 ++ ++ VPCMP $0, %YMM2, %YMMZERO, %k0 ++ VPCMP $0, %YMM4, %YMMZERO, %k1 ++ subq $-(VEC_SIZE * 4), %rdi ++ kortestd %k0, %k1 ++ jz L(loop_4x_vec) ++ ++ /* Check if end was in first half. */ ++ kmovd %k0, %eax ++ subq %rdx, %rdi ++# ifdef USE_AS_WCSLEN ++ shrq $2, %rdi ++# endif ++ testl %eax, %eax ++ jz L(second_vec_return) + +- VPMINU %YMM5, %YMM6, %YMM5 +- VPCMP $0, %YMM5, %YMMZERO, %k0 +- ktestd %k0, %k0 +- jnz L(4x_vec_end) ++ VPCMP $0, %YMM1, %YMMZERO, %k2 ++ kmovd %k2, %edx ++ /* Combine VEC1 matches (edx) with VEC2 matches (eax). */ ++# ifdef USE_AS_WCSLEN ++ sall $CHAR_PER_VEC, %eax ++ orl %edx, %eax ++ tzcntl %eax, %eax ++# else ++ salq $CHAR_PER_VEC, %rax ++ orq %rdx, %rax ++ tzcntq %rax, %rax ++# endif ++ addq %rdi, %rax ++ ret + +- addq $(VEC_SIZE * 4), %rdi + +-# ifndef USE_AS_STRNLEN +- jmp L(loop_4x_vec) +-# else +- subq $(VEC_SIZE * 4), %rsi +- ja L(loop_4x_vec) ++# ifdef USE_AS_STRNLEN + ++L(last_4x_vec_or_less_load): ++ /* Depending on entry adjust rdi / prepare first VEC in YMM1. */ ++ VMOVA (VEC_SIZE * 4)(%rdi), %YMM1 ++L(last_4x_vec_or_less_cmpeq): ++ VPCMP $0, %YMM1, %YMMZERO, %k0 ++ addq $(VEC_SIZE * 3), %rdi + L(last_4x_vec_or_less): +- /* Less than 4 * VEC and aligned to VEC_SIZE. */ +- addl $(VEC_SIZE * 2), %esi +- jle L(last_2x_vec) +- +- VPCMP $0, (%rdi), %YMMZERO, %k0 + kmovd %k0, %eax ++ /* If remaining length > VEC_SIZE * 2. This works if esi is off by ++ VEC_SIZE * 4. */ ++ testl $(CHAR_PER_VEC * 2), %esi ++ jnz L(last_4x_vec) ++ ++ /* length may have been negative or positive by an offset of ++ CHAR_PER_VEC * 4 depending on where this was called from. This ++ fixes that. */ ++ andl $(CHAR_PER_VEC * 4 - 1), %esi + testl %eax, %eax +- jnz L(first_vec_x0) ++ jnz L(last_vec_x1_check) + +- VPCMP $0, VEC_SIZE(%rdi), %YMMZERO, %k0 +- kmovd %k0, %eax +- testl %eax, %eax +- jnz L(first_vec_x1) ++ /* Check the end of data. */ ++ subl $CHAR_PER_VEC, %esi ++ jb L(max) + + VPCMP $0, (VEC_SIZE * 2)(%rdi), %YMMZERO, %k0 + kmovd %k0, %eax +- testl %eax, %eax +- jnz L(first_vec_x2_check) +- subl $VEC_SIZE, %esi +- jle L(max) ++ tzcntl %eax, %eax ++ /* Check the end of data. */ ++ cmpl %eax, %esi ++ jb L(max) + +- VPCMP $0, (VEC_SIZE * 3)(%rdi), %YMMZERO, %k0 +- kmovd %k0, %eax +- testl %eax, %eax +- jnz L(first_vec_x3_check) ++ subq %rdx, %rdi ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get the wchar_t count. */ ++ sarq $2, %rdi ++# endif ++ leaq (CHAR_PER_VEC * 2)(%rdi, %rax), %rax ++ ret ++L(max): + movq %r8, %rax ++ ret ++# endif ++ ++ /* Placed here in strnlen so that the jcc L(last_4x_vec_or_less) ++ in the 4x VEC loop can use 2 byte encoding. */ ++ .p2align 4 ++L(second_vec_return): ++ VPCMP $0, %YMM3, %YMMZERO, %k0 ++ /* Combine YMM3 matches (k0) with YMM4 matches (k1). */ ++# ifdef USE_AS_WCSLEN ++ kunpckbw %k0, %k1, %k0 ++ kmovd %k0, %eax ++ tzcntl %eax, %eax ++# else ++ kunpckdq %k0, %k1, %k0 ++ kmovq %k0, %rax ++ tzcntq %rax, %rax ++# endif ++ leaq (CHAR_PER_VEC * 2)(%rdi, %rax), %rax ++ ret ++ ++ ++# ifdef USE_AS_STRNLEN ++L(last_vec_x1_check): ++ tzcntl %eax, %eax ++ /* Check the end of data. */ ++ cmpl %eax, %esi ++ jb L(max) ++ subq %rdx, %rdi + # ifdef USE_AS_WCSLEN +- shrq $2, %rax ++ /* NB: Divide bytes by 4 to get the wchar_t count. */ ++ sarq $2, %rdi + # endif ++ leaq (CHAR_PER_VEC)(%rdi, %rax), %rax + ret + + .p2align 4 +-L(last_2x_vec): +- addl $(VEC_SIZE * 2), %esi ++L(last_4x_vec): ++ /* Test first 2x VEC normally. */ ++ testl %eax, %eax ++ jnz L(last_vec_x1) + +- VPCMP $0, (%rdi), %YMMZERO, %k0 ++ VPCMP $0, (VEC_SIZE * 2)(%rdi), %YMMZERO, %k0 + kmovd %k0, %eax + testl %eax, %eax +- jnz L(first_vec_x0_check) +- subl $VEC_SIZE, %esi +- jle L(max) ++ jnz L(last_vec_x2) + +- VPCMP $0, VEC_SIZE(%rdi), %YMMZERO, %k0 ++ /* Normalize length. */ ++ andl $(CHAR_PER_VEC * 4 - 1), %esi ++ VPCMP $0, (VEC_SIZE * 3)(%rdi), %YMMZERO, %k0 + kmovd %k0, %eax + testl %eax, %eax +- jnz L(first_vec_x1_check) +- movq %r8, %rax +-# ifdef USE_AS_WCSLEN +- shrq $2, %rax +-# endif +- ret ++ jnz L(last_vec_x3) + +- .p2align 4 +-L(first_vec_x0_check): ++ /* Check the end of data. */ ++ subl $(CHAR_PER_VEC * 3), %esi ++ jb L(max) ++ ++ VPCMP $0, (VEC_SIZE * 4)(%rdi), %YMMZERO, %k0 ++ kmovd %k0, %eax + tzcntl %eax, %eax +-# ifdef USE_AS_WCSLEN +- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ +- sall $2, %eax +-# endif + /* Check the end of data. */ +- cmpq %rax, %rsi +- jbe L(max) +- addq %rdi, %rax +- subq %rdx, %rax ++ cmpl %eax, %esi ++ jb L(max_end) ++ ++ subq %rdx, %rdi + # ifdef USE_AS_WCSLEN +- shrq $2, %rax ++ /* NB: Divide bytes by 4 to get the wchar_t count. */ ++ sarq $2, %rdi + # endif ++ leaq (CHAR_PER_VEC * 4)(%rdi, %rax), %rax + ret + + .p2align 4 +-L(first_vec_x1_check): ++L(last_vec_x1): + tzcntl %eax, %eax ++ subq %rdx, %rdi + # ifdef USE_AS_WCSLEN +- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ +- sall $2, %eax +-# endif +- /* Check the end of data. */ +- cmpq %rax, %rsi +- jbe L(max) +- addq $VEC_SIZE, %rax +- addq %rdi, %rax +- subq %rdx, %rax +-# ifdef USE_AS_WCSLEN +- shrq $2, %rax ++ /* NB: Divide bytes by 4 to get the wchar_t count. */ ++ sarq $2, %rdi + # endif ++ leaq (CHAR_PER_VEC)(%rdi, %rax), %rax + ret + + .p2align 4 +-L(first_vec_x2_check): ++L(last_vec_x2): + tzcntl %eax, %eax ++ subq %rdx, %rdi + # ifdef USE_AS_WCSLEN +- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ +- sall $2, %eax +-# endif +- /* Check the end of data. */ +- cmpq %rax, %rsi +- jbe L(max) +- addq $(VEC_SIZE * 2), %rax +- addq %rdi, %rax +- subq %rdx, %rax +-# ifdef USE_AS_WCSLEN +- shrq $2, %rax ++ /* NB: Divide bytes by 4 to get the wchar_t count. */ ++ sarq $2, %rdi + # endif ++ leaq (CHAR_PER_VEC * 2)(%rdi, %rax), %rax + ret + + .p2align 4 +-L(first_vec_x3_check): ++L(last_vec_x3): + tzcntl %eax, %eax +-# ifdef USE_AS_WCSLEN +- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ +- sall $2, %eax +-# endif ++ subl $(CHAR_PER_VEC * 2), %esi + /* Check the end of data. */ +- cmpq %rax, %rsi +- jbe L(max) +- addq $(VEC_SIZE * 3), %rax +- addq %rdi, %rax +- subq %rdx, %rax ++ cmpl %eax, %esi ++ jb L(max_end) ++ subq %rdx, %rdi + # ifdef USE_AS_WCSLEN +- shrq $2, %rax ++ /* NB: Divide bytes by 4 to get the wchar_t count. */ ++ sarq $2, %rdi + # endif ++ leaq (CHAR_PER_VEC * 3)(%rdi, %rax), %rax + ret +- +- .p2align 4 +-L(max): ++L(max_end): + movq %r8, %rax +-# ifdef USE_AS_WCSLEN +- shrq $2, %rax +-# endif +- ret +- +- .p2align 4 +-L(zero): +- xorl %eax, %eax + ret + # endif + ++ /* Cold case for crossing page with first load. */ + .p2align 4 +-L(first_vec_x0): +- tzcntl %eax, %eax +-# ifdef USE_AS_WCSLEN +- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ +- sall $2, %eax +-# endif +- addq %rdi, %rax +- subq %rdx, %rax ++L(cross_page_boundary): ++ movq %rdi, %rdx ++ /* Align data to VEC_SIZE. */ ++ andq $-VEC_SIZE, %rdi ++ VPCMP $0, (%rdi), %YMMZERO, %k0 ++ kmovd %k0, %eax ++ /* Remove the leading bytes. */ + # ifdef USE_AS_WCSLEN +- shrq $2, %rax ++ /* NB: Divide shift count by 4 since each bit in K0 represent 4 ++ bytes. */ ++ movl %edx, %ecx ++ shrl $2, %ecx ++ andl $(CHAR_PER_VEC - 1), %ecx + # endif +- ret +- +- .p2align 4 +-L(first_vec_x1): ++ /* SHIFT_REG is ecx for USE_AS_WCSLEN and edx otherwise. */ ++ sarxl %SHIFT_REG, %eax, %eax ++ testl %eax, %eax ++# ifndef USE_AS_STRNLEN ++ jz L(cross_page_continue) + tzcntl %eax, %eax +-# ifdef USE_AS_WCSLEN +- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ +- sall $2, %eax +-# endif +- addq $VEC_SIZE, %rax +- addq %rdi, %rax +- subq %rdx, %rax +-# ifdef USE_AS_WCSLEN +- shrq $2, %rax +-# endif + ret +- +- .p2align 4 +-L(first_vec_x2): +- tzcntl %eax, %eax +-# ifdef USE_AS_WCSLEN +- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ +- sall $2, %eax +-# endif +- addq $(VEC_SIZE * 2), %rax +- addq %rdi, %rax +- subq %rdx, %rax +-# ifdef USE_AS_WCSLEN +- shrq $2, %rax +-# endif ++# else ++ jnz L(cross_page_less_vec) ++# ifndef USE_AS_WCSLEN ++ movl %edx, %ecx ++ andl $(CHAR_PER_VEC - 1), %ecx ++# endif ++ movl $CHAR_PER_VEC, %eax ++ subl %ecx, %eax ++ /* Check the end of data. */ ++ cmpq %rax, %rsi ++ ja L(cross_page_continue) ++ movl %esi, %eax + ret +- +- .p2align 4 +-L(4x_vec_end): +- VPCMP $0, %YMM1, %YMMZERO, %k0 +- kmovd %k0, %eax +- testl %eax, %eax +- jnz L(first_vec_x0) +- VPCMP $0, %YMM2, %YMMZERO, %k1 +- kmovd %k1, %eax +- testl %eax, %eax +- jnz L(first_vec_x1) +- VPCMP $0, %YMM3, %YMMZERO, %k2 +- kmovd %k2, %eax +- testl %eax, %eax +- jnz L(first_vec_x2) +- VPCMP $0, %YMM4, %YMMZERO, %k3 +- kmovd %k3, %eax +-L(first_vec_x3): ++L(cross_page_less_vec): + tzcntl %eax, %eax +-# ifdef USE_AS_WCSLEN +- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ +- sall $2, %eax +-# endif +- addq $(VEC_SIZE * 3), %rax +- addq %rdi, %rax +- subq %rdx, %rax +-# ifdef USE_AS_WCSLEN +- shrq $2, %rax +-# endif ++ /* Select min of length and position of first null. */ ++ cmpq %rax, %rsi ++ cmovb %esi, %eax + ret ++# endif + + END (STRLEN) + #endif +-- +GitLab + diff --git a/SOURCES/ia-prefer_no_vzeroupper.patch b/SOURCES/ia-prefer_no_vzeroupper.patch new file mode 100644 index 0000000..bcdcc5c --- /dev/null +++ b/SOURCES/ia-prefer_no_vzeroupper.patch @@ -0,0 +1,75 @@ +From 18a0eba6e84524e3a1cbb544ed1f53dce9531b46 Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Wed, 2 Mar 2022 12:52:23 -0800 +Subject: [PATCH] x86: Set Prefer_No_VZEROUPPER and add Prefer_AVX2_STRCMP + +1. Set Prefer_No_VZEROUPPER if RTM is usable to avoid RTM abort triggered +by VZEROUPPER inside a transactionally executing RTM region. +2. Since to compare 2 32-byte strings, 256-bit EVEX strcmp requires 2 +loads, 3 VPCMPs and 2 KORDs while AVX2 strcmp requires 1 load, 2 VPCMPEQs, +1 VPMINU and 1 VPMOVMSKB, AVX2 strcmp is faster than EVEX strcmp. Add +Prefer_AVX2_STRCMP to prefer AVX2 strcmp family functions. + +(cherry picked from commit 1da50d4bda07f04135dca39f40e79fc9eabed1f8) +--- + sysdeps/x86/cpu-features.c | 20 +++++++++++++++++-- + sysdeps/x86/cpu-tunables.c | 2 ++ + ...cpu-features-preferred_feature_index_1.def | 1 + + 3 files changed, 21 insertions(+), 2 deletions(-) + +diff --git a/sysdeps/x86/cpu-features.c b/sysdeps/x86/cpu-features.c +index 91042505..3610ee5c 100644 +--- a/sysdeps/x86/cpu-features.c ++++ b/sysdeps/x86/cpu-features.c +@@ -524,8 +524,24 @@ init_cpu_features (struct cpu_features *cpu_features) + cpu_features->preferred[index_arch_Prefer_No_VZEROUPPER] + |= bit_arch_Prefer_No_VZEROUPPER; + else +- cpu_features->preferred[index_arch_Prefer_No_AVX512] +- |= bit_arch_Prefer_No_AVX512; ++ { ++ cpu_features->preferred[index_arch_Prefer_No_AVX512] ++ |= bit_arch_Prefer_No_AVX512; ++ ++ /* Avoid RTM abort triggered by VZEROUPPER inside a ++ transactionally executing RTM region. */ ++ if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) ++ cpu_features->preferred[index_arch_Prefer_No_VZEROUPPER] ++ |= bit_arch_Prefer_No_VZEROUPPER; ++ ++ /* Since to compare 2 32-byte strings, 256-bit EVEX strcmp ++ requires 2 loads, 3 VPCMPs and 2 KORDs while AVX2 strcmp ++ requires 1 load, 2 VPCMPEQs, 1 VPMINU and 1 VPMOVMSKB, ++ AVX2 strcmp is faster than EVEX strcmp. */ ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX2)) ++ cpu_features->preferred[index_arch_Prefer_AVX2_STRCMP] ++ |= bit_arch_Prefer_AVX2_STRCMP; ++ } + } + /* This spells out "AuthenticAMD". */ + else if (ebx == 0x68747541 && ecx == 0x444d4163 && edx == 0x69746e65) +diff --git a/sysdeps/x86/cpu-tunables.c b/sysdeps/x86/cpu-tunables.c +index 3173b2b9..73adbaba 100644 +--- a/sysdeps/x86/cpu-tunables.c ++++ b/sysdeps/x86/cpu-tunables.c +@@ -239,6 +239,8 @@ TUNABLE_CALLBACK (set_hwcaps) (tunable_val_t *valp) + CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features, + Fast_Copy_Backward, + disable, 18); ++ CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH ++ (n, cpu_features, Prefer_AVX2_STRCMP, AVX2, disable, 18); + } + break; + case 19: +diff --git a/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def b/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def +index 17a5cc42..4ca70b40 100644 +--- a/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def ++++ b/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def +@@ -32,3 +32,4 @@ BIT (Prefer_ERMS) + BIT (Prefer_FSRM) + BIT (Prefer_No_AVX512) + BIT (MathVec_Prefer_No_AVX512) ++BIT (Prefer_AVX2_STRCMP) +-- +GitLab + diff --git a/SOURCES/ia-redirect-roundeven-funct.patch b/SOURCES/ia-redirect-roundeven-funct.patch new file mode 100644 index 0000000..819f0bd --- /dev/null +++ b/SOURCES/ia-redirect-roundeven-funct.patch @@ -0,0 +1,114 @@ +From 9682778522977444546c061170912da902635a4e Mon Sep 17 00:00:00 2001 +From: Shen-Ta Hsieh +Date: Mon, 24 May 2021 09:43:10 +0800 +Subject: [PATCH] math: redirect roundeven function + +This patch redirect roundeven function for futhermore changes. + +Signed-off-by: Shen-Ta Hsieh +Reviewed-by: H.J. Lu +(cherry picked from commit 447954a206837b5f153869cfeeeab44631c3fac9) +--- + include/math.h | 3 ++- + sysdeps/ieee754/dbl-64/s_roundeven.c | 4 +++- + sysdeps/ieee754/float128/s_roundevenf128.c | 1 + + sysdeps/ieee754/flt-32/s_roundevenf.c | 3 +++ + sysdeps/ieee754/ldbl-128/s_roundevenl.c | 1 + + sysdeps/ieee754/ldbl-96/s_roundevenl.c | 1 + + 6 files changed, 11 insertions(+), 2 deletions(-) + +diff --git a/include/math.h b/include/math.h +index e21d34b8..1f9f9a54 100644 +--- a/include/math.h ++++ b/include/math.h +@@ -38,7 +38,6 @@ libm_hidden_proto (__issignaling) + libm_hidden_proto (__issignalingf) + libm_hidden_proto (__exp) + libm_hidden_proto (__expf) +-libm_hidden_proto (__roundeven) + + # ifndef __NO_LONG_DOUBLE_MATH + libm_hidden_proto (__fpclassifyl) +@@ -56,6 +55,8 @@ libm_hidden_proto (__expm1f128) + + # if !(defined __FINITE_MATH_ONLY__ && __FINITE_MATH_ONLY__ > 0) + # ifndef NO_MATH_REDIRECT ++float (roundevenf) (float) asm ("__roundevenf"); ++double (roundeven) (double) asm ("__roundeven"); + /* Declare sqrt for use within GLIBC. Compilers typically inline sqrt as a + single instruction. Use an asm to avoid use of PLTs if it doesn't. */ + float (sqrtf) (float) asm ("__ieee754_sqrtf"); +diff --git a/sysdeps/ieee754/dbl-64/s_roundeven.c b/sysdeps/ieee754/dbl-64/s_roundeven.c +index 1438e81d..61962184 100644 +--- a/sysdeps/ieee754/dbl-64/s_roundeven.c ++++ b/sysdeps/ieee754/dbl-64/s_roundeven.c +@@ -17,6 +17,7 @@ + License along with the GNU C Library; if not, see + . */ + ++#define NO_MATH_REDIRECT + #include + #include + #include +@@ -101,5 +102,6 @@ __roundeven (double x) + INSERT_WORDS (x, hx, lx); + return x; + } +-hidden_def (__roundeven) ++#ifndef __roundeven + libm_alias_double (__roundeven, roundeven) ++#endif +diff --git a/sysdeps/ieee754/float128/s_roundevenf128.c b/sysdeps/ieee754/float128/s_roundevenf128.c +index 5a9b3f39..e0faf727 100644 +--- a/sysdeps/ieee754/float128/s_roundevenf128.c ++++ b/sysdeps/ieee754/float128/s_roundevenf128.c +@@ -1,2 +1,3 @@ ++#define NO_MATH_REDIRECT + #include + #include "../ldbl-128/s_roundevenl.c" +diff --git a/sysdeps/ieee754/flt-32/s_roundevenf.c b/sysdeps/ieee754/flt-32/s_roundevenf.c +index 90f991d5..a661875e 100644 +--- a/sysdeps/ieee754/flt-32/s_roundevenf.c ++++ b/sysdeps/ieee754/flt-32/s_roundevenf.c +@@ -17,6 +17,7 @@ + License along with the GNU C Library; if not, see + . */ + ++#define NO_MATH_REDIRECT + #include + #include + #include +@@ -67,4 +68,6 @@ __roundevenf (float x) + SET_FLOAT_WORD (x, ix); + return x; + } ++#ifndef __roundevenf + libm_alias_float (__roundeven, roundeven) ++#endif +diff --git a/sysdeps/ieee754/ldbl-128/s_roundevenl.c b/sysdeps/ieee754/ldbl-128/s_roundevenl.c +index 5fc59af4..b9375b6c 100644 +--- a/sysdeps/ieee754/ldbl-128/s_roundevenl.c ++++ b/sysdeps/ieee754/ldbl-128/s_roundevenl.c +@@ -17,6 +17,7 @@ + License along with the GNU C Library; if not, see + . */ + ++#define NO_MATH_REDIRECT + #include + #include + #include +diff --git a/sysdeps/ieee754/ldbl-96/s_roundevenl.c b/sysdeps/ieee754/ldbl-96/s_roundevenl.c +index be2e4fa4..65031ab7 100644 +--- a/sysdeps/ieee754/ldbl-96/s_roundevenl.c ++++ b/sysdeps/ieee754/ldbl-96/s_roundevenl.c +@@ -17,6 +17,7 @@ + License along with the GNU C Library; if not, see + . */ + ++#define NO_MATH_REDIRECT + #include + #include + #include +-- +GitLab + diff --git a/SOURCES/ia-refacto-imp-prf-strchr-avx2.patch b/SOURCES/ia-refacto-imp-prf-strchr-avx2.patch new file mode 100644 index 0000000..69316a9 --- /dev/null +++ b/SOURCES/ia-refacto-imp-prf-strchr-avx2.patch @@ -0,0 +1,356 @@ +From 43847e49c2dab633146c9b6c682ed5768ccda7cd Mon Sep 17 00:00:00 2001 +From: noah +Date: Wed, 3 Feb 2021 00:38:59 -0500 +Subject: [PATCH] x86-64: Refactor and improve performance of strchr-avx2.S + +No bug. Just seemed the performance could be improved a bit. Observed +and expected behavior are unchanged. Optimized body of main +loop. Updated page cross logic and optimized accordingly. Made a few +minor instruction selection modifications. No regressions in test +suite. Both test-strchrnul and test-strchr passed. + +(cherry picked from commit 1f745ecc2109890886b161d4791e1406fdfc29b8) +--- + sysdeps/x86_64/multiarch/strchr-avx2.S | 225 ++++++++++++------------- + sysdeps/x86_64/multiarch/strchr.c | 4 +- + 2 files changed, 114 insertions(+), 115 deletions(-) + +diff --git a/sysdeps/x86_64/multiarch/strchr-avx2.S b/sysdeps/x86_64/multiarch/strchr-avx2.S +index da7d2620..919d256c 100644 +--- a/sysdeps/x86_64/multiarch/strchr-avx2.S ++++ b/sysdeps/x86_64/multiarch/strchr-avx2.S +@@ -27,10 +27,12 @@ + # ifdef USE_AS_WCSCHR + # define VPBROADCAST vpbroadcastd + # define VPCMPEQ vpcmpeqd ++# define VPMINU vpminud + # define CHAR_REG esi + # else + # define VPBROADCAST vpbroadcastb + # define VPCMPEQ vpcmpeqb ++# define VPMINU vpminub + # define CHAR_REG sil + # endif + +@@ -43,71 +45,54 @@ + # endif + + # define VEC_SIZE 32 ++# define PAGE_SIZE 4096 + + .section SECTION(.text),"ax",@progbits + ENTRY (STRCHR) + movl %edi, %ecx +- /* Broadcast CHAR to YMM0. */ ++# ifndef USE_AS_STRCHRNUL ++ xorl %edx, %edx ++# endif ++ ++ /* Broadcast CHAR to YMM0. */ + vmovd %esi, %xmm0 + vpxor %xmm9, %xmm9, %xmm9 + VPBROADCAST %xmm0, %ymm0 +- /* Check if we may cross page boundary with one vector load. */ +- andl $(2 * VEC_SIZE - 1), %ecx +- cmpl $VEC_SIZE, %ecx +- ja L(cros_page_boundary) + +- /* Check the first VEC_SIZE bytes. Search for both CHAR and the +- null byte. */ +- vmovdqu (%rdi), %ymm8 +- VPCMPEQ %ymm8, %ymm0, %ymm1 +- VPCMPEQ %ymm8, %ymm9, %ymm2 +- vpor %ymm1, %ymm2, %ymm1 +- vpmovmskb %ymm1, %eax +- testl %eax, %eax +- jnz L(first_vec_x0) ++ /* Check if we cross page boundary with one vector load. */ ++ andl $(PAGE_SIZE - 1), %ecx ++ cmpl $(PAGE_SIZE - VEC_SIZE), %ecx ++ ja L(cross_page_boundary) + +- /* Align data for aligned loads in the loop. */ +- addq $VEC_SIZE, %rdi +- andl $(VEC_SIZE - 1), %ecx +- andq $-VEC_SIZE, %rdi +- +- jmp L(more_4x_vec) +- +- .p2align 4 +-L(cros_page_boundary): +- andl $(VEC_SIZE - 1), %ecx +- andq $-VEC_SIZE, %rdi ++ /* Check the first VEC_SIZE bytes. Search for both CHAR and the ++ null byte. */ + vmovdqu (%rdi), %ymm8 + VPCMPEQ %ymm8, %ymm0, %ymm1 + VPCMPEQ %ymm8, %ymm9, %ymm2 + vpor %ymm1, %ymm2, %ymm1 + vpmovmskb %ymm1, %eax +- /* Remove the leading bytes. */ +- sarl %cl, %eax + testl %eax, %eax +- jz L(aligned_more) +- /* Found CHAR or the null byte. */ ++ jz L(more_vecs) + tzcntl %eax, %eax +- addq %rcx, %rax +-# ifdef USE_AS_STRCHRNUL ++ /* Found CHAR or the null byte. */ + addq %rdi, %rax +-# else +- xorl %edx, %edx +- leaq (%rdi, %rax), %rax +- cmp (%rax), %CHAR_REG ++# ifndef USE_AS_STRCHRNUL ++ cmp (%rax), %CHAR_REG + cmovne %rdx, %rax + # endif + L(return_vzeroupper): + ZERO_UPPER_VEC_REGISTERS_RETURN + + .p2align 4 ++L(more_vecs): ++ /* Align data for aligned loads in the loop. */ ++ andq $-VEC_SIZE, %rdi + L(aligned_more): +- addq $VEC_SIZE, %rdi + +-L(more_4x_vec): +- /* Check the first 4 * VEC_SIZE. Only one VEC_SIZE at a time +- since data is only aligned to VEC_SIZE. */ +- vmovdqa (%rdi), %ymm8 ++ /* Check the next 4 * VEC_SIZE. Only one VEC_SIZE at a time ++ since data is only aligned to VEC_SIZE. */ ++ vmovdqa VEC_SIZE(%rdi), %ymm8 ++ addq $VEC_SIZE, %rdi + VPCMPEQ %ymm8, %ymm0, %ymm1 + VPCMPEQ %ymm8, %ymm9, %ymm2 + vpor %ymm1, %ymm2, %ymm1 +@@ -137,61 +122,24 @@ L(more_4x_vec): + vpor %ymm1, %ymm2, %ymm1 + vpmovmskb %ymm1, %eax + testl %eax, %eax +- jnz L(first_vec_x3) +- +- addq $(VEC_SIZE * 4), %rdi +- +- /* Align data to 4 * VEC_SIZE. */ +- movq %rdi, %rcx +- andl $(4 * VEC_SIZE - 1), %ecx +- andq $-(4 * VEC_SIZE), %rdi +- +- .p2align 4 +-L(loop_4x_vec): +- /* Compare 4 * VEC at a time forward. */ +- vmovdqa (%rdi), %ymm5 +- vmovdqa VEC_SIZE(%rdi), %ymm6 +- vmovdqa (VEC_SIZE * 2)(%rdi), %ymm7 +- vmovdqa (VEC_SIZE * 3)(%rdi), %ymm8 +- +- VPCMPEQ %ymm5, %ymm0, %ymm1 +- VPCMPEQ %ymm6, %ymm0, %ymm2 +- VPCMPEQ %ymm7, %ymm0, %ymm3 +- VPCMPEQ %ymm8, %ymm0, %ymm4 +- +- VPCMPEQ %ymm5, %ymm9, %ymm5 +- VPCMPEQ %ymm6, %ymm9, %ymm6 +- VPCMPEQ %ymm7, %ymm9, %ymm7 +- VPCMPEQ %ymm8, %ymm9, %ymm8 +- +- vpor %ymm1, %ymm5, %ymm1 +- vpor %ymm2, %ymm6, %ymm2 +- vpor %ymm3, %ymm7, %ymm3 +- vpor %ymm4, %ymm8, %ymm4 +- +- vpor %ymm1, %ymm2, %ymm5 +- vpor %ymm3, %ymm4, %ymm6 +- +- vpor %ymm5, %ymm6, %ymm5 +- +- vpmovmskb %ymm5, %eax +- testl %eax, %eax +- jnz L(4x_vec_end) +- +- addq $(VEC_SIZE * 4), %rdi ++ jz L(prep_loop_4x) + +- jmp L(loop_4x_vec) ++ tzcntl %eax, %eax ++ leaq (VEC_SIZE * 3)(%rdi, %rax), %rax ++# ifndef USE_AS_STRCHRNUL ++ cmp (%rax), %CHAR_REG ++ cmovne %rdx, %rax ++# endif ++ VZEROUPPER ++ ret + + .p2align 4 + L(first_vec_x0): +- /* Found CHAR or the null byte. */ + tzcntl %eax, %eax +-# ifdef USE_AS_STRCHRNUL ++ /* Found CHAR or the null byte. */ + addq %rdi, %rax +-# else +- xorl %edx, %edx +- leaq (%rdi, %rax), %rax +- cmp (%rax), %CHAR_REG ++# ifndef USE_AS_STRCHRNUL ++ cmp (%rax), %CHAR_REG + cmovne %rdx, %rax + # endif + VZEROUPPER_RETURN +@@ -199,13 +147,9 @@ L(first_vec_x0): + .p2align 4 + L(first_vec_x1): + tzcntl %eax, %eax +-# ifdef USE_AS_STRCHRNUL +- addq $VEC_SIZE, %rax +- addq %rdi, %rax +-# else +- xorl %edx, %edx + leaq VEC_SIZE(%rdi, %rax), %rax +- cmp (%rax), %CHAR_REG ++# ifndef USE_AS_STRCHRNUL ++ cmp (%rax), %CHAR_REG + cmovne %rdx, %rax + # endif + VZEROUPPER_RETURN +@@ -213,42 +157,97 @@ L(first_vec_x1): + .p2align 4 + L(first_vec_x2): + tzcntl %eax, %eax +-# ifdef USE_AS_STRCHRNUL +- addq $(VEC_SIZE * 2), %rax +- addq %rdi, %rax +-# else +- xorl %edx, %edx ++ /* Found CHAR or the null byte. */ + leaq (VEC_SIZE * 2)(%rdi, %rax), %rax +- cmp (%rax), %CHAR_REG ++# ifndef USE_AS_STRCHRNUL ++ cmp (%rax), %CHAR_REG + cmovne %rdx, %rax + # endif + VZEROUPPER_RETURN + ++L(prep_loop_4x): ++ /* Align data to 4 * VEC_SIZE. */ ++ andq $-(VEC_SIZE * 4), %rdi ++ + .p2align 4 +-L(4x_vec_end): ++L(loop_4x_vec): ++ /* Compare 4 * VEC at a time forward. */ ++ vmovdqa (VEC_SIZE * 4)(%rdi), %ymm5 ++ vmovdqa (VEC_SIZE * 5)(%rdi), %ymm6 ++ vmovdqa (VEC_SIZE * 6)(%rdi), %ymm7 ++ vmovdqa (VEC_SIZE * 7)(%rdi), %ymm8 ++ ++ /* Leaves only CHARS matching esi as 0. */ ++ vpxor %ymm5, %ymm0, %ymm1 ++ vpxor %ymm6, %ymm0, %ymm2 ++ vpxor %ymm7, %ymm0, %ymm3 ++ vpxor %ymm8, %ymm0, %ymm4 ++ ++ VPMINU %ymm1, %ymm5, %ymm1 ++ VPMINU %ymm2, %ymm6, %ymm2 ++ VPMINU %ymm3, %ymm7, %ymm3 ++ VPMINU %ymm4, %ymm8, %ymm4 ++ ++ VPMINU %ymm1, %ymm2, %ymm5 ++ VPMINU %ymm3, %ymm4, %ymm6 ++ ++ VPMINU %ymm5, %ymm6, %ymm5 ++ ++ VPCMPEQ %ymm5, %ymm9, %ymm5 ++ vpmovmskb %ymm5, %eax ++ ++ addq $(VEC_SIZE * 4), %rdi ++ testl %eax, %eax ++ jz L(loop_4x_vec) ++ ++ VPCMPEQ %ymm1, %ymm9, %ymm1 + vpmovmskb %ymm1, %eax + testl %eax, %eax + jnz L(first_vec_x0) ++ ++ VPCMPEQ %ymm2, %ymm9, %ymm2 + vpmovmskb %ymm2, %eax + testl %eax, %eax + jnz L(first_vec_x1) +- vpmovmskb %ymm3, %eax +- testl %eax, %eax +- jnz L(first_vec_x2) ++ ++ VPCMPEQ %ymm3, %ymm9, %ymm3 ++ VPCMPEQ %ymm4, %ymm9, %ymm4 ++ vpmovmskb %ymm3, %ecx + vpmovmskb %ymm4, %eax ++ salq $32, %rax ++ orq %rcx, %rax ++ tzcntq %rax, %rax ++ leaq (VEC_SIZE * 2)(%rdi, %rax), %rax ++# ifndef USE_AS_STRCHRNUL ++ cmp (%rax), %CHAR_REG ++ cmovne %rdx, %rax ++# endif ++ VZEROUPPER ++ ret ++ ++ /* Cold case for crossing page with first load. */ ++ .p2align 4 ++L(cross_page_boundary): ++ andq $-VEC_SIZE, %rdi ++ andl $(VEC_SIZE - 1), %ecx ++ ++ vmovdqa (%rdi), %ymm8 ++ VPCMPEQ %ymm8, %ymm0, %ymm1 ++ VPCMPEQ %ymm8, %ymm9, %ymm2 ++ vpor %ymm1, %ymm2, %ymm1 ++ vpmovmskb %ymm1, %eax ++ /* Remove the leading bits. */ ++ sarxl %ecx, %eax, %eax + testl %eax, %eax +-L(first_vec_x3): ++ jz L(aligned_more) + tzcntl %eax, %eax +-# ifdef USE_AS_STRCHRNUL +- addq $(VEC_SIZE * 3), %rax ++ addq %rcx, %rdi + addq %rdi, %rax +-# else +- xorl %edx, %edx +- leaq (VEC_SIZE * 3)(%rdi, %rax), %rax +- cmp (%rax), %CHAR_REG ++# ifndef USE_AS_STRCHRNUL ++ cmp (%rax), %CHAR_REG + cmovne %rdx, %rax + # endif + VZEROUPPER_RETURN + + END (STRCHR) +-#endif ++# endif +diff --git a/sysdeps/x86_64/multiarch/strchr.c b/sysdeps/x86_64/multiarch/strchr.c +index 7e582f02..5225bd4f 100644 +--- a/sysdeps/x86_64/multiarch/strchr.c ++++ b/sysdeps/x86_64/multiarch/strchr.c +@@ -38,11 +38,11 @@ IFUNC_SELECTOR (void) + const struct cpu_features* cpu_features = __get_cpu_features (); + + if (CPU_FEATURE_USABLE_P (cpu_features, AVX2) ++ && CPU_FEATURE_USABLE_P (cpu_features, BMI2) + && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) + { + if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) +- && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW) +- && CPU_FEATURE_USABLE_P (cpu_features, BMI2)) ++ && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)) + return OPTIMIZE (evex); + + if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) +-- +GitLab + diff --git a/SOURCES/ia-rmv-ofl-chk-wcsnlen-sse4_1.patch b/SOURCES/ia-rmv-ofl-chk-wcsnlen-sse4_1.patch new file mode 100644 index 0000000..f42dbe1 --- /dev/null +++ b/SOURCES/ia-rmv-ofl-chk-wcsnlen-sse4_1.patch @@ -0,0 +1,55 @@ +From 820e5fe6c8c30693cde469829b482665c8129811 Mon Sep 17 00:00:00 2001 +From: Noah Goldstein +Date: Wed, 23 Jun 2021 19:19:34 -0400 +Subject: [PATCH] x86: Remove unnecessary overflow check from wcsnlen-sse4_1.S + +No bug. The way wcsnlen will check if near the end of maxlen +is the following macro: + + mov %r11, %rsi; \ + subq %rax, %rsi; \ + andq $-64, %rax; \ + testq $-64, %rsi; \ + je L(strnlen_ret) + +Which words independently of s + maxlen overflowing. So the +second overflow check is unnecissary for correctness and +just extra overhead in the common no overflow case. + +test-strlen.c, test-wcslen.c, test-strnlen.c and test-wcsnlen.c are +all passing + +Signed-off-by: Noah Goldstein +Reviewed-by: H.J. Lu +(cherry picked from commit 08cbcd4dbc686bb38ec3093aff2f919fbff5ec17) +--- + sysdeps/x86_64/multiarch/strlen-vec.S | 7 ------- + 1 file changed, 7 deletions(-) + +diff --git a/sysdeps/x86_64/multiarch/strlen-vec.S b/sysdeps/x86_64/multiarch/strlen-vec.S +index 439e486a..b7657282 100644 +--- a/sysdeps/x86_64/multiarch/strlen-vec.S ++++ b/sysdeps/x86_64/multiarch/strlen-vec.S +@@ -71,19 +71,12 @@ L(n_nonzero): + suffice. */ + mov %RSI_LP, %R10_LP + sar $62, %R10_LP +- test %R10_LP, %R10_LP + jnz __wcslen_sse4_1 + sal $2, %RSI_LP + # endif + +- + /* Initialize long lived registers. */ +- + add %RDI_LP, %RSI_LP +-# ifdef AS_WCSLEN +-/* Check for overflow again from s + maxlen * sizeof(wchar_t). */ +- jbe __wcslen_sse4_1 +-# endif + mov %RSI_LP, %R10_LP + and $-64, %R10_LP + mov %RSI_LP, %R11_LP +-- +GitLab + diff --git a/SOURCES/ia-rmv-prefer_avx2_strcmp.patch b/SOURCES/ia-rmv-prefer_avx2_strcmp.patch new file mode 100644 index 0000000..f31187d --- /dev/null +++ b/SOURCES/ia-rmv-prefer_avx2_strcmp.patch @@ -0,0 +1,94 @@ +From 69ce2769234e36e62c2db82e1e638124f076e82e Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Wed, 2 Mar 2022 15:42:47 -0800 +Subject: [PATCH] x86-64: Remove Prefer_AVX2_STRCMP + +Remove Prefer_AVX2_STRCMP to enable EVEX strcmp. When comparing 2 32-byte +strings, EVEX strcmp has been improved to require 1 load, 1 VPTESTM, 1 +VPCMP, 1 KMOVD and 1 INCL instead of 2 loads, 3 VPCMPs, 2 KORDs, 1 KMOVD +and 1 TESTL while AVX2 strcmp requires 1 load, 2 VPCMPEQs, 1 VPMINU, 1 +VPMOVMSKB and 1 TESTL. EVEX strcmp is now faster than AVX2 strcmp by up +to 40% on Tiger Lake and Ice Lake. + +(cherry picked from commit 14dbbf46a007ae5df36646b51ad0c9e5f5259f30) +--- + sysdeps/x86/cpu-features.c | 8 -------- + sysdeps/x86/cpu-tunables.c | 2 -- + .../include/cpu-features-preferred_feature_index_1.def | 1 - + sysdeps/x86_64/multiarch/strcmp.c | 3 +-- + sysdeps/x86_64/multiarch/strncmp.c | 3 +-- + 5 files changed, 2 insertions(+), 15 deletions(-) + +diff --git a/sysdeps/x86/cpu-features.c b/sysdeps/x86/cpu-features.c +index 8885b48e..956bfb4f 100644 +--- a/sysdeps/x86/cpu-features.c ++++ b/sysdeps/x86/cpu-features.c +@@ -533,14 +533,6 @@ init_cpu_features (struct cpu_features *cpu_features) + if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) + cpu_features->preferred[index_arch_Prefer_No_VZEROUPPER] + |= bit_arch_Prefer_No_VZEROUPPER; +- +- /* Since to compare 2 32-byte strings, 256-bit EVEX strcmp +- requires 2 loads, 3 VPCMPs and 2 KORDs while AVX2 strcmp +- requires 1 load, 2 VPCMPEQs, 1 VPMINU and 1 VPMOVMSKB, +- AVX2 strcmp is faster than EVEX strcmp. */ +- if (CPU_FEATURE_USABLE_P (cpu_features, AVX2)) +- cpu_features->preferred[index_arch_Prefer_AVX2_STRCMP] +- |= bit_arch_Prefer_AVX2_STRCMP; + } + } + /* This spells out "AuthenticAMD". */ +diff --git a/sysdeps/x86/cpu-tunables.c b/sysdeps/x86/cpu-tunables.c +index 73adbaba..3173b2b9 100644 +--- a/sysdeps/x86/cpu-tunables.c ++++ b/sysdeps/x86/cpu-tunables.c +@@ -239,8 +239,6 @@ TUNABLE_CALLBACK (set_hwcaps) (tunable_val_t *valp) + CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features, + Fast_Copy_Backward, + disable, 18); +- CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH +- (n, cpu_features, Prefer_AVX2_STRCMP, AVX2, disable, 18); + } + break; + case 19: +diff --git a/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def b/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def +index f2340624..e7277b33 100644 +--- a/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def ++++ b/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def +@@ -32,5 +32,4 @@ BIT (Prefer_ERMS) + BIT (Prefer_FSRM) + BIT (Prefer_No_AVX512) + BIT (MathVec_Prefer_No_AVX512) +-BIT (Prefer_AVX2_STRCMP) + BIT (Avoid_Short_Distance_REP_MOVSB) +diff --git a/sysdeps/x86_64/multiarch/strcmp.c b/sysdeps/x86_64/multiarch/strcmp.c +index 11bbea2b..f8a7220e 100644 +--- a/sysdeps/x86_64/multiarch/strcmp.c ++++ b/sysdeps/x86_64/multiarch/strcmp.c +@@ -43,8 +43,7 @@ IFUNC_SELECTOR (void) + { + if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) + && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW) +- && CPU_FEATURE_USABLE_P (cpu_features, BMI2) +- && !CPU_FEATURES_ARCH_P (cpu_features, Prefer_AVX2_STRCMP)) ++ && CPU_FEATURE_USABLE_P (cpu_features, BMI2)) + return OPTIMIZE (evex); + + if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) +diff --git a/sysdeps/x86_64/multiarch/strncmp.c b/sysdeps/x86_64/multiarch/strncmp.c +index 44c85116..7903f807 100644 +--- a/sysdeps/x86_64/multiarch/strncmp.c ++++ b/sysdeps/x86_64/multiarch/strncmp.c +@@ -43,8 +43,7 @@ IFUNC_SELECTOR (void) + { + if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) + && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW) +- && CPU_FEATURE_USABLE_P (cpu_features, BMI2) +- && !CPU_FEATURES_ARCH_P (cpu_features, Prefer_AVX2_STRCMP)) ++ && CPU_FEATURE_USABLE_P (cpu_features, BMI2)) + return OPTIMIZE (evex); + + if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) +-- +GitLab + diff --git a/SOURCES/ia-roundeven_sse4_1.patch b/SOURCES/ia-roundeven_sse4_1.patch new file mode 100644 index 0000000..01f0153 --- /dev/null +++ b/SOURCES/ia-roundeven_sse4_1.patch @@ -0,0 +1,242 @@ +From 125c5c04cccb16dabf9d0e693df154608289d1dd Mon Sep 17 00:00:00 2001 +From: Shen-Ta Hsieh +Date: Mon, 24 May 2021 09:43:11 +0800 +Subject: [PATCH] x86_64: roundeven with sse4.1 support + +This patch adds support for the sse4.1 hardware floating point +roundeven. + +Here is some benchmark results on my systems: + +=AMD Ryzen 9 3900X 12-Core Processor= + +* benchmark result before this commit +| | roundeven | roundevenf | +|------------|--------------|--------------| +| duration | 3.75587e+09 | 3.75114e+09 | +| iterations | 3.93053e+08 | 4.35402e+08 | +| max | 52.592 | 58.71 | +| min | 7.98 | 7.22 | +| mean | 9.55563 | 8.61535 | + +* benchmark result after this commit +| | roundeven | roundevenf | +|------------|---------------|--------------| +| duration | 3.73815e+09 | 3.73738e+09 | +| iterations | 5.82692e+08 | 5.91498e+08 | +| max | 56.468 | 51.642 | +| min | 6.27 | 6.156 | +| mean | 6.41532 | 6.3185 | + +=Intel(R) Pentium(R) CPU D1508 @ 2.20GHz= + +* benchmark result before this commit +| | roundeven | roundevenf | +|------------|--------------|--------------| +| duration | 2.18208e+09 | 2.18258e+09 | +| iterations | 2.39932e+08 | 2.46924e+08 | +| max | 96.378 | 98.035 | +| min | 6.776 | 5.94 | +| mean | 9.09456 | 8.83907 | + +* benchmark result after this commit +| | roundeven | roundevenf | +|------------|--------------|--------------| +| duration | 2.17415e+09 | 2.17005e+09 | +| iterations | 3.56193e+08 | 4.09824e+08 | +| max | 51.693 | 97.192 | +| min | 5.926 | 5.093 | +| mean | 6.10385 | 5.29507 | + +Signed-off-by: Shen-Ta Hsieh +Reviewed-by: H.J. Lu +(cherry picked from commit 1683249d17e14827b6579529742eb895027dfa84) +--- + sysdeps/x86_64/fpu/multiarch/Makefile | 5 +-- + sysdeps/x86_64/fpu/multiarch/s_roundeven-c.c | 2 ++ + .../x86_64/fpu/multiarch/s_roundeven-sse4_1.S | 24 ++++++++++++++ + sysdeps/x86_64/fpu/multiarch/s_roundeven.c | 31 +++++++++++++++++++ + sysdeps/x86_64/fpu/multiarch/s_roundevenf-c.c | 3 ++ + .../fpu/multiarch/s_roundevenf-sse4_1.S | 24 ++++++++++++++ + sysdeps/x86_64/fpu/multiarch/s_roundevenf.c | 31 +++++++++++++++++++ + 7 files changed, 118 insertions(+), 2 deletions(-) + create mode 100644 sysdeps/x86_64/fpu/multiarch/s_roundeven-c.c + create mode 100644 sysdeps/x86_64/fpu/multiarch/s_roundeven-sse4_1.S + create mode 100644 sysdeps/x86_64/fpu/multiarch/s_roundeven.c + create mode 100644 sysdeps/x86_64/fpu/multiarch/s_roundevenf-c.c + create mode 100644 sysdeps/x86_64/fpu/multiarch/s_roundevenf-sse4_1.S + create mode 100644 sysdeps/x86_64/fpu/multiarch/s_roundevenf.c + +diff --git a/sysdeps/x86_64/fpu/multiarch/Makefile b/sysdeps/x86_64/fpu/multiarch/Makefile +index 9f387248..6ddd1c01 100644 +--- a/sysdeps/x86_64/fpu/multiarch/Makefile ++++ b/sysdeps/x86_64/fpu/multiarch/Makefile +@@ -1,11 +1,12 @@ + ifeq ($(subdir),math) + libm-sysdep_routines += s_floor-c s_ceil-c s_floorf-c s_ceilf-c \ + s_rint-c s_rintf-c s_nearbyint-c s_nearbyintf-c \ +- s_trunc-c s_truncf-c ++ s_roundeven-c s_roundevenf-c s_trunc-c s_truncf-c + + libm-sysdep_routines += s_ceil-sse4_1 s_ceilf-sse4_1 s_floor-sse4_1 \ + s_floorf-sse4_1 s_nearbyint-sse4_1 \ +- s_nearbyintf-sse4_1 s_rint-sse4_1 s_rintf-sse4_1 \ ++ s_nearbyintf-sse4_1 s_roundeven-sse4_1 \ ++ s_roundevenf-sse4_1 s_rint-sse4_1 s_rintf-sse4_1 \ + s_trunc-sse4_1 s_truncf-sse4_1 + + libm-sysdep_routines += e_exp-fma e_log-fma e_pow-fma s_atan-fma \ +diff --git a/sysdeps/x86_64/fpu/multiarch/s_roundeven-c.c b/sysdeps/x86_64/fpu/multiarch/s_roundeven-c.c +new file mode 100644 +index 00000000..c7be43cb +--- /dev/null ++++ b/sysdeps/x86_64/fpu/multiarch/s_roundeven-c.c +@@ -0,0 +1,2 @@ ++#define __roundeven __roundeven_c ++#include +diff --git a/sysdeps/x86_64/fpu/multiarch/s_roundeven-sse4_1.S b/sysdeps/x86_64/fpu/multiarch/s_roundeven-sse4_1.S +new file mode 100644 +index 00000000..6ae8f6b1 +--- /dev/null ++++ b/sysdeps/x86_64/fpu/multiarch/s_roundeven-sse4_1.S +@@ -0,0 +1,24 @@ ++/* Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#include ++ ++ .section .text.sse4.1,"ax",@progbits ++ENTRY(__roundeven_sse41) ++ roundsd $8, %xmm0, %xmm0 ++ ret ++END(__roundeven_sse41) +diff --git a/sysdeps/x86_64/fpu/multiarch/s_roundeven.c b/sysdeps/x86_64/fpu/multiarch/s_roundeven.c +new file mode 100644 +index 00000000..d92eda65 +--- /dev/null ++++ b/sysdeps/x86_64/fpu/multiarch/s_roundeven.c +@@ -0,0 +1,31 @@ ++/* Multiple versions of __roundeven. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#include ++ ++#define roundeven __redirect_roundeven ++#define __roundeven __redirect___roundeven ++#include ++#undef roundeven ++#undef __roundeven ++ ++#define SYMBOL_NAME roundeven ++#include "ifunc-sse4_1.h" ++ ++libc_ifunc_redirected (__redirect_roundeven, __roundeven, IFUNC_SELECTOR ()); ++libm_alias_double (__roundeven, roundeven) +diff --git a/sysdeps/x86_64/fpu/multiarch/s_roundevenf-c.c b/sysdeps/x86_64/fpu/multiarch/s_roundevenf-c.c +new file mode 100644 +index 00000000..72a6e7d1 +--- /dev/null ++++ b/sysdeps/x86_64/fpu/multiarch/s_roundevenf-c.c +@@ -0,0 +1,3 @@ ++#undef __roundevenf ++#define __roundevenf __roundevenf_c ++#include +diff --git a/sysdeps/x86_64/fpu/multiarch/s_roundevenf-sse4_1.S b/sysdeps/x86_64/fpu/multiarch/s_roundevenf-sse4_1.S +new file mode 100644 +index 00000000..a76e1080 +--- /dev/null ++++ b/sysdeps/x86_64/fpu/multiarch/s_roundevenf-sse4_1.S +@@ -0,0 +1,24 @@ ++/* Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#include ++ ++ .section .text.sse4.1,"ax",@progbits ++ENTRY(__roundevenf_sse41) ++ roundss $8, %xmm0, %xmm0 ++ ret ++END(__roundevenf_sse41) +diff --git a/sysdeps/x86_64/fpu/multiarch/s_roundevenf.c b/sysdeps/x86_64/fpu/multiarch/s_roundevenf.c +new file mode 100644 +index 00000000..2ee196e6 +--- /dev/null ++++ b/sysdeps/x86_64/fpu/multiarch/s_roundevenf.c +@@ -0,0 +1,31 @@ ++/* Multiple versions of __roundevenf. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#include ++ ++#define roundevenf __redirect_roundevenf ++#define __roundevenf __redirect___roundevenf ++#include ++#undef roundevenf ++#undef __roundevenf ++ ++#define SYMBOL_NAME roundevenf ++#include "ifunc-sse4_1.h" ++ ++libc_ifunc_redirected (__redirect_roundevenf, __roundevenf, IFUNC_SELECTOR ()); ++libm_alias_float (__roundeven, roundeven) +-- +GitLab + diff --git a/SOURCES/ia-rplc-cas-avoid-extra-load.patch b/SOURCES/ia-rplc-cas-avoid-extra-load.patch new file mode 100644 index 0000000..ed94b27 --- /dev/null +++ b/SOURCES/ia-rplc-cas-avoid-extra-load.patch @@ -0,0 +1,72 @@ +From 77950da440c2b12c153b77ab8b4d1d00c88ca3a9 Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Fri, 12 Nov 2021 11:47:42 -0800 +Subject: [PATCH] Move assignment out of the CAS condition + +Update + +commit 49302b8fdf9103b6fc0a398678668a22fa19574c +Author: H.J. Lu +Date: Thu Nov 11 06:54:01 2021 -0800 + + Avoid extra load with CAS in __pthread_mutex_clocklock_common [BZ #28537] + + Replace boolean CAS with value CAS to avoid the extra load. + +and + +commit 0b82747dc48d5bf0871bdc6da8cb6eec1256355f +Author: H.J. Lu +Date: Thu Nov 11 06:31:51 2021 -0800 + + Avoid extra load with CAS in __pthread_mutex_lock_full [BZ #28537] + + Replace boolean CAS with value CAS to avoid the extra load. + +by moving assignment out of the CAS condition. + +(cherry picked from commit 120ac6d238825452e8024e2f627da33b2508dfd3) +--- + nptl/pthread_mutex_lock.c | 7 +++---- + nptl/pthread_mutex_timedlock.c | 7 +++---- + 2 files changed, 6 insertions(+), 8 deletions(-) + +diff --git a/nptl/pthread_mutex_lock.c b/nptl/pthread_mutex_lock.c +index eb4d8baa..a633d95e 100644 +--- a/nptl/pthread_mutex_lock.c ++++ b/nptl/pthread_mutex_lock.c +@@ -299,10 +299,9 @@ __pthread_mutex_lock_full (pthread_mutex_t *mutex) + meantime. */ + if ((oldval & FUTEX_WAITERS) == 0) + { +- int val; +- if ((val = atomic_compare_and_exchange_val_acq +- (&mutex->__data.__lock, oldval | FUTEX_WAITERS, +- oldval)) != oldval) ++ int val = atomic_compare_and_exchange_val_acq ++ (&mutex->__data.__lock, oldval | FUTEX_WAITERS, oldval); ++ if (val != oldval) + { + oldval = val; + continue; +diff --git a/nptl/pthread_mutex_timedlock.c b/nptl/pthread_mutex_timedlock.c +index c4627ef6..a76c30b7 100644 +--- a/nptl/pthread_mutex_timedlock.c ++++ b/nptl/pthread_mutex_timedlock.c +@@ -269,10 +269,9 @@ __pthread_mutex_timedlock (pthread_mutex_t *mutex, + meantime. */ + if ((oldval & FUTEX_WAITERS) == 0) + { +- int val; +- if ((val = atomic_compare_and_exchange_val_acq +- (&mutex->__data.__lock, oldval | FUTEX_WAITERS, +- oldval)) != oldval) ++ int val = atomic_compare_and_exchange_val_acq ++ (&mutex->__data.__lock, oldval | FUTEX_WAITERS, oldval); ++ if (val != oldval) + { + oldval = val; + continue; +-- +GitLab + diff --git a/SOURCES/ia-rplc-movzx-movzbl.patch b/SOURCES/ia-rplc-movzx-movzbl.patch new file mode 100644 index 0000000..6e6d89c --- /dev/null +++ b/SOURCES/ia-rplc-movzx-movzbl.patch @@ -0,0 +1,54 @@ +From d867c25060d86f0223347734ef5005ad0792554c Mon Sep 17 00:00:00 2001 +From: Fangrui Song +Date: Tue, 2 Nov 2021 20:59:52 -0700 +Subject: [PATCH] x86-64: Replace movzx with movzbl + +Clang cannot assemble movzx in the AT&T dialect mode. + +../sysdeps/x86_64/strcmp.S:2232:16: error: invalid operand for instruction + movzx (%rsi), %ecx + ^~~~ + +Change movzx to movzbl, which follows the AT&T dialect and is used +elsewhere in the file. + +Reviewed-by: H.J. Lu +(cherry picked from commit 6720d36b6623c5e48c070d86acf61198b33e144e) +--- + sysdeps/x86_64/multiarch/strcmp-sse42.S | 4 ++-- + sysdeps/x86_64/strcmp.S | 4 ++-- + 2 files changed, 4 insertions(+), 4 deletions(-) + +diff --git a/sysdeps/x86_64/multiarch/strcmp-sse42.S b/sysdeps/x86_64/multiarch/strcmp-sse42.S +index a1ebea46..d8fdeb3a 100644 +--- a/sysdeps/x86_64/multiarch/strcmp-sse42.S ++++ b/sysdeps/x86_64/multiarch/strcmp-sse42.S +@@ -1771,8 +1771,8 @@ LABEL(strcmp_exitz): + .p2align 4 + // XXX Same as code above + LABEL(Byte0): +- movzx (%rsi), %ecx +- movzx (%rdi), %eax ++ movzbl (%rsi), %ecx ++ movzbl (%rdi), %eax + + #if defined USE_AS_STRCASECMP_L || defined USE_AS_STRNCASECMP_L + leaq _nl_C_LC_CTYPE_tolower+128*4(%rip), %rdx +diff --git a/sysdeps/x86_64/strcmp.S b/sysdeps/x86_64/strcmp.S +index f47c8ad4..aa6df898 100644 +--- a/sysdeps/x86_64/strcmp.S ++++ b/sysdeps/x86_64/strcmp.S +@@ -2232,8 +2232,8 @@ LABEL(strcmp_exitz): + + .p2align 4 + LABEL(Byte0): +- movzx (%rsi), %ecx +- movzx (%rdi), %eax ++ movzbl (%rsi), %ecx ++ movzbl (%rdi), %eax + + #if defined USE_AS_STRCASECMP_L || defined USE_AS_STRNCASECMP_L + leaq _nl_C_LC_CTYPE_tolower+128*4(%rip), %rdx +-- +GitLab + diff --git a/SOURCES/ia-rplc-sse2-inst-avx-memcmp-evex-movbe.patch b/SOURCES/ia-rplc-sse2-inst-avx-memcmp-evex-movbe.patch new file mode 100644 index 0000000..a42aa4b --- /dev/null +++ b/SOURCES/ia-rplc-sse2-inst-avx-memcmp-evex-movbe.patch @@ -0,0 +1,46 @@ +From e7e0ac928b21e5f47e5b648723851c5270db24f2 Mon Sep 17 00:00:00 2001 +From: Noah Goldstein +Date: Sat, 23 Oct 2021 01:26:47 -0400 +Subject: [PATCH] x86: Replace sse2 instructions with avx in + memcmp-evex-movbe.S + +This commit replaces two usages of SSE2 'movups' with AVX 'vmovdqu'. + +it could potentially be dangerous to use SSE2 if this function is ever +called without using 'vzeroupper' beforehand. While compilers appear +to use 'vzeroupper' before function calls if AVX2 has been used, using +SSE2 here is more brittle. Since it is not absolutely necessary it +should be avoided. + +It costs 2-extra bytes but the extra bytes should only eat into +alignment padding. +Reviewed-by: H.J. Lu + +(cherry picked from commit bad852b61b79503fcb3c5fc379c70f768df3e1fb) +--- + sysdeps/x86_64/multiarch/memcmp-evex-movbe.S | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S b/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S +index 2761b54f..640f6757 100644 +--- a/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S ++++ b/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S +@@ -561,13 +561,13 @@ L(between_16_31): + /* From 16 to 31 bytes. No branch when size == 16. */ + + /* Use movups to save code size. */ +- movups (%rsi), %xmm2 ++ vmovdqu (%rsi), %xmm2 + VPCMP $4, (%rdi), %xmm2, %k1 + kmovd %k1, %eax + testl %eax, %eax + jnz L(return_vec_0_lv) + /* Use overlapping loads to avoid branches. */ +- movups -16(%rsi, %rdx, CHAR_SIZE), %xmm2 ++ vmovdqu -16(%rsi, %rdx, CHAR_SIZE), %xmm2 + VPCMP $4, -16(%rdi, %rdx, CHAR_SIZE), %xmm2, %k1 + addl $(CHAR_PER_VEC - (16 / CHAR_SIZE)), %edx + kmovd %k1, %eax +-- +GitLab + diff --git a/SOURCES/ia-set-rep_movsb_threshold-2112.patch b/SOURCES/ia-set-rep_movsb_threshold-2112.patch new file mode 100644 index 0000000..fbba9b9 --- /dev/null +++ b/SOURCES/ia-set-rep_movsb_threshold-2112.patch @@ -0,0 +1,57 @@ +From 238aa7dce4d9684b6326665c79eabad72d5679bf Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Wed, 2 Mar 2022 16:21:12 -0800 +Subject: [PATCH] x86: Set rep_movsb_threshold to 2112 on processors with FSRM + +The glibc memcpy benchmark on Intel Core i7-1065G7 (Ice Lake) showed +that REP MOVSB became faster after 2112 bytes: + + Vector Move REP MOVSB +length=2112, align1=0, align2=0: 24.20 24.40 +length=2112, align1=1, align2=0: 26.07 23.13 +length=2112, align1=0, align2=1: 27.18 28.13 +length=2112, align1=1, align2=1: 26.23 25.16 +length=2176, align1=0, align2=0: 23.18 22.52 +length=2176, align1=2, align2=0: 25.45 22.52 +length=2176, align1=0, align2=2: 27.14 27.82 +length=2176, align1=2, align2=2: 22.73 25.56 +length=2240, align1=0, align2=0: 24.62 24.25 +length=2240, align1=3, align2=0: 29.77 27.15 +length=2240, align1=0, align2=3: 35.55 29.93 +length=2240, align1=3, align2=3: 34.49 25.15 +length=2304, align1=0, align2=0: 34.75 26.64 +length=2304, align1=4, align2=0: 32.09 22.63 +length=2304, align1=0, align2=4: 28.43 31.24 + +Use REP MOVSB for data size > 2112 bytes in memcpy on processors with +fast short REP MOVSB (FSRM). + + * sysdeps/x86/dl-cacheinfo.h (dl_init_cacheinfo): Set + rep_movsb_threshold to 2112 on processors with fast short REP + MOVSB (FSRM). + +(cherry picked from commit cf2c57526ba4b57e6863ad4db8a868e2678adce8) +--- + sysdeps/x86/cacheinfo.h | 6 ++++++ + 1 file changed, 6 insertions(+) + +diff --git a/sysdeps/x86/cacheinfo.h b/sysdeps/x86/cacheinfo.h +index f72f634a..cc3941d3 100644 +--- a/sysdeps/x86/cacheinfo.h ++++ b/sysdeps/x86/cacheinfo.h +@@ -430,6 +430,12 @@ init_cacheinfo (void) + rep_movsb_threshold = 2048 * (16 / 16); + minimum_rep_movsb_threshold = 16 * 8; + } ++ ++ /* NB: The default REP MOVSB threshold is 2112 on processors with fast ++ short REP MOVSB (FSRM). */ ++ if (CPU_FEATURE_USABLE_P (cpu_features, FSRM)) ++ rep_movsb_threshold = 2112; ++ + if (cpu_features->rep_movsb_threshold > minimum_rep_movsb_threshold) + __x86_rep_movsb_threshold = cpu_features->rep_movsb_threshold; + else +-- +GitLab + diff --git a/SOURCES/ia-shrink-memcmp-sse4-code-size.patch b/SOURCES/ia-shrink-memcmp-sse4-code-size.patch new file mode 100644 index 0000000..86d4d16 --- /dev/null +++ b/SOURCES/ia-shrink-memcmp-sse4-code-size.patch @@ -0,0 +1,2428 @@ +From 17bb3526d088d8854b446a6faabf1fad3812b2c9 Mon Sep 17 00:00:00 2001 +From: Noah Goldstein +Date: Wed, 10 Nov 2021 16:18:56 -0600 +Subject: [PATCH] x86: Shrink memcmp-sse4.S code size + +No bug. + +This implementation refactors memcmp-sse4.S primarily with minimizing +code size in mind. It does this by removing the lookup table logic and +removing the unrolled check from (256, 512] bytes. + +memcmp-sse4 code size reduction : -3487 bytes +wmemcmp-sse4 code size reduction: -1472 bytes + +The current memcmp-sse4.S implementation has a large code size +cost. This has serious adverse affects on the ICache / ITLB. While +in micro-benchmarks the implementations appears fast, traces of +real-world code have shown that the speed in micro benchmarks does not +translate when the ICache/ITLB are not primed, and that the cost +of the code size has measurable negative affects on overall +application performance. + +See https://research.google/pubs/pub48320/ for more details. + +Signed-off-by: Noah Goldstein +Reviewed-by: H.J. Lu +(cherry picked from commit 2f9062d7171850451e6044ef78d91ff8c017b9c0) +--- + sysdeps/x86_64/multiarch/memcmp-sse4.S | 2267 +++++++----------------- + 1 file changed, 646 insertions(+), 1621 deletions(-) + +diff --git a/sysdeps/x86_64/multiarch/memcmp-sse4.S b/sysdeps/x86_64/multiarch/memcmp-sse4.S +index 302900f5..50060006 100644 +--- a/sysdeps/x86_64/multiarch/memcmp-sse4.S ++++ b/sysdeps/x86_64/multiarch/memcmp-sse4.S +@@ -25,14 +25,14 @@ + # define MEMCMP __memcmp_sse4_1 + # endif + +-# define JMPTBL(I, B) (I - B) ++#ifdef USE_AS_WMEMCMP ++# define CMPEQ pcmpeqd ++# define CHAR_SIZE 4 ++#else ++# define CMPEQ pcmpeqb ++# define CHAR_SIZE 1 ++#endif + +-# define BRANCH_TO_JMPTBL_ENTRY(TABLE, INDEX, SCALE) \ +- lea TABLE(%rip), %r11; \ +- movslq (%r11, INDEX, SCALE), %rcx; \ +- add %r11, %rcx; \ +- _CET_NOTRACK jmp *%rcx; \ +- ud2 + + /* Warning! + wmemcmp has to use SIGNED comparison for elements. +@@ -47,33 +47,253 @@ ENTRY (MEMCMP) + /* Clear the upper 32 bits. */ + mov %edx, %edx + # endif +- pxor %xmm0, %xmm0 + cmp $79, %RDX_LP + ja L(79bytesormore) ++ ++ cmp $CHAR_SIZE, %RDX_LP ++ jbe L(firstbyte) ++ ++ /* N in (CHAR_SIZE, 79) bytes. */ ++ cmpl $32, %edx ++ ja L(more_32_bytes) ++ ++ cmpl $16, %edx ++ jae L(16_to_32_bytes) ++ + # ifndef USE_AS_WMEMCMP +- cmp $1, %RDX_LP +- je L(firstbyte) ++ cmpl $8, %edx ++ jae L(8_to_16_bytes) ++ ++ cmpl $4, %edx ++ jb L(2_to_3_bytes) ++ ++ movl (%rdi), %eax ++ movl (%rsi), %ecx ++ ++ bswap %eax ++ bswap %ecx ++ ++ shlq $32, %rax ++ shlq $32, %rcx ++ ++ movl -4(%rdi, %rdx), %edi ++ movl -4(%rsi, %rdx), %esi ++ ++ bswap %edi ++ bswap %esi ++ ++ orq %rdi, %rax ++ orq %rsi, %rcx ++ subq %rcx, %rax ++ cmovne %edx, %eax ++ sbbl %ecx, %ecx ++ orl %ecx, %eax ++ ret ++ ++ .p2align 4,, 8 ++L(2_to_3_bytes): ++ movzwl (%rdi), %eax ++ movzwl (%rsi), %ecx ++ shll $8, %eax ++ shll $8, %ecx ++ bswap %eax ++ bswap %ecx ++ movzbl -1(%rdi, %rdx), %edi ++ movzbl -1(%rsi, %rdx), %esi ++ orl %edi, %eax ++ orl %esi, %ecx ++ subl %ecx, %eax ++ ret ++ ++ .p2align 4,, 8 ++L(8_to_16_bytes): ++ movq (%rdi), %rax ++ movq (%rsi), %rcx ++ ++ bswap %rax ++ bswap %rcx ++ ++ subq %rcx, %rax ++ jne L(8_to_16_bytes_done) ++ ++ movq -8(%rdi, %rdx), %rax ++ movq -8(%rsi, %rdx), %rcx ++ ++ bswap %rax ++ bswap %rcx ++ ++ subq %rcx, %rax ++ ++L(8_to_16_bytes_done): ++ cmovne %edx, %eax ++ sbbl %ecx, %ecx ++ orl %ecx, %eax ++ ret ++# else ++ xorl %eax, %eax ++ movl (%rdi), %ecx ++ cmpl (%rsi), %ecx ++ jne L(8_to_16_bytes_done) ++ movl 4(%rdi), %ecx ++ cmpl 4(%rsi), %ecx ++ jne L(8_to_16_bytes_done) ++ movl -4(%rdi, %rdx), %ecx ++ cmpl -4(%rsi, %rdx), %ecx ++ jne L(8_to_16_bytes_done) ++ ret + # endif +- add %rdx, %rsi +- add %rdx, %rdi +- BRANCH_TO_JMPTBL_ENTRY(L(table_64bytes), %rdx, 4) + +-# ifndef USE_AS_WMEMCMP +- .p2align 4 ++ .p2align 4,, 3 ++L(ret_zero): ++ xorl %eax, %eax ++L(zero): ++ ret ++ ++ .p2align 4,, 8 + L(firstbyte): ++ jb L(ret_zero) ++# ifdef USE_AS_WMEMCMP ++ xorl %eax, %eax ++ movl (%rdi), %ecx ++ cmpl (%rsi), %ecx ++ je L(zero) ++L(8_to_16_bytes_done): ++ setg %al ++ leal -1(%rax, %rax), %eax ++# else + movzbl (%rdi), %eax + movzbl (%rsi), %ecx + sub %ecx, %eax ++# endif + ret ++ ++ .p2align 4 ++L(vec_return_begin_48): ++ addq $16, %rdi ++ addq $16, %rsi ++L(vec_return_begin_32): ++ bsfl %eax, %eax ++# ifdef USE_AS_WMEMCMP ++ movl 32(%rdi, %rax), %ecx ++ xorl %edx, %edx ++ cmpl 32(%rsi, %rax), %ecx ++ setg %dl ++ leal -1(%rdx, %rdx), %eax ++# else ++ movzbl 32(%rsi, %rax), %ecx ++ movzbl 32(%rdi, %rax), %eax ++ subl %ecx, %eax ++# endif ++ ret ++ ++ .p2align 4 ++L(vec_return_begin_16): ++ addq $16, %rdi ++ addq $16, %rsi ++L(vec_return_begin): ++ bsfl %eax, %eax ++# ifdef USE_AS_WMEMCMP ++ movl (%rdi, %rax), %ecx ++ xorl %edx, %edx ++ cmpl (%rsi, %rax), %ecx ++ setg %dl ++ leal -1(%rdx, %rdx), %eax ++# else ++ movzbl (%rsi, %rax), %ecx ++ movzbl (%rdi, %rax), %eax ++ subl %ecx, %eax ++# endif ++ ret ++ ++ .p2align 4 ++L(vec_return_end_16): ++ subl $16, %edx ++L(vec_return_end): ++ bsfl %eax, %eax ++ addl %edx, %eax ++# ifdef USE_AS_WMEMCMP ++ movl -16(%rdi, %rax), %ecx ++ xorl %edx, %edx ++ cmpl -16(%rsi, %rax), %ecx ++ setg %dl ++ leal -1(%rdx, %rdx), %eax ++# else ++ movzbl -16(%rsi, %rax), %ecx ++ movzbl -16(%rdi, %rax), %eax ++ subl %ecx, %eax + # endif ++ ret ++ ++ .p2align 4,, 8 ++L(more_32_bytes): ++ movdqu (%rdi), %xmm0 ++ movdqu (%rsi), %xmm1 ++ CMPEQ %xmm0, %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_begin) ++ ++ movdqu 16(%rdi), %xmm0 ++ movdqu 16(%rsi), %xmm1 ++ CMPEQ %xmm0, %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_begin_16) ++ ++ cmpl $64, %edx ++ jbe L(32_to_64_bytes) ++ movdqu 32(%rdi), %xmm0 ++ movdqu 32(%rsi), %xmm1 ++ CMPEQ %xmm0, %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_begin_32) ++ ++ .p2align 4,, 6 ++L(32_to_64_bytes): ++ movdqu -32(%rdi, %rdx), %xmm0 ++ movdqu -32(%rsi, %rdx), %xmm1 ++ CMPEQ %xmm0, %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_end_16) ++ ++ movdqu -16(%rdi, %rdx), %xmm0 ++ movdqu -16(%rsi, %rdx), %xmm1 ++ CMPEQ %xmm0, %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_end) ++ ret ++ ++ .p2align 4 ++L(16_to_32_bytes): ++ movdqu (%rdi), %xmm0 ++ movdqu (%rsi), %xmm1 ++ CMPEQ %xmm0, %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_begin) ++ ++ movdqu -16(%rdi, %rdx), %xmm0 ++ movdqu -16(%rsi, %rdx), %xmm1 ++ CMPEQ %xmm0, %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_end) ++ ret ++ + + .p2align 4 + L(79bytesormore): ++ movdqu (%rdi), %xmm0 + movdqu (%rsi), %xmm1 +- movdqu (%rdi), %xmm2 +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(16bytesin256) ++ CMPEQ %xmm0, %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_begin) ++ ++ + mov %rsi, %rcx + and $-16, %rsi + add $16, %rsi +@@ -86,1694 +306,499 @@ L(79bytesormore): + + cmp $128, %rdx + ja L(128bytesormore) +-L(less128bytes): +- sub $64, %rdx +- +- movdqu (%rdi), %xmm2 +- pxor (%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(16bytesin256) + +- movdqu 16(%rdi), %xmm2 +- pxor 16(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(32bytesin256) +- +- movdqu 32(%rdi), %xmm2 +- pxor 32(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(48bytesin256) +- +- movdqu 48(%rdi), %xmm2 +- pxor 48(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(64bytesin256) +- cmp $32, %rdx +- jb L(less32bytesin64) +- +- movdqu 64(%rdi), %xmm2 +- pxor 64(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(80bytesin256) +- +- movdqu 80(%rdi), %xmm2 +- pxor 80(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(96bytesin256) +- sub $32, %rdx +- add $32, %rdi +- add $32, %rsi +-L(less32bytesin64): +- add $64, %rdi +- add $64, %rsi +- add %rdx, %rsi +- add %rdx, %rdi +- BRANCH_TO_JMPTBL_ENTRY(L(table_64bytes), %rdx, 4) ++ .p2align 4,, 6 ++L(less128bytes): ++ movdqu (%rdi), %xmm1 ++ CMPEQ (%rsi), %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_begin) ++ ++ movdqu 16(%rdi), %xmm1 ++ CMPEQ 16(%rsi), %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_begin_16) ++ ++ movdqu 32(%rdi), %xmm1 ++ CMPEQ 32(%rsi), %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_begin_32) ++ ++ movdqu 48(%rdi), %xmm1 ++ CMPEQ 48(%rsi), %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_begin_48) ++ ++ cmp $96, %rdx ++ jb L(32_to_64_bytes) ++ ++ addq $64, %rdi ++ addq $64, %rsi ++ subq $64, %rdx ++ ++ .p2align 4,, 6 ++L(last_64_bytes): ++ movdqu (%rdi), %xmm1 ++ CMPEQ (%rsi), %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_begin) ++ ++ movdqu 16(%rdi), %xmm1 ++ CMPEQ 16(%rsi), %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_begin_16) ++ ++ movdqu -32(%rdi, %rdx), %xmm0 ++ movdqu -32(%rsi, %rdx), %xmm1 ++ CMPEQ %xmm0, %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_end_16) ++ ++ movdqu -16(%rdi, %rdx), %xmm0 ++ movdqu -16(%rsi, %rdx), %xmm1 ++ CMPEQ %xmm0, %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_end) ++ ret + ++ .p2align 4 + L(128bytesormore): +- cmp $512, %rdx +- ja L(512bytesormore) + cmp $256, %rdx +- ja L(less512bytes) ++ ja L(unaligned_loop) + L(less256bytes): +- sub $128, %rdx +- +- movdqu (%rdi), %xmm2 +- pxor (%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(16bytesin256) +- +- movdqu 16(%rdi), %xmm2 +- pxor 16(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(32bytesin256) +- +- movdqu 32(%rdi), %xmm2 +- pxor 32(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(48bytesin256) +- +- movdqu 48(%rdi), %xmm2 +- pxor 48(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(64bytesin256) +- +- movdqu 64(%rdi), %xmm2 +- pxor 64(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(80bytesin256) +- +- movdqu 80(%rdi), %xmm2 +- pxor 80(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(96bytesin256) +- +- movdqu 96(%rdi), %xmm2 +- pxor 96(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(112bytesin256) +- +- movdqu 112(%rdi), %xmm2 +- pxor 112(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(128bytesin256) +- +- add $128, %rsi +- add $128, %rdi +- +- cmp $64, %rdx +- jae L(less128bytes) +- +- cmp $32, %rdx +- jb L(less32bytesin128) +- +- movdqu (%rdi), %xmm2 +- pxor (%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(16bytesin256) +- +- movdqu 16(%rdi), %xmm2 +- pxor 16(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(32bytesin256) +- sub $32, %rdx +- add $32, %rdi +- add $32, %rsi +-L(less32bytesin128): +- add %rdx, %rsi +- add %rdx, %rdi +- BRANCH_TO_JMPTBL_ENTRY(L(table_64bytes), %rdx, 4) +- +-L(less512bytes): +- sub $256, %rdx +- movdqu (%rdi), %xmm2 +- pxor (%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(16bytesin256) +- +- movdqu 16(%rdi), %xmm2 +- pxor 16(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(32bytesin256) +- +- movdqu 32(%rdi), %xmm2 +- pxor 32(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(48bytesin256) +- +- movdqu 48(%rdi), %xmm2 +- pxor 48(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(64bytesin256) +- +- movdqu 64(%rdi), %xmm2 +- pxor 64(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(80bytesin256) +- +- movdqu 80(%rdi), %xmm2 +- pxor 80(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(96bytesin256) +- +- movdqu 96(%rdi), %xmm2 +- pxor 96(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(112bytesin256) +- +- movdqu 112(%rdi), %xmm2 +- pxor 112(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(128bytesin256) +- +- movdqu 128(%rdi), %xmm2 +- pxor 128(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(144bytesin256) +- +- movdqu 144(%rdi), %xmm2 +- pxor 144(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(160bytesin256) +- +- movdqu 160(%rdi), %xmm2 +- pxor 160(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(176bytesin256) +- +- movdqu 176(%rdi), %xmm2 +- pxor 176(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(192bytesin256) +- +- movdqu 192(%rdi), %xmm2 +- pxor 192(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(208bytesin256) +- +- movdqu 208(%rdi), %xmm2 +- pxor 208(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(224bytesin256) +- +- movdqu 224(%rdi), %xmm2 +- pxor 224(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(240bytesin256) +- +- movdqu 240(%rdi), %xmm2 +- pxor 240(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(256bytesin256) +- +- add $256, %rsi +- add $256, %rdi +- +- cmp $128, %rdx +- jae L(less256bytes) ++ movdqu (%rdi), %xmm1 ++ CMPEQ (%rsi), %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_begin) ++ ++ movdqu 16(%rdi), %xmm1 ++ CMPEQ 16(%rsi), %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_begin_16) ++ ++ movdqu 32(%rdi), %xmm1 ++ CMPEQ 32(%rsi), %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_begin_32) ++ ++ movdqu 48(%rdi), %xmm1 ++ CMPEQ 48(%rsi), %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_begin_48) ++ ++ addq $64, %rdi ++ addq $64, %rsi ++ ++ movdqu (%rdi), %xmm1 ++ CMPEQ (%rsi), %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_begin) ++ ++ movdqu 16(%rdi), %xmm1 ++ CMPEQ 16(%rsi), %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_begin_16) ++ ++ movdqu 32(%rdi), %xmm1 ++ CMPEQ 32(%rsi), %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_begin_32) ++ ++ movdqu 48(%rdi), %xmm1 ++ CMPEQ 48(%rsi), %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_begin_48) ++ ++ addq $-128, %rdx ++ subq $-64, %rsi ++ subq $-64, %rdi + + cmp $64, %rdx +- jae L(less128bytes) ++ ja L(less128bytes) + + cmp $32, %rdx +- jb L(less32bytesin256) +- +- movdqu (%rdi), %xmm2 +- pxor (%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(16bytesin256) +- +- movdqu 16(%rdi), %xmm2 +- pxor 16(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(32bytesin256) +- sub $32, %rdx +- add $32, %rdi +- add $32, %rsi +-L(less32bytesin256): +- add %rdx, %rsi +- add %rdx, %rdi +- BRANCH_TO_JMPTBL_ENTRY(L(table_64bytes), %rdx, 4) ++ ja L(last_64_bytes) ++ ++ movdqu -32(%rdi, %rdx), %xmm0 ++ movdqu -32(%rsi, %rdx), %xmm1 ++ CMPEQ %xmm0, %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_end_16) ++ ++ movdqu -16(%rdi, %rdx), %xmm0 ++ movdqu -16(%rsi, %rdx), %xmm1 ++ CMPEQ %xmm0, %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_end) ++ ret + + .p2align 4 +-L(512bytesormore): ++L(unaligned_loop): + # ifdef DATA_CACHE_SIZE_HALF + mov $DATA_CACHE_SIZE_HALF, %R8_LP + # else + mov __x86_data_cache_size_half(%rip), %R8_LP + # endif +- mov %r8, %r9 +- shr $1, %r8 +- add %r9, %r8 +- cmp %r8, %rdx +- ja L(L2_L3_cache_unaglined) ++ movq %r8, %r9 ++ addq %r8, %r8 ++ addq %r9, %r8 ++ cmpq %r8, %rdx ++ ja L(L2_L3_cache_unaligned) + sub $64, %rdx + .p2align 4 + L(64bytesormore_loop): +- movdqu (%rdi), %xmm2 +- pxor (%rsi), %xmm2 +- movdqa %xmm2, %xmm1 ++ movdqu (%rdi), %xmm0 ++ movdqu 16(%rdi), %xmm1 ++ movdqu 32(%rdi), %xmm2 ++ movdqu 48(%rdi), %xmm3 + +- movdqu 16(%rdi), %xmm3 +- pxor 16(%rsi), %xmm3 +- por %xmm3, %xmm1 ++ CMPEQ (%rsi), %xmm0 ++ CMPEQ 16(%rsi), %xmm1 ++ CMPEQ 32(%rsi), %xmm2 ++ CMPEQ 48(%rsi), %xmm3 + +- movdqu 32(%rdi), %xmm4 +- pxor 32(%rsi), %xmm4 +- por %xmm4, %xmm1 ++ pand %xmm0, %xmm1 ++ pand %xmm2, %xmm3 ++ pand %xmm1, %xmm3 + +- movdqu 48(%rdi), %xmm5 +- pxor 48(%rsi), %xmm5 +- por %xmm5, %xmm1 ++ pmovmskb %xmm3, %eax ++ incw %ax ++ jnz L(64bytesormore_loop_end) + +- ptest %xmm1, %xmm0 +- jnc L(64bytesormore_loop_end) + add $64, %rsi + add $64, %rdi + sub $64, %rdx +- jae L(64bytesormore_loop) ++ ja L(64bytesormore_loop) + +- add $64, %rdx +- add %rdx, %rsi +- add %rdx, %rdi +- BRANCH_TO_JMPTBL_ENTRY(L(table_64bytes), %rdx, 4) ++ .p2align 4,, 6 ++L(loop_tail): ++ addq %rdx, %rdi ++ movdqu (%rdi), %xmm0 ++ movdqu 16(%rdi), %xmm1 ++ movdqu 32(%rdi), %xmm2 ++ movdqu 48(%rdi), %xmm3 ++ ++ addq %rdx, %rsi ++ movdqu (%rsi), %xmm4 ++ movdqu 16(%rsi), %xmm5 ++ movdqu 32(%rsi), %xmm6 ++ movdqu 48(%rsi), %xmm7 ++ ++ CMPEQ %xmm4, %xmm0 ++ CMPEQ %xmm5, %xmm1 ++ CMPEQ %xmm6, %xmm2 ++ CMPEQ %xmm7, %xmm3 ++ ++ pand %xmm0, %xmm1 ++ pand %xmm2, %xmm3 ++ pand %xmm1, %xmm3 ++ ++ pmovmskb %xmm3, %eax ++ incw %ax ++ jnz L(64bytesormore_loop_end) ++ ret + +-L(L2_L3_cache_unaglined): +- sub $64, %rdx ++L(L2_L3_cache_unaligned): ++ subq $64, %rdx + .p2align 4 + L(L2_L3_unaligned_128bytes_loop): + prefetchnta 0x1c0(%rdi) + prefetchnta 0x1c0(%rsi) +- movdqu (%rdi), %xmm2 +- pxor (%rsi), %xmm2 +- movdqa %xmm2, %xmm1 + +- movdqu 16(%rdi), %xmm3 +- pxor 16(%rsi), %xmm3 +- por %xmm3, %xmm1 ++ movdqu (%rdi), %xmm0 ++ movdqu 16(%rdi), %xmm1 ++ movdqu 32(%rdi), %xmm2 ++ movdqu 48(%rdi), %xmm3 ++ ++ CMPEQ (%rsi), %xmm0 ++ CMPEQ 16(%rsi), %xmm1 ++ CMPEQ 32(%rsi), %xmm2 ++ CMPEQ 48(%rsi), %xmm3 + +- movdqu 32(%rdi), %xmm4 +- pxor 32(%rsi), %xmm4 +- por %xmm4, %xmm1 ++ pand %xmm0, %xmm1 ++ pand %xmm2, %xmm3 ++ pand %xmm1, %xmm3 + +- movdqu 48(%rdi), %xmm5 +- pxor 48(%rsi), %xmm5 +- por %xmm5, %xmm1 ++ pmovmskb %xmm3, %eax ++ incw %ax ++ jnz L(64bytesormore_loop_end) + +- ptest %xmm1, %xmm0 +- jnc L(64bytesormore_loop_end) + add $64, %rsi + add $64, %rdi + sub $64, %rdx +- jae L(L2_L3_unaligned_128bytes_loop) ++ ja L(L2_L3_unaligned_128bytes_loop) ++ jmp L(loop_tail) + +- add $64, %rdx +- add %rdx, %rsi +- add %rdx, %rdi +- BRANCH_TO_JMPTBL_ENTRY(L(table_64bytes), %rdx, 4) + +-/* +- * This case is for machines which are sensitive for unaligned instructions. +- */ ++ /* This case is for machines which are sensitive for unaligned ++ * instructions. */ + .p2align 4 + L(2aligned): + cmp $128, %rdx + ja L(128bytesormorein2aligned) + L(less128bytesin2aligned): +- sub $64, %rdx +- +- movdqa (%rdi), %xmm2 +- pxor (%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(16bytesin256) +- +- movdqa 16(%rdi), %xmm2 +- pxor 16(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(32bytesin256) +- +- movdqa 32(%rdi), %xmm2 +- pxor 32(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(48bytesin256) +- +- movdqa 48(%rdi), %xmm2 +- pxor 48(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(64bytesin256) +- cmp $32, %rdx +- jb L(less32bytesin64in2alinged) +- +- movdqa 64(%rdi), %xmm2 +- pxor 64(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(80bytesin256) +- +- movdqa 80(%rdi), %xmm2 +- pxor 80(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(96bytesin256) +- sub $32, %rdx +- add $32, %rdi +- add $32, %rsi +-L(less32bytesin64in2alinged): +- add $64, %rdi +- add $64, %rsi +- add %rdx, %rsi +- add %rdx, %rdi +- BRANCH_TO_JMPTBL_ENTRY(L(table_64bytes), %rdx, 4) ++ movdqa (%rdi), %xmm1 ++ CMPEQ (%rsi), %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_begin) ++ ++ movdqa 16(%rdi), %xmm1 ++ CMPEQ 16(%rsi), %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_begin_16) ++ ++ movdqa 32(%rdi), %xmm1 ++ CMPEQ 32(%rsi), %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_begin_32) ++ ++ movdqa 48(%rdi), %xmm1 ++ CMPEQ 48(%rsi), %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_begin_48) ++ ++ cmp $96, %rdx ++ jb L(32_to_64_bytes) ++ ++ addq $64, %rdi ++ addq $64, %rsi ++ subq $64, %rdx ++ ++ .p2align 4,, 6 ++L(aligned_last_64_bytes): ++ movdqa (%rdi), %xmm1 ++ CMPEQ (%rsi), %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_begin) ++ ++ movdqa 16(%rdi), %xmm1 ++ CMPEQ 16(%rsi), %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_begin_16) ++ ++ movdqu -32(%rdi, %rdx), %xmm0 ++ movdqu -32(%rsi, %rdx), %xmm1 ++ CMPEQ %xmm0, %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_end_16) ++ ++ movdqu -16(%rdi, %rdx), %xmm0 ++ movdqu -16(%rsi, %rdx), %xmm1 ++ CMPEQ %xmm0, %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_end) ++ ret + + .p2align 4 + L(128bytesormorein2aligned): +- cmp $512, %rdx +- ja L(512bytesormorein2aligned) + cmp $256, %rdx +- ja L(256bytesormorein2aligned) ++ ja L(aligned_loop) + L(less256bytesin2alinged): +- sub $128, %rdx +- +- movdqa (%rdi), %xmm2 +- pxor (%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(16bytesin256) +- +- movdqa 16(%rdi), %xmm2 +- pxor 16(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(32bytesin256) +- +- movdqa 32(%rdi), %xmm2 +- pxor 32(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(48bytesin256) +- +- movdqa 48(%rdi), %xmm2 +- pxor 48(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(64bytesin256) +- +- movdqa 64(%rdi), %xmm2 +- pxor 64(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(80bytesin256) +- +- movdqa 80(%rdi), %xmm2 +- pxor 80(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(96bytesin256) +- +- movdqa 96(%rdi), %xmm2 +- pxor 96(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(112bytesin256) +- +- movdqa 112(%rdi), %xmm2 +- pxor 112(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(128bytesin256) +- +- add $128, %rsi +- add $128, %rdi ++ movdqa (%rdi), %xmm1 ++ CMPEQ (%rsi), %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_begin) ++ ++ movdqa 16(%rdi), %xmm1 ++ CMPEQ 16(%rsi), %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_begin_16) ++ ++ movdqa 32(%rdi), %xmm1 ++ CMPEQ 32(%rsi), %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_begin_32) ++ ++ movdqa 48(%rdi), %xmm1 ++ CMPEQ 48(%rsi), %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_begin_48) ++ ++ addq $64, %rdi ++ addq $64, %rsi ++ ++ movdqa (%rdi), %xmm1 ++ CMPEQ (%rsi), %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_begin) ++ ++ movdqa 16(%rdi), %xmm1 ++ CMPEQ 16(%rsi), %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_begin_16) ++ ++ movdqa 32(%rdi), %xmm1 ++ CMPEQ 32(%rsi), %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_begin_32) ++ ++ movdqa 48(%rdi), %xmm1 ++ CMPEQ 48(%rsi), %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_begin_48) ++ ++ addq $-128, %rdx ++ subq $-64, %rsi ++ subq $-64, %rdi + + cmp $64, %rdx +- jae L(less128bytesin2aligned) ++ ja L(less128bytesin2aligned) + + cmp $32, %rdx +- jb L(less32bytesin128in2aligned) +- +- movdqu (%rdi), %xmm2 +- pxor (%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(16bytesin256) +- +- movdqu 16(%rdi), %xmm2 +- pxor 16(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(32bytesin256) +- sub $32, %rdx +- add $32, %rdi +- add $32, %rsi +-L(less32bytesin128in2aligned): +- add %rdx, %rsi +- add %rdx, %rdi +- BRANCH_TO_JMPTBL_ENTRY(L(table_64bytes), %rdx, 4) +- +- .p2align 4 +-L(256bytesormorein2aligned): +- +- sub $256, %rdx +- movdqa (%rdi), %xmm2 +- pxor (%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(16bytesin256) +- +- movdqa 16(%rdi), %xmm2 +- pxor 16(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(32bytesin256) +- +- movdqa 32(%rdi), %xmm2 +- pxor 32(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(48bytesin256) +- +- movdqa 48(%rdi), %xmm2 +- pxor 48(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(64bytesin256) +- +- movdqa 64(%rdi), %xmm2 +- pxor 64(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(80bytesin256) +- +- movdqa 80(%rdi), %xmm2 +- pxor 80(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(96bytesin256) +- +- movdqa 96(%rdi), %xmm2 +- pxor 96(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(112bytesin256) +- +- movdqa 112(%rdi), %xmm2 +- pxor 112(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(128bytesin256) +- +- movdqa 128(%rdi), %xmm2 +- pxor 128(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(144bytesin256) +- +- movdqa 144(%rdi), %xmm2 +- pxor 144(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(160bytesin256) +- +- movdqa 160(%rdi), %xmm2 +- pxor 160(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(176bytesin256) +- +- movdqa 176(%rdi), %xmm2 +- pxor 176(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(192bytesin256) +- +- movdqa 192(%rdi), %xmm2 +- pxor 192(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(208bytesin256) +- +- movdqa 208(%rdi), %xmm2 +- pxor 208(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(224bytesin256) +- +- movdqa 224(%rdi), %xmm2 +- pxor 224(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(240bytesin256) +- +- movdqa 240(%rdi), %xmm2 +- pxor 240(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(256bytesin256) +- +- add $256, %rsi +- add $256, %rdi +- +- cmp $128, %rdx +- jae L(less256bytesin2alinged) +- +- cmp $64, %rdx +- jae L(less128bytesin2aligned) +- +- cmp $32, %rdx +- jb L(less32bytesin256in2alinged) +- +- movdqa (%rdi), %xmm2 +- pxor (%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(16bytesin256) +- +- movdqa 16(%rdi), %xmm2 +- pxor 16(%rsi), %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(32bytesin256) +- sub $32, %rdx +- add $32, %rdi +- add $32, %rsi +-L(less32bytesin256in2alinged): +- add %rdx, %rsi +- add %rdx, %rdi +- BRANCH_TO_JMPTBL_ENTRY(L(table_64bytes), %rdx, 4) ++ ja L(aligned_last_64_bytes) ++ ++ movdqu -32(%rdi, %rdx), %xmm0 ++ movdqu -32(%rsi, %rdx), %xmm1 ++ CMPEQ %xmm0, %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_end_16) ++ ++ movdqu -16(%rdi, %rdx), %xmm0 ++ movdqu -16(%rsi, %rdx), %xmm1 ++ CMPEQ %xmm0, %xmm1 ++ pmovmskb %xmm1, %eax ++ incw %ax ++ jnz L(vec_return_end) ++ ret + + .p2align 4 +-L(512bytesormorein2aligned): ++L(aligned_loop): + # ifdef DATA_CACHE_SIZE_HALF + mov $DATA_CACHE_SIZE_HALF, %R8_LP + # else + mov __x86_data_cache_size_half(%rip), %R8_LP + # endif +- mov %r8, %r9 +- shr $1, %r8 +- add %r9, %r8 +- cmp %r8, %rdx +- ja L(L2_L3_cache_aglined) ++ movq %r8, %r9 ++ addq %r8, %r8 ++ addq %r9, %r8 ++ cmpq %r8, %rdx ++ ja L(L2_L3_cache_aligned) + + sub $64, %rdx + .p2align 4 + L(64bytesormore_loopin2aligned): +- movdqa (%rdi), %xmm2 +- pxor (%rsi), %xmm2 +- movdqa %xmm2, %xmm1 +- +- movdqa 16(%rdi), %xmm3 +- pxor 16(%rsi), %xmm3 +- por %xmm3, %xmm1 ++ movdqa (%rdi), %xmm0 ++ movdqa 16(%rdi), %xmm1 ++ movdqa 32(%rdi), %xmm2 ++ movdqa 48(%rdi), %xmm3 + +- movdqa 32(%rdi), %xmm4 +- pxor 32(%rsi), %xmm4 +- por %xmm4, %xmm1 ++ CMPEQ (%rsi), %xmm0 ++ CMPEQ 16(%rsi), %xmm1 ++ CMPEQ 32(%rsi), %xmm2 ++ CMPEQ 48(%rsi), %xmm3 + +- movdqa 48(%rdi), %xmm5 +- pxor 48(%rsi), %xmm5 +- por %xmm5, %xmm1 ++ pand %xmm0, %xmm1 ++ pand %xmm2, %xmm3 ++ pand %xmm1, %xmm3 + +- ptest %xmm1, %xmm0 +- jnc L(64bytesormore_loop_end) ++ pmovmskb %xmm3, %eax ++ incw %ax ++ jnz L(64bytesormore_loop_end) + add $64, %rsi + add $64, %rdi + sub $64, %rdx +- jae L(64bytesormore_loopin2aligned) +- +- add $64, %rdx +- add %rdx, %rsi +- add %rdx, %rdi +- BRANCH_TO_JMPTBL_ENTRY(L(table_64bytes), %rdx, 4) +-L(L2_L3_cache_aglined): +- sub $64, %rdx ++ ja L(64bytesormore_loopin2aligned) ++ jmp L(loop_tail) + ++L(L2_L3_cache_aligned): ++ subq $64, %rdx + .p2align 4 + L(L2_L3_aligned_128bytes_loop): + prefetchnta 0x1c0(%rdi) + prefetchnta 0x1c0(%rsi) +- movdqa (%rdi), %xmm2 +- pxor (%rsi), %xmm2 +- movdqa %xmm2, %xmm1 +- +- movdqa 16(%rdi), %xmm3 +- pxor 16(%rsi), %xmm3 +- por %xmm3, %xmm1 ++ movdqa (%rdi), %xmm0 ++ movdqa 16(%rdi), %xmm1 ++ movdqa 32(%rdi), %xmm2 ++ movdqa 48(%rdi), %xmm3 + +- movdqa 32(%rdi), %xmm4 +- pxor 32(%rsi), %xmm4 +- por %xmm4, %xmm1 ++ CMPEQ (%rsi), %xmm0 ++ CMPEQ 16(%rsi), %xmm1 ++ CMPEQ 32(%rsi), %xmm2 ++ CMPEQ 48(%rsi), %xmm3 + +- movdqa 48(%rdi), %xmm5 +- pxor 48(%rsi), %xmm5 +- por %xmm5, %xmm1 ++ pand %xmm0, %xmm1 ++ pand %xmm2, %xmm3 ++ pand %xmm1, %xmm3 + +- ptest %xmm1, %xmm0 +- jnc L(64bytesormore_loop_end) +- add $64, %rsi +- add $64, %rdi +- sub $64, %rdx +- jae L(L2_L3_aligned_128bytes_loop) +- +- add $64, %rdx +- add %rdx, %rsi +- add %rdx, %rdi +- BRANCH_TO_JMPTBL_ENTRY(L(table_64bytes), %rdx, 4) ++ pmovmskb %xmm3, %eax ++ incw %ax ++ jnz L(64bytesormore_loop_end) + ++ addq $64, %rsi ++ addq $64, %rdi ++ subq $64, %rdx ++ ja L(L2_L3_aligned_128bytes_loop) ++ jmp L(loop_tail) + + .p2align 4 + L(64bytesormore_loop_end): +- add $16, %rdi +- add $16, %rsi +- ptest %xmm2, %xmm0 +- jnc L(16bytes) +- +- add $16, %rdi +- add $16, %rsi +- ptest %xmm3, %xmm0 +- jnc L(16bytes) +- +- add $16, %rdi +- add $16, %rsi +- ptest %xmm4, %xmm0 +- jnc L(16bytes) +- +- add $16, %rdi +- add $16, %rsi +- jmp L(16bytes) +- +-L(256bytesin256): +- add $256, %rdi +- add $256, %rsi +- jmp L(16bytes) +-L(240bytesin256): +- add $240, %rdi +- add $240, %rsi +- jmp L(16bytes) +-L(224bytesin256): +- add $224, %rdi +- add $224, %rsi +- jmp L(16bytes) +-L(208bytesin256): +- add $208, %rdi +- add $208, %rsi +- jmp L(16bytes) +-L(192bytesin256): +- add $192, %rdi +- add $192, %rsi +- jmp L(16bytes) +-L(176bytesin256): +- add $176, %rdi +- add $176, %rsi +- jmp L(16bytes) +-L(160bytesin256): +- add $160, %rdi +- add $160, %rsi +- jmp L(16bytes) +-L(144bytesin256): +- add $144, %rdi +- add $144, %rsi +- jmp L(16bytes) +-L(128bytesin256): +- add $128, %rdi +- add $128, %rsi +- jmp L(16bytes) +-L(112bytesin256): +- add $112, %rdi +- add $112, %rsi +- jmp L(16bytes) +-L(96bytesin256): +- add $96, %rdi +- add $96, %rsi +- jmp L(16bytes) +-L(80bytesin256): +- add $80, %rdi +- add $80, %rsi +- jmp L(16bytes) +-L(64bytesin256): +- add $64, %rdi +- add $64, %rsi +- jmp L(16bytes) +-L(48bytesin256): +- add $16, %rdi +- add $16, %rsi +-L(32bytesin256): +- add $16, %rdi +- add $16, %rsi +-L(16bytesin256): +- add $16, %rdi +- add $16, %rsi +-L(16bytes): +- mov -16(%rdi), %rax +- mov -16(%rsi), %rcx +- cmp %rax, %rcx +- jne L(diffin8bytes) +-L(8bytes): +- mov -8(%rdi), %rax +- mov -8(%rsi), %rcx +- cmp %rax, %rcx +- jne L(diffin8bytes) +- xor %eax, %eax +- ret +- +- .p2align 4 +-L(12bytes): +- mov -12(%rdi), %rax +- mov -12(%rsi), %rcx +- cmp %rax, %rcx +- jne L(diffin8bytes) +-L(4bytes): +- mov -4(%rsi), %ecx +-# ifndef USE_AS_WMEMCMP +- mov -4(%rdi), %eax +- cmp %eax, %ecx +-# else +- cmp -4(%rdi), %ecx +-# endif +- jne L(diffin4bytes) +-L(0bytes): +- xor %eax, %eax +- ret +- +-# ifndef USE_AS_WMEMCMP +-/* unreal case for wmemcmp */ +- .p2align 4 +-L(65bytes): +- movdqu -65(%rdi), %xmm1 +- movdqu -65(%rsi), %xmm2 +- mov $-65, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(49bytes): +- movdqu -49(%rdi), %xmm1 +- movdqu -49(%rsi), %xmm2 +- mov $-49, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(33bytes): +- movdqu -33(%rdi), %xmm1 +- movdqu -33(%rsi), %xmm2 +- mov $-33, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(17bytes): +- mov -17(%rdi), %rax +- mov -17(%rsi), %rcx +- cmp %rax, %rcx +- jne L(diffin8bytes) +-L(9bytes): +- mov -9(%rdi), %rax +- mov -9(%rsi), %rcx +- cmp %rax, %rcx +- jne L(diffin8bytes) +- movzbl -1(%rdi), %eax +- movzbl -1(%rsi), %edx +- sub %edx, %eax +- ret +- +- .p2align 4 +-L(13bytes): +- mov -13(%rdi), %rax +- mov -13(%rsi), %rcx +- cmp %rax, %rcx +- jne L(diffin8bytes) +- mov -8(%rdi), %rax +- mov -8(%rsi), %rcx +- cmp %rax, %rcx +- jne L(diffin8bytes) +- xor %eax, %eax +- ret +- +- .p2align 4 +-L(5bytes): +- mov -5(%rdi), %eax +- mov -5(%rsi), %ecx +- cmp %eax, %ecx +- jne L(diffin4bytes) +- movzbl -1(%rdi), %eax +- movzbl -1(%rsi), %edx +- sub %edx, %eax +- ret +- +- .p2align 4 +-L(66bytes): +- movdqu -66(%rdi), %xmm1 +- movdqu -66(%rsi), %xmm2 +- mov $-66, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(50bytes): +- movdqu -50(%rdi), %xmm1 +- movdqu -50(%rsi), %xmm2 +- mov $-50, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(34bytes): +- movdqu -34(%rdi), %xmm1 +- movdqu -34(%rsi), %xmm2 +- mov $-34, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(18bytes): +- mov -18(%rdi), %rax +- mov -18(%rsi), %rcx +- cmp %rax, %rcx +- jne L(diffin8bytes) +-L(10bytes): +- mov -10(%rdi), %rax +- mov -10(%rsi), %rcx +- cmp %rax, %rcx +- jne L(diffin8bytes) +- movzwl -2(%rdi), %eax +- movzwl -2(%rsi), %ecx +- cmp %cl, %al +- jne L(end) +- and $0xffff, %eax +- and $0xffff, %ecx +- sub %ecx, %eax +- ret +- +- .p2align 4 +-L(14bytes): +- mov -14(%rdi), %rax +- mov -14(%rsi), %rcx +- cmp %rax, %rcx +- jne L(diffin8bytes) +- mov -8(%rdi), %rax +- mov -8(%rsi), %rcx +- cmp %rax, %rcx +- jne L(diffin8bytes) +- xor %eax, %eax +- ret +- +- .p2align 4 +-L(6bytes): +- mov -6(%rdi), %eax +- mov -6(%rsi), %ecx +- cmp %eax, %ecx +- jne L(diffin4bytes) +-L(2bytes): +- movzwl -2(%rsi), %ecx +- movzwl -2(%rdi), %eax +- cmp %cl, %al +- jne L(end) +- and $0xffff, %eax +- and $0xffff, %ecx +- sub %ecx, %eax +- ret +- +- .p2align 4 +-L(67bytes): +- movdqu -67(%rdi), %xmm2 +- movdqu -67(%rsi), %xmm1 +- mov $-67, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(51bytes): +- movdqu -51(%rdi), %xmm2 +- movdqu -51(%rsi), %xmm1 +- mov $-51, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(35bytes): +- movdqu -35(%rsi), %xmm1 +- movdqu -35(%rdi), %xmm2 +- mov $-35, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(19bytes): +- mov -19(%rdi), %rax +- mov -19(%rsi), %rcx +- cmp %rax, %rcx +- jne L(diffin8bytes) +-L(11bytes): +- mov -11(%rdi), %rax +- mov -11(%rsi), %rcx +- cmp %rax, %rcx +- jne L(diffin8bytes) +- mov -4(%rdi), %eax +- mov -4(%rsi), %ecx +- cmp %eax, %ecx +- jne L(diffin4bytes) +- xor %eax, %eax +- ret +- +- .p2align 4 +-L(15bytes): +- mov -15(%rdi), %rax +- mov -15(%rsi), %rcx +- cmp %rax, %rcx +- jne L(diffin8bytes) +- mov -8(%rdi), %rax +- mov -8(%rsi), %rcx +- cmp %rax, %rcx +- jne L(diffin8bytes) +- xor %eax, %eax +- ret +- +- .p2align 4 +-L(7bytes): +- mov -7(%rdi), %eax +- mov -7(%rsi), %ecx +- cmp %eax, %ecx +- jne L(diffin4bytes) +- mov -4(%rdi), %eax +- mov -4(%rsi), %ecx +- cmp %eax, %ecx +- jne L(diffin4bytes) +- xor %eax, %eax +- ret +- +- .p2align 4 +-L(3bytes): +- movzwl -3(%rdi), %eax +- movzwl -3(%rsi), %ecx +- cmp %eax, %ecx +- jne L(diffin2bytes) +-L(1bytes): +- movzbl -1(%rdi), %eax +- movzbl -1(%rsi), %ecx +- sub %ecx, %eax +- ret +-# endif +- +- .p2align 4 +-L(68bytes): +- movdqu -68(%rdi), %xmm2 +- movdqu -68(%rsi), %xmm1 +- mov $-68, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(52bytes): +- movdqu -52(%rdi), %xmm2 +- movdqu -52(%rsi), %xmm1 +- mov $-52, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(36bytes): +- movdqu -36(%rdi), %xmm2 +- movdqu -36(%rsi), %xmm1 +- mov $-36, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(20bytes): +- movdqu -20(%rdi), %xmm2 +- movdqu -20(%rsi), %xmm1 +- mov $-20, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +- mov -4(%rsi), %ecx +- +-# ifndef USE_AS_WMEMCMP +- mov -4(%rdi), %eax +- cmp %eax, %ecx +-# else +- cmp -4(%rdi), %ecx +-# endif +- jne L(diffin4bytes) +- xor %eax, %eax +- ret +- +-# ifndef USE_AS_WMEMCMP +-/* unreal cases for wmemcmp */ +- .p2align 4 +-L(69bytes): +- movdqu -69(%rsi), %xmm1 +- movdqu -69(%rdi), %xmm2 +- mov $-69, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(53bytes): +- movdqu -53(%rsi), %xmm1 +- movdqu -53(%rdi), %xmm2 +- mov $-53, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(37bytes): +- movdqu -37(%rsi), %xmm1 +- movdqu -37(%rdi), %xmm2 +- mov $-37, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(21bytes): +- movdqu -21(%rsi), %xmm1 +- movdqu -21(%rdi), %xmm2 +- mov $-21, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +- mov -8(%rdi), %rax +- mov -8(%rsi), %rcx +- cmp %rax, %rcx +- jne L(diffin8bytes) +- xor %eax, %eax +- ret +- +- .p2align 4 +-L(70bytes): +- movdqu -70(%rsi), %xmm1 +- movdqu -70(%rdi), %xmm2 +- mov $-70, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(54bytes): +- movdqu -54(%rsi), %xmm1 +- movdqu -54(%rdi), %xmm2 +- mov $-54, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(38bytes): +- movdqu -38(%rsi), %xmm1 +- movdqu -38(%rdi), %xmm2 +- mov $-38, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(22bytes): +- movdqu -22(%rsi), %xmm1 +- movdqu -22(%rdi), %xmm2 +- mov $-22, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +- mov -8(%rdi), %rax +- mov -8(%rsi), %rcx +- cmp %rax, %rcx +- jne L(diffin8bytes) +- xor %eax, %eax +- ret +- +- .p2align 4 +-L(71bytes): +- movdqu -71(%rsi), %xmm1 +- movdqu -71(%rdi), %xmm2 +- mov $-71, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(55bytes): +- movdqu -55(%rdi), %xmm2 +- movdqu -55(%rsi), %xmm1 +- mov $-55, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(39bytes): +- movdqu -39(%rdi), %xmm2 +- movdqu -39(%rsi), %xmm1 +- mov $-39, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(23bytes): +- movdqu -23(%rdi), %xmm2 +- movdqu -23(%rsi), %xmm1 +- mov $-23, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +- mov -8(%rdi), %rax +- mov -8(%rsi), %rcx +- cmp %rax, %rcx +- jne L(diffin8bytes) +- xor %eax, %eax +- ret +-# endif +- +- .p2align 4 +-L(72bytes): +- movdqu -72(%rsi), %xmm1 +- movdqu -72(%rdi), %xmm2 +- mov $-72, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(56bytes): +- movdqu -56(%rdi), %xmm2 +- movdqu -56(%rsi), %xmm1 +- mov $-56, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(40bytes): +- movdqu -40(%rdi), %xmm2 +- movdqu -40(%rsi), %xmm1 +- mov $-40, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(24bytes): +- movdqu -24(%rdi), %xmm2 +- movdqu -24(%rsi), %xmm1 +- mov $-24, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +- +- mov -8(%rsi), %rcx +- mov -8(%rdi), %rax +- cmp %rax, %rcx +- jne L(diffin8bytes) +- xor %eax, %eax +- ret +- +-# ifndef USE_AS_WMEMCMP +-/* unreal cases for wmemcmp */ +- .p2align 4 +-L(73bytes): +- movdqu -73(%rsi), %xmm1 +- movdqu -73(%rdi), %xmm2 +- mov $-73, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(57bytes): +- movdqu -57(%rdi), %xmm2 +- movdqu -57(%rsi), %xmm1 +- mov $-57, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(41bytes): +- movdqu -41(%rdi), %xmm2 +- movdqu -41(%rsi), %xmm1 +- mov $-41, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(25bytes): +- movdqu -25(%rdi), %xmm2 +- movdqu -25(%rsi), %xmm1 +- mov $-25, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +- mov -9(%rdi), %rax +- mov -9(%rsi), %rcx +- cmp %rax, %rcx +- jne L(diffin8bytes) +- movzbl -1(%rdi), %eax +- movzbl -1(%rsi), %ecx +- sub %ecx, %eax +- ret +- +- .p2align 4 +-L(74bytes): +- movdqu -74(%rsi), %xmm1 +- movdqu -74(%rdi), %xmm2 +- mov $-74, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(58bytes): +- movdqu -58(%rdi), %xmm2 +- movdqu -58(%rsi), %xmm1 +- mov $-58, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(42bytes): +- movdqu -42(%rdi), %xmm2 +- movdqu -42(%rsi), %xmm1 +- mov $-42, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(26bytes): +- movdqu -26(%rdi), %xmm2 +- movdqu -26(%rsi), %xmm1 +- mov $-26, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +- mov -10(%rdi), %rax +- mov -10(%rsi), %rcx +- cmp %rax, %rcx +- jne L(diffin8bytes) +- movzwl -2(%rdi), %eax +- movzwl -2(%rsi), %ecx +- jmp L(diffin2bytes) +- +- .p2align 4 +-L(75bytes): +- movdqu -75(%rsi), %xmm1 +- movdqu -75(%rdi), %xmm2 +- mov $-75, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(59bytes): +- movdqu -59(%rdi), %xmm2 +- movdqu -59(%rsi), %xmm1 +- mov $-59, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(43bytes): +- movdqu -43(%rdi), %xmm2 +- movdqu -43(%rsi), %xmm1 +- mov $-43, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(27bytes): +- movdqu -27(%rdi), %xmm2 +- movdqu -27(%rsi), %xmm1 +- mov $-27, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +- mov -11(%rdi), %rax +- mov -11(%rsi), %rcx +- cmp %rax, %rcx +- jne L(diffin8bytes) +- mov -4(%rdi), %eax +- mov -4(%rsi), %ecx +- cmp %eax, %ecx +- jne L(diffin4bytes) +- xor %eax, %eax +- ret +-# endif +- .p2align 4 +-L(76bytes): +- movdqu -76(%rsi), %xmm1 +- movdqu -76(%rdi), %xmm2 +- mov $-76, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(60bytes): +- movdqu -60(%rdi), %xmm2 +- movdqu -60(%rsi), %xmm1 +- mov $-60, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(44bytes): +- movdqu -44(%rdi), %xmm2 +- movdqu -44(%rsi), %xmm1 +- mov $-44, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(28bytes): +- movdqu -28(%rdi), %xmm2 +- movdqu -28(%rsi), %xmm1 +- mov $-28, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +- mov -12(%rdi), %rax +- mov -12(%rsi), %rcx +- cmp %rax, %rcx +- jne L(diffin8bytes) +- mov -4(%rsi), %ecx +-# ifndef USE_AS_WMEMCMP +- mov -4(%rdi), %eax +- cmp %eax, %ecx +-# else +- cmp -4(%rdi), %ecx +-# endif +- jne L(diffin4bytes) +- xor %eax, %eax +- ret +- +-# ifndef USE_AS_WMEMCMP +-/* unreal cases for wmemcmp */ +- .p2align 4 +-L(77bytes): +- movdqu -77(%rsi), %xmm1 +- movdqu -77(%rdi), %xmm2 +- mov $-77, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(61bytes): +- movdqu -61(%rdi), %xmm2 +- movdqu -61(%rsi), %xmm1 +- mov $-61, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(45bytes): +- movdqu -45(%rdi), %xmm2 +- movdqu -45(%rsi), %xmm1 +- mov $-45, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(29bytes): +- movdqu -29(%rdi), %xmm2 +- movdqu -29(%rsi), %xmm1 +- mov $-29, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +- +- mov -13(%rdi), %rax +- mov -13(%rsi), %rcx +- cmp %rax, %rcx +- jne L(diffin8bytes) +- +- mov -8(%rdi), %rax +- mov -8(%rsi), %rcx +- cmp %rax, %rcx +- jne L(diffin8bytes) +- xor %eax, %eax +- ret +- +- .p2align 4 +-L(78bytes): +- movdqu -78(%rsi), %xmm1 +- movdqu -78(%rdi), %xmm2 +- mov $-78, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(62bytes): +- movdqu -62(%rdi), %xmm2 +- movdqu -62(%rsi), %xmm1 +- mov $-62, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(46bytes): +- movdqu -46(%rdi), %xmm2 +- movdqu -46(%rsi), %xmm1 +- mov $-46, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(30bytes): +- movdqu -30(%rdi), %xmm2 +- movdqu -30(%rsi), %xmm1 +- mov $-30, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +- mov -14(%rdi), %rax +- mov -14(%rsi), %rcx +- cmp %rax, %rcx +- jne L(diffin8bytes) +- mov -8(%rdi), %rax +- mov -8(%rsi), %rcx +- cmp %rax, %rcx +- jne L(diffin8bytes) +- xor %eax, %eax +- ret +- +- .p2align 4 +-L(79bytes): +- movdqu -79(%rsi), %xmm1 +- movdqu -79(%rdi), %xmm2 +- mov $-79, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(63bytes): +- movdqu -63(%rdi), %xmm2 +- movdqu -63(%rsi), %xmm1 +- mov $-63, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(47bytes): +- movdqu -47(%rdi), %xmm2 +- movdqu -47(%rsi), %xmm1 +- mov $-47, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(31bytes): +- movdqu -31(%rdi), %xmm2 +- movdqu -31(%rsi), %xmm1 +- mov $-31, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +- mov -15(%rdi), %rax +- mov -15(%rsi), %rcx +- cmp %rax, %rcx +- jne L(diffin8bytes) +- mov -8(%rdi), %rax +- mov -8(%rsi), %rcx +- cmp %rax, %rcx +- jne L(diffin8bytes) +- xor %eax, %eax +- ret +-# endif +- .p2align 4 +-L(64bytes): +- movdqu -64(%rdi), %xmm2 +- movdqu -64(%rsi), %xmm1 +- mov $-64, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(48bytes): +- movdqu -48(%rdi), %xmm2 +- movdqu -48(%rsi), %xmm1 +- mov $-48, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +-L(32bytes): +- movdqu -32(%rdi), %xmm2 +- movdqu -32(%rsi), %xmm1 +- mov $-32, %dl +- pxor %xmm1, %xmm2 +- ptest %xmm2, %xmm0 +- jnc L(less16bytes) +- +- mov -16(%rdi), %rax +- mov -16(%rsi), %rcx +- cmp %rax, %rcx +- jne L(diffin8bytes) +- +- mov -8(%rdi), %rax +- mov -8(%rsi), %rcx +- cmp %rax, %rcx +- jne L(diffin8bytes) +- xor %eax, %eax +- ret +- +-/* +- * Aligned 8 bytes to avoid 2 branch "taken" in one 16 alinged code block. +- */ +- .p2align 3 +-L(less16bytes): +- movsbq %dl, %rdx +- mov (%rsi, %rdx), %rcx +- mov (%rdi, %rdx), %rax +- cmp %rax, %rcx +- jne L(diffin8bytes) +- mov 8(%rsi, %rdx), %rcx +- mov 8(%rdi, %rdx), %rax +-L(diffin8bytes): +- cmp %eax, %ecx +- jne L(diffin4bytes) +- shr $32, %rcx +- shr $32, %rax +- ++ pmovmskb %xmm0, %ecx ++ incw %cx ++ jnz L(loop_end_ret) ++ ++ pmovmskb %xmm1, %ecx ++ notw %cx ++ sall $16, %ecx ++ jnz L(loop_end_ret) ++ ++ pmovmskb %xmm2, %ecx ++ notw %cx ++ shlq $32, %rcx ++ jnz L(loop_end_ret) ++ ++ addq $48, %rdi ++ addq $48, %rsi ++ movq %rax, %rcx ++ ++ .p2align 4,, 6 ++L(loop_end_ret): ++ bsfq %rcx, %rcx + # ifdef USE_AS_WMEMCMP +-/* for wmemcmp */ +- cmp %eax, %ecx +- jne L(diffin4bytes) +- xor %eax, %eax +- ret +-# endif +- +-L(diffin4bytes): +-# ifndef USE_AS_WMEMCMP +- cmp %cx, %ax +- jne L(diffin2bytes) +- shr $16, %ecx +- shr $16, %eax +-L(diffin2bytes): +- cmp %cl, %al +- jne L(end) +- and $0xffff, %eax +- and $0xffff, %ecx +- sub %ecx, %eax +- ret +- +- .p2align 4 +-L(end): +- and $0xff, %eax +- and $0xff, %ecx +- sub %ecx, %eax +- ret ++ movl (%rdi, %rcx), %eax ++ xorl %edx, %edx ++ cmpl (%rsi, %rcx), %eax ++ setg %dl ++ leal -1(%rdx, %rdx), %eax + # else +- +-/* for wmemcmp */ +- mov $1, %eax +- jl L(nequal_bigger) +- neg %eax +- ret +- +- .p2align 4 +-L(nequal_bigger): +- ret +- +-L(unreal_case): +- xor %eax, %eax +- ret ++ movzbl (%rdi, %rcx), %eax ++ movzbl (%rsi, %rcx), %ecx ++ subl %ecx, %eax + # endif +- ++ ret + END (MEMCMP) +- +- .section .rodata.sse4.1,"a",@progbits +- .p2align 3 +-# ifndef USE_AS_WMEMCMP +-L(table_64bytes): +- .int JMPTBL (L(0bytes), L(table_64bytes)) +- .int JMPTBL (L(1bytes), L(table_64bytes)) +- .int JMPTBL (L(2bytes), L(table_64bytes)) +- .int JMPTBL (L(3bytes), L(table_64bytes)) +- .int JMPTBL (L(4bytes), L(table_64bytes)) +- .int JMPTBL (L(5bytes), L(table_64bytes)) +- .int JMPTBL (L(6bytes), L(table_64bytes)) +- .int JMPTBL (L(7bytes), L(table_64bytes)) +- .int JMPTBL (L(8bytes), L(table_64bytes)) +- .int JMPTBL (L(9bytes), L(table_64bytes)) +- .int JMPTBL (L(10bytes), L(table_64bytes)) +- .int JMPTBL (L(11bytes), L(table_64bytes)) +- .int JMPTBL (L(12bytes), L(table_64bytes)) +- .int JMPTBL (L(13bytes), L(table_64bytes)) +- .int JMPTBL (L(14bytes), L(table_64bytes)) +- .int JMPTBL (L(15bytes), L(table_64bytes)) +- .int JMPTBL (L(16bytes), L(table_64bytes)) +- .int JMPTBL (L(17bytes), L(table_64bytes)) +- .int JMPTBL (L(18bytes), L(table_64bytes)) +- .int JMPTBL (L(19bytes), L(table_64bytes)) +- .int JMPTBL (L(20bytes), L(table_64bytes)) +- .int JMPTBL (L(21bytes), L(table_64bytes)) +- .int JMPTBL (L(22bytes), L(table_64bytes)) +- .int JMPTBL (L(23bytes), L(table_64bytes)) +- .int JMPTBL (L(24bytes), L(table_64bytes)) +- .int JMPTBL (L(25bytes), L(table_64bytes)) +- .int JMPTBL (L(26bytes), L(table_64bytes)) +- .int JMPTBL (L(27bytes), L(table_64bytes)) +- .int JMPTBL (L(28bytes), L(table_64bytes)) +- .int JMPTBL (L(29bytes), L(table_64bytes)) +- .int JMPTBL (L(30bytes), L(table_64bytes)) +- .int JMPTBL (L(31bytes), L(table_64bytes)) +- .int JMPTBL (L(32bytes), L(table_64bytes)) +- .int JMPTBL (L(33bytes), L(table_64bytes)) +- .int JMPTBL (L(34bytes), L(table_64bytes)) +- .int JMPTBL (L(35bytes), L(table_64bytes)) +- .int JMPTBL (L(36bytes), L(table_64bytes)) +- .int JMPTBL (L(37bytes), L(table_64bytes)) +- .int JMPTBL (L(38bytes), L(table_64bytes)) +- .int JMPTBL (L(39bytes), L(table_64bytes)) +- .int JMPTBL (L(40bytes), L(table_64bytes)) +- .int JMPTBL (L(41bytes), L(table_64bytes)) +- .int JMPTBL (L(42bytes), L(table_64bytes)) +- .int JMPTBL (L(43bytes), L(table_64bytes)) +- .int JMPTBL (L(44bytes), L(table_64bytes)) +- .int JMPTBL (L(45bytes), L(table_64bytes)) +- .int JMPTBL (L(46bytes), L(table_64bytes)) +- .int JMPTBL (L(47bytes), L(table_64bytes)) +- .int JMPTBL (L(48bytes), L(table_64bytes)) +- .int JMPTBL (L(49bytes), L(table_64bytes)) +- .int JMPTBL (L(50bytes), L(table_64bytes)) +- .int JMPTBL (L(51bytes), L(table_64bytes)) +- .int JMPTBL (L(52bytes), L(table_64bytes)) +- .int JMPTBL (L(53bytes), L(table_64bytes)) +- .int JMPTBL (L(54bytes), L(table_64bytes)) +- .int JMPTBL (L(55bytes), L(table_64bytes)) +- .int JMPTBL (L(56bytes), L(table_64bytes)) +- .int JMPTBL (L(57bytes), L(table_64bytes)) +- .int JMPTBL (L(58bytes), L(table_64bytes)) +- .int JMPTBL (L(59bytes), L(table_64bytes)) +- .int JMPTBL (L(60bytes), L(table_64bytes)) +- .int JMPTBL (L(61bytes), L(table_64bytes)) +- .int JMPTBL (L(62bytes), L(table_64bytes)) +- .int JMPTBL (L(63bytes), L(table_64bytes)) +- .int JMPTBL (L(64bytes), L(table_64bytes)) +- .int JMPTBL (L(65bytes), L(table_64bytes)) +- .int JMPTBL (L(66bytes), L(table_64bytes)) +- .int JMPTBL (L(67bytes), L(table_64bytes)) +- .int JMPTBL (L(68bytes), L(table_64bytes)) +- .int JMPTBL (L(69bytes), L(table_64bytes)) +- .int JMPTBL (L(70bytes), L(table_64bytes)) +- .int JMPTBL (L(71bytes), L(table_64bytes)) +- .int JMPTBL (L(72bytes), L(table_64bytes)) +- .int JMPTBL (L(73bytes), L(table_64bytes)) +- .int JMPTBL (L(74bytes), L(table_64bytes)) +- .int JMPTBL (L(75bytes), L(table_64bytes)) +- .int JMPTBL (L(76bytes), L(table_64bytes)) +- .int JMPTBL (L(77bytes), L(table_64bytes)) +- .int JMPTBL (L(78bytes), L(table_64bytes)) +- .int JMPTBL (L(79bytes), L(table_64bytes)) +-# else +-L(table_64bytes): +- .int JMPTBL (L(0bytes), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(4bytes), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(8bytes), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(12bytes), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(16bytes), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(20bytes), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(24bytes), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(28bytes), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(32bytes), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(36bytes), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(40bytes), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(44bytes), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(48bytes), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(52bytes), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(56bytes), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(60bytes), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(64bytes), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(68bytes), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(72bytes), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(76bytes), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +- .int JMPTBL (L(unreal_case), L(table_64bytes)) +-# endif + #endif +-- +GitLab + diff --git a/SOURCES/ia-string-funct-test-rtm.patch b/SOURCES/ia-string-funct-test-rtm.patch new file mode 100644 index 0000000..a0675d9 --- /dev/null +++ b/SOURCES/ia-string-funct-test-rtm.patch @@ -0,0 +1,736 @@ +From 294c22ed9aa3f576f58b9af69c9a34225fcf02aa Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Tue, 23 Feb 2021 06:33:10 -0800 +Subject: [PATCH] x86: Add string/memory function tests in RTM region + +At function exit, AVX optimized string/memory functions have VZEROUPPER +which triggers RTM abort. When such functions are called inside a +transactionally executing RTM region, RTM abort causes severe performance +degradation. Add tests to verify that string/memory functions won't +cause RTM abort in RTM region. + +(cherry picked from commit 4bd660be40967cd69072f69ebc2ad32bfcc1f206) +--- + sysdeps/x86/Makefile | 23 +++++++++++ + sysdeps/x86/tst-memchr-rtm.c | 54 ++++++++++++++++++++++++++ + sysdeps/x86/tst-memcmp-rtm.c | 52 +++++++++++++++++++++++++ + sysdeps/x86/tst-memmove-rtm.c | 53 ++++++++++++++++++++++++++ + sysdeps/x86/tst-memrchr-rtm.c | 54 ++++++++++++++++++++++++++ + sysdeps/x86/tst-memset-rtm.c | 45 ++++++++++++++++++++++ + sysdeps/x86/tst-strchr-rtm.c | 54 ++++++++++++++++++++++++++ + sysdeps/x86/tst-strcpy-rtm.c | 53 ++++++++++++++++++++++++++ + sysdeps/x86/tst-string-rtm.h | 72 +++++++++++++++++++++++++++++++++++ + sysdeps/x86/tst-strlen-rtm.c | 53 ++++++++++++++++++++++++++ + sysdeps/x86/tst-strncmp-rtm.c | 52 +++++++++++++++++++++++++ + sysdeps/x86/tst-strrchr-rtm.c | 53 ++++++++++++++++++++++++++ + 12 files changed, 618 insertions(+) + create mode 100644 sysdeps/x86/tst-memchr-rtm.c + create mode 100644 sysdeps/x86/tst-memcmp-rtm.c + create mode 100644 sysdeps/x86/tst-memmove-rtm.c + create mode 100644 sysdeps/x86/tst-memrchr-rtm.c + create mode 100644 sysdeps/x86/tst-memset-rtm.c + create mode 100644 sysdeps/x86/tst-strchr-rtm.c + create mode 100644 sysdeps/x86/tst-strcpy-rtm.c + create mode 100644 sysdeps/x86/tst-string-rtm.h + create mode 100644 sysdeps/x86/tst-strlen-rtm.c + create mode 100644 sysdeps/x86/tst-strncmp-rtm.c + create mode 100644 sysdeps/x86/tst-strrchr-rtm.c + +diff --git a/sysdeps/x86/Makefile b/sysdeps/x86/Makefile +index 59e928e9..5be71ada 100644 +--- a/sysdeps/x86/Makefile ++++ b/sysdeps/x86/Makefile +@@ -17,6 +17,29 @@ endif + + ifeq ($(subdir),string) + sysdep_routines += cacheinfo ++ ++tests += \ ++ tst-memchr-rtm \ ++ tst-memcmp-rtm \ ++ tst-memmove-rtm \ ++ tst-memrchr-rtm \ ++ tst-memset-rtm \ ++ tst-strchr-rtm \ ++ tst-strcpy-rtm \ ++ tst-strlen-rtm \ ++ tst-strncmp-rtm \ ++ tst-strrchr-rtm ++ ++CFLAGS-tst-memchr-rtm.c += -mrtm ++CFLAGS-tst-memcmp-rtm.c += -mrtm ++CFLAGS-tst-memmove-rtm.c += -mrtm ++CFLAGS-tst-memrchr-rtm.c += -mrtm ++CFLAGS-tst-memset-rtm.c += -mrtm ++CFLAGS-tst-strchr-rtm.c += -mrtm ++CFLAGS-tst-strcpy-rtm.c += -mrtm ++CFLAGS-tst-strlen-rtm.c += -mrtm ++CFLAGS-tst-strncmp-rtm.c += -mrtm ++CFLAGS-tst-strrchr-rtm.c += -mrtm + endif + + ifneq ($(enable-cet),no) +diff --git a/sysdeps/x86/tst-memchr-rtm.c b/sysdeps/x86/tst-memchr-rtm.c +new file mode 100644 +index 00000000..e4749401 +--- /dev/null ++++ b/sysdeps/x86/tst-memchr-rtm.c +@@ -0,0 +1,54 @@ ++/* Test case for memchr inside a transactionally executing RTM region. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#include ++ ++#define LOOP 3000 ++#define STRING_SIZE 1024 ++char string1[STRING_SIZE]; ++ ++__attribute__ ((noinline, noclone)) ++static int ++prepare (void) ++{ ++ memset (string1, 'a', STRING_SIZE); ++ string1[100] = 'c'; ++ string1[STRING_SIZE - 100] = 'c'; ++ char *p = memchr (string1, 'c', STRING_SIZE); ++ if (p == &string1[100]) ++ return EXIT_SUCCESS; ++ else ++ return EXIT_FAILURE; ++} ++ ++__attribute__ ((noinline, noclone)) ++static int ++function (void) ++{ ++ char *p = memchr (string1, 'c', STRING_SIZE); ++ if (p == &string1[100]) ++ return 0; ++ else ++ return 1; ++} ++ ++static int ++do_test (void) ++{ ++ return do_test_1 ("memchr", LOOP, prepare, function); ++} +diff --git a/sysdeps/x86/tst-memcmp-rtm.c b/sysdeps/x86/tst-memcmp-rtm.c +new file mode 100644 +index 00000000..e4c8a623 +--- /dev/null ++++ b/sysdeps/x86/tst-memcmp-rtm.c +@@ -0,0 +1,52 @@ ++/* Test case for memcmp inside a transactionally executing RTM region. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#include ++ ++#define LOOP 3000 ++#define STRING_SIZE 1024 ++char string1[STRING_SIZE]; ++char string2[STRING_SIZE]; ++ ++__attribute__ ((noinline, noclone)) ++static int ++prepare (void) ++{ ++ memset (string1, 'a', STRING_SIZE); ++ memset (string2, 'a', STRING_SIZE); ++ if (memcmp (string1, string2, STRING_SIZE) == 0) ++ return EXIT_SUCCESS; ++ else ++ return EXIT_FAILURE; ++} ++ ++__attribute__ ((noinline, noclone)) ++static int ++function (void) ++{ ++ if (memcmp (string1, string2, STRING_SIZE) == 0) ++ return 0; ++ else ++ return 1; ++} ++ ++static int ++do_test (void) ++{ ++ return do_test_1 ("memcmp", LOOP, prepare, function); ++} +diff --git a/sysdeps/x86/tst-memmove-rtm.c b/sysdeps/x86/tst-memmove-rtm.c +new file mode 100644 +index 00000000..4bf97ef1 +--- /dev/null ++++ b/sysdeps/x86/tst-memmove-rtm.c +@@ -0,0 +1,53 @@ ++/* Test case for memmove inside a transactionally executing RTM region. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#include ++ ++#define LOOP 3000 ++#define STRING_SIZE 1024 ++char string1[STRING_SIZE]; ++char string2[STRING_SIZE]; ++ ++__attribute__ ((noinline, noclone)) ++static int ++prepare (void) ++{ ++ memset (string1, 'a', STRING_SIZE); ++ if (memmove (string2, string1, STRING_SIZE) == string2 ++ && memcmp (string2, string1, STRING_SIZE) == 0) ++ return EXIT_SUCCESS; ++ else ++ return EXIT_FAILURE; ++} ++ ++__attribute__ ((noinline, noclone)) ++static int ++function (void) ++{ ++ if (memmove (string2, string1, STRING_SIZE) == string2 ++ && memcmp (string2, string1, STRING_SIZE) == 0) ++ return 0; ++ else ++ return 1; ++} ++ ++static int ++do_test (void) ++{ ++ return do_test_1 ("memmove", LOOP, prepare, function); ++} +diff --git a/sysdeps/x86/tst-memrchr-rtm.c b/sysdeps/x86/tst-memrchr-rtm.c +new file mode 100644 +index 00000000..a57a5a8e +--- /dev/null ++++ b/sysdeps/x86/tst-memrchr-rtm.c +@@ -0,0 +1,54 @@ ++/* Test case for memrchr inside a transactionally executing RTM region. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#include ++ ++#define LOOP 3000 ++#define STRING_SIZE 1024 ++char string1[STRING_SIZE]; ++ ++__attribute__ ((noinline, noclone)) ++static int ++prepare (void) ++{ ++ memset (string1, 'a', STRING_SIZE); ++ string1[100] = 'c'; ++ string1[STRING_SIZE - 100] = 'c'; ++ char *p = memrchr (string1, 'c', STRING_SIZE); ++ if (p == &string1[STRING_SIZE - 100]) ++ return EXIT_SUCCESS; ++ else ++ return EXIT_FAILURE; ++} ++ ++__attribute__ ((noinline, noclone)) ++static int ++function (void) ++{ ++ char *p = memrchr (string1, 'c', STRING_SIZE); ++ if (p == &string1[STRING_SIZE - 100]) ++ return 0; ++ else ++ return 1; ++} ++ ++static int ++do_test (void) ++{ ++ return do_test_1 ("memrchr", LOOP, prepare, function); ++} +diff --git a/sysdeps/x86/tst-memset-rtm.c b/sysdeps/x86/tst-memset-rtm.c +new file mode 100644 +index 00000000..bf343a4d +--- /dev/null ++++ b/sysdeps/x86/tst-memset-rtm.c +@@ -0,0 +1,45 @@ ++/* Test case for memset inside a transactionally executing RTM region. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#include ++ ++#define LOOP 3000 ++#define STRING_SIZE 1024 ++char string1[STRING_SIZE]; ++ ++__attribute__ ((noinline, noclone)) ++static int ++prepare (void) ++{ ++ memset (string1, 'a', STRING_SIZE); ++ return EXIT_SUCCESS; ++} ++ ++__attribute__ ((noinline, noclone)) ++static int ++function (void) ++{ ++ memset (string1, 'a', STRING_SIZE); ++ return 0; ++} ++ ++static int ++do_test (void) ++{ ++ return do_test_1 ("memset", LOOP, prepare, function); ++} +diff --git a/sysdeps/x86/tst-strchr-rtm.c b/sysdeps/x86/tst-strchr-rtm.c +new file mode 100644 +index 00000000..a82e29c0 +--- /dev/null ++++ b/sysdeps/x86/tst-strchr-rtm.c +@@ -0,0 +1,54 @@ ++/* Test case for strchr inside a transactionally executing RTM region. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#include ++ ++#define LOOP 3000 ++#define STRING_SIZE 1024 ++char string1[STRING_SIZE]; ++ ++__attribute__ ((noinline, noclone)) ++static int ++prepare (void) ++{ ++ memset (string1, 'a', STRING_SIZE - 1); ++ string1[100] = 'c'; ++ string1[STRING_SIZE - 100] = 'c'; ++ char *p = strchr (string1, 'c'); ++ if (p == &string1[100]) ++ return EXIT_SUCCESS; ++ else ++ return EXIT_FAILURE; ++} ++ ++__attribute__ ((noinline, noclone)) ++static int ++function (void) ++{ ++ char *p = strchr (string1, 'c'); ++ if (p == &string1[100]) ++ return 0; ++ else ++ return 1; ++} ++ ++static int ++do_test (void) ++{ ++ return do_test_1 ("strchr", LOOP, prepare, function); ++} +diff --git a/sysdeps/x86/tst-strcpy-rtm.c b/sysdeps/x86/tst-strcpy-rtm.c +new file mode 100644 +index 00000000..2b2a583f +--- /dev/null ++++ b/sysdeps/x86/tst-strcpy-rtm.c +@@ -0,0 +1,53 @@ ++/* Test case for strcpy inside a transactionally executing RTM region. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#include ++ ++#define LOOP 3000 ++#define STRING_SIZE 1024 ++char string1[STRING_SIZE]; ++char string2[STRING_SIZE]; ++ ++__attribute__ ((noinline, noclone)) ++static int ++prepare (void) ++{ ++ memset (string1, 'a', STRING_SIZE - 1); ++ if (strcpy (string2, string1) == string2 ++ && strcmp (string2, string1) == 0) ++ return EXIT_SUCCESS; ++ else ++ return EXIT_FAILURE; ++} ++ ++__attribute__ ((noinline, noclone)) ++static int ++function (void) ++{ ++ if (strcpy (string2, string1) == string2 ++ && strcmp (string2, string1) == 0) ++ return 0; ++ else ++ return 1; ++} ++ ++static int ++do_test (void) ++{ ++ return do_test_1 ("strcpy", LOOP, prepare, function); ++} +diff --git a/sysdeps/x86/tst-string-rtm.h b/sysdeps/x86/tst-string-rtm.h +new file mode 100644 +index 00000000..d2470afa +--- /dev/null ++++ b/sysdeps/x86/tst-string-rtm.h +@@ -0,0 +1,72 @@ ++/* Test string function in a transactionally executing RTM region. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#include ++#include ++#include ++#include ++#include ++ ++static int ++do_test_1 (const char *name, unsigned int loop, int (*prepare) (void), ++ int (*function) (void)) ++{ ++ if (!CPU_FEATURE_USABLE (RTM)) ++ return EXIT_UNSUPPORTED; ++ ++ int status = prepare (); ++ if (status != EXIT_SUCCESS) ++ return status; ++ ++ unsigned int i; ++ unsigned int naborts = 0; ++ unsigned int failed = 0; ++ for (i = 0; i < loop; i++) ++ { ++ failed |= function (); ++ if (_xbegin() == _XBEGIN_STARTED) ++ { ++ failed |= function (); ++ _xend(); ++ } ++ else ++ { ++ failed |= function (); ++ ++naborts; ++ } ++ } ++ ++ if (failed) ++ FAIL_EXIT1 ("%s() failed", name); ++ ++ if (naborts) ++ { ++ /* NB: Low single digit (<= 5%) noise-level aborts are normal for ++ TSX. */ ++ double rate = 100 * ((double) naborts) / ((double) loop); ++ if (rate > 5) ++ FAIL_EXIT1 ("TSX abort rate: %.2f%% (%d out of %d)", ++ rate, naborts, loop); ++ } ++ ++ return EXIT_SUCCESS; ++} ++ ++static int do_test (void); ++ ++#include +diff --git a/sysdeps/x86/tst-strlen-rtm.c b/sysdeps/x86/tst-strlen-rtm.c +new file mode 100644 +index 00000000..0dcf14db +--- /dev/null ++++ b/sysdeps/x86/tst-strlen-rtm.c +@@ -0,0 +1,53 @@ ++/* Test case for strlen inside a transactionally executing RTM region. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#include ++ ++#define LOOP 3000 ++#define STRING_SIZE 1024 ++char string1[STRING_SIZE]; ++ ++__attribute__ ((noinline, noclone)) ++static int ++prepare (void) ++{ ++ memset (string1, 'a', STRING_SIZE - 1); ++ string1[STRING_SIZE - 100] = '\0'; ++ size_t len = strlen (string1); ++ if (len == STRING_SIZE - 100) ++ return EXIT_SUCCESS; ++ else ++ return EXIT_FAILURE; ++} ++ ++__attribute__ ((noinline, noclone)) ++static int ++function (void) ++{ ++ size_t len = strlen (string1); ++ if (len == STRING_SIZE - 100) ++ return 0; ++ else ++ return 1; ++} ++ ++static int ++do_test (void) ++{ ++ return do_test_1 ("strlen", LOOP, prepare, function); ++} +diff --git a/sysdeps/x86/tst-strncmp-rtm.c b/sysdeps/x86/tst-strncmp-rtm.c +new file mode 100644 +index 00000000..236ad951 +--- /dev/null ++++ b/sysdeps/x86/tst-strncmp-rtm.c +@@ -0,0 +1,52 @@ ++/* Test case for strncmp inside a transactionally executing RTM region. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#include ++ ++#define LOOP 3000 ++#define STRING_SIZE 1024 ++char string1[STRING_SIZE]; ++char string2[STRING_SIZE]; ++ ++__attribute__ ((noinline, noclone)) ++static int ++prepare (void) ++{ ++ memset (string1, 'a', STRING_SIZE - 1); ++ memset (string2, 'a', STRING_SIZE - 1); ++ if (strncmp (string1, string2, STRING_SIZE) == 0) ++ return EXIT_SUCCESS; ++ else ++ return EXIT_FAILURE; ++} ++ ++__attribute__ ((noinline, noclone)) ++static int ++function (void) ++{ ++ if (strncmp (string1, string2, STRING_SIZE) == 0) ++ return 0; ++ else ++ return 1; ++} ++ ++static int ++do_test (void) ++{ ++ return do_test_1 ("strncmp", LOOP, prepare, function); ++} +diff --git a/sysdeps/x86/tst-strrchr-rtm.c b/sysdeps/x86/tst-strrchr-rtm.c +new file mode 100644 +index 00000000..e32bfaf5 +--- /dev/null ++++ b/sysdeps/x86/tst-strrchr-rtm.c +@@ -0,0 +1,53 @@ ++/* Test case for strrchr inside a transactionally executing RTM region. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#include ++ ++#define LOOP 3000 ++#define STRING_SIZE 1024 ++char string1[STRING_SIZE]; ++ ++__attribute__ ((noinline, noclone)) ++static int ++prepare (void) ++{ ++ memset (string1, 'a', STRING_SIZE - 1); ++ string1[STRING_SIZE - 100] = 'c'; ++ char *p = strrchr (string1, 'c'); ++ if (p == &string1[STRING_SIZE - 100]) ++ return EXIT_SUCCESS; ++ else ++ return EXIT_FAILURE; ++} ++ ++__attribute__ ((noinline, noclone)) ++static int ++function (void) ++{ ++ char *p = strrchr (string1, 'c'); ++ if (p == &string1[STRING_SIZE - 100]) ++ return 0; ++ else ++ return 1; ++} ++ ++static int ++do_test (void) ++{ ++ return do_test_1 ("strrchr", LOOP, prepare, function); ++} +-- +GitLab + diff --git a/SOURCES/ia-strlen-opt-avx2.patch b/SOURCES/ia-strlen-opt-avx2.patch new file mode 100644 index 0000000..b4f2fa6 --- /dev/null +++ b/SOURCES/ia-strlen-opt-avx2.patch @@ -0,0 +1,767 @@ +From f97e72cd3b822c04bfdda110dcf252b19afd2bcd Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Wed, 2 Mar 2022 14:29:44 -0800 +Subject: [PATCH] x86: Optimize strlen-avx2.S + +No bug. This commit optimizes strlen-avx2.S. The optimizations are +mostly small things but they add up to roughly 10-30% performance +improvement for strlen. The results for strnlen are bit more +ambiguous. test-strlen, test-strnlen, test-wcslen, and test-wcsnlen +are all passing. + +Signed-off-by: Noah Goldstein +(cherry picked from commit aaa23c35071537e2dcf5807e956802ed215210aa) +--- + sysdeps/x86_64/multiarch/ifunc-impl-list.c | 16 +- + sysdeps/x86_64/multiarch/strlen-avx2.S | 532 +++++++++++++-------- + 2 files changed, 334 insertions(+), 214 deletions(-) + +diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c +index cbfc1a5d..f1a6460a 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c ++++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c +@@ -285,10 +285,12 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + /* Support sysdeps/x86_64/multiarch/strlen.c. */ + IFUNC_IMPL (i, name, strlen, + IFUNC_IMPL_ADD (array, i, strlen, +- CPU_FEATURE_USABLE (AVX2), ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (BMI2)), + __strlen_avx2) + IFUNC_IMPL_ADD (array, i, strlen, + (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (BMI2) + && CPU_FEATURE_USABLE (RTM)), + __strlen_avx2_rtm) + IFUNC_IMPL_ADD (array, i, strlen, +@@ -301,10 +303,12 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + /* Support sysdeps/x86_64/multiarch/strnlen.c. */ + IFUNC_IMPL (i, name, strnlen, + IFUNC_IMPL_ADD (array, i, strnlen, +- CPU_FEATURE_USABLE (AVX2), ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (BMI2)), + __strnlen_avx2) + IFUNC_IMPL_ADD (array, i, strnlen, + (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (BMI2) + && CPU_FEATURE_USABLE (RTM)), + __strnlen_avx2_rtm) + IFUNC_IMPL_ADD (array, i, strnlen, +@@ -640,10 +644,12 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + /* Support sysdeps/x86_64/multiarch/wcslen.c. */ + IFUNC_IMPL (i, name, wcslen, + IFUNC_IMPL_ADD (array, i, wcslen, +- CPU_FEATURE_USABLE (AVX2), ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (BMI2)), + __wcslen_avx2) + IFUNC_IMPL_ADD (array, i, wcslen, + (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (BMI2) + && CPU_FEATURE_USABLE (RTM)), + __wcslen_avx2_rtm) + IFUNC_IMPL_ADD (array, i, wcslen, +@@ -656,10 +662,12 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + /* Support sysdeps/x86_64/multiarch/wcsnlen.c. */ + IFUNC_IMPL (i, name, wcsnlen, + IFUNC_IMPL_ADD (array, i, wcsnlen, +- CPU_FEATURE_USABLE (AVX2), ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (BMI2)), + __wcsnlen_avx2) + IFUNC_IMPL_ADD (array, i, wcsnlen, + (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (BMI2) + && CPU_FEATURE_USABLE (RTM)), + __wcsnlen_avx2_rtm) + IFUNC_IMPL_ADD (array, i, wcsnlen, +diff --git a/sysdeps/x86_64/multiarch/strlen-avx2.S b/sysdeps/x86_64/multiarch/strlen-avx2.S +index 82826e10..be8a5db5 100644 +--- a/sysdeps/x86_64/multiarch/strlen-avx2.S ++++ b/sysdeps/x86_64/multiarch/strlen-avx2.S +@@ -27,9 +27,11 @@ + # ifdef USE_AS_WCSLEN + # define VPCMPEQ vpcmpeqd + # define VPMINU vpminud ++# define CHAR_SIZE 4 + # else + # define VPCMPEQ vpcmpeqb + # define VPMINU vpminub ++# define CHAR_SIZE 1 + # endif + + # ifndef VZEROUPPER +@@ -41,349 +43,459 @@ + # endif + + # define VEC_SIZE 32 ++# define PAGE_SIZE 4096 + + .section SECTION(.text),"ax",@progbits + ENTRY (STRLEN) + # ifdef USE_AS_STRNLEN +- /* Check for zero length. */ ++ /* Check zero length. */ + test %RSI_LP, %RSI_LP + jz L(zero) ++ /* Store max len in R8_LP before adjusting if using WCSLEN. */ ++ mov %RSI_LP, %R8_LP + # ifdef USE_AS_WCSLEN + shl $2, %RSI_LP + # elif defined __ILP32__ + /* Clear the upper 32 bits. */ + movl %esi, %esi + # endif +- mov %RSI_LP, %R8_LP + # endif +- movl %edi, %ecx ++ movl %edi, %eax + movq %rdi, %rdx + vpxor %xmm0, %xmm0, %xmm0 +- ++ /* Clear high bits from edi. Only keeping bits relevant to page ++ cross check. */ ++ andl $(PAGE_SIZE - 1), %eax + /* Check if we may cross page boundary with one vector load. */ +- andl $(2 * VEC_SIZE - 1), %ecx +- cmpl $VEC_SIZE, %ecx +- ja L(cros_page_boundary) ++ cmpl $(PAGE_SIZE - VEC_SIZE), %eax ++ ja L(cross_page_boundary) + + /* Check the first VEC_SIZE bytes. */ +- VPCMPEQ (%rdi), %ymm0, %ymm1 +- vpmovmskb %ymm1, %eax +- testl %eax, %eax +- ++ VPCMPEQ (%rdi), %ymm0, %ymm1 ++ vpmovmskb %ymm1, %eax + # ifdef USE_AS_STRNLEN +- jnz L(first_vec_x0_check) +- /* Adjust length and check the end of data. */ +- subq $VEC_SIZE, %rsi +- jbe L(max) +-# else +- jnz L(first_vec_x0) ++ /* If length < VEC_SIZE handle special. */ ++ cmpq $VEC_SIZE, %rsi ++ jbe L(first_vec_x0) + # endif +- +- /* Align data for aligned loads in the loop. */ +- addq $VEC_SIZE, %rdi +- andl $(VEC_SIZE - 1), %ecx +- andq $-VEC_SIZE, %rdi ++ /* If empty continue to aligned_more. Otherwise return bit ++ position of first match. */ ++ testl %eax, %eax ++ jz L(aligned_more) ++ tzcntl %eax, %eax ++# ifdef USE_AS_WCSLEN ++ shrl $2, %eax ++# endif ++ VZEROUPPER_RETURN + + # ifdef USE_AS_STRNLEN +- /* Adjust length. */ +- addq %rcx, %rsi ++L(zero): ++ xorl %eax, %eax ++ ret + +- subq $(VEC_SIZE * 4), %rsi +- jbe L(last_4x_vec_or_less) ++ .p2align 4 ++L(first_vec_x0): ++ /* Set bit for max len so that tzcnt will return min of max len ++ and position of first match. */ ++ btsq %rsi, %rax ++ tzcntl %eax, %eax ++# ifdef USE_AS_WCSLEN ++ shrl $2, %eax ++# endif ++ VZEROUPPER_RETURN + # endif +- jmp L(more_4x_vec) + + .p2align 4 +-L(cros_page_boundary): +- andl $(VEC_SIZE - 1), %ecx +- andq $-VEC_SIZE, %rdi +- VPCMPEQ (%rdi), %ymm0, %ymm1 +- vpmovmskb %ymm1, %eax +- /* Remove the leading bytes. */ +- sarl %cl, %eax +- testl %eax, %eax +- jz L(aligned_more) ++L(first_vec_x1): + tzcntl %eax, %eax ++ /* Safe to use 32 bit instructions as these are only called for ++ size = [1, 159]. */ + # ifdef USE_AS_STRNLEN +- /* Check the end of data. */ +- cmpq %rax, %rsi +- jbe L(max) ++ /* Use ecx which was computed earlier to compute correct value. ++ */ ++ subl $(VEC_SIZE * 4 + 1), %ecx ++ addl %ecx, %eax ++# else ++ subl %edx, %edi ++ incl %edi ++ addl %edi, %eax + # endif +- addq %rdi, %rax +- addq %rcx, %rax +- subq %rdx, %rax + # ifdef USE_AS_WCSLEN +- shrq $2, %rax ++ shrl $2, %eax + # endif +-L(return_vzeroupper): +- ZERO_UPPER_VEC_REGISTERS_RETURN ++ VZEROUPPER_RETURN + + .p2align 4 +-L(aligned_more): ++L(first_vec_x2): ++ tzcntl %eax, %eax ++ /* Safe to use 32 bit instructions as these are only called for ++ size = [1, 159]. */ + # ifdef USE_AS_STRNLEN +- /* "rcx" is less than VEC_SIZE. Calculate "rdx + rcx - VEC_SIZE" +- with "rdx - (VEC_SIZE - rcx)" instead of "(rdx + rcx) - VEC_SIZE" +- to void possible addition overflow. */ +- negq %rcx +- addq $VEC_SIZE, %rcx +- +- /* Check the end of data. */ +- subq %rcx, %rsi +- jbe L(max) ++ /* Use ecx which was computed earlier to compute correct value. ++ */ ++ subl $(VEC_SIZE * 3 + 1), %ecx ++ addl %ecx, %eax ++# else ++ subl %edx, %edi ++ addl $(VEC_SIZE + 1), %edi ++ addl %edi, %eax + # endif ++# ifdef USE_AS_WCSLEN ++ shrl $2, %eax ++# endif ++ VZEROUPPER_RETURN + +- addq $VEC_SIZE, %rdi ++ .p2align 4 ++L(first_vec_x3): ++ tzcntl %eax, %eax ++ /* Safe to use 32 bit instructions as these are only called for ++ size = [1, 159]. */ ++# ifdef USE_AS_STRNLEN ++ /* Use ecx which was computed earlier to compute correct value. ++ */ ++ subl $(VEC_SIZE * 2 + 1), %ecx ++ addl %ecx, %eax ++# else ++ subl %edx, %edi ++ addl $(VEC_SIZE * 2 + 1), %edi ++ addl %edi, %eax ++# endif ++# ifdef USE_AS_WCSLEN ++ shrl $2, %eax ++# endif ++ VZEROUPPER_RETURN + ++ .p2align 4 ++L(first_vec_x4): ++ tzcntl %eax, %eax ++ /* Safe to use 32 bit instructions as these are only called for ++ size = [1, 159]. */ + # ifdef USE_AS_STRNLEN +- subq $(VEC_SIZE * 4), %rsi +- jbe L(last_4x_vec_or_less) ++ /* Use ecx which was computed earlier to compute correct value. ++ */ ++ subl $(VEC_SIZE + 1), %ecx ++ addl %ecx, %eax ++# else ++ subl %edx, %edi ++ addl $(VEC_SIZE * 3 + 1), %edi ++ addl %edi, %eax + # endif ++# ifdef USE_AS_WCSLEN ++ shrl $2, %eax ++# endif ++ VZEROUPPER_RETURN + +-L(more_4x_vec): ++ .p2align 5 ++L(aligned_more): ++ /* Align data to VEC_SIZE - 1. This is the same number of ++ instructions as using andq with -VEC_SIZE but saves 4 bytes of ++ code on the x4 check. */ ++ orq $(VEC_SIZE - 1), %rdi ++L(cross_page_continue): + /* Check the first 4 * VEC_SIZE. Only one VEC_SIZE at a time + since data is only aligned to VEC_SIZE. */ +- VPCMPEQ (%rdi), %ymm0, %ymm1 +- vpmovmskb %ymm1, %eax +- testl %eax, %eax +- jnz L(first_vec_x0) +- +- VPCMPEQ VEC_SIZE(%rdi), %ymm0, %ymm1 +- vpmovmskb %ymm1, %eax ++# ifdef USE_AS_STRNLEN ++ /* + 1 because rdi is aligned to VEC_SIZE - 1. + CHAR_SIZE because ++ it simplies the logic in last_4x_vec_or_less. */ ++ leaq (VEC_SIZE * 4 + CHAR_SIZE + 1)(%rdi), %rcx ++ subq %rdx, %rcx ++# endif ++ /* Load first VEC regardless. */ ++ VPCMPEQ 1(%rdi), %ymm0, %ymm1 ++# ifdef USE_AS_STRNLEN ++ /* Adjust length. If near end handle specially. */ ++ subq %rcx, %rsi ++ jb L(last_4x_vec_or_less) ++# endif ++ vpmovmskb %ymm1, %eax + testl %eax, %eax + jnz L(first_vec_x1) + +- VPCMPEQ (VEC_SIZE * 2)(%rdi), %ymm0, %ymm1 +- vpmovmskb %ymm1, %eax ++ VPCMPEQ (VEC_SIZE + 1)(%rdi), %ymm0, %ymm1 ++ vpmovmskb %ymm1, %eax + testl %eax, %eax + jnz L(first_vec_x2) + +- VPCMPEQ (VEC_SIZE * 3)(%rdi), %ymm0, %ymm1 +- vpmovmskb %ymm1, %eax ++ VPCMPEQ (VEC_SIZE * 2 + 1)(%rdi), %ymm0, %ymm1 ++ vpmovmskb %ymm1, %eax + testl %eax, %eax + jnz L(first_vec_x3) + +- addq $(VEC_SIZE * 4), %rdi +- +-# ifdef USE_AS_STRNLEN +- subq $(VEC_SIZE * 4), %rsi +- jbe L(last_4x_vec_or_less) +-# endif +- +- /* Align data to 4 * VEC_SIZE. */ +- movq %rdi, %rcx +- andl $(4 * VEC_SIZE - 1), %ecx +- andq $-(4 * VEC_SIZE), %rdi ++ VPCMPEQ (VEC_SIZE * 3 + 1)(%rdi), %ymm0, %ymm1 ++ vpmovmskb %ymm1, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x4) + ++ /* Align data to VEC_SIZE * 4 - 1. */ + # ifdef USE_AS_STRNLEN +- /* Adjust length. */ ++ /* Before adjusting length check if at last VEC_SIZE * 4. */ ++ cmpq $(VEC_SIZE * 4 - 1), %rsi ++ jbe L(last_4x_vec_or_less_load) ++ incq %rdi ++ movl %edi, %ecx ++ orq $(VEC_SIZE * 4 - 1), %rdi ++ andl $(VEC_SIZE * 4 - 1), %ecx ++ /* Readjust length. */ + addq %rcx, %rsi ++# else ++ incq %rdi ++ orq $(VEC_SIZE * 4 - 1), %rdi + # endif +- ++ /* Compare 4 * VEC at a time forward. */ + .p2align 4 + L(loop_4x_vec): +- /* Compare 4 * VEC at a time forward. */ +- vmovdqa (%rdi), %ymm1 +- vmovdqa VEC_SIZE(%rdi), %ymm2 +- vmovdqa (VEC_SIZE * 2)(%rdi), %ymm3 +- vmovdqa (VEC_SIZE * 3)(%rdi), %ymm4 +- VPMINU %ymm1, %ymm2, %ymm5 +- VPMINU %ymm3, %ymm4, %ymm6 +- VPMINU %ymm5, %ymm6, %ymm5 +- +- VPCMPEQ %ymm5, %ymm0, %ymm5 +- vpmovmskb %ymm5, %eax +- testl %eax, %eax +- jnz L(4x_vec_end) +- +- addq $(VEC_SIZE * 4), %rdi +- +-# ifndef USE_AS_STRNLEN +- jmp L(loop_4x_vec) +-# else ++# ifdef USE_AS_STRNLEN ++ /* Break if at end of length. */ + subq $(VEC_SIZE * 4), %rsi +- ja L(loop_4x_vec) +- +-L(last_4x_vec_or_less): +- /* Less than 4 * VEC and aligned to VEC_SIZE. */ +- addl $(VEC_SIZE * 2), %esi +- jle L(last_2x_vec) ++ jb L(last_4x_vec_or_less_cmpeq) ++# endif ++ /* Save some code size by microfusing VPMINU with the load. Since ++ the matches in ymm2/ymm4 can only be returned if there where no ++ matches in ymm1/ymm3 respectively there is no issue with overlap. ++ */ ++ vmovdqa 1(%rdi), %ymm1 ++ VPMINU (VEC_SIZE + 1)(%rdi), %ymm1, %ymm2 ++ vmovdqa (VEC_SIZE * 2 + 1)(%rdi), %ymm3 ++ VPMINU (VEC_SIZE * 3 + 1)(%rdi), %ymm3, %ymm4 ++ ++ VPMINU %ymm2, %ymm4, %ymm5 ++ VPCMPEQ %ymm5, %ymm0, %ymm5 ++ vpmovmskb %ymm5, %ecx + +- VPCMPEQ (%rdi), %ymm0, %ymm1 +- vpmovmskb %ymm1, %eax +- testl %eax, %eax +- jnz L(first_vec_x0) ++ subq $-(VEC_SIZE * 4), %rdi ++ testl %ecx, %ecx ++ jz L(loop_4x_vec) + +- VPCMPEQ VEC_SIZE(%rdi), %ymm0, %ymm1 +- vpmovmskb %ymm1, %eax +- testl %eax, %eax +- jnz L(first_vec_x1) + +- VPCMPEQ (VEC_SIZE * 2)(%rdi), %ymm0, %ymm1 +- vpmovmskb %ymm1, %eax ++ VPCMPEQ %ymm1, %ymm0, %ymm1 ++ vpmovmskb %ymm1, %eax ++ subq %rdx, %rdi + testl %eax, %eax ++ jnz L(last_vec_return_x0) + +- jnz L(first_vec_x2_check) +- subl $VEC_SIZE, %esi +- jle L(max) +- +- VPCMPEQ (VEC_SIZE * 3)(%rdi), %ymm0, %ymm1 +- vpmovmskb %ymm1, %eax ++ VPCMPEQ %ymm2, %ymm0, %ymm2 ++ vpmovmskb %ymm2, %eax + testl %eax, %eax +- +- jnz L(first_vec_x3_check) +- movq %r8, %rax +-# ifdef USE_AS_WCSLEN ++ jnz L(last_vec_return_x1) ++ ++ /* Combine last 2 VEC. */ ++ VPCMPEQ %ymm3, %ymm0, %ymm3 ++ vpmovmskb %ymm3, %eax ++ /* rcx has combined result from all 4 VEC. It will only be used if ++ the first 3 other VEC all did not contain a match. */ ++ salq $32, %rcx ++ orq %rcx, %rax ++ tzcntq %rax, %rax ++ subq $(VEC_SIZE * 2 - 1), %rdi ++ addq %rdi, %rax ++# ifdef USE_AS_WCSLEN + shrq $2, %rax +-# endif ++# endif + VZEROUPPER_RETURN + ++ ++# ifdef USE_AS_STRNLEN + .p2align 4 +-L(last_2x_vec): +- addl $(VEC_SIZE * 2), %esi +- VPCMPEQ (%rdi), %ymm0, %ymm1 +- vpmovmskb %ymm1, %eax +- testl %eax, %eax ++L(last_4x_vec_or_less_load): ++ /* Depending on entry adjust rdi / prepare first VEC in ymm1. */ ++ subq $-(VEC_SIZE * 4), %rdi ++L(last_4x_vec_or_less_cmpeq): ++ VPCMPEQ 1(%rdi), %ymm0, %ymm1 ++L(last_4x_vec_or_less): + +- jnz L(first_vec_x0_check) +- subl $VEC_SIZE, %esi +- jle L(max) ++ vpmovmskb %ymm1, %eax ++ /* If remaining length > VEC_SIZE * 2. This works if esi is off by ++ VEC_SIZE * 4. */ ++ testl $(VEC_SIZE * 2), %esi ++ jnz L(last_4x_vec) + +- VPCMPEQ VEC_SIZE(%rdi), %ymm0, %ymm1 +- vpmovmskb %ymm1, %eax ++ /* length may have been negative or positive by an offset of ++ VEC_SIZE * 4 depending on where this was called from. This fixes ++ that. */ ++ andl $(VEC_SIZE * 4 - 1), %esi + testl %eax, %eax +- jnz L(first_vec_x1_check) +- movq %r8, %rax +-# ifdef USE_AS_WCSLEN +- shrq $2, %rax +-# endif +- VZEROUPPER_RETURN ++ jnz L(last_vec_x1_check) + +- .p2align 4 +-L(first_vec_x0_check): ++ subl $VEC_SIZE, %esi ++ jb L(max) ++ ++ VPCMPEQ (VEC_SIZE + 1)(%rdi), %ymm0, %ymm1 ++ vpmovmskb %ymm1, %eax + tzcntl %eax, %eax + /* Check the end of data. */ +- cmpq %rax, %rsi +- jbe L(max) ++ cmpl %eax, %esi ++ jb L(max) ++ subq %rdx, %rdi ++ addl $(VEC_SIZE + 1), %eax + addq %rdi, %rax +- subq %rdx, %rax + # ifdef USE_AS_WCSLEN + shrq $2, %rax + # endif + VZEROUPPER_RETURN ++# endif + + .p2align 4 +-L(first_vec_x1_check): ++L(last_vec_return_x0): + tzcntl %eax, %eax +- /* Check the end of data. */ +- cmpq %rax, %rsi +- jbe L(max) +- addq $VEC_SIZE, %rax ++ subq $(VEC_SIZE * 4 - 1), %rdi + addq %rdi, %rax +- subq %rdx, %rax +-# ifdef USE_AS_WCSLEN ++# ifdef USE_AS_WCSLEN + shrq $2, %rax +-# endif ++# endif + VZEROUPPER_RETURN + + .p2align 4 +-L(first_vec_x2_check): ++L(last_vec_return_x1): + tzcntl %eax, %eax +- /* Check the end of data. */ +- cmpq %rax, %rsi +- jbe L(max) +- addq $(VEC_SIZE * 2), %rax ++ subq $(VEC_SIZE * 3 - 1), %rdi + addq %rdi, %rax +- subq %rdx, %rax +-# ifdef USE_AS_WCSLEN ++# ifdef USE_AS_WCSLEN + shrq $2, %rax +-# endif ++# endif + VZEROUPPER_RETURN + ++# ifdef USE_AS_STRNLEN + .p2align 4 +-L(first_vec_x3_check): ++L(last_vec_x1_check): ++ + tzcntl %eax, %eax + /* Check the end of data. */ +- cmpq %rax, %rsi +- jbe L(max) +- addq $(VEC_SIZE * 3), %rax ++ cmpl %eax, %esi ++ jb L(max) ++ subq %rdx, %rdi ++ incl %eax + addq %rdi, %rax +- subq %rdx, %rax + # ifdef USE_AS_WCSLEN + shrq $2, %rax + # endif + VZEROUPPER_RETURN + +- .p2align 4 + L(max): + movq %r8, %rax ++ VZEROUPPER_RETURN ++ ++ .p2align 4 ++L(last_4x_vec): ++ /* Test first 2x VEC normally. */ ++ testl %eax, %eax ++ jnz L(last_vec_x1) ++ ++ VPCMPEQ (VEC_SIZE + 1)(%rdi), %ymm0, %ymm1 ++ vpmovmskb %ymm1, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x2) ++ ++ /* Normalize length. */ ++ andl $(VEC_SIZE * 4 - 1), %esi ++ VPCMPEQ (VEC_SIZE * 2 + 1)(%rdi), %ymm0, %ymm1 ++ vpmovmskb %ymm1, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x3) ++ ++ subl $(VEC_SIZE * 3), %esi ++ jb L(max) ++ ++ VPCMPEQ (VEC_SIZE * 3 + 1)(%rdi), %ymm0, %ymm1 ++ vpmovmskb %ymm1, %eax ++ tzcntl %eax, %eax ++ /* Check the end of data. */ ++ cmpl %eax, %esi ++ jb L(max) ++ subq %rdx, %rdi ++ addl $(VEC_SIZE * 3 + 1), %eax ++ addq %rdi, %rax + # ifdef USE_AS_WCSLEN + shrq $2, %rax + # endif + VZEROUPPER_RETURN + +- .p2align 4 +-L(zero): +- xorl %eax, %eax +- ret +-# endif + + .p2align 4 +-L(first_vec_x0): ++L(last_vec_x1): ++ /* essentially duplicates of first_vec_x1 but use 64 bit ++ instructions. */ + tzcntl %eax, %eax ++ subq %rdx, %rdi ++ incl %eax + addq %rdi, %rax +- subq %rdx, %rax +-# ifdef USE_AS_WCSLEN ++# ifdef USE_AS_WCSLEN + shrq $2, %rax +-# endif ++# endif + VZEROUPPER_RETURN + + .p2align 4 +-L(first_vec_x1): ++L(last_vec_x2): ++ /* essentially duplicates of first_vec_x1 but use 64 bit ++ instructions. */ + tzcntl %eax, %eax +- addq $VEC_SIZE, %rax ++ subq %rdx, %rdi ++ addl $(VEC_SIZE + 1), %eax + addq %rdi, %rax +- subq %rdx, %rax +-# ifdef USE_AS_WCSLEN ++# ifdef USE_AS_WCSLEN + shrq $2, %rax +-# endif ++# endif + VZEROUPPER_RETURN + + .p2align 4 +-L(first_vec_x2): ++L(last_vec_x3): + tzcntl %eax, %eax +- addq $(VEC_SIZE * 2), %rax ++ subl $(VEC_SIZE * 2), %esi ++ /* Check the end of data. */ ++ cmpl %eax, %esi ++ jb L(max_end) ++ subq %rdx, %rdi ++ addl $(VEC_SIZE * 2 + 1), %eax + addq %rdi, %rax +- subq %rdx, %rax +-# ifdef USE_AS_WCSLEN ++# ifdef USE_AS_WCSLEN + shrq $2, %rax +-# endif ++# endif ++ VZEROUPPER_RETURN ++L(max_end): ++ movq %r8, %rax + VZEROUPPER_RETURN ++# endif + ++ /* Cold case for crossing page with first load. */ + .p2align 4 +-L(4x_vec_end): +- VPCMPEQ %ymm1, %ymm0, %ymm1 +- vpmovmskb %ymm1, %eax +- testl %eax, %eax +- jnz L(first_vec_x0) +- VPCMPEQ %ymm2, %ymm0, %ymm2 +- vpmovmskb %ymm2, %eax ++L(cross_page_boundary): ++ /* Align data to VEC_SIZE - 1. */ ++ orq $(VEC_SIZE - 1), %rdi ++ VPCMPEQ -(VEC_SIZE - 1)(%rdi), %ymm0, %ymm1 ++ vpmovmskb %ymm1, %eax ++ /* Remove the leading bytes. sarxl only uses bits [5:0] of COUNT ++ so no need to manually mod rdx. */ ++ sarxl %edx, %eax, %eax ++# ifdef USE_AS_STRNLEN + testl %eax, %eax +- jnz L(first_vec_x1) +- VPCMPEQ %ymm3, %ymm0, %ymm3 +- vpmovmskb %ymm3, %eax ++ jnz L(cross_page_less_vec) ++ leaq 1(%rdi), %rcx ++ subq %rdx, %rcx ++ /* Check length. */ ++ cmpq %rsi, %rcx ++ jb L(cross_page_continue) ++ movq %r8, %rax ++# else + testl %eax, %eax +- jnz L(first_vec_x2) +- VPCMPEQ %ymm4, %ymm0, %ymm4 +- vpmovmskb %ymm4, %eax +-L(first_vec_x3): ++ jz L(cross_page_continue) + tzcntl %eax, %eax +- addq $(VEC_SIZE * 3), %rax +- addq %rdi, %rax +- subq %rdx, %rax +-# ifdef USE_AS_WCSLEN +- shrq $2, %rax ++# ifdef USE_AS_WCSLEN ++ shrl $2, %eax ++# endif + # endif ++L(return_vzeroupper): ++ ZERO_UPPER_VEC_REGISTERS_RETURN ++ ++# ifdef USE_AS_STRNLEN ++ .p2align 4 ++L(cross_page_less_vec): ++ tzcntl %eax, %eax ++ cmpq %rax, %rsi ++ cmovb %esi, %eax ++# ifdef USE_AS_WCSLEN ++ shrl $2, %eax ++# endif + VZEROUPPER_RETURN ++# endif + + END (STRLEN) + #endif +-- +GitLab + diff --git a/SOURCES/ia-testl-x86_string_control.patch b/SOURCES/ia-testl-x86_string_control.patch new file mode 100644 index 0000000..e89ee57 --- /dev/null +++ b/SOURCES/ia-testl-x86_string_control.patch @@ -0,0 +1,39 @@ +From 7af32e1d2ba4a5a98a2033bf18de200912ae90a2 Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Sat, 28 Aug 2021 06:10:38 -0700 +Subject: [PATCH] x86-64: Use testl to check __x86_string_control + +Use testl, instead of andl, to check __x86_string_control to avoid +updating __x86_string_control. + +Reviewed-by: Carlos O'Donell +(cherry picked from commit 3c8b9879cab6d41787bc5b14c1748f62fd6d0e5f) +--- + sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S +index 0469bf99..c0809b1b 100644 +--- a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S ++++ b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S +@@ -325,7 +325,7 @@ L(movsb): + /* Avoid slow backward REP MOVSB. */ + jb L(more_8x_vec_backward) + # if AVOID_SHORT_DISTANCE_REP_MOVSB +- andl $X86_STRING_CONTROL_AVOID_SHORT_DISTANCE_REP_MOVSB, __x86_string_control(%rip) ++ testl $X86_STRING_CONTROL_AVOID_SHORT_DISTANCE_REP_MOVSB, __x86_string_control(%rip) + jz 3f + movq %rdi, %rcx + subq %rsi, %rcx +@@ -333,7 +333,7 @@ L(movsb): + # endif + 1: + # if AVOID_SHORT_DISTANCE_REP_MOVSB +- andl $X86_STRING_CONTROL_AVOID_SHORT_DISTANCE_REP_MOVSB, __x86_string_control(%rip) ++ testl $X86_STRING_CONTROL_AVOID_SHORT_DISTANCE_REP_MOVSB, __x86_string_control(%rip) + jz 3f + movq %rsi, %rcx + subq %rdi, %rcx +-- +GitLab + diff --git a/SOURCES/ia-unk-vector-opr-memchr-evex.patch b/SOURCES/ia-unk-vector-opr-memchr-evex.patch new file mode 100644 index 0000000..d0070d0 --- /dev/null +++ b/SOURCES/ia-unk-vector-opr-memchr-evex.patch @@ -0,0 +1,30 @@ +From 3210a5e4acc812b94b8514d8d9664c7b4b6451af Mon Sep 17 00:00:00 2001 +From: Alice Xu +Date: Fri, 7 May 2021 19:03:21 -0700 +Subject: [PATCH] x86-64: Fix an unknown vector operation in memchr-evex.S + +An unknown vector operation occurred in commit 2a76821c308. Fixed it +by using "ymm{k1}{z}" but not "ymm {k1} {z}". + +Reviewed-by: H.J. Lu +(cherry picked from commit 6ea916adfa0ab9af6e7dc6adcf6f977dfe017835) +--- + sysdeps/x86_64/multiarch/memchr-evex.S | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/sysdeps/x86_64/multiarch/memchr-evex.S b/sysdeps/x86_64/multiarch/memchr-evex.S +index 81d5cd64..f3fdad4f 100644 +--- a/sysdeps/x86_64/multiarch/memchr-evex.S ++++ b/sysdeps/x86_64/multiarch/memchr-evex.S +@@ -271,7 +271,7 @@ L(loop_4x_vec): + vpxorq (VEC_SIZE * 6)(%rdi), %YMMMATCH, %YMM3 + VPCMP $0, (VEC_SIZE * 7)(%rdi), %YMMMATCH, %k3 + /* Reduce VEC2 / VEC3 with min and VEC1 with zero mask. */ +- VPMINU %YMM2, %YMM3, %YMM3 {%k1} {z} ++ VPMINU %YMM2, %YMM3, %YMM3{%k1}{z} + VPCMP $0, %YMM3, %YMMZERO, %k2 + # ifdef USE_AS_RAWMEMCHR + subq $-(VEC_SIZE * 4), %rdi +-- +GitLab + diff --git a/SOURCES/ia-upd-256bit-evex-instr-1.patch b/SOURCES/ia-upd-256bit-evex-instr-1.patch new file mode 100644 index 0000000..4a299bf --- /dev/null +++ b/SOURCES/ia-upd-256bit-evex-instr-1.patch @@ -0,0 +1,3407 @@ +From 22a1b88414d40b700c84689d08a6026e3fdee874 Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Fri, 5 Mar 2021 06:24:52 -0800 +Subject: [PATCH] x86-64: Add ifunc-avx2.h functions with 256-bit EVEX + +Update ifunc-avx2.h, strchr.c, strcmp.c, strncmp.c and wcsnlen.c to +select the function optimized with 256-bit EVEX instructions using +YMM16-YMM31 registers to avoid RTM abort with usable AVX512VL, AVX512BW +and BMI2 since VZEROUPPER isn't needed at function exit. + +For strcmp/strncmp, prefer AVX2 strcmp/strncmp if Prefer_AVX2_STRCMP +is set. + +(cherry picked from commit 1fd8c163a83d96ace1ff78fa6bac7aee084f6f77) +--- + sysdeps/x86_64/multiarch/Makefile | 21 +- + sysdeps/x86_64/multiarch/ifunc-avx2.h | 14 +- + sysdeps/x86_64/multiarch/ifunc-impl-list.c | 81 ++ + sysdeps/x86_64/multiarch/memchr-evex.S | 381 +++++++ + sysdeps/x86_64/multiarch/memrchr-evex.S | 337 +++++++ + sysdeps/x86_64/multiarch/rawmemchr-evex.S | 4 + + sysdeps/x86_64/multiarch/strchr-evex.S | 335 +++++++ + sysdeps/x86_64/multiarch/strchr.c | 14 +- + sysdeps/x86_64/multiarch/strchrnul-evex.S | 3 + + sysdeps/x86_64/multiarch/strcmp-evex.S | 1043 ++++++++++++++++++++ + sysdeps/x86_64/multiarch/strcmp.c | 15 +- + sysdeps/x86_64/multiarch/strlen-evex.S | 436 ++++++++ + sysdeps/x86_64/multiarch/strncmp-evex.S | 3 + + sysdeps/x86_64/multiarch/strncmp.c | 15 +- + sysdeps/x86_64/multiarch/strnlen-evex.S | 4 + + sysdeps/x86_64/multiarch/strrchr-evex.S | 265 +++++ + sysdeps/x86_64/multiarch/wcschr-evex.S | 3 + + sysdeps/x86_64/multiarch/wcscmp-evex.S | 4 + + sysdeps/x86_64/multiarch/wcslen-evex.S | 4 + + sysdeps/x86_64/multiarch/wcsncmp-evex.S | 5 + + sysdeps/x86_64/multiarch/wcsnlen-evex.S | 5 + + sysdeps/x86_64/multiarch/wcsnlen.c | 14 +- + sysdeps/x86_64/multiarch/wcsrchr-evex.S | 3 + + sysdeps/x86_64/multiarch/wmemchr-evex.S | 4 + + 24 files changed, 2996 insertions(+), 17 deletions(-) + create mode 100644 sysdeps/x86_64/multiarch/memchr-evex.S + create mode 100644 sysdeps/x86_64/multiarch/memrchr-evex.S + create mode 100644 sysdeps/x86_64/multiarch/rawmemchr-evex.S + create mode 100644 sysdeps/x86_64/multiarch/strchr-evex.S + create mode 100644 sysdeps/x86_64/multiarch/strchrnul-evex.S + create mode 100644 sysdeps/x86_64/multiarch/strcmp-evex.S + create mode 100644 sysdeps/x86_64/multiarch/strlen-evex.S + create mode 100644 sysdeps/x86_64/multiarch/strncmp-evex.S + create mode 100644 sysdeps/x86_64/multiarch/strnlen-evex.S + create mode 100644 sysdeps/x86_64/multiarch/strrchr-evex.S + create mode 100644 sysdeps/x86_64/multiarch/wcschr-evex.S + create mode 100644 sysdeps/x86_64/multiarch/wcscmp-evex.S + create mode 100644 sysdeps/x86_64/multiarch/wcslen-evex.S + create mode 100644 sysdeps/x86_64/multiarch/wcsncmp-evex.S + create mode 100644 sysdeps/x86_64/multiarch/wcsnlen-evex.S + create mode 100644 sysdeps/x86_64/multiarch/wcsrchr-evex.S + create mode 100644 sysdeps/x86_64/multiarch/wmemchr-evex.S + +diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile +index 9477538a..5ce85882 100644 +--- a/sysdeps/x86_64/multiarch/Makefile ++++ b/sysdeps/x86_64/multiarch/Makefile +@@ -39,7 +39,17 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c \ + memmove-avx512-unaligned-erms \ + memset-sse2-unaligned-erms \ + memset-avx2-unaligned-erms \ +- memset-avx512-unaligned-erms ++ memset-avx512-unaligned-erms \ ++ memchr-evex \ ++ memrchr-evex \ ++ rawmemchr-evex \ ++ strchr-evex \ ++ strchrnul-evex \ ++ strcmp-evex \ ++ strlen-evex \ ++ strncmp-evex \ ++ strnlen-evex \ ++ strrchr-evex + CFLAGS-varshift.c += -msse4 + CFLAGS-strcspn-c.c += -msse4 + CFLAGS-strpbrk-c.c += -msse4 +@@ -56,7 +66,14 @@ sysdep_routines += wmemcmp-sse4 wmemcmp-ssse3 wmemcmp-c \ + wcschr-sse2 wcschr-avx2 \ + wcsrchr-sse2 wcsrchr-avx2 \ + wcsnlen-sse4_1 wcsnlen-c \ +- wcslen-sse2 wcslen-avx2 wcsnlen-avx2 ++ wcslen-sse2 wcslen-avx2 wcsnlen-avx2 \ ++ wcschr-evex \ ++ wcscmp-evex \ ++ wcslen-evex \ ++ wcsncmp-evex \ ++ wcsnlen-evex \ ++ wcsrchr-evex \ ++ wmemchr-evex + endif + + ifeq ($(subdir),debug) +diff --git a/sysdeps/x86_64/multiarch/ifunc-avx2.h b/sysdeps/x86_64/multiarch/ifunc-avx2.h +index 5c88640a..7081b0c9 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-avx2.h ++++ b/sysdeps/x86_64/multiarch/ifunc-avx2.h +@@ -21,16 +21,24 @@ + + extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden; + + static inline void * + IFUNC_SELECTOR (void) + { + const struct cpu_features* cpu_features = __get_cpu_features (); + +- if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER) +- && CPU_FEATURE_USABLE_P (cpu_features, AVX2) ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX2) + && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) +- return OPTIMIZE (avx2); ++ { ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) ++ && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW) ++ && CPU_FEATURE_USABLE_P (cpu_features, BMI2)) ++ return OPTIMIZE (evex); ++ ++ if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) ++ return OPTIMIZE (avx2); ++ } + + return OPTIMIZE (sse2); + } +diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c +index fe13505c..bd7d9f19 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c ++++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c +@@ -43,6 +43,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, memchr, + CPU_FEATURE_USABLE (AVX2), + __memchr_avx2) ++ IFUNC_IMPL_ADD (array, i, memchr, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), ++ __memchr_evex) + IFUNC_IMPL_ADD (array, i, memchr, 1, __memchr_sse2)) + + /* Support sysdeps/x86_64/multiarch/memcmp.c. */ +@@ -121,6 +126,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, memrchr, + CPU_FEATURE_USABLE (AVX2), + __memrchr_avx2) ++ IFUNC_IMPL_ADD (array, i, memrchr, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW)), ++ __memrchr_evex) ++ + IFUNC_IMPL_ADD (array, i, memrchr, 1, __memrchr_sse2)) + + #ifdef SHARED +@@ -179,6 +189,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, rawmemchr, + CPU_FEATURE_USABLE (AVX2), + __rawmemchr_avx2) ++ IFUNC_IMPL_ADD (array, i, rawmemchr, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), ++ __rawmemchr_evex) + IFUNC_IMPL_ADD (array, i, rawmemchr, 1, __rawmemchr_sse2)) + + /* Support sysdeps/x86_64/multiarch/strlen.c. */ +@@ -186,6 +201,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, strlen, + CPU_FEATURE_USABLE (AVX2), + __strlen_avx2) ++ IFUNC_IMPL_ADD (array, i, strlen, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW)), ++ __strlen_evex) + IFUNC_IMPL_ADD (array, i, strlen, 1, __strlen_sse2)) + + /* Support sysdeps/x86_64/multiarch/strnlen.c. */ +@@ -193,6 +212,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, strnlen, + CPU_FEATURE_USABLE (AVX2), + __strnlen_avx2) ++ IFUNC_IMPL_ADD (array, i, strnlen, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW)), ++ __strnlen_evex) + IFUNC_IMPL_ADD (array, i, strnlen, 1, __strnlen_sse2)) + + /* Support sysdeps/x86_64/multiarch/stpncpy.c. */ +@@ -255,6 +278,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, strchr, + CPU_FEATURE_USABLE (AVX2), + __strchr_avx2) ++ IFUNC_IMPL_ADD (array, i, strchr, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), ++ __strchr_evex) + IFUNC_IMPL_ADD (array, i, strchr, 1, __strchr_sse2_no_bsf) + IFUNC_IMPL_ADD (array, i, strchr, 1, __strchr_sse2)) + +@@ -263,6 +291,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, strchrnul, + CPU_FEATURE_USABLE (AVX2), + __strchrnul_avx2) ++ IFUNC_IMPL_ADD (array, i, strchrnul, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), ++ __strchrnul_evex) + IFUNC_IMPL_ADD (array, i, strchrnul, 1, __strchrnul_sse2)) + + /* Support sysdeps/x86_64/multiarch/strrchr.c. */ +@@ -270,6 +303,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, strrchr, + CPU_FEATURE_USABLE (AVX2), + __strrchr_avx2) ++ IFUNC_IMPL_ADD (array, i, strrchr, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW)), ++ __strrchr_evex) + IFUNC_IMPL_ADD (array, i, strrchr, 1, __strrchr_sse2)) + + /* Support sysdeps/x86_64/multiarch/strcmp.c. */ +@@ -277,6 +314,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, strcmp, + CPU_FEATURE_USABLE (AVX2), + __strcmp_avx2) ++ IFUNC_IMPL_ADD (array, i, strcmp, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), ++ __strcmp_evex) + IFUNC_IMPL_ADD (array, i, strcmp, CPU_FEATURE_USABLE (SSE4_2), + __strcmp_sse42) + IFUNC_IMPL_ADD (array, i, strcmp, CPU_FEATURE_USABLE (SSSE3), +@@ -370,6 +412,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, wcschr, + CPU_FEATURE_USABLE (AVX2), + __wcschr_avx2) ++ IFUNC_IMPL_ADD (array, i, wcschr, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), ++ __wcschr_evex) + IFUNC_IMPL_ADD (array, i, wcschr, 1, __wcschr_sse2)) + + /* Support sysdeps/x86_64/multiarch/wcsrchr.c. */ +@@ -377,6 +424,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, wcsrchr, + CPU_FEATURE_USABLE (AVX2), + __wcsrchr_avx2) ++ IFUNC_IMPL_ADD (array, i, wcsrchr, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), ++ __wcsrchr_evex) + IFUNC_IMPL_ADD (array, i, wcsrchr, 1, __wcsrchr_sse2)) + + /* Support sysdeps/x86_64/multiarch/wcscmp.c. */ +@@ -384,6 +436,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, wcscmp, + CPU_FEATURE_USABLE (AVX2), + __wcscmp_avx2) ++ IFUNC_IMPL_ADD (array, i, wcscmp, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), ++ __wcscmp_evex) + IFUNC_IMPL_ADD (array, i, wcscmp, 1, __wcscmp_sse2)) + + /* Support sysdeps/x86_64/multiarch/wcsncmp.c. */ +@@ -391,6 +448,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, wcsncmp, + CPU_FEATURE_USABLE (AVX2), + __wcsncmp_avx2) ++ IFUNC_IMPL_ADD (array, i, wcsncmp, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), ++ __wcsncmp_evex) + IFUNC_IMPL_ADD (array, i, wcsncmp, 1, __wcsncmp_sse2)) + + /* Support sysdeps/x86_64/multiarch/wcscpy.c. */ +@@ -404,6 +466,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, wcslen, + CPU_FEATURE_USABLE (AVX2), + __wcslen_avx2) ++ IFUNC_IMPL_ADD (array, i, wcslen, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), ++ __wcslen_evex) + IFUNC_IMPL_ADD (array, i, wcslen, 1, __wcslen_sse2)) + + /* Support sysdeps/x86_64/multiarch/wcsnlen.c. */ +@@ -411,6 +478,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, wcsnlen, + CPU_FEATURE_USABLE (AVX2), + __wcsnlen_avx2) ++ IFUNC_IMPL_ADD (array, i, wcsnlen, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), ++ __wcsnlen_evex) + IFUNC_IMPL_ADD (array, i, wcsnlen, + CPU_FEATURE_USABLE (SSE4_1), + __wcsnlen_sse4_1) +@@ -421,6 +493,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, wmemchr, + CPU_FEATURE_USABLE (AVX2), + __wmemchr_avx2) ++ IFUNC_IMPL_ADD (array, i, wmemchr, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), ++ __wmemchr_evex) + IFUNC_IMPL_ADD (array, i, wmemchr, 1, __wmemchr_sse2)) + + /* Support sysdeps/x86_64/multiarch/wmemcmp.c. */ +@@ -568,6 +645,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, strncmp, + CPU_FEATURE_USABLE (AVX2), + __strncmp_avx2) ++ IFUNC_IMPL_ADD (array, i, strncmp, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW)), ++ __strncmp_evex) + IFUNC_IMPL_ADD (array, i, strncmp, CPU_FEATURE_USABLE (SSE4_2), + __strncmp_sse42) + IFUNC_IMPL_ADD (array, i, strncmp, CPU_FEATURE_USABLE (SSSE3), +diff --git a/sysdeps/x86_64/multiarch/memchr-evex.S b/sysdeps/x86_64/multiarch/memchr-evex.S +new file mode 100644 +index 00000000..6dd5d67b +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/memchr-evex.S +@@ -0,0 +1,381 @@ ++/* memchr/wmemchr optimized with 256-bit EVEX instructions. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#if IS_IN (libc) ++ ++# include ++ ++# ifndef MEMCHR ++# define MEMCHR __memchr_evex ++# endif ++ ++# ifdef USE_AS_WMEMCHR ++# define VPBROADCAST vpbroadcastd ++# define VPCMP vpcmpd ++# define SHIFT_REG r8d ++# else ++# define VPBROADCAST vpbroadcastb ++# define VPCMP vpcmpb ++# define SHIFT_REG ecx ++# endif ++ ++# define XMMMATCH xmm16 ++# define YMMMATCH ymm16 ++# define YMM1 ymm17 ++# define YMM2 ymm18 ++# define YMM3 ymm19 ++# define YMM4 ymm20 ++# define YMM5 ymm21 ++# define YMM6 ymm22 ++ ++# define VEC_SIZE 32 ++ ++ .section .text.evex,"ax",@progbits ++ENTRY (MEMCHR) ++# ifndef USE_AS_RAWMEMCHR ++ /* Check for zero length. */ ++ test %RDX_LP, %RDX_LP ++ jz L(zero) ++# endif ++ movl %edi, %ecx ++# ifdef USE_AS_WMEMCHR ++ shl $2, %RDX_LP ++# else ++# ifdef __ILP32__ ++ /* Clear the upper 32 bits. */ ++ movl %edx, %edx ++# endif ++# endif ++ /* Broadcast CHAR to YMMMATCH. */ ++ VPBROADCAST %esi, %YMMMATCH ++ /* Check if we may cross page boundary with one vector load. */ ++ andl $(2 * VEC_SIZE - 1), %ecx ++ cmpl $VEC_SIZE, %ecx ++ ja L(cros_page_boundary) ++ ++ /* Check the first VEC_SIZE bytes. */ ++ VPCMP $0, (%rdi), %YMMMATCH, %k1 ++ kmovd %k1, %eax ++ testl %eax, %eax ++ ++# ifndef USE_AS_RAWMEMCHR ++ jnz L(first_vec_x0_check) ++ /* Adjust length and check the end of data. */ ++ subq $VEC_SIZE, %rdx ++ jbe L(zero) ++# else ++ jnz L(first_vec_x0) ++# endif ++ ++ /* Align data for aligned loads in the loop. */ ++ addq $VEC_SIZE, %rdi ++ andl $(VEC_SIZE - 1), %ecx ++ andq $-VEC_SIZE, %rdi ++ ++# ifndef USE_AS_RAWMEMCHR ++ /* Adjust length. */ ++ addq %rcx, %rdx ++ ++ subq $(VEC_SIZE * 4), %rdx ++ jbe L(last_4x_vec_or_less) ++# endif ++ jmp L(more_4x_vec) ++ ++ .p2align 4 ++L(cros_page_boundary): ++ andl $(VEC_SIZE - 1), %ecx ++# ifdef USE_AS_WMEMCHR ++ /* NB: Divide shift count by 4 since each bit in K1 represent 4 ++ bytes. */ ++ movl %ecx, %SHIFT_REG ++ sarl $2, %SHIFT_REG ++# endif ++ andq $-VEC_SIZE, %rdi ++ VPCMP $0, (%rdi), %YMMMATCH, %k1 ++ kmovd %k1, %eax ++ /* Remove the leading bytes. */ ++ sarxl %SHIFT_REG, %eax, %eax ++ testl %eax, %eax ++ jz L(aligned_more) ++ tzcntl %eax, %eax ++# ifdef USE_AS_WMEMCHR ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ sall $2, %eax ++# endif ++# ifndef USE_AS_RAWMEMCHR ++ /* Check the end of data. */ ++ cmpq %rax, %rdx ++ jbe L(zero) ++# endif ++ addq %rdi, %rax ++ addq %rcx, %rax ++ ret ++ ++ .p2align 4 ++L(aligned_more): ++# ifndef USE_AS_RAWMEMCHR ++ /* Calculate "rdx + rcx - VEC_SIZE" with "rdx - (VEC_SIZE - rcx)" ++ instead of "(rdx + rcx) - VEC_SIZE" to void possible addition ++ overflow. */ ++ negq %rcx ++ addq $VEC_SIZE, %rcx ++ ++ /* Check the end of data. */ ++ subq %rcx, %rdx ++ jbe L(zero) ++# endif ++ ++ addq $VEC_SIZE, %rdi ++ ++# ifndef USE_AS_RAWMEMCHR ++ subq $(VEC_SIZE * 4), %rdx ++ jbe L(last_4x_vec_or_less) ++# endif ++ ++L(more_4x_vec): ++ /* Check the first 4 * VEC_SIZE. Only one VEC_SIZE at a time ++ since data is only aligned to VEC_SIZE. */ ++ VPCMP $0, (%rdi), %YMMMATCH, %k1 ++ kmovd %k1, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x0) ++ ++ VPCMP $0, VEC_SIZE(%rdi), %YMMMATCH, %k1 ++ kmovd %k1, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x1) ++ ++ VPCMP $0, (VEC_SIZE * 2)(%rdi), %YMMMATCH, %k1 ++ kmovd %k1, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x2) ++ ++ VPCMP $0, (VEC_SIZE * 3)(%rdi), %YMMMATCH, %k1 ++ kmovd %k1, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x3) ++ ++ addq $(VEC_SIZE * 4), %rdi ++ ++# ifndef USE_AS_RAWMEMCHR ++ subq $(VEC_SIZE * 4), %rdx ++ jbe L(last_4x_vec_or_less) ++# endif ++ ++ /* Align data to 4 * VEC_SIZE. */ ++ movq %rdi, %rcx ++ andl $(4 * VEC_SIZE - 1), %ecx ++ andq $-(4 * VEC_SIZE), %rdi ++ ++# ifndef USE_AS_RAWMEMCHR ++ /* Adjust length. */ ++ addq %rcx, %rdx ++# endif ++ ++ .p2align 4 ++L(loop_4x_vec): ++ /* Compare 4 * VEC at a time forward. */ ++ VPCMP $0, (%rdi), %YMMMATCH, %k1 ++ VPCMP $0, VEC_SIZE(%rdi), %YMMMATCH, %k2 ++ kord %k1, %k2, %k5 ++ VPCMP $0, (VEC_SIZE * 2)(%rdi), %YMMMATCH, %k3 ++ VPCMP $0, (VEC_SIZE * 3)(%rdi), %YMMMATCH, %k4 ++ ++ kord %k3, %k4, %k6 ++ kortestd %k5, %k6 ++ jnz L(4x_vec_end) ++ ++ addq $(VEC_SIZE * 4), %rdi ++ ++# ifdef USE_AS_RAWMEMCHR ++ jmp L(loop_4x_vec) ++# else ++ subq $(VEC_SIZE * 4), %rdx ++ ja L(loop_4x_vec) ++ ++L(last_4x_vec_or_less): ++ /* Less than 4 * VEC and aligned to VEC_SIZE. */ ++ addl $(VEC_SIZE * 2), %edx ++ jle L(last_2x_vec) ++ ++ VPCMP $0, (%rdi), %YMMMATCH, %k1 ++ kmovd %k1, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x0) ++ ++ VPCMP $0, VEC_SIZE(%rdi), %YMMMATCH, %k1 ++ kmovd %k1, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x1) ++ ++ VPCMP $0, (VEC_SIZE * 2)(%rdi), %YMMMATCH, %k1 ++ kmovd %k1, %eax ++ testl %eax, %eax ++ ++ jnz L(first_vec_x2_check) ++ subl $VEC_SIZE, %edx ++ jle L(zero) ++ ++ VPCMP $0, (VEC_SIZE * 3)(%rdi), %YMMMATCH, %k1 ++ kmovd %k1, %eax ++ testl %eax, %eax ++ ++ jnz L(first_vec_x3_check) ++ xorl %eax, %eax ++ ret ++ ++ .p2align 4 ++L(last_2x_vec): ++ addl $(VEC_SIZE * 2), %edx ++ VPCMP $0, (%rdi), %YMMMATCH, %k1 ++ kmovd %k1, %eax ++ testl %eax, %eax ++ ++ jnz L(first_vec_x0_check) ++ subl $VEC_SIZE, %edx ++ jle L(zero) ++ ++ VPCMP $0, VEC_SIZE(%rdi), %YMMMATCH, %k1 ++ kmovd %k1, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x1_check) ++ xorl %eax, %eax ++ ret ++ ++ .p2align 4 ++L(first_vec_x0_check): ++ tzcntl %eax, %eax ++# ifdef USE_AS_WMEMCHR ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ sall $2, %eax ++# endif ++ /* Check the end of data. */ ++ cmpq %rax, %rdx ++ jbe L(zero) ++ addq %rdi, %rax ++ ret ++ ++ .p2align 4 ++L(first_vec_x1_check): ++ tzcntl %eax, %eax ++# ifdef USE_AS_WMEMCHR ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ sall $2, %eax ++# endif ++ /* Check the end of data. */ ++ cmpq %rax, %rdx ++ jbe L(zero) ++ addq $VEC_SIZE, %rax ++ addq %rdi, %rax ++ ret ++ ++ .p2align 4 ++L(first_vec_x2_check): ++ tzcntl %eax, %eax ++# ifdef USE_AS_WMEMCHR ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ sall $2, %eax ++# endif ++ /* Check the end of data. */ ++ cmpq %rax, %rdx ++ jbe L(zero) ++ addq $(VEC_SIZE * 2), %rax ++ addq %rdi, %rax ++ ret ++ ++ .p2align 4 ++L(first_vec_x3_check): ++ tzcntl %eax, %eax ++# ifdef USE_AS_WMEMCHR ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ sall $2, %eax ++# endif ++ /* Check the end of data. */ ++ cmpq %rax, %rdx ++ jbe L(zero) ++ addq $(VEC_SIZE * 3), %rax ++ addq %rdi, %rax ++ ret ++ ++ .p2align 4 ++L(zero): ++ xorl %eax, %eax ++ ret ++# endif ++ ++ .p2align 4 ++L(first_vec_x0): ++ tzcntl %eax, %eax ++# ifdef USE_AS_WMEMCHR ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ leaq (%rdi, %rax, 4), %rax ++# else ++ addq %rdi, %rax ++# endif ++ ret ++ ++ .p2align 4 ++L(first_vec_x1): ++ tzcntl %eax, %eax ++# ifdef USE_AS_WMEMCHR ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ leaq VEC_SIZE(%rdi, %rax, 4), %rax ++# else ++ addq $VEC_SIZE, %rax ++ addq %rdi, %rax ++# endif ++ ret ++ ++ .p2align 4 ++L(first_vec_x2): ++ tzcntl %eax, %eax ++# ifdef USE_AS_WMEMCHR ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ leaq (VEC_SIZE * 2)(%rdi, %rax, 4), %rax ++# else ++ addq $(VEC_SIZE * 2), %rax ++ addq %rdi, %rax ++# endif ++ ret ++ ++ .p2align 4 ++L(4x_vec_end): ++ kmovd %k1, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x0) ++ kmovd %k2, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x1) ++ kmovd %k3, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x2) ++ kmovd %k4, %eax ++ testl %eax, %eax ++L(first_vec_x3): ++ tzcntl %eax, %eax ++# ifdef USE_AS_WMEMCHR ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ leaq (VEC_SIZE * 3)(%rdi, %rax, 4), %rax ++# else ++ addq $(VEC_SIZE * 3), %rax ++ addq %rdi, %rax ++# endif ++ ret ++ ++END (MEMCHR) ++#endif +diff --git a/sysdeps/x86_64/multiarch/memrchr-evex.S b/sysdeps/x86_64/multiarch/memrchr-evex.S +new file mode 100644 +index 00000000..16bf8e02 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/memrchr-evex.S +@@ -0,0 +1,337 @@ ++/* memrchr optimized with 256-bit EVEX instructions. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#if IS_IN (libc) ++ ++# include ++ ++# define VMOVA vmovdqa64 ++ ++# define YMMMATCH ymm16 ++ ++# define VEC_SIZE 32 ++ ++ .section .text.evex,"ax",@progbits ++ENTRY (__memrchr_evex) ++ /* Broadcast CHAR to YMMMATCH. */ ++ vpbroadcastb %esi, %YMMMATCH ++ ++ sub $VEC_SIZE, %RDX_LP ++ jbe L(last_vec_or_less) ++ ++ add %RDX_LP, %RDI_LP ++ ++ /* Check the last VEC_SIZE bytes. */ ++ vpcmpb $0, (%rdi), %YMMMATCH, %k1 ++ kmovd %k1, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x0) ++ ++ subq $(VEC_SIZE * 4), %rdi ++ movl %edi, %ecx ++ andl $(VEC_SIZE - 1), %ecx ++ jz L(aligned_more) ++ ++ /* Align data for aligned loads in the loop. */ ++ addq $VEC_SIZE, %rdi ++ addq $VEC_SIZE, %rdx ++ andq $-VEC_SIZE, %rdi ++ subq %rcx, %rdx ++ ++ .p2align 4 ++L(aligned_more): ++ subq $(VEC_SIZE * 4), %rdx ++ jbe L(last_4x_vec_or_less) ++ ++ /* Check the last 4 * VEC_SIZE. Only one VEC_SIZE at a time ++ since data is only aligned to VEC_SIZE. */ ++ vpcmpb $0, (VEC_SIZE * 3)(%rdi), %YMMMATCH, %k1 ++ kmovd %k1, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x3) ++ ++ vpcmpb $0, (VEC_SIZE * 2)(%rdi), %YMMMATCH, %k2 ++ kmovd %k2, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x2) ++ ++ vpcmpb $0, VEC_SIZE(%rdi), %YMMMATCH, %k3 ++ kmovd %k3, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x1) ++ ++ vpcmpb $0, (%rdi), %YMMMATCH, %k4 ++ kmovd %k4, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x0) ++ ++ /* Align data to 4 * VEC_SIZE for loop with fewer branches. ++ There are some overlaps with above if data isn't aligned ++ to 4 * VEC_SIZE. */ ++ movl %edi, %ecx ++ andl $(VEC_SIZE * 4 - 1), %ecx ++ jz L(loop_4x_vec) ++ ++ addq $(VEC_SIZE * 4), %rdi ++ addq $(VEC_SIZE * 4), %rdx ++ andq $-(VEC_SIZE * 4), %rdi ++ subq %rcx, %rdx ++ ++ .p2align 4 ++L(loop_4x_vec): ++ /* Compare 4 * VEC at a time forward. */ ++ subq $(VEC_SIZE * 4), %rdi ++ subq $(VEC_SIZE * 4), %rdx ++ jbe L(last_4x_vec_or_less) ++ ++ vpcmpb $0, (%rdi), %YMMMATCH, %k1 ++ vpcmpb $0, VEC_SIZE(%rdi), %YMMMATCH, %k2 ++ kord %k1, %k2, %k5 ++ vpcmpb $0, (VEC_SIZE * 2)(%rdi), %YMMMATCH, %k3 ++ vpcmpb $0, (VEC_SIZE * 3)(%rdi), %YMMMATCH, %k4 ++ ++ kord %k3, %k4, %k6 ++ kortestd %k5, %k6 ++ jz L(loop_4x_vec) ++ ++ /* There is a match. */ ++ kmovd %k4, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x3) ++ ++ kmovd %k3, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x2) ++ ++ kmovd %k2, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x1) ++ ++ kmovd %k1, %eax ++ bsrl %eax, %eax ++ addq %rdi, %rax ++ ret ++ ++ .p2align 4 ++L(last_4x_vec_or_less): ++ addl $(VEC_SIZE * 4), %edx ++ cmpl $(VEC_SIZE * 2), %edx ++ jbe L(last_2x_vec) ++ ++ vpcmpb $0, (VEC_SIZE * 3)(%rdi), %YMMMATCH, %k1 ++ kmovd %k1, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x3) ++ ++ vpcmpb $0, (VEC_SIZE * 2)(%rdi), %YMMMATCH, %k2 ++ kmovd %k2, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x2) ++ ++ vpcmpb $0, VEC_SIZE(%rdi), %YMMMATCH, %k3 ++ kmovd %k3, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x1_check) ++ cmpl $(VEC_SIZE * 3), %edx ++ jbe L(zero) ++ ++ vpcmpb $0, (%rdi), %YMMMATCH, %k4 ++ kmovd %k4, %eax ++ testl %eax, %eax ++ jz L(zero) ++ bsrl %eax, %eax ++ subq $(VEC_SIZE * 4), %rdx ++ addq %rax, %rdx ++ jl L(zero) ++ addq %rdi, %rax ++ ret ++ ++ .p2align 4 ++L(last_2x_vec): ++ vpcmpb $0, (VEC_SIZE * 3)(%rdi), %YMMMATCH, %k1 ++ kmovd %k1, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x3_check) ++ cmpl $VEC_SIZE, %edx ++ jbe L(zero) ++ ++ vpcmpb $0, (VEC_SIZE * 2)(%rdi), %YMMMATCH, %k1 ++ kmovd %k1, %eax ++ testl %eax, %eax ++ jz L(zero) ++ bsrl %eax, %eax ++ subq $(VEC_SIZE * 2), %rdx ++ addq %rax, %rdx ++ jl L(zero) ++ addl $(VEC_SIZE * 2), %eax ++ addq %rdi, %rax ++ ret ++ ++ .p2align 4 ++L(last_vec_x0): ++ bsrl %eax, %eax ++ addq %rdi, %rax ++ ret ++ ++ .p2align 4 ++L(last_vec_x1): ++ bsrl %eax, %eax ++ addl $VEC_SIZE, %eax ++ addq %rdi, %rax ++ ret ++ ++ .p2align 4 ++L(last_vec_x2): ++ bsrl %eax, %eax ++ addl $(VEC_SIZE * 2), %eax ++ addq %rdi, %rax ++ ret ++ ++ .p2align 4 ++L(last_vec_x3): ++ bsrl %eax, %eax ++ addl $(VEC_SIZE * 3), %eax ++ addq %rdi, %rax ++ ret ++ ++ .p2align 4 ++L(last_vec_x1_check): ++ bsrl %eax, %eax ++ subq $(VEC_SIZE * 3), %rdx ++ addq %rax, %rdx ++ jl L(zero) ++ addl $VEC_SIZE, %eax ++ addq %rdi, %rax ++ ret ++ ++ .p2align 4 ++L(last_vec_x3_check): ++ bsrl %eax, %eax ++ subq $VEC_SIZE, %rdx ++ addq %rax, %rdx ++ jl L(zero) ++ addl $(VEC_SIZE * 3), %eax ++ addq %rdi, %rax ++ ret ++ ++ .p2align 4 ++L(zero): ++ xorl %eax, %eax ++ ret ++ ++ .p2align 4 ++L(last_vec_or_less_aligned): ++ movl %edx, %ecx ++ ++ vpcmpb $0, (%rdi), %YMMMATCH, %k1 ++ ++ movl $1, %edx ++ /* Support rdx << 32. */ ++ salq %cl, %rdx ++ subq $1, %rdx ++ ++ kmovd %k1, %eax ++ ++ /* Remove the trailing bytes. */ ++ andl %edx, %eax ++ testl %eax, %eax ++ jz L(zero) ++ ++ bsrl %eax, %eax ++ addq %rdi, %rax ++ ret ++ ++ .p2align 4 ++L(last_vec_or_less): ++ addl $VEC_SIZE, %edx ++ ++ /* Check for zero length. */ ++ testl %edx, %edx ++ jz L(zero) ++ ++ movl %edi, %ecx ++ andl $(VEC_SIZE - 1), %ecx ++ jz L(last_vec_or_less_aligned) ++ ++ movl %ecx, %esi ++ movl %ecx, %r8d ++ addl %edx, %esi ++ andq $-VEC_SIZE, %rdi ++ ++ subl $VEC_SIZE, %esi ++ ja L(last_vec_2x_aligned) ++ ++ /* Check the last VEC. */ ++ vpcmpb $0, (%rdi), %YMMMATCH, %k1 ++ kmovd %k1, %eax ++ ++ /* Remove the leading and trailing bytes. */ ++ sarl %cl, %eax ++ movl %edx, %ecx ++ ++ movl $1, %edx ++ sall %cl, %edx ++ subl $1, %edx ++ ++ andl %edx, %eax ++ testl %eax, %eax ++ jz L(zero) ++ ++ bsrl %eax, %eax ++ addq %rdi, %rax ++ addq %r8, %rax ++ ret ++ ++ .p2align 4 ++L(last_vec_2x_aligned): ++ movl %esi, %ecx ++ ++ /* Check the last VEC. */ ++ vpcmpb $0, VEC_SIZE(%rdi), %YMMMATCH, %k1 ++ ++ movl $1, %edx ++ sall %cl, %edx ++ subl $1, %edx ++ ++ kmovd %k1, %eax ++ ++ /* Remove the trailing bytes. */ ++ andl %edx, %eax ++ ++ testl %eax, %eax ++ jnz L(last_vec_x1) ++ ++ /* Check the second last VEC. */ ++ vpcmpb $0, (%rdi), %YMMMATCH, %k1 ++ ++ movl %r8d, %ecx ++ ++ kmovd %k1, %eax ++ ++ /* Remove the leading bytes. Must use unsigned right shift for ++ bsrl below. */ ++ shrl %cl, %eax ++ testl %eax, %eax ++ jz L(zero) ++ ++ bsrl %eax, %eax ++ addq %rdi, %rax ++ addq %r8, %rax ++ ret ++END (__memrchr_evex) ++#endif +diff --git a/sysdeps/x86_64/multiarch/rawmemchr-evex.S b/sysdeps/x86_64/multiarch/rawmemchr-evex.S +new file mode 100644 +index 00000000..ec942b77 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/rawmemchr-evex.S +@@ -0,0 +1,4 @@ ++#define MEMCHR __rawmemchr_evex ++#define USE_AS_RAWMEMCHR 1 ++ ++#include "memchr-evex.S" +diff --git a/sysdeps/x86_64/multiarch/strchr-evex.S b/sysdeps/x86_64/multiarch/strchr-evex.S +new file mode 100644 +index 00000000..ddc86a70 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strchr-evex.S +@@ -0,0 +1,335 @@ ++/* strchr/strchrnul optimized with 256-bit EVEX instructions. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#if IS_IN (libc) ++ ++# include ++ ++# ifndef STRCHR ++# define STRCHR __strchr_evex ++# endif ++ ++# define VMOVU vmovdqu64 ++# define VMOVA vmovdqa64 ++ ++# ifdef USE_AS_WCSCHR ++# define VPBROADCAST vpbroadcastd ++# define VPCMP vpcmpd ++# define VPMINU vpminud ++# define CHAR_REG esi ++# define SHIFT_REG r8d ++# else ++# define VPBROADCAST vpbroadcastb ++# define VPCMP vpcmpb ++# define VPMINU vpminub ++# define CHAR_REG sil ++# define SHIFT_REG ecx ++# endif ++ ++# define XMMZERO xmm16 ++ ++# define YMMZERO ymm16 ++# define YMM0 ymm17 ++# define YMM1 ymm18 ++# define YMM2 ymm19 ++# define YMM3 ymm20 ++# define YMM4 ymm21 ++# define YMM5 ymm22 ++# define YMM6 ymm23 ++# define YMM7 ymm24 ++# define YMM8 ymm25 ++ ++# define VEC_SIZE 32 ++# define PAGE_SIZE 4096 ++ ++ .section .text.evex,"ax",@progbits ++ENTRY (STRCHR) ++ movl %edi, %ecx ++# ifndef USE_AS_STRCHRNUL ++ xorl %edx, %edx ++# endif ++ ++ /* Broadcast CHAR to YMM0. */ ++ VPBROADCAST %esi, %YMM0 ++ ++ vpxorq %XMMZERO, %XMMZERO, %XMMZERO ++ ++ /* Check if we cross page boundary with one vector load. */ ++ andl $(PAGE_SIZE - 1), %ecx ++ cmpl $(PAGE_SIZE - VEC_SIZE), %ecx ++ ja L(cross_page_boundary) ++ ++ /* Check the first VEC_SIZE bytes. Search for both CHAR and the ++ null bytes. */ ++ VMOVU (%rdi), %YMM1 ++ ++ /* Leaves only CHARS matching esi as 0. */ ++ vpxorq %YMM1, %YMM0, %YMM2 ++ VPMINU %YMM2, %YMM1, %YMM2 ++ /* Each bit in K0 represents a CHAR or a null byte in YMM1. */ ++ VPCMP $0, %YMMZERO, %YMM2, %k0 ++ ktestd %k0, %k0 ++ jz L(more_vecs) ++ kmovd %k0, %eax ++ tzcntl %eax, %eax ++ /* Found CHAR or the null byte. */ ++# ifdef USE_AS_WCSCHR ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ leaq (%rdi, %rax, 4), %rax ++# else ++ addq %rdi, %rax ++# endif ++# ifndef USE_AS_STRCHRNUL ++ cmp (%rax), %CHAR_REG ++ cmovne %rdx, %rax ++# endif ++ ret ++ ++ .p2align 4 ++L(more_vecs): ++ /* Align data for aligned loads in the loop. */ ++ andq $-VEC_SIZE, %rdi ++L(aligned_more): ++ ++ /* Check the next 4 * VEC_SIZE. Only one VEC_SIZE at a time ++ since data is only aligned to VEC_SIZE. */ ++ VMOVA VEC_SIZE(%rdi), %YMM1 ++ addq $VEC_SIZE, %rdi ++ ++ /* Leaves only CHARS matching esi as 0. */ ++ vpxorq %YMM1, %YMM0, %YMM2 ++ VPMINU %YMM2, %YMM1, %YMM2 ++ /* Each bit in K0 represents a CHAR or a null byte in YMM1. */ ++ VPCMP $0, %YMMZERO, %YMM2, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x0) ++ ++ VMOVA VEC_SIZE(%rdi), %YMM1 ++ /* Leaves only CHARS matching esi as 0. */ ++ vpxorq %YMM1, %YMM0, %YMM2 ++ VPMINU %YMM2, %YMM1, %YMM2 ++ /* Each bit in K0 represents a CHAR or a null byte in YMM1. */ ++ VPCMP $0, %YMMZERO, %YMM2, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x1) ++ ++ VMOVA (VEC_SIZE * 2)(%rdi), %YMM1 ++ /* Leaves only CHARS matching esi as 0. */ ++ vpxorq %YMM1, %YMM0, %YMM2 ++ VPMINU %YMM2, %YMM1, %YMM2 ++ /* Each bit in K0 represents a CHAR or a null byte in YMM1. */ ++ VPCMP $0, %YMMZERO, %YMM2, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x2) ++ ++ VMOVA (VEC_SIZE * 3)(%rdi), %YMM1 ++ /* Leaves only CHARS matching esi as 0. */ ++ vpxorq %YMM1, %YMM0, %YMM2 ++ VPMINU %YMM2, %YMM1, %YMM2 ++ /* Each bit in K0 represents a CHAR or a null byte in YMM1. */ ++ VPCMP $0, %YMMZERO, %YMM2, %k0 ++ ktestd %k0, %k0 ++ jz L(prep_loop_4x) ++ ++ kmovd %k0, %eax ++ tzcntl %eax, %eax ++ /* Found CHAR or the null byte. */ ++# ifdef USE_AS_WCSCHR ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ leaq (VEC_SIZE * 3)(%rdi, %rax, 4), %rax ++# else ++ leaq (VEC_SIZE * 3)(%rdi, %rax), %rax ++# endif ++# ifndef USE_AS_STRCHRNUL ++ cmp (%rax), %CHAR_REG ++ cmovne %rdx, %rax ++# endif ++ ret ++ ++ .p2align 4 ++L(first_vec_x0): ++ tzcntl %eax, %eax ++ /* Found CHAR or the null byte. */ ++# ifdef USE_AS_WCSCHR ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ leaq (%rdi, %rax, 4), %rax ++# else ++ addq %rdi, %rax ++# endif ++# ifndef USE_AS_STRCHRNUL ++ cmp (%rax), %CHAR_REG ++ cmovne %rdx, %rax ++# endif ++ ret ++ ++ .p2align 4 ++L(first_vec_x1): ++ tzcntl %eax, %eax ++ /* Found CHAR or the null byte. */ ++# ifdef USE_AS_WCSCHR ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ leaq VEC_SIZE(%rdi, %rax, 4), %rax ++# else ++ leaq VEC_SIZE(%rdi, %rax), %rax ++# endif ++# ifndef USE_AS_STRCHRNUL ++ cmp (%rax), %CHAR_REG ++ cmovne %rdx, %rax ++# endif ++ ret ++ ++ .p2align 4 ++L(first_vec_x2): ++ tzcntl %eax, %eax ++ /* Found CHAR or the null byte. */ ++# ifdef USE_AS_WCSCHR ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ leaq (VEC_SIZE * 2)(%rdi, %rax, 4), %rax ++# else ++ leaq (VEC_SIZE * 2)(%rdi, %rax), %rax ++# endif ++# ifndef USE_AS_STRCHRNUL ++ cmp (%rax), %CHAR_REG ++ cmovne %rdx, %rax ++# endif ++ ret ++ ++L(prep_loop_4x): ++ /* Align data to 4 * VEC_SIZE. */ ++ andq $-(VEC_SIZE * 4), %rdi ++ ++ .p2align 4 ++L(loop_4x_vec): ++ /* Compare 4 * VEC at a time forward. */ ++ VMOVA (VEC_SIZE * 4)(%rdi), %YMM1 ++ VMOVA (VEC_SIZE * 5)(%rdi), %YMM2 ++ VMOVA (VEC_SIZE * 6)(%rdi), %YMM3 ++ VMOVA (VEC_SIZE * 7)(%rdi), %YMM4 ++ ++ /* Leaves only CHARS matching esi as 0. */ ++ vpxorq %YMM1, %YMM0, %YMM5 ++ vpxorq %YMM2, %YMM0, %YMM6 ++ vpxorq %YMM3, %YMM0, %YMM7 ++ vpxorq %YMM4, %YMM0, %YMM8 ++ ++ VPMINU %YMM5, %YMM1, %YMM5 ++ VPMINU %YMM6, %YMM2, %YMM6 ++ VPMINU %YMM7, %YMM3, %YMM7 ++ VPMINU %YMM8, %YMM4, %YMM8 ++ ++ VPMINU %YMM5, %YMM6, %YMM1 ++ VPMINU %YMM7, %YMM8, %YMM2 ++ ++ VPMINU %YMM1, %YMM2, %YMM1 ++ ++ /* Each bit in K0 represents a CHAR or a null byte. */ ++ VPCMP $0, %YMMZERO, %YMM1, %k0 ++ ++ addq $(VEC_SIZE * 4), %rdi ++ ++ ktestd %k0, %k0 ++ jz L(loop_4x_vec) ++ ++ /* Each bit in K0 represents a CHAR or a null byte in YMM1. */ ++ VPCMP $0, %YMMZERO, %YMM5, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x0) ++ ++ /* Each bit in K1 represents a CHAR or a null byte in YMM2. */ ++ VPCMP $0, %YMMZERO, %YMM6, %k1 ++ kmovd %k1, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x1) ++ ++ /* Each bit in K2 represents a CHAR or a null byte in YMM3. */ ++ VPCMP $0, %YMMZERO, %YMM7, %k2 ++ /* Each bit in K3 represents a CHAR or a null byte in YMM4. */ ++ VPCMP $0, %YMMZERO, %YMM8, %k3 ++ ++# ifdef USE_AS_WCSCHR ++ /* NB: Each bit in K2/K3 represents 4-byte element. */ ++ kshiftlw $8, %k3, %k1 ++# else ++ kshiftlq $32, %k3, %k1 ++# endif ++ ++ /* Each bit in K1 represents a NULL or a mismatch. */ ++ korq %k1, %k2, %k1 ++ kmovq %k1, %rax ++ ++ tzcntq %rax, %rax ++# ifdef USE_AS_WCSCHR ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ leaq (VEC_SIZE * 2)(%rdi, %rax, 4), %rax ++# else ++ leaq (VEC_SIZE * 2)(%rdi, %rax), %rax ++# endif ++# ifndef USE_AS_STRCHRNUL ++ cmp (%rax), %CHAR_REG ++ cmovne %rdx, %rax ++# endif ++ ret ++ ++ /* Cold case for crossing page with first load. */ ++ .p2align 4 ++L(cross_page_boundary): ++ andq $-VEC_SIZE, %rdi ++ andl $(VEC_SIZE - 1), %ecx ++ ++ VMOVA (%rdi), %YMM1 ++ ++ /* Leaves only CHARS matching esi as 0. */ ++ vpxorq %YMM1, %YMM0, %YMM2 ++ VPMINU %YMM2, %YMM1, %YMM2 ++ /* Each bit in K0 represents a CHAR or a null byte in YMM1. */ ++ VPCMP $0, %YMMZERO, %YMM2, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ ++# ifdef USE_AS_WCSCHR ++ /* NB: Divide shift count by 4 since each bit in K1 represent 4 ++ bytes. */ ++ movl %ecx, %SHIFT_REG ++ sarl $2, %SHIFT_REG ++# endif ++ ++ /* Remove the leading bits. */ ++ sarxl %SHIFT_REG, %eax, %eax ++ testl %eax, %eax ++ ++ jz L(aligned_more) ++ tzcntl %eax, %eax ++ addq %rcx, %rdi ++# ifdef USE_AS_WCSCHR ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ leaq (%rdi, %rax, 4), %rax ++# else ++ addq %rdi, %rax ++# endif ++# ifndef USE_AS_STRCHRNUL ++ cmp (%rax), %CHAR_REG ++ cmovne %rdx, %rax ++# endif ++ ret ++ ++END (STRCHR) ++# endif +diff --git a/sysdeps/x86_64/multiarch/strchr.c b/sysdeps/x86_64/multiarch/strchr.c +index 32954713..be05e197 100644 +--- a/sysdeps/x86_64/multiarch/strchr.c ++++ b/sysdeps/x86_64/multiarch/strchr.c +@@ -29,16 +29,24 @@ + extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_no_bsf) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden; + + static inline void * + IFUNC_SELECTOR (void) + { + const struct cpu_features* cpu_features = __get_cpu_features (); + +- if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER) +- && CPU_FEATURE_USABLE_P (cpu_features, AVX2) ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX2) + && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) +- return OPTIMIZE (avx2); ++ { ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) ++ && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW) ++ && CPU_FEATURE_USABLE_P (cpu_features, BMI2)) ++ return OPTIMIZE (evex); ++ ++ if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) ++ return OPTIMIZE (avx2); ++ } + + if (CPU_FEATURES_ARCH_P (cpu_features, Slow_BSF)) + return OPTIMIZE (sse2_no_bsf); +diff --git a/sysdeps/x86_64/multiarch/strchrnul-evex.S b/sysdeps/x86_64/multiarch/strchrnul-evex.S +new file mode 100644 +index 00000000..064fe7ca +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strchrnul-evex.S +@@ -0,0 +1,3 @@ ++#define STRCHR __strchrnul_evex ++#define USE_AS_STRCHRNUL 1 ++#include "strchr-evex.S" +diff --git a/sysdeps/x86_64/multiarch/strcmp-evex.S b/sysdeps/x86_64/multiarch/strcmp-evex.S +new file mode 100644 +index 00000000..459eeed0 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strcmp-evex.S +@@ -0,0 +1,1043 @@ ++/* strcmp/wcscmp/strncmp/wcsncmp optimized with 256-bit EVEX instructions. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#if IS_IN (libc) ++ ++# include ++ ++# ifndef STRCMP ++# define STRCMP __strcmp_evex ++# endif ++ ++# define PAGE_SIZE 4096 ++ ++/* VEC_SIZE = Number of bytes in a ymm register */ ++# define VEC_SIZE 32 ++ ++/* Shift for dividing by (VEC_SIZE * 4). */ ++# define DIVIDE_BY_VEC_4_SHIFT 7 ++# if (VEC_SIZE * 4) != (1 << DIVIDE_BY_VEC_4_SHIFT) ++# error (VEC_SIZE * 4) != (1 << DIVIDE_BY_VEC_4_SHIFT) ++# endif ++ ++# define VMOVU vmovdqu64 ++# define VMOVA vmovdqa64 ++ ++# ifdef USE_AS_WCSCMP ++/* Compare packed dwords. */ ++# define VPCMP vpcmpd ++# define SHIFT_REG32 r8d ++# define SHIFT_REG64 r8 ++/* 1 dword char == 4 bytes. */ ++# define SIZE_OF_CHAR 4 ++# else ++/* Compare packed bytes. */ ++# define VPCMP vpcmpb ++# define SHIFT_REG32 ecx ++# define SHIFT_REG64 rcx ++/* 1 byte char == 1 byte. */ ++# define SIZE_OF_CHAR 1 ++# endif ++ ++# define XMMZERO xmm16 ++# define XMM0 xmm17 ++# define XMM1 xmm18 ++ ++# define YMMZERO ymm16 ++# define YMM0 ymm17 ++# define YMM1 ymm18 ++# define YMM2 ymm19 ++# define YMM3 ymm20 ++# define YMM4 ymm21 ++# define YMM5 ymm22 ++# define YMM6 ymm23 ++# define YMM7 ymm24 ++ ++/* Warning! ++ wcscmp/wcsncmp have to use SIGNED comparison for elements. ++ strcmp/strncmp have to use UNSIGNED comparison for elements. ++*/ ++ ++/* The main idea of the string comparison (byte or dword) using 256-bit ++ EVEX instructions consists of comparing (VPCMP) two ymm vectors. The ++ latter can be on either packed bytes or dwords depending on ++ USE_AS_WCSCMP. In order to check the null char, algorithm keeps the ++ matched bytes/dwords, requiring 5 EVEX instructions (3 VPCMP and 2 ++ KORD). In general, the costs of comparing VEC_SIZE bytes (32-bytes) ++ are 3 VPCMP and 2 KORD instructions, together with VMOVU and ktestd ++ instructions. Main loop (away from from page boundary) compares 4 ++ vectors are a time, effectively comparing 4 x VEC_SIZE bytes (128 ++ bytes) on each loop. ++ ++ The routine strncmp/wcsncmp (enabled by defining USE_AS_STRNCMP) logic ++ is the same as strcmp, except that an a maximum offset is tracked. If ++ the maximum offset is reached before a difference is found, zero is ++ returned. */ ++ ++ .section .text.evex,"ax",@progbits ++ENTRY (STRCMP) ++# ifdef USE_AS_STRNCMP ++ /* Check for simple cases (0 or 1) in offset. */ ++ cmp $1, %RDX_LP ++ je L(char0) ++ jb L(zero) ++# ifdef USE_AS_WCSCMP ++ /* Convert units: from wide to byte char. */ ++ shl $2, %RDX_LP ++# endif ++ /* Register %r11 tracks the maximum offset. */ ++ mov %RDX_LP, %R11_LP ++# endif ++ movl %edi, %eax ++ xorl %edx, %edx ++ /* Make %XMMZERO (%YMMZERO) all zeros in this function. */ ++ vpxorq %XMMZERO, %XMMZERO, %XMMZERO ++ orl %esi, %eax ++ andl $(PAGE_SIZE - 1), %eax ++ cmpl $(PAGE_SIZE - (VEC_SIZE * 4)), %eax ++ jg L(cross_page) ++ /* Start comparing 4 vectors. */ ++ VMOVU (%rdi), %YMM0 ++ VMOVU (%rsi), %YMM1 ++ ++ /* Each bit in K0 represents a mismatch in YMM0 and YMM1. */ ++ VPCMP $4, %YMM0, %YMM1, %k0 ++ ++ /* Check for NULL in YMM0. */ ++ VPCMP $0, %YMMZERO, %YMM0, %k1 ++ /* Check for NULL in YMM1. */ ++ VPCMP $0, %YMMZERO, %YMM1, %k2 ++ /* Each bit in K1 represents a NULL in YMM0 or YMM1. */ ++ kord %k1, %k2, %k1 ++ ++ /* Each bit in K1 represents: ++ 1. A mismatch in YMM0 and YMM1. Or ++ 2. A NULL in YMM0 or YMM1. ++ */ ++ kord %k0, %k1, %k1 ++ ++ ktestd %k1, %k1 ++ je L(next_3_vectors) ++ kmovd %k1, %ecx ++ tzcntl %ecx, %edx ++# ifdef USE_AS_WCSCMP ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ sall $2, %edx ++# endif ++# ifdef USE_AS_STRNCMP ++ /* Return 0 if the mismatched index (%rdx) is after the maximum ++ offset (%r11). */ ++ cmpq %r11, %rdx ++ jae L(zero) ++# endif ++# ifdef USE_AS_WCSCMP ++ xorl %eax, %eax ++ movl (%rdi, %rdx), %ecx ++ cmpl (%rsi, %rdx), %ecx ++ je L(return) ++L(wcscmp_return): ++ setl %al ++ negl %eax ++ orl $1, %eax ++L(return): ++# else ++ movzbl (%rdi, %rdx), %eax ++ movzbl (%rsi, %rdx), %edx ++ subl %edx, %eax ++# endif ++ ret ++ ++ .p2align 4 ++L(return_vec_size): ++ kmovd %k1, %ecx ++ tzcntl %ecx, %edx ++# ifdef USE_AS_WCSCMP ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ sall $2, %edx ++# endif ++# ifdef USE_AS_STRNCMP ++ /* Return 0 if the mismatched index (%rdx + VEC_SIZE) is after ++ the maximum offset (%r11). */ ++ addq $VEC_SIZE, %rdx ++ cmpq %r11, %rdx ++ jae L(zero) ++# ifdef USE_AS_WCSCMP ++ xorl %eax, %eax ++ movl (%rdi, %rdx), %ecx ++ cmpl (%rsi, %rdx), %ecx ++ jne L(wcscmp_return) ++# else ++ movzbl (%rdi, %rdx), %eax ++ movzbl (%rsi, %rdx), %edx ++ subl %edx, %eax ++# endif ++# else ++# ifdef USE_AS_WCSCMP ++ xorl %eax, %eax ++ movl VEC_SIZE(%rdi, %rdx), %ecx ++ cmpl VEC_SIZE(%rsi, %rdx), %ecx ++ jne L(wcscmp_return) ++# else ++ movzbl VEC_SIZE(%rdi, %rdx), %eax ++ movzbl VEC_SIZE(%rsi, %rdx), %edx ++ subl %edx, %eax ++# endif ++# endif ++ ret ++ ++ .p2align 4 ++L(return_2_vec_size): ++ kmovd %k1, %ecx ++ tzcntl %ecx, %edx ++# ifdef USE_AS_WCSCMP ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ sall $2, %edx ++# endif ++# ifdef USE_AS_STRNCMP ++ /* Return 0 if the mismatched index (%rdx + 2 * VEC_SIZE) is ++ after the maximum offset (%r11). */ ++ addq $(VEC_SIZE * 2), %rdx ++ cmpq %r11, %rdx ++ jae L(zero) ++# ifdef USE_AS_WCSCMP ++ xorl %eax, %eax ++ movl (%rdi, %rdx), %ecx ++ cmpl (%rsi, %rdx), %ecx ++ jne L(wcscmp_return) ++# else ++ movzbl (%rdi, %rdx), %eax ++ movzbl (%rsi, %rdx), %edx ++ subl %edx, %eax ++# endif ++# else ++# ifdef USE_AS_WCSCMP ++ xorl %eax, %eax ++ movl (VEC_SIZE * 2)(%rdi, %rdx), %ecx ++ cmpl (VEC_SIZE * 2)(%rsi, %rdx), %ecx ++ jne L(wcscmp_return) ++# else ++ movzbl (VEC_SIZE * 2)(%rdi, %rdx), %eax ++ movzbl (VEC_SIZE * 2)(%rsi, %rdx), %edx ++ subl %edx, %eax ++# endif ++# endif ++ ret ++ ++ .p2align 4 ++L(return_3_vec_size): ++ kmovd %k1, %ecx ++ tzcntl %ecx, %edx ++# ifdef USE_AS_WCSCMP ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ sall $2, %edx ++# endif ++# ifdef USE_AS_STRNCMP ++ /* Return 0 if the mismatched index (%rdx + 3 * VEC_SIZE) is ++ after the maximum offset (%r11). */ ++ addq $(VEC_SIZE * 3), %rdx ++ cmpq %r11, %rdx ++ jae L(zero) ++# ifdef USE_AS_WCSCMP ++ xorl %eax, %eax ++ movl (%rdi, %rdx), %ecx ++ cmpl (%rsi, %rdx), %ecx ++ jne L(wcscmp_return) ++# else ++ movzbl (%rdi, %rdx), %eax ++ movzbl (%rsi, %rdx), %edx ++ subl %edx, %eax ++# endif ++# else ++# ifdef USE_AS_WCSCMP ++ xorl %eax, %eax ++ movl (VEC_SIZE * 3)(%rdi, %rdx), %ecx ++ cmpl (VEC_SIZE * 3)(%rsi, %rdx), %ecx ++ jne L(wcscmp_return) ++# else ++ movzbl (VEC_SIZE * 3)(%rdi, %rdx), %eax ++ movzbl (VEC_SIZE * 3)(%rsi, %rdx), %edx ++ subl %edx, %eax ++# endif ++# endif ++ ret ++ ++ .p2align 4 ++L(next_3_vectors): ++ VMOVU VEC_SIZE(%rdi), %YMM0 ++ VMOVU VEC_SIZE(%rsi), %YMM1 ++ /* Each bit in K0 represents a mismatch in YMM0 and YMM1. */ ++ VPCMP $4, %YMM0, %YMM1, %k0 ++ VPCMP $0, %YMMZERO, %YMM0, %k1 ++ VPCMP $0, %YMMZERO, %YMM1, %k2 ++ /* Each bit in K1 represents a NULL in YMM0 or YMM1. */ ++ kord %k1, %k2, %k1 ++ /* Each bit in K1 represents a NULL or a mismatch. */ ++ kord %k0, %k1, %k1 ++ ktestd %k1, %k1 ++ jne L(return_vec_size) ++ ++ VMOVU (VEC_SIZE * 2)(%rdi), %YMM2 ++ VMOVU (VEC_SIZE * 3)(%rdi), %YMM3 ++ VMOVU (VEC_SIZE * 2)(%rsi), %YMM4 ++ VMOVU (VEC_SIZE * 3)(%rsi), %YMM5 ++ ++ /* Each bit in K0 represents a mismatch in YMM2 and YMM4. */ ++ VPCMP $4, %YMM2, %YMM4, %k0 ++ VPCMP $0, %YMMZERO, %YMM2, %k1 ++ VPCMP $0, %YMMZERO, %YMM4, %k2 ++ /* Each bit in K1 represents a NULL in YMM2 or YMM4. */ ++ kord %k1, %k2, %k1 ++ /* Each bit in K1 represents a NULL or a mismatch. */ ++ kord %k0, %k1, %k1 ++ ktestd %k1, %k1 ++ jne L(return_2_vec_size) ++ ++ /* Each bit in K0 represents a mismatch in YMM3 and YMM5. */ ++ VPCMP $4, %YMM3, %YMM5, %k0 ++ VPCMP $0, %YMMZERO, %YMM3, %k1 ++ VPCMP $0, %YMMZERO, %YMM5, %k2 ++ /* Each bit in K1 represents a NULL in YMM3 or YMM5. */ ++ kord %k1, %k2, %k1 ++ /* Each bit in K1 represents a NULL or a mismatch. */ ++ kord %k0, %k1, %k1 ++ ktestd %k1, %k1 ++ jne L(return_3_vec_size) ++L(main_loop_header): ++ leaq (VEC_SIZE * 4)(%rdi), %rdx ++ movl $PAGE_SIZE, %ecx ++ /* Align load via RAX. */ ++ andq $-(VEC_SIZE * 4), %rdx ++ subq %rdi, %rdx ++ leaq (%rdi, %rdx), %rax ++# ifdef USE_AS_STRNCMP ++ /* Starting from this point, the maximum offset, or simply the ++ 'offset', DECREASES by the same amount when base pointers are ++ moved forward. Return 0 when: ++ 1) On match: offset <= the matched vector index. ++ 2) On mistmach, offset is before the mistmatched index. ++ */ ++ subq %rdx, %r11 ++ jbe L(zero) ++# endif ++ addq %rsi, %rdx ++ movq %rdx, %rsi ++ andl $(PAGE_SIZE - 1), %esi ++ /* Number of bytes before page crossing. */ ++ subq %rsi, %rcx ++ /* Number of VEC_SIZE * 4 blocks before page crossing. */ ++ shrq $DIVIDE_BY_VEC_4_SHIFT, %rcx ++ /* ESI: Number of VEC_SIZE * 4 blocks before page crossing. */ ++ movl %ecx, %esi ++ jmp L(loop_start) ++ ++ .p2align 4 ++L(loop): ++# ifdef USE_AS_STRNCMP ++ /* Base pointers are moved forward by 4 * VEC_SIZE. Decrease ++ the maximum offset (%r11) by the same amount. */ ++ subq $(VEC_SIZE * 4), %r11 ++ jbe L(zero) ++# endif ++ addq $(VEC_SIZE * 4), %rax ++ addq $(VEC_SIZE * 4), %rdx ++L(loop_start): ++ testl %esi, %esi ++ leal -1(%esi), %esi ++ je L(loop_cross_page) ++L(back_to_loop): ++ /* Main loop, comparing 4 vectors are a time. */ ++ VMOVA (%rax), %YMM0 ++ VMOVA VEC_SIZE(%rax), %YMM2 ++ VMOVA (VEC_SIZE * 2)(%rax), %YMM4 ++ VMOVA (VEC_SIZE * 3)(%rax), %YMM6 ++ VMOVU (%rdx), %YMM1 ++ VMOVU VEC_SIZE(%rdx), %YMM3 ++ VMOVU (VEC_SIZE * 2)(%rdx), %YMM5 ++ VMOVU (VEC_SIZE * 3)(%rdx), %YMM7 ++ ++ VPCMP $4, %YMM0, %YMM1, %k0 ++ VPCMP $0, %YMMZERO, %YMM0, %k1 ++ VPCMP $0, %YMMZERO, %YMM1, %k2 ++ kord %k1, %k2, %k1 ++ /* Each bit in K4 represents a NULL or a mismatch in YMM0 and ++ YMM1. */ ++ kord %k0, %k1, %k4 ++ ++ VPCMP $4, %YMM2, %YMM3, %k0 ++ VPCMP $0, %YMMZERO, %YMM2, %k1 ++ VPCMP $0, %YMMZERO, %YMM3, %k2 ++ kord %k1, %k2, %k1 ++ /* Each bit in K5 represents a NULL or a mismatch in YMM2 and ++ YMM3. */ ++ kord %k0, %k1, %k5 ++ ++ VPCMP $4, %YMM4, %YMM5, %k0 ++ VPCMP $0, %YMMZERO, %YMM4, %k1 ++ VPCMP $0, %YMMZERO, %YMM5, %k2 ++ kord %k1, %k2, %k1 ++ /* Each bit in K6 represents a NULL or a mismatch in YMM4 and ++ YMM5. */ ++ kord %k0, %k1, %k6 ++ ++ VPCMP $4, %YMM6, %YMM7, %k0 ++ VPCMP $0, %YMMZERO, %YMM6, %k1 ++ VPCMP $0, %YMMZERO, %YMM7, %k2 ++ kord %k1, %k2, %k1 ++ /* Each bit in K7 represents a NULL or a mismatch in YMM6 and ++ YMM7. */ ++ kord %k0, %k1, %k7 ++ ++ kord %k4, %k5, %k0 ++ kord %k6, %k7, %k1 ++ ++ /* Test each mask (32 bits) individually because for VEC_SIZE ++ == 32 is not possible to OR the four masks and keep all bits ++ in a 64-bit integer register, differing from SSE2 strcmp ++ where ORing is possible. */ ++ kortestd %k0, %k1 ++ je L(loop) ++ ktestd %k4, %k4 ++ je L(test_vec) ++ kmovd %k4, %edi ++ tzcntl %edi, %ecx ++# ifdef USE_AS_WCSCMP ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ sall $2, %ecx ++# endif ++# ifdef USE_AS_STRNCMP ++ cmpq %rcx, %r11 ++ jbe L(zero) ++# ifdef USE_AS_WCSCMP ++ movq %rax, %rsi ++ xorl %eax, %eax ++ movl (%rsi, %rcx), %edi ++ cmpl (%rdx, %rcx), %edi ++ jne L(wcscmp_return) ++# else ++ movzbl (%rax, %rcx), %eax ++ movzbl (%rdx, %rcx), %edx ++ subl %edx, %eax ++# endif ++# else ++# ifdef USE_AS_WCSCMP ++ movq %rax, %rsi ++ xorl %eax, %eax ++ movl (%rsi, %rcx), %edi ++ cmpl (%rdx, %rcx), %edi ++ jne L(wcscmp_return) ++# else ++ movzbl (%rax, %rcx), %eax ++ movzbl (%rdx, %rcx), %edx ++ subl %edx, %eax ++# endif ++# endif ++ ret ++ ++ .p2align 4 ++L(test_vec): ++# ifdef USE_AS_STRNCMP ++ /* The first vector matched. Return 0 if the maximum offset ++ (%r11) <= VEC_SIZE. */ ++ cmpq $VEC_SIZE, %r11 ++ jbe L(zero) ++# endif ++ ktestd %k5, %k5 ++ je L(test_2_vec) ++ kmovd %k5, %ecx ++ tzcntl %ecx, %edi ++# ifdef USE_AS_WCSCMP ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ sall $2, %edi ++# endif ++# ifdef USE_AS_STRNCMP ++ addq $VEC_SIZE, %rdi ++ cmpq %rdi, %r11 ++ jbe L(zero) ++# ifdef USE_AS_WCSCMP ++ movq %rax, %rsi ++ xorl %eax, %eax ++ movl (%rsi, %rdi), %ecx ++ cmpl (%rdx, %rdi), %ecx ++ jne L(wcscmp_return) ++# else ++ movzbl (%rax, %rdi), %eax ++ movzbl (%rdx, %rdi), %edx ++ subl %edx, %eax ++# endif ++# else ++# ifdef USE_AS_WCSCMP ++ movq %rax, %rsi ++ xorl %eax, %eax ++ movl VEC_SIZE(%rsi, %rdi), %ecx ++ cmpl VEC_SIZE(%rdx, %rdi), %ecx ++ jne L(wcscmp_return) ++# else ++ movzbl VEC_SIZE(%rax, %rdi), %eax ++ movzbl VEC_SIZE(%rdx, %rdi), %edx ++ subl %edx, %eax ++# endif ++# endif ++ ret ++ ++ .p2align 4 ++L(test_2_vec): ++# ifdef USE_AS_STRNCMP ++ /* The first 2 vectors matched. Return 0 if the maximum offset ++ (%r11) <= 2 * VEC_SIZE. */ ++ cmpq $(VEC_SIZE * 2), %r11 ++ jbe L(zero) ++# endif ++ ktestd %k6, %k6 ++ je L(test_3_vec) ++ kmovd %k6, %ecx ++ tzcntl %ecx, %edi ++# ifdef USE_AS_WCSCMP ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ sall $2, %edi ++# endif ++# ifdef USE_AS_STRNCMP ++ addq $(VEC_SIZE * 2), %rdi ++ cmpq %rdi, %r11 ++ jbe L(zero) ++# ifdef USE_AS_WCSCMP ++ movq %rax, %rsi ++ xorl %eax, %eax ++ movl (%rsi, %rdi), %ecx ++ cmpl (%rdx, %rdi), %ecx ++ jne L(wcscmp_return) ++# else ++ movzbl (%rax, %rdi), %eax ++ movzbl (%rdx, %rdi), %edx ++ subl %edx, %eax ++# endif ++# else ++# ifdef USE_AS_WCSCMP ++ movq %rax, %rsi ++ xorl %eax, %eax ++ movl (VEC_SIZE * 2)(%rsi, %rdi), %ecx ++ cmpl (VEC_SIZE * 2)(%rdx, %rdi), %ecx ++ jne L(wcscmp_return) ++# else ++ movzbl (VEC_SIZE * 2)(%rax, %rdi), %eax ++ movzbl (VEC_SIZE * 2)(%rdx, %rdi), %edx ++ subl %edx, %eax ++# endif ++# endif ++ ret ++ ++ .p2align 4 ++L(test_3_vec): ++# ifdef USE_AS_STRNCMP ++ /* The first 3 vectors matched. Return 0 if the maximum offset ++ (%r11) <= 3 * VEC_SIZE. */ ++ cmpq $(VEC_SIZE * 3), %r11 ++ jbe L(zero) ++# endif ++ kmovd %k7, %esi ++ tzcntl %esi, %ecx ++# ifdef USE_AS_WCSCMP ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ sall $2, %ecx ++# endif ++# ifdef USE_AS_STRNCMP ++ addq $(VEC_SIZE * 3), %rcx ++ cmpq %rcx, %r11 ++ jbe L(zero) ++# ifdef USE_AS_WCSCMP ++ movq %rax, %rsi ++ xorl %eax, %eax ++ movl (%rsi, %rcx), %esi ++ cmpl (%rdx, %rcx), %esi ++ jne L(wcscmp_return) ++# else ++ movzbl (%rax, %rcx), %eax ++ movzbl (%rdx, %rcx), %edx ++ subl %edx, %eax ++# endif ++# else ++# ifdef USE_AS_WCSCMP ++ movq %rax, %rsi ++ xorl %eax, %eax ++ movl (VEC_SIZE * 3)(%rsi, %rcx), %esi ++ cmpl (VEC_SIZE * 3)(%rdx, %rcx), %esi ++ jne L(wcscmp_return) ++# else ++ movzbl (VEC_SIZE * 3)(%rax, %rcx), %eax ++ movzbl (VEC_SIZE * 3)(%rdx, %rcx), %edx ++ subl %edx, %eax ++# endif ++# endif ++ ret ++ ++ .p2align 4 ++L(loop_cross_page): ++ xorl %r10d, %r10d ++ movq %rdx, %rcx ++ /* Align load via RDX. We load the extra ECX bytes which should ++ be ignored. */ ++ andl $((VEC_SIZE * 4) - 1), %ecx ++ /* R10 is -RCX. */ ++ subq %rcx, %r10 ++ ++ /* This works only if VEC_SIZE * 2 == 64. */ ++# if (VEC_SIZE * 2) != 64 ++# error (VEC_SIZE * 2) != 64 ++# endif ++ ++ /* Check if the first VEC_SIZE * 2 bytes should be ignored. */ ++ cmpl $(VEC_SIZE * 2), %ecx ++ jge L(loop_cross_page_2_vec) ++ ++ VMOVU (%rax, %r10), %YMM2 ++ VMOVU VEC_SIZE(%rax, %r10), %YMM3 ++ VMOVU (%rdx, %r10), %YMM4 ++ VMOVU VEC_SIZE(%rdx, %r10), %YMM5 ++ ++ VPCMP $4, %YMM4, %YMM2, %k0 ++ VPCMP $0, %YMMZERO, %YMM2, %k1 ++ VPCMP $0, %YMMZERO, %YMM4, %k2 ++ kord %k1, %k2, %k1 ++ /* Each bit in K1 represents a NULL or a mismatch in YMM2 and ++ YMM4. */ ++ kord %k0, %k1, %k1 ++ ++ VPCMP $4, %YMM5, %YMM3, %k3 ++ VPCMP $0, %YMMZERO, %YMM3, %k4 ++ VPCMP $0, %YMMZERO, %YMM5, %k5 ++ kord %k4, %k5, %k4 ++ /* Each bit in K3 represents a NULL or a mismatch in YMM3 and ++ YMM5. */ ++ kord %k3, %k4, %k3 ++ ++# ifdef USE_AS_WCSCMP ++ /* NB: Each bit in K1/K3 represents 4-byte element. */ ++ kshiftlw $8, %k3, %k2 ++ /* NB: Divide shift count by 4 since each bit in K1 represent 4 ++ bytes. */ ++ movl %ecx, %SHIFT_REG32 ++ sarl $2, %SHIFT_REG32 ++# else ++ kshiftlq $32, %k3, %k2 ++# endif ++ ++ /* Each bit in K1 represents a NULL or a mismatch. */ ++ korq %k1, %k2, %k1 ++ kmovq %k1, %rdi ++ ++ /* Since ECX < VEC_SIZE * 2, simply skip the first ECX bytes. */ ++ shrxq %SHIFT_REG64, %rdi, %rdi ++ testq %rdi, %rdi ++ je L(loop_cross_page_2_vec) ++ tzcntq %rdi, %rcx ++# ifdef USE_AS_WCSCMP ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ sall $2, %ecx ++# endif ++# ifdef USE_AS_STRNCMP ++ cmpq %rcx, %r11 ++ jbe L(zero) ++# ifdef USE_AS_WCSCMP ++ movq %rax, %rsi ++ xorl %eax, %eax ++ movl (%rsi, %rcx), %edi ++ cmpl (%rdx, %rcx), %edi ++ jne L(wcscmp_return) ++# else ++ movzbl (%rax, %rcx), %eax ++ movzbl (%rdx, %rcx), %edx ++ subl %edx, %eax ++# endif ++# else ++# ifdef USE_AS_WCSCMP ++ movq %rax, %rsi ++ xorl %eax, %eax ++ movl (%rsi, %rcx), %edi ++ cmpl (%rdx, %rcx), %edi ++ jne L(wcscmp_return) ++# else ++ movzbl (%rax, %rcx), %eax ++ movzbl (%rdx, %rcx), %edx ++ subl %edx, %eax ++# endif ++# endif ++ ret ++ ++ .p2align 4 ++L(loop_cross_page_2_vec): ++ /* The first VEC_SIZE * 2 bytes match or are ignored. */ ++ VMOVU (VEC_SIZE * 2)(%rax, %r10), %YMM0 ++ VMOVU (VEC_SIZE * 3)(%rax, %r10), %YMM1 ++ VMOVU (VEC_SIZE * 2)(%rdx, %r10), %YMM2 ++ VMOVU (VEC_SIZE * 3)(%rdx, %r10), %YMM3 ++ ++ VPCMP $4, %YMM0, %YMM2, %k0 ++ VPCMP $0, %YMMZERO, %YMM0, %k1 ++ VPCMP $0, %YMMZERO, %YMM2, %k2 ++ kord %k1, %k2, %k1 ++ /* Each bit in K1 represents a NULL or a mismatch in YMM0 and ++ YMM2. */ ++ kord %k0, %k1, %k1 ++ ++ VPCMP $4, %YMM1, %YMM3, %k3 ++ VPCMP $0, %YMMZERO, %YMM1, %k4 ++ VPCMP $0, %YMMZERO, %YMM3, %k5 ++ kord %k4, %k5, %k4 ++ /* Each bit in K3 represents a NULL or a mismatch in YMM1 and ++ YMM3. */ ++ kord %k3, %k4, %k3 ++ ++# ifdef USE_AS_WCSCMP ++ /* NB: Each bit in K1/K3 represents 4-byte element. */ ++ kshiftlw $8, %k3, %k2 ++# else ++ kshiftlq $32, %k3, %k2 ++# endif ++ ++ /* Each bit in K1 represents a NULL or a mismatch. */ ++ korq %k1, %k2, %k1 ++ kmovq %k1, %rdi ++ ++ xorl %r8d, %r8d ++ /* If ECX > VEC_SIZE * 2, skip ECX - (VEC_SIZE * 2) bytes. */ ++ subl $(VEC_SIZE * 2), %ecx ++ jle 1f ++ /* R8 has number of bytes skipped. */ ++ movl %ecx, %r8d ++# ifdef USE_AS_WCSCMP ++ /* NB: Divide shift count by 4 since each bit in K1 represent 4 ++ bytes. */ ++ sarl $2, %ecx ++# endif ++ /* Skip ECX bytes. */ ++ shrq %cl, %rdi ++1: ++ /* Before jumping back to the loop, set ESI to the number of ++ VEC_SIZE * 4 blocks before page crossing. */ ++ movl $(PAGE_SIZE / (VEC_SIZE * 4) - 1), %esi ++ ++ testq %rdi, %rdi ++# ifdef USE_AS_STRNCMP ++ /* At this point, if %rdi value is 0, it already tested ++ VEC_SIZE*4+%r10 byte starting from %rax. This label ++ checks whether strncmp maximum offset reached or not. */ ++ je L(string_nbyte_offset_check) ++# else ++ je L(back_to_loop) ++# endif ++ tzcntq %rdi, %rcx ++# ifdef USE_AS_WCSCMP ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ sall $2, %ecx ++# endif ++ addq %r10, %rcx ++ /* Adjust for number of bytes skipped. */ ++ addq %r8, %rcx ++# ifdef USE_AS_STRNCMP ++ addq $(VEC_SIZE * 2), %rcx ++ subq %rcx, %r11 ++ jbe L(zero) ++# ifdef USE_AS_WCSCMP ++ movq %rax, %rsi ++ xorl %eax, %eax ++ movl (%rsi, %rcx), %edi ++ cmpl (%rdx, %rcx), %edi ++ jne L(wcscmp_return) ++# else ++ movzbl (%rax, %rcx), %eax ++ movzbl (%rdx, %rcx), %edx ++ subl %edx, %eax ++# endif ++# else ++# ifdef USE_AS_WCSCMP ++ movq %rax, %rsi ++ xorl %eax, %eax ++ movl (VEC_SIZE * 2)(%rsi, %rcx), %edi ++ cmpl (VEC_SIZE * 2)(%rdx, %rcx), %edi ++ jne L(wcscmp_return) ++# else ++ movzbl (VEC_SIZE * 2)(%rax, %rcx), %eax ++ movzbl (VEC_SIZE * 2)(%rdx, %rcx), %edx ++ subl %edx, %eax ++# endif ++# endif ++ ret ++ ++# ifdef USE_AS_STRNCMP ++L(string_nbyte_offset_check): ++ leaq (VEC_SIZE * 4)(%r10), %r10 ++ cmpq %r10, %r11 ++ jbe L(zero) ++ jmp L(back_to_loop) ++# endif ++ ++ .p2align 4 ++L(cross_page_loop): ++ /* Check one byte/dword at a time. */ ++# ifdef USE_AS_WCSCMP ++ cmpl %ecx, %eax ++# else ++ subl %ecx, %eax ++# endif ++ jne L(different) ++ addl $SIZE_OF_CHAR, %edx ++ cmpl $(VEC_SIZE * 4), %edx ++ je L(main_loop_header) ++# ifdef USE_AS_STRNCMP ++ cmpq %r11, %rdx ++ jae L(zero) ++# endif ++# ifdef USE_AS_WCSCMP ++ movl (%rdi, %rdx), %eax ++ movl (%rsi, %rdx), %ecx ++# else ++ movzbl (%rdi, %rdx), %eax ++ movzbl (%rsi, %rdx), %ecx ++# endif ++ /* Check null char. */ ++ testl %eax, %eax ++ jne L(cross_page_loop) ++ /* Since %eax == 0, subtract is OK for both SIGNED and UNSIGNED ++ comparisons. */ ++ subl %ecx, %eax ++# ifndef USE_AS_WCSCMP ++L(different): ++# endif ++ ret ++ ++# ifdef USE_AS_WCSCMP ++ .p2align 4 ++L(different): ++ /* Use movl to avoid modifying EFLAGS. */ ++ movl $0, %eax ++ setl %al ++ negl %eax ++ orl $1, %eax ++ ret ++# endif ++ ++# ifdef USE_AS_STRNCMP ++ .p2align 4 ++L(zero): ++ xorl %eax, %eax ++ ret ++ ++ .p2align 4 ++L(char0): ++# ifdef USE_AS_WCSCMP ++ xorl %eax, %eax ++ movl (%rdi), %ecx ++ cmpl (%rsi), %ecx ++ jne L(wcscmp_return) ++# else ++ movzbl (%rsi), %ecx ++ movzbl (%rdi), %eax ++ subl %ecx, %eax ++# endif ++ ret ++# endif ++ ++ .p2align 4 ++L(last_vector): ++ addq %rdx, %rdi ++ addq %rdx, %rsi ++# ifdef USE_AS_STRNCMP ++ subq %rdx, %r11 ++# endif ++ tzcntl %ecx, %edx ++# ifdef USE_AS_WCSCMP ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ sall $2, %edx ++# endif ++# ifdef USE_AS_STRNCMP ++ cmpq %r11, %rdx ++ jae L(zero) ++# endif ++# ifdef USE_AS_WCSCMP ++ xorl %eax, %eax ++ movl (%rdi, %rdx), %ecx ++ cmpl (%rsi, %rdx), %ecx ++ jne L(wcscmp_return) ++# else ++ movzbl (%rdi, %rdx), %eax ++ movzbl (%rsi, %rdx), %edx ++ subl %edx, %eax ++# endif ++ ret ++ ++ /* Comparing on page boundary region requires special treatment: ++ It must done one vector at the time, starting with the wider ++ ymm vector if possible, if not, with xmm. If fetching 16 bytes ++ (xmm) still passes the boundary, byte comparison must be done. ++ */ ++ .p2align 4 ++L(cross_page): ++ /* Try one ymm vector at a time. */ ++ cmpl $(PAGE_SIZE - VEC_SIZE), %eax ++ jg L(cross_page_1_vector) ++L(loop_1_vector): ++ VMOVU (%rdi, %rdx), %YMM0 ++ VMOVU (%rsi, %rdx), %YMM1 ++ ++ /* Each bit in K0 represents a mismatch in YMM0 and YMM1. */ ++ VPCMP $4, %YMM0, %YMM1, %k0 ++ VPCMP $0, %YMMZERO, %YMM0, %k1 ++ VPCMP $0, %YMMZERO, %YMM1, %k2 ++ /* Each bit in K1 represents a NULL in YMM0 or YMM1. */ ++ kord %k1, %k2, %k1 ++ /* Each bit in K1 represents a NULL or a mismatch. */ ++ kord %k0, %k1, %k1 ++ kmovd %k1, %ecx ++ testl %ecx, %ecx ++ jne L(last_vector) ++ ++ addl $VEC_SIZE, %edx ++ ++ addl $VEC_SIZE, %eax ++# ifdef USE_AS_STRNCMP ++ /* Return 0 if the current offset (%rdx) >= the maximum offset ++ (%r11). */ ++ cmpq %r11, %rdx ++ jae L(zero) ++# endif ++ cmpl $(PAGE_SIZE - VEC_SIZE), %eax ++ jle L(loop_1_vector) ++L(cross_page_1_vector): ++ /* Less than 32 bytes to check, try one xmm vector. */ ++ cmpl $(PAGE_SIZE - 16), %eax ++ jg L(cross_page_1_xmm) ++ VMOVU (%rdi, %rdx), %XMM0 ++ VMOVU (%rsi, %rdx), %XMM1 ++ ++ /* Each bit in K0 represents a mismatch in XMM0 and XMM1. */ ++ VPCMP $4, %XMM0, %XMM1, %k0 ++ VPCMP $0, %XMMZERO, %XMM0, %k1 ++ VPCMP $0, %XMMZERO, %XMM1, %k2 ++ /* Each bit in K1 represents a NULL in XMM0 or XMM1. */ ++ korw %k1, %k2, %k1 ++ /* Each bit in K1 represents a NULL or a mismatch. */ ++ korw %k0, %k1, %k1 ++ kmovw %k1, %ecx ++ testl %ecx, %ecx ++ jne L(last_vector) ++ ++ addl $16, %edx ++# ifndef USE_AS_WCSCMP ++ addl $16, %eax ++# endif ++# ifdef USE_AS_STRNCMP ++ /* Return 0 if the current offset (%rdx) >= the maximum offset ++ (%r11). */ ++ cmpq %r11, %rdx ++ jae L(zero) ++# endif ++ ++L(cross_page_1_xmm): ++# ifndef USE_AS_WCSCMP ++ /* Less than 16 bytes to check, try 8 byte vector. NB: No need ++ for wcscmp nor wcsncmp since wide char is 4 bytes. */ ++ cmpl $(PAGE_SIZE - 8), %eax ++ jg L(cross_page_8bytes) ++ vmovq (%rdi, %rdx), %XMM0 ++ vmovq (%rsi, %rdx), %XMM1 ++ ++ /* Each bit in K0 represents a mismatch in XMM0 and XMM1. */ ++ VPCMP $4, %XMM0, %XMM1, %k0 ++ VPCMP $0, %XMMZERO, %XMM0, %k1 ++ VPCMP $0, %XMMZERO, %XMM1, %k2 ++ /* Each bit in K1 represents a NULL in XMM0 or XMM1. */ ++ kord %k1, %k2, %k1 ++ /* Each bit in K1 represents a NULL or a mismatch. */ ++ kord %k0, %k1, %k1 ++ kmovd %k1, %ecx ++ ++# ifdef USE_AS_WCSCMP ++ /* Only last 2 bits are valid. */ ++ andl $0x3, %ecx ++# else ++ /* Only last 8 bits are valid. */ ++ andl $0xff, %ecx ++# endif ++ ++ testl %ecx, %ecx ++ jne L(last_vector) ++ ++ addl $8, %edx ++ addl $8, %eax ++# ifdef USE_AS_STRNCMP ++ /* Return 0 if the current offset (%rdx) >= the maximum offset ++ (%r11). */ ++ cmpq %r11, %rdx ++ jae L(zero) ++# endif ++ ++L(cross_page_8bytes): ++ /* Less than 8 bytes to check, try 4 byte vector. */ ++ cmpl $(PAGE_SIZE - 4), %eax ++ jg L(cross_page_4bytes) ++ vmovd (%rdi, %rdx), %XMM0 ++ vmovd (%rsi, %rdx), %XMM1 ++ ++ /* Each bit in K0 represents a mismatch in XMM0 and XMM1. */ ++ VPCMP $4, %XMM0, %XMM1, %k0 ++ VPCMP $0, %XMMZERO, %XMM0, %k1 ++ VPCMP $0, %XMMZERO, %XMM1, %k2 ++ /* Each bit in K1 represents a NULL in XMM0 or XMM1. */ ++ kord %k1, %k2, %k1 ++ /* Each bit in K1 represents a NULL or a mismatch. */ ++ kord %k0, %k1, %k1 ++ kmovd %k1, %ecx ++ ++# ifdef USE_AS_WCSCMP ++ /* Only the last bit is valid. */ ++ andl $0x1, %ecx ++# else ++ /* Only last 4 bits are valid. */ ++ andl $0xf, %ecx ++# endif ++ ++ testl %ecx, %ecx ++ jne L(last_vector) ++ ++ addl $4, %edx ++# ifdef USE_AS_STRNCMP ++ /* Return 0 if the current offset (%rdx) >= the maximum offset ++ (%r11). */ ++ cmpq %r11, %rdx ++ jae L(zero) ++# endif ++ ++L(cross_page_4bytes): ++# endif ++ /* Less than 4 bytes to check, try one byte/dword at a time. */ ++# ifdef USE_AS_STRNCMP ++ cmpq %r11, %rdx ++ jae L(zero) ++# endif ++# ifdef USE_AS_WCSCMP ++ movl (%rdi, %rdx), %eax ++ movl (%rsi, %rdx), %ecx ++# else ++ movzbl (%rdi, %rdx), %eax ++ movzbl (%rsi, %rdx), %ecx ++# endif ++ testl %eax, %eax ++ jne L(cross_page_loop) ++ subl %ecx, %eax ++ ret ++END (STRCMP) ++#endif +diff --git a/sysdeps/x86_64/multiarch/strcmp.c b/sysdeps/x86_64/multiarch/strcmp.c +index 3f433fbc..c5f38510 100644 +--- a/sysdeps/x86_64/multiarch/strcmp.c ++++ b/sysdeps/x86_64/multiarch/strcmp.c +@@ -30,16 +30,25 @@ extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden; + + static inline void * + IFUNC_SELECTOR (void) + { + const struct cpu_features* cpu_features = __get_cpu_features (); + +- if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER) +- && CPU_FEATURE_USABLE_P (cpu_features, AVX2) ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX2) + && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) +- return OPTIMIZE (avx2); ++ { ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) ++ && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW) ++ && CPU_FEATURE_USABLE_P (cpu_features, BMI2) ++ && !CPU_FEATURES_ARCH_P (cpu_features, Prefer_AVX2_STRCMP)) ++ return OPTIMIZE (evex); ++ ++ if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) ++ return OPTIMIZE (avx2); ++ } + + if (CPU_FEATURES_ARCH_P (cpu_features, Fast_Unaligned_Load)) + return OPTIMIZE (sse2_unaligned); +diff --git a/sysdeps/x86_64/multiarch/strlen-evex.S b/sysdeps/x86_64/multiarch/strlen-evex.S +new file mode 100644 +index 00000000..cd022509 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strlen-evex.S +@@ -0,0 +1,436 @@ ++/* strlen/strnlen/wcslen/wcsnlen optimized with 256-bit EVEX instructions. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#if IS_IN (libc) ++ ++# include ++ ++# ifndef STRLEN ++# define STRLEN __strlen_evex ++# endif ++ ++# define VMOVA vmovdqa64 ++ ++# ifdef USE_AS_WCSLEN ++# define VPCMP vpcmpd ++# define VPMINU vpminud ++# define SHIFT_REG r9d ++# else ++# define VPCMP vpcmpb ++# define VPMINU vpminub ++# define SHIFT_REG ecx ++# endif ++ ++# define XMMZERO xmm16 ++# define YMMZERO ymm16 ++# define YMM1 ymm17 ++# define YMM2 ymm18 ++# define YMM3 ymm19 ++# define YMM4 ymm20 ++# define YMM5 ymm21 ++# define YMM6 ymm22 ++ ++# define VEC_SIZE 32 ++ ++ .section .text.evex,"ax",@progbits ++ENTRY (STRLEN) ++# ifdef USE_AS_STRNLEN ++ /* Check for zero length. */ ++ test %RSI_LP, %RSI_LP ++ jz L(zero) ++# ifdef USE_AS_WCSLEN ++ shl $2, %RSI_LP ++# elif defined __ILP32__ ++ /* Clear the upper 32 bits. */ ++ movl %esi, %esi ++# endif ++ mov %RSI_LP, %R8_LP ++# endif ++ movl %edi, %ecx ++ movq %rdi, %rdx ++ vpxorq %XMMZERO, %XMMZERO, %XMMZERO ++ ++ /* Check if we may cross page boundary with one vector load. */ ++ andl $(2 * VEC_SIZE - 1), %ecx ++ cmpl $VEC_SIZE, %ecx ++ ja L(cros_page_boundary) ++ ++ /* Check the first VEC_SIZE bytes. Each bit in K0 represents a ++ null byte. */ ++ VPCMP $0, (%rdi), %YMMZERO, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ ++# ifdef USE_AS_STRNLEN ++ jnz L(first_vec_x0_check) ++ /* Adjust length and check the end of data. */ ++ subq $VEC_SIZE, %rsi ++ jbe L(max) ++# else ++ jnz L(first_vec_x0) ++# endif ++ ++ /* Align data for aligned loads in the loop. */ ++ addq $VEC_SIZE, %rdi ++ andl $(VEC_SIZE - 1), %ecx ++ andq $-VEC_SIZE, %rdi ++ ++# ifdef USE_AS_STRNLEN ++ /* Adjust length. */ ++ addq %rcx, %rsi ++ ++ subq $(VEC_SIZE * 4), %rsi ++ jbe L(last_4x_vec_or_less) ++# endif ++ jmp L(more_4x_vec) ++ ++ .p2align 4 ++L(cros_page_boundary): ++ andl $(VEC_SIZE - 1), %ecx ++ andq $-VEC_SIZE, %rdi ++ ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide shift count by 4 since each bit in K0 represent 4 ++ bytes. */ ++ movl %ecx, %SHIFT_REG ++ sarl $2, %SHIFT_REG ++# endif ++ VPCMP $0, (%rdi), %YMMZERO, %k0 ++ kmovd %k0, %eax ++ ++ /* Remove the leading bytes. */ ++ sarxl %SHIFT_REG, %eax, %eax ++ testl %eax, %eax ++ jz L(aligned_more) ++ tzcntl %eax, %eax ++# ifdef USE_AS_WCSLEN ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ sall $2, %eax ++# endif ++# ifdef USE_AS_STRNLEN ++ /* Check the end of data. */ ++ cmpq %rax, %rsi ++ jbe L(max) ++# endif ++ addq %rdi, %rax ++ addq %rcx, %rax ++ subq %rdx, %rax ++# ifdef USE_AS_WCSLEN ++ shrq $2, %rax ++# endif ++ ret ++ ++ .p2align 4 ++L(aligned_more): ++# ifdef USE_AS_STRNLEN ++ /* "rcx" is less than VEC_SIZE. Calculate "rdx + rcx - VEC_SIZE" ++ with "rdx - (VEC_SIZE - rcx)" instead of "(rdx + rcx) - VEC_SIZE" ++ to void possible addition overflow. */ ++ negq %rcx ++ addq $VEC_SIZE, %rcx ++ ++ /* Check the end of data. */ ++ subq %rcx, %rsi ++ jbe L(max) ++# endif ++ ++ addq $VEC_SIZE, %rdi ++ ++# ifdef USE_AS_STRNLEN ++ subq $(VEC_SIZE * 4), %rsi ++ jbe L(last_4x_vec_or_less) ++# endif ++ ++L(more_4x_vec): ++ /* Check the first 4 * VEC_SIZE. Only one VEC_SIZE at a time ++ since data is only aligned to VEC_SIZE. */ ++ VPCMP $0, (%rdi), %YMMZERO, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x0) ++ ++ VPCMP $0, VEC_SIZE(%rdi), %YMMZERO, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x1) ++ ++ VPCMP $0, (VEC_SIZE * 2)(%rdi), %YMMZERO, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x2) ++ ++ VPCMP $0, (VEC_SIZE * 3)(%rdi), %YMMZERO, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x3) ++ ++ addq $(VEC_SIZE * 4), %rdi ++ ++# ifdef USE_AS_STRNLEN ++ subq $(VEC_SIZE * 4), %rsi ++ jbe L(last_4x_vec_or_less) ++# endif ++ ++ /* Align data to 4 * VEC_SIZE. */ ++ movq %rdi, %rcx ++ andl $(4 * VEC_SIZE - 1), %ecx ++ andq $-(4 * VEC_SIZE), %rdi ++ ++# ifdef USE_AS_STRNLEN ++ /* Adjust length. */ ++ addq %rcx, %rsi ++# endif ++ ++ .p2align 4 ++L(loop_4x_vec): ++ /* Compare 4 * VEC at a time forward. */ ++ VMOVA (%rdi), %YMM1 ++ VMOVA VEC_SIZE(%rdi), %YMM2 ++ VMOVA (VEC_SIZE * 2)(%rdi), %YMM3 ++ VMOVA (VEC_SIZE * 3)(%rdi), %YMM4 ++ ++ VPMINU %YMM1, %YMM2, %YMM5 ++ VPMINU %YMM3, %YMM4, %YMM6 ++ ++ VPMINU %YMM5, %YMM6, %YMM5 ++ VPCMP $0, %YMM5, %YMMZERO, %k0 ++ ktestd %k0, %k0 ++ jnz L(4x_vec_end) ++ ++ addq $(VEC_SIZE * 4), %rdi ++ ++# ifndef USE_AS_STRNLEN ++ jmp L(loop_4x_vec) ++# else ++ subq $(VEC_SIZE * 4), %rsi ++ ja L(loop_4x_vec) ++ ++L(last_4x_vec_or_less): ++ /* Less than 4 * VEC and aligned to VEC_SIZE. */ ++ addl $(VEC_SIZE * 2), %esi ++ jle L(last_2x_vec) ++ ++ VPCMP $0, (%rdi), %YMMZERO, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x0) ++ ++ VPCMP $0, VEC_SIZE(%rdi), %YMMZERO, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x1) ++ ++ VPCMP $0, (VEC_SIZE * 2)(%rdi), %YMMZERO, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x2_check) ++ subl $VEC_SIZE, %esi ++ jle L(max) ++ ++ VPCMP $0, (VEC_SIZE * 3)(%rdi), %YMMZERO, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x3_check) ++ movq %r8, %rax ++# ifdef USE_AS_WCSLEN ++ shrq $2, %rax ++# endif ++ ret ++ ++ .p2align 4 ++L(last_2x_vec): ++ addl $(VEC_SIZE * 2), %esi ++ ++ VPCMP $0, (%rdi), %YMMZERO, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x0_check) ++ subl $VEC_SIZE, %esi ++ jle L(max) ++ ++ VPCMP $0, VEC_SIZE(%rdi), %YMMZERO, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x1_check) ++ movq %r8, %rax ++# ifdef USE_AS_WCSLEN ++ shrq $2, %rax ++# endif ++ ret ++ ++ .p2align 4 ++L(first_vec_x0_check): ++ tzcntl %eax, %eax ++# ifdef USE_AS_WCSLEN ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ sall $2, %eax ++# endif ++ /* Check the end of data. */ ++ cmpq %rax, %rsi ++ jbe L(max) ++ addq %rdi, %rax ++ subq %rdx, %rax ++# ifdef USE_AS_WCSLEN ++ shrq $2, %rax ++# endif ++ ret ++ ++ .p2align 4 ++L(first_vec_x1_check): ++ tzcntl %eax, %eax ++# ifdef USE_AS_WCSLEN ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ sall $2, %eax ++# endif ++ /* Check the end of data. */ ++ cmpq %rax, %rsi ++ jbe L(max) ++ addq $VEC_SIZE, %rax ++ addq %rdi, %rax ++ subq %rdx, %rax ++# ifdef USE_AS_WCSLEN ++ shrq $2, %rax ++# endif ++ ret ++ ++ .p2align 4 ++L(first_vec_x2_check): ++ tzcntl %eax, %eax ++# ifdef USE_AS_WCSLEN ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ sall $2, %eax ++# endif ++ /* Check the end of data. */ ++ cmpq %rax, %rsi ++ jbe L(max) ++ addq $(VEC_SIZE * 2), %rax ++ addq %rdi, %rax ++ subq %rdx, %rax ++# ifdef USE_AS_WCSLEN ++ shrq $2, %rax ++# endif ++ ret ++ ++ .p2align 4 ++L(first_vec_x3_check): ++ tzcntl %eax, %eax ++# ifdef USE_AS_WCSLEN ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ sall $2, %eax ++# endif ++ /* Check the end of data. */ ++ cmpq %rax, %rsi ++ jbe L(max) ++ addq $(VEC_SIZE * 3), %rax ++ addq %rdi, %rax ++ subq %rdx, %rax ++# ifdef USE_AS_WCSLEN ++ shrq $2, %rax ++# endif ++ ret ++ ++ .p2align 4 ++L(max): ++ movq %r8, %rax ++# ifdef USE_AS_WCSLEN ++ shrq $2, %rax ++# endif ++ ret ++ ++ .p2align 4 ++L(zero): ++ xorl %eax, %eax ++ ret ++# endif ++ ++ .p2align 4 ++L(first_vec_x0): ++ tzcntl %eax, %eax ++# ifdef USE_AS_WCSLEN ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ sall $2, %eax ++# endif ++ addq %rdi, %rax ++ subq %rdx, %rax ++# ifdef USE_AS_WCSLEN ++ shrq $2, %rax ++# endif ++ ret ++ ++ .p2align 4 ++L(first_vec_x1): ++ tzcntl %eax, %eax ++# ifdef USE_AS_WCSLEN ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ sall $2, %eax ++# endif ++ addq $VEC_SIZE, %rax ++ addq %rdi, %rax ++ subq %rdx, %rax ++# ifdef USE_AS_WCSLEN ++ shrq $2, %rax ++# endif ++ ret ++ ++ .p2align 4 ++L(first_vec_x2): ++ tzcntl %eax, %eax ++# ifdef USE_AS_WCSLEN ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ sall $2, %eax ++# endif ++ addq $(VEC_SIZE * 2), %rax ++ addq %rdi, %rax ++ subq %rdx, %rax ++# ifdef USE_AS_WCSLEN ++ shrq $2, %rax ++# endif ++ ret ++ ++ .p2align 4 ++L(4x_vec_end): ++ VPCMP $0, %YMM1, %YMMZERO, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x0) ++ VPCMP $0, %YMM2, %YMMZERO, %k1 ++ kmovd %k1, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x1) ++ VPCMP $0, %YMM3, %YMMZERO, %k2 ++ kmovd %k2, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x2) ++ VPCMP $0, %YMM4, %YMMZERO, %k3 ++ kmovd %k3, %eax ++L(first_vec_x3): ++ tzcntl %eax, %eax ++# ifdef USE_AS_WCSLEN ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ sall $2, %eax ++# endif ++ addq $(VEC_SIZE * 3), %rax ++ addq %rdi, %rax ++ subq %rdx, %rax ++# ifdef USE_AS_WCSLEN ++ shrq $2, %rax ++# endif ++ ret ++ ++END (STRLEN) ++#endif +diff --git a/sysdeps/x86_64/multiarch/strncmp-evex.S b/sysdeps/x86_64/multiarch/strncmp-evex.S +new file mode 100644 +index 00000000..a1d53e8c +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strncmp-evex.S +@@ -0,0 +1,3 @@ ++#define STRCMP __strncmp_evex ++#define USE_AS_STRNCMP 1 ++#include "strcmp-evex.S" +diff --git a/sysdeps/x86_64/multiarch/strncmp.c b/sysdeps/x86_64/multiarch/strncmp.c +index 686d654f..4c15542f 100644 +--- a/sysdeps/x86_64/multiarch/strncmp.c ++++ b/sysdeps/x86_64/multiarch/strncmp.c +@@ -30,16 +30,25 @@ extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (sse42) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden; + + static inline void * + IFUNC_SELECTOR (void) + { + const struct cpu_features* cpu_features = __get_cpu_features (); + +- if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER) +- && CPU_FEATURE_USABLE_P (cpu_features, AVX2) ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX2) + && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) +- return OPTIMIZE (avx2); ++ { ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) ++ && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW) ++ && CPU_FEATURE_USABLE_P (cpu_features, BMI2) ++ && !CPU_FEATURES_ARCH_P (cpu_features, Prefer_AVX2_STRCMP)) ++ return OPTIMIZE (evex); ++ ++ if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) ++ return OPTIMIZE (avx2); ++ } + + if (CPU_FEATURE_USABLE_P (cpu_features, SSE4_2) + && !CPU_FEATURES_ARCH_P (cpu_features, Slow_SSE4_2)) +diff --git a/sysdeps/x86_64/multiarch/strnlen-evex.S b/sysdeps/x86_64/multiarch/strnlen-evex.S +new file mode 100644 +index 00000000..722022f3 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strnlen-evex.S +@@ -0,0 +1,4 @@ ++#define STRLEN __strnlen_evex ++#define USE_AS_STRNLEN 1 ++ ++#include "strlen-evex.S" +diff --git a/sysdeps/x86_64/multiarch/strrchr-evex.S b/sysdeps/x86_64/multiarch/strrchr-evex.S +new file mode 100644 +index 00000000..f920b5a5 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strrchr-evex.S +@@ -0,0 +1,265 @@ ++/* strrchr/wcsrchr optimized with 256-bit EVEX instructions. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#if IS_IN (libc) ++ ++# include ++ ++# ifndef STRRCHR ++# define STRRCHR __strrchr_evex ++# endif ++ ++# define VMOVU vmovdqu64 ++# define VMOVA vmovdqa64 ++ ++# ifdef USE_AS_WCSRCHR ++# define VPBROADCAST vpbroadcastd ++# define VPCMP vpcmpd ++# define SHIFT_REG r8d ++# else ++# define VPBROADCAST vpbroadcastb ++# define VPCMP vpcmpb ++# define SHIFT_REG ecx ++# endif ++ ++# define XMMZERO xmm16 ++# define YMMZERO ymm16 ++# define YMMMATCH ymm17 ++# define YMM1 ymm18 ++ ++# define VEC_SIZE 32 ++ ++ .section .text.evex,"ax",@progbits ++ENTRY (STRRCHR) ++ movl %edi, %ecx ++ /* Broadcast CHAR to YMMMATCH. */ ++ VPBROADCAST %esi, %YMMMATCH ++ ++ vpxorq %XMMZERO, %XMMZERO, %XMMZERO ++ ++ /* Check if we may cross page boundary with one vector load. */ ++ andl $(2 * VEC_SIZE - 1), %ecx ++ cmpl $VEC_SIZE, %ecx ++ ja L(cros_page_boundary) ++ ++ VMOVU (%rdi), %YMM1 ++ ++ /* Each bit in K0 represents a null byte in YMM1. */ ++ VPCMP $0, %YMMZERO, %YMM1, %k0 ++ /* Each bit in K1 represents a CHAR in YMM1. */ ++ VPCMP $0, %YMMMATCH, %YMM1, %k1 ++ kmovd %k0, %ecx ++ kmovd %k1, %eax ++ ++ addq $VEC_SIZE, %rdi ++ ++ testl %eax, %eax ++ jnz L(first_vec) ++ ++ testl %ecx, %ecx ++ jnz L(return_null) ++ ++ andq $-VEC_SIZE, %rdi ++ xorl %edx, %edx ++ jmp L(aligned_loop) ++ ++ .p2align 4 ++L(first_vec): ++ /* Check if there is a null byte. */ ++ testl %ecx, %ecx ++ jnz L(char_and_nul_in_first_vec) ++ ++ /* Remember the match and keep searching. */ ++ movl %eax, %edx ++ movq %rdi, %rsi ++ andq $-VEC_SIZE, %rdi ++ jmp L(aligned_loop) ++ ++ .p2align 4 ++L(cros_page_boundary): ++ andl $(VEC_SIZE - 1), %ecx ++ andq $-VEC_SIZE, %rdi ++ ++# ifdef USE_AS_WCSRCHR ++ /* NB: Divide shift count by 4 since each bit in K1 represent 4 ++ bytes. */ ++ movl %ecx, %SHIFT_REG ++ sarl $2, %SHIFT_REG ++# endif ++ ++ VMOVA (%rdi), %YMM1 ++ ++ /* Each bit in K0 represents a null byte in YMM1. */ ++ VPCMP $0, %YMMZERO, %YMM1, %k0 ++ /* Each bit in K1 represents a CHAR in YMM1. */ ++ VPCMP $0, %YMMMATCH, %YMM1, %k1 ++ kmovd %k0, %edx ++ kmovd %k1, %eax ++ ++ shrxl %SHIFT_REG, %edx, %edx ++ shrxl %SHIFT_REG, %eax, %eax ++ addq $VEC_SIZE, %rdi ++ ++ /* Check if there is a CHAR. */ ++ testl %eax, %eax ++ jnz L(found_char) ++ ++ testl %edx, %edx ++ jnz L(return_null) ++ ++ jmp L(aligned_loop) ++ ++ .p2align 4 ++L(found_char): ++ testl %edx, %edx ++ jnz L(char_and_nul) ++ ++ /* Remember the match and keep searching. */ ++ movl %eax, %edx ++ leaq (%rdi, %rcx), %rsi ++ ++ .p2align 4 ++L(aligned_loop): ++ VMOVA (%rdi), %YMM1 ++ addq $VEC_SIZE, %rdi ++ ++ /* Each bit in K0 represents a null byte in YMM1. */ ++ VPCMP $0, %YMMZERO, %YMM1, %k0 ++ /* Each bit in K1 represents a CHAR in YMM1. */ ++ VPCMP $0, %YMMMATCH, %YMM1, %k1 ++ kmovd %k0, %ecx ++ kmovd %k1, %eax ++ orl %eax, %ecx ++ jnz L(char_nor_null) ++ ++ VMOVA (%rdi), %YMM1 ++ add $VEC_SIZE, %rdi ++ ++ /* Each bit in K0 represents a null byte in YMM1. */ ++ VPCMP $0, %YMMZERO, %YMM1, %k0 ++ /* Each bit in K1 represents a CHAR in YMM1. */ ++ VPCMP $0, %YMMMATCH, %YMM1, %k1 ++ kmovd %k0, %ecx ++ kmovd %k1, %eax ++ orl %eax, %ecx ++ jnz L(char_nor_null) ++ ++ VMOVA (%rdi), %YMM1 ++ addq $VEC_SIZE, %rdi ++ ++ /* Each bit in K0 represents a null byte in YMM1. */ ++ VPCMP $0, %YMMZERO, %YMM1, %k0 ++ /* Each bit in K1 represents a CHAR in YMM1. */ ++ VPCMP $0, %YMMMATCH, %YMM1, %k1 ++ kmovd %k0, %ecx ++ kmovd %k1, %eax ++ orl %eax, %ecx ++ jnz L(char_nor_null) ++ ++ VMOVA (%rdi), %YMM1 ++ addq $VEC_SIZE, %rdi ++ ++ /* Each bit in K0 represents a null byte in YMM1. */ ++ VPCMP $0, %YMMZERO, %YMM1, %k0 ++ /* Each bit in K1 represents a CHAR in YMM1. */ ++ VPCMP $0, %YMMMATCH, %YMM1, %k1 ++ kmovd %k0, %ecx ++ kmovd %k1, %eax ++ orl %eax, %ecx ++ jz L(aligned_loop) ++ ++ .p2align 4 ++L(char_nor_null): ++ /* Find a CHAR or a null byte in a loop. */ ++ testl %eax, %eax ++ jnz L(match) ++L(return_value): ++ testl %edx, %edx ++ jz L(return_null) ++ movl %edx, %eax ++ movq %rsi, %rdi ++ bsrl %eax, %eax ++# ifdef USE_AS_WCSRCHR ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ leaq -VEC_SIZE(%rdi, %rax, 4), %rax ++# else ++ leaq -VEC_SIZE(%rdi, %rax), %rax ++# endif ++ ret ++ ++ .p2align 4 ++L(match): ++ /* Find a CHAR. Check if there is a null byte. */ ++ kmovd %k0, %ecx ++ testl %ecx, %ecx ++ jnz L(find_nul) ++ ++ /* Remember the match and keep searching. */ ++ movl %eax, %edx ++ movq %rdi, %rsi ++ jmp L(aligned_loop) ++ ++ .p2align 4 ++L(find_nul): ++ /* Mask out any matching bits after the null byte. */ ++ movl %ecx, %r8d ++ subl $1, %r8d ++ xorl %ecx, %r8d ++ andl %r8d, %eax ++ testl %eax, %eax ++ /* If there is no CHAR here, return the remembered one. */ ++ jz L(return_value) ++ bsrl %eax, %eax ++# ifdef USE_AS_WCSRCHR ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ leaq -VEC_SIZE(%rdi, %rax, 4), %rax ++# else ++ leaq -VEC_SIZE(%rdi, %rax), %rax ++# endif ++ ret ++ ++ .p2align 4 ++L(char_and_nul): ++ /* Find both a CHAR and a null byte. */ ++ addq %rcx, %rdi ++ movl %edx, %ecx ++L(char_and_nul_in_first_vec): ++ /* Mask out any matching bits after the null byte. */ ++ movl %ecx, %r8d ++ subl $1, %r8d ++ xorl %ecx, %r8d ++ andl %r8d, %eax ++ testl %eax, %eax ++ /* Return null pointer if the null byte comes first. */ ++ jz L(return_null) ++ bsrl %eax, %eax ++# ifdef USE_AS_WCSRCHR ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ leaq -VEC_SIZE(%rdi, %rax, 4), %rax ++# else ++ leaq -VEC_SIZE(%rdi, %rax), %rax ++# endif ++ ret ++ ++ .p2align 4 ++L(return_null): ++ xorl %eax, %eax ++ ret ++ ++END (STRRCHR) ++#endif +diff --git a/sysdeps/x86_64/multiarch/wcschr-evex.S b/sysdeps/x86_64/multiarch/wcschr-evex.S +new file mode 100644 +index 00000000..7cb8f1e4 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/wcschr-evex.S +@@ -0,0 +1,3 @@ ++#define STRCHR __wcschr_evex ++#define USE_AS_WCSCHR 1 ++#include "strchr-evex.S" +diff --git a/sysdeps/x86_64/multiarch/wcscmp-evex.S b/sysdeps/x86_64/multiarch/wcscmp-evex.S +new file mode 100644 +index 00000000..42e73e51 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/wcscmp-evex.S +@@ -0,0 +1,4 @@ ++#define STRCMP __wcscmp_evex ++#define USE_AS_WCSCMP 1 ++ ++#include "strcmp-evex.S" +diff --git a/sysdeps/x86_64/multiarch/wcslen-evex.S b/sysdeps/x86_64/multiarch/wcslen-evex.S +new file mode 100644 +index 00000000..bdafa83b +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/wcslen-evex.S +@@ -0,0 +1,4 @@ ++#define STRLEN __wcslen_evex ++#define USE_AS_WCSLEN 1 ++ ++#include "strlen-evex.S" +diff --git a/sysdeps/x86_64/multiarch/wcsncmp-evex.S b/sysdeps/x86_64/multiarch/wcsncmp-evex.S +new file mode 100644 +index 00000000..8a8e3107 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/wcsncmp-evex.S +@@ -0,0 +1,5 @@ ++#define STRCMP __wcsncmp_evex ++#define USE_AS_STRNCMP 1 ++#define USE_AS_WCSCMP 1 ++ ++#include "strcmp-evex.S" +diff --git a/sysdeps/x86_64/multiarch/wcsnlen-evex.S b/sysdeps/x86_64/multiarch/wcsnlen-evex.S +new file mode 100644 +index 00000000..24773bb4 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/wcsnlen-evex.S +@@ -0,0 +1,5 @@ ++#define STRLEN __wcsnlen_evex ++#define USE_AS_WCSLEN 1 ++#define USE_AS_STRNLEN 1 ++ ++#include "strlen-evex.S" +diff --git a/sysdeps/x86_64/multiarch/wcsnlen.c b/sysdeps/x86_64/multiarch/wcsnlen.c +index b3144c93..84254b83 100644 +--- a/sysdeps/x86_64/multiarch/wcsnlen.c ++++ b/sysdeps/x86_64/multiarch/wcsnlen.c +@@ -29,16 +29,24 @@ + extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (sse4_1) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden; + + static inline void * + IFUNC_SELECTOR (void) + { + const struct cpu_features* cpu_features = __get_cpu_features (); + +- if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER) +- && CPU_FEATURE_USABLE_P (cpu_features, AVX2) ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX2) + && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) +- return OPTIMIZE (avx2); ++ { ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) ++ && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW) ++ && CPU_FEATURE_USABLE_P (cpu_features, BMI2)) ++ return OPTIMIZE (evex); ++ ++ if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) ++ return OPTIMIZE (avx2); ++ } + + if (CPU_FEATURE_USABLE_P (cpu_features, SSE4_1)) + return OPTIMIZE (sse4_1); +diff --git a/sysdeps/x86_64/multiarch/wcsrchr-evex.S b/sysdeps/x86_64/multiarch/wcsrchr-evex.S +new file mode 100644 +index 00000000..c64602f7 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/wcsrchr-evex.S +@@ -0,0 +1,3 @@ ++#define STRRCHR __wcsrchr_evex ++#define USE_AS_WCSRCHR 1 ++#include "strrchr-evex.S" +diff --git a/sysdeps/x86_64/multiarch/wmemchr-evex.S b/sysdeps/x86_64/multiarch/wmemchr-evex.S +new file mode 100644 +index 00000000..06cd0f9f +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/wmemchr-evex.S +@@ -0,0 +1,4 @@ ++#define MEMCHR __wmemchr_evex ++#define USE_AS_WMEMCHR 1 ++ ++#include "memchr-evex.S" +-- +GitLab + diff --git a/SOURCES/ia-upd-256bit-evex-instr-2.patch b/SOURCES/ia-upd-256bit-evex-instr-2.patch new file mode 100644 index 0000000..9db964f --- /dev/null +++ b/SOURCES/ia-upd-256bit-evex-instr-2.patch @@ -0,0 +1,1489 @@ +From 98192464b47c056515b6ac5ff218c197bd75618d Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Fri, 5 Mar 2021 06:36:50 -0800 +Subject: [PATCH] x86-64: Add strcpy family functions with 256-bit EVEX + +Update ifunc-strcpy.h to select the function optimized with 256-bit EVEX +instructions using YMM16-YMM31 registers to avoid RTM abort with usable +AVX512VL and AVX512BW since VZEROUPPER isn't needed at function exit. + +(cherry picked from commit 525bc2a32c9710df40371f951217c6ae7a923aee) +--- + sysdeps/x86_64/multiarch/Makefile | 6 + + sysdeps/x86_64/multiarch/ifunc-impl-list.c | 24 + + sysdeps/x86_64/multiarch/ifunc-strcpy.h | 13 +- + sysdeps/x86_64/multiarch/stpcpy-evex.S | 3 + + sysdeps/x86_64/multiarch/stpncpy-evex.S | 4 + + sysdeps/x86_64/multiarch/strcat-evex.S | 283 ++++++ + sysdeps/x86_64/multiarch/strcpy-evex.S | 1003 ++++++++++++++++++++ + sysdeps/x86_64/multiarch/strncat-evex.S | 3 + + sysdeps/x86_64/multiarch/strncpy-evex.S | 3 + + 9 files changed, 1339 insertions(+), 3 deletions(-) + create mode 100644 sysdeps/x86_64/multiarch/stpcpy-evex.S + create mode 100644 sysdeps/x86_64/multiarch/stpncpy-evex.S + create mode 100644 sysdeps/x86_64/multiarch/strcat-evex.S + create mode 100644 sysdeps/x86_64/multiarch/strcpy-evex.S + create mode 100644 sysdeps/x86_64/multiarch/strncat-evex.S + create mode 100644 sysdeps/x86_64/multiarch/strncpy-evex.S + +diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile +index 5ce85882..46783cd1 100644 +--- a/sysdeps/x86_64/multiarch/Makefile ++++ b/sysdeps/x86_64/multiarch/Makefile +@@ -43,11 +43,17 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c \ + memchr-evex \ + memrchr-evex \ + rawmemchr-evex \ ++ stpcpy-evex \ ++ stpncpy-evex \ ++ strcat-evex \ + strchr-evex \ + strchrnul-evex \ + strcmp-evex \ ++ strcpy-evex \ + strlen-evex \ ++ strncat-evex \ + strncmp-evex \ ++ strncpy-evex \ + strnlen-evex \ + strrchr-evex + CFLAGS-varshift.c += -msse4 +diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c +index bd7d9f19..082e4da3 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c ++++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c +@@ -224,6 +224,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + __stpncpy_ssse3) + IFUNC_IMPL_ADD (array, i, stpncpy, CPU_FEATURE_USABLE (AVX2), + __stpncpy_avx2) ++ IFUNC_IMPL_ADD (array, i, stpncpy, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW)), ++ __stpncpy_evex) + IFUNC_IMPL_ADD (array, i, stpncpy, 1, + __stpncpy_sse2_unaligned) + IFUNC_IMPL_ADD (array, i, stpncpy, 1, __stpncpy_sse2)) +@@ -234,6 +238,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + __stpcpy_ssse3) + IFUNC_IMPL_ADD (array, i, stpcpy, CPU_FEATURE_USABLE (AVX2), + __stpcpy_avx2) ++ IFUNC_IMPL_ADD (array, i, stpcpy, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW)), ++ __stpcpy_evex) + IFUNC_IMPL_ADD (array, i, stpcpy, 1, __stpcpy_sse2_unaligned) + IFUNC_IMPL_ADD (array, i, stpcpy, 1, __stpcpy_sse2)) + +@@ -268,6 +276,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL (i, name, strcat, + IFUNC_IMPL_ADD (array, i, strcat, CPU_FEATURE_USABLE (AVX2), + __strcat_avx2) ++ IFUNC_IMPL_ADD (array, i, strcat, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW)), ++ __strcat_evex) + IFUNC_IMPL_ADD (array, i, strcat, CPU_FEATURE_USABLE (SSSE3), + __strcat_ssse3) + IFUNC_IMPL_ADD (array, i, strcat, 1, __strcat_sse2_unaligned) +@@ -330,6 +342,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL (i, name, strcpy, + IFUNC_IMPL_ADD (array, i, strcpy, CPU_FEATURE_USABLE (AVX2), + __strcpy_avx2) ++ IFUNC_IMPL_ADD (array, i, strcpy, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW)), ++ __strcpy_evex) + IFUNC_IMPL_ADD (array, i, strcpy, CPU_FEATURE_USABLE (SSSE3), + __strcpy_ssse3) + IFUNC_IMPL_ADD (array, i, strcpy, 1, __strcpy_sse2_unaligned) +@@ -373,6 +389,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL (i, name, strncat, + IFUNC_IMPL_ADD (array, i, strncat, CPU_FEATURE_USABLE (AVX2), + __strncat_avx2) ++ IFUNC_IMPL_ADD (array, i, strncat, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW)), ++ __strncat_evex) + IFUNC_IMPL_ADD (array, i, strncat, CPU_FEATURE_USABLE (SSSE3), + __strncat_ssse3) + IFUNC_IMPL_ADD (array, i, strncat, 1, +@@ -383,6 +403,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL (i, name, strncpy, + IFUNC_IMPL_ADD (array, i, strncpy, CPU_FEATURE_USABLE (AVX2), + __strncpy_avx2) ++ IFUNC_IMPL_ADD (array, i, strncpy, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW)), ++ __strncpy_evex) + IFUNC_IMPL_ADD (array, i, strncpy, CPU_FEATURE_USABLE (SSSE3), + __strncpy_ssse3) + IFUNC_IMPL_ADD (array, i, strncpy, 1, +diff --git a/sysdeps/x86_64/multiarch/ifunc-strcpy.h b/sysdeps/x86_64/multiarch/ifunc-strcpy.h +index 100dca5c..deae6348 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-strcpy.h ++++ b/sysdeps/x86_64/multiarch/ifunc-strcpy.h +@@ -25,16 +25,23 @@ extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned) + attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden; + + static inline void * + IFUNC_SELECTOR (void) + { + const struct cpu_features* cpu_features = __get_cpu_features (); + +- if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER) +- && CPU_FEATURE_USABLE_P (cpu_features, AVX2) ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX2) + && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) +- return OPTIMIZE (avx2); ++ { ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) ++ && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)) ++ return OPTIMIZE (evex); ++ ++ if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) ++ return OPTIMIZE (avx2); ++ } + + if (CPU_FEATURES_ARCH_P (cpu_features, Fast_Unaligned_Load)) + return OPTIMIZE (sse2_unaligned); +diff --git a/sysdeps/x86_64/multiarch/stpcpy-evex.S b/sysdeps/x86_64/multiarch/stpcpy-evex.S +new file mode 100644 +index 00000000..7c6f26cd +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/stpcpy-evex.S +@@ -0,0 +1,3 @@ ++#define USE_AS_STPCPY ++#define STRCPY __stpcpy_evex ++#include "strcpy-evex.S" +diff --git a/sysdeps/x86_64/multiarch/stpncpy-evex.S b/sysdeps/x86_64/multiarch/stpncpy-evex.S +new file mode 100644 +index 00000000..1570014d +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/stpncpy-evex.S +@@ -0,0 +1,4 @@ ++#define USE_AS_STPCPY ++#define USE_AS_STRNCPY ++#define STRCPY __stpncpy_evex ++#include "strcpy-evex.S" +diff --git a/sysdeps/x86_64/multiarch/strcat-evex.S b/sysdeps/x86_64/multiarch/strcat-evex.S +new file mode 100644 +index 00000000..97c3d85b +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strcat-evex.S +@@ -0,0 +1,283 @@ ++/* strcat with 256-bit EVEX instructions. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#if IS_IN (libc) ++ ++# include ++ ++# ifndef STRCAT ++# define STRCAT __strcat_evex ++# endif ++ ++# define VMOVU vmovdqu64 ++# define VMOVA vmovdqa64 ++ ++/* zero register */ ++# define XMMZERO xmm16 ++# define YMMZERO ymm16 ++# define YMM0 ymm17 ++# define YMM1 ymm18 ++ ++# define USE_AS_STRCAT ++ ++/* Number of bytes in a vector register */ ++# define VEC_SIZE 32 ++ ++ .section .text.evex,"ax",@progbits ++ENTRY (STRCAT) ++ mov %rdi, %r9 ++# ifdef USE_AS_STRNCAT ++ mov %rdx, %r8 ++# endif ++ ++ xor %eax, %eax ++ mov %edi, %ecx ++ and $((VEC_SIZE * 4) - 1), %ecx ++ vpxorq %XMMZERO, %XMMZERO, %XMMZERO ++ cmp $(VEC_SIZE * 3), %ecx ++ ja L(fourth_vector_boundary) ++ vpcmpb $0, (%rdi), %YMMZERO, %k0 ++ kmovd %k0, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_first_vector) ++ mov %rdi, %rax ++ and $-VEC_SIZE, %rax ++ jmp L(align_vec_size_start) ++L(fourth_vector_boundary): ++ mov %rdi, %rax ++ and $-VEC_SIZE, %rax ++ vpcmpb $0, (%rax), %YMMZERO, %k0 ++ mov $-1, %r10d ++ sub %rax, %rcx ++ shl %cl, %r10d ++ kmovd %k0, %edx ++ and %r10d, %edx ++ jnz L(exit) ++ ++L(align_vec_size_start): ++ vpcmpb $0, VEC_SIZE(%rax), %YMMZERO, %k0 ++ kmovd %k0, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_second_vector) ++ ++ vpcmpb $0, (VEC_SIZE * 2)(%rax), %YMMZERO, %k1 ++ kmovd %k1, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_third_vector) ++ ++ vpcmpb $0, (VEC_SIZE * 3)(%rax), %YMMZERO, %k2 ++ kmovd %k2, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_fourth_vector) ++ ++ vpcmpb $0, (VEC_SIZE * 4)(%rax), %YMMZERO, %k3 ++ kmovd %k3, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_fifth_vector) ++ ++ vpcmpb $0, (VEC_SIZE * 5)(%rax), %YMMZERO, %k4 ++ add $(VEC_SIZE * 4), %rax ++ kmovd %k4, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_second_vector) ++ ++ vpcmpb $0, (VEC_SIZE * 2)(%rax), %YMMZERO, %k1 ++ kmovd %k1, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_third_vector) ++ ++ vpcmpb $0, (VEC_SIZE * 3)(%rax), %YMMZERO, %k2 ++ kmovd %k2, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_fourth_vector) ++ ++ vpcmpb $0, (VEC_SIZE * 4)(%rax), %YMMZERO, %k3 ++ kmovd %k3, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_fifth_vector) ++ ++ vpcmpb $0, (VEC_SIZE * 5)(%rax), %YMMZERO, %k4 ++ kmovd %k4, %edx ++ add $(VEC_SIZE * 4), %rax ++ test %edx, %edx ++ jnz L(exit_null_on_second_vector) ++ ++ vpcmpb $0, (VEC_SIZE * 2)(%rax), %YMMZERO, %k1 ++ kmovd %k1, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_third_vector) ++ ++ vpcmpb $0, (VEC_SIZE * 3)(%rax), %YMMZERO, %k2 ++ kmovd %k2, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_fourth_vector) ++ ++ vpcmpb $0, (VEC_SIZE * 4)(%rax), %YMMZERO, %k3 ++ kmovd %k3, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_fifth_vector) ++ ++ vpcmpb $0, (VEC_SIZE * 5)(%rax), %YMMZERO, %k4 ++ add $(VEC_SIZE * 4), %rax ++ kmovd %k4, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_second_vector) ++ ++ vpcmpb $0, (VEC_SIZE * 2)(%rax), %YMMZERO, %k1 ++ kmovd %k1, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_third_vector) ++ ++ vpcmpb $0, (VEC_SIZE * 3)(%rax), %YMMZERO, %k2 ++ kmovd %k2, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_fourth_vector) ++ ++ vpcmpb $0, (VEC_SIZE * 4)(%rax), %YMMZERO, %k3 ++ kmovd %k3, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_fifth_vector) ++ ++ test $((VEC_SIZE * 4) - 1), %rax ++ jz L(align_four_vec_loop) ++ ++ vpcmpb $0, (VEC_SIZE * 5)(%rax), %YMMZERO, %k4 ++ add $(VEC_SIZE * 5), %rax ++ kmovd %k4, %edx ++ test %edx, %edx ++ jnz L(exit) ++ ++ test $((VEC_SIZE * 4) - 1), %rax ++ jz L(align_four_vec_loop) ++ ++ vpcmpb $0, VEC_SIZE(%rax), %YMMZERO, %k0 ++ add $VEC_SIZE, %rax ++ kmovd %k0, %edx ++ test %edx, %edx ++ jnz L(exit) ++ ++ test $((VEC_SIZE * 4) - 1), %rax ++ jz L(align_four_vec_loop) ++ ++ vpcmpb $0, VEC_SIZE(%rax), %YMMZERO, %k0 ++ add $VEC_SIZE, %rax ++ kmovd %k0, %edx ++ test %edx, %edx ++ jnz L(exit) ++ ++ test $((VEC_SIZE * 4) - 1), %rax ++ jz L(align_four_vec_loop) ++ ++ vpcmpb $0, VEC_SIZE(%rax), %YMMZERO, %k1 ++ add $VEC_SIZE, %rax ++ kmovd %k1, %edx ++ test %edx, %edx ++ jnz L(exit) ++ ++ add $VEC_SIZE, %rax ++ ++ .p2align 4 ++L(align_four_vec_loop): ++ VMOVA (%rax), %YMM0 ++ VMOVA (VEC_SIZE * 2)(%rax), %YMM1 ++ vpminub VEC_SIZE(%rax), %YMM0, %YMM0 ++ vpminub (VEC_SIZE * 3)(%rax), %YMM1, %YMM1 ++ vpminub %YMM0, %YMM1, %YMM0 ++ /* If K0 != 0, there is a null byte. */ ++ vpcmpb $0, %YMM0, %YMMZERO, %k0 ++ add $(VEC_SIZE * 4), %rax ++ ktestd %k0, %k0 ++ jz L(align_four_vec_loop) ++ ++ vpcmpb $0, -(VEC_SIZE * 4)(%rax), %YMMZERO, %k0 ++ sub $(VEC_SIZE * 5), %rax ++ kmovd %k0, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_second_vector) ++ ++ vpcmpb $0, (VEC_SIZE * 2)(%rax), %YMMZERO, %k1 ++ kmovd %k1, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_third_vector) ++ ++ vpcmpb $0, (VEC_SIZE * 3)(%rax), %YMMZERO, %k2 ++ kmovd %k2, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_fourth_vector) ++ ++ vpcmpb $0, (VEC_SIZE * 4)(%rax), %YMMZERO, %k3 ++ kmovd %k3, %edx ++ sub %rdi, %rax ++ bsf %rdx, %rdx ++ add %rdx, %rax ++ add $(VEC_SIZE * 4), %rax ++ jmp L(StartStrcpyPart) ++ ++ .p2align 4 ++L(exit): ++ sub %rdi, %rax ++L(exit_null_on_first_vector): ++ bsf %rdx, %rdx ++ add %rdx, %rax ++ jmp L(StartStrcpyPart) ++ ++ .p2align 4 ++L(exit_null_on_second_vector): ++ sub %rdi, %rax ++ bsf %rdx, %rdx ++ add %rdx, %rax ++ add $VEC_SIZE, %rax ++ jmp L(StartStrcpyPart) ++ ++ .p2align 4 ++L(exit_null_on_third_vector): ++ sub %rdi, %rax ++ bsf %rdx, %rdx ++ add %rdx, %rax ++ add $(VEC_SIZE * 2), %rax ++ jmp L(StartStrcpyPart) ++ ++ .p2align 4 ++L(exit_null_on_fourth_vector): ++ sub %rdi, %rax ++ bsf %rdx, %rdx ++ add %rdx, %rax ++ add $(VEC_SIZE * 3), %rax ++ jmp L(StartStrcpyPart) ++ ++ .p2align 4 ++L(exit_null_on_fifth_vector): ++ sub %rdi, %rax ++ bsf %rdx, %rdx ++ add %rdx, %rax ++ add $(VEC_SIZE * 4), %rax ++ ++ .p2align 4 ++L(StartStrcpyPart): ++ lea (%r9, %rax), %rdi ++ mov %rsi, %rcx ++ mov %r9, %rax /* save result */ ++ ++# ifdef USE_AS_STRNCAT ++ test %r8, %r8 ++ jz L(ExitZero) ++# define USE_AS_STRNCPY ++# endif ++ ++# include "strcpy-evex.S" ++#endif +diff --git a/sysdeps/x86_64/multiarch/strcpy-evex.S b/sysdeps/x86_64/multiarch/strcpy-evex.S +new file mode 100644 +index 00000000..a343a1a6 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strcpy-evex.S +@@ -0,0 +1,1003 @@ ++/* strcpy with 256-bit EVEX instructions. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#if IS_IN (libc) ++ ++# ifndef USE_AS_STRCAT ++# include ++ ++# ifndef STRCPY ++# define STRCPY __strcpy_evex ++# endif ++ ++# endif ++ ++# define VMOVU vmovdqu64 ++# define VMOVA vmovdqa64 ++ ++/* Number of bytes in a vector register */ ++# ifndef VEC_SIZE ++# define VEC_SIZE 32 ++# endif ++ ++# define XMM2 xmm18 ++# define XMM3 xmm19 ++ ++# define YMM2 ymm18 ++# define YMM3 ymm19 ++# define YMM4 ymm20 ++# define YMM5 ymm21 ++# define YMM6 ymm22 ++# define YMM7 ymm23 ++ ++# ifndef USE_AS_STRCAT ++ ++/* zero register */ ++# define XMMZERO xmm16 ++# define YMMZERO ymm16 ++# define YMM1 ymm17 ++ ++ .section .text.evex,"ax",@progbits ++ENTRY (STRCPY) ++# ifdef USE_AS_STRNCPY ++ mov %RDX_LP, %R8_LP ++ test %R8_LP, %R8_LP ++ jz L(ExitZero) ++# endif ++ mov %rsi, %rcx ++# ifndef USE_AS_STPCPY ++ mov %rdi, %rax /* save result */ ++# endif ++ ++ vpxorq %XMMZERO, %XMMZERO, %XMMZERO ++# endif ++ ++ and $((VEC_SIZE * 4) - 1), %ecx ++ cmp $(VEC_SIZE * 2), %ecx ++ jbe L(SourceStringAlignmentLessTwoVecSize) ++ ++ and $-VEC_SIZE, %rsi ++ and $(VEC_SIZE - 1), %ecx ++ ++ vpcmpb $0, (%rsi), %YMMZERO, %k0 ++ kmovd %k0, %edx ++ shr %cl, %rdx ++ ++# ifdef USE_AS_STRNCPY ++# if defined USE_AS_STPCPY || defined USE_AS_STRCAT ++ mov $VEC_SIZE, %r10 ++ sub %rcx, %r10 ++ cmp %r10, %r8 ++# else ++ mov $(VEC_SIZE + 1), %r10 ++ sub %rcx, %r10 ++ cmp %r10, %r8 ++# endif ++ jbe L(CopyVecSizeTailCase2OrCase3) ++# endif ++ test %edx, %edx ++ jnz L(CopyVecSizeTail) ++ ++ vpcmpb $0, VEC_SIZE(%rsi), %YMMZERO, %k1 ++ kmovd %k1, %edx ++ ++# ifdef USE_AS_STRNCPY ++ add $VEC_SIZE, %r10 ++ cmp %r10, %r8 ++ jbe L(CopyTwoVecSizeCase2OrCase3) ++# endif ++ test %edx, %edx ++ jnz L(CopyTwoVecSize) ++ ++ VMOVU (%rsi, %rcx), %YMM2 /* copy VEC_SIZE bytes */ ++ VMOVU %YMM2, (%rdi) ++ ++/* If source address alignment != destination address alignment */ ++ .p2align 4 ++L(UnalignVecSizeBoth): ++ sub %rcx, %rdi ++# ifdef USE_AS_STRNCPY ++ add %rcx, %r8 ++ sbb %rcx, %rcx ++ or %rcx, %r8 ++# endif ++ mov $VEC_SIZE, %rcx ++ VMOVA (%rsi, %rcx), %YMM2 ++ VMOVU %YMM2, (%rdi, %rcx) ++ VMOVA VEC_SIZE(%rsi, %rcx), %YMM2 ++ vpcmpb $0, %YMM2, %YMMZERO, %k0 ++ kmovd %k0, %edx ++ add $VEC_SIZE, %rcx ++# ifdef USE_AS_STRNCPY ++ sub $(VEC_SIZE * 3), %r8 ++ jbe L(CopyVecSizeCase2OrCase3) ++# endif ++ test %edx, %edx ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++ jnz L(CopyVecSizeUnalignedVec2) ++# else ++ jnz L(CopyVecSize) ++# endif ++ ++ VMOVU %YMM2, (%rdi, %rcx) ++ VMOVA VEC_SIZE(%rsi, %rcx), %YMM3 ++ vpcmpb $0, %YMM3, %YMMZERO, %k0 ++ kmovd %k0, %edx ++ add $VEC_SIZE, %rcx ++# ifdef USE_AS_STRNCPY ++ sub $VEC_SIZE, %r8 ++ jbe L(CopyVecSizeCase2OrCase3) ++# endif ++ test %edx, %edx ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++ jnz L(CopyVecSizeUnalignedVec3) ++# else ++ jnz L(CopyVecSize) ++# endif ++ ++ VMOVU %YMM3, (%rdi, %rcx) ++ VMOVA VEC_SIZE(%rsi, %rcx), %YMM4 ++ vpcmpb $0, %YMM4, %YMMZERO, %k0 ++ kmovd %k0, %edx ++ add $VEC_SIZE, %rcx ++# ifdef USE_AS_STRNCPY ++ sub $VEC_SIZE, %r8 ++ jbe L(CopyVecSizeCase2OrCase3) ++# endif ++ test %edx, %edx ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++ jnz L(CopyVecSizeUnalignedVec4) ++# else ++ jnz L(CopyVecSize) ++# endif ++ ++ VMOVU %YMM4, (%rdi, %rcx) ++ VMOVA VEC_SIZE(%rsi, %rcx), %YMM2 ++ vpcmpb $0, %YMM2, %YMMZERO, %k0 ++ kmovd %k0, %edx ++ add $VEC_SIZE, %rcx ++# ifdef USE_AS_STRNCPY ++ sub $VEC_SIZE, %r8 ++ jbe L(CopyVecSizeCase2OrCase3) ++# endif ++ test %edx, %edx ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++ jnz L(CopyVecSizeUnalignedVec2) ++# else ++ jnz L(CopyVecSize) ++# endif ++ ++ VMOVU %YMM2, (%rdi, %rcx) ++ VMOVA VEC_SIZE(%rsi, %rcx), %YMM2 ++ vpcmpb $0, %YMM2, %YMMZERO, %k0 ++ kmovd %k0, %edx ++ add $VEC_SIZE, %rcx ++# ifdef USE_AS_STRNCPY ++ sub $VEC_SIZE, %r8 ++ jbe L(CopyVecSizeCase2OrCase3) ++# endif ++ test %edx, %edx ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++ jnz L(CopyVecSizeUnalignedVec2) ++# else ++ jnz L(CopyVecSize) ++# endif ++ ++ VMOVA VEC_SIZE(%rsi, %rcx), %YMM3 ++ VMOVU %YMM2, (%rdi, %rcx) ++ vpcmpb $0, %YMM3, %YMMZERO, %k0 ++ kmovd %k0, %edx ++ add $VEC_SIZE, %rcx ++# ifdef USE_AS_STRNCPY ++ sub $VEC_SIZE, %r8 ++ jbe L(CopyVecSizeCase2OrCase3) ++# endif ++ test %edx, %edx ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++ jnz L(CopyVecSizeUnalignedVec3) ++# else ++ jnz L(CopyVecSize) ++# endif ++ ++ VMOVU %YMM3, (%rdi, %rcx) ++ mov %rsi, %rdx ++ lea VEC_SIZE(%rsi, %rcx), %rsi ++ and $-(VEC_SIZE * 4), %rsi ++ sub %rsi, %rdx ++ sub %rdx, %rdi ++# ifdef USE_AS_STRNCPY ++ lea (VEC_SIZE * 8)(%r8, %rdx), %r8 ++# endif ++L(UnalignedFourVecSizeLoop): ++ VMOVA (%rsi), %YMM4 ++ VMOVA VEC_SIZE(%rsi), %YMM5 ++ VMOVA (VEC_SIZE * 2)(%rsi), %YMM6 ++ VMOVA (VEC_SIZE * 3)(%rsi), %YMM7 ++ vpminub %YMM5, %YMM4, %YMM2 ++ vpminub %YMM7, %YMM6, %YMM3 ++ vpminub %YMM2, %YMM3, %YMM2 ++ /* If K7 != 0, there is a null byte. */ ++ vpcmpb $0, %YMM2, %YMMZERO, %k7 ++ kmovd %k7, %edx ++# ifdef USE_AS_STRNCPY ++ sub $(VEC_SIZE * 4), %r8 ++ jbe L(UnalignedLeaveCase2OrCase3) ++# endif ++ test %edx, %edx ++ jnz L(UnalignedFourVecSizeLeave) ++ ++L(UnalignedFourVecSizeLoop_start): ++ add $(VEC_SIZE * 4), %rdi ++ add $(VEC_SIZE * 4), %rsi ++ VMOVU %YMM4, -(VEC_SIZE * 4)(%rdi) ++ VMOVA (%rsi), %YMM4 ++ VMOVU %YMM5, -(VEC_SIZE * 3)(%rdi) ++ VMOVA VEC_SIZE(%rsi), %YMM5 ++ vpminub %YMM5, %YMM4, %YMM2 ++ VMOVU %YMM6, -(VEC_SIZE * 2)(%rdi) ++ VMOVA (VEC_SIZE * 2)(%rsi), %YMM6 ++ VMOVU %YMM7, -VEC_SIZE(%rdi) ++ VMOVA (VEC_SIZE * 3)(%rsi), %YMM7 ++ vpminub %YMM7, %YMM6, %YMM3 ++ vpminub %YMM2, %YMM3, %YMM2 ++ /* If K7 != 0, there is a null byte. */ ++ vpcmpb $0, %YMM2, %YMMZERO, %k7 ++ kmovd %k7, %edx ++# ifdef USE_AS_STRNCPY ++ sub $(VEC_SIZE * 4), %r8 ++ jbe L(UnalignedLeaveCase2OrCase3) ++# endif ++ test %edx, %edx ++ jz L(UnalignedFourVecSizeLoop_start) ++ ++L(UnalignedFourVecSizeLeave): ++ vpcmpb $0, %YMM4, %YMMZERO, %k1 ++ kmovd %k1, %edx ++ test %edx, %edx ++ jnz L(CopyVecSizeUnaligned_0) ++ ++ vpcmpb $0, %YMM5, %YMMZERO, %k2 ++ kmovd %k2, %ecx ++ test %ecx, %ecx ++ jnz L(CopyVecSizeUnaligned_16) ++ ++ vpcmpb $0, %YMM6, %YMMZERO, %k3 ++ kmovd %k3, %edx ++ test %edx, %edx ++ jnz L(CopyVecSizeUnaligned_32) ++ ++ vpcmpb $0, %YMM7, %YMMZERO, %k4 ++ kmovd %k4, %ecx ++ bsf %ecx, %edx ++ VMOVU %YMM4, (%rdi) ++ VMOVU %YMM5, VEC_SIZE(%rdi) ++ VMOVU %YMM6, (VEC_SIZE * 2)(%rdi) ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++# ifdef USE_AS_STPCPY ++ lea (VEC_SIZE * 3)(%rdi, %rdx), %rax ++# endif ++ VMOVU %YMM7, (VEC_SIZE * 3)(%rdi) ++ add $(VEC_SIZE - 1), %r8 ++ sub %rdx, %r8 ++ lea ((VEC_SIZE * 3) + 1)(%rdi, %rdx), %rdi ++ jmp L(StrncpyFillTailWithZero) ++# else ++ add $(VEC_SIZE * 3), %rsi ++ add $(VEC_SIZE * 3), %rdi ++ jmp L(CopyVecSizeExit) ++# endif ++ ++/* If source address alignment == destination address alignment */ ++ ++L(SourceStringAlignmentLessTwoVecSize): ++ VMOVU (%rsi), %YMM3 ++ VMOVU VEC_SIZE(%rsi), %YMM2 ++ vpcmpb $0, %YMM3, %YMMZERO, %k0 ++ kmovd %k0, %edx ++ ++# ifdef USE_AS_STRNCPY ++# if defined USE_AS_STPCPY || defined USE_AS_STRCAT ++ cmp $VEC_SIZE, %r8 ++# else ++ cmp $(VEC_SIZE + 1), %r8 ++# endif ++ jbe L(CopyVecSizeTail1Case2OrCase3) ++# endif ++ test %edx, %edx ++ jnz L(CopyVecSizeTail1) ++ ++ VMOVU %YMM3, (%rdi) ++ vpcmpb $0, %YMM2, %YMMZERO, %k0 ++ kmovd %k0, %edx ++ ++# ifdef USE_AS_STRNCPY ++# if defined USE_AS_STPCPY || defined USE_AS_STRCAT ++ cmp $(VEC_SIZE * 2), %r8 ++# else ++ cmp $((VEC_SIZE * 2) + 1), %r8 ++# endif ++ jbe L(CopyTwoVecSize1Case2OrCase3) ++# endif ++ test %edx, %edx ++ jnz L(CopyTwoVecSize1) ++ ++ and $-VEC_SIZE, %rsi ++ and $(VEC_SIZE - 1), %ecx ++ jmp L(UnalignVecSizeBoth) ++ ++/*------End of main part with loops---------------------*/ ++ ++/* Case1 */ ++ ++# if (!defined USE_AS_STRNCPY) || (defined USE_AS_STRCAT) ++ .p2align 4 ++L(CopyVecSize): ++ add %rcx, %rdi ++# endif ++L(CopyVecSizeTail): ++ add %rcx, %rsi ++L(CopyVecSizeTail1): ++ bsf %edx, %edx ++L(CopyVecSizeExit): ++ cmp $32, %edx ++ jae L(Exit32_63) ++ cmp $16, %edx ++ jae L(Exit16_31) ++ cmp $8, %edx ++ jae L(Exit8_15) ++ cmp $4, %edx ++ jae L(Exit4_7) ++ cmp $3, %edx ++ je L(Exit3) ++ cmp $1, %edx ++ ja L(Exit2) ++ je L(Exit1) ++ movb $0, (%rdi) ++# ifdef USE_AS_STPCPY ++ lea (%rdi), %rax ++# endif ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++ sub $1, %r8 ++ lea 1(%rdi), %rdi ++ jnz L(StrncpyFillTailWithZero) ++# endif ++ ret ++ ++ .p2align 4 ++L(CopyTwoVecSize1): ++ add $VEC_SIZE, %rsi ++ add $VEC_SIZE, %rdi ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++ sub $VEC_SIZE, %r8 ++# endif ++ jmp L(CopyVecSizeTail1) ++ ++ .p2align 4 ++L(CopyTwoVecSize): ++ bsf %edx, %edx ++ add %rcx, %rsi ++ add $VEC_SIZE, %edx ++ sub %ecx, %edx ++ jmp L(CopyVecSizeExit) ++ ++ .p2align 4 ++L(CopyVecSizeUnaligned_0): ++ bsf %edx, %edx ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++# ifdef USE_AS_STPCPY ++ lea (%rdi, %rdx), %rax ++# endif ++ VMOVU %YMM4, (%rdi) ++ add $((VEC_SIZE * 4) - 1), %r8 ++ sub %rdx, %r8 ++ lea 1(%rdi, %rdx), %rdi ++ jmp L(StrncpyFillTailWithZero) ++# else ++ jmp L(CopyVecSizeExit) ++# endif ++ ++ .p2align 4 ++L(CopyVecSizeUnaligned_16): ++ bsf %ecx, %edx ++ VMOVU %YMM4, (%rdi) ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++# ifdef USE_AS_STPCPY ++ lea VEC_SIZE(%rdi, %rdx), %rax ++# endif ++ VMOVU %YMM5, VEC_SIZE(%rdi) ++ add $((VEC_SIZE * 3) - 1), %r8 ++ sub %rdx, %r8 ++ lea (VEC_SIZE + 1)(%rdi, %rdx), %rdi ++ jmp L(StrncpyFillTailWithZero) ++# else ++ add $VEC_SIZE, %rsi ++ add $VEC_SIZE, %rdi ++ jmp L(CopyVecSizeExit) ++# endif ++ ++ .p2align 4 ++L(CopyVecSizeUnaligned_32): ++ bsf %edx, %edx ++ VMOVU %YMM4, (%rdi) ++ VMOVU %YMM5, VEC_SIZE(%rdi) ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++# ifdef USE_AS_STPCPY ++ lea (VEC_SIZE * 2)(%rdi, %rdx), %rax ++# endif ++ VMOVU %YMM6, (VEC_SIZE * 2)(%rdi) ++ add $((VEC_SIZE * 2) - 1), %r8 ++ sub %rdx, %r8 ++ lea ((VEC_SIZE * 2) + 1)(%rdi, %rdx), %rdi ++ jmp L(StrncpyFillTailWithZero) ++# else ++ add $(VEC_SIZE * 2), %rsi ++ add $(VEC_SIZE * 2), %rdi ++ jmp L(CopyVecSizeExit) ++# endif ++ ++# ifdef USE_AS_STRNCPY ++# ifndef USE_AS_STRCAT ++ .p2align 4 ++L(CopyVecSizeUnalignedVec6): ++ VMOVU %YMM6, (%rdi, %rcx) ++ jmp L(CopyVecSizeVecExit) ++ ++ .p2align 4 ++L(CopyVecSizeUnalignedVec5): ++ VMOVU %YMM5, (%rdi, %rcx) ++ jmp L(CopyVecSizeVecExit) ++ ++ .p2align 4 ++L(CopyVecSizeUnalignedVec4): ++ VMOVU %YMM4, (%rdi, %rcx) ++ jmp L(CopyVecSizeVecExit) ++ ++ .p2align 4 ++L(CopyVecSizeUnalignedVec3): ++ VMOVU %YMM3, (%rdi, %rcx) ++ jmp L(CopyVecSizeVecExit) ++# endif ++ ++/* Case2 */ ++ ++ .p2align 4 ++L(CopyVecSizeCase2): ++ add $VEC_SIZE, %r8 ++ add %rcx, %rdi ++ add %rcx, %rsi ++ bsf %edx, %edx ++ cmp %r8d, %edx ++ jb L(CopyVecSizeExit) ++ jmp L(StrncpyExit) ++ ++ .p2align 4 ++L(CopyTwoVecSizeCase2): ++ add %rcx, %rsi ++ bsf %edx, %edx ++ add $VEC_SIZE, %edx ++ sub %ecx, %edx ++ cmp %r8d, %edx ++ jb L(CopyVecSizeExit) ++ jmp L(StrncpyExit) ++ ++L(CopyVecSizeTailCase2): ++ add %rcx, %rsi ++ bsf %edx, %edx ++ cmp %r8d, %edx ++ jb L(CopyVecSizeExit) ++ jmp L(StrncpyExit) ++ ++L(CopyVecSizeTail1Case2): ++ bsf %edx, %edx ++ cmp %r8d, %edx ++ jb L(CopyVecSizeExit) ++ jmp L(StrncpyExit) ++ ++/* Case2 or Case3, Case3 */ ++ ++ .p2align 4 ++L(CopyVecSizeCase2OrCase3): ++ test %rdx, %rdx ++ jnz L(CopyVecSizeCase2) ++L(CopyVecSizeCase3): ++ add $VEC_SIZE, %r8 ++ add %rcx, %rdi ++ add %rcx, %rsi ++ jmp L(StrncpyExit) ++ ++ .p2align 4 ++L(CopyTwoVecSizeCase2OrCase3): ++ test %rdx, %rdx ++ jnz L(CopyTwoVecSizeCase2) ++ add %rcx, %rsi ++ jmp L(StrncpyExit) ++ ++ .p2align 4 ++L(CopyVecSizeTailCase2OrCase3): ++ test %rdx, %rdx ++ jnz L(CopyVecSizeTailCase2) ++ add %rcx, %rsi ++ jmp L(StrncpyExit) ++ ++ .p2align 4 ++L(CopyTwoVecSize1Case2OrCase3): ++ add $VEC_SIZE, %rdi ++ add $VEC_SIZE, %rsi ++ sub $VEC_SIZE, %r8 ++L(CopyVecSizeTail1Case2OrCase3): ++ test %rdx, %rdx ++ jnz L(CopyVecSizeTail1Case2) ++ jmp L(StrncpyExit) ++# endif ++ ++/*------------End labels regarding with copying 1-VEC_SIZE bytes--and 1-(VEC_SIZE*2) bytes----*/ ++ ++ .p2align 4 ++L(Exit1): ++ movzwl (%rsi), %edx ++ mov %dx, (%rdi) ++# ifdef USE_AS_STPCPY ++ lea 1(%rdi), %rax ++# endif ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++ sub $2, %r8 ++ lea 2(%rdi), %rdi ++ jnz L(StrncpyFillTailWithZero) ++# endif ++ ret ++ ++ .p2align 4 ++L(Exit2): ++ movzwl (%rsi), %ecx ++ mov %cx, (%rdi) ++ movb $0, 2(%rdi) ++# ifdef USE_AS_STPCPY ++ lea 2(%rdi), %rax ++# endif ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++ sub $3, %r8 ++ lea 3(%rdi), %rdi ++ jnz L(StrncpyFillTailWithZero) ++# endif ++ ret ++ ++ .p2align 4 ++L(Exit3): ++ mov (%rsi), %edx ++ mov %edx, (%rdi) ++# ifdef USE_AS_STPCPY ++ lea 3(%rdi), %rax ++# endif ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++ sub $4, %r8 ++ lea 4(%rdi), %rdi ++ jnz L(StrncpyFillTailWithZero) ++# endif ++ ret ++ ++ .p2align 4 ++L(Exit4_7): ++ mov (%rsi), %ecx ++ mov %ecx, (%rdi) ++ mov -3(%rsi, %rdx), %ecx ++ mov %ecx, -3(%rdi, %rdx) ++# ifdef USE_AS_STPCPY ++ lea (%rdi, %rdx), %rax ++# endif ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++ sub %rdx, %r8 ++ sub $1, %r8 ++ lea 1(%rdi, %rdx), %rdi ++ jnz L(StrncpyFillTailWithZero) ++# endif ++ ret ++ ++ .p2align 4 ++L(Exit8_15): ++ mov (%rsi), %rcx ++ mov -7(%rsi, %rdx), %r9 ++ mov %rcx, (%rdi) ++ mov %r9, -7(%rdi, %rdx) ++# ifdef USE_AS_STPCPY ++ lea (%rdi, %rdx), %rax ++# endif ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++ sub %rdx, %r8 ++ sub $1, %r8 ++ lea 1(%rdi, %rdx), %rdi ++ jnz L(StrncpyFillTailWithZero) ++# endif ++ ret ++ ++ .p2align 4 ++L(Exit16_31): ++ VMOVU (%rsi), %XMM2 ++ VMOVU -15(%rsi, %rdx), %XMM3 ++ VMOVU %XMM2, (%rdi) ++ VMOVU %XMM3, -15(%rdi, %rdx) ++# ifdef USE_AS_STPCPY ++ lea (%rdi, %rdx), %rax ++# endif ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++ sub %rdx, %r8 ++ sub $1, %r8 ++ lea 1(%rdi, %rdx), %rdi ++ jnz L(StrncpyFillTailWithZero) ++# endif ++ ret ++ ++ .p2align 4 ++L(Exit32_63): ++ VMOVU (%rsi), %YMM2 ++ VMOVU -31(%rsi, %rdx), %YMM3 ++ VMOVU %YMM2, (%rdi) ++ VMOVU %YMM3, -31(%rdi, %rdx) ++# ifdef USE_AS_STPCPY ++ lea (%rdi, %rdx), %rax ++# endif ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++ sub %rdx, %r8 ++ sub $1, %r8 ++ lea 1(%rdi, %rdx), %rdi ++ jnz L(StrncpyFillTailWithZero) ++# endif ++ ret ++ ++# ifdef USE_AS_STRNCPY ++ ++ .p2align 4 ++L(StrncpyExit1): ++ movzbl (%rsi), %edx ++ mov %dl, (%rdi) ++# ifdef USE_AS_STPCPY ++ lea 1(%rdi), %rax ++# endif ++# ifdef USE_AS_STRCAT ++ movb $0, 1(%rdi) ++# endif ++ ret ++ ++ .p2align 4 ++L(StrncpyExit2): ++ movzwl (%rsi), %edx ++ mov %dx, (%rdi) ++# ifdef USE_AS_STPCPY ++ lea 2(%rdi), %rax ++# endif ++# ifdef USE_AS_STRCAT ++ movb $0, 2(%rdi) ++# endif ++ ret ++ ++ .p2align 4 ++L(StrncpyExit3_4): ++ movzwl (%rsi), %ecx ++ movzwl -2(%rsi, %r8), %edx ++ mov %cx, (%rdi) ++ mov %dx, -2(%rdi, %r8) ++# ifdef USE_AS_STPCPY ++ lea (%rdi, %r8), %rax ++# endif ++# ifdef USE_AS_STRCAT ++ movb $0, (%rdi, %r8) ++# endif ++ ret ++ ++ .p2align 4 ++L(StrncpyExit5_8): ++ mov (%rsi), %ecx ++ mov -4(%rsi, %r8), %edx ++ mov %ecx, (%rdi) ++ mov %edx, -4(%rdi, %r8) ++# ifdef USE_AS_STPCPY ++ lea (%rdi, %r8), %rax ++# endif ++# ifdef USE_AS_STRCAT ++ movb $0, (%rdi, %r8) ++# endif ++ ret ++ ++ .p2align 4 ++L(StrncpyExit9_16): ++ mov (%rsi), %rcx ++ mov -8(%rsi, %r8), %rdx ++ mov %rcx, (%rdi) ++ mov %rdx, -8(%rdi, %r8) ++# ifdef USE_AS_STPCPY ++ lea (%rdi, %r8), %rax ++# endif ++# ifdef USE_AS_STRCAT ++ movb $0, (%rdi, %r8) ++# endif ++ ret ++ ++ .p2align 4 ++L(StrncpyExit17_32): ++ VMOVU (%rsi), %XMM2 ++ VMOVU -16(%rsi, %r8), %XMM3 ++ VMOVU %XMM2, (%rdi) ++ VMOVU %XMM3, -16(%rdi, %r8) ++# ifdef USE_AS_STPCPY ++ lea (%rdi, %r8), %rax ++# endif ++# ifdef USE_AS_STRCAT ++ movb $0, (%rdi, %r8) ++# endif ++ ret ++ ++ .p2align 4 ++L(StrncpyExit33_64): ++ /* 0/32, 31/16 */ ++ VMOVU (%rsi), %YMM2 ++ VMOVU -VEC_SIZE(%rsi, %r8), %YMM3 ++ VMOVU %YMM2, (%rdi) ++ VMOVU %YMM3, -VEC_SIZE(%rdi, %r8) ++# ifdef USE_AS_STPCPY ++ lea (%rdi, %r8), %rax ++# endif ++# ifdef USE_AS_STRCAT ++ movb $0, (%rdi, %r8) ++# endif ++ ret ++ ++ .p2align 4 ++L(StrncpyExit65): ++ /* 0/32, 32/32, 64/1 */ ++ VMOVU (%rsi), %YMM2 ++ VMOVU 32(%rsi), %YMM3 ++ mov 64(%rsi), %cl ++ VMOVU %YMM2, (%rdi) ++ VMOVU %YMM3, 32(%rdi) ++ mov %cl, 64(%rdi) ++# ifdef USE_AS_STPCPY ++ lea 65(%rdi), %rax ++# endif ++# ifdef USE_AS_STRCAT ++ movb $0, 65(%rdi) ++# endif ++ ret ++ ++# ifndef USE_AS_STRCAT ++ ++ .p2align 4 ++L(Fill1): ++ mov %dl, (%rdi) ++ ret ++ ++ .p2align 4 ++L(Fill2): ++ mov %dx, (%rdi) ++ ret ++ ++ .p2align 4 ++L(Fill3_4): ++ mov %dx, (%rdi) ++ mov %dx, -2(%rdi, %r8) ++ ret ++ ++ .p2align 4 ++L(Fill5_8): ++ mov %edx, (%rdi) ++ mov %edx, -4(%rdi, %r8) ++ ret ++ ++ .p2align 4 ++L(Fill9_16): ++ mov %rdx, (%rdi) ++ mov %rdx, -8(%rdi, %r8) ++ ret ++ ++ .p2align 4 ++L(Fill17_32): ++ VMOVU %XMMZERO, (%rdi) ++ VMOVU %XMMZERO, -16(%rdi, %r8) ++ ret ++ ++ .p2align 4 ++L(CopyVecSizeUnalignedVec2): ++ VMOVU %YMM2, (%rdi, %rcx) ++ ++ .p2align 4 ++L(CopyVecSizeVecExit): ++ bsf %edx, %edx ++ add $(VEC_SIZE - 1), %r8 ++ add %rcx, %rdi ++# ifdef USE_AS_STPCPY ++ lea (%rdi, %rdx), %rax ++# endif ++ sub %rdx, %r8 ++ lea 1(%rdi, %rdx), %rdi ++ ++ .p2align 4 ++L(StrncpyFillTailWithZero): ++ xor %edx, %edx ++ sub $VEC_SIZE, %r8 ++ jbe L(StrncpyFillExit) ++ ++ VMOVU %YMMZERO, (%rdi) ++ add $VEC_SIZE, %rdi ++ ++ mov %rdi, %rsi ++ and $(VEC_SIZE - 1), %esi ++ sub %rsi, %rdi ++ add %rsi, %r8 ++ sub $(VEC_SIZE * 4), %r8 ++ jb L(StrncpyFillLessFourVecSize) ++ ++L(StrncpyFillLoopVmovdqa): ++ VMOVA %YMMZERO, (%rdi) ++ VMOVA %YMMZERO, VEC_SIZE(%rdi) ++ VMOVA %YMMZERO, (VEC_SIZE * 2)(%rdi) ++ VMOVA %YMMZERO, (VEC_SIZE * 3)(%rdi) ++ add $(VEC_SIZE * 4), %rdi ++ sub $(VEC_SIZE * 4), %r8 ++ jae L(StrncpyFillLoopVmovdqa) ++ ++L(StrncpyFillLessFourVecSize): ++ add $(VEC_SIZE * 2), %r8 ++ jl L(StrncpyFillLessTwoVecSize) ++ VMOVA %YMMZERO, (%rdi) ++ VMOVA %YMMZERO, VEC_SIZE(%rdi) ++ add $(VEC_SIZE * 2), %rdi ++ sub $VEC_SIZE, %r8 ++ jl L(StrncpyFillExit) ++ VMOVA %YMMZERO, (%rdi) ++ add $VEC_SIZE, %rdi ++ jmp L(Fill) ++ ++ .p2align 4 ++L(StrncpyFillLessTwoVecSize): ++ add $VEC_SIZE, %r8 ++ jl L(StrncpyFillExit) ++ VMOVA %YMMZERO, (%rdi) ++ add $VEC_SIZE, %rdi ++ jmp L(Fill) ++ ++ .p2align 4 ++L(StrncpyFillExit): ++ add $VEC_SIZE, %r8 ++L(Fill): ++ cmp $17, %r8d ++ jae L(Fill17_32) ++ cmp $9, %r8d ++ jae L(Fill9_16) ++ cmp $5, %r8d ++ jae L(Fill5_8) ++ cmp $3, %r8d ++ jae L(Fill3_4) ++ cmp $1, %r8d ++ ja L(Fill2) ++ je L(Fill1) ++ ret ++ ++/* end of ifndef USE_AS_STRCAT */ ++# endif ++ ++ .p2align 4 ++L(UnalignedLeaveCase2OrCase3): ++ test %rdx, %rdx ++ jnz L(UnalignedFourVecSizeLeaveCase2) ++L(UnalignedFourVecSizeLeaveCase3): ++ lea (VEC_SIZE * 4)(%r8), %rcx ++ and $-VEC_SIZE, %rcx ++ add $(VEC_SIZE * 3), %r8 ++ jl L(CopyVecSizeCase3) ++ VMOVU %YMM4, (%rdi) ++ sub $VEC_SIZE, %r8 ++ jb L(CopyVecSizeCase3) ++ VMOVU %YMM5, VEC_SIZE(%rdi) ++ sub $VEC_SIZE, %r8 ++ jb L(CopyVecSizeCase3) ++ VMOVU %YMM6, (VEC_SIZE * 2)(%rdi) ++ sub $VEC_SIZE, %r8 ++ jb L(CopyVecSizeCase3) ++ VMOVU %YMM7, (VEC_SIZE * 3)(%rdi) ++# ifdef USE_AS_STPCPY ++ lea (VEC_SIZE * 4)(%rdi), %rax ++# endif ++# ifdef USE_AS_STRCAT ++ movb $0, (VEC_SIZE * 4)(%rdi) ++# endif ++ ret ++ ++ .p2align 4 ++L(UnalignedFourVecSizeLeaveCase2): ++ xor %ecx, %ecx ++ vpcmpb $0, %YMM4, %YMMZERO, %k1 ++ kmovd %k1, %edx ++ add $(VEC_SIZE * 3), %r8 ++ jle L(CopyVecSizeCase2OrCase3) ++ test %edx, %edx ++# ifndef USE_AS_STRCAT ++ jnz L(CopyVecSizeUnalignedVec4) ++# else ++ jnz L(CopyVecSize) ++# endif ++ vpcmpb $0, %YMM5, %YMMZERO, %k2 ++ kmovd %k2, %edx ++ VMOVU %YMM4, (%rdi) ++ add $VEC_SIZE, %rcx ++ sub $VEC_SIZE, %r8 ++ jbe L(CopyVecSizeCase2OrCase3) ++ test %edx, %edx ++# ifndef USE_AS_STRCAT ++ jnz L(CopyVecSizeUnalignedVec5) ++# else ++ jnz L(CopyVecSize) ++# endif ++ ++ vpcmpb $0, %YMM6, %YMMZERO, %k3 ++ kmovd %k3, %edx ++ VMOVU %YMM5, VEC_SIZE(%rdi) ++ add $VEC_SIZE, %rcx ++ sub $VEC_SIZE, %r8 ++ jbe L(CopyVecSizeCase2OrCase3) ++ test %edx, %edx ++# ifndef USE_AS_STRCAT ++ jnz L(CopyVecSizeUnalignedVec6) ++# else ++ jnz L(CopyVecSize) ++# endif ++ ++ vpcmpb $0, %YMM7, %YMMZERO, %k4 ++ kmovd %k4, %edx ++ VMOVU %YMM6, (VEC_SIZE * 2)(%rdi) ++ lea VEC_SIZE(%rdi, %rcx), %rdi ++ lea VEC_SIZE(%rsi, %rcx), %rsi ++ bsf %edx, %edx ++ cmp %r8d, %edx ++ jb L(CopyVecSizeExit) ++L(StrncpyExit): ++ cmp $65, %r8d ++ je L(StrncpyExit65) ++ cmp $33, %r8d ++ jae L(StrncpyExit33_64) ++ cmp $17, %r8d ++ jae L(StrncpyExit17_32) ++ cmp $9, %r8d ++ jae L(StrncpyExit9_16) ++ cmp $5, %r8d ++ jae L(StrncpyExit5_8) ++ cmp $3, %r8d ++ jae L(StrncpyExit3_4) ++ cmp $1, %r8d ++ ja L(StrncpyExit2) ++ je L(StrncpyExit1) ++# ifdef USE_AS_STPCPY ++ mov %rdi, %rax ++# endif ++# ifdef USE_AS_STRCAT ++ movb $0, (%rdi) ++# endif ++ ret ++ ++ .p2align 4 ++L(ExitZero): ++# ifndef USE_AS_STRCAT ++ mov %rdi, %rax ++# endif ++ ret ++ ++# endif ++ ++# ifndef USE_AS_STRCAT ++END (STRCPY) ++# else ++END (STRCAT) ++# endif ++#endif +diff --git a/sysdeps/x86_64/multiarch/strncat-evex.S b/sysdeps/x86_64/multiarch/strncat-evex.S +new file mode 100644 +index 00000000..8884f023 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strncat-evex.S +@@ -0,0 +1,3 @@ ++#define USE_AS_STRNCAT ++#define STRCAT __strncat_evex ++#include "strcat-evex.S" +diff --git a/sysdeps/x86_64/multiarch/strncpy-evex.S b/sysdeps/x86_64/multiarch/strncpy-evex.S +new file mode 100644 +index 00000000..40e391f0 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strncpy-evex.S +@@ -0,0 +1,3 @@ ++#define USE_AS_STRNCPY ++#define STRCPY __strncpy_evex ++#include "strcpy-evex.S" +-- +GitLab + diff --git a/SOURCES/ia-upd-256bit-evex-instr-3.patch b/SOURCES/ia-upd-256bit-evex-instr-3.patch new file mode 100644 index 0000000..1fc1f59 --- /dev/null +++ b/SOURCES/ia-upd-256bit-evex-instr-3.patch @@ -0,0 +1,243 @@ +From c2440c1e45d53140531105de024f7b9ceb53c51e Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Fri, 5 Mar 2021 06:46:08 -0800 +Subject: [PATCH] x86-64: Add memmove family functions with 256-bit EVEX + +Update ifunc-memmove.h to select the function optimized with 256-bit EVEX +instructions using YMM16-YMM31 registers to avoid RTM abort with usable +AVX512VL since VZEROUPPER isn't needed at function exit. + +(cherry picked from commit 63ad43566f7a25d140dc723598aeb441ad657eed) +--- + sysdeps/x86_64/multiarch/Makefile | 1 + + sysdeps/x86_64/multiarch/ifunc-impl-list.c | 36 +++++++++++++++++++ + sysdeps/x86_64/multiarch/ifunc-memmove.h | 21 +++++++++-- + .../multiarch/memmove-evex-unaligned-erms.S | 33 +++++++++++++++++ + .../multiarch/memmove-vec-unaligned-erms.S | 24 ++++++++----- + 5 files changed, 104 insertions(+), 11 deletions(-) + create mode 100644 sysdeps/x86_64/multiarch/memmove-evex-unaligned-erms.S + +diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile +index 46783cd1..4563fc56 100644 +--- a/sysdeps/x86_64/multiarch/Makefile ++++ b/sysdeps/x86_64/multiarch/Makefile +@@ -41,6 +41,7 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c \ + memset-avx2-unaligned-erms \ + memset-avx512-unaligned-erms \ + memchr-evex \ ++ memmove-evex-unaligned-erms \ + memrchr-evex \ + rawmemchr-evex \ + stpcpy-evex \ +diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c +index 082e4da3..6bd3abfc 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c ++++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c +@@ -80,6 +80,12 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, __memmove_chk, + CPU_FEATURE_USABLE (AVX), + __memmove_chk_avx_unaligned_erms) ++ IFUNC_IMPL_ADD (array, i, __memmove_chk, ++ CPU_FEATURE_USABLE (AVX512VL), ++ __memmove_chk_evex_unaligned) ++ IFUNC_IMPL_ADD (array, i, __memmove_chk, ++ CPU_FEATURE_USABLE (AVX512VL), ++ __memmove_chk_evex_unaligned_erms) + IFUNC_IMPL_ADD (array, i, __memmove_chk, + CPU_FEATURE_USABLE (SSSE3), + __memmove_chk_ssse3_back) +@@ -102,6 +108,12 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, memmove, + CPU_FEATURE_USABLE (AVX), + __memmove_avx_unaligned_erms) ++ IFUNC_IMPL_ADD (array, i, memmove, ++ CPU_FEATURE_USABLE (AVX512VL), ++ __memmove_evex_unaligned) ++ IFUNC_IMPL_ADD (array, i, memmove, ++ CPU_FEATURE_USABLE (AVX512VL), ++ __memmove_evex_unaligned_erms) + IFUNC_IMPL_ADD (array, i, memmove, + CPU_FEATURE_USABLE (AVX512F), + __memmove_avx512_no_vzeroupper) +@@ -565,6 +577,12 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, __memcpy_chk, + CPU_FEATURE_USABLE (AVX), + __memcpy_chk_avx_unaligned_erms) ++ IFUNC_IMPL_ADD (array, i, __memcpy_chk, ++ CPU_FEATURE_USABLE (AVX512VL), ++ __memcpy_chk_evex_unaligned) ++ IFUNC_IMPL_ADD (array, i, __memcpy_chk, ++ CPU_FEATURE_USABLE (AVX512VL), ++ __memcpy_chk_evex_unaligned_erms) + IFUNC_IMPL_ADD (array, i, __memcpy_chk, + CPU_FEATURE_USABLE (SSSE3), + __memcpy_chk_ssse3_back) +@@ -587,6 +605,12 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, memcpy, + CPU_FEATURE_USABLE (AVX), + __memcpy_avx_unaligned_erms) ++ IFUNC_IMPL_ADD (array, i, memcpy, ++ CPU_FEATURE_USABLE (AVX512VL), ++ __memcpy_evex_unaligned) ++ IFUNC_IMPL_ADD (array, i, memcpy, ++ CPU_FEATURE_USABLE (AVX512VL), ++ __memcpy_evex_unaligned_erms) + IFUNC_IMPL_ADD (array, i, memcpy, CPU_FEATURE_USABLE (SSSE3), + __memcpy_ssse3_back) + IFUNC_IMPL_ADD (array, i, memcpy, CPU_FEATURE_USABLE (SSSE3), +@@ -623,6 +647,12 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, __mempcpy_chk, + CPU_FEATURE_USABLE (AVX), + __mempcpy_chk_avx_unaligned_erms) ++ IFUNC_IMPL_ADD (array, i, __mempcpy_chk, ++ CPU_FEATURE_USABLE (AVX512VL), ++ __mempcpy_chk_evex_unaligned) ++ IFUNC_IMPL_ADD (array, i, __mempcpy_chk, ++ CPU_FEATURE_USABLE (AVX512VL), ++ __mempcpy_chk_evex_unaligned_erms) + IFUNC_IMPL_ADD (array, i, __mempcpy_chk, + CPU_FEATURE_USABLE (SSSE3), + __mempcpy_chk_ssse3_back) +@@ -654,6 +684,12 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, mempcpy, + CPU_FEATURE_USABLE (AVX), + __mempcpy_avx_unaligned_erms) ++ IFUNC_IMPL_ADD (array, i, mempcpy, ++ CPU_FEATURE_USABLE (AVX512VL), ++ __mempcpy_evex_unaligned) ++ IFUNC_IMPL_ADD (array, i, mempcpy, ++ CPU_FEATURE_USABLE (AVX512VL), ++ __mempcpy_evex_unaligned_erms) + IFUNC_IMPL_ADD (array, i, mempcpy, CPU_FEATURE_USABLE (SSSE3), + __mempcpy_ssse3_back) + IFUNC_IMPL_ADD (array, i, mempcpy, CPU_FEATURE_USABLE (SSSE3), +diff --git a/sysdeps/x86_64/multiarch/ifunc-memmove.h b/sysdeps/x86_64/multiarch/ifunc-memmove.h +index 5e5f0299..6f8bce5f 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-memmove.h ++++ b/sysdeps/x86_64/multiarch/ifunc-memmove.h +@@ -29,6 +29,10 @@ extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3_back) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned_erms) + attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_unaligned) ++ attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_unaligned_erms) ++ attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_unaligned) + attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_unaligned_erms) +@@ -59,10 +63,21 @@ IFUNC_SELECTOR (void) + + if (CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) + { +- if (CPU_FEATURE_USABLE_P (cpu_features, ERMS)) +- return OPTIMIZE (avx_unaligned_erms); ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)) ++ { ++ if (CPU_FEATURE_USABLE_P (cpu_features, ERMS)) ++ return OPTIMIZE (evex_unaligned_erms); ++ ++ return OPTIMIZE (evex_unaligned); ++ } ++ ++ if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) ++ { ++ if (CPU_FEATURE_USABLE_P (cpu_features, ERMS)) ++ return OPTIMIZE (avx_unaligned_erms); + +- return OPTIMIZE (avx_unaligned); ++ return OPTIMIZE (avx_unaligned); ++ } + } + + if (!CPU_FEATURE_USABLE_P (cpu_features, SSSE3) +diff --git a/sysdeps/x86_64/multiarch/memmove-evex-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-evex-unaligned-erms.S +new file mode 100644 +index 00000000..0cbce8f9 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/memmove-evex-unaligned-erms.S +@@ -0,0 +1,33 @@ ++#if IS_IN (libc) ++# define VEC_SIZE 32 ++# define XMM0 xmm16 ++# define XMM1 xmm17 ++# define YMM0 ymm16 ++# define YMM1 ymm17 ++# define VEC0 ymm16 ++# define VEC1 ymm17 ++# define VEC2 ymm18 ++# define VEC3 ymm19 ++# define VEC4 ymm20 ++# define VEC5 ymm21 ++# define VEC6 ymm22 ++# define VEC7 ymm23 ++# define VEC8 ymm24 ++# define VEC9 ymm25 ++# define VEC10 ymm26 ++# define VEC11 ymm27 ++# define VEC12 ymm28 ++# define VEC13 ymm29 ++# define VEC14 ymm30 ++# define VEC15 ymm31 ++# define VEC(i) VEC##i ++# define VMOVNT vmovntdq ++# define VMOVU vmovdqu64 ++# define VMOVA vmovdqa64 ++# define VZEROUPPER ++ ++# define SECTION(p) p##.evex ++# define MEMMOVE_SYMBOL(p,s) p##_evex_##s ++ ++# include "memmove-vec-unaligned-erms.S" ++#endif +diff --git a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S +index 274aa1c7..08e21692 100644 +--- a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S ++++ b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S +@@ -48,6 +48,14 @@ + # define MEMMOVE_CHK_SYMBOL(p,s) MEMMOVE_SYMBOL(p, s) + #endif + ++#ifndef XMM0 ++# define XMM0 xmm0 ++#endif ++ ++#ifndef YMM0 ++# define YMM0 ymm0 ++#endif ++ + #ifndef VZEROUPPER + # if VEC_SIZE > 16 + # define VZEROUPPER vzeroupper +@@ -277,20 +285,20 @@ L(less_vec): + #if VEC_SIZE > 32 + L(between_32_63): + /* From 32 to 63. No branch when size == 32. */ +- vmovdqu (%rsi), %ymm0 +- vmovdqu -32(%rsi,%rdx), %ymm1 +- vmovdqu %ymm0, (%rdi) +- vmovdqu %ymm1, -32(%rdi,%rdx) ++ VMOVU (%rsi), %YMM0 ++ VMOVU -32(%rsi,%rdx), %YMM1 ++ VMOVU %YMM0, (%rdi) ++ VMOVU %YMM1, -32(%rdi,%rdx) + VZEROUPPER + ret + #endif + #if VEC_SIZE > 16 + /* From 16 to 31. No branch when size == 16. */ + L(between_16_31): +- vmovdqu (%rsi), %xmm0 +- vmovdqu -16(%rsi,%rdx), %xmm1 +- vmovdqu %xmm0, (%rdi) +- vmovdqu %xmm1, -16(%rdi,%rdx) ++ VMOVU (%rsi), %XMM0 ++ VMOVU -16(%rsi,%rdx), %XMM1 ++ VMOVU %XMM0, (%rdi) ++ VMOVU %XMM1, -16(%rdi,%rdx) + ret + #endif + L(between_8_15): +-- +GitLab + diff --git a/SOURCES/ia-upd-256bit-evex-instr-4.patch b/SOURCES/ia-upd-256bit-evex-instr-4.patch new file mode 100644 index 0000000..4fcabfb --- /dev/null +++ b/SOURCES/ia-upd-256bit-evex-instr-4.patch @@ -0,0 +1,255 @@ +From 3d5101ddb7a4004459ca3f894caa47cfe9208be6 Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Fri, 5 Mar 2021 07:15:03 -0800 +Subject: [PATCH] x86-64: Add memset family functions with 256-bit EVEX + +Update ifunc-memset.h/ifunc-wmemset.h to select the function optimized +with 256-bit EVEX instructions using YMM16-YMM31 registers to avoid RTM +abort with usable AVX512VL and AVX512BW since VZEROUPPER isn't needed at +function exit. + +(cherry picked from commit 1b968b6b9b3aac702ac2f133e0dd16cfdbb415ee) +--- + sysdeps/x86_64/multiarch/Makefile | 1 + + sysdeps/x86_64/multiarch/ifunc-impl-list.c | 22 +++++++++++++++++ + sysdeps/x86_64/multiarch/ifunc-memset.h | 24 +++++++++++++++---- + sysdeps/x86_64/multiarch/ifunc-wmemset.h | 13 ++++++---- + .../multiarch/memset-evex-unaligned-erms.S | 24 +++++++++++++++++++ + .../multiarch/memset-vec-unaligned-erms.S | 20 +++++++++++----- + 6 files changed, 90 insertions(+), 14 deletions(-) + create mode 100644 sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S + +diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile +index 4563fc56..1cc0a10e 100644 +--- a/sysdeps/x86_64/multiarch/Makefile ++++ b/sysdeps/x86_64/multiarch/Makefile +@@ -43,6 +43,7 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c \ + memchr-evex \ + memmove-evex-unaligned-erms \ + memrchr-evex \ ++ memset-evex-unaligned-erms \ + rawmemchr-evex \ + stpcpy-evex \ + stpncpy-evex \ +diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c +index 6bd3abfc..7cf83485 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c ++++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c +@@ -160,6 +160,14 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, __memset_chk, + CPU_FEATURE_USABLE (AVX2), + __memset_chk_avx2_unaligned_erms) ++ IFUNC_IMPL_ADD (array, i, __memset_chk, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW)), ++ __memset_chk_evex_unaligned) ++ IFUNC_IMPL_ADD (array, i, __memset_chk, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW)), ++ __memset_chk_evex_unaligned_erms) + IFUNC_IMPL_ADD (array, i, __memset_chk, + CPU_FEATURE_USABLE (AVX512F), + __memset_chk_avx512_unaligned_erms) +@@ -185,6 +193,14 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, memset, + CPU_FEATURE_USABLE (AVX2), + __memset_avx2_unaligned_erms) ++ IFUNC_IMPL_ADD (array, i, memset, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW)), ++ __memset_evex_unaligned) ++ IFUNC_IMPL_ADD (array, i, memset, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW)), ++ __memset_evex_unaligned_erms) + IFUNC_IMPL_ADD (array, i, memset, + CPU_FEATURE_USABLE (AVX512F), + __memset_avx512_unaligned_erms) +@@ -555,6 +571,9 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, wmemset, + CPU_FEATURE_USABLE (AVX2), + __wmemset_avx2_unaligned) ++ IFUNC_IMPL_ADD (array, i, wmemset, ++ CPU_FEATURE_USABLE (AVX512VL), ++ __wmemset_evex_unaligned) + IFUNC_IMPL_ADD (array, i, wmemset, + CPU_FEATURE_USABLE (AVX512F), + __wmemset_avx512_unaligned)) +@@ -723,6 +742,9 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, __wmemset_chk, + CPU_FEATURE_USABLE (AVX2), + __wmemset_chk_avx2_unaligned) ++ IFUNC_IMPL_ADD (array, i, __wmemset_chk, ++ CPU_FEATURE_USABLE (AVX512VL), ++ __wmemset_chk_evex_unaligned) + IFUNC_IMPL_ADD (array, i, __wmemset_chk, + CPU_FEATURE_USABLE (AVX512F), + __wmemset_chk_avx512_unaligned)) +diff --git a/sysdeps/x86_64/multiarch/ifunc-memset.h b/sysdeps/x86_64/multiarch/ifunc-memset.h +index 708bd72e..6f31f4dc 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-memset.h ++++ b/sysdeps/x86_64/multiarch/ifunc-memset.h +@@ -27,6 +27,10 @@ extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned_erms) + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_unaligned) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_unaligned_erms) + attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_unaligned) ++ attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_unaligned_erms) ++ attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_unaligned) + attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_unaligned_erms) +@@ -56,10 +60,22 @@ IFUNC_SELECTOR (void) + + if (CPU_FEATURE_USABLE_P (cpu_features, AVX2)) + { +- if (CPU_FEATURE_USABLE_P (cpu_features, ERMS)) +- return OPTIMIZE (avx2_unaligned_erms); +- else +- return OPTIMIZE (avx2_unaligned); ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) ++ && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)) ++ { ++ if (CPU_FEATURE_USABLE_P (cpu_features, ERMS)) ++ return OPTIMIZE (evex_unaligned_erms); ++ ++ return OPTIMIZE (evex_unaligned); ++ } ++ ++ if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) ++ { ++ if (CPU_FEATURE_USABLE_P (cpu_features, ERMS)) ++ return OPTIMIZE (avx2_unaligned_erms); ++ ++ return OPTIMIZE (avx2_unaligned); ++ } + } + + if (CPU_FEATURE_USABLE_P (cpu_features, ERMS)) +diff --git a/sysdeps/x86_64/multiarch/ifunc-wmemset.h b/sysdeps/x86_64/multiarch/ifunc-wmemset.h +index eb242210..9290c4bf 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-wmemset.h ++++ b/sysdeps/x86_64/multiarch/ifunc-wmemset.h +@@ -20,6 +20,7 @@ + + extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_unaligned) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_unaligned) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_unaligned) attribute_hidden; + + static inline void * +@@ -27,14 +28,18 @@ IFUNC_SELECTOR (void) + { + const struct cpu_features* cpu_features = __get_cpu_features (); + +- if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER) +- && CPU_FEATURE_USABLE_P (cpu_features, AVX2) ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX2) + && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) + { + if (CPU_FEATURE_USABLE_P (cpu_features, AVX512F) +- && !CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_AVX512)) ++ && !CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_AVX512) ++ && !CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) + return OPTIMIZE (avx512_unaligned); +- else ++ ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)) ++ return OPTIMIZE (evex_unaligned); ++ ++ if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) + return OPTIMIZE (avx2_unaligned); + } + +diff --git a/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S +new file mode 100644 +index 00000000..ae0a4d6e +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S +@@ -0,0 +1,24 @@ ++#if IS_IN (libc) ++# define VEC_SIZE 32 ++# define XMM0 xmm16 ++# define YMM0 ymm16 ++# define VEC0 ymm16 ++# define VEC(i) VEC##i ++# define VMOVU vmovdqu64 ++# define VMOVA vmovdqa64 ++# define VZEROUPPER ++ ++# define MEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ ++ movq r, %rax; \ ++ vpbroadcastb d, %VEC0 ++ ++# define WMEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ ++ movq r, %rax; \ ++ vpbroadcastd d, %VEC0 ++ ++# define SECTION(p) p##.evex ++# define MEMSET_SYMBOL(p,s) p##_evex_##s ++# define WMEMSET_SYMBOL(p,s) p##_evex_##s ++ ++# include "memset-vec-unaligned-erms.S" ++#endif +diff --git a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S +index 9a0fd818..71e91a8f 100644 +--- a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S ++++ b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S +@@ -34,6 +34,14 @@ + # define WMEMSET_CHK_SYMBOL(p,s) WMEMSET_SYMBOL(p, s) + #endif + ++#ifndef XMM0 ++# define XMM0 xmm0 ++#endif ++ ++#ifndef YMM0 ++# define YMM0 ymm0 ++#endif ++ + #ifndef VZEROUPPER + # if VEC_SIZE > 16 + # define VZEROUPPER vzeroupper +@@ -67,7 +75,7 @@ + ENTRY (__bzero) + mov %RDI_LP, %RAX_LP /* Set return value. */ + mov %RSI_LP, %RDX_LP /* Set n. */ +- pxor %xmm0, %xmm0 ++ pxor %XMM0, %XMM0 + jmp L(entry_from_bzero) + END (__bzero) + weak_alias (__bzero, bzero) +@@ -223,7 +231,7 @@ L(less_vec): + cmpb $16, %dl + jae L(between_16_31) + # endif +- MOVQ %xmm0, %rcx ++ MOVQ %XMM0, %rcx + cmpb $8, %dl + jae L(between_8_15) + cmpb $4, %dl +@@ -238,16 +246,16 @@ L(less_vec): + # if VEC_SIZE > 32 + /* From 32 to 63. No branch when size == 32. */ + L(between_32_63): +- vmovdqu %ymm0, -32(%rdi,%rdx) +- vmovdqu %ymm0, (%rdi) ++ VMOVU %YMM0, -32(%rdi,%rdx) ++ VMOVU %YMM0, (%rdi) + VZEROUPPER + ret + # endif + # if VEC_SIZE > 16 + /* From 16 to 31. No branch when size == 16. */ + L(between_16_31): +- vmovdqu %xmm0, -16(%rdi,%rdx) +- vmovdqu %xmm0, (%rdi) ++ VMOVU %XMM0, -16(%rdi,%rdx) ++ VMOVU %XMM0, (%rdi) + VZEROUPPER + ret + # endif +-- +GitLab + diff --git a/SOURCES/ia-upd-256bit-evex-instr-5.patch b/SOURCES/ia-upd-256bit-evex-instr-5.patch new file mode 100644 index 0000000..a2d8191 --- /dev/null +++ b/SOURCES/ia-upd-256bit-evex-instr-5.patch @@ -0,0 +1,562 @@ +From cd2eeb1be618b5edfc9c6929c07201ff941b31d9 Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Fri, 5 Mar 2021 07:20:28 -0800 +Subject: [PATCH] x86-64: Add memcmp family functions with 256-bit EVEX + +Update ifunc-memcmp.h to select the function optimized with 256-bit EVEX +instructions using YMM16-YMM31 registers to avoid RTM abort with usable +AVX512VL, AVX512BW and MOVBE since VZEROUPPER isn't needed at function +exit. + +(cherry picked from commit 91264fe3577fe887b4860923fa6142b5274c8965) +--- + sysdeps/x86_64/multiarch/Makefile | 4 +- + sysdeps/x86_64/multiarch/ifunc-impl-list.c | 10 + + sysdeps/x86_64/multiarch/ifunc-memcmp.h | 13 +- + sysdeps/x86_64/multiarch/memcmp-evex-movbe.S | 440 ++++++++++++++++++ + sysdeps/x86_64/multiarch/wmemcmp-evex-movbe.S | 4 + + 5 files changed, 467 insertions(+), 4 deletions(-) + create mode 100644 sysdeps/x86_64/multiarch/memcmp-evex-movbe.S + create mode 100644 sysdeps/x86_64/multiarch/wmemcmp-evex-movbe.S + +diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile +index 1cc0a10e..9d79b138 100644 +--- a/sysdeps/x86_64/multiarch/Makefile ++++ b/sysdeps/x86_64/multiarch/Makefile +@@ -41,6 +41,7 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c \ + memset-avx2-unaligned-erms \ + memset-avx512-unaligned-erms \ + memchr-evex \ ++ memcmp-evex-movbe \ + memmove-evex-unaligned-erms \ + memrchr-evex \ + memset-evex-unaligned-erms \ +@@ -81,7 +82,8 @@ sysdep_routines += wmemcmp-sse4 wmemcmp-ssse3 wmemcmp-c \ + wcsncmp-evex \ + wcsnlen-evex \ + wcsrchr-evex \ +- wmemchr-evex ++ wmemchr-evex \ ++ wmemcmp-evex-movbe + endif + + ifeq ($(subdir),debug) +diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c +index 7cf83485..c8da910e 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c ++++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c +@@ -56,6 +56,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + (CPU_FEATURE_USABLE (AVX2) + && CPU_FEATURE_USABLE (MOVBE)), + __memcmp_avx2_movbe) ++ IFUNC_IMPL_ADD (array, i, memcmp, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (MOVBE)), ++ __memcmp_evex_movbe) + IFUNC_IMPL_ADD (array, i, memcmp, CPU_FEATURE_USABLE (SSE4_1), + __memcmp_sse4_1) + IFUNC_IMPL_ADD (array, i, memcmp, CPU_FEATURE_USABLE (SSSE3), +@@ -558,6 +563,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + (CPU_FEATURE_USABLE (AVX2) + && CPU_FEATURE_USABLE (MOVBE)), + __wmemcmp_avx2_movbe) ++ IFUNC_IMPL_ADD (array, i, wmemcmp, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (MOVBE)), ++ __wmemcmp_evex_movbe) + IFUNC_IMPL_ADD (array, i, wmemcmp, CPU_FEATURE_USABLE (SSE4_1), + __wmemcmp_sse4_1) + IFUNC_IMPL_ADD (array, i, wmemcmp, CPU_FEATURE_USABLE (SSSE3), +diff --git a/sysdeps/x86_64/multiarch/ifunc-memcmp.h b/sysdeps/x86_64/multiarch/ifunc-memcmp.h +index 6c1f3153..3ca1f0a6 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-memcmp.h ++++ b/sysdeps/x86_64/multiarch/ifunc-memcmp.h +@@ -23,17 +23,24 @@ extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (sse4_1) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_movbe) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_movbe) attribute_hidden; + + static inline void * + IFUNC_SELECTOR (void) + { + const struct cpu_features* cpu_features = __get_cpu_features (); + +- if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER) +- && CPU_FEATURE_USABLE_P (cpu_features, AVX2) ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX2) + && CPU_FEATURE_USABLE_P (cpu_features, MOVBE) + && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) +- return OPTIMIZE (avx2_movbe); ++ { ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) ++ && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)) ++ return OPTIMIZE (evex_movbe); ++ ++ if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) ++ return OPTIMIZE (avx2_movbe); ++ } + + if (CPU_FEATURE_USABLE_P (cpu_features, SSE4_1)) + return OPTIMIZE (sse4_1); +diff --git a/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S b/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S +new file mode 100644 +index 00000000..9c093972 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S +@@ -0,0 +1,440 @@ ++/* memcmp/wmemcmp optimized with 256-bit EVEX instructions. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#if IS_IN (libc) ++ ++/* memcmp/wmemcmp is implemented as: ++ 1. For size from 2 to 7 bytes, load as big endian with movbe and bswap ++ to avoid branches. ++ 2. Use overlapping compare to avoid branch. ++ 3. Use vector compare when size >= 4 bytes for memcmp or size >= 8 ++ bytes for wmemcmp. ++ 4. If size is 8 * VEC_SIZE or less, unroll the loop. ++ 5. Compare 4 * VEC_SIZE at a time with the aligned first memory ++ area. ++ 6. Use 2 vector compares when size is 2 * VEC_SIZE or less. ++ 7. Use 4 vector compares when size is 4 * VEC_SIZE or less. ++ 8. Use 8 vector compares when size is 8 * VEC_SIZE or less. */ ++ ++# include ++ ++# ifndef MEMCMP ++# define MEMCMP __memcmp_evex_movbe ++# endif ++ ++# define VMOVU vmovdqu64 ++ ++# ifdef USE_AS_WMEMCMP ++# define VPCMPEQ vpcmpeqd ++# else ++# define VPCMPEQ vpcmpeqb ++# endif ++ ++# define XMM1 xmm17 ++# define XMM2 xmm18 ++# define YMM1 ymm17 ++# define YMM2 ymm18 ++# define YMM3 ymm19 ++# define YMM4 ymm20 ++# define YMM5 ymm21 ++# define YMM6 ymm22 ++ ++# define VEC_SIZE 32 ++# ifdef USE_AS_WMEMCMP ++# define VEC_MASK 0xff ++# define XMM_MASK 0xf ++# else ++# define VEC_MASK 0xffffffff ++# define XMM_MASK 0xffff ++# endif ++ ++/* Warning! ++ wmemcmp has to use SIGNED comparison for elements. ++ memcmp has to use UNSIGNED comparison for elemnts. ++*/ ++ ++ .section .text.evex,"ax",@progbits ++ENTRY (MEMCMP) ++# ifdef USE_AS_WMEMCMP ++ shl $2, %RDX_LP ++# elif defined __ILP32__ ++ /* Clear the upper 32 bits. */ ++ movl %edx, %edx ++# endif ++ cmp $VEC_SIZE, %RDX_LP ++ jb L(less_vec) ++ ++ /* From VEC to 2 * VEC. No branch when size == VEC_SIZE. */ ++ VMOVU (%rsi), %YMM2 ++ VPCMPEQ (%rdi), %YMM2, %k1 ++ kmovd %k1, %eax ++ subl $VEC_MASK, %eax ++ jnz L(first_vec) ++ ++ cmpq $(VEC_SIZE * 2), %rdx ++ jbe L(last_vec) ++ ++ /* More than 2 * VEC. */ ++ cmpq $(VEC_SIZE * 8), %rdx ++ ja L(more_8x_vec) ++ cmpq $(VEC_SIZE * 4), %rdx ++ jb L(last_4x_vec) ++ ++ /* From 4 * VEC to 8 * VEC, inclusively. */ ++ VMOVU (%rsi), %YMM1 ++ VPCMPEQ (%rdi), %YMM1, %k1 ++ ++ VMOVU VEC_SIZE(%rsi), %YMM2 ++ VPCMPEQ VEC_SIZE(%rdi), %YMM2, %k2 ++ ++ VMOVU (VEC_SIZE * 2)(%rsi), %YMM3 ++ VPCMPEQ (VEC_SIZE * 2)(%rdi), %YMM3, %k3 ++ ++ VMOVU (VEC_SIZE * 3)(%rsi), %YMM4 ++ VPCMPEQ (VEC_SIZE * 3)(%rdi), %YMM4, %k4 ++ ++ kandd %k1, %k2, %k5 ++ kandd %k3, %k4, %k6 ++ kandd %k5, %k6, %k6 ++ ++ kmovd %k6, %eax ++ cmpl $VEC_MASK, %eax ++ jne L(4x_vec_end) ++ ++ leaq -(4 * VEC_SIZE)(%rdi, %rdx), %rdi ++ leaq -(4 * VEC_SIZE)(%rsi, %rdx), %rsi ++ VMOVU (%rsi), %YMM1 ++ VPCMPEQ (%rdi), %YMM1, %k1 ++ ++ VMOVU VEC_SIZE(%rsi), %YMM2 ++ VPCMPEQ VEC_SIZE(%rdi), %YMM2, %k2 ++ kandd %k1, %k2, %k5 ++ ++ VMOVU (VEC_SIZE * 2)(%rsi), %YMM3 ++ VPCMPEQ (VEC_SIZE * 2)(%rdi), %YMM3, %k3 ++ kandd %k3, %k5, %k5 ++ ++ VMOVU (VEC_SIZE * 3)(%rsi), %YMM4 ++ VPCMPEQ (VEC_SIZE * 3)(%rdi), %YMM4, %k4 ++ kandd %k4, %k5, %k5 ++ ++ kmovd %k5, %eax ++ cmpl $VEC_MASK, %eax ++ jne L(4x_vec_end) ++ xorl %eax, %eax ++ ret ++ ++ .p2align 4 ++L(last_2x_vec): ++ /* From VEC to 2 * VEC. No branch when size == VEC_SIZE. */ ++ VMOVU (%rsi), %YMM2 ++ VPCMPEQ (%rdi), %YMM2, %k2 ++ kmovd %k2, %eax ++ subl $VEC_MASK, %eax ++ jnz L(first_vec) ++ ++L(last_vec): ++ /* Use overlapping loads to avoid branches. */ ++ leaq -VEC_SIZE(%rdi, %rdx), %rdi ++ leaq -VEC_SIZE(%rsi, %rdx), %rsi ++ VMOVU (%rsi), %YMM2 ++ VPCMPEQ (%rdi), %YMM2, %k2 ++ kmovd %k2, %eax ++ subl $VEC_MASK, %eax ++ jnz L(first_vec) ++ ret ++ ++ .p2align 4 ++L(first_vec): ++ /* A byte or int32 is different within 16 or 32 bytes. */ ++ tzcntl %eax, %ecx ++# ifdef USE_AS_WMEMCMP ++ xorl %eax, %eax ++ movl (%rdi, %rcx, 4), %edx ++ cmpl (%rsi, %rcx, 4), %edx ++L(wmemcmp_return): ++ setl %al ++ negl %eax ++ orl $1, %eax ++# else ++ movzbl (%rdi, %rcx), %eax ++ movzbl (%rsi, %rcx), %edx ++ sub %edx, %eax ++# endif ++ ret ++ ++# ifdef USE_AS_WMEMCMP ++ .p2align 4 ++L(4): ++ xorl %eax, %eax ++ movl (%rdi), %edx ++ cmpl (%rsi), %edx ++ jne L(wmemcmp_return) ++ ret ++# else ++ .p2align 4 ++L(between_4_7): ++ /* Load as big endian with overlapping movbe to avoid branches. */ ++ movbe (%rdi), %eax ++ movbe (%rsi), %ecx ++ shlq $32, %rax ++ shlq $32, %rcx ++ movbe -4(%rdi, %rdx), %edi ++ movbe -4(%rsi, %rdx), %esi ++ orq %rdi, %rax ++ orq %rsi, %rcx ++ subq %rcx, %rax ++ je L(exit) ++ sbbl %eax, %eax ++ orl $1, %eax ++ ret ++ ++ .p2align 4 ++L(exit): ++ ret ++ ++ .p2align 4 ++L(between_2_3): ++ /* Load as big endian to avoid branches. */ ++ movzwl (%rdi), %eax ++ movzwl (%rsi), %ecx ++ shll $8, %eax ++ shll $8, %ecx ++ bswap %eax ++ bswap %ecx ++ movb -1(%rdi, %rdx), %al ++ movb -1(%rsi, %rdx), %cl ++ /* Subtraction is okay because the upper 8 bits are zero. */ ++ subl %ecx, %eax ++ ret ++ ++ .p2align 4 ++L(1): ++ movzbl (%rdi), %eax ++ movzbl (%rsi), %ecx ++ subl %ecx, %eax ++ ret ++# endif ++ ++ .p2align 4 ++L(zero): ++ xorl %eax, %eax ++ ret ++ ++ .p2align 4 ++L(less_vec): ++# ifdef USE_AS_WMEMCMP ++ /* It can only be 0, 4, 8, 12, 16, 20, 24, 28 bytes. */ ++ cmpb $4, %dl ++ je L(4) ++ jb L(zero) ++# else ++ cmpb $1, %dl ++ je L(1) ++ jb L(zero) ++ cmpb $4, %dl ++ jb L(between_2_3) ++ cmpb $8, %dl ++ jb L(between_4_7) ++# endif ++ cmpb $16, %dl ++ jae L(between_16_31) ++ /* It is between 8 and 15 bytes. */ ++ vmovq (%rdi), %XMM1 ++ vmovq (%rsi), %XMM2 ++ VPCMPEQ %XMM1, %XMM2, %k2 ++ kmovw %k2, %eax ++ subl $XMM_MASK, %eax ++ jnz L(first_vec) ++ /* Use overlapping loads to avoid branches. */ ++ leaq -8(%rdi, %rdx), %rdi ++ leaq -8(%rsi, %rdx), %rsi ++ vmovq (%rdi), %XMM1 ++ vmovq (%rsi), %XMM2 ++ VPCMPEQ %XMM1, %XMM2, %k2 ++ kmovw %k2, %eax ++ subl $XMM_MASK, %eax ++ jnz L(first_vec) ++ ret ++ ++ .p2align 4 ++L(between_16_31): ++ /* From 16 to 31 bytes. No branch when size == 16. */ ++ VMOVU (%rsi), %XMM2 ++ VPCMPEQ (%rdi), %XMM2, %k2 ++ kmovw %k2, %eax ++ subl $XMM_MASK, %eax ++ jnz L(first_vec) ++ ++ /* Use overlapping loads to avoid branches. */ ++ leaq -16(%rdi, %rdx), %rdi ++ leaq -16(%rsi, %rdx), %rsi ++ VMOVU (%rsi), %XMM2 ++ VPCMPEQ (%rdi), %XMM2, %k2 ++ kmovw %k2, %eax ++ subl $XMM_MASK, %eax ++ jnz L(first_vec) ++ ret ++ ++ .p2align 4 ++L(more_8x_vec): ++ /* More than 8 * VEC. Check the first VEC. */ ++ VMOVU (%rsi), %YMM2 ++ VPCMPEQ (%rdi), %YMM2, %k2 ++ kmovd %k2, %eax ++ subl $VEC_MASK, %eax ++ jnz L(first_vec) ++ ++ /* Align the first memory area for aligned loads in the loop. ++ Compute how much the first memory area is misaligned. */ ++ movq %rdi, %rcx ++ andl $(VEC_SIZE - 1), %ecx ++ /* Get the negative of offset for alignment. */ ++ subq $VEC_SIZE, %rcx ++ /* Adjust the second memory area. */ ++ subq %rcx, %rsi ++ /* Adjust the first memory area which should be aligned now. */ ++ subq %rcx, %rdi ++ /* Adjust length. */ ++ addq %rcx, %rdx ++ ++L(loop_4x_vec): ++ /* Compare 4 * VEC at a time forward. */ ++ VMOVU (%rsi), %YMM1 ++ VPCMPEQ (%rdi), %YMM1, %k1 ++ ++ VMOVU VEC_SIZE(%rsi), %YMM2 ++ VPCMPEQ VEC_SIZE(%rdi), %YMM2, %k2 ++ kandd %k2, %k1, %k5 ++ ++ VMOVU (VEC_SIZE * 2)(%rsi), %YMM3 ++ VPCMPEQ (VEC_SIZE * 2)(%rdi), %YMM3, %k3 ++ kandd %k3, %k5, %k5 ++ ++ VMOVU (VEC_SIZE * 3)(%rsi), %YMM4 ++ VPCMPEQ (VEC_SIZE * 3)(%rdi), %YMM4, %k4 ++ kandd %k4, %k5, %k5 ++ ++ kmovd %k5, %eax ++ cmpl $VEC_MASK, %eax ++ jne L(4x_vec_end) ++ ++ addq $(VEC_SIZE * 4), %rdi ++ addq $(VEC_SIZE * 4), %rsi ++ ++ subq $(VEC_SIZE * 4), %rdx ++ cmpq $(VEC_SIZE * 4), %rdx ++ jae L(loop_4x_vec) ++ ++ /* Less than 4 * VEC. */ ++ cmpq $VEC_SIZE, %rdx ++ jbe L(last_vec) ++ cmpq $(VEC_SIZE * 2), %rdx ++ jbe L(last_2x_vec) ++ ++L(last_4x_vec): ++ /* From 2 * VEC to 4 * VEC. */ ++ VMOVU (%rsi), %YMM2 ++ VPCMPEQ (%rdi), %YMM2, %k2 ++ kmovd %k2, %eax ++ subl $VEC_MASK, %eax ++ jnz L(first_vec) ++ ++ addq $VEC_SIZE, %rdi ++ addq $VEC_SIZE, %rsi ++ VMOVU (%rsi), %YMM2 ++ VPCMPEQ (%rdi), %YMM2, %k2 ++ kmovd %k2, %eax ++ subl $VEC_MASK, %eax ++ jnz L(first_vec) ++ ++ /* Use overlapping loads to avoid branches. */ ++ leaq -(3 * VEC_SIZE)(%rdi, %rdx), %rdi ++ leaq -(3 * VEC_SIZE)(%rsi, %rdx), %rsi ++ VMOVU (%rsi), %YMM2 ++ VPCMPEQ (%rdi), %YMM2, %k2 ++ kmovd %k2, %eax ++ subl $VEC_MASK, %eax ++ jnz L(first_vec) ++ ++ addq $VEC_SIZE, %rdi ++ addq $VEC_SIZE, %rsi ++ VMOVU (%rsi), %YMM2 ++ VPCMPEQ (%rdi), %YMM2, %k2 ++ kmovd %k2, %eax ++ subl $VEC_MASK, %eax ++ jnz L(first_vec) ++ ret ++ ++ .p2align 4 ++L(4x_vec_end): ++ kmovd %k1, %eax ++ subl $VEC_MASK, %eax ++ jnz L(first_vec) ++ kmovd %k2, %eax ++ subl $VEC_MASK, %eax ++ jnz L(first_vec_x1) ++ kmovd %k3, %eax ++ subl $VEC_MASK, %eax ++ jnz L(first_vec_x2) ++ kmovd %k4, %eax ++ subl $VEC_MASK, %eax ++ tzcntl %eax, %ecx ++# ifdef USE_AS_WMEMCMP ++ xorl %eax, %eax ++ movl (VEC_SIZE * 3)(%rdi, %rcx, 4), %edx ++ cmpl (VEC_SIZE * 3)(%rsi, %rcx, 4), %edx ++ jmp L(wmemcmp_return) ++# else ++ movzbl (VEC_SIZE * 3)(%rdi, %rcx), %eax ++ movzbl (VEC_SIZE * 3)(%rsi, %rcx), %edx ++ sub %edx, %eax ++# endif ++ ret ++ ++ .p2align 4 ++L(first_vec_x1): ++ tzcntl %eax, %ecx ++# ifdef USE_AS_WMEMCMP ++ xorl %eax, %eax ++ movl VEC_SIZE(%rdi, %rcx, 4), %edx ++ cmpl VEC_SIZE(%rsi, %rcx, 4), %edx ++ jmp L(wmemcmp_return) ++# else ++ movzbl VEC_SIZE(%rdi, %rcx), %eax ++ movzbl VEC_SIZE(%rsi, %rcx), %edx ++ sub %edx, %eax ++# endif ++ ret ++ ++ .p2align 4 ++L(first_vec_x2): ++ tzcntl %eax, %ecx ++# ifdef USE_AS_WMEMCMP ++ xorl %eax, %eax ++ movl (VEC_SIZE * 2)(%rdi, %rcx, 4), %edx ++ cmpl (VEC_SIZE * 2)(%rsi, %rcx, 4), %edx ++ jmp L(wmemcmp_return) ++# else ++ movzbl (VEC_SIZE * 2)(%rdi, %rcx), %eax ++ movzbl (VEC_SIZE * 2)(%rsi, %rcx), %edx ++ sub %edx, %eax ++# endif ++ ret ++END (MEMCMP) ++#endif +diff --git a/sysdeps/x86_64/multiarch/wmemcmp-evex-movbe.S b/sysdeps/x86_64/multiarch/wmemcmp-evex-movbe.S +new file mode 100644 +index 00000000..4726d74a +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/wmemcmp-evex-movbe.S +@@ -0,0 +1,4 @@ ++#define MEMCMP __wmemcmp_evex_movbe ++#define USE_AS_WMEMCMP 1 ++ ++#include "memcmp-evex-movbe.S" +-- +GitLab + diff --git a/SOURCES/ia-upd-large-memcpy.patch b/SOURCES/ia-upd-large-memcpy.patch new file mode 100644 index 0000000..2ae72f0 --- /dev/null +++ b/SOURCES/ia-upd-large-memcpy.patch @@ -0,0 +1,445 @@ +From 8f4f10bfb4793f47accee8ea86438879a889b595 Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Wed, 2 Mar 2022 15:20:14 -0800 +Subject: [PATCH] x86: Update large memcpy case in memmove-vec-unaligned-erms.S + +No Bug. This commit updates the large memcpy case (no overlap). The +update is to perform memcpy on either 2 or 4 contiguous pages at +once. This 1) helps to alleviate the affects of false memory aliasing +when destination and source have a close 4k alignment and 2) In most +cases and for most DRAM units is a modestly more efficient access +pattern. These changes are a clear performance improvement for +VEC_SIZE =16/32, though more ambiguous for VEC_SIZE=64. test-memcpy, +test-memccpy, test-mempcpy, test-memmove, and tst-memmove-overflow all +pass. + +Signed-off-by: Noah Goldstein +(cherry picked from commit 1a8605b6cd257e8a74e29b5b71c057211f5fb847) +--- + .../multiarch/memmove-vec-unaligned-erms.S | 338 ++++++++++++++---- + 1 file changed, 265 insertions(+), 73 deletions(-) + +diff --git a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S +index c475fed4..3e2dd6bc 100644 +--- a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S ++++ b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S +@@ -32,7 +32,16 @@ + overlapping addresses. + 6. If size >= __x86_shared_non_temporal_threshold and there is no + overlap between destination and source, use non-temporal store +- instead of aligned store. */ ++ instead of aligned store copying from either 2 or 4 pages at ++ once. ++ 8. For point 7) if size < 16 * __x86_shared_non_temporal_threshold ++ and source and destination do not page alias, copy from 2 pages ++ at once using non-temporal stores. Page aliasing in this case is ++ considered true if destination's page alignment - sources' page ++ alignment is less than 8 * VEC_SIZE. ++ 9. If size >= 16 * __x86_shared_non_temporal_threshold or source ++ and destination do page alias copy from 4 pages at once using ++ non-temporal stores. */ + + #include + +@@ -64,6 +73,34 @@ + # endif + #endif + ++#ifndef PAGE_SIZE ++# define PAGE_SIZE 4096 ++#endif ++ ++#if PAGE_SIZE != 4096 ++# error Unsupported PAGE_SIZE ++#endif ++ ++#ifndef LOG_PAGE_SIZE ++# define LOG_PAGE_SIZE 12 ++#endif ++ ++#if PAGE_SIZE != (1 << LOG_PAGE_SIZE) ++# error Invalid LOG_PAGE_SIZE ++#endif ++ ++/* Byte per page for large_memcpy inner loop. */ ++#if VEC_SIZE == 64 ++# define LARGE_LOAD_SIZE (VEC_SIZE * 2) ++#else ++# define LARGE_LOAD_SIZE (VEC_SIZE * 4) ++#endif ++ ++/* Amount to shift rdx by to compare for memcpy_large_4x. */ ++#ifndef LOG_4X_MEMCPY_THRESH ++# define LOG_4X_MEMCPY_THRESH 4 ++#endif ++ + /* Avoid short distance rep movsb only with non-SSE vector. */ + #ifndef AVOID_SHORT_DISTANCE_REP_MOVSB + # define AVOID_SHORT_DISTANCE_REP_MOVSB (VEC_SIZE > 16) +@@ -103,6 +140,28 @@ + # error Unsupported PREFETCH_SIZE! + #endif + ++#if LARGE_LOAD_SIZE == (VEC_SIZE * 2) ++# define LOAD_ONE_SET(base, offset, vec0, vec1, ...) \ ++ VMOVU (offset)base, vec0; \ ++ VMOVU ((offset) + VEC_SIZE)base, vec1; ++# define STORE_ONE_SET(base, offset, vec0, vec1, ...) \ ++ VMOVNT vec0, (offset)base; \ ++ VMOVNT vec1, ((offset) + VEC_SIZE)base; ++#elif LARGE_LOAD_SIZE == (VEC_SIZE * 4) ++# define LOAD_ONE_SET(base, offset, vec0, vec1, vec2, vec3) \ ++ VMOVU (offset)base, vec0; \ ++ VMOVU ((offset) + VEC_SIZE)base, vec1; \ ++ VMOVU ((offset) + VEC_SIZE * 2)base, vec2; \ ++ VMOVU ((offset) + VEC_SIZE * 3)base, vec3; ++# define STORE_ONE_SET(base, offset, vec0, vec1, vec2, vec3) \ ++ VMOVNT vec0, (offset)base; \ ++ VMOVNT vec1, ((offset) + VEC_SIZE)base; \ ++ VMOVNT vec2, ((offset) + VEC_SIZE * 2)base; \ ++ VMOVNT vec3, ((offset) + VEC_SIZE * 3)base; ++#else ++# error Invalid LARGE_LOAD_SIZE ++#endif ++ + #ifndef SECTION + # error SECTION is not defined! + #endif +@@ -390,6 +449,15 @@ L(last_4x_vec): + VZEROUPPER_RETURN + + L(more_8x_vec): ++ /* Check if non-temporal move candidate. */ ++#if (defined USE_MULTIARCH || VEC_SIZE == 16) && IS_IN (libc) ++ /* Check non-temporal store threshold. */ ++ cmp __x86_shared_non_temporal_threshold(%rip), %RDX_LP ++ ja L(large_memcpy_2x) ++#endif ++ /* Entry if rdx is greater than non-temporal threshold but there ++ is overlap. */ ++L(more_8x_vec_check): + cmpq %rsi, %rdi + ja L(more_8x_vec_backward) + /* Source == destination is less common. */ +@@ -416,24 +484,21 @@ L(more_8x_vec): + subq %r8, %rdi + /* Adjust length. */ + addq %r8, %rdx +-#if (defined USE_MULTIARCH || VEC_SIZE == 16) && IS_IN (libc) +- /* Check non-temporal store threshold. */ +- cmp __x86_shared_non_temporal_threshold(%rip), %RDX_LP +- ja L(large_forward) +-#endif ++ ++ .p2align 4 + L(loop_4x_vec_forward): + /* Copy 4 * VEC a time forward. */ + VMOVU (%rsi), %VEC(0) + VMOVU VEC_SIZE(%rsi), %VEC(1) + VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2) + VMOVU (VEC_SIZE * 3)(%rsi), %VEC(3) +- addq $(VEC_SIZE * 4), %rsi +- subq $(VEC_SIZE * 4), %rdx ++ subq $-(VEC_SIZE * 4), %rsi ++ addq $-(VEC_SIZE * 4), %rdx + VMOVA %VEC(0), (%rdi) + VMOVA %VEC(1), VEC_SIZE(%rdi) + VMOVA %VEC(2), (VEC_SIZE * 2)(%rdi) + VMOVA %VEC(3), (VEC_SIZE * 3)(%rdi) +- addq $(VEC_SIZE * 4), %rdi ++ subq $-(VEC_SIZE * 4), %rdi + cmpq $(VEC_SIZE * 4), %rdx + ja L(loop_4x_vec_forward) + /* Store the last 4 * VEC. */ +@@ -467,24 +532,21 @@ L(more_8x_vec_backward): + subq %r8, %r9 + /* Adjust length. */ + subq %r8, %rdx +-#if (defined USE_MULTIARCH || VEC_SIZE == 16) && IS_IN (libc) +- /* Check non-temporal store threshold. */ +- cmp __x86_shared_non_temporal_threshold(%rip), %RDX_LP +- ja L(large_backward) +-#endif ++ ++ .p2align 4 + L(loop_4x_vec_backward): + /* Copy 4 * VEC a time backward. */ + VMOVU (%rcx), %VEC(0) + VMOVU -VEC_SIZE(%rcx), %VEC(1) + VMOVU -(VEC_SIZE * 2)(%rcx), %VEC(2) + VMOVU -(VEC_SIZE * 3)(%rcx), %VEC(3) +- subq $(VEC_SIZE * 4), %rcx +- subq $(VEC_SIZE * 4), %rdx ++ addq $-(VEC_SIZE * 4), %rcx ++ addq $-(VEC_SIZE * 4), %rdx + VMOVA %VEC(0), (%r9) + VMOVA %VEC(1), -VEC_SIZE(%r9) + VMOVA %VEC(2), -(VEC_SIZE * 2)(%r9) + VMOVA %VEC(3), -(VEC_SIZE * 3)(%r9) +- subq $(VEC_SIZE * 4), %r9 ++ addq $-(VEC_SIZE * 4), %r9 + cmpq $(VEC_SIZE * 4), %rdx + ja L(loop_4x_vec_backward) + /* Store the first 4 * VEC. */ +@@ -497,72 +559,202 @@ L(loop_4x_vec_backward): + VZEROUPPER_RETURN + + #if (defined USE_MULTIARCH || VEC_SIZE == 16) && IS_IN (libc) +-L(large_forward): ++ .p2align 4 ++L(large_memcpy_2x): ++ /* Compute absolute value of difference between source and ++ destination. */ ++ movq %rdi, %r9 ++ subq %rsi, %r9 ++ movq %r9, %r8 ++ leaq -1(%r9), %rcx ++ sarq $63, %r8 ++ xorq %r8, %r9 ++ subq %r8, %r9 + /* Don't use non-temporal store if there is overlap between +- destination and source since destination may be in cache +- when source is loaded. */ +- leaq (%rdi, %rdx), %r10 +- cmpq %r10, %rsi +- jb L(loop_4x_vec_forward) +-L(loop_large_forward): ++ destination and source since destination may be in cache when ++ source is loaded. */ ++ cmpq %r9, %rdx ++ ja L(more_8x_vec_check) ++ ++ /* Cache align destination. First store the first 64 bytes then ++ adjust alignments. */ ++ VMOVU (%rsi), %VEC(8) ++#if VEC_SIZE < 64 ++ VMOVU VEC_SIZE(%rsi), %VEC(9) ++#if VEC_SIZE < 32 ++ VMOVU (VEC_SIZE * 2)(%rsi), %VEC(10) ++ VMOVU (VEC_SIZE * 3)(%rsi), %VEC(11) ++#endif ++#endif ++ VMOVU %VEC(8), (%rdi) ++#if VEC_SIZE < 64 ++ VMOVU %VEC(9), VEC_SIZE(%rdi) ++#if VEC_SIZE < 32 ++ VMOVU %VEC(10), (VEC_SIZE * 2)(%rdi) ++ VMOVU %VEC(11), (VEC_SIZE * 3)(%rdi) ++#endif ++#endif ++ /* Adjust source, destination, and size. */ ++ movq %rdi, %r8 ++ andq $63, %r8 ++ /* Get the negative of offset for alignment. */ ++ subq $64, %r8 ++ /* Adjust source. */ ++ subq %r8, %rsi ++ /* Adjust destination which should be aligned now. */ ++ subq %r8, %rdi ++ /* Adjust length. */ ++ addq %r8, %rdx ++ ++ /* Test if source and destination addresses will alias. If they do ++ the larger pipeline in large_memcpy_4x alleviated the ++ performance drop. */ ++ testl $(PAGE_SIZE - VEC_SIZE * 8), %ecx ++ jz L(large_memcpy_4x) ++ ++ movq %rdx, %r10 ++ shrq $LOG_4X_MEMCPY_THRESH, %r10 ++ cmp __x86_shared_non_temporal_threshold(%rip), %r10 ++ jae L(large_memcpy_4x) ++ ++ /* edx will store remainder size for copying tail. */ ++ andl $(PAGE_SIZE * 2 - 1), %edx ++ /* r10 stores outer loop counter. */ ++ shrq $((LOG_PAGE_SIZE + 1) - LOG_4X_MEMCPY_THRESH), %r10 ++ /* Copy 4x VEC at a time from 2 pages. */ ++ .p2align 4 ++L(loop_large_memcpy_2x_outer): ++ /* ecx stores inner loop counter. */ ++ movl $(PAGE_SIZE / LARGE_LOAD_SIZE), %ecx ++L(loop_large_memcpy_2x_inner): ++ PREFETCH_ONE_SET(1, (%rsi), PREFETCHED_LOAD_SIZE) ++ PREFETCH_ONE_SET(1, (%rsi), PREFETCHED_LOAD_SIZE * 2) ++ PREFETCH_ONE_SET(1, (%rsi), PAGE_SIZE + PREFETCHED_LOAD_SIZE) ++ PREFETCH_ONE_SET(1, (%rsi), PAGE_SIZE + PREFETCHED_LOAD_SIZE * 2) ++ /* Load vectors from rsi. */ ++ LOAD_ONE_SET((%rsi), 0, %VEC(0), %VEC(1), %VEC(2), %VEC(3)) ++ LOAD_ONE_SET((%rsi), PAGE_SIZE, %VEC(4), %VEC(5), %VEC(6), %VEC(7)) ++ subq $-LARGE_LOAD_SIZE, %rsi ++ /* Non-temporal store vectors to rdi. */ ++ STORE_ONE_SET((%rdi), 0, %VEC(0), %VEC(1), %VEC(2), %VEC(3)) ++ STORE_ONE_SET((%rdi), PAGE_SIZE, %VEC(4), %VEC(5), %VEC(6), %VEC(7)) ++ subq $-LARGE_LOAD_SIZE, %rdi ++ decl %ecx ++ jnz L(loop_large_memcpy_2x_inner) ++ addq $PAGE_SIZE, %rdi ++ addq $PAGE_SIZE, %rsi ++ decq %r10 ++ jne L(loop_large_memcpy_2x_outer) ++ sfence ++ ++ /* Check if only last 4 loads are needed. */ ++ cmpl $(VEC_SIZE * 4), %edx ++ jbe L(large_memcpy_2x_end) ++ ++ /* Handle the last 2 * PAGE_SIZE bytes. */ ++L(loop_large_memcpy_2x_tail): + /* Copy 4 * VEC a time forward with non-temporal stores. */ +- PREFETCH_ONE_SET (1, (%rsi), PREFETCHED_LOAD_SIZE * 2) +- PREFETCH_ONE_SET (1, (%rsi), PREFETCHED_LOAD_SIZE * 3) ++ PREFETCH_ONE_SET (1, (%rsi), PREFETCHED_LOAD_SIZE) ++ PREFETCH_ONE_SET (1, (%rdi), PREFETCHED_LOAD_SIZE) + VMOVU (%rsi), %VEC(0) + VMOVU VEC_SIZE(%rsi), %VEC(1) + VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2) + VMOVU (VEC_SIZE * 3)(%rsi), %VEC(3) +- addq $PREFETCHED_LOAD_SIZE, %rsi +- subq $PREFETCHED_LOAD_SIZE, %rdx +- VMOVNT %VEC(0), (%rdi) +- VMOVNT %VEC(1), VEC_SIZE(%rdi) +- VMOVNT %VEC(2), (VEC_SIZE * 2)(%rdi) +- VMOVNT %VEC(3), (VEC_SIZE * 3)(%rdi) +- addq $PREFETCHED_LOAD_SIZE, %rdi +- cmpq $PREFETCHED_LOAD_SIZE, %rdx +- ja L(loop_large_forward) +- sfence ++ subq $-(VEC_SIZE * 4), %rsi ++ addl $-(VEC_SIZE * 4), %edx ++ VMOVA %VEC(0), (%rdi) ++ VMOVA %VEC(1), VEC_SIZE(%rdi) ++ VMOVA %VEC(2), (VEC_SIZE * 2)(%rdi) ++ VMOVA %VEC(3), (VEC_SIZE * 3)(%rdi) ++ subq $-(VEC_SIZE * 4), %rdi ++ cmpl $(VEC_SIZE * 4), %edx ++ ja L(loop_large_memcpy_2x_tail) ++ ++L(large_memcpy_2x_end): + /* Store the last 4 * VEC. */ +- VMOVU %VEC(5), (%rcx) +- VMOVU %VEC(6), -VEC_SIZE(%rcx) +- VMOVU %VEC(7), -(VEC_SIZE * 2)(%rcx) +- VMOVU %VEC(8), -(VEC_SIZE * 3)(%rcx) +- /* Store the first VEC. */ +- VMOVU %VEC(4), (%r11) ++ VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VEC(0) ++ VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VEC(1) ++ VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VEC(2) ++ VMOVU -VEC_SIZE(%rsi, %rdx), %VEC(3) ++ ++ VMOVU %VEC(0), -(VEC_SIZE * 4)(%rdi, %rdx) ++ VMOVU %VEC(1), -(VEC_SIZE * 3)(%rdi, %rdx) ++ VMOVU %VEC(2), -(VEC_SIZE * 2)(%rdi, %rdx) ++ VMOVU %VEC(3), -VEC_SIZE(%rdi, %rdx) + VZEROUPPER_RETURN + +-L(large_backward): +- /* Don't use non-temporal store if there is overlap between +- destination and source since destination may be in cache +- when source is loaded. */ +- leaq (%rcx, %rdx), %r10 +- cmpq %r10, %r9 +- jb L(loop_4x_vec_backward) +-L(loop_large_backward): +- /* Copy 4 * VEC a time backward with non-temporal stores. */ +- PREFETCH_ONE_SET (-1, (%rcx), -PREFETCHED_LOAD_SIZE * 2) +- PREFETCH_ONE_SET (-1, (%rcx), -PREFETCHED_LOAD_SIZE * 3) +- VMOVU (%rcx), %VEC(0) +- VMOVU -VEC_SIZE(%rcx), %VEC(1) +- VMOVU -(VEC_SIZE * 2)(%rcx), %VEC(2) +- VMOVU -(VEC_SIZE * 3)(%rcx), %VEC(3) +- subq $PREFETCHED_LOAD_SIZE, %rcx +- subq $PREFETCHED_LOAD_SIZE, %rdx +- VMOVNT %VEC(0), (%r9) +- VMOVNT %VEC(1), -VEC_SIZE(%r9) +- VMOVNT %VEC(2), -(VEC_SIZE * 2)(%r9) +- VMOVNT %VEC(3), -(VEC_SIZE * 3)(%r9) +- subq $PREFETCHED_LOAD_SIZE, %r9 +- cmpq $PREFETCHED_LOAD_SIZE, %rdx +- ja L(loop_large_backward) ++ .p2align 4 ++L(large_memcpy_4x): ++ movq %rdx, %r10 ++ /* edx will store remainder size for copying tail. */ ++ andl $(PAGE_SIZE * 4 - 1), %edx ++ /* r10 stores outer loop counter. */ ++ shrq $(LOG_PAGE_SIZE + 2), %r10 ++ /* Copy 4x VEC at a time from 4 pages. */ ++ .p2align 4 ++L(loop_large_memcpy_4x_outer): ++ /* ecx stores inner loop counter. */ ++ movl $(PAGE_SIZE / LARGE_LOAD_SIZE), %ecx ++L(loop_large_memcpy_4x_inner): ++ /* Only one prefetch set per page as doing 4 pages give more time ++ for prefetcher to keep up. */ ++ PREFETCH_ONE_SET(1, (%rsi), PREFETCHED_LOAD_SIZE) ++ PREFETCH_ONE_SET(1, (%rsi), PAGE_SIZE + PREFETCHED_LOAD_SIZE) ++ PREFETCH_ONE_SET(1, (%rsi), PAGE_SIZE * 2 + PREFETCHED_LOAD_SIZE) ++ PREFETCH_ONE_SET(1, (%rsi), PAGE_SIZE * 3 + PREFETCHED_LOAD_SIZE) ++ /* Load vectors from rsi. */ ++ LOAD_ONE_SET((%rsi), 0, %VEC(0), %VEC(1), %VEC(2), %VEC(3)) ++ LOAD_ONE_SET((%rsi), PAGE_SIZE, %VEC(4), %VEC(5), %VEC(6), %VEC(7)) ++ LOAD_ONE_SET((%rsi), PAGE_SIZE * 2, %VEC(8), %VEC(9), %VEC(10), %VEC(11)) ++ LOAD_ONE_SET((%rsi), PAGE_SIZE * 3, %VEC(12), %VEC(13), %VEC(14), %VEC(15)) ++ subq $-LARGE_LOAD_SIZE, %rsi ++ /* Non-temporal store vectors to rdi. */ ++ STORE_ONE_SET((%rdi), 0, %VEC(0), %VEC(1), %VEC(2), %VEC(3)) ++ STORE_ONE_SET((%rdi), PAGE_SIZE, %VEC(4), %VEC(5), %VEC(6), %VEC(7)) ++ STORE_ONE_SET((%rdi), PAGE_SIZE * 2, %VEC(8), %VEC(9), %VEC(10), %VEC(11)) ++ STORE_ONE_SET((%rdi), PAGE_SIZE * 3, %VEC(12), %VEC(13), %VEC(14), %VEC(15)) ++ subq $-LARGE_LOAD_SIZE, %rdi ++ decl %ecx ++ jnz L(loop_large_memcpy_4x_inner) ++ addq $(PAGE_SIZE * 3), %rdi ++ addq $(PAGE_SIZE * 3), %rsi ++ decq %r10 ++ jne L(loop_large_memcpy_4x_outer) + sfence +- /* Store the first 4 * VEC. */ +- VMOVU %VEC(4), (%rdi) +- VMOVU %VEC(5), VEC_SIZE(%rdi) +- VMOVU %VEC(6), (VEC_SIZE * 2)(%rdi) +- VMOVU %VEC(7), (VEC_SIZE * 3)(%rdi) +- /* Store the last VEC. */ +- VMOVU %VEC(8), (%r11) ++ /* Check if only last 4 loads are needed. */ ++ cmpl $(VEC_SIZE * 4), %edx ++ jbe L(large_memcpy_4x_end) ++ ++ /* Handle the last 4 * PAGE_SIZE bytes. */ ++L(loop_large_memcpy_4x_tail): ++ /* Copy 4 * VEC a time forward with non-temporal stores. */ ++ PREFETCH_ONE_SET (1, (%rsi), PREFETCHED_LOAD_SIZE) ++ PREFETCH_ONE_SET (1, (%rdi), PREFETCHED_LOAD_SIZE) ++ VMOVU (%rsi), %VEC(0) ++ VMOVU VEC_SIZE(%rsi), %VEC(1) ++ VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2) ++ VMOVU (VEC_SIZE * 3)(%rsi), %VEC(3) ++ subq $-(VEC_SIZE * 4), %rsi ++ addl $-(VEC_SIZE * 4), %edx ++ VMOVA %VEC(0), (%rdi) ++ VMOVA %VEC(1), VEC_SIZE(%rdi) ++ VMOVA %VEC(2), (VEC_SIZE * 2)(%rdi) ++ VMOVA %VEC(3), (VEC_SIZE * 3)(%rdi) ++ subq $-(VEC_SIZE * 4), %rdi ++ cmpl $(VEC_SIZE * 4), %edx ++ ja L(loop_large_memcpy_4x_tail) ++ ++L(large_memcpy_4x_end): ++ /* Store the last 4 * VEC. */ ++ VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VEC(0) ++ VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VEC(1) ++ VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VEC(2) ++ VMOVU -VEC_SIZE(%rsi, %rdx), %VEC(3) ++ ++ VMOVU %VEC(0), -(VEC_SIZE * 4)(%rdi, %rdx) ++ VMOVU %VEC(1), -(VEC_SIZE * 3)(%rdi, %rdx) ++ VMOVU %VEC(2), -(VEC_SIZE * 2)(%rdi, %rdx) ++ VMOVU %VEC(3), -VEC_SIZE(%rdi, %rdx) + VZEROUPPER_RETURN + #endif + END (MEMMOVE_SYMBOL (__memmove, unaligned_erms)) +-- +GitLab + diff --git a/SOURCES/ia-upperbound-enh-rep_movsb.patch b/SOURCES/ia-upperbound-enh-rep_movsb.patch new file mode 100644 index 0000000..55b8646 --- /dev/null +++ b/SOURCES/ia-upperbound-enh-rep_movsb.patch @@ -0,0 +1,91 @@ +From 571a3ddd938b742af8fc2b02f26b4b3296ea8a94 Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Wed, 2 Mar 2022 16:12:40 -0800 +Subject: [PATCH] x86: Adding an upper bound for Enhanced REP MOVSB. + +In the process of optimizing memcpy for AMD machines, we have found the +vector move operations are outperforming enhanced REP MOVSB for data +transfers above the L2 cache size on Zen3 architectures. +To handle this use case, we are adding an upper bound parameter on +enhanced REP MOVSB:'__x86_rep_movsb_stop_threshold'. +As per large-bench results, we are configuring this parameter to the +L2 cache size for AMD machines and applicable from Zen3 architecture +supporting the ERMS feature. +For architectures other than AMD, it is the computed value of +non-temporal threshold parameter. + +Reviewed-by: Premachandra Mallappa + +(cherry picked from commit 6e02b3e9327b7dbb063958d2b124b64fcb4bbe3f) +--- + sysdeps/x86/cacheinfo.h | 14 ++++++++++++++ + .../x86_64/multiarch/memmove-vec-unaligned-erms.S | 7 +++++-- + 2 files changed, 19 insertions(+), 2 deletions(-) + +diff --git a/sysdeps/x86/cacheinfo.h b/sysdeps/x86/cacheinfo.h +index 02556961..b982982f 100644 +--- a/sysdeps/x86/cacheinfo.h ++++ b/sysdeps/x86/cacheinfo.h +@@ -45,6 +45,9 @@ long int __x86_rep_movsb_threshold attribute_hidden = 2048; + /* Threshold to use Enhanced REP STOSB. */ + long int __x86_rep_stosb_threshold attribute_hidden = 2048; + ++/* Threshold to stop using Enhanced REP MOVSB. */ ++long int __x86_rep_movsb_stop_threshold attribute_hidden; ++ + static void + get_common_cache_info (long int *shared_ptr, unsigned int *threads_ptr, + long int core) +@@ -352,6 +355,12 @@ init_cacheinfo (void) + shared += core; + } + } ++ ++ /* ERMS feature is implemented from AMD Zen3 architecture and it is ++ performing poorly for data above L2 cache size. Henceforth, adding ++ an upper bound threshold parameter to limit the usage of Enhanced ++ REP MOVSB operations and setting its value to L2 cache size. */ ++ __x86_rep_movsb_stop_threshold = core; + } + + if (cpu_features->data_cache_size != 0) +@@ -421,6 +430,11 @@ init_cacheinfo (void) + else + __x86_rep_movsb_threshold = rep_movsb_threshold; + ++ /* Setting the upper bound of ERMS to the computed value of ++ non-temporal threshold for architectures other than AMD. */ ++ if (cpu_features->basic.kind != arch_kind_amd) ++ __x86_rep_movsb_stop_threshold = __x86_shared_non_temporal_threshold; ++ + # if HAVE_TUNABLES + __x86_rep_stosb_threshold = cpu_features->rep_stosb_threshold; + # endif +diff --git a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S +index 572cef04..620ce3a8 100644 +--- a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S ++++ b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S +@@ -30,7 +30,10 @@ + load and aligned store. Load the last 4 * VEC and first VEC + before the loop and store them after the loop to support + overlapping addresses. +- 6. If size >= __x86_shared_non_temporal_threshold and there is no ++ 6. On machines with ERMS feature, if size greater than equal or to ++ __x86_rep_movsb_threshold and less than ++ __x86_rep_movsb_stop_threshold, then REP MOVSB will be used. ++ 7. If size >= __x86_shared_non_temporal_threshold and there is no + overlap between destination and source, use non-temporal store + instead of aligned store copying from either 2 or 4 pages at + once. +@@ -311,7 +314,7 @@ L(return): + #endif + + L(movsb): +- cmp __x86_shared_non_temporal_threshold(%rip), %RDX_LP ++ cmp __x86_rep_movsb_stop_threshold(%rip), %RDX_LP + jae L(more_8x_vec) + cmpq %rsi, %rdi + jb 1f +-- +GitLab + diff --git a/SOURCES/ia-use-xmmn-vpxor.patch b/SOURCES/ia-use-xmmn-vpxor.patch new file mode 100644 index 0000000..a35cf75 --- /dev/null +++ b/SOURCES/ia-use-xmmn-vpxor.patch @@ -0,0 +1,45 @@ +From e7e6385876577e04101a9b9ed8d10c34f634c905 Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Thu, 11 Jun 2020 12:41:18 -0700 +Subject: [PATCH] x86_64: Use %xmmN with vpxor to clear a vector register + +Since "vpxor %xmmN, %xmmN, %xmmN" clears the whole vector register, use +%xmmN, instead of %ymmN, with vpxor to clear a vector register. + +(cherry picked from commit a35a59036ebae3efcdf5e8167610e0656fca9770) +--- + sysdeps/x86_64/multiarch/strcmp-avx2.S | 4 ++-- + sysdeps/x86_64/multiarch/strrchr-avx2.S | 2 +- + 2 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/sysdeps/x86_64/multiarch/strcmp-avx2.S b/sysdeps/x86_64/multiarch/strcmp-avx2.S +index 433ae047..70d8499b 100644 +--- a/sysdeps/x86_64/multiarch/strcmp-avx2.S ++++ b/sysdeps/x86_64/multiarch/strcmp-avx2.S +@@ -105,8 +105,8 @@ ENTRY (STRCMP) + # endif + movl %edi, %eax + xorl %edx, %edx +- /* Make %ymm7 all zeros in this function. */ +- vpxor %ymm7, %ymm7, %ymm7 ++ /* Make %xmm7 (%ymm7) all zeros in this function. */ ++ vpxor %xmm7, %xmm7, %xmm7 + orl %esi, %eax + andl $(PAGE_SIZE - 1), %eax + cmpl $(PAGE_SIZE - (VEC_SIZE * 4)), %eax +diff --git a/sysdeps/x86_64/multiarch/strrchr-avx2.S b/sysdeps/x86_64/multiarch/strrchr-avx2.S +index 9f22a15e..c949410b 100644 +--- a/sysdeps/x86_64/multiarch/strrchr-avx2.S ++++ b/sysdeps/x86_64/multiarch/strrchr-avx2.S +@@ -48,7 +48,7 @@ ENTRY (STRRCHR) + movl %edi, %ecx + /* Broadcast CHAR to YMM4. */ + VPBROADCAST %xmm4, %ymm4 +- vpxor %ymm0, %ymm0, %ymm0 ++ vpxor %xmm0, %xmm0, %xmm0 + + /* Check if we may cross page boundary with one vector load. */ + andl $(2 * VEC_SIZE - 1), %ecx +-- +GitLab + diff --git a/SOURCES/ia-use-zmm16-zmm31-avx512-1.patch b/SOURCES/ia-use-zmm16-zmm31-avx512-1.patch new file mode 100644 index 0000000..851e2ac --- /dev/null +++ b/SOURCES/ia-use-zmm16-zmm31-avx512-1.patch @@ -0,0 +1,149 @@ +From df380c9a1a74df30968bacb067ed77e583bd960d Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Sun, 7 Mar 2021 09:44:18 -0800 +Subject: [PATCH] x86-64: Use ZMM16-ZMM31 in AVX512 memset family functions + +Update ifunc-memset.h/ifunc-wmemset.h to select the function optimized +with AVX512 instructions using ZMM16-ZMM31 registers to avoid RTM abort +with usable AVX512VL and AVX512BW since VZEROUPPER isn't needed at +function exit. + +(cherry picked from commit 4e2d8f352774b56078c34648b14a2412c38384f4) +--- + sysdeps/x86_64/multiarch/ifunc-impl-list.c | 14 +++++++++----- + sysdeps/x86_64/multiarch/ifunc-memset.h | 13 ++++++++----- + sysdeps/x86_64/multiarch/ifunc-wmemset.h | 12 ++++++------ + .../multiarch/memset-avx512-unaligned-erms.S | 16 ++++++++-------- + 4 files changed, 31 insertions(+), 24 deletions(-) + +diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c +index c1efeec0..d969a156 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c ++++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c +@@ -211,10 +211,12 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + && CPU_FEATURE_USABLE (AVX512BW)), + __memset_chk_evex_unaligned_erms) + IFUNC_IMPL_ADD (array, i, __memset_chk, +- CPU_FEATURE_USABLE (AVX512F), ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW)), + __memset_chk_avx512_unaligned_erms) + IFUNC_IMPL_ADD (array, i, __memset_chk, +- CPU_FEATURE_USABLE (AVX512F), ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW)), + __memset_chk_avx512_unaligned) + IFUNC_IMPL_ADD (array, i, __memset_chk, + CPU_FEATURE_USABLE (AVX512F), +@@ -252,10 +254,12 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + && CPU_FEATURE_USABLE (AVX512BW)), + __memset_evex_unaligned_erms) + IFUNC_IMPL_ADD (array, i, memset, +- CPU_FEATURE_USABLE (AVX512F), ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW)), + __memset_avx512_unaligned_erms) + IFUNC_IMPL_ADD (array, i, memset, +- CPU_FEATURE_USABLE (AVX512F), ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW)), + __memset_avx512_unaligned) + IFUNC_IMPL_ADD (array, i, memset, + CPU_FEATURE_USABLE (AVX512F), +@@ -719,7 +723,7 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + CPU_FEATURE_USABLE (AVX512VL), + __wmemset_evex_unaligned) + IFUNC_IMPL_ADD (array, i, wmemset, +- CPU_FEATURE_USABLE (AVX512F), ++ CPU_FEATURE_USABLE (AVX512VL), + __wmemset_avx512_unaligned)) + + #ifdef SHARED +diff --git a/sysdeps/x86_64/multiarch/ifunc-memset.h b/sysdeps/x86_64/multiarch/ifunc-memset.h +index 6f3375cc..19795938 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-memset.h ++++ b/sysdeps/x86_64/multiarch/ifunc-memset.h +@@ -53,13 +53,16 @@ IFUNC_SELECTOR (void) + if (CPU_FEATURE_USABLE_P (cpu_features, AVX512F) + && !CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_AVX512)) + { +- if (CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) +- return OPTIMIZE (avx512_no_vzeroupper); ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) ++ && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)) ++ { ++ if (CPU_FEATURE_USABLE_P (cpu_features, ERMS)) ++ return OPTIMIZE (avx512_unaligned_erms); + +- if (CPU_FEATURE_USABLE_P (cpu_features, ERMS)) +- return OPTIMIZE (avx512_unaligned_erms); ++ return OPTIMIZE (avx512_unaligned); ++ } + +- return OPTIMIZE (avx512_unaligned); ++ return OPTIMIZE (avx512_no_vzeroupper); + } + + if (CPU_FEATURE_USABLE_P (cpu_features, AVX2)) +diff --git a/sysdeps/x86_64/multiarch/ifunc-wmemset.h b/sysdeps/x86_64/multiarch/ifunc-wmemset.h +index bdc94c6c..98c5d406 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-wmemset.h ++++ b/sysdeps/x86_64/multiarch/ifunc-wmemset.h +@@ -33,13 +33,13 @@ IFUNC_SELECTOR (void) + if (CPU_FEATURE_USABLE_P (cpu_features, AVX2) + && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) + { +- if (CPU_FEATURE_USABLE_P (cpu_features, AVX512F) +- && !CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_AVX512) +- && !CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) +- return OPTIMIZE (avx512_unaligned); +- + if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)) +- return OPTIMIZE (evex_unaligned); ++ { ++ if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_AVX512)) ++ return OPTIMIZE (avx512_unaligned); ++ ++ return OPTIMIZE (evex_unaligned); ++ } + + if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) + return OPTIMIZE (avx2_unaligned_rtm); +diff --git a/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S +index 0783979c..22e7b187 100644 +--- a/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S ++++ b/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S +@@ -1,22 +1,22 @@ + #if IS_IN (libc) + # define VEC_SIZE 64 +-# define VEC(i) zmm##i ++# define XMM0 xmm16 ++# define YMM0 ymm16 ++# define VEC0 zmm16 ++# define VEC(i) VEC##i + # define VMOVU vmovdqu64 + # define VMOVA vmovdqa64 ++# define VZEROUPPER + + # define MEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ +- vmovd d, %xmm0; \ + movq r, %rax; \ +- vpbroadcastb %xmm0, %xmm0; \ +- vpbroadcastq %xmm0, %zmm0 ++ vpbroadcastb d, %VEC0 + + # define WMEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ +- vmovd d, %xmm0; \ + movq r, %rax; \ +- vpbroadcastd %xmm0, %xmm0; \ +- vpbroadcastq %xmm0, %zmm0 ++ vpbroadcastd d, %VEC0 + +-# define SECTION(p) p##.avx512 ++# define SECTION(p) p##.evex512 + # define MEMSET_SYMBOL(p,s) p##_avx512_##s + # define WMEMSET_SYMBOL(p,s) p##_avx512_##s + +-- +GitLab + diff --git a/SOURCES/ia-use-zmm16-zmm31-avx512-2.patch b/SOURCES/ia-use-zmm16-zmm31-avx512-2.patch new file mode 100644 index 0000000..756f4d0 --- /dev/null +++ b/SOURCES/ia-use-zmm16-zmm31-avx512-2.patch @@ -0,0 +1,165 @@ +From edb998421aef6f9b3bc7716f618d913dcfff64f7 Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Sun, 7 Mar 2021 09:45:23 -0800 +Subject: [PATCH] x86-64: Use ZMM16-ZMM31 in AVX512 memmove family functions + +Update ifunc-memmove.h to select the function optimized with AVX512 +instructions using ZMM16-ZMM31 registers to avoid RTM abort with usable +AVX512VL since VZEROUPPER isn't needed at function exit. + +(cherry picked from commit e4fda4631017e49d4ee5a2755db34289b6860fa4) +--- + sysdeps/x86_64/multiarch/ifunc-impl-list.c | 24 +++++++++--------- + sysdeps/x86_64/multiarch/ifunc-memmove.h | 12 +++++---- + .../multiarch/memmove-avx512-unaligned-erms.S | 25 +++++++++++++++++-- + 3 files changed, 42 insertions(+), 19 deletions(-) + +diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c +index d969a156..fec384f6 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c ++++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c +@@ -83,10 +83,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + CPU_FEATURE_USABLE (AVX512F), + __memmove_chk_avx512_no_vzeroupper) + IFUNC_IMPL_ADD (array, i, __memmove_chk, +- CPU_FEATURE_USABLE (AVX512F), ++ CPU_FEATURE_USABLE (AVX512VL), + __memmove_chk_avx512_unaligned) + IFUNC_IMPL_ADD (array, i, __memmove_chk, +- CPU_FEATURE_USABLE (AVX512F), ++ CPU_FEATURE_USABLE (AVX512VL), + __memmove_chk_avx512_unaligned_erms) + IFUNC_IMPL_ADD (array, i, __memmove_chk, + CPU_FEATURE_USABLE (AVX), +@@ -148,10 +148,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + CPU_FEATURE_USABLE (AVX512F), + __memmove_avx512_no_vzeroupper) + IFUNC_IMPL_ADD (array, i, memmove, +- CPU_FEATURE_USABLE (AVX512F), ++ CPU_FEATURE_USABLE (AVX512VL), + __memmove_avx512_unaligned) + IFUNC_IMPL_ADD (array, i, memmove, +- CPU_FEATURE_USABLE (AVX512F), ++ CPU_FEATURE_USABLE (AVX512VL), + __memmove_avx512_unaligned_erms) + IFUNC_IMPL_ADD (array, i, memmove, CPU_FEATURE_USABLE (SSSE3), + __memmove_ssse3_back) +@@ -733,10 +733,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + CPU_FEATURE_USABLE (AVX512F), + __memcpy_chk_avx512_no_vzeroupper) + IFUNC_IMPL_ADD (array, i, __memcpy_chk, +- CPU_FEATURE_USABLE (AVX512F), ++ CPU_FEATURE_USABLE (AVX512VL), + __memcpy_chk_avx512_unaligned) + IFUNC_IMPL_ADD (array, i, __memcpy_chk, +- CPU_FEATURE_USABLE (AVX512F), ++ CPU_FEATURE_USABLE (AVX512VL), + __memcpy_chk_avx512_unaligned_erms) + IFUNC_IMPL_ADD (array, i, __memcpy_chk, + CPU_FEATURE_USABLE (AVX), +@@ -802,10 +802,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + CPU_FEATURE_USABLE (AVX512F), + __memcpy_avx512_no_vzeroupper) + IFUNC_IMPL_ADD (array, i, memcpy, +- CPU_FEATURE_USABLE (AVX512F), ++ CPU_FEATURE_USABLE (AVX512VL), + __memcpy_avx512_unaligned) + IFUNC_IMPL_ADD (array, i, memcpy, +- CPU_FEATURE_USABLE (AVX512F), ++ CPU_FEATURE_USABLE (AVX512VL), + __memcpy_avx512_unaligned_erms) + IFUNC_IMPL_ADD (array, i, memcpy, 1, __memcpy_sse2_unaligned) + IFUNC_IMPL_ADD (array, i, memcpy, 1, +@@ -819,10 +819,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + CPU_FEATURE_USABLE (AVX512F), + __mempcpy_chk_avx512_no_vzeroupper) + IFUNC_IMPL_ADD (array, i, __mempcpy_chk, +- CPU_FEATURE_USABLE (AVX512F), ++ CPU_FEATURE_USABLE (AVX512VL), + __mempcpy_chk_avx512_unaligned) + IFUNC_IMPL_ADD (array, i, __mempcpy_chk, +- CPU_FEATURE_USABLE (AVX512F), ++ CPU_FEATURE_USABLE (AVX512VL), + __mempcpy_chk_avx512_unaligned_erms) + IFUNC_IMPL_ADD (array, i, __mempcpy_chk, + CPU_FEATURE_USABLE (AVX), +@@ -864,10 +864,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + CPU_FEATURE_USABLE (AVX512F), + __mempcpy_avx512_no_vzeroupper) + IFUNC_IMPL_ADD (array, i, mempcpy, +- CPU_FEATURE_USABLE (AVX512F), ++ CPU_FEATURE_USABLE (AVX512VL), + __mempcpy_avx512_unaligned) + IFUNC_IMPL_ADD (array, i, mempcpy, +- CPU_FEATURE_USABLE (AVX512F), ++ CPU_FEATURE_USABLE (AVX512VL), + __mempcpy_avx512_unaligned_erms) + IFUNC_IMPL_ADD (array, i, mempcpy, + CPU_FEATURE_USABLE (AVX), +diff --git a/sysdeps/x86_64/multiarch/ifunc-memmove.h b/sysdeps/x86_64/multiarch/ifunc-memmove.h +index fa09b9fb..014e95c7 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-memmove.h ++++ b/sysdeps/x86_64/multiarch/ifunc-memmove.h +@@ -56,13 +56,15 @@ IFUNC_SELECTOR (void) + if (CPU_FEATURE_USABLE_P (cpu_features, AVX512F) + && !CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_AVX512)) + { +- if (CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) +- return OPTIMIZE (avx512_no_vzeroupper); ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)) ++ { ++ if (CPU_FEATURE_USABLE_P (cpu_features, ERMS)) ++ return OPTIMIZE (avx512_unaligned_erms); + +- if (CPU_FEATURE_USABLE_P (cpu_features, ERMS)) +- return OPTIMIZE (avx512_unaligned_erms); ++ return OPTIMIZE (avx512_unaligned); ++ } + +- return OPTIMIZE (avx512_unaligned); ++ return OPTIMIZE (avx512_no_vzeroupper); + } + + if (CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) +diff --git a/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S +index aac1515c..848848ab 100644 +--- a/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S ++++ b/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S +@@ -1,11 +1,32 @@ + #if IS_IN (libc) + # define VEC_SIZE 64 +-# define VEC(i) zmm##i ++# define XMM0 xmm16 ++# define XMM1 xmm17 ++# define YMM0 ymm16 ++# define YMM1 ymm17 ++# define VEC0 zmm16 ++# define VEC1 zmm17 ++# define VEC2 zmm18 ++# define VEC3 zmm19 ++# define VEC4 zmm20 ++# define VEC5 zmm21 ++# define VEC6 zmm22 ++# define VEC7 zmm23 ++# define VEC8 zmm24 ++# define VEC9 zmm25 ++# define VEC10 zmm26 ++# define VEC11 zmm27 ++# define VEC12 zmm28 ++# define VEC13 zmm29 ++# define VEC14 zmm30 ++# define VEC15 zmm31 ++# define VEC(i) VEC##i + # define VMOVNT vmovntdq + # define VMOVU vmovdqu64 + # define VMOVA vmovdqa64 ++# define VZEROUPPER + +-# define SECTION(p) p##.avx512 ++# define SECTION(p) p##.evex512 + # define MEMMOVE_SYMBOL(p,s) p##_avx512_##s + + # include "memmove-vec-unaligned-erms.S" +-- +GitLab + diff --git a/SOURCES/ia-wcslen-opt-sse4_1.patch b/SOURCES/ia-wcslen-opt-sse4_1.patch new file mode 100644 index 0000000..1169d8d --- /dev/null +++ b/SOURCES/ia-wcslen-opt-sse4_1.patch @@ -0,0 +1,181 @@ +From f98c4e939dc562ee4f687cead51b6fc5fb5ad18f Mon Sep 17 00:00:00 2001 +From: Noah Goldstein +Date: Wed, 23 Jun 2021 01:19:34 -0400 +Subject: [PATCH] x86-64: Add wcslen optimize for sse4.1 + +No bug. This comment adds the ifunc / build infrastructure +necessary for wcslen to prefer the sse4.1 implementation +in strlen-vec.S. test-wcslen.c is passing. + +Signed-off-by: Noah Goldstein +Reviewed-by: H.J. Lu +(cherry picked from commit 6f573a27b6c8b4236445810a44660612323f5a73) +--- + sysdeps/x86_64/multiarch/Makefile | 4 +- + sysdeps/x86_64/multiarch/ifunc-impl-list.c | 3 ++ + sysdeps/x86_64/multiarch/ifunc-wcslen.h | 52 ++++++++++++++++++++++ + sysdeps/x86_64/multiarch/wcslen-sse4_1.S | 4 ++ + sysdeps/x86_64/multiarch/wcslen.c | 2 +- + sysdeps/x86_64/multiarch/wcsnlen.c | 34 +------------- + 6 files changed, 63 insertions(+), 36 deletions(-) + create mode 100644 sysdeps/x86_64/multiarch/ifunc-wcslen.h + create mode 100644 sysdeps/x86_64/multiarch/wcslen-sse4_1.S + +diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile +index 491c7698..65fde4eb 100644 +--- a/sysdeps/x86_64/multiarch/Makefile ++++ b/sysdeps/x86_64/multiarch/Makefile +@@ -93,8 +93,8 @@ sysdep_routines += wmemcmp-sse4 wmemcmp-ssse3 wmemcmp-c \ + wcscpy-ssse3 wcscpy-c \ + wcschr-sse2 wcschr-avx2 \ + wcsrchr-sse2 wcsrchr-avx2 \ +- wcsnlen-sse4_1 wcsnlen-c \ +- wcslen-sse2 wcslen-avx2 wcsnlen-avx2 \ ++ wcslen-sse2 wcslen-sse4_1 wcslen-avx2 \ ++ wcsnlen-c wcsnlen-sse4_1 wcsnlen-avx2 \ + wcschr-avx2-rtm \ + wcscmp-avx2-rtm \ + wcslen-avx2-rtm \ +diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c +index f1a6460a..580913ca 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c ++++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c +@@ -657,6 +657,9 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + && CPU_FEATURE_USABLE (AVX512BW) + && CPU_FEATURE_USABLE (BMI2)), + __wcslen_evex) ++ IFUNC_IMPL_ADD (array, i, wcsnlen, ++ CPU_FEATURE_USABLE (SSE4_1), ++ __wcsnlen_sse4_1) + IFUNC_IMPL_ADD (array, i, wcslen, 1, __wcslen_sse2)) + + /* Support sysdeps/x86_64/multiarch/wcsnlen.c. */ +diff --git a/sysdeps/x86_64/multiarch/ifunc-wcslen.h b/sysdeps/x86_64/multiarch/ifunc-wcslen.h +new file mode 100644 +index 00000000..39e33473 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/ifunc-wcslen.h +@@ -0,0 +1,52 @@ ++/* Common definition for ifunc selections for wcslen and wcsnlen ++ All versions must be listed in ifunc-impl-list.c. ++ Copyright (C) 2017-2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#include ++ ++extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (sse4_1) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden; ++ ++static inline void * ++IFUNC_SELECTOR (void) ++{ ++ const struct cpu_features* cpu_features = __get_cpu_features (); ++ ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX2) ++ && CPU_FEATURE_USABLE_P (cpu_features, BMI2) ++ && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) ++ { ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) ++ && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)) ++ return OPTIMIZE (evex); ++ ++ if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) ++ return OPTIMIZE (avx2_rtm); ++ ++ if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) ++ return OPTIMIZE (avx2); ++ } ++ ++ if (CPU_FEATURE_USABLE_P (cpu_features, SSE4_1)) ++ return OPTIMIZE (sse4_1); ++ ++ return OPTIMIZE (sse2); ++} +diff --git a/sysdeps/x86_64/multiarch/wcslen-sse4_1.S b/sysdeps/x86_64/multiarch/wcslen-sse4_1.S +new file mode 100644 +index 00000000..7e62621a +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/wcslen-sse4_1.S +@@ -0,0 +1,4 @@ ++#define AS_WCSLEN ++#define strlen __wcslen_sse4_1 ++ ++#include "strlen-vec.S" +diff --git a/sysdeps/x86_64/multiarch/wcslen.c b/sysdeps/x86_64/multiarch/wcslen.c +index 6d06e47c..3b04b75b 100644 +--- a/sysdeps/x86_64/multiarch/wcslen.c ++++ b/sysdeps/x86_64/multiarch/wcslen.c +@@ -24,7 +24,7 @@ + # undef __wcslen + + # define SYMBOL_NAME wcslen +-# include "ifunc-avx2.h" ++# include "ifunc-wcslen.h" + + libc_ifunc_redirected (__redirect_wcslen, __wcslen, IFUNC_SELECTOR ()); + weak_alias (__wcslen, wcslen); +diff --git a/sysdeps/x86_64/multiarch/wcsnlen.c b/sysdeps/x86_64/multiarch/wcsnlen.c +index 20b731ae..06736410 100644 +--- a/sysdeps/x86_64/multiarch/wcsnlen.c ++++ b/sysdeps/x86_64/multiarch/wcsnlen.c +@@ -24,39 +24,7 @@ + # undef __wcsnlen + + # define SYMBOL_NAME wcsnlen +-# include +- +-extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden; +-extern __typeof (REDIRECT_NAME) OPTIMIZE (sse4_1) attribute_hidden; +-extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden; +-extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden; +-extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden; +- +-static inline void * +-IFUNC_SELECTOR (void) +-{ +- const struct cpu_features* cpu_features = __get_cpu_features (); +- +- if (CPU_FEATURE_USABLE_P (cpu_features, AVX2) +- && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) +- { +- if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) +- && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW) +- && CPU_FEATURE_USABLE_P (cpu_features, BMI2)) +- return OPTIMIZE (evex); +- +- if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) +- return OPTIMIZE (avx2_rtm); +- +- if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) +- return OPTIMIZE (avx2); +- } +- +- if (CPU_FEATURE_USABLE_P (cpu_features, SSE4_1)) +- return OPTIMIZE (sse4_1); +- +- return OPTIMIZE (sse2); +-} ++# include "ifunc-wcslen.h" + + libc_ifunc_redirected (__redirect_wcsnlen, __wcsnlen, IFUNC_SELECTOR ()); + weak_alias (__wcsnlen, wcsnlen); +-- +GitLab + diff --git a/SOURCES/ia-wordsize-64-roundeven.patch b/SOURCES/ia-wordsize-64-roundeven.patch new file mode 100644 index 0000000..c11a83f --- /dev/null +++ b/SOURCES/ia-wordsize-64-roundeven.patch @@ -0,0 +1,39 @@ +From 12932b371cd6eb8c3edbfb037e615ffbb546be86 Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Tue, 22 Feb 2022 06:49:39 -0800 +Subject: [PATCH] math: Redirect wordsize-64 roundeven function + +Redirect wordsize-64 roundeven function when there is no + +commit 9e97f239eae1f2b1d2e694d844c0f6fd7c4dd271 +Author: Wilco Dijkstra +Date: Thu Jan 7 15:26:26 2021 +0000 + + Remove dbl-64/wordsize-64 (part 2) +--- + sysdeps/ieee754/dbl-64/wordsize-64/s_roundeven.c | 4 +++- + 1 file changed, 3 insertions(+), 1 deletion(-) + +diff --git a/sysdeps/ieee754/dbl-64/wordsize-64/s_roundeven.c b/sysdeps/ieee754/dbl-64/wordsize-64/s_roundeven.c +index 7bbbb2dc..8728d0f2 100644 +--- a/sysdeps/ieee754/dbl-64/wordsize-64/s_roundeven.c ++++ b/sysdeps/ieee754/dbl-64/wordsize-64/s_roundeven.c +@@ -17,6 +17,7 @@ + License along with the GNU C Library; if not, see + . */ + ++#define NO_MATH_REDIRECT + #include + #include + #include +@@ -67,5 +68,6 @@ __roundeven (double x) + INSERT_WORDS64 (x, ix); + return x; + } +-hidden_def (__roundeven) ++#ifndef __roundeven + libm_alias_double (__roundeven, roundeven) ++#endif +-- +GitLab + diff --git a/SPECS/glibc.spec b/SPECS/glibc.spec index 2c37db1..bd945a7 100644 --- a/SPECS/glibc.spec +++ b/SPECS/glibc.spec @@ -1,6 +1,6 @@ %define glibcsrcdir glibc-2.28 %define glibcversion 2.28 -%define glibcrelease 200%{?dist} +%define glibcrelease 204%{?dist} # Pre-release tarballs are pulled in from git using a command that is # effectively: # @@ -893,11 +893,89 @@ Patch698: glibc-rh2065588-11.patch Patch699: glibc-rh2065588-12.patch Patch700: glibc-rh2065588-13.patch Patch701: glibc-rh2072329.patch +Patch702: glibc-rh1982608.patch +Patch703: glibc-rh1961109.patch +Patch704: glibc-rh2086853.patch +Patch705: glibc-rh2077835.patch # Intel Optimizations -Patch1001: 0001-void-short-distance-rep-movsb.patch -Patch1002: 0002-pack-up-patches.patch -Patch1003: 0003-only-avoid-short-distance-rep-mobsb-on-icx.patch +Patch10001: glibc-sw24097-1.patch +Patch10002: glibc-sw24097-2.patch +Patch10003: glibc-sw24097-3.patch +Patch10004: glibc-sw24097-4.patch +Patch10005: glibc-sw24097-5.patch +Patch10006: glibc-sw24097-6.patch +Patch10007: glibc-sw24097-7.patch +Patch10008: glibc-sw24097-8.patch +Patch10009: glibc-sw24155.patch +Patch10010: glibc-sw28755-1.patch +Patch10011: ia-prefer_no_vzeroupper.patch +Patch10012: ia-upd-256bit-evex-instr-1.patch +Patch10013: ia-upd-256bit-evex-instr-2.patch +Patch10014: ia-upd-256bit-evex-instr-3.patch +Patch10015: ia-upd-256bit-evex-instr-4.patch +Patch10016: ia-upd-256bit-evex-instr-5.patch +Patch10017: ia-avx-opt-funct-rtm.patch +Patch10018: ia-string-funct-test-rtm.patch +Patch10019: ia-use-zmm16-zmm31-avx512-1.patch +Patch10020: ia-use-zmm16-zmm31-avx512-2.patch +Patch10021: ia-ifdef-indt-strlen-evex.patch +Patch10022: ia-bmi2-req-strlen-strnlen.patch +Patch10023: ia-memchr-opt-avx2.patch +Patch10024: glibc-sw27974-1.patch +Patch10025: ia-strlen-opt-avx2.patch +Patch10026: ia-opt-memchr-evex.patch +Patch10027: ia-unk-vector-opr-memchr-evex.patch +Patch10028: ia-move-strlen-multiarch.patch +Patch10029: ia-wcslen-opt-sse4_1.patch +Patch10030: glibc-sw27974-2.patch +Patch10031: ia-opt-strlen-evex.patch +Patch10032: glibc-sw28033.patch +Patch10033: glibc-sw28064.patch +Patch10034: glibc-sw28896.patch +Patch10035: glibc-sw25966.patch +Patch10036: ia-use-xmmn-vpxor.patch +Patch10037: ia-refacto-imp-prf-strchr-avx2.patch +Patch10038: glibc-sw27130.patch +Patch10039: ia-upd-large-memcpy.patch +Patch10040: ia-bmi2-req-strchr-avx2.patch +Patch10041: ia-opt-less_vec-memset-avx512.patch +Patch10042: ia-opt-strchr-avx2.patch +Patch10043: ia-opt-strchr-evex.patch +Patch10044: ia-opt-memchr-evex-rtm.patch +Patch10045: ia-opt-memcmp-avx2-movbe.patch +Patch10046: ia-opt-memcmp-evex-movbe-1.patch +Patch10047: ia-impr-memset-vec-unaligned-erms.patch +Patch10048: ia-impr-memmove-vec-unaligned-erms.patch +Patch10049: ia-rmv-ofl-chk-wcsnlen-sse4_1.patch +Patch10050: ia-upperbound-enh-rep_movsb.patch +Patch10051: ia-avoid_short_distance_rep_movsb.patch +Patch10052: ia-testl-x86_string_control.patch +Patch10053: ia-wordsize-64-roundeven.patch +Patch10054: ia-redirect-roundeven-funct.patch +Patch10055: ia-roundeven_sse4_1.patch +Patch10056: glibc-sw28755-2.patch +Patch10057: glibc-sw28252.patch +Patch10058: ia-new-macro-entry_p2align.patch +Patch10059: ia-opt-memcmp-evex-movbe-2.patch +Patch10060: ia-opt-memset-vec-unaligned-erms.patch +Patch10061: ia-rplc-sse2-inst-avx-memcmp-evex-movbe.patch +Patch10062: ia-imp-strcmp-evex.patch +Patch10063: ia-rmv-prefer_avx2_strcmp.patch +Patch10064: ia-rplc-movzx-movzbl.patch +Patch10065: ia-set-rep_movsb_threshold-2112.patch +Patch10066: ia-opt-memmove-vec-unaligned-erms.patch +Patch10067: ia-double-rep_movsb_threshold-erms.patch +Patch10068: ia-shrink-memcmp-sse4-code-size.patch +Patch10069: glibc-sw28537-1.patch +Patch10070: glibc-sw28537-2.patch +Patch10071: glibc-sw28537-3.patch +Patch10072: ia-rplc-cas-avoid-extra-load.patch +Patch10073: glibc-sw28646.patch +Patch10074: ia-no-index_arch_prefer_no_avx512-avx-vnni.patch +Patch10075: ia-opt-less_vec-memcmp-evex-movb.patch +Patch10076: glibc-sw28537-4.patch + ############################################################################## # Continued list of core "glibc" package information: @@ -2728,8 +2806,20 @@ fi %files -f compat-libpthread-nonshared.filelist -n compat-libpthread-nonshared %changelog -* Tue May 03 2022 Ali Erdinc Koroglu - 2.28-200 -- Intel optimizations added +* Fri May 20 2022 Ali Erdinc Koroglu - 2.28.204 +- Intel architecture optimizations + +* Tue May 17 2022 Patsy Griffin - 2.28-203 +- 390x: Add support for IBM z16. (#2077835) + +* Mon May 16 2022 Siddhesh Poyarekar - 2.28-202 +- Ensure that condition in __glibc_fortify is a constant (#2086853) + +* Tue May 10 2022 Arjun Shankar - 2.28-201 +- Add missing MACRON to EBCDIC character sets (#1961109) + +* Wed May 4 2022 DJ Delorie - 2.28-200 +- Fix glob defects on certain XFS filesystems (#1982608) * Tue Apr 26 2022 Siddhesh Poyarekar - 2.28-199 - Fix fortify false positive with mbsrtowcs and mbsnrtowcs (#2072329).