Blame SOURCES/ia-use-zmm16-zmm31-avx512-2.patch

190885
From edb998421aef6f9b3bc7716f618d913dcfff64f7 Mon Sep 17 00:00:00 2001
190885
From: "H.J. Lu" <hjl.tools@gmail.com>
190885
Date: Sun, 7 Mar 2021 09:45:23 -0800
190885
Subject: [PATCH] x86-64: Use ZMM16-ZMM31 in AVX512 memmove family functions
190885
190885
Update ifunc-memmove.h to select the function optimized with AVX512
190885
instructions using ZMM16-ZMM31 registers to avoid RTM abort with usable
190885
AVX512VL since VZEROUPPER isn't needed at function exit.
190885
190885
(cherry picked from commit e4fda4631017e49d4ee5a2755db34289b6860fa4)
190885
---
190885
 sysdeps/x86_64/multiarch/ifunc-impl-list.c    | 24 +++++++++---------
190885
 sysdeps/x86_64/multiarch/ifunc-memmove.h      | 12 +++++----
190885
 .../multiarch/memmove-avx512-unaligned-erms.S | 25 +++++++++++++++++--
190885
 3 files changed, 42 insertions(+), 19 deletions(-)
190885
190885
diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
190885
index d969a156..fec384f6 100644
190885
--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c
190885
+++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
190885
@@ -83,10 +83,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
190885
 			      CPU_FEATURE_USABLE (AVX512F),
190885
 			      __memmove_chk_avx512_no_vzeroupper)
190885
 	      IFUNC_IMPL_ADD (array, i, __memmove_chk,
190885
-			      CPU_FEATURE_USABLE (AVX512F),
190885
+			      CPU_FEATURE_USABLE (AVX512VL),
190885
 			      __memmove_chk_avx512_unaligned)
190885
 	      IFUNC_IMPL_ADD (array, i, __memmove_chk,
190885
-			      CPU_FEATURE_USABLE (AVX512F),
190885
+			      CPU_FEATURE_USABLE (AVX512VL),
190885
 			      __memmove_chk_avx512_unaligned_erms)
190885
 	      IFUNC_IMPL_ADD (array, i, __memmove_chk,
190885
 			      CPU_FEATURE_USABLE (AVX),
190885
@@ -148,10 +148,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
190885
 			      CPU_FEATURE_USABLE (AVX512F),
190885
 			      __memmove_avx512_no_vzeroupper)
190885
 	      IFUNC_IMPL_ADD (array, i, memmove,
190885
-			      CPU_FEATURE_USABLE (AVX512F),
190885
+			      CPU_FEATURE_USABLE (AVX512VL),
190885
 			      __memmove_avx512_unaligned)
190885
 	      IFUNC_IMPL_ADD (array, i, memmove,
190885
-			      CPU_FEATURE_USABLE (AVX512F),
190885
+			      CPU_FEATURE_USABLE (AVX512VL),
190885
 			      __memmove_avx512_unaligned_erms)
190885
 	      IFUNC_IMPL_ADD (array, i, memmove, CPU_FEATURE_USABLE (SSSE3),
190885
 			      __memmove_ssse3_back)
190885
@@ -733,10 +733,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
190885
 			      CPU_FEATURE_USABLE (AVX512F),
190885
 			      __memcpy_chk_avx512_no_vzeroupper)
190885
 	      IFUNC_IMPL_ADD (array, i, __memcpy_chk,
190885
-			      CPU_FEATURE_USABLE (AVX512F),
190885
+			      CPU_FEATURE_USABLE (AVX512VL),
190885
 			      __memcpy_chk_avx512_unaligned)
190885
 	      IFUNC_IMPL_ADD (array, i, __memcpy_chk,
190885
-			      CPU_FEATURE_USABLE (AVX512F),
190885
+			      CPU_FEATURE_USABLE (AVX512VL),
190885
 			      __memcpy_chk_avx512_unaligned_erms)
190885
 	      IFUNC_IMPL_ADD (array, i, __memcpy_chk,
190885
 			      CPU_FEATURE_USABLE (AVX),
190885
@@ -802,10 +802,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
190885
 			      CPU_FEATURE_USABLE (AVX512F),
190885
 			      __memcpy_avx512_no_vzeroupper)
190885
 	      IFUNC_IMPL_ADD (array, i, memcpy,
190885
-			      CPU_FEATURE_USABLE (AVX512F),
190885
+			      CPU_FEATURE_USABLE (AVX512VL),
190885
 			      __memcpy_avx512_unaligned)
190885
 	      IFUNC_IMPL_ADD (array, i, memcpy,
190885
-			      CPU_FEATURE_USABLE (AVX512F),
190885
+			      CPU_FEATURE_USABLE (AVX512VL),
190885
 			      __memcpy_avx512_unaligned_erms)
190885
 	      IFUNC_IMPL_ADD (array, i, memcpy, 1, __memcpy_sse2_unaligned)
190885
 	      IFUNC_IMPL_ADD (array, i, memcpy, 1,
190885
@@ -819,10 +819,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
190885
 			      CPU_FEATURE_USABLE (AVX512F),
190885
 			      __mempcpy_chk_avx512_no_vzeroupper)
190885
 	      IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
190885
-			      CPU_FEATURE_USABLE (AVX512F),
190885
+			      CPU_FEATURE_USABLE (AVX512VL),
190885
 			      __mempcpy_chk_avx512_unaligned)
190885
 	      IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
190885
-			      CPU_FEATURE_USABLE (AVX512F),
190885
+			      CPU_FEATURE_USABLE (AVX512VL),
190885
 			      __mempcpy_chk_avx512_unaligned_erms)
190885
 	      IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
190885
 			      CPU_FEATURE_USABLE (AVX),
190885
@@ -864,10 +864,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
190885
 			      CPU_FEATURE_USABLE (AVX512F),
190885
 			      __mempcpy_avx512_no_vzeroupper)
190885
 	      IFUNC_IMPL_ADD (array, i, mempcpy,
190885
-			      CPU_FEATURE_USABLE (AVX512F),
190885
+			      CPU_FEATURE_USABLE (AVX512VL),
190885
 			      __mempcpy_avx512_unaligned)
190885
 	      IFUNC_IMPL_ADD (array, i, mempcpy,
190885
-			      CPU_FEATURE_USABLE (AVX512F),
190885
+			      CPU_FEATURE_USABLE (AVX512VL),
190885
 			      __mempcpy_avx512_unaligned_erms)
190885
 	      IFUNC_IMPL_ADD (array, i, mempcpy,
190885
 			      CPU_FEATURE_USABLE (AVX),
190885
diff --git a/sysdeps/x86_64/multiarch/ifunc-memmove.h b/sysdeps/x86_64/multiarch/ifunc-memmove.h
190885
index fa09b9fb..014e95c7 100644
190885
--- a/sysdeps/x86_64/multiarch/ifunc-memmove.h
190885
+++ b/sysdeps/x86_64/multiarch/ifunc-memmove.h
190885
@@ -56,13 +56,15 @@ IFUNC_SELECTOR (void)
190885
   if (CPU_FEATURE_USABLE_P (cpu_features, AVX512F)
190885
       && !CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_AVX512))
190885
     {
190885
-      if (CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER))
190885
-	return OPTIMIZE (avx512_no_vzeroupper);
190885
+      if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL))
190885
+	{
190885
+	  if (CPU_FEATURE_USABLE_P (cpu_features, ERMS))
190885
+	    return OPTIMIZE (avx512_unaligned_erms);
190885
 
190885
-      if (CPU_FEATURE_USABLE_P (cpu_features, ERMS))
190885
-	return OPTIMIZE (avx512_unaligned_erms);
190885
+	  return OPTIMIZE (avx512_unaligned);
190885
+	}
190885
 
190885
-      return OPTIMIZE (avx512_unaligned);
190885
+      return OPTIMIZE (avx512_no_vzeroupper);
190885
     }
190885
 
190885
   if (CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load))
190885
diff --git a/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S
190885
index aac1515c..848848ab 100644
190885
--- a/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S
190885
+++ b/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S
190885
@@ -1,11 +1,32 @@
190885
 #if IS_IN (libc)
190885
 # define VEC_SIZE	64
190885
-# define VEC(i)		zmm##i
190885
+# define XMM0		xmm16
190885
+# define XMM1		xmm17
190885
+# define YMM0		ymm16
190885
+# define YMM1		ymm17
190885
+# define VEC0		zmm16
190885
+# define VEC1		zmm17
190885
+# define VEC2		zmm18
190885
+# define VEC3		zmm19
190885
+# define VEC4		zmm20
190885
+# define VEC5		zmm21
190885
+# define VEC6		zmm22
190885
+# define VEC7		zmm23
190885
+# define VEC8		zmm24
190885
+# define VEC9		zmm25
190885
+# define VEC10		zmm26
190885
+# define VEC11		zmm27
190885
+# define VEC12		zmm28
190885
+# define VEC13		zmm29
190885
+# define VEC14		zmm30
190885
+# define VEC15		zmm31
190885
+# define VEC(i)		VEC##i
190885
 # define VMOVNT		vmovntdq
190885
 # define VMOVU		vmovdqu64
190885
 # define VMOVA		vmovdqa64
190885
+# define VZEROUPPER
190885
 
190885
-# define SECTION(p)		p##.avx512
190885
+# define SECTION(p)		p##.evex512
190885
 # define MEMMOVE_SYMBOL(p,s)	p##_avx512_##s
190885
 
190885
 # include "memmove-vec-unaligned-erms.S"
190885
-- 
190885
GitLab
190885