|
 |
076f82 |
commit 35f9c72c8bd7bc30deb412e966e2f548241b15d2
|
|
|
076f82 |
Author: Noah Goldstein <goldstein.w.n@gmail.com>
|
|
|
076f82 |
Date: Wed Jun 29 16:07:15 2022 -0700
|
|
|
076f82 |
|
|
|
076f82 |
x86: Move mem{p}{mov|cpy}_{chk_}erms to its own file
|
|
|
076f82 |
|
|
|
076f82 |
The primary memmove_{impl}_unaligned_erms implementations don't
|
|
|
076f82 |
interact with this function. Putting them in same file both
|
|
|
076f82 |
wastes space and unnecessarily bloats a hot code section.
|
|
|
076f82 |
|
|
|
076f82 |
(cherry picked from commit 21925f64730d52eb7d8b2fb62b412f8ab92b0caf)
|
|
|
076f82 |
|
|
|
076f82 |
diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile
|
|
|
076f82 |
index da9f16286a763556..b9ea5b60c2be1b0a 100644
|
|
|
076f82 |
--- a/sysdeps/x86_64/multiarch/Makefile
|
|
|
076f82 |
+++ b/sysdeps/x86_64/multiarch/Makefile
|
|
|
076f82 |
@@ -17,6 +17,7 @@ sysdep_routines += \
|
|
|
076f82 |
memmove-avx-unaligned-erms-rtm \
|
|
|
076f82 |
memmove-avx512-no-vzeroupper \
|
|
|
076f82 |
memmove-avx512-unaligned-erms \
|
|
|
076f82 |
+ memmove-erms \
|
|
|
076f82 |
memmove-evex-unaligned-erms \
|
|
|
076f82 |
memmove-sse2-unaligned-erms \
|
|
|
076f82 |
memmove-ssse3 \
|
|
|
076f82 |
diff --git a/sysdeps/x86_64/multiarch/memmove-erms.S b/sysdeps/x86_64/multiarch/memmove-erms.S
|
|
|
076f82 |
new file mode 100644
|
|
|
076f82 |
index 0000000000000000..2d3a6ccb76d77052
|
|
|
076f82 |
--- /dev/null
|
|
|
076f82 |
+++ b/sysdeps/x86_64/multiarch/memmove-erms.S
|
|
|
076f82 |
@@ -0,0 +1,72 @@
|
|
|
076f82 |
+/* memcpy/mempcpy/memmove implement with rep movsb
|
|
|
076f82 |
+ Copyright (C) 2022 Free Software Foundation, Inc.
|
|
|
076f82 |
+ This file is part of the GNU C Library.
|
|
|
076f82 |
+
|
|
|
076f82 |
+ The GNU C Library is free software; you can redistribute it and/or
|
|
|
076f82 |
+ modify it under the terms of the GNU Lesser General Public
|
|
|
076f82 |
+ License as published by the Free Software Foundation; either
|
|
|
076f82 |
+ version 2.1 of the License, or (at your option) any later version.
|
|
|
076f82 |
+
|
|
|
076f82 |
+ The GNU C Library is distributed in the hope that it will be useful,
|
|
|
076f82 |
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
076f82 |
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
076f82 |
+ Lesser General Public License for more details.
|
|
|
076f82 |
+
|
|
|
076f82 |
+ You should have received a copy of the GNU Lesser General Public
|
|
|
076f82 |
+ License along with the GNU C Library; if not, see
|
|
|
076f82 |
+ <https://www.gnu.org/licenses/>. */
|
|
|
076f82 |
+
|
|
|
076f82 |
+
|
|
|
076f82 |
+#include <sysdep.h>
|
|
|
076f82 |
+
|
|
|
076f82 |
+#if defined USE_MULTIARCH && IS_IN (libc)
|
|
|
076f82 |
+ .text
|
|
|
076f82 |
+ENTRY (__mempcpy_chk_erms)
|
|
|
076f82 |
+ cmp %RDX_LP, %RCX_LP
|
|
|
076f82 |
+ jb HIDDEN_JUMPTARGET (__chk_fail)
|
|
|
076f82 |
+END (__mempcpy_chk_erms)
|
|
|
076f82 |
+
|
|
|
076f82 |
+/* Only used to measure performance of REP MOVSB. */
|
|
|
076f82 |
+ENTRY (__mempcpy_erms)
|
|
|
076f82 |
+ mov %RDI_LP, %RAX_LP
|
|
|
076f82 |
+ /* Skip zero length. */
|
|
|
076f82 |
+ test %RDX_LP, %RDX_LP
|
|
|
076f82 |
+ jz 2f
|
|
|
076f82 |
+ add %RDX_LP, %RAX_LP
|
|
|
076f82 |
+ jmp L(start_movsb)
|
|
|
076f82 |
+END (__mempcpy_erms)
|
|
|
076f82 |
+
|
|
|
076f82 |
+ENTRY (__memmove_chk_erms)
|
|
|
076f82 |
+ cmp %RDX_LP, %RCX_LP
|
|
|
076f82 |
+ jb HIDDEN_JUMPTARGET (__chk_fail)
|
|
|
076f82 |
+END (__memmove_chk_erms)
|
|
|
076f82 |
+
|
|
|
076f82 |
+ENTRY (__memmove_erms)
|
|
|
076f82 |
+ movq %rdi, %rax
|
|
|
076f82 |
+ /* Skip zero length. */
|
|
|
076f82 |
+ test %RDX_LP, %RDX_LP
|
|
|
076f82 |
+ jz 2f
|
|
|
076f82 |
+L(start_movsb):
|
|
|
076f82 |
+ mov %RDX_LP, %RCX_LP
|
|
|
076f82 |
+ cmp %RSI_LP, %RDI_LP
|
|
|
076f82 |
+ jb 1f
|
|
|
076f82 |
+ /* Source == destination is less common. */
|
|
|
076f82 |
+ je 2f
|
|
|
076f82 |
+ lea (%rsi,%rcx), %RDX_LP
|
|
|
076f82 |
+ cmp %RDX_LP, %RDI_LP
|
|
|
076f82 |
+ jb L(movsb_backward)
|
|
|
076f82 |
+1:
|
|
|
076f82 |
+ rep movsb
|
|
|
076f82 |
+2:
|
|
|
076f82 |
+ ret
|
|
|
076f82 |
+L(movsb_backward):
|
|
|
076f82 |
+ leaq -1(%rdi,%rcx), %rdi
|
|
|
076f82 |
+ leaq -1(%rsi,%rcx), %rsi
|
|
|
076f82 |
+ std
|
|
|
076f82 |
+ rep movsb
|
|
|
076f82 |
+ cld
|
|
|
076f82 |
+ ret
|
|
|
076f82 |
+END (__memmove_erms)
|
|
|
076f82 |
+strong_alias (__memmove_erms, __memcpy_erms)
|
|
|
076f82 |
+strong_alias (__memmove_chk_erms, __memcpy_chk_erms)
|
|
|
076f82 |
+#endif
|
|
|
076f82 |
diff --git a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
|
|
|
076f82 |
index 618d46d8ce28828c..93c7e6883a254434 100644
|
|
|
076f82 |
--- a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
|
|
|
076f82 |
+++ b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
|
|
|
076f82 |
@@ -239,56 +239,6 @@ L(start):
|
|
|
076f82 |
#endif
|
|
|
076f82 |
#if defined USE_MULTIARCH && IS_IN (libc)
|
|
|
076f82 |
END (MEMMOVE_SYMBOL (__memmove, unaligned))
|
|
|
076f82 |
-# if VEC_SIZE == 16
|
|
|
076f82 |
-ENTRY (__mempcpy_chk_erms)
|
|
|
076f82 |
- cmp %RDX_LP, %RCX_LP
|
|
|
076f82 |
- jb HIDDEN_JUMPTARGET (__chk_fail)
|
|
|
076f82 |
-END (__mempcpy_chk_erms)
|
|
|
076f82 |
-
|
|
|
076f82 |
-/* Only used to measure performance of REP MOVSB. */
|
|
|
076f82 |
-ENTRY (__mempcpy_erms)
|
|
|
076f82 |
- mov %RDI_LP, %RAX_LP
|
|
|
076f82 |
- /* Skip zero length. */
|
|
|
076f82 |
- test %RDX_LP, %RDX_LP
|
|
|
076f82 |
- jz 2f
|
|
|
076f82 |
- add %RDX_LP, %RAX_LP
|
|
|
076f82 |
- jmp L(start_movsb)
|
|
|
076f82 |
-END (__mempcpy_erms)
|
|
|
076f82 |
-
|
|
|
076f82 |
-ENTRY (__memmove_chk_erms)
|
|
|
076f82 |
- cmp %RDX_LP, %RCX_LP
|
|
|
076f82 |
- jb HIDDEN_JUMPTARGET (__chk_fail)
|
|
|
076f82 |
-END (__memmove_chk_erms)
|
|
|
076f82 |
-
|
|
|
076f82 |
-ENTRY (__memmove_erms)
|
|
|
076f82 |
- movq %rdi, %rax
|
|
|
076f82 |
- /* Skip zero length. */
|
|
|
076f82 |
- test %RDX_LP, %RDX_LP
|
|
|
076f82 |
- jz 2f
|
|
|
076f82 |
-L(start_movsb):
|
|
|
076f82 |
- mov %RDX_LP, %RCX_LP
|
|
|
076f82 |
- cmp %RSI_LP, %RDI_LP
|
|
|
076f82 |
- jb 1f
|
|
|
076f82 |
- /* Source == destination is less common. */
|
|
|
076f82 |
- je 2f
|
|
|
076f82 |
- lea (%rsi,%rcx), %RDX_LP
|
|
|
076f82 |
- cmp %RDX_LP, %RDI_LP
|
|
|
076f82 |
- jb L(movsb_backward)
|
|
|
076f82 |
-1:
|
|
|
076f82 |
- rep movsb
|
|
|
076f82 |
-2:
|
|
|
076f82 |
- ret
|
|
|
076f82 |
-L(movsb_backward):
|
|
|
076f82 |
- leaq -1(%rdi,%rcx), %rdi
|
|
|
076f82 |
- leaq -1(%rsi,%rcx), %rsi
|
|
|
076f82 |
- std
|
|
|
076f82 |
- rep movsb
|
|
|
076f82 |
- cld
|
|
|
076f82 |
- ret
|
|
|
076f82 |
-END (__memmove_erms)
|
|
|
076f82 |
-strong_alias (__memmove_erms, __memcpy_erms)
|
|
|
076f82 |
-strong_alias (__memmove_chk_erms, __memcpy_chk_erms)
|
|
|
076f82 |
-# endif
|
|
|
076f82 |
|
|
|
076f82 |
# ifdef SHARED
|
|
|
076f82 |
ENTRY (MEMMOVE_CHK_SYMBOL (__mempcpy_chk, unaligned_erms))
|