|
 |
08c3a6 |
commit ba1c3f23d9ba63c38333116eec6043c471c378c4
|
|
|
08c3a6 |
Author: Noah Goldstein <goldstein.w.n@gmail.com>
|
|
|
08c3a6 |
Date: Wed Jun 15 10:41:28 2022 -0700
|
|
|
08c3a6 |
|
|
|
08c3a6 |
x86: Cleanup bounds checking in large memcpy case
|
|
|
08c3a6 |
|
|
|
08c3a6 |
1. Fix incorrect lower-bound threshold in L(large_memcpy_2x).
|
|
|
08c3a6 |
Previously was using `__x86_rep_movsb_threshold` and should
|
|
|
08c3a6 |
have been using `__x86_shared_non_temporal_threshold`.
|
|
|
08c3a6 |
|
|
|
08c3a6 |
2. Avoid reloading __x86_shared_non_temporal_threshold before
|
|
|
08c3a6 |
the L(large_memcpy_4x) bounds check.
|
|
|
08c3a6 |
|
|
|
08c3a6 |
3. Document the second bounds check for L(large_memcpy_4x)
|
|
|
08c3a6 |
more clearly.
|
|
|
08c3a6 |
|
|
|
08c3a6 |
(cherry picked from commit 89a25c6f64746732b87eaf433af0964b564d4a92)
|
|
|
08c3a6 |
|
|
|
08c3a6 |
diff --git a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
|
|
|
08c3a6 |
index 7b27cbdda5fb99f7..618d46d8ce28828c 100644
|
|
|
08c3a6 |
--- a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
|
|
|
08c3a6 |
+++ b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
|
|
|
08c3a6 |
@@ -118,7 +118,13 @@
|
|
|
08c3a6 |
# define LARGE_LOAD_SIZE (VEC_SIZE * 4)
|
|
|
08c3a6 |
#endif
|
|
|
08c3a6 |
|
|
|
08c3a6 |
-/* Amount to shift rdx by to compare for memcpy_large_4x. */
|
|
|
08c3a6 |
+/* Amount to shift __x86_shared_non_temporal_threshold by for
|
|
|
08c3a6 |
+ bound for memcpy_large_4x. This is essentially use to to
|
|
|
08c3a6 |
+ indicate that the copy is far beyond the scope of L3
|
|
|
08c3a6 |
+ (assuming no user config x86_non_temporal_threshold) and to
|
|
|
08c3a6 |
+ use a more aggressively unrolled loop. NB: before
|
|
|
08c3a6 |
+ increasing the value also update initialization of
|
|
|
08c3a6 |
+ x86_non_temporal_threshold. */
|
|
|
08c3a6 |
#ifndef LOG_4X_MEMCPY_THRESH
|
|
|
08c3a6 |
# define LOG_4X_MEMCPY_THRESH 4
|
|
|
08c3a6 |
#endif
|
|
|
08c3a6 |
@@ -724,9 +730,14 @@ L(skip_short_movsb_check):
|
|
|
08c3a6 |
.p2align 4,, 10
|
|
|
08c3a6 |
#if (defined USE_MULTIARCH || VEC_SIZE == 16) && IS_IN (libc)
|
|
|
08c3a6 |
L(large_memcpy_2x_check):
|
|
|
08c3a6 |
- cmp __x86_rep_movsb_threshold(%rip), %RDX_LP
|
|
|
08c3a6 |
- jb L(more_8x_vec_check)
|
|
|
08c3a6 |
+ /* Entry from L(large_memcpy_2x) has a redundant load of
|
|
|
08c3a6 |
+ __x86_shared_non_temporal_threshold(%rip). L(large_memcpy_2x)
|
|
|
08c3a6 |
+ is only use for the non-erms memmove which is generally less
|
|
|
08c3a6 |
+ common. */
|
|
|
08c3a6 |
L(large_memcpy_2x):
|
|
|
08c3a6 |
+ mov __x86_shared_non_temporal_threshold(%rip), %R11_LP
|
|
|
08c3a6 |
+ cmp %R11_LP, %RDX_LP
|
|
|
08c3a6 |
+ jb L(more_8x_vec_check)
|
|
|
08c3a6 |
/* To reach this point it is impossible for dst > src and
|
|
|
08c3a6 |
overlap. Remaining to check is src > dst and overlap. rcx
|
|
|
08c3a6 |
already contains dst - src. Negate rcx to get src - dst. If
|
|
|
08c3a6 |
@@ -774,18 +785,21 @@ L(large_memcpy_2x):
|
|
|
08c3a6 |
/* ecx contains -(dst - src). not ecx will return dst - src - 1
|
|
|
08c3a6 |
which works for testing aliasing. */
|
|
|
08c3a6 |
notl %ecx
|
|
|
08c3a6 |
+ movq %rdx, %r10
|
|
|
08c3a6 |
testl $(PAGE_SIZE - VEC_SIZE * 8), %ecx
|
|
|
08c3a6 |
jz L(large_memcpy_4x)
|
|
|
08c3a6 |
|
|
|
08c3a6 |
- movq %rdx, %r10
|
|
|
08c3a6 |
- shrq $LOG_4X_MEMCPY_THRESH, %r10
|
|
|
08c3a6 |
- cmp __x86_shared_non_temporal_threshold(%rip), %r10
|
|
|
08c3a6 |
+ /* r11 has __x86_shared_non_temporal_threshold. Shift it left
|
|
|
08c3a6 |
+ by LOG_4X_MEMCPY_THRESH to get L(large_memcpy_4x) threshold.
|
|
|
08c3a6 |
+ */
|
|
|
08c3a6 |
+ shlq $LOG_4X_MEMCPY_THRESH, %r11
|
|
|
08c3a6 |
+ cmp %r11, %rdx
|
|
|
08c3a6 |
jae L(large_memcpy_4x)
|
|
|
08c3a6 |
|
|
|
08c3a6 |
/* edx will store remainder size for copying tail. */
|
|
|
08c3a6 |
andl $(PAGE_SIZE * 2 - 1), %edx
|
|
|
08c3a6 |
/* r10 stores outer loop counter. */
|
|
|
08c3a6 |
- shrq $((LOG_PAGE_SIZE + 1) - LOG_4X_MEMCPY_THRESH), %r10
|
|
|
08c3a6 |
+ shrq $(LOG_PAGE_SIZE + 1), %r10
|
|
|
08c3a6 |
/* Copy 4x VEC at a time from 2 pages. */
|
|
|
08c3a6 |
.p2align 4
|
|
|
08c3a6 |
L(loop_large_memcpy_2x_outer):
|
|
|
08c3a6 |
@@ -850,7 +864,6 @@ L(large_memcpy_2x_end):
|
|
|
08c3a6 |
|
|
|
08c3a6 |
.p2align 4
|
|
|
08c3a6 |
L(large_memcpy_4x):
|
|
|
08c3a6 |
- movq %rdx, %r10
|
|
|
08c3a6 |
/* edx will store remainder size for copying tail. */
|
|
|
08c3a6 |
andl $(PAGE_SIZE * 4 - 1), %edx
|
|
|
08c3a6 |
/* r10 stores outer loop counter. */
|