|
 |
c5d972 |
From 9bc2ed8f46d80859a5596789cc9e8cc2de84b0e7 Mon Sep 17 00:00:00 2001
|
|
|
c5d972 |
From: Wilco Dijkstra <wdijkstr@arm.com>
|
|
|
c5d972 |
Date: Tue, 10 Aug 2021 13:39:37 +0100
|
|
|
c5d972 |
Subject: [PATCH] [2/5] AArch64: Improve A64FX memset for large sizes
|
|
|
c5d972 |
|
|
|
c5d972 |
Improve performance of large memsets. Simplify alignment code. For zero memset
|
|
|
c5d972 |
use DC ZVA, which almost doubles performance. For non-zero memsets use the
|
|
|
c5d972 |
unroll8 loop which is about 10% faster.
|
|
|
c5d972 |
|
|
|
c5d972 |
Reviewed-by: Naohiro Tamura <naohirot@fujitsu.com>
|
|
|
c5d972 |
---
|
|
|
c5d972 |
sysdeps/aarch64/multiarch/memset_a64fx.S | 85 +++++++-----------------
|
|
|
c5d972 |
1 file changed, 25 insertions(+), 60 deletions(-)
|
|
|
c5d972 |
|
|
|
c5d972 |
diff --git a/sysdeps/aarch64/multiarch/memset_a64fx.S b/sysdeps/aarch64/multiarch/memset_a64fx.S
|
|
|
c5d972 |
index cf3d402ef6..75cf43ae79 100644
|
|
|
c5d972 |
--- a/sysdeps/aarch64/multiarch/memset_a64fx.S
|
|
|
c5d972 |
+++ b/sysdeps/aarch64/multiarch/memset_a64fx.S
|
|
|
c5d972 |
@@ -27,14 +27,11 @@
|
|
|
c5d972 |
*/
|
|
|
c5d972 |
|
|
|
c5d972 |
#define L1_SIZE (64*1024) // L1 64KB
|
|
|
c5d972 |
-#define L2_SIZE (8*1024*1024) // L2 8MB - 1MB
|
|
|
c5d972 |
+#define L2_SIZE (8*1024*1024) // L2 8MB
|
|
|
c5d972 |
#define CACHE_LINE_SIZE 256
|
|
|
c5d972 |
#define PF_DIST_L1 (CACHE_LINE_SIZE * 16) // Prefetch distance L1
|
|
|
c5d972 |
-#define ZF_DIST (CACHE_LINE_SIZE * 21) // Zerofill distance
|
|
|
c5d972 |
-#define rest x8
|
|
|
c5d972 |
+#define rest x2
|
|
|
c5d972 |
#define vector_length x9
|
|
|
c5d972 |
-#define vl_remainder x10 // vector_length remainder
|
|
|
c5d972 |
-#define cl_remainder x11 // CACHE_LINE_SIZE remainder
|
|
|
c5d972 |
|
|
|
c5d972 |
#if HAVE_AARCH64_SVE_ASM
|
|
|
c5d972 |
# if IS_IN (libc)
|
|
|
c5d972 |
@@ -42,14 +39,6 @@
|
|
|
c5d972 |
|
|
|
c5d972 |
.arch armv8.2-a+sve
|
|
|
c5d972 |
|
|
|
c5d972 |
- .macro dc_zva times
|
|
|
c5d972 |
- dc zva, tmp1
|
|
|
c5d972 |
- add tmp1, tmp1, CACHE_LINE_SIZE
|
|
|
c5d972 |
- .if \times-1
|
|
|
c5d972 |
- dc_zva "(\times-1)"
|
|
|
c5d972 |
- .endif
|
|
|
c5d972 |
- .endm
|
|
|
c5d972 |
-
|
|
|
c5d972 |
.macro st1b_unroll first=0, last=7
|
|
|
c5d972 |
st1b z0.b, p0, [dst, \first, mul vl]
|
|
|
c5d972 |
.if \last-\first
|
|
|
c5d972 |
@@ -188,54 +177,30 @@ L(L1_prefetch): // if rest >= L1_SIZE
|
|
|
c5d972 |
cbnz rest, L(unroll32)
|
|
|
c5d972 |
ret
|
|
|
c5d972 |
|
|
|
c5d972 |
-L(L2):
|
|
|
c5d972 |
- // align dst address at vector_length byte boundary
|
|
|
c5d972 |
- sub tmp1, vector_length, 1
|
|
|
c5d972 |
- ands tmp2, dst, tmp1
|
|
|
c5d972 |
- // if vl_remainder == 0
|
|
|
c5d972 |
- b.eq 1f
|
|
|
c5d972 |
- sub vl_remainder, vector_length, tmp2
|
|
|
c5d972 |
- // process remainder until the first vector_length boundary
|
|
|
c5d972 |
- whilelt p2.b, xzr, vl_remainder
|
|
|
c5d972 |
- st1b z0.b, p2, [dst]
|
|
|
c5d972 |
- add dst, dst, vl_remainder
|
|
|
c5d972 |
- sub rest, rest, vl_remainder
|
|
|
c5d972 |
- // align dstin address at CACHE_LINE_SIZE byte boundary
|
|
|
c5d972 |
-1: mov tmp1, CACHE_LINE_SIZE
|
|
|
c5d972 |
- ands tmp2, dst, CACHE_LINE_SIZE - 1
|
|
|
c5d972 |
- // if cl_remainder == 0
|
|
|
c5d972 |
- b.eq L(L2_dc_zva)
|
|
|
c5d972 |
- sub cl_remainder, tmp1, tmp2
|
|
|
c5d972 |
- // process remainder until the first CACHE_LINE_SIZE boundary
|
|
|
c5d972 |
- mov tmp1, xzr // index
|
|
|
c5d972 |
-2: whilelt p2.b, tmp1, cl_remainder
|
|
|
c5d972 |
- st1b z0.b, p2, [dst, tmp1]
|
|
|
c5d972 |
- incb tmp1
|
|
|
c5d972 |
- cmp tmp1, cl_remainder
|
|
|
c5d972 |
- b.lo 2b
|
|
|
c5d972 |
- add dst, dst, cl_remainder
|
|
|
c5d972 |
- sub rest, rest, cl_remainder
|
|
|
c5d972 |
-
|
|
|
c5d972 |
-L(L2_dc_zva):
|
|
|
c5d972 |
- // zero fill
|
|
|
c5d972 |
- mov tmp1, dst
|
|
|
c5d972 |
- dc_zva (ZF_DIST / CACHE_LINE_SIZE) - 1
|
|
|
c5d972 |
- mov zva_len, ZF_DIST
|
|
|
c5d972 |
- add tmp1, zva_len, CACHE_LINE_SIZE * 2
|
|
|
c5d972 |
- // unroll
|
|
|
c5d972 |
+ // count >= L2_SIZE
|
|
|
c5d972 |
.p2align 3
|
|
|
c5d972 |
-1: st1b_unroll 0, 3
|
|
|
c5d972 |
- add tmp2, dst, zva_len
|
|
|
c5d972 |
- dc zva, tmp2
|
|
|
c5d972 |
- st1b_unroll 4, 7
|
|
|
c5d972 |
- add tmp2, tmp2, CACHE_LINE_SIZE
|
|
|
c5d972 |
- dc zva, tmp2
|
|
|
c5d972 |
- add dst, dst, CACHE_LINE_SIZE * 2
|
|
|
c5d972 |
- sub rest, rest, CACHE_LINE_SIZE * 2
|
|
|
c5d972 |
- cmp rest, tmp1 // ZF_DIST + CACHE_LINE_SIZE * 2
|
|
|
c5d972 |
- b.ge 1b
|
|
|
c5d972 |
- cbnz rest, L(unroll8)
|
|
|
c5d972 |
- ret
|
|
|
c5d972 |
+L(L2):
|
|
|
c5d972 |
+ tst valw, 255
|
|
|
c5d972 |
+ b.ne L(unroll8)
|
|
|
c5d972 |
+ // align dst to CACHE_LINE_SIZE byte boundary
|
|
|
c5d972 |
+ and tmp2, dst, CACHE_LINE_SIZE - 1
|
|
|
c5d972 |
+ st1b z0.b, p0, [dst, 0, mul vl]
|
|
|
c5d972 |
+ st1b z0.b, p0, [dst, 1, mul vl]
|
|
|
c5d972 |
+ st1b z0.b, p0, [dst, 2, mul vl]
|
|
|
c5d972 |
+ st1b z0.b, p0, [dst, 3, mul vl]
|
|
|
c5d972 |
+ sub dst, dst, tmp2
|
|
|
c5d972 |
+ add count, count, tmp2
|
|
|
c5d972 |
+
|
|
|
c5d972 |
+ // clear cachelines using DC ZVA
|
|
|
c5d972 |
+ sub count, count, CACHE_LINE_SIZE * 2
|
|
|
c5d972 |
+ .p2align 4
|
|
|
c5d972 |
+1: add dst, dst, CACHE_LINE_SIZE
|
|
|
c5d972 |
+ dc zva, dst
|
|
|
c5d972 |
+ subs count, count, CACHE_LINE_SIZE
|
|
|
c5d972 |
+ b.hi 1b
|
|
|
c5d972 |
+ add count, count, CACHE_LINE_SIZE
|
|
|
c5d972 |
+ add dst, dst, CACHE_LINE_SIZE
|
|
|
c5d972 |
+ b L(last)
|
|
|
c5d972 |
|
|
|
c5d972 |
END (MEMSET)
|
|
|
c5d972 |
libc_hidden_builtin_def (MEMSET)
|
|
|
c5d972 |
--
|
|
|
c5d972 |
2.31.1
|
|
|
c5d972 |
|