|
 |
a41fe4 |
From 07b427296b8d59f439144029d9a948f6c1ce0a31 Mon Sep 17 00:00:00 2001
|
|
|
a41fe4 |
From: Wilco Dijkstra <wdijkstr@arm.com>
|
|
|
a41fe4 |
Date: Tue, 10 Aug 2021 13:30:27 +0100
|
|
|
a41fe4 |
Subject: [PATCH] [1/5] AArch64: Improve A64FX memset for small sizes
|
|
|
a41fe4 |
|
|
|
a41fe4 |
Improve performance of small memsets by reducing instruction counts and
|
|
|
a41fe4 |
improving code alignment. Bench-memset shows 35-45% performance gain for
|
|
|
a41fe4 |
small sizes.
|
|
|
a41fe4 |
|
|
|
a41fe4 |
Reviewed-by: Naohiro Tamura <naohirot@fujitsu.com>
|
|
|
a41fe4 |
---
|
|
|
a41fe4 |
sysdeps/aarch64/multiarch/memset_a64fx.S | 96 +++++++++---------------
|
|
|
a41fe4 |
1 file changed, 36 insertions(+), 60 deletions(-)
|
|
|
a41fe4 |
|
|
|
a41fe4 |
diff --git a/sysdeps/aarch64/multiarch/memset_a64fx.S b/sysdeps/aarch64/multiarch/memset_a64fx.S
|
|
|
a41fe4 |
index ce54e5418b..cf3d402ef6 100644
|
|
|
a41fe4 |
--- a/sysdeps/aarch64/multiarch/memset_a64fx.S
|
|
|
a41fe4 |
+++ b/sysdeps/aarch64/multiarch/memset_a64fx.S
|
|
|
a41fe4 |
@@ -51,78 +51,54 @@
|
|
|
a41fe4 |
.endm
|
|
|
a41fe4 |
|
|
|
a41fe4 |
.macro st1b_unroll first=0, last=7
|
|
|
a41fe4 |
- st1b z0.b, p0, [dst, #\first, mul vl]
|
|
|
a41fe4 |
+ st1b z0.b, p0, [dst, \first, mul vl]
|
|
|
a41fe4 |
.if \last-\first
|
|
|
a41fe4 |
st1b_unroll "(\first+1)", \last
|
|
|
a41fe4 |
.endif
|
|
|
a41fe4 |
.endm
|
|
|
a41fe4 |
|
|
|
a41fe4 |
- .macro shortcut_for_small_size exit
|
|
|
a41fe4 |
- // if rest <= vector_length * 2
|
|
|
a41fe4 |
- whilelo p0.b, xzr, count
|
|
|
a41fe4 |
- whilelo p1.b, vector_length, count
|
|
|
a41fe4 |
- b.last 1f
|
|
|
a41fe4 |
- st1b z0.b, p0, [dstin, #0, mul vl]
|
|
|
a41fe4 |
- st1b z0.b, p1, [dstin, #1, mul vl]
|
|
|
a41fe4 |
- ret
|
|
|
a41fe4 |
-1: // if rest > vector_length * 8
|
|
|
a41fe4 |
- cmp count, vector_length, lsl 3 // vector_length * 8
|
|
|
a41fe4 |
- b.hi \exit
|
|
|
a41fe4 |
- // if rest <= vector_length * 4
|
|
|
a41fe4 |
- lsl tmp1, vector_length, 1 // vector_length * 2
|
|
|
a41fe4 |
- whilelo p2.b, tmp1, count
|
|
|
a41fe4 |
- incb tmp1
|
|
|
a41fe4 |
- whilelo p3.b, tmp1, count
|
|
|
a41fe4 |
- b.last 1f
|
|
|
a41fe4 |
- st1b z0.b, p0, [dstin, #0, mul vl]
|
|
|
a41fe4 |
- st1b z0.b, p1, [dstin, #1, mul vl]
|
|
|
a41fe4 |
- st1b z0.b, p2, [dstin, #2, mul vl]
|
|
|
a41fe4 |
- st1b z0.b, p3, [dstin, #3, mul vl]
|
|
|
a41fe4 |
- ret
|
|
|
a41fe4 |
-1: // if rest <= vector_length * 8
|
|
|
a41fe4 |
- lsl tmp1, vector_length, 2 // vector_length * 4
|
|
|
a41fe4 |
- whilelo p4.b, tmp1, count
|
|
|
a41fe4 |
- incb tmp1
|
|
|
a41fe4 |
- whilelo p5.b, tmp1, count
|
|
|
a41fe4 |
- b.last 1f
|
|
|
a41fe4 |
- st1b z0.b, p0, [dstin, #0, mul vl]
|
|
|
a41fe4 |
- st1b z0.b, p1, [dstin, #1, mul vl]
|
|
|
a41fe4 |
- st1b z0.b, p2, [dstin, #2, mul vl]
|
|
|
a41fe4 |
- st1b z0.b, p3, [dstin, #3, mul vl]
|
|
|
a41fe4 |
- st1b z0.b, p4, [dstin, #4, mul vl]
|
|
|
a41fe4 |
- st1b z0.b, p5, [dstin, #5, mul vl]
|
|
|
a41fe4 |
- ret
|
|
|
a41fe4 |
-1: lsl tmp1, vector_length, 2 // vector_length * 4
|
|
|
a41fe4 |
- incb tmp1 // vector_length * 5
|
|
|
a41fe4 |
- incb tmp1 // vector_length * 6
|
|
|
a41fe4 |
- whilelo p6.b, tmp1, count
|
|
|
a41fe4 |
- incb tmp1
|
|
|
a41fe4 |
- whilelo p7.b, tmp1, count
|
|
|
a41fe4 |
- st1b z0.b, p0, [dstin, #0, mul vl]
|
|
|
a41fe4 |
- st1b z0.b, p1, [dstin, #1, mul vl]
|
|
|
a41fe4 |
- st1b z0.b, p2, [dstin, #2, mul vl]
|
|
|
a41fe4 |
- st1b z0.b, p3, [dstin, #3, mul vl]
|
|
|
a41fe4 |
- st1b z0.b, p4, [dstin, #4, mul vl]
|
|
|
a41fe4 |
- st1b z0.b, p5, [dstin, #5, mul vl]
|
|
|
a41fe4 |
- st1b z0.b, p6, [dstin, #6, mul vl]
|
|
|
a41fe4 |
- st1b z0.b, p7, [dstin, #7, mul vl]
|
|
|
a41fe4 |
- ret
|
|
|
a41fe4 |
- .endm
|
|
|
a41fe4 |
|
|
|
a41fe4 |
-ENTRY (MEMSET)
|
|
|
a41fe4 |
+#undef BTI_C
|
|
|
a41fe4 |
+#define BTI_C
|
|
|
a41fe4 |
|
|
|
a41fe4 |
+ENTRY (MEMSET)
|
|
|
a41fe4 |
PTR_ARG (0)
|
|
|
a41fe4 |
SIZE_ARG (2)
|
|
|
a41fe4 |
|
|
|
a41fe4 |
- cbnz count, 1f
|
|
|
a41fe4 |
- ret
|
|
|
a41fe4 |
-1: dup z0.b, valw
|
|
|
a41fe4 |
cntb vector_length
|
|
|
a41fe4 |
- // shortcut for less than vector_length * 8
|
|
|
a41fe4 |
- // gives a free ptrue to p0.b for n >= vector_length
|
|
|
a41fe4 |
- shortcut_for_small_size L(vl_agnostic)
|
|
|
a41fe4 |
- // end of shortcut
|
|
|
a41fe4 |
+ dup z0.b, valw
|
|
|
a41fe4 |
+ whilelo p0.b, vector_length, count
|
|
|
a41fe4 |
+ b.last 1f
|
|
|
a41fe4 |
+ whilelo p1.b, xzr, count
|
|
|
a41fe4 |
+ st1b z0.b, p1, [dstin, 0, mul vl]
|
|
|
a41fe4 |
+ st1b z0.b, p0, [dstin, 1, mul vl]
|
|
|
a41fe4 |
+ ret
|
|
|
a41fe4 |
+
|
|
|
a41fe4 |
+ // count >= vector_length * 2
|
|
|
a41fe4 |
+1: cmp count, vector_length, lsl 2
|
|
|
a41fe4 |
+ add dstend, dstin, count
|
|
|
a41fe4 |
+ b.hi 1f
|
|
|
a41fe4 |
+ st1b z0.b, p0, [dstin, 0, mul vl]
|
|
|
a41fe4 |
+ st1b z0.b, p0, [dstin, 1, mul vl]
|
|
|
a41fe4 |
+ st1b z0.b, p0, [dstend, -2, mul vl]
|
|
|
a41fe4 |
+ st1b z0.b, p0, [dstend, -1, mul vl]
|
|
|
a41fe4 |
+ ret
|
|
|
a41fe4 |
+
|
|
|
a41fe4 |
+ // count > vector_length * 4
|
|
|
a41fe4 |
+1: lsl tmp1, vector_length, 3
|
|
|
a41fe4 |
+ cmp count, tmp1
|
|
|
a41fe4 |
+ b.hi L(vl_agnostic)
|
|
|
a41fe4 |
+ st1b z0.b, p0, [dstin, 0, mul vl]
|
|
|
a41fe4 |
+ st1b z0.b, p0, [dstin, 1, mul vl]
|
|
|
a41fe4 |
+ st1b z0.b, p0, [dstin, 2, mul vl]
|
|
|
a41fe4 |
+ st1b z0.b, p0, [dstin, 3, mul vl]
|
|
|
a41fe4 |
+ st1b z0.b, p0, [dstend, -4, mul vl]
|
|
|
a41fe4 |
+ st1b z0.b, p0, [dstend, -3, mul vl]
|
|
|
a41fe4 |
+ st1b z0.b, p0, [dstend, -2, mul vl]
|
|
|
a41fe4 |
+ st1b z0.b, p0, [dstend, -1, mul vl]
|
|
|
a41fe4 |
+ ret
|
|
|
a41fe4 |
|
|
|
a41fe4 |
+ .p2align 4
|
|
|
a41fe4 |
L(vl_agnostic): // VL Agnostic
|
|
|
a41fe4 |
mov rest, count
|
|
|
a41fe4 |
mov dst, dstin
|
|
|
a41fe4 |
--
|
|
|
a41fe4 |
2.31.1
|
|
|
a41fe4 |
|