c5d972
From 07b427296b8d59f439144029d9a948f6c1ce0a31 Mon Sep 17 00:00:00 2001
c5d972
From: Wilco Dijkstra <wdijkstr@arm.com>
c5d972
Date: Tue, 10 Aug 2021 13:30:27 +0100
c5d972
Subject: [PATCH] [1/5] AArch64: Improve A64FX memset for small sizes
c5d972
c5d972
Improve performance of small memsets by reducing instruction counts and
c5d972
improving code alignment. Bench-memset shows 35-45% performance gain for
c5d972
small sizes.
c5d972
c5d972
Reviewed-by: Naohiro Tamura <naohirot@fujitsu.com>
c5d972
---
c5d972
 sysdeps/aarch64/multiarch/memset_a64fx.S | 96 +++++++++---------------
c5d972
 1 file changed, 36 insertions(+), 60 deletions(-)
c5d972
c5d972
diff --git a/sysdeps/aarch64/multiarch/memset_a64fx.S b/sysdeps/aarch64/multiarch/memset_a64fx.S
c5d972
index ce54e5418b..cf3d402ef6 100644
c5d972
--- a/sysdeps/aarch64/multiarch/memset_a64fx.S
c5d972
+++ b/sysdeps/aarch64/multiarch/memset_a64fx.S
c5d972
@@ -51,78 +51,54 @@
c5d972
 	.endm
c5d972
 
c5d972
 	.macro st1b_unroll first=0, last=7
c5d972
-	st1b	z0.b, p0, [dst, #\first, mul vl]
c5d972
+	st1b	z0.b, p0, [dst, \first, mul vl]
c5d972
 	.if \last-\first
c5d972
 	st1b_unroll "(\first+1)", \last
c5d972
 	.endif
c5d972
 	.endm
c5d972
 
c5d972
-	.macro shortcut_for_small_size exit
c5d972
-	// if rest <= vector_length * 2
c5d972
-	whilelo	p0.b, xzr, count
c5d972
-	whilelo	p1.b, vector_length, count
c5d972
-	b.last	1f
c5d972
-	st1b	z0.b, p0, [dstin, #0, mul vl]
c5d972
-	st1b	z0.b, p1, [dstin, #1, mul vl]
c5d972
-	ret
c5d972
-1:	// if rest > vector_length * 8
c5d972
-	cmp	count, vector_length, lsl 3	// vector_length * 8
c5d972
-	b.hi	\exit
c5d972
-	// if rest <= vector_length * 4
c5d972
-	lsl	tmp1, vector_length, 1	// vector_length * 2
c5d972
-	whilelo	p2.b, tmp1, count
c5d972
-	incb	tmp1
c5d972
-	whilelo	p3.b, tmp1, count
c5d972
-	b.last	1f
c5d972
-	st1b	z0.b, p0, [dstin, #0, mul vl]
c5d972
-	st1b	z0.b, p1, [dstin, #1, mul vl]
c5d972
-	st1b	z0.b, p2, [dstin, #2, mul vl]
c5d972
-	st1b	z0.b, p3, [dstin, #3, mul vl]
c5d972
-	ret
c5d972
-1:	// if rest <= vector_length * 8
c5d972
-	lsl	tmp1, vector_length, 2	// vector_length * 4
c5d972
-	whilelo	p4.b, tmp1, count
c5d972
-	incb	tmp1
c5d972
-	whilelo	p5.b, tmp1, count
c5d972
-	b.last	1f
c5d972
-	st1b	z0.b, p0, [dstin, #0, mul vl]
c5d972
-	st1b	z0.b, p1, [dstin, #1, mul vl]
c5d972
-	st1b	z0.b, p2, [dstin, #2, mul vl]
c5d972
-	st1b	z0.b, p3, [dstin, #3, mul vl]
c5d972
-	st1b	z0.b, p4, [dstin, #4, mul vl]
c5d972
-	st1b	z0.b, p5, [dstin, #5, mul vl]
c5d972
-	ret
c5d972
-1:	lsl	tmp1, vector_length, 2	// vector_length * 4
c5d972
-	incb	tmp1			// vector_length * 5
c5d972
-	incb	tmp1			// vector_length * 6
c5d972
-	whilelo	p6.b, tmp1, count
c5d972
-	incb	tmp1
c5d972
-	whilelo	p7.b, tmp1, count
c5d972
-	st1b	z0.b, p0, [dstin, #0, mul vl]
c5d972
-	st1b	z0.b, p1, [dstin, #1, mul vl]
c5d972
-	st1b	z0.b, p2, [dstin, #2, mul vl]
c5d972
-	st1b	z0.b, p3, [dstin, #3, mul vl]
c5d972
-	st1b	z0.b, p4, [dstin, #4, mul vl]
c5d972
-	st1b	z0.b, p5, [dstin, #5, mul vl]
c5d972
-	st1b	z0.b, p6, [dstin, #6, mul vl]
c5d972
-	st1b	z0.b, p7, [dstin, #7, mul vl]
c5d972
-	ret
c5d972
-	.endm
c5d972
 
c5d972
-ENTRY (MEMSET)
c5d972
+#undef BTI_C
c5d972
+#define BTI_C
c5d972
 
c5d972
+ENTRY (MEMSET)
c5d972
 	PTR_ARG (0)
c5d972
 	SIZE_ARG (2)
c5d972
 
c5d972
-	cbnz	count, 1f
c5d972
-	ret
c5d972
-1:	dup	z0.b, valw
c5d972
 	cntb	vector_length
c5d972
-	// shortcut for less than vector_length * 8
c5d972
-	// gives a free ptrue to p0.b for n >= vector_length
c5d972
-	shortcut_for_small_size L(vl_agnostic)
c5d972
-	// end of shortcut
c5d972
+	dup	z0.b, valw
c5d972
+	whilelo	p0.b, vector_length, count
c5d972
+	b.last	1f
c5d972
+	whilelo	p1.b, xzr, count
c5d972
+	st1b	z0.b, p1, [dstin, 0, mul vl]
c5d972
+	st1b	z0.b, p0, [dstin, 1, mul vl]
c5d972
+	ret
c5d972
+
c5d972
+	// count >= vector_length * 2
c5d972
+1:	cmp	count, vector_length, lsl 2
c5d972
+	add	dstend, dstin, count
c5d972
+	b.hi	1f
c5d972
+	st1b	z0.b, p0, [dstin, 0, mul vl]
c5d972
+	st1b	z0.b, p0, [dstin, 1, mul vl]
c5d972
+	st1b	z0.b, p0, [dstend, -2, mul vl]
c5d972
+	st1b	z0.b, p0, [dstend, -1, mul vl]
c5d972
+	ret
c5d972
+
c5d972
+	// count > vector_length * 4
c5d972
+1:	lsl	tmp1, vector_length, 3
c5d972
+	cmp	count, tmp1
c5d972
+	b.hi	L(vl_agnostic)
c5d972
+	st1b	z0.b, p0, [dstin, 0, mul vl]
c5d972
+	st1b	z0.b, p0, [dstin, 1, mul vl]
c5d972
+	st1b	z0.b, p0, [dstin, 2, mul vl]
c5d972
+	st1b	z0.b, p0, [dstin, 3, mul vl]
c5d972
+	st1b	z0.b, p0, [dstend, -4, mul vl]
c5d972
+	st1b	z0.b, p0, [dstend, -3, mul vl]
c5d972
+	st1b	z0.b, p0, [dstend, -2, mul vl]
c5d972
+	st1b	z0.b, p0, [dstend, -1, mul vl]
c5d972
+	ret
c5d972
 
c5d972
+	.p2align 4
c5d972
 L(vl_agnostic): // VL Agnostic
c5d972
 	mov	rest, count
c5d972
 	mov	dst, dstin
c5d972
-- 
c5d972
2.31.1
c5d972