c5d972
From b31bd11454fade731e5158b1aea40b133ae19926 Mon Sep 17 00:00:00 2001
c5d972
From: Wilco Dijkstra <wdijkstr@arm.com>
c5d972
Date: Thu, 2 Dec 2021 18:33:26 +0000
c5d972
Subject: [PATCH] AArch64: Improve A64FX memcpy
c5d972
c5d972
v2 is a complete rewrite of the A64FX memcpy. Performance is improved
c5d972
by streamlining the code, aligning all large copies and using a single
c5d972
unrolled loop for all sizes. The code size for memcpy and memmove goes
c5d972
down from 1796 bytes to 868 bytes. Performance is better in all cases:
c5d972
bench-memcpy-random is 2.3% faster overall, bench-memcpy-large is ~33%
c5d972
faster for large sizes, bench-memcpy-walk is 25% faster for small sizes
c5d972
and 20% for the largest sizes. The geomean of all tests in bench-memcpy
c5d972
is 5.1% faster, and total time is reduced by 4%.
c5d972
c5d972
Reviewed-by: Szabolcs Nagy <szabolcs.nagy@arm.com>
c5d972
---
c5d972
 sysdeps/aarch64/multiarch/memcpy_a64fx.S | 546 ++++++++++-------------
c5d972
 1 file changed, 225 insertions(+), 321 deletions(-)
c5d972
c5d972
diff --git a/sysdeps/aarch64/multiarch/memcpy_a64fx.S b/sysdeps/aarch64/multiarch/memcpy_a64fx.S
c5d972
index ae7464e09f..0b306925e6 100644
c5d972
--- a/sysdeps/aarch64/multiarch/memcpy_a64fx.S
c5d972
+++ b/sysdeps/aarch64/multiarch/memcpy_a64fx.S
c5d972
@@ -28,20 +28,15 @@
c5d972
  *
c5d972
  */
c5d972
 
c5d972
-#define L2_SIZE		(8*1024*1024)/2	// L2 8MB/2
c5d972
-#define CACHE_LINE_SIZE	256
c5d972
-#define ZF_DIST		(CACHE_LINE_SIZE * 21)	// Zerofill distance
c5d972
-#define dest		x0
c5d972
-#define src		x1
c5d972
-#define n		x2	// size
c5d972
-#define tmp1		x3
c5d972
-#define tmp2		x4
c5d972
-#define tmp3		x5
c5d972
-#define rest		x6
c5d972
-#define dest_ptr	x7
c5d972
-#define src_ptr		x8
c5d972
-#define vector_length	x9
c5d972
-#define cl_remainder	x10	// CACHE_LINE_SIZE remainder
c5d972
+#define dstin	x0
c5d972
+#define src	x1
c5d972
+#define n	x2
c5d972
+#define dst	x3
c5d972
+#define dstend	x4
c5d972
+#define srcend	x5
c5d972
+#define tmp	x6
c5d972
+#define vlen	x7
c5d972
+#define vlen8	x8
c5d972
 
c5d972
 #if HAVE_AARCH64_SVE_ASM
c5d972
 # if IS_IN (libc)
c5d972
@@ -50,45 +45,37 @@
c5d972
 
c5d972
 	.arch armv8.2-a+sve
c5d972
 
c5d972
-	.macro dc_zva times
c5d972
-	dc	zva, tmp1
c5d972
-	add	tmp1, tmp1, CACHE_LINE_SIZE
c5d972
-	.if \times-1
c5d972
-	dc_zva "(\times-1)"
c5d972
-	.endif
c5d972
-	.endm
c5d972
-
c5d972
 	.macro ld1b_unroll8
c5d972
-	ld1b	z0.b, p0/z, [src_ptr, #0, mul vl]
c5d972
-	ld1b	z1.b, p0/z, [src_ptr, #1, mul vl]
c5d972
-	ld1b	z2.b, p0/z, [src_ptr, #2, mul vl]
c5d972
-	ld1b	z3.b, p0/z, [src_ptr, #3, mul vl]
c5d972
-	ld1b	z4.b, p0/z, [src_ptr, #4, mul vl]
c5d972
-	ld1b	z5.b, p0/z, [src_ptr, #5, mul vl]
c5d972
-	ld1b	z6.b, p0/z, [src_ptr, #6, mul vl]
c5d972
-	ld1b	z7.b, p0/z, [src_ptr, #7, mul vl]
c5d972
+	ld1b	z0.b, p0/z, [src, 0, mul vl]
c5d972
+	ld1b	z1.b, p0/z, [src, 1, mul vl]
c5d972
+	ld1b	z2.b, p0/z, [src, 2, mul vl]
c5d972
+	ld1b	z3.b, p0/z, [src, 3, mul vl]
c5d972
+	ld1b	z4.b, p0/z, [src, 4, mul vl]
c5d972
+	ld1b	z5.b, p0/z, [src, 5, mul vl]
c5d972
+	ld1b	z6.b, p0/z, [src, 6, mul vl]
c5d972
+	ld1b	z7.b, p0/z, [src, 7, mul vl]
c5d972
 	.endm
c5d972
 
c5d972
 	.macro stld1b_unroll4a
c5d972
-	st1b	z0.b, p0,   [dest_ptr, #0, mul vl]
c5d972
-	st1b	z1.b, p0,   [dest_ptr, #1, mul vl]
c5d972
-	ld1b	z0.b, p0/z, [src_ptr,  #0, mul vl]
c5d972
-	ld1b	z1.b, p0/z, [src_ptr,  #1, mul vl]
c5d972
-	st1b	z2.b, p0,   [dest_ptr, #2, mul vl]
c5d972
-	st1b	z3.b, p0,   [dest_ptr, #3, mul vl]
c5d972
-	ld1b	z2.b, p0/z, [src_ptr,  #2, mul vl]
c5d972
-	ld1b	z3.b, p0/z, [src_ptr,  #3, mul vl]
c5d972
+	st1b	z0.b, p0,   [dst, 0, mul vl]
c5d972
+	st1b	z1.b, p0,   [dst, 1, mul vl]
c5d972
+	ld1b	z0.b, p0/z, [src, 0, mul vl]
c5d972
+	ld1b	z1.b, p0/z, [src, 1, mul vl]
c5d972
+	st1b	z2.b, p0,   [dst, 2, mul vl]
c5d972
+	st1b	z3.b, p0,   [dst, 3, mul vl]
c5d972
+	ld1b	z2.b, p0/z, [src, 2, mul vl]
c5d972
+	ld1b	z3.b, p0/z, [src, 3, mul vl]
c5d972
 	.endm
c5d972
 
c5d972
 	.macro stld1b_unroll4b
c5d972
-	st1b	z4.b, p0,   [dest_ptr, #4, mul vl]
c5d972
-	st1b	z5.b, p0,   [dest_ptr, #5, mul vl]
c5d972
-	ld1b	z4.b, p0/z, [src_ptr,  #4, mul vl]
c5d972
-	ld1b	z5.b, p0/z, [src_ptr,  #5, mul vl]
c5d972
-	st1b	z6.b, p0,   [dest_ptr, #6, mul vl]
c5d972
-	st1b	z7.b, p0,   [dest_ptr, #7, mul vl]
c5d972
-	ld1b	z6.b, p0/z, [src_ptr,  #6, mul vl]
c5d972
-	ld1b	z7.b, p0/z, [src_ptr,  #7, mul vl]
c5d972
+	st1b	z4.b, p0,   [dst, 4, mul vl]
c5d972
+	st1b	z5.b, p0,   [dst, 5, mul vl]
c5d972
+	ld1b	z4.b, p0/z, [src, 4, mul vl]
c5d972
+	ld1b	z5.b, p0/z, [src, 5, mul vl]
c5d972
+	st1b	z6.b, p0,   [dst, 6, mul vl]
c5d972
+	st1b	z7.b, p0,   [dst, 7, mul vl]
c5d972
+	ld1b	z6.b, p0/z, [src, 6, mul vl]
c5d972
+	ld1b	z7.b, p0/z, [src, 7, mul vl]
c5d972
 	.endm
c5d972
 
c5d972
 	.macro stld1b_unroll8
c5d972
@@ -97,87 +84,18 @@
c5d972
 	.endm
c5d972
 
c5d972
 	.macro st1b_unroll8
c5d972
-	st1b	z0.b, p0, [dest_ptr, #0, mul vl]
c5d972
-	st1b	z1.b, p0, [dest_ptr, #1, mul vl]
c5d972
-	st1b	z2.b, p0, [dest_ptr, #2, mul vl]
c5d972
-	st1b	z3.b, p0, [dest_ptr, #3, mul vl]
c5d972
-	st1b	z4.b, p0, [dest_ptr, #4, mul vl]
c5d972
-	st1b	z5.b, p0, [dest_ptr, #5, mul vl]
c5d972
-	st1b	z6.b, p0, [dest_ptr, #6, mul vl]
c5d972
-	st1b	z7.b, p0, [dest_ptr, #7, mul vl]
c5d972
+	st1b	z0.b, p0, [dst, 0, mul vl]
c5d972
+	st1b	z1.b, p0, [dst, 1, mul vl]
c5d972
+	st1b	z2.b, p0, [dst, 2, mul vl]
c5d972
+	st1b	z3.b, p0, [dst, 3, mul vl]
c5d972
+	st1b	z4.b, p0, [dst, 4, mul vl]
c5d972
+	st1b	z5.b, p0, [dst, 5, mul vl]
c5d972
+	st1b	z6.b, p0, [dst, 6, mul vl]
c5d972
+	st1b	z7.b, p0, [dst, 7, mul vl]
c5d972
 	.endm
c5d972
 
c5d972
-	.macro shortcut_for_small_size exit
c5d972
-	// if rest <= vector_length * 2
c5d972
-	whilelo	p0.b, xzr, n
c5d972
-	whilelo	p1.b, vector_length, n
c5d972
-	b.last	1f
c5d972
-	ld1b	z0.b, p0/z, [src, #0, mul vl]
c5d972
-	ld1b	z1.b, p1/z, [src, #1, mul vl]
c5d972
-	st1b	z0.b, p0, [dest, #0, mul vl]
c5d972
-	st1b	z1.b, p1, [dest, #1, mul vl]
c5d972
-	ret
c5d972
-1:	// if rest > vector_length * 8
c5d972
-	cmp	n, vector_length, lsl 3 // vector_length * 8
c5d972
-	b.hi	\exit
c5d972
-	// if rest <= vector_length * 4
c5d972
-	lsl	tmp1, vector_length, 1  // vector_length * 2
c5d972
-	whilelo	p2.b, tmp1, n
c5d972
-	incb	tmp1
c5d972
-	whilelo	p3.b, tmp1, n
c5d972
-	b.last	1f
c5d972
-	ld1b	z0.b, p0/z, [src, #0, mul vl]
c5d972
-	ld1b	z1.b, p1/z, [src, #1, mul vl]
c5d972
-	ld1b	z2.b, p2/z, [src, #2, mul vl]
c5d972
-	ld1b	z3.b, p3/z, [src, #3, mul vl]
c5d972
-	st1b	z0.b, p0, [dest, #0, mul vl]
c5d972
-	st1b	z1.b, p1, [dest, #1, mul vl]
c5d972
-	st1b	z2.b, p2, [dest, #2, mul vl]
c5d972
-	st1b	z3.b, p3, [dest, #3, mul vl]
c5d972
-	ret
c5d972
-1:	// if rest <= vector_length * 8
c5d972
-	lsl	tmp1, vector_length, 2  // vector_length * 4
c5d972
-	whilelo	p4.b, tmp1, n
c5d972
-	incb	tmp1
c5d972
-	whilelo	p5.b, tmp1, n
c5d972
-	b.last	1f
c5d972
-	ld1b	z0.b, p0/z, [src, #0, mul vl]
c5d972
-	ld1b	z1.b, p1/z, [src, #1, mul vl]
c5d972
-	ld1b	z2.b, p2/z, [src, #2, mul vl]
c5d972
-	ld1b	z3.b, p3/z, [src, #3, mul vl]
c5d972
-	ld1b	z4.b, p4/z, [src, #4, mul vl]
c5d972
-	ld1b	z5.b, p5/z, [src, #5, mul vl]
c5d972
-	st1b	z0.b, p0, [dest, #0, mul vl]
c5d972
-	st1b	z1.b, p1, [dest, #1, mul vl]
c5d972
-	st1b	z2.b, p2, [dest, #2, mul vl]
c5d972
-	st1b	z3.b, p3, [dest, #3, mul vl]
c5d972
-	st1b	z4.b, p4, [dest, #4, mul vl]
c5d972
-	st1b	z5.b, p5, [dest, #5, mul vl]
c5d972
-	ret
c5d972
-1:	lsl	tmp1, vector_length, 2	// vector_length * 4
c5d972
-	incb	tmp1			// vector_length * 5
c5d972
-	incb	tmp1			// vector_length * 6
c5d972
-	whilelo	p6.b, tmp1, n
c5d972
-	incb	tmp1
c5d972
-	whilelo	p7.b, tmp1, n
c5d972
-	ld1b	z0.b, p0/z, [src, #0, mul vl]
c5d972
-	ld1b	z1.b, p1/z, [src, #1, mul vl]
c5d972
-	ld1b	z2.b, p2/z, [src, #2, mul vl]
c5d972
-	ld1b	z3.b, p3/z, [src, #3, mul vl]
c5d972
-	ld1b	z4.b, p4/z, [src, #4, mul vl]
c5d972
-	ld1b	z5.b, p5/z, [src, #5, mul vl]
c5d972
-	ld1b	z6.b, p6/z, [src, #6, mul vl]
c5d972
-	ld1b	z7.b, p7/z, [src, #7, mul vl]
c5d972
-	st1b	z0.b, p0, [dest, #0, mul vl]
c5d972
-	st1b	z1.b, p1, [dest, #1, mul vl]
c5d972
-	st1b	z2.b, p2, [dest, #2, mul vl]
c5d972
-	st1b	z3.b, p3, [dest, #3, mul vl]
c5d972
-	st1b	z4.b, p4, [dest, #4, mul vl]
c5d972
-	st1b	z5.b, p5, [dest, #5, mul vl]
c5d972
-	st1b	z6.b, p6, [dest, #6, mul vl]
c5d972
-	st1b	z7.b, p7, [dest, #7, mul vl]
c5d972
-	ret
c5d972
-	.endm
c5d972
+#undef BTI_C
c5d972
+#define BTI_C
c5d972
 
c5d972
 ENTRY (MEMCPY)
c5d972
 
c5d972
@@ -185,223 +103,209 @@ ENTRY (MEMCPY)
c5d972
 	PTR_ARG (1)
c5d972
 	SIZE_ARG (2)
c5d972
 
c5d972
-L(memcpy):
c5d972
-	cntb	vector_length
c5d972
-	// shortcut for less than vector_length * 8
c5d972
-	// gives a free ptrue to p0.b for n >= vector_length
c5d972
-	shortcut_for_small_size L(vl_agnostic)
c5d972
-	// end of shortcut
c5d972
-
c5d972
-L(vl_agnostic): // VL Agnostic
c5d972
-	mov	rest, n
c5d972
-	mov	dest_ptr, dest
c5d972
-	mov	src_ptr, src
c5d972
-	// if rest >= L2_SIZE && vector_length == 64 then L(L2)
c5d972
-	mov	tmp1, 64
c5d972
-	cmp	rest, L2_SIZE
c5d972
-	ccmp	vector_length, tmp1, 0, cs
c5d972
-	b.eq	L(L2)
c5d972
-
c5d972
-L(unroll8): // unrolling and software pipeline
c5d972
-	lsl	tmp1, vector_length, 3	// vector_length * 8
c5d972
-	.p2align 3
c5d972
-	cmp	 rest, tmp1
c5d972
-	b.cc	L(last)
c5d972
+	cntb	vlen
c5d972
+	cmp	n, vlen, lsl 1
c5d972
+	b.hi	L(copy_small)
c5d972
+	whilelo	p1.b, vlen, n
c5d972
+	whilelo	p0.b, xzr, n
c5d972
+	ld1b	z0.b, p0/z, [src, 0, mul vl]
c5d972
+	ld1b	z1.b, p1/z, [src, 1, mul vl]
c5d972
+	st1b	z0.b, p0, [dstin, 0, mul vl]
c5d972
+	st1b	z1.b, p1, [dstin, 1, mul vl]
c5d972
+	ret
c5d972
+
c5d972
+	.p2align 4
c5d972
+
c5d972
+L(copy_small):
c5d972
+	cmp	n, vlen, lsl 3
c5d972
+	b.hi	L(copy_large)
c5d972
+	add	dstend, dstin, n
c5d972
+	add	srcend, src, n
c5d972
+	cmp	n, vlen, lsl 2
c5d972
+	b.hi	1f
c5d972
+
c5d972
+	/* Copy 2-4 vectors.  */
c5d972
+	ptrue	p0.b
c5d972
+	ld1b	z0.b, p0/z, [src, 0, mul vl]
c5d972
+	ld1b	z1.b, p0/z, [src, 1, mul vl]
c5d972
+	ld1b	z2.b, p0/z, [srcend, -2, mul vl]
c5d972
+	ld1b	z3.b, p0/z, [srcend, -1, mul vl]
c5d972
+	st1b	z0.b, p0, [dstin, 0, mul vl]
c5d972
+	st1b	z1.b, p0, [dstin, 1, mul vl]
c5d972
+	st1b	z2.b, p0, [dstend, -2, mul vl]
c5d972
+	st1b	z3.b, p0, [dstend, -1, mul vl]
c5d972
+	ret
c5d972
+
c5d972
+	.p2align 4
c5d972
+	/* Copy 4-8 vectors.  */
c5d972
+1:	ptrue	p0.b
c5d972
+	ld1b	z0.b, p0/z, [src, 0, mul vl]
c5d972
+	ld1b	z1.b, p0/z, [src, 1, mul vl]
c5d972
+	ld1b	z2.b, p0/z, [src, 2, mul vl]
c5d972
+	ld1b	z3.b, p0/z, [src, 3, mul vl]
c5d972
+	ld1b	z4.b, p0/z, [srcend, -4, mul vl]
c5d972
+	ld1b	z5.b, p0/z, [srcend, -3, mul vl]
c5d972
+	ld1b	z6.b, p0/z, [srcend, -2, mul vl]
c5d972
+	ld1b	z7.b, p0/z, [srcend, -1, mul vl]
c5d972
+	st1b	z0.b, p0, [dstin, 0, mul vl]
c5d972
+	st1b	z1.b, p0, [dstin, 1, mul vl]
c5d972
+	st1b	z2.b, p0, [dstin, 2, mul vl]
c5d972
+	st1b	z3.b, p0, [dstin, 3, mul vl]
c5d972
+	st1b	z4.b, p0, [dstend, -4, mul vl]
c5d972
+	st1b	z5.b, p0, [dstend, -3, mul vl]
c5d972
+	st1b	z6.b, p0, [dstend, -2, mul vl]
c5d972
+	st1b	z7.b, p0, [dstend, -1, mul vl]
c5d972
+	ret
c5d972
+
c5d972
+	.p2align 4
c5d972
+	/* At least 8 vectors - always align to vector length for
c5d972
+	   higher and consistent write performance.  */
c5d972
+L(copy_large):
c5d972
+	sub	tmp, vlen, 1
c5d972
+	and	tmp, dstin, tmp
c5d972
+	sub	tmp, vlen, tmp
c5d972
+	whilelo	p1.b, xzr, tmp
c5d972
+	ld1b	z1.b, p1/z, [src]
c5d972
+	st1b	z1.b, p1, [dstin]
c5d972
+	add	dst, dstin, tmp
c5d972
+	add	src, src, tmp
c5d972
+	sub	n, n, tmp
c5d972
+	ptrue	p0.b
c5d972
+
c5d972
+	lsl	vlen8, vlen, 3
c5d972
+	subs	n, n, vlen8
c5d972
+	b.ls	3f
c5d972
 	ld1b_unroll8
c5d972
-	add	src_ptr, src_ptr, tmp1
c5d972
-	sub	rest, rest, tmp1
c5d972
-	cmp	rest, tmp1
c5d972
-	b.cc	2f
c5d972
-	.p2align 3
c5d972
+	add	src, src, vlen8
c5d972
+	subs	n, n, vlen8
c5d972
+	b.ls	2f
c5d972
+
c5d972
+	.p2align 4
c5d972
+	/* 8x unrolled and software pipelined loop.  */
c5d972
 1:	stld1b_unroll8
c5d972
-	add	dest_ptr, dest_ptr, tmp1
c5d972
-	add	src_ptr, src_ptr, tmp1
c5d972
-	sub	rest, rest, tmp1
c5d972
-	cmp	rest, tmp1
c5d972
-	b.ge	1b
c5d972
+	add	dst, dst, vlen8
c5d972
+	add	src, src, vlen8
c5d972
+	subs	n, n, vlen8
c5d972
+	b.hi	1b
c5d972
 2:	st1b_unroll8
c5d972
-	add	dest_ptr, dest_ptr, tmp1
c5d972
-
c5d972
-	.p2align 3
c5d972
-L(last):
c5d972
-	whilelo	p0.b, xzr, rest
c5d972
-	whilelo	p1.b, vector_length, rest
c5d972
-	b.last	1f
c5d972
-	ld1b	z0.b, p0/z, [src_ptr, #0, mul vl]
c5d972
-	ld1b	z1.b, p1/z, [src_ptr, #1, mul vl]
c5d972
-	st1b	z0.b, p0, [dest_ptr, #0, mul vl]
c5d972
-	st1b	z1.b, p1, [dest_ptr, #1, mul vl]
c5d972
-	ret
c5d972
-1:	lsl	tmp1, vector_length, 1	// vector_length * 2
c5d972
-	whilelo	p2.b, tmp1, rest
c5d972
-	incb	tmp1
c5d972
-	whilelo	p3.b, tmp1, rest
c5d972
-	b.last	1f
c5d972
-	ld1b	z0.b, p0/z, [src_ptr, #0, mul vl]
c5d972
-	ld1b	z1.b, p1/z, [src_ptr, #1, mul vl]
c5d972
-	ld1b	z2.b, p2/z, [src_ptr, #2, mul vl]
c5d972
-	ld1b	z3.b, p3/z, [src_ptr, #3, mul vl]
c5d972
-	st1b	z0.b, p0, [dest_ptr, #0, mul vl]
c5d972
-	st1b	z1.b, p1, [dest_ptr, #1, mul vl]
c5d972
-	st1b	z2.b, p2, [dest_ptr, #2, mul vl]
c5d972
-	st1b	z3.b, p3, [dest_ptr, #3, mul vl]
c5d972
+	add	dst, dst, vlen8
c5d972
+3:	add	n, n, vlen8
c5d972
+
c5d972
+	/* Move last 0-8 vectors.  */
c5d972
+L(last_bytes):
c5d972
+	cmp	n, vlen, lsl 1
c5d972
+	b.hi	1f
c5d972
+	whilelo	p0.b, xzr, n
c5d972
+	whilelo	p1.b, vlen, n
c5d972
+	ld1b	z0.b, p0/z, [src, 0, mul vl]
c5d972
+	ld1b	z1.b, p1/z, [src, 1, mul vl]
c5d972
+	st1b	z0.b, p0, [dst, 0, mul vl]
c5d972
+	st1b	z1.b, p1, [dst, 1, mul vl]
c5d972
 	ret
c5d972
-1:	lsl	tmp1, vector_length, 2	// vector_length * 4
c5d972
-	whilelo	p4.b, tmp1, rest
c5d972
-	incb	tmp1
c5d972
-	whilelo	p5.b, tmp1, rest
c5d972
-	incb	tmp1
c5d972
-	whilelo	p6.b, tmp1, rest
c5d972
-	incb	tmp1
c5d972
-	whilelo	p7.b, tmp1, rest
c5d972
-	ld1b	z0.b, p0/z, [src_ptr, #0, mul vl]
c5d972
-	ld1b	z1.b, p1/z, [src_ptr, #1, mul vl]
c5d972
-	ld1b	z2.b, p2/z, [src_ptr, #2, mul vl]
c5d972
-	ld1b	z3.b, p3/z, [src_ptr, #3, mul vl]
c5d972
-	ld1b	z4.b, p4/z, [src_ptr, #4, mul vl]
c5d972
-	ld1b	z5.b, p5/z, [src_ptr, #5, mul vl]
c5d972
-	ld1b	z6.b, p6/z, [src_ptr, #6, mul vl]
c5d972
-	ld1b	z7.b, p7/z, [src_ptr, #7, mul vl]
c5d972
-	st1b	z0.b, p0, [dest_ptr, #0, mul vl]
c5d972
-	st1b	z1.b, p1, [dest_ptr, #1, mul vl]
c5d972
-	st1b	z2.b, p2, [dest_ptr, #2, mul vl]
c5d972
-	st1b	z3.b, p3, [dest_ptr, #3, mul vl]
c5d972
-	st1b	z4.b, p4, [dest_ptr, #4, mul vl]
c5d972
-	st1b	z5.b, p5, [dest_ptr, #5, mul vl]
c5d972
-	st1b	z6.b, p6, [dest_ptr, #6, mul vl]
c5d972
-	st1b	z7.b, p7, [dest_ptr, #7, mul vl]
c5d972
+
c5d972
+	.p2align 4
c5d972
+
c5d972
+1:	add	srcend, src, n
c5d972
+	add	dstend, dst, n
c5d972
+	ld1b	z0.b, p0/z, [src, 0, mul vl]
c5d972
+	ld1b	z1.b, p0/z, [src, 1, mul vl]
c5d972
+	ld1b	z2.b, p0/z, [srcend, -2, mul vl]
c5d972
+	ld1b	z3.b, p0/z, [srcend, -1, mul vl]
c5d972
+	cmp	n, vlen, lsl 2
c5d972
+	b.hi	1f
c5d972
+
c5d972
+	st1b	z0.b, p0, [dst, 0, mul vl]
c5d972
+	st1b	z1.b, p0, [dst, 1, mul vl]
c5d972
+	st1b	z2.b, p0, [dstend, -2, mul vl]
c5d972
+	st1b	z3.b, p0, [dstend, -1, mul vl]
c5d972
 	ret
c5d972
 
c5d972
-L(L2):
c5d972
-	// align dest address at CACHE_LINE_SIZE byte boundary
c5d972
-	mov	tmp1, CACHE_LINE_SIZE
c5d972
-	ands	tmp2, dest_ptr, CACHE_LINE_SIZE - 1
c5d972
-	// if cl_remainder == 0
c5d972
-	b.eq	L(L2_dc_zva)
c5d972
-	sub	cl_remainder, tmp1, tmp2
c5d972
-	// process remainder until the first CACHE_LINE_SIZE boundary
c5d972
-	whilelo	p1.b, xzr, cl_remainder	// keep p0.b all true
c5d972
-	whilelo	p2.b, vector_length, cl_remainder
c5d972
-	b.last	1f
c5d972
-	ld1b	z1.b, p1/z, [src_ptr, #0, mul vl]
c5d972
-	ld1b	z2.b, p2/z, [src_ptr, #1, mul vl]
c5d972
-	st1b	z1.b, p1, [dest_ptr, #0, mul vl]
c5d972
-	st1b	z2.b, p2, [dest_ptr, #1, mul vl]
c5d972
-	b	2f
c5d972
-1:	lsl	tmp1, vector_length, 1	// vector_length * 2
c5d972
-	whilelo	p3.b, tmp1, cl_remainder
c5d972
-	incb	tmp1
c5d972
-	whilelo	p4.b, tmp1, cl_remainder
c5d972
-	ld1b	z1.b, p1/z, [src_ptr, #0, mul vl]
c5d972
-	ld1b	z2.b, p2/z, [src_ptr, #1, mul vl]
c5d972
-	ld1b	z3.b, p3/z, [src_ptr, #2, mul vl]
c5d972
-	ld1b	z4.b, p4/z, [src_ptr, #3, mul vl]
c5d972
-	st1b	z1.b, p1, [dest_ptr, #0, mul vl]
c5d972
-	st1b	z2.b, p2, [dest_ptr, #1, mul vl]
c5d972
-	st1b	z3.b, p3, [dest_ptr, #2, mul vl]
c5d972
-	st1b	z4.b, p4, [dest_ptr, #3, mul vl]
c5d972
-2:	add	dest_ptr, dest_ptr, cl_remainder
c5d972
-	add	src_ptr, src_ptr, cl_remainder
c5d972
-	sub	rest, rest, cl_remainder
c5d972
-
c5d972
-L(L2_dc_zva):
c5d972
-	// zero fill
c5d972
-	and	tmp1, dest, 0xffffffffffffff
c5d972
-	and	tmp2, src, 0xffffffffffffff
c5d972
-	subs	tmp1, tmp1, tmp2	// diff
c5d972
-	b.ge	1f
c5d972
-	neg	tmp1, tmp1
c5d972
-1:	mov	tmp3, ZF_DIST + CACHE_LINE_SIZE * 2
c5d972
-	cmp	tmp1, tmp3
c5d972
-	b.lo	L(unroll8)
c5d972
-	mov	tmp1, dest_ptr
c5d972
-	dc_zva	(ZF_DIST / CACHE_LINE_SIZE) - 1
c5d972
-	// unroll
c5d972
-	ld1b_unroll8	// this line has to be after "b.lo L(unroll8)"
c5d972
-	add	 src_ptr, src_ptr, CACHE_LINE_SIZE * 2
c5d972
-	sub	 rest, rest, CACHE_LINE_SIZE * 2
c5d972
-	mov	 tmp1, ZF_DIST
c5d972
-	.p2align 3
c5d972
-1:	stld1b_unroll4a
c5d972
-	add	tmp2, dest_ptr, tmp1	// dest_ptr + ZF_DIST
c5d972
-	dc	zva, tmp2
c5d972
-	stld1b_unroll4b
c5d972
-	add	tmp2, tmp2, CACHE_LINE_SIZE
c5d972
-	dc	zva, tmp2
c5d972
-	add	dest_ptr, dest_ptr, CACHE_LINE_SIZE * 2
c5d972
-	add	src_ptr, src_ptr, CACHE_LINE_SIZE * 2
c5d972
-	sub	rest, rest, CACHE_LINE_SIZE * 2
c5d972
-	cmp	rest, tmp3	// ZF_DIST + CACHE_LINE_SIZE * 2
c5d972
-	b.ge	1b
c5d972
-	st1b_unroll8
c5d972
-	add	dest_ptr, dest_ptr, CACHE_LINE_SIZE * 2
c5d972
-	b	L(unroll8)
c5d972
+1:	ld1b	z4.b, p0/z, [src, 2, mul vl]
c5d972
+	ld1b	z5.b, p0/z, [src, 3, mul vl]
c5d972
+	ld1b	z6.b, p0/z, [srcend, -4, mul vl]
c5d972
+	ld1b	z7.b, p0/z, [srcend, -3, mul vl]
c5d972
+	st1b	z0.b, p0, [dst, 0, mul vl]
c5d972
+	st1b	z1.b, p0, [dst, 1, mul vl]
c5d972
+	st1b	z4.b, p0, [dst, 2, mul vl]
c5d972
+	st1b	z5.b, p0, [dst, 3, mul vl]
c5d972
+	st1b	z6.b, p0, [dstend, -4, mul vl]
c5d972
+	st1b	z7.b, p0, [dstend, -3, mul vl]
c5d972
+	st1b	z2.b, p0, [dstend, -2, mul vl]
c5d972
+	st1b	z3.b, p0, [dstend, -1, mul vl]
c5d972
+	ret
c5d972
 
c5d972
 END (MEMCPY)
c5d972
 libc_hidden_builtin_def (MEMCPY)
c5d972
 
c5d972
 
c5d972
-ENTRY (MEMMOVE)
c5d972
+ENTRY_ALIGN (MEMMOVE, 4)
c5d972
 
c5d972
 	PTR_ARG (0)
c5d972
 	PTR_ARG (1)
c5d972
 	SIZE_ARG (2)
c5d972
 
c5d972
-	// remove tag address
c5d972
-	// dest has to be immutable because it is the return value
c5d972
-	// src has to be immutable because it is used in L(bwd_last)
c5d972
-	and	tmp2, dest, 0xffffffffffffff	// save dest_notag into tmp2
c5d972
-	and	tmp3, src, 0xffffffffffffff	// save src_notag intp tmp3
c5d972
-	cmp	n, 0
c5d972
-	ccmp	tmp2, tmp3, 4, ne
c5d972
-	b.ne	1f
c5d972
+	/* Fast case for up to 2 vectors.  */
c5d972
+	cntb	vlen
c5d972
+	cmp	n, vlen, lsl 1
c5d972
+	b.hi	1f
c5d972
+	whilelo	p0.b, xzr, n
c5d972
+	whilelo	p1.b, vlen, n
c5d972
+	ld1b	z0.b, p0/z, [src, 0, mul vl]
c5d972
+	ld1b	z1.b, p1/z, [src, 1, mul vl]
c5d972
+	st1b	z0.b, p0, [dstin, 0, mul vl]
c5d972
+	st1b	z1.b, p1, [dstin, 1, mul vl]
c5d972
+L(full_overlap):
c5d972
 	ret
c5d972
-1:	cntb	vector_length
c5d972
-	// shortcut for less than vector_length * 8
c5d972
-	// gives a free ptrue to p0.b for n >= vector_length
c5d972
-	// tmp2 and tmp3 should not be used in this macro to keep
c5d972
-	// notag addresses
c5d972
-	shortcut_for_small_size L(dispatch)
c5d972
-	// end of shortcut
c5d972
-
c5d972
-L(dispatch):
c5d972
-	// tmp2 = dest_notag, tmp3 = src_notag
c5d972
-	// diff = dest_notag - src_notag
c5d972
-	sub	tmp1, tmp2, tmp3
c5d972
-	// if diff <= 0 || diff >= n then memcpy
c5d972
-	cmp	tmp1, 0
c5d972
-	ccmp	tmp1, n, 2, gt
c5d972
-	b.cs	L(vl_agnostic)
c5d972
-
c5d972
-L(bwd_start):
c5d972
-	mov	rest, n
c5d972
-	add	dest_ptr, dest, n	// dest_end
c5d972
-	add	src_ptr, src, n		// src_end
c5d972
-
c5d972
-L(bwd_unroll8): // unrolling and software pipeline
c5d972
-	lsl	tmp1, vector_length, 3	// vector_length * 8
c5d972
-	.p2align 3
c5d972
-	cmp	rest, tmp1
c5d972
-	b.cc	L(bwd_last)
c5d972
-	sub	src_ptr, src_ptr, tmp1
c5d972
+
c5d972
+	.p2align 4
c5d972
+	/* Check for overlapping moves. Return if there is a full overlap.
c5d972
+	   Small moves up to 8 vectors use the overlap-safe copy_small code.
c5d972
+	   Non-overlapping or overlapping moves with dst < src use memcpy.
c5d972
+	   Overlapping moves with dst > src use a backward copy loop.  */
c5d972
+1:	sub	tmp, dstin, src
c5d972
+	ands	tmp, tmp, 0xffffffffffffff	/* Clear special tag bits.  */
c5d972
+	b.eq	L(full_overlap)
c5d972
+	cmp	n, vlen, lsl 3
c5d972
+	b.ls	L(copy_small)
c5d972
+	cmp	tmp, n
c5d972
+	b.hs	L(copy_large)
c5d972
+
c5d972
+	/* Align to vector length.  */
c5d972
+	add	dst, dstin, n
c5d972
+	sub	tmp, vlen, 1
c5d972
+	ands	tmp, dst, tmp
c5d972
+	csel	tmp, tmp, vlen, ne
c5d972
+	whilelo	p1.b, xzr, tmp
c5d972
+	sub	n, n, tmp
c5d972
+	ld1b	z1.b, p1/z, [src, n]
c5d972
+	st1b	z1.b, p1, [dstin, n]
c5d972
+	add	src, src, n
c5d972
+	add	dst, dstin, n
c5d972
+
c5d972
+	ptrue	p0.b
c5d972
+	lsl	vlen8, vlen, 3
c5d972
+	subs	n, n, vlen8
c5d972
+	b.ls	3f
c5d972
+	sub	src, src, vlen8
c5d972
 	ld1b_unroll8
c5d972
-	sub	rest, rest, tmp1
c5d972
-	cmp	rest, tmp1
c5d972
-	b.cc	2f
c5d972
-	.p2align 3
c5d972
-1:	sub	src_ptr, src_ptr, tmp1
c5d972
-	sub	dest_ptr, dest_ptr, tmp1
c5d972
+	subs	n, n, vlen8
c5d972
+	b.ls	2f
c5d972
+
c5d972
+	.p2align 4
c5d972
+	/* 8x unrolled and software pipelined backward copy loop.  */
c5d972
+1:	sub	src, src, vlen8
c5d972
+	sub	dst, dst, vlen8
c5d972
 	stld1b_unroll8
c5d972
-	sub	rest, rest, tmp1
c5d972
-	cmp	rest, tmp1
c5d972
-	b.ge	1b
c5d972
-2:	sub	dest_ptr, dest_ptr, tmp1
c5d972
+	subs	n, n, vlen8
c5d972
+	b.hi	1b
c5d972
+2:	sub	dst, dst, vlen8
c5d972
 	st1b_unroll8
c5d972
+3:	add	n, n, vlen8
c5d972
 
c5d972
-L(bwd_last):
c5d972
-	mov	dest_ptr, dest
c5d972
-	mov	src_ptr, src
c5d972
-	b	L(last)
c5d972
+	/* Adjust src/dst for last 0-8 vectors.  */
c5d972
+	sub	src, src, n
c5d972
+	mov	dst, dstin
c5d972
+	b	L(last_bytes)
c5d972
 
c5d972
 END (MEMMOVE)
c5d972
 libc_hidden_builtin_def (MEMMOVE)
c5d972
-- 
c5d972
2.31.1
c5d972