5de29b
# commit 3be87c77d24c4456ccca4034363b6d1814cd0c84
5de29b
# Author: Alan Modra <amodra@gmail.com>
5de29b
# Date:   Sat Aug 17 18:47:59 2013 +0930
5de29b
# 
5de29b
#     PowerPC LE memset
5de29b
#     http://sourceware.org/ml/libc-alpha/2013-08/msg00104.html
5de29b
#     
5de29b
#     One of the things I noticed when looking at power7 timing is that rlwimi
5de29b
#     is cracked and the two resulting insns have a register dependency.
5de29b
#     That makes it a little slower than the equivalent rldimi.
5de29b
#     
5de29b
#         * sysdeps/powerpc/powerpc64/memset.S: Replace rlwimi with
5de29b
#             insrdi.  Formatting.
5de29b
#         * sysdeps/powerpc/powerpc64/power4/memset.S: Likewise.
5de29b
#         * sysdeps/powerpc/powerpc64/power6/memset.S: Likewise.
5de29b
#         * sysdeps/powerpc/powerpc64/power7/memset.S: Likewise.
5de29b
#         * sysdeps/powerpc/powerpc32/power4/memset.S: Likewise.
5de29b
#         * sysdeps/powerpc/powerpc32/power6/memset.S: Likewise.
5de29b
#         * sysdeps/powerpc/powerpc32/power7/memset.S: Likewise.
5de29b
# 
12745e
diff -urN glibc-2.17-c758a686/sysdeps/powerpc/powerpc32/power4/memset.S glibc-2.17-c758a686/sysdeps/powerpc/powerpc32/power4/memset.S
12745e
--- glibc-2.17-c758a686/sysdeps/powerpc/powerpc32/power4/memset.S	2014-05-29 13:07:41.000000000 -0500
12745e
+++ glibc-2.17-c758a686/sysdeps/powerpc/powerpc32/power4/memset.S	2014-05-29 13:07:46.000000000 -0500
5de29b
@@ -52,7 +52,7 @@
5de29b
 
5de29b
 /* Align to word boundary.  */
5de29b
 	cmplwi	cr5, rLEN, 31
5de29b
-	rlwimi	rCHR, rCHR, 8, 16, 23 /* Replicate byte to halfword.  */
5de29b
+	insrdi	rCHR, rCHR, 8, 48     /* Replicate byte to halfword.  */
5de29b
 	beq+	L(aligned)
5de29b
 	mtcrf	0x01, rMEMP0
5de29b
 	subfic	rALIGN, rALIGN, 4
5de29b
@@ -67,7 +67,7 @@
5de29b
 /* Handle the case of size < 31.  */
5de29b
 L(aligned):
5de29b
 	mtcrf	0x01, rLEN
5de29b
-	rlwimi	rCHR, rCHR, 16, 0, 15 /* Replicate halfword to word.  */
5de29b
+	insrdi	rCHR, rCHR, 16, 32    /* Replicate halfword to word.  */
5de29b
 	ble	cr5, L(medium)
5de29b
 /* Align to 32-byte boundary.  */
5de29b
 	andi.	rALIGN, rMEMP, 0x1C
12745e
diff -urN glibc-2.17-c758a686/sysdeps/powerpc/powerpc32/power6/memset.S glibc-2.17-c758a686/sysdeps/powerpc/powerpc32/power6/memset.S
12745e
--- glibc-2.17-c758a686/sysdeps/powerpc/powerpc32/power6/memset.S	2014-05-29 13:07:41.000000000 -0500
12745e
+++ glibc-2.17-c758a686/sysdeps/powerpc/powerpc32/power6/memset.S	2014-05-29 13:07:46.000000000 -0500
5de29b
@@ -50,7 +50,7 @@
5de29b
 	ble-	cr1, L(small)
5de29b
 /* Align to word boundary.  */
5de29b
 	cmplwi	cr5, rLEN, 31
5de29b
-	rlwimi	rCHR, rCHR, 8, 16, 23 /* Replicate byte to halfword.  */
5de29b
+	insrdi	rCHR, rCHR, 8, 48	/* Replicate byte to halfword.  */
5de29b
 	beq+	L(aligned)
5de29b
 	mtcrf	0x01, rMEMP0
5de29b
 	subfic	rALIGN, rALIGN, 4
5de29b
@@ -66,7 +66,7 @@
5de29b
 /* Handle the case of size < 31.  */
5de29b
 L(aligned):
5de29b
 	mtcrf	0x01, rLEN
5de29b
-	rlwimi	rCHR, rCHR, 16, 0, 15 /* Replicate halfword to word.  */
5de29b
+	insrdi	rCHR, rCHR, 16, 32	/* Replicate halfword to word.  */
5de29b
 	ble	cr5, L(medium)
5de29b
 /* Align to 32-byte boundary.  */
5de29b
 	andi.	rALIGN, rMEMP, 0x1C
12745e
diff -urN glibc-2.17-c758a686/sysdeps/powerpc/powerpc32/power7/memset.S glibc-2.17-c758a686/sysdeps/powerpc/powerpc32/power7/memset.S
12745e
--- glibc-2.17-c758a686/sysdeps/powerpc/powerpc32/power7/memset.S	2014-05-29 13:07:41.000000000 -0500
12745e
+++ glibc-2.17-c758a686/sysdeps/powerpc/powerpc32/power7/memset.S	2014-05-29 13:07:46.000000000 -0500
5de29b
@@ -37,8 +37,8 @@
5de29b
 	cfi_offset(31,-8)
5de29b
 
5de29b
 	/* Replicate byte to word.  */
5de29b
-	rlwimi	4,4,8,16,23
5de29b
-	rlwimi	4,4,16,0,15
5de29b
+	insrdi	4,4,8,48
5de29b
+	insrdi	4,4,16,32
5de29b
 
5de29b
 	ble	cr6,L(small)	/* If length <= 8, use short copy code.  */
5de29b
 
12745e
diff -urN glibc-2.17-c758a686/sysdeps/powerpc/powerpc64/memset.S glibc-2.17-c758a686/sysdeps/powerpc/powerpc64/memset.S
12745e
--- glibc-2.17-c758a686/sysdeps/powerpc/powerpc64/memset.S	2014-05-29 13:07:41.000000000 -0500
12745e
+++ glibc-2.17-c758a686/sysdeps/powerpc/powerpc64/memset.S	2014-05-29 13:07:46.000000000 -0500
5de29b
@@ -73,14 +73,14 @@
5de29b
 
5de29b
 /* Align to doubleword boundary.  */
5de29b
 	cmpldi	cr5, rLEN, 31
5de29b
-	rlwimi	rCHR, rCHR, 8, 16, 23 /* Replicate byte to halfword.  */
5de29b
+	insrdi	rCHR, rCHR, 8, 48	/* Replicate byte to halfword.  */
5de29b
 	beq+	L(aligned2)
5de29b
 	mtcrf	0x01, rMEMP0
5de29b
 	subfic	rALIGN, rALIGN, 8
5de29b
 	cror	28,30,31		/* Detect odd word aligned.  */
5de29b
 	add	rMEMP, rMEMP, rALIGN
5de29b
 	sub	rLEN, rLEN, rALIGN
5de29b
-	rlwimi	rCHR, rCHR, 16, 0, 15 /* Replicate halfword to word.  */
5de29b
+	insrdi	rCHR, rCHR, 16, 32	/* Replicate halfword to word.  */
5de29b
 	bt	29, L(g4)
5de29b
 /* Process the even word of doubleword.  */
5de29b
 	bf+	31, L(g2)
5de29b
@@ -102,14 +102,14 @@
5de29b
 
5de29b
 /* Handle the case of size < 31.  */
5de29b
 L(aligned2):
5de29b
-	rlwimi	rCHR, rCHR, 16, 0, 15 /* Replicate halfword to word.  */
5de29b
+	insrdi	rCHR, rCHR, 16, 32	/* Replicate halfword to word.  */
5de29b
 L(aligned):
5de29b
 	mtcrf	0x01, rLEN
5de29b
 	ble	cr5, L(medium)
5de29b
 /* Align to 32-byte boundary.  */
5de29b
 	andi.	rALIGN, rMEMP, 0x18
5de29b
 	subfic	rALIGN, rALIGN, 0x20
5de29b
-	insrdi	rCHR,rCHR,32,0 /* Replicate word to double word. */
5de29b
+	insrdi	rCHR, rCHR, 32, 0	/* Replicate word to double word. */
5de29b
 	beq	L(caligned)
5de29b
 	mtcrf	0x01, rALIGN
5de29b
 	add	rMEMP, rMEMP, rALIGN
5de29b
@@ -230,7 +230,7 @@
5de29b
 /* Memset of 0-31 bytes.  */
5de29b
 	.align 5
5de29b
 L(medium):
5de29b
-	insrdi	rCHR,rCHR,32,0 /* Replicate word to double word.  */
5de29b
+	insrdi	rCHR, rCHR, 32, 0	/* Replicate word to double word.  */
5de29b
 	cmpldi	cr1, rLEN, 16
5de29b
 L(medium_tail2):
5de29b
 	add	rMEMP, rMEMP, rLEN
12745e
diff -urN glibc-2.17-c758a686/sysdeps/powerpc/powerpc64/power4/memset.S glibc-2.17-c758a686/sysdeps/powerpc/powerpc64/power4/memset.S
12745e
--- glibc-2.17-c758a686/sysdeps/powerpc/powerpc64/power4/memset.S	2014-05-29 13:07:41.000000000 -0500
12745e
+++ glibc-2.17-c758a686/sysdeps/powerpc/powerpc64/power4/memset.S	2014-05-29 13:07:46.000000000 -0500
5de29b
@@ -68,14 +68,14 @@
5de29b
 
5de29b
 /* Align to doubleword boundary.  */
5de29b
 	cmpldi	cr5, rLEN, 31
5de29b
-	rlwimi	rCHR, rCHR, 8, 16, 23 /* Replicate byte to halfword.  */
5de29b
+	insrdi	rCHR, rCHR, 8, 48	/* Replicate byte to halfword.  */
5de29b
 	beq+	L(aligned2)
5de29b
 	mtcrf	0x01, rMEMP0
5de29b
 	subfic	rALIGN, rALIGN, 8
5de29b
 	cror	28,30,31		/* Detect odd word aligned.  */
5de29b
 	add	rMEMP, rMEMP, rALIGN
5de29b
 	sub	rLEN, rLEN, rALIGN
5de29b
-	rlwimi	rCHR, rCHR, 16, 0, 15 /* Replicate halfword to word.  */
5de29b
+	insrdi	rCHR, rCHR, 16, 32	/* Replicate halfword to word.  */
5de29b
 	bt	29, L(g4)
5de29b
 /* Process the even word of doubleword.  */
5de29b
 	bf+	31, L(g2)
5de29b
@@ -97,14 +97,14 @@
5de29b
 
5de29b
 /* Handle the case of size < 31.  */
5de29b
 L(aligned2):
5de29b
-	rlwimi	rCHR, rCHR, 16, 0, 15 /* Replicate halfword to word.  */
5de29b
+	insrdi	rCHR, rCHR, 16, 32	/* Replicate halfword to word.  */
5de29b
 L(aligned):
5de29b
 	mtcrf	0x01, rLEN
5de29b
 	ble	cr5, L(medium)
5de29b
 /* Align to 32-byte boundary.  */
5de29b
 	andi.	rALIGN, rMEMP, 0x18
5de29b
 	subfic	rALIGN, rALIGN, 0x20
5de29b
-	insrdi	rCHR,rCHR,32,0 /* Replicate word to double word. */
5de29b
+	insrdi	rCHR, rCHR, 32, 0	/* Replicate word to double word. */
5de29b
 	beq	L(caligned)
5de29b
 	mtcrf	0x01, rALIGN
5de29b
 	add	rMEMP, rMEMP, rALIGN
5de29b
@@ -164,24 +164,24 @@
5de29b
 L(getCacheAligned):
5de29b
 	cmpldi	cr1,rLEN,32
5de29b
 	andi.	rTMP,rMEMP,127
5de29b
-	blt		cr1,L(handletail32)
5de29b
-	beq		L(cacheAligned)
5de29b
+	blt	cr1,L(handletail32)
5de29b
+	beq	L(cacheAligned)
5de29b
 	addi	rMEMP,rMEMP,32
5de29b
 	addi	rLEN,rLEN,-32
5de29b
-	std		rCHR,-32(rMEMP)
5de29b
-	std		rCHR,-24(rMEMP)
5de29b
-	std		rCHR,-16(rMEMP)
5de29b
-	std		rCHR,-8(rMEMP)
5de29b
-	b		L(getCacheAligned)
5de29b
+	std	rCHR,-32(rMEMP)
5de29b
+	std	rCHR,-24(rMEMP)
5de29b
+	std	rCHR,-16(rMEMP)
5de29b
+	std	rCHR,-8(rMEMP)
5de29b
+	b	L(getCacheAligned)
5de29b
 
5de29b
 /* Now we are aligned to the cache line and can use dcbz.  */
5de29b
 L(cacheAligned):
5de29b
 	cmpld	cr1,rLEN,rCLS
5de29b
-	blt		cr1,L(handletail32)
5de29b
+	blt	cr1,L(handletail32)
5de29b
 	dcbz	0,rMEMP
5de29b
 	subf	rLEN,rCLS,rLEN
5de29b
-	add		rMEMP,rMEMP,rCLS
5de29b
-	b		L(cacheAligned)
5de29b
+	add	rMEMP,rMEMP,rCLS
5de29b
+	b	L(cacheAligned)
5de29b
 
5de29b
 /* We are here because the cache line size was set and was not 32-bytes
5de29b
    and the remainder (rLEN) is less than the actual cache line size.
5de29b
@@ -218,7 +218,7 @@
5de29b
 /* Memset of 0-31 bytes.  */
5de29b
 	.align 5
5de29b
 L(medium):
5de29b
-	insrdi	rCHR,rCHR,32,0 /* Replicate word to double word.  */
5de29b
+	insrdi	rCHR, rCHR, 32, 0	/* Replicate word to double word.  */
5de29b
 	cmpldi	cr1, rLEN, 16
5de29b
 L(medium_tail2):
5de29b
 	add	rMEMP, rMEMP, rLEN
12745e
diff -urN glibc-2.17-c758a686/sysdeps/powerpc/powerpc64/power6/memset.S glibc-2.17-c758a686/sysdeps/powerpc/powerpc64/power6/memset.S
12745e
--- glibc-2.17-c758a686/sysdeps/powerpc/powerpc64/power6/memset.S	2014-05-29 13:07:41.000000000 -0500
12745e
+++ glibc-2.17-c758a686/sysdeps/powerpc/powerpc64/power6/memset.S	2014-05-29 13:07:46.000000000 -0500
5de29b
@@ -65,14 +65,14 @@
5de29b
 
5de29b
 /* Align to doubleword boundary.  */
5de29b
 	cmpldi	cr5, rLEN, 31
5de29b
-	rlwimi	rCHR, rCHR, 8, 16, 23 /* Replicate byte to halfword.  */
5de29b
+	insrdi	rCHR, rCHR, 8, 48	/* Replicate byte to halfword.  */
5de29b
 	beq+	L(aligned2)
5de29b
 	mtcrf	0x01, rMEMP0
5de29b
 	subfic	rALIGN, rALIGN, 8
5de29b
 	cror	28,30,31		/* Detect odd word aligned.  */
5de29b
 	add	rMEMP, rMEMP, rALIGN
5de29b
 	sub	rLEN, rLEN, rALIGN
5de29b
-	rlwimi	rCHR, rCHR, 16, 0, 15 /* Replicate halfword to word.  */
5de29b
+	insrdi	rCHR, rCHR, 16, 32	/* Replicate halfword to word.  */
5de29b
 	bt	29, L(g4)
5de29b
 /* Process the even word of doubleword.  */
5de29b
 	bf+	31, L(g2)
5de29b
@@ -94,14 +94,14 @@
5de29b
 
5de29b
 /* Handle the case of size < 31.  */
5de29b
 L(aligned2):
5de29b
-	rlwimi	rCHR, rCHR, 16, 0, 15 /* Replicate halfword to word.  */
5de29b
+	insrdi	rCHR, rCHR, 16, 32	/* Replicate halfword to word.  */
5de29b
 L(aligned):
5de29b
 	mtcrf	0x01, rLEN
5de29b
 	ble	cr5, L(medium)
5de29b
 /* Align to 32-byte boundary.  */
5de29b
 	andi.	rALIGN, rMEMP, 0x18
5de29b
 	subfic	rALIGN, rALIGN, 0x20
5de29b
-	insrdi	rCHR,rCHR,32,0 /* Replicate word to double word. */
5de29b
+	insrdi	rCHR, rCHR, 32, 0	/* Replicate word to double word. */
5de29b
 	beq	L(caligned)
5de29b
 	mtcrf	0x01, rALIGN
5de29b
 	add	rMEMP, rMEMP, rALIGN
5de29b
@@ -362,7 +362,7 @@
5de29b
 /* Memset of 0-31 bytes.  */
5de29b
 	.align 5
5de29b
 L(medium):
5de29b
-	insrdi	rCHR,rCHR,32,0 /* Replicate word to double word.  */
5de29b
+	insrdi	rCHR, rCHR, 32, 0	/* Replicate word to double word.  */
5de29b
 	cmpldi	cr1, rLEN, 16
5de29b
 L(medium_tail2):
5de29b
 	add	rMEMP, rMEMP, rLEN
12745e
diff -urN glibc-2.17-c758a686/sysdeps/powerpc/powerpc64/power7/memset.S glibc-2.17-c758a686/sysdeps/powerpc/powerpc64/power7/memset.S
12745e
--- glibc-2.17-c758a686/sysdeps/powerpc/powerpc64/power7/memset.S	2014-05-29 13:07:41.000000000 -0500
12745e
+++ glibc-2.17-c758a686/sysdeps/powerpc/powerpc64/power7/memset.S	2014-05-29 13:07:46.000000000 -0500
5de29b
@@ -34,8 +34,8 @@
5de29b
 	mr	10,3
5de29b
 
5de29b
 	/* Replicate byte to word.  */
5de29b
-	rlwimi	4,4,8,16,23
5de29b
-	rlwimi	4,4,16,0,15
5de29b
+	insrdi	4,4,8,48
5de29b
+	insrdi	4,4,16,32
5de29b
 	ble	cr6,L(small)	/* If length <= 8, use short copy code.  */
5de29b
 
5de29b
 	neg	0,3
5de29b
@@ -323,7 +323,7 @@
5de29b
 	clrldi	0,0,62
5de29b
 	beq	L(medium_aligned)
5de29b
 
5de29b
-	/* Force 4-bytes alignment for SRC.  */
5de29b
+	/* Force 4-bytes alignment for DST.  */
5de29b
 	mtocrf	0x01,0
5de29b
 	subf	5,0,5
5de29b
 1:	/* Copy 1 byte.  */