|
 |
5de29b |
# commit 3be87c77d24c4456ccca4034363b6d1814cd0c84
|
|
|
5de29b |
# Author: Alan Modra <amodra@gmail.com>
|
|
|
5de29b |
# Date: Sat Aug 17 18:47:59 2013 +0930
|
|
|
5de29b |
#
|
|
|
5de29b |
# PowerPC LE memset
|
|
|
5de29b |
# http://sourceware.org/ml/libc-alpha/2013-08/msg00104.html
|
|
|
5de29b |
#
|
|
|
5de29b |
# One of the things I noticed when looking at power7 timing is that rlwimi
|
|
|
5de29b |
# is cracked and the two resulting insns have a register dependency.
|
|
|
5de29b |
# That makes it a little slower than the equivalent rldimi.
|
|
|
5de29b |
#
|
|
|
5de29b |
# * sysdeps/powerpc/powerpc64/memset.S: Replace rlwimi with
|
|
|
5de29b |
# insrdi. Formatting.
|
|
|
5de29b |
# * sysdeps/powerpc/powerpc64/power4/memset.S: Likewise.
|
|
|
5de29b |
# * sysdeps/powerpc/powerpc64/power6/memset.S: Likewise.
|
|
|
5de29b |
# * sysdeps/powerpc/powerpc64/power7/memset.S: Likewise.
|
|
|
5de29b |
# * sysdeps/powerpc/powerpc32/power4/memset.S: Likewise.
|
|
|
5de29b |
# * sysdeps/powerpc/powerpc32/power6/memset.S: Likewise.
|
|
|
5de29b |
# * sysdeps/powerpc/powerpc32/power7/memset.S: Likewise.
|
|
|
5de29b |
#
|
|
|
12745e |
diff -urN glibc-2.17-c758a686/sysdeps/powerpc/powerpc32/power4/memset.S glibc-2.17-c758a686/sysdeps/powerpc/powerpc32/power4/memset.S
|
|
|
12745e |
--- glibc-2.17-c758a686/sysdeps/powerpc/powerpc32/power4/memset.S 2014-05-29 13:07:41.000000000 -0500
|
|
|
12745e |
+++ glibc-2.17-c758a686/sysdeps/powerpc/powerpc32/power4/memset.S 2014-05-29 13:07:46.000000000 -0500
|
|
|
5de29b |
@@ -52,7 +52,7 @@
|
|
|
5de29b |
|
|
|
5de29b |
/* Align to word boundary. */
|
|
|
5de29b |
cmplwi cr5, rLEN, 31
|
|
|
5de29b |
- rlwimi rCHR, rCHR, 8, 16, 23 /* Replicate byte to halfword. */
|
|
|
5de29b |
+ insrdi rCHR, rCHR, 8, 48 /* Replicate byte to halfword. */
|
|
|
5de29b |
beq+ L(aligned)
|
|
|
5de29b |
mtcrf 0x01, rMEMP0
|
|
|
5de29b |
subfic rALIGN, rALIGN, 4
|
|
|
5de29b |
@@ -67,7 +67,7 @@
|
|
|
5de29b |
/* Handle the case of size < 31. */
|
|
|
5de29b |
L(aligned):
|
|
|
5de29b |
mtcrf 0x01, rLEN
|
|
|
5de29b |
- rlwimi rCHR, rCHR, 16, 0, 15 /* Replicate halfword to word. */
|
|
|
5de29b |
+ insrdi rCHR, rCHR, 16, 32 /* Replicate halfword to word. */
|
|
|
5de29b |
ble cr5, L(medium)
|
|
|
5de29b |
/* Align to 32-byte boundary. */
|
|
|
5de29b |
andi. rALIGN, rMEMP, 0x1C
|
|
|
12745e |
diff -urN glibc-2.17-c758a686/sysdeps/powerpc/powerpc32/power6/memset.S glibc-2.17-c758a686/sysdeps/powerpc/powerpc32/power6/memset.S
|
|
|
12745e |
--- glibc-2.17-c758a686/sysdeps/powerpc/powerpc32/power6/memset.S 2014-05-29 13:07:41.000000000 -0500
|
|
|
12745e |
+++ glibc-2.17-c758a686/sysdeps/powerpc/powerpc32/power6/memset.S 2014-05-29 13:07:46.000000000 -0500
|
|
|
5de29b |
@@ -50,7 +50,7 @@
|
|
|
5de29b |
ble- cr1, L(small)
|
|
|
5de29b |
/* Align to word boundary. */
|
|
|
5de29b |
cmplwi cr5, rLEN, 31
|
|
|
5de29b |
- rlwimi rCHR, rCHR, 8, 16, 23 /* Replicate byte to halfword. */
|
|
|
5de29b |
+ insrdi rCHR, rCHR, 8, 48 /* Replicate byte to halfword. */
|
|
|
5de29b |
beq+ L(aligned)
|
|
|
5de29b |
mtcrf 0x01, rMEMP0
|
|
|
5de29b |
subfic rALIGN, rALIGN, 4
|
|
|
5de29b |
@@ -66,7 +66,7 @@
|
|
|
5de29b |
/* Handle the case of size < 31. */
|
|
|
5de29b |
L(aligned):
|
|
|
5de29b |
mtcrf 0x01, rLEN
|
|
|
5de29b |
- rlwimi rCHR, rCHR, 16, 0, 15 /* Replicate halfword to word. */
|
|
|
5de29b |
+ insrdi rCHR, rCHR, 16, 32 /* Replicate halfword to word. */
|
|
|
5de29b |
ble cr5, L(medium)
|
|
|
5de29b |
/* Align to 32-byte boundary. */
|
|
|
5de29b |
andi. rALIGN, rMEMP, 0x1C
|
|
|
12745e |
diff -urN glibc-2.17-c758a686/sysdeps/powerpc/powerpc32/power7/memset.S glibc-2.17-c758a686/sysdeps/powerpc/powerpc32/power7/memset.S
|
|
|
12745e |
--- glibc-2.17-c758a686/sysdeps/powerpc/powerpc32/power7/memset.S 2014-05-29 13:07:41.000000000 -0500
|
|
|
12745e |
+++ glibc-2.17-c758a686/sysdeps/powerpc/powerpc32/power7/memset.S 2014-05-29 13:07:46.000000000 -0500
|
|
|
5de29b |
@@ -37,8 +37,8 @@
|
|
|
5de29b |
cfi_offset(31,-8)
|
|
|
5de29b |
|
|
|
5de29b |
/* Replicate byte to word. */
|
|
|
5de29b |
- rlwimi 4,4,8,16,23
|
|
|
5de29b |
- rlwimi 4,4,16,0,15
|
|
|
5de29b |
+ insrdi 4,4,8,48
|
|
|
5de29b |
+ insrdi 4,4,16,32
|
|
|
5de29b |
|
|
|
5de29b |
ble cr6,L(small) /* If length <= 8, use short copy code. */
|
|
|
5de29b |
|
|
|
12745e |
diff -urN glibc-2.17-c758a686/sysdeps/powerpc/powerpc64/memset.S glibc-2.17-c758a686/sysdeps/powerpc/powerpc64/memset.S
|
|
|
12745e |
--- glibc-2.17-c758a686/sysdeps/powerpc/powerpc64/memset.S 2014-05-29 13:07:41.000000000 -0500
|
|
|
12745e |
+++ glibc-2.17-c758a686/sysdeps/powerpc/powerpc64/memset.S 2014-05-29 13:07:46.000000000 -0500
|
|
|
5de29b |
@@ -73,14 +73,14 @@
|
|
|
5de29b |
|
|
|
5de29b |
/* Align to doubleword boundary. */
|
|
|
5de29b |
cmpldi cr5, rLEN, 31
|
|
|
5de29b |
- rlwimi rCHR, rCHR, 8, 16, 23 /* Replicate byte to halfword. */
|
|
|
5de29b |
+ insrdi rCHR, rCHR, 8, 48 /* Replicate byte to halfword. */
|
|
|
5de29b |
beq+ L(aligned2)
|
|
|
5de29b |
mtcrf 0x01, rMEMP0
|
|
|
5de29b |
subfic rALIGN, rALIGN, 8
|
|
|
5de29b |
cror 28,30,31 /* Detect odd word aligned. */
|
|
|
5de29b |
add rMEMP, rMEMP, rALIGN
|
|
|
5de29b |
sub rLEN, rLEN, rALIGN
|
|
|
5de29b |
- rlwimi rCHR, rCHR, 16, 0, 15 /* Replicate halfword to word. */
|
|
|
5de29b |
+ insrdi rCHR, rCHR, 16, 32 /* Replicate halfword to word. */
|
|
|
5de29b |
bt 29, L(g4)
|
|
|
5de29b |
/* Process the even word of doubleword. */
|
|
|
5de29b |
bf+ 31, L(g2)
|
|
|
5de29b |
@@ -102,14 +102,14 @@
|
|
|
5de29b |
|
|
|
5de29b |
/* Handle the case of size < 31. */
|
|
|
5de29b |
L(aligned2):
|
|
|
5de29b |
- rlwimi rCHR, rCHR, 16, 0, 15 /* Replicate halfword to word. */
|
|
|
5de29b |
+ insrdi rCHR, rCHR, 16, 32 /* Replicate halfword to word. */
|
|
|
5de29b |
L(aligned):
|
|
|
5de29b |
mtcrf 0x01, rLEN
|
|
|
5de29b |
ble cr5, L(medium)
|
|
|
5de29b |
/* Align to 32-byte boundary. */
|
|
|
5de29b |
andi. rALIGN, rMEMP, 0x18
|
|
|
5de29b |
subfic rALIGN, rALIGN, 0x20
|
|
|
5de29b |
- insrdi rCHR,rCHR,32,0 /* Replicate word to double word. */
|
|
|
5de29b |
+ insrdi rCHR, rCHR, 32, 0 /* Replicate word to double word. */
|
|
|
5de29b |
beq L(caligned)
|
|
|
5de29b |
mtcrf 0x01, rALIGN
|
|
|
5de29b |
add rMEMP, rMEMP, rALIGN
|
|
|
5de29b |
@@ -230,7 +230,7 @@
|
|
|
5de29b |
/* Memset of 0-31 bytes. */
|
|
|
5de29b |
.align 5
|
|
|
5de29b |
L(medium):
|
|
|
5de29b |
- insrdi rCHR,rCHR,32,0 /* Replicate word to double word. */
|
|
|
5de29b |
+ insrdi rCHR, rCHR, 32, 0 /* Replicate word to double word. */
|
|
|
5de29b |
cmpldi cr1, rLEN, 16
|
|
|
5de29b |
L(medium_tail2):
|
|
|
5de29b |
add rMEMP, rMEMP, rLEN
|
|
|
12745e |
diff -urN glibc-2.17-c758a686/sysdeps/powerpc/powerpc64/power4/memset.S glibc-2.17-c758a686/sysdeps/powerpc/powerpc64/power4/memset.S
|
|
|
12745e |
--- glibc-2.17-c758a686/sysdeps/powerpc/powerpc64/power4/memset.S 2014-05-29 13:07:41.000000000 -0500
|
|
|
12745e |
+++ glibc-2.17-c758a686/sysdeps/powerpc/powerpc64/power4/memset.S 2014-05-29 13:07:46.000000000 -0500
|
|
|
5de29b |
@@ -68,14 +68,14 @@
|
|
|
5de29b |
|
|
|
5de29b |
/* Align to doubleword boundary. */
|
|
|
5de29b |
cmpldi cr5, rLEN, 31
|
|
|
5de29b |
- rlwimi rCHR, rCHR, 8, 16, 23 /* Replicate byte to halfword. */
|
|
|
5de29b |
+ insrdi rCHR, rCHR, 8, 48 /* Replicate byte to halfword. */
|
|
|
5de29b |
beq+ L(aligned2)
|
|
|
5de29b |
mtcrf 0x01, rMEMP0
|
|
|
5de29b |
subfic rALIGN, rALIGN, 8
|
|
|
5de29b |
cror 28,30,31 /* Detect odd word aligned. */
|
|
|
5de29b |
add rMEMP, rMEMP, rALIGN
|
|
|
5de29b |
sub rLEN, rLEN, rALIGN
|
|
|
5de29b |
- rlwimi rCHR, rCHR, 16, 0, 15 /* Replicate halfword to word. */
|
|
|
5de29b |
+ insrdi rCHR, rCHR, 16, 32 /* Replicate halfword to word. */
|
|
|
5de29b |
bt 29, L(g4)
|
|
|
5de29b |
/* Process the even word of doubleword. */
|
|
|
5de29b |
bf+ 31, L(g2)
|
|
|
5de29b |
@@ -97,14 +97,14 @@
|
|
|
5de29b |
|
|
|
5de29b |
/* Handle the case of size < 31. */
|
|
|
5de29b |
L(aligned2):
|
|
|
5de29b |
- rlwimi rCHR, rCHR, 16, 0, 15 /* Replicate halfword to word. */
|
|
|
5de29b |
+ insrdi rCHR, rCHR, 16, 32 /* Replicate halfword to word. */
|
|
|
5de29b |
L(aligned):
|
|
|
5de29b |
mtcrf 0x01, rLEN
|
|
|
5de29b |
ble cr5, L(medium)
|
|
|
5de29b |
/* Align to 32-byte boundary. */
|
|
|
5de29b |
andi. rALIGN, rMEMP, 0x18
|
|
|
5de29b |
subfic rALIGN, rALIGN, 0x20
|
|
|
5de29b |
- insrdi rCHR,rCHR,32,0 /* Replicate word to double word. */
|
|
|
5de29b |
+ insrdi rCHR, rCHR, 32, 0 /* Replicate word to double word. */
|
|
|
5de29b |
beq L(caligned)
|
|
|
5de29b |
mtcrf 0x01, rALIGN
|
|
|
5de29b |
add rMEMP, rMEMP, rALIGN
|
|
|
5de29b |
@@ -164,24 +164,24 @@
|
|
|
5de29b |
L(getCacheAligned):
|
|
|
5de29b |
cmpldi cr1,rLEN,32
|
|
|
5de29b |
andi. rTMP,rMEMP,127
|
|
|
5de29b |
- blt cr1,L(handletail32)
|
|
|
5de29b |
- beq L(cacheAligned)
|
|
|
5de29b |
+ blt cr1,L(handletail32)
|
|
|
5de29b |
+ beq L(cacheAligned)
|
|
|
5de29b |
addi rMEMP,rMEMP,32
|
|
|
5de29b |
addi rLEN,rLEN,-32
|
|
|
5de29b |
- std rCHR,-32(rMEMP)
|
|
|
5de29b |
- std rCHR,-24(rMEMP)
|
|
|
5de29b |
- std rCHR,-16(rMEMP)
|
|
|
5de29b |
- std rCHR,-8(rMEMP)
|
|
|
5de29b |
- b L(getCacheAligned)
|
|
|
5de29b |
+ std rCHR,-32(rMEMP)
|
|
|
5de29b |
+ std rCHR,-24(rMEMP)
|
|
|
5de29b |
+ std rCHR,-16(rMEMP)
|
|
|
5de29b |
+ std rCHR,-8(rMEMP)
|
|
|
5de29b |
+ b L(getCacheAligned)
|
|
|
5de29b |
|
|
|
5de29b |
/* Now we are aligned to the cache line and can use dcbz. */
|
|
|
5de29b |
L(cacheAligned):
|
|
|
5de29b |
cmpld cr1,rLEN,rCLS
|
|
|
5de29b |
- blt cr1,L(handletail32)
|
|
|
5de29b |
+ blt cr1,L(handletail32)
|
|
|
5de29b |
dcbz 0,rMEMP
|
|
|
5de29b |
subf rLEN,rCLS,rLEN
|
|
|
5de29b |
- add rMEMP,rMEMP,rCLS
|
|
|
5de29b |
- b L(cacheAligned)
|
|
|
5de29b |
+ add rMEMP,rMEMP,rCLS
|
|
|
5de29b |
+ b L(cacheAligned)
|
|
|
5de29b |
|
|
|
5de29b |
/* We are here because the cache line size was set and was not 32-bytes
|
|
|
5de29b |
and the remainder (rLEN) is less than the actual cache line size.
|
|
|
5de29b |
@@ -218,7 +218,7 @@
|
|
|
5de29b |
/* Memset of 0-31 bytes. */
|
|
|
5de29b |
.align 5
|
|
|
5de29b |
L(medium):
|
|
|
5de29b |
- insrdi rCHR,rCHR,32,0 /* Replicate word to double word. */
|
|
|
5de29b |
+ insrdi rCHR, rCHR, 32, 0 /* Replicate word to double word. */
|
|
|
5de29b |
cmpldi cr1, rLEN, 16
|
|
|
5de29b |
L(medium_tail2):
|
|
|
5de29b |
add rMEMP, rMEMP, rLEN
|
|
|
12745e |
diff -urN glibc-2.17-c758a686/sysdeps/powerpc/powerpc64/power6/memset.S glibc-2.17-c758a686/sysdeps/powerpc/powerpc64/power6/memset.S
|
|
|
12745e |
--- glibc-2.17-c758a686/sysdeps/powerpc/powerpc64/power6/memset.S 2014-05-29 13:07:41.000000000 -0500
|
|
|
12745e |
+++ glibc-2.17-c758a686/sysdeps/powerpc/powerpc64/power6/memset.S 2014-05-29 13:07:46.000000000 -0500
|
|
|
5de29b |
@@ -65,14 +65,14 @@
|
|
|
5de29b |
|
|
|
5de29b |
/* Align to doubleword boundary. */
|
|
|
5de29b |
cmpldi cr5, rLEN, 31
|
|
|
5de29b |
- rlwimi rCHR, rCHR, 8, 16, 23 /* Replicate byte to halfword. */
|
|
|
5de29b |
+ insrdi rCHR, rCHR, 8, 48 /* Replicate byte to halfword. */
|
|
|
5de29b |
beq+ L(aligned2)
|
|
|
5de29b |
mtcrf 0x01, rMEMP0
|
|
|
5de29b |
subfic rALIGN, rALIGN, 8
|
|
|
5de29b |
cror 28,30,31 /* Detect odd word aligned. */
|
|
|
5de29b |
add rMEMP, rMEMP, rALIGN
|
|
|
5de29b |
sub rLEN, rLEN, rALIGN
|
|
|
5de29b |
- rlwimi rCHR, rCHR, 16, 0, 15 /* Replicate halfword to word. */
|
|
|
5de29b |
+ insrdi rCHR, rCHR, 16, 32 /* Replicate halfword to word. */
|
|
|
5de29b |
bt 29, L(g4)
|
|
|
5de29b |
/* Process the even word of doubleword. */
|
|
|
5de29b |
bf+ 31, L(g2)
|
|
|
5de29b |
@@ -94,14 +94,14 @@
|
|
|
5de29b |
|
|
|
5de29b |
/* Handle the case of size < 31. */
|
|
|
5de29b |
L(aligned2):
|
|
|
5de29b |
- rlwimi rCHR, rCHR, 16, 0, 15 /* Replicate halfword to word. */
|
|
|
5de29b |
+ insrdi rCHR, rCHR, 16, 32 /* Replicate halfword to word. */
|
|
|
5de29b |
L(aligned):
|
|
|
5de29b |
mtcrf 0x01, rLEN
|
|
|
5de29b |
ble cr5, L(medium)
|
|
|
5de29b |
/* Align to 32-byte boundary. */
|
|
|
5de29b |
andi. rALIGN, rMEMP, 0x18
|
|
|
5de29b |
subfic rALIGN, rALIGN, 0x20
|
|
|
5de29b |
- insrdi rCHR,rCHR,32,0 /* Replicate word to double word. */
|
|
|
5de29b |
+ insrdi rCHR, rCHR, 32, 0 /* Replicate word to double word. */
|
|
|
5de29b |
beq L(caligned)
|
|
|
5de29b |
mtcrf 0x01, rALIGN
|
|
|
5de29b |
add rMEMP, rMEMP, rALIGN
|
|
|
5de29b |
@@ -362,7 +362,7 @@
|
|
|
5de29b |
/* Memset of 0-31 bytes. */
|
|
|
5de29b |
.align 5
|
|
|
5de29b |
L(medium):
|
|
|
5de29b |
- insrdi rCHR,rCHR,32,0 /* Replicate word to double word. */
|
|
|
5de29b |
+ insrdi rCHR, rCHR, 32, 0 /* Replicate word to double word. */
|
|
|
5de29b |
cmpldi cr1, rLEN, 16
|
|
|
5de29b |
L(medium_tail2):
|
|
|
5de29b |
add rMEMP, rMEMP, rLEN
|
|
|
12745e |
diff -urN glibc-2.17-c758a686/sysdeps/powerpc/powerpc64/power7/memset.S glibc-2.17-c758a686/sysdeps/powerpc/powerpc64/power7/memset.S
|
|
|
12745e |
--- glibc-2.17-c758a686/sysdeps/powerpc/powerpc64/power7/memset.S 2014-05-29 13:07:41.000000000 -0500
|
|
|
12745e |
+++ glibc-2.17-c758a686/sysdeps/powerpc/powerpc64/power7/memset.S 2014-05-29 13:07:46.000000000 -0500
|
|
|
5de29b |
@@ -34,8 +34,8 @@
|
|
|
5de29b |
mr 10,3
|
|
|
5de29b |
|
|
|
5de29b |
/* Replicate byte to word. */
|
|
|
5de29b |
- rlwimi 4,4,8,16,23
|
|
|
5de29b |
- rlwimi 4,4,16,0,15
|
|
|
5de29b |
+ insrdi 4,4,8,48
|
|
|
5de29b |
+ insrdi 4,4,16,32
|
|
|
5de29b |
ble cr6,L(small) /* If length <= 8, use short copy code. */
|
|
|
5de29b |
|
|
|
5de29b |
neg 0,3
|
|
|
5de29b |
@@ -323,7 +323,7 @@
|
|
|
5de29b |
clrldi 0,0,62
|
|
|
5de29b |
beq L(medium_aligned)
|
|
|
5de29b |
|
|
|
5de29b |
- /* Force 4-bytes alignment for SRC. */
|
|
|
5de29b |
+ /* Force 4-bytes alignment for DST. */
|
|
|
5de29b |
mtocrf 0x01,0
|
|
|
5de29b |
subf 5,0,5
|
|
|
5de29b |
1: /* Copy 1 byte. */
|