|
 |
ce426f |
Backport of
|
|
|
ce426f |
commit ce6615c9c686acd34672a9f4eba9bcf5553496f6
|
|
|
ce426f |
Author: Adhemerval Zanella <azanella@linux.vnet.ibm.com>
|
|
|
ce426f |
Date: Sun Jan 11 19:33:17 2015 -0600
|
|
|
ce426f |
|
|
|
ce426f |
powerpc: Fix POWER7/PPC64 performance regression on LE
|
|
|
ce426f |
|
|
|
ce426f |
This patch fixes a performance regression on the POWER7/PPC64 memcmp
|
|
|
ce426f |
porting for Little Endian. The LE code uses 'ldbrx' instruction to read
|
|
|
ce426f |
the memory on byte reversed form, however ISA 2.06 just provide the indexed
|
|
|
ce426f |
form which uses a register value as additional index, instead of a fixed value
|
|
|
ce426f |
enconded in the instruction.
|
|
|
ce426f |
|
|
|
ce426f |
And the port strategy for LE uses r0 index value and update the address
|
|
|
ce426f |
value on each compare loop interation. For large compare size values,
|
|
|
ce426f |
it adds 8 more instructions plus some more depending of trailing
|
|
|
ce426f |
size. This patch fixes it by adding pre-calculate indexes to remove the
|
|
|
ce426f |
address update on loops and tailing sizes.
|
|
|
ce426f |
|
|
|
ce426f |
For large sizes it shows a considerable gain, with double performance
|
|
|
ce426f |
pairing with BE.
|
|
|
ce426f |
|
|
|
ce426f |
ChangeLog:
|
|
|
ce426f |
2015-01-13 Adhemerval Zanella <azanella@linux.vnet.ibm.com>
|
|
|
ce426f |
|
|
|
ce426f |
* sysdeps/powerpc/powerpc64/power7/memcmp.S (memcmp): Fix performance
|
|
|
ce426f |
regression on LE.
|
|
|
ce426f |
|
|
|
ce426f |
diff --git a/sysdeps/powerpc/powerpc64/power7/memcmp.S b/sysdeps/powerpc/powerpc64/power7/memcmp.S
|
|
|
ce426f |
index 09bff69..98b9e54 100644
|
|
|
ce426f |
--- a/sysdeps/powerpc/powerpc64/power7/memcmp.S
|
|
|
ce426f |
+++ b/sysdeps/powerpc/powerpc64/power7/memcmp.S
|
|
|
ce426f |
@@ -26,18 +26,48 @@
|
|
|
ce426f |
EALIGN (memcmp, 4, 0)
|
|
|
ce426f |
CALL_MCOUNT 3
|
|
|
ce426f |
|
|
|
ce426f |
-#define rRTN r3
|
|
|
ce426f |
-#define rSTR1 r3 /* first string arg */
|
|
|
ce426f |
-#define rSTR2 r4 /* second string arg */
|
|
|
ce426f |
-#define rN r5 /* max string length */
|
|
|
ce426f |
-#define rWORD1 r6 /* current word in s1 */
|
|
|
ce426f |
-#define rWORD2 r7 /* current word in s2 */
|
|
|
ce426f |
-#define rWORD3 r8 /* next word in s1 */
|
|
|
ce426f |
-#define rWORD4 r9 /* next word in s2 */
|
|
|
ce426f |
-#define rWORD5 r10 /* next word in s1 */
|
|
|
ce426f |
-#define rWORD6 r11 /* next word in s2 */
|
|
|
ce426f |
-#define rWORD7 r30 /* next word in s1 */
|
|
|
ce426f |
-#define rWORD8 r31 /* next word in s2 */
|
|
|
ce426f |
+#define rRTN r3
|
|
|
ce426f |
+#define rSTR1 r3 /* first string arg */
|
|
|
ce426f |
+#define rSTR2 r4 /* second string arg */
|
|
|
ce426f |
+#define rN r5 /* max string length */
|
|
|
ce426f |
+#define rWORD1 r6 /* current word in s1 */
|
|
|
ce426f |
+#define rWORD2 r7 /* current word in s2 */
|
|
|
ce426f |
+#define rWORD3 r8 /* next word in s1 */
|
|
|
ce426f |
+#define rWORD4 r9 /* next word in s2 */
|
|
|
ce426f |
+#define rWORD5 r10 /* next word in s1 */
|
|
|
ce426f |
+#define rWORD6 r11 /* next word in s2 */
|
|
|
ce426f |
+
|
|
|
ce426f |
+#define rOFF8 r20 /* 8 bytes offset. */
|
|
|
ce426f |
+#define rOFF16 r21 /* 16 bytes offset. */
|
|
|
ce426f |
+#define rOFF24 r22 /* 24 bytes offset. */
|
|
|
ce426f |
+#define rOFF32 r23 /* 24 bytes offset. */
|
|
|
ce426f |
+#define rWORD6_SHIFT r24 /* Left rotation temp for rWORD8. */
|
|
|
ce426f |
+#define rWORD4_SHIFT r25 /* Left rotation temp for rWORD6. */
|
|
|
ce426f |
+#define rWORD2_SHIFT r26 /* Left rotation temp for rWORD4. */
|
|
|
ce426f |
+#define rWORD8_SHIFT r27 /* Left rotation temp for rWORD2. */
|
|
|
ce426f |
+#define rSHR r28 /* Unaligned shift right count. */
|
|
|
ce426f |
+#define rSHL r29 /* Unaligned shift left count. */
|
|
|
ce426f |
+#define rWORD7 r30 /* next word in s1 */
|
|
|
ce426f |
+#define rWORD8 r31 /* next word in s2 */
|
|
|
ce426f |
+
|
|
|
ce426f |
+#define rWORD8SAVE (-8)
|
|
|
ce426f |
+#define rWORD7SAVE (-16)
|
|
|
ce426f |
+#define rOFF8SAVE (-24)
|
|
|
ce426f |
+#define rOFF16SAVE (-32)
|
|
|
ce426f |
+#define rOFF24SAVE (-40)
|
|
|
ce426f |
+#define rOFF32SAVE (-48)
|
|
|
ce426f |
+#define rSHRSAVE (-56)
|
|
|
ce426f |
+#define rSHLSAVE (-64)
|
|
|
ce426f |
+#define rWORD8SHIFTSAVE (-72)
|
|
|
ce426f |
+#define rWORD2SHIFTSAVE (-80)
|
|
|
ce426f |
+#define rWORD4SHIFTSAVE (-88)
|
|
|
ce426f |
+#define rWORD6SHIFTSAVE (-96)
|
|
|
ce426f |
+
|
|
|
ce426f |
+#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
+# define LD ldbrx
|
|
|
ce426f |
+#else
|
|
|
ce426f |
+# define LD ldx
|
|
|
ce426f |
+#endif
|
|
|
ce426f |
|
|
|
ce426f |
xor r0, rSTR2, rSTR1
|
|
|
ce426f |
cmpldi cr6, rN, 0
|
|
|
ce426f |
@@ -51,10 +81,24 @@ EALIGN (memcmp, 4, 0)
|
|
|
ce426f |
/* If less than 8 bytes or not aligned, use the unaligned
|
|
|
ce426f |
byte loop. */
|
|
|
ce426f |
blt cr1, L(bytealigned)
|
|
|
ce426f |
- std rWORD8, -8(r1)
|
|
|
ce426f |
- cfi_offset(rWORD8, -8)
|
|
|
ce426f |
- std rWORD7, -16(r1)
|
|
|
ce426f |
- cfi_offset(rWORD7, -16)
|
|
|
ce426f |
+ std rWORD8, rWORD8SAVE(r1)
|
|
|
ce426f |
+ cfi_offset(rWORD8, rWORD8SAVE)
|
|
|
ce426f |
+ std rWORD7, rWORD7SAVE(r1)
|
|
|
ce426f |
+ cfi_offset(rWORD7, rWORD7SAVE)
|
|
|
ce426f |
+ std rOFF8, rOFF8SAVE(r1)
|
|
|
ce426f |
+ cfi_offset(rWORD7, rOFF8SAVE)
|
|
|
ce426f |
+ std rOFF16, rOFF16SAVE(r1)
|
|
|
ce426f |
+ cfi_offset(rWORD7, rOFF16SAVE)
|
|
|
ce426f |
+ std rOFF24, rOFF24SAVE(r1)
|
|
|
ce426f |
+ cfi_offset(rWORD7, rOFF24SAVE)
|
|
|
ce426f |
+ std rOFF32, rOFF32SAVE(r1)
|
|
|
ce426f |
+ cfi_offset(rWORD7, rOFF32SAVE)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ li rOFF8,8
|
|
|
ce426f |
+ li rOFF16,16
|
|
|
ce426f |
+ li rOFF24,24
|
|
|
ce426f |
+ li rOFF32,32
|
|
|
ce426f |
+
|
|
|
ce426f |
bne L(unaligned)
|
|
|
ce426f |
/* At this point we know both strings have the same alignment and the
|
|
|
ce426f |
compare length is at least 8 bytes. r12 contains the low order
|
|
|
ce426f |
@@ -79,15 +123,8 @@ L(samealignment):
|
|
|
ce426f |
sldi rWORD6, r12, 3
|
|
|
ce426f |
srdi r0, rN, 5 /* Divide by 32 */
|
|
|
ce426f |
andi. r12, rN, 24 /* Get the DW remainder */
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD1, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD2, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD1, 0(rSTR1)
|
|
|
ce426f |
- ld rWORD2, 0(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD1, 0, rSTR1
|
|
|
ce426f |
+ LD rWORD2, 0, rSTR2
|
|
|
ce426f |
cmpldi cr1, r12, 16
|
|
|
ce426f |
cmpldi cr7, rN, 32
|
|
|
ce426f |
clrldi rN, rN, 61
|
|
|
ce426f |
@@ -104,15 +141,8 @@ L(dsP1):
|
|
|
ce426f |
cmpld cr5, rWORD5, rWORD6
|
|
|
ce426f |
blt cr7, L(dP1x)
|
|
|
ce426f |
/* Do something useful in this cycle since we have to branch anyway. */
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD1, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD2, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD1, 8(rSTR1)
|
|
|
ce426f |
- ld rWORD2, 8(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD1, rOFF8, rSTR1
|
|
|
ce426f |
+ LD rWORD2, rOFF8, rSTR2
|
|
|
ce426f |
cmpld cr7, rWORD1, rWORD2
|
|
|
ce426f |
b L(dP1e)
|
|
|
ce426f |
/* Remainder is 16 */
|
|
|
ce426f |
@@ -123,15 +153,8 @@ L(dPs2):
|
|
|
ce426f |
cmpld cr6, rWORD5, rWORD6
|
|
|
ce426f |
blt cr7, L(dP2x)
|
|
|
ce426f |
/* Do something useful in this cycle since we have to branch anyway. */
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD7, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD8, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD7, 8(rSTR1)
|
|
|
ce426f |
- ld rWORD8, 8(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD7, rOFF8, rSTR1
|
|
|
ce426f |
+ LD rWORD8, rOFF8, rSTR2
|
|
|
ce426f |
cmpld cr5, rWORD7, rWORD8
|
|
|
ce426f |
b L(dP2e)
|
|
|
ce426f |
/* Remainder is 24 */
|
|
|
ce426f |
@@ -173,72 +196,43 @@ L(dP1):
|
|
|
ce426f |
change any on the early exit path. The key here is the non-early
|
|
|
ce426f |
exit path only cares about the condition code (cr5), not about which
|
|
|
ce426f |
register pair was used. */
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD5, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD6, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD5, 0(rSTR1)
|
|
|
ce426f |
- ld rWORD6, 0(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD5, 0, rSTR1
|
|
|
ce426f |
+ LD rWORD6, 0, rSTR2
|
|
|
ce426f |
cmpld cr5, rWORD5, rWORD6
|
|
|
ce426f |
blt cr7, L(dP1x)
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD1, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD2, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD1, 8(rSTR1)
|
|
|
ce426f |
- ld rWORD2, 8(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD1, rOFF8, rSTR1
|
|
|
ce426f |
+ LD rWORD2, rOFF8, rSTR2
|
|
|
ce426f |
cmpld cr7, rWORD1, rWORD2
|
|
|
ce426f |
L(dP1e):
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD3, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD4, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD3, 16(rSTR1)
|
|
|
ce426f |
- ld rWORD4, 16(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD3, rOFF16, rSTR1
|
|
|
ce426f |
+ LD rWORD4, rOFF16, rSTR2
|
|
|
ce426f |
cmpld cr1, rWORD3, rWORD4
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD5, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD6, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD5, 24(rSTR1)
|
|
|
ce426f |
- ld rWORD6, 24(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD5, rOFF24, rSTR1
|
|
|
ce426f |
+ LD rWORD6, rOFF24, rSTR2
|
|
|
ce426f |
cmpld cr6, rWORD5, rWORD6
|
|
|
ce426f |
bne cr5, L(dLcr5x)
|
|
|
ce426f |
bne cr7, L(dLcr7x)
|
|
|
ce426f |
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD7, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD8, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ldu rWORD7, 32(rSTR1)
|
|
|
ce426f |
- ldu rWORD8, 32(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD7, rOFF32, rSTR1
|
|
|
ce426f |
+ LD rWORD8, rOFF32, rSTR2
|
|
|
ce426f |
+ addi rSTR1, rSTR1, 32
|
|
|
ce426f |
+ addi rSTR2, rSTR2, 32
|
|
|
ce426f |
bne cr1, L(dLcr1)
|
|
|
ce426f |
cmpld cr5, rWORD7, rWORD8
|
|
|
ce426f |
bdnz L(dLoop)
|
|
|
ce426f |
bne cr6, L(dLcr6)
|
|
|
ce426f |
- ld rWORD8, -8(r1)
|
|
|
ce426f |
- ld rWORD7, -16(r1)
|
|
|
ce426f |
+ ld rWORD8, rWORD8SAVE(r1)
|
|
|
ce426f |
+ ld rWORD7, rWORD7SAVE(r1)
|
|
|
ce426f |
.align 3
|
|
|
ce426f |
L(dP1x):
|
|
|
ce426f |
sldi. r12, rN, 3
|
|
|
ce426f |
bne cr5, L(dLcr5x)
|
|
|
ce426f |
subfic rN, r12, 64 /* Shift count is 64 - (rN * 8). */
|
|
|
ce426f |
bne L(d00)
|
|
|
ce426f |
+ ld rOFF8, rOFF8SAVE(r1)
|
|
|
ce426f |
+ ld rOFF16, rOFF16SAVE(r1)
|
|
|
ce426f |
+ ld rOFF24, rOFF24SAVE(r1)
|
|
|
ce426f |
+ ld rOFF32, rOFF32SAVE(r1)
|
|
|
ce426f |
li rRTN, 0
|
|
|
ce426f |
blr
|
|
|
ce426f |
|
|
|
ce426f |
@@ -246,79 +240,41 @@ L(dP1x):
|
|
|
ce426f |
.align 4
|
|
|
ce426f |
L(dP2):
|
|
|
ce426f |
mtctr r0
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD5, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD6, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD5, 0(rSTR1)
|
|
|
ce426f |
- ld rWORD6, 0(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD5, 0, rSTR1
|
|
|
ce426f |
+ LD rWORD6, 0, rSTR2
|
|
|
ce426f |
cmpld cr6, rWORD5, rWORD6
|
|
|
ce426f |
blt cr7, L(dP2x)
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD7, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD8, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD7, 8(rSTR1)
|
|
|
ce426f |
- ld rWORD8, 8(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD7, rOFF8, rSTR1
|
|
|
ce426f |
+ LD rWORD8, rOFF8, rSTR2
|
|
|
ce426f |
cmpld cr5, rWORD7, rWORD8
|
|
|
ce426f |
L(dP2e):
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD1, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD2, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD1, 16(rSTR1)
|
|
|
ce426f |
- ld rWORD2, 16(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD1, rOFF16, rSTR1
|
|
|
ce426f |
+ LD rWORD2, rOFF16, rSTR2
|
|
|
ce426f |
cmpld cr7, rWORD1, rWORD2
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD3, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD4, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD3, 24(rSTR1)
|
|
|
ce426f |
- ld rWORD4, 24(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD3, rOFF24, rSTR1
|
|
|
ce426f |
+ LD rWORD4, rOFF24, rSTR2
|
|
|
ce426f |
cmpld cr1, rWORD3, rWORD4
|
|
|
ce426f |
-#ifndef __LITTLE_ENDIAN__
|
|
|
ce426f |
addi rSTR1, rSTR1, 8
|
|
|
ce426f |
addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
bne cr6, L(dLcr6)
|
|
|
ce426f |
bne cr5, L(dLcr5)
|
|
|
ce426f |
b L(dLoop2)
|
|
|
ce426f |
-/* Again we are on a early exit path (16-23 byte compare), we want to
|
|
|
ce426f |
- only use volatile registers and avoid restoring non-volatile
|
|
|
ce426f |
- registers. */
|
|
|
ce426f |
.align 4
|
|
|
ce426f |
L(dP2x):
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD3, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD4, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD3, 8(rSTR1)
|
|
|
ce426f |
- ld rWORD4, 8(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD3, rOFF8, rSTR1
|
|
|
ce426f |
+ LD rWORD4, rOFF8, rSTR2
|
|
|
ce426f |
cmpld cr1, rWORD3, rWORD4
|
|
|
ce426f |
sldi. r12, rN, 3
|
|
|
ce426f |
bne cr6, L(dLcr6x)
|
|
|
ce426f |
-#ifndef __LITTLE_ENDIAN__
|
|
|
ce426f |
addi rSTR1, rSTR1, 8
|
|
|
ce426f |
addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
bne cr1, L(dLcr1x)
|
|
|
ce426f |
subfic rN, r12, 64 /* Shift count is 64 - (rN * 8). */
|
|
|
ce426f |
bne L(d00)
|
|
|
ce426f |
+ ld rOFF8, rOFF8SAVE(r1)
|
|
|
ce426f |
+ ld rOFF16, rOFF16SAVE(r1)
|
|
|
ce426f |
+ ld rOFF24, rOFF24SAVE(r1)
|
|
|
ce426f |
+ ld rOFF32, rOFF32SAVE(r1)
|
|
|
ce426f |
li rRTN, 0
|
|
|
ce426f |
blr
|
|
|
ce426f |
|
|
|
ce426f |
@@ -326,52 +282,22 @@ L(dP2x):
|
|
|
ce426f |
.align 4
|
|
|
ce426f |
L(dP3):
|
|
|
ce426f |
mtctr r0
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD3, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD4, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD3, 0(rSTR1)
|
|
|
ce426f |
- ld rWORD4, 0(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD3, 0, rSTR1
|
|
|
ce426f |
+ LD rWORD4, 0, rSTR2
|
|
|
ce426f |
cmpld cr1, rWORD3, rWORD4
|
|
|
ce426f |
L(dP3e):
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD5, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD6, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD5, 8(rSTR1)
|
|
|
ce426f |
- ld rWORD6, 8(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD5, rOFF8, rSTR1
|
|
|
ce426f |
+ LD rWORD6, rOFF8, rSTR2
|
|
|
ce426f |
cmpld cr6, rWORD5, rWORD6
|
|
|
ce426f |
blt cr7, L(dP3x)
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD7, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD8, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD7, 16(rSTR1)
|
|
|
ce426f |
- ld rWORD8, 16(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD7, rOFF16, rSTR1
|
|
|
ce426f |
+ LD rWORD8, rOFF16, rSTR2
|
|
|
ce426f |
cmpld cr5, rWORD7, rWORD8
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD1, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD2, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD1, 24(rSTR1)
|
|
|
ce426f |
- ld rWORD2, 24(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD1, rOFF24, rSTR1
|
|
|
ce426f |
+ LD rWORD2, rOFF24, rSTR2
|
|
|
ce426f |
cmpld cr7, rWORD1, rWORD2
|
|
|
ce426f |
-#ifndef __LITTLE_ENDIAN__
|
|
|
ce426f |
addi rSTR1, rSTR1, 16
|
|
|
ce426f |
addi rSTR2, rSTR2, 16
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
bne cr1, L(dLcr1)
|
|
|
ce426f |
bne cr6, L(dLcr6)
|
|
|
ce426f |
b L(dLoop1)
|
|
|
ce426f |
@@ -380,26 +306,21 @@ L(dP3e):
|
|
|
ce426f |
registers. */
|
|
|
ce426f |
.align 4
|
|
|
ce426f |
L(dP3x):
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD1, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD2, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD1, 16(rSTR1)
|
|
|
ce426f |
- ld rWORD2, 16(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD1, rOFF16, rSTR1
|
|
|
ce426f |
+ LD rWORD2, rOFF16, rSTR2
|
|
|
ce426f |
cmpld cr7, rWORD1, rWORD2
|
|
|
ce426f |
sldi. r12, rN, 3
|
|
|
ce426f |
bne cr1, L(dLcr1x)
|
|
|
ce426f |
-#ifndef __LITTLE_ENDIAN__
|
|
|
ce426f |
addi rSTR1, rSTR1, 16
|
|
|
ce426f |
addi rSTR2, rSTR2, 16
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
bne cr6, L(dLcr6x)
|
|
|
ce426f |
subfic rN, r12, 64 /* Shift count is 64 - (rN * 8). */
|
|
|
ce426f |
bne cr7, L(dLcr7x)
|
|
|
ce426f |
bne L(d00)
|
|
|
ce426f |
+ ld rOFF8, rOFF8SAVE(r1)
|
|
|
ce426f |
+ ld rOFF16, rOFF16SAVE(r1)
|
|
|
ce426f |
+ ld rOFF24, rOFF24SAVE(r1)
|
|
|
ce426f |
+ ld rOFF32, rOFF32SAVE(r1)
|
|
|
ce426f |
li rRTN, 0
|
|
|
ce426f |
blr
|
|
|
ce426f |
|
|
|
ce426f |
@@ -407,46 +328,20 @@ L(dP3x):
|
|
|
ce426f |
.align 4
|
|
|
ce426f |
L(dP4):
|
|
|
ce426f |
mtctr r0
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD1, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD2, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD1, 0(rSTR1)
|
|
|
ce426f |
- ld rWORD2, 0(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD1, 0, rSTR1
|
|
|
ce426f |
+ LD rWORD2, 0, rSTR2
|
|
|
ce426f |
cmpld cr7, rWORD1, rWORD2
|
|
|
ce426f |
L(dP4e):
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD3, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD4, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD3, 8(rSTR1)
|
|
|
ce426f |
- ld rWORD4, 8(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD3, rOFF8, rSTR1
|
|
|
ce426f |
+ LD rWORD4, rOFF8, rSTR2
|
|
|
ce426f |
cmpld cr1, rWORD3, rWORD4
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD5, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD6, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD5, 16(rSTR1)
|
|
|
ce426f |
- ld rWORD6, 16(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD5, rOFF16, rSTR1
|
|
|
ce426f |
+ LD rWORD6, rOFF16, rSTR2
|
|
|
ce426f |
cmpld cr6, rWORD5, rWORD6
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD7, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD8, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ldu rWORD7, 24(rSTR1)
|
|
|
ce426f |
- ldu rWORD8, 24(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD7, rOFF24, rSTR1
|
|
|
ce426f |
+ LD rWORD8, rOFF24, rSTR2
|
|
|
ce426f |
+ addi rSTR1, rSTR1, 24
|
|
|
ce426f |
+ addi rSTR2, rSTR2, 24
|
|
|
ce426f |
cmpld cr5, rWORD7, rWORD8
|
|
|
ce426f |
bne cr7, L(dLcr7)
|
|
|
ce426f |
bne cr1, L(dLcr1)
|
|
|
ce426f |
@@ -454,51 +349,25 @@ L(dP4e):
|
|
|
ce426f |
/* This is the primary loop */
|
|
|
ce426f |
.align 4
|
|
|
ce426f |
L(dLoop):
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD1, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD2, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD1, 8(rSTR1)
|
|
|
ce426f |
- ld rWORD2, 8(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD1, rOFF8, rSTR1
|
|
|
ce426f |
+ LD rWORD2, rOFF8, rSTR2
|
|
|
ce426f |
cmpld cr1, rWORD3, rWORD4
|
|
|
ce426f |
bne cr6, L(dLcr6)
|
|
|
ce426f |
L(dLoop1):
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD3, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD4, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD3, 16(rSTR1)
|
|
|
ce426f |
- ld rWORD4, 16(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD3, rOFF16, rSTR1
|
|
|
ce426f |
+ LD rWORD4, rOFF16, rSTR2
|
|
|
ce426f |
cmpld cr6, rWORD5, rWORD6
|
|
|
ce426f |
bne cr5, L(dLcr5)
|
|
|
ce426f |
L(dLoop2):
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD5, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD6, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD5, 24(rSTR1)
|
|
|
ce426f |
- ld rWORD6, 24(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD5, rOFF24, rSTR1
|
|
|
ce426f |
+ LD rWORD6, rOFF24, rSTR2
|
|
|
ce426f |
cmpld cr5, rWORD7, rWORD8
|
|
|
ce426f |
bne cr7, L(dLcr7)
|
|
|
ce426f |
L(dLoop3):
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD7, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD8, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ldu rWORD7, 32(rSTR1)
|
|
|
ce426f |
- ldu rWORD8, 32(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD7, rOFF32, rSTR1
|
|
|
ce426f |
+ LD rWORD8, rOFF32, rSTR2
|
|
|
ce426f |
+ addi rSTR1, rSTR1, 32
|
|
|
ce426f |
+ addi rSTR2, rSTR2, 32
|
|
|
ce426f |
bne cr1, L(dLcr1)
|
|
|
ce426f |
cmpld cr7, rWORD1, rWORD2
|
|
|
ce426f |
bdnz L(dLoop)
|
|
|
ce426f |
@@ -519,62 +388,75 @@ L(d14):
|
|
|
ce426f |
sldi. r12, rN, 3
|
|
|
ce426f |
bne cr5, L(dLcr5)
|
|
|
ce426f |
L(d04):
|
|
|
ce426f |
- ld rWORD8, -8(r1)
|
|
|
ce426f |
- ld rWORD7, -16(r1)
|
|
|
ce426f |
+ ld rWORD8, rWORD8SAVE(r1)
|
|
|
ce426f |
+ ld rWORD7, rWORD7SAVE(r1)
|
|
|
ce426f |
subfic rN, r12, 64 /* Shift count is 64 - (rN * 8). */
|
|
|
ce426f |
- beq L(zeroLength)
|
|
|
ce426f |
+ beq L(duzeroLength)
|
|
|
ce426f |
/* At this point we have a remainder of 1 to 7 bytes to compare. Since
|
|
|
ce426f |
we are aligned it is safe to load the whole double word, and use
|
|
|
ce426f |
shift right double to eliminate bits beyond the compare length. */
|
|
|
ce426f |
L(d00):
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD1, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD2, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD1, 8(rSTR1)
|
|
|
ce426f |
- ld rWORD2, 8(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD1, rOFF8, rSTR1
|
|
|
ce426f |
+ LD rWORD2, rOFF8, rSTR2
|
|
|
ce426f |
srd rWORD1, rWORD1, rN
|
|
|
ce426f |
srd rWORD2, rWORD2, rN
|
|
|
ce426f |
cmpld cr7, rWORD1, rWORD2
|
|
|
ce426f |
bne cr7, L(dLcr7x)
|
|
|
ce426f |
+ ld rOFF8, rOFF8SAVE(r1)
|
|
|
ce426f |
+ ld rOFF16, rOFF16SAVE(r1)
|
|
|
ce426f |
+ ld rOFF24, rOFF24SAVE(r1)
|
|
|
ce426f |
+ ld rOFF32, rOFF32SAVE(r1)
|
|
|
ce426f |
li rRTN, 0
|
|
|
ce426f |
blr
|
|
|
ce426f |
|
|
|
ce426f |
.align 4
|
|
|
ce426f |
L(dLcr7):
|
|
|
ce426f |
- ld rWORD8, -8(r1)
|
|
|
ce426f |
- ld rWORD7, -16(r1)
|
|
|
ce426f |
+ ld rWORD8, rWORD8SAVE(r1)
|
|
|
ce426f |
+ ld rWORD7, rWORD7SAVE(r1)
|
|
|
ce426f |
L(dLcr7x):
|
|
|
ce426f |
+ ld rOFF8, rOFF8SAVE(r1)
|
|
|
ce426f |
+ ld rOFF16, rOFF16SAVE(r1)
|
|
|
ce426f |
+ ld rOFF24, rOFF24SAVE(r1)
|
|
|
ce426f |
+ ld rOFF32, rOFF32SAVE(r1)
|
|
|
ce426f |
li rRTN, 1
|
|
|
ce426f |
bgtlr cr7
|
|
|
ce426f |
li rRTN, -1
|
|
|
ce426f |
blr
|
|
|
ce426f |
.align 4
|
|
|
ce426f |
L(dLcr1):
|
|
|
ce426f |
- ld rWORD8, -8(r1)
|
|
|
ce426f |
- ld rWORD7, -16(r1)
|
|
|
ce426f |
+ ld rWORD8, rWORD8SAVE(r1)
|
|
|
ce426f |
+ ld rWORD7, rWORD7SAVE(r1)
|
|
|
ce426f |
L(dLcr1x):
|
|
|
ce426f |
+ ld rOFF8, rOFF8SAVE(r1)
|
|
|
ce426f |
+ ld rOFF16, rOFF16SAVE(r1)
|
|
|
ce426f |
+ ld rOFF24, rOFF24SAVE(r1)
|
|
|
ce426f |
+ ld rOFF32, rOFF32SAVE(r1)
|
|
|
ce426f |
li rRTN, 1
|
|
|
ce426f |
bgtlr cr1
|
|
|
ce426f |
li rRTN, -1
|
|
|
ce426f |
blr
|
|
|
ce426f |
.align 4
|
|
|
ce426f |
L(dLcr6):
|
|
|
ce426f |
- ld rWORD8, -8(r1)
|
|
|
ce426f |
- ld rWORD7, -16(r1)
|
|
|
ce426f |
+ ld rWORD8, rWORD8SAVE(r1)
|
|
|
ce426f |
+ ld rWORD7, rWORD7SAVE(r1)
|
|
|
ce426f |
L(dLcr6x):
|
|
|
ce426f |
+ ld rOFF8, rOFF8SAVE(r1)
|
|
|
ce426f |
+ ld rOFF16, rOFF16SAVE(r1)
|
|
|
ce426f |
+ ld rOFF24, rOFF24SAVE(r1)
|
|
|
ce426f |
+ ld rOFF32, rOFF32SAVE(r1)
|
|
|
ce426f |
li rRTN, 1
|
|
|
ce426f |
bgtlr cr6
|
|
|
ce426f |
li rRTN, -1
|
|
|
ce426f |
blr
|
|
|
ce426f |
.align 4
|
|
|
ce426f |
L(dLcr5):
|
|
|
ce426f |
- ld rWORD8, -8(r1)
|
|
|
ce426f |
- ld rWORD7, -16(r1)
|
|
|
ce426f |
+ ld rWORD8, rWORD8SAVE(r1)
|
|
|
ce426f |
+ ld rWORD7, rWORD7SAVE(r1)
|
|
|
ce426f |
L(dLcr5x):
|
|
|
ce426f |
+ ld rOFF8, rOFF8SAVE(r1)
|
|
|
ce426f |
+ ld rOFF16, rOFF16SAVE(r1)
|
|
|
ce426f |
+ ld rOFF24, rOFF24SAVE(r1)
|
|
|
ce426f |
+ ld rOFF32, rOFF32SAVE(r1)
|
|
|
ce426f |
li rRTN, 1
|
|
|
ce426f |
bgtlr cr5
|
|
|
ce426f |
li rRTN, -1
|
|
|
ce426f |
@@ -583,10 +465,6 @@ L(dLcr5x):
|
|
|
ce426f |
.align 4
|
|
|
ce426f |
L(bytealigned):
|
|
|
ce426f |
mtctr rN
|
|
|
ce426f |
-#if 0
|
|
|
ce426f |
-/* Huh? We've already branched on cr6! */
|
|
|
ce426f |
- beq cr6, L(zeroLength)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
|
|
|
ce426f |
/* We need to prime this loop. This loop is swing modulo scheduled
|
|
|
ce426f |
to avoid pipe delays. The dependent instruction latencies (load to
|
|
|
ce426f |
@@ -685,6 +563,7 @@ L(b11):
|
|
|
ce426f |
L(bx12):
|
|
|
ce426f |
sub rRTN, rWORD1, rWORD2
|
|
|
ce426f |
blr
|
|
|
ce426f |
+
|
|
|
ce426f |
.align 4
|
|
|
ce426f |
L(zeroLength):
|
|
|
ce426f |
li rRTN, 0
|
|
|
ce426f |
@@ -705,42 +584,36 @@ L(zeroLength):
|
|
|
ce426f |
we need to adjust the length (rN) and special case the loop
|
|
|
ce426f |
versioning for the first DW. This ensures that the loop count is
|
|
|
ce426f |
correct and the first DW (shifted) is in the expected resister pair. */
|
|
|
ce426f |
-#define rSHL r29 /* Unaligned shift left count. */
|
|
|
ce426f |
-#define rSHR r28 /* Unaligned shift right count. */
|
|
|
ce426f |
-#define rWORD8_SHIFT r27 /* Left rotation temp for rWORD2. */
|
|
|
ce426f |
-#define rWORD2_SHIFT r26 /* Left rotation temp for rWORD4. */
|
|
|
ce426f |
-#define rWORD4_SHIFT r25 /* Left rotation temp for rWORD6. */
|
|
|
ce426f |
-#define rWORD6_SHIFT r24 /* Left rotation temp for rWORD8. */
|
|
|
ce426f |
L(unaligned):
|
|
|
ce426f |
- std rSHL, -24(r1)
|
|
|
ce426f |
- cfi_offset(rSHL, -24)
|
|
|
ce426f |
+ std rSHL, rSHLSAVE(r1)
|
|
|
ce426f |
+ cfi_offset(rSHL, rSHLSAVE)
|
|
|
ce426f |
clrldi rSHL, rSTR2, 61
|
|
|
ce426f |
beq cr6, L(duzeroLength)
|
|
|
ce426f |
- std rSHR, -32(r1)
|
|
|
ce426f |
- cfi_offset(rSHR, -32)
|
|
|
ce426f |
+ std rSHR, rSHRSAVE(r1)
|
|
|
ce426f |
+ cfi_offset(rSHR, rSHRSAVE)
|
|
|
ce426f |
beq cr5, L(DWunaligned)
|
|
|
ce426f |
- std rWORD8_SHIFT, -40(r1)
|
|
|
ce426f |
- cfi_offset(rWORD8_SHIFT, -40)
|
|
|
ce426f |
+ std rWORD8_SHIFT, rWORD8SHIFTSAVE(r1)
|
|
|
ce426f |
+ cfi_offset(rWORD8_SHIFT, rWORD8SHIFTSAVE)
|
|
|
ce426f |
/* Adjust the logical start of rSTR2 to compensate for the extra bits
|
|
|
ce426f |
in the 1st rSTR1 DW. */
|
|
|
ce426f |
sub rWORD8_SHIFT, rSTR2, r12
|
|
|
ce426f |
/* But do not attempt to address the DW before that DW that contains
|
|
|
ce426f |
the actual start of rSTR2. */
|
|
|
ce426f |
clrrdi rSTR2, rSTR2, 3
|
|
|
ce426f |
- std rWORD2_SHIFT, -48(r1)
|
|
|
ce426f |
- cfi_offset(rWORD2_SHIFT, -48)
|
|
|
ce426f |
+ std rWORD2_SHIFT, rWORD2SHIFTSAVE(r1)
|
|
|
ce426f |
+ cfi_offset(rWORD2_SHIFT, rWORD2SHIFTSAVE)
|
|
|
ce426f |
/* Compute the left/right shift counts for the unaligned rSTR2,
|
|
|
ce426f |
compensating for the logical (DW aligned) start of rSTR1. */
|
|
|
ce426f |
clrldi rSHL, rWORD8_SHIFT, 61
|
|
|
ce426f |
clrrdi rSTR1, rSTR1, 3
|
|
|
ce426f |
- std rWORD4_SHIFT, -56(r1)
|
|
|
ce426f |
- cfi_offset(rWORD4_SHIFT, -56)
|
|
|
ce426f |
+ std rWORD4_SHIFT, rWORD4SHIFTSAVE(r1)
|
|
|
ce426f |
+ cfi_offset(rWORD4_SHIFT, rWORD4SHIFTSAVE)
|
|
|
ce426f |
sldi rSHL, rSHL, 3
|
|
|
ce426f |
cmpld cr5, rWORD8_SHIFT, rSTR2
|
|
|
ce426f |
add rN, rN, r12
|
|
|
ce426f |
sldi rWORD6, r12, 3
|
|
|
ce426f |
- std rWORD6_SHIFT, -64(r1)
|
|
|
ce426f |
- cfi_offset(rWORD6_SHIFT, -64)
|
|
|
ce426f |
+ std rWORD6_SHIFT, rWORD6SHIFTSAVE(r1)
|
|
|
ce426f |
+ cfi_offset(rWORD6_SHIFT, rWORD6SHIFTSAVE)
|
|
|
ce426f |
subfic rSHR, rSHL, 64
|
|
|
ce426f |
srdi r0, rN, 5 /* Divide by 32 */
|
|
|
ce426f |
andi. r12, rN, 24 /* Get the DW remainder */
|
|
|
ce426f |
@@ -750,25 +623,13 @@ L(unaligned):
|
|
|
ce426f |
this may cross a page boundary and cause a page fault. */
|
|
|
ce426f |
li rWORD8, 0
|
|
|
ce426f |
blt cr5, L(dus0)
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD8, 0, rSTR2
|
|
|
ce426f |
+ LD rWORD8, 0, rSTR2
|
|
|
ce426f |
addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD8, 0(rSTR2)
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
sld rWORD8, rWORD8, rSHL
|
|
|
ce426f |
|
|
|
ce426f |
L(dus0):
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD1, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD2, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD1, 0(rSTR1)
|
|
|
ce426f |
- ld rWORD2, 0(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD1, 0, rSTR1
|
|
|
ce426f |
+ LD rWORD2, 0, rSTR2
|
|
|
ce426f |
cmpldi cr1, r12, 16
|
|
|
ce426f |
cmpldi cr7, rN, 32
|
|
|
ce426f |
srd r12, rWORD2, rSHR
|
|
|
ce426f |
@@ -796,12 +657,7 @@ L(dusP1):
|
|
|
ce426f |
beq L(duZeroReturn)
|
|
|
ce426f |
li r0, 0
|
|
|
ce426f |
ble cr7, L(dutrim)
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD2, 0, rSTR2
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD2, 8(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD2, rOFF8, rSTR2
|
|
|
ce426f |
srd r0, rWORD2, rSHR
|
|
|
ce426f |
b L(dutrim)
|
|
|
ce426f |
/* Remainder is 16 */
|
|
|
ce426f |
@@ -832,27 +688,21 @@ L(duPs4):
|
|
|
ce426f |
compare length is at least 8 bytes. */
|
|
|
ce426f |
.align 4
|
|
|
ce426f |
L(DWunaligned):
|
|
|
ce426f |
- std rWORD8_SHIFT, -40(r1)
|
|
|
ce426f |
- cfi_offset(rWORD8_SHIFT, -40)
|
|
|
ce426f |
+ std rWORD8_SHIFT, rWORD8SHIFTSAVE(r1)
|
|
|
ce426f |
+ cfi_offset(rWORD8_SHIFT, rWORD8SHIFTSAVE)
|
|
|
ce426f |
clrrdi rSTR2, rSTR2, 3
|
|
|
ce426f |
- std rWORD2_SHIFT, -48(r1)
|
|
|
ce426f |
- cfi_offset(rWORD2_SHIFT, -48)
|
|
|
ce426f |
+ std rWORD2_SHIFT, rWORD2SHIFTSAVE(r1)
|
|
|
ce426f |
+ cfi_offset(rWORD2_SHIFT, rWORD2SHIFTSAVE)
|
|
|
ce426f |
srdi r0, rN, 5 /* Divide by 32 */
|
|
|
ce426f |
- std rWORD4_SHIFT, -56(r1)
|
|
|
ce426f |
- cfi_offset(rWORD4_SHIFT, -56)
|
|
|
ce426f |
+ std rWORD4_SHIFT, rWORD4SHIFTSAVE(r1)
|
|
|
ce426f |
+ cfi_offset(rWORD4_SHIFT, rWORD4SHIFTSAVE)
|
|
|
ce426f |
andi. r12, rN, 24 /* Get the DW remainder */
|
|
|
ce426f |
- std rWORD6_SHIFT, -64(r1)
|
|
|
ce426f |
- cfi_offset(rWORD6_SHIFT, -64)
|
|
|
ce426f |
+ std rWORD6_SHIFT, rWORD6SHIFTSAVE(r1)
|
|
|
ce426f |
+ cfi_offset(rWORD6_SHIFT, rWORD6SHIFTSAVE)
|
|
|
ce426f |
sldi rSHL, rSHL, 3
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD6, 0, rSTR2
|
|
|
ce426f |
+ LD rWORD6, 0, rSTR2
|
|
|
ce426f |
+ LD rWORD8, rOFF8, rSTR2
|
|
|
ce426f |
addi rSTR2, rSTR2, 8
|
|
|
ce426f |
- ldbrx rWORD8, 0, rSTR2
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD6, 0(rSTR2)
|
|
|
ce426f |
- ldu rWORD8, 8(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
cmpldi cr1, r12, 16
|
|
|
ce426f |
cmpldi cr7, rN, 32
|
|
|
ce426f |
clrldi rN, rN, 61
|
|
|
ce426f |
@@ -867,52 +717,26 @@ L(DWunaligned):
|
|
|
ce426f |
.align 4
|
|
|
ce426f |
L(duP1):
|
|
|
ce426f |
srd r12, rWORD8, rSHR
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD7, 0, rSTR1
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD7, 0(rSTR1)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD7, 0, rSTR1
|
|
|
ce426f |
sld rWORD8_SHIFT, rWORD8, rSHL
|
|
|
ce426f |
or rWORD8, r12, rWORD6_SHIFT
|
|
|
ce426f |
blt cr7, L(duP1x)
|
|
|
ce426f |
L(duP1e):
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD1, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD2, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD1, 8(rSTR1)
|
|
|
ce426f |
- ld rWORD2, 8(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD1, rOFF8, rSTR1
|
|
|
ce426f |
+ LD rWORD2, rOFF8, rSTR2
|
|
|
ce426f |
cmpld cr5, rWORD7, rWORD8
|
|
|
ce426f |
srd r0, rWORD2, rSHR
|
|
|
ce426f |
sld rWORD2_SHIFT, rWORD2, rSHL
|
|
|
ce426f |
or rWORD2, r0, rWORD8_SHIFT
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD3, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD4, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD3, 16(rSTR1)
|
|
|
ce426f |
- ld rWORD4, 16(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD3, rOFF16, rSTR1
|
|
|
ce426f |
+ LD rWORD4, rOFF16, rSTR2
|
|
|
ce426f |
cmpld cr7, rWORD1, rWORD2
|
|
|
ce426f |
srd r12, rWORD4, rSHR
|
|
|
ce426f |
sld rWORD4_SHIFT, rWORD4, rSHL
|
|
|
ce426f |
bne cr5, L(duLcr5)
|
|
|
ce426f |
or rWORD4, r12, rWORD2_SHIFT
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD5, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD6, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD5, 24(rSTR1)
|
|
|
ce426f |
- ld rWORD6, 24(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD5, rOFF24, rSTR1
|
|
|
ce426f |
+ LD rWORD6, rOFF24, rSTR2
|
|
|
ce426f |
cmpld cr1, rWORD3, rWORD4
|
|
|
ce426f |
srd r0, rWORD6, rSHR
|
|
|
ce426f |
sld rWORD6_SHIFT, rWORD6, rSHL
|
|
|
ce426f |
@@ -932,82 +756,47 @@ L(duP1x):
|
|
|
ce426f |
beq L(duZeroReturn)
|
|
|
ce426f |
li r0, 0
|
|
|
ce426f |
ble cr7, L(dutrim)
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD2, 0, rSTR2
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD2, 8(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD2, rOFF8, rSTR2
|
|
|
ce426f |
srd r0, rWORD2, rSHR
|
|
|
ce426f |
b L(dutrim)
|
|
|
ce426f |
/* Remainder is 16 */
|
|
|
ce426f |
.align 4
|
|
|
ce426f |
L(duP2):
|
|
|
ce426f |
srd r0, rWORD8, rSHR
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD5, 0, rSTR1
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD5, 0(rSTR1)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD5, 0, rSTR1
|
|
|
ce426f |
or rWORD6, r0, rWORD6_SHIFT
|
|
|
ce426f |
sld rWORD6_SHIFT, rWORD8, rSHL
|
|
|
ce426f |
L(duP2e):
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD7, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD8, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD7, 8(rSTR1)
|
|
|
ce426f |
- ld rWORD8, 8(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD7, rOFF8, rSTR1
|
|
|
ce426f |
+ LD rWORD8, rOFF8, rSTR2
|
|
|
ce426f |
cmpld cr6, rWORD5, rWORD6
|
|
|
ce426f |
srd r12, rWORD8, rSHR
|
|
|
ce426f |
sld rWORD8_SHIFT, rWORD8, rSHL
|
|
|
ce426f |
or rWORD8, r12, rWORD6_SHIFT
|
|
|
ce426f |
blt cr7, L(duP2x)
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD1, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD2, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD1, 16(rSTR1)
|
|
|
ce426f |
- ld rWORD2, 16(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD1, rOFF16, rSTR1
|
|
|
ce426f |
+ LD rWORD2, rOFF16, rSTR2
|
|
|
ce426f |
cmpld cr5, rWORD7, rWORD8
|
|
|
ce426f |
bne cr6, L(duLcr6)
|
|
|
ce426f |
srd r0, rWORD2, rSHR
|
|
|
ce426f |
sld rWORD2_SHIFT, rWORD2, rSHL
|
|
|
ce426f |
or rWORD2, r0, rWORD8_SHIFT
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD3, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD4, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD3, 24(rSTR1)
|
|
|
ce426f |
- ld rWORD4, 24(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD3, rOFF24, rSTR1
|
|
|
ce426f |
+ LD rWORD4, rOFF24, rSTR2
|
|
|
ce426f |
cmpld cr7, rWORD1, rWORD2
|
|
|
ce426f |
bne cr5, L(duLcr5)
|
|
|
ce426f |
srd r12, rWORD4, rSHR
|
|
|
ce426f |
sld rWORD4_SHIFT, rWORD4, rSHL
|
|
|
ce426f |
or rWORD4, r12, rWORD2_SHIFT
|
|
|
ce426f |
-#ifndef __LITTLE_ENDIAN__
|
|
|
ce426f |
addi rSTR1, rSTR1, 8
|
|
|
ce426f |
addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
cmpld cr1, rWORD3, rWORD4
|
|
|
ce426f |
b L(duLoop2)
|
|
|
ce426f |
.align 4
|
|
|
ce426f |
L(duP2x):
|
|
|
ce426f |
cmpld cr5, rWORD7, rWORD8
|
|
|
ce426f |
-#ifndef __LITTLE_ENDIAN__
|
|
|
ce426f |
addi rSTR1, rSTR1, 8
|
|
|
ce426f |
addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
bne cr6, L(duLcr6)
|
|
|
ce426f |
sldi. rN, rN, 3
|
|
|
ce426f |
bne cr5, L(duLcr5)
|
|
|
ce426f |
@@ -1015,12 +804,7 @@ L(duP2x):
|
|
|
ce426f |
beq L(duZeroReturn)
|
|
|
ce426f |
li r0, 0
|
|
|
ce426f |
ble cr7, L(dutrim)
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD2, 0, rSTR2
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD2, 8(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD2, rOFF8, rSTR2
|
|
|
ce426f |
srd r0, rWORD2, rSHR
|
|
|
ce426f |
b L(dutrim)
|
|
|
ce426f |
|
|
|
ce426f |
@@ -1028,73 +812,39 @@ L(duP2x):
|
|
|
ce426f |
.align 4
|
|
|
ce426f |
L(duP3):
|
|
|
ce426f |
srd r12, rWORD8, rSHR
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD3, 0, rSTR1
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD3, 0(rSTR1)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD3, 0, rSTR1
|
|
|
ce426f |
sld rWORD4_SHIFT, rWORD8, rSHL
|
|
|
ce426f |
or rWORD4, r12, rWORD6_SHIFT
|
|
|
ce426f |
L(duP3e):
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD5, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD6, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD5, 8(rSTR1)
|
|
|
ce426f |
- ld rWORD6, 8(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD5, rOFF8, rSTR1
|
|
|
ce426f |
+ LD rWORD6, rOFF8, rSTR2
|
|
|
ce426f |
cmpld cr1, rWORD3, rWORD4
|
|
|
ce426f |
srd r0, rWORD6, rSHR
|
|
|
ce426f |
sld rWORD6_SHIFT, rWORD6, rSHL
|
|
|
ce426f |
or rWORD6, r0, rWORD4_SHIFT
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD7, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD8, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD7, 16(rSTR1)
|
|
|
ce426f |
- ld rWORD8, 16(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD7, rOFF16, rSTR1
|
|
|
ce426f |
+ LD rWORD8, rOFF16, rSTR2
|
|
|
ce426f |
cmpld cr6, rWORD5, rWORD6
|
|
|
ce426f |
bne cr1, L(duLcr1)
|
|
|
ce426f |
srd r12, rWORD8, rSHR
|
|
|
ce426f |
sld rWORD8_SHIFT, rWORD8, rSHL
|
|
|
ce426f |
or rWORD8, r12, rWORD6_SHIFT
|
|
|
ce426f |
blt cr7, L(duP3x)
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD1, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD2, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD1, 24(rSTR1)
|
|
|
ce426f |
- ld rWORD2, 24(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD1, rOFF24, rSTR1
|
|
|
ce426f |
+ LD rWORD2, rOFF24, rSTR2
|
|
|
ce426f |
cmpld cr5, rWORD7, rWORD8
|
|
|
ce426f |
bne cr6, L(duLcr6)
|
|
|
ce426f |
srd r0, rWORD2, rSHR
|
|
|
ce426f |
sld rWORD2_SHIFT, rWORD2, rSHL
|
|
|
ce426f |
or rWORD2, r0, rWORD8_SHIFT
|
|
|
ce426f |
-#ifndef __LITTLE_ENDIAN__
|
|
|
ce426f |
addi rSTR1, rSTR1, 16
|
|
|
ce426f |
addi rSTR2, rSTR2, 16
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
cmpld cr7, rWORD1, rWORD2
|
|
|
ce426f |
b L(duLoop1)
|
|
|
ce426f |
.align 4
|
|
|
ce426f |
L(duP3x):
|
|
|
ce426f |
-#ifndef __LITTLE_ENDIAN__
|
|
|
ce426f |
addi rSTR1, rSTR1, 16
|
|
|
ce426f |
addi rSTR2, rSTR2, 16
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
-#if 0
|
|
|
ce426f |
-/* Huh? We've already branched on cr1! */
|
|
|
ce426f |
- bne cr1, L(duLcr1)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
cmpld cr5, rWORD7, rWORD8
|
|
|
ce426f |
bne cr6, L(duLcr6)
|
|
|
ce426f |
sldi. rN, rN, 3
|
|
|
ce426f |
@@ -1103,12 +853,7 @@ L(duP3x):
|
|
|
ce426f |
beq L(duZeroReturn)
|
|
|
ce426f |
li r0, 0
|
|
|
ce426f |
ble cr7, L(dutrim)
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD2, 0, rSTR2
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD2, 8(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD2, rOFF8, rSTR2
|
|
|
ce426f |
srd r0, rWORD2, rSHR
|
|
|
ce426f |
b L(dutrim)
|
|
|
ce426f |
|
|
|
ce426f |
@@ -1117,51 +862,27 @@ L(duP3x):
|
|
|
ce426f |
L(duP4):
|
|
|
ce426f |
mtctr r0
|
|
|
ce426f |
srd r0, rWORD8, rSHR
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD1, 0, rSTR1
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD1, 0(rSTR1)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD1, 0, rSTR1
|
|
|
ce426f |
sld rWORD2_SHIFT, rWORD8, rSHL
|
|
|
ce426f |
or rWORD2, r0, rWORD6_SHIFT
|
|
|
ce426f |
L(duP4e):
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD3, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD4, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD3, 8(rSTR1)
|
|
|
ce426f |
- ld rWORD4, 8(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD3, rOFF8, rSTR1
|
|
|
ce426f |
+ LD rWORD4, rOFF8, rSTR2
|
|
|
ce426f |
cmpld cr7, rWORD1, rWORD2
|
|
|
ce426f |
srd r12, rWORD4, rSHR
|
|
|
ce426f |
sld rWORD4_SHIFT, rWORD4, rSHL
|
|
|
ce426f |
or rWORD4, r12, rWORD2_SHIFT
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD5, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD6, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD5, 16(rSTR1)
|
|
|
ce426f |
- ld rWORD6, 16(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD5, rOFF16, rSTR1
|
|
|
ce426f |
+ LD rWORD6, rOFF16, rSTR2
|
|
|
ce426f |
cmpld cr1, rWORD3, rWORD4
|
|
|
ce426f |
bne cr7, L(duLcr7)
|
|
|
ce426f |
srd r0, rWORD6, rSHR
|
|
|
ce426f |
sld rWORD6_SHIFT, rWORD6, rSHL
|
|
|
ce426f |
or rWORD6, r0, rWORD4_SHIFT
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD7, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD8, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ldu rWORD7, 24(rSTR1)
|
|
|
ce426f |
- ldu rWORD8, 24(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD7, rOFF24, rSTR1
|
|
|
ce426f |
+ LD rWORD8, rOFF24, rSTR2
|
|
|
ce426f |
+ addi rSTR1, rSTR1, 24
|
|
|
ce426f |
+ addi rSTR2, rSTR2, 24
|
|
|
ce426f |
cmpld cr6, rWORD5, rWORD6
|
|
|
ce426f |
bne cr1, L(duLcr1)
|
|
|
ce426f |
srd r12, rWORD8, rSHR
|
|
|
ce426f |
@@ -1172,60 +893,34 @@ L(duP4e):
|
|
|
ce426f |
/* This is the primary loop */
|
|
|
ce426f |
.align 4
|
|
|
ce426f |
L(duLoop):
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD1, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD2, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD1, 8(rSTR1)
|
|
|
ce426f |
- ld rWORD2, 8(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD1, rOFF8, rSTR1
|
|
|
ce426f |
+ LD rWORD2, rOFF8, rSTR2
|
|
|
ce426f |
cmpld cr1, rWORD3, rWORD4
|
|
|
ce426f |
bne cr6, L(duLcr6)
|
|
|
ce426f |
srd r0, rWORD2, rSHR
|
|
|
ce426f |
sld rWORD2_SHIFT, rWORD2, rSHL
|
|
|
ce426f |
or rWORD2, r0, rWORD8_SHIFT
|
|
|
ce426f |
L(duLoop1):
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD3, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD4, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD3, 16(rSTR1)
|
|
|
ce426f |
- ld rWORD4, 16(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD3, rOFF16, rSTR1
|
|
|
ce426f |
+ LD rWORD4, rOFF16, rSTR2
|
|
|
ce426f |
cmpld cr6, rWORD5, rWORD6
|
|
|
ce426f |
bne cr5, L(duLcr5)
|
|
|
ce426f |
srd r12, rWORD4, rSHR
|
|
|
ce426f |
sld rWORD4_SHIFT, rWORD4, rSHL
|
|
|
ce426f |
or rWORD4, r12, rWORD2_SHIFT
|
|
|
ce426f |
L(duLoop2):
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD5, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD6, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD5, 24(rSTR1)
|
|
|
ce426f |
- ld rWORD6, 24(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD5, rOFF24, rSTR1
|
|
|
ce426f |
+ LD rWORD6, rOFF24, rSTR2
|
|
|
ce426f |
cmpld cr5, rWORD7, rWORD8
|
|
|
ce426f |
bne cr7, L(duLcr7)
|
|
|
ce426f |
srd r0, rWORD6, rSHR
|
|
|
ce426f |
sld rWORD6_SHIFT, rWORD6, rSHL
|
|
|
ce426f |
or rWORD6, r0, rWORD4_SHIFT
|
|
|
ce426f |
L(duLoop3):
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD7, 0, rSTR1
|
|
|
ce426f |
- ldbrx rWORD8, 0, rSTR2
|
|
|
ce426f |
- addi rSTR1, rSTR1, 8
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ldu rWORD7, 32(rSTR1)
|
|
|
ce426f |
- ldu rWORD8, 32(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD7, rOFF32, rSTR1
|
|
|
ce426f |
+ LD rWORD8, rOFF32, rSTR2
|
|
|
ce426f |
+ addi rSTR1, rSTR1, 32
|
|
|
ce426f |
+ addi rSTR2, rSTR2, 32
|
|
|
ce426f |
cmpld cr7, rWORD1, rWORD2
|
|
|
ce426f |
bne cr1, L(duLcr1)
|
|
|
ce426f |
srd r12, rWORD8, rSHR
|
|
|
ce426f |
@@ -1234,10 +929,6 @@ L(duLoop3):
|
|
|
ce426f |
bdnz L(duLoop)
|
|
|
ce426f |
|
|
|
ce426f |
L(duL4):
|
|
|
ce426f |
-#if 0
|
|
|
ce426f |
-/* Huh? We've already branched on cr1! */
|
|
|
ce426f |
- bne cr1, L(duLcr1)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
cmpld cr1, rWORD3, rWORD4
|
|
|
ce426f |
bne cr6, L(duLcr6)
|
|
|
ce426f |
cmpld cr6, rWORD5, rWORD6
|
|
|
ce426f |
@@ -1264,99 +955,102 @@ L(du14):
|
|
|
ce426f |
beq L(duZeroReturn)
|
|
|
ce426f |
li r0, 0
|
|
|
ce426f |
ble cr7, L(dutrim)
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD2, 0, rSTR2
|
|
|
ce426f |
- addi rSTR2, rSTR2, 8
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD2, 8(rSTR2)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD2, rOFF8, rSTR2
|
|
|
ce426f |
srd r0, rWORD2, rSHR
|
|
|
ce426f |
.align 4
|
|
|
ce426f |
L(dutrim):
|
|
|
ce426f |
-#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
- ldbrx rWORD1, 0, rSTR1
|
|
|
ce426f |
-#else
|
|
|
ce426f |
- ld rWORD1, 8(rSTR1)
|
|
|
ce426f |
-#endif
|
|
|
ce426f |
+ LD rWORD1, rOFF8, rSTR1
|
|
|
ce426f |
ld rWORD8, -8(r1)
|
|
|
ce426f |
subfic rN, rN, 64 /* Shift count is 64 - (rN * 8). */
|
|
|
ce426f |
or rWORD2, r0, rWORD8_SHIFT
|
|
|
ce426f |
- ld rWORD7, -16(r1)
|
|
|
ce426f |
- ld rSHL, -24(r1)
|
|
|
ce426f |
+ ld rWORD7, rWORD7SAVE(r1)
|
|
|
ce426f |
+ ld rSHL, rSHLSAVE(r1)
|
|
|
ce426f |
srd rWORD1, rWORD1, rN
|
|
|
ce426f |
srd rWORD2, rWORD2, rN
|
|
|
ce426f |
- ld rSHR, -32(r1)
|
|
|
ce426f |
- ld rWORD8_SHIFT, -40(r1)
|
|
|
ce426f |
+ ld rSHR, rSHRSAVE(r1)
|
|
|
ce426f |
+ ld rWORD8_SHIFT, rWORD8SHIFTSAVE(r1)
|
|
|
ce426f |
li rRTN, 0
|
|
|
ce426f |
cmpld cr7, rWORD1, rWORD2
|
|
|
ce426f |
- ld rWORD2_SHIFT, -48(r1)
|
|
|
ce426f |
- ld rWORD4_SHIFT, -56(r1)
|
|
|
ce426f |
+ ld rWORD2_SHIFT, rWORD2SHIFTSAVE(r1)
|
|
|
ce426f |
+ ld rWORD4_SHIFT, rWORD4SHIFTSAVE(r1)
|
|
|
ce426f |
beq cr7, L(dureturn24)
|
|
|
ce426f |
li rRTN, 1
|
|
|
ce426f |
- ld rWORD6_SHIFT, -64(r1)
|
|
|
ce426f |
+ ld rWORD6_SHIFT, rWORD6SHIFTSAVE(r1)
|
|
|
ce426f |
+ ld rOFF8, rOFF8SAVE(r1)
|
|
|
ce426f |
+ ld rOFF16, rOFF16SAVE(r1)
|
|
|
ce426f |
+ ld rOFF24, rOFF24SAVE(r1)
|
|
|
ce426f |
+ ld rOFF32, rOFF32SAVE(r1)
|
|
|
ce426f |
bgtlr cr7
|
|
|
ce426f |
li rRTN, -1
|
|
|
ce426f |
blr
|
|
|
ce426f |
.align 4
|
|
|
ce426f |
L(duLcr7):
|
|
|
ce426f |
- ld rWORD8, -8(r1)
|
|
|
ce426f |
- ld rWORD7, -16(r1)
|
|
|
ce426f |
+ ld rWORD8, rWORD8SAVE(r1)
|
|
|
ce426f |
+ ld rWORD7, rWORD7SAVE(r1)
|
|
|
ce426f |
li rRTN, 1
|
|
|
ce426f |
bgt cr7, L(dureturn29)
|
|
|
ce426f |
- ld rSHL, -24(r1)
|
|
|
ce426f |
- ld rSHR, -32(r1)
|
|
|
ce426f |
+ ld rSHL, rSHLSAVE(r1)
|
|
|
ce426f |
+ ld rSHR, rSHRSAVE(r1)
|
|
|
ce426f |
li rRTN, -1
|
|
|
ce426f |
b L(dureturn27)
|
|
|
ce426f |
.align 4
|
|
|
ce426f |
L(duLcr1):
|
|
|
ce426f |
- ld rWORD8, -8(r1)
|
|
|
ce426f |
- ld rWORD7, -16(r1)
|
|
|
ce426f |
+ ld rWORD8, rWORD8SAVE(r1)
|
|
|
ce426f |
+ ld rWORD7, rWORD7SAVE(r1)
|
|
|
ce426f |
li rRTN, 1
|
|
|
ce426f |
bgt cr1, L(dureturn29)
|
|
|
ce426f |
- ld rSHL, -24(r1)
|
|
|
ce426f |
- ld rSHR, -32(r1)
|
|
|
ce426f |
+ ld rSHL, rSHLSAVE(r1)
|
|
|
ce426f |
+ ld rSHR, rSHRSAVE(r1)
|
|
|
ce426f |
li rRTN, -1
|
|
|
ce426f |
b L(dureturn27)
|
|
|
ce426f |
.align 4
|
|
|
ce426f |
L(duLcr6):
|
|
|
ce426f |
- ld rWORD8, -8(r1)
|
|
|
ce426f |
- ld rWORD7, -16(r1)
|
|
|
ce426f |
+ ld rWORD8, rWORD8SAVE(r1)
|
|
|
ce426f |
+ ld rWORD7, rWORD7SAVE(r1)
|
|
|
ce426f |
li rRTN, 1
|
|
|
ce426f |
bgt cr6, L(dureturn29)
|
|
|
ce426f |
- ld rSHL, -24(r1)
|
|
|
ce426f |
- ld rSHR, -32(r1)
|
|
|
ce426f |
+ ld rSHL, rSHLSAVE(r1)
|
|
|
ce426f |
+ ld rSHR, rSHRSAVE(r1)
|
|
|
ce426f |
li rRTN, -1
|
|
|
ce426f |
b L(dureturn27)
|
|
|
ce426f |
.align 4
|
|
|
ce426f |
L(duLcr5):
|
|
|
ce426f |
- ld rWORD8, -8(r1)
|
|
|
ce426f |
- ld rWORD7, -16(r1)
|
|
|
ce426f |
+ ld rWORD8, rWORD8SAVE(r1)
|
|
|
ce426f |
+ ld rWORD7, rWORD7SAVE(r1)
|
|
|
ce426f |
li rRTN, 1
|
|
|
ce426f |
bgt cr5, L(dureturn29)
|
|
|
ce426f |
- ld rSHL, -24(r1)
|
|
|
ce426f |
- ld rSHR, -32(r1)
|
|
|
ce426f |
+ ld rSHL, rSHLSAVE(r1)
|
|
|
ce426f |
+ ld rSHR, rSHRSAVE(r1)
|
|
|
ce426f |
li rRTN, -1
|
|
|
ce426f |
b L(dureturn27)
|
|
|
ce426f |
+
|
|
|
ce426f |
.align 3
|
|
|
ce426f |
L(duZeroReturn):
|
|
|
ce426f |
li rRTN, 0
|
|
|
ce426f |
.align 4
|
|
|
ce426f |
L(dureturn):
|
|
|
ce426f |
- ld rWORD8, -8(r1)
|
|
|
ce426f |
- ld rWORD7, -16(r1)
|
|
|
ce426f |
+ ld rWORD8, rWORD8SAVE(r1)
|
|
|
ce426f |
+ ld rWORD7, rWORD7SAVE(r1)
|
|
|
ce426f |
L(dureturn29):
|
|
|
ce426f |
- ld rSHL, -24(r1)
|
|
|
ce426f |
- ld rSHR, -32(r1)
|
|
|
ce426f |
+ ld rSHL, rSHLSAVE(r1)
|
|
|
ce426f |
+ ld rSHR, rSHRSAVE(r1)
|
|
|
ce426f |
L(dureturn27):
|
|
|
ce426f |
- ld rWORD8_SHIFT, -40(r1)
|
|
|
ce426f |
-L(dureturn26):
|
|
|
ce426f |
- ld rWORD2_SHIFT, -48(r1)
|
|
|
ce426f |
-L(dureturn25):
|
|
|
ce426f |
- ld rWORD4_SHIFT, -56(r1)
|
|
|
ce426f |
+ ld rWORD8_SHIFT, rWORD8SHIFTSAVE(r1)
|
|
|
ce426f |
+ ld rWORD2_SHIFT, rWORD2SHIFTSAVE(r1)
|
|
|
ce426f |
+ ld rWORD4_SHIFT, rWORD4SHIFTSAVE(r1)
|
|
|
ce426f |
L(dureturn24):
|
|
|
ce426f |
- ld rWORD6_SHIFT, -64(r1)
|
|
|
ce426f |
+ ld rWORD6_SHIFT, rWORD6SHIFTSAVE(r1)
|
|
|
ce426f |
+ ld rOFF8, rOFF8SAVE(r1)
|
|
|
ce426f |
+ ld rOFF16, rOFF16SAVE(r1)
|
|
|
ce426f |
+ ld rOFF24, rOFF24SAVE(r1)
|
|
|
ce426f |
+ ld rOFF32, rOFF32SAVE(r1)
|
|
|
ce426f |
blr
|
|
|
ce426f |
+
|
|
|
ce426f |
L(duzeroLength):
|
|
|
ce426f |
+ ld rOFF8, rOFF8SAVE(r1)
|
|
|
ce426f |
+ ld rOFF16, rOFF16SAVE(r1)
|
|
|
ce426f |
+ ld rOFF24, rOFF24SAVE(r1)
|
|
|
ce426f |
+ ld rOFF32, rOFF32SAVE(r1)
|
|
|
ce426f |
li rRTN, 0
|
|
|
ce426f |
blr
|
|
|
ce426f |
|