|
 |
3b386f |
commit 765de945efc5d5602999b2999fe8abdf04881370
|
|
|
3b386f |
Author: Anton Blanchard <anton@ozlabs.org>
|
|
|
3b386f |
Date: Thu May 14 21:49:16 2020 +1000
|
|
|
3b386f |
|
|
|
3b386f |
powerpc: Optimized rawmemchr for POWER9
|
|
|
3b386f |
|
|
|
3b386f |
This version uses vector instructions and is up to 60% faster on medium
|
|
|
3b386f |
matches and up to 90% faster on long matches, compared to the POWER7
|
|
|
3b386f |
version. A few examples:
|
|
|
3b386f |
|
|
|
3b386f |
__rawmemchr_power9 __rawmemchr_power7
|
|
|
3b386f |
Length 32, alignment 0: 2.27566 3.77765
|
|
|
3b386f |
Length 64, alignment 2: 2.46231 3.51064
|
|
|
3b386f |
Length 1024, alignment 0: 17.3059 32.6678
|
|
|
3b386f |
|
|
|
3b386f |
diff --git a/sysdeps/powerpc/powerpc64/le/power9/rawmemchr.S b/sysdeps/powerpc/powerpc64/le/power9/rawmemchr.S
|
|
|
3b386f |
new file mode 100644
|
|
|
3b386f |
index 0000000000000000..9d0276c9315af5c8
|
|
|
3b386f |
--- /dev/null
|
|
|
3b386f |
+++ b/sysdeps/powerpc/powerpc64/le/power9/rawmemchr.S
|
|
|
3b386f |
@@ -0,0 +1,107 @@
|
|
|
3b386f |
+/* Optimized rawmemchr implementation for PowerPC64/POWER9.
|
|
|
3b386f |
+ Copyright (C) 2020 Free Software Foundation, Inc.
|
|
|
3b386f |
+ This file is part of the GNU C Library.
|
|
|
3b386f |
+
|
|
|
3b386f |
+ The GNU C Library is free software; you can redistribute it and/or
|
|
|
3b386f |
+ modify it under the terms of the GNU Lesser General Public
|
|
|
3b386f |
+ License as published by the Free Software Foundation; either
|
|
|
3b386f |
+ version 2.1 of the License, or (at your option) any later version.
|
|
|
3b386f |
+
|
|
|
3b386f |
+ The GNU C Library is distributed in the hope that it will be useful,
|
|
|
3b386f |
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
3b386f |
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
3b386f |
+ Lesser General Public License for more details.
|
|
|
3b386f |
+
|
|
|
3b386f |
+ You should have received a copy of the GNU Lesser General Public
|
|
|
3b386f |
+ License along with the GNU C Library; if not, see
|
|
|
3b386f |
+ <https://www.gnu.org/licenses/>. */
|
|
|
3b386f |
+
|
|
|
3b386f |
+#include <sysdep.h>
|
|
|
3b386f |
+
|
|
|
3b386f |
+#ifndef RAWMEMCHR
|
|
|
3b386f |
+# define RAWMEMCHR __rawmemchr
|
|
|
3b386f |
+#endif
|
|
|
3b386f |
+
|
|
|
3b386f |
+/* Implements the function
|
|
|
3b386f |
+
|
|
|
3b386f |
+ int [r3] rawmemchr (void *s [r3], int c [r4])
|
|
|
3b386f |
+
|
|
|
3b386f |
+ The implementation can load bytes past a matching byte, but only
|
|
|
3b386f |
+ up to the next 16B boundary, so it never crosses a page. */
|
|
|
3b386f |
+
|
|
|
3b386f |
+.machine power9
|
|
|
3b386f |
+ENTRY_TOCLESS (RAWMEMCHR, 4)
|
|
|
3b386f |
+ CALL_MCOUNT 2
|
|
|
3b386f |
+
|
|
|
3b386f |
+ xori r5,r4,0xff
|
|
|
3b386f |
+
|
|
|
3b386f |
+ mtvsrd v18+32,r4 /* matching char in v18 */
|
|
|
3b386f |
+ mtvsrd v19+32,r5 /* non matching char in v19 */
|
|
|
3b386f |
+
|
|
|
3b386f |
+ vspltb v18,v18,7 /* replicate */
|
|
|
3b386f |
+ vspltb v19,v19,7 /* replicate */
|
|
|
3b386f |
+
|
|
|
3b386f |
+ neg r5,r3
|
|
|
3b386f |
+ rldicl r9,r5,0,60 /* How many bytes to get source 16B aligned? */
|
|
|
3b386f |
+
|
|
|
3b386f |
+ /* Align data and fill bytes not loaded with non matching char */
|
|
|
3b386f |
+ lvx v0,0,r3
|
|
|
3b386f |
+ lvsr v1,0,r3
|
|
|
3b386f |
+ vperm v0,v19,v0,v1
|
|
|
3b386f |
+
|
|
|
3b386f |
+ vcmpequb. v6,v0,v18 /* 0xff if byte matches, 0x00 otherwise */
|
|
|
3b386f |
+ beq cr6,L(aligned)
|
|
|
3b386f |
+
|
|
|
3b386f |
+ vctzlsbb r0,v6
|
|
|
3b386f |
+ add r3,r3,r0
|
|
|
3b386f |
+ blr
|
|
|
3b386f |
+
|
|
|
3b386f |
+L(aligned):
|
|
|
3b386f |
+ add r3,r3,r9
|
|
|
3b386f |
+
|
|
|
3b386f |
+L(loop):
|
|
|
3b386f |
+ lxv v0+32,0(r3)
|
|
|
3b386f |
+ vcmpequb. v6,v0,v18 /* 0xff if byte matches, 0x00 otherwise */
|
|
|
3b386f |
+ bne cr6,L(tail1)
|
|
|
3b386f |
+
|
|
|
3b386f |
+ lxv v0+32,16(r3)
|
|
|
3b386f |
+ vcmpequb. v6,v0,v18 /* 0xff if byte matches, 0x00 otherwise */
|
|
|
3b386f |
+ bne cr6,L(tail2)
|
|
|
3b386f |
+
|
|
|
3b386f |
+ lxv v0+32,32(r3)
|
|
|
3b386f |
+ vcmpequb. v6,v0,v18 /* 0xff if byte matches, 0x00 otherwise */
|
|
|
3b386f |
+ bne cr6,L(tail3)
|
|
|
3b386f |
+
|
|
|
3b386f |
+ lxv v0+32,48(r3)
|
|
|
3b386f |
+ vcmpequb. v6,v0,v18 /* 0xff if byte matches, 0x00 otherwise */
|
|
|
3b386f |
+ bne cr6,L(tail4)
|
|
|
3b386f |
+
|
|
|
3b386f |
+ addi r3,r3,64
|
|
|
3b386f |
+ b L(loop)
|
|
|
3b386f |
+
|
|
|
3b386f |
+L(tail1):
|
|
|
3b386f |
+ vctzlsbb r0,v6
|
|
|
3b386f |
+ add r3,r3,r0
|
|
|
3b386f |
+ blr
|
|
|
3b386f |
+
|
|
|
3b386f |
+L(tail2):
|
|
|
3b386f |
+ vctzlsbb r0,v6
|
|
|
3b386f |
+ add r3,r3,r0
|
|
|
3b386f |
+ addi r3,r3,16
|
|
|
3b386f |
+ blr
|
|
|
3b386f |
+
|
|
|
3b386f |
+L(tail3):
|
|
|
3b386f |
+ vctzlsbb r0,v6
|
|
|
3b386f |
+ add r3,r3,r0
|
|
|
3b386f |
+ addi r3,r3,32
|
|
|
3b386f |
+ blr
|
|
|
3b386f |
+
|
|
|
3b386f |
+L(tail4):
|
|
|
3b386f |
+ vctzlsbb r0,v6
|
|
|
3b386f |
+ add r3,r3,r0
|
|
|
3b386f |
+ addi r3,r3,48
|
|
|
3b386f |
+ blr
|
|
|
3b386f |
+
|
|
|
3b386f |
+END (RAWMEMCHR)
|
|
|
3b386f |
+weak_alias (__rawmemchr,rawmemchr)
|
|
|
3b386f |
+libc_hidden_builtin_def (__rawmemchr)
|
|
|
3b386f |
diff --git a/sysdeps/powerpc/powerpc64/multiarch/Makefile b/sysdeps/powerpc/powerpc64/multiarch/Makefile
|
|
|
3b386f |
index cada6b19bf3c8fab..1a8ef5fb73c3b0db 100644
|
|
|
3b386f |
--- a/sysdeps/powerpc/powerpc64/multiarch/Makefile
|
|
|
3b386f |
+++ b/sysdeps/powerpc/powerpc64/multiarch/Makefile
|
|
|
3b386f |
@@ -32,7 +32,8 @@ sysdep_routines += memcpy-power8-cached memcpy-power7 memcpy-a2 memcpy-power6 \
|
|
|
3b386f |
strncase-power8
|
|
|
3b386f |
|
|
|
3b386f |
ifneq (,$(filter %le,$(config-machine)))
|
|
|
3b386f |
-sysdep_routines += strcmp-power9 strncmp-power9 strcpy-power9 stpcpy-power9
|
|
|
3b386f |
+sysdep_routines += strcmp-power9 strncmp-power9 strcpy-power9 stpcpy-power9 \
|
|
|
3b386f |
+ rawmemchr-power9
|
|
|
3b386f |
endif
|
|
|
3b386f |
CFLAGS-strncase-power7.c += -mcpu=power7 -funroll-loops
|
|
|
3b386f |
CFLAGS-strncase_l-power7.c += -mcpu=power7 -funroll-loops
|
|
|
3b386f |
diff --git a/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c b/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c
|
|
|
3b386f |
index b0abc6b61dc15f19..297935863e44c0e1 100644
|
|
|
3b386f |
--- a/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c
|
|
|
3b386f |
+++ b/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c
|
|
|
3b386f |
@@ -216,6 +216,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
|
|
|
3b386f |
|
|
|
3b386f |
/* Support sysdeps/powerpc/powerpc64/multiarch/rawmemchr.c. */
|
|
|
3b386f |
IFUNC_IMPL (i, name, rawmemchr,
|
|
|
3b386f |
+#ifdef __LITTLE_ENDIAN__
|
|
|
3b386f |
+ IFUNC_IMPL_ADD (array, i, rawmemchr,
|
|
|
3b386f |
+ hwcap2 & PPC_FEATURE2_ARCH_3_00,
|
|
|
3b386f |
+ __rawmemchr_power9)
|
|
|
3b386f |
+#endif
|
|
|
3b386f |
IFUNC_IMPL_ADD (array, i, rawmemchr,
|
|
|
3b386f |
hwcap & PPC_FEATURE_HAS_VSX,
|
|
|
3b386f |
__rawmemchr_power7)
|
|
|
3b386f |
diff --git a/sysdeps/powerpc/powerpc64/multiarch/rawmemchr-power9.S b/sysdeps/powerpc/powerpc64/multiarch/rawmemchr-power9.S
|
|
|
3b386f |
new file mode 100644
|
|
|
3b386f |
index 0000000000000000..bac0a9090e7a07f8
|
|
|
3b386f |
--- /dev/null
|
|
|
3b386f |
+++ b/sysdeps/powerpc/powerpc64/multiarch/rawmemchr-power9.S
|
|
|
3b386f |
@@ -0,0 +1,21 @@
|
|
|
3b386f |
+/* Optimized rawmemchr implementation for PowerPC64/POWER9.
|
|
|
3b386f |
+ Copyright (C) 2020 Free Software Foundation, Inc.
|
|
|
3b386f |
+ This file is part of the GNU C Library.
|
|
|
3b386f |
+
|
|
|
3b386f |
+ The GNU C Library is free software; you can redistribute it and/or
|
|
|
3b386f |
+ modify it under the terms of the GNU Lesser General Public
|
|
|
3b386f |
+ License as published by the Free Software Foundation; either
|
|
|
3b386f |
+ version 2.1 of the License, or (at your option) any later version.
|
|
|
3b386f |
+
|
|
|
3b386f |
+ The GNU C Library is distributed in the hope that it will be useful,
|
|
|
3b386f |
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
3b386f |
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
3b386f |
+ Lesser General Public License for more details.
|
|
|
3b386f |
+
|
|
|
3b386f |
+ You should have received a copy of the GNU Lesser General Public
|
|
|
3b386f |
+ License along with the GNU C Library; if not, see
|
|
|
3b386f |
+ <https://www.gnu.org/licenses/>. */
|
|
|
3b386f |
+
|
|
|
3b386f |
+#define RAWMEMCHR __rawmemchr_power9
|
|
|
3b386f |
+
|
|
|
3b386f |
+#include <sysdeps/powerpc/powerpc64/le/power9/rawmemchr.S>
|
|
|
3b386f |
diff --git a/sysdeps/powerpc/powerpc64/multiarch/rawmemchr.c b/sysdeps/powerpc/powerpc64/multiarch/rawmemchr.c
|
|
|
3b386f |
index 02bac49b53d52411..2a7ae5a1ed02e556 100644
|
|
|
3b386f |
--- a/sysdeps/powerpc/powerpc64/multiarch/rawmemchr.c
|
|
|
3b386f |
+++ b/sysdeps/powerpc/powerpc64/multiarch/rawmemchr.c
|
|
|
3b386f |
@@ -24,13 +24,21 @@
|
|
|
3b386f |
|
|
|
3b386f |
extern __typeof (__rawmemchr) __rawmemchr_ppc attribute_hidden;
|
|
|
3b386f |
extern __typeof (__rawmemchr) __rawmemchr_power7 attribute_hidden;
|
|
|
3b386f |
+# ifdef __LITTLE_ENDIAN__
|
|
|
3b386f |
+extern __typeof (__rawmemchr) __rawmemchr_power9 attribute_hidden;
|
|
|
3b386f |
+# endif
|
|
|
3b386f |
+
|
|
|
3b386f |
# undef __rawmemchr
|
|
|
3b386f |
|
|
|
3b386f |
/* Avoid DWARF definition DIE on ifunc symbol so that GDB can handle
|
|
|
3b386f |
ifunc symbol properly. */
|
|
|
3b386f |
libc_ifunc_redirected (__redirect___rawmemchr, __rawmemchr,
|
|
|
3b386f |
- (hwcap & PPC_FEATURE_HAS_VSX)
|
|
|
3b386f |
- ? __rawmemchr_power7
|
|
|
3b386f |
+# ifdef __LITTLE_ENDIAN__
|
|
|
3b386f |
+ (hwcap2 & PPC_FEATURE2_ARCH_3_00)
|
|
|
3b386f |
+ ? __rawmemchr_power9 :
|
|
|
3b386f |
+# endif
|
|
|
3b386f |
+ (hwcap & PPC_FEATURE_HAS_VSX)
|
|
|
3b386f |
+ ? __rawmemchr_power7
|
|
|
3b386f |
: __rawmemchr_ppc);
|
|
|
3b386f |
|
|
|
3b386f |
weak_alias (__rawmemchr, rawmemchr)
|