From 1cf3bb5ec15f28245a6840b5b0443685c828a467 Mon Sep 17 00:00:00 2001

From: "Paul E. Murphy" <murphyp@linux.vnet.ibm.com>

Date: Mon, 14 Mar 2016 17:40:46 -0400

Subject: [PATCH] powerpc: Add optimized P8 strspn

This utilizes vectors and bitmasks.  For small needle, large

haystack, the performance improvement is upto 8x.  For short

strings (0-4B), the cost of computing the bitmask dominates,

and is a tad slower.

(cherry picked from commit 25dba0ad054723196fb633ba5d8a463ef5cb775c)

---

 ChangeLog                                          |  15 ++

 sysdeps/powerpc/powerpc64/multiarch/Makefile       |   3 +-

 .../powerpc/powerpc64/multiarch/ifunc-impl-list.c  |   8 +

 .../powerpc/powerpc64/multiarch/strspn-power8.S    |  40 +++++

 sysdeps/powerpc/powerpc64/multiarch/strspn-ppc64.c |  25 +++

 sysdeps/powerpc/powerpc64/multiarch/strspn.c       |  35 ++++

 sysdeps/powerpc/powerpc64/power8/strspn.S          | 179 +++++++++++++++++++++

 7 files changed, 304 insertions(+), 1 deletion(-)

 create mode 100644 sysdeps/powerpc/powerpc64/multiarch/strspn-power8.S

 create mode 100644 sysdeps/powerpc/powerpc64/multiarch/strspn-ppc64.c

 create mode 100644 sysdeps/powerpc/powerpc64/multiarch/strspn.c

 create mode 100644 sysdeps/powerpc/powerpc64/power8/strspn.S

diff --git a/ChangeLog b/ChangeLog

index 496ef12..f030b68 100644

diff --git a/sysdeps/powerpc/powerpc64/multiarch/Makefile b/sysdeps/powerpc/powerpc64/multiarch/Makefile

index 3b0e3a0..7ed56bf 100644

--- a/sysdeps/powerpc/powerpc64/multiarch/Makefile

+++ b/sysdeps/powerpc/powerpc64/multiarch/Makefile

@@ -19,6 +19,7 @@ sysdep_routines += memcpy-power7 memcpy-a2 memcpy-power6 memcpy-cell \

                   strncpy-power8 strncpy-power7 strncpy-ppc64 \

                   strncat-power7 \

                   strstr-power7 strstr-ppc64 \

+                  strspn-power8 strspn-ppc64 \

                   rawmemchr-ppc64 strlen-power7 strlen-ppc64 strnlen-power7 \

                   strnlen-ppc64 strcasecmp-power7 strcasecmp_l-power7 \

                   strncase-power7 strncase_l-power7 \

diff --git a/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c b/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c

index 364385b..f6c70ba 100644

--- a/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c

+++ b/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c

@@ -322,6 +322,14 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,

                 IFUNC_IMPL_ADD (array, i, strcat, 1,

                                 __strcat_ppc))

+  /* Support sysdeps/powerpc/powerpc64/multiarch/strspn.c.  */

+  IFUNC_IMPL (i, name, strspn,

+             IFUNC_IMPL_ADD (array, i, strspn,

+                             hwcap2 & PPC_FEATURE2_ARCH_2_07,

+                             __strspn_power8)

+             IFUNC_IMPL_ADD (array, i, strspn, 1,

+                             __strspn_ppc))

+

   /* Support sysdeps/powerpc/powerpc64/multiarch/strstr.c.  */

   IFUNC_IMPL (i, name, strstr,

              IFUNC_IMPL_ADD (array, i, strstr,

diff --git a/sysdeps/powerpc/powerpc64/multiarch/strspn-power8.S b/sysdeps/powerpc/powerpc64/multiarch/strspn-power8.S

new file mode 100644

index 0000000..86a4e09

--- /dev/null

+++ b/sysdeps/powerpc/powerpc64/multiarch/strspn-power8.S

@@ -0,0 +1,40 @@

+/* Optimized strspn implementation for POWER8.

+   Copyright (C) 2016 Free Software Foundation, Inc.

+   This file is part of the GNU C Library.

+

+   The GNU C Library is free software; you can redistribute it and/or

+   modify it under the terms of the GNU Lesser General Public

+   License as published by the Free Software Foundation; either

+   version 2.1 of the License, or (at your option) any later version.

+

+   The GNU C Library is distributed in the hope that it will be useful,

+   but WITHOUT ANY WARRANTY; without even the implied warranty of

+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

+   Lesser General Public License for more details.

+

+   You should have received a copy of the GNU Lesser General Public

+   License along with the GNU C Library; if not, see

+   <http://www.gnu.org/licenses/>.  */

+

+#include <sysdep.h>

+

+#undef EALIGN

+#define EALIGN(name, alignt, words)				\

+  .section ".text";						\

+  ENTRY_2(__strspn_power8)					\

+  .align ALIGNARG(alignt);					\

+  EALIGN_W_##words;						\

+  BODY_LABEL(__strspn_power8):					\

+  cfi_startproc;						\

+  LOCALENTRY(__strspn_power8)

+

+#undef END

+#define END(name)						\

+  cfi_endproc;							\

+  TRACEBACK(__strspn_power8)					\

+  END_2(__strspn_power8)

+

+#undef libc_hidden_builtin_def

+#define libc_hidden_builtin_def(name)

+

+#include <sysdeps/powerpc/powerpc64/power8/strspn.S>

diff --git a/sysdeps/powerpc/powerpc64/multiarch/strspn-ppc64.c b/sysdeps/powerpc/powerpc64/multiarch/strspn-ppc64.c

new file mode 100644

index 0000000..4c63665

--- /dev/null

+++ b/sysdeps/powerpc/powerpc64/multiarch/strspn-ppc64.c

@@ -0,0 +1,25 @@

+/* Default strspn implementation for PowerPC64.

+   Copyright (C) 2016 Free Software Foundation, Inc.

+   This file is part of the GNU C Library.

+

+   The GNU C Library is free software; you can redistribute it and/or

+   modify it under the terms of the GNU Lesser General Public

+   License as published by the Free Software Foundation; either

+   version 2.1 of the License, or (at your option) any later version.

+

+   The GNU C Library is distributed in the hope that it will be useful,

+   but WITHOUT ANY WARRANTY; without even the implied warranty of

+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

+   Lesser General Public License for more details.

+

+   You should have received a copy of the GNU Lesser General Public

+   License along with the GNU C Library; if not, see

+   <http://www.gnu.org/licenses/>.  */

+

+#define STRSPN __strspn_ppc

+#ifdef SHARED

+#undef libc_hidden_def

+#define libc_hidden_def(name)

+#endif

+

+#include <string/strspn.c>

diff --git a/sysdeps/powerpc/powerpc64/multiarch/strspn.c b/sysdeps/powerpc/powerpc64/multiarch/strspn.c

new file mode 100644

index 0000000..0e653f3

--- /dev/null

+++ b/sysdeps/powerpc/powerpc64/multiarch/strspn.c

@@ -0,0 +1,35 @@

+/* Multiple versions of strspn. PowerPC64 version.

+   Copyright (C) 2016 Free Software Foundation, Inc.

+   This file is part of the GNU C Library.

+

+   The GNU C Library is free software; you can redistribute it and/or

+   modify it under the terms of the GNU Lesser General Public

+   License as published by the Free Software Foundation; either

+   version 2.1 of the License, or (at your option) any later version.

+

+   The GNU C Library is distributed in the hope that it will be useful,

+   but WITHOUT ANY WARRANTY; without even the implied warranty of

+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

+   Lesser General Public License for more details.

+

+   You should have received a copy of the GNU Lesser General Public

+   License along with the GNU C Library; if not, see

+   <http://www.gnu.org/licenses/>.  */

+

+# include <string.h>

+# include <shlib-compat.h>

+# include "init-arch.h"

+

+#undef strspn

+extern __typeof (strspn) __libc_strspn;

+

+extern __typeof (strspn) __strspn_ppc attribute_hidden;

+extern __typeof (strspn) __strspn_power8 attribute_hidden;

+

+libc_ifunc (__libc_strspn,

+	    (hwcap2 & PPC_FEATURE2_ARCH_2_07)

+	    ? __strspn_power8

+	    : __strspn_ppc);

+

+weak_alias (__libc_strspn, strspn)

+libc_hidden_builtin_def (strspn)

diff --git a/sysdeps/powerpc/powerpc64/power8/strspn.S b/sysdeps/powerpc/powerpc64/power8/strspn.S

new file mode 100644

index 0000000..0dda437

--- /dev/null

+++ b/sysdeps/powerpc/powerpc64/power8/strspn.S

@@ -0,0 +1,179 @@

+/* Optimized strspn implementation for Power8.

+

+   Copyright (C) 2016 Free Software Foundation, Inc.

+   This file is part of the GNU C Library.

+

+   The GNU C Library is free software; you can redistribute it and/or

+   modify it under the terms of the GNU Lesser General Public

+   License as published by the Free Software Foundation; either

+   version 2.1 of the License, or (at your option) any later version.

+

+   The GNU C Library is distributed in the hope that it will be useful,

+   but WITHOUT ANY WARRANTY; without even the implied warranty of

+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

+   Lesser General Public License for more details.

+

+   You should have received a copy of the GNU Lesser General Public

+   License along with the GNU C Library; if not, see

+   <http://www.gnu.org/licenses/>.  */

+

+/* size_t [r3] strspn (const char *string [r3],

+                       const char *needleAccept [r4])  */

+

+/* This takes a novel approach by computing a 256 bit mask whereby

+   each set bit implies the byte is "accepted".  P8 vector hardware

+   has extremely efficient hardware for selecting bits from a mask.

+

+   One might ask "why not use bpermd for short strings"?  It is

+   so slow that its performance about matches the generic PPC64

+   variant without any fancy masking, with the added expense of

+   making the mask.  That was the first variant of this.  */

+

+

+

+#include "sysdep.h"

+

+/* Simple macro to use VSX instructions in overlapping VR's.  */

+#define XXVR(insn, vrt, vra, vrb) \

+	insn 32+vrt, 32+vra, 32+vrb

+

+/* ISA 2.07B instructions are not all defined for older binutils.

+   Macros are defined below for these newer instructions in order

+   to maintain compatibility.  */

+

+/* Note, TX/SX is always set as VMX regs are the high 32 VSX regs.  */

+#define MTVRD(v,r) .long (0x7c000167 | ((v)<<(32-11)) | ((r)<<(32-16)))

+#define MFVRD(r,v) .long (0x7c000067 | ((v)<<(32-11)) | ((r)<<(32-16)))

+

+#define VBPERMQ(t,a,b) .long (0x1000054c \

+			      | ((t)<<(32-11))	\

+			      | ((a)<<(32-16))	\

+			      | ((b)<<(32-21)) )

+

+	/* This can be updated to power8 once the minimum version of

+	   binutils supports power8 and the above instructions.  */

+	.machine power7

+EALIGN(strspn, 4, 0)

+	CALL_MCOUNT 2

+

+	/* Generate useful constants for later on.  */

+	vspltisb v1, 7

+	vspltisb v2, -1

+	vslb	v1, v1, v1	/* 0x80 to swap high bit for vbpermq.  */

+	vspltisb v10, 0

+	vsldoi	v4, v10, v2, 2	/* 0xFFFF into vr4.  */

+	XXVR(xxmrgld, v4, v4, v10) /* Mask for checking matches.  */

+

+	/* Prepare to compute 256b mask.  */

+	addi	r4, r4, -1

+	li	r5, 0

+	li	r6, 0

+	li	r7, 0

+	li	r8, 0

+	li	r11, 1

+	sldi	r11, r11, 63

+

+	/* Start interleaved Mask computation.

+	   This will eventually or 1's into ignored bits from vbpermq.  */

+	lvsr	v11, 0, r3

+	vspltb  v11, v11, 0	/* Splat shift constant.  */

+

+	/* Build a 256b mask in r5-r8.  */

+	.align 4

+L(next_needle):

+	lbzu	r9, 1(r4)

+

+	cmpldi	cr0, r9, 0

+	cmpldi	cr1, r9, 128

+

+	/* This is a little tricky.  srd only uses the first 7 bits,

+	   and if bit 7 is set, value is always 0.  So, we can

+	   effectively shift 128b in this case.  */

+	xori	r12, r9,  0x40	/* Invert bit 6.  */

+	srd	r10, r11, r9	/* Mask for bits 0-63.  */

+	srd	r12, r11, r12	/* Mask for bits 64-127.  */

+

+	beq	cr0, L(start_cmp)

+

+	/* Now, or the value into the correct GPR.  */

+	bge cr1,L(needle_gt128)

+	or	r5, r5, r10	/* 0 - 63.  */

+	or	r6, r6, r12	/* 64 - 127.  */

+	b L(next_needle)

+

+	.align 4

+L(needle_gt128):

+	or	r7, r7, r10	/* 128 - 191.  */

+	or	r8, r8, r12	/* 192 - 255.  */

+	b L(next_needle)

+

+

+	.align 4

+L(start_cmp):

+	/* Move and merge bitmap into 2 VRs.  bpermd is slower on P8.  */

+	mr	r0, r3		/* Save r3 for final length computation.  */

+	MTVRD (v5, r5)

+	MTVRD (v6, r6)

+	MTVRD (v7, r7)

+	MTVRD (v8, r8)

+

+	/* Continue interleaved mask generation.  */

+#ifdef __LITTLE_ENDIAN__

+	vsrw	v11, v2, v11	/* Note, shift ignores higher order bits.  */

+	vsplth  v11, v11, 0	/* Only care about the high 16 bits of v10.  */

+#else

+	vslw	v11, v2, v11	/* Note, shift ignores higher order bits.  */

+	vsplth  v11, v11, 1	/* Only care about the low 16 bits of v10.  */

+#endif

+	lvx	v0, 0, r3	/* Note, unaligned load ignores lower bits.  */

+

+	/* Do the merging of the bitmask.  */

+	XXVR(xxmrghd, v5, v5, v6)

+	XXVR(xxmrghd, v6, v7, v8)

+

+	/* Finish mask generation.  */

+	vand	v11, v11, v4	/* Throwaway bits not in the mask.  */

+

+	/* Compare the first 1-16B, while masking unwanted bytes.  */

+	clrrdi  r3, r3, 4	/* Note,  counts from qw boundaries.  */

+	vxor	v9, v0, v1	/* Swap high bit.  */

+	VBPERMQ (v8, v5, v0)

+	VBPERMQ (v7, v6, v9)

+	vor	v7, v7, v8

+	vor	v7, v7, v11	/* Ignore non-participating bytes.  */

+	vcmpequh. v8, v7, v4

+	bnl	cr6, L(done)

+

+	addi	r3, r3, 16

+

+	.align 4

+L(vec):

+	lvx	v0, 0, r3

+	addi	r3, r3, 16

+	vxor	v9, v0, v1	/* Swap high bit.  */

+	VBPERMQ (v8, v5, v0)

+	VBPERMQ (v7, v6, v9)

+	vor	v7, v7, v8

+	vcmpequh. v8, v7, v4

+	blt	cr6, L(vec)

+

+	addi	r3, r3, -16

+L(done):

+	subf	r3, r0, r3

+	MFVRD (r10, v7)

+

+#ifdef __LITTLE_ENDIAN__

+	addi	r0,  r10, 1	/* Count the trailing 1's.  */

+	andc	r10, r10, r0

+	popcntd	r10, r10

+#else

+	xori	r10, r10, 0xffff /* Count leading 1's by inverting.  */

+	addi	r3,  r3,  -48	/* Account for the extra leading zeros.  */

+	cntlzd  r10, r10

+#endif

+

+	add	r3, r3, r10

+	blr

+

+END(strspn)

+libc_hidden_builtin_def (strspn)

-- 

2.1.0