077c9d
commit 6f47401bd5fc71209219779a0426170a9a7395b0
077c9d
Author: Stefan Liebler <stli@linux.ibm.com>
077c9d
Date:   Fri Mar 22 11:14:08 2019 +0100
077c9d
077c9d
    S390: Add arch13 strstr ifunc variant.
077c9d
    
077c9d
    This patch introduces the new arch13 ifunc variant for strstr.
077c9d
    For needles longer than 9 charachters it is relying on the common-code
077c9d
    implementation.  For shorter needles it is using the new vstrs instruction
077c9d
    which is able to search a substring within a vector register.
077c9d
    
077c9d
    ChangeLog:
077c9d
    
077c9d
            * sysdeps/s390/Makefile (sysdep_routines): Add strstr-arch13.
077c9d
            * sysdeps/s390/ifunc-strstr.h (HAVE_STRSTR_ARCH13, STRSTR_ARCH13,
077c9d
            STRSTR_Z13_ONLY_USED_AS_FALLBACK, HAVE_STRSTR_IFUNC_AND_ARCH13_SUPPORT):
077c9d
            New defines.
077c9d
            * sysdeps/s390/multiarch/ifunc-impl-list.c
077c9d
            (__libc_ifunc_impl_list): Add ifunc variant for arch13 strstr.
077c9d
            * sysdeps/s390/strstr-arch13.S: New file.
077c9d
            * sysdeps/s390/strstr-vx.c: Omit GI symbol for z13 strstr ifunc variant
077c9d
            if it is only used as fallback.
077c9d
            * sysdeps/s390/strstr.c (strstr): Add arch13 variant in ifunc selector.
077c9d
077c9d
diff --git a/sysdeps/s390/Makefile b/sysdeps/s390/Makefile
077c9d
index 3f7de6613c343819..7287b1833da9500f 100644
077c9d
--- a/sysdeps/s390/Makefile
077c9d
+++ b/sysdeps/s390/Makefile
077c9d
@@ -35,7 +35,7 @@ sysdep_routines += bzero memset memset-z900 \
077c9d
 		   memcmp memcmp-z900 \
077c9d
 		   mempcpy memcpy memcpy-z900 \
077c9d
 		   memmove memmove-c \
077c9d
-		   strstr strstr-vx strstr-c \
077c9d
+		   strstr strstr-arch13 strstr-vx strstr-c \
077c9d
 		   memmem memmem-vx memmem-c \
077c9d
 		   strlen strlen-vx strlen-c \
077c9d
 		   strnlen strnlen-vx strnlen-c \
077c9d
diff --git a/sysdeps/s390/ifunc-strstr.h b/sysdeps/s390/ifunc-strstr.h
077c9d
index e6ccfd4e44a1a790..809184d425ad06b0 100644
077c9d
--- a/sysdeps/s390/ifunc-strstr.h
077c9d
+++ b/sysdeps/s390/ifunc-strstr.h
077c9d
@@ -17,7 +17,7 @@
077c9d
    <http://www.gnu.org/licenses/>.  */
077c9d
 
077c9d
 #if defined USE_MULTIARCH && IS_IN (libc)		\
077c9d
-  && ! defined HAVE_S390_MIN_Z13_ZARCH_ASM_SUPPORT
077c9d
+  && ! defined HAVE_S390_MIN_ARCH13_ZARCH_ASM_SUPPORT
077c9d
 # define HAVE_STRSTR_IFUNC	1
077c9d
 #else
077c9d
 # define HAVE_STRSTR_IFUNC	0
077c9d
@@ -29,14 +29,32 @@
077c9d
 # define HAVE_STRSTR_IFUNC_AND_VX_SUPPORT 0
077c9d
 #endif
077c9d
 
077c9d
-#if defined HAVE_S390_MIN_Z13_ZARCH_ASM_SUPPORT
077c9d
+#ifdef HAVE_S390_ARCH13_ASM_SUPPORT
077c9d
+# define HAVE_STRSTR_IFUNC_AND_ARCH13_SUPPORT HAVE_STRSTR_IFUNC
077c9d
+#else
077c9d
+# define HAVE_STRSTR_IFUNC_AND_ARCH13_SUPPORT 0
077c9d
+#endif
077c9d
+
077c9d
+#if defined HAVE_S390_MIN_ARCH13_ZARCH_ASM_SUPPORT
077c9d
+# define STRSTR_DEFAULT		STRSTR_ARCH13
077c9d
+# define HAVE_STRSTR_C		0
077c9d
+# define HAVE_STRSTR_Z13	1
077c9d
+# define STRSTR_Z13_ONLY_USED_AS_FALLBACK 1
077c9d
+# define HAVE_STRSTR_ARCH13	1
077c9d
+#elif defined HAVE_S390_MIN_Z13_ZARCH_ASM_SUPPORT
077c9d
 # define STRSTR_DEFAULT		STRSTR_Z13
077c9d
 # define HAVE_STRSTR_C		0
077c9d
 # define HAVE_STRSTR_Z13	1
077c9d
+# define HAVE_STRSTR_ARCH13	HAVE_STRSTR_IFUNC_AND_ARCH13_SUPPORT
077c9d
 #else
077c9d
 # define STRSTR_DEFAULT		STRSTR_C
077c9d
 # define HAVE_STRSTR_C		1
077c9d
 # define HAVE_STRSTR_Z13	HAVE_STRSTR_IFUNC_AND_VX_SUPPORT
077c9d
+# define HAVE_STRSTR_ARCH13	HAVE_STRSTR_IFUNC_AND_ARCH13_SUPPORT
077c9d
+#endif
077c9d
+
077c9d
+#ifndef STRSTR_Z13_ONLY_USED_AS_FALLBACK
077c9d
+# define STRSTR_Z13_ONLY_USED_AS_FALLBACK 0
077c9d
 #endif
077c9d
 
077c9d
 #if HAVE_STRSTR_C
077c9d
@@ -50,3 +68,9 @@
077c9d
 #else
077c9d
 # define STRSTR_Z13		NULL
077c9d
 #endif
077c9d
+
077c9d
+#if HAVE_STRSTR_ARCH13
077c9d
+# define STRSTR_ARCH13		__strstr_arch13
077c9d
+#else
077c9d
+# define STRSTR_ARCH13		NULL
077c9d
+#endif
077c9d
diff --git a/sysdeps/s390/multiarch/ifunc-impl-list.c b/sysdeps/s390/multiarch/ifunc-impl-list.c
077c9d
index c24bfc95f2d7a22d..67a6a9c94afccd48 100644
077c9d
--- a/sysdeps/s390/multiarch/ifunc-impl-list.c
077c9d
+++ b/sysdeps/s390/multiarch/ifunc-impl-list.c
077c9d
@@ -186,6 +186,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
077c9d
 
077c9d
 #if HAVE_STRSTR_IFUNC
077c9d
     IFUNC_IMPL (i, name, strstr,
077c9d
+# if HAVE_STRSTR_ARCH13
077c9d
+		IFUNC_IMPL_ADD (array, i, strstr,
077c9d
+				dl_hwcap & HWCAP_S390_VXRS_EXT2, STRSTR_ARCH13)
077c9d
+# endif
077c9d
 # if HAVE_STRSTR_Z13
077c9d
 		IFUNC_IMPL_ADD (array, i, strstr,
077c9d
 				dl_hwcap & HWCAP_S390_VX, STRSTR_Z13)
077c9d
diff --git a/sysdeps/s390/strstr-arch13.S b/sysdeps/s390/strstr-arch13.S
077c9d
new file mode 100644
077c9d
index 0000000000000000..929b026adfeba740
077c9d
--- /dev/null
077c9d
+++ b/sysdeps/s390/strstr-arch13.S
077c9d
@@ -0,0 +1,179 @@
077c9d
+/* Vector optimized 32/64 bit S/390 version of strstr.
077c9d
+   Copyright (C) 2019 Free Software Foundation, Inc.
077c9d
+   This file is part of the GNU C Library.
077c9d
+
077c9d
+   The GNU C Library is free software; you can redistribute it and/or
077c9d
+   modify it under the terms of the GNU Lesser General Public
077c9d
+   License as published by the Free Software Foundation; either
077c9d
+   version 2.1 of the License, or (at your option) any later version.
077c9d
+
077c9d
+   The GNU C Library is distributed in the hope that it will be useful,
077c9d
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
077c9d
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
077c9d
+   Lesser General Public License for more details.
077c9d
+
077c9d
+   You should have received a copy of the GNU Lesser General Public
077c9d
+   License along with the GNU C Library; if not, see
077c9d
+   <http://www.gnu.org/licenses/>.  */
077c9d
+
077c9d
+#include <ifunc-strstr.h>
077c9d
+#if HAVE_STRSTR_ARCH13
077c9d
+# include "sysdep.h"
077c9d
+# include "asm-syntax.h"
077c9d
+	.text
077c9d
+
077c9d
+/* char *strstr (const char *haystack=r2, const char *needle=r3)
077c9d
+   Locate a substring.  */
077c9d
+ENTRY(STRSTR_ARCH13)
077c9d
+	.machine "arch13"
077c9d
+	.machinemode "zarch_nohighgprs"
077c9d
+	lcbb	%r1,0(%r3),6
077c9d
+	jo	.Lneedle_on_bb	/* Needle on block-boundary?  */
077c9d
+	vl	%v18,0(%r3),6	/* Load needle.  */
077c9d
+	vfenezb %v19,%v18,%v18	/* v19[7] contains the length of needle.  */
077c9d
+.Lneedle_loaded:
077c9d
+	vlgvb	%r4,%v19,7	/* Get index of zero or 16 if not found.  */
077c9d
+	lghi	%r5,17		/* See below: min-skip-partial-match-index.  */
077c9d
+	cgibe	%r4,0,0(%r14)	/* Test if needle is zero and return.  */
077c9d
+
077c9d
+	/* The vstrs instruction is able to handle needles up to a length of 16,
077c9d
+	   but then we may have to load the next part of haystack with a
077c9d
+	   small offset.  This will be slow - see examples:
077c9d
+	   haystack =mmmmmmmmmmmmmmmm mmmmmmmmmmmmmmmmmm...mmmmmmmmmmmmmmmmmmma
077c9d
+	   needle   =  mmmmmmmmmmmmmma0
077c9d
+	   => needle_len=15; vstrs reports a partial match; haystack+=2
077c9d
+	   haystack =mmmmmmmmmmmmmmmm mmmmmmmmmmmmmmmmmm...mmmmmmmmmmmmmmmmmmma
077c9d
+	   needle   =        mmmmmmmma0000000
077c9d
+	   => needle_len=9; vstrs reports a partial match; haystack+=8  */
077c9d
+# if ! HAVE_STRSTR_Z13
077c9d
+#  error The arch13 variant of strstr needs the z13 variant of strstr!
077c9d
+# endif
077c9d
+	clgfi	%r4,9
077c9d
+	jh	STRSTR_Z13
077c9d
+
077c9d
+	/* In case of a partial match, the vstrs instruction returns the index
077c9d
+	   of the partial match in a vector-register.  Then we have to
077c9d
+	   reload the string at the "current-position plus this index" and run
077c9d
+	   vstrs again in order to determine if it was a full match or no match.
077c9d
+	   Transferring this index from vr to gr, compute the haystack-address
077c9d
+	   and loading with vl is quite slow as all instructions have data
077c9d
+	   dependencies.  Thus we assume, that a partial match is always at the
077c9d
+	   first possible index and just load the next part of haystack from
077c9d
+	   there instead of waiting until the correct index is computed:
077c9d
+	   min-skip-partial-match-index = (16 - n_len) + 1  */
077c9d
+	sgr	%r5,%r4
077c9d
+
077c9d
+.Lloop:
077c9d
+	lcbb	%r1,0(%r2),6
077c9d
+	jo	.Lloop_haystack_on_bb	/* Haystack on block-boundary?  */
077c9d
+	vl	%v16,0(%r2)		/* Load next part of haystack.  */
077c9d
+.Lloop_haystack_loaded:
077c9d
+	/* Vector string search with zero search (cc=0 => no match).  */
077c9d
+	vstrs	%v20,%v16,%v18,%v19,0,2
077c9d
+	jne	.Lloop_vstrs_nonzero_cc
077c9d
+	lcbb	%r1,16(%r2),6		/* Next part of haystack.  */
077c9d
+	jo	.Lloop_haystack_on_bb16
077c9d
+	vl	%v16,16(%r2)
077c9d
+	vstrs	%v20,%v16,%v18,%v19,0,2
077c9d
+	jne	.Lloop_vstrs_nonzero_cc16
077c9d
+	lcbb	%r1,32(%r2),6		/* Next part of haystack.  */
077c9d
+	jo	.Lloop_haystack_on_bb32
077c9d
+	vl	%v16,32(%r2)
077c9d
+	vstrs	%v20,%v16,%v18,%v19,0,2
077c9d
+	jne	.Lloop_vstrs_nonzero_cc32
077c9d
+	lcbb	%r1,48(%r2),6		/* Next part of haystack.  */
077c9d
+	jo	.Lloop_haystack_on_bb48
077c9d
+	vl	%v16,48(%r2)
077c9d
+	vstrs	%v20,%v16,%v18,%v19,0,2
077c9d
+	jne	.Lloop_vstrs_nonzero_cc48
077c9d
+	la	%r2,64(%r2)
077c9d
+	j	.Lloop
077c9d
+
077c9d
+.Lloop_vstrs_nonzero_cc48:
077c9d
+	la	%r2,16(%r2)
077c9d
+.Lloop_vstrs_nonzero_cc32:
077c9d
+	la	%r2,16(%r2)
077c9d
+.Lloop_vstrs_nonzero_cc16:
077c9d
+	la	%r2,16(%r2)
077c9d
+.Lloop_vstrs_nonzero_cc:
077c9d
+	jh	.Lend_match_found /* cc == 2 (full match)  */
077c9d
+	jl	.Lend_no_match	/* cc == 1 (no match, end of string)  */
077c9d
+	/* cc == 3 (partial match) See above: min-skip-partial-match-index!  */
077c9d
+	lcbb	%r1,0(%r5,%r2),6
077c9d
+	la	%r2,0(%r5,%r2)
077c9d
+	jo	.Lloop_haystack_on_bb
077c9d
+	vl	%v16,0(%r2)
077c9d
+	vstrs	%v20,%v16,%v18,%v19,0,2
077c9d
+.Lloop_vstrs_nonzero_cc_loop:
077c9d
+	jh	.Lend_match_found
077c9d
+	jl	.Lend_no_match
077c9d
+	la	%r2,0(%r5,%r2)
077c9d
+	je	.Lloop
077c9d
+	lcbb	%r1,0(%r2),6		/* Next part of haystack.  */
077c9d
+	jo	.Lloop_haystack_on_bb
077c9d
+	vl	%v16,0(%r2)
077c9d
+	vstrs	%v20,%v16,%v18,%v19,0,2
077c9d
+	jh	.Lend_match_found
077c9d
+	jl	.Lend_no_match
077c9d
+	la	%r2,0(%r5,%r2)
077c9d
+	je	.Lloop
077c9d
+	lcbb	%r1,0(%r2),6		/* Next part of haystack.  */
077c9d
+	jo	.Lloop_haystack_on_bb
077c9d
+	vl	%v16,0(%r2)
077c9d
+	vstrs	%v20,%v16,%v18,%v19,0,2
077c9d
+	jh	.Lend_match_found
077c9d
+	jl	.Lend_no_match
077c9d
+	la	%r2,0(%r5,%r2)
077c9d
+	je	.Lloop
077c9d
+	lcbb	%r1,0(%r2),6		/* Next part of haystack.  */
077c9d
+	jo	.Lloop_haystack_on_bb
077c9d
+	vl	%v16,0(%r2)
077c9d
+	vstrs	%v20,%v16,%v18,%v19,0,2
077c9d
+	j	.Lloop_vstrs_nonzero_cc_loop
077c9d
+
077c9d
+.Lend_no_match:
077c9d
+	lghi	%r2,0
077c9d
+	br	%r14
077c9d
+.Lend_match_found:
077c9d
+	vlgvb	%r4,%v20,7
077c9d
+	la	%r2,0(%r4,%r2)
077c9d
+	br	%r14
077c9d
+
077c9d
+.Lloop_haystack_on_bb48:
077c9d
+	la	%r2,16(%r2)
077c9d
+.Lloop_haystack_on_bb32:
077c9d
+	la	%r2,16(%r2)
077c9d
+.Lloop_haystack_on_bb16:
077c9d
+	la	%r2,16(%r2)
077c9d
+.Lloop_haystack_on_bb:
077c9d
+	/* Haystack located on page-boundary.  */
077c9d
+	ahi	%r1,-1		/* vll needs highest index instead of count.  */
077c9d
+	vll	%v16,%r1,0(%r2)
077c9d
+	vlvgb	%v21,%r1,7
077c9d
+	vfenezb	%v17,%v16,%v16	/* Search zero in loaded haystack bytes.  */
077c9d
+	veclb	%v17,%v21		/* Zero index <= loaded byte index?  */
077c9d
+	jle	.Lloop_haystack_loaded	/* -> v16 contains full haystack.  */
077c9d
+	vl	%v16,0(%r2)	/* Load haystack beyond page boundary.  */
077c9d
+	j	.Lloop_haystack_loaded
077c9d
+
077c9d
+.Lneedle_on_bb:
077c9d
+	/* Needle located on page-boundary.  */
077c9d
+	ahi	%r1,-1		/* vll needs highest index instead of count.  */
077c9d
+	vll	%v18,%r1,0(%r3)
077c9d
+	vlvgb	%v21,%r1,7
077c9d
+	vfenezb	%v19,%v18,%v18	/* Search zero in loaded needle bytes.  */
077c9d
+	veclb	%v19,%v21	/* Zero index <= max loaded byte index?  */
077c9d
+	jle	.Lneedle_loaded	/* -> v18 contains full needle.  */
077c9d
+	vl	%v16,0(%r3)	/* Load needle beyond page boundary.  */
077c9d
+	vfenezb	%v19,%v18,%v18
077c9d
+	j	.Lneedle_loaded
077c9d
+END(STRSTR_ARCH13)
077c9d
+
077c9d
+# if ! HAVE_STRSTR_IFUNC
077c9d
+strong_alias (STRSTR_ARCH13, strstr)
077c9d
+# endif
077c9d
+
077c9d
+# if STRSTR_Z13_ONLY_USED_AS_FALLBACK && defined SHARED && IS_IN (libc)
077c9d
+strong_alias (STRSTR_ARCH13, __GI_strstr)
077c9d
+# endif
077c9d
+#endif
077c9d
diff --git a/sysdeps/s390/strstr-vx.c b/sysdeps/s390/strstr-vx.c
077c9d
index effae9d5eb7d2fb1..f69159ffc198b10b 100644
077c9d
--- a/sysdeps/s390/strstr-vx.c
077c9d
+++ b/sysdeps/s390/strstr-vx.c
077c9d
@@ -19,11 +19,11 @@
077c9d
 #include <ifunc-strstr.h>
077c9d
 
077c9d
 #if HAVE_STRSTR_Z13
077c9d
-# if HAVE_STRSTR_IFUNC
077c9d
+# if HAVE_STRSTR_IFUNC || STRSTR_Z13_ONLY_USED_AS_FALLBACK
077c9d
 #  define STRSTR STRSTR_Z13
077c9d
 #  if defined SHARED && IS_IN (libc)
077c9d
 #   undef libc_hidden_builtin_def
077c9d
-#   if HAVE_STRSTR_C
077c9d
+#   if HAVE_STRSTR_C || STRSTR_Z13_ONLY_USED_AS_FALLBACK
077c9d
 #    define libc_hidden_builtin_def(name)
077c9d
 #   else
077c9d
 #    define libc_hidden_builtin_def(name)		\
077c9d
diff --git a/sysdeps/s390/strstr.c b/sysdeps/s390/strstr.c
077c9d
index f8432349a7254cc6..d2969fd0a68fadbf 100644
077c9d
--- a/sysdeps/s390/strstr.c
077c9d
+++ b/sysdeps/s390/strstr.c
077c9d
@@ -32,8 +32,14 @@ extern __typeof (__redirect_strstr) STRSTR_C attribute_hidden;
077c9d
 extern __typeof (__redirect_strstr) STRSTR_Z13 attribute_hidden;
077c9d
 # endif
077c9d
 
077c9d
+# if HAVE_STRSTR_ARCH13
077c9d
+extern __typeof (__redirect_strstr) STRSTR_ARCH13 attribute_hidden;
077c9d
+# endif
077c9d
+
077c9d
 s390_libc_ifunc_expr (__redirect_strstr, strstr,
077c9d
-		      (HAVE_STRSTR_Z13 && (hwcap & HWCAP_S390_VX))
077c9d
+		      (HAVE_STRSTR_ARCH13 && (hwcap & HWCAP_S390_VXRS_EXT2))
077c9d
+		      ? STRSTR_ARCH13
077c9d
+		      : (HAVE_STRSTR_Z13 && (hwcap & HWCAP_S390_VX))
077c9d
 		      ? STRSTR_Z13
077c9d
 		      : STRSTR_DEFAULT
077c9d
 		      )