d8307d
commit 6f47401bd5fc71209219779a0426170a9a7395b0
d8307d
Author: Stefan Liebler <stli@linux.ibm.com>
d8307d
Date:   Fri Mar 22 11:14:08 2019 +0100
d8307d
d8307d
    S390: Add arch13 strstr ifunc variant.
d8307d
    
d8307d
    This patch introduces the new arch13 ifunc variant for strstr.
d8307d
    For needles longer than 9 charachters it is relying on the common-code
d8307d
    implementation.  For shorter needles it is using the new vstrs instruction
d8307d
    which is able to search a substring within a vector register.
d8307d
    
d8307d
    ChangeLog:
d8307d
    
d8307d
            * sysdeps/s390/Makefile (sysdep_routines): Add strstr-arch13.
d8307d
            * sysdeps/s390/ifunc-strstr.h (HAVE_STRSTR_ARCH13, STRSTR_ARCH13,
d8307d
            STRSTR_Z13_ONLY_USED_AS_FALLBACK, HAVE_STRSTR_IFUNC_AND_ARCH13_SUPPORT):
d8307d
            New defines.
d8307d
            * sysdeps/s390/multiarch/ifunc-impl-list.c
d8307d
            (__libc_ifunc_impl_list): Add ifunc variant for arch13 strstr.
d8307d
            * sysdeps/s390/strstr-arch13.S: New file.
d8307d
            * sysdeps/s390/strstr-vx.c: Omit GI symbol for z13 strstr ifunc variant
d8307d
            if it is only used as fallback.
d8307d
            * sysdeps/s390/strstr.c (strstr): Add arch13 variant in ifunc selector.
d8307d
d8307d
diff --git a/sysdeps/s390/Makefile b/sysdeps/s390/Makefile
d8307d
index 3f7de6613c343819..7287b1833da9500f 100644
d8307d
--- a/sysdeps/s390/Makefile
d8307d
+++ b/sysdeps/s390/Makefile
d8307d
@@ -35,7 +35,7 @@ sysdep_routines += bzero memset memset-z900 \
d8307d
 		   memcmp memcmp-z900 \
d8307d
 		   mempcpy memcpy memcpy-z900 \
d8307d
 		   memmove memmove-c \
d8307d
-		   strstr strstr-vx strstr-c \
d8307d
+		   strstr strstr-arch13 strstr-vx strstr-c \
d8307d
 		   memmem memmem-vx memmem-c \
d8307d
 		   strlen strlen-vx strlen-c \
d8307d
 		   strnlen strnlen-vx strnlen-c \
d8307d
diff --git a/sysdeps/s390/ifunc-strstr.h b/sysdeps/s390/ifunc-strstr.h
d8307d
index e6ccfd4e44a1a790..809184d425ad06b0 100644
d8307d
--- a/sysdeps/s390/ifunc-strstr.h
d8307d
+++ b/sysdeps/s390/ifunc-strstr.h
d8307d
@@ -17,7 +17,7 @@
d8307d
    <http://www.gnu.org/licenses/>.  */
d8307d
 
d8307d
 #if defined USE_MULTIARCH && IS_IN (libc)		\
d8307d
-  && ! defined HAVE_S390_MIN_Z13_ZARCH_ASM_SUPPORT
d8307d
+  && ! defined HAVE_S390_MIN_ARCH13_ZARCH_ASM_SUPPORT
d8307d
 # define HAVE_STRSTR_IFUNC	1
d8307d
 #else
d8307d
 # define HAVE_STRSTR_IFUNC	0
d8307d
@@ -29,14 +29,32 @@
d8307d
 # define HAVE_STRSTR_IFUNC_AND_VX_SUPPORT 0
d8307d
 #endif
d8307d
 
d8307d
-#if defined HAVE_S390_MIN_Z13_ZARCH_ASM_SUPPORT
d8307d
+#ifdef HAVE_S390_ARCH13_ASM_SUPPORT
d8307d
+# define HAVE_STRSTR_IFUNC_AND_ARCH13_SUPPORT HAVE_STRSTR_IFUNC
d8307d
+#else
d8307d
+# define HAVE_STRSTR_IFUNC_AND_ARCH13_SUPPORT 0
d8307d
+#endif
d8307d
+
d8307d
+#if defined HAVE_S390_MIN_ARCH13_ZARCH_ASM_SUPPORT
d8307d
+# define STRSTR_DEFAULT		STRSTR_ARCH13
d8307d
+# define HAVE_STRSTR_C		0
d8307d
+# define HAVE_STRSTR_Z13	1
d8307d
+# define STRSTR_Z13_ONLY_USED_AS_FALLBACK 1
d8307d
+# define HAVE_STRSTR_ARCH13	1
d8307d
+#elif defined HAVE_S390_MIN_Z13_ZARCH_ASM_SUPPORT
d8307d
 # define STRSTR_DEFAULT		STRSTR_Z13
d8307d
 # define HAVE_STRSTR_C		0
d8307d
 # define HAVE_STRSTR_Z13	1
d8307d
+# define HAVE_STRSTR_ARCH13	HAVE_STRSTR_IFUNC_AND_ARCH13_SUPPORT
d8307d
 #else
d8307d
 # define STRSTR_DEFAULT		STRSTR_C
d8307d
 # define HAVE_STRSTR_C		1
d8307d
 # define HAVE_STRSTR_Z13	HAVE_STRSTR_IFUNC_AND_VX_SUPPORT
d8307d
+# define HAVE_STRSTR_ARCH13	HAVE_STRSTR_IFUNC_AND_ARCH13_SUPPORT
d8307d
+#endif
d8307d
+
d8307d
+#ifndef STRSTR_Z13_ONLY_USED_AS_FALLBACK
d8307d
+# define STRSTR_Z13_ONLY_USED_AS_FALLBACK 0
d8307d
 #endif
d8307d
 
d8307d
 #if HAVE_STRSTR_C
d8307d
@@ -50,3 +68,9 @@
d8307d
 #else
d8307d
 # define STRSTR_Z13		NULL
d8307d
 #endif
d8307d
+
d8307d
+#if HAVE_STRSTR_ARCH13
d8307d
+# define STRSTR_ARCH13		__strstr_arch13
d8307d
+#else
d8307d
+# define STRSTR_ARCH13		NULL
d8307d
+#endif
d8307d
diff --git a/sysdeps/s390/multiarch/ifunc-impl-list.c b/sysdeps/s390/multiarch/ifunc-impl-list.c
d8307d
index c24bfc95f2d7a22d..67a6a9c94afccd48 100644
d8307d
--- a/sysdeps/s390/multiarch/ifunc-impl-list.c
d8307d
+++ b/sysdeps/s390/multiarch/ifunc-impl-list.c
d8307d
@@ -186,6 +186,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
d8307d
 
d8307d
 #if HAVE_STRSTR_IFUNC
d8307d
     IFUNC_IMPL (i, name, strstr,
d8307d
+# if HAVE_STRSTR_ARCH13
d8307d
+		IFUNC_IMPL_ADD (array, i, strstr,
d8307d
+				dl_hwcap & HWCAP_S390_VXRS_EXT2, STRSTR_ARCH13)
d8307d
+# endif
d8307d
 # if HAVE_STRSTR_Z13
d8307d
 		IFUNC_IMPL_ADD (array, i, strstr,
d8307d
 				dl_hwcap & HWCAP_S390_VX, STRSTR_Z13)
d8307d
diff --git a/sysdeps/s390/strstr-arch13.S b/sysdeps/s390/strstr-arch13.S
d8307d
new file mode 100644
d8307d
index 0000000000000000..929b026adfeba740
d8307d
--- /dev/null
d8307d
+++ b/sysdeps/s390/strstr-arch13.S
d8307d
@@ -0,0 +1,179 @@
d8307d
+/* Vector optimized 32/64 bit S/390 version of strstr.
d8307d
+   Copyright (C) 2019 Free Software Foundation, Inc.
d8307d
+   This file is part of the GNU C Library.
d8307d
+
d8307d
+   The GNU C Library is free software; you can redistribute it and/or
d8307d
+   modify it under the terms of the GNU Lesser General Public
d8307d
+   License as published by the Free Software Foundation; either
d8307d
+   version 2.1 of the License, or (at your option) any later version.
d8307d
+
d8307d
+   The GNU C Library is distributed in the hope that it will be useful,
d8307d
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
d8307d
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
d8307d
+   Lesser General Public License for more details.
d8307d
+
d8307d
+   You should have received a copy of the GNU Lesser General Public
d8307d
+   License along with the GNU C Library; if not, see
d8307d
+   <http://www.gnu.org/licenses/>.  */
d8307d
+
d8307d
+#include <ifunc-strstr.h>
d8307d
+#if HAVE_STRSTR_ARCH13
d8307d
+# include "sysdep.h"
d8307d
+# include "asm-syntax.h"
d8307d
+	.text
d8307d
+
d8307d
+/* char *strstr (const char *haystack=r2, const char *needle=r3)
d8307d
+   Locate a substring.  */
d8307d
+ENTRY(STRSTR_ARCH13)
d8307d
+	.machine "arch13"
d8307d
+	.machinemode "zarch_nohighgprs"
d8307d
+	lcbb	%r1,0(%r3),6
d8307d
+	jo	.Lneedle_on_bb	/* Needle on block-boundary?  */
d8307d
+	vl	%v18,0(%r3),6	/* Load needle.  */
d8307d
+	vfenezb %v19,%v18,%v18	/* v19[7] contains the length of needle.  */
d8307d
+.Lneedle_loaded:
d8307d
+	vlgvb	%r4,%v19,7	/* Get index of zero or 16 if not found.  */
d8307d
+	lghi	%r5,17		/* See below: min-skip-partial-match-index.  */
d8307d
+	cgibe	%r4,0,0(%r14)	/* Test if needle is zero and return.  */
d8307d
+
d8307d
+	/* The vstrs instruction is able to handle needles up to a length of 16,
d8307d
+	   but then we may have to load the next part of haystack with a
d8307d
+	   small offset.  This will be slow - see examples:
d8307d
+	   haystack =mmmmmmmmmmmmmmmm mmmmmmmmmmmmmmmmmm...mmmmmmmmmmmmmmmmmmma
d8307d
+	   needle   =  mmmmmmmmmmmmmma0
d8307d
+	   => needle_len=15; vstrs reports a partial match; haystack+=2
d8307d
+	   haystack =mmmmmmmmmmmmmmmm mmmmmmmmmmmmmmmmmm...mmmmmmmmmmmmmmmmmmma
d8307d
+	   needle   =        mmmmmmmma0000000
d8307d
+	   => needle_len=9; vstrs reports a partial match; haystack+=8  */
d8307d
+# if ! HAVE_STRSTR_Z13
d8307d
+#  error The arch13 variant of strstr needs the z13 variant of strstr!
d8307d
+# endif
d8307d
+	clgfi	%r4,9
d8307d
+	jh	STRSTR_Z13
d8307d
+
d8307d
+	/* In case of a partial match, the vstrs instruction returns the index
d8307d
+	   of the partial match in a vector-register.  Then we have to
d8307d
+	   reload the string at the "current-position plus this index" and run
d8307d
+	   vstrs again in order to determine if it was a full match or no match.
d8307d
+	   Transferring this index from vr to gr, compute the haystack-address
d8307d
+	   and loading with vl is quite slow as all instructions have data
d8307d
+	   dependencies.  Thus we assume, that a partial match is always at the
d8307d
+	   first possible index and just load the next part of haystack from
d8307d
+	   there instead of waiting until the correct index is computed:
d8307d
+	   min-skip-partial-match-index = (16 - n_len) + 1  */
d8307d
+	sgr	%r5,%r4
d8307d
+
d8307d
+.Lloop:
d8307d
+	lcbb	%r1,0(%r2),6
d8307d
+	jo	.Lloop_haystack_on_bb	/* Haystack on block-boundary?  */
d8307d
+	vl	%v16,0(%r2)		/* Load next part of haystack.  */
d8307d
+.Lloop_haystack_loaded:
d8307d
+	/* Vector string search with zero search (cc=0 => no match).  */
d8307d
+	vstrs	%v20,%v16,%v18,%v19,0,2
d8307d
+	jne	.Lloop_vstrs_nonzero_cc
d8307d
+	lcbb	%r1,16(%r2),6		/* Next part of haystack.  */
d8307d
+	jo	.Lloop_haystack_on_bb16
d8307d
+	vl	%v16,16(%r2)
d8307d
+	vstrs	%v20,%v16,%v18,%v19,0,2
d8307d
+	jne	.Lloop_vstrs_nonzero_cc16
d8307d
+	lcbb	%r1,32(%r2),6		/* Next part of haystack.  */
d8307d
+	jo	.Lloop_haystack_on_bb32
d8307d
+	vl	%v16,32(%r2)
d8307d
+	vstrs	%v20,%v16,%v18,%v19,0,2
d8307d
+	jne	.Lloop_vstrs_nonzero_cc32
d8307d
+	lcbb	%r1,48(%r2),6		/* Next part of haystack.  */
d8307d
+	jo	.Lloop_haystack_on_bb48
d8307d
+	vl	%v16,48(%r2)
d8307d
+	vstrs	%v20,%v16,%v18,%v19,0,2
d8307d
+	jne	.Lloop_vstrs_nonzero_cc48
d8307d
+	la	%r2,64(%r2)
d8307d
+	j	.Lloop
d8307d
+
d8307d
+.Lloop_vstrs_nonzero_cc48:
d8307d
+	la	%r2,16(%r2)
d8307d
+.Lloop_vstrs_nonzero_cc32:
d8307d
+	la	%r2,16(%r2)
d8307d
+.Lloop_vstrs_nonzero_cc16:
d8307d
+	la	%r2,16(%r2)
d8307d
+.Lloop_vstrs_nonzero_cc:
d8307d
+	jh	.Lend_match_found /* cc == 2 (full match)  */
d8307d
+	jl	.Lend_no_match	/* cc == 1 (no match, end of string)  */
d8307d
+	/* cc == 3 (partial match) See above: min-skip-partial-match-index!  */
d8307d
+	lcbb	%r1,0(%r5,%r2),6
d8307d
+	la	%r2,0(%r5,%r2)
d8307d
+	jo	.Lloop_haystack_on_bb
d8307d
+	vl	%v16,0(%r2)
d8307d
+	vstrs	%v20,%v16,%v18,%v19,0,2
d8307d
+.Lloop_vstrs_nonzero_cc_loop:
d8307d
+	jh	.Lend_match_found
d8307d
+	jl	.Lend_no_match
d8307d
+	la	%r2,0(%r5,%r2)
d8307d
+	je	.Lloop
d8307d
+	lcbb	%r1,0(%r2),6		/* Next part of haystack.  */
d8307d
+	jo	.Lloop_haystack_on_bb
d8307d
+	vl	%v16,0(%r2)
d8307d
+	vstrs	%v20,%v16,%v18,%v19,0,2
d8307d
+	jh	.Lend_match_found
d8307d
+	jl	.Lend_no_match
d8307d
+	la	%r2,0(%r5,%r2)
d8307d
+	je	.Lloop
d8307d
+	lcbb	%r1,0(%r2),6		/* Next part of haystack.  */
d8307d
+	jo	.Lloop_haystack_on_bb
d8307d
+	vl	%v16,0(%r2)
d8307d
+	vstrs	%v20,%v16,%v18,%v19,0,2
d8307d
+	jh	.Lend_match_found
d8307d
+	jl	.Lend_no_match
d8307d
+	la	%r2,0(%r5,%r2)
d8307d
+	je	.Lloop
d8307d
+	lcbb	%r1,0(%r2),6		/* Next part of haystack.  */
d8307d
+	jo	.Lloop_haystack_on_bb
d8307d
+	vl	%v16,0(%r2)
d8307d
+	vstrs	%v20,%v16,%v18,%v19,0,2
d8307d
+	j	.Lloop_vstrs_nonzero_cc_loop
d8307d
+
d8307d
+.Lend_no_match:
d8307d
+	lghi	%r2,0
d8307d
+	br	%r14
d8307d
+.Lend_match_found:
d8307d
+	vlgvb	%r4,%v20,7
d8307d
+	la	%r2,0(%r4,%r2)
d8307d
+	br	%r14
d8307d
+
d8307d
+.Lloop_haystack_on_bb48:
d8307d
+	la	%r2,16(%r2)
d8307d
+.Lloop_haystack_on_bb32:
d8307d
+	la	%r2,16(%r2)
d8307d
+.Lloop_haystack_on_bb16:
d8307d
+	la	%r2,16(%r2)
d8307d
+.Lloop_haystack_on_bb:
d8307d
+	/* Haystack located on page-boundary.  */
d8307d
+	ahi	%r1,-1		/* vll needs highest index instead of count.  */
d8307d
+	vll	%v16,%r1,0(%r2)
d8307d
+	vlvgb	%v21,%r1,7
d8307d
+	vfenezb	%v17,%v16,%v16	/* Search zero in loaded haystack bytes.  */
d8307d
+	veclb	%v17,%v21		/* Zero index <= loaded byte index?  */
d8307d
+	jle	.Lloop_haystack_loaded	/* -> v16 contains full haystack.  */
d8307d
+	vl	%v16,0(%r2)	/* Load haystack beyond page boundary.  */
d8307d
+	j	.Lloop_haystack_loaded
d8307d
+
d8307d
+.Lneedle_on_bb:
d8307d
+	/* Needle located on page-boundary.  */
d8307d
+	ahi	%r1,-1		/* vll needs highest index instead of count.  */
d8307d
+	vll	%v18,%r1,0(%r3)
d8307d
+	vlvgb	%v21,%r1,7
d8307d
+	vfenezb	%v19,%v18,%v18	/* Search zero in loaded needle bytes.  */
d8307d
+	veclb	%v19,%v21	/* Zero index <= max loaded byte index?  */
d8307d
+	jle	.Lneedle_loaded	/* -> v18 contains full needle.  */
d8307d
+	vl	%v16,0(%r3)	/* Load needle beyond page boundary.  */
d8307d
+	vfenezb	%v19,%v18,%v18
d8307d
+	j	.Lneedle_loaded
d8307d
+END(STRSTR_ARCH13)
d8307d
+
d8307d
+# if ! HAVE_STRSTR_IFUNC
d8307d
+strong_alias (STRSTR_ARCH13, strstr)
d8307d
+# endif
d8307d
+
d8307d
+# if STRSTR_Z13_ONLY_USED_AS_FALLBACK && defined SHARED && IS_IN (libc)
d8307d
+strong_alias (STRSTR_ARCH13, __GI_strstr)
d8307d
+# endif
d8307d
+#endif
d8307d
diff --git a/sysdeps/s390/strstr-vx.c b/sysdeps/s390/strstr-vx.c
d8307d
index effae9d5eb7d2fb1..f69159ffc198b10b 100644
d8307d
--- a/sysdeps/s390/strstr-vx.c
d8307d
+++ b/sysdeps/s390/strstr-vx.c
d8307d
@@ -19,11 +19,11 @@
d8307d
 #include <ifunc-strstr.h>
d8307d
 
d8307d
 #if HAVE_STRSTR_Z13
d8307d
-# if HAVE_STRSTR_IFUNC
d8307d
+# if HAVE_STRSTR_IFUNC || STRSTR_Z13_ONLY_USED_AS_FALLBACK
d8307d
 #  define STRSTR STRSTR_Z13
d8307d
 #  if defined SHARED && IS_IN (libc)
d8307d
 #   undef libc_hidden_builtin_def
d8307d
-#   if HAVE_STRSTR_C
d8307d
+#   if HAVE_STRSTR_C || STRSTR_Z13_ONLY_USED_AS_FALLBACK
d8307d
 #    define libc_hidden_builtin_def(name)
d8307d
 #   else
d8307d
 #    define libc_hidden_builtin_def(name)		\
d8307d
diff --git a/sysdeps/s390/strstr.c b/sysdeps/s390/strstr.c
d8307d
index f8432349a7254cc6..d2969fd0a68fadbf 100644
d8307d
--- a/sysdeps/s390/strstr.c
d8307d
+++ b/sysdeps/s390/strstr.c
d8307d
@@ -32,8 +32,14 @@ extern __typeof (__redirect_strstr) STRSTR_C attribute_hidden;
d8307d
 extern __typeof (__redirect_strstr) STRSTR_Z13 attribute_hidden;
d8307d
 # endif
d8307d
 
d8307d
+# if HAVE_STRSTR_ARCH13
d8307d
+extern __typeof (__redirect_strstr) STRSTR_ARCH13 attribute_hidden;
d8307d
+# endif
d8307d
+
d8307d
 s390_libc_ifunc_expr (__redirect_strstr, strstr,
d8307d
-		      (HAVE_STRSTR_Z13 && (hwcap & HWCAP_S390_VX))
d8307d
+		      (HAVE_STRSTR_ARCH13 && (hwcap & HWCAP_S390_VXRS_EXT2))
d8307d
+		      ? STRSTR_ARCH13
d8307d
+		      : (HAVE_STRSTR_Z13 && (hwcap & HWCAP_S390_VX))
d8307d
 		      ? STRSTR_Z13
d8307d
 		      : STRSTR_DEFAULT
d8307d
 		      )