commit cdd927d98cc38acf55e1c6594b5c9451df8f239f

Author: Stefan Liebler <stli@linux.ibm.com>

Date:   Tue Dec 18 13:57:08 2018 +0100

    S390: Add z13 memmove ifunc variant.

    This patch introduces a z13 specific ifunc variant for memmove.

    As the common code implementation, it checks if we can copy from

    the beginning to the end - with z196 memcpy implementation - or

    if we have to copy from the end to the beginning.

    The latter case is done by using vector load/store instructions.

    If vector instructions are not available, the common-code is

    used as fallback.  Therefore it is implemented in memmove-c with

    a different name.

    Furthermore the ifunc logic decides if we need the common-code

    implementation at all.  If vector instructions are supported

    due to the minimum architecture level set we can skip the

    common-code ifunc variant.

    ChangeLog:

            * sysdeps/s390/Makefile (sysdep_routines): Add memmove-c.

            * sysdeps/s390/ifunc-memcpy.h (HAVE_MEMMOVE_IFUNC,

            HAVE_MEMMOVE_IFUNC_AND_VX_SUPPORT, MEMMOVE_DEFAULT,

            HAVE_MEMMOVE_C, MEMMOVE_C,  HAVE_MEMMOVE_Z13, MEMMOVE_Z13):

            New defines.

            * sysdeps/s390/memcpy-z900.S: Add z13 memmove implementation.

            * sysdeps/s390/memmove-c.c: New file.

            * sysdeps/s390/memmove.c: Likewise.

            * sysdeps/s390/multiarch/ifunc-impl-list.c

            (__libc_ifunc_impl_list): Add ifunc variants for memmove.

diff --git a/sysdeps/s390/Makefile b/sysdeps/s390/Makefile

index 838950a5ab958e31..3a7cccdf8f147398 100644

--- a/sysdeps/s390/Makefile

+++ b/sysdeps/s390/Makefile

@@ -33,5 +33,6 @@ endif

 ifeq ($(subdir),string)

 sysdep_routines += bzero memset memset-z900 \

 		   memcmp memcmp-z900 \

-		   mempcpy memcpy memcpy-z900

+		   mempcpy memcpy memcpy-z900 \

+		   memmove memmove-c

 endif

diff --git a/sysdeps/s390/ifunc-memcpy.h b/sysdeps/s390/ifunc-memcpy.h

index 51c71baa2c0b0452..0e701968c8f39014 100644

--- a/sysdeps/s390/ifunc-memcpy.h

+++ b/sysdeps/s390/ifunc-memcpy.h

@@ -43,6 +43,29 @@

 # define HAVE_MEMCPY_Z196	HAVE_MEMCPY_IFUNC

 #endif

+#if defined SHARED && defined USE_MULTIARCH && IS_IN (libc)	\

+  && ! defined HAVE_S390_MIN_Z13_ZARCH_ASM_SUPPORT

+# define HAVE_MEMMOVE_IFUNC	1

+#else

+# define HAVE_MEMMOVE_IFUNC	0

+#endif

+

+#ifdef HAVE_S390_VX_ASM_SUPPORT

+# define HAVE_MEMMOVE_IFUNC_AND_VX_SUPPORT HAVE_MEMMOVE_IFUNC

+#else

+# define HAVE_MEMMOVE_IFUNC_AND_VX_SUPPORT 0

+#endif

+

+#if defined HAVE_S390_MIN_Z13_ZARCH_ASM_SUPPORT

+# define MEMMOVE_DEFAULT	MEMMOVE_Z13

+# define HAVE_MEMMOVE_C		0

+# define HAVE_MEMMOVE_Z13	1

+#else

+# define MEMMOVE_DEFAULT	MEMMOVE_C

+# define HAVE_MEMMOVE_C		1

+# define HAVE_MEMMOVE_Z13	HAVE_MEMMOVE_IFUNC_AND_VX_SUPPORT

+#endif

+

 #if HAVE_MEMCPY_Z900_G5

 # define MEMCPY_Z900_G5		__memcpy_default

 # define MEMPCPY_Z900_G5	__mempcpy_default

@@ -66,3 +89,15 @@

 # define MEMCPY_Z196		NULL

 # define MEMPCPY_Z196		NULL

 #endif

+

+#if HAVE_MEMMOVE_C

+# define MEMMOVE_C		__memmove_c

+#else

+# define MEMMOVE_C		NULL

+#endif

+

+#if HAVE_MEMMOVE_Z13

+# define MEMMOVE_Z13		__memmove_z13

+#else

+# define MEMMOVE_Z13		NULL

+#endif

diff --git a/sysdeps/s390/memcpy-z900.S b/sysdeps/s390/memcpy-z900.S

index 3a50cf44d85d2417..bd3b1950ee442c0c 100644

--- a/sysdeps/s390/memcpy-z900.S

+++ b/sysdeps/s390/memcpy-z900.S

@@ -182,6 +182,7 @@ ENTRY(MEMCPY_Z196)

 # endif /* !defined __s390x__  */

 	ltgr    %r4,%r4

 	je      .L_Z196_4

+.L_Z196_start2:

 	aghi    %r4,-1

 	srlg    %r5,%r4,8

 	ltgr    %r5,%r5

@@ -207,6 +208,75 @@ ENTRY(MEMCPY_Z196)

 END(MEMCPY_Z196)

 #endif /* HAVE_MEMCPY_Z196  */

+#if HAVE_MEMMOVE_Z13

+ENTRY(MEMMOVE_Z13)

+	.machine "z13"

+	.machinemode "zarch_nohighgprs"

+# if !defined __s390x__

+	/* Note: The 31bit dst and src pointers are prefixed with zeroes.  */

+	llgfr	%r4,%r4

+	llgfr	%r3,%r3

+	llgfr	%r2,%r2

+# endif /* !defined __s390x__ */

+	sgrk	%r0,%r2,%r3

+	clgijh	%r4,16,.L_MEMMOVE_Z13_LARGE

+	aghik	%r5,%r4,-1

+.L_MEMMOVE_Z13_SMALL:

+	jl .L_MEMMOVE_Z13_END		/* Jump away if len was zero.  */

+	/* Store up to 16 bytes with vll/vstl which needs the index

+	   instead of lengths.  */

+	vll	%v16,%r5,0(%r3)

+	vstl	%v16,%r5,0(%r2)

+.L_MEMMOVE_Z13_END:

+	br      %r14

+.L_MEMMOVE_Z13_LARGE:

+	lgr     %r1,%r2			/* For memcpy: r1: Use as dest ;

+					   r2: Return dest  */

+	/* The unsigned comparison (dst - src >= len) determines if we can

+	   execute the forward case with memcpy.  */

+#if ! HAVE_MEMCPY_Z196

+# error The z13 variant of memmove needs the z196 variant of memcpy!

+#endif

+	clgrjhe %r0,%r4,.L_Z196_start2

+	risbgn	%r5,%r4,4,128+63,60	/* r5 = r4 / 16  */

+	aghi	%r4,-16

+	clgijhe	%r5,8,.L_MEMMOVE_Z13_LARGE_64B

+.L_MEMMOVE_Z13_LARGE_16B_LOOP:

+	/* Store at least 16 bytes with vl/vst. The number of 16byte blocks

+	   is stored in r5.  */

+	vl	%v16,0(%r4,%r3)

+	vst	%v16,0(%r4,%r2)

+	aghi	%r4,-16

+	brctg	%r5,.L_MEMMOVE_Z13_LARGE_16B_LOOP

+	aghik	%r5,%r4,15

+	j	.L_MEMMOVE_Z13_SMALL

+.L_MEMMOVE_Z13_LARGE_64B:

+	/* Store at least 128 bytes with 4x vl/vst. The number of 64byte blocks

+	   will be stored in r0.  */

+	aghi	%r4,-48

+	srlg	%r0,%r5,2		/* r5 = %r0 / 4

+					   => Number of 64byte blocks.  */

+.L_MEMMOVE_Z13_LARGE_64B_LOOP:

+	vl	%v20,48(%r4,%r3)

+	vl	%v19,32(%r4,%r3)

+	vl	%v18,16(%r4,%r3)

+	vl	%v17,0(%r4,%r3)

+	vst	%v20,48(%r4,%r2)

+	vst	%v19,32(%r4,%r2)

+	vst	%v18,16(%r4,%r2)

+	vst	%v17,0(%r4,%r2)

+	aghi	%r4,-64

+	brctg	%r0,.L_MEMMOVE_Z13_LARGE_64B_LOOP

+	aghi	%r4,48

+	/* Recalculate the number of 16byte blocks.  */

+	risbg	%r5,%r5,62,128+63,0	/* r5 = r5 & 3

+					   => Remaining 16byte blocks.  */

+	jne	.L_MEMMOVE_Z13_LARGE_16B_LOOP

+	aghik	%r5,%r4,15

+	j	.L_MEMMOVE_Z13_SMALL

+END(MEMMOVE_Z13)

+#endif /* HAVE_MEMMOVE_Z13  */

+

 #if ! HAVE_MEMCPY_IFUNC

 /* If we don't use ifunc, define an alias for mem[p]cpy here.

    Otherwise see sysdeps/s390/mem[p]cpy.c.  */

@@ -215,10 +285,27 @@ strong_alias (MEMPCPY_DEFAULT, __mempcpy)

 weak_alias (__mempcpy, mempcpy)

 #endif

+#if ! HAVE_MEMMOVE_IFUNC

+/* If we don't use ifunc, define an alias for memmove here.

+   Otherwise see sysdeps/s390/memmove.c.  */

+# if ! HAVE_MEMMOVE_C

+/* If the c variant is needed, then sysdeps/s390/memmove-c.c

+   defines memmove.

+   Otherwise MEMMOVE_DEFAULT is implemented here and we have to define it.  */

+strong_alias (MEMMOVE_DEFAULT, memmove)

+# endif

+#endif

+

 #if defined SHARED && IS_IN (libc)

 /* Defines the internal symbols.

    Compare to libc_hidden_[builtin_]def (mem[p]cpy) in string/mem[p]cpy.c.  */

 strong_alias (MEMCPY_DEFAULT, __GI_memcpy)

 strong_alias (MEMPCPY_DEFAULT, __GI_mempcpy)

 strong_alias (MEMPCPY_DEFAULT, __GI___mempcpy)

+# if ! HAVE_MEMMOVE_C

+/* If the c variant is needed, then sysdeps/s390/memmove-c.c

+   defines the internal symbol.

+   Otherwise MEMMOVE_DEFAULT is implemented here and we have to define it.  */

+strong_alias (MEMMOVE_DEFAULT, __GI_memmove)

+# endif

 #endif

diff --git a/sysdeps/s390/memmove-c.c b/sysdeps/s390/memmove-c.c

new file mode 100644

index 0000000000000000..be571093e019a38d

--- /dev/null

+++ b/sysdeps/s390/memmove-c.c

@@ -0,0 +1,37 @@

+/* Fallback C version of memmove.

+   Copyright (C) 2018 Free Software Foundation, Inc.

+   This file is part of the GNU C Library.

+

+   The GNU C Library is free software; you can redistribute it and/or

+   modify it under the terms of the GNU Lesser General Public

+   License as published by the Free Software Foundation; either

+   version 2.1 of the License, or (at your option) any later version.

+

+   The GNU C Library is distributed in the hope that it will be useful,

+   but WITHOUT ANY WARRANTY; without even the implied warranty of

+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

+   Lesser General Public License for more details.

+

+   You should have received a copy of the GNU Lesser General Public

+   License along with the GNU C Library; if not, see

+   <http://www.gnu.org/licenses/>.  */

+

+#include <ifunc-memcpy.h>

+

+#if HAVE_MEMMOVE_C

+# if HAVE_MEMMOVE_IFUNC

+/* If we use ifunc, then the memmove symbol is defined

+   in sysdeps/s390/memmove.c and we use a different name here.

+   Otherwise, we have to define memmove here or in

+   sysdeps/s390/memcpy.S depending on the used default implementation.  */

+#  define MEMMOVE MEMMOVE_C

+#  if defined SHARED && IS_IN (libc)

+/* Define the internal symbol.  */

+#   undef libc_hidden_builtin_def

+#   define libc_hidden_builtin_def(name)			\

+  __hidden_ver1 (__memmove_c, __GI_memmove, __memmove_c);

+#  endif

+# endif

+

+# include <string/memmove.c>

+#endif

diff --git a/sysdeps/s390/memmove.c b/sysdeps/s390/memmove.c

new file mode 100644

index 0000000000000000..ac34edf80f2678cd

--- /dev/null

+++ b/sysdeps/s390/memmove.c

@@ -0,0 +1,44 @@

+/* Multiple versions of memmove.

+   Copyright (C) 2016-2018 Free Software Foundation, Inc.

+   This file is part of the GNU C Library.

+

+   The GNU C Library is free software; you can redistribute it and/or

+   modify it under the terms of the GNU Lesser General Public

+   License as published by the Free Software Foundation; either

+   version 2.1 of the License, or (at your option) any later version.

+

+   The GNU C Library is distributed in the hope that it will be useful,

+   but WITHOUT ANY WARRANTY; without even the implied warranty of

+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

+   Lesser General Public License for more details.

+

+   You should have received a copy of the GNU Lesser General Public

+   License along with the GNU C Library; if not, see

+   <http://www.gnu.org/licenses/>.  */

+

+#include <ifunc-memcpy.h>

+

+#if HAVE_MEMMOVE_IFUNC

+/* If we don't use ifunc, an alias is defined for memmove

+   in sysdeps/s390/memmove-c.c or sysdeps/s390/memcpy.S

+   depending on the used default implementation.  */

+# undef memmove

+# define memmove __redirect_memmove

+# include <string.h>

+# include <ifunc-resolve.h>

+# undef memmove

+

+# if HAVE_MEMMOVE_C

+extern __typeof (__redirect_memmove) MEMMOVE_C attribute_hidden;

+# endif

+

+# if HAVE_MEMMOVE_Z13

+extern __typeof (__redirect_memmove) MEMMOVE_Z13 attribute_hidden;

+# endif

+

+s390_libc_ifunc_expr (__redirect_memmove, memmove,

+		      (HAVE_MEMMOVE_Z13 && (hwcap & HWCAP_S390_VX))

+		      ? MEMMOVE_Z13

+		      : MEMMOVE_DEFAULT

+		      )

+#endif

diff --git a/sysdeps/s390/multiarch/ifunc-impl-list.c b/sysdeps/s390/multiarch/ifunc-impl-list.c

index 6969c480cc40e0e2..c05c63e00608dcd7 100644

--- a/sysdeps/s390/multiarch/ifunc-impl-list.c

+++ b/sysdeps/s390/multiarch/ifunc-impl-list.c

@@ -126,6 +126,18 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,

 	      )

 #endif /* HAVE_MEMCPY_IFUNC  */

+#if HAVE_MEMMOVE_IFUNC

+    IFUNC_IMPL (i, name, memmove,

+# if HAVE_MEMMOVE_Z13

+		IFUNC_IMPL_ADD (array, i, memmove,

+				dl_hwcap & HWCAP_S390_VX, MEMMOVE_Z13)

+# endif

+# if HAVE_MEMMOVE_C

+		IFUNC_IMPL_ADD (array, i, memmove, 1, MEMMOVE_C)

+# endif

+		)

+#endif /* HAVE_MEMMOVE_IFUNC  */

+

 #ifdef HAVE_S390_VX_ASM_SUPPORT

 # define IFUNC_VX_IMPL(FUNC)						\