|
 |
ce426f |
Backport of:
|
|
|
ce426f |
|
|
|
ce426f |
commit f06a4faf8a2b4d046eb40e94b47948cc47d79902
|
|
|
ce426f |
Author: Adhemerval Zanella <azanella@linux.vnet.ibm.com>
|
|
|
ce426f |
Date: Wed Dec 31 11:47:41 2014 -0500
|
|
|
ce426f |
|
|
|
ce426f |
powerpc: Optimized st{r,p}ncpy for POWER8/PPC64
|
|
|
ce426f |
|
|
|
ce426f |
This patch adds an optimized POWER8 st{r,p}ncpy using unaligned accesses.
|
|
|
ce426f |
It shows 10%-80% improvement over the optimized POWER7 one that uses
|
|
|
ce426f |
only aligned accesses, specially on unaligned inputs.
|
|
|
ce426f |
|
|
|
ce426f |
The algorithm first read and check 16 bytes (if inputs do not cross a 4K
|
|
|
ce426f |
page size). The it realign source to 16-bytes and issue a 16 bytes read
|
|
|
ce426f |
and compare loop to speedup null byte checks for large strings. Also,
|
|
|
ce426f |
different from POWER7 optimization, the null pad is done inline in the
|
|
|
ce426f |
implementation using possible unaligned accesses, instead of realying on
|
|
|
ce426f |
a memset call. Special case is added for page cross reads.
|
|
|
ce426f |
|
|
|
ce426f |
ChangeLog:
|
|
|
ce426f |
2015-01-13 Adhemerval Zanella <azanella@linux.vnet.ibm.com>
|
|
|
ce426f |
|
|
|
ce426f |
* sysdeps/powerpc/powerpc64/multiarch/Makefile [sysdep_routines]:
|
|
|
ce426f |
Add strncpy-power8 and stpncpy-power8 objects.
|
|
|
ce426f |
* sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c
|
|
|
ce426f |
(__libc_ifunc_impl_list): Add __strncpy_power8 and stpncpy_power8
|
|
|
ce426f |
implementations.
|
|
|
ce426f |
* sysdeps/powerpc/powerpc64/multiarch/stpncpy-power8.S: New file.
|
|
|
ce426f |
* sysdeps/powerpc/powerpc64/multiarch/stpncpy.c (__stpncpy): Add
|
|
|
ce426f |
__stpncpy_power8 implementation.
|
|
|
ce426f |
* sysdeps/powerpc/powerpc64/multiarch/strncpy-power8.S: New file.
|
|
|
ce426f |
* sysdeps/powerpc/powerpc64/multiarch/strncpy.c (strncpy): Add
|
|
|
ce426f |
__strncpy_power8 implementation.
|
|
|
ce426f |
* sysdeps/powerpc/powerpc64/power8/stpncpy.S: New file.
|
|
|
ce426f |
* sysdeps/powerpc/powerpc64/power8/strncpy.S: New file.
|
|
|
ce426f |
* NEWS: Update.
|
|
|
ce426f |
|
|
|
ce426f |
and its dependency:
|
|
|
ce426f |
|
|
|
ce426f |
commit f360f94a05570045be615649e9a411cefba2e210
|
|
|
ce426f |
Author: Vidya Ranganathan <vidya@linux.vnet.ibm.com>
|
|
|
ce426f |
Date: Mon May 5 19:10:45 2014 -0500
|
|
|
ce426f |
|
|
|
ce426f |
PowerPC: strncpy/stpncpy optimization for PPC64/POWER7
|
|
|
ce426f |
|
|
|
ce426f |
The optimization is achieved by following techniques:
|
|
|
ce426f |
> data alignment [gain from aligned memory access on read/write]
|
|
|
ce426f |
> POWER7 gains performance with loop unrolling/unwinding
|
|
|
ce426f |
[gain by reduction of branch penalty].
|
|
|
ce426f |
> zero padding done by calling optimized memset
|
|
|
ce426f |
|
|
|
ce426f |
diff --git a/sysdeps/powerpc/powerpc64/multiarch/Makefile b/sysdeps/powerpc/powerpc64/multiarch/Makefile
|
|
|
ce426f |
index 74ae710..ef39917 100644
|
|
|
ce426f |
--- a/sysdeps/powerpc/powerpc64/multiarch/Makefile
|
|
|
ce426f |
+++ b/sysdeps/powerpc/powerpc64/multiarch/Makefile
|
|
|
ce426f |
@@ -8,6 +8,8 @@ sysdep_routines += memcpy-power7 memcpy-a2 memcpy-power6 memcpy-cell \
|
|
|
ce426f |
stpcpy-power8 stpcpy-power7 stpcpy-ppc64 \
|
|
|
ce426f |
strcat-power8 strcat-power7 strcat-ppc64 \
|
|
|
ce426f |
strcpy-power8 strcpy-power7 strcpy-ppc64 \
|
|
|
ce426f |
+ stpncpy-power8 stpncpy-power7 stpncpy-ppc64 \
|
|
|
ce426f |
+ strncpy-power8 strncpy-power7 strncpy-ppc64
|
|
|
ce426f |
strncat-power7 \
|
|
|
ce426f |
rawmemchr-ppc64 strlen-power7 strlen-ppc64 strnlen-power7 \
|
|
|
ce426f |
strnlen-ppc64 strcasecmp-power7 strcasecmp_l-power7 \
|
|
|
ce426f |
diff --git a/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c b/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c
|
|
|
ce426f |
index 4e5bb17..23bf5dc 100644
|
|
|
ce426f |
--- a/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c
|
|
|
ce426f |
+++ b/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c
|
|
|
ce426f |
@@ -255,5 +255,27 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
|
|
|
ce426f |
IFUNC_IMPL_ADD (array, i, wcscpy, 1,
|
|
|
ce426f |
__wcscpy_ppc))
|
|
|
ce426f |
|
|
|
ce426f |
+ /* Support sysdeps/powerpc/powerpc64/multiarch/strncpy.c. */
|
|
|
ce426f |
+ IFUNC_IMPL (i, name, strncpy,
|
|
|
ce426f |
+ IFUNC_IMPL_ADD (array, i, strncpy,
|
|
|
ce426f |
+ hwcap2 & PPC_FEATURE2_ARCH_2_07,
|
|
|
ce426f |
+ __strncpy_power8)
|
|
|
ce426f |
+ IFUNC_IMPL_ADD (array, i, strncpy,
|
|
|
ce426f |
+ hwcap & PPC_FEATURE_HAS_VSX,
|
|
|
ce426f |
+ __strncpy_power7)
|
|
|
ce426f |
+ IFUNC_IMPL_ADD (array, i, strncpy, 1,
|
|
|
ce426f |
+ __strncpy_ppc))
|
|
|
ce426f |
+
|
|
|
ce426f |
+ /* Support sysdeps/powerpc/powerpc64/multiarch/stpncpy.c. */
|
|
|
ce426f |
+ IFUNC_IMPL (i, name, stpncpy,
|
|
|
ce426f |
+ IFUNC_IMPL_ADD (array, i, stpncpy,
|
|
|
ce426f |
+ hwcap2 & PPC_FEATURE2_ARCH_2_07,
|
|
|
ce426f |
+ __stpncpy_power8)
|
|
|
ce426f |
+ IFUNC_IMPL_ADD (array, i, stpncpy,
|
|
|
ce426f |
+ hwcap & PPC_FEATURE_HAS_VSX,
|
|
|
ce426f |
+ __stpncpy_power7)
|
|
|
ce426f |
+ IFUNC_IMPL_ADD (array, i, stpncpy, 1,
|
|
|
ce426f |
+ __stpncpy_ppc))
|
|
|
ce426f |
+
|
|
|
ce426f |
return i;
|
|
|
ce426f |
}
|
|
|
ce426f |
diff --git a/sysdeps/powerpc/powerpc64/multiarch/stpncpy-power7.S b/sysdeps/powerpc/powerpc64/multiarch/stpncpy-power7.S
|
|
|
ce426f |
new file mode 100644
|
|
|
ce426f |
index 0000000..e29674f
|
|
|
ce426f |
--- /dev/null
|
|
|
ce426f |
+++ b/sysdeps/powerpc/powerpc64/multiarch/stpncpy-power7.S
|
|
|
ce426f |
@@ -0,0 +1,44 @@
|
|
|
ce426f |
+/* Optimized stpncpy implementation for POWER7.
|
|
|
ce426f |
+ Copyright (C) 2014 Free Software Foundation, Inc.
|
|
|
ce426f |
+ This file is part of the GNU C Library.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ The GNU C Library is free software; you can redistribute it and/or
|
|
|
ce426f |
+ modify it under the terms of the GNU Lesser General Public
|
|
|
ce426f |
+ License as published by the Free Software Foundation; either
|
|
|
ce426f |
+ version 2.1 of the License, or (at your option) any later version.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ The GNU C Library is distributed in the hope that it will be useful,
|
|
|
ce426f |
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
ce426f |
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
ce426f |
+ Lesser General Public License for more details.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ You should have received a copy of the GNU Lesser General Public
|
|
|
ce426f |
+ License along with the GNU C Library; if not, see
|
|
|
ce426f |
+ <http://www.gnu.org/licenses/>. */
|
|
|
ce426f |
+
|
|
|
ce426f |
+#include <sysdep.h>
|
|
|
ce426f |
+
|
|
|
ce426f |
+#define USE_AS_STPNCPY
|
|
|
ce426f |
+
|
|
|
ce426f |
+#undef EALIGN
|
|
|
ce426f |
+#define EALIGN(name, alignt, words) \
|
|
|
ce426f |
+ .section ".text"; \
|
|
|
ce426f |
+ ENTRY_2(__stpncpy_power7) \
|
|
|
ce426f |
+ .align ALIGNARG(alignt); \
|
|
|
ce426f |
+ EALIGN_W_##words; \
|
|
|
ce426f |
+ BODY_LABEL(__stpncpy_power7): \
|
|
|
ce426f |
+ cfi_startproc; \
|
|
|
ce426f |
+ LOCALENTRY(__stpncpy_power7)
|
|
|
ce426f |
+
|
|
|
ce426f |
+#undef END
|
|
|
ce426f |
+#define END(name) \
|
|
|
ce426f |
+ cfi_endproc; \
|
|
|
ce426f |
+ TRACEBACK(__stpncpy_power7) \
|
|
|
ce426f |
+ END_2(__stpncpy_power7)
|
|
|
ce426f |
+
|
|
|
ce426f |
+#undef libc_hidden_builtin_def
|
|
|
ce426f |
+#define libc_hidden_builtin_def(name)
|
|
|
ce426f |
+
|
|
|
ce426f |
+#define MEMSET __memset_power7
|
|
|
ce426f |
+
|
|
|
ce426f |
+#include <sysdeps/powerpc/powerpc64/power7/stpncpy.S>
|
|
|
ce426f |
diff --git a/sysdeps/powerpc/powerpc64/multiarch/stpncpy-power8.S b/sysdeps/powerpc/powerpc64/multiarch/stpncpy-power8.S
|
|
|
ce426f |
new file mode 100644
|
|
|
ce426f |
index 0000000..d5d835d
|
|
|
ce426f |
--- /dev/null
|
|
|
ce426f |
+++ b/sysdeps/powerpc/powerpc64/multiarch/stpncpy-power8.S
|
|
|
ce426f |
@@ -0,0 +1,39 @@
|
|
|
ce426f |
+/* Optimized stpncpy implementation for POWER8.
|
|
|
ce426f |
+ Copyright (C) 2015 Free Software Foundation, Inc.
|
|
|
ce426f |
+ This file is part of the GNU C Library.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ The GNU C Library is free software; you can redistribute it and/or
|
|
|
ce426f |
+ modify it under the terms of the GNU Lesser General Public
|
|
|
ce426f |
+ License as published by the Free Software Foundation; either
|
|
|
ce426f |
+ version 2.1 of the License, or (at your option) any later version.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ The GNU C Library is distributed in the hope that it will be useful,
|
|
|
ce426f |
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
ce426f |
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
ce426f |
+ Lesser General Public License for more details.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ You should have received a copy of the GNU Lesser General Public
|
|
|
ce426f |
+ License along with the GNU C Library; if not, see
|
|
|
ce426f |
+ <http://www.gnu.org/licenses/>. */
|
|
|
ce426f |
+
|
|
|
ce426f |
+#include <sysdep.h>
|
|
|
ce426f |
+
|
|
|
ce426f |
+#define USE_AS_STPNCPY
|
|
|
ce426f |
+
|
|
|
ce426f |
+#undef EALIGN
|
|
|
ce426f |
+#define EALIGN(name, alignt, words) \
|
|
|
ce426f |
+ .section ".text"; \
|
|
|
ce426f |
+ ENTRY_2(__stpncpy_power8) \
|
|
|
ce426f |
+ .align ALIGNARG(alignt); \
|
|
|
ce426f |
+ EALIGN_W_##words; \
|
|
|
ce426f |
+ BODY_LABEL(__stpncpy_power8): \
|
|
|
ce426f |
+ cfi_startproc; \
|
|
|
ce426f |
+ LOCALENTRY(__stpncpy_power8)
|
|
|
ce426f |
+
|
|
|
ce426f |
+#undef END
|
|
|
ce426f |
+#define END(name) \
|
|
|
ce426f |
+ cfi_endproc; \
|
|
|
ce426f |
+ TRACEBACK(__stpncpy_power8) \
|
|
|
ce426f |
+ END_2(__stpncpy_power8)
|
|
|
ce426f |
+
|
|
|
ce426f |
+#include <sysdeps/powerpc/powerpc64/power8/stpncpy.S>
|
|
|
ce426f |
diff --git a/sysdeps/powerpc/powerpc64/multiarch/stpncpy-ppc64.c b/sysdeps/powerpc/powerpc64/multiarch/stpncpy-ppc64.c
|
|
|
ce426f |
new file mode 100644
|
|
|
ce426f |
index 0000000..74f47a7
|
|
|
ce426f |
--- /dev/null
|
|
|
ce426f |
+++ b/sysdeps/powerpc/powerpc64/multiarch/stpncpy-ppc64.c
|
|
|
ce426f |
@@ -0,0 +1,26 @@
|
|
|
ce426f |
+/* Default stpncpy implementation for PowerPC64.
|
|
|
ce426f |
+ Copyright (C) 2014 Free Software Foundation, Inc.
|
|
|
ce426f |
+ This file is part of the GNU C Library.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ The GNU C Library is free software; you can redistribute it and/or
|
|
|
ce426f |
+ modify it under the terms of the GNU Lesser General Public
|
|
|
ce426f |
+ License as published by the Free Software Foundation; either
|
|
|
ce426f |
+ version 2.1 of the License, or (at your option) any later version.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ The GNU C Library is distributed in the hope that it will be useful,
|
|
|
ce426f |
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
ce426f |
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
ce426f |
+ Lesser General Public License for more details.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ You should have received a copy of the GNU Lesser General Public
|
|
|
ce426f |
+ License along with the GNU C Library; if not, see
|
|
|
ce426f |
+ <http://www.gnu.org/licenses/>. */
|
|
|
ce426f |
+
|
|
|
ce426f |
+#define STPNCPY __stpncpy_ppc
|
|
|
ce426f |
+#ifdef SHARED
|
|
|
ce426f |
+#undef libc_hidden_def
|
|
|
ce426f |
+#define libc_hidden_def(name) \
|
|
|
ce426f |
+ __hidden_ver1 (__stpncpy_ppc, __GI___stpncpy, __stpncpy_ppc);
|
|
|
ce426f |
+#endif
|
|
|
ce426f |
+
|
|
|
ce426f |
+#include <string/stpncpy.c>
|
|
|
ce426f |
diff --git a/sysdeps/powerpc/powerpc64/multiarch/stpncpy.c b/sysdeps/powerpc/powerpc64/multiarch/stpncpy.c
|
|
|
ce426f |
new file mode 100644
|
|
|
ce426f |
index 0000000..3ee50e5
|
|
|
ce426f |
--- /dev/null
|
|
|
ce426f |
+++ b/sysdeps/powerpc/powerpc64/multiarch/stpncpy.c
|
|
|
ce426f |
@@ -0,0 +1,36 @@
|
|
|
ce426f |
+/* Multiple versions of stpncpy. PowerPC64 version.
|
|
|
ce426f |
+ Copyright (C) 2014 Free Software Foundation, Inc.
|
|
|
ce426f |
+ This file is part of the GNU C Library.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ The GNU C Library is free software; you can redistribute it and/or
|
|
|
ce426f |
+ modify it under the terms of the GNU Lesser General Public
|
|
|
ce426f |
+ License as published by the Free Software Foundation; either
|
|
|
ce426f |
+ version 2.1 of the License, or (at your option) any later version.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ The GNU C Library is distributed in the hope that it will be useful,
|
|
|
ce426f |
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
ce426f |
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
ce426f |
+ Lesser General Public License for more details.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ You should have received a copy of the GNU Lesser General Public
|
|
|
ce426f |
+ License along with the GNU C Library; if not, see
|
|
|
ce426f |
+ <http://www.gnu.org/licenses/>. */
|
|
|
ce426f |
+
|
|
|
ce426f |
+#ifndef NOT_IN_libc
|
|
|
ce426f |
+# include <string.h>
|
|
|
ce426f |
+# include <shlib-compat.h>
|
|
|
ce426f |
+# include "init-arch.h"
|
|
|
ce426f |
+
|
|
|
ce426f |
+extern __typeof (__stpncpy) __stpncpy_ppc attribute_hidden;
|
|
|
ce426f |
+extern __typeof (__stpncpy) __stpncpy_power7 attribute_hidden;
|
|
|
ce426f |
+extern __typeof (__stpncpy) __stpncpy_power8 attribute_hidden;
|
|
|
ce426f |
+
|
|
|
ce426f |
+libc_ifunc (__stpncpy,
|
|
|
ce426f |
+ (hwcap2 & PPC_FEATURE2_ARCH_2_07)
|
|
|
ce426f |
+ ? __stpncpy_power8 :
|
|
|
ce426f |
+ (hwcap & PPC_FEATURE_HAS_VSX)
|
|
|
ce426f |
+ ? __stpncpy_power7
|
|
|
ce426f |
+ : __stpncpy_ppc);
|
|
|
ce426f |
+
|
|
|
ce426f |
+weak_alias (__stpncpy, stpncpy)
|
|
|
ce426f |
+#endif
|
|
|
ce426f |
diff --git a/sysdeps/powerpc/powerpc64/multiarch/strncpy-power7.S b/sysdeps/powerpc/powerpc64/multiarch/strncpy-power7.S
|
|
|
ce426f |
new file mode 100644
|
|
|
ce426f |
index 0000000..be349f9
|
|
|
ce426f |
--- /dev/null
|
|
|
ce426f |
+++ b/sysdeps/powerpc/powerpc64/multiarch/strncpy-power7.S
|
|
|
ce426f |
@@ -0,0 +1,42 @@
|
|
|
ce426f |
+/* Optimized strncpy implementation for POWER7.
|
|
|
ce426f |
+ Copyright (C) 2014 Free Software Foundation, Inc.
|
|
|
ce426f |
+ This file is part of the GNU C Library.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ The GNU C Library is free software; you can redistribute it and/or
|
|
|
ce426f |
+ modify it under the terms of the GNU Lesser General Public
|
|
|
ce426f |
+ License as published by the Free Software Foundation; either
|
|
|
ce426f |
+ version 2.1 of the License, or (at your option) any later version.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ The GNU C Library is distributed in the hope that it will be useful,
|
|
|
ce426f |
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
ce426f |
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
ce426f |
+ Lesser General Public License for more details.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ You should have received a copy of the GNU Lesser General Public
|
|
|
ce426f |
+ License along with the GNU C Library; if not, see
|
|
|
ce426f |
+ <http://www.gnu.org/licenses/>. */
|
|
|
ce426f |
+
|
|
|
ce426f |
+#include <sysdep.h>
|
|
|
ce426f |
+
|
|
|
ce426f |
+#undef EALIGN
|
|
|
ce426f |
+#define EALIGN(name, alignt, words) \
|
|
|
ce426f |
+ .section ".text"; \
|
|
|
ce426f |
+ ENTRY_2(__strncpy_power7) \
|
|
|
ce426f |
+ .align ALIGNARG(alignt); \
|
|
|
ce426f |
+ EALIGN_W_##words; \
|
|
|
ce426f |
+ BODY_LABEL(__strncpy_power7): \
|
|
|
ce426f |
+ cfi_startproc; \
|
|
|
ce426f |
+ LOCALENTRY(__strncpy_power7)
|
|
|
ce426f |
+
|
|
|
ce426f |
+#undef END
|
|
|
ce426f |
+#define END(name) \
|
|
|
ce426f |
+ cfi_endproc; \
|
|
|
ce426f |
+ TRACEBACK(__strncpy_power7) \
|
|
|
ce426f |
+ END_2(__strncpy_power7)
|
|
|
ce426f |
+
|
|
|
ce426f |
+#undef libc_hidden_builtin_def
|
|
|
ce426f |
+#define libc_hidden_builtin_def(name)
|
|
|
ce426f |
+
|
|
|
ce426f |
+#define MEMSET __memset_power7
|
|
|
ce426f |
+
|
|
|
ce426f |
+#include <sysdeps/powerpc/powerpc64/power7/strncpy.S>
|
|
|
ce426f |
diff --git a/sysdeps/powerpc/powerpc64/multiarch/strncpy-power8.S b/sysdeps/powerpc/powerpc64/multiarch/strncpy-power8.S
|
|
|
ce426f |
new file mode 100644
|
|
|
ce426f |
index 0000000..ed906a4
|
|
|
ce426f |
--- /dev/null
|
|
|
ce426f |
+++ b/sysdeps/powerpc/powerpc64/multiarch/strncpy-power8.S
|
|
|
ce426f |
@@ -0,0 +1,40 @@
|
|
|
ce426f |
+/* Optimized strncpy implementation for POWER8.
|
|
|
ce426f |
+ Copyright (C) 2015 Free Software Foundation, Inc.
|
|
|
ce426f |
+ This file is part of the GNU C Library.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ The GNU C Library is free software; you can redistribute it and/or
|
|
|
ce426f |
+ modify it under the terms of the GNU Lesser General Public
|
|
|
ce426f |
+ License as published by the Free Software Foundation; either
|
|
|
ce426f |
+ version 2.1 of the License, or (at your option) any later version.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ The GNU C Library is distributed in the hope that it will be useful,
|
|
|
ce426f |
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
ce426f |
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
ce426f |
+ Lesser General Public License for more details.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ You should have received a copy of the GNU Lesser General Public
|
|
|
ce426f |
+ License along with the GNU C Library; if not, see
|
|
|
ce426f |
+ <http://www.gnu.org/licenses/>. */
|
|
|
ce426f |
+
|
|
|
ce426f |
+#include <sysdep.h>
|
|
|
ce426f |
+
|
|
|
ce426f |
+#undef EALIGN
|
|
|
ce426f |
+#define EALIGN(name, alignt, words) \
|
|
|
ce426f |
+ .section ".text"; \
|
|
|
ce426f |
+ ENTRY_2(__strncpy_power8) \
|
|
|
ce426f |
+ .align ALIGNARG(alignt); \
|
|
|
ce426f |
+ EALIGN_W_##words; \
|
|
|
ce426f |
+ BODY_LABEL(__strncpy_power8): \
|
|
|
ce426f |
+ cfi_startproc; \
|
|
|
ce426f |
+ LOCALENTRY(__strncpy_power8)
|
|
|
ce426f |
+
|
|
|
ce426f |
+#undef END
|
|
|
ce426f |
+#define END(name) \
|
|
|
ce426f |
+ cfi_endproc; \
|
|
|
ce426f |
+ TRACEBACK(__strncpy_power8) \
|
|
|
ce426f |
+ END_2(__strncpy_power8)
|
|
|
ce426f |
+
|
|
|
ce426f |
+#undef libc_hidden_builtin_def
|
|
|
ce426f |
+#define libc_hidden_builtin_def(name)
|
|
|
ce426f |
+
|
|
|
ce426f |
+#include <sysdeps/powerpc/powerpc64/power8/strncpy.S>
|
|
|
ce426f |
diff --git a/sysdeps/powerpc/powerpc64/multiarch/strncpy-ppc64.c b/sysdeps/powerpc/powerpc64/multiarch/strncpy-ppc64.c
|
|
|
ce426f |
new file mode 100644
|
|
|
ce426f |
index 0000000..e3111d2
|
|
|
ce426f |
--- /dev/null
|
|
|
ce426f |
+++ b/sysdeps/powerpc/powerpc64/multiarch/strncpy-ppc64.c
|
|
|
ce426f |
@@ -0,0 +1,33 @@
|
|
|
ce426f |
+/* Copyright (C) 2014 Free Software Foundation, Inc.
|
|
|
ce426f |
+ This file is part of the GNU C Library.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ The GNU C Library is free software; you can redistribute it and/or
|
|
|
ce426f |
+ modify it under the terms of the GNU Lesser General Public
|
|
|
ce426f |
+ License as published by the Free Software Foundation; either
|
|
|
ce426f |
+ version 2.1 of the License, or (at your option) any later version.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ The GNU C Library is distributed in the hope that it will be useful,
|
|
|
ce426f |
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
ce426f |
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
ce426f |
+ Lesser General Public License for more details.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ You should have received a copy of the GNU Lesser General Public
|
|
|
ce426f |
+ License along with the GNU C Library; if not, see
|
|
|
ce426f |
+ <http://www.gnu.org/licenses/>. */
|
|
|
ce426f |
+
|
|
|
ce426f |
+#include <string.h>
|
|
|
ce426f |
+
|
|
|
ce426f |
+#define STRNCPY __strncpy_ppc
|
|
|
ce426f |
+#undef weak_alias
|
|
|
ce426f |
+#define weak_alias(name, aliasname) \
|
|
|
ce426f |
+ extern __typeof (__strncpy_ppc) aliasname \
|
|
|
ce426f |
+ __attribute__ ((weak, alias ("__strncpy_ppc")));
|
|
|
ce426f |
+#if !defined(NOT_IN_libc) && defined(SHARED)
|
|
|
ce426f |
+# undef libc_hidden_builtin_def
|
|
|
ce426f |
+# define libc_hidden_builtin_def(name) \
|
|
|
ce426f |
+ __hidden_ver1(__strncpy_ppc, __GI_strncpy, __strncpy_ppc);
|
|
|
ce426f |
+#endif
|
|
|
ce426f |
+
|
|
|
ce426f |
+extern __typeof (strncpy) __strncpy_ppc attribute_hidden;
|
|
|
ce426f |
+
|
|
|
ce426f |
+#include <string/strncpy.c>
|
|
|
ce426f |
diff --git a/sysdeps/powerpc/powerpc64/multiarch/strncpy.c b/sysdeps/powerpc/powerpc64/multiarch/strncpy.c
|
|
|
ce426f |
new file mode 100644
|
|
|
ce426f |
index 0000000..19927bc
|
|
|
ce426f |
--- /dev/null
|
|
|
ce426f |
+++ b/sysdeps/powerpc/powerpc64/multiarch/strncpy.c
|
|
|
ce426f |
@@ -0,0 +1,38 @@
|
|
|
ce426f |
+/* Multiple versions of strncpy.
|
|
|
ce426f |
+ Copyright (C) 2014 Free Software Foundation, Inc.
|
|
|
ce426f |
+ This file is part of the GNU C Library.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ The GNU C Library is free software; you can redistribute it and/or
|
|
|
ce426f |
+ modify it under the terms of the GNU Lesser General Public
|
|
|
ce426f |
+ License as published by the Free Software Foundation; either
|
|
|
ce426f |
+ version 2.1 of the License, or (at your option) any later version.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ The GNU C Library is distributed in the hope that it will be useful,
|
|
|
ce426f |
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
ce426f |
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
ce426f |
+ Lesser General Public License for more details.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ You should have received a copy of the GNU Lesser General Public
|
|
|
ce426f |
+ License along with the GNU C Library; if not, see
|
|
|
ce426f |
+ <http://www.gnu.org/licenses/ >. */
|
|
|
ce426f |
+
|
|
|
ce426f |
+/* Define multiple versions only for definition in libc. */
|
|
|
ce426f |
+#ifndef NOT_IN_libc
|
|
|
ce426f |
+# include <string.h>
|
|
|
ce426f |
+# include <shlib-compat.h>
|
|
|
ce426f |
+# include "init-arch.h"
|
|
|
ce426f |
+
|
|
|
ce426f |
+extern __typeof (strncpy) __strncpy_ppc attribute_hidden;
|
|
|
ce426f |
+extern __typeof (strncpy) __strncpy_power7 attribute_hidden;
|
|
|
ce426f |
+extern __typeof (strncpy) __strncpy_power8 attribute_hidden;
|
|
|
ce426f |
+
|
|
|
ce426f |
+/* Avoid DWARF definition DIE on ifunc symbol so that GDB can handle
|
|
|
ce426f |
+ ifunc symbol properly. */
|
|
|
ce426f |
+libc_ifunc (strncpy,
|
|
|
ce426f |
+ (hwcap2 & PPC_FEATURE2_ARCH_2_07)
|
|
|
ce426f |
+ ? __strncpy_power8 :
|
|
|
ce426f |
+ (hwcap & PPC_FEATURE_HAS_VSX)
|
|
|
ce426f |
+ ? __strncpy_power7
|
|
|
ce426f |
+ : __strncpy_ppc);
|
|
|
ce426f |
+
|
|
|
ce426f |
+#endif
|
|
|
ce426f |
diff --git a/sysdeps/powerpc/powerpc64/power7/stpncpy.S b/sysdeps/powerpc/powerpc64/power7/stpncpy.S
|
|
|
ce426f |
new file mode 100644
|
|
|
ce426f |
index 0000000..a539093
|
|
|
ce426f |
--- /dev/null
|
|
|
ce426f |
+++ b/sysdeps/powerpc/powerpc64/power7/stpncpy.S
|
|
|
ce426f |
@@ -0,0 +1,24 @@
|
|
|
ce426f |
+/* Optimized stpncpy implementation for PowerPC64/POWER7.
|
|
|
ce426f |
+ Copyright (C) 2014 Free Software Foundation, Inc.
|
|
|
ce426f |
+ This file is part of the GNU C Library.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ The GNU C Library is free software; you can redistribute it and/or
|
|
|
ce426f |
+ modify it under the terms of the GNU Lesser General Public
|
|
|
ce426f |
+ License as published by the Free Software Foundation; either
|
|
|
ce426f |
+ version 2.1 of the License, or (at your option) any later version.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ The GNU C Library is distributed in the hope that it will be useful,
|
|
|
ce426f |
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
ce426f |
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
ce426f |
+ Lesser General Public License for more details.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ You should have received a copy of the GNU Lesser General Public
|
|
|
ce426f |
+ License along with the GNU C Library; if not, see
|
|
|
ce426f |
+ <http://www.gnu.org/licenses/>. */
|
|
|
ce426f |
+
|
|
|
ce426f |
+#define USE_AS_STPNCPY
|
|
|
ce426f |
+#include <sysdeps/powerpc/powerpc64/power7/strncpy.S>
|
|
|
ce426f |
+
|
|
|
ce426f |
+weak_alias (__stpncpy, stpncpy)
|
|
|
ce426f |
+libc_hidden_def (__stpncpy)
|
|
|
ce426f |
+libc_hidden_builtin_def (stpncpy)
|
|
|
ce426f |
diff --git a/sysdeps/powerpc/powerpc64/power7/strncpy.S b/sysdeps/powerpc/powerpc64/power7/strncpy.S
|
|
|
ce426f |
new file mode 100644
|
|
|
ce426f |
index 0000000..51860df
|
|
|
ce426f |
--- /dev/null
|
|
|
ce426f |
+++ b/sysdeps/powerpc/powerpc64/power7/strncpy.S
|
|
|
ce426f |
@@ -0,0 +1,338 @@
|
|
|
ce426f |
+/* Copyright (C) 2014 Free Software Foundation, Inc.
|
|
|
ce426f |
+ This file is part of the GNU C Library.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ The GNU C Library is free software; you can redistribute it and/or
|
|
|
ce426f |
+ modify it under the terms of the GNU Lesser General Public
|
|
|
ce426f |
+ License as published by the Free Software Foundation; either
|
|
|
ce426f |
+ version 2.1 of the License, or (at your option) any later version.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ The GNU C Library is distributed in the hope that it will be useful,
|
|
|
ce426f |
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
ce426f |
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
ce426f |
+ Lesser General Public License for more details.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ You should have received a copy of the GNU Lesser General Public
|
|
|
ce426f |
+ License along with the GNU C Library; if not, see
|
|
|
ce426f |
+ <http://www.gnu.org/licenses/>. */
|
|
|
ce426f |
+
|
|
|
ce426f |
+#include <sysdep.h>
|
|
|
ce426f |
+
|
|
|
ce426f |
+/* Implements the functions
|
|
|
ce426f |
+
|
|
|
ce426f |
+ char * [r3] strncpy (char *dst [r3], const char *src [r4], size_t n [r5])
|
|
|
ce426f |
+
|
|
|
ce426f |
+ AND
|
|
|
ce426f |
+
|
|
|
ce426f |
+ char * [r3] stpncpy (char *dst [r3], const char *src [r4], size_t n [r5])
|
|
|
ce426f |
+
|
|
|
ce426f |
+ The algorithm is as follows:
|
|
|
ce426f |
+ > if src and dest are 8 byte aligned, perform double word copy
|
|
|
ce426f |
+ else
|
|
|
ce426f |
+ > copy byte by byte on unaligned addresses.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ The aligned comparison are made using cmpb instructions. */
|
|
|
ce426f |
+
|
|
|
ce426f |
+/* The focus on optimization for performance improvements are as follows:
|
|
|
ce426f |
+ 1. data alignment [gain from aligned memory access on read/write]
|
|
|
ce426f |
+ 2. POWER7 gains performance with loop unrolling/unwinding
|
|
|
ce426f |
+ [gain by reduction of branch penalty].
|
|
|
ce426f |
+ 3. The final pad with null bytes is done by calling an optimized
|
|
|
ce426f |
+ memset. */
|
|
|
ce426f |
+
|
|
|
ce426f |
+#ifdef USE_AS_STPNCPY
|
|
|
ce426f |
+# define FUNC_NAME __stpncpy
|
|
|
ce426f |
+#else
|
|
|
ce426f |
+# define FUNC_NAME strncpy
|
|
|
ce426f |
+#endif
|
|
|
ce426f |
+
|
|
|
ce426f |
+#define FRAMESIZE (FRAME_MIN_SIZE+32)
|
|
|
ce426f |
+
|
|
|
ce426f |
+#ifndef MEMSET
|
|
|
ce426f |
+/* For builds with no IFUNC support, local calls should be made to internal
|
|
|
ce426f |
+ GLIBC symbol (created by libc_hidden_builtin_def). */
|
|
|
ce426f |
+# ifdef SHARED
|
|
|
ce426f |
+# define MEMSET __GI_memset
|
|
|
ce426f |
+# else
|
|
|
ce426f |
+# define MEMSET memset
|
|
|
ce426f |
+# endif
|
|
|
ce426f |
+#endif
|
|
|
ce426f |
+
|
|
|
ce426f |
+ .machine power7
|
|
|
ce426f |
+EALIGN(FUNC_NAME, 4, 0)
|
|
|
ce426f |
+ CALL_MCOUNT 3
|
|
|
ce426f |
+
|
|
|
ce426f |
+ mflr r0 /* load link register LR to r0 */
|
|
|
ce426f |
+ or r10, r3, r4 /* to verify source and destination */
|
|
|
ce426f |
+ rldicl. r8, r10, 0, 61 /* is double word aligned .. ? */
|
|
|
ce426f |
+
|
|
|
ce426f |
+ std r19, -8(r1) /* save callers register , r19 */
|
|
|
ce426f |
+ std r18, -16(r1) /* save callers register , r18 */
|
|
|
ce426f |
+ std r0, 16(r1) /* store the link register */
|
|
|
ce426f |
+ stdu r1, -FRAMESIZE(r1) /* create the stack frame */
|
|
|
ce426f |
+
|
|
|
ce426f |
+ mr r9, r3 /* save r3 into r9 for use */
|
|
|
ce426f |
+ mr r18, r3 /* save r3 for retCode of strncpy */
|
|
|
ce426f |
+ bne 0, L(byte_by_byte)
|
|
|
ce426f |
+
|
|
|
ce426f |
+
|
|
|
ce426f |
+ srdi r11, r5, 3 /* compute count for CTR ; count = n/8 */
|
|
|
ce426f |
+ cmpldi cr7, r11, 3 /* if count > 4 ; perform unrolling 4 times */
|
|
|
ce426f |
+ ble 7, L(update1)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ ld r10, 0(r4) /* load doubleWord from src */
|
|
|
ce426f |
+ cmpb r8, r10, r8 /* compare src with NULL ,we read just now */
|
|
|
ce426f |
+ cmpdi cr7, r8, 0 /* if cmpb returned NULL ; we continue */
|
|
|
ce426f |
+ bne cr7, L(update3)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ std r10, 0(r3) /* copy doubleword at offset=0 */
|
|
|
ce426f |
+ ld r10, 8(r4) /* load next doubleword from offset=8 */
|
|
|
ce426f |
+ cmpb r8, r10, r8 /* compare src with NULL , we read just now */
|
|
|
ce426f |
+ cmpdi cr7, r8, 0 /* if cmpb returned NULL ; we continue */
|
|
|
ce426f |
+ bne 7,L(HopBy8)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ addi r8, r11, -4
|
|
|
ce426f |
+ mr r7, r3
|
|
|
ce426f |
+ srdi r8, r8, 2
|
|
|
ce426f |
+ mr r6, r4
|
|
|
ce426f |
+ addi r8, r8, 1
|
|
|
ce426f |
+ li r12, 0
|
|
|
ce426f |
+ mtctr r8
|
|
|
ce426f |
+ b L(dwordCopy)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ .p2align 4
|
|
|
ce426f |
+L(dWordUnroll):
|
|
|
ce426f |
+ std r8, 16(r9)
|
|
|
ce426f |
+ ld r8, 24(r4) /* load dword,perform loop unrolling again */
|
|
|
ce426f |
+ cmpb r10, r8, r10
|
|
|
ce426f |
+ cmpdi cr7, r10, 0
|
|
|
ce426f |
+ bne cr7, L(HopBy24)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ std r8, 24(r7) /* copy dword at offset=24 */
|
|
|
ce426f |
+ addi r9, r9, 32
|
|
|
ce426f |
+ addi r4, r4, 32
|
|
|
ce426f |
+ bdz L(leftDwords) /* continue with loop on counter */
|
|
|
ce426f |
+
|
|
|
ce426f |
+ ld r3, 32(r6)
|
|
|
ce426f |
+ cmpb r8, r3, r10
|
|
|
ce426f |
+ cmpdi cr7, r8, 0
|
|
|
ce426f |
+ bne cr7, L(update2)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ std r3, 32(r7)
|
|
|
ce426f |
+ ld r10, 40(r6)
|
|
|
ce426f |
+ cmpb r8, r10, r8
|
|
|
ce426f |
+ cmpdi cr7, r8, 0
|
|
|
ce426f |
+ bne cr7, L(HopBy40)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ mr r6, r4 /* update values */
|
|
|
ce426f |
+ mr r7, r9
|
|
|
ce426f |
+ mr r11, r0
|
|
|
ce426f |
+ mr r5, r19
|
|
|
ce426f |
+
|
|
|
ce426f |
+L(dwordCopy):
|
|
|
ce426f |
+ std r10, 8(r9) /* copy dword at offset=8 */
|
|
|
ce426f |
+ addi r19, r5, -32
|
|
|
ce426f |
+ addi r0, r11, -4
|
|
|
ce426f |
+ ld r8, 16(r4)
|
|
|
ce426f |
+ cmpb r10, r8, r12
|
|
|
ce426f |
+ cmpdi cr7, r10, 0
|
|
|
ce426f |
+ beq cr7, L(dWordUnroll)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ addi r9, r9, 16 /* increment dst by 16 */
|
|
|
ce426f |
+ addi r4, r4, 16 /* increment src by 16 */
|
|
|
ce426f |
+ addi r5, r5, -16 /* decrement length 'n' by 16 */
|
|
|
ce426f |
+ addi r0, r11, -2 /* decrement loop counter */
|
|
|
ce426f |
+
|
|
|
ce426f |
+L(dWordUnrollOFF):
|
|
|
ce426f |
+ ld r10, 0(r4) /* load first dword */
|
|
|
ce426f |
+ li r8, 0 /* load mask */
|
|
|
ce426f |
+ cmpb r8, r10, r8
|
|
|
ce426f |
+ cmpdi cr7, r8, 0
|
|
|
ce426f |
+ bne cr7, L(byte_by_byte)
|
|
|
ce426f |
+ mtctr r0
|
|
|
ce426f |
+ li r7, 0
|
|
|
ce426f |
+ b L(CopyDword)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ .p2align 4
|
|
|
ce426f |
+L(loadDWordandCompare):
|
|
|
ce426f |
+ ld r10, 0(r4)
|
|
|
ce426f |
+ cmpb r8, r10, r7
|
|
|
ce426f |
+ cmpdi cr7, r8, 0
|
|
|
ce426f |
+ bne cr7, L(byte_by_byte)
|
|
|
ce426f |
+
|
|
|
ce426f |
+L(CopyDword):
|
|
|
ce426f |
+ addi r9, r9, 8
|
|
|
ce426f |
+ std r10, -8(r9)
|
|
|
ce426f |
+ addi r4, r4, 8
|
|
|
ce426f |
+ addi r5, r5, -8
|
|
|
ce426f |
+ bdnz L(loadDWordandCompare)
|
|
|
ce426f |
+
|
|
|
ce426f |
+L(byte_by_byte):
|
|
|
ce426f |
+ cmpldi cr7, r5, 3
|
|
|
ce426f |
+ ble cr7, L(verifyByte)
|
|
|
ce426f |
+ srdi r10, r5, 2
|
|
|
ce426f |
+ mr r19, r9
|
|
|
ce426f |
+ mtctr r10
|
|
|
ce426f |
+ b L(firstByteUnroll)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ .p2align 4
|
|
|
ce426f |
+L(bytes_unroll):
|
|
|
ce426f |
+ lbz r10, 1(r4) /* load byte from src */
|
|
|
ce426f |
+ cmpdi cr7, r10, 0 /* compare for NULL */
|
|
|
ce426f |
+ stb r10, 1(r19) /* store byte to dst */
|
|
|
ce426f |
+ beq cr7, L(updtDestComputeN2ndByte)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ addi r4, r4, 4 /* advance src */
|
|
|
ce426f |
+
|
|
|
ce426f |
+ lbz r10, -2(r4) /* perform loop unrolling for byte r/w */
|
|
|
ce426f |
+ cmpdi cr7, r10, 0
|
|
|
ce426f |
+ stb r10, 2(r19)
|
|
|
ce426f |
+ beq cr7, L(updtDestComputeN3rdByte)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ lbz r10, -1(r4) /* perform loop unrolling for byte r/w */
|
|
|
ce426f |
+ addi r19, r19, 4
|
|
|
ce426f |
+ cmpdi cr7, r10, 0
|
|
|
ce426f |
+ stb r10, -1(r19)
|
|
|
ce426f |
+ beq cr7, L(ComputeNByte)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ bdz L(update0)
|
|
|
ce426f |
+
|
|
|
ce426f |
+L(firstByteUnroll):
|
|
|
ce426f |
+ lbz r10, 0(r4) /* perform loop unrolling for byte r/w */
|
|
|
ce426f |
+ cmpdi cr7, 10, 0
|
|
|
ce426f |
+ stb r10, 0(r19)
|
|
|
ce426f |
+ bne cr7, L(bytes_unroll)
|
|
|
ce426f |
+ addi r19, r19, 1
|
|
|
ce426f |
+
|
|
|
ce426f |
+L(ComputeNByte):
|
|
|
ce426f |
+ subf r9, r19, r9 /* compute 'n'n bytes to fill */
|
|
|
ce426f |
+ add r8, r9, r5
|
|
|
ce426f |
+
|
|
|
ce426f |
+L(zeroFill):
|
|
|
ce426f |
+ cmpdi cr7, r8, 0 /* compare if length is zero */
|
|
|
ce426f |
+ beq cr7, L(update3return)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ mr r3, r19 /* fill buffer with */
|
|
|
ce426f |
+ li r4, 0 /* zero fill buffer */
|
|
|
ce426f |
+ mr r5, r8 /* how many bytes to fill buffer with */
|
|
|
ce426f |
+ bl MEMSET /* call optimized memset */
|
|
|
ce426f |
+ nop
|
|
|
ce426f |
+
|
|
|
ce426f |
+L(update3return):
|
|
|
ce426f |
+#ifdef USE_AS_STPNCPY
|
|
|
ce426f |
+ addi r3, r19, -1 /* update return value */
|
|
|
ce426f |
+#endif
|
|
|
ce426f |
+
|
|
|
ce426f |
+L(hop2return):
|
|
|
ce426f |
+#ifndef USE_AS_STPNCPY
|
|
|
ce426f |
+ mr r3, r18 /* set return value */
|
|
|
ce426f |
+#endif
|
|
|
ce426f |
+ addi r1, r1, FRAMESIZE /* restore stack pointer */
|
|
|
ce426f |
+ ld r0, 16(r1) /* read the saved link register */
|
|
|
ce426f |
+ ld r18, -16(r1) /* restore callers save register, r18 */
|
|
|
ce426f |
+ ld r19, -8(r1) /* restore callers save register, r19 */
|
|
|
ce426f |
+ mtlr r0 /* branch to link register */
|
|
|
ce426f |
+ blr /* return */
|
|
|
ce426f |
+
|
|
|
ce426f |
+ .p2align 4
|
|
|
ce426f |
+L(update0):
|
|
|
ce426f |
+ mr r9, r19
|
|
|
ce426f |
+
|
|
|
ce426f |
+ .p2align 4
|
|
|
ce426f |
+L(verifyByte):
|
|
|
ce426f |
+ rldicl. r8, r5, 0, 62
|
|
|
ce426f |
+#ifdef USE_AS_STPNCPY
|
|
|
ce426f |
+ mr r3, r9
|
|
|
ce426f |
+#endif
|
|
|
ce426f |
+ beq cr0, L(hop2return)
|
|
|
ce426f |
+ mtctr r8
|
|
|
ce426f |
+ addi r4, r4, -1
|
|
|
ce426f |
+ mr r19, r9
|
|
|
ce426f |
+ b L(oneBYone)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ .p2align 4
|
|
|
ce426f |
+L(proceed):
|
|
|
ce426f |
+ bdz L(done)
|
|
|
ce426f |
+
|
|
|
ce426f |
+L(oneBYone):
|
|
|
ce426f |
+ lbzu r10, 1(r4) /* copy byte */
|
|
|
ce426f |
+ addi r19, r19, 1
|
|
|
ce426f |
+ addi r8, r8, -1
|
|
|
ce426f |
+ cmpdi cr7, r10, 0
|
|
|
ce426f |
+ stb r10, -1(r19)
|
|
|
ce426f |
+ bne cr7, L(proceed)
|
|
|
ce426f |
+ b L(zeroFill)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ .p2align 4
|
|
|
ce426f |
+L(done):
|
|
|
ce426f |
+ addi r1, r1, FRAMESIZE /* restore stack pointer */
|
|
|
ce426f |
+#ifdef USE_AS_STPNCPY
|
|
|
ce426f |
+ mr r3, r19 /* set the return value */
|
|
|
ce426f |
+#else
|
|
|
ce426f |
+ mr r3, r18 /* set the return value */
|
|
|
ce426f |
+#endif
|
|
|
ce426f |
+ ld r0, 16(r1) /* read the saved link register */
|
|
|
ce426f |
+ ld r18, -16(r1) /* restore callers save register, r18 */
|
|
|
ce426f |
+ ld r19, -8(r1) /* restore callers save register, r19 */
|
|
|
ce426f |
+ mtlr r0 /* branch to link register */
|
|
|
ce426f |
+ blr /* return */
|
|
|
ce426f |
+
|
|
|
ce426f |
+L(update1):
|
|
|
ce426f |
+ mr r0, r11
|
|
|
ce426f |
+ mr r19, r5
|
|
|
ce426f |
+
|
|
|
ce426f |
+ .p2align 4
|
|
|
ce426f |
+L(leftDwords):
|
|
|
ce426f |
+ cmpdi cr7, r0, 0
|
|
|
ce426f |
+ mr r5, r19
|
|
|
ce426f |
+ bne cr7, L(dWordUnrollOFF)
|
|
|
ce426f |
+ b L(byte_by_byte)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ .p2align 4
|
|
|
ce426f |
+L(updtDestComputeN2ndByte):
|
|
|
ce426f |
+ addi r19, r19, 2 /* update dst by 2 */
|
|
|
ce426f |
+ subf r9, r19, r9 /* compute distance covered */
|
|
|
ce426f |
+ add r8, r9, r5
|
|
|
ce426f |
+ b L(zeroFill)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ .p2align 4
|
|
|
ce426f |
+L(updtDestComputeN3rdByte):
|
|
|
ce426f |
+ addi r19, r19, 3 /* update dst by 3 */
|
|
|
ce426f |
+ subf r9, r19, r9 /* compute distance covered */
|
|
|
ce426f |
+ add r8, r9, r5
|
|
|
ce426f |
+ b L(zeroFill)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ .p2align 4
|
|
|
ce426f |
+L(HopBy24):
|
|
|
ce426f |
+ addi r9, r9, 24 /* increment dst by 24 */
|
|
|
ce426f |
+ addi r4, r4, 24 /* increment src by 24 */
|
|
|
ce426f |
+ addi r5, r5, -24 /* decrement length 'n' by 24 */
|
|
|
ce426f |
+ addi r0, r11, -3 /* decrement loop counter */
|
|
|
ce426f |
+ b L(dWordUnrollOFF)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ .p2align 4
|
|
|
ce426f |
+L(update2):
|
|
|
ce426f |
+ mr r5, r19
|
|
|
ce426f |
+ b L(dWordUnrollOFF)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ .p2align 4
|
|
|
ce426f |
+L(HopBy40):
|
|
|
ce426f |
+ addi r9, r7, 40 /* increment dst by 40 */
|
|
|
ce426f |
+ addi r4, r6, 40 /* increment src by 40 */
|
|
|
ce426f |
+ addi r5, r5, -40 /* decrement length 'n' by 40 */
|
|
|
ce426f |
+ addi r0, r11, -5 /* decrement loop counter */
|
|
|
ce426f |
+ b L(dWordUnrollOFF)
|
|
|
ce426f |
+
|
|
|
ce426f |
+L(update3):
|
|
|
ce426f |
+ mr r0, r11
|
|
|
ce426f |
+ b L(dWordUnrollOFF)
|
|
|
ce426f |
+
|
|
|
ce426f |
+L(HopBy8):
|
|
|
ce426f |
+ addi r9, r3, 8 /* increment dst by 8 */
|
|
|
ce426f |
+ addi r4, r4, 8 /* increment src by 8 */
|
|
|
ce426f |
+ addi r5, r5, -8 /* decrement length 'n' by 8 */
|
|
|
ce426f |
+ addi r0, r11, -1 /* decrement loop counter */
|
|
|
ce426f |
+ b L(dWordUnrollOFF)
|
|
|
ce426f |
+END(FUNC_NAME)
|
|
|
ce426f |
+#ifndef USE_AS_STPNCPY
|
|
|
ce426f |
+libc_hidden_builtin_def (strncpy)
|
|
|
ce426f |
+#endif
|
|
|
ce426f |
diff --git a/sysdeps/powerpc/powerpc64/power8/stpncpy.S b/sysdeps/powerpc/powerpc64/power8/stpncpy.S
|
|
|
ce426f |
new file mode 100644
|
|
|
ce426f |
index 0000000..76a1466
|
|
|
ce426f |
--- /dev/null
|
|
|
ce426f |
+++ b/sysdeps/powerpc/powerpc64/power8/stpncpy.S
|
|
|
ce426f |
@@ -0,0 +1,20 @@
|
|
|
ce426f |
+/* Optimized stpncpy implementation for PowerPC64/POWER8.
|
|
|
ce426f |
+ Copyright (C) 2015 Free Software Foundation, Inc.
|
|
|
ce426f |
+ This file is part of the GNU C Library.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ The GNU C Library is free software; you can redistribute it and/or
|
|
|
ce426f |
+ modify it under the terms of the GNU Lesser General Public
|
|
|
ce426f |
+ License as published by the Free Software Foundation; either
|
|
|
ce426f |
+ version 2.1 of the License, or (at your option) any later version.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ The GNU C Library is distributed in the hope that it will be useful,
|
|
|
ce426f |
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
ce426f |
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
ce426f |
+ Lesser General Public License for more details.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ You should have received a copy of the GNU Lesser General Public
|
|
|
ce426f |
+ License along with the GNU C Library; if not, see
|
|
|
ce426f |
+ <http://www.gnu.org/licenses/>. */
|
|
|
ce426f |
+
|
|
|
ce426f |
+#define USE_AS_STPNCPY
|
|
|
ce426f |
+#include <sysdeps/powerpc/powerpc64/power8/strncpy.S>
|
|
|
ce426f |
diff --git a/sysdeps/powerpc/powerpc64/power8/strncpy.S b/sysdeps/powerpc/powerpc64/power8/strncpy.S
|
|
|
ce426f |
new file mode 100644
|
|
|
ce426f |
index 0000000..5fda953
|
|
|
ce426f |
--- /dev/null
|
|
|
ce426f |
+++ b/sysdeps/powerpc/powerpc64/power8/strncpy.S
|
|
|
ce426f |
@@ -0,0 +1,424 @@
|
|
|
ce426f |
+/* Optimized strncpy/stpncpy implementation for PowerPC64/POWER8.
|
|
|
ce426f |
+ Copyright (C) 2015 Free Software Foundation, Inc.
|
|
|
ce426f |
+ This file is part of the GNU C Library.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ The GNU C Library is free software; you can redistribute it and/or
|
|
|
ce426f |
+ modify it under the terms of the GNU Lesser General Public
|
|
|
ce426f |
+ License as published by the Free Software Foundation; either
|
|
|
ce426f |
+ version 2.1 of the License, or (at your option) any later version.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ The GNU C Library is distributed in the hope that it will be useful,
|
|
|
ce426f |
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
ce426f |
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
ce426f |
+ Lesser General Public License for more details.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ You should have received a copy of the GNU Lesser General Public
|
|
|
ce426f |
+ License along with the GNU C Library; if not, see
|
|
|
ce426f |
+ <http://www.gnu.org/licenses/>. */
|
|
|
ce426f |
+
|
|
|
ce426f |
+#include <sysdep.h>
|
|
|
ce426f |
+
|
|
|
ce426f |
+#ifdef USE_AS_STPNCPY
|
|
|
ce426f |
+# define FUNC_NAME __stpncpy
|
|
|
ce426f |
+#else
|
|
|
ce426f |
+# define FUNC_NAME strncpy
|
|
|
ce426f |
+#endif
|
|
|
ce426f |
+
|
|
|
ce426f |
+/* Implements the function
|
|
|
ce426f |
+
|
|
|
ce426f |
+ char * [r3] strncpy (char *dest [r3], const char *src [r4], size_t n [r5])
|
|
|
ce426f |
+
|
|
|
ce426f |
+ or
|
|
|
ce426f |
+
|
|
|
ce426f |
+ char * [r3] stpncpy (char *dest [r3], const char *src [r4], size_t n [r5])
|
|
|
ce426f |
+
|
|
|
ce426f |
+ if USE_AS_STPCPY is defined.
|
|
|
ce426f |
+
|
|
|
ce426f |
+ The implementation uses unaligned doubleword access to avoid specialized
|
|
|
ce426f |
+ code paths depending of data alignment. Although recent powerpc64 uses
|
|
|
ce426f |
+ 64K as default, the page cross handling assumes minimum page size of
|
|
|
ce426f |
+ 4k. */
|
|
|
ce426f |
+
|
|
|
ce426f |
+ .machine power7
|
|
|
ce426f |
+EALIGN (FUNC_NAME, 4, 0)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ /* Check if the [src]+15 will cross a 4K page by checking if the bit
|
|
|
ce426f |
+ indicating the page size changes. Basically:
|
|
|
ce426f |
+
|
|
|
ce426f |
+ uint64_t srcin = (uint64_t)src;
|
|
|
ce426f |
+ uint64_t ob = srcin & 4096UL;
|
|
|
ce426f |
+ uint64_t nb = (srcin+15UL) & 4096UL;
|
|
|
ce426f |
+ if (ob ^ nb)
|
|
|
ce426f |
+ goto pagecross; */
|
|
|
ce426f |
+
|
|
|
ce426f |
+ addi r10,r4,16
|
|
|
ce426f |
+ rlwinm r9,r4,0,19,19
|
|
|
ce426f |
+
|
|
|
ce426f |
+ /* Since it is a leaf function, save some non-volatile registers on the
|
|
|
ce426f |
+ protected/red zone. */
|
|
|
ce426f |
+ std r26,-48(r1)
|
|
|
ce426f |
+ std r27,-40(r1)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ rlwinm r8,r10,0,19,19
|
|
|
ce426f |
+
|
|
|
ce426f |
+ std r28,-32(r1)
|
|
|
ce426f |
+ std r29,-24(r1)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ cmpld r7,r9,r8
|
|
|
ce426f |
+
|
|
|
ce426f |
+ std r30,-16(r1)
|
|
|
ce426f |
+ std r31,-8(r1)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ beq cr7,L(unaligned_lt_16)
|
|
|
ce426f |
+ rldicl r9,r4,0,61
|
|
|
ce426f |
+ subfic r8,r9,8
|
|
|
ce426f |
+ cmpld cr7,r5,r8
|
|
|
ce426f |
+ bgt cr7,L(pagecross)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ /* At this points there is 1 to 15 bytes to check and write. Since it could
|
|
|
ce426f |
+ be either from first unaligned 16 bytes access or from bulk copy, the code
|
|
|
ce426f |
+ uses an unrolled byte read/write instead of trying to analyze the cmpb
|
|
|
ce426f |
+ results. */
|
|
|
ce426f |
+L(short_path):
|
|
|
ce426f |
+ mr r9,r3
|
|
|
ce426f |
+L(short_path_1):
|
|
|
ce426f |
+ cmpdi cr7,r5,0
|
|
|
ce426f |
+ beq cr7,L(short_path_loop_end_1)
|
|
|
ce426f |
+L(short_path_2):
|
|
|
ce426f |
+ lbz r10,0(r4)
|
|
|
ce426f |
+ cmpdi cr7,r10,0
|
|
|
ce426f |
+ stb r10,0(r9)
|
|
|
ce426f |
+ beq cr7,L(zero_pad_start_1)
|
|
|
ce426f |
+ cmpdi cr0,r5,1
|
|
|
ce426f |
+ addi r8,r9,1
|
|
|
ce426f |
+ addi r6,r5,-1
|
|
|
ce426f |
+ beq cr0,L(short_path_loop_end_0)
|
|
|
ce426f |
+ lbz r10,1(r4)
|
|
|
ce426f |
+ cmpdi cr7,r10,0
|
|
|
ce426f |
+ stb r10,1(r9)
|
|
|
ce426f |
+ beq cr7,L(zero_pad_start_prepare_1)
|
|
|
ce426f |
+ addi r10,r5,-3
|
|
|
ce426f |
+ b L(short_path_loop_1)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ .align 4
|
|
|
ce426f |
+L(short_path_loop):
|
|
|
ce426f |
+ lbz r8,0(r4)
|
|
|
ce426f |
+ addi r7,r10,-2
|
|
|
ce426f |
+ cmpdi cr5,r8,0
|
|
|
ce426f |
+ stb r8,0(r9)
|
|
|
ce426f |
+ beq cr5,L(zero_pad_start_1)
|
|
|
ce426f |
+ beq r7,L(short_path_loop_end_0)
|
|
|
ce426f |
+ lbz r8,1(r4)
|
|
|
ce426f |
+ cmpdi cr7,r8,0
|
|
|
ce426f |
+ stb r8,1(r9)
|
|
|
ce426f |
+ beq cr7,L(zero_pad_start)
|
|
|
ce426f |
+ mr r10,r7
|
|
|
ce426f |
+L(short_path_loop_1):
|
|
|
ce426f |
+ addic. r5,r5,-2
|
|
|
ce426f |
+ addi r9,r9,2
|
|
|
ce426f |
+ cmpdi cr7,r10,0
|
|
|
ce426f |
+ addi r4,r4,2
|
|
|
ce426f |
+ addi r6,r9,1
|
|
|
ce426f |
+ bne cr0,L(short_path_loop)
|
|
|
ce426f |
+#ifdef USE_AS_STPNCPY
|
|
|
ce426f |
+ mr r3,r9
|
|
|
ce426f |
+ b L(short_path_loop_end)
|
|
|
ce426f |
+#endif
|
|
|
ce426f |
+
|
|
|
ce426f |
+L(short_path_loop_end_0):
|
|
|
ce426f |
+#ifdef USE_AS_STPNCPY
|
|
|
ce426f |
+ addi r3,r9,1
|
|
|
ce426f |
+ b L(short_path_loop_end)
|
|
|
ce426f |
+#endif
|
|
|
ce426f |
+L(short_path_loop_end_1):
|
|
|
ce426f |
+#ifdef USE_AS_STPNCPY
|
|
|
ce426f |
+ mr r3,r9
|
|
|
ce426f |
+#endif
|
|
|
ce426f |
+L(short_path_loop_end):
|
|
|
ce426f |
+ /* Restore non-volatile registers. */
|
|
|
ce426f |
+ ld r26,-48(r1)
|
|
|
ce426f |
+ ld r27,-40(r1)
|
|
|
ce426f |
+ ld r28,-32(r1)
|
|
|
ce426f |
+ ld r29,-24(r1)
|
|
|
ce426f |
+ ld r30,-16(r1)
|
|
|
ce426f |
+ ld r31,-8(r1)
|
|
|
ce426f |
+ blr
|
|
|
ce426f |
+
|
|
|
ce426f |
+ /* This code pads the remainder dest with NULL bytes. The algorithm
|
|
|
ce426f |
+ calculate the remanining size and issues a doubleword unrolled
|
|
|
ce426f |
+ loops followed by a byte a byte set. */
|
|
|
ce426f |
+ .align 4
|
|
|
ce426f |
+L(zero_pad_start):
|
|
|
ce426f |
+ mr r5,r10
|
|
|
ce426f |
+ mr r9,r6
|
|
|
ce426f |
+L(zero_pad_start_1):
|
|
|
ce426f |
+ srdi. r8,r5,r3
|
|
|
ce426f |
+ mr r10,r9
|
|
|
ce426f |
+#ifdef USE_AS_STPNCPY
|
|
|
ce426f |
+ mr r3,r9
|
|
|
ce426f |
+#endif
|
|
|
ce426f |
+ beq- cr0,L(zero_pad_loop_b_start)
|
|
|
ce426f |
+ cmpldi cr7,r8,1
|
|
|
ce426f |
+ li cr7,0
|
|
|
ce426f |
+ std r7,0(r9)
|
|
|
ce426f |
+ beq cr7,L(zero_pad_loop_b_prepare)
|
|
|
ce426f |
+ addic. r8,r8,-2
|
|
|
ce426f |
+ addi r10,r9,r16
|
|
|
ce426f |
+ std r7,8(r9)
|
|
|
ce426f |
+ beq cr0,L(zero_pad_loop_dw_2)
|
|
|
ce426f |
+ std r7,16(r9)
|
|
|
ce426f |
+ li r9,0
|
|
|
ce426f |
+ b L(zero_pad_loop_dw_1)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ .align 4
|
|
|
ce426f |
+L(zero_pad_loop_dw):
|
|
|
ce426f |
+ addi r10,r10,16
|
|
|
ce426f |
+ std r9,-8(r10)
|
|
|
ce426f |
+ beq cr0,L(zero_pad_loop_dw_2)
|
|
|
ce426f |
+ std r9,0(r10)
|
|
|
ce426f |
+L(zero_pad_loop_dw_1):
|
|
|
ce426f |
+ cmpldi cr7,r8,1
|
|
|
ce426f |
+ std r9,0(r10)
|
|
|
ce426f |
+ addic. r8,r8,-2
|
|
|
ce426f |
+ bne cr7,L(zero_pad_loop_dw)
|
|
|
ce426f |
+ addi r10,r10,8
|
|
|
ce426f |
+L(zero_pad_loop_dw_2):
|
|
|
ce426f |
+ rldicl r5,r5,0,61
|
|
|
ce426f |
+L(zero_pad_loop_b_start):
|
|
|
ce426f |
+ cmpdi cr7,r5,0
|
|
|
ce426f |
+ addi r5,r5,-1
|
|
|
ce426f |
+ addi r9,r10,-1
|
|
|
ce426f |
+ add r10,r10,5
|
|
|
ce426f |
+ subf r10,r9,r10
|
|
|
ce426f |
+ li r8,0
|
|
|
ce426f |
+ beq- cr7,L(short_path_loop_end)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ /* Write remaining 1-8 bytes. */
|
|
|
ce426f |
+ .align 4
|
|
|
ce426f |
+ addi r9,r9,1
|
|
|
ce426f |
+ mtocrf 0x1,r10
|
|
|
ce426f |
+ bf 29,4f
|
|
|
ce426f |
+ stw r8,0(r9)
|
|
|
ce426f |
+ addi r9,r9,4
|
|
|
ce426f |
+
|
|
|
ce426f |
+ .align 4
|
|
|
ce426f |
+4: bf 30,2f
|
|
|
ce426f |
+ sth r8,0(r9)
|
|
|
ce426f |
+ addi r9,r9,2
|
|
|
ce426f |
+
|
|
|
ce426f |
+ .align 4
|
|
|
ce426f |
+2: bf 31,1f
|
|
|
ce426f |
+ stb r8,0(r9)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ /* Restore non-volatile registers. */
|
|
|
ce426f |
+1: ld r26,-48(r1)
|
|
|
ce426f |
+ ld r27,-40(r1)
|
|
|
ce426f |
+ ld r28,-32(r1)
|
|
|
ce426f |
+ ld r29,-24(r1)
|
|
|
ce426f |
+ ld r30,-16(r1)
|
|
|
ce426f |
+ ld r31,-8(r1)
|
|
|
ce426f |
+ blr
|
|
|
ce426f |
+
|
|
|
ce426f |
+ /* The common case where [src]+16 will not cross a 4K page boundary.
|
|
|
ce426f |
+ In this case the code fast check the first 16 bytes by using doubleword
|
|
|
ce426f |
+ read/compares and update destiny if neither total size or null byte
|
|
|
ce426f |
+ is found in destiny. */
|
|
|
ce426f |
+ .align 4
|
|
|
ce426f |
+L(unaligned_lt_16):
|
|
|
ce426f |
+ cmpldi cr7,r5,7
|
|
|
ce426f |
+ ble cr7,L(short_path)
|
|
|
ce426f |
+ ld r7,0(r4)
|
|
|
ce426f |
+ li r8,0
|
|
|
ce426f |
+ cmpb r8,r7,r8
|
|
|
ce426f |
+ cmpdi cr7,r8,0
|
|
|
ce426f |
+ bne cr7,L(short_path_prepare_2)
|
|
|
ce426f |
+ addi r6,r5,-8
|
|
|
ce426f |
+ std r7,0(r3)
|
|
|
ce426f |
+ addi r9,r3,r8
|
|
|
ce426f |
+ cmpldi cr7,r6,7
|
|
|
ce426f |
+ addi r7,r4,8
|
|
|
ce426f |
+ ble cr7,L(short_path_prepare_1_1)
|
|
|
ce426f |
+ ld r4,8(r4)
|
|
|
ce426f |
+ cmpb r8,r4,r8
|
|
|
ce426f |
+ cmpdi cr7,r8,0
|
|
|
ce426f |
+ bne cr7,L(short_path_prepare_2_1)
|
|
|
ce426f |
+ std r4,8(r3)
|
|
|
ce426f |
+ addi r29,r3,16
|
|
|
ce426f |
+ addi r5,r5,-16
|
|
|
ce426f |
+ /* Neither the null byte was found or total length was reached,
|
|
|
ce426f |
+ align to 16 bytes and issue a bulk copy/compare. */
|
|
|
ce426f |
+ b L(align_to_16b)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ /* In the case of 4k page boundary cross, the algorithm first align
|
|
|
ce426f |
+ the address to a doubleword, calculate a mask based on alignment
|
|
|
ce426f |
+ to ignore the bytes and continue using doubleword. */
|
|
|
ce426f |
+ .align 4
|
|
|
ce426f |
+L(pagecross):
|
|
|
ce426f |
+ rldicr r11,r4,0,59 /* Align the address to 8 bytes boundary. */
|
|
|
ce426f |
+ li r6,-1 /* MASK = 0xffffffffffffffffUL. */
|
|
|
ce426f |
+ sldi r9,r9,3 /* Calculate padding. */
|
|
|
ce426f |
+ ld r7,0(r11) /* Load doubleword from memory. */
|
|
|
ce426f |
+#ifdef __LITTLE_ENDIAN__
|
|
|
ce426f |
+ sld r9,r6,r9 /* MASK = MASK << padding. */
|
|
|
ce426f |
+#else
|
|
|
ce426f |
+ srd r9,r6,r9 /* MASK = MASK >> padding. */
|
|
|
ce426f |
+#endif
|
|
|
ce426f |
+ orc r9,r7,r9 /* Mask bits that are not part of the
|
|
|
ce426f |
+ string. */
|
|
|
ce426f |
+ li cr7,0
|
|
|
ce426f |
+ cmpb r9,r9,r7 /* Check for null bytes in DWORD1. */
|
|
|
ce426f |
+ cmpdi cr7,r9,0
|
|
|
ce426f |
+ bne cr7,L(short_path_prepare_2)
|
|
|
ce426f |
+ subf r8,r8,r5 /* Adjust total length. */
|
|
|
ce426f |
+ cmpldi cr7,r8,8 /* Check if length was reached. */
|
|
|
ce426f |
+ ble cr7,L(short_path_prepare_2)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ /* For next checks we have aligned address, so we check for more
|
|
|
ce426f |
+ three doublewords to make sure we can read 16 unaligned bytes
|
|
|
ce426f |
+ to start the bulk copy with 16 aligned addresses. */
|
|
|
ce426f |
+ ld cr7,8(r11)
|
|
|
ce426f |
+ cmpb r9,r7,r9
|
|
|
ce426f |
+ cmpdi cr7,r9,0
|
|
|
ce426f |
+ bne cr7,L(short_path_prepare_2)
|
|
|
ce426f |
+ addi cr7,r8,-8
|
|
|
ce426f |
+ cmpldi cr7,r7,8
|
|
|
ce426f |
+ ble cr7,L(short_path_prepare_2)
|
|
|
ce426f |
+ ld cr7,16(r11)
|
|
|
ce426f |
+ cmpb r9,r7,r9
|
|
|
ce426f |
+ cmpdi cr7,r9,0
|
|
|
ce426f |
+ bne cr7,L(short_path_prepare_2)
|
|
|
ce426f |
+ addi r8,r8,-16
|
|
|
ce426f |
+ cmpldi r7,r8,8
|
|
|
ce426f |
+ ble cr7,L(short_path_prepare_2)
|
|
|
ce426f |
+ ld r8,24(r11)
|
|
|
ce426f |
+ cmpb r9,r8,r9
|
|
|
ce426f |
+ cmpdi r7,r9,0
|
|
|
ce426f |
+ bne cr7,L(short_path_prepare_2)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ /* No null byte found in the 32 bytes readed and length not reached,
|
|
|
ce426f |
+ read source again using unaligned loads and store them. */
|
|
|
ce426f |
+ ld r9,0(r4)
|
|
|
ce426f |
+ addi r29,r3,16
|
|
|
ce426f |
+ addi r5,r5,-16
|
|
|
ce426f |
+ std r9,0(r3)
|
|
|
ce426f |
+ ld r9,8(r4)
|
|
|
ce426f |
+ std r9,8(r3)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ /* Align source to 16 bytes and adjust destiny and size. */
|
|
|
ce426f |
+L(align_to_16b):
|
|
|
ce426f |
+ rldicl r9,r10,0,60
|
|
|
ce426f |
+ rldicr r28,r10,0,59
|
|
|
ce426f |
+ add r12,r5,r9
|
|
|
ce426f |
+ subf r29,r9,r29
|
|
|
ce426f |
+
|
|
|
ce426f |
+ /* The bulk read/compare/copy loads two doublewords, compare and merge
|
|
|
ce426f |
+ in a single register for speed. This is an attempt to speed up the
|
|
|
ce426f |
+ null-checking process for bigger strings. */
|
|
|
ce426f |
+
|
|
|
ce426f |
+ cmpldi cr7,r12,15
|
|
|
ce426f |
+ ble cr7,L(short_path_prepare_1_2)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ /* Main loop for large sizes, unrolled 2 times to get better use of
|
|
|
ce426f |
+ pipeline. */
|
|
|
ce426f |
+ ld r8,0(28)
|
|
|
ce426f |
+ ld r10,8(28)
|
|
|
ce426f |
+ li r9,0
|
|
|
ce426f |
+ cmpb r7,r8,r9
|
|
|
ce426f |
+ cmpb r9,r10,r9
|
|
|
ce426f |
+ or. r6,r9,r7
|
|
|
ce426f |
+ bne cr0,L(short_path_prepare_2_3)
|
|
|
ce426f |
+ addi r5,r12,-16
|
|
|
ce426f |
+ addi r4,r28,16
|
|
|
ce426f |
+ std r8,0(r29)
|
|
|
ce426f |
+ std r10,8(r29)
|
|
|
ce426f |
+ cmpldi cr7,r5,15
|
|
|
ce426f |
+ addi r9,r29,16
|
|
|
ce426f |
+ ble cr7,L(short_path_1)
|
|
|
ce426f |
+ mr r11,r28
|
|
|
ce426f |
+ mr r6,r29
|
|
|
ce426f |
+ li r30,0
|
|
|
ce426f |
+ subfic r26,r4,48
|
|
|
ce426f |
+ subfic r27,r9,48
|
|
|
ce426f |
+
|
|
|
ce426f |
+ b L(loop_16b)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ .align 4
|
|
|
ce426f |
+L(loop_start):
|
|
|
ce426f |
+ ld r31,0(r11)
|
|
|
ce426f |
+ ld r10,8(r11)
|
|
|
ce426f |
+ cmpb r0,r31,r7
|
|
|
ce426f |
+ cmpb r8,r10,r7
|
|
|
ce426f |
+ or. r7,r0,r8
|
|
|
ce426f |
+ addi r5,r5,-32
|
|
|
ce426f |
+ cmpldi cr7,r5,15
|
|
|
ce426f |
+ add r4,r4,r26
|
|
|
ce426f |
+ add r9,r9,r27
|
|
|
ce426f |
+ bne cr0,L(short_path_prepare_2_2)
|
|
|
ce426f |
+ add r4,r28,r4
|
|
|
ce426f |
+ std r31,0(r6)
|
|
|
ce426f |
+ add r9,r29,r9
|
|
|
ce426f |
+ std r10,8(r6)
|
|
|
ce426f |
+ ble cr7,L(short_path_1)
|
|
|
ce426f |
+
|
|
|
ce426f |
+L(loop_16b):
|
|
|
ce426f |
+ ld r10,16(r11)
|
|
|
ce426f |
+ ld r0,24(r11)
|
|
|
ce426f |
+ cmpb r8,r10,r30
|
|
|
ce426f |
+ cmpb r7,r0,r30
|
|
|
ce426f |
+ or. r7,r8,r7
|
|
|
ce426f |
+ addi r12,r12,-32
|
|
|
ce426f |
+ cmpldi r7,r12,15
|
|
|
ce426f |
+ addi r11,r11,32
|
|
|
ce426f |
+ bne cr0,L(short_path_2)
|
|
|
ce426f |
+ std r10,16(r6)
|
|
|
ce426f |
+ addi r6,r6,32
|
|
|
ce426f |
+ std r0,-8(r6)
|
|
|
ce426f |
+ bgt cr7,L(loop_start)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ mr r5,r12
|
|
|
ce426f |
+ mr r4,r11
|
|
|
ce426f |
+ mr r9,r6
|
|
|
ce426f |
+ b L(short_path_1)
|
|
|
ce426f |
+
|
|
|
ce426f |
+ .align 4
|
|
|
ce426f |
+L(short_path_prepare_1_1):
|
|
|
ce426f |
+ mr r5,r6
|
|
|
ce426f |
+ mr r4,r7
|
|
|
ce426f |
+ b L(short_path_1)
|
|
|
ce426f |
+L(short_path_prepare_1_2):
|
|
|
ce426f |
+ mr r5,r12
|
|
|
ce426f |
+ mr r4,r28
|
|
|
ce426f |
+ mr r9,r29
|
|
|
ce426f |
+ b L(short_path_1)
|
|
|
ce426f |
+L(short_path_prepare_2):
|
|
|
ce426f |
+ mr r9,r3
|
|
|
ce426f |
+ b L(short_path_2)
|
|
|
ce426f |
+L(short_path_prepare_2_1):
|
|
|
ce426f |
+ mr r5,r6
|
|
|
ce426f |
+ mr r4,r7
|
|
|
ce426f |
+ b L(short_path_2)
|
|
|
ce426f |
+L(short_path_prepare_2_2):
|
|
|
ce426f |
+ mr r5,r12
|
|
|
ce426f |
+ mr r4,r11
|
|
|
ce426f |
+ mr r9,r6
|
|
|
ce426f |
+ b L(short_path_2)
|
|
|
ce426f |
+L(short_path_prepare_2_3):
|
|
|
ce426f |
+ mr r5,r12
|
|
|
ce426f |
+ mr r4,r28
|
|
|
ce426f |
+ mr r9,r29
|
|
|
ce426f |
+ b L(short_path_2)
|
|
|
ce426f |
+L(zero_pad_loop_b_prepare):
|
|
|
ce426f |
+ addi r10,r9,8
|
|
|
ce426f |
+ rldicl r5,r5,0,61
|
|
|
ce426f |
+ b L(zero_pad_loop_b_start)
|
|
|
ce426f |
+L(zero_pad_start_prepare_1):
|
|
|
ce426f |
+ mr r5,r6
|
|
|
ce426f |
+ mr r9,r8
|
|
|
ce426f |
+ b L(zero_pad_start_1)
|
|
|
ce426f |
+END (FUNC_NAME)
|
|
|
ce426f |
+
|
|
|
ce426f |
+#ifdef USE_AS_STPNCPY
|
|
|
ce426f |
+libc_hidden_def (__stpncpy)
|
|
|
ce426f |
+#else
|
|
|
ce426f |
+libc_hidden_builtin_def (strncpy)
|
|
|
ce426f |
+#endif
|