|
 |
190885 |
From 3848cd2cab96c673c98ea339aeefd5a27837f587 Mon Sep 17 00:00:00 2001
|
|
|
190885 |
From: "H.J. Lu" <hjl.tools@gmail.com>
|
|
|
190885 |
Date: Tue, 22 Jun 2021 20:42:10 -0700
|
|
|
190885 |
Subject: [PATCH] x86-64: Move strlen.S to multiarch/strlen-vec.S
|
|
|
190885 |
|
|
|
190885 |
Since strlen.S contains SSE2 version of strlen/strnlen and SSE4.1
|
|
|
190885 |
version of wcslen/wcsnlen, move strlen.S to multiarch/strlen-vec.S
|
|
|
190885 |
and include multiarch/strlen-vec.S from SSE2 and SSE4.1 variants.
|
|
|
190885 |
This also removes the unused symbols, __GI___strlen_sse2 and
|
|
|
190885 |
__GI___wcsnlen_sse4_1.
|
|
|
190885 |
|
|
|
190885 |
(cherry picked from commit a0db678071c60b6c47c468d231dd0b3694ba7a98)
|
|
|
190885 |
---
|
|
|
190885 |
sysdeps/x86_64/multiarch/strlen-sse2.S | 2 +-
|
|
|
190885 |
sysdeps/x86_64/multiarch/strlen-vec.S | 257 ++++++++++++++++++++++
|
|
|
190885 |
sysdeps/x86_64/multiarch/wcsnlen-sse4_1.S | 2 +-
|
|
|
190885 |
sysdeps/x86_64/strlen.S | 243 +-------------------
|
|
|
190885 |
4 files changed, 262 insertions(+), 242 deletions(-)
|
|
|
190885 |
create mode 100644 sysdeps/x86_64/multiarch/strlen-vec.S
|
|
|
190885 |
|
|
|
190885 |
diff --git a/sysdeps/x86_64/multiarch/strlen-sse2.S b/sysdeps/x86_64/multiarch/strlen-sse2.S
|
|
|
190885 |
index 7bc57b8d..449c8a7f 100644
|
|
|
190885 |
--- a/sysdeps/x86_64/multiarch/strlen-sse2.S
|
|
|
190885 |
+++ b/sysdeps/x86_64/multiarch/strlen-sse2.S
|
|
|
190885 |
@@ -20,4 +20,4 @@
|
|
|
190885 |
# define strlen __strlen_sse2
|
|
|
190885 |
#endif
|
|
|
190885 |
|
|
|
190885 |
-#include "../strlen.S"
|
|
|
190885 |
+#include "strlen-vec.S"
|
|
|
190885 |
diff --git a/sysdeps/x86_64/multiarch/strlen-vec.S b/sysdeps/x86_64/multiarch/strlen-vec.S
|
|
|
190885 |
new file mode 100644
|
|
|
190885 |
index 00000000..8f660bb9
|
|
|
190885 |
--- /dev/null
|
|
|
190885 |
+++ b/sysdeps/x86_64/multiarch/strlen-vec.S
|
|
|
190885 |
@@ -0,0 +1,257 @@
|
|
|
190885 |
+/* SSE2 version of strlen and SSE4.1 version of wcslen.
|
|
|
190885 |
+ Copyright (C) 2012-2021 Free Software Foundation, Inc.
|
|
|
190885 |
+ This file is part of the GNU C Library.
|
|
|
190885 |
+
|
|
|
190885 |
+ The GNU C Library is free software; you can redistribute it and/or
|
|
|
190885 |
+ modify it under the terms of the GNU Lesser General Public
|
|
|
190885 |
+ License as published by the Free Software Foundation; either
|
|
|
190885 |
+ version 2.1 of the License, or (at your option) any later version.
|
|
|
190885 |
+
|
|
|
190885 |
+ The GNU C Library is distributed in the hope that it will be useful,
|
|
|
190885 |
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
190885 |
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
190885 |
+ Lesser General Public License for more details.
|
|
|
190885 |
+
|
|
|
190885 |
+ You should have received a copy of the GNU Lesser General Public
|
|
|
190885 |
+ License along with the GNU C Library; if not, see
|
|
|
190885 |
+ <https://www.gnu.org/licenses/>. */
|
|
|
190885 |
+
|
|
|
190885 |
+#include <sysdep.h>
|
|
|
190885 |
+
|
|
|
190885 |
+#ifdef AS_WCSLEN
|
|
|
190885 |
+# define PMINU pminud
|
|
|
190885 |
+# define PCMPEQ pcmpeqd
|
|
|
190885 |
+# define SHIFT_RETURN shrq $2, %rax
|
|
|
190885 |
+#else
|
|
|
190885 |
+# define PMINU pminub
|
|
|
190885 |
+# define PCMPEQ pcmpeqb
|
|
|
190885 |
+# define SHIFT_RETURN
|
|
|
190885 |
+#endif
|
|
|
190885 |
+
|
|
|
190885 |
+/* Long lived register in strlen(s), strnlen(s, n) are:
|
|
|
190885 |
+
|
|
|
190885 |
+ %xmm3 - zero
|
|
|
190885 |
+ %rdi - s
|
|
|
190885 |
+ %r10 (s+n) & (~(64-1))
|
|
|
190885 |
+ %r11 s+n
|
|
|
190885 |
+*/
|
|
|
190885 |
+
|
|
|
190885 |
+
|
|
|
190885 |
+.text
|
|
|
190885 |
+ENTRY(strlen)
|
|
|
190885 |
+
|
|
|
190885 |
+/* Test 64 bytes from %rax for zero. Save result as bitmask in %rdx. */
|
|
|
190885 |
+#define FIND_ZERO \
|
|
|
190885 |
+ PCMPEQ (%rax), %xmm0; \
|
|
|
190885 |
+ PCMPEQ 16(%rax), %xmm1; \
|
|
|
190885 |
+ PCMPEQ 32(%rax), %xmm2; \
|
|
|
190885 |
+ PCMPEQ 48(%rax), %xmm3; \
|
|
|
190885 |
+ pmovmskb %xmm0, %esi; \
|
|
|
190885 |
+ pmovmskb %xmm1, %edx; \
|
|
|
190885 |
+ pmovmskb %xmm2, %r8d; \
|
|
|
190885 |
+ pmovmskb %xmm3, %ecx; \
|
|
|
190885 |
+ salq $16, %rdx; \
|
|
|
190885 |
+ salq $16, %rcx; \
|
|
|
190885 |
+ orq %rsi, %rdx; \
|
|
|
190885 |
+ orq %r8, %rcx; \
|
|
|
190885 |
+ salq $32, %rcx; \
|
|
|
190885 |
+ orq %rcx, %rdx;
|
|
|
190885 |
+
|
|
|
190885 |
+#ifdef AS_STRNLEN
|
|
|
190885 |
+/* Do not read anything when n==0. */
|
|
|
190885 |
+ test %RSI_LP, %RSI_LP
|
|
|
190885 |
+ jne L(n_nonzero)
|
|
|
190885 |
+ xor %rax, %rax
|
|
|
190885 |
+ ret
|
|
|
190885 |
+L(n_nonzero):
|
|
|
190885 |
+# ifdef AS_WCSLEN
|
|
|
190885 |
+ shl $2, %RSI_LP
|
|
|
190885 |
+# endif
|
|
|
190885 |
+
|
|
|
190885 |
+/* Initialize long lived registers. */
|
|
|
190885 |
+
|
|
|
190885 |
+ add %RDI_LP, %RSI_LP
|
|
|
190885 |
+ mov %RSI_LP, %R10_LP
|
|
|
190885 |
+ and $-64, %R10_LP
|
|
|
190885 |
+ mov %RSI_LP, %R11_LP
|
|
|
190885 |
+#endif
|
|
|
190885 |
+
|
|
|
190885 |
+ pxor %xmm0, %xmm0
|
|
|
190885 |
+ pxor %xmm1, %xmm1
|
|
|
190885 |
+ pxor %xmm2, %xmm2
|
|
|
190885 |
+ pxor %xmm3, %xmm3
|
|
|
190885 |
+ movq %rdi, %rax
|
|
|
190885 |
+ movq %rdi, %rcx
|
|
|
190885 |
+ andq $4095, %rcx
|
|
|
190885 |
+/* Offsets 4032-4047 will be aligned into 4032 thus fit into page. */
|
|
|
190885 |
+ cmpq $4047, %rcx
|
|
|
190885 |
+/* We cannot unify this branching as it would be ~6 cycles slower. */
|
|
|
190885 |
+ ja L(cross_page)
|
|
|
190885 |
+
|
|
|
190885 |
+#ifdef AS_STRNLEN
|
|
|
190885 |
+/* Test if end is among first 64 bytes. */
|
|
|
190885 |
+# define STRNLEN_PROLOG \
|
|
|
190885 |
+ mov %r11, %rsi; \
|
|
|
190885 |
+ subq %rax, %rsi; \
|
|
|
190885 |
+ andq $-64, %rax; \
|
|
|
190885 |
+ testq $-64, %rsi; \
|
|
|
190885 |
+ je L(strnlen_ret)
|
|
|
190885 |
+#else
|
|
|
190885 |
+# define STRNLEN_PROLOG andq $-64, %rax;
|
|
|
190885 |
+#endif
|
|
|
190885 |
+
|
|
|
190885 |
+/* Ignore bits in mask that come before start of string. */
|
|
|
190885 |
+#define PROLOG(lab) \
|
|
|
190885 |
+ movq %rdi, %rcx; \
|
|
|
190885 |
+ xorq %rax, %rcx; \
|
|
|
190885 |
+ STRNLEN_PROLOG; \
|
|
|
190885 |
+ sarq %cl, %rdx; \
|
|
|
190885 |
+ test %rdx, %rdx; \
|
|
|
190885 |
+ je L(lab); \
|
|
|
190885 |
+ bsfq %rdx, %rax; \
|
|
|
190885 |
+ SHIFT_RETURN; \
|
|
|
190885 |
+ ret
|
|
|
190885 |
+
|
|
|
190885 |
+#ifdef AS_STRNLEN
|
|
|
190885 |
+ andq $-16, %rax
|
|
|
190885 |
+ FIND_ZERO
|
|
|
190885 |
+#else
|
|
|
190885 |
+ /* Test first 16 bytes unaligned. */
|
|
|
190885 |
+ movdqu (%rax), %xmm4
|
|
|
190885 |
+ PCMPEQ %xmm0, %xmm4
|
|
|
190885 |
+ pmovmskb %xmm4, %edx
|
|
|
190885 |
+ test %edx, %edx
|
|
|
190885 |
+ je L(next48_bytes)
|
|
|
190885 |
+ bsf %edx, %eax /* If eax is zeroed 16bit bsf can be used. */
|
|
|
190885 |
+ SHIFT_RETURN
|
|
|
190885 |
+ ret
|
|
|
190885 |
+
|
|
|
190885 |
+L(next48_bytes):
|
|
|
190885 |
+/* Same as FIND_ZERO except we do not check first 16 bytes. */
|
|
|
190885 |
+ andq $-16, %rax
|
|
|
190885 |
+ PCMPEQ 16(%rax), %xmm1
|
|
|
190885 |
+ PCMPEQ 32(%rax), %xmm2
|
|
|
190885 |
+ PCMPEQ 48(%rax), %xmm3
|
|
|
190885 |
+ pmovmskb %xmm1, %edx
|
|
|
190885 |
+ pmovmskb %xmm2, %r8d
|
|
|
190885 |
+ pmovmskb %xmm3, %ecx
|
|
|
190885 |
+ salq $16, %rdx
|
|
|
190885 |
+ salq $16, %rcx
|
|
|
190885 |
+ orq %r8, %rcx
|
|
|
190885 |
+ salq $32, %rcx
|
|
|
190885 |
+ orq %rcx, %rdx
|
|
|
190885 |
+#endif
|
|
|
190885 |
+
|
|
|
190885 |
+ /* When no zero byte is found xmm1-3 are zero so we do not have to
|
|
|
190885 |
+ zero them. */
|
|
|
190885 |
+ PROLOG(loop)
|
|
|
190885 |
+
|
|
|
190885 |
+ .p2align 4
|
|
|
190885 |
+L(cross_page):
|
|
|
190885 |
+ andq $-64, %rax
|
|
|
190885 |
+ FIND_ZERO
|
|
|
190885 |
+ PROLOG(loop_init)
|
|
|
190885 |
+
|
|
|
190885 |
+#ifdef AS_STRNLEN
|
|
|
190885 |
+/* We must do this check to correctly handle strnlen (s, -1). */
|
|
|
190885 |
+L(strnlen_ret):
|
|
|
190885 |
+ bts %rsi, %rdx
|
|
|
190885 |
+ sarq %cl, %rdx
|
|
|
190885 |
+ test %rdx, %rdx
|
|
|
190885 |
+ je L(loop_init)
|
|
|
190885 |
+ bsfq %rdx, %rax
|
|
|
190885 |
+ SHIFT_RETURN
|
|
|
190885 |
+ ret
|
|
|
190885 |
+#endif
|
|
|
190885 |
+ .p2align 4
|
|
|
190885 |
+L(loop_init):
|
|
|
190885 |
+ pxor %xmm1, %xmm1
|
|
|
190885 |
+ pxor %xmm2, %xmm2
|
|
|
190885 |
+ pxor %xmm3, %xmm3
|
|
|
190885 |
+#ifdef AS_STRNLEN
|
|
|
190885 |
+ .p2align 4
|
|
|
190885 |
+L(loop):
|
|
|
190885 |
+
|
|
|
190885 |
+ addq $64, %rax
|
|
|
190885 |
+ cmpq %rax, %r10
|
|
|
190885 |
+ je L(exit_end)
|
|
|
190885 |
+
|
|
|
190885 |
+ movdqa (%rax), %xmm0
|
|
|
190885 |
+ PMINU 16(%rax), %xmm0
|
|
|
190885 |
+ PMINU 32(%rax), %xmm0
|
|
|
190885 |
+ PMINU 48(%rax), %xmm0
|
|
|
190885 |
+ PCMPEQ %xmm3, %xmm0
|
|
|
190885 |
+ pmovmskb %xmm0, %edx
|
|
|
190885 |
+ testl %edx, %edx
|
|
|
190885 |
+ jne L(exit)
|
|
|
190885 |
+ jmp L(loop)
|
|
|
190885 |
+
|
|
|
190885 |
+ .p2align 4
|
|
|
190885 |
+L(exit_end):
|
|
|
190885 |
+ cmp %rax, %r11
|
|
|
190885 |
+ je L(first) /* Do not read when end is at page boundary. */
|
|
|
190885 |
+ pxor %xmm0, %xmm0
|
|
|
190885 |
+ FIND_ZERO
|
|
|
190885 |
+
|
|
|
190885 |
+L(first):
|
|
|
190885 |
+ bts %r11, %rdx
|
|
|
190885 |
+ bsfq %rdx, %rdx
|
|
|
190885 |
+ addq %rdx, %rax
|
|
|
190885 |
+ subq %rdi, %rax
|
|
|
190885 |
+ SHIFT_RETURN
|
|
|
190885 |
+ ret
|
|
|
190885 |
+
|
|
|
190885 |
+ .p2align 4
|
|
|
190885 |
+L(exit):
|
|
|
190885 |
+ pxor %xmm0, %xmm0
|
|
|
190885 |
+ FIND_ZERO
|
|
|
190885 |
+
|
|
|
190885 |
+ bsfq %rdx, %rdx
|
|
|
190885 |
+ addq %rdx, %rax
|
|
|
190885 |
+ subq %rdi, %rax
|
|
|
190885 |
+ SHIFT_RETURN
|
|
|
190885 |
+ ret
|
|
|
190885 |
+
|
|
|
190885 |
+#else
|
|
|
190885 |
+
|
|
|
190885 |
+ /* Main loop. Unrolled twice to improve L2 cache performance on core2. */
|
|
|
190885 |
+ .p2align 4
|
|
|
190885 |
+L(loop):
|
|
|
190885 |
+
|
|
|
190885 |
+ movdqa 64(%rax), %xmm0
|
|
|
190885 |
+ PMINU 80(%rax), %xmm0
|
|
|
190885 |
+ PMINU 96(%rax), %xmm0
|
|
|
190885 |
+ PMINU 112(%rax), %xmm0
|
|
|
190885 |
+ PCMPEQ %xmm3, %xmm0
|
|
|
190885 |
+ pmovmskb %xmm0, %edx
|
|
|
190885 |
+ testl %edx, %edx
|
|
|
190885 |
+ jne L(exit64)
|
|
|
190885 |
+
|
|
|
190885 |
+ subq $-128, %rax
|
|
|
190885 |
+
|
|
|
190885 |
+ movdqa (%rax), %xmm0
|
|
|
190885 |
+ PMINU 16(%rax), %xmm0
|
|
|
190885 |
+ PMINU 32(%rax), %xmm0
|
|
|
190885 |
+ PMINU 48(%rax), %xmm0
|
|
|
190885 |
+ PCMPEQ %xmm3, %xmm0
|
|
|
190885 |
+ pmovmskb %xmm0, %edx
|
|
|
190885 |
+ testl %edx, %edx
|
|
|
190885 |
+ jne L(exit0)
|
|
|
190885 |
+ jmp L(loop)
|
|
|
190885 |
+
|
|
|
190885 |
+ .p2align 4
|
|
|
190885 |
+L(exit64):
|
|
|
190885 |
+ addq $64, %rax
|
|
|
190885 |
+L(exit0):
|
|
|
190885 |
+ pxor %xmm0, %xmm0
|
|
|
190885 |
+ FIND_ZERO
|
|
|
190885 |
+
|
|
|
190885 |
+ bsfq %rdx, %rdx
|
|
|
190885 |
+ addq %rdx, %rax
|
|
|
190885 |
+ subq %rdi, %rax
|
|
|
190885 |
+ SHIFT_RETURN
|
|
|
190885 |
+ ret
|
|
|
190885 |
+
|
|
|
190885 |
+#endif
|
|
|
190885 |
+
|
|
|
190885 |
+END(strlen)
|
|
|
190885 |
diff --git a/sysdeps/x86_64/multiarch/wcsnlen-sse4_1.S b/sysdeps/x86_64/multiarch/wcsnlen-sse4_1.S
|
|
|
190885 |
index a8cab0cb..5fa51fe0 100644
|
|
|
190885 |
--- a/sysdeps/x86_64/multiarch/wcsnlen-sse4_1.S
|
|
|
190885 |
+++ b/sysdeps/x86_64/multiarch/wcsnlen-sse4_1.S
|
|
|
190885 |
@@ -2,4 +2,4 @@
|
|
|
190885 |
#define AS_STRNLEN
|
|
|
190885 |
#define strlen __wcsnlen_sse4_1
|
|
|
190885 |
|
|
|
190885 |
-#include "../strlen.S"
|
|
|
190885 |
+#include "strlen-vec.S"
|
|
|
190885 |
diff --git a/sysdeps/x86_64/strlen.S b/sysdeps/x86_64/strlen.S
|
|
|
190885 |
index f845f3d4..ad047d84 100644
|
|
|
190885 |
--- a/sysdeps/x86_64/strlen.S
|
|
|
190885 |
+++ b/sysdeps/x86_64/strlen.S
|
|
|
190885 |
@@ -1,5 +1,5 @@
|
|
|
190885 |
-/* SSE2 version of strlen/wcslen.
|
|
|
190885 |
- Copyright (C) 2012-2018 Free Software Foundation, Inc.
|
|
|
190885 |
+/* SSE2 version of strlen.
|
|
|
190885 |
+ Copyright (C) 2021 Free Software Foundation, Inc.
|
|
|
190885 |
This file is part of the GNU C Library.
|
|
|
190885 |
|
|
|
190885 |
The GNU C Library is free software; you can redistribute it and/or
|
|
|
190885 |
@@ -16,243 +16,6 @@
|
|
|
190885 |
License along with the GNU C Library; if not, see
|
|
|
190885 |
<http://www.gnu.org/licenses/>. */
|
|
|
190885 |
|
|
|
190885 |
-#include <sysdep.h>
|
|
|
190885 |
+#include "multiarch/strlen-vec.S"
|
|
|
190885 |
|
|
|
190885 |
-#ifdef AS_WCSLEN
|
|
|
190885 |
-# define PMINU pminud
|
|
|
190885 |
-# define PCMPEQ pcmpeqd
|
|
|
190885 |
-# define SHIFT_RETURN shrq $2, %rax
|
|
|
190885 |
-#else
|
|
|
190885 |
-# define PMINU pminub
|
|
|
190885 |
-# define PCMPEQ pcmpeqb
|
|
|
190885 |
-# define SHIFT_RETURN
|
|
|
190885 |
-#endif
|
|
|
190885 |
-
|
|
|
190885 |
-/* Long lived register in strlen(s), strnlen(s, n) are:
|
|
|
190885 |
-
|
|
|
190885 |
- %xmm3 - zero
|
|
|
190885 |
- %rdi - s
|
|
|
190885 |
- %r10 (s+n) & (~(64-1))
|
|
|
190885 |
- %r11 s+n
|
|
|
190885 |
-*/
|
|
|
190885 |
-
|
|
|
190885 |
-
|
|
|
190885 |
-.text
|
|
|
190885 |
-ENTRY(strlen)
|
|
|
190885 |
-
|
|
|
190885 |
-/* Test 64 bytes from %rax for zero. Save result as bitmask in %rdx. */
|
|
|
190885 |
-#define FIND_ZERO \
|
|
|
190885 |
- PCMPEQ (%rax), %xmm0; \
|
|
|
190885 |
- PCMPEQ 16(%rax), %xmm1; \
|
|
|
190885 |
- PCMPEQ 32(%rax), %xmm2; \
|
|
|
190885 |
- PCMPEQ 48(%rax), %xmm3; \
|
|
|
190885 |
- pmovmskb %xmm0, %esi; \
|
|
|
190885 |
- pmovmskb %xmm1, %edx; \
|
|
|
190885 |
- pmovmskb %xmm2, %r8d; \
|
|
|
190885 |
- pmovmskb %xmm3, %ecx; \
|
|
|
190885 |
- salq $16, %rdx; \
|
|
|
190885 |
- salq $16, %rcx; \
|
|
|
190885 |
- orq %rsi, %rdx; \
|
|
|
190885 |
- orq %r8, %rcx; \
|
|
|
190885 |
- salq $32, %rcx; \
|
|
|
190885 |
- orq %rcx, %rdx;
|
|
|
190885 |
-
|
|
|
190885 |
-#ifdef AS_STRNLEN
|
|
|
190885 |
-/* Do not read anything when n==0. */
|
|
|
190885 |
- test %RSI_LP, %RSI_LP
|
|
|
190885 |
- jne L(n_nonzero)
|
|
|
190885 |
- xor %rax, %rax
|
|
|
190885 |
- ret
|
|
|
190885 |
-L(n_nonzero):
|
|
|
190885 |
-# ifdef AS_WCSLEN
|
|
|
190885 |
- shl $2, %RSI_LP
|
|
|
190885 |
-# endif
|
|
|
190885 |
-
|
|
|
190885 |
-/* Initialize long lived registers. */
|
|
|
190885 |
-
|
|
|
190885 |
- add %RDI_LP, %RSI_LP
|
|
|
190885 |
- mov %RSI_LP, %R10_LP
|
|
|
190885 |
- and $-64, %R10_LP
|
|
|
190885 |
- mov %RSI_LP, %R11_LP
|
|
|
190885 |
-#endif
|
|
|
190885 |
-
|
|
|
190885 |
- pxor %xmm0, %xmm0
|
|
|
190885 |
- pxor %xmm1, %xmm1
|
|
|
190885 |
- pxor %xmm2, %xmm2
|
|
|
190885 |
- pxor %xmm3, %xmm3
|
|
|
190885 |
- movq %rdi, %rax
|
|
|
190885 |
- movq %rdi, %rcx
|
|
|
190885 |
- andq $4095, %rcx
|
|
|
190885 |
-/* Offsets 4032-4047 will be aligned into 4032 thus fit into page. */
|
|
|
190885 |
- cmpq $4047, %rcx
|
|
|
190885 |
-/* We cannot unify this branching as it would be ~6 cycles slower. */
|
|
|
190885 |
- ja L(cross_page)
|
|
|
190885 |
-
|
|
|
190885 |
-#ifdef AS_STRNLEN
|
|
|
190885 |
-/* Test if end is among first 64 bytes. */
|
|
|
190885 |
-# define STRNLEN_PROLOG \
|
|
|
190885 |
- mov %r11, %rsi; \
|
|
|
190885 |
- subq %rax, %rsi; \
|
|
|
190885 |
- andq $-64, %rax; \
|
|
|
190885 |
- testq $-64, %rsi; \
|
|
|
190885 |
- je L(strnlen_ret)
|
|
|
190885 |
-#else
|
|
|
190885 |
-# define STRNLEN_PROLOG andq $-64, %rax;
|
|
|
190885 |
-#endif
|
|
|
190885 |
-
|
|
|
190885 |
-/* Ignore bits in mask that come before start of string. */
|
|
|
190885 |
-#define PROLOG(lab) \
|
|
|
190885 |
- movq %rdi, %rcx; \
|
|
|
190885 |
- xorq %rax, %rcx; \
|
|
|
190885 |
- STRNLEN_PROLOG; \
|
|
|
190885 |
- sarq %cl, %rdx; \
|
|
|
190885 |
- test %rdx, %rdx; \
|
|
|
190885 |
- je L(lab); \
|
|
|
190885 |
- bsfq %rdx, %rax; \
|
|
|
190885 |
- SHIFT_RETURN; \
|
|
|
190885 |
- ret
|
|
|
190885 |
-
|
|
|
190885 |
-#ifdef AS_STRNLEN
|
|
|
190885 |
- andq $-16, %rax
|
|
|
190885 |
- FIND_ZERO
|
|
|
190885 |
-#else
|
|
|
190885 |
- /* Test first 16 bytes unaligned. */
|
|
|
190885 |
- movdqu (%rax), %xmm4
|
|
|
190885 |
- PCMPEQ %xmm0, %xmm4
|
|
|
190885 |
- pmovmskb %xmm4, %edx
|
|
|
190885 |
- test %edx, %edx
|
|
|
190885 |
- je L(next48_bytes)
|
|
|
190885 |
- bsf %edx, %eax /* If eax is zeroed 16bit bsf can be used. */
|
|
|
190885 |
- SHIFT_RETURN
|
|
|
190885 |
- ret
|
|
|
190885 |
-
|
|
|
190885 |
-L(next48_bytes):
|
|
|
190885 |
-/* Same as FIND_ZERO except we do not check first 16 bytes. */
|
|
|
190885 |
- andq $-16, %rax
|
|
|
190885 |
- PCMPEQ 16(%rax), %xmm1
|
|
|
190885 |
- PCMPEQ 32(%rax), %xmm2
|
|
|
190885 |
- PCMPEQ 48(%rax), %xmm3
|
|
|
190885 |
- pmovmskb %xmm1, %edx
|
|
|
190885 |
- pmovmskb %xmm2, %r8d
|
|
|
190885 |
- pmovmskb %xmm3, %ecx
|
|
|
190885 |
- salq $16, %rdx
|
|
|
190885 |
- salq $16, %rcx
|
|
|
190885 |
- orq %r8, %rcx
|
|
|
190885 |
- salq $32, %rcx
|
|
|
190885 |
- orq %rcx, %rdx
|
|
|
190885 |
-#endif
|
|
|
190885 |
-
|
|
|
190885 |
- /* When no zero byte is found xmm1-3 are zero so we do not have to
|
|
|
190885 |
- zero them. */
|
|
|
190885 |
- PROLOG(loop)
|
|
|
190885 |
-
|
|
|
190885 |
- .p2align 4
|
|
|
190885 |
-L(cross_page):
|
|
|
190885 |
- andq $-64, %rax
|
|
|
190885 |
- FIND_ZERO
|
|
|
190885 |
- PROLOG(loop_init)
|
|
|
190885 |
-
|
|
|
190885 |
-#ifdef AS_STRNLEN
|
|
|
190885 |
-/* We must do this check to correctly handle strnlen (s, -1). */
|
|
|
190885 |
-L(strnlen_ret):
|
|
|
190885 |
- bts %rsi, %rdx
|
|
|
190885 |
- sarq %cl, %rdx
|
|
|
190885 |
- test %rdx, %rdx
|
|
|
190885 |
- je L(loop_init)
|
|
|
190885 |
- bsfq %rdx, %rax
|
|
|
190885 |
- SHIFT_RETURN
|
|
|
190885 |
- ret
|
|
|
190885 |
-#endif
|
|
|
190885 |
- .p2align 4
|
|
|
190885 |
-L(loop_init):
|
|
|
190885 |
- pxor %xmm1, %xmm1
|
|
|
190885 |
- pxor %xmm2, %xmm2
|
|
|
190885 |
- pxor %xmm3, %xmm3
|
|
|
190885 |
-#ifdef AS_STRNLEN
|
|
|
190885 |
- .p2align 4
|
|
|
190885 |
-L(loop):
|
|
|
190885 |
-
|
|
|
190885 |
- addq $64, %rax
|
|
|
190885 |
- cmpq %rax, %r10
|
|
|
190885 |
- je L(exit_end)
|
|
|
190885 |
-
|
|
|
190885 |
- movdqa (%rax), %xmm0
|
|
|
190885 |
- PMINU 16(%rax), %xmm0
|
|
|
190885 |
- PMINU 32(%rax), %xmm0
|
|
|
190885 |
- PMINU 48(%rax), %xmm0
|
|
|
190885 |
- PCMPEQ %xmm3, %xmm0
|
|
|
190885 |
- pmovmskb %xmm0, %edx
|
|
|
190885 |
- testl %edx, %edx
|
|
|
190885 |
- jne L(exit)
|
|
|
190885 |
- jmp L(loop)
|
|
|
190885 |
-
|
|
|
190885 |
- .p2align 4
|
|
|
190885 |
-L(exit_end):
|
|
|
190885 |
- cmp %rax, %r11
|
|
|
190885 |
- je L(first) /* Do not read when end is at page boundary. */
|
|
|
190885 |
- pxor %xmm0, %xmm0
|
|
|
190885 |
- FIND_ZERO
|
|
|
190885 |
-
|
|
|
190885 |
-L(first):
|
|
|
190885 |
- bts %r11, %rdx
|
|
|
190885 |
- bsfq %rdx, %rdx
|
|
|
190885 |
- addq %rdx, %rax
|
|
|
190885 |
- subq %rdi, %rax
|
|
|
190885 |
- SHIFT_RETURN
|
|
|
190885 |
- ret
|
|
|
190885 |
-
|
|
|
190885 |
- .p2align 4
|
|
|
190885 |
-L(exit):
|
|
|
190885 |
- pxor %xmm0, %xmm0
|
|
|
190885 |
- FIND_ZERO
|
|
|
190885 |
-
|
|
|
190885 |
- bsfq %rdx, %rdx
|
|
|
190885 |
- addq %rdx, %rax
|
|
|
190885 |
- subq %rdi, %rax
|
|
|
190885 |
- SHIFT_RETURN
|
|
|
190885 |
- ret
|
|
|
190885 |
-
|
|
|
190885 |
-#else
|
|
|
190885 |
-
|
|
|
190885 |
- /* Main loop. Unrolled twice to improve L2 cache performance on core2. */
|
|
|
190885 |
- .p2align 4
|
|
|
190885 |
-L(loop):
|
|
|
190885 |
-
|
|
|
190885 |
- movdqa 64(%rax), %xmm0
|
|
|
190885 |
- PMINU 80(%rax), %xmm0
|
|
|
190885 |
- PMINU 96(%rax), %xmm0
|
|
|
190885 |
- PMINU 112(%rax), %xmm0
|
|
|
190885 |
- PCMPEQ %xmm3, %xmm0
|
|
|
190885 |
- pmovmskb %xmm0, %edx
|
|
|
190885 |
- testl %edx, %edx
|
|
|
190885 |
- jne L(exit64)
|
|
|
190885 |
-
|
|
|
190885 |
- subq $-128, %rax
|
|
|
190885 |
-
|
|
|
190885 |
- movdqa (%rax), %xmm0
|
|
|
190885 |
- PMINU 16(%rax), %xmm0
|
|
|
190885 |
- PMINU 32(%rax), %xmm0
|
|
|
190885 |
- PMINU 48(%rax), %xmm0
|
|
|
190885 |
- PCMPEQ %xmm3, %xmm0
|
|
|
190885 |
- pmovmskb %xmm0, %edx
|
|
|
190885 |
- testl %edx, %edx
|
|
|
190885 |
- jne L(exit0)
|
|
|
190885 |
- jmp L(loop)
|
|
|
190885 |
-
|
|
|
190885 |
- .p2align 4
|
|
|
190885 |
-L(exit64):
|
|
|
190885 |
- addq $64, %rax
|
|
|
190885 |
-L(exit0):
|
|
|
190885 |
- pxor %xmm0, %xmm0
|
|
|
190885 |
- FIND_ZERO
|
|
|
190885 |
-
|
|
|
190885 |
- bsfq %rdx, %rdx
|
|
|
190885 |
- addq %rdx, %rax
|
|
|
190885 |
- subq %rdi, %rax
|
|
|
190885 |
- SHIFT_RETURN
|
|
|
190885 |
- ret
|
|
|
190885 |
-
|
|
|
190885 |
-#endif
|
|
|
190885 |
-
|
|
|
190885 |
-END(strlen)
|
|
|
190885 |
libc_hidden_builtin_def (strlen)
|
|
|
190885 |
--
|
|
|
190885 |
GitLab
|
|
|
190885 |
|