Blame SOURCES/ia-code_cleanup-strchr-evex.patch

513694
From 23371c15467ef5d3225018278d6691e110119e4b Mon Sep 17 00:00:00 2001
513694
From: Noah Goldstein <goldstein.w.n@gmail.com>
513694
Date: Wed, 23 Mar 2022 16:57:18 -0500
513694
Subject: [PATCH] x86: Code cleanup in strchr-evex and comment justifying
513694
 branch
513694
513694
Small code cleanup for size: -81 bytes.
513694
513694
Add comment justifying using a branch to do NULL/non-null return.
513694
513694
All string/memory tests pass and no regressions in benchtests.
513694
513694
geometric_mean(N=20) of all benchmarks New / Original: .985
513694
Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
513694
513694
(cherry picked from commit ec285ea90415458225623ddc0492ae3f705af043)
513694
---
513694
 sysdeps/x86_64/multiarch/strchr-evex.S | 146 ++++++++++++++-----------
513694
 1 file changed, 80 insertions(+), 66 deletions(-)
513694
513694
diff --git a/sysdeps/x86_64/multiarch/strchr-evex.S b/sysdeps/x86_64/multiarch/strchr-evex.S
513694
index 7f9d4ee4..0b49e0ac 100644
513694
--- a/sysdeps/x86_64/multiarch/strchr-evex.S
513694
+++ b/sysdeps/x86_64/multiarch/strchr-evex.S
513694
@@ -30,6 +30,7 @@
513694
 # ifdef USE_AS_WCSCHR
513694
 #  define VPBROADCAST	vpbroadcastd
513694
 #  define VPCMP		vpcmpd
513694
+#  define VPTESTN	vptestnmd
513694
 #  define VPMINU	vpminud
513694
 #  define CHAR_REG	esi
513694
 #  define SHIFT_REG	ecx
513694
@@ -37,6 +38,7 @@
513694
 # else
513694
 #  define VPBROADCAST	vpbroadcastb
513694
 #  define VPCMP		vpcmpb
513694
+#  define VPTESTN	vptestnmb
513694
 #  define VPMINU	vpminub
513694
 #  define CHAR_REG	sil
513694
 #  define SHIFT_REG	edx
513694
@@ -61,13 +63,11 @@
513694
 # define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE)
513694
 
513694
 	.section .text.evex,"ax",@progbits
513694
-ENTRY (STRCHR)
513694
+ENTRY_P2ALIGN (STRCHR, 5)
513694
 	/* Broadcast CHAR to YMM0.	*/
513694
 	VPBROADCAST	%esi, %YMM0
513694
 	movl	%edi, %eax
513694
 	andl	$(PAGE_SIZE - 1), %eax
513694
-	vpxorq	%XMMZERO, %XMMZERO, %XMMZERO
513694
-
513694
 	/* Check if we cross page boundary with one vector load.
513694
 	   Otherwise it is safe to use an unaligned load.  */
513694
 	cmpl	$(PAGE_SIZE - VEC_SIZE), %eax
513694
@@ -81,49 +81,35 @@ ENTRY (STRCHR)
513694
 	vpxorq	%YMM1, %YMM0, %YMM2
513694
 	VPMINU	%YMM2, %YMM1, %YMM2
513694
 	/* Each bit in K0 represents a CHAR or a null byte in YMM1.  */
513694
-	VPCMP	$0, %YMMZERO, %YMM2, %k0
513694
+	VPTESTN	%YMM2, %YMM2, %k0
513694
 	kmovd	%k0, %eax
513694
 	testl	%eax, %eax
513694
 	jz	L(aligned_more)
513694
 	tzcntl	%eax, %eax
513694
+# ifndef USE_AS_STRCHRNUL
513694
+	/* Found CHAR or the null byte.  */
513694
+	cmp	(%rdi, %rax, CHAR_SIZE), %CHAR_REG
513694
+	/* NB: Use a branch instead of cmovcc here. The expectation is
513694
+	   that with strchr the user will branch based on input being
513694
+	   null. Since this branch will be 100% predictive of the user
513694
+	   branch a branch miss here should save what otherwise would
513694
+	   be branch miss in the user code. Otherwise using a branch 1)
513694
+	   saves code size and 2) is faster in highly predictable
513694
+	   environments.  */
513694
+	jne	L(zero)
513694
+# endif
513694
 # ifdef USE_AS_WCSCHR
513694
 	/* NB: Multiply wchar_t count by 4 to get the number of bytes.
513694
 	 */
513694
 	leaq	(%rdi, %rax, CHAR_SIZE), %rax
513694
 # else
513694
 	addq	%rdi, %rax
513694
-# endif
513694
-# ifndef USE_AS_STRCHRNUL
513694
-	/* Found CHAR or the null byte.	 */
513694
-	cmp	(%rax), %CHAR_REG
513694
-	jne	L(zero)
513694
 # endif
513694
 	ret
513694
 
513694
-	/* .p2align 5 helps keep performance more consistent if ENTRY()
513694
-	   alignment % 32 was either 16 or 0. As well this makes the
513694
-	   alignment % 32 of the loop_4x_vec fixed which makes tuning it
513694
-	   easier.  */
513694
-	.p2align 5
513694
-L(first_vec_x3):
513694
-	tzcntl	%eax, %eax
513694
-# ifndef USE_AS_STRCHRNUL
513694
-	/* Found CHAR or the null byte.	 */
513694
-	cmp	(VEC_SIZE * 3)(%rdi, %rax, CHAR_SIZE), %CHAR_REG
513694
-	jne	L(zero)
513694
-# endif
513694
-	/* NB: Multiply sizeof char type (1 or 4) to get the number of
513694
-	   bytes.  */
513694
-	leaq	(VEC_SIZE * 3)(%rdi, %rax, CHAR_SIZE), %rax
513694
-	ret
513694
 
513694
-# ifndef USE_AS_STRCHRNUL
513694
-L(zero):
513694
-	xorl	%eax, %eax
513694
-	ret
513694
-# endif
513694
 
513694
-	.p2align 4
513694
+	.p2align 4,, 10
513694
 L(first_vec_x4):
513694
 # ifndef USE_AS_STRCHRNUL
513694
 	/* Check to see if first match was CHAR (k0) or null (k1).  */
513694
@@ -144,9 +130,18 @@ L(first_vec_x4):
513694
 	leaq	(VEC_SIZE * 4)(%rdi, %rax, CHAR_SIZE), %rax
513694
 	ret
513694
 
513694
+# ifndef USE_AS_STRCHRNUL
513694
+L(zero):
513694
+	xorl	%eax, %eax
513694
+	ret
513694
+# endif
513694
+
513694
+
513694
 	.p2align 4
513694
 L(first_vec_x1):
513694
-	tzcntl	%eax, %eax
513694
+	/* Use bsf here to save 1-byte keeping keeping the block in 1x
513694
+	   fetch block. eax guranteed non-zero.  */
513694
+	bsfl	%eax, %eax
513694
 # ifndef USE_AS_STRCHRNUL
513694
 	/* Found CHAR or the null byte.	 */
513694
 	cmp	(VEC_SIZE)(%rdi, %rax, CHAR_SIZE), %CHAR_REG
513694
@@ -158,7 +153,7 @@ L(first_vec_x1):
513694
 	leaq	(VEC_SIZE)(%rdi, %rax, CHAR_SIZE), %rax
513694
 	ret
513694
 
513694
-	.p2align 4
513694
+	.p2align 4,, 10
513694
 L(first_vec_x2):
513694
 # ifndef USE_AS_STRCHRNUL
513694
 	/* Check to see if first match was CHAR (k0) or null (k1).  */
513694
@@ -179,6 +174,21 @@ L(first_vec_x2):
513694
 	leaq	(VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %rax
513694
 	ret
513694
 
513694
+	.p2align 4,, 10
513694
+L(first_vec_x3):
513694
+	/* Use bsf here to save 1-byte keeping keeping the block in 1x
513694
+	   fetch block. eax guranteed non-zero.  */
513694
+	bsfl	%eax, %eax
513694
+# ifndef USE_AS_STRCHRNUL
513694
+	/* Found CHAR or the null byte.	 */
513694
+	cmp	(VEC_SIZE * 3)(%rdi, %rax, CHAR_SIZE), %CHAR_REG
513694
+	jne	L(zero)
513694
+# endif
513694
+	/* NB: Multiply sizeof char type (1 or 4) to get the number of
513694
+	   bytes.  */
513694
+	leaq	(VEC_SIZE * 3)(%rdi, %rax, CHAR_SIZE), %rax
513694
+	ret
513694
+
513694
 	.p2align 4
513694
 L(aligned_more):
513694
 	/* Align data to VEC_SIZE.  */
513694
@@ -195,7 +205,7 @@ L(cross_page_continue):
513694
 	vpxorq	%YMM1, %YMM0, %YMM2
513694
 	VPMINU	%YMM2, %YMM1, %YMM2
513694
 	/* Each bit in K0 represents a CHAR or a null byte in YMM1.  */
513694
-	VPCMP	$0, %YMMZERO, %YMM2, %k0
513694
+	VPTESTN	%YMM2, %YMM2, %k0
513694
 	kmovd	%k0, %eax
513694
 	testl	%eax, %eax
513694
 	jnz	L(first_vec_x1)
513694
@@ -206,7 +216,7 @@ L(cross_page_continue):
513694
 	/* Each bit in K0 represents a CHAR in YMM1.  */
513694
 	VPCMP	$0, %YMM1, %YMM0, %k0
513694
 	/* Each bit in K1 represents a CHAR in YMM1.  */
513694
-	VPCMP	$0, %YMM1, %YMMZERO, %k1
513694
+	VPTESTN	%YMM1, %YMM1, %k1
513694
 	kortestd	%k0, %k1
513694
 	jnz	L(first_vec_x2)
513694
 
513694
@@ -215,7 +225,7 @@ L(cross_page_continue):
513694
 	vpxorq	%YMM1, %YMM0, %YMM2
513694
 	VPMINU	%YMM2, %YMM1, %YMM2
513694
 	/* Each bit in K0 represents a CHAR or a null byte in YMM1.  */
513694
-	VPCMP	$0, %YMMZERO, %YMM2, %k0
513694
+	VPTESTN	%YMM2, %YMM2, %k0
513694
 	kmovd	%k0, %eax
513694
 	testl	%eax, %eax
513694
 	jnz	L(first_vec_x3)
513694
@@ -224,7 +234,7 @@ L(cross_page_continue):
513694
 	/* Each bit in K0 represents a CHAR in YMM1.  */
513694
 	VPCMP	$0, %YMM1, %YMM0, %k0
513694
 	/* Each bit in K1 represents a CHAR in YMM1.  */
513694
-	VPCMP	$0, %YMM1, %YMMZERO, %k1
513694
+	VPTESTN	%YMM1, %YMM1, %k1
513694
 	kortestd	%k0, %k1
513694
 	jnz	L(first_vec_x4)
513694
 
513694
@@ -265,33 +275,33 @@ L(loop_4x_vec):
513694
 	VPMINU	%YMM3, %YMM4, %YMM4
513694
 	VPMINU	%YMM2, %YMM4, %YMM4{%k4}{z}
513694
 
513694
-	VPCMP	$0, %YMMZERO, %YMM4, %k1
513694
+	VPTESTN	%YMM4, %YMM4, %k1
513694
 	kmovd	%k1, %ecx
513694
 	subq	$-(VEC_SIZE * 4), %rdi
513694
 	testl	%ecx, %ecx
513694
 	jz	L(loop_4x_vec)
513694
 
513694
-	VPCMP	$0, %YMMZERO, %YMM1, %k0
513694
+	VPTESTN	%YMM1, %YMM1, %k0
513694
 	kmovd	%k0, %eax
513694
 	testl	%eax, %eax
513694
 	jnz	L(last_vec_x1)
513694
 
513694
-	VPCMP	$0, %YMMZERO, %YMM2, %k0
513694
+	VPTESTN	%YMM2, %YMM2, %k0
513694
 	kmovd	%k0, %eax
513694
 	testl	%eax, %eax
513694
 	jnz	L(last_vec_x2)
513694
 
513694
-	VPCMP	$0, %YMMZERO, %YMM3, %k0
513694
+	VPTESTN	%YMM3, %YMM3, %k0
513694
 	kmovd	%k0, %eax
513694
 	/* Combine YMM3 matches (eax) with YMM4 matches (ecx).  */
513694
 # ifdef USE_AS_WCSCHR
513694
 	sall	$8, %ecx
513694
 	orl	%ecx, %eax
513694
-	tzcntl	%eax, %eax
513694
+	bsfl	%eax, %eax
513694
 # else
513694
 	salq	$32, %rcx
513694
 	orq	%rcx, %rax
513694
-	tzcntq	%rax, %rax
513694
+	bsfq	%rax, %rax
513694
 # endif
513694
 # ifndef USE_AS_STRCHRNUL
513694
 	/* Check if match was CHAR or null.  */
513694
@@ -303,28 +313,28 @@ L(loop_4x_vec):
513694
 	leaq	(VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %rax
513694
 	ret
513694
 
513694
-# ifndef USE_AS_STRCHRNUL
513694
-L(zero_end):
513694
-	xorl	%eax, %eax
513694
-	ret
513694
+	.p2align 4,, 8
513694
+L(last_vec_x1):
513694
+	bsfl	%eax, %eax
513694
+# ifdef USE_AS_WCSCHR
513694
+	/* NB: Multiply wchar_t count by 4 to get the number of bytes.
513694
+	   */
513694
+	leaq	(%rdi, %rax, CHAR_SIZE), %rax
513694
+# else
513694
+	addq	%rdi, %rax
513694
 # endif
513694
 
513694
-	.p2align 4
513694
-L(last_vec_x1):
513694
-	tzcntl	%eax, %eax
513694
 # ifndef USE_AS_STRCHRNUL
513694
 	/* Check if match was null.  */
513694
-	cmp	(%rdi, %rax, CHAR_SIZE), %CHAR_REG
513694
+	cmp	(%rax), %CHAR_REG
513694
 	jne	L(zero_end)
513694
 # endif
513694
-	/* NB: Multiply sizeof char type (1 or 4) to get the number of
513694
-	   bytes.  */
513694
-	leaq	(%rdi, %rax, CHAR_SIZE), %rax
513694
+
513694
 	ret
513694
 
513694
-	.p2align 4
513694
+	.p2align 4,, 8
513694
 L(last_vec_x2):
513694
-	tzcntl	%eax, %eax
513694
+	bsfl	%eax, %eax
513694
 # ifndef USE_AS_STRCHRNUL
513694
 	/* Check if match was null.  */
513694
 	cmp	(VEC_SIZE)(%rdi, %rax, CHAR_SIZE), %CHAR_REG
513694
@@ -336,7 +346,7 @@ L(last_vec_x2):
513694
 	ret
513694
 
513694
 	/* Cold case for crossing page with first load.	 */
513694
-	.p2align 4
513694
+	.p2align 4,, 8
513694
 L(cross_page_boundary):
513694
 	movq	%rdi, %rdx
513694
 	/* Align rdi.  */
513694
@@ -346,9 +356,9 @@ L(cross_page_boundary):
513694
 	vpxorq	%YMM1, %YMM0, %YMM2
513694
 	VPMINU	%YMM2, %YMM1, %YMM2
513694
 	/* Each bit in K0 represents a CHAR or a null byte in YMM1.  */
513694
-	VPCMP	$0, %YMMZERO, %YMM2, %k0
513694
+	VPTESTN	%YMM2, %YMM2, %k0
513694
 	kmovd	%k0, %eax
513694
-	/* Remove the leading bits.	 */
513694
+	/* Remove the leading bits.  */
513694
 # ifdef USE_AS_WCSCHR
513694
 	movl	%edx, %SHIFT_REG
513694
 	/* NB: Divide shift count by 4 since each bit in K1 represent 4
513694
@@ -360,20 +370,24 @@ L(cross_page_boundary):
513694
 	/* If eax is zero continue.  */
513694
 	testl	%eax, %eax
513694
 	jz	L(cross_page_continue)
513694
-	tzcntl	%eax, %eax
513694
-# ifndef USE_AS_STRCHRNUL
513694
-	/* Check to see if match was CHAR or null.  */
513694
-	cmp	(%rdx, %rax, CHAR_SIZE), %CHAR_REG
513694
-	jne	L(zero_end)
513694
-# endif
513694
+	bsfl	%eax, %eax
513694
+
513694
 # ifdef USE_AS_WCSCHR
513694
 	/* NB: Multiply wchar_t count by 4 to get the number of
513694
 	   bytes.  */
513694
 	leaq	(%rdx, %rax, CHAR_SIZE), %rax
513694
 # else
513694
 	addq	%rdx, %rax
513694
+# endif
513694
+# ifndef USE_AS_STRCHRNUL
513694
+	/* Check to see if match was CHAR or null.  */
513694
+	cmp	(%rax), %CHAR_REG
513694
+	je	L(cross_page_ret)
513694
+L(zero_end):
513694
+	xorl	%eax, %eax
513694
+L(cross_page_ret):
513694
 # endif
513694
 	ret
513694
 
513694
 END (STRCHR)
513694
-# endif
513694
+#endif
513694
-- 
513694
GitLab
513694