From 6c6a4eae9d8131531c9231f9f177d2db4130df01 Mon Sep 17 00:00:00 2001

From: Noah Goldstein <goldstein.w.n@gmail.com>

Date: Fri, 24 Dec 2021 18:54:41 -0600

Subject: [PATCH] x86: Optimize L(less_vec) case in memcmp-evex-movbe.S

No bug.

Optimizations are twofold.

1) Replace page cross and 0/1 checks with masked load instructions in

   L(less_vec). In applications this reduces branch-misses in the

   hot [0, 32] case.

2) Change controlflow so that L(less_vec) case gets the fall through.

Change 2) helps copies in the [0, 32] size range but comes at the cost

of copies in the [33, 64] size range.  From profiles of GCC and

Python3, 94%+ and 99%+ of calls are in the [0, 32] range so this

appears to the the right tradeoff.

Signed-off-by: Noah Goldstein <goldstein.w.n@gmail.com>

Reviewed-by: H.J. Lu <hjl.tools@gmail.com>

(cherry picked from commit abddd61de090ae84e380aff68a98bd94ef704667)

---

 sysdeps/x86_64/multiarch/memcmp-evex-movbe.S | 249 +++++--------------

 1 file changed, 56 insertions(+), 193 deletions(-)

diff --git a/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S b/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S

index 640f6757..d2899e7c 100644

--- a/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S

+++ b/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S

@@ -62,15 +62,18 @@ Latency:

 # define VMOVU		vmovdqu64

 # ifdef USE_AS_WMEMCMP

+#  define VMOVU_MASK	vmovdqu32

 #  define CHAR_SIZE	4

 #  define VPCMP	vpcmpd

 #  define VPTEST	vptestmd

 # else

+#  define VMOVU_MASK	vmovdqu8

 #  define CHAR_SIZE	1

 #  define VPCMP	vpcmpub

 #  define VPTEST	vptestmb

 # endif

+

 # define VEC_SIZE	32

 # define PAGE_SIZE	4096

 # define CHAR_PER_VEC	(VEC_SIZE / CHAR_SIZE)

@@ -102,12 +105,48 @@ ENTRY_P2ALIGN (MEMCMP, 6)

 	movl	%edx, %edx

 # endif

 	cmp	$CHAR_PER_VEC, %RDX_LP

-	jb	L(less_vec)

+	/* Fall through for [0, VEC_SIZE] as its the hottest.  */

+	ja	L(more_1x_vec)

+

+	/* Create mask for CHAR's we want to compare. This allows us to

+	   avoid having to include page cross logic.  */

+	movl	$-1, %ecx

+	bzhil	%edx, %ecx, %ecx

+	kmovd	%ecx, %k2

+

+	/* Safe to load full ymm with mask.  */

+	VMOVU_MASK (%rsi), %YMM2{%k2}

+	VPCMP	$4,(%rdi), %YMM2, %k1{%k2}

+	kmovd	%k1, %eax

+	testl	%eax, %eax

+	jnz	L(return_vec_0)

+	ret

+	.p2align 4

+L(return_vec_0):

+	tzcntl	%eax, %eax

+# ifdef USE_AS_WMEMCMP

+	movl	(%rdi, %rax, CHAR_SIZE), %ecx

+	xorl	%edx, %edx

+	cmpl	(%rsi, %rax, CHAR_SIZE), %ecx

+	/* NB: no partial register stall here because xorl zero idiom

+	   above.  */

+	setg	%dl

+	leal	-1(%rdx, %rdx), %eax

+# else

+	movzbl	(%rsi, %rax), %ecx

+	movzbl	(%rdi, %rax), %eax

+	subl	%ecx, %eax

+# endif

+	ret

+

+

+	.p2align 4

+L(more_1x_vec):

 	/* From VEC to 2 * VEC.  No branch when size == VEC_SIZE.  */

 	VMOVU	(%rsi), %YMM1

 	/* Use compare not equals to directly check for mismatch.  */

-	VPCMP	$4, (%rdi), %YMM1, %k1

+	VPCMP	$4,(%rdi), %YMM1, %k1

 	kmovd	%k1, %eax

 	/* NB: eax must be destination register if going to

 	   L(return_vec_[0,2]). For L(return_vec_3) destination register

@@ -131,13 +170,13 @@ ENTRY_P2ALIGN (MEMCMP, 6)

 	/* Check third and fourth VEC no matter what.  */

 	VMOVU	(VEC_SIZE * 2)(%rsi), %YMM3

-	VPCMP	$4, (VEC_SIZE * 2)(%rdi), %YMM3, %k1

+	VPCMP	$4,(VEC_SIZE * 2)(%rdi), %YMM3, %k1

 	kmovd	%k1, %eax

 	testl	%eax, %eax

 	jnz	L(return_vec_2)

 	VMOVU	(VEC_SIZE * 3)(%rsi), %YMM4

-	VPCMP	$4, (VEC_SIZE * 3)(%rdi), %YMM4, %k1

+	VPCMP	$4,(VEC_SIZE * 3)(%rdi), %YMM4, %k1

 	kmovd	%k1, %ecx

 	testl	%ecx, %ecx

 	jnz	L(return_vec_3)

@@ -169,7 +208,7 @@ ENTRY_P2ALIGN (MEMCMP, 6)

 	VMOVU	(VEC_SIZE * 3)(%rsi), %YMM4

 	/* Ternary logic to xor (VEC_SIZE * 3)(%rdi) with YMM4 while

 	   oring with YMM1. Result is stored in YMM4.  */

-	vpternlogd $0xde, (VEC_SIZE * 3)(%rdi), %YMM1, %YMM4

+	vpternlogd $0xde,(VEC_SIZE * 3)(%rdi), %YMM1, %YMM4

 	/* Or together YMM2, YMM3, and YMM4 into YMM4.  */

 	vpternlogd $0xfe, %YMM2, %YMM3, %YMM4

@@ -184,7 +223,8 @@ ENTRY_P2ALIGN (MEMCMP, 6)

 	/* NB: eax must be zero to reach here.  */

 	ret

-	.p2align 4

+

+	.p2align 4,, 8

 L(8x_end_return_vec_0_1_2_3):

 	movq	%rdx, %rdi

 L(8x_return_vec_0_1_2_3):

@@ -222,23 +262,6 @@ L(return_vec_3):

 # endif

 	ret

-	.p2align 4

-L(return_vec_0):

-	tzcntl	%eax, %eax

-# ifdef USE_AS_WMEMCMP

-	movl	(%rdi, %rax, CHAR_SIZE), %ecx

-	xorl	%edx, %edx

-	cmpl	(%rsi, %rax, CHAR_SIZE), %ecx

-	/* NB: no partial register stall here because xorl zero idiom

-	   above.  */

-	setg	%dl

-	leal	-1(%rdx, %rdx), %eax

-# else

-	movzbl	(%rsi, %rax), %ecx

-	movzbl	(%rdi, %rax), %eax

-	subl	%ecx, %eax

-# endif

-	ret

 	.p2align 4

 L(return_vec_1):

@@ -297,7 +320,7 @@ L(loop_4x_vec):

 	VMOVU	(VEC_SIZE * 2)(%rsi, %rdi), %YMM3

 	vpxorq	(VEC_SIZE * 2)(%rdi), %YMM3, %YMM3

 	VMOVU	(VEC_SIZE * 3)(%rsi, %rdi), %YMM4

-	vpternlogd $0xde, (VEC_SIZE * 3)(%rdi), %YMM1, %YMM4

+	vpternlogd $0xde,(VEC_SIZE * 3)(%rdi), %YMM1, %YMM4

 	vpternlogd $0xfe, %YMM2, %YMM3, %YMM4

 	VPTEST	%YMM4, %YMM4, %k1

 	kmovd	%k1, %ecx

@@ -324,7 +347,7 @@ L(loop_4x_vec):

 	VMOVU	VEC_SIZE(%rsi, %rdx), %YMM2

 	vpxorq	VEC_SIZE(%rdx), %YMM2, %YMM2

 	VMOVU	(VEC_SIZE * 3)(%rsi, %rdx), %YMM4

-	vpternlogd $0xde, (VEC_SIZE * 3)(%rdx), %YMM1, %YMM4

+	vpternlogd $0xde,(VEC_SIZE * 3)(%rdx), %YMM1, %YMM4

 	vpternlogd $0xfe, %YMM2, %YMM3, %YMM4

 	VPTEST	%YMM4, %YMM4, %k1

 	kmovd	%k1, %ecx

@@ -336,14 +359,14 @@ L(loop_4x_vec):

 	/* Only entry is from L(more_8x_vec).  */

 	.p2align 4,, 10

 L(8x_last_2x_vec):

-	VPCMP	$4, (VEC_SIZE * 2)(%rdx), %YMM3, %k1

+	VPCMP	$4,(VEC_SIZE * 2)(%rdx), %YMM3, %k1

 	kmovd	%k1, %eax

 	testl	%eax, %eax

 	jnz	L(8x_return_vec_2)

 	/* Naturally aligned to 16 bytes.  */

 L(8x_last_1x_vec):

 	VMOVU	(VEC_SIZE * 3)(%rsi, %rdx), %YMM1

-	VPCMP	$4, (VEC_SIZE * 3)(%rdx), %YMM1, %k1

+	VPCMP	$4,(VEC_SIZE * 3)(%rdx), %YMM1, %k1

 	kmovd	%k1, %eax

 	testl	%eax, %eax

 	jnz	L(8x_return_vec_3)

@@ -392,7 +415,9 @@ L(last_1x_vec):

 	jnz	L(return_vec_0_end)

 	ret

-	.p2align 4,, 10

+

+	/* Don't align. Takes 2-fetch blocks either way and aligning

+	   will cause code to spill into another cacheline.  */

 L(return_vec_1_end):

 	/* Use bsf to save code size. This is necessary to have

 	   L(one_or_less) fit in aligning bytes between.  */

@@ -411,31 +436,8 @@ L(return_vec_1_end):

 # endif

 	ret

-	/* NB: L(one_or_less) fits in alignment padding between

-	   L(return_vec_1_end) and L(return_vec_0_end).  */

-# ifdef USE_AS_WMEMCMP

-L(one_or_less):

-	jb	L(zero)

-	movl	(%rdi), %ecx

-	xorl	%edx, %edx

-	cmpl	(%rsi), %ecx

-	je	L(zero)

-	setg	%dl

-	leal	-1(%rdx, %rdx), %eax

-	ret

-# else

-L(one_or_less):

-	jb	L(zero)

-	movzbl	(%rsi), %ecx

-	movzbl	(%rdi), %eax

-	subl	%ecx, %eax

-	ret

-# endif

-L(zero):

-	xorl	%eax, %eax

-	ret

-

-	.p2align 4

+	/* Don't align. Takes 2-fetch blocks either way and aligning

+	   will cause code to spill into another cacheline.  */

 L(return_vec_0_end):

 	tzcntl	%eax, %eax

 	addl	%edx, %eax

@@ -451,146 +453,7 @@ L(return_vec_0_end):

 	subl	%ecx, %eax

 # endif

 	ret

+	/* 1-byte until next cache line.  */

-	.p2align 4

-L(less_vec):

-	/* Check if one or less CHAR. This is necessary for size == 0

-	   but is also faster for size == CHAR_SIZE.  */

-	cmpl	$1, %edx

-	jbe	L(one_or_less)

-

-	/* Check if loading one VEC from either s1 or s2 could cause a

-	   page cross. This can have false positives but is by far the

-	   fastest method.  */

-	movl	%edi, %eax

-	orl	%esi, %eax

-	andl	$(PAGE_SIZE - 1), %eax

-	cmpl	$(PAGE_SIZE - VEC_SIZE), %eax

-	jg	L(page_cross_less_vec)

-

-	/* No page cross possible.  */

-	VMOVU	(%rsi), %YMM2

-	VPCMP	$4, (%rdi), %YMM2, %k1

-	kmovd	%k1, %eax

-	/* Check if any matches where in bounds. Intentionally not

-	   storing result in eax to limit dependency chain if it goes to

-	   L(return_vec_0_lv).  */

-	bzhil	%edx, %eax, %edx

-	jnz	L(return_vec_0_lv)

-	xorl	%eax, %eax

-	ret

-

-	/* Essentially duplicate of L(return_vec_0). Ends up not costing

-	   any code as shrinks L(less_vec) by allowing 2-byte encoding of

-	   the jump and ends up fitting in aligning bytes. As well fits on

-	   same cache line as L(less_vec) so also saves a line from having

-	   to be fetched on cold calls to memcmp.  */

-	.p2align 4,, 4

-L(return_vec_0_lv):

-	tzcntl	%eax, %eax

-# ifdef USE_AS_WMEMCMP

-	movl	(%rdi, %rax, CHAR_SIZE), %ecx

-	xorl	%edx, %edx

-	cmpl	(%rsi, %rax, CHAR_SIZE), %ecx

-	/* NB: no partial register stall here because xorl zero idiom

-	   above.  */

-	setg	%dl

-	leal	-1(%rdx, %rdx), %eax

-# else

-	movzbl	(%rsi, %rax), %ecx

-	movzbl	(%rdi, %rax), %eax

-	subl	%ecx, %eax

-# endif

-	ret

-

-	.p2align 4

-L(page_cross_less_vec):

-	/* if USE_AS_WMEMCMP it can only be 0, 4, 8, 12, 16, 20, 24, 28

-	   bytes.  */

-	cmpl	$(16 / CHAR_SIZE), %edx

-	jae	L(between_16_31)

-# ifndef USE_AS_WMEMCMP

-	cmpl	$8, %edx

-	jae	L(between_8_15)

-	cmpl	$4, %edx

-	jb	L(between_2_3)

-

-	/* Load as big endian with overlapping movbe to avoid branches.

-	 */

-	movbe	(%rdi), %eax

-	movbe	(%rsi), %ecx

-	shlq	$32, %rax

-	shlq	$32, %rcx

-	movbe	-4(%rdi, %rdx), %edi

-	movbe	-4(%rsi, %rdx), %esi

-	orq	%rdi, %rax

-	orq	%rsi, %rcx

-	subq	%rcx, %rax

-	/* edx is guranteed to be positive int32 in range [4, 7].  */

-	cmovne	%edx, %eax

-	/* ecx is -1 if rcx > rax. Otherwise 0.  */

-	sbbl	%ecx, %ecx

-	/* If rcx > rax, then ecx is 0 and eax is positive. If rcx ==

-	   rax then eax and ecx are zero. If rax < rax then ecx is -1 so

-	   eax doesn't matter.  */

-	orl	%ecx, %eax

-	ret

-

-	.p2align 4,, 8

-L(between_8_15):

-# endif

-	/* If USE_AS_WMEMCMP fall through into 8-15 byte case.  */

-	vmovq	(%rdi), %xmm1

-	vmovq	(%rsi), %xmm2

-	VPCMP	$4, %xmm1, %xmm2, %k1

-	kmovd	%k1, %eax

-	testl	%eax, %eax

-	jnz	L(return_vec_0_lv)

-	/* Use overlapping loads to avoid branches.  */

-	vmovq	-8(%rdi, %rdx, CHAR_SIZE), %xmm1

-	vmovq	-8(%rsi, %rdx, CHAR_SIZE), %xmm2

-	VPCMP	$4, %xmm1, %xmm2, %k1

-	addl	$(CHAR_PER_VEC - (8 / CHAR_SIZE)), %edx

-	kmovd	%k1, %eax

-	testl	%eax, %eax

-	jnz	L(return_vec_0_end)

-	ret

-

-	.p2align 4,, 8

-L(between_16_31):

-	/* From 16 to 31 bytes.  No branch when size == 16.  */

-

-	/* Use movups to save code size.  */

-	vmovdqu	(%rsi), %xmm2

-	VPCMP	$4, (%rdi), %xmm2, %k1

-	kmovd	%k1, %eax

-	testl	%eax, %eax

-	jnz	L(return_vec_0_lv)

-	/* Use overlapping loads to avoid branches.  */

-	vmovdqu	-16(%rsi, %rdx, CHAR_SIZE), %xmm2

-	VPCMP	$4, -16(%rdi, %rdx, CHAR_SIZE), %xmm2, %k1

-	addl	$(CHAR_PER_VEC - (16 / CHAR_SIZE)), %edx

-	kmovd	%k1, %eax

-	testl	%eax, %eax

-	jnz	L(return_vec_0_end)

-	ret

-

-# ifndef USE_AS_WMEMCMP

-L(between_2_3):

-	/* Load as big endian to avoid branches.  */

-	movzwl	(%rdi), %eax

-	movzwl	(%rsi), %ecx

-	shll	$8, %eax

-	shll	$8, %ecx

-	bswap	%eax

-	bswap	%ecx

-	movzbl	-1(%rdi, %rdx), %edi

-	movzbl	-1(%rsi, %rdx), %esi

-	orl	%edi, %eax

-	orl	%esi, %ecx

-	/* Subtraction is okay because the upper 8 bits are zero.  */

-	subl	%ecx, %eax

-	ret

-# endif

 END (MEMCMP)

 #endif

-- 

GitLab