commit f35ad30da4880a1574996df0674986ecf82fa7ae

Author: H.J. Lu <hjl.tools@gmail.com>

Date:   Fri Oct 29 12:40:20 2021 -0700

    x86-64: Improve EVEX strcmp with masked load

    In strcmp-evex.S, to compare 2 32-byte strings, replace

            VMOVU   (%rdi, %rdx), %YMM0

            VMOVU   (%rsi, %rdx), %YMM1

            /* Each bit in K0 represents a mismatch in YMM0 and YMM1.  */

            VPCMP   $4, %YMM0, %YMM1, %k0

            VPCMP   $0, %YMMZERO, %YMM0, %k1

            VPCMP   $0, %YMMZERO, %YMM1, %k2

            /* Each bit in K1 represents a NULL in YMM0 or YMM1.  */

            kord    %k1, %k2, %k1

            /* Each bit in K1 represents a NULL or a mismatch.  */

            kord    %k0, %k1, %k1

            kmovd   %k1, %ecx

            testl   %ecx, %ecx

            jne     L(last_vector)

    with

            VMOVU   (%rdi, %rdx), %YMM0

            VPTESTM %YMM0, %YMM0, %k2

            /* Each bit cleared in K1 represents a mismatch or a null CHAR

               in YMM0 and 32 bytes at (%rsi, %rdx).  */

            VPCMP   $0, (%rsi, %rdx), %YMM0, %k1{%k2}

            kmovd   %k1, %ecx

            incl    %ecx

            jne     L(last_vector)

    It makes EVEX strcmp faster than AVX2 strcmp by up to 40% on Tiger Lake

    and Ice Lake.

    Co-Authored-By: Noah Goldstein <goldstein.w.n@gmail.com>

    (cherry picked from commit c46e9afb2df5fc9e39ff4d13777e4b4c26e04e55)

diff --git a/sysdeps/x86_64/multiarch/strcmp-evex.S b/sysdeps/x86_64/multiarch/strcmp-evex.S

index d5aa6daa46c7ed25..82f12ac89bcae20b 100644

--- a/sysdeps/x86_64/multiarch/strcmp-evex.S

+++ b/sysdeps/x86_64/multiarch/strcmp-evex.S

@@ -41,6 +41,8 @@

 # ifdef USE_AS_WCSCMP

 /* Compare packed dwords.  */

 #  define VPCMP		vpcmpd

+#  define VPMINU	vpminud

+#  define VPTESTM	vptestmd

 #  define SHIFT_REG32	r8d

 #  define SHIFT_REG64	r8

 /* 1 dword char == 4 bytes.  */

@@ -48,6 +50,8 @@

 # else

 /* Compare packed bytes.  */

 #  define VPCMP		vpcmpb

+#  define VPMINU	vpminub

+#  define VPTESTM	vptestmb

 #  define SHIFT_REG32	ecx

 #  define SHIFT_REG64	rcx

 /* 1 byte char == 1 byte.  */

@@ -67,6 +71,9 @@

 # define YMM5		ymm22

 # define YMM6		ymm23

 # define YMM7		ymm24

+# define YMM8		ymm25

+# define YMM9		ymm26

+# define YMM10		ymm27

 /* Warning!

            wcscmp/wcsncmp have to use SIGNED comparison for elements.

@@ -76,7 +83,7 @@

 /* The main idea of the string comparison (byte or dword) using 256-bit

    EVEX instructions consists of comparing (VPCMP) two ymm vectors. The

    latter can be on either packed bytes or dwords depending on

-   USE_AS_WCSCMP. In order to check the null char, algorithm keeps the

+   USE_AS_WCSCMP. In order to check the null CHAR, algorithm keeps the

    matched bytes/dwords, requiring 5 EVEX instructions (3 VPCMP and 2

    KORD). In general, the costs of comparing VEC_SIZE bytes (32-bytes)

    are 3 VPCMP and 2 KORD instructions, together with VMOVU and ktestd

@@ -123,27 +130,21 @@ ENTRY (STRCMP)

 	jg	L(cross_page)

 	/* Start comparing 4 vectors.  */

 	VMOVU	(%rdi), %YMM0

-	VMOVU	(%rsi), %YMM1

-	/* Each bit in K0 represents a mismatch in YMM0 and YMM1.  */

-	VPCMP	$4, %YMM0, %YMM1, %k0

+	/* Each bit set in K2 represents a non-null CHAR in YMM0.  */

+	VPTESTM	%YMM0, %YMM0, %k2

-	/* Check for NULL in YMM0.  */

-	VPCMP	$0, %YMMZERO, %YMM0, %k1

-	/* Check for NULL in YMM1.  */

-	VPCMP	$0, %YMMZERO, %YMM1, %k2

-	/* Each bit in K1 represents a NULL in YMM0 or YMM1.  */

-	kord	%k1, %k2, %k1

+	/* Each bit cleared in K1 represents a mismatch or a null CHAR

+	   in YMM0 and 32 bytes at (%rsi).  */

+	VPCMP	$0, (%rsi), %YMM0, %k1{%k2}

-	/* Each bit in K1 represents:

-	   1. A mismatch in YMM0 and YMM1.  Or

-	   2. A NULL in YMM0 or YMM1.

-	 */

-	kord	%k0, %k1, %k1

-

-	ktestd	%k1, %k1

-	je	L(next_3_vectors)

 	kmovd	%k1, %ecx

+# ifdef USE_AS_WCSCMP

+	subl	$0xff, %ecx

+# else

+	incl	%ecx

+# endif

+	je	L(next_3_vectors)

 	tzcntl	%ecx, %edx

 # ifdef USE_AS_WCSCMP

 	/* NB: Multiply wchar_t count by 4 to get the number of bytes.  */

@@ -172,9 +173,7 @@ L(return):

 # endif

 	ret

-	.p2align 4

 L(return_vec_size):

-	kmovd	%k1, %ecx

 	tzcntl	%ecx, %edx

 # ifdef USE_AS_WCSCMP

 	/* NB: Multiply wchar_t count by 4 to get the number of bytes.  */

@@ -210,9 +209,7 @@ L(return_vec_size):

 # endif

 	ret

-	.p2align 4

 L(return_2_vec_size):

-	kmovd	%k1, %ecx

 	tzcntl	%ecx, %edx

 # ifdef USE_AS_WCSCMP

 	/* NB: Multiply wchar_t count by 4 to get the number of bytes.  */

@@ -248,9 +245,7 @@ L(return_2_vec_size):

 # endif

 	ret

-	.p2align 4

 L(return_3_vec_size):

-	kmovd	%k1, %ecx

 	tzcntl	%ecx, %edx

 # ifdef USE_AS_WCSCMP

 	/* NB: Multiply wchar_t count by 4 to get the number of bytes.  */

@@ -289,43 +284,45 @@ L(return_3_vec_size):

 	.p2align 4

 L(next_3_vectors):

 	VMOVU	VEC_SIZE(%rdi), %YMM0

-	VMOVU	VEC_SIZE(%rsi), %YMM1

-	/* Each bit in K0 represents a mismatch in YMM0 and YMM1.  */

-	VPCMP	$4, %YMM0, %YMM1, %k0

-	VPCMP	$0, %YMMZERO, %YMM0, %k1

-	VPCMP	$0, %YMMZERO, %YMM1, %k2

-	/* Each bit in K1 represents a NULL in YMM0 or YMM1.  */

-	kord	%k1, %k2, %k1

-	/* Each bit in K1 represents a NULL or a mismatch.  */

-	kord	%k0, %k1, %k1

-	ktestd	%k1, %k1

+	/* Each bit set in K2 represents a non-null CHAR in YMM0.  */

+	VPTESTM	%YMM0, %YMM0, %k2

+	/* Each bit cleared in K1 represents a mismatch or a null CHAR

+	   in YMM0 and 32 bytes at VEC_SIZE(%rsi).  */

+	VPCMP	$0, VEC_SIZE(%rsi), %YMM0, %k1{%k2}

+	kmovd	%k1, %ecx

+# ifdef USE_AS_WCSCMP

+	subl	$0xff, %ecx

+# else

+	incl	%ecx

+# endif

 	jne	L(return_vec_size)

-	VMOVU	(VEC_SIZE * 2)(%rdi), %YMM2

-	VMOVU	(VEC_SIZE * 3)(%rdi), %YMM3

-	VMOVU	(VEC_SIZE * 2)(%rsi), %YMM4

-	VMOVU	(VEC_SIZE * 3)(%rsi), %YMM5

-

-	/* Each bit in K0 represents a mismatch in YMM2 and YMM4.  */

-	VPCMP	$4, %YMM2, %YMM4, %k0

-	VPCMP	$0, %YMMZERO, %YMM2, %k1

-	VPCMP	$0, %YMMZERO, %YMM4, %k2

-	/* Each bit in K1 represents a NULL in YMM2 or YMM4.  */

-	kord	%k1, %k2, %k1

-	/* Each bit in K1 represents a NULL or a mismatch.  */

-	kord	%k0, %k1, %k1

-	ktestd	%k1, %k1

+	VMOVU	(VEC_SIZE * 2)(%rdi), %YMM0

+	/* Each bit set in K2 represents a non-null CHAR in YMM0.  */

+	VPTESTM	%YMM0, %YMM0, %k2

+	/* Each bit cleared in K1 represents a mismatch or a null CHAR

+	   in YMM0 and 32 bytes at (VEC_SIZE * 2)(%rsi).  */

+	VPCMP	$0, (VEC_SIZE * 2)(%rsi), %YMM0, %k1{%k2}

+	kmovd	%k1, %ecx

+# ifdef USE_AS_WCSCMP

+	subl	$0xff, %ecx

+# else

+	incl	%ecx

+# endif

 	jne	L(return_2_vec_size)

-	/* Each bit in K0 represents a mismatch in YMM3 and YMM5.  */

-	VPCMP	$4, %YMM3, %YMM5, %k0

-	VPCMP	$0, %YMMZERO, %YMM3, %k1

-	VPCMP	$0, %YMMZERO, %YMM5, %k2

-	/* Each bit in K1 represents a NULL in YMM3 or YMM5.  */

-	kord	%k1, %k2, %k1

-	/* Each bit in K1 represents a NULL or a mismatch.  */

-	kord	%k0, %k1, %k1

-	ktestd	%k1, %k1

+	VMOVU	(VEC_SIZE * 3)(%rdi), %YMM0

+	/* Each bit set in K2 represents a non-null CHAR in YMM0.  */

+	VPTESTM	%YMM0, %YMM0, %k2

+	/* Each bit cleared in K1 represents a mismatch or a null CHAR

+	   in YMM0 and 32 bytes at (VEC_SIZE * 2)(%rsi).  */

+	VPCMP	$0, (VEC_SIZE * 3)(%rsi), %YMM0, %k1{%k2}

+	kmovd	%k1, %ecx

+# ifdef USE_AS_WCSCMP

+	subl	$0xff, %ecx

+# else

+	incl	%ecx

+# endif

 	jne	L(return_3_vec_size)

 L(main_loop_header):

 	leaq	(VEC_SIZE * 4)(%rdi), %rdx

@@ -375,56 +372,51 @@ L(back_to_loop):

 	VMOVA	VEC_SIZE(%rax), %YMM2

 	VMOVA	(VEC_SIZE * 2)(%rax), %YMM4

 	VMOVA	(VEC_SIZE * 3)(%rax), %YMM6

-	VMOVU	(%rdx), %YMM1

-	VMOVU	VEC_SIZE(%rdx), %YMM3

-	VMOVU	(VEC_SIZE * 2)(%rdx), %YMM5

-	VMOVU	(VEC_SIZE * 3)(%rdx), %YMM7

-

-	VPCMP	$4, %YMM0, %YMM1, %k0

-	VPCMP	$0, %YMMZERO, %YMM0, %k1

-	VPCMP	$0, %YMMZERO, %YMM1, %k2

-	kord	%k1, %k2, %k1

-	/* Each bit in K4 represents a NULL or a mismatch in YMM0 and

-	   YMM1.  */

-	kord	%k0, %k1, %k4

-

-	VPCMP	$4, %YMM2, %YMM3, %k0

-	VPCMP	$0, %YMMZERO, %YMM2, %k1

-	VPCMP	$0, %YMMZERO, %YMM3, %k2

-	kord	%k1, %k2, %k1

-	/* Each bit in K5 represents a NULL or a mismatch in YMM2 and

-	   YMM3.  */

-	kord	%k0, %k1, %k5

-

-	VPCMP	$4, %YMM4, %YMM5, %k0

-	VPCMP	$0, %YMMZERO, %YMM4, %k1

-	VPCMP	$0, %YMMZERO, %YMM5, %k2

-	kord	%k1, %k2, %k1

-	/* Each bit in K6 represents a NULL or a mismatch in YMM4 and

-	   YMM5.  */

-	kord	%k0, %k1, %k6

-

-	VPCMP	$4, %YMM6, %YMM7, %k0

-	VPCMP	$0, %YMMZERO, %YMM6, %k1

-	VPCMP	$0, %YMMZERO, %YMM7, %k2

-	kord	%k1, %k2, %k1

-	/* Each bit in K7 represents a NULL or a mismatch in YMM6 and

-	   YMM7.  */

-	kord	%k0, %k1, %k7

-

-	kord	%k4, %k5, %k0

-	kord	%k6, %k7, %k1

-

-	/* Test each mask (32 bits) individually because for VEC_SIZE

-	   == 32 is not possible to OR the four masks and keep all bits

-	   in a 64-bit integer register, differing from SSE2 strcmp

-	   where ORing is possible.  */

-	kortestd %k0, %k1

-	je	L(loop)

-	ktestd	%k4, %k4

+

+	VPMINU	%YMM0, %YMM2, %YMM8

+	VPMINU	%YMM4, %YMM6, %YMM9

+

+	/* A zero CHAR in YMM8 means that there is a null CHAR.  */

+	VPMINU	%YMM8, %YMM9, %YMM8

+

+	/* Each bit set in K1 represents a non-null CHAR in YMM8.  */

+	VPTESTM	%YMM8, %YMM8, %k1

+

+	/* (YMM ^ YMM): A non-zero CHAR represents a mismatch.  */

+	vpxorq	(%rdx), %YMM0, %YMM1

+	vpxorq	VEC_SIZE(%rdx), %YMM2, %YMM3

+	vpxorq	(VEC_SIZE * 2)(%rdx), %YMM4, %YMM5

+	vpxorq	(VEC_SIZE * 3)(%rdx), %YMM6, %YMM7

+

+	vporq	%YMM1, %YMM3, %YMM9

+	vporq	%YMM5, %YMM7, %YMM10

+

+	/* A non-zero CHAR in YMM9 represents a mismatch.  */

+	vporq	%YMM9, %YMM10, %YMM9

+

+	/* Each bit cleared in K0 represents a mismatch or a null CHAR.  */

+	VPCMP	$0, %YMMZERO, %YMM9, %k0{%k1}

+	kmovd   %k0, %ecx

+# ifdef USE_AS_WCSCMP

+	subl	$0xff, %ecx

+# else

+	incl	%ecx

+# endif

+	je	 L(loop)

+

+	/* Each bit set in K1 represents a non-null CHAR in YMM0.  */

+	VPTESTM	%YMM0, %YMM0, %k1

+	/* Each bit cleared in K0 represents a mismatch or a null CHAR

+	   in YMM0 and (%rdx).  */

+	VPCMP	$0, %YMMZERO, %YMM1, %k0{%k1}

+	kmovd	%k0, %ecx

+# ifdef USE_AS_WCSCMP

+	subl	$0xff, %ecx

+# else

+	incl	%ecx

+# endif

 	je	L(test_vec)

-	kmovd	%k4, %edi

-	tzcntl	%edi, %ecx

+	tzcntl	%ecx, %ecx

 # ifdef USE_AS_WCSCMP

 	/* NB: Multiply wchar_t count by 4 to get the number of bytes.  */

 	sall	$2, %ecx

@@ -466,9 +458,18 @@ L(test_vec):

 	cmpq	$VEC_SIZE, %r11

 	jbe	L(zero)

 # endif

-	ktestd	%k5, %k5

+	/* Each bit set in K1 represents a non-null CHAR in YMM2.  */

+	VPTESTM	%YMM2, %YMM2, %k1

+	/* Each bit cleared in K0 represents a mismatch or a null CHAR

+	   in YMM2 and VEC_SIZE(%rdx).  */

+	VPCMP	$0, %YMMZERO, %YMM3, %k0{%k1}

+	kmovd	%k0, %ecx

+# ifdef USE_AS_WCSCMP

+	subl	$0xff, %ecx

+# else

+	incl	%ecx

+# endif

 	je	L(test_2_vec)

-	kmovd	%k5, %ecx

 	tzcntl	%ecx, %edi

 # ifdef USE_AS_WCSCMP

 	/* NB: Multiply wchar_t count by 4 to get the number of bytes.  */

@@ -512,9 +513,18 @@ L(test_2_vec):

 	cmpq	$(VEC_SIZE * 2), %r11

 	jbe	L(zero)

 # endif

-	ktestd	%k6, %k6

+	/* Each bit set in K1 represents a non-null CHAR in YMM4.  */

+	VPTESTM	%YMM4, %YMM4, %k1

+	/* Each bit cleared in K0 represents a mismatch or a null CHAR

+	   in YMM4 and (VEC_SIZE * 2)(%rdx).  */

+	VPCMP	$0, %YMMZERO, %YMM5, %k0{%k1}

+	kmovd	%k0, %ecx

+# ifdef USE_AS_WCSCMP

+	subl	$0xff, %ecx

+# else

+	incl	%ecx

+# endif

 	je	L(test_3_vec)

-	kmovd	%k6, %ecx

 	tzcntl	%ecx, %edi

 # ifdef USE_AS_WCSCMP

 	/* NB: Multiply wchar_t count by 4 to get the number of bytes.  */

@@ -558,8 +568,18 @@ L(test_3_vec):

 	cmpq	$(VEC_SIZE * 3), %r11

 	jbe	L(zero)

 # endif

-	kmovd	%k7, %esi

-	tzcntl	%esi, %ecx

+	/* Each bit set in K1 represents a non-null CHAR in YMM6.  */

+	VPTESTM	%YMM6, %YMM6, %k1

+	/* Each bit cleared in K0 represents a mismatch or a null CHAR

+	   in YMM6 and (VEC_SIZE * 3)(%rdx).  */

+	VPCMP	$0, %YMMZERO, %YMM7, %k0{%k1}

+	kmovd	%k0, %ecx

+# ifdef USE_AS_WCSCMP

+	subl	$0xff, %ecx

+# else

+	incl	%ecx

+# endif

+	tzcntl	%ecx, %ecx

 # ifdef USE_AS_WCSCMP

 	/* NB: Multiply wchar_t count by 4 to get the number of bytes.  */

 	sall	$2, %ecx

@@ -615,39 +635,51 @@ L(loop_cross_page):

 	VMOVU	(%rax, %r10), %YMM2

 	VMOVU	VEC_SIZE(%rax, %r10), %YMM3

-	VMOVU	(%rdx, %r10), %YMM4

-	VMOVU	VEC_SIZE(%rdx, %r10), %YMM5

-

-	VPCMP	$4, %YMM4, %YMM2, %k0

-	VPCMP	$0, %YMMZERO, %YMM2, %k1

-	VPCMP	$0, %YMMZERO, %YMM4, %k2

-	kord	%k1, %k2, %k1

-	/* Each bit in K1 represents a NULL or a mismatch in YMM2 and

-	   YMM4.  */

-	kord	%k0, %k1, %k1

-

-	VPCMP	$4, %YMM5, %YMM3, %k3

-	VPCMP	$0, %YMMZERO, %YMM3, %k4

-	VPCMP	$0, %YMMZERO, %YMM5, %k5

-	kord	%k4, %k5, %k4

-	/* Each bit in K3 represents a NULL or a mismatch in YMM3 and

-	   YMM5.  */

-	kord	%k3, %k4, %k3

+

+	/* Each bit set in K2 represents a non-null CHAR in YMM2.  */

+	VPTESTM	%YMM2, %YMM2, %k2

+	/* Each bit cleared in K1 represents a mismatch or a null CHAR

+	   in YMM2 and 32 bytes at (%rdx, %r10).  */

+	VPCMP	$0, (%rdx, %r10), %YMM2, %k1{%k2}

+	kmovd	%k1, %r9d

+	/* Don't use subl since it is the lower 16/32 bits of RDI

+	   below.  */

+	notl	%r9d

+# ifdef USE_AS_WCSCMP

+	/* Only last 8 bits are valid.  */

+	andl	$0xff, %r9d

+# endif

+

+	/* Each bit set in K4 represents a non-null CHAR in YMM3.  */

+	VPTESTM	%YMM3, %YMM3, %k4

+	/* Each bit cleared in K3 represents a mismatch or a null CHAR

+	   in YMM3 and 32 bytes at VEC_SIZE(%rdx, %r10).  */

+	VPCMP	$0, VEC_SIZE(%rdx, %r10), %YMM3, %k3{%k4}

+	kmovd	%k3, %edi

+# ifdef USE_AS_WCSCMP

+	/* Don't use subl since it is the upper 8 bits of EDI below.  */

+	notl	%edi

+	andl	$0xff, %edi

+# else

+	incl	%edi

+# endif

 # ifdef USE_AS_WCSCMP

-	/* NB: Each bit in K1/K3 represents 4-byte element.  */

-	kshiftlw $8, %k3, %k2

+	/* NB: Each bit in EDI/R9D represents 4-byte element.  */

+	sall	$8, %edi

 	/* NB: Divide shift count by 4 since each bit in K1 represent 4

 	   bytes.  */

 	movl	%ecx, %SHIFT_REG32

 	sarl	$2, %SHIFT_REG32

+

+	/* Each bit in EDI represents a null CHAR or a mismatch.  */

+	orl	%r9d, %edi

 # else

-	kshiftlq $32, %k3, %k2

-# endif

+	salq	$32, %rdi

-	/* Each bit in K1 represents a NULL or a mismatch.  */

-	korq	%k1, %k2, %k1

-	kmovq	%k1, %rdi

+	/* Each bit in RDI represents a null CHAR or a mismatch.  */

+	orq	%r9, %rdi

+# endif

 	/* Since ECX < VEC_SIZE * 2, simply skip the first ECX bytes.  */

 	shrxq	%SHIFT_REG64, %rdi, %rdi

@@ -692,35 +724,45 @@ L(loop_cross_page_2_vec):

 	/* The first VEC_SIZE * 2 bytes match or are ignored.  */

 	VMOVU	(VEC_SIZE * 2)(%rax, %r10), %YMM0

 	VMOVU	(VEC_SIZE * 3)(%rax, %r10), %YMM1

-	VMOVU	(VEC_SIZE * 2)(%rdx, %r10), %YMM2

-	VMOVU	(VEC_SIZE * 3)(%rdx, %r10), %YMM3

-

-	VPCMP	$4, %YMM0, %YMM2, %k0

-	VPCMP	$0, %YMMZERO, %YMM0, %k1

-	VPCMP	$0, %YMMZERO, %YMM2, %k2

-	kord	%k1, %k2, %k1

-	/* Each bit in K1 represents a NULL or a mismatch in YMM0 and

-	   YMM2.  */

-	kord	%k0, %k1, %k1

-

-	VPCMP	$4, %YMM1, %YMM3, %k3

-	VPCMP	$0, %YMMZERO, %YMM1, %k4

-	VPCMP	$0, %YMMZERO, %YMM3, %k5

-	kord	%k4, %k5, %k4

-	/* Each bit in K3 represents a NULL or a mismatch in YMM1 and

-	   YMM3.  */

-	kord	%k3, %k4, %k3

+	VPTESTM	%YMM0, %YMM0, %k2

+	/* Each bit cleared in K1 represents a mismatch or a null CHAR

+	   in YMM0 and 32 bytes at (VEC_SIZE * 2)(%rdx, %r10).  */

+	VPCMP	$0, (VEC_SIZE * 2)(%rdx, %r10), %YMM0, %k1{%k2}

+	kmovd	%k1, %r9d

+	/* Don't use subl since it is the lower 16/32 bits of RDI

+	   below.  */

+	notl	%r9d

 # ifdef USE_AS_WCSCMP

-	/* NB: Each bit in K1/K3 represents 4-byte element.  */

-	kshiftlw $8, %k3, %k2

+	/* Only last 8 bits are valid.  */

+	andl	$0xff, %r9d

+# endif

+

+	VPTESTM	%YMM1, %YMM1, %k4

+	/* Each bit cleared in K3 represents a mismatch or a null CHAR

+	   in YMM1 and 32 bytes at (VEC_SIZE * 3)(%rdx, %r10).  */

+	VPCMP	$0, (VEC_SIZE * 3)(%rdx, %r10), %YMM1, %k3{%k4}

+	kmovd	%k3, %edi

+# ifdef USE_AS_WCSCMP

+	/* Don't use subl since it is the upper 8 bits of EDI below.  */

+	notl	%edi

+	andl	$0xff, %edi

 # else

-	kshiftlq $32, %k3, %k2

+	incl	%edi

 # endif

-	/* Each bit in K1 represents a NULL or a mismatch.  */

-	korq	%k1, %k2, %k1

-	kmovq	%k1, %rdi

+# ifdef USE_AS_WCSCMP

+	/* NB: Each bit in EDI/R9D represents 4-byte element.  */

+	sall	$8, %edi

+

+	/* Each bit in EDI represents a null CHAR or a mismatch.  */

+	orl	%r9d, %edi

+# else

+	salq	$32, %rdi

+

+	/* Each bit in RDI represents a null CHAR or a mismatch.  */

+	orq	%r9, %rdi

+# endif

 	xorl	%r8d, %r8d

 	/* If ECX > VEC_SIZE * 2, skip ECX - (VEC_SIZE * 2) bytes.  */

@@ -729,12 +771,15 @@ L(loop_cross_page_2_vec):

 	/* R8 has number of bytes skipped.  */

 	movl	%ecx, %r8d

 # ifdef USE_AS_WCSCMP

-	/* NB: Divide shift count by 4 since each bit in K1 represent 4

+	/* NB: Divide shift count by 4 since each bit in RDI represent 4

 	   bytes.  */

 	sarl	$2, %ecx

-# endif

+	/* Skip ECX bytes.  */

+	shrl	%cl, %edi

+# else

 	/* Skip ECX bytes.  */

 	shrq	%cl, %rdi

+# endif

 1:

 	/* Before jumping back to the loop, set ESI to the number of

 	   VEC_SIZE * 4 blocks before page crossing.  */

@@ -818,7 +863,7 @@ L(cross_page_loop):

 	movzbl	(%rdi, %rdx), %eax

 	movzbl	(%rsi, %rdx), %ecx

 # endif

-	/* Check null char.  */

+	/* Check null CHAR.  */

 	testl	%eax, %eax

 	jne	L(cross_page_loop)

 	/* Since %eax == 0, subtract is OK for both SIGNED and UNSIGNED

@@ -901,18 +946,17 @@ L(cross_page):

 	jg	L(cross_page_1_vector)

 L(loop_1_vector):

 	VMOVU	(%rdi, %rdx), %YMM0

-	VMOVU	(%rsi, %rdx), %YMM1

-

-	/* Each bit in K0 represents a mismatch in YMM0 and YMM1.  */

-	VPCMP	$4, %YMM0, %YMM1, %k0

-	VPCMP	$0, %YMMZERO, %YMM0, %k1

-	VPCMP	$0, %YMMZERO, %YMM1, %k2

-	/* Each bit in K1 represents a NULL in YMM0 or YMM1.  */

-	kord	%k1, %k2, %k1

-	/* Each bit in K1 represents a NULL or a mismatch.  */

-	kord	%k0, %k1, %k1

+

+	VPTESTM	%YMM0, %YMM0, %k2

+	/* Each bit cleared in K1 represents a mismatch or a null CHAR

+	   in YMM0 and 32 bytes at (%rsi, %rdx).  */

+	VPCMP	$0, (%rsi, %rdx), %YMM0, %k1{%k2}

 	kmovd	%k1, %ecx

-	testl	%ecx, %ecx

+# ifdef USE_AS_WCSCMP

+	subl	$0xff, %ecx

+# else

+	incl	%ecx

+# endif

 	jne	L(last_vector)

 	addl	$VEC_SIZE, %edx

@@ -931,18 +975,17 @@ L(cross_page_1_vector):

 	cmpl	$(PAGE_SIZE - 16), %eax

 	jg	L(cross_page_1_xmm)

 	VMOVU	(%rdi, %rdx), %XMM0

-	VMOVU	(%rsi, %rdx), %XMM1

-

-	/* Each bit in K0 represents a mismatch in XMM0 and XMM1.  */

-	VPCMP	$4, %XMM0, %XMM1, %k0

-	VPCMP	$0, %XMMZERO, %XMM0, %k1

-	VPCMP	$0, %XMMZERO, %XMM1, %k2

-	/* Each bit in K1 represents a NULL in XMM0 or XMM1.  */

-	korw	%k1, %k2, %k1

-	/* Each bit in K1 represents a NULL or a mismatch.  */

-	korw	%k0, %k1, %k1

-	kmovw	%k1, %ecx

-	testl	%ecx, %ecx

+

+	VPTESTM	%YMM0, %YMM0, %k2

+	/* Each bit cleared in K1 represents a mismatch or a null CHAR

+	   in XMM0 and 16 bytes at (%rsi, %rdx).  */

+	VPCMP	$0, (%rsi, %rdx), %XMM0, %k1{%k2}

+	kmovd	%k1, %ecx

+# ifdef USE_AS_WCSCMP

+	subl	$0xf, %ecx

+# else

+	subl	$0xffff, %ecx

+# endif

 	jne	L(last_vector)

 	addl	$16, %edx

@@ -965,25 +1008,16 @@ L(cross_page_1_xmm):

 	vmovq	(%rdi, %rdx), %XMM0

 	vmovq	(%rsi, %rdx), %XMM1

-	/* Each bit in K0 represents a mismatch in XMM0 and XMM1.  */

-	VPCMP	$4, %XMM0, %XMM1, %k0

-	VPCMP	$0, %XMMZERO, %XMM0, %k1

-	VPCMP	$0, %XMMZERO, %XMM1, %k2

-	/* Each bit in K1 represents a NULL in XMM0 or XMM1.  */

-	kord	%k1, %k2, %k1

-	/* Each bit in K1 represents a NULL or a mismatch.  */

-	kord	%k0, %k1, %k1

-	kmovd	%k1, %ecx

-

+	VPTESTM	%YMM0, %YMM0, %k2

+	/* Each bit cleared in K1 represents a mismatch or a null CHAR

+	   in XMM0 and XMM1.  */

+	VPCMP	$0, %XMM1, %XMM0, %k1{%k2}

+	kmovb	%k1, %ecx

 # ifdef USE_AS_WCSCMP

-	/* Only last 2 bits are valid.  */

-	andl	$0x3, %ecx

+	subl	$0x3, %ecx

 # else

-	/* Only last 8 bits are valid.  */

-	andl	$0xff, %ecx

+	subl	$0xff, %ecx

 # endif

-

-	testl	%ecx, %ecx

 	jne	L(last_vector)

 	addl	$8, %edx

@@ -1002,25 +1036,16 @@ L(cross_page_8bytes):

 	vmovd	(%rdi, %rdx), %XMM0

 	vmovd	(%rsi, %rdx), %XMM1

-	/* Each bit in K0 represents a mismatch in XMM0 and XMM1.  */

-	VPCMP	$4, %XMM0, %XMM1, %k0

-	VPCMP	$0, %XMMZERO, %XMM0, %k1

-	VPCMP	$0, %XMMZERO, %XMM1, %k2

-	/* Each bit in K1 represents a NULL in XMM0 or XMM1.  */

-	kord	%k1, %k2, %k1

-	/* Each bit in K1 represents a NULL or a mismatch.  */

-	kord	%k0, %k1, %k1

+	VPTESTM	%YMM0, %YMM0, %k2

+	/* Each bit cleared in K1 represents a mismatch or a null CHAR

+	   in XMM0 and XMM1.  */

+	VPCMP	$0, %XMM1, %XMM0, %k1{%k2}

 	kmovd	%k1, %ecx

-

 # ifdef USE_AS_WCSCMP

-	/* Only the last bit is valid.  */

-	andl	$0x1, %ecx

+	subl	$0x1, %ecx

 # else

-	/* Only last 4 bits are valid.  */

-	andl	$0xf, %ecx

+	subl	$0xf, %ecx

 # endif

-

-	testl	%ecx, %ecx

 	jne	L(last_vector)

 	addl	$4, %edx