|
 |
8fbb1c |
diff -up openssl-1.0.1e/crypto/bn/asm/x86_64-mont5.pl.rsa-const openssl-1.0.1e/crypto/bn/asm/x86_64-mont5.pl
|
|
|
8fbb1c |
--- openssl-1.0.1e/crypto/bn/asm/x86_64-mont5.pl.rsa-const 2016-01-14 17:38:50.127212987 +0100
|
|
|
8fbb1c |
+++ openssl-1.0.1e/crypto/bn/asm/x86_64-mont5.pl 2016-02-24 12:03:28.180178677 +0100
|
|
|
8fbb1c |
@@ -66,60 +66,113 @@ bn_mul_mont_gather5:
|
|
|
8fbb1c |
.align 16
|
|
|
8fbb1c |
.Lmul_enter:
|
|
|
8fbb1c |
mov ${num}d,${num}d
|
|
|
8fbb1c |
- mov `($win64?56:8)`(%rsp),%r10d # load 7th argument
|
|
|
8fbb1c |
+ movd `($win64?56:8)`(%rsp),%xmm5 # load 7th argument
|
|
|
8fbb1c |
+ lea .Linc(%rip),%r10
|
|
|
8fbb1c |
push %rbx
|
|
|
8fbb1c |
push %rbp
|
|
|
8fbb1c |
push %r12
|
|
|
8fbb1c |
push %r13
|
|
|
8fbb1c |
push %r14
|
|
|
8fbb1c |
push %r15
|
|
|
8fbb1c |
-___
|
|
|
8fbb1c |
-$code.=<<___ if ($win64);
|
|
|
8fbb1c |
- lea -0x28(%rsp),%rsp
|
|
|
8fbb1c |
- movaps %xmm6,(%rsp)
|
|
|
8fbb1c |
- movaps %xmm7,0x10(%rsp)
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
.Lmul_alloca:
|
|
|
8fbb1c |
-___
|
|
|
8fbb1c |
-$code.=<<___;
|
|
|
8fbb1c |
mov %rsp,%rax
|
|
|
8fbb1c |
lea 2($num),%r11
|
|
|
8fbb1c |
neg %r11
|
|
|
8fbb1c |
- lea (%rsp,%r11,8),%rsp # tp=alloca(8*(num+2))
|
|
|
8fbb1c |
+ lea -264(%rsp,%r11,8),%rsp # tp=alloca(8*(num+2)+256+8)
|
|
|
8fbb1c |
and \$-1024,%rsp # minimize TLB usage
|
|
|
8fbb1c |
|
|
|
8fbb1c |
mov %rax,8(%rsp,$num,8) # tp[num+1]=%rsp
|
|
|
8fbb1c |
.Lmul_body:
|
|
|
8fbb1c |
- mov $bp,%r12 # reassign $bp
|
|
|
8fbb1c |
+ lea 128($bp),%r12 # reassign $bp (+size optimization)
|
|
|
8fbb1c |
___
|
|
|
8fbb1c |
$bp="%r12";
|
|
|
8fbb1c |
$STRIDE=2**5*8; # 5 is "window size"
|
|
|
8fbb1c |
$N=$STRIDE/4; # should match cache line size
|
|
|
8fbb1c |
$code.=<<___;
|
|
|
8fbb1c |
- mov %r10,%r11
|
|
|
8fbb1c |
- shr \$`log($N/8)/log(2)`,%r10
|
|
|
8fbb1c |
- and \$`$N/8-1`,%r11
|
|
|
8fbb1c |
- not %r10
|
|
|
8fbb1c |
- lea .Lmagic_masks(%rip),%rax
|
|
|
8fbb1c |
- and \$`2**5/($N/8)-1`,%r10 # 5 is "window size"
|
|
|
8fbb1c |
- lea 96($bp,%r11,8),$bp # pointer within 1st cache line
|
|
|
8fbb1c |
- movq 0(%rax,%r10,8),%xmm4 # set of masks denoting which
|
|
|
8fbb1c |
- movq 8(%rax,%r10,8),%xmm5 # cache line contains element
|
|
|
8fbb1c |
- movq 16(%rax,%r10,8),%xmm6 # denoted by 7th argument
|
|
|
8fbb1c |
- movq 24(%rax,%r10,8),%xmm7
|
|
|
8fbb1c |
-
|
|
|
8fbb1c |
- movq `0*$STRIDE/4-96`($bp),%xmm0
|
|
|
8fbb1c |
- movq `1*$STRIDE/4-96`($bp),%xmm1
|
|
|
8fbb1c |
- pand %xmm4,%xmm0
|
|
|
8fbb1c |
- movq `2*$STRIDE/4-96`($bp),%xmm2
|
|
|
8fbb1c |
- pand %xmm5,%xmm1
|
|
|
8fbb1c |
- movq `3*$STRIDE/4-96`($bp),%xmm3
|
|
|
8fbb1c |
- pand %xmm6,%xmm2
|
|
|
8fbb1c |
- por %xmm1,%xmm0
|
|
|
8fbb1c |
- pand %xmm7,%xmm3
|
|
|
8fbb1c |
+ movdqa 0(%r10),%xmm0 # 00000001000000010000000000000000
|
|
|
8fbb1c |
+ movdqa 16(%r10),%xmm1 # 00000002000000020000000200000002
|
|
|
8fbb1c |
+ lea 24-112(%rsp,$num,8),%r10# place the mask after tp[num+3] (+ICache optimization)
|
|
|
8fbb1c |
+ and \$-16,%r10
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+ pshufd \$0,%xmm5,%xmm5 # broadcast index
|
|
|
8fbb1c |
+ movdqa %xmm1,%xmm4
|
|
|
8fbb1c |
+ movdqa %xmm1,%xmm2
|
|
|
8fbb1c |
+___
|
|
|
8fbb1c |
+########################################################################
|
|
|
8fbb1c |
+# calculate mask by comparing 0..31 to index and save result to stack
|
|
|
8fbb1c |
+#
|
|
|
8fbb1c |
+$code.=<<___;
|
|
|
8fbb1c |
+ paddd %xmm0,%xmm1
|
|
|
8fbb1c |
+ pcmpeqd %xmm5,%xmm0 # compare to 1,0
|
|
|
8fbb1c |
+ .byte 0x67
|
|
|
8fbb1c |
+ movdqa %xmm4,%xmm3
|
|
|
8fbb1c |
+___
|
|
|
8fbb1c |
+for($k=0;$k<$STRIDE/16-4;$k+=4) {
|
|
|
8fbb1c |
+$code.=<<___;
|
|
|
8fbb1c |
+ paddd %xmm1,%xmm2
|
|
|
8fbb1c |
+ pcmpeqd %xmm5,%xmm1 # compare to 3,2
|
|
|
8fbb1c |
+ movdqa %xmm0,`16*($k+0)+112`(%r10)
|
|
|
8fbb1c |
+ movdqa %xmm4,%xmm0
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+ paddd %xmm2,%xmm3
|
|
|
8fbb1c |
+ pcmpeqd %xmm5,%xmm2 # compare to 5,4
|
|
|
8fbb1c |
+ movdqa %xmm1,`16*($k+1)+112`(%r10)
|
|
|
8fbb1c |
+ movdqa %xmm4,%xmm1
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+ paddd %xmm3,%xmm0
|
|
|
8fbb1c |
+ pcmpeqd %xmm5,%xmm3 # compare to 7,6
|
|
|
8fbb1c |
+ movdqa %xmm2,`16*($k+2)+112`(%r10)
|
|
|
8fbb1c |
+ movdqa %xmm4,%xmm2
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+ paddd %xmm0,%xmm1
|
|
|
8fbb1c |
+ pcmpeqd %xmm5,%xmm0
|
|
|
8fbb1c |
+ movdqa %xmm3,`16*($k+3)+112`(%r10)
|
|
|
8fbb1c |
+ movdqa %xmm4,%xmm3
|
|
|
8fbb1c |
+___
|
|
|
8fbb1c |
+}
|
|
|
8fbb1c |
+$code.=<<___; # last iteration can be optimized
|
|
|
8fbb1c |
+ paddd %xmm1,%xmm2
|
|
|
8fbb1c |
+ pcmpeqd %xmm5,%xmm1
|
|
|
8fbb1c |
+ movdqa %xmm0,`16*($k+0)+112`(%r10)
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+ paddd %xmm2,%xmm3
|
|
|
8fbb1c |
+ .byte 0x67
|
|
|
8fbb1c |
+ pcmpeqd %xmm5,%xmm2
|
|
|
8fbb1c |
+ movdqa %xmm1,`16*($k+1)+112`(%r10)
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+ pcmpeqd %xmm5,%xmm3
|
|
|
8fbb1c |
+ movdqa %xmm2,`16*($k+2)+112`(%r10)
|
|
|
8fbb1c |
+ pand `16*($k+0)-128`($bp),%xmm0 # while it's still in register
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+ pand `16*($k+1)-128`($bp),%xmm1
|
|
|
8fbb1c |
+ pand `16*($k+2)-128`($bp),%xmm2
|
|
|
8fbb1c |
+ movdqa %xmm3,`16*($k+3)+112`(%r10)
|
|
|
8fbb1c |
+ pand `16*($k+3)-128`($bp),%xmm3
|
|
|
8fbb1c |
por %xmm2,%xmm0
|
|
|
8fbb1c |
+ por %xmm3,%xmm1
|
|
|
8fbb1c |
+___
|
|
|
8fbb1c |
+for($k=0;$k<$STRIDE/16-4;$k+=4) {
|
|
|
8fbb1c |
+$code.=<<___;
|
|
|
8fbb1c |
+ movdqa `16*($k+0)-128`($bp),%xmm4
|
|
|
8fbb1c |
+ movdqa `16*($k+1)-128`($bp),%xmm5
|
|
|
8fbb1c |
+ movdqa `16*($k+2)-128`($bp),%xmm2
|
|
|
8fbb1c |
+ pand `16*($k+0)+112`(%r10),%xmm4
|
|
|
8fbb1c |
+ movdqa `16*($k+3)-128`($bp),%xmm3
|
|
|
8fbb1c |
+ pand `16*($k+1)+112`(%r10),%xmm5
|
|
|
8fbb1c |
+ por %xmm4,%xmm0
|
|
|
8fbb1c |
+ pand `16*($k+2)+112`(%r10),%xmm2
|
|
|
8fbb1c |
+ por %xmm5,%xmm1
|
|
|
8fbb1c |
+ pand `16*($k+3)+112`(%r10),%xmm3
|
|
|
8fbb1c |
+ por %xmm2,%xmm0
|
|
|
8fbb1c |
+ por %xmm3,%xmm1
|
|
|
8fbb1c |
+___
|
|
|
8fbb1c |
+}
|
|
|
8fbb1c |
+$code.=<<___;
|
|
|
8fbb1c |
+ por %xmm1,%xmm0
|
|
|
8fbb1c |
+ pshufd \$0x4e,%xmm0,%xmm1
|
|
|
8fbb1c |
+ por %xmm1,%xmm0
|
|
|
8fbb1c |
lea $STRIDE($bp),$bp
|
|
|
8fbb1c |
- por %xmm3,%xmm0
|
|
|
8fbb1c |
-
|
|
|
8fbb1c |
movq %xmm0,$m0 # m0=bp[0]
|
|
|
8fbb1c |
|
|
|
8fbb1c |
mov ($n0),$n0 # pull n0[0] value
|
|
|
8fbb1c |
@@ -128,29 +181,14 @@ $code.=<<___;
|
|
|
8fbb1c |
xor $i,$i # i=0
|
|
|
8fbb1c |
xor $j,$j # j=0
|
|
|
8fbb1c |
|
|
|
8fbb1c |
- movq `0*$STRIDE/4-96`($bp),%xmm0
|
|
|
8fbb1c |
- movq `1*$STRIDE/4-96`($bp),%xmm1
|
|
|
8fbb1c |
- pand %xmm4,%xmm0
|
|
|
8fbb1c |
- movq `2*$STRIDE/4-96`($bp),%xmm2
|
|
|
8fbb1c |
- pand %xmm5,%xmm1
|
|
|
8fbb1c |
-
|
|
|
8fbb1c |
mov $n0,$m1
|
|
|
8fbb1c |
mulq $m0 # ap[0]*bp[0]
|
|
|
8fbb1c |
mov %rax,$lo0
|
|
|
8fbb1c |
mov ($np),%rax
|
|
|
8fbb1c |
|
|
|
8fbb1c |
- movq `3*$STRIDE/4-96`($bp),%xmm3
|
|
|
8fbb1c |
- pand %xmm6,%xmm2
|
|
|
8fbb1c |
- por %xmm1,%xmm0
|
|
|
8fbb1c |
- pand %xmm7,%xmm3
|
|
|
8fbb1c |
-
|
|
|
8fbb1c |
imulq $lo0,$m1 # "tp[0]"*n0
|
|
|
8fbb1c |
mov %rdx,$hi0
|
|
|
8fbb1c |
|
|
|
8fbb1c |
- por %xmm2,%xmm0
|
|
|
8fbb1c |
- lea $STRIDE($bp),$bp
|
|
|
8fbb1c |
- por %xmm3,%xmm0
|
|
|
8fbb1c |
-
|
|
|
8fbb1c |
mulq $m1 # np[0]*m1
|
|
|
8fbb1c |
add %rax,$lo0 # discarded
|
|
|
8fbb1c |
mov 8($ap),%rax
|
|
|
8fbb1c |
@@ -183,8 +221,6 @@ $code.=<<___;
|
|
|
8fbb1c |
cmp $num,$j
|
|
|
8fbb1c |
jne .L1st
|
|
|
8fbb1c |
|
|
|
8fbb1c |
- movq %xmm0,$m0 # bp[1]
|
|
|
8fbb1c |
-
|
|
|
8fbb1c |
add %rax,$hi1
|
|
|
8fbb1c |
mov ($ap),%rax # ap[0]
|
|
|
8fbb1c |
adc \$0,%rdx
|
|
|
8fbb1c |
@@ -204,33 +240,46 @@ $code.=<<___;
|
|
|
8fbb1c |
jmp .Louter
|
|
|
8fbb1c |
.align 16
|
|
|
8fbb1c |
.Louter:
|
|
|
8fbb1c |
+ lea 24+128(%rsp,$num,8),%rdx # where 256-byte mask is (+size optimization)
|
|
|
8fbb1c |
+ and \$-16,%rdx
|
|
|
8fbb1c |
+ pxor %xmm4,%xmm4
|
|
|
8fbb1c |
+ pxor %xmm5,%xmm5
|
|
|
8fbb1c |
+___
|
|
|
8fbb1c |
+for($k=0;$k<$STRIDE/16;$k+=4) {
|
|
|
8fbb1c |
+$code.=<<___;
|
|
|
8fbb1c |
+ movdqa `16*($k+0)-128`($bp),%xmm0
|
|
|
8fbb1c |
+ movdqa `16*($k+1)-128`($bp),%xmm1
|
|
|
8fbb1c |
+ movdqa `16*($k+2)-128`($bp),%xmm2
|
|
|
8fbb1c |
+ movdqa `16*($k+3)-128`($bp),%xmm3
|
|
|
8fbb1c |
+ pand `16*($k+0)-128`(%rdx),%xmm0
|
|
|
8fbb1c |
+ pand `16*($k+1)-128`(%rdx),%xmm1
|
|
|
8fbb1c |
+ por %xmm0,%xmm4
|
|
|
8fbb1c |
+ pand `16*($k+2)-128`(%rdx),%xmm2
|
|
|
8fbb1c |
+ por %xmm1,%xmm5
|
|
|
8fbb1c |
+ pand `16*($k+3)-128`(%rdx),%xmm3
|
|
|
8fbb1c |
+ por %xmm2,%xmm4
|
|
|
8fbb1c |
+ por %xmm3,%xmm5
|
|
|
8fbb1c |
+___
|
|
|
8fbb1c |
+}
|
|
|
8fbb1c |
+$code.=<<___;
|
|
|
8fbb1c |
+ por %xmm5,%xmm4
|
|
|
8fbb1c |
+ pshufd \$0x4e,%xmm4,%xmm0
|
|
|
8fbb1c |
+ por %xmm4,%xmm0
|
|
|
8fbb1c |
+ lea $STRIDE($bp),$bp
|
|
|
8fbb1c |
+ movq %xmm0,$m0 # m0=bp[i]
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
xor $j,$j # j=0
|
|
|
8fbb1c |
mov $n0,$m1
|
|
|
8fbb1c |
mov (%rsp),$lo0
|
|
|
8fbb1c |
|
|
|
8fbb1c |
- movq `0*$STRIDE/4-96`($bp),%xmm0
|
|
|
8fbb1c |
- movq `1*$STRIDE/4-96`($bp),%xmm1
|
|
|
8fbb1c |
- pand %xmm4,%xmm0
|
|
|
8fbb1c |
- movq `2*$STRIDE/4-96`($bp),%xmm2
|
|
|
8fbb1c |
- pand %xmm5,%xmm1
|
|
|
8fbb1c |
-
|
|
|
8fbb1c |
mulq $m0 # ap[0]*bp[i]
|
|
|
8fbb1c |
add %rax,$lo0 # ap[0]*bp[i]+tp[0]
|
|
|
8fbb1c |
mov ($np),%rax
|
|
|
8fbb1c |
adc \$0,%rdx
|
|
|
8fbb1c |
|
|
|
8fbb1c |
- movq `3*$STRIDE/4-96`($bp),%xmm3
|
|
|
8fbb1c |
- pand %xmm6,%xmm2
|
|
|
8fbb1c |
- por %xmm1,%xmm0
|
|
|
8fbb1c |
- pand %xmm7,%xmm3
|
|
|
8fbb1c |
-
|
|
|
8fbb1c |
imulq $lo0,$m1 # tp[0]*n0
|
|
|
8fbb1c |
mov %rdx,$hi0
|
|
|
8fbb1c |
|
|
|
8fbb1c |
- por %xmm2,%xmm0
|
|
|
8fbb1c |
- lea $STRIDE($bp),$bp
|
|
|
8fbb1c |
- por %xmm3,%xmm0
|
|
|
8fbb1c |
-
|
|
|
8fbb1c |
mulq $m1 # np[0]*m1
|
|
|
8fbb1c |
add %rax,$lo0 # discarded
|
|
|
8fbb1c |
mov 8($ap),%rax
|
|
|
8fbb1c |
@@ -266,8 +315,6 @@ $code.=<<___;
|
|
|
8fbb1c |
cmp $num,$j
|
|
|
8fbb1c |
jne .Linner
|
|
|
8fbb1c |
|
|
|
8fbb1c |
- movq %xmm0,$m0 # bp[i+1]
|
|
|
8fbb1c |
-
|
|
|
8fbb1c |
add %rax,$hi1
|
|
|
8fbb1c |
mov ($ap),%rax # ap[0]
|
|
|
8fbb1c |
adc \$0,%rdx
|
|
|
8fbb1c |
@@ -321,13 +368,7 @@ $code.=<<___;
|
|
|
8fbb1c |
|
|
|
8fbb1c |
mov 8(%rsp,$num,8),%rsi # restore %rsp
|
|
|
8fbb1c |
mov \$1,%rax
|
|
|
8fbb1c |
-___
|
|
|
8fbb1c |
-$code.=<<___ if ($win64);
|
|
|
8fbb1c |
- movaps (%rsi),%xmm6
|
|
|
8fbb1c |
- movaps 0x10(%rsi),%xmm7
|
|
|
8fbb1c |
- lea 0x28(%rsi),%rsi
|
|
|
8fbb1c |
-___
|
|
|
8fbb1c |
-$code.=<<___;
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
mov (%rsi),%r15
|
|
|
8fbb1c |
mov 8(%rsi),%r14
|
|
|
8fbb1c |
mov 16(%rsi),%r13
|
|
|
8fbb1c |
@@ -348,91 +389,130 @@ $code.=<<___;
|
|
|
8fbb1c |
bn_mul4x_mont_gather5:
|
|
|
8fbb1c |
.Lmul4x_enter:
|
|
|
8fbb1c |
mov ${num}d,${num}d
|
|
|
8fbb1c |
- mov `($win64?56:8)`(%rsp),%r10d # load 7th argument
|
|
|
8fbb1c |
+ movd `($win64?56:8)`(%rsp),%xmm5 # load 7th argument
|
|
|
8fbb1c |
+ lea .Linc(%rip),%r10
|
|
|
8fbb1c |
push %rbx
|
|
|
8fbb1c |
push %rbp
|
|
|
8fbb1c |
push %r12
|
|
|
8fbb1c |
push %r13
|
|
|
8fbb1c |
push %r14
|
|
|
8fbb1c |
push %r15
|
|
|
8fbb1c |
-___
|
|
|
8fbb1c |
-$code.=<<___ if ($win64);
|
|
|
8fbb1c |
- lea -0x28(%rsp),%rsp
|
|
|
8fbb1c |
- movaps %xmm6,(%rsp)
|
|
|
8fbb1c |
- movaps %xmm7,0x10(%rsp)
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
.Lmul4x_alloca:
|
|
|
8fbb1c |
-___
|
|
|
8fbb1c |
-$code.=<<___;
|
|
|
8fbb1c |
mov %rsp,%rax
|
|
|
8fbb1c |
lea 4($num),%r11
|
|
|
8fbb1c |
neg %r11
|
|
|
8fbb1c |
- lea (%rsp,%r11,8),%rsp # tp=alloca(8*(num+4))
|
|
|
8fbb1c |
+ lea -256(%rsp,%r11,8),%rsp # tp=alloca(8*(num+4)+256)
|
|
|
8fbb1c |
and \$-1024,%rsp # minimize TLB usage
|
|
|
8fbb1c |
|
|
|
8fbb1c |
mov %rax,8(%rsp,$num,8) # tp[num+1]=%rsp
|
|
|
8fbb1c |
.Lmul4x_body:
|
|
|
8fbb1c |
mov $rp,16(%rsp,$num,8) # tp[num+2]=$rp
|
|
|
8fbb1c |
- mov %rdx,%r12 # reassign $bp
|
|
|
8fbb1c |
+ lea 128(%rdx),%r12 # reassign $bp (+size optimization)
|
|
|
8fbb1c |
___
|
|
|
8fbb1c |
$bp="%r12";
|
|
|
8fbb1c |
$STRIDE=2**5*8; # 5 is "window size"
|
|
|
8fbb1c |
$N=$STRIDE/4; # should match cache line size
|
|
|
8fbb1c |
$code.=<<___;
|
|
|
8fbb1c |
- mov %r10,%r11
|
|
|
8fbb1c |
- shr \$`log($N/8)/log(2)`,%r10
|
|
|
8fbb1c |
- and \$`$N/8-1`,%r11
|
|
|
8fbb1c |
- not %r10
|
|
|
8fbb1c |
- lea .Lmagic_masks(%rip),%rax
|
|
|
8fbb1c |
- and \$`2**5/($N/8)-1`,%r10 # 5 is "window size"
|
|
|
8fbb1c |
- lea 96($bp,%r11,8),$bp # pointer within 1st cache line
|
|
|
8fbb1c |
- movq 0(%rax,%r10,8),%xmm4 # set of masks denoting which
|
|
|
8fbb1c |
- movq 8(%rax,%r10,8),%xmm5 # cache line contains element
|
|
|
8fbb1c |
- movq 16(%rax,%r10,8),%xmm6 # denoted by 7th argument
|
|
|
8fbb1c |
- movq 24(%rax,%r10,8),%xmm7
|
|
|
8fbb1c |
-
|
|
|
8fbb1c |
- movq `0*$STRIDE/4-96`($bp),%xmm0
|
|
|
8fbb1c |
- movq `1*$STRIDE/4-96`($bp),%xmm1
|
|
|
8fbb1c |
- pand %xmm4,%xmm0
|
|
|
8fbb1c |
- movq `2*$STRIDE/4-96`($bp),%xmm2
|
|
|
8fbb1c |
- pand %xmm5,%xmm1
|
|
|
8fbb1c |
- movq `3*$STRIDE/4-96`($bp),%xmm3
|
|
|
8fbb1c |
- pand %xmm6,%xmm2
|
|
|
8fbb1c |
- por %xmm1,%xmm0
|
|
|
8fbb1c |
- pand %xmm7,%xmm3
|
|
|
8fbb1c |
+ movdqa 0(%r10),%xmm0 # 00000001000000010000000000000000
|
|
|
8fbb1c |
+ movdqa 16(%r10),%xmm1 # 00000002000000020000000200000002
|
|
|
8fbb1c |
+ lea 32-112(%rsp,$num,8),%r10# place the mask after tp[num+4] (+ICache optimization)
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+ pshufd \$0,%xmm5,%xmm5 # broadcast index
|
|
|
8fbb1c |
+ movdqa %xmm1,%xmm4
|
|
|
8fbb1c |
+ .byte 0x67,0x67
|
|
|
8fbb1c |
+ movdqa %xmm1,%xmm2
|
|
|
8fbb1c |
+___
|
|
|
8fbb1c |
+########################################################################
|
|
|
8fbb1c |
+# calculate mask by comparing 0..31 to index and save result to stack
|
|
|
8fbb1c |
+#
|
|
|
8fbb1c |
+$code.=<<___;
|
|
|
8fbb1c |
+ paddd %xmm0,%xmm1
|
|
|
8fbb1c |
+ pcmpeqd %xmm5,%xmm0 # compare to 1,0
|
|
|
8fbb1c |
+ .byte 0x67
|
|
|
8fbb1c |
+ movdqa %xmm4,%xmm3
|
|
|
8fbb1c |
+___
|
|
|
8fbb1c |
+for($k=0;$k<$STRIDE/16-4;$k+=4) {
|
|
|
8fbb1c |
+$code.=<<___;
|
|
|
8fbb1c |
+ paddd %xmm1,%xmm2
|
|
|
8fbb1c |
+ pcmpeqd %xmm5,%xmm1 # compare to 3,2
|
|
|
8fbb1c |
+ movdqa %xmm0,`16*($k+0)+112`(%r10)
|
|
|
8fbb1c |
+ movdqa %xmm4,%xmm0
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+ paddd %xmm2,%xmm3
|
|
|
8fbb1c |
+ pcmpeqd %xmm5,%xmm2 # compare to 5,4
|
|
|
8fbb1c |
+ movdqa %xmm1,`16*($k+1)+112`(%r10)
|
|
|
8fbb1c |
+ movdqa %xmm4,%xmm1
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+ paddd %xmm3,%xmm0
|
|
|
8fbb1c |
+ pcmpeqd %xmm5,%xmm3 # compare to 7,6
|
|
|
8fbb1c |
+ movdqa %xmm2,`16*($k+2)+112`(%r10)
|
|
|
8fbb1c |
+ movdqa %xmm4,%xmm2
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+ paddd %xmm0,%xmm1
|
|
|
8fbb1c |
+ pcmpeqd %xmm5,%xmm0
|
|
|
8fbb1c |
+ movdqa %xmm3,`16*($k+3)+112`(%r10)
|
|
|
8fbb1c |
+ movdqa %xmm4,%xmm3
|
|
|
8fbb1c |
+___
|
|
|
8fbb1c |
+}
|
|
|
8fbb1c |
+$code.=<<___; # last iteration can be optimized
|
|
|
8fbb1c |
+ paddd %xmm1,%xmm2
|
|
|
8fbb1c |
+ pcmpeqd %xmm5,%xmm1
|
|
|
8fbb1c |
+ movdqa %xmm0,`16*($k+0)+112`(%r10)
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+ paddd %xmm2,%xmm3
|
|
|
8fbb1c |
+ .byte 0x67
|
|
|
8fbb1c |
+ pcmpeqd %xmm5,%xmm2
|
|
|
8fbb1c |
+ movdqa %xmm1,`16*($k+1)+112`(%r10)
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+ pcmpeqd %xmm5,%xmm3
|
|
|
8fbb1c |
+ movdqa %xmm2,`16*($k+2)+112`(%r10)
|
|
|
8fbb1c |
+ pand `16*($k+0)-128`($bp),%xmm0 # while it's still in register
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+ pand `16*($k+1)-128`($bp),%xmm1
|
|
|
8fbb1c |
+ pand `16*($k+2)-128`($bp),%xmm2
|
|
|
8fbb1c |
+ movdqa %xmm3,`16*($k+3)+112`(%r10)
|
|
|
8fbb1c |
+ pand `16*($k+3)-128`($bp),%xmm3
|
|
|
8fbb1c |
+ por %xmm2,%xmm0
|
|
|
8fbb1c |
+ por %xmm3,%xmm1
|
|
|
8fbb1c |
+___
|
|
|
8fbb1c |
+for($k=0;$k<$STRIDE/16-4;$k+=4) {
|
|
|
8fbb1c |
+$code.=<<___;
|
|
|
8fbb1c |
+ movdqa `16*($k+0)-128`($bp),%xmm4
|
|
|
8fbb1c |
+ movdqa `16*($k+1)-128`($bp),%xmm5
|
|
|
8fbb1c |
+ movdqa `16*($k+2)-128`($bp),%xmm2
|
|
|
8fbb1c |
+ pand `16*($k+0)+112`(%r10),%xmm4
|
|
|
8fbb1c |
+ movdqa `16*($k+3)-128`($bp),%xmm3
|
|
|
8fbb1c |
+ pand `16*($k+1)+112`(%r10),%xmm5
|
|
|
8fbb1c |
+ por %xmm4,%xmm0
|
|
|
8fbb1c |
+ pand `16*($k+2)+112`(%r10),%xmm2
|
|
|
8fbb1c |
+ por %xmm5,%xmm1
|
|
|
8fbb1c |
+ pand `16*($k+3)+112`(%r10),%xmm3
|
|
|
8fbb1c |
por %xmm2,%xmm0
|
|
|
8fbb1c |
+ por %xmm3,%xmm1
|
|
|
8fbb1c |
+___
|
|
|
8fbb1c |
+}
|
|
|
8fbb1c |
+$code.=<<___;
|
|
|
8fbb1c |
+ por %xmm1,%xmm0
|
|
|
8fbb1c |
+ pshufd \$0x4e,%xmm0,%xmm1
|
|
|
8fbb1c |
+ por %xmm1,%xmm0
|
|
|
8fbb1c |
lea $STRIDE($bp),$bp
|
|
|
8fbb1c |
- por %xmm3,%xmm0
|
|
|
8fbb1c |
-
|
|
|
8fbb1c |
movq %xmm0,$m0 # m0=bp[0]
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
mov ($n0),$n0 # pull n0[0] value
|
|
|
8fbb1c |
mov ($ap),%rax
|
|
|
8fbb1c |
|
|
|
8fbb1c |
xor $i,$i # i=0
|
|
|
8fbb1c |
xor $j,$j # j=0
|
|
|
8fbb1c |
|
|
|
8fbb1c |
- movq `0*$STRIDE/4-96`($bp),%xmm0
|
|
|
8fbb1c |
- movq `1*$STRIDE/4-96`($bp),%xmm1
|
|
|
8fbb1c |
- pand %xmm4,%xmm0
|
|
|
8fbb1c |
- movq `2*$STRIDE/4-96`($bp),%xmm2
|
|
|
8fbb1c |
- pand %xmm5,%xmm1
|
|
|
8fbb1c |
-
|
|
|
8fbb1c |
mov $n0,$m1
|
|
|
8fbb1c |
mulq $m0 # ap[0]*bp[0]
|
|
|
8fbb1c |
mov %rax,$A[0]
|
|
|
8fbb1c |
mov ($np),%rax
|
|
|
8fbb1c |
|
|
|
8fbb1c |
- movq `3*$STRIDE/4-96`($bp),%xmm3
|
|
|
8fbb1c |
- pand %xmm6,%xmm2
|
|
|
8fbb1c |
- por %xmm1,%xmm0
|
|
|
8fbb1c |
- pand %xmm7,%xmm3
|
|
|
8fbb1c |
-
|
|
|
8fbb1c |
imulq $A[0],$m1 # "tp[0]"*n0
|
|
|
8fbb1c |
mov %rdx,$A[1]
|
|
|
8fbb1c |
|
|
|
8fbb1c |
- por %xmm2,%xmm0
|
|
|
8fbb1c |
- lea $STRIDE($bp),$bp
|
|
|
8fbb1c |
- por %xmm3,%xmm0
|
|
|
8fbb1c |
-
|
|
|
8fbb1c |
mulq $m1 # np[0]*m1
|
|
|
8fbb1c |
add %rax,$A[0] # discarded
|
|
|
8fbb1c |
mov 8($ap),%rax
|
|
|
8fbb1c |
@@ -550,8 +630,6 @@ $code.=<<___;
|
|
|
8fbb1c |
mov $N[1],-16(%rsp,$j,8) # tp[j-1]
|
|
|
8fbb1c |
mov %rdx,$N[0]
|
|
|
8fbb1c |
|
|
|
8fbb1c |
- movq %xmm0,$m0 # bp[1]
|
|
|
8fbb1c |
-
|
|
|
8fbb1c |
xor $N[1],$N[1]
|
|
|
8fbb1c |
add $A[0],$N[0]
|
|
|
8fbb1c |
adc \$0,$N[1]
|
|
|
8fbb1c |
@@ -561,12 +639,34 @@ $code.=<<___;
|
|
|
8fbb1c |
lea 1($i),$i # i++
|
|
|
8fbb1c |
.align 4
|
|
|
8fbb1c |
.Louter4x:
|
|
|
8fbb1c |
+ lea 32+128(%rsp,$num,8),%rdx # where 256-byte mask is (+size optimization)
|
|
|
8fbb1c |
+ pxor %xmm4,%xmm4
|
|
|
8fbb1c |
+ pxor %xmm5,%xmm5
|
|
|
8fbb1c |
+___
|
|
|
8fbb1c |
+for($k=0;$k<$STRIDE/16;$k+=4) {
|
|
|
8fbb1c |
+$code.=<<___;
|
|
|
8fbb1c |
+ movdqa `16*($k+0)-128`($bp),%xmm0
|
|
|
8fbb1c |
+ movdqa `16*($k+1)-128`($bp),%xmm1
|
|
|
8fbb1c |
+ movdqa `16*($k+2)-128`($bp),%xmm2
|
|
|
8fbb1c |
+ movdqa `16*($k+3)-128`($bp),%xmm3
|
|
|
8fbb1c |
+ pand `16*($k+0)-128`(%rdx),%xmm0
|
|
|
8fbb1c |
+ pand `16*($k+1)-128`(%rdx),%xmm1
|
|
|
8fbb1c |
+ por %xmm0,%xmm4
|
|
|
8fbb1c |
+ pand `16*($k+2)-128`(%rdx),%xmm2
|
|
|
8fbb1c |
+ por %xmm1,%xmm5
|
|
|
8fbb1c |
+ pand `16*($k+3)-128`(%rdx),%xmm3
|
|
|
8fbb1c |
+ por %xmm2,%xmm4
|
|
|
8fbb1c |
+ por %xmm3,%xmm5
|
|
|
8fbb1c |
+___
|
|
|
8fbb1c |
+}
|
|
|
8fbb1c |
+$code.=<<___;
|
|
|
8fbb1c |
+ por %xmm5,%xmm4
|
|
|
8fbb1c |
+ pshufd \$0x4e,%xmm4,%xmm0
|
|
|
8fbb1c |
+ por %xmm4,%xmm0
|
|
|
8fbb1c |
+ lea $STRIDE($bp),$bp
|
|
|
8fbb1c |
+ movq %xmm0,$m0 # m0=bp[i]
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
xor $j,$j # j=0
|
|
|
8fbb1c |
- movq `0*$STRIDE/4-96`($bp),%xmm0
|
|
|
8fbb1c |
- movq `1*$STRIDE/4-96`($bp),%xmm1
|
|
|
8fbb1c |
- pand %xmm4,%xmm0
|
|
|
8fbb1c |
- movq `2*$STRIDE/4-96`($bp),%xmm2
|
|
|
8fbb1c |
- pand %xmm5,%xmm1
|
|
|
8fbb1c |
|
|
|
8fbb1c |
mov (%rsp),$A[0]
|
|
|
8fbb1c |
mov $n0,$m1
|
|
|
8fbb1c |
@@ -575,18 +675,9 @@ $code.=<<___;
|
|
|
8fbb1c |
mov ($np),%rax
|
|
|
8fbb1c |
adc \$0,%rdx
|
|
|
8fbb1c |
|
|
|
8fbb1c |
- movq `3*$STRIDE/4-96`($bp),%xmm3
|
|
|
8fbb1c |
- pand %xmm6,%xmm2
|
|
|
8fbb1c |
- por %xmm1,%xmm0
|
|
|
8fbb1c |
- pand %xmm7,%xmm3
|
|
|
8fbb1c |
-
|
|
|
8fbb1c |
imulq $A[0],$m1 # tp[0]*n0
|
|
|
8fbb1c |
mov %rdx,$A[1]
|
|
|
8fbb1c |
|
|
|
8fbb1c |
- por %xmm2,%xmm0
|
|
|
8fbb1c |
- lea $STRIDE($bp),$bp
|
|
|
8fbb1c |
- por %xmm3,%xmm0
|
|
|
8fbb1c |
-
|
|
|
8fbb1c |
mulq $m1 # np[0]*m1
|
|
|
8fbb1c |
add %rax,$A[0] # "$N[0]", discarded
|
|
|
8fbb1c |
mov 8($ap),%rax
|
|
|
8fbb1c |
@@ -718,7 +809,6 @@ $code.=<<___;
|
|
|
8fbb1c |
mov $N[0],-24(%rsp,$j,8) # tp[j-1]
|
|
|
8fbb1c |
mov %rdx,$N[0]
|
|
|
8fbb1c |
|
|
|
8fbb1c |
- movq %xmm0,$m0 # bp[i+1]
|
|
|
8fbb1c |
mov $N[1],-16(%rsp,$j,8) # tp[j-1]
|
|
|
8fbb1c |
|
|
|
8fbb1c |
xor $N[1],$N[1]
|
|
|
8fbb1c |
@@ -809,13 +899,7 @@ ___
|
|
|
8fbb1c |
$code.=<<___;
|
|
|
8fbb1c |
mov 8(%rsp,$num,8),%rsi # restore %rsp
|
|
|
8fbb1c |
mov \$1,%rax
|
|
|
8fbb1c |
-___
|
|
|
8fbb1c |
-$code.=<<___ if ($win64);
|
|
|
8fbb1c |
- movaps (%rsi),%xmm6
|
|
|
8fbb1c |
- movaps 0x10(%rsi),%xmm7
|
|
|
8fbb1c |
- lea 0x28(%rsi),%rsi
|
|
|
8fbb1c |
-___
|
|
|
8fbb1c |
-$code.=<<___;
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
mov (%rsi),%r15
|
|
|
8fbb1c |
mov 8(%rsi),%r14
|
|
|
8fbb1c |
mov 16(%rsi),%r13
|
|
|
8fbb1c |
@@ -830,8 +914,8 @@ ___
|
|
|
8fbb1c |
}}}
|
|
|
8fbb1c |
|
|
|
8fbb1c |
{
|
|
|
8fbb1c |
-my ($inp,$num,$tbl,$idx)=$win64?("%rcx","%rdx","%r8", "%r9") : # Win64 order
|
|
|
8fbb1c |
- ("%rdi","%rsi","%rdx","%rcx"); # Unix order
|
|
|
8fbb1c |
+my ($inp,$num,$tbl,$idx)=$win64?("%rcx","%rdx","%r8", "%r9d") : # Win64 order
|
|
|
8fbb1c |
+ ("%rdi","%rsi","%rdx","%ecx"); # Unix order
|
|
|
8fbb1c |
my $out=$inp;
|
|
|
8fbb1c |
my $STRIDE=2**5*8;
|
|
|
8fbb1c |
my $N=$STRIDE/4;
|
|
|
8fbb1c |
@@ -859,53 +943,89 @@ bn_scatter5:
|
|
|
8fbb1c |
.type bn_gather5,\@abi-omnipotent
|
|
|
8fbb1c |
.align 16
|
|
|
8fbb1c |
bn_gather5:
|
|
|
8fbb1c |
-___
|
|
|
8fbb1c |
-$code.=<<___ if ($win64);
|
|
|
8fbb1c |
-.LSEH_begin_bn_gather5:
|
|
|
8fbb1c |
+.LSEH_begin_bn_gather5: # Win64 thing, but harmless in other cases
|
|
|
8fbb1c |
# I can't trust assembler to use specific encoding:-(
|
|
|
8fbb1c |
- .byte 0x48,0x83,0xec,0x28 #sub \$0x28,%rsp
|
|
|
8fbb1c |
- .byte 0x0f,0x29,0x34,0x24 #movaps %xmm6,(%rsp)
|
|
|
8fbb1c |
- .byte 0x0f,0x29,0x7c,0x24,0x10 #movdqa %xmm7,0x10(%rsp)
|
|
|
8fbb1c |
-___
|
|
|
8fbb1c |
-$code.=<<___;
|
|
|
8fbb1c |
- mov $idx,%r11
|
|
|
8fbb1c |
- shr \$`log($N/8)/log(2)`,$idx
|
|
|
8fbb1c |
- and \$`$N/8-1`,%r11
|
|
|
8fbb1c |
- not $idx
|
|
|
8fbb1c |
- lea .Lmagic_masks(%rip),%rax
|
|
|
8fbb1c |
- and \$`2**5/($N/8)-1`,$idx # 5 is "window size"
|
|
|
8fbb1c |
- lea 96($tbl,%r11,8),$tbl # pointer within 1st cache line
|
|
|
8fbb1c |
- movq 0(%rax,$idx,8),%xmm4 # set of masks denoting which
|
|
|
8fbb1c |
- movq 8(%rax,$idx,8),%xmm5 # cache line contains element
|
|
|
8fbb1c |
- movq 16(%rax,$idx,8),%xmm6 # denoted by 7th argument
|
|
|
8fbb1c |
- movq 24(%rax,$idx,8),%xmm7
|
|
|
8fbb1c |
+ .byte 0x4c,0x8d,0x14,0x24 # lea (%rsp),%r10
|
|
|
8fbb1c |
+ .byte 0x48,0x81,0xec,0x08,0x01,0x00,0x00 # sub $0x108,%rsp
|
|
|
8fbb1c |
+ lea .Linc(%rip),%rax
|
|
|
8fbb1c |
+ and \$-16,%rsp # shouldn't be formally required
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+ movd $idx,%xmm5
|
|
|
8fbb1c |
+ movdqa 0(%rax),%xmm0 # 00000001000000010000000000000000
|
|
|
8fbb1c |
+ movdqa 16(%rax),%xmm1 # 00000002000000020000000200000002
|
|
|
8fbb1c |
+ lea 128($tbl),%r11 # size optimization
|
|
|
8fbb1c |
+ lea 128(%rsp),%rax # size optimization
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+ pshufd \$0,%xmm5,%xmm5 # broadcast $idx
|
|
|
8fbb1c |
+ movdqa %xmm1,%xmm4
|
|
|
8fbb1c |
+ movdqa %xmm1,%xmm2
|
|
|
8fbb1c |
+___
|
|
|
8fbb1c |
+########################################################################
|
|
|
8fbb1c |
+# calculate mask by comparing 0..31 to $idx and save result to stack
|
|
|
8fbb1c |
+#
|
|
|
8fbb1c |
+for($i=0;$i<$STRIDE/16;$i+=4) {
|
|
|
8fbb1c |
+$code.=<<___;
|
|
|
8fbb1c |
+ paddd %xmm0,%xmm1
|
|
|
8fbb1c |
+ pcmpeqd %xmm5,%xmm0 # compare to 1,0
|
|
|
8fbb1c |
+___
|
|
|
8fbb1c |
+$code.=<<___ if ($i);
|
|
|
8fbb1c |
+ movdqa %xmm3,`16*($i-1)-128`(%rax)
|
|
|
8fbb1c |
+___
|
|
|
8fbb1c |
+$code.=<<___;
|
|
|
8fbb1c |
+ movdqa %xmm4,%xmm3
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+ paddd %xmm1,%xmm2
|
|
|
8fbb1c |
+ pcmpeqd %xmm5,%xmm1 # compare to 3,2
|
|
|
8fbb1c |
+ movdqa %xmm0,`16*($i+0)-128`(%rax)
|
|
|
8fbb1c |
+ movdqa %xmm4,%xmm0
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+ paddd %xmm2,%xmm3
|
|
|
8fbb1c |
+ pcmpeqd %xmm5,%xmm2 # compare to 5,4
|
|
|
8fbb1c |
+ movdqa %xmm1,`16*($i+1)-128`(%rax)
|
|
|
8fbb1c |
+ movdqa %xmm4,%xmm1
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+ paddd %xmm3,%xmm0
|
|
|
8fbb1c |
+ pcmpeqd %xmm5,%xmm3 # compare to 7,6
|
|
|
8fbb1c |
+ movdqa %xmm2,`16*($i+2)-128`(%rax)
|
|
|
8fbb1c |
+ movdqa %xmm4,%xmm2
|
|
|
8fbb1c |
+___
|
|
|
8fbb1c |
+}
|
|
|
8fbb1c |
+$code.=<<___;
|
|
|
8fbb1c |
+ movdqa %xmm3,`16*($i-1)-128`(%rax)
|
|
|
8fbb1c |
jmp .Lgather
|
|
|
8fbb1c |
-.align 16
|
|
|
8fbb1c |
-.Lgather:
|
|
|
8fbb1c |
- movq `0*$STRIDE/4-96`($tbl),%xmm0
|
|
|
8fbb1c |
- movq `1*$STRIDE/4-96`($tbl),%xmm1
|
|
|
8fbb1c |
- pand %xmm4,%xmm0
|
|
|
8fbb1c |
- movq `2*$STRIDE/4-96`($tbl),%xmm2
|
|
|
8fbb1c |
- pand %xmm5,%xmm1
|
|
|
8fbb1c |
- movq `3*$STRIDE/4-96`($tbl),%xmm3
|
|
|
8fbb1c |
- pand %xmm6,%xmm2
|
|
|
8fbb1c |
- por %xmm1,%xmm0
|
|
|
8fbb1c |
- pand %xmm7,%xmm3
|
|
|
8fbb1c |
- por %xmm2,%xmm0
|
|
|
8fbb1c |
- lea $STRIDE($tbl),$tbl
|
|
|
8fbb1c |
- por %xmm3,%xmm0
|
|
|
8fbb1c |
|
|
|
8fbb1c |
+.align 32
|
|
|
8fbb1c |
+.Lgather:
|
|
|
8fbb1c |
+ pxor %xmm4,%xmm4
|
|
|
8fbb1c |
+ pxor %xmm5,%xmm5
|
|
|
8fbb1c |
+___
|
|
|
8fbb1c |
+for($i=0;$i<$STRIDE/16;$i+=4) {
|
|
|
8fbb1c |
+$code.=<<___;
|
|
|
8fbb1c |
+ movdqa `16*($i+0)-128`(%r11),%xmm0
|
|
|
8fbb1c |
+ movdqa `16*($i+1)-128`(%r11),%xmm1
|
|
|
8fbb1c |
+ movdqa `16*($i+2)-128`(%r11),%xmm2
|
|
|
8fbb1c |
+ pand `16*($i+0)-128`(%rax),%xmm0
|
|
|
8fbb1c |
+ movdqa `16*($i+3)-128`(%r11),%xmm3
|
|
|
8fbb1c |
+ pand `16*($i+1)-128`(%rax),%xmm1
|
|
|
8fbb1c |
+ por %xmm0,%xmm4
|
|
|
8fbb1c |
+ pand `16*($i+2)-128`(%rax),%xmm2
|
|
|
8fbb1c |
+ por %xmm1,%xmm5
|
|
|
8fbb1c |
+ pand `16*($i+3)-128`(%rax),%xmm3
|
|
|
8fbb1c |
+ por %xmm2,%xmm4
|
|
|
8fbb1c |
+ por %xmm3,%xmm5
|
|
|
8fbb1c |
+___
|
|
|
8fbb1c |
+}
|
|
|
8fbb1c |
+$code.=<<___;
|
|
|
8fbb1c |
+ por %xmm5,%xmm4
|
|
|
8fbb1c |
+ lea $STRIDE(%r11),%r11
|
|
|
8fbb1c |
+ pshufd \$0x4e,%xmm4,%xmm0
|
|
|
8fbb1c |
+ por %xmm4,%xmm0
|
|
|
8fbb1c |
movq %xmm0,($out) # m0=bp[0]
|
|
|
8fbb1c |
lea 8($out),$out
|
|
|
8fbb1c |
sub \$1,$num
|
|
|
8fbb1c |
jnz .Lgather
|
|
|
8fbb1c |
-___
|
|
|
8fbb1c |
-$code.=<<___ if ($win64);
|
|
|
8fbb1c |
- movaps %xmm6,(%rsp)
|
|
|
8fbb1c |
- movaps %xmm7,0x10(%rsp)
|
|
|
8fbb1c |
- lea 0x28(%rsp),%rsp
|
|
|
8fbb1c |
-___
|
|
|
8fbb1c |
-$code.=<<___;
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+ lea (%r10),%rsp
|
|
|
8fbb1c |
ret
|
|
|
8fbb1c |
.LSEH_end_bn_gather5:
|
|
|
8fbb1c |
.size bn_gather5,.-bn_gather5
|
|
|
8fbb1c |
@@ -913,9 +1033,9 @@ ___
|
|
|
8fbb1c |
}
|
|
|
8fbb1c |
$code.=<<___;
|
|
|
8fbb1c |
.align 64
|
|
|
8fbb1c |
-.Lmagic_masks:
|
|
|
8fbb1c |
- .long 0,0, 0,0, 0,0, -1,-1
|
|
|
8fbb1c |
- .long 0,0, 0,0, 0,0, 0,0
|
|
|
8fbb1c |
+.Linc:
|
|
|
8fbb1c |
+ .long 0,0, 1,1
|
|
|
8fbb1c |
+ .long 2,2, 2,2
|
|
|
8fbb1c |
.asciz "Montgomery Multiplication with scatter/gather for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
|
|
|
8fbb1c |
___
|
|
|
8fbb1c |
|
|
|
8fbb1c |
@@ -954,7 +1074,7 @@ mul_handler:
|
|
|
8fbb1c |
cmp %r10,%rbx # context->Rip
|
|
|
8fbb1c |
jb .Lcommon_seh_tail
|
|
|
8fbb1c |
|
|
|
8fbb1c |
- lea `40+48`(%rax),%rax
|
|
|
8fbb1c |
+ lea 48(%rax),%rax
|
|
|
8fbb1c |
|
|
|
8fbb1c |
mov 4(%r11),%r10d # HandlerData[1]
|
|
|
8fbb1c |
lea (%rsi,%r10),%r10 # end of alloca label
|
|
|
8fbb1c |
@@ -971,9 +1091,7 @@ mul_handler:
|
|
|
8fbb1c |
mov 192($context),%r10 # pull $num
|
|
|
8fbb1c |
mov 8(%rax,%r10,8),%rax # pull saved stack pointer
|
|
|
8fbb1c |
|
|
|
8fbb1c |
- movaps (%rax),%xmm0
|
|
|
8fbb1c |
- movaps 16(%rax),%xmm1
|
|
|
8fbb1c |
- lea `40+48`(%rax),%rax
|
|
|
8fbb1c |
+ lea 48(%rax),%rax
|
|
|
8fbb1c |
|
|
|
8fbb1c |
mov -8(%rax),%rbx
|
|
|
8fbb1c |
mov -16(%rax),%rbp
|
|
|
8fbb1c |
@@ -987,8 +1105,6 @@ mul_handler:
|
|
|
8fbb1c |
mov %r13,224($context) # restore context->R13
|
|
|
8fbb1c |
mov %r14,232($context) # restore context->R14
|
|
|
8fbb1c |
mov %r15,240($context) # restore context->R15
|
|
|
8fbb1c |
- movups %xmm0,512($context) # restore context->Xmm6
|
|
|
8fbb1c |
- movups %xmm1,528($context) # restore context->Xmm7
|
|
|
8fbb1c |
|
|
|
8fbb1c |
.Lcommon_seh_tail:
|
|
|
8fbb1c |
mov 8(%rax),%rdi
|
|
|
8fbb1c |
@@ -1057,10 +1173,9 @@ mul_handler:
|
|
|
8fbb1c |
.rva .Lmul4x_alloca,.Lmul4x_body,.Lmul4x_epilogue # HandlerData[]
|
|
|
8fbb1c |
.align 8
|
|
|
8fbb1c |
.LSEH_info_bn_gather5:
|
|
|
8fbb1c |
- .byte 0x01,0x0d,0x05,0x00
|
|
|
8fbb1c |
- .byte 0x0d,0x78,0x01,0x00 #movaps 0x10(rsp),xmm7
|
|
|
8fbb1c |
- .byte 0x08,0x68,0x00,0x00 #movaps (rsp),xmm6
|
|
|
8fbb1c |
- .byte 0x04,0x42,0x00,0x00 #sub rsp,0x28
|
|
|
8fbb1c |
+ .byte 0x01,0x0b,0x03,0x0a
|
|
|
8fbb1c |
+ .byte 0x0b,0x01,0x21,0x00 # sub rsp,0x108
|
|
|
8fbb1c |
+ .byte 0x04,0xa3,0x00,0x00 # lea r10,(rsp), set_frame r10
|
|
|
8fbb1c |
.align 8
|
|
|
8fbb1c |
___
|
|
|
8fbb1c |
}
|
|
|
8fbb1c |
diff -up openssl-1.0.1e/crypto/bn/bn_exp.c.rsa-const openssl-1.0.1e/crypto/bn/bn_exp.c
|
|
|
8fbb1c |
--- openssl-1.0.1e/crypto/bn/bn_exp.c.rsa-const 2013-02-11 16:26:04.000000000 +0100
|
|
|
8fbb1c |
+++ openssl-1.0.1e/crypto/bn/bn_exp.c 2016-02-24 12:00:20.807856735 +0100
|
|
|
8fbb1c |
@@ -111,6 +111,7 @@
|
|
|
8fbb1c |
|
|
|
8fbb1c |
|
|
|
8fbb1c |
#include "cryptlib.h"
|
|
|
8fbb1c |
+#include "constant_time_locl.h"
|
|
|
8fbb1c |
#include "bn_lcl.h"
|
|
|
8fbb1c |
|
|
|
8fbb1c |
#include <stdlib.h>
|
|
|
8fbb1c |
@@ -534,30 +535,74 @@ err:
|
|
|
8fbb1c |
* as cache lines are concerned. The following functions are used to transfer a BIGNUM
|
|
|
8fbb1c |
* from/to that table. */
|
|
|
8fbb1c |
|
|
|
8fbb1c |
-static int MOD_EXP_CTIME_COPY_TO_PREBUF(const BIGNUM *b, int top, unsigned char *buf, int idx, int width)
|
|
|
8fbb1c |
+static int MOD_EXP_CTIME_COPY_TO_PREBUF(const BIGNUM *b, int top, unsigned char *buf, int idx, int window)
|
|
|
8fbb1c |
{
|
|
|
8fbb1c |
- size_t i, j;
|
|
|
8fbb1c |
+ int i, j;
|
|
|
8fbb1c |
+ int width = 1 << window;
|
|
|
8fbb1c |
+ BN_ULONG *table = (BN_ULONG *)buf;
|
|
|
8fbb1c |
|
|
|
8fbb1c |
if (top > b->top)
|
|
|
8fbb1c |
top = b->top; /* this works because 'buf' is explicitly zeroed */
|
|
|
8fbb1c |
- for (i = 0, j=idx; i < top * sizeof b->d[0]; i++, j+=width)
|
|
|
8fbb1c |
+ for (i = 0, j=idx; i < top; i++, j+=width)
|
|
|
8fbb1c |
{
|
|
|
8fbb1c |
- buf[j] = ((unsigned char*)b->d)[i];
|
|
|
8fbb1c |
+ table[j] = b->d[i];
|
|
|
8fbb1c |
}
|
|
|
8fbb1c |
|
|
|
8fbb1c |
return 1;
|
|
|
8fbb1c |
}
|
|
|
8fbb1c |
|
|
|
8fbb1c |
-static int MOD_EXP_CTIME_COPY_FROM_PREBUF(BIGNUM *b, int top, unsigned char *buf, int idx, int width)
|
|
|
8fbb1c |
+static int MOD_EXP_CTIME_COPY_FROM_PREBUF(BIGNUM *b, int top, unsigned char *buf, int idx, int window)
|
|
|
8fbb1c |
{
|
|
|
8fbb1c |
- size_t i, j;
|
|
|
8fbb1c |
+ int i, j;
|
|
|
8fbb1c |
+ int width = 1 << window;
|
|
|
8fbb1c |
+ volatile BN_ULONG *table = (volatile BN_ULONG *)buf;
|
|
|
8fbb1c |
|
|
|
8fbb1c |
if (bn_wexpand(b, top) == NULL)
|
|
|
8fbb1c |
return 0;
|
|
|
8fbb1c |
|
|
|
8fbb1c |
- for (i=0, j=idx; i < top * sizeof b->d[0]; i++, j+=width)
|
|
|
8fbb1c |
+ if (window <= 3)
|
|
|
8fbb1c |
{
|
|
|
8fbb1c |
- ((unsigned char*)b->d)[i] = buf[j];
|
|
|
8fbb1c |
+ for (i = 0; i < top; i++, table += width)
|
|
|
8fbb1c |
+ {
|
|
|
8fbb1c |
+ BN_ULONG acc = 0;
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+ for (j = 0; j < width; j++)
|
|
|
8fbb1c |
+ {
|
|
|
8fbb1c |
+ acc |= table[j] &
|
|
|
8fbb1c |
+ ((BN_ULONG)0 - (constant_time_eq_int(j,idx)&1));
|
|
|
8fbb1c |
+ }
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+ b->d[i] = acc;
|
|
|
8fbb1c |
+ }
|
|
|
8fbb1c |
+ }
|
|
|
8fbb1c |
+ else
|
|
|
8fbb1c |
+ {
|
|
|
8fbb1c |
+ int xstride = 1 << (window - 2);
|
|
|
8fbb1c |
+ BN_ULONG y0, y1, y2, y3;
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+ i = idx >> (window - 2); /* equivalent of idx / xstride */
|
|
|
8fbb1c |
+ idx &= xstride - 1; /* equivalent of idx % xstride */
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+ y0 = (BN_ULONG)0 - (constant_time_eq_int(i,0)&1;;
|
|
|
8fbb1c |
+ y1 = (BN_ULONG)0 - (constant_time_eq_int(i,1)&1;;
|
|
|
8fbb1c |
+ y2 = (BN_ULONG)0 - (constant_time_eq_int(i,2)&1;;
|
|
|
8fbb1c |
+ y3 = (BN_ULONG)0 - (constant_time_eq_int(i,3)&1;;
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+ for (i = 0; i < top; i++, table += width)
|
|
|
8fbb1c |
+ {
|
|
|
8fbb1c |
+ BN_ULONG acc = 0;
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+ for (j = 0; j < xstride; j++)
|
|
|
8fbb1c |
+ {
|
|
|
8fbb1c |
+ acc |= ( (table[j + 0 * xstride] & y0) |
|
|
|
8fbb1c |
+ (table[j + 1 * xstride] & y1) |
|
|
|
8fbb1c |
+ (table[j + 2 * xstride] & y2) |
|
|
|
8fbb1c |
+ (table[j + 3 * xstride] & y3) )
|
|
|
8fbb1c |
+ & ((BN_ULONG)0 - (constant_time_eq_int(j,idx)&1));
|
|
|
8fbb1c |
+ }
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+ b->d[i] = acc;
|
|
|
8fbb1c |
+ }
|
|
|
8fbb1c |
}
|
|
|
8fbb1c |
|
|
|
8fbb1c |
b->top = top;
|
|
|
8fbb1c |
@@ -592,17 +637,27 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr
|
|
|
8fbb1c |
bn_check_top(p);
|
|
|
8fbb1c |
bn_check_top(m);
|
|
|
8fbb1c |
|
|
|
8fbb1c |
- top = m->top;
|
|
|
8fbb1c |
-
|
|
|
8fbb1c |
- if (!(m->d[0] & 1))
|
|
|
8fbb1c |
+ if (!BN_is_odd(m))
|
|
|
8fbb1c |
{
|
|
|
8fbb1c |
BNerr(BN_F_BN_MOD_EXP_MONT_CONSTTIME,BN_R_CALLED_WITH_EVEN_MODULUS);
|
|
|
8fbb1c |
return(0);
|
|
|
8fbb1c |
}
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+ top = m->top;
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
bits=BN_num_bits(p);
|
|
|
8fbb1c |
if (bits == 0)
|
|
|
8fbb1c |
{
|
|
|
8fbb1c |
- ret = BN_one(rr);
|
|
|
8fbb1c |
+ /* x**0 mod 1 is still zero. */
|
|
|
8fbb1c |
+ if (BN_is_one(m))
|
|
|
8fbb1c |
+ {
|
|
|
8fbb1c |
+ ret = 1;
|
|
|
8fbb1c |
+ BN_zero(rr);
|
|
|
8fbb1c |
+ }
|
|
|
8fbb1c |
+ else
|
|
|
8fbb1c |
+ {
|
|
|
8fbb1c |
+ ret = BN_one(rr);
|
|
|
8fbb1c |
+ }
|
|
|
8fbb1c |
return ret;
|
|
|
8fbb1c |
}
|
|
|
8fbb1c |
|
|
|
8fbb1c |
@@ -680,7 +735,7 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr
|
|
|
8fbb1c |
|
|
|
8fbb1c |
/* Dedicated window==4 case improves 512-bit RSA sign by ~15%, but as
|
|
|
8fbb1c |
* 512-bit RSA is hardly relevant, we omit it to spare size... */
|
|
|
8fbb1c |
- if (window==5)
|
|
|
8fbb1c |
+ if (window==5 && top>1)
|
|
|
8fbb1c |
{
|
|
|
8fbb1c |
void bn_mul_mont_gather5(BN_ULONG *rp,const BN_ULONG *ap,
|
|
|
8fbb1c |
const void *table,const BN_ULONG *np,
|
|
|
8fbb1c |
@@ -767,8 +822,8 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr
|
|
|
8fbb1c |
else
|
|
|
8fbb1c |
#endif
|
|
|
8fbb1c |
{
|
|
|
8fbb1c |
- if (!MOD_EXP_CTIME_COPY_TO_PREBUF(&tmp, top, powerbuf, 0, numPowers)) goto err;
|
|
|
8fbb1c |
- if (!MOD_EXP_CTIME_COPY_TO_PREBUF(&am, top, powerbuf, 1, numPowers)) goto err;
|
|
|
8fbb1c |
+ if (!MOD_EXP_CTIME_COPY_TO_PREBUF(&tmp, top, powerbuf, 0, window)) goto err;
|
|
|
8fbb1c |
+ if (!MOD_EXP_CTIME_COPY_TO_PREBUF(&am, top, powerbuf, 1, window)) goto err;
|
|
|
8fbb1c |
|
|
|
8fbb1c |
/* If the window size is greater than 1, then calculate
|
|
|
8fbb1c |
* val[i=2..2^winsize-1]. Powers are computed as a*a^(i-1)
|
|
|
8fbb1c |
@@ -778,20 +833,20 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr
|
|
|
8fbb1c |
if (window > 1)
|
|
|
8fbb1c |
{
|
|
|
8fbb1c |
if (!BN_mod_mul_montgomery(&tmp,&am,&am,mont,ctx)) goto err;
|
|
|
8fbb1c |
- if (!MOD_EXP_CTIME_COPY_TO_PREBUF(&tmp, top, powerbuf, 2, numPowers)) goto err;
|
|
|
8fbb1c |
+ if (!MOD_EXP_CTIME_COPY_TO_PREBUF(&tmp, top, powerbuf, 2, window)) goto err;
|
|
|
8fbb1c |
for (i=3; i
|
|
|
8fbb1c |
{
|
|
|
8fbb1c |
/* Calculate a^i = a^(i-1) * a */
|
|
|
8fbb1c |
if (!BN_mod_mul_montgomery(&tmp,&am,&tmp,mont,ctx))
|
|
|
8fbb1c |
goto err;
|
|
|
8fbb1c |
- if (!MOD_EXP_CTIME_COPY_TO_PREBUF(&tmp, top, powerbuf, i, numPowers)) goto err;
|
|
|
8fbb1c |
+ if (!MOD_EXP_CTIME_COPY_TO_PREBUF(&tmp, top, powerbuf, i, window)) goto err;
|
|
|
8fbb1c |
}
|
|
|
8fbb1c |
}
|
|
|
8fbb1c |
|
|
|
8fbb1c |
bits--;
|
|
|
8fbb1c |
for (wvalue=0, i=bits%window; i>=0; i--,bits--)
|
|
|
8fbb1c |
wvalue = (wvalue<<1)+BN_is_bit_set(p,bits);
|
|
|
8fbb1c |
- if (!MOD_EXP_CTIME_COPY_FROM_PREBUF(&tmp,top,powerbuf,wvalue,numPowers)) goto err;
|
|
|
8fbb1c |
+ if (!MOD_EXP_CTIME_COPY_FROM_PREBUF(&tmp,top,powerbuf,wvalue,window)) goto err;
|
|
|
8fbb1c |
|
|
|
8fbb1c |
/* Scan the exponent one window at a time starting from the most
|
|
|
8fbb1c |
* significant bits.
|
|
|
8fbb1c |
@@ -808,7 +863,7 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr
|
|
|
8fbb1c |
}
|
|
|
8fbb1c |
|
|
|
8fbb1c |
/* Fetch the appropriate pre-computed value from the pre-buf */
|
|
|
8fbb1c |
- if (!MOD_EXP_CTIME_COPY_FROM_PREBUF(&am, top, powerbuf, wvalue, numPowers)) goto err;
|
|
|
8fbb1c |
+ if (!MOD_EXP_CTIME_COPY_FROM_PREBUF(&am, top, powerbuf, wvalue, window)) goto err;
|
|
|
8fbb1c |
|
|
|
8fbb1c |
/* Multiply the result into the intermediate result */
|
|
|
8fbb1c |
if (!BN_mod_mul_montgomery(&tmp,&tmp,&am,mont,ctx)) goto err;
|
|
|
8fbb1c |
@@ -874,7 +929,16 @@ int BN_mod_exp_mont_word(BIGNUM *rr, BN_
|
|
|
8fbb1c |
bits = BN_num_bits(p);
|
|
|
8fbb1c |
if (bits == 0)
|
|
|
8fbb1c |
{
|
|
|
8fbb1c |
- ret = BN_one(rr);
|
|
|
8fbb1c |
+ /* x**0 mod 1 is still zero. */
|
|
|
8fbb1c |
+ if (BN_is_one(m))
|
|
|
8fbb1c |
+ {
|
|
|
8fbb1c |
+ ret = 1;
|
|
|
8fbb1c |
+ BN_zero(rr);
|
|
|
8fbb1c |
+ }
|
|
|
8fbb1c |
+ else
|
|
|
8fbb1c |
+ {
|
|
|
8fbb1c |
+ ret = BN_one(rr);
|
|
|
8fbb1c |
+ }
|
|
|
8fbb1c |
return ret;
|
|
|
8fbb1c |
}
|
|
|
8fbb1c |
if (a == 0)
|
|
|
8fbb1c |
@@ -997,10 +1061,18 @@ int BN_mod_exp_simple(BIGNUM *r, const B
|
|
|
8fbb1c |
}
|
|
|
8fbb1c |
|
|
|
8fbb1c |
bits=BN_num_bits(p);
|
|
|
8fbb1c |
-
|
|
|
8fbb1c |
if (bits == 0)
|
|
|
8fbb1c |
{
|
|
|
8fbb1c |
- ret = BN_one(r);
|
|
|
8fbb1c |
+ /* x**0 mod 1 is still zero. */
|
|
|
8fbb1c |
+ if (BN_is_one(m))
|
|
|
8fbb1c |
+ {
|
|
|
8fbb1c |
+ ret = 1;
|
|
|
8fbb1c |
+ BN_zero(r);
|
|
|
8fbb1c |
+ }
|
|
|
8fbb1c |
+ else
|
|
|
8fbb1c |
+ {
|
|
|
8fbb1c |
+ ret = BN_one(r);
|
|
|
8fbb1c |
+ }
|
|
|
8fbb1c |
return ret;
|
|
|
8fbb1c |
}
|
|
|
8fbb1c |
|
|
|
8fbb1c |
diff -up openssl-1.0.1e/crypto/constant_time_locl.h.rsa-const openssl-1.0.1e/crypto/constant_time_locl.h
|
|
|
8fbb1c |
--- openssl-1.0.1e/crypto/constant_time_locl.h.rsa-const 2016-02-24 11:29:44.635303119 +0100
|
|
|
8fbb1c |
+++ openssl-1.0.1e/crypto/constant_time_locl.h 2016-02-24 11:27:44.000000000 +0100
|
|
|
8fbb1c |
@@ -0,0 +1,211 @@
|
|
|
8fbb1c |
+/* crypto/constant_time_locl.h */
|
|
|
8fbb1c |
+/*-
|
|
|
8fbb1c |
+ * Utilities for constant-time cryptography.
|
|
|
8fbb1c |
+ *
|
|
|
8fbb1c |
+ * Author: Emilia Kasper (emilia@openssl.org)
|
|
|
8fbb1c |
+ * Based on previous work by Bodo Moeller, Emilia Kasper, Adam Langley
|
|
|
8fbb1c |
+ * (Google).
|
|
|
8fbb1c |
+ * ====================================================================
|
|
|
8fbb1c |
+ * Copyright (c) 2014 The OpenSSL Project. All rights reserved.
|
|
|
8fbb1c |
+ *
|
|
|
8fbb1c |
+ * Redistribution and use in source and binary forms, with or without
|
|
|
8fbb1c |
+ * modification, are permitted provided that the following conditions
|
|
|
8fbb1c |
+ * are met:
|
|
|
8fbb1c |
+ * 1. Redistributions of source code must retain the copyright
|
|
|
8fbb1c |
+ * notice, this list of conditions and the following disclaimer.
|
|
|
8fbb1c |
+ * 2. Redistributions in binary form must reproduce the above copyright
|
|
|
8fbb1c |
+ * notice, this list of conditions and the following disclaimer in the
|
|
|
8fbb1c |
+ * documentation and/or other materials provided with the distribution.
|
|
|
8fbb1c |
+ * 3. All advertising materials mentioning features or use of this software
|
|
|
8fbb1c |
+ * must display the following acknowledgement:
|
|
|
8fbb1c |
+ * "This product includes cryptographic software written by
|
|
|
8fbb1c |
+ * Eric Young (eay@cryptsoft.com)"
|
|
|
8fbb1c |
+ * The word 'cryptographic' can be left out if the rouines from the library
|
|
|
8fbb1c |
+ * being used are not cryptographic related :-).
|
|
|
8fbb1c |
+ * 4. If you include any Windows specific code (or a derivative thereof) from
|
|
|
8fbb1c |
+ * the apps directory (application code) you must include an acknowledgement:
|
|
|
8fbb1c |
+ * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)"
|
|
|
8fbb1c |
+ *
|
|
|
8fbb1c |
+ * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND
|
|
|
8fbb1c |
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
|
8fbb1c |
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
|
8fbb1c |
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
|
|
8fbb1c |
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
|
8fbb1c |
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
|
8fbb1c |
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
|
8fbb1c |
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
|
8fbb1c |
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
|
8fbb1c |
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
|
8fbb1c |
+ * SUCH DAMAGE.
|
|
|
8fbb1c |
+ *
|
|
|
8fbb1c |
+ * The licence and distribution terms for any publically available version or
|
|
|
8fbb1c |
+ * derivative of this code cannot be changed. i.e. this code cannot simply be
|
|
|
8fbb1c |
+ * copied and put under another distribution licence
|
|
|
8fbb1c |
+ * [including the GNU Public Licence.]
|
|
|
8fbb1c |
+ */
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+#ifndef HEADER_CONSTANT_TIME_LOCL_H
|
|
|
8fbb1c |
+# define HEADER_CONSTANT_TIME_LOCL_H
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+# include "e_os.h" /* For 'inline' */
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+#ifdef __cplusplus
|
|
|
8fbb1c |
+extern "C" {
|
|
|
8fbb1c |
+#endif
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+/*-
|
|
|
8fbb1c |
+ * The boolean methods return a bitmask of all ones (0xff...f) for true
|
|
|
8fbb1c |
+ * and 0 for false. This is useful for choosing a value based on the result
|
|
|
8fbb1c |
+ * of a conditional in constant time. For example,
|
|
|
8fbb1c |
+ *
|
|
|
8fbb1c |
+ * if (a < b) {
|
|
|
8fbb1c |
+ * c = a;
|
|
|
8fbb1c |
+ * } else {
|
|
|
8fbb1c |
+ * c = b;
|
|
|
8fbb1c |
+ * }
|
|
|
8fbb1c |
+ *
|
|
|
8fbb1c |
+ * can be written as
|
|
|
8fbb1c |
+ *
|
|
|
8fbb1c |
+ * unsigned int lt = constant_time_lt(a, b);
|
|
|
8fbb1c |
+ * c = constant_time_select(lt, a, b);
|
|
|
8fbb1c |
+ */
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+/*
|
|
|
8fbb1c |
+ * Returns the given value with the MSB copied to all the other
|
|
|
8fbb1c |
+ * bits. Uses the fact that arithmetic shift shifts-in the sign bit.
|
|
|
8fbb1c |
+ * However, this is not ensured by the C standard so you may need to
|
|
|
8fbb1c |
+ * replace this with something else on odd CPUs.
|
|
|
8fbb1c |
+ */
|
|
|
8fbb1c |
+static inline unsigned int constant_time_msb(unsigned int a);
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+/*
|
|
|
8fbb1c |
+ * Returns 0xff..f if a < b and 0 otherwise.
|
|
|
8fbb1c |
+ */
|
|
|
8fbb1c |
+static inline unsigned int constant_time_lt(unsigned int a, unsigned int b);
|
|
|
8fbb1c |
+/* Convenience method for getting an 8-bit mask. */
|
|
|
8fbb1c |
+static inline unsigned char constant_time_lt_8(unsigned int a,
|
|
|
8fbb1c |
+ unsigned int b);
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+/*
|
|
|
8fbb1c |
+ * Returns 0xff..f if a >= b and 0 otherwise.
|
|
|
8fbb1c |
+ */
|
|
|
8fbb1c |
+static inline unsigned int constant_time_ge(unsigned int a, unsigned int b);
|
|
|
8fbb1c |
+/* Convenience method for getting an 8-bit mask. */
|
|
|
8fbb1c |
+static inline unsigned char constant_time_ge_8(unsigned int a,
|
|
|
8fbb1c |
+ unsigned int b);
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+/*
|
|
|
8fbb1c |
+ * Returns 0xff..f if a == 0 and 0 otherwise.
|
|
|
8fbb1c |
+ */
|
|
|
8fbb1c |
+static inline unsigned int constant_time_is_zero(unsigned int a);
|
|
|
8fbb1c |
+/* Convenience method for getting an 8-bit mask. */
|
|
|
8fbb1c |
+static inline unsigned char constant_time_is_zero_8(unsigned int a);
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+/*
|
|
|
8fbb1c |
+ * Returns 0xff..f if a == b and 0 otherwise.
|
|
|
8fbb1c |
+ */
|
|
|
8fbb1c |
+static inline unsigned int constant_time_eq(unsigned int a, unsigned int b);
|
|
|
8fbb1c |
+/* Convenience method for getting an 8-bit mask. */
|
|
|
8fbb1c |
+static inline unsigned char constant_time_eq_8(unsigned int a,
|
|
|
8fbb1c |
+ unsigned int b);
|
|
|
8fbb1c |
+/* Signed integers. */
|
|
|
8fbb1c |
+static inline unsigned int constant_time_eq_int(int a, int b);
|
|
|
8fbb1c |
+/* Convenience method for getting an 8-bit mask. */
|
|
|
8fbb1c |
+static inline unsigned char constant_time_eq_int_8(int a, int b);
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+/*-
|
|
|
8fbb1c |
+ * Returns (mask & a) | (~mask & b).
|
|
|
8fbb1c |
+ *
|
|
|
8fbb1c |
+ * When |mask| is all 1s or all 0s (as returned by the methods above),
|
|
|
8fbb1c |
+ * the select methods return either |a| (if |mask| is nonzero) or |b|
|
|
|
8fbb1c |
+ * (if |mask| is zero).
|
|
|
8fbb1c |
+ */
|
|
|
8fbb1c |
+static inline unsigned int constant_time_select(unsigned int mask,
|
|
|
8fbb1c |
+ unsigned int a,
|
|
|
8fbb1c |
+ unsigned int b);
|
|
|
8fbb1c |
+/* Convenience method for unsigned chars. */
|
|
|
8fbb1c |
+static inline unsigned char constant_time_select_8(unsigned char mask,
|
|
|
8fbb1c |
+ unsigned char a,
|
|
|
8fbb1c |
+ unsigned char b);
|
|
|
8fbb1c |
+/* Convenience method for signed integers. */
|
|
|
8fbb1c |
+static inline int constant_time_select_int(unsigned int mask, int a, int b);
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+static inline unsigned int constant_time_msb(unsigned int a)
|
|
|
8fbb1c |
+{
|
|
|
8fbb1c |
+ return 0 - (a >> (sizeof(a) * 8 - 1));
|
|
|
8fbb1c |
+}
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+static inline unsigned int constant_time_lt(unsigned int a, unsigned int b)
|
|
|
8fbb1c |
+{
|
|
|
8fbb1c |
+ return constant_time_msb(a ^ ((a ^ b) | ((a - b) ^ b)));
|
|
|
8fbb1c |
+}
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+static inline unsigned char constant_time_lt_8(unsigned int a, unsigned int b)
|
|
|
8fbb1c |
+{
|
|
|
8fbb1c |
+ return (unsigned char)(constant_time_lt(a, b));
|
|
|
8fbb1c |
+}
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+static inline unsigned int constant_time_ge(unsigned int a, unsigned int b)
|
|
|
8fbb1c |
+{
|
|
|
8fbb1c |
+ return ~constant_time_lt(a, b);
|
|
|
8fbb1c |
+}
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+static inline unsigned char constant_time_ge_8(unsigned int a, unsigned int b)
|
|
|
8fbb1c |
+{
|
|
|
8fbb1c |
+ return (unsigned char)(constant_time_ge(a, b));
|
|
|
8fbb1c |
+}
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+static inline unsigned int constant_time_is_zero(unsigned int a)
|
|
|
8fbb1c |
+{
|
|
|
8fbb1c |
+ return constant_time_msb(~a & (a - 1));
|
|
|
8fbb1c |
+}
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+static inline unsigned char constant_time_is_zero_8(unsigned int a)
|
|
|
8fbb1c |
+{
|
|
|
8fbb1c |
+ return (unsigned char)(constant_time_is_zero(a));
|
|
|
8fbb1c |
+}
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+static inline unsigned int constant_time_eq(unsigned int a, unsigned int b)
|
|
|
8fbb1c |
+{
|
|
|
8fbb1c |
+ return constant_time_is_zero(a ^ b);
|
|
|
8fbb1c |
+}
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+static inline unsigned char constant_time_eq_8(unsigned int a, unsigned int b)
|
|
|
8fbb1c |
+{
|
|
|
8fbb1c |
+ return (unsigned char)(constant_time_eq(a, b));
|
|
|
8fbb1c |
+}
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+static inline unsigned int constant_time_eq_int(int a, int b)
|
|
|
8fbb1c |
+{
|
|
|
8fbb1c |
+ return constant_time_eq((unsigned)(a), (unsigned)(b));
|
|
|
8fbb1c |
+}
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+static inline unsigned char constant_time_eq_int_8(int a, int b)
|
|
|
8fbb1c |
+{
|
|
|
8fbb1c |
+ return constant_time_eq_8((unsigned)(a), (unsigned)(b));
|
|
|
8fbb1c |
+}
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+static inline unsigned int constant_time_select(unsigned int mask,
|
|
|
8fbb1c |
+ unsigned int a,
|
|
|
8fbb1c |
+ unsigned int b)
|
|
|
8fbb1c |
+{
|
|
|
8fbb1c |
+ return (mask & a) | (~mask & b);
|
|
|
8fbb1c |
+}
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+static inline unsigned char constant_time_select_8(unsigned char mask,
|
|
|
8fbb1c |
+ unsigned char a,
|
|
|
8fbb1c |
+ unsigned char b)
|
|
|
8fbb1c |
+{
|
|
|
8fbb1c |
+ return (unsigned char)(constant_time_select(mask, a, b));
|
|
|
8fbb1c |
+}
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+static inline int constant_time_select_int(unsigned int mask, int a, int b)
|
|
|
8fbb1c |
+{
|
|
|
8fbb1c |
+ return (int)(constant_time_select(mask, (unsigned)(a), (unsigned)(b)));
|
|
|
8fbb1c |
+}
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+#ifdef __cplusplus
|
|
|
8fbb1c |
+}
|
|
|
8fbb1c |
+#endif
|
|
|
8fbb1c |
+
|
|
|
8fbb1c |
+#endif /* HEADER_CONSTANT_TIME_LOCL_H */
|
|
|
8fbb1c |
diff -up openssl-1.0.1e/crypto/perlasm/x86_64-xlate.pl.rsa-const openssl-1.0.1e/crypto/perlasm/x86_64-xlate.pl
|
|
|
8fbb1c |
--- openssl-1.0.1e/crypto/perlasm/x86_64-xlate.pl.rsa-const 2016-01-14 17:38:50.121212850 +0100
|
|
|
8fbb1c |
+++ openssl-1.0.1e/crypto/perlasm/x86_64-xlate.pl 2016-02-24 11:26:40.316912890 +0100
|
|
|
8fbb1c |
@@ -121,7 +121,7 @@ my %globals;
|
|
|
8fbb1c |
$self->{sz} = "";
|
|
|
8fbb1c |
} elsif ($self->{op} =~ /^v/) { # VEX
|
|
|
8fbb1c |
$self->{sz} = "";
|
|
|
8fbb1c |
- } elsif ($self->{op} =~ /movq/ && $line =~ /%xmm/) {
|
|
|
8fbb1c |
+ } elsif ($self->{op} =~ /mov[dq]/ && $line =~ /%xmm/) {
|
|
|
8fbb1c |
$self->{sz} = "";
|
|
|
8fbb1c |
} elsif ($self->{op} =~ /([a-z]{3,})([qlwb])$/) {
|
|
|
8fbb1c |
$self->{op} = $1;
|