|
 |
fd2893 |
diff -up openssl-1.0.2k/crypto/aes/asm/aesp8-ppc.pl.ppc-update openssl-1.0.2k/crypto/aes/asm/aesp8-ppc.pl
|
|
|
fd2893 |
--- openssl-1.0.2k/crypto/aes/asm/aesp8-ppc.pl.ppc-update 2017-01-26 14:22:03.000000000 +0100
|
|
|
fd2893 |
+++ openssl-1.0.2k/crypto/aes/asm/aesp8-ppc.pl 2017-04-13 09:51:40.611133165 +0200
|
|
|
fd2893 |
@@ -20,6 +20,19 @@
|
|
|
fd2893 |
# instructions are interleaved. It's reckoned that eventual
|
|
|
fd2893 |
# misalignment penalties at page boundaries are in average lower
|
|
|
fd2893 |
# than additional overhead in pure AltiVec approach.
|
|
|
fd2893 |
+#
|
|
|
fd2893 |
+# May 2016
|
|
|
fd2893 |
+#
|
|
|
fd2893 |
+# Add XTS subroutine, 9x on little- and 12x improvement on big-endian
|
|
|
fd2893 |
+# systems were measured.
|
|
|
fd2893 |
+#
|
|
|
fd2893 |
+######################################################################
|
|
|
fd2893 |
+# Current large-block performance in cycles per byte processed with
|
|
|
fd2893 |
+# 128-bit key (less is better).
|
|
|
fd2893 |
+#
|
|
|
fd2893 |
+# CBC en-/decrypt CTR XTS
|
|
|
fd2893 |
+# POWER8[le] 3.96/0.72 0.74 1.1
|
|
|
fd2893 |
+# POWER8[be] 3.75/0.65 0.66 1.0
|
|
|
fd2893 |
|
|
|
fd2893 |
$flavour = shift;
|
|
|
fd2893 |
|
|
|
fd2893 |
@@ -486,6 +499,8 @@ $code.=<<___;
|
|
|
fd2893 |
.globl .${prefix}_cbc_encrypt
|
|
|
fd2893 |
.align 5
|
|
|
fd2893 |
.${prefix}_cbc_encrypt:
|
|
|
fd2893 |
+ li $idx,15
|
|
|
fd2893 |
+ andc $len,$len,$idx
|
|
|
fd2893 |
${UCMP}i $len,16
|
|
|
fd2893 |
bltlr-
|
|
|
fd2893 |
|
|
|
fd2893 |
@@ -494,7 +509,6 @@ $code.=<<___;
|
|
|
fd2893 |
mfspr $vrsave,256
|
|
|
fd2893 |
mtspr 256,r0
|
|
|
fd2893 |
|
|
|
fd2893 |
- li $idx,15
|
|
|
fd2893 |
vxor $rndkey0,$rndkey0,$rndkey0
|
|
|
fd2893 |
le?vspltisb $tmp,0x0f
|
|
|
fd2893 |
|
|
|
fd2893 |
@@ -1887,6 +1901,1849 @@ Lctr32_enc8x_done:
|
|
|
fd2893 |
___
|
|
|
fd2893 |
}} }}}
|
|
|
fd2893 |
|
|
|
fd2893 |
+#########################################################################
|
|
|
fd2893 |
+{{{ # XTS procedures #
|
|
|
fd2893 |
+# int aes_p8_xts_[en|de]crypt(const char *inp, char *out, size_t len, #
|
|
|
fd2893 |
+# const AES_KEY *key1, const AES_KEY *key2, #
|
|
|
fd2893 |
+# [const] unsigned char iv[16]); #
|
|
|
fd2893 |
+# If $key2 is NULL, then a "tweak chaining" mode is engaged, in which #
|
|
|
fd2893 |
+# input tweak value is assumed to be encrypted already, and last tweak #
|
|
|
fd2893 |
+# value, one suitable for consecutive call on same chunk of data, is #
|
|
|
fd2893 |
+# written back to original buffer. In addition, in "tweak chaining" #
|
|
|
fd2893 |
+# mode only complete input blocks are processed. #
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+my ($inp,$out,$len,$key1,$key2,$ivp,$rounds,$idx) = map("r$_",(3..10));
|
|
|
fd2893 |
+my ($rndkey0,$rndkey1,$inout) = map("v$_",(0..2));
|
|
|
fd2893 |
+my ($output,$inptail,$inpperm,$leperm,$keyperm) = map("v$_",(3..7));
|
|
|
fd2893 |
+my ($tweak,$seven,$eighty7,$tmp,$tweak1) = map("v$_",(8..12));
|
|
|
fd2893 |
+my $taillen = $key2;
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ ($inp,$idx) = ($idx,$inp); # reassign
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+$code.=<<___;
|
|
|
fd2893 |
+.globl .${prefix}_xts_encrypt
|
|
|
fd2893 |
+.align 5
|
|
|
fd2893 |
+.${prefix}_xts_encrypt:
|
|
|
fd2893 |
+ mr $inp,r3 # reassign
|
|
|
fd2893 |
+ li r3,-1
|
|
|
fd2893 |
+ ${UCMP}i $len,16
|
|
|
fd2893 |
+ bltlr-
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ lis r0,0xfff0
|
|
|
fd2893 |
+ mfspr r12,256 # save vrsave
|
|
|
fd2893 |
+ li r11,0
|
|
|
fd2893 |
+ mtspr 256,r0
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vspltisb $seven,0x07 # 0x070707..07
|
|
|
fd2893 |
+ le?lvsl $leperm,r11,r11
|
|
|
fd2893 |
+ le?vspltisb $tmp,0x0f
|
|
|
fd2893 |
+ le?vxor $leperm,$leperm,$seven
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ li $idx,15
|
|
|
fd2893 |
+ lvx $tweak,0,$ivp # load [unaligned] iv
|
|
|
fd2893 |
+ lvsl $inpperm,0,$ivp
|
|
|
fd2893 |
+ lvx $inptail,$idx,$ivp
|
|
|
fd2893 |
+ le?vxor $inpperm,$inpperm,$tmp
|
|
|
fd2893 |
+ vperm $tweak,$tweak,$inptail,$inpperm
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ neg r11,$inp
|
|
|
fd2893 |
+ lvsr $inpperm,0,r11 # prepare for unaligned load
|
|
|
fd2893 |
+ lvx $inout,0,$inp
|
|
|
fd2893 |
+ addi $inp,$inp,15 # 15 is not typo
|
|
|
fd2893 |
+ le?vxor $inpperm,$inpperm,$tmp
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ ${UCMP}i $key2,0 # key2==NULL?
|
|
|
fd2893 |
+ beq Lxts_enc_no_key2
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ ?lvsl $keyperm,0,$key2 # prepare for unaligned key
|
|
|
fd2893 |
+ lwz $rounds,240($key2)
|
|
|
fd2893 |
+ srwi $rounds,$rounds,1
|
|
|
fd2893 |
+ subi $rounds,$rounds,1
|
|
|
fd2893 |
+ li $idx,16
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ lvx $rndkey0,0,$key2
|
|
|
fd2893 |
+ lvx $rndkey1,$idx,$key2
|
|
|
fd2893 |
+ addi $idx,$idx,16
|
|
|
fd2893 |
+ ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
|
|
|
fd2893 |
+ vxor $tweak,$tweak,$rndkey0
|
|
|
fd2893 |
+ lvx $rndkey0,$idx,$key2
|
|
|
fd2893 |
+ addi $idx,$idx,16
|
|
|
fd2893 |
+ mtctr $rounds
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+Ltweak_xts_enc:
|
|
|
fd2893 |
+ ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
|
|
|
fd2893 |
+ vcipher $tweak,$tweak,$rndkey1
|
|
|
fd2893 |
+ lvx $rndkey1,$idx,$key2
|
|
|
fd2893 |
+ addi $idx,$idx,16
|
|
|
fd2893 |
+ ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
|
|
|
fd2893 |
+ vcipher $tweak,$tweak,$rndkey0
|
|
|
fd2893 |
+ lvx $rndkey0,$idx,$key2
|
|
|
fd2893 |
+ addi $idx,$idx,16
|
|
|
fd2893 |
+ bdnz Ltweak_xts_enc
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
|
|
|
fd2893 |
+ vcipher $tweak,$tweak,$rndkey1
|
|
|
fd2893 |
+ lvx $rndkey1,$idx,$key2
|
|
|
fd2893 |
+ ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
|
|
|
fd2893 |
+ vcipherlast $tweak,$tweak,$rndkey0
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ li $ivp,0 # don't chain the tweak
|
|
|
fd2893 |
+ b Lxts_enc
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+Lxts_enc_no_key2:
|
|
|
fd2893 |
+ li $idx,-16
|
|
|
fd2893 |
+ and $len,$len,$idx # in "tweak chaining"
|
|
|
fd2893 |
+ # mode only complete
|
|
|
fd2893 |
+ # blocks are processed
|
|
|
fd2893 |
+Lxts_enc:
|
|
|
fd2893 |
+ lvx $inptail,0,$inp
|
|
|
fd2893 |
+ addi $inp,$inp,16
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ ?lvsl $keyperm,0,$key1 # prepare for unaligned key
|
|
|
fd2893 |
+ lwz $rounds,240($key1)
|
|
|
fd2893 |
+ srwi $rounds,$rounds,1
|
|
|
fd2893 |
+ subi $rounds,$rounds,1
|
|
|
fd2893 |
+ li $idx,16
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vslb $eighty7,$seven,$seven # 0x808080..80
|
|
|
fd2893 |
+ vor $eighty7,$eighty7,$seven # 0x878787..87
|
|
|
fd2893 |
+ vspltisb $tmp,1 # 0x010101..01
|
|
|
fd2893 |
+ vsldoi $eighty7,$eighty7,$tmp,15 # 0x870101..01
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ ${UCMP}i $len,96
|
|
|
fd2893 |
+ bge _aesp8_xts_encrypt6x
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ andi. $taillen,$len,15
|
|
|
fd2893 |
+ subic r0,$len,32
|
|
|
fd2893 |
+ subi $taillen,$taillen,16
|
|
|
fd2893 |
+ subfe r0,r0,r0
|
|
|
fd2893 |
+ and r0,r0,$taillen
|
|
|
fd2893 |
+ add $inp,$inp,r0
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ lvx $rndkey0,0,$key1
|
|
|
fd2893 |
+ lvx $rndkey1,$idx,$key1
|
|
|
fd2893 |
+ addi $idx,$idx,16
|
|
|
fd2893 |
+ vperm $inout,$inout,$inptail,$inpperm
|
|
|
fd2893 |
+ ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
|
|
|
fd2893 |
+ vxor $inout,$inout,$tweak
|
|
|
fd2893 |
+ vxor $inout,$inout,$rndkey0
|
|
|
fd2893 |
+ lvx $rndkey0,$idx,$key1
|
|
|
fd2893 |
+ addi $idx,$idx,16
|
|
|
fd2893 |
+ mtctr $rounds
|
|
|
fd2893 |
+ b Loop_xts_enc
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+.align 5
|
|
|
fd2893 |
+Loop_xts_enc:
|
|
|
fd2893 |
+ ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
|
|
|
fd2893 |
+ vcipher $inout,$inout,$rndkey1
|
|
|
fd2893 |
+ lvx $rndkey1,$idx,$key1
|
|
|
fd2893 |
+ addi $idx,$idx,16
|
|
|
fd2893 |
+ ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
|
|
|
fd2893 |
+ vcipher $inout,$inout,$rndkey0
|
|
|
fd2893 |
+ lvx $rndkey0,$idx,$key1
|
|
|
fd2893 |
+ addi $idx,$idx,16
|
|
|
fd2893 |
+ bdnz Loop_xts_enc
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
|
|
|
fd2893 |
+ vcipher $inout,$inout,$rndkey1
|
|
|
fd2893 |
+ lvx $rndkey1,$idx,$key1
|
|
|
fd2893 |
+ li $idx,16
|
|
|
fd2893 |
+ ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
|
|
|
fd2893 |
+ vxor $rndkey0,$rndkey0,$tweak
|
|
|
fd2893 |
+ vcipherlast $output,$inout,$rndkey0
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ le?vperm $tmp,$output,$output,$leperm
|
|
|
fd2893 |
+ be?nop
|
|
|
fd2893 |
+ le?stvx_u $tmp,0,$out
|
|
|
fd2893 |
+ be?stvx_u $output,0,$out
|
|
|
fd2893 |
+ addi $out,$out,16
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ subic. $len,$len,16
|
|
|
fd2893 |
+ beq Lxts_enc_done
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vmr $inout,$inptail
|
|
|
fd2893 |
+ lvx $inptail,0,$inp
|
|
|
fd2893 |
+ addi $inp,$inp,16
|
|
|
fd2893 |
+ lvx $rndkey0,0,$key1
|
|
|
fd2893 |
+ lvx $rndkey1,$idx,$key1
|
|
|
fd2893 |
+ addi $idx,$idx,16
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ subic r0,$len,32
|
|
|
fd2893 |
+ subfe r0,r0,r0
|
|
|
fd2893 |
+ and r0,r0,$taillen
|
|
|
fd2893 |
+ add $inp,$inp,r0
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vsrab $tmp,$tweak,$seven # next tweak value
|
|
|
fd2893 |
+ vaddubm $tweak,$tweak,$tweak
|
|
|
fd2893 |
+ vsldoi $tmp,$tmp,$tmp,15
|
|
|
fd2893 |
+ vand $tmp,$tmp,$eighty7
|
|
|
fd2893 |
+ vxor $tweak,$tweak,$tmp
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vperm $inout,$inout,$inptail,$inpperm
|
|
|
fd2893 |
+ ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
|
|
|
fd2893 |
+ vxor $inout,$inout,$tweak
|
|
|
fd2893 |
+ vxor $output,$output,$rndkey0 # just in case $len<16
|
|
|
fd2893 |
+ vxor $inout,$inout,$rndkey0
|
|
|
fd2893 |
+ lvx $rndkey0,$idx,$key1
|
|
|
fd2893 |
+ addi $idx,$idx,16
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ mtctr $rounds
|
|
|
fd2893 |
+ ${UCMP}i $len,16
|
|
|
fd2893 |
+ bge Loop_xts_enc
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vxor $output,$output,$tweak
|
|
|
fd2893 |
+ lvsr $inpperm,0,$len # $inpperm is no longer needed
|
|
|
fd2893 |
+ vxor $inptail,$inptail,$inptail # $inptail is no longer needed
|
|
|
fd2893 |
+ vspltisb $tmp,-1
|
|
|
fd2893 |
+ vperm $inptail,$inptail,$tmp,$inpperm
|
|
|
fd2893 |
+ vsel $inout,$inout,$output,$inptail
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ subi r11,$out,17
|
|
|
fd2893 |
+ subi $out,$out,16
|
|
|
fd2893 |
+ mtctr $len
|
|
|
fd2893 |
+ li $len,16
|
|
|
fd2893 |
+Loop_xts_enc_steal:
|
|
|
fd2893 |
+ lbzu r0,1(r11)
|
|
|
fd2893 |
+ stb r0,16(r11)
|
|
|
fd2893 |
+ bdnz Loop_xts_enc_steal
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ mtctr $rounds
|
|
|
fd2893 |
+ b Loop_xts_enc # one more time...
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+Lxts_enc_done:
|
|
|
fd2893 |
+ ${UCMP}i $ivp,0
|
|
|
fd2893 |
+ beq Lxts_enc_ret
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vsrab $tmp,$tweak,$seven # next tweak value
|
|
|
fd2893 |
+ vaddubm $tweak,$tweak,$tweak
|
|
|
fd2893 |
+ vsldoi $tmp,$tmp,$tmp,15
|
|
|
fd2893 |
+ vand $tmp,$tmp,$eighty7
|
|
|
fd2893 |
+ vxor $tweak,$tweak,$tmp
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ le?vperm $tweak,$tweak,$tweak,$leperm
|
|
|
fd2893 |
+ stvx_u $tweak,0,$ivp
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+Lxts_enc_ret:
|
|
|
fd2893 |
+ mtspr 256,r12 # restore vrsave
|
|
|
fd2893 |
+ li r3,0
|
|
|
fd2893 |
+ blr
|
|
|
fd2893 |
+ .long 0
|
|
|
fd2893 |
+ .byte 0,12,0x04,0,0x80,6,6,0
|
|
|
fd2893 |
+ .long 0
|
|
|
fd2893 |
+.size .${prefix}_xts_encrypt,.-.${prefix}_xts_encrypt
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+.globl .${prefix}_xts_decrypt
|
|
|
fd2893 |
+.align 5
|
|
|
fd2893 |
+.${prefix}_xts_decrypt:
|
|
|
fd2893 |
+ mr $inp,r3 # reassign
|
|
|
fd2893 |
+ li r3,-1
|
|
|
fd2893 |
+ ${UCMP}i $len,16
|
|
|
fd2893 |
+ bltlr-
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ lis r0,0xfff8
|
|
|
fd2893 |
+ mfspr r12,256 # save vrsave
|
|
|
fd2893 |
+ li r11,0
|
|
|
fd2893 |
+ mtspr 256,r0
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ andi. r0,$len,15
|
|
|
fd2893 |
+ neg r0,r0
|
|
|
fd2893 |
+ andi. r0,r0,16
|
|
|
fd2893 |
+ sub $len,$len,r0
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vspltisb $seven,0x07 # 0x070707..07
|
|
|
fd2893 |
+ le?lvsl $leperm,r11,r11
|
|
|
fd2893 |
+ le?vspltisb $tmp,0x0f
|
|
|
fd2893 |
+ le?vxor $leperm,$leperm,$seven
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ li $idx,15
|
|
|
fd2893 |
+ lvx $tweak,0,$ivp # load [unaligned] iv
|
|
|
fd2893 |
+ lvsl $inpperm,0,$ivp
|
|
|
fd2893 |
+ lvx $inptail,$idx,$ivp
|
|
|
fd2893 |
+ le?vxor $inpperm,$inpperm,$tmp
|
|
|
fd2893 |
+ vperm $tweak,$tweak,$inptail,$inpperm
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ neg r11,$inp
|
|
|
fd2893 |
+ lvsr $inpperm,0,r11 # prepare for unaligned load
|
|
|
fd2893 |
+ lvx $inout,0,$inp
|
|
|
fd2893 |
+ addi $inp,$inp,15 # 15 is not typo
|
|
|
fd2893 |
+ le?vxor $inpperm,$inpperm,$tmp
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ ${UCMP}i $key2,0 # key2==NULL?
|
|
|
fd2893 |
+ beq Lxts_dec_no_key2
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ ?lvsl $keyperm,0,$key2 # prepare for unaligned key
|
|
|
fd2893 |
+ lwz $rounds,240($key2)
|
|
|
fd2893 |
+ srwi $rounds,$rounds,1
|
|
|
fd2893 |
+ subi $rounds,$rounds,1
|
|
|
fd2893 |
+ li $idx,16
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ lvx $rndkey0,0,$key2
|
|
|
fd2893 |
+ lvx $rndkey1,$idx,$key2
|
|
|
fd2893 |
+ addi $idx,$idx,16
|
|
|
fd2893 |
+ ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
|
|
|
fd2893 |
+ vxor $tweak,$tweak,$rndkey0
|
|
|
fd2893 |
+ lvx $rndkey0,$idx,$key2
|
|
|
fd2893 |
+ addi $idx,$idx,16
|
|
|
fd2893 |
+ mtctr $rounds
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+Ltweak_xts_dec:
|
|
|
fd2893 |
+ ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
|
|
|
fd2893 |
+ vcipher $tweak,$tweak,$rndkey1
|
|
|
fd2893 |
+ lvx $rndkey1,$idx,$key2
|
|
|
fd2893 |
+ addi $idx,$idx,16
|
|
|
fd2893 |
+ ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
|
|
|
fd2893 |
+ vcipher $tweak,$tweak,$rndkey0
|
|
|
fd2893 |
+ lvx $rndkey0,$idx,$key2
|
|
|
fd2893 |
+ addi $idx,$idx,16
|
|
|
fd2893 |
+ bdnz Ltweak_xts_dec
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
|
|
|
fd2893 |
+ vcipher $tweak,$tweak,$rndkey1
|
|
|
fd2893 |
+ lvx $rndkey1,$idx,$key2
|
|
|
fd2893 |
+ ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
|
|
|
fd2893 |
+ vcipherlast $tweak,$tweak,$rndkey0
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ li $ivp,0 # don't chain the tweak
|
|
|
fd2893 |
+ b Lxts_dec
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+Lxts_dec_no_key2:
|
|
|
fd2893 |
+ neg $idx,$len
|
|
|
fd2893 |
+ andi. $idx,$idx,15
|
|
|
fd2893 |
+ add $len,$len,$idx # in "tweak chaining"
|
|
|
fd2893 |
+ # mode only complete
|
|
|
fd2893 |
+ # blocks are processed
|
|
|
fd2893 |
+Lxts_dec:
|
|
|
fd2893 |
+ lvx $inptail,0,$inp
|
|
|
fd2893 |
+ addi $inp,$inp,16
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ ?lvsl $keyperm,0,$key1 # prepare for unaligned key
|
|
|
fd2893 |
+ lwz $rounds,240($key1)
|
|
|
fd2893 |
+ srwi $rounds,$rounds,1
|
|
|
fd2893 |
+ subi $rounds,$rounds,1
|
|
|
fd2893 |
+ li $idx,16
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vslb $eighty7,$seven,$seven # 0x808080..80
|
|
|
fd2893 |
+ vor $eighty7,$eighty7,$seven # 0x878787..87
|
|
|
fd2893 |
+ vspltisb $tmp,1 # 0x010101..01
|
|
|
fd2893 |
+ vsldoi $eighty7,$eighty7,$tmp,15 # 0x870101..01
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ ${UCMP}i $len,96
|
|
|
fd2893 |
+ bge _aesp8_xts_decrypt6x
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ lvx $rndkey0,0,$key1
|
|
|
fd2893 |
+ lvx $rndkey1,$idx,$key1
|
|
|
fd2893 |
+ addi $idx,$idx,16
|
|
|
fd2893 |
+ vperm $inout,$inout,$inptail,$inpperm
|
|
|
fd2893 |
+ ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
|
|
|
fd2893 |
+ vxor $inout,$inout,$tweak
|
|
|
fd2893 |
+ vxor $inout,$inout,$rndkey0
|
|
|
fd2893 |
+ lvx $rndkey0,$idx,$key1
|
|
|
fd2893 |
+ addi $idx,$idx,16
|
|
|
fd2893 |
+ mtctr $rounds
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ ${UCMP}i $len,16
|
|
|
fd2893 |
+ blt Ltail_xts_dec
|
|
|
fd2893 |
+ be?b Loop_xts_dec
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+.align 5
|
|
|
fd2893 |
+Loop_xts_dec:
|
|
|
fd2893 |
+ ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
|
|
|
fd2893 |
+ vncipher $inout,$inout,$rndkey1
|
|
|
fd2893 |
+ lvx $rndkey1,$idx,$key1
|
|
|
fd2893 |
+ addi $idx,$idx,16
|
|
|
fd2893 |
+ ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
|
|
|
fd2893 |
+ vncipher $inout,$inout,$rndkey0
|
|
|
fd2893 |
+ lvx $rndkey0,$idx,$key1
|
|
|
fd2893 |
+ addi $idx,$idx,16
|
|
|
fd2893 |
+ bdnz Loop_xts_dec
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
|
|
|
fd2893 |
+ vncipher $inout,$inout,$rndkey1
|
|
|
fd2893 |
+ lvx $rndkey1,$idx,$key1
|
|
|
fd2893 |
+ li $idx,16
|
|
|
fd2893 |
+ ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
|
|
|
fd2893 |
+ vxor $rndkey0,$rndkey0,$tweak
|
|
|
fd2893 |
+ vncipherlast $output,$inout,$rndkey0
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ le?vperm $tmp,$output,$output,$leperm
|
|
|
fd2893 |
+ be?nop
|
|
|
fd2893 |
+ le?stvx_u $tmp,0,$out
|
|
|
fd2893 |
+ be?stvx_u $output,0,$out
|
|
|
fd2893 |
+ addi $out,$out,16
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ subic. $len,$len,16
|
|
|
fd2893 |
+ beq Lxts_dec_done
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vmr $inout,$inptail
|
|
|
fd2893 |
+ lvx $inptail,0,$inp
|
|
|
fd2893 |
+ addi $inp,$inp,16
|
|
|
fd2893 |
+ lvx $rndkey0,0,$key1
|
|
|
fd2893 |
+ lvx $rndkey1,$idx,$key1
|
|
|
fd2893 |
+ addi $idx,$idx,16
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vsrab $tmp,$tweak,$seven # next tweak value
|
|
|
fd2893 |
+ vaddubm $tweak,$tweak,$tweak
|
|
|
fd2893 |
+ vsldoi $tmp,$tmp,$tmp,15
|
|
|
fd2893 |
+ vand $tmp,$tmp,$eighty7
|
|
|
fd2893 |
+ vxor $tweak,$tweak,$tmp
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vperm $inout,$inout,$inptail,$inpperm
|
|
|
fd2893 |
+ ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
|
|
|
fd2893 |
+ vxor $inout,$inout,$tweak
|
|
|
fd2893 |
+ vxor $inout,$inout,$rndkey0
|
|
|
fd2893 |
+ lvx $rndkey0,$idx,$key1
|
|
|
fd2893 |
+ addi $idx,$idx,16
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ mtctr $rounds
|
|
|
fd2893 |
+ ${UCMP}i $len,16
|
|
|
fd2893 |
+ bge Loop_xts_dec
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+Ltail_xts_dec:
|
|
|
fd2893 |
+ vsrab $tmp,$tweak,$seven # next tweak value
|
|
|
fd2893 |
+ vaddubm $tweak1,$tweak,$tweak
|
|
|
fd2893 |
+ vsldoi $tmp,$tmp,$tmp,15
|
|
|
fd2893 |
+ vand $tmp,$tmp,$eighty7
|
|
|
fd2893 |
+ vxor $tweak1,$tweak1,$tmp
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ subi $inp,$inp,16
|
|
|
fd2893 |
+ add $inp,$inp,$len
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vxor $inout,$inout,$tweak # :-(
|
|
|
fd2893 |
+ vxor $inout,$inout,$tweak1 # :-)
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+Loop_xts_dec_short:
|
|
|
fd2893 |
+ ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
|
|
|
fd2893 |
+ vncipher $inout,$inout,$rndkey1
|
|
|
fd2893 |
+ lvx $rndkey1,$idx,$key1
|
|
|
fd2893 |
+ addi $idx,$idx,16
|
|
|
fd2893 |
+ ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
|
|
|
fd2893 |
+ vncipher $inout,$inout,$rndkey0
|
|
|
fd2893 |
+ lvx $rndkey0,$idx,$key1
|
|
|
fd2893 |
+ addi $idx,$idx,16
|
|
|
fd2893 |
+ bdnz Loop_xts_dec_short
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
|
|
|
fd2893 |
+ vncipher $inout,$inout,$rndkey1
|
|
|
fd2893 |
+ lvx $rndkey1,$idx,$key1
|
|
|
fd2893 |
+ li $idx,16
|
|
|
fd2893 |
+ ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
|
|
|
fd2893 |
+ vxor $rndkey0,$rndkey0,$tweak1
|
|
|
fd2893 |
+ vncipherlast $output,$inout,$rndkey0
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ le?vperm $tmp,$output,$output,$leperm
|
|
|
fd2893 |
+ be?nop
|
|
|
fd2893 |
+ le?stvx_u $tmp,0,$out
|
|
|
fd2893 |
+ be?stvx_u $output,0,$out
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vmr $inout,$inptail
|
|
|
fd2893 |
+ lvx $inptail,0,$inp
|
|
|
fd2893 |
+ #addi $inp,$inp,16
|
|
|
fd2893 |
+ lvx $rndkey0,0,$key1
|
|
|
fd2893 |
+ lvx $rndkey1,$idx,$key1
|
|
|
fd2893 |
+ addi $idx,$idx,16
|
|
|
fd2893 |
+ vperm $inout,$inout,$inptail,$inpperm
|
|
|
fd2893 |
+ ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ lvsr $inpperm,0,$len # $inpperm is no longer needed
|
|
|
fd2893 |
+ vxor $inptail,$inptail,$inptail # $inptail is no longer needed
|
|
|
fd2893 |
+ vspltisb $tmp,-1
|
|
|
fd2893 |
+ vperm $inptail,$inptail,$tmp,$inpperm
|
|
|
fd2893 |
+ vsel $inout,$inout,$output,$inptail
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vxor $rndkey0,$rndkey0,$tweak
|
|
|
fd2893 |
+ vxor $inout,$inout,$rndkey0
|
|
|
fd2893 |
+ lvx $rndkey0,$idx,$key1
|
|
|
fd2893 |
+ addi $idx,$idx,16
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ subi r11,$out,1
|
|
|
fd2893 |
+ mtctr $len
|
|
|
fd2893 |
+ li $len,16
|
|
|
fd2893 |
+Loop_xts_dec_steal:
|
|
|
fd2893 |
+ lbzu r0,1(r11)
|
|
|
fd2893 |
+ stb r0,16(r11)
|
|
|
fd2893 |
+ bdnz Loop_xts_dec_steal
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ mtctr $rounds
|
|
|
fd2893 |
+ b Loop_xts_dec # one more time...
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+Lxts_dec_done:
|
|
|
fd2893 |
+ ${UCMP}i $ivp,0
|
|
|
fd2893 |
+ beq Lxts_dec_ret
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vsrab $tmp,$tweak,$seven # next tweak value
|
|
|
fd2893 |
+ vaddubm $tweak,$tweak,$tweak
|
|
|
fd2893 |
+ vsldoi $tmp,$tmp,$tmp,15
|
|
|
fd2893 |
+ vand $tmp,$tmp,$eighty7
|
|
|
fd2893 |
+ vxor $tweak,$tweak,$tmp
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ le?vperm $tweak,$tweak,$tweak,$leperm
|
|
|
fd2893 |
+ stvx_u $tweak,0,$ivp
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+Lxts_dec_ret:
|
|
|
fd2893 |
+ mtspr 256,r12 # restore vrsave
|
|
|
fd2893 |
+ li r3,0
|
|
|
fd2893 |
+ blr
|
|
|
fd2893 |
+ .long 0
|
|
|
fd2893 |
+ .byte 0,12,0x04,0,0x80,6,6,0
|
|
|
fd2893 |
+ .long 0
|
|
|
fd2893 |
+.size .${prefix}_xts_decrypt,.-.${prefix}_xts_decrypt
|
|
|
fd2893 |
+___
|
|
|
fd2893 |
+#########################################################################
|
|
|
fd2893 |
+{{ # Optimized XTS procedures #
|
|
|
fd2893 |
+my $key_=$key2;
|
|
|
fd2893 |
+my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,3,26..31));
|
|
|
fd2893 |
+ $x00=0 if ($flavour =~ /osx/);
|
|
|
fd2893 |
+my ($in0, $in1, $in2, $in3, $in4, $in5 )=map("v$_",(0..5));
|
|
|
fd2893 |
+my ($out0, $out1, $out2, $out3, $out4, $out5)=map("v$_",(7,12..16));
|
|
|
fd2893 |
+my ($twk0, $twk1, $twk2, $twk3, $twk4, $twk5)=map("v$_",(17..22));
|
|
|
fd2893 |
+my $rndkey0="v23"; # v24-v25 rotating buffer for first found keys
|
|
|
fd2893 |
+ # v26-v31 last 6 round keys
|
|
|
fd2893 |
+my ($keyperm)=($out0); # aliases with "caller", redundant assignment
|
|
|
fd2893 |
+my $taillen=$x70;
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+$code.=<<___;
|
|
|
fd2893 |
+.align 5
|
|
|
fd2893 |
+_aesp8_xts_encrypt6x:
|
|
|
fd2893 |
+ $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
|
|
|
fd2893 |
+ mflr r11
|
|
|
fd2893 |
+ li r7,`$FRAME+8*16+15`
|
|
|
fd2893 |
+ li r3,`$FRAME+8*16+31`
|
|
|
fd2893 |
+ $PUSH r11,`$FRAME+21*16+6*$SIZE_T+$LRSAVE`($sp)
|
|
|
fd2893 |
+ stvx v20,r7,$sp # ABI says so
|
|
|
fd2893 |
+ addi r7,r7,32
|
|
|
fd2893 |
+ stvx v21,r3,$sp
|
|
|
fd2893 |
+ addi r3,r3,32
|
|
|
fd2893 |
+ stvx v22,r7,$sp
|
|
|
fd2893 |
+ addi r7,r7,32
|
|
|
fd2893 |
+ stvx v23,r3,$sp
|
|
|
fd2893 |
+ addi r3,r3,32
|
|
|
fd2893 |
+ stvx v24,r7,$sp
|
|
|
fd2893 |
+ addi r7,r7,32
|
|
|
fd2893 |
+ stvx v25,r3,$sp
|
|
|
fd2893 |
+ addi r3,r3,32
|
|
|
fd2893 |
+ stvx v26,r7,$sp
|
|
|
fd2893 |
+ addi r7,r7,32
|
|
|
fd2893 |
+ stvx v27,r3,$sp
|
|
|
fd2893 |
+ addi r3,r3,32
|
|
|
fd2893 |
+ stvx v28,r7,$sp
|
|
|
fd2893 |
+ addi r7,r7,32
|
|
|
fd2893 |
+ stvx v29,r3,$sp
|
|
|
fd2893 |
+ addi r3,r3,32
|
|
|
fd2893 |
+ stvx v30,r7,$sp
|
|
|
fd2893 |
+ stvx v31,r3,$sp
|
|
|
fd2893 |
+ li r0,-1
|
|
|
fd2893 |
+ stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave
|
|
|
fd2893 |
+ li $x10,0x10
|
|
|
fd2893 |
+ $PUSH r26,`$FRAME+21*16+0*$SIZE_T`($sp)
|
|
|
fd2893 |
+ li $x20,0x20
|
|
|
fd2893 |
+ $PUSH r27,`$FRAME+21*16+1*$SIZE_T`($sp)
|
|
|
fd2893 |
+ li $x30,0x30
|
|
|
fd2893 |
+ $PUSH r28,`$FRAME+21*16+2*$SIZE_T`($sp)
|
|
|
fd2893 |
+ li $x40,0x40
|
|
|
fd2893 |
+ $PUSH r29,`$FRAME+21*16+3*$SIZE_T`($sp)
|
|
|
fd2893 |
+ li $x50,0x50
|
|
|
fd2893 |
+ $PUSH r30,`$FRAME+21*16+4*$SIZE_T`($sp)
|
|
|
fd2893 |
+ li $x60,0x60
|
|
|
fd2893 |
+ $PUSH r31,`$FRAME+21*16+5*$SIZE_T`($sp)
|
|
|
fd2893 |
+ li $x70,0x70
|
|
|
fd2893 |
+ mtspr 256,r0
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ subi $rounds,$rounds,3 # -4 in total
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ lvx $rndkey0,$x00,$key1 # load key schedule
|
|
|
fd2893 |
+ lvx v30,$x10,$key1
|
|
|
fd2893 |
+ addi $key1,$key1,0x20
|
|
|
fd2893 |
+ lvx v31,$x00,$key1
|
|
|
fd2893 |
+ ?vperm $rndkey0,$rndkey0,v30,$keyperm
|
|
|
fd2893 |
+ addi $key_,$sp,$FRAME+15
|
|
|
fd2893 |
+ mtctr $rounds
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+Load_xts_enc_key:
|
|
|
fd2893 |
+ ?vperm v24,v30,v31,$keyperm
|
|
|
fd2893 |
+ lvx v30,$x10,$key1
|
|
|
fd2893 |
+ addi $key1,$key1,0x20
|
|
|
fd2893 |
+ stvx v24,$x00,$key_ # off-load round[1]
|
|
|
fd2893 |
+ ?vperm v25,v31,v30,$keyperm
|
|
|
fd2893 |
+ lvx v31,$x00,$key1
|
|
|
fd2893 |
+ stvx v25,$x10,$key_ # off-load round[2]
|
|
|
fd2893 |
+ addi $key_,$key_,0x20
|
|
|
fd2893 |
+ bdnz Load_xts_enc_key
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ lvx v26,$x10,$key1
|
|
|
fd2893 |
+ ?vperm v24,v30,v31,$keyperm
|
|
|
fd2893 |
+ lvx v27,$x20,$key1
|
|
|
fd2893 |
+ stvx v24,$x00,$key_ # off-load round[3]
|
|
|
fd2893 |
+ ?vperm v25,v31,v26,$keyperm
|
|
|
fd2893 |
+ lvx v28,$x30,$key1
|
|
|
fd2893 |
+ stvx v25,$x10,$key_ # off-load round[4]
|
|
|
fd2893 |
+ addi $key_,$sp,$FRAME+15 # rewind $key_
|
|
|
fd2893 |
+ ?vperm v26,v26,v27,$keyperm
|
|
|
fd2893 |
+ lvx v29,$x40,$key1
|
|
|
fd2893 |
+ ?vperm v27,v27,v28,$keyperm
|
|
|
fd2893 |
+ lvx v30,$x50,$key1
|
|
|
fd2893 |
+ ?vperm v28,v28,v29,$keyperm
|
|
|
fd2893 |
+ lvx v31,$x60,$key1
|
|
|
fd2893 |
+ ?vperm v29,v29,v30,$keyperm
|
|
|
fd2893 |
+ lvx $twk5,$x70,$key1 # borrow $twk5
|
|
|
fd2893 |
+ ?vperm v30,v30,v31,$keyperm
|
|
|
fd2893 |
+ lvx v24,$x00,$key_ # pre-load round[1]
|
|
|
fd2893 |
+ ?vperm v31,v31,$twk5,$keyperm
|
|
|
fd2893 |
+ lvx v25,$x10,$key_ # pre-load round[2]
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vperm $in0,$inout,$inptail,$inpperm
|
|
|
fd2893 |
+ subi $inp,$inp,31 # undo "caller"
|
|
|
fd2893 |
+ vxor $twk0,$tweak,$rndkey0
|
|
|
fd2893 |
+ vsrab $tmp,$tweak,$seven # next tweak value
|
|
|
fd2893 |
+ vaddubm $tweak,$tweak,$tweak
|
|
|
fd2893 |
+ vsldoi $tmp,$tmp,$tmp,15
|
|
|
fd2893 |
+ vand $tmp,$tmp,$eighty7
|
|
|
fd2893 |
+ vxor $out0,$in0,$twk0
|
|
|
fd2893 |
+ vxor $tweak,$tweak,$tmp
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ lvx_u $in1,$x10,$inp
|
|
|
fd2893 |
+ vxor $twk1,$tweak,$rndkey0
|
|
|
fd2893 |
+ vsrab $tmp,$tweak,$seven # next tweak value
|
|
|
fd2893 |
+ vaddubm $tweak,$tweak,$tweak
|
|
|
fd2893 |
+ vsldoi $tmp,$tmp,$tmp,15
|
|
|
fd2893 |
+ le?vperm $in1,$in1,$in1,$leperm
|
|
|
fd2893 |
+ vand $tmp,$tmp,$eighty7
|
|
|
fd2893 |
+ vxor $out1,$in1,$twk1
|
|
|
fd2893 |
+ vxor $tweak,$tweak,$tmp
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ lvx_u $in2,$x20,$inp
|
|
|
fd2893 |
+ andi. $taillen,$len,15
|
|
|
fd2893 |
+ vxor $twk2,$tweak,$rndkey0
|
|
|
fd2893 |
+ vsrab $tmp,$tweak,$seven # next tweak value
|
|
|
fd2893 |
+ vaddubm $tweak,$tweak,$tweak
|
|
|
fd2893 |
+ vsldoi $tmp,$tmp,$tmp,15
|
|
|
fd2893 |
+ le?vperm $in2,$in2,$in2,$leperm
|
|
|
fd2893 |
+ vand $tmp,$tmp,$eighty7
|
|
|
fd2893 |
+ vxor $out2,$in2,$twk2
|
|
|
fd2893 |
+ vxor $tweak,$tweak,$tmp
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ lvx_u $in3,$x30,$inp
|
|
|
fd2893 |
+ sub $len,$len,$taillen
|
|
|
fd2893 |
+ vxor $twk3,$tweak,$rndkey0
|
|
|
fd2893 |
+ vsrab $tmp,$tweak,$seven # next tweak value
|
|
|
fd2893 |
+ vaddubm $tweak,$tweak,$tweak
|
|
|
fd2893 |
+ vsldoi $tmp,$tmp,$tmp,15
|
|
|
fd2893 |
+ le?vperm $in3,$in3,$in3,$leperm
|
|
|
fd2893 |
+ vand $tmp,$tmp,$eighty7
|
|
|
fd2893 |
+ vxor $out3,$in3,$twk3
|
|
|
fd2893 |
+ vxor $tweak,$tweak,$tmp
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ lvx_u $in4,$x40,$inp
|
|
|
fd2893 |
+ subi $len,$len,0x60
|
|
|
fd2893 |
+ vxor $twk4,$tweak,$rndkey0
|
|
|
fd2893 |
+ vsrab $tmp,$tweak,$seven # next tweak value
|
|
|
fd2893 |
+ vaddubm $tweak,$tweak,$tweak
|
|
|
fd2893 |
+ vsldoi $tmp,$tmp,$tmp,15
|
|
|
fd2893 |
+ le?vperm $in4,$in4,$in4,$leperm
|
|
|
fd2893 |
+ vand $tmp,$tmp,$eighty7
|
|
|
fd2893 |
+ vxor $out4,$in4,$twk4
|
|
|
fd2893 |
+ vxor $tweak,$tweak,$tmp
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ lvx_u $in5,$x50,$inp
|
|
|
fd2893 |
+ addi $inp,$inp,0x60
|
|
|
fd2893 |
+ vxor $twk5,$tweak,$rndkey0
|
|
|
fd2893 |
+ vsrab $tmp,$tweak,$seven # next tweak value
|
|
|
fd2893 |
+ vaddubm $tweak,$tweak,$tweak
|
|
|
fd2893 |
+ vsldoi $tmp,$tmp,$tmp,15
|
|
|
fd2893 |
+ le?vperm $in5,$in5,$in5,$leperm
|
|
|
fd2893 |
+ vand $tmp,$tmp,$eighty7
|
|
|
fd2893 |
+ vxor $out5,$in5,$twk5
|
|
|
fd2893 |
+ vxor $tweak,$tweak,$tmp
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vxor v31,v31,$rndkey0
|
|
|
fd2893 |
+ mtctr $rounds
|
|
|
fd2893 |
+ b Loop_xts_enc6x
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+.align 5
|
|
|
fd2893 |
+Loop_xts_enc6x:
|
|
|
fd2893 |
+ vcipher $out0,$out0,v24
|
|
|
fd2893 |
+ vcipher $out1,$out1,v24
|
|
|
fd2893 |
+ vcipher $out2,$out2,v24
|
|
|
fd2893 |
+ vcipher $out3,$out3,v24
|
|
|
fd2893 |
+ vcipher $out4,$out4,v24
|
|
|
fd2893 |
+ vcipher $out5,$out5,v24
|
|
|
fd2893 |
+ lvx v24,$x20,$key_ # round[3]
|
|
|
fd2893 |
+ addi $key_,$key_,0x20
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vcipher $out0,$out0,v25
|
|
|
fd2893 |
+ vcipher $out1,$out1,v25
|
|
|
fd2893 |
+ vcipher $out2,$out2,v25
|
|
|
fd2893 |
+ vcipher $out3,$out3,v25
|
|
|
fd2893 |
+ vcipher $out4,$out4,v25
|
|
|
fd2893 |
+ vcipher $out5,$out5,v25
|
|
|
fd2893 |
+ lvx v25,$x10,$key_ # round[4]
|
|
|
fd2893 |
+ bdnz Loop_xts_enc6x
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ subic $len,$len,96 # $len-=96
|
|
|
fd2893 |
+ vxor $in0,$twk0,v31 # xor with last round key
|
|
|
fd2893 |
+ vcipher $out0,$out0,v24
|
|
|
fd2893 |
+ vcipher $out1,$out1,v24
|
|
|
fd2893 |
+ vsrab $tmp,$tweak,$seven # next tweak value
|
|
|
fd2893 |
+ vxor $twk0,$tweak,$rndkey0
|
|
|
fd2893 |
+ vaddubm $tweak,$tweak,$tweak
|
|
|
fd2893 |
+ vcipher $out2,$out2,v24
|
|
|
fd2893 |
+ vcipher $out3,$out3,v24
|
|
|
fd2893 |
+ vsldoi $tmp,$tmp,$tmp,15
|
|
|
fd2893 |
+ vcipher $out4,$out4,v24
|
|
|
fd2893 |
+ vcipher $out5,$out5,v24
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ subfe. r0,r0,r0 # borrow?-1:0
|
|
|
fd2893 |
+ vand $tmp,$tmp,$eighty7
|
|
|
fd2893 |
+ vcipher $out0,$out0,v25
|
|
|
fd2893 |
+ vcipher $out1,$out1,v25
|
|
|
fd2893 |
+ vxor $tweak,$tweak,$tmp
|
|
|
fd2893 |
+ vcipher $out2,$out2,v25
|
|
|
fd2893 |
+ vcipher $out3,$out3,v25
|
|
|
fd2893 |
+ vxor $in1,$twk1,v31
|
|
|
fd2893 |
+ vsrab $tmp,$tweak,$seven # next tweak value
|
|
|
fd2893 |
+ vxor $twk1,$tweak,$rndkey0
|
|
|
fd2893 |
+ vcipher $out4,$out4,v25
|
|
|
fd2893 |
+ vcipher $out5,$out5,v25
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ and r0,r0,$len
|
|
|
fd2893 |
+ vaddubm $tweak,$tweak,$tweak
|
|
|
fd2893 |
+ vsldoi $tmp,$tmp,$tmp,15
|
|
|
fd2893 |
+ vcipher $out0,$out0,v26
|
|
|
fd2893 |
+ vcipher $out1,$out1,v26
|
|
|
fd2893 |
+ vand $tmp,$tmp,$eighty7
|
|
|
fd2893 |
+ vcipher $out2,$out2,v26
|
|
|
fd2893 |
+ vcipher $out3,$out3,v26
|
|
|
fd2893 |
+ vxor $tweak,$tweak,$tmp
|
|
|
fd2893 |
+ vcipher $out4,$out4,v26
|
|
|
fd2893 |
+ vcipher $out5,$out5,v26
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ add $inp,$inp,r0 # $inp is adjusted in such
|
|
|
fd2893 |
+ # way that at exit from the
|
|
|
fd2893 |
+ # loop inX-in5 are loaded
|
|
|
fd2893 |
+ # with last "words"
|
|
|
fd2893 |
+ vxor $in2,$twk2,v31
|
|
|
fd2893 |
+ vsrab $tmp,$tweak,$seven # next tweak value
|
|
|
fd2893 |
+ vxor $twk2,$tweak,$rndkey0
|
|
|
fd2893 |
+ vaddubm $tweak,$tweak,$tweak
|
|
|
fd2893 |
+ vcipher $out0,$out0,v27
|
|
|
fd2893 |
+ vcipher $out1,$out1,v27
|
|
|
fd2893 |
+ vsldoi $tmp,$tmp,$tmp,15
|
|
|
fd2893 |
+ vcipher $out2,$out2,v27
|
|
|
fd2893 |
+ vcipher $out3,$out3,v27
|
|
|
fd2893 |
+ vand $tmp,$tmp,$eighty7
|
|
|
fd2893 |
+ vcipher $out4,$out4,v27
|
|
|
fd2893 |
+ vcipher $out5,$out5,v27
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ addi $key_,$sp,$FRAME+15 # rewind $key_
|
|
|
fd2893 |
+ vxor $tweak,$tweak,$tmp
|
|
|
fd2893 |
+ vcipher $out0,$out0,v28
|
|
|
fd2893 |
+ vcipher $out1,$out1,v28
|
|
|
fd2893 |
+ vxor $in3,$twk3,v31
|
|
|
fd2893 |
+ vsrab $tmp,$tweak,$seven # next tweak value
|
|
|
fd2893 |
+ vxor $twk3,$tweak,$rndkey0
|
|
|
fd2893 |
+ vcipher $out2,$out2,v28
|
|
|
fd2893 |
+ vcipher $out3,$out3,v28
|
|
|
fd2893 |
+ vaddubm $tweak,$tweak,$tweak
|
|
|
fd2893 |
+ vsldoi $tmp,$tmp,$tmp,15
|
|
|
fd2893 |
+ vcipher $out4,$out4,v28
|
|
|
fd2893 |
+ vcipher $out5,$out5,v28
|
|
|
fd2893 |
+ lvx v24,$x00,$key_ # re-pre-load round[1]
|
|
|
fd2893 |
+ vand $tmp,$tmp,$eighty7
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vcipher $out0,$out0,v29
|
|
|
fd2893 |
+ vcipher $out1,$out1,v29
|
|
|
fd2893 |
+ vxor $tweak,$tweak,$tmp
|
|
|
fd2893 |
+ vcipher $out2,$out2,v29
|
|
|
fd2893 |
+ vcipher $out3,$out3,v29
|
|
|
fd2893 |
+ vxor $in4,$twk4,v31
|
|
|
fd2893 |
+ vsrab $tmp,$tweak,$seven # next tweak value
|
|
|
fd2893 |
+ vxor $twk4,$tweak,$rndkey0
|
|
|
fd2893 |
+ vcipher $out4,$out4,v29
|
|
|
fd2893 |
+ vcipher $out5,$out5,v29
|
|
|
fd2893 |
+ lvx v25,$x10,$key_ # re-pre-load round[2]
|
|
|
fd2893 |
+ vaddubm $tweak,$tweak,$tweak
|
|
|
fd2893 |
+ vsldoi $tmp,$tmp,$tmp,15
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vcipher $out0,$out0,v30
|
|
|
fd2893 |
+ vcipher $out1,$out1,v30
|
|
|
fd2893 |
+ vand $tmp,$tmp,$eighty7
|
|
|
fd2893 |
+ vcipher $out2,$out2,v30
|
|
|
fd2893 |
+ vcipher $out3,$out3,v30
|
|
|
fd2893 |
+ vxor $tweak,$tweak,$tmp
|
|
|
fd2893 |
+ vcipher $out4,$out4,v30
|
|
|
fd2893 |
+ vcipher $out5,$out5,v30
|
|
|
fd2893 |
+ vxor $in5,$twk5,v31
|
|
|
fd2893 |
+ vsrab $tmp,$tweak,$seven # next tweak value
|
|
|
fd2893 |
+ vxor $twk5,$tweak,$rndkey0
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vcipherlast $out0,$out0,$in0
|
|
|
fd2893 |
+ lvx_u $in0,$x00,$inp # load next input block
|
|
|
fd2893 |
+ vaddubm $tweak,$tweak,$tweak
|
|
|
fd2893 |
+ vsldoi $tmp,$tmp,$tmp,15
|
|
|
fd2893 |
+ vcipherlast $out1,$out1,$in1
|
|
|
fd2893 |
+ lvx_u $in1,$x10,$inp
|
|
|
fd2893 |
+ vcipherlast $out2,$out2,$in2
|
|
|
fd2893 |
+ le?vperm $in0,$in0,$in0,$leperm
|
|
|
fd2893 |
+ lvx_u $in2,$x20,$inp
|
|
|
fd2893 |
+ vand $tmp,$tmp,$eighty7
|
|
|
fd2893 |
+ vcipherlast $out3,$out3,$in3
|
|
|
fd2893 |
+ le?vperm $in1,$in1,$in1,$leperm
|
|
|
fd2893 |
+ lvx_u $in3,$x30,$inp
|
|
|
fd2893 |
+ vcipherlast $out4,$out4,$in4
|
|
|
fd2893 |
+ le?vperm $in2,$in2,$in2,$leperm
|
|
|
fd2893 |
+ lvx_u $in4,$x40,$inp
|
|
|
fd2893 |
+ vxor $tweak,$tweak,$tmp
|
|
|
fd2893 |
+ vcipherlast $tmp,$out5,$in5 # last block might be needed
|
|
|
fd2893 |
+ # in stealing mode
|
|
|
fd2893 |
+ le?vperm $in3,$in3,$in3,$leperm
|
|
|
fd2893 |
+ lvx_u $in5,$x50,$inp
|
|
|
fd2893 |
+ addi $inp,$inp,0x60
|
|
|
fd2893 |
+ le?vperm $in4,$in4,$in4,$leperm
|
|
|
fd2893 |
+ le?vperm $in5,$in5,$in5,$leperm
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ le?vperm $out0,$out0,$out0,$leperm
|
|
|
fd2893 |
+ le?vperm $out1,$out1,$out1,$leperm
|
|
|
fd2893 |
+ stvx_u $out0,$x00,$out # store output
|
|
|
fd2893 |
+ vxor $out0,$in0,$twk0
|
|
|
fd2893 |
+ le?vperm $out2,$out2,$out2,$leperm
|
|
|
fd2893 |
+ stvx_u $out1,$x10,$out
|
|
|
fd2893 |
+ vxor $out1,$in1,$twk1
|
|
|
fd2893 |
+ le?vperm $out3,$out3,$out3,$leperm
|
|
|
fd2893 |
+ stvx_u $out2,$x20,$out
|
|
|
fd2893 |
+ vxor $out2,$in2,$twk2
|
|
|
fd2893 |
+ le?vperm $out4,$out4,$out4,$leperm
|
|
|
fd2893 |
+ stvx_u $out3,$x30,$out
|
|
|
fd2893 |
+ vxor $out3,$in3,$twk3
|
|
|
fd2893 |
+ le?vperm $out5,$tmp,$tmp,$leperm
|
|
|
fd2893 |
+ stvx_u $out4,$x40,$out
|
|
|
fd2893 |
+ vxor $out4,$in4,$twk4
|
|
|
fd2893 |
+ le?stvx_u $out5,$x50,$out
|
|
|
fd2893 |
+ be?stvx_u $tmp, $x50,$out
|
|
|
fd2893 |
+ vxor $out5,$in5,$twk5
|
|
|
fd2893 |
+ addi $out,$out,0x60
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ mtctr $rounds
|
|
|
fd2893 |
+ beq Loop_xts_enc6x # did $len-=96 borrow?
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ addic. $len,$len,0x60
|
|
|
fd2893 |
+ beq Lxts_enc6x_zero
|
|
|
fd2893 |
+ cmpwi $len,0x20
|
|
|
fd2893 |
+ blt Lxts_enc6x_one
|
|
|
fd2893 |
+ nop
|
|
|
fd2893 |
+ beq Lxts_enc6x_two
|
|
|
fd2893 |
+ cmpwi $len,0x40
|
|
|
fd2893 |
+ blt Lxts_enc6x_three
|
|
|
fd2893 |
+ nop
|
|
|
fd2893 |
+ beq Lxts_enc6x_four
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+Lxts_enc6x_five:
|
|
|
fd2893 |
+ vxor $out0,$in1,$twk0
|
|
|
fd2893 |
+ vxor $out1,$in2,$twk1
|
|
|
fd2893 |
+ vxor $out2,$in3,$twk2
|
|
|
fd2893 |
+ vxor $out3,$in4,$twk3
|
|
|
fd2893 |
+ vxor $out4,$in5,$twk4
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ bl _aesp8_xts_enc5x
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ le?vperm $out0,$out0,$out0,$leperm
|
|
|
fd2893 |
+ vmr $twk0,$twk5 # unused tweak
|
|
|
fd2893 |
+ le?vperm $out1,$out1,$out1,$leperm
|
|
|
fd2893 |
+ stvx_u $out0,$x00,$out # store output
|
|
|
fd2893 |
+ le?vperm $out2,$out2,$out2,$leperm
|
|
|
fd2893 |
+ stvx_u $out1,$x10,$out
|
|
|
fd2893 |
+ le?vperm $out3,$out3,$out3,$leperm
|
|
|
fd2893 |
+ stvx_u $out2,$x20,$out
|
|
|
fd2893 |
+ vxor $tmp,$out4,$twk5 # last block prep for stealing
|
|
|
fd2893 |
+ le?vperm $out4,$out4,$out4,$leperm
|
|
|
fd2893 |
+ stvx_u $out3,$x30,$out
|
|
|
fd2893 |
+ stvx_u $out4,$x40,$out
|
|
|
fd2893 |
+ addi $out,$out,0x50
|
|
|
fd2893 |
+ bne Lxts_enc6x_steal
|
|
|
fd2893 |
+ b Lxts_enc6x_done
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+.align 4
|
|
|
fd2893 |
+Lxts_enc6x_four:
|
|
|
fd2893 |
+ vxor $out0,$in2,$twk0
|
|
|
fd2893 |
+ vxor $out1,$in3,$twk1
|
|
|
fd2893 |
+ vxor $out2,$in4,$twk2
|
|
|
fd2893 |
+ vxor $out3,$in5,$twk3
|
|
|
fd2893 |
+ vxor $out4,$out4,$out4
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ bl _aesp8_xts_enc5x
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ le?vperm $out0,$out0,$out0,$leperm
|
|
|
fd2893 |
+ vmr $twk0,$twk4 # unused tweak
|
|
|
fd2893 |
+ le?vperm $out1,$out1,$out1,$leperm
|
|
|
fd2893 |
+ stvx_u $out0,$x00,$out # store output
|
|
|
fd2893 |
+ le?vperm $out2,$out2,$out2,$leperm
|
|
|
fd2893 |
+ stvx_u $out1,$x10,$out
|
|
|
fd2893 |
+ vxor $tmp,$out3,$twk4 # last block prep for stealing
|
|
|
fd2893 |
+ le?vperm $out3,$out3,$out3,$leperm
|
|
|
fd2893 |
+ stvx_u $out2,$x20,$out
|
|
|
fd2893 |
+ stvx_u $out3,$x30,$out
|
|
|
fd2893 |
+ addi $out,$out,0x40
|
|
|
fd2893 |
+ bne Lxts_enc6x_steal
|
|
|
fd2893 |
+ b Lxts_enc6x_done
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+.align 4
|
|
|
fd2893 |
+Lxts_enc6x_three:
|
|
|
fd2893 |
+ vxor $out0,$in3,$twk0
|
|
|
fd2893 |
+ vxor $out1,$in4,$twk1
|
|
|
fd2893 |
+ vxor $out2,$in5,$twk2
|
|
|
fd2893 |
+ vxor $out3,$out3,$out3
|
|
|
fd2893 |
+ vxor $out4,$out4,$out4
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ bl _aesp8_xts_enc5x
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ le?vperm $out0,$out0,$out0,$leperm
|
|
|
fd2893 |
+ vmr $twk0,$twk3 # unused tweak
|
|
|
fd2893 |
+ le?vperm $out1,$out1,$out1,$leperm
|
|
|
fd2893 |
+ stvx_u $out0,$x00,$out # store output
|
|
|
fd2893 |
+ vxor $tmp,$out2,$twk3 # last block prep for stealing
|
|
|
fd2893 |
+ le?vperm $out2,$out2,$out2,$leperm
|
|
|
fd2893 |
+ stvx_u $out1,$x10,$out
|
|
|
fd2893 |
+ stvx_u $out2,$x20,$out
|
|
|
fd2893 |
+ addi $out,$out,0x30
|
|
|
fd2893 |
+ bne Lxts_enc6x_steal
|
|
|
fd2893 |
+ b Lxts_enc6x_done
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+.align 4
|
|
|
fd2893 |
+Lxts_enc6x_two:
|
|
|
fd2893 |
+ vxor $out0,$in4,$twk0
|
|
|
fd2893 |
+ vxor $out1,$in5,$twk1
|
|
|
fd2893 |
+ vxor $out2,$out2,$out2
|
|
|
fd2893 |
+ vxor $out3,$out3,$out3
|
|
|
fd2893 |
+ vxor $out4,$out4,$out4
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ bl _aesp8_xts_enc5x
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ le?vperm $out0,$out0,$out0,$leperm
|
|
|
fd2893 |
+ vmr $twk0,$twk2 # unused tweak
|
|
|
fd2893 |
+ vxor $tmp,$out1,$twk2 # last block prep for stealing
|
|
|
fd2893 |
+ le?vperm $out1,$out1,$out1,$leperm
|
|
|
fd2893 |
+ stvx_u $out0,$x00,$out # store output
|
|
|
fd2893 |
+ stvx_u $out1,$x10,$out
|
|
|
fd2893 |
+ addi $out,$out,0x20
|
|
|
fd2893 |
+ bne Lxts_enc6x_steal
|
|
|
fd2893 |
+ b Lxts_enc6x_done
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+.align 4
|
|
|
fd2893 |
+Lxts_enc6x_one:
|
|
|
fd2893 |
+ vxor $out0,$in5,$twk0
|
|
|
fd2893 |
+ nop
|
|
|
fd2893 |
+Loop_xts_enc1x:
|
|
|
fd2893 |
+ vcipher $out0,$out0,v24
|
|
|
fd2893 |
+ lvx v24,$x20,$key_ # round[3]
|
|
|
fd2893 |
+ addi $key_,$key_,0x20
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vcipher $out0,$out0,v25
|
|
|
fd2893 |
+ lvx v25,$x10,$key_ # round[4]
|
|
|
fd2893 |
+ bdnz Loop_xts_enc1x
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ add $inp,$inp,$taillen
|
|
|
fd2893 |
+ cmpwi $taillen,0
|
|
|
fd2893 |
+ vcipher $out0,$out0,v24
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ subi $inp,$inp,16
|
|
|
fd2893 |
+ vcipher $out0,$out0,v25
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ lvsr $inpperm,0,$taillen
|
|
|
fd2893 |
+ vcipher $out0,$out0,v26
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ lvx_u $in0,0,$inp
|
|
|
fd2893 |
+ vcipher $out0,$out0,v27
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ addi $key_,$sp,$FRAME+15 # rewind $key_
|
|
|
fd2893 |
+ vcipher $out0,$out0,v28
|
|
|
fd2893 |
+ lvx v24,$x00,$key_ # re-pre-load round[1]
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vcipher $out0,$out0,v29
|
|
|
fd2893 |
+ lvx v25,$x10,$key_ # re-pre-load round[2]
|
|
|
fd2893 |
+ vxor $twk0,$twk0,v31
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ le?vperm $in0,$in0,$in0,$leperm
|
|
|
fd2893 |
+ vcipher $out0,$out0,v30
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vperm $in0,$in0,$in0,$inpperm
|
|
|
fd2893 |
+ vcipherlast $out0,$out0,$twk0
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vmr $twk0,$twk1 # unused tweak
|
|
|
fd2893 |
+ vxor $tmp,$out0,$twk1 # last block prep for stealing
|
|
|
fd2893 |
+ le?vperm $out0,$out0,$out0,$leperm
|
|
|
fd2893 |
+ stvx_u $out0,$x00,$out # store output
|
|
|
fd2893 |
+ addi $out,$out,0x10
|
|
|
fd2893 |
+ bne Lxts_enc6x_steal
|
|
|
fd2893 |
+ b Lxts_enc6x_done
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+.align 4
|
|
|
fd2893 |
+Lxts_enc6x_zero:
|
|
|
fd2893 |
+ cmpwi $taillen,0
|
|
|
fd2893 |
+ beq Lxts_enc6x_done
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ add $inp,$inp,$taillen
|
|
|
fd2893 |
+ subi $inp,$inp,16
|
|
|
fd2893 |
+ lvx_u $in0,0,$inp
|
|
|
fd2893 |
+ lvsr $inpperm,0,$taillen # $in5 is no more
|
|
|
fd2893 |
+ le?vperm $in0,$in0,$in0,$leperm
|
|
|
fd2893 |
+ vperm $in0,$in0,$in0,$inpperm
|
|
|
fd2893 |
+ vxor $tmp,$tmp,$twk0
|
|
|
fd2893 |
+Lxts_enc6x_steal:
|
|
|
fd2893 |
+ vxor $in0,$in0,$twk0
|
|
|
fd2893 |
+ vxor $out0,$out0,$out0
|
|
|
fd2893 |
+ vspltisb $out1,-1
|
|
|
fd2893 |
+ vperm $out0,$out0,$out1,$inpperm
|
|
|
fd2893 |
+ vsel $out0,$in0,$tmp,$out0 # $tmp is last block, remember?
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ subi r30,$out,17
|
|
|
fd2893 |
+ subi $out,$out,16
|
|
|
fd2893 |
+ mtctr $taillen
|
|
|
fd2893 |
+Loop_xts_enc6x_steal:
|
|
|
fd2893 |
+ lbzu r0,1(r30)
|
|
|
fd2893 |
+ stb r0,16(r30)
|
|
|
fd2893 |
+ bdnz Loop_xts_enc6x_steal
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ li $taillen,0
|
|
|
fd2893 |
+ mtctr $rounds
|
|
|
fd2893 |
+ b Loop_xts_enc1x # one more time...
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+.align 4
|
|
|
fd2893 |
+Lxts_enc6x_done:
|
|
|
fd2893 |
+ ${UCMP}i $ivp,0
|
|
|
fd2893 |
+ beq Lxts_enc6x_ret
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vxor $tweak,$twk0,$rndkey0
|
|
|
fd2893 |
+ le?vperm $tweak,$tweak,$tweak,$leperm
|
|
|
fd2893 |
+ stvx_u $tweak,0,$ivp
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+Lxts_enc6x_ret:
|
|
|
fd2893 |
+ mtlr r11
|
|
|
fd2893 |
+ li r10,`$FRAME+15`
|
|
|
fd2893 |
+ li r11,`$FRAME+31`
|
|
|
fd2893 |
+ stvx $seven,r10,$sp # wipe copies of round keys
|
|
|
fd2893 |
+ addi r10,r10,32
|
|
|
fd2893 |
+ stvx $seven,r11,$sp
|
|
|
fd2893 |
+ addi r11,r11,32
|
|
|
fd2893 |
+ stvx $seven,r10,$sp
|
|
|
fd2893 |
+ addi r10,r10,32
|
|
|
fd2893 |
+ stvx $seven,r11,$sp
|
|
|
fd2893 |
+ addi r11,r11,32
|
|
|
fd2893 |
+ stvx $seven,r10,$sp
|
|
|
fd2893 |
+ addi r10,r10,32
|
|
|
fd2893 |
+ stvx $seven,r11,$sp
|
|
|
fd2893 |
+ addi r11,r11,32
|
|
|
fd2893 |
+ stvx $seven,r10,$sp
|
|
|
fd2893 |
+ addi r10,r10,32
|
|
|
fd2893 |
+ stvx $seven,r11,$sp
|
|
|
fd2893 |
+ addi r11,r11,32
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ mtspr 256,$vrsave
|
|
|
fd2893 |
+ lvx v20,r10,$sp # ABI says so
|
|
|
fd2893 |
+ addi r10,r10,32
|
|
|
fd2893 |
+ lvx v21,r11,$sp
|
|
|
fd2893 |
+ addi r11,r11,32
|
|
|
fd2893 |
+ lvx v22,r10,$sp
|
|
|
fd2893 |
+ addi r10,r10,32
|
|
|
fd2893 |
+ lvx v23,r11,$sp
|
|
|
fd2893 |
+ addi r11,r11,32
|
|
|
fd2893 |
+ lvx v24,r10,$sp
|
|
|
fd2893 |
+ addi r10,r10,32
|
|
|
fd2893 |
+ lvx v25,r11,$sp
|
|
|
fd2893 |
+ addi r11,r11,32
|
|
|
fd2893 |
+ lvx v26,r10,$sp
|
|
|
fd2893 |
+ addi r10,r10,32
|
|
|
fd2893 |
+ lvx v27,r11,$sp
|
|
|
fd2893 |
+ addi r11,r11,32
|
|
|
fd2893 |
+ lvx v28,r10,$sp
|
|
|
fd2893 |
+ addi r10,r10,32
|
|
|
fd2893 |
+ lvx v29,r11,$sp
|
|
|
fd2893 |
+ addi r11,r11,32
|
|
|
fd2893 |
+ lvx v30,r10,$sp
|
|
|
fd2893 |
+ lvx v31,r11,$sp
|
|
|
fd2893 |
+ $POP r26,`$FRAME+21*16+0*$SIZE_T`($sp)
|
|
|
fd2893 |
+ $POP r27,`$FRAME+21*16+1*$SIZE_T`($sp)
|
|
|
fd2893 |
+ $POP r28,`$FRAME+21*16+2*$SIZE_T`($sp)
|
|
|
fd2893 |
+ $POP r29,`$FRAME+21*16+3*$SIZE_T`($sp)
|
|
|
fd2893 |
+ $POP r30,`$FRAME+21*16+4*$SIZE_T`($sp)
|
|
|
fd2893 |
+ $POP r31,`$FRAME+21*16+5*$SIZE_T`($sp)
|
|
|
fd2893 |
+ addi $sp,$sp,`$FRAME+21*16+6*$SIZE_T`
|
|
|
fd2893 |
+ blr
|
|
|
fd2893 |
+ .long 0
|
|
|
fd2893 |
+ .byte 0,12,0x04,1,0x80,6,6,0
|
|
|
fd2893 |
+ .long 0
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+.align 5
|
|
|
fd2893 |
+_aesp8_xts_enc5x:
|
|
|
fd2893 |
+ vcipher $out0,$out0,v24
|
|
|
fd2893 |
+ vcipher $out1,$out1,v24
|
|
|
fd2893 |
+ vcipher $out2,$out2,v24
|
|
|
fd2893 |
+ vcipher $out3,$out3,v24
|
|
|
fd2893 |
+ vcipher $out4,$out4,v24
|
|
|
fd2893 |
+ lvx v24,$x20,$key_ # round[3]
|
|
|
fd2893 |
+ addi $key_,$key_,0x20
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vcipher $out0,$out0,v25
|
|
|
fd2893 |
+ vcipher $out1,$out1,v25
|
|
|
fd2893 |
+ vcipher $out2,$out2,v25
|
|
|
fd2893 |
+ vcipher $out3,$out3,v25
|
|
|
fd2893 |
+ vcipher $out4,$out4,v25
|
|
|
fd2893 |
+ lvx v25,$x10,$key_ # round[4]
|
|
|
fd2893 |
+ bdnz _aesp8_xts_enc5x
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ add $inp,$inp,$taillen
|
|
|
fd2893 |
+ cmpwi $taillen,0
|
|
|
fd2893 |
+ vcipher $out0,$out0,v24
|
|
|
fd2893 |
+ vcipher $out1,$out1,v24
|
|
|
fd2893 |
+ vcipher $out2,$out2,v24
|
|
|
fd2893 |
+ vcipher $out3,$out3,v24
|
|
|
fd2893 |
+ vcipher $out4,$out4,v24
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ subi $inp,$inp,16
|
|
|
fd2893 |
+ vcipher $out0,$out0,v25
|
|
|
fd2893 |
+ vcipher $out1,$out1,v25
|
|
|
fd2893 |
+ vcipher $out2,$out2,v25
|
|
|
fd2893 |
+ vcipher $out3,$out3,v25
|
|
|
fd2893 |
+ vcipher $out4,$out4,v25
|
|
|
fd2893 |
+ vxor $twk0,$twk0,v31
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vcipher $out0,$out0,v26
|
|
|
fd2893 |
+ lvsr $inpperm,r0,$taillen # $in5 is no more
|
|
|
fd2893 |
+ vcipher $out1,$out1,v26
|
|
|
fd2893 |
+ vcipher $out2,$out2,v26
|
|
|
fd2893 |
+ vcipher $out3,$out3,v26
|
|
|
fd2893 |
+ vcipher $out4,$out4,v26
|
|
|
fd2893 |
+ vxor $in1,$twk1,v31
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vcipher $out0,$out0,v27
|
|
|
fd2893 |
+ lvx_u $in0,0,$inp
|
|
|
fd2893 |
+ vcipher $out1,$out1,v27
|
|
|
fd2893 |
+ vcipher $out2,$out2,v27
|
|
|
fd2893 |
+ vcipher $out3,$out3,v27
|
|
|
fd2893 |
+ vcipher $out4,$out4,v27
|
|
|
fd2893 |
+ vxor $in2,$twk2,v31
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ addi $key_,$sp,$FRAME+15 # rewind $key_
|
|
|
fd2893 |
+ vcipher $out0,$out0,v28
|
|
|
fd2893 |
+ vcipher $out1,$out1,v28
|
|
|
fd2893 |
+ vcipher $out2,$out2,v28
|
|
|
fd2893 |
+ vcipher $out3,$out3,v28
|
|
|
fd2893 |
+ vcipher $out4,$out4,v28
|
|
|
fd2893 |
+ lvx v24,$x00,$key_ # re-pre-load round[1]
|
|
|
fd2893 |
+ vxor $in3,$twk3,v31
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vcipher $out0,$out0,v29
|
|
|
fd2893 |
+ le?vperm $in0,$in0,$in0,$leperm
|
|
|
fd2893 |
+ vcipher $out1,$out1,v29
|
|
|
fd2893 |
+ vcipher $out2,$out2,v29
|
|
|
fd2893 |
+ vcipher $out3,$out3,v29
|
|
|
fd2893 |
+ vcipher $out4,$out4,v29
|
|
|
fd2893 |
+ lvx v25,$x10,$key_ # re-pre-load round[2]
|
|
|
fd2893 |
+ vxor $in4,$twk4,v31
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vcipher $out0,$out0,v30
|
|
|
fd2893 |
+ vperm $in0,$in0,$in0,$inpperm
|
|
|
fd2893 |
+ vcipher $out1,$out1,v30
|
|
|
fd2893 |
+ vcipher $out2,$out2,v30
|
|
|
fd2893 |
+ vcipher $out3,$out3,v30
|
|
|
fd2893 |
+ vcipher $out4,$out4,v30
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vcipherlast $out0,$out0,$twk0
|
|
|
fd2893 |
+ vcipherlast $out1,$out1,$in1
|
|
|
fd2893 |
+ vcipherlast $out2,$out2,$in2
|
|
|
fd2893 |
+ vcipherlast $out3,$out3,$in3
|
|
|
fd2893 |
+ vcipherlast $out4,$out4,$in4
|
|
|
fd2893 |
+ blr
|
|
|
fd2893 |
+ .long 0
|
|
|
fd2893 |
+ .byte 0,12,0x14,0,0,0,0,0
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+.align 5
|
|
|
fd2893 |
+_aesp8_xts_decrypt6x:
|
|
|
fd2893 |
+ $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
|
|
|
fd2893 |
+ mflr r11
|
|
|
fd2893 |
+ li r7,`$FRAME+8*16+15`
|
|
|
fd2893 |
+ li r3,`$FRAME+8*16+31`
|
|
|
fd2893 |
+ $PUSH r11,`$FRAME+21*16+6*$SIZE_T+$LRSAVE`($sp)
|
|
|
fd2893 |
+ stvx v20,r7,$sp # ABI says so
|
|
|
fd2893 |
+ addi r7,r7,32
|
|
|
fd2893 |
+ stvx v21,r3,$sp
|
|
|
fd2893 |
+ addi r3,r3,32
|
|
|
fd2893 |
+ stvx v22,r7,$sp
|
|
|
fd2893 |
+ addi r7,r7,32
|
|
|
fd2893 |
+ stvx v23,r3,$sp
|
|
|
fd2893 |
+ addi r3,r3,32
|
|
|
fd2893 |
+ stvx v24,r7,$sp
|
|
|
fd2893 |
+ addi r7,r7,32
|
|
|
fd2893 |
+ stvx v25,r3,$sp
|
|
|
fd2893 |
+ addi r3,r3,32
|
|
|
fd2893 |
+ stvx v26,r7,$sp
|
|
|
fd2893 |
+ addi r7,r7,32
|
|
|
fd2893 |
+ stvx v27,r3,$sp
|
|
|
fd2893 |
+ addi r3,r3,32
|
|
|
fd2893 |
+ stvx v28,r7,$sp
|
|
|
fd2893 |
+ addi r7,r7,32
|
|
|
fd2893 |
+ stvx v29,r3,$sp
|
|
|
fd2893 |
+ addi r3,r3,32
|
|
|
fd2893 |
+ stvx v30,r7,$sp
|
|
|
fd2893 |
+ stvx v31,r3,$sp
|
|
|
fd2893 |
+ li r0,-1
|
|
|
fd2893 |
+ stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave
|
|
|
fd2893 |
+ li $x10,0x10
|
|
|
fd2893 |
+ $PUSH r26,`$FRAME+21*16+0*$SIZE_T`($sp)
|
|
|
fd2893 |
+ li $x20,0x20
|
|
|
fd2893 |
+ $PUSH r27,`$FRAME+21*16+1*$SIZE_T`($sp)
|
|
|
fd2893 |
+ li $x30,0x30
|
|
|
fd2893 |
+ $PUSH r28,`$FRAME+21*16+2*$SIZE_T`($sp)
|
|
|
fd2893 |
+ li $x40,0x40
|
|
|
fd2893 |
+ $PUSH r29,`$FRAME+21*16+3*$SIZE_T`($sp)
|
|
|
fd2893 |
+ li $x50,0x50
|
|
|
fd2893 |
+ $PUSH r30,`$FRAME+21*16+4*$SIZE_T`($sp)
|
|
|
fd2893 |
+ li $x60,0x60
|
|
|
fd2893 |
+ $PUSH r31,`$FRAME+21*16+5*$SIZE_T`($sp)
|
|
|
fd2893 |
+ li $x70,0x70
|
|
|
fd2893 |
+ mtspr 256,r0
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ subi $rounds,$rounds,3 # -4 in total
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ lvx $rndkey0,$x00,$key1 # load key schedule
|
|
|
fd2893 |
+ lvx v30,$x10,$key1
|
|
|
fd2893 |
+ addi $key1,$key1,0x20
|
|
|
fd2893 |
+ lvx v31,$x00,$key1
|
|
|
fd2893 |
+ ?vperm $rndkey0,$rndkey0,v30,$keyperm
|
|
|
fd2893 |
+ addi $key_,$sp,$FRAME+15
|
|
|
fd2893 |
+ mtctr $rounds
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+Load_xts_dec_key:
|
|
|
fd2893 |
+ ?vperm v24,v30,v31,$keyperm
|
|
|
fd2893 |
+ lvx v30,$x10,$key1
|
|
|
fd2893 |
+ addi $key1,$key1,0x20
|
|
|
fd2893 |
+ stvx v24,$x00,$key_ # off-load round[1]
|
|
|
fd2893 |
+ ?vperm v25,v31,v30,$keyperm
|
|
|
fd2893 |
+ lvx v31,$x00,$key1
|
|
|
fd2893 |
+ stvx v25,$x10,$key_ # off-load round[2]
|
|
|
fd2893 |
+ addi $key_,$key_,0x20
|
|
|
fd2893 |
+ bdnz Load_xts_dec_key
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ lvx v26,$x10,$key1
|
|
|
fd2893 |
+ ?vperm v24,v30,v31,$keyperm
|
|
|
fd2893 |
+ lvx v27,$x20,$key1
|
|
|
fd2893 |
+ stvx v24,$x00,$key_ # off-load round[3]
|
|
|
fd2893 |
+ ?vperm v25,v31,v26,$keyperm
|
|
|
fd2893 |
+ lvx v28,$x30,$key1
|
|
|
fd2893 |
+ stvx v25,$x10,$key_ # off-load round[4]
|
|
|
fd2893 |
+ addi $key_,$sp,$FRAME+15 # rewind $key_
|
|
|
fd2893 |
+ ?vperm v26,v26,v27,$keyperm
|
|
|
fd2893 |
+ lvx v29,$x40,$key1
|
|
|
fd2893 |
+ ?vperm v27,v27,v28,$keyperm
|
|
|
fd2893 |
+ lvx v30,$x50,$key1
|
|
|
fd2893 |
+ ?vperm v28,v28,v29,$keyperm
|
|
|
fd2893 |
+ lvx v31,$x60,$key1
|
|
|
fd2893 |
+ ?vperm v29,v29,v30,$keyperm
|
|
|
fd2893 |
+ lvx $twk5,$x70,$key1 # borrow $twk5
|
|
|
fd2893 |
+ ?vperm v30,v30,v31,$keyperm
|
|
|
fd2893 |
+ lvx v24,$x00,$key_ # pre-load round[1]
|
|
|
fd2893 |
+ ?vperm v31,v31,$twk5,$keyperm
|
|
|
fd2893 |
+ lvx v25,$x10,$key_ # pre-load round[2]
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vperm $in0,$inout,$inptail,$inpperm
|
|
|
fd2893 |
+ subi $inp,$inp,31 # undo "caller"
|
|
|
fd2893 |
+ vxor $twk0,$tweak,$rndkey0
|
|
|
fd2893 |
+ vsrab $tmp,$tweak,$seven # next tweak value
|
|
|
fd2893 |
+ vaddubm $tweak,$tweak,$tweak
|
|
|
fd2893 |
+ vsldoi $tmp,$tmp,$tmp,15
|
|
|
fd2893 |
+ vand $tmp,$tmp,$eighty7
|
|
|
fd2893 |
+ vxor $out0,$in0,$twk0
|
|
|
fd2893 |
+ vxor $tweak,$tweak,$tmp
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ lvx_u $in1,$x10,$inp
|
|
|
fd2893 |
+ vxor $twk1,$tweak,$rndkey0
|
|
|
fd2893 |
+ vsrab $tmp,$tweak,$seven # next tweak value
|
|
|
fd2893 |
+ vaddubm $tweak,$tweak,$tweak
|
|
|
fd2893 |
+ vsldoi $tmp,$tmp,$tmp,15
|
|
|
fd2893 |
+ le?vperm $in1,$in1,$in1,$leperm
|
|
|
fd2893 |
+ vand $tmp,$tmp,$eighty7
|
|
|
fd2893 |
+ vxor $out1,$in1,$twk1
|
|
|
fd2893 |
+ vxor $tweak,$tweak,$tmp
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ lvx_u $in2,$x20,$inp
|
|
|
fd2893 |
+ andi. $taillen,$len,15
|
|
|
fd2893 |
+ vxor $twk2,$tweak,$rndkey0
|
|
|
fd2893 |
+ vsrab $tmp,$tweak,$seven # next tweak value
|
|
|
fd2893 |
+ vaddubm $tweak,$tweak,$tweak
|
|
|
fd2893 |
+ vsldoi $tmp,$tmp,$tmp,15
|
|
|
fd2893 |
+ le?vperm $in2,$in2,$in2,$leperm
|
|
|
fd2893 |
+ vand $tmp,$tmp,$eighty7
|
|
|
fd2893 |
+ vxor $out2,$in2,$twk2
|
|
|
fd2893 |
+ vxor $tweak,$tweak,$tmp
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ lvx_u $in3,$x30,$inp
|
|
|
fd2893 |
+ sub $len,$len,$taillen
|
|
|
fd2893 |
+ vxor $twk3,$tweak,$rndkey0
|
|
|
fd2893 |
+ vsrab $tmp,$tweak,$seven # next tweak value
|
|
|
fd2893 |
+ vaddubm $tweak,$tweak,$tweak
|
|
|
fd2893 |
+ vsldoi $tmp,$tmp,$tmp,15
|
|
|
fd2893 |
+ le?vperm $in3,$in3,$in3,$leperm
|
|
|
fd2893 |
+ vand $tmp,$tmp,$eighty7
|
|
|
fd2893 |
+ vxor $out3,$in3,$twk3
|
|
|
fd2893 |
+ vxor $tweak,$tweak,$tmp
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ lvx_u $in4,$x40,$inp
|
|
|
fd2893 |
+ subi $len,$len,0x60
|
|
|
fd2893 |
+ vxor $twk4,$tweak,$rndkey0
|
|
|
fd2893 |
+ vsrab $tmp,$tweak,$seven # next tweak value
|
|
|
fd2893 |
+ vaddubm $tweak,$tweak,$tweak
|
|
|
fd2893 |
+ vsldoi $tmp,$tmp,$tmp,15
|
|
|
fd2893 |
+ le?vperm $in4,$in4,$in4,$leperm
|
|
|
fd2893 |
+ vand $tmp,$tmp,$eighty7
|
|
|
fd2893 |
+ vxor $out4,$in4,$twk4
|
|
|
fd2893 |
+ vxor $tweak,$tweak,$tmp
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ lvx_u $in5,$x50,$inp
|
|
|
fd2893 |
+ addi $inp,$inp,0x60
|
|
|
fd2893 |
+ vxor $twk5,$tweak,$rndkey0
|
|
|
fd2893 |
+ vsrab $tmp,$tweak,$seven # next tweak value
|
|
|
fd2893 |
+ vaddubm $tweak,$tweak,$tweak
|
|
|
fd2893 |
+ vsldoi $tmp,$tmp,$tmp,15
|
|
|
fd2893 |
+ le?vperm $in5,$in5,$in5,$leperm
|
|
|
fd2893 |
+ vand $tmp,$tmp,$eighty7
|
|
|
fd2893 |
+ vxor $out5,$in5,$twk5
|
|
|
fd2893 |
+ vxor $tweak,$tweak,$tmp
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vxor v31,v31,$rndkey0
|
|
|
fd2893 |
+ mtctr $rounds
|
|
|
fd2893 |
+ b Loop_xts_dec6x
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+.align 5
|
|
|
fd2893 |
+Loop_xts_dec6x:
|
|
|
fd2893 |
+ vncipher $out0,$out0,v24
|
|
|
fd2893 |
+ vncipher $out1,$out1,v24
|
|
|
fd2893 |
+ vncipher $out2,$out2,v24
|
|
|
fd2893 |
+ vncipher $out3,$out3,v24
|
|
|
fd2893 |
+ vncipher $out4,$out4,v24
|
|
|
fd2893 |
+ vncipher $out5,$out5,v24
|
|
|
fd2893 |
+ lvx v24,$x20,$key_ # round[3]
|
|
|
fd2893 |
+ addi $key_,$key_,0x20
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vncipher $out0,$out0,v25
|
|
|
fd2893 |
+ vncipher $out1,$out1,v25
|
|
|
fd2893 |
+ vncipher $out2,$out2,v25
|
|
|
fd2893 |
+ vncipher $out3,$out3,v25
|
|
|
fd2893 |
+ vncipher $out4,$out4,v25
|
|
|
fd2893 |
+ vncipher $out5,$out5,v25
|
|
|
fd2893 |
+ lvx v25,$x10,$key_ # round[4]
|
|
|
fd2893 |
+ bdnz Loop_xts_dec6x
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ subic $len,$len,96 # $len-=96
|
|
|
fd2893 |
+ vxor $in0,$twk0,v31 # xor with last round key
|
|
|
fd2893 |
+ vncipher $out0,$out0,v24
|
|
|
fd2893 |
+ vncipher $out1,$out1,v24
|
|
|
fd2893 |
+ vsrab $tmp,$tweak,$seven # next tweak value
|
|
|
fd2893 |
+ vxor $twk0,$tweak,$rndkey0
|
|
|
fd2893 |
+ vaddubm $tweak,$tweak,$tweak
|
|
|
fd2893 |
+ vncipher $out2,$out2,v24
|
|
|
fd2893 |
+ vncipher $out3,$out3,v24
|
|
|
fd2893 |
+ vsldoi $tmp,$tmp,$tmp,15
|
|
|
fd2893 |
+ vncipher $out4,$out4,v24
|
|
|
fd2893 |
+ vncipher $out5,$out5,v24
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ subfe. r0,r0,r0 # borrow?-1:0
|
|
|
fd2893 |
+ vand $tmp,$tmp,$eighty7
|
|
|
fd2893 |
+ vncipher $out0,$out0,v25
|
|
|
fd2893 |
+ vncipher $out1,$out1,v25
|
|
|
fd2893 |
+ vxor $tweak,$tweak,$tmp
|
|
|
fd2893 |
+ vncipher $out2,$out2,v25
|
|
|
fd2893 |
+ vncipher $out3,$out3,v25
|
|
|
fd2893 |
+ vxor $in1,$twk1,v31
|
|
|
fd2893 |
+ vsrab $tmp,$tweak,$seven # next tweak value
|
|
|
fd2893 |
+ vxor $twk1,$tweak,$rndkey0
|
|
|
fd2893 |
+ vncipher $out4,$out4,v25
|
|
|
fd2893 |
+ vncipher $out5,$out5,v25
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ and r0,r0,$len
|
|
|
fd2893 |
+ vaddubm $tweak,$tweak,$tweak
|
|
|
fd2893 |
+ vsldoi $tmp,$tmp,$tmp,15
|
|
|
fd2893 |
+ vncipher $out0,$out0,v26
|
|
|
fd2893 |
+ vncipher $out1,$out1,v26
|
|
|
fd2893 |
+ vand $tmp,$tmp,$eighty7
|
|
|
fd2893 |
+ vncipher $out2,$out2,v26
|
|
|
fd2893 |
+ vncipher $out3,$out3,v26
|
|
|
fd2893 |
+ vxor $tweak,$tweak,$tmp
|
|
|
fd2893 |
+ vncipher $out4,$out4,v26
|
|
|
fd2893 |
+ vncipher $out5,$out5,v26
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ add $inp,$inp,r0 # $inp is adjusted in such
|
|
|
fd2893 |
+ # way that at exit from the
|
|
|
fd2893 |
+ # loop inX-in5 are loaded
|
|
|
fd2893 |
+ # with last "words"
|
|
|
fd2893 |
+ vxor $in2,$twk2,v31
|
|
|
fd2893 |
+ vsrab $tmp,$tweak,$seven # next tweak value
|
|
|
fd2893 |
+ vxor $twk2,$tweak,$rndkey0
|
|
|
fd2893 |
+ vaddubm $tweak,$tweak,$tweak
|
|
|
fd2893 |
+ vncipher $out0,$out0,v27
|
|
|
fd2893 |
+ vncipher $out1,$out1,v27
|
|
|
fd2893 |
+ vsldoi $tmp,$tmp,$tmp,15
|
|
|
fd2893 |
+ vncipher $out2,$out2,v27
|
|
|
fd2893 |
+ vncipher $out3,$out3,v27
|
|
|
fd2893 |
+ vand $tmp,$tmp,$eighty7
|
|
|
fd2893 |
+ vncipher $out4,$out4,v27
|
|
|
fd2893 |
+ vncipher $out5,$out5,v27
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ addi $key_,$sp,$FRAME+15 # rewind $key_
|
|
|
fd2893 |
+ vxor $tweak,$tweak,$tmp
|
|
|
fd2893 |
+ vncipher $out0,$out0,v28
|
|
|
fd2893 |
+ vncipher $out1,$out1,v28
|
|
|
fd2893 |
+ vxor $in3,$twk3,v31
|
|
|
fd2893 |
+ vsrab $tmp,$tweak,$seven # next tweak value
|
|
|
fd2893 |
+ vxor $twk3,$tweak,$rndkey0
|
|
|
fd2893 |
+ vncipher $out2,$out2,v28
|
|
|
fd2893 |
+ vncipher $out3,$out3,v28
|
|
|
fd2893 |
+ vaddubm $tweak,$tweak,$tweak
|
|
|
fd2893 |
+ vsldoi $tmp,$tmp,$tmp,15
|
|
|
fd2893 |
+ vncipher $out4,$out4,v28
|
|
|
fd2893 |
+ vncipher $out5,$out5,v28
|
|
|
fd2893 |
+ lvx v24,$x00,$key_ # re-pre-load round[1]
|
|
|
fd2893 |
+ vand $tmp,$tmp,$eighty7
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vncipher $out0,$out0,v29
|
|
|
fd2893 |
+ vncipher $out1,$out1,v29
|
|
|
fd2893 |
+ vxor $tweak,$tweak,$tmp
|
|
|
fd2893 |
+ vncipher $out2,$out2,v29
|
|
|
fd2893 |
+ vncipher $out3,$out3,v29
|
|
|
fd2893 |
+ vxor $in4,$twk4,v31
|
|
|
fd2893 |
+ vsrab $tmp,$tweak,$seven # next tweak value
|
|
|
fd2893 |
+ vxor $twk4,$tweak,$rndkey0
|
|
|
fd2893 |
+ vncipher $out4,$out4,v29
|
|
|
fd2893 |
+ vncipher $out5,$out5,v29
|
|
|
fd2893 |
+ lvx v25,$x10,$key_ # re-pre-load round[2]
|
|
|
fd2893 |
+ vaddubm $tweak,$tweak,$tweak
|
|
|
fd2893 |
+ vsldoi $tmp,$tmp,$tmp,15
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vncipher $out0,$out0,v30
|
|
|
fd2893 |
+ vncipher $out1,$out1,v30
|
|
|
fd2893 |
+ vand $tmp,$tmp,$eighty7
|
|
|
fd2893 |
+ vncipher $out2,$out2,v30
|
|
|
fd2893 |
+ vncipher $out3,$out3,v30
|
|
|
fd2893 |
+ vxor $tweak,$tweak,$tmp
|
|
|
fd2893 |
+ vncipher $out4,$out4,v30
|
|
|
fd2893 |
+ vncipher $out5,$out5,v30
|
|
|
fd2893 |
+ vxor $in5,$twk5,v31
|
|
|
fd2893 |
+ vsrab $tmp,$tweak,$seven # next tweak value
|
|
|
fd2893 |
+ vxor $twk5,$tweak,$rndkey0
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vncipherlast $out0,$out0,$in0
|
|
|
fd2893 |
+ lvx_u $in0,$x00,$inp # load next input block
|
|
|
fd2893 |
+ vaddubm $tweak,$tweak,$tweak
|
|
|
fd2893 |
+ vsldoi $tmp,$tmp,$tmp,15
|
|
|
fd2893 |
+ vncipherlast $out1,$out1,$in1
|
|
|
fd2893 |
+ lvx_u $in1,$x10,$inp
|
|
|
fd2893 |
+ vncipherlast $out2,$out2,$in2
|
|
|
fd2893 |
+ le?vperm $in0,$in0,$in0,$leperm
|
|
|
fd2893 |
+ lvx_u $in2,$x20,$inp
|
|
|
fd2893 |
+ vand $tmp,$tmp,$eighty7
|
|
|
fd2893 |
+ vncipherlast $out3,$out3,$in3
|
|
|
fd2893 |
+ le?vperm $in1,$in1,$in1,$leperm
|
|
|
fd2893 |
+ lvx_u $in3,$x30,$inp
|
|
|
fd2893 |
+ vncipherlast $out4,$out4,$in4
|
|
|
fd2893 |
+ le?vperm $in2,$in2,$in2,$leperm
|
|
|
fd2893 |
+ lvx_u $in4,$x40,$inp
|
|
|
fd2893 |
+ vxor $tweak,$tweak,$tmp
|
|
|
fd2893 |
+ vncipherlast $out5,$out5,$in5
|
|
|
fd2893 |
+ le?vperm $in3,$in3,$in3,$leperm
|
|
|
fd2893 |
+ lvx_u $in5,$x50,$inp
|
|
|
fd2893 |
+ addi $inp,$inp,0x60
|
|
|
fd2893 |
+ le?vperm $in4,$in4,$in4,$leperm
|
|
|
fd2893 |
+ le?vperm $in5,$in5,$in5,$leperm
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ le?vperm $out0,$out0,$out0,$leperm
|
|
|
fd2893 |
+ le?vperm $out1,$out1,$out1,$leperm
|
|
|
fd2893 |
+ stvx_u $out0,$x00,$out # store output
|
|
|
fd2893 |
+ vxor $out0,$in0,$twk0
|
|
|
fd2893 |
+ le?vperm $out2,$out2,$out2,$leperm
|
|
|
fd2893 |
+ stvx_u $out1,$x10,$out
|
|
|
fd2893 |
+ vxor $out1,$in1,$twk1
|
|
|
fd2893 |
+ le?vperm $out3,$out3,$out3,$leperm
|
|
|
fd2893 |
+ stvx_u $out2,$x20,$out
|
|
|
fd2893 |
+ vxor $out2,$in2,$twk2
|
|
|
fd2893 |
+ le?vperm $out4,$out4,$out4,$leperm
|
|
|
fd2893 |
+ stvx_u $out3,$x30,$out
|
|
|
fd2893 |
+ vxor $out3,$in3,$twk3
|
|
|
fd2893 |
+ le?vperm $out5,$out5,$out5,$leperm
|
|
|
fd2893 |
+ stvx_u $out4,$x40,$out
|
|
|
fd2893 |
+ vxor $out4,$in4,$twk4
|
|
|
fd2893 |
+ stvx_u $out5,$x50,$out
|
|
|
fd2893 |
+ vxor $out5,$in5,$twk5
|
|
|
fd2893 |
+ addi $out,$out,0x60
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ mtctr $rounds
|
|
|
fd2893 |
+ beq Loop_xts_dec6x # did $len-=96 borrow?
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ addic. $len,$len,0x60
|
|
|
fd2893 |
+ beq Lxts_dec6x_zero
|
|
|
fd2893 |
+ cmpwi $len,0x20
|
|
|
fd2893 |
+ blt Lxts_dec6x_one
|
|
|
fd2893 |
+ nop
|
|
|
fd2893 |
+ beq Lxts_dec6x_two
|
|
|
fd2893 |
+ cmpwi $len,0x40
|
|
|
fd2893 |
+ blt Lxts_dec6x_three
|
|
|
fd2893 |
+ nop
|
|
|
fd2893 |
+ beq Lxts_dec6x_four
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+Lxts_dec6x_five:
|
|
|
fd2893 |
+ vxor $out0,$in1,$twk0
|
|
|
fd2893 |
+ vxor $out1,$in2,$twk1
|
|
|
fd2893 |
+ vxor $out2,$in3,$twk2
|
|
|
fd2893 |
+ vxor $out3,$in4,$twk3
|
|
|
fd2893 |
+ vxor $out4,$in5,$twk4
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ bl _aesp8_xts_dec5x
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ le?vperm $out0,$out0,$out0,$leperm
|
|
|
fd2893 |
+ vmr $twk0,$twk5 # unused tweak
|
|
|
fd2893 |
+ vxor $twk1,$tweak,$rndkey0
|
|
|
fd2893 |
+ le?vperm $out1,$out1,$out1,$leperm
|
|
|
fd2893 |
+ stvx_u $out0,$x00,$out # store output
|
|
|
fd2893 |
+ vxor $out0,$in0,$twk1
|
|
|
fd2893 |
+ le?vperm $out2,$out2,$out2,$leperm
|
|
|
fd2893 |
+ stvx_u $out1,$x10,$out
|
|
|
fd2893 |
+ le?vperm $out3,$out3,$out3,$leperm
|
|
|
fd2893 |
+ stvx_u $out2,$x20,$out
|
|
|
fd2893 |
+ le?vperm $out4,$out4,$out4,$leperm
|
|
|
fd2893 |
+ stvx_u $out3,$x30,$out
|
|
|
fd2893 |
+ stvx_u $out4,$x40,$out
|
|
|
fd2893 |
+ addi $out,$out,0x50
|
|
|
fd2893 |
+ bne Lxts_dec6x_steal
|
|
|
fd2893 |
+ b Lxts_dec6x_done
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+.align 4
|
|
|
fd2893 |
+Lxts_dec6x_four:
|
|
|
fd2893 |
+ vxor $out0,$in2,$twk0
|
|
|
fd2893 |
+ vxor $out1,$in3,$twk1
|
|
|
fd2893 |
+ vxor $out2,$in4,$twk2
|
|
|
fd2893 |
+ vxor $out3,$in5,$twk3
|
|
|
fd2893 |
+ vxor $out4,$out4,$out4
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ bl _aesp8_xts_dec5x
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ le?vperm $out0,$out0,$out0,$leperm
|
|
|
fd2893 |
+ vmr $twk0,$twk4 # unused tweak
|
|
|
fd2893 |
+ vmr $twk1,$twk5
|
|
|
fd2893 |
+ le?vperm $out1,$out1,$out1,$leperm
|
|
|
fd2893 |
+ stvx_u $out0,$x00,$out # store output
|
|
|
fd2893 |
+ vxor $out0,$in0,$twk5
|
|
|
fd2893 |
+ le?vperm $out2,$out2,$out2,$leperm
|
|
|
fd2893 |
+ stvx_u $out1,$x10,$out
|
|
|
fd2893 |
+ le?vperm $out3,$out3,$out3,$leperm
|
|
|
fd2893 |
+ stvx_u $out2,$x20,$out
|
|
|
fd2893 |
+ stvx_u $out3,$x30,$out
|
|
|
fd2893 |
+ addi $out,$out,0x40
|
|
|
fd2893 |
+ bne Lxts_dec6x_steal
|
|
|
fd2893 |
+ b Lxts_dec6x_done
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+.align 4
|
|
|
fd2893 |
+Lxts_dec6x_three:
|
|
|
fd2893 |
+ vxor $out0,$in3,$twk0
|
|
|
fd2893 |
+ vxor $out1,$in4,$twk1
|
|
|
fd2893 |
+ vxor $out2,$in5,$twk2
|
|
|
fd2893 |
+ vxor $out3,$out3,$out3
|
|
|
fd2893 |
+ vxor $out4,$out4,$out4
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ bl _aesp8_xts_dec5x
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ le?vperm $out0,$out0,$out0,$leperm
|
|
|
fd2893 |
+ vmr $twk0,$twk3 # unused tweak
|
|
|
fd2893 |
+ vmr $twk1,$twk4
|
|
|
fd2893 |
+ le?vperm $out1,$out1,$out1,$leperm
|
|
|
fd2893 |
+ stvx_u $out0,$x00,$out # store output
|
|
|
fd2893 |
+ vxor $out0,$in0,$twk4
|
|
|
fd2893 |
+ le?vperm $out2,$out2,$out2,$leperm
|
|
|
fd2893 |
+ stvx_u $out1,$x10,$out
|
|
|
fd2893 |
+ stvx_u $out2,$x20,$out
|
|
|
fd2893 |
+ addi $out,$out,0x30
|
|
|
fd2893 |
+ bne Lxts_dec6x_steal
|
|
|
fd2893 |
+ b Lxts_dec6x_done
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+.align 4
|
|
|
fd2893 |
+Lxts_dec6x_two:
|
|
|
fd2893 |
+ vxor $out0,$in4,$twk0
|
|
|
fd2893 |
+ vxor $out1,$in5,$twk1
|
|
|
fd2893 |
+ vxor $out2,$out2,$out2
|
|
|
fd2893 |
+ vxor $out3,$out3,$out3
|
|
|
fd2893 |
+ vxor $out4,$out4,$out4
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ bl _aesp8_xts_dec5x
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ le?vperm $out0,$out0,$out0,$leperm
|
|
|
fd2893 |
+ vmr $twk0,$twk2 # unused tweak
|
|
|
fd2893 |
+ vmr $twk1,$twk3
|
|
|
fd2893 |
+ le?vperm $out1,$out1,$out1,$leperm
|
|
|
fd2893 |
+ stvx_u $out0,$x00,$out # store output
|
|
|
fd2893 |
+ vxor $out0,$in0,$twk3
|
|
|
fd2893 |
+ stvx_u $out1,$x10,$out
|
|
|
fd2893 |
+ addi $out,$out,0x20
|
|
|
fd2893 |
+ bne Lxts_dec6x_steal
|
|
|
fd2893 |
+ b Lxts_dec6x_done
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+.align 4
|
|
|
fd2893 |
+Lxts_dec6x_one:
|
|
|
fd2893 |
+ vxor $out0,$in5,$twk0
|
|
|
fd2893 |
+ nop
|
|
|
fd2893 |
+Loop_xts_dec1x:
|
|
|
fd2893 |
+ vncipher $out0,$out0,v24
|
|
|
fd2893 |
+ lvx v24,$x20,$key_ # round[3]
|
|
|
fd2893 |
+ addi $key_,$key_,0x20
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vncipher $out0,$out0,v25
|
|
|
fd2893 |
+ lvx v25,$x10,$key_ # round[4]
|
|
|
fd2893 |
+ bdnz Loop_xts_dec1x
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ subi r0,$taillen,1
|
|
|
fd2893 |
+ vncipher $out0,$out0,v24
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ andi. r0,r0,16
|
|
|
fd2893 |
+ cmpwi $taillen,0
|
|
|
fd2893 |
+ vncipher $out0,$out0,v25
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ sub $inp,$inp,r0
|
|
|
fd2893 |
+ vncipher $out0,$out0,v26
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ lvx_u $in0,0,$inp
|
|
|
fd2893 |
+ vncipher $out0,$out0,v27
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ addi $key_,$sp,$FRAME+15 # rewind $key_
|
|
|
fd2893 |
+ vncipher $out0,$out0,v28
|
|
|
fd2893 |
+ lvx v24,$x00,$key_ # re-pre-load round[1]
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vncipher $out0,$out0,v29
|
|
|
fd2893 |
+ lvx v25,$x10,$key_ # re-pre-load round[2]
|
|
|
fd2893 |
+ vxor $twk0,$twk0,v31
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ le?vperm $in0,$in0,$in0,$leperm
|
|
|
fd2893 |
+ vncipher $out0,$out0,v30
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ mtctr $rounds
|
|
|
fd2893 |
+ vncipherlast $out0,$out0,$twk0
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vmr $twk0,$twk1 # unused tweak
|
|
|
fd2893 |
+ vmr $twk1,$twk2
|
|
|
fd2893 |
+ le?vperm $out0,$out0,$out0,$leperm
|
|
|
fd2893 |
+ stvx_u $out0,$x00,$out # store output
|
|
|
fd2893 |
+ addi $out,$out,0x10
|
|
|
fd2893 |
+ vxor $out0,$in0,$twk2
|
|
|
fd2893 |
+ bne Lxts_dec6x_steal
|
|
|
fd2893 |
+ b Lxts_dec6x_done
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+.align 4
|
|
|
fd2893 |
+Lxts_dec6x_zero:
|
|
|
fd2893 |
+ cmpwi $taillen,0
|
|
|
fd2893 |
+ beq Lxts_dec6x_done
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ lvx_u $in0,0,$inp
|
|
|
fd2893 |
+ le?vperm $in0,$in0,$in0,$leperm
|
|
|
fd2893 |
+ vxor $out0,$in0,$twk1
|
|
|
fd2893 |
+Lxts_dec6x_steal:
|
|
|
fd2893 |
+ vncipher $out0,$out0,v24
|
|
|
fd2893 |
+ lvx v24,$x20,$key_ # round[3]
|
|
|
fd2893 |
+ addi $key_,$key_,0x20
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vncipher $out0,$out0,v25
|
|
|
fd2893 |
+ lvx v25,$x10,$key_ # round[4]
|
|
|
fd2893 |
+ bdnz Lxts_dec6x_steal
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ add $inp,$inp,$taillen
|
|
|
fd2893 |
+ vncipher $out0,$out0,v24
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ cmpwi $taillen,0
|
|
|
fd2893 |
+ vncipher $out0,$out0,v25
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ lvx_u $in0,0,$inp
|
|
|
fd2893 |
+ vncipher $out0,$out0,v26
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ lvsr $inpperm,0,$taillen # $in5 is no more
|
|
|
fd2893 |
+ vncipher $out0,$out0,v27
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ addi $key_,$sp,$FRAME+15 # rewind $key_
|
|
|
fd2893 |
+ vncipher $out0,$out0,v28
|
|
|
fd2893 |
+ lvx v24,$x00,$key_ # re-pre-load round[1]
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vncipher $out0,$out0,v29
|
|
|
fd2893 |
+ lvx v25,$x10,$key_ # re-pre-load round[2]
|
|
|
fd2893 |
+ vxor $twk1,$twk1,v31
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ le?vperm $in0,$in0,$in0,$leperm
|
|
|
fd2893 |
+ vncipher $out0,$out0,v30
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vperm $in0,$in0,$in0,$inpperm
|
|
|
fd2893 |
+ vncipherlast $tmp,$out0,$twk1
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ le?vperm $out0,$tmp,$tmp,$leperm
|
|
|
fd2893 |
+ le?stvx_u $out0,0,$out
|
|
|
fd2893 |
+ be?stvx_u $tmp,0,$out
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vxor $out0,$out0,$out0
|
|
|
fd2893 |
+ vspltisb $out1,-1
|
|
|
fd2893 |
+ vperm $out0,$out0,$out1,$inpperm
|
|
|
fd2893 |
+ vsel $out0,$in0,$tmp,$out0
|
|
|
fd2893 |
+ vxor $out0,$out0,$twk0
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ subi r30,$out,1
|
|
|
fd2893 |
+ mtctr $taillen
|
|
|
fd2893 |
+Loop_xts_dec6x_steal:
|
|
|
fd2893 |
+ lbzu r0,1(r30)
|
|
|
fd2893 |
+ stb r0,16(r30)
|
|
|
fd2893 |
+ bdnz Loop_xts_dec6x_steal
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ li $taillen,0
|
|
|
fd2893 |
+ mtctr $rounds
|
|
|
fd2893 |
+ b Loop_xts_dec1x # one more time...
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+.align 4
|
|
|
fd2893 |
+Lxts_dec6x_done:
|
|
|
fd2893 |
+ ${UCMP}i $ivp,0
|
|
|
fd2893 |
+ beq Lxts_dec6x_ret
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vxor $tweak,$twk0,$rndkey0
|
|
|
fd2893 |
+ le?vperm $tweak,$tweak,$tweak,$leperm
|
|
|
fd2893 |
+ stvx_u $tweak,0,$ivp
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+Lxts_dec6x_ret:
|
|
|
fd2893 |
+ mtlr r11
|
|
|
fd2893 |
+ li r10,`$FRAME+15`
|
|
|
fd2893 |
+ li r11,`$FRAME+31`
|
|
|
fd2893 |
+ stvx $seven,r10,$sp # wipe copies of round keys
|
|
|
fd2893 |
+ addi r10,r10,32
|
|
|
fd2893 |
+ stvx $seven,r11,$sp
|
|
|
fd2893 |
+ addi r11,r11,32
|
|
|
fd2893 |
+ stvx $seven,r10,$sp
|
|
|
fd2893 |
+ addi r10,r10,32
|
|
|
fd2893 |
+ stvx $seven,r11,$sp
|
|
|
fd2893 |
+ addi r11,r11,32
|
|
|
fd2893 |
+ stvx $seven,r10,$sp
|
|
|
fd2893 |
+ addi r10,r10,32
|
|
|
fd2893 |
+ stvx $seven,r11,$sp
|
|
|
fd2893 |
+ addi r11,r11,32
|
|
|
fd2893 |
+ stvx $seven,r10,$sp
|
|
|
fd2893 |
+ addi r10,r10,32
|
|
|
fd2893 |
+ stvx $seven,r11,$sp
|
|
|
fd2893 |
+ addi r11,r11,32
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ mtspr 256,$vrsave
|
|
|
fd2893 |
+ lvx v20,r10,$sp # ABI says so
|
|
|
fd2893 |
+ addi r10,r10,32
|
|
|
fd2893 |
+ lvx v21,r11,$sp
|
|
|
fd2893 |
+ addi r11,r11,32
|
|
|
fd2893 |
+ lvx v22,r10,$sp
|
|
|
fd2893 |
+ addi r10,r10,32
|
|
|
fd2893 |
+ lvx v23,r11,$sp
|
|
|
fd2893 |
+ addi r11,r11,32
|
|
|
fd2893 |
+ lvx v24,r10,$sp
|
|
|
fd2893 |
+ addi r10,r10,32
|
|
|
fd2893 |
+ lvx v25,r11,$sp
|
|
|
fd2893 |
+ addi r11,r11,32
|
|
|
fd2893 |
+ lvx v26,r10,$sp
|
|
|
fd2893 |
+ addi r10,r10,32
|
|
|
fd2893 |
+ lvx v27,r11,$sp
|
|
|
fd2893 |
+ addi r11,r11,32
|
|
|
fd2893 |
+ lvx v28,r10,$sp
|
|
|
fd2893 |
+ addi r10,r10,32
|
|
|
fd2893 |
+ lvx v29,r11,$sp
|
|
|
fd2893 |
+ addi r11,r11,32
|
|
|
fd2893 |
+ lvx v30,r10,$sp
|
|
|
fd2893 |
+ lvx v31,r11,$sp
|
|
|
fd2893 |
+ $POP r26,`$FRAME+21*16+0*$SIZE_T`($sp)
|
|
|
fd2893 |
+ $POP r27,`$FRAME+21*16+1*$SIZE_T`($sp)
|
|
|
fd2893 |
+ $POP r28,`$FRAME+21*16+2*$SIZE_T`($sp)
|
|
|
fd2893 |
+ $POP r29,`$FRAME+21*16+3*$SIZE_T`($sp)
|
|
|
fd2893 |
+ $POP r30,`$FRAME+21*16+4*$SIZE_T`($sp)
|
|
|
fd2893 |
+ $POP r31,`$FRAME+21*16+5*$SIZE_T`($sp)
|
|
|
fd2893 |
+ addi $sp,$sp,`$FRAME+21*16+6*$SIZE_T`
|
|
|
fd2893 |
+ blr
|
|
|
fd2893 |
+ .long 0
|
|
|
fd2893 |
+ .byte 0,12,0x04,1,0x80,6,6,0
|
|
|
fd2893 |
+ .long 0
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+.align 5
|
|
|
fd2893 |
+_aesp8_xts_dec5x:
|
|
|
fd2893 |
+ vncipher $out0,$out0,v24
|
|
|
fd2893 |
+ vncipher $out1,$out1,v24
|
|
|
fd2893 |
+ vncipher $out2,$out2,v24
|
|
|
fd2893 |
+ vncipher $out3,$out3,v24
|
|
|
fd2893 |
+ vncipher $out4,$out4,v24
|
|
|
fd2893 |
+ lvx v24,$x20,$key_ # round[3]
|
|
|
fd2893 |
+ addi $key_,$key_,0x20
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vncipher $out0,$out0,v25
|
|
|
fd2893 |
+ vncipher $out1,$out1,v25
|
|
|
fd2893 |
+ vncipher $out2,$out2,v25
|
|
|
fd2893 |
+ vncipher $out3,$out3,v25
|
|
|
fd2893 |
+ vncipher $out4,$out4,v25
|
|
|
fd2893 |
+ lvx v25,$x10,$key_ # round[4]
|
|
|
fd2893 |
+ bdnz _aesp8_xts_dec5x
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ subi r0,$taillen,1
|
|
|
fd2893 |
+ vncipher $out0,$out0,v24
|
|
|
fd2893 |
+ vncipher $out1,$out1,v24
|
|
|
fd2893 |
+ vncipher $out2,$out2,v24
|
|
|
fd2893 |
+ vncipher $out3,$out3,v24
|
|
|
fd2893 |
+ vncipher $out4,$out4,v24
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ andi. r0,r0,16
|
|
|
fd2893 |
+ cmpwi $taillen,0
|
|
|
fd2893 |
+ vncipher $out0,$out0,v25
|
|
|
fd2893 |
+ vncipher $out1,$out1,v25
|
|
|
fd2893 |
+ vncipher $out2,$out2,v25
|
|
|
fd2893 |
+ vncipher $out3,$out3,v25
|
|
|
fd2893 |
+ vncipher $out4,$out4,v25
|
|
|
fd2893 |
+ vxor $twk0,$twk0,v31
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ sub $inp,$inp,r0
|
|
|
fd2893 |
+ vncipher $out0,$out0,v26
|
|
|
fd2893 |
+ vncipher $out1,$out1,v26
|
|
|
fd2893 |
+ vncipher $out2,$out2,v26
|
|
|
fd2893 |
+ vncipher $out3,$out3,v26
|
|
|
fd2893 |
+ vncipher $out4,$out4,v26
|
|
|
fd2893 |
+ vxor $in1,$twk1,v31
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vncipher $out0,$out0,v27
|
|
|
fd2893 |
+ lvx_u $in0,0,$inp
|
|
|
fd2893 |
+ vncipher $out1,$out1,v27
|
|
|
fd2893 |
+ vncipher $out2,$out2,v27
|
|
|
fd2893 |
+ vncipher $out3,$out3,v27
|
|
|
fd2893 |
+ vncipher $out4,$out4,v27
|
|
|
fd2893 |
+ vxor $in2,$twk2,v31
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ addi $key_,$sp,$FRAME+15 # rewind $key_
|
|
|
fd2893 |
+ vncipher $out0,$out0,v28
|
|
|
fd2893 |
+ vncipher $out1,$out1,v28
|
|
|
fd2893 |
+ vncipher $out2,$out2,v28
|
|
|
fd2893 |
+ vncipher $out3,$out3,v28
|
|
|
fd2893 |
+ vncipher $out4,$out4,v28
|
|
|
fd2893 |
+ lvx v24,$x00,$key_ # re-pre-load round[1]
|
|
|
fd2893 |
+ vxor $in3,$twk3,v31
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vncipher $out0,$out0,v29
|
|
|
fd2893 |
+ le?vperm $in0,$in0,$in0,$leperm
|
|
|
fd2893 |
+ vncipher $out1,$out1,v29
|
|
|
fd2893 |
+ vncipher $out2,$out2,v29
|
|
|
fd2893 |
+ vncipher $out3,$out3,v29
|
|
|
fd2893 |
+ vncipher $out4,$out4,v29
|
|
|
fd2893 |
+ lvx v25,$x10,$key_ # re-pre-load round[2]
|
|
|
fd2893 |
+ vxor $in4,$twk4,v31
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vncipher $out0,$out0,v30
|
|
|
fd2893 |
+ vncipher $out1,$out1,v30
|
|
|
fd2893 |
+ vncipher $out2,$out2,v30
|
|
|
fd2893 |
+ vncipher $out3,$out3,v30
|
|
|
fd2893 |
+ vncipher $out4,$out4,v30
|
|
|
fd2893 |
+
|
|
|
fd2893 |
+ vncipherlast $out0,$out0,$twk0
|
|
|
fd2893 |
+ vncipherlast $out1,$out1,$in1
|
|
|
fd2893 |
+ vncipherlast $out2,$out2,$in2
|
|
|
fd2893 |
+ vncipherlast $out3,$out3,$in3
|
|
|
fd2893 |
+ vncipherlast $out4,$out4,$in4
|
|
|
fd2893 |
+ mtctr $rounds
|
|
|
fd2893 |
+ blr
|
|
|
fd2893 |
+ .long 0
|
|
|
fd2893 |
+ .byte 0,12,0x14,0,0,0,0,0
|
|
|
fd2893 |
+___
|
|
|
fd2893 |
+}} }}}
|
|
|
fd2893 |
+
|
|
|
fd2893 |
my $consts=1;
|
|
|
fd2893 |
foreach(split("\n",$code)) {
|
|
|
fd2893 |
s/\`([^\`]*)\`/eval($1)/geo;
|
|
|
fd2893 |
diff -up openssl-1.0.2k/crypto/evp/e_aes.c.ppc-update openssl-1.0.2k/crypto/evp/e_aes.c
|
|
|
fd2893 |
--- openssl-1.0.2k/crypto/evp/e_aes.c.ppc-update 2017-03-09 17:59:26.303232439 +0100
|
|
|
fd2893 |
+++ openssl-1.0.2k/crypto/evp/e_aes.c 2017-03-09 17:59:26.314232696 +0100
|
|
|
fd2893 |
@@ -172,6 +172,8 @@ void AES_xts_decrypt(const unsigned char
|
|
|
fd2893 |
# define HWAES_decrypt aes_p8_decrypt
|
|
|
fd2893 |
# define HWAES_cbc_encrypt aes_p8_cbc_encrypt
|
|
|
fd2893 |
# define HWAES_ctr32_encrypt_blocks aes_p8_ctr32_encrypt_blocks
|
|
|
fd2893 |
+# define HWAES_xts_encrypt aes_p8_xts_encrypt
|
|
|
fd2893 |
+# define HWAES_xts_decrypt aes_p8_xts_decrypt
|
|
|
fd2893 |
# endif
|
|
|
fd2893 |
|
|
|
fd2893 |
# if defined(AES_ASM) && !defined(I386_ONLY) && ( \
|
|
|
fd2893 |
@@ -911,6 +913,12 @@ void HWAES_cbc_encrypt(const unsigned ch
|
|
|
fd2893 |
void HWAES_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out,
|
|
|
fd2893 |
size_t len, const AES_KEY *key,
|
|
|
fd2893 |
const unsigned char ivec[16]);
|
|
|
fd2893 |
+void HWAES_xts_encrypt(const unsigned char *inp, unsigned char *out,
|
|
|
fd2893 |
+ size_t len, const AES_KEY *key1,
|
|
|
fd2893 |
+ const AES_KEY *key2, const unsigned char iv[16]);
|
|
|
fd2893 |
+void HWAES_xts_decrypt(const unsigned char *inp, unsigned char *out,
|
|
|
fd2893 |
+ size_t len, const AES_KEY *key1,
|
|
|
fd2893 |
+ const AES_KEY *key2, const unsigned char iv[16]);
|
|
|
fd2893 |
# endif
|
|
|
fd2893 |
|
|
|
fd2893 |
# define BLOCK_CIPHER_generic_pack(nid,keylen,flags) \
|
|
|
fd2893 |
@@ -1664,10 +1672,16 @@ static int aes_xts_init_key(EVP_CIPHER_C
|
|
|
fd2893 |
HWAES_set_encrypt_key(key, ctx->key_len * 4,
|
|
|
fd2893 |
&xctx->ks1.ks);
|
|
|
fd2893 |
xctx->xts.block1 = (block128_f) HWAES_encrypt;
|
|
|
fd2893 |
+# ifdef HWAES_xts_encrypt
|
|
|
fd2893 |
+ xctx->stream = HWAES_xts_encrypt;
|
|
|
fd2893 |
+# endif
|
|
|
fd2893 |
} else {
|
|
|
fd2893 |
HWAES_set_decrypt_key(key, ctx->key_len * 4,
|
|
|
fd2893 |
&xctx->ks1.ks);
|
|
|
fd2893 |
xctx->xts.block1 = (block128_f) HWAES_decrypt;
|
|
|
fd2893 |
+# ifdef HWAES_xts_decrypt
|
|
|
fd2893 |
+ xctx->stream = HWAES_xts_decrypt;
|
|
|
fd2893 |
+#endif
|
|
|
fd2893 |
}
|
|
|
fd2893 |
|
|
|
fd2893 |
HWAES_set_encrypt_key(key + ctx->key_len / 2,
|