Blame SOURCES/openssl-1.0.2k-ppc-update.patch

fd2893
diff -up openssl-1.0.2k/crypto/aes/asm/aesp8-ppc.pl.ppc-update openssl-1.0.2k/crypto/aes/asm/aesp8-ppc.pl
fd2893
--- openssl-1.0.2k/crypto/aes/asm/aesp8-ppc.pl.ppc-update	2017-01-26 14:22:03.000000000 +0100
fd2893
+++ openssl-1.0.2k/crypto/aes/asm/aesp8-ppc.pl	2017-04-13 09:51:40.611133165 +0200
fd2893
@@ -20,6 +20,19 @@
fd2893
 # instructions are interleaved. It's reckoned that eventual
fd2893
 # misalignment penalties at page boundaries are in average lower
fd2893
 # than additional overhead in pure AltiVec approach.
fd2893
+#
fd2893
+# May 2016
fd2893
+#
fd2893
+# Add XTS subroutine, 9x on little- and 12x improvement on big-endian
fd2893
+# systems were measured.
fd2893
+#
fd2893
+######################################################################
fd2893
+# Current large-block performance in cycles per byte processed with
fd2893
+# 128-bit key (less is better).
fd2893
+#
fd2893
+#		CBC en-/decrypt	CTR	XTS
fd2893
+# POWER8[le]	3.96/0.72	0.74	1.1
fd2893
+# POWER8[be]	3.75/0.65	0.66	1.0
fd2893
 
fd2893
 $flavour = shift;
fd2893
 
fd2893
@@ -486,6 +499,8 @@ $code.=<<___;
fd2893
 .globl	.${prefix}_cbc_encrypt
fd2893
 .align	5
fd2893
 .${prefix}_cbc_encrypt:
fd2893
+	li		$idx,15
fd2893
+	andc		$len,$len,$idx
fd2893
 	${UCMP}i	$len,16
fd2893
 	bltlr-
fd2893
 
fd2893
@@ -494,7 +509,6 @@ $code.=<<___;
fd2893
 	mfspr		$vrsave,256
fd2893
 	mtspr		256,r0
fd2893
 
fd2893
-	li		$idx,15
fd2893
 	vxor		$rndkey0,$rndkey0,$rndkey0
fd2893
 	le?vspltisb	$tmp,0x0f
fd2893
 
fd2893
@@ -1887,6 +1901,1849 @@ Lctr32_enc8x_done:
fd2893
 ___
fd2893
 }}	}}}
fd2893
 
fd2893
+#########################################################################
fd2893
+{{{	# XTS procedures						#
fd2893
+# int aes_p8_xts_[en|de]crypt(const char *inp, char *out, size_t len,	#
fd2893
+#                             const AES_KEY *key1, const AES_KEY *key2,	#
fd2893
+#                             [const] unsigned char iv[16]);		#
fd2893
+# If $key2 is NULL, then a "tweak chaining" mode is engaged, in which	#
fd2893
+# input tweak value is assumed to be encrypted already, and last tweak	#
fd2893
+# value, one suitable for consecutive call on same chunk of data, is	#
fd2893
+# written back to original buffer. In addition, in "tweak chaining"	#
fd2893
+# mode only complete input blocks are processed.			#
fd2893
+
fd2893
+my ($inp,$out,$len,$key1,$key2,$ivp,$rounds,$idx) =	map("r$_",(3..10));
fd2893
+my ($rndkey0,$rndkey1,$inout) =				map("v$_",(0..2));
fd2893
+my ($output,$inptail,$inpperm,$leperm,$keyperm) =	map("v$_",(3..7));
fd2893
+my ($tweak,$seven,$eighty7,$tmp,$tweak1) =		map("v$_",(8..12));
fd2893
+my $taillen = $key2;
fd2893
+
fd2893
+   ($inp,$idx) = ($idx,$inp);				# reassign
fd2893
+
fd2893
+$code.=<<___;
fd2893
+.globl	.${prefix}_xts_encrypt
fd2893
+.align	5
fd2893
+.${prefix}_xts_encrypt:
fd2893
+	mr		$inp,r3				# reassign
fd2893
+	li		r3,-1
fd2893
+	${UCMP}i	$len,16
fd2893
+	bltlr-
fd2893
+
fd2893
+	lis		r0,0xfff0
fd2893
+	mfspr		r12,256				# save vrsave
fd2893
+	li		r11,0
fd2893
+	mtspr		256,r0
fd2893
+
fd2893
+	vspltisb	$seven,0x07			# 0x070707..07
fd2893
+	le?lvsl		$leperm,r11,r11
fd2893
+	le?vspltisb	$tmp,0x0f
fd2893
+	le?vxor		$leperm,$leperm,$seven
fd2893
+
fd2893
+	li		$idx,15
fd2893
+	lvx		$tweak,0,$ivp			# load [unaligned] iv
fd2893
+	lvsl		$inpperm,0,$ivp
fd2893
+	lvx		$inptail,$idx,$ivp
fd2893
+	le?vxor		$inpperm,$inpperm,$tmp
fd2893
+	vperm		$tweak,$tweak,$inptail,$inpperm
fd2893
+
fd2893
+	neg		r11,$inp
fd2893
+	lvsr		$inpperm,0,r11			# prepare for unaligned load
fd2893
+	lvx		$inout,0,$inp
fd2893
+	addi		$inp,$inp,15			# 15 is not typo
fd2893
+	le?vxor		$inpperm,$inpperm,$tmp
fd2893
+
fd2893
+	${UCMP}i	$key2,0				# key2==NULL?
fd2893
+	beq		Lxts_enc_no_key2
fd2893
+
fd2893
+	?lvsl		$keyperm,0,$key2		# prepare for unaligned key
fd2893
+	lwz		$rounds,240($key2)
fd2893
+	srwi		$rounds,$rounds,1
fd2893
+	subi		$rounds,$rounds,1
fd2893
+	li		$idx,16
fd2893
+
fd2893
+	lvx		$rndkey0,0,$key2
fd2893
+	lvx		$rndkey1,$idx,$key2
fd2893
+	addi		$idx,$idx,16
fd2893
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
fd2893
+	vxor		$tweak,$tweak,$rndkey0
fd2893
+	lvx		$rndkey0,$idx,$key2
fd2893
+	addi		$idx,$idx,16
fd2893
+	mtctr		$rounds
fd2893
+
fd2893
+Ltweak_xts_enc:
fd2893
+	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
fd2893
+	vcipher		$tweak,$tweak,$rndkey1
fd2893
+	lvx		$rndkey1,$idx,$key2
fd2893
+	addi		$idx,$idx,16
fd2893
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
fd2893
+	vcipher		$tweak,$tweak,$rndkey0
fd2893
+	lvx		$rndkey0,$idx,$key2
fd2893
+	addi		$idx,$idx,16
fd2893
+	bdnz		Ltweak_xts_enc
fd2893
+
fd2893
+	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
fd2893
+	vcipher		$tweak,$tweak,$rndkey1
fd2893
+	lvx		$rndkey1,$idx,$key2
fd2893
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
fd2893
+	vcipherlast	$tweak,$tweak,$rndkey0
fd2893
+
fd2893
+	li		$ivp,0				# don't chain the tweak
fd2893
+	b		Lxts_enc
fd2893
+
fd2893
+Lxts_enc_no_key2:
fd2893
+	li		$idx,-16
fd2893
+	and		$len,$len,$idx			# in "tweak chaining"
fd2893
+							# mode only complete
fd2893
+							# blocks are processed
fd2893
+Lxts_enc:
fd2893
+	lvx		$inptail,0,$inp
fd2893
+	addi		$inp,$inp,16
fd2893
+
fd2893
+	?lvsl		$keyperm,0,$key1		# prepare for unaligned key
fd2893
+	lwz		$rounds,240($key1)
fd2893
+	srwi		$rounds,$rounds,1
fd2893
+	subi		$rounds,$rounds,1
fd2893
+	li		$idx,16
fd2893
+
fd2893
+	vslb		$eighty7,$seven,$seven		# 0x808080..80
fd2893
+	vor		$eighty7,$eighty7,$seven	# 0x878787..87
fd2893
+	vspltisb	$tmp,1				# 0x010101..01
fd2893
+	vsldoi		$eighty7,$eighty7,$tmp,15	# 0x870101..01
fd2893
+
fd2893
+	${UCMP}i	$len,96
fd2893
+	bge		_aesp8_xts_encrypt6x
fd2893
+
fd2893
+	andi.		$taillen,$len,15
fd2893
+	subic		r0,$len,32
fd2893
+	subi		$taillen,$taillen,16
fd2893
+	subfe		r0,r0,r0
fd2893
+	and		r0,r0,$taillen
fd2893
+	add		$inp,$inp,r0
fd2893
+
fd2893
+	lvx		$rndkey0,0,$key1
fd2893
+	lvx		$rndkey1,$idx,$key1
fd2893
+	addi		$idx,$idx,16
fd2893
+	vperm		$inout,$inout,$inptail,$inpperm
fd2893
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
fd2893
+	vxor		$inout,$inout,$tweak
fd2893
+	vxor		$inout,$inout,$rndkey0
fd2893
+	lvx		$rndkey0,$idx,$key1
fd2893
+	addi		$idx,$idx,16
fd2893
+	mtctr		$rounds
fd2893
+	b		Loop_xts_enc
fd2893
+
fd2893
+.align	5
fd2893
+Loop_xts_enc:
fd2893
+	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
fd2893
+	vcipher		$inout,$inout,$rndkey1
fd2893
+	lvx		$rndkey1,$idx,$key1
fd2893
+	addi		$idx,$idx,16
fd2893
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
fd2893
+	vcipher		$inout,$inout,$rndkey0
fd2893
+	lvx		$rndkey0,$idx,$key1
fd2893
+	addi		$idx,$idx,16
fd2893
+	bdnz		Loop_xts_enc
fd2893
+
fd2893
+	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
fd2893
+	vcipher		$inout,$inout,$rndkey1
fd2893
+	lvx		$rndkey1,$idx,$key1
fd2893
+	li		$idx,16
fd2893
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
fd2893
+	vxor		$rndkey0,$rndkey0,$tweak
fd2893
+	vcipherlast	$output,$inout,$rndkey0
fd2893
+
fd2893
+	le?vperm	$tmp,$output,$output,$leperm
fd2893
+	be?nop
fd2893
+	le?stvx_u	$tmp,0,$out
fd2893
+	be?stvx_u	$output,0,$out
fd2893
+	addi		$out,$out,16
fd2893
+
fd2893
+	subic.		$len,$len,16
fd2893
+	beq		Lxts_enc_done
fd2893
+
fd2893
+	vmr		$inout,$inptail
fd2893
+	lvx		$inptail,0,$inp
fd2893
+	addi		$inp,$inp,16
fd2893
+	lvx		$rndkey0,0,$key1
fd2893
+	lvx		$rndkey1,$idx,$key1
fd2893
+	addi		$idx,$idx,16
fd2893
+
fd2893
+	subic		r0,$len,32
fd2893
+	subfe		r0,r0,r0
fd2893
+	and		r0,r0,$taillen
fd2893
+	add		$inp,$inp,r0
fd2893
+
fd2893
+	vsrab		$tmp,$tweak,$seven		# next tweak value
fd2893
+	vaddubm		$tweak,$tweak,$tweak
fd2893
+	vsldoi		$tmp,$tmp,$tmp,15
fd2893
+	vand		$tmp,$tmp,$eighty7
fd2893
+	vxor		$tweak,$tweak,$tmp
fd2893
+
fd2893
+	vperm		$inout,$inout,$inptail,$inpperm
fd2893
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
fd2893
+	vxor		$inout,$inout,$tweak
fd2893
+	vxor		$output,$output,$rndkey0	# just in case $len<16
fd2893
+	vxor		$inout,$inout,$rndkey0
fd2893
+	lvx		$rndkey0,$idx,$key1
fd2893
+	addi		$idx,$idx,16
fd2893
+
fd2893
+	mtctr		$rounds
fd2893
+	${UCMP}i	$len,16
fd2893
+	bge		Loop_xts_enc
fd2893
+
fd2893
+	vxor		$output,$output,$tweak
fd2893
+	lvsr		$inpperm,0,$len			# $inpperm is no longer needed
fd2893
+	vxor		$inptail,$inptail,$inptail	# $inptail is no longer needed
fd2893
+	vspltisb	$tmp,-1
fd2893
+	vperm		$inptail,$inptail,$tmp,$inpperm
fd2893
+	vsel		$inout,$inout,$output,$inptail
fd2893
+
fd2893
+	subi		r11,$out,17
fd2893
+	subi		$out,$out,16
fd2893
+	mtctr		$len
fd2893
+	li		$len,16
fd2893
+Loop_xts_enc_steal:
fd2893
+	lbzu		r0,1(r11)
fd2893
+	stb		r0,16(r11)
fd2893
+	bdnz		Loop_xts_enc_steal
fd2893
+
fd2893
+	mtctr		$rounds
fd2893
+	b		Loop_xts_enc			# one more time...
fd2893
+
fd2893
+Lxts_enc_done:
fd2893
+	${UCMP}i	$ivp,0
fd2893
+	beq		Lxts_enc_ret
fd2893
+
fd2893
+	vsrab		$tmp,$tweak,$seven		# next tweak value
fd2893
+	vaddubm		$tweak,$tweak,$tweak
fd2893
+	vsldoi		$tmp,$tmp,$tmp,15
fd2893
+	vand		$tmp,$tmp,$eighty7
fd2893
+	vxor		$tweak,$tweak,$tmp
fd2893
+
fd2893
+	le?vperm	$tweak,$tweak,$tweak,$leperm
fd2893
+	stvx_u		$tweak,0,$ivp
fd2893
+
fd2893
+Lxts_enc_ret:
fd2893
+	mtspr		256,r12				# restore vrsave
fd2893
+	li		r3,0
fd2893
+	blr
fd2893
+	.long		0
fd2893
+	.byte		0,12,0x04,0,0x80,6,6,0
fd2893
+	.long		0
fd2893
+.size	.${prefix}_xts_encrypt,.-.${prefix}_xts_encrypt
fd2893
+
fd2893
+.globl	.${prefix}_xts_decrypt
fd2893
+.align	5
fd2893
+.${prefix}_xts_decrypt:
fd2893
+	mr		$inp,r3				# reassign
fd2893
+	li		r3,-1
fd2893
+	${UCMP}i	$len,16
fd2893
+	bltlr-
fd2893
+
fd2893
+	lis		r0,0xfff8
fd2893
+	mfspr		r12,256				# save vrsave
fd2893
+	li		r11,0
fd2893
+	mtspr		256,r0
fd2893
+
fd2893
+	andi.		r0,$len,15
fd2893
+	neg		r0,r0
fd2893
+	andi.		r0,r0,16
fd2893
+	sub		$len,$len,r0
fd2893
+
fd2893
+	vspltisb	$seven,0x07			# 0x070707..07
fd2893
+	le?lvsl		$leperm,r11,r11
fd2893
+	le?vspltisb	$tmp,0x0f
fd2893
+	le?vxor		$leperm,$leperm,$seven
fd2893
+
fd2893
+	li		$idx,15
fd2893
+	lvx		$tweak,0,$ivp			# load [unaligned] iv
fd2893
+	lvsl		$inpperm,0,$ivp
fd2893
+	lvx		$inptail,$idx,$ivp
fd2893
+	le?vxor		$inpperm,$inpperm,$tmp
fd2893
+	vperm		$tweak,$tweak,$inptail,$inpperm
fd2893
+
fd2893
+	neg		r11,$inp
fd2893
+	lvsr		$inpperm,0,r11			# prepare for unaligned load
fd2893
+	lvx		$inout,0,$inp
fd2893
+	addi		$inp,$inp,15			# 15 is not typo
fd2893
+	le?vxor		$inpperm,$inpperm,$tmp
fd2893
+
fd2893
+	${UCMP}i	$key2,0				# key2==NULL?
fd2893
+	beq		Lxts_dec_no_key2
fd2893
+
fd2893
+	?lvsl		$keyperm,0,$key2		# prepare for unaligned key
fd2893
+	lwz		$rounds,240($key2)
fd2893
+	srwi		$rounds,$rounds,1
fd2893
+	subi		$rounds,$rounds,1
fd2893
+	li		$idx,16
fd2893
+
fd2893
+	lvx		$rndkey0,0,$key2
fd2893
+	lvx		$rndkey1,$idx,$key2
fd2893
+	addi		$idx,$idx,16
fd2893
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
fd2893
+	vxor		$tweak,$tweak,$rndkey0
fd2893
+	lvx		$rndkey0,$idx,$key2
fd2893
+	addi		$idx,$idx,16
fd2893
+	mtctr		$rounds
fd2893
+
fd2893
+Ltweak_xts_dec:
fd2893
+	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
fd2893
+	vcipher		$tweak,$tweak,$rndkey1
fd2893
+	lvx		$rndkey1,$idx,$key2
fd2893
+	addi		$idx,$idx,16
fd2893
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
fd2893
+	vcipher		$tweak,$tweak,$rndkey0
fd2893
+	lvx		$rndkey0,$idx,$key2
fd2893
+	addi		$idx,$idx,16
fd2893
+	bdnz		Ltweak_xts_dec
fd2893
+
fd2893
+	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
fd2893
+	vcipher		$tweak,$tweak,$rndkey1
fd2893
+	lvx		$rndkey1,$idx,$key2
fd2893
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
fd2893
+	vcipherlast	$tweak,$tweak,$rndkey0
fd2893
+
fd2893
+	li		$ivp,0				# don't chain the tweak
fd2893
+	b		Lxts_dec
fd2893
+
fd2893
+Lxts_dec_no_key2:
fd2893
+	neg		$idx,$len
fd2893
+	andi.		$idx,$idx,15
fd2893
+	add		$len,$len,$idx			# in "tweak chaining"
fd2893
+							# mode only complete
fd2893
+							# blocks are processed
fd2893
+Lxts_dec:
fd2893
+	lvx		$inptail,0,$inp
fd2893
+	addi		$inp,$inp,16
fd2893
+
fd2893
+	?lvsl		$keyperm,0,$key1		# prepare for unaligned key
fd2893
+	lwz		$rounds,240($key1)
fd2893
+	srwi		$rounds,$rounds,1
fd2893
+	subi		$rounds,$rounds,1
fd2893
+	li		$idx,16
fd2893
+
fd2893
+	vslb		$eighty7,$seven,$seven		# 0x808080..80
fd2893
+	vor		$eighty7,$eighty7,$seven	# 0x878787..87
fd2893
+	vspltisb	$tmp,1				# 0x010101..01
fd2893
+	vsldoi		$eighty7,$eighty7,$tmp,15	# 0x870101..01
fd2893
+
fd2893
+	${UCMP}i	$len,96
fd2893
+	bge		_aesp8_xts_decrypt6x
fd2893
+
fd2893
+	lvx		$rndkey0,0,$key1
fd2893
+	lvx		$rndkey1,$idx,$key1
fd2893
+	addi		$idx,$idx,16
fd2893
+	vperm		$inout,$inout,$inptail,$inpperm
fd2893
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
fd2893
+	vxor		$inout,$inout,$tweak
fd2893
+	vxor		$inout,$inout,$rndkey0
fd2893
+	lvx		$rndkey0,$idx,$key1
fd2893
+	addi		$idx,$idx,16
fd2893
+	mtctr		$rounds
fd2893
+
fd2893
+	${UCMP}i	$len,16
fd2893
+	blt		Ltail_xts_dec
fd2893
+	be?b		Loop_xts_dec
fd2893
+
fd2893
+.align	5
fd2893
+Loop_xts_dec:
fd2893
+	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
fd2893
+	vncipher	$inout,$inout,$rndkey1
fd2893
+	lvx		$rndkey1,$idx,$key1
fd2893
+	addi		$idx,$idx,16
fd2893
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
fd2893
+	vncipher	$inout,$inout,$rndkey0
fd2893
+	lvx		$rndkey0,$idx,$key1
fd2893
+	addi		$idx,$idx,16
fd2893
+	bdnz		Loop_xts_dec
fd2893
+
fd2893
+	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
fd2893
+	vncipher	$inout,$inout,$rndkey1
fd2893
+	lvx		$rndkey1,$idx,$key1
fd2893
+	li		$idx,16
fd2893
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
fd2893
+	vxor		$rndkey0,$rndkey0,$tweak
fd2893
+	vncipherlast	$output,$inout,$rndkey0
fd2893
+
fd2893
+	le?vperm	$tmp,$output,$output,$leperm
fd2893
+	be?nop
fd2893
+	le?stvx_u	$tmp,0,$out
fd2893
+	be?stvx_u	$output,0,$out
fd2893
+	addi		$out,$out,16
fd2893
+
fd2893
+	subic.		$len,$len,16
fd2893
+	beq		Lxts_dec_done
fd2893
+
fd2893
+	vmr		$inout,$inptail
fd2893
+	lvx		$inptail,0,$inp
fd2893
+	addi		$inp,$inp,16
fd2893
+	lvx		$rndkey0,0,$key1
fd2893
+	lvx		$rndkey1,$idx,$key1
fd2893
+	addi		$idx,$idx,16
fd2893
+
fd2893
+	vsrab		$tmp,$tweak,$seven		# next tweak value
fd2893
+	vaddubm		$tweak,$tweak,$tweak
fd2893
+	vsldoi		$tmp,$tmp,$tmp,15
fd2893
+	vand		$tmp,$tmp,$eighty7
fd2893
+	vxor		$tweak,$tweak,$tmp
fd2893
+
fd2893
+	vperm		$inout,$inout,$inptail,$inpperm
fd2893
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
fd2893
+	vxor		$inout,$inout,$tweak
fd2893
+	vxor		$inout,$inout,$rndkey0
fd2893
+	lvx		$rndkey0,$idx,$key1
fd2893
+	addi		$idx,$idx,16
fd2893
+
fd2893
+	mtctr		$rounds
fd2893
+	${UCMP}i	$len,16
fd2893
+	bge		Loop_xts_dec
fd2893
+
fd2893
+Ltail_xts_dec:
fd2893
+	vsrab		$tmp,$tweak,$seven		# next tweak value
fd2893
+	vaddubm		$tweak1,$tweak,$tweak
fd2893
+	vsldoi		$tmp,$tmp,$tmp,15
fd2893
+	vand		$tmp,$tmp,$eighty7
fd2893
+	vxor		$tweak1,$tweak1,$tmp
fd2893
+
fd2893
+	subi		$inp,$inp,16
fd2893
+	add		$inp,$inp,$len
fd2893
+
fd2893
+	vxor		$inout,$inout,$tweak		# :-(
fd2893
+	vxor		$inout,$inout,$tweak1		# :-)
fd2893
+
fd2893
+Loop_xts_dec_short:
fd2893
+	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
fd2893
+	vncipher	$inout,$inout,$rndkey1
fd2893
+	lvx		$rndkey1,$idx,$key1
fd2893
+	addi		$idx,$idx,16
fd2893
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
fd2893
+	vncipher	$inout,$inout,$rndkey0
fd2893
+	lvx		$rndkey0,$idx,$key1
fd2893
+	addi		$idx,$idx,16
fd2893
+	bdnz		Loop_xts_dec_short
fd2893
+
fd2893
+	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
fd2893
+	vncipher	$inout,$inout,$rndkey1
fd2893
+	lvx		$rndkey1,$idx,$key1
fd2893
+	li		$idx,16
fd2893
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
fd2893
+	vxor		$rndkey0,$rndkey0,$tweak1
fd2893
+	vncipherlast	$output,$inout,$rndkey0
fd2893
+
fd2893
+	le?vperm	$tmp,$output,$output,$leperm
fd2893
+	be?nop
fd2893
+	le?stvx_u	$tmp,0,$out
fd2893
+	be?stvx_u	$output,0,$out
fd2893
+
fd2893
+	vmr		$inout,$inptail
fd2893
+	lvx		$inptail,0,$inp
fd2893
+	#addi		$inp,$inp,16
fd2893
+	lvx		$rndkey0,0,$key1
fd2893
+	lvx		$rndkey1,$idx,$key1
fd2893
+	addi		$idx,$idx,16
fd2893
+	vperm		$inout,$inout,$inptail,$inpperm
fd2893
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
fd2893
+
fd2893
+	lvsr		$inpperm,0,$len			# $inpperm is no longer needed
fd2893
+	vxor		$inptail,$inptail,$inptail	# $inptail is no longer needed
fd2893
+	vspltisb	$tmp,-1
fd2893
+	vperm		$inptail,$inptail,$tmp,$inpperm
fd2893
+	vsel		$inout,$inout,$output,$inptail
fd2893
+
fd2893
+	vxor		$rndkey0,$rndkey0,$tweak
fd2893
+	vxor		$inout,$inout,$rndkey0
fd2893
+	lvx		$rndkey0,$idx,$key1
fd2893
+	addi		$idx,$idx,16
fd2893
+
fd2893
+	subi		r11,$out,1
fd2893
+	mtctr		$len
fd2893
+	li		$len,16
fd2893
+Loop_xts_dec_steal:
fd2893
+	lbzu		r0,1(r11)
fd2893
+	stb		r0,16(r11)
fd2893
+	bdnz		Loop_xts_dec_steal
fd2893
+
fd2893
+	mtctr		$rounds
fd2893
+	b		Loop_xts_dec			# one more time...
fd2893
+
fd2893
+Lxts_dec_done:
fd2893
+	${UCMP}i	$ivp,0
fd2893
+	beq		Lxts_dec_ret
fd2893
+
fd2893
+	vsrab		$tmp,$tweak,$seven		# next tweak value
fd2893
+	vaddubm		$tweak,$tweak,$tweak
fd2893
+	vsldoi		$tmp,$tmp,$tmp,15
fd2893
+	vand		$tmp,$tmp,$eighty7
fd2893
+	vxor		$tweak,$tweak,$tmp
fd2893
+
fd2893
+	le?vperm	$tweak,$tweak,$tweak,$leperm
fd2893
+	stvx_u		$tweak,0,$ivp
fd2893
+
fd2893
+Lxts_dec_ret:
fd2893
+	mtspr		256,r12				# restore vrsave
fd2893
+	li		r3,0
fd2893
+	blr
fd2893
+	.long		0
fd2893
+	.byte		0,12,0x04,0,0x80,6,6,0
fd2893
+	.long		0
fd2893
+.size	.${prefix}_xts_decrypt,.-.${prefix}_xts_decrypt
fd2893
+___
fd2893
+#########################################################################
fd2893
+{{	# Optimized XTS procedures					#
fd2893
+my $key_=$key2;
fd2893
+my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,3,26..31));
fd2893
+    $x00=0 if ($flavour =~ /osx/);
fd2893
+my ($in0,  $in1,  $in2,  $in3,  $in4,  $in5 )=map("v$_",(0..5));
fd2893
+my ($out0, $out1, $out2, $out3, $out4, $out5)=map("v$_",(7,12..16));
fd2893
+my ($twk0, $twk1, $twk2, $twk3, $twk4, $twk5)=map("v$_",(17..22));
fd2893
+my $rndkey0="v23";	# v24-v25 rotating buffer for first found keys
fd2893
+			# v26-v31 last 6 round keys
fd2893
+my ($keyperm)=($out0);	# aliases with "caller", redundant assignment
fd2893
+my $taillen=$x70;
fd2893
+
fd2893
+$code.=<<___;
fd2893
+.align	5
fd2893
+_aesp8_xts_encrypt6x:
fd2893
+	$STU		$sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
fd2893
+	mflr		r11
fd2893
+	li		r7,`$FRAME+8*16+15`
fd2893
+	li		r3,`$FRAME+8*16+31`
fd2893
+	$PUSH		r11,`$FRAME+21*16+6*$SIZE_T+$LRSAVE`($sp)
fd2893
+	stvx		v20,r7,$sp		# ABI says so
fd2893
+	addi		r7,r7,32
fd2893
+	stvx		v21,r3,$sp
fd2893
+	addi		r3,r3,32
fd2893
+	stvx		v22,r7,$sp
fd2893
+	addi		r7,r7,32
fd2893
+	stvx		v23,r3,$sp
fd2893
+	addi		r3,r3,32
fd2893
+	stvx		v24,r7,$sp
fd2893
+	addi		r7,r7,32
fd2893
+	stvx		v25,r3,$sp
fd2893
+	addi		r3,r3,32
fd2893
+	stvx		v26,r7,$sp
fd2893
+	addi		r7,r7,32
fd2893
+	stvx		v27,r3,$sp
fd2893
+	addi		r3,r3,32
fd2893
+	stvx		v28,r7,$sp
fd2893
+	addi		r7,r7,32
fd2893
+	stvx		v29,r3,$sp
fd2893
+	addi		r3,r3,32
fd2893
+	stvx		v30,r7,$sp
fd2893
+	stvx		v31,r3,$sp
fd2893
+	li		r0,-1
fd2893
+	stw		$vrsave,`$FRAME+21*16-4`($sp)	# save vrsave
fd2893
+	li		$x10,0x10
fd2893
+	$PUSH		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
fd2893
+	li		$x20,0x20
fd2893
+	$PUSH		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
fd2893
+	li		$x30,0x30
fd2893
+	$PUSH		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
fd2893
+	li		$x40,0x40
fd2893
+	$PUSH		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
fd2893
+	li		$x50,0x50
fd2893
+	$PUSH		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
fd2893
+	li		$x60,0x60
fd2893
+	$PUSH		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
fd2893
+	li		$x70,0x70
fd2893
+	mtspr		256,r0
fd2893
+
fd2893
+	subi		$rounds,$rounds,3	# -4 in total
fd2893
+
fd2893
+	lvx		$rndkey0,$x00,$key1	# load key schedule
fd2893
+	lvx		v30,$x10,$key1
fd2893
+	addi		$key1,$key1,0x20
fd2893
+	lvx		v31,$x00,$key1
fd2893
+	?vperm		$rndkey0,$rndkey0,v30,$keyperm
fd2893
+	addi		$key_,$sp,$FRAME+15
fd2893
+	mtctr		$rounds
fd2893
+
fd2893
+Load_xts_enc_key:
fd2893
+	?vperm		v24,v30,v31,$keyperm
fd2893
+	lvx		v30,$x10,$key1
fd2893
+	addi		$key1,$key1,0x20
fd2893
+	stvx		v24,$x00,$key_		# off-load round[1]
fd2893
+	?vperm		v25,v31,v30,$keyperm
fd2893
+	lvx		v31,$x00,$key1
fd2893
+	stvx		v25,$x10,$key_		# off-load round[2]
fd2893
+	addi		$key_,$key_,0x20
fd2893
+	bdnz		Load_xts_enc_key
fd2893
+
fd2893
+	lvx		v26,$x10,$key1
fd2893
+	?vperm		v24,v30,v31,$keyperm
fd2893
+	lvx		v27,$x20,$key1
fd2893
+	stvx		v24,$x00,$key_		# off-load round[3]
fd2893
+	?vperm		v25,v31,v26,$keyperm
fd2893
+	lvx		v28,$x30,$key1
fd2893
+	stvx		v25,$x10,$key_		# off-load round[4]
fd2893
+	addi		$key_,$sp,$FRAME+15	# rewind $key_
fd2893
+	?vperm		v26,v26,v27,$keyperm
fd2893
+	lvx		v29,$x40,$key1
fd2893
+	?vperm		v27,v27,v28,$keyperm
fd2893
+	lvx		v30,$x50,$key1
fd2893
+	?vperm		v28,v28,v29,$keyperm
fd2893
+	lvx		v31,$x60,$key1
fd2893
+	?vperm		v29,v29,v30,$keyperm
fd2893
+	lvx		$twk5,$x70,$key1	# borrow $twk5
fd2893
+	?vperm		v30,v30,v31,$keyperm
fd2893
+	lvx		v24,$x00,$key_		# pre-load round[1]
fd2893
+	?vperm		v31,v31,$twk5,$keyperm
fd2893
+	lvx		v25,$x10,$key_		# pre-load round[2]
fd2893
+
fd2893
+	 vperm		$in0,$inout,$inptail,$inpperm
fd2893
+	 subi		$inp,$inp,31		# undo "caller"
fd2893
+	vxor		$twk0,$tweak,$rndkey0
fd2893
+	vsrab		$tmp,$tweak,$seven	# next tweak value
fd2893
+	vaddubm		$tweak,$tweak,$tweak
fd2893
+	vsldoi		$tmp,$tmp,$tmp,15
fd2893
+	vand		$tmp,$tmp,$eighty7
fd2893
+	 vxor		$out0,$in0,$twk0
fd2893
+	vxor		$tweak,$tweak,$tmp
fd2893
+
fd2893
+	 lvx_u		$in1,$x10,$inp
fd2893
+	vxor		$twk1,$tweak,$rndkey0
fd2893
+	vsrab		$tmp,$tweak,$seven	# next tweak value
fd2893
+	vaddubm		$tweak,$tweak,$tweak
fd2893
+	vsldoi		$tmp,$tmp,$tmp,15
fd2893
+	 le?vperm	$in1,$in1,$in1,$leperm
fd2893
+	vand		$tmp,$tmp,$eighty7
fd2893
+	 vxor		$out1,$in1,$twk1
fd2893
+	vxor		$tweak,$tweak,$tmp
fd2893
+
fd2893
+	 lvx_u		$in2,$x20,$inp
fd2893
+	 andi.		$taillen,$len,15
fd2893
+	vxor		$twk2,$tweak,$rndkey0
fd2893
+	vsrab		$tmp,$tweak,$seven	# next tweak value
fd2893
+	vaddubm		$tweak,$tweak,$tweak
fd2893
+	vsldoi		$tmp,$tmp,$tmp,15
fd2893
+	 le?vperm	$in2,$in2,$in2,$leperm
fd2893
+	vand		$tmp,$tmp,$eighty7
fd2893
+	 vxor		$out2,$in2,$twk2
fd2893
+	vxor		$tweak,$tweak,$tmp
fd2893
+
fd2893
+	 lvx_u		$in3,$x30,$inp
fd2893
+	 sub		$len,$len,$taillen
fd2893
+	vxor		$twk3,$tweak,$rndkey0
fd2893
+	vsrab		$tmp,$tweak,$seven	# next tweak value
fd2893
+	vaddubm		$tweak,$tweak,$tweak
fd2893
+	vsldoi		$tmp,$tmp,$tmp,15
fd2893
+	 le?vperm	$in3,$in3,$in3,$leperm
fd2893
+	vand		$tmp,$tmp,$eighty7
fd2893
+	 vxor		$out3,$in3,$twk3
fd2893
+	vxor		$tweak,$tweak,$tmp
fd2893
+
fd2893
+	 lvx_u		$in4,$x40,$inp
fd2893
+	 subi		$len,$len,0x60
fd2893
+	vxor		$twk4,$tweak,$rndkey0
fd2893
+	vsrab		$tmp,$tweak,$seven	# next tweak value
fd2893
+	vaddubm		$tweak,$tweak,$tweak
fd2893
+	vsldoi		$tmp,$tmp,$tmp,15
fd2893
+	 le?vperm	$in4,$in4,$in4,$leperm
fd2893
+	vand		$tmp,$tmp,$eighty7
fd2893
+	 vxor		$out4,$in4,$twk4
fd2893
+	vxor		$tweak,$tweak,$tmp
fd2893
+
fd2893
+	 lvx_u		$in5,$x50,$inp
fd2893
+	 addi		$inp,$inp,0x60
fd2893
+	vxor		$twk5,$tweak,$rndkey0
fd2893
+	vsrab		$tmp,$tweak,$seven	# next tweak value
fd2893
+	vaddubm		$tweak,$tweak,$tweak
fd2893
+	vsldoi		$tmp,$tmp,$tmp,15
fd2893
+	 le?vperm	$in5,$in5,$in5,$leperm
fd2893
+	vand		$tmp,$tmp,$eighty7
fd2893
+	 vxor		$out5,$in5,$twk5
fd2893
+	vxor		$tweak,$tweak,$tmp
fd2893
+
fd2893
+	vxor		v31,v31,$rndkey0
fd2893
+	mtctr		$rounds
fd2893
+	b		Loop_xts_enc6x
fd2893
+
fd2893
+.align	5
fd2893
+Loop_xts_enc6x:
fd2893
+	vcipher		$out0,$out0,v24
fd2893
+	vcipher		$out1,$out1,v24
fd2893
+	vcipher		$out2,$out2,v24
fd2893
+	vcipher		$out3,$out3,v24
fd2893
+	vcipher		$out4,$out4,v24
fd2893
+	vcipher		$out5,$out5,v24
fd2893
+	lvx		v24,$x20,$key_		# round[3]
fd2893
+	addi		$key_,$key_,0x20
fd2893
+
fd2893
+	vcipher		$out0,$out0,v25
fd2893
+	vcipher		$out1,$out1,v25
fd2893
+	vcipher		$out2,$out2,v25
fd2893
+	vcipher		$out3,$out3,v25
fd2893
+	vcipher		$out4,$out4,v25
fd2893
+	vcipher		$out5,$out5,v25
fd2893
+	lvx		v25,$x10,$key_		# round[4]
fd2893
+	bdnz		Loop_xts_enc6x
fd2893
+
fd2893
+	subic		$len,$len,96		# $len-=96
fd2893
+	 vxor		$in0,$twk0,v31		# xor with last round key
fd2893
+	vcipher		$out0,$out0,v24
fd2893
+	vcipher		$out1,$out1,v24
fd2893
+	 vsrab		$tmp,$tweak,$seven	# next tweak value
fd2893
+	 vxor		$twk0,$tweak,$rndkey0
fd2893
+	 vaddubm	$tweak,$tweak,$tweak
fd2893
+	vcipher		$out2,$out2,v24
fd2893
+	vcipher		$out3,$out3,v24
fd2893
+	 vsldoi		$tmp,$tmp,$tmp,15
fd2893
+	vcipher		$out4,$out4,v24
fd2893
+	vcipher		$out5,$out5,v24
fd2893
+
fd2893
+	subfe.		r0,r0,r0		# borrow?-1:0
fd2893
+	 vand		$tmp,$tmp,$eighty7
fd2893
+	vcipher		$out0,$out0,v25
fd2893
+	vcipher		$out1,$out1,v25
fd2893
+	 vxor		$tweak,$tweak,$tmp
fd2893
+	vcipher		$out2,$out2,v25
fd2893
+	vcipher		$out3,$out3,v25
fd2893
+	 vxor		$in1,$twk1,v31
fd2893
+	 vsrab		$tmp,$tweak,$seven	# next tweak value
fd2893
+	 vxor		$twk1,$tweak,$rndkey0
fd2893
+	vcipher		$out4,$out4,v25
fd2893
+	vcipher		$out5,$out5,v25
fd2893
+
fd2893
+	and		r0,r0,$len
fd2893
+	 vaddubm	$tweak,$tweak,$tweak
fd2893
+	 vsldoi		$tmp,$tmp,$tmp,15
fd2893
+	vcipher		$out0,$out0,v26
fd2893
+	vcipher		$out1,$out1,v26
fd2893
+	 vand		$tmp,$tmp,$eighty7
fd2893
+	vcipher		$out2,$out2,v26
fd2893
+	vcipher		$out3,$out3,v26
fd2893
+	 vxor		$tweak,$tweak,$tmp
fd2893
+	vcipher		$out4,$out4,v26
fd2893
+	vcipher		$out5,$out5,v26
fd2893
+
fd2893
+	add		$inp,$inp,r0		# $inp is adjusted in such
fd2893
+						# way that at exit from the
fd2893
+						# loop inX-in5 are loaded
fd2893
+						# with last "words"
fd2893
+	 vxor		$in2,$twk2,v31
fd2893
+	 vsrab		$tmp,$tweak,$seven	# next tweak value
fd2893
+	 vxor		$twk2,$tweak,$rndkey0
fd2893
+	 vaddubm	$tweak,$tweak,$tweak
fd2893
+	vcipher		$out0,$out0,v27
fd2893
+	vcipher		$out1,$out1,v27
fd2893
+	 vsldoi		$tmp,$tmp,$tmp,15
fd2893
+	vcipher		$out2,$out2,v27
fd2893
+	vcipher		$out3,$out3,v27
fd2893
+	 vand		$tmp,$tmp,$eighty7
fd2893
+	vcipher		$out4,$out4,v27
fd2893
+	vcipher		$out5,$out5,v27
fd2893
+
fd2893
+	addi		$key_,$sp,$FRAME+15	# rewind $key_
fd2893
+	 vxor		$tweak,$tweak,$tmp
fd2893
+	vcipher		$out0,$out0,v28
fd2893
+	vcipher		$out1,$out1,v28
fd2893
+	 vxor		$in3,$twk3,v31
fd2893
+	 vsrab		$tmp,$tweak,$seven	# next tweak value
fd2893
+	 vxor		$twk3,$tweak,$rndkey0
fd2893
+	vcipher		$out2,$out2,v28
fd2893
+	vcipher		$out3,$out3,v28
fd2893
+	 vaddubm	$tweak,$tweak,$tweak
fd2893
+	 vsldoi		$tmp,$tmp,$tmp,15
fd2893
+	vcipher		$out4,$out4,v28
fd2893
+	vcipher		$out5,$out5,v28
fd2893
+	lvx		v24,$x00,$key_		# re-pre-load round[1]
fd2893
+	 vand		$tmp,$tmp,$eighty7
fd2893
+
fd2893
+	vcipher		$out0,$out0,v29
fd2893
+	vcipher		$out1,$out1,v29
fd2893
+	 vxor		$tweak,$tweak,$tmp
fd2893
+	vcipher		$out2,$out2,v29
fd2893
+	vcipher		$out3,$out3,v29
fd2893
+	 vxor		$in4,$twk4,v31
fd2893
+	 vsrab		$tmp,$tweak,$seven	# next tweak value
fd2893
+	 vxor		$twk4,$tweak,$rndkey0
fd2893
+	vcipher		$out4,$out4,v29
fd2893
+	vcipher		$out5,$out5,v29
fd2893
+	lvx		v25,$x10,$key_		# re-pre-load round[2]
fd2893
+	 vaddubm	$tweak,$tweak,$tweak
fd2893
+	 vsldoi		$tmp,$tmp,$tmp,15
fd2893
+
fd2893
+	vcipher		$out0,$out0,v30
fd2893
+	vcipher		$out1,$out1,v30
fd2893
+	 vand		$tmp,$tmp,$eighty7
fd2893
+	vcipher		$out2,$out2,v30
fd2893
+	vcipher		$out3,$out3,v30
fd2893
+	 vxor		$tweak,$tweak,$tmp
fd2893
+	vcipher		$out4,$out4,v30
fd2893
+	vcipher		$out5,$out5,v30
fd2893
+	 vxor		$in5,$twk5,v31
fd2893
+	 vsrab		$tmp,$tweak,$seven	# next tweak value
fd2893
+	 vxor		$twk5,$tweak,$rndkey0
fd2893
+
fd2893
+	vcipherlast	$out0,$out0,$in0
fd2893
+	 lvx_u		$in0,$x00,$inp		# load next input block
fd2893
+	 vaddubm	$tweak,$tweak,$tweak
fd2893
+	 vsldoi		$tmp,$tmp,$tmp,15
fd2893
+	vcipherlast	$out1,$out1,$in1
fd2893
+	 lvx_u		$in1,$x10,$inp
fd2893
+	vcipherlast	$out2,$out2,$in2
fd2893
+	 le?vperm	$in0,$in0,$in0,$leperm
fd2893
+	 lvx_u		$in2,$x20,$inp
fd2893
+	 vand		$tmp,$tmp,$eighty7
fd2893
+	vcipherlast	$out3,$out3,$in3
fd2893
+	 le?vperm	$in1,$in1,$in1,$leperm
fd2893
+	 lvx_u		$in3,$x30,$inp
fd2893
+	vcipherlast	$out4,$out4,$in4
fd2893
+	 le?vperm	$in2,$in2,$in2,$leperm
fd2893
+	 lvx_u		$in4,$x40,$inp
fd2893
+	 vxor		$tweak,$tweak,$tmp
fd2893
+	vcipherlast	$tmp,$out5,$in5		# last block might be needed
fd2893
+						# in stealing mode
fd2893
+	 le?vperm	$in3,$in3,$in3,$leperm
fd2893
+	 lvx_u		$in5,$x50,$inp
fd2893
+	 addi		$inp,$inp,0x60
fd2893
+	 le?vperm	$in4,$in4,$in4,$leperm
fd2893
+	 le?vperm	$in5,$in5,$in5,$leperm
fd2893
+
fd2893
+	le?vperm	$out0,$out0,$out0,$leperm
fd2893
+	le?vperm	$out1,$out1,$out1,$leperm
fd2893
+	stvx_u		$out0,$x00,$out		# store output
fd2893
+	 vxor		$out0,$in0,$twk0
fd2893
+	le?vperm	$out2,$out2,$out2,$leperm
fd2893
+	stvx_u		$out1,$x10,$out
fd2893
+	 vxor		$out1,$in1,$twk1
fd2893
+	le?vperm	$out3,$out3,$out3,$leperm
fd2893
+	stvx_u		$out2,$x20,$out
fd2893
+	 vxor		$out2,$in2,$twk2
fd2893
+	le?vperm	$out4,$out4,$out4,$leperm
fd2893
+	stvx_u		$out3,$x30,$out
fd2893
+	 vxor		$out3,$in3,$twk3
fd2893
+	le?vperm	$out5,$tmp,$tmp,$leperm
fd2893
+	stvx_u		$out4,$x40,$out
fd2893
+	 vxor		$out4,$in4,$twk4
fd2893
+	le?stvx_u	$out5,$x50,$out
fd2893
+	be?stvx_u	$tmp, $x50,$out
fd2893
+	 vxor		$out5,$in5,$twk5
fd2893
+	addi		$out,$out,0x60
fd2893
+
fd2893
+	mtctr		$rounds
fd2893
+	beq		Loop_xts_enc6x		# did $len-=96 borrow?
fd2893
+
fd2893
+	addic.		$len,$len,0x60
fd2893
+	beq		Lxts_enc6x_zero
fd2893
+	cmpwi		$len,0x20
fd2893
+	blt		Lxts_enc6x_one
fd2893
+	nop
fd2893
+	beq		Lxts_enc6x_two
fd2893
+	cmpwi		$len,0x40
fd2893
+	blt		Lxts_enc6x_three
fd2893
+	nop
fd2893
+	beq		Lxts_enc6x_four
fd2893
+
fd2893
+Lxts_enc6x_five:
fd2893
+	vxor		$out0,$in1,$twk0
fd2893
+	vxor		$out1,$in2,$twk1
fd2893
+	vxor		$out2,$in3,$twk2
fd2893
+	vxor		$out3,$in4,$twk3
fd2893
+	vxor		$out4,$in5,$twk4
fd2893
+
fd2893
+	bl		_aesp8_xts_enc5x
fd2893
+
fd2893
+	le?vperm	$out0,$out0,$out0,$leperm
fd2893
+	vmr		$twk0,$twk5		# unused tweak
fd2893
+	le?vperm	$out1,$out1,$out1,$leperm
fd2893
+	stvx_u		$out0,$x00,$out		# store output
fd2893
+	le?vperm	$out2,$out2,$out2,$leperm
fd2893
+	stvx_u		$out1,$x10,$out
fd2893
+	le?vperm	$out3,$out3,$out3,$leperm
fd2893
+	stvx_u		$out2,$x20,$out
fd2893
+	vxor		$tmp,$out4,$twk5	# last block prep for stealing
fd2893
+	le?vperm	$out4,$out4,$out4,$leperm
fd2893
+	stvx_u		$out3,$x30,$out
fd2893
+	stvx_u		$out4,$x40,$out
fd2893
+	addi		$out,$out,0x50
fd2893
+	bne		Lxts_enc6x_steal
fd2893
+	b		Lxts_enc6x_done
fd2893
+
fd2893
+.align	4
fd2893
+Lxts_enc6x_four:
fd2893
+	vxor		$out0,$in2,$twk0
fd2893
+	vxor		$out1,$in3,$twk1
fd2893
+	vxor		$out2,$in4,$twk2
fd2893
+	vxor		$out3,$in5,$twk3
fd2893
+	vxor		$out4,$out4,$out4
fd2893
+
fd2893
+	bl		_aesp8_xts_enc5x
fd2893
+
fd2893
+	le?vperm	$out0,$out0,$out0,$leperm
fd2893
+	vmr		$twk0,$twk4		# unused tweak
fd2893
+	le?vperm	$out1,$out1,$out1,$leperm
fd2893
+	stvx_u		$out0,$x00,$out		# store output
fd2893
+	le?vperm	$out2,$out2,$out2,$leperm
fd2893
+	stvx_u		$out1,$x10,$out
fd2893
+	vxor		$tmp,$out3,$twk4	# last block prep for stealing
fd2893
+	le?vperm	$out3,$out3,$out3,$leperm
fd2893
+	stvx_u		$out2,$x20,$out
fd2893
+	stvx_u		$out3,$x30,$out
fd2893
+	addi		$out,$out,0x40
fd2893
+	bne		Lxts_enc6x_steal
fd2893
+	b		Lxts_enc6x_done
fd2893
+
fd2893
+.align	4
fd2893
+Lxts_enc6x_three:
fd2893
+	vxor		$out0,$in3,$twk0
fd2893
+	vxor		$out1,$in4,$twk1
fd2893
+	vxor		$out2,$in5,$twk2
fd2893
+	vxor		$out3,$out3,$out3
fd2893
+	vxor		$out4,$out4,$out4
fd2893
+
fd2893
+	bl		_aesp8_xts_enc5x
fd2893
+
fd2893
+	le?vperm	$out0,$out0,$out0,$leperm
fd2893
+	vmr		$twk0,$twk3		# unused tweak
fd2893
+	le?vperm	$out1,$out1,$out1,$leperm
fd2893
+	stvx_u		$out0,$x00,$out		# store output
fd2893
+	vxor		$tmp,$out2,$twk3	# last block prep for stealing
fd2893
+	le?vperm	$out2,$out2,$out2,$leperm
fd2893
+	stvx_u		$out1,$x10,$out
fd2893
+	stvx_u		$out2,$x20,$out
fd2893
+	addi		$out,$out,0x30
fd2893
+	bne		Lxts_enc6x_steal
fd2893
+	b		Lxts_enc6x_done
fd2893
+
fd2893
+.align	4
fd2893
+Lxts_enc6x_two:
fd2893
+	vxor		$out0,$in4,$twk0
fd2893
+	vxor		$out1,$in5,$twk1
fd2893
+	vxor		$out2,$out2,$out2
fd2893
+	vxor		$out3,$out3,$out3
fd2893
+	vxor		$out4,$out4,$out4
fd2893
+
fd2893
+	bl		_aesp8_xts_enc5x
fd2893
+
fd2893
+	le?vperm	$out0,$out0,$out0,$leperm
fd2893
+	vmr		$twk0,$twk2		# unused tweak
fd2893
+	vxor		$tmp,$out1,$twk2	# last block prep for stealing
fd2893
+	le?vperm	$out1,$out1,$out1,$leperm
fd2893
+	stvx_u		$out0,$x00,$out		# store output
fd2893
+	stvx_u		$out1,$x10,$out
fd2893
+	addi		$out,$out,0x20
fd2893
+	bne		Lxts_enc6x_steal
fd2893
+	b		Lxts_enc6x_done
fd2893
+
fd2893
+.align	4
fd2893
+Lxts_enc6x_one:
fd2893
+	vxor		$out0,$in5,$twk0
fd2893
+	nop
fd2893
+Loop_xts_enc1x:
fd2893
+	vcipher		$out0,$out0,v24
fd2893
+	lvx		v24,$x20,$key_		# round[3]
fd2893
+	addi		$key_,$key_,0x20
fd2893
+
fd2893
+	vcipher		$out0,$out0,v25
fd2893
+	lvx		v25,$x10,$key_		# round[4]
fd2893
+	bdnz		Loop_xts_enc1x
fd2893
+
fd2893
+	add		$inp,$inp,$taillen
fd2893
+	cmpwi		$taillen,0
fd2893
+	vcipher		$out0,$out0,v24
fd2893
+
fd2893
+	subi		$inp,$inp,16
fd2893
+	vcipher		$out0,$out0,v25
fd2893
+
fd2893
+	lvsr		$inpperm,0,$taillen
fd2893
+	vcipher		$out0,$out0,v26
fd2893
+
fd2893
+	lvx_u		$in0,0,$inp
fd2893
+	vcipher		$out0,$out0,v27
fd2893
+
fd2893
+	addi		$key_,$sp,$FRAME+15	# rewind $key_
fd2893
+	vcipher		$out0,$out0,v28
fd2893
+	lvx		v24,$x00,$key_		# re-pre-load round[1]
fd2893
+
fd2893
+	vcipher		$out0,$out0,v29
fd2893
+	lvx		v25,$x10,$key_		# re-pre-load round[2]
fd2893
+	 vxor		$twk0,$twk0,v31
fd2893
+
fd2893
+	le?vperm	$in0,$in0,$in0,$leperm
fd2893
+	vcipher		$out0,$out0,v30
fd2893
+
fd2893
+	vperm		$in0,$in0,$in0,$inpperm
fd2893
+	vcipherlast	$out0,$out0,$twk0
fd2893
+
fd2893
+	vmr		$twk0,$twk1		# unused tweak
fd2893
+	vxor		$tmp,$out0,$twk1	# last block prep for stealing
fd2893
+	le?vperm	$out0,$out0,$out0,$leperm
fd2893
+	stvx_u		$out0,$x00,$out		# store output
fd2893
+	addi		$out,$out,0x10
fd2893
+	bne		Lxts_enc6x_steal
fd2893
+	b		Lxts_enc6x_done
fd2893
+
fd2893
+.align	4
fd2893
+Lxts_enc6x_zero:
fd2893
+	cmpwi		$taillen,0
fd2893
+	beq		Lxts_enc6x_done
fd2893
+
fd2893
+	add		$inp,$inp,$taillen
fd2893
+	subi		$inp,$inp,16
fd2893
+	lvx_u		$in0,0,$inp
fd2893
+	lvsr		$inpperm,0,$taillen	# $in5 is no more
fd2893
+	le?vperm	$in0,$in0,$in0,$leperm
fd2893
+	vperm		$in0,$in0,$in0,$inpperm
fd2893
+	vxor		$tmp,$tmp,$twk0
fd2893
+Lxts_enc6x_steal:
fd2893
+	vxor		$in0,$in0,$twk0
fd2893
+	vxor		$out0,$out0,$out0
fd2893
+	vspltisb	$out1,-1
fd2893
+	vperm		$out0,$out0,$out1,$inpperm
fd2893
+	vsel		$out0,$in0,$tmp,$out0	# $tmp is last block, remember?
fd2893
+
fd2893
+	subi		r30,$out,17
fd2893
+	subi		$out,$out,16
fd2893
+	mtctr		$taillen
fd2893
+Loop_xts_enc6x_steal:
fd2893
+	lbzu		r0,1(r30)
fd2893
+	stb		r0,16(r30)
fd2893
+	bdnz		Loop_xts_enc6x_steal
fd2893
+
fd2893
+	li		$taillen,0
fd2893
+	mtctr		$rounds
fd2893
+	b		Loop_xts_enc1x		# one more time...
fd2893
+
fd2893
+.align	4
fd2893
+Lxts_enc6x_done:
fd2893
+	${UCMP}i	$ivp,0
fd2893
+	beq		Lxts_enc6x_ret
fd2893
+
fd2893
+	vxor		$tweak,$twk0,$rndkey0
fd2893
+	le?vperm	$tweak,$tweak,$tweak,$leperm
fd2893
+	stvx_u		$tweak,0,$ivp
fd2893
+
fd2893
+Lxts_enc6x_ret:
fd2893
+	mtlr		r11
fd2893
+	li		r10,`$FRAME+15`
fd2893
+	li		r11,`$FRAME+31`
fd2893
+	stvx		$seven,r10,$sp		# wipe copies of round keys
fd2893
+	addi		r10,r10,32
fd2893
+	stvx		$seven,r11,$sp
fd2893
+	addi		r11,r11,32
fd2893
+	stvx		$seven,r10,$sp
fd2893
+	addi		r10,r10,32
fd2893
+	stvx		$seven,r11,$sp
fd2893
+	addi		r11,r11,32
fd2893
+	stvx		$seven,r10,$sp
fd2893
+	addi		r10,r10,32
fd2893
+	stvx		$seven,r11,$sp
fd2893
+	addi		r11,r11,32
fd2893
+	stvx		$seven,r10,$sp
fd2893
+	addi		r10,r10,32
fd2893
+	stvx		$seven,r11,$sp
fd2893
+	addi		r11,r11,32
fd2893
+
fd2893
+	mtspr		256,$vrsave
fd2893
+	lvx		v20,r10,$sp		# ABI says so
fd2893
+	addi		r10,r10,32
fd2893
+	lvx		v21,r11,$sp
fd2893
+	addi		r11,r11,32
fd2893
+	lvx		v22,r10,$sp
fd2893
+	addi		r10,r10,32
fd2893
+	lvx		v23,r11,$sp
fd2893
+	addi		r11,r11,32
fd2893
+	lvx		v24,r10,$sp
fd2893
+	addi		r10,r10,32
fd2893
+	lvx		v25,r11,$sp
fd2893
+	addi		r11,r11,32
fd2893
+	lvx		v26,r10,$sp
fd2893
+	addi		r10,r10,32
fd2893
+	lvx		v27,r11,$sp
fd2893
+	addi		r11,r11,32
fd2893
+	lvx		v28,r10,$sp
fd2893
+	addi		r10,r10,32
fd2893
+	lvx		v29,r11,$sp
fd2893
+	addi		r11,r11,32
fd2893
+	lvx		v30,r10,$sp
fd2893
+	lvx		v31,r11,$sp
fd2893
+	$POP		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
fd2893
+	$POP		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
fd2893
+	$POP		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
fd2893
+	$POP		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
fd2893
+	$POP		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
fd2893
+	$POP		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
fd2893
+	addi		$sp,$sp,`$FRAME+21*16+6*$SIZE_T`
fd2893
+	blr
fd2893
+	.long		0
fd2893
+	.byte		0,12,0x04,1,0x80,6,6,0
fd2893
+	.long		0
fd2893
+
fd2893
+.align	5
fd2893
+_aesp8_xts_enc5x:
fd2893
+	vcipher		$out0,$out0,v24
fd2893
+	vcipher		$out1,$out1,v24
fd2893
+	vcipher		$out2,$out2,v24
fd2893
+	vcipher		$out3,$out3,v24
fd2893
+	vcipher		$out4,$out4,v24
fd2893
+	lvx		v24,$x20,$key_		# round[3]
fd2893
+	addi		$key_,$key_,0x20
fd2893
+
fd2893
+	vcipher		$out0,$out0,v25
fd2893
+	vcipher		$out1,$out1,v25
fd2893
+	vcipher		$out2,$out2,v25
fd2893
+	vcipher		$out3,$out3,v25
fd2893
+	vcipher		$out4,$out4,v25
fd2893
+	lvx		v25,$x10,$key_		# round[4]
fd2893
+	bdnz		_aesp8_xts_enc5x
fd2893
+
fd2893
+	add		$inp,$inp,$taillen
fd2893
+	cmpwi		$taillen,0
fd2893
+	vcipher		$out0,$out0,v24
fd2893
+	vcipher		$out1,$out1,v24
fd2893
+	vcipher		$out2,$out2,v24
fd2893
+	vcipher		$out3,$out3,v24
fd2893
+	vcipher		$out4,$out4,v24
fd2893
+
fd2893
+	subi		$inp,$inp,16
fd2893
+	vcipher		$out0,$out0,v25
fd2893
+	vcipher		$out1,$out1,v25
fd2893
+	vcipher		$out2,$out2,v25
fd2893
+	vcipher		$out3,$out3,v25
fd2893
+	vcipher		$out4,$out4,v25
fd2893
+	 vxor		$twk0,$twk0,v31
fd2893
+
fd2893
+	vcipher		$out0,$out0,v26
fd2893
+	lvsr		$inpperm,r0,$taillen	# $in5 is no more
fd2893
+	vcipher		$out1,$out1,v26
fd2893
+	vcipher		$out2,$out2,v26
fd2893
+	vcipher		$out3,$out3,v26
fd2893
+	vcipher		$out4,$out4,v26
fd2893
+	 vxor		$in1,$twk1,v31
fd2893
+
fd2893
+	vcipher		$out0,$out0,v27
fd2893
+	lvx_u		$in0,0,$inp
fd2893
+	vcipher		$out1,$out1,v27
fd2893
+	vcipher		$out2,$out2,v27
fd2893
+	vcipher		$out3,$out3,v27
fd2893
+	vcipher		$out4,$out4,v27
fd2893
+	 vxor		$in2,$twk2,v31
fd2893
+
fd2893
+	addi		$key_,$sp,$FRAME+15	# rewind $key_
fd2893
+	vcipher		$out0,$out0,v28
fd2893
+	vcipher		$out1,$out1,v28
fd2893
+	vcipher		$out2,$out2,v28
fd2893
+	vcipher		$out3,$out3,v28
fd2893
+	vcipher		$out4,$out4,v28
fd2893
+	lvx		v24,$x00,$key_		# re-pre-load round[1]
fd2893
+	 vxor		$in3,$twk3,v31
fd2893
+
fd2893
+	vcipher		$out0,$out0,v29
fd2893
+	le?vperm	$in0,$in0,$in0,$leperm
fd2893
+	vcipher		$out1,$out1,v29
fd2893
+	vcipher		$out2,$out2,v29
fd2893
+	vcipher		$out3,$out3,v29
fd2893
+	vcipher		$out4,$out4,v29
fd2893
+	lvx		v25,$x10,$key_		# re-pre-load round[2]
fd2893
+	 vxor		$in4,$twk4,v31
fd2893
+
fd2893
+	vcipher		$out0,$out0,v30
fd2893
+	vperm		$in0,$in0,$in0,$inpperm
fd2893
+	vcipher		$out1,$out1,v30
fd2893
+	vcipher		$out2,$out2,v30
fd2893
+	vcipher		$out3,$out3,v30
fd2893
+	vcipher		$out4,$out4,v30
fd2893
+
fd2893
+	vcipherlast	$out0,$out0,$twk0
fd2893
+	vcipherlast	$out1,$out1,$in1
fd2893
+	vcipherlast	$out2,$out2,$in2
fd2893
+	vcipherlast	$out3,$out3,$in3
fd2893
+	vcipherlast	$out4,$out4,$in4
fd2893
+	blr
fd2893
+        .long   	0
fd2893
+        .byte   	0,12,0x14,0,0,0,0,0
fd2893
+
fd2893
+.align	5
fd2893
+_aesp8_xts_decrypt6x:
fd2893
+	$STU		$sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
fd2893
+	mflr		r11
fd2893
+	li		r7,`$FRAME+8*16+15`
fd2893
+	li		r3,`$FRAME+8*16+31`
fd2893
+	$PUSH		r11,`$FRAME+21*16+6*$SIZE_T+$LRSAVE`($sp)
fd2893
+	stvx		v20,r7,$sp		# ABI says so
fd2893
+	addi		r7,r7,32
fd2893
+	stvx		v21,r3,$sp
fd2893
+	addi		r3,r3,32
fd2893
+	stvx		v22,r7,$sp
fd2893
+	addi		r7,r7,32
fd2893
+	stvx		v23,r3,$sp
fd2893
+	addi		r3,r3,32
fd2893
+	stvx		v24,r7,$sp
fd2893
+	addi		r7,r7,32
fd2893
+	stvx		v25,r3,$sp
fd2893
+	addi		r3,r3,32
fd2893
+	stvx		v26,r7,$sp
fd2893
+	addi		r7,r7,32
fd2893
+	stvx		v27,r3,$sp
fd2893
+	addi		r3,r3,32
fd2893
+	stvx		v28,r7,$sp
fd2893
+	addi		r7,r7,32
fd2893
+	stvx		v29,r3,$sp
fd2893
+	addi		r3,r3,32
fd2893
+	stvx		v30,r7,$sp
fd2893
+	stvx		v31,r3,$sp
fd2893
+	li		r0,-1
fd2893
+	stw		$vrsave,`$FRAME+21*16-4`($sp)	# save vrsave
fd2893
+	li		$x10,0x10
fd2893
+	$PUSH		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
fd2893
+	li		$x20,0x20
fd2893
+	$PUSH		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
fd2893
+	li		$x30,0x30
fd2893
+	$PUSH		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
fd2893
+	li		$x40,0x40
fd2893
+	$PUSH		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
fd2893
+	li		$x50,0x50
fd2893
+	$PUSH		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
fd2893
+	li		$x60,0x60
fd2893
+	$PUSH		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
fd2893
+	li		$x70,0x70
fd2893
+	mtspr		256,r0
fd2893
+
fd2893
+	subi		$rounds,$rounds,3	# -4 in total
fd2893
+
fd2893
+	lvx		$rndkey0,$x00,$key1	# load key schedule
fd2893
+	lvx		v30,$x10,$key1
fd2893
+	addi		$key1,$key1,0x20
fd2893
+	lvx		v31,$x00,$key1
fd2893
+	?vperm		$rndkey0,$rndkey0,v30,$keyperm
fd2893
+	addi		$key_,$sp,$FRAME+15
fd2893
+	mtctr		$rounds
fd2893
+
fd2893
+Load_xts_dec_key:
fd2893
+	?vperm		v24,v30,v31,$keyperm
fd2893
+	lvx		v30,$x10,$key1
fd2893
+	addi		$key1,$key1,0x20
fd2893
+	stvx		v24,$x00,$key_		# off-load round[1]
fd2893
+	?vperm		v25,v31,v30,$keyperm
fd2893
+	lvx		v31,$x00,$key1
fd2893
+	stvx		v25,$x10,$key_		# off-load round[2]
fd2893
+	addi		$key_,$key_,0x20
fd2893
+	bdnz		Load_xts_dec_key
fd2893
+
fd2893
+	lvx		v26,$x10,$key1
fd2893
+	?vperm		v24,v30,v31,$keyperm
fd2893
+	lvx		v27,$x20,$key1
fd2893
+	stvx		v24,$x00,$key_		# off-load round[3]
fd2893
+	?vperm		v25,v31,v26,$keyperm
fd2893
+	lvx		v28,$x30,$key1
fd2893
+	stvx		v25,$x10,$key_		# off-load round[4]
fd2893
+	addi		$key_,$sp,$FRAME+15	# rewind $key_
fd2893
+	?vperm		v26,v26,v27,$keyperm
fd2893
+	lvx		v29,$x40,$key1
fd2893
+	?vperm		v27,v27,v28,$keyperm
fd2893
+	lvx		v30,$x50,$key1
fd2893
+	?vperm		v28,v28,v29,$keyperm
fd2893
+	lvx		v31,$x60,$key1
fd2893
+	?vperm		v29,v29,v30,$keyperm
fd2893
+	lvx		$twk5,$x70,$key1	# borrow $twk5
fd2893
+	?vperm		v30,v30,v31,$keyperm
fd2893
+	lvx		v24,$x00,$key_		# pre-load round[1]
fd2893
+	?vperm		v31,v31,$twk5,$keyperm
fd2893
+	lvx		v25,$x10,$key_		# pre-load round[2]
fd2893
+
fd2893
+	 vperm		$in0,$inout,$inptail,$inpperm
fd2893
+	 subi		$inp,$inp,31		# undo "caller"
fd2893
+	vxor		$twk0,$tweak,$rndkey0
fd2893
+	vsrab		$tmp,$tweak,$seven	# next tweak value
fd2893
+	vaddubm		$tweak,$tweak,$tweak
fd2893
+	vsldoi		$tmp,$tmp,$tmp,15
fd2893
+	vand		$tmp,$tmp,$eighty7
fd2893
+	 vxor		$out0,$in0,$twk0
fd2893
+	vxor		$tweak,$tweak,$tmp
fd2893
+
fd2893
+	 lvx_u		$in1,$x10,$inp
fd2893
+	vxor		$twk1,$tweak,$rndkey0
fd2893
+	vsrab		$tmp,$tweak,$seven	# next tweak value
fd2893
+	vaddubm		$tweak,$tweak,$tweak
fd2893
+	vsldoi		$tmp,$tmp,$tmp,15
fd2893
+	 le?vperm	$in1,$in1,$in1,$leperm
fd2893
+	vand		$tmp,$tmp,$eighty7
fd2893
+	 vxor		$out1,$in1,$twk1
fd2893
+	vxor		$tweak,$tweak,$tmp
fd2893
+
fd2893
+	 lvx_u		$in2,$x20,$inp
fd2893
+	 andi.		$taillen,$len,15
fd2893
+	vxor		$twk2,$tweak,$rndkey0
fd2893
+	vsrab		$tmp,$tweak,$seven	# next tweak value
fd2893
+	vaddubm		$tweak,$tweak,$tweak
fd2893
+	vsldoi		$tmp,$tmp,$tmp,15
fd2893
+	 le?vperm	$in2,$in2,$in2,$leperm
fd2893
+	vand		$tmp,$tmp,$eighty7
fd2893
+	 vxor		$out2,$in2,$twk2
fd2893
+	vxor		$tweak,$tweak,$tmp
fd2893
+
fd2893
+	 lvx_u		$in3,$x30,$inp
fd2893
+	 sub		$len,$len,$taillen
fd2893
+	vxor		$twk3,$tweak,$rndkey0
fd2893
+	vsrab		$tmp,$tweak,$seven	# next tweak value
fd2893
+	vaddubm		$tweak,$tweak,$tweak
fd2893
+	vsldoi		$tmp,$tmp,$tmp,15
fd2893
+	 le?vperm	$in3,$in3,$in3,$leperm
fd2893
+	vand		$tmp,$tmp,$eighty7
fd2893
+	 vxor		$out3,$in3,$twk3
fd2893
+	vxor		$tweak,$tweak,$tmp
fd2893
+
fd2893
+	 lvx_u		$in4,$x40,$inp
fd2893
+	 subi		$len,$len,0x60
fd2893
+	vxor		$twk4,$tweak,$rndkey0
fd2893
+	vsrab		$tmp,$tweak,$seven	# next tweak value
fd2893
+	vaddubm		$tweak,$tweak,$tweak
fd2893
+	vsldoi		$tmp,$tmp,$tmp,15
fd2893
+	 le?vperm	$in4,$in4,$in4,$leperm
fd2893
+	vand		$tmp,$tmp,$eighty7
fd2893
+	 vxor		$out4,$in4,$twk4
fd2893
+	vxor		$tweak,$tweak,$tmp
fd2893
+
fd2893
+	 lvx_u		$in5,$x50,$inp
fd2893
+	 addi		$inp,$inp,0x60
fd2893
+	vxor		$twk5,$tweak,$rndkey0
fd2893
+	vsrab		$tmp,$tweak,$seven	# next tweak value
fd2893
+	vaddubm		$tweak,$tweak,$tweak
fd2893
+	vsldoi		$tmp,$tmp,$tmp,15
fd2893
+	 le?vperm	$in5,$in5,$in5,$leperm
fd2893
+	vand		$tmp,$tmp,$eighty7
fd2893
+	 vxor		$out5,$in5,$twk5
fd2893
+	vxor		$tweak,$tweak,$tmp
fd2893
+
fd2893
+	vxor		v31,v31,$rndkey0
fd2893
+	mtctr		$rounds
fd2893
+	b		Loop_xts_dec6x
fd2893
+
fd2893
+.align	5
fd2893
+Loop_xts_dec6x:
fd2893
+	vncipher	$out0,$out0,v24
fd2893
+	vncipher	$out1,$out1,v24
fd2893
+	vncipher	$out2,$out2,v24
fd2893
+	vncipher	$out3,$out3,v24
fd2893
+	vncipher	$out4,$out4,v24
fd2893
+	vncipher	$out5,$out5,v24
fd2893
+	lvx		v24,$x20,$key_		# round[3]
fd2893
+	addi		$key_,$key_,0x20
fd2893
+
fd2893
+	vncipher	$out0,$out0,v25
fd2893
+	vncipher	$out1,$out1,v25
fd2893
+	vncipher	$out2,$out2,v25
fd2893
+	vncipher	$out3,$out3,v25
fd2893
+	vncipher	$out4,$out4,v25
fd2893
+	vncipher	$out5,$out5,v25
fd2893
+	lvx		v25,$x10,$key_		# round[4]
fd2893
+	bdnz		Loop_xts_dec6x
fd2893
+
fd2893
+	subic		$len,$len,96		# $len-=96
fd2893
+	 vxor		$in0,$twk0,v31		# xor with last round key
fd2893
+	vncipher	$out0,$out0,v24
fd2893
+	vncipher	$out1,$out1,v24
fd2893
+	 vsrab		$tmp,$tweak,$seven	# next tweak value
fd2893
+	 vxor		$twk0,$tweak,$rndkey0
fd2893
+	 vaddubm	$tweak,$tweak,$tweak
fd2893
+	vncipher	$out2,$out2,v24
fd2893
+	vncipher	$out3,$out3,v24
fd2893
+	 vsldoi		$tmp,$tmp,$tmp,15
fd2893
+	vncipher	$out4,$out4,v24
fd2893
+	vncipher	$out5,$out5,v24
fd2893
+
fd2893
+	subfe.		r0,r0,r0		# borrow?-1:0
fd2893
+	 vand		$tmp,$tmp,$eighty7
fd2893
+	vncipher	$out0,$out0,v25
fd2893
+	vncipher	$out1,$out1,v25
fd2893
+	 vxor		$tweak,$tweak,$tmp
fd2893
+	vncipher	$out2,$out2,v25
fd2893
+	vncipher	$out3,$out3,v25
fd2893
+	 vxor		$in1,$twk1,v31
fd2893
+	 vsrab		$tmp,$tweak,$seven	# next tweak value
fd2893
+	 vxor		$twk1,$tweak,$rndkey0
fd2893
+	vncipher	$out4,$out4,v25
fd2893
+	vncipher	$out5,$out5,v25
fd2893
+
fd2893
+	and		r0,r0,$len
fd2893
+	 vaddubm	$tweak,$tweak,$tweak
fd2893
+	 vsldoi		$tmp,$tmp,$tmp,15
fd2893
+	vncipher	$out0,$out0,v26
fd2893
+	vncipher	$out1,$out1,v26
fd2893
+	 vand		$tmp,$tmp,$eighty7
fd2893
+	vncipher	$out2,$out2,v26
fd2893
+	vncipher	$out3,$out3,v26
fd2893
+	 vxor		$tweak,$tweak,$tmp
fd2893
+	vncipher	$out4,$out4,v26
fd2893
+	vncipher	$out5,$out5,v26
fd2893
+
fd2893
+	add		$inp,$inp,r0		# $inp is adjusted in such
fd2893
+						# way that at exit from the
fd2893
+						# loop inX-in5 are loaded
fd2893
+						# with last "words"
fd2893
+	 vxor		$in2,$twk2,v31
fd2893
+	 vsrab		$tmp,$tweak,$seven	# next tweak value
fd2893
+	 vxor		$twk2,$tweak,$rndkey0
fd2893
+	 vaddubm	$tweak,$tweak,$tweak
fd2893
+	vncipher	$out0,$out0,v27
fd2893
+	vncipher	$out1,$out1,v27
fd2893
+	 vsldoi		$tmp,$tmp,$tmp,15
fd2893
+	vncipher	$out2,$out2,v27
fd2893
+	vncipher	$out3,$out3,v27
fd2893
+	 vand		$tmp,$tmp,$eighty7
fd2893
+	vncipher	$out4,$out4,v27
fd2893
+	vncipher	$out5,$out5,v27
fd2893
+
fd2893
+	addi		$key_,$sp,$FRAME+15	# rewind $key_
fd2893
+	 vxor		$tweak,$tweak,$tmp
fd2893
+	vncipher	$out0,$out0,v28
fd2893
+	vncipher	$out1,$out1,v28
fd2893
+	 vxor		$in3,$twk3,v31
fd2893
+	 vsrab		$tmp,$tweak,$seven	# next tweak value
fd2893
+	 vxor		$twk3,$tweak,$rndkey0
fd2893
+	vncipher	$out2,$out2,v28
fd2893
+	vncipher	$out3,$out3,v28
fd2893
+	 vaddubm	$tweak,$tweak,$tweak
fd2893
+	 vsldoi		$tmp,$tmp,$tmp,15
fd2893
+	vncipher	$out4,$out4,v28
fd2893
+	vncipher	$out5,$out5,v28
fd2893
+	lvx		v24,$x00,$key_		# re-pre-load round[1]
fd2893
+	 vand		$tmp,$tmp,$eighty7
fd2893
+
fd2893
+	vncipher	$out0,$out0,v29
fd2893
+	vncipher	$out1,$out1,v29
fd2893
+	 vxor		$tweak,$tweak,$tmp
fd2893
+	vncipher	$out2,$out2,v29
fd2893
+	vncipher	$out3,$out3,v29
fd2893
+	 vxor		$in4,$twk4,v31
fd2893
+	 vsrab		$tmp,$tweak,$seven	# next tweak value
fd2893
+	 vxor		$twk4,$tweak,$rndkey0
fd2893
+	vncipher	$out4,$out4,v29
fd2893
+	vncipher	$out5,$out5,v29
fd2893
+	lvx		v25,$x10,$key_		# re-pre-load round[2]
fd2893
+	 vaddubm	$tweak,$tweak,$tweak
fd2893
+	 vsldoi		$tmp,$tmp,$tmp,15
fd2893
+
fd2893
+	vncipher	$out0,$out0,v30
fd2893
+	vncipher	$out1,$out1,v30
fd2893
+	 vand		$tmp,$tmp,$eighty7
fd2893
+	vncipher	$out2,$out2,v30
fd2893
+	vncipher	$out3,$out3,v30
fd2893
+	 vxor		$tweak,$tweak,$tmp
fd2893
+	vncipher	$out4,$out4,v30
fd2893
+	vncipher	$out5,$out5,v30
fd2893
+	 vxor		$in5,$twk5,v31
fd2893
+	 vsrab		$tmp,$tweak,$seven	# next tweak value
fd2893
+	 vxor		$twk5,$tweak,$rndkey0
fd2893
+
fd2893
+	vncipherlast	$out0,$out0,$in0
fd2893
+	 lvx_u		$in0,$x00,$inp		# load next input block
fd2893
+	 vaddubm	$tweak,$tweak,$tweak
fd2893
+	 vsldoi		$tmp,$tmp,$tmp,15
fd2893
+	vncipherlast	$out1,$out1,$in1
fd2893
+	 lvx_u		$in1,$x10,$inp
fd2893
+	vncipherlast	$out2,$out2,$in2
fd2893
+	 le?vperm	$in0,$in0,$in0,$leperm
fd2893
+	 lvx_u		$in2,$x20,$inp
fd2893
+	 vand		$tmp,$tmp,$eighty7
fd2893
+	vncipherlast	$out3,$out3,$in3
fd2893
+	 le?vperm	$in1,$in1,$in1,$leperm
fd2893
+	 lvx_u		$in3,$x30,$inp
fd2893
+	vncipherlast	$out4,$out4,$in4
fd2893
+	 le?vperm	$in2,$in2,$in2,$leperm
fd2893
+	 lvx_u		$in4,$x40,$inp
fd2893
+	 vxor		$tweak,$tweak,$tmp
fd2893
+	vncipherlast	$out5,$out5,$in5
fd2893
+	 le?vperm	$in3,$in3,$in3,$leperm
fd2893
+	 lvx_u		$in5,$x50,$inp
fd2893
+	 addi		$inp,$inp,0x60
fd2893
+	 le?vperm	$in4,$in4,$in4,$leperm
fd2893
+	 le?vperm	$in5,$in5,$in5,$leperm
fd2893
+
fd2893
+	le?vperm	$out0,$out0,$out0,$leperm
fd2893
+	le?vperm	$out1,$out1,$out1,$leperm
fd2893
+	stvx_u		$out0,$x00,$out		# store output
fd2893
+	 vxor		$out0,$in0,$twk0
fd2893
+	le?vperm	$out2,$out2,$out2,$leperm
fd2893
+	stvx_u		$out1,$x10,$out
fd2893
+	 vxor		$out1,$in1,$twk1
fd2893
+	le?vperm	$out3,$out3,$out3,$leperm
fd2893
+	stvx_u		$out2,$x20,$out
fd2893
+	 vxor		$out2,$in2,$twk2
fd2893
+	le?vperm	$out4,$out4,$out4,$leperm
fd2893
+	stvx_u		$out3,$x30,$out
fd2893
+	 vxor		$out3,$in3,$twk3
fd2893
+	le?vperm	$out5,$out5,$out5,$leperm
fd2893
+	stvx_u		$out4,$x40,$out
fd2893
+	 vxor		$out4,$in4,$twk4
fd2893
+	stvx_u		$out5,$x50,$out
fd2893
+	 vxor		$out5,$in5,$twk5
fd2893
+	addi		$out,$out,0x60
fd2893
+
fd2893
+	mtctr		$rounds
fd2893
+	beq		Loop_xts_dec6x		# did $len-=96 borrow?
fd2893
+
fd2893
+	addic.		$len,$len,0x60
fd2893
+	beq		Lxts_dec6x_zero
fd2893
+	cmpwi		$len,0x20
fd2893
+	blt		Lxts_dec6x_one
fd2893
+	nop
fd2893
+	beq		Lxts_dec6x_two
fd2893
+	cmpwi		$len,0x40
fd2893
+	blt		Lxts_dec6x_three
fd2893
+	nop
fd2893
+	beq		Lxts_dec6x_four
fd2893
+
fd2893
+Lxts_dec6x_five:
fd2893
+	vxor		$out0,$in1,$twk0
fd2893
+	vxor		$out1,$in2,$twk1
fd2893
+	vxor		$out2,$in3,$twk2
fd2893
+	vxor		$out3,$in4,$twk3
fd2893
+	vxor		$out4,$in5,$twk4
fd2893
+
fd2893
+	bl		_aesp8_xts_dec5x
fd2893
+
fd2893
+	le?vperm	$out0,$out0,$out0,$leperm
fd2893
+	vmr		$twk0,$twk5		# unused tweak
fd2893
+	vxor		$twk1,$tweak,$rndkey0
fd2893
+	le?vperm	$out1,$out1,$out1,$leperm
fd2893
+	stvx_u		$out0,$x00,$out		# store output
fd2893
+	vxor		$out0,$in0,$twk1
fd2893
+	le?vperm	$out2,$out2,$out2,$leperm
fd2893
+	stvx_u		$out1,$x10,$out
fd2893
+	le?vperm	$out3,$out3,$out3,$leperm
fd2893
+	stvx_u		$out2,$x20,$out
fd2893
+	le?vperm	$out4,$out4,$out4,$leperm
fd2893
+	stvx_u		$out3,$x30,$out
fd2893
+	stvx_u		$out4,$x40,$out
fd2893
+	addi		$out,$out,0x50
fd2893
+	bne		Lxts_dec6x_steal
fd2893
+	b		Lxts_dec6x_done
fd2893
+
fd2893
+.align	4
fd2893
+Lxts_dec6x_four:
fd2893
+	vxor		$out0,$in2,$twk0
fd2893
+	vxor		$out1,$in3,$twk1
fd2893
+	vxor		$out2,$in4,$twk2
fd2893
+	vxor		$out3,$in5,$twk3
fd2893
+	vxor		$out4,$out4,$out4
fd2893
+
fd2893
+	bl		_aesp8_xts_dec5x
fd2893
+
fd2893
+	le?vperm	$out0,$out0,$out0,$leperm
fd2893
+	vmr		$twk0,$twk4		# unused tweak
fd2893
+	vmr		$twk1,$twk5
fd2893
+	le?vperm	$out1,$out1,$out1,$leperm
fd2893
+	stvx_u		$out0,$x00,$out		# store output
fd2893
+	vxor		$out0,$in0,$twk5
fd2893
+	le?vperm	$out2,$out2,$out2,$leperm
fd2893
+	stvx_u		$out1,$x10,$out
fd2893
+	le?vperm	$out3,$out3,$out3,$leperm
fd2893
+	stvx_u		$out2,$x20,$out
fd2893
+	stvx_u		$out3,$x30,$out
fd2893
+	addi		$out,$out,0x40
fd2893
+	bne		Lxts_dec6x_steal
fd2893
+	b		Lxts_dec6x_done
fd2893
+
fd2893
+.align	4
fd2893
+Lxts_dec6x_three:
fd2893
+	vxor		$out0,$in3,$twk0
fd2893
+	vxor		$out1,$in4,$twk1
fd2893
+	vxor		$out2,$in5,$twk2
fd2893
+	vxor		$out3,$out3,$out3
fd2893
+	vxor		$out4,$out4,$out4
fd2893
+
fd2893
+	bl		_aesp8_xts_dec5x
fd2893
+
fd2893
+	le?vperm	$out0,$out0,$out0,$leperm
fd2893
+	vmr		$twk0,$twk3		# unused tweak
fd2893
+	vmr		$twk1,$twk4
fd2893
+	le?vperm	$out1,$out1,$out1,$leperm
fd2893
+	stvx_u		$out0,$x00,$out		# store output
fd2893
+	vxor		$out0,$in0,$twk4
fd2893
+	le?vperm	$out2,$out2,$out2,$leperm
fd2893
+	stvx_u		$out1,$x10,$out
fd2893
+	stvx_u		$out2,$x20,$out
fd2893
+	addi		$out,$out,0x30
fd2893
+	bne		Lxts_dec6x_steal
fd2893
+	b		Lxts_dec6x_done
fd2893
+
fd2893
+.align	4
fd2893
+Lxts_dec6x_two:
fd2893
+	vxor		$out0,$in4,$twk0
fd2893
+	vxor		$out1,$in5,$twk1
fd2893
+	vxor		$out2,$out2,$out2
fd2893
+	vxor		$out3,$out3,$out3
fd2893
+	vxor		$out4,$out4,$out4
fd2893
+
fd2893
+	bl		_aesp8_xts_dec5x
fd2893
+
fd2893
+	le?vperm	$out0,$out0,$out0,$leperm
fd2893
+	vmr		$twk0,$twk2		# unused tweak
fd2893
+	vmr		$twk1,$twk3
fd2893
+	le?vperm	$out1,$out1,$out1,$leperm
fd2893
+	stvx_u		$out0,$x00,$out		# store output
fd2893
+	vxor		$out0,$in0,$twk3
fd2893
+	stvx_u		$out1,$x10,$out
fd2893
+	addi		$out,$out,0x20
fd2893
+	bne		Lxts_dec6x_steal
fd2893
+	b		Lxts_dec6x_done
fd2893
+
fd2893
+.align	4
fd2893
+Lxts_dec6x_one:
fd2893
+	vxor		$out0,$in5,$twk0
fd2893
+	nop
fd2893
+Loop_xts_dec1x:
fd2893
+	vncipher	$out0,$out0,v24
fd2893
+	lvx		v24,$x20,$key_		# round[3]
fd2893
+	addi		$key_,$key_,0x20
fd2893
+
fd2893
+	vncipher	$out0,$out0,v25
fd2893
+	lvx		v25,$x10,$key_		# round[4]
fd2893
+	bdnz		Loop_xts_dec1x
fd2893
+
fd2893
+	subi		r0,$taillen,1
fd2893
+	vncipher	$out0,$out0,v24
fd2893
+
fd2893
+	andi.		r0,r0,16
fd2893
+	cmpwi		$taillen,0
fd2893
+	vncipher	$out0,$out0,v25
fd2893
+
fd2893
+	sub		$inp,$inp,r0
fd2893
+	vncipher	$out0,$out0,v26
fd2893
+
fd2893
+	lvx_u		$in0,0,$inp
fd2893
+	vncipher	$out0,$out0,v27
fd2893
+
fd2893
+	addi		$key_,$sp,$FRAME+15	# rewind $key_
fd2893
+	vncipher	$out0,$out0,v28
fd2893
+	lvx		v24,$x00,$key_		# re-pre-load round[1]
fd2893
+
fd2893
+	vncipher	$out0,$out0,v29
fd2893
+	lvx		v25,$x10,$key_		# re-pre-load round[2]
fd2893
+	 vxor		$twk0,$twk0,v31
fd2893
+
fd2893
+	le?vperm	$in0,$in0,$in0,$leperm
fd2893
+	vncipher	$out0,$out0,v30
fd2893
+
fd2893
+	mtctr		$rounds
fd2893
+	vncipherlast	$out0,$out0,$twk0
fd2893
+
fd2893
+	vmr		$twk0,$twk1		# unused tweak
fd2893
+	vmr		$twk1,$twk2
fd2893
+	le?vperm	$out0,$out0,$out0,$leperm
fd2893
+	stvx_u		$out0,$x00,$out		# store output
fd2893
+	addi		$out,$out,0x10
fd2893
+	vxor		$out0,$in0,$twk2
fd2893
+	bne		Lxts_dec6x_steal
fd2893
+	b		Lxts_dec6x_done
fd2893
+
fd2893
+.align	4
fd2893
+Lxts_dec6x_zero:
fd2893
+	cmpwi		$taillen,0
fd2893
+	beq		Lxts_dec6x_done
fd2893
+
fd2893
+	lvx_u		$in0,0,$inp
fd2893
+	le?vperm	$in0,$in0,$in0,$leperm
fd2893
+	vxor		$out0,$in0,$twk1
fd2893
+Lxts_dec6x_steal:
fd2893
+	vncipher	$out0,$out0,v24
fd2893
+	lvx		v24,$x20,$key_		# round[3]
fd2893
+	addi		$key_,$key_,0x20
fd2893
+
fd2893
+	vncipher	$out0,$out0,v25
fd2893
+	lvx		v25,$x10,$key_		# round[4]
fd2893
+	bdnz		Lxts_dec6x_steal
fd2893
+
fd2893
+	add		$inp,$inp,$taillen
fd2893
+	vncipher	$out0,$out0,v24
fd2893
+
fd2893
+	cmpwi		$taillen,0
fd2893
+	vncipher	$out0,$out0,v25
fd2893
+
fd2893
+	lvx_u		$in0,0,$inp
fd2893
+	vncipher	$out0,$out0,v26
fd2893
+
fd2893
+	lvsr		$inpperm,0,$taillen	# $in5 is no more
fd2893
+	vncipher	$out0,$out0,v27
fd2893
+
fd2893
+	addi		$key_,$sp,$FRAME+15	# rewind $key_
fd2893
+	vncipher	$out0,$out0,v28
fd2893
+	lvx		v24,$x00,$key_		# re-pre-load round[1]
fd2893
+
fd2893
+	vncipher	$out0,$out0,v29
fd2893
+	lvx		v25,$x10,$key_		# re-pre-load round[2]
fd2893
+	 vxor		$twk1,$twk1,v31
fd2893
+
fd2893
+	le?vperm	$in0,$in0,$in0,$leperm
fd2893
+	vncipher	$out0,$out0,v30
fd2893
+
fd2893
+	vperm		$in0,$in0,$in0,$inpperm
fd2893
+	vncipherlast	$tmp,$out0,$twk1
fd2893
+
fd2893
+	le?vperm	$out0,$tmp,$tmp,$leperm
fd2893
+	le?stvx_u	$out0,0,$out
fd2893
+	be?stvx_u	$tmp,0,$out
fd2893
+
fd2893
+	vxor		$out0,$out0,$out0
fd2893
+	vspltisb	$out1,-1
fd2893
+	vperm		$out0,$out0,$out1,$inpperm
fd2893
+	vsel		$out0,$in0,$tmp,$out0
fd2893
+	vxor		$out0,$out0,$twk0
fd2893
+
fd2893
+	subi		r30,$out,1
fd2893
+	mtctr		$taillen
fd2893
+Loop_xts_dec6x_steal:
fd2893
+	lbzu		r0,1(r30)
fd2893
+	stb		r0,16(r30)
fd2893
+	bdnz		Loop_xts_dec6x_steal
fd2893
+
fd2893
+	li		$taillen,0
fd2893
+	mtctr		$rounds
fd2893
+	b		Loop_xts_dec1x		# one more time...
fd2893
+
fd2893
+.align	4
fd2893
+Lxts_dec6x_done:
fd2893
+	${UCMP}i	$ivp,0
fd2893
+	beq		Lxts_dec6x_ret
fd2893
+
fd2893
+	vxor		$tweak,$twk0,$rndkey0
fd2893
+	le?vperm	$tweak,$tweak,$tweak,$leperm
fd2893
+	stvx_u		$tweak,0,$ivp
fd2893
+
fd2893
+Lxts_dec6x_ret:
fd2893
+	mtlr		r11
fd2893
+	li		r10,`$FRAME+15`
fd2893
+	li		r11,`$FRAME+31`
fd2893
+	stvx		$seven,r10,$sp		# wipe copies of round keys
fd2893
+	addi		r10,r10,32
fd2893
+	stvx		$seven,r11,$sp
fd2893
+	addi		r11,r11,32
fd2893
+	stvx		$seven,r10,$sp
fd2893
+	addi		r10,r10,32
fd2893
+	stvx		$seven,r11,$sp
fd2893
+	addi		r11,r11,32
fd2893
+	stvx		$seven,r10,$sp
fd2893
+	addi		r10,r10,32
fd2893
+	stvx		$seven,r11,$sp
fd2893
+	addi		r11,r11,32
fd2893
+	stvx		$seven,r10,$sp
fd2893
+	addi		r10,r10,32
fd2893
+	stvx		$seven,r11,$sp
fd2893
+	addi		r11,r11,32
fd2893
+
fd2893
+	mtspr		256,$vrsave
fd2893
+	lvx		v20,r10,$sp		# ABI says so
fd2893
+	addi		r10,r10,32
fd2893
+	lvx		v21,r11,$sp
fd2893
+	addi		r11,r11,32
fd2893
+	lvx		v22,r10,$sp
fd2893
+	addi		r10,r10,32
fd2893
+	lvx		v23,r11,$sp
fd2893
+	addi		r11,r11,32
fd2893
+	lvx		v24,r10,$sp
fd2893
+	addi		r10,r10,32
fd2893
+	lvx		v25,r11,$sp
fd2893
+	addi		r11,r11,32
fd2893
+	lvx		v26,r10,$sp
fd2893
+	addi		r10,r10,32
fd2893
+	lvx		v27,r11,$sp
fd2893
+	addi		r11,r11,32
fd2893
+	lvx		v28,r10,$sp
fd2893
+	addi		r10,r10,32
fd2893
+	lvx		v29,r11,$sp
fd2893
+	addi		r11,r11,32
fd2893
+	lvx		v30,r10,$sp
fd2893
+	lvx		v31,r11,$sp
fd2893
+	$POP		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
fd2893
+	$POP		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
fd2893
+	$POP		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
fd2893
+	$POP		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
fd2893
+	$POP		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
fd2893
+	$POP		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
fd2893
+	addi		$sp,$sp,`$FRAME+21*16+6*$SIZE_T`
fd2893
+	blr
fd2893
+	.long		0
fd2893
+	.byte		0,12,0x04,1,0x80,6,6,0
fd2893
+	.long		0
fd2893
+
fd2893
+.align	5
fd2893
+_aesp8_xts_dec5x:
fd2893
+	vncipher	$out0,$out0,v24
fd2893
+	vncipher	$out1,$out1,v24
fd2893
+	vncipher	$out2,$out2,v24
fd2893
+	vncipher	$out3,$out3,v24
fd2893
+	vncipher	$out4,$out4,v24
fd2893
+	lvx		v24,$x20,$key_		# round[3]
fd2893
+	addi		$key_,$key_,0x20
fd2893
+
fd2893
+	vncipher	$out0,$out0,v25
fd2893
+	vncipher	$out1,$out1,v25
fd2893
+	vncipher	$out2,$out2,v25
fd2893
+	vncipher	$out3,$out3,v25
fd2893
+	vncipher	$out4,$out4,v25
fd2893
+	lvx		v25,$x10,$key_		# round[4]
fd2893
+	bdnz		_aesp8_xts_dec5x
fd2893
+
fd2893
+	subi		r0,$taillen,1
fd2893
+	vncipher	$out0,$out0,v24
fd2893
+	vncipher	$out1,$out1,v24
fd2893
+	vncipher	$out2,$out2,v24
fd2893
+	vncipher	$out3,$out3,v24
fd2893
+	vncipher	$out4,$out4,v24
fd2893
+
fd2893
+	andi.		r0,r0,16
fd2893
+	cmpwi		$taillen,0
fd2893
+	vncipher	$out0,$out0,v25
fd2893
+	vncipher	$out1,$out1,v25
fd2893
+	vncipher	$out2,$out2,v25
fd2893
+	vncipher	$out3,$out3,v25
fd2893
+	vncipher	$out4,$out4,v25
fd2893
+	 vxor		$twk0,$twk0,v31
fd2893
+
fd2893
+	sub		$inp,$inp,r0
fd2893
+	vncipher	$out0,$out0,v26
fd2893
+	vncipher	$out1,$out1,v26
fd2893
+	vncipher	$out2,$out2,v26
fd2893
+	vncipher	$out3,$out3,v26
fd2893
+	vncipher	$out4,$out4,v26
fd2893
+	 vxor		$in1,$twk1,v31
fd2893
+
fd2893
+	vncipher	$out0,$out0,v27
fd2893
+	lvx_u		$in0,0,$inp
fd2893
+	vncipher	$out1,$out1,v27
fd2893
+	vncipher	$out2,$out2,v27
fd2893
+	vncipher	$out3,$out3,v27
fd2893
+	vncipher	$out4,$out4,v27
fd2893
+	 vxor		$in2,$twk2,v31
fd2893
+
fd2893
+	addi		$key_,$sp,$FRAME+15	# rewind $key_
fd2893
+	vncipher	$out0,$out0,v28
fd2893
+	vncipher	$out1,$out1,v28
fd2893
+	vncipher	$out2,$out2,v28
fd2893
+	vncipher	$out3,$out3,v28
fd2893
+	vncipher	$out4,$out4,v28
fd2893
+	lvx		v24,$x00,$key_		# re-pre-load round[1]
fd2893
+	 vxor		$in3,$twk3,v31
fd2893
+
fd2893
+	vncipher	$out0,$out0,v29
fd2893
+	le?vperm	$in0,$in0,$in0,$leperm
fd2893
+	vncipher	$out1,$out1,v29
fd2893
+	vncipher	$out2,$out2,v29
fd2893
+	vncipher	$out3,$out3,v29
fd2893
+	vncipher	$out4,$out4,v29
fd2893
+	lvx		v25,$x10,$key_		# re-pre-load round[2]
fd2893
+	 vxor		$in4,$twk4,v31
fd2893
+
fd2893
+	vncipher	$out0,$out0,v30
fd2893
+	vncipher	$out1,$out1,v30
fd2893
+	vncipher	$out2,$out2,v30
fd2893
+	vncipher	$out3,$out3,v30
fd2893
+	vncipher	$out4,$out4,v30
fd2893
+
fd2893
+	vncipherlast	$out0,$out0,$twk0
fd2893
+	vncipherlast	$out1,$out1,$in1
fd2893
+	vncipherlast	$out2,$out2,$in2
fd2893
+	vncipherlast	$out3,$out3,$in3
fd2893
+	vncipherlast	$out4,$out4,$in4
fd2893
+	mtctr		$rounds
fd2893
+	blr
fd2893
+        .long   	0
fd2893
+        .byte   	0,12,0x14,0,0,0,0,0
fd2893
+___
fd2893
+}}	}}}
fd2893
+
fd2893
 my $consts=1;
fd2893
 foreach(split("\n",$code)) {
fd2893
         s/\`([^\`]*)\`/eval($1)/geo;
fd2893
diff -up openssl-1.0.2k/crypto/evp/e_aes.c.ppc-update openssl-1.0.2k/crypto/evp/e_aes.c
fd2893
--- openssl-1.0.2k/crypto/evp/e_aes.c.ppc-update	2017-03-09 17:59:26.303232439 +0100
fd2893
+++ openssl-1.0.2k/crypto/evp/e_aes.c	2017-03-09 17:59:26.314232696 +0100
fd2893
@@ -172,6 +172,8 @@ void AES_xts_decrypt(const unsigned char
fd2893
 #  define HWAES_decrypt aes_p8_decrypt
fd2893
 #  define HWAES_cbc_encrypt aes_p8_cbc_encrypt
fd2893
 #  define HWAES_ctr32_encrypt_blocks aes_p8_ctr32_encrypt_blocks
fd2893
+#  define HWAES_xts_encrypt aes_p8_xts_encrypt
fd2893
+#  define HWAES_xts_decrypt aes_p8_xts_decrypt
fd2893
 # endif
fd2893
 
fd2893
 # if     defined(AES_ASM) && !defined(I386_ONLY) &&      (  \
fd2893
@@ -911,6 +913,12 @@ void HWAES_cbc_encrypt(const unsigned ch
fd2893
 void HWAES_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out,
fd2893
                                 size_t len, const AES_KEY *key,
fd2893
                                 const unsigned char ivec[16]);
fd2893
+void HWAES_xts_encrypt(const unsigned char *inp, unsigned char *out,
fd2893
+                       size_t len, const AES_KEY *key1,
fd2893
+                       const AES_KEY *key2, const unsigned char iv[16]);
fd2893
+void HWAES_xts_decrypt(const unsigned char *inp, unsigned char *out,
fd2893
+                       size_t len, const AES_KEY *key1,
fd2893
+                       const AES_KEY *key2, const unsigned char iv[16]);
fd2893
 # endif
fd2893
 
fd2893
 # define BLOCK_CIPHER_generic_pack(nid,keylen,flags)             \
fd2893
@@ -1664,10 +1672,16 @@ static int aes_xts_init_key(EVP_CIPHER_C
fd2893
                     HWAES_set_encrypt_key(key, ctx->key_len * 4,
fd2893
                                           &xctx->ks1.ks);
fd2893
                     xctx->xts.block1 = (block128_f) HWAES_encrypt;
fd2893
+# ifdef HWAES_xts_encrypt
fd2893
+                    xctx->stream = HWAES_xts_encrypt;
fd2893
+# endif
fd2893
                 } else {
fd2893
                     HWAES_set_decrypt_key(key, ctx->key_len * 4,
fd2893
                                           &xctx->ks1.ks);
fd2893
                     xctx->xts.block1 = (block128_f) HWAES_decrypt;
fd2893
+# ifdef HWAES_xts_decrypt
fd2893
+                    xctx->stream = HWAES_xts_decrypt;
fd2893
+#endif
fd2893
                 }
fd2893
 
fd2893
                 HWAES_set_encrypt_key(key + ctx->key_len / 2,