Pablo Greco e6a3ae
From 8e7643e39fc0c539f55d85263e87f64dca52efea Mon Sep 17 00:00:00 2001
Pablo Greco e6a3ae
From: "Daniel P. Berrange" <berrange@redhat.com>
Pablo Greco e6a3ae
Date: Wed, 24 Apr 2019 09:56:39 +0100
Pablo Greco e6a3ae
Subject: [PATCH 5/9] crypto: convert xts_tweak_encdec to use xts_uint128 type
Pablo Greco e6a3ae
MIME-Version: 1.0
Pablo Greco e6a3ae
Content-Type: text/plain; charset=UTF-8
Pablo Greco e6a3ae
Content-Transfer-Encoding: 8bit
Pablo Greco e6a3ae
Pablo Greco e6a3ae
RH-Author: Daniel P. Berrange <berrange@redhat.com>
Pablo Greco e6a3ae
Message-id: <20190424095643.796-6-berrange@redhat.com>
Pablo Greco e6a3ae
Patchwork-id: 85883
Pablo Greco e6a3ae
O-Subject: [RHEL-8.1.0 qemu-kvm PATCH 5/9] crypto: convert xts_tweak_encdec to use xts_uint128 type
Pablo Greco e6a3ae
Bugzilla: 1680231
Pablo Greco e6a3ae
RH-Acked-by: Philippe Mathieu-Daudé <philmd@redhat.com>
Pablo Greco e6a3ae
RH-Acked-by: John Snow <jsnow@redhat.com>
Pablo Greco e6a3ae
RH-Acked-by: Eric Blake <eblake@redhat.com>
Pablo Greco e6a3ae
Pablo Greco e6a3ae
Using 64-bit arithmetic increases the performance for xts-aes-128
Pablo Greco e6a3ae
when built with gcrypt:
Pablo Greco e6a3ae
Pablo Greco e6a3ae
  Encrypt: 272 MB/s -> 355 MB/s
Pablo Greco e6a3ae
  Decrypt: 275 MB/s -> 362 MB/s
Pablo Greco e6a3ae
Pablo Greco e6a3ae
Reviewed-by: Alberto Garcia <berto@igalia.com>
Pablo Greco e6a3ae
Signed-off-by: Daniel P. Berrangé <berrange@redhat.com>
Pablo Greco e6a3ae
(cherry picked from commit db217c69f0849add67cfa2cd6601c329398be12c)
Pablo Greco e6a3ae
Signed-off-by: Danilo C. L. de Paula <ddepaula@redhat.com>
Pablo Greco e6a3ae
---
Pablo Greco e6a3ae
 crypto/xts.c | 84 +++++++++++++++++++++++++++++++++++++++++-------------------
Pablo Greco e6a3ae
 1 file changed, 58 insertions(+), 26 deletions(-)
Pablo Greco e6a3ae
Pablo Greco e6a3ae
diff --git a/crypto/xts.c b/crypto/xts.c
Pablo Greco e6a3ae
index bee23f8..0ad231f 100644
Pablo Greco e6a3ae
--- a/crypto/xts.c
Pablo Greco e6a3ae
+++ b/crypto/xts.c
Pablo Greco e6a3ae
@@ -31,6 +31,13 @@ typedef union {
Pablo Greco e6a3ae
     uint64_t u[2];
Pablo Greco e6a3ae
 } xts_uint128;
Pablo Greco e6a3ae
 
Pablo Greco e6a3ae
+static inline void xts_uint128_xor(xts_uint128 *D,
Pablo Greco e6a3ae
+                                   const xts_uint128 *S1,
Pablo Greco e6a3ae
+                                   const xts_uint128 *S2)
Pablo Greco e6a3ae
+{
Pablo Greco e6a3ae
+    D->u[0] = S1->u[0] ^ S2->u[0];
Pablo Greco e6a3ae
+    D->u[1] = S1->u[1] ^ S2->u[1];
Pablo Greco e6a3ae
+}
Pablo Greco e6a3ae
 
Pablo Greco e6a3ae
 static void xts_mult_x(uint8_t *I)
Pablo Greco e6a3ae
 {
Pablo Greco e6a3ae
@@ -60,25 +67,19 @@ static void xts_mult_x(uint8_t *I)
Pablo Greco e6a3ae
  */
Pablo Greco e6a3ae
 static void xts_tweak_encdec(const void *ctx,
Pablo Greco e6a3ae
                              xts_cipher_func *func,
Pablo Greco e6a3ae
-                             const uint8_t *src,
Pablo Greco e6a3ae
-                             uint8_t *dst,
Pablo Greco e6a3ae
-                             uint8_t *iv)
Pablo Greco e6a3ae
+                             const xts_uint128 *src,
Pablo Greco e6a3ae
+                             xts_uint128 *dst,
Pablo Greco e6a3ae
+                             xts_uint128 *iv)
Pablo Greco e6a3ae
 {
Pablo Greco e6a3ae
-    unsigned long x;
Pablo Greco e6a3ae
-
Pablo Greco e6a3ae
     /* tweak encrypt block i */
Pablo Greco e6a3ae
-    for (x = 0; x < XTS_BLOCK_SIZE; x++) {
Pablo Greco e6a3ae
-        dst[x] = src[x] ^ iv[x];
Pablo Greco e6a3ae
-    }
Pablo Greco e6a3ae
+    xts_uint128_xor(dst, src, iv);
Pablo Greco e6a3ae
 
Pablo Greco e6a3ae
-    func(ctx, XTS_BLOCK_SIZE, dst, dst);
Pablo Greco e6a3ae
+    func(ctx, XTS_BLOCK_SIZE, dst->b, dst->b);
Pablo Greco e6a3ae
 
Pablo Greco e6a3ae
-    for (x = 0; x < XTS_BLOCK_SIZE; x++) {
Pablo Greco e6a3ae
-        dst[x] = dst[x] ^ iv[x];
Pablo Greco e6a3ae
-    }
Pablo Greco e6a3ae
+    xts_uint128_xor(dst, dst, iv);
Pablo Greco e6a3ae
 
Pablo Greco e6a3ae
     /* LFSR the tweak */
Pablo Greco e6a3ae
-    xts_mult_x(iv);
Pablo Greco e6a3ae
+    xts_mult_x(iv->b);
Pablo Greco e6a3ae
 }
Pablo Greco e6a3ae
 
Pablo Greco e6a3ae
 
Pablo Greco e6a3ae
@@ -110,20 +111,34 @@ void xts_decrypt(const void *datactx,
Pablo Greco e6a3ae
     /* encrypt the iv */
Pablo Greco e6a3ae
     encfunc(tweakctx, XTS_BLOCK_SIZE, T.b, iv);
Pablo Greco e6a3ae
 
Pablo Greco e6a3ae
-    for (i = 0; i < lim; i++) {
Pablo Greco e6a3ae
-        xts_tweak_encdec(datactx, decfunc, src, dst, T.b);
Pablo Greco e6a3ae
-
Pablo Greco e6a3ae
-        src += XTS_BLOCK_SIZE;
Pablo Greco e6a3ae
-        dst += XTS_BLOCK_SIZE;
Pablo Greco e6a3ae
+    if (QEMU_PTR_IS_ALIGNED(src, sizeof(uint64_t)) &&
Pablo Greco e6a3ae
+        QEMU_PTR_IS_ALIGNED(dst, sizeof(uint64_t))) {
Pablo Greco e6a3ae
+        xts_uint128 *S = (xts_uint128 *)src;
Pablo Greco e6a3ae
+        xts_uint128 *D = (xts_uint128 *)dst;
Pablo Greco e6a3ae
+        for (i = 0; i < lim; i++, S++, D++) {
Pablo Greco e6a3ae
+            xts_tweak_encdec(datactx, decfunc, S, D, &T);
Pablo Greco e6a3ae
+        }
Pablo Greco e6a3ae
+    } else {
Pablo Greco e6a3ae
+        xts_uint128 D;
Pablo Greco e6a3ae
+
Pablo Greco e6a3ae
+        for (i = 0; i < lim; i++) {
Pablo Greco e6a3ae
+            memcpy(&D, src, XTS_BLOCK_SIZE);
Pablo Greco e6a3ae
+            xts_tweak_encdec(datactx, decfunc, &D, &D, &T);
Pablo Greco e6a3ae
+            memcpy(dst, &D, XTS_BLOCK_SIZE);
Pablo Greco e6a3ae
+            src += XTS_BLOCK_SIZE;
Pablo Greco e6a3ae
+            dst += XTS_BLOCK_SIZE;
Pablo Greco e6a3ae
+        }
Pablo Greco e6a3ae
     }
Pablo Greco e6a3ae
 
Pablo Greco e6a3ae
     /* if length is not a multiple of XTS_BLOCK_SIZE then */
Pablo Greco e6a3ae
     if (mo > 0) {
Pablo Greco e6a3ae
+        xts_uint128 S, D;
Pablo Greco e6a3ae
         memcpy(&CC, &T, XTS_BLOCK_SIZE);
Pablo Greco e6a3ae
         xts_mult_x(CC.b);
Pablo Greco e6a3ae
 
Pablo Greco e6a3ae
         /* PP = tweak decrypt block m-1 */
Pablo Greco e6a3ae
-        xts_tweak_encdec(datactx, decfunc, src, PP.b, CC.b);
Pablo Greco e6a3ae
+        memcpy(&S, src, XTS_BLOCK_SIZE);
Pablo Greco e6a3ae
+        xts_tweak_encdec(datactx, decfunc, &S, &PP, &CC);
Pablo Greco e6a3ae
 
Pablo Greco e6a3ae
         /* Pm = first length % XTS_BLOCK_SIZE bytes of PP */
Pablo Greco e6a3ae
         for (i = 0; i < mo; i++) {
Pablo Greco e6a3ae
@@ -135,7 +150,8 @@ void xts_decrypt(const void *datactx,
Pablo Greco e6a3ae
         }
Pablo Greco e6a3ae
 
Pablo Greco e6a3ae
         /* Pm-1 = Tweak uncrypt CC */
Pablo Greco e6a3ae
-        xts_tweak_encdec(datactx, decfunc, CC.b, dst, T.b);
Pablo Greco e6a3ae
+        xts_tweak_encdec(datactx, decfunc, &CC, &D, &T);
Pablo Greco e6a3ae
+        memcpy(dst, &D, XTS_BLOCK_SIZE);
Pablo Greco e6a3ae
     }
Pablo Greco e6a3ae
 
Pablo Greco e6a3ae
     /* Decrypt the iv back */
Pablo Greco e6a3ae
@@ -171,17 +187,32 @@ void xts_encrypt(const void *datactx,
Pablo Greco e6a3ae
     /* encrypt the iv */
Pablo Greco e6a3ae
     encfunc(tweakctx, XTS_BLOCK_SIZE, T.b, iv);
Pablo Greco e6a3ae
 
Pablo Greco e6a3ae
-    for (i = 0; i < lim; i++) {
Pablo Greco e6a3ae
-        xts_tweak_encdec(datactx, encfunc, src, dst, T.b);
Pablo Greco e6a3ae
+    if (QEMU_PTR_IS_ALIGNED(src, sizeof(uint64_t)) &&
Pablo Greco e6a3ae
+        QEMU_PTR_IS_ALIGNED(dst, sizeof(uint64_t))) {
Pablo Greco e6a3ae
+        xts_uint128 *S = (xts_uint128 *)src;
Pablo Greco e6a3ae
+        xts_uint128 *D = (xts_uint128 *)dst;
Pablo Greco e6a3ae
+        for (i = 0; i < lim; i++, S++, D++) {
Pablo Greco e6a3ae
+            xts_tweak_encdec(datactx, encfunc, S, D, &T);
Pablo Greco e6a3ae
+        }
Pablo Greco e6a3ae
+    } else {
Pablo Greco e6a3ae
+        xts_uint128 D;
Pablo Greco e6a3ae
+
Pablo Greco e6a3ae
+        for (i = 0; i < lim; i++) {
Pablo Greco e6a3ae
+            memcpy(&D, src, XTS_BLOCK_SIZE);
Pablo Greco e6a3ae
+            xts_tweak_encdec(datactx, encfunc, &D, &D, &T);
Pablo Greco e6a3ae
+            memcpy(dst, &D, XTS_BLOCK_SIZE);
Pablo Greco e6a3ae
 
Pablo Greco e6a3ae
-        dst += XTS_BLOCK_SIZE;
Pablo Greco e6a3ae
-        src += XTS_BLOCK_SIZE;
Pablo Greco e6a3ae
+            dst += XTS_BLOCK_SIZE;
Pablo Greco e6a3ae
+            src += XTS_BLOCK_SIZE;
Pablo Greco e6a3ae
+        }
Pablo Greco e6a3ae
     }
Pablo Greco e6a3ae
 
Pablo Greco e6a3ae
     /* if length is not a multiple of XTS_BLOCK_SIZE then */
Pablo Greco e6a3ae
     if (mo > 0) {
Pablo Greco e6a3ae
+        xts_uint128 S, D;
Pablo Greco e6a3ae
         /* CC = tweak encrypt block m-1 */
Pablo Greco e6a3ae
-        xts_tweak_encdec(datactx, encfunc, src, CC.b, T.b);
Pablo Greco e6a3ae
+        memcpy(&S, src, XTS_BLOCK_SIZE);
Pablo Greco e6a3ae
+        xts_tweak_encdec(datactx, encfunc, &S, &CC, &T);
Pablo Greco e6a3ae
 
Pablo Greco e6a3ae
         /* Cm = first length % XTS_BLOCK_SIZE bytes of CC */
Pablo Greco e6a3ae
         for (i = 0; i < mo; i++) {
Pablo Greco e6a3ae
@@ -194,7 +225,8 @@ void xts_encrypt(const void *datactx,
Pablo Greco e6a3ae
         }
Pablo Greco e6a3ae
 
Pablo Greco e6a3ae
         /* Cm-1 = Tweak encrypt PP */
Pablo Greco e6a3ae
-        xts_tweak_encdec(datactx, encfunc, PP.b, dst, T.b);
Pablo Greco e6a3ae
+        xts_tweak_encdec(datactx, encfunc, &PP, &D, &T);
Pablo Greco e6a3ae
+        memcpy(dst, &D, XTS_BLOCK_SIZE);
Pablo Greco e6a3ae
     }
Pablo Greco e6a3ae
 
Pablo Greco e6a3ae
     /* Decrypt the iv back */
Pablo Greco e6a3ae
-- 
Pablo Greco e6a3ae
1.8.3.1
Pablo Greco e6a3ae