yeahuh / rpms / qemu-kvm

Forked from rpms/qemu-kvm 2 years ago
Clone

Blame SOURCES/kvm-migration-stop-compression-to-allocate-and-free-memo.patch

ae23c9
From d9c3b12460d40900dbc2c0f4b6b03886488178d6 Mon Sep 17 00:00:00 2001
ae23c9
From: "Dr. David Alan Gilbert" <dgilbert@redhat.com>
ae23c9
Date: Wed, 1 Aug 2018 13:55:06 +0100
ae23c9
Subject: [PATCH 02/21] migration: stop compression to allocate and free memory
ae23c9
 frequently
ae23c9
ae23c9
RH-Author: Dr. David Alan Gilbert <dgilbert@redhat.com>
ae23c9
Message-id: <20180801135522.11658-3-dgilbert@redhat.com>
ae23c9
Patchwork-id: 81579
ae23c9
O-Subject: [qemu-kvm RHEL8/virt212 PATCH 02/18] migration: stop compression to allocate and free memory frequently
ae23c9
Bugzilla: 1594384
ae23c9
RH-Acked-by: Peter Xu <peterx@redhat.com>
ae23c9
RH-Acked-by: John Snow <jsnow@redhat.com>
ae23c9
RH-Acked-by: Juan Quintela <quintela@redhat.com>
ae23c9
ae23c9
From: Xiao Guangrong <xiaoguangrong@tencent.com>
ae23c9
ae23c9
Current code uses compress2() to compress memory which manages memory
ae23c9
internally, that causes huge memory is allocated and freed very
ae23c9
frequently
ae23c9
ae23c9
More worse, frequently returning memory to kernel will flush TLBs
ae23c9
and trigger invalidation callbacks on mmu-notification which
ae23c9
interacts with KVM MMU, that dramatically reduce the performance
ae23c9
of VM
ae23c9
ae23c9
So, we maintain the memory by ourselves and reuse it for each
ae23c9
compression
ae23c9
ae23c9
Reviewed-by: Peter Xu <peterx@redhat.com>
ae23c9
Reviewed-by: Jiang Biao <jiang.biao2@zte.com.cn>
ae23c9
Signed-off-by: Xiao Guangrong <xiaoguangrong@tencent.com>
ae23c9
Message-Id: <20180330075128.26919-3-xiaoguangrong@tencent.com>
ae23c9
Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
ae23c9
(cherry picked from commit dcaf446ebda5d87e05eb41cdbafb7ae4a7cc4a62)
ae23c9
Signed-off-by: Danilo C. L. de Paula <ddepaula@redhat.com>
ae23c9
---
ae23c9
 migration/qemu-file.c | 39 ++++++++++++++++++++++++++++++++-------
ae23c9
 migration/qemu-file.h |  6 ++++--
ae23c9
 migration/ram.c       | 41 ++++++++++++++++++++++++++++++++---------
ae23c9
 3 files changed, 68 insertions(+), 18 deletions(-)
ae23c9
ae23c9
diff --git a/migration/qemu-file.c b/migration/qemu-file.c
ae23c9
index bb63c77..bafe3a0 100644
ae23c9
--- a/migration/qemu-file.c
ae23c9
+++ b/migration/qemu-file.c
ae23c9
@@ -658,8 +658,32 @@ uint64_t qemu_get_be64(QEMUFile *f)
ae23c9
     return v;
ae23c9
 }
ae23c9
 
ae23c9
-/* Compress size bytes of data start at p with specific compression
ae23c9
- * level and store the compressed data to the buffer of f.
ae23c9
+/* return the size after compression, or negative value on error */
ae23c9
+static int qemu_compress_data(z_stream *stream, uint8_t *dest, size_t dest_len,
ae23c9
+                              const uint8_t *source, size_t source_len)
ae23c9
+{
ae23c9
+    int err;
ae23c9
+
ae23c9
+    err = deflateReset(stream);
ae23c9
+    if (err != Z_OK) {
ae23c9
+        return -1;
ae23c9
+    }
ae23c9
+
ae23c9
+    stream->avail_in = source_len;
ae23c9
+    stream->next_in = (uint8_t *)source;
ae23c9
+    stream->avail_out = dest_len;
ae23c9
+    stream->next_out = dest;
ae23c9
+
ae23c9
+    err = deflate(stream, Z_FINISH);
ae23c9
+    if (err != Z_STREAM_END) {
ae23c9
+        return -1;
ae23c9
+    }
ae23c9
+
ae23c9
+    return stream->next_out - dest;
ae23c9
+}
ae23c9
+
ae23c9
+/* Compress size bytes of data start at p and store the compressed
ae23c9
+ * data to the buffer of f.
ae23c9
  *
ae23c9
  * When f is not writable, return -1 if f has no space to save the
ae23c9
  * compressed data.
ae23c9
@@ -667,9 +691,8 @@ uint64_t qemu_get_be64(QEMUFile *f)
ae23c9
  * do fflush first, if f still has no space to save the compressed
ae23c9
  * data, return -1.
ae23c9
  */
ae23c9
-
ae23c9
-ssize_t qemu_put_compression_data(QEMUFile *f, const uint8_t *p, size_t size,
ae23c9
-                                  int level)
ae23c9
+ssize_t qemu_put_compression_data(QEMUFile *f, z_stream *stream,
ae23c9
+                                  const uint8_t *p, size_t size)
ae23c9
 {
ae23c9
     ssize_t blen = IO_BUF_SIZE - f->buf_index - sizeof(int32_t);
ae23c9
 
ae23c9
@@ -683,8 +706,10 @@ ssize_t qemu_put_compression_data(QEMUFile *f, const uint8_t *p, size_t size,
ae23c9
             return -1;
ae23c9
         }
ae23c9
     }
ae23c9
-    if (compress2(f->buf + f->buf_index + sizeof(int32_t), (uLongf *)&blen,
ae23c9
-                  (Bytef *)p, size, level) != Z_OK) {
ae23c9
+
ae23c9
+    blen = qemu_compress_data(stream, f->buf + f->buf_index + sizeof(int32_t),
ae23c9
+                              blen, p, size);
ae23c9
+    if (blen < 0) {
ae23c9
         error_report("Compress Failed!");
ae23c9
         return 0;
ae23c9
     }
ae23c9
diff --git a/migration/qemu-file.h b/migration/qemu-file.h
ae23c9
index f4f356a..2ccfcfb 100644
ae23c9
--- a/migration/qemu-file.h
ae23c9
+++ b/migration/qemu-file.h
ae23c9
@@ -25,6 +25,8 @@
ae23c9
 #ifndef MIGRATION_QEMU_FILE_H
ae23c9
 #define MIGRATION_QEMU_FILE_H
ae23c9
 
ae23c9
+#include <zlib.h>
ae23c9
+
ae23c9
 /* Read a chunk of data from a file at the given position.  The pos argument
ae23c9
  * can be ignored if the file is only be used for streaming.  The number of
ae23c9
  * bytes actually read should be returned.
ae23c9
@@ -132,8 +134,8 @@ bool qemu_file_is_writable(QEMUFile *f);
ae23c9
 
ae23c9
 size_t qemu_peek_buffer(QEMUFile *f, uint8_t **buf, size_t size, size_t offset);
ae23c9
 size_t qemu_get_buffer_in_place(QEMUFile *f, uint8_t **buf, size_t size);
ae23c9
-ssize_t qemu_put_compression_data(QEMUFile *f, const uint8_t *p, size_t size,
ae23c9
-                                  int level);
ae23c9
+ssize_t qemu_put_compression_data(QEMUFile *f, z_stream *stream,
ae23c9
+                                  const uint8_t *p, size_t size);
ae23c9
 int qemu_put_qemu_file(QEMUFile *f_des, QEMUFile *f_src);
ae23c9
 
ae23c9
 /*
ae23c9
diff --git a/migration/ram.c b/migration/ram.c
ae23c9
index f27038a..7d3b1da 100644
ae23c9
--- a/migration/ram.c
ae23c9
+++ b/migration/ram.c
ae23c9
@@ -269,6 +269,7 @@ struct CompressParam {
ae23c9
     QemuCond cond;
ae23c9
     RAMBlock *block;
ae23c9
     ram_addr_t offset;
ae23c9
+    z_stream stream;
ae23c9
 };
ae23c9
 typedef struct CompressParam CompressParam;
ae23c9
 
ae23c9
@@ -299,7 +300,7 @@ static QemuThread *decompress_threads;
ae23c9
 static QemuMutex decomp_done_lock;
ae23c9
 static QemuCond decomp_done_cond;
ae23c9
 
ae23c9
-static int do_compress_ram_page(QEMUFile *f, RAMBlock *block,
ae23c9
+static int do_compress_ram_page(QEMUFile *f, z_stream *stream, RAMBlock *block,
ae23c9
                                 ram_addr_t offset);
ae23c9
 
ae23c9
 static void *do_data_compress(void *opaque)
ae23c9
@@ -316,7 +317,7 @@ static void *do_data_compress(void *opaque)
ae23c9
             param->block = NULL;
ae23c9
             qemu_mutex_unlock(&param->mutex);
ae23c9
 
ae23c9
-            do_compress_ram_page(param->file, block, offset);
ae23c9
+            do_compress_ram_page(param->file, &param->stream, block, offset);
ae23c9
 
ae23c9
             qemu_mutex_lock(&comp_done_lock);
ae23c9
             param->done = true;
ae23c9
@@ -357,10 +358,19 @@ static void compress_threads_save_cleanup(void)
ae23c9
     terminate_compression_threads();
ae23c9
     thread_count = migrate_compress_threads();
ae23c9
     for (i = 0; i < thread_count; i++) {
ae23c9
+        /*
ae23c9
+         * we use it as a indicator which shows if the thread is
ae23c9
+         * properly init'd or not
ae23c9
+         */
ae23c9
+        if (!comp_param[i].file) {
ae23c9
+            break;
ae23c9
+        }
ae23c9
         qemu_thread_join(compress_threads + i);
ae23c9
-        qemu_fclose(comp_param[i].file);
ae23c9
         qemu_mutex_destroy(&comp_param[i].mutex);
ae23c9
         qemu_cond_destroy(&comp_param[i].cond);
ae23c9
+        deflateEnd(&comp_param[i].stream);
ae23c9
+        qemu_fclose(comp_param[i].file);
ae23c9
+        comp_param[i].file = NULL;
ae23c9
     }
ae23c9
     qemu_mutex_destroy(&comp_done_lock);
ae23c9
     qemu_cond_destroy(&comp_done_cond);
ae23c9
@@ -370,12 +380,12 @@ static void compress_threads_save_cleanup(void)
ae23c9
     comp_param = NULL;
ae23c9
 }
ae23c9
 
ae23c9
-static void compress_threads_save_setup(void)
ae23c9
+static int compress_threads_save_setup(void)
ae23c9
 {
ae23c9
     int i, thread_count;
ae23c9
 
ae23c9
     if (!migrate_use_compression()) {
ae23c9
-        return;
ae23c9
+        return 0;
ae23c9
     }
ae23c9
     thread_count = migrate_compress_threads();
ae23c9
     compress_threads = g_new0(QemuThread, thread_count);
ae23c9
@@ -383,6 +393,11 @@ static void compress_threads_save_setup(void)
ae23c9
     qemu_cond_init(&comp_done_cond);
ae23c9
     qemu_mutex_init(&comp_done_lock);
ae23c9
     for (i = 0; i < thread_count; i++) {
ae23c9
+        if (deflateInit(&comp_param[i].stream,
ae23c9
+                        migrate_compress_level()) != Z_OK) {
ae23c9
+            goto exit;
ae23c9
+        }
ae23c9
+
ae23c9
         /* comp_param[i].file is just used as a dummy buffer to save data,
ae23c9
          * set its ops to empty.
ae23c9
          */
ae23c9
@@ -395,6 +410,11 @@ static void compress_threads_save_setup(void)
ae23c9
                            do_data_compress, comp_param + i,
ae23c9
                            QEMU_THREAD_JOINABLE);
ae23c9
     }
ae23c9
+    return 0;
ae23c9
+
ae23c9
+exit:
ae23c9
+    compress_threads_save_cleanup();
ae23c9
+    return -1;
ae23c9
 }
ae23c9
 
ae23c9
 /* Multiple fd's */
ae23c9
@@ -1032,7 +1052,7 @@ static int ram_save_page(RAMState *rs, PageSearchStatus *pss, bool last_stage)
ae23c9
     return pages;
ae23c9
 }
ae23c9
 
ae23c9
-static int do_compress_ram_page(QEMUFile *f, RAMBlock *block,
ae23c9
+static int do_compress_ram_page(QEMUFile *f, z_stream *stream, RAMBlock *block,
ae23c9
                                 ram_addr_t offset)
ae23c9
 {
ae23c9
     RAMState *rs = ram_state;
ae23c9
@@ -1041,8 +1061,7 @@ static int do_compress_ram_page(QEMUFile *f, RAMBlock *block,
ae23c9
 
ae23c9
     bytes_sent = save_page_header(rs, f, block, offset |
ae23c9
                                   RAM_SAVE_FLAG_COMPRESS_PAGE);
ae23c9
-    blen = qemu_put_compression_data(f, p, TARGET_PAGE_SIZE,
ae23c9
-                                     migrate_compress_level());
ae23c9
+    blen = qemu_put_compression_data(f, stream, p, TARGET_PAGE_SIZE);
ae23c9
     if (blen < 0) {
ae23c9
         bytes_sent = 0;
ae23c9
         qemu_file_set_error(migrate_get_current()->to_dst_file, blen);
ae23c9
@@ -2215,9 +2234,14 @@ static int ram_save_setup(QEMUFile *f, void *opaque)
ae23c9
     RAMState **rsp = opaque;
ae23c9
     RAMBlock *block;
ae23c9
 
ae23c9
+    if (compress_threads_save_setup()) {
ae23c9
+        return -1;
ae23c9
+    }
ae23c9
+
ae23c9
     /* migration has already setup the bitmap, reuse it. */
ae23c9
     if (!migration_in_colo_state()) {
ae23c9
         if (ram_init_all(rsp) != 0) {
ae23c9
+            compress_threads_save_cleanup();
ae23c9
             return -1;
ae23c9
         }
ae23c9
     }
ae23c9
@@ -2237,7 +2261,6 @@ static int ram_save_setup(QEMUFile *f, void *opaque)
ae23c9
     }
ae23c9
 
ae23c9
     rcu_read_unlock();
ae23c9
-    compress_threads_save_setup();
ae23c9
 
ae23c9
     ram_control_before_iterate(f, RAM_CONTROL_SETUP);
ae23c9
     ram_control_after_iterate(f, RAM_CONTROL_SETUP);
ae23c9
-- 
ae23c9
1.8.3.1
ae23c9