From 5d7481d76a57e533f521a5d99ba8d35b5d69625c Mon Sep 17 00:00:00 2001

Message-Id: <5d7481d76a57e533f521a5d99ba8d35b5d69625c.1387382496.git.minovotn@redhat.com>

In-Reply-To: <c5386144fbf09f628148101bc674e2421cdd16e3.1387382496.git.minovotn@redhat.com>

References: <c5386144fbf09f628148101bc674e2421cdd16e3.1387382496.git.minovotn@redhat.com>

From: Nigel Croxon <ncroxon@redhat.com>

Date: Thu, 14 Nov 2013 22:52:51 +0100

Subject: [PATCH 15/46] rdma: core logic

RH-Author: Nigel Croxon <ncroxon@redhat.com>

Message-id: <1384469598-13137-16-git-send-email-ncroxon@redhat.com>

Patchwork-id: 55697

O-Subject: [RHEL7.0 PATCH 15/42] rdma: core logic

Bugzilla: 1011720

RH-Acked-by: Orit Wasserman <owasserm@redhat.com>

RH-Acked-by: Amit Shah <amit.shah@redhat.com>

RH-Acked-by: Paolo Bonzini <pbonzini@redhat.com>

Bugzilla: 1011720

https://bugzilla.redhat.com/show_bug.cgi?id=1011720

>From commit ID:

commit 2da776db4846eadcb808598a5d3484d149773c05

Author: Michael R. Hines <mrhines@us.ibm.com>

Date:   Mon Jul 22 10:01:54 2013 -0400

    rdma: core logic

    Code that does need to be visible is kept

    well contained inside this file and this is the only

    new additional file to the entire patch.

    This file includes the entire protocol and interfaces

    required to perform RDMA migration.

    Also, the configure and Makefile modifications to link

    this file are included.

    Full documentation is in docs/rdma.txt

    Reviewed-by: Paolo Bonzini <pbonzini@redhat.com>

    Reviewed-by: Chegu Vinod <chegu_vinod@hp.com>

    Tested-by: Chegu Vinod <chegu_vinod@hp.com>

    Tested-by: Michael R. Hines <mrhines@us.ibm.com>

    Signed-off-by: Michael R. Hines <mrhines@us.ibm.com>

    Signed-off-by: Juan Quintela <quintela@redhat.com>

---

 Makefile.objs                 |    1 +

 configure                     |   40 +

 include/migration/migration.h |    4 +

 migration-rdma.c              | 3249 +++++++++++++++++++++++++++++++++++++++++

 migration.c                   |    8 +

 5 files changed, 3302 insertions(+), 0 deletions(-)

 create mode 100644 migration-rdma.c

Signed-off-by: Michal Novotny <minovotn@redhat.com>

---

 Makefile.objs                 |    1 +

 configure                     |   40 +

 include/migration/migration.h |    4 +

 migration-rdma.c              | 3249 +++++++++++++++++++++++++++++++++++++++++

 migration.c                   |    8 +

 5 files changed, 3302 insertions(+)

 create mode 100644 migration-rdma.c

diff --git a/Makefile.objs b/Makefile.objs

index 286ce06..67b4a28 100644

--- a/Makefile.objs

+++ b/Makefile.objs

@@ -50,6 +50,7 @@ common-obj-$(CONFIG_POSIX) += os-posix.o

 common-obj-$(CONFIG_LINUX) += fsdev/

 common-obj-y += migration.o migration-tcp.o

+common-obj-$(CONFIG_RDMA) += migration-rdma.o

 common-obj-y += qemu-char.o #aio.o

 common-obj-y += block-migration.o

 common-obj-y += page_cache.o xbzrle.o

diff --git a/configure b/configure

index 0a729ac..33235c4 100755

--- a/configure

+++ b/configure

@@ -181,6 +181,7 @@ xfs=""

 vhost_net="no"

 vhost_scsi="no"

 kvm="no"

+rdma=""

 gprof="no"

 debug_tcg="no"

 debug="no"

@@ -925,6 +926,10 @@ for opt do

   ;;

   --enable-gtk) gtk="yes"

   ;;

+  --enable-rdma) rdma="yes"

+  ;;

+  --disable-rdma) rdma="no"

+  ;;

   --with-gtkabi=*) gtkabi="$optarg"

   ;;

   --enable-tpm) tpm="yes"

@@ -1151,6 +1156,8 @@ echo "  --enable-bluez           enable bluez stack connectivity"

 echo "  --disable-slirp          disable SLIRP userspace network connectivity"

 echo "  --disable-kvm            disable KVM acceleration support"

 echo "  --enable-kvm             enable KVM acceleration support"

+echo "  --disable-rdma           disable RDMA-based migration support"

+echo "  --enable-rdma            enable RDMA-based migration support"

 echo "  --enable-tcg-interpreter enable TCG with bytecode interpreter (TCI)"

 echo "  --disable-nptl           disable usermode NPTL support"

 echo "  --enable-nptl            enable usermode NPTL support"

@@ -1846,6 +1853,30 @@ EOF

 fi

 ##########################################

+# RDMA needs OpenFabrics libraries

+if test "$rdma" != "no" ; then

+  cat > $TMPC <

+#include <rdma/rdma_cma.h>

+int main(void) { return 0; }

+EOF

+  rdma_libs="-lrdmacm -libverbs"

+  if compile_prog "" "$rdma_libs" ; then

+    rdma="yes"

+    libs_softmmu="$libs_softmmu $rdma_libs"

+  else

+    if test "$rdma" = "yes" ; then

+        error_exit \

+            " OpenFabrics librdmacm/libibverbs not present." \

+            " Your options:" \

+            "  (1) Fast: Install infiniband packages from your distro." \

+            "  (2) Cleanest: Install libraries from www.openfabrics.org" \

+            "  (3) Also: Install softiwarp if you don't have RDMA hardware"

+    fi

+    rdma="no"

+  fi

+fi

+

+##########################################

 # VNC TLS/WS detection

 if test "$vnc" = "yes" -a \( "$vnc_tls" != "no" -o "$vnc_ws" != "no" \) ; then

   cat > $TMPC <

@@ -3550,6 +3581,7 @@ echo "Linux AIO support $linux_aio"

 echo "ATTR/XATTR support $attr"

 echo "Install blobs     $blobs"

 echo "KVM support       $kvm"

+echo "RDMA support      $rdma"

 echo "TCG interpreter   $tcg_interpreter"

 echo "fdt support       $fdt"

 echo "preadv support    $preadv"

@@ -4032,6 +4064,10 @@ if test "$trace_default" = "yes"; then

   echo "CONFIG_TRACE_DEFAULT=y" >> $config_host_mak

 fi

+if test "$rdma" = "yes" ; then

+  echo "CONFIG_RDMA=y" >> $config_host_mak

+fi

+

 if test "$tcg_interpreter" = "yes"; then

   QEMU_INCLUDES="-I\$(SRC_PATH)/tcg/tci $QEMU_INCLUDES"

 elif test "$ARCH" = "sparc64" ; then

@@ -4506,6 +4542,10 @@ if [ "$pixman" = "internal" ]; then

   echo "config-host.h: subdir-pixman" >> $config_host_mak

 fi

+if test "$rdma" = "yes" ; then

+echo "CONFIG_RDMA=y" >> $config_host_mak

+fi

+

 if [ "$dtc_internal" = "yes" ]; then

   echo "config-host.h: subdir-dtc" >> $config_host_mak

 fi

diff --git a/include/migration/migration.h b/include/migration/migration.h

index 90b5021..13a9629 100644

--- a/include/migration/migration.h

+++ b/include/migration/migration.h

@@ -77,6 +77,10 @@ void fd_start_incoming_migration(const char *path, Error **errp);

 void fd_start_outgoing_migration(MigrationState *s, const char *fdname, Error **errp);

+void rdma_start_outgoing_migration(void *opaque, const char *host_port, Error **errp);

+

+void rdma_start_incoming_migration(const char *host_port, Error **errp);

+

 void migrate_fd_error(MigrationState *s);

 void migrate_fd_connect(MigrationState *s);

diff --git a/migration-rdma.c b/migration-rdma.c

new file mode 100644

index 0000000..d044830

--- /dev/null

+++ b/migration-rdma.c

@@ -0,0 +1,3249 @@

+/*

+ * RDMA protocol and interfaces

+ *

+ * Copyright IBM, Corp. 2010-2013

+ *

+ * Authors:

+ *  Michael R. Hines <mrhines@us.ibm.com>

+ *  Jiuxing Liu <jl@us.ibm.com>

+ *

+ * This work is licensed under the terms of the GNU GPL, version 2 or

+ * later.  See the COPYING file in the top-level directory.

+ *

+ */

+#include "qemu-common.h"

+#include "migration/migration.h"

+#include "migration/qemu-file.h"

+#include "exec/cpu-common.h"

+#include "qemu/main-loop.h"

+#include "qemu/sockets.h"

+#include "qemu/bitmap.h"

+#include "block/coroutine.h"

+#include <stdio.h>

+#include <sys/types.h>

+#include <sys/socket.h>

+#include <netdb.h>

+#include <arpa/inet.h>

+#include <string.h>

+#include <rdma/rdma_cma.h>

+

+#define DEBUG_RDMA

+//#define DEBUG_RDMA_VERBOSE

+//#define DEBUG_RDMA_REALLY_VERBOSE

+

+#ifdef DEBUG_RDMA

+#define DPRINTF(fmt, ...) \

+    do { printf("rdma: " fmt, ## __VA_ARGS__); } while (0)

+#else

+#define DPRINTF(fmt, ...) \

+    do { } while (0)

+#endif

+

+#ifdef DEBUG_RDMA_VERBOSE

+#define DDPRINTF(fmt, ...) \

+    do { printf("rdma: " fmt, ## __VA_ARGS__); } while (0)

+#else

+#define DDPRINTF(fmt, ...) \

+    do { } while (0)

+#endif

+

+#ifdef DEBUG_RDMA_REALLY_VERBOSE

+#define DDDPRINTF(fmt, ...) \

+    do { printf("rdma: " fmt, ## __VA_ARGS__); } while (0)

+#else

+#define DDDPRINTF(fmt, ...) \

+    do { } while (0)

+#endif

+

+/*

+ * Print and error on both the Monitor and the Log file.

+ */

+#define ERROR(errp, fmt, ...) \

+    do { \

+        fprintf(stderr, "RDMA ERROR: " fmt, ## __VA_ARGS__); \

+        if (errp && (*(errp) == NULL)) { \

+            error_setg(errp, "RDMA ERROR: " fmt, ## __VA_ARGS__); \

+        } \

+    } while (0)

+

+#define RDMA_RESOLVE_TIMEOUT_MS 10000

+

+/* Do not merge data if larger than this. */

+#define RDMA_MERGE_MAX (2 * 1024 * 1024)

+#define RDMA_SIGNALED_SEND_MAX (RDMA_MERGE_MAX / 4096)

+

+#define RDMA_REG_CHUNK_SHIFT 20 /* 1 MB */

+

+/*

+ * This is only for non-live state being migrated.

+ * Instead of RDMA_WRITE messages, we use RDMA_SEND

+ * messages for that state, which requires a different

+ * delivery design than main memory.

+ */

+#define RDMA_SEND_INCREMENT 32768

+

+/*

+ * Maximum size infiniband SEND message

+ */

+#define RDMA_CONTROL_MAX_BUFFER (512 * 1024)

+#define RDMA_CONTROL_MAX_COMMANDS_PER_MESSAGE 4096

+

+#define RDMA_CONTROL_VERSION_CURRENT 1

+/*

+ * Capabilities for negotiation.

+ */

+#define RDMA_CAPABILITY_PIN_ALL 0x01

+

+/*

+ * Add the other flags above to this list of known capabilities

+ * as they are introduced.

+ */

+static uint32_t known_capabilities = RDMA_CAPABILITY_PIN_ALL;

+

+#define CHECK_ERROR_STATE() \

+    do { \

+        if (rdma->error_state) { \

+            if (!rdma->error_reported) { \

+                fprintf(stderr, "RDMA is in an error state waiting migration" \

+                                " to abort!\n"); \

+                rdma->error_reported = 1; \

+            } \

+            return rdma->error_state; \

+        } \

+    } while (0);

+

+/*

+ * A work request ID is 64-bits and we split up these bits

+ * into 3 parts:

+ *

+ * bits 0-15 : type of control message, 2^16

+ * bits 16-29: ram block index, 2^14

+ * bits 30-63: ram block chunk number, 2^34

+ *

+ * The last two bit ranges are only used for RDMA writes,

+ * in order to track their completion and potentially

+ * also track unregistration status of the message.

+ */

+#define RDMA_WRID_TYPE_SHIFT  0UL

+#define RDMA_WRID_BLOCK_SHIFT 16UL

+#define RDMA_WRID_CHUNK_SHIFT 30UL

+

+#define RDMA_WRID_TYPE_MASK \

+    ((1UL << RDMA_WRID_BLOCK_SHIFT) - 1UL)

+

+#define RDMA_WRID_BLOCK_MASK \

+    (~RDMA_WRID_TYPE_MASK & ((1UL << RDMA_WRID_CHUNK_SHIFT) - 1UL))

+

+#define RDMA_WRID_CHUNK_MASK (~RDMA_WRID_BLOCK_MASK & ~RDMA_WRID_TYPE_MASK)

+

+/*

+ * RDMA migration protocol:

+ * 1. RDMA Writes (data messages, i.e. RAM)

+ * 2. IB Send/Recv (control channel messages)

+ */

+enum {

+    RDMA_WRID_NONE = 0,

+    RDMA_WRID_RDMA_WRITE = 1,

+    RDMA_WRID_SEND_CONTROL = 2000,

+    RDMA_WRID_RECV_CONTROL = 4000,

+};

+

+const char *wrid_desc[] = {

+    [RDMA_WRID_NONE] = "NONE",

+    [RDMA_WRID_RDMA_WRITE] = "WRITE RDMA",

+    [RDMA_WRID_SEND_CONTROL] = "CONTROL SEND",

+    [RDMA_WRID_RECV_CONTROL] = "CONTROL RECV",

+};

+

+/*

+ * Work request IDs for IB SEND messages only (not RDMA writes).

+ * This is used by the migration protocol to transmit

+ * control messages (such as device state and registration commands)

+ *

+ * We could use more WRs, but we have enough for now.

+ */

+enum {

+    RDMA_WRID_READY = 0,

+    RDMA_WRID_DATA,

+    RDMA_WRID_CONTROL,

+    RDMA_WRID_MAX,

+};

+

+/*

+ * SEND/RECV IB Control Messages.

+ */

+enum {

+    RDMA_CONTROL_NONE = 0,

+    RDMA_CONTROL_ERROR,

+    RDMA_CONTROL_READY,               /* ready to receive */

+    RDMA_CONTROL_QEMU_FILE,           /* QEMUFile-transmitted bytes */

+    RDMA_CONTROL_RAM_BLOCKS_REQUEST,  /* RAMBlock synchronization */

+    RDMA_CONTROL_RAM_BLOCKS_RESULT,   /* RAMBlock synchronization */

+    RDMA_CONTROL_COMPRESS,            /* page contains repeat values */

+    RDMA_CONTROL_REGISTER_REQUEST,    /* dynamic page registration */

+    RDMA_CONTROL_REGISTER_RESULT,     /* key to use after registration */

+    RDMA_CONTROL_REGISTER_FINISHED,   /* current iteration finished */

+    RDMA_CONTROL_UNREGISTER_REQUEST,  /* dynamic UN-registration */

+    RDMA_CONTROL_UNREGISTER_FINISHED, /* unpinning finished */

+};

+

+const char *control_desc[] = {

+    [RDMA_CONTROL_NONE] = "NONE",

+    [RDMA_CONTROL_ERROR] = "ERROR",

+    [RDMA_CONTROL_READY] = "READY",

+    [RDMA_CONTROL_QEMU_FILE] = "QEMU FILE",

+    [RDMA_CONTROL_RAM_BLOCKS_REQUEST] = "RAM BLOCKS REQUEST",

+    [RDMA_CONTROL_RAM_BLOCKS_RESULT] = "RAM BLOCKS RESULT",

+    [RDMA_CONTROL_COMPRESS] = "COMPRESS",

+    [RDMA_CONTROL_REGISTER_REQUEST] = "REGISTER REQUEST",

+    [RDMA_CONTROL_REGISTER_RESULT] = "REGISTER RESULT",

+    [RDMA_CONTROL_REGISTER_FINISHED] = "REGISTER FINISHED",

+    [RDMA_CONTROL_UNREGISTER_REQUEST] = "UNREGISTER REQUEST",

+    [RDMA_CONTROL_UNREGISTER_FINISHED] = "UNREGISTER FINISHED",

+};

+

+/*

+ * Memory and MR structures used to represent an IB Send/Recv work request.

+ * This is *not* used for RDMA writes, only IB Send/Recv.

+ */

+typedef struct {

+    uint8_t  control[RDMA_CONTROL_MAX_BUFFER]; /* actual buffer to register */

+    struct   ibv_mr *control_mr;               /* registration metadata */

+    size_t   control_len;                      /* length of the message */

+    uint8_t *control_curr;                     /* start of unconsumed bytes */

+} RDMAWorkRequestData;

+

+/*

+ * Negotiate RDMA capabilities during connection-setup time.

+ */

+typedef struct {

+    uint32_t version;

+    uint32_t flags;

+} RDMACapabilities;

+

+static void caps_to_network(RDMACapabilities *cap)

+{

+    cap->version = htonl(cap->version);

+    cap->flags = htonl(cap->flags);

+}

+

+static void network_to_caps(RDMACapabilities *cap)

+{

+    cap->version = ntohl(cap->version);

+    cap->flags = ntohl(cap->flags);

+}

+

+/*

+ * Representation of a RAMBlock from an RDMA perspective.

+ * This is not transmitted, only local.

+ * This and subsequent structures cannot be linked lists

+ * because we're using a single IB message to transmit

+ * the information. It's small anyway, so a list is overkill.

+ */

+typedef struct RDMALocalBlock {

+    uint8_t  *local_host_addr; /* local virtual address */

+    uint64_t remote_host_addr; /* remote virtual address */

+    uint64_t offset;

+    uint64_t length;

+    struct   ibv_mr **pmr;     /* MRs for chunk-level registration */

+    struct   ibv_mr *mr;       /* MR for non-chunk-level registration */

+    uint32_t *remote_keys;     /* rkeys for chunk-level registration */

+    uint32_t remote_rkey;      /* rkeys for non-chunk-level registration */

+    int      index;            /* which block are we */

+    bool     is_ram_block;

+    int      nb_chunks;

+    unsigned long *transit_bitmap;

+    unsigned long *unregister_bitmap;

+} RDMALocalBlock;

+

+/*

+ * Also represents a RAMblock, but only on the dest.

+ * This gets transmitted by the dest during connection-time

+ * to the source VM and then is used to populate the

+ * corresponding RDMALocalBlock with

+ * the information needed to perform the actual RDMA.

+ */

+typedef struct QEMU_PACKED RDMARemoteBlock {

+    uint64_t remote_host_addr;

+    uint64_t offset;

+    uint64_t length;

+    uint32_t remote_rkey;

+    uint32_t padding;

+} RDMARemoteBlock;

+

+static uint64_t htonll(uint64_t v)

+{

+    union { uint32_t lv[2]; uint64_t llv; } u;

+    u.lv[0] = htonl(v >> 32);

+    u.lv[1] = htonl(v & 0xFFFFFFFFULL);

+    return u.llv;

+}

+

+static uint64_t ntohll(uint64_t v) {

+    union { uint32_t lv[2]; uint64_t llv; } u;

+    u.llv = v;

+    return ((uint64_t)ntohl(u.lv[0]) << 32) | (uint64_t) ntohl(u.lv[1]);

+}

+

+static void remote_block_to_network(RDMARemoteBlock *rb)

+{

+    rb->remote_host_addr = htonll(rb->remote_host_addr);

+    rb->offset = htonll(rb->offset);

+    rb->length = htonll(rb->length);

+    rb->remote_rkey = htonl(rb->remote_rkey);

+}

+

+static void network_to_remote_block(RDMARemoteBlock *rb)

+{

+    rb->remote_host_addr = ntohll(rb->remote_host_addr);

+    rb->offset = ntohll(rb->offset);

+    rb->length = ntohll(rb->length);

+    rb->remote_rkey = ntohl(rb->remote_rkey);

+}

+

+/*

+ * Virtual address of the above structures used for transmitting

+ * the RAMBlock descriptions at connection-time.

+ * This structure is *not* transmitted.

+ */

+typedef struct RDMALocalBlocks {

+    int nb_blocks;

+    bool     init;             /* main memory init complete */

+    RDMALocalBlock *block;

+} RDMALocalBlocks;

+

+/*

+ * Main data structure for RDMA state.

+ * While there is only one copy of this structure being allocated right now,

+ * this is the place where one would start if you wanted to consider

+ * having more than one RDMA connection open at the same time.

+ */

+typedef struct RDMAContext {

+    char *host;

+    int port;

+

+    RDMAWorkRequestData wr_data[RDMA_WRID_MAX + 1];

+

+    /*

+     * This is used by *_exchange_send() to figure out whether or not

+     * the initial "READY" message has already been received or not.

+     * This is because other functions may potentially poll() and detect

+     * the READY message before send() does, in which case we need to

+     * know if it completed.

+     */

+    int control_ready_expected;

+

+    /* number of outstanding writes */

+    int nb_sent;

+

+    /* store info about current buffer so that we can

+       merge it with future sends */

+    uint64_t current_addr;

+    uint64_t current_length;

+    /* index of ram block the current buffer belongs to */

+    int current_index;

+    /* index of the chunk in the current ram block */

+    int current_chunk;

+

+    bool pin_all;

+

+    /*

+     * infiniband-specific variables for opening the device

+     * and maintaining connection state and so forth.

+     *

+     * cm_id also has ibv_context, rdma_event_channel, and ibv_qp in

+     * cm_id->verbs, cm_id->channel, and cm_id->qp.

+     */

+    struct rdma_cm_id *cm_id;               /* connection manager ID */

+    struct rdma_cm_id *listen_id;

+

+    struct ibv_context          *verbs;

+    struct rdma_event_channel   *channel;

+    struct ibv_qp *qp;                      /* queue pair */

+    struct ibv_comp_channel *comp_channel;  /* completion channel */

+    struct ibv_pd *pd;                      /* protection domain */

+    struct ibv_cq *cq;                      /* completion queue */

+

+    /*

+     * If a previous write failed (perhaps because of a failed

+     * memory registration, then do not attempt any future work

+     * and remember the error state.

+     */

+    int error_state;

+    int error_reported;

+

+    /*

+     * Description of ram blocks used throughout the code.

+     */

+    RDMALocalBlocks local_ram_blocks;

+    RDMARemoteBlock *block;

+

+    /*

+     * Migration on *destination* started.

+     * Then use coroutine yield function.

+     * Source runs in a thread, so we don't care.

+     */

+    int migration_started_on_destination;

+

+    int total_registrations;

+    int total_writes;

+

+    int unregister_current, unregister_next;

+    uint64_t unregistrations[RDMA_SIGNALED_SEND_MAX];

+

+    GHashTable *blockmap;

+} RDMAContext;

+

+/*

+ * Interface to the rest of the migration call stack.

+ */

+typedef struct QEMUFileRDMA {

+    RDMAContext *rdma;

+    size_t len;

+    void *file;

+} QEMUFileRDMA;

+

+/*

+ * Main structure for IB Send/Recv control messages.

+ * This gets prepended at the beginning of every Send/Recv.

+ */

+typedef struct QEMU_PACKED {

+    uint32_t len;     /* Total length of data portion */

+    uint32_t type;    /* which control command to perform */

+    uint32_t repeat;  /* number of commands in data portion of same type */

+    uint32_t padding;

+} RDMAControlHeader;

+

+static void control_to_network(RDMAControlHeader *control)

+{

+    control->type = htonl(control->type);

+    control->len = htonl(control->len);

+    control->repeat = htonl(control->repeat);

+}

+

+static void network_to_control(RDMAControlHeader *control)

+{

+    control->type = ntohl(control->type);

+    control->len = ntohl(control->len);

+    control->repeat = ntohl(control->repeat);

+}

+

+/*

+ * Register a single Chunk.

+ * Information sent by the source VM to inform the dest

+ * to register an single chunk of memory before we can perform

+ * the actual RDMA operation.

+ */

+typedef struct QEMU_PACKED {

+    union QEMU_PACKED {

+        uint64_t current_addr;  /* offset into the ramblock of the chunk */

+        uint64_t chunk;         /* chunk to lookup if unregistering */

+    } key;

+    uint32_t current_index; /* which ramblock the chunk belongs to */

+    uint32_t padding;

+    uint64_t chunks;            /* how many sequential chunks to register */

+} RDMARegister;

+

+static void register_to_network(RDMARegister *reg)

+{

+    reg->key.current_addr = htonll(reg->key.current_addr);

+    reg->current_index = htonl(reg->current_index);

+    reg->chunks = htonll(reg->chunks);

+}

+

+static void network_to_register(RDMARegister *reg)

+{

+    reg->key.current_addr = ntohll(reg->key.current_addr);

+    reg->current_index = ntohl(reg->current_index);

+    reg->chunks = ntohll(reg->chunks);

+}

+

+typedef struct QEMU_PACKED {

+    uint32_t value;     /* if zero, we will madvise() */

+    uint32_t block_idx; /* which ram block index */

+    uint64_t offset;    /* where in the remote ramblock this chunk */

+    uint64_t length;    /* length of the chunk */

+} RDMACompress;

+

+static void compress_to_network(RDMACompress *comp)

+{

+    comp->value = htonl(comp->value);

+    comp->block_idx = htonl(comp->block_idx);

+    comp->offset = htonll(comp->offset);

+    comp->length = htonll(comp->length);

+}

+

+static void network_to_compress(RDMACompress *comp)

+{

+    comp->value = ntohl(comp->value);

+    comp->block_idx = ntohl(comp->block_idx);

+    comp->offset = ntohll(comp->offset);

+    comp->length = ntohll(comp->length);

+}

+

+/*

+ * The result of the dest's memory registration produces an "rkey"

+ * which the source VM must reference in order to perform

+ * the RDMA operation.

+ */

+typedef struct QEMU_PACKED {

+    uint32_t rkey;

+    uint32_t padding;

+    uint64_t host_addr;

+} RDMARegisterResult;

+

+static void result_to_network(RDMARegisterResult *result)

+{

+    result->rkey = htonl(result->rkey);

+    result->host_addr = htonll(result->host_addr);

+};

+

+static void network_to_result(RDMARegisterResult *result)

+{

+    result->rkey = ntohl(result->rkey);

+    result->host_addr = ntohll(result->host_addr);

+};

+

+const char *print_wrid(int wrid);

+static int qemu_rdma_exchange_send(RDMAContext *rdma, RDMAControlHeader *head,

+                                   uint8_t *data, RDMAControlHeader *resp,

+                                   int *resp_idx,

+                                   int (*callback)(RDMAContext *rdma));

+

+static inline uint64_t ram_chunk_index(uint8_t *start, uint8_t *host)

+{

+    return ((uintptr_t) host - (uintptr_t) start) >> RDMA_REG_CHUNK_SHIFT;

+}

+

+static inline uint8_t *ram_chunk_start(RDMALocalBlock *rdma_ram_block,

+                                       uint64_t i)

+{

+    return (uint8_t *) (((uintptr_t) rdma_ram_block->local_host_addr)

+                                    + (i << RDMA_REG_CHUNK_SHIFT));

+}

+

+static inline uint8_t *ram_chunk_end(RDMALocalBlock *rdma_ram_block, uint64_t i)

+{

+    uint8_t *result = ram_chunk_start(rdma_ram_block, i) +

+                                         (1UL << RDMA_REG_CHUNK_SHIFT);

+