From 1b7bbc5ff814406dfe9f64203462d92d8f71ef3a Mon Sep 17 00:00:00 2001

From: Jeffrey Cody <jcody@redhat.com>

Date: Mon, 8 May 2017 22:11:16 +0200

Subject: [PATCH 6/7] block/vxhs: modularize VXHS via g_module

RH-Author: Jeffrey Cody <jcody@redhat.com>

Message-id: <8a91a423440b7a5a14e868279c772e99b865bfc6.1494281291.git.jcody@redhat.com>

Patchwork-id: 75046

O-Subject: [RHEV-7.4 qemu-kvm-rhev 4/4] block/vxhs: modularize VXHS via g_module

Bugzilla: 1265869

RH-Acked-by: Stefan Hajnoczi <stefanha@redhat.com>

RH-Acked-by: Laurent Vivier <lvivier@redhat.com>

RH-Acked-by: Miroslav Rezanina <mrezanin@redhat.com>

This converts the VXHS driver to be a runtime loaded module, with no

external build dependencies on libvxhs / libqnio.   This (libvxhs) is a

3rd party library, written by Veritas, to interface with the VXHS

server.

Red Hat is not going to distribute libvxhs, nor will Red Hat use libvxhs

in the build systems.  So that creates two criteria for the

modularization, for business reasons:

1. No runtime dependencies on libvxhs (aside from opening the library on

vxhs open)

2. No build dependencies on libvxhs packages.

There is support in QEMU for modular block drivers, however there are

two issues with using the built-in support:

    A. It is all-or-none; if modules are enabled all protocols are built

       as modules.  This wouldn't be that bad, as it would of course

       enable more granular dependencies for qemu rpm packages. But...

    B. It is not designed with criteria #2 in mind; it reduces runtime

       dependencies, not build dependencies.  The protocol libraries

       that are still built linked against external libraries and using

       external headers.

This patch uses g_module to load the libvxhs library, and incorporates

the libvxhs.h header in the build tree.  If block driver protocols are

also built as modules, libvxhs will still work and be built as a module

too, except the shared library will still not have any dependency on

libvxhs.

There are a few changes in this patch from upstream (aside from the

module loading aspects):

1. It enables VXHS support to be built as a protocl module if

--enable-modules is used during configure.

2. If the init call to iio_init() fails, populate errp with a

meaningful error message.

3. Since we are loading the library dynamically, make check the min and

max supported versions in the libvxhs library on load.

Patches for items #1 and #2 have been posted upstream.

It is expected that the libvxhs library is located at the following

pathname: /usr/lib64/qemu/libvxhs.so.1

VXHS support is only built for x86_64 in RHEV.

Signed-off-by: Jeff Cody <jcody@redhat.com>

Signed-off-by: Miroslav Rezanina <mrezanin@redhat.com>

---

 block/vxhs.c                  | 122 +++++++++++++++++++++++++++++++----

 configure                     |  33 +---------

 include/block/vxhs_shim.h     | 143 ++++++++++++++++++++++++++++++++++++++++++

 redhat/build_configure.sh     |   9 ++-

 redhat/qemu-kvm.spec.template |   7 +++

 5 files changed, 272 insertions(+), 42 deletions(-)

 create mode 100644 include/block/vxhs_shim.h

diff --git a/block/vxhs.c b/block/vxhs.c

index 9ffe9d3..fbce83e 100644

--- a/block/vxhs.c

+++ b/block/vxhs.c

@@ -9,7 +9,8 @@

  */

 #include "qemu/osdep.h"

-#include <qnio/qnio_api.h>

+#include "block/vxhs_shim.h"

+#include <gmodule.h>

 #include <sys/param.h>

 #include "block/block_int.h"

 #include "qapi/qmp/qerror.h"

@@ -58,6 +59,96 @@ typedef struct BDRVVXHSState {

     char *tlscredsid; /* tlscredsid */

 } BDRVVXHSState;

+#define LIBVXHS_FULL_PATHNAME "/usr/lib64/qemu/libvxhs.so.1"

+static bool libvxhs_loaded;

+static GModule *libvxhs_handle;

+

+static LibVXHSFuncs libvxhs;

+

+typedef struct LibVXHSSymbols {

+    const char *name;

+    gpointer *addr;

+} LibVXHSSymbols;

+

+static LibVXHSSymbols libvxhs_symbols[] = {

+    {"iio_init",        (gpointer *) &libvxhs.iio_init},

+    {"iio_fini",        (gpointer *) &libvxhs.iio_fini},

+    {"iio_min_version", (gpointer *) &libvxhs.iio_min_version},

+    {"iio_max_version", (gpointer *) &libvxhs.iio_max_version},

+    {"iio_open",        (gpointer *) &libvxhs.iio_open},

+    {"iio_close",       (gpointer *) &libvxhs.iio_close},

+    {"iio_writev",      (gpointer *) &libvxhs.iio_writev},

+    {"iio_readv",       (gpointer *) &libvxhs.iio_readv},

+    {"iio_ioctl",       (gpointer *) &libvxhs.iio_ioctl},

+    {NULL}

+};

+

+static void bdrv_vxhs_set_funcs(GModule *handle, Error **errp)

+{

+    int i = 0;

+    while (libvxhs_symbols[i].name) {

+        const char *name = libvxhs_symbols[i].name;

+        if (!g_module_symbol(handle, name, libvxhs_symbols[i].addr)) {

+            error_setg(errp, "%s could not be loaded from libvxhs: %s",

+                       name, g_module_error());

+            return;

+        }

+        ++i;

+    }

+}

+

+static void bdrv_vxhs_load_libs(Error **errp)

+{

+    Error *local_err = NULL;

+    int32_t ver;

+

+    if (libvxhs_loaded) {

+        return;

+    }

+

+    if (!g_module_supported()) {

+        error_setg(errp, "modules are not supported on this platform: %s",

+                     g_module_error());

+        return;

+    }

+

+    libvxhs_handle = g_module_open(LIBVXHS_FULL_PATHNAME,

+                                   G_MODULE_BIND_LAZY | G_MODULE_BIND_LOCAL);

+    if (!libvxhs_handle) {

+        error_setg(errp, "error loading libvxhs: %s", g_module_error());

+        return;

+    }

+

+    g_module_make_resident(libvxhs_handle);

+

+    bdrv_vxhs_set_funcs(libvxhs_handle, &local_err);

+    if (local_err) {

+        error_propagate(errp, local_err);

+        return;

+    }

+

+    /* Now check to see if the libvxhs we are using here is supported

+     * by the loaded version */

+

+    ver = (*libvxhs.iio_min_version)();

+    if (ver > QNIO_VERSION) {

+        error_setg(errp, "Trying to use libvxhs version %"PRId32" API, but "

+                         "only %"PRId32" or newer is supported by %s",

+                          QNIO_VERSION, ver, LIBVXHS_FULL_PATHNAME);

+        return;

+    }

+

+    ver = (*libvxhs.iio_max_version)();

+    if (ver < QNIO_VERSION) {

+        error_setg(errp, "Trying to use libvxhs version %"PRId32" API, but "

+                         "only %"PRId32" or earlier is supported by %s",

+                          QNIO_VERSION, ver, LIBVXHS_FULL_PATHNAME);

+        return;

+    }

+

+    libvxhs_loaded = true;

+}

+

 static void vxhs_complete_aio_bh(void *opaque)

 {

     VXHSAIOCB *acb = opaque;

@@ -219,7 +310,7 @@ static void vxhs_parse_filename(const char *filename, QDict *options,

 static int vxhs_init_and_ref(void)

 {

     if (vxhs_ref++ == 0) {

-        if (iio_init(QNIO_VERSION, vxhs_iio_callback)) {

+        if ((*libvxhs.iio_init)(QNIO_VERSION, vxhs_iio_callback)) {

             return -ENODEV;

         }

     }

@@ -229,7 +320,7 @@ static int vxhs_init_and_ref(void)

 static void vxhs_unref(void)

 {

     if (--vxhs_ref == 0) {

-        iio_fini();

+        (*libvxhs.iio_fini)();

     }

 }

@@ -299,8 +390,17 @@ static int vxhs_open(BlockDriverState *bs, QDict *options,

     char *client_key = NULL;

     char *client_cert = NULL;

+    bdrv_vxhs_load_libs(&local_err);

+    if (local_err) {

+        error_propagate(errp, local_err);

+        /* on error, cannot cleanup because the iio_fini() function

+         * is not loaded */

+        return -EINVAL;

+    }

+

     ret = vxhs_init_and_ref();

     if (ret < 0) {

+        error_setg(&local_err, "libvxhs iio_init() failed");

         ret = -EINVAL;

         goto out;

     }

@@ -385,8 +485,8 @@ static int vxhs_open(BlockDriverState *bs, QDict *options,

     /*

      * Open qnio channel to storage agent if not opened before

      */

-    dev_handlep = iio_open(of_vsa_addr, s->vdisk_guid, 0,

-                           cacert, client_key, client_cert);

+    dev_handlep = (*libvxhs.iio_open)(of_vsa_addr, s->vdisk_guid, 0,

+                                      cacert, client_key, client_cert);

     if (dev_handlep == NULL) {

         trace_vxhs_open_iio_open(of_vsa_addr);

         ret = -ENODEV;

@@ -450,12 +550,12 @@ static BlockAIOCB *vxhs_aio_rw(BlockDriverState *bs, int64_t sector_num,

     switch (iodir) {

     case VDISK_AIO_WRITE:

-            ret = iio_writev(dev_handle, acb, qiov->iov, qiov->niov,

-                             offset, (uint64_t)size, iio_flags);

+            ret = (*libvxhs.iio_writev)(dev_handle, acb, qiov->iov, qiov->niov,

+                                        offset, (uint64_t)size, iio_flags);

             break;

     case VDISK_AIO_READ:

-            ret = iio_readv(dev_handle, acb, qiov->iov, qiov->niov,

-                            offset, (uint64_t)size, iio_flags);

+            ret = (*libvxhs.iio_readv)(dev_handle, acb, qiov->iov, qiov->niov,

+                                       offset, (uint64_t)size, iio_flags);

             break;

     default:

             trace_vxhs_aio_rw_invalid(iodir);

@@ -505,7 +605,7 @@ static void vxhs_close(BlockDriverState *bs)

      * Close vDisk device

      */

     if (s->vdisk_hostinfo.dev_handle) {

-        iio_close(s->vdisk_hostinfo.dev_handle);

+        (*libvxhs.iio_close)(s->vdisk_hostinfo.dev_handle);

         s->vdisk_hostinfo.dev_handle = NULL;

     }

@@ -527,7 +627,7 @@ static int64_t vxhs_get_vdisk_stat(BDRVVXHSState *s)

     int ret = 0;

     void *dev_handle = s->vdisk_hostinfo.dev_handle;

-    ret = iio_ioctl(dev_handle, IOR_VDISK_STAT, &vdisk_size, 0);

+    ret = (*libvxhs.iio_ioctl)(dev_handle, IOR_VDISK_STAT, &vdisk_size, 0);

     if (ret < 0) {

         trace_vxhs_get_vdisk_stat_err(s->vdisk_guid, ret, errno);

         return -EIO;

diff --git a/configure b/configure

index 47b2475..2ca95f4 100755

--- a/configure

+++ b/configure

@@ -3091,7 +3091,7 @@ else

     glib_req_ver=2.22

 fi

 glib_modules=gthread-2.0

-if test "$modules" = yes; then

+if test "$modules" = yes -o "$vxhs" = yes; then

     glib_modules="$glib_modules gmodule-2.0"

 fi

@@ -4793,33 +4793,6 @@ if compile_prog "" "" ; then

 fi

 ##########################################

-# Veritas HyperScale block driver VxHS

-# Check if libvxhs is installed

-

-if test "$vxhs" != "no" ; then

-  cat > $TMPC <

-#include <stdint.h>

-#include <qnio/qnio_api.h>

-

-void *vxhs_callback;

-

-int main(void) {

-    iio_init(QNIO_VERSION, vxhs_callback);

-    return 0;

-}

-EOF

-  vxhs_libs="-lvxhs -lssl"

-  if compile_prog "" "$vxhs_libs" ; then

-    vxhs=yes

-  else

-    if test "$vxhs" = "yes" ; then

-      feature_not_found "vxhs block device" "Install libvxhs See github"

-    fi

-    vxhs=no

-  fi

-fi

-

-##########################################

 # End of CC checks

 # After here, no more $cc or $ld runs

@@ -5831,8 +5804,8 @@ if test "$pthread_setname_np" = "yes" ; then

 fi

 if test "$vxhs" = "yes" ; then

-  echo "CONFIG_VXHS=y" >> $config_host_mak

-  echo "VXHS_LIBS=$vxhs_libs" >> $config_host_mak

+  echo "CONFIG_VXHS=m" >> $config_host_mak

+  echo "VXHS_LIBS= -lssl" >> $config_host_mak

 fi

 if test "$tcg_interpreter" = "yes"; then

diff --git a/include/block/vxhs_shim.h b/include/block/vxhs_shim.h

new file mode 100644

index 0000000..42519ae

--- /dev/null

+++ b/include/block/vxhs_shim.h

@@ -0,0 +1,143 @@

+/*

+ * Network IO library for VxHS QEMU block driver (Veritas Technologies)

+ *

+ * This work is licensed under the terms of the GNU GPL, version 2.  See

+ * the COPYING file in the top-level directory.

+ *

+ * Contributions after 2014-08-15 are licensed under the terms of the

+ * GNU GPL, version 2 or (at your option) any later version.

+ */

+

+#ifndef QNIO_API_H

+#define QNIO_API_H

+

+#include <sys/uio.h>

+

+/*

+ * Bump up the version everytime this file is modified

+ */

+#define QNIO_VERSION    34

+

+/*

+ * These are the opcodes referenced by callback routine.

+ */

+#define IRP_READ_REQUEST                    0x1FFF

+#define IRP_WRITE_REQUEST                   0x2FFF

+#define IRP_VDISK_CHECK_IO_FAILOVER_READY   2020

+

+/*

+ * opcodes for iio_ioctl.

+ */

+#define IOR_VDISK_STAT                      1005

+

+/*

+ * Error values for iio_cb_t callback function.

+ */

+#define QNIOERROR_HUP                       901 /* Retriable error */

+#define QNIOERROR_NOCONN                    902 /* Non-retriable error */

+

+

+/* Operation Flags */

+#define IIO_FLAG_ASYNC        0x0001   /* Do an async send */

+

+/*

+ * INPUT:

+ *     ctx - opaque context

+ *     opcode - Operation

+ *     error - 0 for sucess, non-zero for failure.

+ * RETURNS:

+ *     void

+ * DESCRIPTION:

+ *     This callback is called, after Async request completes.

+ *

+ * CONTEXT:

+ *     The callback should be wait-free.

+ */

+typedef void (*iio_cb_t) (void *ctx, uint32_t opcode, uint32_t error);

+

+typedef struct LibVXHSFuncs {

+/*

+ * RETURNS:

+ *     0 for sucess, non-zero for failure.

+ * DESCRIPTION:

+ *     Intilize the library state. This should be called at the

+ *     begining before issuing any library call.

+ */

+    int     (*iio_init)(int32_t version, iio_cb_t cb);

+/*

+ * RETURNS:

+ *     void

+ * DESCRIPTION:

+ *     Relinquish library resources. This should be called on the

+ *     close of last open device.

+ */

+    void    (*iio_fini)(void);

+/*

+ * DESCRIPTION:

+ *     Returns minimum QNIO API version supported by library.

+ */

+    int32_t (*iio_min_version)(void);

+/*

+ * DESCRIPTION:

+ *     Returns maximum QNIO API version supported by library.

+ */

+    int32_t (*iio_max_version)(void);

+/*

+ * INPUT:

+ *    uri - const string of the format of://<hostname|ip>:port

+ *    devid - Device ID.

+ *    flags - currently unused, this must be set to 0

+ *    cacert - CA certificates file in PEM format

+ *    client_key - Client private key file in PEM format

+ *    client_cert - Client certificate file in PEM format

+ * RETURNS:

+ *    opeque device handle on success, NULL on failure.

+ * DESCRIPTION:

+ *    This call returns device handle on success. Returns NULL on

+ *    failure with errno set

+ *    errno can be one of:

+ *        ENODEV - remote device not found

+ *        EBADF  - Unable to open communication channel.

+ *        EBUSY  - The call cannot be completed right now

+ */

+    void   *(*iio_open)(const char *uri, const char *devid, uint32_t flags,

+                        const char *cacert, const char *client_key,

+                        const char *client_cert);

+/*

+ * Close the device.

+ *    For every matching iio_open() there should be a matching iio_close()

+ *    The last close free all data structures associated with the device.

+ */

+    int32_t (*iio_close)(void *dev_handle);

+/*

+ * INPUT:

+ *    dev_handle - device descriptor on which read/write needs to be performed

+ *    ctx - an opaque context that is not interpreted This is set for

+ *          async calls only. It can be NULL.

+ *    iov    - an array of iovecs (This is a scatter gather operation)

+ *    iovcnt  - the number of iovecs

+ *    offset - an offset to perform the write

+ *    size   - I/O size

+ *    flags  - can be one of

+ *        IIO_FLAG_ASYNC - indicating this is a aio call.

+ * RETURNS:

+ *        -1 on error, sets errno

+ *        EBADF  - the remote fd is bad

+ *        EBUSY  - The call cannot be completed right now

+ *        EPIPE  - the channel got disconnected, call back would be called in

+ *                 addition to this.

+ */

+    int32_t (*iio_writev)(void *dev_handle, void *ctx, struct iovec *iov,

+                          int iovcnt, uint64_t offset, uint64_t size,

+                          uint32_t flags);

+

+    int32_t (*iio_readv)(void *dev_handle, void *ctx, struct iovec *iov,

+                         int iovcnt, uint64_t offset, uint64_t size,

+                         uint32_t flags);

+

+    int32_t (*iio_ioctl)(void *dev_handle, uint32_t opcode, void *opaque,

+                         uint32_t flags);

+

+} LibVXHSFuncs;

+

+#endif

-- 

1.8.3.1