From e58ce3910208fead9e24eb08e19a11bb2eba2f1e Mon Sep 17 00:00:00 2001
From: "Danilo C. L. de Paula" <ddepaula@redhat.com>
Date: Wed, 7 Mar 2018 13:05:43 -0300
Subject: block/vxhs: modularize VXHS via g_module
RH-Author: Jeffrey Cody <jcody@redhat.com>
Message-id: <8a91a423440b7a5a14e868279c772e99b865bfc6.1494281291.git.jcody@redhat.com>
Patchwork-id: 75046
O-Subject: [RHEV-7.4 qemu-kvm-rhev 4/4] block/vxhs: modularize VXHS via g_module Bugzilla: 1265869
RH-Acked-by: Stefan Hajnoczi <stefanha@redhat.com>
RH-Acked-by: Laurent Vivier <lvivier@redhat.com>
RH-Acked-by: Miroslav Rezanina <mrezanin@redhat.com>
This converts the VXHS driver to be a runtime loaded module, with no
external build dependencies on libvxhs / libqnio. This (libvxhs) is a
3rd party library, written by Veritas, to interface with the VXHS
server.
Red Hat is not going to distribute libvxhs, nor will Red Hat use libvxhs
in the build systems. So that creates two criteria for the
modularization, for business reasons:
1. No runtime dependencies on libvxhs (aside from opening the library on
vxhs open)
2. No build dependencies on libvxhs packages.
There is support in QEMU for modular block drivers, however there are
two issues with using the built-in support:
A. It is all-or-none; if modules are enabled all protocols are built
as modules. This wouldn't be that bad, as it would of course
enable more granular dependencies for qemu rpm packages. But...
B. It is not designed with criteria #2 in mind; it reduces runtime
dependencies, not build dependencies. The protocol libraries
that are still built linked against external libraries and using
external headers.
This patch uses g_module to load the libvxhs library, and incorporates
the libvxhs.h header in the build tree. If block driver protocols are
also built as modules, libvxhs will still work and be built as a module
too, except the shared library will still not have any dependency on
libvxhs.
There are a few changes in this patch from upstream (aside from the
module loading aspects):
1. It enables VXHS support to be built as a protocl module if
--enable-modules is used during configure.
2. If the init call to iio_init() fails, populate errp with a
meaningful error message.
3. Since we are loading the library dynamically, make check the min and
max supported versions in the libvxhs library on load.
Patches for items #1 and #2 have been posted upstream.
It is expected that the libvxhs library is located at the following
pathname: /usr/lib64/qemu/libvxhs.so.1
VXHS support is only built for x86_64 in RHEV.
Signed-off-by: Jeff Cody <jcody@redhat.com>
Signed-off-by: Miroslav Rezanina <mrezanin@redhat.com>
block/vxhs: improve error message for missing / bad vxhs module
RH-Author: Jeffrey Cody <jcody@redhat.com>
Message-id: <59af10d83125fff42beacd30dbca83d50409bbed.1513031708.git.jcody@redhat.com>
Patchwork-id: 78305
O-Subject: [RHV7.5 qemu-kvm-rhev PATCH 1/1] block/vxhs: improve error message for missing / bad vxhs module
Bugzilla: 1505654
RH-Acked-by: Markus Armbruster <armbru@redhat.com>
RH-Acked-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
RH-Acked-by: Stefan Hajnoczi <stefanha@redhat.com>
RH-Acked-by: Laszlo Ersek <lersek@redhat.com>
[Downstream only, as the module load of libvxhs is downstream only]
In the case of missing libvxhs libraries, the original error message,
while technically accurate, may lead a user to think there is a QEMU bug
if trying to using the VXHS protocol. Update the message so that it is
clear that the likely issue is that the Veritas QEMU libvxhs RPM is not
installed (or not installed correctly, if there are permission or file
corruption issues, etc.).
An example error message before this change:
> qemu-img info vxhs://localhost/test
qemu-img: Could not open 'vxhs://localhost/test': \
error loading libvxhs: /usr/lib64/qemu/libvxhs.so.1: \
cannot open shared object file: No such file or directory
An example error message after this change:
> qemu-img info vxhs://localhost/test
qemu-img: Could not open 'vxhs://localhost/test': \
The VXHS library from Veritas might not be installed correctly \
(/usr/lib64/qemu/libvxhs.so.1: \
cannot open shared object file: No such file or directory)
Signed-off-by: Jeff Cody <jcody@redhat.com>
Signed-off-by: Miroslav Rezanina <mrezanin@redhat.com>
---
block/vxhs.c | 123 ++++++++++++++++++++++++++++++++----
configure | 33 +---------
include/block/vxhs_shim.h | 143 ++++++++++++++++++++++++++++++++++++++++++
redhat/build_configure.sh | 9 ++-
redhat/qemu-kvm.spec.template | 7 +++
5 files changed, 273 insertions(+), 42 deletions(-)
create mode 100644 include/block/vxhs_shim.h
diff --git a/block/vxhs.c b/block/vxhs.c
index 75cc6c8..68edb51 100644
--- a/block/vxhs.c
+++ b/block/vxhs.c
@@ -9,7 +9,8 @@
*/
#include "qemu/osdep.h"
-#include <qnio/qnio_api.h>
+#include "block/vxhs_shim.h"
+#include <gmodule.h>
#include <sys/param.h>
#include "block/block_int.h"
#include "qapi/qmp/qerror.h"
@@ -58,6 +59,97 @@ typedef struct BDRVVXHSState {
char *tlscredsid; /* tlscredsid */
} BDRVVXHSState;
+#define LIBVXHS_FULL_PATHNAME "/usr/lib64/qemu/libvxhs.so.1"
+static bool libvxhs_loaded;
+static GModule *libvxhs_handle;
+
+static LibVXHSFuncs libvxhs;
+
+typedef struct LibVXHSSymbols {
+ const char *name;
+ gpointer *addr;
+} LibVXHSSymbols;
+
+static LibVXHSSymbols libvxhs_symbols[] = {
+ {"iio_init", (gpointer *) &libvxhs.iio_init},
+ {"iio_fini", (gpointer *) &libvxhs.iio_fini},
+ {"iio_min_version", (gpointer *) &libvxhs.iio_min_version},
+ {"iio_max_version", (gpointer *) &libvxhs.iio_max_version},
+ {"iio_open", (gpointer *) &libvxhs.iio_open},
+ {"iio_close", (gpointer *) &libvxhs.iio_close},
+ {"iio_writev", (gpointer *) &libvxhs.iio_writev},
+ {"iio_readv", (gpointer *) &libvxhs.iio_readv},
+ {"iio_ioctl", (gpointer *) &libvxhs.iio_ioctl},
+ {NULL}
+};
+
+static void bdrv_vxhs_set_funcs(GModule *handle, Error **errp)
+{
+ int i = 0;
+ while (libvxhs_symbols[i].name) {
+ const char *name = libvxhs_symbols[i].name;
+ if (!g_module_symbol(handle, name, libvxhs_symbols[i].addr)) {
+ error_setg(errp, "%s could not be loaded from libvxhs: %s",
+ name, g_module_error());
+ return;
+ }
+ ++i;
+ }
+}
+
+static void bdrv_vxhs_load_libs(Error **errp)
+{
+ Error *local_err = NULL;
+ int32_t ver;
+
+ if (libvxhs_loaded) {
+ return;
+ }
+
+ if (!g_module_supported()) {
+ error_setg(errp, "modules are not supported on this platform: %s",
+ g_module_error());
+ return;
+ }
+
+ libvxhs_handle = g_module_open(LIBVXHS_FULL_PATHNAME,
+ G_MODULE_BIND_LAZY | G_MODULE_BIND_LOCAL);
+ if (!libvxhs_handle) {
+ error_setg(errp, "The VXHS library from Veritas might not be installed "
+ "correctly (%s)", g_module_error());
+ return;
+ }
+
+ g_module_make_resident(libvxhs_handle);
+
+ bdrv_vxhs_set_funcs(libvxhs_handle, &local_err);
+ if (local_err) {
+ error_propagate(errp, local_err);
+ return;
+ }
+
+ /* Now check to see if the libvxhs we are using here is supported
+ * by the loaded version */
+
+ ver = (*libvxhs.iio_min_version)();
+ if (ver > QNIO_VERSION) {
+ error_setg(errp, "Trying to use libvxhs version %"PRId32" API, but "
+ "only %"PRId32" or newer is supported by %s",
+ QNIO_VERSION, ver, LIBVXHS_FULL_PATHNAME);
+ return;
+ }
+
+ ver = (*libvxhs.iio_max_version)();
+ if (ver < QNIO_VERSION) {
+ error_setg(errp, "Trying to use libvxhs version %"PRId32" API, but "
+ "only %"PRId32" or earlier is supported by %s",
+ QNIO_VERSION, ver, LIBVXHS_FULL_PATHNAME);
+ return;
+ }
+
+ libvxhs_loaded = true;
+}
+
static void vxhs_complete_aio_bh(void *opaque)
{
VXHSAIOCB *acb = opaque;
@@ -219,7 +311,7 @@ static void vxhs_parse_filename(const char *filename, QDict *options,
static int vxhs_init_and_ref(void)
{
if (vxhs_ref++ == 0) {
- if (iio_init(QNIO_VERSION, vxhs_iio_callback)) {
+ if ((*libvxhs.iio_init)(QNIO_VERSION, vxhs_iio_callback)) {
return -ENODEV;
}
}
@@ -229,7 +321,7 @@ static int vxhs_init_and_ref(void)
static void vxhs_unref(void)
{
if (--vxhs_ref == 0) {
- iio_fini();
+ (*libvxhs.iio_fini)();
}
}
@@ -299,8 +391,17 @@ static int vxhs_open(BlockDriverState *bs, QDict *options,
char *client_key = NULL;
char *client_cert = NULL;
+ bdrv_vxhs_load_libs(&local_err);
+ if (local_err) {
+ error_propagate(errp, local_err);
+ /* on error, cannot cleanup because the iio_fini() function
+ * is not loaded */
+ return -EINVAL;
+ }
+
ret = vxhs_init_and_ref();
if (ret < 0) {
+ error_setg(&local_err, "libvxhs iio_init() failed");
ret = -EINVAL;
goto out;
}
@@ -385,8 +486,8 @@ static int vxhs_open(BlockDriverState *bs, QDict *options,
/*
* Open qnio channel to storage agent if not opened before
*/
- dev_handlep = iio_open(of_vsa_addr, s->vdisk_guid, 0,
- cacert, client_key, client_cert);
+ dev_handlep = (*libvxhs.iio_open)(of_vsa_addr, s->vdisk_guid, 0,
+ cacert, client_key, client_cert);
if (dev_handlep == NULL) {
trace_vxhs_open_iio_open(of_vsa_addr);
ret = -ENODEV;
@@ -450,12 +551,12 @@ static BlockAIOCB *vxhs_aio_rw(BlockDriverState *bs, int64_t sector_num,
switch (iodir) {
case VDISK_AIO_WRITE:
- ret = iio_writev(dev_handle, acb, qiov->iov, qiov->niov,
- offset, (uint64_t)size, iio_flags);
+ ret = (*libvxhs.iio_writev)(dev_handle, acb, qiov->iov, qiov->niov,
+ offset, (uint64_t)size, iio_flags);
break;
case VDISK_AIO_READ:
- ret = iio_readv(dev_handle, acb, qiov->iov, qiov->niov,
- offset, (uint64_t)size, iio_flags);
+ ret = (*libvxhs.iio_readv)(dev_handle, acb, qiov->iov, qiov->niov,
+ offset, (uint64_t)size, iio_flags);
break;
default:
trace_vxhs_aio_rw_invalid(iodir);
@@ -505,7 +606,7 @@ static void vxhs_close(BlockDriverState *bs)
* Close vDisk device
*/
if (s->vdisk_hostinfo.dev_handle) {
- iio_close(s->vdisk_hostinfo.dev_handle);
+ (*libvxhs.iio_close)(s->vdisk_hostinfo.dev_handle);
s->vdisk_hostinfo.dev_handle = NULL;
}
@@ -527,7 +628,7 @@ static int64_t vxhs_get_vdisk_stat(BDRVVXHSState *s)
int ret = 0;
void *dev_handle = s->vdisk_hostinfo.dev_handle;
- ret = iio_ioctl(dev_handle, IOR_VDISK_STAT, &vdisk_size, 0);
+ ret = (*libvxhs.iio_ioctl)(dev_handle, IOR_VDISK_STAT, &vdisk_size, 0);
if (ret < 0) {
trace_vxhs_get_vdisk_stat_err(s->vdisk_guid, ret, errno);
return -EIO;
diff --git a/configure b/configure
index 0a19b03..7358269 100755
--- a/configure
+++ b/configure
@@ -3369,7 +3369,7 @@ else
glib_req_ver=2.22
fi
glib_modules=gthread-2.0
-if test "$modules" = yes; then
+if test "$modules" = yes -o "$vxhs" = yes; then
glib_modules="$glib_modules gmodule-export-2.0"
fi
@@ -5314,33 +5314,6 @@ if compile_prog "" "" ; then
fi
##########################################
-# Veritas HyperScale block driver VxHS
-# Check if libvxhs is installed
-
-if test "$vxhs" != "no" ; then
- cat > $TMPC <<EOF
-#include <stdint.h>
-#include <qnio/qnio_api.h>
-
-void *vxhs_callback;
-
-int main(void) {
- iio_init(QNIO_VERSION, vxhs_callback);
- return 0;
-}
-EOF
- vxhs_libs="-lvxhs -lssl"
- if compile_prog "" "$vxhs_libs" ; then
- vxhs=yes
- else
- if test "$vxhs" = "yes" ; then
- feature_not_found "vxhs block device" "Install libvxhs See github"
- fi
- vxhs=no
- fi
-fi
-
-##########################################
# check for _Static_assert()
have_static_assert=no
@@ -6614,8 +6587,8 @@ if test "$pthread_setname_np" = "yes" ; then
fi
if test "$vxhs" = "yes" ; then
- echo "CONFIG_VXHS=y" >> $config_host_mak
- echo "VXHS_LIBS=$vxhs_libs" >> $config_host_mak
+ echo "CONFIG_VXHS=m" >> $config_host_mak
+ echo "VXHS_LIBS= -lssl" >> $config_host_mak
fi
if test "$tcg_interpreter" = "yes"; then
diff --git a/include/block/vxhs_shim.h b/include/block/vxhs_shim.h
new file mode 100644
index 0000000..42519ae
--- /dev/null
+++ b/include/block/vxhs_shim.h
@@ -0,0 +1,143 @@
+/*
+ * Network IO library for VxHS QEMU block driver (Veritas Technologies)
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2. See
+ * the COPYING file in the top-level directory.
+ *
+ * Contributions after 2014-08-15 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
+ */
+
+#ifndef QNIO_API_H
+#define QNIO_API_H
+
+#include <sys/uio.h>
+
+/*
+ * Bump up the version everytime this file is modified
+ */
+#define QNIO_VERSION 34
+
+/*
+ * These are the opcodes referenced by callback routine.
+ */
+#define IRP_READ_REQUEST 0x1FFF
+#define IRP_WRITE_REQUEST 0x2FFF
+#define IRP_VDISK_CHECK_IO_FAILOVER_READY 2020
+
+/*
+ * opcodes for iio_ioctl.
+ */
+#define IOR_VDISK_STAT 1005
+
+/*
+ * Error values for iio_cb_t callback function.
+ */
+#define QNIOERROR_HUP 901 /* Retriable error */
+#define QNIOERROR_NOCONN 902 /* Non-retriable error */
+
+
+/* Operation Flags */
+#define IIO_FLAG_ASYNC 0x0001 /* Do an async send */
+
+/*
+ * INPUT:
+ * ctx - opaque context
+ * opcode - Operation
+ * error - 0 for sucess, non-zero for failure.
+ * RETURNS:
+ * void
+ * DESCRIPTION:
+ * This callback is called, after Async request completes.
+ *
+ * CONTEXT:
+ * The callback should be wait-free.
+ */
+typedef void (*iio_cb_t) (void *ctx, uint32_t opcode, uint32_t error);
+
+typedef struct LibVXHSFuncs {
+/*
+ * RETURNS:
+ * 0 for sucess, non-zero for failure.
+ * DESCRIPTION:
+ * Intilize the library state. This should be called at the
+ * begining before issuing any library call.
+ */
+ int (*iio_init)(int32_t version, iio_cb_t cb);
+/*
+ * RETURNS:
+ * void
+ * DESCRIPTION:
+ * Relinquish library resources. This should be called on the
+ * close of last open device.
+ */
+ void (*iio_fini)(void);
+/*
+ * DESCRIPTION:
+ * Returns minimum QNIO API version supported by library.
+ */
+ int32_t (*iio_min_version)(void);
+/*
+ * DESCRIPTION:
+ * Returns maximum QNIO API version supported by library.
+ */
+ int32_t (*iio_max_version)(void);
+/*
+ * INPUT:
+ * uri - const string of the format of://<hostname|ip>:port
+ * devid - Device ID.
+ * flags - currently unused, this must be set to 0
+ * cacert - CA certificates file in PEM format
+ * client_key - Client private key file in PEM format
+ * client_cert - Client certificate file in PEM format
+ * RETURNS:
+ * opeque device handle on success, NULL on failure.
+ * DESCRIPTION:
+ * This call returns device handle on success. Returns NULL on
+ * failure with errno set
+ * errno can be one of:
+ * ENODEV - remote device not found
+ * EBADF - Unable to open communication channel.
+ * EBUSY - The call cannot be completed right now
+ */
+ void *(*iio_open)(const char *uri, const char *devid, uint32_t flags,
+ const char *cacert, const char *client_key,
+ const char *client_cert);
+/*
+ * Close the device.
+ * For every matching iio_open() there should be a matching iio_close()
+ * The last close free all data structures associated with the device.
+ */
+ int32_t (*iio_close)(void *dev_handle);
+/*
+ * INPUT:
+ * dev_handle - device descriptor on which read/write needs to be performed
+ * ctx - an opaque context that is not interpreted This is set for
+ * async calls only. It can be NULL.
+ * iov - an array of iovecs (This is a scatter gather operation)
+ * iovcnt - the number of iovecs
+ * offset - an offset to perform the write
+ * size - I/O size
+ * flags - can be one of
+ * IIO_FLAG_ASYNC - indicating this is a aio call.
+ * RETURNS:
+ * -1 on error, sets errno
+ * EBADF - the remote fd is bad
+ * EBUSY - The call cannot be completed right now
+ * EPIPE - the channel got disconnected, call back would be called in
+ * addition to this.
+ */
+ int32_t (*iio_writev)(void *dev_handle, void *ctx, struct iovec *iov,
+ int iovcnt, uint64_t offset, uint64_t size,
+ uint32_t flags);
+
+ int32_t (*iio_readv)(void *dev_handle, void *ctx, struct iovec *iov,
+ int iovcnt, uint64_t offset, uint64_t size,
+ uint32_t flags);
+
+ int32_t (*iio_ioctl)(void *dev_handle, uint32_t opcode, void *opaque,
+ uint32_t flags);
+
+} LibVXHSFuncs;
+
+#endif