From e58ce3910208fead9e24eb08e19a11bb2eba2f1e Mon Sep 17 00:00:00 2001 From: "Danilo C. L. de Paula" Date: Wed, 7 Mar 2018 13:05:43 -0300 Subject: block/vxhs: modularize VXHS via g_module RH-Author: Jeffrey Cody Message-id: <8a91a423440b7a5a14e868279c772e99b865bfc6.1494281291.git.jcody@redhat.com> Patchwork-id: 75046 O-Subject: [RHEV-7.4 qemu-kvm-rhev 4/4] block/vxhs: modularize VXHS via g_module Bugzilla: 1265869 RH-Acked-by: Stefan Hajnoczi RH-Acked-by: Laurent Vivier RH-Acked-by: Miroslav Rezanina This converts the VXHS driver to be a runtime loaded module, with no external build dependencies on libvxhs / libqnio. This (libvxhs) is a 3rd party library, written by Veritas, to interface with the VXHS server. Red Hat is not going to distribute libvxhs, nor will Red Hat use libvxhs in the build systems. So that creates two criteria for the modularization, for business reasons: 1. No runtime dependencies on libvxhs (aside from opening the library on vxhs open) 2. No build dependencies on libvxhs packages. There is support in QEMU for modular block drivers, however there are two issues with using the built-in support: A. It is all-or-none; if modules are enabled all protocols are built as modules. This wouldn't be that bad, as it would of course enable more granular dependencies for qemu rpm packages. But... B. It is not designed with criteria #2 in mind; it reduces runtime dependencies, not build dependencies. The protocol libraries that are still built linked against external libraries and using external headers. This patch uses g_module to load the libvxhs library, and incorporates the libvxhs.h header in the build tree. If block driver protocols are also built as modules, libvxhs will still work and be built as a module too, except the shared library will still not have any dependency on libvxhs. There are a few changes in this patch from upstream (aside from the module loading aspects): 1. It enables VXHS support to be built as a protocl module if --enable-modules is used during configure. 2. If the init call to iio_init() fails, populate errp with a meaningful error message. 3. Since we are loading the library dynamically, make check the min and max supported versions in the libvxhs library on load. Patches for items #1 and #2 have been posted upstream. It is expected that the libvxhs library is located at the following pathname: /usr/lib64/qemu/libvxhs.so.1 VXHS support is only built for x86_64 in RHEV. Signed-off-by: Jeff Cody Signed-off-by: Miroslav Rezanina block/vxhs: improve error message for missing / bad vxhs module RH-Author: Jeffrey Cody Message-id: <59af10d83125fff42beacd30dbca83d50409bbed.1513031708.git.jcody@redhat.com> Patchwork-id: 78305 O-Subject: [RHV7.5 qemu-kvm-rhev PATCH 1/1] block/vxhs: improve error message for missing / bad vxhs module Bugzilla: 1505654 RH-Acked-by: Markus Armbruster RH-Acked-by: Dr. David Alan Gilbert RH-Acked-by: Stefan Hajnoczi RH-Acked-by: Laszlo Ersek [Downstream only, as the module load of libvxhs is downstream only] In the case of missing libvxhs libraries, the original error message, while technically accurate, may lead a user to think there is a QEMU bug if trying to using the VXHS protocol. Update the message so that it is clear that the likely issue is that the Veritas QEMU libvxhs RPM is not installed (or not installed correctly, if there are permission or file corruption issues, etc.). An example error message before this change: > qemu-img info vxhs://localhost/test qemu-img: Could not open 'vxhs://localhost/test': \ error loading libvxhs: /usr/lib64/qemu/libvxhs.so.1: \ cannot open shared object file: No such file or directory An example error message after this change: > qemu-img info vxhs://localhost/test qemu-img: Could not open 'vxhs://localhost/test': \ The VXHS library from Veritas might not be installed correctly \ (/usr/lib64/qemu/libvxhs.so.1: \ cannot open shared object file: No such file or directory) Signed-off-by: Jeff Cody Signed-off-by: Miroslav Rezanina --- block/vxhs.c | 123 ++++++++++++++++++++++++++++++++---- configure | 33 +--------- include/block/vxhs_shim.h | 143 ++++++++++++++++++++++++++++++++++++++++++ redhat/build_configure.sh | 9 ++- redhat/qemu-kvm.spec.template | 7 +++ 5 files changed, 273 insertions(+), 42 deletions(-) create mode 100644 include/block/vxhs_shim.h diff --git a/block/vxhs.c b/block/vxhs.c index 75cc6c8..68edb51 100644 --- a/block/vxhs.c +++ b/block/vxhs.c @@ -9,7 +9,8 @@ */ #include "qemu/osdep.h" -#include +#include "block/vxhs_shim.h" +#include #include #include "block/block_int.h" #include "qapi/qmp/qerror.h" @@ -58,6 +59,97 @@ typedef struct BDRVVXHSState { char *tlscredsid; /* tlscredsid */ } BDRVVXHSState; +#define LIBVXHS_FULL_PATHNAME "/usr/lib64/qemu/libvxhs.so.1" +static bool libvxhs_loaded; +static GModule *libvxhs_handle; + +static LibVXHSFuncs libvxhs; + +typedef struct LibVXHSSymbols { + const char *name; + gpointer *addr; +} LibVXHSSymbols; + +static LibVXHSSymbols libvxhs_symbols[] = { + {"iio_init", (gpointer *) &libvxhs.iio_init}, + {"iio_fini", (gpointer *) &libvxhs.iio_fini}, + {"iio_min_version", (gpointer *) &libvxhs.iio_min_version}, + {"iio_max_version", (gpointer *) &libvxhs.iio_max_version}, + {"iio_open", (gpointer *) &libvxhs.iio_open}, + {"iio_close", (gpointer *) &libvxhs.iio_close}, + {"iio_writev", (gpointer *) &libvxhs.iio_writev}, + {"iio_readv", (gpointer *) &libvxhs.iio_readv}, + {"iio_ioctl", (gpointer *) &libvxhs.iio_ioctl}, + {NULL} +}; + +static void bdrv_vxhs_set_funcs(GModule *handle, Error **errp) +{ + int i = 0; + while (libvxhs_symbols[i].name) { + const char *name = libvxhs_symbols[i].name; + if (!g_module_symbol(handle, name, libvxhs_symbols[i].addr)) { + error_setg(errp, "%s could not be loaded from libvxhs: %s", + name, g_module_error()); + return; + } + ++i; + } +} + +static void bdrv_vxhs_load_libs(Error **errp) +{ + Error *local_err = NULL; + int32_t ver; + + if (libvxhs_loaded) { + return; + } + + if (!g_module_supported()) { + error_setg(errp, "modules are not supported on this platform: %s", + g_module_error()); + return; + } + + libvxhs_handle = g_module_open(LIBVXHS_FULL_PATHNAME, + G_MODULE_BIND_LAZY | G_MODULE_BIND_LOCAL); + if (!libvxhs_handle) { + error_setg(errp, "The VXHS library from Veritas might not be installed " + "correctly (%s)", g_module_error()); + return; + } + + g_module_make_resident(libvxhs_handle); + + bdrv_vxhs_set_funcs(libvxhs_handle, &local_err); + if (local_err) { + error_propagate(errp, local_err); + return; + } + + /* Now check to see if the libvxhs we are using here is supported + * by the loaded version */ + + ver = (*libvxhs.iio_min_version)(); + if (ver > QNIO_VERSION) { + error_setg(errp, "Trying to use libvxhs version %"PRId32" API, but " + "only %"PRId32" or newer is supported by %s", + QNIO_VERSION, ver, LIBVXHS_FULL_PATHNAME); + return; + } + + ver = (*libvxhs.iio_max_version)(); + if (ver < QNIO_VERSION) { + error_setg(errp, "Trying to use libvxhs version %"PRId32" API, but " + "only %"PRId32" or earlier is supported by %s", + QNIO_VERSION, ver, LIBVXHS_FULL_PATHNAME); + return; + } + + libvxhs_loaded = true; +} + static void vxhs_complete_aio_bh(void *opaque) { VXHSAIOCB *acb = opaque; @@ -219,7 +311,7 @@ static void vxhs_parse_filename(const char *filename, QDict *options, static int vxhs_init_and_ref(void) { if (vxhs_ref++ == 0) { - if (iio_init(QNIO_VERSION, vxhs_iio_callback)) { + if ((*libvxhs.iio_init)(QNIO_VERSION, vxhs_iio_callback)) { return -ENODEV; } } @@ -229,7 +321,7 @@ static int vxhs_init_and_ref(void) static void vxhs_unref(void) { if (--vxhs_ref == 0) { - iio_fini(); + (*libvxhs.iio_fini)(); } } @@ -299,8 +391,17 @@ static int vxhs_open(BlockDriverState *bs, QDict *options, char *client_key = NULL; char *client_cert = NULL; + bdrv_vxhs_load_libs(&local_err); + if (local_err) { + error_propagate(errp, local_err); + /* on error, cannot cleanup because the iio_fini() function + * is not loaded */ + return -EINVAL; + } + ret = vxhs_init_and_ref(); if (ret < 0) { + error_setg(&local_err, "libvxhs iio_init() failed"); ret = -EINVAL; goto out; } @@ -385,8 +486,8 @@ static int vxhs_open(BlockDriverState *bs, QDict *options, /* * Open qnio channel to storage agent if not opened before */ - dev_handlep = iio_open(of_vsa_addr, s->vdisk_guid, 0, - cacert, client_key, client_cert); + dev_handlep = (*libvxhs.iio_open)(of_vsa_addr, s->vdisk_guid, 0, + cacert, client_key, client_cert); if (dev_handlep == NULL) { trace_vxhs_open_iio_open(of_vsa_addr); ret = -ENODEV; @@ -450,12 +551,12 @@ static BlockAIOCB *vxhs_aio_rw(BlockDriverState *bs, int64_t sector_num, switch (iodir) { case VDISK_AIO_WRITE: - ret = iio_writev(dev_handle, acb, qiov->iov, qiov->niov, - offset, (uint64_t)size, iio_flags); + ret = (*libvxhs.iio_writev)(dev_handle, acb, qiov->iov, qiov->niov, + offset, (uint64_t)size, iio_flags); break; case VDISK_AIO_READ: - ret = iio_readv(dev_handle, acb, qiov->iov, qiov->niov, - offset, (uint64_t)size, iio_flags); + ret = (*libvxhs.iio_readv)(dev_handle, acb, qiov->iov, qiov->niov, + offset, (uint64_t)size, iio_flags); break; default: trace_vxhs_aio_rw_invalid(iodir); @@ -505,7 +606,7 @@ static void vxhs_close(BlockDriverState *bs) * Close vDisk device */ if (s->vdisk_hostinfo.dev_handle) { - iio_close(s->vdisk_hostinfo.dev_handle); + (*libvxhs.iio_close)(s->vdisk_hostinfo.dev_handle); s->vdisk_hostinfo.dev_handle = NULL; } @@ -527,7 +628,7 @@ static int64_t vxhs_get_vdisk_stat(BDRVVXHSState *s) int ret = 0; void *dev_handle = s->vdisk_hostinfo.dev_handle; - ret = iio_ioctl(dev_handle, IOR_VDISK_STAT, &vdisk_size, 0); + ret = (*libvxhs.iio_ioctl)(dev_handle, IOR_VDISK_STAT, &vdisk_size, 0); if (ret < 0) { trace_vxhs_get_vdisk_stat_err(s->vdisk_guid, ret, errno); return -EIO; diff --git a/configure b/configure index 0a19b03..7358269 100755 --- a/configure +++ b/configure @@ -3369,7 +3369,7 @@ else glib_req_ver=2.22 fi glib_modules=gthread-2.0 -if test "$modules" = yes; then +if test "$modules" = yes -o "$vxhs" = yes; then glib_modules="$glib_modules gmodule-export-2.0" fi @@ -5314,33 +5314,6 @@ if compile_prog "" "" ; then fi ########################################## -# Veritas HyperScale block driver VxHS -# Check if libvxhs is installed - -if test "$vxhs" != "no" ; then - cat > $TMPC < -#include - -void *vxhs_callback; - -int main(void) { - iio_init(QNIO_VERSION, vxhs_callback); - return 0; -} -EOF - vxhs_libs="-lvxhs -lssl" - if compile_prog "" "$vxhs_libs" ; then - vxhs=yes - else - if test "$vxhs" = "yes" ; then - feature_not_found "vxhs block device" "Install libvxhs See github" - fi - vxhs=no - fi -fi - -########################################## # check for _Static_assert() have_static_assert=no @@ -6614,8 +6587,8 @@ if test "$pthread_setname_np" = "yes" ; then fi if test "$vxhs" = "yes" ; then - echo "CONFIG_VXHS=y" >> $config_host_mak - echo "VXHS_LIBS=$vxhs_libs" >> $config_host_mak + echo "CONFIG_VXHS=m" >> $config_host_mak + echo "VXHS_LIBS= -lssl" >> $config_host_mak fi if test "$tcg_interpreter" = "yes"; then diff --git a/include/block/vxhs_shim.h b/include/block/vxhs_shim.h new file mode 100644 index 0000000..42519ae --- /dev/null +++ b/include/block/vxhs_shim.h @@ -0,0 +1,143 @@ +/* + * Network IO library for VxHS QEMU block driver (Veritas Technologies) + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + * + * Contributions after 2014-08-15 are licensed under the terms of the + * GNU GPL, version 2 or (at your option) any later version. + */ + +#ifndef QNIO_API_H +#define QNIO_API_H + +#include + +/* + * Bump up the version everytime this file is modified + */ +#define QNIO_VERSION 34 + +/* + * These are the opcodes referenced by callback routine. + */ +#define IRP_READ_REQUEST 0x1FFF +#define IRP_WRITE_REQUEST 0x2FFF +#define IRP_VDISK_CHECK_IO_FAILOVER_READY 2020 + +/* + * opcodes for iio_ioctl. + */ +#define IOR_VDISK_STAT 1005 + +/* + * Error values for iio_cb_t callback function. + */ +#define QNIOERROR_HUP 901 /* Retriable error */ +#define QNIOERROR_NOCONN 902 /* Non-retriable error */ + + +/* Operation Flags */ +#define IIO_FLAG_ASYNC 0x0001 /* Do an async send */ + +/* + * INPUT: + * ctx - opaque context + * opcode - Operation + * error - 0 for sucess, non-zero for failure. + * RETURNS: + * void + * DESCRIPTION: + * This callback is called, after Async request completes. + * + * CONTEXT: + * The callback should be wait-free. + */ +typedef void (*iio_cb_t) (void *ctx, uint32_t opcode, uint32_t error); + +typedef struct LibVXHSFuncs { +/* + * RETURNS: + * 0 for sucess, non-zero for failure. + * DESCRIPTION: + * Intilize the library state. This should be called at the + * begining before issuing any library call. + */ + int (*iio_init)(int32_t version, iio_cb_t cb); +/* + * RETURNS: + * void + * DESCRIPTION: + * Relinquish library resources. This should be called on the + * close of last open device. + */ + void (*iio_fini)(void); +/* + * DESCRIPTION: + * Returns minimum QNIO API version supported by library. + */ + int32_t (*iio_min_version)(void); +/* + * DESCRIPTION: + * Returns maximum QNIO API version supported by library. + */ + int32_t (*iio_max_version)(void); +/* + * INPUT: + * uri - const string of the format of://:port + * devid - Device ID. + * flags - currently unused, this must be set to 0 + * cacert - CA certificates file in PEM format + * client_key - Client private key file in PEM format + * client_cert - Client certificate file in PEM format + * RETURNS: + * opeque device handle on success, NULL on failure. + * DESCRIPTION: + * This call returns device handle on success. Returns NULL on + * failure with errno set + * errno can be one of: + * ENODEV - remote device not found + * EBADF - Unable to open communication channel. + * EBUSY - The call cannot be completed right now + */ + void *(*iio_open)(const char *uri, const char *devid, uint32_t flags, + const char *cacert, const char *client_key, + const char *client_cert); +/* + * Close the device. + * For every matching iio_open() there should be a matching iio_close() + * The last close free all data structures associated with the device. + */ + int32_t (*iio_close)(void *dev_handle); +/* + * INPUT: + * dev_handle - device descriptor on which read/write needs to be performed + * ctx - an opaque context that is not interpreted This is set for + * async calls only. It can be NULL. + * iov - an array of iovecs (This is a scatter gather operation) + * iovcnt - the number of iovecs + * offset - an offset to perform the write + * size - I/O size + * flags - can be one of + * IIO_FLAG_ASYNC - indicating this is a aio call. + * RETURNS: + * -1 on error, sets errno + * EBADF - the remote fd is bad + * EBUSY - The call cannot be completed right now + * EPIPE - the channel got disconnected, call back would be called in + * addition to this. + */ + int32_t (*iio_writev)(void *dev_handle, void *ctx, struct iovec *iov, + int iovcnt, uint64_t offset, uint64_t size, + uint32_t flags); + + int32_t (*iio_readv)(void *dev_handle, void *ctx, struct iovec *iov, + int iovcnt, uint64_t offset, uint64_t size, + uint32_t flags); + + int32_t (*iio_ioctl)(void *dev_handle, uint32_t opcode, void *opaque, + uint32_t flags); + +} LibVXHSFuncs; + +#endif