From 064e2489caf7aeb9cd01dfcdc1b6a73b091b7f23 Mon Sep 17 00:00:00 2001

From: Adrien Mazarguil <adrien.mazarguil@6wind.com>

Date: Tue, 30 Jan 2018 16:34:52 +0100

Subject: [PATCH 1/9] net/mlx4: move rdma-core calls to separate file

This lays the groundwork for externalizing rdma-core as an optional

run-time dependency instead of a mandatory one.

No functional change.

Signed-off-by: Adrien Mazarguil <adrien.mazarguil@6wind.com>

Acked-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>

(cherry picked from commit 5f03b79b97d77804620e220a7715b696d8073b0e)

---

 drivers/net/mlx4/Makefile      |   1 +

 drivers/net/mlx4/mlx4.c        |  33 ++---

 drivers/net/mlx4/mlx4_ethdev.c |   1 +

 drivers/net/mlx4/mlx4_flow.c   |  32 ++---

 drivers/net/mlx4/mlx4_glue.c   | 275 +++++++++++++++++++++++++++++++++++++++++

 drivers/net/mlx4/mlx4_glue.h   |  80 ++++++++++++

 drivers/net/mlx4/mlx4_intr.c   |  10 +-

 drivers/net/mlx4/mlx4_mr.c     |   7 +-

 drivers/net/mlx4/mlx4_rxq.c    |  53 ++++----

 drivers/net/mlx4/mlx4_txq.c    |  17 +--

 10 files changed, 438 insertions(+), 71 deletions(-)

 create mode 100644 drivers/net/mlx4/mlx4_glue.c

 create mode 100644 drivers/net/mlx4/mlx4_glue.h

diff --git a/drivers/net/mlx4/Makefile b/drivers/net/mlx4/Makefile

index f1f47c2..7ba304b 100644

--- a/drivers/net/mlx4/Makefile

+++ b/drivers/net/mlx4/Makefile

@@ -38,6 +38,7 @@ LIB = librte_pmd_mlx4.a

 SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4.c

 SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_ethdev.c

 SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_flow.c

+SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_glue.c

 SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_intr.c

 SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_mr.c

 SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_rxq.c

diff --git a/drivers/net/mlx4/mlx4.c b/drivers/net/mlx4/mlx4.c

index f9e4f9d..7f58e26 100644

--- a/drivers/net/mlx4/mlx4.c

+++ b/drivers/net/mlx4/mlx4.c

@@ -67,6 +67,7 @@

 #include <rte_mbuf.h>

 #include "mlx4.h"

+#include "mlx4_glue.h"

 #include "mlx4_flow.h"

 #include "mlx4_rxtx.h"

 #include "mlx4_utils.h"

@@ -218,8 +219,8 @@ struct mlx4_conf {

 		mlx4_tx_queue_release(dev->data->tx_queues[i]);

 	if (priv->pd != NULL) {

 		assert(priv->ctx != NULL);

-		claim_zero(ibv_dealloc_pd(priv->pd));

-		claim_zero(ibv_close_device(priv->ctx));

+		claim_zero(mlx4_glue->dealloc_pd(priv->pd));

+		claim_zero(mlx4_glue->close_device(priv->ctx));

 	} else

 		assert(priv->ctx == NULL);

 	mlx4_intr_uninstall(priv);

@@ -434,7 +435,7 @@ struct mlx4_conf {

 	(void)pci_drv;

 	assert(pci_drv == &mlx4_driver);

-	list = ibv_get_device_list(&i);

+	list = mlx4_glue->get_device_list(&i);

 	if (list == NULL) {

 		rte_errno = errno;

 		assert(rte_errno);

@@ -463,12 +464,12 @@ struct mlx4_conf {

 		      PCI_DEVICE_ID_MELLANOX_CONNECTX3VF);

 		INFO("PCI information matches, using device \"%s\" (VF: %s)",

 		     list[i]->name, (vf ? "true" : "false"));

-		attr_ctx = ibv_open_device(list[i]);

+		attr_ctx = mlx4_glue->open_device(list[i]);

 		err = errno;

 		break;

 	}

 	if (attr_ctx == NULL) {

-		ibv_free_device_list(list);

+		mlx4_glue->free_device_list(list);

 		switch (err) {

 		case 0:

 			rte_errno = ENODEV;

@@ -485,7 +486,7 @@ struct mlx4_conf {

 	}

 	ibv_dev = list[i];

 	DEBUG("device opened");

-	if (ibv_query_device(attr_ctx, &device_attr)) {

+	if (mlx4_glue->query_device(attr_ctx, &device_attr)) {

 		rte_errno = ENODEV;

 		goto error;

 	}

@@ -512,13 +513,13 @@ struct mlx4_conf {

 		if (!(conf.ports.enabled & (1 << i)))

 			continue;

 		DEBUG("using port %u", port);

-		ctx = ibv_open_device(ibv_dev);

+		ctx = mlx4_glue->open_device(ibv_dev);

 		if (ctx == NULL) {

 			rte_errno = ENODEV;

 			goto port_error;

 		}

 		/* Check port status. */

-		err = ibv_query_port(ctx, port, &port_attr);

+		err = mlx4_glue->query_port(ctx, port, &port_attr);

 		if (err) {

 			rte_errno = err;

 			ERROR("port query failed: %s", strerror(rte_errno));

@@ -532,7 +533,7 @@ struct mlx4_conf {

 		}

 		if (port_attr.state != IBV_PORT_ACTIVE)

 			DEBUG("port %d is not active: \"%s\" (%d)",

-			      port, ibv_port_state_str(port_attr.state),

+			      port, mlx4_glue->port_state_str(port_attr.state),

 			      port_attr.state);

 		/* Make asynchronous FD non-blocking to handle interrupts. */

 		if (mlx4_fd_set_non_blocking(ctx->async_fd) < 0) {

@@ -541,7 +542,7 @@ struct mlx4_conf {

 			goto port_error;

 		}

 		/* Allocate protection domain. */

-		pd = ibv_alloc_pd(ctx);

+		pd = mlx4_glue->alloc_pd(ctx);

 		if (pd == NULL) {

 			rte_errno = ENOMEM;

 			ERROR("PD allocation failure");

@@ -605,7 +606,7 @@ struct mlx4_conf {

 			char name[RTE_ETH_NAME_MAX_LEN];

 			snprintf(name, sizeof(name), "%s port %u",

-				 ibv_get_device_name(ibv_dev), port);

+				 mlx4_glue->get_device_name(ibv_dev), port);

 			eth_dev = rte_eth_dev_allocate(name);

 		}

 		if (eth_dev == NULL) {

@@ -648,9 +649,9 @@ struct mlx4_conf {

 port_error:

 		rte_free(priv);

 		if (pd)

-			claim_zero(ibv_dealloc_pd(pd));

+			claim_zero(mlx4_glue->dealloc_pd(pd));

 		if (ctx)

-			claim_zero(ibv_close_device(ctx));

+			claim_zero(mlx4_glue->close_device(ctx));

 		if (eth_dev)

 			rte_eth_dev_release_port(eth_dev);

 		break;

@@ -665,9 +666,9 @@ struct mlx4_conf {

 	 */

 error:

 	if (attr_ctx)

-		claim_zero(ibv_close_device(attr_ctx));

+		claim_zero(mlx4_glue->close_device(attr_ctx));

 	if (list)

-		ibv_free_device_list(list);

+		mlx4_glue->free_device_list(list);

 	assert(rte_errno >= 0);

 	return -rte_errno;

 }

@@ -714,7 +715,7 @@ struct mlx4_conf {

 	 * using this PMD, which is not supported in forked processes.

 	 */

 	setenv("RDMAV_HUGEPAGES_SAFE", "1", 1);

-	ibv_fork_init();

+	mlx4_glue->fork_init();

 	rte_pci_register(&mlx4_driver);

 }

diff --git a/drivers/net/mlx4/mlx4_ethdev.c b/drivers/net/mlx4/mlx4_ethdev.c

index 2f69e7d..e2f9509 100644

--- a/drivers/net/mlx4/mlx4_ethdev.c

+++ b/drivers/net/mlx4/mlx4_ethdev.c

@@ -70,6 +70,7 @@

 #include "mlx4.h"

 #include "mlx4_flow.h"

+#include "mlx4_glue.h"

 #include "mlx4_rxtx.h"

 #include "mlx4_utils.h"

diff --git a/drivers/net/mlx4/mlx4_flow.c b/drivers/net/mlx4/mlx4_flow.c

index 8b87b29..4a6b8a4 100644

--- a/drivers/net/mlx4/mlx4_flow.c

+++ b/drivers/net/mlx4/mlx4_flow.c

@@ -65,6 +65,7 @@

 /* PMD headers. */

 #include "mlx4.h"

+#include "mlx4_glue.h"

 #include "mlx4_flow.h"

 #include "mlx4_rxtx.h"

 #include "mlx4_utils.h"

@@ -914,24 +915,25 @@ struct mlx4_drop {

 		.priv = priv,

 		.refcnt = 1,

 	};

-	drop->cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0);

+	drop->cq = mlx4_glue->create_cq(priv->ctx, 1, NULL, NULL, 0);

 	if (!drop->cq)

 		goto error;

-	drop->qp = ibv_create_qp(priv->pd,

-				 &(struct ibv_qp_init_attr){

-					.send_cq = drop->cq,

-					.recv_cq = drop->cq,

-					.qp_type = IBV_QPT_RAW_PACKET,

-				 });

+	drop->qp = mlx4_glue->create_qp

+		(priv->pd,

+		 &(struct ibv_qp_init_attr){

+			.send_cq = drop->cq,

+			.recv_cq = drop->cq,

+			.qp_type = IBV_QPT_RAW_PACKET,

+		 });

 	if (!drop->qp)

 		goto error;

 	priv->drop = drop;

 	return drop;

 error:

 	if (drop->qp)

-		claim_zero(ibv_destroy_qp(drop->qp));

+		claim_zero(mlx4_glue->destroy_qp(drop->qp));

 	if (drop->cq)

-		claim_zero(ibv_destroy_cq(drop->cq));

+		claim_zero(mlx4_glue->destroy_cq(drop->cq));

 	if (drop)

 		rte_free(drop);

 	rte_errno = ENOMEM;

@@ -951,8 +953,8 @@ struct mlx4_drop {

 	if (--drop->refcnt)

 		return;

 	drop->priv->drop = NULL;

-	claim_zero(ibv_destroy_qp(drop->qp));

-	claim_zero(ibv_destroy_cq(drop->cq));

+	claim_zero(mlx4_glue->destroy_qp(drop->qp));

+	claim_zero(mlx4_glue->destroy_cq(drop->cq));

 	rte_free(drop);

 }

@@ -984,7 +986,7 @@ struct mlx4_drop {

 	if (!enable) {

 		if (!flow->ibv_flow)

 			return 0;

-		claim_zero(ibv_destroy_flow(flow->ibv_flow));

+		claim_zero(mlx4_glue->destroy_flow(flow->ibv_flow));

 		flow->ibv_flow = NULL;

 		if (flow->drop)

 			mlx4_drop_put(priv->drop);

@@ -997,7 +999,7 @@ struct mlx4_drop {

 	    !priv->isolated &&

 	    flow->ibv_attr->priority == MLX4_FLOW_PRIORITY_LAST) {

 		if (flow->ibv_flow) {

-			claim_zero(ibv_destroy_flow(flow->ibv_flow));

+			claim_zero(mlx4_glue->destroy_flow(flow->ibv_flow));

 			flow->ibv_flow = NULL;

 			if (flow->drop)

 				mlx4_drop_put(priv->drop);

@@ -1027,7 +1029,7 @@ struct mlx4_drop {

 			if (missing ^ !flow->drop)

 				return 0;

 			/* Verbs flow needs updating. */

-			claim_zero(ibv_destroy_flow(flow->ibv_flow));

+			claim_zero(mlx4_glue->destroy_flow(flow->ibv_flow));

 			flow->ibv_flow = NULL;

 			if (flow->drop)

 				mlx4_drop_put(priv->drop);

@@ -1059,7 +1061,7 @@ struct mlx4_drop {

 	assert(qp);

 	if (flow->ibv_flow)

 		return 0;

-	flow->ibv_flow = ibv_create_flow(qp, flow->ibv_attr);

+	flow->ibv_flow = mlx4_glue->create_flow(qp, flow->ibv_attr);

 	if (flow->ibv_flow)

 		return 0;

 	if (flow->drop)

diff --git a/drivers/net/mlx4/mlx4_glue.c b/drivers/net/mlx4/mlx4_glue.c

new file mode 100644

index 0000000..30797bd

--- /dev/null

+++ b/drivers/net/mlx4/mlx4_glue.c

@@ -0,0 +1,275 @@

+/* SPDX-License-Identifier: BSD-3-Clause

+ * Copyright 2018 6WIND S.A.

+ * Copyright 2018 Mellanox

+ */

+

+/* Verbs headers do not support -pedantic. */

+#ifdef PEDANTIC

+#pragma GCC diagnostic ignored "-Wpedantic"

+#endif

+#include <infiniband/mlx4dv.h>

+#include <infiniband/verbs.h>

+#ifdef PEDANTIC

+#pragma GCC diagnostic error "-Wpedantic"

+#endif

+

+#include "mlx4_glue.h"

+

+static int

+mlx4_glue_fork_init(void)

+{

+	return ibv_fork_init();

+}

+

+static int

+mlx4_glue_get_async_event(struct ibv_context *context,

+			  struct ibv_async_event *event)

+{

+	return ibv_get_async_event(context, event);

+}

+

+static void

+mlx4_glue_ack_async_event(struct ibv_async_event *event)

+{

+	ibv_ack_async_event(event);

+}

+

+static struct ibv_pd *

+mlx4_glue_alloc_pd(struct ibv_context *context)

+{

+	return ibv_alloc_pd(context);

+}

+

+static int

+mlx4_glue_dealloc_pd(struct ibv_pd *pd)

+{

+	return ibv_dealloc_pd(pd);

+}

+

+static struct ibv_device **

+mlx4_glue_get_device_list(int *num_devices)

+{

+	return ibv_get_device_list(num_devices);

+}

+

+static void

+mlx4_glue_free_device_list(struct ibv_device **list)

+{

+	ibv_free_device_list(list);

+}

+

+static struct ibv_context *

+mlx4_glue_open_device(struct ibv_device *device)

+{

+	return ibv_open_device(device);

+}

+

+static int

+mlx4_glue_close_device(struct ibv_context *context)

+{

+	return ibv_close_device(context);

+}

+

+static const char *

+mlx4_glue_get_device_name(struct ibv_device *device)

+{

+	return ibv_get_device_name(device);

+}

+

+static int

+mlx4_glue_query_device(struct ibv_context *context,

+		       struct ibv_device_attr *device_attr)

+{

+	return ibv_query_device(context, device_attr);

+}

+

+static int

+mlx4_glue_query_device_ex(struct ibv_context *context,

+			  const struct ibv_query_device_ex_input *input,

+			  struct ibv_device_attr_ex *attr)

+{

+	return ibv_query_device_ex(context, input, attr);

+}

+

+static int

+mlx4_glue_query_port(struct ibv_context *context, uint8_t port_num,

+		     struct ibv_port_attr *port_attr)

+{

+	return ibv_query_port(context, port_num, port_attr);

+}

+

+static const char *

+mlx4_glue_port_state_str(enum ibv_port_state port_state)

+{

+	return ibv_port_state_str(port_state);

+}

+

+static struct ibv_comp_channel *

+mlx4_glue_create_comp_channel(struct ibv_context *context)

+{

+	return ibv_create_comp_channel(context);

+}

+

+static int

+mlx4_glue_destroy_comp_channel(struct ibv_comp_channel *channel)

+{

+	return ibv_destroy_comp_channel(channel);

+}

+

+static struct ibv_cq *

+mlx4_glue_create_cq(struct ibv_context *context, int cqe, void *cq_context,

+		    struct ibv_comp_channel *channel, int comp_vector)

+{

+	return ibv_create_cq(context, cqe, cq_context, channel, comp_vector);

+}

+

+static int

+mlx4_glue_destroy_cq(struct ibv_cq *cq)

+{

+	return ibv_destroy_cq(cq);

+}

+

+static int

+mlx4_glue_get_cq_event(struct ibv_comp_channel *channel, struct ibv_cq **cq,

+		       void **cq_context)

+{

+	return ibv_get_cq_event(channel, cq, cq_context);

+}

+

+static void

+mlx4_glue_ack_cq_events(struct ibv_cq *cq, unsigned int nevents)

+{

+	ibv_ack_cq_events(cq, nevents);

+}

+

+static struct ibv_flow *

+mlx4_glue_create_flow(struct ibv_qp *qp, struct ibv_flow_attr *flow)

+{

+	return ibv_create_flow(qp, flow);

+}

+

+static int

+mlx4_glue_destroy_flow(struct ibv_flow *flow_id)

+{

+	return ibv_destroy_flow(flow_id);

+}

+

+static struct ibv_qp *

+mlx4_glue_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr)

+{

+	return ibv_create_qp(pd, qp_init_attr);

+}

+

+static struct ibv_qp *

+mlx4_glue_create_qp_ex(struct ibv_context *context,

+		       struct ibv_qp_init_attr_ex *qp_init_attr_ex)

+{

+	return ibv_create_qp_ex(context, qp_init_attr_ex);

+}

+

+static int

+mlx4_glue_destroy_qp(struct ibv_qp *qp)

+{

+	return ibv_destroy_qp(qp);

+}

+

+static int

+mlx4_glue_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask)

+{

+	return ibv_modify_qp(qp, attr, attr_mask);

+}

+

+static struct ibv_mr *

+mlx4_glue_reg_mr(struct ibv_pd *pd, void *addr, size_t length, int access)

+{

+	return ibv_reg_mr(pd, addr, length, access);

+}

+

+static int

+mlx4_glue_dereg_mr(struct ibv_mr *mr)

+{

+	return ibv_dereg_mr(mr);

+}

+

+static struct ibv_rwq_ind_table *

+mlx4_glue_create_rwq_ind_table(struct ibv_context *context,

+			       struct ibv_rwq_ind_table_init_attr *init_attr)

+{

+	return ibv_create_rwq_ind_table(context, init_attr);

+}

+

+static int

+mlx4_glue_destroy_rwq_ind_table(struct ibv_rwq_ind_table *rwq_ind_table)

+{

+	return ibv_destroy_rwq_ind_table(rwq_ind_table);

+}

+

+static struct ibv_wq *

+mlx4_glue_create_wq(struct ibv_context *context,

+		    struct ibv_wq_init_attr *wq_init_attr)

+{

+	return ibv_create_wq(context, wq_init_attr);

+}

+

+static int

+mlx4_glue_destroy_wq(struct ibv_wq *wq)

+{

+	return ibv_destroy_wq(wq);

+}

+static int

+mlx4_glue_modify_wq(struct ibv_wq *wq, struct ibv_wq_attr *wq_attr)

+{

+	return ibv_modify_wq(wq, wq_attr);

+}

+

+static int

+mlx4_glue_dv_init_obj(struct mlx4dv_obj *obj, uint64_t obj_type)

+{

+	return mlx4dv_init_obj(obj, obj_type);

+}

+

+static int

+mlx4_glue_dv_set_context_attr(struct ibv_context *context,

+			      enum mlx4dv_set_ctx_attr_type attr_type,

+			      void *attr)

+{

+	return mlx4dv_set_context_attr(context, attr_type, attr);

+}

+

+const struct mlx4_glue *mlx4_glue = &(const struct mlx4_glue){

+	.fork_init = mlx4_glue_fork_init,

+	.get_async_event = mlx4_glue_get_async_event,

+	.ack_async_event = mlx4_glue_ack_async_event,

+	.alloc_pd = mlx4_glue_alloc_pd,

+	.dealloc_pd = mlx4_glue_dealloc_pd,

+	.get_device_list = mlx4_glue_get_device_list,

+	.free_device_list = mlx4_glue_free_device_list,

+	.open_device = mlx4_glue_open_device,

+	.close_device = mlx4_glue_close_device,

+	.get_device_name = mlx4_glue_get_device_name,

+	.query_device = mlx4_glue_query_device,

+	.query_device_ex = mlx4_glue_query_device_ex,

+	.query_port = mlx4_glue_query_port,

+	.port_state_str = mlx4_glue_port_state_str,

+	.create_comp_channel = mlx4_glue_create_comp_channel,

+	.destroy_comp_channel = mlx4_glue_destroy_comp_channel,

+	.create_cq = mlx4_glue_create_cq,

+	.destroy_cq = mlx4_glue_destroy_cq,

+	.get_cq_event = mlx4_glue_get_cq_event,

+	.ack_cq_events = mlx4_glue_ack_cq_events,

+	.create_flow = mlx4_glue_create_flow,

+	.destroy_flow = mlx4_glue_destroy_flow,

+	.create_qp = mlx4_glue_create_qp,

+	.create_qp_ex = mlx4_glue_create_qp_ex,

+	.destroy_qp = mlx4_glue_destroy_qp,

+	.modify_qp = mlx4_glue_modify_qp,

+	.reg_mr = mlx4_glue_reg_mr,

+	.dereg_mr = mlx4_glue_dereg_mr,

+	.create_rwq_ind_table = mlx4_glue_create_rwq_ind_table,

+	.destroy_rwq_ind_table = mlx4_glue_destroy_rwq_ind_table,

+	.create_wq = mlx4_glue_create_wq,

+	.destroy_wq = mlx4_glue_destroy_wq,

+	.modify_wq = mlx4_glue_modify_wq,

+	.dv_init_obj = mlx4_glue_dv_init_obj,

+	.dv_set_context_attr = mlx4_glue_dv_set_context_attr,

+};

diff --git a/drivers/net/mlx4/mlx4_glue.h b/drivers/net/mlx4/mlx4_glue.h

new file mode 100644

index 0000000..0623511

--- /dev/null

+++ b/drivers/net/mlx4/mlx4_glue.h

@@ -0,0 +1,80 @@

+/* SPDX-License-Identifier: BSD-3-Clause

+ * Copyright 2018 6WIND S.A.

+ * Copyright 2018 Mellanox

+ */

+

+#ifndef MLX4_GLUE_H_

+#define MLX4_GLUE_H_

+

+/* Verbs headers do not support -pedantic. */

+#ifdef PEDANTIC

+#pragma GCC diagnostic ignored "-Wpedantic"

+#endif

+#include <infiniband/mlx4dv.h>

+#include <infiniband/verbs.h>

+#ifdef PEDANTIC

+#pragma GCC diagnostic error "-Wpedantic"

+#endif

+

+struct mlx4_glue {

+	int (*fork_init)(void);

+	int (*get_async_event)(struct ibv_context *context,

+			       struct ibv_async_event *event);

+	void (*ack_async_event)(struct ibv_async_event *event);

+	struct ibv_pd *(*alloc_pd)(struct ibv_context *context);

+	int (*dealloc_pd)(struct ibv_pd *pd);

+	struct ibv_device **(*get_device_list)(int *num_devices);

+	void (*free_device_list)(struct ibv_device **list);

+	struct ibv_context *(*open_device)(struct ibv_device *device);

+	int (*close_device)(struct ibv_context *context);

+	const char *(*get_device_name)(struct ibv_device *device);

+	int (*query_device)(struct ibv_context *context,

+			    struct ibv_device_attr *device_attr);

+	int (*query_device_ex)(struct ibv_context *context,

+			       const struct ibv_query_device_ex_input *input,

+			       struct ibv_device_attr_ex *attr);

+	int (*query_port)(struct ibv_context *context, uint8_t port_num,

+			  struct ibv_port_attr *port_attr);

+	const char *(*port_state_str)(enum ibv_port_state port_state);

+	struct ibv_comp_channel *(*create_comp_channel)

+		(struct ibv_context *context);

+	int (*destroy_comp_channel)(struct ibv_comp_channel *channel);

+	struct ibv_cq *(*create_cq)(struct ibv_context *context, int cqe,

+				    void *cq_context,

+				    struct ibv_comp_channel *channel,

+				    int comp_vector);

+	int (*destroy_cq)(struct ibv_cq *cq);

+	int (*get_cq_event)(struct ibv_comp_channel *channel,

+			    struct ibv_cq **cq, void **cq_context);

+	void (*ack_cq_events)(struct ibv_cq *cq, unsigned int nevents);

+	struct ibv_flow *(*create_flow)(struct ibv_qp *qp,

+					struct ibv_flow_attr *flow);

+	int (*destroy_flow)(struct ibv_flow *flow_id);

+	struct ibv_qp *(*create_qp)(struct ibv_pd *pd,

+				    struct ibv_qp_init_attr *qp_init_attr);

+	struct ibv_qp *(*create_qp_ex)

+		(struct ibv_context *context,

+		 struct ibv_qp_init_attr_ex *qp_init_attr_ex);

+	int (*destroy_qp)(struct ibv_qp *qp);

+	int (*modify_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr,

+			 int attr_mask);

+	struct ibv_mr *(*reg_mr)(struct ibv_pd *pd, void *addr,

+				 size_t length, int access);

+	int (*dereg_mr)(struct ibv_mr *mr);

+	struct ibv_rwq_ind_table *(*create_rwq_ind_table)

+		(struct ibv_context *context,

+		 struct ibv_rwq_ind_table_init_attr *init_attr);

+	int (*destroy_rwq_ind_table)(struct ibv_rwq_ind_table *rwq_ind_table);

+	struct ibv_wq *(*create_wq)(struct ibv_context *context,

+				    struct ibv_wq_init_attr *wq_init_attr);

+	int (*destroy_wq)(struct ibv_wq *wq);

+	int (*modify_wq)(struct ibv_wq *wq, struct ibv_wq_attr *wq_attr);

+	int (*dv_init_obj)(struct mlx4dv_obj *obj, uint64_t obj_type);

+	int (*dv_set_context_attr)(struct ibv_context *context,

+				   enum mlx4dv_set_ctx_attr_type attr_type,

+				   void *attr);

+};

+

+const struct mlx4_glue *mlx4_glue;

+

+#endif /* MLX4_GLUE_H_ */

diff --git a/drivers/net/mlx4/mlx4_intr.c b/drivers/net/mlx4/mlx4_intr.c

index 50d1976..c63806d 100644

--- a/drivers/net/mlx4/mlx4_intr.c

+++ b/drivers/net/mlx4/mlx4_intr.c

@@ -57,6 +57,7 @@

 #include <rte_interrupts.h>

 #include "mlx4.h"

+#include "mlx4_glue.h"

 #include "mlx4_rxtx.h"

 #include "mlx4_utils.h"

@@ -216,7 +217,7 @@

 	unsigned int i;

 	/* Read all message and acknowledge them. */

-	while (!ibv_get_async_event(priv->ctx, &event)) {

+	while (!mlx4_glue->get_async_event(priv->ctx, &event)) {

 		switch (event.event_type) {

 		case IBV_EVENT_PORT_ACTIVE:

 		case IBV_EVENT_PORT_ERR:

@@ -231,7 +232,7 @@

 			DEBUG("event type %d on physical port %d not handled",

 			      event.event_type, event.element.port_num);

 		}

-		ibv_ack_async_event(&event);

+		mlx4_glue->ack_async_event(&event);

 	}

 	for (i = 0; i != RTE_DIM(caught); ++i)

 		if (caught[i])

@@ -354,7 +355,8 @@

 	if (!rxq || !rxq->channel) {

 		ret = EINVAL;

 	} else {

-		ret = ibv_get_cq_event(rxq->cq->channel, &ev_cq, &ev_ctx);

+		ret = mlx4_glue->get_cq_event(rxq->cq->channel, &ev_cq,

+					      &ev_ctx);

 		if (ret || ev_cq != rxq->cq)

 			ret = EINVAL;

 	}

@@ -364,7 +366,7 @@

 		     idx);

 	} else {

 		rxq->mcq.arm_sn++;

-		ibv_ack_cq_events(rxq->cq, 1);

+		mlx4_glue->ack_cq_events(rxq->cq, 1);

 	}

 	return -ret;

 }

diff --git a/drivers/net/mlx4/mlx4_mr.c b/drivers/net/mlx4/mlx4_mr.c

index 2a3e269..493c008 100644

--- a/drivers/net/mlx4/mlx4_mr.c

+++ b/drivers/net/mlx4/mlx4_mr.c

@@ -60,6 +60,7 @@

 #include <rte_mempool.h>

 #include <rte_spinlock.h>

+#include "mlx4_glue.h"

 #include "mlx4_rxtx.h"

 #include "mlx4_utils.h"

@@ -200,8 +201,8 @@ struct mlx4_mr *

 		.end = end,

 		.refcnt = 1,

 		.priv = priv,

-		.mr = ibv_reg_mr(priv->pd, (void *)start, end - start,

-				 IBV_ACCESS_LOCAL_WRITE),

+		.mr = mlx4_glue->reg_mr(priv->pd, (void *)start, end - start,

+					IBV_ACCESS_LOCAL_WRITE),

 		.mp = mp,

 	};

 	if (mr->mr) {

@@ -240,7 +241,7 @@ struct mlx4_mr *

 	if (--mr->refcnt)