From 2432a74c845b7b529cb6e9044140b9445922a7ae Mon Sep 17 00:00:00 2001
From: Alex Vesker <valex@nvidia.com>
Date: Wed, 28 Oct 2020 12:26:32 +0200
Subject: [PATCH] mlx5: DR, Fix incorrect use of fl_roce_enabled capability
[ Upstream commit 2337d6790ad21b1d0c5373cf2aa6f8e70a510434 ]
Creating a FL QP should be allowed only when RoCE is enabled (roce_en)
and FL is supported with RoCE enabled. Previously we relied on the
general HCA cap whether RoCE is enabled and ignored its real state.
Creating a FL QP even if RoCE is disabled which could results in a
failure to modify QP.
Fixes: 6724f6530d3e ("mlx5: DR, Query RoCE capabilities")
Signed-off-by: Alex Vesker <valex@nvidia.com>
Signed-off-by: Yishai Hadas <yishaih@nvidia.com>
Signed-off-by: Nicolas Morey-Chaisemartin <nmoreychaisemartin@suse.com>
---
providers/mlx5/dr_devx.c | 24 ++++++++++++++++++++++++
providers/mlx5/dr_domain.c | 2 +-
providers/mlx5/dr_send.c | 8 +++++++-
providers/mlx5/mlx5_ifc.h | 29 +++++++++++++++++++++++++++++
providers/mlx5/mlx5dv_dr.h | 2 ++
5 files changed, 63 insertions(+), 2 deletions(-)
diff --git a/providers/mlx5/dr_devx.c b/providers/mlx5/dr_devx.c
index cd0f8bbc2e44..74a8155b8777 100644
--- a/providers/mlx5/dr_devx.c
+++ b/providers/mlx5/dr_devx.c
@@ -66,6 +66,26 @@ int dr_devx_query_esw_vport_context(struct ibv_context *ctx,
return 0;
}
+static int dr_devx_query_nic_vport_context(struct ibv_context *ctx,
+ bool *roce_en)
+{
+ uint32_t out[DEVX_ST_SZ_DW(query_nic_vport_context_out)] = {};
+ uint32_t in[DEVX_ST_SZ_DW(query_nic_vport_context_in)] = {};
+ int err;
+
+ DEVX_SET(query_nic_vport_context_in, in, opcode,
+ MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT);
+ err = mlx5dv_devx_general_cmd(ctx, in, sizeof(in), out, sizeof(out));
+ if (err) {
+ dr_dbg_ctx(ctx, "Query nic vport context failed %d\n", err);
+ return err;
+ }
+
+ *roce_en = DEVX_GET(query_nic_vport_context_out, out,
+ nic_vport_context.roce_en);
+ return 0;
+}
+
int dr_devx_query_gvmi(struct ibv_context *ctx, bool other_vport,
uint16_t vport_number, uint16_t *gvmi)
{
@@ -225,6 +245,10 @@ int dr_devx_query_device(struct ibv_context *ctx, struct dr_devx_caps *caps)
/* RoCE caps */
if (roce) {
+ err = dr_devx_query_nic_vport_context(ctx, &caps->roce_caps.roce_en);
+ if (err)
+ return err;
+
DEVX_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP);
DEVX_SET(query_hca_cap_in, in, op_mod,
MLX5_SET_HCA_CAP_OP_MOD_ROCE |
diff --git a/providers/mlx5/dr_domain.c b/providers/mlx5/dr_domain.c
index 916283e505aa..b47c5841f624 100644
--- a/providers/mlx5/dr_domain.c
+++ b/providers/mlx5/dr_domain.c
@@ -199,7 +199,7 @@ static int dr_domain_caps_init(struct ibv_context *ctx,
* force-loopback.
*/
if ((dmn->type != MLX5DV_DR_DOMAIN_TYPE_FDB) &&
- !dmn->info.caps.roce_caps.fl_rc_qp_when_roce_enabled)
+ !dr_send_allow_fl(&dmn->info.caps))
return 0;
ret = dr_domain_query_fdb_caps(ctx, dmn);
diff --git a/providers/mlx5/dr_send.c b/providers/mlx5/dr_send.c
index dfda549d7f01..67ab1c7eb40f 100644
--- a/providers/mlx5/dr_send.c
+++ b/providers/mlx5/dr_send.c
@@ -820,6 +820,12 @@ int dr_send_postsend_action(struct mlx5dv_dr_domain *dmn,
return ret;
}
+bool dr_send_allow_fl(struct dr_devx_caps *caps)
+{
+ return (caps->roce_caps.roce_en &&
+ caps->roce_caps.fl_rc_qp_when_roce_enabled);
+}
+
static int dr_prepare_qp_to_rts(struct mlx5dv_dr_domain *dmn)
{
struct dr_devx_qp_rts_attr rts_attr = {};
@@ -844,7 +850,7 @@ static int dr_prepare_qp_to_rts(struct mlx5dv_dr_domain *dmn)
rtr_attr.port_num = port;
/* Enable force-loopback on the QP */
- if (dmn->info.caps.roce_caps.fl_rc_qp_when_roce_enabled) {
+ if (dr_send_allow_fl(&dmn->info.caps)) {
rtr_attr.fl = true;
} else {
ret = dr_devx_query_gid(dmn->ctx, port, gid_index, &rtr_attr.dgid_attr);
diff --git a/providers/mlx5/mlx5_ifc.h b/providers/mlx5/mlx5_ifc.h
index 815207a435a8..58b7da23b3aa 100644
--- a/providers/mlx5/mlx5_ifc.h
+++ b/providers/mlx5/mlx5_ifc.h
@@ -49,6 +49,7 @@ enum {
MLX5_CMD_OP_RTS2RTS_QP = 0x505,
MLX5_CMD_OP_QUERY_QP = 0x50b,
MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT = 0x752,
+ MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT = 0x754,
MLX5_CMD_OP_QUERY_ROCE_ADDRESS = 0x760,
MLX5_CMD_OP_QUERY_LAG = 0x842,
MLX5_CMD_OP_CREATE_TIR = 0x900,
@@ -1963,6 +1964,34 @@ struct mlx5_ifc_query_esw_vport_context_in_bits {
u8 reserved_at_60[0x20];
};
+struct mlx5_ifc_nic_vport_context_bits {
+ u8 reserved_at_0[0x1f];
+ u8 roce_en[0x1];
+
+ u8 reserved_at_20[0x7e0];
+};
+
+struct mlx5_ifc_query_nic_vport_context_out_bits {
+ u8 status[0x8];
+ u8 reserved_at_8[0x18];
+
+ u8 syndrome[0x20];
+
+ u8 reserved_at_40[0x40];
+
+ struct mlx5_ifc_nic_vport_context_bits nic_vport_context;
+};
+
+struct mlx5_ifc_query_nic_vport_context_in_bits {
+ u8 opcode[0x10];
+ u8 reserved_at_10[0x10];
+
+ u8 reserved_at_20[0x10];
+ u8 op_mod[0x10];
+
+ u8 reserved_at_40[0x40];
+};
+
enum {
MLX5_QPC_ST_RC = 0x0,
};
diff --git a/providers/mlx5/mlx5dv_dr.h b/providers/mlx5/mlx5dv_dr.h
index 22e99eef4f32..a130211c15f5 100644
--- a/providers/mlx5/mlx5dv_dr.h
+++ b/providers/mlx5/mlx5dv_dr.h
@@ -582,6 +582,7 @@ struct dr_devx_vport_cap {
};
struct dr_devx_roce_cap {
+ bool roce_en;
bool fl_rc_qp_when_roce_enabled;
};
@@ -1032,6 +1033,7 @@ struct dr_send_ring {
int dr_send_ring_alloc(struct mlx5dv_dr_domain *dmn);
void dr_send_ring_free(struct dr_send_ring *send_ring);
int dr_send_ring_force_drain(struct mlx5dv_dr_domain *dmn);
+bool dr_send_allow_fl(struct dr_devx_caps *caps);
int dr_send_postsend_ste(struct mlx5dv_dr_domain *dmn, struct dr_ste *ste,
uint8_t *data, uint16_t size, uint16_t offset);
int dr_send_postsend_htbl(struct mlx5dv_dr_domain *dmn, struct dr_ste_htbl *htbl,
--
2.25.4