From 2432a74c845b7b529cb6e9044140b9445922a7ae Mon Sep 17 00:00:00 2001 From: Alex Vesker Date: Wed, 28 Oct 2020 12:26:32 +0200 Subject: [PATCH] mlx5: DR, Fix incorrect use of fl_roce_enabled capability [ Upstream commit 2337d6790ad21b1d0c5373cf2aa6f8e70a510434 ] Creating a FL QP should be allowed only when RoCE is enabled (roce_en) and FL is supported with RoCE enabled. Previously we relied on the general HCA cap whether RoCE is enabled and ignored its real state. Creating a FL QP even if RoCE is disabled which could results in a failure to modify QP. Fixes: 6724f6530d3e ("mlx5: DR, Query RoCE capabilities") Signed-off-by: Alex Vesker Signed-off-by: Yishai Hadas Signed-off-by: Nicolas Morey-Chaisemartin --- providers/mlx5/dr_devx.c | 24 ++++++++++++++++++++++++ providers/mlx5/dr_domain.c | 2 +- providers/mlx5/dr_send.c | 8 +++++++- providers/mlx5/mlx5_ifc.h | 29 +++++++++++++++++++++++++++++ providers/mlx5/mlx5dv_dr.h | 2 ++ 5 files changed, 63 insertions(+), 2 deletions(-) diff --git a/providers/mlx5/dr_devx.c b/providers/mlx5/dr_devx.c index cd0f8bbc2e44..74a8155b8777 100644 --- a/providers/mlx5/dr_devx.c +++ b/providers/mlx5/dr_devx.c @@ -66,6 +66,26 @@ int dr_devx_query_esw_vport_context(struct ibv_context *ctx, return 0; } +static int dr_devx_query_nic_vport_context(struct ibv_context *ctx, + bool *roce_en) +{ + uint32_t out[DEVX_ST_SZ_DW(query_nic_vport_context_out)] = {}; + uint32_t in[DEVX_ST_SZ_DW(query_nic_vport_context_in)] = {}; + int err; + + DEVX_SET(query_nic_vport_context_in, in, opcode, + MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT); + err = mlx5dv_devx_general_cmd(ctx, in, sizeof(in), out, sizeof(out)); + if (err) { + dr_dbg_ctx(ctx, "Query nic vport context failed %d\n", err); + return err; + } + + *roce_en = DEVX_GET(query_nic_vport_context_out, out, + nic_vport_context.roce_en); + return 0; +} + int dr_devx_query_gvmi(struct ibv_context *ctx, bool other_vport, uint16_t vport_number, uint16_t *gvmi) { @@ -225,6 +245,10 @@ int dr_devx_query_device(struct ibv_context *ctx, struct dr_devx_caps *caps) /* RoCE caps */ if (roce) { + err = dr_devx_query_nic_vport_context(ctx, &caps->roce_caps.roce_en); + if (err) + return err; + DEVX_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP); DEVX_SET(query_hca_cap_in, in, op_mod, MLX5_SET_HCA_CAP_OP_MOD_ROCE | diff --git a/providers/mlx5/dr_domain.c b/providers/mlx5/dr_domain.c index 916283e505aa..b47c5841f624 100644 --- a/providers/mlx5/dr_domain.c +++ b/providers/mlx5/dr_domain.c @@ -199,7 +199,7 @@ static int dr_domain_caps_init(struct ibv_context *ctx, * force-loopback. */ if ((dmn->type != MLX5DV_DR_DOMAIN_TYPE_FDB) && - !dmn->info.caps.roce_caps.fl_rc_qp_when_roce_enabled) + !dr_send_allow_fl(&dmn->info.caps)) return 0; ret = dr_domain_query_fdb_caps(ctx, dmn); diff --git a/providers/mlx5/dr_send.c b/providers/mlx5/dr_send.c index dfda549d7f01..67ab1c7eb40f 100644 --- a/providers/mlx5/dr_send.c +++ b/providers/mlx5/dr_send.c @@ -820,6 +820,12 @@ int dr_send_postsend_action(struct mlx5dv_dr_domain *dmn, return ret; } +bool dr_send_allow_fl(struct dr_devx_caps *caps) +{ + return (caps->roce_caps.roce_en && + caps->roce_caps.fl_rc_qp_when_roce_enabled); +} + static int dr_prepare_qp_to_rts(struct mlx5dv_dr_domain *dmn) { struct dr_devx_qp_rts_attr rts_attr = {}; @@ -844,7 +850,7 @@ static int dr_prepare_qp_to_rts(struct mlx5dv_dr_domain *dmn) rtr_attr.port_num = port; /* Enable force-loopback on the QP */ - if (dmn->info.caps.roce_caps.fl_rc_qp_when_roce_enabled) { + if (dr_send_allow_fl(&dmn->info.caps)) { rtr_attr.fl = true; } else { ret = dr_devx_query_gid(dmn->ctx, port, gid_index, &rtr_attr.dgid_attr); diff --git a/providers/mlx5/mlx5_ifc.h b/providers/mlx5/mlx5_ifc.h index 815207a435a8..58b7da23b3aa 100644 --- a/providers/mlx5/mlx5_ifc.h +++ b/providers/mlx5/mlx5_ifc.h @@ -49,6 +49,7 @@ enum { MLX5_CMD_OP_RTS2RTS_QP = 0x505, MLX5_CMD_OP_QUERY_QP = 0x50b, MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT = 0x752, + MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT = 0x754, MLX5_CMD_OP_QUERY_ROCE_ADDRESS = 0x760, MLX5_CMD_OP_QUERY_LAG = 0x842, MLX5_CMD_OP_CREATE_TIR = 0x900, @@ -1963,6 +1964,34 @@ struct mlx5_ifc_query_esw_vport_context_in_bits { u8 reserved_at_60[0x20]; }; +struct mlx5_ifc_nic_vport_context_bits { + u8 reserved_at_0[0x1f]; + u8 roce_en[0x1]; + + u8 reserved_at_20[0x7e0]; +}; + +struct mlx5_ifc_query_nic_vport_context_out_bits { + u8 status[0x8]; + u8 reserved_at_8[0x18]; + + u8 syndrome[0x20]; + + u8 reserved_at_40[0x40]; + + struct mlx5_ifc_nic_vport_context_bits nic_vport_context; +}; + +struct mlx5_ifc_query_nic_vport_context_in_bits { + u8 opcode[0x10]; + u8 reserved_at_10[0x10]; + + u8 reserved_at_20[0x10]; + u8 op_mod[0x10]; + + u8 reserved_at_40[0x40]; +}; + enum { MLX5_QPC_ST_RC = 0x0, }; diff --git a/providers/mlx5/mlx5dv_dr.h b/providers/mlx5/mlx5dv_dr.h index 22e99eef4f32..a130211c15f5 100644 --- a/providers/mlx5/mlx5dv_dr.h +++ b/providers/mlx5/mlx5dv_dr.h @@ -582,6 +582,7 @@ struct dr_devx_vport_cap { }; struct dr_devx_roce_cap { + bool roce_en; bool fl_rc_qp_when_roce_enabled; }; @@ -1032,6 +1033,7 @@ struct dr_send_ring { int dr_send_ring_alloc(struct mlx5dv_dr_domain *dmn); void dr_send_ring_free(struct dr_send_ring *send_ring); int dr_send_ring_force_drain(struct mlx5dv_dr_domain *dmn); +bool dr_send_allow_fl(struct dr_devx_caps *caps); int dr_send_postsend_ste(struct mlx5dv_dr_domain *dmn, struct dr_ste *ste, uint8_t *data, uint16_t size, uint16_t offset); int dr_send_postsend_htbl(struct mlx5dv_dr_domain *dmn, struct dr_ste_htbl *htbl, -- 2.25.4