From 8aae7abe241c81ef7d461940d7bb7f2973172b99 Mon Sep 17 00:00:00 2001
From: Ido Kalir <idok@mellanox.com>
Date: Tue, 18 Aug 2020 16:03:05 +0300
Subject: [PATCH] tests: Add mlx5 CQ tests
Add tests for mlx5dv CQ, including traffic and some bad creation flows.
Signed-off-by: Ido Kalir <idok@mellanox.com>
Signed-off-by: Edward Srouji <edwards@nvidia.com>
---
tests/CMakeLists.txt | 1 +
tests/base.py | 28 ++++-
tests/mlx5_base.py | 6 +-
tests/test_mlx5_cq.py | 237 ++++++++++++++++++++++++++++++++++++++++++
4 files changed, 269 insertions(+), 3 deletions(-)
create mode 100644 tests/test_mlx5_cq.py
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 4de98d08a81e..ce3b5ef25b81 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -14,6 +14,7 @@ rdma_python_test(tests
test_cqex.py
test_device.py
test_efadv.py
+ test_mlx5_cq.py
test_mlx5_dc.py
test_mlx5_lag_affinity.py
test_mlx5_pp.py
diff --git a/tests/base.py b/tests/base.py
index 3eb5f5db9648..1ca52f0ce5bf 100644
--- a/tests/base.py
+++ b/tests/base.py
@@ -104,6 +104,29 @@ class RDMATestCase(unittest.TestCase):
self.gid_index = gid_index
self.pkey_index = pkey_index
self.ip_addr = None
+ self.pre_environment = {}
+
+ def set_env_variable(self, var, value):
+ """
+ Set environment variable. The current value for each variable is stored
+ and is set back at the end of the test.
+ :param var: The name of the environment variable
+ :param value: The requested new value of this environment variable
+ """
+ if var not in self.pre_environment.keys():
+ self.pre_environment[var] = os.environ.get(var)
+ os.environ[var] = value
+
+ def tearDown(self):
+ """
+ Restore the previous environment variables values before ending the test.
+ """
+ for k, v in self.pre_environment.items():
+ if v is None:
+ os.environ.pop(k)
+ else:
+ os.environ[k] = v
+ super().tearDown()
def is_eth_and_has_roce_hw_bug(self):
"""
@@ -256,7 +279,7 @@ class TrafficResources(BaseResources):
needed for traffic.
"""
def __init__(self, dev_name, ib_port, gid_index, with_srq=False,
- qp_count=1):
+ qp_count=1, msg_size=1024):
"""
Initializes a TrafficResources object with the given values and creates
basic RDMA resources.
@@ -265,11 +288,12 @@ class TrafficResources(BaseResources):
:param gid_index: Which GID index to use
:param with_srq: If True, create SRQ and attach to QPs
:param qp_count: Number of QPs to create
+ :param msg_size: Size of resource msg. If None, use 1024 as default.
"""
super(TrafficResources, self).__init__(dev_name=dev_name,
ib_port=ib_port,
gid_index=gid_index)
- self.msg_size = 1024
+ self.msg_size = msg_size
self.num_msgs = 1000
self.port_attr = None
self.mr = None
diff --git a/tests/mlx5_base.py b/tests/mlx5_base.py
index 099906f35129..a4202bae6a0d 100644
--- a/tests/mlx5_base.py
+++ b/tests/mlx5_base.py
@@ -18,8 +18,9 @@ from pyverbs.mr import MR
class Mlx5DcResources(TrafficResources):
def __init__(self, dev_name, ib_port, gid_index, send_ops_flags,
- qp_count=1):
+ qp_count=1, create_flags=0):
self.send_ops_flags = send_ops_flags
+ self.create_flags = create_flags
super().__init__(dev_name, ib_port, gid_index, with_srq=True,
qp_count=qp_count)
@@ -77,7 +78,10 @@ class Mlx5DcResources(TrafficResources):
try:
for _ in range(self.qp_count):
comp_mask = dve.MLX5DV_QP_INIT_ATTR_MASK_DC
+ if self.create_flags:
+ comp_mask |= dve.MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS
attr = Mlx5DVQPInitAttr(comp_mask=comp_mask,
+ create_flags=self.create_flags,
dc_init_attr=Mlx5DVDCInitAttr())
qp = Mlx5QP(self.ctx, qp_init_attr, attr)
self.qps.append(qp)
diff --git a/tests/test_mlx5_cq.py b/tests/test_mlx5_cq.py
new file mode 100644
index 000000000000..1f757c27345e
--- /dev/null
+++ b/tests/test_mlx5_cq.py
@@ -0,0 +1,237 @@
+import unittest
+import errno
+
+from pyverbs.providers.mlx5.mlx5dv import Mlx5Context, Mlx5DVContextAttr, \
+ Mlx5DVCQInitAttr, Mlx5CQ, context_flags_to_str
+from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsUserError
+from tests.base import RDMATestCase, RCResources
+import pyverbs.providers.mlx5.mlx5_enums as dve
+from tests.mlx5_base import Mlx5DcResources
+from pyverbs.cq import CqInitAttrEx
+import pyverbs.enums as e
+import tests.utils as u
+
+
+def create_dv_cq(res):
+ """
+ Create Mlx5 DV CQ.
+ :param res: An instance of BaseResources.
+ :return: None
+ """
+ dvcq_init_attr = Mlx5DVCQInitAttr()
+ if res.cqe_comp_res_format:
+ dvcq_init_attr.cqe_comp_res_format = res.cqe_comp_res_format
+ dvcq_init_attr.comp_mask |= dve.MLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE
+ if res.flags:
+ dvcq_init_attr.flags = res.flags
+ dvcq_init_attr.comp_mask |= dve.MLX5DV_CQ_INIT_ATTR_MASK_FLAGS
+ if res.cqe_size:
+ dvcq_init_attr.cqe_size = res.cqe_size
+ dvcq_init_attr.comp_mask |= dve.MLX5DV_CQ_INIT_ATTR_MASK_CQE_SIZE
+ try:
+ res.cq = Mlx5CQ(res.ctx, CqInitAttrEx(), dvcq_init_attr)
+ except PyverbsRDMAError as ex:
+ if ex.error_code == errno.EOPNOTSUPP:
+ raise unittest.SkipTest('Create Mlx5DV CQ is not supported')
+ raise ex
+
+
+class Mlx5CQRes(RCResources):
+ def __init__(self, dev_name, ib_port, gid_index, cqe_comp_res_format=None,
+ flags=None, cqe_size=None, msg_size=1024, requested_dev_cap=None):
+ """
+ Initialize Mlx5 DV CQ resources based on RC resources that include RC
+ QP.
+ :param dev_name: Device name to be used
+ :param ib_port: IB port of the device to use
+ :param gid_index: Which GID index to use
+ :param cqe_comp_res_format: Type of compression to use
+ :param flags: DV CQ specific flags
+ :param cqe_size: The CQE size
+ :param msg_size: The resource msg size
+ :param requested_dev_cap: A necessary device cap. If it's not supported
+ by the device, the test will be skipped.
+ """
+ self.cqe_comp_res_format = cqe_comp_res_format
+ self.flags = flags
+ self.cqe_size = cqe_size
+ self.requested_dev_cap = requested_dev_cap
+ super().__init__(dev_name, ib_port, gid_index, msg_size=msg_size)
+
+ def create_context(self):
+ mlx5dv_attr = Mlx5DVContextAttr()
+ try:
+ self.ctx = Mlx5Context(mlx5dv_attr, name=self.dev_name)
+ except PyverbsUserError as ex:
+ raise unittest.SkipTest(f'Could not open mlx5 context ({ex})')
+ except PyverbsRDMAError:
+ raise unittest.SkipTest('Opening mlx5 context is not supported')
+ if self.requested_dev_cap:
+ if not self.ctx.query_mlx5_device().flags & self.requested_dev_cap:
+ miss_caps = context_flags_to_str(self.requested_dev_cap)
+ raise unittest.SkipTest(f'Device caps doesn\'t support {miss_caps}')
+
+ def create_cq(self):
+ create_dv_cq(self)
+
+
+class Mlx5DvCqDcRes(Mlx5DcResources):
+ def __init__(self, dev_name, ib_port, gid_index, cqe_comp_res_format=None,
+ flags=None, cqe_size=None, create_flags=None):
+ """
+ Initialize Mlx5 DV CQ resources based on RC resources that include RC
+ QP.
+ :param dev_name: Device name to be used
+ :param ib_port: IB port of the device to use
+ :param gid_index: Which GID index to use
+ :param cqe_comp_res_format: Type of compression to use
+ :param flags: DV CQ specific flags
+ :param cqe_size: The CQ's CQe size
+ :param create_flags: DV QP specific flags
+ """
+ self.cqe_comp_res_format = cqe_comp_res_format
+ self.flags = flags
+ self.cqe_size = cqe_size
+ super().__init__(dev_name, ib_port, gid_index,
+ send_ops_flags=e.IBV_QP_EX_WITH_SEND,
+ create_flags=create_flags)
+
+ def create_cq(self):
+ create_dv_cq(self)
+
+
+class DvCqTest(RDMATestCase):
+ def setUp(self):
+ super().setUp()
+ self.iters = 10
+ self.server = None
+ self.client = None
+ self.traffic_args = None
+
+ def create_players(self, resource, **resource_arg):
+ """
+ Init DV CQ tests resources.
+ :param resource: The RDMA resources to use.
+ :param resource_arg: Dict of args that specify the resource specific
+ attributes.
+ :return: None
+ """
+ self.client = resource(**self.dev_info, **resource_arg)
+ self.server = resource(**self.dev_info, **resource_arg)
+ self.client.pre_run(self.server.psns, self.server.qps_num)
+ self.server.pre_run(self.client.psns, self.client.qps_num)
+ if resource == Mlx5DvCqDcRes:
+ self.client.remote_dct_num = self.server.dct_qp.qp_num
+ self.server.remote_dct_num = self.client.dct_qp.qp_num
+ self.traffic_args = {'client': self.client, 'server': self.server,
+ 'iters': self.iters, 'gid_idx': self.gid_index,
+ 'port': self.ib_port}
+
+ def test_dv_cq_traffic(self):
+ """
+ Run SEND traffic using DC CQ.
+ """
+ self.create_players(Mlx5CQRes)
+ u.traffic(**self.traffic_args, is_cq_ex=True)
+
+ def test_dv_cq_compression_flags(self):
+ """
+ Create DV CQ with different types of CQE compression formats. The test
+ also does bad flow and try to use more than one compression formats.
+ """
+ # Create DV CQ with all legal compression flags.
+ for comp_type in [dve.MLX5DV_CQE_RES_FORMAT_CSUM_STRIDX,
+ dve.MLX5DV_CQE_RES_FORMAT_CSUM,
+ dve.MLX5DV_CQE_RES_FORMAT_HASH]:
+ self.create_players(Mlx5CQRes, cqe_comp_res_format=comp_type,
+ requested_dev_cap=dve.MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP)
+ u.traffic(**self.traffic_args, is_cq_ex=True)
+
+ # Try to create DV CQ with more than one compression flags.
+ cqe_multi_format = dve.MLX5DV_CQE_RES_FORMAT_HASH | \
+ dve.MLX5DV_CQE_RES_FORMAT_CSUM
+ with self.assertRaises(PyverbsRDMAError) as ex:
+ self.create_players(Mlx5CQRes, cqe_comp_res_format=cqe_multi_format)
+ self.assertEqual(ex.exception.error_code, errno.EINVAL)
+
+ def test_dv_cq_padding(self):
+ """
+ Create DV CQ with padding flag.
+ """
+ self.create_players(Mlx5CQRes, cqe_size=128,
+ flags=dve.MLX5DV_CQ_INIT_ATTR_FLAGS_CQE_PAD,
+ requested_dev_cap=dve.MLX5DV_CONTEXT_FLAGS_CQE_128B_PAD)
+ u.traffic(**self.traffic_args, is_cq_ex=True)
+
+ def test_dv_cq_padding_not_aligned_cqe_size(self):
+ """
+ Create DV CQ with padding flag when CQE size is not 128B. The creation
+ should fail because padding is supported only with CQE size of 128B.
+ """
+ # Padding flag works only when the cqe size is 128.
+ with self.assertRaises(PyverbsRDMAError) as ex:
+ self.create_players(Mlx5CQRes, cqe_size=64,
+ flags=dve.MLX5DV_CQ_INIT_ATTR_FLAGS_CQE_PAD,
+ requested_dev_cap=dve.MLX5DV_CONTEXT_FLAGS_CQE_128B_PAD)
+ self.assertEqual(ex.exception.error_code, errno.EINVAL)
+
+ def test_dv_cq_cqe_size_128(self):
+ """
+ Test multiple sizes of msg using CQE size of 128B.
+ """
+ msg_sizes = [60, # Lower than 64B
+ 70, # In range of 64B - 128B
+ 140] # Bigger than 128B
+ for size in msg_sizes:
+ self.create_players(Mlx5CQRes, cqe_size=128, msg_size=size)
+ u.traffic(**self.traffic_args, is_cq_ex=True)
+
+ def test_dv_cq_cqe_size_64(self):
+ """
+ Test multiple sizes of msg using CQE size of 64B.
+ """
+ msg_sizes = [16, # Lower than 32B
+ 60, # In range of 32B - 64B
+ 70] # Bigger than 64B
+ for size in msg_sizes:
+ self.create_players(Mlx5CQRes, cqe_size=64, msg_size=size)
+ u.traffic(**self.traffic_args, is_cq_ex=True)
+
+ def test_dv_cq_cqe_size_with_bad_size(self):
+ """
+ Create CQ with ilegal cqe_size value.
+ """
+ # Set the CQE size in the CQE creation.
+ with self.assertRaises(PyverbsRDMAError) as ex:
+ self.create_players(Mlx5CQRes, cqe_size=100)
+ self.assertEqual(ex.exception.error_code, errno.EINVAL)
+
+ # Set the CQE size using the environment value.
+ self.set_env_variable('MLX5_CQE_SIZE', '100')
+ with self.assertRaises(PyverbsRDMAError) as ex:
+ self.create_players(Mlx5CQRes)
+ self.assertEqual(ex.exception.error_code, errno.EINVAL)
+
+ def test_dv_cq_cqe_size_environment_var(self):
+ """
+ Create DV CQs with all the legal cqe_size values using the environment
+ variable mechanism.
+ """
+ for cqe_size in ['64', '128']:
+ self.set_env_variable('MLX5_CQE_SIZE', cqe_size)
+ self.create_players(Mlx5CQRes)
+
+ def test_scatter_to_cqe_control_by_qp(self):
+ """
+ Create QP with specific SCATTER_TO_CQE flags. The test set different
+ values in the scatter2cqe environment variable and create the QP with
+ enable/disable flags. The QP should ignore the environment variable
+ value and behave according to the specific creation flag.
+ """
+ for s2c_env_val in ['0', '1']:
+ for qp_s2c_value in [dve.MLX5DV_QP_CREATE_DISABLE_SCATTER_TO_CQE,
+ dve.MLX5DV_QP_CREATE_ALLOW_SCATTER_TO_CQE]:
+ self.set_env_variable('MLX5_SCATTER_TO_CQE', s2c_env_val)
+ self.create_players(Mlx5DvCqDcRes, create_flags=qp_s2c_value)
+ u.traffic(**self.traffic_args, new_send=True,
+ send_op=e.IBV_QP_EX_WITH_SEND, is_cq_ex=True)
--
2.25.4