From dadee6df65387ba0fea0d78cc9c99af0350d3c37 Mon Sep 17 00:00:00 2001
From: Alex Vesker <valex@nvidia.com>
Date: Tue, 8 Dec 2020 18:41:02 +0200
Subject: [PATCH] mlx5: DR, Avoid ICM depletion on multiple domains

[ Upstream commit 951fdedd1ad580a0281e9bca22477942f5256c69 ]

When running multiple domains on the same PF, large ICM allocation can
deplete all of the resource and lead to failures on other domains, this
was seen on ConnectX6DX devices with limited size of action ICM (modify
header).
The solution is take into consideration the total available resource
size.

Fixes: c86f095752f2 ("mlx5: DR, Increase ICM action memory allocation size up-to 8MB")
Signed-off-by: Alex Vesker <valex@nvidia.com>
Signed-off-by: Erez Shitrit <erezsh@nvidia.com>
Signed-off-by: Yishai Hadas <yishaih@nvidia.com>
Signed-off-by: Nicolas Morey-Chaisemartin <nmoreychaisemartin@suse.com>
---
 providers/mlx5/dr_domain.c | 26 ++++++++++++++++++++------
 1 file changed, 20 insertions(+), 6 deletions(-)

diff --git a/providers/mlx5/dr_domain.c b/providers/mlx5/dr_domain.c
index 0a4b565ef85b..1a999965cb9b 100644
--- a/providers/mlx5/dr_domain.c
+++ b/providers/mlx5/dr_domain.c
@@ -283,24 +283,38 @@ static void dr_domain_caps_uninit(struct mlx5dv_dr_domain *dmn)
 
 static int dr_domain_check_icm_memory_caps(struct mlx5dv_dr_domain *dmn)
 {
+	uint32_t max_req_bytes_log, max_req_chunks_log;
+
+	/* Check for minimum ICM log byte size requirements */
 	if (dmn->info.caps.log_modify_hdr_icm_size < DR_CHUNK_SIZE_4K +
 	    DR_MODIFY_ACTION_LOG_SIZE) {
 		errno = ENOMEM;
 		return errno;
 	}
 
-	dmn->info.max_log_action_icm_sz = min_t(uint32_t,
-						DR_CHUNK_SIZE_1024K,
-						dmn->info.caps.log_modify_hdr_icm_size
-						- DR_MODIFY_ACTION_LOG_SIZE);
-
 	if (dmn->info.caps.log_icm_size < DR_CHUNK_SIZE_1024K +
 	    DR_STE_LOG_SIZE) {
 		errno = ENOMEM;
 		return errno;
 	}
 
-	dmn->info.max_log_sw_icm_sz = DR_CHUNK_SIZE_1024K;
+	/* Current code tries to use large allocations to improve our internal
+	 * memory allocation (less DMs and less FW calls).
+	 * When creating multiple domains on the same PF, we want to make sure
+	 * we don't deplete all of the ICM resources on a single domain.
+	 * To provide some functionality with a limited resource we will use
+	 * up to 1/8 of the total available size allowing opening a domain
+	 * of each type.
+	 */
+	max_req_bytes_log = dmn->info.caps.log_modify_hdr_icm_size - 3;
+	max_req_chunks_log = max_req_bytes_log - DR_MODIFY_ACTION_LOG_SIZE;
+	dmn->info.max_log_action_icm_sz =
+		min_t(uint32_t, DR_CHUNK_SIZE_1024K, max_req_chunks_log);
+
+	max_req_bytes_log = dmn->info.caps.log_icm_size - 3;
+	max_req_chunks_log = max_req_bytes_log - DR_STE_LOG_SIZE;
+	dmn->info.max_log_sw_icm_sz =
+		min_t(uint32_t, DR_CHUNK_SIZE_1024K, max_req_chunks_log);
 
 	return 0;
 }
-- 
2.25.4