From 4b7203f835727e9314ef42db682b578730783d7d Mon Sep 17 00:00:00 2001
From: Gal Pressman <galpress@amazon.com>
Date: Wed, 11 Nov 2020 14:21:13 +0200
Subject: [PATCH] efa: Flush write combining writes before writing to the LLQ

[ Upstream commit 9a0d3830da11a187fb6bffe4f6f361560a0b2f40 ]

An mmio_wc_start() is needed before writing to the LLQ memory in order
to prevent the WQEs copy (WC memory) from being reordered relative to
other mmio writes, such as tx doorbells (NC memory).

This prevents the provider to issue more than max_tx_batch LLQ writes
between two doorbells. This is especially relevant when the user calls
the _post API with more WQEs than max_tx_batch.

Fixes: 7aad28d11981 ("efa: Respect maximum TX doorbell batch")
Signed-off-by: Shadi Ammouri <sammouri@amazon.com>
Signed-off-by: Gal Pressman <galpress@amazon.com>
Signed-off-by: Nicolas Morey-Chaisemartin <nmoreychaisemartin@suse.com>
---
 providers/efa/verbs.c | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/providers/efa/verbs.c b/providers/efa/verbs.c
index e179ff24e911..e80660d1907f 100644
--- a/providers/efa/verbs.c
+++ b/providers/efa/verbs.c
@@ -1389,7 +1389,6 @@ static inline void efa_rq_ring_doorbell(struct efa_rq *rq, uint16_t pc)
 
 static inline void efa_sq_ring_doorbell(struct efa_sq *sq, uint16_t pc)
 {
-	mmio_flush_writes();
 	mmio_write32(sq->wq.db, pc);
 }
 
@@ -1510,15 +1509,19 @@ int efa_post_send(struct ibv_qp *ibvqp, struct ibv_send_wr *wr,
 
 		if (curbatch == qp->sq.max_batch_wr) {
 			curbatch = 0;
+			mmio_flush_writes();
 			efa_sq_ring_doorbell(&qp->sq, qp->sq.wq.pc);
+			mmio_wc_start();
 		}
 
 		wr = wr->next;
 	}
 
 ring_db:
-	if (curbatch)
+	if (curbatch) {
+		mmio_flush_writes();
 		efa_sq_ring_doorbell(&qp->sq, qp->sq.wq.pc);
+	}
 
 	/*
 	 * Not using mmio_wc_spinunlock as the doorbell write should be done
@@ -1774,6 +1777,7 @@ static int efa_send_wr_complete(struct ibv_qp_ex *ibvqpx)
 	pc = qp->sq.wq.pc - qp->sq.num_wqe_pending;
 	sq_desc_idx = pc & qp->sq.wq.desc_mask;
 
+	/* mmio_wc_start() comes from efa_send_wr_start() */
 	while (qp->sq.num_wqe_pending) {
 		num_wqe_to_copy = min3(qp->sq.num_wqe_pending,
 				       qp->sq.wq.wqe_cnt - sq_desc_idx,
@@ -1792,13 +1796,17 @@ static int efa_send_wr_complete(struct ibv_qp_ex *ibvqpx)
 			      qp->sq.wq.desc_mask;
 
 		if (curbatch == max_txbatch) {
+			mmio_flush_writes();
 			efa_sq_ring_doorbell(&qp->sq, pc);
 			curbatch = 0;
+			mmio_wc_start();
 		}
 	}
 
-	if (curbatch)
+	if (curbatch) {
+		mmio_flush_writes();
 		efa_sq_ring_doorbell(&qp->sq, qp->sq.wq.pc);
+	}
 out:
 	/*
 	 * Not using mmio_wc_spinunlock as the doorbell write should be done
-- 
2.25.4