ena module for Driver Update Program
CentOS Sources
2019-01-09 fc384122a7a805dde4dbbf3c4c4ef94a5ca0f2fe
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
Date: Tue, 30 Oct 2018 13:25:24 -0400
From: linville@redhat.com
To: rhkernel-list@redhat.com
Cc: Lin Liu <linl@redhat.com>
Subject: [RHEL7 09/28] net: ena: fix incorrect usage of memory barriers
 
From: "John W. Linville" <linville@redhat.com>
 
BZ: https://bugzilla.redhat.com/show_bug.cgi?id=1633418
Brew: https://brewweb.engineering.redhat.com/brew/taskinfo?taskID=18850510
 
Added memory barriers where they were missing to support multiple
architectures, and removed redundant ones.
 
As part of removing the redundant memory barriers and improving
performance, we moved to more relaxed versions of memory barriers,
as well as to the more relaxed version of writel - writel_relaxed,
while maintaining correctness.
 
Signed-off-by: Netanel Belgazal <netanel@amazon.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
(cherry picked from commit 37dff155dcf57f6c08bf1641c5ddf9abd45f2b1f)
Signed-off-by: John W. Linville <linville@redhat.com>
---
 drivers/net/ethernet/amazon/ena/ena_com.c     | 16 +++++-----
 drivers/net/ethernet/amazon/ena/ena_eth_com.c |  6 ++++
 drivers/net/ethernet/amazon/ena/ena_eth_com.h |  8 ++---
 drivers/net/ethernet/amazon/ena/ena_netdev.c  | 30 +++++++------------
 4 files changed, 26 insertions(+), 34 deletions(-)
 
diff --git a/drivers/net/ethernet/amazon/ena/ena_com.c b/drivers/net/ethernet/amazon/ena/ena_com.c
index c37deef3bcf1..7635c38e77dd 100644
--- a/drivers/net/ethernet/amazon/ena/ena_com.c
+++ b/drivers/net/ethernet/amazon/ena/ena_com.c
@@ -464,7 +464,7 @@ static void ena_com_handle_admin_completion(struct ena_com_admin_queue *admin_qu
         /* Do not read the rest of the completion entry before the
          * phase bit was validated
          */
-        rmb();
+        dma_rmb();
         ena_com_handle_single_admin_completion(admin_queue, cqe);
 
         head_masked++;
@@ -627,15 +627,8 @@ static u32 ena_com_reg_bar_read32(struct ena_com_dev *ena_dev, u16 offset)
     mmio_read_reg |= mmio_read->seq_num &
             ENA_REGS_MMIO_REG_READ_REQ_ID_MASK;
 
-    /* make sure read_resp->req_id get updated before the hw can write
-     * there
-     */
-    wmb();
-
-    writel_relaxed(mmio_read_reg,
-               ena_dev->reg_bar + ENA_REGS_MMIO_REG_READ_OFF);
+    writel(mmio_read_reg, ena_dev->reg_bar + ENA_REGS_MMIO_REG_READ_OFF);
 
-    mmiowb();
     for (i = 0; i < timeout; i++) {
         if (READ_ONCE(read_resp->req_id) == mmio_read->seq_num)
             break;
@@ -1798,6 +1791,11 @@ void ena_com_aenq_intr_handler(struct ena_com_dev *dev, void *data)
     /* Go over all the events */
     while ((READ_ONCE(aenq_common->flags) &
         ENA_ADMIN_AENQ_COMMON_DESC_PHASE_MASK) == phase) {
+        /* Make sure the phase bit (ownership) is as expected before
+         * reading the rest of the descriptor.
+         */
+        dma_rmb();
+
         pr_debug("AENQ! Group[%x] Syndrom[%x] timestamp: [%llus]\n",
              aenq_common->group, aenq_common->syndrom,
              (u64)aenq_common->timestamp_low +
diff --git a/drivers/net/ethernet/amazon/ena/ena_eth_com.c b/drivers/net/ethernet/amazon/ena/ena_eth_com.c
index ea149c134e15..1c682b76190f 100644
--- a/drivers/net/ethernet/amazon/ena/ena_eth_com.c
+++ b/drivers/net/ethernet/amazon/ena/ena_eth_com.c
@@ -51,6 +51,11 @@ static inline struct ena_eth_io_rx_cdesc_base *ena_com_get_next_rx_cdesc(
     if (desc_phase != expected_phase)
         return NULL;
 
+    /* Make sure we read the rest of the descriptor after the phase bit
+     * has been read
+     */
+    dma_rmb();
+
     return cdesc;
 }
 
@@ -493,6 +498,7 @@ int ena_com_tx_comp_req_id_get(struct ena_com_io_cq *io_cq, u16 *req_id)
     if (cdesc_phase != expected_phase)
         return -EAGAIN;
 
+    dma_rmb();
     if (unlikely(cdesc->req_id >= io_cq->q_depth)) {
         pr_err("Invalid req id %d\n", cdesc->req_id);
         return -EINVAL;
diff --git a/drivers/net/ethernet/amazon/ena/ena_eth_com.h b/drivers/net/ethernet/amazon/ena/ena_eth_com.h
index 6fdc753d9483..2f7657227cfe 100644
--- a/drivers/net/ethernet/amazon/ena/ena_eth_com.h
+++ b/drivers/net/ethernet/amazon/ena/ena_eth_com.h
@@ -107,8 +107,7 @@ static inline int ena_com_sq_empty_space(struct ena_com_io_sq *io_sq)
     return io_sq->q_depth - 1 - cnt;
 }
 
-static inline int ena_com_write_sq_doorbell(struct ena_com_io_sq *io_sq,
-                        bool relaxed)
+static inline int ena_com_write_sq_doorbell(struct ena_com_io_sq *io_sq)
 {
     u16 tail;
 
@@ -117,10 +116,7 @@ static inline int ena_com_write_sq_doorbell(struct ena_com_io_sq *io_sq,
     pr_debug("write submission queue doorbell for queue: %d tail: %d\n",
          io_sq->qid, tail);
 
-    if (relaxed)
-        writel_relaxed(tail, io_sq->db_addr);
-    else
-        writel(tail, io_sq->db_addr);
+    writel(tail, io_sq->db_addr);
 
     return 0;
 }
diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.c b/drivers/net/ethernet/amazon/ena/ena_netdev.c
index 9497db03bec8..4ca0af957654 100644
--- a/drivers/net/ethernet/amazon/ena/ena_netdev.c
+++ b/drivers/net/ethernet/amazon/ena/ena_netdev.c
@@ -551,14 +551,9 @@ static int ena_refill_rx_bufs(struct ena_ring *rx_ring, u32 num)
                 rx_ring->qid, i, num);
     }
 
-    if (likely(i)) {
-        /* Add memory barrier to make sure the desc were written before
-         * issue a doorbell
-         */
-        wmb();
-        ena_com_write_sq_doorbell(rx_ring->ena_com_io_sq, true);
-        mmiowb();
-    }
+    /* ena_com_write_sq_doorbell issues a wmb() */
+    if (likely(i))
+        ena_com_write_sq_doorbell(rx_ring->ena_com_io_sq);
 
     rx_ring->next_to_use = next_to_use;
 
@@ -2105,12 +2100,6 @@ static netdev_tx_t ena_start_xmit(struct sk_buff *skb, struct net_device *dev)
     tx_ring->next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use,
         tx_ring->ring_size);
 
-    /* This WMB is aimed to:
-     * 1 - perform smp barrier before reading next_to_completion
-     * 2 - make sure the desc were written before trigger DB
-     */
-    wmb();
-
     /* stop the queue when no more space available, the packet can have up
      * to sgl_size + 2. one for the meta descriptor and one for header
      * (if the header is larger than tx_max_header_size).
@@ -2129,10 +2118,11 @@ static netdev_tx_t ena_start_xmit(struct sk_buff *skb, struct net_device *dev)
          * stop the queue but meanwhile clean_tx_irq updates
          * next_to_completion and terminates.
          * The queue will remain stopped forever.
-         * To solve this issue this function perform rmb, check
-         * the wakeup condition and wake up the queue if needed.
+         * To solve this issue add a mb() to make sure that
+         * netif_tx_stop_queue() write is vissible before checking if
+         * there is additional space in the queue.
          */
-        smp_rmb();
+        smp_mb();
 
         if (ena_com_sq_empty_space(tx_ring->ena_com_io_sq)
                 > ENA_TX_WAKEUP_THRESH) {
@@ -2144,8 +2134,10 @@ static netdev_tx_t ena_start_xmit(struct sk_buff *skb, struct net_device *dev)
     }
 
     if (netif_xmit_stopped(txq) || !skb->xmit_more) {
-        /* trigger the dma engine */
-        ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq, false);
+        /* trigger the dma engine. ena_com_write_sq_doorbell()
+         * has a mb
+         */
+        ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq);
         u64_stats_update_begin(&tx_ring->syncp);
         tx_ring->tx_stats.doorbells++;
         u64_stats_update_end(&tx_ring->syncp);
-- 
2.17.1