From 371f6ca7f5cbad70f6e5fafc12d5448d7b6f0750 Mon Sep 17 00:00:00 2001

From: Arthur Kiyanovski <akiyano@amazon.com>

Date: Thu, 11 Oct 2018 11:26:16 +0300

Subject: [PATCH 02/16] net: ena: minor performance improvement

Reduce fastpath overhead by making ena_com_tx_comp_req_id_get() inline.

Also move it to ena_eth_com.h file with its dependency function

ena_com_cq_inc_head().

Signed-off-by: Arthur Kiyanovski <akiyano@amazon.com>

Signed-off-by: David S. Miller <davem@davemloft.net>

---

 drivers/net/ethernet/amazon/ena/ena_eth_com.c | 43 -----------------

 drivers/net/ethernet/amazon/ena/ena_eth_com.h | 46 ++++++++++++++++++-

 2 files changed, 44 insertions(+), 45 deletions(-)

diff --git a/drivers/net/ethernet/amazon/ena/ena_eth_com.c b/drivers/net/ethernet/amazon/ena/ena_eth_com.c

index 2b3ff0c20155..9c0511e9f9a2 100644

--- a/drivers/net/ethernet/amazon/ena/ena_eth_com.c

+++ b/drivers/net/ethernet/amazon/ena/ena_eth_com.c

@@ -59,15 +59,6 @@ static inline struct ena_eth_io_rx_cdesc_base *ena_com_get_next_rx_cdesc(

 	return cdesc;

 }

-static inline void ena_com_cq_inc_head(struct ena_com_io_cq *io_cq)

-{

-	io_cq->head++;

-

-	/* Switch phase bit in case of wrap around */

-	if (unlikely((io_cq->head & (io_cq->q_depth - 1)) == 0))

-		io_cq->phase ^= 1;

-}

-

 static inline void *get_sq_desc(struct ena_com_io_sq *io_sq)

 {

 	u16 tail_masked;

@@ -477,40 +468,6 @@ int ena_com_add_single_rx_desc(struct ena_com_io_sq *io_sq,

 	return 0;

 }

-int ena_com_tx_comp_req_id_get(struct ena_com_io_cq *io_cq, u16 *req_id)

-{

-	u8 expected_phase, cdesc_phase;

-	struct ena_eth_io_tx_cdesc *cdesc;

-	u16 masked_head;

-

-	masked_head = io_cq->head & (io_cq->q_depth - 1);

-	expected_phase = io_cq->phase;

-

-	cdesc = (struct ena_eth_io_tx_cdesc *)

-		((uintptr_t)io_cq->cdesc_addr.virt_addr +

-		(masked_head * io_cq->cdesc_entry_size_in_bytes));

-

-	/* When the current completion descriptor phase isn't the same as the

-	 * expected, it mean that the device still didn't update

-	 * this completion.

-	 */

-	cdesc_phase = READ_ONCE(cdesc->flags) & ENA_ETH_IO_TX_CDESC_PHASE_MASK;

-	if (cdesc_phase != expected_phase)

-		return -EAGAIN;

-

-	dma_rmb();

-	if (unlikely(cdesc->req_id >= io_cq->q_depth)) {

-		pr_err("Invalid req id %d\n", cdesc->req_id);

-		return -EINVAL;

-	}

-

-	ena_com_cq_inc_head(io_cq);

-

-	*req_id = READ_ONCE(cdesc->req_id);

-

-	return 0;

-}

-

 bool ena_com_cq_empty(struct ena_com_io_cq *io_cq)

 {

 	struct ena_eth_io_rx_cdesc_base *cdesc;

diff --git a/drivers/net/ethernet/amazon/ena/ena_eth_com.h b/drivers/net/ethernet/amazon/ena/ena_eth_com.h

index 2f7657227cfe..4930324e9d8d 100644

--- a/drivers/net/ethernet/amazon/ena/ena_eth_com.h

+++ b/drivers/net/ethernet/amazon/ena/ena_eth_com.h

@@ -86,8 +86,6 @@ int ena_com_add_single_rx_desc(struct ena_com_io_sq *io_sq,

 			       struct ena_com_buf *ena_buf,

 			       u16 req_id);

-int ena_com_tx_comp_req_id_get(struct ena_com_io_cq *io_cq, u16 *req_id);

-

 bool ena_com_cq_empty(struct ena_com_io_cq *io_cq);

 static inline void ena_com_unmask_intr(struct ena_com_io_cq *io_cq,

@@ -159,4 +157,48 @@ static inline void ena_com_comp_ack(struct ena_com_io_sq *io_sq, u16 elem)

 	io_sq->next_to_comp += elem;

 }

+static inline void ena_com_cq_inc_head(struct ena_com_io_cq *io_cq)

+{

+	io_cq->head++;

+

+	/* Switch phase bit in case of wrap around */

+	if (unlikely((io_cq->head & (io_cq->q_depth - 1)) == 0))

+		io_cq->phase ^= 1;

+}

+

+static inline int ena_com_tx_comp_req_id_get(struct ena_com_io_cq *io_cq,

+					     u16 *req_id)

+{

+	u8 expected_phase, cdesc_phase;

+	struct ena_eth_io_tx_cdesc *cdesc;

+	u16 masked_head;

+

+	masked_head = io_cq->head & (io_cq->q_depth - 1);

+	expected_phase = io_cq->phase;

+

+	cdesc = (struct ena_eth_io_tx_cdesc *)

+		((uintptr_t)io_cq->cdesc_addr.virt_addr +

+		(masked_head * io_cq->cdesc_entry_size_in_bytes));

+

+	/* When the current completion descriptor phase isn't the same as the

+	 * expected, it mean that the device still didn't update

+	 * this completion.

+	 */

+	cdesc_phase = READ_ONCE(cdesc->flags) & ENA_ETH_IO_TX_CDESC_PHASE_MASK;

+	if (cdesc_phase != expected_phase)

+		return -EAGAIN;

+

+	dma_rmb();

+

+	*req_id = READ_ONCE(cdesc->req_id);

+	if (unlikely(*req_id >= io_cq->q_depth)) {

+		pr_err("Invalid req id %d\n", cdesc->req_id);

+		return -EINVAL;

+	}

+

+	ena_com_cq_inc_head(io_cq);

+

+	return 0;

+}

+

 #endif /* ENA_ETH_COM_H_ */

-- 

2.19.1

From df44a6755f48dfc5c94d878e80807931460c3846 Mon Sep 17 00:00:00 2001

From: Arthur Kiyanovski <akiyano@amazon.com>

Date: Thu, 11 Oct 2018 11:26:17 +0300

Subject: [PATCH 03/16] net: ena: complete host info to match latest ENA spec

Add new fields and definitions to host info and fill them

according to the latest ENA spec version.

Signed-off-by: Arthur Kiyanovski <akiyano@amazon.com>

Signed-off-by: David S. Miller <davem@davemloft.net>

---

 .../net/ethernet/amazon/ena/ena_admin_defs.h  | 31 ++++++++++++++++++-

 drivers/net/ethernet/amazon/ena/ena_com.c     | 12 +++----

 .../net/ethernet/amazon/ena/ena_common_defs.h |  4 +--

 drivers/net/ethernet/amazon/ena/ena_netdev.c  | 10 ++++--

 4 files changed, 43 insertions(+), 14 deletions(-)

diff --git a/drivers/net/ethernet/amazon/ena/ena_admin_defs.h b/drivers/net/ethernet/amazon/ena/ena_admin_defs.h

index 4532e574ebcd..d735164efea3 100644

--- a/drivers/net/ethernet/amazon/ena/ena_admin_defs.h

+++ b/drivers/net/ethernet/amazon/ena/ena_admin_defs.h

@@ -63,6 +63,8 @@ enum ena_admin_aq_completion_status {

 	ENA_ADMIN_ILLEGAL_PARAMETER		= 5,

 	ENA_ADMIN_UNKNOWN_ERROR			= 6,

+

+	ENA_ADMIN_RESOURCE_BUSY                 = 7,

 };

 enum ena_admin_aq_feature_id {

@@ -702,6 +704,10 @@ enum ena_admin_os_type {

 	ENA_ADMIN_OS_FREEBSD	= 4,

 	ENA_ADMIN_OS_IPXE	= 5,

+

+	ENA_ADMIN_OS_ESXI       = 6,

+

+	ENA_ADMIN_OS_GROUPS_NUM = 6,

 };

 struct ena_admin_host_info {

@@ -723,11 +729,27 @@ struct ena_admin_host_info {

 	/* 7:0 : major

 	 * 15:8 : minor

 	 * 23:16 : sub_minor

+	 * 31:24 : module_type

 	 */

 	u32 driver_version;

 	/* features bitmap */

-	u32 supported_network_features[4];

+	u32 supported_network_features[2];

+

+	/* ENA spec version of driver */

+	u16 ena_spec_version;

+

+	/* ENA device's Bus, Device and Function

+	 * 2:0 : function

+	 * 7:3 : device

+	 * 15:8 : bus

+	 */

+	u16 bdf;

+

+	/* Number of CPUs */

+	u16 num_cpus;

+

+	u16 reserved;

 };

 struct ena_admin_rss_ind_table_entry {

@@ -1008,6 +1030,13 @@ struct ena_admin_ena_mmio_req_read_less_resp {

 #define ENA_ADMIN_HOST_INFO_MINOR_MASK GENMASK(15, 8)

 #define ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT 16

 #define ENA_ADMIN_HOST_INFO_SUB_MINOR_MASK GENMASK(23, 16)

+#define ENA_ADMIN_HOST_INFO_MODULE_TYPE_SHIFT 24

+#define ENA_ADMIN_HOST_INFO_MODULE_TYPE_MASK GENMASK(31, 24)

+#define ENA_ADMIN_HOST_INFO_FUNCTION_MASK GENMASK(2, 0)

+#define ENA_ADMIN_HOST_INFO_DEVICE_SHIFT 3

+#define ENA_ADMIN_HOST_INFO_DEVICE_MASK GENMASK(7, 3)

+#define ENA_ADMIN_HOST_INFO_BUS_SHIFT 8

+#define ENA_ADMIN_HOST_INFO_BUS_MASK GENMASK(15, 8)

 /* aenq_common_desc */

 #define ENA_ADMIN_AENQ_COMMON_DESC_PHASE_MASK BIT(0)

diff --git a/drivers/net/ethernet/amazon/ena/ena_com.c b/drivers/net/ethernet/amazon/ena/ena_com.c

index 7635c38e77dd..b6e6a4721931 100644

--- a/drivers/net/ethernet/amazon/ena/ena_com.c

+++ b/drivers/net/ethernet/amazon/ena/ena_com.c

@@ -41,9 +41,6 @@

 #define ENA_ASYNC_QUEUE_DEPTH 16

 #define ENA_ADMIN_QUEUE_DEPTH 32

-#define MIN_ENA_VER (((ENA_COMMON_SPEC_VERSION_MAJOR) << \

-		ENA_REGS_VERSION_MAJOR_VERSION_SHIFT) \

-		| (ENA_COMMON_SPEC_VERSION_MINOR))

 #define ENA_CTRL_MAJOR		0

 #define ENA_CTRL_MINOR		0

@@ -1400,11 +1397,6 @@ int ena_com_validate_version(struct ena_com_dev *ena_dev)

 			ENA_REGS_VERSION_MAJOR_VERSION_SHIFT,

 		ver & ENA_REGS_VERSION_MINOR_VERSION_MASK);

-	if (ver < MIN_ENA_VER) {

-		pr_err("ENA version is lower than the minimal version the driver supports\n");

-		return -1;

-	}

-

 	pr_info("ena controller version: %d.%d.%d implementation version %d\n",

 		(ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_MASK) >>

 			ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_SHIFT,

@@ -2441,6 +2433,10 @@ int ena_com_allocate_host_info(struct ena_com_dev *ena_dev)

 	if (unlikely(!host_attr->host_info))

 		return -ENOMEM;

+	host_attr->host_info->ena_spec_version =

+		((ENA_COMMON_SPEC_VERSION_MAJOR << ENA_REGS_VERSION_MAJOR_VERSION_SHIFT) |

+		(ENA_COMMON_SPEC_VERSION_MINOR));

+

 	return 0;

 }

diff --git a/drivers/net/ethernet/amazon/ena/ena_common_defs.h b/drivers/net/ethernet/amazon/ena/ena_common_defs.h

index bb8d73676eab..23beb7e7ed7b 100644

--- a/drivers/net/ethernet/amazon/ena/ena_common_defs.h

+++ b/drivers/net/ethernet/amazon/ena/ena_common_defs.h

@@ -32,8 +32,8 @@

 #ifndef _ENA_COMMON_H_

 #define _ENA_COMMON_H_

-#define ENA_COMMON_SPEC_VERSION_MAJOR	0 /*  */

-#define ENA_COMMON_SPEC_VERSION_MINOR	10 /*  */

+#define ENA_COMMON_SPEC_VERSION_MAJOR        2

+#define ENA_COMMON_SPEC_VERSION_MINOR        0

 /* ENA operates with 48-bit memory addresses. ena_mem_addr_t */

 struct ena_common_mem_addr {

diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.c b/drivers/net/ethernet/amazon/ena/ena_netdev.c

index 69a49784b204..0c9c0d3ce856 100644

--- a/drivers/net/ethernet/amazon/ena/ena_netdev.c

+++ b/drivers/net/ethernet/amazon/ena/ena_netdev.c

@@ -2206,7 +2206,8 @@ static u16 ena_select_queue(struct net_device *dev, struct sk_buff *skb,

 	return qid;

 }

-static void ena_config_host_info(struct ena_com_dev *ena_dev)

+static void ena_config_host_info(struct ena_com_dev *ena_dev,

+				 struct pci_dev *pdev)

 {

 	struct ena_admin_host_info *host_info;

 	int rc;

@@ -2220,6 +2221,7 @@ static void ena_config_host_info(struct ena_com_dev *ena_dev)

 	host_info = ena_dev->host_attr.host_info;

+	host_info->bdf = (pdev->bus->number << 8) | pdev->devfn;

 	host_info->os_type = ENA_ADMIN_OS_LINUX;

 	host_info->kernel_ver = LINUX_VERSION_CODE;

 	strlcpy(host_info->kernel_ver_str, utsname()->version,

@@ -2230,7 +2232,9 @@ static void ena_config_host_info(struct ena_com_dev *ena_dev)

 	host_info->driver_version =

 		(DRV_MODULE_VER_MAJOR) |

 		(DRV_MODULE_VER_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) |

-		(DRV_MODULE_VER_SUBMINOR << ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT);

+		(DRV_MODULE_VER_SUBMINOR << ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT) |

+		("K"[0] << ENA_ADMIN_HOST_INFO_MODULE_TYPE_SHIFT);

+	host_info->num_cpus = num_online_cpus();

 	rc = ena_com_set_host_attributes(ena_dev);

 	if (rc) {

@@ -2454,7 +2458,7 @@ static int ena_device_init(struct ena_com_dev *ena_dev, struct pci_dev *pdev,

 	 */

 	ena_com_set_admin_polling_mode(ena_dev, true);

-	ena_config_host_info(ena_dev);

+	ena_config_host_info(ena_dev, pdev);

 	/* Get Device Attributes*/

 	rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx);

-- 

2.19.1

From 0a66e6d1fe86cb3d49fcd76057b4f7a50e0fe49a Mon Sep 17 00:00:00 2001

From: Arthur Kiyanovski <akiyano@amazon.com>

Date: Thu, 11 Oct 2018 11:26:18 +0300

Subject: [PATCH 04/16] net: ena: introduce Low Latency Queues data structures

 according to ENA spec

Low Latency Queues(LLQ) allow usage of device's memory for descriptors

and headers. Such queues decrease processing time since data is already

located on the device when driver rings the doorbell.

Signed-off-by: Arthur Kiyanovski <akiyano@amazon.com>

Signed-off-by: David S. Miller <davem@davemloft.net>

---

 .../net/ethernet/amazon/ena/ena_admin_defs.h  | 90 ++++++++++++++++++-

 drivers/net/ethernet/amazon/ena/ena_com.h     | 38 ++++++++

 drivers/net/ethernet/amazon/ena/ena_netdev.c  |  6 +-

 3 files changed, 128 insertions(+), 6 deletions(-)

diff --git a/drivers/net/ethernet/amazon/ena/ena_admin_defs.h b/drivers/net/ethernet/amazon/ena/ena_admin_defs.h

index d735164efea3..b439ec1b3edb 100644

--- a/drivers/net/ethernet/amazon/ena/ena_admin_defs.h

+++ b/drivers/net/ethernet/amazon/ena/ena_admin_defs.h

@@ -74,6 +74,8 @@ enum ena_admin_aq_feature_id {

 	ENA_ADMIN_HW_HINTS			= 3,

+	ENA_ADMIN_LLQ                           = 4,

+

 	ENA_ADMIN_RSS_HASH_FUNCTION		= 10,

 	ENA_ADMIN_STATELESS_OFFLOAD_CONFIG	= 11,

@@ -485,8 +487,85 @@ struct ena_admin_device_attr_feature_desc {

 	u32 max_mtu;

 };

+enum ena_admin_llq_header_location {

+	/* header is in descriptor list */

+	ENA_ADMIN_INLINE_HEADER                     = 1,

+	/* header in a separate ring, implies 16B descriptor list entry */

+	ENA_ADMIN_HEADER_RING                       = 2,

+};

+

+enum ena_admin_llq_ring_entry_size {

+	ENA_ADMIN_LIST_ENTRY_SIZE_128B              = 1,

+	ENA_ADMIN_LIST_ENTRY_SIZE_192B              = 2,

+	ENA_ADMIN_LIST_ENTRY_SIZE_256B              = 4,

+};

+

+enum ena_admin_llq_num_descs_before_header {

+	ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_0     = 0,

+	ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_1     = 1,

+	ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2     = 2,

+	ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_4     = 4,

+	ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_8     = 8,

+};

+

+/* packet descriptor list entry always starts with one or more descriptors,

+ * followed by a header. The rest of the descriptors are located in the

+ * beginning of the subsequent entry. Stride refers to how the rest of the

+ * descriptors are placed. This field is relevant only for inline header

+ * mode

+ */

+enum ena_admin_llq_stride_ctrl {

+	ENA_ADMIN_SINGLE_DESC_PER_ENTRY             = 1,

+	ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY          = 2,

+};

+

+struct ena_admin_feature_llq_desc {

+	u32 max_llq_num;

+

+	u32 max_llq_depth;

+

+	/*  specify the header locations the device supports. bitfield of

+	 *    enum ena_admin_llq_header_location.

+	 */

+	u16 header_location_ctrl_supported;

+

+	/* the header location the driver selected to use. */

+	u16 header_location_ctrl_enabled;

+

+	/* if inline header is specified - this is the size of descriptor

+	 *    list entry. If header in a separate ring is specified - this is

+	 *    the size of header ring entry. bitfield of enum

+	 *    ena_admin_llq_ring_entry_size. specify the entry sizes the device

+	 *    supports

+	 */

+	u16 entry_size_ctrl_supported;

+

+	/* the entry size the driver selected to use. */

+	u16 entry_size_ctrl_enabled;

+

+	/* valid only if inline header is specified. First entry associated

+	 *    with the packet includes descriptors and header. Rest of the

+	 *    entries occupied by descriptors. This parameter defines the max

+	 *    number of descriptors precedding the header in the first entry.

+	 *    The field is bitfield of enum

+	 *    ena_admin_llq_num_descs_before_header and specify the values the

+	 *    device supports

+	 */

+	u16 desc_num_before_header_supported;

+

+	/* the desire field the driver selected to use */

+	u16 desc_num_before_header_enabled;

+

+	/* valid only if inline was chosen. bitfield of enum

+	 *    ena_admin_llq_stride_ctrl

+	 */

+	u16 descriptors_stride_ctrl_supported;

+

+	/* the stride control the driver selected to use */

+	u16 descriptors_stride_ctrl_enabled;

+};

+

 struct ena_admin_queue_feature_desc {

-	/* including LLQs */

 	u32 max_sq_num;

 	u32 max_sq_depth;

@@ -495,9 +574,9 @@ struct ena_admin_queue_feature_desc {

 	u32 max_cq_depth;

-	u32 max_llq_num;

+	u32 max_legacy_llq_num;

-	u32 max_llq_depth;

+	u32 max_legacy_llq_depth;

 	u32 max_header_size;

@@ -822,6 +901,8 @@ struct ena_admin_get_feat_resp {

 		struct ena_admin_device_attr_feature_desc dev_attr;

+		struct ena_admin_feature_llq_desc llq;

+

 		struct ena_admin_queue_feature_desc max_queue;

 		struct ena_admin_feature_aenq_desc aenq;

@@ -869,6 +950,9 @@ struct ena_admin_set_feat_cmd {

 		/* rss indirection table */

 		struct ena_admin_feature_rss_ind_table ind_table;

+

+		/* LLQ configuration */

+		struct ena_admin_feature_llq_desc llq;

 	} u;

 };

diff --git a/drivers/net/ethernet/amazon/ena/ena_com.h b/drivers/net/ethernet/amazon/ena/ena_com.h

index 7b784f8a06a6..50e6c8f6f138 100644

--- a/drivers/net/ethernet/amazon/ena/ena_com.h

+++ b/drivers/net/ethernet/amazon/ena/ena_com.h

@@ -108,6 +108,14 @@ enum ena_intr_moder_level {

 	ENA_INTR_MAX_NUM_OF_LEVELS,

 };

+struct ena_llq_configurations {

+	enum ena_admin_llq_header_location llq_header_location;

+	enum ena_admin_llq_ring_entry_size llq_ring_entry_size;

+	enum ena_admin_llq_stride_ctrl  llq_stride_ctrl;

+	enum ena_admin_llq_num_descs_before_header llq_num_decs_before_header;

+	u16 llq_ring_entry_size_value;

+};

+

 struct ena_intr_moder_entry {

 	unsigned int intr_moder_interval;

 	unsigned int pkts_per_interval;

@@ -142,6 +150,15 @@ struct ena_com_tx_meta {

 	u16 l4_hdr_len; /* In words */

 };

+struct ena_com_llq_info {

+	u16 header_location_ctrl;

+	u16 desc_stride_ctrl;

+	u16 desc_list_entry_size_ctrl;

+	u16 desc_list_entry_size;

+	u16 descs_num_before_header;

+	u16 descs_per_entry;

+};

+

 struct ena_com_io_cq {

 	struct ena_com_io_desc_addr cdesc_addr;

@@ -179,6 +196,20 @@ struct ena_com_io_cq {

 } ____cacheline_aligned;

+struct ena_com_io_bounce_buffer_control {

+	u8 *base_buffer;

+	u16 next_to_use;

+	u16 buffer_size;

+	u16 buffers_num;  /* Must be a power of 2 */

+};

+

+/* This struct is to keep tracking the current location of the next llq entry */

+struct ena_com_llq_pkt_ctrl {

+	u8 *curr_bounce_buf;

+	u16 idx;

+	u16 descs_left_in_line;

+};

+

 struct ena_com_io_sq {

 	struct ena_com_io_desc_addr desc_addr;

@@ -190,6 +221,9 @@ struct ena_com_io_sq {

 	u32 msix_vector;

 	struct ena_com_tx_meta cached_tx_meta;

+	struct ena_com_llq_info llq_info;

+	struct ena_com_llq_pkt_ctrl llq_buf_ctrl;

+	struct ena_com_io_bounce_buffer_control bounce_buf_ctrl;

 	u16 q_depth;

 	u16 qid;

@@ -197,6 +231,7 @@ struct ena_com_io_sq {

 	u16 idx;

 	u16 tail;

 	u16 next_to_comp;

+	u16 llq_last_copy_tail;

 	u32 tx_max_header_size;

 	u8 phase;

 	u8 desc_entry_size;

@@ -334,6 +369,8 @@ struct ena_com_dev {

 	u16 intr_delay_resolution;

 	u32 intr_moder_tx_interval;

 	struct ena_intr_moder_entry *intr_moder_tbl;

+

+	struct ena_com_llq_info llq_info;

 };

 struct ena_com_dev_get_features_ctx {

@@ -342,6 +379,7 @@ struct ena_com_dev_get_features_ctx {

 	struct ena_admin_feature_aenq_desc aenq;

 	struct ena_admin_feature_offload_desc offload;

 	struct ena_admin_ena_hw_hints hw_hints;

+	struct ena_admin_feature_llq_desc llq;

 };

 struct ena_com_create_io_ctx {

diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.c b/drivers/net/ethernet/amazon/ena/ena_netdev.c

index 0c9c0d3ce856..789556960b8e 100644

--- a/drivers/net/ethernet/amazon/ena/ena_netdev.c

+++ b/drivers/net/ethernet/amazon/ena/ena_netdev.c

@@ -2959,7 +2959,7 @@ static int ena_calc_io_queue_num(struct pci_dev *pdev,

 	/* In case of LLQ use the llq number in the get feature cmd */

 	if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {

-		io_sq_num = get_feat_ctx->max_queues.max_llq_num;

+		io_sq_num = get_feat_ctx->max_queues.max_legacy_llq_num;

 		if (io_sq_num == 0) {

 			dev_err(&pdev->dev,

@@ -2995,7 +2995,7 @@ static void ena_set_push_mode(struct pci_dev *pdev, struct ena_com_dev *ena_dev,

 	has_mem_bar = pci_select_bars(pdev, IORESOURCE_MEM) & BIT(ENA_MEM_BAR);

 	/* Enable push mode if device supports LLQ */

-	if (has_mem_bar && (get_feat_ctx->max_queues.max_llq_num > 0))

+	if (has_mem_bar && get_feat_ctx->max_queues.max_legacy_llq_num > 0)

 		ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_DEV;

 	else

 		ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;

@@ -3131,7 +3131,7 @@ static int ena_calc_queue_size(struct pci_dev *pdev,

 	if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)

 		queue_size = min_t(u32, queue_size,

-				   get_feat_ctx->max_queues.max_llq_depth);

+				   get_feat_ctx->max_queues.max_legacy_llq_depth);

 	queue_size = rounddown_pow_of_two(queue_size);

-- 

2.19.1

From 8e9ebea20ab8db4f3a993e815e0b6b84ce98bbfb Mon Sep 17 00:00:00 2001

From: Arthur Kiyanovski <akiyano@amazon.com>

Date: Thu, 11 Oct 2018 11:26:19 +0300

Subject: [PATCH 05/16] net: ena: add functions for handling Low Latency Queues

 in ena_com

This patch introduces APIs for detection, initialization, configuration

and actual usage of low latency queues(LLQ). It extends transmit API with

creation of LLQ descriptors in device memory (which include host buffers

descriptors as well as packet header)

Signed-off-by: Arthur Kiyanovski <akiyano@amazon.com>

Signed-off-by: David S. Miller <davem@davemloft.net>

---

 drivers/net/ethernet/amazon/ena/ena_com.c     | 249 +++++++++++++++++-

 drivers/net/ethernet/amazon/ena/ena_com.h     |  28 ++

 drivers/net/ethernet/amazon/ena/ena_eth_com.c | 231 ++++++++++++----

 drivers/net/ethernet/amazon/ena/ena_eth_com.h |  25 +-

 drivers/net/ethernet/amazon/ena/ena_netdev.c  |  21 +-

 5 files changed, 474 insertions(+), 80 deletions(-)

diff --git a/drivers/net/ethernet/amazon/ena/ena_com.c b/drivers/net/ethernet/amazon/ena/ena_com.c

index b6e6a4721931..5220c7578d6b 100644

--- a/drivers/net/ethernet/amazon/ena/ena_com.c

+++ b/drivers/net/ethernet/amazon/ena/ena_com.c

@@ -58,6 +58,8 @@

 #define ENA_MMIO_READ_TIMEOUT 0xFFFFFFFF

+#define ENA_COM_BOUNCE_BUFFER_CNTRL_CNT	4

+

 #define ENA_REGS_ADMIN_INTR_MASK 1

 #define ENA_POLL_MS	5

@@ -352,21 +354,48 @@ static int ena_com_init_io_sq(struct ena_com_dev *ena_dev,

 						    &io_sq->desc_addr.phys_addr,

 						    GFP_KERNEL);

 		}

-	} else {

+

+		if (!io_sq->desc_addr.virt_addr) {

+			pr_err("memory allocation failed");

+			return -ENOMEM;

+		}

+	}

+

+	if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {

+		/* Allocate bounce buffers */

+		io_sq->bounce_buf_ctrl.buffer_size =

+			ena_dev->llq_info.desc_list_entry_size;

+		io_sq->bounce_buf_ctrl.buffers_num =

+			ENA_COM_BOUNCE_BUFFER_CNTRL_CNT;

+		io_sq->bounce_buf_ctrl.next_to_use = 0;

+

+		size = io_sq->bounce_buf_ctrl.buffer_size *

+			 io_sq->bounce_buf_ctrl.buffers_num;

+

 		dev_node = dev_to_node(ena_dev->dmadev);

 		set_dev_node(ena_dev->dmadev, ctx->numa_node);

-		io_sq->desc_addr.virt_addr =

+		io_sq->bounce_buf_ctrl.base_buffer =

 			devm_kzalloc(ena_dev->dmadev, size, GFP_KERNEL);

 		set_dev_node(ena_dev->dmadev, dev_node);

-		if (!io_sq->desc_addr.virt_addr) {

-			io_sq->desc_addr.virt_addr =

+		if (!io_sq->bounce_buf_ctrl.base_buffer)

+			io_sq->bounce_buf_ctrl.base_buffer =

 				devm_kzalloc(ena_dev->dmadev, size, GFP_KERNEL);

+

+		if (!io_sq->bounce_buf_ctrl.base_buffer) {

+			pr_err("bounce buffer memory allocation failed");

+			return -ENOMEM;

 		}

-	}

-	if (!io_sq->desc_addr.virt_addr) {

-		pr_err("memory allocation failed");

-		return -ENOMEM;

+		memcpy(&io_sq->llq_info, &ena_dev->llq_info,

+		       sizeof(io_sq->llq_info));

+

+		/* Initiate the first bounce buffer */

+		io_sq->llq_buf_ctrl.curr_bounce_buf =

+			ena_com_get_next_bounce_buffer(&io_sq->bounce_buf_ctrl);

+		memset(io_sq->llq_buf_ctrl.curr_bounce_buf,

+		       0x0, io_sq->llq_info.desc_list_entry_size);

+		io_sq->llq_buf_ctrl.descs_left_in_line =

+			io_sq->llq_info.descs_num_before_header;

 	}

 	io_sq->tail = 0;

@@ -554,6 +583,156 @@ static int ena_com_wait_and_process_admin_cq_polling(struct ena_comp_ctx *comp_c

 	return ret;

 }

+/**

+ * Set the LLQ configurations of the firmware

+ *

+ * The driver provides only the enabled feature values to the device,

+ * which in turn, checks if they are supported.

+ */

+static int ena_com_set_llq(struct ena_com_dev *ena_dev)

+{

+	struct ena_com_admin_queue *admin_queue;

+	struct ena_admin_set_feat_cmd cmd;

+	struct ena_admin_set_feat_resp resp;

+	struct ena_com_llq_info *llq_info = &ena_dev->llq_info;

+	int ret;

+

+	memset(&cmd, 0x0, sizeof(cmd));

+	admin_queue = &ena_dev->admin_queue;

+

+	cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;

+	cmd.feat_common.feature_id = ENA_ADMIN_LLQ;

+

+	cmd.u.llq.header_location_ctrl_enabled = llq_info->header_location_ctrl;

+	cmd.u.llq.entry_size_ctrl_enabled = llq_info->desc_list_entry_size_ctrl;

+	cmd.u.llq.desc_num_before_header_enabled = llq_info->descs_num_before_header;

+	cmd.u.llq.descriptors_stride_ctrl_enabled = llq_info->desc_stride_ctrl;

+

+	ret = ena_com_execute_admin_command(admin_queue,

+					    (struct ena_admin_aq_entry *)&cmd,