From 4519eada8a75bdad412bcf39f7aa09356b0ac9b9 Mon Sep 17 00:00:00 2001 From: Open vSwitch CI Date: Oct 24 2024 12:13:19 +0000 Subject: Import openvswitch3.3-3.3.0-56 from Fast DataPath --- diff --git a/SOURCES/openvswitch-3.3.0.patch b/SOURCES/openvswitch-3.3.0.patch index da4b9e6..781d4d8 100644 --- a/SOURCES/openvswitch-3.3.0.patch +++ b/SOURCES/openvswitch-3.3.0.patch @@ -7788,3 +7788,30780 @@ index 95a65fcdcd..0352030fec 100644 } } +diff --git a/dpdk/.github/workflows/build.yml b/dpdk/.github/workflows/build.yml +index 272a6ffc7f..6b35c56da0 100644 +--- a/dpdk/.github/workflows/build.yml ++++ b/dpdk/.github/workflows/build.yml +@@ -72,7 +72,7 @@ jobs: + + steps: + - name: Checkout sources +- uses: actions/checkout@v3 ++ uses: actions/checkout@v4 + - name: Generate cache keys + id: get_ref_keys + run: | +@@ -80,7 +80,7 @@ jobs: + echo 'libabigail=libabigail-${{ env.LIBABIGAIL_VERSION }}-${{ matrix.config.os }}' >> $GITHUB_OUTPUT + echo 'abi=abi-${{ matrix.config.os }}-${{ matrix.config.compiler }}-${{ matrix.config.cross }}-${{ env.REF_GIT_TAG }}' >> $GITHUB_OUTPUT + - name: Retrieve ccache cache +- uses: actions/cache@v3 ++ uses: actions/cache@v4 + with: + path: ~/.ccache + key: ${{ steps.get_ref_keys.outputs.ccache }}-${{ github.ref }} +@@ -88,13 +88,13 @@ jobs: + ${{ steps.get_ref_keys.outputs.ccache }}-refs/heads/main + - name: Retrieve libabigail cache + id: libabigail-cache +- uses: actions/cache@v3 ++ uses: actions/cache@v4 + if: env.ABI_CHECKS == 'true' + with: + path: libabigail + key: ${{ steps.get_ref_keys.outputs.libabigail }} + - name: Retrieve ABI reference cache +- uses: actions/cache@v3 ++ uses: actions/cache@v4 + if: env.ABI_CHECKS == 'true' + with: + path: reference +@@ -143,7 +143,7 @@ jobs: + run: .ci/linux-build.sh + - name: Upload logs on failure + if: failure() +- uses: actions/upload-artifact@v3 ++ uses: actions/upload-artifact@v4 + with: + name: meson-logs-${{ join(matrix.config.*, '-') }} + path: | +@@ -171,7 +171,7 @@ jobs: + echo 'image=image-${{ matrix.config.image }}-'$(date -u +%Y-%m-%d) >> $GITHUB_OUTPUT + - name: Retrieve image cache + id: image_cache +- uses: actions/cache@v3 ++ uses: actions/cache@v4 + with: + path: ~/.image + key: ${{ steps.get_keys.outputs.image }} +@@ -218,7 +218,7 @@ jobs: + + steps: + - name: Checkout sources +- uses: actions/checkout@v3 ++ uses: actions/checkout@v4 + - name: Generate various keys + id: get_keys + run: | +@@ -226,7 +226,7 @@ jobs: + echo 'logs=meson-logs-${{ join(matrix.config.*, '-') }}' | tr -d ':' >> $GITHUB_OUTPUT + - name: Retrieve image cache + id: image_cache +- uses: actions/cache@v3 ++ uses: actions/cache@v4 + with: + path: ~/.image + key: ${{ needs.prepare-container-images.outputs.image }} +@@ -236,7 +236,7 @@ jobs: + echo 'Image ${{ matrix.config.image }} is not cached.' + false + - name: Retrieve ccache cache +- uses: actions/cache@v3 ++ uses: actions/cache@v4 + with: + path: ~/.ccache + key: ${{ steps.get_keys.outputs.ccache }}-${{ github.ref }} +@@ -276,7 +276,7 @@ jobs: + run: docker kill dpdk + - name: Upload logs on failure + if: failure() +- uses: actions/upload-artifact@v3 ++ uses: actions/upload-artifact@v4 + with: + name: ${{ steps.get_keys.outputs.logs }} + path: | +diff --git a/dpdk/.mailmap b/dpdk/.mailmap +index ab0742a382..f2883144f3 100644 +--- a/dpdk/.mailmap ++++ b/dpdk/.mailmap +@@ -2,7 +2,7 @@ Aakash Sasidharan + Aaro Koskinen + Aaron Campbell + Aaron Conole +-Abdullah Ömer Yamaç ++Abdullah Ömer Yamaç + Abdullah Sevincer + Abed Kamaluddin + Abhijit Gangurde +@@ -29,10 +29,12 @@ Akash Saxena + Akeem G Abodunrin + Akhil Goyal + Akihiko Odaki ++Akshay Dorwat + Alain Leon + Alan Brady + Alan Carew + Alan Dewar ++Alan Elder + Alan Liu + Alan Winkowski + Alejandro Lucero +@@ -42,12 +44,14 @@ Aleksandr Miloshenko + Aleksey Baulin + Aleksey Katargin + Ales Musil ++Alessio Igor Bogani + Alexander Bechikov + Alexander Belyakov + Alexander Chernavin + Alexander Guy + Alexander Kozyrev + Alexander Matushevsky ++Alexander Skorichenko + Alexander Solganik + Alexander V Gutkin + Alexandre Ferrieux +@@ -224,6 +228,7 @@ Cheng Liu + Cheng Peng + Chengwen Feng + Chenmin Sun ++Chenming Chang + Chenxu Di + Chenyu Huang + Cheryl Houser +@@ -367,6 +372,7 @@ Elad Persiko + Elena Agostini + Eli Britstein + Elza Mathew ++Emi Aoki + Emma Finn + Emma Kenny + Emmanuel Roullit +@@ -401,9 +407,11 @@ Fengtian Guo + Ferdinand Thiessen + Ferruh Yigit + Fidaullah Noonari ++Fidel Castro + Fiona Trahe + Flavia Musatescu + Flavio Leitner ++Flore Norceide + Forrest Shi + Francesco Mancino + Francesco Santoro +@@ -483,6 +491,7 @@ Hanoch Haim + Hanumanth Pothula + Hao Chen + Hao Wu ++Haoqian He + Hari Kumar Vemula + Harini Ramakrishnan + Hariprasad Govindharajan +@@ -518,6 +527,8 @@ Hiral Shah + Hiroki Shirokura + Hiroshi Shimamoto + Hiroyuki Mikita ++Holly Nichols ++Hongbo Li + Hongbo Zheng + Hongjun Ni + Hongzhi Guo +@@ -654,6 +665,7 @@ Jin Yu + Jiri Slaby + Job Abraham + Jochen Behrens ++Joel Kavanagh + Joey Xing + Johan Faltstrom + Johan Källström +@@ -675,7 +687,7 @@ John Ousterhout + John Romein + John W. Linville + Jonas Pfefferle +-Jonathan Erb ++Jonathan Erb + Jonathan Tsai + Jon DeVree + Jon Loeliger +@@ -707,6 +719,7 @@ Junjie Wan + Jun Qiu + Jun W Zhou + Junxiao Shi ++Jun Wang + Jun Yang + Junyu Jiang + Juraj Linkeš +@@ -721,7 +734,7 @@ Kamalakshitha Aligeri + Kamil Bednarczyk + Kamil Chalupnik + Kamil Rytarowski +-Kamil Vojanec ++Kamil Vojanec + Kanaka Durga Kotamarthy + Karen Kelly + Karen Sornek +@@ -785,9 +798,11 @@ Leszek Zygo + Levend Sayar + Lev Faerman + Lewei Yang ++Lewis Donzis + Leyi Rong + Liang Ma + Liang-Min Larry Wang ++Liangxing Wang + Liang Xu + Liang Zhang + Li Feng +@@ -853,7 +868,6 @@ Manish Chopra + Manish Kurup + Manish Tomar + Mao Jiang +-Mao YingMing + Marcel Apfelbaum + Marcel Cornu + Marcelo Ricardo Leitner +@@ -870,6 +884,7 @@ Marcin Wojtas + Marcin Zapolski + Marco Varlese + Marc Sune ++Marek Mical + Maria Lingemark + Mario Carrillo + Mário Kuka +@@ -891,8 +906,9 @@ Martin Klozik + Martin Spinler + Martin Weiser + Martyna Szapar-Mudlaw +-Maryam Tahhan ++Maryam Tahhan + Masoud Hasanifard ++Masoumeh Farhadi Nia + Matan Azrad + Matej Vido + Mateusz Kowalski +@@ -936,6 +952,7 @@ Michael Santana + Michael Savisko + Michael Shamis + Michael S. Tsirkin ++Michael Theodore Stolarchuk + Michael Wildt + Michal Berger + Michal Jastrzebski +@@ -1009,7 +1026,8 @@ Nemanja Marjanovic + Netanel Belgazal + Netanel Gonen + Niall Power +-Nick Connolly ++Nicholas Pratte ++Nick Connolly + Nick Nunley + Niclas Storm + Nicolas Chautru +@@ -1127,6 +1145,7 @@ Przemyslaw Czesnowicz + Przemyslaw Patynowski + Przemyslaw Zegan + Pu Xu <583493798@qq.com> ++Qian Hao + Qian Xu + Qiao Liu + Qi Fu +@@ -1142,6 +1161,7 @@ Quentin Armitage + Qun Wan + Radha Mohan Chintakuntla + Radoslaw Biernacki ++Radoslaw Tyl + Radu Bulie + Radu Nicolau + Rafael Ávila de Espíndola +@@ -1205,7 +1225,7 @@ Roman Kapl + Roman Korynkevych + Roman Storozhenko + Roman Zhukov +-Ronak Doshi ++Ronak Doshi + Ron Beider + Ronghua Zhang + RongQiang Xie +@@ -1275,9 +1295,11 @@ Shahed Shaikh + Shai Brandes + Shailendra Bhatnagar + Shally Verma ++Shani Peretz + Shannon Nelson + Shannon Zhao + Shaopeng He ++Shaowei Sun <1819846787@qq.com> + Sharmila Podury + Sharon Haroni + Shay Agroskin +@@ -1298,6 +1320,7 @@ Shiyang He + Shlomi Gridish + Shougang Wang + Shraddha Joshi ++Shreesh Adiga <16567adigashreesh@gmail.com> + Shreyansh Jain + Shrikrishna Khare + Shuai Zhu +@@ -1306,6 +1329,7 @@ Shuki Katzenelson + Shun Hao + Shu Shen + Shujing Dong ++Shuo Li + Shweta Choudaha + Shyam Kumar Shrivastav + Shy Shyman +@@ -1424,7 +1448,9 @@ Timothy McDaniel + Timothy Miskell + Timothy Redaelli + Tim Shearer ++Ting-Kai Ku + Ting Xu ++Tingting Liao + Tiwei Bie + Todd Fujinaka + Tomasz Cel +@@ -1437,6 +1463,7 @@ Tomasz Kulasek + Tomasz Zawadzki + Tom Barbette + Tom Crugnale ++Tom Jones + Tom Millington + Tom Rix + Tomer Shmilovich +@@ -1457,6 +1484,7 @@ Vadim Suraev + Vakul Garg + Vamsi Attunuru + Vanshika Shukla ++Varun Sethi + Vasily Philipov + Veerasenareddy Burru + Venkata Suresh Kumar P +@@ -1485,6 +1513,8 @@ Vincent Guo + Vincent Jardin + Vincent Li + Vincent S. Cojot ++Vinh Tran ++Vipin Padmam Ramesh + Vipin Varghese + Vipul Ashri + Visa Hankala +@@ -1507,6 +1537,7 @@ Walter Heymans + Wang Sheng-Hui + Wangyu (Eric) + Waterman Cao ++Wathsala Vithanage + Weichun Chen + Wei Dai + Weifeng Li +@@ -1604,6 +1635,7 @@ Yi Lu + Yilun Xu + Yinan Wang + Ying A Wang ++Yingming Mao + Yingya Han + Yinjun Zhang + Yipeng Wang +diff --git a/dpdk/VERSION b/dpdk/VERSION +index 94c0153b26..d1bc17f504 100644 +--- a/dpdk/VERSION ++++ b/dpdk/VERSION +@@ -1 +1 @@ +-23.11.0 ++23.11.2 +diff --git a/dpdk/app/dumpcap/main.c b/dpdk/app/dumpcap/main.c +index fc28e2d702..76c7475114 100644 +--- a/dpdk/app/dumpcap/main.c ++++ b/dpdk/app/dumpcap/main.c +@@ -628,6 +628,11 @@ static void dpdk_init(void) + eal_argv[i++] = strdup(file_prefix); + } + ++ for (i = 0; i < (unsigned int)eal_argc; i++) { ++ if (eal_argv[i] == NULL) ++ rte_panic("No memory\n"); ++ } ++ + if (rte_eal_init(eal_argc, eal_argv) < 0) + rte_exit(EXIT_FAILURE, "EAL init failed: is primary process running?\n"); + } +@@ -934,6 +939,11 @@ int main(int argc, char **argv) + { + struct rte_ring *r; + struct rte_mempool *mp; ++ struct sigaction action = { ++ .sa_flags = SA_RESTART, ++ .sa_handler = signal_handler, ++ }; ++ struct sigaction origaction; + dumpcap_out_t out; + char *p; + +@@ -959,8 +969,13 @@ int main(int argc, char **argv) + + compile_filters(); + +- signal(SIGINT, signal_handler); +- signal(SIGPIPE, SIG_IGN); ++ sigemptyset(&action.sa_mask); ++ sigaction(SIGTERM, &action, NULL); ++ sigaction(SIGINT, &action, NULL); ++ sigaction(SIGPIPE, &action, NULL); ++ sigaction(SIGHUP, NULL, &origaction); ++ if (origaction.sa_handler == SIG_DFL) ++ sigaction(SIGHUP, &action, NULL); + + enable_primary_monitor(); + +diff --git a/dpdk/app/graph/meson.build b/dpdk/app/graph/meson.build +index 5b0f966d99..8aefdf505c 100644 +--- a/dpdk/app/graph/meson.build ++++ b/dpdk/app/graph/meson.build +@@ -5,6 +5,7 @@ + name = 'graph' + build = cc.has_header('sys/epoll.h') + if not build ++ reason = 'only supported on Linux' + subdir_done() + endif + +diff --git a/dpdk/app/meson.build b/dpdk/app/meson.build +index 8aaed59f39..21b6da29b3 100644 +--- a/dpdk/app/meson.build ++++ b/dpdk/app/meson.build +@@ -93,7 +93,7 @@ foreach app:apps + if not build + if reason != '' + dpdk_apps_disabled += app +- set_variable(app.underscorify() + '_disable_reason', reason) ++ set_variable('app_' + app.underscorify() + '_disable_reason', reason) + endif + continue + endif +diff --git a/dpdk/app/pdump/main.c b/dpdk/app/pdump/main.c +index 7a1c7bdf60..3592f8a865 100644 +--- a/dpdk/app/pdump/main.c ++++ b/dpdk/app/pdump/main.c +@@ -171,6 +171,9 @@ parse_device_id(const char *key __rte_unused, const char *value, + struct pdump_tuples *pt = extra_args; + + pt->device_id = strdup(value); ++ if (pt->device_id == NULL) ++ return -1; ++ + pt->dump_by_type = DEVICE_ID; + + return 0; +@@ -568,11 +571,9 @@ disable_primary_monitor(void) + } + + static void +-signal_handler(int sig_num) ++signal_handler(int sig_num __rte_unused) + { +- if (sig_num == SIGINT) { +- quit_signal = 1; +- } ++ quit_signal = 1; + } + + static inline int +@@ -972,6 +973,11 @@ enable_primary_monitor(void) + int + main(int argc, char **argv) + { ++ struct sigaction action = { ++ .sa_flags = SA_RESTART, ++ .sa_handler = signal_handler, ++ }; ++ struct sigaction origaction; + int diag; + int ret; + int i; +@@ -980,8 +986,14 @@ main(int argc, char **argv) + char mp_flag[] = "--proc-type=secondary"; + char *argp[argc + 2]; + +- /* catch ctrl-c so we can print on exit */ +- signal(SIGINT, signal_handler); ++ /* catch ctrl-c so we can cleanup on exit */ ++ sigemptyset(&action.sa_mask); ++ sigaction(SIGTERM, &action, NULL); ++ sigaction(SIGINT, &action, NULL); ++ sigaction(SIGPIPE, &action, NULL); ++ sigaction(SIGHUP, NULL, &origaction); ++ if (origaction.sa_handler == SIG_DFL) ++ sigaction(SIGHUP, &action, NULL); + + argp[0] = argv[0]; + argp[1] = n_flag; +diff --git a/dpdk/app/test-bbdev/test_bbdev_perf.c b/dpdk/app/test-bbdev/test_bbdev_perf.c +index dcce00aa0a..5c1755ae0d 100644 +--- a/dpdk/app/test-bbdev/test_bbdev_perf.c ++++ b/dpdk/app/test-bbdev/test_bbdev_perf.c +@@ -94,6 +94,8 @@ + #define K0_2_2 25 /* K0 fraction numerator for rv 2 and BG 2 */ + #define K0_3_1 56 /* K0 fraction numerator for rv 3 and BG 1 */ + #define K0_3_2 43 /* K0 fraction numerator for rv 3 and BG 2 */ ++#define NUM_SC_PER_RB (12) /* Number of subcarriers in a RB in 3GPP. */ ++#define BITS_PER_LLR (8) /* Number of bits in a LLR. */ + + #define HARQ_MEM_TOLERANCE 256 + static struct test_bbdev_vector test_vector; +@@ -2131,7 +2133,8 @@ validate_op_chain(struct rte_bbdev_op_data *op, + uint16_t data_len = rte_pktmbuf_data_len(m) - offset; + total_data_size += orig_op->segments[i].length; + +- if (orig_op->segments[i].length > RTE_BBDEV_LDPC_E_MAX_MBUF) ++ if ((orig_op->segments[i].length + RTE_PKTMBUF_HEADROOM) ++ > RTE_BBDEV_LDPC_E_MAX_MBUF) + ignore_mbuf = true; + if (!ignore_mbuf) + TEST_ASSERT(orig_op->segments[i].length == data_len, +@@ -2895,8 +2898,14 @@ calc_fft_size(struct rte_bbdev_fft_op *op) + static uint32_t + calc_mldts_size(struct rte_bbdev_mldts_op *op) + { +- uint32_t output_size; +- output_size = op->mldts.num_layers * op->mldts.num_rbs * op->mldts.c_rep; ++ uint32_t output_size = 0; ++ uint16_t i; ++ ++ for (i = 0; i < op->mldts.num_layers; i++) ++ output_size += op->mldts.q_m[i]; ++ ++ output_size *= NUM_SC_PER_RB * BITS_PER_LLR * op->mldts.num_rbs * (op->mldts.c_rep + 1); ++ + return output_size; + } + +@@ -3399,15 +3408,6 @@ throughput_intr_lcore_ldpc_dec(void *arg) + if (unlikely(num_to_process - enqueued < num_to_enq)) + num_to_enq = num_to_process - enqueued; + +- enq = 0; +- do { +- enq += rte_bbdev_enqueue_ldpc_dec_ops( +- tp->dev_id, +- queue_id, &ops[enqueued], +- num_to_enq); +- } while (unlikely(num_to_enq != enq)); +- enqueued += enq; +- + /* Write to thread burst_sz current number of enqueued + * descriptors. It ensures that proper number of + * descriptors will be dequeued in callback +@@ -3417,6 +3417,15 @@ throughput_intr_lcore_ldpc_dec(void *arg) + */ + __atomic_store_n(&tp->burst_sz, num_to_enq, __ATOMIC_RELAXED); + ++ enq = 0; ++ do { ++ enq += rte_bbdev_enqueue_ldpc_dec_ops( ++ tp->dev_id, ++ queue_id, &ops[enqueued], ++ num_to_enq); ++ } while (unlikely(num_to_enq != enq)); ++ enqueued += enq; ++ + /* Wait until processing of previous batch is + * completed + */ +@@ -3491,14 +3500,6 @@ throughput_intr_lcore_dec(void *arg) + if (unlikely(num_to_process - enqueued < num_to_enq)) + num_to_enq = num_to_process - enqueued; + +- enq = 0; +- do { +- enq += rte_bbdev_enqueue_dec_ops(tp->dev_id, +- queue_id, &ops[enqueued], +- num_to_enq); +- } while (unlikely(num_to_enq != enq)); +- enqueued += enq; +- + /* Write to thread burst_sz current number of enqueued + * descriptors. It ensures that proper number of + * descriptors will be dequeued in callback +@@ -3508,6 +3509,14 @@ throughput_intr_lcore_dec(void *arg) + */ + __atomic_store_n(&tp->burst_sz, num_to_enq, __ATOMIC_RELAXED); + ++ enq = 0; ++ do { ++ enq += rte_bbdev_enqueue_dec_ops(tp->dev_id, ++ queue_id, &ops[enqueued], ++ num_to_enq); ++ } while (unlikely(num_to_enq != enq)); ++ enqueued += enq; ++ + /* Wait until processing of previous batch is + * completed + */ +@@ -3577,14 +3586,6 @@ throughput_intr_lcore_enc(void *arg) + if (unlikely(num_to_process - enqueued < num_to_enq)) + num_to_enq = num_to_process - enqueued; + +- enq = 0; +- do { +- enq += rte_bbdev_enqueue_enc_ops(tp->dev_id, +- queue_id, &ops[enqueued], +- num_to_enq); +- } while (unlikely(enq != num_to_enq)); +- enqueued += enq; +- + /* Write to thread burst_sz current number of enqueued + * descriptors. It ensures that proper number of + * descriptors will be dequeued in callback +@@ -3594,6 +3595,14 @@ throughput_intr_lcore_enc(void *arg) + */ + __atomic_store_n(&tp->burst_sz, num_to_enq, __ATOMIC_RELAXED); + ++ enq = 0; ++ do { ++ enq += rte_bbdev_enqueue_enc_ops(tp->dev_id, ++ queue_id, &ops[enqueued], ++ num_to_enq); ++ } while (unlikely(enq != num_to_enq)); ++ enqueued += enq; ++ + /* Wait until processing of previous batch is + * completed + */ +@@ -3665,15 +3674,6 @@ throughput_intr_lcore_ldpc_enc(void *arg) + if (unlikely(num_to_process - enqueued < num_to_enq)) + num_to_enq = num_to_process - enqueued; + +- enq = 0; +- do { +- enq += rte_bbdev_enqueue_ldpc_enc_ops( +- tp->dev_id, +- queue_id, &ops[enqueued], +- num_to_enq); +- } while (unlikely(enq != num_to_enq)); +- enqueued += enq; +- + /* Write to thread burst_sz current number of enqueued + * descriptors. It ensures that proper number of + * descriptors will be dequeued in callback +@@ -3683,6 +3683,15 @@ throughput_intr_lcore_ldpc_enc(void *arg) + */ + __atomic_store_n(&tp->burst_sz, num_to_enq, __ATOMIC_RELAXED); + ++ enq = 0; ++ do { ++ enq += rte_bbdev_enqueue_ldpc_enc_ops( ++ tp->dev_id, ++ queue_id, &ops[enqueued], ++ num_to_enq); ++ } while (unlikely(enq != num_to_enq)); ++ enqueued += enq; ++ + /* Wait until processing of previous batch is + * completed + */ +@@ -3754,14 +3763,6 @@ throughput_intr_lcore_fft(void *arg) + if (unlikely(num_to_process - enqueued < num_to_enq)) + num_to_enq = num_to_process - enqueued; + +- enq = 0; +- do { +- enq += rte_bbdev_enqueue_fft_ops(tp->dev_id, +- queue_id, &ops[enqueued], +- num_to_enq); +- } while (unlikely(enq != num_to_enq)); +- enqueued += enq; +- + /* Write to thread burst_sz current number of enqueued + * descriptors. It ensures that proper number of + * descriptors will be dequeued in callback +@@ -3771,6 +3772,14 @@ throughput_intr_lcore_fft(void *arg) + */ + __atomic_store_n(&tp->burst_sz, num_to_enq, __ATOMIC_RELAXED); + ++ enq = 0; ++ do { ++ enq += rte_bbdev_enqueue_fft_ops(tp->dev_id, ++ queue_id, &ops[enqueued], ++ num_to_enq); ++ } while (unlikely(enq != num_to_enq)); ++ enqueued += enq; ++ + /* Wait until processing of previous batch is + * completed + */ +@@ -3837,13 +3846,6 @@ throughput_intr_lcore_mldts(void *arg) + if (unlikely(num_to_process - enqueued < num_to_enq)) + num_to_enq = num_to_process - enqueued; + +- enq = 0; +- do { +- enq += rte_bbdev_enqueue_mldts_ops(tp->dev_id, +- queue_id, &ops[enqueued], num_to_enq); +- } while (unlikely(enq != num_to_enq)); +- enqueued += enq; +- + /* Write to thread burst_sz current number of enqueued + * descriptors. It ensures that proper number of + * descriptors will be dequeued in callback +@@ -3853,6 +3855,13 @@ throughput_intr_lcore_mldts(void *arg) + */ + __atomic_store_n(&tp->burst_sz, num_to_enq, __ATOMIC_RELAXED); + ++ enq = 0; ++ do { ++ enq += rte_bbdev_enqueue_mldts_ops(tp->dev_id, ++ queue_id, &ops[enqueued], num_to_enq); ++ } while (unlikely(enq != num_to_enq)); ++ enqueued += enq; ++ + /* Wait until processing of previous batch is + * completed + */ +diff --git a/dpdk/app/test-crypto-perf/cperf_ops.c b/dpdk/app/test-crypto-perf/cperf_ops.c +index 84945d1313..0e79133310 100644 +--- a/dpdk/app/test-crypto-perf/cperf_ops.c ++++ b/dpdk/app/test-crypto-perf/cperf_ops.c +@@ -21,7 +21,6 @@ cperf_set_ops_asym(struct rte_crypto_op **ops, + uint64_t *tsc_start __rte_unused) + { + uint16_t i; +- void *asym_sess = (void *)sess; + + for (i = 0; i < nb_ops; i++) { + struct rte_crypto_asym_op *asym_op = ops[i]->asym; +@@ -31,7 +30,7 @@ cperf_set_ops_asym(struct rte_crypto_op **ops, + asym_op->modex.base.length = options->modex_data->base.len; + asym_op->modex.result.data = options->modex_data->result.data; + asym_op->modex.result.length = options->modex_data->result.len; +- rte_crypto_op_attach_asym_session(ops[i], asym_sess); ++ rte_crypto_op_attach_asym_session(ops[i], sess); + } + } + +@@ -64,7 +63,6 @@ cperf_set_ops_security(struct rte_crypto_op **ops, + + for (i = 0; i < nb_ops; i++) { + struct rte_crypto_sym_op *sym_op = ops[i]->sym; +- void *sec_sess = (void *)sess; + uint32_t buf_sz; + + uint32_t *per_pkt_hfn = rte_crypto_op_ctod_offset(ops[i], +@@ -72,7 +70,7 @@ cperf_set_ops_security(struct rte_crypto_op **ops, + *per_pkt_hfn = options->pdcp_ses_hfn_en ? 0 : PDCP_DEFAULT_HFN; + + ops[i]->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; +- rte_security_attach_session(ops[i], sec_sess); ++ rte_security_attach_session(ops[i], sess); + sym_op->m_src = (struct rte_mbuf *)((uint8_t *)ops[i] + + src_buf_offset); + +@@ -129,7 +127,6 @@ cperf_set_ops_security_ipsec(struct rte_crypto_op **ops, + uint16_t iv_offset __rte_unused, uint32_t *imix_idx, + uint64_t *tsc_start) + { +- void *sec_sess = sess; + const uint32_t test_buffer_size = options->test_buffer_size; + const uint32_t headroom_sz = options->headroom_sz; + const uint32_t segment_sz = options->segment_sz; +@@ -143,7 +140,7 @@ cperf_set_ops_security_ipsec(struct rte_crypto_op **ops, + struct rte_mbuf *m = sym_op->m_src; + + ops[i]->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; +- rte_security_attach_session(ops[i], sec_sess); ++ rte_security_attach_session(ops[i], sess); + sym_op->m_src = (struct rte_mbuf *)((uint8_t *)ops[i] + + src_buf_offset); + +diff --git a/dpdk/app/test-crypto-perf/cperf_options_parsing.c b/dpdk/app/test-crypto-perf/cperf_options_parsing.c +index 75afedc7fd..8909b5690d 100644 +--- a/dpdk/app/test-crypto-perf/cperf_options_parsing.c ++++ b/dpdk/app/test-crypto-perf/cperf_options_parsing.c +@@ -516,6 +516,10 @@ parse_test_file(struct cperf_options *opts, + const char *arg) + { + opts->test_file = strdup(arg); ++ if (opts->test_file == NULL) { ++ RTE_LOG(ERR, USER1, "Dup vector file failed!\n"); ++ return -1; ++ } + if (access(opts->test_file, F_OK) != -1) + return 0; + RTE_LOG(ERR, USER1, "Test vector file doesn't exist\n"); +diff --git a/dpdk/app/test-crypto-perf/cperf_test_common.c b/dpdk/app/test-crypto-perf/cperf_test_common.c +index 932aab16df..6b8ab65731 100644 +--- a/dpdk/app/test-crypto-perf/cperf_test_common.c ++++ b/dpdk/app/test-crypto-perf/cperf_test_common.c +@@ -49,7 +49,6 @@ fill_multi_seg_mbuf(struct rte_mbuf *m, struct rte_mempool *mp, + { + uint16_t mbuf_hdr_size = sizeof(struct rte_mbuf); + uint16_t remaining_segments = segments_nb; +- struct rte_mbuf *next_mbuf; + rte_iova_t next_seg_phys_addr = rte_mempool_virt2iova(obj) + + mbuf_offset + mbuf_hdr_size; + +@@ -70,15 +69,15 @@ fill_multi_seg_mbuf(struct rte_mbuf *m, struct rte_mempool *mp, + m->nb_segs = segments_nb; + m->port = 0xff; + rte_mbuf_refcnt_set(m, 1); +- next_mbuf = (struct rte_mbuf *) ((uint8_t *) m + +- mbuf_hdr_size + segment_sz); +- m->next = next_mbuf; +- m = next_mbuf; +- remaining_segments--; + ++ remaining_segments--; ++ if (remaining_segments > 0) { ++ m->next = (struct rte_mbuf *)((uint8_t *) m + mbuf_hdr_size + segment_sz); ++ m = m->next; ++ } else { ++ m->next = NULL; ++ } + } while (remaining_segments > 0); +- +- m->next = NULL; + } + + static void +@@ -150,11 +149,11 @@ cperf_alloc_common_memory(const struct cperf_options *options, + int ret; + + /* Calculate the object size */ +- uint16_t crypto_op_size = sizeof(struct rte_crypto_op) + +- sizeof(struct rte_crypto_sym_op); ++ uint16_t crypto_op_size = sizeof(struct rte_crypto_op); + uint16_t crypto_op_private_size; + + if (options->op_type == CPERF_ASYM_MODEX) { ++ crypto_op_size += sizeof(struct rte_crypto_asym_op); + snprintf(pool_name, RTE_MEMPOOL_NAMESIZE, "perf_asym_op_pool%u", + rte_socket_id()); + *pool = rte_crypto_op_pool_create( +@@ -171,6 +170,8 @@ cperf_alloc_common_memory(const struct cperf_options *options, + return 0; + } + ++ crypto_op_size += sizeof(struct rte_crypto_sym_op); ++ + /* + * If doing AES-CCM, IV field needs to be 16 bytes long, + * and AAD field needs to be long enough to have 18 bytes, +@@ -227,7 +228,8 @@ cperf_alloc_common_memory(const struct cperf_options *options, + (mbuf_size * segments_nb); + params.dst_buf_offset = *dst_buf_offset; + /* Destination buffer will be one segment only */ +- obj_size += max_size + sizeof(struct rte_mbuf); ++ obj_size += max_size + sizeof(struct rte_mbuf) + ++ options->headroom_sz + options->tailroom_sz; + } + + *pool = rte_mempool_create_empty(pool_name, +@@ -269,7 +271,7 @@ cperf_mbuf_set(struct rte_mbuf *mbuf, + const struct cperf_options *options, + const struct cperf_test_vector *test_vector) + { +- uint32_t segment_sz = options->segment_sz; ++ uint32_t segment_sz = options->segment_sz - options->headroom_sz - options->tailroom_sz; + uint8_t *mbuf_data; + uint8_t *test_data; + uint32_t remaining_bytes = options->max_buffer_size; +diff --git a/dpdk/app/test-crypto-perf/cperf_test_latency.c b/dpdk/app/test-crypto-perf/cperf_test_latency.c +index 484bc9eb4e..e55d293db3 100644 +--- a/dpdk/app/test-crypto-perf/cperf_test_latency.c ++++ b/dpdk/app/test-crypto-perf/cperf_test_latency.c +@@ -121,7 +121,11 @@ store_timestamp(struct rte_crypto_op *op, uint64_t timestamp) + { + struct priv_op_data *priv_data; + +- priv_data = (struct priv_op_data *) (op->sym + 1); ++ if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) ++ priv_data = (struct priv_op_data *) (op->sym + 1); ++ else ++ priv_data = (struct priv_op_data *) (op->asym + 1); ++ + priv_data->result->status = op->status; + priv_data->result->tsc_end = timestamp; + } +@@ -250,9 +254,13 @@ cperf_latency_test_runner(void *arg) + ctx->res[tsc_idx].tsc_start = tsc_start; + /* + * Private data structure starts after the end of the +- * rte_crypto_sym_op structure. ++ * rte_crypto_sym_op (or rte_crypto_asym_op) structure. + */ +- priv_data = (struct priv_op_data *) (ops[i]->sym + 1); ++ if (ops[i]->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) ++ priv_data = (struct priv_op_data *) (ops[i]->sym + 1); ++ else ++ priv_data = (struct priv_op_data *) (ops[i]->asym + 1); ++ + priv_data->result = (void *)&ctx->res[tsc_idx]; + tsc_idx++; + } +diff --git a/dpdk/app/test-crypto-perf/cperf_test_verify.c b/dpdk/app/test-crypto-perf/cperf_test_verify.c +index a6c0ffe813..10172a53a0 100644 +--- a/dpdk/app/test-crypto-perf/cperf_test_verify.c ++++ b/dpdk/app/test-crypto-perf/cperf_test_verify.c +@@ -111,8 +111,10 @@ cperf_verify_op(struct rte_crypto_op *op, + uint32_t len; + uint16_t nb_segs; + uint8_t *data; +- uint32_t cipher_offset, auth_offset; +- uint8_t cipher, auth; ++ uint32_t cipher_offset, auth_offset = 0; ++ bool cipher = false; ++ bool digest_verify = false; ++ bool is_encrypt = false; + int res = 0; + + if (op->status != RTE_CRYPTO_OP_STATUS_SUCCESS) +@@ -150,57 +152,54 @@ cperf_verify_op(struct rte_crypto_op *op, + + switch (options->op_type) { + case CPERF_CIPHER_ONLY: +- cipher = 1; ++ cipher = true; + cipher_offset = 0; +- auth = 0; +- auth_offset = 0; +- break; +- case CPERF_CIPHER_THEN_AUTH: +- cipher = 1; +- cipher_offset = 0; +- auth = 1; +- auth_offset = options->test_buffer_size; ++ is_encrypt = options->cipher_op == RTE_CRYPTO_CIPHER_OP_ENCRYPT; + break; + case CPERF_AUTH_ONLY: +- cipher = 0; + cipher_offset = 0; +- auth = 1; +- auth_offset = options->test_buffer_size; ++ if (options->auth_op == RTE_CRYPTO_AUTH_OP_GENERATE) { ++ auth_offset = options->test_buffer_size; ++ digest_verify = true; ++ } + break; ++ case CPERF_CIPHER_THEN_AUTH: + case CPERF_AUTH_THEN_CIPHER: +- cipher = 1; ++ cipher = true; + cipher_offset = 0; +- auth = 1; +- auth_offset = options->test_buffer_size; ++ if (options->cipher_op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { ++ auth_offset = options->test_buffer_size; ++ digest_verify = true; ++ is_encrypt = true; ++ } + break; + case CPERF_AEAD: +- cipher = 1; ++ cipher = true; + cipher_offset = 0; +- auth = 1; +- auth_offset = options->test_buffer_size; ++ if (options->aead_op == RTE_CRYPTO_AEAD_OP_ENCRYPT) { ++ auth_offset = options->test_buffer_size; ++ digest_verify = true; ++ is_encrypt = true; ++ } + break; + default: + res = 1; + goto out; + } + +- if (cipher == 1) { +- if (options->cipher_op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) +- res += memcmp(data + cipher_offset, ++ if (cipher) { ++ if (is_encrypt) ++ res += !!memcmp(data + cipher_offset, + vector->ciphertext.data, + options->test_buffer_size); + else +- res += memcmp(data + cipher_offset, ++ res += !!memcmp(data + cipher_offset, + vector->plaintext.data, + options->test_buffer_size); + } + +- if (auth == 1) { +- if (options->auth_op == RTE_CRYPTO_AUTH_OP_GENERATE) +- res += memcmp(data + auth_offset, +- vector->digest.data, +- options->digest_sz); +- } ++ if (digest_verify) ++ res += !!memcmp(data + auth_offset, vector->digest.data, options->digest_sz); + + out: + rte_free(data); +@@ -276,7 +275,6 @@ cperf_verify_test_runner(void *test_ctx) + ops_needed, ctx->sess, ctx->options, + ctx->test_vector, iv_offset, &imix_idx, NULL); + +- + /* Populate the mbuf with the test vector, for verification */ + for (i = 0; i < ops_needed; i++) + cperf_mbuf_set(ops[i]->sym->m_src, +@@ -294,6 +292,17 @@ cperf_verify_test_runner(void *test_ctx) + } + #endif /* CPERF_LINEARIZATION_ENABLE */ + ++ /** ++ * When ops_needed is smaller than ops_enqd, the ++ * unused ops need to be moved to the front for ++ * next round use. ++ */ ++ if (unlikely(ops_enqd > ops_needed)) { ++ size_t nb_b_to_mov = ops_unused * sizeof(struct rte_crypto_op *); ++ ++ memmove(&ops[ops_needed], &ops[ops_enqd], nb_b_to_mov); ++ } ++ + /* Enqueue burst of ops on crypto device */ + ops_enqd = rte_cryptodev_enqueue_burst(ctx->dev_id, ctx->qp_id, + ops, burst_size); +diff --git a/dpdk/app/test-dma-perf/main.c b/dpdk/app/test-dma-perf/main.c +index 5f8bab8f45..544784df50 100644 +--- a/dpdk/app/test-dma-perf/main.c ++++ b/dpdk/app/test-dma-perf/main.c +@@ -206,6 +206,8 @@ parse_lcore_dma(struct test_configure *test_case, const char *value) + return -1; + + input = strndup(value, strlen(value) + 1); ++ if (input == NULL) ++ return -1; + addrs = input; + + while (*addrs == '\0') +diff --git a/dpdk/app/test-pmd/bpf_cmd.c b/dpdk/app/test-pmd/bpf_cmd.c +index 46f6b7d6d2..24d34f983e 100644 +--- a/dpdk/app/test-pmd/bpf_cmd.c ++++ b/dpdk/app/test-pmd/bpf_cmd.c +@@ -139,7 +139,7 @@ static cmdline_parse_token_string_t cmd_load_bpf_prm = + cmdline_parse_inst_t cmd_operate_bpf_ld_parse = { + .f = cmd_operate_bpf_ld_parsed, + .data = NULL, +- .help_str = "bpf-load rx|tx ", ++ .help_str = "bpf-load rx|tx ", + .tokens = { + (void *)&cmd_load_bpf_start, + (void *)&cmd_load_bpf_dir, +diff --git a/dpdk/app/test-pmd/cmdline.c b/dpdk/app/test-pmd/cmdline.c +index 9369d3b4c5..d9304e4a32 100644 +--- a/dpdk/app/test-pmd/cmdline.c ++++ b/dpdk/app/test-pmd/cmdline.c +@@ -3528,6 +3528,8 @@ parse_hdrs_list(const char *str, const char *item_name, unsigned int max_items, + + nb_item = 0; + char *str2 = strdup(str); ++ if (str2 == NULL) ++ return nb_item; + cur = strtok_r(str2, ",", &tmp); + while (cur != NULL) { + parsed_items[nb_item] = get_ptype(cur); +diff --git a/dpdk/app/test-pmd/cmdline_flow.c b/dpdk/app/test-pmd/cmdline_flow.c +index ce71818705..7e6e06a04f 100644 +--- a/dpdk/app/test-pmd/cmdline_flow.c ++++ b/dpdk/app/test-pmd/cmdline_flow.c +@@ -3520,7 +3520,7 @@ static const struct token token_list[] = { + [QUEUE_DESTROY] = { + .name = "destroy", + .help = "destroy a flow rule", +- .next = NEXT(NEXT_ENTRY(QUEUE_DESTROY_ID), ++ .next = NEXT(NEXT_ENTRY(QUEUE_DESTROY_POSTPONE), + NEXT_ENTRY(COMMON_QUEUE_ID)), + .args = ARGS(ARGS_ENTRY(struct buffer, queue)), + .call = parse_qo_destroy, +@@ -5543,9 +5543,12 @@ static const struct token token_list[] = { + [ITEM_CONNTRACK] = { + .name = "conntrack", + .help = "conntrack state", ++ .priv = PRIV_ITEM(CONNTRACK, ++ sizeof(struct rte_flow_item_conntrack)), + .next = NEXT(NEXT_ENTRY(ITEM_NEXT), NEXT_ENTRY(COMMON_UNSIGNED), + item_param), + .args = ARGS(ARGS_ENTRY(struct rte_flow_item_conntrack, flags)), ++ .call = parse_vc, + }, + [ITEM_PORT_REPRESENTOR] = { + .name = "port_representor", +@@ -6905,7 +6908,7 @@ static const struct token token_list[] = { + .comp = comp_none, + }, + [ACTION_MODIFY_FIELD_SRC_TAG_INDEX] = { +- .name = "stc_tag_index", ++ .name = "src_tag_index", + .help = "source field tag array", + .next = NEXT(action_modify_field_src, + NEXT_ENTRY(COMMON_UNSIGNED)), +@@ -7395,11 +7398,13 @@ static const struct token token_list[] = { + .type = "UNSIGNED", + .help = "unsigned integer value", + .call = parse_indlst_id2ptr, ++ .comp = comp_none, + }, + [INDIRECT_LIST_ACTION_ID2PTR_CONF] = { + .type = "UNSIGNED", + .help = "unsigned integer value", + .call = parse_indlst_id2ptr, ++ .comp = comp_none, + }, + [ACTION_SHARED_INDIRECT] = { + .name = "shared_indirect", +@@ -11334,34 +11339,36 @@ parse_indlst_id2ptr(struct context *ctx, const struct token *token, + uint32_t id; + int ret; + +- if (!action) +- return -1; + ctx->objdata = 0; + ctx->object = &id; + ctx->objmask = NULL; + ret = parse_int(ctx, token, str, len, ctx->object, sizeof(id)); ++ ctx->object = action; + if (ret != (int)len) + return ret; +- ctx->object = action; +- action_conf = (void *)(uintptr_t)action->conf; +- action_conf->conf = NULL; +- switch (ctx->curr) { +- case INDIRECT_LIST_ACTION_ID2PTR_HANDLE: +- action_conf->handle = (typeof(action_conf->handle)) +- port_action_handle_get_by_id(ctx->port, id); +- if (!action_conf->handle) { +- printf("no indirect list handle for id %u\n", id); +- return -1; ++ ++ /* set handle and conf */ ++ if (action) { ++ action_conf = (void *)(uintptr_t)action->conf; ++ action_conf->conf = NULL; ++ switch (ctx->curr) { ++ case INDIRECT_LIST_ACTION_ID2PTR_HANDLE: ++ action_conf->handle = (typeof(action_conf->handle)) ++ port_action_handle_get_by_id(ctx->port, id); ++ if (!action_conf->handle) { ++ printf("no indirect list handle for id %u\n", id); ++ return -1; ++ } ++ break; ++ case INDIRECT_LIST_ACTION_ID2PTR_CONF: ++ indlst_conf = indirect_action_list_conf_get(id); ++ if (!indlst_conf) ++ return -1; ++ action_conf->conf = (const void **)indlst_conf->conf; ++ break; ++ default: ++ break; + } +- break; +- case INDIRECT_LIST_ACTION_ID2PTR_CONF: +- indlst_conf = indirect_action_list_conf_get(id); +- if (!indlst_conf) +- return -1; +- action_conf->conf = (const void **)indlst_conf->conf; +- break; +- default: +- break; + } + return ret; + } +@@ -12609,6 +12616,7 @@ cmd_flow_parsed(const struct buffer *in) + port_queue_action_handle_create( + in->port, in->queue, in->postpone, + in->args.vc.attr.group, ++ in->command == QUEUE_INDIRECT_ACTION_LIST_CREATE, + &((const struct rte_flow_indir_action_conf) { + .ingress = in->args.vc.attr.ingress, + .egress = in->args.vc.attr.egress, +diff --git a/dpdk/app/test-pmd/config.c b/dpdk/app/test-pmd/config.c +index cad7537bc6..40e4e83fb8 100644 +--- a/dpdk/app/test-pmd/config.c ++++ b/dpdk/app/test-pmd/config.c +@@ -1891,8 +1891,7 @@ port_action_handle_flush(portid_t port_id) + /* Poisoning to make sure PMDs update it in case of error. */ + memset(&error, 0x44, sizeof(error)); + if (pia->handle != NULL) { +- ret = pia->type == +- RTE_FLOW_ACTION_TYPE_INDIRECT_LIST ? ++ ret = pia->type == RTE_FLOW_ACTION_TYPE_INDIRECT_LIST ? + rte_flow_action_list_handle_destroy + (port_id, pia->list_handle, &error) : + rte_flow_action_handle_destroy +@@ -1902,11 +1901,9 @@ port_action_handle_flush(portid_t port_id) + pia->id); + ret = port_flow_complain(&error); + } +- tmp = &pia->next; +- } else { +- *tmp = pia->next; +- free(pia); + } ++ *tmp = pia->next; ++ free(pia); + } + return ret; + } +@@ -2789,8 +2786,7 @@ port_queue_flow_create(portid_t port_id, queueid_t queue_id, + flow = rte_flow_async_create_by_index(port_id, queue_id, &op_attr, pt->table, + rule_idx, actions, actions_idx, job, &error); + if (!flow) { +- uint64_t flow_id = pf->id; +- port_queue_flow_destroy(port_id, queue_id, true, 1, &flow_id); ++ free(pf); + free(job); + return port_flow_complain(&error); + } +@@ -2997,6 +2993,7 @@ port_queue_flow_update(portid_t port_id, queueid_t queue_id, + int + port_queue_action_handle_create(portid_t port_id, uint32_t queue_id, + bool postpone, uint32_t id, ++ bool indirect_list, + const struct rte_flow_indir_action_conf *conf, + const struct rte_flow_action *action) + { +@@ -3006,8 +3003,6 @@ port_queue_action_handle_create(portid_t port_id, uint32_t queue_id, + int ret; + struct rte_flow_error error; + struct queue_job *job; +- bool is_indirect_list = action[1].type != RTE_FLOW_ACTION_TYPE_END; +- + + ret = action_alloc(port_id, id, &pia); + if (ret) +@@ -3029,7 +3024,7 @@ port_queue_action_handle_create(portid_t port_id, uint32_t queue_id, + /* Poisoning to make sure PMDs update it in case of error. */ + memset(&error, 0x88, sizeof(error)); + +- if (is_indirect_list) ++ if (indirect_list) + queue_action_list_handle_create(port_id, queue_id, pia, job, + &attr, conf, action, &error); + else +@@ -4611,9 +4606,9 @@ fwd_stream_on_other_lcores(uint16_t domain_id, lcoreid_t src_lc, + continue; + printf("Shared Rx queue group %u queue %hu can't be scheduled on different cores:\n", + share_group, share_rxq); +- printf(" lcore %hhu Port %hu queue %hu\n", ++ printf(" lcore %u Port %hu queue %hu\n", + src_lc, src_port, src_rxq); +- printf(" lcore %hhu Port %hu queue %hu\n", ++ printf(" lcore %u Port %hu queue %hu\n", + lc_id, fs->rx_port, fs->rx_queue); + printf("Please use --nb-cores=%hu to limit number of forwarding cores\n", + nb_rxq); +@@ -4794,7 +4789,6 @@ rss_fwd_config_setup(void) + queueid_t nb_q; + streamid_t sm_id; + int start; +- int end; + + nb_q = nb_rxq; + if (nb_q > nb_txq) +@@ -4802,7 +4796,7 @@ rss_fwd_config_setup(void) + cur_fwd_config.nb_fwd_lcores = (lcoreid_t) nb_fwd_lcores; + cur_fwd_config.nb_fwd_ports = nb_fwd_ports; + cur_fwd_config.nb_fwd_streams = +- (streamid_t) (nb_q * cur_fwd_config.nb_fwd_ports); ++ (streamid_t) (nb_q / num_procs * cur_fwd_config.nb_fwd_ports); + + if (cur_fwd_config.nb_fwd_streams < cur_fwd_config.nb_fwd_lcores) + cur_fwd_config.nb_fwd_lcores = +@@ -4824,7 +4818,6 @@ rss_fwd_config_setup(void) + * the 2~3 queue for secondary process. + */ + start = proc_id * nb_q / num_procs; +- end = start + nb_q / num_procs; + rxp = 0; + rxq = start; + for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { +@@ -4843,8 +4836,6 @@ rss_fwd_config_setup(void) + continue; + rxp = 0; + rxq++; +- if (rxq >= end) +- rxq = start; + } + } + +@@ -4989,7 +4980,7 @@ icmp_echo_config_setup(void) + lcoreid_t lc_id; + uint16_t sm_id; + +- if ((nb_txq * nb_fwd_ports) < nb_fwd_lcores) ++ if ((lcoreid_t)(nb_txq * nb_fwd_ports) < nb_fwd_lcores) + cur_fwd_config.nb_fwd_lcores = (lcoreid_t) + (nb_txq * nb_fwd_ports); + else +diff --git a/dpdk/app/test-pmd/csumonly.c b/dpdk/app/test-pmd/csumonly.c +index 21210aff43..71add6ca47 100644 +--- a/dpdk/app/test-pmd/csumonly.c ++++ b/dpdk/app/test-pmd/csumonly.c +@@ -577,21 +577,23 @@ static uint64_t + process_outer_cksums(void *outer_l3_hdr, struct testpmd_offload_info *info, + uint64_t tx_offloads, int tso_enabled, struct rte_mbuf *m) + { +- struct rte_ipv4_hdr *ipv4_hdr = outer_l3_hdr; +- struct rte_ipv6_hdr *ipv6_hdr = outer_l3_hdr; + struct rte_udp_hdr *udp_hdr; + uint64_t ol_flags = 0; + + if (info->outer_ethertype == _htons(RTE_ETHER_TYPE_IPV4)) { +- ipv4_hdr->hdr_checksum = 0; + ol_flags |= RTE_MBUF_F_TX_OUTER_IPV4; + +- if (tx_offloads & RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM) ++ if (tx_offloads & RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM) { + ol_flags |= RTE_MBUF_F_TX_OUTER_IP_CKSUM; +- else ++ } else { ++ struct rte_ipv4_hdr *ipv4_hdr = outer_l3_hdr; ++ ++ ipv4_hdr->hdr_checksum = 0; + ipv4_hdr->hdr_checksum = rte_ipv4_cksum(ipv4_hdr); +- } else ++ } ++ } else { + ol_flags |= RTE_MBUF_F_TX_OUTER_IPV6; ++ } + + if (info->outer_l4_proto != IPPROTO_UDP) + return ol_flags; +@@ -606,13 +608,6 @@ process_outer_cksums(void *outer_l3_hdr, struct testpmd_offload_info *info, + + /* Skip SW outer UDP checksum generation if HW supports it */ + if (tx_offloads & RTE_ETH_TX_OFFLOAD_OUTER_UDP_CKSUM) { +- if (info->outer_ethertype == _htons(RTE_ETHER_TYPE_IPV4)) +- udp_hdr->dgram_cksum +- = rte_ipv4_phdr_cksum(ipv4_hdr, ol_flags); +- else +- udp_hdr->dgram_cksum +- = rte_ipv6_phdr_cksum(ipv6_hdr, ol_flags); +- + ol_flags |= RTE_MBUF_F_TX_OUTER_UDP_CKSUM; + return ol_flags; + } +@@ -870,16 +865,28 @@ pkt_burst_checksum_forward(struct fwd_stream *fs) + + /* receive a burst of packet */ + nb_rx = common_fwd_stream_receive(fs, pkts_burst, nb_pkt_per_burst); +- if (unlikely(nb_rx == 0)) ++ if (unlikely(nb_rx == 0)) { ++#ifndef RTE_LIB_GRO + return false; ++#else ++ gro_enable = gro_ports[fs->rx_port].enable; ++ /* ++ * Check if packets need to be flushed in the GRO context ++ * due to a timeout. ++ * ++ * Continue only in GRO heavyweight mode and if there are ++ * packets in the GRO context. ++ */ ++ if (!gro_enable || (gro_flush_cycles == GRO_DEFAULT_FLUSH_CYCLES) || ++ (rte_gro_get_pkt_count(current_fwd_lcore()->gro_ctx) == 0)) ++ return false; ++#endif ++ } + + rx_bad_ip_csum = 0; + rx_bad_l4_csum = 0; + rx_bad_outer_l4_csum = 0; + rx_bad_outer_ip_csum = 0; +-#ifdef RTE_LIB_GRO +- gro_enable = gro_ports[fs->rx_port].enable; +-#endif + + txp = &ports[fs->tx_port]; + tx_offloads = txp->dev_conf.txmode.offloads; +@@ -1110,6 +1117,7 @@ tunnel_update: + } + + #ifdef RTE_LIB_GRO ++ gro_enable = gro_ports[fs->rx_port].enable; + if (unlikely(gro_enable)) { + if (gro_flush_cycles == GRO_DEFAULT_FLUSH_CYCLES) { + nb_rx = rte_gro_reassemble_burst(pkts_burst, nb_rx, +@@ -1129,6 +1137,8 @@ tunnel_update: + gro_pkts_num); + fs->gro_times = 0; + } ++ if (nb_rx == 0) ++ return false; + } + + pkts_ip_csum_recalc(pkts_burst, nb_rx, tx_offloads); +diff --git a/dpdk/app/test-pmd/ieee1588fwd.c b/dpdk/app/test-pmd/ieee1588fwd.c +index 386d9f10e6..52ae551e2f 100644 +--- a/dpdk/app/test-pmd/ieee1588fwd.c ++++ b/dpdk/app/test-pmd/ieee1588fwd.c +@@ -197,14 +197,23 @@ ieee1588_packet_fwd(struct fwd_stream *fs) + static int + port_ieee1588_fwd_begin(portid_t pi) + { +- rte_eth_timesync_enable(pi); +- return 0; ++ int ret; ++ ++ ret = rte_eth_timesync_enable(pi); ++ if (ret) ++ printf("Port %u enable PTP failed, ret = %d\n", pi, ret); ++ ++ return ret; + } + + static void + port_ieee1588_fwd_end(portid_t pi) + { +- rte_eth_timesync_disable(pi); ++ int ret; ++ ++ ret = rte_eth_timesync_disable(pi); ++ if (ret) ++ printf("Port %u disable PTP failed, ret = %d\n", pi, ret); + } + + static void +diff --git a/dpdk/app/test-pmd/parameters.c b/dpdk/app/test-pmd/parameters.c +index a9ca58339d..7e23393392 100644 +--- a/dpdk/app/test-pmd/parameters.c ++++ b/dpdk/app/test-pmd/parameters.c +@@ -100,10 +100,6 @@ usage(char* progname) + "of peer ports.\n"); + printf(" --eth-peer=X,M:M:M:M:M:M: set the MAC address of the X peer " + "port (0 <= X < %d).\n", RTE_MAX_ETHPORTS); +-#endif +-#ifdef RTE_LIB_LATENCYSTATS +- printf(" --latencystats=N: enable latency and jitter statistics " +- "monitoring on forwarding lcore id N.\n"); + #endif + printf(" --disable-crc-strip: disable CRC stripping by hardware.\n"); + printf(" --enable-scatter: enable scattered Rx.\n"); +@@ -167,8 +163,14 @@ usage(char* progname) + printf(" --disable-device-start: do not automatically start port\n"); + printf(" --no-lsc-interrupt: disable link status change interrupt.\n"); + printf(" --no-rmv-interrupt: disable device removal interrupt.\n"); ++#ifdef RTE_LIB_BITRATESTATS + printf(" --bitrate-stats=N: set the logical core N to perform " + "bit-rate calculation.\n"); ++#endif ++#ifdef RTE_LIB_LATENCYSTATS ++ printf(" --latencystats=N: enable latency and jitter statistics " ++ "monitoring on forwarding lcore id N.\n"); ++#endif + printf(" --print-event : " + "enable print of designated event or all of them.\n"); + printf(" --mask-event : " +@@ -774,7 +776,7 @@ launch_args_parse(int argc, char** argv) + n = strtoul(optarg, &end, 10); + if ((optarg[0] == '\0') || (end == NULL) || + (*end != '\0')) +- break; ++ rte_exit(EXIT_FAILURE, "Invalid stats-period value\n"); + + stats_period = n; + break; +@@ -875,8 +877,8 @@ launch_args_parse(int argc, char** argv) + } + if (!strcmp(lgopts[opt_idx].name, "nb-cores")) { + n = atoi(optarg); +- if (n > 0 && n <= nb_lcores) +- nb_fwd_lcores = (uint8_t) n; ++ if (n > 0 && (lcoreid_t)n <= nb_lcores) ++ nb_fwd_lcores = (lcoreid_t) n; + else + rte_exit(EXIT_FAILURE, + "nb-cores should be > 0 and <= %d\n", +@@ -1126,7 +1128,9 @@ launch_args_parse(int argc, char** argv) + 0, + &dev_info); + if (ret != 0) +- return; ++ rte_exit(EXIT_FAILURE, "Failed to get driver " ++ "recommended burst size, please provide a " ++ "value between 1 and %d\n", MAX_PKT_BURST); + + rec_nb_pkts = dev_info + .default_rxportconf.burst_size; +@@ -1493,7 +1497,7 @@ launch_args_parse(int argc, char** argv) + break; + default: + usage(argv[0]); +- fprintf(stderr, "Invalid option: %s\n", argv[optind]); ++ fprintf(stderr, "Invalid option: %s\n", argv[optind - 1]); + rte_exit(EXIT_FAILURE, + "Command line is incomplete or incorrect\n"); + break; +diff --git a/dpdk/app/test-pmd/testpmd.h b/dpdk/app/test-pmd/testpmd.h +index 9b10a9ea1c..db6d0d35f4 100644 +--- a/dpdk/app/test-pmd/testpmd.h ++++ b/dpdk/app/test-pmd/testpmd.h +@@ -84,7 +84,7 @@ extern volatile uint8_t f_quit; + /* Maximum number of pools supported per Rx queue */ + #define MAX_MEMPOOL 8 + +-typedef uint8_t lcoreid_t; ++typedef uint32_t lcoreid_t; + typedef uint16_t portid_t; + typedef uint16_t queueid_t; + typedef uint16_t streamid_t; +@@ -996,6 +996,7 @@ int port_queue_flow_update(portid_t port_id, queueid_t queue_id, + const struct rte_flow_action *actions); + int port_queue_action_handle_create(portid_t port_id, uint32_t queue_id, + bool postpone, uint32_t id, ++ bool indirect_list, + const struct rte_flow_indir_action_conf *conf, + const struct rte_flow_action *action); + int port_queue_action_handle_destroy(portid_t port_id, +diff --git a/dpdk/app/test/process.h b/dpdk/app/test/process.h +index af7bc3e0de..9fb2bf481c 100644 +--- a/dpdk/app/test/process.h ++++ b/dpdk/app/test/process.h +@@ -17,6 +17,7 @@ + #include + + #include /* strlcpy */ ++#include + + #ifdef RTE_EXEC_ENV_FREEBSD + #define self "curproc" +@@ -34,6 +35,34 @@ extern uint16_t flag_for_send_pkts; + #endif + #endif + ++#define PREFIX_ALLOW "--allow=" ++ ++static int ++add_parameter_allow(char **argv, int max_capacity) ++{ ++ struct rte_devargs *devargs; ++ int count = 0; ++ ++ RTE_EAL_DEVARGS_FOREACH(NULL, devargs) { ++ if (strlen(devargs->name) == 0) ++ continue; ++ ++ if (devargs->data == NULL || strlen(devargs->data) == 0) { ++ if (asprintf(&argv[count], PREFIX_ALLOW"%s", devargs->name) < 0) ++ break; ++ } else { ++ if (asprintf(&argv[count], PREFIX_ALLOW"%s,%s", ++ devargs->name, devargs->data) < 0) ++ break; ++ } ++ ++ if (++count == max_capacity) ++ break; ++ } ++ ++ return count; ++} ++ + /* + * launches a second copy of the test process using the given argv parameters, + * which should include argv[0] as the process name. To identify in the +@@ -43,8 +72,10 @@ extern uint16_t flag_for_send_pkts; + static inline int + process_dup(const char *const argv[], int numargs, const char *env_value) + { +- int num; +- char *argv_cpy[numargs + 1]; ++ int num = 0; ++ char **argv_cpy; ++ int allow_num; ++ int argv_num; + int i, status; + char path[32]; + #ifdef RTE_LIB_PDUMP +@@ -58,11 +89,21 @@ process_dup(const char *const argv[], int numargs, const char *env_value) + if (pid < 0) + return -1; + else if (pid == 0) { ++ allow_num = rte_devargs_type_count(RTE_DEVTYPE_ALLOWED); ++ argv_num = numargs + allow_num + 1; ++ argv_cpy = calloc(argv_num, sizeof(char *)); ++ if (!argv_cpy) ++ rte_panic("Memory allocation failed\n"); ++ + /* make a copy of the arguments to be passed to exec */ +- for (i = 0; i < numargs; i++) ++ for (i = 0; i < numargs; i++) { + argv_cpy[i] = strdup(argv[i]); +- argv_cpy[i] = NULL; +- num = numargs; ++ if (argv_cpy[i] == NULL) ++ rte_panic("Error dup args\n"); ++ } ++ if (allow_num > 0) ++ num = add_parameter_allow(&argv_cpy[i], allow_num); ++ num += numargs; + + #ifdef RTE_EXEC_ENV_LINUX + { +diff --git a/dpdk/app/test/test.c b/dpdk/app/test/test.c +index bfa9ea52e3..7b882a59de 100644 +--- a/dpdk/app/test/test.c ++++ b/dpdk/app/test/test.c +@@ -375,11 +375,13 @@ unit_test_suite_runner(struct unit_test_suite *suite) + + if (test_success == TEST_SUCCESS) + suite->succeeded++; +- else if (test_success == TEST_SKIPPED) ++ else if (test_success == TEST_SKIPPED) { + suite->skipped++; +- else if (test_success == -ENOTSUP) ++ suite->executed--; ++ } else if (test_success == -ENOTSUP) { + suite->unsupported++; +- else ++ suite->executed--; ++ } else + suite->failed++; + } else if (test_success == -ENOTSUP) { + suite->unsupported++; +diff --git a/dpdk/app/test/test_cfgfile.c b/dpdk/app/test/test_cfgfile.c +index 2f596affee..a5e3d8699c 100644 +--- a/dpdk/app/test/test_cfgfile.c ++++ b/dpdk/app/test/test_cfgfile.c +@@ -168,7 +168,7 @@ test_cfgfile_invalid_section_header(void) + struct rte_cfgfile *cfgfile; + + cfgfile = rte_cfgfile_load(CFG_FILES_ETC "/invalid_section.ini", 0); +- TEST_ASSERT_NULL(cfgfile, "Expected failured did not occur"); ++ TEST_ASSERT_NULL(cfgfile, "Expected failure did not occur"); + + return 0; + } +@@ -185,7 +185,7 @@ test_cfgfile_invalid_comment(void) + + cfgfile = rte_cfgfile_load_with_params(CFG_FILES_ETC "/sample2.ini", 0, + ¶ms); +- TEST_ASSERT_NULL(cfgfile, "Expected failured did not occur"); ++ TEST_ASSERT_NULL(cfgfile, "Expected failure did not occur"); + + return 0; + } +@@ -196,7 +196,7 @@ test_cfgfile_invalid_key_value_pair(void) + struct rte_cfgfile *cfgfile; + + cfgfile = rte_cfgfile_load(CFG_FILES_ETC "/empty_key_value.ini", 0); +- TEST_ASSERT_NULL(cfgfile, "Expected failured did not occur"); ++ TEST_ASSERT_NULL(cfgfile, "Expected failure did not occur"); + + return 0; + } +@@ -236,7 +236,7 @@ test_cfgfile_missing_section(void) + struct rte_cfgfile *cfgfile; + + cfgfile = rte_cfgfile_load(CFG_FILES_ETC "/missing_section.ini", 0); +- TEST_ASSERT_NULL(cfgfile, "Expected failured did not occur"); ++ TEST_ASSERT_NULL(cfgfile, "Expected failure did not occur"); + + return 0; + } +diff --git a/dpdk/app/test/test_cryptodev.c b/dpdk/app/test/test_cryptodev.c +index 58561ededf..6cd38aefae 100644 +--- a/dpdk/app/test/test_cryptodev.c ++++ b/dpdk/app/test/test_cryptodev.c +@@ -196,6 +196,8 @@ post_process_raw_dp_op(void *user_data, uint32_t index __rte_unused, + static struct crypto_testsuite_params testsuite_params = { NULL }; + struct crypto_testsuite_params *p_testsuite_params = &testsuite_params; + static struct crypto_unittest_params unittest_params; ++static bool enq_cb_called; ++static bool deq_cb_called; + + int + process_sym_raw_dp_op(uint8_t dev_id, uint16_t qp_id, +@@ -9121,7 +9123,7 @@ static int test_pdcp_proto(int i, int oop, enum rte_crypto_cipher_operation opc, + /* Out of place support */ + if (oop) { + /* +- * For out-op-place we need to alloc another mbuf ++ * For out-of-place we need to alloc another mbuf + */ + ut_params->obuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + rte_pktmbuf_append(ut_params->obuf, output_vec_len); +@@ -9330,7 +9332,7 @@ test_pdcp_proto_SGL(int i, int oop, + /* Out of place support */ + if (oop) { + /* +- * For out-op-place we need to alloc another mbuf ++ * For out-of-place we need to alloc another mbuf + */ + ut_params->obuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + rte_pktmbuf_append(ut_params->obuf, frag_size_oop); +@@ -13540,6 +13542,7 @@ test_enq_callback(uint16_t dev_id, uint16_t qp_id, struct rte_crypto_op **ops, + RTE_SET_USED(ops); + RTE_SET_USED(user_param); + ++ enq_cb_called = true; + printf("crypto enqueue callback called\n"); + return nb_ops; + } +@@ -13553,21 +13556,58 @@ test_deq_callback(uint16_t dev_id, uint16_t qp_id, struct rte_crypto_op **ops, + RTE_SET_USED(ops); + RTE_SET_USED(user_param); + ++ deq_cb_called = true; + printf("crypto dequeue callback called\n"); + return nb_ops; + } + + /* +- * Thread using enqueue/dequeue callback with RCU. ++ * Process enqueue/dequeue NULL crypto request to verify callback with RCU. + */ + static int +-test_enqdeq_callback_thread(void *arg) ++test_enqdeq_callback_null_cipher(void) + { +- RTE_SET_USED(arg); +- /* DP thread calls rte_cryptodev_enqueue_burst()/ +- * rte_cryptodev_dequeue_burst() and invokes callback. +- */ +- test_null_burst_operation(); ++ struct crypto_testsuite_params *ts_params = &testsuite_params; ++ struct crypto_unittest_params *ut_params = &unittest_params; ++ ++ /* Setup Cipher Parameters */ ++ ut_params->cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER; ++ ut_params->cipher_xform.next = &ut_params->auth_xform; ++ ++ ut_params->cipher_xform.cipher.algo = RTE_CRYPTO_CIPHER_NULL; ++ ut_params->cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT; ++ ++ /* Setup Auth Parameters */ ++ ut_params->auth_xform.type = RTE_CRYPTO_SYM_XFORM_AUTH; ++ ut_params->auth_xform.next = NULL; ++ ++ ut_params->auth_xform.auth.algo = RTE_CRYPTO_AUTH_NULL; ++ ut_params->auth_xform.auth.op = RTE_CRYPTO_AUTH_OP_GENERATE; ++ ++ /* Create Crypto session */ ++ ut_params->sess = rte_cryptodev_sym_session_create(ts_params->valid_devs[0], ++ &ut_params->auth_xform, ts_params->session_mpool); ++ TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); ++ ++ ut_params->op = rte_crypto_op_alloc(ts_params->op_mpool, RTE_CRYPTO_OP_TYPE_SYMMETRIC); ++ TEST_ASSERT_NOT_NULL(ut_params->op, "Failed to allocate symmetric crypto op"); ++ ++ /* Allocate mbuf */ ++ ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); ++ TEST_ASSERT_NOT_NULL(ut_params->ibuf, "Failed to allocate mbuf"); ++ ++ /* Append some random data */ ++ TEST_ASSERT_NOT_NULL(rte_pktmbuf_append(ut_params->ibuf, sizeof(unsigned int)), ++ "no room to append data"); ++ ++ rte_crypto_op_attach_sym_session(ut_params->op, ut_params->sess); ++ ++ ut_params->op->sym->m_src = ut_params->ibuf; ++ ++ /* Process crypto operation */ ++ TEST_ASSERT_NOT_NULL(process_crypto_request(ts_params->valid_devs[0], ut_params->op), ++ "failed to process sym crypto op"); ++ + return 0; + } + +@@ -13575,6 +13615,7 @@ static int + test_enq_callback_setup(void) + { + struct crypto_testsuite_params *ts_params = &testsuite_params; ++ struct rte_cryptodev_sym_capability_idx cap_idx; + struct rte_cryptodev_info dev_info; + struct rte_cryptodev_qp_conf qp_conf = { + .nb_descriptors = MAX_NUM_OPS_INFLIGHT +@@ -13582,6 +13623,19 @@ test_enq_callback_setup(void) + + struct rte_cryptodev_cb *cb; + uint16_t qp_id = 0; ++ int j = 0; ++ ++ /* Verify the crypto capabilities for which enqueue/dequeue is done. */ ++ cap_idx.type = RTE_CRYPTO_SYM_XFORM_AUTH; ++ cap_idx.algo.auth = RTE_CRYPTO_AUTH_NULL; ++ if (rte_cryptodev_sym_capability_get(ts_params->valid_devs[0], ++ &cap_idx) == NULL) ++ return TEST_SKIPPED; ++ cap_idx.type = RTE_CRYPTO_SYM_XFORM_CIPHER; ++ cap_idx.algo.cipher = RTE_CRYPTO_CIPHER_NULL; ++ if (rte_cryptodev_sym_capability_get(ts_params->valid_devs[0], ++ &cap_idx) == NULL) ++ return TEST_SKIPPED; + + /* Stop the device in case it's started so it can be configured */ + rte_cryptodev_stop(ts_params->valid_devs[0]); +@@ -13605,9 +13659,16 @@ test_enq_callback_setup(void) + qp_conf.nb_descriptors, qp_id, + ts_params->valid_devs[0]); + ++ enq_cb_called = false; + /* Test with invalid crypto device */ + cb = rte_cryptodev_add_enq_callback(RTE_CRYPTO_MAX_DEVS, + qp_id, test_enq_callback, NULL); ++ if (rte_errno == ENOTSUP) { ++ RTE_LOG(ERR, USER1, "%s line %d: " ++ "rte_cryptodev_add_enq_callback() " ++ "Not supported, skipped\n", __func__, __LINE__); ++ return TEST_SKIPPED; ++ } + TEST_ASSERT_NULL(cb, "Add callback on qp %u on " + "cryptodev %u did not fail", + qp_id, RTE_CRYPTO_MAX_DEVS); +@@ -13637,12 +13698,11 @@ test_enq_callback_setup(void) + + rte_cryptodev_start(ts_params->valid_devs[0]); + +- /* Launch a thread */ +- rte_eal_remote_launch(test_enqdeq_callback_thread, NULL, +- rte_get_next_lcore(-1, 1, 0)); ++ TEST_ASSERT_SUCCESS(test_enqdeq_callback_null_cipher(), "Crypto Processing failed"); + +- /* Wait until reader exited. */ +- rte_eal_mp_wait_lcore(); ++ /* Wait until callback not called. */ ++ while (!enq_cb_called && (j++ < 10)) ++ rte_delay_ms(10); + + /* Test with invalid crypto device */ + TEST_ASSERT_FAIL(rte_cryptodev_remove_enq_callback( +@@ -13667,6 +13727,8 @@ test_enq_callback_setup(void) + "qp %u on cryptodev %u", + qp_id, ts_params->valid_devs[0]); + ++ TEST_ASSERT(deq_cb_called == true, "Crypto dequeue callback not called"); ++ + return TEST_SUCCESS; + } + +@@ -13674,6 +13736,7 @@ static int + test_deq_callback_setup(void) + { + struct crypto_testsuite_params *ts_params = &testsuite_params; ++ struct rte_cryptodev_sym_capability_idx cap_idx; + struct rte_cryptodev_info dev_info; + struct rte_cryptodev_qp_conf qp_conf = { + .nb_descriptors = MAX_NUM_OPS_INFLIGHT +@@ -13681,6 +13744,19 @@ test_deq_callback_setup(void) + + struct rte_cryptodev_cb *cb; + uint16_t qp_id = 0; ++ int j = 0; ++ ++ /* Verify the crypto capabilities for which enqueue/dequeue is done. */ ++ cap_idx.type = RTE_CRYPTO_SYM_XFORM_AUTH; ++ cap_idx.algo.auth = RTE_CRYPTO_AUTH_NULL; ++ if (rte_cryptodev_sym_capability_get(ts_params->valid_devs[0], ++ &cap_idx) == NULL) ++ return TEST_SKIPPED; ++ cap_idx.type = RTE_CRYPTO_SYM_XFORM_CIPHER; ++ cap_idx.algo.cipher = RTE_CRYPTO_CIPHER_NULL; ++ if (rte_cryptodev_sym_capability_get(ts_params->valid_devs[0], ++ &cap_idx) == NULL) ++ return TEST_SKIPPED; + + /* Stop the device in case it's started so it can be configured */ + rte_cryptodev_stop(ts_params->valid_devs[0]); +@@ -13704,9 +13780,16 @@ test_deq_callback_setup(void) + qp_conf.nb_descriptors, qp_id, + ts_params->valid_devs[0]); + ++ deq_cb_called = false; + /* Test with invalid crypto device */ + cb = rte_cryptodev_add_deq_callback(RTE_CRYPTO_MAX_DEVS, + qp_id, test_deq_callback, NULL); ++ if (rte_errno == ENOTSUP) { ++ RTE_LOG(ERR, USER1, "%s line %d: " ++ "rte_cryptodev_add_deq_callback() " ++ "Not supported, skipped\n", __func__, __LINE__); ++ return TEST_SKIPPED; ++ } + TEST_ASSERT_NULL(cb, "Add callback on qp %u on " + "cryptodev %u did not fail", + qp_id, RTE_CRYPTO_MAX_DEVS); +@@ -13736,12 +13819,11 @@ test_deq_callback_setup(void) + + rte_cryptodev_start(ts_params->valid_devs[0]); + +- /* Launch a thread */ +- rte_eal_remote_launch(test_enqdeq_callback_thread, NULL, +- rte_get_next_lcore(-1, 1, 0)); ++ TEST_ASSERT_SUCCESS(test_enqdeq_callback_null_cipher(), "Crypto processing failed"); + +- /* Wait until reader exited. */ +- rte_eal_mp_wait_lcore(); ++ /* Wait until callback not called. */ ++ while (!deq_cb_called && (j++ < 10)) ++ rte_delay_ms(10); + + /* Test with invalid crypto device */ + TEST_ASSERT_FAIL(rte_cryptodev_remove_deq_callback( +@@ -13766,6 +13848,8 @@ test_deq_callback_setup(void) + "qp %u on cryptodev %u", + qp_id, ts_params->valid_devs[0]); + ++ TEST_ASSERT(enq_cb_called == true, "Crypto enqueue callback not called"); ++ + return TEST_SUCCESS; + } + +@@ -15486,7 +15570,7 @@ test_authenticated_encryption_SGL(const struct aead_test_data *tdata, + } + + /* +- * For out-op-place we need to alloc another mbuf ++ * For out-of-place we need to alloc another mbuf + */ + if (oop) { + ut_params->obuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); +diff --git a/dpdk/app/test/test_cryptodev_asym.c b/dpdk/app/test/test_cryptodev_asym.c +index 17daf734e8..3b57a0249b 100644 +--- a/dpdk/app/test/test_cryptodev_asym.c ++++ b/dpdk/app/test/test_cryptodev_asym.c +@@ -547,8 +547,6 @@ ut_setup_asym(void) + qp_id, ts_params->valid_devs[0]); + } + +- rte_cryptodev_stats_reset(ts_params->valid_devs[0]); +- + /* Start the device */ + TEST_ASSERT_SUCCESS(rte_cryptodev_start(ts_params->valid_devs[0]), + "Failed to start cryptodev %u", +@@ -561,7 +559,6 @@ static void + ut_teardown_asym(void) + { + struct crypto_testsuite_params_asym *ts_params = &testsuite_params; +- struct rte_cryptodev_stats stats; + uint8_t dev_id = ts_params->valid_devs[0]; + + if (self->sess != NULL) +@@ -571,8 +568,6 @@ ut_teardown_asym(void) + self->op = NULL; + self->result_op = NULL; + +- rte_cryptodev_stats_get(ts_params->valid_devs[0], &stats); +- + /* Stop the device */ + rte_cryptodev_stop(ts_params->valid_devs[0]); + } +@@ -631,7 +626,7 @@ test_capability(void) + RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO)) { + RTE_LOG(INFO, USER1, + "Device doesn't support asymmetric. Test Skipped\n"); +- return TEST_SUCCESS; ++ return TEST_SKIPPED; + } + + /* print xform capability */ +@@ -646,6 +641,7 @@ test_capability(void) + capa = rte_cryptodev_asym_capability_get(dev_id, + (const struct + rte_cryptodev_asym_capability_idx *) &idx); ++ TEST_ASSERT_NOT_NULL(capa, "Failed to get asymmetric capability"); + print_asym_capa(capa); + } + } +@@ -3200,6 +3196,32 @@ static int send_one(void) + return TEST_SUCCESS; + } + ++static int ++modular_cmpeq(const uint8_t *a, size_t a_len, const uint8_t *b, size_t b_len) ++{ ++ const uint8_t *new_a, *new_b; ++ size_t i, j; ++ ++ /* Strip leading NUL bytes */ ++ for (i = 0; i < a_len; i++) ++ if (a[i] != 0) ++ break; ++ ++ for (j = 0; j < b_len; j++) ++ if (b[j] != 0) ++ break; ++ ++ if (a_len - i != b_len - j) ++ return 1; ++ ++ new_a = &a[i]; ++ new_b = &b[j]; ++ if (memcmp(new_a, new_b, a_len - i)) ++ return 1; ++ ++ return 0; ++} ++ + static int + modular_exponentiation(const void *test_data) + { +@@ -3234,9 +3256,9 @@ modular_exponentiation(const void *test_data) + + TEST_ASSERT_SUCCESS(send_one(), + "Failed to process crypto op"); +- TEST_ASSERT_BUFFERS_ARE_EQUAL(vector->reminder.data, ++ TEST_ASSERT_SUCCESS(modular_cmpeq(vector->reminder.data, vector->reminder.len, + self->result_op->asym->modex.result.data, +- self->result_op->asym->modex.result.length, ++ self->result_op->asym->modex.result.length), + "operation verification failed\n"); + + return TEST_SUCCESS; +@@ -3292,11 +3314,8 @@ modular_multiplicative_inverse(const void *test_data) + arg.qt.coef.data = coef; \ + arg.qt.coef.length = vector->coef.len + +-typedef void (*rsa_key_init_t)(struct rte_crypto_asym_xform *, +- const struct rsa_test_data_2 *); +- + static int +-RSA_Encrypt(const struct rsa_test_data_2 *vector, uint8_t *cipher_buf) ++rsa_encrypt(const struct rsa_test_data_2 *vector, uint8_t *cipher_buf) + { + self->result_op = NULL; + /* Compute encryption on the test vector */ +@@ -3314,7 +3333,7 @@ RSA_Encrypt(const struct rsa_test_data_2 *vector, uint8_t *cipher_buf) + } + + static int +-RSA_Decrypt(const struct rsa_test_data_2 *vector, uint8_t *plaintext, ++rsa_decrypt(const struct rsa_test_data_2 *vector, uint8_t *plaintext, + const int use_op) + { + uint8_t cipher[TEST_DATA_SIZE] = { 0 }; +@@ -3335,41 +3354,14 @@ RSA_Decrypt(const struct rsa_test_data_2 *vector, uint8_t *plaintext, + return 0; + } + +-static void +-RSA_key_init_Exp(struct rte_crypto_asym_xform *xform, +- const struct rsa_test_data_2 *vector) +-{ +- SET_RSA_PARAM(xform->rsa, vector, n); +- SET_RSA_PARAM(xform->rsa, vector, e); +- SET_RSA_PARAM(xform->rsa, vector, d); +- xform->rsa.key_type = RTE_RSA_KEY_TYPE_EXP; +-} +- +-static void +-RSA_key_init_CRT(struct rte_crypto_asym_xform *xform, +- const struct rsa_test_data_2 *vector) +-{ +- SET_RSA_PARAM(xform->rsa, vector, n); +- SET_RSA_PARAM(xform->rsa, vector, e); +- SET_RSA_PARAM_QT(xform->rsa, vector, p); +- SET_RSA_PARAM_QT(xform->rsa, vector, q); +- SET_RSA_PARAM_QT(xform->rsa, vector, dP); +- SET_RSA_PARAM_QT(xform->rsa, vector, dQ); +- SET_RSA_PARAM_QT(xform->rsa, vector, qInv); +- xform->rsa.key_type = RTE_RSA_KEY_TYPE_QT; +-} +- + static int +-RSA_Init_Session(const struct rsa_test_data_2 *vector, +- rsa_key_init_t key_init) ++rsa_init_session(struct rte_crypto_asym_xform *xform) + { + const uint8_t dev_id = params->valid_devs[0]; + struct rte_cryptodev_info dev_info; +- struct rte_crypto_asym_xform xform = { }; + int ret = 0; + +- key_init(&xform, vector); +- xform.xform_type = RTE_CRYPTO_ASYM_XFORM_RSA; ++ xform->xform_type = RTE_CRYPTO_ASYM_XFORM_RSA; + + rte_cryptodev_info_get(dev_id, &dev_info); + if (!(dev_info.feature_flags & RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_QT)) { +@@ -3377,7 +3369,7 @@ RSA_Init_Session(const struct rsa_test_data_2 *vector, + "Device doesn't support decrypt op with quintuple key type. Test skipped\n"); + return TEST_SKIPPED; + } +- ret = rte_cryptodev_asym_session_create(dev_id, &xform, ++ ret = rte_cryptodev_asym_session_create(dev_id, xform, + params->session_mpool, &self->sess); + if (ret < 0) { + RTE_LOG(ERR, USER1, +@@ -3388,17 +3380,23 @@ RSA_Init_Session(const struct rsa_test_data_2 *vector, + } + + static int +-KAT_RSA_Encrypt(const void *data) ++kat_rsa_encrypt(const void *data) + { + uint8_t cipher_buf[TEST_DATA_SIZE] = {0}; + const struct rsa_test_data_2 *vector = data; +- int ret = RSA_Init_Session(vector, RSA_key_init_Exp); ++ struct rte_crypto_asym_xform xform = { }; ++ ++ SET_RSA_PARAM(xform.rsa, vector, n); ++ SET_RSA_PARAM(xform.rsa, vector, e); ++ SET_RSA_PARAM(xform.rsa, vector, d); ++ xform.rsa.key_type = RTE_RSA_KEY_TYPE_EXP; ++ int ret = rsa_init_session(&xform); + + if (ret) { + RTE_LOG(ERR, USER1, "Failed to init session for RSA\n"); + return ret; + } +- TEST_ASSERT_SUCCESS(RSA_Encrypt(vector, cipher_buf), ++ TEST_ASSERT_SUCCESS(rsa_encrypt(vector, cipher_buf), + "RSA: Failed to encrypt"); + TEST_ASSERT_BUFFERS_ARE_EQUAL(vector->cipher.data, + self->result_op->asym->rsa.cipher.data, +@@ -3408,17 +3406,26 @@ KAT_RSA_Encrypt(const void *data) + } + + static int +-KAT_RSA_Encrypt_CRT(const void *data) ++kat_rsa_encrypt_crt(const void *data) + { + uint8_t cipher_buf[TEST_DATA_SIZE] = {0}; + const struct rsa_test_data_2 *vector = data; +- int ret = RSA_Init_Session(vector, RSA_key_init_CRT); ++ struct rte_crypto_asym_xform xform = { }; + ++ SET_RSA_PARAM(xform.rsa, vector, n); ++ SET_RSA_PARAM(xform.rsa, vector, e); ++ SET_RSA_PARAM_QT(xform.rsa, vector, p); ++ SET_RSA_PARAM_QT(xform.rsa, vector, q); ++ SET_RSA_PARAM_QT(xform.rsa, vector, dP); ++ SET_RSA_PARAM_QT(xform.rsa, vector, dQ); ++ SET_RSA_PARAM_QT(xform.rsa, vector, qInv); ++ xform.rsa.key_type = RTE_RSA_KEY_TYPE_QT; ++ int ret = rsa_init_session(&xform); + if (ret) { + RTE_LOG(ERR, USER1, "Failed to init session for RSA\n"); + return ret; + } +- TEST_ASSERT_SUCCESS(RSA_Encrypt(vector, cipher_buf), ++ TEST_ASSERT_SUCCESS(rsa_encrypt(vector, cipher_buf), + "RSA: Failed to encrypt"); + TEST_ASSERT_BUFFERS_ARE_EQUAL(vector->cipher.data, + self->result_op->asym->rsa.cipher.data, +@@ -3428,17 +3435,23 @@ KAT_RSA_Encrypt_CRT(const void *data) + } + + static int +-KAT_RSA_Decrypt(const void *data) ++kat_rsa_decrypt(const void *data) + { + uint8_t message[TEST_DATA_SIZE] = {0}; + const struct rsa_test_data_2 *vector = data; +- int ret = RSA_Init_Session(vector, RSA_key_init_Exp); ++ struct rte_crypto_asym_xform xform = { }; ++ ++ SET_RSA_PARAM(xform.rsa, vector, n); ++ SET_RSA_PARAM(xform.rsa, vector, e); ++ SET_RSA_PARAM(xform.rsa, vector, d); ++ xform.rsa.key_type = RTE_RSA_KEY_TYPE_EXP; ++ int ret = rsa_init_session(&xform); + + if (ret) { + RTE_LOG(ERR, USER1, "Failed to init session for RSA\n"); + return ret; + } +- TEST_ASSERT_SUCCESS(RSA_Decrypt(vector, message, 0), ++ TEST_ASSERT_SUCCESS(rsa_decrypt(vector, message, 0), + "RSA: Failed to encrypt"); + TEST_ASSERT_BUFFERS_ARE_EQUAL(vector->message.data, + self->result_op->asym->rsa.message.data, +@@ -3448,17 +3461,26 @@ KAT_RSA_Decrypt(const void *data) + } + + static int +-KAT_RSA_Decrypt_CRT(const void *data) ++kat_rsa_decrypt_crt(const void *data) + { + uint8_t message[TEST_DATA_SIZE] = {0}; + const struct rsa_test_data_2 *vector = data; +- int ret = RSA_Init_Session(vector, RSA_key_init_CRT); ++ struct rte_crypto_asym_xform xform = { }; + ++ SET_RSA_PARAM(xform.rsa, vector, n); ++ SET_RSA_PARAM(xform.rsa, vector, e); ++ SET_RSA_PARAM_QT(xform.rsa, vector, p); ++ SET_RSA_PARAM_QT(xform.rsa, vector, q); ++ SET_RSA_PARAM_QT(xform.rsa, vector, dP); ++ SET_RSA_PARAM_QT(xform.rsa, vector, dQ); ++ SET_RSA_PARAM_QT(xform.rsa, vector, qInv); ++ xform.rsa.key_type = RTE_RSA_KEY_TYPE_QT; ++ int ret = rsa_init_session(&xform); + if (ret) { + RTE_LOG(ERR, USER1, "Failed to init session for RSA\n"); + return ret; + } +- TEST_ASSERT_SUCCESS(RSA_Decrypt(vector, message, 0), ++ TEST_ASSERT_SUCCESS(rsa_decrypt(vector, message, 0), + "RSA: Failed to encrypt"); + TEST_ASSERT_BUFFERS_ARE_EQUAL(vector->message.data, + self->result_op->asym->rsa.message.data, +@@ -3535,20 +3557,20 @@ static struct unit_test_suite cryptodev_qat_asym_testsuite = { + TEST_CASE_NAMED_WITH_DATA( + "RSA Encryption (n=128, pt=20, e=3) EXP, Padding: NONE", + ut_setup_asym, ut_teardown_asym, +- KAT_RSA_Encrypt, &RSA_vector_128_20_3_None), ++ kat_rsa_encrypt, &rsa_vector_128_20_3_none), + TEST_CASE_NAMED_WITH_DATA( + "RSA Decryption (n=128, pt=20, e=3) EXP, Padding: NONE", + ut_setup_asym, ut_teardown_asym, +- KAT_RSA_Decrypt, &RSA_vector_128_20_3_None), ++ kat_rsa_decrypt, &rsa_vector_128_20_3_none), + /* RSA CRT */ + TEST_CASE_NAMED_WITH_DATA( + "RSA Encryption (n=128, pt=20, e=3) CRT, Padding: NONE", + ut_setup_asym, ut_teardown_asym, +- KAT_RSA_Encrypt_CRT, &RSA_vector_128_20_3_None), ++ kat_rsa_encrypt_crt, &rsa_vector_128_20_3_none), + TEST_CASE_NAMED_WITH_DATA( + "RSA Decryption (n=128, pt=20, e=3) CRT, Padding: NONE", + ut_setup_asym, ut_teardown_asym, +- KAT_RSA_Decrypt_CRT, &RSA_vector_128_20_3_None), ++ kat_rsa_decrypt_crt, &rsa_vector_128_20_3_none), + TEST_CASES_END() /**< NULL terminate unit test array */ + } + }; +diff --git a/dpdk/app/test/test_cryptodev_rsa_test_vectors.h b/dpdk/app/test/test_cryptodev_rsa_test_vectors.h +index b4982014a2..89981f13f0 100644 +--- a/dpdk/app/test/test_cryptodev_rsa_test_vectors.h ++++ b/dpdk/app/test/test_cryptodev_rsa_test_vectors.h +@@ -72,7 +72,7 @@ struct rsa_test_data_2 { + }; + + static const struct +-rsa_test_data_2 RSA_vector_128_20_3_None = { ++rsa_test_data_2 rsa_vector_128_20_3_none = { + .description = + "RSA Encryption Decryption (n=128, pt=20, e=3) EXP, QT", + .xform_type = RTE_CRYPTO_ASYM_XFORM_RSA, +diff --git a/dpdk/app/test/test_event_eth_tx_adapter.c b/dpdk/app/test/test_event_eth_tx_adapter.c +index dbd22f6800..482b8e69e3 100644 +--- a/dpdk/app/test/test_event_eth_tx_adapter.c ++++ b/dpdk/app/test/test_event_eth_tx_adapter.c +@@ -484,6 +484,10 @@ tx_adapter_service(void) + int internal_port; + uint32_t cap; + ++ /* Initialize mbufs */ ++ for (i = 0; i < RING_SIZE; i++) ++ rte_pktmbuf_reset(&bufs[i]); ++ + memset(&dev_conf, 0, sizeof(dev_conf)); + err = rte_event_eth_tx_adapter_caps_get(TEST_DEV_ID, TEST_ETHDEV_ID, + &cap); +diff --git a/dpdk/app/test/test_eventdev.c b/dpdk/app/test/test_eventdev.c +index 71de947ce4..e4e234dc98 100644 +--- a/dpdk/app/test/test_eventdev.c ++++ b/dpdk/app/test/test_eventdev.c +@@ -33,9 +33,15 @@ testsuite_setup(void) + uint8_t count; + count = rte_event_dev_count(); + if (!count) { ++ int ret; ++ + printf("Failed to find a valid event device," +- " testing with event_skeleton device\n"); +- return rte_vdev_init("event_skeleton", NULL); ++ " trying with event_skeleton device\n"); ++ ret = rte_vdev_init("event_skeleton", NULL); ++ if (ret != 0) { ++ printf("No event device, skipping\n"); ++ return TEST_SKIPPED; ++ } + } + return TEST_SUCCESS; + } +diff --git a/dpdk/app/test/test_fbarray.c b/dpdk/app/test/test_fbarray.c +index 26a51e2a3e..09f6907fb1 100644 +--- a/dpdk/app/test/test_fbarray.c ++++ b/dpdk/app/test/test_fbarray.c +@@ -21,23 +21,41 @@ struct fbarray_testsuite_params { + }; + + static struct fbarray_testsuite_params param; ++static struct fbarray_testsuite_params unaligned; + + #define FBARRAY_TEST_ARR_NAME "fbarray_autotest" + #define FBARRAY_TEST_LEN 256 ++#define FBARRAY_UNALIGNED_TEST_ARR_NAME "fbarray_unaligned_autotest" ++#define FBARRAY_UNALIGNED_TEST_LEN 60 + #define FBARRAY_TEST_ELT_SZ (sizeof(int)) + + static int autotest_setup(void) + { +- return rte_fbarray_init(¶m.arr, FBARRAY_TEST_ARR_NAME, ++ int ret; ++ ++ ret = rte_fbarray_init(¶m.arr, FBARRAY_TEST_ARR_NAME, + FBARRAY_TEST_LEN, FBARRAY_TEST_ELT_SZ); ++ if (ret) { ++ printf("Failed to initialize test array\n"); ++ return -1; ++ } ++ ret = rte_fbarray_init(&unaligned.arr, FBARRAY_UNALIGNED_TEST_ARR_NAME, ++ FBARRAY_UNALIGNED_TEST_LEN, FBARRAY_TEST_ELT_SZ); ++ if (ret) { ++ printf("Failed to initialize unaligned test array\n"); ++ rte_fbarray_destroy(¶m.arr); ++ return -1; ++ } ++ return 0; + } + + static void autotest_teardown(void) + { + rte_fbarray_destroy(¶m.arr); ++ rte_fbarray_destroy(&unaligned.arr); + } + +-static int init_array(void) ++static int init_aligned(void) + { + int i; + for (i = param.start; i <= param.end; i++) { +@@ -47,11 +65,35 @@ static int init_array(void) + return 0; + } + +-static void reset_array(void) ++static int init_unaligned(void) ++{ ++ int i; ++ for (i = unaligned.start; i <= unaligned.end; i++) { ++ if (rte_fbarray_set_used(&unaligned.arr, i)) ++ return -1; ++ } ++ return 0; ++} ++ ++static void reset_aligned(void) + { + int i; + for (i = 0; i < FBARRAY_TEST_LEN; i++) + rte_fbarray_set_free(¶m.arr, i); ++ /* reset param as well */ ++ param.start = -1; ++ param.end = -1; ++} ++ ++static void reset_unaligned(void) ++{ ++ int i; ++ for (i = 0; i < FBARRAY_UNALIGNED_TEST_LEN; i++) ++ rte_fbarray_set_free(&unaligned.arr, i); ++ /* reset param as well */ ++ unaligned.start = -1; ++ unaligned.end = -1; ++ + } + + static int first_msk_test_setup(void) +@@ -59,7 +101,7 @@ static int first_msk_test_setup(void) + /* put all within first mask */ + param.start = 3; + param.end = 10; +- return init_array(); ++ return init_aligned(); + } + + static int cross_msk_test_setup(void) +@@ -67,7 +109,7 @@ static int cross_msk_test_setup(void) + /* put all within second and third mask */ + param.start = 70; + param.end = 160; +- return init_array(); ++ return init_aligned(); + } + + static int multi_msk_test_setup(void) +@@ -75,7 +117,7 @@ static int multi_msk_test_setup(void) + /* put all within first and last mask */ + param.start = 3; + param.end = FBARRAY_TEST_LEN - 20; +- return init_array(); ++ return init_aligned(); + } + + static int last_msk_test_setup(void) +@@ -83,7 +125,7 @@ static int last_msk_test_setup(void) + /* put all within last mask */ + param.start = FBARRAY_TEST_LEN - 20; + param.end = FBARRAY_TEST_LEN - 1; +- return init_array(); ++ return init_aligned(); + } + + static int full_msk_test_setup(void) +@@ -91,16 +133,31 @@ static int full_msk_test_setup(void) + /* fill entire mask */ + param.start = 0; + param.end = FBARRAY_TEST_LEN - 1; +- return init_array(); ++ return init_aligned(); + } + +-static int empty_msk_test_setup(void) ++static int lookahead_test_setup(void) + { +- /* do not fill anything in */ +- reset_array(); +- param.start = -1; +- param.end = -1; +- return 0; ++ /* set index 64 as used */ ++ param.start = 64; ++ param.end = 64; ++ return init_aligned(); ++} ++ ++static int lookbehind_test_setup(void) ++{ ++ /* set index 63 as used */ ++ param.start = 63; ++ param.end = 63; ++ return init_aligned(); ++} ++ ++static int unaligned_test_setup(void) ++{ ++ unaligned.start = 0; ++ /* leave one free bit at the end */ ++ unaligned.end = FBARRAY_UNALIGNED_TEST_LEN - 2; ++ return init_unaligned(); + } + + static int test_invalid(void) +@@ -454,7 +511,7 @@ static int test_basic(void) + if (check_free()) + return TEST_FAILED; + +- reset_array(); ++ reset_aligned(); + + return TEST_SUCCESS; + } +@@ -697,6 +754,26 @@ static int test_find(void) + return TEST_SUCCESS; + } + ++static int test_find_unaligned(void) ++{ ++ TEST_ASSERT_EQUAL((int)unaligned.arr.count, unaligned.end - unaligned.start + 1, ++ "Wrong element count\n"); ++ /* ensure space is free before start */ ++ if (ensure_correct(&unaligned.arr, 0, unaligned.start - 1, false)) ++ return TEST_FAILED; ++ /* ensure space is occupied where it's supposed to be */ ++ if (ensure_correct(&unaligned.arr, unaligned.start, unaligned.end, true)) ++ return TEST_FAILED; ++ /* ensure space after end is free as well */ ++ if (ensure_correct(&unaligned.arr, unaligned.end + 1, FBARRAY_UNALIGNED_TEST_LEN - 1, ++ false)) ++ return TEST_FAILED; ++ /* test if find_biggest API's work correctly */ ++ if (test_biggest(&unaligned.arr, unaligned.start, unaligned.end)) ++ return TEST_FAILED; ++ return TEST_SUCCESS; ++} ++ + static int test_empty(void) + { + TEST_ASSERT_EQUAL((int)param.arr.count, 0, "Wrong element count\n"); +@@ -709,6 +786,87 @@ static int test_empty(void) + return TEST_SUCCESS; + } + ++static int test_lookahead(void) ++{ ++ int ret; ++ ++ /* run regular test first */ ++ ret = test_find(); ++ if (ret != TEST_SUCCESS) ++ return ret; ++ ++ /* test if we can find free chunk while not starting with 0 */ ++ TEST_ASSERT_EQUAL(rte_fbarray_find_next_n_free(¶m.arr, 1, param.start), ++ param.start + 1, "Free chunk index is wrong\n"); ++ return TEST_SUCCESS; ++} ++ ++static int test_lookbehind(void) ++{ ++ int ret, free_len = 2; ++ ++ /* run regular test first */ ++ ret = test_find(); ++ if (ret != TEST_SUCCESS) ++ return ret; ++ ++ /* test if we can find free chunk while crossing mask boundary */ ++ TEST_ASSERT_EQUAL(rte_fbarray_find_prev_n_free(¶m.arr, param.start + 1, free_len), ++ param.start - free_len, "Free chunk index is wrong\n"); ++ return TEST_SUCCESS; ++} ++ ++static int test_lookahead_mask(void) ++{ ++ /* ++ * There is a certain type of lookahead behavior we want to test here, ++ * namely masking of bits that were scanned with lookahead but that we ++ * know do not match our criteria. This is achieved in following steps: ++ * ++ * 0. Look for a big enough chunk of free space (say, 62 elements) ++ * 1. Trigger lookahead by breaking a run somewhere inside mask 0 ++ * (indices 0-63) ++ * 2. Fail lookahead by breaking the run somewhere inside mask 1 ++ * (indices 64-127) ++ * 3. Ensure that we can still find free space in mask 1 afterwards ++ */ ++ ++ /* break run on first mask */ ++ rte_fbarray_set_used(¶m.arr, 61); ++ /* break run on second mask */ ++ rte_fbarray_set_used(¶m.arr, 70); ++ ++ /* we expect to find free space at 71 */ ++ TEST_ASSERT_EQUAL(rte_fbarray_find_next_n_free(¶m.arr, 0, 62), ++ 71, "Free chunk index is wrong\n"); ++ return TEST_SUCCESS; ++} ++ ++static int test_lookbehind_mask(void) ++{ ++ /* ++ * There is a certain type of lookbehind behavior we want to test here, ++ * namely masking of bits that were scanned with lookbehind but that we ++ * know do not match our criteria. This is achieved in two steps: ++ * ++ * 0. Look for a big enough chunk of free space (say, 62 elements) ++ * 1. Trigger lookbehind by breaking a run somewhere inside mask 2 ++ * (indices 128-191) ++ * 2. Fail lookbehind by breaking the run somewhere inside mask 1 ++ * (indices 64-127) ++ * 3. Ensure that we can still find free space in mask 1 afterwards ++ */ ++ ++ /* break run on mask 2 */ ++ rte_fbarray_set_used(¶m.arr, 130); ++ /* break run on mask 1 */ ++ rte_fbarray_set_used(¶m.arr, 70); ++ ++ /* start from 190, we expect to find free space at 8 */ ++ TEST_ASSERT_EQUAL(rte_fbarray_find_prev_n_free(¶m.arr, 190, 62), ++ 8, "Free chunk index is wrong\n"); ++ return TEST_SUCCESS; ++} + + static struct unit_test_suite fbarray_test_suite = { + .suite_name = "fbarray autotest", +@@ -717,12 +875,19 @@ static struct unit_test_suite fbarray_test_suite = { + .unit_test_cases = { + TEST_CASE(test_invalid), + TEST_CASE(test_basic), +- TEST_CASE_ST(first_msk_test_setup, reset_array, test_find), +- TEST_CASE_ST(cross_msk_test_setup, reset_array, test_find), +- TEST_CASE_ST(multi_msk_test_setup, reset_array, test_find), +- TEST_CASE_ST(last_msk_test_setup, reset_array, test_find), +- TEST_CASE_ST(full_msk_test_setup, reset_array, test_find), +- TEST_CASE_ST(empty_msk_test_setup, reset_array, test_empty), ++ TEST_CASE_ST(first_msk_test_setup, reset_aligned, test_find), ++ TEST_CASE_ST(cross_msk_test_setup, reset_aligned, test_find), ++ TEST_CASE_ST(multi_msk_test_setup, reset_aligned, test_find), ++ TEST_CASE_ST(last_msk_test_setup, reset_aligned, test_find), ++ TEST_CASE_ST(full_msk_test_setup, reset_aligned, test_find), ++ /* empty test does not need setup */ ++ TEST_CASE_ST(NULL, reset_aligned, test_empty), ++ TEST_CASE_ST(lookahead_test_setup, reset_aligned, test_lookahead), ++ TEST_CASE_ST(lookbehind_test_setup, reset_aligned, test_lookbehind), ++ /* setup for these tests is more complex so do it in test func */ ++ TEST_CASE_ST(NULL, reset_aligned, test_lookahead_mask), ++ TEST_CASE_ST(NULL, reset_aligned, test_lookbehind_mask), ++ TEST_CASE_ST(unaligned_test_setup, reset_unaligned, test_find_unaligned), + TEST_CASES_END() + } + }; +diff --git a/dpdk/app/test/test_graph.c b/dpdk/app/test/test_graph.c +index 3dd017ebfb..eb4f9a61e3 100644 +--- a/dpdk/app/test/test_graph.c ++++ b/dpdk/app/test/test_graph.c +@@ -695,6 +695,77 @@ test_graph_clone(void) + return ret; + } + ++static int ++test_graph_id_collisions(void) ++{ ++ static const char *node_patterns[] = {"test_node_source1", "test_node00"}; ++ struct rte_graph_param gconf = { ++ .socket_id = SOCKET_ID_ANY, ++ .nb_node_patterns = 2, ++ .node_patterns = node_patterns, ++ }; ++ rte_graph_t g1, g2, g3, g4; ++ ++ g1 = rte_graph_create("worker1", &gconf); ++ if (g1 == RTE_GRAPH_ID_INVALID) { ++ printf("Graph 1 creation failed with error = %d\n", rte_errno); ++ return -1; ++ } ++ g2 = rte_graph_create("worker2", &gconf); ++ if (g2 == RTE_GRAPH_ID_INVALID) { ++ printf("Graph 2 creation failed with error = %d\n", rte_errno); ++ return -1; ++ } ++ g3 = rte_graph_create("worker3", &gconf); ++ if (g3 == RTE_GRAPH_ID_INVALID) { ++ printf("Graph 3 creation failed with error = %d\n", rte_errno); ++ return -1; ++ } ++ if (g1 == g2 || g2 == g3 || g1 == g3) { ++ printf("Graph ids should be different\n"); ++ return -1; ++ } ++ if (rte_graph_destroy(g2) < 0) { ++ printf("Graph 2 suppression failed\n"); ++ return -1; ++ } ++ g4 = rte_graph_create("worker4", &gconf); ++ if (g4 == RTE_GRAPH_ID_INVALID) { ++ printf("Graph 4 creation failed with error = %d\n", rte_errno); ++ return -1; ++ } ++ if (g1 == g3 || g1 == g4 || g3 == g4) { ++ printf("Graph ids should be different\n"); ++ return -1; ++ } ++ g2 = rte_graph_clone(g1, "worker2", &gconf); ++ if (g2 == RTE_GRAPH_ID_INVALID) { ++ printf("Graph 4 creation failed with error = %d\n", rte_errno); ++ return -1; ++ } ++ if (g1 == g2 || g1 == g3 || g1 == g4 || g2 == g3 || g2 == g4 || g3 == g4) { ++ printf("Graph ids should be different\n"); ++ return -1; ++ } ++ if (rte_graph_destroy(g1) < 0) { ++ printf("Graph 1 suppression failed\n"); ++ return -1; ++ } ++ if (rte_graph_destroy(g2) < 0) { ++ printf("Graph 2 suppression failed\n"); ++ return -1; ++ } ++ if (rte_graph_destroy(g3) < 0) { ++ printf("Graph 3 suppression failed\n"); ++ return -1; ++ } ++ if (rte_graph_destroy(g4) < 0) { ++ printf("Graph 4 suppression failed\n"); ++ return -1; ++ } ++ return 0; ++} ++ + static int + test_graph_model_mcore_dispatch_node_lcore_affinity_set(void) + { +@@ -976,6 +1047,7 @@ static struct unit_test_suite graph_testsuite = { + TEST_CASE(test_lookup_functions), + TEST_CASE(test_create_graph), + TEST_CASE(test_graph_clone), ++ TEST_CASE(test_graph_id_collisions), + TEST_CASE(test_graph_model_mcore_dispatch_node_lcore_affinity_set), + TEST_CASE(test_graph_model_mcore_dispatch_core_bind_unbind), + TEST_CASE(test_graph_worker_model_set_get), +diff --git a/dpdk/app/test/test_mbuf.c b/dpdk/app/test/test_mbuf.c +index d7393df7eb..a39288a5f8 100644 +--- a/dpdk/app/test/test_mbuf.c ++++ b/dpdk/app/test/test_mbuf.c +@@ -2345,16 +2345,13 @@ test_pktmbuf_ext_shinfo_init_helper(struct rte_mempool *pktmbuf_pool) + GOTO_FAIL("%s: External buffer is not attached to mbuf\n", + __func__); + +- /* allocate one more mbuf */ ++ /* allocate one more mbuf, it is attached to the same external buffer */ + clone = rte_pktmbuf_clone(m, pktmbuf_pool); + if (clone == NULL) + GOTO_FAIL("%s: mbuf clone allocation failed!\n", __func__); + if (rte_pktmbuf_pkt_len(clone) != 0) + GOTO_FAIL("%s: Bad packet length\n", __func__); + +- /* attach the same external buffer to the cloned mbuf */ +- rte_pktmbuf_attach_extbuf(clone, ext_buf_addr, buf_iova, buf_len, +- ret_shinfo); + if (clone->ol_flags != RTE_MBUF_F_EXTERNAL) + GOTO_FAIL("%s: External buffer is not attached to mbuf\n", + __func__); +diff --git a/dpdk/app/test/test_power.c b/dpdk/app/test/test_power.c +index f1e80299d3..403adc22d6 100644 +--- a/dpdk/app/test/test_power.c ++++ b/dpdk/app/test/test_power.c +@@ -143,7 +143,7 @@ test_power(void) + /* Test setting a valid environment */ + ret = rte_power_set_env(envs[i]); + if (ret != 0) { +- printf("Unexpectedly unsucceeded on setting a valid environment\n"); ++ printf("Unexpectedly unsuccessful on setting a valid environment\n"); + return -1; + } + +diff --git a/dpdk/app/test/test_power_intel_uncore.c b/dpdk/app/test/test_power_intel_uncore.c +index 80b45ce46e..049658627d 100644 +--- a/dpdk/app/test/test_power_intel_uncore.c ++++ b/dpdk/app/test/test_power_intel_uncore.c +@@ -17,14 +17,12 @@ test_power_intel_uncore(void) + #include + #include + +-#define MAX_UNCORE_FREQS 32 +- + #define VALID_PKG 0 + #define VALID_DIE 0 + #define INVALID_PKG (rte_power_uncore_get_num_pkgs() + 1) + #define INVALID_DIE (rte_power_uncore_get_num_dies(VALID_PKG) + 1) + #define VALID_INDEX 1 +-#define INVALID_INDEX (MAX_UNCORE_FREQS + 1) ++#define INVALID_INDEX (RTE_MAX_UNCORE_FREQS + 1) + + static int check_power_uncore_init(void) + { +diff --git a/dpdk/buildtools/dpdk-cmdline-gen.py b/dpdk/buildtools/dpdk-cmdline-gen.py +index 49b03bee4a..30d32ac183 100755 +--- a/dpdk/buildtools/dpdk-cmdline-gen.py ++++ b/dpdk/buildtools/dpdk-cmdline-gen.py +@@ -71,8 +71,8 @@ def process_command(lineno, tokens, comment): + elif t_type in ["IP", "IP_ADDR", "IPADDR"]: + result_struct.append(f"\tcmdline_ipaddr_t {t_name};") + initializers.append( +- f"cmdline_parse_token_ipaddr_t cmd_{name}_{t_name}_tok =\n" +- f"\tTOKEN_IPV4_INITIALIZER(struct cmd_{name}_result, {t_name});" ++ f"static cmdline_parse_token_ipaddr_t cmd_{name}_{t_name}_tok =\n" ++ f"\tTOKEN_IPADDR_INITIALIZER(struct cmd_{name}_result, {t_name});" + ) + elif t_type.startswith("(") and t_type.endswith(")"): + result_struct.append(f"\tcmdline_fixed_string_t {t_name};") +diff --git a/dpdk/buildtools/get-test-suites.py b/dpdk/buildtools/get-test-suites.py +index 574c233aa8..c61f6a273f 100644 +--- a/dpdk/buildtools/get-test-suites.py ++++ b/dpdk/buildtools/get-test-suites.py +@@ -19,7 +19,7 @@ def get_fast_test_params(test_name, ln): + return f":{nohuge.strip().lower()}:{asan.strip().lower()}" + + for fname in input_list: +- with open(fname) as f: ++ with open(fname, "r", encoding="utf-8") as f: + contents = [ln.strip() for ln in f.readlines()] + test_lines = [ln for ln in contents if test_def_regex.match(ln)] + non_suite_tests.extend([non_suite_regex.match(ln).group(1) +diff --git a/dpdk/buildtools/map-list-symbol.sh b/dpdk/buildtools/map-list-symbol.sh +index a834399816..878c5880df 100755 +--- a/dpdk/buildtools/map-list-symbol.sh ++++ b/dpdk/buildtools/map-list-symbol.sh +@@ -5,6 +5,7 @@ + section=all + symbol=all + quiet= ++version= + + while getopts 'S:s:qV:' name; do + case $name in +diff --git a/dpdk/buildtools/meson.build b/dpdk/buildtools/meson.build +index 72447b60a0..3adf34e1a8 100644 +--- a/dpdk/buildtools/meson.build ++++ b/dpdk/buildtools/meson.build +@@ -5,7 +5,11 @@ pkgconf = find_program('pkg-config', 'pkgconf', required: false) + check_symbols = find_program('check-symbols.sh') + ldflags_ibverbs_static = find_program('options-ibverbs-static.sh') + +-python3 = import('python').find_installation(required: false) ++python3_required_modules = [] ++if host_machine.system() != 'windows' ++ python3_required_modules = ['elftools'] ++endif ++python3 = import('python').find_installation('python3', required: false, modules: python3_required_modules) + if python3.found() + py3 = [python3] + else +@@ -45,16 +49,3 @@ else + pmdinfogen += 'elf' + endif + +-# TODO: starting from Meson 0.51.0 use +-# python3 = import('python').find_installation('python', +-# modules : python3_required_modules) +-python3_required_modules = [] +-if host_machine.system() != 'windows' +- python3_required_modules = ['elftools'] +-endif +-foreach module : python3_required_modules +- script = 'import importlib.util; import sys; exit(importlib.util.find_spec("@0@") is None)' +- if run_command(py3, '-c', script.format(module), check: false).returncode() != 0 +- error('missing python module: @0@'.format(module)) +- endif +-endforeach +diff --git a/dpdk/buildtools/pmdinfogen.py b/dpdk/buildtools/pmdinfogen.py +index 2a44f17bda..dfb89500c0 100755 +--- a/dpdk/buildtools/pmdinfogen.py ++++ b/dpdk/buildtools/pmdinfogen.py +@@ -6,6 +6,7 @@ + import argparse + import ctypes + import json ++import re + import sys + import tempfile + +@@ -66,11 +67,11 @@ class ELFImage: + return [symbol] + return None + +- def find_by_prefix(self, prefix): +- prefix = prefix.encode("utf-8") if self._legacy_elftools else prefix ++ def find_by_pattern(self, pattern): ++ pattern = pattern.encode("utf-8") if self._legacy_elftools else pattern + for i in range(self._symtab.num_symbols()): + symbol = self._symtab.get_symbol(i) +- if symbol.name.startswith(prefix): ++ if re.match(pattern, symbol.name): + yield ELFSymbol(self._image, symbol) + + +@@ -97,9 +98,9 @@ class COFFImage: + def is_big_endian(self): + return False + +- def find_by_prefix(self, prefix): ++ def find_by_pattern(self, pattern): + for symbol in self._image.symbols: +- if symbol.name.startswith(prefix): ++ if re.match(pattern, symbol.name): + yield COFFSymbol(self._image, symbol) + + def find_by_name(self, name): +@@ -199,7 +200,7 @@ class Driver: + + def load_drivers(image): + drivers = [] +- for symbol in image.find_by_prefix("this_pmd_name"): ++ for symbol in image.find_by_pattern("^this_pmd_name[0-9]+$"): + drivers.append(Driver.load(image, symbol)) + return drivers + +diff --git a/dpdk/buildtools/subproject/meson.build b/dpdk/buildtools/subproject/meson.build +index 3192efaa40..203c5d36c6 100644 +--- a/dpdk/buildtools/subproject/meson.build ++++ b/dpdk/buildtools/subproject/meson.build +@@ -2,18 +2,23 @@ + # Copyright(c) 2022 Intel Corporation + + message('DPDK subproject linking: ' + get_option('default_library')) ++subproject_cflags = ['-include', 'rte_config.h'] + machine_args ++if is_freebsd ++ subproject_cflags += ['-D__BSD_VISIBLE'] ++endif + if get_option('default_library') == 'static' + dpdk_dep = declare_dependency( + version: meson.project_version(), + dependencies: dpdk_static_lib_deps, ++ compile_args: subproject_cflags, + # static library deps in DPDK build don't include "link_with" parameters, + # so explicitly link-in both libs and drivers +- link_with: dpdk_static_libraries, +- link_whole: dpdk_drivers, ++ link_whole: dpdk_static_libraries + dpdk_drivers, + link_args: dpdk_extra_ldflags) + else + dpdk_dep = declare_dependency( + version: meson.project_version(), ++ compile_args: subproject_cflags, + # shared library deps include all necessary linking parameters + dependencies: dpdk_shared_lib_deps) + endif +diff --git a/dpdk/config/arm/arm32_armv8_linux_gcc b/dpdk/config/arm/arm32_armv8_linux_gcc +index 269a60ba19..abcb182b16 100644 +--- a/dpdk/config/arm/arm32_armv8_linux_gcc ++++ b/dpdk/config/arm/arm32_armv8_linux_gcc +@@ -4,6 +4,7 @@ cpp = ['ccache', 'arm-linux-gnueabihf-g++'] + ar = 'arm-linux-gnueabihf-gcc-ar' + strip = 'arm-linux-gnueabihf-strip' + pkgconfig = 'arm-linux-gnueabihf-pkg-config' ++pkg-config = 'arm-linux-gnueabihf-pkg-config' + pcap-config = '' + + [host_machine] +diff --git a/dpdk/config/arm/arm64_altra_linux_gcc b/dpdk/config/arm/arm64_altra_linux_gcc +index ce0667ebe2..769503d936 100644 +--- a/dpdk/config/arm/arm64_altra_linux_gcc ++++ b/dpdk/config/arm/arm64_altra_linux_gcc +@@ -4,6 +4,7 @@ cpp = ['ccache', 'aarch64-linux-gnu-g++'] + ar = 'aarch64-linux-gnu-gcc-ar' + strip = 'aarch64-linux-gnu-strip' + pkgconfig = 'aarch64-linux-gnu-pkg-config' ++pkg-config = 'aarch64-linux-gnu-pkg-config' + pcap-config = '' + + [host_machine] +diff --git a/dpdk/config/arm/arm64_ampereone_linux_gcc b/dpdk/config/arm/arm64_ampereone_linux_gcc +index 8964432a74..bb6acfc4cf 100644 +--- a/dpdk/config/arm/arm64_ampereone_linux_gcc ++++ b/dpdk/config/arm/arm64_ampereone_linux_gcc +@@ -4,6 +4,7 @@ cpp = ['ccache', 'aarch64-linux-gnu-g++'] + ar = 'aarch64-linux-gnu-gcc-ar' + strip = 'aarch64-linux-gnu-strip' + pkgconfig = 'aarch64-linux-gnu-pkg-config' ++pkg-config = 'aarch64-linux-gnu-pkg-config' + pcap-config = '' + + [host_machine] +diff --git a/dpdk/config/arm/arm64_armada_linux_gcc b/dpdk/config/arm/arm64_armada_linux_gcc +index 635b4946a3..8f36d895da 100644 +--- a/dpdk/config/arm/arm64_armada_linux_gcc ++++ b/dpdk/config/arm/arm64_armada_linux_gcc +@@ -5,6 +5,7 @@ ar = 'aarch64-linux-gnu-ar' + as = 'aarch64-linux-gnu-as' + strip = 'aarch64-linux-gnu-strip' + pkgconfig = 'aarch64-linux-gnu-pkg-config' ++pkg-config = 'aarch64-linux-gnu-pkg-config' + pcap-config = '' + + [host_machine] +diff --git a/dpdk/config/arm/arm64_armv8_linux_clang_ubuntu b/dpdk/config/arm/arm64_armv8_linux_clang_ubuntu +index 86ae43937b..e34fabebe5 100644 +--- a/dpdk/config/arm/arm64_armv8_linux_clang_ubuntu ++++ b/dpdk/config/arm/arm64_armv8_linux_clang_ubuntu +@@ -6,6 +6,7 @@ strip = 'llvm-strip' + llvm-config = 'llvm-config' + pcap-config = 'llvm-config' + pkgconfig = 'aarch64-linux-gnu-pkg-config' ++pkg-config = 'aarch64-linux-gnu-pkg-config' + + [host_machine] + system = 'linux' +diff --git a/dpdk/config/arm/arm64_armv8_linux_gcc b/dpdk/config/arm/arm64_armv8_linux_gcc +index 529694b49d..a9b136cf48 100644 +--- a/dpdk/config/arm/arm64_armv8_linux_gcc ++++ b/dpdk/config/arm/arm64_armv8_linux_gcc +@@ -4,6 +4,7 @@ cpp = ['ccache', 'aarch64-linux-gnu-g++'] + ar = 'aarch64-linux-gnu-gcc-ar' + strip = 'aarch64-linux-gnu-strip' + pkgconfig = 'aarch64-linux-gnu-pkg-config' ++pkg-config = 'aarch64-linux-gnu-pkg-config' + pcap-config = '' + + [host_machine] +diff --git a/dpdk/config/arm/arm64_bluefield3_linux_gcc b/dpdk/config/arm/arm64_bluefield3_linux_gcc +index 775cf5883d..d900e72a6d 100644 +--- a/dpdk/config/arm/arm64_bluefield3_linux_gcc ++++ b/dpdk/config/arm/arm64_bluefield3_linux_gcc +@@ -4,6 +4,7 @@ cpp = ['ccache', 'aarch64-linux-gnu-g++'] + ar = 'aarch64-linux-gnu-gcc-ar' + strip = 'aarch64-linux-gnu-strip' + pkgconfig = 'aarch64-linux-gnu-pkg-config' ++pkg-config = 'aarch64-linux-gnu-pkg-config' + pcap-config = '' + + [host_machine] +diff --git a/dpdk/config/arm/arm64_bluefield_linux_gcc b/dpdk/config/arm/arm64_bluefield_linux_gcc +index 1286227915..bcffb6534b 100644 +--- a/dpdk/config/arm/arm64_bluefield_linux_gcc ++++ b/dpdk/config/arm/arm64_bluefield_linux_gcc +@@ -4,6 +4,7 @@ cpp = ['ccache', 'aarch64-linux-gnu-g++'] + ar = 'aarch64-linux-gnu-gcc-ar' + strip = 'aarch64-linux-gnu-strip' + pkgconfig = 'aarch64-linux-gnu-pkg-config' ++pkg-config = 'aarch64-linux-gnu-pkg-config' + pcap-config = '' + + [host_machine] +diff --git a/dpdk/config/arm/arm64_cdx_linux_gcc b/dpdk/config/arm/arm64_cdx_linux_gcc +index 8e6d619dae..bf494b776e 100644 +--- a/dpdk/config/arm/arm64_cdx_linux_gcc ++++ b/dpdk/config/arm/arm64_cdx_linux_gcc +@@ -5,6 +5,7 @@ ar = 'aarch64-linux-gnu-ar' + as = 'aarch64-linux-gnu-as' + strip = 'aarch64-linux-gnu-strip' + pkgconfig = 'aarch64-linux-gnu-pkg-config' ++pkg-config = 'aarch64-linux-gnu-pkg-config' + pcap-config = '' + + [host_machine] +diff --git a/dpdk/config/arm/arm64_centriq2400_linux_gcc b/dpdk/config/arm/arm64_centriq2400_linux_gcc +index bc8737e072..33cb5ef675 100644 +--- a/dpdk/config/arm/arm64_centriq2400_linux_gcc ++++ b/dpdk/config/arm/arm64_centriq2400_linux_gcc +@@ -4,6 +4,7 @@ cpp = ['ccache', 'aarch64-linux-gnu-g++'] + ar = 'aarch64-linux-gnu-gcc-ar' + strip = 'aarch64-linux-gnu-strip' + pkgconfig = 'aarch64-linux-gnu-pkg-config' ++pkg-config = 'aarch64-linux-gnu-pkg-config' + pcap-config = '' + + [host_machine] +diff --git a/dpdk/config/arm/arm64_cn10k_linux_gcc b/dpdk/config/arm/arm64_cn10k_linux_gcc +index fa904af5d0..63fcca9cbe 100644 +--- a/dpdk/config/arm/arm64_cn10k_linux_gcc ++++ b/dpdk/config/arm/arm64_cn10k_linux_gcc +@@ -4,6 +4,7 @@ cpp = ['ccache', 'aarch64-linux-gnu-g++'] + ar = 'aarch64-linux-gnu-gcc-ar' + strip = 'aarch64-linux-gnu-strip' + pkgconfig = 'aarch64-linux-gnu-pkg-config' ++pkg-config = 'aarch64-linux-gnu-pkg-config' + pcap-config = '' + cmake = 'cmake' + +diff --git a/dpdk/config/arm/arm64_cn9k_linux_gcc b/dpdk/config/arm/arm64_cn9k_linux_gcc +index 646ce4b5d3..fa4618e370 100644 +--- a/dpdk/config/arm/arm64_cn9k_linux_gcc ++++ b/dpdk/config/arm/arm64_cn9k_linux_gcc +@@ -4,6 +4,7 @@ cpp = ['ccache', 'aarch64-linux-gnu-g++'] + ar = 'aarch64-linux-gnu-gcc-ar' + strip = 'aarch64-linux-gnu-strip' + pkgconfig = 'aarch64-linux-gnu-pkg-config' ++pkg-config = 'aarch64-linux-gnu-pkg-config' + pcap-config = '' + cmake = 'cmake' + +diff --git a/dpdk/config/arm/arm64_dpaa_linux_gcc b/dpdk/config/arm/arm64_dpaa_linux_gcc +index 8465b5097b..bf0eab18e2 100644 +--- a/dpdk/config/arm/arm64_dpaa_linux_gcc ++++ b/dpdk/config/arm/arm64_dpaa_linux_gcc +@@ -5,6 +5,7 @@ ar = 'aarch64-linux-gnu-ar' + as = 'aarch64-linux-gnu-as' + strip = 'aarch64-linux-gnu-strip' + pkgconfig = 'aarch64-linux-gnu-pkg-config' ++pkg-config = 'aarch64-linux-gnu-pkg-config' + pcap-config = '' + + [host_machine] +diff --git a/dpdk/config/arm/arm64_emag_linux_gcc b/dpdk/config/arm/arm64_emag_linux_gcc +index 248169ed68..3e8c15f8a4 100644 +--- a/dpdk/config/arm/arm64_emag_linux_gcc ++++ b/dpdk/config/arm/arm64_emag_linux_gcc +@@ -4,6 +4,7 @@ cpp = ['ccache', 'aarch64-linux-gnu-g++'] + ar = 'aarch64-linux-gnu-gcc-ar' + strip = 'aarch64-linux-gnu-strip' + pkgconfig = 'aarch64-linux-gnu-pkg-config' ++pkg-config = 'aarch64-linux-gnu-pkg-config' + pcap-config = '' + + [host_machine] +diff --git a/dpdk/config/arm/arm64_ft2000plus_linux_gcc b/dpdk/config/arm/arm64_ft2000plus_linux_gcc +index ae9f779056..4ecb251d3f 100644 +--- a/dpdk/config/arm/arm64_ft2000plus_linux_gcc ++++ b/dpdk/config/arm/arm64_ft2000plus_linux_gcc +@@ -4,6 +4,7 @@ cpp = ['ccache', 'aarch64-linux-gnu-g++'] + ar = 'aarch64-linux-gnu-gcc-ar' + strip = 'aarch64-linux-gnu-strip' + pkgconfig = 'aarch64-linux-gnu-pkg-config' ++pkg-config = 'aarch64-linux-gnu-pkg-config' + pcap-config = '' + + [host_machine] +diff --git a/dpdk/config/arm/arm64_graviton2_linux_gcc b/dpdk/config/arm/arm64_graviton2_linux_gcc +index fdb298bb11..b5f681bc27 100644 +--- a/dpdk/config/arm/arm64_graviton2_linux_gcc ++++ b/dpdk/config/arm/arm64_graviton2_linux_gcc +@@ -4,6 +4,7 @@ cpp = ['ccache', 'aarch64-linux-gnu-g++'] + ar = 'aarch64-linux-gnu-gcc-ar' + strip = 'aarch64-linux-gnu-strip' + pkgconfig = 'aarch64-linux-gnu-pkg-config' ++pkg-config = 'aarch64-linux-gnu-pkg-config' + pcap-config = '' + + [host_machine] +diff --git a/dpdk/config/arm/arm64_graviton3_linux_gcc b/dpdk/config/arm/arm64_graviton3_linux_gcc +index 19b422075d..77b5168836 100644 +--- a/dpdk/config/arm/arm64_graviton3_linux_gcc ++++ b/dpdk/config/arm/arm64_graviton3_linux_gcc +@@ -4,6 +4,7 @@ cpp = ['ccache', 'aarch64-linux-gnu-g++'] + ar = 'aarch64-linux-gnu-gcc-ar' + strip = 'aarch64-linux-gnu-strip' + pkgconfig = 'aarch64-linux-gnu-pkg-config' ++pkg-config = 'aarch64-linux-gnu-pkg-config' + pcap-config = '' + + [host_machine] +diff --git a/dpdk/config/arm/arm64_hip10_linux_gcc b/dpdk/config/arm/arm64_hip10_linux_gcc +index 2943e4abbf..85bcd05887 100644 +--- a/dpdk/config/arm/arm64_hip10_linux_gcc ++++ b/dpdk/config/arm/arm64_hip10_linux_gcc +@@ -4,6 +4,7 @@ cpp = ['ccache', 'aarch64-linux-gnu-g++'] + ar = 'aarch64-linux-gnu-gcc-ar' + strip = 'aarch64-linux-gnu-strip' + pkgconfig = 'aarch64-linux-gnu-pkg-config' ++pkg-config = 'aarch64-linux-gnu-pkg-config' + pcap-config = '' + + [host_machine] +diff --git a/dpdk/config/arm/arm64_kunpeng920_linux_gcc b/dpdk/config/arm/arm64_kunpeng920_linux_gcc +index 193fb48a61..6e7d8ac667 100644 +--- a/dpdk/config/arm/arm64_kunpeng920_linux_gcc ++++ b/dpdk/config/arm/arm64_kunpeng920_linux_gcc +@@ -4,6 +4,7 @@ cpp = ['ccache', 'aarch64-linux-gnu-g++'] + ar = 'aarch64-linux-gnu-gcc-ar' + strip = 'aarch64-linux-gnu-strip' + pkgconfig = 'aarch64-linux-gnu-pkg-config' ++pkg-config = 'aarch64-linux-gnu-pkg-config' + pcap-config = '' + + [host_machine] +diff --git a/dpdk/config/arm/arm64_kunpeng930_linux_gcc b/dpdk/config/arm/arm64_kunpeng930_linux_gcc +index e4281ceb4f..61fe482a2e 100644 +--- a/dpdk/config/arm/arm64_kunpeng930_linux_gcc ++++ b/dpdk/config/arm/arm64_kunpeng930_linux_gcc +@@ -4,6 +4,7 @@ cpp = ['ccache', 'aarch64-linux-gnu-g++'] + ar = 'aarch64-linux-gnu-gcc-ar' + strip = 'aarch64-linux-gnu-strip' + pkgconfig = 'aarch64-linux-gnu-pkg-config' ++pkg-config = 'aarch64-linux-gnu-pkg-config' + pcap-config = '' + + [host_machine] +diff --git a/dpdk/config/arm/arm64_n1sdp_linux_gcc b/dpdk/config/arm/arm64_n1sdp_linux_gcc +index 2806a4241b..c3c12098d8 100644 +--- a/dpdk/config/arm/arm64_n1sdp_linux_gcc ++++ b/dpdk/config/arm/arm64_n1sdp_linux_gcc +@@ -4,6 +4,7 @@ cpp = ['ccache', 'aarch64-linux-gnu-g++'] + ar = 'aarch64-linux-gnu-gcc-ar' + strip = 'aarch64-linux-gnu-strip' + pkgconfig = 'aarch64-linux-gnu-pkg-config' ++pkg-config = 'aarch64-linux-gnu-pkg-config' + pcap-config = '' + + [host_machine] +diff --git a/dpdk/config/arm/arm64_n2_linux_gcc b/dpdk/config/arm/arm64_n2_linux_gcc +index 7404bd197b..89200861c4 100644 +--- a/dpdk/config/arm/arm64_n2_linux_gcc ++++ b/dpdk/config/arm/arm64_n2_linux_gcc +@@ -4,6 +4,7 @@ cpp = ['ccache', 'aarch64-linux-gnu-g++'] + ar = 'aarch64-linux-gnu-gcc-ar' + strip = 'aarch64-linux-gnu-strip' + pkgconfig = 'aarch64-linux-gnu-pkg-config' ++pkg-config = 'aarch64-linux-gnu-pkg-config' + pcap-config = '' + + [host_machine] +diff --git a/dpdk/config/arm/arm64_stingray_linux_gcc b/dpdk/config/arm/arm64_stingray_linux_gcc +index 08148b5c3d..358f3b43d3 100644 +--- a/dpdk/config/arm/arm64_stingray_linux_gcc ++++ b/dpdk/config/arm/arm64_stingray_linux_gcc +@@ -4,6 +4,7 @@ cpp = ['ccache', 'aarch64-linux-gnu-g++'] + ar = 'aarch64-linux-gnu-gcc-ar' + strip = 'aarch64-linux-gnu-strip' + pkgconfig = 'aarch64-linux-gnu-pkg-config' ++pkg-config = 'aarch64-linux-gnu-pkg-config' + pcap-config = '' + + [host_machine] +diff --git a/dpdk/config/arm/arm64_thunderx2_linux_gcc b/dpdk/config/arm/arm64_thunderx2_linux_gcc +index 32ae938e95..124a97da01 100644 +--- a/dpdk/config/arm/arm64_thunderx2_linux_gcc ++++ b/dpdk/config/arm/arm64_thunderx2_linux_gcc +@@ -4,6 +4,7 @@ cpp = ['ccache', 'aarch64-linux-gnu-g++'] + ar = 'aarch64-linux-gnu-gcc-ar' + strip = 'aarch64-linux-gnu-strip' + pkgconfig = 'aarch64-linux-gnu-pkg-config' ++pkg-config = 'aarch64-linux-gnu-pkg-config' + pcap-config = '' + + [host_machine] +diff --git a/dpdk/config/arm/arm64_thunderxt83_linux_gcc b/dpdk/config/arm/arm64_thunderxt83_linux_gcc +index e9d9e62d44..433c3cb4e3 100644 +--- a/dpdk/config/arm/arm64_thunderxt83_linux_gcc ++++ b/dpdk/config/arm/arm64_thunderxt83_linux_gcc +@@ -4,6 +4,7 @@ cpp = ['ccache', 'aarch64-linux-gnu-g++'] + ar = 'aarch64-linux-gnu-gcc-ar' + strip = 'aarch64-linux-gnu-strip' + pkgconfig = 'aarch64-linux-gnu-pkg-config' ++pkg-config = 'aarch64-linux-gnu-pkg-config' + pcap-config = '' + + [host_machine] +diff --git a/dpdk/config/arm/arm64_thunderxt88_linux_gcc b/dpdk/config/arm/arm64_thunderxt88_linux_gcc +index c6e5a5656a..81975aea14 100644 +--- a/dpdk/config/arm/arm64_thunderxt88_linux_gcc ++++ b/dpdk/config/arm/arm64_thunderxt88_linux_gcc +@@ -4,6 +4,7 @@ cpp = ['ccache', 'aarch64-linux-gnu-g++'] + ar = 'aarch64-linux-gnu-gcc-ar' + strip = 'aarch64-linux-gnu-strip' + pkgconfig = 'aarch64-linux-gnu-pkg-config' ++pkg-config = 'aarch64-linux-gnu-pkg-config' + pcap-config = '' + + [host_machine] +diff --git a/dpdk/config/arm/arm64_tys2500_linux_gcc b/dpdk/config/arm/arm64_tys2500_linux_gcc +index fce85fb0d8..24e6539a15 100644 +--- a/dpdk/config/arm/arm64_tys2500_linux_gcc ++++ b/dpdk/config/arm/arm64_tys2500_linux_gcc +@@ -4,6 +4,7 @@ cpp = ['ccache', 'aarch64-linux-gnu-g++'] + ar = 'aarch64-linux-gnu-gcc-ar' + strip = 'aarch64-linux-gnu-strip' + pkgconfig = 'aarch64-linux-gnu-pkg-config' ++pkg-config = 'aarch64-linux-gnu-pkg-config' + pcap-config = '' + + [host_machine] +diff --git a/dpdk/config/meson.build b/dpdk/config/meson.build +index a9ccd56deb..898b719929 100644 +--- a/dpdk/config/meson.build ++++ b/dpdk/config/meson.build +@@ -121,13 +121,14 @@ else + cpu_instruction_set = 'generic' + endif + endif ++ if platform == 'native' ++ if cpu_instruction_set == 'auto' ++ cpu_instruction_set = 'native' ++ endif ++ endif + endif + +-if platform == 'native' +- if cpu_instruction_set == 'auto' +- cpu_instruction_set = 'native' +- endif +-elif platform == 'generic' ++if platform == 'generic' + if cpu_instruction_set == 'auto' + cpu_instruction_set = 'generic' + endif +@@ -188,6 +189,9 @@ dpdk_conf.set('RTE_ARCH_32', cc.sizeof('void *') == 4) + + if not is_windows + add_project_link_arguments('-Wl,--no-as-needed', language: 'c') ++ if cc.has_link_argument('-Wl,--undefined-version') ++ add_project_link_arguments('-Wl,--undefined-version', language: 'c') ++ endif + endif + + # use pthreads if available for the platform +diff --git a/dpdk/config/x86/cross-mingw b/dpdk/config/x86/cross-mingw +index cddebda5b5..11597eaa26 100644 +--- a/dpdk/config/x86/cross-mingw ++++ b/dpdk/config/x86/cross-mingw +@@ -5,6 +5,7 @@ ld = 'x86_64-w64-mingw32-ld' + ar = 'x86_64-w64-mingw32-ar' + strip = 'x86_64-w64-mingw32-strip' + pkgconfig = 'x86_64-w64-mingw32-pkg-config' ++pkg-config = 'x86_64-w64-mingw32-pkg-config' + objdump = 'x86_64-w64-mingw32-objdump' + + [host_machine] +diff --git a/dpdk/doc/api/doxy-api-index.md b/dpdk/doc/api/doxy-api-index.md +index a6a768bd7c..29eaad6523 100644 +--- a/dpdk/doc/api/doxy-api-index.md ++++ b/dpdk/doc/api/doxy-api-index.md +@@ -102,6 +102,7 @@ The public API headers are grouped by topics: + [service cores](@ref rte_service.h), + [keepalive](@ref rte_keepalive.h), + [power/freq](@ref rte_power.h), ++ [power/uncore](@ref rte_power_uncore.h), + [PMD power](@ref rte_power_pmd_mgmt.h) + + - **layers**: +diff --git a/dpdk/doc/guides/bbdevs/fpga_5gnr_fec.rst b/dpdk/doc/guides/bbdevs/fpga_5gnr_fec.rst +index 956dd6bed5..99fc936829 100644 +--- a/dpdk/doc/guides/bbdevs/fpga_5gnr_fec.rst ++++ b/dpdk/doc/guides/bbdevs/fpga_5gnr_fec.rst +@@ -100,7 +100,6 @@ parameters defined in ``rte_fpga_5gnr_fec_conf`` structure: + uint8_t dl_bandwidth; + uint8_t ul_load_balance; + uint8_t dl_load_balance; +- uint16_t flr_time_out; + }; + + - ``pf_mode_en``: identifies whether only PF is to be used, or the VFs. PF and +@@ -126,10 +125,6 @@ parameters defined in ``rte_fpga_5gnr_fec_conf`` structure: + If all hardware queues exceeds the watermark, no code blocks will be + streamed in from UL/DL code block FIFO. + +-- ``flr_time_out``: specifies how many 16.384us to be FLR time out. The +- time_out = flr_time_out x 16.384us. For instance, if you want to set 10ms for +- the FLR time out then set this setting to 0x262=610. +- + + An example configuration code calling the function ``rte_fpga_5gnr_fec_configure()`` is shown + below: +@@ -154,7 +149,7 @@ below: + /* setup FPGA PF */ + ret = rte_fpga_5gnr_fec_configure(info->dev_name, &conf); + TEST_ASSERT_SUCCESS(ret, +- "Failed to configure 4G FPGA PF for bbdev %s", ++ "Failed to configure 5GNR FPGA PF for bbdev %s", + info->dev_name); + + +diff --git a/dpdk/doc/guides/cryptodevs/cnxk.rst b/dpdk/doc/guides/cryptodevs/cnxk.rst +index fbe67475be..89a6556c1d 100644 +--- a/dpdk/doc/guides/cryptodevs/cnxk.rst ++++ b/dpdk/doc/guides/cryptodevs/cnxk.rst +@@ -197,7 +197,7 @@ Debugging Options + +---+------------+-------------------------------------------------------+ + | # | Component | EAL log command | + +===+============+=======================================================+ +- | 1 | CPT | --log-level='pmd\.crypto\.cnxk,8' | ++ | 1 | CPT | --log-level='pmd\.common\.cnxk\.crypto,8' | + +---+------------+-------------------------------------------------------+ + + Testing +diff --git a/dpdk/doc/guides/cryptodevs/overview.rst b/dpdk/doc/guides/cryptodevs/overview.rst +index d754b0cfc6..b068d0d19c 100644 +--- a/dpdk/doc/guides/cryptodevs/overview.rst ++++ b/dpdk/doc/guides/cryptodevs/overview.rst +@@ -20,17 +20,17 @@ Supported Feature Flags + - "OOP SGL In SGL Out" feature flag stands for + "Out-of-place Scatter-gather list Input, Scatter-gather list Output", + which means PMD supports different scatter-gather styled input and output buffers +- (i.e. both can consists of multiple segments). ++ (i.e. both can consist of multiple segments). + + - "OOP SGL In LB Out" feature flag stands for + "Out-of-place Scatter-gather list Input, Linear Buffers Output", +- which means PMD supports input from scatter-gathered styled buffers, ++ which means PMD supports input from scatter-gather styled buffers, + outputting linear buffers (i.e. single segment). + + - "OOP LB In SGL Out" feature flag stands for + "Out-of-place Linear Buffers Input, Scatter-gather list Output", + which means PMD supports input from linear buffer, outputting +- scatter-gathered styled buffers. ++ scatter-gather styled buffers. + + - "OOP LB In LB Out" feature flag stands for + "Out-of-place Linear Buffers Input, Linear Buffers Output", +diff --git a/dpdk/doc/guides/dmadevs/hisilicon.rst b/dpdk/doc/guides/dmadevs/hisilicon.rst +index 8c1f0f8886..974bc49376 100644 +--- a/dpdk/doc/guides/dmadevs/hisilicon.rst ++++ b/dpdk/doc/guides/dmadevs/hisilicon.rst +@@ -13,7 +13,6 @@ Supported Kunpeng SoCs + ---------------------- + + * Kunpeng 920 +-* Kunpeng 930 + + + Device Setup +diff --git a/dpdk/doc/guides/eventdevs/cnxk.rst b/dpdk/doc/guides/eventdevs/cnxk.rst +index cccb8a0304..c259d37109 100644 +--- a/dpdk/doc/guides/eventdevs/cnxk.rst ++++ b/dpdk/doc/guides/eventdevs/cnxk.rst +@@ -208,9 +208,9 @@ Debugging Options + +---+------------+-------------------------------------------------------+ + | # | Component | EAL log command | + +===+============+=======================================================+ +- | 1 | SSO | --log-level='pmd\.event\.cnxk,8' | ++ | 1 | SSO | --log-level='pmd\.common\.cnxk\.event,8' | + +---+------------+-------------------------------------------------------+ +- | 2 | TIM | --log-level='pmd\.event\.cnxk\.timer,8' | ++ | 2 | TIM | --log-level='pmd\.common\.cnxk\.timer,8' | + +---+------------+-------------------------------------------------------+ + + Limitations +diff --git a/dpdk/doc/guides/howto/af_xdp_cni.rst b/dpdk/doc/guides/howto/af_xdp_cni.rst +deleted file mode 100644 +index a1a6d5b99c..0000000000 +--- a/dpdk/doc/guides/howto/af_xdp_cni.rst ++++ /dev/null +@@ -1,253 +0,0 @@ +-.. SPDX-License-Identifier: BSD-3-Clause +- Copyright(c) 2023 Intel Corporation. +- +-Using a CNI with the AF_XDP driver +-================================== +- +-Introduction +------------- +- +-CNI, the Container Network Interface, is a technology for configuring +-container network interfaces +-and which can be used to setup Kubernetes networking. +-AF_XDP is a Linux socket Address Family that enables an XDP program +-to redirect packets to a memory buffer in userspace. +- +-This document explains how to enable the `AF_XDP Plugin for Kubernetes`_ within +-a DPDK application using the :doc:`../nics/af_xdp` to connect and use these technologies. +- +-.. _AF_XDP Plugin for Kubernetes: https://github.com/intel/afxdp-plugins-for-kubernetes +- +- +-Background +----------- +- +-The standard :doc:`../nics/af_xdp` initialization process involves loading an eBPF program +-onto the kernel netdev to be used by the PMD. +-This operation requires root or escalated Linux privileges +-and thus prevents the PMD from working in an unprivileged container. +-The AF_XDP CNI plugin handles this situation +-by providing a device plugin that performs the program loading. +- +-At a technical level the CNI opens a Unix Domain Socket and listens for a client +-to make requests over that socket. +-A DPDK application acting as a client connects and initiates a configuration "handshake". +-The client then receives a file descriptor which points to the XSKMAP +-associated with the loaded eBPF program. +-The XSKMAP is a BPF map of AF_XDP sockets (XSK). +-The client can then proceed with creating an AF_XDP socket +-and inserting that socket into the XSKMAP pointed to by the descriptor. +- +-The EAL vdev argument ``use_cni`` is used to indicate that the user wishes +-to run the PMD in unprivileged mode and to receive the XSKMAP file descriptor +-from the CNI. +-When this flag is set, +-the ``XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD`` libbpf flag +-should be used when creating the socket +-to instruct libbpf not to load the default libbpf program on the netdev. +-Instead the loading is handled by the CNI. +- +-.. note:: +- +- The Unix Domain Socket file path appear in the end user is "/tmp/afxdp.sock". +- +- +-Prerequisites +-------------- +- +-Docker and container prerequisites: +- +-* Set up the device plugin +- as described in the instructions for `AF_XDP Plugin for Kubernetes`_. +- +-* The Docker image should contain the libbpf and libxdp libraries, +- which are dependencies for AF_XDP, +- and should include support for the ``ethtool`` command. +- +-* The Pod should have enabled the capabilities ``CAP_NET_RAW`` and ``CAP_BPF`` +- for AF_XDP along with support for hugepages. +- +-* Increase locked memory limit so containers have enough memory for packet buffers. +- For example: +- +- .. code-block:: console +- +- cat << EOF | sudo tee /etc/systemd/system/containerd.service.d/limits.conf +- [Service] +- LimitMEMLOCK=infinity +- EOF +- +-* dpdk-testpmd application should have AF_XDP feature enabled. +- +- For further information see the docs for the: :doc:`../../nics/af_xdp`. +- +- +-Example +-------- +- +-Howto run dpdk-testpmd with CNI plugin: +- +-* Clone the CNI plugin +- +- .. code-block:: console +- +- # git clone https://github.com/intel/afxdp-plugins-for-kubernetes.git +- +-* Build the CNI plugin +- +- .. code-block:: console +- +- # cd afxdp-plugins-for-kubernetes/ +- # make build +- +- .. note:: +- +- CNI plugin has a dependence on the config.json. +- +- Sample Config.json +- +- .. code-block:: json +- +- { +- "logLevel":"debug", +- "logFile":"afxdp-dp-e2e.log", +- "pools":[ +- { +- "name":"e2e", +- "mode":"primary", +- "timeout":30, +- "ethtoolCmds" : ["-L -device- combined 1"], +- "devices":[ +- { +- "name":"ens785f0" +- } +- ] +- } +- ] +- } +- +- For further reference please use the `config.json`_ +- +- .. _config.json: https://github.com/intel/afxdp-plugins-for-kubernetes/blob/v0.0.2/test/e2e/config.json +- +-* Create the Network Attachment definition +- +- .. code-block:: console +- +- # kubectl create -f nad.yaml +- +- Sample nad.yml +- +- .. code-block:: yaml +- +- apiVersion: "k8s.cni.cncf.io/v1" +- kind: NetworkAttachmentDefinition +- metadata: +- name: afxdp-e2e-test +- annotations: +- k8s.v1.cni.cncf.io/resourceName: afxdp/e2e +- spec: +- config: '{ +- "cniVersion": "0.3.0", +- "type": "afxdp", +- "mode": "cdq", +- "logFile": "afxdp-cni-e2e.log", +- "logLevel": "debug", +- "ipam": { +- "type": "host-local", +- "subnet": "192.168.1.0/24", +- "rangeStart": "192.168.1.200", +- "rangeEnd": "192.168.1.216", +- "routes": [ +- { "dst": "0.0.0.0/0" } +- ], +- "gateway": "192.168.1.1" +- } +- }' +- +- For further reference please use the `nad.yaml`_ +- +- .. _nad.yaml: https://github.com/intel/afxdp-plugins-for-kubernetes/blob/v0.0.2/test/e2e/nad.yaml +- +-* Build the Docker image +- +- .. code-block:: console +- +- # docker build -t afxdp-e2e-test -f Dockerfile . +- +- Sample Dockerfile: +- +- .. code-block:: console +- +- FROM ubuntu:20.04 +- RUN apt-get update -y +- RUN apt install build-essential libelf-dev -y +- RUN apt-get install iproute2 acl -y +- RUN apt install python3-pyelftools ethtool -y +- RUN apt install libnuma-dev libjansson-dev libpcap-dev net-tools -y +- RUN apt-get install clang llvm -y +- COPY ./libbpf.tar.gz /tmp +- RUN cd /tmp && tar -xvmf libbpf.tar.gz && cd libbpf/src && make install +- COPY ./libxdp.tar.gz /tmp +- RUN cd /tmp && tar -xvmf libxdp.tar.gz && cd libxdp && make install +- +- .. note:: +- +- All the files that need to COPY-ed should be in the same directory as the Dockerfile +- +-* Run the Pod +- +- .. code-block:: console +- +- # kubectl create -f pod.yaml +- +- Sample pod.yaml: +- +- .. code-block:: yaml +- +- apiVersion: v1 +- kind: Pod +- metadata: +- name: afxdp-e2e-test +- annotations: +- k8s.v1.cni.cncf.io/networks: afxdp-e2e-test +- spec: +- containers: +- - name: afxdp +- image: afxdp-e2e-test:latest +- imagePullPolicy: Never +- env: +- - name: LD_LIBRARY_PATH +- value: /usr/lib64/:/usr/local/lib/ +- command: ["tail", "-f", "/dev/null"] +- securityContext: +- capabilities: +- add: +- - CAP_NET_RAW +- - CAP_BPF +- resources: +- requests: +- hugepages-2Mi: 2Gi +- memory: 2Gi +- afxdp/e2e: '1' +- limits: +- hugepages-2Mi: 2Gi +- memory: 2Gi +- afxdp/e2e: '1' +- +- For further reference please use the `pod.yaml`_ +- +- .. _pod.yaml: https://github.com/intel/afxdp-plugins-for-kubernetes/blob/v0.0.2/test/e2e/pod-1c1d.yaml +- +-* Run DPDK with a command like the following: +- +- .. code-block:: console +- +- kubectl exec -i --container -- \ +- //dpdk-testpmd -l 0,1 --no-pci \ +- --vdev=net_af_xdp0,use_cni=1,iface= \ +- -- --no-mlockall --in-memory +- +-For further reference please use the `e2e`_ test case in `AF_XDP Plugin for Kubernetes`_ +- +- .. _e2e: https://github.com/intel/afxdp-plugins-for-kubernetes/tree/v0.0.2/test/e2e +diff --git a/dpdk/doc/guides/howto/af_xdp_dp.rst b/dpdk/doc/guides/howto/af_xdp_dp.rst +new file mode 100644 +index 0000000000..4ca5462c69 +--- /dev/null ++++ b/dpdk/doc/guides/howto/af_xdp_dp.rst +@@ -0,0 +1,323 @@ ++.. SPDX-License-Identifier: BSD-3-Clause ++ Copyright(c) 2023 Intel Corporation. ++ ++Using the AF_XDP driver in Kubernetes ++===================================== ++ ++Introduction ++------------ ++ ++Two infrastructure components are needed in order to provision a pod ++that is using the AF_XDP PMD in Kubernetes: ++ ++1. AF_XDP Device Plugin (DP). ++2. AF_XDP Container Network Interface (CNI) binary. ++ ++Both of these components are available through ++the `AF_XDP Device Plugin for Kubernetes`_ repository. ++ ++The AF_XDP DP provisions and advertises networking interfaces to Kubernetes, ++while the CNI configures and plumbs network interfaces for the Pod. ++ ++This document explains how to use the `AF_XDP Device Plugin for Kubernetes`_ ++with a DPDK application using the :doc:`../nics/af_xdp`. ++ ++.. _AF_XDP Device Plugin for Kubernetes: https://github.com/redhat-et/afxdp-plugins-for-kubernetes ++ ++ ++Background ++---------- ++ ++The standard :doc:`../nics/af_xdp` initialization process involves loading an eBPF program ++onto the kernel netdev to be used by the PMD. ++This operation requires root or escalated Linux privileges ++and thus prevents the PMD from working in an unprivileged container. ++The AF_XDP Device Plugin handles this situation ++by managing the eBPF program(s) on behalf of the Pod, outside of the pod context. ++ ++At a technical level the AF_XDP Device Plugin opens a Unix Domain Socket (UDS) ++and listens for a client to make requests over that socket. ++A DPDK application acting as a client connects and initiates a configuration "handshake". ++After some validation on the Device Plugin side, ++the client receives a file descriptor which points to the XSKMAP ++associated with the loaded eBPF program. ++The XSKMAP is an eBPF map of AF_XDP sockets (XSK). ++The client can then proceed with creating an AF_XDP socket ++and inserting that socket into the XSKMAP pointed to by the descriptor. ++ ++The EAL vdev argument ``use_cni`` is used to indicate that the user wishes ++to run the PMD in unprivileged mode and to receive the XSKMAP file descriptor ++from the CNI. ++When this flag is set, ++the ``XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD`` libbpf flag ++should be used when creating the socket ++to instruct libbpf not to load the default libbpf program on the netdev. ++Instead the loading is handled by the AF_XDP Device Plugin. ++ ++The EAL vdev argument ``dp_path`` is used alongside the ``use_cni`` argument ++to explicitly tell the AF_XDP PMD where to find the UDS ++to interact with the AF_XDP Device Plugin. ++If this argument is not passed alongside the ``use_cni`` argument ++then the AF_XDP PMD configures it internally. ++ ++.. note:: ++ ++ DPDK AF_XDP PMD <= v23.11 will only work with ++ the AF_XDP Device Plugin <= commit id `38317c2`_. ++ ++.. note:: ++ ++ DPDK AF_XDP PMD > v23.11 will work with latest version of the AF_XDP Device Plugin ++ through a combination of the ``dp_path`` and/or the ``use_cni`` parameter. ++ In these versions of the PMD if a user doesn't explicitly set the ``dp_path`` parameter ++ when using ``use_cni`` then that path is transparently configured in the AF_XDP PMD ++ to the default `AF_XDP Device Plugin for Kubernetes`_ mount point path. ++ The path can be overridden by explicitly setting the ``dp_path`` param. ++ ++.. note:: ++ ++ DPDK AF_XDP PMD > v23.11 is backwards compatible ++ with (older) versions of the AF_XDP DP <= commit id `38317c2`_ ++ by explicitly setting ``dp_path`` to ``/tmp/afxdp.sock``. ++ ++.. _38317c2: https://github.com/redhat-et/afxdp-plugins-for-kubernetes/commit/38317c256b5c7dfb39e013a0f76010c2ded03669 ++ ++Prerequisites ++------------- ++ ++Device Plugin and DPDK container prerequisites: ++ ++* Create a DPDK container image. ++ ++* Set up the device plugin and prepare the Pod Spec as described in ++ the instructions for `AF_XDP Device Plugin for Kubernetes`_. ++ ++* The Docker image should contain the libbpf and libxdp libraries, ++ which are dependencies for AF_XDP, ++ and should include support for the ``ethtool`` command. ++ ++* The Pod should have enabled the capabilities ++ ``CAP_NET_RAW`` for AF_XDP socket creation, ++ ``IPC_LOCK`` for umem creation and ++ ``CAP_BPF`` (for Kernel < 5.19) along with support for hugepages. ++ ++ .. note:: ++ ++ For Kernel versions < 5.19, all BPF sys calls required CAP_BPF, ++ to access maps shared between the eBFP program and the userspace program. ++ Kernels >= 5.19, only requires CAP_BPF for map creation (BPF_MAP_CREATE) ++ and loading programs (BPF_PROG_LOAD). ++ ++* Increase locked memory limit so containers have enough memory for packet buffers. ++ For example: ++ ++ .. code-block:: console ++ ++ cat << EOF | sudo tee /etc/systemd/system/containerd.service.d/limits.conf ++ [Service] ++ LimitMEMLOCK=infinity ++ EOF ++ ++* dpdk-testpmd application should have AF_XDP feature enabled. ++ ++ For further information see the docs for the: :doc:`../../nics/af_xdp`. ++ ++ ++Example ++------- ++ ++Build a DPDK container image (using Docker) ++~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ++ ++1. Create a Dockerfile (should be placed in top level DPDK directory): ++ ++ .. code-block:: console ++ ++ FROM fedora:38 ++ ++ # Setup container to build DPDK applications ++ RUN dnf -y upgrade && dnf -y install \ ++ libbsd-devel \ ++ numactl-libs \ ++ libbpf-devel \ ++ libbpf \ ++ meson \ ++ ninja-build \ ++ libxdp-devel \ ++ libxdp \ ++ numactl-devel \ ++ python3-pyelftools \ ++ python38 \ ++ iproute ++ RUN dnf groupinstall -y 'Development Tools' ++ ++ # Create DPDK dir and copy over sources ++ # Create DPDK dir and copy over sources ++ COPY ./ /dpdk ++ WORKDIR /dpdk ++ ++ # Build DPDK ++ RUN meson setup build ++ RUN ninja -C build ++ ++2. Build a DPDK container image (using Docker) ++ ++ .. code-block:: console ++ ++ # docker build -t dpdk -f Dockerfile ++ ++Run dpdk-testpmd with the AF_XDP Device Plugin + CNI ++~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ++ ++* Clone the AF_XDP Device plugin and CNI ++ ++ .. code-block:: console ++ ++ # git clone https://github.com/redhat-et/afxdp-plugins-for-kubernetes.git ++ ++ .. note:: ++ ++ Ensure you have the AF_XDP Device Plugin + CNI prerequisites installed. ++ ++* Build the AF_XDP Device plugin and CNI ++ ++ .. code-block:: console ++ ++ # cd afxdp-plugins-for-kubernetes/ ++ # make image ++ ++* Make sure to modify the image used by the `daemonset.yml`_ file ++ in the deployments directory with the following configuration: ++ ++ .. _daemonset.yml : https://github.com/redhat-et/afxdp-plugins-for-kubernetes/blob/main/deployments/daemonset.yml ++ ++ .. code-block:: yaml ++ ++ image: afxdp-device-plugin:latest ++ ++ .. note:: ++ ++ This will select the AF_XDP DP image that was built locally. ++ Detailed configuration options can be found in the AF_XDP Device Plugin `readme`_ . ++ ++ .. _readme: https://github.com/redhat-et/afxdp-plugins-for-kubernetes#readme ++ ++* Deploy the AF_XDP Device Plugin and CNI ++ ++ .. code-block:: console ++ ++ # kubectl create -f deployments/daemonset.yml ++ ++* Create the Network Attachment definition ++ ++ .. code-block:: console ++ ++ # kubectl create -f nad.yaml ++ ++ Sample nad.yml ++ ++ .. code-block:: yaml ++ ++ apiVersion: "k8s.cni.cncf.io/v1" ++ kind: NetworkAttachmentDefinition ++ metadata: ++ name: afxdp-network ++ annotations: ++ k8s.v1.cni.cncf.io/resourceName: afxdp/myPool ++ spec: ++ config: '{ ++ "cniVersion": "0.3.0", ++ "type": "afxdp", ++ "mode": "primary", ++ "logFile": "afxdp-cni.log", ++ "logLevel": "debug", ++ "ethtoolCmds" : ["-N -device- rx-flow-hash udp4 fn", ++ "-N -device- flow-type udp4 dst-port 2152 action 22" ++ ], ++ "ipam": { ++ "type": "host-local", ++ "subnet": "192.168.1.0/24", ++ "rangeStart": "192.168.1.200", ++ "rangeEnd": "192.168.1.220", ++ "routes": [ ++ { "dst": "0.0.0.0/0" } ++ ], ++ "gateway": "192.168.1.1" ++ } ++ }' ++ ++ For further reference please use the example provided by the AF_XDP DP `nad.yaml`_ ++ ++ .. _nad.yaml: https://github.com/redhat-et/afxdp-plugins-for-kubernetes/blob/main/examples/network-attachment-definition.yaml ++ ++* Run the Pod ++ ++ .. code-block:: console ++ ++ # kubectl create -f pod.yaml ++ ++ Sample pod.yaml: ++ ++ .. code-block:: yaml ++ ++ apiVersion: v1 ++ kind: Pod ++ metadata: ++ name: dpdk ++ annotations: ++ k8s.v1.cni.cncf.io/networks: afxdp-network ++ spec: ++ containers: ++ - name: testpmd ++ image: dpdk:latest ++ command: ["tail", "-f", "/dev/null"] ++ securityContext: ++ capabilities: ++ add: ++ - NET_RAW ++ - IPC_LOCK ++ resources: ++ requests: ++ afxdp/myPool: '1' ++ limits: ++ hugepages-1Gi: 2Gi ++ cpu: 2 ++ memory: 256Mi ++ afxdp/myPool: '1' ++ volumeMounts: ++ - name: hugepages ++ mountPath: /dev/hugepages ++ volumes: ++ - name: hugepages ++ emptyDir: ++ medium: HugePages ++ ++ For further reference please see the `pod.yaml`_ ++ ++ .. _pod.yaml: https://github.com/redhat-et/afxdp-plugins-for-kubernetes/blob/main/examples/pod-spec.yaml ++ ++* Run DPDK with a command like the following: ++ ++ .. code-block:: console ++ ++ kubectl exec -i --container -- \ ++ //dpdk-testpmd -l 0,1 --no-pci \ ++ --vdev=net_af_xdp0,use_cni=1,iface= \ ++ --no-mlockall --in-memory \ ++ -- -i --a --nb-cores=2 --rxq=1 --txq=1 --forward-mode=macswap; ++ ++ Or ++ ++ .. code-block:: console ++ ++ kubectl exec -i --container -- \ ++ //dpdk-testpmd -l 0,1 --no-pci \ ++ --vdev=net_af_xdp0,use_cni=1,iface=,dp_path="/tmp/afxdp_dp//afxdp.sock" \ ++ --no-mlockall --in-memory \ ++ -- -i --a --nb-cores=2 --rxq=1 --txq=1 --forward-mode=macswap; ++ ++.. note:: ++ ++ If the ``dp_path`` parameter isn't explicitly set (like the example above), ++ the AF_XDP PMD will set the parameter value to ++ ``/tmp/afxdp_dp/<>/afxdp.sock``. +diff --git a/dpdk/doc/guides/howto/index.rst b/dpdk/doc/guides/howto/index.rst +index 71a3381c36..a7692e8a97 100644 +--- a/dpdk/doc/guides/howto/index.rst ++++ b/dpdk/doc/guides/howto/index.rst +@@ -8,7 +8,7 @@ HowTo Guides + :maxdepth: 2 + :numbered: + +- af_xdp_cni ++ af_xdp_dp + lm_bond_virtio_sriov + lm_virtio_vhost_user + flow_bifurcation +diff --git a/dpdk/doc/guides/linux_gsg/enable_func.rst b/dpdk/doc/guides/linux_gsg/enable_func.rst +index 2344d97403..5511640cb8 100644 +--- a/dpdk/doc/guides/linux_gsg/enable_func.rst ++++ b/dpdk/doc/guides/linux_gsg/enable_func.rst +@@ -85,8 +85,7 @@ need to be adjusted in order to ensure normal DPDK operation: + The above limits can usually be adjusted by editing + ``/etc/security/limits.conf`` file, and rebooting. + +-See `Hugepage Mapping `_ +-section to learn how these limits affect EAL. ++See :ref:`Hugepage Mapping ` section to learn how these limits affect EAL. + + Device Control + ~~~~~~~~~~~~~~ +diff --git a/dpdk/doc/guides/mempool/cnxk.rst b/dpdk/doc/guides/mempool/cnxk.rst +index d883b83f7b..ffd9a6de86 100644 +--- a/dpdk/doc/guides/mempool/cnxk.rst ++++ b/dpdk/doc/guides/mempool/cnxk.rst +@@ -72,7 +72,7 @@ Debugging Options + +---+------------+-------------------------------------------------------+ + | # | Component | EAL log command | + +===+============+=======================================================+ +- | 1 | NPA | --log-level='pmd\.mempool.cnxk,8' | ++ | 1 | NPA | --log-level='pmd\.common\.cnxk\.mempool,8' | + +---+------------+-------------------------------------------------------+ + + Standalone mempool device +diff --git a/dpdk/doc/guides/mldevs/cnxk.rst b/dpdk/doc/guides/mldevs/cnxk.rst +index ae9e1fae5f..fc1bcd9cdb 100644 +--- a/dpdk/doc/guides/mldevs/cnxk.rst ++++ b/dpdk/doc/guides/mldevs/cnxk.rst +@@ -383,7 +383,7 @@ Debugging Options + +---+------------+-------------------------------------------------------+ + | # | Component | EAL log command | + +===+============+=======================================================+ +- | 1 | ML | --log-level='pmd\.ml\.cnxk,8' | ++ | 1 | ML | --log-level='pmd\.common\.cnxk\.ml,8' | + +---+------------+-------------------------------------------------------+ + + +diff --git a/dpdk/doc/guides/nics/af_xdp.rst b/dpdk/doc/guides/nics/af_xdp.rst +index 1932525d4d..60a88c467e 100644 +--- a/dpdk/doc/guides/nics/af_xdp.rst ++++ b/dpdk/doc/guides/nics/af_xdp.rst +@@ -155,9 +155,9 @@ use_cni + ~~~~~~~ + + The EAL vdev argument ``use_cni`` is used to indicate that the user wishes to +-enable the `AF_XDP Plugin for Kubernetes`_ within a DPDK application. ++enable the `AF_XDP Device Plugin for Kubernetes`_ with a DPDK application/pod. + +-.. _AF_XDP Plugin for Kubernetes: https://github.com/intel/afxdp-plugins-for-kubernetes ++.. _AF_XDP Device Plugin for Kubernetes: https://github.com/redhat-et/afxdp-plugins-for-kubernetes + + .. code-block:: console + +@@ -171,6 +171,21 @@ enable the `AF_XDP Plugin for Kubernetes`_ within a DPDK application. + so enabling and disabling of the promiscuous mode through the DPDK application + is also not supported. + ++dp_path ++~~~~~~~ ++ ++The EAL vdev argument ``dp_path`` is used alongside the ``use_cni`` argument ++to explicitly tell the AF_XDP PMD where to find the UDS ++to interact with the `AF_XDP Device Plugin for Kubernetes`_. ++If this argument is not passed alongside the ``use_cni`` argument ++then the AF_XDP PMD configures it internally. ++ ++.. _AF_XDP Device Plugin for Kubernetes: https://github.com/redhat-et/afxdp-plugins-for-kubernetes ++ ++.. code-block:: console ++ ++ --vdev=net_af_xdp0,use_cni=1,dp_path="/tmp/afxdp_dp/<>/afxdp.sock" ++ + Limitations + ----------- + +diff --git a/dpdk/doc/guides/nics/cnxk.rst b/dpdk/doc/guides/nics/cnxk.rst +index 9ec52e380f..501ef1f826 100644 +--- a/dpdk/doc/guides/nics/cnxk.rst ++++ b/dpdk/doc/guides/nics/cnxk.rst +@@ -416,6 +416,18 @@ Runtime Config Options + With the above configuration, PMD would allocate meta buffers of size 512 for + inline inbound IPsec processing second pass. + ++- ``NPC MCAM Aging poll frequency in seconds`` (default ``10``) ++ ++ Poll frequency for aging control thread can be specified by ++ ``aging_poll_freq`` devargs parameter. ++ ++ For example:: ++ ++ -a 0002:01:00.2,aging_poll_freq=50 ++ ++ With the above configuration, driver would poll for aging flows ++ every 50 seconds. ++ + .. note:: + + Above devarg parameters are configurable per device, user needs to pass the +@@ -601,18 +613,6 @@ Runtime Config Options for inline device + With the above configuration, driver would poll for soft expiry events every + 1000 usec. + +-- ``NPC MCAM Aging poll frequency in seconds`` (default ``10``) +- +- Poll frequency for aging control thread can be specified by +- ``aging_poll_freq`` ``devargs`` parameter. +- +- For example:: +- +- -a 0002:01:00.2,aging_poll_freq=50 +- +- With the above configuration, driver would poll for aging flows every 50 +- seconds. +- + Debugging Options + ----------------- + +@@ -623,7 +623,7 @@ Debugging Options + +---+------------+-------------------------------------------------------+ + | # | Component | EAL log command | + +===+============+=======================================================+ +- | 1 | NIX | --log-level='pmd\.net.cnxk,8' | ++ | 1 | NIX | --log-level='pmd\.common.cnxk\.nix,8' | + +---+------------+-------------------------------------------------------+ +- | 2 | NPC | --log-level='pmd\.net.cnxk\.flow,8' | ++ | 2 | NPC | --log-level='pmd\.common.cnxk\.flow,8' | + +---+------------+-------------------------------------------------------+ +diff --git a/dpdk/doc/guides/nics/features.rst b/dpdk/doc/guides/nics/features.rst +index f7d9980849..cf9fabb8b8 100644 +--- a/dpdk/doc/guides/nics/features.rst ++++ b/dpdk/doc/guides/nics/features.rst +@@ -34,6 +34,17 @@ Supports getting the speed capabilities that the current device is capable of. + * **[related] API**: ``rte_eth_dev_info_get()``. + + ++.. _nic_features_link_speeds_config: ++ ++Link speed configuration ++------------------------ ++ ++Supports configurating fixed speed and link autonegotiation. ++ ++* **[uses] user config**: ``dev_conf.link_speeds:RTE_ETH_LINK_SPEED_*``. ++* **[related] API**: ``rte_eth_dev_configure()``. ++ ++ + .. _nic_features_link_status: + + Link status +@@ -751,6 +762,19 @@ Supports congestion management. + ``rte_eth_cman_config_set()``, ``rte_eth_cman_config_get()``. + + ++.. _nic_features_traffic_manager: ++ ++Traffic manager ++--------------- ++ ++Supports Traffic manager. ++ ++* **[implements] rte_tm_ops**: ``capabilities_get``, ``shaper_profile_add``, ++ ``hierarchy_commit`` and so on. ++* **[related] API**: ``rte_tm_capabilities_get()``, ``rte_tm_shaper_profile_add()``, ++ ``rte_tm_hierarchy_commit()`` and so on. ++ ++ + .. _nic_features_fw_version: + + FW version +diff --git a/dpdk/doc/guides/nics/features/atlantic.ini b/dpdk/doc/guides/nics/features/atlantic.ini +index ef4155027c..29969c1493 100644 +--- a/dpdk/doc/guides/nics/features/atlantic.ini ++++ b/dpdk/doc/guides/nics/features/atlantic.ini +@@ -5,6 +5,7 @@ + ; + [Features] + Speed capabilities = Y ++Link speed configuration = Y + Link status = Y + Link status event = Y + Queue start/stop = Y +diff --git a/dpdk/doc/guides/nics/features/bnxt.ini b/dpdk/doc/guides/nics/features/bnxt.ini +index bd4e2295dc..c33889663d 100644 +--- a/dpdk/doc/guides/nics/features/bnxt.ini ++++ b/dpdk/doc/guides/nics/features/bnxt.ini +@@ -5,6 +5,7 @@ + ; + [Features] + Speed capabilities = Y ++Link speed configuration = Y + Link status = Y + Link status event = Y + Rx interrupt = Y +diff --git a/dpdk/doc/guides/nics/features/cnxk.ini b/dpdk/doc/guides/nics/features/cnxk.ini +index ac7de9a0f0..f85813ab52 100644 +--- a/dpdk/doc/guides/nics/features/cnxk.ini ++++ b/dpdk/doc/guides/nics/features/cnxk.ini +@@ -28,6 +28,7 @@ RSS key update = Y + RSS reta update = Y + Inner RSS = Y + Congestion management = Y ++Traffic manager = Y + Inline protocol = Y + Flow control = Y + Scattered Rx = Y +diff --git a/dpdk/doc/guides/nics/features/default.ini b/dpdk/doc/guides/nics/features/default.ini +index 806cb033ff..c30702c72e 100644 +--- a/dpdk/doc/guides/nics/features/default.ini ++++ b/dpdk/doc/guides/nics/features/default.ini +@@ -8,6 +8,7 @@ + ; + [Features] + Speed capabilities = ++Link speed configuration = + Link status = + Link status event = + Removal event = +@@ -42,6 +43,7 @@ VLAN filter = + Flow control = + Rate limitation = + Congestion management = ++Traffic manager = + Inline crypto = + Inline protocol = + CRC offload = +diff --git a/dpdk/doc/guides/nics/features/dpaa.ini b/dpdk/doc/guides/nics/features/dpaa.ini +index a382c7160c..b136ed191a 100644 +--- a/dpdk/doc/guides/nics/features/dpaa.ini ++++ b/dpdk/doc/guides/nics/features/dpaa.ini +@@ -5,6 +5,7 @@ + ; + [Features] + Speed capabilities = Y ++Link speed configuration = Y + Link status = Y + Link status event = Y + Burst mode info = Y +diff --git a/dpdk/doc/guides/nics/features/dpaa2.ini b/dpdk/doc/guides/nics/features/dpaa2.ini +index 26dc8c2178..f02da463d9 100644 +--- a/dpdk/doc/guides/nics/features/dpaa2.ini ++++ b/dpdk/doc/guides/nics/features/dpaa2.ini +@@ -17,6 +17,7 @@ Unicast MAC filter = Y + RSS hash = Y + VLAN filter = Y + Flow control = Y ++Traffic manager = Y + VLAN offload = Y + L3 checksum offload = Y + L4 checksum offload = Y +diff --git a/dpdk/doc/guides/nics/features/hns3.ini b/dpdk/doc/guides/nics/features/hns3.ini +index 338b4e6864..8b623d3077 100644 +--- a/dpdk/doc/guides/nics/features/hns3.ini ++++ b/dpdk/doc/guides/nics/features/hns3.ini +@@ -5,6 +5,7 @@ + ; + [Features] + Speed capabilities = Y ++Link speed configuration = Y + Link status = Y + Link status event = Y + Rx interrupt = Y +@@ -28,6 +29,7 @@ RSS reta update = Y + DCB = Y + VLAN filter = Y + Flow control = Y ++Traffic manager = Y + CRC offload = Y + VLAN offload = Y + FEC = Y +diff --git a/dpdk/doc/guides/nics/features/i40e.ini b/dpdk/doc/guides/nics/features/i40e.ini +index e241dad047..ef7514c44b 100644 +--- a/dpdk/doc/guides/nics/features/i40e.ini ++++ b/dpdk/doc/guides/nics/features/i40e.ini +@@ -5,6 +5,7 @@ + ; + [Features] + Speed capabilities = Y ++Link speed configuration = Y + Link status = Y + Link status event = Y + Rx interrupt = Y +@@ -27,6 +28,7 @@ SR-IOV = Y + DCB = Y + VLAN filter = Y + Flow control = Y ++Traffic manager = Y + CRC offload = Y + VLAN offload = Y + QinQ offload = P +diff --git a/dpdk/doc/guides/nics/features/iavf.ini b/dpdk/doc/guides/nics/features/iavf.ini +index db4f92ce71..ce9860e963 100644 +--- a/dpdk/doc/guides/nics/features/iavf.ini ++++ b/dpdk/doc/guides/nics/features/iavf.ini +@@ -25,17 +25,18 @@ RSS hash = Y + RSS key update = Y + RSS reta update = Y + VLAN filter = Y ++Traffic manager = Y ++Inline crypto = Y + CRC offload = Y + VLAN offload = P + L3 checksum offload = Y + L4 checksum offload = Y + Timestamp offload = Y + Inner L3 checksum = Y +-Inner L4 checksum = Y ++Inner L4 checksum = P + Packet type parsing = Y + Rx descriptor status = Y + Tx descriptor status = Y +-Inline crypto = Y + Basic stats = Y + Multiprocess aware = Y + FreeBSD = Y +diff --git a/dpdk/doc/guides/nics/features/ice.ini b/dpdk/doc/guides/nics/features/ice.ini +index 13f8871dcc..62869ef0a0 100644 +--- a/dpdk/doc/guides/nics/features/ice.ini ++++ b/dpdk/doc/guides/nics/features/ice.ini +@@ -8,6 +8,7 @@ + ; + [Features] + Speed capabilities = Y ++Link speed configuration = Y + Link status = Y + Link status event = Y + Rx interrupt = Y +@@ -26,6 +27,7 @@ RSS hash = Y + RSS key update = Y + RSS reta update = Y + VLAN filter = Y ++Traffic manager = Y + CRC offload = Y + VLAN offload = Y + QinQ offload = P +diff --git a/dpdk/doc/guides/nics/features/ice_dcf.ini b/dpdk/doc/guides/nics/features/ice_dcf.ini +index 3b11622d4c..0e86338990 100644 +--- a/dpdk/doc/guides/nics/features/ice_dcf.ini ++++ b/dpdk/doc/guides/nics/features/ice_dcf.ini +@@ -22,6 +22,7 @@ Promiscuous mode = Y + Allmulticast mode = Y + Unicast MAC filter = Y + VLAN filter = Y ++Traffic manager = Y + VLAN offload = Y + Extended stats = Y + Basic stats = Y +diff --git a/dpdk/doc/guides/nics/features/igb.ini b/dpdk/doc/guides/nics/features/igb.ini +index 7b4af6f86c..ee2408f3ee 100644 +--- a/dpdk/doc/guides/nics/features/igb.ini ++++ b/dpdk/doc/guides/nics/features/igb.ini +@@ -5,6 +5,7 @@ + ; + [Features] + Speed capabilities = P ++Link speed configuration = Y + Link status = Y + Link status event = Y + Rx interrupt = Y +diff --git a/dpdk/doc/guides/nics/features/igc.ini b/dpdk/doc/guides/nics/features/igc.ini +index 47d9344435..d6db18c1e8 100644 +--- a/dpdk/doc/guides/nics/features/igc.ini ++++ b/dpdk/doc/guides/nics/features/igc.ini +@@ -4,6 +4,7 @@ + ; + [Features] + Speed capabilities = Y ++Link speed configuration = Y + Link status = Y + Link status event = Y + FW version = Y +diff --git a/dpdk/doc/guides/nics/features/ionic.ini b/dpdk/doc/guides/nics/features/ionic.ini +index af0fc5462a..64b2316288 100644 +--- a/dpdk/doc/guides/nics/features/ionic.ini ++++ b/dpdk/doc/guides/nics/features/ionic.ini +@@ -5,6 +5,7 @@ + ; + [Features] + Speed capabilities = Y ++Link speed configuration = Y + Link status = Y + Link status event = Y + Fast mbuf free = Y +diff --git a/dpdk/doc/guides/nics/features/ipn3ke.ini b/dpdk/doc/guides/nics/features/ipn3ke.ini +index 1f6b780273..e412978820 100644 +--- a/dpdk/doc/guides/nics/features/ipn3ke.ini ++++ b/dpdk/doc/guides/nics/features/ipn3ke.ini +@@ -25,6 +25,7 @@ SR-IOV = Y + DCB = Y + VLAN filter = Y + Flow control = Y ++Traffic manager = Y + CRC offload = Y + VLAN offload = Y + QinQ offload = Y +diff --git a/dpdk/doc/guides/nics/features/ixgbe.ini b/dpdk/doc/guides/nics/features/ixgbe.ini +index 8590ac857f..cb9331dbcd 100644 +--- a/dpdk/doc/guides/nics/features/ixgbe.ini ++++ b/dpdk/doc/guides/nics/features/ixgbe.ini +@@ -5,6 +5,7 @@ + ; + [Features] + Speed capabilities = Y ++Link speed configuration = Y + Link status = Y + Link status event = Y + Rx interrupt = Y +@@ -27,6 +28,7 @@ DCB = Y + VLAN filter = Y + Flow control = Y + Rate limitation = Y ++Traffic manager = Y + Inline crypto = Y + CRC offload = P + VLAN offload = P +diff --git a/dpdk/doc/guides/nics/features/mvpp2.ini b/dpdk/doc/guides/nics/features/mvpp2.ini +index 653c9d08cb..ccc2c2d4f8 100644 +--- a/dpdk/doc/guides/nics/features/mvpp2.ini ++++ b/dpdk/doc/guides/nics/features/mvpp2.ini +@@ -12,8 +12,9 @@ Allmulticast mode = Y + Unicast MAC filter = Y + Multicast MAC filter = Y + RSS hash = Y +-Flow control = Y + VLAN filter = Y ++Flow control = Y ++Traffic manager = Y + CRC offload = Y + L3 checksum offload = Y + L4 checksum offload = Y +diff --git a/dpdk/doc/guides/nics/features/ngbe.ini b/dpdk/doc/guides/nics/features/ngbe.ini +index 2701c5f051..1dfd92e96b 100644 +--- a/dpdk/doc/guides/nics/features/ngbe.ini ++++ b/dpdk/doc/guides/nics/features/ngbe.ini +@@ -5,6 +5,7 @@ + ; + [Features] + Speed capabilities = Y ++Link speed configuration = Y + Link status = Y + Link status event = Y + Free Tx mbuf on demand = Y +diff --git a/dpdk/doc/guides/nics/features/octeontx.ini b/dpdk/doc/guides/nics/features/octeontx.ini +index fa1e18b120..46ae8318a9 100644 +--- a/dpdk/doc/guides/nics/features/octeontx.ini ++++ b/dpdk/doc/guides/nics/features/octeontx.ini +@@ -5,6 +5,7 @@ + ; + [Features] + Speed capabilities = Y ++Link speed configuration = Y + Link status = Y + Link status event = Y + Lock-free Tx queue = Y +diff --git a/dpdk/doc/guides/nics/features/sfc.ini b/dpdk/doc/guides/nics/features/sfc.ini +index 8a9198adcb..f9654e69ed 100644 +--- a/dpdk/doc/guides/nics/features/sfc.ini ++++ b/dpdk/doc/guides/nics/features/sfc.ini +@@ -5,6 +5,7 @@ + ; + [Features] + Speed capabilities = Y ++Link speed configuration = Y + Link status = Y + Link status event = Y + Rx interrupt = Y +diff --git a/dpdk/doc/guides/nics/features/thunderx.ini b/dpdk/doc/guides/nics/features/thunderx.ini +index b33bb37c82..2ab8db7239 100644 +--- a/dpdk/doc/guides/nics/features/thunderx.ini ++++ b/dpdk/doc/guides/nics/features/thunderx.ini +@@ -5,6 +5,7 @@ + ; + [Features] + Speed capabilities = Y ++Link speed configuration = Y + Link status = Y + Link status event = Y + Queue start/stop = Y +diff --git a/dpdk/doc/guides/nics/features/txgbe.ini b/dpdk/doc/guides/nics/features/txgbe.ini +index e21083052c..be0af3dfad 100644 +--- a/dpdk/doc/guides/nics/features/txgbe.ini ++++ b/dpdk/doc/guides/nics/features/txgbe.ini +@@ -5,6 +5,7 @@ + ; + [Features] + Speed capabilities = Y ++Link speed configuration = Y + Link status = Y + Link status event = Y + Rx interrupt = Y +@@ -26,6 +27,7 @@ DCB = Y + VLAN filter = Y + Flow control = Y + Rate limitation = Y ++Traffic manager = Y + Inline crypto = Y + CRC offload = P + VLAN offload = P +diff --git a/dpdk/doc/guides/nics/hns3.rst b/dpdk/doc/guides/nics/hns3.rst +index 3b0613fc1b..3e84d1ff1c 100644 +--- a/dpdk/doc/guides/nics/hns3.rst ++++ b/dpdk/doc/guides/nics/hns3.rst +@@ -6,7 +6,7 @@ HNS3 Poll Mode Driver + + The hns3 PMD (**librte_net_hns3**) provides poll mode driver support + for the inbuilt HiSilicon Network Subsystem(HNS) network engine +-found in the HiSilicon Kunpeng 920 SoC and Kunpeng 930 SoC . ++found in the HiSilicon Kunpeng 920 SoC (HIP08) and Kunpeng 930 SoC (HIP09/HIP10). + + Features + -------- +diff --git a/dpdk/doc/guides/nics/mlx5.rst b/dpdk/doc/guides/nics/mlx5.rst +index 6b52fb93c5..2c59b24d78 100644 +--- a/dpdk/doc/guides/nics/mlx5.rst ++++ b/dpdk/doc/guides/nics/mlx5.rst +@@ -245,6 +245,26 @@ Limitations + - Matching on ICMP6 following IPv6 routing extension header, + should match ``ipv6_routing_ext_next_hdr`` instead of ICMP6. + ++ - The supported actions order is as below:: ++ ++ MARK (a) ++ *_DECAP (b) ++ OF_POP_VLAN ++ COUNT | AGE ++ METER_MARK | CONNTRACK ++ OF_PUSH_VLAN ++ MODIFY_FIELD ++ *_ENCAP (c) ++ JUMP | DROP | RSS (a) | QUEUE (a) | REPRESENTED_PORT (d) ++ ++ a. Only supported on ingress. ++ b. Any decapsulation action, including the combination of RAW_ENCAP and RAW_DECAP actions ++ which results in L3 decapsulation. ++ Not supported on egress. ++ c. Any encapsulation action, including the combination of RAW_ENCAP and RAW_DECAP actions ++ which results in L3 encap. ++ d. Only in transfer (switchdev) mode. ++ + - When using Verbs flow engine (``dv_flow_en`` = 0), flow pattern without any + specific VLAN will match for VLAN packets as well: + +@@ -708,8 +728,8 @@ Limitations + + - Cannot co-exist with ASO meter, ASO age action in a single flow rule. + - Flow rules insertion rate and memory consumption need more optimization. +- - 256 ports maximum. +- - 4M connections maximum with ``dv_flow_en`` 1 mode. 16M with ``dv_flow_en`` 2. ++ - 16 ports maximum. ++ - 32M connections maximum. + + - Multi-thread flow insertion: + +diff --git a/dpdk/doc/guides/nics/nfp.rst b/dpdk/doc/guides/nics/nfp.rst +index fee1860f4a..b577229bda 100644 +--- a/dpdk/doc/guides/nics/nfp.rst ++++ b/dpdk/doc/guides/nics/nfp.rst +@@ -299,10 +299,6 @@ be wrote N times in the heads. It is the same with NFD3. + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | Data for field 5 | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +- | Data for field 6 | +- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +- | Data for field 7 | +- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | Packet Data | + | ... | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +diff --git a/dpdk/doc/guides/platform/cnxk.rst b/dpdk/doc/guides/platform/cnxk.rst +index 70065e3d96..0e61bc91d9 100644 +--- a/dpdk/doc/guides/platform/cnxk.rst ++++ b/dpdk/doc/guides/platform/cnxk.rst +@@ -250,9 +250,9 @@ Debugging Options + +---+------------+-------------------------------------------------------+ + | # | Component | EAL log command | + +===+============+=======================================================+ +- | 1 | Common | --log-level='pmd\.cnxk\.base,8' | ++ | 1 | Common | --log-level='pmd\.common\.cnxk\.base,8' | + +---+------------+-------------------------------------------------------+ +- | 2 | Mailbox | --log-level='pmd\.cnxk\.mbox,8' | ++ | 2 | Mailbox | --log-level='pmd\.common\.cnxk\.mbox,8' | + +---+------------+-------------------------------------------------------+ + + Debugfs support +diff --git a/dpdk/doc/guides/platform/mlx5.rst b/dpdk/doc/guides/platform/mlx5.rst +index 400000e284..d64599699e 100644 +--- a/dpdk/doc/guides/platform/mlx5.rst ++++ b/dpdk/doc/guides/platform/mlx5.rst +@@ -230,7 +230,7 @@ DevX SDK Installation + The DevX SDK must be installed on the machine building the Windows PMD. + Additional information can be found at + `How to Integrate Windows DevX in Your Development Environment +-`_. ++`_. + The minimal supported WinOF2 version is 2.60. + + +diff --git a/dpdk/doc/guides/prog_guide/img/mbuf1.svg b/dpdk/doc/guides/prog_guide/img/mbuf1.svg +index a08bf3b6c0..111a874c00 100644 +--- a/dpdk/doc/guides/prog_guide/img/mbuf1.svg ++++ b/dpdk/doc/guides/prog_guide/img/mbuf1.svg +@@ -487,7 +487,7 @@ + sodipodi:role="line" + id="tspan5256" + x="59.842155" +- y="282.37683">m->pkt.next = NULL ++ y="282.37683">m->next = NULL + m->pkt.next = NULL ++ y="628.45935">m->next = NULL + m->pkt.next = mseg3 ++ y="628.45935">m->next = mseg3 + m->pkt.next = mseg2 ++ y="628.45935">m->next = mseg2 + l2_len = len(out_eth) + mb->l3_len = len(out_ip) +- mb->ol_flags |= RTE_MBUF_F_TX_IPV4 | RTE_MBUF_F_TX_IP_CSUM ++ mb->ol_flags |= RTE_MBUF_F_TX_IPV4 | RTE_MBUF_F_TX_IP_CKSUM + set out_ip checksum to 0 in the packet + + This is supported on hardware advertising RTE_ETH_TX_OFFLOAD_IPV4_CKSUM. +@@ -143,7 +143,7 @@ a vxlan-encapsulated tcp packet: + + mb->l2_len = len(out_eth) + mb->l3_len = len(out_ip) +- mb->ol_flags |= RTE_MBUF_F_TX_IPV4 | RTE_MBUF_F_TX_IP_CSUM | RTE_MBUF_F_TX_UDP_CKSUM ++ mb->ol_flags |= RTE_MBUF_F_TX_IPV4 | RTE_MBUF_F_TX_IP_CKSUM | RTE_MBUF_F_TX_UDP_CKSUM + set out_ip checksum to 0 in the packet + set out_udp checksum to pseudo header using rte_ipv4_phdr_cksum() + +@@ -154,7 +154,7 @@ a vxlan-encapsulated tcp packet: + + mb->l2_len = len(out_eth + out_ip + out_udp + vxlan + in_eth) + mb->l3_len = len(in_ip) +- mb->ol_flags |= RTE_MBUF_F_TX_IPV4 | RTE_MBUF_F_TX_IP_CSUM ++ mb->ol_flags |= RTE_MBUF_F_TX_IPV4 | RTE_MBUF_F_TX_IP_CKSUM + set in_ip checksum to 0 in the packet + + This is similar to case 1), but l2_len is different. It is supported +@@ -165,7 +165,7 @@ a vxlan-encapsulated tcp packet: + + mb->l2_len = len(out_eth + out_ip + out_udp + vxlan + in_eth) + mb->l3_len = len(in_ip) +- mb->ol_flags |= RTE_MBUF_F_TX_IPV4 | RTE_MBUF_F_TX_IP_CSUM | RTE_MBUF_F_TX_TCP_CKSUM ++ mb->ol_flags |= RTE_MBUF_F_TX_IPV4 | RTE_MBUF_F_TX_IP_CKSUM | RTE_MBUF_F_TX_TCP_CKSUM + set in_ip checksum to 0 in the packet + set in_tcp checksum to pseudo header using rte_ipv4_phdr_cksum() + +diff --git a/dpdk/doc/guides/prog_guide/packet_framework.rst b/dpdk/doc/guides/prog_guide/packet_framework.rst +index ebc69d8c3e..9987ead6c5 100644 +--- a/dpdk/doc/guides/prog_guide/packet_framework.rst ++++ b/dpdk/doc/guides/prog_guide/packet_framework.rst +@@ -509,7 +509,7 @@ the number of L2 or L3 cache memory misses is greatly reduced, hence one of the + This is because the cost of L2/L3 cache memory miss on memory read accesses is high, as usually due to data dependency between instructions, + the CPU execution units have to stall until the read operation is completed from L3 cache memory or external DRAM memory. + By using prefetch instructions, the latency of memory read accesses is hidden, +-provided that it is preformed early enough before the respective data structure is actually used. ++provided that it is performed early enough before the respective data structure is actually used. + + By splitting the processing into several stages that are executed on different packets (the packets from the input burst are interlaced), + enough work is created to allow the prefetch instructions to complete successfully (before the prefetched data structures are actually accessed) and +diff --git a/dpdk/doc/guides/prog_guide/profile_app.rst b/dpdk/doc/guides/prog_guide/profile_app.rst +index 14292d4c25..a6b5fb4d5e 100644 +--- a/dpdk/doc/guides/prog_guide/profile_app.rst ++++ b/dpdk/doc/guides/prog_guide/profile_app.rst +@@ -59,7 +59,7 @@ addition to the standard events, ``perf`` can be used to profile arm64 + specific PMU (Performance Monitor Unit) events through raw events (``-e`` + ``-rXX``). + +-For more derails refer to the ++For more details refer to the + `ARM64 specific PMU events enumeration `_. + + +diff --git a/dpdk/doc/guides/rel_notes/deprecation.rst b/dpdk/doc/guides/rel_notes/deprecation.rst +index 81b93515cb..10630ba255 100644 +--- a/dpdk/doc/guides/rel_notes/deprecation.rst ++++ b/dpdk/doc/guides/rel_notes/deprecation.rst +@@ -27,10 +27,6 @@ Deprecation Notices + * kvargs: The function ``rte_kvargs_process`` will get a new parameter + for returning key match count. It will ease handling of no-match case. + +-* cmdline: The function ``cmdline_poll`` does not work correctly on either +- Linux or Windows and is unused by any part of DPDK. +- This function is now deprecated and will be removed in DPDK 23.11. +- + * telemetry: The functions ``rte_tel_data_add_array_u64`` and ``rte_tel_data_add_dict_u64``, + used by telemetry callbacks for adding unsigned integer values to be returned to the user, + are renamed to ``rte_tel_data_add_array_uint`` and ``rte_tel_data_add_dict_uint`` respectively. +diff --git a/dpdk/doc/guides/rel_notes/release_23_11.rst b/dpdk/doc/guides/rel_notes/release_23_11.rst +index 6d83682d73..538e06aede 100644 +--- a/dpdk/doc/guides/rel_notes/release_23_11.rst ++++ b/dpdk/doc/guides/rel_notes/release_23_11.rst +@@ -669,3 +669,744 @@ Tested Platforms + * OFED: + + * MLNX_OFED 23.07-0.5.1.2 ++ ++23.11.1 Release Notes ++--------------------- ++ ++ ++23.11.1 Fixes ++~~~~~~~~~~~~~ ++ ++* 23.11.1-rc1 ++* app/crypto-perf: add missing op resubmission ++* app/crypto-perf: fix copy segment size ++* app/crypto-perf: fix data comparison ++* app/crypto-perf: fix encrypt operation verification ++* app/crypto-perf: fix next segment mbuf ++* app/crypto-perf: fix out-of-place mbuf size ++* app/crypto-perf: verify strdup return ++* app/dma-perf: verify strdup return ++* app/dumpcap: verify strdup return ++* app/graph: fix build reason ++* app/pdump: verify strdup return ++* app/testpmd: fix --stats-period option check ++* app/testpmd: fix GRO packets flush on timeout ++* app/testpmd: fix async flow create failure handling ++* app/testpmd: fix async indirect action list creation ++* app/testpmd: fix auto-completion for indirect action list ++* app/testpmd: fix burst option parsing ++* app/testpmd: fix crash in multi-process forwarding ++* app/testpmd: fix error message for invalid option ++* app/testpmd: fix flow modify tag typo ++* app/testpmd: hide --bitrate-stats in help if disabled ++* app/testpmd: return if no packets in GRO heavy weight mode ++* app/testpmd: verify strdup return ++* build: fix linker warnings about undefined symbols ++* build: fix reasons conflict ++* build: link static libs with whole-archive in subproject ++* build: pass cflags in subproject ++* buildtools/cmdline: fix IP address initializer ++* buildtools/cmdline: fix generated code for IP addresses ++* bus/dpaa: verify strdup return ++* bus/fslmc: verify strdup return ++* bus/vdev: fix devargs in secondary process ++* bus/vdev: verify strdup return ++* ci: update versions of actions in GHA ++* common/cnxk: fix RSS RETA configuration ++* common/cnxk: fix Tx MTU configuration ++* common/cnxk: fix VLAN check for inner header ++* common/cnxk: fix inline device pointer check ++* common/cnxk: fix link config for SDP ++* common/cnxk: fix mbox region copy ++* common/cnxk: fix mbox struct attributes ++* common/cnxk: fix memory leak in CPT init ++* common/cnxk: fix possible out-of-bounds access ++* common/cnxk: remove CN9K inline IPsec FP opcodes ++* common/cnxk: remove dead code ++* common/mlx5: fix calloc parameters ++* common/mlx5: fix duplicate read of general capabilities ++* common/mlx5: fix query sample info capability ++* common/qat: fix legacy flag ++* common/sfc_efx/base: use C11 static assert ++* config: fix CPU instruction set for cross-build ++* crypto/cnxk: fix CN9K ECDH public key verification ++* crypto/qat: fix crash with CCM null AAD pointer ++* cryptodev: remove unused extern variable ++* dma/dpaa2: fix logtype register ++* dma/idxd: verify strdup return ++* dmadev: fix calloc parameters ++* doc: add --latencystats option in testpmd guide ++* doc: add link speeds configuration in features table ++* doc: add traffic manager in features table ++* doc: fix aging poll frequency option in cnxk guide ++* doc: fix commands in eventdev test tool guide ++* doc: fix configuration in baseband 5GNR driver guide ++* doc: fix default IP fragments maximum in programmer guide ++* doc: fix typo in packet framework guide ++* doc: fix typo in profiling guide ++* doc: fix typos in cryptodev overview ++* doc: remove cmdline polling mode deprecation notice ++* doc: update link to Windows DevX in mlx5 guide ++* drivers/net: fix buffer overflow for packet types list ++* dts: fix smoke tests driver regex ++* dts: strip whitespaces from stdout and stderr ++* eal/x86: add AMD vendor check for TSC calibration ++* eal: verify strdup return ++* ethdev: fix NVGRE encap flow action description ++* event/cnxk: fix dequeue timeout configuration ++* event/cnxk: verify strdup return ++* event/dlb2: remove superfluous memcpy ++* eventdev/crypto: fix enqueueing ++* eventdev: fix Doxygen processing of vector struct ++* eventdev: fix calloc parameters ++* eventdev: improve Doxygen comments on configure struct ++* examples/ipsec-secgw: fix Rx queue ID in Rx callback ++* examples/ipsec-secgw: fix cryptodev to SA mapping ++* examples/ipsec-secgw: fix typo in error message ++* examples/ipsec-secgw: fix width of variables ++* examples/l3fwd: fix Rx over not ready port ++* examples/l3fwd: fix Rx queue configuration ++* examples/packet_ordering: fix Rx with reorder mode disabled ++* examples/qos_sched: fix memory leak in args parsing ++* examples/vhost: verify strdup return ++* gro: fix reordering of packets ++* hash: remove some dead code ++* kernel/freebsd: fix module build on FreeBSD 14 ++* lib: add newline in logs ++* lib: remove redundant newline from logs ++* lib: use dedicated logtypes and macros ++* ml/cnxk: fix xstats calculation ++* net/af_xdp: fix leak on XSK configuration failure ++* net/af_xdp: fix memzone leak on config failure ++* net/bnx2x: fix calloc parameters ++* net/bnx2x: fix warnings about memcpy lengths ++* net/bnxt: fix 50G and 100G forced speed ++* net/bnxt: fix array overflow ++* net/bnxt: fix backward firmware compatibility ++* net/bnxt: fix deadlock in ULP timer callback ++* net/bnxt: fix null pointer dereference ++* net/bnxt: fix number of Tx queues being created ++* net/bnxt: fix speed change from 200G to 25G on Thor ++* net/bnxt: modify locking for representor Tx ++* net/bonding: fix flow count query ++* net/cnxk: add cookies check for multi-segment offload ++* net/cnxk: fix MTU limit ++* net/cnxk: fix Rx packet format check condition ++* net/cnxk: fix aged flow query ++* net/cnxk: fix buffer size configuration ++* net/cnxk: fix flow RSS configuration ++* net/cnxk: fix indirect mbuf handling in Tx ++* net/cnxk: fix mbuf fields in multi-segment Tx ++* net/cnxk: improve Tx performance for SW mbuf free ++* net/ena/base: fix metrics excessive memory consumption ++* net/ena/base: limit exponential backoff ++* net/ena/base: restructure interrupt handling ++* net/ena: fix fast mbuf free ++* net/ena: fix mbuf double free in fast free mode ++* net/failsafe: fix memory leak in args parsing ++* net/gve: fix DQO for chained descriptors ++* net/hns3: enable PFC for all user priorities ++* net/hns3: fix VF multiple count on one reset ++* net/hns3: fix disable command with firmware ++* net/hns3: fix reset level comparison ++* net/hns3: refactor PF mailbox message struct ++* net/hns3: refactor VF mailbox message struct ++* net/hns3: refactor handle mailbox function ++* net/hns3: refactor send mailbox function ++* net/hns3: remove QinQ insert support for VF ++* net/hns3: support new device ++* net/i40e: remove incorrect 16B descriptor read block ++* net/i40e: remove redundant judgment in flow parsing ++* net/iavf: fix crash on VF start ++* net/iavf: fix memory leak on security context error ++* net/iavf: fix no polling mode switching ++* net/iavf: remove error logs for VLAN offloading ++* net/iavf: remove incorrect 16B descriptor read block ++* net/ice: fix link update ++* net/ice: fix memory leaks ++* net/ice: fix tunnel TSO capabilities ++* net/ice: remove incorrect 16B descriptor read block ++* net/igc: fix timesync disable ++* net/ionic: fix RSS query ++* net/ionic: fix device close ++* net/ionic: fix missing volatile type for cqe pointers ++* net/ixgbe: fix memoy leak after device init failure ++* net/ixgbe: increase VF reset timeout ++* net/ixgbevf: fix RSS init for x550 NICs ++* net/mana: fix memory leak on MR allocation ++* net/mana: handle MR cache expansion failure ++* net/mana: prevent values overflow returned from RDMA layer ++* net/memif: fix crash with Tx burst larger than 255 ++* net/memif: fix extra mbuf refcnt update in zero copy Tx ++* net/mlx5/hws: check not supported fields in VXLAN ++* net/mlx5/hws: enable multiple integrity items ++* net/mlx5/hws: fix ESP flow matching validation ++* net/mlx5/hws: fix VLAN inner type ++* net/mlx5/hws: fix VLAN item in non-relaxed mode ++* net/mlx5/hws: fix direct index insert on depend WQE ++* net/mlx5/hws: fix memory access in L3 decapsulation ++* net/mlx5/hws: fix port ID for root table ++* net/mlx5/hws: fix tunnel protocol checks ++* net/mlx5/hws: skip item when inserting rules by index ++* net/mlx5: fix DR context release ordering ++* net/mlx5: fix GENEVE TLV option management ++* net/mlx5: fix GENEVE option item translation ++* net/mlx5: fix HWS meter actions availability ++* net/mlx5: fix HWS registers initialization ++* net/mlx5: fix IP-in-IP tunnels recognition ++* net/mlx5: fix VLAN ID in flow modify ++* net/mlx5: fix VLAN handling in meter split ++* net/mlx5: fix age position in hairpin split ++* net/mlx5: fix async flow create error handling ++* net/mlx5: fix condition of LACP miss flow ++* net/mlx5: fix connection tracking action validation ++* net/mlx5: fix conntrack action handle representation ++* net/mlx5: fix counters map in bonding mode ++* net/mlx5: fix drop action release timing ++* net/mlx5: fix error packets drop in regular Rx ++* net/mlx5: fix flow action template expansion ++* net/mlx5: fix flow configure validation ++* net/mlx5: fix flow counter cache starvation ++* net/mlx5: fix flow tag modification ++* net/mlx5: fix indirect action async job initialization ++* net/mlx5: fix jump action validation ++* net/mlx5: fix meter policy priority ++* net/mlx5: fix modify flex item ++* net/mlx5: fix non-masked indirect list meter translation ++* net/mlx5: fix parameters verification in HWS table create ++* net/mlx5: fix rollback on failed flow configure ++* net/mlx5: fix stats query crash in secondary process ++* net/mlx5: fix sync flow meter action ++* net/mlx5: fix sync meter processing in HWS ++* net/mlx5: fix template clean up of FDB control flow rule ++* net/mlx5: fix use after free when releasing Tx queues ++* net/mlx5: fix warning about copy length ++* net/mlx5: prevent ioctl failure log flooding ++* net/mlx5: prevent querying aged flows on uninit port ++* net/mlx5: remove GENEVE options length limitation ++* net/mlx5: remove device status check in flow creation ++* net/mlx5: remove duplication of L3 flow item validation ++* net/netvsc: fix VLAN metadata parsing ++* net/nfp: fix IPsec data endianness ++* net/nfp: fix NFD3 metadata process ++* net/nfp: fix NFDk metadata process ++* net/nfp: fix Rx descriptor ++* net/nfp: fix Rx memory leak ++* net/nfp: fix calloc parameters ++* net/nfp: fix device close ++* net/nfp: fix device resource freeing ++* net/nfp: fix getting firmware VNIC version ++* net/nfp: fix initialization failure flow ++* net/nfp: fix resource leak for CoreNIC firmware ++* net/nfp: fix resource leak for PF initialization ++* net/nfp: fix resource leak for VF ++* net/nfp: fix resource leak for device initialization ++* net/nfp: fix resource leak for exit of CoreNIC firmware ++* net/nfp: fix resource leak for exit of flower firmware ++* net/nfp: fix resource leak for flower firmware ++* net/nfp: fix switch domain free check ++* net/nfp: fix uninitialized variable ++* net/nfp: free switch domain ID on close ++* net/nfp: verify strdup return ++* net/sfc: fix calloc parameters ++* net/softnic: fix include of log library ++* net/tap: do not overwrite flow API errors ++* net/tap: fix traffic control handle calculation ++* net/thunderx: fix DMAC control register update ++* net/virtio: fix vDPA device init advertising control queue ++* net/virtio: remove duplicate queue xstats ++* net/vmxnet3: fix initialization on FreeBSD ++* net/vmxnet3: ignore Rx queue interrupt setup on FreeBSD ++* net: add macros for VLAN metadata parsing ++* net: fix TCP/UDP checksum with padding data ++* pipeline: fix calloc parameters ++* rawdev: fix calloc parameters ++* rcu: fix acked token in debug log ++* rcu: use atomic operation on acked token ++* regexdev: fix logtype register ++* telemetry: fix connected clients count ++* telemetry: fix empty JSON dictionaries ++* test/cfgfile: fix typo in error messages ++* test/event: fix crash in Tx adapter freeing ++* test/event: skip test if no driver is present ++* test/mbuf: fix external mbuf case with assert enabled ++* test/power: fix typo in error message ++* test: assume C source files are UTF-8 encoded ++* test: do not count skipped tests as executed ++* test: fix probing in secondary process ++* test: verify strdup return ++* vdpa/mlx5: fix queue enable drain CQ ++* version: 23.11.1-rc2 ++* vhost: fix VDUSE device destruction failure ++* vhost: fix deadlock during vDPA SW live migration ++* vhost: fix memory leak in Virtio Tx split path ++* vhost: fix virtqueue access check in VDUSE setup ++* vhost: fix virtqueue access check in datapath ++* vhost: fix virtqueue access check in vhost-user setup ++ ++23.11.1 Validation ++~~~~~~~~~~~~~~~~~~ ++ ++* RedHat Testing: ++ ++ * Test scenarios: ++ ++ * VM with device assignment(PF) throughput testing(1G hugepage size) ++ * VM with device assignment(PF) throughput testing(2M hugepage size) ++ * VM with device assignment(VF) throughput testing ++ * PVP (host dpdk testpmd as vswitch) 1Q: throughput testing ++ * PVP vhost-user 2Q throughput testing ++ * PVP vhost-user 1Q - cross numa node throughput testing ++ * VM with vhost-user 2 queues throughput testing ++ * vhost-user reconnect with dpdk-client, qemu-server qemu reconnect ++ * vhost-user reconnect with dpdk-client, qemu-server ovs reconnect ++ * PVP reconnect with dpdk-client, qemu-server ++ * PVP 1Q live migration testing ++ * PVP 1Q cross numa node live migration testing ++ * VM with ovs+dpdk+vhost-user 1Q live migration testing ++ * VM with ovs+dpdk+vhost-user 1Q live migration testing (2M) ++ * VM with ovs+dpdk+vhost-user 2Q live migration testing ++ * VM with ovs+dpdk+vhost-user 4Q live migration testing ++ * Host PF + DPDK testing ++ * Host VF + DPDK testing ++ ++ * Test Versions and device: ++ ++ * RHEL 9.4 ++ * qemu-kvm-8.2.0 ++ * kernel 5.14 ++ * libvirt 10.0 ++ * X540-AT2 NIC(ixgbe, 10G) ++ ++* Nvidia(R) Testing: ++ ++ * Test scenarios: ++ ++ * Send and receive multiple types of traffic. ++ * testpmd xstats counter test. ++ * testpmd timestamp test. ++ * Changing/checking link status through testpmd. ++ * rte_flow tests (https://doc.dpdk.org/guides/nics/mlx5.html#supported-hardware-offloads) ++ * RSS tests. ++ * VLAN filtering, stripping, and insertion tests. ++ * Checksum and TSO tests. ++ * ptype tests. ++ * link_status_interrupt example application tests. ++ * l3fwd-power example application tests. ++ * Multi-process example applications tests. ++ * Hardware LRO tests. ++ * Buffer Split tests. ++ * Tx scheduling tests. ++ ++ * Test platform: ++ ++ * NIC: ConnectX-6 Dx / OS: Ubuntu 20.04 / Driver: MLNX_OFED_LINUX-24.04-0.6.6.0 / Firmware: 22.41.1000 ++ * NIC: ConnectX-7 / OS: Ubuntu 20.04 / Driver: MLNX_OFED_LINUX-24.04-0.6.6.0 / Firmware: 28.41.1000 ++ * DPU: BlueField-2 / DOCA SW version: 2.7.0 / Firmware: 24.41.1000 ++ ++ * OS/driver combinations: ++ ++ * Debian 12 with MLNX_OFED_LINUX-24.01-0.3.3.1. ++ * Ubuntu 20.04.6 with MLNX_OFED_LINUX-24.01-0.3.3.1. ++ * Ubuntu 22.04.4 with MLNX_OFED_LINUX-24.04-0.6.6.0. ++ * Ubuntu 20.04.6 with rdma-core master (311c591). ++ * Ubuntu 20.04.6 with rdma-core v28.0. ++ * Fedora 40 with rdma-core v48.0. ++ * Fedora 41 (Rawhide) with rdma-core v51.0. ++ * OpenSUSE Leap 15.5 with rdma-core v42.0. ++ * Windows Server 2019 with Clang 16.0.6. ++ ++* Intel(R) Testing: ++ ++ * Basic NIC testing ++ ++ * Build & CFLAG compile: cover the build test combination with latest GCC/Clang version and the popular OS revision such as Ubuntu23.10, Ubuntu22.04, Fedora39, RHEL8.9, RHEL9.2, FreeBSD14.0, SUSE15, CentOS7.9, openEuler22.03-SP2，OpenAnolis8.8 etc. ++ * PF(i40e, ixgbe): test scenarios including RTE_FLOW/TSO/Jumboframe/checksum offload/VLAN/VXLAN, etc. ++ * VF(i40e, ixgbe): test scenarios including VF-RTE_FLOW/TSO/Jumboframe/checksum offload/VLAN/VXLAN, etc. ++ * PF/VF(ice): test scenarios including Switch features/Package Management/Flow Director/Advanced Tx/Advanced RSS/ACL/DCF/Flexible Descriptor, etc. ++ * Intel NIC single core/NIC performance: test scenarios including PF/VF single core performance test, etc. ++ * IPsec: test scenarios including ipsec/ipsec-gw/ipsec library basic test - QAT&SW/FIB library, etc. ++ ++ * Basic cryptodev and virtio testing ++ ++ * Virtio: both function and performance test are covered. Such as PVP/Virtio_loopback/virtio-user loopback/virtio-net VM2VM perf testing/VMAWARE ESXI 8.0, etc. ++ * Cryptodev Function test: Cryptodev API testing/CompressDev ISA-L/QAT/ZLIB PMD Testing/FIPS, etc. ++ * Cryptodev Performance test: test scenarios including Thoughput Performance/Cryptodev Latency, etc. ++ ++23.11.1 Known Issues ++~~~~~~~~~~~~~~~~~~~~ ++ ++* Start dpdk-pdump in VM with virtio-0.95 protocol failed ++ ++ Fix available in upstream. ++ ++23.11.2 Release Notes ++--------------------- ++ ++ ++23.11.2 Fixes ++~~~~~~~~~~~~~ ++ ++* app/bbdev: fix MLD output size computation ++* app/bbdev: fix TB logic ++* app/bbdev: fix interrupt tests ++* app/crypto-perf: fix result for asymmetric ++* app/crypto-perf: remove redundant local variable ++* app/dumpcap: handle SIGTERM and SIGHUP ++* app/pdump: handle SIGTERM and SIGHUP ++* app/testpmd: add postpone option to async flow destroy ++* app/testpmd: fix build on signed comparison ++* app/testpmd: fix help string of BPF load command ++* app/testpmd: fix indirect action flush ++* app/testpmd: fix lcore ID restriction ++* app/testpmd: fix outer IP checksum offload ++* app/testpmd: fix parsing for connection tracking item ++* app/testpmd: handle IEEE1588 init failure ++* baseband/acc: fix memory barrier ++* baseband/la12xx: forbid secondary process ++* bpf: disable on 32-bit x86 ++* bpf: fix MOV instruction evaluation ++* bpf: fix load hangs with six IPv6 addresses ++* build: use builtin helper for python dependencies ++* buildtools: fix build with clang 17 and ASan ++* bus/dpaa: fix bus scan for DMA devices ++* bus/dpaa: fix memory leak in bus scan ++* bus/dpaa: remove redundant file descriptor check ++* bus/pci: fix FD in secondary process ++* bus/pci: fix UIO resource mapping in secondary process ++* bus/pci: fix build with musl 1.2.4 / Alpine 3.19 ++* bus/vdev: fix device reinitialization ++* common/cnxk: fix flow aging cleanup ++* common/cnxk: fix flow aging on application exit ++* common/cnxk: fix integer overflow ++* common/cnxk: fix segregation of logs based on module ++* common/dpaax/caamflib: fix PDCP AES-AES watchdog error ++* common/dpaax/caamflib: fix PDCP-SDAP watchdog error ++* common/dpaax: fix IOVA table cleanup ++* common/dpaax: fix node array overrun ++* common/idpf: fix PTP message validation ++* common/idpf: fix flex descriptor mask ++* common/mlx5: fix PRM structs ++* common/mlx5: fix unsigned/signed mismatch ++* common/mlx5: remove unneeded field when modify RQ table ++* config: fix warning for cross build with meson >= 1.3.0 ++* crypto/cnxk: fix ECDH public key verification ++* crypto/cnxk: fix minimal input normalization ++* crypto/cnxk: fix out-of-bound access ++* crypto/dpaa2_sec: fix event queue user context ++* crypto/dpaa_sec: fix IPsec descriptor ++* crypto/ipsec_mb: fix function comment ++* crypto/openssl: fix GCM and CCM thread unsafe contexts ++* crypto/openssl: make per-QP auth context clones ++* crypto/openssl: make per-QP cipher context clones ++* crypto/openssl: optimize 3DES-CTR context init ++* crypto/openssl: set cipher padding once ++* crypto/qat: fix GEN4 write ++* crypto/qat: fix log message typo ++* crypto/qat: fix placement of OOP offset ++* cryptodev: fix build without crypto callbacks ++* cryptodev: validate crypto callbacks from next node ++* devtools: fix symbol listing ++* dma/hisilicon: remove support for HIP09 platform ++* dma/idxd: fix setup with Ubuntu 24.04 ++* dmadev: fix structure alignment ++* doc: add baseline mode in l3fwd-power guide ++* doc: add power uncore in API index ++* doc: describe mlx5 HWS actions order ++* doc: fix AF_XDP device plugin howto ++* doc: fix DMA performance test invocation ++* doc: fix link to hugepage mapping from Linux guide ++* doc: fix mbuf flags ++* doc: fix testpmd ring size command ++* doc: fix typo in l2fwd-crypto guide ++* doc: remove empty section from testpmd guide ++* doc: remove reference to mbuf pkt field ++* doc: update AF_XDP device plugin repository ++* doc: update metadata description in nfp guide ++* eal/linux: lower log level on allocation attempt failure ++* eal/unix: support ZSTD compression for firmware ++* eal/windows: install sched.h file ++* eal: fix type in destructor macro for MSVC ++* ethdev: fix GENEVE option item conversion ++* ethdev: fix device init without socket-local memory ++* ethdev: fix strict aliasing in link up ++* event/sw: fix warning from useless snprintf ++* eventdev/crypto: fix opaque field handling ++* examples/fips_validation: fix dereference and out-of-bound ++* examples/ipsec-secgw: fix SA salt endianness ++* examples/ipsec-secgw: revert SA salt endianness ++* examples/l3fwd: fix crash in ACL mode for mixed traffic ++* examples/l3fwd: fix crash on multiple sockets ++* examples: fix lcore ID restriction ++* examples: fix port ID restriction ++* examples: fix queue ID restriction ++* fbarray: fix finding for unaligned length ++* fbarray: fix incorrect lookahead behavior ++* fbarray: fix incorrect lookbehind behavior ++* fbarray: fix lookahead ignore mask handling ++* fbarray: fix lookbehind ignore mask handling ++* graph: fix ID collisions ++* graph: fix mcore dispatch walk ++* graph: fix stats retrieval while destroying a graph ++* hash: check name when creating a hash ++* hash: fix RCU reclamation size ++* hash: fix return code description in Doxygen ++* latencystats: fix literal float suffix ++* malloc: fix multi-process wait condition handling ++* mbuf: fix dynamic fields copy ++* net/af_packet: align Rx/Tx structs to cache line ++* net/af_xdp: count mbuf allocation failures ++* net/af_xdp: fix multi-interface support for k8s ++* net/af_xdp: fix port ID in Rx mbuf ++* net/af_xdp: fix stats reset ++* net/af_xdp: remove unused local statistic ++* net/ark: fix index arithmetic ++* net/axgbe: check only minimum speed for cables ++* net/axgbe: delay AN timeout during KR training ++* net/axgbe: disable RRC for yellow carp devices ++* net/axgbe: disable interrupts during device removal ++* net/axgbe: enable PLL control for fixed PHY modes only ++* net/axgbe: fix MDIO access for non-zero ports and CL45 PHYs ++* net/axgbe: fix SFP codes check for DAC cables ++* net/axgbe: fix Tx flow on 30H HW ++* net/axgbe: fix connection for SFP+ active cables ++* net/axgbe: fix fluctuations for 1G Bel Fuse SFP ++* net/axgbe: fix linkup in PHY status ++* net/axgbe: reset link when link never comes back ++* net/axgbe: update DMA coherency values ++* net/bonding: fix failover time of LACP with mode 4 ++* net/cnxk: fix RSS config ++* net/cnxk: fix extbuf handling for multisegment packet ++* net/cnxk: fix outbound security with higher packet burst ++* net/cnxk: fix promiscuous state after MAC change ++* net/cnxk: update SA userdata and keep original cookie ++* net/cpfl: add checks on control queue messages ++* net/cpfl: fix 32-bit build ++* net/dpaa: forbid MTU configuration for shared interface ++* net/e1000/base: fix link power down ++* net/ena: fix bad checksum handling ++* net/ena: fix checksum handling ++* net/ena: fix return value check ++* net/fm10k: fix cleanup during init failure ++* net/gve: fix RSS hash endianness in DQO format ++* net/gve: fix Tx queue state on queue start ++* net/hns3: check Rx DMA address alignmnent ++* net/hns3: disable SCTP verification tag for RSS hash input ++* net/hns3: fix Rx timestamp flag ++* net/hns3: fix double free for Rx/Tx queue ++* net/hns3: fix offload flag of IEEE 1588 ++* net/hns3: fix uninitialized variable in FEC query ++* net/hns3: fix variable overflow ++* net/i40e: fix outer UDP checksum offload for X710 ++* net/iavf: fix VF reset when using DCF ++* net/iavf: remove outer UDP checksum offload for X710 VF ++* net/ice/base: fix GCS descriptor field offsets ++* net/ice/base: fix board type definition ++* net/ice/base: fix check for existing switch rule ++* net/ice/base: fix masking when reading context ++* net/ice/base: fix memory leak in firmware version check ++* net/ice/base: fix pointer to variable outside scope ++* net/ice/base: fix potential TLV length overflow ++* net/ice/base: fix preparing PHY for timesync command ++* net/ice/base: fix resource leak ++* net/ice/base: fix return type of bitmap hamming weight ++* net/ice/base: fix sign extension ++* net/ice/base: fix size when allocating children arrays ++* net/ice/base: fix temporary failures reading NVM ++* net/ice: fix VLAN stripping in double VLAN mode ++* net/ice: fix check for outer UDP checksum offload ++* net/ice: fix memory leaks in raw pattern parsing ++* net/ice: fix return value for raw pattern parsing ++* net/ionic: fix mbuf double-free when emptying array ++* net/ixgbe/base: fix 5G link speed reported on VF ++* net/ixgbe/base: fix PHY ID for X550 ++* net/ixgbe/base: revert advertising for X550 2.5G/5G ++* net/ixgbe: do not create delayed interrupt handler twice ++* net/ixgbe: do not update link status in secondary process ++* net/mana: fix uninitialized return value ++* net/mlx5/hws: add template match none flag ++* net/mlx5/hws: decrease log level for creation failure ++* net/mlx5/hws: extend tag saving for match and jumbo ++* net/mlx5/hws: fix action template dump ++* net/mlx5/hws: fix check of range templates ++* net/mlx5/hws: fix deletion of action vport ++* net/mlx5/hws: fix function comment ++* net/mlx5/hws: fix matcher reconnect ++* net/mlx5/hws: fix memory leak in modify header ++* net/mlx5/hws: fix port ID on root item convert ++* net/mlx5/hws: fix spinlock release on context open ++* net/mlx5/hws: remove unused variable ++* net/mlx5/hws: set default miss when replacing table ++* net/mlx5: break flow resource release loop ++* net/mlx5: fix Arm build with GCC 9.1 ++* net/mlx5: fix MTU configuration ++* net/mlx5: fix access to flow template operations ++* net/mlx5: fix crash on counter pool destroy ++* net/mlx5: fix disabling E-Switch default flow rules ++* net/mlx5: fix end condition of reading xstats ++* net/mlx5: fix flow template indirect action failure ++* net/mlx5: fix hash Rx queue release in flow sample ++* net/mlx5: fix indexed pool with invalid index ++* net/mlx5: fix shared Rx queue data access race ++* net/mlx5: fix start without duplicate flow patterns ++* net/mlx5: fix uplink port probing in bonding mode ++* net/mlx5: support jump in meter hierarchy ++* net/netvsc: fix MTU set ++* net/netvsc: use ethdev API to set VF MTU ++* net/nfp: adapt reverse sequence card ++* net/nfp: disable ctrl VNIC queues on close ++* net/nfp: fix IPv6 TTL and DSCP flow action ++* net/nfp: fix allocation of switch domain ++* net/nfp: fix configuration BAR ++* net/nfp: fix dereference of null pointer ++* net/nfp: fix disabling 32-bit build ++* net/nfp: fix firmware abnormal cleanup ++* net/nfp: fix flow mask table entry ++* net/nfp: fix getting firmware version ++* net/nfp: fix repeat disable port ++* net/nfp: fix representor port queue release ++* net/nfp: fix resource leak in secondary process ++* net/nfp: fix xstats for multi PF firmware ++* net/nfp: forbid offload flow rules with empty action list ++* net/nfp: remove redundant function call ++* net/nfp: remove unneeded logic for VLAN layer ++* net/ngbe: add special config for YT8531SH-CA PHY ++* net/ngbe: fix MTU range ++* net/ngbe: fix hotplug remove ++* net/ngbe: fix memory leaks ++* net/ngbe: keep PHY power down while device probing ++* net/tap: fix file descriptor check in isolated flow ++* net/txgbe: fix MTU range ++* net/txgbe: fix Rx interrupt ++* net/txgbe: fix Tx hang on queue disable ++* net/txgbe: fix VF promiscuous and allmulticast ++* net/txgbe: fix flow filters in VT mode ++* net/txgbe: fix hotplug remove ++* net/txgbe: fix memory leaks ++* net/txgbe: fix tunnel packet parsing ++* net/txgbe: reconfigure more MAC Rx registers ++* net/txgbe: restrict configuration of VLAN strip offload ++* net/virtio-user: add memcpy check ++* net/virtio-user: fix control queue allocation ++* net/virtio-user: fix control queue allocation for non-vDPA ++* net/virtio-user: fix control queue destruction ++* net/virtio-user: fix shadow control queue notification init ++* net/virtio: fix MAC table update ++* net/vmxnet3: add missing register command ++* net/vmxnet3: fix init logs ++* net: fix outer UDP checksum in Intel prepare helper ++* pcapng: add memcpy check ++* power: fix number of uncore frequencies ++* telemetry: fix connection parameter parsing ++* telemetry: lower log level on socket error ++* test/crypto: fix RSA cases in QAT suite ++* test/crypto: fix allocation comment ++* test/crypto: fix asymmetric capability test ++* test/crypto: fix enqueue/dequeue callback case ++* test/crypto: fix modex comparison ++* test/crypto: remove unused stats in setup ++* test/crypto: validate modex from first non-zero ++* v23.11.2-rc1 ++* vdpa/sfc: remove dead code ++* version: 23.11.2-rc2 ++* vhost: cleanup resubmit info before inflight setup ++* vhost: fix build with GCC 13 ++ ++23.11.2 Validation ++~~~~~~~~~~~~~~~~~~ ++ ++* RedHat Testing: ++ ++ * Test scenarios: ++ ++ * VM with device assignment(PF) throughput testing(1G hugepage size) ++ * VM with device assignment(PF) throughput testing(2M hugepage size) ++ * VM with device assignment(VF) throughput testing ++ * PVP (host dpdk testpmd as vswitch) 1Q: throughput testing ++ * PVP vhost-user 2Q throughput testing ++ * PVP vhost-user 1Q - cross numa node throughput testing ++ * VM with vhost-user 2 queues throughput testing ++ * vhost-user reconnect with dpdk-client, qemu-server qemu reconnect ++ * vhost-user reconnect with dpdk-client, qemu-server ovs reconnect ++ * PVP reconnect with dpdk-client, qemu-server ++ * PVP 1Q live migration testing ++ * PVP 1Q cross numa node live migration testing ++ * VM with ovs+dpdk+vhost-user 1Q live migration testing ++ * VM with ovs+dpdk+vhost-user 1Q live migration testing (2M) ++ * VM with ovs+dpdk+vhost-user 2Q live migration testing ++ * VM with ovs+dpdk+vhost-user 4Q live migration testing ++ * Host PF + DPDK testing ++ * Host VF + DPDK testing ++ ++ * Test Versions and device: ++ ++ * RHEL 9.4 ++ * qemu-kvm-8.2.0 ++ * kernel 5.14 ++ * libvirt 10.0 ++ * openvswitch 3.3 ++ * X540-AT2 NIC(ixgbe, 10G) ++ ++* Nvidia(R) Testing: ++ ++ * Test scenarios: ++ ++ * Send and receive multiple types of traffic. ++ * testpmd xstats counter test. ++ * testpmd timestamp test. ++ * Changing/checking link status through testpmd. ++ * rte_flow tests (https://doc.dpdk.org/guides/nics/mlx5.html#supported-hardware-offloads) ++ * RSS tests. ++ * VLAN filtering, stripping, and insertion tests. ++ * Checksum and TSO tests. ++ * ptype tests. ++ * link_status_interrupt example application tests. ++ * l3fwd-power example application tests. ++ * Multi-process example applications tests. ++ * Hardware LRO tests. ++ * Buffer Split tests. ++ * Tx scheduling tests. ++ ++ * Test platform: ++ ++ * NIC: ConnectX-6 Dx / OS: Ubuntu 20.04 / Driver: MLNX_OFED_LINUX-24.07-0.6.1.0 / Firmware: 22.42.1000 ++ * NIC: ConnectX-7 / OS: Ubuntu 20.04 / Driver: MLNX_OFED_LINUX-24.07-0.6.1.0 / Firmware: 28.42.1000 ++ * DPU: BlueField-2 / DOCA SW version: 2.8.0 / Firmware: 24.42.1000 ++ ++ * OS/driver combinations: ++ ++ * Debian 12 with MLNX_OFED_LINUX-24.04-0.7.0.0. ++ * Ubuntu 20.04.6 with MLNX_OFED_LINUX-24.07-0.6.1.0. ++ * Ubuntu 20.04.6 with rdma-core master (dd9c687). ++ * Ubuntu 20.04.6 with rdma-core v28.0. ++ * Fedora 40 with rdma-core v48.0. ++ * Fedora 42 (Rawhide) with rdma-core v51.0. ++ * OpenSUSE Leap 15.6 with rdma-core v49.1. ++ ++* Intel(R) Testing: ++ ++ * Basic NIC testing ++ ++ * Build & CFLAG compile: cover the build test combination with latest GCC/Clang version and the popular OS revision such as Ubuntu23.10, Ubuntu22.04, Fedora39, RHEL8.9, RHEL9.2, FreeBSD14.0, SUSE15, CentOS7.9, openEuler22.03-SP2，OpenAnolis8.8 etc. ++ * PF(i40e, ixgbe): test scenarios including RTE_FLOW/TSO/Jumboframe/checksum offload/VLAN/VXLAN, etc. ++ * VF(i40e, ixgbe): test scenarios including VF-RTE_FLOW/TSO/Jumboframe/checksum offload/VLAN/VXLAN, etc. ++ * PF/VF(ice): test scenarios including Switch features/Package Management/Flow Director/Advanced Tx/Advanced RSS/ACL/DCF/Flexible Descriptor, etc. ++ * Intel NIC single core/NIC performance: test scenarios including PF/VF single core performance test, etc. ++ * IPsec: test scenarios including ipsec/ipsec-gw/ipsec library basic test - QAT&SW/FIB library, etc. ++ ++ * Basic cryptodev and virtio testing ++ ++ * Virtio: both function and performance test are covered. Such as PVP/Virtio_loopback/virtio-user loopback/virtio-net VM2VM perf testing/VMAWARE ESXI 8.0, etc. ++ * Cryptodev Function test: Cryptodev API testing/CompressDev ISA-L/QAT/ZLIB PMD Testing/FIPS, etc. ++ * Cryptodev Performance test: test scenarios including Thoughput Performance/Cryptodev Latency, etc. ++ ++23.11.2 Known Issues ++~~~~~~~~~~~~~~~~~~~~ ++ ++* Start dpdk-pdump in VM with virtio-0.95 protocol failed ++ ++ Fix available in upstream. ++ ++* Failed to add vdev when launch dpdk-pdump with vdev secondary process ++ ++ Fix available in upstream. +diff --git a/dpdk/doc/guides/sample_app_ug/l2_forward_crypto.rst b/dpdk/doc/guides/sample_app_ug/l2_forward_crypto.rst +index ce49eab96f..7ff304d05c 100644 +--- a/dpdk/doc/guides/sample_app_ug/l2_forward_crypto.rst ++++ b/dpdk/doc/guides/sample_app_ug/l2_forward_crypto.rst +@@ -30,7 +30,7 @@ Compiling the Application + + To compile the sample application see :doc:`compiling`. + +-The application is located in the ``l2fwd-crypt`` sub-directory. ++The application is located in the ``l2fwd-crypto`` sub-directory. + + Running the Application + ----------------------- +diff --git a/dpdk/doc/guides/sample_app_ug/l3_forward_power_man.rst b/dpdk/doc/guides/sample_app_ug/l3_forward_power_man.rst +index 4a6f33bf4f..9c9684fea7 100644 +--- a/dpdk/doc/guides/sample_app_ug/l3_forward_power_man.rst ++++ b/dpdk/doc/guides/sample_app_ug/l3_forward_power_man.rst +@@ -280,6 +280,9 @@ will use automatic PMD power management. + This mode is limited to one queue per core, + and has three available power management schemes: + ++``baseline`` ++ This mode will not enable any power saving features. ++ + ``monitor`` + This will use ``rte_power_monitor()`` function to enter + a power-optimized state (subject to platform support). +diff --git a/dpdk/doc/guides/testpmd_app_ug/run_app.rst b/dpdk/doc/guides/testpmd_app_ug/run_app.rst +index 24a086401e..1a9b812a7f 100644 +--- a/dpdk/doc/guides/testpmd_app_ug/run_app.rst ++++ b/dpdk/doc/guides/testpmd_app_ug/run_app.rst +@@ -422,6 +422,10 @@ The command line options are: + + Set the logical core N to perform bitrate calculation. + ++* ``--latencystats=N`` ++ ++ Set the logical core N to perform latency and jitter calculations. ++ + * ``--print-event `` + + Enable printing the occurrence of the designated event. Using all will +diff --git a/dpdk/doc/guides/testpmd_app_ug/testpmd_funcs.rst b/dpdk/doc/guides/testpmd_app_ug/testpmd_funcs.rst +index 447e28e694..45d34d2632 100644 +--- a/dpdk/doc/guides/testpmd_app_ug/testpmd_funcs.rst ++++ b/dpdk/doc/guides/testpmd_app_ug/testpmd_funcs.rst +@@ -2033,7 +2033,7 @@ port config - queue ring size + + Configure a rx/tx queue ring size:: + +- testpmd> port (port_id) (rxq|txq) (queue_id) ring_size (value) ++ testpmd> port config (port_id) (rxq|txq) (queue_id) ring_size (value) + + Only take effect after command that (re-)start the port or command that setup specific queue. + +@@ -2910,14 +2910,6 @@ for port 0 and queue 0:: + + testpmd> set port cman config 0 0 obj queue mode red 10 100 1 + +-Filter Functions +----------------- +- +-This section details the available filter functions that are available. +- +-Note these functions interface the deprecated legacy filtering framework, +-superseded by *rte_flow*. See `Flow rules management`_. +- + .. _testpmd_rte_flow: + + Flow rules management +@@ -2927,10 +2919,6 @@ Control of the generic flow API (*rte_flow*) is fully exposed through the + ``flow`` command (configuration, validation, creation, destruction, queries + and operation modes). + +-Considering *rte_flow* overlaps with all `Filter Functions`_, using both +-features simultaneously may cause undefined side-effects and is therefore +-not recommended. +- + ``flow`` syntax + ~~~~~~~~~~~~~~~ + +diff --git a/dpdk/doc/guides/tools/dmaperf.rst b/dpdk/doc/guides/tools/dmaperf.rst +index 9e3e78a6b7..9fc77ca943 100644 +--- a/dpdk/doc/guides/tools/dmaperf.rst ++++ b/dpdk/doc/guides/tools/dmaperf.rst +@@ -119,7 +119,7 @@ Typical command-line invocation to execute the application: + + .. code-block:: console + +- dpdk-test-dma-perf --config=./config_dma.ini --result=./res_dma.csv ++ dpdk-test-dma-perf --config ./config_dma.ini --result ./res_dma.csv + + Where ``config_dma.ini`` is the configuration file, + and ``res_dma.csv`` will be the generated result file. +diff --git a/dpdk/doc/guides/tools/testeventdev.rst b/dpdk/doc/guides/tools/testeventdev.rst +index fc36bfb30c..3fcc2c9894 100644 +--- a/dpdk/doc/guides/tools/testeventdev.rst ++++ b/dpdk/doc/guides/tools/testeventdev.rst +@@ -308,7 +308,7 @@ Example command to run order queue test: + + .. code-block:: console + +- sudo /app/dpdk-test-eventdev --vdev=event_sw0 -- \ ++ sudo /app/dpdk-test-eventdev -c 0x1f -s 0x10 --vdev=event_sw0 -- \ + --test=order_queue --plcores 1 --wlcores 2,3 + + +@@ -371,7 +371,7 @@ Example command to run order ``all types queue`` test: + + .. code-block:: console + +- sudo /app/dpdk-test-eventdev --vdev=event_octeontx -- \ ++ sudo /app/dpdk-test-eventdev -c 0x1f -- \ + --test=order_atq --plcores 1 --wlcores 2,3 + + +@@ -475,14 +475,14 @@ Example command to run perf queue test: + + .. code-block:: console + +- sudo /app/dpdk-test-eventdev -c 0xf -s 0x1 --vdev=event_sw0 -- \ ++ sudo /app/dpdk-test-eventdev -c 0xf -s 0x2 --vdev=event_sw0 -- \ + --test=perf_queue --plcores=2 --wlcore=3 --stlist=p --nb_pkts=0 + + Example command to run perf queue test with producer enqueuing a burst of events: + + .. code-block:: console + +- sudo /app/dpdk-test-eventdev -c 0xf -s 0x1 --vdev=event_sw0 -- \ ++ sudo /app/dpdk-test-eventdev -c 0xf -s 0x2 --vdev=event_sw0 -- \ + --test=perf_queue --plcores=2 --wlcore=3 --stlist=p --nb_pkts=0 \ + --prod_enq_burst_sz=32 + +@@ -490,15 +490,15 @@ Example command to run perf queue test with ethernet ports: + + .. code-block:: console + +- sudo build/app/dpdk-test-eventdev --vdev=event_sw0 -- \ ++ sudo build/app/dpdk-test-eventdev -c 0xf -s 0x2 --vdev=event_sw0 -- \ + --test=perf_queue --plcores=2 --wlcore=3 --stlist=p --prod_type_ethdev + + Example command to run perf queue test with event timer adapter: + + .. code-block:: console + +- sudo /app/dpdk-test-eventdev --vdev="event_octeontx" -- \ +- --wlcores 4 --plcores 12 --test perf_queue --stlist=a \ ++ sudo /app/dpdk-test-eventdev -c 0xfff1 \ ++ -- --wlcores 4 --plcores 12 --test perf_queue --stlist=a \ + --prod_type_timerdev --fwd_latency + + PERF_ATQ Test +@@ -585,15 +585,15 @@ Example command to run perf ``all types queue`` test: + + .. code-block:: console + +- sudo /app/dpdk-test-eventdev --vdev=event_octeontx -- \ ++ sudo /app/dpdk-test-eventdev -c 0xf -- \ + --test=perf_atq --plcores=2 --wlcore=3 --stlist=p --nb_pkts=0 + + Example command to run perf ``all types queue`` test with event timer adapter: + + .. code-block:: console + +- sudo /app/dpdk-test-eventdev --vdev="event_octeontx" -- \ +- --wlcores 4 --plcores 12 --test perf_atq --verbose 20 \ ++ sudo /app/dpdk-test-eventdev -c 0xfff1 \ ++ -- --wlcores 4 --plcores 12 --test perf_atq --verbose 20 \ + --stlist=a --prod_type_timerdev --fwd_latency + + +@@ -817,13 +817,13 @@ Example command to run pipeline atq test: + + .. code-block:: console + +- sudo /app/dpdk-test-eventdev -c 0xf -s 0x8 --vdev=event_sw0 -- \ ++ sudo /app/dpdk-test-eventdev -c 0xf -- \ + --test=pipeline_atq --wlcore=1 --prod_type_ethdev --stlist=a + + Example command to run pipeline atq test with vector events: + + .. code-block:: console + +- sudo /app/dpdk-test-eventdev -c 0xf -s 0x8 --vdev=event_sw0 -- \ ++ sudo /app/dpdk-test-eventdev -c 0xf -- \ + --test=pipeline_atq --wlcore=1 --prod_type_ethdev --stlist=a \ + --enable_vector --vector_size 512 +diff --git a/dpdk/drivers/baseband/acc/acc_common.h b/dpdk/drivers/baseband/acc/acc_common.h +index bda2ad2f7a..6752c256d2 100644 +--- a/dpdk/drivers/baseband/acc/acc_common.h ++++ b/dpdk/drivers/baseband/acc/acc_common.h +@@ -1110,6 +1110,9 @@ acc_dma_enqueue(struct acc_queue *q, uint16_t n, + req_elem_addr, + (void *)q->mmio_reg_enqueue); + ++ q->aq_enqueued++; ++ q->sw_ring_head += enq_batch_size; ++ + rte_wmb(); + + /* Start time measurement for enqueue function offload. */ +@@ -1120,8 +1123,6 @@ acc_dma_enqueue(struct acc_queue *q, uint16_t n, + + queue_stats->acc_offload_cycles += rte_rdtsc_precise() - start_time; + +- q->aq_enqueued++; +- q->sw_ring_head += enq_batch_size; + n -= enq_batch_size; + + } while (n); +diff --git a/dpdk/drivers/baseband/la12xx/bbdev_la12xx.c b/dpdk/drivers/baseband/la12xx/bbdev_la12xx.c +index bb754a5395..1a56e73abd 100644 +--- a/dpdk/drivers/baseband/la12xx/bbdev_la12xx.c ++++ b/dpdk/drivers/baseband/la12xx/bbdev_la12xx.c +@@ -1084,6 +1084,9 @@ la12xx_bbdev_remove(struct rte_vdev_device *vdev) + + PMD_INIT_FUNC_TRACE(); + ++ if (rte_eal_process_type() != RTE_PROC_PRIMARY) ++ return 0; ++ + if (vdev == NULL) + return -EINVAL; + +diff --git a/dpdk/drivers/bus/dpaa/base/qbman/process.c b/dpdk/drivers/bus/dpaa/base/qbman/process.c +index 3504ec97db..3e4622f606 100644 +--- a/dpdk/drivers/bus/dpaa/base/qbman/process.c ++++ b/dpdk/drivers/bus/dpaa/base/qbman/process.c +@@ -1,7 +1,7 @@ + /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0) + * + * Copyright 2011-2016 Freescale Semiconductor Inc. +- * Copyright 2017,2020 NXP ++ * Copyright 2017,2020,2022,2024 NXP + * + */ + #include +@@ -27,15 +27,16 @@ static int check_fd(void) + { + int ret; + +- if (fd >= 0) +- return 0; + ret = pthread_mutex_lock(&fd_init_lock); + assert(!ret); ++ + /* check again with the lock held */ + if (fd < 0) + fd = open(PROCESS_PATH, O_RDWR); ++ + ret = pthread_mutex_unlock(&fd_init_lock); + assert(!ret); ++ + return (fd >= 0) ? 0 : -ENODEV; + } + +diff --git a/dpdk/drivers/bus/dpaa/dpaa_bus.c b/dpdk/drivers/bus/dpaa/dpaa_bus.c +index e57159f5d8..aaf2a5f43e 100644 +--- a/dpdk/drivers/bus/dpaa/dpaa_bus.c ++++ b/dpdk/drivers/bus/dpaa/dpaa_bus.c +@@ -187,6 +187,7 @@ dpaa_create_device_list(void) + if (dev->intr_handle == NULL) { + DPAA_BUS_LOG(ERR, "Failed to allocate intr handle"); + ret = -ENOMEM; ++ free(dev); + goto cleanup; + } + +@@ -220,7 +221,7 @@ dpaa_create_device_list(void) + + if (dpaa_sec_available()) { + DPAA_BUS_LOG(INFO, "DPAA SEC devices are not available"); +- return 0; ++ goto qdma_dpaa; + } + + /* Creating SEC Devices */ +@@ -238,6 +239,7 @@ dpaa_create_device_list(void) + if (dev->intr_handle == NULL) { + DPAA_BUS_LOG(ERR, "Failed to allocate intr handle"); + ret = -ENOMEM; ++ free(dev); + goto cleanup; + } + +@@ -259,6 +261,7 @@ dpaa_create_device_list(void) + + rte_dpaa_bus.device_count += i; + ++qdma_dpaa: + /* Creating QDMA Device */ + for (i = 0; i < RTE_DPAA_QDMA_DEVICES; i++) { + dev = calloc(1, sizeof(struct rte_dpaa_device)); +@@ -791,6 +794,10 @@ dpaa_bus_dev_iterate(const void *start, const char *str, + + /* Now that name=device_name format is available, split */ + dup = strdup(str); ++ if (dup == NULL) { ++ DPAA_BUS_DEBUG("Dup string (%s) failed!\n", str); ++ return NULL; ++ } + dev_name = dup + strlen("name="); + + if (start != NULL) { +diff --git a/dpdk/drivers/bus/fslmc/fslmc_bus.c b/dpdk/drivers/bus/fslmc/fslmc_bus.c +index 57bfb5111a..89f0f329c0 100644 +--- a/dpdk/drivers/bus/fslmc/fslmc_bus.c ++++ b/dpdk/drivers/bus/fslmc/fslmc_bus.c +@@ -634,6 +634,10 @@ fslmc_bus_dev_iterate(const void *start, const char *str, + + /* Now that name=device_name format is available, split */ + dup = strdup(str); ++ if (dup == NULL) { ++ DPAA2_BUS_DEBUG("Dup string (%s) failed!\n", str); ++ return NULL; ++ } + dev_name = dup + strlen("name="); + + if (start != NULL) { +diff --git a/dpdk/drivers/bus/pci/linux/pci_uio.c b/dpdk/drivers/bus/pci/linux/pci_uio.c +index 97d740dfe5..4afda97858 100644 +--- a/dpdk/drivers/bus/pci/linux/pci_uio.c ++++ b/dpdk/drivers/bus/pci/linux/pci_uio.c +@@ -237,7 +237,7 @@ pci_uio_alloc_resource(struct rte_pci_device *dev, + } + snprintf(devname, sizeof(devname), "/dev/uio%u", uio_num); + +- /* save fd if in primary process */ ++ /* save fd */ + fd = open(devname, O_RDWR); + if (fd < 0) { + RTE_LOG(ERR, EAL, "Cannot open %s: %s\n", +@@ -275,6 +275,9 @@ pci_uio_alloc_resource(struct rte_pci_device *dev, + } + } + ++ if (rte_eal_process_type() != RTE_PROC_PRIMARY) ++ return 0; ++ + /* allocate the mapping details for secondary processes*/ + *uio_res = rte_zmalloc("UIO_RES", sizeof(**uio_res), 0); + if (*uio_res == NULL) { +diff --git a/dpdk/drivers/bus/pci/linux/pci_vfio.c b/dpdk/drivers/bus/pci/linux/pci_vfio.c +index 3f3201daf2..baa0b9d9b3 100644 +--- a/dpdk/drivers/bus/pci/linux/pci_vfio.c ++++ b/dpdk/drivers/bus/pci/linux/pci_vfio.c +@@ -80,7 +80,7 @@ pci_vfio_read_config(const struct rte_pci_device *dev, + if ((uint64_t)len + offs > size) + return -1; + +- return pread64(fd, buf, len, offset + offs); ++ return pread(fd, buf, len, offset + offs); + } + + int +@@ -101,7 +101,7 @@ pci_vfio_write_config(const struct rte_pci_device *dev, + if ((uint64_t)len + offs > size) + return -1; + +- return pwrite64(fd, buf, len, offset + offs); ++ return pwrite(fd, buf, len, offset + offs); + } + + /* get PCI BAR number where MSI-X interrupts are */ +@@ -155,7 +155,7 @@ pci_vfio_enable_bus_memory(struct rte_pci_device *dev, int dev_fd) + return -1; + } + +- ret = pread64(dev_fd, &cmd, sizeof(cmd), offset + RTE_PCI_COMMAND); ++ ret = pread(dev_fd, &cmd, sizeof(cmd), offset + RTE_PCI_COMMAND); + + if (ret != sizeof(cmd)) { + RTE_LOG(ERR, EAL, "Cannot read command from PCI config space!\n"); +@@ -166,7 +166,7 @@ pci_vfio_enable_bus_memory(struct rte_pci_device *dev, int dev_fd) + return 0; + + cmd |= RTE_PCI_COMMAND_MEMORY; +- ret = pwrite64(dev_fd, &cmd, sizeof(cmd), offset + RTE_PCI_COMMAND); ++ ret = pwrite(dev_fd, &cmd, sizeof(cmd), offset + RTE_PCI_COMMAND); + + if (ret != sizeof(cmd)) { + RTE_LOG(ERR, EAL, "Cannot write command to PCI config space!\n"); +@@ -425,7 +425,7 @@ pci_vfio_is_ioport_bar(const struct rte_pci_device *dev, int vfio_dev_fd, + return -1; + } + +- ret = pread64(vfio_dev_fd, &ioport_bar, sizeof(ioport_bar), ++ ret = pread(vfio_dev_fd, &ioport_bar, sizeof(ioport_bar), + offset + RTE_PCI_BASE_ADDRESS_0 + bar_index * 4); + if (ret != sizeof(ioport_bar)) { + RTE_LOG(ERR, EAL, "Cannot read command (%x) from config space!\n", +@@ -1250,7 +1250,7 @@ pci_vfio_ioport_read(struct rte_pci_ioport *p, + if (vfio_dev_fd < 0) + return; + +- if (pread64(vfio_dev_fd, data, ++ if (pread(vfio_dev_fd, data, + len, p->base + offset) <= 0) + RTE_LOG(ERR, EAL, + "Can't read from PCI bar (%" PRIu64 ") : offset (%x)\n", +@@ -1267,7 +1267,7 @@ pci_vfio_ioport_write(struct rte_pci_ioport *p, + if (vfio_dev_fd < 0) + return; + +- if (pwrite64(vfio_dev_fd, data, ++ if (pwrite(vfio_dev_fd, data, + len, p->base + offset) <= 0) + RTE_LOG(ERR, EAL, + "Can't write to PCI bar (%" PRIu64 ") : offset (%x)\n", +@@ -1298,7 +1298,7 @@ pci_vfio_mmio_read(const struct rte_pci_device *dev, int bar, + if ((uint64_t)len + offs > size) + return -1; + +- return pread64(fd, buf, len, offset + offs); ++ return pread(fd, buf, len, offset + offs); + } + + int +@@ -1318,7 +1318,7 @@ pci_vfio_mmio_write(const struct rte_pci_device *dev, int bar, + if ((uint64_t)len + offs > size) + return -1; + +- return pwrite64(fd, buf, len, offset + offs); ++ return pwrite(fd, buf, len, offset + offs); + } + + int +diff --git a/dpdk/drivers/bus/pci/pci_common_uio.c b/dpdk/drivers/bus/pci/pci_common_uio.c +index 76c661f054..a06378b239 100644 +--- a/dpdk/drivers/bus/pci/pci_common_uio.c ++++ b/dpdk/drivers/bus/pci/pci_common_uio.c +@@ -26,7 +26,7 @@ EAL_REGISTER_TAILQ(rte_uio_tailq) + static int + pci_uio_map_secondary(struct rte_pci_device *dev) + { +- int fd, i, j; ++ int fd, i = 0, j, res_idx; + struct mapped_pci_resource *uio_res; + struct mapped_pci_res_list *uio_res_list = + RTE_TAILQ_CAST(rte_uio_tailq.head, mapped_pci_res_list); +@@ -37,7 +37,15 @@ pci_uio_map_secondary(struct rte_pci_device *dev) + if (rte_pci_addr_cmp(&uio_res->pci_addr, &dev->addr)) + continue; + +- for (i = 0; i != uio_res->nb_maps; i++) { ++ /* Map all BARs */ ++ for (res_idx = 0; res_idx != PCI_MAX_RESOURCE; res_idx++) { ++ /* skip empty BAR */ ++ if (dev->mem_resource[res_idx].phys_addr == 0) ++ continue; ++ ++ if (i >= uio_res->nb_maps) ++ return -1; ++ + /* + * open devname, to mmap it + */ +@@ -71,7 +79,9 @@ pci_uio_map_secondary(struct rte_pci_device *dev) + } + return -1; + } +- dev->mem_resource[i].addr = mapaddr; ++ dev->mem_resource[res_idx].addr = mapaddr; ++ ++ i++; + } + return 0; + } +@@ -96,15 +106,15 @@ pci_uio_map_resource(struct rte_pci_device *dev) + if (rte_intr_dev_fd_set(dev->intr_handle, -1)) + return -1; + +- /* secondary processes - use already recorded details */ +- if (rte_eal_process_type() != RTE_PROC_PRIMARY) +- return pci_uio_map_secondary(dev); +- + /* allocate uio resource */ + ret = pci_uio_alloc_resource(dev, &uio_res); + if (ret) + return ret; + ++ /* secondary processes - use already recorded details */ ++ if (rte_eal_process_type() != RTE_PROC_PRIMARY) ++ return pci_uio_map_secondary(dev); ++ + /* Map all BARs */ + for (i = 0; i != PCI_MAX_RESOURCE; i++) { + /* skip empty BAR */ +@@ -220,6 +230,18 @@ pci_uio_unmap_resource(struct rte_pci_device *dev) + if (uio_res == NULL) + return; + ++ /* close fd */ ++ if (rte_intr_fd_get(dev->intr_handle) >= 0) ++ close(rte_intr_fd_get(dev->intr_handle)); ++ uio_cfg_fd = rte_intr_dev_fd_get(dev->intr_handle); ++ if (uio_cfg_fd >= 0) { ++ close(uio_cfg_fd); ++ rte_intr_dev_fd_set(dev->intr_handle, -1); ++ } ++ ++ rte_intr_fd_set(dev->intr_handle, -1); ++ rte_intr_type_set(dev->intr_handle, RTE_INTR_HANDLE_UNKNOWN); ++ + /* secondary processes - just free maps */ + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return pci_uio_unmap(uio_res); +@@ -231,16 +253,4 @@ pci_uio_unmap_resource(struct rte_pci_device *dev) + + /* free uio resource */ + rte_free(uio_res); +- +- /* close fd if in primary process */ +- if (rte_intr_fd_get(dev->intr_handle) >= 0) +- close(rte_intr_fd_get(dev->intr_handle)); +- uio_cfg_fd = rte_intr_dev_fd_get(dev->intr_handle); +- if (uio_cfg_fd >= 0) { +- close(uio_cfg_fd); +- rte_intr_dev_fd_set(dev->intr_handle, -1); +- } +- +- rte_intr_fd_set(dev->intr_handle, -1); +- rte_intr_type_set(dev->intr_handle, RTE_INTR_HANDLE_UNKNOWN); + } +diff --git a/dpdk/drivers/bus/vdev/vdev.c b/dpdk/drivers/bus/vdev/vdev.c +index 7974b27295..dcedd0d4a0 100644 +--- a/dpdk/drivers/bus/vdev/vdev.c ++++ b/dpdk/drivers/bus/vdev/vdev.c +@@ -247,6 +247,10 @@ alloc_devargs(const char *name, const char *args) + devargs->data = strdup(args); + else + devargs->data = strdup(""); ++ if (devargs->data == NULL) { ++ free(devargs); ++ return NULL; ++ } + devargs->args = devargs->data; + + ret = strlcpy(devargs->name, name, sizeof(devargs->name)); +@@ -259,6 +263,22 @@ alloc_devargs(const char *name, const char *args) + return devargs; + } + ++static struct rte_devargs * ++vdev_devargs_lookup(const char *name) ++{ ++ struct rte_devargs *devargs; ++ char dev_name[32]; ++ ++ RTE_EAL_DEVARGS_FOREACH("vdev", devargs) { ++ devargs->bus->parse(devargs->name, &dev_name); ++ if (strcmp(dev_name, name) == 0) { ++ VDEV_LOG(INFO, "devargs matched %s", dev_name); ++ return devargs; ++ } ++ } ++ return NULL; ++} ++ + static int + insert_vdev(const char *name, const char *args, + struct rte_vdev_device **p_dev, +@@ -271,7 +291,11 @@ insert_vdev(const char *name, const char *args, + if (name == NULL) + return -EINVAL; + +- devargs = alloc_devargs(name, args); ++ if (rte_eal_process_type() == RTE_PROC_PRIMARY) ++ devargs = alloc_devargs(name, args); ++ else ++ devargs = vdev_devargs_lookup(name); ++ + if (!devargs) + return -ENOMEM; + +@@ -283,7 +307,6 @@ insert_vdev(const char *name, const char *args, + + dev->device.bus = &rte_vdev_bus; + dev->device.numa_node = SOCKET_ID_ANY; +- dev->device.name = devargs->name; + + if (find_vdev(name)) { + /* +@@ -298,6 +321,7 @@ insert_vdev(const char *name, const char *args, + if (init) + rte_devargs_insert(&devargs); + dev->device.devargs = devargs; ++ dev->device.name = devargs->name; + TAILQ_INSERT_TAIL(&vdev_device_list, dev, next); + + if (p_dev) +diff --git a/dpdk/drivers/common/cnxk/cnxk_security.c b/dpdk/drivers/common/cnxk/cnxk_security.c +index a8c3ba90cd..40685d0912 100644 +--- a/dpdk/drivers/common/cnxk/cnxk_security.c ++++ b/dpdk/drivers/common/cnxk/cnxk_security.c +@@ -618,235 +618,6 @@ cnxk_ot_ipsec_outb_sa_valid(struct roc_ot_ipsec_outb_sa *sa) + return !!sa->w2.s.valid; + } + +-static inline int +-ipsec_xfrm_verify(struct rte_security_ipsec_xform *ipsec_xfrm, +- struct rte_crypto_sym_xform *crypto_xfrm) +-{ +- if (crypto_xfrm->next == NULL) +- return -EINVAL; +- +- if (ipsec_xfrm->direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS) { +- if (crypto_xfrm->type != RTE_CRYPTO_SYM_XFORM_AUTH || +- crypto_xfrm->next->type != RTE_CRYPTO_SYM_XFORM_CIPHER) +- return -EINVAL; +- } else { +- if (crypto_xfrm->type != RTE_CRYPTO_SYM_XFORM_CIPHER || +- crypto_xfrm->next->type != RTE_CRYPTO_SYM_XFORM_AUTH) +- return -EINVAL; +- } +- +- return 0; +-} +- +-static int +-onf_ipsec_sa_common_param_fill(struct roc_ie_onf_sa_ctl *ctl, uint8_t *salt, +- uint8_t *cipher_key, uint8_t *hmac_opad_ipad, +- struct rte_security_ipsec_xform *ipsec_xfrm, +- struct rte_crypto_sym_xform *crypto_xfrm) +-{ +- struct rte_crypto_sym_xform *auth_xfrm, *cipher_xfrm; +- int rc, length, auth_key_len; +- const uint8_t *key = NULL; +- uint8_t ccm_flag = 0; +- +- /* Set direction */ +- switch (ipsec_xfrm->direction) { +- case RTE_SECURITY_IPSEC_SA_DIR_INGRESS: +- ctl->direction = ROC_IE_SA_DIR_INBOUND; +- auth_xfrm = crypto_xfrm; +- cipher_xfrm = crypto_xfrm->next; +- break; +- case RTE_SECURITY_IPSEC_SA_DIR_EGRESS: +- ctl->direction = ROC_IE_SA_DIR_OUTBOUND; +- cipher_xfrm = crypto_xfrm; +- auth_xfrm = crypto_xfrm->next; +- break; +- default: +- return -EINVAL; +- } +- +- /* Set protocol - ESP vs AH */ +- switch (ipsec_xfrm->proto) { +- case RTE_SECURITY_IPSEC_SA_PROTO_ESP: +- ctl->ipsec_proto = ROC_IE_SA_PROTOCOL_ESP; +- break; +- case RTE_SECURITY_IPSEC_SA_PROTO_AH: +- return -ENOTSUP; +- default: +- return -EINVAL; +- } +- +- /* Set mode - transport vs tunnel */ +- switch (ipsec_xfrm->mode) { +- case RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT: +- ctl->ipsec_mode = ROC_IE_SA_MODE_TRANSPORT; +- break; +- case RTE_SECURITY_IPSEC_SA_MODE_TUNNEL: +- ctl->ipsec_mode = ROC_IE_SA_MODE_TUNNEL; +- break; +- default: +- return -EINVAL; +- } +- +- /* Set encryption algorithm */ +- if (crypto_xfrm->type == RTE_CRYPTO_SYM_XFORM_AEAD) { +- length = crypto_xfrm->aead.key.length; +- +- switch (crypto_xfrm->aead.algo) { +- case RTE_CRYPTO_AEAD_AES_GCM: +- ctl->enc_type = ROC_IE_ON_SA_ENC_AES_GCM; +- ctl->auth_type = ROC_IE_ON_SA_AUTH_NULL; +- memcpy(salt, &ipsec_xfrm->salt, 4); +- key = crypto_xfrm->aead.key.data; +- break; +- case RTE_CRYPTO_AEAD_AES_CCM: +- ctl->enc_type = ROC_IE_ON_SA_ENC_AES_CCM; +- ctl->auth_type = ROC_IE_ON_SA_AUTH_NULL; +- ccm_flag = 0x07 & ~ROC_CPT_AES_CCM_CTR_LEN; +- *salt = ccm_flag; +- memcpy(PLT_PTR_ADD(salt, 1), &ipsec_xfrm->salt, 3); +- key = crypto_xfrm->aead.key.data; +- break; +- default: +- return -ENOTSUP; +- } +- +- } else { +- rc = ipsec_xfrm_verify(ipsec_xfrm, crypto_xfrm); +- if (rc) +- return rc; +- +- switch (cipher_xfrm->cipher.algo) { +- case RTE_CRYPTO_CIPHER_AES_CBC: +- ctl->enc_type = ROC_IE_ON_SA_ENC_AES_CBC; +- break; +- case RTE_CRYPTO_CIPHER_AES_CTR: +- ctl->enc_type = ROC_IE_ON_SA_ENC_AES_CTR; +- break; +- default: +- return -ENOTSUP; +- } +- +- switch (auth_xfrm->auth.algo) { +- case RTE_CRYPTO_AUTH_SHA1_HMAC: +- ctl->auth_type = ROC_IE_ON_SA_AUTH_SHA1; +- break; +- default: +- return -ENOTSUP; +- } +- auth_key_len = auth_xfrm->auth.key.length; +- if (auth_key_len < 20 || auth_key_len > 64) +- return -ENOTSUP; +- +- key = cipher_xfrm->cipher.key.data; +- length = cipher_xfrm->cipher.key.length; +- +- ipsec_hmac_opad_ipad_gen(auth_xfrm, hmac_opad_ipad); +- } +- +- switch (length) { +- case ROC_CPT_AES128_KEY_LEN: +- ctl->aes_key_len = ROC_IE_SA_AES_KEY_LEN_128; +- break; +- case ROC_CPT_AES192_KEY_LEN: +- ctl->aes_key_len = ROC_IE_SA_AES_KEY_LEN_192; +- break; +- case ROC_CPT_AES256_KEY_LEN: +- ctl->aes_key_len = ROC_IE_SA_AES_KEY_LEN_256; +- break; +- default: +- return -EINVAL; +- } +- +- memcpy(cipher_key, key, length); +- +- if (ipsec_xfrm->options.esn) +- ctl->esn_en = 1; +- +- ctl->spi = rte_cpu_to_be_32(ipsec_xfrm->spi); +- return 0; +-} +- +-int +-cnxk_onf_ipsec_inb_sa_fill(struct roc_onf_ipsec_inb_sa *sa, +- struct rte_security_ipsec_xform *ipsec_xfrm, +- struct rte_crypto_sym_xform *crypto_xfrm) +-{ +- struct roc_ie_onf_sa_ctl *ctl = &sa->ctl; +- int rc; +- +- rc = onf_ipsec_sa_common_param_fill(ctl, sa->nonce, sa->cipher_key, +- sa->hmac_key, ipsec_xfrm, +- crypto_xfrm); +- if (rc) +- return rc; +- +- rte_wmb(); +- +- /* Enable SA */ +- ctl->valid = 1; +- return 0; +-} +- +-int +-cnxk_onf_ipsec_outb_sa_fill(struct roc_onf_ipsec_outb_sa *sa, +- struct rte_security_ipsec_xform *ipsec_xfrm, +- struct rte_crypto_sym_xform *crypto_xfrm) +-{ +- struct rte_security_ipsec_tunnel_param *tunnel = &ipsec_xfrm->tunnel; +- struct roc_ie_onf_sa_ctl *ctl = &sa->ctl; +- int rc; +- +- /* Fill common params */ +- rc = onf_ipsec_sa_common_param_fill(ctl, sa->nonce, sa->cipher_key, +- sa->hmac_key, ipsec_xfrm, +- crypto_xfrm); +- if (rc) +- return rc; +- +- if (ipsec_xfrm->mode != RTE_SECURITY_IPSEC_SA_MODE_TUNNEL) +- goto skip_tunnel_info; +- +- /* Tunnel header info */ +- switch (tunnel->type) { +- case RTE_SECURITY_IPSEC_TUNNEL_IPV4: +- memcpy(&sa->ip_src, &tunnel->ipv4.src_ip, +- sizeof(struct in_addr)); +- memcpy(&sa->ip_dst, &tunnel->ipv4.dst_ip, +- sizeof(struct in_addr)); +- break; +- case RTE_SECURITY_IPSEC_TUNNEL_IPV6: +- return -ENOTSUP; +- default: +- return -EINVAL; +- } +- +- /* Update udp encap ports */ +- if (ipsec_xfrm->options.udp_encap == 1) { +- sa->udp_src = 4500; +- sa->udp_dst = 4500; +- } +- +-skip_tunnel_info: +- rte_wmb(); +- +- /* Enable SA */ +- ctl->valid = 1; +- return 0; +-} +- +-bool +-cnxk_onf_ipsec_inb_sa_valid(struct roc_onf_ipsec_inb_sa *sa) +-{ +- return !!sa->ctl.valid; +-} +- +-bool +-cnxk_onf_ipsec_outb_sa_valid(struct roc_onf_ipsec_outb_sa *sa) +-{ +- return !!sa->ctl.valid; +-} +- + uint8_t + cnxk_ipsec_ivlen_get(enum rte_crypto_cipher_algorithm c_algo, + enum rte_crypto_auth_algorithm a_algo, +diff --git a/dpdk/drivers/common/cnxk/cnxk_security.h b/dpdk/drivers/common/cnxk/cnxk_security.h +index 2277ce9144..72628ef3b8 100644 +--- a/dpdk/drivers/common/cnxk/cnxk_security.h ++++ b/dpdk/drivers/common/cnxk/cnxk_security.h +@@ -48,18 +48,6 @@ cnxk_ot_ipsec_outb_sa_fill(struct roc_ot_ipsec_outb_sa *sa, + bool __roc_api cnxk_ot_ipsec_inb_sa_valid(struct roc_ot_ipsec_inb_sa *sa); + bool __roc_api cnxk_ot_ipsec_outb_sa_valid(struct roc_ot_ipsec_outb_sa *sa); + +-/* [CN9K, CN10K) */ +-int __roc_api +-cnxk_onf_ipsec_inb_sa_fill(struct roc_onf_ipsec_inb_sa *sa, +- struct rte_security_ipsec_xform *ipsec_xfrm, +- struct rte_crypto_sym_xform *crypto_xfrm); +-int __roc_api +-cnxk_onf_ipsec_outb_sa_fill(struct roc_onf_ipsec_outb_sa *sa, +- struct rte_security_ipsec_xform *ipsec_xfrm, +- struct rte_crypto_sym_xform *crypto_xfrm); +-bool __roc_api cnxk_onf_ipsec_inb_sa_valid(struct roc_onf_ipsec_inb_sa *sa); +-bool __roc_api cnxk_onf_ipsec_outb_sa_valid(struct roc_onf_ipsec_outb_sa *sa); +- + /* [CN9K] */ + int __roc_api + cnxk_on_ipsec_inb_sa_create(struct rte_security_ipsec_xform *ipsec, +diff --git a/dpdk/drivers/common/cnxk/roc_ae.h b/dpdk/drivers/common/cnxk/roc_ae.h +index a9a08d9fb9..7886b9d107 100644 +--- a/dpdk/drivers/common/cnxk/roc_ae.h ++++ b/dpdk/drivers/common/cnxk/roc_ae.h +@@ -53,29 +53,31 @@ typedef enum { + ROC_AE_ERR_ECC_POINT_NOT_ON_CURVE = 0x11 + } roc_ae_error_code; + ++#define ROC_AE_EC_DATA_MAX 66 ++ + /* Prime and order fields of built-in elliptic curves */ + struct roc_ae_ec_group { + struct { + /* P521 maximum length */ +- uint8_t data[66]; ++ uint8_t data[ROC_AE_EC_DATA_MAX]; + unsigned int length; + } prime; + + struct { + /* P521 maximum length */ +- uint8_t data[66]; ++ uint8_t data[ROC_AE_EC_DATA_MAX]; + unsigned int length; + } order; + + struct { + /* P521 maximum length */ +- uint8_t data[66]; ++ uint8_t data[ROC_AE_EC_DATA_MAX]; + unsigned int length; + } consta; + + struct { + /* P521 maximum length */ +- uint8_t data[66]; ++ uint8_t data[ROC_AE_EC_DATA_MAX]; + unsigned int length; + } constb; + }; +@@ -86,18 +88,18 @@ struct roc_ae_ec_ctx { + + /* Private key */ + struct { +- uint8_t data[66]; ++ uint8_t data[ROC_AE_EC_DATA_MAX]; + unsigned int length; + } pkey; + + /* Public key */ + struct { + struct { +- uint8_t data[66]; ++ uint8_t data[ROC_AE_EC_DATA_MAX]; + unsigned int length; + } x; + struct { +- uint8_t data[66]; ++ uint8_t data[ROC_AE_EC_DATA_MAX]; + unsigned int length; + } y; + } q; +diff --git a/dpdk/drivers/common/cnxk/roc_cpt.c b/dpdk/drivers/common/cnxk/roc_cpt.c +index 981e85a204..4e23d8c135 100644 +--- a/dpdk/drivers/common/cnxk/roc_cpt.c ++++ b/dpdk/drivers/common/cnxk/roc_cpt.c +@@ -756,7 +756,7 @@ roc_cpt_dev_init(struct roc_cpt *roc_cpt) + rc = dev_init(dev, pci_dev); + if (rc) { + plt_err("Failed to init roc device"); +- goto fail; ++ return rc; + } + + cpt->pci_dev = pci_dev; +@@ -788,6 +788,7 @@ roc_cpt_dev_init(struct roc_cpt *roc_cpt) + return 0; + + fail: ++ dev_fini(dev, pci_dev); + return rc; + } + +diff --git a/dpdk/drivers/common/cnxk/roc_dev.c b/dpdk/drivers/common/cnxk/roc_dev.c +index e7e89bf3d6..14aff233d5 100644 +--- a/dpdk/drivers/common/cnxk/roc_dev.c ++++ b/dpdk/drivers/common/cnxk/roc_dev.c +@@ -198,9 +198,8 @@ af_pf_wait_msg(struct dev *dev, uint16_t vf, int num_msg) + vf_msg = mbox_alloc_msg(&dev->mbox_vfpf_up, vf, sz); + if (vf_msg) { + mbox_req_init(MBOX_MSG_CGX_LINK_EVENT, vf_msg); +- memcpy((uint8_t *)vf_msg + +- sizeof(struct mbox_msghdr), &linfo, +- sizeof(struct cgx_link_user_info)); ++ mbox_memcpy((uint8_t *)vf_msg + sizeof(struct mbox_msghdr), &linfo, ++ sizeof(struct cgx_link_user_info)); + + vf_msg->rc = msg->rc; + vf_msg->pcifunc = msg->pcifunc; +@@ -503,6 +502,8 @@ pf_vf_mbox_send_up_msg(struct dev *dev, void *rec_msg) + size_t size; + + size = PLT_ALIGN(mbox_id2size(msg->hdr.id), MBOX_MSG_ALIGN); ++ if (size < sizeof(struct mbox_msghdr)) ++ return; + /* Send UP message to all VF's */ + for (vf = 0; vf < vf_mbox->ndevs; vf++) { + /* VF active */ +diff --git a/dpdk/drivers/common/cnxk/roc_ie_on.h b/dpdk/drivers/common/cnxk/roc_ie_on.h +index 9933ffa148..11c995e9d1 100644 +--- a/dpdk/drivers/common/cnxk/roc_ie_on.h ++++ b/dpdk/drivers/common/cnxk/roc_ie_on.h +@@ -269,66 +269,6 @@ struct roc_ie_on_inb_sa { + #define ROC_IE_ON_UCC_L2_HDR_INFO_ERR 0xCF + #define ROC_IE_ON_UCC_L2_HDR_LEN_ERR 0xE0 + +-struct roc_ie_onf_sa_ctl { +- uint32_t spi; +- uint64_t exp_proto_inter_frag : 8; +- uint64_t rsvd_41_40 : 2; +- /* Disable SPI, SEQ data in RPTR for Inbound inline */ +- uint64_t spi_seq_dis : 1; +- uint64_t esn_en : 1; +- uint64_t rsvd_44_45 : 2; +- uint64_t encap_type : 2; +- uint64_t enc_type : 3; +- uint64_t rsvd_48 : 1; +- uint64_t auth_type : 4; +- uint64_t valid : 1; +- uint64_t direction : 1; +- uint64_t outer_ip_ver : 1; +- uint64_t inner_ip_ver : 1; +- uint64_t ipsec_mode : 1; +- uint64_t ipsec_proto : 1; +- uint64_t aes_key_len : 2; +-}; +- +-struct roc_onf_ipsec_outb_sa { +- /* w0 */ +- struct roc_ie_onf_sa_ctl ctl; +- +- /* w1 */ +- uint8_t nonce[4]; +- uint16_t udp_src; +- uint16_t udp_dst; +- +- /* w2 */ +- uint32_t ip_src; +- uint32_t ip_dst; +- +- /* w3-w6 */ +- uint8_t cipher_key[32]; +- +- /* w7-w12 */ +- uint8_t hmac_key[48]; +-}; +- +-struct roc_onf_ipsec_inb_sa { +- /* w0 */ +- struct roc_ie_onf_sa_ctl ctl; +- +- /* w1 */ +- uint8_t nonce[4]; /* Only for AES-GCM */ +- uint32_t unused; +- +- /* w2 */ +- uint32_t esn_hi; +- uint32_t esn_low; +- +- /* w3-w6 */ +- uint8_t cipher_key[32]; +- +- /* w7-w12 */ +- uint8_t hmac_key[48]; +-}; +- + #define ROC_ONF_IPSEC_INB_MAX_L2_SZ 32UL + #define ROC_ONF_IPSEC_OUTB_MAX_L2_SZ 30UL + #define ROC_ONF_IPSEC_OUTB_MAX_L2_INFO_SZ (ROC_ONF_IPSEC_OUTB_MAX_L2_SZ + 2) +diff --git a/dpdk/drivers/common/cnxk/roc_mbox.h b/dpdk/drivers/common/cnxk/roc_mbox.h +index 05434aec5a..7eaff0a0eb 100644 +--- a/dpdk/drivers/common/cnxk/roc_mbox.h ++++ b/dpdk/drivers/common/cnxk/roc_mbox.h +@@ -1420,12 +1420,12 @@ struct nix_cn10k_aq_enq_req { + struct nix_cn10k_aq_enq_rsp { + struct mbox_msghdr hdr; + union { +- struct nix_cn10k_rq_ctx_s rq; +- struct nix_cn10k_sq_ctx_s sq; +- struct nix_cq_ctx_s cq; +- struct nix_rsse_s rss; +- struct nix_rx_mce_s mce; +- struct nix_band_prof_s prof; ++ __io struct nix_cn10k_rq_ctx_s rq; ++ __io struct nix_cn10k_sq_ctx_s sq; ++ __io struct nix_cq_ctx_s cq; ++ __io struct nix_rsse_s rss; ++ __io struct nix_rx_mce_s mce; ++ __io struct nix_band_prof_s prof; + }; + }; + +@@ -1661,11 +1661,11 @@ struct nix_rq_cpt_field_mask_cfg_req { + #define RQ_CTX_MASK_MAX 6 + union { + uint64_t __io rq_ctx_word_set[RQ_CTX_MASK_MAX]; +- struct nix_cn10k_rq_ctx_s rq_set; ++ __io struct nix_cn10k_rq_ctx_s rq_set; + }; + union { + uint64_t __io rq_ctx_word_mask[RQ_CTX_MASK_MAX]; +- struct nix_cn10k_rq_ctx_s rq_mask; ++ __io struct nix_cn10k_rq_ctx_s rq_mask; + }; + struct nix_lf_rx_ipec_cfg1_req { + uint32_t __io spb_cpt_aura; +diff --git a/dpdk/drivers/common/cnxk/roc_nix.c b/dpdk/drivers/common/cnxk/roc_nix.c +index f64933a1d9..afbc3eb901 100644 +--- a/dpdk/drivers/common/cnxk/roc_nix.c ++++ b/dpdk/drivers/common/cnxk/roc_nix.c +@@ -482,7 +482,7 @@ skip_dev_init: + sdp_lbk_id_update(pci_dev, nix); + nix->pci_dev = pci_dev; + nix->reta_sz = reta_sz; +- nix->mtu = ROC_NIX_DEFAULT_HW_FRS; ++ nix->mtu = roc_nix_max_pkt_len(roc_nix); + nix->dmac_flt_idx = -1; + + /* Register error and ras interrupts */ +diff --git a/dpdk/drivers/common/cnxk/roc_nix.h b/dpdk/drivers/common/cnxk/roc_nix.h +index acdd1c4cbc..250d710c07 100644 +--- a/dpdk/drivers/common/cnxk/roc_nix.h ++++ b/dpdk/drivers/common/cnxk/roc_nix.h +@@ -267,8 +267,6 @@ struct roc_nix_eeprom_info { + #define ROC_NIX_RSS_KEY_LEN 48 /* 352 Bits */ + #define ROC_NIX_RSS_MCAM_IDX_DEFAULT (-1) + +-#define ROC_NIX_DEFAULT_HW_FRS 1514 +- + #define ROC_NIX_VWQE_MAX_SIZE_LOG2 11 + #define ROC_NIX_VWQE_MIN_SIZE_LOG2 2 + +diff --git a/dpdk/drivers/common/cnxk/roc_nix_inl.c b/dpdk/drivers/common/cnxk/roc_nix_inl.c +index 750fd08355..bc9cc2f429 100644 +--- a/dpdk/drivers/common/cnxk/roc_nix_inl.c ++++ b/dpdk/drivers/common/cnxk/roc_nix_inl.c +@@ -620,8 +620,7 @@ roc_nix_reassembly_configure(uint32_t max_wait_time, uint16_t max_frags) + return -EFAULT; + + PLT_SET_USED(max_frags); +- if (idev == NULL) +- return -ENOTSUP; ++ + roc_cpt = idev->cpt; + if (!roc_cpt) { + plt_err("Cannot support inline inbound, cryptodev not probed"); +@@ -876,7 +875,8 @@ roc_nix_inl_inb_init(struct roc_nix *roc_nix) + inl_dev = idev->nix_inl_dev; + + roc_nix->custom_meta_aura_ena = (roc_nix->local_meta_aura_ena && +- (inl_dev->is_multi_channel || roc_nix->custom_sa_action)); ++ ((inl_dev && inl_dev->is_multi_channel) || ++ roc_nix->custom_sa_action)); + if (!roc_model_is_cn9k() && !roc_errata_nix_no_meta_aura()) { + nix->need_meta_aura = true; + if (!roc_nix->local_meta_aura_ena || roc_nix->custom_meta_aura_ena) +diff --git a/dpdk/drivers/common/cnxk/roc_nix_inl.h b/dpdk/drivers/common/cnxk/roc_nix_inl.h +index ab1e9c0f98..f5ce26f03f 100644 +--- a/dpdk/drivers/common/cnxk/roc_nix_inl.h ++++ b/dpdk/drivers/common/cnxk/roc_nix_inl.h +@@ -4,24 +4,6 @@ + #ifndef _ROC_NIX_INL_H_ + #define _ROC_NIX_INL_H_ + +-/* ONF INB HW area */ +-#define ROC_NIX_INL_ONF_IPSEC_INB_HW_SZ \ +- PLT_ALIGN(sizeof(struct roc_onf_ipsec_inb_sa), ROC_ALIGN) +-/* ONF INB SW reserved area */ +-#define ROC_NIX_INL_ONF_IPSEC_INB_SW_RSVD 384 +-#define ROC_NIX_INL_ONF_IPSEC_INB_SA_SZ \ +- (ROC_NIX_INL_ONF_IPSEC_INB_HW_SZ + ROC_NIX_INL_ONF_IPSEC_INB_SW_RSVD) +-#define ROC_NIX_INL_ONF_IPSEC_INB_SA_SZ_LOG2 9 +- +-/* ONF OUTB HW area */ +-#define ROC_NIX_INL_ONF_IPSEC_OUTB_HW_SZ \ +- PLT_ALIGN(sizeof(struct roc_onf_ipsec_outb_sa), ROC_ALIGN) +-/* ONF OUTB SW reserved area */ +-#define ROC_NIX_INL_ONF_IPSEC_OUTB_SW_RSVD 128 +-#define ROC_NIX_INL_ONF_IPSEC_OUTB_SA_SZ \ +- (ROC_NIX_INL_ONF_IPSEC_OUTB_HW_SZ + ROC_NIX_INL_ONF_IPSEC_OUTB_SW_RSVD) +-#define ROC_NIX_INL_ONF_IPSEC_OUTB_SA_SZ_LOG2 8 +- + /* ON INB HW area */ + #define ROC_NIX_INL_ON_IPSEC_INB_HW_SZ \ + PLT_ALIGN(sizeof(struct roc_ie_on_inb_sa), ROC_ALIGN) +@@ -31,10 +13,10 @@ + (ROC_NIX_INL_ON_IPSEC_INB_HW_SZ + ROC_NIX_INL_ON_IPSEC_INB_SW_RSVD) + #define ROC_NIX_INL_ON_IPSEC_INB_SA_SZ_LOG2 10 + +-/* ONF OUTB HW area */ ++/* ON OUTB HW area */ + #define ROC_NIX_INL_ON_IPSEC_OUTB_HW_SZ \ + PLT_ALIGN(sizeof(struct roc_ie_on_outb_sa), ROC_ALIGN) +-/* ONF OUTB SW reserved area */ ++/* ON OUTB SW reserved area */ + #define ROC_NIX_INL_ON_IPSEC_OUTB_SW_RSVD 256 + #define ROC_NIX_INL_ON_IPSEC_OUTB_SA_SZ \ + (ROC_NIX_INL_ON_IPSEC_OUTB_HW_SZ + ROC_NIX_INL_ON_IPSEC_OUTB_SW_RSVD) +@@ -86,34 +68,6 @@ roc_nix_inl_on_ipsec_outb_sa_sw_rsvd(void *sa) + return PLT_PTR_ADD(sa, ROC_NIX_INL_ON_IPSEC_OUTB_HW_SZ); + } + +-static inline struct roc_onf_ipsec_inb_sa * +-roc_nix_inl_onf_ipsec_inb_sa(uintptr_t base, uint64_t idx) +-{ +- uint64_t off = idx << ROC_NIX_INL_ONF_IPSEC_INB_SA_SZ_LOG2; +- +- return PLT_PTR_ADD(base, off); +-} +- +-static inline struct roc_onf_ipsec_outb_sa * +-roc_nix_inl_onf_ipsec_outb_sa(uintptr_t base, uint64_t idx) +-{ +- uint64_t off = idx << ROC_NIX_INL_ONF_IPSEC_OUTB_SA_SZ_LOG2; +- +- return PLT_PTR_ADD(base, off); +-} +- +-static inline void * +-roc_nix_inl_onf_ipsec_inb_sa_sw_rsvd(void *sa) +-{ +- return PLT_PTR_ADD(sa, ROC_NIX_INL_ONF_IPSEC_INB_HW_SZ); +-} +- +-static inline void * +-roc_nix_inl_onf_ipsec_outb_sa_sw_rsvd(void *sa) +-{ +- return PLT_PTR_ADD(sa, ROC_NIX_INL_ONF_IPSEC_OUTB_HW_SZ); +-} +- + /* Inline device SSO Work callback */ + typedef void (*roc_nix_inl_sso_work_cb_t)(uint64_t *gw, void *args, + uint32_t soft_exp_event); +diff --git a/dpdk/drivers/common/cnxk/roc_nix_rss.c b/dpdk/drivers/common/cnxk/roc_nix_rss.c +index 3599eb9bae..2b88e1360d 100644 +--- a/dpdk/drivers/common/cnxk/roc_nix_rss.c ++++ b/dpdk/drivers/common/cnxk/roc_nix_rss.c +@@ -196,7 +196,7 @@ roc_nix_rss_reta_set(struct roc_nix *roc_nix, uint8_t group, + if (rc) + return rc; + +- memcpy(&nix->reta[group], reta, ROC_NIX_RSS_RETA_MAX); ++ memcpy(&nix->reta[group], reta, sizeof(uint16_t) * ROC_NIX_RSS_RETA_MAX); + return 0; + } + +@@ -209,7 +209,7 @@ roc_nix_rss_reta_get(struct roc_nix *roc_nix, uint8_t group, + if (group >= ROC_NIX_RSS_GRPS) + return NIX_ERR_PARAM; + +- memcpy(reta, &nix->reta[group], ROC_NIX_RSS_RETA_MAX); ++ memcpy(reta, &nix->reta[group], sizeof(uint16_t) * ROC_NIX_RSS_RETA_MAX); + return 0; + } + +diff --git a/dpdk/drivers/common/cnxk/roc_nix_tm.c b/dpdk/drivers/common/cnxk/roc_nix_tm.c +index ece88b5e99..9e5e614b3b 100644 +--- a/dpdk/drivers/common/cnxk/roc_nix_tm.c ++++ b/dpdk/drivers/common/cnxk/roc_nix_tm.c +@@ -328,6 +328,9 @@ nix_tm_bp_config_set(struct roc_nix *roc_nix, uint16_t sq, uint16_t tc, + uint8_t k = 0; + int rc = 0, i; + ++ if (roc_nix_is_sdp(roc_nix)) ++ return 0; ++ + sq_s = nix->sqs[sq]; + if (!sq_s) + return -ENOENT; +diff --git a/dpdk/drivers/common/cnxk/roc_npc.c b/dpdk/drivers/common/cnxk/roc_npc.c +index a0d88c0743..fcede1d0b7 100644 +--- a/dpdk/drivers/common/cnxk/roc_npc.c ++++ b/dpdk/drivers/common/cnxk/roc_npc.c +@@ -351,6 +351,8 @@ roc_npc_fini(struct roc_npc *roc_npc) + struct npc *npc = roc_npc_to_npc_priv(roc_npc); + int rc; + ++ npc_aging_ctrl_thread_destroy(roc_npc); ++ + rc = npc_flow_free_all_resources(npc); + if (rc) { + plt_err("Error when deleting NPC MCAM entries, counters"); +@@ -1626,8 +1628,7 @@ roc_npc_flow_destroy(struct roc_npc *roc_npc, struct roc_npc_flow *flow) + if (flow->has_age_action) + npc_age_flow_list_entry_delete(roc_npc, flow); + +- if (roc_npc->flow_age.age_flow_refcnt == 0 && +- plt_thread_is_valid(roc_npc->flow_age.aged_flows_poll_thread)) ++ if (roc_npc->flow_age.age_flow_refcnt == 0) + npc_aging_ctrl_thread_destroy(roc_npc); + + done: +diff --git a/dpdk/drivers/common/cnxk/roc_npc_aging.c b/dpdk/drivers/common/cnxk/roc_npc_aging.c +index 254dd2139b..258c15e341 100644 +--- a/dpdk/drivers/common/cnxk/roc_npc_aging.c ++++ b/dpdk/drivers/common/cnxk/roc_npc_aging.c +@@ -133,6 +133,21 @@ exit: + return rc; + } + ++static void ++npc_age_wait_until(struct roc_npc_flow_age *flow_age) ++{ ++#define NPC_AGE_WAIT_TIMEOUT_MS 1000 ++#define NPC_AGE_WAIT_TIMEOUT_US (NPC_AGE_WAIT_TIMEOUT_MS * NPC_AGE_WAIT_TIMEOUT_MS) ++ uint64_t timeout = 0; ++ uint64_t sleep = 10 * NPC_AGE_WAIT_TIMEOUT_MS; ++ ++ do { ++ plt_delay_us(sleep); ++ timeout += sleep; ++ } while (!flow_age->aged_flows_get_thread_exit && ++ (timeout < ((uint64_t)flow_age->aging_poll_freq * NPC_AGE_WAIT_TIMEOUT_US))); ++} ++ + uint32_t + npc_aged_flows_get(void *args) + { +@@ -197,7 +212,7 @@ npc_aged_flows_get(void *args) + plt_seqcount_write_end(&flow_age->seq_cnt); + + lbl_sleep: +- sleep(flow_age->aging_poll_freq); ++ npc_age_wait_until(flow_age); + } + + return 0; +@@ -234,8 +249,11 @@ npc_age_flow_list_entry_delete(struct roc_npc *roc_npc, + { + struct npc *npc = roc_npc_to_npc_priv(roc_npc); + struct npc_age_flow_list_head *list; ++ struct roc_npc_flow_age *flow_age; + struct npc_age_flow_entry *curr; + ++ flow_age = &roc_npc->flow_age; ++ + list = &npc->age_flow_list; + curr = TAILQ_FIRST(list); + +@@ -244,6 +262,7 @@ npc_age_flow_list_entry_delete(struct roc_npc *roc_npc, + + while (curr) { + if (flow->mcam_id == curr->flow->mcam_id) { ++ plt_bitmap_clear(flow_age->aged_flows, flow->mcam_id); + TAILQ_REMOVE(list, curr, next); + plt_free(curr); + break; +@@ -299,9 +318,11 @@ npc_aging_ctrl_thread_destroy(struct roc_npc *roc_npc) + struct roc_npc_flow_age *flow_age; + + flow_age = &roc_npc->flow_age; +- flow_age->aged_flows_get_thread_exit = true; +- plt_thread_join(flow_age->aged_flows_poll_thread, NULL); +- npc_aged_flows_bitmap_free(roc_npc); ++ if (plt_thread_is_valid(flow_age->aged_flows_poll_thread)) { ++ flow_age->aged_flows_get_thread_exit = true; ++ plt_thread_join(flow_age->aged_flows_poll_thread, NULL); ++ npc_aged_flows_bitmap_free(roc_npc); ++ } + } + + void * +diff --git a/dpdk/drivers/common/cnxk/roc_npc_parse.c b/dpdk/drivers/common/cnxk/roc_npc_parse.c +index ecd1b3e13b..3c288070fb 100644 +--- a/dpdk/drivers/common/cnxk/roc_npc_parse.c ++++ b/dpdk/drivers/common/cnxk/roc_npc_parse.c +@@ -1092,6 +1092,7 @@ npc_parse_lf(struct npc_parse_state *pst) + { + const struct roc_npc_item_info *pattern, *last_pattern; + char hw_mask[NPC_MAX_EXTRACT_HW_LEN]; ++ const struct roc_npc_flow_item_eth *eth_item; + struct npc_parse_item_info info; + int lid, lt, lflags; + int nr_vlans = 0; +@@ -1108,10 +1109,12 @@ npc_parse_lf(struct npc_parse_state *pst) + lt = NPC_LT_LF_TU_ETHER; + lflags = 0; + ++ eth_item = pst->pattern->spec; ++ + /* No match support for vlan tags */ + info.def_mask = NULL; + info.hw_mask = NULL; +- info.len = pst->pattern->size; ++ info.len = sizeof(eth_item->hdr); + info.spec = NULL; + info.mask = NULL; + info.hw_hdr_len = 0; +@@ -1142,12 +1145,15 @@ npc_parse_lf(struct npc_parse_state *pst) + } + + info.hw_mask = &hw_mask; +- info.len = pst->pattern->size; ++ info.len = sizeof(eth_item->hdr); + info.hw_hdr_len = 0; + npc_get_hw_supp_mask(pst, &info, lid, lt); + info.spec = NULL; + info.mask = NULL; + ++ if (eth_item && eth_item->has_vlan) ++ pst->set_vlan_ltype_mask = true; ++ + rc = npc_parse_item_basic(pst->pattern, &info); + if (rc != 0) + return rc; +diff --git a/dpdk/drivers/common/cnxk/roc_platform.c b/dpdk/drivers/common/cnxk/roc_platform.c +index 15cbb6d68f..80d81742a2 100644 +--- a/dpdk/drivers/common/cnxk/roc_platform.c ++++ b/dpdk/drivers/common/cnxk/roc_platform.c +@@ -85,15 +85,15 @@ roc_plt_init(void) + return 0; + } + +-RTE_LOG_REGISTER_DEFAULT(cnxk_logtype_base, NOTICE); +-RTE_LOG_REGISTER_DEFAULT(cnxk_logtype_mbox, NOTICE); +-RTE_LOG_REGISTER_DEFAULT(cnxk_logtype_cpt, NOTICE); +-RTE_LOG_REGISTER_DEFAULT(cnxk_logtype_ml, NOTICE); +-RTE_LOG_REGISTER_DEFAULT(cnxk_logtype_npa, NOTICE); +-RTE_LOG_REGISTER_DEFAULT(cnxk_logtype_nix, NOTICE); +-RTE_LOG_REGISTER_DEFAULT(cnxk_logtype_npc, NOTICE); +-RTE_LOG_REGISTER_DEFAULT(cnxk_logtype_sso, NOTICE); +-RTE_LOG_REGISTER_DEFAULT(cnxk_logtype_tim, NOTICE); +-RTE_LOG_REGISTER_DEFAULT(cnxk_logtype_tm, NOTICE); +-RTE_LOG_REGISTER_DEFAULT(cnxk_logtype_dpi, NOTICE); +-RTE_LOG_REGISTER_DEFAULT(cnxk_logtype_ree, NOTICE); ++RTE_LOG_REGISTER_SUFFIX(cnxk_logtype_base, base, NOTICE); ++RTE_LOG_REGISTER_SUFFIX(cnxk_logtype_mbox, mbox, NOTICE); ++RTE_LOG_REGISTER_SUFFIX(cnxk_logtype_cpt, crypto, NOTICE); ++RTE_LOG_REGISTER_SUFFIX(cnxk_logtype_ml, ml, NOTICE); ++RTE_LOG_REGISTER_SUFFIX(cnxk_logtype_npa, mempool, NOTICE); ++RTE_LOG_REGISTER_SUFFIX(cnxk_logtype_nix, nix, NOTICE); ++RTE_LOG_REGISTER_SUFFIX(cnxk_logtype_npc, flow, NOTICE); ++RTE_LOG_REGISTER_SUFFIX(cnxk_logtype_sso, event, NOTICE); ++RTE_LOG_REGISTER_SUFFIX(cnxk_logtype_tim, timer, NOTICE); ++RTE_LOG_REGISTER_SUFFIX(cnxk_logtype_tm, tm, NOTICE); ++RTE_LOG_REGISTER_SUFFIX(cnxk_logtype_dpi, dpi, NOTICE); ++RTE_LOG_REGISTER_SUFFIX(cnxk_logtype_ree, ree, NOTICE); +diff --git a/dpdk/drivers/common/cnxk/version.map b/dpdk/drivers/common/cnxk/version.map +index aa884a8fe2..e718c13acb 100644 +--- a/dpdk/drivers/common/cnxk/version.map ++++ b/dpdk/drivers/common/cnxk/version.map +@@ -17,10 +17,6 @@ INTERNAL { + cnxk_logtype_sso; + cnxk_logtype_tim; + cnxk_logtype_tm; +- cnxk_onf_ipsec_inb_sa_fill; +- cnxk_onf_ipsec_outb_sa_fill; +- cnxk_onf_ipsec_inb_sa_valid; +- cnxk_onf_ipsec_outb_sa_valid; + cnxk_ot_ipsec_inb_sa_fill; + cnxk_ot_ipsec_outb_sa_fill; + cnxk_ot_ipsec_inb_sa_valid; +diff --git a/dpdk/drivers/common/dpaax/caamflib/desc/ipsec.h b/dpdk/drivers/common/dpaax/caamflib/desc/ipsec.h +index 95fc3ea5ba..54fca3bc67 100644 +--- a/dpdk/drivers/common/dpaax/caamflib/desc/ipsec.h ++++ b/dpdk/drivers/common/dpaax/caamflib/desc/ipsec.h +@@ -731,6 +731,79 @@ static inline void __gen_auth_key(struct program *program, + authdata->key, authdata->key_type); + } + ++/** ++ * rta_inline_ipsec_query() - Provide indications on which data items can be inlined ++ * and which shall be referenced in IPsec shared descriptor. ++ * @sd_base_len: Shared descriptor base length - bytes consumed by the commands, ++ * excluding the data items to be inlined (or corresponding ++ * pointer if an item is not inlined). Each cnstr_* function that ++ * generates descriptors should have a define mentioning ++ * corresponding length. ++ * @jd_len: Maximum length of the job descriptor(s) that will be used ++ * together with the shared descriptor. ++ * @data_len: Array of lengths of the data items trying to be inlined ++ * @inl_mask: 32bit mask with bit x = 1 if data item x can be inlined, 0 ++ * otherwise. ++ * @count: Number of data items (size of @data_len array); must be <= 32 ++ * @auth_algtype: Authentication algorithm type. ++ * @auth_index: Index value of data_len for authentication key length. ++ * -1 if authentication key length is not present in data_len. ++ * ++ * Return: 0 if data can be inlined / referenced, negative value if not. If 0, ++ * check @inl_mask for details. ++ */ ++static inline int ++rta_inline_ipsec_query(unsigned int sd_base_len, ++ unsigned int jd_len, ++ unsigned int *data_len, ++ uint32_t *inl_mask, ++ unsigned int count, ++ uint32_t auth_algtype, ++ int32_t auth_index) ++{ ++ uint32_t dkp_protid; ++ ++ switch (auth_algtype & OP_PCL_IPSEC_AUTH_MASK) { ++ case OP_PCL_IPSEC_HMAC_MD5_96: ++ case OP_PCL_IPSEC_HMAC_MD5_128: ++ dkp_protid = OP_PCLID_DKP_MD5; ++ break; ++ case OP_PCL_IPSEC_HMAC_SHA1_96: ++ case OP_PCL_IPSEC_HMAC_SHA1_160: ++ dkp_protid = OP_PCLID_DKP_SHA1; ++ break; ++ case OP_PCL_IPSEC_HMAC_SHA2_256_128: ++ dkp_protid = OP_PCLID_DKP_SHA256; ++ break; ++ case OP_PCL_IPSEC_HMAC_SHA2_384_192: ++ dkp_protid = OP_PCLID_DKP_SHA384; ++ break; ++ case OP_PCL_IPSEC_HMAC_SHA2_512_256: ++ dkp_protid = OP_PCLID_DKP_SHA512; ++ break; ++ case OP_PCL_IPSEC_HMAC_SHA2_224_96: ++ case OP_PCL_IPSEC_HMAC_SHA2_224_112: ++ case OP_PCL_IPSEC_HMAC_SHA2_224_224: ++ dkp_protid = OP_PCLID_DKP_SHA224; ++ break; ++ default: ++ return rta_inline_query(sd_base_len, ++ jd_len, ++ data_len, ++ inl_mask, count); ++ } ++ ++ /* Updating the maximum supported inline key length */ ++ if (auth_index != -1) { ++ if (split_key_len(dkp_protid) > data_len[auth_index]) ++ data_len[auth_index] = split_key_len(dkp_protid); ++ } ++ return rta_inline_query(sd_base_len, ++ jd_len, ++ data_len, ++ inl_mask, count); ++} ++ + /** + * cnstr_shdsc_ipsec_encap - IPSec ESP encapsulation protocol-level shared + * descriptor. +diff --git a/dpdk/drivers/common/dpaax/caamflib/desc/pdcp.h b/dpdk/drivers/common/dpaax/caamflib/desc/pdcp.h +index 7d16c66d79..0ed9eec816 100644 +--- a/dpdk/drivers/common/dpaax/caamflib/desc/pdcp.h ++++ b/dpdk/drivers/common/dpaax/caamflib/desc/pdcp.h +@@ -1023,6 +1023,11 @@ pdcp_insert_uplane_aes_aes_op(struct program *p, + SEQFIFOLOAD(p, MSG1, 0, VLF | LAST1 | FLUSH1); + MOVEB(p, CONTEXT1, 0, MATH3, 0, 4, WAITCOMP | IMMED); + ++ /* conditional jump with calm added to ensure that the ++ * previous processing has been completed ++ */ ++ JUMP(p, 1, LOCAL_JUMP, ALL_TRUE, CALM); ++ + LOAD(p, CLRW_RESET_CLS1_CHA | + CLRW_CLR_C1KEY | + CLRW_CLR_C1CTX | +@@ -1070,6 +1075,11 @@ pdcp_insert_uplane_aes_aes_op(struct program *p, + + MOVEB(p, OFIFO, 0, MATH3, 0, 4, IMMED); + ++ /* conditional jump with calm added to ensure that the ++ * previous processing has been completed ++ */ ++ JUMP(p, 1, LOCAL_JUMP, ALL_TRUE, CALM); ++ + LOAD(p, CLRW_RESET_CLS1_CHA | + CLRW_CLR_C1KEY | + CLRW_CLR_C1CTX | +diff --git a/dpdk/drivers/common/dpaax/caamflib/desc/sdap.h b/dpdk/drivers/common/dpaax/caamflib/desc/sdap.h +index b38c15a24f..d41bacf8f9 100644 +--- a/dpdk/drivers/common/dpaax/caamflib/desc/sdap.h ++++ b/dpdk/drivers/common/dpaax/caamflib/desc/sdap.h +@@ -1,5 +1,5 @@ + /* SPDX-License-Identifier: BSD-3-Clause +- * Copyright 2020-2022 NXP ++ * Copyright 2020-2023 NXP + */ + + #ifndef __DESC_SDAP_H__ +@@ -628,6 +628,10 @@ static inline int pdcp_sdap_insert_no_snoop_op( + /* Save the ICV generated */ + MOVEB(p, CONTEXT1, 0, MATH3, 0, 4, WAITCOMP | IMMED); + ++ /* conditional jump with calm added to ensure that the ++ * previous processing has been completed ++ */ ++ JUMP(p, 1, LOCAL_JUMP, ALL_TRUE, CALM); + /* The CHA will be reused so we need to clear it */ + LOAD(p, CLRW_RESET_CLS1_CHA | + CLRW_CLR_C1KEY | +@@ -718,6 +722,10 @@ static inline int pdcp_sdap_insert_no_snoop_op( + /* Save the ICV which is stalling in output FIFO to MATH3 */ + MOVEB(p, OFIFO, 0, MATH3, 0, 4, IMMED); + ++ /* conditional jump with calm added to ensure that the ++ * previous processing has been completed ++ */ ++ JUMP(p, 1, LOCAL_JUMP, ALL_TRUE, CALM); + /* Reset class 1 CHA */ + LOAD(p, CLRW_RESET_CLS1_CHA | + CLRW_CLR_C1KEY | +diff --git a/dpdk/drivers/common/dpaax/dpaax_iova_table.c b/dpdk/drivers/common/dpaax/dpaax_iova_table.c +index 9daac4bc03..860e702333 100644 +--- a/dpdk/drivers/common/dpaax/dpaax_iova_table.c ++++ b/dpdk/drivers/common/dpaax/dpaax_iova_table.c +@@ -1,5 +1,5 @@ + /* SPDX-License-Identifier: BSD-3-Clause +- * Copyright 2018 NXP ++ * Copyright 2018-2023 NXP + */ + + #include +@@ -139,10 +139,12 @@ read_memory_node(unsigned int *count) + } + + DPAAX_DEBUG("Device-tree memory node data:"); +- do { ++ ++ while (j > 0) { ++ --j; + DPAAX_DEBUG(" %08" PRIx64 " %08zu", + nodes[j].addr, nodes[j].len); +- } while (--j); ++ } + + cleanup: + close(fd); +@@ -255,10 +257,7 @@ dpaax_iova_table_populate(void) + void + dpaax_iova_table_depopulate(void) + { +- if (dpaax_iova_table_p == NULL) +- return; +- +- rte_free(dpaax_iova_table_p->entries); ++ rte_free(dpaax_iova_table_p); + dpaax_iova_table_p = NULL; + + DPAAX_DEBUG("IOVA Table cleaned"); +diff --git a/dpdk/drivers/common/idpf/base/virtchnl2.h b/dpdk/drivers/common/idpf/base/virtchnl2.h +index 3900b784d0..21b2039aa2 100644 +--- a/dpdk/drivers/common/idpf/base/virtchnl2.h ++++ b/dpdk/drivers/common/idpf/base/virtchnl2.h +@@ -1872,7 +1872,7 @@ virtchnl2_vc_validate_vf_msg(__rte_unused struct virtchnl2_version_info *ver, u3 + case VIRTCHNL2_OP_GET_PTP_CAPS: + valid_len = sizeof(struct virtchnl2_get_ptp_caps); + +- if (msglen >= valid_len) { ++ if (msglen > valid_len) { + struct virtchnl2_get_ptp_caps *ptp_caps = + (struct virtchnl2_get_ptp_caps *)msg; + +@@ -1888,7 +1888,7 @@ virtchnl2_vc_validate_vf_msg(__rte_unused struct virtchnl2_version_info *ver, u3 + case VIRTCHNL2_OP_GET_PTP_TX_TSTAMP_LATCHES: + valid_len = sizeof(struct virtchnl2_ptp_tx_tstamp_latches); + +- if (msglen >= valid_len) { ++ if (msglen > valid_len) { + struct virtchnl2_ptp_tx_tstamp_latches *tx_tstamp_latches = + (struct virtchnl2_ptp_tx_tstamp_latches *)msg; + +diff --git a/dpdk/drivers/common/idpf/base/virtchnl2_lan_desc.h b/dpdk/drivers/common/idpf/base/virtchnl2_lan_desc.h +index e6e782a219..a5e3f05014 100644 +--- a/dpdk/drivers/common/idpf/base/virtchnl2_lan_desc.h ++++ b/dpdk/drivers/common/idpf/base/virtchnl2_lan_desc.h +@@ -110,7 +110,7 @@ + BIT_ULL(VIRTCHNL2_RX_FLEX_DESC_ADV_SPH_S) + #define VIRTCHNL2_RX_FLEX_DESC_ADV_FF1_S 12 + #define VIRTCHNL2_RX_FLEX_DESC_ADV_FF1_M \ +- IDPF_M(0x7UL, VIRTCHNL2_RX_FLEX_DESC_ADV_FF1_M) ++ IDPF_M(0x7UL, VIRTCHNL2_RX_FLEX_DESC_ADV_FF1_S) + #define VIRTCHNL2_RX_FLEX_DESC_ADV_MISS_S 15 + #define VIRTCHNL2_RX_FLEX_DESC_ADV_MISS_M \ + BIT_ULL(VIRTCHNL2_RX_FLEX_DESC_ADV_MISS_S) +diff --git a/dpdk/drivers/common/mlx5/mlx5_common_mr.c b/dpdk/drivers/common/mlx5/mlx5_common_mr.c +index 40ff9153bd..85ec10d2ee 100644 +--- a/dpdk/drivers/common/mlx5/mlx5_common_mr.c ++++ b/dpdk/drivers/common/mlx5/mlx5_common_mr.c +@@ -1381,7 +1381,7 @@ mlx5_mempool_get_chunks(struct rte_mempool *mp, struct mlx5_range **out, + + DRV_LOG(DEBUG, "Collecting chunks of regular mempool %s", mp->name); + n = mp->nb_mem_chunks; +- *out = calloc(sizeof(**out), n); ++ *out = calloc(n, sizeof(**out)); + if (*out == NULL) + return -1; + rte_mempool_mem_iter(mp, mlx5_range_from_mempool_chunk, *out); +diff --git a/dpdk/drivers/common/mlx5/mlx5_devx_cmds.c b/dpdk/drivers/common/mlx5/mlx5_devx_cmds.c +index 4d8818924a..630ab96a8f 100644 +--- a/dpdk/drivers/common/mlx5/mlx5_devx_cmds.c ++++ b/dpdk/drivers/common/mlx5/mlx5_devx_cmds.c +@@ -965,19 +965,9 @@ mlx5_devx_cmd_query_hca_attr(void *ctx, + max_geneve_tlv_options); + attr->max_geneve_tlv_option_data_len = MLX5_GET(cmd_hca_cap, hcattr, + max_geneve_tlv_option_data_len); ++ attr->query_match_sample_info = MLX5_GET(cmd_hca_cap, hcattr, ++ query_match_sample_info); + attr->qos.sup = MLX5_GET(cmd_hca_cap, hcattr, qos); +- attr->qos.flow_meter_aso_sup = !!(MLX5_GET64(cmd_hca_cap, hcattr, +- general_obj_types) & +- MLX5_GENERAL_OBJ_TYPES_CAP_FLOW_METER_ASO); +- attr->vdpa.valid = !!(MLX5_GET64(cmd_hca_cap, hcattr, +- general_obj_types) & +- MLX5_GENERAL_OBJ_TYPES_CAP_VIRTQ_NET_Q); +- attr->vdpa.queue_counters_valid = !!(MLX5_GET64(cmd_hca_cap, hcattr, +- general_obj_types) & +- MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_Q_COUNTERS); +- attr->parse_graph_flex_node = !!(MLX5_GET64(cmd_hca_cap, hcattr, +- general_obj_types) & +- MLX5_GENERAL_OBJ_TYPES_CAP_PARSE_GRAPH_FLEX_NODE); + attr->wqe_index_ignore = MLX5_GET(cmd_hca_cap, hcattr, + wqe_index_ignore_cap); + attr->cross_channel = MLX5_GET(cmd_hca_cap, hcattr, cd); +@@ -1001,6 +991,9 @@ mlx5_devx_cmd_query_hca_attr(void *ctx, + /* Read the general_obj_types bitmap and extract the relevant bits. */ + general_obj_types_supported = MLX5_GET64(cmd_hca_cap, hcattr, + general_obj_types); ++ attr->qos.flow_meter_aso_sup = ++ !!(general_obj_types_supported & ++ MLX5_GENERAL_OBJ_TYPES_CAP_FLOW_METER_ASO); + attr->vdpa.valid = !!(general_obj_types_supported & + MLX5_GENERAL_OBJ_TYPES_CAP_VIRTQ_NET_Q); + attr->vdpa.queue_counters_valid = +@@ -1074,8 +1067,7 @@ mlx5_devx_cmd_query_hca_attr(void *ctx, + MLX5_GET(cmd_hca_cap, hcattr, umr_modify_entity_size_disabled); + attr->wait_on_time = MLX5_GET(cmd_hca_cap, hcattr, wait_on_time); + attr->crypto = MLX5_GET(cmd_hca_cap, hcattr, crypto); +- attr->ct_offload = !!(MLX5_GET64(cmd_hca_cap, hcattr, +- general_obj_types) & ++ attr->ct_offload = !!(general_obj_types_supported & + MLX5_GENERAL_OBJ_TYPES_CAP_CONN_TRACK_OFFLOAD); + attr->rq_delay_drop = MLX5_GET(cmd_hca_cap, hcattr, rq_delay_drop); + attr->nic_flow_table = MLX5_GET(cmd_hca_cap, hcattr, nic_flow_table); +@@ -1104,8 +1096,8 @@ mlx5_devx_cmd_query_hca_attr(void *ctx, + (ctx, &attr->flex); + if (rc) + return -1; +- attr->flex.query_match_sample_info = MLX5_GET(cmd_hca_cap, hcattr, +- query_match_sample_info); ++ attr->flex.query_match_sample_info = ++ attr->query_match_sample_info; + } + if (attr->crypto) { + attr->aes_xts = MLX5_GET(cmd_hca_cap, hcattr, aes_xts) || +@@ -1814,7 +1806,7 @@ mlx5_devx_cmd_create_rqt(void *ctx, + uint32_t out[MLX5_ST_SZ_DW(create_rqt_out)] = {0}; + void *rqt_ctx; + struct mlx5_devx_obj *rqt = NULL; +- int i; ++ unsigned int i; + + in = mlx5_malloc(MLX5_MEM_ZERO, inlen, 0, SOCKET_ID_ANY); + if (!in) { +@@ -1867,7 +1859,7 @@ mlx5_devx_cmd_modify_rqt(struct mlx5_devx_obj *rqt, + uint32_t out[MLX5_ST_SZ_DW(modify_rqt_out)] = {0}; + uint32_t *in = mlx5_malloc(MLX5_MEM_ZERO, inlen, 0, SOCKET_ID_ANY); + void *rqt_ctx; +- int i; ++ unsigned int i; + int ret; + + if (!in) { +@@ -1880,7 +1872,6 @@ mlx5_devx_cmd_modify_rqt(struct mlx5_devx_obj *rqt, + MLX5_SET64(modify_rqt_in, in, modify_bitmask, 0x1); + rqt_ctx = MLX5_ADDR_OF(modify_rqt_in, in, rqt_context); + MLX5_SET(rqtc, rqt_ctx, list_q_type, rqt_attr->rq_type); +- MLX5_SET(rqtc, rqt_ctx, rqt_max_size, rqt_attr->rqt_max_size); + MLX5_SET(rqtc, rqt_ctx, rqt_actual_size, rqt_attr->rqt_actual_size); + for (i = 0; i < rqt_attr->rqt_actual_size; i++) + MLX5_SET(rqtc, rqt_ctx, rq_num[i], rqt_attr->rq_list[i]); +diff --git a/dpdk/drivers/common/mlx5/mlx5_devx_cmds.h b/dpdk/drivers/common/mlx5/mlx5_devx_cmds.h +index 7f23e925a5..b814c8becc 100644 +--- a/dpdk/drivers/common/mlx5/mlx5_devx_cmds.h ++++ b/dpdk/drivers/common/mlx5/mlx5_devx_cmds.h +@@ -315,6 +315,7 @@ struct mlx5_hca_attr { + uint32_t flow_counter_bulk_log_granularity:5; + uint32_t alloc_flow_counter_pd:1; + uint32_t flow_counter_access_aso:1; ++ uint32_t query_match_sample_info:1; + uint32_t flow_access_aso_opc_mod:8; + uint32_t cross_vhca:1; + uint32_t lag_rx_port_affinity:1; +diff --git a/dpdk/drivers/common/mlx5/mlx5_prm.h b/dpdk/drivers/common/mlx5/mlx5_prm.h +index 9e22dce6da..3cbb1179c0 100644 +--- a/dpdk/drivers/common/mlx5/mlx5_prm.h ++++ b/dpdk/drivers/common/mlx5/mlx5_prm.h +@@ -2334,8 +2334,8 @@ struct mlx5_ifc_cmd_hca_cap_2_bits { + u8 reserved_at_d0[0x3]; + u8 log_conn_track_max_alloc[0x5]; + u8 reserved_at_d8[0x3]; +- u8 log_max_conn_track_offload[0x5]; +- u8 cross_vhca_object_to_object_supported[0x20]; /* End of DW7. */ ++ u8 log_max_conn_track_offload[0x5]; /* End of DW7. */ ++ u8 cross_vhca_object_to_object_supported[0x20]; + u8 allowed_object_for_other_vhca_access_high[0x20]; + u8 allowed_object_for_other_vhca_access[0x20]; + u8 reserved_at_140[0x20]; +@@ -3606,7 +3606,7 @@ struct mlx5_ifc_stc_ste_param_vport_bits { + u8 eswitch_owner_vhca_id[0x10]; + u8 vport_number[0x10]; + u8 eswitch_owner_vhca_id_valid[0x1]; +- u8 reserved_at_21[0x59]; ++ u8 reserved_at_21[0x5f]; + }; + + union mlx5_ifc_stc_param_bits { +diff --git a/dpdk/drivers/common/qat/meson.build b/dpdk/drivers/common/qat/meson.build +index 5c36fbb270..62abcb6fe3 100644 +--- a/dpdk/drivers/common/qat/meson.build ++++ b/dpdk/drivers/common/qat/meson.build +@@ -17,13 +17,13 @@ qat_compress_relpath = '../../' + qat_compress_path + if disable_drivers.contains(qat_crypto_path) + qat_crypto = false + dpdk_drvs_disabled += qat_crypto_path +- set_variable(qat_crypto_path.underscorify() + '_disable_reason', ++ set_variable('drv_' + qat_crypto_path.underscorify() + '_disable_reason', + 'Explicitly disabled via build config') + endif + if disable_drivers.contains(qat_compress_path) + qat_compress = false + dpdk_drvs_disabled += qat_compress_path +- set_variable(qat_compress_path.underscorify() + '_disable_reason', ++ set_variable('drv_' + qat_compress_path.underscorify() + '_disable_reason', + 'Explicitly disabled via build config') + endif + +@@ -36,7 +36,7 @@ if arch_subdir == 'arm' + else + qat_crypto = false + dpdk_drvs_disabled += qat_crypto_path +- set_variable(qat_crypto_path.underscorify() + '_disable_reason', ++ set_variable('drv_' + qat_crypto_path.underscorify() + '_disable_reason', + 'missing dependency for Arm, libcrypto') + endif + else +@@ -57,7 +57,7 @@ else + else + qat_crypto = false + dpdk_drvs_disabled += qat_crypto_path +- set_variable(qat_crypto_path.underscorify() + '_disable_reason', ++ set_variable('drv_' + qat_crypto_path.underscorify() + '_disable_reason', + 'missing dependency, libipsecmb or libcrypto') + endif + elif libcrypto.found() +@@ -66,7 +66,7 @@ else + else + qat_crypto = false + dpdk_drvs_disabled += qat_crypto_path +- set_variable(qat_crypto_path.underscorify() + '_disable_reason', ++ set_variable('drv_' + qat_crypto_path.underscorify() + '_disable_reason', + 'missing dependency, libipsecmb or libcrypto') + endif + endif +diff --git a/dpdk/drivers/common/qat/qat_device.c b/dpdk/drivers/common/qat/qat_device.c +index f55dc3c6f0..eceb5c89c4 100644 +--- a/dpdk/drivers/common/qat/qat_device.c ++++ b/dpdk/drivers/common/qat/qat_device.c +@@ -29,6 +29,7 @@ struct qat_dev_hw_spec_funcs *qat_dev_hw_spec[QAT_N_GENS]; + /* per-process array of device data */ + struct qat_device_info qat_pci_devs[RTE_PMD_QAT_MAX_PCI_DEVICES]; + static int qat_nb_pci_devices; ++int qat_legacy_capa; + + /* + * The set of PCI devices this driver supports +diff --git a/dpdk/drivers/common/sfc_efx/base/efx.h b/dpdk/drivers/common/sfc_efx/base/efx.h +index 3312c2fa8f..5773cb00b3 100644 +--- a/dpdk/drivers/common/sfc_efx/base/efx.h ++++ b/dpdk/drivers/common/sfc_efx/base/efx.h +@@ -7,6 +7,8 @@ + #ifndef _SYS_EFX_H + #define _SYS_EFX_H + ++#include ++ + #include "efx_annote.h" + #include "efsys.h" + #include "efx_types.h" +@@ -17,14 +19,20 @@ + extern "C" { + #endif + +-#define EFX_STATIC_ASSERT(_cond) \ +- ((void)sizeof (char[(_cond) ? 1 : -1])) ++/* ++ * Triggers an error at compilation time if the condition is false. ++ * ++ * The { } exists to workaround a bug in clang (#55821) ++ * where it would not handle _Static_assert in a switch case. ++ */ ++#define EFX_STATIC_ASSERT(_cond) \ ++ { static_assert((_cond), #_cond); } + + #define EFX_ARRAY_SIZE(_array) \ + (sizeof (_array) / sizeof ((_array)[0])) + + #define EFX_FIELD_OFFSET(_type, _field) \ +- ((size_t)&(((_type *)0)->_field)) ++ offsetof(_type, _field) + + /* The macro expands divider twice */ + #define EFX_DIV_ROUND_UP(_n, _d) (((_n) + (_d) - 1) / (_d)) +diff --git a/dpdk/drivers/crypto/cnxk/cn10k_cryptodev_ops.c b/dpdk/drivers/crypto/cnxk/cn10k_cryptodev_ops.c +index 997110e3d3..c96cf2b3a1 100644 +--- a/dpdk/drivers/crypto/cnxk/cn10k_cryptodev_ops.c ++++ b/dpdk/drivers/crypto/cnxk/cn10k_cryptodev_ops.c +@@ -861,15 +861,20 @@ cn10k_cpt_dequeue_post_process(struct cnxk_cpt_qp *qp, + + return; + } else if (cop->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC && +- cop->sess_type == RTE_CRYPTO_OP_WITH_SESSION && +- cop->asym->ecdh.ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_VERIFY) { +- if (likely(compcode == CPT_COMP_GOOD)) { +- if (uc_compcode == ROC_AE_ERR_ECC_POINT_NOT_ON_CURVE) { +- cop->status = RTE_CRYPTO_OP_STATUS_ERROR; +- return; +- } else if (uc_compcode == ROC_AE_ERR_ECC_PAI) { +- cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; +- return; ++ cop->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { ++ struct cnxk_ae_sess *sess; ++ ++ sess = (struct cnxk_ae_sess *)cop->asym->session; ++ if (sess->xfrm_type == RTE_CRYPTO_ASYM_XFORM_ECDH && ++ cop->asym->ecdh.ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_VERIFY) { ++ if (likely(compcode == CPT_COMP_GOOD)) { ++ if (uc_compcode == ROC_AE_ERR_ECC_POINT_NOT_ON_CURVE) { ++ cop->status = RTE_CRYPTO_OP_STATUS_ERROR; ++ return; ++ } else if (uc_compcode == ROC_AE_ERR_ECC_PAI) { ++ cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; ++ return; ++ } + } + } + } +diff --git a/dpdk/drivers/crypto/cnxk/cn9k_cryptodev_ops.c b/dpdk/drivers/crypto/cnxk/cn9k_cryptodev_ops.c +index 34d40b07d4..eb5575b7ec 100644 +--- a/dpdk/drivers/crypto/cnxk/cn9k_cryptodev_ops.c ++++ b/dpdk/drivers/crypto/cnxk/cn9k_cryptodev_ops.c +@@ -578,7 +578,22 @@ cn9k_cpt_dequeue_post_process(struct cnxk_cpt_qp *qp, struct rte_crypto_op *cop, + if (unlikely(res->uc_compcode)) { + if (res->uc_compcode == ROC_SE_ERR_GC_ICV_MISCOMPARE) + cop->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; +- else ++ else if (cop->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC && ++ cop->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { ++ struct cnxk_ae_sess *sess; ++ ++ sess = (struct cnxk_ae_sess *)cop->asym->session; ++ if (sess->xfrm_type == RTE_CRYPTO_ASYM_XFORM_ECDH && ++ cop->asym->ecdh.ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_VERIFY) { ++ if (res->uc_compcode == ROC_AE_ERR_ECC_POINT_NOT_ON_CURVE) { ++ cop->status = RTE_CRYPTO_OP_STATUS_ERROR; ++ return; ++ } else if (res->uc_compcode == ROC_AE_ERR_ECC_PAI) { ++ cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; ++ return; ++ } ++ } ++ } else + cop->status = RTE_CRYPTO_OP_STATUS_ERROR; + + plt_dp_info("Request failed with microcode error"); +diff --git a/dpdk/drivers/crypto/cnxk/cnxk_ae.h b/dpdk/drivers/crypto/cnxk/cnxk_ae.h +index ea11e093bf..ef9cb5eb91 100644 +--- a/dpdk/drivers/crypto/cnxk/cnxk_ae.h ++++ b/dpdk/drivers/crypto/cnxk/cnxk_ae.h +@@ -49,13 +49,22 @@ struct cnxk_ae_sess { + }; + + static __rte_always_inline void +-cnxk_ae_modex_param_normalize(uint8_t **data, size_t *len) ++cnxk_ae_modex_param_normalize(uint8_t **data, size_t *len, size_t max) + { ++ uint8_t msw_len = *len % 8; ++ uint64_t msw_val = 0; + size_t i; + +- /* Strip leading NUL bytes */ +- for (i = 0; i < *len; i++) { +- if ((*data)[i] != 0) ++ if (*len <= 8) ++ return; ++ ++ memcpy(&msw_val, *data, msw_len); ++ if (msw_val != 0) ++ return; ++ ++ for (i = msw_len; i < *len && (*len - i) < max; i += 8) { ++ memcpy(&msw_val, &(*data)[i], 8); ++ if (msw_val != 0) + break; + } + *data += i; +@@ -72,8 +81,8 @@ cnxk_ae_fill_modex_params(struct cnxk_ae_sess *sess, + uint8_t *exp = xform->modex.exponent.data; + uint8_t *mod = xform->modex.modulus.data; + +- cnxk_ae_modex_param_normalize(&mod, &mod_len); +- cnxk_ae_modex_param_normalize(&exp, &exp_len); ++ cnxk_ae_modex_param_normalize(&mod, &mod_len, SIZE_MAX); ++ cnxk_ae_modex_param_normalize(&exp, &exp_len, mod_len); + + if (unlikely(exp_len == 0 || mod_len == 0)) + return -EINVAL; +@@ -205,16 +214,22 @@ cnxk_ae_fill_ec_params(struct cnxk_ae_sess *sess, + return 0; + + ec->pkey.length = xform->ec.pkey.length; +- if (xform->ec.pkey.length) +- rte_memcpy(ec->pkey.data, xform->ec.pkey.data, xform->ec.pkey.length); ++ if (ec->pkey.length > ROC_AE_EC_DATA_MAX) ++ ec->pkey.length = ROC_AE_EC_DATA_MAX; ++ if (ec->pkey.length) ++ rte_memcpy(ec->pkey.data, xform->ec.pkey.data, ec->pkey.length); + + ec->q.x.length = xform->ec.q.x.length; +- if (xform->ec.q.x.length) +- rte_memcpy(ec->q.x.data, xform->ec.q.x.data, xform->ec.q.x.length); ++ if (ec->q.x.length > ROC_AE_EC_DATA_MAX) ++ ec->q.x.length = ROC_AE_EC_DATA_MAX; ++ if (ec->q.x.length) ++ rte_memcpy(ec->q.x.data, xform->ec.q.x.data, ec->q.x.length); + + ec->q.y.length = xform->ec.q.y.length; ++ if (ec->q.y.length > ROC_AE_EC_DATA_MAX) ++ ec->q.y.length = ROC_AE_EC_DATA_MAX; + if (xform->ec.q.y.length) +- rte_memcpy(ec->q.y.data, xform->ec.q.y.data, xform->ec.q.y.length); ++ rte_memcpy(ec->q.y.data, xform->ec.q.y.data, ec->q.y.length); + + return 0; + } +@@ -282,7 +297,7 @@ cnxk_ae_modex_prep(struct rte_crypto_op *op, struct roc_ae_buf_ptr *meta_buf, + struct rte_crypto_mod_op_param mod_op; + uint64_t total_key_len; + union cpt_inst_w4 w4; +- uint32_t base_len; ++ size_t base_len; + uint32_t dlen; + uint8_t *dptr; + +@@ -290,8 +305,11 @@ cnxk_ae_modex_prep(struct rte_crypto_op *op, struct roc_ae_buf_ptr *meta_buf, + + base_len = mod_op.base.length; + if (unlikely(base_len > mod_len)) { +- op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; +- return -ENOTSUP; ++ cnxk_ae_modex_param_normalize(&mod_op.base.data, &base_len, mod_len); ++ if (base_len > mod_len) { ++ op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; ++ return -ENOTSUP; ++ } + } + + total_key_len = mod_len + exp_len; +@@ -735,7 +753,11 @@ cnxk_ae_sm2_sign_prep(struct rte_crypto_sm2_op_param *sm2, + uint8_t *dptr; + + prime_len = ec_grp->prime.length; ++ if (prime_len > ROC_AE_EC_DATA_MAX) ++ prime_len = ROC_AE_EC_DATA_MAX; + order_len = ec_grp->order.length; ++ if (order_len > ROC_AE_EC_DATA_MAX) ++ order_len = ROC_AE_EC_DATA_MAX; + + /* Truncate input length to curve prime length */ + if (message_len > prime_len) +@@ -822,7 +844,11 @@ cnxk_ae_sm2_verify_prep(struct rte_crypto_sm2_op_param *sm2, + uint8_t *dptr; + + prime_len = ec_grp->prime.length; ++ if (prime_len > ROC_AE_EC_DATA_MAX) ++ prime_len = ROC_AE_EC_DATA_MAX; + order_len = ec_grp->order.length; ++ if (order_len > ROC_AE_EC_DATA_MAX) ++ order_len = ROC_AE_EC_DATA_MAX; + + /* Truncate input length to curve prime length */ + if (message_len > prime_len) +diff --git a/dpdk/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c b/dpdk/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c +index bb5a2c629e..6ae356ace0 100644 +--- a/dpdk/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c ++++ b/dpdk/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c +@@ -4124,7 +4124,7 @@ dpaa2_sec_eventq_attach(const struct rte_cryptodev *dev, + cfg.dest_cfg.priority = priority; + + cfg.options |= DPSECI_QUEUE_OPT_USER_CTX; +- cfg.user_ctx = (size_t)(qp); ++ cfg.user_ctx = (size_t)(&qp->rx_vq); + if (event->sched_type == RTE_SCHED_TYPE_ATOMIC) { + cfg.options |= DPSECI_QUEUE_OPT_ORDER_PRESERVATION; + cfg.order_preservation_en = 1; +diff --git a/dpdk/drivers/crypto/dpaa_sec/dpaa_sec.c b/dpdk/drivers/crypto/dpaa_sec/dpaa_sec.c +index a301e8edb2..906ea39047 100644 +--- a/dpdk/drivers/crypto/dpaa_sec/dpaa_sec.c ++++ b/dpdk/drivers/crypto/dpaa_sec/dpaa_sec.c +@@ -395,10 +395,10 @@ dpaa_sec_prep_ipsec_cdb(dpaa_sec_session *ses) + + cdb->sh_desc[0] = cipherdata.keylen; + cdb->sh_desc[1] = authdata.keylen; +- err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN, ++ err = rta_inline_ipsec_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN, + DESC_JOB_IO_LEN, + (unsigned int *)cdb->sh_desc, +- &cdb->sh_desc[2], 2); ++ &cdb->sh_desc[2], 2, authdata.algtype, 1); + + if (err < 0) { + DPAA_SEC_ERR("Crypto: Incorrect key lengths"); +diff --git a/dpdk/drivers/crypto/ipsec_mb/ipsec_mb_ops.c b/dpdk/drivers/crypto/ipsec_mb/ipsec_mb_ops.c +index 30f919cd40..2a5599b7d8 100644 +--- a/dpdk/drivers/crypto/ipsec_mb/ipsec_mb_ops.c ++++ b/dpdk/drivers/crypto/ipsec_mb/ipsec_mb_ops.c +@@ -406,7 +406,7 @@ ipsec_mb_ipc_request(const struct rte_mp_msg *mp_msg, const void *peer) + resp_param->result = ipsec_mb_qp_release(dev, qp_id); + break; + default: +- CDEV_LOG_ERR("invalid mp request type\n"); ++ CDEV_LOG_ERR("invalid mp request type"); + } + + out: +diff --git a/dpdk/drivers/crypto/ipsec_mb/pmd_aesni_mb.c b/dpdk/drivers/crypto/ipsec_mb/pmd_aesni_mb.c +index 4de4866cf3..80de25c65b 100644 +--- a/dpdk/drivers/crypto/ipsec_mb/pmd_aesni_mb.c ++++ b/dpdk/drivers/crypto/ipsec_mb/pmd_aesni_mb.c +@@ -1500,7 +1500,7 @@ aesni_mb_digest_appended_in_src(struct rte_crypto_op *op, IMB_JOB *job, + * + * @return + * - 0 on success, the IMB_JOB will be filled +- * - -1 if invalid session or errors allocationg SGL linear buffer, ++ * - -1 if invalid session or errors allocating SGL linear buffer, + * IMB_JOB will not be filled + */ + static inline int +diff --git a/dpdk/drivers/crypto/openssl/compat.h b/dpdk/drivers/crypto/openssl/compat.h +index 9f9167c4f1..e1814fea8c 100644 +--- a/dpdk/drivers/crypto/openssl/compat.h ++++ b/dpdk/drivers/crypto/openssl/compat.h +@@ -5,6 +5,32 @@ + #ifndef __RTA_COMPAT_H__ + #define __RTA_COMPAT_H__ + ++#if OPENSSL_VERSION_NUMBER >= 0x30000000L ++static __rte_always_inline void ++free_hmac_ctx(EVP_MAC_CTX *ctx) ++{ ++ EVP_MAC_CTX_free(ctx); ++} ++ ++static __rte_always_inline void ++free_cmac_ctx(EVP_MAC_CTX *ctx) ++{ ++ EVP_MAC_CTX_free(ctx); ++} ++#else ++static __rte_always_inline void ++free_hmac_ctx(HMAC_CTX *ctx) ++{ ++ HMAC_CTX_free(ctx); ++} ++ ++static __rte_always_inline void ++free_cmac_ctx(CMAC_CTX *ctx) ++{ ++ CMAC_CTX_free(ctx); ++} ++#endif ++ + #if (OPENSSL_VERSION_NUMBER < 0x10100000L) + + static __rte_always_inline int +diff --git a/dpdk/drivers/crypto/openssl/openssl_pmd_private.h b/dpdk/drivers/crypto/openssl/openssl_pmd_private.h +index 334912d335..aa3f466e74 100644 +--- a/dpdk/drivers/crypto/openssl/openssl_pmd_private.h ++++ b/dpdk/drivers/crypto/openssl/openssl_pmd_private.h +@@ -80,6 +80,20 @@ struct openssl_qp { + */ + } __rte_cache_aligned; + ++struct evp_ctx_pair { ++ EVP_CIPHER_CTX *cipher; ++ union { ++ EVP_MD_CTX *auth; ++#if OPENSSL_VERSION_NUMBER >= 0x30000000L ++ EVP_MAC_CTX *hmac; ++ EVP_MAC_CTX *cmac; ++#else ++ HMAC_CTX *hmac; ++ CMAC_CTX *cmac; ++#endif ++ }; ++}; ++ + /** OPENSSL crypto private session structure */ + struct openssl_session { + enum openssl_chain_order chain_order; +@@ -166,6 +180,15 @@ struct openssl_session { + /**< digest length */ + } auth; + ++ uint16_t ctx_copies_len; ++ /* < number of entries in ctx_copies */ ++ struct evp_ctx_pair qp_ctx[]; ++ /**< Flexible array member of per-queue-pair structures, each containing ++ * pointers to copies of the cipher and auth EVP contexts. Cipher ++ * contexts are not safe to use from multiple cores simultaneously, so ++ * maintaining these copies allows avoiding per-buffer copying into a ++ * temporary context. ++ */ + } __rte_cache_aligned; + + /** OPENSSL crypto private asymmetric session structure */ +@@ -217,7 +240,8 @@ struct openssl_asym_session { + /** Set and validate OPENSSL crypto session parameters */ + extern int + openssl_set_session_parameters(struct openssl_session *sess, +- const struct rte_crypto_sym_xform *xform); ++ const struct rte_crypto_sym_xform *xform, ++ uint16_t nb_queue_pairs); + + /** Reset OPENSSL crypto session parameters */ + extern void +diff --git a/dpdk/drivers/crypto/openssl/rte_openssl_pmd.c b/dpdk/drivers/crypto/openssl/rte_openssl_pmd.c +index e8cb09defc..101111e85b 100644 +--- a/dpdk/drivers/crypto/openssl/rte_openssl_pmd.c ++++ b/dpdk/drivers/crypto/openssl/rte_openssl_pmd.c +@@ -350,7 +350,8 @@ get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen, + static int + openssl_set_sess_aead_enc_param(struct openssl_session *sess, + enum rte_crypto_aead_algorithm algo, +- uint8_t tag_len, const uint8_t *key) ++ uint8_t tag_len, const uint8_t *key, ++ EVP_CIPHER_CTX **ctx) + { + int iv_type = 0; + unsigned int do_ccm; +@@ -378,7 +379,7 @@ openssl_set_sess_aead_enc_param(struct openssl_session *sess, + } + + sess->cipher.mode = OPENSSL_CIPHER_LIB; +- sess->cipher.ctx = EVP_CIPHER_CTX_new(); ++ *ctx = EVP_CIPHER_CTX_new(); + + if (get_aead_algo(algo, sess->cipher.key.length, + &sess->cipher.evp_algo) != 0) +@@ -388,19 +389,19 @@ openssl_set_sess_aead_enc_param(struct openssl_session *sess, + + sess->chain_order = OPENSSL_CHAIN_COMBINED; + +- if (EVP_EncryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, ++ if (EVP_EncryptInit_ex(*ctx, sess->cipher.evp_algo, + NULL, NULL, NULL) <= 0) + return -EINVAL; + +- if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, sess->iv.length, ++ if (EVP_CIPHER_CTX_ctrl(*ctx, iv_type, sess->iv.length, + NULL) <= 0) + return -EINVAL; + + if (do_ccm) +- EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, ++ EVP_CIPHER_CTX_ctrl(*ctx, EVP_CTRL_CCM_SET_TAG, + tag_len, NULL); + +- if (EVP_EncryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) ++ if (EVP_EncryptInit_ex(*ctx, NULL, NULL, key, NULL) <= 0) + return -EINVAL; + + return 0; +@@ -410,7 +411,8 @@ openssl_set_sess_aead_enc_param(struct openssl_session *sess, + static int + openssl_set_sess_aead_dec_param(struct openssl_session *sess, + enum rte_crypto_aead_algorithm algo, +- uint8_t tag_len, const uint8_t *key) ++ uint8_t tag_len, const uint8_t *key, ++ EVP_CIPHER_CTX **ctx) + { + int iv_type = 0; + unsigned int do_ccm = 0; +@@ -437,7 +439,7 @@ openssl_set_sess_aead_dec_param(struct openssl_session *sess, + } + + sess->cipher.mode = OPENSSL_CIPHER_LIB; +- sess->cipher.ctx = EVP_CIPHER_CTX_new(); ++ *ctx = EVP_CIPHER_CTX_new(); + + if (get_aead_algo(algo, sess->cipher.key.length, + &sess->cipher.evp_algo) != 0) +@@ -447,24 +449,46 @@ openssl_set_sess_aead_dec_param(struct openssl_session *sess, + + sess->chain_order = OPENSSL_CHAIN_COMBINED; + +- if (EVP_DecryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, ++ if (EVP_DecryptInit_ex(*ctx, sess->cipher.evp_algo, + NULL, NULL, NULL) <= 0) + return -EINVAL; + +- if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, ++ if (EVP_CIPHER_CTX_ctrl(*ctx, iv_type, + sess->iv.length, NULL) <= 0) + return -EINVAL; + + if (do_ccm) +- EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, ++ EVP_CIPHER_CTX_ctrl(*ctx, EVP_CTRL_CCM_SET_TAG, + tag_len, NULL); + +- if (EVP_DecryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) ++ if (EVP_DecryptInit_ex(*ctx, NULL, NULL, key, NULL) <= 0) + return -EINVAL; + + return 0; + } + ++#if (OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x30200000L) ++static int openssl_aesni_ctx_clone(EVP_CIPHER_CTX **dest, ++ struct openssl_session *sess) ++{ ++ /* OpenSSL versions 3.0.0 <= V < 3.2.0 have no dupctx() implementation ++ * for AES-GCM and AES-CCM. In this case, we have to create new empty ++ * contexts and initialise, as we did the original context. ++ */ ++ if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) ++ sess->aead_algo = RTE_CRYPTO_AEAD_AES_GCM; ++ ++ if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ++ return openssl_set_sess_aead_enc_param(sess, sess->aead_algo, ++ sess->auth.digest_length, sess->cipher.key.data, ++ dest); ++ else ++ return openssl_set_sess_aead_dec_param(sess, sess->aead_algo, ++ sess->auth.digest_length, sess->cipher.key.data, ++ dest); ++} ++#endif ++ + /** Set session cipher parameters */ + static int + openssl_set_session_cipher_parameters(struct openssl_session *sess, +@@ -521,6 +545,15 @@ openssl_set_session_cipher_parameters(struct openssl_session *sess, + sess->cipher.key.length, + sess->cipher.key.data) != 0) + return -EINVAL; ++ ++ ++ /* We use 3DES encryption also for decryption. ++ * IV is not important for 3DES ECB. ++ */ ++ if (EVP_EncryptInit_ex(sess->cipher.ctx, EVP_des_ede3_ecb(), ++ NULL, sess->cipher.key.data, NULL) != 1) ++ return -EINVAL; ++ + break; + + case RTE_CRYPTO_CIPHER_DES_CBC: +@@ -586,6 +619,8 @@ openssl_set_session_cipher_parameters(struct openssl_session *sess, + return -ENOTSUP; + } + ++ EVP_CIPHER_CTX_set_padding(sess->cipher.ctx, 0); ++ + return 0; + } + +@@ -623,12 +658,14 @@ openssl_set_session_auth_parameters(struct openssl_session *sess, + return openssl_set_sess_aead_enc_param(sess, + RTE_CRYPTO_AEAD_AES_GCM, + xform->auth.digest_length, +- xform->auth.key.data); ++ xform->auth.key.data, ++ &sess->cipher.ctx); + else + return openssl_set_sess_aead_dec_param(sess, + RTE_CRYPTO_AEAD_AES_GCM, + xform->auth.digest_length, +- xform->auth.key.data); ++ xform->auth.key.data, ++ &sess->cipher.ctx); + break; + + case RTE_CRYPTO_AUTH_MD5: +@@ -770,16 +807,19 @@ openssl_set_session_aead_parameters(struct openssl_session *sess, + /* Select cipher direction */ + if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) + return openssl_set_sess_aead_enc_param(sess, xform->aead.algo, +- xform->aead.digest_length, xform->aead.key.data); ++ xform->aead.digest_length, xform->aead.key.data, ++ &sess->cipher.ctx); + else + return openssl_set_sess_aead_dec_param(sess, xform->aead.algo, +- xform->aead.digest_length, xform->aead.key.data); ++ xform->aead.digest_length, xform->aead.key.data, ++ &sess->cipher.ctx); + } + + /** Parse crypto xform chain and set private session parameters */ + int + openssl_set_session_parameters(struct openssl_session *sess, +- const struct rte_crypto_sym_xform *xform) ++ const struct rte_crypto_sym_xform *xform, ++ uint16_t nb_queue_pairs) + { + const struct rte_crypto_sym_xform *cipher_xform = NULL; + const struct rte_crypto_sym_xform *auth_xform = NULL; +@@ -841,6 +881,12 @@ openssl_set_session_parameters(struct openssl_session *sess, + } + } + ++ /* ++ * With only one queue pair, the array of copies is not needed. ++ * Otherwise, one entry per queue pair is required. ++ */ ++ sess->ctx_copies_len = nb_queue_pairs > 1 ? nb_queue_pairs : 0; ++ + return 0; + } + +@@ -848,33 +894,45 @@ openssl_set_session_parameters(struct openssl_session *sess, + void + openssl_reset_session(struct openssl_session *sess) + { +- EVP_CIPHER_CTX_free(sess->cipher.ctx); ++ /* Free all the qp_ctx entries. */ ++ for (uint16_t i = 0; i < sess->ctx_copies_len; i++) { ++ if (sess->qp_ctx[i].cipher != NULL) { ++ EVP_CIPHER_CTX_free(sess->qp_ctx[i].cipher); ++ sess->qp_ctx[i].cipher = NULL; ++ } + +- if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI) +- EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx); ++ switch (sess->auth.mode) { ++ case OPENSSL_AUTH_AS_AUTH: ++ EVP_MD_CTX_destroy(sess->qp_ctx[i].auth); ++ sess->qp_ctx[i].auth = NULL; ++ break; ++ case OPENSSL_AUTH_AS_HMAC: ++ free_hmac_ctx(sess->qp_ctx[i].hmac); ++ sess->qp_ctx[i].hmac = NULL; ++ break; ++ case OPENSSL_AUTH_AS_CMAC: ++ free_cmac_ctx(sess->qp_ctx[i].cmac); ++ sess->qp_ctx[i].cmac = NULL; ++ break; ++ } ++ } ++ ++ EVP_CIPHER_CTX_free(sess->cipher.ctx); + + switch (sess->auth.mode) { + case OPENSSL_AUTH_AS_AUTH: + EVP_MD_CTX_destroy(sess->auth.auth.ctx); + break; + case OPENSSL_AUTH_AS_HMAC: +- EVP_PKEY_free(sess->auth.hmac.pkey); +-# if OPENSSL_VERSION_NUMBER >= 0x30000000L +- EVP_MAC_CTX_free(sess->auth.hmac.ctx); +-# else +- HMAC_CTX_free(sess->auth.hmac.ctx); +-# endif ++ free_hmac_ctx(sess->auth.hmac.ctx); + break; + case OPENSSL_AUTH_AS_CMAC: +-# if OPENSSL_VERSION_NUMBER >= 0x30000000L +- EVP_MAC_CTX_free(sess->auth.cmac.ctx); +-# else +- CMAC_CTX_free(sess->auth.cmac.ctx); +-# endif +- break; +- default: ++ free_cmac_ctx(sess->auth.cmac.ctx); + break; + } ++ ++ if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI) ++ EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx); + } + + /** Provide session for operation */ +@@ -914,7 +972,7 @@ get_session(struct openssl_qp *qp, struct rte_crypto_op *op) + sess = (struct openssl_session *)_sess->driver_priv_data; + + if (unlikely(openssl_set_session_parameters(sess, +- op->sym->xform) != 0)) { ++ op->sym->xform, 1) != 0)) { + rte_mempool_put(qp->sess_mp, _sess); + sess = NULL; + } +@@ -1068,8 +1126,6 @@ process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, + if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) + goto process_cipher_encrypt_err; + +- EVP_CIPHER_CTX_set_padding(ctx, 0); +- + if (process_openssl_encryption_update(mbuf_src, offset, &dst, + srclen, ctx, inplace)) + goto process_cipher_encrypt_err; +@@ -1118,8 +1174,6 @@ process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, + if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) + goto process_cipher_decrypt_err; + +- EVP_CIPHER_CTX_set_padding(ctx, 0); +- + if (process_openssl_decryption_update(mbuf_src, offset, &dst, + srclen, ctx, inplace)) + goto process_cipher_decrypt_err; +@@ -1136,8 +1190,7 @@ process_cipher_decrypt_err: + /** Process cipher des 3 ctr encryption, decryption algorithm */ + static int + process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst, +- int offset, uint8_t *iv, uint8_t *key, int srclen, +- EVP_CIPHER_CTX *ctx) ++ int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx) + { + uint8_t ebuf[8], ctr[8]; + int unused, n; +@@ -1155,12 +1208,6 @@ process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst, + src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); + l = rte_pktmbuf_data_len(m) - offset; + +- /* We use 3DES encryption also for decryption. +- * IV is not important for 3DES ecb +- */ +- if (EVP_EncryptInit_ex(ctx, EVP_des_ede3_ecb(), NULL, key, NULL) <= 0) +- goto process_cipher_des3ctr_err; +- + memcpy(ctr, iv, 8); + + for (n = 0; n < srclen; n++) { +@@ -1427,6 +1474,9 @@ process_openssl_auth_mac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, + if (m == 0) + goto process_auth_err; + ++ if (EVP_MAC_init(ctx, NULL, 0, NULL) <= 0) ++ goto process_auth_err; ++ + src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); + + l = rte_pktmbuf_data_len(m) - offset; +@@ -1453,11 +1503,9 @@ process_auth_final: + if (EVP_MAC_final(ctx, dst, &dstlen, DIGEST_LENGTH_MAX) != 1) + goto process_auth_err; + +- EVP_MAC_CTX_free(ctx); + return 0; + + process_auth_err: +- EVP_MAC_CTX_free(ctx); + OPENSSL_LOG(ERR, "Process openssl auth failed"); + return -EINVAL; + } +@@ -1569,11 +1617,151 @@ process_auth_err: + # endif + /*----------------------------------------------------------------------------*/ + ++static inline EVP_CIPHER_CTX * ++get_local_cipher_ctx(struct openssl_session *sess, struct openssl_qp *qp) ++{ ++ /* If the array is not being used, just return the main context. */ ++ if (sess->ctx_copies_len == 0) ++ return sess->cipher.ctx; ++ ++ EVP_CIPHER_CTX **lctx = &sess->qp_ctx[qp->id].cipher; ++ ++ if (unlikely(*lctx == NULL)) { ++#if OPENSSL_VERSION_NUMBER >= 0x30200000L ++ /* EVP_CIPHER_CTX_dup() added in OSSL 3.2 */ ++ *lctx = EVP_CIPHER_CTX_dup(sess->cipher.ctx); ++ return *lctx; ++#elif OPENSSL_VERSION_NUMBER >= 0x30000000L ++ if (sess->chain_order == OPENSSL_CHAIN_COMBINED) { ++ /* AESNI special-cased to use openssl_aesni_ctx_clone() ++ * to allow for working around lack of ++ * EVP_CIPHER_CTX_copy support for 3.0.0 <= OSSL Version ++ * < 3.2.0. ++ */ ++ if (openssl_aesni_ctx_clone(lctx, sess) != 0) ++ *lctx = NULL; ++ return *lctx; ++ } ++#endif ++ ++ *lctx = EVP_CIPHER_CTX_new(); ++ EVP_CIPHER_CTX_copy(*lctx, sess->cipher.ctx); ++ } ++ ++ return *lctx; ++} ++ ++static inline EVP_MD_CTX * ++get_local_auth_ctx(struct openssl_session *sess, struct openssl_qp *qp) ++{ ++ /* If the array is not being used, just return the main context. */ ++ if (sess->ctx_copies_len == 0) ++ return sess->auth.auth.ctx; ++ ++ EVP_MD_CTX **lctx = &sess->qp_ctx[qp->id].auth; ++ ++ if (unlikely(*lctx == NULL)) { ++#if OPENSSL_VERSION_NUMBER >= 0x30100000L ++ /* EVP_MD_CTX_dup() added in OSSL 3.1 */ ++ *lctx = EVP_MD_CTX_dup(sess->auth.auth.ctx); ++#else ++ *lctx = EVP_MD_CTX_new(); ++ EVP_MD_CTX_copy(*lctx, sess->auth.auth.ctx); ++#endif ++ } ++ ++ return *lctx; ++} ++ ++#if OPENSSL_VERSION_NUMBER >= 0x30000000L ++static inline EVP_MAC_CTX * ++#else ++static inline HMAC_CTX * ++#endif ++get_local_hmac_ctx(struct openssl_session *sess, struct openssl_qp *qp) ++{ ++#if (OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x30003000L) ++ /* For OpenSSL versions 3.0.0 <= v < 3.0.3, re-initing of ++ * EVP_MAC_CTXs is broken, and doesn't actually reset their ++ * state. This was fixed in OSSL commit c9ddc5af5199 ("Avoid ++ * undefined behavior of provided macs on EVP_MAC ++ * reinitialization"). In cases where the fix is not present, ++ * fall back to duplicating the context every buffer as a ++ * workaround, at the cost of performance. ++ */ ++ RTE_SET_USED(qp); ++ return EVP_MAC_CTX_dup(sess->auth.hmac.ctx); ++#else ++ if (sess->ctx_copies_len == 0) ++ return sess->auth.hmac.ctx; ++ ++#if OPENSSL_VERSION_NUMBER >= 0x30000000L ++ EVP_MAC_CTX **lctx = ++#else ++ HMAC_CTX **lctx = ++#endif ++ &sess->qp_ctx[qp->id].hmac; ++ ++ if (unlikely(*lctx == NULL)) { ++#if OPENSSL_VERSION_NUMBER >= 0x30000000L ++ *lctx = EVP_MAC_CTX_dup(sess->auth.hmac.ctx); ++#else ++ *lctx = HMAC_CTX_new(); ++ HMAC_CTX_copy(*lctx, sess->auth.hmac.ctx); ++#endif ++ } ++ ++ return *lctx; ++#endif ++} ++ ++#if OPENSSL_VERSION_NUMBER >= 0x30000000L ++static inline EVP_MAC_CTX * ++#else ++static inline CMAC_CTX * ++#endif ++get_local_cmac_ctx(struct openssl_session *sess, struct openssl_qp *qp) ++{ ++#if (OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x30003000L) ++ /* For OpenSSL versions 3.0.0 <= v < 3.0.3, re-initing of ++ * EVP_MAC_CTXs is broken, and doesn't actually reset their ++ * state. This was fixed in OSSL commit c9ddc5af5199 ("Avoid ++ * undefined behavior of provided macs on EVP_MAC ++ * reinitialization"). In cases where the fix is not present, ++ * fall back to duplicating the context every buffer as a ++ * workaround, at the cost of performance. ++ */ ++ RTE_SET_USED(qp); ++ return EVP_MAC_CTX_dup(sess->auth.cmac.ctx); ++#else ++ if (sess->ctx_copies_len == 0) ++ return sess->auth.cmac.ctx; ++ ++#if OPENSSL_VERSION_NUMBER >= 0x30000000L ++ EVP_MAC_CTX **lctx = ++#else ++ CMAC_CTX **lctx = ++#endif ++ &sess->qp_ctx[qp->id].cmac; ++ ++ if (unlikely(*lctx == NULL)) { ++#if OPENSSL_VERSION_NUMBER >= 0x30000000L ++ *lctx = EVP_MAC_CTX_dup(sess->auth.cmac.ctx); ++#else ++ *lctx = CMAC_CTX_new(); ++ CMAC_CTX_copy(*lctx, sess->auth.cmac.ctx); ++#endif ++ } ++ ++ return *lctx; ++#endif ++} ++ + /** Process auth/cipher combined operation */ + static void +-process_openssl_combined_op +- (struct rte_crypto_op *op, struct openssl_session *sess, +- struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) ++process_openssl_combined_op(struct openssl_qp *qp, struct rte_crypto_op *op, ++ struct openssl_session *sess, struct rte_mbuf *mbuf_src, ++ struct rte_mbuf *mbuf_dst) + { + /* cipher */ + uint8_t *dst = NULL, *iv, *tag, *aad; +@@ -1590,6 +1778,8 @@ process_openssl_combined_op + return; + } + ++ EVP_CIPHER_CTX *ctx = get_local_cipher_ctx(sess, qp); ++ + iv = rte_crypto_op_ctod_offset(op, uint8_t *, + sess->iv.offset); + if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) { +@@ -1623,12 +1813,12 @@ process_openssl_combined_op + status = process_openssl_auth_encryption_gcm( + mbuf_src, offset, srclen, + aad, aadlen, iv, +- dst, tag, sess->cipher.ctx); ++ dst, tag, ctx); + else + status = process_openssl_auth_encryption_ccm( + mbuf_src, offset, srclen, + aad, aadlen, iv, +- dst, tag, taglen, sess->cipher.ctx); ++ dst, tag, taglen, ctx); + + } else { + if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || +@@ -1636,12 +1826,12 @@ process_openssl_combined_op + status = process_openssl_auth_decryption_gcm( + mbuf_src, offset, srclen, + aad, aadlen, iv, +- dst, tag, sess->cipher.ctx); ++ dst, tag, ctx); + else + status = process_openssl_auth_decryption_ccm( + mbuf_src, offset, srclen, + aad, aadlen, iv, +- dst, tag, taglen, sess->cipher.ctx); ++ dst, tag, taglen, ctx); + } + + if (status != 0) { +@@ -1656,14 +1846,13 @@ process_openssl_combined_op + + /** Process cipher operation */ + static void +-process_openssl_cipher_op +- (struct rte_crypto_op *op, struct openssl_session *sess, +- struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) ++process_openssl_cipher_op(struct openssl_qp *qp, struct rte_crypto_op *op, ++ struct openssl_session *sess, struct rte_mbuf *mbuf_src, ++ struct rte_mbuf *mbuf_dst) + { + uint8_t *dst, *iv; + int srclen, status; + uint8_t inplace = (mbuf_src == mbuf_dst) ? 1 : 0; +- EVP_CIPHER_CTX *ctx_copy; + + /* + * Segmented OOP destination buffer is not supported for encryption/ +@@ -1682,25 +1871,22 @@ process_openssl_cipher_op + + iv = rte_crypto_op_ctod_offset(op, uint8_t *, + sess->iv.offset); +- ctx_copy = EVP_CIPHER_CTX_new(); +- EVP_CIPHER_CTX_copy(ctx_copy, sess->cipher.ctx); ++ ++ EVP_CIPHER_CTX *ctx = get_local_cipher_ctx(sess, qp); + + if (sess->cipher.mode == OPENSSL_CIPHER_LIB) + if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) + status = process_openssl_cipher_encrypt(mbuf_src, dst, + op->sym->cipher.data.offset, iv, +- srclen, ctx_copy, inplace); ++ srclen, ctx, inplace); + else + status = process_openssl_cipher_decrypt(mbuf_src, dst, + op->sym->cipher.data.offset, iv, +- srclen, ctx_copy, inplace); ++ srclen, ctx, inplace); + else + status = process_openssl_cipher_des3ctr(mbuf_src, dst, +- op->sym->cipher.data.offset, iv, +- sess->cipher.key.data, srclen, +- ctx_copy); ++ op->sym->cipher.data.offset, iv, srclen, ctx); + +- EVP_CIPHER_CTX_free(ctx_copy); + if (status != 0) + op->status = RTE_CRYPTO_OP_STATUS_ERROR; + } +@@ -1819,42 +2005,40 @@ process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op, + + switch (sess->auth.mode) { + case OPENSSL_AUTH_AS_AUTH: +- ctx_a = EVP_MD_CTX_create(); +- EVP_MD_CTX_copy_ex(ctx_a, sess->auth.auth.ctx); ++ ctx_a = get_local_auth_ctx(sess, qp); + status = process_openssl_auth(mbuf_src, dst, + op->sym->auth.data.offset, NULL, NULL, srclen, + ctx_a, sess->auth.auth.evp_algo); +- EVP_MD_CTX_destroy(ctx_a); + break; + case OPENSSL_AUTH_AS_HMAC: ++ ctx_h = get_local_hmac_ctx(sess, qp); + # if OPENSSL_VERSION_NUMBER >= 0x30000000L +- ctx_h = EVP_MAC_CTX_dup(sess->auth.hmac.ctx); + status = process_openssl_auth_mac(mbuf_src, dst, + op->sym->auth.data.offset, srclen, + ctx_h); + # else +- ctx_h = HMAC_CTX_new(); +- HMAC_CTX_copy(ctx_h, sess->auth.hmac.ctx); + status = process_openssl_auth_hmac(mbuf_src, dst, + op->sym->auth.data.offset, srclen, + ctx_h); +- HMAC_CTX_free(ctx_h); + # endif ++#if (OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x30003000L) ++ EVP_MAC_CTX_free(ctx_h); ++#endif + break; + case OPENSSL_AUTH_AS_CMAC: ++ ctx_c = get_local_cmac_ctx(sess, qp); + # if OPENSSL_VERSION_NUMBER >= 0x30000000L +- ctx_c = EVP_MAC_CTX_dup(sess->auth.cmac.ctx); + status = process_openssl_auth_mac(mbuf_src, dst, + op->sym->auth.data.offset, srclen, + ctx_c); + # else +- ctx_c = CMAC_CTX_new(); +- CMAC_CTX_copy(ctx_c, sess->auth.cmac.ctx); + status = process_openssl_auth_cmac(mbuf_src, dst, + op->sym->auth.data.offset, srclen, + ctx_c); +- CMAC_CTX_free(ctx_c); + # endif ++#if (OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x30003000L) ++ EVP_MAC_CTX_free(ctx_c); ++#endif + break; + default: + status = -1; +@@ -3105,13 +3289,13 @@ process_op(struct openssl_qp *qp, struct rte_crypto_op *op, + + switch (sess->chain_order) { + case OPENSSL_CHAIN_ONLY_CIPHER: +- process_openssl_cipher_op(op, sess, msrc, mdst); ++ process_openssl_cipher_op(qp, op, sess, msrc, mdst); + break; + case OPENSSL_CHAIN_ONLY_AUTH: + process_openssl_auth_op(qp, op, sess, msrc, mdst); + break; + case OPENSSL_CHAIN_CIPHER_AUTH: +- process_openssl_cipher_op(op, sess, msrc, mdst); ++ process_openssl_cipher_op(qp, op, sess, msrc, mdst); + /* OOP */ + if (msrc != mdst) + copy_plaintext(msrc, mdst, op); +@@ -3119,10 +3303,10 @@ process_op(struct openssl_qp *qp, struct rte_crypto_op *op, + break; + case OPENSSL_CHAIN_AUTH_CIPHER: + process_openssl_auth_op(qp, op, sess, msrc, mdst); +- process_openssl_cipher_op(op, sess, msrc, mdst); ++ process_openssl_cipher_op(qp, op, sess, msrc, mdst); + break; + case OPENSSL_CHAIN_COMBINED: +- process_openssl_combined_op(op, sess, msrc, mdst); ++ process_openssl_combined_op(qp, op, sess, msrc, mdst); + break; + case OPENSSL_CHAIN_CIPHER_BPI: + process_openssl_docsis_bpi_op(op, sess, msrc, mdst); +diff --git a/dpdk/drivers/crypto/openssl/rte_openssl_pmd_ops.c b/dpdk/drivers/crypto/openssl/rte_openssl_pmd_ops.c +index b16baaa08f..1bbb855a59 100644 +--- a/dpdk/drivers/crypto/openssl/rte_openssl_pmd_ops.c ++++ b/dpdk/drivers/crypto/openssl/rte_openssl_pmd_ops.c +@@ -794,9 +794,35 @@ qp_setup_cleanup: + + /** Returns the size of the symmetric session structure */ + static unsigned +-openssl_pmd_sym_session_get_size(struct rte_cryptodev *dev __rte_unused) ++openssl_pmd_sym_session_get_size(struct rte_cryptodev *dev) + { +- return sizeof(struct openssl_session); ++ /* ++ * For 0 qps, return the max size of the session - this is necessary if ++ * the user calls into this function to create the session mempool, ++ * without first configuring the number of qps for the cryptodev. ++ */ ++ if (dev->data->nb_queue_pairs == 0) { ++ unsigned int max_nb_qps = ((struct openssl_private *) ++ dev->data->dev_private)->max_nb_qpairs; ++ return sizeof(struct openssl_session) + ++ (sizeof(struct evp_ctx_pair) * max_nb_qps); ++ } ++ ++ /* ++ * With only one queue pair, the thread safety of multiple context ++ * copies is not necessary, so don't allocate extra memory for the ++ * array. ++ */ ++ if (dev->data->nb_queue_pairs == 1) ++ return sizeof(struct openssl_session); ++ ++ /* ++ * Otherwise, the size of the flexible array member should be enough to ++ * fit pointers to per-qp contexts. This is twice the number of queue ++ * pairs, to allow for auth and cipher contexts. ++ */ ++ return sizeof(struct openssl_session) + ++ (sizeof(struct evp_ctx_pair) * dev->data->nb_queue_pairs); + } + + /** Returns the size of the asymmetric session structure */ +@@ -808,7 +834,7 @@ openssl_pmd_asym_session_get_size(struct rte_cryptodev *dev __rte_unused) + + /** Configure the session from a crypto xform chain */ + static int +-openssl_pmd_sym_session_configure(struct rte_cryptodev *dev __rte_unused, ++openssl_pmd_sym_session_configure(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess) + { +@@ -820,7 +846,8 @@ openssl_pmd_sym_session_configure(struct rte_cryptodev *dev __rte_unused, + return -EINVAL; + } + +- ret = openssl_set_session_parameters(sess_private_data, xform); ++ ret = openssl_set_session_parameters(sess_private_data, xform, ++ dev->data->nb_queue_pairs); + if (ret != 0) { + OPENSSL_LOG(ERR, "failed configure session parameters"); + +diff --git a/dpdk/drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c b/dpdk/drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +index de72383d4b..b44acece7c 100644 +--- a/dpdk/drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c ++++ b/dpdk/drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +@@ -9,6 +9,7 @@ + #include "qat_asym.h" + #include "qat_crypto.h" + #include "qat_crypto_pmd_gens.h" ++#include "adf_transport_access_macros_gen4vf.h" + + + static struct rte_cryptodev_capabilities qat_sym_crypto_legacy_caps_gen4[] = { +@@ -233,6 +234,78 @@ qat_sym_build_op_aead_gen4(void *in_op, struct qat_sym_session *ctx, + return 0; + } + ++int ++qat_sym_dp_enqueue_done_gen4(void *qp_data, uint8_t *drv_ctx, uint32_t n) ++{ ++ struct qat_qp *qp = qp_data; ++ struct qat_queue *tx_queue = &qp->tx_q; ++ struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx; ++ ++ if (unlikely(dp_ctx->cached_enqueue != n)) ++ return -1; ++ ++ qp->enqueued += n; ++ qp->stats.enqueued_count += n; ++ ++ tx_queue->tail = dp_ctx->tail; ++ ++ WRITE_CSR_RING_TAIL_GEN4VF(qp->mmap_bar_addr, ++ tx_queue->hw_bundle_number, ++ tx_queue->hw_queue_number, tx_queue->tail); ++ ++ tx_queue->csr_tail = tx_queue->tail; ++ dp_ctx->cached_enqueue = 0; ++ ++ return 0; ++} ++ ++int ++qat_sym_dp_dequeue_done_gen4(void *qp_data, uint8_t *drv_ctx, uint32_t n) ++{ ++ struct qat_qp *qp = qp_data; ++ struct qat_queue *rx_queue = &qp->rx_q; ++ struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx; ++ ++ if (unlikely(dp_ctx->cached_dequeue != n)) ++ return -1; ++ ++ rx_queue->head = dp_ctx->head; ++ rx_queue->nb_processed_responses += n; ++ qp->dequeued += n; ++ qp->stats.dequeued_count += n; ++ if (rx_queue->nb_processed_responses > QAT_CSR_HEAD_WRITE_THRESH) { ++ uint32_t old_head, new_head; ++ uint32_t max_head; ++ ++ old_head = rx_queue->csr_head; ++ new_head = rx_queue->head; ++ max_head = qp->nb_descriptors * rx_queue->msg_size; ++ ++ /* write out free descriptors */ ++ void *cur_desc = (uint8_t *)rx_queue->base_addr + old_head; ++ ++ if (new_head < old_head) { ++ memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, ++ max_head - old_head); ++ memset(rx_queue->base_addr, ADF_RING_EMPTY_SIG_BYTE, ++ new_head); ++ } else { ++ memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, new_head - ++ old_head); ++ } ++ rx_queue->nb_processed_responses = 0; ++ rx_queue->csr_head = new_head; ++ ++ /* write current head to CSR */ ++ WRITE_CSR_RING_HEAD_GEN4VF(qp->mmap_bar_addr, ++ rx_queue->hw_bundle_number, rx_queue->hw_queue_number, ++ new_head); ++ } ++ ++ dp_ctx->cached_dequeue = 0; ++ return 0; ++} ++ + static int + qat_sym_crypto_set_session_gen4(void *cdev, void *session) + { +@@ -390,11 +463,51 @@ qat_sym_configure_raw_dp_ctx_gen4(void *_raw_dp_ctx, void *_ctx) + { + struct rte_crypto_raw_dp_ctx *raw_dp_ctx = _raw_dp_ctx; + struct qat_sym_session *ctx = _ctx; +- int ret; + +- ret = qat_sym_configure_raw_dp_ctx_gen1(_raw_dp_ctx, _ctx); +- if (ret < 0) +- return ret; ++ raw_dp_ctx->enqueue_done = qat_sym_dp_enqueue_done_gen4; ++ raw_dp_ctx->dequeue_burst = qat_sym_dp_dequeue_burst_gen1; ++ raw_dp_ctx->dequeue = qat_sym_dp_dequeue_single_gen1; ++ raw_dp_ctx->dequeue_done = qat_sym_dp_dequeue_done_gen4; ++ ++ if ((ctx->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER || ++ ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_HASH) && ++ !ctx->is_gmac) { ++ /* AES-GCM or AES-CCM */ ++ if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 || ++ ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64 || ++ (ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_AES128 ++ && ctx->qat_mode == ICP_QAT_HW_CIPHER_CTR_MODE ++ && ctx->qat_hash_alg == ++ ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC)) { ++ raw_dp_ctx->enqueue_burst = ++ qat_sym_dp_enqueue_aead_jobs_gen1; ++ raw_dp_ctx->enqueue = ++ qat_sym_dp_enqueue_single_aead_gen1; ++ } else { ++ raw_dp_ctx->enqueue_burst = ++ qat_sym_dp_enqueue_chain_jobs_gen1; ++ raw_dp_ctx->enqueue = ++ qat_sym_dp_enqueue_single_chain_gen1; ++ } ++ } else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_AUTH || ctx->is_gmac) { ++ raw_dp_ctx->enqueue_burst = qat_sym_dp_enqueue_auth_jobs_gen1; ++ raw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_auth_gen1; ++ } else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER) { ++ if (ctx->qat_mode == ICP_QAT_HW_CIPHER_AEAD_MODE || ++ ctx->qat_cipher_alg == ++ ICP_QAT_HW_CIPHER_ALGO_CHACHA20_POLY1305) { ++ raw_dp_ctx->enqueue_burst = ++ qat_sym_dp_enqueue_aead_jobs_gen1; ++ raw_dp_ctx->enqueue = ++ qat_sym_dp_enqueue_single_aead_gen1; ++ } else { ++ raw_dp_ctx->enqueue_burst = ++ qat_sym_dp_enqueue_cipher_jobs_gen1; ++ raw_dp_ctx->enqueue = ++ qat_sym_dp_enqueue_single_cipher_gen1; ++ } ++ } else ++ return -1; + + if (ctx->is_single_pass && ctx->is_ucs) { + raw_dp_ctx->enqueue_burst = qat_sym_dp_enqueue_aead_jobs_gen4; +diff --git a/dpdk/drivers/crypto/qat/dev/qat_crypto_pmd_gens.h b/dpdk/drivers/crypto/qat/dev/qat_crypto_pmd_gens.h +index b8ddf42d6f..64e892d022 100644 +--- a/dpdk/drivers/crypto/qat/dev/qat_crypto_pmd_gens.h ++++ b/dpdk/drivers/crypto/qat/dev/qat_crypto_pmd_gens.h +@@ -394,7 +394,7 @@ qat_sym_convert_op_to_vec_chain(struct rte_crypto_op *op, + struct qat_sym_op_cookie *cookie) + { + union rte_crypto_sym_ofs ofs; +- uint32_t max_len = 0; ++ uint32_t max_len = 0, oop_offset = 0; + uint32_t cipher_len = 0, cipher_ofs = 0; + uint32_t auth_len = 0, auth_ofs = 0; + int is_oop = (op->sym->m_dst != NULL) && +@@ -468,6 +468,16 @@ qat_sym_convert_op_to_vec_chain(struct rte_crypto_op *op, + + max_len = RTE_MAX(cipher_ofs + cipher_len, auth_ofs + auth_len); + ++ /* If OOP, we need to keep in mind that offset needs to start where ++ * cipher/auth starts, namely no offset on the smaller one ++ */ ++ if (is_oop) { ++ oop_offset = RTE_MIN(auth_ofs, cipher_ofs); ++ auth_ofs -= oop_offset; ++ cipher_ofs -= oop_offset; ++ max_len -= oop_offset; ++ } ++ + /* digest in buffer check. Needed only for wireless algos + * or combined cipher-crc operations + */ +@@ -508,9 +518,7 @@ qat_sym_convert_op_to_vec_chain(struct rte_crypto_op *op, + max_len = RTE_MAX(max_len, auth_ofs + auth_len + + ctx->digest_length); + } +- +- /* Passing 0 as cipher & auth offsets are assigned into ofs later */ +- n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, 0, max_len, ++ n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, oop_offset, max_len, + in_sgl->vec, QAT_SYM_SGL_MAX_NUMBER); + if (unlikely(n_src < 0 || n_src > op->sym->m_src->nb_segs)) { + op->status = RTE_CRYPTO_OP_STATUS_ERROR; +@@ -520,7 +528,7 @@ qat_sym_convert_op_to_vec_chain(struct rte_crypto_op *op, + + if (unlikely((op->sym->m_dst != NULL) && + (op->sym->m_dst != op->sym->m_src))) { +- int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, 0, ++ int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, oop_offset, + max_len, out_sgl->vec, QAT_SYM_SGL_MAX_NUMBER); + + if (n_dst < 0 || n_dst > op->sym->m_dst->nb_segs) { +@@ -894,10 +902,12 @@ enqueue_one_aead_job_gen1(struct qat_sym_session *ctx, + *(uint8_t *)&cipher_param->u.cipher_IV_array[0] = + q - ICP_QAT_HW_CCM_NONCE_OFFSET; + +- rte_memcpy((uint8_t *)aad->va + +- ICP_QAT_HW_CCM_NONCE_OFFSET, +- (uint8_t *)iv->va + ICP_QAT_HW_CCM_NONCE_OFFSET, +- ctx->cipher_iv.length); ++ if (ctx->aad_len > 0) { ++ rte_memcpy((uint8_t *)aad->va + ++ ICP_QAT_HW_CCM_NONCE_OFFSET, ++ (uint8_t *)iv->va + ICP_QAT_HW_CCM_NONCE_OFFSET, ++ ctx->cipher_iv.length); ++ } + break; + default: + break; +@@ -1007,6 +1017,12 @@ qat_sym_dp_enqueue_done_gen1(void *qp_data, uint8_t *drv_ctx, uint32_t n); + int + qat_sym_dp_dequeue_done_gen1(void *qp_data, uint8_t *drv_ctx, uint32_t n); + ++int ++qat_sym_dp_enqueue_done_gen4(void *qp_data, uint8_t *drv_ctx, uint32_t n); ++ ++int ++qat_sym_dp_dequeue_done_gen4(void *qp_data, uint8_t *drv_ctx, uint32_t n); ++ + int + qat_sym_configure_raw_dp_ctx_gen1(void *_raw_dp_ctx, void *_ctx); + +diff --git a/dpdk/drivers/crypto/qat/qat_sym.c b/dpdk/drivers/crypto/qat/qat_sym.c +index 6e03bde841..8235fc0a5a 100644 +--- a/dpdk/drivers/crypto/qat/qat_sym.c ++++ b/dpdk/drivers/crypto/qat/qat_sym.c +@@ -18,7 +18,6 @@ + #include "qat_qp.h" + + uint8_t qat_sym_driver_id; +-int qat_legacy_capa; + + struct qat_crypto_gen_dev_ops qat_sym_gen_dev_ops[QAT_N_GENS]; + +@@ -266,7 +265,7 @@ qat_sym_dev_create(struct qat_pci_device *qat_pci_dev, + } + + cryptodev->feature_flags |= RTE_CRYPTODEV_FF_SECURITY; +- QAT_LOG(INFO, "Device %s rte_security support ensabled", name); ++ QAT_LOG(INFO, "Device %s rte_security support enabled", name); + } else { + QAT_LOG(INFO, "Device %s rte_security support disabled", name); + } +diff --git a/dpdk/drivers/dma/dpaa2/dpaa2_qdma.c b/dpdk/drivers/dma/dpaa2/dpaa2_qdma.c +index 8968bb853b..2c91ceec13 100644 +--- a/dpdk/drivers/dma/dpaa2/dpaa2_qdma.c ++++ b/dpdk/drivers/dma/dpaa2/dpaa2_qdma.c +@@ -16,9 +16,6 @@ + + #define DPAA2_QDMA_PREFETCH "prefetch" + +-/* Dynamic log type identifier */ +-int dpaa2_qdma_logtype; +- + uint32_t dpaa2_coherent_no_alloc_cache; + uint32_t dpaa2_coherent_alloc_cache; + +@@ -1699,4 +1696,4 @@ static struct rte_dpaa2_driver rte_dpaa2_qdma_pmd = { + RTE_PMD_REGISTER_DPAA2(dpaa2_qdma, rte_dpaa2_qdma_pmd); + RTE_PMD_REGISTER_PARAM_STRING(dpaa2_qdma, + "no_prefetch= "); +-RTE_LOG_REGISTER_DEFAULT(dpaa_qdma2_logtype, INFO); ++RTE_LOG_REGISTER_DEFAULT(dpaa2_qdma_logtype, INFO); +diff --git a/dpdk/drivers/dma/hisilicon/hisi_dmadev.c b/dpdk/drivers/dma/hisilicon/hisi_dmadev.c +index 0e11ca14cc..4db3b0554c 100644 +--- a/dpdk/drivers/dma/hisilicon/hisi_dmadev.c ++++ b/dpdk/drivers/dma/hisilicon/hisi_dmadev.c +@@ -39,8 +39,6 @@ hisi_dma_queue_base(struct hisi_dma_dev *hw) + { + if (hw->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) + return HISI_DMA_HIP08_QUEUE_BASE; +- else if (hw->reg_layout == HISI_DMA_REG_LAYOUT_HIP09) +- return HISI_DMA_HIP09_QUEUE_BASE; + else + return 0; + } +@@ -216,25 +214,6 @@ hisi_dma_init_hw(struct hisi_dma_dev *hw) + HISI_DMA_HIP08_QUEUE_INT_MASK_M, true); + hisi_dma_update_queue_mbit(hw, HISI_DMA_QUEUE_INT_MASK_REG, + HISI_DMA_HIP08_QUEUE_INT_MASK_M, true); +- } else if (hw->reg_layout == HISI_DMA_REG_LAYOUT_HIP09) { +- hisi_dma_update_queue_mbit(hw, HISI_DMA_QUEUE_CTRL0_REG, +- HISI_DMA_HIP09_QUEUE_CTRL0_ERR_ABORT_M, false); +- hisi_dma_update_queue_mbit(hw, HISI_DMA_QUEUE_INT_STATUS_REG, +- HISI_DMA_HIP09_QUEUE_INT_MASK_M, true); +- hisi_dma_update_queue_mbit(hw, HISI_DMA_QUEUE_INT_MASK_REG, +- HISI_DMA_HIP09_QUEUE_INT_MASK_M, true); +- hisi_dma_update_queue_mbit(hw, +- HISI_DMA_HIP09_QUEUE_ERR_INT_STATUS_REG, +- HISI_DMA_HIP09_QUEUE_ERR_INT_MASK_M, true); +- hisi_dma_update_queue_mbit(hw, +- HISI_DMA_HIP09_QUEUE_ERR_INT_MASK_REG, +- HISI_DMA_HIP09_QUEUE_ERR_INT_MASK_M, true); +- hisi_dma_update_queue_bit(hw, HISI_DMA_QUEUE_CTRL1_REG, +- HISI_DMA_HIP09_QUEUE_CTRL1_VA_ENABLE_B, true); +- hisi_dma_update_bit(hw, +- HISI_DMA_HIP09_QUEUE_CFG_REG(hw->queue_id), +- HISI_DMA_HIP09_QUEUE_CFG_LINK_DOWN_MASK_B, +- true); + } + } + +@@ -256,8 +235,6 @@ hisi_dma_reg_layout(uint8_t revision) + { + if (revision == HISI_DMA_REVISION_HIP08B) + return HISI_DMA_REG_LAYOUT_HIP08; +- else if (revision >= HISI_DMA_REVISION_HIP09A) +- return HISI_DMA_REG_LAYOUT_HIP09; + else + return HISI_DMA_REG_LAYOUT_INVALID; + } +@@ -328,14 +305,11 @@ hisi_dma_info_get(const struct rte_dma_dev *dev, + struct rte_dma_info *dev_info, + uint32_t info_sz) + { +- struct hisi_dma_dev *hw = dev->data->dev_private; ++ RTE_SET_USED(dev); + RTE_SET_USED(info_sz); + + dev_info->dev_capa = RTE_DMA_CAPA_MEM_TO_MEM | + RTE_DMA_CAPA_OPS_COPY; +- if (hw->reg_layout == HISI_DMA_REG_LAYOUT_HIP09) +- dev_info->dev_capa |= RTE_DMA_CAPA_HANDLES_ERRORS; +- + dev_info->max_vchans = 1; + dev_info->max_desc = HISI_DMA_MAX_DESC_NUM; + dev_info->min_desc = HISI_DMA_MIN_DESC_NUM; +@@ -514,18 +488,6 @@ hisi_dma_dump_common(struct hisi_dma_dev *hw, FILE *f) + { HISI_DMA_REG_LAYOUT_HIP08, + HISI_DMA_HIP08_DUMP_START_REG, + HISI_DMA_HIP08_DUMP_END_REG }, +- { HISI_DMA_REG_LAYOUT_HIP09, +- HISI_DMA_HIP09_DUMP_REGION_A_START_REG, +- HISI_DMA_HIP09_DUMP_REGION_A_END_REG }, +- { HISI_DMA_REG_LAYOUT_HIP09, +- HISI_DMA_HIP09_DUMP_REGION_B_START_REG, +- HISI_DMA_HIP09_DUMP_REGION_B_END_REG }, +- { HISI_DMA_REG_LAYOUT_HIP09, +- HISI_DMA_HIP09_DUMP_REGION_C_START_REG, +- HISI_DMA_HIP09_DUMP_REGION_C_END_REG }, +- { HISI_DMA_REG_LAYOUT_HIP09, +- HISI_DMA_HIP09_DUMP_REGION_D_START_REG, +- HISI_DMA_HIP09_DUMP_REGION_D_END_REG }, + }; + uint32_t i; + +diff --git a/dpdk/drivers/dma/hisilicon/hisi_dmadev.h b/dpdk/drivers/dma/hisilicon/hisi_dmadev.h +index 5a17f9f69e..a57b5c759a 100644 +--- a/dpdk/drivers/dma/hisilicon/hisi_dmadev.h ++++ b/dpdk/drivers/dma/hisilicon/hisi_dmadev.h +@@ -25,22 +25,14 @@ + #define HISI_DMA_DEVICE_ID 0xA122 + #define HISI_DMA_PCI_REVISION_ID_REG 0x08 + #define HISI_DMA_REVISION_HIP08B 0x21 +-#define HISI_DMA_REVISION_HIP09A 0x30 + + #define HISI_DMA_MAX_HW_QUEUES 4 + #define HISI_DMA_MAX_DESC_NUM 8192 + #define HISI_DMA_MIN_DESC_NUM 32 + +-/** +- * The HIP08B(HiSilicon IP08) and HIP09B(HiSilicon IP09) are DMA iEPs, they +- * have the same pci device id but different pci revision. +- * Unfortunately, they have different register layouts, so two layout +- * enumerations are defined. +- */ + enum { + HISI_DMA_REG_LAYOUT_INVALID = 0, +- HISI_DMA_REG_LAYOUT_HIP08, +- HISI_DMA_REG_LAYOUT_HIP09 ++ HISI_DMA_REG_LAYOUT_HIP08 + }; + + /** +@@ -69,9 +61,6 @@ enum { + * length of queue-region. The global offset for a single queue register is + * calculated by: + * offset = queue-base + (queue-id * queue-region) + reg-offset-in-region. +- * +- * The first part of queue region is basically the same for HIP08 and HIP09 +- * register layouts, therefore, HISI_QUEUE_* registers are defined for it. + */ + #define HISI_DMA_QUEUE_SQ_BASE_L_REG 0x0 + #define HISI_DMA_QUEUE_SQ_BASE_H_REG 0x4 +@@ -110,28 +99,6 @@ enum { + #define HISI_DMA_HIP08_DUMP_START_REG 0x2000 + #define HISI_DMA_HIP08_DUMP_END_REG 0x2280 + +-/** +- * HiSilicon IP09 DMA register and field define: +- */ +-#define HISI_DMA_HIP09_QUEUE_BASE 0x2000 +-#define HISI_DMA_HIP09_QUEUE_CTRL0_ERR_ABORT_M GENMASK(31, 28) +-#define HISI_DMA_HIP09_QUEUE_CTRL1_VA_ENABLE_B 2 +-#define HISI_DMA_HIP09_QUEUE_INT_MASK_M 0x1 +-#define HISI_DMA_HIP09_QUEUE_ERR_INT_STATUS_REG 0x48 +-#define HISI_DMA_HIP09_QUEUE_ERR_INT_MASK_REG 0x4C +-#define HISI_DMA_HIP09_QUEUE_ERR_INT_MASK_M GENMASK(18, 1) +-#define HISI_DMA_HIP09_QUEUE_CFG_REG(queue_id) (0x800 + \ +- (queue_id) * 0x20) +-#define HISI_DMA_HIP09_QUEUE_CFG_LINK_DOWN_MASK_B 16 +-#define HISI_DMA_HIP09_DUMP_REGION_A_START_REG 0x0 +-#define HISI_DMA_HIP09_DUMP_REGION_A_END_REG 0x368 +-#define HISI_DMA_HIP09_DUMP_REGION_B_START_REG 0x800 +-#define HISI_DMA_HIP09_DUMP_REGION_B_END_REG 0xA08 +-#define HISI_DMA_HIP09_DUMP_REGION_C_START_REG 0x1800 +-#define HISI_DMA_HIP09_DUMP_REGION_C_END_REG 0x1A4C +-#define HISI_DMA_HIP09_DUMP_REGION_D_START_REG 0x1C00 +-#define HISI_DMA_HIP09_DUMP_REGION_D_END_REG 0x1CC4 +- + /** + * In fact, there are multiple states, but it need to pay attention to + * the following three states for the driver: +diff --git a/dpdk/drivers/dma/idxd/dpdk_idxd_cfg.py b/dpdk/drivers/dma/idxd/dpdk_idxd_cfg.py +index c0c833ade9..5c9572b49d 100755 +--- a/dpdk/drivers/dma/idxd/dpdk_idxd_cfg.py ++++ b/dpdk/drivers/dma/idxd/dpdk_idxd_cfg.py +@@ -104,8 +104,10 @@ def configure_dsa(dsa_id, args): + "priority": 1, + "max_batch_size": 1024, + "size": int(max_work_queues_size / nb_queues)} +- wqcfg.update(parse_wq_opts(args.wq_option)) + wq_dir = SysfsDir(os.path.join(dsa_dir.path, f"wq{dsa_id}.{q}")) ++ if os.path.exists(os.path.join(wq_dir.path, f"driver_name")): ++ wqcfg.update({"driver_name": "user"}) ++ wqcfg.update(parse_wq_opts(args.wq_option)) + wq_dir.write_values(wqcfg) + + # enable device and then queues +diff --git a/dpdk/drivers/dma/idxd/idxd_bus.c b/dpdk/drivers/dma/idxd/idxd_bus.c +index 3b2d4c2b65..ba8076715d 100644 +--- a/dpdk/drivers/dma/idxd/idxd_bus.c ++++ b/dpdk/drivers/dma/idxd/idxd_bus.c +@@ -261,9 +261,15 @@ static int + is_for_this_process_use(struct rte_dsa_device *dev, const char *name) + { + char *runtime_dir = strdup(rte_eal_get_runtime_dir()); +- char *prefix = basename(runtime_dir); +- int prefixlen = strlen(prefix); + int retval = 0; ++ int prefixlen; ++ char *prefix; ++ ++ if (runtime_dir == NULL) ++ return retval; ++ ++ prefix = basename(runtime_dir); ++ prefixlen = strlen(prefix); + + if (strncmp(name, "dpdk_", 5) == 0) + retval = 1; +diff --git a/dpdk/drivers/event/cnxk/cn10k_tx_worker.h b/dpdk/drivers/event/cnxk/cn10k_tx_worker.h +index 53e0dde20c..256237b895 100644 +--- a/dpdk/drivers/event/cnxk/cn10k_tx_worker.h ++++ b/dpdk/drivers/event/cnxk/cn10k_tx_worker.h +@@ -70,6 +70,7 @@ cn10k_sso_tx_one(struct cn10k_sso_hws *ws, struct rte_mbuf *m, uint64_t *cmd, + const uint64_t *txq_data, const uint32_t flags) + { + uint8_t lnum = 0, loff = 0, shft = 0; ++ struct rte_mbuf *extm = NULL; + struct cn10k_eth_txq *txq; + uintptr_t laddr; + uint16_t segdw; +@@ -90,7 +91,7 @@ cn10k_sso_tx_one(struct cn10k_sso_hws *ws, struct rte_mbuf *m, uint64_t *cmd, + if (flags & NIX_TX_OFFLOAD_TSO_F) + cn10k_nix_xmit_prepare_tso(m, flags); + +- cn10k_nix_xmit_prepare(txq, m, cmd, flags, txq->lso_tun_fmt, &sec, ++ cn10k_nix_xmit_prepare(txq, m, &extm, cmd, flags, txq->lso_tun_fmt, &sec, + txq->mark_flag, txq->mark_fmt); + + laddr = lmt_addr; +@@ -105,7 +106,7 @@ cn10k_sso_tx_one(struct cn10k_sso_hws *ws, struct rte_mbuf *m, uint64_t *cmd, + cn10k_nix_xmit_mv_lmt_base(laddr, cmd, flags); + + if (flags & NIX_TX_MULTI_SEG_F) +- segdw = cn10k_nix_prepare_mseg(txq, m, (uint64_t *)laddr, flags); ++ segdw = cn10k_nix_prepare_mseg(txq, m, &extm, (uint64_t *)laddr, flags); + else + segdw = cn10k_nix_tx_ext_subs(flags) + 2; + +@@ -127,6 +128,9 @@ cn10k_sso_tx_one(struct cn10k_sso_hws *ws, struct rte_mbuf *m, uint64_t *cmd, + /* Memory barrier to make sure lmtst store completes */ + rte_io_wmb(); + ++ if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F && !txq->tx_compl.ena) ++ cn10k_nix_free_extmbuf(extm); ++ + return 1; + } + +diff --git a/dpdk/drivers/event/cnxk/cn9k_worker.h b/dpdk/drivers/event/cnxk/cn9k_worker.h +index 0451157812..107265d54b 100644 +--- a/dpdk/drivers/event/cnxk/cn9k_worker.h ++++ b/dpdk/drivers/event/cnxk/cn9k_worker.h +@@ -746,7 +746,7 @@ static __rte_always_inline uint16_t + cn9k_sso_hws_event_tx(uint64_t base, struct rte_event *ev, uint64_t *cmd, + uint64_t *txq_data, const uint32_t flags) + { +- struct rte_mbuf *m = ev->mbuf; ++ struct rte_mbuf *m = ev->mbuf, *extm = NULL; + struct cn9k_eth_txq *txq; + + /* Perform header writes before barrier for TSO */ +@@ -767,7 +767,7 @@ cn9k_sso_hws_event_tx(uint64_t base, struct rte_event *ev, uint64_t *cmd, + if (cn9k_sso_sq_depth(txq) <= 0) + return 0; + cn9k_nix_tx_skeleton(txq, cmd, flags, 0); +- cn9k_nix_xmit_prepare(txq, m, cmd, flags, txq->lso_tun_fmt, txq->mark_flag, ++ cn9k_nix_xmit_prepare(txq, m, &extm, cmd, flags, txq->lso_tun_fmt, txq->mark_flag, + txq->mark_fmt); + + if (flags & NIX_TX_OFFLOAD_SECURITY_F) { +@@ -789,7 +789,7 @@ cn9k_sso_hws_event_tx(uint64_t base, struct rte_event *ev, uint64_t *cmd, + } + + if (flags & NIX_TX_MULTI_SEG_F) { +- const uint16_t segdw = cn9k_nix_prepare_mseg(txq, m, cmd, flags); ++ const uint16_t segdw = cn9k_nix_prepare_mseg(txq, m, &extm, cmd, flags); + cn9k_nix_xmit_prepare_tstamp(txq, cmd, m->ol_flags, segdw, + flags); + if (!CNXK_TT_FROM_EVENT(ev->event)) { +@@ -819,6 +819,9 @@ cn9k_sso_hws_event_tx(uint64_t base, struct rte_event *ev, uint64_t *cmd, + } + + done: ++ if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F && !txq->tx_compl.ena) ++ cn9k_nix_free_extmbuf(extm); ++ + return 1; + } + +diff --git a/dpdk/drivers/event/cnxk/cnxk_eventdev.c b/dpdk/drivers/event/cnxk/cnxk_eventdev.c +index 0c61f4c20e..20f7f0d6df 100644 +--- a/dpdk/drivers/event/cnxk/cnxk_eventdev.c ++++ b/dpdk/drivers/event/cnxk/cnxk_eventdev.c +@@ -162,16 +162,17 @@ cnxk_sso_dev_validate(const struct rte_eventdev *event_dev, uint32_t deq_depth, + + deq_tmo_ns = conf->dequeue_timeout_ns; + +- if (deq_tmo_ns == 0) +- deq_tmo_ns = dev->min_dequeue_timeout_ns; +- if (deq_tmo_ns < dev->min_dequeue_timeout_ns || +- deq_tmo_ns > dev->max_dequeue_timeout_ns) { ++ if (deq_tmo_ns && (deq_tmo_ns < dev->min_dequeue_timeout_ns || ++ deq_tmo_ns > dev->max_dequeue_timeout_ns)) { + plt_err("Unsupported dequeue timeout requested"); + return -EINVAL; + } + +- if (conf->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT) ++ if (conf->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT) { ++ if (deq_tmo_ns == 0) ++ deq_tmo_ns = dev->min_dequeue_timeout_ns; + dev->is_timeout_deq = 1; ++ } + + dev->deq_tmo_ns = deq_tmo_ns; + +@@ -553,6 +554,9 @@ parse_list(const char *value, void *opaque, param_parse_t fn) + char *end = NULL; + char *f = s; + ++ if (s == NULL) ++ return; ++ + while (*s) { + if (*s == '[') + start = s; +@@ -663,7 +667,7 @@ cnxk_sso_init(struct rte_eventdev *event_dev) + } + + dev->is_timeout_deq = 0; +- dev->min_dequeue_timeout_ns = 0; ++ dev->min_dequeue_timeout_ns = USEC2NSEC(1); + dev->max_dequeue_timeout_ns = USEC2NSEC(0x3FF); + dev->max_num_events = -1; + dev->nb_event_queues = 0; +diff --git a/dpdk/drivers/event/dlb2/dlb2.c b/dpdk/drivers/event/dlb2/dlb2.c +index 050ace0904..5044cb17ef 100644 +--- a/dpdk/drivers/event/dlb2/dlb2.c ++++ b/dpdk/drivers/event/dlb2/dlb2.c +@@ -160,7 +160,6 @@ static int + dlb2_hw_query_resources(struct dlb2_eventdev *dlb2) + { + struct dlb2_hw_dev *handle = &dlb2->qm_instance; +- struct dlb2_hw_resource_info *dlb2_info = &handle->info; + int num_ldb_ports; + int ret; + +@@ -222,8 +221,6 @@ dlb2_hw_query_resources(struct dlb2_eventdev *dlb2) + handle->info.hw_rsrc_max.reorder_window_size = + dlb2->hw_rsrc_query_results.num_hist_list_entries; + +- rte_memcpy(dlb2_info, &handle->info.hw_rsrc_max, sizeof(*dlb2_info)); +- + return 0; + } + +diff --git a/dpdk/drivers/event/sw/iq_chunk.h b/dpdk/drivers/event/sw/iq_chunk.h +index 31d013eab7..7820815c38 100644 +--- a/dpdk/drivers/event/sw/iq_chunk.h ++++ b/dpdk/drivers/event/sw/iq_chunk.h +@@ -9,8 +9,6 @@ + #include + #include + +-#define IQ_ROB_NAMESIZE 12 +- + struct sw_queue_chunk { + struct rte_event events[SW_EVS_PER_Q_CHUNK]; + struct sw_queue_chunk *next; +diff --git a/dpdk/drivers/event/sw/sw_evdev.c b/dpdk/drivers/event/sw/sw_evdev.c +index 55e7735cb0..2096496917 100644 +--- a/dpdk/drivers/event/sw/sw_evdev.c ++++ b/dpdk/drivers/event/sw/sw_evdev.c +@@ -228,9 +228,7 @@ qid_init(struct sw_evdev *sw, unsigned int idx, int type, + const struct rte_event_queue_conf *queue_conf) + { + unsigned int i; +- int dev_id = sw->data->dev_id; + int socket_id = sw->data->socket_id; +- char buf[IQ_ROB_NAMESIZE]; + struct sw_qid *qid = &sw->qids[idx]; + + /* Initialize the FID structures to no pinning (-1), and zero packets */ +@@ -260,8 +258,7 @@ qid_init(struct sw_evdev *sw, unsigned int idx, int type, + goto cleanup; + } + +- snprintf(buf, sizeof(buf), "sw%d_iq_%d_rob", dev_id, i); +- qid->reorder_buffer = rte_zmalloc_socket(buf, ++ qid->reorder_buffer = rte_zmalloc_socket(NULL, + window_size * sizeof(qid->reorder_buffer[0]), + 0, socket_id); + if (!qid->reorder_buffer) { +diff --git a/dpdk/drivers/meson.build b/dpdk/drivers/meson.build +index 5ba534049a..f2be71bc05 100644 +--- a/dpdk/drivers/meson.build ++++ b/dpdk/drivers/meson.build +@@ -93,7 +93,7 @@ foreach subpath:subdirs + if skip_class + drv_path = join_paths(class, '*') + dpdk_drvs_disabled += drv_path +- set_variable(drv_path.underscorify() + '_disable_reason', reason) ++ set_variable('drv_' + drv_path.underscorify() + '_disable_reason', reason) + continue + endif + endif +@@ -199,7 +199,7 @@ foreach subpath:subdirs + # component disable printout in those cases + if reason != '' + dpdk_drvs_disabled += drv_path +- set_variable(drv_path.underscorify() + '_disable_reason', reason) ++ set_variable('drv_' + drv_path.underscorify() + '_disable_reason', reason) + endif + continue + endif +diff --git a/dpdk/drivers/ml/cnxk/cn10k_ml_ops.c b/dpdk/drivers/ml/cnxk/cn10k_ml_ops.c +index 7f7e5efcea..5370038733 100644 +--- a/dpdk/drivers/ml/cnxk/cn10k_ml_ops.c ++++ b/dpdk/drivers/ml/cnxk/cn10k_ml_ops.c +@@ -288,6 +288,7 @@ cn10k_ml_model_xstat_get(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_layer *l + static int + cn10k_ml_cache_model_data(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_layer *layer) + { ++ struct cn10k_ml_layer_xstats *xstats; + char str[RTE_MEMZONE_NAMESIZE]; + const struct plt_memzone *mz; + uint64_t isize = 0; +@@ -309,6 +310,16 @@ cn10k_ml_cache_model_data(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_layer * + PLT_PTR_ADD(mz->addr, isize), 1); + plt_memzone_free(mz); + ++ /* Reset sync xstats. */ ++ xstats = layer->glow.sync_xstats; ++ xstats->hw_latency_tot = 0; ++ xstats->hw_latency_min = UINT64_MAX; ++ xstats->hw_latency_max = 0; ++ xstats->fw_latency_tot = 0; ++ xstats->fw_latency_min = UINT64_MAX; ++ xstats->fw_latency_max = 0; ++ xstats->dequeued_count = 0; ++ + return ret; + } + +diff --git a/dpdk/drivers/net/af_packet/rte_eth_af_packet.c b/dpdk/drivers/net/af_packet/rte_eth_af_packet.c +index 397a32db58..6b7b16f348 100644 +--- a/dpdk/drivers/net/af_packet/rte_eth_af_packet.c ++++ b/dpdk/drivers/net/af_packet/rte_eth_af_packet.c +@@ -6,6 +6,7 @@ + * All rights reserved. + */ + ++#include + #include + #include + #include +@@ -39,7 +40,7 @@ + #define DFLT_FRAME_SIZE (1 << 11) + #define DFLT_FRAME_COUNT (1 << 9) + +-struct pkt_rx_queue { ++struct __rte_cache_aligned pkt_rx_queue { + int sockfd; + + struct iovec *rd; +@@ -55,7 +56,7 @@ struct pkt_rx_queue { + volatile unsigned long rx_bytes; + }; + +-struct pkt_tx_queue { ++struct __rte_cache_aligned pkt_tx_queue { + int sockfd; + unsigned int frame_data_size; + +diff --git a/dpdk/drivers/net/af_xdp/compat.h b/dpdk/drivers/net/af_xdp/compat.h +index 28ea64aeaa..3b5a5c1ed5 100644 +--- a/dpdk/drivers/net/af_xdp/compat.h ++++ b/dpdk/drivers/net/af_xdp/compat.h +@@ -46,6 +46,21 @@ create_shared_socket(struct xsk_socket **xsk_ptr __rte_unused, + } + #endif + ++#ifdef ETH_AF_XDP_UPDATE_XSKMAP ++static __rte_always_inline int ++update_xskmap(struct xsk_socket *xsk, int map_fd, int xsk_queue_idx __rte_unused) ++{ ++ return xsk_socket__update_xskmap(xsk, map_fd); ++} ++#else ++static __rte_always_inline int ++update_xskmap(struct xsk_socket *xsk, int map_fd, int xsk_queue_idx) ++{ ++ int fd = xsk_socket__fd(xsk); ++ return bpf_map_update_elem(map_fd, &xsk_queue_idx, &fd, 0); ++} ++#endif ++ + #ifdef XDP_USE_NEED_WAKEUP + static int + tx_syscall_needed(struct xsk_ring_prod *q) +diff --git a/dpdk/drivers/net/af_xdp/meson.build b/dpdk/drivers/net/af_xdp/meson.build +index 9f33e57fa2..69d109ff46 100644 +--- a/dpdk/drivers/net/af_xdp/meson.build ++++ b/dpdk/drivers/net/af_xdp/meson.build +@@ -7,6 +7,12 @@ if is_windows + subdir_done() + endif + ++if arch_subdir == 'x86' and dpdk_conf.get('RTE_ARCH_32') ++ build = false ++ reason = 'not supported on 32-bit x86' ++ subdir_done() ++endif ++ + sources = files('rte_eth_af_xdp.c') + + libxdp_ver = '>=1.2.2' +@@ -77,6 +83,10 @@ if build + dependencies : bpf_dep, args: cflags) + cflags += ['-DRTE_NET_AF_XDP_LIBBPF_XDP_ATTACH'] + endif ++ if cc.has_function('xsk_socket__update_xskmap', prefix : xsk_check_prefix, ++ dependencies : ext_deps, args: cflags) ++ cflags += ['-DETH_AF_XDP_UPDATE_XSKMAP'] ++ endif + endif + + require_iova_in_mbuf = false +diff --git a/dpdk/drivers/net/af_xdp/rte_eth_af_xdp.c b/dpdk/drivers/net/af_xdp/rte_eth_af_xdp.c +index 353c8688ec..74f750dbb3 100644 +--- a/dpdk/drivers/net/af_xdp/rte_eth_af_xdp.c ++++ b/dpdk/drivers/net/af_xdp/rte_eth_af_xdp.c +@@ -83,12 +83,13 @@ RTE_LOG_REGISTER_DEFAULT(af_xdp_logtype, NOTICE); + + #define ETH_AF_XDP_MP_KEY "afxdp_mp_send_fds" + ++#define DP_BASE_PATH "/tmp/afxdp_dp" ++#define DP_UDS_SOCK "afxdp.sock" + #define MAX_LONG_OPT_SZ 64 + #define UDS_MAX_FD_NUM 2 + #define UDS_MAX_CMD_LEN 64 + #define UDS_MAX_CMD_RESP 128 + #define UDS_XSK_MAP_FD_MSG "/xsk_map_fd" +-#define UDS_SOCK "/tmp/afxdp.sock" + #define UDS_CONNECT_MSG "/connect" + #define UDS_HOST_OK_MSG "/host_ok" + #define UDS_HOST_NAK_MSG "/host_nak" +@@ -123,7 +124,7 @@ struct xsk_umem_info { + struct rx_stats { + uint64_t rx_pkts; + uint64_t rx_bytes; +- uint64_t rx_dropped; ++ uint64_t imissed_offset; + }; + + struct pkt_rx_queue { +@@ -131,6 +132,7 @@ struct pkt_rx_queue { + struct xsk_umem_info *umem; + struct xsk_socket *xsk; + struct rte_mempool *mb_pool; ++ uint16_t port; + + struct rx_stats stats; + +@@ -171,6 +173,7 @@ struct pmd_internals { + bool custom_prog_configured; + bool force_copy; + bool use_cni; ++ char dp_path[PATH_MAX]; + struct bpf_map *map; + + struct rte_ether_addr eth_addr; +@@ -191,6 +194,7 @@ struct pmd_process_private { + #define ETH_AF_XDP_BUDGET_ARG "busy_budget" + #define ETH_AF_XDP_FORCE_COPY_ARG "force_copy" + #define ETH_AF_XDP_USE_CNI_ARG "use_cni" ++#define ETH_AF_XDP_DP_PATH_ARG "dp_path" + + static const char * const valid_arguments[] = { + ETH_AF_XDP_IFACE_ARG, +@@ -201,6 +205,7 @@ static const char * const valid_arguments[] = { + ETH_AF_XDP_BUDGET_ARG, + ETH_AF_XDP_FORCE_COPY_ARG, + ETH_AF_XDP_USE_CNI_ARG, ++ ETH_AF_XDP_DP_PATH_ARG, + NULL + }; + +@@ -311,6 +316,7 @@ af_xdp_rx_zc(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) + unsigned long rx_bytes = 0; + int i; + struct rte_mbuf *fq_bufs[ETH_AF_XDP_RX_BATCH_SIZE]; ++ struct rte_eth_dev *dev = &rte_eth_devices[rxq->port]; + + nb_pkts = xsk_ring_cons__peek(rx, nb_pkts, &idx_rx); + +@@ -338,6 +344,8 @@ af_xdp_rx_zc(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) + * xsk_ring_cons__peek + */ + rx->cached_cons -= nb_pkts; ++ dev->data->rx_mbuf_alloc_failed += nb_pkts; ++ + return 0; + } + +@@ -360,6 +368,7 @@ af_xdp_rx_zc(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) + bufs[i]->data_off = offset - sizeof(struct rte_mbuf) - + rte_pktmbuf_priv_size(umem->mb_pool) - + umem->mb_pool->header_size; ++ bufs[i]->port = rxq->port; + + rte_pktmbuf_pkt_len(bufs[i]) = len; + rte_pktmbuf_data_len(bufs[i]) = len; +@@ -388,6 +397,7 @@ af_xdp_rx_cp(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) + int i; + uint32_t free_thresh = fq->size >> 1; + struct rte_mbuf *mbufs[ETH_AF_XDP_RX_BATCH_SIZE]; ++ struct rte_eth_dev *dev = &rte_eth_devices[rxq->port]; + + if (xsk_prod_nb_free(fq, free_thresh) >= free_thresh) + (void)reserve_fill_queue(umem, nb_pkts, NULL, fq); +@@ -406,6 +416,7 @@ af_xdp_rx_cp(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) + * xsk_ring_cons__peek + */ + rx->cached_cons -= nb_pkts; ++ dev->data->rx_mbuf_alloc_failed += nb_pkts; + return 0; + } + +@@ -426,6 +437,7 @@ af_xdp_rx_cp(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) + rte_pktmbuf_data_len(mbufs[i]) = len; + rx_bytes += len; + bufs[i] = mbufs[i]; ++ bufs[i]->port = rxq->port; + } + + xsk_ring_cons__release(rx, nb_pkts); +@@ -867,7 +879,6 @@ eth_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) + + stats->ipackets += stats->q_ipackets[i]; + stats->ibytes += stats->q_ibytes[i]; +- stats->imissed += rxq->stats.rx_dropped; + stats->oerrors += txq->stats.tx_dropped; + fd = process_private->rxq_xsk_fds[i]; + ret = fd >= 0 ? getsockopt(fd, SOL_XDP, XDP_STATISTICS, +@@ -876,7 +887,7 @@ eth_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) + AF_XDP_LOG(ERR, "getsockopt() failed for XDP_STATISTICS.\n"); + return -1; + } +- stats->imissed += xdp_stats.rx_dropped; ++ stats->imissed += xdp_stats.rx_dropped - rxq->stats.imissed_offset; + + stats->opackets += stats->q_opackets[i]; + stats->obytes += stats->q_obytes[i]; +@@ -889,13 +900,25 @@ static int + eth_stats_reset(struct rte_eth_dev *dev) + { + struct pmd_internals *internals = dev->data->dev_private; +- int i; ++ struct pmd_process_private *process_private = dev->process_private; ++ struct xdp_statistics xdp_stats; ++ socklen_t optlen; ++ int i, ret, fd; + + for (i = 0; i < internals->queue_cnt; i++) { + memset(&internals->rx_queues[i].stats, 0, + sizeof(struct rx_stats)); + memset(&internals->tx_queues[i].stats, 0, + sizeof(struct tx_stats)); ++ fd = process_private->rxq_xsk_fds[i]; ++ optlen = sizeof(struct xdp_statistics); ++ ret = fd >= 0 ? getsockopt(fd, SOL_XDP, XDP_STATISTICS, ++ &xdp_stats, &optlen) : -1; ++ if (ret != 0) { ++ AF_XDP_LOG(ERR, "getsockopt() failed for XDP_STATISTICS.\n"); ++ return -1; ++ } ++ internals->rx_queues[i].stats.imissed_offset = xdp_stats.rx_dropped; + } + + return 0; +@@ -960,6 +983,9 @@ remove_xdp_program(struct pmd_internals *internals) + static void + xdp_umem_destroy(struct xsk_umem_info *umem) + { ++ (void)xsk_umem__delete(umem->umem); ++ umem->umem = NULL; ++ + #if defined(XDP_UMEM_UNALIGNED_CHUNK_FLAG) + umem->mb_pool = NULL; + #else +@@ -992,11 +1018,8 @@ eth_dev_close(struct rte_eth_dev *dev) + break; + xsk_socket__delete(rxq->xsk); + +- if (__atomic_fetch_sub(&rxq->umem->refcnt, 1, __ATOMIC_ACQUIRE) - 1 +- == 0) { +- (void)xsk_umem__delete(rxq->umem->umem); ++ if (__atomic_fetch_sub(&rxq->umem->refcnt, 1, __ATOMIC_ACQUIRE) - 1 == 0) + xdp_umem_destroy(rxq->umem); +- } + + /* free pkt_tx_queue */ + rte_free(rxq->pair); +@@ -1234,6 +1257,7 @@ xsk_umem_info *xdp_umem_configure(struct pmd_internals *internals, + AF_XDP_LOG(ERR, "Failed to reserve memzone for af_xdp umem.\n"); + goto err; + } ++ umem->mz = mz; + + ret = xsk_umem__create(&umem->umem, mz->addr, + ETH_AF_XDP_NUM_BUFFERS * ETH_AF_XDP_FRAME_SIZE, +@@ -1244,7 +1268,6 @@ xsk_umem_info *xdp_umem_configure(struct pmd_internals *internals, + AF_XDP_LOG(ERR, "Failed to create umem\n"); + goto err; + } +- umem->mz = mz; + + return umem; + +@@ -1351,7 +1374,7 @@ err_prefer: + } + + static int +-init_uds_sock(struct sockaddr_un *server) ++init_uds_sock(struct sockaddr_un *server, const char *dp_path) + { + int sock; + +@@ -1362,7 +1385,7 @@ init_uds_sock(struct sockaddr_un *server) + } + + server->sun_family = AF_UNIX; +- strlcpy(server->sun_path, UDS_SOCK, sizeof(server->sun_path)); ++ strlcpy(server->sun_path, dp_path, sizeof(server->sun_path)); + + if (connect(sock, (struct sockaddr *)server, sizeof(struct sockaddr_un)) < 0) { + close(sock); +@@ -1382,7 +1405,7 @@ struct msg_internal { + }; + + static int +-send_msg(int sock, char *request, int *fd) ++send_msg(int sock, char *request, int *fd, const char *dp_path) + { + int snd; + struct iovec iov; +@@ -1393,7 +1416,7 @@ send_msg(int sock, char *request, int *fd) + + memset(&dst, 0, sizeof(dst)); + dst.sun_family = AF_UNIX; +- strlcpy(dst.sun_path, UDS_SOCK, sizeof(dst.sun_path)); ++ strlcpy(dst.sun_path, dp_path, sizeof(dst.sun_path)); + + /* Initialize message header structure */ + memset(&msgh, 0, sizeof(msgh)); +@@ -1470,8 +1493,8 @@ read_msg(int sock, char *response, struct sockaddr_un *s, int *fd) + } + + static int +-make_request_cni(int sock, struct sockaddr_un *server, char *request, +- int *req_fd, char *response, int *out_fd) ++make_request_dp(int sock, struct sockaddr_un *server, char *request, ++ int *req_fd, char *response, int *out_fd, const char *dp_path) + { + int rval; + +@@ -1483,7 +1506,7 @@ make_request_cni(int sock, struct sockaddr_un *server, char *request, + if (req_fd == NULL) + rval = write(sock, request, strlen(request)); + else +- rval = send_msg(sock, request, req_fd); ++ rval = send_msg(sock, request, req_fd, dp_path); + + if (rval < 0) { + AF_XDP_LOG(ERR, "Write error %s\n", strerror(errno)); +@@ -1507,7 +1530,7 @@ check_response(char *response, char *exp_resp, long size) + } + + static int +-get_cni_fd(char *if_name) ++uds_get_xskmap_fd(char *if_name, const char *dp_path) + { + char request[UDS_MAX_CMD_LEN], response[UDS_MAX_CMD_RESP]; + char hostname[MAX_LONG_OPT_SZ], exp_resp[UDS_MAX_CMD_RESP]; +@@ -1520,14 +1543,14 @@ get_cni_fd(char *if_name) + return -1; + + memset(&server, 0, sizeof(server)); +- sock = init_uds_sock(&server); ++ sock = init_uds_sock(&server, dp_path); + if (sock < 0) + return -1; + +- /* Initiates handshake to CNI send: /connect,hostname */ ++ /* Initiates handshake to the AF_XDP Device Plugin send: /connect,hostname */ + snprintf(request, sizeof(request), "%s,%s", UDS_CONNECT_MSG, hostname); + memset(response, 0, sizeof(response)); +- if (make_request_cni(sock, &server, request, NULL, response, &out_fd) < 0) { ++ if (make_request_dp(sock, &server, request, NULL, response, &out_fd, dp_path) < 0) { + AF_XDP_LOG(ERR, "Error in processing cmd [%s]\n", request); + goto err_close; + } +@@ -1541,7 +1564,7 @@ get_cni_fd(char *if_name) + /* Request for "/version" */ + strlcpy(request, UDS_VERSION_MSG, UDS_MAX_CMD_LEN); + memset(response, 0, sizeof(response)); +- if (make_request_cni(sock, &server, request, NULL, response, &out_fd) < 0) { ++ if (make_request_dp(sock, &server, request, NULL, response, &out_fd, dp_path) < 0) { + AF_XDP_LOG(ERR, "Error in processing cmd [%s]\n", request); + goto err_close; + } +@@ -1549,7 +1572,7 @@ get_cni_fd(char *if_name) + /* Request for file descriptor for netdev name*/ + snprintf(request, sizeof(request), "%s,%s", UDS_XSK_MAP_FD_MSG, if_name); + memset(response, 0, sizeof(response)); +- if (make_request_cni(sock, &server, request, NULL, response, &out_fd) < 0) { ++ if (make_request_dp(sock, &server, request, NULL, response, &out_fd, dp_path) < 0) { + AF_XDP_LOG(ERR, "Error in processing cmd [%s]\n", request); + goto err_close; + } +@@ -1571,7 +1594,7 @@ get_cni_fd(char *if_name) + /* Initiate close connection */ + strlcpy(request, UDS_FIN_MSG, UDS_MAX_CMD_LEN); + memset(response, 0, sizeof(response)); +- if (make_request_cni(sock, &server, request, NULL, response, &out_fd) < 0) { ++ if (make_request_dp(sock, &server, request, NULL, response, &out_fd, dp_path) < 0) { + AF_XDP_LOG(ERR, "Error in processing cmd [%s]\n", request); + goto err_close; + } +@@ -1695,21 +1718,21 @@ xsk_configure(struct pmd_internals *internals, struct pkt_rx_queue *rxq, + } + + if (internals->use_cni) { +- int err, fd, map_fd; ++ int err, map_fd; + +- /* get socket fd from CNI plugin */ +- map_fd = get_cni_fd(internals->if_name); ++ /* get socket fd from AF_XDP Device Plugin */ ++ map_fd = uds_get_xskmap_fd(internals->if_name, internals->dp_path); + if (map_fd < 0) { +- AF_XDP_LOG(ERR, "Failed to receive CNI plugin fd\n"); ++ AF_XDP_LOG(ERR, "Failed to receive xskmap fd from AF_XDP Device Plugin\n"); + goto out_xsk; + } +- /* get socket fd */ +- fd = xsk_socket__fd(rxq->xsk); +- err = bpf_map_update_elem(map_fd, &rxq->xsk_queue_idx, &fd, 0); ++ ++ err = update_xskmap(rxq->xsk, map_fd, rxq->xsk_queue_idx); + if (err) { +- AF_XDP_LOG(ERR, "Failed to insert unprivileged xsk in map.\n"); ++ AF_XDP_LOG(ERR, "Failed to insert xsk in map.\n"); + goto out_xsk; + } ++ + } else if (rxq->busy_budget) { + ret = configure_preferred_busy_poll(rxq); + if (ret) { +@@ -1779,6 +1802,8 @@ eth_rx_queue_setup(struct rte_eth_dev *dev, + + process_private->rxq_xsk_fds[rx_queue_id] = rxq->fds[0].fd; + ++ rxq->port = dev->data->port_id; ++ + dev->data->rx_queues[rx_queue_id] = rxq; + return 0; + +@@ -1881,13 +1906,13 @@ static const struct eth_dev_ops ops = { + .get_monitor_addr = eth_get_monitor_addr, + }; + +-/* CNI option works in unprivileged container environment +- * and ethernet device functionality will be reduced. So +- * additional customiszed eth_dev_ops struct is needed +- * for cni. Promiscuous enable and disable functionality +- * is removed. ++/* AF_XDP Device Plugin option works in unprivileged ++ * container environments and ethernet device functionality ++ * will be reduced. So additional customised eth_dev_ops ++ * struct is needed for the Device Plugin. Promiscuous ++ * enable and disable functionality is removed. + **/ +-static const struct eth_dev_ops ops_cni = { ++static const struct eth_dev_ops ops_afxdp_dp = { + .dev_start = eth_dev_start, + .dev_stop = eth_dev_stop, + .dev_close = eth_dev_close, +@@ -2023,7 +2048,8 @@ xdp_get_channels_info(const char *if_name, int *max_queues, + static int + parse_parameters(struct rte_kvargs *kvlist, char *if_name, int *start_queue, + int *queue_cnt, int *shared_umem, char *prog_path, +- int *busy_budget, int *force_copy, int *use_cni) ++ int *busy_budget, int *force_copy, int *use_cni, ++ char *dp_path) + { + int ret; + +@@ -2069,6 +2095,11 @@ parse_parameters(struct rte_kvargs *kvlist, char *if_name, int *start_queue, + if (ret < 0) + goto free_kvlist; + ++ ret = rte_kvargs_process(kvlist, ETH_AF_XDP_DP_PATH_ARG, ++ &parse_prog_arg, dp_path); ++ if (ret < 0) ++ goto free_kvlist; ++ + free_kvlist: + rte_kvargs_free(kvlist); + return ret; +@@ -2108,7 +2139,7 @@ static struct rte_eth_dev * + init_internals(struct rte_vdev_device *dev, const char *if_name, + int start_queue_idx, int queue_cnt, int shared_umem, + const char *prog_path, int busy_budget, int force_copy, +- int use_cni) ++ int use_cni, const char *dp_path) + { + const char *name = rte_vdev_device_name(dev); + const unsigned int numa_node = dev->device.numa_node; +@@ -2138,6 +2169,7 @@ init_internals(struct rte_vdev_device *dev, const char *if_name, + internals->shared_umem = shared_umem; + internals->force_copy = force_copy; + internals->use_cni = use_cni; ++ strlcpy(internals->dp_path, dp_path, PATH_MAX); + + if (xdp_get_channels_info(if_name, &internals->max_queue_cnt, + &internals->combined_queue_cnt)) { +@@ -2199,7 +2231,7 @@ init_internals(struct rte_vdev_device *dev, const char *if_name, + if (!internals->use_cni) + eth_dev->dev_ops = &ops; + else +- eth_dev->dev_ops = &ops_cni; ++ eth_dev->dev_ops = &ops_afxdp_dp; + + eth_dev->rx_pkt_burst = eth_af_xdp_rx; + eth_dev->tx_pkt_burst = eth_af_xdp_tx; +@@ -2328,6 +2360,7 @@ rte_pmd_af_xdp_probe(struct rte_vdev_device *dev) + int busy_budget = -1, ret; + int force_copy = 0; + int use_cni = 0; ++ char dp_path[PATH_MAX] = {'\0'}; + struct rte_eth_dev *eth_dev = NULL; + const char *name = rte_vdev_device_name(dev); + +@@ -2370,7 +2403,7 @@ rte_pmd_af_xdp_probe(struct rte_vdev_device *dev) + + if (parse_parameters(kvlist, if_name, &xsk_start_queue_idx, + &xsk_queue_cnt, &shared_umem, prog_path, +- &busy_budget, &force_copy, &use_cni) < 0) { ++ &busy_budget, &force_copy, &use_cni, dp_path) < 0) { + AF_XDP_LOG(ERR, "Invalid kvargs value\n"); + return -EINVAL; + } +@@ -2384,7 +2417,19 @@ rte_pmd_af_xdp_probe(struct rte_vdev_device *dev) + if (use_cni && strnlen(prog_path, PATH_MAX)) { + AF_XDP_LOG(ERR, "When '%s' parameter is used, '%s' parameter is not valid\n", + ETH_AF_XDP_USE_CNI_ARG, ETH_AF_XDP_PROG_ARG); +- return -EINVAL; ++ return -EINVAL; ++ } ++ ++ if (use_cni && !strnlen(dp_path, PATH_MAX)) { ++ snprintf(dp_path, sizeof(dp_path), "%s/%s/%s", DP_BASE_PATH, if_name, DP_UDS_SOCK); ++ AF_XDP_LOG(INFO, "'%s' parameter not provided, setting value to '%s'\n", ++ ETH_AF_XDP_DP_PATH_ARG, dp_path); ++ } ++ ++ if (!use_cni && strnlen(dp_path, PATH_MAX)) { ++ AF_XDP_LOG(ERR, "'%s' parameter is set, but '%s' was not enabled\n", ++ ETH_AF_XDP_DP_PATH_ARG, ETH_AF_XDP_USE_CNI_ARG); ++ return -EINVAL; + } + + if (strlen(if_name) == 0) { +@@ -2410,7 +2455,7 @@ rte_pmd_af_xdp_probe(struct rte_vdev_device *dev) + + eth_dev = init_internals(dev, if_name, xsk_start_queue_idx, + xsk_queue_cnt, shared_umem, prog_path, +- busy_budget, force_copy, use_cni); ++ busy_budget, force_copy, use_cni, dp_path); + if (eth_dev == NULL) { + AF_XDP_LOG(ERR, "Failed to init internals\n"); + return -1; +@@ -2471,4 +2516,5 @@ RTE_PMD_REGISTER_PARAM_STRING(net_af_xdp, + "xdp_prog= " + "busy_budget= " + "force_copy= " +- "use_cni= "); ++ "use_cni= " ++ "dp_path= "); +diff --git a/dpdk/drivers/net/ark/ark_ethdev_tx.c b/dpdk/drivers/net/ark/ark_ethdev_tx.c +index 4792754f19..8f1f90b1a4 100644 +--- a/dpdk/drivers/net/ark/ark_ethdev_tx.c ++++ b/dpdk/drivers/net/ark/ark_ethdev_tx.c +@@ -39,8 +39,8 @@ struct ark_tx_queue { + uint32_t queue_mask; + + /* 3 indexes to the paired data rings. */ +- int32_t prod_index; /* where to put the next one */ +- int32_t free_index; /* mbuf has been freed */ ++ uint32_t prod_index; /* where to put the next one */ ++ uint32_t free_index; /* mbuf has been freed */ + + /* The queue Id is used to identify the HW Q */ + uint16_t phys_qid; +@@ -49,7 +49,7 @@ struct ark_tx_queue { + + /* next cache line - fields written by device */ + RTE_MARKER cacheline1 __rte_cache_min_aligned; +- volatile int32_t cons_index; /* hw is done, can be freed */ ++ volatile uint32_t cons_index; /* hw is done, can be freed */ + } __rte_cache_aligned; + + /* Forward declarations */ +@@ -108,7 +108,7 @@ eth_ark_xmit_pkts(void *vtxq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) + uint32_t user_meta[5]; + + int stat; +- int32_t prod_index_limit; ++ uint32_t prod_index_limit; + uint16_t nb; + uint8_t user_len = 0; + const uint32_t min_pkt_len = ARK_MIN_TX_PKTLEN; +@@ -123,8 +123,13 @@ eth_ark_xmit_pkts(void *vtxq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) + /* leave 4 elements mpu data */ + prod_index_limit = queue->queue_size + queue->free_index - 4; + ++ /* Populate the buffer bringing prod_index up to or slightly beyond ++ * prod_index_limit. Prod_index will increment by 2 or more each ++ * iteration. Note: indexes are uint32_t, cast to (signed) int32_t ++ * to catch the slight overage case; e.g. (200 - 201) ++ */ + for (nb = 0; +- (nb < nb_pkts) && (prod_index_limit - queue->prod_index) > 0; ++ (nb < nb_pkts) && (int32_t)(prod_index_limit - queue->prod_index) > 0; + ++nb) { + mbuf = tx_pkts[nb]; + +@@ -194,13 +199,13 @@ eth_ark_tx_jumbo(struct ark_tx_queue *queue, struct rte_mbuf *mbuf, + uint32_t *user_meta, uint8_t meta_cnt) + { + struct rte_mbuf *next; +- int32_t free_queue_space; ++ uint32_t free_queue_space; + uint8_t flags = ARK_DDM_SOP; + + free_queue_space = queue->queue_mask - + (queue->prod_index - queue->free_index); + /* We need up to 4 mbufs for first header and 2 for subsequent ones */ +- if (unlikely(free_queue_space < (2 + (2 * mbuf->nb_segs)))) ++ if (unlikely(free_queue_space < (2U + (2U * mbuf->nb_segs)))) + return -1; + + while (mbuf != NULL) { +@@ -392,10 +397,11 @@ free_completed_tx(struct ark_tx_queue *queue) + { + struct rte_mbuf *mbuf; + union ark_tx_meta *meta; +- int32_t top_index; ++ uint32_t top_index; + + top_index = queue->cons_index; /* read once */ +- while ((top_index - queue->free_index) > 0) { ++ ++ while ((int32_t)(top_index - queue->free_index) > 0) { + meta = &queue->meta_q[queue->free_index & queue->queue_mask]; + if (likely((meta->flags & ARK_DDM_SOP) != 0)) { + mbuf = queue->bufs[queue->free_index & +diff --git a/dpdk/drivers/net/axgbe/axgbe_common.h b/dpdk/drivers/net/axgbe/axgbe_common.h +index a5d11c5832..51532fb34a 100644 +--- a/dpdk/drivers/net/axgbe/axgbe_common.h ++++ b/dpdk/drivers/net/axgbe/axgbe_common.h +@@ -407,8 +407,6 @@ + #define MAC_MDIOSCAR_PA_WIDTH 5 + #define MAC_MDIOSCAR_RA_INDEX 0 + #define MAC_MDIOSCAR_RA_WIDTH 16 +-#define MAC_MDIOSCAR_REG_INDEX 0 +-#define MAC_MDIOSCAR_REG_WIDTH 21 + #define MAC_MDIOSCCDR_BUSY_INDEX 22 + #define MAC_MDIOSCCDR_BUSY_WIDTH 1 + #define MAC_MDIOSCCDR_CMD_INDEX 16 +diff --git a/dpdk/drivers/net/axgbe/axgbe_dev.c b/dpdk/drivers/net/axgbe/axgbe_dev.c +index 6a7fddffca..5233633a53 100644 +--- a/dpdk/drivers/net/axgbe/axgbe_dev.c ++++ b/dpdk/drivers/net/axgbe/axgbe_dev.c +@@ -63,15 +63,27 @@ static int mdio_complete(struct axgbe_port *pdata) + return 0; + } + ++static unsigned int axgbe_create_mdio_sca(int port, int reg) ++{ ++ unsigned int mdio_sca, da; ++ ++ da = (reg & MII_ADDR_C45) ? reg >> 16 : 0; ++ ++ mdio_sca = 0; ++ AXGMAC_SET_BITS(mdio_sca, MAC_MDIOSCAR, RA, reg); ++ AXGMAC_SET_BITS(mdio_sca, MAC_MDIOSCAR, PA, port); ++ AXGMAC_SET_BITS(mdio_sca, MAC_MDIOSCAR, DA, da); ++ ++ return mdio_sca; ++} ++ + static int axgbe_write_ext_mii_regs(struct axgbe_port *pdata, int addr, + int reg, u16 val) + { + unsigned int mdio_sca, mdio_sccd; + uint64_t timeout; + +- mdio_sca = 0; +- AXGMAC_SET_BITS(mdio_sca, MAC_MDIOSCAR, REG, reg); +- AXGMAC_SET_BITS(mdio_sca, MAC_MDIOSCAR, DA, addr); ++ mdio_sca = axgbe_create_mdio_sca(addr, reg); + AXGMAC_IOWRITE(pdata, MAC_MDIOSCAR, mdio_sca); + + mdio_sccd = 0; +@@ -97,9 +109,7 @@ static int axgbe_read_ext_mii_regs(struct axgbe_port *pdata, int addr, + unsigned int mdio_sca, mdio_sccd; + uint64_t timeout; + +- mdio_sca = 0; +- AXGMAC_SET_BITS(mdio_sca, MAC_MDIOSCAR, REG, reg); +- AXGMAC_SET_BITS(mdio_sca, MAC_MDIOSCAR, DA, addr); ++ mdio_sca = axgbe_create_mdio_sca(addr, reg); + AXGMAC_IOWRITE(pdata, MAC_MDIOSCAR, mdio_sca); + + mdio_sccd = 0; +@@ -259,20 +269,28 @@ static int axgbe_set_speed(struct axgbe_port *pdata, int speed) + return 0; + } + ++static unsigned int axgbe_get_fc_queue_count(struct axgbe_port *pdata) ++{ ++ unsigned int max_q_count = AXGMAC_MAX_FLOW_CONTROL_QUEUES; ++ ++ /* From MAC ver 30H the TFCR is per priority, instead of per queue */ ++ if (AXGMAC_GET_BITS(pdata->hw_feat.version, MAC_VR, SNPSVER) >= 0x30) ++ return max_q_count; ++ else ++ return (RTE_MIN(pdata->tx_q_count, max_q_count)); ++} ++ + static int axgbe_disable_tx_flow_control(struct axgbe_port *pdata) + { +- unsigned int max_q_count, q_count; + unsigned int reg, reg_val; +- unsigned int i; ++ unsigned int i, q_count; + + /* Clear MTL flow control */ + for (i = 0; i < pdata->rx_q_count; i++) + AXGMAC_MTL_IOWRITE_BITS(pdata, i, MTL_Q_RQOMR, EHFC, 0); + + /* Clear MAC flow control */ +- max_q_count = AXGMAC_MAX_FLOW_CONTROL_QUEUES; +- q_count = RTE_MIN(pdata->tx_q_count, +- max_q_count); ++ q_count = axgbe_get_fc_queue_count(pdata); + reg = MAC_Q0TFCR; + for (i = 0; i < q_count; i++) { + reg_val = AXGMAC_IOREAD(pdata, reg); +@@ -287,9 +305,8 @@ static int axgbe_disable_tx_flow_control(struct axgbe_port *pdata) + + static int axgbe_enable_tx_flow_control(struct axgbe_port *pdata) + { +- unsigned int max_q_count, q_count; + unsigned int reg, reg_val; +- unsigned int i; ++ unsigned int i, q_count; + + /* Set MTL flow control */ + for (i = 0; i < pdata->rx_q_count; i++) { +@@ -306,9 +323,7 @@ static int axgbe_enable_tx_flow_control(struct axgbe_port *pdata) + } + + /* Set MAC flow control */ +- max_q_count = AXGMAC_MAX_FLOW_CONTROL_QUEUES; +- q_count = RTE_MIN(pdata->tx_q_count, +- max_q_count); ++ q_count = axgbe_get_fc_queue_count(pdata); + reg = MAC_Q0TFCR; + for (i = 0; i < q_count; i++) { + reg_val = AXGMAC_IOREAD(pdata, reg); +@@ -637,23 +652,21 @@ static void axgbe_config_dma_cache(struct axgbe_port *pdata) + unsigned int arcache, awcache, arwcache; + + arcache = 0; +- AXGMAC_SET_BITS(arcache, DMA_AXIARCR, DRC, 0x3); ++ AXGMAC_SET_BITS(arcache, DMA_AXIARCR, DRC, 0xf); ++ AXGMAC_SET_BITS(arcache, DMA_AXIARCR, TEC, 0xf); ++ AXGMAC_SET_BITS(arcache, DMA_AXIARCR, THC, 0xf); + AXGMAC_IOWRITE(pdata, DMA_AXIARCR, arcache); + + awcache = 0; +- AXGMAC_SET_BITS(awcache, DMA_AXIAWCR, DWC, 0x3); +- AXGMAC_SET_BITS(awcache, DMA_AXIAWCR, RPC, 0x3); +- AXGMAC_SET_BITS(awcache, DMA_AXIAWCR, RPD, 0x1); +- AXGMAC_SET_BITS(awcache, DMA_AXIAWCR, RHC, 0x3); +- AXGMAC_SET_BITS(awcache, DMA_AXIAWCR, RHD, 0x1); +- AXGMAC_SET_BITS(awcache, DMA_AXIAWCR, RDC, 0x3); +- AXGMAC_SET_BITS(awcache, DMA_AXIAWCR, RDD, 0x1); ++ AXGMAC_SET_BITS(awcache, DMA_AXIAWCR, DWC, 0xf); ++ AXGMAC_SET_BITS(awcache, DMA_AXIAWCR, RPC, 0xf); ++ AXGMAC_SET_BITS(awcache, DMA_AXIAWCR, RHC, 0xf); ++ AXGMAC_SET_BITS(awcache, DMA_AXIAWCR, RDC, 0xf); + AXGMAC_IOWRITE(pdata, DMA_AXIAWCR, awcache); + + arwcache = 0; +- AXGMAC_SET_BITS(arwcache, DMA_AXIAWRCR, TDWD, 0x1); +- AXGMAC_SET_BITS(arwcache, DMA_AXIAWRCR, TDWC, 0x3); +- AXGMAC_SET_BITS(arwcache, DMA_AXIAWRCR, RDRC, 0x3); ++ AXGMAC_SET_BITS(arwcache, DMA_AXIAWRCR, TDWC, 0xf); ++ AXGMAC_SET_BITS(arwcache, DMA_AXIAWRCR, RDRC, 0xf); + AXGMAC_IOWRITE(pdata, DMA_AXIAWRCR, arwcache); + } + +diff --git a/dpdk/drivers/net/axgbe/axgbe_ethdev.c b/dpdk/drivers/net/axgbe/axgbe_ethdev.c +index f174d46143..6ce87f83f4 100644 +--- a/dpdk/drivers/net/axgbe/axgbe_ethdev.c ++++ b/dpdk/drivers/net/axgbe/axgbe_ethdev.c +@@ -207,6 +207,7 @@ static struct axgbe_version_data axgbe_v2a = { + .ecc_support = 1, + .i2c_support = 1, + .an_cdr_workaround = 1, ++ .enable_rrc = 1, + }; + + static struct axgbe_version_data axgbe_v2b = { +@@ -219,6 +220,7 @@ static struct axgbe_version_data axgbe_v2b = { + .ecc_support = 1, + .i2c_support = 1, + .an_cdr_workaround = 1, ++ .enable_rrc = 1, + }; + + static const struct rte_eth_desc_lim rx_desc_lim = { +@@ -2267,6 +2269,9 @@ eth_axgbe_dev_init(struct rte_eth_dev *eth_dev) + + /* Yellow Carp devices do not need cdr workaround */ + pdata->vdata->an_cdr_workaround = 0; ++ ++ /* Yellow Carp devices do not need rrc */ ++ pdata->vdata->enable_rrc = 0; + } else { + unknown_cpu = 1; + } +@@ -2404,12 +2409,14 @@ static int + axgbe_dev_close(struct rte_eth_dev *eth_dev) + { + struct rte_pci_device *pci_dev; ++ struct axgbe_port *pdata; + + PMD_INIT_FUNC_TRACE(); + + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return 0; + ++ pdata = eth_dev->data->dev_private; + pci_dev = RTE_DEV_TO_PCI(eth_dev->device); + axgbe_dev_clear_queues(eth_dev); + +@@ -2419,6 +2426,9 @@ axgbe_dev_close(struct rte_eth_dev *eth_dev) + axgbe_dev_interrupt_handler, + (void *)eth_dev); + ++ /* Disable all interrupts in the hardware */ ++ XP_IOWRITE(pdata, XP_INT_EN, 0x0); ++ + return 0; + } + +diff --git a/dpdk/drivers/net/axgbe/axgbe_ethdev.h b/dpdk/drivers/net/axgbe/axgbe_ethdev.h +index 7f19321d88..b4bd56e239 100644 +--- a/dpdk/drivers/net/axgbe/axgbe_ethdev.h ++++ b/dpdk/drivers/net/axgbe/axgbe_ethdev.h +@@ -111,6 +111,7 @@ + /* Auto-negotiation */ + #define AXGBE_AN_MS_TIMEOUT 500 + #define AXGBE_LINK_TIMEOUT 5 ++#define AXGBE_KR_TRAINING_WAIT_ITER 50 + + #define AXGBE_SGMII_AN_LINK_STATUS BIT(1) + #define AXGBE_SGMII_AN_LINK_SPEED (BIT(2) | BIT(3)) +@@ -463,6 +464,7 @@ struct axgbe_version_data { + unsigned int ecc_support; + unsigned int i2c_support; + unsigned int an_cdr_workaround; ++ unsigned int enable_rrc; + }; + + struct axgbe_mmc_stats { +@@ -653,6 +655,7 @@ struct axgbe_port { + unsigned int parallel_detect; + unsigned int fec_ability; + unsigned long an_start; ++ unsigned long kr_start_time; + enum axgbe_an_mode an_mode; + + /* I2C support */ +diff --git a/dpdk/drivers/net/axgbe/axgbe_mdio.c b/dpdk/drivers/net/axgbe/axgbe_mdio.c +index 913ceada0d..d95a52659e 100644 +--- a/dpdk/drivers/net/axgbe/axgbe_mdio.c ++++ b/dpdk/drivers/net/axgbe/axgbe_mdio.c +@@ -200,13 +200,14 @@ static void axgbe_switch_mode(struct axgbe_port *pdata) + axgbe_change_mode(pdata, pdata->phy_if.phy_impl.switch_mode(pdata)); + } + +-static void axgbe_set_mode(struct axgbe_port *pdata, ++static bool axgbe_set_mode(struct axgbe_port *pdata, + enum axgbe_mode mode) + { + if (mode == axgbe_cur_mode(pdata)) +- return; ++ return false; + + axgbe_change_mode(pdata, mode); ++ return true; + } + + static bool axgbe_use_mode(struct axgbe_port *pdata, +@@ -357,6 +358,7 @@ static enum axgbe_an axgbe_an73_tx_training(struct axgbe_port *pdata, + reg |= AXGBE_KR_TRAINING_ENABLE; + reg |= AXGBE_KR_TRAINING_START; + XMDIO_WRITE(pdata, MDIO_MMD_PMAPMD, MDIO_PMA_10GBR_PMD_CTRL, reg); ++ pdata->kr_start_time = rte_get_timer_cycles(); + + PMD_DRV_LOG(DEBUG, "KR training initiated\n"); + if (pdata->phy_if.phy_impl.kr_training_post) +@@ -487,6 +489,7 @@ static enum axgbe_an axgbe_an73_incompat_link(struct axgbe_port *pdata) + + axgbe_an_disable(pdata); + axgbe_switch_mode(pdata); ++ pdata->an_result = AXGBE_AN_READY; + axgbe_an_restart(pdata); + + return AXGBE_AN_INCOMPAT_LINK; +@@ -967,11 +970,34 @@ static void axgbe_check_link_timeout(struct axgbe_port *pdata) + { + unsigned long link_timeout; + unsigned long ticks; ++ unsigned long kr_time; ++ int wait; + + link_timeout = pdata->link_check + (AXGBE_LINK_TIMEOUT * + 2 * rte_get_timer_hz()); + ticks = rte_get_timer_cycles(); + if (time_after(ticks, link_timeout)) { ++ if ((axgbe_cur_mode(pdata) == AXGBE_MODE_KR) && ++ pdata->phy.autoneg == AUTONEG_ENABLE) { ++ /* AN restart should not happen while KR training is in progress. ++ * The while loop ensures no AN restart during KR training, ++ * waits up to 500ms and AN restart is triggered only if KR ++ * training is failed. ++ */ ++ wait = AXGBE_KR_TRAINING_WAIT_ITER; ++ while (wait--) { ++ kr_time = pdata->kr_start_time + ++ msecs_to_timer_cycles(AXGBE_AN_MS_TIMEOUT); ++ ticks = rte_get_timer_cycles(); ++ if (time_after(ticks, kr_time)) ++ break; ++ /* AN restart is not required, if AN result is COMPLETE */ ++ if (pdata->an_result == AXGBE_AN_COMPLETE) ++ return; ++ rte_delay_us(10500); ++ } ++ } ++ + PMD_DRV_LOG(NOTICE, "AN link timeout\n"); + axgbe_phy_config_aneg(pdata); + } +@@ -982,7 +1008,7 @@ static enum axgbe_mode axgbe_phy_status_aneg(struct axgbe_port *pdata) + return pdata->phy_if.phy_impl.an_outcome(pdata); + } + +-static void axgbe_phy_status_result(struct axgbe_port *pdata) ++static bool axgbe_phy_status_result(struct axgbe_port *pdata) + { + enum axgbe_mode mode; + +@@ -1016,7 +1042,10 @@ static void axgbe_phy_status_result(struct axgbe_port *pdata) + + pdata->phy.duplex = DUPLEX_FULL; + +- axgbe_set_mode(pdata, mode); ++ if (axgbe_set_mode(pdata, mode)) ++ return true; ++ else ++ return false; + } + + static int autoneg_time_out(unsigned long autoneg_start_time) +@@ -1051,7 +1080,7 @@ static void axgbe_phy_status(struct axgbe_port *pdata) + &an_restart); + if (an_restart) { + axgbe_phy_config_aneg(pdata); +- return; ++ goto adjust_link; + } + + if (pdata->phy.link) { +@@ -1083,7 +1112,10 @@ static void axgbe_phy_status(struct axgbe_port *pdata) + return; + } + } +- axgbe_phy_status_result(pdata); ++ ++ if (axgbe_phy_status_result(pdata)) ++ return; ++ + if (rte_bit_relaxed_get32(AXGBE_LINK_INIT, &pdata->dev_state)) + rte_bit_relaxed_clear32(AXGBE_LINK_INIT, + &pdata->dev_state); +diff --git a/dpdk/drivers/net/axgbe/axgbe_phy_impl.c b/dpdk/drivers/net/axgbe/axgbe_phy_impl.c +index d97fbbfddd..12908d4e6f 100644 +--- a/dpdk/drivers/net/axgbe/axgbe_phy_impl.c ++++ b/dpdk/drivers/net/axgbe/axgbe_phy_impl.c +@@ -69,6 +69,7 @@ enum axgbe_sfp_cable { + AXGBE_SFP_CABLE_UNKNOWN = 0, + AXGBE_SFP_CABLE_ACTIVE, + AXGBE_SFP_CABLE_PASSIVE, ++ AXGBE_SFP_CABLE_FIBER, + }; + + enum axgbe_sfp_base { +@@ -116,9 +117,7 @@ enum axgbe_sfp_speed { + + #define AXGBE_SFP_BASE_BR 12 + #define AXGBE_SFP_BASE_BR_1GBE_MIN 0x0a +-#define AXGBE_SFP_BASE_BR_1GBE_MAX 0x0d + #define AXGBE_SFP_BASE_BR_10GBE_MIN 0x64 +-#define AXGBE_SFP_BASE_BR_10GBE_MAX 0x68 + + #define AXGBE_SFP_BASE_CU_CABLE_LEN 18 + +@@ -535,25 +534,22 @@ static void axgbe_phy_sfp_phy_settings(struct axgbe_port *pdata) + static bool axgbe_phy_sfp_bit_rate(struct axgbe_sfp_eeprom *sfp_eeprom, + enum axgbe_sfp_speed sfp_speed) + { +- u8 *sfp_base, min, max; ++ u8 *sfp_base, min; + + sfp_base = sfp_eeprom->base; + + switch (sfp_speed) { + case AXGBE_SFP_SPEED_1000: + min = AXGBE_SFP_BASE_BR_1GBE_MIN; +- max = AXGBE_SFP_BASE_BR_1GBE_MAX; + break; + case AXGBE_SFP_SPEED_10000: + min = AXGBE_SFP_BASE_BR_10GBE_MIN; +- max = AXGBE_SFP_BASE_BR_10GBE_MAX; + break; + default: + return false; + } + +- return ((sfp_base[AXGBE_SFP_BASE_BR] >= min) && +- (sfp_base[AXGBE_SFP_BASE_BR] <= max)); ++ return sfp_base[AXGBE_SFP_BASE_BR] >= min; + } + + static void axgbe_phy_sfp_external_phy(struct axgbe_port *pdata) +@@ -578,6 +574,9 @@ static bool axgbe_phy_belfuse_parse_quirks(struct axgbe_port *pdata) + AXGBE_BEL_FUSE_VENDOR, strlen(AXGBE_BEL_FUSE_VENDOR))) + return false; + ++ /* Reset PHY - wait for self-clearing reset bit to clear */ ++ pdata->phy_if.phy_impl.reset(pdata); ++ + if (!memcmp(&sfp_eeprom->base[AXGBE_SFP_BASE_VENDOR_PN], + AXGBE_BEL_FUSE_PARTNO, strlen(AXGBE_BEL_FUSE_PARTNO))) { + phy_data->sfp_base = AXGBE_SFP_BASE_1000_SX; +@@ -613,16 +612,21 @@ static void axgbe_phy_sfp_parse_eeprom(struct axgbe_port *pdata) + + axgbe_phy_sfp_parse_quirks(pdata); + +- /* Assume ACTIVE cable unless told it is PASSIVE */ ++ /* Assume FIBER cable unless told otherwise */ + if (sfp_base[AXGBE_SFP_BASE_CABLE] & AXGBE_SFP_BASE_CABLE_PASSIVE) { + phy_data->sfp_cable = AXGBE_SFP_CABLE_PASSIVE; + phy_data->sfp_cable_len = sfp_base[AXGBE_SFP_BASE_CU_CABLE_LEN]; +- } else { ++ } else if (sfp_base[AXGBE_SFP_BASE_CABLE] & AXGBE_SFP_BASE_CABLE_ACTIVE) { + phy_data->sfp_cable = AXGBE_SFP_CABLE_ACTIVE; ++ } else { ++ phy_data->sfp_cable = AXGBE_SFP_CABLE_FIBER; + } + + /* Determine the type of SFP */ +- if (sfp_base[AXGBE_SFP_BASE_10GBE_CC] & AXGBE_SFP_BASE_10GBE_CC_SR) ++ if (phy_data->sfp_cable != AXGBE_SFP_CABLE_FIBER && ++ axgbe_phy_sfp_bit_rate(sfp_eeprom, AXGBE_SFP_SPEED_10000)) ++ phy_data->sfp_base = AXGBE_SFP_BASE_10000_CR; ++ else if (sfp_base[AXGBE_SFP_BASE_10GBE_CC] & AXGBE_SFP_BASE_10GBE_CC_SR) + phy_data->sfp_base = AXGBE_SFP_BASE_10000_SR; + else if (sfp_base[AXGBE_SFP_BASE_10GBE_CC] & AXGBE_SFP_BASE_10GBE_CC_LR) + phy_data->sfp_base = AXGBE_SFP_BASE_10000_LR; +@@ -639,9 +643,6 @@ static void axgbe_phy_sfp_parse_eeprom(struct axgbe_port *pdata) + phy_data->sfp_base = AXGBE_SFP_BASE_1000_CX; + else if (sfp_base[AXGBE_SFP_BASE_1GBE_CC] & AXGBE_SFP_BASE_1GBE_CC_T) + phy_data->sfp_base = AXGBE_SFP_BASE_1000_T; +- else if ((phy_data->sfp_cable == AXGBE_SFP_CABLE_PASSIVE) && +- axgbe_phy_sfp_bit_rate(sfp_eeprom, AXGBE_SFP_SPEED_10000)) +- phy_data->sfp_base = AXGBE_SFP_BASE_10000_CR; + + switch (phy_data->sfp_base) { + case AXGBE_SFP_BASE_1000_T: +@@ -1225,6 +1226,10 @@ static void axgbe_phy_rx_reset(struct axgbe_port *pdata) + + static void axgbe_phy_pll_ctrl(struct axgbe_port *pdata, bool enable) + { ++ /* PLL_CTRL feature needs to be enabled for fixed PHY modes (Non-Autoneg) only */ ++ if (pdata->phy.autoneg != AUTONEG_DISABLE) ++ return; ++ + XMDIO_WRITE_BITS(pdata, MDIO_MMD_PMAPMD, MDIO_VEND2_PMA_MISC_CTRL0, + XGBE_PMA_PLL_CTRL_MASK, + enable ? XGBE_PMA_PLL_CTRL_SET +@@ -1269,8 +1274,9 @@ static void axgbe_phy_perform_ratechange(struct axgbe_port *pdata, + axgbe_phy_rx_reset(pdata); + + reenable_pll: +- /* Re-enable the PLL control */ +- axgbe_phy_pll_ctrl(pdata, true); ++ /* Enable PLL re-initialization, not needed for PHY Power Off and RRC cmds */ ++ if (cmd != 0 && cmd != 5) ++ axgbe_phy_pll_ctrl(pdata, true); + + PMD_DRV_LOG(NOTICE, "firmware mailbox command did not complete\n"); + } +@@ -1697,8 +1703,15 @@ static int axgbe_phy_link_status(struct axgbe_port *pdata, int *an_restart) + if (reg & MDIO_STAT1_LSTATUS) + return 1; + ++ if (pdata->phy.autoneg == AUTONEG_ENABLE && ++ phy_data->port_mode == AXGBE_PORT_MODE_BACKPLANE) { ++ if (rte_bit_relaxed_get32(AXGBE_LINK_INIT, &pdata->dev_state)) { ++ *an_restart = 1; ++ } ++ } ++ + /* No link, attempt a receiver reset cycle */ +- if (phy_data->rrc_count++) { ++ if (pdata->vdata->enable_rrc && phy_data->rrc_count++) { + phy_data->rrc_count = 0; + axgbe_phy_rrc(pdata); + } +diff --git a/dpdk/drivers/net/bnx2x/bnx2x.c b/dpdk/drivers/net/bnx2x/bnx2x.c +index c3283c94f3..597ee43359 100644 +--- a/dpdk/drivers/net/bnx2x/bnx2x.c ++++ b/dpdk/drivers/net/bnx2x/bnx2x.c +@@ -2389,7 +2389,7 @@ int bnx2x_alloc_ilt_mem(struct bnx2x_softc *sc) + static int bnx2x_alloc_ilt_lines_mem(struct bnx2x_softc *sc) + { + sc->ilt->lines = rte_calloc("", +- sizeof(struct ilt_line), ILT_MAX_LINES, ++ ILT_MAX_LINES, sizeof(struct ilt_line), + RTE_CACHE_LINE_SIZE); + return sc->ilt->lines == NULL; + } +diff --git a/dpdk/drivers/net/bnx2x/bnx2x_stats.c b/dpdk/drivers/net/bnx2x/bnx2x_stats.c +index c07b01510a..69132c7c80 100644 +--- a/dpdk/drivers/net/bnx2x/bnx2x_stats.c ++++ b/dpdk/drivers/net/bnx2x/bnx2x_stats.c +@@ -114,7 +114,7 @@ bnx2x_hw_stats_post(struct bnx2x_softc *sc) + + /* Update MCP's statistics if possible */ + if (sc->func_stx) { +- rte_memcpy(BNX2X_SP(sc, func_stats), &sc->func_stats, ++ memcpy(BNX2X_SP(sc, func_stats), &sc->func_stats, + sizeof(sc->func_stats)); + } + +@@ -817,10 +817,10 @@ bnx2x_hw_stats_update(struct bnx2x_softc *sc) + etherstatspktsover1522octets); + } + +- rte_memcpy(old, new, sizeof(struct nig_stats)); ++ memcpy(old, new, sizeof(struct nig_stats)); + +- rte_memcpy(&(estats->rx_stat_ifhcinbadoctets_hi), &(pstats->mac_stx[1]), +- sizeof(struct mac_stx)); ++ memcpy(RTE_PTR_ADD(estats, offsetof(struct bnx2x_eth_stats, rx_stat_ifhcinbadoctets_hi)), ++ &pstats->mac_stx[1], sizeof(struct mac_stx)); + estats->brb_drop_hi = pstats->brb_drop_hi; + estats->brb_drop_lo = pstats->brb_drop_lo; + +@@ -1492,9 +1492,11 @@ bnx2x_stats_init(struct bnx2x_softc *sc) + REG_RD(sc, NIG_REG_STAT0_BRB_TRUNCATE + port*0x38); + if (!CHIP_IS_E3(sc)) { + REG_RD_DMAE(sc, NIG_REG_STAT0_EGRESS_MAC_PKT0 + port*0x50, +- &(sc->port.old_nig_stats.egress_mac_pkt0_lo), 2); ++ RTE_PTR_ADD(&sc->port.old_nig_stats, ++ offsetof(struct nig_stats, egress_mac_pkt0_lo)), 2); + REG_RD_DMAE(sc, NIG_REG_STAT0_EGRESS_MAC_PKT1 + port*0x50, +- &(sc->port.old_nig_stats.egress_mac_pkt1_lo), 2); ++ RTE_PTR_ADD(&sc->port.old_nig_stats, ++ offsetof(struct nig_stats, egress_mac_pkt1_lo)), 2); + } + + /* function stats */ +diff --git a/dpdk/drivers/net/bnx2x/bnx2x_vfpf.c b/dpdk/drivers/net/bnx2x/bnx2x_vfpf.c +index 63953c2979..5411df3a38 100644 +--- a/dpdk/drivers/net/bnx2x/bnx2x_vfpf.c ++++ b/dpdk/drivers/net/bnx2x/bnx2x_vfpf.c +@@ -52,9 +52,9 @@ bnx2x_check_bull(struct bnx2x_softc *sc) + + /* check the mac address and VLAN and allocate memory if valid */ + if (valid_bitmap & (1 << MAC_ADDR_VALID) && memcmp(bull->mac, sc->old_bulletin.mac, ETH_ALEN)) +- rte_memcpy(&sc->link_params.mac_addr, bull->mac, ETH_ALEN); ++ memcpy(&sc->link_params.mac_addr, bull->mac, ETH_ALEN); + if (valid_bitmap & (1 << VLAN_VALID)) +- rte_memcpy(&bull->vlan, &sc->old_bulletin.vlan, RTE_VLAN_HLEN); ++ memcpy(&bull->vlan, &sc->old_bulletin.vlan, sizeof(bull->vlan)); + + sc->old_bulletin = *bull; + +@@ -569,7 +569,7 @@ bnx2x_vf_set_mac(struct bnx2x_softc *sc, int set) + + bnx2x_check_bull(sc); + +- rte_memcpy(query->filters[0].mac, sc->link_params.mac_addr, ETH_ALEN); ++ memcpy(query->filters[0].mac, sc->link_params.mac_addr, ETH_ALEN); + + bnx2x_add_tlv(sc, query, query->first_tlv.tl.length, + BNX2X_VF_TLV_LIST_END, +@@ -583,9 +583,9 @@ bnx2x_vf_set_mac(struct bnx2x_softc *sc, int set) + while (BNX2X_VF_STATUS_FAILURE == reply->status && + bnx2x_check_bull(sc)) { + /* A new mac was configured by PF for us */ +- rte_memcpy(sc->link_params.mac_addr, sc->pf2vf_bulletin->mac, ++ memcpy(sc->link_params.mac_addr, sc->pf2vf_bulletin->mac, + ETH_ALEN); +- rte_memcpy(query->filters[0].mac, sc->pf2vf_bulletin->mac, ++ memcpy(query->filters[0].mac, sc->pf2vf_bulletin->mac, + ETH_ALEN); + + rc = bnx2x_do_req4pf(sc, sc->vf2pf_mbox_mapping.paddr); +@@ -622,10 +622,10 @@ bnx2x_vf_config_rss(struct bnx2x_softc *sc, + BNX2X_VF_TLV_LIST_END, + sizeof(struct channel_list_end_tlv)); + +- rte_memcpy(query->rss_key, params->rss_key, sizeof(params->rss_key)); ++ memcpy(query->rss_key, params->rss_key, sizeof(params->rss_key)); + query->rss_key_size = T_ETH_RSS_KEY; + +- rte_memcpy(query->ind_table, params->ind_table, T_ETH_INDIRECTION_TABLE_SIZE); ++ memcpy(query->ind_table, params->ind_table, T_ETH_INDIRECTION_TABLE_SIZE); + query->ind_table_size = T_ETH_INDIRECTION_TABLE_SIZE; + + query->rss_result_mask = params->rss_result_mask; +diff --git a/dpdk/drivers/net/bnxt/bnxt.h b/dpdk/drivers/net/bnxt/bnxt.h +index 0e01b1d4ba..be2fd689bb 100644 +--- a/dpdk/drivers/net/bnxt/bnxt.h ++++ b/dpdk/drivers/net/bnxt/bnxt.h +@@ -449,8 +449,8 @@ struct bnxt_ring_mem_info { + + struct bnxt_ctx_pg_info { + uint32_t entries; +- void *ctx_pg_arr[MAX_CTX_PAGES]; +- rte_iova_t ctx_dma_arr[MAX_CTX_PAGES]; ++ void **ctx_pg_arr; ++ rte_iova_t *ctx_dma_arr; + struct bnxt_ring_mem_info ring_mem; + }; + +@@ -550,7 +550,6 @@ struct bnxt_mark_info { + + struct bnxt_rep_info { + struct rte_eth_dev *vfr_eth_dev; +- pthread_mutex_t vfr_lock; + pthread_mutex_t vfr_start_lock; + bool conduit_valid; + }; +@@ -896,6 +895,7 @@ struct bnxt { + struct rte_ether_addr *mcast_addr_list; + rte_iova_t mc_list_dma_addr; + uint32_t nb_mc_addr; ++#define BNXT_DFLT_MAX_MC_ADDR 16 /* for compatibility with older firmware */ + uint32_t max_mcast_addr; /* maximum number of mcast filters supported */ + + struct rte_eth_rss_conf rss_conf; /* RSS configuration. */ +diff --git a/dpdk/drivers/net/bnxt/bnxt_ethdev.c b/dpdk/drivers/net/bnxt/bnxt_ethdev.c +index acf7e6e46e..0fc561d258 100644 +--- a/dpdk/drivers/net/bnxt/bnxt_ethdev.c ++++ b/dpdk/drivers/net/bnxt/bnxt_ethdev.c +@@ -1673,10 +1673,8 @@ bnxt_uninit_locks(struct bnxt *bp) + pthread_mutex_destroy(&bp->def_cp_lock); + pthread_mutex_destroy(&bp->health_check_lock); + pthread_mutex_destroy(&bp->err_recovery_lock); +- if (bp->rep_info) { +- pthread_mutex_destroy(&bp->rep_info->vfr_lock); ++ if (bp->rep_info) + pthread_mutex_destroy(&bp->rep_info->vfr_start_lock); +- } + } + + static void bnxt_drv_uninit(struct bnxt *bp) +@@ -4750,7 +4748,7 @@ static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp, + { + struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem; + const struct rte_memzone *mz = NULL; +- char mz_name[RTE_MEMZONE_NAMESIZE]; ++ char name[RTE_MEMZONE_NAMESIZE]; + rte_iova_t mz_phys_addr; + uint64_t valid_bits = 0; + uint32_t sz; +@@ -4762,6 +4760,19 @@ static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp, + rmem->nr_pages = RTE_ALIGN_MUL_CEIL(mem_size, BNXT_PAGE_SIZE) / + BNXT_PAGE_SIZE; + rmem->page_size = BNXT_PAGE_SIZE; ++ ++ snprintf(name, RTE_MEMZONE_NAMESIZE, "bnxt_ctx_pg_arr%s_%x_%d", ++ suffix, idx, bp->eth_dev->data->port_id); ++ ctx_pg->ctx_pg_arr = rte_zmalloc(name, sizeof(void *) * rmem->nr_pages, 0); ++ if (ctx_pg->ctx_pg_arr == NULL) ++ return -ENOMEM; ++ ++ snprintf(name, RTE_MEMZONE_NAMESIZE, "bnxt_ctx_dma_arr%s_%x_%d", ++ suffix, idx, bp->eth_dev->data->port_id); ++ ctx_pg->ctx_dma_arr = rte_zmalloc(name, sizeof(rte_iova_t *) * rmem->nr_pages, 0); ++ if (ctx_pg->ctx_dma_arr == NULL) ++ return -ENOMEM; ++ + rmem->pg_arr = ctx_pg->ctx_pg_arr; + rmem->dma_arr = ctx_pg->ctx_dma_arr; + rmem->flags = BNXT_RMEM_VALID_PTE_FLAG; +@@ -4769,13 +4780,13 @@ static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp, + valid_bits = PTU_PTE_VALID; + + if (rmem->nr_pages > 1) { +- snprintf(mz_name, RTE_MEMZONE_NAMESIZE, ++ snprintf(name, RTE_MEMZONE_NAMESIZE, + "bnxt_ctx_pg_tbl%s_%x_%d", + suffix, idx, bp->eth_dev->data->port_id); +- mz_name[RTE_MEMZONE_NAMESIZE - 1] = 0; +- mz = rte_memzone_lookup(mz_name); ++ name[RTE_MEMZONE_NAMESIZE - 1] = 0; ++ mz = rte_memzone_lookup(name); + if (!mz) { +- mz = rte_memzone_reserve_aligned(mz_name, ++ mz = rte_memzone_reserve_aligned(name, + rmem->nr_pages * 8, + bp->eth_dev->device->numa_node, + RTE_MEMZONE_2MB | +@@ -4794,11 +4805,11 @@ static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp, + rmem->pg_tbl_mz = mz; + } + +- snprintf(mz_name, RTE_MEMZONE_NAMESIZE, "bnxt_ctx_%s_%x_%d", ++ snprintf(name, RTE_MEMZONE_NAMESIZE, "bnxt_ctx_%s_%x_%d", + suffix, idx, bp->eth_dev->data->port_id); +- mz = rte_memzone_lookup(mz_name); ++ mz = rte_memzone_lookup(name); + if (!mz) { +- mz = rte_memzone_reserve_aligned(mz_name, ++ mz = rte_memzone_reserve_aligned(name, + mem_size, + bp->eth_dev->device->numa_node, + RTE_MEMZONE_1GB | +@@ -4844,6 +4855,17 @@ static void bnxt_free_ctx_mem(struct bnxt *bp) + return; + + bp->ctx->flags &= ~BNXT_CTX_FLAG_INITED; ++ rte_free(bp->ctx->qp_mem.ctx_pg_arr); ++ rte_free(bp->ctx->srq_mem.ctx_pg_arr); ++ rte_free(bp->ctx->cq_mem.ctx_pg_arr); ++ rte_free(bp->ctx->vnic_mem.ctx_pg_arr); ++ rte_free(bp->ctx->stat_mem.ctx_pg_arr); ++ rte_free(bp->ctx->qp_mem.ctx_dma_arr); ++ rte_free(bp->ctx->srq_mem.ctx_dma_arr); ++ rte_free(bp->ctx->cq_mem.ctx_dma_arr); ++ rte_free(bp->ctx->vnic_mem.ctx_dma_arr); ++ rte_free(bp->ctx->stat_mem.ctx_dma_arr); ++ + rte_memzone_free(bp->ctx->qp_mem.ring_mem.mz); + rte_memzone_free(bp->ctx->srq_mem.ring_mem.mz); + rte_memzone_free(bp->ctx->cq_mem.ring_mem.mz); +@@ -4856,6 +4878,8 @@ static void bnxt_free_ctx_mem(struct bnxt *bp) + rte_memzone_free(bp->ctx->stat_mem.ring_mem.pg_tbl_mz); + + for (i = 0; i < bp->ctx->tqm_fp_rings_count + 1; i++) { ++ rte_free(bp->ctx->tqm_mem[i]->ctx_pg_arr); ++ rte_free(bp->ctx->tqm_mem[i]->ctx_dma_arr); + if (bp->ctx->tqm_mem[i]) + rte_memzone_free(bp->ctx->tqm_mem[i]->ring_mem.mz); + } +@@ -6173,13 +6197,6 @@ static int bnxt_init_rep_info(struct bnxt *bp) + for (i = 0; i < BNXT_MAX_CFA_CODE; i++) + bp->cfa_code_map[i] = BNXT_VF_IDX_INVALID; + +- rc = pthread_mutex_init(&bp->rep_info->vfr_lock, NULL); +- if (rc) { +- PMD_DRV_LOG(ERR, "Unable to initialize vfr_lock\n"); +- bnxt_free_rep_info(bp); +- return rc; +- } +- + rc = pthread_mutex_init(&bp->rep_info->vfr_start_lock, NULL); + if (rc) { + PMD_DRV_LOG(ERR, "Unable to initialize vfr_start_lock\n"); +diff --git a/dpdk/drivers/net/bnxt/bnxt_hwrm.c b/dpdk/drivers/net/bnxt/bnxt_hwrm.c +index 06f196760f..94c3249ae4 100644 +--- a/dpdk/drivers/net/bnxt/bnxt_hwrm.c ++++ b/dpdk/drivers/net/bnxt/bnxt_hwrm.c +@@ -863,6 +863,8 @@ static int __bnxt_hwrm_func_qcaps(struct bnxt *bp) + bp->max_l2_ctx, bp->max_vnics); + bp->max_stat_ctx = rte_le_to_cpu_16(resp->max_stat_ctx); + bp->max_mcast_addr = rte_le_to_cpu_32(resp->max_mcast_filters); ++ if (!bp->max_mcast_addr) ++ bp->max_mcast_addr = BNXT_DFLT_MAX_MC_ADDR; + memcpy(bp->dsn, resp->device_serial_number, sizeof(bp->dsn)); + + if (BNXT_PF(bp)) +@@ -3039,6 +3041,8 @@ static uint16_t bnxt_check_eth_link_autoneg(uint32_t conf_link) + static uint16_t bnxt_parse_eth_link_speed(uint32_t conf_link_speed, + struct bnxt_link_info *link_info) + { ++ uint16_t support_pam4_speeds = link_info->support_pam4_speeds; ++ uint16_t support_speeds = link_info->support_speeds; + uint16_t eth_link_speed = 0; + + if (conf_link_speed == RTE_ETH_LINK_SPEED_AUTONEG) +@@ -3070,29 +3074,30 @@ static uint16_t bnxt_parse_eth_link_speed(uint32_t conf_link_speed, + case RTE_ETH_LINK_SPEED_25G: + eth_link_speed = + HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_25GB; ++ link_info->link_signal_mode = BNXT_SIG_MODE_NRZ; + break; + case RTE_ETH_LINK_SPEED_40G: + eth_link_speed = + HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_40GB; + break; + case RTE_ETH_LINK_SPEED_50G: +- if (link_info->support_pam4_speeds & +- HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_PAM4_SPEEDS_50G) { +- eth_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_PAM4_LINK_SPEED_50GB; +- link_info->link_signal_mode = BNXT_SIG_MODE_PAM4; +- } else { ++ if (support_speeds & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_50GB) { + eth_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_50GB; + link_info->link_signal_mode = BNXT_SIG_MODE_NRZ; ++ } else if (support_pam4_speeds & ++ HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_PAM4_SPEEDS_50G) { ++ eth_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_PAM4_LINK_SPEED_50GB; ++ link_info->link_signal_mode = BNXT_SIG_MODE_PAM4; + } + break; + case RTE_ETH_LINK_SPEED_100G: +- if (link_info->support_pam4_speeds & +- HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_PAM4_SPEEDS_100G) { +- eth_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_PAM4_LINK_SPEED_100GB; +- link_info->link_signal_mode = BNXT_SIG_MODE_PAM4; +- } else { ++ if (support_speeds & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_100GB) { + eth_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_100GB; + link_info->link_signal_mode = BNXT_SIG_MODE_NRZ; ++ } else if (support_pam4_speeds & ++ HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_PAM4_SPEEDS_100G) { ++ eth_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_PAM4_LINK_SPEED_100GB; ++ link_info->link_signal_mode = BNXT_SIG_MODE_PAM4; + } + break; + case RTE_ETH_LINK_SPEED_200G: +diff --git a/dpdk/drivers/net/bnxt/bnxt_reps.c b/dpdk/drivers/net/bnxt/bnxt_reps.c +index 78337431af..6d6b8252e2 100644 +--- a/dpdk/drivers/net/bnxt/bnxt_reps.c ++++ b/dpdk/drivers/net/bnxt/bnxt_reps.c +@@ -32,6 +32,14 @@ static const struct eth_dev_ops bnxt_rep_dev_ops = { + .flow_ops_get = bnxt_flow_ops_get_op + }; + ++static bool bnxt_rep_check_parent(struct bnxt_representor *rep) ++{ ++ if (!rep->parent_dev->data->dev_private) ++ return false; ++ ++ return true; ++} ++ + uint16_t + bnxt_vfr_recv(uint16_t port_id, uint16_t queue_id, struct rte_mbuf *mbuf) + { +@@ -124,8 +132,8 @@ bnxt_rep_tx_burst(void *tx_queue, + qid = vfr_txq->txq->queue_id; + vf_rep_bp = vfr_txq->bp; + parent = vf_rep_bp->parent_dev->data->dev_private; +- pthread_mutex_lock(&parent->rep_info->vfr_lock); + ptxq = parent->tx_queues[qid]; ++ pthread_mutex_lock(&ptxq->txq_lock); + + ptxq->vfr_tx_cfa_action = vf_rep_bp->vfr_tx_cfa_action; + +@@ -134,9 +142,9 @@ bnxt_rep_tx_burst(void *tx_queue, + vf_rep_bp->tx_pkts[qid]++; + } + +- rc = bnxt_xmit_pkts(ptxq, tx_pkts, nb_pkts); ++ rc = _bnxt_xmit_pkts(ptxq, tx_pkts, nb_pkts); + ptxq->vfr_tx_cfa_action = 0; +- pthread_mutex_unlock(&parent->rep_info->vfr_lock); ++ pthread_mutex_unlock(&ptxq->txq_lock); + + return rc; + } +@@ -266,12 +274,12 @@ int bnxt_representor_uninit(struct rte_eth_dev *eth_dev) + PMD_DRV_LOG(DEBUG, "BNXT Port:%d VFR uninit\n", eth_dev->data->port_id); + eth_dev->data->mac_addrs = NULL; + +- parent_bp = rep->parent_dev->data->dev_private; +- if (!parent_bp) { ++ if (!bnxt_rep_check_parent(rep)) { + PMD_DRV_LOG(DEBUG, "BNXT Port:%d already freed\n", + eth_dev->data->port_id); + return 0; + } ++ parent_bp = rep->parent_dev->data->dev_private; + + parent_bp->num_reps--; + vf_id = rep->vf_id; +@@ -539,11 +547,12 @@ int bnxt_rep_dev_info_get_op(struct rte_eth_dev *eth_dev, + int rc = 0; + + /* MAC Specifics */ +- parent_bp = rep_bp->parent_dev->data->dev_private; +- if (!parent_bp) { +- PMD_DRV_LOG(ERR, "Rep parent NULL!\n"); ++ if (!bnxt_rep_check_parent(rep_bp)) { ++ /* Need not be an error scenario, if parent is closed first */ ++ PMD_DRV_LOG(INFO, "Rep parent port does not exist.\n"); + return rc; + } ++ parent_bp = rep_bp->parent_dev->data->dev_private; + PMD_DRV_LOG(DEBUG, "Representor dev_info_get_op\n"); + dev_info->max_mac_addrs = parent_bp->max_l2_ctx; + dev_info->max_hash_mac_addrs = 0; +@@ -730,10 +739,10 @@ int bnxt_rep_tx_queue_setup_op(struct rte_eth_dev *eth_dev, + struct bnxt_tx_queue *parent_txq, *txq; + struct bnxt_vf_rep_tx_queue *vfr_txq; + +- if (queue_idx >= rep_bp->rx_nr_rings) { ++ if (queue_idx >= rep_bp->tx_nr_rings) { + PMD_DRV_LOG(ERR, + "Cannot create Tx rings %d. %d rings available\n", +- queue_idx, rep_bp->rx_nr_rings); ++ queue_idx, rep_bp->tx_nr_rings); + return -EINVAL; + } + +diff --git a/dpdk/drivers/net/bnxt/bnxt_txq.c b/dpdk/drivers/net/bnxt/bnxt_txq.c +index 4df4604975..696603757b 100644 +--- a/dpdk/drivers/net/bnxt/bnxt_txq.c ++++ b/dpdk/drivers/net/bnxt/bnxt_txq.c +@@ -112,6 +112,7 @@ void bnxt_tx_queue_release_op(struct rte_eth_dev *dev, uint16_t queue_idx) + txq->mz = NULL; + + rte_free(txq->free); ++ pthread_mutex_destroy(&txq->txq_lock); + rte_free(txq); + dev->data->tx_queues[queue_idx] = NULL; + } +@@ -195,6 +196,11 @@ int bnxt_tx_queue_setup_op(struct rte_eth_dev *eth_dev, + goto err; + } + ++ rc = pthread_mutex_init(&txq->txq_lock, NULL); ++ if (rc != 0) { ++ PMD_DRV_LOG(ERR, "TxQ mutex init failed!"); ++ goto err; ++ } + return 0; + err: + bnxt_tx_queue_release_op(eth_dev, queue_idx); +diff --git a/dpdk/drivers/net/bnxt/bnxt_txq.h b/dpdk/drivers/net/bnxt/bnxt_txq.h +index 3a483ad5c3..9e54985c4c 100644 +--- a/dpdk/drivers/net/bnxt/bnxt_txq.h ++++ b/dpdk/drivers/net/bnxt/bnxt_txq.h +@@ -26,6 +26,7 @@ struct bnxt_tx_queue { + int index; + int tx_wake_thresh; + uint32_t vfr_tx_cfa_action; ++ pthread_mutex_t txq_lock; + struct bnxt_tx_ring_info *tx_ring; + + unsigned int cp_nr_rings; +diff --git a/dpdk/drivers/net/bnxt/bnxt_txr.c b/dpdk/drivers/net/bnxt/bnxt_txr.c +index 899986764f..cef14427a8 100644 +--- a/dpdk/drivers/net/bnxt/bnxt_txr.c ++++ b/dpdk/drivers/net/bnxt/bnxt_txr.c +@@ -562,6 +562,19 @@ static int bnxt_handle_tx_cp(struct bnxt_tx_queue *txq) + + uint16_t bnxt_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) ++{ ++ struct bnxt_tx_queue *txq = tx_queue; ++ uint16_t rc; ++ ++ pthread_mutex_lock(&txq->txq_lock); ++ rc = _bnxt_xmit_pkts(tx_queue, tx_pkts, nb_pkts); ++ pthread_mutex_unlock(&txq->txq_lock); ++ ++ return rc; ++} ++ ++uint16_t _bnxt_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, ++ uint16_t nb_pkts) + { + int rc; + uint16_t nb_tx_pkts = 0; +diff --git a/dpdk/drivers/net/bnxt/bnxt_txr.h b/dpdk/drivers/net/bnxt/bnxt_txr.h +index e64ea2c7d1..09078d545d 100644 +--- a/dpdk/drivers/net/bnxt/bnxt_txr.h ++++ b/dpdk/drivers/net/bnxt/bnxt_txr.h +@@ -47,7 +47,9 @@ void bnxt_free_tx_rings(struct bnxt *bp); + int bnxt_init_one_tx_ring(struct bnxt_tx_queue *txq); + int bnxt_init_tx_ring_struct(struct bnxt_tx_queue *txq, unsigned int socket_id); + uint16_t bnxt_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, +- uint16_t nb_pkts); ++ uint16_t nb_pkts); ++uint16_t _bnxt_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, ++ uint16_t nb_pkts); + #if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64) + uint16_t bnxt_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts); +diff --git a/dpdk/drivers/net/bnxt/tf_ulp/ulp_ha_mgr.c b/dpdk/drivers/net/bnxt/tf_ulp/ulp_ha_mgr.c +index f3f5bda890..852deef3b4 100644 +--- a/dpdk/drivers/net/bnxt/tf_ulp/ulp_ha_mgr.c ++++ b/dpdk/drivers/net/bnxt/tf_ulp/ulp_ha_mgr.c +@@ -253,6 +253,7 @@ ulp_ha_mgr_timer_cb(void *arg) + + myclient_cnt = bnxt_ulp_cntxt_num_shared_clients_get(ulp_ctx); + if (myclient_cnt == 0) { ++ bnxt_ulp_cntxt_entry_release(); + BNXT_TF_DBG(ERR, + "PANIC Client Count is zero kill timer\n."); + return; +diff --git a/dpdk/drivers/net/bonding/rte_eth_bond_8023ad.c b/dpdk/drivers/net/bonding/rte_eth_bond_8023ad.c +index 79f1b3f1a0..06c21ebe6d 100644 +--- a/dpdk/drivers/net/bonding/rte_eth_bond_8023ad.c ++++ b/dpdk/drivers/net/bonding/rte_eth_bond_8023ad.c +@@ -865,7 +865,6 @@ bond_mode_8023ad_periodic_cb(void *arg) + struct bond_dev_private *internals = bond_dev->data->dev_private; + struct port *port; + struct rte_eth_link link_info; +- struct rte_ether_addr member_addr; + struct rte_mbuf *lacp_pkt = NULL; + uint16_t member_id; + uint16_t i; +@@ -892,7 +891,6 @@ bond_mode_8023ad_periodic_cb(void *arg) + key = 0; + } + +- rte_eth_macaddr_get(member_id, &member_addr); + port = &bond_mode_8023ad_ports[member_id]; + + key = rte_cpu_to_be_16(key); +@@ -904,8 +902,8 @@ bond_mode_8023ad_periodic_cb(void *arg) + SM_FLAG_SET(port, NTT); + } + +- if (!rte_is_same_ether_addr(&port->actor.system, &member_addr)) { +- rte_ether_addr_copy(&member_addr, &port->actor.system); ++ if (!rte_is_same_ether_addr(&internals->mode4.mac_addr, &port->actor.system)) { ++ rte_ether_addr_copy(&internals->mode4.mac_addr, &port->actor.system); + if (port->aggregator_port_id == member_id) + SM_FLAG_SET(port, NTT); + } +@@ -1173,21 +1171,20 @@ void + bond_mode_8023ad_mac_address_update(struct rte_eth_dev *bond_dev) + { + struct bond_dev_private *internals = bond_dev->data->dev_private; +- struct rte_ether_addr member_addr; + struct port *member, *agg_member; + uint16_t member_id, i, j; + + bond_mode_8023ad_stop(bond_dev); + ++ rte_eth_macaddr_get(internals->port_id, &internals->mode4.mac_addr); + for (i = 0; i < internals->active_member_count; i++) { + member_id = internals->active_members[i]; + member = &bond_mode_8023ad_ports[member_id]; +- rte_eth_macaddr_get(member_id, &member_addr); + +- if (rte_is_same_ether_addr(&member_addr, &member->actor.system)) ++ if (rte_is_same_ether_addr(&internals->mode4.mac_addr, &member->actor.system)) + continue; + +- rte_ether_addr_copy(&member_addr, &member->actor.system); ++ rte_ether_addr_copy(&internals->mode4.mac_addr, &member->actor.system); + /* Do nothing if this port is not an aggregator. In other case + * Set NTT flag on every port that use this aggregator. */ + if (member->aggregator_port_id != member_id) +diff --git a/dpdk/drivers/net/bonding/rte_eth_bond_flow.c b/dpdk/drivers/net/bonding/rte_eth_bond_flow.c +index 71a91675f7..5d0be5caf5 100644 +--- a/dpdk/drivers/net/bonding/rte_eth_bond_flow.c ++++ b/dpdk/drivers/net/bonding/rte_eth_bond_flow.c +@@ -180,6 +180,8 @@ bond_flow_query_count(struct rte_eth_dev *dev, struct rte_flow *flow, + + count->bytes = 0; + count->hits = 0; ++ count->bytes_set = 0; ++ count->hits_set = 0; + rte_memcpy(&member_count, count, sizeof(member_count)); + for (i = 0; i < internals->member_count; i++) { + ret = rte_flow_query(internals->members[i].port_id, +@@ -192,8 +194,12 @@ bond_flow_query_count(struct rte_eth_dev *dev, struct rte_flow *flow, + } + count->bytes += member_count.bytes; + count->hits += member_count.hits; ++ count->bytes_set |= member_count.bytes_set; ++ count->hits_set |= member_count.hits_set; + member_count.bytes = 0; + member_count.hits = 0; ++ member_count.bytes_set = 0; ++ member_count.hits_set = 0; + } + return 0; + } +diff --git a/dpdk/drivers/net/cnxk/cn10k_ethdev.c b/dpdk/drivers/net/cnxk/cn10k_ethdev.c +index 4a4e97287c..29b7f2ba5e 100644 +--- a/dpdk/drivers/net/cnxk/cn10k_ethdev.c ++++ b/dpdk/drivers/net/cnxk/cn10k_ethdev.c +@@ -389,7 +389,13 @@ cn10k_nix_tx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t qidx) + struct roc_nix_sq *sq = &dev->sqs[qidx]; + do { + handle_tx_completion_pkts(txq, flags & NIX_TX_VWQE_F); ++ /* Check if SQ is empty */ + roc_nix_sq_head_tail_get(nix, sq->qid, &head, &tail); ++ if (head != tail) ++ continue; ++ ++ /* Check if completion CQ is empty */ ++ roc_nix_cq_head_tail_get(nix, sq->cqid, &head, &tail); + } while (head != tail); + } + +diff --git a/dpdk/drivers/net/cnxk/cn10k_ethdev_sec.c b/dpdk/drivers/net/cnxk/cn10k_ethdev_sec.c +index 575d0fabd5..4719f6b863 100644 +--- a/dpdk/drivers/net/cnxk/cn10k_ethdev_sec.c ++++ b/dpdk/drivers/net/cnxk/cn10k_ethdev_sec.c +@@ -1087,8 +1087,8 @@ cn10k_eth_sec_session_update(void *device, struct rte_security_session *sess, + { + struct rte_eth_dev *eth_dev = (struct rte_eth_dev *)device; + struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); +- struct roc_ot_ipsec_inb_sa *inb_sa_dptr; + struct rte_security_ipsec_xform *ipsec; ++ struct cn10k_sec_sess_priv sess_priv; + struct rte_crypto_sym_xform *crypto; + struct cnxk_eth_sec_sess *eth_sec; + bool inbound; +@@ -1109,6 +1109,11 @@ cn10k_eth_sec_session_update(void *device, struct rte_security_session *sess, + eth_sec->spi = conf->ipsec.spi; + + if (inbound) { ++ struct roc_ot_ipsec_inb_sa *inb_sa_dptr, *inb_sa; ++ struct cn10k_inb_priv_data *inb_priv; ++ ++ inb_sa = eth_sec->sa; ++ inb_priv = roc_nix_inl_ot_ipsec_inb_sa_sw_rsvd(inb_sa); + inb_sa_dptr = (struct roc_ot_ipsec_inb_sa *)dev->inb.sa_dptr; + memset(inb_sa_dptr, 0, sizeof(struct roc_ot_ipsec_inb_sa)); + +@@ -1116,26 +1121,74 @@ cn10k_eth_sec_session_update(void *device, struct rte_security_session *sess, + true); + if (rc) + return -EINVAL; ++ /* Use cookie for original data */ ++ inb_sa_dptr->w1.s.cookie = inb_sa->w1.s.cookie; ++ ++ if (ipsec->options.stats == 1) { ++ /* Enable mib counters */ ++ inb_sa_dptr->w0.s.count_mib_bytes = 1; ++ inb_sa_dptr->w0.s.count_mib_pkts = 1; ++ } ++ ++ /* Enable out-of-place processing */ ++ if (ipsec->options.ingress_oop) ++ inb_sa_dptr->w0.s.pkt_format = ROC_IE_OT_SA_PKT_FMT_FULL; + + rc = roc_nix_inl_ctx_write(&dev->nix, inb_sa_dptr, eth_sec->sa, + eth_sec->inb, + sizeof(struct roc_ot_ipsec_inb_sa)); + if (rc) + return -EINVAL; ++ ++ /* Save userdata in inb private area */ ++ inb_priv->userdata = conf->userdata; + } else { +- struct roc_ot_ipsec_outb_sa *outb_sa_dptr; ++ struct roc_ot_ipsec_outb_sa *outb_sa_dptr, *outb_sa; ++ struct cn10k_outb_priv_data *outb_priv; ++ struct cnxk_ipsec_outb_rlens *rlens; + ++ outb_sa = eth_sec->sa; ++ outb_priv = roc_nix_inl_ot_ipsec_outb_sa_sw_rsvd(outb_sa); ++ rlens = &outb_priv->rlens; + outb_sa_dptr = (struct roc_ot_ipsec_outb_sa *)dev->outb.sa_dptr; + memset(outb_sa_dptr, 0, sizeof(struct roc_ot_ipsec_outb_sa)); + + rc = cnxk_ot_ipsec_outb_sa_fill(outb_sa_dptr, ipsec, crypto); + if (rc) + return -EINVAL; ++ ++ /* Save rlen info */ ++ cnxk_ipsec_outb_rlens_get(rlens, ipsec, crypto); ++ ++ if (ipsec->options.stats == 1) { ++ /* Enable mib counters */ ++ outb_sa_dptr->w0.s.count_mib_bytes = 1; ++ outb_sa_dptr->w0.s.count_mib_pkts = 1; ++ } ++ ++ sess_priv.u64 = 0; ++ sess_priv.sa_idx = outb_priv->sa_idx; ++ sess_priv.roundup_byte = rlens->roundup_byte; ++ sess_priv.roundup_len = rlens->roundup_len; ++ sess_priv.partial_len = rlens->partial_len; ++ sess_priv.mode = outb_sa_dptr->w2.s.ipsec_mode; ++ sess_priv.outer_ip_ver = outb_sa_dptr->w2.s.outer_ip_ver; ++ /* Propagate inner checksum enable from SA to fast path */ ++ sess_priv.chksum = ++ (!ipsec->options.ip_csum_enable << 1 | !ipsec->options.l4_csum_enable); ++ sess_priv.dec_ttl = ipsec->options.dec_ttl; ++ if (roc_feature_nix_has_inl_ipsec_mseg() && dev->outb.cpt_eng_caps & BIT_ULL(35)) ++ sess_priv.nixtx_off = 1; ++ + rc = roc_nix_inl_ctx_write(&dev->nix, outb_sa_dptr, eth_sec->sa, + eth_sec->inb, + sizeof(struct roc_ot_ipsec_outb_sa)); + if (rc) + return -EINVAL; ++ ++ /* Save userdata */ ++ outb_priv->userdata = conf->userdata; ++ sess->fast_mdata = sess_priv.u64; + } + + return 0; +diff --git a/dpdk/drivers/net/cnxk/cn10k_rx.h b/dpdk/drivers/net/cnxk/cn10k_rx.h +index 7bb4c86d75..86e4233dc7 100644 +--- a/dpdk/drivers/net/cnxk/cn10k_rx.h ++++ b/dpdk/drivers/net/cnxk/cn10k_rx.h +@@ -705,7 +705,7 @@ nix_cqe_xtract_mseg(const union nix_rx_parse_u *rx, struct rte_mbuf *mbuf, + if (cq_w1 & BIT(11) && flags & NIX_RX_OFFLOAD_SECURITY_F) { + const uint64_t *wqe = (const uint64_t *)(mbuf + 1); + +- if (hdr->w0.pkt_fmt != ROC_IE_OT_SA_PKT_FMT_FULL) ++ if (!(flags & NIX_RX_REAS_F) || hdr->w0.pkt_fmt != ROC_IE_OT_SA_PKT_FMT_FULL) + rx = (const union nix_rx_parse_u *)(wqe + 1); + } + +diff --git a/dpdk/drivers/net/cnxk/cn10k_rxtx.h b/dpdk/drivers/net/cnxk/cn10k_rxtx.h +index aeffc4ac92..9f33d0192e 100644 +--- a/dpdk/drivers/net/cnxk/cn10k_rxtx.h ++++ b/dpdk/drivers/net/cnxk/cn10k_rxtx.h +@@ -177,6 +177,7 @@ handle_tx_completion_pkts(struct cn10k_eth_txq *txq, uint8_t mt_safe) + m = m_next; + } + rte_pktmbuf_free_seg(m); ++ txq->tx_compl.ptr[tx_compl_s0->sqe_id] = NULL; + + head++; + head &= qmask; +diff --git a/dpdk/drivers/net/cnxk/cn10k_tx.h b/dpdk/drivers/net/cnxk/cn10k_tx.h +index 467f0ccc65..c84154ee84 100644 +--- a/dpdk/drivers/net/cnxk/cn10k_tx.h ++++ b/dpdk/drivers/net/cnxk/cn10k_tx.h +@@ -784,19 +784,35 @@ cn10k_nix_prep_sec(struct rte_mbuf *m, uint64_t *cmd, uintptr_t *nixtx_addr, + } + #endif + ++static inline void ++cn10k_nix_free_extmbuf(struct rte_mbuf *m) ++{ ++ struct rte_mbuf *m_next; ++ while (m != NULL) { ++ m_next = m->next; ++ rte_pktmbuf_free_seg(m); ++ m = m_next; ++ } ++} ++ + static __rte_always_inline uint64_t +-cn10k_nix_prefree_seg(struct rte_mbuf *m, struct cn10k_eth_txq *txq, +- struct nix_send_hdr_s *send_hdr) ++cn10k_nix_prefree_seg(struct rte_mbuf *m, struct rte_mbuf **extm, struct cn10k_eth_txq *txq, ++ struct nix_send_hdr_s *send_hdr, uint64_t *aura) + { ++ struct rte_mbuf *prev = NULL; + uint32_t sqe_id; + + if (RTE_MBUF_HAS_EXTBUF(m)) { + if (unlikely(txq->tx_compl.ena == 0)) { +- rte_pktmbuf_free_seg(m); ++ m->next = *extm; ++ *extm = m; + return 1; + } + if (send_hdr->w0.pnc) { +- txq->tx_compl.ptr[send_hdr->w1.sqe_id]->next = m; ++ sqe_id = send_hdr->w1.sqe_id; ++ prev = txq->tx_compl.ptr[sqe_id]; ++ m->next = prev; ++ txq->tx_compl.ptr[sqe_id] = m; + } else { + sqe_id = __atomic_fetch_add(&txq->tx_compl.sqe_id, 1, __ATOMIC_RELAXED); + send_hdr->w0.pnc = 1; +@@ -806,10 +822,160 @@ cn10k_nix_prefree_seg(struct rte_mbuf *m, struct cn10k_eth_txq *txq, + } + return 1; + } else { +- return cnxk_nix_prefree_seg(m); ++ return cnxk_nix_prefree_seg(m, aura); + } + } + ++#if defined(RTE_ARCH_ARM64) ++/* Only called for first segments of single segmented mbufs */ ++static __rte_always_inline void ++cn10k_nix_prefree_seg_vec(struct rte_mbuf **mbufs, struct rte_mbuf **extm, ++ struct cn10k_eth_txq *txq, ++ uint64x2_t *senddesc01_w0, uint64x2_t *senddesc23_w0, ++ uint64x2_t *senddesc01_w1, uint64x2_t *senddesc23_w1) ++{ ++ struct rte_mbuf **tx_compl_ptr = txq->tx_compl.ptr; ++ uint32_t nb_desc_mask = txq->tx_compl.nb_desc_mask; ++ bool tx_compl_ena = txq->tx_compl.ena; ++ struct rte_mbuf *m0, *m1, *m2, *m3; ++ struct rte_mbuf *cookie; ++ uint64_t w0, w1, aura; ++ uint64_t sqe_id; ++ ++ m0 = mbufs[0]; ++ m1 = mbufs[1]; ++ m2 = mbufs[2]; ++ m3 = mbufs[3]; ++ ++ /* mbuf 0 */ ++ w0 = vgetq_lane_u64(*senddesc01_w0, 0); ++ if (RTE_MBUF_HAS_EXTBUF(m0)) { ++ w0 |= BIT_ULL(19); ++ w1 = vgetq_lane_u64(*senddesc01_w1, 0); ++ w1 &= ~0xFFFF000000000000UL; ++ if (unlikely(!tx_compl_ena)) { ++ m0->next = *extm; ++ *extm = m0; ++ } else { ++ sqe_id = rte_atomic_fetch_add_explicit(&txq->tx_compl.sqe_id, 1, ++ rte_memory_order_relaxed); ++ sqe_id = sqe_id & nb_desc_mask; ++ /* Set PNC */ ++ w0 |= BIT_ULL(43); ++ w1 |= sqe_id << 48; ++ tx_compl_ptr[sqe_id] = m0; ++ *senddesc01_w1 = vsetq_lane_u64(w1, *senddesc01_w1, 0); ++ } ++ } else { ++ cookie = RTE_MBUF_DIRECT(m0) ? m0 : rte_mbuf_from_indirect(m0); ++ aura = (w0 >> 20) & 0xFFFFF; ++ w0 &= ~0xFFFFF00000UL; ++ w0 |= cnxk_nix_prefree_seg(m0, &aura) << 19; ++ w0 |= aura << 20; ++ ++ if ((w0 & BIT_ULL(19)) == 0) ++ RTE_MEMPOOL_CHECK_COOKIES(cookie->pool, (void **)&cookie, 1, 0); ++ } ++ *senddesc01_w0 = vsetq_lane_u64(w0, *senddesc01_w0, 0); ++ ++ /* mbuf1 */ ++ w0 = vgetq_lane_u64(*senddesc01_w0, 1); ++ if (RTE_MBUF_HAS_EXTBUF(m1)) { ++ w0 |= BIT_ULL(19); ++ w1 = vgetq_lane_u64(*senddesc01_w1, 1); ++ w1 &= ~0xFFFF000000000000UL; ++ if (unlikely(!tx_compl_ena)) { ++ m1->next = *extm; ++ *extm = m1; ++ } else { ++ sqe_id = rte_atomic_fetch_add_explicit(&txq->tx_compl.sqe_id, 1, ++ rte_memory_order_relaxed); ++ sqe_id = sqe_id & nb_desc_mask; ++ /* Set PNC */ ++ w0 |= BIT_ULL(43); ++ w1 |= sqe_id << 48; ++ tx_compl_ptr[sqe_id] = m1; ++ *senddesc01_w1 = vsetq_lane_u64(w1, *senddesc01_w1, 1); ++ } ++ } else { ++ cookie = RTE_MBUF_DIRECT(m1) ? m1 : rte_mbuf_from_indirect(m1); ++ aura = (w0 >> 20) & 0xFFFFF; ++ w0 &= ~0xFFFFF00000UL; ++ w0 |= cnxk_nix_prefree_seg(m1, &aura) << 19; ++ w0 |= aura << 20; ++ ++ if ((w0 & BIT_ULL(19)) == 0) ++ RTE_MEMPOOL_CHECK_COOKIES(cookie->pool, (void **)&cookie, 1, 0); ++ } ++ *senddesc01_w0 = vsetq_lane_u64(w0, *senddesc01_w0, 1); ++ ++ /* mbuf 2 */ ++ w0 = vgetq_lane_u64(*senddesc23_w0, 0); ++ if (RTE_MBUF_HAS_EXTBUF(m2)) { ++ w0 |= BIT_ULL(19); ++ w1 = vgetq_lane_u64(*senddesc23_w1, 0); ++ w1 &= ~0xFFFF000000000000UL; ++ if (unlikely(!tx_compl_ena)) { ++ m2->next = *extm; ++ *extm = m2; ++ } else { ++ sqe_id = rte_atomic_fetch_add_explicit(&txq->tx_compl.sqe_id, 1, ++ rte_memory_order_relaxed); ++ sqe_id = sqe_id & nb_desc_mask; ++ /* Set PNC */ ++ w0 |= BIT_ULL(43); ++ w1 |= sqe_id << 48; ++ tx_compl_ptr[sqe_id] = m2; ++ *senddesc23_w1 = vsetq_lane_u64(w1, *senddesc23_w1, 0); ++ } ++ } else { ++ cookie = RTE_MBUF_DIRECT(m2) ? m2 : rte_mbuf_from_indirect(m2); ++ aura = (w0 >> 20) & 0xFFFFF; ++ w0 &= ~0xFFFFF00000UL; ++ w0 |= cnxk_nix_prefree_seg(m2, &aura) << 19; ++ w0 |= aura << 20; ++ ++ if ((w0 & BIT_ULL(19)) == 0) ++ RTE_MEMPOOL_CHECK_COOKIES(cookie->pool, (void **)&cookie, 1, 0); ++ } ++ *senddesc23_w0 = vsetq_lane_u64(w0, *senddesc23_w0, 0); ++ ++ /* mbuf3 */ ++ w0 = vgetq_lane_u64(*senddesc23_w0, 1); ++ if (RTE_MBUF_HAS_EXTBUF(m3)) { ++ w0 |= BIT_ULL(19); ++ w1 = vgetq_lane_u64(*senddesc23_w1, 1); ++ w1 &= ~0xFFFF000000000000UL; ++ if (unlikely(!tx_compl_ena)) { ++ m3->next = *extm; ++ *extm = m3; ++ } else { ++ sqe_id = rte_atomic_fetch_add_explicit(&txq->tx_compl.sqe_id, 1, ++ rte_memory_order_relaxed); ++ sqe_id = sqe_id & nb_desc_mask; ++ /* Set PNC */ ++ w0 |= BIT_ULL(43); ++ w1 |= sqe_id << 48; ++ tx_compl_ptr[sqe_id] = m3; ++ *senddesc23_w1 = vsetq_lane_u64(w1, *senddesc23_w1, 1); ++ } ++ } else { ++ cookie = RTE_MBUF_DIRECT(m3) ? m3 : rte_mbuf_from_indirect(m3); ++ aura = (w0 >> 20) & 0xFFFFF; ++ w0 &= ~0xFFFFF00000UL; ++ w0 |= cnxk_nix_prefree_seg(m3, &aura) << 19; ++ w0 |= aura << 20; ++ ++ if ((w0 & BIT_ULL(19)) == 0) ++ RTE_MEMPOOL_CHECK_COOKIES(cookie->pool, (void **)&cookie, 1, 0); ++ } ++ *senddesc23_w0 = vsetq_lane_u64(w0, *senddesc23_w0, 1); ++#ifndef RTE_LIBRTE_MEMPOOL_DEBUG ++ RTE_SET_USED(cookie); ++#endif ++} ++#endif ++ + static __rte_always_inline void + cn10k_nix_xmit_prepare_tso(struct rte_mbuf *m, const uint64_t flags) + { +@@ -864,9 +1030,9 @@ cn10k_nix_xmit_prepare_tso(struct rte_mbuf *m, const uint64_t flags) + + static __rte_always_inline void + cn10k_nix_xmit_prepare(struct cn10k_eth_txq *txq, +- struct rte_mbuf *m, uint64_t *cmd, const uint16_t flags, +- const uint64_t lso_tun_fmt, bool *sec, uint8_t mark_flag, +- uint64_t mark_fmt) ++ struct rte_mbuf *m, struct rte_mbuf **extm, uint64_t *cmd, ++ const uint16_t flags, const uint64_t lso_tun_fmt, bool *sec, ++ uint8_t mark_flag, uint64_t mark_fmt) + { + uint8_t mark_off = 0, mark_vlan = 0, markptr = 0; + struct nix_send_ext_s *send_hdr_ext; +@@ -889,6 +1055,9 @@ cn10k_nix_xmit_prepare(struct cn10k_eth_txq *txq, + sg = (union nix_send_sg_s *)(cmd + 2); + } + ++ if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F) ++ send_hdr->w0.pnc = 0; ++ + if (flags & (NIX_TX_NEED_SEND_HDR_W1 | NIX_TX_OFFLOAD_SECURITY_F)) { + ol_flags = m->ol_flags; + w1.u = 0; +@@ -1049,19 +1218,30 @@ cn10k_nix_xmit_prepare(struct cn10k_eth_txq *txq, + send_hdr->w1.u = w1.u; + + if (!(flags & NIX_TX_MULTI_SEG_F)) { ++ struct rte_mbuf *cookie; ++ + sg->seg1_size = send_hdr->w0.total; + *(rte_iova_t *)(sg + 1) = rte_mbuf_data_iova(m); ++ cookie = RTE_MBUF_DIRECT(m) ? m : rte_mbuf_from_indirect(m); + + if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F) { ++ uint64_t aura; ++ + /* DF bit = 1 if refcount of current mbuf or parent mbuf + * is greater than 1 + * DF bit = 0 otherwise + */ +- send_hdr->w0.df = cn10k_nix_prefree_seg(m, txq, send_hdr); ++ aura = send_hdr->w0.aura; ++ send_hdr->w0.df = cn10k_nix_prefree_seg(m, extm, txq, send_hdr, &aura); ++ send_hdr->w0.aura = aura; + } ++#ifdef RTE_LIBRTE_MEMPOOL_DEBUG + /* Mark mempool object as "put" since it is freed by NIX */ + if (!send_hdr->w0.df) +- RTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0); ++ RTE_MEMPOOL_CHECK_COOKIES(cookie->pool, (void **)&cookie, 1, 0); ++#else ++ RTE_SET_USED(cookie); ++#endif + } else { + sg->seg1_size = m->data_len; + *(rte_iova_t *)(sg + 1) = rte_mbuf_data_iova(m); +@@ -1113,7 +1293,7 @@ cn10k_nix_xmit_prepare_tstamp(struct cn10k_eth_txq *txq, uintptr_t lmt_addr, + struct nix_send_mem_s *send_mem; + + send_mem = (struct nix_send_mem_s *)(lmt + off); +- /* Packets for which PKT_TX_IEEE1588_TMST is not set, tx tstamp ++ /* Packets for which RTE_MBUF_F_TX_IEEE1588_TMST is not set, Tx tstamp + * should not be recorded, hence changing the alg type to + * NIX_SENDMEMALG_SUB and also changing send mem addr field to + * next 8 bytes as it corrupts the actual Tx tstamp registered +@@ -1128,13 +1308,14 @@ cn10k_nix_xmit_prepare_tstamp(struct cn10k_eth_txq *txq, uintptr_t lmt_addr, + } + + static __rte_always_inline uint16_t +-cn10k_nix_prepare_mseg(struct cn10k_eth_txq *txq, +- struct rte_mbuf *m, uint64_t *cmd, const uint16_t flags) ++cn10k_nix_prepare_mseg(struct cn10k_eth_txq *txq, struct rte_mbuf *m, struct rte_mbuf **extm, ++ uint64_t *cmd, const uint16_t flags) + { + uint64_t prefree = 0, aura0, aura, nb_segs, segdw; + struct nix_send_hdr_s *send_hdr; + union nix_send_sg_s *sg, l_sg; + union nix_send_sg2_s l_sg2; ++ struct rte_mbuf *cookie; + struct rte_mbuf *m_next; + uint8_t off, is_sg2; + uint64_t len, dlen; +@@ -1163,21 +1344,27 @@ cn10k_nix_prepare_mseg(struct cn10k_eth_txq *txq, + len -= dlen; + nb_segs = m->nb_segs - 1; + m_next = m->next; ++ m->next = NULL; ++ m->nb_segs = 1; + slist = &cmd[3 + off + 1]; + ++ cookie = RTE_MBUF_DIRECT(m) ? m : rte_mbuf_from_indirect(m); + /* Set invert df if buffer is not to be freed by H/W */ + if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F) { +- prefree = cn10k_nix_prefree_seg(m, txq, send_hdr); ++ aura = send_hdr->w0.aura; ++ prefree = cn10k_nix_prefree_seg(m, extm, txq, send_hdr, &aura); ++ send_hdr->w0.aura = aura; + l_sg.i1 = prefree; + } + + #ifdef RTE_LIBRTE_MEMPOOL_DEBUG + /* Mark mempool object as "put" since it is freed by NIX */ + if (!prefree) +- RTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0); ++ RTE_MEMPOOL_CHECK_COOKIES(cookie->pool, (void **)&cookie, 1, 0); + rte_io_wmb(); ++#else ++ RTE_SET_USED(cookie); + #endif +- m->next = NULL; + + /* Quickly handle single segmented packets. With this if-condition + * compiler will completely optimize out the below do-while loop +@@ -1207,9 +1394,12 @@ cn10k_nix_prepare_mseg(struct cn10k_eth_txq *txq, + aura = aura0; + prefree = 0; + ++ m->next = NULL; ++ ++ cookie = RTE_MBUF_DIRECT(m) ? m : rte_mbuf_from_indirect(m); + if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F) { + aura = roc_npa_aura_handle_to_aura(m->pool->pool_id); +- prefree = cn10k_nix_prefree_seg(m, txq, send_hdr); ++ prefree = cn10k_nix_prefree_seg(m, extm, txq, send_hdr, &aura); + is_sg2 = aura != aura0 && !prefree; + } + +@@ -1259,13 +1449,14 @@ cn10k_nix_prepare_mseg(struct cn10k_eth_txq *txq, + l_sg.subdc = NIX_SUBDC_SG; + slist++; + } +- m->next = NULL; + + #ifdef RTE_LIBRTE_MEMPOOL_DEBUG + /* Mark mempool object as "put" since it is freed by NIX + */ + if (!prefree) +- RTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0); ++ RTE_MEMPOOL_CHECK_COOKIES(cookie->pool, (void **)&cookie, 1, 0); ++#else ++ RTE_SET_USED(cookie); + #endif + m = m_next; + } while (nb_segs); +@@ -1302,6 +1493,7 @@ cn10k_nix_xmit_pkts(void *tx_queue, uint64_t *ws, struct rte_mbuf **tx_pkts, + uint8_t lnum, c_lnum, c_shft, c_loff; + uintptr_t pa, lbase = txq->lmt_base; + uint16_t lmt_id, burst, left, i; ++ struct rte_mbuf *extm = NULL; + uintptr_t c_lbase = lbase; + uint64_t lso_tun_fmt = 0; + uint64_t mark_fmt = 0; +@@ -1356,7 +1548,7 @@ again: + if (flags & NIX_TX_OFFLOAD_TSO_F) + cn10k_nix_xmit_prepare_tso(tx_pkts[i], flags); + +- cn10k_nix_xmit_prepare(txq, tx_pkts[i], cmd, flags, lso_tun_fmt, ++ cn10k_nix_xmit_prepare(txq, tx_pkts[i], &extm, cmd, flags, lso_tun_fmt, + &sec, mark_flag, mark_fmt); + + laddr = (uintptr_t)LMT_OFF(lbase, lnum, 0); +@@ -1431,6 +1623,11 @@ again: + } + + rte_io_wmb(); ++ if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F && !txq->tx_compl.ena) { ++ cn10k_nix_free_extmbuf(extm); ++ extm = NULL; ++ } ++ + if (left) + goto again; + +@@ -1446,6 +1643,7 @@ cn10k_nix_xmit_pkts_mseg(void *tx_queue, uint64_t *ws, + uintptr_t pa0, pa1, lbase = txq->lmt_base; + const rte_iova_t io_addr = txq->io_addr; + uint16_t segdw, lmt_id, burst, left, i; ++ struct rte_mbuf *extm = NULL; + uint8_t lnum, c_lnum, c_loff; + uintptr_t c_lbase = lbase; + uint64_t lso_tun_fmt = 0; +@@ -1507,7 +1705,7 @@ again: + if (flags & NIX_TX_OFFLOAD_TSO_F) + cn10k_nix_xmit_prepare_tso(tx_pkts[i], flags); + +- cn10k_nix_xmit_prepare(txq, tx_pkts[i], cmd, flags, lso_tun_fmt, ++ cn10k_nix_xmit_prepare(txq, tx_pkts[i], &extm, cmd, flags, lso_tun_fmt, + &sec, mark_flag, mark_fmt); + + laddr = (uintptr_t)LMT_OFF(lbase, lnum, 0); +@@ -1521,7 +1719,7 @@ again: + /* Move NIX desc to LMT/NIXTX area */ + cn10k_nix_xmit_mv_lmt_base(laddr, cmd, flags); + /* Store sg list directly on lmt line */ +- segdw = cn10k_nix_prepare_mseg(txq, tx_pkts[i], (uint64_t *)laddr, ++ segdw = cn10k_nix_prepare_mseg(txq, tx_pkts[i], &extm, (uint64_t *)laddr, + flags); + cn10k_nix_xmit_prepare_tstamp(txq, laddr, tx_pkts[i]->ol_flags, + segdw, flags); +@@ -1594,6 +1792,11 @@ again: + } + + rte_io_wmb(); ++ if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F && !txq->tx_compl.ena) { ++ cn10k_nix_free_extmbuf(extm); ++ extm = NULL; ++ } ++ + if (left) + goto again; + +@@ -1644,7 +1847,7 @@ cn10k_nix_prepare_tso(struct rte_mbuf *m, union nix_send_hdr_w1_u *w1, + + static __rte_always_inline uint16_t + cn10k_nix_prepare_mseg_vec_noff(struct cn10k_eth_txq *txq, +- struct rte_mbuf *m, uint64_t *cmd, ++ struct rte_mbuf *m, struct rte_mbuf **extm, uint64_t *cmd, + uint64x2_t *cmd0, uint64x2_t *cmd1, + uint64x2_t *cmd2, uint64x2_t *cmd3, + const uint32_t flags) +@@ -1659,7 +1862,7 @@ cn10k_nix_prepare_mseg_vec_noff(struct cn10k_eth_txq *txq, + vst1q_u64(cmd + 2, *cmd1); /* sg */ + } + +- segdw = cn10k_nix_prepare_mseg(txq, m, cmd, flags); ++ segdw = cn10k_nix_prepare_mseg(txq, m, extm, cmd, flags); + + if (flags & NIX_TX_OFFLOAD_TSTAMP_F) + vst1q_u64(cmd + segdw * 2 - 2, *cmd3); +@@ -1694,9 +1897,13 @@ cn10k_nix_prepare_mseg_vec_list(struct rte_mbuf *m, uint64_t *cmd, + len -= dlen; + sg_u = sg_u | ((uint64_t)dlen); + ++ /* Mark mempool object as "put" since it is freed by NIX */ ++ RTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0); ++ + nb_segs = m->nb_segs - 1; + m_next = m->next; + m->next = NULL; ++ m->nb_segs = 1; + m = m_next; + /* Fill mbuf segments */ + do { +@@ -1719,6 +1926,9 @@ cn10k_nix_prepare_mseg_vec_list(struct rte_mbuf *m, uint64_t *cmd, + slist++; + } + m->next = NULL; ++ /* Mark mempool object as "put" since it is freed by NIX */ ++ RTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0); ++ + m = m_next; + } while (nb_segs); + +@@ -1742,8 +1952,11 @@ cn10k_nix_prepare_mseg_vec(struct rte_mbuf *m, uint64_t *cmd, uint64x2_t *cmd0, + union nix_send_hdr_w0_u sh; + union nix_send_sg_s sg; + +- if (m->nb_segs == 1) ++ if (m->nb_segs == 1) { ++ /* Mark mempool object as "put" since it is freed by NIX */ ++ RTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0); + return; ++ } + + sh.u = vgetq_lane_u64(cmd0[0], 0); + sg.u = vgetq_lane_u64(cmd1[0], 0); +@@ -1759,7 +1972,7 @@ cn10k_nix_prepare_mseg_vec(struct rte_mbuf *m, uint64_t *cmd, uint64x2_t *cmd0, + + static __rte_always_inline uint8_t + cn10k_nix_prep_lmt_mseg_vector(struct cn10k_eth_txq *txq, +- struct rte_mbuf **mbufs, uint64x2_t *cmd0, ++ struct rte_mbuf **mbufs, struct rte_mbuf **extm, uint64x2_t *cmd0, + uint64x2_t *cmd1, uint64x2_t *cmd2, + uint64x2_t *cmd3, uint8_t *segdw, + uint64_t *lmt_addr, __uint128_t *data128, +@@ -1777,7 +1990,7 @@ cn10k_nix_prep_lmt_mseg_vector(struct cn10k_eth_txq *txq, + lmt_addr += 16; + off = 0; + } +- off += cn10k_nix_prepare_mseg_vec_noff(txq, mbufs[j], ++ off += cn10k_nix_prepare_mseg_vec_noff(txq, mbufs[j], extm, + lmt_addr + off * 2, &cmd0[j], &cmd1[j], + &cmd2[j], &cmd3[j], flags); + } +@@ -1803,6 +2016,11 @@ cn10k_nix_prep_lmt_mseg_vector(struct cn10k_eth_txq *txq, + *data128 |= ((__uint128_t)7) << *shift; + *shift += 3; + ++ /* Mark mempool object as "put" since it is freed by NIX */ ++ RTE_MEMPOOL_CHECK_COOKIES(mbufs[0]->pool, (void **)&mbufs[0], 1, 0); ++ RTE_MEMPOOL_CHECK_COOKIES(mbufs[1]->pool, (void **)&mbufs[1], 1, 0); ++ RTE_MEMPOOL_CHECK_COOKIES(mbufs[2]->pool, (void **)&mbufs[2], 1, 0); ++ RTE_MEMPOOL_CHECK_COOKIES(mbufs[3]->pool, (void **)&mbufs[3], 1, 0); + return 1; + } + } +@@ -1821,6 +2039,11 @@ cn10k_nix_prep_lmt_mseg_vector(struct cn10k_eth_txq *txq, + vst1q_u64(lmt_addr + 10, cmd2[j + 1]); + vst1q_u64(lmt_addr + 12, cmd1[j + 1]); + vst1q_u64(lmt_addr + 14, cmd3[j + 1]); ++ ++ /* Mark mempool object as "put" since it is freed by NIX */ ++ RTE_MEMPOOL_CHECK_COOKIES(mbufs[j]->pool, (void **)&mbufs[j], 1, 0); ++ RTE_MEMPOOL_CHECK_COOKIES(mbufs[j + 1]->pool, ++ (void **)&mbufs[j + 1], 1, 0); + } else if (flags & NIX_TX_NEED_EXT_HDR) { + /* EXT header take 3 each, space for 2 segs.*/ + cn10k_nix_prepare_mseg_vec(mbufs[j], +@@ -1920,14 +2143,14 @@ cn10k_nix_lmt_next(uint8_t dw, uintptr_t laddr, uint8_t *lnum, uint8_t *loff, + + static __rte_always_inline void + cn10k_nix_xmit_store(struct cn10k_eth_txq *txq, +- struct rte_mbuf *mbuf, uint8_t segdw, uintptr_t laddr, ++ struct rte_mbuf *mbuf, struct rte_mbuf **extm, uint8_t segdw, uintptr_t laddr, + uint64x2_t cmd0, uint64x2_t cmd1, uint64x2_t cmd2, + uint64x2_t cmd3, const uint16_t flags) + { + uint8_t off; + + if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F) { +- cn10k_nix_prepare_mseg_vec_noff(txq, mbuf, LMT_OFF(laddr, 0, 0), ++ cn10k_nix_prepare_mseg_vec_noff(txq, mbuf, extm, LMT_OFF(laddr, 0, 0), + &cmd0, &cmd1, &cmd2, &cmd3, + flags); + return; +@@ -1997,13 +2220,10 @@ cn10k_nix_xmit_pkts_vector(void *tx_queue, uint64_t *ws, + uint64x2_t sgdesc01_w0, sgdesc23_w0; + uint64x2_t sgdesc01_w1, sgdesc23_w1; + struct cn10k_eth_txq *txq = tx_queue; +- uint64x2_t xmask01_w0, xmask23_w0; +- uint64x2_t xmask01_w1, xmask23_w1; + rte_iova_t io_addr = txq->io_addr; + uint8_t lnum, shift = 0, loff = 0; + uintptr_t laddr = txq->lmt_base; + uint8_t c_lnum, c_shft, c_loff; +- struct nix_send_hdr_s send_hdr; + uint64x2_t ltypes01, ltypes23; + uint64x2_t xtmp128, ytmp128; + uint64x2_t xmask01, xmask23; +@@ -2014,6 +2234,7 @@ cn10k_nix_xmit_pkts_vector(void *tx_queue, uint64_t *ws, + __uint128_t data128; + uint64_t data[2]; + } wd; ++ struct rte_mbuf *extm = NULL; + + if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F && txq->tx_compl.ena) + handle_tx_completion_pkts(txq, flags & NIX_TX_VWQE_F); +@@ -2098,7 +2319,8 @@ again: + } + + for (i = 0; i < burst; i += NIX_DESCS_PER_LOOP) { +- if (flags & NIX_TX_OFFLOAD_SECURITY_F && c_lnum + 2 > 16) { ++ if (flags & NIX_TX_OFFLOAD_SECURITY_F && ++ (((int)((16 - c_lnum) << 1) - c_loff) < 4)) { + burst = i; + break; + } +@@ -2153,7 +2375,7 @@ again: + } + /* Clear lower 32bit of SEND_HDR_W0 and SEND_SG_W0 */ + senddesc01_w0 = +- vbicq_u64(senddesc01_w0, vdupq_n_u64(0xFFFFFFFF)); ++ vbicq_u64(senddesc01_w0, vdupq_n_u64(0x800FFFFFFFF)); + sgdesc01_w0 = vbicq_u64(sgdesc01_w0, vdupq_n_u64(0xFFFFFFFF)); + + senddesc23_w0 = senddesc01_w0; +@@ -2859,73 +3081,8 @@ again: + !(flags & NIX_TX_MULTI_SEG_F) && + !(flags & NIX_TX_OFFLOAD_SECURITY_F)) { + /* Set don't free bit if reference count > 1 */ +- xmask01_w0 = vdupq_n_u64(0); +- xmask01_w1 = vdupq_n_u64(0); +- xmask23_w0 = xmask01_w0; +- xmask23_w1 = xmask01_w1; +- +- /* Move mbufs to iova */ +- mbuf0 = (uint64_t *)tx_pkts[0]; +- mbuf1 = (uint64_t *)tx_pkts[1]; +- mbuf2 = (uint64_t *)tx_pkts[2]; +- mbuf3 = (uint64_t *)tx_pkts[3]; +- +- send_hdr.w0.u = 0; +- send_hdr.w1.u = 0; +- +- if (cn10k_nix_prefree_seg((struct rte_mbuf *)mbuf0, txq, &send_hdr)) { +- send_hdr.w0.df = 1; +- xmask01_w0 = vsetq_lane_u64(send_hdr.w0.u, xmask01_w0, 0); +- xmask01_w1 = vsetq_lane_u64(send_hdr.w1.u, xmask01_w1, 0); +- } else { +- RTE_MEMPOOL_CHECK_COOKIES( +- ((struct rte_mbuf *)mbuf0)->pool, +- (void **)&mbuf0, 1, 0); +- } +- +- send_hdr.w0.u = 0; +- send_hdr.w1.u = 0; +- +- if (cn10k_nix_prefree_seg((struct rte_mbuf *)mbuf1, txq, &send_hdr)) { +- send_hdr.w0.df = 1; +- xmask01_w0 = vsetq_lane_u64(send_hdr.w0.u, xmask01_w0, 1); +- xmask01_w1 = vsetq_lane_u64(send_hdr.w1.u, xmask01_w1, 1); +- } else { +- RTE_MEMPOOL_CHECK_COOKIES( +- ((struct rte_mbuf *)mbuf1)->pool, +- (void **)&mbuf1, 1, 0); +- } +- +- send_hdr.w0.u = 0; +- send_hdr.w1.u = 0; +- +- if (cn10k_nix_prefree_seg((struct rte_mbuf *)mbuf2, txq, &send_hdr)) { +- send_hdr.w0.df = 1; +- xmask23_w0 = vsetq_lane_u64(send_hdr.w0.u, xmask23_w0, 0); +- xmask23_w1 = vsetq_lane_u64(send_hdr.w1.u, xmask23_w1, 0); +- } else { +- RTE_MEMPOOL_CHECK_COOKIES( +- ((struct rte_mbuf *)mbuf2)->pool, +- (void **)&mbuf2, 1, 0); +- } +- +- send_hdr.w0.u = 0; +- send_hdr.w1.u = 0; +- +- if (cn10k_nix_prefree_seg((struct rte_mbuf *)mbuf3, txq, &send_hdr)) { +- send_hdr.w0.df = 1; +- xmask23_w0 = vsetq_lane_u64(send_hdr.w0.u, xmask23_w0, 1); +- xmask23_w1 = vsetq_lane_u64(send_hdr.w1.u, xmask23_w1, 1); +- } else { +- RTE_MEMPOOL_CHECK_COOKIES( +- ((struct rte_mbuf *)mbuf3)->pool, +- (void **)&mbuf3, 1, 0); +- } +- +- senddesc01_w0 = vorrq_u64(senddesc01_w0, xmask01_w0); +- senddesc23_w0 = vorrq_u64(senddesc23_w0, xmask23_w0); +- senddesc01_w1 = vorrq_u64(senddesc01_w1, xmask01_w1); +- senddesc23_w1 = vorrq_u64(senddesc23_w1, xmask23_w1); ++ cn10k_nix_prefree_seg_vec(tx_pkts, &extm, txq, &senddesc01_w0, ++ &senddesc23_w0, &senddesc01_w1, &senddesc23_w1); + } else if (!(flags & NIX_TX_MULTI_SEG_F) && + !(flags & NIX_TX_OFFLOAD_SECURITY_F)) { + /* Move mbufs to iova */ +@@ -2997,7 +3154,7 @@ again: + &shift, &wd.data128, &next); + + /* Store mbuf0 to LMTLINE/CPT NIXTX area */ +- cn10k_nix_xmit_store(txq, tx_pkts[0], segdw[0], next, ++ cn10k_nix_xmit_store(txq, tx_pkts[0], &extm, segdw[0], next, + cmd0[0], cmd1[0], cmd2[0], cmd3[0], + flags); + +@@ -3013,7 +3170,7 @@ again: + &shift, &wd.data128, &next); + + /* Store mbuf1 to LMTLINE/CPT NIXTX area */ +- cn10k_nix_xmit_store(txq, tx_pkts[1], segdw[1], next, ++ cn10k_nix_xmit_store(txq, tx_pkts[1], &extm, segdw[1], next, + cmd0[1], cmd1[1], cmd2[1], cmd3[1], + flags); + +@@ -3029,7 +3186,7 @@ again: + &shift, &wd.data128, &next); + + /* Store mbuf2 to LMTLINE/CPT NIXTX area */ +- cn10k_nix_xmit_store(txq, tx_pkts[2], segdw[2], next, ++ cn10k_nix_xmit_store(txq, tx_pkts[2], &extm, segdw[2], next, + cmd0[2], cmd1[2], cmd2[2], cmd3[2], + flags); + +@@ -3045,7 +3202,7 @@ again: + &shift, &wd.data128, &next); + + /* Store mbuf3 to LMTLINE/CPT NIXTX area */ +- cn10k_nix_xmit_store(txq, tx_pkts[3], segdw[3], next, ++ cn10k_nix_xmit_store(txq, tx_pkts[3], &extm, segdw[3], next, + cmd0[3], cmd1[3], cmd2[3], cmd3[3], + flags); + +@@ -3053,7 +3210,7 @@ again: + uint8_t j; + + segdw[4] = 8; +- j = cn10k_nix_prep_lmt_mseg_vector(txq, tx_pkts, cmd0, cmd1, ++ j = cn10k_nix_prep_lmt_mseg_vector(txq, tx_pkts, &extm, cmd0, cmd1, + cmd2, cmd3, segdw, + (uint64_t *) + LMT_OFF(laddr, lnum, +@@ -3203,6 +3360,11 @@ again: + } + + rte_io_wmb(); ++ if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F && !txq->tx_compl.ena) { ++ cn10k_nix_free_extmbuf(extm); ++ extm = NULL; ++ } ++ + if (left) + goto again; + +diff --git a/dpdk/drivers/net/cnxk/cn9k_ethdev.c b/dpdk/drivers/net/cnxk/cn9k_ethdev.c +index bae4dda5e2..b92b978a27 100644 +--- a/dpdk/drivers/net/cnxk/cn9k_ethdev.c ++++ b/dpdk/drivers/net/cnxk/cn9k_ethdev.c +@@ -347,7 +347,13 @@ cn9k_nix_tx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t qidx) + struct roc_nix_sq *sq = &dev->sqs[qidx]; + do { + handle_tx_completion_pkts(txq, 0); ++ /* Check if SQ is empty */ + roc_nix_sq_head_tail_get(nix, sq->qid, &head, &tail); ++ if (head != tail) ++ continue; ++ ++ /* Check if completion CQ is empty */ ++ roc_nix_cq_head_tail_get(nix, sq->cqid, &head, &tail); + } while (head != tail); + } + +diff --git a/dpdk/drivers/net/cnxk/cn9k_ethdev.h b/dpdk/drivers/net/cnxk/cn9k_ethdev.h +index 9e0a3c5bb2..6ae0db62ca 100644 +--- a/dpdk/drivers/net/cnxk/cn9k_ethdev.h ++++ b/dpdk/drivers/net/cnxk/cn9k_ethdev.h +@@ -169,6 +169,7 @@ handle_tx_completion_pkts(struct cn9k_eth_txq *txq, uint8_t mt_safe) + m = m_next; + } + rte_pktmbuf_free_seg(m); ++ txq->tx_compl.ptr[tx_compl_s0->sqe_id] = NULL; + + head++; + head &= qmask; +diff --git a/dpdk/drivers/net/cnxk/cn9k_tx.h b/dpdk/drivers/net/cnxk/cn9k_tx.h +index fba4bb4215..4715bf8a65 100644 +--- a/dpdk/drivers/net/cnxk/cn9k_tx.h ++++ b/dpdk/drivers/net/cnxk/cn9k_tx.h +@@ -82,32 +82,198 @@ cn9k_nix_tx_skeleton(struct cn9k_eth_txq *txq, uint64_t *cmd, + } + } + ++static __rte_always_inline void ++cn9k_nix_free_extmbuf(struct rte_mbuf *m) ++{ ++ struct rte_mbuf *m_next; ++ while (m != NULL) { ++ m_next = m->next; ++ rte_pktmbuf_free_seg(m); ++ m = m_next; ++ } ++} ++ + static __rte_always_inline uint64_t +-cn9k_nix_prefree_seg(struct rte_mbuf *m, struct cn9k_eth_txq *txq, +- struct nix_send_hdr_s *send_hdr) ++cn9k_nix_prefree_seg(struct rte_mbuf *m, struct rte_mbuf **extm, struct cn9k_eth_txq *txq, ++ struct nix_send_hdr_s *send_hdr, uint64_t *aura) + { ++ struct rte_mbuf *prev; + uint32_t sqe_id; + + if (RTE_MBUF_HAS_EXTBUF(m)) { + if (unlikely(txq->tx_compl.ena == 0)) { +- rte_pktmbuf_free_seg(m); ++ m->next = *extm; ++ *extm = m; + return 1; + } + if (send_hdr->w0.pnc) { +- txq->tx_compl.ptr[send_hdr->w1.sqe_id]->next = m; ++ sqe_id = send_hdr->w1.sqe_id; ++ prev = txq->tx_compl.ptr[sqe_id]; ++ m->next = prev; ++ txq->tx_compl.ptr[sqe_id] = m; + } else { + sqe_id = __atomic_fetch_add(&txq->tx_compl.sqe_id, 1, __ATOMIC_RELAXED); + send_hdr->w0.pnc = 1; + send_hdr->w1.sqe_id = sqe_id & + txq->tx_compl.nb_desc_mask; + txq->tx_compl.ptr[send_hdr->w1.sqe_id] = m; ++ m->next = NULL; + } + return 1; + } else { +- return cnxk_nix_prefree_seg(m); ++ return cnxk_nix_prefree_seg(m, aura); + } + } + ++#if defined(RTE_ARCH_ARM64) ++/* Only called for first segments of single segmented mbufs */ ++static __rte_always_inline void ++cn9k_nix_prefree_seg_vec(struct rte_mbuf **mbufs, struct rte_mbuf **extm, struct cn9k_eth_txq *txq, ++ uint64x2_t *senddesc01_w0, uint64x2_t *senddesc23_w0, ++ uint64x2_t *senddesc01_w1, uint64x2_t *senddesc23_w1) ++{ ++ struct rte_mbuf **tx_compl_ptr = txq->tx_compl.ptr; ++ uint32_t nb_desc_mask = txq->tx_compl.nb_desc_mask; ++ bool tx_compl_ena = txq->tx_compl.ena; ++ struct rte_mbuf *m0, *m1, *m2, *m3; ++ struct rte_mbuf *cookie; ++ uint64_t w0, w1, aura; ++ uint64_t sqe_id; ++ ++ m0 = mbufs[0]; ++ m1 = mbufs[1]; ++ m2 = mbufs[2]; ++ m3 = mbufs[3]; ++ ++ /* mbuf 0 */ ++ w0 = vgetq_lane_u64(*senddesc01_w0, 0); ++ if (RTE_MBUF_HAS_EXTBUF(m0)) { ++ w0 |= BIT_ULL(19); ++ w1 = vgetq_lane_u64(*senddesc01_w1, 0); ++ w1 &= ~0xFFFF000000000000UL; ++ if (unlikely(!tx_compl_ena)) { ++ m0->next = *extm; ++ *extm = m0; ++ } else { ++ sqe_id = rte_atomic_fetch_add_explicit(&txq->tx_compl.sqe_id, 1, ++ rte_memory_order_relaxed); ++ sqe_id = sqe_id & nb_desc_mask; ++ /* Set PNC */ ++ w0 |= BIT_ULL(43); ++ w1 |= sqe_id << 48; ++ tx_compl_ptr[sqe_id] = m0; ++ *senddesc01_w1 = vsetq_lane_u64(w1, *senddesc01_w1, 0); ++ } ++ } else { ++ cookie = RTE_MBUF_DIRECT(m0) ? m0 : rte_mbuf_from_indirect(m0); ++ aura = (w0 >> 20) & 0xFFFFF; ++ w0 &= ~0xFFFFF00000UL; ++ w0 |= cnxk_nix_prefree_seg(m0, &aura) << 19; ++ w0 |= aura << 20; ++ ++ if ((w0 & BIT_ULL(19)) == 0) ++ RTE_MEMPOOL_CHECK_COOKIES(cookie->pool, (void **)&cookie, 1, 0); ++ } ++ *senddesc01_w0 = vsetq_lane_u64(w0, *senddesc01_w0, 0); ++ ++ /* mbuf1 */ ++ w0 = vgetq_lane_u64(*senddesc01_w0, 1); ++ if (RTE_MBUF_HAS_EXTBUF(m1)) { ++ w0 |= BIT_ULL(19); ++ w1 = vgetq_lane_u64(*senddesc01_w1, 1); ++ w1 &= ~0xFFFF000000000000UL; ++ if (unlikely(!tx_compl_ena)) { ++ m1->next = *extm; ++ *extm = m1; ++ } else { ++ sqe_id = rte_atomic_fetch_add_explicit(&txq->tx_compl.sqe_id, 1, ++ rte_memory_order_relaxed); ++ sqe_id = sqe_id & nb_desc_mask; ++ /* Set PNC */ ++ w0 |= BIT_ULL(43); ++ w1 |= sqe_id << 48; ++ tx_compl_ptr[sqe_id] = m1; ++ *senddesc01_w1 = vsetq_lane_u64(w1, *senddesc01_w1, 1); ++ } ++ } else { ++ cookie = RTE_MBUF_DIRECT(m1) ? m1 : rte_mbuf_from_indirect(m1); ++ aura = (w0 >> 20) & 0xFFFFF; ++ w0 &= ~0xFFFFF00000UL; ++ w0 |= cnxk_nix_prefree_seg(m1, &aura) << 19; ++ w0 |= aura << 20; ++ ++ if ((w0 & BIT_ULL(19)) == 0) ++ RTE_MEMPOOL_CHECK_COOKIES(cookie->pool, (void **)&cookie, 1, 0); ++ } ++ *senddesc01_w0 = vsetq_lane_u64(w0, *senddesc01_w0, 1); ++ ++ /* mbuf 2 */ ++ w0 = vgetq_lane_u64(*senddesc23_w0, 0); ++ if (RTE_MBUF_HAS_EXTBUF(m2)) { ++ w0 |= BIT_ULL(19); ++ w1 = vgetq_lane_u64(*senddesc23_w1, 0); ++ w1 &= ~0xFFFF000000000000UL; ++ if (unlikely(!tx_compl_ena)) { ++ m2->next = *extm; ++ *extm = m2; ++ } else { ++ sqe_id = rte_atomic_fetch_add_explicit(&txq->tx_compl.sqe_id, 1, ++ rte_memory_order_relaxed); ++ sqe_id = sqe_id & nb_desc_mask; ++ /* Set PNC */ ++ w0 |= BIT_ULL(43); ++ w1 |= sqe_id << 48; ++ tx_compl_ptr[sqe_id] = m2; ++ *senddesc23_w1 = vsetq_lane_u64(w1, *senddesc23_w1, 0); ++ } ++ } else { ++ cookie = RTE_MBUF_DIRECT(m2) ? m2 : rte_mbuf_from_indirect(m2); ++ aura = (w0 >> 20) & 0xFFFFF; ++ w0 &= ~0xFFFFF00000UL; ++ w0 |= cnxk_nix_prefree_seg(m2, &aura) << 19; ++ w0 |= aura << 20; ++ ++ if ((w0 & BIT_ULL(19)) == 0) ++ RTE_MEMPOOL_CHECK_COOKIES(cookie->pool, (void **)&cookie, 1, 0); ++ } ++ *senddesc23_w0 = vsetq_lane_u64(w0, *senddesc23_w0, 0); ++ ++ /* mbuf3 */ ++ w0 = vgetq_lane_u64(*senddesc23_w0, 1); ++ if (RTE_MBUF_HAS_EXTBUF(m3)) { ++ w0 |= BIT_ULL(19); ++ w1 = vgetq_lane_u64(*senddesc23_w1, 1); ++ w1 &= ~0xFFFF000000000000UL; ++ if (unlikely(!tx_compl_ena)) { ++ m3->next = *extm; ++ *extm = m3; ++ } else { ++ sqe_id = rte_atomic_fetch_add_explicit(&txq->tx_compl.sqe_id, 1, ++ rte_memory_order_relaxed); ++ sqe_id = sqe_id & nb_desc_mask; ++ /* Set PNC */ ++ w0 |= BIT_ULL(43); ++ w1 |= sqe_id << 48; ++ tx_compl_ptr[sqe_id] = m3; ++ *senddesc23_w1 = vsetq_lane_u64(w1, *senddesc23_w1, 1); ++ } ++ } else { ++ cookie = RTE_MBUF_DIRECT(m3) ? m3 : rte_mbuf_from_indirect(m3); ++ aura = (w0 >> 20) & 0xFFFFF; ++ w0 &= ~0xFFFFF00000UL; ++ w0 |= cnxk_nix_prefree_seg(m3, &aura) << 19; ++ w0 |= aura << 20; ++ ++ if ((w0 & BIT_ULL(19)) == 0) ++ RTE_MEMPOOL_CHECK_COOKIES(cookie->pool, (void **)&cookie, 1, 0); ++ } ++ *senddesc23_w0 = vsetq_lane_u64(w0, *senddesc23_w0, 1); ++#ifndef RTE_LIBRTE_MEMPOOL_DEBUG ++ RTE_SET_USED(cookie); ++#endif ++} ++#endif ++ + static __rte_always_inline void + cn9k_nix_xmit_prepare_tso(struct rte_mbuf *m, const uint64_t flags) + { +@@ -161,10 +327,9 @@ cn9k_nix_xmit_prepare_tso(struct rte_mbuf *m, const uint64_t flags) + } + + static __rte_always_inline void +-cn9k_nix_xmit_prepare(struct cn9k_eth_txq *txq, +- struct rte_mbuf *m, uint64_t *cmd, const uint16_t flags, +- const uint64_t lso_tun_fmt, uint8_t mark_flag, +- uint64_t mark_fmt) ++cn9k_nix_xmit_prepare(struct cn9k_eth_txq *txq, struct rte_mbuf *m, struct rte_mbuf **extm, ++ uint64_t *cmd, const uint16_t flags, const uint64_t lso_tun_fmt, ++ uint8_t mark_flag, uint64_t mark_fmt) + { + uint8_t mark_off = 0, mark_vlan = 0, markptr = 0; + struct nix_send_ext_s *send_hdr_ext; +@@ -191,6 +356,8 @@ cn9k_nix_xmit_prepare(struct cn9k_eth_txq *txq, + ol_flags = m->ol_flags; + w1.u = 0; + } ++ if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F) ++ send_hdr->w0.pnc = 0; + + if (!(flags & NIX_TX_MULTI_SEG_F)) + send_hdr->w0.total = m->data_len; +@@ -345,23 +512,33 @@ cn9k_nix_xmit_prepare(struct cn9k_eth_txq *txq, + send_hdr->w1.u = w1.u; + + if (!(flags & NIX_TX_MULTI_SEG_F)) { ++ struct rte_mbuf *cookie; ++ + sg->seg1_size = m->data_len; + *(rte_iova_t *)(++sg) = rte_mbuf_data_iova(m); ++ cookie = RTE_MBUF_DIRECT(m) ? m : rte_mbuf_from_indirect(m); + + if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F) { ++ uint64_t aura; + /* DF bit = 1 if refcount of current mbuf or parent mbuf + * is greater than 1 + * DF bit = 0 otherwise + */ +- send_hdr->w0.df = cn9k_nix_prefree_seg(m, txq, send_hdr); ++ aura = send_hdr->w0.aura; ++ send_hdr->w0.df = cn9k_nix_prefree_seg(m, extm, txq, send_hdr, &aura); ++ send_hdr->w0.aura = aura; + /* Ensuring mbuf fields which got updated in + * cnxk_nix_prefree_seg are written before LMTST. + */ + rte_io_wmb(); + } ++#ifdef RTE_LIBRTE_MEMPOOL_DEBUG + /* Mark mempool object as "put" since it is freed by NIX */ + if (!send_hdr->w0.df) +- RTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0); ++ RTE_MEMPOOL_CHECK_COOKIES(cookie->pool, (void **)&cookie, 1, 0); ++#else ++ RTE_SET_USED(cookie); ++#endif + } else { + sg->seg1_size = m->data_len; + *(rte_iova_t *)(sg + 1) = rte_mbuf_data_iova(m); +@@ -383,7 +560,7 @@ cn9k_nix_xmit_prepare_tstamp(struct cn9k_eth_txq *txq, uint64_t *cmd, + + send_mem = (struct nix_send_mem_s *)(cmd + off); + +- /* Packets for which PKT_TX_IEEE1588_TMST is not set, tx tstamp ++ /* Packets for which RTE_MBUF_F_TX_IEEE1588_TMST is not set, Tx tstamp + * should not be recorded, hence changing the alg type to + * NIX_SENDMEMALG_SUB and also changing send mem addr field to + * next 8 bytes as it corrupts the actual Tx tstamp registered +@@ -439,10 +616,12 @@ cn9k_nix_xmit_submit_lmt_release(const rte_iova_t io_addr) + } + + static __rte_always_inline uint16_t +-cn9k_nix_prepare_mseg(struct cn9k_eth_txq *txq, +- struct rte_mbuf *m, uint64_t *cmd, const uint16_t flags) ++cn9k_nix_prepare_mseg(struct cn9k_eth_txq *txq, struct rte_mbuf *m, struct rte_mbuf **extm, ++ uint64_t *cmd, const uint16_t flags) + { + struct nix_send_hdr_s *send_hdr; ++ uint64_t prefree = 0, aura; ++ struct rte_mbuf *cookie; + union nix_send_sg_s *sg; + struct rte_mbuf *m_next; + uint64_t *slist, sg_u; +@@ -467,17 +646,27 @@ cn9k_nix_prepare_mseg(struct cn9k_eth_txq *txq, + m_next = m->next; + slist = &cmd[3 + off + 1]; + ++ cookie = RTE_MBUF_DIRECT(m) ? m : rte_mbuf_from_indirect(m); + /* Set invert df if buffer is not to be freed by H/W */ + if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F) { +- sg_u |= (cn9k_nix_prefree_seg(m, txq, send_hdr) << 55); ++ aura = send_hdr->w0.aura; ++ prefree = (cn9k_nix_prefree_seg(m, extm, txq, send_hdr, &aura) << 55); ++ send_hdr->w0.aura = aura; ++ sg_u |= prefree; + rte_io_wmb(); + } + + /* Mark mempool object as "put" since it is freed by NIX */ + #ifdef RTE_LIBRTE_MEMPOOL_DEBUG + if (!(sg_u & (1ULL << 55))) +- RTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0); ++ RTE_MEMPOOL_CHECK_COOKIES(cookie->pool, (void **)&cookie, 1, 0); + rte_io_wmb(); ++#else ++ RTE_SET_USED(cookie); ++#endif ++#ifdef RTE_ENABLE_ASSERT ++ m->next = NULL; ++ m->nb_segs = 1; + #endif + m = m_next; + if (!m) +@@ -488,16 +677,17 @@ cn9k_nix_prepare_mseg(struct cn9k_eth_txq *txq, + m_next = m->next; + sg_u = sg_u | ((uint64_t)m->data_len << (i << 4)); + *slist = rte_mbuf_data_iova(m); ++ cookie = RTE_MBUF_DIRECT(m) ? m : rte_mbuf_from_indirect(m); + /* Set invert df if buffer is not to be freed by H/W */ + if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F) { +- sg_u |= (cn9k_nix_prefree_seg(m, txq, send_hdr) << (i + 55)); ++ sg_u |= (cn9k_nix_prefree_seg(m, extm, txq, send_hdr, NULL) << (i + 55)); + /* Commit changes to mbuf */ + rte_io_wmb(); + } + /* Mark mempool object as "put" since it is freed by NIX */ + #ifdef RTE_LIBRTE_MEMPOOL_DEBUG + if (!(sg_u & (1ULL << (i + 55)))) +- RTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0); ++ RTE_MEMPOOL_CHECK_COOKIES(cookie->pool, (void **)&cookie, 1, 0); + rte_io_wmb(); + #endif + slist++; +@@ -513,6 +703,9 @@ cn9k_nix_prepare_mseg(struct cn9k_eth_txq *txq, + sg_u = sg->u; + slist++; + } ++#ifdef RTE_ENABLE_ASSERT ++ m->next = NULL; ++#endif + m = m_next; + } while (nb_segs); + +@@ -526,6 +719,9 @@ done: + segdw += (off >> 1) + 1 + !!(flags & NIX_TX_OFFLOAD_TSTAMP_F); + send_hdr->w0.sizem1 = segdw - 1; + ++#ifdef RTE_ENABLE_ASSERT ++ rte_io_wmb(); ++#endif + return segdw; + } + +@@ -568,6 +764,7 @@ cn9k_nix_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t pkts, + const rte_iova_t io_addr = txq->io_addr; + uint64_t lso_tun_fmt = 0, mark_fmt = 0; + void *lmt_addr = txq->lmt_addr; ++ struct rte_mbuf *extm = NULL; + uint8_t mark_flag = 0; + uint16_t i; + +@@ -598,13 +795,16 @@ cn9k_nix_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t pkts, + rte_io_wmb(); + + for (i = 0; i < pkts; i++) { +- cn9k_nix_xmit_prepare(txq, tx_pkts[i], cmd, flags, lso_tun_fmt, ++ cn9k_nix_xmit_prepare(txq, tx_pkts[i], &extm, cmd, flags, lso_tun_fmt, + mark_flag, mark_fmt); + cn9k_nix_xmit_prepare_tstamp(txq, cmd, tx_pkts[i]->ol_flags, 4, + flags); + cn9k_nix_xmit_one(cmd, lmt_addr, io_addr, flags); + } + ++ if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F && !txq->tx_compl.ena) ++ cn9k_nix_free_extmbuf(extm); ++ + /* Reduce the cached count */ + txq->fc_cache_pkts -= pkts; + +@@ -619,6 +819,7 @@ cn9k_nix_xmit_pkts_mseg(void *tx_queue, struct rte_mbuf **tx_pkts, + const rte_iova_t io_addr = txq->io_addr; + uint64_t lso_tun_fmt = 0, mark_fmt = 0; + void *lmt_addr = txq->lmt_addr; ++ struct rte_mbuf *extm = NULL; + uint8_t mark_flag = 0; + uint16_t segdw; + uint64_t i; +@@ -650,14 +851,17 @@ cn9k_nix_xmit_pkts_mseg(void *tx_queue, struct rte_mbuf **tx_pkts, + rte_io_wmb(); + + for (i = 0; i < pkts; i++) { +- cn9k_nix_xmit_prepare(txq, tx_pkts[i], cmd, flags, lso_tun_fmt, ++ cn9k_nix_xmit_prepare(txq, tx_pkts[i], &extm, cmd, flags, lso_tun_fmt, + mark_flag, mark_fmt); +- segdw = cn9k_nix_prepare_mseg(txq, tx_pkts[i], cmd, flags); ++ segdw = cn9k_nix_prepare_mseg(txq, tx_pkts[i], &extm, cmd, flags); + cn9k_nix_xmit_prepare_tstamp(txq, cmd, tx_pkts[i]->ol_flags, + segdw, flags); + cn9k_nix_xmit_mseg_one(cmd, lmt_addr, io_addr, segdw); + } + ++ if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F && !txq->tx_compl.ena) ++ cn9k_nix_free_extmbuf(extm); ++ + /* Reduce the cached count */ + txq->fc_cache_pkts -= pkts; + +@@ -705,12 +909,12 @@ cn9k_nix_prepare_tso(struct rte_mbuf *m, union nix_send_hdr_w1_u *w1, + + static __rte_always_inline uint8_t + cn9k_nix_prepare_mseg_vec_list(struct cn9k_eth_txq *txq, +- struct rte_mbuf *m, uint64_t *cmd, ++ struct rte_mbuf *m, struct rte_mbuf **extm, uint64_t *cmd, + struct nix_send_hdr_s *send_hdr, + union nix_send_sg_s *sg, const uint32_t flags) + { +- struct rte_mbuf *m_next; +- uint64_t *slist, sg_u; ++ struct rte_mbuf *m_next, *cookie; ++ uint64_t *slist, sg_u, aura; + uint16_t nb_segs; + uint64_t segdw; + int i = 1; +@@ -727,29 +931,40 @@ cn9k_nix_prepare_mseg_vec_list(struct cn9k_eth_txq *txq, + m_next = m->next; + + /* Set invert df if buffer is not to be freed by H/W */ +- if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F) +- sg_u |= (cn9k_nix_prefree_seg(m, txq, send_hdr) << 55); +- /* Mark mempool object as "put" since it is freed by NIX */ ++ cookie = RTE_MBUF_DIRECT(m) ? m : rte_mbuf_from_indirect(m); ++ if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F) { ++ aura = send_hdr->w0.aura; ++ sg_u |= (cn9k_nix_prefree_seg(m, extm, txq, send_hdr, &aura) << 55); ++ send_hdr->w0.aura = aura; ++ } ++ /* Mark mempool object as "put" since it is freed by NIX */ + #ifdef RTE_LIBRTE_MEMPOOL_DEBUG + if (!(sg_u & (1ULL << 55))) +- RTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0); ++ RTE_MEMPOOL_CHECK_COOKIES(cookie->pool, (void **)&cookie, 1, 0); + rte_io_wmb(); ++#else ++ RTE_SET_USED(cookie); + #endif + ++#ifdef RTE_ENABLE_ASSERT ++ m->next = NULL; ++ m->nb_segs = 1; ++#endif + m = m_next; + /* Fill mbuf segments */ + do { + m_next = m->next; + sg_u = sg_u | ((uint64_t)m->data_len << (i << 4)); + *slist = rte_mbuf_data_iova(m); ++ cookie = RTE_MBUF_DIRECT(m) ? m : rte_mbuf_from_indirect(m); + /* Set invert df if buffer is not to be freed by H/W */ + if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F) +- sg_u |= (cn9k_nix_prefree_seg(m, txq, send_hdr) << (i + 55)); ++ sg_u |= (cn9k_nix_prefree_seg(m, extm, txq, send_hdr, &aura) << (i + 55)); + /* Mark mempool object as "put" since it is freed by NIX + */ + #ifdef RTE_LIBRTE_MEMPOOL_DEBUG + if (!(sg_u & (1ULL << (i + 55)))) +- RTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0); ++ RTE_MEMPOOL_CHECK_COOKIES(cookie->pool, (void **)&cookie, 1, 0); + rte_io_wmb(); + #endif + slist++; +@@ -765,6 +980,9 @@ cn9k_nix_prepare_mseg_vec_list(struct cn9k_eth_txq *txq, + sg_u = sg->u; + slist++; + } ++#ifdef RTE_ENABLE_ASSERT ++ m->next = NULL; ++#endif + m = m_next; + } while (nb_segs); + +@@ -780,24 +998,31 @@ cn9k_nix_prepare_mseg_vec_list(struct cn9k_eth_txq *txq, + !!(flags & NIX_TX_OFFLOAD_TSTAMP_F); + send_hdr->w0.sizem1 = segdw - 1; + ++#ifdef RTE_ENABLE_ASSERT ++ rte_io_wmb(); ++#endif + return segdw; + } + + static __rte_always_inline uint8_t +-cn9k_nix_prepare_mseg_vec(struct cn9k_eth_txq *txq, +- struct rte_mbuf *m, uint64_t *cmd, uint64x2_t *cmd0, +- uint64x2_t *cmd1, const uint32_t flags) ++cn9k_nix_prepare_mseg_vec(struct cn9k_eth_txq *txq, struct rte_mbuf *m, struct rte_mbuf **extm, ++ uint64_t *cmd, uint64x2_t *cmd0, uint64x2_t *cmd1, const uint32_t flags) + { + struct nix_send_hdr_s send_hdr; ++ struct rte_mbuf *cookie; + union nix_send_sg_s sg; ++ uint64_t aura; + uint8_t ret; + + if (m->nb_segs == 1) { ++ cookie = RTE_MBUF_DIRECT(m) ? m : rte_mbuf_from_indirect(m); + if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F) { + send_hdr.w0.u = vgetq_lane_u64(cmd0[0], 0); + send_hdr.w1.u = vgetq_lane_u64(cmd0[0], 1); + sg.u = vgetq_lane_u64(cmd1[0], 0); +- sg.u |= (cn9k_nix_prefree_seg(m, txq, &send_hdr) << 55); ++ aura = send_hdr.w0.aura; ++ sg.u |= (cn9k_nix_prefree_seg(m, extm, txq, &send_hdr, &aura) << 55); ++ send_hdr.w0.aura = aura; + cmd1[0] = vsetq_lane_u64(sg.u, cmd1[0], 0); + cmd0[0] = vsetq_lane_u64(send_hdr.w0.u, cmd0[0], 0); + cmd0[0] = vsetq_lane_u64(send_hdr.w1.u, cmd0[0], 1); +@@ -806,8 +1031,10 @@ cn9k_nix_prepare_mseg_vec(struct cn9k_eth_txq *txq, + #ifdef RTE_LIBRTE_MEMPOOL_DEBUG + sg.u = vgetq_lane_u64(cmd1[0], 0); + if (!(sg.u & (1ULL << 55))) +- RTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0); ++ RTE_MEMPOOL_CHECK_COOKIES(cookie->pool, (void **)&cookie, 1, 0); + rte_io_wmb(); ++#else ++ RTE_SET_USED(cookie); + #endif + return 2 + !!(flags & NIX_TX_NEED_EXT_HDR) + + !!(flags & NIX_TX_OFFLOAD_TSTAMP_F); +@@ -817,7 +1044,7 @@ cn9k_nix_prepare_mseg_vec(struct cn9k_eth_txq *txq, + send_hdr.w1.u = vgetq_lane_u64(cmd0[0], 1); + sg.u = vgetq_lane_u64(cmd1[0], 0); + +- ret = cn9k_nix_prepare_mseg_vec_list(txq, m, cmd, &send_hdr, &sg, flags); ++ ret = cn9k_nix_prepare_mseg_vec_list(txq, m, extm, cmd, &send_hdr, &sg, flags); + + cmd0[0] = vsetq_lane_u64(send_hdr.w0.u, cmd0[0], 0); + cmd0[0] = vsetq_lane_u64(send_hdr.w1.u, cmd0[0], 1); +@@ -962,11 +1189,9 @@ cn9k_nix_xmit_pkts_vector(void *tx_queue, struct rte_mbuf **tx_pkts, + uint64x2_t sgdesc01_w1, sgdesc23_w1; + struct cn9k_eth_txq *txq = tx_queue; + uint64_t *lmt_addr = txq->lmt_addr; +- uint64x2_t xmask01_w0, xmask23_w0; +- uint64x2_t xmask01_w1, xmask23_w1; + rte_iova_t io_addr = txq->io_addr; +- struct nix_send_hdr_s send_hdr; + uint64x2_t ltypes01, ltypes23; ++ struct rte_mbuf *extm = NULL; + uint64x2_t xtmp128, ytmp128; + uint64x2_t xmask01, xmask23; + uint64_t lmt_status, i; +@@ -1028,7 +1253,7 @@ cn9k_nix_xmit_pkts_vector(void *tx_queue, struct rte_mbuf **tx_pkts, + for (i = 0; i < pkts; i += NIX_DESCS_PER_LOOP) { + /* Clear lower 32bit of SEND_HDR_W0 and SEND_SG_W0 */ + senddesc01_w0 = +- vbicq_u64(senddesc01_w0, vdupq_n_u64(0xFFFFFFFF)); ++ vbicq_u64(senddesc01_w0, vdupq_n_u64(0x800FFFFFFFF)); + sgdesc01_w0 = vbicq_u64(sgdesc01_w0, vdupq_n_u64(0xFFFFFFFF)); + + senddesc23_w0 = senddesc01_w0; +@@ -1732,74 +1957,8 @@ cn9k_nix_xmit_pkts_vector(void *tx_queue, struct rte_mbuf **tx_pkts, + if ((flags & NIX_TX_OFFLOAD_MBUF_NOFF_F) && + !(flags & NIX_TX_MULTI_SEG_F)) { + /* Set don't free bit if reference count > 1 */ +- xmask01_w0 = vdupq_n_u64(0); +- xmask01_w1 = vdupq_n_u64(0); +- xmask23_w0 = xmask01_w0; +- xmask23_w1 = xmask01_w1; +- +- /* Move mbufs to iova */ +- mbuf0 = (uint64_t *)tx_pkts[0]; +- mbuf1 = (uint64_t *)tx_pkts[1]; +- mbuf2 = (uint64_t *)tx_pkts[2]; +- mbuf3 = (uint64_t *)tx_pkts[3]; +- +- send_hdr.w0.u = 0; +- send_hdr.w1.u = 0; +- +- if (cn9k_nix_prefree_seg((struct rte_mbuf *)mbuf0, txq, &send_hdr)) { +- send_hdr.w0.df = 1; +- xmask01_w0 = vsetq_lane_u64(send_hdr.w0.u, xmask01_w0, 0); +- xmask01_w1 = vsetq_lane_u64(send_hdr.w1.u, xmask01_w1, 0); +- } else { +- RTE_MEMPOOL_CHECK_COOKIES( +- ((struct rte_mbuf *)mbuf0)->pool, +- (void **)&mbuf0, 1, 0); +- } +- +- send_hdr.w0.u = 0; +- send_hdr.w1.u = 0; +- +- if (cn9k_nix_prefree_seg((struct rte_mbuf *)mbuf1, txq, &send_hdr)) { +- send_hdr.w0.df = 1; +- xmask01_w0 = vsetq_lane_u64(send_hdr.w0.u, xmask01_w0, 1); +- xmask01_w1 = vsetq_lane_u64(send_hdr.w1.u, xmask01_w1, 1); +- } else { +- RTE_MEMPOOL_CHECK_COOKIES( +- ((struct rte_mbuf *)mbuf1)->pool, +- (void **)&mbuf1, 1, 0); +- } +- +- send_hdr.w0.u = 0; +- send_hdr.w1.u = 0; +- +- if (cn9k_nix_prefree_seg((struct rte_mbuf *)mbuf2, txq, &send_hdr)) { +- send_hdr.w0.df = 1; +- xmask23_w0 = vsetq_lane_u64(send_hdr.w0.u, xmask23_w0, 0); +- xmask23_w1 = vsetq_lane_u64(send_hdr.w1.u, xmask23_w1, 0); +- } else { +- RTE_MEMPOOL_CHECK_COOKIES( +- ((struct rte_mbuf *)mbuf2)->pool, +- (void **)&mbuf2, 1, 0); +- } +- +- send_hdr.w0.u = 0; +- send_hdr.w1.u = 0; +- +- if (cn9k_nix_prefree_seg((struct rte_mbuf *)mbuf3, txq, &send_hdr)) { +- send_hdr.w0.df = 1; +- xmask23_w0 = vsetq_lane_u64(send_hdr.w0.u, xmask23_w0, 1); +- xmask23_w1 = vsetq_lane_u64(send_hdr.w1.u, xmask23_w1, 1); +- } else { +- RTE_MEMPOOL_CHECK_COOKIES( +- ((struct rte_mbuf *)mbuf3)->pool, +- (void **)&mbuf3, 1, 0); +- } +- +- senddesc01_w0 = vorrq_u64(senddesc01_w0, xmask01_w0); +- senddesc23_w0 = vorrq_u64(senddesc23_w0, xmask23_w0); +- senddesc01_w1 = vorrq_u64(senddesc01_w1, xmask01_w1); +- senddesc23_w1 = vorrq_u64(senddesc23_w1, xmask23_w1); +- ++ cn9k_nix_prefree_seg_vec(tx_pkts, &extm, txq, &senddesc01_w0, ++ &senddesc23_w0, &senddesc01_w1, &senddesc23_w1); + /* Ensuring mbuf fields which got updated in + * cnxk_nix_prefree_seg are written before LMTST. + */ +@@ -1860,7 +2019,7 @@ cn9k_nix_xmit_pkts_vector(void *tx_queue, struct rte_mbuf **tx_pkts, + /* Build mseg list for each packet individually. */ + for (j = 0; j < NIX_DESCS_PER_LOOP; j++) + segdw[j] = cn9k_nix_prepare_mseg_vec(txq, +- tx_pkts[j], ++ tx_pkts[j], &extm, + seg_list[j], &cmd0[j], + &cmd1[j], flags); + segdw[4] = 8; +@@ -1935,6 +2094,9 @@ cn9k_nix_xmit_pkts_vector(void *tx_queue, struct rte_mbuf **tx_pkts, + tx_pkts = tx_pkts + NIX_DESCS_PER_LOOP; + } + ++ if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F && !txq->tx_compl.ena) ++ cn9k_nix_free_extmbuf(extm); ++ + if (unlikely(pkts_left)) { + if (flags & NIX_TX_MULTI_SEG_F) + pkts += cn9k_nix_xmit_pkts_mseg(tx_queue, tx_pkts, +diff --git a/dpdk/drivers/net/cnxk/cnxk_ethdev.c b/dpdk/drivers/net/cnxk/cnxk_ethdev.c +index 5e11bbb017..c841b31051 100644 +--- a/dpdk/drivers/net/cnxk/cnxk_ethdev.c ++++ b/dpdk/drivers/net/cnxk/cnxk_ethdev.c +@@ -1384,6 +1384,13 @@ cnxk_nix_configure(struct rte_eth_dev *eth_dev) + goto free_nix_lf; + } + ++ /* Overwrite default RSS setup if requested by user */ ++ rc = cnxk_nix_rss_hash_update(eth_dev, &conf->rx_adv_conf.rss_conf); ++ if (rc) { ++ plt_err("Failed to configure rss rc=%d", rc); ++ goto free_nix_lf; ++ } ++ + /* Init the default TM scheduler hierarchy */ + rc = roc_nix_tm_init(nix); + if (rc) { +diff --git a/dpdk/drivers/net/cnxk/cnxk_ethdev_devargs.c b/dpdk/drivers/net/cnxk/cnxk_ethdev_devargs.c +index 8e862be933..a0e9300cff 100644 +--- a/dpdk/drivers/net/cnxk/cnxk_ethdev_devargs.c ++++ b/dpdk/drivers/net/cnxk/cnxk_ethdev_devargs.c +@@ -75,7 +75,7 @@ parse_ipsec_out_max_sa(const char *key, const char *value, void *extra_args) + if (errno) + val = 0; + +- *(uint16_t *)extra_args = val; ++ *(uint32_t *)extra_args = val; + + return 0; + } +diff --git a/dpdk/drivers/net/cnxk/cnxk_ethdev_dp.h b/dpdk/drivers/net/cnxk/cnxk_ethdev_dp.h +index c1f99a2616..67f40b8e25 100644 +--- a/dpdk/drivers/net/cnxk/cnxk_ethdev_dp.h ++++ b/dpdk/drivers/net/cnxk/cnxk_ethdev_dp.h +@@ -84,7 +84,7 @@ struct cnxk_timesync_info { + + /* Inlines */ + static __rte_always_inline uint64_t +-cnxk_pktmbuf_detach(struct rte_mbuf *m) ++cnxk_pktmbuf_detach(struct rte_mbuf *m, uint64_t *aura) + { + struct rte_mempool *mp = m->pool; + uint32_t mbuf_size, buf_len; +@@ -94,6 +94,8 @@ cnxk_pktmbuf_detach(struct rte_mbuf *m) + + /* Update refcount of direct mbuf */ + md = rte_mbuf_from_indirect(m); ++ if (aura) ++ *aura = roc_npa_aura_handle_to_aura(md->pool->pool_id); + refcount = rte_mbuf_refcnt_update(md, -1); + + priv_size = rte_pktmbuf_priv_size(mp); +@@ -126,18 +128,18 @@ cnxk_pktmbuf_detach(struct rte_mbuf *m) + } + + static __rte_always_inline uint64_t +-cnxk_nix_prefree_seg(struct rte_mbuf *m) ++cnxk_nix_prefree_seg(struct rte_mbuf *m, uint64_t *aura) + { + if (likely(rte_mbuf_refcnt_read(m) == 1)) { + if (!RTE_MBUF_DIRECT(m)) +- return cnxk_pktmbuf_detach(m); ++ return cnxk_pktmbuf_detach(m, aura); + + m->next = NULL; + m->nb_segs = 1; + return 0; + } else if (rte_mbuf_refcnt_update(m, -1) == 0) { + if (!RTE_MBUF_DIRECT(m)) +- return cnxk_pktmbuf_detach(m); ++ return cnxk_pktmbuf_detach(m, aura); + + rte_mbuf_refcnt_set(m, 1); + m->next = NULL; +diff --git a/dpdk/drivers/net/cnxk/cnxk_ethdev_ops.c b/dpdk/drivers/net/cnxk/cnxk_ethdev_ops.c +index 5de2919047..c8f4848f92 100644 +--- a/dpdk/drivers/net/cnxk/cnxk_ethdev_ops.c ++++ b/dpdk/drivers/net/cnxk/cnxk_ethdev_ops.c +@@ -20,8 +20,7 @@ cnxk_nix_info_get(struct rte_eth_dev *eth_dev, struct rte_eth_dev_info *devinfo) + devinfo->max_tx_queues = RTE_MAX_QUEUES_PER_PORT; + devinfo->max_mac_addrs = dev->max_mac_entries; + devinfo->max_vfs = pci_dev->max_vfs; +- devinfo->max_mtu = devinfo->max_rx_pktlen - +- (RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN); ++ devinfo->max_mtu = devinfo->max_rx_pktlen - CNXK_NIX_L2_OVERHEAD; + devinfo->min_mtu = devinfo->min_rx_bufsize - CNXK_NIX_L2_OVERHEAD; + + devinfo->rx_offload_capa = dev->rx_offload_capa; +@@ -448,6 +447,13 @@ cnxk_nix_mac_addr_set(struct rte_eth_dev *eth_dev, struct rte_ether_addr *addr) + roc_nix_npc_mac_addr_set(nix, dev->mac_addr); + goto exit; + } ++ ++ if (eth_dev->data->promiscuous) { ++ rc = roc_nix_mac_promisc_mode_enable(nix, true); ++ if (rc) ++ plt_err("Failed to setup promisc mode in mac, rc=%d(%s)", rc, ++ roc_error_msg_get(rc)); ++ } + } + + /* Update mac address to cnxk ethernet device */ +@@ -544,8 +550,9 @@ cnxk_nix_mtu_set(struct rte_eth_dev *eth_dev, uint16_t mtu) + struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); + struct rte_eth_dev_data *data = eth_dev->data; + struct roc_nix *nix = &dev->nix; ++ struct cnxk_eth_rxq_sp *rxq_sp; ++ uint32_t buffsz = 0; + int rc = -EINVAL; +- uint32_t buffsz; + + frame_size += CNXK_NIX_TIMESYNC_RX_OFFSET * dev->ptp_en; + +@@ -561,8 +568,24 @@ cnxk_nix_mtu_set(struct rte_eth_dev *eth_dev, uint16_t mtu) + goto exit; + } + +- buffsz = data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM; +- old_frame_size = data->mtu + CNXK_NIX_L2_OVERHEAD; ++ if (!eth_dev->data->nb_rx_queues) ++ goto skip_buffsz_check; ++ ++ /* Perform buff size check */ ++ if (data->min_rx_buf_size) { ++ buffsz = data->min_rx_buf_size; ++ } else if (eth_dev->data->rx_queues && eth_dev->data->rx_queues[0]) { ++ rxq_sp = cnxk_eth_rxq_to_sp(data->rx_queues[0]); ++ ++ if (rxq_sp->qconf.mp) ++ buffsz = rte_pktmbuf_data_room_size(rxq_sp->qconf.mp); ++ } ++ ++ /* Skip validation if RQ's are not yet setup */ ++ if (!buffsz) ++ goto skip_buffsz_check; ++ ++ buffsz -= RTE_PKTMBUF_HEADROOM; + + /* Refuse MTU that requires the support of scattered packets + * when this feature has not been enabled before. +@@ -580,6 +603,8 @@ cnxk_nix_mtu_set(struct rte_eth_dev *eth_dev, uint16_t mtu) + goto exit; + } + ++skip_buffsz_check: ++ old_frame_size = data->mtu + CNXK_NIX_L2_OVERHEAD; + /* if new MTU was smaller than old one, then flush all SQs before MTU change */ + if (old_frame_size > frame_size) { + if (data->dev_started) { +@@ -591,19 +616,9 @@ cnxk_nix_mtu_set(struct rte_eth_dev *eth_dev, uint16_t mtu) + + frame_size -= RTE_ETHER_CRC_LEN; + +- /* Update mtu on Tx */ +- rc = roc_nix_mac_mtu_set(nix, frame_size); +- if (rc) { +- plt_err("Failed to set MTU, rc=%d", rc); +- goto exit; +- } +- +- /* Sync same frame size on Rx */ ++ /* Set frame size on Rx */ + rc = roc_nix_mac_max_rx_len_set(nix, frame_size); + if (rc) { +- /* Rollback to older mtu */ +- roc_nix_mac_mtu_set(nix, +- old_frame_size - RTE_ETHER_CRC_LEN); + plt_err("Failed to max Rx frame length, rc=%d", rc); + goto exit; + } +diff --git a/dpdk/drivers/net/cnxk/cnxk_flow.c b/dpdk/drivers/net/cnxk/cnxk_flow.c +index 08ab75e2bb..be0330fa04 100644 +--- a/dpdk/drivers/net/cnxk/cnxk_flow.c ++++ b/dpdk/drivers/net/cnxk/cnxk_flow.c +@@ -102,15 +102,19 @@ npc_rss_action_validate(struct rte_eth_dev *eth_dev, + } + + static void +-npc_rss_flowkey_get(struct cnxk_eth_dev *eth_dev, +- const struct roc_npc_action *rss_action, +- uint32_t *flowkey_cfg) ++npc_rss_flowkey_get(struct cnxk_eth_dev *eth_dev, const struct roc_npc_action *rss_action, ++ uint32_t *flowkey_cfg, uint64_t default_rss_types) + { + const struct roc_npc_action_rss *rss; ++ uint64_t rss_types; + + rss = (const struct roc_npc_action_rss *)rss_action->conf; ++ rss_types = rss->types; ++ /* If no RSS types are specified, use default one */ ++ if (rss_types == 0) ++ rss_types = default_rss_types; + +- *flowkey_cfg = cnxk_rss_ethdev_to_nix(eth_dev, rss->types, rss->level); ++ *flowkey_cfg = cnxk_rss_ethdev_to_nix(eth_dev, rss_types, rss->level); + } + + static int +@@ -204,7 +208,8 @@ cnxk_map_actions(struct rte_eth_dev *eth_dev, const struct rte_flow_attr *attr, + goto err_exit; + in_actions[i].type = ROC_NPC_ACTION_TYPE_RSS; + in_actions[i].conf = actions->conf; +- npc_rss_flowkey_get(dev, &in_actions[i], flowkey_cfg); ++ npc_rss_flowkey_get(dev, &in_actions[i], flowkey_cfg, ++ eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf); + break; + + case RTE_FLOW_ACTION_TYPE_SECURITY: +@@ -503,6 +508,9 @@ cnxk_flow_get_aged_flows(struct rte_eth_dev *eth_dev, void **context, + + flow_age = &roc_npc->flow_age; + ++ if (!flow_age->age_flow_refcnt) ++ return 0; ++ + do { + sn = plt_seqcount_read_begin(&flow_age->seq_cnt); + +diff --git a/dpdk/drivers/net/cpfl/cpfl_flow_engine_fxp.c b/dpdk/drivers/net/cpfl/cpfl_flow_engine_fxp.c +index 8a4e1419b4..f6bd1f7599 100644 +--- a/dpdk/drivers/net/cpfl/cpfl_flow_engine_fxp.c ++++ b/dpdk/drivers/net/cpfl/cpfl_flow_engine_fxp.c +@@ -95,7 +95,7 @@ cpfl_fxp_create(struct rte_eth_dev *dev, + + ret = cpfl_rule_process(itf, ad->ctlqp[cpq_id], ad->ctlqp[cpq_id + 1], + rim->rules, rim->rule_num, true); +- if (ret < 0) { ++ if (ret != 0) { + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "cpfl filter create flow fail"); + rte_free(rim); +diff --git a/dpdk/drivers/net/cpfl/cpfl_flow_parser.c b/dpdk/drivers/net/cpfl/cpfl_flow_parser.c +index a8f0488f21..40569ddc6f 100644 +--- a/dpdk/drivers/net/cpfl/cpfl_flow_parser.c ++++ b/dpdk/drivers/net/cpfl/cpfl_flow_parser.c +@@ -1696,7 +1696,7 @@ cpfl_parse_check_prog_action(struct cpfl_flow_js_mr_key_action *key_act, + bool check_name; + + check_name = key_act->prog.has_name ? strcmp(prog->name, key_act->prog.name) == 0 +- : atol(prog->name) == key_act->prog.id; ++ : (uint32_t)atol(prog->name) == key_act->prog.id; + if (!check_name) { + PMD_DRV_LOG(ERR, "Not support this prog type: %s.", prog->name); + return -EINVAL; +diff --git a/dpdk/drivers/net/cpfl/cpfl_fxp_rule.c b/dpdk/drivers/net/cpfl/cpfl_fxp_rule.c +index 0e710a007b..be34da9fa2 100644 +--- a/dpdk/drivers/net/cpfl/cpfl_fxp_rule.c ++++ b/dpdk/drivers/net/cpfl/cpfl_fxp_rule.c +@@ -92,6 +92,14 @@ cpfl_receive_ctlq_msg(struct idpf_hw *hw, struct idpf_ctlq_info *cq, u16 num_q_m + + /* TODO - process rx controlq message */ + for (i = 0; i < num_q_msg; i++) { ++ ret = q_msg[i].status; ++ if (ret != CPFL_CFG_PKT_ERR_OK && ++ q_msg[i].opcode != cpfl_ctlq_sem_query_del_rule_hash_addr) { ++ PMD_INIT_LOG(ERR, "Failed to process rx_ctrlq msg: %s", ++ cpfl_cfg_pkt_errormsg[ret]); ++ return ret; ++ } ++ + if (q_msg[i].data_len > 0) + dma = q_msg[i].ctx.indirect.payload; + else +diff --git a/dpdk/drivers/net/cpfl/cpfl_rules.h b/dpdk/drivers/net/cpfl/cpfl_rules.h +index d23eae8e91..10569b1fdc 100644 +--- a/dpdk/drivers/net/cpfl/cpfl_rules.h ++++ b/dpdk/drivers/net/cpfl/cpfl_rules.h +@@ -62,6 +62,17 @@ enum cpfl_cfg_pkt_error_code { + CPFL_CFG_PKT_ERR_EMAXCOL = 9 /* Max Hash Collision */ + }; + ++static const char * const cpfl_cfg_pkt_errormsg[] = { ++ [CPFL_CFG_PKT_ERR_ESRCH] = "Bad opcode", ++ [CPFL_CFG_PKT_ERR_EEXIST] = "The rule conflicts with already existed one", ++ [CPFL_CFG_PKT_ERR_ENOSPC] = "No space left in the table", ++ [CPFL_CFG_PKT_ERR_ERANGE] = "Parameter out of range", ++ [CPFL_CFG_PKT_ERR_ESBCOMP] = "Completion error", ++ [CPFL_CFG_PKT_ERR_ENOPIN] = "Entry cannot be pinned in cache", ++ [CPFL_CFG_PKT_ERR_ENOTFND] = "Entry does not exist", ++ [CPFL_CFG_PKT_ERR_EMAXCOL] = "Maximum Hash Collisions reached", ++}; ++ + /* macros for creating context for rule descriptor */ + #define MEV_RULE_VSI_ID_S 0 + #define MEV_RULE_VSI_ID_M \ +diff --git a/dpdk/drivers/net/cxgbe/cxgbe_ethdev.c b/dpdk/drivers/net/cxgbe/cxgbe_ethdev.c +index 8cc3d9f257..781f48cfac 100644 +--- a/dpdk/drivers/net/cxgbe/cxgbe_ethdev.c ++++ b/dpdk/drivers/net/cxgbe/cxgbe_ethdev.c +@@ -211,9 +211,9 @@ int cxgbe_dev_link_update(struct rte_eth_dev *eth_dev, + unsigned int i, work_done, budget = 32; + struct link_config *lc = &pi->link_cfg; + struct adapter *adapter = pi->adapter; +- struct rte_eth_link new_link = { 0 }; + u8 old_link = pi->link_cfg.link_ok; + struct sge *s = &adapter->sge; ++ struct rte_eth_link new_link; + + for (i = 0; i < CXGBE_LINK_STATUS_POLL_CNT; i++) { + if (!s->fw_evtq.desc) +@@ -232,6 +232,7 @@ int cxgbe_dev_link_update(struct rte_eth_dev *eth_dev, + rte_delay_ms(CXGBE_LINK_STATUS_POLL_MS); + } + ++ memset(&new_link, 0, sizeof(new_link)); + new_link.link_status = cxgbe_force_linkup(adapter) ? + RTE_ETH_LINK_UP : pi->link_cfg.link_ok; + new_link.link_autoneg = (lc->link_caps & FW_PORT_CAP32_ANEG) ? 1 : 0; +diff --git a/dpdk/drivers/net/dpaa/dpaa_ethdev.c b/dpdk/drivers/net/dpaa/dpaa_ethdev.c +index ef4c06db6a..bcb28f33ee 100644 +--- a/dpdk/drivers/net/dpaa/dpaa_ethdev.c ++++ b/dpdk/drivers/net/dpaa/dpaa_ethdev.c +@@ -14,6 +14,7 @@ + #include + #include + #include ++#include + + #include + #include +@@ -165,9 +166,15 @@ dpaa_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) + uint32_t frame_size = mtu + RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN + + VLAN_TAG_SIZE; + uint32_t buffsz = dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM; ++ struct fman_if *fif = dev->process_private; + + PMD_INIT_FUNC_TRACE(); + ++ if (fif->is_shared_mac) { ++ DPAA_PMD_ERR("Cannot configure mtu from DPDK in VSP mode."); ++ return -ENOTSUP; ++ } ++ + /* + * Refuse mtu that requires the support of scattered packets + * when this feature has not been enabled before. +@@ -206,7 +213,8 @@ dpaa_eth_dev_configure(struct rte_eth_dev *dev) + struct rte_intr_handle *intr_handle; + uint32_t max_rx_pktlen; + int speed, duplex; +- int ret, rx_status; ++ int ret, rx_status, socket_fd; ++ struct ifreq ifr; + + PMD_INIT_FUNC_TRACE(); + +@@ -222,6 +230,26 @@ dpaa_eth_dev_configure(struct rte_eth_dev *dev) + dpaa_intf->name); + return -EHOSTDOWN; + } ++ ++ socket_fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_IP); ++ if (socket_fd == -1) { ++ DPAA_PMD_ERR("Cannot open IF socket"); ++ return -errno; ++ } ++ ++ strncpy(ifr.ifr_name, dpaa_intf->name, IFNAMSIZ - 1); ++ ++ if (ioctl(socket_fd, SIOCGIFMTU, &ifr) < 0) { ++ DPAA_PMD_ERR("Cannot get interface mtu"); ++ close(socket_fd); ++ return -errno; ++ } ++ ++ close(socket_fd); ++ DPAA_PMD_INFO("Using kernel configured mtu size(%u)", ++ ifr.ifr_mtu); ++ ++ eth_conf->rxmode.mtu = ifr.ifr_mtu; + } + + /* Rx offloads which are enabled by default */ +@@ -249,7 +277,8 @@ dpaa_eth_dev_configure(struct rte_eth_dev *dev) + max_rx_pktlen = DPAA_MAX_RX_PKT_LEN; + } + +- fman_if_set_maxfrm(dev->process_private, max_rx_pktlen); ++ if (!fif->is_shared_mac) ++ fman_if_set_maxfrm(dev->process_private, max_rx_pktlen); + + if (rx_offloads & RTE_ETH_RX_OFFLOAD_SCATTER) { + DPAA_PMD_DEBUG("enabling scatter mode"); +@@ -363,7 +392,8 @@ dpaa_supported_ptypes_get(struct rte_eth_dev *dev) + RTE_PTYPE_L4_TCP, + RTE_PTYPE_L4_UDP, + RTE_PTYPE_L4_SCTP, +- RTE_PTYPE_TUNNEL_ESP ++ RTE_PTYPE_TUNNEL_ESP, ++ RTE_PTYPE_UNKNOWN + }; + + PMD_INIT_FUNC_TRACE(); +diff --git a/dpdk/drivers/net/e1000/base/e1000_base.c b/dpdk/drivers/net/e1000/base/e1000_base.c +index ab73e1e59e..3ec32e7240 100644 +--- a/dpdk/drivers/net/e1000/base/e1000_base.c ++++ b/dpdk/drivers/net/e1000/base/e1000_base.c +@@ -107,7 +107,7 @@ void e1000_power_down_phy_copper_base(struct e1000_hw *hw) + return; + + /* If the management interface is not enabled, then power down */ +- if (phy->ops.check_reset_block(hw)) ++ if (!(e1000_enable_mng_pass_thru(hw) || phy->ops.check_reset_block(hw))) + e1000_power_down_phy_copper(hw); + } + +diff --git a/dpdk/drivers/net/ena/base/ena_com.c b/dpdk/drivers/net/ena/base/ena_com.c +index 6953a1fa33..2f438597e6 100644 +--- a/dpdk/drivers/net/ena/base/ena_com.c ++++ b/dpdk/drivers/net/ena/base/ena_com.c +@@ -34,6 +34,8 @@ + + #define ENA_REGS_ADMIN_INTR_MASK 1 + ++#define ENA_MAX_BACKOFF_DELAY_EXP 16U ++ + #define ENA_MIN_ADMIN_POLL_US 100 + + #define ENA_MAX_ADMIN_POLL_US 5000 +@@ -177,6 +179,7 @@ static int ena_com_admin_init_aenq(struct ena_com_dev *ena_dev, + static void comp_ctxt_release(struct ena_com_admin_queue *queue, + struct ena_comp_ctx *comp_ctx) + { ++ comp_ctx->user_cqe = NULL; + comp_ctx->occupied = false; + ATOMIC32_DEC(&queue->outstanding_cmds); + } +@@ -470,6 +473,9 @@ static void ena_com_handle_single_admin_completion(struct ena_com_admin_queue *a + return; + } + ++ if (!comp_ctx->occupied) ++ return; ++ + comp_ctx->status = ENA_CMD_COMPLETED; + comp_ctx->comp_status = cqe->acq_common_descriptor.status; + +@@ -545,8 +551,9 @@ static int ena_com_comp_status_to_errno(struct ena_com_admin_queue *admin_queue, + + static void ena_delay_exponential_backoff_us(u32 exp, u32 delay_us) + { ++ exp = ENA_MIN32(ENA_MAX_BACKOFF_DELAY_EXP, exp); + delay_us = ENA_MAX32(ENA_MIN_ADMIN_POLL_US, delay_us); +- delay_us = ENA_MIN32(delay_us * (1U << exp), ENA_MAX_ADMIN_POLL_US); ++ delay_us = ENA_MIN32(ENA_MAX_ADMIN_POLL_US, delay_us * (1U << exp)); + ENA_USLEEP(delay_us); + } + +@@ -3134,16 +3141,18 @@ int ena_com_allocate_debug_area(struct ena_com_dev *ena_dev, + int ena_com_allocate_customer_metrics_buffer(struct ena_com_dev *ena_dev) + { + struct ena_customer_metrics *customer_metrics = &ena_dev->customer_metrics; ++ customer_metrics->buffer_len = ENA_CUSTOMER_METRICS_BUFFER_SIZE; ++ customer_metrics->buffer_virt_addr = NULL; + + ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, + customer_metrics->buffer_len, + customer_metrics->buffer_virt_addr, + customer_metrics->buffer_dma_addr, + customer_metrics->buffer_dma_handle); +- if (unlikely(customer_metrics->buffer_virt_addr == NULL)) ++ if (unlikely(customer_metrics->buffer_virt_addr == NULL)) { ++ customer_metrics->buffer_len = 0; + return ENA_COM_NO_MEM; +- +- customer_metrics->buffer_len = ENA_CUSTOMER_METRICS_BUFFER_SIZE; ++ } + + return 0; + } +diff --git a/dpdk/drivers/net/ena/ena_ethdev.c b/dpdk/drivers/net/ena/ena_ethdev.c +index dc846d2e84..f3962aa76e 100644 +--- a/dpdk/drivers/net/ena/ena_ethdev.c ++++ b/dpdk/drivers/net/ena/ena_ethdev.c +@@ -37,10 +37,10 @@ + #define ENA_MIN_RING_DESC 128 + + /* +- * We should try to keep ENA_CLEANUP_BUF_SIZE lower than ++ * We should try to keep ENA_CLEANUP_BUF_THRESH lower than + * RTE_MEMPOOL_CACHE_MAX_SIZE, so we can fit this in mempool local cache. + */ +-#define ENA_CLEANUP_BUF_SIZE 256 ++#define ENA_CLEANUP_BUF_THRESH 256 + + #define ENA_PTYPE_HAS_HASH (RTE_PTYPE_L4_TCP | RTE_PTYPE_L4_UDP) + +@@ -648,18 +648,13 @@ static inline void ena_rx_mbuf_prepare(struct ena_ring *rx_ring, + packet_type |= RTE_PTYPE_L3_IPV6; + } + +- if (!ena_rx_ctx->l4_csum_checked || ena_rx_ctx->frag) { ++ if (!ena_rx_ctx->l4_csum_checked || ena_rx_ctx->frag || ++ !(packet_type & (RTE_PTYPE_L4_TCP | RTE_PTYPE_L4_UDP))) { + ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_UNKNOWN; + } else { + if (unlikely(ena_rx_ctx->l4_csum_err)) { + ++rx_stats->l4_csum_bad; +- /* +- * For the L4 Rx checksum offload the HW may indicate +- * bad checksum although it's valid. Because of that, +- * we're setting the UNKNOWN flag to let the app +- * re-verify the checksum. +- */ +- ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_UNKNOWN; ++ ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_BAD; + } else { + ++rx_stats->l4_csum_good; + ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_GOOD; +@@ -797,7 +792,7 @@ static void ena_config_host_info(struct ena_com_dev *ena_dev) + + rc = ena_com_set_host_attributes(ena_dev); + if (rc) { +- if (rc == -ENA_COM_UNSUPPORTED) ++ if (rc == ENA_COM_UNSUPPORTED) + PMD_DRV_LOG(WARNING, "Cannot set host attributes\n"); + else + PMD_DRV_LOG(ERR, "Cannot set host attributes\n"); +@@ -841,7 +836,7 @@ static void ena_config_debug_area(struct ena_adapter *adapter) + + rc = ena_com_set_host_attributes(&adapter->ena_dev); + if (rc) { +- if (rc == -ENA_COM_UNSUPPORTED) ++ if (rc == ENA_COM_UNSUPPORTED) + PMD_DRV_LOG(WARNING, "Cannot set host attributes\n"); + else + PMD_DRV_LOG(ERR, "Cannot set host attributes\n"); +@@ -3105,33 +3100,12 @@ static int ena_xmit_mbuf(struct ena_ring *tx_ring, struct rte_mbuf *mbuf) + return 0; + } + +-static __rte_always_inline size_t +-ena_tx_cleanup_mbuf_fast(struct rte_mbuf **mbufs_to_clean, +- struct rte_mbuf *mbuf, +- size_t mbuf_cnt, +- size_t buf_size) +-{ +- struct rte_mbuf *m_next; +- +- while (mbuf != NULL) { +- m_next = mbuf->next; +- mbufs_to_clean[mbuf_cnt++] = mbuf; +- if (mbuf_cnt == buf_size) { +- rte_mempool_put_bulk(mbufs_to_clean[0]->pool, (void **)mbufs_to_clean, +- (unsigned int)mbuf_cnt); +- mbuf_cnt = 0; +- } +- mbuf = m_next; +- } +- +- return mbuf_cnt; +-} +- + static int ena_tx_cleanup(void *txp, uint32_t free_pkt_cnt) + { +- struct rte_mbuf *mbufs_to_clean[ENA_CLEANUP_BUF_SIZE]; ++ struct rte_mbuf *pkts_to_clean[ENA_CLEANUP_BUF_THRESH]; + struct ena_ring *tx_ring = (struct ena_ring *)txp; + size_t mbuf_cnt = 0; ++ size_t pkt_cnt = 0; + unsigned int total_tx_descs = 0; + unsigned int total_tx_pkts = 0; + uint16_t cleanup_budget; +@@ -3162,8 +3136,13 @@ static int ena_tx_cleanup(void *txp, uint32_t free_pkt_cnt) + + mbuf = tx_info->mbuf; + if (fast_free) { +- mbuf_cnt = ena_tx_cleanup_mbuf_fast(mbufs_to_clean, mbuf, mbuf_cnt, +- ENA_CLEANUP_BUF_SIZE); ++ pkts_to_clean[pkt_cnt++] = mbuf; ++ mbuf_cnt += mbuf->nb_segs; ++ if (mbuf_cnt >= ENA_CLEANUP_BUF_THRESH) { ++ rte_pktmbuf_free_bulk(pkts_to_clean, pkt_cnt); ++ mbuf_cnt = 0; ++ pkt_cnt = 0; ++ } + } else { + rte_pktmbuf_free(mbuf); + } +@@ -3186,8 +3165,7 @@ static int ena_tx_cleanup(void *txp, uint32_t free_pkt_cnt) + } + + if (mbuf_cnt != 0) +- rte_mempool_put_bulk(mbufs_to_clean[0]->pool, +- (void **)mbufs_to_clean, mbuf_cnt); ++ rte_pktmbuf_free_bulk(pkts_to_clean, pkt_cnt); + + /* Notify completion handler that full cleanup was performed */ + if (free_pkt_cnt == 0 || total_tx_pkts < cleanup_budget) +diff --git a/dpdk/drivers/net/failsafe/failsafe_args.c b/dpdk/drivers/net/failsafe/failsafe_args.c +index b203e02d9a..3b867437d7 100644 +--- a/dpdk/drivers/net/failsafe/failsafe_args.c ++++ b/dpdk/drivers/net/failsafe/failsafe_args.c +@@ -248,7 +248,7 @@ fs_parse_device_param(struct rte_eth_dev *dev, const char *param, + goto free_args; + } else { + ERROR("Unrecognized device type: %.*s", (int)b, param); +- return -EINVAL; ++ ret = -EINVAL; + } + free_args: + free(args); +diff --git a/dpdk/drivers/net/fm10k/fm10k_ethdev.c b/dpdk/drivers/net/fm10k/fm10k_ethdev.c +index 4d3c4c10cf..cc2012786d 100644 +--- a/dpdk/drivers/net/fm10k/fm10k_ethdev.c ++++ b/dpdk/drivers/net/fm10k/fm10k_ethdev.c +@@ -3057,7 +3057,7 @@ eth_fm10k_dev_init(struct rte_eth_dev *dev) + struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = pdev->intr_handle; +- int diag, i; ++ int diag, i, ret; + struct fm10k_macvlan_filter_info *macvlan; + + PMD_INIT_FUNC_TRACE(); +@@ -3146,21 +3146,24 @@ eth_fm10k_dev_init(struct rte_eth_dev *dev) + diag = fm10k_stats_reset(dev); + if (diag != 0) { + PMD_INIT_LOG(ERR, "Stats reset failed: %d", diag); +- return diag; ++ ret = diag; ++ goto err_stat; + } + + /* Reset the hw */ + diag = fm10k_reset_hw(hw); + if (diag != FM10K_SUCCESS) { + PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag); +- return -EIO; ++ ret = -EIO; ++ goto err_reset_hw; + } + + /* Setup mailbox service */ + diag = fm10k_setup_mbx_service(hw); + if (diag != FM10K_SUCCESS) { + PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag); +- return -EIO; ++ ret = -EIO; ++ goto err_mbx; + } + + /*PF/VF has different interrupt handling mechanism */ +@@ -3199,7 +3202,8 @@ eth_fm10k_dev_init(struct rte_eth_dev *dev) + + if (switch_ready == false) { + PMD_INIT_LOG(ERR, "switch is not ready"); +- return -1; ++ ret = -1; ++ goto err_switch_ready; + } + } + +@@ -3234,7 +3238,8 @@ eth_fm10k_dev_init(struct rte_eth_dev *dev) + + if (!hw->mac.default_vid) { + PMD_INIT_LOG(ERR, "default VID is not ready"); +- return -1; ++ ret = -1; ++ goto err_vid; + } + } + +@@ -3243,6 +3248,28 @@ eth_fm10k_dev_init(struct rte_eth_dev *dev) + MAIN_VSI_POOL_NUMBER); + + return 0; ++ ++err_vid: ++err_switch_ready: ++ rte_intr_disable(intr_handle); ++ ++ if (hw->mac.type == fm10k_mac_pf) { ++ fm10k_dev_disable_intr_pf(dev); ++ rte_intr_callback_unregister(intr_handle, ++ fm10k_dev_interrupt_handler_pf, (void *)dev); ++ } else { ++ fm10k_dev_disable_intr_vf(dev); ++ rte_intr_callback_unregister(intr_handle, ++ fm10k_dev_interrupt_handler_vf, (void *)dev); ++ } ++ ++err_mbx: ++err_reset_hw: ++err_stat: ++ rte_free(dev->data->mac_addrs); ++ dev->data->mac_addrs = NULL; ++ ++ return ret; + } + + static int +diff --git a/dpdk/drivers/net/gve/gve_rx_dqo.c b/dpdk/drivers/net/gve/gve_rx_dqo.c +index 7c7a8c48d0..a56cdbf11b 100644 +--- a/dpdk/drivers/net/gve/gve_rx_dqo.c ++++ b/dpdk/drivers/net/gve/gve_rx_dqo.c +@@ -127,7 +127,7 @@ gve_rx_burst_dqo(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) + rxm->ol_flags = 0; + + rxm->ol_flags |= RTE_MBUF_F_RX_RSS_HASH; +- rxm->hash.rss = rte_be_to_cpu_32(rx_desc->hash); ++ rxm->hash.rss = rte_le_to_cpu_32(rx_desc->hash); + + rx_pkts[nb_rx++] = rxm; + bytes += pkt_len; +diff --git a/dpdk/drivers/net/gve/gve_tx.c b/dpdk/drivers/net/gve/gve_tx.c +index 2e0d001109..bb21b90635 100644 +--- a/dpdk/drivers/net/gve/gve_tx.c ++++ b/dpdk/drivers/net/gve/gve_tx.c +@@ -681,7 +681,7 @@ gve_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id) + + rte_write32(rte_cpu_to_be_32(GVE_IRQ_MASK), txq->ntfy_addr); + +- dev->data->rx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; ++ dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; + + return 0; + } +diff --git a/dpdk/drivers/net/gve/gve_tx_dqo.c b/dpdk/drivers/net/gve/gve_tx_dqo.c +index 16101de84f..97d9c6549b 100644 +--- a/dpdk/drivers/net/gve/gve_tx_dqo.c ++++ b/dpdk/drivers/net/gve/gve_tx_dqo.c +@@ -13,7 +13,7 @@ gve_tx_clean_dqo(struct gve_tx_queue *txq) + struct gve_tx_compl_desc *compl_desc; + struct gve_tx_queue *aim_txq; + uint16_t nb_desc_clean; +- struct rte_mbuf *txe; ++ struct rte_mbuf *txe, *txe_next; + uint16_t compl_tag; + uint16_t next; + +@@ -43,10 +43,15 @@ gve_tx_clean_dqo(struct gve_tx_queue *txq) + PMD_DRV_LOG(DEBUG, "GVE_COMPL_TYPE_DQO_REINJECTION !!!"); + /* FALLTHROUGH */ + case GVE_COMPL_TYPE_DQO_PKT: ++ /* free all segments. */ + txe = aim_txq->sw_ring[compl_tag]; +- if (txe != NULL) { ++ while (txe != NULL) { ++ txe_next = txe->next; + rte_pktmbuf_free_seg(txe); +- txe = NULL; ++ if (aim_txq->sw_ring[compl_tag] == txe) ++ aim_txq->sw_ring[compl_tag] = NULL; ++ txe = txe_next; ++ compl_tag = (compl_tag + 1) & (aim_txq->sw_size - 1); + } + break; + case GVE_COMPL_TYPE_DQO_MISS: +@@ -83,6 +88,7 @@ gve_tx_burst_dqo(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) + uint16_t tx_id; + uint16_t sw_id; + uint64_t bytes; ++ uint16_t first_sw_id; + + sw_ring = txq->sw_ring; + txr = txq->tx_ring; +@@ -107,23 +113,25 @@ gve_tx_burst_dqo(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) + + ol_flags = tx_pkt->ol_flags; + nb_used = tx_pkt->nb_segs; +- ++ first_sw_id = sw_id; + do { +- txd = &txr[tx_id]; ++ if (sw_ring[sw_id] != NULL) ++ PMD_DRV_LOG(DEBUG, "Overwriting an entry in sw_ring"); + ++ txd = &txr[tx_id]; + sw_ring[sw_id] = tx_pkt; + + /* fill Tx descriptor */ + txd->pkt.buf_addr = rte_cpu_to_le_64(rte_mbuf_data_iova(tx_pkt)); + txd->pkt.dtype = GVE_TX_PKT_DESC_DTYPE_DQO; +- txd->pkt.compl_tag = rte_cpu_to_le_16(sw_id); ++ txd->pkt.compl_tag = rte_cpu_to_le_16(first_sw_id); + txd->pkt.buf_size = RTE_MIN(tx_pkt->data_len, GVE_TX_MAX_BUF_SIZE_DQO); + + /* size of desc_ring and sw_ring could be different */ + tx_id = (tx_id + 1) & mask; + sw_id = (sw_id + 1) & sw_mask; + +- bytes += tx_pkt->pkt_len; ++ bytes += tx_pkt->data_len; + tx_pkt = tx_pkt->next; + } while (tx_pkt); + +@@ -384,7 +392,7 @@ gve_tx_queue_start_dqo(struct rte_eth_dev *dev, uint16_t tx_queue_id) + + rte_write32(rte_cpu_to_be_32(GVE_IRQ_MASK), txq->ntfy_addr); + +- dev->data->rx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; ++ dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; + + return 0; + } +diff --git a/dpdk/drivers/net/hns3/hns3_cmd.c b/dpdk/drivers/net/hns3/hns3_cmd.c +index 2c1664485b..001ff49b36 100644 +--- a/dpdk/drivers/net/hns3/hns3_cmd.c ++++ b/dpdk/drivers/net/hns3/hns3_cmd.c +@@ -545,7 +545,9 @@ hns3_set_dcb_capability(struct hns3_hw *hw) + if (device_id == HNS3_DEV_ID_25GE_RDMA || + device_id == HNS3_DEV_ID_50GE_RDMA || + device_id == HNS3_DEV_ID_100G_RDMA_MACSEC || +- device_id == HNS3_DEV_ID_200G_RDMA) ++ device_id == HNS3_DEV_ID_200G_RDMA || ++ device_id == HNS3_DEV_ID_100G_ROH || ++ device_id == HNS3_DEV_ID_200G_ROH) + hns3_set_bit(hw->capability, HNS3_DEV_SUPPORT_DCB_B, 1); + } + +diff --git a/dpdk/drivers/net/hns3/hns3_common.c b/dpdk/drivers/net/hns3/hns3_common.c +index 8f224aa00c..5e6cdfdaa0 100644 +--- a/dpdk/drivers/net/hns3/hns3_common.c ++++ b/dpdk/drivers/net/hns3/hns3_common.c +@@ -85,7 +85,7 @@ hns3_dev_infos_get(struct rte_eth_dev *eth_dev, struct rte_eth_dev_info *info) + RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE | + RTE_ETH_TX_OFFLOAD_VLAN_INSERT); + +- if (!hw->port_base_vlan_cfg.state) ++ if (!hns->is_vf && !hw->port_base_vlan_cfg.state) + info->tx_offload_capa |= RTE_ETH_TX_OFFLOAD_QINQ_INSERT; + + if (hns3_dev_get_support(hw, OUTER_UDP_CKSUM)) +@@ -224,7 +224,7 @@ hns3_parse_dev_caps_mask(const char *key, const char *value, void *extra_args) + static int + hns3_parse_mbx_time_limit(const char *key, const char *value, void *extra_args) + { +- uint32_t val; ++ uint64_t val; + + RTE_SET_USED(key); + +diff --git a/dpdk/drivers/net/hns3/hns3_dcb.c b/dpdk/drivers/net/hns3/hns3_dcb.c +index 2831d3dc62..915e4eb768 100644 +--- a/dpdk/drivers/net/hns3/hns3_dcb.c ++++ b/dpdk/drivers/net/hns3/hns3_dcb.c +@@ -1499,7 +1499,6 @@ hns3_dcb_info_update(struct hns3_adapter *hns, uint8_t num_tc) + static int + hns3_dcb_hw_configure(struct hns3_adapter *hns) + { +- struct rte_eth_dcb_rx_conf *dcb_rx_conf; + struct hns3_pf *pf = &hns->pf; + struct hns3_hw *hw = &hns->hw; + enum hns3_fc_status fc_status = hw->current_fc_status; +@@ -1519,12 +1518,8 @@ hns3_dcb_hw_configure(struct hns3_adapter *hns) + } + + if (hw->data->dev_conf.dcb_capability_en & RTE_ETH_DCB_PFC_SUPPORT) { +- dcb_rx_conf = &hw->data->dev_conf.rx_adv_conf.dcb_rx_conf; +- if (dcb_rx_conf->nb_tcs == 0) +- hw->dcb_info.pfc_en = 1; /* tc0 only */ +- else +- hw->dcb_info.pfc_en = +- RTE_LEN2MASK((uint8_t)dcb_rx_conf->nb_tcs, uint8_t); ++ hw->dcb_info.pfc_en = ++ RTE_LEN2MASK((uint8_t)HNS3_MAX_USER_PRIO, uint8_t); + + hw->dcb_info.hw_pfc_map = + hns3_dcb_undrop_tc_map(hw, hw->dcb_info.pfc_en); +diff --git a/dpdk/drivers/net/hns3/hns3_ethdev.c b/dpdk/drivers/net/hns3/hns3_ethdev.c +index ae81368f68..6e72730d75 100644 +--- a/dpdk/drivers/net/hns3/hns3_ethdev.c ++++ b/dpdk/drivers/net/hns3/hns3_ethdev.c +@@ -380,7 +380,7 @@ hns3_interrupt_handler(void *param) + hns3_warn(hw, "received reset interrupt"); + hns3_schedule_reset(hns); + } else if (event_cause == HNS3_VECTOR0_EVENT_MBX) { +- hns3_dev_handle_mbx_msg(hw); ++ hns3pf_handle_mbx_msg(hw); + } else if (event_cause != HNS3_VECTOR0_EVENT_PTP) { + hns3_warn(hw, "received unknown event: vector0_int_stat:0x%x " + "ras_int_stat:0x%x cmdq_int_stat:0x%x", +@@ -2738,6 +2738,7 @@ hns3_get_capability(struct hns3_hw *hw) + hw->rss_info.ipv6_sctp_offload_supported = false; + hw->udp_cksum_mode = HNS3_SPECIAL_PORT_SW_CKSUM_MODE; + pf->support_multi_tc_pause = false; ++ hw->rx_dma_addr_align = HNS3_RX_DMA_ADDR_ALIGN_64; + return 0; + } + +@@ -2758,6 +2759,7 @@ hns3_get_capability(struct hns3_hw *hw) + hw->rss_info.ipv6_sctp_offload_supported = true; + hw->udp_cksum_mode = HNS3_SPECIAL_PORT_HW_CKSUM_MODE; + pf->support_multi_tc_pause = true; ++ hw->rx_dma_addr_align = HNS3_RX_DMA_ADDR_ALIGN_128; + + return 0; + } +@@ -5545,28 +5547,14 @@ is_pf_reset_done(struct hns3_hw *hw) + static enum hns3_reset_level + hns3_detect_reset_event(struct hns3_hw *hw) + { +- struct hns3_adapter *hns = HNS3_DEV_HW_TO_ADAPTER(hw); + enum hns3_reset_level new_req = HNS3_NONE_RESET; +- enum hns3_reset_level last_req; + uint32_t vector0_intr_state; + +- last_req = hns3_get_reset_level(hns, &hw->reset.pending); + vector0_intr_state = hns3_read_dev(hw, HNS3_VECTOR0_OTHER_INT_STS_REG); +- if (BIT(HNS3_VECTOR0_IMPRESET_INT_B) & vector0_intr_state) { +- __atomic_store_n(&hw->reset.disable_cmd, 1, __ATOMIC_RELAXED); ++ if (BIT(HNS3_VECTOR0_IMPRESET_INT_B) & vector0_intr_state) + new_req = HNS3_IMP_RESET; +- } else if (BIT(HNS3_VECTOR0_GLOBALRESET_INT_B) & vector0_intr_state) { +- __atomic_store_n(&hw->reset.disable_cmd, 1, __ATOMIC_RELAXED); ++ else if (BIT(HNS3_VECTOR0_GLOBALRESET_INT_B) & vector0_intr_state) + new_req = HNS3_GLOBAL_RESET; +- } +- +- if (new_req == HNS3_NONE_RESET) +- return HNS3_NONE_RESET; +- +- if (last_req == HNS3_NONE_RESET || last_req < new_req) { +- hns3_schedule_delayed_reset(hns); +- hns3_warn(hw, "High level reset detected, delay do reset"); +- } + + return new_req; + } +@@ -5586,10 +5574,14 @@ hns3_is_reset_pending(struct hns3_adapter *hns) + return false; + + new_req = hns3_detect_reset_event(hw); ++ if (new_req == HNS3_NONE_RESET) ++ return false; ++ + last_req = hns3_get_reset_level(hns, &hw->reset.pending); +- if (last_req != HNS3_NONE_RESET && new_req != HNS3_NONE_RESET && +- new_req < last_req) { +- hns3_warn(hw, "High level reset %d is pending", last_req); ++ if (last_req == HNS3_NONE_RESET || last_req < new_req) { ++ __atomic_store_n(&hw->reset.disable_cmd, 1, __ATOMIC_RELAXED); ++ hns3_schedule_delayed_reset(hns); ++ hns3_warn(hw, "High level reset detected, delay do reset"); + return true; + } + last_req = hns3_get_reset_level(hns, &hw->reset.request); +@@ -6054,7 +6046,7 @@ hns3_fec_get_internal(struct hns3_hw *hw, uint32_t *fec_capa) + { + struct hns3_sfp_info_cmd *resp; + uint32_t tmp_fec_capa; +- uint8_t auto_state; ++ uint8_t auto_state = 0; + struct hns3_cmd_desc desc; + int ret; + +@@ -6658,6 +6650,8 @@ static const struct rte_pci_id pci_id_hns3_map[] = { + { RTE_PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, HNS3_DEV_ID_50GE_RDMA) }, + { RTE_PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, HNS3_DEV_ID_100G_RDMA_MACSEC) }, + { RTE_PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, HNS3_DEV_ID_200G_RDMA) }, ++ { RTE_PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, HNS3_DEV_ID_100G_ROH) }, ++ { RTE_PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, HNS3_DEV_ID_200G_ROH) }, + { .vendor_id = 0, }, /* sentinel */ + }; + +diff --git a/dpdk/drivers/net/hns3/hns3_ethdev.h b/dpdk/drivers/net/hns3/hns3_ethdev.h +index 12d8299def..c190d5109b 100644 +--- a/dpdk/drivers/net/hns3/hns3_ethdev.h ++++ b/dpdk/drivers/net/hns3/hns3_ethdev.h +@@ -28,7 +28,9 @@ + #define HNS3_DEV_ID_25GE_RDMA 0xA222 + #define HNS3_DEV_ID_50GE_RDMA 0xA224 + #define HNS3_DEV_ID_100G_RDMA_MACSEC 0xA226 ++#define HNS3_DEV_ID_100G_ROH 0xA227 + #define HNS3_DEV_ID_200G_RDMA 0xA228 ++#define HNS3_DEV_ID_200G_ROH 0xA22C + #define HNS3_DEV_ID_100G_VF 0xA22E + #define HNS3_DEV_ID_100G_RDMA_PFC_VF 0xA22F + +@@ -485,6 +487,9 @@ struct hns3_queue_intr { + #define HNS3_PKTS_DROP_STATS_MODE1 0 + #define HNS3_PKTS_DROP_STATS_MODE2 1 + ++#define HNS3_RX_DMA_ADDR_ALIGN_128 128 ++#define HNS3_RX_DMA_ADDR_ALIGN_64 64 ++ + struct hns3_hw { + struct rte_eth_dev_data *data; + void *io_base; +@@ -552,6 +557,11 @@ struct hns3_hw { + * direction. + */ + uint8_t min_tx_pkt_len; ++ /* ++ * The required alignment of the DMA address of the RX buffer. ++ * See HNS3_RX_DMA_ADDR_ALIGN_XXX for available values. ++ */ ++ uint16_t rx_dma_addr_align; + + struct hns3_queue_intr intr; + /* +diff --git a/dpdk/drivers/net/hns3/hns3_ethdev_vf.c b/dpdk/drivers/net/hns3/hns3_ethdev_vf.c +index 916cc0fb1b..d4d691ad86 100644 +--- a/dpdk/drivers/net/hns3/hns3_ethdev_vf.c ++++ b/dpdk/drivers/net/hns3/hns3_ethdev_vf.c +@@ -91,11 +91,13 @@ hns3vf_add_uc_mac_addr(struct hns3_hw *hw, struct rte_ether_addr *mac_addr) + { + /* mac address was checked by upper level interface */ + char mac_str[RTE_ETHER_ADDR_FMT_SIZE]; ++ struct hns3_vf_to_pf_msg req; + int ret; + +- ret = hns3_send_mbx_msg(hw, HNS3_MBX_SET_UNICAST, +- HNS3_MBX_MAC_VLAN_UC_ADD, mac_addr->addr_bytes, +- RTE_ETHER_ADDR_LEN, false, NULL, 0); ++ hns3vf_mbx_setup(&req, HNS3_MBX_SET_UNICAST, ++ HNS3_MBX_MAC_VLAN_UC_ADD); ++ memcpy(req.data, mac_addr->addr_bytes, RTE_ETHER_ADDR_LEN); ++ ret = hns3vf_mbx_send(hw, &req, false, NULL, 0); + if (ret) { + hns3_ether_format_addr(mac_str, RTE_ETHER_ADDR_FMT_SIZE, + mac_addr); +@@ -110,12 +112,13 @@ hns3vf_remove_uc_mac_addr(struct hns3_hw *hw, struct rte_ether_addr *mac_addr) + { + /* mac address was checked by upper level interface */ + char mac_str[RTE_ETHER_ADDR_FMT_SIZE]; ++ struct hns3_vf_to_pf_msg req; + int ret; + +- ret = hns3_send_mbx_msg(hw, HNS3_MBX_SET_UNICAST, +- HNS3_MBX_MAC_VLAN_UC_REMOVE, +- mac_addr->addr_bytes, RTE_ETHER_ADDR_LEN, +- false, NULL, 0); ++ hns3vf_mbx_setup(&req, HNS3_MBX_SET_UNICAST, ++ HNS3_MBX_MAC_VLAN_UC_REMOVE); ++ memcpy(req.data, mac_addr->addr_bytes, RTE_ETHER_ADDR_LEN); ++ ret = hns3vf_mbx_send(hw, &req, false, NULL, 0); + if (ret) { + hns3_ether_format_addr(mac_str, RTE_ETHER_ADDR_FMT_SIZE, + mac_addr); +@@ -134,6 +137,7 @@ hns3vf_set_default_mac_addr(struct rte_eth_dev *dev, + struct rte_ether_addr *old_addr; + uint8_t addr_bytes[HNS3_TWO_ETHER_ADDR_LEN]; /* for 2 MAC addresses */ + char mac_str[RTE_ETHER_ADDR_FMT_SIZE]; ++ struct hns3_vf_to_pf_msg req; + int ret; + + /* +@@ -146,9 +150,10 @@ hns3vf_set_default_mac_addr(struct rte_eth_dev *dev, + memcpy(&addr_bytes[RTE_ETHER_ADDR_LEN], old_addr->addr_bytes, + RTE_ETHER_ADDR_LEN); + +- ret = hns3_send_mbx_msg(hw, HNS3_MBX_SET_UNICAST, +- HNS3_MBX_MAC_VLAN_UC_MODIFY, addr_bytes, +- HNS3_TWO_ETHER_ADDR_LEN, true, NULL, 0); ++ hns3vf_mbx_setup(&req, HNS3_MBX_SET_UNICAST, ++ HNS3_MBX_MAC_VLAN_UC_MODIFY); ++ memcpy(req.data, addr_bytes, HNS3_TWO_ETHER_ADDR_LEN); ++ ret = hns3vf_mbx_send(hw, &req, true, NULL, 0); + if (ret) { + /* + * The hns3 VF PMD depends on the hns3 PF kernel ethdev +@@ -185,12 +190,13 @@ hns3vf_add_mc_mac_addr(struct hns3_hw *hw, + struct rte_ether_addr *mac_addr) + { + char mac_str[RTE_ETHER_ADDR_FMT_SIZE]; ++ struct hns3_vf_to_pf_msg req; + int ret; + +- ret = hns3_send_mbx_msg(hw, HNS3_MBX_SET_MULTICAST, +- HNS3_MBX_MAC_VLAN_MC_ADD, +- mac_addr->addr_bytes, RTE_ETHER_ADDR_LEN, false, +- NULL, 0); ++ hns3vf_mbx_setup(&req, HNS3_MBX_SET_MULTICAST, ++ HNS3_MBX_MAC_VLAN_MC_ADD); ++ memcpy(req.data, mac_addr->addr_bytes, RTE_ETHER_ADDR_LEN); ++ ret = hns3vf_mbx_send(hw, &req, false, NULL, 0); + if (ret) { + hns3_ether_format_addr(mac_str, RTE_ETHER_ADDR_FMT_SIZE, + mac_addr); +@@ -206,12 +212,13 @@ hns3vf_remove_mc_mac_addr(struct hns3_hw *hw, + struct rte_ether_addr *mac_addr) + { + char mac_str[RTE_ETHER_ADDR_FMT_SIZE]; ++ struct hns3_vf_to_pf_msg req; + int ret; + +- ret = hns3_send_mbx_msg(hw, HNS3_MBX_SET_MULTICAST, +- HNS3_MBX_MAC_VLAN_MC_REMOVE, +- mac_addr->addr_bytes, RTE_ETHER_ADDR_LEN, false, +- NULL, 0); ++ hns3vf_mbx_setup(&req, HNS3_MBX_SET_MULTICAST, ++ HNS3_MBX_MAC_VLAN_MC_REMOVE); ++ memcpy(req.data, mac_addr->addr_bytes, RTE_ETHER_ADDR_LEN); ++ ret = hns3vf_mbx_send(hw, &req, false, NULL, 0); + if (ret) { + hns3_ether_format_addr(mac_str, RTE_ETHER_ADDR_FMT_SIZE, + mac_addr); +@@ -254,11 +261,12 @@ hns3vf_set_promisc_mode(struct hns3_hw *hw, bool en_bc_pmc, + * the packets with vlan tag in promiscuous mode. + */ + hns3_cmd_setup_basic_desc(&desc, HNS3_OPC_MBX_VF_TO_PF, false); +- req->msg[0] = HNS3_MBX_SET_PROMISC_MODE; +- req->msg[1] = en_bc_pmc ? 1 : 0; +- req->msg[2] = en_uc_pmc ? 1 : 0; +- req->msg[3] = en_mc_pmc ? 1 : 0; +- req->msg[4] = hw->promisc_mode == HNS3_LIMIT_PROMISC_MODE ? 1 : 0; ++ req->msg.code = HNS3_MBX_SET_PROMISC_MODE; ++ req->msg.en_bc = en_bc_pmc ? 1 : 0; ++ req->msg.en_uc = en_uc_pmc ? 1 : 0; ++ req->msg.en_mc = en_mc_pmc ? 1 : 0; ++ req->msg.en_limit_promisc = ++ hw->promisc_mode == HNS3_LIMIT_PROMISC_MODE ? 1 : 0; + + ret = hns3_cmd_send(hw, &desc, 1); + if (ret) +@@ -347,30 +355,26 @@ hns3vf_bind_ring_with_vector(struct hns3_hw *hw, uint16_t vector_id, + bool mmap, enum hns3_ring_type queue_type, + uint16_t queue_id) + { +- struct hns3_vf_bind_vector_msg bind_msg; ++ struct hns3_vf_to_pf_msg req = {0}; + const char *op_str; +- uint16_t code; + int ret; + +- memset(&bind_msg, 0, sizeof(bind_msg)); +- code = mmap ? HNS3_MBX_MAP_RING_TO_VECTOR : ++ req.code = mmap ? HNS3_MBX_MAP_RING_TO_VECTOR : + HNS3_MBX_UNMAP_RING_TO_VECTOR; +- bind_msg.vector_id = (uint8_t)vector_id; ++ req.vector_id = (uint8_t)vector_id; ++ req.ring_num = 1; + + if (queue_type == HNS3_RING_TYPE_RX) +- bind_msg.param[0].int_gl_index = HNS3_RING_GL_RX; ++ req.ring_param[0].int_gl_index = HNS3_RING_GL_RX; + else +- bind_msg.param[0].int_gl_index = HNS3_RING_GL_TX; +- +- bind_msg.param[0].ring_type = queue_type; +- bind_msg.ring_num = 1; +- bind_msg.param[0].tqp_index = queue_id; ++ req.ring_param[0].int_gl_index = HNS3_RING_GL_TX; ++ req.ring_param[0].ring_type = queue_type; ++ req.ring_param[0].tqp_index = queue_id; + op_str = mmap ? "Map" : "Unmap"; +- ret = hns3_send_mbx_msg(hw, code, 0, (uint8_t *)&bind_msg, +- sizeof(bind_msg), false, NULL, 0); ++ ret = hns3vf_mbx_send(hw, &req, false, NULL, 0); + if (ret) +- hns3_err(hw, "%s TQP %u fail, vector_id is %u, ret is %d.", +- op_str, queue_id, bind_msg.vector_id, ret); ++ hns3_err(hw, "%s TQP %u fail, vector_id is %u, ret = %d.", ++ op_str, queue_id, req.vector_id, ret); + + return ret; + } +@@ -453,10 +457,12 @@ cfg_err: + static int + hns3vf_config_mtu(struct hns3_hw *hw, uint16_t mtu) + { ++ struct hns3_vf_to_pf_msg req; + int ret; + +- ret = hns3_send_mbx_msg(hw, HNS3_MBX_SET_MTU, 0, (const uint8_t *)&mtu, +- sizeof(mtu), true, NULL, 0); ++ hns3vf_mbx_setup(&req, HNS3_MBX_SET_MTU, 0); ++ memcpy(req.data, &mtu, sizeof(mtu)); ++ ret = hns3vf_mbx_send(hw, &req, true, NULL, 0); + if (ret) + hns3_err(hw, "Failed to set mtu (%u) for vf: %d", mtu, ret); + +@@ -563,13 +569,8 @@ hns3vf_check_event_cause(struct hns3_adapter *hns, uint32_t *clearval) + val = hns3_read_dev(hw, HNS3_VF_RST_ING); + hns3_write_dev(hw, HNS3_VF_RST_ING, val | HNS3_VF_RST_ING_BIT); + val = cmdq_stat_reg & ~BIT(HNS3_VECTOR0_RST_INT_B); +- if (clearval) { +- hw->reset.stats.global_cnt++; +- hns3_warn(hw, "Global reset detected, clear reset status"); +- } else { +- hns3_schedule_delayed_reset(hns); +- hns3_warn(hw, "Global reset detected, don't clear reset status"); +- } ++ hw->reset.stats.global_cnt++; ++ hns3_warn(hw, "Global reset detected, clear reset status"); + + ret = HNS3VF_VECTOR0_EVENT_RST; + goto out; +@@ -584,9 +585,9 @@ hns3vf_check_event_cause(struct hns3_adapter *hns, uint32_t *clearval) + + val = 0; + ret = HNS3VF_VECTOR0_EVENT_OTHER; ++ + out: +- if (clearval) +- *clearval = val; ++ *clearval = val; + return ret; + } + +@@ -612,7 +613,7 @@ hns3vf_interrupt_handler(void *param) + hns3_schedule_reset(hns); + break; + case HNS3VF_VECTOR0_EVENT_MBX: +- hns3_dev_handle_mbx_msg(hw); ++ hns3vf_handle_mbx_msg(hw); + break; + default: + break; +@@ -647,12 +648,13 @@ hns3vf_get_push_lsc_cap(struct hns3_hw *hw) + uint16_t val = HNS3_PF_PUSH_LSC_CAP_NOT_SUPPORTED; + uint16_t exp = HNS3_PF_PUSH_LSC_CAP_UNKNOWN; + struct hns3_vf *vf = HNS3_DEV_HW_TO_VF(hw); ++ struct hns3_vf_to_pf_msg req; + + __atomic_store_n(&vf->pf_push_lsc_cap, HNS3_PF_PUSH_LSC_CAP_UNKNOWN, + __ATOMIC_RELEASE); + +- (void)hns3_send_mbx_msg(hw, HNS3_MBX_GET_LINK_STATUS, 0, NULL, 0, false, +- NULL, 0); ++ hns3vf_mbx_setup(&req, HNS3_MBX_GET_LINK_STATUS, 0); ++ (void)hns3vf_mbx_send(hw, &req, false, NULL, 0); + + while (remain_ms > 0) { + rte_delay_ms(HNS3_POLL_RESPONE_MS); +@@ -663,7 +665,7 @@ hns3vf_get_push_lsc_cap(struct hns3_hw *hw) + * driver has to actively handle the HNS3_MBX_LINK_STAT_CHANGE + * mailbox from PF driver to get this capability. + */ +- hns3_dev_handle_mbx_msg(hw); ++ hns3vf_handle_mbx_msg(hw); + if (__atomic_load_n(&vf->pf_push_lsc_cap, __ATOMIC_ACQUIRE) != + HNS3_PF_PUSH_LSC_CAP_UNKNOWN) + break; +@@ -705,6 +707,7 @@ hns3vf_get_capability(struct hns3_hw *hw) + hw->min_tx_pkt_len = HNS3_HIP08_MIN_TX_PKT_LEN; + hw->rss_info.ipv6_sctp_offload_supported = false; + hw->promisc_mode = HNS3_UNLIMIT_PROMISC_MODE; ++ hw->rx_dma_addr_align = HNS3_RX_DMA_ADDR_ALIGN_64; + return 0; + } + +@@ -722,6 +725,7 @@ hns3vf_get_capability(struct hns3_hw *hw) + hw->drop_stats_mode = HNS3_PKTS_DROP_STATS_MODE2; + hw->rss_info.ipv6_sctp_offload_supported = true; + hw->promisc_mode = HNS3_LIMIT_PROMISC_MODE; ++ hw->rx_dma_addr_align = HNS3_RX_DMA_ADDR_ALIGN_128; + + return 0; + } +@@ -747,12 +751,13 @@ hns3vf_check_tqp_info(struct hns3_hw *hw) + static int + hns3vf_get_port_base_vlan_filter_state(struct hns3_hw *hw) + { ++ struct hns3_vf_to_pf_msg req; + uint8_t resp_msg; + int ret; + +- ret = hns3_send_mbx_msg(hw, HNS3_MBX_SET_VLAN, +- HNS3_MBX_GET_PORT_BASE_VLAN_STATE, NULL, 0, +- true, &resp_msg, sizeof(resp_msg)); ++ hns3vf_mbx_setup(&req, HNS3_MBX_SET_VLAN, ++ HNS3_MBX_GET_PORT_BASE_VLAN_STATE); ++ ret = hns3vf_mbx_send(hw, &req, true, &resp_msg, sizeof(resp_msg)); + if (ret) { + if (ret == -ETIME) { + /* +@@ -793,10 +798,12 @@ hns3vf_get_queue_info(struct hns3_hw *hw) + { + #define HNS3VF_TQPS_RSS_INFO_LEN 6 + uint8_t resp_msg[HNS3VF_TQPS_RSS_INFO_LEN]; ++ struct hns3_vf_to_pf_msg req; + int ret; + +- ret = hns3_send_mbx_msg(hw, HNS3_MBX_GET_QINFO, 0, NULL, 0, true, +- resp_msg, HNS3VF_TQPS_RSS_INFO_LEN); ++ hns3vf_mbx_setup(&req, HNS3_MBX_GET_QINFO, 0); ++ ret = hns3vf_mbx_send(hw, &req, true, ++ resp_msg, HNS3VF_TQPS_RSS_INFO_LEN); + if (ret) { + PMD_INIT_LOG(ERR, "Failed to get tqp info from PF: %d", ret); + return ret; +@@ -834,10 +841,11 @@ hns3vf_get_basic_info(struct hns3_hw *hw) + { + uint8_t resp_msg[HNS3_MBX_MAX_RESP_DATA_SIZE]; + struct hns3_basic_info *basic_info; ++ struct hns3_vf_to_pf_msg req; + int ret; + +- ret = hns3_send_mbx_msg(hw, HNS3_MBX_GET_BASIC_INFO, 0, NULL, 0, +- true, resp_msg, sizeof(resp_msg)); ++ hns3vf_mbx_setup(&req, HNS3_MBX_GET_BASIC_INFO, 0); ++ ret = hns3vf_mbx_send(hw, &req, true, resp_msg, sizeof(resp_msg)); + if (ret) { + hns3_err(hw, "failed to get basic info from PF, ret = %d.", + ret); +@@ -857,10 +865,11 @@ static int + hns3vf_get_host_mac_addr(struct hns3_hw *hw) + { + uint8_t host_mac[RTE_ETHER_ADDR_LEN]; ++ struct hns3_vf_to_pf_msg req; + int ret; + +- ret = hns3_send_mbx_msg(hw, HNS3_MBX_GET_MAC_ADDR, 0, NULL, 0, +- true, host_mac, RTE_ETHER_ADDR_LEN); ++ hns3vf_mbx_setup(&req, HNS3_MBX_GET_MAC_ADDR, 0); ++ ret = hns3vf_mbx_send(hw, &req, true, host_mac, RTE_ETHER_ADDR_LEN); + if (ret) { + hns3_err(hw, "Failed to get mac addr from PF: %d", ret); + return ret; +@@ -909,6 +918,7 @@ static void + hns3vf_request_link_info(struct hns3_hw *hw) + { + struct hns3_vf *vf = HNS3_DEV_HW_TO_VF(hw); ++ struct hns3_vf_to_pf_msg req; + bool send_req; + int ret; + +@@ -920,8 +930,8 @@ hns3vf_request_link_info(struct hns3_hw *hw) + if (!send_req) + return; + +- ret = hns3_send_mbx_msg(hw, HNS3_MBX_GET_LINK_STATUS, 0, NULL, 0, false, +- NULL, 0); ++ hns3vf_mbx_setup(&req, HNS3_MBX_GET_LINK_STATUS, 0); ++ ret = hns3vf_mbx_send(hw, &req, false, NULL, 0); + if (ret) { + hns3_err(hw, "failed to fetch link status, ret = %d", ret); + return; +@@ -965,19 +975,18 @@ hns3vf_update_link_status(struct hns3_hw *hw, uint8_t link_status, + static int + hns3vf_vlan_filter_configure(struct hns3_adapter *hns, uint16_t vlan_id, int on) + { +-#define HNS3VF_VLAN_MBX_MSG_LEN 5 ++ struct hns3_mbx_vlan_filter *vlan_filter; ++ struct hns3_vf_to_pf_msg req = {0}; + struct hns3_hw *hw = &hns->hw; +- uint8_t msg_data[HNS3VF_VLAN_MBX_MSG_LEN]; +- uint16_t proto = htons(RTE_ETHER_TYPE_VLAN); +- uint8_t is_kill = on ? 0 : 1; + +- msg_data[0] = is_kill; +- memcpy(&msg_data[1], &vlan_id, sizeof(vlan_id)); +- memcpy(&msg_data[3], &proto, sizeof(proto)); ++ req.code = HNS3_MBX_SET_VLAN; ++ req.subcode = HNS3_MBX_VLAN_FILTER; ++ vlan_filter = (struct hns3_mbx_vlan_filter *)req.data; ++ vlan_filter->is_kill = on ? 0 : 1; ++ vlan_filter->proto = rte_cpu_to_le_16(RTE_ETHER_TYPE_VLAN); ++ vlan_filter->vlan_id = rte_cpu_to_le_16(vlan_id); + +- return hns3_send_mbx_msg(hw, HNS3_MBX_SET_VLAN, HNS3_MBX_VLAN_FILTER, +- msg_data, HNS3VF_VLAN_MBX_MSG_LEN, true, NULL, +- 0); ++ return hns3vf_mbx_send(hw, &req, true, NULL, 0); + } + + static int +@@ -1006,6 +1015,7 @@ hns3vf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) + static int + hns3vf_en_vlan_filter(struct hns3_hw *hw, bool enable) + { ++ struct hns3_vf_to_pf_msg req; + uint8_t msg_data; + int ret; + +@@ -1013,9 +1023,10 @@ hns3vf_en_vlan_filter(struct hns3_hw *hw, bool enable) + return 0; + + msg_data = enable ? 1 : 0; +- ret = hns3_send_mbx_msg(hw, HNS3_MBX_SET_VLAN, +- HNS3_MBX_ENABLE_VLAN_FILTER, &msg_data, +- sizeof(msg_data), true, NULL, 0); ++ hns3vf_mbx_setup(&req, HNS3_MBX_SET_VLAN, ++ HNS3_MBX_ENABLE_VLAN_FILTER); ++ memcpy(req.data, &msg_data, sizeof(msg_data)); ++ ret = hns3vf_mbx_send(hw, &req, true, NULL, 0); + if (ret) + hns3_err(hw, "%s vlan filter failed, ret = %d.", + enable ? "enable" : "disable", ret); +@@ -1026,12 +1037,15 @@ hns3vf_en_vlan_filter(struct hns3_hw *hw, bool enable) + static int + hns3vf_en_hw_strip_rxvtag(struct hns3_hw *hw, bool enable) + { ++ struct hns3_vf_to_pf_msg req; + uint8_t msg_data; + int ret; + + msg_data = enable ? 1 : 0; +- ret = hns3_send_mbx_msg(hw, HNS3_MBX_SET_VLAN, HNS3_MBX_VLAN_RX_OFF_CFG, +- &msg_data, sizeof(msg_data), false, NULL, 0); ++ hns3vf_mbx_setup(&req, HNS3_MBX_SET_VLAN, ++ HNS3_MBX_VLAN_RX_OFF_CFG); ++ memcpy(req.data, &msg_data, sizeof(msg_data)); ++ ret = hns3vf_mbx_send(hw, &req, false, NULL, 0); + if (ret) + hns3_err(hw, "vf %s strip failed, ret = %d.", + enable ? "enable" : "disable", ret); +@@ -1175,11 +1189,13 @@ hns3vf_dev_configure_vlan(struct rte_eth_dev *dev) + static int + hns3vf_set_alive(struct hns3_hw *hw, bool alive) + { ++ struct hns3_vf_to_pf_msg req; + uint8_t msg_data; + + msg_data = alive ? 1 : 0; +- return hns3_send_mbx_msg(hw, HNS3_MBX_SET_ALIVE, 0, &msg_data, +- sizeof(msg_data), false, NULL, 0); ++ hns3vf_mbx_setup(&req, HNS3_MBX_SET_ALIVE, 0); ++ memcpy(req.data, &msg_data, sizeof(msg_data)); ++ return hns3vf_mbx_send(hw, &req, false, NULL, 0); + } + + static void +@@ -1187,11 +1203,12 @@ hns3vf_keep_alive_handler(void *param) + { + struct rte_eth_dev *eth_dev = (struct rte_eth_dev *)param; + struct hns3_adapter *hns = eth_dev->data->dev_private; ++ struct hns3_vf_to_pf_msg req; + struct hns3_hw *hw = &hns->hw; + int ret; + +- ret = hns3_send_mbx_msg(hw, HNS3_MBX_KEEP_ALIVE, 0, NULL, 0, +- false, NULL, 0); ++ hns3vf_mbx_setup(&req, HNS3_MBX_KEEP_ALIVE, 0); ++ ret = hns3vf_mbx_send(hw, &req, false, NULL, 0); + if (ret) + hns3_err(hw, "VF sends keeping alive cmd failed(=%d)", + ret); +@@ -1330,9 +1347,11 @@ err_init_hardware: + static int + hns3vf_clear_vport_list(struct hns3_hw *hw) + { +- return hns3_send_mbx_msg(hw, HNS3_MBX_HANDLE_VF_TBL, +- HNS3_MBX_VPORT_LIST_CLEAR, NULL, 0, false, +- NULL, 0); ++ struct hns3_vf_to_pf_msg req; ++ ++ hns3vf_mbx_setup(&req, HNS3_MBX_HANDLE_VF_TBL, ++ HNS3_MBX_VPORT_LIST_CLEAR); ++ return hns3vf_mbx_send(hw, &req, false, NULL, 0); + } + + static int +@@ -1709,11 +1728,25 @@ is_vf_reset_done(struct hns3_hw *hw) + return true; + } + ++static enum hns3_reset_level ++hns3vf_detect_reset_event(struct hns3_hw *hw) ++{ ++ enum hns3_reset_level reset = HNS3_NONE_RESET; ++ uint32_t cmdq_stat_reg; ++ ++ cmdq_stat_reg = hns3_read_dev(hw, HNS3_VECTOR0_CMDQ_STAT_REG); ++ if (BIT(HNS3_VECTOR0_RST_INT_B) & cmdq_stat_reg) ++ reset = HNS3_VF_RESET; ++ ++ return reset; ++} ++ + bool + hns3vf_is_reset_pending(struct hns3_adapter *hns) + { ++ enum hns3_reset_level last_req; + struct hns3_hw *hw = &hns->hw; +- enum hns3_reset_level reset; ++ enum hns3_reset_level new_req; + + /* + * According to the protocol of PCIe, FLR to a PF device resets the PF +@@ -1736,13 +1769,18 @@ hns3vf_is_reset_pending(struct hns3_adapter *hns) + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return false; + +- hns3vf_check_event_cause(hns, NULL); +- reset = hns3vf_get_reset_level(hw, &hw->reset.pending); +- if (hw->reset.level != HNS3_NONE_RESET && reset != HNS3_NONE_RESET && +- hw->reset.level < reset) { +- hns3_warn(hw, "High level reset %d is pending", reset); ++ new_req = hns3vf_detect_reset_event(hw); ++ if (new_req == HNS3_NONE_RESET) ++ return false; ++ ++ last_req = hns3vf_get_reset_level(hw, &hw->reset.pending); ++ if (last_req == HNS3_NONE_RESET || last_req < new_req) { ++ __atomic_store_n(&hw->reset.disable_cmd, 1, __ATOMIC_RELAXED); ++ hns3_schedule_delayed_reset(hns); ++ hns3_warn(hw, "High level reset detected, delay do reset"); + return true; + } ++ + return false; + } + +@@ -1801,12 +1839,13 @@ hns3vf_wait_hardware_ready(struct hns3_adapter *hns) + static int + hns3vf_prepare_reset(struct hns3_adapter *hns) + { ++ struct hns3_vf_to_pf_msg req; + struct hns3_hw *hw = &hns->hw; + int ret; + + if (hw->reset.level == HNS3_VF_FUNC_RESET) { +- ret = hns3_send_mbx_msg(hw, HNS3_MBX_RESET, 0, NULL, +- 0, true, NULL, 0); ++ hns3vf_mbx_setup(&req, HNS3_MBX_RESET, 0); ++ ret = hns3vf_mbx_send(hw, &req, true, NULL, 0); + if (ret) + return ret; + } +diff --git a/dpdk/drivers/net/hns3/hns3_mbx.c b/dpdk/drivers/net/hns3/hns3_mbx.c +index f1743c195e..9cdbc1668a 100644 +--- a/dpdk/drivers/net/hns3/hns3_mbx.c ++++ b/dpdk/drivers/net/hns3/hns3_mbx.c +@@ -11,8 +11,6 @@ + #include "hns3_intr.h" + #include "hns3_rxtx.h" + +-#define HNS3_CMD_CODE_OFFSET 2 +- + static const struct errno_respcode_map err_code_map[] = { + {0, 0}, + {1, -EPERM}, +@@ -26,6 +24,14 @@ static const struct errno_respcode_map err_code_map[] = { + {95, -EOPNOTSUPP}, + }; + ++void ++hns3vf_mbx_setup(struct hns3_vf_to_pf_msg *req, uint8_t code, uint8_t subcode) ++{ ++ memset(req, 0, sizeof(struct hns3_vf_to_pf_msg)); ++ req->code = code; ++ req->subcode = subcode; ++} ++ + static int + hns3_resp_to_errno(uint16_t resp_code) + { +@@ -72,7 +78,7 @@ hns3_get_mbx_resp(struct hns3_hw *hw, uint16_t code, uint16_t subcode, + return -EIO; + } + +- hns3_dev_handle_mbx_msg(hw); ++ hns3vf_handle_mbx_msg(hw); + rte_delay_us(HNS3_WAIT_RESP_US); + + if (hw->mbx_resp.received_match_resp) +@@ -120,44 +126,24 @@ hns3_mbx_prepare_resp(struct hns3_hw *hw, uint16_t code, uint16_t subcode) + } + + int +-hns3_send_mbx_msg(struct hns3_hw *hw, uint16_t code, uint16_t subcode, +- const uint8_t *msg_data, uint8_t msg_len, bool need_resp, +- uint8_t *resp_data, uint16_t resp_len) ++hns3vf_mbx_send(struct hns3_hw *hw, ++ struct hns3_vf_to_pf_msg *req, bool need_resp, ++ uint8_t *resp_data, uint16_t resp_len) + { +- struct hns3_mbx_vf_to_pf_cmd *req; ++ struct hns3_mbx_vf_to_pf_cmd *cmd; + struct hns3_cmd_desc desc; +- bool is_ring_vector_msg; +- int offset; + int ret; + +- req = (struct hns3_mbx_vf_to_pf_cmd *)desc.data; +- +- /* first two bytes are reserved for code & subcode */ +- if (msg_len > (HNS3_MBX_MAX_MSG_SIZE - HNS3_CMD_CODE_OFFSET)) { +- hns3_err(hw, +- "VF send mbx msg fail, msg len %u exceeds max payload len %d", +- msg_len, HNS3_MBX_MAX_MSG_SIZE - HNS3_CMD_CODE_OFFSET); +- return -EINVAL; +- } +- + hns3_cmd_setup_basic_desc(&desc, HNS3_OPC_MBX_VF_TO_PF, false); +- req->msg[0] = code; +- is_ring_vector_msg = (code == HNS3_MBX_MAP_RING_TO_VECTOR) || +- (code == HNS3_MBX_UNMAP_RING_TO_VECTOR) || +- (code == HNS3_MBX_GET_RING_VECTOR_MAP); +- if (!is_ring_vector_msg) +- req->msg[1] = subcode; +- if (msg_data) { +- offset = is_ring_vector_msg ? 1 : HNS3_CMD_CODE_OFFSET; +- memcpy(&req->msg[offset], msg_data, msg_len); +- } ++ cmd = (struct hns3_mbx_vf_to_pf_cmd *)desc.data; ++ cmd->msg = *req; + + /* synchronous send */ + if (need_resp) { +- req->mbx_need_resp |= HNS3_MBX_NEED_RESP_BIT; ++ cmd->mbx_need_resp |= HNS3_MBX_NEED_RESP_BIT; + rte_spinlock_lock(&hw->mbx_resp.lock); +- hns3_mbx_prepare_resp(hw, code, subcode); +- req->match_id = hw->mbx_resp.match_id; ++ hns3_mbx_prepare_resp(hw, req->code, req->subcode); ++ cmd->match_id = hw->mbx_resp.match_id; + ret = hns3_cmd_send(hw, &desc, 1); + if (ret) { + rte_spinlock_unlock(&hw->mbx_resp.lock); +@@ -166,7 +152,8 @@ hns3_send_mbx_msg(struct hns3_hw *hw, uint16_t code, uint16_t subcode, + return ret; + } + +- ret = hns3_get_mbx_resp(hw, code, subcode, resp_data, resp_len); ++ ret = hns3_get_mbx_resp(hw, req->code, req->subcode, ++ resp_data, resp_len); + rte_spinlock_unlock(&hw->mbx_resp.lock); + } else { + /* asynchronous send */ +@@ -193,17 +180,17 @@ static void + hns3vf_handle_link_change_event(struct hns3_hw *hw, + struct hns3_mbx_pf_to_vf_cmd *req) + { ++ struct hns3_mbx_link_status *link_info = ++ (struct hns3_mbx_link_status *)req->msg.msg_data; + uint8_t link_status, link_duplex; +- uint16_t *msg_q = req->msg; + uint8_t support_push_lsc; + uint32_t link_speed; + +- memcpy(&link_speed, &msg_q[2], sizeof(link_speed)); +- link_status = rte_le_to_cpu_16(msg_q[1]); +- link_duplex = (uint8_t)rte_le_to_cpu_16(msg_q[4]); +- hns3vf_update_link_status(hw, link_status, link_speed, +- link_duplex); +- support_push_lsc = (*(uint8_t *)&msg_q[5]) & 1u; ++ link_status = (uint8_t)rte_le_to_cpu_16(link_info->link_status); ++ link_speed = rte_le_to_cpu_32(link_info->speed); ++ link_duplex = (uint8_t)rte_le_to_cpu_16(link_info->duplex); ++ hns3vf_update_link_status(hw, link_status, link_speed, link_duplex); ++ support_push_lsc = (link_info->flag) & 1u; + hns3vf_update_push_lsc_cap(hw, support_push_lsc); + } + +@@ -212,7 +199,6 @@ hns3_handle_asserting_reset(struct hns3_hw *hw, + struct hns3_mbx_pf_to_vf_cmd *req) + { + enum hns3_reset_level reset_level; +- uint16_t *msg_q = req->msg; + + /* + * PF has asserted reset hence VF should go in pending +@@ -220,7 +206,7 @@ hns3_handle_asserting_reset(struct hns3_hw *hw, + * has been completely reset. After this stack should + * eventually be re-initialized. + */ +- reset_level = rte_le_to_cpu_16(msg_q[1]); ++ reset_level = rte_le_to_cpu_16(req->msg.reset_level); + hns3_atomic_set_bit(reset_level, &hw->reset.pending); + + hns3_warn(hw, "PF inform reset level %d", reset_level); +@@ -242,8 +228,9 @@ hns3_handle_mbx_response(struct hns3_hw *hw, struct hns3_mbx_pf_to_vf_cmd *req) + * to match the request. + */ + if (req->match_id == resp->match_id) { +- resp->resp_status = hns3_resp_to_errno(req->msg[3]); +- memcpy(resp->additional_info, &req->msg[4], ++ resp->resp_status = ++ hns3_resp_to_errno(req->msg.resp_status); ++ memcpy(resp->additional_info, &req->msg.resp_data, + HNS3_MBX_MAX_RESP_DATA_SIZE); + rte_io_wmb(); + resp->received_match_resp = true; +@@ -256,7 +243,8 @@ hns3_handle_mbx_response(struct hns3_hw *hw, struct hns3_mbx_pf_to_vf_cmd *req) + * support copy request's match_id to its response. So VF follows the + * original scheme to process. + */ +- msg_data = (uint32_t)req->msg[1] << HNS3_MBX_RESP_CODE_OFFSET | req->msg[2]; ++ msg_data = (uint32_t)req->msg.vf_mbx_msg_code << ++ HNS3_MBX_RESP_CODE_OFFSET | req->msg.vf_mbx_msg_subcode; + if (resp->req_msg_data != msg_data) { + hns3_warn(hw, + "received response tag (%u) is mismatched with requested tag (%u)", +@@ -264,8 +252,8 @@ hns3_handle_mbx_response(struct hns3_hw *hw, struct hns3_mbx_pf_to_vf_cmd *req) + return; + } + +- resp->resp_status = hns3_resp_to_errno(req->msg[3]); +- memcpy(resp->additional_info, &req->msg[4], ++ resp->resp_status = hns3_resp_to_errno(req->msg.resp_status); ++ memcpy(resp->additional_info, &req->msg.resp_data, + HNS3_MBX_MAX_RESP_DATA_SIZE); + rte_io_wmb(); + resp->received_match_resp = true; +@@ -296,11 +284,8 @@ static void + hns3pf_handle_link_change_event(struct hns3_hw *hw, + struct hns3_mbx_vf_to_pf_cmd *req) + { +-#define LINK_STATUS_OFFSET 1 +-#define LINK_FAIL_CODE_OFFSET 2 +- +- if (!req->msg[LINK_STATUS_OFFSET]) +- hns3_link_fail_parse(hw, req->msg[LINK_FAIL_CODE_OFFSET]); ++ if (!req->msg.link_status) ++ hns3_link_fail_parse(hw, req->msg.link_fail_code); + + hns3_update_linkstatus_and_event(hw, true); + } +@@ -309,8 +294,7 @@ static void + hns3_update_port_base_vlan_info(struct hns3_hw *hw, + struct hns3_mbx_pf_to_vf_cmd *req) + { +-#define PVID_STATE_OFFSET 1 +- uint16_t new_pvid_state = req->msg[PVID_STATE_OFFSET] ? ++ uint16_t new_pvid_state = req->msg.pvid_state ? + HNS3_PORT_BASE_VLAN_ENABLE : HNS3_PORT_BASE_VLAN_DISABLE; + /* + * Currently, hardware doesn't support more than two layers VLAN offload +@@ -359,7 +343,7 @@ hns3_handle_mbx_msg_out_intr(struct hns3_hw *hw) + while (next_to_use != tail) { + desc = &crq->desc[next_to_use]; + req = (struct hns3_mbx_pf_to_vf_cmd *)desc->data; +- opcode = req->msg[0] & 0xff; ++ opcode = req->msg.code & 0xff; + + flag = rte_le_to_cpu_16(crq->desc[next_to_use].flag); + if (!hns3_get_bit(flag, HNS3_CMDQ_RX_OUTVLD_B)) +@@ -388,9 +372,57 @@ scan_next: + } + + void +-hns3_dev_handle_mbx_msg(struct hns3_hw *hw) ++hns3pf_handle_mbx_msg(struct hns3_hw *hw) ++{ ++ struct hns3_cmq_ring *crq = &hw->cmq.crq; ++ struct hns3_mbx_vf_to_pf_cmd *req; ++ struct hns3_cmd_desc *desc; ++ uint16_t flag; ++ ++ rte_spinlock_lock(&hw->cmq.crq.lock); ++ ++ while (!hns3_cmd_crq_empty(hw)) { ++ if (__atomic_load_n(&hw->reset.disable_cmd, __ATOMIC_RELAXED)) { ++ rte_spinlock_unlock(&hw->cmq.crq.lock); ++ return; ++ } ++ desc = &crq->desc[crq->next_to_use]; ++ req = (struct hns3_mbx_vf_to_pf_cmd *)desc->data; ++ ++ flag = rte_le_to_cpu_16(crq->desc[crq->next_to_use].flag); ++ if (unlikely(!hns3_get_bit(flag, HNS3_CMDQ_RX_OUTVLD_B))) { ++ hns3_warn(hw, ++ "dropped invalid mailbox message, code = %u", ++ req->msg.code); ++ ++ /* dropping/not processing this invalid message */ ++ crq->desc[crq->next_to_use].flag = 0; ++ hns3_mbx_ring_ptr_move_crq(crq); ++ continue; ++ } ++ ++ switch (req->msg.code) { ++ case HNS3_MBX_PUSH_LINK_STATUS: ++ hns3pf_handle_link_change_event(hw, req); ++ break; ++ default: ++ hns3_err(hw, "received unsupported(%u) mbx msg", ++ req->msg.code); ++ break; ++ } ++ crq->desc[crq->next_to_use].flag = 0; ++ hns3_mbx_ring_ptr_move_crq(crq); ++ } ++ ++ /* Write back CMDQ_RQ header pointer, IMP need this pointer */ ++ hns3_write_dev(hw, HNS3_CMDQ_RX_HEAD_REG, crq->next_to_use); ++ ++ rte_spinlock_unlock(&hw->cmq.crq.lock); ++} ++ ++void ++hns3vf_handle_mbx_msg(struct hns3_hw *hw) + { +- struct hns3_adapter *hns = HNS3_DEV_HW_TO_ADAPTER(hw); + struct hns3_cmq_ring *crq = &hw->cmq.crq; + struct hns3_mbx_pf_to_vf_cmd *req; + struct hns3_cmd_desc *desc; +@@ -401,7 +433,7 @@ hns3_dev_handle_mbx_msg(struct hns3_hw *hw) + rte_spinlock_lock(&hw->cmq.crq.lock); + + handle_out = (rte_eal_process_type() != RTE_PROC_PRIMARY || +- !rte_thread_is_intr()) && hns->is_vf; ++ !rte_thread_is_intr()); + if (handle_out) { + /* + * Currently, any threads in the primary and secondary processes +@@ -432,7 +464,7 @@ hns3_dev_handle_mbx_msg(struct hns3_hw *hw) + + desc = &crq->desc[crq->next_to_use]; + req = (struct hns3_mbx_pf_to_vf_cmd *)desc->data; +- opcode = req->msg[0] & 0xff; ++ opcode = req->msg.code & 0xff; + + flag = rte_le_to_cpu_16(crq->desc[crq->next_to_use].flag); + if (unlikely(!hns3_get_bit(flag, HNS3_CMDQ_RX_OUTVLD_B))) { +@@ -446,8 +478,7 @@ hns3_dev_handle_mbx_msg(struct hns3_hw *hw) + continue; + } + +- handle_out = hns->is_vf && desc->opcode == 0; +- if (handle_out) { ++ if (desc->opcode == 0) { + /* Message already processed by other thread */ + crq->desc[crq->next_to_use].flag = 0; + hns3_mbx_ring_ptr_move_crq(crq); +@@ -464,16 +495,6 @@ hns3_dev_handle_mbx_msg(struct hns3_hw *hw) + case HNS3_MBX_ASSERTING_RESET: + hns3_handle_asserting_reset(hw, req); + break; +- case HNS3_MBX_PUSH_LINK_STATUS: +- /* +- * This message is reported by the firmware and is +- * reported in 'struct hns3_mbx_vf_to_pf_cmd' format. +- * Therefore, we should cast the req variable to +- * 'struct hns3_mbx_vf_to_pf_cmd' and then process it. +- */ +- hns3pf_handle_link_change_event(hw, +- (struct hns3_mbx_vf_to_pf_cmd *)req); +- break; + case HNS3_MBX_PUSH_VLAN_INFO: + /* + * When the PVID configuration status of VF device is +@@ -488,7 +509,7 @@ hns3_dev_handle_mbx_msg(struct hns3_hw *hw) + * hns3 PF kernel driver, VF driver will receive this + * mailbox message from PF driver. + */ +- hns3_handle_promisc_info(hw, req->msg[1]); ++ hns3_handle_promisc_info(hw, req->msg.promisc_en); + break; + default: + hns3_err(hw, "received unsupported(%u) mbx msg", +diff --git a/dpdk/drivers/net/hns3/hns3_mbx.h b/dpdk/drivers/net/hns3/hns3_mbx.h +index 4a328802b9..2b6cb8f513 100644 +--- a/dpdk/drivers/net/hns3/hns3_mbx.h ++++ b/dpdk/drivers/net/hns3/hns3_mbx.h +@@ -89,7 +89,6 @@ enum hns3_mbx_link_fail_subcode { + HNS3_MBX_LF_XSFP_ABSENT, + }; + +-#define HNS3_MBX_MAX_MSG_SIZE 16 + #define HNS3_MBX_MAX_RESP_DATA_SIZE 8 + #define HNS3_MBX_DEF_TIME_LIMIT_MS 500 + +@@ -107,6 +106,69 @@ struct hns3_mbx_resp_status { + uint8_t additional_info[HNS3_MBX_MAX_RESP_DATA_SIZE]; + }; + ++struct hns3_ring_chain_param { ++ uint8_t ring_type; ++ uint8_t tqp_index; ++ uint8_t int_gl_index; ++}; ++ ++struct hns3_mbx_vlan_filter { ++ uint8_t is_kill; ++ uint16_t vlan_id; ++ uint16_t proto; ++} __rte_packed; ++ ++struct hns3_mbx_link_status { ++ uint16_t link_status; ++ uint32_t speed; ++ uint16_t duplex; ++ uint8_t flag; ++} __rte_packed; ++ ++#define HNS3_MBX_MSG_MAX_DATA_SIZE 14 ++#define HNS3_MBX_MAX_RING_CHAIN_PARAM_NUM 4 ++struct hns3_vf_to_pf_msg { ++ uint8_t code; ++ union { ++ struct { ++ uint8_t subcode; ++ uint8_t data[HNS3_MBX_MSG_MAX_DATA_SIZE]; ++ }; ++ struct { ++ uint8_t en_bc; ++ uint8_t en_uc; ++ uint8_t en_mc; ++ uint8_t en_limit_promisc; ++ }; ++ struct { ++ uint8_t vector_id; ++ uint8_t ring_num; ++ struct hns3_ring_chain_param ++ ring_param[HNS3_MBX_MAX_RING_CHAIN_PARAM_NUM]; ++ }; ++ struct { ++ uint8_t link_status; ++ uint8_t link_fail_code; ++ }; ++ }; ++}; ++ ++struct hns3_pf_to_vf_msg { ++ uint16_t code; ++ union { ++ struct { ++ uint16_t vf_mbx_msg_code; ++ uint16_t vf_mbx_msg_subcode; ++ uint16_t resp_status; ++ uint8_t resp_data[HNS3_MBX_MAX_RESP_DATA_SIZE]; ++ }; ++ uint16_t promisc_en; ++ uint16_t reset_level; ++ uint16_t pvid_state; ++ uint8_t msg_data[HNS3_MBX_MSG_MAX_DATA_SIZE]; ++ }; ++}; ++ + struct errno_respcode_map { + uint16_t resp_code; + int err_no; +@@ -122,7 +184,7 @@ struct hns3_mbx_vf_to_pf_cmd { + uint8_t msg_len; + uint8_t rsv2; + uint16_t match_id; +- uint8_t msg[HNS3_MBX_MAX_MSG_SIZE]; ++ struct hns3_vf_to_pf_msg msg; + }; + + struct hns3_mbx_pf_to_vf_cmd { +@@ -131,20 +193,7 @@ struct hns3_mbx_pf_to_vf_cmd { + uint8_t msg_len; + uint8_t rsv1; + uint16_t match_id; +- uint16_t msg[8]; +-}; +- +-struct hns3_ring_chain_param { +- uint8_t ring_type; +- uint8_t tqp_index; +- uint8_t int_gl_index; +-}; +- +-#define HNS3_MBX_MAX_RING_CHAIN_PARAM_NUM 4 +-struct hns3_vf_bind_vector_msg { +- uint8_t vector_id; +- uint8_t ring_num; +- struct hns3_ring_chain_param param[HNS3_MBX_MAX_RING_CHAIN_PARAM_NUM]; ++ struct hns3_pf_to_vf_msg msg; + }; + + struct hns3_pf_rst_done_cmd { +@@ -158,8 +207,11 @@ struct hns3_pf_rst_done_cmd { + ((crq)->next_to_use = ((crq)->next_to_use + 1) % (crq)->desc_num) + + struct hns3_hw; +-void hns3_dev_handle_mbx_msg(struct hns3_hw *hw); +-int hns3_send_mbx_msg(struct hns3_hw *hw, uint16_t code, uint16_t subcode, +- const uint8_t *msg_data, uint8_t msg_len, bool need_resp, +- uint8_t *resp_data, uint16_t resp_len); ++void hns3pf_handle_mbx_msg(struct hns3_hw *hw); ++void hns3vf_handle_mbx_msg(struct hns3_hw *hw); ++void hns3vf_mbx_setup(struct hns3_vf_to_pf_msg *req, ++ uint8_t code, uint8_t subcode); ++int hns3vf_mbx_send(struct hns3_hw *hw, ++ struct hns3_vf_to_pf_msg *req_msg, bool need_resp, ++ uint8_t *resp_data, uint16_t resp_len); + #endif /* HNS3_MBX_H */ +diff --git a/dpdk/drivers/net/hns3/hns3_rss.c b/dpdk/drivers/net/hns3/hns3_rss.c +index 15feb26043..3eae4caf52 100644 +--- a/dpdk/drivers/net/hns3/hns3_rss.c ++++ b/dpdk/drivers/net/hns3/hns3_rss.c +@@ -153,8 +153,7 @@ static const struct { + BIT_ULL(HNS3_RSS_FIELD_IPV4_SCTP_EN_IP_S) | + BIT_ULL(HNS3_RSS_FIELD_IPV4_SCTP_EN_IP_D) | + BIT_ULL(HNS3_RSS_FIELD_IPV4_SCTP_EN_SCTP_S) | +- BIT_ULL(HNS3_RSS_FIELD_IPV4_SCTP_EN_SCTP_D) | +- BIT_ULL(HNS3_RSS_FIELD_IPV4_SCTP_EN_SCTP_VER), ++ BIT_ULL(HNS3_RSS_FIELD_IPV4_SCTP_EN_SCTP_D), + HNS3_RSS_TUPLE_IPV4_SCTP_M }, + + /* IPV6-FRAG */ +@@ -274,8 +273,7 @@ static const struct { + BIT_ULL(HNS3_RSS_FIELD_IPV6_SCTP_EN_IP_S) | + BIT_ULL(HNS3_RSS_FIELD_IPV6_SCTP_EN_IP_D) | + BIT_ULL(HNS3_RSS_FIELD_IPV6_SCTP_EN_SCTP_D) | +- BIT_ULL(HNS3_RSS_FIELD_IPV6_SCTP_EN_SCTP_S) | +- BIT_ULL(HNS3_RSS_FIELD_IPV6_SCTP_EN_SCTP_VER), ++ BIT_ULL(HNS3_RSS_FIELD_IPV6_SCTP_EN_SCTP_S), + HNS3_RSS_TUPLE_IPV6_SCTP_M }, + }; + +diff --git a/dpdk/drivers/net/hns3/hns3_rss.h b/dpdk/drivers/net/hns3/hns3_rss.h +index 9d182a8025..0755760b45 100644 +--- a/dpdk/drivers/net/hns3/hns3_rss.h ++++ b/dpdk/drivers/net/hns3/hns3_rss.h +@@ -49,7 +49,6 @@ enum hns3_tuple_field { + HNS3_RSS_FIELD_IPV4_SCTP_EN_SCTP_S, + HNS3_RSS_FIELD_IPV4_SCTP_EN_IP_D, + HNS3_RSS_FIELD_IPV4_SCTP_EN_IP_S, +- HNS3_RSS_FIELD_IPV4_SCTP_EN_SCTP_VER, + + /* IPV4 ENABLE FIELD */ + HNS3_RSS_FIELD_IPV4_EN_NONFRAG_IP_D = 24, +@@ -74,7 +73,6 @@ enum hns3_tuple_field { + HNS3_RSS_FIELD_IPV6_SCTP_EN_SCTP_S, + HNS3_RSS_FIELD_IPV6_SCTP_EN_IP_D, + HNS3_RSS_FIELD_IPV6_SCTP_EN_IP_S, +- HNS3_RSS_FIELD_IPV6_SCTP_EN_SCTP_VER, + + /* IPV6 ENABLE FIELD */ + HNS3_RSS_FIELD_IPV6_NONFRAG_IP_D = 56, +@@ -96,12 +94,12 @@ enum hns3_tuple_field { + + #define HNS3_RSS_TUPLE_IPV4_TCP_M GENMASK(3, 0) + #define HNS3_RSS_TUPLE_IPV4_UDP_M GENMASK(11, 8) +-#define HNS3_RSS_TUPLE_IPV4_SCTP_M GENMASK(20, 16) ++#define HNS3_RSS_TUPLE_IPV4_SCTP_M GENMASK(19, 16) + #define HNS3_RSS_TUPLE_IPV4_NONF_M GENMASK(25, 24) + #define HNS3_RSS_TUPLE_IPV4_FLAG_M GENMASK(27, 26) + #define HNS3_RSS_TUPLE_IPV6_TCP_M GENMASK(35, 32) + #define HNS3_RSS_TUPLE_IPV6_UDP_M GENMASK(43, 40) +-#define HNS3_RSS_TUPLE_IPV6_SCTP_M GENMASK(52, 48) ++#define HNS3_RSS_TUPLE_IPV6_SCTP_M GENMASK(51, 48) + #define HNS3_RSS_TUPLE_IPV6_NONF_M GENMASK(57, 56) + #define HNS3_RSS_TUPLE_IPV6_FLAG_M GENMASK(59, 58) + +diff --git a/dpdk/drivers/net/hns3/hns3_rxtx.c b/dpdk/drivers/net/hns3/hns3_rxtx.c +index 09b7e90c70..8d0db134d2 100644 +--- a/dpdk/drivers/net/hns3/hns3_rxtx.c ++++ b/dpdk/drivers/net/hns3/hns3_rxtx.c +@@ -86,9 +86,14 @@ hns3_rx_queue_release(void *queue) + struct hns3_rx_queue *rxq = queue; + if (rxq) { + hns3_rx_queue_release_mbufs(rxq); +- if (rxq->mz) ++ if (rxq->mz) { + rte_memzone_free(rxq->mz); +- rte_free(rxq->sw_ring); ++ rxq->mz = NULL; ++ } ++ if (rxq->sw_ring) { ++ rte_free(rxq->sw_ring); ++ rxq->sw_ring = NULL; ++ } + rte_free(rxq); + } + } +@@ -99,10 +104,18 @@ hns3_tx_queue_release(void *queue) + struct hns3_tx_queue *txq = queue; + if (txq) { + hns3_tx_queue_release_mbufs(txq); +- if (txq->mz) ++ if (txq->mz) { + rte_memzone_free(txq->mz); +- rte_free(txq->sw_ring); +- rte_free(txq->free); ++ txq->mz = NULL; ++ } ++ if (txq->sw_ring) { ++ rte_free(txq->sw_ring); ++ txq->sw_ring = NULL; ++ } ++ if (txq->free) { ++ rte_free(txq->free); ++ txq->free = NULL; ++ } + rte_free(txq); + } + } +@@ -260,12 +273,27 @@ hns3_free_all_queues(struct rte_eth_dev *dev) + hns3_free_tx_queues(dev); + } + ++static int ++hns3_check_rx_dma_addr(struct hns3_hw *hw, uint64_t dma_addr) ++{ ++ uint64_t rem; ++ ++ rem = dma_addr & (hw->rx_dma_addr_align - 1); ++ if (rem > 0) { ++ hns3_err(hw, "The IO address of the beginning of the mbuf data " ++ "must be %u-byte aligned", hw->rx_dma_addr_align); ++ return -EINVAL; ++ } ++ return 0; ++} ++ + static int + hns3_alloc_rx_queue_mbufs(struct hns3_hw *hw, struct hns3_rx_queue *rxq) + { + struct rte_mbuf *mbuf; + uint64_t dma_addr; + uint16_t i; ++ int ret; + + for (i = 0; i < rxq->nb_rx_desc; i++) { + mbuf = rte_mbuf_raw_alloc(rxq->mb_pool); +@@ -286,6 +314,12 @@ hns3_alloc_rx_queue_mbufs(struct hns3_hw *hw, struct hns3_rx_queue *rxq) + dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf)); + rxq->rx_ring[i].addr = dma_addr; + rxq->rx_ring[i].rx.bd_base_info = 0; ++ ++ ret = hns3_check_rx_dma_addr(hw, dma_addr); ++ if (ret != 0) { ++ hns3_rx_queue_release_mbufs(rxq); ++ return ret; ++ } + } + + return 0; +@@ -686,13 +720,12 @@ tqp_reset_fail: + static int + hns3vf_reset_tqp(struct hns3_hw *hw, uint16_t queue_id) + { +- uint8_t msg_data[2]; ++ struct hns3_vf_to_pf_msg req; + int ret; + +- memcpy(msg_data, &queue_id, sizeof(uint16_t)); +- +- ret = hns3_send_mbx_msg(hw, HNS3_MBX_QUEUE_RESET, 0, msg_data, +- sizeof(msg_data), true, NULL, 0); ++ hns3vf_mbx_setup(&req, HNS3_MBX_QUEUE_RESET, 0); ++ memcpy(req.data, &queue_id, sizeof(uint16_t)); ++ ret = hns3vf_mbx_send(hw, &req, true, NULL, 0); + if (ret) + hns3_err(hw, "fail to reset tqp, queue_id = %u, ret = %d.", + queue_id, ret); +@@ -769,15 +802,14 @@ static int + hns3vf_reset_all_tqps(struct hns3_hw *hw) + { + #define HNS3VF_RESET_ALL_TQP_DONE 1U ++ struct hns3_vf_to_pf_msg req; + uint8_t reset_status; +- uint8_t msg_data[2]; + int ret; + uint16_t i; + +- memset(msg_data, 0, sizeof(msg_data)); +- ret = hns3_send_mbx_msg(hw, HNS3_MBX_QUEUE_RESET, 0, msg_data, +- sizeof(msg_data), true, &reset_status, +- sizeof(reset_status)); ++ hns3vf_mbx_setup(&req, HNS3_MBX_QUEUE_RESET, 0); ++ ret = hns3vf_mbx_send(hw, &req, true, ++ &reset_status, sizeof(reset_status)); + if (ret) { + hns3_err(hw, "fail to send rcb reset mbx, ret = %d.", ret); + return ret; +@@ -2390,8 +2422,7 @@ hns3_rx_ptp_timestamp_handle(struct hns3_rx_queue *rxq, struct rte_mbuf *mbuf, + { + struct hns3_pf *pf = HNS3_DEV_PRIVATE_TO_PF(rxq->hns); + +- mbuf->ol_flags |= RTE_MBUF_F_RX_IEEE1588_PTP | +- RTE_MBUF_F_RX_IEEE1588_TMST; ++ mbuf->ol_flags |= RTE_MBUF_F_RX_IEEE1588_TMST; + if (hns3_timestamp_rx_dynflag > 0) { + *RTE_MBUF_DYNFIELD(mbuf, hns3_timestamp_dynfield_offset, + rte_mbuf_timestamp_t *) = timestamp; +@@ -2670,6 +2701,7 @@ hns3_recv_scattered_pkts(void *rx_queue, + continue; + } + ++ first_seg->ol_flags = 0; + if (unlikely(bd_base_info & BIT(HNS3_RXD_TS_VLD_B))) + hns3_rx_ptp_timestamp_handle(rxq, first_seg, timestamp); + +@@ -2699,7 +2731,7 @@ hns3_recv_scattered_pkts(void *rx_queue, + + first_seg->port = rxq->port_id; + first_seg->hash.rss = rte_le_to_cpu_32(rxd.rx.rss_hash); +- first_seg->ol_flags = RTE_MBUF_F_RX_RSS_HASH; ++ first_seg->ol_flags |= RTE_MBUF_F_RX_RSS_HASH; + if (unlikely(bd_base_info & BIT(HNS3_RXD_LUM_B))) { + first_seg->hash.fdir.hi = + rte_le_to_cpu_16(rxd.rx.fd_id); +@@ -3617,58 +3649,6 @@ hns3_pkt_need_linearized(struct rte_mbuf *tx_pkts, uint32_t bd_num, + return false; + } + +-static bool +-hns3_outer_ipv4_cksum_prepared(struct rte_mbuf *m, uint64_t ol_flags, +- uint32_t *l4_proto) +-{ +- struct rte_ipv4_hdr *ipv4_hdr; +- ipv4_hdr = rte_pktmbuf_mtod_offset(m, struct rte_ipv4_hdr *, +- m->outer_l2_len); +- if (ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) +- ipv4_hdr->hdr_checksum = 0; +- if (ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM) { +- struct rte_udp_hdr *udp_hdr; +- /* +- * If OUTER_UDP_CKSUM is support, HW can calculate the pseudo +- * header for TSO packets +- */ +- if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) +- return true; +- udp_hdr = rte_pktmbuf_mtod_offset(m, struct rte_udp_hdr *, +- m->outer_l2_len + m->outer_l3_len); +- udp_hdr->dgram_cksum = rte_ipv4_phdr_cksum(ipv4_hdr, ol_flags); +- +- return true; +- } +- *l4_proto = ipv4_hdr->next_proto_id; +- return false; +-} +- +-static bool +-hns3_outer_ipv6_cksum_prepared(struct rte_mbuf *m, uint64_t ol_flags, +- uint32_t *l4_proto) +-{ +- struct rte_ipv6_hdr *ipv6_hdr; +- ipv6_hdr = rte_pktmbuf_mtod_offset(m, struct rte_ipv6_hdr *, +- m->outer_l2_len); +- if (ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM) { +- struct rte_udp_hdr *udp_hdr; +- /* +- * If OUTER_UDP_CKSUM is support, HW can calculate the pseudo +- * header for TSO packets +- */ +- if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) +- return true; +- udp_hdr = rte_pktmbuf_mtod_offset(m, struct rte_udp_hdr *, +- m->outer_l2_len + m->outer_l3_len); +- udp_hdr->dgram_cksum = rte_ipv6_phdr_cksum(ipv6_hdr, ol_flags); +- +- return true; +- } +- *l4_proto = ipv6_hdr->proto; +- return false; +-} +- + static void + hns3_outer_header_cksum_prepare(struct rte_mbuf *m) + { +@@ -3676,29 +3656,38 @@ hns3_outer_header_cksum_prepare(struct rte_mbuf *m) + uint32_t paylen, hdr_len, l4_proto; + struct rte_udp_hdr *udp_hdr; + +- if (!(ol_flags & (RTE_MBUF_F_TX_OUTER_IPV4 | RTE_MBUF_F_TX_OUTER_IPV6))) ++ if (!(ol_flags & (RTE_MBUF_F_TX_OUTER_IPV4 | RTE_MBUF_F_TX_OUTER_IPV6)) && ++ ((ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM) || ++ !(ol_flags & RTE_MBUF_F_TX_TCP_SEG))) + return; + + if (ol_flags & RTE_MBUF_F_TX_OUTER_IPV4) { +- if (hns3_outer_ipv4_cksum_prepared(m, ol_flags, &l4_proto)) +- return; ++ struct rte_ipv4_hdr *ipv4_hdr; ++ ++ ipv4_hdr = rte_pktmbuf_mtod_offset(m, struct rte_ipv4_hdr *, ++ m->outer_l2_len); ++ l4_proto = ipv4_hdr->next_proto_id; + } else { +- if (hns3_outer_ipv6_cksum_prepared(m, ol_flags, &l4_proto)) +- return; ++ struct rte_ipv6_hdr *ipv6_hdr; ++ ++ ipv6_hdr = rte_pktmbuf_mtod_offset(m, struct rte_ipv6_hdr *, ++ m->outer_l2_len); ++ l4_proto = ipv6_hdr->proto; + } + ++ if (l4_proto != IPPROTO_UDP) ++ return; ++ + /* driver should ensure the outer udp cksum is 0 for TUNNEL TSO */ +- if (l4_proto == IPPROTO_UDP && (ol_flags & RTE_MBUF_F_TX_TCP_SEG)) { +- hdr_len = m->l2_len + m->l3_len + m->l4_len; +- hdr_len += m->outer_l2_len + m->outer_l3_len; +- paylen = m->pkt_len - hdr_len; +- if (paylen <= m->tso_segsz) +- return; +- udp_hdr = rte_pktmbuf_mtod_offset(m, struct rte_udp_hdr *, +- m->outer_l2_len + +- m->outer_l3_len); +- udp_hdr->dgram_cksum = 0; +- } ++ hdr_len = m->l2_len + m->l3_len + m->l4_len; ++ hdr_len += m->outer_l2_len + m->outer_l3_len; ++ paylen = m->pkt_len - hdr_len; ++ if (paylen <= m->tso_segsz) ++ return; ++ udp_hdr = rte_pktmbuf_mtod_offset(m, struct rte_udp_hdr *, ++ m->outer_l2_len + ++ m->outer_l3_len); ++ udp_hdr->dgram_cksum = 0; + } + + static int +diff --git a/dpdk/drivers/net/i40e/i40e_ethdev.c b/dpdk/drivers/net/i40e/i40e_ethdev.c +index 3ca226156b..ffc1f6d874 100644 +--- a/dpdk/drivers/net/i40e/i40e_ethdev.c ++++ b/dpdk/drivers/net/i40e/i40e_ethdev.c +@@ -3724,8 +3724,12 @@ i40e_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) + RTE_ETH_TX_OFFLOAD_IPIP_TNL_TSO | + RTE_ETH_TX_OFFLOAD_GENEVE_TNL_TSO | + RTE_ETH_TX_OFFLOAD_MULTI_SEGS | +- RTE_ETH_TX_OFFLOAD_OUTER_UDP_CKSUM | + dev_info->tx_queue_offload_capa; ++ if (hw->mac.type == I40E_MAC_X722) { ++ dev_info->tx_offload_capa |= ++ RTE_ETH_TX_OFFLOAD_OUTER_UDP_CKSUM; ++ } ++ + dev_info->dev_capa = + RTE_ETH_DEV_CAPA_RUNTIME_RX_QUEUE_SETUP | + RTE_ETH_DEV_CAPA_RUNTIME_TX_QUEUE_SETUP; +diff --git a/dpdk/drivers/net/i40e/i40e_flow.c b/dpdk/drivers/net/i40e/i40e_flow.c +index 877e49151e..92165c8422 100644 +--- a/dpdk/drivers/net/i40e/i40e_flow.c ++++ b/dpdk/drivers/net/i40e/i40e_flow.c +@@ -1708,8 +1708,7 @@ i40e_flow_parse_fdir_pattern(struct rte_eth_dev *dev, + + ether_type = rte_be_to_cpu_16(eth_spec->hdr.ether_type); + +- if (next_type == RTE_FLOW_ITEM_TYPE_VLAN || +- ether_type == RTE_ETHER_TYPE_IPV4 || ++ if (ether_type == RTE_ETHER_TYPE_IPV4 || + ether_type == RTE_ETHER_TYPE_IPV6 || + ether_type == i40e_get_outer_vlan(dev)) { + rte_flow_error_set(error, EINVAL, +diff --git a/dpdk/drivers/net/i40e/i40e_rxtx.c b/dpdk/drivers/net/i40e/i40e_rxtx.c +index 9aa5facb53..5e693cb1ea 100644 +--- a/dpdk/drivers/net/i40e/i40e_rxtx.c ++++ b/dpdk/drivers/net/i40e/i40e_rxtx.c +@@ -295,6 +295,15 @@ i40e_parse_tunneling_params(uint64_t ol_flags, + */ + *cd_tunneling |= (tx_offload.l2_len >> 1) << + I40E_TXD_CTX_QW0_NATLEN_SHIFT; ++ ++ /** ++ * Calculate the tunneling UDP checksum (only supported with X722). ++ * Shall be set only if L4TUNT = 01b and EIPT is not zero ++ */ ++ if ((*cd_tunneling & I40E_TXD_CTX_QW0_EXT_IP_MASK) && ++ (*cd_tunneling & I40E_TXD_CTX_UDP_TUNNELING) && ++ (ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM)) ++ *cd_tunneling |= I40E_TXD_CTX_QW0_L4T_CS_MASK; + } + + static inline void +diff --git a/dpdk/drivers/net/i40e/i40e_rxtx_vec_avx2.c b/dpdk/drivers/net/i40e/i40e_rxtx_vec_avx2.c +index f468c1fd90..19cf0ac718 100644 +--- a/dpdk/drivers/net/i40e/i40e_rxtx_vec_avx2.c ++++ b/dpdk/drivers/net/i40e/i40e_rxtx_vec_avx2.c +@@ -276,46 +276,30 @@ _recv_raw_pkts_vec_avx2(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_pkts, + _mm256_loadu_si256((void *)&sw_ring[i + 4])); + #endif + +- __m256i raw_desc0_1, raw_desc2_3, raw_desc4_5, raw_desc6_7; +-#ifdef RTE_LIBRTE_I40E_16BYTE_RX_DESC +- /* for AVX we need alignment otherwise loads are not atomic */ +- if (avx_aligned) { +- /* load in descriptors, 2 at a time, in reverse order */ +- raw_desc6_7 = _mm256_load_si256((void *)(rxdp + 6)); +- rte_compiler_barrier(); +- raw_desc4_5 = _mm256_load_si256((void *)(rxdp + 4)); +- rte_compiler_barrier(); +- raw_desc2_3 = _mm256_load_si256((void *)(rxdp + 2)); +- rte_compiler_barrier(); +- raw_desc0_1 = _mm256_load_si256((void *)(rxdp + 0)); +- } else +-#endif +- do { +- const __m128i raw_desc7 = _mm_load_si128((void *)(rxdp + 7)); +- rte_compiler_barrier(); +- const __m128i raw_desc6 = _mm_load_si128((void *)(rxdp + 6)); +- rte_compiler_barrier(); +- const __m128i raw_desc5 = _mm_load_si128((void *)(rxdp + 5)); +- rte_compiler_barrier(); +- const __m128i raw_desc4 = _mm_load_si128((void *)(rxdp + 4)); +- rte_compiler_barrier(); +- const __m128i raw_desc3 = _mm_load_si128((void *)(rxdp + 3)); +- rte_compiler_barrier(); +- const __m128i raw_desc2 = _mm_load_si128((void *)(rxdp + 2)); +- rte_compiler_barrier(); +- const __m128i raw_desc1 = _mm_load_si128((void *)(rxdp + 1)); +- rte_compiler_barrier(); +- const __m128i raw_desc0 = _mm_load_si128((void *)(rxdp + 0)); +- +- raw_desc6_7 = _mm256_inserti128_si256( +- _mm256_castsi128_si256(raw_desc6), raw_desc7, 1); +- raw_desc4_5 = _mm256_inserti128_si256( +- _mm256_castsi128_si256(raw_desc4), raw_desc5, 1); +- raw_desc2_3 = _mm256_inserti128_si256( +- _mm256_castsi128_si256(raw_desc2), raw_desc3, 1); +- raw_desc0_1 = _mm256_inserti128_si256( +- _mm256_castsi128_si256(raw_desc0), raw_desc1, 1); +- } while (0); ++ const __m128i raw_desc7 = _mm_load_si128((void *)(rxdp + 7)); ++ rte_compiler_barrier(); ++ const __m128i raw_desc6 = _mm_load_si128((void *)(rxdp + 6)); ++ rte_compiler_barrier(); ++ const __m128i raw_desc5 = _mm_load_si128((void *)(rxdp + 5)); ++ rte_compiler_barrier(); ++ const __m128i raw_desc4 = _mm_load_si128((void *)(rxdp + 4)); ++ rte_compiler_barrier(); ++ const __m128i raw_desc3 = _mm_load_si128((void *)(rxdp + 3)); ++ rte_compiler_barrier(); ++ const __m128i raw_desc2 = _mm_load_si128((void *)(rxdp + 2)); ++ rte_compiler_barrier(); ++ const __m128i raw_desc1 = _mm_load_si128((void *)(rxdp + 1)); ++ rte_compiler_barrier(); ++ const __m128i raw_desc0 = _mm_load_si128((void *)(rxdp + 0)); ++ ++ const __m256i raw_desc6_7 = _mm256_inserti128_si256( ++ _mm256_castsi128_si256(raw_desc6), raw_desc7, 1); ++ const __m256i raw_desc4_5 = _mm256_inserti128_si256( ++ _mm256_castsi128_si256(raw_desc4), raw_desc5, 1); ++ const __m256i raw_desc2_3 = _mm256_inserti128_si256( ++ _mm256_castsi128_si256(raw_desc2), raw_desc3, 1); ++ const __m256i raw_desc0_1 = _mm256_inserti128_si256( ++ _mm256_castsi128_si256(raw_desc0), raw_desc1, 1); + + if (split_packet) { + int j; +diff --git a/dpdk/drivers/net/iavf/iavf.h b/dpdk/drivers/net/iavf/iavf.h +index 10868f2c30..d273d884f5 100644 +--- a/dpdk/drivers/net/iavf/iavf.h ++++ b/dpdk/drivers/net/iavf/iavf.h +@@ -18,7 +18,8 @@ + + #define IAVF_AQ_LEN 32 + #define IAVF_AQ_BUF_SZ 4096 +-#define IAVF_RESET_WAIT_CNT 500 ++#define IAVF_RESET_WAIT_CNT 2000 ++#define IAVF_RESET_DETECTED_CNT 500 + #define IAVF_BUF_SIZE_MIN 1024 + #define IAVF_FRAME_SIZE_MAX 9728 + #define IAVF_QUEUE_BASE_ADDR_UNIT 128 +@@ -511,5 +512,6 @@ int iavf_flow_sub_check(struct iavf_adapter *adapter, + struct iavf_fsub_conf *filter); + void iavf_dev_watchdog_enable(struct iavf_adapter *adapter); + void iavf_dev_watchdog_disable(struct iavf_adapter *adapter); +-int iavf_handle_hw_reset(struct rte_eth_dev *dev); ++void iavf_handle_hw_reset(struct rte_eth_dev *dev); ++void iavf_set_no_poll(struct iavf_adapter *adapter, bool link_change); + #endif /* _IAVF_ETHDEV_H_ */ +diff --git a/dpdk/drivers/net/iavf/iavf_ethdev.c b/dpdk/drivers/net/iavf/iavf_ethdev.c +index d1edb0dd5c..54bff05675 100644 +--- a/dpdk/drivers/net/iavf/iavf_ethdev.c ++++ b/dpdk/drivers/net/iavf/iavf_ethdev.c +@@ -296,6 +296,7 @@ iavf_dev_watchdog(void *cb_arg) + PMD_DRV_LOG(INFO, "VF \"%s\" reset has completed", + adapter->vf.eth_dev->data->name); + adapter->vf.vf_reset = false; ++ iavf_set_no_poll(adapter, false); + } + /* If not in reset then poll vfr_inprogress register for VFLR event */ + } else { +@@ -308,6 +309,7 @@ iavf_dev_watchdog(void *cb_arg) + + /* enter reset state with VFLR event */ + adapter->vf.vf_reset = true; ++ iavf_set_no_poll(adapter, false); + adapter->vf.link_up = false; + + iavf_dev_event_post(adapter->vf.eth_dev, RTE_ETH_EVENT_INTR_RESET, +@@ -628,7 +630,8 @@ iavf_dev_init_vlan(struct rte_eth_dev *dev) + RTE_ETH_VLAN_FILTER_MASK | + RTE_ETH_VLAN_EXTEND_MASK); + if (err) { +- PMD_DRV_LOG(ERR, "Failed to update vlan offload"); ++ PMD_DRV_LOG(INFO, ++ "VLAN offloading is not supported, or offloading was refused by the PF"); + return err; + } + +@@ -704,9 +707,7 @@ iavf_dev_configure(struct rte_eth_dev *dev) + vf->max_rss_qregion = IAVF_MAX_NUM_QUEUES_DFLT; + } + +- ret = iavf_dev_init_vlan(dev); +- if (ret) +- PMD_DRV_LOG(ERR, "configure VLAN failed: %d", ret); ++ iavf_dev_init_vlan(dev); + + if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) { + if (iavf_init_rss(ad) != 0) { +@@ -1086,9 +1087,6 @@ iavf_dev_stop(struct rte_eth_dev *dev) + + PMD_INIT_FUNC_TRACE(); + +- if (vf->vf_reset) +- return 0; +- + if (adapter->closed) + return -1; + +@@ -1165,7 +1163,6 @@ iavf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) + RTE_ETH_TX_OFFLOAD_TCP_CKSUM | + RTE_ETH_TX_OFFLOAD_SCTP_CKSUM | + RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM | +- RTE_ETH_TX_OFFLOAD_OUTER_UDP_CKSUM | + RTE_ETH_TX_OFFLOAD_TCP_TSO | + RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | + RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | +@@ -1174,6 +1171,10 @@ iavf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) + RTE_ETH_TX_OFFLOAD_MULTI_SEGS | + RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; + ++ /* X710 does not support outer udp checksum */ ++ if (adapter->hw.mac.type != IAVF_MAC_XL710) ++ dev_info->tx_offload_capa |= RTE_ETH_TX_OFFLOAD_OUTER_UDP_CKSUM; ++ + if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_CRC) + dev_info->rx_offload_capa |= RTE_ETH_RX_OFFLOAD_KEEP_CRC; + +@@ -2916,8 +2917,10 @@ iavf_dev_close(struct rte_eth_dev *dev) + * effect. + */ + out: +- if (vf->vf_reset && !rte_pci_set_bus_master(pci_dev, true)) ++ if (vf->vf_reset && !rte_pci_set_bus_master(pci_dev, true)) { + vf->vf_reset = false; ++ iavf_set_no_poll(adapter, false); ++ } + + /* disable watchdog */ + iavf_dev_watchdog_disable(adapter); +@@ -2948,9 +2951,20 @@ static int + iavf_dev_reset(struct rte_eth_dev *dev) + { + int ret; ++ struct iavf_adapter *adapter = ++ IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); + ++ if (!vf->in_reset_recovery) { ++ ret = iavf_aq_send_msg_to_pf(hw, VIRTCHNL_OP_RESET_VF, ++ IAVF_SUCCESS, NULL, 0, NULL); ++ if (ret) { ++ PMD_DRV_LOG(ERR, "fail to send cmd VIRTCHNL_OP_RESET_VF"); ++ return ret; ++ } ++ } ++ + /* + * Check whether the VF reset has been done and inform application, + * to avoid calling the virtual channel command, which may cause +@@ -2961,7 +2975,7 @@ iavf_dev_reset(struct rte_eth_dev *dev) + PMD_DRV_LOG(ERR, "Wait too long for reset done!\n"); + return ret; + } +- vf->vf_reset = false; ++ iavf_set_no_poll(adapter, false); + + PMD_DRV_LOG(DEBUG, "Start dev_reset ...\n"); + ret = iavf_dev_uninit(dev); +@@ -2971,16 +2985,49 @@ iavf_dev_reset(struct rte_eth_dev *dev) + return iavf_dev_init(dev); + } + ++static inline bool ++iavf_is_reset(struct iavf_hw *hw) ++{ ++ return !(IAVF_READ_REG(hw, IAVF_VF_ARQLEN1) & ++ IAVF_VF_ARQLEN1_ARQENABLE_MASK); ++} ++ ++static bool ++iavf_is_reset_detected(struct iavf_adapter *adapter) ++{ ++ struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter); ++ int i; ++ ++ /* poll until we see the reset actually happen */ ++ for (i = 0; i < IAVF_RESET_DETECTED_CNT; i++) { ++ if (iavf_is_reset(hw)) ++ return true; ++ rte_delay_ms(20); ++ } ++ ++ return false; ++} ++ + /* + * Handle hardware reset + */ +-int ++void + iavf_handle_hw_reset(struct rte_eth_dev *dev) + { + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); ++ struct iavf_adapter *adapter = dev->data->dev_private; + int ret; + ++ if (!dev->data->dev_started) ++ return; ++ ++ if (!iavf_is_reset_detected(adapter)) { ++ PMD_DRV_LOG(DEBUG, "reset not start\n"); ++ return; ++ } ++ + vf->in_reset_recovery = true; ++ iavf_set_no_poll(adapter, false); + + ret = iavf_dev_reset(dev); + if (ret) +@@ -2997,15 +3044,26 @@ iavf_handle_hw_reset(struct rte_eth_dev *dev) + ret = iavf_dev_start(dev); + if (ret) + goto error; +- dev->data->dev_started = 1; + +- vf->in_reset_recovery = false; +- return 0; ++ dev->data->dev_started = 1; ++ goto exit; + + error: + PMD_DRV_LOG(DEBUG, "RESET recover with error code=%d\n", ret); ++exit: + vf->in_reset_recovery = false; +- return ret; ++ iavf_set_no_poll(adapter, false); ++ ++ return; ++} ++ ++void ++iavf_set_no_poll(struct iavf_adapter *adapter, bool link_change) ++{ ++ struct iavf_info *vf = &adapter->vf; ++ ++ adapter->no_poll = (link_change & !vf->link_up) || ++ vf->vf_reset || vf->in_reset_recovery; + } + + static int +diff --git a/dpdk/drivers/net/iavf/iavf_ipsec_crypto.c b/dpdk/drivers/net/iavf/iavf_ipsec_crypto.c +index 07a69db540..d6c0180ffd 100644 +--- a/dpdk/drivers/net/iavf/iavf_ipsec_crypto.c ++++ b/dpdk/drivers/net/iavf/iavf_ipsec_crypto.c +@@ -1518,8 +1518,11 @@ iavf_security_ctx_create(struct iavf_adapter *adapter) + if (adapter->security_ctx == NULL) { + adapter->security_ctx = rte_malloc("iavf_security_ctx", + sizeof(struct iavf_security_ctx), 0); +- if (adapter->security_ctx == NULL) ++ if (adapter->security_ctx == NULL) { ++ rte_free(adapter->vf.eth_dev->security_ctx); ++ adapter->vf.eth_dev->security_ctx = NULL; + return -ENOMEM; ++ } + } + + return 0; +diff --git a/dpdk/drivers/net/iavf/iavf_rxtx_vec_avx2.c b/dpdk/drivers/net/iavf/iavf_rxtx_vec_avx2.c +index 510b4d8f1c..49d41af953 100644 +--- a/dpdk/drivers/net/iavf/iavf_rxtx_vec_avx2.c ++++ b/dpdk/drivers/net/iavf/iavf_rxtx_vec_avx2.c +@@ -193,62 +193,30 @@ _iavf_recv_raw_pkts_vec_avx2(struct iavf_rx_queue *rxq, + _mm256_loadu_si256((void *)&sw_ring[i + 4])); + #endif + +- __m256i raw_desc0_1, raw_desc2_3, raw_desc4_5, raw_desc6_7; +-#ifdef RTE_LIBRTE_IAVF_16BYTE_RX_DESC +- /* for AVX we need alignment otherwise loads are not atomic */ +- if (avx_aligned) { +- /* load in descriptors, 2 at a time, in reverse order */ +- raw_desc6_7 = _mm256_load_si256((void *)(rxdp + 6)); +- rte_compiler_barrier(); +- raw_desc4_5 = _mm256_load_si256((void *)(rxdp + 4)); +- rte_compiler_barrier(); +- raw_desc2_3 = _mm256_load_si256((void *)(rxdp + 2)); +- rte_compiler_barrier(); +- raw_desc0_1 = _mm256_load_si256((void *)(rxdp + 0)); +- } else +-#endif +- { +- const __m128i raw_desc7 = +- _mm_load_si128((void *)(rxdp + 7)); +- rte_compiler_barrier(); +- const __m128i raw_desc6 = +- _mm_load_si128((void *)(rxdp + 6)); +- rte_compiler_barrier(); +- const __m128i raw_desc5 = +- _mm_load_si128((void *)(rxdp + 5)); +- rte_compiler_barrier(); +- const __m128i raw_desc4 = +- _mm_load_si128((void *)(rxdp + 4)); +- rte_compiler_barrier(); +- const __m128i raw_desc3 = +- _mm_load_si128((void *)(rxdp + 3)); +- rte_compiler_barrier(); +- const __m128i raw_desc2 = +- _mm_load_si128((void *)(rxdp + 2)); +- rte_compiler_barrier(); +- const __m128i raw_desc1 = +- _mm_load_si128((void *)(rxdp + 1)); +- rte_compiler_barrier(); +- const __m128i raw_desc0 = +- _mm_load_si128((void *)(rxdp + 0)); +- +- raw_desc6_7 = +- _mm256_inserti128_si256 +- (_mm256_castsi128_si256(raw_desc6), +- raw_desc7, 1); +- raw_desc4_5 = +- _mm256_inserti128_si256 +- (_mm256_castsi128_si256(raw_desc4), +- raw_desc5, 1); +- raw_desc2_3 = +- _mm256_inserti128_si256 +- (_mm256_castsi128_si256(raw_desc2), +- raw_desc3, 1); +- raw_desc0_1 = +- _mm256_inserti128_si256 +- (_mm256_castsi128_si256(raw_desc0), +- raw_desc1, 1); +- } ++ const __m128i raw_desc7 = _mm_load_si128((void *)(rxdp + 7)); ++ rte_compiler_barrier(); ++ const __m128i raw_desc6 = _mm_load_si128((void *)(rxdp + 6)); ++ rte_compiler_barrier(); ++ const __m128i raw_desc5 = _mm_load_si128((void *)(rxdp + 5)); ++ rte_compiler_barrier(); ++ const __m128i raw_desc4 = _mm_load_si128((void *)(rxdp + 4)); ++ rte_compiler_barrier(); ++ const __m128i raw_desc3 = _mm_load_si128((void *)(rxdp + 3)); ++ rte_compiler_barrier(); ++ const __m128i raw_desc2 = _mm_load_si128((void *)(rxdp + 2)); ++ rte_compiler_barrier(); ++ const __m128i raw_desc1 = _mm_load_si128((void *)(rxdp + 1)); ++ rte_compiler_barrier(); ++ const __m128i raw_desc0 = _mm_load_si128((void *)(rxdp + 0)); ++ ++ const __m256i raw_desc6_7 = ++ _mm256_inserti128_si256(_mm256_castsi128_si256(raw_desc6), raw_desc7, 1); ++ const __m256i raw_desc4_5 = ++ _mm256_inserti128_si256(_mm256_castsi128_si256(raw_desc4), raw_desc5, 1); ++ const __m256i raw_desc2_3 = ++ _mm256_inserti128_si256(_mm256_castsi128_si256(raw_desc2), raw_desc3, 1); ++ const __m256i raw_desc0_1 = ++ _mm256_inserti128_si256(_mm256_castsi128_si256(raw_desc0), raw_desc1, 1); + + if (split_packet) { + int j; +diff --git a/dpdk/drivers/net/iavf/iavf_vchnl.c b/dpdk/drivers/net/iavf/iavf_vchnl.c +index 0a3e1d082c..1111d30f57 100644 +--- a/dpdk/drivers/net/iavf/iavf_vchnl.c ++++ b/dpdk/drivers/net/iavf/iavf_vchnl.c +@@ -273,20 +273,18 @@ iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len, + iavf_dev_watchdog_enable(adapter); + } + if (adapter->devargs.no_poll_on_link_down) { +- if (vf->link_up && adapter->no_poll) { +- adapter->no_poll = false; +- PMD_DRV_LOG(DEBUG, "VF no poll turned off"); +- } +- if (!vf->link_up) { +- adapter->no_poll = true; ++ iavf_set_no_poll(adapter, true); ++ if (adapter->no_poll) + PMD_DRV_LOG(DEBUG, "VF no poll turned on"); +- } ++ else ++ PMD_DRV_LOG(DEBUG, "VF no poll turned off"); + } + PMD_DRV_LOG(INFO, "Link status update:%s", + vf->link_up ? "up" : "down"); + break; + case VIRTCHNL_EVENT_RESET_IMPENDING: + vf->vf_reset = true; ++ iavf_set_no_poll(adapter, false); + PMD_DRV_LOG(INFO, "VF is resetting"); + break; + case VIRTCHNL_EVENT_PF_DRIVER_CLOSE: +@@ -462,6 +460,7 @@ iavf_handle_pf_event_msg(struct rte_eth_dev *dev, uint8_t *msg, + vf->link_up = false; + if (!vf->vf_reset) { + vf->vf_reset = true; ++ iavf_set_no_poll(adapter, false); + iavf_dev_event_post(dev, RTE_ETH_EVENT_INTR_RESET, + NULL, 0); + } +@@ -485,14 +484,11 @@ iavf_handle_pf_event_msg(struct rte_eth_dev *dev, uint8_t *msg, + iavf_dev_watchdog_enable(adapter); + } + if (adapter->devargs.no_poll_on_link_down) { +- if (vf->link_up && adapter->no_poll) { +- adapter->no_poll = false; +- PMD_DRV_LOG(DEBUG, "VF no poll turned off"); +- } +- if (!vf->link_up) { +- adapter->no_poll = true; ++ iavf_set_no_poll(adapter, true); ++ if (adapter->no_poll) + PMD_DRV_LOG(DEBUG, "VF no poll turned on"); +- } ++ else ++ PMD_DRV_LOG(DEBUG, "VF no poll turned off"); + } + iavf_dev_event_post(dev, RTE_ETH_EVENT_INTR_LSC, NULL, 0); + break; +diff --git a/dpdk/drivers/net/ice/base/ice_adminq_cmd.h b/dpdk/drivers/net/ice/base/ice_adminq_cmd.h +index 844e90bbce..1131379d63 100644 +--- a/dpdk/drivers/net/ice/base/ice_adminq_cmd.h ++++ b/dpdk/drivers/net/ice/base/ice_adminq_cmd.h +@@ -1728,8 +1728,8 @@ struct ice_aqc_link_topo_addr { + #define ICE_AQC_LINK_TOPO_HANDLE_M (0x3FF << ICE_AQC_LINK_TOPO_HANDLE_S) + /* Used to decode the handle field */ + #define ICE_AQC_LINK_TOPO_HANDLE_BRD_TYPE_M BIT(9) +-#define ICE_AQC_LINK_TOPO_HANDLE_BRD_TYPE_LOM BIT(9) +-#define ICE_AQC_LINK_TOPO_HANDLE_BRD_TYPE_MEZZ 0 ++#define ICE_AQC_LINK_TOPO_HANDLE_BRD_TYPE_LOM 0 ++#define ICE_AQC_LINK_TOPO_HANDLE_BRD_TYPE_MEZZ BIT(9) + #define ICE_AQC_LINK_TOPO_HANDLE_NODE_S 0 + /* In case of a Mezzanine type */ + #define ICE_AQC_LINK_TOPO_HANDLE_MEZZ_NODE_M \ +diff --git a/dpdk/drivers/net/ice/base/ice_bitops.h b/dpdk/drivers/net/ice/base/ice_bitops.h +index 3b71c1b7f5..5c17bcb674 100644 +--- a/dpdk/drivers/net/ice/base/ice_bitops.h ++++ b/dpdk/drivers/net/ice/base/ice_bitops.h +@@ -418,10 +418,10 @@ ice_bitmap_set(ice_bitmap_t *dst, u16 pos, u16 num_bits) + * Note that this function assumes it is operating on a bitmap declared using + * ice_declare_bitmap. + */ +-static inline int ++static inline u16 + ice_bitmap_hweight(ice_bitmap_t *bm, u16 size) + { +- int count = 0; ++ u16 count = 0; + u16 bit = 0; + + while (size > (bit = ice_find_next_bit(bm, size, bit))) { +diff --git a/dpdk/drivers/net/ice/base/ice_common.c b/dpdk/drivers/net/ice/base/ice_common.c +index 8867279c28..7a50a0f9f0 100644 +--- a/dpdk/drivers/net/ice/base/ice_common.c ++++ b/dpdk/drivers/net/ice/base/ice_common.c +@@ -3890,8 +3890,10 @@ ice_cfg_phy_fec(struct ice_port_info *pi, struct ice_aqc_set_phy_cfg_data *cfg, + break; + case ICE_FEC_DIS_AUTO: + /* Set No FEC and auto FEC */ +- if (!ice_fw_supports_fec_dis_auto(hw)) +- return ICE_ERR_NOT_SUPPORTED; ++ if (!ice_fw_supports_fec_dis_auto(hw)) { ++ status = ICE_ERR_NOT_SUPPORTED; ++ goto out; ++ } + cfg->link_fec_opt |= ICE_AQC_PHY_FEC_DIS; + /* fall-through */ + case ICE_FEC_AUTO: +@@ -4904,7 +4906,7 @@ ice_read_byte(u8 *src_ctx, u8 *dest_ctx, const struct ice_ctx_ele *ce_info) + + ice_memcpy(&dest_byte, src, sizeof(dest_byte), ICE_DMA_TO_NONDMA); + +- dest_byte &= ~(mask); ++ dest_byte &= mask; + + dest_byte >>= shift_width; + +@@ -4944,7 +4946,7 @@ ice_read_word(u8 *src_ctx, u8 *dest_ctx, const struct ice_ctx_ele *ce_info) + /* the data in the memory is stored as little endian so mask it + * correctly + */ +- src_word &= ~(CPU_TO_LE16(mask)); ++ src_word &= CPU_TO_LE16(mask); + + /* get the data back into host order before shifting */ + dest_word = LE16_TO_CPU(src_word); +@@ -4995,7 +4997,7 @@ ice_read_dword(u8 *src_ctx, u8 *dest_ctx, const struct ice_ctx_ele *ce_info) + /* the data in the memory is stored as little endian so mask it + * correctly + */ +- src_dword &= ~(CPU_TO_LE32(mask)); ++ src_dword &= CPU_TO_LE32(mask); + + /* get the data back into host order before shifting */ + dest_dword = LE32_TO_CPU(src_dword); +@@ -5046,7 +5048,7 @@ ice_read_qword(u8 *src_ctx, u8 *dest_ctx, const struct ice_ctx_ele *ce_info) + /* the data in the memory is stored as little endian so mask it + * correctly + */ +- src_qword &= ~(CPU_TO_LE64(mask)); ++ src_qword &= CPU_TO_LE64(mask); + + /* get the data back into host order before shifting */ + dest_qword = LE64_TO_CPU(src_qword); +diff --git a/dpdk/drivers/net/ice/base/ice_flex_pipe.c b/dpdk/drivers/net/ice/base/ice_flex_pipe.c +index f9266447d9..a0e4f5fa27 100644 +--- a/dpdk/drivers/net/ice/base/ice_flex_pipe.c ++++ b/dpdk/drivers/net/ice/base/ice_flex_pipe.c +@@ -1534,16 +1534,14 @@ ice_write_prof_mask_reg(struct ice_hw *hw, enum ice_block blk, u16 mask_idx, + switch (blk) { + case ICE_BLK_RSS: + offset = GLQF_HMASK(mask_idx); +- val = (idx << GLQF_HMASK_MSK_INDEX_S) & +- GLQF_HMASK_MSK_INDEX_M; +- val |= (mask << GLQF_HMASK_MASK_S) & GLQF_HMASK_MASK_M; ++ val = (idx << GLQF_HMASK_MSK_INDEX_S) & GLQF_HMASK_MSK_INDEX_M; ++ val |= ((u32)mask << GLQF_HMASK_MASK_S) & GLQF_HMASK_MASK_M; + break; + case ICE_BLK_FD: + offset = GLQF_FDMASK(mask_idx); + val = (idx << GLQF_FDMASK_MSK_INDEX_S) & + GLQF_FDMASK_MSK_INDEX_M; +- val |= (mask << GLQF_FDMASK_MASK_S) & +- GLQF_FDMASK_MASK_M; ++ val |= ((u32)mask << GLQF_FDMASK_MASK_S) & GLQF_FDMASK_MASK_M; + break; + default: + ice_debug(hw, ICE_DBG_PKG, "No profile masks for block %d\n", +diff --git a/dpdk/drivers/net/ice/base/ice_lan_tx_rx.h b/dpdk/drivers/net/ice/base/ice_lan_tx_rx.h +index d816df0ff6..39673e36f7 100644 +--- a/dpdk/drivers/net/ice/base/ice_lan_tx_rx.h ++++ b/dpdk/drivers/net/ice/base/ice_lan_tx_rx.h +@@ -1074,10 +1074,9 @@ struct ice_tx_ctx_desc { + __le64 qw1; + }; + +-#define ICE_TX_GSC_DESC_START 0 /* 7 BITS */ +-#define ICE_TX_GSC_DESC_OFFSET 7 /* 4 BITS */ +-#define ICE_TX_GSC_DESC_TYPE 11 /* 2 BITS */ +-#define ICE_TX_GSC_DESC_ENA 13 /* 1 BIT */ ++#define ICE_TX_GCS_DESC_START 0 /* 8 BITS */ ++#define ICE_TX_GCS_DESC_OFFSET 8 /* 4 BITS */ ++#define ICE_TX_GCS_DESC_TYPE 12 /* 3 BITS */ + + #define ICE_TXD_CTX_QW1_DTYPE_S 0 + #define ICE_TXD_CTX_QW1_DTYPE_M (0xFUL << ICE_TXD_CTX_QW1_DTYPE_S) +diff --git a/dpdk/drivers/net/ice/base/ice_nvm.c b/dpdk/drivers/net/ice/base/ice_nvm.c +index e46aded12a..6b0794f562 100644 +--- a/dpdk/drivers/net/ice/base/ice_nvm.c ++++ b/dpdk/drivers/net/ice/base/ice_nvm.c +@@ -72,6 +72,7 @@ ice_read_flat_nvm(struct ice_hw *hw, u32 offset, u32 *length, u8 *data, + enum ice_status status; + u32 inlen = *length; + u32 bytes_read = 0; ++ int retry_cnt = 0; + bool last_cmd; + + ice_debug(hw, ICE_DBG_TRACE, "%s\n", __func__); +@@ -106,11 +107,24 @@ ice_read_flat_nvm(struct ice_hw *hw, u32 offset, u32 *length, u8 *data, + offset, (u16)read_size, + data + bytes_read, last_cmd, + read_shadow_ram, NULL); +- if (status) +- break; +- +- bytes_read += read_size; +- offset += read_size; ++ if (status) { ++ if (hw->adminq.sq_last_status != ICE_AQ_RC_EBUSY || ++ retry_cnt > ICE_SQ_SEND_MAX_EXECUTE) ++ break; ++ ice_debug(hw, ICE_DBG_NVM, ++ "NVM read EBUSY error, retry %d\n", ++ retry_cnt + 1); ++ ice_release_nvm(hw); ++ msleep(ICE_SQ_SEND_DELAY_TIME_MS); ++ status = ice_acquire_nvm(hw, ICE_RES_READ); ++ if (status) ++ break; ++ retry_cnt++; ++ } else { ++ bytes_read += read_size; ++ offset += read_size; ++ retry_cnt = 0; ++ } + } while (!last_cmd); + + *length = bytes_read; +@@ -474,7 +488,7 @@ ice_get_pfa_module_tlv(struct ice_hw *hw, u16 *module_tlv, u16 *module_tlv_len, + { + enum ice_status status; + u16 pfa_len, pfa_ptr; +- u16 next_tlv; ++ u32 next_tlv; + + status = ice_read_sr_word(hw, ICE_SR_PFA_PTR, &pfa_ptr); + if (status != ICE_SUCCESS) { +@@ -490,25 +504,30 @@ ice_get_pfa_module_tlv(struct ice_hw *hw, u16 *module_tlv, u16 *module_tlv_len, + * of TLVs to find the requested one. + */ + next_tlv = pfa_ptr + 1; +- while (next_tlv < pfa_ptr + pfa_len) { ++ while (next_tlv < ((u32)pfa_ptr + pfa_len)) { + u16 tlv_sub_module_type; + u16 tlv_len; + + /* Read TLV type */ +- status = ice_read_sr_word(hw, next_tlv, &tlv_sub_module_type); +- if (status != ICE_SUCCESS) { ++ status = ice_read_sr_word(hw, (u16)next_tlv, ++ &tlv_sub_module_type); ++ if (status) { + ice_debug(hw, ICE_DBG_INIT, "Failed to read TLV type.\n"); + break; + } + /* Read TLV length */ +- status = ice_read_sr_word(hw, next_tlv + 1, &tlv_len); ++ status = ice_read_sr_word(hw, (u16)(next_tlv + 1), &tlv_len); + if (status != ICE_SUCCESS) { + ice_debug(hw, ICE_DBG_INIT, "Failed to read TLV length.\n"); + break; + } ++ if (tlv_len > pfa_len) { ++ ice_debug(hw, ICE_DBG_INIT, "Invalid TLV length.\n"); ++ return ICE_ERR_INVAL_SIZE; ++ } + if (tlv_sub_module_type == module_type) { + if (tlv_len) { +- *module_tlv = next_tlv; ++ *module_tlv = (u16)next_tlv; + *module_tlv_len = tlv_len; + return ICE_SUCCESS; + } +@@ -749,7 +768,7 @@ ice_get_orom_civd_data(struct ice_hw *hw, enum ice_bank_select bank, + orom_data, hw->flash.banks.orom_size); + if (status) { + ice_debug(hw, ICE_DBG_NVM, "Unable to read Option ROM data\n"); +- return status; ++ goto exit_error;; + } + + /* Scan the memory buffer to locate the CIVD data section */ +@@ -773,7 +792,8 @@ ice_get_orom_civd_data(struct ice_hw *hw, enum ice_bank_select bank, + if (sum) { + ice_debug(hw, ICE_DBG_NVM, "Found CIVD data with invalid checksum of %u\n", + sum); +- goto err_invalid_checksum; ++ status = ICE_ERR_NVM; ++ goto exit_error; + } + + *civd = *tmp; +@@ -781,11 +801,12 @@ ice_get_orom_civd_data(struct ice_hw *hw, enum ice_bank_select bank, + return ICE_SUCCESS; + } + ++ status = ICE_ERR_NVM; + ice_debug(hw, ICE_DBG_NVM, "Unable to locate CIVD data within the Option ROM\n"); + +-err_invalid_checksum: ++exit_error: + ice_free(hw, orom_data); +- return ICE_ERR_NVM; ++ return status; + } + + /** +diff --git a/dpdk/drivers/net/ice/base/ice_ptp_hw.c b/dpdk/drivers/net/ice/base/ice_ptp_hw.c +index 548ef5e820..c507f211df 100644 +--- a/dpdk/drivers/net/ice/base/ice_ptp_hw.c ++++ b/dpdk/drivers/net/ice/base/ice_ptp_hw.c +@@ -2817,8 +2817,8 @@ ice_ptp_one_port_cmd_e822(struct ice_hw *hw, u8 port, enum ice_ptp_tmr_cmd cmd, + val &= ~TS_CMD_MASK; + val |= cmd_val; + +- status = ice_write_phy_reg_e822_lp(hw, port, P_REG_TX_TMR_CMD, val, +- lock_sbq); ++ status = ice_write_phy_reg_e822_lp(hw, port, P_REG_TX_TMR_CMD, ++ val | TS_CMD_RX_TYPE, lock_sbq); + if (status) { + ice_debug(hw, ICE_DBG_PTP, "Failed to write back TX_TMR_CMD, status %d\n", + status); +diff --git a/dpdk/drivers/net/ice/base/ice_ptp_hw.h b/dpdk/drivers/net/ice/base/ice_ptp_hw.h +index 3667c9882d..f53b9e3ecc 100644 +--- a/dpdk/drivers/net/ice/base/ice_ptp_hw.h ++++ b/dpdk/drivers/net/ice/base/ice_ptp_hw.h +@@ -295,6 +295,8 @@ enum ice_status ice_ptp_init_phy_cfg(struct ice_hw *hw); + #define TS_CMD_MASK_E810 0xFF + #define TS_CMD_MASK 0xF + #define SYNC_EXEC_CMD 0x3 ++#define TS_CMD_RX_TYPE_S 0x4 ++#define TS_CMD_RX_TYPE MAKEMASK(0x18, TS_CMD_RX_TYPE_S) + + /* Macros to derive port low and high addresses on both quads */ + #define P_Q0_L(a, p) ((((a) + (0x2000 * (p)))) & 0xFFFF) +diff --git a/dpdk/drivers/net/ice/base/ice_sched.c b/dpdk/drivers/net/ice/base/ice_sched.c +index a4d31647fe..21cfe53a6d 100644 +--- a/dpdk/drivers/net/ice/base/ice_sched.c ++++ b/dpdk/drivers/net/ice/base/ice_sched.c +@@ -28,9 +28,8 @@ ice_sched_add_root_node(struct ice_port_info *pi, + if (!root) + return ICE_ERR_NO_MEMORY; + +- /* coverity[suspicious_sizeof] */ + root->children = (struct ice_sched_node **) +- ice_calloc(hw, hw->max_children[0], sizeof(*root)); ++ ice_calloc(hw, hw->max_children[0], sizeof(*root->children)); + if (!root->children) { + ice_free(hw, root); + return ICE_ERR_NO_MEMORY; +@@ -186,9 +185,9 @@ ice_sched_add_node(struct ice_port_info *pi, u8 layer, + if (!node) + return ICE_ERR_NO_MEMORY; + if (hw->max_children[layer]) { +- /* coverity[suspicious_sizeof] */ + node->children = (struct ice_sched_node **) +- ice_calloc(hw, hw->max_children[layer], sizeof(*node)); ++ ice_calloc(hw, hw->max_children[layer], ++ sizeof(*node->children)); + if (!node->children) { + ice_free(hw, node); + return ICE_ERR_NO_MEMORY; +@@ -1069,11 +1068,11 @@ ice_sched_add_nodes_to_layer(struct ice_port_info *pi, + u32 *first_teid_ptr = first_node_teid; + u16 new_num_nodes = num_nodes; + enum ice_status status = ICE_SUCCESS; ++ u32 temp; + + *num_nodes_added = 0; + while (*num_nodes_added < num_nodes) { + u16 max_child_nodes, num_added = 0; +- u32 temp; + + status = ice_sched_add_nodes_to_hw_layer(pi, tc_node, parent, + layer, new_num_nodes, +diff --git a/dpdk/drivers/net/ice/base/ice_switch.c b/dpdk/drivers/net/ice/base/ice_switch.c +index f7fcc3a8d4..c4fd07199e 100644 +--- a/dpdk/drivers/net/ice/base/ice_switch.c ++++ b/dpdk/drivers/net/ice/base/ice_switch.c +@@ -4603,7 +4603,7 @@ ice_add_update_vsi_list(struct ice_hw *hw, + u16 vsi_handle_arr[2]; + + /* A rule already exists with the new VSI being added */ +- if (cur_fltr->fwd_id.hw_vsi_id == new_fltr->fwd_id.hw_vsi_id) ++ if (cur_fltr->vsi_handle == new_fltr->vsi_handle) + return ICE_ERR_ALREADY_EXISTS; + + vsi_handle_arr[0] = cur_fltr->vsi_handle; +@@ -4651,7 +4651,7 @@ ice_add_update_vsi_list(struct ice_hw *hw, + + /* A rule already exists with the new VSI being added */ + if (ice_is_bit_set(m_entry->vsi_list_info->vsi_map, vsi_handle)) +- return ICE_SUCCESS; ++ return ICE_ERR_ALREADY_EXISTS; + + /* Update the previously created VSI list set with + * the new VSI ID passed in +@@ -7390,7 +7390,7 @@ ice_find_free_recp_res_idx(struct ice_hw *hw, const ice_bitmap_t *profiles, + ice_xor_bitmap(free_idx, used_idx, possible_idx, ICE_MAX_FV_WORDS); + + /* return number of free indexes */ +- return (u16)ice_bitmap_hweight(free_idx, ICE_MAX_FV_WORDS); ++ return ice_bitmap_hweight(free_idx, ICE_MAX_FV_WORDS); + } + + static void ice_set_recipe_index(unsigned long idx, u8 *bitmap) +@@ -8101,6 +8101,7 @@ ice_add_adv_recipe(struct ice_hw *hw, struct ice_adv_lkup_elem *lkups, + enum ice_status status = ICE_SUCCESS; + struct ice_sw_recipe *rm; + u8 i; ++ u16 cnt; + + if (!ice_is_prof_rule(rinfo->tun_type) && !lkups_cnt) + return ICE_ERR_PARAM; +diff --git a/dpdk/drivers/net/ice/ice_ethdev.c b/dpdk/drivers/net/ice/ice_ethdev.c +index 3ccba4db80..c1d2b91ad7 100644 +--- a/dpdk/drivers/net/ice/ice_ethdev.c ++++ b/dpdk/drivers/net/ice/ice_ethdev.c +@@ -1804,6 +1804,7 @@ ice_pf_setup(struct ice_pf *pf) + } + + pf->main_vsi = vsi; ++ rte_spinlock_init(&pf->link_lock); + + return 0; + } +@@ -3621,17 +3622,31 @@ ice_rxq_intr_setup(struct rte_eth_dev *dev) + return 0; + } + ++static enum ice_status ++ice_get_link_info_safe(struct ice_pf *pf, bool ena_lse, ++ struct ice_link_status *link) ++{ ++ struct ice_hw *hw = ICE_PF_TO_HW(pf); ++ int ret; ++ ++ rte_spinlock_lock(&pf->link_lock); ++ ++ ret = ice_aq_get_link_info(hw->port_info, ena_lse, link, NULL); ++ ++ rte_spinlock_unlock(&pf->link_lock); ++ ++ return ret; ++} ++ + static void + ice_get_init_link_status(struct rte_eth_dev *dev) + { +- struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private); + bool enable_lse = dev->data->dev_conf.intr_conf.lsc ? true : false; + struct ice_link_status link_status; + int ret; + +- ret = ice_aq_get_link_info(hw->port_info, enable_lse, +- &link_status, NULL); ++ ret = ice_get_link_info_safe(pf, enable_lse, &link_status); + if (ret != ICE_SUCCESS) { + PMD_DRV_LOG(ERR, "Failed to get link info"); + pf->init_link_up = false; +@@ -3735,7 +3750,10 @@ ice_dev_start(struct rte_eth_dev *dev) + ice_set_tx_function(dev); + + mask = RTE_ETH_VLAN_STRIP_MASK | RTE_ETH_VLAN_FILTER_MASK | +- RTE_ETH_VLAN_EXTEND_MASK | RTE_ETH_QINQ_STRIP_MASK; ++ RTE_ETH_VLAN_EXTEND_MASK; ++ if (ice_is_dvm_ena(hw)) ++ mask |= RTE_ETH_QINQ_STRIP_MASK; ++ + ret = ice_vlan_offload_set(dev, mask); + if (ret) { + PMD_INIT_LOG(ERR, "Unable to set VLAN offload"); +@@ -3876,7 +3894,11 @@ ice_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) + RTE_ETH_TX_OFFLOAD_TCP_CKSUM | + RTE_ETH_TX_OFFLOAD_SCTP_CKSUM | + RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM | +- RTE_ETH_TX_OFFLOAD_OUTER_UDP_CKSUM; ++ RTE_ETH_TX_OFFLOAD_OUTER_UDP_CKSUM | ++ RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | ++ RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | ++ RTE_ETH_TX_OFFLOAD_IPIP_TNL_TSO | ++ RTE_ETH_TX_OFFLOAD_GENEVE_TNL_TSO; + dev_info->flow_type_rss_offloads |= ICE_RSS_OFFLOAD_ALL; + } + +@@ -3996,7 +4018,7 @@ ice_link_update(struct rte_eth_dev *dev, int wait_to_complete) + { + #define CHECK_INTERVAL 50 /* 50ms */ + #define MAX_REPEAT_TIME 40 /* 2s (40 * 50ms) in total */ +- struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private); ++ struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private); + struct ice_link_status link_status; + struct rte_eth_link link, old; + int status; +@@ -4010,8 +4032,7 @@ ice_link_update(struct rte_eth_dev *dev, int wait_to_complete) + + do { + /* Get link status information from hardware */ +- status = ice_aq_get_link_info(hw->port_info, enable_lse, +- &link_status, NULL); ++ status = ice_get_link_info_safe(pf, enable_lse, &link_status); + if (status != ICE_SUCCESS) { + link.link_speed = RTE_ETH_SPEED_NUM_100M; + link.link_duplex = RTE_ETH_LINK_FULL_DUPLEX; +@@ -4802,19 +4823,35 @@ ice_vlan_offload_set(struct rte_eth_dev *dev, int mask) + ice_vsi_config_vlan_filter(vsi, false); + } + +- if (mask & RTE_ETH_VLAN_STRIP_MASK) { +- if (rxmode->offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP) +- ice_vsi_config_vlan_stripping(vsi, true); +- else +- ice_vsi_config_vlan_stripping(vsi, false); +- } ++ struct ice_hw *hw = ICE_VSI_TO_HW(vsi); ++ if (!ice_is_dvm_ena(hw)) { ++ if (mask & RTE_ETH_VLAN_STRIP_MASK) { ++ if (rxmode->offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP) ++ ice_vsi_config_vlan_stripping(vsi, true); ++ else ++ ice_vsi_config_vlan_stripping(vsi, false); ++ } + +- if (mask & RTE_ETH_QINQ_STRIP_MASK) { +- /* Enable or disable outer VLAN stripping */ +- if (rxmode->offloads & RTE_ETH_RX_OFFLOAD_QINQ_STRIP) +- ice_vsi_config_outer_vlan_stripping(vsi, true); +- else +- ice_vsi_config_outer_vlan_stripping(vsi, false); ++ if (mask & RTE_ETH_QINQ_STRIP_MASK) { ++ PMD_DRV_LOG(ERR, "Single VLAN mode (SVM) does not support qinq"); ++ return -ENOTSUP; ++ } ++ } else { ++ if ((mask & RTE_ETH_VLAN_STRIP_MASK) | ++ (mask & RTE_ETH_QINQ_STRIP_MASK)) { ++ if (rxmode->offloads & (RTE_ETH_RX_OFFLOAD_VLAN_STRIP | ++ RTE_ETH_RX_OFFLOAD_QINQ_STRIP)) ++ ice_vsi_config_outer_vlan_stripping(vsi, true); ++ else ++ ice_vsi_config_outer_vlan_stripping(vsi, false); ++ } ++ ++ if (mask & RTE_ETH_QINQ_STRIP_MASK) { ++ if (rxmode->offloads & RTE_ETH_RX_OFFLOAD_QINQ_STRIP) ++ ice_vsi_config_vlan_stripping(vsi, true); ++ else ++ ice_vsi_config_vlan_stripping(vsi, false); ++ } + } + + return 0; +diff --git a/dpdk/drivers/net/ice/ice_ethdev.h b/dpdk/drivers/net/ice/ice_ethdev.h +index abe6dcdc23..d607f028e0 100644 +--- a/dpdk/drivers/net/ice/ice_ethdev.h ++++ b/dpdk/drivers/net/ice/ice_ethdev.h +@@ -548,6 +548,10 @@ struct ice_pf { + uint64_t rss_hf; + struct ice_tm_conf tm_conf; + uint16_t outer_ethertype; ++ /* lock prevent race condition between lsc interrupt handler ++ * and link status update during dev_start. ++ */ ++ rte_spinlock_t link_lock; + }; + + #define ICE_MAX_QUEUE_NUM 2048 +diff --git a/dpdk/drivers/net/ice/ice_hash.c b/dpdk/drivers/net/ice/ice_hash.c +index f923641533..d8c46347d2 100644 +--- a/dpdk/drivers/net/ice/ice_hash.c ++++ b/dpdk/drivers/net/ice/ice_hash.c +@@ -650,10 +650,10 @@ ice_hash_parse_raw_pattern(struct ice_adapter *ad, + uint8_t *pkt_buf, *msk_buf; + uint8_t tmp_val = 0; + uint8_t tmp_c = 0; +- int i, j; ++ int i, j, ret = 0; + + if (ad->psr == NULL) +- return -rte_errno; ++ return -ENOTSUP; + + raw_spec = item->spec; + raw_mask = item->mask; +@@ -670,8 +670,10 @@ ice_hash_parse_raw_pattern(struct ice_adapter *ad, + return -ENOMEM; + + msk_buf = rte_zmalloc(NULL, pkt_len, 0); +- if (!msk_buf) ++ if (!msk_buf) { ++ rte_free(pkt_buf); + return -ENOMEM; ++ } + + /* convert string to int array */ + for (i = 0, j = 0; i < spec_len; i += 2, j++) { +@@ -708,18 +710,22 @@ ice_hash_parse_raw_pattern(struct ice_adapter *ad, + msk_buf[j] = tmp_val * 16 + tmp_c - '0'; + } + +- if (ice_parser_run(ad->psr, pkt_buf, pkt_len, &rslt)) +- return -rte_errno; ++ ret = ice_parser_run(ad->psr, pkt_buf, pkt_len, &rslt); ++ if (ret) ++ goto free_mem; + +- if (ice_parser_profile_init(&rslt, pkt_buf, msk_buf, +- pkt_len, ICE_BLK_RSS, true, &prof)) +- return -rte_errno; ++ ret = ice_parser_profile_init(&rslt, pkt_buf, msk_buf, ++ pkt_len, ICE_BLK_RSS, true, &prof); ++ if (ret) ++ goto free_mem; + + rte_memcpy(&meta->raw.prof, &prof, sizeof(prof)); + ++free_mem: + rte_free(pkt_buf); + rte_free(msk_buf); +- return 0; ++ ++ return ret; + } + + static void +diff --git a/dpdk/drivers/net/ice/ice_rxtx.c b/dpdk/drivers/net/ice/ice_rxtx.c +index 73e47ae92d..dea6a5b535 100644 +--- a/dpdk/drivers/net/ice/ice_rxtx.c ++++ b/dpdk/drivers/net/ice/ice_rxtx.c +@@ -2734,9 +2734,9 @@ ice_parse_tunneling_params(uint64_t ol_flags, + * Calculate the tunneling UDP checksum. + * Shall be set only if L4TUNT = 01b and EIPT is not zero + */ +- if (!(*cd_tunneling & ICE_TX_CTX_EIPT_NONE) && +- (*cd_tunneling & ICE_TXD_CTX_UDP_TUNNELING) && +- (ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM)) ++ if ((*cd_tunneling & ICE_TXD_CTX_QW0_EIPT_M) && ++ (*cd_tunneling & ICE_TXD_CTX_UDP_TUNNELING) && ++ (ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM)) + *cd_tunneling |= ICE_TXD_CTX_QW0_L4T_CS_M; + } + +diff --git a/dpdk/drivers/net/ice/ice_rxtx_vec_avx2.c b/dpdk/drivers/net/ice/ice_rxtx_vec_avx2.c +index 6f6d790967..d6e88dbb29 100644 +--- a/dpdk/drivers/net/ice/ice_rxtx_vec_avx2.c ++++ b/dpdk/drivers/net/ice/ice_rxtx_vec_avx2.c +@@ -254,62 +254,30 @@ _ice_recv_raw_pkts_vec_avx2(struct ice_rx_queue *rxq, struct rte_mbuf **rx_pkts, + _mm256_loadu_si256((void *)&sw_ring[i + 4])); + #endif + +- __m256i raw_desc0_1, raw_desc2_3, raw_desc4_5, raw_desc6_7; +-#ifdef RTE_LIBRTE_ICE_16BYTE_RX_DESC +- /* for AVX we need alignment otherwise loads are not atomic */ +- if (avx_aligned) { +- /* load in descriptors, 2 at a time, in reverse order */ +- raw_desc6_7 = _mm256_load_si256((void *)(rxdp + 6)); +- rte_compiler_barrier(); +- raw_desc4_5 = _mm256_load_si256((void *)(rxdp + 4)); +- rte_compiler_barrier(); +- raw_desc2_3 = _mm256_load_si256((void *)(rxdp + 2)); +- rte_compiler_barrier(); +- raw_desc0_1 = _mm256_load_si256((void *)(rxdp + 0)); +- } else +-#endif +- { +- const __m128i raw_desc7 = +- _mm_load_si128((void *)(rxdp + 7)); +- rte_compiler_barrier(); +- const __m128i raw_desc6 = +- _mm_load_si128((void *)(rxdp + 6)); +- rte_compiler_barrier(); +- const __m128i raw_desc5 = +- _mm_load_si128((void *)(rxdp + 5)); +- rte_compiler_barrier(); +- const __m128i raw_desc4 = +- _mm_load_si128((void *)(rxdp + 4)); +- rte_compiler_barrier(); +- const __m128i raw_desc3 = +- _mm_load_si128((void *)(rxdp + 3)); +- rte_compiler_barrier(); +- const __m128i raw_desc2 = +- _mm_load_si128((void *)(rxdp + 2)); +- rte_compiler_barrier(); +- const __m128i raw_desc1 = +- _mm_load_si128((void *)(rxdp + 1)); +- rte_compiler_barrier(); +- const __m128i raw_desc0 = +- _mm_load_si128((void *)(rxdp + 0)); +- +- raw_desc6_7 = +- _mm256_inserti128_si256 +- (_mm256_castsi128_si256(raw_desc6), +- raw_desc7, 1); +- raw_desc4_5 = +- _mm256_inserti128_si256 +- (_mm256_castsi128_si256(raw_desc4), +- raw_desc5, 1); +- raw_desc2_3 = +- _mm256_inserti128_si256 +- (_mm256_castsi128_si256(raw_desc2), +- raw_desc3, 1); +- raw_desc0_1 = +- _mm256_inserti128_si256 +- (_mm256_castsi128_si256(raw_desc0), +- raw_desc1, 1); +- } ++ const __m128i raw_desc7 = _mm_load_si128((void *)(rxdp + 7)); ++ rte_compiler_barrier(); ++ const __m128i raw_desc6 = _mm_load_si128((void *)(rxdp + 6)); ++ rte_compiler_barrier(); ++ const __m128i raw_desc5 = _mm_load_si128((void *)(rxdp + 5)); ++ rte_compiler_barrier(); ++ const __m128i raw_desc4 = _mm_load_si128((void *)(rxdp + 4)); ++ rte_compiler_barrier(); ++ const __m128i raw_desc3 = _mm_load_si128((void *)(rxdp + 3)); ++ rte_compiler_barrier(); ++ const __m128i raw_desc2 = _mm_load_si128((void *)(rxdp + 2)); ++ rte_compiler_barrier(); ++ const __m128i raw_desc1 = _mm_load_si128((void *)(rxdp + 1)); ++ rte_compiler_barrier(); ++ const __m128i raw_desc0 = _mm_load_si128((void *)(rxdp + 0)); ++ ++ const __m256i raw_desc6_7 = ++ _mm256_inserti128_si256(_mm256_castsi128_si256(raw_desc6), raw_desc7, 1); ++ const __m256i raw_desc4_5 = ++ _mm256_inserti128_si256(_mm256_castsi128_si256(raw_desc4), raw_desc5, 1); ++ const __m256i raw_desc2_3 = ++ _mm256_inserti128_si256(_mm256_castsi128_si256(raw_desc2), raw_desc3, 1); ++ const __m256i raw_desc0_1 = ++ _mm256_inserti128_si256(_mm256_castsi128_si256(raw_desc0), raw_desc1, 1); + + if (split_packet) { + int j; +diff --git a/dpdk/drivers/net/ice/ice_rxtx_vec_common.h b/dpdk/drivers/net/ice/ice_rxtx_vec_common.h +index 55840cf170..4b73465af5 100644 +--- a/dpdk/drivers/net/ice/ice_rxtx_vec_common.h ++++ b/dpdk/drivers/net/ice/ice_rxtx_vec_common.h +@@ -251,6 +251,10 @@ ice_rxq_vec_setup_default(struct ice_rx_queue *rxq) + RTE_ETH_TX_OFFLOAD_MULTI_SEGS | \ + RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM | \ + RTE_ETH_TX_OFFLOAD_TCP_TSO | \ ++ RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | \ ++ RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | \ ++ RTE_ETH_TX_OFFLOAD_IPIP_TNL_TSO | \ ++ RTE_ETH_TX_OFFLOAD_GENEVE_TNL_TSO | \ + RTE_ETH_TX_OFFLOAD_OUTER_UDP_CKSUM) + + #define ICE_TX_VECTOR_OFFLOAD ( \ +diff --git a/dpdk/drivers/net/ice/ice_tm.c b/dpdk/drivers/net/ice/ice_tm.c +index f5ea47ae83..65b9fdf320 100644 +--- a/dpdk/drivers/net/ice/ice_tm.c ++++ b/dpdk/drivers/net/ice/ice_tm.c +@@ -58,8 +58,15 @@ void + ice_tm_conf_uninit(struct rte_eth_dev *dev) + { + struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private); ++ struct ice_tm_shaper_profile *shaper_profile; + struct ice_tm_node *tm_node; + ++ /* clear profile */ ++ while ((shaper_profile = TAILQ_FIRST(&pf->tm_conf.shaper_profile_list))) { ++ TAILQ_REMOVE(&pf->tm_conf.shaper_profile_list, shaper_profile, node); ++ rte_free(shaper_profile); ++ } ++ + /* clear node configuration */ + while ((tm_node = TAILQ_FIRST(&pf->tm_conf.queue_list))) { + TAILQ_REMOVE(&pf->tm_conf.queue_list, tm_node, node); +@@ -648,6 +655,8 @@ static int ice_move_recfg_lan_txq(struct rte_eth_dev *dev, + uint16_t buf_size = ice_struct_size(buf, txqs, 1); + + buf = (struct ice_aqc_move_txqs_data *)ice_malloc(hw, sizeof(*buf)); ++ if (buf == NULL) ++ return -ENOMEM; + + queue_parent_node = queue_sched_node->parent; + buf->src_teid = queue_parent_node->info.node_teid; +@@ -659,6 +668,7 @@ static int ice_move_recfg_lan_txq(struct rte_eth_dev *dev, + NULL, buf, buf_size, &txqs_moved, NULL); + if (ret || txqs_moved == 0) { + PMD_DRV_LOG(ERR, "move lan queue %u failed", queue_id); ++ rte_free(buf); + return ICE_ERR_PARAM; + } + +@@ -668,12 +678,14 @@ static int ice_move_recfg_lan_txq(struct rte_eth_dev *dev, + } else { + PMD_DRV_LOG(ERR, "invalid children number %d for queue %u", + queue_parent_node->num_children, queue_id); ++ rte_free(buf); + return ICE_ERR_PARAM; + } + dst_node->children[dst_node->num_children++] = queue_sched_node; + queue_sched_node->parent = dst_node; + ice_sched_query_elem(hw, queue_sched_node->info.node_teid, &queue_sched_node->info); + ++ rte_free(buf); + return ret; + } + +diff --git a/dpdk/drivers/net/igc/igc_ethdev.c b/dpdk/drivers/net/igc/igc_ethdev.c +index 58c4f80927..690736b6d1 100644 +--- a/dpdk/drivers/net/igc/igc_ethdev.c ++++ b/dpdk/drivers/net/igc/igc_ethdev.c +@@ -2853,7 +2853,7 @@ eth_igc_timesync_disable(struct rte_eth_dev *dev) + IGC_WRITE_REG(hw, IGC_TSYNCRXCTL, 0); + + val = IGC_READ_REG(hw, IGC_RXPBS); +- val &= IGC_RXPBS_CFG_TS_EN; ++ val &= ~IGC_RXPBS_CFG_TS_EN; + IGC_WRITE_REG(hw, IGC_RXPBS, val); + + val = IGC_READ_REG(hw, IGC_SRRCTL(0)); +diff --git a/dpdk/drivers/net/ionic/ionic_ethdev.c b/dpdk/drivers/net/ionic/ionic_ethdev.c +index 340fd0cd59..4ec9598b8e 100644 +--- a/dpdk/drivers/net/ionic/ionic_ethdev.c ++++ b/dpdk/drivers/net/ionic/ionic_ethdev.c +@@ -561,7 +561,7 @@ ionic_dev_rss_reta_query(struct rte_eth_dev *eth_dev, + struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); + struct ionic_adapter *adapter = lif->adapter; + struct ionic_identity *ident = &adapter->ident; +- int i, num; ++ int i, j, num; + uint16_t tbl_sz = rte_le_to_cpu_16(ident->lif.eth.rss_ind_tbl_sz); + + IONIC_PRINT_CALL(); +@@ -582,9 +582,10 @@ ionic_dev_rss_reta_query(struct rte_eth_dev *eth_dev, + num = reta_size / RTE_ETH_RETA_GROUP_SIZE; + + for (i = 0; i < num; i++) { +- memcpy(reta_conf->reta, +- &lif->rss_ind_tbl[i * RTE_ETH_RETA_GROUP_SIZE], +- RTE_ETH_RETA_GROUP_SIZE); ++ for (j = 0; j < RTE_ETH_RETA_GROUP_SIZE; j++) { ++ reta_conf->reta[j] = ++ lif->rss_ind_tbl[(i * RTE_ETH_RETA_GROUP_SIZE) + j]; ++ } + reta_conf++; + } + +@@ -969,19 +970,21 @@ ionic_dev_close(struct rte_eth_dev *eth_dev) + + ionic_lif_stop(lif); + +- ionic_lif_free_queues(lif); +- + IONIC_PRINT(NOTICE, "Removing device %s", eth_dev->device->name); + if (adapter->intf->unconfigure_intr) + (*adapter->intf->unconfigure_intr)(adapter); + +- rte_eth_dev_destroy(eth_dev, eth_ionic_dev_uninit); +- + ionic_port_reset(adapter); + ionic_reset(adapter); ++ ++ ionic_lif_free_queues(lif); ++ ionic_lif_deinit(lif); ++ ionic_lif_free(lif); /* Does not free LIF object */ ++ + if (adapter->intf->unmap_bars) + (*adapter->intf->unmap_bars)(adapter); + ++ lif->adapter = NULL; + rte_free(adapter); + + return 0; +@@ -1058,21 +1061,18 @@ err: + static int + eth_ionic_dev_uninit(struct rte_eth_dev *eth_dev) + { +- struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); +- struct ionic_adapter *adapter = lif->adapter; +- + IONIC_PRINT_CALL(); + + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return 0; + +- adapter->lif = NULL; +- +- ionic_lif_deinit(lif); +- ionic_lif_free(lif); ++ if (eth_dev->state != RTE_ETH_DEV_UNUSED) ++ ionic_dev_close(eth_dev); + +- if (!(lif->state & IONIC_LIF_F_FW_RESET)) +- ionic_lif_reset(lif); ++ eth_dev->dev_ops = NULL; ++ eth_dev->rx_pkt_burst = NULL; ++ eth_dev->tx_pkt_burst = NULL; ++ eth_dev->tx_pkt_prepare = NULL; + + return 0; + } +@@ -1227,17 +1227,18 @@ eth_ionic_dev_remove(struct rte_device *rte_dev) + { + char name[RTE_ETH_NAME_MAX_LEN]; + struct rte_eth_dev *eth_dev; ++ int ret = 0; + + /* Adapter lookup is using the eth_dev name */ + snprintf(name, sizeof(name), "%s_lif", rte_dev->name); + + eth_dev = rte_eth_dev_allocated(name); + if (eth_dev) +- ionic_dev_close(eth_dev); ++ ret = rte_eth_dev_destroy(eth_dev, eth_ionic_dev_uninit); + else + IONIC_PRINT(DEBUG, "Cannot find device %s", rte_dev->name); + +- return 0; ++ return ret; + } + + RTE_LOG_REGISTER_DEFAULT(ionic_logtype, NOTICE); +diff --git a/dpdk/drivers/net/ionic/ionic_rxtx.c b/dpdk/drivers/net/ionic/ionic_rxtx.c +index b9e73b4871..170d3b0802 100644 +--- a/dpdk/drivers/net/ionic/ionic_rxtx.c ++++ b/dpdk/drivers/net/ionic/ionic_rxtx.c +@@ -26,38 +26,40 @@ + #include "ionic_logs.h" + + static void +-ionic_empty_array(void **array, uint32_t cnt, uint16_t idx) ++ionic_empty_array(void **array, uint32_t free_idx, uint32_t zero_idx) + { + uint32_t i; + +- for (i = idx; i < cnt; i++) ++ for (i = 0; i < free_idx; i++) + if (array[i]) + rte_pktmbuf_free_seg(array[i]); + +- memset(array, 0, sizeof(void *) * cnt); ++ memset(array, 0, sizeof(void *) * zero_idx); + } + + static void __rte_cold + ionic_tx_empty(struct ionic_tx_qcq *txq) + { + struct ionic_queue *q = &txq->qcq.q; ++ uint32_t info_len = q->num_descs * q->num_segs; + +- ionic_empty_array(q->info, q->num_descs * q->num_segs, 0); ++ ionic_empty_array(q->info, info_len, info_len); + } + + static void __rte_cold + ionic_rx_empty(struct ionic_rx_qcq *rxq) + { + struct ionic_queue *q = &rxq->qcq.q; ++ uint32_t info_len = q->num_descs * q->num_segs; + + /* + * Walk the full info array so that the clean up includes any + * fragments that were left dangling for later reuse + */ +- ionic_empty_array(q->info, q->num_descs * q->num_segs, 0); ++ ionic_empty_array(q->info, info_len, info_len); + +- ionic_empty_array((void **)rxq->mbs, +- IONIC_MBUF_BULK_ALLOC, rxq->mb_idx); ++ ionic_empty_array((void **)rxq->mbs, rxq->mb_idx, ++ IONIC_MBUF_BULK_ALLOC); + rxq->mb_idx = 0; + } + +@@ -752,7 +754,7 @@ ionic_dev_rx_descriptor_status(void *rx_queue, uint16_t offset) + { + struct ionic_rx_qcq *rxq = rx_queue; + struct ionic_qcq *qcq = &rxq->qcq; +- struct ionic_rxq_comp *cq_desc; ++ volatile struct ionic_rxq_comp *cq_desc; + uint16_t mask, head, tail, pos; + bool done_color; + +@@ -791,7 +793,7 @@ ionic_dev_tx_descriptor_status(void *tx_queue, uint16_t offset) + { + struct ionic_tx_qcq *txq = tx_queue; + struct ionic_qcq *qcq = &txq->qcq; +- struct ionic_txq_comp *cq_desc; ++ volatile struct ionic_txq_comp *cq_desc; + uint16_t mask, head, tail, pos, cq_pos; + bool done_color; + +diff --git a/dpdk/drivers/net/ionic/ionic_rxtx_sg.c b/dpdk/drivers/net/ionic/ionic_rxtx_sg.c +index ab8e56e91c..241b6f8587 100644 +--- a/dpdk/drivers/net/ionic/ionic_rxtx_sg.c ++++ b/dpdk/drivers/net/ionic/ionic_rxtx_sg.c +@@ -27,7 +27,8 @@ ionic_tx_flush_sg(struct ionic_tx_qcq *txq) + struct ionic_cq *cq = &txq->qcq.cq; + struct ionic_queue *q = &txq->qcq.q; + struct rte_mbuf *txm; +- struct ionic_txq_comp *cq_desc, *cq_desc_base = cq->base; ++ struct ionic_txq_comp *cq_desc_base = cq->base; ++ volatile struct ionic_txq_comp *cq_desc; + void **info; + uint32_t i; + +@@ -252,7 +253,7 @@ ionic_xmit_pkts_sg(void *tx_queue, struct rte_mbuf **tx_pkts, + */ + static __rte_always_inline void + ionic_rx_clean_one_sg(struct ionic_rx_qcq *rxq, +- struct ionic_rxq_comp *cq_desc, ++ volatile struct ionic_rxq_comp *cq_desc, + struct ionic_rx_service *rx_svc) + { + struct ionic_queue *q = &rxq->qcq.q; +@@ -438,7 +439,8 @@ ionic_rxq_service_sg(struct ionic_rx_qcq *rxq, uint32_t work_to_do, + struct ionic_cq *cq = &rxq->qcq.cq; + struct ionic_queue *q = &rxq->qcq.q; + struct ionic_rxq_desc *q_desc_base = q->base; +- struct ionic_rxq_comp *cq_desc, *cq_desc_base = cq->base; ++ struct ionic_rxq_comp *cq_desc_base = cq->base; ++ volatile struct ionic_rxq_comp *cq_desc; + uint32_t work_done = 0; + uint64_t then, now, hz, delta; + +diff --git a/dpdk/drivers/net/ionic/ionic_rxtx_simple.c b/dpdk/drivers/net/ionic/ionic_rxtx_simple.c +index 5f81856256..0992177afc 100644 +--- a/dpdk/drivers/net/ionic/ionic_rxtx_simple.c ++++ b/dpdk/drivers/net/ionic/ionic_rxtx_simple.c +@@ -27,7 +27,8 @@ ionic_tx_flush(struct ionic_tx_qcq *txq) + struct ionic_cq *cq = &txq->qcq.cq; + struct ionic_queue *q = &txq->qcq.q; + struct rte_mbuf *txm; +- struct ionic_txq_comp *cq_desc, *cq_desc_base = cq->base; ++ struct ionic_txq_comp *cq_desc_base = cq->base; ++ volatile struct ionic_txq_comp *cq_desc; + void **info; + + cq_desc = &cq_desc_base[cq->tail_idx]; +@@ -225,7 +226,7 @@ ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + */ + static __rte_always_inline void + ionic_rx_clean_one(struct ionic_rx_qcq *rxq, +- struct ionic_rxq_comp *cq_desc, ++ volatile struct ionic_rxq_comp *cq_desc, + struct ionic_rx_service *rx_svc) + { + struct ionic_queue *q = &rxq->qcq.q; +@@ -359,7 +360,8 @@ ionic_rxq_service(struct ionic_rx_qcq *rxq, uint32_t work_to_do, + struct ionic_cq *cq = &rxq->qcq.cq; + struct ionic_queue *q = &rxq->qcq.q; + struct ionic_rxq_desc *q_desc_base = q->base; +- struct ionic_rxq_comp *cq_desc, *cq_desc_base = cq->base; ++ struct ionic_rxq_comp *cq_desc_base = cq->base; ++ volatile struct ionic_rxq_comp *cq_desc; + uint32_t work_done = 0; + uint64_t then, now, hz, delta; + +diff --git a/dpdk/drivers/net/ixgbe/base/ixgbe_phy.c b/dpdk/drivers/net/ixgbe/base/ixgbe_phy.c +index 74c5db16fa..56267bb00d 100644 +--- a/dpdk/drivers/net/ixgbe/base/ixgbe_phy.c ++++ b/dpdk/drivers/net/ixgbe/base/ixgbe_phy.c +@@ -432,8 +432,7 @@ enum ixgbe_phy_type ixgbe_get_phy_type_from_id(u32 phy_id) + case TN1010_PHY_ID: + phy_type = ixgbe_phy_tn; + break; +- case X550_PHY_ID2: +- case X550_PHY_ID3: ++ case X550_PHY_ID: + case X540_PHY_ID: + phy_type = ixgbe_phy_aq; + break; +@@ -915,6 +914,10 @@ static s32 ixgbe_get_copper_speeds_supported(struct ixgbe_hw *hw) + hw->phy.speeds_supported |= IXGBE_LINK_SPEED_100_FULL; + + switch (hw->mac.type) { ++ case ixgbe_mac_X550: ++ hw->phy.speeds_supported |= IXGBE_LINK_SPEED_2_5GB_FULL; ++ hw->phy.speeds_supported |= IXGBE_LINK_SPEED_5GB_FULL; ++ break; + case ixgbe_mac_X550EM_x: + case ixgbe_mac_X550EM_a: + hw->phy.speeds_supported &= ~IXGBE_LINK_SPEED_100_FULL; +diff --git a/dpdk/drivers/net/ixgbe/base/ixgbe_type.h b/dpdk/drivers/net/ixgbe/base/ixgbe_type.h +index 1094df5891..f709681df2 100644 +--- a/dpdk/drivers/net/ixgbe/base/ixgbe_type.h ++++ b/dpdk/drivers/net/ixgbe/base/ixgbe_type.h +@@ -1664,6 +1664,7 @@ struct ixgbe_dmac_config { + #define TN1010_PHY_ID 0x00A19410 + #define TNX_FW_REV 0xB + #define X540_PHY_ID 0x01540200 ++#define X550_PHY_ID 0x01540220 + #define X550_PHY_ID2 0x01540223 + #define X550_PHY_ID3 0x01540221 + #define X557_PHY_ID 0x01540240 +@@ -1800,7 +1801,7 @@ enum { + /* VFRE bitmask */ + #define IXGBE_VFRE_ENABLE_ALL 0xFFFFFFFF + +-#define IXGBE_VF_INIT_TIMEOUT 200 /* Number of retries to clear RSTI */ ++#define IXGBE_VF_INIT_TIMEOUT 10000 /* Number of retries to clear RSTI */ + + /* RDHMPN and TDHMPN bitmasks */ + #define IXGBE_RDHMPN_RDICADDR 0x007FF800 +diff --git a/dpdk/drivers/net/ixgbe/base/ixgbe_vf.c b/dpdk/drivers/net/ixgbe/base/ixgbe_vf.c +index 5e3ae1b519..11dbbe2a86 100644 +--- a/dpdk/drivers/net/ixgbe/base/ixgbe_vf.c ++++ b/dpdk/drivers/net/ixgbe/base/ixgbe_vf.c +@@ -585,7 +585,7 @@ s32 ixgbe_check_mac_link_vf(struct ixgbe_hw *hw, ixgbe_link_speed *speed, + switch (links_reg & IXGBE_LINKS_SPEED_82599) { + case IXGBE_LINKS_SPEED_10G_82599: + *speed = IXGBE_LINK_SPEED_10GB_FULL; +- if (hw->mac.type >= ixgbe_mac_X550) { ++ if (hw->mac.type >= ixgbe_mac_X550_vf) { + if (links_reg & IXGBE_LINKS_SPEED_NON_STD) + *speed = IXGBE_LINK_SPEED_2_5GB_FULL; + } +@@ -595,7 +595,7 @@ s32 ixgbe_check_mac_link_vf(struct ixgbe_hw *hw, ixgbe_link_speed *speed, + break; + case IXGBE_LINKS_SPEED_100_82599: + *speed = IXGBE_LINK_SPEED_100_FULL; +- if (hw->mac.type == ixgbe_mac_X550) { ++ if (hw->mac.type == ixgbe_mac_X550_vf) { + if (links_reg & IXGBE_LINKS_SPEED_NON_STD) + *speed = IXGBE_LINK_SPEED_5GB_FULL; + } +@@ -603,7 +603,7 @@ s32 ixgbe_check_mac_link_vf(struct ixgbe_hw *hw, ixgbe_link_speed *speed, + case IXGBE_LINKS_SPEED_10_X550EM_A: + *speed = IXGBE_LINK_SPEED_UNKNOWN; + /* Since Reserved in older MAC's */ +- if (hw->mac.type >= ixgbe_mac_X550) ++ if (hw->mac.type >= ixgbe_mac_X550_vf) + *speed = IXGBE_LINK_SPEED_10_FULL; + break; + default: +diff --git a/dpdk/drivers/net/ixgbe/ixgbe_ethdev.c b/dpdk/drivers/net/ixgbe/ixgbe_ethdev.c +index d6cf00317e..a44497ce51 100644 +--- a/dpdk/drivers/net/ixgbe/ixgbe_ethdev.c ++++ b/dpdk/drivers/net/ixgbe/ixgbe_ethdev.c +@@ -1190,7 +1190,8 @@ eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev, void *init_params __rte_unused) + diag = ixgbe_validate_eeprom_checksum(hw, &csum); + if (diag != IXGBE_SUCCESS) { + PMD_INIT_LOG(ERR, "The EEPROM checksum is not valid: %d", diag); +- return -EIO; ++ ret = -EIO; ++ goto err_exit; + } + + #ifdef RTE_LIBRTE_IXGBE_BYPASS +@@ -1228,7 +1229,8 @@ eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev, void *init_params __rte_unused) + PMD_INIT_LOG(ERR, "Unsupported SFP+ Module"); + if (diag) { + PMD_INIT_LOG(ERR, "Hardware Initialization Failure: %d", diag); +- return -EIO; ++ ret = -EIO; ++ goto err_exit; + } + + /* Reset the hw statistics */ +@@ -1248,7 +1250,8 @@ eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev, void *init_params __rte_unused) + "Failed to allocate %u bytes needed to store " + "MAC addresses", + RTE_ETHER_ADDR_LEN * hw->mac.num_rar_entries); +- return -ENOMEM; ++ ret = -ENOMEM; ++ goto err_exit; + } + /* Copy the permanent MAC address */ + rte_ether_addr_copy((struct rte_ether_addr *)hw->mac.perm_addr, +@@ -1263,7 +1266,8 @@ eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev, void *init_params __rte_unused) + RTE_ETHER_ADDR_LEN * IXGBE_VMDQ_NUM_UC_MAC); + rte_free(eth_dev->data->mac_addrs); + eth_dev->data->mac_addrs = NULL; +- return -ENOMEM; ++ ret = -ENOMEM; ++ goto err_exit; + } + + /* initialize the vfta */ +@@ -1347,6 +1351,11 @@ err_pf_host_init: + eth_dev->data->mac_addrs = NULL; + rte_free(eth_dev->data->hash_mac_addrs); + eth_dev->data->hash_mac_addrs = NULL; ++err_exit: ++#ifdef RTE_LIB_SECURITY ++ rte_free(eth_dev->security_ctx); ++ eth_dev->security_ctx = NULL; ++#endif + return ret; + } + +@@ -4280,6 +4289,9 @@ ixgbe_dev_link_update_share(struct rte_eth_dev *dev, + int wait = 1; + u32 esdp_reg; + ++ if (rte_eal_process_type() != RTE_PROC_PRIMARY) ++ return -1; ++ + memset(&link, 0, sizeof(link)); + link.link_status = RTE_ETH_LINK_DOWN; + link.link_speed = RTE_ETH_SPEED_NUM_NONE; +@@ -4654,14 +4666,20 @@ ixgbe_dev_interrupt_action(struct rte_eth_dev *dev) + timeout = IXGBE_LINK_DOWN_CHECK_TIMEOUT; + + ixgbe_dev_link_status_print(dev); +- if (rte_eal_alarm_set(timeout * 1000, +- ixgbe_dev_interrupt_delayed_handler, (void *)dev) < 0) +- PMD_DRV_LOG(ERR, "Error setting alarm"); +- else { +- /* remember original mask */ +- intr->mask_original = intr->mask; +- /* only disable lsc interrupt */ +- intr->mask &= ~IXGBE_EIMS_LSC; ++ ++ /* Don't program delayed handler if LSC interrupt is disabled. ++ * It means one is already programmed. ++ */ ++ if (intr->mask & IXGBE_EIMS_LSC) { ++ if (rte_eal_alarm_set(timeout * 1000, ++ ixgbe_dev_interrupt_delayed_handler, (void *)dev) < 0) ++ PMD_DRV_LOG(ERR, "Error setting alarm"); ++ else { ++ /* remember original mask */ ++ intr->mask_original = intr->mask; ++ /* only disable lsc interrupt */ ++ intr->mask &= ~IXGBE_EIMS_LSC; ++ } + } + } + +diff --git a/dpdk/drivers/net/ixgbe/ixgbe_rxtx.c b/dpdk/drivers/net/ixgbe/ixgbe_rxtx.c +index 90b0a7004f..f6c17d4efb 100644 +--- a/dpdk/drivers/net/ixgbe/ixgbe_rxtx.c ++++ b/dpdk/drivers/net/ixgbe/ixgbe_rxtx.c +@@ -5844,6 +5844,25 @@ ixgbevf_dev_rx_init(struct rte_eth_dev *dev) + IXGBE_PSRTYPE_RQPL_SHIFT; + IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype); + ++ /* Initialize the rss for x550_vf cards if enabled */ ++ switch (hw->mac.type) { ++ case ixgbe_mac_X550_vf: ++ case ixgbe_mac_X550EM_x_vf: ++ case ixgbe_mac_X550EM_a_vf: ++ switch (dev->data->dev_conf.rxmode.mq_mode) { ++ case RTE_ETH_MQ_RX_RSS: ++ case RTE_ETH_MQ_RX_DCB_RSS: ++ case RTE_ETH_MQ_RX_VMDQ_RSS: ++ ixgbe_rss_configure(dev); ++ break; ++ default: ++ break; ++ } ++ break; ++ default: ++ break; ++ } ++ + ixgbe_set_rx_function(dev); + + return 0; +diff --git a/dpdk/drivers/net/mana/mana.c b/dpdk/drivers/net/mana/mana.c +index 781ed76139..65ca139be5 100644 +--- a/dpdk/drivers/net/mana/mana.c ++++ b/dpdk/drivers/net/mana/mana.c +@@ -296,8 +296,8 @@ mana_dev_info_get(struct rte_eth_dev *dev, + dev_info->min_rx_bufsize = MIN_RX_BUF_SIZE; + dev_info->max_rx_pktlen = MANA_MAX_MTU + RTE_ETHER_HDR_LEN; + +- dev_info->max_rx_queues = priv->max_rx_queues; +- dev_info->max_tx_queues = priv->max_tx_queues; ++ dev_info->max_rx_queues = RTE_MIN(priv->max_rx_queues, UINT16_MAX); ++ dev_info->max_tx_queues = RTE_MIN(priv->max_tx_queues, UINT16_MAX); + + dev_info->max_mac_addrs = MANA_MAX_MAC_ADDR; + dev_info->max_hash_mac_addrs = 0; +@@ -338,16 +338,20 @@ mana_dev_info_get(struct rte_eth_dev *dev, + + /* Buffer limits */ + dev_info->rx_desc_lim.nb_min = MIN_BUFFERS_PER_QUEUE; +- dev_info->rx_desc_lim.nb_max = priv->max_rx_desc; ++ dev_info->rx_desc_lim.nb_max = RTE_MIN(priv->max_rx_desc, UINT16_MAX); + dev_info->rx_desc_lim.nb_align = MIN_BUFFERS_PER_QUEUE; +- dev_info->rx_desc_lim.nb_seg_max = priv->max_recv_sge; +- dev_info->rx_desc_lim.nb_mtu_seg_max = priv->max_recv_sge; ++ dev_info->rx_desc_lim.nb_seg_max = ++ RTE_MIN(priv->max_recv_sge, UINT16_MAX); ++ dev_info->rx_desc_lim.nb_mtu_seg_max = ++ RTE_MIN(priv->max_recv_sge, UINT16_MAX); + + dev_info->tx_desc_lim.nb_min = MIN_BUFFERS_PER_QUEUE; +- dev_info->tx_desc_lim.nb_max = priv->max_tx_desc; ++ dev_info->tx_desc_lim.nb_max = RTE_MIN(priv->max_tx_desc, UINT16_MAX); + dev_info->tx_desc_lim.nb_align = MIN_BUFFERS_PER_QUEUE; +- dev_info->tx_desc_lim.nb_seg_max = priv->max_send_sge; +- dev_info->rx_desc_lim.nb_mtu_seg_max = priv->max_recv_sge; ++ dev_info->tx_desc_lim.nb_seg_max = ++ RTE_MIN(priv->max_send_sge, UINT16_MAX); ++ dev_info->tx_desc_lim.nb_mtu_seg_max = ++ RTE_MIN(priv->max_send_sge, UINT16_MAX); + + /* Speed */ + dev_info->speed_capa = RTE_ETH_LINK_SPEED_100G; +@@ -707,7 +711,7 @@ mana_dev_stats_reset(struct rte_eth_dev *dev __rte_unused) + static int + mana_get_ifname(const struct mana_priv *priv, char (*ifname)[IF_NAMESIZE]) + { +- int ret; ++ int ret = -ENODEV; + DIR *dir; + struct dirent *dent; + +@@ -1385,9 +1389,9 @@ mana_probe_port(struct ibv_device *ibdev, struct ibv_device_attr_ex *dev_attr, + priv->max_mr = dev_attr->orig_attr.max_mr; + priv->max_mr_size = dev_attr->orig_attr.max_mr_size; + +- DRV_LOG(INFO, "dev %s max queues %d desc %d sge %d", ++ DRV_LOG(INFO, "dev %s max queues %d desc %d sge %d mr %" PRIu64, + name, priv->max_rx_queues, priv->max_rx_desc, +- priv->max_send_sge); ++ priv->max_send_sge, priv->max_mr_size); + + rte_eth_copy_pci_info(eth_dev, pci_dev); + +diff --git a/dpdk/drivers/net/mana/mana.h b/dpdk/drivers/net/mana/mana.h +index 6836872dc2..822b8a1f15 100644 +--- a/dpdk/drivers/net/mana/mana.h ++++ b/dpdk/drivers/net/mana/mana.h +@@ -522,9 +522,9 @@ void mana_del_pmd_mr(struct mana_mr_cache *mr); + void mana_mempool_chunk_cb(struct rte_mempool *mp, void *opaque, + struct rte_mempool_memhdr *memhdr, unsigned int idx); + +-struct mana_mr_cache *mana_mr_btree_lookup(struct mana_mr_btree *bt, +- uint16_t *idx, +- uintptr_t addr, size_t len); ++int mana_mr_btree_lookup(struct mana_mr_btree *bt, uint16_t *idx, ++ uintptr_t addr, size_t len, ++ struct mana_mr_cache **cache); + int mana_mr_btree_insert(struct mana_mr_btree *bt, struct mana_mr_cache *entry); + int mana_mr_btree_init(struct mana_mr_btree *bt, int n, int socket); + void mana_mr_btree_free(struct mana_mr_btree *bt); +diff --git a/dpdk/drivers/net/mana/mr.c b/dpdk/drivers/net/mana/mr.c +index b8e6ea0bbf..eb6d073a95 100644 +--- a/dpdk/drivers/net/mana/mr.c ++++ b/dpdk/drivers/net/mana/mr.c +@@ -40,7 +40,7 @@ mana_new_pmd_mr(struct mana_mr_btree *local_tree, struct mana_priv *priv, + struct ibv_mr *ibv_mr; + struct mana_range ranges[pool->nb_mem_chunks]; + uint32_t i; +- struct mana_mr_cache *mr; ++ struct mana_mr_cache mr; + int ret; + + rte_mempool_mem_iter(pool, mana_mempool_chunk_cb, ranges); +@@ -75,14 +75,13 @@ mana_new_pmd_mr(struct mana_mr_btree *local_tree, struct mana_priv *priv, + DP_LOG(DEBUG, "MR lkey %u addr %p len %zu", + ibv_mr->lkey, ibv_mr->addr, ibv_mr->length); + +- mr = rte_calloc("MANA MR", 1, sizeof(*mr), 0); +- mr->lkey = ibv_mr->lkey; +- mr->addr = (uintptr_t)ibv_mr->addr; +- mr->len = ibv_mr->length; +- mr->verb_obj = ibv_mr; ++ mr.lkey = ibv_mr->lkey; ++ mr.addr = (uintptr_t)ibv_mr->addr; ++ mr.len = ibv_mr->length; ++ mr.verb_obj = ibv_mr; + + rte_spinlock_lock(&priv->mr_btree_lock); +- ret = mana_mr_btree_insert(&priv->mr_btree, mr); ++ ret = mana_mr_btree_insert(&priv->mr_btree, &mr); + rte_spinlock_unlock(&priv->mr_btree_lock); + if (ret) { + ibv_dereg_mr(ibv_mr); +@@ -90,7 +89,7 @@ mana_new_pmd_mr(struct mana_mr_btree *local_tree, struct mana_priv *priv, + return ret; + } + +- ret = mana_mr_btree_insert(local_tree, mr); ++ ret = mana_mr_btree_insert(local_tree, &mr); + if (ret) { + /* Don't need to clean up MR as it's already + * in the global tree +@@ -138,8 +137,12 @@ mana_find_pmd_mr(struct mana_mr_btree *local_mr_btree, struct mana_priv *priv, + + try_again: + /* First try to find the MR in local queue tree */ +- mr = mana_mr_btree_lookup(local_mr_btree, &idx, +- (uintptr_t)mbuf->buf_addr, mbuf->buf_len); ++ ret = mana_mr_btree_lookup(local_mr_btree, &idx, ++ (uintptr_t)mbuf->buf_addr, mbuf->buf_len, ++ &mr); ++ if (ret) ++ return NULL; ++ + if (mr) { + DP_LOG(DEBUG, "Local mr lkey %u addr 0x%" PRIxPTR " len %zu", + mr->lkey, mr->addr, mr->len); +@@ -148,11 +151,14 @@ try_again: + + /* If not found, try to find the MR in global tree */ + rte_spinlock_lock(&priv->mr_btree_lock); +- mr = mana_mr_btree_lookup(&priv->mr_btree, &idx, +- (uintptr_t)mbuf->buf_addr, +- mbuf->buf_len); ++ ret = mana_mr_btree_lookup(&priv->mr_btree, &idx, ++ (uintptr_t)mbuf->buf_addr, ++ mbuf->buf_len, &mr); + rte_spinlock_unlock(&priv->mr_btree_lock); + ++ if (ret) ++ return NULL; ++ + /* If found in the global tree, add it to the local tree */ + if (mr) { + ret = mana_mr_btree_insert(local_mr_btree, mr); +@@ -228,22 +234,23 @@ mana_mr_btree_expand(struct mana_mr_btree *bt, int n) + /* + * Look for a region of memory in MR cache. + */ +-struct mana_mr_cache * +-mana_mr_btree_lookup(struct mana_mr_btree *bt, uint16_t *idx, +- uintptr_t addr, size_t len) ++int mana_mr_btree_lookup(struct mana_mr_btree *bt, uint16_t *idx, ++ uintptr_t addr, size_t len, ++ struct mana_mr_cache **cache) + { + struct mana_mr_cache *table; + uint16_t n; + uint16_t base = 0; + int ret; + +- n = bt->len; ++ *cache = NULL; + ++ n = bt->len; + /* Try to double the cache if it's full */ + if (n == bt->size) { + ret = mana_mr_btree_expand(bt, bt->size << 1); + if (ret) +- return NULL; ++ return ret; + } + + table = bt->table; +@@ -262,14 +269,16 @@ mana_mr_btree_lookup(struct mana_mr_btree *bt, uint16_t *idx, + + *idx = base; + +- if (addr + len <= table[base].addr + table[base].len) +- return &table[base]; ++ if (addr + len <= table[base].addr + table[base].len) { ++ *cache = &table[base]; ++ return 0; ++ } + + DP_LOG(DEBUG, + "addr 0x%" PRIxPTR " len %zu idx %u sum 0x%" PRIxPTR " not found", + addr, len, *idx, addr + len); + +- return NULL; ++ return 0; + } + + int +@@ -314,14 +323,21 @@ mana_mr_btree_insert(struct mana_mr_btree *bt, struct mana_mr_cache *entry) + struct mana_mr_cache *table; + uint16_t idx = 0; + uint16_t shift; ++ int ret; ++ ++ ret = mana_mr_btree_lookup(bt, &idx, entry->addr, entry->len, &table); ++ if (ret) ++ return ret; + +- if (mana_mr_btree_lookup(bt, &idx, entry->addr, entry->len)) { ++ if (table) { + DP_LOG(DEBUG, "Addr 0x%" PRIxPTR " len %zu exists in btree", + entry->addr, entry->len); + return 0; + } + + if (bt->len >= bt->size) { ++ DP_LOG(ERR, "Btree overflow detected len %u size %u", ++ bt->len, bt->size); + bt->overflow = 1; + return -1; + } +diff --git a/dpdk/drivers/net/memif/rte_eth_memif.c b/dpdk/drivers/net/memif/rte_eth_memif.c +index 7cc8c0da91..18377d9caf 100644 +--- a/dpdk/drivers/net/memif/rte_eth_memif.c ++++ b/dpdk/drivers/net/memif/rte_eth_memif.c +@@ -265,8 +265,6 @@ memif_free_stored_mbufs(struct pmd_process_private *proc_private, struct memif_q + cur_tail = __atomic_load_n(&ring->tail, __ATOMIC_ACQUIRE); + while (mq->last_tail != cur_tail) { + RTE_MBUF_PREFETCH_TO_FREE(mq->buffers[(mq->last_tail + 1) & mask]); +- /* Decrement refcnt and free mbuf. (current segment) */ +- rte_mbuf_refcnt_update(mq->buffers[mq->last_tail & mask], -1); + rte_pktmbuf_free_seg(mq->buffers[mq->last_tail & mask]); + mq->last_tail++; + } +@@ -684,7 +682,7 @@ eth_memif_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) + n_free = __atomic_load_n(&ring->head, __ATOMIC_ACQUIRE) - slot; + } + +- uint8_t i; ++ uint16_t i; + struct rte_mbuf **buf_tmp = bufs; + mbuf_head = *buf_tmp++; + struct rte_mempool *mp = mbuf_head->pool; +@@ -825,10 +823,6 @@ memif_tx_one_zc(struct pmd_process_private *proc_private, struct memif_queue *mq + next_in_chain: + /* store pointer to mbuf to free it later */ + mq->buffers[slot & mask] = mbuf; +- /* Increment refcnt to make sure the buffer is not freed before server +- * receives it. (current segment) +- */ +- rte_mbuf_refcnt_update(mbuf, 1); + /* populate descriptor */ + d0 = &ring->desc[slot & mask]; + d0->length = rte_pktmbuf_data_len(mbuf); +diff --git a/dpdk/drivers/net/mlx5/hws/mlx5dr.h b/dpdk/drivers/net/mlx5/hws/mlx5dr.h +index d88f73ab57..f003d9f446 100644 +--- a/dpdk/drivers/net/mlx5/hws/mlx5dr.h ++++ b/dpdk/drivers/net/mlx5/hws/mlx5dr.h +@@ -80,6 +80,7 @@ enum mlx5dr_action_aso_ct_flags { + }; + + enum mlx5dr_match_template_flags { ++ MLX5DR_MATCH_TEMPLATE_FLAG_NONE = 0, + /* Allow relaxed matching by skipping derived dependent match fields. */ + MLX5DR_MATCH_TEMPLATE_FLAG_RELAXED_MATCH = 1, + }; +diff --git a/dpdk/drivers/net/mlx5/hws/mlx5dr_action.c b/dpdk/drivers/net/mlx5/hws/mlx5dr_action.c +index 862ee3e332..a068f100c5 100644 +--- a/dpdk/drivers/net/mlx5/hws/mlx5dr_action.c ++++ b/dpdk/drivers/net/mlx5/hws/mlx5dr_action.c +@@ -1465,7 +1465,9 @@ mlx5dr_action_handle_tunnel_l3_to_l2(struct mlx5dr_action *action, + + /* Create a full modify header action list in case shared */ + mlx5dr_action_prepare_decap_l3_actions(hdrs->sz, mh_data, &num_of_actions); +- mlx5dr_action_prepare_decap_l3_data(hdrs->data, mh_data, num_of_actions); ++ ++ if (action->flags & MLX5DR_ACTION_FLAG_SHARED) ++ mlx5dr_action_prepare_decap_l3_data(hdrs->data, mh_data, num_of_actions); + + /* All DecapL3 cases require the same max arg size */ + arg_obj = mlx5dr_arg_create_modify_header_arg(ctx, +@@ -1489,6 +1491,7 @@ mlx5dr_action_handle_tunnel_l3_to_l2(struct mlx5dr_action *action, + + action[i].modify_header.max_num_of_actions = num_of_actions; + action[i].modify_header.num_of_actions = num_of_actions; ++ action[i].modify_header.num_of_patterns = num_of_hdrs; + action[i].modify_header.arg_obj = arg_obj; + action[i].modify_header.pat_obj = pat_obj; + action[i].modify_header.require_reparse = +@@ -2547,6 +2550,7 @@ static void mlx5dr_action_destroy_hws(struct mlx5dr_action *action) + case MLX5DR_ACTION_TYP_ASO_CT: + case MLX5DR_ACTION_TYP_PUSH_VLAN: + case MLX5DR_ACTION_TYP_REMOVE_HEADER: ++ case MLX5DR_ACTION_TYP_VPORT: + mlx5dr_action_destroy_stcs(action); + break; + case MLX5DR_ACTION_TYP_DEST_ROOT: +@@ -2600,6 +2604,9 @@ static void mlx5dr_action_destroy_hws(struct mlx5dr_action *action) + if (action->ipv6_route_ext.action[i]) + mlx5dr_action_destroy(action->ipv6_route_ext.action[i]); + break; ++ default: ++ DR_LOG(ERR, "Not supported action type: %d", action->type); ++ assert(false); + } + } + +diff --git a/dpdk/drivers/net/mlx5/hws/mlx5dr_cmd.c b/dpdk/drivers/net/mlx5/hws/mlx5dr_cmd.c +index 876a47147d..0fb764df32 100644 +--- a/dpdk/drivers/net/mlx5/hws/mlx5dr_cmd.c ++++ b/dpdk/drivers/net/mlx5/hws/mlx5dr_cmd.c +@@ -1027,7 +1027,8 @@ int mlx5dr_cmd_generate_wqe(struct ibv_context *ctx, + + ret = mlx5_glue->devx_general_cmd(ctx, in, sizeof(in), out, sizeof(out)); + if (ret) { +- DR_LOG(ERR, "Failed to write GTA WQE using FW"); ++ DR_LOG(ERR, "Failed to write GTA WQE using FW (syndrome: %#x)", ++ mlx5dr_cmd_get_syndrome(out)); + rte_errno = errno; + return rte_errno; + } +diff --git a/dpdk/drivers/net/mlx5/hws/mlx5dr_context.c b/dpdk/drivers/net/mlx5/hws/mlx5dr_context.c +index 15d53c578a..7f120b3b1b 100644 +--- a/dpdk/drivers/net/mlx5/hws/mlx5dr_context.c ++++ b/dpdk/drivers/net/mlx5/hws/mlx5dr_context.c +@@ -263,6 +263,7 @@ struct mlx5dr_context *mlx5dr_context_open(struct ibv_context *ibv_ctx, + free_caps: + simple_free(ctx->caps); + free_ctx: ++ pthread_spin_destroy(&ctx->ctrl_lock); + simple_free(ctx); + return NULL; + } +diff --git a/dpdk/drivers/net/mlx5/hws/mlx5dr_debug.c b/dpdk/drivers/net/mlx5/hws/mlx5dr_debug.c +index 11557bcab8..f11c81ffee 100644 +--- a/dpdk/drivers/net/mlx5/hws/mlx5dr_debug.c ++++ b/dpdk/drivers/net/mlx5/hws/mlx5dr_debug.c +@@ -150,7 +150,7 @@ mlx5dr_debug_dump_matcher_action_template(FILE *f, struct mlx5dr_matcher *matche + MLX5DR_DEBUG_RES_TYPE_MATCHER_ACTION_TEMPLATE, + (uint64_t)(uintptr_t)at, + (uint64_t)(uintptr_t)matcher, +- at->only_term ? 0 : 1, ++ at->only_term, + is_root ? 0 : at->num_of_action_stes, + at->num_actions); + if (ret < 0) { +diff --git a/dpdk/drivers/net/mlx5/hws/mlx5dr_definer.c b/dpdk/drivers/net/mlx5/hws/mlx5dr_definer.c +index 0b60479406..031e87bc0c 100644 +--- a/dpdk/drivers/net/mlx5/hws/mlx5dr_definer.c ++++ b/dpdk/drivers/net/mlx5/hws/mlx5dr_definer.c +@@ -8,8 +8,9 @@ + #define BAD_PORT 0xBAD + #define ETH_TYPE_IPV4_VXLAN 0x0800 + #define ETH_TYPE_IPV6_VXLAN 0x86DD +-#define ETH_VXLAN_DEFAULT_PORT 4789 +-#define IP_UDP_PORT_MPLS 6635 ++#define UDP_GTPU_PORT 2152 ++#define UDP_VXLAN_PORT 4789 ++#define UDP_PORT_MPLS 6635 + #define UDP_ROCEV2_PORT 4791 + #define DR_FLOW_LAYER_TUNNEL_NO_MPLS (MLX5_FLOW_LAYER_TUNNEL & ~MLX5_FLOW_LAYER_MPLS) + +@@ -41,6 +42,10 @@ + (bit_off))); \ + } while (0) + ++/* Getter function based on bit offset and mask, for 32bit DW*/ ++#define DR_GET_32(p, byte_off, bit_off, mask) \ ++ ((rte_be_to_cpu_32(*((const rte_be32_t *)(p) + ((byte_off) / 4))) >> (bit_off)) & (mask)) ++ + /* Setter function based on bit offset and mask */ + #define DR_SET(p, v, byte_off, bit_off, mask) \ + do { \ +@@ -158,7 +163,7 @@ struct mlx5dr_definer_conv_data { + X(SET, tcp_protocol, STE_TCP, rte_flow_item_tcp) \ + X(SET_BE16, tcp_src_port, v->hdr.src_port, rte_flow_item_tcp) \ + X(SET_BE16, tcp_dst_port, v->hdr.dst_port, rte_flow_item_tcp) \ +- X(SET, gtp_udp_port, RTE_GTPU_UDP_PORT, rte_flow_item_gtp) \ ++ X(SET, gtp_udp_port, UDP_GTPU_PORT, rte_flow_item_gtp) \ + X(SET_BE32, gtp_teid, v->hdr.teid, rte_flow_item_gtp) \ + X(SET, gtp_msg_type, v->hdr.msg_type, rte_flow_item_gtp) \ + X(SET, gtp_ext_flag, !!v->hdr.gtp_hdr_info, rte_flow_item_gtp) \ +@@ -166,8 +171,8 @@ struct mlx5dr_definer_conv_data { + X(SET, gtp_ext_hdr_pdu, v->hdr.type, rte_flow_item_gtp_psc) \ + X(SET, gtp_ext_hdr_qfi, v->hdr.qfi, rte_flow_item_gtp_psc) \ + X(SET, vxlan_flags, v->flags, rte_flow_item_vxlan) \ +- X(SET, vxlan_udp_port, ETH_VXLAN_DEFAULT_PORT, rte_flow_item_vxlan) \ +- X(SET, mpls_udp_port, IP_UDP_PORT_MPLS, rte_flow_item_mpls) \ ++ X(SET, vxlan_udp_port, UDP_VXLAN_PORT, rte_flow_item_vxlan) \ ++ X(SET, mpls_udp_port, UDP_PORT_MPLS, rte_flow_item_mpls) \ + X(SET, source_qp, v->queue, mlx5_rte_flow_item_sq) \ + X(SET, tag, v->data, rte_flow_item_tag) \ + X(SET, metadata, v->data, rte_flow_item_meta) \ +@@ -183,6 +188,8 @@ struct mlx5dr_definer_conv_data { + X(SET, ib_l4_udp_port, UDP_ROCEV2_PORT, rte_flow_item_ib_bth) \ + X(SET, ib_l4_opcode, v->hdr.opcode, rte_flow_item_ib_bth) \ + X(SET, ib_l4_bth_a, v->hdr.a, rte_flow_item_ib_bth) \ ++ X(SET, cvlan, STE_CVLAN, rte_flow_item_vlan) \ ++ X(SET_BE16, inner_type, v->inner_type, rte_flow_item_vlan) \ + + /* Item set function format */ + #define X(set_type, func_name, value, item_type) \ +@@ -377,7 +384,7 @@ mlx5dr_definer_integrity_set(struct mlx5dr_definer_fc *fc, + { + bool inner = (fc->fname == MLX5DR_DEFINER_FNAME_INTEGRITY_I); + const struct rte_flow_item_integrity *v = item_spec; +- uint32_t ok1_bits = 0; ++ uint32_t ok1_bits = DR_GET_32(tag, fc->byte_off, fc->bit_off, fc->bit_mask); + + if (v->l3_ok) + ok1_bits |= inner ? BIT(MLX5DR_DEFINER_OKS1_SECOND_L3_OK) : +@@ -769,6 +776,15 @@ mlx5dr_definer_conv_item_vlan(struct mlx5dr_definer_conv_data *cd, + struct mlx5dr_definer_fc *fc; + bool inner = cd->tunnel; + ++ if (!cd->relaxed) { ++ /* Mark packet as tagged (CVLAN) */ ++ fc = &cd->fc[DR_CALC_FNAME(VLAN_TYPE, inner)]; ++ fc->item_idx = item_idx; ++ fc->tag_mask_set = &mlx5dr_definer_ones_set; ++ fc->tag_set = &mlx5dr_definer_cvlan_set; ++ DR_CALC_SET(fc, eth_l2, first_vlan_qualifier, inner); ++ } ++ + if (!m) + return 0; + +@@ -777,8 +793,7 @@ mlx5dr_definer_conv_item_vlan(struct mlx5dr_definer_conv_data *cd, + return rte_errno; + } + +- if (!cd->relaxed || m->has_more_vlan) { +- /* Mark packet as tagged (CVLAN or SVLAN) even if TCI is not specified.*/ ++ if (m->has_more_vlan) { + fc = &cd->fc[DR_CALC_FNAME(VLAN_TYPE, inner)]; + fc->item_idx = item_idx; + fc->tag_mask_set = &mlx5dr_definer_ones_set; +@@ -796,7 +811,7 @@ mlx5dr_definer_conv_item_vlan(struct mlx5dr_definer_conv_data *cd, + if (m->hdr.eth_proto) { + fc = &cd->fc[DR_CALC_FNAME(ETH_TYPE, inner)]; + fc->item_idx = item_idx; +- fc->tag_set = &mlx5dr_definer_eth_type_set; ++ fc->tag_set = &mlx5dr_definer_inner_type_set; + DR_CALC_SET(fc, eth_l2, l3_ethertype, inner); + } + +@@ -1170,6 +1185,12 @@ mlx5dr_definer_conv_item_gtp(struct mlx5dr_definer_conv_data *cd, + const struct rte_flow_item_gtp *m = item->mask; + struct mlx5dr_definer_fc *fc; + ++ if (cd->tunnel) { ++ DR_LOG(ERR, "Inner GTPU item not supported"); ++ rte_errno = ENOTSUP; ++ return rte_errno; ++ } ++ + /* Overwrite GTPU dest port if not present */ + fc = &cd->fc[DR_CALC_FNAME(L4_DPORT, false)]; + if (!fc->tag_set && !cd->relaxed) { +@@ -1344,9 +1365,20 @@ mlx5dr_definer_conv_item_vxlan(struct mlx5dr_definer_conv_data *cd, + struct mlx5dr_definer_fc *fc; + bool inner = cd->tunnel; + +- /* In order to match on VXLAN we must match on ether_type, ip_protocol +- * and l4_dport. +- */ ++ if (inner) { ++ DR_LOG(ERR, "Inner VXLAN item not supported"); ++ rte_errno = ENOTSUP; ++ return rte_errno; ++ } ++ ++ /* In order to match on VXLAN we must match on ip_protocol and l4_dport */ ++ if (m && (m->rsvd0[0] != 0 || m->rsvd0[1] != 0 || m->rsvd0[2] != 0 || ++ m->rsvd1 != 0)) { ++ DR_LOG(ERR, "reserved fields are not supported"); ++ rte_errno = ENOTSUP; ++ return rte_errno; ++ } ++ + if (!cd->relaxed) { + fc = &cd->fc[DR_CALC_FNAME(IP_PROTOCOL, inner)]; + if (!fc->tag_set) { +@@ -1369,12 +1401,6 @@ mlx5dr_definer_conv_item_vxlan(struct mlx5dr_definer_conv_data *cd, + return 0; + + if (m->flags) { +- if (inner) { +- DR_LOG(ERR, "Inner VXLAN flags item not supported"); +- rte_errno = ENOTSUP; +- return rte_errno; +- } +- + fc = &cd->fc[MLX5DR_DEFINER_FNAME_VXLAN_FLAGS]; + fc->item_idx = item_idx; + fc->tag_set = &mlx5dr_definer_vxlan_flags_set; +@@ -1384,12 +1410,6 @@ mlx5dr_definer_conv_item_vxlan(struct mlx5dr_definer_conv_data *cd, + } + + if (!is_mem_zero(m->vni, 3)) { +- if (inner) { +- DR_LOG(ERR, "Inner VXLAN vni item not supported"); +- rte_errno = ENOTSUP; +- return rte_errno; +- } +- + fc = &cd->fc[MLX5DR_DEFINER_FNAME_VXLAN_VNI]; + fc->item_idx = item_idx; + fc->tag_set = &mlx5dr_definer_vxlan_vni_set; +@@ -2240,11 +2260,6 @@ mlx5dr_definer_conv_item_esp(struct mlx5dr_definer_conv_data *cd, + const struct rte_flow_item_esp *m = item->mask; + struct mlx5dr_definer_fc *fc; + +- if (!cd->ctx->caps->ipsec_offload) { +- rte_errno = ENOTSUP; +- return rte_errno; +- } +- + if (!m) + return 0; + if (m->hdr.spi) { +@@ -2842,7 +2857,7 @@ mlx5dr_definer_find_best_match_fit(struct mlx5dr_context *ctx, + return 0; + } + +- DR_LOG(ERR, "Unable to find supporting match/jumbo definer combination"); ++ DR_LOG(DEBUG, "Unable to find supporting match/jumbo definer combination"); + rte_errno = ENOTSUP; + return rte_errno; + } +@@ -2975,7 +2990,7 @@ mlx5dr_definer_calc_layout(struct mlx5dr_matcher *matcher, + /* Find the match definer layout for header layout match union */ + ret = mlx5dr_definer_find_best_match_fit(ctx, match_definer, match_hl); + if (ret) { +- DR_LOG(ERR, "Failed to create match definer from header layout"); ++ DR_LOG(DEBUG, "Failed to create match definer from header layout"); + goto free_fc; + } + +@@ -3191,15 +3206,18 @@ mlx5dr_definer_matcher_range_init(struct mlx5dr_context *ctx, + + /* Create optional range definers */ + for (i = 0; i < matcher->num_of_mt; i++) { +- if (!mt[i].fcr_sz) +- continue; +- + /* All must use range if requested */ +- if (i && !mt[i - 1].range_definer) { ++ bool is_range = !!mt[i].fcr_sz; ++ bool has_range = matcher->flags & MLX5DR_MATCHER_FLAGS_RANGE_DEFINER; ++ ++ if (i && ((is_range && !has_range) || (!is_range && has_range))) { + DR_LOG(ERR, "Using range and non range templates is not allowed"); + goto free_definers; + } + ++ if (!mt[i].fcr_sz) ++ continue; ++ + matcher->flags |= MLX5DR_MATCHER_FLAGS_RANGE_DEFINER; + /* Create definer without fcr binding, already binded */ + mt[i].range_definer = mlx5dr_definer_alloc(ctx, +@@ -3320,7 +3338,7 @@ int mlx5dr_definer_matcher_init(struct mlx5dr_context *ctx, + + ret = mlx5dr_definer_calc_layout(matcher, &match_layout, &range_layout); + if (ret) { +- DR_LOG(ERR, "Failed to calculate matcher definer layout"); ++ DR_LOG(DEBUG, "Failed to calculate matcher definer layout"); + return ret; + } + +diff --git a/dpdk/drivers/net/mlx5/hws/mlx5dr_matcher.c b/dpdk/drivers/net/mlx5/hws/mlx5dr_matcher.c +index 4ea161eae6..36be96c668 100644 +--- a/dpdk/drivers/net/mlx5/hws/mlx5dr_matcher.c ++++ b/dpdk/drivers/net/mlx5/hws/mlx5dr_matcher.c +@@ -340,7 +340,7 @@ static int mlx5dr_matcher_disconnect(struct mlx5dr_matcher *matcher) + return 0; + + matcher_reconnect: +- if (LIST_EMPTY(&tbl->head)) ++ if (LIST_EMPTY(&tbl->head) || prev_matcher == matcher) + LIST_INSERT_HEAD(&matcher->tbl->head, matcher, next); + else + LIST_INSERT_AFTER(prev_matcher, matcher, next); +@@ -807,7 +807,7 @@ static int mlx5dr_matcher_bind_mt(struct mlx5dr_matcher *matcher) + /* Calculate match, range and hash definers */ + ret = mlx5dr_definer_matcher_init(ctx, matcher); + if (ret) { +- DR_LOG(ERR, "Failed to set matcher templates with match definers"); ++ DR_LOG(DEBUG, "Failed to set matcher templates with match definers"); + return ret; + } + +@@ -1171,6 +1171,13 @@ static int mlx5dr_matcher_init_root(struct mlx5dr_matcher *matcher) + return rte_errno; + } + ++ ret = flow_hw_get_port_id_from_ctx(ctx, &flow_attr.port_id); ++ if (ret) { ++ DR_LOG(ERR, "Failed to get port id for dev %s", ctx->ibv_ctx->device->name); ++ rte_errno = EINVAL; ++ return rte_errno; ++ } ++ + mask = simple_calloc(1, MLX5_ST_SZ_BYTES(fte_match_param) + + offsetof(struct mlx5dv_flow_match_parameters, match_buf)); + if (!mask) { +diff --git a/dpdk/drivers/net/mlx5/hws/mlx5dr_pat_arg.h b/dpdk/drivers/net/mlx5/hws/mlx5dr_pat_arg.h +index bbe313102f..c4e0cbc843 100644 +--- a/dpdk/drivers/net/mlx5/hws/mlx5dr_pat_arg.h ++++ b/dpdk/drivers/net/mlx5/hws/mlx5dr_pat_arg.h +@@ -30,7 +30,6 @@ struct mlx5dr_pattern_cache { + struct mlx5dr_pattern_cache_item { + struct { + struct mlx5dr_devx_obj *pattern_obj; +- struct dr_icm_chunk *chunk; + uint8_t *data; + uint16_t num_of_actions; + } mh_data; +diff --git a/dpdk/drivers/net/mlx5/hws/mlx5dr_rule.c b/dpdk/drivers/net/mlx5/hws/mlx5dr_rule.c +index fa19303b91..cc7a30d6d0 100644 +--- a/dpdk/drivers/net/mlx5/hws/mlx5dr_rule.c ++++ b/dpdk/drivers/net/mlx5/hws/mlx5dr_rule.c +@@ -23,6 +23,9 @@ static void mlx5dr_rule_skip(struct mlx5dr_matcher *matcher, + *skip_rx = false; + *skip_tx = false; + ++ if (unlikely(mlx5dr_matcher_is_insert_by_idx(matcher))) ++ return; ++ + if (mt->item_flags & MLX5_FLOW_ITEM_REPRESENTED_PORT) { + v = items[mt->vport_item_id].spec; + vport = flow_hw_conv_port_id(v->port_id); +@@ -55,14 +58,16 @@ static void mlx5dr_rule_init_dep_wqe(struct mlx5dr_send_ring_dep_wqe *dep_wqe, + struct mlx5dr_rule *rule, + const struct rte_flow_item *items, + struct mlx5dr_match_template *mt, +- void *user_data) ++ struct mlx5dr_rule_attr *attr) + { + struct mlx5dr_matcher *matcher = rule->matcher; + struct mlx5dr_table *tbl = matcher->tbl; + bool skip_rx, skip_tx; + + dep_wqe->rule = rule; +- dep_wqe->user_data = user_data; ++ dep_wqe->user_data = attr->user_data; ++ dep_wqe->direct_index = mlx5dr_matcher_is_insert_by_idx(matcher) ? ++ attr->rule_idx : 0; + + if (!items) { /* rule update */ + dep_wqe->rtc_0 = rule->rtc_0; +@@ -145,8 +150,13 @@ mlx5dr_rule_save_delete_info(struct mlx5dr_rule *rule, + rule->tag_ptr = simple_calloc(2, sizeof(*rule->tag_ptr)); + assert(rule->tag_ptr); + +- src_tag = (uint8_t *)ste_attr->wqe_data->tag; +- memcpy(rule->tag_ptr[0].match, src_tag, MLX5DR_MATCH_TAG_SZ); ++ if (is_jumbo) ++ memcpy(rule->tag_ptr[0].jumbo, ste_attr->wqe_data->action, ++ MLX5DR_JUMBO_TAG_SZ); ++ else ++ memcpy(rule->tag_ptr[0].match, ste_attr->wqe_data->tag, ++ MLX5DR_MATCH_TAG_SZ); ++ + rule->tag_ptr[1].reserved[0] = ste_attr->send_attr.match_definer_id; + + /* Save range definer id and tag for delete */ +@@ -289,8 +299,8 @@ static int mlx5dr_rule_create_hws_fw_wqe(struct mlx5dr_rule *rule, + } + + mlx5dr_rule_create_init(rule, &ste_attr, &apply, false); +- mlx5dr_rule_init_dep_wqe(&match_wqe, rule, items, mt, attr->user_data); +- mlx5dr_rule_init_dep_wqe(&range_wqe, rule, items, mt, attr->user_data); ++ mlx5dr_rule_init_dep_wqe(&match_wqe, rule, items, mt, attr); ++ mlx5dr_rule_init_dep_wqe(&range_wqe, rule, items, mt, attr); + + ste_attr.direct_index = 0; + ste_attr.rtc_0 = match_wqe.rtc_0; +@@ -395,7 +405,7 @@ static int mlx5dr_rule_create_hws(struct mlx5dr_rule *rule, + * dep_wqe buffers (ctrl, data) are also reused for all STE writes. + */ + dep_wqe = mlx5dr_send_add_new_dep_wqe(queue); +- mlx5dr_rule_init_dep_wqe(dep_wqe, rule, items, mt, attr->user_data); ++ mlx5dr_rule_init_dep_wqe(dep_wqe, rule, items, mt, attr); + + ste_attr.wqe_ctrl = &dep_wqe->wqe_ctrl; + ste_attr.wqe_data = &dep_wqe->wqe_data; +@@ -457,8 +467,7 @@ static int mlx5dr_rule_create_hws(struct mlx5dr_rule *rule, + ste_attr.used_id_rtc_1 = &rule->rtc_1; + ste_attr.retry_rtc_0 = dep_wqe->retry_rtc_0; + ste_attr.retry_rtc_1 = dep_wqe->retry_rtc_1; +- ste_attr.direct_index = mlx5dr_matcher_is_insert_by_idx(matcher) ? +- attr->rule_idx : 0; ++ ste_attr.direct_index = dep_wqe->direct_index; + } else { + apply.next_direct_idx = --ste_attr.direct_index; + } +@@ -594,6 +603,13 @@ static int mlx5dr_rule_create_root(struct mlx5dr_rule *rule, + uint8_t match_criteria; + int ret; + ++ ret = flow_hw_get_port_id_from_ctx(ctx, &flow_attr.port_id); ++ if (ret) { ++ DR_LOG(ERR, "Failed to get port id for dev %s", ctx->ibv_ctx->device->name); ++ rte_errno = EINVAL; ++ return rte_errno; ++ } ++ + attr = simple_calloc(num_actions, sizeof(*attr)); + if (!attr) { + rte_errno = ENOMEM; +diff --git a/dpdk/drivers/net/mlx5/hws/mlx5dr_send.c b/dpdk/drivers/net/mlx5/hws/mlx5dr_send.c +index 622d574bfa..4c279ba42a 100644 +--- a/dpdk/drivers/net/mlx5/hws/mlx5dr_send.c ++++ b/dpdk/drivers/net/mlx5/hws/mlx5dr_send.c +@@ -50,6 +50,7 @@ void mlx5dr_send_all_dep_wqe(struct mlx5dr_send_engine *queue) + ste_attr.used_id_rtc_1 = &dep_wqe->rule->rtc_1; + ste_attr.wqe_ctrl = &dep_wqe->wqe_ctrl; + ste_attr.wqe_data = &dep_wqe->wqe_data; ++ ste_attr.direct_index = dep_wqe->direct_index; + + mlx5dr_send_ste(queue, &ste_attr); + +diff --git a/dpdk/drivers/net/mlx5/hws/mlx5dr_send.h b/dpdk/drivers/net/mlx5/hws/mlx5dr_send.h +index c1e8616f7e..0c89faa8a7 100644 +--- a/dpdk/drivers/net/mlx5/hws/mlx5dr_send.h ++++ b/dpdk/drivers/net/mlx5/hws/mlx5dr_send.h +@@ -106,6 +106,7 @@ struct mlx5dr_send_ring_dep_wqe { + uint32_t rtc_1; + uint32_t retry_rtc_0; + uint32_t retry_rtc_1; ++ uint32_t direct_index; + void *user_data; + }; + +@@ -202,8 +203,6 @@ struct mlx5dr_send_ste_attr { + * value to write in CPU endian format. + * @param addr + * Address to write to. +- * @param lock +- * Address of the lock to use for that UAR access. + */ + static __rte_always_inline void + mlx5dr_uar_write64_relaxed(uint64_t val, void *addr) +diff --git a/dpdk/drivers/net/mlx5/hws/mlx5dr_table.c b/dpdk/drivers/net/mlx5/hws/mlx5dr_table.c +index 55b9b20150..ab73017ade 100644 +--- a/dpdk/drivers/net/mlx5/hws/mlx5dr_table.c ++++ b/dpdk/drivers/net/mlx5/hws/mlx5dr_table.c +@@ -611,8 +611,7 @@ static int mlx5dr_table_set_default_miss_not_valid(struct mlx5dr_table *tbl, + + if (mlx5dr_table_is_root(tbl) || + (miss_tbl && mlx5dr_table_is_root(miss_tbl)) || +- (miss_tbl && miss_tbl->type != tbl->type) || +- (miss_tbl && tbl->default_miss.miss_tbl)) { ++ (miss_tbl && miss_tbl->type != tbl->type)) { + DR_LOG(ERR, "Invalid arguments"); + rte_errno = EINVAL; + return -rte_errno; +@@ -625,6 +624,7 @@ int mlx5dr_table_set_default_miss(struct mlx5dr_table *tbl, + struct mlx5dr_table *miss_tbl) + { + struct mlx5dr_context *ctx = tbl->ctx; ++ struct mlx5dr_table *old_miss_tbl; + int ret; + + ret = mlx5dr_table_set_default_miss_not_valid(tbl, miss_tbl); +@@ -632,15 +632,16 @@ int mlx5dr_table_set_default_miss(struct mlx5dr_table *tbl, + return ret; + + pthread_spin_lock(&ctx->ctrl_lock); +- ++ old_miss_tbl = tbl->default_miss.miss_tbl; + ret = mlx5dr_table_connect_to_miss_table(tbl, miss_tbl); + if (ret) + goto out; + ++ if (old_miss_tbl) ++ LIST_REMOVE(tbl, default_miss.next); ++ + if (miss_tbl) + LIST_INSERT_HEAD(&miss_tbl->default_miss.head, tbl, default_miss.next); +- else +- LIST_REMOVE(tbl, default_miss.next); + + pthread_spin_unlock(&ctx->ctrl_lock); + return 0; +diff --git a/dpdk/drivers/net/mlx5/linux/mlx5_ethdev_os.c b/dpdk/drivers/net/mlx5/linux/mlx5_ethdev_os.c +index dd5a0c546d..1d999ef66b 100644 +--- a/dpdk/drivers/net/mlx5/linux/mlx5_ethdev_os.c ++++ b/dpdk/drivers/net/mlx5/linux/mlx5_ethdev_os.c +@@ -671,7 +671,7 @@ mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) + ifr.ifr_data = (void *)ðpause; + ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr); + if (ret) { +- DRV_LOG(WARNING, ++ DRV_LOG(DEBUG, + "port %u ioctl(SIOCETHTOOL, ETHTOOL_GPAUSEPARAM) failed:" + " %s", + dev->data->port_id, strerror(rte_errno)); +@@ -1286,13 +1286,17 @@ _mlx5_os_read_dev_counters(struct rte_eth_dev *dev, int pf, uint64_t *stats) + struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; + unsigned int i; + struct ifreq ifr; +- unsigned int stats_sz = xstats_ctrl->stats_n * sizeof(uint64_t); ++ unsigned int max_stats_n = RTE_MAX(xstats_ctrl->stats_n, xstats_ctrl->stats_n_2nd); ++ unsigned int stats_sz = max_stats_n * sizeof(uint64_t); + unsigned char et_stat_buf[sizeof(struct ethtool_stats) + stats_sz]; + struct ethtool_stats *et_stats = (struct ethtool_stats *)et_stat_buf; + int ret; ++ uint16_t i_idx, o_idx; ++ uint32_t total_stats = xstats_n; + + et_stats->cmd = ETHTOOL_GSTATS; +- et_stats->n_stats = xstats_ctrl->stats_n; ++ /* Pass the maximum value, the driver may ignore this. */ ++ et_stats->n_stats = max_stats_n; + ifr.ifr_data = (caddr_t)et_stats; + if (pf >= 0) + ret = mlx5_ifreq_by_ifname(priv->sh->bond.ports[pf].ifname, +@@ -1305,21 +1309,34 @@ _mlx5_os_read_dev_counters(struct rte_eth_dev *dev, int pf, uint64_t *stats) + dev->data->port_id); + return ret; + } +- for (i = 0; i != xstats_ctrl->mlx5_stats_n; ++i) { +- if (xstats_ctrl->info[i].dev) +- continue; +- stats[i] += (uint64_t) +- et_stats->data[xstats_ctrl->dev_table_idx[i]]; ++ if (pf <= 0) { ++ for (i = 0; i != total_stats; i++) { ++ i_idx = xstats_ctrl->dev_table_idx[i]; ++ o_idx = xstats_ctrl->xstats_o_idx[i]; ++ if (i_idx == UINT16_MAX || xstats_ctrl->info[o_idx].dev) ++ continue; ++ stats[o_idx] += (uint64_t)et_stats->data[i_idx]; ++ } ++ } else { ++ for (i = 0; i != total_stats; i++) { ++ i_idx = xstats_ctrl->dev_table_idx_2nd[i]; ++ o_idx = xstats_ctrl->xstats_o_idx_2nd[i]; ++ if (i_idx == UINT16_MAX || xstats_ctrl->info[o_idx].dev) ++ continue; ++ stats[o_idx] += (uint64_t)et_stats->data[i_idx]; ++ } + } + return 0; + } + +-/** ++/* + * Read device counters. + * + * @param dev + * Pointer to Ethernet device. +- * @param[out] stats ++ * @param bond_master ++ * Indicate if the device is a bond master. ++ * @param stats + * Counters table output buffer. + * + * @return +@@ -1327,7 +1344,7 @@ _mlx5_os_read_dev_counters(struct rte_eth_dev *dev, int pf, uint64_t *stats) + * rte_errno is set. + */ + int +-mlx5_os_read_dev_counters(struct rte_eth_dev *dev, uint64_t *stats) ++mlx5_os_read_dev_counters(struct rte_eth_dev *dev, bool bond_master, uint64_t *stats) + { + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; +@@ -1335,7 +1352,7 @@ mlx5_os_read_dev_counters(struct rte_eth_dev *dev, uint64_t *stats) + + memset(stats, 0, sizeof(*stats) * xstats_ctrl->mlx5_stats_n); + /* Read ifreq counters. */ +- if (priv->master && priv->pf_bond >= 0) { ++ if (bond_master) { + /* Sum xstats from bonding device member ports. */ + for (i = 0; i < priv->sh->bond.n_port; i++) { + ret = _mlx5_os_read_dev_counters(dev, i, stats); +@@ -1347,13 +1364,17 @@ mlx5_os_read_dev_counters(struct rte_eth_dev *dev, uint64_t *stats) + if (ret) + return ret; + } +- /* Read IB counters. */ +- for (i = 0; i != xstats_ctrl->mlx5_stats_n; ++i) { ++ /* ++ * Read IB dev counters. ++ * The counters are unique per IB device but not per netdev IF. ++ * In bonding mode, getting the stats name only from 1 port is enough. ++ */ ++ for (i = xstats_ctrl->dev_cnt_start; i < xstats_ctrl->mlx5_stats_n; i++) { + if (!xstats_ctrl->info[i].dev) + continue; + /* return last xstats counter if fail to read. */ + if (mlx5_os_read_dev_stat(priv, xstats_ctrl->info[i].ctr_name, +- &stats[i]) == 0) ++ &stats[i]) == 0) + xstats_ctrl->xstats[i] = stats[i]; + else + stats[i] = xstats_ctrl->xstats[i]; +@@ -1361,18 +1382,24 @@ mlx5_os_read_dev_counters(struct rte_eth_dev *dev, uint64_t *stats) + return ret; + } + +-/** ++/* + * Query the number of statistics provided by ETHTOOL. + * + * @param dev + * Pointer to Ethernet device. ++ * @param bond_master ++ * Indicate if the device is a bond master. ++ * @param n_stats ++ * Pointer to number of stats to store. ++ * @param n_stats_sec ++ * Pointer to number of stats to store for the 2nd port of the bond. + * + * @return +- * Number of statistics on success, negative errno value otherwise and +- * rte_errno is set. ++ * 0 on success, negative errno value otherwise and rte_errno is set. + */ + int +-mlx5_os_get_stats_n(struct rte_eth_dev *dev) ++mlx5_os_get_stats_n(struct rte_eth_dev *dev, bool bond_master, ++ uint16_t *n_stats, uint16_t *n_stats_sec) + { + struct mlx5_priv *priv = dev->data->dev_private; + struct ethtool_drvinfo drvinfo; +@@ -1381,18 +1408,34 @@ mlx5_os_get_stats_n(struct rte_eth_dev *dev) + + drvinfo.cmd = ETHTOOL_GDRVINFO; + ifr.ifr_data = (caddr_t)&drvinfo; +- if (priv->master && priv->pf_bond >= 0) +- /* Bonding PF. */ ++ /* Bonding PFs. */ ++ if (bond_master) { + ret = mlx5_ifreq_by_ifname(priv->sh->bond.ports[0].ifname, + SIOCETHTOOL, &ifr); +- else ++ if (ret) { ++ DRV_LOG(WARNING, "bonding port %u unable to query number of" ++ " statistics for the 1st slave, %d", PORT_ID(priv), ret); ++ return ret; ++ } ++ *n_stats = drvinfo.n_stats; ++ ret = mlx5_ifreq_by_ifname(priv->sh->bond.ports[1].ifname, ++ SIOCETHTOOL, &ifr); ++ if (ret) { ++ DRV_LOG(WARNING, "bonding port %u unable to query number of" ++ " statistics for the 2nd slave, %d", PORT_ID(priv), ret); ++ return ret; ++ } ++ *n_stats_sec = drvinfo.n_stats; ++ } else { + ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr); +- if (ret) { +- DRV_LOG(WARNING, "port %u unable to query number of statistics", +- dev->data->port_id); +- return ret; ++ if (ret) { ++ DRV_LOG(WARNING, "port %u unable to query number of statistics", ++ PORT_ID(priv)); ++ return ret; ++ } ++ *n_stats = drvinfo.n_stats; + } +- return drvinfo.n_stats; ++ return 0; + } + + static const struct mlx5_counter_ctrl mlx5_counters_init[] = { +@@ -1576,7 +1619,104 @@ static const struct mlx5_counter_ctrl mlx5_counters_init[] = { + }, + }; + +-static const unsigned int xstats_n = RTE_DIM(mlx5_counters_init); ++const unsigned int xstats_n = RTE_DIM(mlx5_counters_init); ++ ++static int ++mlx5_os_get_stats_strings(struct rte_eth_dev *dev, bool bond_master, ++ struct ethtool_gstrings *strings, ++ uint32_t stats_n, uint32_t stats_n_2nd) ++{ ++ struct mlx5_priv *priv = dev->data->dev_private; ++ struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; ++ struct ifreq ifr; ++ int ret; ++ uint32_t i, j, idx; ++ ++ /* Ensure no out of bounds access before. */ ++ MLX5_ASSERT(xstats_n <= MLX5_MAX_XSTATS); ++ strings->cmd = ETHTOOL_GSTRINGS; ++ strings->string_set = ETH_SS_STATS; ++ strings->len = stats_n; ++ ifr.ifr_data = (caddr_t)strings; ++ if (bond_master) ++ ret = mlx5_ifreq_by_ifname(priv->sh->bond.ports[0].ifname, ++ SIOCETHTOOL, &ifr); ++ else ++ ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr); ++ if (ret) { ++ DRV_LOG(WARNING, "port %u unable to get statistic names with %d", ++ PORT_ID(priv), ret); ++ return ret; ++ } ++ /* Reorganize the orders to reduce the iterations. */ ++ for (j = 0; j < xstats_n; j++) { ++ xstats_ctrl->dev_table_idx[j] = UINT16_MAX; ++ for (i = 0; i < stats_n; i++) { ++ const char *curr_string = ++ (const char *)&strings->data[i * ETH_GSTRING_LEN]; ++ ++ if (!strcmp(mlx5_counters_init[j].ctr_name, curr_string)) { ++ idx = xstats_ctrl->mlx5_stats_n++; ++ xstats_ctrl->dev_table_idx[j] = i; ++ xstats_ctrl->xstats_o_idx[j] = idx; ++ xstats_ctrl->info[idx] = mlx5_counters_init[j]; ++ } ++ } ++ } ++ if (!bond_master) { ++ /* Add dev counters, unique per IB device. */ ++ xstats_ctrl->dev_cnt_start = xstats_ctrl->mlx5_stats_n; ++ for (j = 0; j != xstats_n; j++) { ++ if (mlx5_counters_init[j].dev) { ++ idx = xstats_ctrl->mlx5_stats_n++; ++ xstats_ctrl->info[idx] = mlx5_counters_init[j]; ++ xstats_ctrl->hw_stats[idx] = 0; ++ } ++ } ++ return 0; ++ } ++ ++ strings->len = stats_n_2nd; ++ ret = mlx5_ifreq_by_ifname(priv->sh->bond.ports[1].ifname, ++ SIOCETHTOOL, &ifr); ++ if (ret) { ++ DRV_LOG(WARNING, "port %u unable to get statistic names for 2nd slave with %d", ++ PORT_ID(priv), ret); ++ return ret; ++ } ++ /* The 2nd slave port may have a different strings set, based on the configuration. */ ++ for (j = 0; j != xstats_n; j++) { ++ xstats_ctrl->dev_table_idx_2nd[j] = UINT16_MAX; ++ for (i = 0; i != stats_n_2nd; i++) { ++ const char *curr_string = ++ (const char *)&strings->data[i * ETH_GSTRING_LEN]; ++ ++ if (!strcmp(mlx5_counters_init[j].ctr_name, curr_string)) { ++ xstats_ctrl->dev_table_idx_2nd[j] = i; ++ if (xstats_ctrl->dev_table_idx[j] != UINT16_MAX) { ++ /* Already mapped in the 1st slave port. */ ++ idx = xstats_ctrl->xstats_o_idx[j]; ++ xstats_ctrl->xstats_o_idx_2nd[j] = idx; ++ } else { ++ /* Append the new items to the end of the map. */ ++ idx = xstats_ctrl->mlx5_stats_n++; ++ xstats_ctrl->xstats_o_idx_2nd[j] = idx; ++ xstats_ctrl->info[idx] = mlx5_counters_init[j]; ++ } ++ } ++ } ++ } ++ /* Dev counters are always at the last now. */ ++ xstats_ctrl->dev_cnt_start = xstats_ctrl->mlx5_stats_n; ++ for (j = 0; j != xstats_n; j++) { ++ if (mlx5_counters_init[j].dev) { ++ idx = xstats_ctrl->mlx5_stats_n++; ++ xstats_ctrl->info[idx] = mlx5_counters_init[j]; ++ xstats_ctrl->hw_stats[idx] = 0; ++ } ++ } ++ return 0; ++} + + /** + * Init the structures to read device counters. +@@ -1590,76 +1730,44 @@ mlx5_os_stats_init(struct rte_eth_dev *dev) + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; + struct mlx5_stats_ctrl *stats_ctrl = &priv->stats_ctrl; +- unsigned int i; +- unsigned int j; +- struct ifreq ifr; + struct ethtool_gstrings *strings = NULL; +- unsigned int dev_stats_n; ++ uint16_t dev_stats_n = 0; ++ uint16_t dev_stats_n_2nd = 0; ++ unsigned int max_stats_n; + unsigned int str_sz; + int ret; ++ bool bond_master = (priv->master && priv->pf_bond >= 0); + + /* So that it won't aggregate for each init. */ + xstats_ctrl->mlx5_stats_n = 0; +- ret = mlx5_os_get_stats_n(dev); ++ ret = mlx5_os_get_stats_n(dev, bond_master, &dev_stats_n, &dev_stats_n_2nd); + if (ret < 0) { + DRV_LOG(WARNING, "port %u no extended statistics available", + dev->data->port_id); + return; + } +- dev_stats_n = ret; ++ max_stats_n = RTE_MAX(dev_stats_n, dev_stats_n_2nd); + /* Allocate memory to grab stat names and values. */ +- str_sz = dev_stats_n * ETH_GSTRING_LEN; ++ str_sz = max_stats_n * ETH_GSTRING_LEN; + strings = (struct ethtool_gstrings *) + mlx5_malloc(0, str_sz + sizeof(struct ethtool_gstrings), 0, + SOCKET_ID_ANY); + if (!strings) { + DRV_LOG(WARNING, "port %u unable to allocate memory for xstats", +- dev->data->port_id); ++ dev->data->port_id); + return; + } +- strings->cmd = ETHTOOL_GSTRINGS; +- strings->string_set = ETH_SS_STATS; +- strings->len = dev_stats_n; +- ifr.ifr_data = (caddr_t)strings; +- if (priv->master && priv->pf_bond >= 0) +- /* Bonding master. */ +- ret = mlx5_ifreq_by_ifname(priv->sh->bond.ports[0].ifname, +- SIOCETHTOOL, &ifr); +- else +- ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr); +- if (ret) { +- DRV_LOG(WARNING, "port %u unable to get statistic names", ++ ret = mlx5_os_get_stats_strings(dev, bond_master, strings, ++ dev_stats_n, dev_stats_n_2nd); ++ if (ret < 0) { ++ DRV_LOG(WARNING, "port %u failed to get the stats strings", + dev->data->port_id); + goto free; + } +- for (i = 0; i != dev_stats_n; ++i) { +- const char *curr_string = (const char *) +- &strings->data[i * ETH_GSTRING_LEN]; +- +- for (j = 0; j != xstats_n; ++j) { +- if (!strcmp(mlx5_counters_init[j].ctr_name, +- curr_string)) { +- unsigned int idx = xstats_ctrl->mlx5_stats_n++; +- +- xstats_ctrl->dev_table_idx[idx] = i; +- xstats_ctrl->info[idx] = mlx5_counters_init[j]; +- break; +- } +- } +- } +- /* Add dev counters. */ +- MLX5_ASSERT(xstats_ctrl->mlx5_stats_n <= MLX5_MAX_XSTATS); +- for (i = 0; i != xstats_n; ++i) { +- if (mlx5_counters_init[i].dev) { +- unsigned int idx = xstats_ctrl->mlx5_stats_n++; +- +- xstats_ctrl->info[idx] = mlx5_counters_init[i]; +- xstats_ctrl->hw_stats[idx] = 0; +- } +- } + xstats_ctrl->stats_n = dev_stats_n; ++ xstats_ctrl->stats_n_2nd = dev_stats_n_2nd; + /* Copy to base at first time. */ +- ret = mlx5_os_read_dev_counters(dev, xstats_ctrl->base); ++ ret = mlx5_os_read_dev_counters(dev, bond_master, xstats_ctrl->base); + if (ret) + DRV_LOG(ERR, "port %u cannot read device counters: %s", + dev->data->port_id, strerror(rte_errno)); +diff --git a/dpdk/drivers/net/mlx5/linux/mlx5_os.c b/dpdk/drivers/net/mlx5/linux/mlx5_os.c +index ae82e1e5d8..2241e84341 100644 +--- a/dpdk/drivers/net/mlx5/linux/mlx5_os.c ++++ b/dpdk/drivers/net/mlx5/linux/mlx5_os.c +@@ -455,15 +455,16 @@ __mlx5_discovery_misc5_cap(struct mlx5_priv *priv) + * Routine checks the reference counter and does actual + * resources creation/initialization only if counter is zero. + * +- * @param[in] priv +- * Pointer to the private device data structure. ++ * @param[in] eth_dev ++ * Pointer to the device. + * + * @return + * Zero on success, positive error code otherwise. + */ + static int +-mlx5_alloc_shared_dr(struct mlx5_priv *priv) ++mlx5_alloc_shared_dr(struct rte_eth_dev *eth_dev) + { ++ struct mlx5_priv *priv = eth_dev->data->dev_private; + struct mlx5_dev_ctx_shared *sh = priv->sh; + char s[MLX5_NAME_SIZE] __rte_unused; + int err; +@@ -578,6 +579,44 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv) + err = errno; + goto error; + } ++ ++ if (sh->config.dv_flow_en == 1) { ++ /* Query availability of metadata reg_c's. */ ++ if (!priv->sh->metadata_regc_check_flag) { ++ err = mlx5_flow_discover_mreg_c(eth_dev); ++ if (err < 0) { ++ err = -err; ++ goto error; ++ } ++ } ++ if (!mlx5_flow_ext_mreg_supported(eth_dev)) { ++ DRV_LOG(DEBUG, ++ "port %u extensive metadata register is not supported", ++ eth_dev->data->port_id); ++ if (sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { ++ DRV_LOG(ERR, "metadata mode %u is not supported " ++ "(no metadata registers available)", ++ sh->config.dv_xmeta_en); ++ err = ENOTSUP; ++ goto error; ++ } ++ } ++ if (sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && ++ mlx5_flow_ext_mreg_supported(eth_dev) && sh->dv_regc0_mask) { ++ sh->mreg_cp_tbl = mlx5_hlist_create(MLX5_FLOW_MREG_HNAME, ++ MLX5_FLOW_MREG_HTABLE_SZ, ++ false, true, eth_dev, ++ flow_dv_mreg_create_cb, ++ flow_dv_mreg_match_cb, ++ flow_dv_mreg_remove_cb, ++ flow_dv_mreg_clone_cb, ++ flow_dv_mreg_clone_free_cb); ++ if (!sh->mreg_cp_tbl) { ++ err = ENOMEM; ++ goto error; ++ } ++ } ++ } + #endif + if (!sh->tunnel_hub && sh->config.dv_miss_info) + err = mlx5_alloc_tunnel_hub(sh); +@@ -662,6 +701,10 @@ error: + mlx5_list_destroy(sh->dest_array_list); + sh->dest_array_list = NULL; + } ++ if (sh->mreg_cp_tbl) { ++ mlx5_hlist_destroy(sh->mreg_cp_tbl); ++ sh->mreg_cp_tbl = NULL; ++ } + return err; + } + +@@ -759,6 +802,10 @@ mlx5_os_free_shared_dr(struct mlx5_priv *priv) + mlx5_list_destroy(sh->dest_array_list); + sh->dest_array_list = NULL; + } ++ if (sh->mreg_cp_tbl) { ++ mlx5_hlist_destroy(sh->mreg_cp_tbl); ++ sh->mreg_cp_tbl = NULL; ++ } + } + + /** +@@ -1545,13 +1592,6 @@ err_secondary: + } + /* Create context for virtual machine VLAN workaround. */ + priv->vmwa_context = mlx5_vlan_vmwa_init(eth_dev, spawn->ifindex); +- if (sh->config.dv_flow_en) { +- err = mlx5_alloc_shared_dr(priv); +- if (err) +- goto error; +- if (mlx5_flex_item_port_init(eth_dev) < 0) +- goto error; +- } + if (mlx5_devx_obj_ops_en(sh)) { + priv->obj_ops = devx_obj_ops; + mlx5_queue_counter_id_prepare(eth_dev); +@@ -1602,6 +1642,13 @@ err_secondary: + goto error; + } + rte_rwlock_init(&priv->ind_tbls_lock); ++ if (sh->config.dv_flow_en) { ++ err = mlx5_alloc_shared_dr(eth_dev); ++ if (err) ++ goto error; ++ if (mlx5_flex_item_port_init(eth_dev) < 0) ++ goto error; ++ } + if (priv->sh->config.dv_flow_en == 2) { + #ifdef HAVE_MLX5_HWS_SUPPORT + if (priv->sh->config.dv_esw_en) { +@@ -1682,43 +1729,6 @@ err_secondary: + err = -err; + goto error; + } +- /* Query availability of metadata reg_c's. */ +- if (!priv->sh->metadata_regc_check_flag) { +- err = mlx5_flow_discover_mreg_c(eth_dev); +- if (err < 0) { +- err = -err; +- goto error; +- } +- } +- if (!mlx5_flow_ext_mreg_supported(eth_dev)) { +- DRV_LOG(DEBUG, +- "port %u extensive metadata register is not supported", +- eth_dev->data->port_id); +- if (sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { +- DRV_LOG(ERR, "metadata mode %u is not supported " +- "(no metadata registers available)", +- sh->config.dv_xmeta_en); +- err = ENOTSUP; +- goto error; +- } +- } +- if (sh->config.dv_flow_en && +- sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && +- mlx5_flow_ext_mreg_supported(eth_dev) && +- priv->sh->dv_regc0_mask) { +- priv->mreg_cp_tbl = mlx5_hlist_create(MLX5_FLOW_MREG_HNAME, +- MLX5_FLOW_MREG_HTABLE_SZ, +- false, true, eth_dev, +- flow_dv_mreg_create_cb, +- flow_dv_mreg_match_cb, +- flow_dv_mreg_remove_cb, +- flow_dv_mreg_clone_cb, +- flow_dv_mreg_clone_free_cb); +- if (!priv->mreg_cp_tbl) { +- err = ENOMEM; +- goto error; +- } +- } + rte_spinlock_init(&priv->shared_act_sl); + mlx5_flow_counter_mode_config(eth_dev); + mlx5_flow_drop_action_config(eth_dev); +@@ -1737,8 +1747,6 @@ error: + priv->sh->config.dv_esw_en) + flow_hw_destroy_vport_action(eth_dev); + #endif +- if (priv->mreg_cp_tbl) +- mlx5_hlist_destroy(priv->mreg_cp_tbl); + if (priv->sh) + mlx5_os_free_shared_dr(priv); + if (priv->nl_socket_route >= 0) +@@ -2429,8 +2437,7 @@ mlx5_os_pci_probe_pf(struct mlx5_common_device *cdev, + list[ns].info.master = 0; + list[ns].info.representor = 0; + } +- if (list[ns].info.port_name == bd) +- ns++; ++ ns++; + break; + case MLX5_PHYS_PORT_NAME_TYPE_PFHPF: + /* Fallthrough */ +@@ -2993,9 +3000,15 @@ mlx5_os_read_dev_stat(struct mlx5_priv *priv, const char *ctr_name, + + if (priv->sh) { + if (priv->q_counters != NULL && +- strcmp(ctr_name, "out_of_buffer") == 0) ++ strcmp(ctr_name, "out_of_buffer") == 0) { ++ if (rte_eal_process_type() == RTE_PROC_SECONDARY) { ++ DRV_LOG(WARNING, "Devx out_of_buffer counter is not supported in the secondary process"); ++ rte_errno = ENOTSUP; ++ return 1; ++ } + return mlx5_devx_cmd_queue_counter_query + (priv->q_counters, 0, (uint32_t *)stat); ++ } + MKSTR(path, "%s/ports/%d/hw_counters/%s", + priv->sh->ibdev_path, + priv->dev_port, +diff --git a/dpdk/drivers/net/mlx5/mlx5.c b/dpdk/drivers/net/mlx5/mlx5.c +index 3a182de248..8d4a0a3dda 100644 +--- a/dpdk/drivers/net/mlx5/mlx5.c ++++ b/dpdk/drivers/net/mlx5/mlx5.c +@@ -1689,7 +1689,8 @@ mlx5_init_shared_dev_registers(struct mlx5_dev_ctx_shared *sh) + } else { + DRV_LOG(DEBUG, "ASO register: NONE"); + } +- mlx5_init_hws_flow_tags_registers(sh); ++ if (sh->config.dv_flow_en == 2) ++ mlx5_init_hws_flow_tags_registers(sh); + } + + /** +@@ -2267,6 +2268,7 @@ mlx5_dev_close(struct rte_eth_dev *dev) + mlx5_indirect_list_handles_release(dev); + #ifdef HAVE_MLX5_HWS_SUPPORT + flow_hw_destroy_vport_action(dev); ++ /* dr context will be closed after mlx5_os_free_shared_dr. */ + flow_hw_resource_release(dev); + flow_hw_clear_port_info(dev); + #endif +@@ -2279,7 +2281,7 @@ mlx5_dev_close(struct rte_eth_dev *dev) + mlx5_free(priv->rxq_privs); + priv->rxq_privs = NULL; + } +- if (priv->txqs != NULL) { ++ if (priv->txqs != NULL && dev->data->tx_queues != NULL) { + /* XXX race condition if mlx5_tx_burst() is still running. */ + rte_delay_us_sleep(1000); + for (i = 0; (i != priv->txqs_n); ++i) +@@ -2288,16 +2290,20 @@ mlx5_dev_close(struct rte_eth_dev *dev) + priv->txqs = NULL; + } + mlx5_proc_priv_uninit(dev); ++ if (priv->drop_queue.hrxq) ++ mlx5_drop_action_destroy(dev); + if (priv->q_counters) { + mlx5_devx_cmd_destroy(priv->q_counters); + priv->q_counters = NULL; + } +- if (priv->drop_queue.hrxq) +- mlx5_drop_action_destroy(dev); +- if (priv->mreg_cp_tbl) +- mlx5_hlist_destroy(priv->mreg_cp_tbl); + mlx5_mprq_free_mp(dev); + mlx5_os_free_shared_dr(priv); ++#ifdef HAVE_MLX5_HWS_SUPPORT ++ if (priv->dr_ctx) { ++ claim_zero(mlx5dr_context_close(priv->dr_ctx)); ++ priv->dr_ctx = NULL; ++ } ++#endif + if (priv->rss_conf.rss_key != NULL) + mlx5_free(priv->rss_conf.rss_key); + if (priv->reta_idx != NULL) +diff --git a/dpdk/drivers/net/mlx5/mlx5.h b/dpdk/drivers/net/mlx5/mlx5.h +index 263ebead7f..0c81bcab9f 100644 +--- a/dpdk/drivers/net/mlx5/mlx5.h ++++ b/dpdk/drivers/net/mlx5/mlx5.h +@@ -263,16 +263,29 @@ struct mlx5_counter_ctrl { + struct mlx5_xstats_ctrl { + /* Number of device stats. */ + uint16_t stats_n; ++ /* Number of device stats, for the 2nd port in bond. */ ++ uint16_t stats_n_2nd; + /* Number of device stats identified by PMD. */ +- uint16_t mlx5_stats_n; ++ uint16_t mlx5_stats_n; ++ /* First device counters index. */ ++ uint16_t dev_cnt_start; + /* Index in the device counters table. */ + uint16_t dev_table_idx[MLX5_MAX_XSTATS]; ++ /* Index in the output table. */ ++ uint16_t xstats_o_idx[MLX5_MAX_XSTATS]; + uint64_t base[MLX5_MAX_XSTATS]; + uint64_t xstats[MLX5_MAX_XSTATS]; + uint64_t hw_stats[MLX5_MAX_XSTATS]; + struct mlx5_counter_ctrl info[MLX5_MAX_XSTATS]; ++ /* Index in the device counters table, for the 2nd port in bond. */ ++ uint16_t dev_table_idx_2nd[MLX5_MAX_XSTATS]; ++ /* Index in the output table, for the 2nd port in bond. */ ++ uint16_t xstats_o_idx_2nd[MLX5_MAX_XSTATS]; + }; + ++/* xstats array size. */ ++extern const unsigned int xstats_n; ++ + struct mlx5_stats_ctrl { + /* Base for imissed counter. */ + uint64_t imissed_base; +@@ -1473,6 +1486,8 @@ struct mlx5_dev_ctx_shared { + struct mlx5_hlist *flow_tbls; /* SWS flow table. */ + struct mlx5_hlist *groups; /* HWS flow group. */ + }; ++ struct mlx5_hlist *mreg_cp_tbl; ++ /* Hash table of Rx metadata register copy table. */ + struct mlx5_flow_tunnel_hub *tunnel_hub; + /* Direct Rules tables for FDB, NIC TX+RX */ + void *dr_drop_action; /* Pointer to DR drop action, any domain. */ +@@ -1862,11 +1877,7 @@ struct mlx5_priv { + rte_spinlock_t hw_ctrl_lock; + LIST_HEAD(hw_ctrl_flow, mlx5_hw_ctrl_flow) hw_ctrl_flows; + LIST_HEAD(hw_ext_ctrl_flow, mlx5_hw_ctrl_flow) hw_ext_ctrl_flows; +- struct rte_flow_template_table *hw_esw_sq_miss_root_tbl; +- struct rte_flow_template_table *hw_esw_sq_miss_tbl; +- struct rte_flow_template_table *hw_esw_zero_tbl; +- struct rte_flow_template_table *hw_tx_meta_cpy_tbl; +- struct rte_flow_template_table *hw_lacp_rx_tbl; ++ struct mlx5_flow_hw_ctrl_fdb *hw_ctrl_fdb; + struct rte_flow_pattern_template *hw_tx_repr_tagging_pt; + struct rte_flow_actions_template *hw_tx_repr_tagging_at; + struct rte_flow_template_table *hw_tx_repr_tagging_tbl; +@@ -1900,8 +1911,6 @@ struct mlx5_priv { + int nl_socket_rdma; /* Netlink socket (NETLINK_RDMA). */ + int nl_socket_route; /* Netlink socket (NETLINK_ROUTE). */ + struct mlx5_nl_vlan_vmwa_context *vmwa_context; /* VLAN WA context. */ +- struct mlx5_hlist *mreg_cp_tbl; +- /* Hash table of Rx metadata register copy table. */ + struct mlx5_mtr_config mtr_config; /* Meter configuration */ + uint8_t mtr_sfx_reg; /* Meter prefix-suffix flow match REG_C. */ + struct mlx5_legacy_flow_meters flow_meters; /* MTR list. */ +@@ -1989,6 +1998,30 @@ enum dr_dump_rec_type { + DR_DUMP_REC_TYPE_PMD_COUNTER = 4430, + }; + ++#if defined(HAVE_MLX5_HWS_SUPPORT) ++static __rte_always_inline struct mlx5_hw_q_job * ++flow_hw_job_get(struct mlx5_priv *priv, uint32_t queue) ++{ ++ MLX5_ASSERT(priv->hw_q[queue].job_idx <= priv->hw_q[queue].size); ++ return priv->hw_q[queue].job_idx ? ++ priv->hw_q[queue].job[--priv->hw_q[queue].job_idx] : NULL; ++} ++ ++static __rte_always_inline void ++flow_hw_job_put(struct mlx5_priv *priv, struct mlx5_hw_q_job *job, uint32_t queue) ++{ ++ MLX5_ASSERT(priv->hw_q[queue].job_idx < priv->hw_q[queue].size); ++ priv->hw_q[queue].job[priv->hw_q[queue].job_idx++] = job; ++} ++ ++struct mlx5_hw_q_job * ++mlx5_flow_action_job_init(struct mlx5_priv *priv, uint32_t queue, ++ const struct rte_flow_action_handle *handle, ++ void *user_data, void *query_data, ++ enum mlx5_hw_job_type type, ++ struct rte_flow_error *error); ++#endif ++ + /** + * Indicates whether HW objects operations can be created by DevX. + * +@@ -2131,8 +2164,9 @@ int mlx5_get_module_eeprom(struct rte_eth_dev *dev, + struct rte_dev_eeprom_info *info); + int mlx5_os_read_dev_stat(struct mlx5_priv *priv, + const char *ctr_name, uint64_t *stat); +-int mlx5_os_read_dev_counters(struct rte_eth_dev *dev, uint64_t *stats); +-int mlx5_os_get_stats_n(struct rte_eth_dev *dev); ++int mlx5_os_read_dev_counters(struct rte_eth_dev *dev, bool bond_master, uint64_t *stats); ++int mlx5_os_get_stats_n(struct rte_eth_dev *dev, bool bond_master, ++ uint16_t *n_stats, uint16_t *n_stats_sec); + void mlx5_os_stats_init(struct rte_eth_dev *dev); + int mlx5_get_flag_dropless_rq(struct rte_eth_dev *dev); + +@@ -2394,11 +2428,12 @@ int mlx5_aso_flow_hit_queue_poll_start(struct mlx5_dev_ctx_shared *sh); + int mlx5_aso_flow_hit_queue_poll_stop(struct mlx5_dev_ctx_shared *sh); + void mlx5_aso_queue_uninit(struct mlx5_dev_ctx_shared *sh, + enum mlx5_access_aso_opc_mod aso_opc_mod); +-int mlx5_aso_meter_update_by_wqe(struct mlx5_dev_ctx_shared *sh, uint32_t queue, +- struct mlx5_aso_mtr *mtr, struct mlx5_mtr_bulk *bulk, +- void *user_data, bool push); +-int mlx5_aso_mtr_wait(struct mlx5_dev_ctx_shared *sh, uint32_t queue, +- struct mlx5_aso_mtr *mtr); ++int mlx5_aso_meter_update_by_wqe(struct mlx5_priv *priv, uint32_t queue, ++ struct mlx5_aso_mtr *mtr, ++ struct mlx5_mtr_bulk *bulk, ++ struct mlx5_hw_q_job *job, bool push); ++int mlx5_aso_mtr_wait(struct mlx5_priv *priv, ++ struct mlx5_aso_mtr *mtr, bool is_tmpl_api); + int mlx5_aso_ct_update_by_wqe(struct mlx5_dev_ctx_shared *sh, uint32_t queue, + struct mlx5_aso_ct_action *ct, + const struct rte_flow_action_conntrack *profile, +diff --git a/dpdk/drivers/net/mlx5/mlx5_devx.c b/dpdk/drivers/net/mlx5/mlx5_devx.c +index 9fa400fc48..4f08ddf899 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_devx.c ++++ b/dpdk/drivers/net/mlx5/mlx5_devx.c +@@ -592,7 +592,8 @@ mlx5_rxq_devx_obj_new(struct mlx5_rxq_priv *rxq) + DRV_LOG(ERR, "Failed to create CQ."); + goto error; + } +- rxq_data->delay_drop = priv->config.std_delay_drop; ++ if (!rxq_data->shared || !rxq_ctrl->started) ++ rxq_data->delay_drop = priv->config.std_delay_drop; + /* Create RQ using DevX API. */ + ret = mlx5_rxq_create_devx_rq_resources(rxq); + if (ret) { +diff --git a/dpdk/drivers/net/mlx5/mlx5_ethdev.c b/dpdk/drivers/net/mlx5/mlx5_ethdev.c +index ab30e2c215..ec4bdd8af1 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_ethdev.c ++++ b/dpdk/drivers/net/mlx5/mlx5_ethdev.c +@@ -146,6 +146,12 @@ mlx5_dev_configure(struct rte_eth_dev *dev) + ret = mlx5_proc_priv_init(dev); + if (ret) + return ret; ++ ret = mlx5_dev_set_mtu(dev, dev->data->mtu); ++ if (ret) { ++ DRV_LOG(ERR, "port %u failed to set MTU to %u", dev->data->port_id, ++ dev->data->mtu); ++ return ret; ++ } + return 0; + } + +diff --git a/dpdk/drivers/net/mlx5/mlx5_flow.c b/dpdk/drivers/net/mlx5/mlx5_flow.c +index 85e8c77c81..fdc7c3ea54 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_flow.c ++++ b/dpdk/drivers/net/mlx5/mlx5_flow.c +@@ -1953,18 +1953,20 @@ mlx5_flow_rxq_dynf_set(struct rte_eth_dev *dev) + if (rxq == NULL || rxq->ctrl == NULL) + continue; + data = &rxq->ctrl->rxq; +- if (!rte_flow_dynf_metadata_avail()) { +- data->dynf_meta = 0; +- data->flow_meta_mask = 0; +- data->flow_meta_offset = -1; +- data->flow_meta_port_mask = 0; +- } else { +- data->dynf_meta = 1; +- data->flow_meta_mask = rte_flow_dynf_metadata_mask; +- data->flow_meta_offset = rte_flow_dynf_metadata_offs; +- data->flow_meta_port_mask = priv->sh->dv_meta_mask; ++ if (!data->shared || !rxq->ctrl->started) { ++ if (!rte_flow_dynf_metadata_avail()) { ++ data->dynf_meta = 0; ++ data->flow_meta_mask = 0; ++ data->flow_meta_offset = -1; ++ data->flow_meta_port_mask = 0; ++ } else { ++ data->dynf_meta = 1; ++ data->flow_meta_mask = rte_flow_dynf_metadata_mask; ++ data->flow_meta_offset = rte_flow_dynf_metadata_offs; ++ data->flow_meta_port_mask = priv->sh->dv_meta_mask; ++ } ++ data->mark_flag = mark_flag; + } +- data->mark_flag = mark_flag; + } + } + +@@ -2504,7 +2506,7 @@ int + flow_validate_modify_field_level(const struct rte_flow_action_modify_data *data, + struct rte_flow_error *error) + { +- if (data->level == 0) ++ if (data->level == 0 || data->field == RTE_FLOW_FIELD_FLEX_ITEM) + return 0; + if (data->field != RTE_FLOW_FIELD_TAG && + data->field != (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG) +@@ -5228,8 +5230,8 @@ flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, + }; + + /* Check if already registered. */ +- MLX5_ASSERT(priv->mreg_cp_tbl); +- entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); ++ MLX5_ASSERT(priv->sh->mreg_cp_tbl); ++ entry = mlx5_hlist_register(priv->sh->mreg_cp_tbl, mark_id, &ctx); + if (!entry) + return NULL; + return container_of(entry, struct mlx5_flow_mreg_copy_resource, +@@ -5268,10 +5270,10 @@ flow_mreg_del_copy_action(struct rte_eth_dev *dev, + return; + mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], + flow->rix_mreg_copy); +- if (!mcp_res || !priv->mreg_cp_tbl) ++ if (!mcp_res || !priv->sh->mreg_cp_tbl) + return; + MLX5_ASSERT(mcp_res->rix_flow); +- mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); ++ mlx5_hlist_unregister(priv->sh->mreg_cp_tbl, &mcp_res->hlist_ent); + flow->rix_mreg_copy = 0; + } + +@@ -5293,14 +5295,14 @@ flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) + uint32_t mark_id; + + /* Check if default flow is registered. */ +- if (!priv->mreg_cp_tbl) ++ if (!priv->sh->mreg_cp_tbl) + return; + mark_id = MLX5_DEFAULT_COPY_ID; + ctx.data = &mark_id; +- entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx); ++ entry = mlx5_hlist_lookup(priv->sh->mreg_cp_tbl, mark_id, &ctx); + if (!entry) + return; +- mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); ++ mlx5_hlist_unregister(priv->sh->mreg_cp_tbl, entry); + } + + /** +@@ -5338,7 +5340,7 @@ flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, + */ + mark_id = MLX5_DEFAULT_COPY_ID; + ctx.data = &mark_id; +- if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx)) ++ if (mlx5_hlist_lookup(priv->sh->mreg_cp_tbl, mark_id, &ctx)) + return 0; + mcp_res = flow_mreg_add_copy_action(dev, mark_id, error); + if (!mcp_res) +@@ -5492,6 +5494,7 @@ flow_hairpin_split(struct rte_eth_dev *dev, + } + break; + case RTE_FLOW_ACTION_TYPE_COUNT: ++ case RTE_FLOW_ACTION_TYPE_AGE: + if (encap) { + rte_memcpy(actions_tx, actions, + sizeof(struct rte_flow_action)); +@@ -5817,8 +5820,8 @@ flow_meter_split_prep(struct rte_eth_dev *dev, + struct mlx5_rte_flow_item_tag *tag_item_spec; + struct mlx5_rte_flow_item_tag *tag_item_mask; + uint32_t tag_id = 0; +- struct rte_flow_item *vlan_item_dst = NULL; +- const struct rte_flow_item *vlan_item_src = NULL; ++ bool vlan_actions; ++ struct rte_flow_item *orig_sfx_items = sfx_items; + const struct rte_flow_item *orig_items = items; + struct rte_flow_action *hw_mtr_action; + struct rte_flow_action *action_pre_head = NULL; +@@ -5835,6 +5838,7 @@ flow_meter_split_prep(struct rte_eth_dev *dev, + + /* Prepare the suffix subflow items. */ + tag_item = sfx_items++; ++ tag_item->type = (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_TAG; + for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { + int item_type = items->type; + +@@ -5857,10 +5861,13 @@ flow_meter_split_prep(struct rte_eth_dev *dev, + sfx_items++; + break; + case RTE_FLOW_ITEM_TYPE_VLAN: +- /* Determine if copy vlan item below. */ +- vlan_item_src = items; +- vlan_item_dst = sfx_items++; +- vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID; ++ /* ++ * Copy VLAN items in case VLAN actions are performed. ++ * If there are no VLAN actions, these items will be VOID. ++ */ ++ memcpy(sfx_items, items, sizeof(*sfx_items)); ++ sfx_items->type = (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_VLAN; ++ sfx_items++; + break; + default: + break; +@@ -5877,6 +5884,7 @@ flow_meter_split_prep(struct rte_eth_dev *dev, + tag_action = actions_pre++; + } + /* Prepare the actions for prefix and suffix flow. */ ++ vlan_actions = false; + for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { + struct rte_flow_action *action_cur = NULL; + +@@ -5907,16 +5915,7 @@ flow_meter_split_prep(struct rte_eth_dev *dev, + break; + case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: + case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: +- if (vlan_item_dst && vlan_item_src) { +- memcpy(vlan_item_dst, vlan_item_src, +- sizeof(*vlan_item_dst)); +- /* +- * Convert to internal match item, it is used +- * for vlan push and set vid. +- */ +- vlan_item_dst->type = (enum rte_flow_item_type) +- MLX5_RTE_FLOW_ITEM_TYPE_VLAN; +- } ++ vlan_actions = true; + break; + case RTE_FLOW_ACTION_TYPE_COUNT: + if (fm->def_policy) +@@ -5931,6 +5930,14 @@ flow_meter_split_prep(struct rte_eth_dev *dev, + actions_sfx++ : actions_pre++; + memcpy(action_cur, actions, sizeof(struct rte_flow_action)); + } ++ /* If there are no VLAN actions, convert VLAN items to VOID in suffix flow items. */ ++ if (!vlan_actions) { ++ struct rte_flow_item *it = orig_sfx_items; ++ ++ for (; it->type != RTE_FLOW_ITEM_TYPE_END; it++) ++ if (it->type == (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_VLAN) ++ it->type = RTE_FLOW_ITEM_TYPE_VOID; ++ } + /* Add end action to the actions. */ + actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; + if (priv->sh->meter_aso_en) { +@@ -6020,8 +6027,6 @@ flow_meter_split_prep(struct rte_eth_dev *dev, + tag_action->type = (enum rte_flow_action_type) + MLX5_RTE_FLOW_ACTION_TYPE_TAG; + tag_action->conf = set_tag; +- tag_item->type = (enum rte_flow_item_type) +- MLX5_RTE_FLOW_ITEM_TYPE_TAG; + tag_item->spec = tag_item_spec; + tag_item->last = NULL; + tag_item->mask = tag_item_mask; +@@ -6849,6 +6854,19 @@ flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, + &drop_split_info, error); + } + ++static int ++flow_count_vlan_items(const struct rte_flow_item items[]) ++{ ++ int items_n = 0; ++ ++ for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { ++ if (items->type == RTE_FLOW_ITEM_TYPE_VLAN || ++ items->type == (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_VLAN) ++ items_n++; ++ } ++ return items_n; ++} ++ + /** + * The splitting for meter feature. + * +@@ -6904,6 +6922,7 @@ flow_create_split_meter(struct rte_eth_dev *dev, + size_t act_size; + size_t item_size; + int actions_n = 0; ++ int vlan_items_n = 0; + int ret = 0; + + if (priv->mtr_en) +@@ -6963,9 +6982,11 @@ flow_create_split_meter(struct rte_eth_dev *dev, + act_size = (sizeof(struct rte_flow_action) * + (actions_n + METER_PREFIX_ACTION)) + + sizeof(struct mlx5_rte_flow_action_set_tag); +- /* Suffix items: tag, vlan, port id, end. */ +-#define METER_SUFFIX_ITEM 4 +- item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + ++ /* Flow can have multiple VLAN items. Account for them in suffix items. */ ++ vlan_items_n = flow_count_vlan_items(items); ++ /* Suffix items: tag, [vlans], port id, end. */ ++#define METER_SUFFIX_ITEM 3 ++ item_size = sizeof(struct rte_flow_item) * (METER_SUFFIX_ITEM + vlan_items_n) + + sizeof(struct mlx5_rte_flow_item_tag) * 2; + sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), + 0, SOCKET_ID_ANY); +diff --git a/dpdk/drivers/net/mlx5/mlx5_flow.h b/dpdk/drivers/net/mlx5/mlx5_flow.h +index 6dde9de688..bde7dc43a8 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_flow.h ++++ b/dpdk/drivers/net/mlx5/mlx5_flow.h +@@ -77,7 +77,7 @@ enum mlx5_indirect_type { + /* Now, the maximal ports will be supported is 16, action number is 32M. */ + #define MLX5_INDIRECT_ACT_CT_MAX_PORT 0x10 + +-#define MLX5_INDIRECT_ACT_CT_OWNER_SHIFT 22 ++#define MLX5_INDIRECT_ACT_CT_OWNER_SHIFT 25 + #define MLX5_INDIRECT_ACT_CT_OWNER_MASK (MLX5_INDIRECT_ACT_CT_MAX_PORT - 1) + + /* 29-31: type, 25-28: owner port, 0-24: index */ +@@ -1759,6 +1759,28 @@ flow_hw_get_reg_id_from_ctx(void *dr_ctx, + return REG_NON; + } + ++static __rte_always_inline int ++flow_hw_get_port_id_from_ctx(void *dr_ctx, uint32_t *port_val) ++{ ++#if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) ++ uint32_t port; ++ ++ MLX5_ETH_FOREACH_DEV(port, NULL) { ++ struct mlx5_priv *priv; ++ priv = rte_eth_devices[port].data->dev_private; ++ ++ if (priv->dr_ctx == dr_ctx) { ++ *port_val = port; ++ return 0; ++ } ++ } ++#else ++ RTE_SET_USED(dr_ctx); ++ RTE_SET_USED(port_val); ++#endif ++ return -EINVAL; ++} ++ + void flow_hw_set_port_info(struct rte_eth_dev *dev); + void flow_hw_clear_port_info(struct rte_eth_dev *dev); + int flow_hw_create_vport_action(struct rte_eth_dev *dev); +@@ -2446,6 +2468,25 @@ struct mlx5_flow_hw_ctrl_rx { + [MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_MAX]; + }; + ++/* Contains all templates required for control flow rules in FDB with HWS. */ ++struct mlx5_flow_hw_ctrl_fdb { ++ struct rte_flow_pattern_template *esw_mgr_items_tmpl; ++ struct rte_flow_actions_template *regc_jump_actions_tmpl; ++ struct rte_flow_template_table *hw_esw_sq_miss_root_tbl; ++ struct rte_flow_pattern_template *regc_sq_items_tmpl; ++ struct rte_flow_actions_template *port_actions_tmpl; ++ struct rte_flow_template_table *hw_esw_sq_miss_tbl; ++ struct rte_flow_pattern_template *port_items_tmpl; ++ struct rte_flow_actions_template *jump_one_actions_tmpl; ++ struct rte_flow_template_table *hw_esw_zero_tbl; ++ struct rte_flow_pattern_template *tx_meta_items_tmpl; ++ struct rte_flow_actions_template *tx_meta_actions_tmpl; ++ struct rte_flow_template_table *hw_tx_meta_cpy_tbl; ++ struct rte_flow_pattern_template *lacp_rx_items_tmpl; ++ struct rte_flow_actions_template *lacp_rx_actions_tmpl; ++ struct rte_flow_template_table *hw_lacp_rx_tbl; ++}; ++ + #define MLX5_CTRL_PROMISCUOUS (RTE_BIT32(0)) + #define MLX5_CTRL_ALL_MULTICAST (RTE_BIT32(1)) + #define MLX5_CTRL_BROADCAST (RTE_BIT32(2)) +diff --git a/dpdk/drivers/net/mlx5/mlx5_flow_aso.c b/dpdk/drivers/net/mlx5/mlx5_flow_aso.c +index f311443472..ab9eb21e01 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_flow_aso.c ++++ b/dpdk/drivers/net/mlx5/mlx5_flow_aso.c +@@ -792,7 +792,7 @@ mlx5_aso_mtr_sq_enqueue_single(struct mlx5_dev_ctx_shared *sh, + struct mlx5_aso_mtr *aso_mtr, + struct mlx5_mtr_bulk *bulk, + bool need_lock, +- void *user_data, ++ struct mlx5_hw_q_job *job, + bool push) + { + volatile struct mlx5_aso_wqe *wqe = NULL; +@@ -819,7 +819,7 @@ mlx5_aso_mtr_sq_enqueue_single(struct mlx5_dev_ctx_shared *sh, + rte_prefetch0(&sq->sq_obj.aso_wqes[(sq->head + 1) & mask]); + /* Fill next WQE. */ + fm = &aso_mtr->fm; +- sq->elts[sq->head & mask].mtr = user_data ? user_data : aso_mtr; ++ sq->elts[sq->head & mask].user_data = job ? job : (void *)aso_mtr; + if (aso_mtr->type == ASO_METER_INDIRECT) { + if (likely(sh->config.dv_flow_en == 2)) + pool = aso_mtr->pool; +@@ -897,24 +897,6 @@ mlx5_aso_mtr_sq_enqueue_single(struct mlx5_dev_ctx_shared *sh, + return 1; + } + +-static void +-mlx5_aso_mtrs_status_update(struct mlx5_aso_sq *sq, uint16_t aso_mtrs_nums) +-{ +- uint16_t size = 1 << sq->log_desc_n; +- uint16_t mask = size - 1; +- uint16_t i; +- struct mlx5_aso_mtr *aso_mtr = NULL; +- uint8_t exp_state = ASO_METER_WAIT; +- +- for (i = 0; i < aso_mtrs_nums; ++i) { +- aso_mtr = sq->elts[(sq->tail + i) & mask].mtr; +- MLX5_ASSERT(aso_mtr); +- (void)__atomic_compare_exchange_n(&aso_mtr->state, +- &exp_state, ASO_METER_READY, +- false, __ATOMIC_RELAXED, __ATOMIC_RELAXED); +- } +-} +- + static void + mlx5_aso_mtr_completion_handle(struct mlx5_aso_sq *sq, bool need_lock) + { +@@ -925,7 +907,7 @@ mlx5_aso_mtr_completion_handle(struct mlx5_aso_sq *sq, bool need_lock) + uint32_t idx; + uint32_t next_idx = cq->cq_ci & mask; + uint16_t max; +- uint16_t n = 0; ++ uint16_t i, n = 0; + int ret; + + if (need_lock) +@@ -957,7 +939,19 @@ mlx5_aso_mtr_completion_handle(struct mlx5_aso_sq *sq, bool need_lock) + cq->cq_ci++; + } while (1); + if (likely(n)) { +- mlx5_aso_mtrs_status_update(sq, n); ++ uint8_t exp_state = ASO_METER_WAIT; ++ struct mlx5_aso_mtr *aso_mtr; ++ __rte_unused bool verdict; ++ ++ for (i = 0; i < n; ++i) { ++ aso_mtr = sq->elts[(sq->tail + i) & mask].mtr; ++ MLX5_ASSERT(aso_mtr); ++ verdict = __atomic_compare_exchange_n(&aso_mtr->state, ++ &exp_state, ASO_METER_READY, ++ false, __ATOMIC_RELAXED, ++ __ATOMIC_RELAXED); ++ MLX5_ASSERT(verdict); ++ } + sq->tail += n; + rte_io_wmb(); + cq->cq_obj.db_rec[0] = rte_cpu_to_be_32(cq->cq_ci); +@@ -966,6 +960,82 @@ mlx5_aso_mtr_completion_handle(struct mlx5_aso_sq *sq, bool need_lock) + rte_spinlock_unlock(&sq->sqsl); + } + ++static __rte_always_inline struct mlx5_aso_sq * ++mlx5_aso_mtr_select_sq(struct mlx5_dev_ctx_shared *sh, uint32_t queue, ++ struct mlx5_aso_mtr *mtr, bool *need_lock) ++{ ++ struct mlx5_aso_sq *sq; ++ ++ if (likely(sh->config.dv_flow_en == 2) && ++ mtr->type == ASO_METER_INDIRECT) { ++ if (queue == MLX5_HW_INV_QUEUE) { ++ sq = &mtr->pool->sq[mtr->pool->nb_sq - 1]; ++ *need_lock = true; ++ } else { ++ sq = &mtr->pool->sq[queue]; ++ *need_lock = false; ++ } ++ } else { ++ sq = &sh->mtrmng->pools_mng.sq; ++ *need_lock = true; ++ } ++ return sq; ++} ++ ++#if defined(HAVE_MLX5_HWS_SUPPORT) ++static void ++mlx5_aso_poll_cq_mtr_hws(struct mlx5_priv *priv, struct mlx5_aso_sq *sq) ++{ ++#define MLX5_HWS_MTR_CMPL_NUM 4 ++ ++ int i, ret; ++ struct mlx5_aso_mtr *mtr; ++ uint8_t exp_state = ASO_METER_WAIT; ++ struct rte_flow_op_result res[MLX5_HWS_MTR_CMPL_NUM]; ++ __rte_unused bool verdict; ++ ++ rte_spinlock_lock(&sq->sqsl); ++repeat: ++ ret = mlx5_aso_pull_completion(sq, res, MLX5_HWS_MTR_CMPL_NUM); ++ if (ret) { ++ for (i = 0; i < ret; i++) { ++ struct mlx5_hw_q_job *job = res[i].user_data; ++ ++ MLX5_ASSERT(job); ++ mtr = mlx5_ipool_get(priv->hws_mpool->idx_pool, ++ MLX5_INDIRECT_ACTION_IDX_GET(job->action)); ++ MLX5_ASSERT(mtr); ++ verdict = __atomic_compare_exchange_n(&mtr->state, ++ &exp_state, ASO_METER_READY, ++ false, __ATOMIC_RELAXED, ++ __ATOMIC_RELAXED); ++ MLX5_ASSERT(verdict); ++ flow_hw_job_put(priv, job, CTRL_QUEUE_ID(priv)); ++ } ++ if (ret == MLX5_HWS_MTR_CMPL_NUM) ++ goto repeat; ++ } ++ rte_spinlock_unlock(&sq->sqsl); ++ ++#undef MLX5_HWS_MTR_CMPL_NUM ++} ++#else ++static void ++mlx5_aso_poll_cq_mtr_hws(__rte_unused struct mlx5_priv *priv, __rte_unused struct mlx5_aso_sq *sq) ++{ ++ MLX5_ASSERT(false); ++} ++#endif ++ ++static void ++mlx5_aso_poll_cq_mtr_sws(__rte_unused struct mlx5_priv *priv, ++ struct mlx5_aso_sq *sq) ++{ ++ mlx5_aso_mtr_completion_handle(sq, true); ++} ++ ++typedef void (*poll_cq_t)(struct mlx5_priv *, struct mlx5_aso_sq *); ++ + /** + * Update meter parameter by send WQE. + * +@@ -980,39 +1050,29 @@ mlx5_aso_mtr_completion_handle(struct mlx5_aso_sq *sq, bool need_lock) + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ + int +-mlx5_aso_meter_update_by_wqe(struct mlx5_dev_ctx_shared *sh, uint32_t queue, +- struct mlx5_aso_mtr *mtr, +- struct mlx5_mtr_bulk *bulk, +- void *user_data, +- bool push) ++mlx5_aso_meter_update_by_wqe(struct mlx5_priv *priv, uint32_t queue, ++ struct mlx5_aso_mtr *mtr, ++ struct mlx5_mtr_bulk *bulk, ++ struct mlx5_hw_q_job *job, bool push) + { +- struct mlx5_aso_sq *sq; +- uint32_t poll_wqe_times = MLX5_MTR_POLL_WQE_CQE_TIMES; + bool need_lock; ++ struct mlx5_dev_ctx_shared *sh = priv->sh; ++ struct mlx5_aso_sq *sq = ++ mlx5_aso_mtr_select_sq(sh, queue, mtr, &need_lock); ++ uint32_t poll_wqe_times = MLX5_MTR_POLL_WQE_CQE_TIMES; ++ poll_cq_t poll_mtr_cq = ++ job ? mlx5_aso_poll_cq_mtr_hws : mlx5_aso_poll_cq_mtr_sws; + int ret; + +- if (likely(sh->config.dv_flow_en == 2) && +- mtr->type == ASO_METER_INDIRECT) { +- if (queue == MLX5_HW_INV_QUEUE) { +- sq = &mtr->pool->sq[mtr->pool->nb_sq - 1]; +- need_lock = true; +- } else { +- sq = &mtr->pool->sq[queue]; +- need_lock = false; +- } +- } else { +- sq = &sh->mtrmng->pools_mng.sq; +- need_lock = true; +- } + if (queue != MLX5_HW_INV_QUEUE) { + ret = mlx5_aso_mtr_sq_enqueue_single(sh, sq, mtr, bulk, +- need_lock, user_data, push); ++ need_lock, job, push); + return ret > 0 ? 0 : -1; + } + do { +- mlx5_aso_mtr_completion_handle(sq, need_lock); ++ poll_mtr_cq(priv, sq); + if (mlx5_aso_mtr_sq_enqueue_single(sh, sq, mtr, bulk, +- need_lock, NULL, true)) ++ need_lock, job, true)) + return 0; + /* Waiting for wqe resource. */ + rte_delay_us_sleep(MLX5_ASO_WQE_CQE_RESPONSE_DELAY); +@@ -1036,32 +1096,22 @@ mlx5_aso_meter_update_by_wqe(struct mlx5_dev_ctx_shared *sh, uint32_t queue, + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ + int +-mlx5_aso_mtr_wait(struct mlx5_dev_ctx_shared *sh, uint32_t queue, +- struct mlx5_aso_mtr *mtr) ++mlx5_aso_mtr_wait(struct mlx5_priv *priv, ++ struct mlx5_aso_mtr *mtr, bool is_tmpl_api) + { ++ bool need_lock; + struct mlx5_aso_sq *sq; ++ struct mlx5_dev_ctx_shared *sh = priv->sh; + uint32_t poll_cqe_times = MLX5_MTR_POLL_WQE_CQE_TIMES; +- uint8_t state; +- bool need_lock; ++ uint8_t state = __atomic_load_n(&mtr->state, __ATOMIC_RELAXED); ++ poll_cq_t poll_mtr_cq = ++ is_tmpl_api ? mlx5_aso_poll_cq_mtr_hws : mlx5_aso_poll_cq_mtr_sws; + +- if (likely(sh->config.dv_flow_en == 2) && +- mtr->type == ASO_METER_INDIRECT) { +- if (queue == MLX5_HW_INV_QUEUE) { +- sq = &mtr->pool->sq[mtr->pool->nb_sq - 1]; +- need_lock = true; +- } else { +- sq = &mtr->pool->sq[queue]; +- need_lock = false; +- } +- } else { +- sq = &sh->mtrmng->pools_mng.sq; +- need_lock = true; +- } +- state = __atomic_load_n(&mtr->state, __ATOMIC_RELAXED); + if (state == ASO_METER_READY || state == ASO_METER_WAIT_ASYNC) + return 0; ++ sq = mlx5_aso_mtr_select_sq(sh, MLX5_HW_INV_QUEUE, mtr, &need_lock); + do { +- mlx5_aso_mtr_completion_handle(sq, need_lock); ++ poll_mtr_cq(priv, sq); + if (__atomic_load_n(&mtr->state, __ATOMIC_RELAXED) == + ASO_METER_READY) + return 0; +diff --git a/dpdk/drivers/net/mlx5/mlx5_flow_dv.c b/dpdk/drivers/net/mlx5/mlx5_flow_dv.c +index 115d730317..863737ceba 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_flow_dv.c ++++ b/dpdk/drivers/net/mlx5/mlx5_flow_dv.c +@@ -267,21 +267,41 @@ struct field_modify_info modify_tcp[] = { + {0, 0, 0}, + }; + +-static void ++enum mlx5_l3_tunnel_detection { ++ l3_tunnel_none, ++ l3_tunnel_outer, ++ l3_tunnel_inner ++}; ++ ++static enum mlx5_l3_tunnel_detection + mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused, +- uint8_t next_protocol, uint64_t *item_flags, +- int *tunnel) ++ uint8_t next_protocol, uint64_t item_flags, ++ uint64_t *l3_tunnel_flag) + { ++ enum mlx5_l3_tunnel_detection td = l3_tunnel_none; ++ + MLX5_ASSERT(item->type == RTE_FLOW_ITEM_TYPE_IPV4 || + item->type == RTE_FLOW_ITEM_TYPE_IPV6); +- if (next_protocol == IPPROTO_IPIP) { +- *item_flags |= MLX5_FLOW_LAYER_IPIP; +- *tunnel = 1; +- } +- if (next_protocol == IPPROTO_IPV6) { +- *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP; +- *tunnel = 1; ++ if ((item_flags & MLX5_FLOW_LAYER_OUTER_L3) == 0) { ++ switch (next_protocol) { ++ case IPPROTO_IPIP: ++ td = l3_tunnel_outer; ++ *l3_tunnel_flag = MLX5_FLOW_LAYER_IPIP; ++ break; ++ case IPPROTO_IPV6: ++ td = l3_tunnel_outer; ++ *l3_tunnel_flag = MLX5_FLOW_LAYER_IPV6_ENCAP; ++ break; ++ default: ++ break; ++ } ++ } else { ++ td = l3_tunnel_inner; ++ *l3_tunnel_flag = item->type == RTE_FLOW_ITEM_TYPE_IPV4 ? ++ MLX5_FLOW_LAYER_IPIP : ++ MLX5_FLOW_LAYER_IPV6_ENCAP; + } ++ return td; + } + + static inline struct mlx5_hlist * +@@ -1925,7 +1945,7 @@ mlx5_flow_field_id_to_modify_info + if (priv->sh->config.dv_flow_en == 2) + reg = flow_hw_get_reg_id(dev, + RTE_FLOW_ITEM_TYPE_TAG, +- data->level); ++ tag_index); + else + reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, + tag_index, error); +@@ -5484,13 +5504,6 @@ flow_dv_validate_action_jump(struct rte_eth_dev *dev, + &grp_info, error); + if (ret) + return ret; +- if (attributes->group == target_group && +- !(action_flags & (MLX5_FLOW_ACTION_TUNNEL_SET | +- MLX5_FLOW_ACTION_TUNNEL_MATCH))) +- return rte_flow_error_set(error, EINVAL, +- RTE_FLOW_ERROR_TYPE_ACTION, NULL, +- "target group must be other than" +- " the current flow group"); + if (table == 0) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_CONF, +@@ -5952,7 +5965,7 @@ flow_dv_modify_create_cb(void *tool_ctx, void *cb_ctx) + "cannot allocate resource memory"); + return NULL; + } +- rte_memcpy(&entry->ft_type, ++ rte_memcpy(RTE_PTR_ADD(entry, offsetof(typeof(*entry), ft_type)), + RTE_PTR_ADD(ref, offsetof(typeof(*ref), ft_type)), + key_len + data_len); + if (entry->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) +@@ -7062,11 +7075,13 @@ flow_dv_validate_attributes(struct rte_eth_dev *dev, + } + + static int +-validate_integrity_bits(const struct rte_flow_item_integrity *mask, ++validate_integrity_bits(const void *arg, + int64_t pattern_flags, uint64_t l3_flags, + uint64_t l4_flags, uint64_t ip4_flag, + struct rte_flow_error *error) + { ++ const struct rte_flow_item_integrity *mask = arg; ++ + if (mask->l3_ok && !(pattern_flags & l3_flags)) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, +@@ -7255,6 +7270,40 @@ flow_dv_validate_item_flex(struct rte_eth_dev *dev, + return 0; + } + ++static __rte_always_inline uint8_t ++mlx5_flow_l3_next_protocol(const struct rte_flow_item *l3_item, ++ enum MLX5_SET_MATCHER key_type) ++{ ++#define MLX5_L3_NEXT_PROTOCOL(i, ms) \ ++ ((i)->type == RTE_FLOW_ITEM_TYPE_IPV4 ? \ ++ ((const struct rte_flow_item_ipv4 *)(i)->ms)->hdr.next_proto_id : \ ++ (i)->type == RTE_FLOW_ITEM_TYPE_IPV6 ? \ ++ ((const struct rte_flow_item_ipv6 *)(i)->ms)->hdr.proto : \ ++ (i)->type == RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT ? \ ++ ((const struct rte_flow_item_ipv6_frag_ext *)(i)->ms)->hdr.next_header :\ ++ 0xff) ++ ++ uint8_t next_protocol; ++ ++ if (l3_item->mask != NULL && l3_item->spec != NULL) { ++ next_protocol = MLX5_L3_NEXT_PROTOCOL(l3_item, mask); ++ if (next_protocol) ++ next_protocol &= MLX5_L3_NEXT_PROTOCOL(l3_item, spec); ++ else ++ next_protocol = 0xff; ++ } else if (key_type == MLX5_SET_MATCHER_HS_M && l3_item->mask != NULL) { ++ next_protocol = MLX5_L3_NEXT_PROTOCOL(l3_item, mask); ++ } else if (key_type == MLX5_SET_MATCHER_HS_V && l3_item->spec != NULL) { ++ next_protocol = MLX5_L3_NEXT_PROTOCOL(l3_item, spec); ++ } else { ++ /* Reset for inner layer. */ ++ next_protocol = 0xff; ++ } ++ return next_protocol; ++ ++#undef MLX5_L3_NEXT_PROTOCOL ++} ++ + /** + * Validate IB BTH item. + * +@@ -7451,6 +7500,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, + return ret; + is_root = (uint64_t)ret; + for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { ++ enum mlx5_l3_tunnel_detection l3_tunnel_detection; ++ uint64_t l3_tunnel_flag; + int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); + int type = items->type; + +@@ -7528,8 +7579,16 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, + vlan_m = items->mask; + break; + case RTE_FLOW_ITEM_TYPE_IPV4: +- mlx5_flow_tunnel_ip_check(items, next_protocol, +- &item_flags, &tunnel); ++ next_protocol = mlx5_flow_l3_next_protocol ++ (items, (enum MLX5_SET_MATCHER)-1); ++ l3_tunnel_detection = ++ mlx5_flow_tunnel_ip_check(items, next_protocol, ++ item_flags, ++ &l3_tunnel_flag); ++ if (l3_tunnel_detection == l3_tunnel_inner) { ++ item_flags |= l3_tunnel_flag; ++ tunnel = 1; ++ } + ret = flow_dv_validate_item_ipv4(dev, items, item_flags, + last_item, ether_type, + error); +@@ -7537,23 +7596,20 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, + return ret; + last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : + MLX5_FLOW_LAYER_OUTER_L3_IPV4; +- if (items->mask != NULL && +- ((const struct rte_flow_item_ipv4 *) +- items->mask)->hdr.next_proto_id) { +- next_protocol = +- ((const struct rte_flow_item_ipv4 *) +- (items->spec))->hdr.next_proto_id; +- next_protocol &= +- ((const struct rte_flow_item_ipv4 *) +- (items->mask))->hdr.next_proto_id; +- } else { +- /* Reset for inner layer. */ +- next_protocol = 0xff; +- } ++ if (l3_tunnel_detection == l3_tunnel_outer) ++ item_flags |= l3_tunnel_flag; + break; + case RTE_FLOW_ITEM_TYPE_IPV6: +- mlx5_flow_tunnel_ip_check(items, next_protocol, +- &item_flags, &tunnel); ++ next_protocol = mlx5_flow_l3_next_protocol ++ (items, (enum MLX5_SET_MATCHER)-1); ++ l3_tunnel_detection = ++ mlx5_flow_tunnel_ip_check(items, next_protocol, ++ item_flags, ++ &l3_tunnel_flag); ++ if (l3_tunnel_detection == l3_tunnel_inner) { ++ item_flags |= l3_tunnel_flag; ++ tunnel = 1; ++ } + ret = mlx5_flow_validate_item_ipv6(items, item_flags, + last_item, + ether_type, +@@ -7563,22 +7619,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, + return ret; + last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : + MLX5_FLOW_LAYER_OUTER_L3_IPV6; +- if (items->mask != NULL && +- ((const struct rte_flow_item_ipv6 *) +- items->mask)->hdr.proto) { +- item_ipv6_proto = +- ((const struct rte_flow_item_ipv6 *) +- items->spec)->hdr.proto; +- next_protocol = +- ((const struct rte_flow_item_ipv6 *) +- items->spec)->hdr.proto; +- next_protocol &= +- ((const struct rte_flow_item_ipv6 *) +- items->mask)->hdr.proto; +- } else { +- /* Reset for inner layer. */ +- next_protocol = 0xff; +- } ++ if (l3_tunnel_detection == l3_tunnel_outer) ++ item_flags |= l3_tunnel_flag; + break; + case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: + ret = flow_dv_validate_item_ipv6_frag_ext(items, +@@ -7589,19 +7631,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, + last_item = tunnel ? + MLX5_FLOW_LAYER_INNER_L3_IPV6_FRAG_EXT : + MLX5_FLOW_LAYER_OUTER_L3_IPV6_FRAG_EXT; +- if (items->mask != NULL && +- ((const struct rte_flow_item_ipv6_frag_ext *) +- items->mask)->hdr.next_header) { +- next_protocol = +- ((const struct rte_flow_item_ipv6_frag_ext *) +- items->spec)->hdr.next_header; +- next_protocol &= +- ((const struct rte_flow_item_ipv6_frag_ext *) +- items->mask)->hdr.next_header; +- } else { +- /* Reset for inner layer. */ +- next_protocol = 0xff; +- } ++ next_protocol = mlx5_flow_l3_next_protocol ++ (items, (enum MLX5_SET_MATCHER)-1); + break; + case RTE_FLOW_ITEM_TYPE_TCP: + ret = mlx5_flow_validate_item_tcp +@@ -9985,14 +10016,13 @@ flow_dv_translate_item_geneve_opt(struct rte_eth_dev *dev, void *key, + { + const struct rte_flow_item_geneve_opt *geneve_opt_m; + const struct rte_flow_item_geneve_opt *geneve_opt_v; +- const struct rte_flow_item_geneve_opt *geneve_opt_vv = item->spec; +- void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); ++ const struct rte_flow_item_geneve_opt *orig_spec = item->spec; + void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3); + rte_be32_t opt_data_key = 0, opt_data_mask = 0; +- uint32_t *data; ++ size_t option_byte_len; + int ret = 0; + +- if (MLX5_ITEM_VALID(item, key_type)) ++ if (MLX5_ITEM_VALID(item, key_type) || !orig_spec) + return -1; + MLX5_ITEM_UPDATE(item, key_type, geneve_opt_v, geneve_opt_m, + &rte_flow_item_geneve_opt_mask); +@@ -10005,36 +10035,15 @@ flow_dv_translate_item_geneve_opt(struct rte_eth_dev *dev, void *key, + return ret; + } + } +- /* +- * Set the option length in GENEVE header if not requested. +- * The GENEVE TLV option length is expressed by the option length field +- * in the GENEVE header. +- * If the option length was not requested but the GENEVE TLV option item +- * is present we set the option length field implicitly. +- */ +- if (!MLX5_GET16(fte_match_set_misc, misc_v, geneve_opt_len)) { +- if (key_type & MLX5_SET_MATCHER_M) +- MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len, +- MLX5_GENEVE_OPTLEN_MASK); +- else +- MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len, +- geneve_opt_v->option_len + 1); +- } +- /* Set the data. */ +- if (key_type == MLX5_SET_MATCHER_SW_V) +- data = geneve_opt_vv->data; +- else +- data = geneve_opt_v->data; +- if (data) { +- memcpy(&opt_data_key, data, +- RTE_MIN((uint32_t)(geneve_opt_v->option_len * 4), +- sizeof(opt_data_key))); +- memcpy(&opt_data_mask, geneve_opt_m->data, +- RTE_MIN((uint32_t)(geneve_opt_v->option_len * 4), +- sizeof(opt_data_mask))); ++ /* Convert the option length from DW to bytes for using memcpy. */ ++ option_byte_len = RTE_MIN((size_t)(orig_spec->option_len * 4), ++ sizeof(rte_be32_t)); ++ if (geneve_opt_v->data) { ++ memcpy(&opt_data_key, geneve_opt_v->data, option_byte_len); ++ memcpy(&opt_data_mask, geneve_opt_m->data, option_byte_len); + MLX5_SET(fte_match_set_misc3, misc3_v, +- geneve_tlv_option_0_data, +- rte_be_to_cpu_32(opt_data_key & opt_data_mask)); ++ geneve_tlv_option_0_data, ++ rte_be_to_cpu_32(opt_data_key & opt_data_mask)); + } + return ret; + } +@@ -13658,6 +13667,13 @@ flow_dv_translate_create_conntrack(struct rte_eth_dev *dev, + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "Connection is not supported"); ++ if (dev->data->port_id >= MLX5_INDIRECT_ACT_CT_MAX_PORT) { ++ rte_flow_error_set(error, EINVAL, ++ RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, ++ "CT supports port indexes up to " ++ RTE_STR(MLX5_ACTION_CTX_CT_MAX_PORT)); ++ return 0; ++ } + idx = flow_dv_aso_ct_alloc(dev, error); + if (!idx) + return rte_flow_error_set(error, rte_errno, +@@ -13707,6 +13723,8 @@ flow_dv_translate_items(struct rte_eth_dev *dev, + int tunnel = !!(wks->item_flags & MLX5_FLOW_LAYER_TUNNEL); + int item_type = items->type; + uint64_t last_item = wks->last_item; ++ enum mlx5_l3_tunnel_detection l3_tunnel_detection; ++ uint64_t l3_tunnel_flag; + int ret; + + switch (item_type) { +@@ -13750,94 +13768,47 @@ flow_dv_translate_items(struct rte_eth_dev *dev, + MLX5_FLOW_LAYER_OUTER_VLAN); + break; + case RTE_FLOW_ITEM_TYPE_IPV4: +- mlx5_flow_tunnel_ip_check(items, next_protocol, +- &wks->item_flags, &tunnel); ++ next_protocol = mlx5_flow_l3_next_protocol(items, key_type); ++ l3_tunnel_detection = ++ mlx5_flow_tunnel_ip_check(items, next_protocol, ++ wks->item_flags, ++ &l3_tunnel_flag); ++ if (l3_tunnel_detection == l3_tunnel_inner) { ++ wks->item_flags |= l3_tunnel_flag; ++ tunnel = 1; ++ } + flow_dv_translate_item_ipv4(key, items, tunnel, + wks->group, key_type); + wks->priority = MLX5_PRIORITY_MAP_L3; + last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : + MLX5_FLOW_LAYER_OUTER_L3_IPV4; +- if (items->mask != NULL && +- items->spec != NULL && +- ((const struct rte_flow_item_ipv4 *) +- items->mask)->hdr.next_proto_id) { +- next_protocol = +- ((const struct rte_flow_item_ipv4 *) +- (items->spec))->hdr.next_proto_id; +- next_protocol &= +- ((const struct rte_flow_item_ipv4 *) +- (items->mask))->hdr.next_proto_id; +- } else if (key_type == MLX5_SET_MATCHER_HS_M && +- items->mask != NULL) { +- next_protocol = ((const struct rte_flow_item_ipv4 *) +- (items->mask))->hdr.next_proto_id; +- } else if (key_type == MLX5_SET_MATCHER_HS_V && +- items->spec != NULL) { +- next_protocol = ((const struct rte_flow_item_ipv4 *) +- (items->spec))->hdr.next_proto_id; +- } else { +- /* Reset for inner layer. */ +- next_protocol = 0xff; +- } ++ if (l3_tunnel_detection == l3_tunnel_outer) ++ wks->item_flags |= l3_tunnel_flag; + break; + case RTE_FLOW_ITEM_TYPE_IPV6: +- mlx5_flow_tunnel_ip_check(items, next_protocol, +- &wks->item_flags, &tunnel); ++ next_protocol = mlx5_flow_l3_next_protocol(items, key_type); ++ l3_tunnel_detection = ++ mlx5_flow_tunnel_ip_check(items, next_protocol, ++ wks->item_flags, ++ &l3_tunnel_flag); ++ if (l3_tunnel_detection == l3_tunnel_inner) { ++ wks->item_flags |= l3_tunnel_flag; ++ tunnel = 1; ++ } + flow_dv_translate_item_ipv6(key, items, tunnel, + wks->group, key_type); + wks->priority = MLX5_PRIORITY_MAP_L3; + last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : + MLX5_FLOW_LAYER_OUTER_L3_IPV6; +- if (items->mask != NULL && +- items->spec != NULL && +- ((const struct rte_flow_item_ipv6 *) +- items->mask)->hdr.proto) { +- next_protocol = +- ((const struct rte_flow_item_ipv6 *) +- items->spec)->hdr.proto; +- next_protocol &= +- ((const struct rte_flow_item_ipv6 *) +- items->mask)->hdr.proto; +- } else if (key_type == MLX5_SET_MATCHER_HS_M && +- items->mask != NULL) { +- next_protocol = ((const struct rte_flow_item_ipv6 *) +- (items->mask))->hdr.proto; +- } else if (key_type == MLX5_SET_MATCHER_HS_V && +- items->spec != NULL) { +- next_protocol = ((const struct rte_flow_item_ipv6 *) +- (items->spec))->hdr.proto; +- } else { +- /* Reset for inner layer. */ +- next_protocol = 0xff; +- } ++ if (l3_tunnel_detection == l3_tunnel_outer) ++ wks->item_flags |= l3_tunnel_flag; + break; + case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: + flow_dv_translate_item_ipv6_frag_ext + (key, items, tunnel, key_type); + last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6_FRAG_EXT : + MLX5_FLOW_LAYER_OUTER_L3_IPV6_FRAG_EXT; +- if (items->mask != NULL && +- items->spec != NULL && +- ((const struct rte_flow_item_ipv6_frag_ext *) +- items->mask)->hdr.next_header) { +- next_protocol = +- ((const struct rte_flow_item_ipv6_frag_ext *) +- items->spec)->hdr.next_header; +- next_protocol &= +- ((const struct rte_flow_item_ipv6_frag_ext *) +- items->mask)->hdr.next_header; +- } else if (key_type == MLX5_SET_MATCHER_HS_M && +- items->mask != NULL) { +- next_protocol = ((const struct rte_flow_item_ipv6_frag_ext *) +- (items->mask))->hdr.next_header; +- } else if (key_type == MLX5_SET_MATCHER_HS_V && +- items->spec != NULL) { +- next_protocol = ((const struct rte_flow_item_ipv6_frag_ext *) +- (items->spec))->hdr.next_header; +- } else { +- /* Reset for inner layer. */ +- next_protocol = 0xff; +- } ++ next_protocol = mlx5_flow_l3_next_protocol(items, key_type); + break; + case RTE_FLOW_ITEM_TYPE_TCP: + flow_dv_translate_item_tcp(key, items, tunnel, key_type); +@@ -14280,7 +14251,7 @@ flow_dv_translate_items_sws(struct rte_eth_dev *dev, + * Avoid be overwritten by other sub mlx5_flows. + */ + if (wks.geneve_tlv_option) +- dev_flow->flow->geneve_tlv_option = wks.geneve_tlv_option; ++ dev_flow->flow->geneve_tlv_option += wks.geneve_tlv_option; + return 0; + } + +@@ -15420,7 +15391,8 @@ error: + SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, + handle_idx, dh, next) { + /* hrxq is union, don't clear it if the flag is not set. */ +- if (dh->fate_action == MLX5_FLOW_FATE_QUEUE && dh->rix_hrxq) { ++ if (dh->fate_action == MLX5_FLOW_FATE_QUEUE && dh->rix_hrxq && ++ !dh->dvh.rix_sample && !dh->dvh.rix_dest_array) { + mlx5_hrxq_release(dev, dh->rix_hrxq); + dh->rix_hrxq = 0; + } else if (dh->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { +@@ -15884,9 +15856,9 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) + flow_dv_aso_ct_release(dev, flow->ct, NULL); + else if (flow->age) + flow_dv_aso_age_release(dev, flow->age); +- if (flow->geneve_tlv_option) { ++ while (flow->geneve_tlv_option) { + flow_dev_geneve_tlv_option_resource_release(priv->sh); +- flow->geneve_tlv_option = 0; ++ flow->geneve_tlv_option--; + } + while (flow->dev_handles) { + uint32_t tmp_idx = flow->dev_handles; +@@ -16350,6 +16322,8 @@ flow_dv_action_create(struct rte_eth_dev *dev, + case RTE_FLOW_ACTION_TYPE_CONNTRACK: + ret = flow_dv_translate_create_conntrack(dev, action->conf, + err); ++ if (!ret) ++ break; + idx = MLX5_INDIRECT_ACT_CT_GEN_IDX(PORT_ID(priv), ret); + break; + default: +@@ -17675,9 +17649,8 @@ __flow_dv_create_policy_matcher(struct rte_eth_dev *dev, + } + } + tbl_data = container_of(tbl_rsc, struct mlx5_flow_tbl_data_entry, tbl); +- if (priority < RTE_COLOR_RED) +- flow_dv_match_meta_reg(matcher.mask.buf, +- (enum modify_reg)color_reg_c_idx, color_mask, color_mask); ++ flow_dv_match_meta_reg(matcher.mask.buf, ++ (enum modify_reg)color_reg_c_idx, color_mask, color_mask); + matcher.priority = priority; + matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, + matcher.mask.size); +@@ -17711,7 +17684,7 @@ __flow_dv_create_policy_matcher(struct rte_eth_dev *dev, + static int + __flow_dv_create_domain_policy_rules(struct rte_eth_dev *dev, + struct mlx5_flow_meter_sub_policy *sub_policy, +- uint8_t egress, uint8_t transfer, bool match_src_port, ++ uint8_t egress, uint8_t transfer, bool *match_src_port, + struct mlx5_meter_policy_acts acts[RTE_COLORS]) + { + struct mlx5_priv *priv = dev->data->dev_private; +@@ -17726,9 +17699,9 @@ __flow_dv_create_domain_policy_rules(struct rte_eth_dev *dev, + .reserved = 0, + }; + int i; ++ uint16_t priority; + int ret = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, &flow_err); + struct mlx5_sub_policy_color_rule *color_rule; +- bool svport_match; + struct mlx5_sub_policy_color_rule *tmp_rules[RTE_COLORS] = {NULL}; + + if (ret < 0) +@@ -17761,13 +17734,12 @@ __flow_dv_create_domain_policy_rules(struct rte_eth_dev *dev, + TAILQ_INSERT_TAIL(&sub_policy->color_rules[i], + color_rule, next_port); + color_rule->src_port = priv->representor_id; +- /* No use. */ +- attr.priority = i; ++ priority = (match_src_port[i] == match_src_port[RTE_COLOR_GREEN]) ? ++ MLX5_MTR_POLICY_MATCHER_PRIO : (MLX5_MTR_POLICY_MATCHER_PRIO + 1); + /* Create matchers for colors. */ +- svport_match = (i != RTE_COLOR_RED) ? match_src_port : false; + if (__flow_dv_create_policy_matcher(dev, color_reg_c_idx, +- MLX5_MTR_POLICY_MATCHER_PRIO, sub_policy, +- &attr, svport_match, NULL, ++ priority, sub_policy, ++ &attr, match_src_port[i], NULL, + &color_rule->matcher, &flow_err)) { + DRV_LOG(ERR, "Failed to create color%u matcher.", i); + goto err_exit; +@@ -17777,7 +17749,7 @@ __flow_dv_create_domain_policy_rules(struct rte_eth_dev *dev, + color_reg_c_idx, (enum rte_color)i, + color_rule->matcher, + acts[i].actions_n, acts[i].dv_actions, +- svport_match, NULL, &color_rule->rule, ++ match_src_port[i], NULL, &color_rule->rule, + &attr)) { + DRV_LOG(ERR, "Failed to create color%u rule.", i); + goto err_exit; +@@ -17825,7 +17797,7 @@ __flow_dv_create_policy_acts_rules(struct rte_eth_dev *dev, + uint8_t egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0; + uint8_t transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0; + bool mtr_first = egress || (transfer && priv->representor_id != UINT16_MAX); +- bool match_src_port = false; ++ bool match_src_port[RTE_COLORS] = {false}; + int i; + + /* If RSS or Queue, no previous actions / rules is created. */ +@@ -17896,7 +17868,7 @@ __flow_dv_create_policy_acts_rules(struct rte_eth_dev *dev, + acts[i].dv_actions[acts[i].actions_n] = + port_action->action; + acts[i].actions_n++; +- match_src_port = true; ++ match_src_port[i] = true; + break; + case MLX5_FLOW_FATE_DROP: + case MLX5_FLOW_FATE_JUMP: +@@ -17948,7 +17920,7 @@ __flow_dv_create_policy_acts_rules(struct rte_eth_dev *dev, + acts[i].dv_actions[acts[i].actions_n++] = + tbl_data->jump.action; + if (mtr_policy->act_cnt[i].modify_hdr) +- match_src_port = !!transfer; ++ match_src_port[i] = !!transfer; + break; + default: + /*Queue action do nothing*/ +@@ -17962,9 +17934,9 @@ __flow_dv_create_policy_acts_rules(struct rte_eth_dev *dev, + "Failed to create policy rules per domain."); + goto err_exit; + } +- if (match_src_port) { +- mtr_policy->match_port = match_src_port; +- mtr_policy->hierarchy_match_port = match_src_port; ++ if (match_src_port[RTE_COLOR_GREEN] || match_src_port[RTE_COLOR_YELLOW]) { ++ mtr_policy->match_port = 1; ++ mtr_policy->hierarchy_match_port = 1; + } + return 0; + err_exit: +@@ -18026,6 +17998,7 @@ __flow_dv_create_domain_def_policy(struct rte_eth_dev *dev, uint32_t domain) + uint8_t egress, transfer; + struct rte_flow_error error; + struct mlx5_meter_policy_acts acts[RTE_COLORS]; ++ bool match_src_port[RTE_COLORS] = {false}; + int ret; + + egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0; +@@ -18101,7 +18074,7 @@ __flow_dv_create_domain_def_policy(struct rte_eth_dev *dev, uint32_t domain) + /* Create default policy rules. */ + ret = __flow_dv_create_domain_policy_rules(dev, + &def_policy->sub_policy, +- egress, transfer, false, acts); ++ egress, transfer, match_src_port, acts); + if (ret) { + DRV_LOG(ERR, "Failed to create default policy rules."); + goto def_policy_error; +@@ -18660,7 +18633,7 @@ flow_dv_meter_hierarchy_rule_create(struct rte_eth_dev *dev, + struct { + struct mlx5_flow_meter_policy *fm_policy; + struct mlx5_flow_meter_info *next_fm; +- struct mlx5_sub_policy_color_rule *tag_rule[MLX5_MTR_RTE_COLORS]; ++ struct mlx5_sub_policy_color_rule *tag_rule[RTE_COLORS]; + } fm_info[MLX5_MTR_CHAIN_MAX_NUM] = { {0} }; + uint32_t fm_cnt = 0; + uint32_t i, j; +@@ -18694,14 +18667,22 @@ flow_dv_meter_hierarchy_rule_create(struct rte_eth_dev *dev, + mtr_policy = fm_info[i].fm_policy; + rte_spinlock_lock(&mtr_policy->sl); + sub_policy = mtr_policy->sub_policys[domain][0]; +- for (j = 0; j < MLX5_MTR_RTE_COLORS; j++) { ++ for (j = 0; j < RTE_COLORS; j++) { + uint8_t act_n = 0; +- struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; ++ struct mlx5_flow_dv_modify_hdr_resource *modify_hdr = NULL; + struct mlx5_flow_dv_port_id_action_resource *port_action; ++ uint8_t fate_action; + +- if (mtr_policy->act_cnt[j].fate_action != MLX5_FLOW_FATE_MTR && +- mtr_policy->act_cnt[j].fate_action != MLX5_FLOW_FATE_PORT_ID) +- continue; ++ if (j == RTE_COLOR_RED) { ++ fate_action = MLX5_FLOW_FATE_DROP; ++ } else { ++ fate_action = mtr_policy->act_cnt[j].fate_action; ++ modify_hdr = mtr_policy->act_cnt[j].modify_hdr; ++ if (fate_action != MLX5_FLOW_FATE_MTR && ++ fate_action != MLX5_FLOW_FATE_PORT_ID && ++ fate_action != MLX5_FLOW_FATE_DROP) ++ continue; ++ } + color_rule = mlx5_malloc(MLX5_MEM_ZERO, + sizeof(struct mlx5_sub_policy_color_rule), + 0, SOCKET_ID_ANY); +@@ -18713,9 +18694,8 @@ flow_dv_meter_hierarchy_rule_create(struct rte_eth_dev *dev, + goto err_exit; + } + color_rule->src_port = src_port; +- modify_hdr = mtr_policy->act_cnt[j].modify_hdr; + /* Prepare to create color rule. */ +- if (mtr_policy->act_cnt[j].fate_action == MLX5_FLOW_FATE_MTR) { ++ if (fate_action == MLX5_FLOW_FATE_MTR) { + next_fm = fm_info[i].next_fm; + if (mlx5_flow_meter_attach(priv, next_fm, &attr, error)) { + mlx5_free(color_rule); +@@ -18742,7 +18722,7 @@ flow_dv_meter_hierarchy_rule_create(struct rte_eth_dev *dev, + } + acts.dv_actions[act_n++] = tbl_data->jump.action; + acts.actions_n = act_n; +- } else { ++ } else if (fate_action == MLX5_FLOW_FATE_PORT_ID) { + port_action = + mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID], + mtr_policy->act_cnt[j].rix_port_id_action); +@@ -18755,6 +18735,9 @@ flow_dv_meter_hierarchy_rule_create(struct rte_eth_dev *dev, + acts.dv_actions[act_n++] = modify_hdr->action; + acts.dv_actions[act_n++] = port_action->action; + acts.actions_n = act_n; ++ } else { ++ acts.dv_actions[act_n++] = mtr_policy->dr_drop_action[domain]; ++ acts.actions_n = act_n; + } + fm_info[i].tag_rule[j] = color_rule; + TAILQ_INSERT_TAIL(&sub_policy->color_rules[j], color_rule, next_port); +@@ -18786,7 +18769,7 @@ err_exit: + mtr_policy = fm_info[i].fm_policy; + rte_spinlock_lock(&mtr_policy->sl); + sub_policy = mtr_policy->sub_policys[domain][0]; +- for (j = 0; j < MLX5_MTR_RTE_COLORS; j++) { ++ for (j = 0; j < RTE_COLORS; j++) { + color_rule = fm_info[i].tag_rule[j]; + if (!color_rule) + continue; +@@ -19116,8 +19099,7 @@ flow_dv_get_aged_flows(struct rte_eth_dev *dev, + LIST_FOREACH(act, &age_info->aged_aso, next) { + nb_flows++; + if (nb_contexts) { +- context[nb_flows - 1] = +- act->age_params.context; ++ context[nb_flows - 1] = act->age_params.context; + if (!(--nb_contexts)) + break; + } +@@ -19675,11 +19657,13 @@ flow_dv_validate_mtr_policy_acts(struct rte_eth_dev *dev, + } + } + if (next_mtr && *policy_mode == MLX5_MTR_POLICY_MODE_ALL) { +- if (!(action_flags[RTE_COLOR_GREEN] & action_flags[RTE_COLOR_YELLOW] & +- MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY)) ++ uint64_t hierarchy_type_flag = ++ MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY | MLX5_FLOW_ACTION_JUMP; ++ if (!(action_flags[RTE_COLOR_GREEN] & hierarchy_type_flag) || ++ !(action_flags[RTE_COLOR_YELLOW] & hierarchy_type_flag)) + return -rte_mtr_error_set(error, EINVAL, RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, +- "Meter hierarchy supports meter action only."); ++ "Unsupported action in meter hierarchy."); + } + /* If both colors have RSS, the attributes should be the same. */ + if (flow_dv_mtr_policy_rss_compare(rss_color[RTE_COLOR_GREEN], +diff --git a/dpdk/drivers/net/mlx5/mlx5_flow_hw.c b/dpdk/drivers/net/mlx5/mlx5_flow_hw.c +index da873ae2e2..af4df13b2f 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_flow_hw.c ++++ b/dpdk/drivers/net/mlx5/mlx5_flow_hw.c +@@ -104,12 +104,40 @@ struct mlx5_tbl_multi_pattern_ctx { + + #define MLX5_EMPTY_MULTI_PATTERN_CTX {{{0,}},} + ++static __rte_always_inline struct mlx5_hw_q_job * ++flow_hw_action_job_init(struct mlx5_priv *priv, uint32_t queue, ++ const struct rte_flow_action_handle *handle, ++ void *user_data, void *query_data, ++ enum mlx5_hw_job_type type, ++ enum mlx5_hw_indirect_type indirect_type, ++ struct rte_flow_error *error); ++static void ++flow_hw_age_count_release(struct mlx5_priv *priv, uint32_t queue, struct rte_flow_hw *flow, ++ struct rte_flow_error *error); ++ + static int + mlx5_tbl_multi_pattern_process(struct rte_eth_dev *dev, + struct rte_flow_template_table *tbl, + struct mlx5_tbl_multi_pattern_ctx *mpat, + struct rte_flow_error *error); + ++static __rte_always_inline enum mlx5_indirect_list_type ++flow_hw_inlist_type_get(const struct rte_flow_action *actions); ++ ++static bool ++mlx5_hw_ctx_validate(const struct rte_eth_dev *dev, struct rte_flow_error *error) ++{ ++ const struct mlx5_priv *priv = dev->data->dev_private; ++ ++ if (!priv->dr_ctx) { ++ rte_flow_error_set(error, EINVAL, ++ RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, ++ "non-template flow engine was not configured"); ++ return false; ++ } ++ return true; ++} ++ + static __rte_always_inline int + mlx5_multi_pattern_reformat_to_index(enum mlx5dr_action_type type) + { +@@ -274,21 +302,6 @@ static const struct rte_flow_item_eth ctrl_rx_eth_bcast_spec = { + .hdr.ether_type = 0, + }; + +-static __rte_always_inline struct mlx5_hw_q_job * +-flow_hw_job_get(struct mlx5_priv *priv, uint32_t queue) +-{ +- MLX5_ASSERT(priv->hw_q[queue].job_idx <= priv->hw_q[queue].size); +- return priv->hw_q[queue].job_idx ? +- priv->hw_q[queue].job[--priv->hw_q[queue].job_idx] : NULL; +-} +- +-static __rte_always_inline void +-flow_hw_job_put(struct mlx5_priv *priv, struct mlx5_hw_q_job *job, uint32_t queue) +-{ +- MLX5_ASSERT(priv->hw_q[queue].job_idx < priv->hw_q[queue].size); +- priv->hw_q[queue].job[priv->hw_q[queue].job_idx++] = job; +-} +- + static inline enum mlx5dr_matcher_insert_mode + flow_hw_matcher_insert_mode_get(enum rte_flow_table_insertion_type insert_type) + { +@@ -1010,15 +1023,19 @@ flow_hw_shared_action_translate(struct rte_eth_dev *dev, + if (!shared_rss || __flow_hw_act_data_shared_rss_append + (priv, acts, + (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_RSS, +- action_src, action_dst, idx, shared_rss)) ++ action_src, action_dst, idx, shared_rss)) { ++ DRV_LOG(WARNING, "Indirect RSS action index %d translate failed", act_idx); + return -1; ++ } + break; + case MLX5_INDIRECT_ACTION_TYPE_COUNT: + if (__flow_hw_act_data_shared_cnt_append(priv, acts, + (enum rte_flow_action_type) + MLX5_RTE_FLOW_ACTION_TYPE_COUNT, +- action_src, action_dst, act_idx)) ++ action_src, action_dst, act_idx)) { ++ DRV_LOG(WARNING, "Indirect count action translate failed"); + return -1; ++ } + break; + case MLX5_INDIRECT_ACTION_TYPE_AGE: + /* Not supported, prevent by validate function. */ +@@ -1026,15 +1043,19 @@ flow_hw_shared_action_translate(struct rte_eth_dev *dev, + break; + case MLX5_INDIRECT_ACTION_TYPE_CT: + if (flow_hw_ct_compile(dev, MLX5_HW_INV_QUEUE, +- idx, &acts->rule_acts[action_dst])) ++ idx, &acts->rule_acts[action_dst])) { ++ DRV_LOG(WARNING, "Indirect CT action translate failed"); + return -1; ++ } + break; + case MLX5_INDIRECT_ACTION_TYPE_METER_MARK: + if (__flow_hw_act_data_shared_mtr_append(priv, acts, + (enum rte_flow_action_type) + MLX5_RTE_FLOW_ACTION_TYPE_METER_MARK, +- action_src, action_dst, idx)) ++ action_src, action_dst, idx)) { ++ DRV_LOG(WARNING, "Indirect meter mark action translate failed"); + return -1; ++ } + break; + case MLX5_INDIRECT_ACTION_TYPE_QUOTA: + flow_hw_construct_quota(priv, &acts->rule_acts[action_dst], idx); +@@ -1455,7 +1476,7 @@ flow_hw_meter_compile(struct rte_eth_dev *dev, + acts->rule_acts[jump_pos].action = (!!group) ? + acts->jump->hws_action : + acts->jump->root_action; +- if (mlx5_aso_mtr_wait(priv->sh, MLX5_HW_INV_QUEUE, aso_mtr)) ++ if (mlx5_aso_mtr_wait(priv, aso_mtr, true)) + return -ENOMEM; + return 0; + } +@@ -1532,7 +1553,7 @@ static rte_be32_t vlan_hdr_to_be32(const struct rte_flow_action *actions) + static __rte_always_inline struct mlx5_aso_mtr * + flow_hw_meter_mark_alloc(struct rte_eth_dev *dev, uint32_t queue, + const struct rte_flow_action *action, +- void *user_data, bool push) ++ struct mlx5_hw_q_job *job, bool push) + { + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_aso_mtr_pool *pool = priv->hws_mpool; +@@ -1540,6 +1561,8 @@ flow_hw_meter_mark_alloc(struct rte_eth_dev *dev, uint32_t queue, + struct mlx5_aso_mtr *aso_mtr; + struct mlx5_flow_meter_info *fm; + uint32_t mtr_id; ++ uintptr_t handle = (uintptr_t)MLX5_INDIRECT_ACTION_TYPE_METER_MARK << ++ MLX5_INDIRECT_ACTION_TYPE_OFFSET; + + if (meter_mark->profile == NULL) + return NULL; +@@ -1558,15 +1581,16 @@ flow_hw_meter_mark_alloc(struct rte_eth_dev *dev, uint32_t queue, + ASO_METER_WAIT : ASO_METER_WAIT_ASYNC; + aso_mtr->offset = mtr_id - 1; + aso_mtr->init_color = fm->color_aware ? RTE_COLORS : RTE_COLOR_GREEN; ++ job->action = (void *)(handle | mtr_id); + /* Update ASO flow meter by wqe. */ +- if (mlx5_aso_meter_update_by_wqe(priv->sh, queue, aso_mtr, +- &priv->mtr_bulk, user_data, push)) { ++ if (mlx5_aso_meter_update_by_wqe(priv, queue, aso_mtr, ++ &priv->mtr_bulk, job, push)) { + mlx5_ipool_free(pool->idx_pool, mtr_id); + return NULL; + } + /* Wait for ASO object completion. */ + if (queue == MLX5_HW_INV_QUEUE && +- mlx5_aso_mtr_wait(priv->sh, MLX5_HW_INV_QUEUE, aso_mtr)) { ++ mlx5_aso_mtr_wait(priv, aso_mtr, true)) { + mlx5_ipool_free(pool->idx_pool, mtr_id); + return NULL; + } +@@ -1584,10 +1608,18 @@ flow_hw_meter_mark_compile(struct rte_eth_dev *dev, + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_aso_mtr_pool *pool = priv->hws_mpool; + struct mlx5_aso_mtr *aso_mtr; ++ struct mlx5_hw_q_job *job = ++ flow_hw_action_job_init(priv, queue, NULL, NULL, NULL, ++ MLX5_HW_Q_JOB_TYPE_CREATE, ++ MLX5_HW_INDIRECT_TYPE_LEGACY, NULL); + +- aso_mtr = flow_hw_meter_mark_alloc(dev, queue, action, NULL, true); +- if (!aso_mtr) ++ if (!job) + return -1; ++ aso_mtr = flow_hw_meter_mark_alloc(dev, queue, action, job, true); ++ if (!aso_mtr) { ++ flow_hw_job_put(priv, job, queue); ++ return -1; ++ } + + /* Compile METER_MARK action */ + acts[aso_mtr_pos].action = pool->action; +@@ -1722,15 +1754,9 @@ flow_hw_translate_indirect_meter(struct rte_eth_dev *dev, + const struct rte_flow_indirect_update_flow_meter_mark **flow_conf = + (typeof(flow_conf))action_conf->conf; + +- /* +- * Masked indirect handle set dr5 action during template table +- * translation. +- */ +- if (!dr_rule->action) { +- ret = flow_dr_set_meter(priv, dr_rule, action_conf); +- if (ret) +- return ret; +- } ++ ret = flow_dr_set_meter(priv, dr_rule, action_conf); ++ if (ret) ++ return ret; + if (!act_data->shared_meter.conf_masked) { + if (flow_conf && flow_conf[0] && flow_conf[0]->init_color < RTE_COLORS) + flow_dr_mtr_flow_color(dr_rule, flow_conf[0]->init_color); +@@ -2512,6 +2538,9 @@ __flow_hw_actions_translate(struct rte_eth_dev *dev, + } + return 0; + err: ++ /* If rte_errno was not initialized and reached error state. */ ++ if (!rte_errno) ++ rte_errno = EINVAL; + err = rte_errno; + __flow_hw_action_template_destroy(dev, acts); + return rte_flow_error_set(error, err, +@@ -2865,6 +2894,30 @@ flow_hw_modify_field_construct(struct mlx5_hw_q_job *job, + return 0; + } + ++/** ++ * Release any actions allocated for the flow rule during actions construction. ++ * ++ * @param[in] flow ++ * Pointer to flow structure. ++ */ ++static void ++flow_hw_release_actions(struct rte_eth_dev *dev, ++ uint32_t queue, ++ struct rte_flow_hw *flow) ++{ ++ struct mlx5_priv *priv = dev->data->dev_private; ++ struct mlx5_aso_mtr_pool *pool = priv->hws_mpool; ++ ++ if (flow->fate_type == MLX5_FLOW_FATE_JUMP) ++ flow_hw_jump_release(dev, flow->jump); ++ else if (flow->fate_type == MLX5_FLOW_FATE_QUEUE) ++ mlx5_hrxq_obj_release(dev, flow->hrxq); ++ if (mlx5_hws_cnt_id_valid(flow->cnt_id)) ++ flow_hw_age_count_release(priv, queue, flow, NULL); ++ if (flow->mtr_id) ++ mlx5_ipool_free(pool->idx_pool, flow->mtr_id); ++} ++ + /** + * Construct flow action array. + * +@@ -2980,7 +3033,7 @@ flow_hw_actions_construct(struct rte_eth_dev *dev, + (dev, queue, action, table, it_idx, + at->action_flags, job->flow, + &rule_acts[act_data->action_dst])) +- return -1; ++ goto error; + break; + case RTE_FLOW_ACTION_TYPE_VOID: + break; +@@ -3000,7 +3053,7 @@ flow_hw_actions_construct(struct rte_eth_dev *dev, + jump = flow_hw_jump_action_register + (dev, &table->cfg, jump_group, NULL); + if (!jump) +- return -1; ++ goto error; + rule_acts[act_data->action_dst].action = + (!!attr.group) ? jump->hws_action : jump->root_action; + job->flow->jump = jump; +@@ -3012,7 +3065,7 @@ flow_hw_actions_construct(struct rte_eth_dev *dev, + ft_flag, + action); + if (!hrxq) +- return -1; ++ goto error; + rule_acts[act_data->action_dst].action = hrxq->action; + job->flow->hrxq = hrxq; + job->flow->fate_type = MLX5_FLOW_FATE_QUEUE; +@@ -3022,19 +3075,19 @@ flow_hw_actions_construct(struct rte_eth_dev *dev, + if (flow_hw_shared_action_get + (dev, act_data, item_flags, + &rule_acts[act_data->action_dst])) +- return -1; ++ goto error; + break; + case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: + enc_item = ((const struct rte_flow_action_vxlan_encap *) + action->conf)->definition; + if (flow_dv_convert_encap_data(enc_item, buf, &encap_len, NULL)) +- return -1; ++ goto error; + break; + case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: + enc_item = ((const struct rte_flow_action_nvgre_encap *) + action->conf)->definition; + if (flow_dv_convert_encap_data(enc_item, buf, &encap_len, NULL)) +- return -1; ++ goto error; + break; + case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: + raw_encap_data = +@@ -3063,12 +3116,12 @@ flow_hw_actions_construct(struct rte_eth_dev *dev, + hw_acts, + action); + if (ret) +- return -1; ++ goto error; + break; + case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT: + port_action = action->conf; + if (!priv->hw_vport[port_action->port_id]) +- return -1; ++ goto error; + rule_acts[act_data->action_dst].action = + priv->hw_vport[port_action->port_id]; + break; +@@ -3088,7 +3141,7 @@ flow_hw_actions_construct(struct rte_eth_dev *dev, + jump = flow_hw_jump_action_register + (dev, &table->cfg, aso_mtr->fm.group, NULL); + if (!jump) +- return -1; ++ goto error; + MLX5_ASSERT + (!rule_acts[act_data->action_dst + 1].action); + rule_acts[act_data->action_dst + 1].action = +@@ -3096,8 +3149,8 @@ flow_hw_actions_construct(struct rte_eth_dev *dev, + jump->root_action; + job->flow->jump = jump; + job->flow->fate_type = MLX5_FLOW_FATE_JUMP; +- if (mlx5_aso_mtr_wait(priv->sh, MLX5_HW_INV_QUEUE, aso_mtr)) +- return -1; ++ if (mlx5_aso_mtr_wait(priv, aso_mtr, true)) ++ goto error; + break; + case RTE_FLOW_ACTION_TYPE_AGE: + age = action->conf; +@@ -3112,7 +3165,7 @@ flow_hw_actions_construct(struct rte_eth_dev *dev, + job->flow->res_idx, + error); + if (age_idx == 0) +- return -rte_errno; ++ goto error; + job->flow->age_idx = age_idx; + if (at->action_flags & MLX5_FLOW_ACTION_INDIRECT_COUNT) + /* +@@ -3123,11 +3176,10 @@ flow_hw_actions_construct(struct rte_eth_dev *dev, + break; + /* Fall-through. */ + case RTE_FLOW_ACTION_TYPE_COUNT: +- /* If the port is engaged in resource sharing, do not use queue cache. */ +- cnt_queue = mlx5_hws_cnt_is_pool_shared(priv) ? NULL : &queue; ++ cnt_queue = mlx5_hws_cnt_get_queue(priv, &queue); + ret = mlx5_hws_cnt_pool_get(priv->hws_cpool, cnt_queue, &cnt_id, age_idx); + if (ret != 0) +- return ret; ++ goto error; + ret = mlx5_hws_cnt_pool_get_action_offset + (priv->hws_cpool, + cnt_id, +@@ -3135,7 +3187,7 @@ flow_hw_actions_construct(struct rte_eth_dev *dev, + &rule_acts[act_data->action_dst].counter.offset + ); + if (ret != 0) +- return ret; ++ goto error; + job->flow->cnt_id = cnt_id; + break; + case MLX5_RTE_FLOW_ACTION_TYPE_COUNT: +@@ -3146,7 +3198,7 @@ flow_hw_actions_construct(struct rte_eth_dev *dev, + &rule_acts[act_data->action_dst].counter.offset + ); + if (ret != 0) +- return ret; ++ goto error; + job->flow->cnt_id = act_data->shared_counter.id; + break; + case RTE_FLOW_ACTION_TYPE_CONNTRACK: +@@ -3154,7 +3206,7 @@ flow_hw_actions_construct(struct rte_eth_dev *dev, + ((uint32_t)(uintptr_t)action->conf); + if (flow_hw_ct_compile(dev, queue, ct_idx, + &rule_acts[act_data->action_dst])) +- return -1; ++ goto error; + break; + case MLX5_RTE_FLOW_ACTION_TYPE_METER_MARK: + mtr_id = act_data->shared_meter.id & +@@ -3162,7 +3214,7 @@ flow_hw_actions_construct(struct rte_eth_dev *dev, + /* Find ASO object. */ + aso_mtr = mlx5_ipool_get(pool->idx_pool, mtr_id); + if (!aso_mtr) +- return -1; ++ goto error; + rule_acts[act_data->action_dst].action = + pool->action; + rule_acts[act_data->action_dst].aso_meter.offset = +@@ -3177,7 +3229,7 @@ flow_hw_actions_construct(struct rte_eth_dev *dev, + act_data->action_dst, action, + rule_acts, &job->flow->mtr_id, MLX5_HW_INV_QUEUE); + if (ret != 0) +- return ret; ++ goto error; + break; + default: + break; +@@ -3215,6 +3267,11 @@ flow_hw_actions_construct(struct rte_eth_dev *dev, + if (mlx5_hws_cnt_id_valid(hw_acts->cnt_id)) + job->flow->cnt_id = hw_acts->cnt_id; + return 0; ++ ++error: ++ flow_hw_release_actions(dev, queue, job->flow); ++ rte_errno = EINVAL; ++ return -rte_errno; + } + + static const struct rte_flow_item * +@@ -3320,10 +3377,6 @@ flow_hw_async_flow_create(struct rte_eth_dev *dev, + uint32_t res_idx = 0; + int ret; + +- if (unlikely((!dev->data->dev_started))) { +- rte_errno = EINVAL; +- goto error; +- } + job = flow_hw_job_get(priv, queue); + if (!job) { + rte_errno = ENOMEM; +@@ -3368,10 +3421,8 @@ flow_hw_async_flow_create(struct rte_eth_dev *dev, + if (flow_hw_actions_construct(dev, job, + &table->ats[action_template_index], + pattern_template_index, actions, +- rule_acts, queue, error)) { +- rte_errno = EINVAL; ++ rule_acts, queue, error)) + goto error; +- } + rule_items = flow_hw_get_rule_items(dev, table, items, + pattern_template_index, job); + if (!rule_items) +@@ -3722,8 +3773,7 @@ flow_hw_age_count_release(struct mlx5_priv *priv, uint32_t queue, + } + return; + } +- /* If the port is engaged in resource sharing, do not use queue cache. */ +- cnt_queue = mlx5_hws_cnt_is_pool_shared(priv) ? NULL : &queue; ++ cnt_queue = mlx5_hws_cnt_get_queue(priv, &queue); + /* Put the counter first to reduce the race risk in BG thread. */ + mlx5_hws_cnt_pool_put(priv->hws_cpool, cnt_queue, &flow->cnt_id); + flow->cnt_id = 0; +@@ -3780,13 +3830,6 @@ flow_hw_pull_legacy_indirect_comp(struct rte_eth_dev *dev, struct mlx5_hw_q_job + job->query.hw); + aso_ct->state = ASO_CONNTRACK_READY; + } +- } else { +- /* +- * rte_flow_op_result::user data can point to +- * struct mlx5_aso_mtr object as well +- */ +- if (queue != CTRL_QUEUE_ID(priv)) +- MLX5_ASSERT(false); + } + } + +@@ -4368,12 +4411,23 @@ flow_hw_table_create(struct rte_eth_dev *dev, + matcher_attr.rule.num_log = rte_log2_u32(nb_flows); + /* Parse hints information. */ + if (attr->specialize) { +- if (attr->specialize == RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_WIRE_ORIG) +- matcher_attr.optimize_flow_src = MLX5DR_MATCHER_FLOW_SRC_WIRE; +- else if (attr->specialize == RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_VPORT_ORIG) +- matcher_attr.optimize_flow_src = MLX5DR_MATCHER_FLOW_SRC_VPORT; +- else +- DRV_LOG(INFO, "Unsupported hint value %x", attr->specialize); ++ uint32_t val = RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_WIRE_ORIG | ++ RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_VPORT_ORIG; ++ ++ if ((attr->specialize & val) == val) { ++ DRV_LOG(INFO, "Invalid hint value %x", ++ attr->specialize); ++ rte_errno = EINVAL; ++ goto it_error; ++ } ++ if (attr->specialize & ++ RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_WIRE_ORIG) ++ matcher_attr.optimize_flow_src = ++ MLX5DR_MATCHER_FLOW_SRC_WIRE; ++ else if (attr->specialize & ++ RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_VPORT_ORIG) ++ matcher_attr.optimize_flow_src = ++ MLX5DR_MATCHER_FLOW_SRC_VPORT; + } + /* Build the item template. */ + for (i = 0; i < nb_item_templates; i++) { +@@ -4623,7 +4677,7 @@ flow_hw_table_destroy(struct rte_eth_dev *dev, + return rte_flow_error_set(error, EBUSY, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, +- "table in use"); ++ "table is in use"); + } + LIST_REMOVE(table, next); + for (i = 0; i < table->nb_item_templates; i++) +@@ -4975,15 +5029,17 @@ flow_hw_validate_action_modify_field(struct rte_eth_dev *dev, + ret = flow_validate_modify_field_level(&action_conf->dst, error); + if (ret) + return ret; +- if (action_conf->dst.tag_index && +- !flow_modify_field_support_tag_array(action_conf->dst.field)) +- return rte_flow_error_set(error, EINVAL, +- RTE_FLOW_ERROR_TYPE_ACTION, action, +- "destination tag index is not supported"); +- if (action_conf->dst.class_id) +- return rte_flow_error_set(error, EINVAL, +- RTE_FLOW_ERROR_TYPE_ACTION, action, +- "destination class id is not supported"); ++ if (action_conf->dst.field != RTE_FLOW_FIELD_FLEX_ITEM) { ++ if (action_conf->dst.tag_index && ++ !flow_modify_field_support_tag_array(action_conf->dst.field)) ++ return rte_flow_error_set(error, EINVAL, ++ RTE_FLOW_ERROR_TYPE_ACTION, action, ++ "destination tag index is not supported"); ++ if (action_conf->dst.class_id) ++ return rte_flow_error_set(error, EINVAL, ++ RTE_FLOW_ERROR_TYPE_ACTION, action, ++ "destination class id is not supported"); ++ } + if (mask_conf->dst.level != UINT8_MAX) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, action, +@@ -4998,15 +5054,17 @@ flow_hw_validate_action_modify_field(struct rte_eth_dev *dev, + "destination field mask and template are not equal"); + if (action_conf->src.field != RTE_FLOW_FIELD_POINTER && + action_conf->src.field != RTE_FLOW_FIELD_VALUE) { +- if (action_conf->src.tag_index && +- !flow_modify_field_support_tag_array(action_conf->src.field)) +- return rte_flow_error_set(error, EINVAL, +- RTE_FLOW_ERROR_TYPE_ACTION, action, +- "source tag index is not supported"); +- if (action_conf->src.class_id) +- return rte_flow_error_set(error, EINVAL, +- RTE_FLOW_ERROR_TYPE_ACTION, action, +- "source class id is not supported"); ++ if (action_conf->src.field != RTE_FLOW_FIELD_FLEX_ITEM) { ++ if (action_conf->src.tag_index && ++ !flow_modify_field_support_tag_array(action_conf->src.field)) ++ return rte_flow_error_set(error, EINVAL, ++ RTE_FLOW_ERROR_TYPE_ACTION, action, ++ "source tag index is not supported"); ++ if (action_conf->src.class_id) ++ return rte_flow_error_set(error, EINVAL, ++ RTE_FLOW_ERROR_TYPE_ACTION, action, ++ "source class id is not supported"); ++ } + if (mask_conf->src.level != UINT8_MAX) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, action, +@@ -5443,6 +5501,69 @@ mlx5_decap_encap_reformat_type(const struct rte_flow_action *actions, + MLX5_FLOW_ACTION_ENCAP : MLX5_FLOW_ACTION_DECAP; + } + ++enum mlx5_hw_indirect_list_relative_position { ++ MLX5_INDIRECT_LIST_POSITION_UNKNOWN = -1, ++ MLX5_INDIRECT_LIST_POSITION_BEFORE_MH = 0, ++ MLX5_INDIRECT_LIST_POSITION_AFTER_MH, ++}; ++ ++static enum mlx5_hw_indirect_list_relative_position ++mlx5_hw_indirect_list_mh_position(const struct rte_flow_action *action) ++{ ++ const struct rte_flow_action_indirect_list *conf = action->conf; ++ enum mlx5_indirect_list_type list_type = mlx5_get_indirect_list_type(conf->handle); ++ enum mlx5_hw_indirect_list_relative_position pos = MLX5_INDIRECT_LIST_POSITION_UNKNOWN; ++ const union { ++ struct mlx5_indlst_legacy *legacy; ++ struct mlx5_hw_encap_decap_action *reformat; ++ struct rte_flow_action_list_handle *handle; ++ } h = { .handle = conf->handle}; ++ ++ switch (list_type) { ++ case MLX5_INDIRECT_ACTION_LIST_TYPE_LEGACY: ++ switch (h.legacy->legacy_type) { ++ case RTE_FLOW_ACTION_TYPE_AGE: ++ case RTE_FLOW_ACTION_TYPE_COUNT: ++ case RTE_FLOW_ACTION_TYPE_CONNTRACK: ++ case RTE_FLOW_ACTION_TYPE_METER_MARK: ++ case RTE_FLOW_ACTION_TYPE_QUOTA: ++ pos = MLX5_INDIRECT_LIST_POSITION_BEFORE_MH; ++ break; ++ case RTE_FLOW_ACTION_TYPE_RSS: ++ pos = MLX5_INDIRECT_LIST_POSITION_AFTER_MH; ++ break; ++ default: ++ pos = MLX5_INDIRECT_LIST_POSITION_UNKNOWN; ++ break; ++ } ++ break; ++ case MLX5_INDIRECT_ACTION_LIST_TYPE_MIRROR: ++ pos = MLX5_INDIRECT_LIST_POSITION_AFTER_MH; ++ break; ++ case MLX5_INDIRECT_ACTION_LIST_TYPE_REFORMAT: ++ switch (h.reformat->action_type) { ++ case MLX5DR_ACTION_TYP_REFORMAT_TNL_L2_TO_L2: ++ case MLX5DR_ACTION_TYP_REFORMAT_TNL_L3_TO_L2: ++ pos = MLX5_INDIRECT_LIST_POSITION_BEFORE_MH; ++ break; ++ case MLX5DR_ACTION_TYP_REFORMAT_L2_TO_TNL_L2: ++ case MLX5DR_ACTION_TYP_REFORMAT_L2_TO_TNL_L3: ++ pos = MLX5_INDIRECT_LIST_POSITION_AFTER_MH; ++ break; ++ default: ++ pos = MLX5_INDIRECT_LIST_POSITION_UNKNOWN; ++ break; ++ } ++ break; ++ default: ++ pos = MLX5_INDIRECT_LIST_POSITION_UNKNOWN; ++ break; ++ } ++ return pos; ++} ++ ++#define MLX5_HW_EXPAND_MH_FAILED 0xffff ++ + static inline uint16_t + flow_hw_template_expand_modify_field(struct rte_flow_action actions[], + struct rte_flow_action masks[], +@@ -5479,6 +5600,7 @@ flow_hw_template_expand_modify_field(struct rte_flow_action actions[], + * @see action_order_arr[] + */ + for (i = act_num - 2; (int)i >= 0; i--) { ++ enum mlx5_hw_indirect_list_relative_position pos; + enum rte_flow_action_type type = actions[i].type; + uint64_t reformat_type; + +@@ -5509,6 +5631,13 @@ flow_hw_template_expand_modify_field(struct rte_flow_action actions[], + if (actions[i - 1].type == RTE_FLOW_ACTION_TYPE_RAW_DECAP) + i--; + break; ++ case RTE_FLOW_ACTION_TYPE_INDIRECT_LIST: ++ pos = mlx5_hw_indirect_list_mh_position(&actions[i]); ++ if (pos == MLX5_INDIRECT_LIST_POSITION_UNKNOWN) ++ return MLX5_HW_EXPAND_MH_FAILED; ++ if (pos == MLX5_INDIRECT_LIST_POSITION_BEFORE_MH) ++ goto insert; ++ break; + default: + i++; /* new MF inserted AFTER actions[i] */ + goto insert; +@@ -5639,6 +5768,8 @@ mlx5_flow_hw_actions_validate(struct rte_eth_dev *dev, + int ret; + const struct rte_flow_action_ipv6_ext_remove *remove_data; + ++ if (!mlx5_hw_ctx_validate(dev, error)) ++ return -rte_errno; + /* FDB actions are only valid to proxy port. */ + if (attr->transfer && (!priv->sh->config.dv_esw_en || !priv->master)) + return rte_flow_error_set(error, EINVAL, +@@ -6151,7 +6282,6 @@ flow_hw_set_vlan_vid(struct rte_eth_dev *dev, + rm[set_vlan_vid_ix].conf)->vlan_vid != 0); + const struct rte_flow_action_of_set_vlan_vid *conf = + ra[set_vlan_vid_ix].conf; +- rte_be16_t vid = masked ? conf->vlan_vid : 0; + int width = mlx5_flow_item_field_width(dev, RTE_FLOW_FIELD_VLAN_ID, 0, + NULL, &error); + *spec = (typeof(*spec)) { +@@ -6162,8 +6292,6 @@ flow_hw_set_vlan_vid(struct rte_eth_dev *dev, + }, + .src = { + .field = RTE_FLOW_FIELD_VALUE, +- .level = vid, +- .offset = 0, + }, + .width = width, + }; +@@ -6175,11 +6303,15 @@ flow_hw_set_vlan_vid(struct rte_eth_dev *dev, + }, + .src = { + .field = RTE_FLOW_FIELD_VALUE, +- .level = masked ? (1U << width) - 1 : 0, +- .offset = 0, + }, + .width = 0xffffffff, + }; ++ if (masked) { ++ uint32_t mask_val = 0xffffffff; ++ ++ rte_memcpy(spec->src.value, &conf->vlan_vid, sizeof(conf->vlan_vid)); ++ rte_memcpy(mask->src.value, &mask_val, sizeof(mask_val)); ++ } + ra[set_vlan_vid_ix].type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD; + ra[set_vlan_vid_ix].conf = spec; + rm[set_vlan_vid_ix].type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD; +@@ -6206,8 +6338,6 @@ flow_hw_set_vlan_vid_construct(struct rte_eth_dev *dev, + }, + .src = { + .field = RTE_FLOW_FIELD_VALUE, +- .level = vid, +- .offset = 0, + }, + .width = width, + }; +@@ -6216,6 +6346,7 @@ flow_hw_set_vlan_vid_construct(struct rte_eth_dev *dev, + .conf = &conf + }; + ++ rte_memcpy(conf.src.value, &vid, sizeof(vid)); + return flow_hw_modify_field_construct(job, act_data, hw_acts, + &modify_action); + } +@@ -6463,6 +6594,12 @@ flow_hw_actions_template_create(struct rte_eth_dev *dev, + action_flags, + act_num, + expand_mf_num); ++ if (pos == MLX5_HW_EXPAND_MH_FAILED) { ++ rte_flow_error_set(error, ENOMEM, ++ RTE_FLOW_ERROR_TYPE_UNSPECIFIED, ++ NULL, "modify header expansion failed"); ++ return NULL; ++ } + act_num += expand_mf_num; + for (i = pos + expand_mf_num; i < act_num; i++) + src_off[i] += expand_mf_num; +@@ -6585,7 +6722,7 @@ flow_hw_actions_template_destroy(struct rte_eth_dev *dev, + return rte_flow_error_set(error, EBUSY, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, +- "action template in using"); ++ "action template is in use"); + } + if (template->action_flags & flag) + mlx5_free_srh_flex_parser(dev); +@@ -6645,6 +6782,8 @@ flow_hw_pattern_validate(struct rte_eth_dev *dev, + bool items_end = false; + uint32_t tag_bitmap = 0; + ++ if (!mlx5_hw_ctx_validate(dev, error)) ++ return -rte_errno; + if (!attr->ingress && !attr->egress && !attr->transfer) + return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ATTR, NULL, + "at least one of the direction attributes" +@@ -7003,7 +7142,7 @@ flow_hw_pattern_template_destroy(struct rte_eth_dev *dev, + return rte_flow_error_set(error, EBUSY, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, +- "item template in using"); ++ "item template is in use"); + } + if (template->item_flags & (MLX5_FLOW_ITEM_OUTER_IPV6_ROUTING_EXT | + MLX5_FLOW_ITEM_INNER_IPV6_ROUTING_EXT)) +@@ -8366,6 +8505,72 @@ flow_hw_create_ctrl_jump_table(struct rte_eth_dev *dev, + return flow_hw_table_create(dev, &cfg, &it, 1, &at, 1, error); + } + ++/** ++ * Cleans up all template tables and pattern, and actions templates used for ++ * FDB control flow rules. ++ * ++ * @param dev ++ * Pointer to Ethernet device. ++ */ ++static void ++flow_hw_cleanup_ctrl_fdb_tables(struct rte_eth_dev *dev) ++{ ++ struct mlx5_priv *priv = dev->data->dev_private; ++ struct mlx5_flow_hw_ctrl_fdb *hw_ctrl_fdb; ++ ++ if (!priv->hw_ctrl_fdb) ++ return; ++ hw_ctrl_fdb = priv->hw_ctrl_fdb; ++ /* Clean up templates used for LACP default miss table. */ ++ if (hw_ctrl_fdb->hw_lacp_rx_tbl) ++ claim_zero(flow_hw_table_destroy(dev, hw_ctrl_fdb->hw_lacp_rx_tbl, NULL)); ++ if (hw_ctrl_fdb->lacp_rx_actions_tmpl) ++ claim_zero(flow_hw_actions_template_destroy(dev, hw_ctrl_fdb->lacp_rx_actions_tmpl, ++ NULL)); ++ if (hw_ctrl_fdb->lacp_rx_items_tmpl) ++ claim_zero(flow_hw_pattern_template_destroy(dev, hw_ctrl_fdb->lacp_rx_items_tmpl, ++ NULL)); ++ /* Clean up templates used for default Tx metadata copy. */ ++ if (hw_ctrl_fdb->hw_tx_meta_cpy_tbl) ++ claim_zero(flow_hw_table_destroy(dev, hw_ctrl_fdb->hw_tx_meta_cpy_tbl, NULL)); ++ if (hw_ctrl_fdb->tx_meta_actions_tmpl) ++ claim_zero(flow_hw_actions_template_destroy(dev, hw_ctrl_fdb->tx_meta_actions_tmpl, ++ NULL)); ++ if (hw_ctrl_fdb->tx_meta_items_tmpl) ++ claim_zero(flow_hw_pattern_template_destroy(dev, hw_ctrl_fdb->tx_meta_items_tmpl, ++ NULL)); ++ /* Clean up templates used for default FDB jump rule. */ ++ if (hw_ctrl_fdb->hw_esw_zero_tbl) ++ claim_zero(flow_hw_table_destroy(dev, hw_ctrl_fdb->hw_esw_zero_tbl, NULL)); ++ if (hw_ctrl_fdb->jump_one_actions_tmpl) ++ claim_zero(flow_hw_actions_template_destroy(dev, hw_ctrl_fdb->jump_one_actions_tmpl, ++ NULL)); ++ if (hw_ctrl_fdb->port_items_tmpl) ++ claim_zero(flow_hw_pattern_template_destroy(dev, hw_ctrl_fdb->port_items_tmpl, ++ NULL)); ++ /* Clean up templates used for default SQ miss flow rules - non-root table. */ ++ if (hw_ctrl_fdb->hw_esw_sq_miss_tbl) ++ claim_zero(flow_hw_table_destroy(dev, hw_ctrl_fdb->hw_esw_sq_miss_tbl, NULL)); ++ if (hw_ctrl_fdb->regc_sq_items_tmpl) ++ claim_zero(flow_hw_pattern_template_destroy(dev, hw_ctrl_fdb->regc_sq_items_tmpl, ++ NULL)); ++ if (hw_ctrl_fdb->port_actions_tmpl) ++ claim_zero(flow_hw_actions_template_destroy(dev, hw_ctrl_fdb->port_actions_tmpl, ++ NULL)); ++ /* Clean up templates used for default SQ miss flow rules - root table. */ ++ if (hw_ctrl_fdb->hw_esw_sq_miss_root_tbl) ++ claim_zero(flow_hw_table_destroy(dev, hw_ctrl_fdb->hw_esw_sq_miss_root_tbl, NULL)); ++ if (hw_ctrl_fdb->regc_jump_actions_tmpl) ++ claim_zero(flow_hw_actions_template_destroy(dev, ++ hw_ctrl_fdb->regc_jump_actions_tmpl, NULL)); ++ if (hw_ctrl_fdb->esw_mgr_items_tmpl) ++ claim_zero(flow_hw_pattern_template_destroy(dev, hw_ctrl_fdb->esw_mgr_items_tmpl, ++ NULL)); ++ /* Clean up templates structure for FDB control flow rules. */ ++ mlx5_free(hw_ctrl_fdb); ++ priv->hw_ctrl_fdb = NULL; ++} ++ + /* + * Create a table on the root group to for the LACP traffic redirecting. + * +@@ -8415,182 +8620,154 @@ flow_hw_create_lacp_rx_table(struct rte_eth_dev *dev, + * @return + * 0 on success, negative values otherwise + */ +-static __rte_unused int ++static int + flow_hw_create_ctrl_tables(struct rte_eth_dev *dev, struct rte_flow_error *error) + { + struct mlx5_priv *priv = dev->data->dev_private; +- struct rte_flow_pattern_template *esw_mgr_items_tmpl = NULL; +- struct rte_flow_pattern_template *regc_sq_items_tmpl = NULL; +- struct rte_flow_pattern_template *port_items_tmpl = NULL; +- struct rte_flow_pattern_template *tx_meta_items_tmpl = NULL; +- struct rte_flow_pattern_template *lacp_rx_items_tmpl = NULL; +- struct rte_flow_actions_template *regc_jump_actions_tmpl = NULL; +- struct rte_flow_actions_template *port_actions_tmpl = NULL; +- struct rte_flow_actions_template *jump_one_actions_tmpl = NULL; +- struct rte_flow_actions_template *tx_meta_actions_tmpl = NULL; +- struct rte_flow_actions_template *lacp_rx_actions_tmpl = NULL; ++ struct mlx5_flow_hw_ctrl_fdb *hw_ctrl_fdb; + uint32_t xmeta = priv->sh->config.dv_xmeta_en; + uint32_t repr_matching = priv->sh->config.repr_matching; +- int ret; ++ uint32_t fdb_def_rule = priv->sh->config.fdb_def_rule; + +- /* Create templates and table for default SQ miss flow rules - root table. */ +- esw_mgr_items_tmpl = flow_hw_create_ctrl_esw_mgr_pattern_template(dev, error); +- if (!esw_mgr_items_tmpl) { +- DRV_LOG(ERR, "port %u failed to create E-Switch Manager item" +- " template for control flows", dev->data->port_id); +- goto err; +- } +- regc_jump_actions_tmpl = flow_hw_create_ctrl_regc_jump_actions_template(dev, error); +- if (!regc_jump_actions_tmpl) { +- DRV_LOG(ERR, "port %u failed to create REG_C set and jump action template" +- " for control flows", dev->data->port_id); +- goto err; +- } +- MLX5_ASSERT(priv->hw_esw_sq_miss_root_tbl == NULL); +- priv->hw_esw_sq_miss_root_tbl = flow_hw_create_ctrl_sq_miss_root_table +- (dev, esw_mgr_items_tmpl, regc_jump_actions_tmpl, error); +- if (!priv->hw_esw_sq_miss_root_tbl) { +- DRV_LOG(ERR, "port %u failed to create table for default sq miss (root table)" +- " for control flows", dev->data->port_id); +- goto err; +- } +- /* Create templates and table for default SQ miss flow rules - non-root table. */ +- regc_sq_items_tmpl = flow_hw_create_ctrl_regc_sq_pattern_template(dev, error); +- if (!regc_sq_items_tmpl) { +- DRV_LOG(ERR, "port %u failed to create SQ item template for" +- " control flows", dev->data->port_id); +- goto err; +- } +- port_actions_tmpl = flow_hw_create_ctrl_port_actions_template(dev, error); +- if (!port_actions_tmpl) { +- DRV_LOG(ERR, "port %u failed to create port action template" +- " for control flows", dev->data->port_id); +- goto err; +- } +- MLX5_ASSERT(priv->hw_esw_sq_miss_tbl == NULL); +- priv->hw_esw_sq_miss_tbl = flow_hw_create_ctrl_sq_miss_table(dev, regc_sq_items_tmpl, +- port_actions_tmpl, error); +- if (!priv->hw_esw_sq_miss_tbl) { +- DRV_LOG(ERR, "port %u failed to create table for default sq miss (non-root table)" +- " for control flows", dev->data->port_id); +- goto err; +- } +- /* Create templates and table for default FDB jump flow rules. */ +- port_items_tmpl = flow_hw_create_ctrl_port_pattern_template(dev, error); +- if (!port_items_tmpl) { +- DRV_LOG(ERR, "port %u failed to create SQ item template for" +- " control flows", dev->data->port_id); +- goto err; +- } +- jump_one_actions_tmpl = flow_hw_create_ctrl_jump_actions_template +- (dev, MLX5_HW_LOWEST_USABLE_GROUP, error); +- if (!jump_one_actions_tmpl) { +- DRV_LOG(ERR, "port %u failed to create jump action template" +- " for control flows", dev->data->port_id); ++ MLX5_ASSERT(priv->hw_ctrl_fdb == NULL); ++ hw_ctrl_fdb = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*hw_ctrl_fdb), 0, SOCKET_ID_ANY); ++ if (!hw_ctrl_fdb) { ++ DRV_LOG(ERR, "port %u failed to allocate memory for FDB control flow templates", ++ dev->data->port_id); ++ rte_errno = ENOMEM; + goto err; + } +- MLX5_ASSERT(priv->hw_esw_zero_tbl == NULL); +- priv->hw_esw_zero_tbl = flow_hw_create_ctrl_jump_table(dev, port_items_tmpl, +- jump_one_actions_tmpl, +- error); +- if (!priv->hw_esw_zero_tbl) { +- DRV_LOG(ERR, "port %u failed to create table for default jump to group 1" +- " for control flows", dev->data->port_id); +- goto err; ++ priv->hw_ctrl_fdb = hw_ctrl_fdb; ++ if (fdb_def_rule) { ++ /* Create templates and table for default SQ miss flow rules - root table. */ ++ hw_ctrl_fdb->esw_mgr_items_tmpl = ++ flow_hw_create_ctrl_esw_mgr_pattern_template(dev, error); ++ if (!hw_ctrl_fdb->esw_mgr_items_tmpl) { ++ DRV_LOG(ERR, "port %u failed to create E-Switch Manager item" ++ " template for control flows", dev->data->port_id); ++ goto err; ++ } ++ hw_ctrl_fdb->regc_jump_actions_tmpl = ++ flow_hw_create_ctrl_regc_jump_actions_template(dev, error); ++ if (!hw_ctrl_fdb->regc_jump_actions_tmpl) { ++ DRV_LOG(ERR, "port %u failed to create REG_C set and jump action template" ++ " for control flows", dev->data->port_id); ++ goto err; ++ } ++ hw_ctrl_fdb->hw_esw_sq_miss_root_tbl = ++ flow_hw_create_ctrl_sq_miss_root_table ++ (dev, hw_ctrl_fdb->esw_mgr_items_tmpl, ++ hw_ctrl_fdb->regc_jump_actions_tmpl, error); ++ if (!hw_ctrl_fdb->hw_esw_sq_miss_root_tbl) { ++ DRV_LOG(ERR, "port %u failed to create table for default sq miss (root table)" ++ " for control flows", dev->data->port_id); ++ goto err; ++ } ++ /* Create templates and table for default SQ miss flow rules - non-root table. */ ++ hw_ctrl_fdb->regc_sq_items_tmpl = ++ flow_hw_create_ctrl_regc_sq_pattern_template(dev, error); ++ if (!hw_ctrl_fdb->regc_sq_items_tmpl) { ++ DRV_LOG(ERR, "port %u failed to create SQ item template for" ++ " control flows", dev->data->port_id); ++ goto err; ++ } ++ hw_ctrl_fdb->port_actions_tmpl = ++ flow_hw_create_ctrl_port_actions_template(dev, error); ++ if (!hw_ctrl_fdb->port_actions_tmpl) { ++ DRV_LOG(ERR, "port %u failed to create port action template" ++ " for control flows", dev->data->port_id); ++ goto err; ++ } ++ hw_ctrl_fdb->hw_esw_sq_miss_tbl = ++ flow_hw_create_ctrl_sq_miss_table ++ (dev, hw_ctrl_fdb->regc_sq_items_tmpl, ++ hw_ctrl_fdb->port_actions_tmpl, error); ++ if (!hw_ctrl_fdb->hw_esw_sq_miss_tbl) { ++ DRV_LOG(ERR, "port %u failed to create table for default sq miss (non-root table)" ++ " for control flows", dev->data->port_id); ++ goto err; ++ } ++ /* Create templates and table for default FDB jump flow rules. */ ++ hw_ctrl_fdb->port_items_tmpl = ++ flow_hw_create_ctrl_port_pattern_template(dev, error); ++ if (!hw_ctrl_fdb->port_items_tmpl) { ++ DRV_LOG(ERR, "port %u failed to create SQ item template for" ++ " control flows", dev->data->port_id); ++ goto err; ++ } ++ hw_ctrl_fdb->jump_one_actions_tmpl = ++ flow_hw_create_ctrl_jump_actions_template ++ (dev, MLX5_HW_LOWEST_USABLE_GROUP, error); ++ if (!hw_ctrl_fdb->jump_one_actions_tmpl) { ++ DRV_LOG(ERR, "port %u failed to create jump action template" ++ " for control flows", dev->data->port_id); ++ goto err; ++ } ++ hw_ctrl_fdb->hw_esw_zero_tbl = flow_hw_create_ctrl_jump_table ++ (dev, hw_ctrl_fdb->port_items_tmpl, ++ hw_ctrl_fdb->jump_one_actions_tmpl, error); ++ if (!hw_ctrl_fdb->hw_esw_zero_tbl) { ++ DRV_LOG(ERR, "port %u failed to create table for default jump to group 1" ++ " for control flows", dev->data->port_id); ++ goto err; ++ } + } + /* Create templates and table for default Tx metadata copy flow rule. */ + if (!repr_matching && xmeta == MLX5_XMETA_MODE_META32_HWS) { +- tx_meta_items_tmpl = ++ hw_ctrl_fdb->tx_meta_items_tmpl = + flow_hw_create_tx_default_mreg_copy_pattern_template(dev, error); +- if (!tx_meta_items_tmpl) { ++ if (!hw_ctrl_fdb->tx_meta_items_tmpl) { + DRV_LOG(ERR, "port %u failed to Tx metadata copy pattern" + " template for control flows", dev->data->port_id); + goto err; + } +- tx_meta_actions_tmpl = ++ hw_ctrl_fdb->tx_meta_actions_tmpl = + flow_hw_create_tx_default_mreg_copy_actions_template(dev, error); +- if (!tx_meta_actions_tmpl) { ++ if (!hw_ctrl_fdb->tx_meta_actions_tmpl) { + DRV_LOG(ERR, "port %u failed to Tx metadata copy actions" + " template for control flows", dev->data->port_id); + goto err; + } +- MLX5_ASSERT(priv->hw_tx_meta_cpy_tbl == NULL); +- priv->hw_tx_meta_cpy_tbl = +- flow_hw_create_tx_default_mreg_copy_table(dev, tx_meta_items_tmpl, +- tx_meta_actions_tmpl, error); +- if (!priv->hw_tx_meta_cpy_tbl) { ++ hw_ctrl_fdb->hw_tx_meta_cpy_tbl = ++ flow_hw_create_tx_default_mreg_copy_table ++ (dev, hw_ctrl_fdb->tx_meta_items_tmpl, ++ hw_ctrl_fdb->tx_meta_actions_tmpl, error); ++ if (!hw_ctrl_fdb->hw_tx_meta_cpy_tbl) { + DRV_LOG(ERR, "port %u failed to create table for default" + " Tx metadata copy flow rule", dev->data->port_id); + goto err; + } + } + /* Create LACP default miss table. */ +- if (!priv->sh->config.lacp_by_user && priv->pf_bond >= 0) { +- lacp_rx_items_tmpl = flow_hw_create_lacp_rx_pattern_template(dev, error); +- if (!lacp_rx_items_tmpl) { ++ if (!priv->sh->config.lacp_by_user && priv->pf_bond >= 0 && priv->master) { ++ hw_ctrl_fdb->lacp_rx_items_tmpl = ++ flow_hw_create_lacp_rx_pattern_template(dev, error); ++ if (!hw_ctrl_fdb->lacp_rx_items_tmpl) { + DRV_LOG(ERR, "port %u failed to create pattern template" + " for LACP Rx traffic", dev->data->port_id); + goto err; + } +- lacp_rx_actions_tmpl = flow_hw_create_lacp_rx_actions_template(dev, error); +- if (!lacp_rx_actions_tmpl) { ++ hw_ctrl_fdb->lacp_rx_actions_tmpl = ++ flow_hw_create_lacp_rx_actions_template(dev, error); ++ if (!hw_ctrl_fdb->lacp_rx_actions_tmpl) { + DRV_LOG(ERR, "port %u failed to create actions template" + " for LACP Rx traffic", dev->data->port_id); + goto err; + } +- priv->hw_lacp_rx_tbl = flow_hw_create_lacp_rx_table(dev, lacp_rx_items_tmpl, +- lacp_rx_actions_tmpl, error); +- if (!priv->hw_lacp_rx_tbl) { ++ hw_ctrl_fdb->hw_lacp_rx_tbl = flow_hw_create_lacp_rx_table ++ (dev, hw_ctrl_fdb->lacp_rx_items_tmpl, ++ hw_ctrl_fdb->lacp_rx_actions_tmpl, error); ++ if (!hw_ctrl_fdb->hw_lacp_rx_tbl) { + DRV_LOG(ERR, "port %u failed to create template table for" + " for LACP Rx traffic", dev->data->port_id); + goto err; + } + } + return 0; ++ + err: +- /* Do not overwrite the rte_errno. */ +- ret = -rte_errno; +- if (ret == 0) +- ret = rte_flow_error_set(error, EINVAL, +- RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, +- "Failed to create control tables."); +- if (priv->hw_tx_meta_cpy_tbl) { +- flow_hw_table_destroy(dev, priv->hw_tx_meta_cpy_tbl, NULL); +- priv->hw_tx_meta_cpy_tbl = NULL; +- } +- if (priv->hw_esw_zero_tbl) { +- flow_hw_table_destroy(dev, priv->hw_esw_zero_tbl, NULL); +- priv->hw_esw_zero_tbl = NULL; +- } +- if (priv->hw_esw_sq_miss_tbl) { +- flow_hw_table_destroy(dev, priv->hw_esw_sq_miss_tbl, NULL); +- priv->hw_esw_sq_miss_tbl = NULL; +- } +- if (priv->hw_esw_sq_miss_root_tbl) { +- flow_hw_table_destroy(dev, priv->hw_esw_sq_miss_root_tbl, NULL); +- priv->hw_esw_sq_miss_root_tbl = NULL; +- } +- if (lacp_rx_actions_tmpl) +- flow_hw_actions_template_destroy(dev, lacp_rx_actions_tmpl, NULL); +- if (tx_meta_actions_tmpl) +- flow_hw_actions_template_destroy(dev, tx_meta_actions_tmpl, NULL); +- if (jump_one_actions_tmpl) +- flow_hw_actions_template_destroy(dev, jump_one_actions_tmpl, NULL); +- if (port_actions_tmpl) +- flow_hw_actions_template_destroy(dev, port_actions_tmpl, NULL); +- if (regc_jump_actions_tmpl) +- flow_hw_actions_template_destroy(dev, regc_jump_actions_tmpl, NULL); +- if (lacp_rx_items_tmpl) +- flow_hw_pattern_template_destroy(dev, lacp_rx_items_tmpl, NULL); +- if (tx_meta_items_tmpl) +- flow_hw_pattern_template_destroy(dev, tx_meta_items_tmpl, NULL); +- if (port_items_tmpl) +- flow_hw_pattern_template_destroy(dev, port_items_tmpl, NULL); +- if (regc_sq_items_tmpl) +- flow_hw_pattern_template_destroy(dev, regc_sq_items_tmpl, NULL); +- if (esw_mgr_items_tmpl) +- flow_hw_pattern_template_destroy(dev, esw_mgr_items_tmpl, NULL); +- return ret; ++ flow_hw_cleanup_ctrl_fdb_tables(dev); ++ return -EINVAL; + } + + static void +@@ -9184,6 +9361,38 @@ flow_hw_compare_config(const struct mlx5_flow_hw_attr *hw_attr, + return true; + } + ++static int ++flow_hw_validate_attributes(const struct rte_flow_port_attr *port_attr, ++ uint16_t nb_queue, ++ const struct rte_flow_queue_attr *queue_attr[], ++ struct rte_flow_error *error) ++{ ++ uint32_t size; ++ unsigned int i; ++ ++ if (port_attr == NULL) ++ return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, ++ "Port attributes must be non-NULL"); ++ ++ if (nb_queue == 0) ++ return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, ++ "At least one flow queue is required"); ++ ++ if (queue_attr == NULL) ++ return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, ++ "Queue attributes must be non-NULL"); ++ ++ size = queue_attr[0]->size; ++ for (i = 1; i < nb_queue; ++i) { ++ if (queue_attr[i]->size != size) ++ return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, ++ NULL, ++ "All flow queues must have the same size"); ++ } ++ ++ return 0; ++} ++ + /** + * Configure port HWS resources. + * +@@ -9235,10 +9444,8 @@ flow_hw_configure(struct rte_eth_dev *dev, + int ret = 0; + uint32_t action_flags; + +- if (!port_attr || !nb_queue || !queue_attr) { +- rte_errno = EINVAL; +- goto err; +- } ++ if (flow_hw_validate_attributes(port_attr, nb_queue, queue_attr, error)) ++ return -rte_errno; + /* + * Calling rte_flow_configure() again is allowed if and only if + * provided configuration matches the initially provided one. +@@ -9285,14 +9492,6 @@ flow_hw_configure(struct rte_eth_dev *dev, + /* Allocate the queue job descriptor LIFO. */ + mem_size = sizeof(priv->hw_q[0]) * nb_q_updated; + for (i = 0; i < nb_q_updated; i++) { +- /* +- * Check if the queues' size are all the same as the +- * limitation from HWS layer. +- */ +- if (_queue_attr[i]->size != _queue_attr[0]->size) { +- rte_errno = EINVAL; +- goto err; +- } + mem_size += (sizeof(struct mlx5_hw_q_job *) + + sizeof(struct mlx5_hw_q_job) + + sizeof(uint8_t) * MLX5_ENCAP_MAX_LEN + +@@ -9545,6 +9744,14 @@ flow_hw_configure(struct rte_eth_dev *dev, + priv->hws_strict_queue = 1; + return 0; + err: ++ priv->hws_strict_queue = 0; ++ flow_hw_destroy_vlan(dev); ++ if (priv->hws_age_req) ++ mlx5_hws_age_pool_destroy(priv); ++ if (priv->hws_cpool) { ++ mlx5_hws_cnt_pool_destroy(priv->sh, priv->hws_cpool); ++ priv->hws_cpool = NULL; ++ } + if (priv->hws_ctpool) { + flow_hw_ct_pool_destroy(dev, priv->hws_ctpool); + priv->hws_ctpool = NULL; +@@ -9553,44 +9760,54 @@ err: + flow_hw_ct_mng_destroy(dev, priv->ct_mng); + priv->ct_mng = NULL; + } +- if (priv->hws_age_req) +- mlx5_hws_age_pool_destroy(priv); +- if (priv->hws_cpool) { +- mlx5_hws_cnt_pool_destroy(priv->sh, priv->hws_cpool); +- priv->hws_cpool = NULL; +- } +- mlx5_flow_quota_destroy(dev); + flow_hw_destroy_send_to_kernel_action(priv); ++ flow_hw_cleanup_ctrl_fdb_tables(dev); + flow_hw_free_vport_actions(priv); ++ if (priv->hw_def_miss) { ++ mlx5dr_action_destroy(priv->hw_def_miss); ++ priv->hw_def_miss = NULL; ++ } ++ flow_hw_cleanup_tx_repr_tagging(dev); + for (i = 0; i < MLX5_HW_ACTION_FLAG_MAX; i++) { +- if (priv->hw_drop[i]) ++ if (priv->hw_drop[i]) { + mlx5dr_action_destroy(priv->hw_drop[i]); +- if (priv->hw_tag[i]) ++ priv->hw_drop[i] = NULL; ++ } ++ if (priv->hw_tag[i]) { + mlx5dr_action_destroy(priv->hw_tag[i]); ++ priv->hw_tag[i] = NULL; ++ } + } +- if (priv->hw_def_miss) +- mlx5dr_action_destroy(priv->hw_def_miss); +- flow_hw_destroy_vlan(dev); +- if (dr_ctx) ++ mlx5_flow_meter_uninit(dev); ++ mlx5_flow_quota_destroy(dev); ++ flow_hw_cleanup_ctrl_rx_tables(dev); ++ if (dr_ctx) { + claim_zero(mlx5dr_context_close(dr_ctx)); +- for (i = 0; i < nb_q_updated; i++) { +- rte_ring_free(priv->hw_q[i].indir_iq); +- rte_ring_free(priv->hw_q[i].indir_cq); ++ priv->dr_ctx = NULL; + } +- mlx5_free(priv->hw_q); +- priv->hw_q = NULL; +- if (priv->acts_ipool) { +- mlx5_ipool_destroy(priv->acts_ipool); +- priv->acts_ipool = NULL; +- } +- if (_queue_attr) +- mlx5_free(_queue_attr); + if (priv->shared_host) { ++ struct mlx5_priv *host_priv = priv->shared_host->data->dev_private; ++ + __atomic_fetch_sub(&host_priv->shared_refcnt, 1, __ATOMIC_RELAXED); + priv->shared_host = NULL; + } ++ if (priv->hw_q) { ++ for (i = 0; i < nb_q_updated; i++) { ++ rte_ring_free(priv->hw_q[i].indir_iq); ++ rte_ring_free(priv->hw_q[i].indir_cq); ++ } ++ mlx5_free(priv->hw_q); ++ priv->hw_q = NULL; ++ } ++ if (priv->acts_ipool) { ++ mlx5_ipool_destroy(priv->acts_ipool); ++ priv->acts_ipool = NULL; ++ } + mlx5_free(priv->hw_attr); + priv->hw_attr = NULL; ++ priv->nb_queue = 0; ++ if (_queue_attr) ++ mlx5_free(_queue_attr); + /* Do not overwrite the internal errno information. */ + if (ret) + return ret; +@@ -9609,37 +9826,48 @@ void + flow_hw_resource_release(struct rte_eth_dev *dev) + { + struct mlx5_priv *priv = dev->data->dev_private; +- struct rte_flow_template_table *tbl; +- struct rte_flow_pattern_template *it; +- struct rte_flow_actions_template *at; +- struct mlx5_flow_group *grp; ++ struct rte_flow_template_table *tbl, *temp_tbl; ++ struct rte_flow_pattern_template *it, *temp_it; ++ struct rte_flow_actions_template *at, *temp_at; ++ struct mlx5_flow_group *grp, *temp_grp; + uint32_t i; + + if (!priv->dr_ctx) + return; + flow_hw_rxq_flag_set(dev, false); + flow_hw_flush_all_ctrl_flows(dev); ++ flow_hw_cleanup_ctrl_fdb_tables(dev); + flow_hw_cleanup_tx_repr_tagging(dev); + flow_hw_cleanup_ctrl_rx_tables(dev); +- while (!LIST_EMPTY(&priv->flow_hw_grp)) { +- grp = LIST_FIRST(&priv->flow_hw_grp); +- flow_hw_group_unset_miss_group(dev, grp, NULL); +- } +- while (!LIST_EMPTY(&priv->flow_hw_tbl_ongo)) { +- tbl = LIST_FIRST(&priv->flow_hw_tbl_ongo); +- flow_hw_table_destroy(dev, tbl, NULL); +- } +- while (!LIST_EMPTY(&priv->flow_hw_tbl)) { +- tbl = LIST_FIRST(&priv->flow_hw_tbl); +- flow_hw_table_destroy(dev, tbl, NULL); +- } +- while (!LIST_EMPTY(&priv->flow_hw_itt)) { +- it = LIST_FIRST(&priv->flow_hw_itt); +- flow_hw_pattern_template_destroy(dev, it, NULL); +- } +- while (!LIST_EMPTY(&priv->flow_hw_at)) { +- at = LIST_FIRST(&priv->flow_hw_at); +- flow_hw_actions_template_destroy(dev, at, NULL); ++ grp = LIST_FIRST(&priv->flow_hw_grp); ++ while (grp) { ++ temp_grp = LIST_NEXT(grp, next); ++ claim_zero(flow_hw_group_unset_miss_group(dev, grp, NULL)); ++ grp = temp_grp; ++ } ++ tbl = LIST_FIRST(&priv->flow_hw_tbl_ongo); ++ while (tbl) { ++ temp_tbl = LIST_NEXT(tbl, next); ++ claim_zero(flow_hw_table_destroy(dev, tbl, NULL)); ++ tbl = temp_tbl; ++ } ++ tbl = LIST_FIRST(&priv->flow_hw_tbl); ++ while (tbl) { ++ temp_tbl = LIST_NEXT(tbl, next); ++ claim_zero(flow_hw_table_destroy(dev, tbl, NULL)); ++ tbl = temp_tbl; ++ } ++ it = LIST_FIRST(&priv->flow_hw_itt); ++ while (it) { ++ temp_it = LIST_NEXT(it, next); ++ claim_zero(flow_hw_pattern_template_destroy(dev, it, NULL)); ++ it = temp_it; ++ } ++ at = LIST_FIRST(&priv->flow_hw_at); ++ while (at) { ++ temp_at = LIST_NEXT(at, next); ++ claim_zero(flow_hw_actions_template_destroy(dev, at, NULL)); ++ at = temp_at; + } + for (i = 0; i < MLX5_HW_ACTION_FLAG_MAX; i++) { + if (priv->hw_drop[i]) +@@ -9677,13 +9905,11 @@ flow_hw_resource_release(struct rte_eth_dev *dev) + } + mlx5_free(priv->hw_q); + priv->hw_q = NULL; +- claim_zero(mlx5dr_context_close(priv->dr_ctx)); + if (priv->shared_host) { + struct mlx5_priv *host_priv = priv->shared_host->data->dev_private; + __atomic_fetch_sub(&host_priv->shared_refcnt, 1, __ATOMIC_RELAXED); + priv->shared_host = NULL; + } +- priv->dr_ctx = NULL; + mlx5_free(priv->hw_attr); + priv->hw_attr = NULL; + priv->nb_queue = 0; +@@ -9853,6 +10079,13 @@ flow_hw_conntrack_create(struct rte_eth_dev *dev, uint32_t queue, + "CT is not enabled"); + return 0; + } ++ if (dev->data->port_id >= MLX5_INDIRECT_ACT_CT_MAX_PORT) { ++ rte_flow_error_set(error, EINVAL, ++ RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, ++ "CT supports port indexes up to " ++ RTE_STR(MLX5_ACTION_CTX_CT_MAX_PORT)); ++ return 0; ++ } + ct = mlx5_ipool_zmalloc(pool->cts, &ct_idx); + if (!ct) { + rte_flow_error_set(error, rte_errno, +@@ -9967,11 +10200,13 @@ flow_hw_action_job_init(struct mlx5_priv *priv, uint32_t queue, + const struct rte_flow_action_handle *handle, + void *user_data, void *query_data, + enum mlx5_hw_job_type type, ++ enum mlx5_hw_indirect_type indirect_type, + struct rte_flow_error *error) + { + struct mlx5_hw_q_job *job; + +- MLX5_ASSERT(queue != MLX5_HW_INV_QUEUE); ++ if (queue == MLX5_HW_INV_QUEUE) ++ queue = CTRL_QUEUE_ID(priv); + job = flow_hw_job_get(priv, queue); + if (!job) { + rte_flow_error_set(error, ENOMEM, +@@ -9983,9 +10218,21 @@ flow_hw_action_job_init(struct mlx5_priv *priv, uint32_t queue, + job->action = handle; + job->user_data = user_data; + job->query.user = query_data; ++ job->indirect_type = indirect_type; + return job; + } + ++struct mlx5_hw_q_job * ++mlx5_flow_action_job_init(struct mlx5_priv *priv, uint32_t queue, ++ const struct rte_flow_action_handle *handle, ++ void *user_data, void *query_data, ++ enum mlx5_hw_job_type type, ++ struct rte_flow_error *error) ++{ ++ return flow_hw_action_job_init(priv, queue, handle, user_data, query_data, ++ type, MLX5_HW_INDIRECT_TYPE_LEGACY, error); ++} ++ + static __rte_always_inline void + flow_hw_action_finalize(struct rte_eth_dev *dev, uint32_t queue, + struct mlx5_hw_q_job *job, +@@ -10045,15 +10292,17 @@ flow_hw_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, + const struct rte_flow_action_age *age; + struct mlx5_aso_mtr *aso_mtr; + cnt_id_t cnt_id; +- uint32_t mtr_id; + uint32_t age_idx; + bool push = flow_hw_action_push(attr); + bool aso = false; ++ bool force_job = action->type == RTE_FLOW_ACTION_TYPE_METER_MARK; + +- if (attr) { ++ if (!mlx5_hw_ctx_validate(dev, error)) ++ return NULL; ++ if (attr || force_job) { + job = flow_hw_action_job_init(priv, queue, NULL, user_data, + NULL, MLX5_HW_Q_JOB_TYPE_CREATE, +- error); ++ MLX5_HW_INDIRECT_TYPE_LEGACY, error); + if (!job) + return NULL; + } +@@ -10105,9 +10354,7 @@ flow_hw_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, + aso_mtr = flow_hw_meter_mark_alloc(dev, queue, action, job, push); + if (!aso_mtr) + break; +- mtr_id = (MLX5_INDIRECT_ACTION_TYPE_METER_MARK << +- MLX5_INDIRECT_ACTION_TYPE_OFFSET) | (aso_mtr->fm.meter_id); +- handle = (struct rte_flow_action_handle *)(uintptr_t)mtr_id; ++ handle = (void *)(uintptr_t)job->action; + break; + case RTE_FLOW_ACTION_TYPE_RSS: + handle = flow_dv_action_create(dev, conf, action, error); +@@ -10122,9 +10369,8 @@ flow_hw_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, + NULL, "action type not supported"); + break; + } +- if (job) { ++ if (job && !force_job) { + job->action = handle; +- job->indirect_type = MLX5_HW_INDIRECT_TYPE_LEGACY; + flow_hw_action_finalize(dev, queue, job, push, aso, + handle != NULL); + } +@@ -10155,15 +10401,17 @@ mlx5_flow_update_meter_mark(struct rte_eth_dev *dev, uint32_t queue, + fm->color_aware = meter_mark->color_mode; + if (upd_meter_mark->state_valid) + fm->is_enable = meter_mark->state; ++ aso_mtr->state = (queue == MLX5_HW_INV_QUEUE) ? ++ ASO_METER_WAIT : ASO_METER_WAIT_ASYNC; + /* Update ASO flow meter by wqe. */ +- if (mlx5_aso_meter_update_by_wqe(priv->sh, queue, ++ if (mlx5_aso_meter_update_by_wqe(priv, queue, + aso_mtr, &priv->mtr_bulk, job, push)) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "Unable to update ASO meter WQE"); + /* Wait for ASO object completion. */ + if (queue == MLX5_HW_INV_QUEUE && +- mlx5_aso_mtr_wait(priv->sh, MLX5_HW_INV_QUEUE, aso_mtr)) ++ mlx5_aso_mtr_wait(priv, aso_mtr, true)) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "Unable to wait for ASO meter CQE"); +@@ -10209,11 +10457,12 @@ flow_hw_action_handle_update(struct rte_eth_dev *dev, uint32_t queue, + int ret = 0; + bool push = flow_hw_action_push(attr); + bool aso = false; ++ bool force_job = type == MLX5_INDIRECT_ACTION_TYPE_METER_MARK; + +- if (attr) { ++ if (attr || force_job) { + job = flow_hw_action_job_init(priv, queue, handle, user_data, + NULL, MLX5_HW_Q_JOB_TYPE_UPDATE, +- error); ++ MLX5_HW_INDIRECT_TYPE_LEGACY, error); + if (!job) + return -rte_errno; + } +@@ -10247,7 +10496,7 @@ flow_hw_action_handle_update(struct rte_eth_dev *dev, uint32_t queue, + "action type not supported"); + break; + } +- if (job) ++ if (job && !force_job) + flow_hw_action_finalize(dev, queue, job, push, aso, ret == 0); + return ret; + } +@@ -10290,11 +10539,12 @@ flow_hw_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, + bool push = flow_hw_action_push(attr); + bool aso = false; + int ret = 0; ++ bool force_job = type == MLX5_INDIRECT_ACTION_TYPE_METER_MARK; + +- if (attr) { ++ if (attr || force_job) { + job = flow_hw_action_job_init(priv, queue, handle, user_data, + NULL, MLX5_HW_Q_JOB_TYPE_DESTROY, +- error); ++ MLX5_HW_INDIRECT_TYPE_LEGACY, error); + if (!job) + return -rte_errno; + } +@@ -10327,7 +10577,7 @@ flow_hw_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, + fm = &aso_mtr->fm; + fm->is_enable = 0; + /* Update ASO flow meter by wqe. */ +- if (mlx5_aso_meter_update_by_wqe(priv->sh, queue, aso_mtr, ++ if (mlx5_aso_meter_update_by_wqe(priv, queue, aso_mtr, + &priv->mtr_bulk, job, push)) { + ret = -EINVAL; + rte_flow_error_set(error, EINVAL, +@@ -10337,17 +10587,14 @@ flow_hw_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, + } + /* Wait for ASO object completion. */ + if (queue == MLX5_HW_INV_QUEUE && +- mlx5_aso_mtr_wait(priv->sh, MLX5_HW_INV_QUEUE, aso_mtr)) { ++ mlx5_aso_mtr_wait(priv, aso_mtr, true)) { + ret = -EINVAL; + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "Unable to wait for ASO meter CQE"); + break; + } +- if (!job) +- mlx5_ipool_free(pool->idx_pool, idx); +- else +- aso = true; ++ aso = true; + break; + case MLX5_INDIRECT_ACTION_TYPE_RSS: + ret = flow_dv_action_destroy(dev, handle, error); +@@ -10361,7 +10608,7 @@ flow_hw_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, + "action type not supported"); + break; + } +- if (job) ++ if (job && !force_job) + flow_hw_action_finalize(dev, queue, job, push, aso, ret == 0); + return ret; + } +@@ -10607,7 +10854,7 @@ flow_hw_action_handle_query(struct rte_eth_dev *dev, uint32_t queue, + if (attr) { + job = flow_hw_action_job_init(priv, queue, handle, user_data, + data, MLX5_HW_Q_JOB_TYPE_QUERY, +- error); ++ MLX5_HW_INDIRECT_TYPE_LEGACY, error); + if (!job) + return -rte_errno; + } +@@ -10661,7 +10908,7 @@ flow_hw_async_action_handle_query_update + job = flow_hw_action_job_init(priv, queue, handle, user_data, + query, + MLX5_HW_Q_JOB_TYPE_UPDATE_QUERY, +- error); ++ MLX5_HW_INDIRECT_TYPE_LEGACY, error); + if (!job) + return -rte_errno; + } +@@ -10742,6 +10989,10 @@ flow_hw_get_q_aged_flows(struct rte_eth_dev *dev, uint32_t queue_id, + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "empty context"); ++ if (!priv->hws_age_req) ++ return rte_flow_error_set(error, ENOENT, ++ RTE_FLOW_ERROR_TYPE_UNSPECIFIED, ++ NULL, "No aging initialized"); + if (priv->hws_strict_queue) { + if (queue_id >= age_info->hw_q_age->nb_rings) + return rte_flow_error_set(error, EINVAL, +@@ -11319,6 +11570,8 @@ flow_hw_async_action_list_handle_create(struct rte_eth_dev *dev, uint32_t queue, + } + }; + ++ if (!mlx5_hw_ctx_validate(dev, error)) ++ return NULL; + if (!actions) { + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "No action list"); +@@ -11337,7 +11590,7 @@ flow_hw_async_action_list_handle_create(struct rte_eth_dev *dev, uint32_t queue, + if (attr) { + job = flow_hw_action_job_init(priv, queue, NULL, user_data, + NULL, MLX5_HW_Q_JOB_TYPE_CREATE, +- error); ++ MLX5_HW_INDIRECT_TYPE_LIST, error); + if (!job) + return NULL; + } +@@ -11357,7 +11610,6 @@ flow_hw_async_action_list_handle_create(struct rte_eth_dev *dev, uint32_t queue, + } + if (job) { + job->action = handle; +- job->indirect_type = MLX5_HW_INDIRECT_TYPE_LIST; + flow_hw_action_finalize(dev, queue, job, push, false, + handle != NULL); + } +@@ -11402,7 +11654,7 @@ flow_hw_async_action_list_handle_destroy + if (attr) { + job = flow_hw_action_job_init(priv, queue, NULL, user_data, + NULL, MLX5_HW_Q_JOB_TYPE_DESTROY, +- error); ++ MLX5_HW_INDIRECT_TYPE_LIST, error); + if (!job) + return rte_errno; + } +@@ -11881,8 +12133,9 @@ mlx5_flow_hw_esw_create_sq_miss_flow(struct rte_eth_dev *dev, uint32_t sqn, bool + proxy_port_id, port_id); + return 0; + } +- if (!proxy_priv->hw_esw_sq_miss_root_tbl || +- !proxy_priv->hw_esw_sq_miss_tbl) { ++ if (!proxy_priv->hw_ctrl_fdb || ++ !proxy_priv->hw_ctrl_fdb->hw_esw_sq_miss_root_tbl || ++ !proxy_priv->hw_ctrl_fdb->hw_esw_sq_miss_tbl) { + DRV_LOG(ERR, "Transfer proxy port (port %u) of port %u was configured, but " + "default flow tables were not created.", + proxy_port_id, port_id); +@@ -11914,7 +12167,8 @@ mlx5_flow_hw_esw_create_sq_miss_flow(struct rte_eth_dev *dev, uint32_t sqn, bool + actions[2] = (struct rte_flow_action) { + .type = RTE_FLOW_ACTION_TYPE_END, + }; +- ret = flow_hw_create_ctrl_flow(dev, proxy_dev, proxy_priv->hw_esw_sq_miss_root_tbl, ++ ret = flow_hw_create_ctrl_flow(dev, proxy_dev, ++ proxy_priv->hw_ctrl_fdb->hw_esw_sq_miss_root_tbl, + items, 0, actions, 0, &flow_info, external); + if (ret) { + DRV_LOG(ERR, "Port %u failed to create root SQ miss flow rule for SQ %u, ret %d", +@@ -11945,7 +12199,8 @@ mlx5_flow_hw_esw_create_sq_miss_flow(struct rte_eth_dev *dev, uint32_t sqn, bool + .type = RTE_FLOW_ACTION_TYPE_END, + }; + flow_info.type = MLX5_HW_CTRL_FLOW_TYPE_SQ_MISS; +- ret = flow_hw_create_ctrl_flow(dev, proxy_dev, proxy_priv->hw_esw_sq_miss_tbl, ++ ret = flow_hw_create_ctrl_flow(dev, proxy_dev, ++ proxy_priv->hw_ctrl_fdb->hw_esw_sq_miss_tbl, + items, 0, actions, 0, &flow_info, external); + if (ret) { + DRV_LOG(ERR, "Port %u failed to create HWS SQ miss flow rule for SQ %u, ret %d", +@@ -11989,10 +12244,13 @@ mlx5_flow_hw_esw_destroy_sq_miss_flow(struct rte_eth_dev *dev, uint32_t sqn) + } + proxy_dev = &rte_eth_devices[proxy_port_id]; + proxy_priv = proxy_dev->data->dev_private; ++ /* FDB default flow rules must be enabled. */ ++ MLX5_ASSERT(proxy_priv->sh->config.fdb_def_rule); + if (!proxy_priv->dr_ctx) + return 0; +- if (!proxy_priv->hw_esw_sq_miss_root_tbl || +- !proxy_priv->hw_esw_sq_miss_tbl) ++ if (!proxy_priv->hw_ctrl_fdb || ++ !proxy_priv->hw_ctrl_fdb->hw_esw_sq_miss_root_tbl || ++ !proxy_priv->hw_ctrl_fdb->hw_esw_sq_miss_tbl) + return 0; + cf = LIST_FIRST(&proxy_priv->hw_ctrl_flows); + while (cf != NULL) { +@@ -12052,6 +12310,8 @@ mlx5_flow_hw_esw_create_default_jump_flow(struct rte_eth_dev *dev) + } + proxy_dev = &rte_eth_devices[proxy_port_id]; + proxy_priv = proxy_dev->data->dev_private; ++ /* FDB default flow rules must be enabled. */ ++ MLX5_ASSERT(proxy_priv->sh->config.fdb_def_rule); + if (!proxy_priv->dr_ctx) { + DRV_LOG(DEBUG, "Transfer proxy port (port %u) of port %u must be configured " + "for HWS to create default FDB jump rule. Default rule will " +@@ -12059,7 +12319,7 @@ mlx5_flow_hw_esw_create_default_jump_flow(struct rte_eth_dev *dev) + proxy_port_id, port_id); + return 0; + } +- if (!proxy_priv->hw_esw_zero_tbl) { ++ if (!proxy_priv->hw_ctrl_fdb || !proxy_priv->hw_ctrl_fdb->hw_esw_zero_tbl) { + DRV_LOG(ERR, "Transfer proxy port (port %u) of port %u was configured, but " + "default flow tables were not created.", + proxy_port_id, port_id); +@@ -12067,7 +12327,7 @@ mlx5_flow_hw_esw_create_default_jump_flow(struct rte_eth_dev *dev) + return -rte_errno; + } + return flow_hw_create_ctrl_flow(dev, proxy_dev, +- proxy_priv->hw_esw_zero_tbl, ++ proxy_priv->hw_ctrl_fdb->hw_esw_zero_tbl, + items, 0, actions, 0, &flow_info, false); + } + +@@ -12119,10 +12379,12 @@ mlx5_flow_hw_create_tx_default_mreg_copy_flow(struct rte_eth_dev *dev) + }; + + MLX5_ASSERT(priv->master); +- if (!priv->dr_ctx || !priv->hw_tx_meta_cpy_tbl) ++ if (!priv->dr_ctx || ++ !priv->hw_ctrl_fdb || ++ !priv->hw_ctrl_fdb->hw_tx_meta_cpy_tbl) + return 0; + return flow_hw_create_ctrl_flow(dev, dev, +- priv->hw_tx_meta_cpy_tbl, ++ priv->hw_ctrl_fdb->hw_tx_meta_cpy_tbl, + eth_all, 0, copy_reg_action, 0, &flow_info, false); + } + +@@ -12214,11 +12476,11 @@ mlx5_flow_hw_lacp_rx_flow(struct rte_eth_dev *dev) + .type = MLX5_HW_CTRL_FLOW_TYPE_LACP_RX, + }; + +- MLX5_ASSERT(priv->master); +- if (!priv->dr_ctx || !priv->hw_lacp_rx_tbl) ++ if (!priv->dr_ctx || !priv->hw_ctrl_fdb || !priv->hw_ctrl_fdb->hw_lacp_rx_tbl) + return 0; +- return flow_hw_create_ctrl_flow(dev, dev, priv->hw_lacp_rx_tbl, eth_lacp, 0, +- miss_action, 0, &flow_info, false); ++ return flow_hw_create_ctrl_flow(dev, dev, ++ priv->hw_ctrl_fdb->hw_lacp_rx_tbl, ++ eth_lacp, 0, miss_action, 0, &flow_info, false); + } + + static uint32_t +diff --git a/dpdk/drivers/net/mlx5/mlx5_flow_meter.c b/dpdk/drivers/net/mlx5/mlx5_flow_meter.c +index 7cbf772ea4..7bf5018c70 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_flow_meter.c ++++ b/dpdk/drivers/net/mlx5/mlx5_flow_meter.c +@@ -618,6 +618,7 @@ mlx5_flow_meter_profile_get(struct rte_eth_dev *dev, + meter_profile_id); + } + ++#if defined(HAVE_MLX5_HWS_SUPPORT) + /** + * Callback to add MTR profile with HWS. + * +@@ -697,6 +698,7 @@ mlx5_flow_meter_profile_hws_delete(struct rte_eth_dev *dev, + memset(fmp, 0, sizeof(struct mlx5_flow_meter_profile)); + return 0; + } ++#endif + + /** + * Find policy by id. +@@ -839,6 +841,7 @@ mlx5_flow_meter_policy_validate(struct rte_eth_dev *dev, + return 0; + } + ++#if defined(HAVE_MLX5_HWS_SUPPORT) + /** + * Callback to check MTR policy action validate for HWS + * +@@ -875,6 +878,7 @@ mlx5_flow_meter_policy_hws_validate(struct rte_eth_dev *dev, + } + return 0; + } ++#endif + + static int + __mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev, +@@ -1201,6 +1205,7 @@ mlx5_flow_meter_policy_get(struct rte_eth_dev *dev, + &policy_idx); + } + ++#if defined(HAVE_MLX5_HWS_SUPPORT) + /** + * Callback to delete MTR policy for HWS. + * +@@ -1523,7 +1528,7 @@ policy_add_err: + RTE_MTR_ERROR_TYPE_UNSPECIFIED, + NULL, "Failed to create meter policy."); + } +- ++#endif + /** + * Check meter validation. + * +@@ -1608,12 +1613,12 @@ mlx5_flow_meter_action_modify(struct mlx5_priv *priv, + if (sh->meter_aso_en) { + fm->is_enable = !!is_enable; + aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); +- ret = mlx5_aso_meter_update_by_wqe(sh, MLX5_HW_INV_QUEUE, ++ ret = mlx5_aso_meter_update_by_wqe(priv, MLX5_HW_INV_QUEUE, + aso_mtr, &priv->mtr_bulk, + NULL, true); + if (ret) + return ret; +- ret = mlx5_aso_mtr_wait(sh, MLX5_HW_INV_QUEUE, aso_mtr); ++ ret = mlx5_aso_mtr_wait(priv, aso_mtr, false); + if (ret) + return ret; + } else { +@@ -1859,7 +1864,7 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, + /* If ASO meter supported, update ASO flow meter by wqe. */ + if (priv->sh->meter_aso_en) { + aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); +- ret = mlx5_aso_meter_update_by_wqe(priv->sh, MLX5_HW_INV_QUEUE, ++ ret = mlx5_aso_meter_update_by_wqe(priv, MLX5_HW_INV_QUEUE, + aso_mtr, &priv->mtr_bulk, NULL, true); + if (ret) + goto error; +@@ -1893,6 +1898,7 @@ error: + NULL, "Failed to create devx meter."); + } + ++#if defined(HAVE_MLX5_HWS_SUPPORT) + /** + * Create meter rules. + * +@@ -1920,6 +1926,7 @@ mlx5_flow_meter_hws_create(struct rte_eth_dev *dev, uint32_t meter_id, + struct mlx5_flow_meter_info *fm; + struct mlx5_flow_meter_policy *policy = NULL; + struct mlx5_aso_mtr *aso_mtr; ++ struct mlx5_hw_q_job *job; + int ret; + + if (!priv->mtr_profile_arr || +@@ -1965,17 +1972,26 @@ mlx5_flow_meter_hws_create(struct rte_eth_dev *dev, uint32_t meter_id, + fm->shared = !!shared; + fm->initialized = 1; + /* Update ASO flow meter by wqe. */ +- ret = mlx5_aso_meter_update_by_wqe(priv->sh, MLX5_HW_INV_QUEUE, aso_mtr, +- &priv->mtr_bulk, NULL, true); +- if (ret) ++ job = mlx5_flow_action_job_init(priv, MLX5_HW_INV_QUEUE, NULL, NULL, ++ NULL, MLX5_HW_Q_JOB_TYPE_CREATE, NULL); ++ if (!job) ++ return -rte_mtr_error_set(error, ENOMEM, ++ RTE_MTR_ERROR_TYPE_MTR_ID, ++ NULL, "No job context."); ++ ret = mlx5_aso_meter_update_by_wqe(priv, MLX5_HW_INV_QUEUE, aso_mtr, ++ &priv->mtr_bulk, job, true); ++ if (ret) { ++ flow_hw_job_put(priv, job, CTRL_QUEUE_ID(priv)); + return -rte_mtr_error_set(error, ENOTSUP, +- RTE_MTR_ERROR_TYPE_UNSPECIFIED, +- NULL, "Failed to create devx meter."); ++ RTE_MTR_ERROR_TYPE_UNSPECIFIED, ++ NULL, "Failed to create devx meter."); ++ } + fm->active_state = params->meter_enable; + __atomic_fetch_add(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED); + __atomic_fetch_add(&policy->ref_cnt, 1, __ATOMIC_RELAXED); + return 0; + } ++#endif + + static int + mlx5_flow_meter_params_flush(struct rte_eth_dev *dev, +@@ -2460,6 +2476,7 @@ static const struct rte_mtr_ops mlx5_flow_mtr_ops = { + .stats_read = mlx5_flow_meter_stats_read, + }; + ++#if defined(HAVE_MLX5_HWS_SUPPORT) + static const struct rte_mtr_ops mlx5_flow_mtr_hws_ops = { + .capabilities_get = mlx5_flow_mtr_cap_get, + .meter_profile_add = mlx5_flow_meter_profile_hws_add, +@@ -2478,6 +2495,7 @@ static const struct rte_mtr_ops mlx5_flow_mtr_hws_ops = { + .stats_update = NULL, + .stats_read = NULL, + }; ++#endif + + /** + * Get meter operations. +@@ -2493,12 +2511,16 @@ static const struct rte_mtr_ops mlx5_flow_mtr_hws_ops = { + int + mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg) + { ++#if defined(HAVE_MLX5_HWS_SUPPORT) + struct mlx5_priv *priv = dev->data->dev_private; + + if (priv->sh->config.dv_flow_en == 2) + *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_hws_ops; + else + *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops; ++#else ++ *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops; ++#endif + return 0; + } + +@@ -2614,7 +2636,7 @@ mlx5_flow_meter_attach(struct mlx5_priv *priv, + struct mlx5_aso_mtr *aso_mtr; + + aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); +- if (mlx5_aso_mtr_wait(priv->sh, MLX5_HW_INV_QUEUE, aso_mtr)) { ++ if (mlx5_aso_mtr_wait(priv, aso_mtr, false)) { + return rte_flow_error_set(error, ENOENT, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, +@@ -2877,7 +2899,6 @@ mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error) + struct mlx5_flow_meter_profile *fmp; + struct mlx5_legacy_flow_meter *legacy_fm; + struct mlx5_flow_meter_info *fm; +- struct mlx5_flow_meter_policy *policy; + struct mlx5_flow_meter_sub_policy *sub_policy; + void *tmp; + uint32_t i, mtr_idx, policy_idx; +@@ -2945,15 +2966,20 @@ mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error) + mlx5_l3t_destroy(priv->policy_idx_tbl); + priv->policy_idx_tbl = NULL; + } ++#if defined(HAVE_MLX5_HWS_SUPPORT) + if (priv->mtr_policy_arr) { ++ struct mlx5_flow_meter_policy *policy; ++ + for (i = 0; i < priv->mtr_config.nb_meter_policies; i++) { + policy = mlx5_flow_meter_policy_find(dev, i, + &policy_idx); +- if (policy->initialized) ++ if (policy->initialized) { + mlx5_flow_meter_policy_hws_delete(dev, i, + error); ++ } + } + } ++#endif + if (priv->mtr_profile_tbl) { + MLX5_L3T_FOREACH(priv->mtr_profile_tbl, i, entry) { + fmp = entry; +@@ -2967,14 +2993,17 @@ mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error) + mlx5_l3t_destroy(priv->mtr_profile_tbl); + priv->mtr_profile_tbl = NULL; + } ++#if defined(HAVE_MLX5_HWS_SUPPORT) + if (priv->mtr_profile_arr) { + for (i = 0; i < priv->mtr_config.nb_meter_profiles; i++) { + fmp = mlx5_flow_meter_profile_find(priv, i); +- if (fmp->initialized) ++ if (fmp->initialized) { + mlx5_flow_meter_profile_hws_delete(dev, i, + error); ++ } + } + } ++#endif + /* Delete default policy table. */ + mlx5_flow_destroy_def_policy(dev); + if (priv->sh->refcnt == 1) +diff --git a/dpdk/drivers/net/mlx5/mlx5_hws_cnt.c b/dpdk/drivers/net/mlx5/mlx5_hws_cnt.c +index a3bea94811..41edd19bb8 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_hws_cnt.c ++++ b/dpdk/drivers/net/mlx5/mlx5_hws_cnt.c +@@ -340,6 +340,55 @@ mlx5_hws_cnt_pool_deinit(struct mlx5_hws_cnt_pool * const cntp) + mlx5_free(cntp); + } + ++static bool ++mlx5_hws_cnt_should_enable_cache(const struct mlx5_hws_cnt_pool_cfg *pcfg, ++ const struct mlx5_hws_cache_param *ccfg) ++{ ++ /* ++ * Enable cache if and only if there are enough counters requested ++ * to populate all of the caches. ++ */ ++ return pcfg->request_num >= ccfg->q_num * ccfg->size; ++} ++ ++static struct mlx5_hws_cnt_pool_caches * ++mlx5_hws_cnt_cache_init(const struct mlx5_hws_cnt_pool_cfg *pcfg, ++ const struct mlx5_hws_cache_param *ccfg) ++{ ++ struct mlx5_hws_cnt_pool_caches *cache; ++ char mz_name[RTE_MEMZONE_NAMESIZE]; ++ uint32_t qidx; ++ ++ /* If counter pool is big enough, setup the counter pool cache. */ ++ cache = mlx5_malloc(MLX5_MEM_ANY | MLX5_MEM_ZERO, ++ sizeof(*cache) + ++ sizeof(((struct mlx5_hws_cnt_pool_caches *)0)->qcache[0]) ++ * ccfg->q_num, 0, SOCKET_ID_ANY); ++ if (cache == NULL) ++ return NULL; ++ /* Store the necessary cache parameters. */ ++ cache->fetch_sz = ccfg->fetch_sz; ++ cache->preload_sz = ccfg->preload_sz; ++ cache->threshold = ccfg->threshold; ++ cache->q_num = ccfg->q_num; ++ for (qidx = 0; qidx < ccfg->q_num; qidx++) { ++ snprintf(mz_name, sizeof(mz_name), "%s_qc/%x", pcfg->name, qidx); ++ cache->qcache[qidx] = rte_ring_create(mz_name, ccfg->size, ++ SOCKET_ID_ANY, ++ RING_F_SP_ENQ | RING_F_SC_DEQ | ++ RING_F_EXACT_SZ); ++ if (cache->qcache[qidx] == NULL) ++ goto error; ++ } ++ return cache; ++ ++error: ++ while (qidx--) ++ rte_ring_free(cache->qcache[qidx]); ++ mlx5_free(cache); ++ return NULL; ++} ++ + static struct mlx5_hws_cnt_pool * + mlx5_hws_cnt_pool_init(struct mlx5_dev_ctx_shared *sh, + const struct mlx5_hws_cnt_pool_cfg *pcfg, +@@ -348,7 +397,6 @@ mlx5_hws_cnt_pool_init(struct mlx5_dev_ctx_shared *sh, + char mz_name[RTE_MEMZONE_NAMESIZE]; + struct mlx5_hws_cnt_pool *cntp; + uint64_t cnt_num = 0; +- uint32_t qidx; + + MLX5_ASSERT(pcfg); + MLX5_ASSERT(ccfg); +@@ -360,17 +408,6 @@ mlx5_hws_cnt_pool_init(struct mlx5_dev_ctx_shared *sh, + cntp->cfg = *pcfg; + if (cntp->cfg.host_cpool) + return cntp; +- cntp->cache = mlx5_malloc(MLX5_MEM_ANY | MLX5_MEM_ZERO, +- sizeof(*cntp->cache) + +- sizeof(((struct mlx5_hws_cnt_pool_caches *)0)->qcache[0]) +- * ccfg->q_num, 0, SOCKET_ID_ANY); +- if (cntp->cache == NULL) +- goto error; +- /* store the necessary cache parameters. */ +- cntp->cache->fetch_sz = ccfg->fetch_sz; +- cntp->cache->preload_sz = ccfg->preload_sz; +- cntp->cache->threshold = ccfg->threshold; +- cntp->cache->q_num = ccfg->q_num; + if (pcfg->request_num > sh->hws_max_nb_counters) { + DRV_LOG(ERR, "Counter number %u " + "is greater than the maximum supported (%u).", +@@ -418,13 +455,10 @@ mlx5_hws_cnt_pool_init(struct mlx5_dev_ctx_shared *sh, + DRV_LOG(ERR, "failed to create reuse list ring"); + goto error; + } +- for (qidx = 0; qidx < ccfg->q_num; qidx++) { +- snprintf(mz_name, sizeof(mz_name), "%s_qc/%x", pcfg->name, qidx); +- cntp->cache->qcache[qidx] = rte_ring_create(mz_name, ccfg->size, +- SOCKET_ID_ANY, +- RING_F_SP_ENQ | RING_F_SC_DEQ | +- RING_F_EXACT_SZ); +- if (cntp->cache->qcache[qidx] == NULL) ++ /* Allocate counter cache only if needed. */ ++ if (mlx5_hws_cnt_should_enable_cache(pcfg, ccfg)) { ++ cntp->cache = mlx5_hws_cnt_cache_init(pcfg, ccfg); ++ if (cntp->cache == NULL) + goto error; + } + /* Initialize the time for aging-out calculation. */ +@@ -685,7 +719,9 @@ mlx5_hws_cnt_pool_destroy(struct mlx5_dev_ctx_shared *sh, + * Maybe blocked for at most 200ms here. + */ + rte_spinlock_lock(&sh->cpool_lock); +- LIST_REMOVE(cpool, next); ++ /* Try to remove cpool before it was added to list caused segfault. */ ++ if (!LIST_EMPTY(&sh->hws_cpool_list) && cpool->next.le_prev) ++ LIST_REMOVE(cpool, next); + rte_spinlock_unlock(&sh->cpool_lock); + if (cpool->cfg.host_cpool == NULL) { + if (--sh->cnt_svc->refcnt == 0) +diff --git a/dpdk/drivers/net/mlx5/mlx5_hws_cnt.h b/dpdk/drivers/net/mlx5/mlx5_hws_cnt.h +index 585b5a83ad..e00596088f 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_hws_cnt.h ++++ b/dpdk/drivers/net/mlx5/mlx5_hws_cnt.h +@@ -557,19 +557,32 @@ mlx5_hws_cnt_pool_get(struct mlx5_hws_cnt_pool *cpool, uint32_t *queue, + } + + /** +- * Check if counter pool allocated for HWS is shared between ports. ++ * Decide if the given queue can be used to perform counter allocation/deallcation ++ * based on counter configuration + * + * @param[in] priv + * Pointer to the port private data structure. ++ * @param[in] queue ++ * Pointer to the queue index. + * + * @return +- * True if counter pools is shared between ports. False otherwise. ++ * @p queue if cache related to the queue can be used. NULL otherwise. + */ +-static __rte_always_inline bool +-mlx5_hws_cnt_is_pool_shared(struct mlx5_priv *priv) ++static __rte_always_inline uint32_t * ++mlx5_hws_cnt_get_queue(struct mlx5_priv *priv, uint32_t *queue) + { +- return priv && priv->hws_cpool && +- (priv->shared_refcnt || priv->hws_cpool->cfg.host_cpool != NULL); ++ if (priv && priv->hws_cpool) { ++ /* Do not use queue cache if counter pool is shared. */ ++ if (priv->shared_refcnt || priv->hws_cpool->cfg.host_cpool != NULL) ++ return NULL; ++ /* Do not use queue cache if counter cache is disabled. */ ++ if (priv->hws_cpool->cache == NULL) ++ return NULL; ++ return queue; ++ } ++ /* This case should not be reached if counter pool was successfully configured. */ ++ MLX5_ASSERT(false); ++ return NULL; + } + + static __rte_always_inline unsigned int +diff --git a/dpdk/drivers/net/mlx5/mlx5_rx.c b/dpdk/drivers/net/mlx5/mlx5_rx.c +index 5bf1a679b2..cc087348a4 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_rx.c ++++ b/dpdk/drivers/net/mlx5/mlx5_rx.c +@@ -613,7 +613,8 @@ mlx5_rx_err_handle(struct mlx5_rxq_data *rxq, uint8_t vec, + * @param mprq + * Indication if it is called from MPRQ. + * @return +- * 0 in case of empty CQE, MLX5_REGULAR_ERROR_CQE_RET in case of error CQE, ++ * 0 in case of empty CQE, ++ * MLX5_REGULAR_ERROR_CQE_RET in case of error CQE, + * MLX5_CRITICAL_ERROR_CQE_RET in case of error CQE lead to Rx queue reset, + * otherwise the packet size in regular RxQ, + * and striding byte count format in mprq case. +@@ -697,6 +698,11 @@ mlx5_rx_poll_len(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe, + if (ret == MLX5_RECOVERY_ERROR_RET || + ret == MLX5_RECOVERY_COMPLETED_RET) + return MLX5_CRITICAL_ERROR_CQE_RET; ++ if (!mprq && ret == MLX5_RECOVERY_IGNORE_RET) { ++ *skip_cnt = 1; ++ ++rxq->cq_ci; ++ return MLX5_ERROR_CQE_MASK; ++ } + } else { + return 0; + } +@@ -971,19 +977,18 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) + cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_mask]; + len = mlx5_rx_poll_len(rxq, cqe, cqe_n, cqe_mask, &mcqe, &skip_cnt, false); + if (unlikely(len & MLX5_ERROR_CQE_MASK)) { ++ /* We drop packets with non-critical errors */ ++ rte_mbuf_raw_free(rep); + if (len == MLX5_CRITICAL_ERROR_CQE_RET) { +- rte_mbuf_raw_free(rep); + rq_ci = rxq->rq_ci << sges_n; + break; + } ++ /* Skip specified amount of error CQEs packets */ + rq_ci >>= sges_n; + rq_ci += skip_cnt; + rq_ci <<= sges_n; +- idx = rq_ci & wqe_mask; +- wqe = &((volatile struct mlx5_wqe_data_seg *)rxq->wqes)[idx]; +- seg = (*rxq->elts)[idx]; +- cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_mask]; +- len = len & ~MLX5_ERROR_CQE_MASK; ++ MLX5_ASSERT(!pkt); ++ continue; + } + if (len == 0) { + rte_mbuf_raw_free(rep); +diff --git a/dpdk/drivers/net/mlx5/mlx5_rx.h b/dpdk/drivers/net/mlx5/mlx5_rx.h +index 2fce908499..d0ceae72ea 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_rx.h ++++ b/dpdk/drivers/net/mlx5/mlx5_rx.h +@@ -101,14 +101,14 @@ struct mlx5_rxq_data { + unsigned int shared:1; /* Shared RXQ. */ + unsigned int delay_drop:1; /* Enable delay drop. */ + unsigned int cqe_comp_layout:1; /* CQE Compression Layout*/ +- unsigned int cq_ci:24; ++ uint16_t port_id; + volatile uint32_t *rq_db; + volatile uint32_t *cq_db; +- uint16_t port_id; + uint32_t elts_ci; + uint32_t rq_ci; + uint16_t consumed_strd; /* Number of consumed strides in WQE. */ + uint32_t rq_pi; ++ uint32_t cq_ci:24; + uint16_t rq_repl_thresh; /* Threshold for buffer replenishment. */ + uint32_t byte_mask; + union { +diff --git a/dpdk/drivers/net/mlx5/mlx5_stats.c b/dpdk/drivers/net/mlx5/mlx5_stats.c +index 615e1d073d..f4ac58e2f9 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_stats.c ++++ b/dpdk/drivers/net/mlx5/mlx5_stats.c +@@ -39,24 +39,36 @@ mlx5_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *stats, + unsigned int n) + { + struct mlx5_priv *priv = dev->data->dev_private; +- unsigned int i; +- uint64_t counters[n]; ++ uint64_t counters[MLX5_MAX_XSTATS]; + struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; ++ unsigned int i; ++ uint16_t stats_n = 0; ++ uint16_t stats_n_2nd = 0; + uint16_t mlx5_stats_n = xstats_ctrl->mlx5_stats_n; ++ bool bond_master = (priv->master && priv->pf_bond >= 0); + + if (n >= mlx5_stats_n && stats) { +- int stats_n; + int ret; + +- stats_n = mlx5_os_get_stats_n(dev); +- if (stats_n < 0) +- return stats_n; +- if (xstats_ctrl->stats_n != stats_n) ++ ret = mlx5_os_get_stats_n(dev, bond_master, &stats_n, &stats_n_2nd); ++ if (ret < 0) ++ return ret; ++ /* ++ * The number of statistics fetched via "ETH_SS_STATS" may vary because ++ * of the port configuration each time. This is also true between 2 ++ * ports. There might be a case that the numbers are the same even if ++ * configurations are different. ++ * It is not recommended to change the configuration without using ++ * RTE API. The port(traffic) restart may trigger another initialization ++ * to make sure the map are correct. ++ */ ++ if (xstats_ctrl->stats_n != stats_n || ++ (bond_master && xstats_ctrl->stats_n_2nd != stats_n_2nd)) + mlx5_os_stats_init(dev); +- ret = mlx5_os_read_dev_counters(dev, counters); +- if (ret) ++ ret = mlx5_os_read_dev_counters(dev, bond_master, counters); ++ if (ret < 0) + return ret; +- for (i = 0; i != mlx5_stats_n; ++i) { ++ for (i = 0; i != mlx5_stats_n; i++) { + stats[i].id = i; + if (xstats_ctrl->info[i].dev) { + uint64_t wrap_n; +@@ -225,30 +237,32 @@ mlx5_xstats_reset(struct rte_eth_dev *dev) + { + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; +- int stats_n; + unsigned int i; + uint64_t *counters; + int ret; ++ uint16_t stats_n = 0; ++ uint16_t stats_n_2nd = 0; ++ bool bond_master = (priv->master && priv->pf_bond >= 0); + +- stats_n = mlx5_os_get_stats_n(dev); +- if (stats_n < 0) { ++ ret = mlx5_os_get_stats_n(dev, bond_master, &stats_n, &stats_n_2nd); ++ if (ret < 0) { + DRV_LOG(ERR, "port %u cannot get stats: %s", dev->data->port_id, +- strerror(-stats_n)); +- return stats_n; ++ strerror(-ret)); ++ return ret; + } +- if (xstats_ctrl->stats_n != stats_n) ++ if (xstats_ctrl->stats_n != stats_n || ++ (bond_master && xstats_ctrl->stats_n_2nd != stats_n_2nd)) + mlx5_os_stats_init(dev); +- counters = mlx5_malloc(MLX5_MEM_SYS, sizeof(*counters) * +- xstats_ctrl->mlx5_stats_n, 0, +- SOCKET_ID_ANY); ++ /* Considering to use stack directly. */ ++ counters = mlx5_malloc(MLX5_MEM_SYS, sizeof(*counters) * xstats_ctrl->mlx5_stats_n, ++ 0, SOCKET_ID_ANY); + if (!counters) { +- DRV_LOG(WARNING, "port %u unable to allocate memory for xstats " +- "counters", ++ DRV_LOG(WARNING, "port %u unable to allocate memory for xstats counters", + dev->data->port_id); + rte_errno = ENOMEM; + return -rte_errno; + } +- ret = mlx5_os_read_dev_counters(dev, counters); ++ ret = mlx5_os_read_dev_counters(dev, bond_master, counters); + if (ret) { + DRV_LOG(ERR, "port %u cannot read device counters: %s", + dev->data->port_id, strerror(rte_errno)); +diff --git a/dpdk/drivers/net/mlx5/mlx5_trigger.c b/dpdk/drivers/net/mlx5/mlx5_trigger.c +index 5ac25d7e2d..fe2c512c5c 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_trigger.c ++++ b/dpdk/drivers/net/mlx5/mlx5_trigger.c +@@ -1498,7 +1498,9 @@ mlx5_traffic_enable_hws(struct rte_eth_dev *dev) + if (!txq) + continue; + queue = mlx5_txq_get_sqn(txq); +- if ((priv->representor || priv->master) && config->dv_esw_en) { ++ if ((priv->representor || priv->master) && ++ config->dv_esw_en && ++ config->fdb_def_rule) { + if (mlx5_flow_hw_esw_create_sq_miss_flow(dev, queue, false)) { + mlx5_txq_release(dev, i); + goto error; +@@ -1524,7 +1526,7 @@ mlx5_traffic_enable_hws(struct rte_eth_dev *dev) + } + if (priv->isolated) + return 0; +- if (!priv->sh->config.lacp_by_user && priv->pf_bond >= 0) ++ if (!priv->sh->config.lacp_by_user && priv->pf_bond >= 0 && priv->master) + if (mlx5_flow_hw_lacp_rx_flow(dev)) + goto error; + if (dev->data->promiscuous) +@@ -1632,14 +1634,14 @@ mlx5_traffic_enable(struct rte_eth_dev *dev) + DRV_LOG(INFO, "port %u FDB default rule is disabled", + dev->data->port_id); + } +- if (!priv->sh->config.lacp_by_user && priv->pf_bond >= 0) { ++ if (!priv->sh->config.lacp_by_user && priv->pf_bond >= 0 && priv->master) { + ret = mlx5_flow_lacp_miss(dev); + if (ret) + DRV_LOG(INFO, "port %u LACP rule cannot be created - " + "forward LACP to kernel.", dev->data->port_id); + else +- DRV_LOG(INFO, "LACP traffic will be missed in port %u." +- , dev->data->port_id); ++ DRV_LOG(INFO, "LACP traffic will be missed in port %u.", ++ dev->data->port_id); + } + if (priv->isolated) + return 0; +diff --git a/dpdk/drivers/net/mlx5/mlx5_txq.c b/dpdk/drivers/net/mlx5/mlx5_txq.c +index 1ac43548b2..aac078a6ed 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_txq.c ++++ b/dpdk/drivers/net/mlx5/mlx5_txq.c +@@ -1311,11 +1311,18 @@ rte_pmd_mlx5_external_sq_enable(uint16_t port_id, uint32_t sq_num) + } + #ifdef HAVE_MLX5_HWS_SUPPORT + if (priv->sh->config.dv_flow_en == 2) { +- if (mlx5_flow_hw_esw_create_sq_miss_flow(dev, sq_num, true)) +- return -rte_errno; ++ bool sq_miss_created = false; ++ ++ if (priv->sh->config.fdb_def_rule) { ++ if (mlx5_flow_hw_esw_create_sq_miss_flow(dev, sq_num, true)) ++ return -rte_errno; ++ sq_miss_created = true; ++ } ++ + if (priv->sh->config.repr_matching && + mlx5_flow_hw_tx_repr_matching_flow(dev, sq_num, true)) { +- mlx5_flow_hw_esw_destroy_sq_miss_flow(dev, sq_num); ++ if (sq_miss_created) ++ mlx5_flow_hw_esw_destroy_sq_miss_flow(dev, sq_num); + return -rte_errno; + } + return 0; +diff --git a/dpdk/drivers/net/mlx5/mlx5_utils.c b/dpdk/drivers/net/mlx5/mlx5_utils.c +index 4db738785f..b5b6c7c728 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_utils.c ++++ b/dpdk/drivers/net/mlx5/mlx5_utils.c +@@ -379,7 +379,8 @@ _mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, int cidx, uint32_t idx) + idx -= 1; + trunk_idx = mlx5_trunk_idx_get(pool, idx); + trunk = lc->trunks[trunk_idx]; +- MLX5_ASSERT(trunk); ++ if (!trunk) ++ return NULL; + entry_idx = idx - mlx5_trunk_idx_offset_get(pool, trunk_idx); + return &trunk->data[entry_idx * pool->cfg.size]; + } +diff --git a/dpdk/drivers/net/mlx5/windows/mlx5_ethdev_os.c b/dpdk/drivers/net/mlx5/windows/mlx5_ethdev_os.c +index a31e1b5494..49f750be68 100644 +--- a/dpdk/drivers/net/mlx5/windows/mlx5_ethdev_os.c ++++ b/dpdk/drivers/net/mlx5/windows/mlx5_ethdev_os.c +@@ -178,20 +178,29 @@ mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) + return -ENOTSUP; + } + +-/** ++/* + * Query the number of statistics provided by ETHTOOL. + * + * @param dev + * Pointer to Ethernet device. ++ * @param bond_master ++ * Indicate if the device is a bond master. ++ * @param n_stats ++ * Pointer to number of stats to store. ++ * @param n_stats_sec ++ * Pointer to number of stats to store for the 2nd port of the bond. + * + * @return +- * Number of statistics on success, negative errno value otherwise and +- * rte_errno is set. ++ * 0 on success, negative errno value otherwise and rte_errno is set. + */ + int +-mlx5_os_get_stats_n(struct rte_eth_dev *dev) ++mlx5_os_get_stats_n(struct rte_eth_dev *dev, bool bond_master, ++ uint16_t *n_stats, uint16_t *n_stats_sec) + { + RTE_SET_USED(dev); ++ RTE_SET_USED(bond_master); ++ RTE_SET_USED(n_stats); ++ RTE_SET_USED(n_stats_sec); + return -ENOTSUP; + } + +@@ -221,6 +230,8 @@ mlx5_os_stats_init(struct rte_eth_dev *dev) + * + * @param dev + * Pointer to Ethernet device. ++ * @param bond_master ++ * Indicate if the device is a bond master. + * @param[out] stats + * Counters table output buffer. + * +@@ -229,9 +240,10 @@ mlx5_os_stats_init(struct rte_eth_dev *dev) + * rte_errno is set. + */ + int +-mlx5_os_read_dev_counters(struct rte_eth_dev *dev, uint64_t *stats) ++mlx5_os_read_dev_counters(struct rte_eth_dev *dev, bool bond_master, uint64_t *stats) + { + RTE_SET_USED(dev); ++ RTE_SET_USED(bond_master); + RTE_SET_USED(stats); + return -ENOTSUP; + } +diff --git a/dpdk/drivers/net/mvneta/mvneta_ethdev.c b/dpdk/drivers/net/mvneta/mvneta_ethdev.c +index daa69e533a..212c300c14 100644 +--- a/dpdk/drivers/net/mvneta/mvneta_ethdev.c ++++ b/dpdk/drivers/net/mvneta/mvneta_ethdev.c +@@ -198,7 +198,8 @@ mvneta_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused) + RTE_PTYPE_L3_IPV4, + RTE_PTYPE_L3_IPV6, + RTE_PTYPE_L4_TCP, +- RTE_PTYPE_L4_UDP ++ RTE_PTYPE_L4_UDP, ++ RTE_PTYPE_UNKNOWN + }; + + return ptypes; +diff --git a/dpdk/drivers/net/mvpp2/mrvl_ethdev.c b/dpdk/drivers/net/mvpp2/mrvl_ethdev.c +index c12364941d..4cc64c7cad 100644 +--- a/dpdk/drivers/net/mvpp2/mrvl_ethdev.c ++++ b/dpdk/drivers/net/mvpp2/mrvl_ethdev.c +@@ -1777,7 +1777,8 @@ mrvl_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused) + RTE_PTYPE_L3_IPV6_EXT, + RTE_PTYPE_L2_ETHER_ARP, + RTE_PTYPE_L4_TCP, +- RTE_PTYPE_L4_UDP ++ RTE_PTYPE_L4_UDP, ++ RTE_PTYPE_UNKNOWN + }; + + return ptypes; +diff --git a/dpdk/drivers/net/netvsc/hn_ethdev.c b/dpdk/drivers/net/netvsc/hn_ethdev.c +index b8a32832d7..f8cb05a118 100644 +--- a/dpdk/drivers/net/netvsc/hn_ethdev.c ++++ b/dpdk/drivers/net/netvsc/hn_ethdev.c +@@ -1127,8 +1127,10 @@ hn_reinit(struct rte_eth_dev *dev, uint16_t mtu) + int i, ret = 0; + + /* Point primary queues at new primary channel */ +- rxqs[0]->chan = hv->channels[0]; +- txqs[0]->chan = hv->channels[0]; ++ if (rxqs[0]) { ++ rxqs[0]->chan = hv->channels[0]; ++ txqs[0]->chan = hv->channels[0]; ++ } + + ret = hn_attach(hv, mtu); + if (ret) +@@ -1140,10 +1142,12 @@ hn_reinit(struct rte_eth_dev *dev, uint16_t mtu) + return ret; + + /* Point any additional queues at new subchannels */ +- for (i = 1; i < dev->data->nb_rx_queues; i++) +- rxqs[i]->chan = hv->channels[i]; +- for (i = 1; i < dev->data->nb_tx_queues; i++) +- txqs[i]->chan = hv->channels[i]; ++ if (rxqs[0]) { ++ for (i = 1; i < dev->data->nb_rx_queues; i++) ++ rxqs[i]->chan = hv->channels[i]; ++ for (i = 1; i < dev->data->nb_tx_queues; i++) ++ txqs[i]->chan = hv->channels[i]; ++ } + + return ret; + } +diff --git a/dpdk/drivers/net/netvsc/hn_rxtx.c b/dpdk/drivers/net/netvsc/hn_rxtx.c +index e4f5015aa3..9bf1ec5509 100644 +--- a/dpdk/drivers/net/netvsc/hn_rxtx.c ++++ b/dpdk/drivers/net/netvsc/hn_rxtx.c +@@ -612,7 +612,9 @@ static void hn_rxpkt(struct hn_rx_queue *rxq, struct hn_rx_bufinfo *rxb, + RTE_PTYPE_L4_MASK); + + if (info->vlan_info != HN_NDIS_VLAN_INFO_INVALID) { +- m->vlan_tci = info->vlan_info; ++ m->vlan_tci = RTE_VLAN_TCI_MAKE(NDIS_VLAN_INFO_ID(info->vlan_info), ++ NDIS_VLAN_INFO_PRI(info->vlan_info), ++ NDIS_VLAN_INFO_CFI(info->vlan_info)); + m->ol_flags |= RTE_MBUF_F_RX_VLAN_STRIPPED | RTE_MBUF_F_RX_VLAN; + + /* NDIS always strips tag, put it back if necessary */ +@@ -1332,7 +1334,9 @@ static void hn_encap(struct rndis_packet_msg *pkt, + if (m->ol_flags & RTE_MBUF_F_TX_VLAN) { + pi_data = hn_rndis_pktinfo_append(pkt, NDIS_VLAN_INFO_SIZE, + NDIS_PKTINFO_TYPE_VLAN); +- *pi_data = m->vlan_tci; ++ *pi_data = NDIS_VLAN_INFO_MAKE(RTE_VLAN_TCI_ID(m->vlan_tci), ++ RTE_VLAN_TCI_PRI(m->vlan_tci), ++ RTE_VLAN_TCI_DEI(m->vlan_tci)); + } + + if (m->ol_flags & RTE_MBUF_F_TX_TCP_SEG) { +diff --git a/dpdk/drivers/net/netvsc/hn_vf.c b/dpdk/drivers/net/netvsc/hn_vf.c +index 90cb6f6923..a4e958419d 100644 +--- a/dpdk/drivers/net/netvsc/hn_vf.c ++++ b/dpdk/drivers/net/netvsc/hn_vf.c +@@ -264,7 +264,7 @@ int hn_vf_add(struct rte_eth_dev *dev, struct hn_data *hv) + goto exit; + } + +- ret = hn_vf_mtu_set(dev, dev->data->mtu); ++ ret = rte_eth_dev_set_mtu(port, dev->data->mtu); + if (ret) { + PMD_DRV_LOG(ERR, "Failed to set VF MTU"); + goto exit; +@@ -794,7 +794,7 @@ int hn_vf_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) + rte_rwlock_read_lock(&hv->vf_lock); + vf_dev = hn_get_vf_dev(hv); + if (hv->vf_ctx.vf_vsc_switched && vf_dev) +- ret = vf_dev->dev_ops->mtu_set(vf_dev, mtu); ++ ret = rte_eth_dev_set_mtu(vf_dev->data->port_id, mtu); + rte_rwlock_read_unlock(&hv->vf_lock); + + return ret; +diff --git a/dpdk/drivers/net/nfp/flower/nfp_flower.c b/dpdk/drivers/net/nfp/flower/nfp_flower.c +index 6b523d98b0..9ecd5f49c7 100644 +--- a/dpdk/drivers/net/nfp/flower/nfp_flower.c ++++ b/dpdk/drivers/net/nfp/flower/nfp_flower.c +@@ -82,63 +82,6 @@ nfp_flower_pf_start(struct rte_eth_dev *dev) + return 0; + } + +-/* Reset and stop device. The device can not be restarted. */ +-static int +-nfp_flower_pf_close(struct rte_eth_dev *dev) +-{ +- uint16_t i; +- struct nfp_net_hw *hw; +- struct nfp_pf_dev *pf_dev; +- struct nfp_net_txq *this_tx_q; +- struct nfp_net_rxq *this_rx_q; +- struct nfp_flower_representor *repr; +- struct nfp_app_fw_flower *app_fw_flower; +- +- if (rte_eal_process_type() != RTE_PROC_PRIMARY) +- return 0; +- +- repr = dev->data->dev_private; +- hw = repr->app_fw_flower->pf_hw; +- pf_dev = hw->pf_dev; +- app_fw_flower = NFP_PRIV_TO_APP_FW_FLOWER(pf_dev->app_fw_priv); +- +- nfp_mtr_priv_uninit(pf_dev); +- +- /* +- * We assume that the DPDK application is stopping all the +- * threads/queues before calling the device close function. +- */ +- nfp_net_disable_queues(dev); +- +- /* Clear queues */ +- for (i = 0; i < dev->data->nb_tx_queues; i++) { +- this_tx_q = dev->data->tx_queues[i]; +- nfp_net_reset_tx_queue(this_tx_q); +- } +- +- for (i = 0; i < dev->data->nb_rx_queues; i++) { +- this_rx_q = dev->data->rx_queues[i]; +- nfp_net_reset_rx_queue(this_rx_q); +- } +- +- /* Cancel possible impending LSC work here before releasing the port */ +- rte_eal_alarm_cancel(nfp_net_dev_interrupt_delayed_handler, (void *)dev); +- +- nn_cfg_writeb(&hw->super, NFP_NET_CFG_LSC, 0xff); +- +- /* Now it is safe to free all PF resources */ +- PMD_DRV_LOG(INFO, "Freeing PF resources"); +- nfp_cpp_area_free(pf_dev->ctrl_area); +- nfp_cpp_area_free(pf_dev->qc_area); +- free(pf_dev->hwinfo); +- free(pf_dev->sym_tbl); +- nfp_cpp_free(pf_dev->cpp); +- rte_free(app_fw_flower); +- rte_free(pf_dev); +- +- return 0; +-} +- + static const struct eth_dev_ops nfp_flower_pf_vnic_ops = { + .dev_infos_get = nfp_net_infos_get, + .link_update = nfp_net_link_update, +@@ -146,7 +89,6 @@ static const struct eth_dev_ops nfp_flower_pf_vnic_ops = { + + .dev_start = nfp_flower_pf_start, + .dev_stop = nfp_net_stop, +- .dev_close = nfp_flower_pf_close, + }; + + static inline struct nfp_flower_representor * +@@ -191,7 +133,9 @@ nfp_flower_pf_dispatch_pkts(struct nfp_net_hw *hw, + return false; + } + +- rte_ring_enqueue(repr->ring, (void *)mbuf); ++ if (rte_ring_enqueue(repr->ring, (void *)mbuf) != 0) ++ return false; ++ + return true; + } + +@@ -567,6 +511,8 @@ nfp_flower_cleanup_ctrl_vnic(struct nfp_net_hw *hw) + + pci_name = strchr(app_fw_flower->pf_hw->pf_dev->pci_dev->name, ':') + 1; + ++ nfp_net_disable_queues(eth_dev); ++ + snprintf(ctrl_txring_name, sizeof(ctrl_txring_name), "%s_cttx_ring", pci_name); + for (i = 0; i < hw->max_tx_queues; i++) { + txq = eth_dev->data->tx_queues[i]; +@@ -858,6 +804,23 @@ app_cleanup: + return ret; + } + ++void ++nfp_uninit_app_fw_flower(struct nfp_pf_dev *pf_dev) ++{ ++ struct nfp_app_fw_flower *app_fw_flower; ++ ++ app_fw_flower = pf_dev->app_fw_priv; ++ nfp_flower_cleanup_ctrl_vnic(app_fw_flower->ctrl_hw); ++ nfp_cpp_area_free(app_fw_flower->ctrl_hw->ctrl_area); ++ nfp_cpp_area_free(pf_dev->ctrl_area); ++ rte_free(app_fw_flower->pf_hw); ++ nfp_mtr_priv_uninit(pf_dev); ++ nfp_flow_priv_uninit(pf_dev); ++ if (rte_eth_switch_domain_free(app_fw_flower->switch_domain_id) != 0) ++ PMD_DRV_LOG(WARNING, "Failed to free switch domain for device"); ++ rte_free(app_fw_flower); ++} ++ + int + nfp_secondary_init_app_fw_flower(struct nfp_pf_dev *pf_dev) + { +diff --git a/dpdk/drivers/net/nfp/flower/nfp_flower.h b/dpdk/drivers/net/nfp/flower/nfp_flower.h +index 6f27c06acc..8393de66c5 100644 +--- a/dpdk/drivers/net/nfp/flower/nfp_flower.h ++++ b/dpdk/drivers/net/nfp/flower/nfp_flower.h +@@ -106,6 +106,7 @@ nfp_flower_support_decap_v2(const struct nfp_app_fw_flower *app_fw_flower) + + int nfp_init_app_fw_flower(struct nfp_pf_dev *pf_dev, + const struct nfp_dev_info *dev_info); ++void nfp_uninit_app_fw_flower(struct nfp_pf_dev *pf_dev); + int nfp_secondary_init_app_fw_flower(struct nfp_pf_dev *pf_dev); + bool nfp_flower_pf_dispatch_pkts(struct nfp_net_hw *hw, + struct rte_mbuf *mbuf, +diff --git a/dpdk/drivers/net/nfp/flower/nfp_flower_ctrl.c b/dpdk/drivers/net/nfp/flower/nfp_flower_ctrl.c +index c25487c277..102daa3d70 100644 +--- a/dpdk/drivers/net/nfp/flower/nfp_flower_ctrl.c ++++ b/dpdk/drivers/net/nfp/flower/nfp_flower_ctrl.c +@@ -441,6 +441,11 @@ nfp_flower_cmsg_port_mod_rx(struct nfp_app_fw_flower *app_fw_flower, + return -EINVAL; + } + ++ if (repr == NULL) { ++ PMD_DRV_LOG(ERR, "Can not get 'repr' for port %#x", port); ++ return -EINVAL; ++ } ++ + repr->link.link_duplex = RTE_ETH_LINK_FULL_DUPLEX; + if ((msg->info & NFP_FLOWER_CMSG_PORT_MOD_INFO_LINK) != 0) + repr->link.link_status = RTE_ETH_LINK_UP; +diff --git a/dpdk/drivers/net/nfp/flower/nfp_flower_representor.c b/dpdk/drivers/net/nfp/flower/nfp_flower_representor.c +index 0f0e63aae0..88fb6975af 100644 +--- a/dpdk/drivers/net/nfp/flower/nfp_flower_representor.c ++++ b/dpdk/drivers/net/nfp/flower/nfp_flower_representor.c +@@ -291,12 +291,156 @@ nfp_flower_repr_tx_burst(void *tx_queue, + return sent; + } + ++static void ++nfp_flower_repr_free_queue(struct nfp_flower_representor *repr) ++{ ++ uint16_t i; ++ struct rte_eth_dev *eth_dev = repr->eth_dev; ++ ++ for (i = 0; i < eth_dev->data->nb_tx_queues; i++) ++ rte_free(eth_dev->data->tx_queues[i]); ++ ++ for (i = 0; i < eth_dev->data->nb_rx_queues; i++) ++ rte_free(eth_dev->data->rx_queues[i]); ++} ++ ++static void ++nfp_flower_pf_repr_close_queue(struct nfp_flower_representor *repr) ++{ ++ struct rte_eth_dev *eth_dev = repr->eth_dev; ++ ++ /* ++ * We assume that the DPDK application is stopping all the ++ * threads/queues before calling the device close function. ++ */ ++ nfp_net_disable_queues(eth_dev); ++ ++ /* Clear queues */ ++ nfp_net_close_tx_queue(eth_dev); ++ nfp_net_close_rx_queue(eth_dev); ++} ++ ++static void ++nfp_flower_repr_close_queue(struct nfp_flower_representor *repr) ++{ ++ switch (repr->repr_type) { ++ case NFP_REPR_TYPE_PHYS_PORT: ++ nfp_flower_repr_free_queue(repr); ++ break; ++ case NFP_REPR_TYPE_PF: ++ nfp_flower_pf_repr_close_queue(repr); ++ break; ++ case NFP_REPR_TYPE_VF: ++ nfp_flower_repr_free_queue(repr); ++ break; ++ default: ++ PMD_DRV_LOG(ERR, "Unsupported repr port type."); ++ break; ++ } ++} ++ ++static int ++nfp_flower_repr_uninit(struct rte_eth_dev *eth_dev) ++{ ++ uint16_t index; ++ struct nfp_flower_representor *repr; ++ ++ repr = eth_dev->data->dev_private; ++ rte_ring_free(repr->ring); ++ ++ if (repr->repr_type == NFP_REPR_TYPE_PHYS_PORT) { ++ index = NFP_FLOWER_CMSG_PORT_PHYS_PORT_NUM(repr->port_id); ++ repr->app_fw_flower->phy_reprs[index] = NULL; ++ } else { ++ index = repr->vf_id; ++ repr->app_fw_flower->vf_reprs[index] = NULL; ++ } ++ ++ return 0; ++} ++ ++static int ++nfp_flower_pf_repr_uninit(struct rte_eth_dev *eth_dev) ++{ ++ struct nfp_flower_representor *repr = eth_dev->data->dev_private; ++ ++ repr->app_fw_flower->pf_repr = NULL; ++ ++ return 0; ++} ++ ++static void ++nfp_flower_repr_free(struct nfp_flower_representor *repr, ++ enum nfp_repr_type repr_type) ++{ ++ switch (repr_type) { ++ case NFP_REPR_TYPE_PHYS_PORT: ++ nfp_flower_repr_uninit(repr->eth_dev); ++ break; ++ case NFP_REPR_TYPE_PF: ++ nfp_flower_pf_repr_uninit(repr->eth_dev); ++ break; ++ case NFP_REPR_TYPE_VF: ++ nfp_flower_repr_uninit(repr->eth_dev); ++ break; ++ default: ++ PMD_DRV_LOG(ERR, "Unsupported repr port type."); ++ break; ++ } ++} ++ ++/* Reset and stop device. The device can not be restarted. */ ++static int ++nfp_flower_repr_dev_close(struct rte_eth_dev *dev) ++{ ++ uint16_t i; ++ struct nfp_net_hw *hw; ++ struct nfp_pf_dev *pf_dev; ++ struct nfp_flower_representor *repr; ++ struct nfp_app_fw_flower *app_fw_flower; ++ ++ if (rte_eal_process_type() != RTE_PROC_PRIMARY) ++ return 0; ++ ++ repr = dev->data->dev_private; ++ app_fw_flower = repr->app_fw_flower; ++ hw = app_fw_flower->pf_hw; ++ pf_dev = hw->pf_dev; ++ ++ if (pf_dev->app_fw_id != NFP_APP_FW_FLOWER_NIC) ++ return -EINVAL; ++ ++ nfp_flower_repr_close_queue(repr); ++ ++ nfp_flower_repr_free(repr, repr->repr_type); ++ ++ for (i = 0; i < MAX_FLOWER_VFS; i++) { ++ if (app_fw_flower->vf_reprs[i] != NULL) ++ return 0; ++ } ++ ++ for (i = 0; i < NFP_MAX_PHYPORTS; i++) { ++ if (app_fw_flower->phy_reprs[i] != NULL) ++ return 0; ++ } ++ ++ if (app_fw_flower->pf_repr != NULL) ++ return 0; ++ ++ /* Now it is safe to free all PF resources */ ++ nfp_uninit_app_fw_flower(pf_dev); ++ nfp_pf_uninit(pf_dev); ++ ++ return 0; ++} ++ + static const struct eth_dev_ops nfp_flower_pf_repr_dev_ops = { + .dev_infos_get = nfp_flower_repr_dev_infos_get, + + .dev_start = nfp_flower_pf_start, + .dev_configure = nfp_net_configure, + .dev_stop = nfp_net_stop, ++ .dev_close = nfp_flower_repr_dev_close, + + .rx_queue_setup = nfp_net_rx_queue_setup, + .tx_queue_setup = nfp_net_tx_queue_setup, +@@ -319,6 +463,7 @@ static const struct eth_dev_ops nfp_flower_repr_dev_ops = { + .dev_start = nfp_flower_repr_dev_start, + .dev_configure = nfp_net_configure, + .dev_stop = nfp_flower_repr_dev_stop, ++ .dev_close = nfp_flower_repr_dev_close, + + .rx_queue_setup = nfp_flower_repr_rx_queue_setup, + .tx_queue_setup = nfp_flower_repr_tx_queue_setup, +@@ -410,6 +555,7 @@ nfp_flower_pf_repr_init(struct rte_eth_dev *eth_dev, + + repr->app_fw_flower->pf_repr = repr; + repr->app_fw_flower->pf_hw->eth_dev = eth_dev; ++ repr->eth_dev = eth_dev; + + return 0; + } +@@ -501,6 +647,8 @@ nfp_flower_repr_init(struct rte_eth_dev *eth_dev, + app_fw_flower->vf_reprs[index] = repr; + } + ++ repr->eth_dev = eth_dev; ++ + return 0; + + mac_cleanup: +@@ -511,6 +659,35 @@ ring_cleanup: + return ret; + } + ++static void ++nfp_flower_repr_free_all(struct nfp_app_fw_flower *app_fw_flower) ++{ ++ uint32_t i; ++ struct nfp_flower_representor *repr; ++ ++ for (i = 0; i < MAX_FLOWER_VFS; i++) { ++ repr = app_fw_flower->vf_reprs[i]; ++ if (repr != NULL) { ++ nfp_flower_repr_free(repr, NFP_REPR_TYPE_VF); ++ app_fw_flower->vf_reprs[i] = NULL; ++ } ++ } ++ ++ for (i = 0; i < NFP_MAX_PHYPORTS; i++) { ++ repr = app_fw_flower->phy_reprs[i]; ++ if (repr != NULL) { ++ nfp_flower_repr_free(repr, NFP_REPR_TYPE_PHYS_PORT); ++ app_fw_flower->phy_reprs[i] = NULL; ++ } ++ } ++ ++ repr = app_fw_flower->pf_repr; ++ if (repr != NULL) { ++ nfp_flower_repr_free(repr, NFP_REPR_TYPE_PF); ++ app_fw_flower->pf_repr = NULL; ++ } ++} ++ + static int + nfp_flower_repr_alloc(struct nfp_app_fw_flower *app_fw_flower) + { +@@ -563,7 +740,7 @@ nfp_flower_repr_alloc(struct nfp_app_fw_flower *app_fw_flower) + eth_port = &nfp_eth_table->ports[i]; + flower_repr.repr_type = NFP_REPR_TYPE_PHYS_PORT; + flower_repr.port_id = nfp_flower_get_phys_port_id(eth_port->index); +- flower_repr.nfp_idx = eth_port->eth_index; ++ flower_repr.nfp_idx = eth_port->index; + flower_repr.vf_id = i + 1; + + /* Copy the real mac of the interface to the representor struct */ +@@ -585,7 +762,7 @@ nfp_flower_repr_alloc(struct nfp_app_fw_flower *app_fw_flower) + } + + if (i < app_fw_flower->num_phyport_reprs) +- return ret; ++ goto repr_free; + + /* + * Now allocate eth_dev's for VF representors. +@@ -614,9 +791,14 @@ nfp_flower_repr_alloc(struct nfp_app_fw_flower *app_fw_flower) + } + + if (i < app_fw_flower->num_vf_reprs) +- return ret; ++ goto repr_free; + + return 0; ++ ++repr_free: ++ nfp_flower_repr_free_all(app_fw_flower); ++ ++ return ret; + } + + int +@@ -634,10 +816,9 @@ nfp_flower_repr_create(struct nfp_app_fw_flower *app_fw_flower) + pci_dev = pf_dev->pci_dev; + + /* Allocate a switch domain for the flower app */ +- if (app_fw_flower->switch_domain_id == RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID && +- rte_eth_switch_domain_alloc(&app_fw_flower->switch_domain_id)) { ++ ret = rte_eth_switch_domain_alloc(&app_fw_flower->switch_domain_id); ++ if (ret != 0) + PMD_INIT_LOG(WARNING, "failed to allocate switch domain for device"); +- } + + /* Now parse PCI device args passed for representor info */ + if (pci_dev->device.devargs != NULL) { +@@ -677,8 +858,15 @@ nfp_flower_repr_create(struct nfp_app_fw_flower *app_fw_flower) + ret = nfp_flower_repr_alloc(app_fw_flower); + if (ret != 0) { + PMD_INIT_LOG(ERR, "representors allocation failed"); +- return -EINVAL; ++ ret = -EINVAL; ++ goto domain_free; + } + + return 0; ++ ++domain_free: ++ if (rte_eth_switch_domain_free(app_fw_flower->switch_domain_id) != 0) ++ PMD_INIT_LOG(WARNING, "failed to free switch domain for device"); ++ ++ return ret; + } +diff --git a/dpdk/drivers/net/nfp/flower/nfp_flower_representor.h b/dpdk/drivers/net/nfp/flower/nfp_flower_representor.h +index bcb4c3cdb5..8053617562 100644 +--- a/dpdk/drivers/net/nfp/flower/nfp_flower_representor.h ++++ b/dpdk/drivers/net/nfp/flower/nfp_flower_representor.h +@@ -20,6 +20,7 @@ struct nfp_flower_representor { + struct rte_ring *ring; + struct rte_eth_link link; + struct rte_eth_stats repr_stats; ++ struct rte_eth_dev *eth_dev; + }; + + int nfp_flower_repr_create(struct nfp_app_fw_flower *app_fw_flower); +diff --git a/dpdk/drivers/net/nfp/meson.build b/dpdk/drivers/net/nfp/meson.build +index cf9c16266d..7bf94710f1 100644 +--- a/dpdk/drivers/net/nfp/meson.build ++++ b/dpdk/drivers/net/nfp/meson.build +@@ -4,6 +4,7 @@ + if not is_linux or not dpdk_conf.get('RTE_ARCH_64') + build = false + reason = 'only supported on 64-bit Linux' ++ subdir_done() + endif + + sources = files( +diff --git a/dpdk/drivers/net/nfp/nfd3/nfp_nfd3_dp.c b/dpdk/drivers/net/nfp/nfd3/nfp_nfd3_dp.c +index ff9b10f046..b9da74bc99 100644 +--- a/dpdk/drivers/net/nfp/nfd3/nfp_nfd3_dp.c ++++ b/dpdk/drivers/net/nfp/nfd3/nfp_nfd3_dp.c +@@ -137,7 +137,7 @@ nfp_net_nfd3_tx_vlan(struct nfp_net_txq *txq, + } + } + +-static inline void ++static inline int + nfp_net_nfd3_set_meta_data(struct nfp_net_meta_raw *meta_data, + struct nfp_net_txq *txq, + struct rte_mbuf *pkt) +@@ -174,7 +174,7 @@ nfp_net_nfd3_set_meta_data(struct nfp_net_meta_raw *meta_data, + } + + if (meta_data->length == 0) +- return; ++ return 0; + + meta_info = meta_data->header; + meta_data->header = rte_cpu_to_be_32(meta_data->header); +@@ -188,15 +188,16 @@ nfp_net_nfd3_set_meta_data(struct nfp_net_meta_raw *meta_data, + case NFP_NET_META_VLAN: + if (vlan_layer > 0) { + PMD_DRV_LOG(ERR, "At most 1 layers of vlan is supported"); +- return; ++ return -EINVAL; + } ++ + nfp_net_set_meta_vlan(meta_data, pkt, layer); + vlan_layer++; + break; + case NFP_NET_META_IPSEC: + if (ipsec_layer > 2) { + PMD_DRV_LOG(ERR, "At most 3 layers of ipsec is supported for now."); +- return; ++ return -EINVAL; + } + + nfp_net_set_meta_ipsec(meta_data, txq, pkt, layer, ipsec_layer); +@@ -204,11 +205,13 @@ nfp_net_nfd3_set_meta_data(struct nfp_net_meta_raw *meta_data, + break; + default: + PMD_DRV_LOG(ERR, "The metadata type not supported"); +- return; ++ return -ENOTSUP; + } + + memcpy(meta, &meta_data->data[layer], sizeof(meta_data->data[layer])); + } ++ ++ return 0; + } + + uint16_t +@@ -225,6 +228,7 @@ nfp_net_nfd3_xmit_pkts_common(void *tx_queue, + uint16_t nb_pkts, + bool repr_flag) + { ++ int ret; + uint16_t i; + uint8_t offset; + uint32_t pkt_size; +@@ -271,7 +275,10 @@ nfp_net_nfd3_xmit_pkts_common(void *tx_queue, + if (!repr_flag) { + struct nfp_net_meta_raw meta_data; + memset(&meta_data, 0, sizeof(meta_data)); +- nfp_net_nfd3_set_meta_data(&meta_data, txq, pkt); ++ ret = nfp_net_nfd3_set_meta_data(&meta_data, txq, pkt); ++ if (unlikely(ret != 0)) ++ goto xmit_end; ++ + offset = meta_data.length; + } else { + offset = FLOWER_PKT_DATA_OFFSET; +diff --git a/dpdk/drivers/net/nfp/nfdk/nfp_nfdk_dp.c b/dpdk/drivers/net/nfp/nfdk/nfp_nfdk_dp.c +index 0141fbcc8f..772c847b9d 100644 +--- a/dpdk/drivers/net/nfp/nfdk/nfp_nfdk_dp.c ++++ b/dpdk/drivers/net/nfp/nfdk/nfp_nfdk_dp.c +@@ -167,7 +167,7 @@ close_block: + return nop_slots; + } + +-static void ++static int + nfp_net_nfdk_set_meta_data(struct rte_mbuf *pkt, + struct nfp_net_txq *txq, + uint64_t *metadata) +@@ -178,7 +178,6 @@ nfp_net_nfdk_set_meta_data(struct rte_mbuf *pkt, + uint32_t cap_extend; + struct nfp_net_hw *hw; + uint32_t header_offset; +- uint8_t vlan_layer = 0; + uint8_t ipsec_layer = 0; + struct nfp_net_meta_raw meta_data; + +@@ -206,8 +205,10 @@ nfp_net_nfdk_set_meta_data(struct rte_mbuf *pkt, + meta_data.length += 3 * NFP_NET_META_FIELD_SIZE; + } + +- if (meta_data.length == 0) +- return; ++ if (meta_data.length == 0) { ++ *metadata = 0; ++ return 0; ++ } + + meta_type = meta_data.header; + header_offset = meta_type << NFP_NET_META_NFDK_LENGTH; +@@ -221,17 +222,13 @@ nfp_net_nfdk_set_meta_data(struct rte_mbuf *pkt, + meta += NFP_NET_META_FIELD_SIZE) { + switch (meta_type & NFP_NET_META_FIELD_MASK) { + case NFP_NET_META_VLAN: +- if (vlan_layer > 0) { +- PMD_DRV_LOG(ERR, "At most 1 layers of vlan is supported"); +- return; +- } ++ + nfp_net_set_meta_vlan(&meta_data, pkt, layer); +- vlan_layer++; + break; + case NFP_NET_META_IPSEC: + if (ipsec_layer > 2) { + PMD_DRV_LOG(ERR, "At most 3 layers of ipsec is supported for now."); +- return; ++ return -EINVAL; + } + + nfp_net_set_meta_ipsec(&meta_data, txq, pkt, layer, ipsec_layer); +@@ -239,13 +236,15 @@ nfp_net_nfdk_set_meta_data(struct rte_mbuf *pkt, + break; + default: + PMD_DRV_LOG(ERR, "The metadata type not supported"); +- return; ++ return -ENOTSUP; + } + + memcpy(meta, &meta_data.data[layer], sizeof(meta_data.data[layer])); + } + + *metadata = NFDK_DESC_TX_CHAIN_META; ++ ++ return 0; + } + + uint16_t +@@ -292,6 +291,7 @@ nfp_net_nfdk_xmit_pkts_common(void *tx_queue, + + /* Sending packets */ + while (npkts < nb_pkts && free_descs > 0) { ++ int ret; + int nop_descs; + uint32_t type; + uint32_t dma_len; +@@ -319,10 +319,13 @@ nfp_net_nfdk_xmit_pkts_common(void *tx_queue, + + temp_pkt = pkt; + +- if (repr_flag) ++ if (repr_flag) { + metadata = NFDK_DESC_TX_CHAIN_META; +- else +- nfp_net_nfdk_set_meta_data(pkt, txq, &metadata); ++ } else { ++ ret = nfp_net_nfdk_set_meta_data(pkt, txq, &metadata); ++ if (unlikely(ret != 0)) ++ goto xmit_end; ++ } + + if (unlikely(pkt->nb_segs > 1 && + (hw->super.cap & NFP_NET_CFG_CTRL_GATHER) == 0)) { +diff --git a/dpdk/drivers/net/nfp/nfp_ethdev.c b/dpdk/drivers/net/nfp/nfp_ethdev.c +index f02caf8056..7495b01f16 100644 +--- a/dpdk/drivers/net/nfp/nfp_ethdev.c ++++ b/dpdk/drivers/net/nfp/nfp_ethdev.c +@@ -310,6 +310,66 @@ nfp_net_keepalive_stop(struct nfp_multi_pf *multi_pf) + rte_eal_alarm_cancel(nfp_net_beat_timer, (void *)multi_pf); + } + ++static void ++nfp_net_uninit(struct rte_eth_dev *eth_dev) ++{ ++ struct nfp_net_hw *net_hw; ++ ++ net_hw = eth_dev->data->dev_private; ++ rte_free(net_hw->eth_xstats_base); ++ nfp_ipsec_uninit(eth_dev); ++} ++ ++static void ++nfp_cleanup_port_app_fw_nic(struct nfp_pf_dev *pf_dev, ++ uint8_t id) ++{ ++ struct rte_eth_dev *eth_dev; ++ struct nfp_app_fw_nic *app_fw_nic; ++ ++ app_fw_nic = pf_dev->app_fw_priv; ++ if (app_fw_nic->ports[id] != NULL) { ++ eth_dev = app_fw_nic->ports[id]->eth_dev; ++ if (eth_dev != NULL) ++ nfp_net_uninit(eth_dev); ++ ++ app_fw_nic->ports[id] = NULL; ++ } ++} ++ ++static void ++nfp_uninit_app_fw_nic(struct nfp_pf_dev *pf_dev) ++{ ++ nfp_cpp_area_release_free(pf_dev->ctrl_area); ++ rte_free(pf_dev->app_fw_priv); ++} ++ ++void ++nfp_pf_uninit(struct nfp_pf_dev *pf_dev) ++{ ++ nfp_cpp_area_release_free(pf_dev->mac_stats_area); ++ nfp_cpp_area_release_free(pf_dev->qc_area); ++ free(pf_dev->sym_tbl); ++ if (pf_dev->multi_pf.enabled) { ++ nfp_net_keepalive_stop(&pf_dev->multi_pf); ++ nfp_net_keepalive_uninit(&pf_dev->multi_pf); ++ } ++ free(pf_dev->nfp_eth_table); ++ free(pf_dev->hwinfo); ++ nfp_cpp_free(pf_dev->cpp); ++ rte_free(pf_dev); ++} ++ ++static int ++nfp_pf_secondary_uninit(struct nfp_pf_dev *pf_dev) ++{ ++ free(pf_dev->sym_tbl); ++ nfp_cpp_free(pf_dev->cpp); ++ rte_free(pf_dev); ++ ++ return 0; ++} ++ + /* Reset and stop device. The device can not be restarted. */ + static int + nfp_net_close(struct rte_eth_dev *dev) +@@ -321,8 +381,19 @@ nfp_net_close(struct rte_eth_dev *dev) + struct rte_pci_device *pci_dev; + struct nfp_app_fw_nic *app_fw_nic; + +- if (rte_eal_process_type() != RTE_PROC_PRIMARY) ++ /* ++ * In secondary process, a released eth device can be found by its name ++ * in shared memory. ++ * If the state of the eth device is RTE_ETH_DEV_UNUSED, it means the ++ * eth device has been released. ++ */ ++ if (rte_eal_process_type() == RTE_PROC_SECONDARY) { ++ if (dev->state == RTE_ETH_DEV_UNUSED) ++ return 0; ++ ++ nfp_pf_secondary_uninit(dev->process_private); + return 0; ++ } + + hw = dev->data->dev_private; + pf_dev = hw->pf_dev; +@@ -339,16 +410,17 @@ nfp_net_close(struct rte_eth_dev *dev) + nfp_net_close_tx_queue(dev); + nfp_net_close_rx_queue(dev); + +- /* Clear ipsec */ +- nfp_ipsec_uninit(dev); +- + /* Cancel possible impending LSC work here before releasing the port */ + rte_eal_alarm_cancel(nfp_net_dev_interrupt_delayed_handler, (void *)dev); + + /* Only free PF resources after all physical ports have been closed */ + /* Mark this port as unused and free device priv resources */ + nn_cfg_writeb(&hw->super, NFP_NET_CFG_LSC, 0xff); +- app_fw_nic->ports[hw->idx] = NULL; ++ ++ if (pf_dev->app_fw_id != NFP_APP_FW_CORE_NIC) ++ return -EINVAL; ++ ++ nfp_cleanup_port_app_fw_nic(pf_dev, hw->idx); + + for (i = 0; i < app_fw_nic->total_phyports; i++) { + id = nfp_function_id_get(pf_dev, i); +@@ -358,26 +430,16 @@ nfp_net_close(struct rte_eth_dev *dev) + return 0; + } + +- /* Now it is safe to free all PF resources */ +- PMD_INIT_LOG(INFO, "Freeing PF resources"); +- if (pf_dev->multi_pf.enabled) { +- nfp_net_keepalive_stop(&pf_dev->multi_pf); +- nfp_net_keepalive_uninit(&pf_dev->multi_pf); +- } +- nfp_cpp_area_free(pf_dev->ctrl_area); +- nfp_cpp_area_free(pf_dev->qc_area); +- free(pf_dev->hwinfo); +- free(pf_dev->sym_tbl); +- nfp_cpp_free(pf_dev->cpp); +- rte_free(app_fw_nic); +- rte_free(pf_dev); +- ++ /* Enable in nfp_net_start() */ + rte_intr_disable(pci_dev->intr_handle); + +- /* Unregister callback func from eal lib */ ++ /* Register in nfp_net_init() */ + rte_intr_callback_unregister(pci_dev->intr_handle, + nfp_net_dev_interrupt_handler, (void *)dev); + ++ nfp_uninit_app_fw_nic(pf_dev); ++ nfp_pf_uninit(pf_dev); ++ + return 0; + } + +@@ -576,28 +638,13 @@ nfp_net_init(struct rte_eth_dev *eth_dev) + + rte_eth_copy_pci_info(eth_dev, pci_dev); + +- if (port == 0 || pf_dev->multi_pf.enabled) { +- uint32_t min_size; +- ++ if (pf_dev->multi_pf.enabled) + hw->ctrl_bar = pf_dev->ctrl_bar; +- min_size = NFP_MAC_STATS_SIZE * net_hw->pf_dev->nfp_eth_table->max_index; +- net_hw->mac_stats_bar = nfp_rtsym_map(net_hw->pf_dev->sym_tbl, "_mac_stats", +- min_size, &net_hw->mac_stats_area); +- if (net_hw->mac_stats_bar == NULL) { +- PMD_INIT_LOG(ERR, "nfp_rtsym_map fails for _mac_stats_bar"); +- return -EIO; +- } +- +- net_hw->mac_stats = net_hw->mac_stats_bar; +- } else { +- if (pf_dev->ctrl_bar == NULL) +- return -ENODEV; +- +- /* Use port offset in pf ctrl_bar for this ports control bar */ ++ else + hw->ctrl_bar = pf_dev->ctrl_bar + (port * NFP_NET_CFG_BAR_SZ); +- net_hw->mac_stats = app_fw_nic->ports[0]->mac_stats_bar + ++ ++ net_hw->mac_stats = pf_dev->mac_stats_bar + + (net_hw->nfp_idx * NFP_MAC_STATS_SIZE); +- } + + PMD_INIT_LOG(DEBUG, "ctrl bar: %p", hw->ctrl_bar); + PMD_INIT_LOG(DEBUG, "MAC stats: %p", net_hw->mac_stats); +@@ -625,7 +672,8 @@ nfp_net_init(struct rte_eth_dev *eth_dev) + if (net_hw->eth_xstats_base == NULL) { + PMD_INIT_LOG(ERR, "no memory for xstats base values on device %s!", + pci_dev->device.name); +- return -ENOMEM; ++ err = -ENOMEM; ++ goto ipsec_exit; + } + + /* Work out where in the BAR the queues start. */ +@@ -655,7 +703,8 @@ nfp_net_init(struct rte_eth_dev *eth_dev) + eth_dev->data->mac_addrs = rte_zmalloc("mac_addr", RTE_ETHER_ADDR_LEN, 0); + if (eth_dev->data->mac_addrs == NULL) { + PMD_INIT_LOG(ERR, "Failed to space for MAC address"); +- return -ENOMEM; ++ err = -ENOMEM; ++ goto xstats_free; + } + + nfp_net_pf_read_mac(app_fw_nic, port); +@@ -693,6 +742,13 @@ nfp_net_init(struct rte_eth_dev *eth_dev) + nfp_net_stats_reset(eth_dev); + + return 0; ++ ++xstats_free: ++ rte_free(net_hw->eth_xstats_base); ++ipsec_exit: ++ nfp_ipsec_uninit(eth_dev); ++ ++ return err; + } + + #define DEFAULT_FW_PATH "/lib/firmware/netronome" +@@ -1120,26 +1176,46 @@ port_cleanup: + app_fw_nic->ports[id]->eth_dev != NULL) { + struct rte_eth_dev *tmp_dev; + tmp_dev = app_fw_nic->ports[id]->eth_dev; +- nfp_ipsec_uninit(tmp_dev); ++ nfp_net_uninit(tmp_dev); + rte_eth_dev_release_port(tmp_dev); +- app_fw_nic->ports[id] = NULL; + } + } +- nfp_cpp_area_free(pf_dev->ctrl_area); ++ nfp_cpp_area_release_free(pf_dev->ctrl_area); + app_cleanup: + rte_free(app_fw_nic); + + return ret; + } + ++/* Force the physical port down to clear the possible DMA error */ + static int +-nfp_pf_init(struct rte_pci_device *pci_dev) ++nfp_net_force_port_down(struct nfp_pf_dev *pf_dev, ++ struct nfp_eth_table *nfp_eth_table, ++ struct nfp_cpp *cpp) + { ++ int ret; + uint32_t i; + uint32_t id; ++ uint32_t index; ++ uint32_t count; ++ ++ count = nfp_net_get_port_num(pf_dev, nfp_eth_table); ++ for (i = 0; i < count; i++) { ++ id = nfp_function_id_get(pf_dev, i); ++ index = nfp_eth_table->ports[id].index; ++ ret = nfp_eth_set_configured(cpp, index, 0); ++ if (ret < 0) ++ return ret; ++ } ++ ++ return 0; ++} ++ ++static int ++nfp_pf_init(struct rte_pci_device *pci_dev) ++{ + int ret = 0; + uint64_t addr; +- uint32_t index; + uint32_t cpp_id; + uint8_t function_id; + struct nfp_cpp *cpp; +@@ -1211,11 +1287,11 @@ nfp_pf_init(struct rte_pci_device *pci_dev) + pf_dev->multi_pf.enabled = nfp_check_multi_pf_from_nsp(pci_dev, cpp); + pf_dev->multi_pf.function_id = function_id; + +- /* Force the physical port down to clear the possible DMA error */ +- for (i = 0; i < nfp_eth_table->count; i++) { +- id = nfp_function_id_get(pf_dev, i); +- index = nfp_eth_table->ports[id].index; +- nfp_eth_set_configured(cpp, index, 0); ++ ret = nfp_net_force_port_down(pf_dev, nfp_eth_table, cpp); ++ if (ret != 0) { ++ PMD_INIT_LOG(ERR, "Failed to force port down"); ++ ret = -EIO; ++ goto eth_table_cleanup; + } + + if (nfp_fw_setup(pci_dev, cpp, nfp_eth_table, hwinfo, +@@ -1264,6 +1340,14 @@ nfp_pf_init(struct rte_pci_device *pci_dev) + + PMD_INIT_LOG(DEBUG, "qc_bar address: %p", pf_dev->qc_bar); + ++ pf_dev->mac_stats_bar = nfp_rtsym_map(sym_tbl, "_mac_stats", ++ NFP_MAC_STATS_SIZE * nfp_eth_table->max_index, ++ &pf_dev->mac_stats_area); ++ if (pf_dev->mac_stats_bar == NULL) { ++ PMD_INIT_LOG(ERR, "nfp_rtsym_map fails for _mac_stats"); ++ goto hwqueues_cleanup; ++ } ++ + /* + * PF initialization has been done at this point. Call app specific + * init code now. +@@ -1273,14 +1357,14 @@ nfp_pf_init(struct rte_pci_device *pci_dev) + if (pf_dev->multi_pf.enabled) { + ret = nfp_enable_multi_pf(pf_dev); + if (ret != 0) +- goto hwqueues_cleanup; ++ goto mac_stats_cleanup; + } + + PMD_INIT_LOG(INFO, "Initializing coreNIC"); + ret = nfp_init_app_fw_nic(pf_dev, dev_info); + if (ret != 0) { + PMD_INIT_LOG(ERR, "Could not initialize coreNIC!"); +- goto hwqueues_cleanup; ++ goto mac_stats_cleanup; + } + break; + case NFP_APP_FW_FLOWER_NIC: +@@ -1288,13 +1372,13 @@ nfp_pf_init(struct rte_pci_device *pci_dev) + ret = nfp_init_app_fw_flower(pf_dev, dev_info); + if (ret != 0) { + PMD_INIT_LOG(ERR, "Could not initialize Flower!"); +- goto hwqueues_cleanup; ++ goto mac_stats_cleanup; + } + break; + default: + PMD_INIT_LOG(ERR, "Unsupported Firmware loaded"); + ret = -EINVAL; +- goto hwqueues_cleanup; ++ goto mac_stats_cleanup; + } + + /* Register the CPP bridge service here for primary use */ +@@ -1304,13 +1388,18 @@ nfp_pf_init(struct rte_pci_device *pci_dev) + + return 0; + ++mac_stats_cleanup: ++ nfp_cpp_area_release_free(pf_dev->mac_stats_area); + hwqueues_cleanup: +- nfp_cpp_area_free(pf_dev->qc_area); ++ nfp_cpp_area_release_free(pf_dev->qc_area); + sym_tbl_cleanup: + free(sym_tbl); + fw_cleanup: + nfp_fw_unload(cpp); +- nfp_net_keepalive_stop(&pf_dev->multi_pf); ++ if (pf_dev->multi_pf.enabled) { ++ nfp_net_keepalive_stop(&pf_dev->multi_pf); ++ nfp_net_keepalive_uninit(&pf_dev->multi_pf); ++ } + eth_table_cleanup: + free(nfp_eth_table); + hwinfo_cleanup: +@@ -1437,7 +1526,7 @@ nfp_pf_secondary_init(struct rte_pci_device *pci_dev) + if (sym_tbl == NULL) { + PMD_INIT_LOG(ERR, "Something is wrong with the firmware symbol table"); + ret = -EIO; +- goto pf_cleanup; ++ goto cpp_cleanup; + } + + /* Read the app ID of the firmware loaded */ +@@ -1484,6 +1573,8 @@ nfp_pf_secondary_init(struct rte_pci_device *pci_dev) + + sym_tbl_cleanup: + free(sym_tbl); ++cpp_cleanup: ++ nfp_cpp_free(cpp); + pf_cleanup: + rte_free(pf_dev); + +diff --git a/dpdk/drivers/net/nfp/nfp_ethdev_vf.c b/dpdk/drivers/net/nfp/nfp_ethdev_vf.c +index 7927f53403..cfe7225ca5 100644 +--- a/dpdk/drivers/net/nfp/nfp_ethdev_vf.c ++++ b/dpdk/drivers/net/nfp/nfp_ethdev_vf.c +@@ -160,13 +160,17 @@ nfp_netvf_set_link_down(struct rte_eth_dev *dev __rte_unused) + static int + nfp_netvf_close(struct rte_eth_dev *dev) + { ++ struct nfp_net_hw *net_hw; + struct rte_pci_device *pci_dev; + + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return 0; + ++ net_hw = dev->data->dev_private; + pci_dev = RTE_ETH_DEV_TO_PCI(dev); + ++ rte_free(net_hw->eth_xstats_base); ++ + /* + * We assume that the DPDK application is stopping all the + * threads/queues before calling the device close function. +@@ -284,8 +288,6 @@ nfp_netvf_init(struct rte_eth_dev *eth_dev) + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return 0; + +- rte_eth_copy_pci_info(eth_dev, pci_dev); +- + net_hw->eth_xstats_base = rte_malloc("rte_eth_xstat", + sizeof(struct rte_eth_xstat) * nfp_net_xstats_size(eth_dev), 0); + if (net_hw->eth_xstats_base == NULL) { +@@ -323,7 +325,7 @@ nfp_netvf_init(struct rte_eth_dev *eth_dev) + if (eth_dev->data->mac_addrs == NULL) { + PMD_INIT_LOG(ERR, "Failed to space for MAC address"); + err = -ENOMEM; +- goto dev_err_ctrl_map; ++ goto free_xstats; + } + + nfp_read_mac(hw); +@@ -360,8 +362,8 @@ nfp_netvf_init(struct rte_eth_dev *eth_dev) + + return 0; + +-dev_err_ctrl_map: +- nfp_cpp_area_free(net_hw->ctrl_area); ++free_xstats: ++ rte_free(net_hw->eth_xstats_base); + + return err; + } +diff --git a/dpdk/drivers/net/nfp/nfp_flow.c b/dpdk/drivers/net/nfp/nfp_flow.c +index f832b52d89..91ebee5db4 100644 +--- a/dpdk/drivers/net/nfp/nfp_flow.c ++++ b/dpdk/drivers/net/nfp/nfp_flow.c +@@ -312,14 +312,14 @@ nfp_check_mask_add(struct nfp_flow_priv *priv, + ret = nfp_mask_table_add(priv, mask_data, mask_len, mask_id); + if (ret != 0) + return false; ++ ++ *meta_flags |= NFP_FL_META_FLAG_MANAGE_MASK; + } else { + /* Mask entry already exist */ + mask_entry->ref_cnt++; + *mask_id = mask_entry->mask_id; + } + +- *meta_flags |= NFP_FL_META_FLAG_MANAGE_MASK; +- + return true; + } + +@@ -3658,7 +3658,7 @@ nfp_flow_compile_action(struct nfp_flower_representor *representor, + ttl_tos_flag = true; + } + } else { +- nfp_flow_action_set_hl(position, action, ttl_tos_flag); ++ nfp_flow_action_set_hl(position, action, tc_hl_flag); + if (!tc_hl_flag) { + position += sizeof(struct nfp_fl_act_set_ipv6_tc_hl_fl); + tc_hl_flag = true; +@@ -3675,7 +3675,7 @@ nfp_flow_compile_action(struct nfp_flower_representor *representor, + break; + case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: + PMD_DRV_LOG(DEBUG, "Process RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP"); +- nfp_flow_action_set_tc(position, action, ttl_tos_flag); ++ nfp_flow_action_set_tc(position, action, tc_hl_flag); + if (!tc_hl_flag) { + position += sizeof(struct nfp_fl_act_set_ipv6_tc_hl_fl); + tc_hl_flag = true; +@@ -3741,6 +3741,11 @@ nfp_flow_compile_action(struct nfp_flower_representor *representor, + total_actions++; + } + ++ if (nfp_flow->install_flag && total_actions == 0) { ++ PMD_DRV_LOG(ERR, "The action list is empty"); ++ return -ENOTSUP; ++ } ++ + if (drop_flag) + nfp_flow_meta->shortcut = rte_cpu_to_be_32(NFP_FL_SC_ACT_DROP); + else if (total_actions > 1) +diff --git a/dpdk/drivers/net/nfp/nfp_ipsec.c b/dpdk/drivers/net/nfp/nfp_ipsec.c +index 452947380e..b10cda570b 100644 +--- a/dpdk/drivers/net/nfp/nfp_ipsec.c ++++ b/dpdk/drivers/net/nfp/nfp_ipsec.c +@@ -18,6 +18,7 @@ + #include "nfp_rxtx.h" + + #define NFP_UDP_ESP_PORT 4500 ++#define NFP_ESP_IV_LENGTH 8 + + static const struct rte_cryptodev_capabilities nfp_crypto_caps[] = { + { +@@ -521,10 +522,14 @@ nfp_aesgcm_iv_update(struct ipsec_add_sa *cfg, + char *save; + char *iv_b; + char *iv_str; +- uint8_t *cfg_iv; ++ const rte_be32_t *iv_value; ++ uint8_t cfg_iv[NFP_ESP_IV_LENGTH] = {}; + + iv_str = strdup(iv_string); +- cfg_iv = (uint8_t *)cfg->aesgcm_fields.iv; ++ if (iv_str == NULL) { ++ PMD_DRV_LOG(ERR, "Failed to strdup iv_string"); ++ return; ++ } + + for (i = 0; i < iv_len; i++) { + iv_b = strtok_r(i ? NULL : iv_str, ",", &save); +@@ -534,8 +539,9 @@ nfp_aesgcm_iv_update(struct ipsec_add_sa *cfg, + cfg_iv[i] = strtoul(iv_b, NULL, 0); + } + +- *(uint32_t *)cfg_iv = rte_be_to_cpu_32(*(uint32_t *)cfg_iv); +- *(uint32_t *)&cfg_iv[4] = rte_be_to_cpu_32(*(uint32_t *)&cfg_iv[4]); ++ iv_value = (const rte_be32_t *)(cfg_iv); ++ cfg->aesgcm_fields.iv[0] = rte_be_to_cpu_32(iv_value[0]); ++ cfg->aesgcm_fields.iv[1] = rte_be_to_cpu_32(iv_value[1]); + + free(iv_str); + } +@@ -576,7 +582,7 @@ nfp_aead_map(struct rte_eth_dev *eth_dev, + uint32_t offset; + uint32_t device_id; + const char *iv_str; +- const uint32_t *key; ++ const rte_be32_t *key; + struct nfp_net_hw *net_hw; + + net_hw = eth_dev->data->dev_private; +@@ -626,7 +632,7 @@ nfp_aead_map(struct rte_eth_dev *eth_dev, + return -EINVAL; + } + +- key = (const uint32_t *)(aead->key.data); ++ key = (const rte_be32_t *)(aead->key.data); + + /* + * The CHACHA20's key order needs to be adjusted based on hardware design. +@@ -638,16 +644,22 @@ nfp_aead_map(struct rte_eth_dev *eth_dev, + + for (i = 0; i < key_length / sizeof(cfg->cipher_key[0]); i++) { + index = (i + offset) % (key_length / sizeof(cfg->cipher_key[0])); +- cfg->cipher_key[index] = rte_cpu_to_be_32(*key++); ++ cfg->cipher_key[index] = rte_be_to_cpu_32(key[i]); + } + + /* +- * The iv of the FW is equal to ESN by default. Reading the +- * iv of the configuration information is not supported. ++ * The iv of the FW is equal to ESN by default. Only the ++ * aead algorithm can offload the iv of configuration and ++ * the length of iv cannot be greater than NFP_ESP_IV_LENGTH. + */ + iv_str = getenv("ETH_SEC_IV_OVR"); + if (iv_str != NULL) { + iv_len = aead->iv.length; ++ if (iv_len > NFP_ESP_IV_LENGTH) { ++ PMD_DRV_LOG(ERR, "Unsupported length of iv data"); ++ return -EINVAL; ++ } ++ + nfp_aesgcm_iv_update(cfg, iv_len, iv_str); + } + +@@ -664,7 +676,7 @@ nfp_cipher_map(struct rte_eth_dev *eth_dev, + int ret; + uint32_t i; + uint32_t device_id; +- const uint32_t *key; ++ const rte_be32_t *key; + struct nfp_net_hw *net_hw; + + net_hw = eth_dev->data->dev_private; +@@ -698,14 +710,14 @@ nfp_cipher_map(struct rte_eth_dev *eth_dev, + return -EINVAL; + } + +- key = (const uint32_t *)(cipher->key.data); ++ key = (const rte_be32_t *)(cipher->key.data); + if (key_length > sizeof(cfg->cipher_key)) { + PMD_DRV_LOG(ERR, "Insufficient space for offloaded key"); + return -EINVAL; + } + + for (i = 0; i < key_length / sizeof(cfg->cipher_key[0]); i++) +- cfg->cipher_key[i] = rte_cpu_to_be_32(*key++); ++ cfg->cipher_key[i] = rte_be_to_cpu_32(key[i]); + + return 0; + } +@@ -800,7 +812,7 @@ nfp_auth_map(struct rte_eth_dev *eth_dev, + uint32_t i; + uint8_t key_length; + uint32_t device_id; +- const uint32_t *key; ++ const rte_be32_t *key; + struct nfp_net_hw *net_hw; + + if (digest_length == 0) { +@@ -847,7 +859,7 @@ nfp_auth_map(struct rte_eth_dev *eth_dev, + return -EINVAL; + } + +- key = (const uint32_t *)(auth->key.data); ++ key = (const rte_be32_t *)(auth->key.data); + key_length = auth->key.length; + if (key_length > sizeof(cfg->auth_key)) { + PMD_DRV_LOG(ERR, "Insufficient space for offloaded auth key!"); +@@ -855,7 +867,7 @@ nfp_auth_map(struct rte_eth_dev *eth_dev, + } + + for (i = 0; i < key_length / sizeof(cfg->auth_key[0]); i++) +- cfg->auth_key[i] = rte_cpu_to_be_32(*key++); ++ cfg->auth_key[i] = rte_be_to_cpu_32(key[i]); + + return 0; + } +@@ -895,7 +907,7 @@ nfp_crypto_msg_build(struct rte_eth_dev *eth_dev, + return ret; + } + +- cfg->aesgcm_fields.salt = rte_cpu_to_be_32(conf->ipsec.salt); ++ cfg->aesgcm_fields.salt = conf->ipsec.salt; + break; + case RTE_CRYPTO_SYM_XFORM_AUTH: + /* Only support Auth + Cipher for inbound */ +@@ -960,7 +972,10 @@ nfp_ipsec_msg_build(struct rte_eth_dev *eth_dev, + struct rte_security_session_conf *conf, + struct nfp_ipsec_msg *msg) + { ++ int i; + int ret; ++ rte_be32_t *src_ip; ++ rte_be32_t *dst_ip; + struct ipsec_add_sa *cfg; + enum rte_security_ipsec_tunnel_type type; + +@@ -1018,12 +1033,18 @@ nfp_ipsec_msg_build(struct rte_eth_dev *eth_dev, + type = conf->ipsec.tunnel.type; + cfg->ctrl_word.mode = NFP_IPSEC_MODE_TUNNEL; + if (type == RTE_SECURITY_IPSEC_TUNNEL_IPV4) { +- cfg->src_ip.v4 = conf->ipsec.tunnel.ipv4.src_ip; +- cfg->dst_ip.v4 = conf->ipsec.tunnel.ipv4.dst_ip; ++ src_ip = (rte_be32_t *)&conf->ipsec.tunnel.ipv4.src_ip.s_addr; ++ dst_ip = (rte_be32_t *)&conf->ipsec.tunnel.ipv4.dst_ip.s_addr; ++ cfg->src_ip[0] = rte_be_to_cpu_32(src_ip[0]); ++ cfg->dst_ip[0] = rte_be_to_cpu_32(dst_ip[0]); + cfg->ipv6 = 0; + } else if (type == RTE_SECURITY_IPSEC_TUNNEL_IPV6) { +- cfg->src_ip.v6 = conf->ipsec.tunnel.ipv6.src_addr; +- cfg->dst_ip.v6 = conf->ipsec.tunnel.ipv6.dst_addr; ++ src_ip = (rte_be32_t *)conf->ipsec.tunnel.ipv6.src_addr.s6_addr; ++ dst_ip = (rte_be32_t *)conf->ipsec.tunnel.ipv6.dst_addr.s6_addr; ++ for (i = 0; i < 4; i++) { ++ cfg->src_ip[i] = rte_be_to_cpu_32(src_ip[i]); ++ cfg->dst_ip[i] = rte_be_to_cpu_32(dst_ip[i]); ++ } + cfg->ipv6 = 1; + } else { + PMD_DRV_LOG(ERR, "Unsupported address family!"); +@@ -1036,9 +1057,11 @@ nfp_ipsec_msg_build(struct rte_eth_dev *eth_dev, + cfg->ctrl_word.mode = NFP_IPSEC_MODE_TRANSPORT; + if (type == RTE_SECURITY_IPSEC_TUNNEL_IPV4) { + memset(&cfg->src_ip, 0, sizeof(cfg->src_ip)); ++ memset(&cfg->dst_ip, 0, sizeof(cfg->dst_ip)); + cfg->ipv6 = 0; + } else if (type == RTE_SECURITY_IPSEC_TUNNEL_IPV6) { + memset(&cfg->src_ip, 0, sizeof(cfg->src_ip)); ++ memset(&cfg->dst_ip, 0, sizeof(cfg->dst_ip)); + cfg->ipv6 = 1; + } else { + PMD_DRV_LOG(ERR, "Unsupported address family!"); +@@ -1172,18 +1195,18 @@ nfp_security_set_pkt_metadata(void *device, + desc_md = RTE_MBUF_DYNFIELD(m, offset, struct nfp_tx_ipsec_desc_msg *); + + if (priv_session->msg.ctrl_word.ext_seq != 0 && sqn != NULL) { +- desc_md->esn.low = rte_cpu_to_be_32(*sqn); +- desc_md->esn.hi = rte_cpu_to_be_32(*sqn >> 32); ++ desc_md->esn.low = (uint32_t)*sqn; ++ desc_md->esn.hi = (uint32_t)(*sqn >> 32); + } else if (priv_session->msg.ctrl_word.ext_seq != 0) { +- desc_md->esn.low = rte_cpu_to_be_32(priv_session->ipsec.esn.low); +- desc_md->esn.hi = rte_cpu_to_be_32(priv_session->ipsec.esn.hi); ++ desc_md->esn.low = priv_session->ipsec.esn.low; ++ desc_md->esn.hi = priv_session->ipsec.esn.hi; + } else { +- desc_md->esn.low = rte_cpu_to_be_32(priv_session->ipsec.esn.value); ++ desc_md->esn.low = priv_session->ipsec.esn.low; + desc_md->esn.hi = 0; + } + + desc_md->enc = 1; +- desc_md->sa_idx = rte_cpu_to_be_32(priv_session->sa_index); ++ desc_md->sa_idx = priv_session->sa_index; + } + + return 0; +diff --git a/dpdk/drivers/net/nfp/nfp_ipsec.h b/dpdk/drivers/net/nfp/nfp_ipsec.h +index d7a729398a..f7c4f3f225 100644 +--- a/dpdk/drivers/net/nfp/nfp_ipsec.h ++++ b/dpdk/drivers/net/nfp/nfp_ipsec.h +@@ -36,11 +36,6 @@ struct sa_ctrl_word { + uint32_t spare2 :1; /**< Must be set to 0 */ + }; + +-union nfp_ip_addr { +- struct in6_addr v6; +- struct in_addr v4; +-}; +- + struct ipsec_add_sa { + uint32_t cipher_key[8]; /**< Cipher Key */ + union { +@@ -60,8 +55,8 @@ struct ipsec_add_sa { + uint8_t spare1; + uint32_t soft_byte_cnt; /**< Soft lifetime byte count */ + uint32_t hard_byte_cnt; /**< Hard lifetime byte count */ +- union nfp_ip_addr src_ip; /**< Src IP addr */ +- union nfp_ip_addr dst_ip; /**< Dst IP addr */ ++ uint32_t src_ip[4]; /**< Src IP addr */ ++ uint32_t dst_ip[4]; /**< Dst IP addr */ + uint16_t natt_dst_port; /**< NAT-T UDP Header dst port */ + uint16_t natt_src_port; /**< NAT-T UDP Header src port */ + uint32_t soft_lifetime_limit; /**< Soft lifetime time limit */ +diff --git a/dpdk/drivers/net/nfp/nfp_net_common.c b/dpdk/drivers/net/nfp/nfp_net_common.c +index e969b840d6..0491912bd3 100644 +--- a/dpdk/drivers/net/nfp/nfp_net_common.c ++++ b/dpdk/drivers/net/nfp/nfp_net_common.c +@@ -189,9 +189,6 @@ nfp_net_notify_port_speed(struct nfp_net_hw *hw, + nfp_net_link_speed_rte2nfp(link->link_speed)); + } + +-/* The length of firmware version string */ +-#define FW_VER_LEN 32 +- + /** + * Reconfigure the firmware via the mailbox + * +@@ -1299,6 +1296,7 @@ nfp_net_supported_ptypes_get(struct rte_eth_dev *dev) + RTE_PTYPE_INNER_L4_NONFRAG, + RTE_PTYPE_INNER_L4_ICMP, + RTE_PTYPE_INNER_L4_SCTP, ++ RTE_PTYPE_UNKNOWN + }; + + if (dev->rx_pkt_burst != nfp_net_recv_pkts) +@@ -2062,17 +2060,22 @@ nfp_net_firmware_version_get(struct rte_eth_dev *dev, + size_t fw_size) + { + struct nfp_net_hw *hw; +- char mip_name[FW_VER_LEN]; +- char app_name[FW_VER_LEN]; +- char nsp_version[FW_VER_LEN]; +- char vnic_version[FW_VER_LEN]; ++ char app_name[FW_VER_LEN] = {0}; ++ char mip_name[FW_VER_LEN] = {0}; ++ char nsp_version[FW_VER_LEN] = {0}; ++ char vnic_version[FW_VER_LEN] = {0}; + + if (fw_size < FW_VER_LEN) + return FW_VER_LEN; + + hw = nfp_net_get_hw(dev); + +- if ((dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR) != 0) { ++ if (hw->fw_version[0] != 0) { ++ snprintf(fw_version, FW_VER_LEN, "%s", hw->fw_version); ++ return 0; ++ } ++ ++ if ((dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR) == 0) { + snprintf(vnic_version, FW_VER_LEN, "%d.%d.%d.%d", + hw->ver.extend, hw->ver.class, + hw->ver.major, hw->ver.minor); +@@ -2084,8 +2087,16 @@ nfp_net_firmware_version_get(struct rte_eth_dev *dev, + nfp_net_get_mip_name(hw, mip_name); + nfp_net_get_app_name(hw, app_name); + +- snprintf(fw_version, FW_VER_LEN, "%s %s %s %s", ++ if (nsp_version[0] == 0 || mip_name[0] == 0) { ++ snprintf(fw_version, FW_VER_LEN, "%s %s %s %s", + vnic_version, nsp_version, mip_name, app_name); ++ return 0; ++ } ++ ++ snprintf(hw->fw_version, FW_VER_LEN, "%s %s %s %s", ++ vnic_version, nsp_version, mip_name, app_name); ++ ++ snprintf(fw_version, FW_VER_LEN, "%s", hw->fw_version); + + return 0; + } +@@ -2249,3 +2260,13 @@ nfp_net_flow_ctrl_set(struct rte_eth_dev *dev, + + return 0; + } ++ ++uint32_t ++nfp_net_get_port_num(struct nfp_pf_dev *pf_dev, ++ struct nfp_eth_table *nfp_eth_table) ++{ ++ if (pf_dev->multi_pf.enabled) ++ return 1; ++ else ++ return nfp_eth_table->count; ++} +diff --git a/dpdk/drivers/net/nfp/nfp_net_common.h b/dpdk/drivers/net/nfp/nfp_net_common.h +index 30fea7ae02..41d59bfa99 100644 +--- a/dpdk/drivers/net/nfp/nfp_net_common.h ++++ b/dpdk/drivers/net/nfp/nfp_net_common.h +@@ -38,6 +38,9 @@ + + #define NFP_BEAT_LENGTH 8 + ++/* The length of firmware version string */ ++#define FW_VER_LEN 32 ++ + /* + * Each PF has corresponding word to beat: + * Offset | Usage +@@ -98,6 +101,9 @@ struct nfp_pf_dev { + + uint8_t *qc_bar; + ++ struct nfp_cpp_area *mac_stats_area; ++ uint8_t *mac_stats_bar; ++ + struct nfp_hwinfo *hwinfo; + struct nfp_rtsym_table *sym_tbl; + +@@ -165,8 +171,6 @@ struct nfp_net_hw { + + struct nfp_cpp *cpp; + struct nfp_cpp_area *ctrl_area; +- struct nfp_cpp_area *mac_stats_area; +- uint8_t *mac_stats_bar; + uint8_t *mac_stats; + + /** Sequential physical port number, only valid for CoreNIC firmware */ +@@ -177,6 +181,9 @@ struct nfp_net_hw { + struct nfp_net_tlv_caps tlv_caps; + + struct nfp_net_ipsec_data *ipsec_data; ++ ++ /** Used for firmware version */ ++ char fw_version[FW_VER_LEN]; + }; + + static inline uint32_t +@@ -272,6 +279,9 @@ int nfp_net_flow_ctrl_get(struct rte_eth_dev *dev, + struct rte_eth_fc_conf *fc_conf); + int nfp_net_flow_ctrl_set(struct rte_eth_dev *dev, + struct rte_eth_fc_conf *fc_conf); ++void nfp_pf_uninit(struct nfp_pf_dev *pf_dev); ++uint32_t nfp_net_get_port_num(struct nfp_pf_dev *pf_dev, ++ struct nfp_eth_table *nfp_eth_table); + + #define NFP_PRIV_TO_APP_FW_NIC(app_fw_priv)\ + ((struct nfp_app_fw_nic *)app_fw_priv) +diff --git a/dpdk/drivers/net/nfp/nfp_rxtx.c b/dpdk/drivers/net/nfp/nfp_rxtx.c +index f21e120a43..8ca651ba55 100644 +--- a/dpdk/drivers/net/nfp/nfp_rxtx.c ++++ b/dpdk/drivers/net/nfp/nfp_rxtx.c +@@ -747,15 +747,6 @@ nfp_net_recv_pkts(void *rx_queue, + /* Checking the checksum flag */ + nfp_net_rx_cksum(rxq, rxds, mb); + +- if (meta.port_id == 0) { +- rx_pkts[avail++] = mb; +- } else if (nfp_flower_pf_dispatch_pkts(hw, mb, meta.port_id)) { +- avail_multiplexed++; +- } else { +- rte_pktmbuf_free(mb); +- break; +- } +- + /* Now resetting and updating the descriptor */ + rxds->vals[0] = 0; + rxds->vals[1] = 0; +@@ -768,6 +759,15 @@ nfp_net_recv_pkts(void *rx_queue, + rxq->rd_p++; + if (unlikely(rxq->rd_p == rxq->rx_count)) /* Wrapping */ + rxq->rd_p = 0; ++ ++ if (meta.port_id == 0) { ++ rx_pkts[avail++] = mb; ++ } else if (nfp_flower_pf_dispatch_pkts(hw, mb, meta.port_id)) { ++ avail_multiplexed++; ++ } else { ++ rte_pktmbuf_free(mb); ++ break; ++ } + } + + if (nb_hold == 0) +diff --git a/dpdk/drivers/net/nfp/nfpcore/nfp6000_pcie.c b/dpdk/drivers/net/nfp/nfpcore/nfp6000_pcie.c +index a6fd89b6c8..ef1ffd6d01 100644 +--- a/dpdk/drivers/net/nfp/nfpcore/nfp6000_pcie.c ++++ b/dpdk/drivers/net/nfp/nfpcore/nfp6000_pcie.c +@@ -263,19 +263,6 @@ nfp_bitsize_calc(uint64_t mask) + return bit_size; + } + +-static int +-nfp_cmp_bars(const void *ptr_a, +- const void *ptr_b) +-{ +- const struct nfp_bar *a = ptr_a; +- const struct nfp_bar *b = ptr_b; +- +- if (a->bitsize == b->bitsize) +- return a->index - b->index; +- else +- return a->bitsize - b->bitsize; +-} +- + static bool + nfp_bars_for_secondary(uint32_t index) + { +@@ -383,9 +370,6 @@ nfp_enable_bars(struct nfp_pcie_user *nfp) + if (nfp_bar_write(nfp, bar, barcfg_msix_general) < 0) + return -EIO; + +- /* Sort bars by bit size - use the smallest possible first. */ +- qsort(&nfp->bar[0], nfp->bars, sizeof(nfp->bar[0]), nfp_cmp_bars); +- + return 0; + } + +@@ -466,16 +450,18 @@ find_matching_bar(struct nfp_pcie_user *nfp, + int width) + { + uint32_t n; ++ uint32_t index; + +- for (n = 0; n < nfp->bars; n++) { +- struct nfp_bar *bar = &nfp->bar[n]; ++ for (n = RTE_DIM(nfp->bar) ; n > 0; n--) { ++ index = n - 1; ++ struct nfp_bar *bar = &nfp->bar[index]; + + if (bar->lock) + continue; + + if (matching_bar_exist(bar, target, action, token, + offset, size, width)) +- return n; ++ return index; + } + + return -1; +@@ -493,10 +479,12 @@ find_unused_bar_noblock(struct nfp_pcie_user *nfp, + { + int ret; + uint32_t n; ++ uint32_t index; + const struct nfp_bar *bar; + +- for (n = 0; n < nfp->bars; n++) { +- bar = &nfp->bar[n]; ++ for (n = RTE_DIM(nfp->bar); n > 0; n--) { ++ index = n - 1; ++ bar = &nfp->bar[index]; + + if (bar->bitsize == 0) + continue; +@@ -508,7 +496,7 @@ find_unused_bar_noblock(struct nfp_pcie_user *nfp, + continue; + + if (!bar->lock) +- return n; ++ return index; + } + + return -EAGAIN; +@@ -561,7 +549,7 @@ nfp_disable_bars(struct nfp_pcie_user *nfp) + uint32_t i; + struct nfp_bar *bar; + +- for (i = 0; i < nfp->bars; i++) { ++ for (i = 0; i < RTE_DIM(nfp->bar); i++) { + bar = &nfp->bar[i]; + if (bar->iomem != NULL) { + bar->iomem = NULL; +diff --git a/dpdk/drivers/net/nfp/nfpcore/nfp_mutex.c b/dpdk/drivers/net/nfp/nfpcore/nfp_mutex.c +index 3c10c7a090..edb78dfdc9 100644 +--- a/dpdk/drivers/net/nfp/nfpcore/nfp_mutex.c ++++ b/dpdk/drivers/net/nfp/nfpcore/nfp_mutex.c +@@ -168,7 +168,7 @@ nfp_cpp_mutex_alloc(struct nfp_cpp *cpp, + if (tmp != key) + return NULL; + +- mutex = calloc(sizeof(*mutex), 1); ++ mutex = calloc(1, sizeof(*mutex)); + if (mutex == NULL) + return NULL; + +diff --git a/dpdk/drivers/net/ngbe/base/ngbe_devids.h b/dpdk/drivers/net/ngbe/base/ngbe_devids.h +index 83eedf423e..e1efa62015 100644 +--- a/dpdk/drivers/net/ngbe/base/ngbe_devids.h ++++ b/dpdk/drivers/net/ngbe/base/ngbe_devids.h +@@ -83,6 +83,7 @@ + #define NGBE_YT8521S_SFP_GPIO 0x0062 + #define NGBE_INTERNAL_YT8521S_SFP_GPIO 0x0064 + #define NGBE_LY_YT8521S_SFP 0x0070 ++#define NGBE_RGMII_FPGA 0x0080 + #define NGBE_WOL_SUP 0x4000 + #define NGBE_NCSI_SUP 0x8000 + +diff --git a/dpdk/drivers/net/ngbe/base/ngbe_hw.c b/dpdk/drivers/net/ngbe/base/ngbe_hw.c +index 22ccdb0b7d..4dced0d328 100644 +--- a/dpdk/drivers/net/ngbe/base/ngbe_hw.c ++++ b/dpdk/drivers/net/ngbe/base/ngbe_hw.c +@@ -173,6 +173,9 @@ s32 ngbe_reset_hw_em(struct ngbe_hw *hw) + ngbe_reset_misc_em(hw); + hw->mac.clear_hw_cntrs(hw); + ++ if (!((hw->sub_device_id & NGBE_OEM_MASK) == NGBE_RGMII_FPGA)) ++ hw->phy.set_phy_power(hw, false); ++ + msec_delay(50); + + /* Store the permanent mac address */ +diff --git a/dpdk/drivers/net/ngbe/base/ngbe_phy_yt.c b/dpdk/drivers/net/ngbe/base/ngbe_phy_yt.c +index ea313cd9a5..a374b015fd 100644 +--- a/dpdk/drivers/net/ngbe/base/ngbe_phy_yt.c ++++ b/dpdk/drivers/net/ngbe/base/ngbe_phy_yt.c +@@ -320,6 +320,10 @@ skip_an_fiber: + value |= value_r4; + ngbe_write_phy_reg_mdi(hw, YT_ANA, 0, value); + ++ /* config for yt8531sh-ca */ ++ ngbe_write_phy_reg_ext_yt(hw, YT_SPEC_CONF, 0, ++ YT_SPEC_CONF_8531SH_CA); ++ + /* software reset to make the above configuration + * take effect + */ +diff --git a/dpdk/drivers/net/ngbe/base/ngbe_phy_yt.h b/dpdk/drivers/net/ngbe/base/ngbe_phy_yt.h +index ddf992e79a..c45bec7ce7 100644 +--- a/dpdk/drivers/net/ngbe/base/ngbe_phy_yt.h ++++ b/dpdk/drivers/net/ngbe/base/ngbe_phy_yt.h +@@ -32,6 +32,8 @@ + #define YT_MISC 0xA006 + #define YT_MISC_FIBER_PRIO MS16(8, 0x1) /* 0 for UTP */ + #define YT_MISC_RESV MS16(0, 0x1) ++#define YT_SPEC_CONF 0xA023 ++#define YT_SPEC_CONF_8531SH_CA 0x4031 + + /* SDS EXT */ + #define YT_AUTO 0xA5 +diff --git a/dpdk/drivers/net/ngbe/ngbe_ethdev.c b/dpdk/drivers/net/ngbe/ngbe_ethdev.c +index 478da014b2..fb86e7b10d 100644 +--- a/dpdk/drivers/net/ngbe/ngbe_ethdev.c ++++ b/dpdk/drivers/net/ngbe/ngbe_ethdev.c +@@ -546,7 +546,7 @@ static int eth_ngbe_pci_remove(struct rte_pci_device *pci_dev) + if (ethdev == NULL) + return 0; + +- return rte_eth_dev_destroy(ethdev, eth_ngbe_dev_uninit); ++ return rte_eth_dev_pci_generic_remove(pci_dev, eth_ngbe_dev_uninit); + } + + static struct rte_pci_driver rte_ngbe_pmd = { +@@ -1811,7 +1811,9 @@ ngbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) + dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues; + dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues; + dev_info->min_rx_bufsize = 1024; +- dev_info->max_rx_pktlen = 15872; ++ dev_info->max_rx_pktlen = NGBE_MAX_MTU + NGBE_ETH_OVERHEAD; ++ dev_info->min_mtu = RTE_ETHER_MIN_MTU; ++ dev_info->max_mtu = NGBE_MAX_MTU; + dev_info->max_mac_addrs = hw->mac.num_rar_entries; + dev_info->max_hash_mac_addrs = NGBE_VMDQ_NUM_UC_MAC; + dev_info->max_vfs = pci_dev->max_vfs; +diff --git a/dpdk/drivers/net/ngbe/ngbe_ethdev.h b/dpdk/drivers/net/ngbe/ngbe_ethdev.h +index 3cde7c8750..9b43d5f20e 100644 +--- a/dpdk/drivers/net/ngbe/ngbe_ethdev.h ++++ b/dpdk/drivers/net/ngbe/ngbe_ethdev.h +@@ -32,6 +32,7 @@ + + #define NGBE_QUEUE_ITR_INTERVAL_DEFAULT 500 /* 500us */ + ++#define NGBE_MAX_MTU 9414 + /* The overhead from MTU to max frame size. */ + #define NGBE_ETH_OVERHEAD (RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN) + +diff --git a/dpdk/drivers/net/ngbe/ngbe_rxtx.c b/dpdk/drivers/net/ngbe/ngbe_rxtx.c +index 8a873b858e..4680ff91f1 100644 +--- a/dpdk/drivers/net/ngbe/ngbe_rxtx.c ++++ b/dpdk/drivers/net/ngbe/ngbe_rxtx.c +@@ -1791,6 +1791,7 @@ ngbe_tx_queue_release(struct ngbe_tx_queue *txq) + if (txq->ops != NULL) { + txq->ops->release_mbufs(txq); + txq->ops->free_swring(txq); ++ rte_memzone_free(txq->mz); + } + rte_free(txq); + } +@@ -1995,6 +1996,7 @@ ngbe_dev_tx_queue_setup(struct rte_eth_dev *dev, + return -ENOMEM; + } + ++ txq->mz = tz; + txq->nb_tx_desc = nb_desc; + txq->tx_free_thresh = tx_free_thresh; + txq->pthresh = tx_conf->tx_thresh.pthresh; +@@ -2097,6 +2099,7 @@ ngbe_rx_queue_release(struct ngbe_rx_queue *rxq) + ngbe_rx_queue_release_mbufs(rxq); + rte_free(rxq->sw_ring); + rte_free(rxq->sw_sc_ring); ++ rte_memzone_free(rxq->mz); + rte_free(rxq); + } + } +@@ -2187,6 +2190,7 @@ ngbe_reset_rx_queue(struct ngbe_adapter *adapter, struct ngbe_rx_queue *rxq) + rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1); + rxq->rx_tail = 0; + rxq->nb_rx_hold = 0; ++ rte_pktmbuf_free(rxq->pkt_first_seg); + rxq->pkt_first_seg = NULL; + rxq->pkt_last_seg = NULL; + } +@@ -2277,6 +2281,7 @@ ngbe_dev_rx_queue_setup(struct rte_eth_dev *dev, + return -ENOMEM; + } + ++ rxq->mz = rz; + /* + * Zero init all the descriptors in the ring. + */ +diff --git a/dpdk/drivers/net/ngbe/ngbe_rxtx.h b/dpdk/drivers/net/ngbe/ngbe_rxtx.h +index 9130f9d0df..2914b9a756 100644 +--- a/dpdk/drivers/net/ngbe/ngbe_rxtx.h ++++ b/dpdk/drivers/net/ngbe/ngbe_rxtx.h +@@ -276,6 +276,7 @@ struct ngbe_rx_queue { + struct rte_mbuf fake_mbuf; + /** hold packets to return to application */ + struct rte_mbuf *rx_stage[RTE_PMD_NGBE_RX_MAX_BURST * 2]; ++ const struct rte_memzone *mz; + }; + + /** +@@ -353,6 +354,7 @@ struct ngbe_tx_queue { + uint8_t tx_deferred_start; /**< not in global dev start */ + + const struct ngbe_txq_ops *ops; /**< txq ops */ ++ const struct rte_memzone *mz; + }; + + struct ngbe_txq_ops { +diff --git a/dpdk/drivers/net/pfe/pfe_ethdev.c b/dpdk/drivers/net/pfe/pfe_ethdev.c +index 551f3cf193..0073dd7405 100644 +--- a/dpdk/drivers/net/pfe/pfe_ethdev.c ++++ b/dpdk/drivers/net/pfe/pfe_ethdev.c +@@ -520,7 +520,8 @@ pfe_supported_ptypes_get(struct rte_eth_dev *dev) + RTE_PTYPE_L3_IPV6_EXT, + RTE_PTYPE_L4_TCP, + RTE_PTYPE_L4_UDP, +- RTE_PTYPE_L4_SCTP ++ RTE_PTYPE_L4_SCTP, ++ RTE_PTYPE_UNKNOWN + }; + + if (dev->rx_pkt_burst == pfe_recv_pkts || +diff --git a/dpdk/drivers/net/sfc/sfc_mae.c b/dpdk/drivers/net/sfc/sfc_mae.c +index e5ec0ae49d..60ff6d2181 100644 +--- a/dpdk/drivers/net/sfc/sfc_mae.c ++++ b/dpdk/drivers/net/sfc/sfc_mae.c +@@ -1350,8 +1350,8 @@ sfc_mae_action_set_list_add(struct sfc_adapter *sa, + + action_set_list->action_sets = + rte_calloc("sfc_mae_action_set_list_action_sets", +- sizeof(struct sfc_mae_action_set *), +- action_set_list->nb_action_sets, 0); ++ action_set_list->nb_action_sets, ++ sizeof(struct sfc_mae_action_set *), 0); + if (action_set_list->action_sets == NULL) { + sfc_err(sa, "failed to allocate action set list"); + rte_free(action_set_list); +diff --git a/dpdk/drivers/net/softnic/rte_eth_softnic_cli.c b/dpdk/drivers/net/softnic/rte_eth_softnic_cli.c +index 085523fe03..95e705c553 100644 +--- a/dpdk/drivers/net/softnic/rte_eth_softnic_cli.c ++++ b/dpdk/drivers/net/softnic/rte_eth_softnic_cli.c +@@ -337,6 +337,7 @@ cmd_softnic_pipeline_libbuild(struct pmd_internals *softnic __rte_unused, + "-I %s/lib/eal/include " + "-I %s/lib/eal/x86/include " + "-I %s/lib/eal/include/generic " ++ "-I %s/lib/log " + "-I %s/lib/meter " + "-I %s/lib/port " + "-I %s/lib/table " +@@ -361,6 +362,7 @@ cmd_softnic_pipeline_libbuild(struct pmd_internals *softnic __rte_unused, + install_dir, + install_dir, + install_dir, ++ install_dir, + log_file, + obj_file, + lib_file, +diff --git a/dpdk/drivers/net/tap/rte_eth_tap.c b/dpdk/drivers/net/tap/rte_eth_tap.c +index b41fa971cb..3fa03cdbee 100644 +--- a/dpdk/drivers/net/tap/rte_eth_tap.c ++++ b/dpdk/drivers/net/tap/rte_eth_tap.c +@@ -1803,6 +1803,7 @@ tap_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused) + RTE_PTYPE_L4_UDP, + RTE_PTYPE_L4_TCP, + RTE_PTYPE_L4_SCTP, ++ RTE_PTYPE_UNKNOWN + }; + + return ptypes; +diff --git a/dpdk/drivers/net/tap/tap_flow.c b/dpdk/drivers/net/tap/tap_flow.c +index ed4d42f92f..79cd6a12ca 100644 +--- a/dpdk/drivers/net/tap/tap_flow.c ++++ b/dpdk/drivers/net/tap/tap_flow.c +@@ -11,6 +11,7 @@ + + #include + #include ++#include + #include + #include + #include +@@ -1082,8 +1083,11 @@ priv_flow_process(struct pmd_internals *pmd, + } + /* use flower filter type */ + tap_nlattr_add(&flow->msg.nh, TCA_KIND, sizeof("flower"), "flower"); +- if (tap_nlattr_nested_start(&flow->msg, TCA_OPTIONS) < 0) +- goto exit_item_not_supported; ++ if (tap_nlattr_nested_start(&flow->msg, TCA_OPTIONS) < 0) { ++ rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_ACTION, ++ actions, "could not allocated netlink msg"); ++ goto exit_return_error; ++ } + } + for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) { + const struct tap_flow_items *token = NULL; +@@ -1199,9 +1203,12 @@ actions: + if (action) + goto exit_action_not_supported; + action = 1; +- if (!queue || +- (queue->index > pmd->dev->data->nb_rx_queues - 1)) +- goto exit_action_not_supported; ++ if (queue->index >= pmd->dev->data->nb_rx_queues) { ++ rte_flow_error_set(error, ERANGE, ++ RTE_FLOW_ERROR_TYPE_ACTION, actions, ++ "queue index out of range"); ++ goto exit_return_error; ++ } + if (flow) { + struct action_data adata = { + .id = "skbedit", +@@ -1227,7 +1234,7 @@ actions: + if (!pmd->rss_enabled) { + err = rss_enable(pmd, attr, error); + if (err) +- goto exit_action_not_supported; ++ goto exit_return_error; + } + if (flow) + err = rss_add_actions(flow, pmd, rss, error); +@@ -1235,7 +1242,7 @@ actions: + goto exit_action_not_supported; + } + if (err) +- goto exit_action_not_supported; ++ goto exit_return_error; + } + /* When fate is unknown, drop traffic. */ + if (!action) { +@@ -1258,6 +1265,7 @@ exit_item_not_supported: + exit_action_not_supported: + rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, + actions, "action not supported"); ++exit_return_error: + return -rte_errno; + } + +@@ -1290,9 +1298,7 @@ tap_flow_validate(struct rte_eth_dev *dev, + * In those rules, the handle (uint32_t) is the part that would identify + * specifically each rule. + * +- * On 32-bit architectures, the handle can simply be the flow's pointer address. +- * On 64-bit architectures, we rely on jhash(flow) to find a (sufficiently) +- * unique handle. ++ * Use jhash of the flow pointer to make a unique handle. + * + * @param[in, out] flow + * The flow that needs its handle set. +@@ -1302,16 +1308,18 @@ tap_flow_set_handle(struct rte_flow *flow) + { + union { + struct rte_flow *flow; +- const void *key; +- } tmp; +- uint32_t handle = 0; ++ uint32_t words[sizeof(flow) / sizeof(uint32_t)]; ++ } tmp = { ++ .flow = flow, ++ }; ++ uint32_t handle; ++ static uint64_t hash_seed; + +- tmp.flow = flow; ++ if (hash_seed == 0) ++ hash_seed = rte_rand(); ++ ++ handle = rte_jhash_32b(tmp.words, sizeof(flow) / sizeof(uint32_t), hash_seed); + +- if (sizeof(flow) > 4) +- handle = rte_jhash(tmp.key, sizeof(flow), 1); +- else +- handle = (uintptr_t)flow; + /* must be at least 1 to avoid letting the kernel choose one for us */ + if (!handle) + handle = 1; +@@ -1587,7 +1595,7 @@ tap_flow_isolate(struct rte_eth_dev *dev, + * If netdevice is there, setup appropriate flow rules immediately. + * Otherwise it will be set when bringing up the netdevice (tun_alloc). + */ +- if (!process_private->rxq_fds[0]) ++ if (process_private->rxq_fds[0] == -1) + return 0; + if (set) { + struct rte_flow *remote_flow; +diff --git a/dpdk/drivers/net/thunderx/base/nicvf_mbox.c b/dpdk/drivers/net/thunderx/base/nicvf_mbox.c +index 5993eec4e6..0e0176974d 100644 +--- a/dpdk/drivers/net/thunderx/base/nicvf_mbox.c ++++ b/dpdk/drivers/net/thunderx/base/nicvf_mbox.c +@@ -485,3 +485,15 @@ nicvf_mbox_reset_xcast(struct nicvf *nic) + mbx.msg.msg = NIC_MBOX_MSG_RESET_XCAST; + nicvf_mbox_send_msg_to_pf(nic, &mbx); + } ++ ++int ++nicvf_mbox_set_xcast(struct nicvf *nic, uint8_t mode, uint64_t mac) ++{ ++ struct nic_mbx mbx = { .msg = { 0 } }; ++ ++ mbx.xcast.msg = NIC_MBOX_MSG_SET_XCAST; ++ mbx.xcast.mode = mode; ++ mbx.xcast.mac = mac; ++ ++ return nicvf_mbox_send_msg_to_pf(nic, &mbx); ++} +diff --git a/dpdk/drivers/net/thunderx/base/nicvf_mbox.h b/dpdk/drivers/net/thunderx/base/nicvf_mbox.h +index 322c8159cb..47f3d13755 100644 +--- a/dpdk/drivers/net/thunderx/base/nicvf_mbox.h ++++ b/dpdk/drivers/net/thunderx/base/nicvf_mbox.h +@@ -45,6 +45,8 @@ + #define NIC_MBOX_MSG_CFG_DONE 0xF0 /* VF configuration done */ + #define NIC_MBOX_MSG_SHUTDOWN 0xF1 /* VF is being shutdown */ + #define NIC_MBOX_MSG_RESET_XCAST 0xF2 /* Reset DCAM filtering mode */ ++#define NIC_MBOX_MSG_ADD_MCAST 0xF3 /* ADD MAC to DCAM filters */ ++#define NIC_MBOX_MSG_SET_XCAST 0xF4 /* Set MCAST/BCAST Rx mode */ + #define NIC_MBOX_MSG_MAX 0x100 /* Maximum number of messages */ + + /* Get vNIC VF configuration */ +@@ -190,6 +192,12 @@ struct change_link_mode_msg { + + }; + ++struct xcast { ++ uint8_t msg; ++ uint8_t mode; ++ uint64_t mac:48; ++}; ++ + struct nic_mbx { + /* 128 bit shared memory between PF and each VF */ + union { +@@ -209,6 +217,7 @@ union { + struct reset_stat_cfg reset_stat; + struct set_link_state set_link; + struct change_link_mode_msg mode; ++ struct xcast xcast; + }; + }; + +@@ -239,5 +248,6 @@ void nicvf_mbox_cfg_done(struct nicvf *nic); + void nicvf_mbox_link_change(struct nicvf *nic); + void nicvf_mbox_reset_xcast(struct nicvf *nic); + int nicvf_mbox_change_mode(struct nicvf *nic, struct change_link_mode *cfg); ++int nicvf_mbox_set_xcast(struct nicvf *nic, uint8_t mode, uint64_t mac); + + #endif /* __THUNDERX_NICVF_MBOX__ */ +diff --git a/dpdk/drivers/net/thunderx/nicvf_ethdev.c b/dpdk/drivers/net/thunderx/nicvf_ethdev.c +index a504d41dfe..ba2ef4058e 100644 +--- a/dpdk/drivers/net/thunderx/nicvf_ethdev.c ++++ b/dpdk/drivers/net/thunderx/nicvf_ethdev.c +@@ -58,6 +58,10 @@ RTE_LOG_REGISTER_SUFFIX(nicvf_logtype_driver, driver, NOTICE); + #define NICVF_QLM_MODE_SGMII 7 + #define NICVF_QLM_MODE_XFI 12 + ++#define BCAST_ACCEPT 0x01 ++#define CAM_ACCEPT (1 << 3) ++#define BGX_MCAST_MODE(x) ((x) << 1) ++ + enum nicvf_link_speed { + NICVF_LINK_SPEED_SGMII, + NICVF_LINK_SPEED_XAUI, +@@ -392,12 +396,14 @@ nicvf_dev_supported_ptypes_get(struct rte_eth_dev *dev) + RTE_PTYPE_L4_TCP, + RTE_PTYPE_L4_UDP, + RTE_PTYPE_L4_FRAG, ++ RTE_PTYPE_UNKNOWN + }; + static const uint32_t ptypes_tunnel[] = { + RTE_PTYPE_TUNNEL_GRE, + RTE_PTYPE_TUNNEL_GENEVE, + RTE_PTYPE_TUNNEL_VXLAN, + RTE_PTYPE_TUNNEL_NVGRE, ++ RTE_PTYPE_UNKNOWN + }; + static const uint32_t ptypes_end = RTE_PTYPE_UNKNOWN; + +@@ -2183,9 +2189,22 @@ nicvf_eth_dev_uninit(struct rte_eth_dev *dev) + nicvf_dev_close(dev); + return 0; + } ++ ++static inline uint64_t ether_addr_to_u64(uint8_t *addr) ++{ ++ uint64_t u = 0; ++ int i; ++ ++ for (i = 0; i < RTE_ETHER_ADDR_LEN; i++) ++ u = u << 8 | addr[i]; ++ ++ return u; ++} ++ + static int + nicvf_eth_dev_init(struct rte_eth_dev *eth_dev) + { ++ uint8_t dmac_ctrl_reg = 0; + int ret; + struct rte_pci_device *pci_dev; + struct nicvf *nic = nicvf_pmd_priv(eth_dev); +@@ -2309,6 +2328,15 @@ nicvf_eth_dev_init(struct rte_eth_dev *eth_dev) + goto malloc_fail; + } + ++ /* set DMAC CTRL reg to allow MAC */ ++ dmac_ctrl_reg = BCAST_ACCEPT | BGX_MCAST_MODE(2) | CAM_ACCEPT; ++ ret = nicvf_mbox_set_xcast(nic, dmac_ctrl_reg, ++ ether_addr_to_u64(nic->mac_addr)); ++ if (ret) { ++ PMD_INIT_LOG(ERR, "Failed to set mac addr"); ++ goto malloc_fail; ++ } ++ + ret = nicvf_set_first_skip(eth_dev); + if (ret) { + PMD_INIT_LOG(ERR, "Failed to configure first skip"); +diff --git a/dpdk/drivers/net/txgbe/base/meson.build b/dpdk/drivers/net/txgbe/base/meson.build +index a81d6890fe..4cf90a394a 100644 +--- a/dpdk/drivers/net/txgbe/base/meson.build ++++ b/dpdk/drivers/net/txgbe/base/meson.build +@@ -22,6 +22,6 @@ foreach flag: error_cflags + endforeach + + base_lib = static_library('txgbe_base', sources, +- dependencies: [static_rte_eal, static_rte_net], ++ dependencies: [static_rte_eal, static_rte_net, static_rte_bus_pci], + c_args: c_args) + base_objs = base_lib.extract_all_objects(recursive: true) +diff --git a/dpdk/drivers/net/txgbe/base/txgbe_hw.c b/dpdk/drivers/net/txgbe/base/txgbe_hw.c +index d19fd0065d..7094551fee 100644 +--- a/dpdk/drivers/net/txgbe/base/txgbe_hw.c ++++ b/dpdk/drivers/net/txgbe/base/txgbe_hw.c +@@ -462,7 +462,7 @@ void txgbe_set_lan_id_multi_port(struct txgbe_hw *hw) + **/ + s32 txgbe_stop_hw(struct txgbe_hw *hw) + { +- u32 reg_val; ++ s32 status = 0; + u16 i; + + /* +@@ -484,16 +484,26 @@ s32 txgbe_stop_hw(struct txgbe_hw *hw) + wr32(hw, TXGBE_ICR(0), TXGBE_ICR_MASK); + wr32(hw, TXGBE_ICR(1), TXGBE_ICR_MASK); + +- /* Disable the transmit unit. Each queue must be disabled. */ +- for (i = 0; i < hw->mac.max_tx_queues; i++) +- wr32(hw, TXGBE_TXCFG(i), TXGBE_TXCFG_FLUSH); ++ wr32(hw, TXGBE_BMECTL, 0x3); + + /* Disable the receive unit by stopping each queue */ +- for (i = 0; i < hw->mac.max_rx_queues; i++) { +- reg_val = rd32(hw, TXGBE_RXCFG(i)); +- reg_val &= ~TXGBE_RXCFG_ENA; +- wr32(hw, TXGBE_RXCFG(i), reg_val); +- } ++ for (i = 0; i < hw->mac.max_rx_queues; i++) ++ wr32(hw, TXGBE_RXCFG(i), 0); ++ ++ /* flush all queues disables */ ++ txgbe_flush(hw); ++ msec_delay(2); ++ ++ /* Prevent the PCI-E bus from hanging by disabling PCI-E master ++ * access and verify no pending requests ++ */ ++ status = txgbe_set_pcie_master(hw, false); ++ if (status) ++ return status; ++ ++ /* Disable the transmit unit. Each queue must be disabled. */ ++ for (i = 0; i < hw->mac.max_tx_queues; i++) ++ wr32(hw, TXGBE_TXCFG(i), 0); + + /* flush all queues disables */ + txgbe_flush(hw); +@@ -1174,6 +1184,38 @@ out: + } + } + ++s32 txgbe_set_pcie_master(struct txgbe_hw *hw, bool enable) ++{ ++ struct rte_pci_device *pci_dev = (struct rte_pci_device *)hw->back; ++ s32 status = 0; ++ u32 i; ++ ++ if (rte_pci_set_bus_master(pci_dev, enable) < 0) { ++ DEBUGOUT("Cannot configure PCI bus master."); ++ return -1; ++ } ++ ++ if (enable) ++ goto out; ++ ++ /* Exit if master requests are blocked */ ++ if (!(rd32(hw, TXGBE_BMEPEND))) ++ goto out; ++ ++ /* Poll for master request bit to clear */ ++ for (i = 0; i < TXGBE_PCI_MASTER_DISABLE_TIMEOUT; i++) { ++ usec_delay(100); ++ if (!(rd32(hw, TXGBE_BMEPEND))) ++ goto out; ++ } ++ ++ DEBUGOUT("PCIe transaction pending bit also did not clear."); ++ status = TXGBE_ERR_MASTER_REQUESTS_PENDING; ++ ++out: ++ return status; ++} ++ + /** + * txgbe_acquire_swfw_sync - Acquire SWFW semaphore + * @hw: pointer to hardware structure +diff --git a/dpdk/drivers/net/txgbe/base/txgbe_hw.h b/dpdk/drivers/net/txgbe/base/txgbe_hw.h +index 7031589f7c..4bf9da2d4c 100644 +--- a/dpdk/drivers/net/txgbe/base/txgbe_hw.h ++++ b/dpdk/drivers/net/txgbe/base/txgbe_hw.h +@@ -40,6 +40,7 @@ s32 txgbe_setup_fc(struct txgbe_hw *hw); + s32 txgbe_validate_mac_addr(u8 *mac_addr); + s32 txgbe_acquire_swfw_sync(struct txgbe_hw *hw, u32 mask); + void txgbe_release_swfw_sync(struct txgbe_hw *hw, u32 mask); ++s32 txgbe_set_pcie_master(struct txgbe_hw *hw, bool enable); + + s32 txgbe_get_san_mac_addr(struct txgbe_hw *hw, u8 *san_mac_addr); + s32 txgbe_set_san_mac_addr(struct txgbe_hw *hw, u8 *san_mac_addr); +diff --git a/dpdk/drivers/net/txgbe/base/txgbe_osdep.h b/dpdk/drivers/net/txgbe/base/txgbe_osdep.h +index b62c0b0824..0d9492c3cb 100644 +--- a/dpdk/drivers/net/txgbe/base/txgbe_osdep.h ++++ b/dpdk/drivers/net/txgbe/base/txgbe_osdep.h +@@ -19,6 +19,7 @@ + #include + #include + #include ++#include + + #include "../txgbe_logs.h" + +diff --git a/dpdk/drivers/net/txgbe/base/txgbe_regs.h b/dpdk/drivers/net/txgbe/base/txgbe_regs.h +index 79290a7afe..a2984f1106 100644 +--- a/dpdk/drivers/net/txgbe/base/txgbe_regs.h ++++ b/dpdk/drivers/net/txgbe/base/txgbe_regs.h +@@ -1022,6 +1022,8 @@ enum txgbe_5tuple_protocol { + #define TXGBE_MACRXFLT_CTL_PASS LS(3, 6, 0x3) + #define TXGBE_MACRXFLT_RXALL MS(31, 0x1) + ++#define TXGBE_MAC_WDG_TIMEOUT 0x01100C ++ + /****************************************************************************** + * Statistic Registers + ******************************************************************************/ +@@ -1236,6 +1238,9 @@ enum txgbe_5tuple_protocol { + #define TXGBE_TCPTMR 0x000170 + #define TXGBE_ITRSEL 0x000180 + ++#define TXGBE_BMECTL 0x012020 ++#define TXGBE_BMEPEND 0x000168 ++ + /* P2V Mailbox */ + #define TXGBE_MBMEM(i) (0x005000 + 0x40 * (i)) /* 0-63 */ + #define TXGBE_MBCTL(i) (0x000600 + 4 * (i)) /* 0-63 */ +diff --git a/dpdk/drivers/net/txgbe/base/txgbe_type.h b/dpdk/drivers/net/txgbe/base/txgbe_type.h +index 75e839b7de..f52736cae9 100644 +--- a/dpdk/drivers/net/txgbe/base/txgbe_type.h ++++ b/dpdk/drivers/net/txgbe/base/txgbe_type.h +@@ -29,6 +29,7 @@ + #define TXGBE_FDIRCMD_CMD_POLL 10 + #define TXGBE_VF_INIT_TIMEOUT 200 /* Number of retries to clear RSTI */ + #define TXGBE_SPI_TIMEOUT 10000 ++#define TXGBE_PCI_MASTER_DISABLE_TIMEOUT 800 + + #define TXGBE_ALIGN 128 /* as intel did */ + +diff --git a/dpdk/drivers/net/txgbe/txgbe_ethdev.c b/dpdk/drivers/net/txgbe/txgbe_ethdev.c +index 6bc231a130..ad29c3cfec 100644 +--- a/dpdk/drivers/net/txgbe/txgbe_ethdev.c ++++ b/dpdk/drivers/net/txgbe/txgbe_ethdev.c +@@ -601,6 +601,7 @@ eth_txgbe_dev_init(struct rte_eth_dev *eth_dev, void *init_params __rte_unused) + hw->hw_addr = (void *)pci_dev->mem_resource[0].addr; + + /* Vendor and Device ID need to be set before init of shared code */ ++ hw->back = pci_dev; + hw->device_id = pci_dev->id.device_id; + hw->vendor_id = pci_dev->id.vendor_id; + if (pci_dev->id.subsystem_vendor_id == PCI_VENDOR_ID_WANGXUN) { +@@ -734,6 +735,8 @@ eth_txgbe_dev_init(struct rte_eth_dev *eth_dev, void *init_params __rte_unused) + PMD_INIT_LOG(ERR, + "Failed to allocate %d bytes needed to store MAC addresses", + RTE_ETHER_ADDR_LEN * TXGBE_VMDQ_NUM_UC_MAC); ++ rte_free(eth_dev->data->mac_addrs); ++ eth_dev->data->mac_addrs = NULL; + return -ENOMEM; + } + +@@ -901,6 +904,7 @@ static int txgbe_fdir_filter_init(struct rte_eth_dev *eth_dev) + if (!fdir_info->hash_map) { + PMD_INIT_LOG(ERR, + "Failed to allocate memory for fdir hash map!"); ++ rte_hash_free(fdir_info->hash_handle); + return -ENOMEM; + } + fdir_info->mask_added = FALSE; +@@ -936,6 +940,7 @@ static int txgbe_l2_tn_filter_init(struct rte_eth_dev *eth_dev) + if (!l2_tn_info->hash_map) { + PMD_INIT_LOG(ERR, + "Failed to allocate memory for L2 TN hash map!"); ++ rte_hash_free(l2_tn_info->hash_handle); + return -ENOMEM; + } + l2_tn_info->e_tag_en = FALSE; +@@ -963,7 +968,7 @@ static int eth_txgbe_pci_remove(struct rte_pci_device *pci_dev) + if (!ethdev) + return 0; + +- return rte_eth_dev_destroy(ethdev, eth_txgbe_dev_uninit); ++ return rte_eth_dev_pci_generic_remove(pci_dev, eth_txgbe_dev_uninit); + } + + static struct rte_pci_driver rte_txgbe_pmd = { +@@ -999,41 +1004,25 @@ txgbe_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) + } + + static void +-txgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on) ++txgbe_vlan_strip_q_set(struct rte_eth_dev *dev, uint16_t queue, int on) + { +- struct txgbe_hw *hw = TXGBE_DEV_HW(dev); +- struct txgbe_rx_queue *rxq; +- bool restart; +- uint32_t rxcfg, rxbal, rxbah; +- + if (on) + txgbe_vlan_hw_strip_enable(dev, queue); + else + txgbe_vlan_hw_strip_disable(dev, queue); ++} + +- rxq = dev->data->rx_queues[queue]; +- rxbal = rd32(hw, TXGBE_RXBAL(rxq->reg_idx)); +- rxbah = rd32(hw, TXGBE_RXBAH(rxq->reg_idx)); +- rxcfg = rd32(hw, TXGBE_RXCFG(rxq->reg_idx)); +- if (rxq->offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP) { +- restart = (rxcfg & TXGBE_RXCFG_ENA) && +- !(rxcfg & TXGBE_RXCFG_VLAN); +- rxcfg |= TXGBE_RXCFG_VLAN; +- } else { +- restart = (rxcfg & TXGBE_RXCFG_ENA) && +- (rxcfg & TXGBE_RXCFG_VLAN); +- rxcfg &= ~TXGBE_RXCFG_VLAN; +- } +- rxcfg &= ~TXGBE_RXCFG_ENA; ++static void ++txgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on) ++{ ++ struct txgbe_hw *hw = TXGBE_DEV_HW(dev); + +- if (restart) { +- /* set vlan strip for ring */ +- txgbe_dev_rx_queue_stop(dev, queue); +- wr32(hw, TXGBE_RXBAL(rxq->reg_idx), rxbal); +- wr32(hw, TXGBE_RXBAH(rxq->reg_idx), rxbah); +- wr32(hw, TXGBE_RXCFG(rxq->reg_idx), rxcfg); +- txgbe_dev_rx_queue_start(dev, queue); ++ if (!hw->adapter_stopped) { ++ PMD_DRV_LOG(ERR, "Please stop port first"); ++ return; + } ++ ++ txgbe_vlan_strip_q_set(dev, queue, on); + } + + static int +@@ -1258,9 +1247,9 @@ txgbe_vlan_hw_strip_config(struct rte_eth_dev *dev) + rxq = dev->data->rx_queues[i]; + + if (rxq->offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP) +- txgbe_vlan_strip_queue_set(dev, i, 1); ++ txgbe_vlan_strip_q_set(dev, i, 1); + else +- txgbe_vlan_strip_queue_set(dev, i, 0); ++ txgbe_vlan_strip_q_set(dev, i, 0); + } + } + +@@ -1322,6 +1311,13 @@ txgbe_vlan_offload_config(struct rte_eth_dev *dev, int mask) + static int + txgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask) + { ++ struct txgbe_hw *hw = TXGBE_DEV_HW(dev); ++ ++ if (!hw->adapter_stopped && (mask & RTE_ETH_VLAN_STRIP_MASK)) { ++ PMD_DRV_LOG(ERR, "Please stop port first"); ++ return -EPERM; ++ } ++ + txgbe_config_vlan_strip_on_all_queues(dev, mask); + + txgbe_vlan_offload_config(dev, mask); +@@ -1716,6 +1712,8 @@ txgbe_dev_start(struct rte_eth_dev *dev) + hw->mac.get_link_status = true; + hw->dev_start = true; + ++ txgbe_set_pcie_master(hw, true); ++ + /* workaround for GPIO intr lost when mng_veto bit is set */ + if (txgbe_check_reset_blocked(hw)) + txgbe_reinit_gpio_intr(hw); +@@ -1979,6 +1977,8 @@ txgbe_dev_stop(struct rte_eth_dev *dev) + adapter->rss_reta_updated = 0; + wr32m(hw, TXGBE_LEDCTL, 0xFFFFFFFF, TXGBE_LEDCTL_SEL_MASK); + ++ txgbe_set_pcie_master(hw, true); ++ + hw->adapter_stopped = true; + dev->data->dev_started = 0; + hw->dev_start = false; +@@ -2061,6 +2061,8 @@ txgbe_dev_close(struct rte_eth_dev *dev) + + txgbe_dev_free_queues(dev); + ++ txgbe_set_pcie_master(hw, false); ++ + /* reprogram the RAR[0] in case user changed it. */ + txgbe_set_rar(hw, 0, hw->mac.addr, 0, true); + +@@ -2671,7 +2673,9 @@ txgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) + dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues; + dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues; + dev_info->min_rx_bufsize = 1024; +- dev_info->max_rx_pktlen = 15872; ++ dev_info->max_rx_pktlen = TXGBE_MAX_MTU + TXGBE_ETH_OVERHEAD; ++ dev_info->min_mtu = RTE_ETHER_MIN_MTU; ++ dev_info->max_mtu = TXGBE_MAX_MTU; + dev_info->max_mac_addrs = hw->mac.num_rar_entries; + dev_info->max_hash_mac_addrs = TXGBE_VMDQ_NUM_UC_MAC; + dev_info->max_vfs = pci_dev->max_vfs; +@@ -2876,6 +2880,7 @@ txgbe_dev_link_update_share(struct rte_eth_dev *dev, + bool link_up; + int err; + int wait = 1; ++ u32 reg; + + memset(&link, 0, sizeof(link)); + link.link_status = RTE_ETH_LINK_DOWN; +@@ -2963,9 +2968,14 @@ txgbe_dev_link_update_share(struct rte_eth_dev *dev, + } + + /* Re configure MAC RX */ +- if (hw->mac.type == txgbe_mac_raptor) ++ if (hw->mac.type == txgbe_mac_raptor) { ++ reg = rd32(hw, TXGBE_MACRXCFG); ++ wr32(hw, TXGBE_MACRXCFG, reg); + wr32m(hw, TXGBE_MACRXFLT, TXGBE_MACRXFLT_PROMISC, + TXGBE_MACRXFLT_PROMISC); ++ reg = rd32(hw, TXGBE_MAC_WDG_TIMEOUT); ++ wr32(hw, TXGBE_MAC_WDG_TIMEOUT, reg); ++ } + + return rte_eth_linkstatus_set(dev, &link); + } +@@ -3683,12 +3693,8 @@ txgbe_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) + return -EINVAL; + } + +- if (hw->mode) +- wr32m(hw, TXGBE_FRMSZ, TXGBE_FRMSZ_MAX_MASK, +- TXGBE_FRAME_SIZE_MAX); +- else +- wr32m(hw, TXGBE_FRMSZ, TXGBE_FRMSZ_MAX_MASK, +- TXGBE_FRMSZ_MAX(frame_size)); ++ wr32m(hw, TXGBE_FRMSZ, TXGBE_FRMSZ_MAX_MASK, ++ TXGBE_FRMSZ_MAX(frame_size)); + + return 0; + } +@@ -3839,13 +3845,13 @@ txgbe_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) + struct txgbe_hw *hw = TXGBE_DEV_HW(dev); + + if (queue_id < 32) { +- mask = rd32(hw, TXGBE_IMS(0)); +- mask &= (1 << queue_id); +- wr32(hw, TXGBE_IMS(0), mask); ++ mask = rd32(hw, TXGBE_IMC(0)); ++ mask |= (1 << queue_id); ++ wr32(hw, TXGBE_IMC(0), mask); + } else if (queue_id < 64) { +- mask = rd32(hw, TXGBE_IMS(1)); +- mask &= (1 << (queue_id - 32)); +- wr32(hw, TXGBE_IMS(1), mask); ++ mask = rd32(hw, TXGBE_IMC(1)); ++ mask |= (1 << (queue_id - 32)); ++ wr32(hw, TXGBE_IMC(1), mask); + } + rte_intr_enable(intr_handle); + +@@ -3860,11 +3866,11 @@ txgbe_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) + + if (queue_id < 32) { + mask = rd32(hw, TXGBE_IMS(0)); +- mask &= ~(1 << queue_id); ++ mask |= (1 << queue_id); + wr32(hw, TXGBE_IMS(0), mask); + } else if (queue_id < 64) { + mask = rd32(hw, TXGBE_IMS(1)); +- mask &= ~(1 << (queue_id - 32)); ++ mask |= (1 << (queue_id - 32)); + wr32(hw, TXGBE_IMS(1), mask); + } + +@@ -3898,7 +3904,7 @@ txgbe_set_ivar_map(struct txgbe_hw *hw, int8_t direction, + wr32(hw, TXGBE_IVARMISC, tmp); + } else { + /* rx or tx causes */ +- /* Workaround for ICR lost */ ++ msix_vector |= TXGBE_IVAR_VLD; /* Workaround for ICR lost */ + idx = ((16 * (queue & 1)) + (8 * direction)); + tmp = rd32(hw, TXGBE_IVAR(queue >> 1)); + tmp &= ~(0xFF << idx); +@@ -4004,6 +4010,7 @@ txgbe_syn_filter_set(struct rte_eth_dev *dev, + struct txgbe_filter_info *filter_info = TXGBE_DEV_FILTER(dev); + uint32_t syn_info; + uint32_t synqf; ++ uint16_t queue; + + if (filter->queue >= TXGBE_MAX_RX_QUEUE_NUM) + return -EINVAL; +@@ -4013,7 +4020,11 @@ txgbe_syn_filter_set(struct rte_eth_dev *dev, + if (add) { + if (syn_info & TXGBE_SYNCLS_ENA) + return -EINVAL; +- synqf = (uint32_t)TXGBE_SYNCLS_QPID(filter->queue); ++ if (RTE_ETH_DEV_SRIOV(dev).active) ++ queue = RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + filter->queue; ++ else ++ queue = filter->queue; ++ synqf = (uint32_t)TXGBE_SYNCLS_QPID(queue); + synqf |= TXGBE_SYNCLS_ENA; + + if (filter->hig_pri) +@@ -4082,7 +4093,10 @@ txgbe_inject_5tuple_filter(struct rte_eth_dev *dev, + wr32(hw, TXGBE_5TFPORT(i), sdpqf); + wr32(hw, TXGBE_5TFCTL0(i), ftqf); + +- l34timir |= TXGBE_5TFCTL1_QP(filter->queue); ++ if (RTE_ETH_DEV_SRIOV(dev).active) ++ l34timir |= TXGBE_5TFCTL1_QP(RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + filter->queue); ++ else ++ l34timir |= TXGBE_5TFCTL1_QP(filter->queue); + wr32(hw, TXGBE_5TFCTL1(i), l34timir); + } + +@@ -4366,7 +4380,17 @@ txgbe_add_del_ethertype_filter(struct rte_eth_dev *dev, + if (add) { + etqf = TXGBE_ETFLT_ENA; + etqf |= TXGBE_ETFLT_ETID(filter->ether_type); +- etqs |= TXGBE_ETCLS_QPID(filter->queue); ++ if (RTE_ETH_DEV_SRIOV(dev).active) { ++ int pool, queue; ++ ++ pool = RTE_ETH_DEV_SRIOV(dev).def_vmdq_idx; ++ queue = RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + filter->queue; ++ etqf |= TXGBE_ETFLT_POOLENA; ++ etqf |= TXGBE_ETFLT_POOL(pool); ++ etqs |= TXGBE_ETCLS_QPID(queue); ++ } else { ++ etqs |= TXGBE_ETCLS_QPID(filter->queue); ++ } + etqs |= TXGBE_ETCLS_QENA; + + ethertype_filter.ethertype = filter->ether_type; +diff --git a/dpdk/drivers/net/txgbe/txgbe_ethdev.h b/dpdk/drivers/net/txgbe/txgbe_ethdev.h +index 7feb45d0cf..d5b58018a4 100644 +--- a/dpdk/drivers/net/txgbe/txgbe_ethdev.h ++++ b/dpdk/drivers/net/txgbe/txgbe_ethdev.h +@@ -56,7 +56,7 @@ + #define TXGBE_5TUPLE_MAX_PRI 7 + #define TXGBE_5TUPLE_MIN_PRI 1 + +- ++#define TXGBE_MAX_MTU 9414 + /* The overhead from MTU to max frame size. */ + #define TXGBE_ETH_OVERHEAD (RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN) + +diff --git a/dpdk/drivers/net/txgbe/txgbe_ethdev_vf.c b/dpdk/drivers/net/txgbe/txgbe_ethdev_vf.c +index f1341fbf7e..92603fccc2 100644 +--- a/dpdk/drivers/net/txgbe/txgbe_ethdev_vf.c ++++ b/dpdk/drivers/net/txgbe/txgbe_ethdev_vf.c +@@ -295,6 +295,8 @@ eth_txgbevf_dev_init(struct rte_eth_dev *eth_dev) + err = hw->mac.start_hw(hw); + if (err) { + PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", err); ++ rte_free(eth_dev->data->mac_addrs); ++ eth_dev->data->mac_addrs = NULL; + return -EIO; + } + +@@ -670,8 +672,10 @@ txgbevf_dev_start(struct rte_eth_dev *dev) + * now only one vector is used for Rx queue + */ + intr_vector = 1; +- if (rte_intr_efd_enable(intr_handle, intr_vector)) ++ if (rte_intr_efd_enable(intr_handle, intr_vector)) { ++ txgbe_dev_clear_queues(dev); + return -1; ++ } + } + + if (rte_intr_dp_is_en(intr_handle)) { +@@ -679,6 +683,7 @@ txgbevf_dev_start(struct rte_eth_dev *dev) + dev->data->nb_rx_queues)) { + PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" + " intr_vec", dev->data->nb_rx_queues); ++ txgbe_dev_clear_queues(dev); + return -ENOMEM; + } + } +@@ -965,7 +970,7 @@ txgbevf_set_ivar_map(struct txgbe_hw *hw, int8_t direction, + wr32(hw, TXGBE_VFIVARMISC, tmp); + } else { + /* rx or tx cause */ +- /* Workaround for ICR lost */ ++ msix_vector |= TXGBE_VFIVAR_VLD; /* Workaround for ICR lost */ + idx = ((16 * (queue & 1)) + (8 * direction)); + tmp = rd32(hw, TXGBE_VFIVAR(queue >> 1)); + tmp &= ~(0xFF << idx); +@@ -1201,9 +1206,13 @@ static int + txgbevf_dev_promiscuous_disable(struct rte_eth_dev *dev) + { + struct txgbe_hw *hw = TXGBE_DEV_HW(dev); ++ int mode = TXGBEVF_XCAST_MODE_NONE; + int ret; + +- switch (hw->mac.update_xcast_mode(hw, TXGBEVF_XCAST_MODE_NONE)) { ++ if (dev->data->all_multicast) ++ mode = TXGBEVF_XCAST_MODE_ALLMULTI; ++ ++ switch (hw->mac.update_xcast_mode(hw, mode)) { + case 0: + ret = 0; + break; +@@ -1224,6 +1233,9 @@ txgbevf_dev_allmulticast_enable(struct rte_eth_dev *dev) + struct txgbe_hw *hw = TXGBE_DEV_HW(dev); + int ret; + ++ if (dev->data->promiscuous) ++ return 0; ++ + switch (hw->mac.update_xcast_mode(hw, TXGBEVF_XCAST_MODE_ALLMULTI)) { + case 0: + ret = 0; +@@ -1245,6 +1257,9 @@ txgbevf_dev_allmulticast_disable(struct rte_eth_dev *dev) + struct txgbe_hw *hw = TXGBE_DEV_HW(dev); + int ret; + ++ if (dev->data->promiscuous) ++ return 0; ++ + switch (hw->mac.update_xcast_mode(hw, TXGBEVF_XCAST_MODE_MULTI)) { + case 0: + ret = 0; +diff --git a/dpdk/drivers/net/txgbe/txgbe_fdir.c b/dpdk/drivers/net/txgbe/txgbe_fdir.c +index a198b6781b..f627ab681d 100644 +--- a/dpdk/drivers/net/txgbe/txgbe_fdir.c ++++ b/dpdk/drivers/net/txgbe/txgbe_fdir.c +@@ -844,6 +844,9 @@ txgbe_fdir_filter_program(struct rte_eth_dev *dev, + return -EINVAL; + } + ++ if (RTE_ETH_DEV_SRIOV(dev).active) ++ queue = RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue; ++ + node = txgbe_fdir_filter_lookup(info, &rule->input); + if (node) { + if (!update) { +diff --git a/dpdk/drivers/net/txgbe/txgbe_rxtx.c b/dpdk/drivers/net/txgbe/txgbe_rxtx.c +index 1cd4b25965..2efc2bcf29 100644 +--- a/dpdk/drivers/net/txgbe/txgbe_rxtx.c ++++ b/dpdk/drivers/net/txgbe/txgbe_rxtx.c +@@ -564,26 +564,17 @@ tx_desc_ol_flags_to_ptype(uint64_t oflags) + switch (oflags & RTE_MBUF_F_TX_TUNNEL_MASK) { + case RTE_MBUF_F_TX_TUNNEL_VXLAN: + case RTE_MBUF_F_TX_TUNNEL_VXLAN_GPE: +- ptype |= RTE_PTYPE_L2_ETHER | +- RTE_PTYPE_L3_IPV4 | +- RTE_PTYPE_TUNNEL_GRENAT; ++ ptype |= RTE_PTYPE_TUNNEL_GRENAT; + break; + case RTE_MBUF_F_TX_TUNNEL_GRE: +- ptype |= RTE_PTYPE_L2_ETHER | +- RTE_PTYPE_L3_IPV4 | +- RTE_PTYPE_TUNNEL_GRE; ++ ptype |= RTE_PTYPE_TUNNEL_GRE; + break; + case RTE_MBUF_F_TX_TUNNEL_GENEVE: +- ptype |= RTE_PTYPE_L2_ETHER | +- RTE_PTYPE_L3_IPV4 | +- RTE_PTYPE_TUNNEL_GENEVE; +- ptype |= RTE_PTYPE_INNER_L2_ETHER; ++ ptype |= RTE_PTYPE_TUNNEL_GENEVE; + break; + case RTE_MBUF_F_TX_TUNNEL_IPIP: + case RTE_MBUF_F_TX_TUNNEL_IP: +- ptype |= RTE_PTYPE_L2_ETHER | +- RTE_PTYPE_L3_IPV4 | +- RTE_PTYPE_TUNNEL_IP; ++ ptype |= RTE_PTYPE_TUNNEL_IP; + break; + } + +@@ -667,11 +658,20 @@ txgbe_xmit_cleanup(struct txgbe_tx_queue *txq) + return 0; + } + ++#define GRE_CHECKSUM_PRESENT 0x8000 ++#define GRE_KEY_PRESENT 0x2000 ++#define GRE_SEQUENCE_PRESENT 0x1000 ++#define GRE_EXT_LEN 4 ++#define GRE_SUPPORTED_FIELDS (GRE_CHECKSUM_PRESENT | GRE_KEY_PRESENT |\ ++ GRE_SEQUENCE_PRESENT) ++ + static inline uint8_t + txgbe_get_tun_len(struct rte_mbuf *mbuf) + { + struct txgbe_genevehdr genevehdr; + const struct txgbe_genevehdr *gh; ++ const struct txgbe_grehdr *grh; ++ struct txgbe_grehdr grehdr; + uint8_t tun_len; + + switch (mbuf->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) { +@@ -684,11 +684,16 @@ txgbe_get_tun_len(struct rte_mbuf *mbuf) + + sizeof(struct txgbe_vxlanhdr); + break; + case RTE_MBUF_F_TX_TUNNEL_GRE: +- tun_len = sizeof(struct txgbe_nvgrehdr); ++ tun_len = sizeof(struct txgbe_grehdr); ++ grh = rte_pktmbuf_read(mbuf, ++ mbuf->outer_l2_len + mbuf->outer_l3_len, ++ sizeof(grehdr), &grehdr); ++ if (grh->flags & rte_cpu_to_be_16(GRE_SUPPORTED_FIELDS)) ++ tun_len += GRE_EXT_LEN; + break; + case RTE_MBUF_F_TX_TUNNEL_GENEVE: +- gh = rte_pktmbuf_read(mbuf, +- mbuf->outer_l2_len + mbuf->outer_l3_len, ++ gh = rte_pktmbuf_read(mbuf, mbuf->outer_l2_len + ++ mbuf->outer_l3_len + sizeof(struct txgbe_udphdr), + sizeof(genevehdr), &genevehdr); + tun_len = sizeof(struct txgbe_udphdr) + + sizeof(struct txgbe_genevehdr) +@@ -702,27 +707,26 @@ txgbe_get_tun_len(struct rte_mbuf *mbuf) + } + + static inline uint8_t +-txgbe_parse_tun_ptid(struct rte_mbuf *tx_pkt) ++txgbe_parse_tun_ptid(struct rte_mbuf *tx_pkt, uint8_t tun_len) + { +- uint64_t l2_vxlan, l2_vxlan_mac, l2_vxlan_mac_vlan; +- uint64_t l2_gre, l2_gre_mac, l2_gre_mac_vlan; ++ uint64_t inner_l2_len; + uint8_t ptid = 0; + +- l2_vxlan = sizeof(struct txgbe_udphdr) + sizeof(struct txgbe_vxlanhdr); +- l2_vxlan_mac = l2_vxlan + sizeof(struct rte_ether_hdr); +- l2_vxlan_mac_vlan = l2_vxlan_mac + sizeof(struct rte_vlan_hdr); ++ inner_l2_len = tx_pkt->l2_len - tun_len; + +- l2_gre = sizeof(struct txgbe_grehdr); +- l2_gre_mac = l2_gre + sizeof(struct rte_ether_hdr); +- l2_gre_mac_vlan = l2_gre_mac + sizeof(struct rte_vlan_hdr); +- +- if (tx_pkt->l2_len == l2_vxlan || tx_pkt->l2_len == l2_gre) ++ switch (inner_l2_len) { ++ case 0: + ptid = TXGBE_PTID_TUN_EIG; +- else if (tx_pkt->l2_len == l2_vxlan_mac || tx_pkt->l2_len == l2_gre_mac) ++ break; ++ case sizeof(struct rte_ether_hdr): + ptid = TXGBE_PTID_TUN_EIGM; +- else if (tx_pkt->l2_len == l2_vxlan_mac_vlan || +- tx_pkt->l2_len == l2_gre_mac_vlan) ++ break; ++ case sizeof(struct rte_ether_hdr) + sizeof(struct rte_vlan_hdr): + ptid = TXGBE_PTID_TUN_EIGMV; ++ break; ++ default: ++ ptid = TXGBE_PTID_TUN_EI; ++ } + + return ptid; + } +@@ -789,8 +793,6 @@ txgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + tx_ol_req = ol_flags & TXGBE_TX_OFFLOAD_MASK; + if (tx_ol_req) { + tx_offload.ptid = tx_desc_ol_flags_to_ptid(tx_ol_req); +- if (tx_offload.ptid & TXGBE_PTID_PKT_TUN) +- tx_offload.ptid |= txgbe_parse_tun_ptid(tx_pkt); + tx_offload.l2_len = tx_pkt->l2_len; + tx_offload.l3_len = tx_pkt->l3_len; + tx_offload.l4_len = tx_pkt->l4_len; +@@ -799,6 +801,9 @@ txgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + tx_offload.outer_l2_len = tx_pkt->outer_l2_len; + tx_offload.outer_l3_len = tx_pkt->outer_l3_len; + tx_offload.outer_tun_len = txgbe_get_tun_len(tx_pkt); ++ if (tx_offload.ptid & TXGBE_PTID_PKT_TUN) ++ tx_offload.ptid |= txgbe_parse_tun_ptid(tx_pkt, ++ tx_offload.outer_tun_len); + + #ifdef RTE_LIB_SECURITY + if (use_ipsec) { +@@ -2130,6 +2135,7 @@ txgbe_tx_queue_release(struct txgbe_tx_queue *txq) + if (txq != NULL && txq->ops != NULL) { + txq->ops->release_mbufs(txq); + txq->ops->free_swring(txq); ++ rte_memzone_free(txq->mz); + rte_free(txq); + } + } +@@ -2341,6 +2347,7 @@ txgbe_dev_tx_queue_setup(struct rte_eth_dev *dev, + return -ENOMEM; + } + ++ txq->mz = tz; + txq->nb_tx_desc = nb_desc; + txq->tx_free_thresh = tx_free_thresh; + txq->pthresh = tx_conf->tx_thresh.pthresh; +@@ -2458,6 +2465,7 @@ txgbe_rx_queue_release(struct txgbe_rx_queue *rxq) + txgbe_rx_queue_release_mbufs(rxq); + rte_free(rxq->sw_ring); + rte_free(rxq->sw_sc_ring); ++ rte_memzone_free(rxq->mz); + rte_free(rxq); + } + } +@@ -2551,6 +2559,7 @@ txgbe_reset_rx_queue(struct txgbe_adapter *adapter, struct txgbe_rx_queue *rxq) + rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1); + rxq->rx_tail = 0; + rxq->nb_rx_hold = 0; ++ rte_pktmbuf_free(rxq->pkt_first_seg); + rxq->pkt_first_seg = NULL; + rxq->pkt_last_seg = NULL; + } +@@ -2631,6 +2640,7 @@ txgbe_dev_rx_queue_setup(struct rte_eth_dev *dev, + return -ENOMEM; + } + ++ rxq->mz = rz; + /* + * Zero init all the descriptors in the ring. + */ +@@ -5069,6 +5079,7 @@ txgbe_config_rss_filter(struct rte_eth_dev *dev, + uint32_t reta; + uint16_t i; + uint16_t j; ++ uint16_t queue; + struct rte_eth_rss_conf rss_conf = { + .rss_key = conf->conf.key_len ? + (void *)(uintptr_t)conf->conf.key : NULL, +@@ -5101,7 +5112,12 @@ txgbe_config_rss_filter(struct rte_eth_dev *dev, + for (i = 0, j = 0; i < RTE_ETH_RSS_RETA_SIZE_128; i++, j++) { + if (j == conf->conf.queue_num) + j = 0; +- reta = (reta >> 8) | LS32(conf->conf.queue[j], 24, 0xFF); ++ if (RTE_ETH_DEV_SRIOV(dev).active) ++ queue = RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + ++ conf->conf.queue[j]; ++ else ++ queue = conf->conf.queue[j]; ++ reta = (reta >> 8) | LS32(queue, 24, 0xFF); + if ((i & 3) == 3) + wr32at(hw, TXGBE_REG_RSSTBL, i >> 2, reta); + } +diff --git a/dpdk/drivers/net/txgbe/txgbe_rxtx.h b/dpdk/drivers/net/txgbe/txgbe_rxtx.h +index 27d4c842c0..c579e1a9f2 100644 +--- a/dpdk/drivers/net/txgbe/txgbe_rxtx.h ++++ b/dpdk/drivers/net/txgbe/txgbe_rxtx.h +@@ -314,6 +314,7 @@ struct txgbe_rx_queue { + struct rte_mbuf fake_mbuf; + /** hold packets to return to application */ + struct rte_mbuf *rx_stage[RTE_PMD_TXGBE_RX_MAX_BURST * 2]; ++ const struct rte_memzone *mz; + }; + + /** +@@ -402,6 +403,7 @@ struct txgbe_tx_queue { + uint8_t using_ipsec; + /**< indicates that IPsec TX feature is in use */ + #endif ++ const struct rte_memzone *mz; + }; + + struct txgbe_txq_ops { +diff --git a/dpdk/drivers/net/virtio/virtio_ethdev.c b/dpdk/drivers/net/virtio/virtio_ethdev.c +index c2c0a1a111..70d4839def 100644 +--- a/dpdk/drivers/net/virtio/virtio_ethdev.c ++++ b/dpdk/drivers/net/virtio/virtio_ethdev.c +@@ -913,6 +913,8 @@ virtio_mac_addr_add(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr, + struct virtio_net_ctrl_mac *tbl + = rte_is_multicast_ether_addr(addr) ? mc : uc; + ++ if (rte_is_zero_ether_addr(addr)) ++ break; + memcpy(&tbl->macs[tbl->entries++], addr, RTE_ETHER_ADDR_LEN); + } + +@@ -1793,8 +1795,6 @@ virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features) + else + eth_dev->data->dev_flags &= ~RTE_ETH_DEV_INTR_LSC; + +- eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS; +- + /* Setting up rx_header size for the device */ + if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF) || + virtio_with_feature(hw, VIRTIO_F_VERSION_1) || +diff --git a/dpdk/drivers/net/virtio/virtio_user/vhost_user.c b/dpdk/drivers/net/virtio/virtio_user/vhost_user.c +index 3c05ac9cc0..c10252506b 100644 +--- a/dpdk/drivers/net/virtio/virtio_user/vhost_user.c ++++ b/dpdk/drivers/net/virtio/virtio_user/vhost_user.c +@@ -128,7 +128,8 @@ vhost_user_write(int fd, struct vhost_user_msg *msg, int *fds, int fd_num) + cmsg->cmsg_len = CMSG_LEN(fd_size); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_RIGHTS; +- memcpy(CMSG_DATA(cmsg), fds, fd_size); ++ if (fd_size > 0) ++ memcpy(CMSG_DATA(cmsg), fds, fd_size); + + do { + r = sendmsg(fd, &msgh, 0); +diff --git a/dpdk/drivers/net/virtio/virtio_user/virtio_user_dev.c b/dpdk/drivers/net/virtio/virtio_user/virtio_user_dev.c +index af1f8c8237..1bfd6aba80 100644 +--- a/dpdk/drivers/net/virtio/virtio_user/virtio_user_dev.c ++++ b/dpdk/drivers/net/virtio/virtio_user/virtio_user_dev.c +@@ -20,6 +20,7 @@ + #include + + #include "vhost.h" ++#include "virtio.h" + #include "virtio_user_dev.h" + #include "../virtio_ethdev.h" + +@@ -32,6 +33,61 @@ const char * const virtio_user_backend_strings[] = { + [VIRTIO_USER_BACKEND_VHOST_VDPA] = "VHOST_VDPA", + }; + ++static int ++virtio_user_uninit_notify_queue(struct virtio_user_dev *dev, uint32_t queue_sel) ++{ ++ if (dev->kickfds[queue_sel] >= 0) { ++ close(dev->kickfds[queue_sel]); ++ dev->kickfds[queue_sel] = -1; ++ } ++ ++ if (dev->callfds[queue_sel] >= 0) { ++ close(dev->callfds[queue_sel]); ++ dev->callfds[queue_sel] = -1; ++ } ++ ++ return 0; ++} ++ ++static int ++virtio_user_init_notify_queue(struct virtio_user_dev *dev, uint32_t queue_sel) ++{ ++ /* May use invalid flag, but some backend uses kickfd and ++ * callfd as criteria to judge if dev is alive. so finally we ++ * use real event_fd. ++ */ ++ dev->callfds[queue_sel] = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); ++ if (dev->callfds[queue_sel] < 0) { ++ PMD_DRV_LOG(ERR, "(%s) Failed to setup callfd for queue %u: %s", ++ dev->path, queue_sel, strerror(errno)); ++ return -1; ++ } ++ dev->kickfds[queue_sel] = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); ++ if (dev->kickfds[queue_sel] < 0) { ++ PMD_DRV_LOG(ERR, "(%s) Failed to setup kickfd for queue %u: %s", ++ dev->path, queue_sel, strerror(errno)); ++ return -1; ++ } ++ ++ return 0; ++} ++ ++static int ++virtio_user_destroy_queue(struct virtio_user_dev *dev, uint32_t queue_sel) ++{ ++ struct vhost_vring_state state; ++ int ret; ++ ++ state.index = queue_sel; ++ ret = dev->ops->get_vring_base(dev, &state); ++ if (ret < 0) { ++ PMD_DRV_LOG(ERR, "(%s) Failed to destroy queue %u", dev->path, queue_sel); ++ return -1; ++ } ++ ++ return 0; ++} ++ + static int + virtio_user_create_queue(struct virtio_user_dev *dev, uint32_t queue_sel) + { +@@ -127,6 +183,22 @@ err: + return -1; + } + ++static int ++virtio_user_foreach_queue(struct virtio_user_dev *dev, int (*fn)(struct virtio_user_dev *, uint32_t)) ++{ ++ uint32_t i, nr_vq; ++ ++ nr_vq = dev->max_queue_pairs * 2; ++ if (dev->hw_cvq) ++ nr_vq++; ++ ++ for (i = 0; i < nr_vq; i++) ++ if (fn(dev, i) < 0) ++ return -1; ++ ++ return 0; ++} ++ + static int + virtio_user_queue_setup(struct virtio_user_dev *dev, + int (*fn)(struct virtio_user_dev *, uint32_t)) +@@ -215,6 +287,12 @@ virtio_user_start_device(struct virtio_user_dev *dev) + if (ret < 0) + goto error; + ++ if (dev->scvq) { ++ ret = dev->ops->cvq_enable(dev, 1); ++ if (ret < 0) ++ goto error; ++ } ++ + dev->started = true; + + pthread_mutex_unlock(&dev->mutex); +@@ -233,7 +311,6 @@ error: + + int virtio_user_stop_device(struct virtio_user_dev *dev) + { +- struct vhost_vring_state state; + uint32_t i; + int ret; + +@@ -247,16 +324,16 @@ int virtio_user_stop_device(struct virtio_user_dev *dev) + goto err; + } + +- /* Stop the backend. */ +- for (i = 0; i < dev->max_queue_pairs * 2; ++i) { +- state.index = i; +- ret = dev->ops->get_vring_base(dev, &state); +- if (ret < 0) { +- PMD_DRV_LOG(ERR, "(%s) get_vring_base failed, index=%u", dev->path, i); ++ if (dev->scvq) { ++ ret = dev->ops->cvq_enable(dev, 0); ++ if (ret < 0) + goto err; +- } + } + ++ /* Stop the backend. */ ++ if (virtio_user_foreach_queue(dev, virtio_user_destroy_queue) < 0) ++ goto err; ++ + dev->started = false; + + out: +@@ -386,46 +463,13 @@ out: + static int + virtio_user_dev_init_notify(struct virtio_user_dev *dev) + { +- uint32_t i, j, nr_vq; +- int callfd; +- int kickfd; +- +- nr_vq = dev->max_queue_pairs * 2; +- if (dev->hw_cvq) +- nr_vq++; + +- for (i = 0; i < nr_vq; i++) { +- /* May use invalid flag, but some backend uses kickfd and +- * callfd as criteria to judge if dev is alive. so finally we +- * use real event_fd. +- */ +- callfd = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); +- if (callfd < 0) { +- PMD_DRV_LOG(ERR, "(%s) callfd error, %s", dev->path, strerror(errno)); +- goto err; +- } +- kickfd = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); +- if (kickfd < 0) { +- close(callfd); +- PMD_DRV_LOG(ERR, "(%s) kickfd error, %s", dev->path, strerror(errno)); +- goto err; +- } +- dev->callfds[i] = callfd; +- dev->kickfds[i] = kickfd; +- } ++ if (virtio_user_foreach_queue(dev, virtio_user_init_notify_queue) < 0) ++ goto err; + + return 0; + err: +- for (j = 0; j < i; j++) { +- if (dev->kickfds[j] >= 0) { +- close(dev->kickfds[j]); +- dev->kickfds[j] = -1; +- } +- if (dev->callfds[j] >= 0) { +- close(dev->callfds[j]); +- dev->callfds[j] = -1; +- } +- } ++ virtio_user_foreach_queue(dev, virtio_user_uninit_notify_queue); + + return -1; + } +@@ -433,18 +477,8 @@ err: + static void + virtio_user_dev_uninit_notify(struct virtio_user_dev *dev) + { +- uint32_t i; ++ virtio_user_foreach_queue(dev, virtio_user_uninit_notify_queue); + +- for (i = 0; i < dev->max_queue_pairs * 2; ++i) { +- if (dev->kickfds[i] >= 0) { +- close(dev->kickfds[i]); +- dev->kickfds[i] = -1; +- } +- if (dev->callfds[i] >= 0) { +- close(dev->callfds[i]); +- dev->callfds[i] = -1; +- } +- } + } + + static int +@@ -575,7 +609,7 @@ virtio_user_alloc_vrings(struct virtio_user_dev *dev) + bool packed_ring = !!(dev->device_features & (1ull << VIRTIO_F_RING_PACKED)); + + nr_vrings = dev->max_queue_pairs * 2; +- if (dev->device_features & (1ull << VIRTIO_NET_F_MQ)) ++ if (dev->frontend_features & (1ull << VIRTIO_NET_F_CTRL_VQ)) + nr_vrings++; + + dev->callfds = rte_zmalloc("virtio_user_dev", nr_vrings * sizeof(*dev->callfds), 0); +@@ -725,7 +759,7 @@ virtio_user_dev_init(struct virtio_user_dev *dev, char *path, uint16_t queues, + if (virtio_user_dev_init_max_queue_pairs(dev, queues)) + dev->unsupported_features |= (1ull << VIRTIO_NET_F_MQ); + +- if (dev->max_queue_pairs > 1) ++ if (dev->max_queue_pairs > 1 || dev->hw_cvq) + cq = 1; + + if (!mrg_rxbuf) +@@ -743,8 +777,9 @@ virtio_user_dev_init(struct virtio_user_dev *dev, char *path, uint16_t queues, + dev->unsupported_features |= (1ull << VIRTIO_NET_F_MAC); + + if (cq) { +- /* device does not really need to know anything about CQ, +- * so if necessary, we just claim to support CQ ++ /* Except for vDPA, the device does not really need to know ++ * anything about CQ, so if necessary, we just claim to support ++ * control queue. + */ + dev->frontend_features |= (1ull << VIRTIO_NET_F_CTRL_VQ); + } else { +@@ -844,9 +879,6 @@ virtio_user_handle_mq(struct virtio_user_dev *dev, uint16_t q_pairs) + for (i = q_pairs; i < dev->max_queue_pairs; ++i) + ret |= dev->ops->enable_qp(dev, i, 0); + +- if (dev->scvq) +- ret |= dev->ops->cvq_enable(dev, 1); +- + dev->queue_pairs = q_pairs; + + return ret; +diff --git a/dpdk/drivers/net/vmxnet3/base/vmxnet3_defs.h b/dpdk/drivers/net/vmxnet3/base/vmxnet3_defs.h +index 24c235876e..a6bb281d8d 100644 +--- a/dpdk/drivers/net/vmxnet3/base/vmxnet3_defs.h ++++ b/dpdk/drivers/net/vmxnet3/base/vmxnet3_defs.h +@@ -126,6 +126,7 @@ typedef enum { + VMXNET3_CMD_RESERVED7, + VMXNET3_CMD_RESERVED8, + VMXNET3_CMD_GET_MAX_QUEUES_CONF, ++ VMXNET3_CMD_RESERVED11, + VMXNET3_CMD_GET_MAX_CAPABILITIES, + VMXNET3_CMD_GET_DCR0_REG, + } Vmxnet3_Cmd; +diff --git a/dpdk/drivers/net/vmxnet3/vmxnet3_ethdev.c b/dpdk/drivers/net/vmxnet3/vmxnet3_ethdev.c +index e49191718a..70ae9c6035 100644 +--- a/dpdk/drivers/net/vmxnet3/vmxnet3_ethdev.c ++++ b/dpdk/drivers/net/vmxnet3/vmxnet3_ethdev.c +@@ -257,6 +257,7 @@ vmxnet3_disable_all_intrs(struct vmxnet3_hw *hw) + vmxnet3_disable_intr(hw, i); + } + ++#ifndef RTE_EXEC_ENV_FREEBSD + /* + * Enable all intrs used by the device + */ +@@ -280,6 +281,7 @@ vmxnet3_enable_all_intrs(struct vmxnet3_hw *hw) + vmxnet3_enable_intr(hw, i); + } + } ++#endif + + /* + * Gets tx data ring descriptor size. +@@ -1129,6 +1131,7 @@ vmxnet3_dev_start(struct rte_eth_dev *dev) + /* Setting proper Rx Mode and issue Rx Mode Update command */ + vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_UCAST | VMXNET3_RXM_BCAST, 1); + ++#ifndef RTE_EXEC_ENV_FREEBSD + /* Setup interrupt callback */ + rte_intr_callback_register(dev->intr_handle, + vmxnet3_interrupt_handler, dev); +@@ -1140,6 +1143,7 @@ vmxnet3_dev_start(struct rte_eth_dev *dev) + + /* enable all intrs */ + vmxnet3_enable_all_intrs(hw); ++#endif + + vmxnet3_process_events(dev); + +@@ -1928,11 +1932,13 @@ done: + static int + vmxnet3_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) + { ++#ifndef RTE_EXEC_ENV_FREEBSD + struct vmxnet3_hw *hw = dev->data->dev_private; + + vmxnet3_enable_intr(hw, + rte_intr_vec_list_index_get(dev->intr_handle, + queue_id)); ++#endif + + return 0; + } +diff --git a/dpdk/drivers/net/vmxnet3/vmxnet3_logs.h b/dpdk/drivers/net/vmxnet3/vmxnet3_logs.h +index 74154e3a1a..ae8542811a 100644 +--- a/dpdk/drivers/net/vmxnet3/vmxnet3_logs.h ++++ b/dpdk/drivers/net/vmxnet3/vmxnet3_logs.h +@@ -7,7 +7,7 @@ + + extern int vmxnet3_logtype_init; + #define PMD_INIT_LOG(level, fmt, args...) \ +- rte_log(RTE_LOG_ ## level, vmxnet3_logtype_driver, \ ++ rte_log(RTE_LOG_ ## level, vmxnet3_logtype_init, \ + "%s(): " fmt "\n", __func__, ## args) + #define PMD_INIT_FUNC_TRACE() PMD_INIT_LOG(DEBUG, " >>") + +diff --git a/dpdk/drivers/vdpa/mlx5/mlx5_vdpa_event.c b/dpdk/drivers/vdpa/mlx5/mlx5_vdpa_event.c +index 9557c1042e..32430614d5 100644 +--- a/dpdk/drivers/vdpa/mlx5/mlx5_vdpa_event.c ++++ b/dpdk/drivers/vdpa/mlx5/mlx5_vdpa_event.c +@@ -244,22 +244,30 @@ mlx5_vdpa_queues_complete(struct mlx5_vdpa_priv *priv) + return max; + } + ++static void ++mlx5_vdpa_drain_cq_one(struct mlx5_vdpa_priv *priv, ++ struct mlx5_vdpa_virtq *virtq) ++{ ++ struct mlx5_vdpa_cq *cq = &virtq->eqp.cq; ++ ++ mlx5_vdpa_queue_complete(cq); ++ if (cq->cq_obj.cq) { ++ cq->cq_obj.cqes[0].wqe_counter = rte_cpu_to_be_16(UINT16_MAX); ++ virtq->eqp.qp_pi = 0; ++ if (!cq->armed) ++ mlx5_vdpa_cq_arm(priv, cq); ++ } ++} ++ + void + mlx5_vdpa_drain_cq(struct mlx5_vdpa_priv *priv) + { ++ struct mlx5_vdpa_virtq *virtq; + unsigned int i; + + for (i = 0; i < priv->caps.max_num_virtio_queues; i++) { +- struct mlx5_vdpa_cq *cq = &priv->virtqs[i].eqp.cq; +- +- mlx5_vdpa_queue_complete(cq); +- if (cq->cq_obj.cq) { +- cq->cq_obj.cqes[0].wqe_counter = +- rte_cpu_to_be_16(UINT16_MAX); +- priv->virtqs[i].eqp.qp_pi = 0; +- if (!cq->armed) +- mlx5_vdpa_cq_arm(priv, cq); +- } ++ virtq = &priv->virtqs[i]; ++ mlx5_vdpa_drain_cq_one(priv, virtq); + } + } + +@@ -632,6 +640,7 @@ mlx5_vdpa_event_qp_prepare(struct mlx5_vdpa_priv *priv, uint16_t desc_n, + if (eqp->cq.cq_obj.cq != NULL && log_desc_n == eqp->cq.log_desc_n) { + /* Reuse existing resources. */ + eqp->cq.callfd = callfd; ++ mlx5_vdpa_drain_cq_one(priv, virtq); + /* FW will set event qp to error state in q destroy. */ + if (reset && !mlx5_vdpa_qps2rst2rts(eqp)) + rte_write32(rte_cpu_to_be_32(RTE_BIT32(log_desc_n)), +diff --git a/dpdk/drivers/vdpa/sfc/sfc_vdpa_hw.c b/dpdk/drivers/vdpa/sfc/sfc_vdpa_hw.c +index edb7e35c2c..7e43719f53 100644 +--- a/dpdk/drivers/vdpa/sfc/sfc_vdpa_hw.c ++++ b/dpdk/drivers/vdpa/sfc/sfc_vdpa_hw.c +@@ -13,8 +13,6 @@ + #include "sfc_vdpa.h" + #include "sfc_vdpa_ops.h" + +-extern uint32_t sfc_logtype_driver; +- + #ifndef PAGE_SIZE + #define PAGE_SIZE (sysconf(_SC_PAGESIZE)) + #endif +diff --git a/dpdk/dts/framework/remote_session/remote/remote_session.py b/dpdk/dts/framework/remote_session/remote/remote_session.py +index 719f7d1ef7..68894a9686 100644 +--- a/dpdk/dts/framework/remote_session/remote/remote_session.py ++++ b/dpdk/dts/framework/remote_session/remote/remote_session.py +@@ -3,8 +3,8 @@ + # Copyright(c) 2022-2023 PANTHEON.tech s.r.o. + # Copyright(c) 2022-2023 University of New Hampshire + +-import dataclasses + from abc import ABC, abstractmethod ++from dataclasses import InitVar, dataclass, field + from pathlib import PurePath + + from framework.config import NodeConfiguration +@@ -13,7 +13,7 @@ from framework.logger import DTSLOG + from framework.settings import SETTINGS + + +-@dataclasses.dataclass(slots=True, frozen=True) ++@dataclass(slots=True, frozen=True) + class CommandResult: + """ + The result of remote execution of a command. +@@ -21,9 +21,25 @@ class CommandResult: + + name: str + command: str +- stdout: str +- stderr: str ++ init_stdout: InitVar[str] ++ init_stderr: InitVar[str] + return_code: int ++ stdout: str = field(init=False) ++ stderr: str = field(init=False) ++ ++ def __post_init__(self, init_stdout: str, init_stderr: str) -> None: ++ """Strip the whitespaces from stdout and stderr. ++ ++ The generated __init__ method uses object.__setattr__() when the dataclass is frozen, ++ so that's what we use here as well. ++ ++ In order to get access to dataclass fields in the __post_init__ method, ++ we have to type them as InitVars. These InitVars are included in the __init__ method's ++ signature, so we have to exclude the actual stdout and stderr fields ++ from the __init__ method's signature, so that we have the proper number of arguments. ++ """ ++ object.__setattr__(self, "stdout", init_stdout.strip()) ++ object.__setattr__(self, "stderr", init_stderr.strip()) + + def __str__(self) -> str: + return ( +diff --git a/dpdk/dts/tests/TestSuite_smoke_tests.py b/dpdk/dts/tests/TestSuite_smoke_tests.py +index 8958f58dac..5e897cf5d2 100644 +--- a/dpdk/dts/tests/TestSuite_smoke_tests.py ++++ b/dpdk/dts/tests/TestSuite_smoke_tests.py +@@ -91,7 +91,7 @@ class SmokeTests(TestSuite): + # with the address for the nic we are on in the loop and then captures the + # name of the driver in a group + devbind_info_for_nic = re.search( +- f"{nic.pci}[^\\n]*drv=([\\d\\w]*) [^\\n]*", ++ f"{nic.pci}[^\\n]*drv=([\\d\\w-]*) [^\\n]*", + all_nics_in_dpdk_devbind, + ) + self.verify( +diff --git a/dpdk/examples/fips_validation/fips_validation_rsa.c b/dpdk/examples/fips_validation/fips_validation_rsa.c +index f675b51051..55f81860a0 100644 +--- a/dpdk/examples/fips_validation/fips_validation_rsa.c ++++ b/dpdk/examples/fips_validation/fips_validation_rsa.c +@@ -328,6 +328,9 @@ parse_test_rsa_json_interim_writeback(struct fips_val *val) + if (prepare_vec_rsa() < 0) + return -1; + ++ if (!vec.rsa.e.val) ++ return -1; ++ + writeback_hex_str("", info.one_line_text, &vec.rsa.n); + obj = json_string(info.one_line_text); + json_object_set_new(json_info.json_write_group, "n", obj); +@@ -474,7 +477,7 @@ fips_test_randomize_message(struct fips_val *msg, struct fips_val *rand) + uint16_t rv_len; + + if (!msg->val || !rand->val || rand->len > RV_BUF_LEN +- || msg->len > FIPS_TEST_JSON_BUF_LEN) ++ || msg->len > (FIPS_TEST_JSON_BUF_LEN - 1)) + return -EINVAL; + + memset(rv, 0, sizeof(rv)); +@@ -503,7 +506,7 @@ fips_test_randomize_message(struct fips_val *msg, struct fips_val *rand) + m[i + j] ^= rv[j]; + + m[i + j] = ((uint8_t *)&rv_bitlen)[0]; +- m[i + j + 1] = (((uint8_t *)&rv_bitlen)[1] >> 8) & 0xFF; ++ m[i + j + 1] = ((uint8_t *)&rv_bitlen)[1]; + + rte_free(msg->val); + msg->len = (rv_bitlen + m_bitlen + 16) / 8; +diff --git a/dpdk/examples/ipsec-secgw/event_helper.h b/dpdk/examples/ipsec-secgw/event_helper.h +index dfb81bfcf1..be635685b4 100644 +--- a/dpdk/examples/ipsec-secgw/event_helper.h ++++ b/dpdk/examples/ipsec-secgw/event_helper.h +@@ -102,7 +102,7 @@ struct eh_event_link_info { + /**< Event port ID */ + uint8_t eventq_id; + /**< Event queue to be linked to the port */ +- uint8_t lcore_id; ++ uint32_t lcore_id; + /**< Lcore to be polling on this port */ + }; + +diff --git a/dpdk/examples/ipsec-secgw/ipsec-secgw.c b/dpdk/examples/ipsec-secgw/ipsec-secgw.c +index bf98d2618b..761b9cf396 100644 +--- a/dpdk/examples/ipsec-secgw/ipsec-secgw.c ++++ b/dpdk/examples/ipsec-secgw/ipsec-secgw.c +@@ -220,8 +220,8 @@ static const char *cfgfile; + + struct lcore_params { + uint16_t port_id; +- uint8_t queue_id; +- uint8_t lcore_id; ++ uint16_t queue_id; ++ uint32_t lcore_id; + } __rte_cache_aligned; + + static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS]; +@@ -568,7 +568,7 @@ process_pkts_outbound_nosp(struct ipsec_ctx *ipsec_ctx, + + static inline void + process_pkts(struct lcore_conf *qconf, struct rte_mbuf **pkts, +- uint8_t nb_pkts, uint16_t portid, void *ctx) ++ uint16_t nb_pkts, uint16_t portid, void *ctx) + { + struct ipsec_traffic traffic; + +@@ -695,9 +695,7 @@ ipsec_poll_mode_worker(void) + struct rte_mbuf *pkts[MAX_PKT_BURST]; + uint32_t lcore_id; + uint64_t prev_tsc, diff_tsc, cur_tsc; +- int32_t i, nb_rx; +- uint16_t portid; +- uint8_t queueid; ++ uint16_t i, nb_rx, portid, queueid; + struct lcore_conf *qconf; + int32_t rc, socket_id; + const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) +@@ -744,7 +742,7 @@ ipsec_poll_mode_worker(void) + portid = rxql[i].port_id; + queueid = rxql[i].queue_id; + RTE_LOG(INFO, IPSEC, +- " -- lcoreid=%u portid=%u rxqueueid=%hhu\n", ++ " -- lcoreid=%u portid=%u rxqueueid=%" PRIu16 "\n", + lcore_id, portid, queueid); + } + +@@ -789,8 +787,7 @@ int + check_flow_params(uint16_t fdir_portid, uint8_t fdir_qid) + { + uint16_t i; +- uint16_t portid; +- uint8_t queueid; ++ uint16_t portid, queueid; + + for (i = 0; i < nb_lcore_params; ++i) { + portid = lcore_params_array[i].port_id; +@@ -810,7 +807,7 @@ check_flow_params(uint16_t fdir_portid, uint8_t fdir_qid) + static int32_t + check_poll_mode_params(struct eh_conf *eh_conf) + { +- uint8_t lcore; ++ uint32_t lcore; + uint16_t portid; + uint16_t i; + int32_t socket_id; +@@ -829,13 +826,13 @@ check_poll_mode_params(struct eh_conf *eh_conf) + for (i = 0; i < nb_lcore_params; ++i) { + lcore = lcore_params[i].lcore_id; + if (!rte_lcore_is_enabled(lcore)) { +- printf("error: lcore %hhu is not enabled in " ++ printf("error: lcore %u is not enabled in " + "lcore mask\n", lcore); + return -1; + } + socket_id = rte_lcore_to_socket_id(lcore); + if (socket_id != 0 && numa_on == 0) { +- printf("warning: lcore %hhu is on socket %d " ++ printf("warning: lcore %u is on socket %d " + "with numa off\n", + lcore, socket_id); + } +@@ -852,7 +849,7 @@ check_poll_mode_params(struct eh_conf *eh_conf) + return 0; + } + +-static uint8_t ++static uint16_t + get_port_nb_rx_queues(const uint16_t port) + { + int32_t queue = -1; +@@ -863,14 +860,14 @@ get_port_nb_rx_queues(const uint16_t port) + lcore_params[i].queue_id > queue) + queue = lcore_params[i].queue_id; + } +- return (uint8_t)(++queue); ++ return (uint16_t)(++queue); + } + + static int32_t + init_lcore_rx_queues(void) + { + uint16_t i, nb_rx_queue; +- uint8_t lcore; ++ uint32_t lcore; + + for (i = 0; i < nb_lcore_params; ++i) { + lcore = lcore_params[i].lcore_id; +@@ -1051,6 +1048,11 @@ parse_config(const char *q_arg) + char *str_fld[_NUM_FLD]; + int32_t i; + uint32_t size; ++ uint32_t max_fld[_NUM_FLD] = { ++ RTE_MAX_ETHPORTS, ++ RTE_MAX_QUEUES_PER_PORT, ++ RTE_MAX_LCORE ++ }; + + nb_lcore_params = 0; + +@@ -1071,7 +1073,7 @@ parse_config(const char *q_arg) + for (i = 0; i < _NUM_FLD; i++) { + errno = 0; + int_fld[i] = strtoul(str_fld[i], &end, 0); +- if (errno != 0 || end == str_fld[i] || int_fld[i] > 255) ++ if (errno != 0 || end == str_fld[i] || int_fld[i] > max_fld[i]) + return -1; + } + if (nb_lcore_params >= MAX_LCORE_PARAMS) { +@@ -1080,11 +1082,11 @@ parse_config(const char *q_arg) + return -1; + } + lcore_params_array[nb_lcore_params].port_id = +- (uint8_t)int_fld[FLD_PORT]; ++ (uint16_t)int_fld[FLD_PORT]; + lcore_params_array[nb_lcore_params].queue_id = +- (uint8_t)int_fld[FLD_QUEUE]; ++ (uint16_t)int_fld[FLD_QUEUE]; + lcore_params_array[nb_lcore_params].lcore_id = +- (uint8_t)int_fld[FLD_LCORE]; ++ (uint32_t)int_fld[FLD_LCORE]; + ++nb_lcore_params; + } + lcore_params = lcore_params_array; +@@ -1920,7 +1922,8 @@ port_init(uint16_t portid, uint64_t req_rx_offloads, uint64_t req_tx_offloads, + struct rte_eth_dev_info dev_info; + struct rte_eth_txconf *txconf; + uint16_t nb_tx_queue, nb_rx_queue; +- uint16_t tx_queueid, rx_queueid, queue, lcore_id; ++ uint16_t tx_queueid, rx_queueid, queue; ++ uint32_t lcore_id; + int32_t ret, socket_id; + struct lcore_conf *qconf; + struct rte_ether_addr ethaddr; +@@ -2094,10 +2097,10 @@ port_init(uint16_t portid, uint64_t req_rx_offloads, uint64_t req_tx_offloads, + + /* Register Rx callback if ptypes are not supported */ + if (!ptype_supported && +- !rte_eth_add_rx_callback(portid, queue, ++ !rte_eth_add_rx_callback(portid, rx_queueid, + parse_ptype_cb, NULL)) { + printf("Failed to add rx callback: port=%d, " +- "queue=%d\n", portid, queue); ++ "rx_queueid=%d\n", portid, rx_queueid); + } + + +diff --git a/dpdk/examples/ipsec-secgw/ipsec.c b/dpdk/examples/ipsec-secgw/ipsec.c +index f5cec4a928..b52b0ffc3d 100644 +--- a/dpdk/examples/ipsec-secgw/ipsec.c ++++ b/dpdk/examples/ipsec-secgw/ipsec.c +@@ -259,7 +259,7 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx_lcore[], + continue; + + /* Looking for cryptodev, which can handle this SA */ +- key.lcore_id = (uint8_t)lcore_id; ++ key.lcore_id = lcore_id; + key.cipher_algo = (uint8_t)sa->cipher_algo; + key.auth_algo = (uint8_t)sa->auth_algo; + key.aead_algo = (uint8_t)sa->aead_algo; +@@ -288,10 +288,21 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx_lcore[], + if (cdev_id == RTE_CRYPTO_MAX_DEVS) + cdev_id = ipsec_ctx->tbl[cdev_id_qp].id; + else if (cdev_id != ipsec_ctx->tbl[cdev_id_qp].id) { +- RTE_LOG(ERR, IPSEC, +- "SA mapping to multiple cryptodevs is " +- "not supported!"); +- return -EINVAL; ++ struct rte_cryptodev_info dev_info_1, dev_info_2; ++ rte_cryptodev_info_get(cdev_id, &dev_info_1); ++ rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id, ++ &dev_info_2); ++ if (dev_info_1.driver_id == dev_info_2.driver_id) { ++ RTE_LOG(WARNING, IPSEC, ++ "SA mapped to multiple cryptodevs for SPI %d\n", ++ sa->spi); ++ ++ } else { ++ RTE_LOG(WARNING, IPSEC, ++ "SA mapped to multiple cryptodevs of different types for SPI %d\n", ++ sa->spi); ++ ++ } + } + + /* Store per core queue pair information */ +@@ -908,6 +919,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, + continue; + } + ++ RTE_ASSERT(sa->cqp[ipsec_ctx->lcore_id] != NULL); + enqueue_cop(sa->cqp[ipsec_ctx->lcore_id], &priv->cop); + } + } +diff --git a/dpdk/examples/ipsec-secgw/ipsec.h b/dpdk/examples/ipsec-secgw/ipsec.h +index bdcada1c40..d4ecfdf08d 100644 +--- a/dpdk/examples/ipsec-secgw/ipsec.h ++++ b/dpdk/examples/ipsec-secgw/ipsec.h +@@ -117,7 +117,7 @@ struct ipsec_sa { + uint32_t spi; + struct cdev_qp *cqp[RTE_MAX_LCORE]; + uint64_t seq; +- uint32_t salt; ++ rte_be32_t salt; + uint32_t fallback_sessions; + enum rte_crypto_cipher_algorithm cipher_algo; + enum rte_crypto_auth_algorithm auth_algo; +@@ -256,11 +256,11 @@ extern struct offloads tx_offloads; + * (hash key calculation reads 8 bytes if this struct is size 5 bytes). + */ + struct cdev_key { +- uint16_t lcore_id; ++ uint32_t lcore_id; + uint8_t cipher_algo; + uint8_t auth_algo; + uint8_t aead_algo; +- uint8_t padding[3]; /* padding to 8-byte size should be zeroed */ ++ uint8_t padding; /* padding to 8-byte size should be zeroed */ + }; + + struct socket_ctx { +@@ -285,7 +285,7 @@ struct cnt_blk { + + struct lcore_rx_queue { + uint16_t port_id; +- uint8_t queue_id; ++ uint16_t queue_id; + void *sec_ctx; + } __rte_cache_aligned; + +diff --git a/dpdk/examples/ipsec-secgw/ipsec_worker.c b/dpdk/examples/ipsec-secgw/ipsec_worker.c +index 8d122e8519..c9c43ebd2b 100644 +--- a/dpdk/examples/ipsec-secgw/ipsec_worker.c ++++ b/dpdk/examples/ipsec-secgw/ipsec_worker.c +@@ -1598,8 +1598,7 @@ ipsec_poll_mode_wrkr_inl_pr(void) + int32_t socket_id; + uint32_t lcore_id; + int32_t i, nb_rx; +- uint16_t portid; +- uint8_t queueid; ++ uint16_t portid, queueid; + + prev_tsc = 0; + lcore_id = rte_lcore_id(); +@@ -1633,7 +1632,7 @@ ipsec_poll_mode_wrkr_inl_pr(void) + portid = rxql[i].port_id; + queueid = rxql[i].queue_id; + RTE_LOG(INFO, IPSEC, +- " -- lcoreid=%u portid=%u rxqueueid=%hhu\n", ++ " -- lcoreid=%u portid=%u rxqueueid=%" PRIu16 "\n", + lcore_id, portid, queueid); + } + +@@ -1729,8 +1728,7 @@ ipsec_poll_mode_wrkr_inl_pr_ss(void) + uint32_t i, nb_rx, j; + int32_t socket_id; + uint32_t lcore_id; +- uint16_t portid; +- uint8_t queueid; ++ uint16_t portid, queueid; + + prev_tsc = 0; + lcore_id = rte_lcore_id(); +@@ -1764,7 +1762,7 @@ ipsec_poll_mode_wrkr_inl_pr_ss(void) + portid = rxql[i].port_id; + queueid = rxql[i].queue_id; + RTE_LOG(INFO, IPSEC, +- " -- lcoreid=%u portid=%u rxqueueid=%hhu\n", ++ " -- lcoreid=%u portid=%u rxqueueid=%" PRIu16 "\n", + lcore_id, portid, queueid); + } + +diff --git a/dpdk/examples/ipsec-secgw/ipsec_worker.h b/dpdk/examples/ipsec-secgw/ipsec_worker.h +index ac980b8bcf..8e937fda3e 100644 +--- a/dpdk/examples/ipsec-secgw/ipsec_worker.h ++++ b/dpdk/examples/ipsec-secgw/ipsec_worker.h +@@ -469,7 +469,7 @@ fail: + + static __rte_always_inline void + route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], +- uint8_t nb_pkts, uint64_t tx_offloads, bool ip_cksum) ++ uint32_t nb_pkts, uint64_t tx_offloads, bool ip_cksum) + { + uint32_t hop[MAX_PKT_BURST * 2]; + uint32_t dst_ip[MAX_PKT_BURST * 2]; +@@ -557,7 +557,7 @@ route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], + } + + static __rte_always_inline void +-route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) ++route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint32_t nb_pkts) + { + int32_t hop[MAX_PKT_BURST * 2]; + uint8_t dst_ip[MAX_PKT_BURST * 2][16]; +diff --git a/dpdk/examples/ipsec-secgw/parser.c b/dpdk/examples/ipsec-secgw/parser.c +index 98f8176651..2bd6df335b 100644 +--- a/dpdk/examples/ipsec-secgw/parser.c ++++ b/dpdk/examples/ipsec-secgw/parser.c +@@ -388,7 +388,7 @@ cfg_parse_neigh(void *parsed_result, __rte_unused struct cmdline *cl, + rc = parse_mac(res->mac, &mac); + APP_CHECK(rc == 0, st, "invalid ether addr:%s", res->mac); + rc = add_dst_ethaddr(res->port, &mac); +- APP_CHECK(rc == 0, st, "invalid port numer:%hu", res->port); ++ APP_CHECK(rc == 0, st, "invalid port number:%hu", res->port); + if (st->status < 0) + return; + } +diff --git a/dpdk/examples/l3fwd-graph/main.c b/dpdk/examples/l3fwd-graph/main.c +index 96cb1c81ff..4ded69b4a0 100644 +--- a/dpdk/examples/l3fwd-graph/main.c ++++ b/dpdk/examples/l3fwd-graph/main.c +@@ -90,7 +90,7 @@ static int pcap_trace_enable; + + struct lcore_rx_queue { + uint16_t port_id; +- uint8_t queue_id; ++ uint16_t queue_id; + char node_name[RTE_NODE_NAMESIZE]; + }; + +@@ -110,8 +110,8 @@ static struct lcore_conf lcore_conf[RTE_MAX_LCORE]; + + struct lcore_params { + uint16_t port_id; +- uint8_t queue_id; +- uint8_t lcore_id; ++ uint16_t queue_id; ++ uint32_t lcore_id; + } __rte_cache_aligned; + + static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS]; +@@ -205,19 +205,19 @@ check_worker_model_params(void) + static int + check_lcore_params(void) + { +- uint8_t queue, lcore; ++ uint16_t queue, i; + int socketid; +- uint16_t i; ++ uint32_t lcore; + + for (i = 0; i < nb_lcore_params; ++i) { + queue = lcore_params[i].queue_id; + if (queue >= MAX_RX_QUEUE_PER_PORT) { +- printf("Invalid queue number: %hhu\n", queue); ++ printf("Invalid queue number: %" PRIu16 "\n", queue); + return -1; + } + lcore = lcore_params[i].lcore_id; + if (!rte_lcore_is_enabled(lcore)) { +- printf("Error: lcore %hhu is not enabled in lcore mask\n", ++ printf("Error: lcore %u is not enabled in lcore mask\n", + lcore); + return -1; + } +@@ -228,7 +228,7 @@ check_lcore_params(void) + } + socketid = rte_lcore_to_socket_id(lcore); + if ((socketid != 0) && (numa_on == 0)) { +- printf("Warning: lcore %hhu is on socket %d with numa off\n", ++ printf("Warning: lcore %u is on socket %d with numa off\n", + lcore, socketid); + } + } +@@ -257,7 +257,7 @@ check_port_config(void) + return 0; + } + +-static uint8_t ++static uint16_t + get_port_n_rx_queues(const uint16_t port) + { + int queue = -1; +@@ -275,14 +275,14 @@ get_port_n_rx_queues(const uint16_t port) + } + } + +- return (uint8_t)(++queue); ++ return (uint16_t)(++queue); + } + + static int + init_lcore_rx_queues(void) + { + uint16_t i, nb_rx_queue; +- uint8_t lcore; ++ uint32_t lcore; + + for (i = 0; i < nb_lcore_params; ++i) { + lcore = lcore_params[i].lcore_id; +@@ -290,7 +290,7 @@ init_lcore_rx_queues(void) + if (nb_rx_queue >= MAX_RX_QUEUE_PER_LCORE) { + printf("Error: too many queues (%u) for lcore: %u\n", + (unsigned int)nb_rx_queue + 1, +- (unsigned int)lcore); ++ lcore); + return -1; + } + +@@ -448,11 +448,11 @@ parse_config(const char *q_arg) + } + + lcore_params_array[nb_lcore_params].port_id = +- (uint8_t)int_fld[FLD_PORT]; ++ (uint16_t)int_fld[FLD_PORT]; + lcore_params_array[nb_lcore_params].queue_id = +- (uint8_t)int_fld[FLD_QUEUE]; ++ (uint16_t)int_fld[FLD_QUEUE]; + lcore_params_array[nb_lcore_params].lcore_id = +- (uint8_t)int_fld[FLD_LCORE]; ++ (uint32_t)int_fld[FLD_LCORE]; + ++nb_lcore_params; + } + lcore_params = lcore_params_array; +@@ -1011,7 +1011,8 @@ main(int argc, char **argv) + "ethdev_tx-*", + "pkt_drop", + }; +- uint8_t nb_rx_queue, queue, socketid; ++ uint8_t socketid; ++ uint16_t nb_rx_queue, queue; + struct rte_graph_param graph_conf; + struct rte_eth_dev_info dev_info; + uint32_t nb_ports, nb_conf = 0; +diff --git a/dpdk/examples/l3fwd-power/main.c b/dpdk/examples/l3fwd-power/main.c +index 9c0dcd343b..996ac6dc56 100644 +--- a/dpdk/examples/l3fwd-power/main.c ++++ b/dpdk/examples/l3fwd-power/main.c +@@ -213,7 +213,7 @@ enum freq_scale_hint_t + + struct lcore_rx_queue { + uint16_t port_id; +- uint8_t queue_id; ++ uint16_t queue_id; + enum freq_scale_hint_t freq_up_hint; + uint32_t zero_rx_packet_count; + uint32_t idle_hint; +@@ -837,7 +837,7 @@ sleep_until_rx_interrupt(int num, int lcore) + struct rte_epoll_event event[num]; + int n, i; + uint16_t port_id; +- uint8_t queue_id; ++ uint16_t queue_id; + void *data; + + if (status[lcore].wakeup) { +@@ -849,9 +849,9 @@ sleep_until_rx_interrupt(int num, int lcore) + n = rte_epoll_wait(RTE_EPOLL_PER_THREAD, event, num, 10); + for (i = 0; i < n; i++) { + data = event[i].epdata.data; +- port_id = ((uintptr_t)data) >> CHAR_BIT; ++ port_id = ((uintptr_t)data) >> (sizeof(uint16_t) * CHAR_BIT); + queue_id = ((uintptr_t)data) & +- RTE_LEN2MASK(CHAR_BIT, uint8_t); ++ RTE_LEN2MASK((sizeof(uint16_t) * CHAR_BIT), uint16_t); + RTE_LOG(INFO, L3FWD_POWER, + "lcore %u is waked up from rx interrupt on" + " port %d queue %d\n", +@@ -866,7 +866,7 @@ static void turn_on_off_intr(struct lcore_conf *qconf, bool on) + { + int i; + struct lcore_rx_queue *rx_queue; +- uint8_t queue_id; ++ uint16_t queue_id; + uint16_t port_id; + + for (i = 0; i < qconf->n_rx_queue; ++i) { +@@ -886,7 +886,7 @@ static void turn_on_off_intr(struct lcore_conf *qconf, bool on) + static int event_register(struct lcore_conf *qconf) + { + struct lcore_rx_queue *rx_queue; +- uint8_t queueid; ++ uint16_t queueid; + uint16_t portid; + uint32_t data; + int ret; +@@ -896,7 +896,7 @@ static int event_register(struct lcore_conf *qconf) + rx_queue = &(qconf->rx_queue_list[i]); + portid = rx_queue->port_id; + queueid = rx_queue->queue_id; +- data = portid << CHAR_BIT | queueid; ++ data = portid << (sizeof(uint16_t) * CHAR_BIT) | queueid; + + ret = rte_eth_dev_rx_intr_ctl_q(portid, queueid, + RTE_EPOLL_PER_THREAD, +@@ -916,8 +916,7 @@ static int main_intr_loop(__rte_unused void *dummy) + unsigned int lcore_id; + uint64_t prev_tsc, diff_tsc, cur_tsc; + int i, j, nb_rx; +- uint8_t queueid; +- uint16_t portid; ++ uint16_t portid, queueid; + struct lcore_conf *qconf; + struct lcore_rx_queue *rx_queue; + uint32_t lcore_rx_idle_count = 0; +@@ -945,7 +944,7 @@ static int main_intr_loop(__rte_unused void *dummy) + portid = qconf->rx_queue_list[i].port_id; + queueid = qconf->rx_queue_list[i].queue_id; + RTE_LOG(INFO, L3FWD_POWER, +- " -- lcoreid=%u portid=%u rxqueueid=%hhu\n", ++ " -- lcoreid=%u portid=%u rxqueueid=%" PRIu16 "\n", + lcore_id, portid, queueid); + } + +@@ -1082,8 +1081,7 @@ main_telemetry_loop(__rte_unused void *dummy) + unsigned int lcore_id; + uint64_t prev_tsc, diff_tsc, cur_tsc, prev_tel_tsc; + int i, j, nb_rx; +- uint8_t queueid; +- uint16_t portid; ++ uint16_t portid, queueid; + struct lcore_conf *qconf; + struct lcore_rx_queue *rx_queue; + uint64_t ep_nep[2] = {0}, fp_nfp[2] = {0}; +@@ -1113,7 +1111,7 @@ main_telemetry_loop(__rte_unused void *dummy) + portid = qconf->rx_queue_list[i].port_id; + queueid = qconf->rx_queue_list[i].queue_id; + RTE_LOG(INFO, L3FWD_POWER, " -- lcoreid=%u portid=%u " +- "rxqueueid=%hhu\n", lcore_id, portid, queueid); ++ "rxqueueid=%" PRIu16 "\n", lcore_id, portid, queueid); + } + + while (!is_done()) { +@@ -1204,8 +1202,7 @@ main_legacy_loop(__rte_unused void *dummy) + uint64_t prev_tsc, diff_tsc, cur_tsc, tim_res_tsc, hz; + uint64_t prev_tsc_power = 0, cur_tsc_power, diff_tsc_power; + int i, j, nb_rx; +- uint8_t queueid; +- uint16_t portid; ++ uint16_t portid, queueid; + struct lcore_conf *qconf; + struct lcore_rx_queue *rx_queue; + enum freq_scale_hint_t lcore_scaleup_hint; +@@ -1233,7 +1230,7 @@ main_legacy_loop(__rte_unused void *dummy) + portid = qconf->rx_queue_list[i].port_id; + queueid = qconf->rx_queue_list[i].queue_id; + RTE_LOG(INFO, L3FWD_POWER, " -- lcoreid=%u portid=%u " +- "rxqueueid=%hhu\n", lcore_id, portid, queueid); ++ "rxqueueid=%" PRIu16 "\n", lcore_id, portid, queueid); + } + + /* add into event wait list */ +@@ -1398,25 +1395,25 @@ start_rx: + static int + check_lcore_params(void) + { +- uint8_t queue, lcore; +- uint16_t i; ++ uint16_t queue, i; ++ uint32_t lcore; + int socketid; + + for (i = 0; i < nb_lcore_params; ++i) { + queue = lcore_params[i].queue_id; + if (queue >= MAX_RX_QUEUE_PER_PORT) { +- printf("invalid queue number: %hhu\n", queue); ++ printf("invalid queue number: %" PRIu16 "\n", queue); + return -1; + } + lcore = lcore_params[i].lcore_id; + if (!rte_lcore_is_enabled(lcore)) { +- printf("error: lcore %hhu is not enabled in lcore " ++ printf("error: lcore %u is not enabled in lcore " + "mask\n", lcore); + return -1; + } + if ((socketid = rte_lcore_to_socket_id(lcore) != 0) && + (numa_on == 0)) { +- printf("warning: lcore %hhu is on socket %d with numa " ++ printf("warning: lcore %u is on socket %d with numa " + "off\n", lcore, socketid); + } + if (app_mode == APP_MODE_TELEMETRY && lcore == rte_lcore_id()) { +@@ -1450,7 +1447,7 @@ check_port_config(void) + return 0; + } + +-static uint8_t ++static uint16_t + get_port_n_rx_queues(const uint16_t port) + { + int queue = -1; +@@ -1461,21 +1458,21 @@ get_port_n_rx_queues(const uint16_t port) + lcore_params[i].queue_id > queue) + queue = lcore_params[i].queue_id; + } +- return (uint8_t)(++queue); ++ return (uint16_t)(++queue); + } + + static int + init_lcore_rx_queues(void) + { + uint16_t i, nb_rx_queue; +- uint8_t lcore; ++ uint32_t lcore; + + for (i = 0; i < nb_lcore_params; ++i) { + lcore = lcore_params[i].lcore_id; + nb_rx_queue = lcore_conf[lcore].n_rx_queue; + if (nb_rx_queue >= MAX_RX_QUEUE_PER_LCORE) { + printf("error: too many queues (%u) for lcore: %u\n", +- (unsigned)nb_rx_queue + 1, (unsigned)lcore); ++ (unsigned int)nb_rx_queue + 1, lcore); + return -1; + } else { + lcore_conf[lcore].rx_queue_list[nb_rx_queue].port_id = +@@ -1660,6 +1657,11 @@ parse_config(const char *q_arg) + char *str_fld[_NUM_FLD]; + int i; + unsigned size; ++ unsigned int max_fld[_NUM_FLD] = { ++ RTE_MAX_ETHPORTS, ++ RTE_MAX_QUEUES_PER_PORT, ++ RTE_MAX_LCORE ++ }; + + nb_lcore_params = 0; + +@@ -1679,8 +1681,7 @@ parse_config(const char *q_arg) + for (i = 0; i < _NUM_FLD; i++){ + errno = 0; + int_fld[i] = strtoul(str_fld[i], &end, 0); +- if (errno != 0 || end == str_fld[i] || int_fld[i] > +- 255) ++ if (errno != 0 || end == str_fld[i] || int_fld[i] > max_fld[i]) + return -1; + } + if (nb_lcore_params >= MAX_LCORE_PARAMS) { +@@ -1689,11 +1690,11 @@ parse_config(const char *q_arg) + return -1; + } + lcore_params_array[nb_lcore_params].port_id = +- (uint8_t)int_fld[FLD_PORT]; ++ (uint16_t)int_fld[FLD_PORT]; + lcore_params_array[nb_lcore_params].queue_id = +- (uint8_t)int_fld[FLD_QUEUE]; ++ (uint16_t)int_fld[FLD_QUEUE]; + lcore_params_array[nb_lcore_params].lcore_id = +- (uint8_t)int_fld[FLD_LCORE]; ++ (uint32_t)int_fld[FLD_LCORE]; + ++nb_lcore_params; + } + lcore_params = lcore_params_array; +@@ -2500,8 +2501,8 @@ main(int argc, char **argv) + uint64_t hz; + uint32_t n_tx_queue, nb_lcores; + uint32_t dev_rxq_num, dev_txq_num; +- uint8_t nb_rx_queue, queue, socketid; +- uint16_t portid; ++ uint8_t socketid; ++ uint16_t portid, nb_rx_queue, queue; + const char *ptr_strings[NUM_TELSTATS]; + + /* init EAL */ +diff --git a/dpdk/examples/l3fwd-power/main.h b/dpdk/examples/l3fwd-power/main.h +index 258de98f5b..194bd82102 100644 +--- a/dpdk/examples/l3fwd-power/main.h ++++ b/dpdk/examples/l3fwd-power/main.h +@@ -9,8 +9,8 @@ + #define MAX_LCORE_PARAMS 1024 + struct lcore_params { + uint16_t port_id; +- uint8_t queue_id; +- uint8_t lcore_id; ++ uint16_t queue_id; ++ uint32_t lcore_id; + } __rte_cache_aligned; + + extern struct lcore_params *lcore_params; +diff --git a/dpdk/examples/l3fwd-power/perf_core.c b/dpdk/examples/l3fwd-power/perf_core.c +index 41ef6d0c9a..e4bdb62121 100644 +--- a/dpdk/examples/l3fwd-power/perf_core.c ++++ b/dpdk/examples/l3fwd-power/perf_core.c +@@ -22,9 +22,9 @@ static uint16_t nb_hp_lcores; + + struct perf_lcore_params { + uint16_t port_id; +- uint8_t queue_id; ++ uint16_t queue_id; + uint8_t high_perf; +- uint8_t lcore_idx; ++ uint32_t lcore_idx; + } __rte_cache_aligned; + + static struct perf_lcore_params prf_lc_prms[MAX_LCORE_PARAMS]; +@@ -132,6 +132,12 @@ parse_perf_config(const char *q_arg) + char *str_fld[_NUM_FLD]; + int i; + unsigned int size; ++ unsigned int max_fld[_NUM_FLD] = { ++ RTE_MAX_ETHPORTS, ++ RTE_MAX_QUEUES_PER_PORT, ++ 255, ++ RTE_MAX_LCORE ++ }; + + nb_prf_lc_prms = 0; + +@@ -152,7 +158,8 @@ parse_perf_config(const char *q_arg) + for (i = 0; i < _NUM_FLD; i++) { + errno = 0; + int_fld[i] = strtoul(str_fld[i], &end, 0); +- if (errno != 0 || end == str_fld[i] || int_fld[i] > 255) ++ if (errno != 0 || end == str_fld[i] || int_fld[i] > max_fld[i]) ++ + return -1; + } + if (nb_prf_lc_prms >= MAX_LCORE_PARAMS) { +@@ -161,13 +168,13 @@ parse_perf_config(const char *q_arg) + return -1; + } + prf_lc_prms[nb_prf_lc_prms].port_id = +- (uint8_t)int_fld[FLD_PORT]; ++ (uint16_t)int_fld[FLD_PORT]; + prf_lc_prms[nb_prf_lc_prms].queue_id = +- (uint8_t)int_fld[FLD_QUEUE]; ++ (uint16_t)int_fld[FLD_QUEUE]; + prf_lc_prms[nb_prf_lc_prms].high_perf = + !!(uint8_t)int_fld[FLD_LCORE_HP]; + prf_lc_prms[nb_prf_lc_prms].lcore_idx = +- (uint8_t)int_fld[FLD_LCORE_IDX]; ++ (uint32_t)int_fld[FLD_LCORE_IDX]; + ++nb_prf_lc_prms; + } + +diff --git a/dpdk/examples/l3fwd/l3fwd.h b/dpdk/examples/l3fwd/l3fwd.h +index e7ae0e5834..12c264cb4c 100644 +--- a/dpdk/examples/l3fwd/l3fwd.h ++++ b/dpdk/examples/l3fwd/l3fwd.h +@@ -74,7 +74,7 @@ struct mbuf_table { + + struct lcore_rx_queue { + uint16_t port_id; +- uint8_t queue_id; ++ uint16_t queue_id; + } __rte_cache_aligned; + + struct lcore_conf { +diff --git a/dpdk/examples/l3fwd/l3fwd_acl.c b/dpdk/examples/l3fwd/l3fwd_acl.c +index 401692bcec..31798ccb10 100644 +--- a/dpdk/examples/l3fwd/l3fwd_acl.c ++++ b/dpdk/examples/l3fwd/l3fwd_acl.c +@@ -962,8 +962,6 @@ setup_acl(const int socket_id) + acl_log("IPv6 ACL entries %u:\n", acl_num_ipv6); + dump_ipv6_rules((struct acl6_rule *)acl_base_ipv6, acl_num_ipv6, 1); + +- memset(&acl_config, 0, sizeof(acl_config)); +- + /* Check sockets a context should be created on */ + if (socket_id >= NB_SOCKETS) { + acl_log("Socket %d is out " +@@ -973,6 +971,9 @@ setup_acl(const int socket_id) + return; + } + ++ rte_acl_free(acl_config.acx_ipv4[socket_id]); ++ rte_acl_free(acl_config.acx_ipv6[socket_id]); ++ + acl_config.acx_ipv4[socket_id] = app_acl_init(route_base_ipv4, + acl_base_ipv4, route_num_ipv4, acl_num_ipv4, + 0, socket_id); +@@ -997,7 +998,7 @@ acl_main_loop(__rte_unused void *dummy) + uint64_t prev_tsc, diff_tsc, cur_tsc; + int i, nb_rx; + uint16_t portid; +- uint8_t queueid; ++ uint16_t queueid; + struct lcore_conf *qconf; + int socketid; + const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) +@@ -1020,7 +1021,7 @@ acl_main_loop(__rte_unused void *dummy) + portid = qconf->rx_queue_list[i].port_id; + queueid = qconf->rx_queue_list[i].queue_id; + RTE_LOG(INFO, L3FWD, +- " -- lcoreid=%u portid=%u rxqueueid=%hhu\n", ++ " -- lcoreid=%u portid=%u rxqueueid=%" PRIu16 "\n", + lcore_id, portid, queueid); + } + +@@ -1073,9 +1074,9 @@ acl_main_loop(__rte_unused void *dummy) + + l3fwd_acl_send_packets( + qconf, +- pkts_burst, ++ acl_search.m_ipv4, + acl_search.res_ipv4, +- nb_rx); ++ acl_search.num_ipv4); + } + + if (acl_search.num_ipv6) { +@@ -1088,9 +1089,9 @@ acl_main_loop(__rte_unused void *dummy) + + l3fwd_acl_send_packets( + qconf, +- pkts_burst, ++ acl_search.m_ipv6, + acl_search.res_ipv6, +- nb_rx); ++ acl_search.num_ipv6); + } + } + } +diff --git a/dpdk/examples/l3fwd/l3fwd_em.c b/dpdk/examples/l3fwd/l3fwd_em.c +index 40e102b38a..f18ac0048b 100644 +--- a/dpdk/examples/l3fwd/l3fwd_em.c ++++ b/dpdk/examples/l3fwd/l3fwd_em.c +@@ -586,7 +586,7 @@ em_main_loop(__rte_unused void *dummy) + unsigned lcore_id; + uint64_t prev_tsc, diff_tsc, cur_tsc; + int i, nb_rx; +- uint8_t queueid; ++ uint16_t queueid; + uint16_t portid; + struct lcore_conf *qconf; + const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / +@@ -609,7 +609,7 @@ em_main_loop(__rte_unused void *dummy) + portid = qconf->rx_queue_list[i].port_id; + queueid = qconf->rx_queue_list[i].queue_id; + RTE_LOG(INFO, L3FWD, +- " -- lcoreid=%u portid=%u rxqueueid=%hhu\n", ++ " -- lcoreid=%u portid=%u rxqueueid=%" PRIu16 "\n", + lcore_id, portid, queueid); + } + +diff --git a/dpdk/examples/l3fwd/l3fwd_event.h b/dpdk/examples/l3fwd/l3fwd_event.h +index 9aad358003..c6a4a89127 100644 +--- a/dpdk/examples/l3fwd/l3fwd_event.h ++++ b/dpdk/examples/l3fwd/l3fwd_event.h +@@ -78,8 +78,8 @@ struct l3fwd_event_resources { + uint8_t deq_depth; + uint8_t has_burst; + uint8_t enabled; +- uint8_t eth_rx_queues; + uint8_t vector_enabled; ++ uint16_t eth_rx_queues; + uint16_t vector_size; + uint64_t vector_tmo_ns; + }; +diff --git a/dpdk/examples/l3fwd/l3fwd_fib.c b/dpdk/examples/l3fwd/l3fwd_fib.c +index 6a21984415..f38b19af3f 100644 +--- a/dpdk/examples/l3fwd/l3fwd_fib.c ++++ b/dpdk/examples/l3fwd/l3fwd_fib.c +@@ -186,7 +186,7 @@ fib_main_loop(__rte_unused void *dummy) + uint64_t prev_tsc, diff_tsc, cur_tsc; + int i, nb_rx; + uint16_t portid; +- uint8_t queueid; ++ uint16_t queueid; + struct lcore_conf *qconf; + const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / + US_PER_S * BURST_TX_DRAIN_US; +@@ -208,7 +208,7 @@ fib_main_loop(__rte_unused void *dummy) + portid = qconf->rx_queue_list[i].port_id; + queueid = qconf->rx_queue_list[i].queue_id; + RTE_LOG(INFO, L3FWD, +- " -- lcoreid=%u portid=%u rxqueueid=%hhu\n", ++ " -- lcoreid=%u portid=%u rxqueueid=%" PRIu16 "\n", + lcore_id, portid, queueid); + } + +diff --git a/dpdk/examples/l3fwd/l3fwd_lpm.c b/dpdk/examples/l3fwd/l3fwd_lpm.c +index a484a33089..e8fd95aae9 100644 +--- a/dpdk/examples/l3fwd/l3fwd_lpm.c ++++ b/dpdk/examples/l3fwd/l3fwd_lpm.c +@@ -148,8 +148,7 @@ lpm_main_loop(__rte_unused void *dummy) + unsigned lcore_id; + uint64_t prev_tsc, diff_tsc, cur_tsc; + int i, nb_rx; +- uint16_t portid; +- uint8_t queueid; ++ uint16_t portid, queueid; + struct lcore_conf *qconf; + const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / + US_PER_S * BURST_TX_DRAIN_US; +@@ -171,7 +170,7 @@ lpm_main_loop(__rte_unused void *dummy) + portid = qconf->rx_queue_list[i].port_id; + queueid = qconf->rx_queue_list[i].queue_id; + RTE_LOG(INFO, L3FWD, +- " -- lcoreid=%u portid=%u rxqueueid=%hhu\n", ++ " -- lcoreid=%u portid=%u rxqueueid=%" PRIu16 "\n", + lcore_id, portid, queueid); + } + +diff --git a/dpdk/examples/l3fwd/main.c b/dpdk/examples/l3fwd/main.c +index 3bf28aec0c..a239869ada 100644 +--- a/dpdk/examples/l3fwd/main.c ++++ b/dpdk/examples/l3fwd/main.c +@@ -98,8 +98,8 @@ struct parm_cfg parm_config; + + struct lcore_params { + uint16_t port_id; +- uint8_t queue_id; +- uint8_t lcore_id; ++ uint16_t queue_id; ++ uint32_t lcore_id; + } __rte_cache_aligned; + + static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS]; +@@ -292,24 +292,24 @@ setup_l3fwd_lookup_tables(void) + static int + check_lcore_params(void) + { +- uint8_t queue, lcore; +- uint16_t i; ++ uint16_t queue, i; ++ uint32_t lcore; + int socketid; + + for (i = 0; i < nb_lcore_params; ++i) { + queue = lcore_params[i].queue_id; + if (queue >= MAX_RX_QUEUE_PER_PORT) { +- printf("invalid queue number: %hhu\n", queue); ++ printf("invalid queue number: %" PRIu16 "\n", queue); + return -1; + } + lcore = lcore_params[i].lcore_id; + if (!rte_lcore_is_enabled(lcore)) { +- printf("error: lcore %hhu is not enabled in lcore mask\n", lcore); ++ printf("error: lcore %u is not enabled in lcore mask\n", lcore); + return -1; + } + if ((socketid = rte_lcore_to_socket_id(lcore) != 0) && + (numa_on == 0)) { +- printf("warning: lcore %hhu is on socket %d with numa off \n", ++ printf("warning: lcore %u is on socket %d with numa off\n", + lcore, socketid); + } + } +@@ -336,7 +336,7 @@ check_port_config(void) + return 0; + } + +-static uint8_t ++static uint16_t + get_port_n_rx_queues(const uint16_t port) + { + int queue = -1; +@@ -352,21 +352,21 @@ get_port_n_rx_queues(const uint16_t port) + lcore_params[i].port_id); + } + } +- return (uint8_t)(++queue); ++ return (uint16_t)(++queue); + } + + static int + init_lcore_rx_queues(void) + { + uint16_t i, nb_rx_queue; +- uint8_t lcore; ++ uint32_t lcore; + + for (i = 0; i < nb_lcore_params; ++i) { + lcore = lcore_params[i].lcore_id; + nb_rx_queue = lcore_conf[lcore].n_rx_queue; + if (nb_rx_queue >= MAX_RX_QUEUE_PER_LCORE) { + printf("error: too many queues (%u) for lcore: %u\n", +- (unsigned)nb_rx_queue + 1, (unsigned)lcore); ++ (unsigned int)nb_rx_queue + 1, lcore); + return -1; + } else { + lcore_conf[lcore].rx_queue_list[nb_rx_queue].port_id = +@@ -500,6 +500,11 @@ parse_config(const char *q_arg) + char *str_fld[_NUM_FLD]; + int i; + unsigned size; ++ uint16_t max_fld[_NUM_FLD] = { ++ RTE_MAX_ETHPORTS, ++ RTE_MAX_QUEUES_PER_PORT, ++ RTE_MAX_LCORE ++ }; + + nb_lcore_params = 0; + +@@ -518,7 +523,7 @@ parse_config(const char *q_arg) + for (i = 0; i < _NUM_FLD; i++){ + errno = 0; + int_fld[i] = strtoul(str_fld[i], &end, 0); +- if (errno != 0 || end == str_fld[i] || int_fld[i] > 255) ++ if (errno != 0 || end == str_fld[i] || int_fld[i] > max_fld[i]) + return -1; + } + if (nb_lcore_params >= MAX_LCORE_PARAMS) { +@@ -527,11 +532,11 @@ parse_config(const char *q_arg) + return -1; + } + lcore_params_array[nb_lcore_params].port_id = +- (uint8_t)int_fld[FLD_PORT]; ++ (uint16_t)int_fld[FLD_PORT]; + lcore_params_array[nb_lcore_params].queue_id = +- (uint8_t)int_fld[FLD_QUEUE]; ++ (uint16_t)int_fld[FLD_QUEUE]; + lcore_params_array[nb_lcore_params].lcore_id = +- (uint8_t)int_fld[FLD_LCORE]; ++ (uint32_t)int_fld[FLD_LCORE]; + ++nb_lcore_params; + } + lcore_params = lcore_params_array; +@@ -630,7 +635,7 @@ parse_event_eth_rx_queues(const char *eth_rx_queues) + { + struct l3fwd_event_resources *evt_rsrc = l3fwd_get_eventdev_rsrc(); + char *end = NULL; +- uint8_t num_eth_rx_queues; ++ uint16_t num_eth_rx_queues; + + /* parse decimal string */ + num_eth_rx_queues = strtoul(eth_rx_queues, &end, 10); +@@ -1211,7 +1216,8 @@ config_port_max_pkt_len(struct rte_eth_conf *conf, + static void + l3fwd_poll_resource_setup(void) + { +- uint8_t nb_rx_queue, queue, socketid; ++ uint8_t socketid; ++ uint16_t nb_rx_queue, queue; + struct rte_eth_dev_info dev_info; + uint32_t n_tx_queue, nb_lcores; + struct rte_eth_txconf *txconf; +@@ -1388,6 +1394,7 @@ l3fwd_poll_resource_setup(void) + fflush(stdout); + /* init RX queues */ + for(queue = 0; queue < qconf->n_rx_queue; ++queue) { ++ struct rte_eth_conf local_conf; + struct rte_eth_rxconf rxq_conf; + + portid = qconf->rx_queue_list[queue].port_id; +@@ -1408,8 +1415,14 @@ l3fwd_poll_resource_setup(void) + "Error during getting device (port %u) info: %s\n", + portid, strerror(-ret)); + ++ ret = rte_eth_dev_conf_get(portid, &local_conf); ++ if (ret != 0) ++ rte_exit(EXIT_FAILURE, ++ "Error during getting device (port %u) configuration: %s\n", ++ portid, strerror(-ret)); ++ + rxq_conf = dev_info.default_rxconf; +- rxq_conf.offloads = port_conf.rxmode.offloads; ++ rxq_conf.offloads = local_conf.rxmode.offloads; + if (!per_port_pool) + ret = rte_eth_rx_queue_setup(portid, queueid, + nb_rxd, socketid, +@@ -1528,7 +1541,7 @@ main(int argc, char **argv) + struct lcore_conf *qconf; + uint16_t queueid, portid; + unsigned int lcore_id; +- uint8_t queue; ++ uint16_t queue; + int ret; + + /* init EAL */ +@@ -1577,7 +1590,6 @@ main(int argc, char **argv) + l3fwd_lkp.main_loop = evt_rsrc->ops.fib_event_loop; + else + l3fwd_lkp.main_loop = evt_rsrc->ops.lpm_event_loop; +- l3fwd_event_service_setup(); + } else + #endif + l3fwd_poll_resource_setup(); +@@ -1609,6 +1621,11 @@ main(int argc, char **argv) + } + } + ++#ifdef RTE_LIB_EVENTDEV ++ if (evt_rsrc->enabled) ++ l3fwd_event_service_setup(); ++#endif ++ + printf("\n"); + + for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { +diff --git a/dpdk/examples/packet_ordering/main.c b/dpdk/examples/packet_ordering/main.c +index d2fd6f77e4..f839db9102 100644 +--- a/dpdk/examples/packet_ordering/main.c ++++ b/dpdk/examples/packet_ordering/main.c +@@ -5,6 +5,7 @@ + #include + #include + #include ++#include + + #include + #include +@@ -427,8 +428,8 @@ int_handler(int sig_num) + * The mbufs are then passed to the worker threads via the rx_to_workers + * ring. + */ +-static int +-rx_thread(struct rte_ring *ring_out) ++static __rte_always_inline int ++rx_thread(struct rte_ring *ring_out, bool disable_reorder_flag) + { + uint32_t seqn = 0; + uint16_t i, ret = 0; +@@ -454,9 +455,11 @@ rx_thread(struct rte_ring *ring_out) + } + app_stats.rx.rx_pkts += nb_rx_pkts; + +- /* mark sequence number */ +- for (i = 0; i < nb_rx_pkts; ) +- *rte_reorder_seqn(pkts[i++]) = seqn++; ++ /* mark sequence number if reorder is enabled */ ++ if (!disable_reorder_flag) { ++ for (i = 0; i < nb_rx_pkts;) ++ *rte_reorder_seqn(pkts[i++]) = seqn++; ++ } + + /* enqueue to rx_to_workers ring */ + ret = rte_ring_enqueue_burst(ring_out, +@@ -473,6 +476,18 @@ rx_thread(struct rte_ring *ring_out) + return 0; + } + ++static __rte_noinline int ++rx_thread_reorder(struct rte_ring *ring_out) ++{ ++ return rx_thread(ring_out, false); ++} ++ ++static __rte_noinline int ++rx_thread_reorder_disabled(struct rte_ring *ring_out) ++{ ++ return rx_thread(ring_out, true); ++} ++ + /** + * This thread takes bursts of packets from the rx_to_workers ring and + * Changes the input port value to output port value. And feds it to +@@ -772,8 +787,11 @@ main(int argc, char **argv) + (void *)&send_args, last_lcore_id); + } + +- /* Start rx_thread() on the main core */ +- rx_thread(rx_to_workers); ++ /* Start rx_thread_xxx() on the main core */ ++ if (disable_reorder) ++ rx_thread_reorder_disabled(rx_to_workers); ++ else ++ rx_thread_reorder(rx_to_workers); + + RTE_LCORE_FOREACH_WORKER(lcore_id) { + if (rte_eal_wait_lcore(lcore_id) < 0) +diff --git a/dpdk/examples/pipeline/cli.c b/dpdk/examples/pipeline/cli.c +index 2ae6cc579f..afb143c01f 100644 +--- a/dpdk/examples/pipeline/cli.c ++++ b/dpdk/examples/pipeline/cli.c +@@ -714,6 +714,7 @@ cmd_pipeline_libbuild(char **tokens, + "-I %s/lib/eal/include " + "-I %s/lib/eal/x86/include " + "-I %s/lib/eal/include/generic " ++ "-I %s/lib/log " + "-I %s/lib/meter " + "-I %s/lib/port " + "-I %s/lib/table " +@@ -738,6 +739,7 @@ cmd_pipeline_libbuild(char **tokens, + install_dir, + install_dir, + install_dir, ++ install_dir, + log_file, + obj_file, + lib_file, +diff --git a/dpdk/examples/qos_sched/args.c b/dpdk/examples/qos_sched/args.c +index e97273152a..886542b3c1 100644 +--- a/dpdk/examples/qos_sched/args.c ++++ b/dpdk/examples/qos_sched/args.c +@@ -103,8 +103,10 @@ app_parse_opt_vals(const char *conf_str, char separator, uint32_t n_vals, uint32 + + n_tokens = rte_strsplit(string, strnlen(string, 32), tokens, n_vals, separator); + +- if (n_tokens > MAX_OPT_VALUES) ++ if (n_tokens > MAX_OPT_VALUES) { ++ free(string); + return -1; ++ } + + for (i = 0; i < n_tokens; i++) + opt_vals[i] = (uint32_t)atol(tokens[i]); +@@ -182,10 +184,10 @@ app_parse_flow_conf(const char *conf_str) + + pconf->rx_port = vals[0]; + pconf->tx_port = vals[1]; +- pconf->rx_core = (uint8_t)vals[2]; +- pconf->wt_core = (uint8_t)vals[3]; ++ pconf->rx_core = vals[2]; ++ pconf->wt_core = vals[3]; + if (ret == 5) +- pconf->tx_core = (uint8_t)vals[4]; ++ pconf->tx_core = vals[4]; + else + pconf->tx_core = pconf->wt_core; + +diff --git a/dpdk/examples/qos_sched/init.c b/dpdk/examples/qos_sched/init.c +index d8abae635a..32964fd57e 100644 +--- a/dpdk/examples/qos_sched/init.c ++++ b/dpdk/examples/qos_sched/init.c +@@ -335,7 +335,7 @@ int app_init(void) + for(i = 0; i < nb_pfc; i++) { + uint32_t socket = rte_lcore_to_socket_id(qos_conf[i].rx_core); + struct rte_ring *ring; +- struct rte_eth_link link = {0}; ++ struct rte_eth_link link; + int retry_count = 100, retry_delay = 100; /* try every 100ms for 10 sec */ + + snprintf(ring_name, MAX_NAME_LEN, "ring-%u-%u", i, qos_conf[i].rx_core); +@@ -367,6 +367,7 @@ int app_init(void) + app_init_port(qos_conf[i].rx_port, qos_conf[i].mbuf_pool); + app_init_port(qos_conf[i].tx_port, qos_conf[i].mbuf_pool); + ++ memset(&link, 0, sizeof(link)); + rte_eth_link_get(qos_conf[i].tx_port, &link); + if (link.link_status == 0) + printf("Waiting for link on port %u\n", qos_conf[i].tx_port); +diff --git a/dpdk/examples/vhost/main.c b/dpdk/examples/vhost/main.c +index ce5c1efddf..3fc1b151d1 100644 +--- a/dpdk/examples/vhost/main.c ++++ b/dpdk/examples/vhost/main.c +@@ -260,6 +260,9 @@ open_dma(const char *value) + char *dma_arg[RTE_MAX_VHOST_DEVICE]; + int args_nr; + ++ if (input == NULL) ++ return -1; ++ + while (isblank(*addrs)) + addrs++; + if (*addrs == '\0') { +diff --git a/dpdk/examples/vm_power_manager/guest_cli/vm_power_cli_guest.c b/dpdk/examples/vm_power_manager/guest_cli/vm_power_cli_guest.c +index 94bfbbaf78..5eddb47847 100644 +--- a/dpdk/examples/vm_power_manager/guest_cli/vm_power_cli_guest.c ++++ b/dpdk/examples/vm_power_manager/guest_cli/vm_power_cli_guest.c +@@ -401,7 +401,7 @@ check_response_cmd(unsigned int lcore_id, int *result) + + struct cmd_set_cpu_freq_result { + cmdline_fixed_string_t set_cpu_freq; +- uint8_t lcore_id; ++ uint32_t lcore_id; + cmdline_fixed_string_t cmd; + }; + +@@ -444,7 +444,7 @@ cmdline_parse_token_string_t cmd_set_cpu_freq = + set_cpu_freq, "set_cpu_freq"); + cmdline_parse_token_num_t cmd_set_cpu_freq_core_num = + TOKEN_NUM_INITIALIZER(struct cmd_set_cpu_freq_result, +- lcore_id, RTE_UINT8); ++ lcore_id, RTE_UINT32); + cmdline_parse_token_string_t cmd_set_cpu_freq_cmd_cmd = + TOKEN_STRING_INITIALIZER(struct cmd_set_cpu_freq_result, + cmd, "up#down#min#max#enable_turbo#disable_turbo"); +diff --git a/dpdk/kernel/freebsd/nic_uio/nic_uio.c b/dpdk/kernel/freebsd/nic_uio/nic_uio.c +index 7a81694c92..0043892870 100644 +--- a/dpdk/kernel/freebsd/nic_uio/nic_uio.c ++++ b/dpdk/kernel/freebsd/nic_uio/nic_uio.c +@@ -78,10 +78,14 @@ struct pci_bdf { + uint32_t function; + }; + +-static devclass_t nic_uio_devclass; +- + DEFINE_CLASS_0(nic_uio, nic_uio_driver, nic_uio_methods, sizeof(struct nic_uio_softc)); ++ ++#if __FreeBSD_version < 1400000 ++static devclass_t nic_uio_devclass; + DRIVER_MODULE(nic_uio, pci, nic_uio_driver, nic_uio_devclass, nic_uio_modevent, 0); ++#else ++DRIVER_MODULE(nic_uio, pci, nic_uio_driver, nic_uio_modevent, 0); ++#endif + + static int + nic_uio_mmap(struct cdev *cdev, vm_ooffset_t offset, vm_paddr_t *paddr, +diff --git a/dpdk/lib/bbdev/rte_bbdev.c b/dpdk/lib/bbdev/rte_bbdev.c +index cfebea09c7..e09bb97abb 100644 +--- a/dpdk/lib/bbdev/rte_bbdev.c ++++ b/dpdk/lib/bbdev/rte_bbdev.c +@@ -1106,12 +1106,12 @@ rte_bbdev_queue_intr_ctl(uint16_t dev_id, uint16_t queue_id, int epfd, int op, + + intr_handle = dev->intr_handle; + if (intr_handle == NULL) { +- rte_bbdev_log(ERR, "Device %u intr handle unset\n", dev_id); ++ rte_bbdev_log(ERR, "Device %u intr handle unset", dev_id); + return -ENOTSUP; + } + + if (queue_id >= RTE_MAX_RXTX_INTR_VEC_ID) { +- rte_bbdev_log(ERR, "Device %u queue_id %u is too big\n", ++ rte_bbdev_log(ERR, "Device %u queue_id %u is too big", + dev_id, queue_id); + return -ENOTSUP; + } +@@ -1120,7 +1120,7 @@ rte_bbdev_queue_intr_ctl(uint16_t dev_id, uint16_t queue_id, int epfd, int op, + ret = rte_intr_rx_ctl(intr_handle, epfd, op, vec, data); + if (ret && (ret != -EEXIST)) { + rte_bbdev_log(ERR, +- "dev %u q %u int ctl error op %d epfd %d vec %u\n", ++ "dev %u q %u int ctl error op %d epfd %d vec %u", + dev_id, queue_id, op, epfd, vec); + return ret; + } +diff --git a/dpdk/lib/bpf/bpf_validate.c b/dpdk/lib/bpf/bpf_validate.c +index 95b9ef99ef..da8d5f3deb 100644 +--- a/dpdk/lib/bpf/bpf_validate.c ++++ b/dpdk/lib/bpf/bpf_validate.c +@@ -29,10 +29,13 @@ struct bpf_reg_val { + }; + + struct bpf_eval_state { ++ SLIST_ENTRY(bpf_eval_state) next; /* for @safe list traversal */ + struct bpf_reg_val rv[EBPF_REG_NUM]; + struct bpf_reg_val sv[MAX_BPF_STACK_SIZE / sizeof(uint64_t)]; + }; + ++SLIST_HEAD(bpf_evst_head, bpf_eval_state); ++ + /* possible instruction node colour */ + enum { + WHITE, +@@ -52,6 +55,9 @@ enum { + + #define MAX_EDGES 2 + ++/* max number of 'safe' evaluated states to track per node */ ++#define NODE_EVST_MAX 32 ++ + struct inst_node { + uint8_t colour; + uint8_t nb_edge:4; +@@ -59,7 +65,18 @@ struct inst_node { + uint8_t edge_type[MAX_EDGES]; + uint32_t edge_dest[MAX_EDGES]; + uint32_t prev_node; +- struct bpf_eval_state *evst; ++ struct { ++ struct bpf_eval_state *cur; /* save/restore for jcc targets */ ++ struct bpf_eval_state *start; ++ struct bpf_evst_head safe; /* safe states for track/prune */ ++ uint32_t nb_safe; ++ } evst; ++}; ++ ++struct evst_pool { ++ uint32_t num; ++ uint32_t cur; ++ struct bpf_eval_state *ent; + }; + + struct bpf_verifier { +@@ -73,11 +90,8 @@ struct bpf_verifier { + uint32_t edge_type[MAX_EDGE_TYPE]; + struct bpf_eval_state *evst; + struct inst_node *evin; +- struct { +- uint32_t num; +- uint32_t cur; +- struct bpf_eval_state *ent; +- } evst_pool; ++ struct evst_pool evst_sr_pool; /* for evst save/restore */ ++ struct evst_pool evst_tp_pool; /* for evst track/prune */ + }; + + struct bpf_ins_check { +@@ -636,14 +650,14 @@ eval_alu(struct bpf_verifier *bvf, const struct ebpf_insn *ins) + { + uint64_t msk; + uint32_t op; +- size_t opsz; ++ size_t opsz, sz; + const char *err; + struct bpf_eval_state *st; + struct bpf_reg_val *rd, rs; + +- opsz = (BPF_CLASS(ins->code) == BPF_ALU) ? ++ sz = (BPF_CLASS(ins->code) == BPF_ALU) ? + sizeof(uint32_t) : sizeof(uint64_t); +- opsz = opsz * CHAR_BIT; ++ opsz = sz * CHAR_BIT; + msk = RTE_LEN2MASK(opsz, uint64_t); + + st = bvf->evst; +@@ -652,8 +666,10 @@ eval_alu(struct bpf_verifier *bvf, const struct ebpf_insn *ins) + if (BPF_SRC(ins->code) == BPF_X) { + rs = st->rv[ins->src_reg]; + eval_apply_mask(&rs, msk); +- } else ++ } else { ++ rs = (struct bpf_reg_val){.v = {.size = sz,},}; + eval_fill_imm(&rs, msk, ins->imm); ++ } + + eval_apply_mask(rd, msk); + +@@ -1083,7 +1099,7 @@ eval_jcc(struct bpf_verifier *bvf, const struct ebpf_insn *ins) + struct bpf_reg_val rvf, rvt; + + tst = bvf->evst; +- fst = bvf->evin->evst; ++ fst = bvf->evin->evst.cur; + + frd = fst->rv + ins->dst_reg; + trd = tst->rv + ins->dst_reg; +@@ -1812,8 +1828,8 @@ add_edge(struct bpf_verifier *bvf, struct inst_node *node, uint32_t nidx) + uint32_t ne; + + if (nidx > bvf->prm->nb_ins) { +- RTE_BPF_LOG(ERR, "%s: program boundary violation at pc: %u, " +- "next pc: %u\n", ++ RTE_BPF_LOG(ERR, ++ "%s: program boundary violation at pc: %u, next pc: %u\n", + __func__, get_node_idx(bvf, node), nidx); + return -EINVAL; + } +@@ -2089,60 +2105,113 @@ validate(struct bpf_verifier *bvf) + * helper functions get/free eval states. + */ + static struct bpf_eval_state * +-pull_eval_state(struct bpf_verifier *bvf) ++pull_eval_state(struct evst_pool *pool) + { + uint32_t n; + +- n = bvf->evst_pool.cur; +- if (n == bvf->evst_pool.num) ++ n = pool->cur; ++ if (n == pool->num) + return NULL; + +- bvf->evst_pool.cur = n + 1; +- return bvf->evst_pool.ent + n; ++ pool->cur = n + 1; ++ return pool->ent + n; + } + + static void +-push_eval_state(struct bpf_verifier *bvf) ++push_eval_state(struct evst_pool *pool) + { +- bvf->evst_pool.cur--; ++ RTE_ASSERT(pool->cur != 0); ++ pool->cur--; + } + + static void + evst_pool_fini(struct bpf_verifier *bvf) + { + bvf->evst = NULL; +- free(bvf->evst_pool.ent); +- memset(&bvf->evst_pool, 0, sizeof(bvf->evst_pool)); ++ free(bvf->evst_sr_pool.ent); ++ memset(&bvf->evst_sr_pool, 0, sizeof(bvf->evst_sr_pool)); ++ memset(&bvf->evst_tp_pool, 0, sizeof(bvf->evst_tp_pool)); + } + + static int + evst_pool_init(struct bpf_verifier *bvf) + { +- uint32_t n; ++ uint32_t k, n; + +- n = bvf->nb_jcc_nodes + 1; ++ /* ++ * We need nb_jcc_nodes + 1 for save_cur/restore_cur ++ * remaining ones will be used for state tracking/pruning. ++ */ ++ k = bvf->nb_jcc_nodes + 1; ++ n = k * 3; + +- bvf->evst_pool.ent = calloc(n, sizeof(bvf->evst_pool.ent[0])); +- if (bvf->evst_pool.ent == NULL) ++ bvf->evst_sr_pool.ent = calloc(n, sizeof(bvf->evst_sr_pool.ent[0])); ++ if (bvf->evst_sr_pool.ent == NULL) + return -ENOMEM; + +- bvf->evst_pool.num = n; +- bvf->evst_pool.cur = 0; ++ bvf->evst_sr_pool.num = k; ++ bvf->evst_sr_pool.cur = 0; + +- bvf->evst = pull_eval_state(bvf); ++ bvf->evst_tp_pool.ent = bvf->evst_sr_pool.ent + k; ++ bvf->evst_tp_pool.num = n - k; ++ bvf->evst_tp_pool.cur = 0; ++ ++ bvf->evst = pull_eval_state(&bvf->evst_sr_pool); + return 0; + } + ++/* ++ * try to allocate and initialise new eval state for given node. ++ * later if no errors will be encountered, this state will be accepted as ++ * one of the possible 'safe' states for that node. ++ */ ++static void ++save_start_eval_state(struct bpf_verifier *bvf, struct inst_node *node) ++{ ++ RTE_ASSERT(node->evst.start == NULL); ++ ++ /* limit number of states for one node with some reasonable value */ ++ if (node->evst.nb_safe >= NODE_EVST_MAX) ++ return; ++ ++ /* try to get new eval_state */ ++ node->evst.start = pull_eval_state(&bvf->evst_tp_pool); ++ ++ /* make a copy of current state */ ++ if (node->evst.start != NULL) { ++ memcpy(node->evst.start, bvf->evst, sizeof(*node->evst.start)); ++ SLIST_NEXT(node->evst.start, next) = NULL; ++ } ++} ++ ++/* ++ * add @start state to the list of @safe states. ++ */ ++static void ++save_safe_eval_state(struct bpf_verifier *bvf, struct inst_node *node) ++{ ++ if (node->evst.start == NULL) ++ return; ++ ++ SLIST_INSERT_HEAD(&node->evst.safe, node->evst.start, next); ++ node->evst.nb_safe++; ++ ++ RTE_BPF_LOG(DEBUG, "%s(bvf=%p,node=%u,state=%p): nb_safe=%u;\n", __func__, bvf, get_node_idx(bvf, node), ++ node->evst.start, node->evst.nb_safe); ++ ++ node->evst.start = NULL; ++} ++ + /* + * Save current eval state. + */ + static int +-save_eval_state(struct bpf_verifier *bvf, struct inst_node *node) ++save_cur_eval_state(struct bpf_verifier *bvf, struct inst_node *node) + { + struct bpf_eval_state *st; + + /* get new eval_state for this node */ +- st = pull_eval_state(bvf); ++ st = pull_eval_state(&bvf->evst_sr_pool); + if (st == NULL) { + RTE_BPF_LOG(ERR, + "%s: internal error (out of space) at pc: %u\n", +@@ -2154,11 +2223,13 @@ save_eval_state(struct bpf_verifier *bvf, struct inst_node *node) + memcpy(st, bvf->evst, sizeof(*st)); + + /* swap current state with new one */ +- node->evst = bvf->evst; ++ RTE_ASSERT(node->evst.cur == NULL); ++ node->evst.cur = bvf->evst; + bvf->evst = st; + + RTE_BPF_LOG(DEBUG, "%s(bvf=%p,node=%u) old/new states: %p/%p;\n", +- __func__, bvf, get_node_idx(bvf, node), node->evst, bvf->evst); ++ __func__, bvf, get_node_idx(bvf, node), node->evst.cur, ++ bvf->evst); + + return 0; + } +@@ -2167,14 +2238,15 @@ save_eval_state(struct bpf_verifier *bvf, struct inst_node *node) + * Restore previous eval state and mark current eval state as free. + */ + static void +-restore_eval_state(struct bpf_verifier *bvf, struct inst_node *node) ++restore_cur_eval_state(struct bpf_verifier *bvf, struct inst_node *node) + { + RTE_BPF_LOG(DEBUG, "%s(bvf=%p,node=%u) old/new states: %p/%p;\n", +- __func__, bvf, get_node_idx(bvf, node), bvf->evst, node->evst); ++ __func__, bvf, get_node_idx(bvf, node), bvf->evst, ++ node->evst.cur); + +- bvf->evst = node->evst; +- node->evst = NULL; +- push_eval_state(bvf); ++ bvf->evst = node->evst.cur; ++ node->evst.cur = NULL; ++ push_eval_state(&bvf->evst_sr_pool); + } + + static void +@@ -2191,26 +2263,124 @@ log_eval_state(const struct bpf_verifier *bvf, const struct ebpf_insn *ins, + + rte_log(loglvl, rte_bpf_logtype, + "r%u={\n" +- "\tv={type=%u, size=%zu},\n" ++ "\tv={type=%u, size=%zu, buf_size=%zu},\n" + "\tmask=0x%" PRIx64 ",\n" + "\tu={min=0x%" PRIx64 ", max=0x%" PRIx64 "},\n" + "\ts={min=%" PRId64 ", max=%" PRId64 "},\n" + "};\n", + ins->dst_reg, +- rv->v.type, rv->v.size, ++ rv->v.type, rv->v.size, rv->v.buf_size, + rv->mask, + rv->u.min, rv->u.max, + rv->s.min, rv->s.max); + } + + /* +- * Do second pass through CFG and try to evaluate instructions +- * via each possible path. +- * Right now evaluation functionality is quite limited. +- * Still need to add extra checks for: +- * - use/return uninitialized registers. +- * - use uninitialized data from the stack. +- * - memory boundaries violation. ++ * compare two evaluation states. ++ * returns zero if @lv is more conservative (safer) then @rv. ++ * returns non-zero value otherwise. ++ */ ++static int ++cmp_reg_val_within(const struct bpf_reg_val *lv, const struct bpf_reg_val *rv) ++{ ++ /* expect @v and @mask to be identical */ ++ if (memcmp(&lv->v, &rv->v, sizeof(lv->v)) != 0 || lv->mask != rv->mask) ++ return -1; ++ ++ /* exact match only for mbuf and stack pointers */ ++ if (lv->v.type == RTE_BPF_ARG_PTR_MBUF || ++ lv->v.type == BPF_ARG_PTR_STACK) ++ return -1; ++ ++ if (lv->u.min <= rv->u.min && lv->u.max >= rv->u.max && ++ lv->s.min <= rv->s.min && lv->s.max >= rv->s.max) ++ return 0; ++ ++ return -1; ++} ++ ++/* ++ * compare two evaluation states. ++ * returns zero if they are identical. ++ * returns positive value if @lv is more conservative (safer) then @rv. ++ * returns negative value otherwise. ++ */ ++static int ++cmp_eval_state(const struct bpf_eval_state *lv, const struct bpf_eval_state *rv) ++{ ++ int32_t rc; ++ uint32_t i, k; ++ ++ /* for stack expect identical values */ ++ rc = memcmp(lv->sv, rv->sv, sizeof(lv->sv)); ++ if (rc != 0) ++ return -(2 * EBPF_REG_NUM); ++ ++ k = 0; ++ /* check register values */ ++ for (i = 0; i != RTE_DIM(lv->rv); i++) { ++ rc = memcmp(&lv->rv[i], &rv->rv[i], sizeof(lv->rv[i])); ++ if (rc != 0 && cmp_reg_val_within(&lv->rv[i], &rv->rv[i]) != 0) ++ return -(i + 1); ++ k += (rc != 0); ++ } ++ ++ return k; ++} ++ ++/* ++ * check did we already evaluated that path and can it be pruned that time. ++ */ ++static int ++prune_eval_state(struct bpf_verifier *bvf, const struct inst_node *node, ++ struct inst_node *next) ++{ ++ int32_t rc; ++ struct bpf_eval_state *safe; ++ ++ rc = INT32_MIN; ++ SLIST_FOREACH(safe, &next->evst.safe, next) { ++ rc = cmp_eval_state(safe, bvf->evst); ++ if (rc >= 0) ++ break; ++ } ++ ++ rc = (rc >= 0) ? 0 : -1; ++ ++ /* ++ * current state doesn't match any safe states, ++ * so no prunning is possible right now, ++ * track current state for future references. ++ */ ++ if (rc != 0) ++ save_start_eval_state(bvf, next); ++ ++ RTE_BPF_LOG(DEBUG, ++ "%s(bvf=%p,node=%u,next=%u) returns %d, " ++ "next->evst.start=%p, next->evst.nb_safe=%u\n", ++ __func__, bvf, get_node_idx(bvf, node), get_node_idx(bvf, next), rc, next->evst.start, ++ next->evst.nb_safe); ++ return rc; ++} ++ ++/* Do second pass through CFG and try to evaluate instructions ++ * via each possible path. The verifier will try all paths, tracking types of ++ * registers used as input to instructions, and updating resulting type via ++ * register state values. Plus for each register and possible stack value it ++ * tries to estimate possible max/min value. ++ * For conditional jumps, a stack is used to save evaluation state, so one ++ * path is explored while the state for the other path is pushed onto the stack. ++ * Then later, we backtrack to the first pushed instruction and repeat the cycle ++ * until the stack is empty and we're done. ++ * For program with many conditional branches walking through all possible path ++ * could be very excessive. So to minimize number of evaluations we use ++ * heuristic similar to what Linux kernel does - state pruning: ++ * If from given instruction for given program state we explore all possible ++ * paths and for each of them reach _exit() without any complaints and a valid ++ * R0 value, then for that instruction, that program state can be marked as ++ * 'safe'. When we later arrive at the same instruction with a state ++ * equivalent to an earlier instruction's 'safe' state, we can prune the search. ++ * For now, only states for JCC targets are saved/examined. + */ + static int + evaluate(struct bpf_verifier *bvf) +@@ -2221,6 +2391,13 @@ evaluate(struct bpf_verifier *bvf) + const struct ebpf_insn *ins; + struct inst_node *next, *node; + ++ struct { ++ uint32_t nb_eval; ++ uint32_t nb_prune; ++ uint32_t nb_save; ++ uint32_t nb_restore; ++ } stats; ++ + /* initial state of frame pointer */ + static const struct bpf_reg_val rvfp = { + .v = { +@@ -2244,6 +2421,8 @@ evaluate(struct bpf_verifier *bvf) + next = node; + rc = 0; + ++ memset(&stats, 0, sizeof(stats)); ++ + while (node != NULL && rc == 0) { + + /* +@@ -2257,11 +2436,14 @@ evaluate(struct bpf_verifier *bvf) + op = ins[idx].code; + + /* for jcc node make a copy of evaluation state */ +- if (node->nb_edge > 1) +- rc |= save_eval_state(bvf, node); ++ if (node->nb_edge > 1) { ++ rc |= save_cur_eval_state(bvf, node); ++ stats.nb_save++; ++ } + + if (ins_chk[op].eval != NULL && rc == 0) { + err = ins_chk[op].eval(bvf, ins + idx); ++ stats.nb_eval++; + if (err != NULL) { + RTE_BPF_LOG(ERR, "%s: %s at pc: %u\n", + __func__, err, idx); +@@ -2275,21 +2457,37 @@ evaluate(struct bpf_verifier *bvf) + + /* proceed through CFG */ + next = get_next_node(bvf, node); ++ + if (next != NULL) { + + /* proceed with next child */ + if (node->cur_edge == node->nb_edge && +- node->evst != NULL) +- restore_eval_state(bvf, node); ++ node->evst.cur != NULL) { ++ restore_cur_eval_state(bvf, node); ++ stats.nb_restore++; ++ } + +- next->prev_node = get_node_idx(bvf, node); +- node = next; ++ /* ++ * for jcc targets: check did we already evaluated ++ * that path and can it's evaluation be skipped that ++ * time. ++ */ ++ if (node->nb_edge > 1 && prune_eval_state(bvf, node, ++ next) == 0) { ++ next = NULL; ++ stats.nb_prune++; ++ } else { ++ next->prev_node = get_node_idx(bvf, node); ++ node = next; ++ } + } else { + /* + * finished with current node and all it's kids, +- * proceed with parent ++ * mark it's @start state as safe for future references, ++ * and proceed with parent. + */ + node->cur_edge = 0; ++ save_safe_eval_state(bvf, node); + node = get_prev_node(bvf, node); + + /* finished */ +@@ -2298,6 +2496,14 @@ evaluate(struct bpf_verifier *bvf) + } + } + ++ RTE_BPF_LOG(DEBUG, ++ "%s(%p) returns %d, stats:\n" ++ "node evaluations=%u;\n" ++ "state pruned=%u;\n" ++ "state saves=%u;\n" ++ "state restores=%u;\n", ++ __func__, bvf, rc, stats.nb_eval, stats.nb_prune, stats.nb_save, stats.nb_restore); ++ + return rc; + } + +diff --git a/dpdk/lib/bpf/meson.build b/dpdk/lib/bpf/meson.build +index cd739bb827..aa258a9061 100644 +--- a/dpdk/lib/bpf/meson.build ++++ b/dpdk/lib/bpf/meson.build +@@ -7,6 +7,12 @@ if is_windows + subdir_done() + endif + ++if arch_subdir == 'x86' and dpdk_conf.get('RTE_ARCH_32') ++ build = false ++ reason = 'not supported on 32-bit x86' ++ subdir_done() ++endif ++ + sources = files('bpf.c', + 'bpf_dump.c', + 'bpf_exec.c', +diff --git a/dpdk/lib/cfgfile/rte_cfgfile.c b/dpdk/lib/cfgfile/rte_cfgfile.c +index eefba6e408..2f9cc0722a 100644 +--- a/dpdk/lib/cfgfile/rte_cfgfile.c ++++ b/dpdk/lib/cfgfile/rte_cfgfile.c +@@ -137,7 +137,7 @@ rte_cfgfile_check_params(const struct rte_cfgfile_parameters *params) + unsigned int i; + + if (!params) { +- CFG_LOG(ERR, "missing cfgfile parameters\n"); ++ CFG_LOG(ERR, "missing cfgfile parameters"); + return -EINVAL; + } + +@@ -150,7 +150,7 @@ rte_cfgfile_check_params(const struct rte_cfgfile_parameters *params) + } + + if (valid_comment == 0) { +- CFG_LOG(ERR, "invalid comment characters %c\n", ++ CFG_LOG(ERR, "invalid comment characters %c", + params->comment_character); + return -ENOTSUP; + } +@@ -188,7 +188,7 @@ rte_cfgfile_load_with_params(const char *filename, int flags, + lineno++; + if ((len >= sizeof(buffer) - 1) && (buffer[len-1] != '\n')) { + CFG_LOG(ERR, " line %d - no \\n found on string. " +- "Check if line too long\n", lineno); ++ "Check if line too long", lineno); + goto error1; + } + /* skip parsing if comment character found */ +@@ -209,7 +209,7 @@ rte_cfgfile_load_with_params(const char *filename, int flags, + char *end = memchr(buffer, ']', len); + if (end == NULL) { + CFG_LOG(ERR, +- "line %d - no terminating ']' character found\n", ++ "line %d - no terminating ']' character found", + lineno); + goto error1; + } +@@ -225,7 +225,7 @@ rte_cfgfile_load_with_params(const char *filename, int flags, + split[1] = memchr(buffer, '=', len); + if (split[1] == NULL) { + CFG_LOG(ERR, +- "line %d - no '=' character found\n", ++ "line %d - no '=' character found", + lineno); + goto error1; + } +@@ -249,7 +249,7 @@ rte_cfgfile_load_with_params(const char *filename, int flags, + if (!(flags & CFG_FLAG_EMPTY_VALUES) && + (*split[1] == '\0')) { + CFG_LOG(ERR, +- "line %d - cannot use empty values\n", ++ "line %d - cannot use empty values", + lineno); + goto error1; + } +@@ -414,7 +414,7 @@ int rte_cfgfile_set_entry(struct rte_cfgfile *cfg, const char *sectionname, + return 0; + } + +- CFG_LOG(ERR, "entry name doesn't exist\n"); ++ CFG_LOG(ERR, "entry name doesn't exist"); + return -EINVAL; + } + +diff --git a/dpdk/lib/compressdev/rte_compressdev_pmd.c b/dpdk/lib/compressdev/rte_compressdev_pmd.c +index 156bccd972..762b44f03e 100644 +--- a/dpdk/lib/compressdev/rte_compressdev_pmd.c ++++ b/dpdk/lib/compressdev/rte_compressdev_pmd.c +@@ -100,12 +100,12 @@ rte_compressdev_pmd_create(const char *name, + struct rte_compressdev *compressdev; + + if (params->name[0] != '\0') { +- COMPRESSDEV_LOG(INFO, "User specified device name = %s\n", ++ COMPRESSDEV_LOG(INFO, "User specified device name = %s", + params->name); + name = params->name; + } + +- COMPRESSDEV_LOG(INFO, "Creating compressdev %s\n", name); ++ COMPRESSDEV_LOG(INFO, "Creating compressdev %s", name); + + COMPRESSDEV_LOG(INFO, "Init parameters - name: %s, socket id: %d", + name, params->socket_id); +diff --git a/dpdk/lib/cryptodev/rte_cryptodev.c b/dpdk/lib/cryptodev/rte_cryptodev.c +index b258827734..c5eb4ecb01 100644 +--- a/dpdk/lib/cryptodev/rte_cryptodev.c ++++ b/dpdk/lib/cryptodev/rte_cryptodev.c +@@ -1489,6 +1489,10 @@ rte_cryptodev_add_enq_callback(uint8_t dev_id, + rte_cryptodev_callback_fn cb_fn, + void *cb_arg) + { ++#ifndef RTE_CRYPTO_CALLBACKS ++ rte_errno = ENOTSUP; ++ return NULL; ++#endif + struct rte_cryptodev *dev; + struct rte_cryptodev_cb_rcu *list; + struct rte_cryptodev_cb *cb, *tail; +@@ -1554,6 +1558,9 @@ rte_cryptodev_remove_enq_callback(uint8_t dev_id, + uint16_t qp_id, + struct rte_cryptodev_cb *cb) + { ++#ifndef RTE_CRYPTO_CALLBACKS ++ return -ENOTSUP; ++#endif + struct rte_cryptodev *dev; + RTE_ATOMIC(struct rte_cryptodev_cb *) *prev_cb; + struct rte_cryptodev_cb *curr_cb; +@@ -1628,6 +1635,10 @@ rte_cryptodev_add_deq_callback(uint8_t dev_id, + rte_cryptodev_callback_fn cb_fn, + void *cb_arg) + { ++#ifndef RTE_CRYPTO_CALLBACKS ++ rte_errno = ENOTSUP; ++ return NULL; ++#endif + struct rte_cryptodev *dev; + struct rte_cryptodev_cb_rcu *list; + struct rte_cryptodev_cb *cb, *tail; +@@ -1694,6 +1705,9 @@ rte_cryptodev_remove_deq_callback(uint8_t dev_id, + uint16_t qp_id, + struct rte_cryptodev_cb *cb) + { ++#ifndef RTE_CRYPTO_CALLBACKS ++ return -ENOTSUP; ++#endif + struct rte_cryptodev *dev; + RTE_ATOMIC(struct rte_cryptodev_cb *) *prev_cb; + struct rte_cryptodev_cb *curr_cb; +@@ -2072,7 +2086,7 @@ rte_cryptodev_sym_session_create(uint8_t dev_id, + } + + if (xforms == NULL) { +- CDEV_LOG_ERR("Invalid xform\n"); ++ CDEV_LOG_ERR("Invalid xform"); + rte_errno = EINVAL; + return NULL; + } +@@ -2682,7 +2696,7 @@ rte_cryptodev_driver_id_get(const char *name) + int driver_id = -1; + + if (name == NULL) { +- RTE_LOG(DEBUG, CRYPTODEV, "name pointer NULL"); ++ CDEV_LOG_DEBUG("name pointer NULL"); + return -1; + } + +diff --git a/dpdk/lib/cryptodev/rte_cryptodev.h b/dpdk/lib/cryptodev/rte_cryptodev.h +index aaeaf294e6..a42a4fc04e 100644 +--- a/dpdk/lib/cryptodev/rte_cryptodev.h ++++ b/dpdk/lib/cryptodev/rte_cryptodev.h +@@ -26,8 +26,6 @@ extern "C" { + + #include "rte_cryptodev_trace_fp.h" + +-extern const char **rte_cyptodev_names; +- + /* Logging Macros */ + + #define CDEV_LOG_ERR(...) \ +@@ -1912,7 +1910,7 @@ rte_cryptodev_dequeue_burst(uint8_t dev_id, uint16_t qp_id, + nb_ops = fp_ops->dequeue_burst(qp, ops, nb_ops); + + #ifdef RTE_CRYPTO_CALLBACKS +- if (unlikely(fp_ops->qp.deq_cb != NULL)) { ++ if (unlikely(fp_ops->qp.deq_cb[qp_id].next != NULL)) { + struct rte_cryptodev_cb_rcu *list; + struct rte_cryptodev_cb *cb; + +@@ -1979,7 +1977,7 @@ rte_cryptodev_enqueue_burst(uint8_t dev_id, uint16_t qp_id, + fp_ops = &rte_crypto_fp_ops[dev_id]; + qp = fp_ops->qp.data[qp_id]; + #ifdef RTE_CRYPTO_CALLBACKS +- if (unlikely(fp_ops->qp.enq_cb != NULL)) { ++ if (unlikely(fp_ops->qp.enq_cb[qp_id].next != NULL)) { + struct rte_cryptodev_cb_rcu *list; + struct rte_cryptodev_cb *cb; + +diff --git a/dpdk/lib/dispatcher/rte_dispatcher.c b/dpdk/lib/dispatcher/rte_dispatcher.c +index 10d02edde9..95dd41b818 100644 +--- a/dpdk/lib/dispatcher/rte_dispatcher.c ++++ b/dpdk/lib/dispatcher/rte_dispatcher.c +@@ -246,7 +246,7 @@ evd_service_register(struct rte_dispatcher *dispatcher) + rc = rte_service_component_register(&service, &dispatcher->service_id); + if (rc != 0) + RTE_EDEV_LOG_ERR("Registration of dispatcher service " +- "%s failed with error code %d\n", ++ "%s failed with error code %d", + service.name, rc); + + return rc; +@@ -260,7 +260,7 @@ evd_service_unregister(struct rte_dispatcher *dispatcher) + rc = rte_service_component_unregister(dispatcher->service_id); + if (rc != 0) + RTE_EDEV_LOG_ERR("Unregistration of dispatcher service " +- "failed with error code %d\n", rc); ++ "failed with error code %d", rc); + + return rc; + } +@@ -279,7 +279,7 @@ rte_dispatcher_create(uint8_t event_dev_id) + RTE_CACHE_LINE_SIZE, socket_id); + + if (dispatcher == NULL) { +- RTE_EDEV_LOG_ERR("Unable to allocate memory for dispatcher\n"); ++ RTE_EDEV_LOG_ERR("Unable to allocate memory for dispatcher"); + rte_errno = ENOMEM; + return NULL; + } +@@ -483,7 +483,7 @@ evd_lcore_uninstall_handler(struct rte_dispatcher_lcore *lcore, + unreg_handler = evd_lcore_get_handler_by_id(lcore, handler_id); + + if (unreg_handler == NULL) { +- RTE_EDEV_LOG_ERR("Invalid handler id %d\n", handler_id); ++ RTE_EDEV_LOG_ERR("Invalid handler id %d", handler_id); + return -EINVAL; + } + +@@ -602,7 +602,7 @@ rte_dispatcher_finalize_unregister(struct rte_dispatcher *dispatcher, + unreg_finalizer = evd_get_finalizer_by_id(dispatcher, finalizer_id); + + if (unreg_finalizer == NULL) { +- RTE_EDEV_LOG_ERR("Invalid finalizer id %d\n", finalizer_id); ++ RTE_EDEV_LOG_ERR("Invalid finalizer id %d", finalizer_id); + return -EINVAL; + } + +@@ -636,7 +636,7 @@ evd_set_service_runstate(struct rte_dispatcher *dispatcher, int state) + */ + if (rc != 0) + RTE_EDEV_LOG_ERR("Unexpected error %d occurred while setting " +- "service component run state to %d\n", rc, ++ "service component run state to %d", rc, + state); + + RTE_VERIFY(rc == 0); +diff --git a/dpdk/lib/dmadev/rte_dmadev.c b/dpdk/lib/dmadev/rte_dmadev.c +index 4e5e420c82..5093c6e38b 100644 +--- a/dpdk/lib/dmadev/rte_dmadev.c ++++ b/dpdk/lib/dmadev/rte_dmadev.c +@@ -158,15 +158,24 @@ static int + dma_dev_data_prepare(void) + { + size_t size; ++ void *ptr; + + if (rte_dma_devices != NULL) + return 0; + +- size = dma_devices_max * sizeof(struct rte_dma_dev); +- rte_dma_devices = malloc(size); +- if (rte_dma_devices == NULL) ++ /* The DMA device object is expected to align cacheline, ++ * but the return value of malloc may not be aligned to the cache line. ++ * Therefore, extra memory is applied for realignment. ++ * Note: posix_memalign/aligned_alloc are not used ++ * because not always available, depending on libc. ++ */ ++ size = dma_devices_max * sizeof(struct rte_dma_dev) + RTE_CACHE_LINE_SIZE; ++ ptr = malloc(size); ++ if (ptr == NULL) + return -ENOMEM; +- memset(rte_dma_devices, 0, size); ++ memset(ptr, 0, size); ++ ++ rte_dma_devices = RTE_PTR_ALIGN(ptr, RTE_CACHE_LINE_SIZE); + + return 0; + } +@@ -726,7 +735,7 @@ rte_dma_vchan_status(int16_t dev_id, uint16_t vchan, enum rte_dma_vchan_status * + return -EINVAL; + + if (vchan >= dev->data->dev_conf.nb_vchans) { +- RTE_DMA_LOG(ERR, "Device %u vchan %u out of range\n", dev_id, vchan); ++ RTE_DMA_LOG(ERR, "Device %u vchan %u out of range", dev_id, vchan); + return -EINVAL; + } + +@@ -1028,7 +1037,7 @@ dmadev_handle_dev_dump(const char *cmd __rte_unused, + if (*end_param != '\0') + RTE_DMA_LOG(WARNING, "Extra parameters passed to dmadev telemetry command, ignoring"); + +- buf = calloc(sizeof(char), RTE_TEL_MAX_SINGLE_STRING_LEN); ++ buf = calloc(RTE_TEL_MAX_SINGLE_STRING_LEN, sizeof(char)); + if (buf == NULL) + return -ENOMEM; + +diff --git a/dpdk/lib/eal/common/eal_common_fbarray.c b/dpdk/lib/eal/common/eal_common_fbarray.c +index 2055bfa57d..253110360e 100644 +--- a/dpdk/lib/eal/common/eal_common_fbarray.c ++++ b/dpdk/lib/eal/common/eal_common_fbarray.c +@@ -173,7 +173,7 @@ find_next_n(const struct rte_fbarray *arr, unsigned int start, unsigned int n, + + /* combine current ignore mask with last index ignore mask */ + if (msk_idx == last) +- ignore_msk |= last_msk; ++ ignore_msk &= last_msk; + + /* if we have an ignore mask, ignore once */ + if (ignore_msk) { +@@ -216,6 +216,8 @@ find_next_n(const struct rte_fbarray *arr, unsigned int start, unsigned int n, + for (lookahead_idx = msk_idx + 1; lookahead_idx < msk->n_masks; + lookahead_idx++) { + unsigned int s_idx, need; ++ uint64_t first_bit = 1; ++ + lookahead_msk = msk->data[lookahead_idx]; + + /* if we're looking for free space, invert the mask */ +@@ -225,18 +227,24 @@ find_next_n(const struct rte_fbarray *arr, unsigned int start, unsigned int n, + /* figure out how many consecutive bits we need here */ + need = RTE_MIN(left, MASK_ALIGN); + +- for (s_idx = 0; s_idx < need - 1; s_idx++) ++ /* count number of shifts we performed */ ++ for (s_idx = 0; s_idx < need - 1; s_idx++) { + lookahead_msk &= lookahead_msk >> 1ULL; ++ /* did we lose the run yet? */ ++ if ((lookahead_msk & first_bit) == 0) ++ break; ++ } + + /* if first bit is not set, we've lost the run */ +- if ((lookahead_msk & 1) == 0) { ++ if ((lookahead_msk & first_bit) == 0) { + /* + * we've scanned this far, so we know there are + * no runs in the space we've lookahead-scanned + * as well, so skip that on next iteration. + */ +- ignore_msk = ~((1ULL << need) - 1); +- msk_idx = lookahead_idx; ++ ignore_msk = ~((1ULL << (s_idx + 1)) - 1); ++ /* outer loop will increment msk_idx so add 1 */ ++ msk_idx = lookahead_idx - 1; + break; + } + +@@ -500,8 +508,13 @@ find_prev_n(const struct rte_fbarray *arr, unsigned int start, unsigned int n, + /* figure out how many consecutive bits we need here */ + need = RTE_MIN(left, MASK_ALIGN); + +- for (s_idx = 0; s_idx < need - 1; s_idx++) ++ /* count number of shifts we performed */ ++ for (s_idx = 0; s_idx < need - 1; s_idx++) { + lookbehind_msk &= lookbehind_msk << 1ULL; ++ /* did we lose the run yet? */ ++ if ((lookbehind_msk & last_bit) == 0) ++ break; ++ } + + /* if last bit is not set, we've lost the run */ + if ((lookbehind_msk & last_bit) == 0) { +@@ -510,8 +523,9 @@ find_prev_n(const struct rte_fbarray *arr, unsigned int start, unsigned int n, + * no runs in the space we've lookbehind-scanned + * as well, so skip that on next iteration. + */ +- ignore_msk = UINT64_MAX << need; +- msk_idx = lookbehind_idx; ++ ignore_msk = ~(UINT64_MAX << (MASK_ALIGN - s_idx - 1)); ++ /* outer loop will decrement msk_idx so add 1 */ ++ msk_idx = lookbehind_idx + 1; + break; + } + +diff --git a/dpdk/lib/eal/common/eal_common_options.c b/dpdk/lib/eal/common/eal_common_options.c +index a6d21f1cba..aec51736f2 100644 +--- a/dpdk/lib/eal/common/eal_common_options.c ++++ b/dpdk/lib/eal/common/eal_common_options.c +@@ -226,6 +226,8 @@ eal_save_args(int argc, char **argv) + if (strcmp(argv[i], "--") == 0) + break; + eal_args[i] = strdup(argv[i]); ++ if (eal_args[i] == NULL) ++ goto error; + } + eal_args[i++] = NULL; /* always finish with NULL */ + +@@ -235,13 +237,31 @@ eal_save_args(int argc, char **argv) + + eal_app_args = calloc(argc - i + 1, sizeof(*eal_args)); + if (eal_app_args == NULL) +- return -1; ++ goto error; + +- for (j = 0; i < argc; j++, i++) ++ for (j = 0; i < argc; j++, i++) { + eal_app_args[j] = strdup(argv[i]); ++ if (eal_app_args[j] == NULL) ++ goto error; ++ } + eal_app_args[j] = NULL; + + return 0; ++ ++error: ++ if (eal_app_args != NULL) { ++ i = 0; ++ while (eal_app_args[i] != NULL) ++ free(eal_app_args[i++]); ++ free(eal_app_args); ++ eal_app_args = NULL; ++ } ++ i = 0; ++ while (eal_args[i] != NULL) ++ free(eal_args[i++]); ++ free(eal_args); ++ eal_args = NULL; ++ return -1; + } + #endif + +@@ -2141,7 +2161,7 @@ rte_vect_set_max_simd_bitwidth(uint16_t bitwidth) + struct internal_config *internal_conf = + eal_get_internal_configuration(); + if (internal_conf->max_simd_bitwidth.forced) { +- RTE_LOG(NOTICE, EAL, "Cannot set max SIMD bitwidth - user runtime override enabled"); ++ RTE_LOG(NOTICE, EAL, "Cannot set max SIMD bitwidth - user runtime override enabled\n"); + return -EPERM; + } + +diff --git a/dpdk/lib/eal/common/malloc_mp.c b/dpdk/lib/eal/common/malloc_mp.c +index 4d62397aba..a704eb01d1 100644 +--- a/dpdk/lib/eal/common/malloc_mp.c ++++ b/dpdk/lib/eal/common/malloc_mp.c +@@ -756,7 +756,8 @@ request_to_primary(struct malloc_mp_req *user_req) + do { + ret = pthread_cond_timedwait(&entry->cond, + &mp_request_list.lock, &ts); +- } while (ret != 0 && ret != ETIMEDOUT); ++ } while ((ret != 0 && ret != ETIMEDOUT) && ++ entry->state == REQ_STATE_ACTIVE); + + if (entry->state != REQ_STATE_COMPLETE) { + RTE_LOG(ERR, EAL, "Request timed out\n"); +diff --git a/dpdk/lib/eal/include/rte_common.h b/dpdk/lib/eal/include/rte_common.h +index c1ba32d00e..40af7164af 100644 +--- a/dpdk/lib/eal/include/rte_common.h ++++ b/dpdk/lib/eal/include/rte_common.h +@@ -267,7 +267,7 @@ static void __attribute__((destructor(RTE_PRIO(prio)), used)) func(void) + #define RTE_FINI_PRIO(name, priority) \ + static void name(void); \ + __pragma(const_seg(DTOR_PRIORITY_TO_SECTION(priority))) \ +- __declspec(allocate(DTOR_PRIORITY_TO_SECTION(priority))) name ## _pointer = &name; \ ++ __declspec(allocate(DTOR_PRIORITY_TO_SECTION(priority))) void *name ## _pointer = &name; \ + __pragma(const_seg()) \ + static void name(void) + #endif +diff --git a/dpdk/lib/eal/linux/eal_dev.c b/dpdk/lib/eal/linux/eal_dev.c +index ac76f6174d..c0ba98852b 100644 +--- a/dpdk/lib/eal/linux/eal_dev.c ++++ b/dpdk/lib/eal/linux/eal_dev.c +@@ -182,6 +182,8 @@ dev_uev_parse(const char *buf, struct rte_dev_event *event, int length) + i += 14; + strlcpy(pci_slot_name, buf, sizeof(subsystem)); + event->devname = strdup(pci_slot_name); ++ if (event->devname == NULL) ++ return -1; + } + for (; i < length; i++) { + if (*buf == '\0') +diff --git a/dpdk/lib/eal/linux/eal_hugepage_info.c b/dpdk/lib/eal/linux/eal_hugepage_info.c +index 581d9dfc91..36a495fb1f 100644 +--- a/dpdk/lib/eal/linux/eal_hugepage_info.c ++++ b/dpdk/lib/eal/linux/eal_hugepage_info.c +@@ -403,7 +403,7 @@ inspect_hugedir_cb(const struct walk_hugedir_data *whd) + struct stat st; + + if (fstat(whd->file_fd, &st) < 0) +- RTE_LOG(DEBUG, EAL, "%s(): stat(\"%s\") failed: %s", ++ RTE_LOG(DEBUG, EAL, "%s(): stat(\"%s\") failed: %s\n", + __func__, whd->file_name, strerror(errno)); + else + (*total_size) += st.st_size; +diff --git a/dpdk/lib/eal/linux/eal_interrupts.c b/dpdk/lib/eal/linux/eal_interrupts.c +index d4919dff45..eabac24992 100644 +--- a/dpdk/lib/eal/linux/eal_interrupts.c ++++ b/dpdk/lib/eal/linux/eal_interrupts.c +@@ -1542,7 +1542,7 @@ rte_intr_efd_enable(struct rte_intr_handle *intr_handle, uint32_t nb_efd) + /* only check, initialization would be done in vdev driver.*/ + if ((uint64_t)rte_intr_efd_counter_size_get(intr_handle) > + sizeof(union rte_intr_read_buffer)) { +- RTE_LOG(ERR, EAL, "the efd_counter_size is oversized"); ++ RTE_LOG(ERR, EAL, "the efd_counter_size is oversized\n"); + return -EINVAL; + } + } else { +diff --git a/dpdk/lib/eal/linux/eal_memalloc.c b/dpdk/lib/eal/linux/eal_memalloc.c +index 9853ec78a2..b9fc83fe6a 100644 +--- a/dpdk/lib/eal/linux/eal_memalloc.c ++++ b/dpdk/lib/eal/linux/eal_memalloc.c +@@ -1061,7 +1061,7 @@ eal_memalloc_alloc_seg_bulk(struct rte_memseg **ms, int n_segs, size_t page_sz, + /* memalloc is locked, so it's safe to use thread-unsafe version */ + ret = rte_memseg_list_walk_thread_unsafe(alloc_seg_walk, &wa); + if (ret == 0) { +- RTE_LOG(ERR, EAL, "%s(): couldn't find suitable memseg_list\n", ++ RTE_LOG(DEBUG, EAL, "%s(): couldn't find suitable memseg_list\n", + __func__); + ret = -1; + } else if (ret > 0) { +diff --git a/dpdk/lib/eal/unix/eal_firmware.c b/dpdk/lib/eal/unix/eal_firmware.c +index 1a7cf8e7b7..9fc2f7763a 100644 +--- a/dpdk/lib/eal/unix/eal_firmware.c ++++ b/dpdk/lib/eal/unix/eal_firmware.c +@@ -15,6 +15,8 @@ + + #include "eal_firmware.h" + ++static const char * const compression_suffixes[] = { "xz", "zst" }; ++ + #ifdef RTE_HAS_LIBARCHIVE + + struct firmware_read_ctx { +@@ -36,7 +38,11 @@ firmware_open(struct firmware_read_ctx *ctx, const char *name, size_t blocksize) + + err = archive_read_support_filter_xz(ctx->a); + if (err != ARCHIVE_OK && err != ARCHIVE_WARN) +- goto error; ++ RTE_LOG(DEBUG, EAL, "could not initialise libarchive for xz compression\n"); ++ ++ err = archive_read_support_filter_zstd(ctx->a); ++ if (err != ARCHIVE_OK && err != ARCHIVE_WARN) ++ RTE_LOG(DEBUG, EAL, "could not initialise libarchive for zstd compression\n"); + + if (archive_read_open_filename(ctx->a, name, blocksize) != ARCHIVE_OK) + goto error; +@@ -147,16 +153,21 @@ rte_firmware_read(const char *name, void **buf, size_t *bufsz) + + ret = firmware_read(name, buf, bufsz); + if (ret < 0) { +- snprintf(path, sizeof(path), "%s.xz", name); +- path[PATH_MAX - 1] = '\0'; ++ unsigned int i; ++ ++ for (i = 0; i < RTE_DIM(compression_suffixes); i++) { ++ snprintf(path, sizeof(path), "%s.%s", name, compression_suffixes[i]); ++ path[PATH_MAX - 1] = '\0'; ++ if (access(path, F_OK) != 0) ++ continue; + #ifndef RTE_HAS_LIBARCHIVE +- if (access(path, F_OK) == 0) { + RTE_LOG(WARNING, EAL, "libarchive not linked, %s cannot be decompressed\n", + path); +- } + #else +- ret = firmware_read(path, buf, bufsz); ++ ret = firmware_read(path, buf, bufsz); + #endif ++ break; ++ } + } + return ret; + } +diff --git a/dpdk/lib/eal/windows/eal_memory.c b/dpdk/lib/eal/windows/eal_memory.c +index 31410a41fd..fd39155163 100644 +--- a/dpdk/lib/eal/windows/eal_memory.c ++++ b/dpdk/lib/eal/windows/eal_memory.c +@@ -110,7 +110,7 @@ eal_mem_win32api_init(void) + VirtualAlloc2_ptr = (VirtualAlloc2_type)( + (void *)GetProcAddress(library, function)); + if (VirtualAlloc2_ptr == NULL) { +- RTE_LOG_WIN32_ERR("GetProcAddress(\"%s\", \"%s\")\n", ++ RTE_LOG_WIN32_ERR("GetProcAddress(\"%s\", \"%s\")", + library_name, function); + + /* Contrary to the docs, Server 2016 is not supported. */ +diff --git a/dpdk/lib/eal/windows/include/meson.build b/dpdk/lib/eal/windows/include/meson.build +index 5fb1962ac7..e985a77d58 100644 +--- a/dpdk/lib/eal/windows/include/meson.build ++++ b/dpdk/lib/eal/windows/include/meson.build +@@ -6,4 +6,5 @@ includes += include_directories('.') + headers += files( + 'rte_os.h', + 'rte_windows.h', ++ 'sched.h', + ) +diff --git a/dpdk/lib/eal/x86/rte_cycles.c b/dpdk/lib/eal/x86/rte_cycles.c +index 69ed59b4f0..f147a5231d 100644 +--- a/dpdk/lib/eal/x86/rte_cycles.c ++++ b/dpdk/lib/eal/x86/rte_cycles.c +@@ -10,6 +10,10 @@ + #include + #endif + ++#define x86_vendor_amd(t1, t2, t3) \ ++ ((t1 == 0x68747541) && /* htuA */ \ ++ (t2 == 0x444d4163) && /* DMAc */ \ ++ (t3 == 0x69746e65)) /* itne */ + + #include "eal_private.h" + +@@ -110,6 +114,18 @@ get_tsc_freq_arch(void) + uint8_t mult, model; + int32_t ret; + ++#ifdef RTE_TOOLCHAIN_MSVC ++ __cpuid(cpuinfo, 0); ++ a = cpuinfo[0]; ++ b = cpuinfo[1]; ++ c = cpuinfo[2]; ++ d = cpuinfo[3]; ++#else ++ __cpuid(0, a, b, c, d); ++#endif ++ if (x86_vendor_amd(b, c, d)) ++ return 0; ++ + /* + * Time Stamp Counter and Nominal Core Crystal Clock + * Information Leaf +diff --git a/dpdk/lib/ethdev/ethdev_driver.c b/dpdk/lib/ethdev/ethdev_driver.c +index fff4b7b4cd..47659d5e8c 100644 +--- a/dpdk/lib/ethdev/ethdev_driver.c ++++ b/dpdk/lib/ethdev/ethdev_driver.c +@@ -297,15 +297,25 @@ rte_eth_dev_create(struct rte_device *device, const char *name, + return -ENODEV; + + if (priv_data_size) { ++ /* try alloc private data on device-local node. */ + ethdev->data->dev_private = rte_zmalloc_socket( + name, priv_data_size, RTE_CACHE_LINE_SIZE, + device->numa_node); + +- if (!ethdev->data->dev_private) { +- RTE_ETHDEV_LOG(ERR, +- "failed to allocate private data\n"); +- retval = -ENOMEM; +- goto probe_failed; ++ /* fall back to alloc on any socket on failure */ ++ if (ethdev->data->dev_private == NULL) { ++ ethdev->data->dev_private = rte_zmalloc(name, ++ priv_data_size, RTE_CACHE_LINE_SIZE); ++ ++ if (ethdev->data->dev_private == NULL) { ++ RTE_ETHDEV_LOG(ERR, "failed to allocate private data\n"); ++ retval = -ENOMEM; ++ goto probe_failed; ++ } ++ /* got memory, but not local, so issue warning */ ++ RTE_ETHDEV_LOG(WARNING, ++ "Private data for ethdev '%s' not allocated on local NUMA node %d\n", ++ device->name, device->numa_node); + } + } + } else { +@@ -487,7 +497,7 @@ rte_eth_devargs_parse(const char *dargs, struct rte_eth_devargs *eth_da) + pair = &args.pairs[i]; + if (strcmp("representor", pair->key) == 0) { + if (eth_da->type != RTE_ETH_REPRESENTOR_NONE) { +- RTE_LOG(ERR, EAL, "duplicated representor key: %s\n", ++ RTE_ETHDEV_LOG(ERR, "duplicated representor key: %s\n", + dargs); + result = -1; + goto parse_cleanup; +@@ -713,7 +723,7 @@ rte_eth_representor_id_get(uint16_t port_id, + if (info->ranges[i].controller != controller) + continue; + if (info->ranges[i].id_end < info->ranges[i].id_base) { +- RTE_LOG(WARNING, EAL, "Port %hu invalid representor ID Range %u - %u, entry %d\n", ++ RTE_ETHDEV_LOG(WARNING, "Port %hu invalid representor ID Range %u - %u, entry %d\n", + port_id, info->ranges[i].id_base, + info->ranges[i].id_end, i); + continue; +diff --git a/dpdk/lib/ethdev/ethdev_driver.h b/dpdk/lib/ethdev/ethdev_driver.h +index b482cd12bb..ec56925882 100644 +--- a/dpdk/lib/ethdev/ethdev_driver.h ++++ b/dpdk/lib/ethdev/ethdev_driver.h +@@ -1655,18 +1655,13 @@ static inline int + rte_eth_linkstatus_set(struct rte_eth_dev *dev, + const struct rte_eth_link *new_link) + { +- RTE_ATOMIC(uint64_t) *dev_link = (uint64_t __rte_atomic *)&(dev->data->dev_link); +- union { +- uint64_t val64; +- struct rte_eth_link link; +- } orig; +- +- RTE_BUILD_BUG_ON(sizeof(*new_link) != sizeof(uint64_t)); ++ struct rte_eth_link old_link; + +- orig.val64 = rte_atomic_exchange_explicit(dev_link, *(const uint64_t *)new_link, +- rte_memory_order_seq_cst); ++ old_link.val64 = rte_atomic_exchange_explicit(&dev->data->dev_link.val64, ++ new_link->val64, ++ rte_memory_order_seq_cst); + +- return (orig.link.link_status == new_link->link_status) ? -1 : 0; ++ return (old_link.link_status == new_link->link_status) ? -1 : 0; + } + + /** +@@ -1682,12 +1677,11 @@ static inline void + rte_eth_linkstatus_get(const struct rte_eth_dev *dev, + struct rte_eth_link *link) + { +- RTE_ATOMIC(uint64_t) *src = (uint64_t __rte_atomic *)&(dev->data->dev_link); +- uint64_t *dst = (uint64_t *)link; +- +- RTE_BUILD_BUG_ON(sizeof(*link) != sizeof(uint64_t)); ++ struct rte_eth_link curr_link; + +- *dst = rte_atomic_load_explicit(src, rte_memory_order_seq_cst); ++ curr_link.val64 = rte_atomic_load_explicit(&dev->data->dev_link.val64, ++ rte_memory_order_seq_cst); ++ rte_atomic_store_explicit(&link->val64, curr_link.val64, rte_memory_order_seq_cst); + } + + /** +diff --git a/dpdk/lib/ethdev/ethdev_pci.h b/dpdk/lib/ethdev/ethdev_pci.h +index 320e3e0093..c40bc2ed02 100644 +--- a/dpdk/lib/ethdev/ethdev_pci.h ++++ b/dpdk/lib/ethdev/ethdev_pci.h +@@ -31,7 +31,7 @@ rte_eth_copy_pci_info(struct rte_eth_dev *eth_dev, + struct rte_pci_device *pci_dev) + { + if ((eth_dev == NULL) || (pci_dev == NULL)) { +- RTE_ETHDEV_LOG(ERR, "NULL pointer eth_dev=%p pci_dev=%p", ++ RTE_ETHDEV_LOG(ERR, "NULL pointer eth_dev=%p pci_dev=%p\n", + (void *)eth_dev, (void *)pci_dev); + return; + } +@@ -93,12 +93,26 @@ rte_eth_dev_pci_allocate(struct rte_pci_device *dev, size_t private_data_size) + return NULL; + + if (private_data_size) { ++ /* Try and alloc the private-data structure on socket local to the device */ + eth_dev->data->dev_private = rte_zmalloc_socket(name, + private_data_size, RTE_CACHE_LINE_SIZE, + dev->device.numa_node); +- if (!eth_dev->data->dev_private) { +- rte_eth_dev_release_port(eth_dev); +- return NULL; ++ ++ /* if cannot allocate memory on the socket local to the device ++ * use rte_malloc to allocate memory on some other socket, if available. ++ */ ++ if (eth_dev->data->dev_private == NULL) { ++ eth_dev->data->dev_private = rte_zmalloc(name, ++ private_data_size, RTE_CACHE_LINE_SIZE); ++ ++ if (eth_dev->data->dev_private == NULL) { ++ rte_eth_dev_release_port(eth_dev); ++ return NULL; ++ } ++ /* got memory, but not local, so issue warning */ ++ RTE_ETHDEV_LOG(WARNING, ++ "Private data for ethdev '%s' not allocated on local NUMA node %d\n", ++ dev->device.name, dev->device.numa_node); + } + } + } else { +diff --git a/dpdk/lib/ethdev/ethdev_private.c b/dpdk/lib/ethdev/ethdev_private.c +index e98b7188b0..0e1c7b23c1 100644 +--- a/dpdk/lib/ethdev/ethdev_private.c ++++ b/dpdk/lib/ethdev/ethdev_private.c +@@ -182,7 +182,7 @@ rte_eth_devargs_parse_representor_ports(char *str, void *data) + RTE_DIM(eth_da->representor_ports)); + done: + if (str == NULL) +- RTE_LOG(ERR, EAL, "wrong representor format: %s\n", str); ++ RTE_ETHDEV_LOG(ERR, "wrong representor format: %s\n", str); + return str == NULL ? -1 : 0; + } + +diff --git a/dpdk/lib/ethdev/rte_class_eth.c b/dpdk/lib/ethdev/rte_class_eth.c +index b61dae849d..311beb17cb 100644 +--- a/dpdk/lib/ethdev/rte_class_eth.c ++++ b/dpdk/lib/ethdev/rte_class_eth.c +@@ -165,7 +165,7 @@ eth_dev_iterate(const void *start, + valid_keys = eth_params_keys; + kvargs = rte_kvargs_parse(str, valid_keys); + if (kvargs == NULL) { +- RTE_LOG(ERR, EAL, "cannot parse argument list\n"); ++ RTE_ETHDEV_LOG(ERR, "cannot parse argument list\n"); + rte_errno = EINVAL; + return NULL; + } +diff --git a/dpdk/lib/ethdev/rte_ethdev.c b/dpdk/lib/ethdev/rte_ethdev.c +index 3858983fcc..b9d99ece15 100644 +--- a/dpdk/lib/ethdev/rte_ethdev.c ++++ b/dpdk/lib/ethdev/rte_ethdev.c +@@ -724,7 +724,7 @@ rte_eth_dev_get_port_by_name(const char *name, uint16_t *port_id) + uint16_t pid; + + if (name == NULL) { +- RTE_ETHDEV_LOG(ERR, "Cannot get port ID from NULL name"); ++ RTE_ETHDEV_LOG(ERR, "Cannot get port ID from NULL name\n"); + return -EINVAL; + } + +@@ -2394,41 +2394,41 @@ rte_eth_rx_hairpin_queue_setup(uint16_t port_id, uint16_t rx_queue_id, + nb_rx_desc = cap.max_nb_desc; + if (nb_rx_desc > cap.max_nb_desc) { + RTE_ETHDEV_LOG(ERR, +- "Invalid value for nb_rx_desc(=%hu), should be: <= %hu", ++ "Invalid value for nb_rx_desc(=%hu), should be: <= %hu\n", + nb_rx_desc, cap.max_nb_desc); + return -EINVAL; + } + if (conf->peer_count > cap.max_rx_2_tx) { + RTE_ETHDEV_LOG(ERR, +- "Invalid value for number of peers for Rx queue(=%u), should be: <= %hu", ++ "Invalid value for number of peers for Rx queue(=%u), should be: <= %hu\n", + conf->peer_count, cap.max_rx_2_tx); + return -EINVAL; + } + if (conf->use_locked_device_memory && !cap.rx_cap.locked_device_memory) { + RTE_ETHDEV_LOG(ERR, +- "Attempt to use locked device memory for Rx queue, which is not supported"); ++ "Attempt to use locked device memory for Rx queue, which is not supported\n"); + return -EINVAL; + } + if (conf->use_rte_memory && !cap.rx_cap.rte_memory) { + RTE_ETHDEV_LOG(ERR, +- "Attempt to use DPDK memory for Rx queue, which is not supported"); ++ "Attempt to use DPDK memory for Rx queue, which is not supported\n"); + return -EINVAL; + } + if (conf->use_locked_device_memory && conf->use_rte_memory) { + RTE_ETHDEV_LOG(ERR, +- "Attempt to use mutually exclusive memory settings for Rx queue"); ++ "Attempt to use mutually exclusive memory settings for Rx queue\n"); + return -EINVAL; + } + if (conf->force_memory && + !conf->use_locked_device_memory && + !conf->use_rte_memory) { + RTE_ETHDEV_LOG(ERR, +- "Attempt to force Rx queue memory settings, but none is set"); ++ "Attempt to force Rx queue memory settings, but none is set\n"); + return -EINVAL; + } + if (conf->peer_count == 0) { + RTE_ETHDEV_LOG(ERR, +- "Invalid value for number of peers for Rx queue(=%u), should be: > 0", ++ "Invalid value for number of peers for Rx queue(=%u), should be: > 0\n", + conf->peer_count); + return -EINVAL; + } +@@ -2438,7 +2438,7 @@ rte_eth_rx_hairpin_queue_setup(uint16_t port_id, uint16_t rx_queue_id, + count++; + } + if (count > cap.max_nb_queues) { +- RTE_ETHDEV_LOG(ERR, "To many Rx hairpin queues max is %d", ++ RTE_ETHDEV_LOG(ERR, "To many Rx hairpin queues max is %d\n", + cap.max_nb_queues); + return -EINVAL; + } +@@ -2597,41 +2597,41 @@ rte_eth_tx_hairpin_queue_setup(uint16_t port_id, uint16_t tx_queue_id, + nb_tx_desc = cap.max_nb_desc; + if (nb_tx_desc > cap.max_nb_desc) { + RTE_ETHDEV_LOG(ERR, +- "Invalid value for nb_tx_desc(=%hu), should be: <= %hu", ++ "Invalid value for nb_tx_desc(=%hu), should be: <= %hu\n", + nb_tx_desc, cap.max_nb_desc); + return -EINVAL; + } + if (conf->peer_count > cap.max_tx_2_rx) { + RTE_ETHDEV_LOG(ERR, +- "Invalid value for number of peers for Tx queue(=%u), should be: <= %hu", ++ "Invalid value for number of peers for Tx queue(=%u), should be: <= %hu\n", + conf->peer_count, cap.max_tx_2_rx); + return -EINVAL; + } + if (conf->use_locked_device_memory && !cap.tx_cap.locked_device_memory) { + RTE_ETHDEV_LOG(ERR, +- "Attempt to use locked device memory for Tx queue, which is not supported"); ++ "Attempt to use locked device memory for Tx queue, which is not supported\n"); + return -EINVAL; + } + if (conf->use_rte_memory && !cap.tx_cap.rte_memory) { + RTE_ETHDEV_LOG(ERR, +- "Attempt to use DPDK memory for Tx queue, which is not supported"); ++ "Attempt to use DPDK memory for Tx queue, which is not supported\n"); + return -EINVAL; + } + if (conf->use_locked_device_memory && conf->use_rte_memory) { + RTE_ETHDEV_LOG(ERR, +- "Attempt to use mutually exclusive memory settings for Tx queue"); ++ "Attempt to use mutually exclusive memory settings for Tx queue\n"); + return -EINVAL; + } + if (conf->force_memory && + !conf->use_locked_device_memory && + !conf->use_rte_memory) { + RTE_ETHDEV_LOG(ERR, +- "Attempt to force Tx queue memory settings, but none is set"); ++ "Attempt to force Tx queue memory settings, but none is set\n"); + return -EINVAL; + } + if (conf->peer_count == 0) { + RTE_ETHDEV_LOG(ERR, +- "Invalid value for number of peers for Tx queue(=%u), should be: > 0", ++ "Invalid value for number of peers for Tx queue(=%u), should be: > 0\n", + conf->peer_count); + return -EINVAL; + } +@@ -2641,7 +2641,7 @@ rte_eth_tx_hairpin_queue_setup(uint16_t port_id, uint16_t tx_queue_id, + count++; + } + if (count > cap.max_nb_queues) { +- RTE_ETHDEV_LOG(ERR, "To many Tx hairpin queues max is %d", ++ RTE_ETHDEV_LOG(ERR, "To many Tx hairpin queues max is %d\n", + cap.max_nb_queues); + return -EINVAL; + } +@@ -6716,7 +6716,7 @@ rte_eth_ip_reassembly_capability_get(uint16_t port_id, + } + + if (reassembly_capa == NULL) { +- RTE_ETHDEV_LOG(ERR, "Cannot get reassembly capability to NULL"); ++ RTE_ETHDEV_LOG(ERR, "Cannot get reassembly capability to NULL\n"); + return -EINVAL; + } + +@@ -6752,7 +6752,7 @@ rte_eth_ip_reassembly_conf_get(uint16_t port_id, + } + + if (conf == NULL) { +- RTE_ETHDEV_LOG(ERR, "Cannot get reassembly info to NULL"); ++ RTE_ETHDEV_LOG(ERR, "Cannot get reassembly info to NULL\n"); + return -EINVAL; + } + +@@ -6780,7 +6780,7 @@ rte_eth_ip_reassembly_conf_set(uint16_t port_id, + if (dev->data->dev_configured == 0) { + RTE_ETHDEV_LOG(ERR, + "Device with port_id=%u is not configured.\n" +- "Cannot set IP reassembly configuration", ++ "Cannot set IP reassembly configuration\n", + port_id); + return -EINVAL; + } +diff --git a/dpdk/lib/ethdev/rte_ethdev.h b/dpdk/lib/ethdev/rte_ethdev.h +index 77331ce652..545799c341 100644 +--- a/dpdk/lib/ethdev/rte_ethdev.h ++++ b/dpdk/lib/ethdev/rte_ethdev.h +@@ -331,13 +331,18 @@ struct rte_eth_stats { + /** + * A structure used to retrieve link-level information of an Ethernet port. + */ +-__extension__ + struct rte_eth_link { +- uint32_t link_speed; /**< RTE_ETH_SPEED_NUM_ */ +- uint16_t link_duplex : 1; /**< RTE_ETH_LINK_[HALF/FULL]_DUPLEX */ +- uint16_t link_autoneg : 1; /**< RTE_ETH_LINK_[AUTONEG/FIXED] */ +- uint16_t link_status : 1; /**< RTE_ETH_LINK_[DOWN/UP] */ +-} __rte_aligned(8); /**< aligned for atomic64 read/write */ ++ union { ++ RTE_ATOMIC(uint64_t) val64; /**< used for atomic64 read/write */ ++ __extension__ ++ struct { ++ uint32_t link_speed; /**< RTE_ETH_SPEED_NUM_ */ ++ uint16_t link_duplex : 1; /**< RTE_ETH_LINK_[HALF/FULL]_DUPLEX */ ++ uint16_t link_autoneg : 1; /**< RTE_ETH_LINK_[AUTONEG/FIXED] */ ++ uint16_t link_status : 1; /**< RTE_ETH_LINK_[DOWN/UP] */ ++ }; ++ }; ++}; + + /**@{@name Link negotiation + * Constants used in link management. +diff --git a/dpdk/lib/ethdev/rte_flow.c b/dpdk/lib/ethdev/rte_flow.c +index 549e329558..fa2a8fedce 100644 +--- a/dpdk/lib/ethdev/rte_flow.c ++++ b/dpdk/lib/ethdev/rte_flow.c +@@ -216,7 +216,7 @@ static const struct rte_flow_desc_data rte_flow_desc_action[] = { + sizeof(struct rte_flow_action_of_push_mpls)), + MK_FLOW_ACTION(VXLAN_ENCAP, sizeof(struct rte_flow_action_vxlan_encap)), + MK_FLOW_ACTION(VXLAN_DECAP, 0), +- MK_FLOW_ACTION(NVGRE_ENCAP, sizeof(struct rte_flow_action_vxlan_encap)), ++ MK_FLOW_ACTION(NVGRE_ENCAP, sizeof(struct rte_flow_action_nvgre_encap)), + MK_FLOW_ACTION(NVGRE_DECAP, 0), + MK_FLOW_ACTION(RAW_ENCAP, sizeof(struct rte_flow_action_raw_encap)), + MK_FLOW_ACTION(RAW_DECAP, sizeof(struct rte_flow_action_raw_decap)), +@@ -616,6 +616,7 @@ rte_flow_conv_item_spec(void *buf, const size_t size, + switch (item->type) { + union { + const struct rte_flow_item_raw *raw; ++ const struct rte_flow_item_geneve_opt *geneve_opt; + } spec; + union { + const struct rte_flow_item_raw *raw; +@@ -625,10 +626,13 @@ rte_flow_conv_item_spec(void *buf, const size_t size, + } mask; + union { + const struct rte_flow_item_raw *raw; ++ const struct rte_flow_item_geneve_opt *geneve_opt; + } src; + union { + struct rte_flow_item_raw *raw; ++ struct rte_flow_item_geneve_opt *geneve_opt; + } dst; ++ void *deep_src; + size_t tmp; + + case RTE_FLOW_ITEM_TYPE_RAW: +@@ -657,13 +661,30 @@ rte_flow_conv_item_spec(void *buf, const size_t size, + tmp = last.raw->length & mask.raw->length; + if (tmp) { + off = RTE_ALIGN_CEIL(off, sizeof(*dst.raw->pattern)); +- if (size >= off + tmp) +- dst.raw->pattern = rte_memcpy +- ((void *)((uintptr_t)dst.raw + off), +- src.raw->pattern, tmp); ++ if (size >= off + tmp) { ++ deep_src = (void *)((uintptr_t)dst.raw + off); ++ dst.raw->pattern = rte_memcpy(deep_src, ++ src.raw->pattern, ++ tmp); ++ } + off += tmp; + } + break; ++ case RTE_FLOW_ITEM_TYPE_GENEVE_OPT: ++ off = rte_flow_conv_copy(buf, data, size, ++ rte_flow_desc_item, item->type); ++ spec.geneve_opt = item->spec; ++ src.geneve_opt = data; ++ dst.geneve_opt = buf; ++ tmp = spec.geneve_opt->option_len << 2; ++ if (size > 0 && src.geneve_opt->data) { ++ deep_src = (void *)((uintptr_t)(dst.geneve_opt + 1)); ++ dst.geneve_opt->data = rte_memcpy(deep_src, ++ src.geneve_opt->data, ++ tmp); ++ } ++ off += tmp; ++ break; + default: + off = rte_flow_conv_copy(buf, data, size, + rte_flow_desc_item, item->type); +diff --git a/dpdk/lib/ethdev/rte_flow.h b/dpdk/lib/ethdev/rte_flow.h +index affdc8121b..4cdc1f1d8f 100644 +--- a/dpdk/lib/ethdev/rte_flow.h ++++ b/dpdk/lib/ethdev/rte_flow.h +@@ -3471,7 +3471,7 @@ struct rte_flow_action_vxlan_encap { + */ + struct rte_flow_action_nvgre_encap { + /** +- * Encapsulating vxlan tunnel definition ++ * Encapsulating nvgre tunnel definition + * (terminated by the END pattern item). + */ + struct rte_flow_item *definition; +diff --git a/dpdk/lib/eventdev/eventdev_pmd.h b/dpdk/lib/eventdev/eventdev_pmd.h +index 30bd90085c..2ec5aec0a8 100644 +--- a/dpdk/lib/eventdev/eventdev_pmd.h ++++ b/dpdk/lib/eventdev/eventdev_pmd.h +@@ -49,14 +49,14 @@ extern "C" { + /* Macros to check for valid device */ + #define RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, retval) do { \ + if (!rte_event_pmd_is_valid_dev((dev_id))) { \ +- RTE_EDEV_LOG_ERR("Invalid dev_id=%d\n", dev_id); \ ++ RTE_EDEV_LOG_ERR("Invalid dev_id=%d", dev_id); \ + return retval; \ + } \ + } while (0) + + #define RTE_EVENTDEV_VALID_DEVID_OR_ERRNO_RET(dev_id, errno, retval) do { \ + if (!rte_event_pmd_is_valid_dev((dev_id))) { \ +- RTE_EDEV_LOG_ERR("Invalid dev_id=%d\n", dev_id); \ ++ RTE_EDEV_LOG_ERR("Invalid dev_id=%d", dev_id); \ + rte_errno = errno; \ + return retval; \ + } \ +@@ -64,7 +64,7 @@ extern "C" { + + #define RTE_EVENTDEV_VALID_DEVID_OR_RET(dev_id) do { \ + if (!rte_event_pmd_is_valid_dev((dev_id))) { \ +- RTE_EDEV_LOG_ERR("Invalid dev_id=%d\n", dev_id); \ ++ RTE_EDEV_LOG_ERR("Invalid dev_id=%d", dev_id); \ + return; \ + } \ + } while (0) +diff --git a/dpdk/lib/eventdev/rte_event_crypto_adapter.c b/dpdk/lib/eventdev/rte_event_crypto_adapter.c +index 1b435c9f0e..d11c0d4be8 100644 +--- a/dpdk/lib/eventdev/rte_event_crypto_adapter.c ++++ b/dpdk/lib/eventdev/rte_event_crypto_adapter.c +@@ -133,11 +133,33 @@ static struct event_crypto_adapter **event_crypto_adapter; + /* Macros to check for valid adapter */ + #define EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, retval) do { \ + if (!eca_valid_id(id)) { \ +- RTE_EDEV_LOG_ERR("Invalid crypto adapter id = %d\n", id); \ ++ RTE_EDEV_LOG_ERR("Invalid crypto adapter id = %d", id); \ + return retval; \ + } \ + } while (0) + ++#define ECA_DYNFIELD_NAME "eca_ev_opaque_data" ++/* Device-specific metadata field type */ ++typedef uint8_t eca_dynfield_t; ++ ++/* mbuf dynamic field offset for device-specific metadata */ ++int eca_dynfield_offset = -1; ++ ++static int ++eca_dynfield_register(void) ++{ ++ static const struct rte_mbuf_dynfield eca_dynfield_desc = { ++ .name = ECA_DYNFIELD_NAME, ++ .size = sizeof(eca_dynfield_t), ++ .align = __alignof__(eca_dynfield_t), ++ .flags = 0, ++ }; ++ ++ eca_dynfield_offset = ++ rte_mbuf_dynfield_register(&eca_dynfield_desc); ++ return eca_dynfield_offset; ++} ++ + static inline int + eca_valid_id(uint8_t id) + { +@@ -245,20 +267,28 @@ eca_circular_buffer_flush_to_cdev(struct crypto_ops_circular_buffer *bufp, + struct rte_crypto_op **ops = bufp->op_buffer; + + if (*tailp > *headp) ++ /* Flush ops from head pointer to (tail - head) OPs */ + n = *tailp - *headp; + else if (*tailp < *headp) ++ /* Circ buffer - Rollover. ++ * Flush OPs from head to max size of buffer. ++ * Rest of the OPs will be flushed in next iteration. ++ */ + n = bufp->size - *headp; + else { /* head == tail case */ + /* when head == tail, + * circ buff is either full(tail pointer roll over) or empty + */ + if (bufp->count != 0) { +- /* circ buffer is full */ +- n = bufp->count; ++ /* Circ buffer - FULL. ++ * Flush OPs from head to max size of buffer. ++ * Rest of the OPS will be flushed in next iteration. ++ */ ++ n = bufp->size - *headp; + } else { +- /* circ buffer is empty */ ++ /* Circ buffer - Empty */ + *nb_ops_flushed = 0; +- return 0; /* buffer empty */ ++ return 0; + } + } + +@@ -309,7 +339,7 @@ eca_default_config_cb(uint8_t id, uint8_t dev_id, + + ret = rte_event_dev_configure(dev_id, &dev_conf); + if (ret) { +- RTE_EDEV_LOG_ERR("failed to configure event dev %u\n", dev_id); ++ RTE_EDEV_LOG_ERR("failed to configure event dev %u", dev_id); + if (started) { + if (rte_event_dev_start(dev_id)) + return -EIO; +@@ -319,7 +349,7 @@ eca_default_config_cb(uint8_t id, uint8_t dev_id, + + ret = rte_event_port_setup(dev_id, port_id, port_conf); + if (ret) { +- RTE_EDEV_LOG_ERR("failed to setup event port %u\n", port_id); ++ RTE_EDEV_LOG_ERR("failed to setup event port %u", port_id); + return ret; + } + +@@ -391,7 +421,7 @@ rte_event_crypto_adapter_create_ext(uint8_t id, uint8_t dev_id, + sizeof(struct crypto_device_info), 0, + socket_id); + if (adapter->cdevs == NULL) { +- RTE_EDEV_LOG_ERR("Failed to get mem for crypto devices\n"); ++ RTE_EDEV_LOG_ERR("Failed to get mem for crypto devices"); + eca_circular_buffer_free(&adapter->ebuf); + rte_free(adapter); + return -ENOMEM; +@@ -483,6 +513,25 @@ eca_enq_to_cryptodev(struct event_crypto_adapter *adapter, struct rte_event *ev, + crypto_op = ev[i].event_ptr; + if (crypto_op == NULL) + continue; ++ ++ /** "struct rte_event::impl_opaque" field passed on from ++ * eventdev PMD could have different value per event. ++ * For session-based crypto operations retain ++ * "struct rte_event::impl_opaque" into mbuf dynamic field and ++ * restore it back after copying event information from ++ * session event metadata. ++ * For session-less, each crypto operation carries event ++ * metadata and retains "struct rte_event:impl_opaque" ++ * information to be passed back to eventdev PMD. ++ */ ++ if (crypto_op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { ++ struct rte_mbuf *mbuf = crypto_op->sym->m_src; ++ ++ *RTE_MBUF_DYNFIELD(mbuf, ++ eca_dynfield_offset, ++ eca_dynfield_t *) = ev[i].impl_opaque; ++ } ++ + m_data = rte_cryptodev_session_event_mdata_get(crypto_op); + if (m_data == NULL) { + rte_pktmbuf_free(crypto_op->sym->m_src); +@@ -649,6 +698,21 @@ eca_ops_enqueue_burst(struct event_crypto_adapter *adapter, + + rte_memcpy(ev, &m_data->response_info, sizeof(*ev)); + ev->event_ptr = ops[i]; ++ ++ /** Restore "struct rte_event::impl_opaque" from mbuf ++ * dynamic field for session based crypto operation. ++ * For session-less, each crypto operations carries event ++ * metadata and retains "struct rte_event::impl_opaque" ++ * information to be passed back to eventdev PMD. ++ */ ++ if (ops[i]->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { ++ struct rte_mbuf *mbuf = ops[i]->sym->m_src; ++ ++ ev->impl_opaque = *RTE_MBUF_DYNFIELD(mbuf, ++ eca_dynfield_offset, ++ eca_dynfield_t *); ++ } ++ + ev->event_type = RTE_EVENT_TYPE_CRYPTODEV; + if (adapter->implicit_release_disabled) + ev->op = RTE_EVENT_OP_FORWARD; +@@ -887,6 +951,18 @@ eca_init_service(struct event_crypto_adapter *adapter, uint8_t id) + } + + adapter->implicit_release_disabled = (uint8_t)impl_rel; ++ ++ /** Register for mbuf dyn field to store/restore ++ * "struct rte_event::impl_opaque" ++ */ ++ eca_dynfield_offset = eca_dynfield_register(); ++ if (eca_dynfield_offset < 0) { ++ RTE_EDEV_LOG_ERR("Failed to register eca mbuf dyn field"); ++ eca_circular_buffer_free(&adapter->ebuf); ++ rte_free(adapter); ++ return -EINVAL; ++ } ++ + adapter->service_inited = 1; + + return ret; +@@ -1403,7 +1479,7 @@ rte_event_crypto_adapter_runtime_params_set(uint8_t id, + EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL); + + if (params == NULL) { +- RTE_EDEV_LOG_ERR("params pointer is NULL\n"); ++ RTE_EDEV_LOG_ERR("params pointer is NULL"); + return -EINVAL; + } + +@@ -1436,7 +1512,7 @@ rte_event_crypto_adapter_runtime_params_get(uint8_t id, + EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL); + + if (params == NULL) { +- RTE_EDEV_LOG_ERR("params pointer is NULL\n"); ++ RTE_EDEV_LOG_ERR("params pointer is NULL"); + return -EINVAL; + } + +diff --git a/dpdk/lib/eventdev/rte_event_dma_adapter.c b/dpdk/lib/eventdev/rte_event_dma_adapter.c +index af4b5ad388..4196164305 100644 +--- a/dpdk/lib/eventdev/rte_event_dma_adapter.c ++++ b/dpdk/lib/eventdev/rte_event_dma_adapter.c +@@ -20,7 +20,7 @@ + #define EVENT_DMA_ADAPTER_ID_VALID_OR_ERR_RET(id, retval) \ + do { \ + if (!edma_adapter_valid_id(id)) { \ +- RTE_EDEV_LOG_ERR("Invalid DMA adapter id = %d\n", id); \ ++ RTE_EDEV_LOG_ERR("Invalid DMA adapter id = %d", id); \ + return retval; \ + } \ + } while (0) +@@ -313,7 +313,7 @@ edma_default_config_cb(uint8_t id, uint8_t evdev_id, struct rte_event_dma_adapte + + ret = rte_event_dev_configure(evdev_id, &dev_conf); + if (ret) { +- RTE_EDEV_LOG_ERR("Failed to configure event dev %u\n", evdev_id); ++ RTE_EDEV_LOG_ERR("Failed to configure event dev %u", evdev_id); + if (started) { + if (rte_event_dev_start(evdev_id)) + return -EIO; +@@ -323,7 +323,7 @@ edma_default_config_cb(uint8_t id, uint8_t evdev_id, struct rte_event_dma_adapte + + ret = rte_event_port_setup(evdev_id, port_id, port_conf); + if (ret) { +- RTE_EDEV_LOG_ERR("Failed to setup event port %u\n", port_id); ++ RTE_EDEV_LOG_ERR("Failed to setup event port %u", port_id); + return ret; + } + +@@ -407,7 +407,7 @@ rte_event_dma_adapter_create_ext(uint8_t id, uint8_t evdev_id, + num_dma_dev * sizeof(struct dma_device_info), 0, + socket_id); + if (adapter->dma_devs == NULL) { +- RTE_EDEV_LOG_ERR("Failed to get memory for DMA devices\n"); ++ RTE_EDEV_LOG_ERR("Failed to get memory for DMA devices"); + edma_circular_buffer_free(&adapter->ebuf); + rte_free(adapter); + return -ENOMEM; +@@ -417,7 +417,7 @@ rte_event_dma_adapter_create_ext(uint8_t id, uint8_t evdev_id, + for (i = 0; i < num_dma_dev; i++) { + ret = rte_dma_info_get(i, &info); + if (ret) { +- RTE_EDEV_LOG_ERR("Failed to get dma device info\n"); ++ RTE_EDEV_LOG_ERR("Failed to get dma device info"); + edma_circular_buffer_free(&adapter->ebuf); + rte_free(adapter); + return ret; +@@ -1046,7 +1046,7 @@ rte_event_dma_adapter_vchan_add(uint8_t id, int16_t dma_dev_id, uint16_t vchan, + sizeof(struct dma_vchan_info), + 0, adapter->socket_id); + if (dev_info->vchanq == NULL) { +- printf("Queue pair add not supported\n"); ++ RTE_EDEV_LOG_ERR("Queue pair add not supported"); + return -ENOMEM; + } + } +@@ -1057,7 +1057,7 @@ rte_event_dma_adapter_vchan_add(uint8_t id, int16_t dma_dev_id, uint16_t vchan, + sizeof(struct dma_vchan_info), + 0, adapter->socket_id); + if (dev_info->tqmap == NULL) { +- printf("tq pair add not supported\n"); ++ RTE_EDEV_LOG_ERR("tq pair add not supported"); + return -ENOMEM; + } + } +@@ -1297,7 +1297,7 @@ rte_event_dma_adapter_runtime_params_set(uint8_t id, + EVENT_DMA_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL); + + if (params == NULL) { +- RTE_EDEV_LOG_ERR("params pointer is NULL\n"); ++ RTE_EDEV_LOG_ERR("params pointer is NULL"); + return -EINVAL; + } + +@@ -1326,7 +1326,7 @@ rte_event_dma_adapter_runtime_params_get(uint8_t id, + EVENT_DMA_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL); + + if (params == NULL) { +- RTE_EDEV_LOG_ERR("params pointer is NULL\n"); ++ RTE_EDEV_LOG_ERR("params pointer is NULL"); + return -EINVAL; + } + +diff --git a/dpdk/lib/eventdev/rte_event_eth_rx_adapter.c b/dpdk/lib/eventdev/rte_event_eth_rx_adapter.c +index 6db03adf04..1b83a55b5c 100644 +--- a/dpdk/lib/eventdev/rte_event_eth_rx_adapter.c ++++ b/dpdk/lib/eventdev/rte_event_eth_rx_adapter.c +@@ -293,14 +293,14 @@ rxa_event_buf_get(struct event_eth_rx_adapter *rx_adapter, uint16_t eth_dev_id, + + #define RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, retval) do { \ + if (!rxa_validate_id(id)) { \ +- RTE_EDEV_LOG_ERR("Invalid eth Rx adapter id = %d\n", id); \ ++ RTE_EDEV_LOG_ERR("Invalid eth Rx adapter id = %d", id); \ + return retval; \ + } \ + } while (0) + + #define RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_GOTO_ERR_RET(id, retval) do { \ + if (!rxa_validate_id(id)) { \ +- RTE_EDEV_LOG_ERR("Invalid eth Rx adapter id = %d\n", id); \ ++ RTE_EDEV_LOG_ERR("Invalid eth Rx adapter id = %d", id); \ + ret = retval; \ + goto error; \ + } \ +@@ -308,15 +308,15 @@ rxa_event_buf_get(struct event_eth_rx_adapter *rx_adapter, uint16_t eth_dev_id, + + #define RTE_EVENT_ETH_RX_ADAPTER_TOKEN_VALID_OR_GOTO_ERR_RET(token, retval) do { \ + if ((token) == NULL || strlen(token) == 0 || !isdigit(*token)) { \ +- RTE_EDEV_LOG_ERR("Invalid eth Rx adapter token\n"); \ ++ RTE_EDEV_LOG_ERR("Invalid eth Rx adapter token"); \ + ret = retval; \ + goto error; \ + } \ + } while (0) + +-#define RTE_ETH_VALID_PORTID_OR_GOTO_ERR_RET(port_id, retval) do { \ ++#define RTE_EVENT_ETH_RX_ADAPTER_PORTID_VALID_OR_GOTO_ERR_RET(port_id, retval) do { \ + if (!rte_eth_dev_is_valid_port(port_id)) { \ +- RTE_ETHDEV_LOG(ERR, "Invalid port_id=%u\n", port_id); \ ++ RTE_EDEV_LOG_ERR("Invalid port_id=%u", port_id); \ + ret = retval; \ + goto error; \ + } \ +@@ -1540,7 +1540,7 @@ rxa_default_conf_cb(uint8_t id, uint8_t dev_id, + + ret = rte_event_dev_configure(dev_id, &dev_conf); + if (ret) { +- RTE_EDEV_LOG_ERR("failed to configure event dev %u\n", ++ RTE_EDEV_LOG_ERR("failed to configure event dev %u", + dev_id); + if (started) { + if (rte_event_dev_start(dev_id)) +@@ -1551,7 +1551,7 @@ rxa_default_conf_cb(uint8_t id, uint8_t dev_id, + + ret = rte_event_port_setup(dev_id, port_id, port_conf); + if (ret) { +- RTE_EDEV_LOG_ERR("failed to setup event port %u\n", ++ RTE_EDEV_LOG_ERR("failed to setup event port %u", + port_id); + return ret; + } +@@ -1628,7 +1628,7 @@ rxa_create_intr_thread(struct event_eth_rx_adapter *rx_adapter) + if (!err) + return 0; + +- RTE_EDEV_LOG_ERR("Failed to create interrupt thread err = %d\n", err); ++ RTE_EDEV_LOG_ERR("Failed to create interrupt thread err = %d", err); + rte_free(rx_adapter->epoll_events); + error: + rte_ring_free(rx_adapter->intr_ring); +@@ -1644,12 +1644,12 @@ rxa_destroy_intr_thread(struct event_eth_rx_adapter *rx_adapter) + + err = pthread_cancel((pthread_t)rx_adapter->rx_intr_thread.opaque_id); + if (err) +- RTE_EDEV_LOG_ERR("Can't cancel interrupt thread err = %d\n", ++ RTE_EDEV_LOG_ERR("Can't cancel interrupt thread err = %d", + err); + + err = rte_thread_join(rx_adapter->rx_intr_thread, NULL); + if (err) +- RTE_EDEV_LOG_ERR("Can't join interrupt thread err = %d\n", err); ++ RTE_EDEV_LOG_ERR("Can't join interrupt thread err = %d", err); + + rte_free(rx_adapter->epoll_events); + rte_ring_free(rx_adapter->intr_ring); +@@ -1915,7 +1915,7 @@ rxa_init_service(struct event_eth_rx_adapter *rx_adapter, uint8_t id) + if (rte_mbuf_dyn_rx_timestamp_register( + &event_eth_rx_timestamp_dynfield_offset, + &event_eth_rx_timestamp_dynflag) != 0) { +- RTE_EDEV_LOG_ERR("Error registering timestamp field in mbuf\n"); ++ RTE_EDEV_LOG_ERR("Error registering timestamp field in mbuf"); + return -rte_errno; + } + +@@ -2445,7 +2445,7 @@ rxa_create(uint8_t id, uint8_t dev_id, + RTE_DIM(default_rss_key)); + + if (rx_adapter->eth_devices == NULL) { +- RTE_EDEV_LOG_ERR("failed to get mem for eth devices\n"); ++ RTE_EDEV_LOG_ERR("failed to get mem for eth devices"); + rte_free(rx_adapter); + return -ENOMEM; + } +@@ -2497,12 +2497,12 @@ rxa_config_params_validate(struct rte_event_eth_rx_adapter_params *rxa_params, + return 0; + } else if (!rxa_params->use_queue_event_buf && + rxa_params->event_buf_size == 0) { +- RTE_EDEV_LOG_ERR("event buffer size can't be zero\n"); ++ RTE_EDEV_LOG_ERR("event buffer size can't be zero"); + return -EINVAL; + } else if (rxa_params->use_queue_event_buf && + rxa_params->event_buf_size != 0) { + RTE_EDEV_LOG_ERR("event buffer size needs to be configured " +- "as part of queue add\n"); ++ "as part of queue add"); + return -EINVAL; + } + +@@ -3597,7 +3597,7 @@ handle_rxa_stats(const char *cmd __rte_unused, + /* Get Rx adapter stats */ + if (rte_event_eth_rx_adapter_stats_get(rx_adapter_id, + &rx_adptr_stats)) { +- RTE_EDEV_LOG_ERR("Failed to get Rx adapter stats\n"); ++ RTE_EDEV_LOG_ERR("Failed to get Rx adapter stats"); + return -1; + } + +@@ -3636,7 +3636,7 @@ handle_rxa_stats_reset(const char *cmd __rte_unused, + + /* Reset Rx adapter stats */ + if (rte_event_eth_rx_adapter_stats_reset(rx_adapter_id)) { +- RTE_EDEV_LOG_ERR("Failed to reset Rx adapter stats\n"); ++ RTE_EDEV_LOG_ERR("Failed to reset Rx adapter stats"); + return -1; + } + +@@ -3671,7 +3671,7 @@ handle_rxa_get_queue_conf(const char *cmd __rte_unused, + + /* Get device ID from parameter string */ + eth_dev_id = strtoul(token, NULL, 10); +- RTE_ETH_VALID_PORTID_OR_GOTO_ERR_RET(eth_dev_id, -EINVAL); ++ RTE_EVENT_ETH_RX_ADAPTER_PORTID_VALID_OR_GOTO_ERR_RET(eth_dev_id, -EINVAL); + + token = strtok(NULL, ","); + RTE_EVENT_ETH_RX_ADAPTER_TOKEN_VALID_OR_GOTO_ERR_RET(token, -1); +@@ -3743,7 +3743,7 @@ handle_rxa_get_queue_stats(const char *cmd __rte_unused, + + /* Get device ID from parameter string */ + eth_dev_id = strtoul(token, NULL, 10); +- RTE_ETH_VALID_PORTID_OR_GOTO_ERR_RET(eth_dev_id, -EINVAL); ++ RTE_EVENT_ETH_RX_ADAPTER_PORTID_VALID_OR_GOTO_ERR_RET(eth_dev_id, -EINVAL); + + token = strtok(NULL, ","); + RTE_EVENT_ETH_RX_ADAPTER_TOKEN_VALID_OR_GOTO_ERR_RET(token, -1); +@@ -3813,7 +3813,7 @@ handle_rxa_queue_stats_reset(const char *cmd __rte_unused, + + /* Get device ID from parameter string */ + eth_dev_id = strtoul(token, NULL, 10); +- RTE_ETH_VALID_PORTID_OR_GOTO_ERR_RET(eth_dev_id, -EINVAL); ++ RTE_EVENT_ETH_RX_ADAPTER_PORTID_VALID_OR_GOTO_ERR_RET(eth_dev_id, -EINVAL); + + token = strtok(NULL, ","); + RTE_EVENT_ETH_RX_ADAPTER_TOKEN_VALID_OR_GOTO_ERR_RET(token, -1); +@@ -3868,7 +3868,7 @@ handle_rxa_instance_get(const char *cmd __rte_unused, + + /* Get device ID from parameter string */ + eth_dev_id = strtoul(token, NULL, 10); +- RTE_ETH_VALID_PORTID_OR_GOTO_ERR_RET(eth_dev_id, -EINVAL); ++ RTE_EVENT_ETH_RX_ADAPTER_PORTID_VALID_OR_GOTO_ERR_RET(eth_dev_id, -EINVAL); + + token = strtok(NULL, ","); + RTE_EVENT_ETH_RX_ADAPTER_TOKEN_VALID_OR_GOTO_ERR_RET(token, -1); +diff --git a/dpdk/lib/eventdev/rte_event_eth_tx_adapter.c b/dpdk/lib/eventdev/rte_event_eth_tx_adapter.c +index 360d5caf6a..56435be991 100644 +--- a/dpdk/lib/eventdev/rte_event_eth_tx_adapter.c ++++ b/dpdk/lib/eventdev/rte_event_eth_tx_adapter.c +@@ -334,7 +334,7 @@ txa_service_conf_cb(uint8_t __rte_unused id, uint8_t dev_id, + + ret = rte_event_port_setup(dev_id, port_id, pc); + if (ret) { +- RTE_EDEV_LOG_ERR("failed to setup event port %u\n", ++ RTE_EDEV_LOG_ERR("failed to setup event port %u", + port_id); + if (started) { + if (rte_event_dev_start(dev_id)) +diff --git a/dpdk/lib/eventdev/rte_event_timer_adapter.c b/dpdk/lib/eventdev/rte_event_timer_adapter.c +index 27466707bc..3f22e85173 100644 +--- a/dpdk/lib/eventdev/rte_event_timer_adapter.c ++++ b/dpdk/lib/eventdev/rte_event_timer_adapter.c +@@ -106,7 +106,7 @@ default_port_conf_cb(uint16_t id, uint8_t event_dev_id, uint8_t *event_port_id, + + ret = rte_event_dev_configure(dev_id, &dev_conf); + if (ret < 0) { +- EVTIM_LOG_ERR("failed to configure event dev %u\n", dev_id); ++ EVTIM_LOG_ERR("failed to configure event dev %u", dev_id); + if (started) + if (rte_event_dev_start(dev_id)) + return -EIO; +@@ -116,7 +116,7 @@ default_port_conf_cb(uint16_t id, uint8_t event_dev_id, uint8_t *event_port_id, + + ret = rte_event_port_setup(dev_id, port_id, port_conf); + if (ret < 0) { +- EVTIM_LOG_ERR("failed to setup event port %u on event dev %u\n", ++ EVTIM_LOG_ERR("failed to setup event port %u on event dev %u", + port_id, dev_id); + return ret; + } +diff --git a/dpdk/lib/eventdev/rte_eventdev.c b/dpdk/lib/eventdev/rte_eventdev.c +index 0ca32d6721..1c865e993f 100644 +--- a/dpdk/lib/eventdev/rte_eventdev.c ++++ b/dpdk/lib/eventdev/rte_eventdev.c +@@ -1007,13 +1007,13 @@ rte_event_port_profile_links_set(uint8_t dev_id, uint8_t port_id, const uint8_t + } + + if (*dev->dev_ops->port_link == NULL) { +- RTE_EDEV_LOG_ERR("Function not supported\n"); ++ RTE_EDEV_LOG_ERR("Function not supported"); + rte_errno = ENOTSUP; + return 0; + } + + if (profile_id && *dev->dev_ops->port_link_profile == NULL) { +- RTE_EDEV_LOG_ERR("Function not supported\n"); ++ RTE_EDEV_LOG_ERR("Function not supported"); + rte_errno = ENOTSUP; + return 0; + } +@@ -1428,8 +1428,8 @@ rte_event_vector_pool_create(const char *name, unsigned int n, + int ret; + + if (!nb_elem) { +- RTE_LOG(ERR, EVENTDEV, +- "Invalid number of elements=%d requested\n", nb_elem); ++ RTE_EDEV_LOG_ERR("Invalid number of elements=%d requested", ++ nb_elem); + rte_errno = EINVAL; + return NULL; + } +@@ -1444,7 +1444,7 @@ rte_event_vector_pool_create(const char *name, unsigned int n, + mp_ops_name = rte_mbuf_best_mempool_ops(); + ret = rte_mempool_set_ops_byname(mp, mp_ops_name, NULL); + if (ret != 0) { +- RTE_LOG(ERR, EVENTDEV, "error setting mempool handler\n"); ++ RTE_EDEV_LOG_ERR("error setting mempool handler"); + goto err; + } + +@@ -2002,7 +2002,7 @@ handle_dev_dump(const char *cmd __rte_unused, + + RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL); + +- buf = calloc(sizeof(char), RTE_TEL_MAX_SINGLE_STRING_LEN); ++ buf = calloc(RTE_TEL_MAX_SINGLE_STRING_LEN, sizeof(char)); + if (buf == NULL) + return -ENOMEM; + +diff --git a/dpdk/lib/eventdev/rte_eventdev.h b/dpdk/lib/eventdev/rte_eventdev.h +index ec9b02455d..7fd9016ca7 100644 +--- a/dpdk/lib/eventdev/rte_eventdev.h ++++ b/dpdk/lib/eventdev/rte_eventdev.h +@@ -515,9 +515,9 @@ rte_event_dev_attr_get(uint8_t dev_id, uint32_t attr_id, + struct rte_event_dev_config { + uint32_t dequeue_timeout_ns; + /**< rte_event_dequeue_burst() timeout on this device. +- * This value should be in the range of *min_dequeue_timeout_ns* and +- * *max_dequeue_timeout_ns* which previously provided in +- * rte_event_dev_info_get() ++ * This value should be in the range of @ref rte_event_dev_info.min_dequeue_timeout_ns and ++ * @ref rte_event_dev_info.max_dequeue_timeout_ns returned by ++ * @ref rte_event_dev_info_get() + * The value 0 is allowed, in which case, default dequeue timeout used. + * @see RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT + */ +@@ -525,40 +525,53 @@ struct rte_event_dev_config { + /**< In a *closed system* this field is the limit on maximum number of + * events that can be inflight in the eventdev at a given time. The + * limit is required to ensure that the finite space in a closed system +- * is not overwhelmed. The value cannot exceed the *max_num_events* +- * as provided by rte_event_dev_info_get(). +- * This value should be set to -1 for *open system*. ++ * is not exhausted. ++ * The value cannot exceed @ref rte_event_dev_info.max_num_events ++ * returned by rte_event_dev_info_get(). ++ * ++ * This value should be set to -1 for *open systems*, that is, ++ * those systems returning -1 in @ref rte_event_dev_info.max_num_events. ++ * ++ * @see rte_event_port_conf.new_event_threshold + */ + uint8_t nb_event_queues; + /**< Number of event queues to configure on this device. +- * This value cannot exceed the *max_event_queues* which previously +- * provided in rte_event_dev_info_get() ++ * This value *includes* any single-link queue-port pairs to be used. ++ * This value cannot exceed @ref rte_event_dev_info.max_event_queues + ++ * @ref rte_event_dev_info.max_single_link_event_port_queue_pairs ++ * returned by rte_event_dev_info_get(). ++ * The number of non-single-link queues i.e. this value less ++ * *nb_single_link_event_port_queues* in this struct, cannot exceed ++ * @ref rte_event_dev_info.max_event_queues + */ + uint8_t nb_event_ports; + /**< Number of event ports to configure on this device. +- * This value cannot exceed the *max_event_ports* which previously +- * provided in rte_event_dev_info_get() ++ * This value *includes* any single-link queue-port pairs to be used. ++ * This value cannot exceed @ref rte_event_dev_info.max_event_ports + ++ * @ref rte_event_dev_info.max_single_link_event_port_queue_pairs ++ * returned by rte_event_dev_info_get(). ++ * The number of non-single-link ports i.e. this value less ++ * *nb_single_link_event_port_queues* in this struct, cannot exceed ++ * @ref rte_event_dev_info.max_event_ports + */ + uint32_t nb_event_queue_flows; +- /**< Number of flows for any event queue on this device. +- * This value cannot exceed the *max_event_queue_flows* which previously +- * provided in rte_event_dev_info_get() ++ /**< Max number of flows needed for a single event queue on this device. ++ * This value cannot exceed @ref rte_event_dev_info.max_event_queue_flows ++ * returned by rte_event_dev_info_get() + */ + uint32_t nb_event_port_dequeue_depth; +- /**< Maximum number of events can be dequeued at a time from an +- * event port by this device. +- * This value cannot exceed the *max_event_port_dequeue_depth* +- * which previously provided in rte_event_dev_info_get(). ++ /**< Max number of events that can be dequeued at a time from an event port on this device. ++ * This value cannot exceed @ref rte_event_dev_info.max_event_port_dequeue_depth ++ * returned by rte_event_dev_info_get(). + * Ignored when device is not RTE_EVENT_DEV_CAP_BURST_MODE capable. +- * @see rte_event_port_setup() ++ * @see rte_event_port_setup() rte_event_dequeue_burst() + */ + uint32_t nb_event_port_enqueue_depth; +- /**< Maximum number of events can be enqueued at a time from an +- * event port by this device. +- * This value cannot exceed the *max_event_port_enqueue_depth* +- * which previously provided in rte_event_dev_info_get(). ++ /**< Maximum number of events can be enqueued at a time to an event port on this device. ++ * This value cannot exceed @ref rte_event_dev_info.max_event_port_enqueue_depth ++ * returned by rte_event_dev_info_get(). + * Ignored when device is not RTE_EVENT_DEV_CAP_BURST_MODE capable. +- * @see rte_event_port_setup() ++ * @see rte_event_port_setup() rte_event_enqueue_burst() + */ + uint32_t event_dev_cfg; + /**< Event device config flags(RTE_EVENT_DEV_CFG_)*/ +@@ -568,7 +581,7 @@ struct rte_event_dev_config { + * queues; this value cannot exceed *nb_event_ports* or + * *nb_event_queues*. If the device has ports and queues that are + * optimized for single-link usage, this field is a hint for how many +- * to allocate; otherwise, regular event ports and queues can be used. ++ * to allocate; otherwise, regular event ports and queues will be used. + */ + }; + +@@ -1098,10 +1111,8 @@ struct rte_event_vector { + * port and queue of the mbufs in the vector + */ + struct { +- uint16_t port; +- /* Ethernet device port id. */ +- uint16_t queue; +- /* Ethernet device queue id. */ ++ uint16_t port; /**< Ethernet device port id. */ ++ uint16_t queue; /**< Ethernet device queue id. */ + }; + }; + /**< Union to hold common attributes of the vector array. */ +@@ -1130,7 +1141,11 @@ struct rte_event_vector { + * vector array can be an array of mbufs or pointers or opaque u64 + * values. + */ ++#ifndef __DOXYGEN__ + } __rte_aligned(16); ++#else ++}; ++#endif + + /* Scheduler type definitions */ + #define RTE_SCHED_TYPE_ORDERED 0 +diff --git a/dpdk/lib/graph/graph.c b/dpdk/lib/graph/graph.c +index 26f0968a97..8ea2109645 100644 +--- a/dpdk/lib/graph/graph.c ++++ b/dpdk/lib/graph/graph.c +@@ -19,11 +19,54 @@ + + static struct graph_head graph_list = STAILQ_HEAD_INITIALIZER(graph_list); + static rte_spinlock_t graph_lock = RTE_SPINLOCK_INITIALIZER; +-static rte_graph_t graph_id; +- +-#define GRAPH_ID_CHECK(id) ID_CHECK(id, graph_id) + + /* Private functions */ ++static struct graph * ++graph_from_id(rte_graph_t id) ++{ ++ struct graph *graph; ++ STAILQ_FOREACH(graph, &graph_list, next) { ++ if (graph->id == id) ++ return graph; ++ } ++ rte_errno = EINVAL; ++ return NULL; ++} ++ ++static rte_graph_t ++graph_next_free_id(void) ++{ ++ struct graph *graph; ++ rte_graph_t id = 0; ++ ++ STAILQ_FOREACH(graph, &graph_list, next) { ++ if (id < graph->id) ++ break; ++ id = graph->id + 1; ++ } ++ ++ return id; ++} ++ ++static void ++graph_insert_ordered(struct graph *graph) ++{ ++ struct graph *after, *g; ++ ++ after = NULL; ++ STAILQ_FOREACH(g, &graph_list, next) { ++ if (g->id < graph->id) ++ after = g; ++ else if (g->id > graph->id) ++ break; ++ } ++ if (after == NULL) { ++ STAILQ_INSERT_HEAD(&graph_list, graph, next); ++ } else { ++ STAILQ_INSERT_AFTER(&graph_list, after, graph, next); ++ } ++} ++ + struct graph_head * + graph_list_head_get(void) + { +@@ -279,7 +322,8 @@ rte_graph_model_mcore_dispatch_core_bind(rte_graph_t id, int lcore) + { + struct graph *graph; + +- GRAPH_ID_CHECK(id); ++ if (graph_from_id(id) == NULL) ++ goto fail; + if (!rte_lcore_is_enabled(lcore)) + SET_ERR_JMP(ENOLINK, fail, "lcore %d not enabled", lcore); + +@@ -309,7 +353,8 @@ rte_graph_model_mcore_dispatch_core_unbind(rte_graph_t id) + { + struct graph *graph; + +- GRAPH_ID_CHECK(id); ++ if (graph_from_id(id) == NULL) ++ goto fail; + STAILQ_FOREACH(graph, &graph_list, next) + if (graph->id == id) + break; +@@ -406,7 +451,7 @@ rte_graph_create(const char *name, struct rte_graph_param *prm) + graph->socket = prm->socket_id; + graph->src_node_count = src_node_count; + graph->node_count = graph_nodes_count(graph); +- graph->id = graph_id; ++ graph->id = graph_next_free_id(); + graph->parent_id = RTE_GRAPH_ID_INVALID; + graph->lcore_id = RTE_MAX_LCORE; + graph->num_pkt_to_capture = prm->num_pkt_to_capture; +@@ -422,8 +467,7 @@ rte_graph_create(const char *name, struct rte_graph_param *prm) + goto graph_mem_destroy; + + /* All good, Lets add the graph to the list */ +- graph_id++; +- STAILQ_INSERT_TAIL(&graph_list, graph, next); ++ graph_insert_ordered(graph); + + graph_spinlock_unlock(); + return graph->id; +@@ -467,7 +511,6 @@ rte_graph_destroy(rte_graph_t id) + graph_cleanup(graph); + STAILQ_REMOVE(&graph_list, graph, graph, next); + free(graph); +- graph_id--; + goto done; + } + graph = tmp; +@@ -520,7 +563,7 @@ graph_clone(struct graph *parent_graph, const char *name, struct rte_graph_param + graph->parent_id = parent_graph->id; + graph->lcore_id = parent_graph->lcore_id; + graph->socket = parent_graph->socket; +- graph->id = graph_id; ++ graph->id = graph_next_free_id(); + + /* Allocate the Graph fast path memory and populate the data */ + if (graph_fp_mem_create(graph)) +@@ -539,8 +582,7 @@ graph_clone(struct graph *parent_graph, const char *name, struct rte_graph_param + goto graph_mem_destroy; + + /* All good, Lets add the graph to the list */ +- graph_id++; +- STAILQ_INSERT_TAIL(&graph_list, graph, next); ++ graph_insert_ordered(graph); + + graph_spinlock_unlock(); + return graph->id; +@@ -561,7 +603,8 @@ rte_graph_clone(rte_graph_t id, const char *name, struct rte_graph_param *prm) + { + struct graph *graph; + +- GRAPH_ID_CHECK(id); ++ if (graph_from_id(id) == NULL) ++ goto fail; + STAILQ_FOREACH(graph, &graph_list, next) + if (graph->id == id) + return graph_clone(graph, name, prm); +@@ -587,7 +630,8 @@ rte_graph_id_to_name(rte_graph_t id) + { + struct graph *graph; + +- GRAPH_ID_CHECK(id); ++ if (graph_from_id(id) == NULL) ++ goto fail; + STAILQ_FOREACH(graph, &graph_list, next) + if (graph->id == id) + return graph->name; +@@ -604,7 +648,8 @@ rte_graph_node_get(rte_graph_t gid, uint32_t nid) + rte_graph_off_t off; + rte_node_t count; + +- GRAPH_ID_CHECK(gid); ++ if (graph_from_id(gid) == NULL) ++ goto fail; + STAILQ_FOREACH(graph, &graph_list, next) + if (graph->id == gid) { + rte_graph_foreach_node(count, off, graph->graph, +@@ -729,7 +774,8 @@ graph_scan_dump(FILE *f, rte_graph_t id, bool all) + struct graph *graph; + + RTE_VERIFY(f); +- GRAPH_ID_CHECK(id); ++ if (graph_from_id(id) == NULL) ++ goto fail; + + STAILQ_FOREACH(graph, &graph_list, next) { + if (all == true) { +@@ -758,7 +804,13 @@ rte_graph_list_dump(FILE *f) + rte_graph_t + rte_graph_max_count(void) + { +- return graph_id; ++ struct graph *graph; ++ rte_graph_t count = 0; ++ ++ STAILQ_FOREACH(graph, &graph_list, next) ++ count++; ++ ++ return count; + } + + RTE_LOG_REGISTER_DEFAULT(rte_graph_logtype, INFO); +diff --git a/dpdk/lib/graph/graph_stats.c b/dpdk/lib/graph/graph_stats.c +index cc32245c05..e99e8cf68a 100644 +--- a/dpdk/lib/graph/graph_stats.c ++++ b/dpdk/lib/graph/graph_stats.c +@@ -34,6 +34,7 @@ struct rte_graph_cluster_stats { + uint32_t cluster_node_size; /* Size of struct cluster_node */ + rte_node_t max_nodes; + int socket_id; ++ bool dispatch; + void *cookie; + size_t sz; + +@@ -74,17 +75,16 @@ print_banner_dispatch(FILE *f) + } + + static inline void +-print_banner(FILE *f) ++print_banner(FILE *f, bool dispatch) + { +- if (rte_graph_worker_model_get(STAILQ_FIRST(graph_list_head_get())->graph) == +- RTE_GRAPH_MODEL_MCORE_DISPATCH) ++ if (dispatch) + print_banner_dispatch(f); + else + print_banner_default(f); + } + + static inline void +-print_node(FILE *f, const struct rte_graph_cluster_node_stats *stat) ++print_node(FILE *f, const struct rte_graph_cluster_node_stats *stat, bool dispatch) + { + double objs_per_call, objs_per_sec, cycles_per_call, ts_per_hz; + const uint64_t prev_calls = stat->prev_calls; +@@ -104,8 +104,7 @@ print_node(FILE *f, const struct rte_graph_cluster_node_stats *stat) + objs_per_sec = ts_per_hz ? (objs - prev_objs) / ts_per_hz : 0; + objs_per_sec /= 1000000; + +- if (rte_graph_worker_model_get(STAILQ_FIRST(graph_list_head_get())->graph) == +- RTE_GRAPH_MODEL_MCORE_DISPATCH) { ++ if (dispatch) { + fprintf(f, + "|%-31s|%-15" PRIu64 "|%-15" PRIu64 "|%-15" PRIu64 + "|%-15" PRIu64 "|%-15" PRIu64 +@@ -123,20 +122,17 @@ print_node(FILE *f, const struct rte_graph_cluster_node_stats *stat) + } + + static int +-graph_cluster_stats_cb(bool is_first, bool is_last, void *cookie, ++graph_cluster_stats_cb(bool dispatch, bool is_first, bool is_last, void *cookie, + const struct rte_graph_cluster_node_stats *stat) + { + FILE *f = cookie; +- int model; +- +- model = rte_graph_worker_model_get(STAILQ_FIRST(graph_list_head_get())->graph); + + if (unlikely(is_first)) +- print_banner(f); ++ print_banner(f, dispatch); + if (stat->objs) +- print_node(f, stat); ++ print_node(f, stat, dispatch); + if (unlikely(is_last)) { +- if (model == RTE_GRAPH_MODEL_MCORE_DISPATCH) ++ if (dispatch) + boarder_model_dispatch(); + else + boarder(); +@@ -145,6 +141,20 @@ graph_cluster_stats_cb(bool is_first, bool is_last, void *cookie, + return 0; + }; + ++static int ++graph_cluster_stats_cb_rtc(bool is_first, bool is_last, void *cookie, ++ const struct rte_graph_cluster_node_stats *stat) ++{ ++ return graph_cluster_stats_cb(false, is_first, is_last, cookie, stat); ++}; ++ ++static int ++graph_cluster_stats_cb_dispatch(bool is_first, bool is_last, void *cookie, ++ const struct rte_graph_cluster_node_stats *stat) ++{ ++ return graph_cluster_stats_cb(true, is_first, is_last, cookie, stat); ++}; ++ + static struct rte_graph_cluster_stats * + stats_mem_init(struct cluster *cluster, + const struct rte_graph_cluster_stats_param *prm) +@@ -157,8 +167,13 @@ stats_mem_init(struct cluster *cluster, + + /* Fix up callback */ + fn = prm->fn; +- if (fn == NULL) +- fn = graph_cluster_stats_cb; ++ if (fn == NULL) { ++ const struct rte_graph *graph = cluster->graphs[0]->graph; ++ if (graph->model == RTE_GRAPH_MODEL_MCORE_DISPATCH) ++ fn = graph_cluster_stats_cb_dispatch; ++ else ++ fn = graph_cluster_stats_cb_rtc; ++ } + + cluster_node_size = sizeof(struct cluster_node); + /* For a given cluster, max nodes will be the max number of graphs */ +@@ -350,6 +365,8 @@ rte_graph_cluster_stats_create(const struct rte_graph_cluster_stats_param *prm) + if (stats_mem_populate(&stats, graph_fp, graph_node)) + goto realloc_fail; + } ++ if (graph->graph->model == RTE_GRAPH_MODEL_MCORE_DISPATCH) ++ stats->dispatch = true; + } + + /* Finally copy to hugepage memory to avoid pressure on rte_realloc */ +@@ -375,20 +392,18 @@ rte_graph_cluster_stats_destroy(struct rte_graph_cluster_stats *stat) + } + + static inline void +-cluster_node_arregate_stats(struct cluster_node *cluster) ++cluster_node_arregate_stats(struct cluster_node *cluster, bool dispatch) + { + uint64_t calls = 0, cycles = 0, objs = 0, realloc_count = 0; + struct rte_graph_cluster_node_stats *stat = &cluster->stat; + uint64_t sched_objs = 0, sched_fail = 0; + struct rte_node *node; + rte_node_t count; +- int model; + +- model = rte_graph_worker_model_get(STAILQ_FIRST(graph_list_head_get())->graph); + for (count = 0; count < cluster->nb_nodes; count++) { + node = cluster->nodes[count]; + +- if (model == RTE_GRAPH_MODEL_MCORE_DISPATCH) { ++ if (dispatch) { + sched_objs += node->dispatch.total_sched_objs; + sched_fail += node->dispatch.total_sched_fail; + } +@@ -403,7 +418,7 @@ cluster_node_arregate_stats(struct cluster_node *cluster) + stat->objs = objs; + stat->cycles = cycles; + +- if (model == RTE_GRAPH_MODEL_MCORE_DISPATCH) { ++ if (dispatch) { + stat->dispatch.sched_objs = sched_objs; + stat->dispatch.sched_fail = sched_fail; + } +@@ -433,7 +448,7 @@ rte_graph_cluster_stats_get(struct rte_graph_cluster_stats *stat, bool skip_cb) + cluster = stat->clusters; + + for (count = 0; count < stat->max_nodes; count++) { +- cluster_node_arregate_stats(cluster); ++ cluster_node_arregate_stats(cluster, stat->dispatch); + if (!skip_cb) + rc = stat->fn(!count, (count == stat->max_nodes - 1), + stat->cookie, &cluster->stat); +diff --git a/dpdk/lib/graph/rte_graph_model_mcore_dispatch.h b/dpdk/lib/graph/rte_graph_model_mcore_dispatch.h +index 75ec388cad..732b89297f 100644 +--- a/dpdk/lib/graph/rte_graph_model_mcore_dispatch.h ++++ b/dpdk/lib/graph/rte_graph_model_mcore_dispatch.h +@@ -100,7 +100,7 @@ rte_graph_walk_mcore_dispatch(struct rte_graph *graph) + node = (struct rte_node *)RTE_PTR_ADD(graph, cir_start[(int32_t)head++]); + + /* skip the src nodes which not bind with current worker */ +- if ((int32_t)head < 0 && node->dispatch.lcore_id != graph->dispatch.lcore_id) ++ if ((int32_t)head < 1 && node->dispatch.lcore_id != graph->dispatch.lcore_id) + continue; + + /* Schedule the node until all task/objs are done */ +diff --git a/dpdk/lib/gro/gro_tcp.h b/dpdk/lib/gro/gro_tcp.h +index d926c4b8cc..2c68b5f23e 100644 +--- a/dpdk/lib/gro/gro_tcp.h ++++ b/dpdk/lib/gro/gro_tcp.h +@@ -19,6 +19,8 @@ + #define INVALID_TCP_HDRLEN(len) \ + (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN)) + ++#define VALID_GRO_TCP_FLAGS (RTE_TCP_ACK_FLAG | RTE_TCP_PSH_FLAG | RTE_TCP_FIN_FLAG) ++ + struct cmn_tcp_key { + struct rte_ether_addr eth_saddr; + struct rte_ether_addr eth_daddr; +@@ -81,11 +83,13 @@ merge_two_tcp_packets(struct gro_tcp_item *item, + struct rte_mbuf *pkt, + int cmp, + uint32_t sent_seq, ++ uint8_t tcp_flags, + uint16_t ip_id, + uint16_t l2_offset) + { + struct rte_mbuf *pkt_head, *pkt_tail, *lastseg; + uint16_t hdr_len, l2_len; ++ struct rte_tcp_hdr *tcp_hdr; + + if (cmp > 0) { + pkt_head = item->firstseg; +@@ -128,6 +132,11 @@ merge_two_tcp_packets(struct gro_tcp_item *item, + /* update MBUF metadata for the merged packet */ + pkt_head->nb_segs += pkt_tail->nb_segs; + pkt_head->pkt_len += pkt_tail->pkt_len; ++ if (tcp_flags != RTE_TCP_ACK_FLAG) { ++ tcp_hdr = rte_pktmbuf_mtod_offset(pkt, struct rte_tcp_hdr *, ++ l2_offset + pkt_head->l2_len + pkt_head->l3_len); ++ tcp_hdr->tcp_flags |= tcp_flags; ++ } + + return 1; + } +diff --git a/dpdk/lib/gro/gro_tcp4.c b/dpdk/lib/gro/gro_tcp4.c +index 6645de592b..c8b8d7990c 100644 +--- a/dpdk/lib/gro/gro_tcp4.c ++++ b/dpdk/lib/gro/gro_tcp4.c +@@ -126,6 +126,7 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt, + uint32_t item_idx; + uint32_t i, max_flow_num, remaining_flow_num; + uint8_t find; ++ uint32_t item_start_idx; + + /* + * Don't process the packet whose TCP header length is greater +@@ -139,11 +140,8 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt, + tcp_hdr = (struct rte_tcp_hdr *)((char *)ipv4_hdr + pkt->l3_len); + hdr_len = pkt->l2_len + pkt->l3_len + pkt->l4_len; + +- /* +- * Don't process the packet which has FIN, SYN, RST, PSH, URG, ECE +- * or CWR set. +- */ +- if (tcp_hdr->tcp_flags != RTE_TCP_ACK_FLAG) ++ /* Return early if the TCP flags are not handled in GRO layer */ ++ if (tcp_hdr->tcp_flags & ~VALID_GRO_TCP_FLAGS) + return -1; + + /* trim the tail padding bytes */ +@@ -183,13 +181,35 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt, + if (tbl->flows[i].start_index != INVALID_ARRAY_INDEX) { + if (is_same_tcp4_flow(tbl->flows[i].key, key)) { + find = 1; ++ item_start_idx = tbl->flows[i].start_index; + break; + } + remaining_flow_num--; + } + } + +- if (find == 0) { ++ if (find == 1) { ++ /* ++ * Any packet with additional flags like PSH,FIN should be processed ++ * and flushed immediately. ++ * Hence marking the start time to 0, so that the packets will be flushed ++ * immediately in timer mode. ++ */ ++ if (tcp_hdr->tcp_flags & (RTE_TCP_ACK_FLAG | RTE_TCP_PSH_FLAG | RTE_TCP_FIN_FLAG)) { ++ if (tcp_hdr->tcp_flags != RTE_TCP_ACK_FLAG) ++ tbl->items[item_start_idx].start_time = 0; ++ return process_tcp_item(pkt, tcp_hdr, tcp_dl, tbl->items, ++ tbl->flows[i].start_index, &tbl->item_num, ++ tbl->max_item_num, ip_id, is_atomic, start_time); ++ } else { ++ return -1; ++ } ++ } ++ /* ++ * Add new flow to the table only if contains ACK flag with data. ++ * Do not add any packets with additional tcp flags to the GRO table ++ */ ++ if (tcp_hdr->tcp_flags == RTE_TCP_ACK_FLAG) { + sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq); + item_idx = insert_new_tcp_item(pkt, tbl->items, &tbl->item_num, + tbl->max_item_num, start_time, +@@ -209,9 +229,7 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt, + return 0; + } + +- return process_tcp_item(pkt, tcp_hdr, tcp_dl, tbl->items, tbl->flows[i].start_index, +- &tbl->item_num, tbl->max_item_num, +- ip_id, is_atomic, start_time); ++ return -1; + } + + /* +diff --git a/dpdk/lib/gro/gro_tcp_internal.h b/dpdk/lib/gro/gro_tcp_internal.h +index cc84abeaeb..e4855da1ad 100644 +--- a/dpdk/lib/gro/gro_tcp_internal.h ++++ b/dpdk/lib/gro/gro_tcp_internal.h +@@ -101,7 +101,7 @@ process_tcp_item(struct rte_mbuf *pkt, + is_atomic); + if (cmp) { + if (merge_two_tcp_packets(&items[cur_idx], +- pkt, cmp, sent_seq, ip_id, 0)) ++ pkt, cmp, sent_seq, tcp_hdr->tcp_flags, ip_id, 0)) + return 1; + /* + * Fail to merge the two packets, as the packet +diff --git a/dpdk/lib/gro/gro_vxlan_tcp4.c b/dpdk/lib/gro/gro_vxlan_tcp4.c +index 6ab7001922..8dd62a949c 100644 +--- a/dpdk/lib/gro/gro_vxlan_tcp4.c ++++ b/dpdk/lib/gro/gro_vxlan_tcp4.c +@@ -239,10 +239,11 @@ merge_two_vxlan_tcp4_packets(struct gro_vxlan_tcp4_item *item, + struct rte_mbuf *pkt, + int cmp, + uint32_t sent_seq, ++ uint8_t tcp_flags, + uint16_t outer_ip_id, + uint16_t ip_id) + { +- if (merge_two_tcp_packets(&item->inner_item, pkt, cmp, sent_seq, ++ if (merge_two_tcp_packets(&item->inner_item, pkt, cmp, sent_seq, tcp_flags, + ip_id, pkt->outer_l2_len + + pkt->outer_l3_len)) { + /* Update the outer IPv4 ID to the large value. */ +@@ -413,7 +414,7 @@ gro_vxlan_tcp4_reassemble(struct rte_mbuf *pkt, + tcp_dl, outer_is_atomic, is_atomic); + if (cmp) { + if (merge_two_vxlan_tcp4_packets(&(tbl->items[cur_idx]), +- pkt, cmp, sent_seq, ++ pkt, cmp, sent_seq, tcp_hdr->tcp_flags, + outer_ip_id, ip_id)) + return 1; + /* +diff --git a/dpdk/lib/hash/rte_cuckoo_hash.c b/dpdk/lib/hash/rte_cuckoo_hash.c +index ccdc3b9894..258b6b7f2b 100644 +--- a/dpdk/lib/hash/rte_cuckoo_hash.c ++++ b/dpdk/lib/hash/rte_cuckoo_hash.c +@@ -166,6 +166,7 @@ rte_hash_create(const struct rte_hash_parameters *params) + /* Check for valid parameters */ + if ((params->entries > RTE_HASH_ENTRIES_MAX) || + (params->entries < RTE_HASH_BUCKET_ENTRIES) || ++ (params->name == NULL) || + (params->key_len == 0)) { + rte_errno = EINVAL; + RTE_LOG(ERR, HASH, "rte_hash_create has invalid parameters\n"); +@@ -1550,6 +1551,7 @@ rte_hash_rcu_qsbr_add(struct rte_hash *h, struct rte_hash_rcu_config *cfg) + if (params.size == 0) + params.size = total_entries; + params.trigger_reclaim_limit = cfg->trigger_reclaim_limit; ++ params.max_reclaim_size = cfg->max_reclaim_size; + if (params.max_reclaim_size == 0) + params.max_reclaim_size = RTE_HASH_RCU_DQ_RECLAIM_MAX; + params.esize = sizeof(struct __rte_hash_rcu_dq_entry); +diff --git a/dpdk/lib/hash/rte_cuckoo_hash.h b/dpdk/lib/hash/rte_cuckoo_hash.h +index f7afc4dd79..8ea793c66e 100644 +--- a/dpdk/lib/hash/rte_cuckoo_hash.h ++++ b/dpdk/lib/hash/rte_cuckoo_hash.h +@@ -29,17 +29,6 @@ + #define RETURN_IF_TRUE(cond, retval) + #endif + +-#if defined(RTE_LIBRTE_HASH_DEBUG) +-#define ERR_IF_TRUE(cond, fmt, args...) do { \ +- if (cond) { \ +- RTE_LOG(ERR, HASH, fmt, ##args); \ +- return; \ +- } \ +-} while (0) +-#else +-#define ERR_IF_TRUE(cond, fmt, args...) +-#endif +- + #include + #include + +diff --git a/dpdk/lib/hash/rte_hash.h b/dpdk/lib/hash/rte_hash.h +index 7ecc021111..ba96521529 100644 +--- a/dpdk/lib/hash/rte_hash.h ++++ b/dpdk/lib/hash/rte_hash.h +@@ -286,7 +286,7 @@ rte_hash_add_key_with_hash_data(const struct rte_hash *h, const void *key, + * @return + * - -EINVAL if the parameters are invalid. + * - -ENOSPC if there is no space in the hash for this key. +- * - A positive value that can be used by the caller as an offset into an ++ * - A non-negative value that can be used by the caller as an offset into an + * array of user data. This value is unique for this key. This + * unique key id may be larger than the user specified entry count + * when RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD flag is set. +@@ -310,7 +310,7 @@ rte_hash_add_key(const struct rte_hash *h, const void *key); + * @return + * - -EINVAL if the parameters are invalid. + * - -ENOSPC if there is no space in the hash for this key. +- * - A positive value that can be used by the caller as an offset into an ++ * - A non-negative value that can be used by the caller as an offset into an + * array of user data. This value is unique for this key. This + * unique key ID may be larger than the user specified entry count + * when RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD flag is set. +@@ -341,7 +341,7 @@ rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key, hash_sig_t + * @return + * - -EINVAL if the parameters are invalid. + * - -ENOENT if the key is not found. +- * - A positive value that can be used by the caller as an offset into an ++ * - A non-negative value that can be used by the caller as an offset into an + * array of user data. This value is unique for this key, and is the same + * value that was returned when the key was added. + */ +@@ -373,7 +373,7 @@ rte_hash_del_key(const struct rte_hash *h, const void *key); + * @return + * - -EINVAL if the parameters are invalid. + * - -ENOENT if the key is not found. +- * - A positive value that can be used by the caller as an offset into an ++ * - A non-negative value that can be used by the caller as an offset into an + * array of user data. This value is unique for this key, and is the same + * value that was returned when the key was added. + */ +@@ -440,7 +440,7 @@ rte_hash_free_key_with_position(const struct rte_hash *h, + * @param data + * Output with pointer to data returned from the hash table. + * @return +- * - A positive value that can be used by the caller as an offset into an ++ * - A non-negative value that can be used by the caller as an offset into an + * array of user data. This value is unique for this key, and is the same + * value that was returned when the key was added. + * - -EINVAL if the parameters are invalid. +@@ -465,7 +465,7 @@ rte_hash_lookup_data(const struct rte_hash *h, const void *key, void **data); + * @param data + * Output with pointer to data returned from the hash table. + * @return +- * - A positive value that can be used by the caller as an offset into an ++ * - A non-negative value that can be used by the caller as an offset into an + * array of user data. This value is unique for this key, and is the same + * value that was returned when the key was added. + * - -EINVAL if the parameters are invalid. +@@ -488,7 +488,7 @@ rte_hash_lookup_with_hash_data(const struct rte_hash *h, const void *key, + * @return + * - -EINVAL if the parameters are invalid. + * - -ENOENT if the key is not found. +- * - A positive value that can be used by the caller as an offset into an ++ * - A non-negative value that can be used by the caller as an offset into an + * array of user data. This value is unique for this key, and is the same + * value that was returned when the key was added. + */ +@@ -510,7 +510,7 @@ rte_hash_lookup(const struct rte_hash *h, const void *key); + * @return + * - -EINVAL if the parameters are invalid. + * - -ENOENT if the key is not found. +- * - A positive value that can be used by the caller as an offset into an ++ * - A non-negative value that can be used by the caller as an offset into an + * array of user data. This value is unique for this key, and is the same + * value that was returned when the key was added. + */ +diff --git a/dpdk/lib/latencystats/rte_latencystats.c b/dpdk/lib/latencystats/rte_latencystats.c +index 8985a377db..e47eac2cf8 100644 +--- a/dpdk/lib/latencystats/rte_latencystats.c ++++ b/dpdk/lib/latencystats/rte_latencystats.c +@@ -164,7 +164,7 @@ calc_latency(uint16_t pid __rte_unused, + * a constant smoothing factor between 0 and 1. The value + * is used below for measuring average latency. + */ +- const float alpha = 0.2; ++ const float alpha = 0.2f; + + now = rte_rdtsc(); + for (i = 0; i < nb_pkts; i++) { +diff --git a/dpdk/lib/lpm/rte_lpm6.c b/dpdk/lib/lpm/rte_lpm6.c +index 8d21aeddb8..79c75d8dfc 100644 +--- a/dpdk/lib/lpm/rte_lpm6.c ++++ b/dpdk/lib/lpm/rte_lpm6.c +@@ -279,7 +279,7 @@ rte_lpm6_create(const char *name, int socket_id, + + rules_tbl = rte_hash_create(&rule_hash_tbl_params); + if (rules_tbl == NULL) { +- RTE_LOG(ERR, LPM, "LPM rules hash table allocation failed: %s (%d)", ++ RTE_LOG(ERR, LPM, "LPM rules hash table allocation failed: %s (%d)\n", + rte_strerror(rte_errno), rte_errno); + goto fail_wo_unlock; + } +@@ -289,7 +289,7 @@ rte_lpm6_create(const char *name, int socket_id, + sizeof(uint32_t) * config->number_tbl8s, + RTE_CACHE_LINE_SIZE); + if (tbl8_pool == NULL) { +- RTE_LOG(ERR, LPM, "LPM tbl8 pool allocation failed: %s (%d)", ++ RTE_LOG(ERR, LPM, "LPM tbl8 pool allocation failed: %s (%d)\n", + rte_strerror(rte_errno), rte_errno); + rte_errno = ENOMEM; + goto fail_wo_unlock; +@@ -300,7 +300,7 @@ rte_lpm6_create(const char *name, int socket_id, + sizeof(struct rte_lpm_tbl8_hdr) * config->number_tbl8s, + RTE_CACHE_LINE_SIZE); + if (tbl8_hdrs == NULL) { +- RTE_LOG(ERR, LPM, "LPM tbl8 headers allocation failed: %s (%d)", ++ RTE_LOG(ERR, LPM, "LPM tbl8 headers allocation failed: %s (%d)\n", + rte_strerror(rte_errno), rte_errno); + rte_errno = ENOMEM; + goto fail_wo_unlock; +diff --git a/dpdk/lib/mbuf/rte_mbuf.h b/dpdk/lib/mbuf/rte_mbuf.h +index 286b32b788..c266727a13 100644 +--- a/dpdk/lib/mbuf/rte_mbuf.h ++++ b/dpdk/lib/mbuf/rte_mbuf.h +@@ -1119,6 +1119,9 @@ rte_pktmbuf_attach_extbuf(struct rte_mbuf *m, void *buf_addr, + static inline void + rte_mbuf_dynfield_copy(struct rte_mbuf *mdst, const struct rte_mbuf *msrc) + { ++#if !RTE_IOVA_IN_MBUF ++ mdst->dynfield2 = msrc->dynfield2; ++#endif + memcpy(&mdst->dynfield1, msrc->dynfield1, sizeof(mdst->dynfield1)); + } + +diff --git a/dpdk/lib/mempool/rte_mempool_ops.c b/dpdk/lib/mempool/rte_mempool_ops.c +index ae1d288f27..e871de9ec9 100644 +--- a/dpdk/lib/mempool/rte_mempool_ops.c ++++ b/dpdk/lib/mempool/rte_mempool_ops.c +@@ -46,7 +46,7 @@ rte_mempool_register_ops(const struct rte_mempool_ops *h) + + if (strlen(h->name) >= sizeof(ops->name) - 1) { + rte_spinlock_unlock(&rte_mempool_ops_table.sl); +- RTE_LOG(DEBUG, EAL, "%s(): mempool_ops <%s>: name too long\n", ++ RTE_LOG(DEBUG, MEMPOOL, "%s(): mempool_ops <%s>: name too long\n", + __func__, h->name); + rte_errno = EEXIST; + return -EEXIST; +diff --git a/dpdk/lib/meson.build b/dpdk/lib/meson.build +index 6c143ce5a6..55d483cd26 100644 +--- a/dpdk/lib/meson.build ++++ b/dpdk/lib/meson.build +@@ -228,7 +228,7 @@ foreach l:libraries + + if not build + dpdk_libs_disabled += name +- set_variable(name.underscorify() + '_disable_reason', reason) ++ set_variable('lib_' + name.underscorify() + '_disable_reason', reason) + continue + endif + +diff --git a/dpdk/lib/metrics/rte_metrics_telemetry.c b/dpdk/lib/metrics/rte_metrics_telemetry.c +index 5be21b2e86..1d133e1f8c 100644 +--- a/dpdk/lib/metrics/rte_metrics_telemetry.c ++++ b/dpdk/lib/metrics/rte_metrics_telemetry.c +@@ -363,7 +363,7 @@ rte_metrics_tel_stat_names_to_ids(const char * const *stat_names, + } + } + if (j == num_metrics) { +- METRICS_LOG_WARN("Invalid stat name %s\n", ++ METRICS_LOG_WARN("Invalid stat name %s", + stat_names[i]); + free(names); + return -EINVAL; +diff --git a/dpdk/lib/mldev/rte_mldev.c b/dpdk/lib/mldev/rte_mldev.c +index cc5f2e0cc6..196b1850e6 100644 +--- a/dpdk/lib/mldev/rte_mldev.c ++++ b/dpdk/lib/mldev/rte_mldev.c +@@ -159,7 +159,7 @@ int + rte_ml_dev_init(size_t dev_max) + { + if (dev_max == 0 || dev_max > INT16_MAX) { +- RTE_MLDEV_LOG(ERR, "Invalid dev_max = %zu (> %d)\n", dev_max, INT16_MAX); ++ RTE_MLDEV_LOG(ERR, "Invalid dev_max = %zu (> %d)", dev_max, INT16_MAX); + rte_errno = EINVAL; + return -rte_errno; + } +@@ -217,7 +217,7 @@ rte_ml_dev_socket_id(int16_t dev_id) + struct rte_ml_dev *dev; + + if (!rte_ml_dev_is_valid_dev(dev_id)) { +- RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d", dev_id); + return -EINVAL; + } + +@@ -232,7 +232,7 @@ rte_ml_dev_info_get(int16_t dev_id, struct rte_ml_dev_info *dev_info) + struct rte_ml_dev *dev; + + if (!rte_ml_dev_is_valid_dev(dev_id)) { +- RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d", dev_id); + return -EINVAL; + } + +@@ -241,7 +241,7 @@ rte_ml_dev_info_get(int16_t dev_id, struct rte_ml_dev_info *dev_info) + return -ENOTSUP; + + if (dev_info == NULL) { +- RTE_MLDEV_LOG(ERR, "Dev %d, dev_info cannot be NULL\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Dev %d, dev_info cannot be NULL", dev_id); + return -EINVAL; + } + memset(dev_info, 0, sizeof(struct rte_ml_dev_info)); +@@ -257,7 +257,7 @@ rte_ml_dev_configure(int16_t dev_id, const struct rte_ml_dev_config *config) + int ret; + + if (!rte_ml_dev_is_valid_dev(dev_id)) { +- RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d", dev_id); + return -EINVAL; + } + +@@ -271,7 +271,7 @@ rte_ml_dev_configure(int16_t dev_id, const struct rte_ml_dev_config *config) + } + + if (config == NULL) { +- RTE_MLDEV_LOG(ERR, "Dev %d, config cannot be NULL\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Dev %d, config cannot be NULL", dev_id); + return -EINVAL; + } + +@@ -280,7 +280,7 @@ rte_ml_dev_configure(int16_t dev_id, const struct rte_ml_dev_config *config) + return ret; + + if (config->nb_queue_pairs > dev_info.max_queue_pairs) { +- RTE_MLDEV_LOG(ERR, "Device %d num of queues %u > %u\n", dev_id, ++ RTE_MLDEV_LOG(ERR, "Device %d num of queues %u > %u", dev_id, + config->nb_queue_pairs, dev_info.max_queue_pairs); + return -EINVAL; + } +@@ -294,7 +294,7 @@ rte_ml_dev_close(int16_t dev_id) + struct rte_ml_dev *dev; + + if (!rte_ml_dev_is_valid_dev(dev_id)) { +- RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d", dev_id); + return -EINVAL; + } + +@@ -318,7 +318,7 @@ rte_ml_dev_start(int16_t dev_id) + int ret; + + if (!rte_ml_dev_is_valid_dev(dev_id)) { +- RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d", dev_id); + return -EINVAL; + } + +@@ -345,7 +345,7 @@ rte_ml_dev_stop(int16_t dev_id) + int ret; + + if (!rte_ml_dev_is_valid_dev(dev_id)) { +- RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d", dev_id); + return -EINVAL; + } + +@@ -372,7 +372,7 @@ rte_ml_dev_queue_pair_setup(int16_t dev_id, uint16_t queue_pair_id, + struct rte_ml_dev *dev; + + if (!rte_ml_dev_is_valid_dev(dev_id)) { +- RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d", dev_id); + return -EINVAL; + } + +@@ -386,7 +386,7 @@ rte_ml_dev_queue_pair_setup(int16_t dev_id, uint16_t queue_pair_id, + } + + if (qp_conf == NULL) { +- RTE_MLDEV_LOG(ERR, "Dev %d, qp_conf cannot be NULL\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Dev %d, qp_conf cannot be NULL", dev_id); + return -EINVAL; + } + +@@ -404,7 +404,7 @@ rte_ml_dev_stats_get(int16_t dev_id, struct rte_ml_dev_stats *stats) + struct rte_ml_dev *dev; + + if (!rte_ml_dev_is_valid_dev(dev_id)) { +- RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d", dev_id); + return -EINVAL; + } + +@@ -413,7 +413,7 @@ rte_ml_dev_stats_get(int16_t dev_id, struct rte_ml_dev_stats *stats) + return -ENOTSUP; + + if (stats == NULL) { +- RTE_MLDEV_LOG(ERR, "Dev %d, stats cannot be NULL\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Dev %d, stats cannot be NULL", dev_id); + return -EINVAL; + } + memset(stats, 0, sizeof(struct rte_ml_dev_stats)); +@@ -427,7 +427,7 @@ rte_ml_dev_stats_reset(int16_t dev_id) + struct rte_ml_dev *dev; + + if (!rte_ml_dev_is_valid_dev(dev_id)) { +- RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d", dev_id); + return; + } + +@@ -445,7 +445,7 @@ rte_ml_dev_xstats_names_get(int16_t dev_id, enum rte_ml_dev_xstats_mode mode, in + struct rte_ml_dev *dev; + + if (!rte_ml_dev_is_valid_dev(dev_id)) { +- RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d", dev_id); + return -EINVAL; + } + +@@ -462,7 +462,7 @@ rte_ml_dev_xstats_by_name_get(int16_t dev_id, const char *name, uint16_t *stat_i + struct rte_ml_dev *dev; + + if (!rte_ml_dev_is_valid_dev(dev_id)) { +- RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d", dev_id); + return -EINVAL; + } + +@@ -471,12 +471,12 @@ rte_ml_dev_xstats_by_name_get(int16_t dev_id, const char *name, uint16_t *stat_i + return -ENOTSUP; + + if (name == NULL) { +- RTE_MLDEV_LOG(ERR, "Dev %d, name cannot be NULL\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Dev %d, name cannot be NULL", dev_id); + return -EINVAL; + } + + if (value == NULL) { +- RTE_MLDEV_LOG(ERR, "Dev %d, value cannot be NULL\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Dev %d, value cannot be NULL", dev_id); + return -EINVAL; + } + +@@ -490,7 +490,7 @@ rte_ml_dev_xstats_get(int16_t dev_id, enum rte_ml_dev_xstats_mode mode, int32_t + struct rte_ml_dev *dev; + + if (!rte_ml_dev_is_valid_dev(dev_id)) { +- RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d", dev_id); + return -EINVAL; + } + +@@ -499,12 +499,12 @@ rte_ml_dev_xstats_get(int16_t dev_id, enum rte_ml_dev_xstats_mode mode, int32_t + return -ENOTSUP; + + if (stat_ids == NULL) { +- RTE_MLDEV_LOG(ERR, "Dev %d, stat_ids cannot be NULL\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Dev %d, stat_ids cannot be NULL", dev_id); + return -EINVAL; + } + + if (values == NULL) { +- RTE_MLDEV_LOG(ERR, "Dev %d, values cannot be NULL\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Dev %d, values cannot be NULL", dev_id); + return -EINVAL; + } + +@@ -518,7 +518,7 @@ rte_ml_dev_xstats_reset(int16_t dev_id, enum rte_ml_dev_xstats_mode mode, int32_ + struct rte_ml_dev *dev; + + if (!rte_ml_dev_is_valid_dev(dev_id)) { +- RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d", dev_id); + return -EINVAL; + } + +@@ -535,7 +535,7 @@ rte_ml_dev_dump(int16_t dev_id, FILE *fd) + struct rte_ml_dev *dev; + + if (!rte_ml_dev_is_valid_dev(dev_id)) { +- RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d", dev_id); + return -EINVAL; + } + +@@ -544,7 +544,7 @@ rte_ml_dev_dump(int16_t dev_id, FILE *fd) + return -ENOTSUP; + + if (fd == NULL) { +- RTE_MLDEV_LOG(ERR, "Dev %d, file descriptor cannot be NULL\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Dev %d, file descriptor cannot be NULL", dev_id); + return -EINVAL; + } + +@@ -557,7 +557,7 @@ rte_ml_dev_selftest(int16_t dev_id) + struct rte_ml_dev *dev; + + if (!rte_ml_dev_is_valid_dev(dev_id)) { +- RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d", dev_id); + return -EINVAL; + } + +@@ -574,7 +574,7 @@ rte_ml_model_load(int16_t dev_id, struct rte_ml_model_params *params, uint16_t * + struct rte_ml_dev *dev; + + if (!rte_ml_dev_is_valid_dev(dev_id)) { +- RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d", dev_id); + return -EINVAL; + } + +@@ -583,12 +583,12 @@ rte_ml_model_load(int16_t dev_id, struct rte_ml_model_params *params, uint16_t * + return -ENOTSUP; + + if (params == NULL) { +- RTE_MLDEV_LOG(ERR, "Dev %d, params cannot be NULL\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Dev %d, params cannot be NULL", dev_id); + return -EINVAL; + } + + if (model_id == NULL) { +- RTE_MLDEV_LOG(ERR, "Dev %d, model_id cannot be NULL\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Dev %d, model_id cannot be NULL", dev_id); + return -EINVAL; + } + +@@ -601,7 +601,7 @@ rte_ml_model_unload(int16_t dev_id, uint16_t model_id) + struct rte_ml_dev *dev; + + if (!rte_ml_dev_is_valid_dev(dev_id)) { +- RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d", dev_id); + return -EINVAL; + } + +@@ -618,7 +618,7 @@ rte_ml_model_start(int16_t dev_id, uint16_t model_id) + struct rte_ml_dev *dev; + + if (!rte_ml_dev_is_valid_dev(dev_id)) { +- RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d", dev_id); + return -EINVAL; + } + +@@ -635,7 +635,7 @@ rte_ml_model_stop(int16_t dev_id, uint16_t model_id) + struct rte_ml_dev *dev; + + if (!rte_ml_dev_is_valid_dev(dev_id)) { +- RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d", dev_id); + return -EINVAL; + } + +@@ -652,7 +652,7 @@ rte_ml_model_info_get(int16_t dev_id, uint16_t model_id, struct rte_ml_model_inf + struct rte_ml_dev *dev; + + if (!rte_ml_dev_is_valid_dev(dev_id)) { +- RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d", dev_id); + return -EINVAL; + } + +@@ -661,7 +661,7 @@ rte_ml_model_info_get(int16_t dev_id, uint16_t model_id, struct rte_ml_model_inf + return -ENOTSUP; + + if (model_info == NULL) { +- RTE_MLDEV_LOG(ERR, "Dev %d, model_id %u, model_info cannot be NULL\n", dev_id, ++ RTE_MLDEV_LOG(ERR, "Dev %d, model_id %u, model_info cannot be NULL", dev_id, + model_id); + return -EINVAL; + } +@@ -675,7 +675,7 @@ rte_ml_model_params_update(int16_t dev_id, uint16_t model_id, void *buffer) + struct rte_ml_dev *dev; + + if (!rte_ml_dev_is_valid_dev(dev_id)) { +- RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d", dev_id); + return -EINVAL; + } + +@@ -684,7 +684,7 @@ rte_ml_model_params_update(int16_t dev_id, uint16_t model_id, void *buffer) + return -ENOTSUP; + + if (buffer == NULL) { +- RTE_MLDEV_LOG(ERR, "Dev %d, buffer cannot be NULL\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Dev %d, buffer cannot be NULL", dev_id); + return -EINVAL; + } + +@@ -698,7 +698,7 @@ rte_ml_io_quantize(int16_t dev_id, uint16_t model_id, struct rte_ml_buff_seg **d + struct rte_ml_dev *dev; + + if (!rte_ml_dev_is_valid_dev(dev_id)) { +- RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d", dev_id); + return -EINVAL; + } + +@@ -707,12 +707,12 @@ rte_ml_io_quantize(int16_t dev_id, uint16_t model_id, struct rte_ml_buff_seg **d + return -ENOTSUP; + + if (dbuffer == NULL) { +- RTE_MLDEV_LOG(ERR, "Dev %d, dbuffer cannot be NULL\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Dev %d, dbuffer cannot be NULL", dev_id); + return -EINVAL; + } + + if (qbuffer == NULL) { +- RTE_MLDEV_LOG(ERR, "Dev %d, qbuffer cannot be NULL\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Dev %d, qbuffer cannot be NULL", dev_id); + return -EINVAL; + } + +@@ -726,7 +726,7 @@ rte_ml_io_dequantize(int16_t dev_id, uint16_t model_id, struct rte_ml_buff_seg * + struct rte_ml_dev *dev; + + if (!rte_ml_dev_is_valid_dev(dev_id)) { +- RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d", dev_id); + return -EINVAL; + } + +@@ -735,12 +735,12 @@ rte_ml_io_dequantize(int16_t dev_id, uint16_t model_id, struct rte_ml_buff_seg * + return -ENOTSUP; + + if (qbuffer == NULL) { +- RTE_MLDEV_LOG(ERR, "Dev %d, qbuffer cannot be NULL\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Dev %d, qbuffer cannot be NULL", dev_id); + return -EINVAL; + } + + if (dbuffer == NULL) { +- RTE_MLDEV_LOG(ERR, "Dev %d, dbuffer cannot be NULL\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Dev %d, dbuffer cannot be NULL", dev_id); + return -EINVAL; + } + +@@ -811,7 +811,7 @@ rte_ml_enqueue_burst(int16_t dev_id, uint16_t qp_id, struct rte_ml_op **ops, uin + + #ifdef RTE_LIBRTE_ML_DEV_DEBUG + if (!rte_ml_dev_is_valid_dev(dev_id)) { +- RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d", dev_id); + rte_errno = -EINVAL; + return 0; + } +@@ -823,13 +823,13 @@ rte_ml_enqueue_burst(int16_t dev_id, uint16_t qp_id, struct rte_ml_op **ops, uin + } + + if (ops == NULL) { +- RTE_MLDEV_LOG(ERR, "Dev %d, ops cannot be NULL\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Dev %d, ops cannot be NULL", dev_id); + rte_errno = -EINVAL; + return 0; + } + + if (qp_id >= dev->data->nb_queue_pairs) { +- RTE_MLDEV_LOG(ERR, "Invalid qp_id %u\n", qp_id); ++ RTE_MLDEV_LOG(ERR, "Invalid qp_id %u", qp_id); + rte_errno = -EINVAL; + return 0; + } +@@ -847,7 +847,7 @@ rte_ml_dequeue_burst(int16_t dev_id, uint16_t qp_id, struct rte_ml_op **ops, uin + + #ifdef RTE_LIBRTE_ML_DEV_DEBUG + if (!rte_ml_dev_is_valid_dev(dev_id)) { +- RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d", dev_id); + rte_errno = -EINVAL; + return 0; + } +@@ -859,13 +859,13 @@ rte_ml_dequeue_burst(int16_t dev_id, uint16_t qp_id, struct rte_ml_op **ops, uin + } + + if (ops == NULL) { +- RTE_MLDEV_LOG(ERR, "Dev %d, ops cannot be NULL\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Dev %d, ops cannot be NULL", dev_id); + rte_errno = -EINVAL; + return 0; + } + + if (qp_id >= dev->data->nb_queue_pairs) { +- RTE_MLDEV_LOG(ERR, "Invalid qp_id %u\n", qp_id); ++ RTE_MLDEV_LOG(ERR, "Invalid qp_id %u", qp_id); + rte_errno = -EINVAL; + return 0; + } +@@ -883,7 +883,7 @@ rte_ml_op_error_get(int16_t dev_id, struct rte_ml_op *op, struct rte_ml_op_error + + #ifdef RTE_LIBRTE_ML_DEV_DEBUG + if (!rte_ml_dev_is_valid_dev(dev_id)) { +- RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Invalid dev_id = %d", dev_id); + return -EINVAL; + } + +@@ -892,12 +892,12 @@ rte_ml_op_error_get(int16_t dev_id, struct rte_ml_op *op, struct rte_ml_op_error + return -ENOTSUP; + + if (op == NULL) { +- RTE_MLDEV_LOG(ERR, "Dev %d, op cannot be NULL\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Dev %d, op cannot be NULL", dev_id); + return -EINVAL; + } + + if (error == NULL) { +- RTE_MLDEV_LOG(ERR, "Dev %d, error cannot be NULL\n", dev_id); ++ RTE_MLDEV_LOG(ERR, "Dev %d, error cannot be NULL", dev_id); + return -EINVAL; + } + #else +diff --git a/dpdk/lib/net/rte_ether.h b/dpdk/lib/net/rte_ether.h +index ce073ea818..75285bdd12 100644 +--- a/dpdk/lib/net/rte_ether.h ++++ b/dpdk/lib/net/rte_ether.h +@@ -46,6 +46,20 @@ extern "C" { + + #define RTE_ETHER_MIN_MTU 68 /**< Minimum MTU for IPv4 packets, see RFC 791. */ + ++/* VLAN header fields */ ++#define RTE_VLAN_DEI_SHIFT 12 ++#define RTE_VLAN_PRI_SHIFT 13 ++#define RTE_VLAN_PRI_MASK 0xe000 /* Priority Code Point */ ++#define RTE_VLAN_DEI_MASK 0x1000 /* Drop Eligible Indicator */ ++#define RTE_VLAN_ID_MASK 0x0fff /* VLAN Identifier */ ++ ++#define RTE_VLAN_TCI_ID(vlan_tci) ((vlan_tci) & RTE_VLAN_ID_MASK) ++#define RTE_VLAN_TCI_PRI(vlan_tci) (((vlan_tci) & RTE_VLAN_PRI_MASK) >> RTE_VLAN_PRI_SHIFT) ++#define RTE_VLAN_TCI_DEI(vlan_tci) (((vlan_tci) & RTE_VLAN_DEI_MASK) >> RTE_VLAN_DEI_SHIFT) ++#define RTE_VLAN_TCI_MAKE(id, pri, dei) ((id) | \ ++ ((pri) << RTE_VLAN_PRI_SHIFT) | \ ++ ((dei) << RTE_VLAN_DEI_SHIFT)) ++ + /** + * Ethernet address: + * A universally administered address is uniquely assigned to a device by its +diff --git a/dpdk/lib/net/rte_ip.h b/dpdk/lib/net/rte_ip.h +index 6fa98a5a0f..0d103d4127 100644 +--- a/dpdk/lib/net/rte_ip.h ++++ b/dpdk/lib/net/rte_ip.h +@@ -419,11 +419,14 @@ __rte_ipv4_udptcp_cksum_mbuf(const struct rte_mbuf *m, + { + uint16_t raw_cksum; + uint32_t cksum; ++ uint16_t len; + +- if (l4_off > m->pkt_len) +- return 0; ++ if (unlikely(l4_off > m->pkt_len)) ++ return 0; /* invalid params, return a dummy value */ ++ ++ len = rte_be_to_cpu_16(ipv4_hdr->total_length) - (uint16_t)rte_ipv4_hdr_len(ipv4_hdr); + +- if (rte_raw_cksum_mbuf(m, l4_off, m->pkt_len - l4_off, &raw_cksum)) ++ if (rte_raw_cksum_mbuf(m, l4_off, len, &raw_cksum)) + return 0; + + cksum = raw_cksum + rte_ipv4_phdr_cksum(ipv4_hdr, 0); +@@ -663,10 +666,10 @@ __rte_ipv6_udptcp_cksum_mbuf(const struct rte_mbuf *m, + uint16_t raw_cksum; + uint32_t cksum; + +- if (l4_off > m->pkt_len) +- return 0; ++ if (unlikely(l4_off > m->pkt_len)) ++ return 0; /* invalid params, return a dummy value */ + +- if (rte_raw_cksum_mbuf(m, l4_off, m->pkt_len - l4_off, &raw_cksum)) ++ if (rte_raw_cksum_mbuf(m, l4_off, rte_be_to_cpu_16(ipv6_hdr->payload_len), &raw_cksum)) + return 0; + + cksum = raw_cksum + rte_ipv6_phdr_cksum(ipv6_hdr, 0); +diff --git a/dpdk/lib/net/rte_net.h b/dpdk/lib/net/rte_net.h +index ef3ff4c6fd..efd9d5f5ee 100644 +--- a/dpdk/lib/net/rte_net.h ++++ b/dpdk/lib/net/rte_net.h +@@ -121,7 +121,8 @@ rte_net_intel_cksum_flags_prepare(struct rte_mbuf *m, uint64_t ol_flags) + * no offloads are requested. + */ + if (!(ol_flags & (RTE_MBUF_F_TX_IP_CKSUM | RTE_MBUF_F_TX_L4_MASK | RTE_MBUF_F_TX_TCP_SEG | +- RTE_MBUF_F_TX_UDP_SEG | RTE_MBUF_F_TX_OUTER_IP_CKSUM))) ++ RTE_MBUF_F_TX_UDP_SEG | RTE_MBUF_F_TX_OUTER_IP_CKSUM | ++ RTE_MBUF_F_TX_OUTER_UDP_CKSUM))) + return 0; + + if (ol_flags & (RTE_MBUF_F_TX_OUTER_IPV4 | RTE_MBUF_F_TX_OUTER_IPV6)) { +@@ -135,6 +136,21 @@ rte_net_intel_cksum_flags_prepare(struct rte_mbuf *m, uint64_t ol_flags) + struct rte_ipv4_hdr *, m->outer_l2_len); + ipv4_hdr->hdr_checksum = 0; + } ++ if (ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM) { ++ if (ol_flags & RTE_MBUF_F_TX_OUTER_IPV4) { ++ ipv4_hdr = rte_pktmbuf_mtod_offset(m, struct rte_ipv4_hdr *, ++ m->outer_l2_len); ++ udp_hdr = (struct rte_udp_hdr *)((char *)ipv4_hdr + ++ m->outer_l3_len); ++ udp_hdr->dgram_cksum = rte_ipv4_phdr_cksum(ipv4_hdr, m->ol_flags); ++ } else { ++ ipv6_hdr = rte_pktmbuf_mtod_offset(m, struct rte_ipv6_hdr *, ++ m->outer_l2_len); ++ udp_hdr = rte_pktmbuf_mtod_offset(m, struct rte_udp_hdr *, ++ m->outer_l2_len + m->outer_l3_len); ++ udp_hdr->dgram_cksum = rte_ipv6_phdr_cksum(ipv6_hdr, m->ol_flags); ++ } ++ } + } + + /* +diff --git a/dpdk/lib/net/rte_net_crc.c b/dpdk/lib/net/rte_net_crc.c +index a685f9e7bb..900d6de7f4 100644 +--- a/dpdk/lib/net/rte_net_crc.c ++++ b/dpdk/lib/net/rte_net_crc.c +@@ -179,7 +179,7 @@ avx512_vpclmulqdq_get_handlers(void) + max_simd_bitwidth >= RTE_VECT_SIMD_512) + return handlers_avx512; + #endif +- NET_LOG(INFO, "Requirements not met, can't use AVX512\n"); ++ NET_LOG(INFO, "Requirements not met, can't use AVX512"); + return NULL; + } + +@@ -205,7 +205,7 @@ sse42_pclmulqdq_get_handlers(void) + max_simd_bitwidth >= RTE_VECT_SIMD_128) + return handlers_sse42; + #endif +- NET_LOG(INFO, "Requirements not met, can't use SSE\n"); ++ NET_LOG(INFO, "Requirements not met, can't use SSE"); + return NULL; + } + +@@ -231,7 +231,7 @@ neon_pmull_get_handlers(void) + max_simd_bitwidth >= RTE_VECT_SIMD_128) + return handlers_neon; + #endif +- NET_LOG(INFO, "Requirements not met, can't use NEON\n"); ++ NET_LOG(INFO, "Requirements not met, can't use NEON"); + return NULL; + } + +diff --git a/dpdk/lib/node/ethdev_rx.c b/dpdk/lib/node/ethdev_rx.c +index 3e8fac1df4..475eff6abe 100644 +--- a/dpdk/lib/node/ethdev_rx.c ++++ b/dpdk/lib/node/ethdev_rx.c +@@ -160,13 +160,13 @@ ethdev_ptype_setup(uint16_t port, uint16_t queue) + + if (!l3_ipv4 || !l3_ipv6) { + node_info("ethdev_rx", +- "Enabling ptype callback for required ptypes on port %u\n", ++ "Enabling ptype callback for required ptypes on port %u", + port); + + if (!rte_eth_add_rx_callback(port, queue, eth_pkt_parse_cb, + NULL)) { + node_err("ethdev_rx", +- "Failed to add rx ptype cb: port=%d, queue=%d\n", ++ "Failed to add rx ptype cb: port=%d, queue=%d", + port, queue); + return -EINVAL; + } +diff --git a/dpdk/lib/node/ip4_lookup.c b/dpdk/lib/node/ip4_lookup.c +index 0dbfde64fe..18955971f6 100644 +--- a/dpdk/lib/node/ip4_lookup.c ++++ b/dpdk/lib/node/ip4_lookup.c +@@ -143,7 +143,7 @@ rte_node_ip4_route_add(uint32_t ip, uint8_t depth, uint16_t next_hop, + ip, depth, val); + if (ret < 0) { + node_err("ip4_lookup", +- "Unable to add entry %s / %d nh (%x) to LPM table on sock %d, rc=%d\n", ++ "Unable to add entry %s / %d nh (%x) to LPM table on sock %d, rc=%d", + abuf, depth, val, socket, ret); + return ret; + } +diff --git a/dpdk/lib/node/ip6_lookup.c b/dpdk/lib/node/ip6_lookup.c +index 6f56eb5ec5..309964f60f 100644 +--- a/dpdk/lib/node/ip6_lookup.c ++++ b/dpdk/lib/node/ip6_lookup.c +@@ -283,7 +283,7 @@ rte_node_ip6_route_add(const uint8_t *ip, uint8_t depth, uint16_t next_hop, + if (ret < 0) { + node_err("ip6_lookup", + "Unable to add entry %s / %d nh (%x) to LPM " +- "table on sock %d, rc=%d\n", ++ "table on sock %d, rc=%d", + abuf, depth, val, socket, ret); + return ret; + } +diff --git a/dpdk/lib/node/kernel_rx.c b/dpdk/lib/node/kernel_rx.c +index 2dba7c8cc7..6c20cdbb1e 100644 +--- a/dpdk/lib/node/kernel_rx.c ++++ b/dpdk/lib/node/kernel_rx.c +@@ -134,7 +134,7 @@ kernel_rx_node_do(struct rte_graph *graph, struct rte_node *node, kernel_rx_node + if (len == 0 || len == 0xFFFF) { + rte_pktmbuf_free(m); + if (rx->idx <= 0) +- node_dbg("kernel_rx", "rx_mbuf array is empty\n"); ++ node_dbg("kernel_rx", "rx_mbuf array is empty"); + rx->idx--; + break; + } +@@ -207,20 +207,20 @@ kernel_rx_node_init(const struct rte_graph *graph, struct rte_node *node) + RTE_VERIFY(elem != NULL); + + if (ctx->pktmbuf_pool == NULL) { +- node_err("kernel_rx", "Invalid mbuf pool on graph %s\n", graph->name); ++ node_err("kernel_rx", "Invalid mbuf pool on graph %s", graph->name); + return -EINVAL; + } + + recv_info = rte_zmalloc_socket("kernel_rx_info", sizeof(kernel_rx_info_t), + RTE_CACHE_LINE_SIZE, graph->socket); + if (!recv_info) { +- node_err("kernel_rx", "Kernel recv_info is NULL\n"); ++ node_err("kernel_rx", "Kernel recv_info is NULL"); + return -ENOMEM; + } + + sock = socket(AF_INET, SOCK_RAW, IPPROTO_RAW); + if (sock < 0) { +- node_err("kernel_rx", "Unable to open RAW socket\n"); ++ node_err("kernel_rx", "Unable to open RAW socket"); + return sock; + } + +diff --git a/dpdk/lib/node/kernel_tx.c b/dpdk/lib/node/kernel_tx.c +index 27d1808c71..3a96741622 100644 +--- a/dpdk/lib/node/kernel_tx.c ++++ b/dpdk/lib/node/kernel_tx.c +@@ -36,7 +36,7 @@ kernel_tx_process_mbuf(struct rte_node *node, struct rte_mbuf **mbufs, uint16_t + sin.sin_addr.s_addr = ip4->dst_addr; + + if (sendto(ctx->sock, buf, len, 0, (struct sockaddr *)&sin, sizeof(sin)) < 0) +- node_err("kernel_tx", "Unable to send packets: %s\n", strerror(errno)); ++ node_err("kernel_tx", "Unable to send packets: %s", strerror(errno)); + } + } + +@@ -87,7 +87,7 @@ kernel_tx_node_init(const struct rte_graph *graph __rte_unused, struct rte_node + + ctx->sock = socket(AF_INET, SOCK_RAW, IPPROTO_RAW); + if (ctx->sock < 0) +- node_err("kernel_tx", "Unable to open RAW socket\n"); ++ node_err("kernel_tx", "Unable to open RAW socket"); + + return 0; + } +diff --git a/dpdk/lib/pcapng/rte_pcapng.c b/dpdk/lib/pcapng/rte_pcapng.c +index f74ec939a9..7254defce7 100644 +--- a/dpdk/lib/pcapng/rte_pcapng.c ++++ b/dpdk/lib/pcapng/rte_pcapng.c +@@ -128,7 +128,8 @@ pcapng_add_option(struct pcapng_option *popt, uint16_t code, + { + popt->code = code; + popt->length = len; +- memcpy(popt->data, data, len); ++ if (len > 0) ++ memcpy(popt->data, data, len); + + return (struct pcapng_option *)((uint8_t *)popt + pcapng_optlen(len)); + } +diff --git a/dpdk/lib/pipeline/rte_swx_pipeline_spec.c b/dpdk/lib/pipeline/rte_swx_pipeline_spec.c +index 2bba0d0524..17419e7b85 100644 +--- a/dpdk/lib/pipeline/rte_swx_pipeline_spec.c ++++ b/dpdk/lib/pipeline/rte_swx_pipeline_spec.c +@@ -2890,7 +2890,7 @@ pipeline_spec_parse(FILE *spec, + } + + /* Memory allocation. */ +- s = calloc(sizeof(struct pipeline_spec), 1); ++ s = calloc(1, sizeof(struct pipeline_spec)); + if (!s) { + if (err_line) + *err_line = n_lines; +@@ -4241,7 +4241,7 @@ pipeline_iospec_parse(FILE *spec, + } + + /* Memory allocation. */ +- s = calloc(sizeof(struct pipeline_iospec), 1); ++ s = calloc(1, sizeof(struct pipeline_iospec)); + if (!s) { + if (err_line) + *err_line = n_lines; +diff --git a/dpdk/lib/power/guest_channel.c b/dpdk/lib/power/guest_channel.c +index 7b2ae0b650..c964332011 100644 +--- a/dpdk/lib/power/guest_channel.c ++++ b/dpdk/lib/power/guest_channel.c +@@ -89,7 +89,7 @@ guest_channel_host_connect(const char *path, unsigned int lcore_id) + flags |= O_NONBLOCK; + if (fcntl(fd, F_SETFL, flags) < 0) { + RTE_LOG(ERR, GUEST_CHANNEL, "Failed on setting non-blocking mode for " +- "file %s", fd_path); ++ "file %s\n", fd_path); + goto error; + } + /* QEMU needs a delay after connection */ +diff --git a/dpdk/lib/power/power_intel_uncore.c b/dpdk/lib/power/power_intel_uncore.c +index 688aebc4ee..be174dce44 100644 +--- a/dpdk/lib/power/power_intel_uncore.c ++++ b/dpdk/lib/power/power_intel_uncore.c +@@ -11,7 +11,6 @@ + #include "power_intel_uncore.h" + #include "power_common.h" + +-#define MAX_UNCORE_FREQS 32 + #define MAX_NUMA_DIE 8 + #define BUS_FREQ 100000 + #define FILTER_LENGTH 18 +@@ -32,7 +31,7 @@ + struct uncore_power_info { + unsigned int die; /* Core die id */ + unsigned int pkg; /* Package id */ +- uint32_t freqs[MAX_UNCORE_FREQS]; /* Frequency array */ ++ uint32_t freqs[RTE_MAX_UNCORE_FREQS]; /* Frequency array */ + uint32_t nb_freqs; /* Number of available freqs */ + FILE *f_cur_min; /* FD of scaling_min */ + FILE *f_cur_max; /* FD of scaling_max */ +@@ -51,7 +50,7 @@ set_uncore_freq_internal(struct uncore_power_info *ui, uint32_t idx) + uint32_t target_uncore_freq, curr_max_freq; + int ret; + +- if (idx >= MAX_UNCORE_FREQS || idx >= ui->nb_freqs) { ++ if (idx >= RTE_MAX_UNCORE_FREQS || idx >= ui->nb_freqs) { + RTE_LOG(DEBUG, POWER, "Invalid uncore frequency index %u, which " + "should be less than %u\n", idx, ui->nb_freqs); + return -1; +@@ -221,7 +220,7 @@ power_get_available_uncore_freqs(struct uncore_power_info *ui) + uint32_t i, num_uncore_freqs = 0; + + num_uncore_freqs = (ui->init_max_freq - ui->init_min_freq) / BUS_FREQ + 1; +- if (num_uncore_freqs >= MAX_UNCORE_FREQS) { ++ if (num_uncore_freqs >= RTE_MAX_UNCORE_FREQS) { + RTE_LOG(ERR, POWER, "Too many available uncore frequencies: %d\n", + num_uncore_freqs); + goto out; +diff --git a/dpdk/lib/power/rte_power_pmd_mgmt.c b/dpdk/lib/power/rte_power_pmd_mgmt.c +index 38f8384085..6f18ed0adf 100644 +--- a/dpdk/lib/power/rte_power_pmd_mgmt.c ++++ b/dpdk/lib/power/rte_power_pmd_mgmt.c +@@ -686,7 +686,7 @@ int + rte_power_pmd_mgmt_set_pause_duration(unsigned int duration) + { + if (duration == 0) { +- RTE_LOG(ERR, POWER, "Pause duration must be greater than 0, value unchanged"); ++ RTE_LOG(ERR, POWER, "Pause duration must be greater than 0, value unchanged\n"); + return -EINVAL; + } + pause_duration = duration; +@@ -709,7 +709,7 @@ rte_power_pmd_mgmt_set_scaling_freq_min(unsigned int lcore, unsigned int min) + } + + if (min > scale_freq_max[lcore]) { +- RTE_LOG(ERR, POWER, "Invalid min frequency: Cannot be greater than max frequency"); ++ RTE_LOG(ERR, POWER, "Invalid min frequency: Cannot be greater than max frequency\n"); + return -EINVAL; + } + scale_freq_min[lcore] = min; +@@ -729,7 +729,7 @@ rte_power_pmd_mgmt_set_scaling_freq_max(unsigned int lcore, unsigned int max) + if (max == 0) + max = UINT32_MAX; + if (max < scale_freq_min[lcore]) { +- RTE_LOG(ERR, POWER, "Invalid max frequency: Cannot be less than min frequency"); ++ RTE_LOG(ERR, POWER, "Invalid max frequency: Cannot be less than min frequency\n"); + return -EINVAL; + } + +diff --git a/dpdk/lib/rawdev/rte_rawdev.c b/dpdk/lib/rawdev/rte_rawdev.c +index 474bdc9540..4f8897b639 100644 +--- a/dpdk/lib/rawdev/rte_rawdev.c ++++ b/dpdk/lib/rawdev/rte_rawdev.c +@@ -656,7 +656,7 @@ handle_dev_dump(const char *cmd __rte_unused, + if (!rte_rawdev_pmd_is_valid_dev(dev_id)) + return -EINVAL; + +- buf = calloc(sizeof(char), RTE_TEL_MAX_SINGLE_STRING_LEN); ++ buf = calloc(RTE_TEL_MAX_SINGLE_STRING_LEN, sizeof(char)); + if (buf == NULL) + return -ENOMEM; + +diff --git a/dpdk/lib/rcu/rte_rcu_qsbr.c b/dpdk/lib/rcu/rte_rcu_qsbr.c +index a9f3d6cc98..41a44be4b9 100644 +--- a/dpdk/lib/rcu/rte_rcu_qsbr.c ++++ b/dpdk/lib/rcu/rte_rcu_qsbr.c +@@ -92,7 +92,7 @@ rte_rcu_qsbr_thread_register(struct rte_rcu_qsbr *v, unsigned int thread_id) + return 1; + } + +- __RTE_RCU_IS_LOCK_CNT_ZERO(v, thread_id, ERR, "Lock counter %u\n", ++ __RTE_RCU_IS_LOCK_CNT_ZERO(v, thread_id, ERR, "Lock counter %u", + v->qsbr_cnt[thread_id].lock_cnt); + + id = thread_id & __RTE_QSBR_THRID_MASK; +@@ -144,7 +144,7 @@ rte_rcu_qsbr_thread_unregister(struct rte_rcu_qsbr *v, unsigned int thread_id) + return 1; + } + +- __RTE_RCU_IS_LOCK_CNT_ZERO(v, thread_id, ERR, "Lock counter %u\n", ++ __RTE_RCU_IS_LOCK_CNT_ZERO(v, thread_id, ERR, "Lock counter %u", + v->qsbr_cnt[thread_id].lock_cnt); + + id = thread_id & __RTE_QSBR_THRID_MASK; +diff --git a/dpdk/lib/rcu/rte_rcu_qsbr.h b/dpdk/lib/rcu/rte_rcu_qsbr.h +index 5979fb0efb..8bda00e911 100644 +--- a/dpdk/lib/rcu/rte_rcu_qsbr.h ++++ b/dpdk/lib/rcu/rte_rcu_qsbr.h +@@ -299,7 +299,7 @@ rte_rcu_qsbr_thread_online(struct rte_rcu_qsbr *v, unsigned int thread_id) + + RTE_ASSERT(v != NULL && thread_id < v->max_threads); + +- __RTE_RCU_IS_LOCK_CNT_ZERO(v, thread_id, ERR, "Lock counter %u\n", ++ __RTE_RCU_IS_LOCK_CNT_ZERO(v, thread_id, ERR, "Lock counter %u", + v->qsbr_cnt[thread_id].lock_cnt); + + /* Copy the current value of token. +@@ -350,7 +350,7 @@ rte_rcu_qsbr_thread_offline(struct rte_rcu_qsbr *v, unsigned int thread_id) + { + RTE_ASSERT(v != NULL && thread_id < v->max_threads); + +- __RTE_RCU_IS_LOCK_CNT_ZERO(v, thread_id, ERR, "Lock counter %u\n", ++ __RTE_RCU_IS_LOCK_CNT_ZERO(v, thread_id, ERR, "Lock counter %u", + v->qsbr_cnt[thread_id].lock_cnt); + + /* The reader can go offline only after the load of the +@@ -427,7 +427,7 @@ rte_rcu_qsbr_unlock(__rte_unused struct rte_rcu_qsbr *v, + 1, rte_memory_order_release); + + __RTE_RCU_IS_LOCK_CNT_ZERO(v, thread_id, WARNING, +- "Lock counter %u. Nested locks?\n", ++ "Lock counter %u. Nested locks?", + v->qsbr_cnt[thread_id].lock_cnt); + #endif + } +@@ -481,7 +481,7 @@ rte_rcu_qsbr_quiescent(struct rte_rcu_qsbr *v, unsigned int thread_id) + + RTE_ASSERT(v != NULL && thread_id < v->max_threads); + +- __RTE_RCU_IS_LOCK_CNT_ZERO(v, thread_id, ERR, "Lock counter %u\n", ++ __RTE_RCU_IS_LOCK_CNT_ZERO(v, thread_id, ERR, "Lock counter %u", + v->qsbr_cnt[thread_id].lock_cnt); + + /* Acquire the changes to the shared data structure released +@@ -664,16 +664,20 @@ __rte_rcu_qsbr_check_all(struct rte_rcu_qsbr *v, uint64_t t, bool wait) + static __rte_always_inline int + rte_rcu_qsbr_check(struct rte_rcu_qsbr *v, uint64_t t, bool wait) + { ++ uint64_t acked_token; ++ + RTE_ASSERT(v != NULL); + + /* Check if all the readers have already acknowledged this token */ +- if (likely(t <= v->acked_token)) { ++ acked_token = rte_atomic_load_explicit(&v->acked_token, ++ rte_memory_order_relaxed); ++ if (likely(t <= acked_token)) { + __RTE_RCU_DP_LOG(DEBUG, + "%s: check: token = %" PRIu64 ", wait = %d", + __func__, t, wait); + __RTE_RCU_DP_LOG(DEBUG, + "%s: status: least acked token = %" PRIu64, +- __func__, v->acked_token); ++ __func__, acked_token); + return 1; + } + +diff --git a/dpdk/lib/regexdev/rte_regexdev.c b/dpdk/lib/regexdev/rte_regexdev.c +index caec069182..d38a85eb0b 100644 +--- a/dpdk/lib/regexdev/rte_regexdev.c ++++ b/dpdk/lib/regexdev/rte_regexdev.c +@@ -19,7 +19,7 @@ static struct { + struct rte_regexdev_data data[RTE_MAX_REGEXDEV_DEVS]; + } *rte_regexdev_shared_data; + +-int rte_regexdev_logtype; ++RTE_LOG_REGISTER_DEFAULT(rte_regexdev_logtype, INFO); + + static uint16_t + regexdev_find_free_dev(void) +diff --git a/dpdk/lib/stack/rte_stack.c b/dpdk/lib/stack/rte_stack.c +index 1fabec2bfe..1dab6d6645 100644 +--- a/dpdk/lib/stack/rte_stack.c ++++ b/dpdk/lib/stack/rte_stack.c +@@ -56,7 +56,7 @@ rte_stack_create(const char *name, unsigned int count, int socket_id, + int ret; + + if (flags & ~(RTE_STACK_F_LF)) { +- STACK_LOG_ERR("Unsupported stack flags %#x\n", flags); ++ STACK_LOG_ERR("Unsupported stack flags %#x", flags); + return NULL; + } + +@@ -65,7 +65,7 @@ rte_stack_create(const char *name, unsigned int count, int socket_id, + #endif + #if !defined(RTE_STACK_LF_SUPPORTED) + if (flags & RTE_STACK_F_LF) { +- STACK_LOG_ERR("Lock-free stack is not supported on your platform\n"); ++ STACK_LOG_ERR("Lock-free stack is not supported on your platform"); + rte_errno = ENOTSUP; + return NULL; + } +@@ -82,7 +82,7 @@ rte_stack_create(const char *name, unsigned int count, int socket_id, + + te = rte_zmalloc("STACK_TAILQ_ENTRY", sizeof(*te), 0); + if (te == NULL) { +- STACK_LOG_ERR("Cannot reserve memory for tailq\n"); ++ STACK_LOG_ERR("Cannot reserve memory for tailq"); + rte_errno = ENOMEM; + return NULL; + } +@@ -92,7 +92,7 @@ rte_stack_create(const char *name, unsigned int count, int socket_id, + mz = rte_memzone_reserve_aligned(mz_name, sz, socket_id, + 0, __alignof__(*s)); + if (mz == NULL) { +- STACK_LOG_ERR("Cannot reserve stack memzone!\n"); ++ STACK_LOG_ERR("Cannot reserve stack memzone!"); + rte_mcfg_tailq_write_unlock(); + rte_free(te); + return NULL; +diff --git a/dpdk/lib/telemetry/telemetry.c b/dpdk/lib/telemetry/telemetry.c +index 92982842a8..f688db7981 100644 +--- a/dpdk/lib/telemetry/telemetry.c ++++ b/dpdk/lib/telemetry/telemetry.c +@@ -169,7 +169,11 @@ container_to_json(const struct rte_tel_data *d, char *out_buf, size_t buf_len) + d->type != TEL_ARRAY_INT && d->type != TEL_ARRAY_STRING) + return snprintf(out_buf, buf_len, "null"); + +- used = rte_tel_json_empty_array(out_buf, buf_len, 0); ++ if (d->type == TEL_DICT) ++ used = rte_tel_json_empty_obj(out_buf, buf_len, 0); ++ else ++ used = rte_tel_json_empty_array(out_buf, buf_len, 0); ++ + if (d->type == TEL_ARRAY_UINT) + for (i = 0; i < d->data_len; i++) + used = rte_tel_json_add_array_uint(out_buf, +@@ -377,8 +381,8 @@ client_handler(void *sock_id) + "{\"version\":\"%s\",\"pid\":%d,\"max_output_len\":%d}", + telemetry_version, getpid(), MAX_OUTPUT_LEN); + if (write(s, info_str, strlen(info_str)) < 0) { +- close(s); +- return NULL; ++ TMTY_LOG(DEBUG, "Socket write base info to client failed\n"); ++ goto exit; + } + + /* receive data is not null terminated */ +@@ -403,6 +407,7 @@ client_handler(void *sock_id) + + bytes = read(s, buffer, sizeof(buffer) - 1); + } ++exit: + close(s); + rte_atomic_fetch_sub_explicit(&v2_clients, 1, rte_memory_order_relaxed); + return NULL; +diff --git a/dpdk/lib/telemetry/telemetry_legacy.c b/dpdk/lib/telemetry/telemetry_legacy.c +index 4c1d1c353a..578230732c 100644 +--- a/dpdk/lib/telemetry/telemetry_legacy.c ++++ b/dpdk/lib/telemetry/telemetry_legacy.c +@@ -94,7 +94,7 @@ register_client(const char *cmd __rte_unused, const char *params, + } + #ifndef RTE_EXEC_ENV_WINDOWS + strlcpy(data, strchr(params, ':'), sizeof(data)); +- memcpy(data, &data[strlen(":\"")], strlen(data)); ++ memmove(data, &data[strlen(":\"")], strlen(data)); + if (!strchr(data, '\"')) { + fprintf(stderr, "Invalid client data\n"); + return -1; +diff --git a/dpdk/lib/vhost/fd_man.c b/dpdk/lib/vhost/fd_man.c +index 134414fb4b..84c5da0793 100644 +--- a/dpdk/lib/vhost/fd_man.c ++++ b/dpdk/lib/vhost/fd_man.c +@@ -307,10 +307,11 @@ fdset_event_dispatch(void *arg) + } + + static void +-fdset_pipe_read_cb(int readfd, void *dat __rte_unused, ++fdset_pipe_read_cb(int readfd, void *dat, + int *remove __rte_unused) + { + char charbuf[16]; ++ struct fdset *fdset = dat; + int r = read(readfd, charbuf, sizeof(charbuf)); + /* + * Just an optimization, we don't care if read() failed +@@ -318,6 +319,11 @@ fdset_pipe_read_cb(int readfd, void *dat __rte_unused, + * compiler happy + */ + RTE_SET_USED(r); ++ ++ pthread_mutex_lock(&fdset->sync_mutex); ++ fdset->sync = true; ++ pthread_cond_broadcast(&fdset->sync_cond); ++ pthread_mutex_unlock(&fdset->sync_mutex); + } + + void +@@ -340,7 +346,7 @@ fdset_pipe_init(struct fdset *fdset) + } + + ret = fdset_add(fdset, fdset->u.readfd, +- fdset_pipe_read_cb, NULL, NULL); ++ fdset_pipe_read_cb, NULL, fdset); + + if (ret < 0) { + RTE_LOG(ERR, VHOST_FDMAN, +@@ -364,5 +370,18 @@ fdset_pipe_notify(struct fdset *fdset) + * compiler happy + */ + RTE_SET_USED(r); ++} ++ ++void ++fdset_pipe_notify_sync(struct fdset *fdset) ++{ ++ pthread_mutex_lock(&fdset->sync_mutex); ++ ++ fdset->sync = false; ++ fdset_pipe_notify(fdset); ++ ++ while (!fdset->sync) ++ pthread_cond_wait(&fdset->sync_cond, &fdset->sync_mutex); + ++ pthread_mutex_unlock(&fdset->sync_mutex); + } +diff --git a/dpdk/lib/vhost/fd_man.h b/dpdk/lib/vhost/fd_man.h +index 6315904c8e..7816fb11ac 100644 +--- a/dpdk/lib/vhost/fd_man.h ++++ b/dpdk/lib/vhost/fd_man.h +@@ -6,6 +6,7 @@ + #define _FD_MAN_H_ + #include + #include ++#include + + #define MAX_FDS 1024 + +@@ -35,6 +36,10 @@ struct fdset { + int writefd; + }; + } u; ++ ++ pthread_mutex_t sync_mutex; ++ pthread_cond_t sync_cond; ++ bool sync; + }; + + +@@ -53,5 +58,6 @@ int fdset_pipe_init(struct fdset *fdset); + void fdset_pipe_uninit(struct fdset *fdset); + + void fdset_pipe_notify(struct fdset *fdset); ++void fdset_pipe_notify_sync(struct fdset *fdset); + + #endif +diff --git a/dpdk/lib/vhost/socket.c b/dpdk/lib/vhost/socket.c +index 5882e44176..0b95c54c5b 100644 +--- a/dpdk/lib/vhost/socket.c ++++ b/dpdk/lib/vhost/socket.c +@@ -93,6 +93,7 @@ static struct vhost_user vhost_user = { + .fd = { [0 ... MAX_FDS - 1] = {-1, NULL, NULL, NULL, 0} }, + .fd_mutex = PTHREAD_MUTEX_INITIALIZER, + .fd_pooling_mutex = PTHREAD_MUTEX_INITIALIZER, ++ .sync_mutex = PTHREAD_MUTEX_INITIALIZER, + .num = 0 + }, + .vsocket_cnt = 0, +diff --git a/dpdk/lib/vhost/vdpa.c b/dpdk/lib/vhost/vdpa.c +index 219eef879c..ce4fb09859 100644 +--- a/dpdk/lib/vhost/vdpa.c ++++ b/dpdk/lib/vhost/vdpa.c +@@ -19,6 +19,7 @@ + #include "rte_vdpa.h" + #include "vdpa_driver.h" + #include "vhost.h" ++#include "iotlb.h" + + /** Double linked list of vDPA devices. */ + TAILQ_HEAD(vdpa_device_list, rte_vdpa_device); +@@ -147,7 +148,6 @@ rte_vdpa_unregister_device(struct rte_vdpa_device *dev) + + int + rte_vdpa_relay_vring_used(int vid, uint16_t qid, void *vring_m) +- __rte_no_thread_safety_analysis /* FIXME: requires iotlb_lock? */ + { + struct virtio_net *dev = get_device(vid); + uint16_t idx, idx_m, desc_id; +@@ -193,17 +193,21 @@ rte_vdpa_relay_vring_used(int vid, uint16_t qid, void *vring_m) + if (unlikely(nr_descs > vq->size)) + return -1; + ++ vhost_user_iotlb_rd_lock(vq); + desc_ring = (struct vring_desc *)(uintptr_t) + vhost_iova_to_vva(dev, vq, + vq->desc[desc_id].addr, &dlen, + VHOST_ACCESS_RO); ++ vhost_user_iotlb_rd_unlock(vq); + if (unlikely(!desc_ring)) + return -1; + + if (unlikely(dlen < vq->desc[desc_id].len)) { ++ vhost_user_iotlb_rd_lock(vq); + idesc = vhost_alloc_copy_ind_table(dev, vq, + vq->desc[desc_id].addr, + vq->desc[desc_id].len); ++ vhost_user_iotlb_rd_unlock(vq); + if (unlikely(!idesc)) + return -1; + +@@ -220,9 +224,12 @@ rte_vdpa_relay_vring_used(int vid, uint16_t qid, void *vring_m) + if (unlikely(nr_descs-- == 0)) + goto fail; + desc = desc_ring[desc_id]; +- if (desc.flags & VRING_DESC_F_WRITE) ++ if (desc.flags & VRING_DESC_F_WRITE) { ++ vhost_user_iotlb_rd_lock(vq); + vhost_log_write_iova(dev, vq, desc.addr, + desc.len); ++ vhost_user_iotlb_rd_unlock(vq); ++ } + desc_id = desc.next; + } while (desc.flags & VRING_DESC_F_NEXT); + +diff --git a/dpdk/lib/vhost/vduse.c b/dpdk/lib/vhost/vduse.c +index 080b58f7de..b46f0e53c7 100644 +--- a/dpdk/lib/vhost/vduse.c ++++ b/dpdk/lib/vhost/vduse.c +@@ -36,6 +36,7 @@ static struct vduse vduse = { + .fd = { [0 ... MAX_FDS - 1] = {-1, NULL, NULL, NULL, 0} }, + .fd_mutex = PTHREAD_MUTEX_INITIALIZER, + .fd_pooling_mutex = PTHREAD_MUTEX_INITIALIZER, ++ .sync_mutex = PTHREAD_MUTEX_INITIALIZER, + .num = 0 + }, + }; +@@ -196,6 +197,7 @@ vduse_vring_setup(struct virtio_net *dev, unsigned int index) + vq->size * sizeof(struct batch_copy_elem), + RTE_CACHE_LINE_SIZE, 0); + ++ rte_rwlock_write_lock(&vq->access_lock); + vhost_user_iotlb_rd_lock(vq); + if (vring_translate(dev, vq)) + VHOST_LOG_CONFIG(dev->ifname, ERR, "Failed to translate vring %d addresses\n", +@@ -206,6 +208,7 @@ vduse_vring_setup(struct virtio_net *dev, unsigned int index) + "Failed to disable guest notifications on vring %d\n", + index); + vhost_user_iotlb_rd_unlock(vq); ++ rte_rwlock_write_unlock(&vq->access_lock); + + vq_efd.index = index; + vq_efd.fd = vq->kickfd; +@@ -259,7 +262,9 @@ vduse_vring_cleanup(struct virtio_net *dev, unsigned int index) + close(vq->kickfd); + vq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD; + ++ rte_rwlock_write_lock(&vq->access_lock); + vring_invalidate(dev, vq); ++ rte_rwlock_write_unlock(&vq->access_lock); + + rte_free(vq->batch_copy_elems); + vq->batch_copy_elems = NULL; +@@ -614,7 +619,7 @@ vduse_device_destroy(const char *path) + vduse_device_stop(dev); + + fdset_del(&vduse.fdset, dev->vduse_dev_fd); +- fdset_pipe_notify(&vduse.fdset); ++ fdset_pipe_notify_sync(&vduse.fdset); + + if (dev->vduse_dev_fd >= 0) { + close(dev->vduse_dev_fd); +diff --git a/dpdk/lib/vhost/vhost_crypto.c b/dpdk/lib/vhost/vhost_crypto.c +index 9bf5ef67b9..7b22281815 100644 +--- a/dpdk/lib/vhost/vhost_crypto.c ++++ b/dpdk/lib/vhost/vhost_crypto.c +@@ -245,7 +245,7 @@ transform_cipher_param(struct rte_crypto_sym_xform *xform, + return ret; + + if (param->cipher_key_len > VHOST_USER_CRYPTO_MAX_CIPHER_KEY_LENGTH) { +- VC_LOG_DBG("Invalid cipher key length\n"); ++ VC_LOG_DBG("Invalid cipher key length"); + return -VIRTIO_CRYPTO_BADMSG; + } + +@@ -301,7 +301,7 @@ transform_chain_param(struct rte_crypto_sym_xform *xforms, + return ret; + + if (param->cipher_key_len > VHOST_USER_CRYPTO_MAX_CIPHER_KEY_LENGTH) { +- VC_LOG_DBG("Invalid cipher key length\n"); ++ VC_LOG_DBG("Invalid cipher key length"); + return -VIRTIO_CRYPTO_BADMSG; + } + +@@ -321,7 +321,7 @@ transform_chain_param(struct rte_crypto_sym_xform *xforms, + return ret; + + if (param->auth_key_len > VHOST_USER_CRYPTO_MAX_HMAC_KEY_LENGTH) { +- VC_LOG_DBG("Invalid auth key length\n"); ++ VC_LOG_DBG("Invalid auth key length"); + return -VIRTIO_CRYPTO_BADMSG; + } + +diff --git a/dpdk/lib/vhost/vhost_user.c b/dpdk/lib/vhost/vhost_user.c +index e36312181a..f8e42dd619 100644 +--- a/dpdk/lib/vhost/vhost_user.c ++++ b/dpdk/lib/vhost/vhost_user.c +@@ -1799,6 +1799,7 @@ vhost_user_set_inflight_fd(struct virtio_net **pdev, + if (!vq) + continue; + ++ cleanup_vq_inflight(dev, vq); + if (vq_is_packed(dev)) { + vq->inflight_packed = addr; + vq->inflight_packed->desc_num = queue_size; +@@ -2198,7 +2199,9 @@ vhost_user_get_vring_base(struct virtio_net **pdev, + + vhost_user_iotlb_flush_all(dev); + ++ rte_rwlock_write_lock(&vq->access_lock); + vring_invalidate(dev, vq); ++ rte_rwlock_write_unlock(&vq->access_lock); + + return RTE_VHOST_MSG_RESULT_REPLY; + } +diff --git a/dpdk/lib/vhost/virtio_net.c b/dpdk/lib/vhost/virtio_net.c +index 8af20f1487..6d53ff932d 100644 +--- a/dpdk/lib/vhost/virtio_net.c ++++ b/dpdk/lib/vhost/virtio_net.c +@@ -1696,6 +1696,17 @@ virtio_dev_rx_packed(struct virtio_net *dev, + return pkt_idx; + } + ++static void ++virtio_dev_vring_translate(struct virtio_net *dev, struct vhost_virtqueue *vq) ++{ ++ rte_rwlock_write_lock(&vq->access_lock); ++ vhost_user_iotlb_rd_lock(vq); ++ if (!vq->access_ok) ++ vring_translate(dev, vq); ++ vhost_user_iotlb_rd_unlock(vq); ++ rte_rwlock_write_unlock(&vq->access_lock); ++} ++ + static __rte_always_inline uint32_t + virtio_dev_rx(struct virtio_net *dev, struct vhost_virtqueue *vq, + struct rte_mbuf **pkts, uint32_t count) +@@ -1710,9 +1721,13 @@ virtio_dev_rx(struct virtio_net *dev, struct vhost_virtqueue *vq, + + vhost_user_iotlb_rd_lock(vq); + +- if (unlikely(!vq->access_ok)) +- if (unlikely(vring_translate(dev, vq) < 0)) +- goto out; ++ if (unlikely(!vq->access_ok)) { ++ vhost_user_iotlb_rd_unlock(vq); ++ rte_rwlock_read_unlock(&vq->access_lock); ++ ++ virtio_dev_vring_translate(dev, vq); ++ goto out_no_unlock; ++ } + + count = RTE_MIN((uint32_t)MAX_PKT_BURST, count); + if (count == 0) +@@ -1731,6 +1746,7 @@ out: + out_access_unlock: + rte_rwlock_read_unlock(&vq->access_lock); + ++out_no_unlock: + return nb_tx; + } + +@@ -1919,7 +1935,7 @@ vhost_enqueue_async_packed(struct virtio_net *dev, + else + max_tries = 1; + +- while (size > 0) { ++ do { + /* + * if we tried all available ring items, and still + * can't get enough buf, it means something abnormal +@@ -1946,7 +1962,7 @@ vhost_enqueue_async_packed(struct virtio_net *dev, + avail_idx += desc_count; + if (avail_idx >= vq->size) + avail_idx -= vq->size; +- } ++ } while (size > 0); + + if (unlikely(mbuf_to_desc(dev, vq, pkt, buf_vec, nr_vec, *nr_buffers, true) < 0)) + return -1; +@@ -2528,9 +2544,13 @@ virtio_dev_rx_async_submit(struct virtio_net *dev, struct vhost_virtqueue *vq, + + vhost_user_iotlb_rd_lock(vq); + +- if (unlikely(!vq->access_ok)) +- if (unlikely(vring_translate(dev, vq) < 0)) +- goto out; ++ if (unlikely(!vq->access_ok)) { ++ vhost_user_iotlb_rd_unlock(vq); ++ rte_rwlock_read_unlock(&vq->access_lock); ++ ++ virtio_dev_vring_translate(dev, vq); ++ goto out_no_unlock; ++ } + + count = RTE_MIN((uint32_t)MAX_PKT_BURST, count); + if (count == 0) +@@ -2551,6 +2571,7 @@ out: + out_access_unlock: + rte_rwlock_write_unlock(&vq->access_lock); + ++out_no_unlock: + return nb_tx; + } + +@@ -3083,7 +3104,6 @@ virtio_dev_tx_split(struct virtio_net *dev, struct vhost_virtqueue *vq, + { + uint16_t i; + uint16_t avail_entries; +- uint16_t dropped = 0; + static bool allocerr_warned; + + /* +@@ -3122,11 +3142,8 @@ virtio_dev_tx_split(struct virtio_net *dev, struct vhost_virtqueue *vq, + + update_shadow_used_ring_split(vq, head_idx, 0); + +- if (unlikely(buf_len <= dev->vhost_hlen)) { +- dropped += 1; +- i++; ++ if (unlikely(buf_len <= dev->vhost_hlen)) + break; +- } + + buf_len -= dev->vhost_hlen; + +@@ -3143,8 +3160,6 @@ virtio_dev_tx_split(struct virtio_net *dev, struct vhost_virtqueue *vq, + buf_len, mbuf_pool->name); + allocerr_warned = true; + } +- dropped += 1; +- i++; + break; + } + +@@ -3155,27 +3170,21 @@ virtio_dev_tx_split(struct virtio_net *dev, struct vhost_virtqueue *vq, + VHOST_LOG_DATA(dev->ifname, ERR, "failed to copy desc to mbuf.\n"); + allocerr_warned = true; + } +- dropped += 1; +- i++; + break; + } +- + } + +- if (dropped) +- rte_pktmbuf_free_bulk(&pkts[i - 1], count - i + 1); +- +- vq->last_avail_idx += i; ++ if (unlikely(count != i)) ++ rte_pktmbuf_free_bulk(&pkts[i], count - i); + +- do_data_copy_dequeue(vq); +- if (unlikely(i < count)) +- vq->shadow_used_idx = i; + if (likely(vq->shadow_used_idx)) { ++ vq->last_avail_idx += vq->shadow_used_idx; ++ do_data_copy_dequeue(vq); + flush_shadow_used_ring_split(dev, vq); + vhost_vring_call_split(dev, vq); + } + +- return (i - dropped); ++ return i; + } + + __rte_noinline +@@ -3581,11 +3590,13 @@ rte_vhost_dequeue_burst(int vid, uint16_t queue_id, + + vhost_user_iotlb_rd_lock(vq); + +- if (unlikely(!vq->access_ok)) +- if (unlikely(vring_translate(dev, vq) < 0)) { +- count = 0; +- goto out; +- } ++ if (unlikely(!vq->access_ok)) { ++ vhost_user_iotlb_rd_unlock(vq); ++ rte_rwlock_read_unlock(&vq->access_lock); ++ ++ virtio_dev_vring_translate(dev, vq); ++ goto out_no_unlock; ++ } + + /* + * Construct a RARP broadcast packet, and inject it to the "pkts" +@@ -3646,6 +3657,7 @@ out_access_unlock: + if (unlikely(rarp_mbuf != NULL)) + count += 1; + ++out_no_unlock: + return count; + } + +@@ -4196,11 +4208,14 @@ rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id, + + vhost_user_iotlb_rd_lock(vq); + +- if (unlikely(vq->access_ok == 0)) +- if (unlikely(vring_translate(dev, vq) < 0)) { +- count = 0; +- goto out; +- } ++ if (unlikely(vq->access_ok == 0)) { ++ vhost_user_iotlb_rd_unlock(vq); ++ rte_rwlock_read_unlock(&vq->access_lock); ++ ++ virtio_dev_vring_translate(dev, vq); ++ count = 0; ++ goto out_no_unlock; ++ } + + /* + * Construct a RARP broadcast packet, and inject it to the "pkts" +@@ -4266,5 +4281,6 @@ out_access_unlock: + if (unlikely(rarp_mbuf != NULL)) + count += 1; + ++out_no_unlock: + return count; + } +diff --git a/dpdk/meson.build b/dpdk/meson.build +index 5e161f43e5..8b248d4505 100644 +--- a/dpdk/meson.build ++++ b/dpdk/meson.build +@@ -164,17 +164,17 @@ message(output_message + '\n') + output_message = '\n=================\nContent Skipped\n=================\n' + output_message += '\napps:\n\t' + foreach app:dpdk_apps_disabled +- reason = get_variable(app.underscorify() + '_disable_reason') ++ reason = get_variable('app_' + app.underscorify() + '_disable_reason') + output_message += app + ':\t' + reason + '\n\t' + endforeach + output_message += '\nlibs:\n\t' + foreach lib:dpdk_libs_disabled +- reason = get_variable(lib.underscorify() + '_disable_reason') ++ reason = get_variable('lib_' + lib.underscorify() + '_disable_reason') + output_message += lib + ':\t' + reason + '\n\t' + endforeach + output_message += '\ndrivers:\n\t' + foreach drv:dpdk_drvs_disabled +- reason = get_variable(drv.underscorify() + '_disable_reason') ++ reason = get_variable('drv_' + drv.underscorify() + '_disable_reason') + output_message += drv + ':\t' + reason + '\n\t' + endforeach + message(output_message + '\n') diff --git a/SPECS/openvswitch3.3.spec b/SPECS/openvswitch3.3.spec index b71099a..3a71839 100644 --- a/SPECS/openvswitch3.3.spec +++ b/SPECS/openvswitch3.3.spec @@ -57,7 +57,7 @@ Summary: Open vSwitch Group: System Environment/Daemons daemon/database/utilities URL: http://www.openvswitch.org/ Version: 3.3.0 -Release: 55%{?dist} +Release: 56%{?dist} # Nearly all of openvswitch is ASL 2.0. The bugtool is LGPLv2+, and the # lib/sflow*.[ch] files are SISSL @@ -769,6 +769,543 @@ exit 0 %endif %changelog +* Thu Oct 24 2024 Open vSwitch CI - 3.3.0-56 +- Merging dpdk subtree [RH git: 61607569a5] + Commit list: + e8eb14e00d version: 23.11.2 + 8401a3e84b version: 23.11.2-rc2 + 50e50f1d99 net/ice/base: fix preparing PHY for timesync command + 7302cab07c net/nfp: fix firmware abnormal cleanup + 92c5aa4387 net/nfp: forbid offload flow rules with empty action list + e9c4dbd5be crypto/openssl: make per-QP auth context clones + 729e0848b7 examples: fix port ID restriction + 3fc9eb2f4f examples: fix lcore ID restriction + 776c4e37ee doc: add baseline mode in l3fwd-power guide + 8437250f9f doc: fix DMA performance test invocation + d01561713d doc: describe mlx5 HWS actions order + 1ef3097094 doc: add power uncore in API index + 09ccd86606 doc: fix mbuf flags + 613a4879b4 examples/ipsec-secgw: revert SA salt endianness + 8b87ae54ed doc: remove reference to mbuf pkt field + 938afb0ab2 examples: fix queue ID restriction + 80da81b6f9 net/ice/base: fix temporary failures reading NVM + 034f533709 net/hns3: fix uninitialized variable in FEC query + 5fa2084ac3 examples/l3fwd: fix crash on multiple sockets + 1708229729 examples/l3fwd: fix crash in ACL mode for mixed traffic + bef8327055 bus/vdev: fix device reinitialization + 971d455e59 malloc: fix multi-process wait condition handling + b53fb811c0 power: fix number of uncore frequencies + 52bf7488c3 app/pdump: handle SIGTERM and SIGHUP + 6bed8020a3 app/dumpcap: handle SIGTERM and SIGHUP + eacf416207 dma/hisilicon: remove support for HIP09 platform + a5375f4492 bus/pci: fix FD in secondary process + c076f02992 bus/pci: fix UIO resource mapping in secondary process + ca12727f09 app/testpmd: fix build on signed comparison + b858eb7a55 net/gve: fix Tx queue state on queue start + 7ff9eeeb71 ethdev: fix device init without socket-local memory + 890b02b907 app/testpmd: add postpone option to async flow destroy + fbd7ac3b83 net/netvsc: use ethdev API to set VF MTU + e8746659b2 ethdev: fix GENEVE option item conversion + e2b3b0a5d4 net/ark: fix index arithmetic + 7deaec6ac8 net/hns3: check Rx DMA address alignmnent + 6bf4a626e4 net/mlx5: fix disabling E-Switch default flow rules + 3b60ef3db6 common/mlx5: remove unneeded field when modify RQ table + 1988c32194 net/mlx5: fix uplink port probing in bonding mode + 07ad92c1a6 net/mlx5: fix end condition of reading xstats + b7c9a02306 net/mlx5/hws: remove unused variable + 1163643a1e net/mlx5/hws: fix port ID on root item convert + 4a80ab31f0 net/mlx5/hws: fix deletion of action vport + dcd02c715e net/mlx5/hws: fix check of range templates + 3c9aff8fbf net/mlx5/hws: fix memory leak in modify header + 85eeb293b3 net/mlx5: fix MTU configuration + c1792007ff net/mlx5: fix Arm build with GCC 9.1 + a8331ab8b2 net/mlx5: fix shared Rx queue data access race + 15e2b0e736 net/ice: fix return value for raw pattern parsing + 17e800edd1 net/ice: fix memory leaks in raw pattern parsing + 8f23521ad7 common/cnxk: fix integer overflow + a408bd0bcb crypto/qat: fix placement of OOP offset + b2acb5218f test/crypto: fix modex comparison + d308cefc96 test/crypto: fix asymmetric capability test + adffcf4383 test/crypto: remove unused stats in setup + 4b0d806bab doc: fix typo in l2fwd-crypto guide + 0e19bfa703 crypto/qat: fix log message typo + 5d66d4a3f0 test/crypto: fix allocation comment + 9fef0db81b crypto/ipsec_mb: fix function comment + c6111cb5fd crypto/qat: fix GEN4 write + f707532cde net/nfp: fix disabling 32-bit build + de0c58c4b8 doc: update AF_XDP device plugin repository + 811fcdf23a net/nfp: adapt reverse sequence card + ea7085704a net/nfp: remove unneeded logic for VLAN layer + 4c45c694ca doc: update metadata description in nfp guide + 174b2b5a9a net/nfp: fix getting firmware version + 241976029a net/nfp: remove redundant function call + c90d304f0e net/gve: fix RSS hash endianness in DQO format + 316baf3f0b net/ena: fix checksum handling + ffba3914ad net/ena: fix return value check + db4ca6f1cb net/ena: fix bad checksum handling + 5453c7a0b0 net/nfp: fix repeat disable port + 51d6936232 net/nfp: fix dereference of null pointer + d0d759188e net/nfp: disable ctrl VNIC queues on close + c22c079e1c net/ionic: fix mbuf double-free when emptying array + 2e84f93745 net/nfp: fix flow mask table entry + 8df473a653 net/nfp: fix allocation of switch domain + 2a50559bfd net/netvsc: fix MTU set + e2ef427581 net/nfp: fix IPv6 TTL and DSCP flow action + 3762e1ed03 net/vmxnet3: fix init logs + fdc5f6074b net/txgbe: fix Rx interrupt + 2dfb54ff33 net/ngbe: fix memory leaks + 638e12515a net/txgbe: fix memory leaks + d0a84e43b3 net/ngbe: fix MTU range + 0fe77d1de3 net/txgbe: fix MTU range + db50b74b3c net/ngbe: fix hotplug remove + 5de5204582 net/txgbe: fix hotplug remove + 6d224307f6 net/ngbe: keep PHY power down while device probing + 658060fffe net/ngbe: add special config for YT8531SH-CA PHY + 85bd339e19 net/txgbe: fix VF promiscuous and allmulticast + ea62ead19d net/txgbe: reconfigure more MAC Rx registers + 55d8be2055 net/txgbe: restrict configuration of VLAN strip offload + ed2250e120 net/txgbe: fix Tx hang on queue disable + 6ea637699a net/txgbe: fix flow filters in VT mode + 842a7baf9c net/txgbe: fix tunnel packet parsing + 708d5a261b net/mana: fix uninitialized return value + e113512712 app/testpmd: fix parsing for connection tracking item + 11b6493c45 doc: remove empty section from testpmd guide + f11711212c app/testpmd: handle IEEE1588 init failure + e7f8c62dfc net/cpfl: fix 32-bit build + ec9de9db2d net/cpfl: add checks on control queue messages + 39b2b4c7de common/idpf: fix PTP message validation + e0f453462f common/idpf: fix flex descriptor mask + bd5b88d172 net/ice/base: fix masking when reading context + 67a40ce4ef net/ice/base: fix board type definition + 5167b4d2d3 net/ice/base: fix potential TLV length overflow + abd055ea63 net/ice/base: fix check for existing switch rule + fddfbdbf49 net/ice/base: fix return type of bitmap hamming weight + c9eae16d5e net/ice/base: fix GCS descriptor field offsets + 8bc9ae6b59 net/ice/base: fix size when allocating children arrays + 1257bf9a7c net/ice/base: fix sign extension + 2458257e56 net/ice/base: fix resource leak + 7a6c0e6212 net/ice/base: fix memory leak in firmware version check + 87d7cf4082 net/ice/base: fix pointer to variable outside scope + aafeb830bc buildtools: fix build with clang 17 and ASan + a4e8a4f488 fbarray: fix finding for unaligned length + d88beb497f net/mlx5: fix start without duplicate flow patterns + 77231b2598 net/dpaa: forbid MTU configuration for shared interface + d21248db89 bus/dpaa: remove redundant file descriptor check + bb85c1fd72 common/dpaax: fix node array overrun + 90c9f938e5 common/dpaax: fix IOVA table cleanup + 0b4bc3a5d1 bus/dpaa: fix memory leak in bus scan + d36efdb2cd bus/dpaa: fix bus scan for DMA devices + daa0d9edd1 app/testpmd: fix help string of BPF load command + 7353cb767f dma/idxd: fix setup with Ubuntu 24.04 + f563086258 eal/linux: lower log level on allocation attempt failure + 8ccf607fad devtools: fix symbol listing + 997166395e fbarray: fix lookbehind ignore mask handling + 8baf379032 fbarray: fix lookahead ignore mask handling + 24869bf93c fbarray: fix incorrect lookbehind behavior + 5e66590575 fbarray: fix incorrect lookahead behavior + 427fa07238 examples/ipsec-secgw: fix SA salt endianness + d0d02993de crypto/dpaa2_sec: fix event queue user context + 6e2def6ca9 crypto/dpaa_sec: fix IPsec descriptor + b977583692 common/dpaax/caamflib: fix PDCP AES-AES watchdog error + 5089ef6c28 common/dpaax/caamflib: fix PDCP-SDAP watchdog error + 4af94ab6e2 crypto/openssl: set cipher padding once + 4f8c97e941 crypto/openssl: make per-QP cipher context clones + ee88b9496c crypto/openssl: optimize 3DES-CTR context init + eb6a1a85e6 crypto/openssl: fix GCM and CCM thread unsafe contexts + cc8ca588a0 examples/fips_validation: fix dereference and out-of-bound + 9b3e235581 cryptodev: validate crypto callbacks from next node + 578ee20720 cryptodev: fix build without crypto callbacks + fbb350108f crypto/cnxk: fix minimal input normalization + 7978b75d1b test/crypto: validate modex from first non-zero + ede34a4359 app/crypto-perf: fix result for asymmetric + 7469762567 app/crypto-perf: remove redundant local variable + e585a0db98 crypto/cnxk: fix ECDH public key verification + 6034788bd6 crypto/cnxk: fix out-of-bound access + ea90bc49fc net/virtio-user: fix control queue allocation for non-vDPA + 15d3dfa07a baseband/la12xx: forbid secondary process + f798848548 telemetry: fix connection parameter parsing + 713520f91d bpf: fix load hangs with six IPv6 addresses + 59523f029e bpf: fix MOV instruction evaluation + c9071e44b7 mbuf: fix dynamic fields copy + c13a819a44 graph: fix mcore dispatch walk + 777f0bc1a5 vdpa/sfc: remove dead code + 583796e298 dmadev: fix structure alignment + d859544e45 common/cnxk: fix flow aging on application exit + c343cb088f app/bbdev: fix interrupt tests + 0c99a3d922 app/bbdev: fix MLD output size computation + 179f1c6e6b app/bbdev: fix TB logic + acdd88c4f9 build: use builtin helper for python dependencies + edfa6a87c8 config: fix warning for cross build with meson >= 1.3.0 + 151a54d0b6 v23.11.2-rc1 + 61b7d1f4c1 doc: fix link to hugepage mapping from Linux guide + 0e68080faf telemetry: lower log level on socket error + 4fe42b5bd5 test/crypto: fix enqueue/dequeue callback case + 4dc08a4d14 test/crypto: fix RSA cases in QAT suite + f1e088abb9 net/mlx5/hws: fix matcher reconnect + db0c8afc11 net/mlx5: fix crash on counter pool destroy + 59e27c048f net/mlx5: support jump in meter hierarchy + 15d0fcf1ac net/mlx5: fix access to flow template operations + 7cc4f4359e net/mlx5: break flow resource release loop + 8f7a4c4861 net/mlx5: fix flow template indirect action failure + 0caa8332a4 net/mlx5: fix hash Rx queue release in flow sample + 5546ccbefe net/mlx5: fix indexed pool with invalid index + c12bd3ffbf net/mlx5/hws: fix action template dump + 096734a9b5 net/mlx5/hws: set default miss when replacing table + df8e365511 net/mlx5/hws: extend tag saving for match and jumbo + 9b84b09d4a net/mlx5/hws: add template match none flag + 8b2eb11323 net/mlx5/hws: fix spinlock release on context open + 3811ef8d25 net/mlx5/hws: fix function comment + 21c0e76d5a common/mlx5: fix PRM structs + aec70880d8 net/mlx5/hws: decrease log level for creation failure + b4d5b769a9 common/mlx5: fix unsigned/signed mismatch + 5405ea2f7a hash: fix RCU reclamation size + 2f62695370 bpf: disable on 32-bit x86 + 61c4175079 graph: fix stats retrieval while destroying a graph + e022af0b88 graph: fix ID collisions + 4541f5810c net/cnxk: fix promiscuous state after MAC change + 144a806a1b net/cnxk: fix outbound security with higher packet burst + 01d4a05a9f net/cnxk: update SA userdata and keep original cookie + 802a3a7d74 net/cnxk: fix extbuf handling for multisegment packet + 67a8e5ba52 common/cnxk: fix segregation of logs based on module + c99b186412 common/cnxk: fix flow aging cleanup + af85590165 net/cnxk: fix RSS config + 1e2d3032e2 net/ixgbe/base: fix PHY ID for X550 + b371343ddc net/ixgbe/base: fix 5G link speed reported on VF + 3d128f41b7 net/ixgbe/base: revert advertising for X550 2.5G/5G + 7ec2441a3e net/e1000/base: fix link power down + 4c8436297f net/ixgbe: do not create delayed interrupt handler twice + f683115cef net/ixgbe: do not update link status in secondary process + 0fc2747f6c net/ice: fix VLAN stripping in double VLAN mode + 729144bdae net/fm10k: fix cleanup during init failure + bb9096e474 net/iavf: fix VF reset when using DCF + b6e445d0d4 eventdev/crypto: fix opaque field handling + 32c7c20981 event/sw: fix warning from useless snprintf + 614773e8c7 baseband/acc: fix memory barrier + ac1bd05172 net/virtio: fix MAC table update + eb821e0ed1 net/virtio-user: fix control queue allocation + 6e4de6f224 net/virtio-user: fix shadow control queue notification init + 1d824e440e net/virtio-user: fix control queue destruction + 2fdb8840ee vhost: cleanup resubmit info before inflight setup + 8c020a6f4d vhost: fix build with GCC 13 + 1af612de7e hash: check name when creating a hash + 9616fce23b hash: fix return code description in Doxygen + 44bcfd6b38 net/nfp: fix xstats for multi PF firmware + 8bf40f1d11 app/testpmd: fix lcore ID restriction + 80c5c9789b net/iavf: remove outer UDP checksum offload for X710 VF + 1970a0ca45 net/i40e: fix outer UDP checksum offload for X710 + e8c2cccfbd net: fix outer UDP checksum in Intel prepare helper + dda814c495 app/testpmd: fix outer IP checksum offload + 4d57f72a5b net/ice: fix check for outer UDP checksum offload + c61b23292e net/axgbe: fix linkup in PHY status + b7eddfc563 net/axgbe: delay AN timeout during KR training + 388f022054 net/axgbe: fix Tx flow on 30H HW + e3632f6bbb net/axgbe: check only minimum speed for cables + 141a4ff6d5 net/axgbe: fix connection for SFP+ active cables + 4eda15db34 net/axgbe: fix SFP codes check for DAC cables + d72913dcad net/axgbe: enable PLL control for fixed PHY modes only + 3cf40bf1c3 net/axgbe: disable RRC for yellow carp devices + ec06a8c3d4 net/axgbe: disable interrupts during device removal + a2be089e35 net/axgbe: update DMA coherency values + 17290bc90b net/axgbe: fix fluctuations for 1G Bel Fuse SFP + a61b3c008a net/axgbe: reset link when link never comes back + 498a5720e3 net/axgbe: fix MDIO access for non-zero ports and CL45 PHYs + dea5481a8f net/tap: fix file descriptor check in isolated flow + c22f99f86c net/nfp: fix configuration BAR + ab2e5cf865 net/nfp: fix resource leak in secondary process + 442ca8b2ec net/af_xdp: remove unused local statistic + b0a4771394 net/af_xdp: fix stats reset + fdda0d4d83 net/af_xdp: count mbuf allocation failures + c6891273d3 net/af_xdp: fix port ID in Rx mbuf + 60f2e572ea doc: fix testpmd ring size command + bd69f5a43d net/af_packet: align Rx/Tx structs to cache line + e9da7f4655 net/vmxnet3: add missing register command + e754c1c6d8 ethdev: fix strict aliasing in link up + b1be619a77 net/af_xdp: fix multi-interface support for k8s + 3bbccce3a9 doc: fix AF_XDP device plugin howto + 02d2453afc net/hns3: disable SCTP verification tag for RSS hash input + 6e37e43fe3 net/hns3: fix variable overflow + 16a24e9f99 net/hns3: fix double free for Rx/Tx queue + 78e4da4546 net/hns3: fix Rx timestamp flag + c491084749 net/hns3: fix offload flag of IEEE 1588 + beda536606 app/testpmd: fix indirect action flush + dae924d0a2 net/bonding: fix failover time of LACP with mode 4 + 453e0c281b net/nfp: fix representor port queue release + 83149f4fea latencystats: fix literal float suffix + 50b99c8b12 eal/windows: install sched.h file + 11af26df38 net/virtio-user: add memcpy check + eb02060534 pcapng: add memcpy check + 2dd9223248 eal/unix: support ZSTD compression for firmware + 09e70301ee eal: fix type in destructor macro for MSVC + 338632b663 bus/pci: fix build with musl 1.2.4 / Alpine 3.19 + a6ec5765cf version: 23.11.1 + 51783c9b60 version: 23.11.1-rc2 + 152600d10e net/mlx5/hws: fix tunnel protocol checks + 67f3179f5c net/mlx5: fix rollback on failed flow configure + 750d393405 net/mlx5: fix async flow create error handling + 41c5baeffc net/mlx5/hws: fix port ID for root table + cfa8a4cb90 net/ena/base: fix metrics excessive memory consumption + a20a3c1129 dts: strip whitespaces from stdout and stderr + abc6816134 examples/ipsec-secgw: fix typo in error message + cf631810bf test/cfgfile: fix typo in error messages + f48e923b46 test/power: fix typo in error message + ed3a625fe6 doc: fix typo in packet framework guide + e85092f875 doc: fix typo in profiling guide + df1119d4a9 net/mlx5: fix sync flow meter action + aeebcd33c0 net/mlx5/hws: fix memory access in L3 decapsulation + 0291a1f49e net/igc: fix timesync disable + 07fde8240d net/vmxnet3: ignore Rx queue interrupt setup on FreeBSD + d2d309e5cf net/ena: fix mbuf double free in fast free mode + f7d1b5cff3 app/testpmd: fix auto-completion for indirect action list + 4aa1a64204 net/nfp: fix uninitialized variable + a5ac9baa7a doc: fix default IP fragments maximum in programmer guide + f7909e3c75 examples/ipsec-secgw: fix Rx queue ID in Rx callback + 0a44e64c41 net/bnxt: fix number of Tx queues being created + 5a8ca987e9 net/mlx5: fix warning about copy length + 2864fd3102 net/mlx5: fix drop action release timing + 9aba4dee4d net/mlx5: fix age position in hairpin split + fc12ccc047 net/mlx5: prevent ioctl failure log flooding + 8117b4b2f7 net/mlx5: fix flow configure validation + b1749f6ed2 net/mlx5: fix template clean up of FDB control flow rule + 3735e8e88c net/mlx5/hws: fix direct index insert on depend WQE + 859bafedf3 net/mlx5: fix DR context release ordering + 9a9f0acac6 net/mlx5: fix IP-in-IP tunnels recognition + c551015ebb net/mlx5: remove duplication of L3 flow item validation + a8b06881d9 net/mlx5: fix meter policy priority + 78d38b5d67 net/mlx5: fix VLAN ID in flow modify + f01fd28181 doc: update link to Windows DevX in mlx5 guide + af41defcf7 net/mlx5: fix non-masked indirect list meter translation + 1994df02c9 net/mlx5: fix indirect action async job initialization + 7192f0ed82 net/mlx5: fix sync meter processing in HWS + 50eb03f8d3 net/mlx5: fix HWS meter actions availability + fe697bbce3 net/hns3: support new device + 97089aa02e app/testpmd: fix error message for invalid option + 92c08367ea app/testpmd: fix burst option parsing + 6c2174ad80 app/testpmd: fix --stats-period option check + 0884b3bd36 net/nfp: fix initialization failure flow + dd48153b15 net/nfp: fix switch domain free check + aa850bad00 net/ena/base: restructure interrupt handling + e1abac3de0 net/ena/base: limit exponential backoff + 2fa8497bd3 net/ena: fix fast mbuf free + 5f75adca7e net/nfp: fix IPsec data endianness + bec3117648 net/nfp: fix getting firmware VNIC version + 5853ebb3b9 doc: add link speeds configuration in features table + 15952c71eb app/testpmd: fix async indirect action list creation + 166c5df810 doc: add traffic manager in features table + cadb90f711 net/hns3: enable PFC for all user priorities + 72d3dfa9de crypto/qat: fix crash with CCM null AAD pointer + 90d0e13d7d examples/ipsec-secgw: fix cryptodev to SA mapping + 9796ac2ab8 build: pass cflags in subproject + 7105c8a299 net/virtio: fix vDPA device init advertising control queue + 587143897e examples/l3fwd: fix Rx queue configuration + 2f8836901c dts: fix smoke tests driver regex + ce95b8c9cd examples/l3fwd: fix Rx over not ready port + 10296d5f50 examples/packet_ordering: fix Rx with reorder mode disabled + e8dccbca30 test: do not count skipped tests as executed + 5c5df0f292 test: assume C source files are UTF-8 encoded + de3976eb27 test/mbuf: fix external mbuf case with assert enabled + ced51dd5ef config: fix CPU instruction set for cross-build + 6148604a43 bus/vdev: fix devargs in secondary process + ef4c8a57f3 test: fix probing in secondary process + 272feb8eb9 net/mlx5: remove device status check in flow creation + a10a65c396 net/mlx5: fix flow action template expansion + 0c31d1220f net/mlx5: fix counters map in bonding mode + 091234f3cb net/mlx5: fix flow counter cache starvation + b90c42e4ff net/mlx5: fix parameters verification in HWS table create + 0198b11a11 net/mlx5: fix VLAN handling in meter split + 86c66608c2 net/mlx5/hws: enable multiple integrity items + ca1084cd48 net/mlx5: fix HWS registers initialization + 527857d5c2 net/mlx5: fix connection tracking action validation + 1d65510ff6 net/mlx5: fix conntrack action handle representation + a5d0545e5d net/mlx5: fix condition of LACP miss flow + 17f644b4a8 net/mlx5/hws: fix VLAN inner type + 99be466799 net/mlx5: prevent querying aged flows on uninit port + bfa6cbba4c net/mlx5: fix error packets drop in regular Rx + 213cb88068 net/mlx5: fix use after free when releasing Tx queues + a06ab8044a net/mlx5/hws: fix VLAN item in non-relaxed mode + 21d51e8848 net/mlx5/hws: check not supported fields in VXLAN + b80ca5960e net/mlx5/hws: skip item when inserting rules by index + 2368f82fd8 doc: fix aging poll frequency option in cnxk guide + 630dbc8a92 net/cnxk: improve Tx performance for SW mbuf free + 37256aa1bf common/cnxk: fix possible out-of-bounds access + 9172348240 common/cnxk: remove dead code + 9cb9b9c8a0 common/cnxk: fix link config for SDP + 6f05d2d461 net/cnxk: fix mbuf fields in multi-segment Tx + a6bd2f39c1 common/cnxk: fix mbox struct attributes + e5450b2bba net/cnxk: add cookies check for multi-segment offload + 0e5159a223 net/cnxk: fix indirect mbuf handling in Tx + 6c6cd1fe53 common/cnxk: fix RSS RETA configuration + f4c83ba01c net/cnxk: fix MTU limit + 3e73021b35 common/cnxk: fix Tx MTU configuration + e71ac13a38 net/cnxk: fix buffer size configuration + fbfaa5ae04 common/cnxk: remove CN9K inline IPsec FP opcodes + b3ef799286 net/bnx2x: fix warnings about memcpy lengths + 2d11f389b0 net/cnxk: fix Rx packet format check condition + 8bc81d5447 common/cnxk: fix inline device pointer check + dbdcd8bb85 net/ice: remove incorrect 16B descriptor read block + 72093d3d41 net/iavf: remove incorrect 16B descriptor read block + 542c8410cb net/i40e: remove incorrect 16B descriptor read block + 33b5bed057 net/ixgbe: increase VF reset timeout + eefc0111de net/iavf: remove error logs for VLAN offloading + 2aa5a75750 net/ixgbevf: fix RSS init for x550 NICs + a71de447a2 net/bnxt: fix null pointer dereference + 1d5bfd9fdf net/tap: fix traffic control handle calculation + e9462a5690 net/tap: do not overwrite flow API errors + 4a1ffc9b02 app/testpmd: fix async flow create failure handling + 92ab2d6da2 app/testpmd: return if no packets in GRO heavy weight mode + 61ce57b13a net/mlx5: fix modify flex item + 4d1331e972 app/testpmd: fix flow modify tag typo + c2d52df599 net/af_xdp: fix leak on XSK configuration failure + b2dba501cf vhost: fix VDUSE device destruction failure + af414b892d common/qat: fix legacy flag + 6cacd0e502 doc: fix typos in cryptodev overview + 14c38e2db1 app/crypto-perf: add missing op resubmission + a1f1843146 app/crypto-perf: fix out-of-place mbuf size + f0cfffc636 app/crypto-perf: fix copy segment size + b2cd908926 eventdev/crypto: fix enqueueing + e5ed464710 eventdev: fix Doxygen processing of vector struct + 2faf71417f eventdev: improve Doxygen comments on configure struct + 7721c9f498 test/event: fix crash in Tx adapter freeing + 524c60f422 event/dlb2: remove superfluous memcpy + 4e8d39a298 doc: fix configuration in baseband 5GNR driver guide + b0b971bf66 23.11.1-rc1 + 05bea47b81 app/testpmd: fix GRO packets flush on timeout + cc670c7833 net/nfp: fix NFDk metadata process + 8e79562a0e net/nfp: fix NFD3 metadata process + 1f3f996269 net/mlx5: fix stats query crash in secondary process + 5982bea06b net/mlx5: fix GENEVE option item translation + 22653f6966 net/mlx5: remove GENEVE options length limitation + 06c494555f common/mlx5: fix query sample info capability + 10061b4047 common/mlx5: fix duplicate read of general capabilities + 1825629903 net/mlx5: fix GENEVE TLV option management + c3eb862979 net/mlx5/hws: fix ESP flow matching validation + d25716a8a0 net/mlx5: fix flow tag modification + 7c8f2e719a net/mlx5: fix jump action validation + 01c5db8d99 net/cnxk: fix aged flow query + ecdb679c52 common/cnxk: fix VLAN check for inner header + 4f69dab88c common/cnxk: fix mbox region copy + 0e5798d30b net/thunderx: fix DMAC control register update + 874fd28866 net/cnxk: fix flow RSS configuration + f047cea926 ml/cnxk: fix xstats calculation + a77f545bd7 net/bnxt: fix deadlock in ULP timer callback + d9e1762f07 net/bnxt: modify locking for representor Tx + c26cb2a644 net/bnxt: fix backward firmware compatibility + 1fb50b8baa net/bnxt: fix speed change from 200G to 25G on Thor + e1f8152ede net/bnxt: fix 50G and 100G forced speed + 3fa018b15a net/bnxt: fix array overflow + c3ccbda492 net/netvsc: fix VLAN metadata parsing + de2d362411 net: add macros for VLAN metadata parsing + 561a3f508f net/gve: fix DQO for chained descriptors + de543e342a net/softnic: fix include of log library + edaeda9ef7 net/memif: fix extra mbuf refcnt update in zero copy Tx + c7b50f40e1 common/sfc_efx/base: use C11 static assert + 216918c28c net/mana: handle MR cache expansion failure + 6679de7a8f net/mana: fix memory leak on MR allocation + 3fb4840708 net/bonding: fix flow count query + 1ce60b941d net/ionic: fix device close + 9583f634f3 net/ionic: fix RSS query + 2ea5bde557 net/ionic: fix missing volatile type for cqe pointers + 49b4ce1f94 app/testpmd: fix crash in multi-process forwarding + db4ba50b3a drivers/net: fix buffer overflow for packet types list + 7de2520f2d net/mana: prevent values overflow returned from RDMA layer + 84e9d93f57 net/nfp: free switch domain ID on close + a581442d9b net/nfp: fix device resource freeing + 52bd57a03b net/nfp: fix device close + c65a2bfc26 net/vmxnet3: fix initialization on FreeBSD + edc0e91ffc app/testpmd: hide --bitrate-stats in help if disabled + ec2260423e doc: add --latencystats option in testpmd guide + e670d64d34 net/hns3: remove QinQ insert support for VF + 48fe88cb3c net/nfp: fix Rx descriptor + e024c471f9 net/nfp: fix Rx memory leak + cb1cef89c4 net/hns3: fix reset level comparison + a3584fcde6 net/hns3: fix disable command with firmware + 48d9241bbd net/hns3: fix VF multiple count on one reset + 8abf8591dc net/hns3: refactor handle mailbox function + 1be4ad59be net/hns3: refactor send mailbox function + f876981e54 net/hns3: refactor PF mailbox message struct + c1c62366ed net/hns3: refactor VF mailbox message struct + 9cf299a873 net/memif: fix crash with Tx burst larger than 255 + 01809245ba net/af_xdp: fix memzone leak on config failure + c2a5c0d085 net/nfp: fix resource leak for VF + ddeb9d64a9 net/nfp: fix resource leak for exit of flower firmware + e65a677895 net/nfp: fix resource leak for exit of CoreNIC firmware + 09e1df883a net/nfp: fix resource leak for flower firmware + 02916557c1 net/nfp: fix resource leak for PF initialization + 1d53d5495b net/nfp: fix resource leak for CoreNIC firmware + f2ee31d52c net/nfp: fix resource leak for device initialization + 8610d2715d ethdev: fix NVGRE encap flow action description + d06a344524 doc: fix commands in eventdev test tool guide + 9350513462 test/event: skip test if no driver is present + 6ccd84cf16 event/cnxk: fix dequeue timeout configuration + b7fd1f73fe app/crypto-perf: fix encrypt operation verification + 04d9dfd665 app/crypto-perf: fix data comparison + dfc9d45365 app/crypto-perf: fix next segment mbuf + 8988726643 crypto/cnxk: fix CN9K ECDH public key verification + ea096d3e48 common/cnxk: fix memory leak in CPT init + f5d6c54154 examples/ipsec-secgw: fix width of variables + 96d48b5b40 cryptodev: remove unused extern variable + e951bbbd18 vhost: fix memory leak in Virtio Tx split path + 19f0cf0927 vdpa/mlx5: fix queue enable drain CQ + 5eb1dd92dc vhost: fix deadlock during vDPA SW live migration + 33fbddf9a4 net/virtio: remove duplicate queue xstats + c8e7cd6c6d vhost: fix virtqueue access check in vhost-user setup + 692a7a0034 vhost: fix virtqueue access check in VDUSE setup + bbba917213 vhost: fix virtqueue access check in datapath + c139df70dd net: fix TCP/UDP checksum with padding data + c30a4f8b31 rcu: fix acked token in debug log + 94b20c14a6 rcu: use atomic operation on acked token + 8878a84e2e build: link static libs with whole-archive in subproject + 5e24d7f2de build: fix linker warnings about undefined symbols + 63241d7662 net/sfc: fix calloc parameters + acad009eed net/nfp: fix calloc parameters + 238a03cdec net/bnx2x: fix calloc parameters + e3ae3295ee common/mlx5: fix calloc parameters + 7c10528d68 rawdev: fix calloc parameters + ad881b0db8 dmadev: fix calloc parameters + 9173abff75 eventdev: fix calloc parameters + 600e30b793 pipeline: fix calloc parameters + 5331b41382 examples/vhost: verify strdup return + 88f1c9af33 examples/qos_sched: fix memory leak in args parsing + fbd04d26f3 test: verify strdup return + c830d9e2af app/testpmd: verify strdup return + 8d5327fcfd app/dma-perf: verify strdup return + fefe40a5ed app/crypto-perf: verify strdup return + e0fd44c6ab app/pdump: verify strdup return + 1387327fa4 app/dumpcap: verify strdup return + c6790ef542 net/nfp: verify strdup return + 2044a179a7 net/failsafe: fix memory leak in args parsing + cedf721f24 event/cnxk: verify strdup return + df74839ea1 dma/idxd: verify strdup return + b4943e7a51 bus/vdev: verify strdup return + 82d4ba69f2 bus/fslmc: verify strdup return + 8c8e7aeb90 bus/dpaa: verify strdup return + 2feed5de50 eal: verify strdup return + bb34c79bf4 doc: remove cmdline polling mode deprecation notice + 5f30c47cc5 eal/x86: add AMD vendor check for TSC calibration + a9e8fc49d9 ci: update versions of actions in GHA + d7a30d20c4 gro: fix reordering of packets + b5c580913f telemetry: fix empty JSON dictionaries + cbd1c165bb telemetry: fix connected clients count + d7dc480432 app/graph: fix build reason + 7872a7b0bd build: fix reasons conflict + 54e4045c78 kernel/freebsd: fix module build on FreeBSD 14 + 943de5c27e net/ice: fix memory leaks + 9aa2da4c02 net/iavf: fix crash on VF start + 0a72821dd9 net/iavf: fix no polling mode switching + c321ba6a9d net/ice: fix tunnel TSO capabilities + 48efa16873 net/ice: fix link update + c655f20c8f net/ixgbe: fix memoy leak after device init failure + 3defa10a78 net/iavf: fix memory leak on security context error + ca47a866b5 net/i40e: remove redundant judgment in flow parsing + ec5fe01a28 dma/dpaa2: fix logtype register + fd6f07da94 lib: remove redundant newline from logs + ec5e780f09 lib: add newline in logs + e421bcd708 lib: use dedicated logtypes and macros + f1e3dec4b4 regexdev: fix logtype register + dc0428a5e4 hash: remove some dead code + 2c512fe65a buildtools/cmdline: fix IP address initializer + 3cedd8b9e4 buildtools/cmdline: fix generated code for IP addresses + + * Tue Oct 22 2024 Open vSwitch CI - 3.3.0-55 - Merging upstream branch-3.3 [RH git: 749598021a] Commit list: