From 443f86dd75ee4d36e225c6f42685a751010f00cc Mon Sep 17 00:00:00 2001 From: Open vSwitch CI Date: Apr 30 2024 19:44:58 +0000 Subject: Import openvswitch3.1-3.1.0-113 from Fast DataPath --- diff --git a/.openvswitch.metadata b/.openvswitch.metadata index e9139a6..bee0497 100644 --- a/.openvswitch.metadata +++ b/.openvswitch.metadata @@ -1,6 +1,6 @@ +d34f96421a86004aa5d26ecf975edefd09f948b1 SOURCES/Pygments-1.4.tar.gz +3a11f130c63b057532ca37fe49c8967d0cbae1d5 SOURCES/Sphinx-1.2.3.tar.gz 002450621b33c5690060345b0aac25bc2426d675 SOURCES/docutils-0.12.tar.gz ecac1f50d82d5b381c8406f7140e6c6bb0dd5c86 SOURCES/openvswitch-3.1.0.tar.gz 8509a716f9f936526f64fb23f313c5a9baf2f123 SOURCES/pyelftools-0.27.tar.gz -d34f96421a86004aa5d26ecf975edefd09f948b1 SOURCES/Pygments-1.4.tar.gz -3a11f130c63b057532ca37fe49c8967d0cbae1d5 SOURCES/Sphinx-1.2.3.tar.gz a6b5bf45bb8f65c704310f9fe592081af9d94aab SOURCES/dpdk-22.11.1.tar.xz diff --git a/SOURCES/openvswitch-3.1.0.patch b/SOURCES/openvswitch-3.1.0.patch index 7a6ade6..500c665 100644 --- a/SOURCES/openvswitch-3.1.0.patch +++ b/SOURCES/openvswitch-3.1.0.patch @@ -1,6 +1,6 @@ diff --git a/.ci/dpdk-build.sh b/.ci/dpdk-build.sh new file mode 100755 -index 000000000..02dcefef6 +index 0000000000..02dcefef61 --- /dev/null +++ b/.ci/dpdk-build.sh @@ -0,0 +1,54 @@ @@ -60,7 +60,7 @@ index 000000000..02dcefef6 +build_dpdk $DPDK_VER diff --git a/.ci/dpdk-prepare.sh b/.ci/dpdk-prepare.sh new file mode 100755 -index 000000000..f7e6215dd +index 0000000000..4424f9eb97 --- /dev/null +++ b/.ci/dpdk-prepare.sh @@ -0,0 +1,11 @@ @@ -74,9 +74,9 @@ index 000000000..f7e6215dd +# https://github.com/pypa/pip/issues/10655 +pip3 install --disable-pip-version-check --user wheel +pip3 install --disable-pip-version-check --user pyelftools -+pip3 install --user 'meson==0.53.2' ++pip3 install --user 'meson>=1.4,<1.5' diff --git a/.ci/linux-build.sh b/.ci/linux-build.sh -index 10021fddb..8227a5748 100755 +index 10021fddb2..8227a57487 100755 --- a/.ci/linux-build.sh +++ b/.ci/linux-build.sh @@ -9,9 +9,7 @@ EXTRA_OPTS="--enable-Werror" @@ -174,7 +174,7 @@ index 10021fddb..8227a5748 100755 if [ "$CC" = "clang" ]; then diff --git a/.ci/linux-prepare.sh b/.ci/linux-prepare.sh -index f414a879c..c28b6819a 100755 +index f414a879c7..133e8d9673 100755 --- a/.ci/linux-prepare.sh +++ b/.ci/linux-prepare.sh @@ -23,8 +23,7 @@ cd .. @@ -183,12 +183,12 @@ index f414a879c..c28b6819a 100755 pip3 install --disable-pip-version-check --user \ - flake8 'hacking>=3.0' netaddr pyparsing sphinx setuptools pyelftools -pip3 install --user 'meson==0.53.2' -+ flake8 'hacking>=3.0' netaddr pyparsing sphinx setuptools ++ flake8 netaddr pyparsing sphinx setuptools # Install python test dependencies pip3 install -r python/test_requirements.txt diff --git a/.ci/osx-build.sh b/.ci/osx-build.sh -index 09df61826..b81744ec9 100755 +index 09df61826f..b81744ec9b 100755 --- a/.ci/osx-build.sh +++ b/.ci/osx-build.sh @@ -10,7 +10,7 @@ function configure_ovs() @@ -201,41 +201,47 @@ index 09df61826..b81744ec9 100755 if [ "$CC" = "clang" ]; then make CFLAGS="$CFLAGS -Wno-error=unused-command-line-argument" diff --git a/.cirrus.yml b/.cirrus.yml -index 952d96431..48931fa08 100644 +index 952d964315..8db385f002 100644 --- a/.cirrus.yml +++ b/.cirrus.yml -@@ -3,7 +3,7 @@ freebsd_build_task: +@@ -2,8 +2,8 @@ freebsd_build_task: + freebsd_instance: matrix: - image_family: freebsd-12-4-snap +- image_family: freebsd-12-4-snap - image_family: freebsd-13-1-snap -+ image_family: freebsd-13-2-snap ++ image_family: freebsd-13-3-snap ++ image_family: freebsd-14-0-snap cpu: 4 memory: 4G diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml -index 82675b973..80c449336 100644 +index 82675b9734..22dc81252d 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml -@@ -3,12 +3,80 @@ name: Build and Test +@@ -2,13 +2,84 @@ name: Build and Test + on: [push, pull_request] ++env: ++ python_default: 3.12 ++ jobs: + build-dpdk: + env: + dependencies: gcc libnuma-dev ninja-build + CC: gcc + DPDK_GIT: https://dpdk.org/git/dpdk-stable -+ DPDK_VER: 22.11.1 ++ DPDK_VER: 22.11.4 + name: dpdk gcc + outputs: + dpdk_key: ${{ steps.gen_dpdk_key.outputs.key }} -+ runs-on: ubuntu-20.04 ++ runs-on: ubuntu-22.04 + timeout-minutes: 30 + + steps: + - name: checkout -+ uses: actions/checkout@v3 ++ uses: actions/checkout@v4 + + - name: update PATH + run: | @@ -263,16 +269,16 @@ index 82675b973..80c449336 100644 + + - name: cache + id: dpdk_cache -+ uses: actions/cache@v3 ++ uses: actions/cache@v4 + with: + path: dpdk-dir + key: ${{ steps.gen_dpdk_key.outputs.key }} + + - name: set up python + if: steps.dpdk_cache.outputs.cache-hit != 'true' -+ uses: actions/setup-python@v4 ++ uses: actions/setup-python@v5 + with: -+ python-version: '3.9' ++ python-version: ${{ env.python_default }} + + - name: update APT cache + if: steps.dpdk_cache.outputs.cache-hit != 'true' @@ -301,7 +307,7 @@ index 82675b973..80c449336 100644 ASAN: ${{ matrix.asan }} UBSAN: ${{ matrix.ubsan }} CC: ${{ matrix.compiler }} -@@ -17,6 +85,7 @@ jobs: +@@ -17,10 +88,11 @@ jobs: LIBS: ${{ matrix.libs }} M32: ${{ matrix.m32 }} OPTS: ${{ matrix.opts }} @@ -309,7 +315,12 @@ index 82675b973..80c449336 100644 TESTSUITE: ${{ matrix.testsuite }} name: linux ${{ join(matrix.*, ' ') }} -@@ -32,6 +101,11 @@ jobs: +- runs-on: ubuntu-20.04 ++ runs-on: ubuntu-22.04 + timeout-minutes: 30 + + strategy: +@@ -32,6 +104,11 @@ jobs: - compiler: clang opts: --disable-ssl @@ -321,10 +332,24 @@ index 82675b973..80c449336 100644 - compiler: gcc testsuite: test - compiler: clang -@@ -104,25 +178,12 @@ jobs: - with: - python-version: '3.9' +@@ -92,7 +169,7 @@ jobs: + + steps: + - name: checkout +- uses: actions/checkout@v3 ++ uses: actions/checkout@v4 + - name: update PATH + run: | +@@ -100,29 +177,16 @@ jobs: + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: set up python +- uses: actions/setup-python@v4 ++ uses: actions/setup-python@v5 + with: +- python-version: '3.9' +- - - name: create ci signature file for the dpdk cache key - if: matrix.dpdk != '' || matrix.dpdk_shared != '' - # This will collect most of DPDK related lines, so hash will be different @@ -334,13 +359,15 @@ index 82675b973..80c449336 100644 - run: | - grep -irE 'RTE_|DPDK|meson|ninja' -r .ci/ > dpdk-ci-signature - cat dpdk-ci-signature -- ++ python-version: ${{ env.python_default }} + - name: cache if: matrix.dpdk != '' || matrix.dpdk_shared != '' - uses: actions/cache@v3 +- uses: actions/cache@v3 - env: - matrix_key: ${{ matrix.dpdk }}${{ matrix.dpdk_shared }} - ci_key: ${{ hashFiles('dpdk-ci-signature') }} ++ uses: actions/cache@v4 with: path: dpdk-dir - key: ${{ env.matrix_key }}-${{ env.ci_key }} @@ -348,8 +375,161 @@ index 82675b973..80c449336 100644 - name: update APT cache run: sudo apt update || true +@@ -136,6 +200,14 @@ jobs: + if: matrix.m32 != '' + run: sudo apt install -y gcc-multilib + ++ - name: Reduce ASLR entropy ++ if: matrix.asan != '' || matrix.ubsan != '' ++ # Asan in llvm 14 provided in ubuntu-22.04 is incompatible with ++ # high-entropy ASLR configured in much newer kernels that GitHub ++ # runners are using leading to random crashes: ++ # https://github.com/actions/runner-images/issues/9491 ++ run: sudo sysctl -w vm.mmap_rnd_bits=28 ++ + - name: prepare + run: ./.ci/linux-prepare.sh + +@@ -145,9 +217,9 @@ jobs: + - name: copy logs on failure + if: failure() || cancelled() + run: | +- # upload-artifact@v2 throws exceptions if it tries to upload socket ++ # upload-artifact throws exceptions if it tries to upload socket + # files and we could have some socket files in testsuite.dir. +- # Also, upload-artifact@v2 doesn't work well enough with wildcards. ++ # Also, upload-artifact doesn't work well enough with wildcards. + # So, we're just archiving everything here to avoid any issues. + mkdir logs + cp config.log ./logs/ +@@ -156,7 +228,7 @@ jobs: + + - name: upload logs on failure + if: failure() || cancelled() +- uses: actions/upload-artifact@v3 ++ uses: actions/upload-artifact@v4 + with: + name: logs-linux-${{ join(matrix.*, '-') }} + path: logs.tgz +@@ -175,15 +247,15 @@ jobs: + + steps: + - name: checkout +- uses: actions/checkout@v3 ++ uses: actions/checkout@v4 + - name: update PATH + run: | + echo "$HOME/bin" >> $GITHUB_PATH + echo "$HOME/.local/bin" >> $GITHUB_PATH + - name: set up python +- uses: actions/setup-python@v4 ++ uses: actions/setup-python@v5 + with: +- python-version: '3.9' ++ python-version: ${{ env.python_default }} + - name: install dependencies + run: brew install automake libtool + - name: prepare +@@ -192,7 +264,7 @@ jobs: + run: ./.ci/osx-build.sh + - name: upload logs on failure + if: failure() +- uses: actions/upload-artifact@v3 ++ uses: actions/upload-artifact@v4 + with: + name: logs-osx-clang---disable-ssl + path: config.log +@@ -216,7 +288,7 @@ jobs: + + steps: + - name: checkout +- uses: actions/checkout@v3 ++ uses: actions/checkout@v4 + + - name: update PATH + run: | +@@ -238,7 +310,7 @@ jobs: + run: ./.ci/linux-build.sh + + - name: upload deb packages +- uses: actions/upload-artifact@v3 ++ uses: actions/upload-artifact@v4 + with: + name: deb-packages-${{ matrix.dpdk }}-dpdk + path: '/home/runner/work/ovs/*.deb' +diff --git a/AUTHORS.rst b/AUTHORS.rst +index 2df76c56f1..b29b1f4051 100644 +--- a/AUTHORS.rst ++++ b/AUTHORS.rst +@@ -561,6 +561,7 @@ David Evans davidjoshuaevans@gmail.com + David Palma palma@onesource.pt + David van Moolenbroek dvmoolenbroek@aimvalley.nl + Derek Cormier derek.cormier@lab.ntt.co.jp ++Derrick Lim derrick.lim@rakuten.com + Dhaval Badiani dbadiani@vmware.com + DK Moon + Ding Zhi zhi.ding@6wind.com +diff --git a/Documentation/faq/releases.rst b/Documentation/faq/releases.rst +index 9e1b422620..a393f78a89 100644 +--- a/Documentation/faq/releases.rst ++++ b/Documentation/faq/releases.rst +@@ -214,9 +214,9 @@ Q: What DPDK version does each Open vSwitch release work with? + 2.14.x 19.11.13 + 2.15.x 20.11.6 + 2.16.x 20.11.6 +- 2.17.x 21.11.2 +- 3.0.x 21.11.2 +- 3.1.x 22.11.1 ++ 2.17.x 21.11.6 ++ 3.0.x 21.11.6 ++ 3.1.x 22.11.4 + ============ ======== + + Q: Are all the DPDK releases that OVS versions work with maintained? +diff --git a/Documentation/intro/install/dpdk.rst b/Documentation/intro/install/dpdk.rst +index 63a0ebb23b..27df48493e 100644 +--- a/Documentation/intro/install/dpdk.rst ++++ b/Documentation/intro/install/dpdk.rst +@@ -42,7 +42,7 @@ Build requirements + In addition to the requirements described in :doc:`general`, building Open + vSwitch with DPDK will require the following: + +-- DPDK 22.11.1 ++- DPDK 22.11.4 + + - A `DPDK supported NIC`_ + +@@ -73,9 +73,9 @@ Install DPDK + #. Download the `DPDK sources`_, extract the file and set ``DPDK_DIR``:: + + $ cd /usr/src/ +- $ wget https://fast.dpdk.org/rel/dpdk-22.11.1.tar.xz +- $ tar xf dpdk-22.11.1.tar.xz +- $ export DPDK_DIR=/usr/src/dpdk-stable-22.11.1 ++ $ wget https://fast.dpdk.org/rel/dpdk-22.11.4.tar.xz ++ $ tar xf dpdk-22.11.4.tar.xz ++ $ export DPDK_DIR=/usr/src/dpdk-stable-22.11.4 + $ cd $DPDK_DIR + + #. Configure and install DPDK using Meson +diff --git a/Documentation/intro/install/general.rst b/Documentation/intro/install/general.rst +index 42b5682fd8..0dd93f722c 100644 +--- a/Documentation/intro/install/general.rst ++++ b/Documentation/intro/install/general.rst +@@ -176,10 +176,7 @@ following to obtain better warnings: + + - clang, version 3.4 or later + +-- flake8 along with the hacking flake8 plugin (for Python code). The automatic +- flake8 check that runs against Python code has some warnings enabled that +- come from the "hacking" flake8 plugin. If it's not installed, the warnings +- just won't occur until it's run on a system with "hacking" installed. ++- flake8 (for Python code) + + - the python packages listed in "python/test_requirements.txt" (compatible + with pip). If they are installed, the pytest-based Python unit tests will diff --git a/Documentation/ref/ovs-actions.7.rst b/Documentation/ref/ovs-actions.7.rst -index b59b7634f..36adcc5db 100644 +index b59b7634fa..36adcc5db2 100644 --- a/Documentation/ref/ovs-actions.7.rst +++ b/Documentation/ref/ovs-actions.7.rst @@ -694,7 +694,8 @@ encapsulated in an OpenFlow ``packet-in`` message. The supported options are: @@ -385,7 +565,7 @@ index b59b7634f..36adcc5db 100644 This action was added in Open vSwitch 2.14. diff --git a/Makefile.am b/Makefile.am -index e605187b8..eddb981ae 100644 +index e605187b81..f2e0650b88 100644 --- a/Makefile.am +++ b/Makefile.am @@ -75,6 +75,8 @@ EXTRA_DIST = \ @@ -406,7 +586,23 @@ index e605187b8..eddb981ae 100644 if grep warning: $@.tmp; then error=:; fi; \ rm -f $@.tmp; \ done; \ -@@ -412,7 +414,7 @@ endif +@@ -396,23 +398,17 @@ ALL_LOCAL += flake8-check + # F811 redefinition of unused from line (only from flake8 v2.0) + # D*** -- warnings from flake8-docstrings plugin + # H*** -- warnings from flake8 hacking plugin (custom style checks beyond PEP8) +-# H231 Python 3.x incompatible 'except x,y:' construct +-# H232 Python 3.x incompatible octal 077 should be written as 0o77 +-# H233 Python 3.x incompatible use of print operator +-# H238 old style class declaration, use new style (inherit from `object`) +-FLAKE8_SELECT = H231,H232,H233,H238 + FLAKE8_IGNORE = E121,E123,E125,E126,E127,E128,E129,E131,E203,E722,W503,W504,F811,D,H,I + flake8-check: $(FLAKE8_PYFILES) + $(FLAKE8_WERROR)$(AM_V_GEN) \ + src='$^' && \ +- flake8 $$src --select=$(FLAKE8_SELECT) $(FLAKE8_FLAGS) && \ + flake8 $$src --ignore=$(FLAKE8_IGNORE) $(FLAKE8_FLAGS) && \ + touch $@ + endif CLEANFILES += flake8-check -include manpages.mk @@ -416,12 +612,20 @@ index e605187b8..eddb981ae 100644 @if cmp -s $(@F).tmp $@; then \ touch $@; \ diff --git a/NEWS b/NEWS -index 37a01dea5..13229cb3a 100644 +index 37a01dea50..aaddb77e8e 100644 --- a/NEWS +++ b/NEWS -@@ -1,3 +1,20 @@ -+v3.1.4 - xx xxx xxxx +@@ -1,3 +1,28 @@ ++v3.1.5 - xx xxx xxxx ++-------------------- ++ ++v3.1.4 - 08 Feb 2024 +-------------------- ++ - Bug fixes ++ - Security: ++ * Fixed vulnerabilities CVE-2023-3966 and CVE-2023-5366. ++ - DPDK: ++ * OVS validated with DPDK 22.11.4. + +v3.1.3 - 17 Oct 2023 +-------------------- @@ -441,7 +645,7 @@ index 37a01dea5..13229cb3a 100644 -------------------- - ovs-vswitchd now detects changes in CPU affinity and adjusts the number diff --git a/build-aux/automake.mk b/build-aux/automake.mk -index b9a77a51c..d65b6da6c 100644 +index b9a77a51cf..d65b6da6c5 100644 --- a/build-aux/automake.mk +++ b/build-aux/automake.mk @@ -1,11 +1,19 @@ @@ -484,7 +688,7 @@ index b9a77a51c..d65b6da6c 100644 + build-aux/soexpand.py \ + build-aux/xml2nroff diff --git a/build-aux/extract-ofp-actions b/build-aux/extract-ofp-actions -index 0aa6c65f3..cc5c1dbb0 100755 +index 0aa6c65f31..cc5c1dbb06 100755 --- a/build-aux/extract-ofp-actions +++ b/build-aux/extract-ofp-actions @@ -17,27 +17,30 @@ version_map = {"1.0": 0x01, @@ -731,7 +935,7 @@ index 0aa6c65f3..cc5c1dbb0 100755 if __name__ == '__main__': argv0 = sys.argv[0] diff --git a/build-aux/extract-ofp-errors b/build-aux/extract-ofp-errors -index 2c3fbfc88..eeefccbee 100755 +index 2c3fbfc881..eeefccbee0 100755 --- a/build-aux/extract-ofp-errors +++ b/build-aux/extract-ofp-errors @@ -22,6 +22,9 @@ tokenRe = "#?" + idRe + "|[0-9]+|." @@ -1045,7 +1249,7 @@ index 2c3fbfc88..eeefccbee 100755 if '--help' in sys.argv: usage() diff --git a/build-aux/extract-ofp-fields b/build-aux/extract-ofp-fields -index efec59c25..89d80c208 100755 +index efec59c25b..89d80c2085 100755 --- a/build-aux/extract-ofp-fields +++ b/build-aux/extract-ofp-fields @@ -4,9 +4,9 @@ import getopt @@ -1226,7 +1430,7 @@ index efec59c25..89d80c208 100755 if __name__ == "__main__": argv0 = sys.argv[0] diff --git a/build-aux/extract-ofp-msgs b/build-aux/extract-ofp-msgs -index 6b3295cf6..c26ea1d35 100755 +index 6b3295cf64..c26ea1d355 100755 --- a/build-aux/extract-ofp-msgs +++ b/build-aux/extract-ofp-msgs @@ -24,6 +24,9 @@ OFPT11_STATS_REQUEST = 18 @@ -1332,7 +1536,7 @@ index 6b3295cf6..c26ea1d35 100755 print(line) - diff --git a/build-aux/gen_ofp_field_decoders b/build-aux/gen_ofp_field_decoders -index 0b797ee8c..0cb6108c2 100755 +index 0b797ee8c8..0cb6108c22 100755 --- a/build-aux/gen_ofp_field_decoders +++ b/build-aux/gen_ofp_field_decoders @@ -2,7 +2,7 @@ @@ -1354,7 +1558,7 @@ index 0b797ee8c..0cb6108c2 100755 field_decoders = {} aliases = {} diff --git a/build-aux/sodepends.py b/build-aux/sodepends.py -index 45812bcbd..ac8dd61a4 100755 +index 45812bcbd7..ac8dd61a4b 100755 --- a/build-aux/sodepends.py +++ b/build-aux/sodepends.py @@ -14,9 +14,10 @@ @@ -1370,7 +1574,7 @@ index 45812bcbd..ac8dd61a4 100755 def sodepends(include_dirs, filenames, dst): ok = True diff --git a/build-aux/soexpand.py b/build-aux/soexpand.py -index 00adcf47a..7d4dc0486 100755 +index 00adcf47a3..7d4dc0486a 100755 --- a/build-aux/soexpand.py +++ b/build-aux/soexpand.py @@ -14,9 +14,10 @@ @@ -1386,7 +1590,7 @@ index 00adcf47a..7d4dc0486 100755 def soexpand(include_dirs, src, dst): ok = True diff --git a/build-aux/xml2nroff b/build-aux/xml2nroff -index ee5553f45..3e937910b 100755 +index ee5553f456..3e937910be 100755 --- a/build-aux/xml2nroff +++ b/build-aux/xml2nroff @@ -18,7 +18,7 @@ import getopt @@ -1422,7 +1626,7 @@ index ee5553f45..3e937910b 100755 sys.exit(1) for line in s.splitlines(): diff --git a/configure.ac b/configure.ac -index 9bf896c01..cd50adcb2 100644 +index 9bf896c013..c30e273c57 100644 --- a/configure.ac +++ b/configure.ac @@ -13,7 +13,7 @@ @@ -1430,7 +1634,7 @@ index 9bf896c01..cd50adcb2 100644 AC_PREREQ(2.63) -AC_INIT(openvswitch, 3.1.0, bugs@openvswitch.org) -+AC_INIT(openvswitch, 3.1.4, bugs@openvswitch.org) ++AC_INIT(openvswitch, 3.1.5, bugs@openvswitch.org) AC_CONFIG_SRCDIR([vswitchd/ovs-vswitchd.c]) AC_CONFIG_MACRO_DIR([m4]) AC_CONFIG_AUX_DIR([build-aux]) @@ -1448,7 +1652,7 @@ index 9bf896c01..cd50adcb2 100644 AC_PROG_CXX AC_PROG_CPP diff --git a/datapath-windows/include/automake.mk b/datapath-windows/include/automake.mk -index a354f007f..185a06b03 100644 +index a354f007fd..185a06b03e 100644 --- a/datapath-windows/include/automake.mk +++ b/datapath-windows/include/automake.mk @@ -7,6 +7,4 @@ $(srcdir)/datapath-windows/include/OvsDpInterface.h: \ @@ -1459,15 +1663,21 @@ index a354f007f..185a06b03 100644 - CLEANFILES += $(srcdir)/datapath-windows/include/OvsDpInterface.h diff --git a/debian/changelog b/debian/changelog -index a5ad222c4..a47e46242 100644 +index a5ad222c4c..232c3d2660 100644 --- a/debian/changelog +++ b/debian/changelog -@@ -1,3 +1,27 @@ +@@ -1,3 +1,33 @@ ++openvswitch (3.1.5-1) unstable; urgency=low ++ [ Open vSwitch team ] ++ * New upstream version ++ ++ -- Open vSwitch team Thu, 08 Feb 2024 17:55:19 +0100 ++ +openvswitch (3.1.4-1) unstable; urgency=low + [ Open vSwitch team ] + * New upstream version + -+ -- Open vSwitch team Tue, 17 Oct 2023 13:02:17 +0200 ++ -- Open vSwitch team Thu, 08 Feb 2024 17:55:19 +0100 + +openvswitch (3.1.3-1) unstable; urgency=low + [ Open vSwitch team ] @@ -1491,7 +1701,7 @@ index a5ad222c4..a47e46242 100644 * New upstream version diff --git a/include/automake.mk b/include/automake.mk -index 1e3390ae0..a276c680b 100644 +index 1e3390ae0d..a276c680b5 100644 --- a/include/automake.mk +++ b/include/automake.mk @@ -8,7 +8,6 @@ include/odp-netlink-macros.h: include/odp-netlink.h \ @@ -1503,7 +1713,7 @@ index 1e3390ae0..a276c680b 100644 include include/openflow/automake.mk diff --git a/include/openflow/automake.mk b/include/openflow/automake.mk -index a1d75756c..820c09f84 100644 +index a1d75756c9..820c09f84b 100644 --- a/include/openflow/automake.mk +++ b/include/openflow/automake.mk @@ -22,6 +22,3 @@ HSTAMP_FILES = $(openflowinclude_HEADERS:.h=.hstamp) @@ -1514,7 +1724,7 @@ index a1d75756c..820c09f84 100644 -EXTRA_DIST += build-aux/check-structs - diff --git a/include/openvswitch/compiler.h b/include/openvswitch/compiler.h -index cf009f826..52614a5ac 100644 +index cf009f8264..52614a5ac0 100644 --- a/include/openvswitch/compiler.h +++ b/include/openvswitch/compiler.h @@ -37,6 +37,16 @@ @@ -1535,7 +1745,7 @@ index cf009f826..52614a5ac 100644 #define OVS_UNUSED __attribute__((__unused__)) #define OVS_PRINTF_FORMAT(FMT, ARG1) __attribute__((__format__(printf, FMT, ARG1))) diff --git a/include/openvswitch/meta-flow.h b/include/openvswitch/meta-flow.h -index 045dce8f5..3b0220aaa 100644 +index 045dce8f5f..3b0220aaa2 100644 --- a/include/openvswitch/meta-flow.h +++ b/include/openvswitch/meta-flow.h @@ -2366,6 +2366,10 @@ void mf_format_subvalue(const union mf_subvalue *subvalue, struct ds *s); @@ -1549,8 +1759,45 @@ index 045dce8f5..3b0220aaa 100644 #ifdef __cplusplus } #endif +diff --git a/ipsec/ovs-monitor-ipsec.in b/ipsec/ovs-monitor-ipsec.in +index 7945162f9f..bc7ac55237 100755 +--- a/ipsec/ovs-monitor-ipsec.in ++++ b/ipsec/ovs-monitor-ipsec.in +@@ -457,14 +457,30 @@ conn prevent_unencrypted_vxlan + CERTKEY_PREFIX = "ovs_certkey_" + + def __init__(self, libreswan_root_prefix, args): ++ # Collect version infromation ++ self.IPSEC = libreswan_root_prefix + "/usr/sbin/ipsec" ++ proc = subprocess.Popen([self.IPSEC, "--version"], ++ stdout=subprocess.PIPE, ++ encoding="latin1") ++ pout, perr = proc.communicate() ++ ++ v = re.match("^Libreswan (.*)$", pout) ++ try: ++ version = int(v.group(1).split(".")[0]) ++ except: ++ version = 0 ++ ++ if version >= 4: ++ ipsec_d = args.ipsec_d if args.ipsec_d else "/var/lib/ipsec/nss" ++ else: ++ ipsec_d = args.ipsec_d if args.ipsec_d else "/etc/ipsec.d" ++ + ipsec_conf = args.ipsec_conf if args.ipsec_conf else "/etc/ipsec.conf" +- ipsec_d = args.ipsec_d if args.ipsec_d else "/etc/ipsec.d" + ipsec_secrets = (args.ipsec_secrets if args.ipsec_secrets + else "/etc/ipsec.secrets") + ipsec_ctl = (args.ipsec_ctl if args.ipsec_ctl + else "/run/pluto/pluto.ctl") + +- self.IPSEC = libreswan_root_prefix + "/usr/sbin/ipsec" + self.IPSEC_CONF = libreswan_root_prefix + ipsec_conf + self.IPSEC_SECRETS = libreswan_root_prefix + ipsec_secrets + self.IPSEC_D = "sql:" + libreswan_root_prefix + ipsec_d diff --git a/lib/automake.mk b/lib/automake.mk -index e64ee76ce..1be13a420 100644 +index e64ee76ce7..1be13a420a 100644 --- a/lib/automake.mk +++ b/lib/automake.mk @@ -451,7 +451,7 @@ lib_libsflow_la_SOURCES = \ @@ -1593,8 +1840,91 @@ index e64ee76ce..1be13a420 100644 # _server IDL OVSIDL_BUILT += lib/ovsdb-server-idl.c lib/ovsdb-server-idl.h lib/ovsdb-server-idl.ovsidl +diff --git a/lib/backtrace.h b/lib/backtrace.h +index 5708bf9c68..1a5b879944 100644 +--- a/lib/backtrace.h ++++ b/lib/backtrace.h +@@ -26,7 +26,7 @@ + #endif + + /* log_backtrace() will save the backtrace of a running program +- * into the log at the DEBUG level. ++ * into the log at the ERROR level. + * + * To use it, insert the following code to where backtrace is + * desired: +diff --git a/lib/bfd.c b/lib/bfd.c +index 9698576d07..b8149e7897 100644 +--- a/lib/bfd.c ++++ b/lib/bfd.c +@@ -586,7 +586,6 @@ bfd_put_packet(struct bfd *bfd, struct dp_packet *p, + { + long long int min_tx, min_rx; + struct udp_header *udp; +- struct eth_header *eth; + struct ip_header *ip; + struct msg *msg; + +@@ -605,15 +604,13 @@ bfd_put_packet(struct bfd *bfd, struct dp_packet *p, + * set. */ + ovs_assert(!(bfd->flags & FLAG_POLL) || !(bfd->flags & FLAG_FINAL)); + +- dp_packet_reserve(p, 2); /* Properly align after the ethernet header. */ +- eth = dp_packet_put_uninit(p, sizeof *eth); +- eth->eth_src = eth_addr_is_zero(bfd->local_eth_src) +- ? eth_src : bfd->local_eth_src; +- eth->eth_dst = eth_addr_is_zero(bfd->local_eth_dst) +- ? eth_addr_bfd : bfd->local_eth_dst; +- eth->eth_type = htons(ETH_TYPE_IP); ++ ip = eth_compose(p, ++ eth_addr_is_zero(bfd->local_eth_dst) ++ ? eth_addr_bfd : bfd->local_eth_dst, ++ eth_addr_is_zero(bfd->local_eth_src) ++ ? eth_src : bfd->local_eth_src, ++ ETH_TYPE_IP, sizeof *ip + sizeof *udp + sizeof *msg); + +- ip = dp_packet_put_zeros(p, sizeof *ip); + ip->ip_ihl_ver = IP_IHL_VER(5, 4); + ip->ip_tot_len = htons(sizeof *ip + sizeof *udp + sizeof *msg); + ip->ip_ttl = MAXTTL; +@@ -621,15 +618,17 @@ bfd_put_packet(struct bfd *bfd, struct dp_packet *p, + ip->ip_proto = IPPROTO_UDP; + put_16aligned_be32(&ip->ip_src, bfd->ip_src); + put_16aligned_be32(&ip->ip_dst, bfd->ip_dst); +- /* Checksum has already been zeroed by put_zeros call. */ ++ /* Checksum has already been zeroed by eth_compose call. */ + ip->ip_csum = csum(ip, sizeof *ip); ++ dp_packet_set_l4(p, ip + 1); + +- udp = dp_packet_put_zeros(p, sizeof *udp); ++ udp = dp_packet_l4(p); + udp->udp_src = htons(bfd->udp_src); + udp->udp_dst = htons(BFD_DEST_PORT); + udp->udp_len = htons(sizeof *udp + sizeof *msg); ++ /* Checksum already zero from eth_compose. */ + +- msg = dp_packet_put_uninit(p, sizeof *msg); ++ msg = (struct msg *)(udp + 1); + msg->vers_diag = (BFD_VERSION << 5) | bfd->diag; + msg->flags = (bfd->state & STATE_MASK) | bfd->flags; + +@@ -1131,10 +1130,11 @@ bfd_set_state(struct bfd *bfd, enum state state, enum diag diag) + if (!VLOG_DROP_INFO(&rl)) { + struct ds ds = DS_EMPTY_INITIALIZER; + +- ds_put_format(&ds, "%s: BFD state change: %s->%s" +- " \"%s\"->\"%s\".\n", ++ ds_put_format(&ds, "%s: BFD state change: (bfd.SessionState: %s," ++ " bfd.LocalDiag: \"%s\") -> (bfd.SessionState: %s," ++ " bfd.LocalDiag: \"%s\")\n", + bfd->name, bfd_state_str(bfd->state), +- bfd_state_str(state), bfd_diag_str(bfd->diag), ++ bfd_diag_str(bfd->diag), bfd_state_str(state), + bfd_diag_str(diag)); + bfd_put_details(&ds, bfd); + VLOG_INFO("%s", ds_cstr(&ds)); diff --git a/lib/classifier.c b/lib/classifier.c -index 0a89626cc..18dbfc83a 100644 +index 0a89626cc3..18dbfc83ad 100644 --- a/lib/classifier.c +++ b/lib/classifier.c @@ -1695,6 +1695,8 @@ find_match_wc(const struct cls_subtable *subtable, ovs_version_t version, @@ -1654,7 +1984,7 @@ index 0a89626cc..18dbfc83a 100644 } diff --git a/lib/conntrack-private.h b/lib/conntrack-private.h -index fae8b3a9b..ba6e75dee 100644 +index fae8b3a9ba..ba6e75dee1 100644 --- a/lib/conntrack-private.h +++ b/lib/conntrack-private.h @@ -49,6 +49,12 @@ struct ct_endpoint { @@ -1705,7 +2035,7 @@ index fae8b3a9b..ba6e75dee 100644 uint32_t tp_id; /* Timeout policy ID. */ }; diff --git a/lib/conntrack-tp.c b/lib/conntrack-tp.c -index 89cb2704a..2149fdc73 100644 +index 89cb2704a6..2149fdc73a 100644 --- a/lib/conntrack-tp.c +++ b/lib/conntrack-tp.c @@ -253,7 +253,8 @@ conn_update_expiration(struct conntrack *ct, struct conn *conn, @@ -1729,7 +2059,7 @@ index 89cb2704a..2149fdc73 100644 conn->expiration = now + val * 1000; } diff --git a/lib/conntrack-tp.h b/lib/conntrack-tp.h -index 4d411d19f..7ece2eae2 100644 +index 4d411d19fd..7ece2eae2f 100644 --- a/lib/conntrack-tp.h +++ b/lib/conntrack-tp.h @@ -17,8 +17,15 @@ @@ -1749,7 +2079,7 @@ index 4d411d19f..7ece2eae2 100644 int timeout_policy_update(struct conntrack *ct, struct timeout_policy *tp); int timeout_policy_delete(struct conntrack *ct, uint32_t tp_id); diff --git a/lib/conntrack.c b/lib/conntrack.c -index 524670e45..2470c1689 100644 +index 524670e45d..d25f501582 100644 --- a/lib/conntrack.c +++ b/lib/conntrack.c @@ -101,7 +101,7 @@ static enum ct_update_res conn_update(struct conntrack *ct, struct conn *conn, @@ -2323,7 +2653,18 @@ index 524670e45..2470c1689 100644 delete_conn__(conn); } -@@ -2569,15 +2482,18 @@ static void +@@ -2556,7 +2469,9 @@ tuple_to_conn_key(const struct ct_dpif_tuple *tuple, uint16_t zone, + key->src.icmp_type = tuple->icmp_type; + key->src.icmp_code = tuple->icmp_code; + key->dst.icmp_id = tuple->icmp_id; +- key->dst.icmp_type = reverse_icmp_type(tuple->icmp_type); ++ key->dst.icmp_type = (tuple->ip_proto == IPPROTO_ICMP) ++ ? reverse_icmp_type(tuple->icmp_type) ++ : reverse_icmp6_type(tuple->icmp_type); + key->dst.icmp_code = tuple->icmp_code; + } else { + key->src.port = tuple->src_port; +@@ -2569,15 +2484,18 @@ static void conn_to_ct_dpif_entry(const struct conn *conn, struct ct_dpif_entry *entry, long long now) { @@ -2345,7 +2686,7 @@ index 524670e45..2470c1689 100644 ovs_mutex_lock(&conn->lock); entry->mark = conn->mark; -@@ -2585,7 +2501,7 @@ conn_to_ct_dpif_entry(const struct conn *conn, struct ct_dpif_entry *entry, +@@ -2585,7 +2503,7 @@ conn_to_ct_dpif_entry(const struct conn *conn, struct ct_dpif_entry *entry, long long expiration = conn_expiration(conn) - now; @@ -2354,19 +2695,35 @@ index 524670e45..2470c1689 100644 if (class->conn_get_protoinfo) { class->conn_get_protoinfo(conn, &entry->protoinfo); } -@@ -2633,15 +2549,20 @@ conntrack_dump_next(struct conntrack_dump *dump, struct ct_dpif_entry *entry) - if (!cm_node) { - break; - } -+ struct conn_key_node *keyn; - struct conn *conn; -- INIT_CONTAINER(conn, cm_node, cm_node); +@@ -2618,30 +2536,29 @@ conntrack_dump_start(struct conntrack *ct, struct conntrack_dump *dump, + + dump->ct = ct; + *ptot_bkts = 1; /* Need to clean up the callers. */ ++ dump->cursor = cmap_cursor_start(&ct->conns); + return 0; + } + + int + conntrack_dump_next(struct conntrack_dump *dump, struct ct_dpif_entry *entry) + { +- struct conntrack *ct = dump->ct; + long long now = time_msec(); -+ INIT_CONTAINER(keyn, cm_node, cm_node); +- for (;;) { +- struct cmap_node *cm_node = cmap_next_position(&ct->conns, +- &dump->cm_pos); +- if (!cm_node) { +- break; ++ struct conn_key_node *keyn; ++ struct conn *conn; ++ ++ CMAP_CURSOR_FOR_EACH_CONTINUE (keyn, cm_node, &dump->cursor) { + if (keyn->dir != CT_DIR_FWD) { + continue; -+ } -+ + } +- struct conn *conn; +- INIT_CONTAINER(conn, cm_node, cm_node); + + conn = CONTAINER_OF(keyn, struct conn, key_node[CT_DIR_FWD]); if (conn_expired(conn, now)) { continue; @@ -2378,7 +2735,7 @@ index 524670e45..2470c1689 100644 conn_to_ct_dpif_entry(conn, entry, now); return 0; } -@@ -2659,14 +2580,15 @@ conntrack_dump_done(struct conntrack_dump *dump OVS_UNUSED) +@@ -2659,14 +2576,15 @@ conntrack_dump_done(struct conntrack_dump *dump OVS_UNUSED) int conntrack_flush(struct conntrack *ct, const uint16_t *zone) { @@ -2398,7 +2755,7 @@ index 524670e45..2470c1689 100644 conn_clean(ct, conn); } } -@@ -2678,18 +2600,18 @@ int +@@ -2678,18 +2596,18 @@ int conntrack_flush_tuple(struct conntrack *ct, const struct ct_dpif_tuple *tuple, uint16_t zone) { @@ -2420,7 +2777,7 @@ index 524670e45..2470c1689 100644 error = ENOENT; } -@@ -2832,50 +2754,54 @@ expectation_create(struct conntrack *ct, ovs_be16 dst_port, +@@ -2832,50 +2750,54 @@ expectation_create(struct conntrack *ct, ovs_be16 dst_port, const struct conn *parent_conn, bool reply, bool src_ip_wc, bool skip_nat) { @@ -2487,7 +2844,7 @@ index 524670e45..2470c1689 100644 sizeof alg_exp_node->parent_key); /* Take the write lock here because it is almost 100% * likely that the lookup will fail and -@@ -3127,12 +3053,16 @@ process_ftp_ctl_v4(struct conntrack *ct, +@@ -3127,12 +3049,16 @@ process_ftp_ctl_v4(struct conntrack *ct, switch (mode) { case CT_FTP_MODE_ACTIVE: @@ -2508,7 +2865,7 @@ index 524670e45..2470c1689 100644 break; case CT_TFTP_MODE: default: -@@ -3164,7 +3094,7 @@ skip_ipv6_digits(char *str) +@@ -3164,7 +3090,7 @@ skip_ipv6_digits(char *str) static enum ftp_ctl_pkt process_ftp_ctl_v6(struct conntrack *ct, struct dp_packet *pkt, @@ -2517,7 +2874,7 @@ index 524670e45..2470c1689 100644 union ct_addr *v6_addr_rep, char **ftp_data_start, size_t *addr_offset_from_ftp_data_start, size_t *addr_size, enum ct_alg_mode *mode) -@@ -3232,24 +3162,25 @@ process_ftp_ctl_v6(struct conntrack *ct, +@@ -3232,24 +3158,25 @@ process_ftp_ctl_v6(struct conntrack *ct, switch (*mode) { case CT_FTP_MODE_ACTIVE: @@ -2547,7 +2904,7 @@ index 524670e45..2470c1689 100644 !!(pkt->md.ct_state & CS_REPLY_DIR), false, false); return CT_FTP_CTL_INTEREST; } -@@ -3403,7 +3334,8 @@ handle_tftp_ctl(struct conntrack *ct, +@@ -3403,7 +3330,8 @@ handle_tftp_ctl(struct conntrack *ct, long long now OVS_UNUSED, enum ftp_ctl_pkt ftp_ctl OVS_UNUSED, bool nat OVS_UNUSED) { @@ -2557,8 +2914,21 @@ index 524670e45..2470c1689 100644 conn_for_expectation, !!(pkt->md.ct_state & CS_REPLY_DIR), false, false); } +diff --git a/lib/conntrack.h b/lib/conntrack.h +index b064abc9fa..bed23a9515 100644 +--- a/lib/conntrack.h ++++ b/lib/conntrack.h +@@ -100,7 +100,7 @@ void conntrack_clear(struct dp_packet *packet); + struct conntrack_dump { + struct conntrack *ct; + unsigned bucket; +- struct cmap_position cm_pos; ++ struct cmap_cursor cursor; + bool filter_zone; + uint16_t zone; + }; diff --git a/lib/cpu.c b/lib/cpu.c -index 0292f715e..fbbea4005 100644 +index 0292f715ec..fbbea40053 100644 --- a/lib/cpu.c +++ b/lib/cpu.c @@ -37,7 +37,9 @@ static bool x86_has_isa(uint32_t leaf, enum x86_reg reg, uint32_t bit) @@ -2573,7 +2943,7 @@ index 0292f715e..fbbea4005 100644 __cpuid_count(leaf, 0, regs[EAX], regs[EBX], regs[ECX], regs[EDX]); return (regs[reg] & ((uint32_t) 1 << bit)) != 0; diff --git a/lib/db-ctl-base.c b/lib/db-ctl-base.c -index 134496ef3..3a8068b12 100644 +index 134496ef3f..3a8068b12c 100644 --- a/lib/db-ctl-base.c +++ b/lib/db-ctl-base.c @@ -820,6 +820,7 @@ check_condition(const struct ovsdb_idl_table_class *table, @@ -2666,7 +3036,7 @@ index 134496ef3..3a8068b12 100644 invalidate_cache(ctx); } diff --git a/lib/dpctl.c b/lib/dpctl.c -index d12d9b8a5..41b23d8ae 100644 +index d12d9b8a5e..41b23d8aeb 100644 --- a/lib/dpctl.c +++ b/lib/dpctl.c @@ -673,7 +673,7 @@ show_dpif(struct dpif *dpif, struct dpctl_params *dpctl_p) @@ -2726,7 +3096,7 @@ index d12d9b8a5..41b23d8ae 100644 free(argcopy); return 0; diff --git a/lib/dpif-netdev.c b/lib/dpif-netdev.c -index c9f7179c3..4c85ab4aa 100644 +index c9f7179c3b..4c85ab4aad 100644 --- a/lib/dpif-netdev.c +++ b/lib/dpif-netdev.c @@ -3365,14 +3365,13 @@ static inline void @@ -2836,7 +3206,7 @@ index c9f7179c3..4c85ab4aa 100644 dpif_netdev_enumerate, dpif_netdev_port_open_type, diff --git a/lib/dpif-netlink.c b/lib/dpif-netlink.c -index 026b0daa8..dc56e863f 100644 +index 026b0daa8d..dc56e863fd 100644 --- a/lib/dpif-netlink.c +++ b/lib/dpif-netlink.c @@ -395,7 +395,7 @@ dpif_netlink_open(const struct dpif_class *class OVS_UNUSED, const char *name, @@ -2866,7 +3236,7 @@ index 026b0daa8..dc56e863f 100644 dpif_netlink_enumerate, NULL, diff --git a/lib/dpif-provider.h b/lib/dpif-provider.h -index 12477a24f..b8ead8a02 100644 +index 12477a24fe..b8ead8a02a 100644 --- a/lib/dpif-provider.h +++ b/lib/dpif-provider.h @@ -127,6 +127,14 @@ struct dpif_class { @@ -2885,7 +3255,7 @@ index 12477a24f..b8ead8a02 100644 * startup. Returning an error from this function will prevent any * datapath with this class from being created. diff --git a/lib/dpif.c b/lib/dpif.c -index fe4db83fb..4397aeaf4 100644 +index fe4db83fbf..4397aeaf4f 100644 --- a/lib/dpif.c +++ b/lib/dpif.c @@ -701,13 +701,14 @@ dpif_port_set_config(struct dpif *dpif, odp_port_t port_no, @@ -2941,7 +3311,7 @@ index fe4db83fb..4397aeaf4 100644 + return dpif->dpif_class->synced_dp_layers; +} diff --git a/lib/dpif.h b/lib/dpif.h -index 6cb4dae6d..2a3649720 100644 +index 6cb4dae6d8..2a36497209 100644 --- a/lib/dpif.h +++ b/lib/dpif.h @@ -461,7 +461,7 @@ void dpif_port_clone(struct dpif_port *, const struct dpif_port *); @@ -2962,7 +3332,7 @@ index 6cb4dae6d..2a3649720 100644 /* Log functions. */ struct vlog_module; diff --git a/lib/fatal-signal.c b/lib/fatal-signal.c -index bbb31ef27..7054f8e74 100644 +index bbb31ef275..7054f8e745 100644 --- a/lib/fatal-signal.c +++ b/lib/fatal-signal.c @@ -78,6 +78,39 @@ static void call_hooks(int sig_nr); @@ -3044,8 +3414,36 @@ index bbb31ef27..7054f8e74 100644 for (i = 0; i < n_hooks; i++) { struct hook *h = &hooks[i]; if (h->cancel_cb) { +diff --git a/lib/jsonrpc.c b/lib/jsonrpc.c +index c8ce5362e1..3db5f76e28 100644 +--- a/lib/jsonrpc.c ++++ b/lib/jsonrpc.c +@@ -221,19 +221,19 @@ jsonrpc_log_msg(const struct jsonrpc *rpc, const char *title, + } + if (msg->params) { + ds_put_cstr(&s, ", params="); +- json_to_ds(msg->params, 0, &s); ++ json_to_ds(msg->params, JSSF_SORT, &s); + } + if (msg->result) { + ds_put_cstr(&s, ", result="); +- json_to_ds(msg->result, 0, &s); ++ json_to_ds(msg->result, JSSF_SORT, &s); + } + if (msg->error) { + ds_put_cstr(&s, ", error="); +- json_to_ds(msg->error, 0, &s); ++ json_to_ds(msg->error, JSSF_SORT, &s); + } + if (msg->id) { + ds_put_cstr(&s, ", id="); +- json_to_ds(msg->id, 0, &s); ++ json_to_ds(msg->id, JSSF_SORT, &s); + } + VLOG_DBG("%s: %s %s%s", rpc->name, title, + jsonrpc_msg_type_to_string(msg->type), ds_cstr(&s)); diff --git a/lib/learning-switch.c b/lib/learning-switch.c -index 8102475ca..cdf42935c 100644 +index 8102475cae..cdf42935c1 100644 --- a/lib/learning-switch.c +++ b/lib/learning-switch.c @@ -569,6 +569,7 @@ process_packet_in(struct lswitch *sw, const struct ofp_header *oh) @@ -3057,7 +3455,7 @@ index 8102475ca..cdf42935c 100644 if (buffer_id == UINT32_MAX) { po.packet = dp_packet_data(&pkt); diff --git a/lib/mcast-snooping.c b/lib/mcast-snooping.c -index 029ca2855..43805ae4d 100644 +index 029ca28558..43805ae4d5 100644 --- a/lib/mcast-snooping.c +++ b/lib/mcast-snooping.c @@ -946,8 +946,9 @@ mcast_snooping_wait(struct mcast_snooping *ms) @@ -3092,7 +3490,7 @@ index 029ca2855..43805ae4d 100644 ovs_rwlock_unlock(&ms->rwlock); } diff --git a/lib/meta-flow.c b/lib/meta-flow.c -index c576ae620..474344194 100644 +index c576ae6202..474344194f 100644 --- a/lib/meta-flow.c +++ b/lib/meta-flow.c @@ -3676,3 +3676,28 @@ mf_bitmap_not(struct mf_bitmap x) @@ -3125,7 +3523,7 @@ index c576ae620..474344194 100644 + } +} diff --git a/lib/meta-flow.xml b/lib/meta-flow.xml -index a1a20366d..ac72a44bc 100644 +index a1a20366d4..ac72a44bce 100644 --- a/lib/meta-flow.xml +++ b/lib/meta-flow.xml @@ -3517,23 +3517,24 @@ actions=clone(load:0->NXM_OF_IN_PORT[],output:123) @@ -3180,7 +3578,7 @@ index a1a20366d..ac72a44bc 100644 diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c -index ab5b8223e..ab4a4e9d8 100644 +index ab5b8223ef..da6829553c 100644 --- a/lib/netdev-dpdk.c +++ b/lib/netdev-dpdk.c @@ -489,6 +489,9 @@ struct netdev_dpdk { @@ -3276,8 +3674,252 @@ index ab5b8223e..ab4a4e9d8 100644 dev->flags = NETDEV_UP | NETDEV_PROMISC; ovs_list_push_back(&dpdk_list, &dev->list_node); +@@ -2019,17 +2075,16 @@ netdev_dpdk_set_config(struct netdev *netdev, const struct smap *args, + struct eth_addr mac; + + if (!dpdk_port_is_representor(dev)) { +- VLOG_WARN_BUF(errp, "'%s' is trying to set the VF MAC '%s' " +- "but 'options:dpdk-vf-mac' is only supported for " +- "VF representors.", +- netdev_get_name(netdev), vf_mac); ++ VLOG_WARN("'%s' is trying to set the VF MAC '%s' " ++ "but 'options:dpdk-vf-mac' is only supported for " ++ "VF representors.", ++ netdev_get_name(netdev), vf_mac); + } else if (!eth_addr_from_string(vf_mac, &mac)) { +- VLOG_WARN_BUF(errp, "interface '%s': cannot parse VF MAC '%s'.", +- netdev_get_name(netdev), vf_mac); ++ VLOG_WARN("interface '%s': cannot parse VF MAC '%s'.", ++ netdev_get_name(netdev), vf_mac); + } else if (eth_addr_is_multicast(mac)) { +- VLOG_WARN_BUF(errp, +- "interface '%s': cannot set VF MAC to multicast " +- "address '%s'.", netdev_get_name(netdev), vf_mac); ++ VLOG_WARN("interface '%s': cannot set VF MAC to multicast " ++ "address '%s'.", netdev_get_name(netdev), vf_mac); + } else if (!eth_addr_equals(dev->requested_hwaddr, mac)) { + dev->requested_hwaddr = mac; + netdev_request_reconfigure(netdev); +diff --git a/lib/netdev-dummy.c b/lib/netdev-dummy.c +index 72cb954711..1c61535068 100644 +--- a/lib/netdev-dummy.c ++++ b/lib/netdev-dummy.c +@@ -39,6 +39,7 @@ + #include "pcap-file.h" + #include "openvswitch/poll-loop.h" + #include "openvswitch/shash.h" ++#include "ovs-router.h" + #include "sset.h" + #include "stream.h" + #include "unaligned.h" +@@ -136,8 +137,7 @@ struct netdev_dummy { + + struct pcap_file *tx_pcap, *rxq_pcap OVS_GUARDED; + +- struct in_addr address, netmask; +- struct in6_addr ipv6, ipv6_mask; ++ struct ovs_list addrs OVS_GUARDED; + struct ovs_list rxes OVS_GUARDED; /* List of child "netdev_rxq_dummy"s. */ + + struct hmap offloaded_flows OVS_GUARDED; +@@ -161,6 +161,12 @@ struct netdev_rxq_dummy { + struct seq *seq; /* Reports newly queued packets. */ + }; + ++struct netdev_addr_dummy { ++ struct in6_addr address; ++ struct in6_addr netmask; ++ struct ovs_list node; /* In netdev_dummy's "addrs" list. */ ++}; ++ + static unixctl_cb_func netdev_dummy_set_admin_state; + static int netdev_dummy_construct(struct netdev *); + static void netdev_dummy_queue_packet(struct netdev_dummy *, +@@ -169,6 +175,7 @@ static void netdev_dummy_queue_packet(struct netdev_dummy *, + static void dummy_packet_stream_close(struct dummy_packet_stream *); + + static void pkt_list_delete(struct ovs_list *); ++static void addr_list_delete(struct ovs_list *); + + static bool + is_dummy_class(const struct netdev_class *class) +@@ -720,6 +727,7 @@ netdev_dummy_construct(struct netdev *netdev_) + dummy_packet_conn_init(&netdev->conn); + + ovs_list_init(&netdev->rxes); ++ ovs_list_init(&netdev->addrs); + hmap_init(&netdev->offloaded_flows); + ovs_mutex_unlock(&netdev->mutex); + +@@ -756,6 +764,7 @@ netdev_dummy_destruct(struct netdev *netdev_) + free(off_flow); + } + hmap_destroy(&netdev->offloaded_flows); ++ addr_list_delete(&netdev->addrs); + + ovs_mutex_unlock(&netdev->mutex); + ovs_mutex_destroy(&netdev->mutex); +@@ -803,32 +812,24 @@ netdev_dummy_get_addr_list(const struct netdev *netdev_, struct in6_addr **paddr + struct netdev_dummy *netdev = netdev_dummy_cast(netdev_); + int cnt = 0, i = 0, err = 0; + struct in6_addr *addr, *mask; ++ struct netdev_addr_dummy *addr_dummy; + + ovs_mutex_lock(&netdev->mutex); +- if (netdev->address.s_addr != INADDR_ANY) { +- cnt++; +- } + +- if (ipv6_addr_is_set(&netdev->ipv6)) { +- cnt++; +- } ++ cnt = ovs_list_size(&netdev->addrs); + if (!cnt) { + err = EADDRNOTAVAIL; + goto out; + } + addr = xmalloc(sizeof *addr * cnt); + mask = xmalloc(sizeof *mask * cnt); +- if (netdev->address.s_addr != INADDR_ANY) { +- in6_addr_set_mapped_ipv4(&addr[i], netdev->address.s_addr); +- in6_addr_set_mapped_ipv4(&mask[i], netdev->netmask.s_addr); +- i++; +- } + +- if (ipv6_addr_is_set(&netdev->ipv6)) { +- memcpy(&addr[i], &netdev->ipv6, sizeof *addr); +- memcpy(&mask[i], &netdev->ipv6_mask, sizeof *mask); ++ LIST_FOR_EACH (addr_dummy, node, &netdev->addrs) { ++ memcpy(&addr[i], &addr_dummy->address, sizeof *addr); ++ memcpy(&mask[i], &addr_dummy->netmask, sizeof *mask); + i++; + } ++ + if (paddr) { + *paddr = addr; + *pmask = mask; +@@ -844,14 +845,16 @@ out: + } + + static int +-netdev_dummy_set_in4(struct netdev *netdev_, struct in_addr address, ++netdev_dummy_add_in4(struct netdev *netdev_, struct in_addr address, + struct in_addr netmask) + { + struct netdev_dummy *netdev = netdev_dummy_cast(netdev_); ++ struct netdev_addr_dummy *addr_dummy = xmalloc(sizeof *addr_dummy); + + ovs_mutex_lock(&netdev->mutex); +- netdev->address = address; +- netdev->netmask = netmask; ++ in6_addr_set_mapped_ipv4(&addr_dummy->address, address.s_addr); ++ in6_addr_set_mapped_ipv4(&addr_dummy->netmask, netmask.s_addr); ++ ovs_list_push_back(&netdev->addrs, &addr_dummy->node); + netdev_change_seq_changed(netdev_); + ovs_mutex_unlock(&netdev->mutex); + +@@ -859,14 +862,16 @@ netdev_dummy_set_in4(struct netdev *netdev_, struct in_addr address, + } + + static int +-netdev_dummy_set_in6(struct netdev *netdev_, struct in6_addr *in6, ++netdev_dummy_add_in6(struct netdev *netdev_, struct in6_addr *in6, + struct in6_addr *mask) + { + struct netdev_dummy *netdev = netdev_dummy_cast(netdev_); ++ struct netdev_addr_dummy *addr_dummy = xmalloc(sizeof *addr_dummy); + + ovs_mutex_lock(&netdev->mutex); +- netdev->ipv6 = *in6; +- netdev->ipv6_mask = *mask; ++ addr_dummy->address = *in6; ++ addr_dummy->netmask = *mask; ++ ovs_list_push_back(&netdev->addrs, &addr_dummy->node); + netdev_change_seq_changed(netdev_); + ovs_mutex_unlock(&netdev->mutex); + +@@ -1178,7 +1183,10 @@ netdev_dummy_send(struct netdev *netdev, int qid, + dummy_packet_conn_send(&dev->conn, buffer, size); + + /* Reply to ARP requests for 'dev''s assigned IP address. */ +- if (dev->address.s_addr) { ++ struct netdev_addr_dummy *addr_dummy; ++ LIST_FOR_EACH (addr_dummy, node, &dev->addrs) { ++ ovs_be32 address = in6_addr_get_mapped_ipv4(&addr_dummy->address); ++ + struct dp_packet dp; + struct flow flow; + +@@ -1186,11 +1194,12 @@ netdev_dummy_send(struct netdev *netdev, int qid, + flow_extract(&dp, &flow); + if (flow.dl_type == htons(ETH_TYPE_ARP) + && flow.nw_proto == ARP_OP_REQUEST +- && flow.nw_dst == dev->address.s_addr) { ++ && flow.nw_dst == address) { + struct dp_packet *reply = dp_packet_new(0); + compose_arp(reply, ARP_OP_REPLY, dev->hwaddr, flow.dl_src, + false, flow.nw_dst, flow.nw_src); + netdev_dummy_queue_packet(dev, reply, NULL, 0); ++ break; + } + } + +@@ -1677,6 +1686,16 @@ pkt_list_delete(struct ovs_list *l) + } + } + ++static void ++addr_list_delete(struct ovs_list *l) ++{ ++ struct netdev_addr_dummy *addr_dummy; ++ ++ LIST_FOR_EACH_POP (addr_dummy, node, l) { ++ free(addr_dummy); ++ } ++} ++ + static struct dp_packet * + eth_from_packet(const char *s) + { +@@ -2005,11 +2024,20 @@ netdev_dummy_ip4addr(struct unixctl_conn *conn, int argc OVS_UNUSED, + + if (netdev && is_dummy_class(netdev->netdev_class)) { + struct in_addr ip, mask; ++ struct in6_addr ip6; ++ uint32_t plen; + char *error; + +- error = ip_parse_masked(argv[2], &ip.s_addr, &mask.s_addr); ++ error = ip_parse_cidr(argv[2], &ip.s_addr, &plen); + if (!error) { +- netdev_dummy_set_in4(netdev, ip, mask); ++ mask.s_addr = be32_prefix_mask(plen); ++ netdev_dummy_add_in4(netdev, ip, mask); ++ ++ /* Insert local route entry for the new address. */ ++ in6_addr_set_mapped_ipv4(&ip6, ip.s_addr); ++ ovs_router_force_insert(0, &ip6, plen + 96, true, argv[1], ++ &in6addr_any); ++ + unixctl_command_reply(conn, "OK"); + } else { + unixctl_command_reply_error(conn, error); +@@ -2038,7 +2066,12 @@ netdev_dummy_ip6addr(struct unixctl_conn *conn, int argc OVS_UNUSED, + struct in6_addr mask; + + mask = ipv6_create_mask(plen); +- netdev_dummy_set_in6(netdev, &ip6, &mask); ++ netdev_dummy_add_in6(netdev, &ip6, &mask); ++ ++ /* Insert local route entry for the new address. */ ++ ovs_router_force_insert(0, &ip6, plen, true, argv[1], ++ &in6addr_any); ++ + unixctl_command_reply(conn, "OK"); + } else { + unixctl_command_reply_error(conn, error); diff --git a/lib/netdev-linux.c b/lib/netdev-linux.c -index 65bdd51db..94b99a7be 100644 +index 65bdd51dba..94b99a7be5 100644 --- a/lib/netdev-linux.c +++ b/lib/netdev-linux.c @@ -2712,8 +2712,16 @@ tc_add_matchall_policer(struct netdev *netdev, uint32_t kbits_rate, @@ -3362,7 +4004,7 @@ index 65bdd51db..94b99a7be 100644 } diff --git a/lib/netdev-native-tnl.c b/lib/netdev-native-tnl.c -index b89dfdd52..6c9094638 100644 +index b89dfdd52a..6c90946388 100644 --- a/lib/netdev-native-tnl.c +++ b/lib/netdev-native-tnl.c @@ -320,7 +320,7 @@ netdev_tnl_ip_build_header(struct ovs_action_push_tnl *data, @@ -3621,7 +4263,7 @@ index b89dfdd52..6c9094638 100644 gnh->options, &crit_opt); diff --git a/lib/netdev-offload-dpdk.c b/lib/netdev-offload-dpdk.c -index b3421c099..2180b5fb5 100644 +index b3421c0996..2180b5fb5f 100644 --- a/lib/netdev-offload-dpdk.c +++ b/lib/netdev-offload-dpdk.c @@ -2345,13 +2345,13 @@ netdev_offload_dpdk_flow_destroy(struct ufid_to_rte_flow_data *rte_flow_data) @@ -3693,7 +4335,7 @@ index b3421c099..2180b5fb5 100644 return 0; } diff --git a/lib/netdev-offload-tc.c b/lib/netdev-offload-tc.c -index 4c78c4816..09f6393f8 100644 +index 4c78c48163..94c17e41a4 100644 --- a/lib/netdev-offload-tc.c +++ b/lib/netdev-offload-tc.c @@ -276,8 +276,9 @@ del_filter_and_ufid_mapping(struct tcf_id *id, const ovs_u128 *ufid, @@ -3807,7 +4449,90 @@ index 4c78c4816..09f6393f8 100644 } else if (!find_ufid(netdev, &id, ufid)) { continue; } -@@ -2490,15 +2505,23 @@ netdev_tc_flow_get(struct netdev *netdev, +@@ -1576,7 +1591,9 @@ parse_put_flow_set_action(struct tc_flower *flower, struct tc_action *action, + } + break; + case OVS_TUNNEL_KEY_ATTR_TP_SRC: { +- action->encap.tp_src = nl_attr_get_be16(tun_attr); ++ /* There is no corresponding attribute in TC. */ ++ VLOG_DBG_RL(&rl, "unsupported tunnel key attribute TP_SRC"); ++ return EOPNOTSUPP; + } + break; + case OVS_TUNNEL_KEY_ATTR_TP_DST: { +@@ -1719,12 +1736,12 @@ test_key_and_mask(struct match *match) + return 0; + } + +-static void ++static int + flower_match_to_tun_opt(struct tc_flower *flower, const struct flow_tnl *tnl, + struct flow_tnl *tnl_mask) + { + struct geneve_opt *opt, *opt_mask; +- int len, cnt = 0; ++ int tot_opt_len, len, cnt = 0; + + /* 'flower' always has an exact match on tunnel metadata length, so having + * it in a wrong format is not acceptable unless it is empty. */ +@@ -1740,7 +1757,7 @@ flower_match_to_tun_opt(struct tc_flower *flower, const struct flow_tnl *tnl, + memset(&tnl_mask->metadata.present.map, 0, + sizeof tnl_mask->metadata.present.map); + } +- return; ++ return 0; + } + + tnl_mask->flags &= ~FLOW_TNL_F_UDPIF; +@@ -1754,7 +1771,7 @@ flower_match_to_tun_opt(struct tc_flower *flower, const struct flow_tnl *tnl, + sizeof tnl_mask->metadata.present.len); + + if (!tnl->metadata.present.len) { +- return; ++ return 0; + } + + memcpy(flower->key.tunnel.metadata.opts.gnv, tnl->metadata.opts.gnv, +@@ -1768,7 +1785,16 @@ flower_match_to_tun_opt(struct tc_flower *flower, const struct flow_tnl *tnl, + * also not masks, but actual lengths in the 'flower' structure. */ + len = flower->key.tunnel.metadata.present.len; + while (len) { ++ if (len < sizeof *opt) { ++ return EOPNOTSUPP; ++ } ++ + opt = &flower->key.tunnel.metadata.opts.gnv[cnt]; ++ tot_opt_len = sizeof *opt + opt->length * 4; ++ if (len < tot_opt_len) { ++ return EOPNOTSUPP; ++ } ++ + opt_mask = &flower->mask.tunnel.metadata.opts.gnv[cnt]; + + opt_mask->length = opt->length; +@@ -1776,6 +1802,8 @@ flower_match_to_tun_opt(struct tc_flower *flower, const struct flow_tnl *tnl, + cnt += sizeof(struct geneve_opt) / 4 + opt->length; + len -= sizeof(struct geneve_opt) + opt->length * 4; + } ++ ++ return 0; + } + + static void +@@ -2213,7 +2241,11 @@ netdev_tc_flow_put(struct netdev *netdev, struct match *match, + tnl_mask->flags &= ~(FLOW_TNL_F_DONT_FRAGMENT | FLOW_TNL_F_CSUM); + + if (!strcmp(netdev_get_type(netdev), "geneve")) { +- flower_match_to_tun_opt(&flower, tnl, tnl_mask); ++ err = flower_match_to_tun_opt(&flower, tnl, tnl_mask); ++ if (err) { ++ VLOG_WARN_RL(&warn_rl, "Unable to parse geneve options"); ++ return err; ++ } + } + flower.tunnel = true; + } else { +@@ -2490,15 +2522,23 @@ netdev_tc_flow_get(struct netdev *netdev, err = tc_get_flower(&id, &flower); if (err) { @@ -3834,7 +4559,7 @@ index 4c78c4816..09f6393f8 100644 if (stats) { struct dpif_flow_stats adjust_stats; -@@ -2860,8 +2883,9 @@ netdev_tc_init_flow_api(struct netdev *netdev) +@@ -2860,8 +2900,9 @@ netdev_tc_init_flow_api(struct netdev *netdev) error = tc_add_del_qdisc(ifindex, true, block_id, hook); if (error && error != EEXIST) { @@ -3847,7 +4572,7 @@ index 4c78c4816..09f6393f8 100644 } diff --git a/lib/netdev-offload.c b/lib/netdev-offload.c -index 4592262bd..931d634e1 100644 +index 4592262bd3..931d634e15 100644 --- a/lib/netdev-offload.c +++ b/lib/netdev-offload.c @@ -485,11 +485,13 @@ netdev_set_hw_info(struct netdev *netdev, int type, int val) @@ -4120,7 +4845,7 @@ index 4592262bd..931d634e1 100644 offload_thread_nb); offload_thread_nb = DEFAULT_OFFLOAD_THREAD_NB; diff --git a/lib/netdev-vport-private.h b/lib/netdev-vport-private.h -index d89a28c66..586231057 100644 +index d89a28c66c..586231057c 100644 --- a/lib/netdev-vport-private.h +++ b/lib/netdev-vport-private.h @@ -22,11 +22,17 @@ @@ -4150,7 +4875,7 @@ index d89a28c66..586231057 100644 bool carrier_status; diff --git a/lib/netdev-vport.c b/lib/netdev-vport.c -index 3b3927865..4cfe15d5a 100644 +index 3b39278650..4cfe15d5a7 100644 --- a/lib/netdev-vport.c +++ b/lib/netdev-vport.c @@ -37,6 +37,7 @@ @@ -4574,7 +5299,7 @@ index 3b3927865..4cfe15d5a 100644 diff --git a/lib/netdev-windows.c b/lib/netdev-windows.c -index 4ad45ffa1..3fad501e3 100644 +index 4ad45ffa1b..3fad501e3e 100644 --- a/lib/netdev-windows.c +++ b/lib/netdev-windows.c @@ -156,6 +156,7 @@ netdev_windows_system_construct(struct netdev *netdev_) @@ -4603,7 +5328,7 @@ index 4ad45ffa1..3fad501e3 100644 netdev->dev_type = info.ovs_type; netdev->port_no = info.port_no; diff --git a/lib/netdev.h b/lib/netdev.h -index acf174927..47c15bde7 100644 +index acf174927d..47c15bde7e 100644 --- a/lib/netdev.h +++ b/lib/netdev.h @@ -72,6 +72,9 @@ struct sset; @@ -4625,7 +5350,7 @@ index acf174927..47c15bde7 100644 uint8_t erspan_ver; uint8_t erspan_dir; diff --git a/lib/netlink-conntrack.c b/lib/netlink-conntrack.c -index 4fcde9ba1..492bfcffb 100644 +index 4fcde9ba1e..492bfcffb8 100644 --- a/lib/netlink-conntrack.c +++ b/lib/netlink-conntrack.c @@ -579,7 +579,8 @@ nl_ct_put_tuple_proto(struct ofpbuf *buf, const struct ct_dpif_tuple *tuple) @@ -4638,8 +5363,143 @@ index 4fcde9ba1..492bfcffb 100644 nl_msg_put_be16(buf, CTA_PROTO_SRC_PORT, tuple->src_port); nl_msg_put_be16(buf, CTA_PROTO_DST_PORT, tuple->dst_port); } else { +diff --git a/lib/netlink-protocol.h b/lib/netlink-protocol.h +index 6eaa7035a4..e4bb28ac9f 100644 +--- a/lib/netlink-protocol.h ++++ b/lib/netlink-protocol.h +@@ -155,6 +155,11 @@ enum { + #define NLA_TYPE_MASK ~(NLA_F_NESTED | NLA_F_NET_BYTEORDER) + #endif + ++/* Introduced in v4.4. */ ++#ifndef NLM_F_DUMP_FILTERED ++#define NLM_F_DUMP_FILTERED 0x20 ++#endif ++ + /* These were introduced all together in 2.6.14. (We want our programs to + * support the newer kernel features even if compiled with older headers.) */ + #ifndef NETLINK_ADD_MEMBERSHIP +@@ -168,6 +173,11 @@ enum { + #define NETLINK_LISTEN_ALL_NSID 8 + #endif + ++/* Strict checking of netlink arguments introduced in Linux kernel v4.20. */ ++#ifndef NETLINK_GET_STRICT_CHK ++#define NETLINK_GET_STRICT_CHK 12 ++#endif ++ + /* These were introduced all together in 2.6.23. (We want our programs to + * support the newer kernel features even if compiled with older headers.) */ + #ifndef CTRL_ATTR_MCAST_GRP_MAX +diff --git a/lib/netlink-socket.c b/lib/netlink-socket.c +index 80da20d9f0..5cb1fc89ae 100644 +--- a/lib/netlink-socket.c ++++ b/lib/netlink-socket.c +@@ -205,6 +205,15 @@ nl_sock_create(int protocol, struct nl_sock **sockp) + } + } + ++ /* Strict checking only supported for NETLINK_ROUTE. */ ++ if (protocol == NETLINK_ROUTE ++ && setsockopt(sock->fd, SOL_NETLINK, NETLINK_GET_STRICT_CHK, ++ &one, sizeof one) < 0) { ++ VLOG_RL(&rl, errno == ENOPROTOOPT ? VLL_DBG : VLL_WARN, ++ "netlink: could not enable strict checking (%s)", ++ ovs_strerror(errno)); ++ } ++ + retval = get_socket_rcvbuf(sock->fd); + if (retval < 0) { + retval = -retval; +diff --git a/lib/odp-util.c b/lib/odp-util.c +index 5fc312f8c0..fa353eab5c 100644 +--- a/lib/odp-util.c ++++ b/lib/odp-util.c +@@ -6405,12 +6405,10 @@ odp_flow_key_from_flow__(const struct odp_flow_key_parms *parms, + icmpv6_key->icmpv6_code = ntohs(data->tp_dst); + + if (is_nd(flow, NULL) +- /* Even though 'tp_src' and 'tp_dst' are 16 bits wide, ICMP +- * type and code are 8 bits wide. Therefore, an exact match +- * looks like htons(0xff), not htons(0xffff). See +- * xlate_wc_finish() for details. */ +- && (!export_mask || (data->tp_src == htons(0xff) +- && data->tp_dst == htons(0xff)))) { ++ /* Even though 'tp_src' is 16 bits wide, ICMP type is 8 bits ++ * wide. Therefore, an exact match looks like htons(0xff), ++ * not htons(0xffff). See xlate_wc_finish() for details. */ ++ && (!export_mask || data->tp_src == htons(0xff))) { + struct ovs_key_nd *nd_key; + nd_key = nl_msg_put_unspec_uninit(buf, OVS_KEY_ATTR_ND, + sizeof *nd_key); +@@ -7126,20 +7124,17 @@ parse_l2_5_onward(const struct nlattr *attrs[OVS_KEY_ATTR_MAX + 1], + flow->arp_sha = nd_key->nd_sll; + flow->arp_tha = nd_key->nd_tll; + if (is_mask) { +- /* Even though 'tp_src' and 'tp_dst' are 16 bits wide, +- * ICMP type and code are 8 bits wide. Therefore, an +- * exact match looks like htons(0xff), not +- * htons(0xffff). See xlate_wc_finish() for details. +- * */ ++ /* Even though 'tp_src' is 16 bits wide, ICMP type ++ * is 8 bits wide. Therefore, an exact match looks ++ * like htons(0xff), not htons(0xffff). See ++ * xlate_wc_finish() for details. */ + if (!is_all_zeros(nd_key, sizeof *nd_key) && +- (flow->tp_src != htons(0xff) || +- flow->tp_dst != htons(0xff))) { ++ flow->tp_src != htons(0xff)) { + odp_parse_error(&rl, errorp, +- "ICMP (src,dst) masks should be " +- "(0xff,0xff) but are actually " +- "(%#"PRIx16",%#"PRIx16")", +- ntohs(flow->tp_src), +- ntohs(flow->tp_dst)); ++ "ICMP src mask should be " ++ "(0xff) but is actually " ++ "(%#"PRIx16")", ++ ntohs(flow->tp_src)); + return ODP_FIT_ERROR; + } else { + *expected_attrs |= UINT64_C(1) << OVS_KEY_ATTR_ND; +diff --git a/lib/ofp-ct.c b/lib/ofp-ct.c +index 85a9d8beca..a140fba470 100644 +--- a/lib/ofp-ct.c ++++ b/lib/ofp-ct.c +@@ -31,6 +31,9 @@ + #include "openvswitch/ofp-prop.h" + #include "openvswitch/ofp-util.h" + #include "openvswitch/packets.h" ++#include "openvswitch/vlog.h" ++ ++VLOG_DEFINE_THIS_MODULE(ofp_ct); + + static void + ofp_ct_tuple_format(struct ds *ds, const struct ofp_ct_tuple *tuple, +@@ -286,6 +289,10 @@ ofp_ct_tuple_decode_nested(struct ofpbuf *property, struct ofp_ct_tuple *tuple, + case NXT_CT_TUPLE_ICMP_CODE: + error = ofpprop_parse_u8(&inner, &tuple->icmp_code); + break; ++ ++ default: ++ error = OFPPROP_UNKNOWN(false, "NXT_CT_TUPLE", type); ++ break; + } + + if (error) { +@@ -377,6 +384,10 @@ ofp_ct_match_decode(struct ofp_ct_match *match, bool *with_zone, + } + error = ofpprop_parse_u16(&property, zone_id); + break; ++ ++ default: ++ error = OFPPROP_UNKNOWN(false, "NXT_CT_FLUSH", type); ++ break; + } + + if (error) { diff --git a/lib/ofp-parse.c b/lib/ofp-parse.c -index a90b926ef..102b183a8 100644 +index a90b926efb..102b183a8f 100644 --- a/lib/ofp-parse.c +++ b/lib/ofp-parse.c @@ -71,16 +71,13 @@ str_to_u16(const char *str, const char *name, uint16_t *valuep) @@ -4662,7 +5522,7 @@ index a90b926ef..102b183a8 100644 } *valuep = value; diff --git a/lib/ofp-table.c b/lib/ofp-table.c -index a956754f2..f9bd3b7f9 100644 +index a956754f2d..f9bd3b7f9c 100644 --- a/lib/ofp-table.c +++ b/lib/ofp-table.c @@ -1416,7 +1416,7 @@ count_common_prefix_run(const char *ids[], size_t n, @@ -4674,8 +5534,79 @@ index a956754f2..f9bd3b7f9 100644 } i++; } +diff --git a/lib/ofpbuf.c b/lib/ofpbuf.c +index d3d42b4148..232ebeb97b 100644 +--- a/lib/ofpbuf.c ++++ b/lib/ofpbuf.c +@@ -197,12 +197,12 @@ ofpbuf_clone_with_headroom(const struct ofpbuf *b, size_t headroom) + struct ofpbuf *new_buffer; + + new_buffer = ofpbuf_clone_data_with_headroom(b->data, b->size, headroom); +- if (b->header) { ++ if (new_buffer->data && b->header) { + ptrdiff_t header_offset = (char *) b->header - (char *) b->data; + + new_buffer->header = (char *) new_buffer->data + header_offset; + } +- if (b->msg) { ++ if (new_buffer->data && b->msg) { + ptrdiff_t msg_offset = (char *) b->msg - (char *) b->data; + + new_buffer->msg = (char *) new_buffer->data + msg_offset; +diff --git a/lib/ovs-atomic.h b/lib/ovs-atomic.h +index 8fdce0cf80..62ee4e0682 100644 +--- a/lib/ovs-atomic.h ++++ b/lib/ovs-atomic.h +@@ -329,7 +329,7 @@ + #if __CHECKER__ + /* sparse doesn't understand some GCC extensions we use. */ + #include "ovs-atomic-pthreads.h" +- #elif __has_extension(c_atomic) ++ #elif __clang__ && __has_extension(c_atomic) + #include "ovs-atomic-clang.h" + #elif HAVE_ATOMIC && __cplusplus >= 201103L + #include "ovs-atomic-c++.h" +diff --git a/lib/ovs-router.c b/lib/ovs-router.c +index 5d0fbd503e..4080134533 100644 +--- a/lib/ovs-router.c ++++ b/lib/ovs-router.c +@@ -278,6 +278,19 @@ ovs_router_insert(uint32_t mark, const struct in6_addr *ip_dst, uint8_t plen, + } + } + ++/* The same as 'ovs_router_insert', but it adds the route even if updates ++ * from the system routing table are disabled. Used for unit tests. */ ++void ++ovs_router_force_insert(uint32_t mark, const struct in6_addr *ip_dst, ++ uint8_t plen, bool local, const char output_bridge[], ++ const struct in6_addr *gw) ++{ ++ uint8_t priority = local ? plen + 64 : plen; ++ ++ ovs_router_insert__(mark, priority, local, ip_dst, plen, ++ output_bridge, gw); ++} ++ + static void + rt_entry_delete__(const struct cls_rule *cr) + { +diff --git a/lib/ovs-router.h b/lib/ovs-router.h +index d8ce3c00de..53a3b5d7e9 100644 +--- a/lib/ovs-router.h ++++ b/lib/ovs-router.h +@@ -33,6 +33,10 @@ void ovs_router_init(void); + void ovs_router_insert(uint32_t mark, const struct in6_addr *ip_dst, + uint8_t plen, bool local, + const char output_bridge[], const struct in6_addr *gw); ++void ovs_router_force_insert(uint32_t mark, const struct in6_addr *ip_dst, ++ uint8_t plen, bool local, ++ const char output_bridge[], ++ const struct in6_addr *gw); + void ovs_router_flush(void); + + void ovs_router_disable_system_routing_table(void); diff --git a/lib/ovs-thread.c b/lib/ovs-thread.c -index 2d382f1e8..ac5d2c3d0 100644 +index 2d382f1e8b..ac5d2c3d02 100644 --- a/lib/ovs-thread.c +++ b/lib/ovs-thread.c @@ -674,7 +674,7 @@ count_cpu_cores(void) @@ -4688,7 +5619,7 @@ index 2d382f1e8..ac5d2c3d0 100644 cpu_cores = count_cpu_cores__(); } diff --git a/lib/ovs.tmac b/lib/ovs.tmac -index 5f8f20afa..97b6fa3df 100644 +index 5f8f20afa4..97b6fa3df7 100644 --- a/lib/ovs.tmac +++ b/lib/ovs.tmac @@ -175,7 +175,7 @@ @@ -4700,8 +5631,267 @@ index 5f8f20afa..97b6fa3df 100644 .. . . +diff --git a/lib/ovsdb-idl.c b/lib/ovsdb-idl.c +index 634fbb56df..ba720474b6 100644 +--- a/lib/ovsdb-idl.c ++++ b/lib/ovsdb-idl.c +@@ -177,6 +177,7 @@ static void ovsdb_idl_row_mark_backrefs_for_reparsing(struct ovsdb_idl_row *); + static void ovsdb_idl_row_track_change(struct ovsdb_idl_row *, + enum ovsdb_idl_change); + static void ovsdb_idl_row_untrack_change(struct ovsdb_idl_row *); ++static void ovsdb_idl_row_clear_changeseqno(struct ovsdb_idl_row *); + + static void ovsdb_idl_txn_abort_all(struct ovsdb_idl *); + static bool ovsdb_idl_txn_extract_mutations(struct ovsdb_idl_row *, +@@ -1374,6 +1375,7 @@ ovsdb_idl_track_clear__(struct ovsdb_idl *idl, bool flush_all) + row->updated = NULL; + } + ovsdb_idl_row_untrack_change(row); ++ ovsdb_idl_row_clear_changeseqno(row); + + if (ovsdb_idl_row_is_orphan(row)) { + ovsdb_idl_row_unparse(row); +@@ -1632,6 +1634,7 @@ ovsdb_idl_process_update(struct ovsdb_idl_table *table, + ru->columns); + } else if (ovsdb_idl_row_is_orphan(row)) { + ovsdb_idl_row_untrack_change(row); ++ ovsdb_idl_row_clear_changeseqno(row); + ovsdb_idl_insert_row(row, ru->columns); + } else { + VLOG_ERR_RL(&semantic_rl, "cannot add existing row "UUID_FMT" to " +@@ -2283,11 +2286,15 @@ ovsdb_idl_row_untrack_change(struct ovsdb_idl_row *row) + return; + } + ++ ovs_list_remove(&row->track_node); ++ ovs_list_init(&row->track_node); ++} ++ ++static void ovsdb_idl_row_clear_changeseqno(struct ovsdb_idl_row *row) ++{ + row->change_seqno[OVSDB_IDL_CHANGE_INSERT] = + row->change_seqno[OVSDB_IDL_CHANGE_MODIFY] = + row->change_seqno[OVSDB_IDL_CHANGE_DELETE] = 0; +- ovs_list_remove(&row->track_node); +- ovs_list_init(&row->track_node); + } + + static struct ovsdb_idl_row * +diff --git a/lib/ovsdb-types.h b/lib/ovsdb-types.h +index b9eb0928df..d2455fc977 100644 +--- a/lib/ovsdb-types.h ++++ b/lib/ovsdb-types.h +@@ -235,6 +235,18 @@ static inline bool ovsdb_type_is_map(const struct ovsdb_type *type) + return type->value.type != OVSDB_TYPE_VOID; + } + ++static inline bool ovsdb_type_has_strong_refs(const struct ovsdb_type *type) ++{ ++ return ovsdb_base_type_is_strong_ref(&type->key) ++ || ovsdb_base_type_is_strong_ref(&type->value); ++} ++ ++static inline bool ovsdb_type_has_weak_refs(const struct ovsdb_type *type) ++{ ++ return ovsdb_base_type_is_weak_ref(&type->key) ++ || ovsdb_base_type_is_weak_ref(&type->value); ++} ++ + #ifdef __cplusplus + } + #endif +diff --git a/lib/route-table.c b/lib/route-table.c +index ac82cf262f..6df7237e10 100644 +--- a/lib/route-table.c ++++ b/lib/route-table.c +@@ -26,6 +26,7 @@ + #include + #include + ++#include "coverage.h" + #include "hash.h" + #include "netdev.h" + #include "netlink.h" +@@ -44,6 +45,8 @@ + + VLOG_DEFINE_THIS_MODULE(route_table); + ++COVERAGE_DEFINE(route_table_dump); ++ + struct route_data { + /* Copied from struct rtmsg. */ + unsigned char rtm_dst_len; +@@ -79,7 +82,7 @@ static struct nln_notifier *name_notifier = NULL; + + static bool route_table_valid = false; + +-static int route_table_reset(void); ++static void route_table_reset(void); + static void route_table_handle_msg(const struct route_table_msg *); + static int route_table_parse(struct ofpbuf *, struct route_table_msg *); + static void route_table_change(const struct route_table_msg *, void *); +@@ -152,26 +155,22 @@ route_table_wait(void) + ovs_mutex_unlock(&route_table_mutex); + } + +-static int +-route_table_reset(void) ++static bool ++route_table_dump_one_table(unsigned char id) + { +- struct nl_dump dump; +- struct rtgenmsg *rtgenmsg; + uint64_t reply_stub[NL_DUMP_BUFSIZE / 8]; + struct ofpbuf request, reply, buf; +- +- route_map_clear(); +- netdev_get_addrs_list_flush(); +- route_table_valid = true; +- rt_change_seq++; ++ struct rtmsg *rq_msg; ++ bool filtered = true; ++ struct nl_dump dump; + + ofpbuf_init(&request, 0); + +- nl_msg_put_nlmsghdr(&request, sizeof *rtgenmsg, RTM_GETROUTE, +- NLM_F_REQUEST); ++ nl_msg_put_nlmsghdr(&request, sizeof *rq_msg, RTM_GETROUTE, NLM_F_REQUEST); + +- rtgenmsg = ofpbuf_put_zeros(&request, sizeof *rtgenmsg); +- rtgenmsg->rtgen_family = AF_UNSPEC; ++ rq_msg = ofpbuf_put_zeros(&request, sizeof *rq_msg); ++ rq_msg->rtm_family = AF_UNSPEC; ++ rq_msg->rtm_table = id; + + nl_dump_start(&dump, NETLINK_ROUTE, &request); + ofpbuf_uninit(&request); +@@ -181,12 +180,43 @@ route_table_reset(void) + struct route_table_msg msg; + + if (route_table_parse(&reply, &msg)) { ++ struct nlmsghdr *nlmsghdr = nl_msg_nlmsghdr(&reply); ++ ++ /* Older kernels do not support filtering. */ ++ if (!(nlmsghdr->nlmsg_flags & NLM_F_DUMP_FILTERED)) { ++ filtered = false; ++ } + route_table_handle_msg(&msg); + } + } + ofpbuf_uninit(&buf); ++ nl_dump_done(&dump); ++ ++ return filtered; ++} ++ ++static void ++route_table_reset(void) ++{ ++ unsigned char tables[] = { ++ RT_TABLE_DEFAULT, ++ RT_TABLE_MAIN, ++ RT_TABLE_LOCAL, ++ }; + +- return nl_dump_done(&dump); ++ route_map_clear(); ++ netdev_get_addrs_list_flush(); ++ route_table_valid = true; ++ rt_change_seq++; ++ ++ COVERAGE_INC(route_table_dump); ++ ++ for (size_t i = 0; i < ARRAY_SIZE(tables); i++) { ++ if (!route_table_dump_one_table(tables[i])) { ++ /* Got unfiltered reply, no need to dump further. */ ++ break; ++ } ++ } + } + + /* Return RTNLGRP_IPV4_ROUTE or RTNLGRP_IPV6_ROUTE on success, 0 on parse +@@ -201,6 +231,7 @@ route_table_parse(struct ofpbuf *buf, struct route_table_msg *change) + [RTA_OIF] = { .type = NL_A_U32, .optional = true }, + [RTA_GATEWAY] = { .type = NL_A_U32, .optional = true }, + [RTA_MARK] = { .type = NL_A_U32, .optional = true }, ++ [RTA_TABLE] = { .type = NL_A_U32, .optional = true }, + }; + + static const struct nl_policy policy6[] = { +@@ -208,6 +239,7 @@ route_table_parse(struct ofpbuf *buf, struct route_table_msg *change) + [RTA_OIF] = { .type = NL_A_U32, .optional = true }, + [RTA_MARK] = { .type = NL_A_U32, .optional = true }, + [RTA_GATEWAY] = { .type = NL_A_IPV6, .optional = true }, ++ [RTA_TABLE] = { .type = NL_A_U32, .optional = true }, + }; + + struct nlattr *attrs[ARRAY_SIZE(policy)]; +@@ -229,6 +261,7 @@ route_table_parse(struct ofpbuf *buf, struct route_table_msg *change) + + if (parsed) { + const struct nlmsghdr *nlmsg; ++ uint32_t table_id; + int rta_oif; /* Output interface index. */ + + nlmsg = buf->data; +@@ -244,6 +277,19 @@ route_table_parse(struct ofpbuf *buf, struct route_table_msg *change) + rtm->rtm_type != RTN_LOCAL) { + change->relevant = false; + } ++ ++ table_id = rtm->rtm_table; ++ if (attrs[RTA_TABLE]) { ++ table_id = nl_attr_get_u32(attrs[RTA_TABLE]); ++ } ++ /* Do not consider changes in non-standard routing tables. */ ++ if (table_id ++ && table_id != RT_TABLE_DEFAULT ++ && table_id != RT_TABLE_MAIN ++ && table_id != RT_TABLE_LOCAL) { ++ change->relevant = false; ++ } ++ + change->nlmsg_type = nlmsg->nlmsg_type; + change->rd.rtm_dst_len = rtm->rtm_dst_len + (ipv4 ? 96 : 0); + change->rd.local = rtm->rtm_type == RTN_LOCAL; +@@ -299,7 +345,9 @@ static void + route_table_change(const struct route_table_msg *change OVS_UNUSED, + void *aux OVS_UNUSED) + { +- route_table_valid = false; ++ if (!change || change->relevant) { ++ route_table_valid = false; ++ } + } + + static void +diff --git a/lib/rstp.c b/lib/rstp.c +index 2f01966f79..90e8094599 100644 +--- a/lib/rstp.c ++++ b/lib/rstp.c +@@ -50,7 +50,7 @@ + + VLOG_DEFINE_THIS_MODULE(rstp); + +-struct ovs_mutex rstp_mutex = OVS_MUTEX_INITIALIZER; ++struct ovs_mutex rstp_mutex; + + static struct ovs_list all_rstps__ = OVS_LIST_INITIALIZER(&all_rstps__); + static struct ovs_list *const all_rstps OVS_GUARDED_BY(rstp_mutex) = &all_rstps__; +@@ -248,6 +248,10 @@ void + rstp_init(void) + OVS_EXCLUDED(rstp_mutex) + { ++ /* We need a recursive mutex because rstp_send_bpdu() could loop back ++ * into the rstp module through a patch port. */ ++ ovs_mutex_init_recursive(&rstp_mutex); ++ + unixctl_command_register("rstp/tcn", "[bridge]", 0, 1, rstp_unixctl_tcn, + NULL); + unixctl_command_register("rstp/show", "[bridge]", 0, 1, rstp_unixctl_show, diff --git a/lib/smap.c b/lib/smap.c -index c1633e2a1..47fb34502 100644 +index c1633e2a18..47fb345020 100644 --- a/lib/smap.c +++ b/lib/smap.c @@ -100,7 +100,7 @@ smap_add_format(struct smap *smap, const char *key, const char *format, ...) @@ -4714,7 +5904,7 @@ index c1633e2a1..47fb34502 100644 char buf[INET6_ADDRSTRLEN]; ipv6_string_mapped(buf, addr); diff --git a/lib/smap.h b/lib/smap.h -index 2fe6c540a..d1d2ae6f2 100644 +index 2fe6c540a7..d1d2ae6f20 100644 --- a/lib/smap.h +++ b/lib/smap.h @@ -100,7 +100,7 @@ struct smap_node *smap_add_nocopy(struct smap *, char *, char *); @@ -4727,7 +5917,7 @@ index 2fe6c540a..d1d2ae6f2 100644 void smap_replace_nocopy(struct smap *, const char *, char *); diff --git a/lib/stream-ssl.c b/lib/stream-ssl.c -index 62da9febb..86747e58b 100644 +index 62da9febb6..86747e58ba 100644 --- a/lib/stream-ssl.c +++ b/lib/stream-ssl.c @@ -1075,7 +1075,13 @@ do_ssl_init(void) @@ -4746,7 +5936,7 @@ index 62da9febb..86747e58b 100644 SSL_CTX_set_tmp_dh_callback(ctx, tmp_dh_callback); #else diff --git a/lib/tc.c b/lib/tc.c -index 4c07e2216..0c0f07767 100644 +index 4c07e22162..0c0f077676 100644 --- a/lib/tc.c +++ b/lib/tc.c @@ -36,6 +36,7 @@ @@ -4923,13 +6113,122 @@ index 4c07e2216..0c0f07767 100644 id->prio = tc_get_major(tc->tcm_info); id->handle = tc->tcm_handle; -diff --git a/ofproto/connmgr.c b/ofproto/connmgr.c -index 7b14cae77..f7f7b1279 100644 ---- a/ofproto/connmgr.c -+++ b/ofproto/connmgr.c -@@ -1209,7 +1209,7 @@ ofconn_create(struct ofservice *ofservice, struct rconn *rconn, - hmap_init(&ofconn->bundles); - ofconn->next_bundle_expiry_check = time_msec() + BUNDLE_EXPIRY_INTERVAL; +diff --git a/lib/tc.h b/lib/tc.h +index cdd3b4f60e..ea2c5adc19 100644 +--- a/lib/tc.h ++++ b/lib/tc.h +@@ -227,7 +227,8 @@ struct tc_action { + struct { + bool id_present; + ovs_be64 id; +- ovs_be16 tp_src; ++ /* ovs_be16 tp_src; Could have been here, but there is no ++ * TCA_TUNNEL_KEY_ENC_ attribute for it in the kernel. */ + ovs_be16 tp_dst; + uint8_t tos; + uint8_t ttl; +diff --git a/lib/vconn.c b/lib/vconn.c +index b556762277..e9603432d2 100644 +--- a/lib/vconn.c ++++ b/lib/vconn.c +@@ -682,7 +682,6 @@ do_send(struct vconn *vconn, struct ofpbuf *msg) + + ofpmsg_update_length(msg); + if (!VLOG_IS_DBG_ENABLED()) { +- COVERAGE_INC(vconn_sent); + retval = (vconn->vclass->send)(vconn, msg); + } else { + char *s = ofp_to_string(msg->data, msg->size, NULL, NULL, 1); +@@ -693,6 +692,9 @@ do_send(struct vconn *vconn, struct ofpbuf *msg) + } + free(s); + } ++ if (!retval) { ++ COVERAGE_INC(vconn_sent); ++ } + return retval; + } + +diff --git a/ofproto/bond.c b/ofproto/bond.c +index cfdf44f854..c31869a4c7 100644 +--- a/ofproto/bond.c ++++ b/ofproto/bond.c +@@ -186,7 +186,7 @@ static struct bond_member *choose_output_member(const struct bond *, + struct flow_wildcards *, + uint16_t vlan) + OVS_REQ_RDLOCK(rwlock); +-static void update_recirc_rules__(struct bond *); ++static void update_recirc_rules(struct bond *) OVS_REQ_WRLOCK(rwlock); + static bool bond_may_recirc(const struct bond *); + static void bond_update_post_recirc_rules__(struct bond *, bool force) + OVS_REQ_WRLOCK(rwlock); +@@ -299,7 +299,10 @@ bond_unref(struct bond *bond) + } + free(bond->hash); + bond->hash = NULL; +- update_recirc_rules__(bond); ++ ++ ovs_rwlock_wrlock(&rwlock); ++ update_recirc_rules(bond); ++ ovs_rwlock_unlock(&rwlock); + + hmap_destroy(&bond->pr_rule_ops); + free(bond->primary); +@@ -331,17 +334,8 @@ add_pr_rule(struct bond *bond, const struct match *match, + hmap_insert(&bond->pr_rule_ops, &pr_op->hmap_node, hash); + } + +-/* This function should almost never be called directly. +- * 'update_recirc_rules()' should be called instead. Since +- * this function modifies 'bond->pr_rule_ops', it is only +- * safe when 'rwlock' is held. +- * +- * However, when the 'bond' is the only reference in the system, +- * calling this function avoid acquiring lock only to satisfy +- * lock annotation. Currently, only 'bond_unref()' calls +- * this function directly. */ + static void +-update_recirc_rules__(struct bond *bond) ++update_recirc_rules(struct bond *bond) OVS_REQ_WRLOCK(rwlock) + { + struct match match; + struct bond_pr_rule_op *pr_op; +@@ -407,6 +401,15 @@ update_recirc_rules__(struct bond *bond) + + VLOG_ERR("failed to remove post recirculation flow %s", err_s); + free(err_s); ++ } else if (bond->hash) { ++ /* If the flow deletion failed, a subsequent call to ++ * ofproto_dpif_add_internal_flow() would just modify the ++ * flow preserving its statistics. Therefore, only reset ++ * the entry's byte counter if it succeeds. */ ++ uint32_t hash = pr_op->match.flow.dp_hash & BOND_MASK; ++ struct bond_entry *entry = &bond->hash[hash]; ++ ++ entry->pr_tx_bytes = 0; + } + + hmap_remove(&bond->pr_rule_ops, &pr_op->hmap_node); +@@ -421,12 +424,6 @@ update_recirc_rules__(struct bond *bond) + ofpbuf_uninit(&ofpacts); + } + +-static void +-update_recirc_rules(struct bond *bond) +- OVS_REQ_RDLOCK(rwlock) +-{ +- update_recirc_rules__(bond); +-} + + /* Updates 'bond''s overall configuration to 's'. + * +diff --git a/ofproto/connmgr.c b/ofproto/connmgr.c +index 7b14cae773..f7f7b12799 100644 +--- a/ofproto/connmgr.c ++++ b/ofproto/connmgr.c +@@ -1209,7 +1209,7 @@ ofconn_create(struct ofservice *ofservice, struct rconn *rconn, + hmap_init(&ofconn->bundles); + ofconn->next_bundle_expiry_check = time_msec() + BUNDLE_EXPIRY_INTERVAL; - ofconn_set_rate_limit(ofconn, settings->rate_limit, settings->burst_limit); + ofservice_reconfigure(ofservice, settings); @@ -4996,7 +6295,7 @@ index 7b14cae77..f7f7b1279 100644 VLOG_INFO("%s: added %s controller \"%s\"", mgr->name, ofconn_type_to_string(ofservice->type), target); diff --git a/ofproto/ofproto-dpif-ipfix.c b/ofproto/ofproto-dpif-ipfix.c -index 742eed399..f13478a88 100644 +index 742eed3998..f13478a884 100644 --- a/ofproto/ofproto-dpif-ipfix.c +++ b/ofproto/ofproto-dpif-ipfix.c @@ -124,11 +124,18 @@ struct dpif_ipfix_port { @@ -5289,11 +6588,75 @@ index 742eed399..f13478a88 100644 { uint64_t export_time_usec; uint32_t export_time_sec; +diff --git a/ofproto/ofproto-dpif-monitor.c b/ofproto/ofproto-dpif-monitor.c +index bb0e490910..5132f9c952 100644 +--- a/ofproto/ofproto-dpif-monitor.c ++++ b/ofproto/ofproto-dpif-monitor.c +@@ -275,19 +275,16 @@ monitor_mport_run(struct mport *mport, struct dp_packet *packet) + long long int lldp_wake_time = LLONG_MAX; + + if (mport->cfm && cfm_should_send_ccm(mport->cfm)) { +- dp_packet_clear(packet); + cfm_compose_ccm(mport->cfm, packet, mport->hw_addr); + ofproto_dpif_send_packet(mport->ofport, false, packet); + } + if (mport->bfd && bfd_should_send_packet(mport->bfd)) { + bool oam; + +- dp_packet_clear(packet); + bfd_put_packet(mport->bfd, packet, mport->hw_addr, &oam); + ofproto_dpif_send_packet(mport->ofport, oam, packet); + } + if (mport->lldp && lldp_should_send_packet(mport->lldp)) { +- dp_packet_clear(packet); + lldp_put_packet(mport->lldp, packet, mport->hw_addr); + ofproto_dpif_send_packet(mport->ofport, false, packet); + } +diff --git a/ofproto/ofproto-dpif-trace.c b/ofproto/ofproto-dpif-trace.c +index 527e2f17ed..4fbe85018e 100644 +--- a/ofproto/ofproto-dpif-trace.c ++++ b/ofproto/ofproto-dpif-trace.c +@@ -845,17 +845,35 @@ ofproto_trace(struct ofproto_dpif *ofproto, const struct flow *flow, + bool names) + { + struct ovs_list recirc_queue = OVS_LIST_INITIALIZER(&recirc_queue); ++ int recirculations = 0; ++ + ofproto_trace__(ofproto, flow, packet, &recirc_queue, + ofpacts, ofpacts_len, output, names); + + struct oftrace_recirc_node *recirc_node; + LIST_FOR_EACH_POP (recirc_node, node, &recirc_queue) { ++ if (recirculations++ > 4096) { ++ ds_put_cstr(output, "\n\n"); ++ ds_put_char_multiple(output, '=', 79); ++ ds_put_cstr(output, "\nTrace reached the recirculation limit." ++ " Sopping the trace here."); ++ ds_put_format(output, ++ "\nQueued but not processed: %"PRIuSIZE ++ " recirculations.", ++ ovs_list_size(&recirc_queue) + 1); ++ oftrace_recirc_node_destroy(recirc_node); ++ break; ++ } + ofproto_trace_recirc_node(recirc_node, next_ct_states, output); + ofproto_trace__(ofproto, &recirc_node->flow, recirc_node->packet, + &recirc_queue, ofpacts, ofpacts_len, output, + names); + oftrace_recirc_node_destroy(recirc_node); + } ++ /* Destroy remaining recirculation nodes, if any. */ ++ LIST_FOR_EACH_POP (recirc_node, node, &recirc_queue) { ++ oftrace_recirc_node_destroy(recirc_node); ++ } + } + + void diff --git a/ofproto/ofproto-dpif-upcall.c b/ofproto/ofproto-dpif-upcall.c -index e05ffe312..0fceeba18 100644 +index e05ffe3128..9c6a63d7fb 100644 --- a/ofproto/ofproto-dpif-upcall.c +++ b/ofproto/ofproto-dpif-upcall.c -@@ -47,17 +47,20 @@ +@@ -47,17 +47,21 @@ #define UPCALL_MAX_BATCH 64 #define REVALIDATE_MAX_BATCH 50 @@ -5309,6 +6672,7 @@ index e05ffe312..0fceeba18 100644 COVERAGE_DEFINE(revalidate_missed_dp_flow); +COVERAGE_DEFINE(ukey_dp_change); +COVERAGE_DEFINE(ukey_invalid_stat_reset); ++COVERAGE_DEFINE(ukey_replace_contention); COVERAGE_DEFINE(upcall_flow_limit_hit); COVERAGE_DEFINE(upcall_flow_limit_kill); +COVERAGE_DEFINE(upcall_ukey_contention); @@ -5316,7 +6680,7 @@ index e05ffe312..0fceeba18 100644 /* A thread that reads upcalls from dpif, forwards each upcall's packet, * and possibly sets up a kernel flow as a cache. */ -@@ -287,6 +290,7 @@ struct udpif_key { +@@ -287,6 +291,7 @@ struct udpif_key { struct ovs_mutex mutex; /* Guards the following. */ struct dpif_flow_stats stats OVS_GUARDED; /* Last known stats.*/ @@ -5324,7 +6688,7 @@ index e05ffe312..0fceeba18 100644 long long int created OVS_GUARDED; /* Estimate of creation time. */ uint64_t dump_seq OVS_GUARDED; /* Tracks udpif->dump_seq. */ uint64_t reval_seq OVS_GUARDED; /* Tracks udpif->reval_seq. */ -@@ -780,6 +784,17 @@ udpif_get_n_flows(struct udpif *udpif) +@@ -780,6 +785,17 @@ udpif_get_n_flows(struct udpif *udpif) atomic_store_relaxed(&udpif->n_flows_timestamp, now); dpif_get_dp_stats(udpif->dpif, &stats); flow_count = stats.n_flows; @@ -5342,7 +6706,7 @@ index e05ffe312..0fceeba18 100644 atomic_store_relaxed(&udpif->n_flows, flow_count); ovs_mutex_unlock(&udpif->n_flows_mutex); } else { -@@ -972,7 +987,7 @@ udpif_revalidator(void *arg) +@@ -972,7 +988,7 @@ udpif_revalidator(void *arg) udpif->reval_exit = latch_is_set(&udpif->exit_latch); start_time = time_msec(); @@ -5351,7 +6715,7 @@ index e05ffe312..0fceeba18 100644 bool terse_dump; terse_dump = udpif_use_ufid(udpif); -@@ -981,10 +996,15 @@ udpif_revalidator(void *arg) +@@ -981,10 +997,15 @@ udpif_revalidator(void *arg) } } @@ -5368,7 +6732,16 @@ index e05ffe312..0fceeba18 100644 } if (udpif->reval_exit) { -@@ -1766,6 +1786,7 @@ ukey_create__(const struct nlattr *key, size_t key_len, +@@ -1395,8 +1416,6 @@ upcall_cb(const struct dp_packet *packet, const struct flow *flow, ovs_u128 *ufi + } + + if (upcall.ukey && !ukey_install(udpif, upcall.ukey)) { +- static struct vlog_rate_limit rll = VLOG_RATE_LIMIT_INIT(1, 1); +- VLOG_WARN_RL(&rll, "upcall_cb failure: ukey installation fails"); + error = ENOSPC; + } + out: +@@ -1766,6 +1785,7 @@ ukey_create__(const struct nlattr *key, size_t key_len, ukey->created = ukey->flow_time = time_msec(); memset(&ukey->stats, 0, sizeof ukey->stats); ukey->stats.used = used; @@ -5376,7 +6749,28 @@ index e05ffe312..0fceeba18 100644 ukey->xcache = NULL; ukey->offloaded = false; -@@ -2095,10 +2116,12 @@ ukey_delete(struct umap *umap, struct udpif_key *ukey) +@@ -1893,15 +1913,15 @@ try_ukey_replace(struct umap *umap, struct udpif_key *old_ukey, + transition_ukey(old_ukey, UKEY_DELETED); + transition_ukey(new_ukey, UKEY_VISIBLE); + replaced = true; ++ COVERAGE_INC(upcall_ukey_replace); ++ } else { ++ COVERAGE_INC(handler_duplicate_upcall); + } + ovs_mutex_unlock(&old_ukey->mutex); +- } +- +- if (replaced) { +- COVERAGE_INC(upcall_ukey_replace); + } else { +- COVERAGE_INC(handler_duplicate_upcall); ++ COVERAGE_INC(ukey_replace_contention); + } ++ + return replaced; + } + +@@ -2095,10 +2115,12 @@ ukey_delete(struct umap *umap, struct udpif_key *ukey) } static bool @@ -5391,7 +6785,7 @@ index e05ffe312..0fceeba18 100644 if (!used) { /* Always revalidate the first time a flow is dumped. */ -@@ -2125,8 +2148,12 @@ should_revalidate(const struct udpif *udpif, uint64_t packets, +@@ -2125,8 +2147,12 @@ should_revalidate(const struct udpif *udpif, uint64_t packets, duration = now - used; metric = duration / packets; @@ -5406,7 +6800,7 @@ index e05ffe312..0fceeba18 100644 return true; } return false; -@@ -2302,6 +2329,27 @@ exit: +@@ -2302,6 +2328,27 @@ exit: return result; } @@ -5434,7 +6828,7 @@ index e05ffe312..0fceeba18 100644 /* Verifies that the datapath actions of 'ukey' are still correct, and pushes * 'stats' for it. * -@@ -2324,7 +2372,7 @@ static enum reval_result +@@ -2324,7 +2371,7 @@ static enum reval_result revalidate_ukey(struct udpif *udpif, struct udpif_key *ukey, const struct dpif_flow_stats *stats, struct ofpbuf *odp_actions, uint64_t reval_seq, @@ -5443,7 +6837,7 @@ index e05ffe312..0fceeba18 100644 OVS_REQUIRES(ukey->mutex) { bool need_revalidate = ukey->reval_seq != reval_seq; -@@ -2335,15 +2383,19 @@ revalidate_ukey(struct udpif *udpif, struct udpif_key *ukey, +@@ -2335,15 +2382,19 @@ revalidate_ukey(struct udpif *udpif, struct udpif_key *ukey, push.used = stats->used; push.tcp_flags = stats->tcp_flags; @@ -5470,7 +6864,7 @@ index e05ffe312..0fceeba18 100644 if (!ukey->xcache) { ukey->xcache = xlate_cache_new(); } else { -@@ -2359,7 +2411,7 @@ revalidate_ukey(struct udpif *udpif, struct udpif_key *ukey, +@@ -2359,7 +2410,7 @@ revalidate_ukey(struct udpif *udpif, struct udpif_key *ukey, /* Stats for deleted flows will be attributed upon flow deletion. Skip. */ if (result != UKEY_DELETE) { @@ -5479,7 +6873,7 @@ index e05ffe312..0fceeba18 100644 ukey->stats = *stats; ukey->reval_seq = reval_seq; } -@@ -2455,6 +2507,15 @@ push_dp_ops(struct udpif *udpif, struct ukey_op *ops, size_t n_ops) +@@ -2455,6 +2506,15 @@ push_dp_ops(struct udpif *udpif, struct ukey_op *ops, size_t n_ops) push->tcp_flags = stats->tcp_flags | op->ukey->stats.tcp_flags; push->n_packets = stats->n_packets - op->ukey->stats.n_packets; push->n_bytes = stats->n_bytes - op->ukey->stats.n_bytes; @@ -5495,7 +6889,7 @@ index e05ffe312..0fceeba18 100644 ovs_mutex_unlock(&op->ukey->mutex); } else { push = stats; -@@ -2759,6 +2820,22 @@ revalidate(struct revalidator *revalidator) +@@ -2759,6 +2819,22 @@ revalidate(struct revalidator *revalidator) continue; } @@ -5518,7 +6912,7 @@ index e05ffe312..0fceeba18 100644 already_dumped = ukey->dump_seq == dump_seq; if (already_dumped) { /* The flow has already been handled during this flow dump -@@ -2790,8 +2867,7 @@ revalidate(struct revalidator *revalidator) +@@ -2790,8 +2866,7 @@ revalidate(struct revalidator *revalidator) result = UKEY_DELETE; } else { result = revalidate_ukey(udpif, ukey, &stats, &odp_actions, @@ -5528,6 +6922,14 @@ index e05ffe312..0fceeba18 100644 } ukey->dump_seq = dump_seq; +@@ -2857,6 +2932,7 @@ revalidator_sweep__(struct revalidator *revalidator, bool purge) + /* Handler threads could be holding a ukey lock while it installs a + * new flow, so don't hang around waiting for access to it. */ + if (ovs_mutex_trylock(&ukey->mutex)) { ++ COVERAGE_INC(upcall_ukey_contention); + continue; + } + ukey_state = ukey->state; @@ -2876,7 +2952,7 @@ revalidator_sweep__(struct revalidator *revalidator, bool purge) COVERAGE_INC(revalidate_missed_dp_flow); memcpy(&stats, &ukey->stats, sizeof stats); @@ -5558,7 +6960,7 @@ index e05ffe312..0fceeba18 100644 unixctl_command_reply(conn, ""); } diff --git a/ofproto/ofproto-dpif-xlate-cache.c b/ofproto/ofproto-dpif-xlate-cache.c -index 9224ee2e6..2e1fcb3a6 100644 +index 9224ee2e6d..2e1fcb3a6f 100644 --- a/ofproto/ofproto-dpif-xlate-cache.c +++ b/ofproto/ofproto-dpif-xlate-cache.c @@ -125,7 +125,7 @@ xlate_push_stats_entry(struct xc_entry *entry, @@ -5571,7 +6973,7 @@ index 9224ee2e6..2e1fcb3a6 100644 static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 5); VLOG_WARN_RL(&rl, "xcache LEARN action execution failed."); diff --git a/ofproto/ofproto-dpif-xlate.c b/ofproto/ofproto-dpif-xlate.c -index a9cf3cbee..18ff9e5a4 100644 +index a9cf3cbee0..ac953d70d8 100644 --- a/ofproto/ofproto-dpif-xlate.c +++ b/ofproto/ofproto-dpif-xlate.c @@ -66,6 +66,7 @@ @@ -5725,7 +7127,16 @@ index a9cf3cbee..18ff9e5a4 100644 xport = xport_lookup(xcfg, tnl_port_should_receive(flow) ? tnl_port_receive(flow) : odp_port_to_ofport(backer, flow->in_port.odp_port)); -@@ -3906,20 +3999,17 @@ static void +@@ -3681,6 +3774,8 @@ native_tunnel_output(struct xlate_ctx *ctx, const struct xport *xport, + + if (flow->tunnel.ip_src) { + in6_addr_set_mapped_ipv4(&s_ip6, flow->tunnel.ip_src); ++ } else if (ipv6_addr_is_set(&flow->tunnel.ipv6_src)) { ++ s_ip6 = flow->tunnel.ipv6_src; + } + + err = tnl_route_lookup_flow(ctx, flow, &d_ip6, &s_ip6, &out_dev); +@@ -3906,20 +4001,17 @@ static void patch_port_output(struct xlate_ctx *ctx, const struct xport *in_dev, struct xport *out_dev, bool is_last_action) { @@ -5753,7 +7164,7 @@ index a9cf3cbee..18ff9e5a4 100644 flow->in_port.ofp_port = out_dev->ofp_port; flow->metadata = htonll(0); memset(&flow->tunnel, 0, sizeof flow->tunnel); -@@ -3958,14 +4048,15 @@ patch_port_output(struct xlate_ctx *ctx, const struct xport *in_dev, +@@ -3958,14 +4050,15 @@ patch_port_output(struct xlate_ctx *ctx, const struct xport *in_dev, } else { /* Forwarding is disabled by STP and RSTP. Let OFPP_NORMAL and * the learning action look at the packet, then drop it. */ @@ -5771,7 +7182,7 @@ index a9cf3cbee..18ff9e5a4 100644 ctx->odp_actions->size = old_size; /* Undo changes that may have been done for freezing. */ -@@ -3977,18 +4068,15 @@ patch_port_output(struct xlate_ctx *ctx, const struct xport *in_dev, +@@ -3977,18 +4070,15 @@ patch_port_output(struct xlate_ctx *ctx, const struct xport *in_dev, if (independent_mirrors) { ctx->mirrors = old_mirrors; } @@ -5794,7 +7205,7 @@ index a9cf3cbee..18ff9e5a4 100644 /* The out bridge popping MPLS should have no effect on the original * bridge. */ -@@ -4238,7 +4326,7 @@ compose_output_action__(struct xlate_ctx *ctx, ofp_port_t ofp_port, +@@ -4238,7 +4328,7 @@ compose_output_action__(struct xlate_ctx *ctx, ofp_port_t ofp_port, const struct xport *xport = get_ofp_port(ctx->xbridge, ofp_port); struct flow_wildcards *wc = ctx->wc; struct flow *flow = &ctx->xin->flow; @@ -5803,7 +7214,7 @@ index a9cf3cbee..18ff9e5a4 100644 union flow_vlan_hdr flow_vlans[FLOW_MAX_VLAN_HEADERS]; uint8_t flow_nw_tos; odp_port_t out_port, odp_port, odp_tnl_port; -@@ -4252,7 +4340,6 @@ compose_output_action__(struct xlate_ctx *ctx, ofp_port_t ofp_port, +@@ -4252,7 +4342,6 @@ compose_output_action__(struct xlate_ctx *ctx, ofp_port_t ofp_port, /* If 'struct flow' gets additional metadata, we'll need to zero it out * before traversing a patch port. */ BUILD_ASSERT_DECL(FLOW_WC_SEQ == 42); @@ -5811,7 +7222,7 @@ index a9cf3cbee..18ff9e5a4 100644 if (!check_output_prerequisites(ctx, xport, flow, check_stp)) { return; -@@ -4296,7 +4383,7 @@ compose_output_action__(struct xlate_ctx *ctx, ofp_port_t ofp_port, +@@ -4296,7 +4385,7 @@ compose_output_action__(struct xlate_ctx *ctx, ofp_port_t ofp_port, * the Logical (tunnel) Port are not visible for any further * matches, while explicit set actions on tunnel metadata are. */ @@ -5820,7 +7231,7 @@ index a9cf3cbee..18ff9e5a4 100644 odp_port = tnl_port_send(xport->ofport, flow, ctx->wc); if (odp_port == ODPP_NONE) { xlate_report(ctx, OFT_WARN, "Tunneling decided against output"); -@@ -4327,7 +4414,7 @@ compose_output_action__(struct xlate_ctx *ctx, ofp_port_t ofp_port, +@@ -4327,7 +4416,7 @@ compose_output_action__(struct xlate_ctx *ctx, ofp_port_t ofp_port, tnl_type = tnl_port_get_type(xport->ofport); commit_odp_tunnel_action(flow, &ctx->base_flow, ctx->odp_actions, tnl_type); @@ -5829,7 +7240,7 @@ index a9cf3cbee..18ff9e5a4 100644 } } else { odp_port = xport->odp_port; -@@ -4371,7 +4458,8 @@ compose_output_action__(struct xlate_ctx *ctx, ofp_port_t ofp_port, +@@ -4371,7 +4460,8 @@ compose_output_action__(struct xlate_ctx *ctx, ofp_port_t ofp_port, /* Output to native tunnel port. */ native_tunnel_output(ctx, xport, flow, odp_port, truncate, is_last_action); @@ -5839,7 +7250,7 @@ index a9cf3cbee..18ff9e5a4 100644 } else if (terminate_native_tunnel(ctx, xport, flow, wc, &odp_tnl_port)) { -@@ -4414,7 +4502,7 @@ compose_output_action__(struct xlate_ctx *ctx, ofp_port_t ofp_port, +@@ -4414,7 +4504,7 @@ compose_output_action__(struct xlate_ctx *ctx, ofp_port_t ofp_port, xport->xbundle)); } @@ -5848,7 +7259,7 @@ index a9cf3cbee..18ff9e5a4 100644 /* Restore flow */ memcpy(flow->vlans, flow_vlans, sizeof flow->vlans); flow->nw_tos = flow_nw_tos; -@@ -4422,6 +4510,7 @@ compose_output_action__(struct xlate_ctx *ctx, ofp_port_t ofp_port, +@@ -4422,6 +4512,7 @@ compose_output_action__(struct xlate_ctx *ctx, ofp_port_t ofp_port, flow->dl_src = flow_dl_src; flow->packet_type = flow_packet_type; flow->dl_type = flow_dl_type; @@ -5856,7 +7267,112 @@ index a9cf3cbee..18ff9e5a4 100644 } static void -@@ -5211,6 +5300,7 @@ compose_dec_ttl(struct xlate_ctx *ctx, struct ofpact_cnt_ids *ids) +@@ -4943,10 +5034,37 @@ put_controller_user_action(struct xlate_ctx *ctx, + bool dont_send, bool continuation, + uint32_t recirc_id, int len, + enum ofp_packet_in_reason reason, ++ uint32_t provider_meter_id, + uint16_t controller_id) + { + struct user_action_cookie cookie; + ++ /* If the controller action didn't request a meter (indicated by a ++ * 'meter_id' argument other than NX_CTLR_NO_METER), see if one was ++ * configured through the "controller" virtual meter. ++ * ++ * Internally, ovs-vswitchd uses UINT32_MAX to indicate no meter is ++ * configured. */ ++ uint32_t meter_id; ++ if (provider_meter_id == UINT32_MAX) { ++ meter_id = ctx->xbridge->ofproto->up.controller_meter_id; ++ } else { ++ meter_id = provider_meter_id; ++ } ++ ++ size_t offset; ++ size_t ac_offset; ++ if (meter_id != UINT32_MAX) { ++ /* If controller meter is configured, generate ++ * clone(meter,userspace) action. */ ++ offset = nl_msg_start_nested(ctx->odp_actions, OVS_ACTION_ATTR_SAMPLE); ++ nl_msg_put_u32(ctx->odp_actions, OVS_SAMPLE_ATTR_PROBABILITY, ++ UINT32_MAX); ++ ac_offset = nl_msg_start_nested(ctx->odp_actions, ++ OVS_SAMPLE_ATTR_ACTIONS); ++ nl_msg_put_u32(ctx->odp_actions, OVS_ACTION_ATTR_METER, meter_id); ++ } ++ + memset(&cookie, 0, sizeof cookie); + cookie.type = USER_ACTION_COOKIE_CONTROLLER; + cookie.ofp_in_port = OFPP_NONE, +@@ -4964,6 +5082,11 @@ put_controller_user_action(struct xlate_ctx *ctx, + uint32_t pid = dpif_port_get_pid(ctx->xbridge->dpif, odp_port); + odp_put_userspace_action(pid, &cookie, sizeof cookie, ODPP_NONE, + false, ctx->odp_actions, NULL); ++ ++ if (meter_id != UINT32_MAX) { ++ nl_msg_end_nested(ctx->odp_actions, ac_offset); ++ nl_msg_end_nested(ctx->odp_actions, offset); ++ } + } + + static void +@@ -5008,32 +5131,6 @@ xlate_controller_action(struct xlate_ctx *ctx, int len, + } + recirc_refs_add(&ctx->xout->recircs, recirc_id); + +- /* If the controller action didn't request a meter (indicated by a +- * 'meter_id' argument other than NX_CTLR_NO_METER), see if one was +- * configured through the "controller" virtual meter. +- * +- * Internally, ovs-vswitchd uses UINT32_MAX to indicate no meter is +- * configured. */ +- uint32_t meter_id; +- if (provider_meter_id == UINT32_MAX) { +- meter_id = ctx->xbridge->ofproto->up.controller_meter_id; +- } else { +- meter_id = provider_meter_id; +- } +- +- size_t offset; +- size_t ac_offset; +- if (meter_id != UINT32_MAX) { +- /* If controller meter is configured, generate clone(meter, userspace) +- * action. */ +- offset = nl_msg_start_nested(ctx->odp_actions, OVS_ACTION_ATTR_SAMPLE); +- nl_msg_put_u32(ctx->odp_actions, OVS_SAMPLE_ATTR_PROBABILITY, +- UINT32_MAX); +- ac_offset = nl_msg_start_nested(ctx->odp_actions, +- OVS_SAMPLE_ATTR_ACTIONS); +- nl_msg_put_u32(ctx->odp_actions, OVS_ACTION_ATTR_METER, meter_id); +- } +- + /* Generate the datapath flows even if we don't send the packet-in + * so that debugging more closely represents normal state. */ + bool dont_send = false; +@@ -5041,12 +5138,7 @@ xlate_controller_action(struct xlate_ctx *ctx, int len, + dont_send = true; + } + put_controller_user_action(ctx, dont_send, false, recirc_id, len, +- reason, controller_id); +- +- if (meter_id != UINT32_MAX) { +- nl_msg_end_nested(ctx->odp_actions, ac_offset); +- nl_msg_end_nested(ctx->odp_actions, offset); +- } ++ reason, provider_meter_id, controller_id); + } + + /* Creates a frozen state, and allocates a unique recirc id for the given +@@ -5098,6 +5190,7 @@ finish_freezing__(struct xlate_ctx *ctx, uint8_t table) + put_controller_user_action(ctx, false, true, recirc_id, + ctx->pause->max_len, + ctx->pause->reason, ++ ctx->pause->provider_meter_id, + ctx->pause->controller_id); + } else { + if (ctx->recirc_update_dp_hash) { +@@ -5211,6 +5304,7 @@ compose_dec_ttl(struct xlate_ctx *ctx, struct ofpact_cnt_ids *ids) } ctx->wc->masks.nw_ttl = 0xff; @@ -5864,7 +7380,7 @@ index a9cf3cbee..18ff9e5a4 100644 if (flow->nw_ttl > 1) { flow->nw_ttl--; return false; -@@ -5399,15 +5489,15 @@ xlate_output_reg_action(struct xlate_ctx *ctx, +@@ -5399,15 +5493,15 @@ xlate_output_reg_action(struct xlate_ctx *ctx, { uint64_t port = mf_get_subfield(&or->src, &ctx->xin->flow); if (port <= UINT16_MAX) { @@ -5885,7 +7401,7 @@ index a9cf3cbee..18ff9e5a4 100644 } else { xlate_report(ctx, OFT_WARN, "output port %"PRIu64" is out of range", port); -@@ -5616,8 +5706,16 @@ xlate_learn_action(struct xlate_ctx *ctx, const struct ofpact_learn *learn) +@@ -5616,8 +5710,16 @@ xlate_learn_action(struct xlate_ctx *ctx, const struct ofpact_learn *learn) if (!error) { bool success = true; if (ctx->xin->allow_side_effects) { @@ -5903,7 +7419,7 @@ index a9cf3cbee..18ff9e5a4 100644 } else if (learn->limit) { if (!ofm->temp_rule || ofm->temp_rule->state != RULE_INSERTED) { -@@ -5748,13 +5846,15 @@ xlate_sample_action(struct xlate_ctx *ctx, +@@ -5748,13 +5850,15 @@ xlate_sample_action(struct xlate_ctx *ctx, struct flow *flow = &ctx->xin->flow; tnl_port_send(xport->ofport, flow, ctx->wc); if (!ovs_native_tunneling_is_on(ctx->xbridge->ofproto)) { @@ -5921,7 +7437,7 @@ index a9cf3cbee..18ff9e5a4 100644 } } else { xlate_report_error(ctx, -@@ -5864,21 +5964,12 @@ clone_xlate_actions(const struct ofpact *actions, size_t actions_len, +@@ -5864,21 +5968,12 @@ clone_xlate_actions(const struct ofpact *actions, size_t actions_len, struct xlate_ctx *ctx, bool is_last_action, bool group_bucket_action OVS_UNUSED) { @@ -5946,7 +7462,7 @@ index a9cf3cbee..18ff9e5a4 100644 do_xlate_actions(actions, actions_len, ctx, is_last_action, false); if (!ctx->freezing) { xlate_action_set(ctx); -@@ -5893,7 +5984,8 @@ clone_xlate_actions(const struct ofpact *actions, size_t actions_len, +@@ -5893,7 +5988,8 @@ clone_xlate_actions(const struct ofpact *actions, size_t actions_len, * avoid emitting those actions twice. Once inside * the clone, another time for the action after clone. */ xlate_commit_actions(ctx); @@ -5956,7 +7472,7 @@ index a9cf3cbee..18ff9e5a4 100644 bool old_was_mpls = ctx->was_mpls; bool old_conntracked = ctx->conntracked; -@@ -5950,14 +6042,10 @@ dp_clone_done: +@@ -5950,14 +6046,10 @@ dp_clone_done: ctx->was_mpls = old_was_mpls; /* Restore the 'base_flow' for the next action. */ @@ -5973,7 +7489,7 @@ index a9cf3cbee..18ff9e5a4 100644 } static void -@@ -6333,8 +6421,8 @@ compose_conntrack_action(struct xlate_ctx *ctx, struct ofpact_conntrack *ofc, +@@ -6333,8 +6425,8 @@ compose_conntrack_action(struct xlate_ctx *ctx, struct ofpact_conntrack *ofc, { uint16_t zone; if (ofc->zone_src.field) { @@ -5984,7 +7500,7 @@ index a9cf3cbee..18ff9e5a4 100644 zone = mf_get_subfield(&ofc->zone_src, &ctx->xin->flow); if (ctx->xin->frozen_state) { -@@ -6344,12 +6432,13 @@ compose_conntrack_action(struct xlate_ctx *ctx, struct ofpact_conntrack *ofc, +@@ -6344,12 +6436,13 @@ compose_conntrack_action(struct xlate_ctx *ctx, struct ofpact_conntrack *ofc, * which will invalidate the megaflow with old the recirc_id. */ if (!mf_is_frozen_metadata(ofc->zone_src.field)) { @@ -6000,7 +7516,7 @@ index a9cf3cbee..18ff9e5a4 100644 } else { zone = ofc->zone_imm; } -@@ -6439,16 +6528,16 @@ xlate_check_pkt_larger(struct xlate_ctx *ctx, +@@ -6439,16 +6532,16 @@ xlate_check_pkt_larger(struct xlate_ctx *ctx, const struct ofpact *remaining_acts, size_t remaining_acts_len) { @@ -6021,7 +7537,7 @@ index a9cf3cbee..18ff9e5a4 100644 &ctx->xin->flow); /* If datapath doesn't support check_pkt_len action, then set the * SLOW_ACTION flag. If we don't set SLOW_ACTION, we -@@ -6458,22 +6547,17 @@ xlate_check_pkt_larger(struct xlate_ctx *ctx, +@@ -6458,22 +6551,17 @@ xlate_check_pkt_larger(struct xlate_ctx *ctx, * the packet length. This results in wrong actions being applied. */ ctx->xout->slow |= SLOW_ACTION; @@ -6049,7 +7565,7 @@ index a9cf3cbee..18ff9e5a4 100644 bool old_was_mpls = ctx->was_mpls; bool old_conntracked = ctx->conntracked; -@@ -6483,8 +6567,8 @@ xlate_check_pkt_larger(struct xlate_ctx *ctx, +@@ -6483,8 +6571,8 @@ xlate_check_pkt_larger(struct xlate_ctx *ctx, check_pkt_larger->pkt_len); size_t offset_attr = nl_msg_start_nested( ctx->odp_actions, OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_GREATER); @@ -6060,7 +7576,7 @@ index a9cf3cbee..18ff9e5a4 100644 do_xlate_actions(remaining_acts, remaining_acts_len, ctx, true, false); if (!ctx->freezing) { xlate_action_set(ctx); -@@ -6494,10 +6578,10 @@ xlate_check_pkt_larger(struct xlate_ctx *ctx, +@@ -6494,10 +6582,10 @@ xlate_check_pkt_larger(struct xlate_ctx *ctx, } nl_msg_end_nested(ctx->odp_actions, offset_attr); @@ -6073,7 +7589,7 @@ index a9cf3cbee..18ff9e5a4 100644 /* If the flow translation for the IF_GREATER case requires freezing, * then ctx->exit would be true. Reset to false so that we can -@@ -6508,8 +6592,8 @@ xlate_check_pkt_larger(struct xlate_ctx *ctx, +@@ -6508,8 +6596,8 @@ xlate_check_pkt_larger(struct xlate_ctx *ctx, offset_attr = nl_msg_start_nested( ctx->odp_actions, OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_LESS_EQUAL); @@ -6084,7 +7600,7 @@ index a9cf3cbee..18ff9e5a4 100644 do_xlate_actions(remaining_acts, remaining_acts_len, ctx, true, false); if (!ctx->freezing) { xlate_action_set(ctx); -@@ -6520,15 +6604,12 @@ xlate_check_pkt_larger(struct xlate_ctx *ctx, +@@ -6520,15 +6608,12 @@ xlate_check_pkt_larger(struct xlate_ctx *ctx, nl_msg_end_nested(ctx->odp_actions, offset_attr); nl_msg_end_nested(ctx->odp_actions, offset); @@ -6103,7 +7619,7 @@ index a9cf3cbee..18ff9e5a4 100644 } static void -@@ -6979,6 +7060,132 @@ xlate_ofpact_unroll_xlate(struct xlate_ctx *ctx, +@@ -6979,6 +7064,132 @@ xlate_ofpact_unroll_xlate(struct xlate_ctx *ctx, "cookie=%#"PRIx64, a->rule_table_id, a->rule_cookie); } @@ -6236,20 +7752,20 @@ index a9cf3cbee..18ff9e5a4 100644 static void do_xlate_actions(const struct ofpact *ofpacts, size_t ofpacts_len, struct xlate_ctx *ctx, bool is_last_action, -@@ -7020,21 +7227,10 @@ do_xlate_actions(const struct ofpact *ofpacts, size_t ofpacts_len, +@@ -7020,21 +7231,10 @@ do_xlate_actions(const struct ofpact *ofpacts, size_t ofpacts_len, break; } - if (OVS_UNLIKELY(ctx->xin->trace)) { - struct ofputil_port_map map = OFPUTIL_PORT_MAP_INITIALIZER(&map); -+ reset_mirror_ctx(ctx, flow, a); - +- - if (ctx->xin->names) { - struct ofproto_dpif *ofprotop; - ofprotop = ofproto_dpif_lookup_by_name(ctx->xbridge->name); - ofproto_append_ports_to_map(&map, ofprotop->up.ports); - } -- ++ reset_mirror_ctx(ctx, flow, a); + - struct ds s = DS_EMPTY_INITIALIZER; - struct ofpact_format_params fp = { .s = &s, .port_map = &map }; - ofpacts_format(a, OFPACT_ALIGN(a->len), &fp); @@ -6261,7 +7777,7 @@ index a9cf3cbee..18ff9e5a4 100644 } switch (a->type) { -@@ -7128,6 +7324,7 @@ do_xlate_actions(const struct ofpact *ofpacts, size_t ofpacts_len, +@@ -7128,6 +7328,7 @@ do_xlate_actions(const struct ofpact *ofpacts, size_t ofpacts_len, case OFPACT_SET_IPV4_SRC: if (flow->dl_type == htons(ETH_TYPE_IP)) { memset(&wc->masks.nw_src, 0xff, sizeof wc->masks.nw_src); @@ -6269,7 +7785,7 @@ index a9cf3cbee..18ff9e5a4 100644 flow->nw_src = ofpact_get_SET_IPV4_SRC(a)->ipv4; } break; -@@ -7135,12 +7332,14 @@ do_xlate_actions(const struct ofpact *ofpacts, size_t ofpacts_len, +@@ -7135,12 +7336,14 @@ do_xlate_actions(const struct ofpact *ofpacts, size_t ofpacts_len, case OFPACT_SET_IPV4_DST: if (flow->dl_type == htons(ETH_TYPE_IP)) { memset(&wc->masks.nw_dst, 0xff, sizeof wc->masks.nw_dst); @@ -6284,7 +7800,7 @@ index a9cf3cbee..18ff9e5a4 100644 wc->masks.nw_tos |= IP_DSCP_MASK; flow->nw_tos &= ~IP_DSCP_MASK; flow->nw_tos |= ofpact_get_SET_IP_DSCP(a)->dscp; -@@ -7149,6 +7348,7 @@ do_xlate_actions(const struct ofpact *ofpacts, size_t ofpacts_len, +@@ -7149,6 +7352,7 @@ do_xlate_actions(const struct ofpact *ofpacts, size_t ofpacts_len, case OFPACT_SET_IP_ECN: if (is_ip_any(flow)) { @@ -6292,7 +7808,7 @@ index a9cf3cbee..18ff9e5a4 100644 wc->masks.nw_tos |= IP_ECN_MASK; flow->nw_tos &= ~IP_ECN_MASK; flow->nw_tos |= ofpact_get_SET_IP_ECN(a)->ecn; -@@ -7157,6 +7357,7 @@ do_xlate_actions(const struct ofpact *ofpacts, size_t ofpacts_len, +@@ -7157,6 +7361,7 @@ do_xlate_actions(const struct ofpact *ofpacts, size_t ofpacts_len, case OFPACT_SET_IP_TTL: if (is_ip_any(flow)) { @@ -6300,7 +7816,7 @@ index a9cf3cbee..18ff9e5a4 100644 wc->masks.nw_ttl = 0xff; flow->nw_ttl = ofpact_get_SET_IP_TTL(a)->ttl; } -@@ -7224,6 +7425,7 @@ do_xlate_actions(const struct ofpact *ofpacts, size_t ofpacts_len, +@@ -7224,6 +7429,7 @@ do_xlate_actions(const struct ofpact *ofpacts, size_t ofpacts_len, /* Set the field only if the packet actually has it. */ if (mf_are_prereqs_ok(mf, flow, wc)) { @@ -6308,7 +7824,7 @@ index a9cf3cbee..18ff9e5a4 100644 mf_mask_field_masked(mf, ofpact_set_field_mask(set_field), wc); mf_set_flow_value_masked(mf, set_field->value, ofpact_set_field_mask(set_field), -@@ -7280,6 +7482,7 @@ do_xlate_actions(const struct ofpact *ofpacts, size_t ofpacts_len, +@@ -7280,6 +7486,7 @@ do_xlate_actions(const struct ofpact *ofpacts, size_t ofpacts_len, case OFPACT_DEC_TTL: wc->masks.nw_ttl = 0xff; @@ -6317,7 +7833,7 @@ index a9cf3cbee..18ff9e5a4 100644 return; } diff --git a/ofproto/ofproto-dpif.c b/ofproto/ofproto-dpif.c -index f87e27a8c..ba5706f6a 100644 +index f87e27a8cd..ff520c150e 100644 --- a/ofproto/ofproto-dpif.c +++ b/ofproto/ofproto-dpif.c @@ -714,12 +714,6 @@ close_dpif_backer(struct dpif_backer *backer, bool del) @@ -6412,7 +7928,35 @@ index f87e27a8c..ba5706f6a 100644 } else if (del) { /* The underlying device is already deleted (e.g. tunctl -d). * Calling dpif_port_remove to do local cleanup for the netdev */ -@@ -4886,7 +4880,7 @@ packet_xlate(struct ofproto *ofproto_, struct ofproto_packet_out *opo) +@@ -3908,15 +3902,21 @@ port_query_by_name(const struct ofproto *ofproto_, const char *devname, + int error; + + if (sset_contains(&ofproto->ghost_ports, devname)) { +- const char *type = netdev_get_type_from_name(devname); +- + /* We may be called before ofproto->up.port_by_name is populated with + * the appropriate ofport. For this reason, we must get the name and +- * type from the netdev layer directly. */ +- if (type) { +- const struct ofport *ofport; ++ * type from the netdev layer directly. ++ * However, when a port deleted, the corresponding netdev is also ++ * removed from netdev_shash. netdev_get_type_from_name returns NULL ++ * in such case and we should try to get type from ofport->netdev. */ ++ const char *type = netdev_get_type_from_name(devname); ++ const struct ofport *ofport = ++ shash_find_data(&ofproto->up.port_by_name, devname); ++ ++ if (!type && ofport && ofport->netdev) { ++ type = netdev_get_type(ofport->netdev); ++ } + +- ofport = shash_find_data(&ofproto->up.port_by_name, devname); ++ if (type) { + ofproto_port->ofp_port = ofport ? ofport->ofp_port : OFPP_NONE; + ofproto_port->name = xstrdup(devname); + ofproto_port->type = xstrdup(type); +@@ -4886,7 +4886,7 @@ packet_xlate(struct ofproto *ofproto_, struct ofproto_packet_out *opo) if (entry->type == XC_LEARN) { struct ofproto_flow_mod *ofm = entry->learn.ofm; @@ -6422,7 +7966,7 @@ index f87e27a8c..ba5706f6a 100644 goto error_out; } diff --git a/ofproto/ofproto-provider.h b/ofproto/ofproto-provider.h -index a84ddc1d0..9f7b8b6e8 100644 +index a84ddc1d06..9f7b8b6e83 100644 --- a/ofproto/ofproto-provider.h +++ b/ofproto/ofproto-provider.h @@ -541,6 +541,11 @@ extern unsigned ofproto_max_revalidator; @@ -6452,7 +7996,7 @@ index a84ddc1d0..9f7b8b6e8 100644 OVS_REQUIRES(ofproto_mutex); void ofproto_flow_mod_learn_revert(struct ofproto_flow_mod *ofm) diff --git a/ofproto/ofproto.c b/ofproto/ofproto.c -index 17f636ed9..9a2f7f801 100644 +index 17f636ed9d..9a2f7f8014 100644 --- a/ofproto/ofproto.c +++ b/ofproto/ofproto.c @@ -311,6 +311,7 @@ unsigned ofproto_flow_limit = OFPROTO_FLOW_LIMIT_DEFAULT; @@ -6593,7 +8137,7 @@ index 17f636ed9..9a2f7f801 100644 } } else { diff --git a/ofproto/ofproto.h b/ofproto/ofproto.h -index 4e15167ab..fa7973ac7 100644 +index 4e15167ab7..fa7973ac72 100644 --- a/ofproto/ofproto.h +++ b/ofproto/ofproto.h @@ -311,6 +311,7 @@ int ofproto_port_dump_done(struct ofproto_port_dump *); @@ -6612,8 +8156,66 @@ index 4e15167ab..fa7973ac7 100644 void ofproto_set_forward_bpdu(struct ofproto *, bool forward_bpdu); void ofproto_set_mac_table_config(struct ofproto *, unsigned idle_time, size_t max_entries); +diff --git a/ofproto/tunnel.c b/ofproto/tunnel.c +index 3455ed233b..80ddee78ac 100644 +--- a/ofproto/tunnel.c ++++ b/ofproto/tunnel.c +@@ -432,6 +432,7 @@ tnl_port_send(const struct ofport_dpif *ofport, struct flow *flow, + flow->tunnel.ipv6_dst = in6addr_any; + } + } ++ flow->tunnel.tp_src = 0; /* Do not carry from a previous tunnel. */ + flow->tunnel.tp_dst = cfg->dst_port; + if (!cfg->out_key_flow) { + flow->tunnel.tun_id = cfg->out_key; +diff --git a/ovsdb/automake.mk b/ovsdb/automake.mk +index eba713bb6d..d484fe9deb 100644 +--- a/ovsdb/automake.mk ++++ b/ovsdb/automake.mk +@@ -114,11 +114,13 @@ $(OVSIDL_BUILT): ovsdb/ovsdb-idlc.in python/ovs/dirs.py + + # ovsdb-doc + EXTRA_DIST += ovsdb/ovsdb-doc ++FLAKE8_PYFILES += ovsdb/ovsdb-doc + OVSDB_DOC = $(run_python) $(srcdir)/ovsdb/ovsdb-doc + ovsdb/ovsdb-doc: python/ovs/dirs.py + + # ovsdb-dot + EXTRA_DIST += ovsdb/ovsdb-dot.in ovsdb/dot2pic ++FLAKE8_PYFILES += ovsdb/ovsdb-dot.in ovsdb/dot2pic + noinst_SCRIPTS += ovsdb/ovsdb-dot + CLEANFILES += ovsdb/ovsdb-dot + OVSDB_DOT = $(run_python) $(srcdir)/ovsdb/ovsdb-dot.in +diff --git a/ovsdb/dot2pic b/ovsdb/dot2pic +index 2f858e19d5..3db6444de6 100755 +--- a/ovsdb/dot2pic ++++ b/ovsdb/dot2pic +@@ -17,6 +17,7 @@ + import getopt + import sys + ++ + def dot2pic(src, dst): + scale = 1.0 + while True: +@@ -49,8 +50,8 @@ def dot2pic(src, dst): + dst.write("box at %f,%f wid %f height %f\n" + % (x, y, width, height)) + elif command == 'edge': +- tail = words[1] +- head = words[2] ++ # tail = words[1] ++ # head = words[2] + n = int(words[3]) + + # Extract x,y coordinates. +@@ -114,4 +115,3 @@ else: + if font_scale: + print(".ps %+d" % font_scale) + print(".PE") +- diff --git a/ovsdb/execution.c b/ovsdb/execution.c -index f9b8067d0..5587ef96f 100644 +index f9b8067d02..5587ef96fe 100644 --- a/ovsdb/execution.c +++ b/ovsdb/execution.c @@ -320,7 +320,7 @@ parse_row(const struct json *json, const struct ovsdb_table *table, @@ -6635,7 +8237,7 @@ index f9b8067d0..5587ef96f 100644 ovsdb_row_destroy(row); break; diff --git a/ovsdb/file.c b/ovsdb/file.c -index fdc289ad1..a3273229b 100644 +index fdc289ad1b..a3273229b6 100644 --- a/ovsdb/file.c +++ b/ovsdb/file.c @@ -107,13 +107,18 @@ ovsdb_file_update_row_from_json(struct ovsdb_row *row, bool converting, @@ -6671,8 +8273,20 @@ index fdc289ad1..a3273229b 100644 if (error) { ovsdb_row_destroy(new); } else { +diff --git a/ovsdb/jsonrpc-server.c b/ovsdb/jsonrpc-server.c +index 17868f5b72..5d10f54293 100644 +--- a/ovsdb/jsonrpc-server.c ++++ b/ovsdb/jsonrpc-server.c +@@ -215,6 +215,7 @@ ovsdb_jsonrpc_default_options(const char *target) + options->probe_interval = (stream_or_pstream_needs_probes(target) + ? RECONNECT_DEFAULT_PROBE_INTERVAL + : 0); ++ options->dscp = DSCP_DEFAULT; + return options; + } + diff --git a/ovsdb/log.c b/ovsdb/log.c -index e42f00246..fff7c6ba1 100644 +index e42f002464..fff7c6ba10 100644 --- a/ovsdb/log.c +++ b/ovsdb/log.c @@ -552,6 +552,23 @@ ovsdb_log_truncate(struct ovsdb_log *file) @@ -6700,7 +8314,7 @@ index e42f00246..fff7c6ba1 100644 * 'data' with a data line (each ending with a new-line). To write the record * to a file, write 'header' followed by 'data'. diff --git a/ovsdb/log.h b/ovsdb/log.h -index 90714ea13..63e5681a0 100644 +index 90714ea131..63e5681a0b 100644 --- a/ovsdb/log.h +++ b/ovsdb/log.h @@ -66,6 +66,9 @@ struct ovsdb_error *ovsdb_log_read(struct ovsdb_log *, struct json **) @@ -6714,7 +8328,7 @@ index 90714ea13..63e5681a0 100644 struct ds *header, struct ds *data); diff --git a/ovsdb/monitor.c b/ovsdb/monitor.c -index 191befcae..bf5d083cc 100644 +index 191befcae3..bf5d083cc5 100644 --- a/ovsdb/monitor.c +++ b/ovsdb/monitor.c @@ -474,6 +474,7 @@ ovsdb_monitor_add_column(struct ovsdb_monitor *dbmon, @@ -6757,20 +8371,276 @@ index 191befcae..bf5d083cc 100644 dbmon->new_change_set = NULL; } diff --git a/ovsdb/ovsdb-doc b/ovsdb/ovsdb-doc -index 10d0c0c13..099770d25 100755 +index 10d0c0c134..2edf487a28 100755 --- a/ovsdb/ovsdb-doc +++ b/ovsdb/ovsdb-doc -@@ -24,7 +24,7 @@ import ovs.json +@@ -14,9 +14,7 @@ + # See the License for the specific language governing permissions and + # limitations under the License. + +-from datetime import date + import getopt +-import os + import sys + import xml.dom.minidom + +@@ -24,10 +22,13 @@ import ovs.json from ovs.db import error import ovs.db.schema -from build.nroff import * -+from ovs_build_helpers.nroff import * ++from ovs_build_helpers.nroff import block_xml_to_nroff ++from ovs_build_helpers.nroff import escape_nroff_literal ++from ovs_build_helpers.nroff import text_to_nroff + + argv0 = sys.argv[0] + ++ + def typeAndConstraintsToNroff(column): + type = column.type.toEnglish(escape_nroff_literal) + constraints = column.type.constraintsToEnglish(escape_nroff_literal, +@@ -38,6 +39,7 @@ def typeAndConstraintsToNroff(column): + type += " (must be unique within table)" + return type + ++ + def columnGroupToNroff(table, groupXml, documented_columns): + introNodes = [] + columnNodes = [] +@@ -49,7 +51,10 @@ def columnGroupToNroff(table, groupXml, documented_columns): + if (columnNodes + and not (node.nodeType == node.TEXT_NODE + and node.data.isspace())): +- raise error.Error("text follows or inside : %s" % node) ++ raise error.Error( ++ "text follows or inside : %s" ++ % node ++ ) + introNodes += [node] + + summary = [] +@@ -65,15 +70,9 @@ def columnGroupToNroff(table, groupXml, documented_columns): + if node.hasAttribute('type'): + type_string = node.attributes['type'].nodeValue + type_json = ovs.json.from_string(str(type_string)) +- # py2 -> py3 means str -> bytes and unicode -> str +- try: +- if type(type_json) in (str, unicode): +- raise error.Error("%s %s:%s has invalid 'type': %s" +- % (table.name, name, key, type_json)) +- except: +- if type(type_json) in (bytes, str): +- raise error.Error("%s %s:%s has invalid 'type': %s" +- % (table.name, name, key, type_json)) ++ if type(type_json) in (bytes, str): ++ raise error.Error("%s %s:%s has invalid 'type': %s" ++ % (table.name, name, key, type_json)) + type_ = ovs.db.types.BaseType.from_json(type_json) + else: + type_ = column.type.value +@@ -91,10 +90,11 @@ def columnGroupToNroff(table, groupXml, documented_columns): + else: + if type_.type != column.type.value.type: + type_english = type_.toEnglish() ++ typeNroff += ", containing " + if type_english[0] in 'aeiou': +- typeNroff += ", containing an %s" % type_english ++ typeNroff += "an %s" % type_english + else: +- typeNroff += ", containing a %s" % type_english ++ typeNroff += "a %s" % type_english + constraints = ( + type_.constraintsToEnglish(escape_nroff_literal, + text_to_nroff)) +@@ -121,6 +121,7 @@ def columnGroupToNroff(table, groupXml, documented_columns): + raise error.Error("unknown element %s in " % node.tagName) + return summary, intro, body + ++ + def tableSummaryToNroff(summary, level=0): + s = "" + for type, name, arg in summary: +@@ -132,6 +133,7 @@ def tableSummaryToNroff(summary, level=0): + s += ".RE\n" + return s + ++ + def tableToNroff(schema, tableXml): + tableName = tableXml.attributes['name'].nodeValue + table = schema.tables[tableName] +@@ -156,20 +158,17 @@ def tableToNroff(schema, tableXml): + + return s + ++ + def docsToNroff(schemaFile, xmlFile, erFile, version=None): + schema = ovs.db.schema.DbSchema.from_json(ovs.json.from_file(schemaFile)) + doc = xml.dom.minidom.parse(xmlFile).documentElement + +- schemaDate = os.stat(schemaFile).st_mtime +- xmlDate = os.stat(xmlFile).st_mtime +- d = date.fromtimestamp(max(schemaDate, xmlDate)) +- + if doc.hasAttribute('name'): + manpage = doc.attributes['name'].nodeValue + else: + manpage = schema.name + +- if version == None: ++ if version is None: + version = "UNKNOWN" + + # Putting '\" p as the first line tells "man" that the manpage +@@ -194,7 +193,6 @@ def docsToNroff(schemaFile, xmlFile, erFile, version=None): + .PP + ''' % (manpage, schema.version, version, text_to_nroff(manpage), schema.name) + +- tables = "" + introNodes = [] + tableNodes = [] + summary = [] +@@ -237,8 +235,8 @@ Purpose + """ % (name, text_to_nroff(title)) + + if erFile: +- s += """ +-.\\" check if in troff mode (TTY) ++ s += r""" ++.\" check if in troff mode (TTY) + .if t \{ + .bp + .SH "TABLE RELATIONSHIPS" +@@ -248,8 +246,8 @@ database. Each node represents a table. Tables that are part of the + ``root set'' are shown with double borders. Each edge leads from the + table that contains it and points to the table that its value + represents. Edges are labeled with their column names, followed by a +-constraint on the number of allowed values: \\fB?\\fR for zero or one, +-\\fB*\\fR for zero or more, \\fB+\\fR for one or more. Thick lines ++constraint on the number of allowed values: \fB?\fR for zero or one, ++\fB*\fR for zero or more, \fB+\fR for one or more. Thick lines + represent strong references; thin lines represent weak references. + .RS -1in + """ +@@ -263,6 +261,7 @@ represent strong references; thin lines represent weak references. + s += tableToNroff(schema, node) + "\n" + return s + ++ + def usage(): + print("""\ + %(argv0)s: ovsdb schema documentation generator +@@ -278,6 +277,7 @@ The following options are also available: + """ % {'argv0': argv0}) + sys.exit(0) + ++ + if __name__ == "__main__": + try: + try: +diff --git a/ovsdb/ovsdb-dot.in b/ovsdb/ovsdb-dot.in +index 41b986c0ac..f1eefd49cb 100755 +--- a/ovsdb/ovsdb-dot.in ++++ b/ovsdb/ovsdb-dot.in +@@ -1,15 +1,13 @@ + #! @PYTHON3@ + +-from datetime import date + import ovs.db.error + import ovs.db.schema + import getopt +-import os +-import re + import sys argv0 = sys.argv[0] ++ + def printEdge(tableName, type, baseType, label): + if baseType.ref_table_name: + if type.n_min == 0: +@@ -31,38 +29,42 @@ def printEdge(tableName, type, baseType, label): + options['label'] = '"%s%s"' % (label, arity) + if baseType.ref_type == 'weak': + options['style'] = 'dotted' +- print ("\t%s -> %s [%s];" % ( ++ print("\t%s -> %s [%s];" % ( + tableName, + baseType.ref_table_name, +- ', '.join(['%s=%s' % (k,v) for k,v in options.items()]))) ++ ', '.join(['%s=%s' % (k, v) for k, v in options.items()]))) ++ + + def schemaToDot(schemaFile, arrows): + schema = ovs.db.schema.DbSchema.from_json(ovs.json.from_file(schemaFile)) + +- print ("digraph %s {" % schema.name) +- print ('\trankdir=LR;') +- print ('\tsize="6.5,4";') +- print ('\tmargin="0";') +- print ("\tnode [shape=box];") ++ print("digraph %s {" % schema.name) ++ print('\trankdir=LR;') ++ print('\tsize="6.5,4";') ++ print('\tmargin="0";') ++ print("\tnode [shape=box];") + if not arrows: +- print ("\tedge [dir=none, arrowhead=none, arrowtail=none];") ++ print("\tedge [dir=none, arrowhead=none, arrowtail=none];") + for tableName, table in schema.tables.items(): + options = {} + if table.is_root: + options['style'] = 'bold' +- print ("\t%s [%s];" % ( ++ print("\t%s [%s];" % ( + tableName, +- ', '.join(['%s=%s' % (k,v) for k,v in options.items()]))) ++ ', '.join(['%s=%s' % (k, v) for k, v in options.items()]))) + for columnName, column in table.columns.items(): + if column.type.value: +- printEdge(tableName, column.type, column.type.key, "%s key" % columnName) +- printEdge(tableName, column.type, column.type.value, "%s value" % columnName) ++ printEdge(tableName, column.type, column.type.key, ++ "%s key" % columnName) ++ printEdge(tableName, column.type, column.type.value, ++ "%s value" % columnName) + else: + printEdge(tableName, column.type, column.type.key, columnName) +- print ("}"); ++ print("}") ++ + + def usage(): +- print ("""\ ++ print("""\ + %(argv0)s: compiles ovsdb schemas to graphviz format + Prints a .dot file that "dot" can render to an entity-relationship diagram + usage: %(argv0)s [OPTIONS] SCHEMA +@@ -75,12 +77,13 @@ The following options are also available: + """ % {'argv0': argv0}) + sys.exit(0) + ++ + if __name__ == "__main__": + try: + try: + options, args = getopt.gnu_getopt(sys.argv[1:], 'hV', + ['no-arrows', +- 'help', 'version',]) ++ 'help', 'version']) + except getopt.GetoptError as geo: + sys.stderr.write("%s: %s\n" % (argv0, geo.msg)) + sys.exit(1) +@@ -92,7 +95,7 @@ if __name__ == "__main__": + elif key in ['-h', '--help']: + usage() + elif key in ['-V', '--version']: +- print ("ovsdb-dot (Open vSwitch) @VERSION@") ++ print("ovsdb-dot (Open vSwitch) @VERSION@") + else: + sys.exit(0) + diff --git a/ovsdb/ovsdb-server.c b/ovsdb/ovsdb-server.c -index 33ca4910d..b7b4d1559 100644 +index 33ca4910d7..b7b4d15593 100644 --- a/ovsdb/ovsdb-server.c +++ b/ovsdb/ovsdb-server.c @@ -233,7 +233,7 @@ main_loop(struct server_config *config, @@ -6900,7 +8770,7 @@ index 33ca4910d..b7b4d1559 100644 static void diff --git a/ovsdb/ovsdb-tool.c b/ovsdb/ovsdb-tool.c -index 60f353197..facd680ff 100644 +index 60f353197b..facd680ff3 100644 --- a/ovsdb/ovsdb-tool.c +++ b/ovsdb/ovsdb-tool.c @@ -1006,7 +1006,8 @@ raft_header_to_standalone_log(const struct raft_header *h, @@ -6990,8 +8860,258 @@ index 60f353197..facd680ff 100644 check_ovsdb_error(ovsdb_log_commit_block(db_log_data)); ovsdb_log_close(db_log_data); ovsdb_log_close(log); +diff --git a/ovsdb/raft.c b/ovsdb/raft.c +index b2361b1737..2066d4869b 100644 +--- a/ovsdb/raft.c ++++ b/ovsdb/raft.c +@@ -80,6 +80,7 @@ enum raft_failure_test { + FT_STOP_RAFT_RPC, + FT_TRANSFER_LEADERSHIP, + FT_TRANSFER_LEADERSHIP_AFTER_SEND_APPEND_REQ, ++ FT_TRANSFER_LEADERSHIP_AFTER_STARTING_TO_ADD, + }; + static enum raft_failure_test failure_test; + +@@ -378,6 +379,7 @@ static void raft_get_servers_from_log(struct raft *, enum vlog_level); + static void raft_get_election_timer_from_log(struct raft *); + + static bool raft_handle_write_error(struct raft *, struct ovsdb_error *); ++static bool raft_has_uncommitted_configuration(const struct raft *); + + static void raft_run_reconfigure(struct raft *); + +@@ -1005,8 +1007,13 @@ raft_conn_update_probe_interval(struct raft *raft, struct raft_conn *r_conn) + * inactivity probe follower will just try to initiate election + * indefinitely staying in 'candidate' role. And the leader will continue + * to send heartbeats to the dead connection thinking that remote server +- * is still part of the cluster. */ +- int probe_interval = raft->election_timer + ELECTION_RANGE_MSEC; ++ * is still part of the cluster. ++ * ++ * While joining, the real value of the election timeout is not known to ++ * this server, so using the maximum. */ ++ int probe_interval = (raft->joining ? ELECTION_MAX_MSEC ++ : raft->election_timer) ++ + ELECTION_RANGE_MSEC; + + jsonrpc_session_set_probe_interval(r_conn->js, probe_interval); + } +@@ -1250,10 +1257,30 @@ raft_transfer_leadership(struct raft *raft, const char *reason) + return; + } + +- struct raft_server *s; ++ struct raft_server **servers, *s; ++ uint64_t threshold = 0; ++ size_t n = 0, start, i; ++ ++ servers = xmalloc(hmap_count(&raft->servers) * sizeof *servers); ++ + HMAP_FOR_EACH (s, hmap_node, &raft->servers) { +- if (!uuid_equals(&raft->sid, &s->sid) +- && s->phase == RAFT_PHASE_STABLE) { ++ if (uuid_equals(&raft->sid, &s->sid) ++ || s->phase != RAFT_PHASE_STABLE) { ++ continue; ++ } ++ if (s->match_index > threshold) { ++ threshold = s->match_index; ++ } ++ servers[n++] = s; ++ } ++ ++ start = n ? random_range(n) : 0; ++ ++retry: ++ for (i = 0; i < n; i++) { ++ s = servers[(start + i) % n]; ++ ++ if (s->match_index >= threshold) { + struct raft_conn *conn = raft_find_conn_by_sid(raft, &s->sid); + if (!conn) { + continue; +@@ -1269,7 +1296,10 @@ raft_transfer_leadership(struct raft *raft, const char *reason) + .term = raft->term, + } + }; +- raft_send_to_conn(raft, &rpc, conn); ++ ++ if (!raft_send_to_conn(raft, &rpc, conn)) { ++ continue; ++ } + + raft_record_note(raft, "transfer leadership", + "transferring leadership to %s because %s", +@@ -1277,6 +1307,23 @@ raft_transfer_leadership(struct raft *raft, const char *reason) + break; + } + } ++ ++ if (n && i == n && threshold) { ++ if (threshold > raft->commit_index) { ++ /* Failed to transfer to servers with the highest 'match_index'. ++ * Try other servers that are not behind the majority. */ ++ threshold = raft->commit_index; ++ } else { ++ /* Try any other server. It is safe, because they either have all ++ * the append requests queued up for them before the leadership ++ * transfer message or their connection is broken and we will not ++ * transfer anyway. */ ++ threshold = 0; ++ } ++ goto retry; ++ } ++ ++ free(servers); + } + + /* Send a RemoveServerRequest to the rest of the servers in the cluster. +@@ -2649,15 +2696,22 @@ raft_become_follower(struct raft *raft) + * new configuration. Our AppendEntries processing will properly update + * the server configuration later, if necessary. + * ++ * However, since we're sending replies about a failure to add, those new ++ * servers has to be cleaned up. Otherwise, they will stuck in a 'CATCHUP' ++ * phase in case this server regains leadership before they join through ++ * the current new leader. They are not yet in 'raft->servers', so not ++ * part of the shared configuration. ++ * + * Also we do not complete commands here, as they can still be completed + * if their log entries have already been replicated to other servers. + * If the entries were actually committed according to the new leader, our + * AppendEntries processing will complete the corresponding commands. + */ + struct raft_server *s; +- HMAP_FOR_EACH (s, hmap_node, &raft->add_servers) { ++ HMAP_FOR_EACH_POP (s, hmap_node, &raft->add_servers) { + raft_send_add_server_reply__(raft, &s->sid, s->address, false, + RAFT_SERVER_LOST_LEADERSHIP); ++ raft_server_destroy(s); + } + if (raft->remove_server) { + raft_send_remove_server_reply__(raft, &raft->remove_server->sid, +@@ -2721,6 +2775,13 @@ raft_send_heartbeats(struct raft *raft) + raft_reset_ping_timer(raft); + } + ++static void ++raft_join_complete(struct raft *raft) ++{ ++ raft->joining = false; ++ raft_update_probe_intervals(raft); ++} ++ + /* Initializes the fields in 's' that represent the leader's view of the + * server. */ + static void +@@ -2758,6 +2819,18 @@ raft_become_leader(struct raft *raft) + raft_reset_election_timer(raft); + raft_reset_ping_timer(raft); + ++ if (raft->joining) { ++ /* It is possible that the server committing this one to the list of ++ * servers lost leadership before the entry is committed but after ++ * it was already replicated to majority of servers. In this case ++ * other servers will recognize this one as a valid cluster member ++ * and may transfer leadership to it and vote for it. This way ++ * we're becoming a cluster leader without receiving reply for a ++ * join request and will commit addition of this server ourselves. */ ++ VLOG_INFO_RL(&rl, "elected as leader while joining"); ++ raft_join_complete(raft); ++ } ++ + struct raft_server *s; + HMAP_FOR_EACH (s, hmap_node, &raft->servers) { + raft_server_init_leader(raft, s); +@@ -2916,12 +2989,12 @@ raft_update_commit_index(struct raft *raft, uint64_t new_commit_index) + } + + while (raft->commit_index < new_commit_index) { ++ static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 5); + uint64_t index = ++raft->commit_index; + const struct raft_entry *e = raft_get_entry(raft, index); + + if (raft_entry_has_data(e)) { + struct raft_command *cmd = raft_find_command_by_eid(raft, &e->eid); +- static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 5); + + if (cmd) { + if (!cmd->index && raft->role == RAFT_LEADER) { +@@ -2965,6 +3038,35 @@ raft_update_commit_index(struct raft *raft, uint64_t new_commit_index) + * reallocate raft->entries, which would invalidate 'e', so + * this case must be last, after the one for 'e->data'. */ + raft_run_reconfigure(raft); ++ } else if (e->servers && !raft_has_uncommitted_configuration(raft)) { ++ struct ovsdb_error *error; ++ struct raft_server *s; ++ struct hmap servers; ++ ++ error = raft_servers_from_json(e->servers, &servers); ++ ovs_assert(!error); ++ HMAP_FOR_EACH (s, hmap_node, &servers) { ++ struct raft_server *server = raft_find_server(raft, &s->sid); ++ ++ if (server && server->phase == RAFT_PHASE_COMMITTING) { ++ /* This server lost leadership while committing ++ * server 's', but it was committed later by a ++ * new leader. */ ++ server->phase = RAFT_PHASE_STABLE; ++ } ++ ++ if (raft->joining && uuid_equals(&s->sid, &raft->sid)) { ++ /* Leadership change happened before previous leader ++ * could commit the change of a servers list, but it ++ * was replicated and a new leader committed it. */ ++ VLOG_INFO_RL(&rl, ++ "added to configuration without reply " ++ "(eid: "UUID_FMT", commit index: %"PRIu64")", ++ UUID_ARGS(&e->eid), index); ++ raft_join_complete(raft); ++ } ++ } ++ raft_servers_destroy(&servers); + } + } + +@@ -3877,6 +3979,10 @@ raft_handle_add_server_request(struct raft *raft, + "to cluster "CID_FMT, s->nickname, SID_ARGS(&s->sid), + rq->address, CID_ARGS(&raft->cid)); + raft_send_append_request(raft, s, 0, "initialize new server"); ++ ++ if (failure_test == FT_TRANSFER_LEADERSHIP_AFTER_STARTING_TO_ADD) { ++ failure_test = FT_TRANSFER_LEADERSHIP; ++ } + } + + static void +@@ -3891,7 +3997,7 @@ raft_handle_add_server_reply(struct raft *raft, + } + + if (rpy->success) { +- raft->joining = false; ++ raft_join_complete(raft); + + /* It is tempting, at this point, to check that this server is part of + * the current configuration. However, this is not necessarily the +@@ -4865,6 +4971,7 @@ raft_get_election_timer_from_log(struct raft *raft) + break; + } + } ++ raft_update_probe_intervals(raft); + } + + static void +@@ -5002,6 +5109,8 @@ raft_unixctl_failure_test(struct unixctl_conn *conn OVS_UNUSED, + } else if (!strcmp(test, + "transfer-leadership-after-sending-append-request")) { + failure_test = FT_TRANSFER_LEADERSHIP_AFTER_SEND_APPEND_REQ; ++ } else if (!strcmp(test, "transfer-leadership-after-starting-to-add")) { ++ failure_test = FT_TRANSFER_LEADERSHIP_AFTER_STARTING_TO_ADD; + } else if (!strcmp(test, "transfer-leadership")) { + failure_test = FT_TRANSFER_LEADERSHIP; + } else if (!strcmp(test, "clear")) { diff --git a/ovsdb/relay.c b/ovsdb/relay.c -index 9ff6ed8f3..5a2b4b3b6 100644 +index 9ff6ed8f39..5a2b4b3b66 100644 --- a/ovsdb/relay.c +++ b/ovsdb/relay.c @@ -301,6 +301,8 @@ static void @@ -7044,7 +9164,7 @@ index 9ff6ed8f3..5a2b4b3b6 100644 } diff --git a/ovsdb/relay.h b/ovsdb/relay.h -index 390ea70c8..2d66b5e5f 100644 +index 390ea70c82..2d66b5e5fa 100644 --- a/ovsdb/relay.h +++ b/ovsdb/relay.h @@ -23,8 +23,11 @@ struct json; @@ -7062,7 +9182,7 @@ index 390ea70c8..2d66b5e5f 100644 void ovsdb_relay_add_db(struct ovsdb *, const char *remote, schema_change_callback schema_change_cb, diff --git a/ovsdb/row.c b/ovsdb/row.c -index d7bfbdd36..2b52b6816 100644 +index d7bfbdd365..2b52b68161 100644 --- a/ovsdb/row.c +++ b/ovsdb/row.c @@ -302,12 +302,14 @@ ovsdb_row_columns_to_string(const struct ovsdb_row *row, @@ -7098,7 +9218,7 @@ index d7bfbdd36..2b52b6816 100644 return error; } diff --git a/ovsdb/row.h b/ovsdb/row.h -index ff91288fe..59f498a20 100644 +index ff91288fed..6f5e58acb3 100644 --- a/ovsdb/row.h +++ b/ovsdb/row.h @@ -114,7 +114,8 @@ void ovsdb_row_columns_to_string(const struct ovsdb_row *, @@ -7111,8 +9231,24 @@ index ff91288fe..59f498a20 100644 OVS_WARN_UNUSED_RESULT; struct json *ovsdb_row_to_json(const struct ovsdb_row *, const struct ovsdb_column_set *include); +@@ -129,6 +130,7 @@ ovsdb_row_get_uuid(const struct ovsdb_row *row) + static inline struct uuid * + ovsdb_row_get_uuid_rw(struct ovsdb_row *row) + { ++ ovsdb_datum_unshare(&row->fields[OVSDB_COL_UUID], &ovsdb_type_uuid); + return &row->fields[OVSDB_COL_UUID].keys[0].uuid; + } + +@@ -141,6 +143,7 @@ ovsdb_row_get_version(const struct ovsdb_row *row) + static inline struct uuid * + ovsdb_row_get_version_rw(struct ovsdb_row *row) + { ++ ovsdb_datum_unshare(&row->fields[OVSDB_COL_VERSION], &ovsdb_type_uuid); + return &row->fields[OVSDB_COL_VERSION].keys[0].uuid; + } + diff --git a/ovsdb/storage.c b/ovsdb/storage.c -index e8f95ce64..6c395106c 100644 +index e8f95ce642..6c395106c0 100644 --- a/ovsdb/storage.c +++ b/ovsdb/storage.c @@ -623,7 +623,7 @@ ovsdb_storage_store_snapshot(struct ovsdb_storage *storage, @@ -7155,7 +9291,7 @@ index e8f95ce64..6c395106c 100644 /* When 'error' and 'command' are both null, it indicates that the * command is complete. This is fine since this unbacked storage drops diff --git a/ovsdb/storage.h b/ovsdb/storage.h -index a1fdaa564..05f40ce93 100644 +index a1fdaa564e..05f40ce934 100644 --- a/ovsdb/storage.h +++ b/ovsdb/storage.h @@ -85,7 +85,7 @@ struct ovsdb_error *ovsdb_storage_store_snapshot(struct ovsdb_storage *storage, @@ -7168,7 +9304,7 @@ index a1fdaa564..05f40ce93 100644 OVS_WARN_UNUSED_RESULT; diff --git a/ovsdb/table.c b/ovsdb/table.c -index 66071ce2f..0792e1580 100644 +index 66071ce2f8..0792e1580e 100644 --- a/ovsdb/table.c +++ b/ovsdb/table.c @@ -368,7 +368,8 @@ ovsdb_table_execute_insert(struct ovsdb_txn *txn, const struct uuid *row_uuid, @@ -7191,10 +9327,65 @@ index 66071ce2f..0792e1580 100644 if (!error && (xor || !ovsdb_row_equal_columns(row, update, &columns))) { error = ovsdb_row_update_columns(ovsdb_txn_row_modify(txn, row), diff --git a/ovsdb/transaction.c b/ovsdb/transaction.c -index 03541af85..f01de2a34 100644 +index 03541af85d..0160fd3b60 100644 --- a/ovsdb/transaction.c +++ b/ovsdb/transaction.c -@@ -1251,7 +1251,7 @@ ovsdb_txn_precheck_prereq(const struct ovsdb *db) +@@ -321,7 +321,8 @@ update_row_ref_count(struct ovsdb_txn *txn, struct ovsdb_txn_row *r) + const struct ovsdb_column *column = node->data; + struct ovsdb_error *error; + +- if (bitmap_is_set(r->changed, column->index)) { ++ if (bitmap_is_set(r->changed, column->index) ++ && ovsdb_type_has_strong_refs(&column->type)) { + if (r->old && !r->new) { + error = ovsdb_txn_adjust_row_refs( + txn, r->old, column, +@@ -716,6 +717,10 @@ assess_weak_refs(struct ovsdb_txn *txn, struct ovsdb_txn_row *txn_row) + unsigned int orig_n; + bool zero = false; + ++ if (!ovsdb_type_has_weak_refs(&column->type)) { ++ continue; ++ } ++ + orig_n = datum->n; + + /* Collecting all key-value pairs that references deleted rows. */ +@@ -731,18 +736,24 @@ assess_weak_refs(struct ovsdb_txn *txn, struct ovsdb_txn_row *txn_row) + ovsdb_datum_sort_unique(&deleted_refs, &column->type); + + /* Removing elements that references deleted rows. */ +- ovsdb_datum_subtract(datum, &column->type, +- &deleted_refs, &column->type); ++ if (deleted_refs.n) { ++ ovsdb_datum_subtract(datum, &column->type, ++ &deleted_refs, &column->type); ++ } + ovsdb_datum_destroy(&deleted_refs, &column->type); + + /* Generating the difference between old and new data. */ +- if (txn_row->old) { +- ovsdb_datum_added_removed(&added, &removed, +- &txn_row->old->fields[column->index], +- datum, &column->type); +- } else { +- ovsdb_datum_init_empty(&removed); +- ovsdb_datum_clone(&added, datum); ++ ovsdb_datum_init_empty(&added); ++ ovsdb_datum_init_empty(&removed); ++ if (datum->n != orig_n ++ || bitmap_is_set(txn_row->changed, column->index)) { ++ if (txn_row->old) { ++ ovsdb_datum_added_removed(&added, &removed, ++ &txn_row->old->fields[column->index], ++ datum, &column->type); ++ } else { ++ ovsdb_datum_clone(&added, datum); ++ } + } + + /* Checking added data and creating new references. */ +@@ -1251,7 +1262,7 @@ ovsdb_txn_precheck_prereq(const struct ovsdb *db) struct ovsdb_txn_progress * ovsdb_txn_propose_schema_change(struct ovsdb *db, @@ -7204,7 +9395,7 @@ index 03541af85..f01de2a34 100644 { struct ovsdb_txn_progress *progress = xzalloc(sizeof *progress); diff --git a/ovsdb/transaction.h b/ovsdb/transaction.h -index 6b5bb7f24..9991f34d2 100644 +index 6b5bb7f24b..9991f34d24 100644 --- a/ovsdb/transaction.h +++ b/ovsdb/transaction.h @@ -21,6 +21,7 @@ @@ -7225,10 +9416,25 @@ index 6b5bb7f24..9991f34d2 100644 bool ovsdb_txn_progress_is_complete(const struct ovsdb_txn_progress *); const struct ovsdb_error *ovsdb_txn_progress_get_error( diff --git a/ovsdb/trigger.c b/ovsdb/trigger.c -index 01bb80e28..3c93ae580 100644 +index 01bb80e282..84dbc96f52 100644 --- a/ovsdb/trigger.c +++ b/ovsdb/trigger.c -@@ -274,8 +274,8 @@ ovsdb_trigger_try(struct ovsdb_trigger *t, long long int now) +@@ -252,6 +252,14 @@ ovsdb_trigger_try(struct ovsdb_trigger *t, long long int now) + return false; + } + ++ if (t->read_only) { ++ trigger_convert_error( ++ t, ovsdb_error("not allowed", "conversion is not allowed " ++ "for read-only database %s", ++ t->db->schema->name)); ++ return false; ++ } ++ + /* Validate parameters. */ + const struct json *params = t->request->params; + if (params->type != JSON_ARRAY || params->array.n != 2) { +@@ -274,8 +282,8 @@ ovsdb_trigger_try(struct ovsdb_trigger *t, long long int now) if (!error) { error = ovsdb_convert(t->db, new_schema, &newdb); } @@ -7238,7 +9444,7 @@ index 01bb80e28..3c93ae580 100644 trigger_convert_error(t, error); return false; } -@@ -286,7 +286,8 @@ ovsdb_trigger_try(struct ovsdb_trigger *t, long long int now) +@@ -286,7 +294,8 @@ ovsdb_trigger_try(struct ovsdb_trigger *t, long long int now) /* Propose the change. */ t->progress = ovsdb_txn_propose_schema_change( @@ -7249,7 +9455,7 @@ index 01bb80e28..3c93ae580 100644 t->reply = jsonrpc_create_reply(json_object_create(), t->request->id); diff --git a/python/automake.mk b/python/automake.mk -index d00911828..b0f444169 100644 +index d00911828c..b0f444169c 100644 --- a/python/automake.mk +++ b/python/automake.mk @@ -64,10 +64,10 @@ ovs_pytests = \ @@ -7301,10 +9507,29 @@ index d00911828..b0f444169 100644 UNINSTALL_LOCAL += ovs-uninstall-local diff --git a/python/ovs/db/idl.py b/python/ovs/db/idl.py -index 9fc2159b0..16ece0334 100644 +index 9fc2159b04..a80da84e7a 100644 --- a/python/ovs/db/idl.py +++ b/python/ovs/db/idl.py -@@ -494,6 +494,7 @@ class Idl(object): +@@ -299,6 +299,7 @@ class Idl(object): + self._server_schema_request_id = None + self._server_monitor_request_id = None + self._db_change_aware_request_id = None ++ self._monitor_cancel_request_id = None + self._server_db_name = '_Server' + self._server_db_table = 'Database' + self.server_tables = None +@@ -481,6 +482,10 @@ class Idl(object): + break + else: + self.__parse_update(msg.params[1], OVSDB_UPDATE) ++ elif self.handle_monitor_canceled(msg): ++ break ++ elif self.handle_monitor_cancel_reply(msg): ++ break + elif (msg.type == ovs.jsonrpc.Message.T_REPLY + and self._monitor_request_id is not None + and self._monitor_request_id == msg.id): +@@ -494,6 +499,7 @@ class Idl(object): if not msg.result[0]: self.__clear() self.__parse_update(msg.result[2], OVSDB_UPDATE3) @@ -7312,8 +9537,156 @@ index 9fc2159b0..16ece0334 100644 elif self.state == self.IDL_S_DATA_MONITOR_COND_REQUESTED: self.__clear() self.__parse_update(msg.result, OVSDB_UPDATE2) +@@ -615,6 +621,33 @@ class Idl(object): + + return initial_change_seqno != self.change_seqno + ++ def handle_monitor_canceled(self, msg): ++ if msg.type != msg.T_NOTIFY: ++ return False ++ if msg.method != "monitor_canceled": ++ return False ++ ++ if msg.params[0] == str(self.uuid): ++ params = [str(self.server_monitor_uuid)] ++ elif msg.params[0] == str(self.server_monitor_uuid): ++ params = [str(self.uuid)] ++ else: ++ return False ++ ++ mc_msg = ovs.jsonrpc.Message.create_request("monitor_cancel", params) ++ self._monitor_cancel_request_id = mc_msg.id ++ self.send_request(mc_msg) ++ self.restart_fsm() ++ return True ++ ++ def handle_monitor_cancel_reply(self, msg): ++ if msg.type != msg.T_REPLY: ++ return False ++ if msg.id != self._monitor_cancel_request_id: ++ return False ++ self._monitor_cancel_request_id = None ++ return True ++ + def compose_cond_change(self): + if not self.cond_changed: + return +diff --git a/python/ovs/flow/odp.py b/python/ovs/flow/odp.py +index db63afc8d6..4bfe5b59a2 100644 +--- a/python/ovs/flow/odp.py ++++ b/python/ovs/flow/odp.py +@@ -204,6 +204,7 @@ class ODPFlow(Flow): + """Generate the arguments for the action KVDecoders.""" + _decoders = { + "drop": decode_flag, ++ "meter": decode_int, + "lb_output": decode_int, + "trunc": decode_int, + "recirc": decode_int, +@@ -334,8 +335,31 @@ class ODPFlow(Flow): + ) + ), + **ODPFlow._tnl_action_decoder_args(), ++ "hash": nested_kv_decoder( ++ KVDecoders( ++ { ++ "l4": decode_int, ++ "sym_l4": decode_int, ++ } ++ ) ++ ), + } + ++ _decoders["sample"] = nested_kv_decoder( ++ KVDecoders( ++ { ++ "sample": (lambda x: float(x.strip("%"))), ++ "actions": nested_kv_decoder( ++ KVDecoders( ++ decoders=_decoders, ++ default_free=decode_free_output, ++ ), ++ is_list=True, ++ ), ++ } ++ ) ++ ) ++ + _decoders["clone"] = nested_kv_decoder( + KVDecoders(decoders=_decoders, default_free=decode_free_output), + is_list=True, +@@ -343,20 +367,6 @@ class ODPFlow(Flow): + + return { + **_decoders, +- "sample": nested_kv_decoder( +- KVDecoders( +- { +- "sample": (lambda x: float(x.strip("%"))), +- "actions": nested_kv_decoder( +- KVDecoders( +- decoders=_decoders, +- default_free=decode_free_output, +- ), +- is_list=True, +- ), +- } +- ) +- ), + "check_pkt_len": nested_kv_decoder( + KVDecoders( + { +@@ -365,13 +375,15 @@ class ODPFlow(Flow): + KVDecoders( + decoders=_decoders, + default_free=decode_free_output, +- ) ++ ), ++ is_list=True, + ), + "le": nested_kv_decoder( + KVDecoders( + decoders=_decoders, + default_free=decode_free_output, +- ) ++ ), ++ is_list=True, + ), + } + ) +diff --git a/python/ovs/flow/ofp.py b/python/ovs/flow/ofp.py +index 20231fd9f3..3d3226c919 100644 +--- a/python/ovs/flow/ofp.py ++++ b/python/ovs/flow/ofp.py +@@ -170,12 +170,13 @@ class OFPFlow(Flow): + args = { + "table": decode_int, + "duration": decode_time, +- "n_packet": decode_int, ++ "n_packets": decode_int, + "n_bytes": decode_int, + "cookie": decode_int, + "idle_timeout": decode_time, + "hard_timeout": decode_time, + "hard_age": decode_time, ++ "idle_age": decode_time, + } + return KVDecoders(args) + +diff --git a/python/ovs/flow/ofp_act.py b/python/ovs/flow/ofp_act.py +index c540443eae..2c85076a34 100644 +--- a/python/ovs/flow/ofp_act.py ++++ b/python/ovs/flow/ofp_act.py +@@ -54,6 +54,7 @@ def decode_controller(value): + "id": decode_int, + "userdata": decode_default, + "pause": decode_flag, ++ "meter_id": decode_int, + } + ) + )(value) diff --git a/python/ovs/jsonrpc.py b/python/ovs/jsonrpc.py -index d5127268a..d9fe27aec 100644 +index d5127268aa..d9fe27aec6 100644 --- a/python/ovs/jsonrpc.py +++ b/python/ovs/jsonrpc.py @@ -377,7 +377,7 @@ class Session(object): @@ -7326,7 +9699,7 @@ index d5127268a..d9fe27aec 100644 self.remotes = remotes random.shuffle(self.remotes) diff --git a/python/ovs/stream.py b/python/ovs/stream.py -index ac5b0fd0c..b32341076 100644 +index ac5b0fd0c6..b32341076c 100644 --- a/python/ovs/stream.py +++ b/python/ovs/stream.py @@ -824,7 +824,8 @@ class SSLStream(Stream): @@ -7339,24 +9712,306 @@ index ac5b0fd0c..b32341076 100644 return ovs.socket_util.get_exception_errno(e) return 0 -diff --git a/python/build/__init__.py b/python/ovs_build_helpers/__init__.py -similarity index 100% -rename from python/build/__init__.py -rename to python/ovs_build_helpers/__init__.py -diff --git a/python/build/extract_ofp_fields.py b/python/ovs_build_helpers/extract_ofp_fields.py -similarity index 100% -rename from python/build/extract_ofp_fields.py -rename to python/ovs_build_helpers/extract_ofp_fields.py -diff --git a/python/build/nroff.py b/python/ovs_build_helpers/nroff.py -similarity index 100% -rename from python/build/nroff.py -rename to python/ovs_build_helpers/nroff.py -diff --git a/python/build/soutil.py b/python/ovs_build_helpers/soutil.py -similarity index 100% -rename from python/build/soutil.py -rename to python/ovs_build_helpers/soutil.py +diff --git a/python/ovs/tests/test_odp.py b/python/ovs/tests/test_odp.py +index f8017ca8a1..e9fd5e207f 100644 +--- a/python/ovs/tests/test_odp.py ++++ b/python/ovs/tests/test_odp.py +@@ -13,6 +13,32 @@ from ovs.flow.decoders import ( + ) + + ++def do_test_section(input_string, section, expected): ++ flow = ODPFlow(input_string) ++ kv_list = flow.section(section).data ++ ++ assert len(expected) == len(kv_list) ++ ++ for i in range(len(expected)): ++ assert expected[i].key == kv_list[i].key ++ assert expected[i].value == kv_list[i].value ++ ++ # Assert positions relative to action string are OK. ++ pos = flow.section(section).pos ++ string = flow.section(section).string ++ ++ kpos = kv_list[i].meta.kpos ++ kstr = kv_list[i].meta.kstring ++ vpos = kv_list[i].meta.vpos ++ vstr = kv_list[i].meta.vstring ++ assert string[kpos : kpos + len(kstr)] == kstr ++ if vpos != -1: ++ assert string[vpos : vpos + len(vstr)] == vstr ++ ++ # Assert string meta is correct. ++ assert input_string[pos : pos + len(string)] == string ++ ++ + @pytest.mark.parametrize( + "input_string,expected", + [ +@@ -109,26 +135,7 @@ from ovs.flow.decoders import ( + ], + ) + def test_odp_fields(input_string, expected): +- odp = ODPFlow(input_string) +- match = odp.match_kv +- for i in range(len(expected)): +- assert expected[i].key == match[i].key +- assert expected[i].value == match[i].value +- +- # Assert positions relative to action string are OK. +- mpos = odp.section("match").pos +- mstring = odp.section("match").string +- +- kpos = match[i].meta.kpos +- kstr = match[i].meta.kstring +- vpos = match[i].meta.vpos +- vstr = match[i].meta.vstring +- assert mstring[kpos : kpos + len(kstr)] == kstr +- if vpos != -1: +- assert mstring[vpos : vpos + len(vstr)] == vstr +- +- # Assert mstring meta is correct. +- assert input_string[mpos : mpos + len(mstring)] == mstring ++ do_test_section(input_string, "match", expected) + + + @pytest.mark.parametrize( +@@ -496,48 +503,41 @@ def test_odp_fields(input_string, expected): + "check_pkt_len", + { + "size": 200, +- "gt": {"output": {"port": 4}}, +- "le": {"output": {"port": 5}}, ++ "gt": [{"output": {"port": 4}}], ++ "le": [{"output": {"port": 5}}], + }, + ), + KeyValue( + "check_pkt_len", + { + "size": 200, +- "gt": {"drop": True}, +- "le": {"output": {"port": 5}}, ++ "gt": [{"drop": True}], ++ "le": [{"output": {"port": 5}}], + }, + ), + KeyValue( + "check_pkt_len", + { + "size": 200, +- "gt": {"ct": {"nat": True}}, +- "le": {"drop": True}, ++ "gt": [{"ct": {"nat": True}}], ++ "le": [{"drop": True}], + }, + ), + ], + ), ++ ( ++ "actions:meter(1),hash(l4(0))", ++ [ ++ KeyValue("meter", 1), ++ KeyValue( ++ "hash", ++ { ++ "l4": 0, ++ } ++ ), ++ ], ++ ), + ], + ) + def test_odp_actions(input_string, expected): +- odp = ODPFlow(input_string) +- actions = odp.actions_kv +- for i in range(len(expected)): +- assert expected[i].key == actions[i].key +- assert expected[i].value == actions[i].value +- +- # Assert positions relative to action string are OK. +- apos = odp.section("actions").pos +- astring = odp.section("actions").string +- +- kpos = actions[i].meta.kpos +- kstr = actions[i].meta.kstring +- vpos = actions[i].meta.vpos +- vstr = actions[i].meta.vstring +- assert astring[kpos : kpos + len(kstr)] == kstr +- if vpos != -1: +- assert astring[vpos : vpos + len(vstr)] == vstr +- +- # Assert astring meta is correct. +- assert input_string[apos : apos + len(astring)] == astring ++ do_test_section(input_string, "actions", expected) +diff --git a/python/ovs/tests/test_ofp.py b/python/ovs/tests/test_ofp.py +index 27bcf0c47c..d098520cae 100644 +--- a/python/ovs/tests/test_ofp.py ++++ b/python/ovs/tests/test_ofp.py +@@ -6,6 +6,32 @@ from ovs.flow.kv import KeyValue, ParseError + from ovs.flow.decoders import EthMask, IPMask, decode_mask + + ++def do_test_section(input_string, section, expected): ++ flow = OFPFlow(input_string) ++ kv_list = flow.section(section).data ++ ++ assert len(expected) == len(kv_list) ++ ++ for i in range(len(expected)): ++ assert expected[i].key == kv_list[i].key ++ assert expected[i].value == kv_list[i].value ++ ++ # Assert positions relative to action string are OK. ++ pos = flow.section(section).pos ++ string = flow.section(section).string ++ ++ kpos = kv_list[i].meta.kpos ++ kstr = kv_list[i].meta.kstring ++ vpos = kv_list[i].meta.vpos ++ vstr = kv_list[i].meta.vstring ++ assert string[kpos : kpos + len(kstr)] == kstr ++ if vpos != -1: ++ assert string[vpos : vpos + len(vstr)] == vstr ++ ++ # Assert string meta is correct. ++ assert input_string[pos : pos + len(string)] == string ++ ++ + @pytest.mark.parametrize( + "input_string,expected", + [ +@@ -26,6 +52,21 @@ from ovs.flow.decoders import EthMask, IPMask, decode_mask + KeyValue("controller", {"max_len": 200}), + ], + ), ++ ( ++ "actions=controller(max_len=123,reason=no_match,id=456,userdata=00.00.00.12.00.00.00.00,meter_id=12)", # noqa: E501 ++ [ ++ KeyValue( ++ "controller", ++ { ++ "max_len": 123, ++ "reason": "no_match", ++ "id": 456, ++ "userdata": "00.00.00.12.00.00.00.00", ++ "meter_id": 12, ++ } ++ ), ++ ], ++ ), + ( + "actions=enqueue(foo,42),enqueue:foo:42,enqueue(bar,4242)", + [ +@@ -545,6 +586,20 @@ from ovs.flow.decoders import EthMask, IPMask, decode_mask + ), + ], + ), ++ ( ++ "actions=LOCAL,clone(sample(probability=123))", ++ [ ++ KeyValue("output", {"port": "LOCAL"}), ++ KeyValue( ++ "clone", ++ [ ++ {"sample": { ++ "probability": 123, ++ }}, ++ ] ++ ), ++ ], ++ ), + ( + "actions=doesnotexist(1234)", + ParseError, +@@ -570,27 +625,41 @@ from ovs.flow.decoders import EthMask, IPMask, decode_mask + def test_act(input_string, expected): + if isinstance(expected, type): + with pytest.raises(expected): +- ofp = OFPFlow(input_string) ++ OFPFlow(input_string) + return + +- ofp = OFPFlow(input_string) +- actions = ofp.actions_kv ++ do_test_section(input_string, "actions", expected) + +- for i in range(len(expected)): +- assert expected[i].key == actions[i].key +- assert expected[i].value == actions[i].value + +- # Assert positions relative to action string are OK. +- apos = ofp.section("actions").pos +- astring = ofp.section("actions").string ++@pytest.mark.parametrize( ++ "input_string,expected", ++ [ ++ ( ++ "cookie=0x35f946ead8d8f9e4, duration=97746.271s, table=0, n_packets=12, n_bytes=254, idle_age=117, priority=4,in_port=1", # noqa: E501 ++ ( ++ [ ++ KeyValue("cookie", 0x35f946ead8d8f9e4), ++ KeyValue("duration", 97746.271), ++ KeyValue("table", 0), ++ KeyValue("n_packets", 12), ++ KeyValue("n_bytes", 254), ++ KeyValue("idle_age", 117), ++ ], ++ [ ++ KeyValue("priority", 4), ++ KeyValue("in_port", 1) ++ ], ++ ), ++ ), ++ ], ++) ++def test_key(input_string, expected): ++ if isinstance(expected, type): ++ with pytest.raises(expected): ++ OFPFlow(input_string) ++ return + +- kpos = actions[i].meta.kpos +- kstr = actions[i].meta.kstring +- vpos = actions[i].meta.vpos +- vstr = actions[i].meta.vstring +- assert astring[kpos : kpos + len(kstr)] == kstr +- if vpos != -1: +- assert astring[vpos : vpos + len(vstr)] == vstr ++ input_string += " actions=drop" + +- # Assert astring meta is correct. +- assert input_string[apos : apos + len(astring)] == astring ++ do_test_section(input_string, "info", expected[0]) ++ do_test_section(input_string, "match", expected[1]) +diff --git a/python/build/__init__.py b/python/ovs_build_helpers/__init__.py +similarity index 100% +rename from python/build/__init__.py +rename to python/ovs_build_helpers/__init__.py +diff --git a/python/build/extract_ofp_fields.py b/python/ovs_build_helpers/extract_ofp_fields.py +similarity index 100% +rename from python/build/extract_ofp_fields.py +rename to python/ovs_build_helpers/extract_ofp_fields.py +diff --git a/python/build/nroff.py b/python/ovs_build_helpers/nroff.py +similarity index 100% +rename from python/build/nroff.py +rename to python/ovs_build_helpers/nroff.py +diff --git a/python/build/soutil.py b/python/ovs_build_helpers/soutil.py +similarity index 100% +rename from python/build/soutil.py +rename to python/ovs_build_helpers/soutil.py +diff --git a/rhel/usr_lib_systemd_system_ovsdb-server.service b/rhel/usr_lib_systemd_system_ovsdb-server.service +index 49dc06e38c..558632320c 100644 +--- a/rhel/usr_lib_systemd_system_ovsdb-server.service ++++ b/rhel/usr_lib_systemd_system_ovsdb-server.service +@@ -29,3 +29,4 @@ ExecStop=/usr/share/openvswitch/scripts/ovs-ctl --no-ovs-vswitchd stop + ExecReload=/usr/share/openvswitch/scripts/ovs-ctl --no-ovs-vswitchd \ + ${OVS_USER_OPT} \ + --no-monitor restart $OPTIONS ++TimeoutSec=300 diff --git a/tests/.gitignore b/tests/.gitignore -index 83b1cb3b4..3a8c45975 100644 +index 83b1cb3b48..3a8c459756 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -3,6 +3,7 @@ @@ -7368,7 +10023,7 @@ index 83b1cb3b4..3a8c45975 100644 /idltest.h /idltest.ovsidl diff --git a/tests/automake.mk b/tests/automake.mk -index c8de3fe28..e39453cd1 100644 +index c8de3fe28d..e39453cd14 100644 --- a/tests/automake.mk +++ b/tests/automake.mk @@ -110,7 +110,8 @@ TESTSUITE_AT = \ @@ -7382,7 +10037,7 @@ index c8de3fe28..e39453cd1 100644 EXTRA_DIST += $(FUZZ_REGRESSION_TESTS) FUZZ_REGRESSION_TESTS = \ diff --git a/tests/classifier.at b/tests/classifier.at -index f652b5983..de2705653 100644 +index f652b59837..de2705653e 100644 --- a/tests/classifier.at +++ b/tests/classifier.at @@ -65,6 +65,94 @@ Datapath actions: 2 @@ -7481,7 +10136,7 @@ index f652b5983..de2705653 100644 AT_SETUP([flow classifier - prefix lookup]) OVS_VSWITCHD_START diff --git a/tests/dpctl.at b/tests/dpctl.at -index 7454a51ec..d2f1046f8 100644 +index 7454a51ec6..d2f1046f8b 100644 --- a/tests/dpctl.at +++ b/tests/dpctl.at @@ -135,3 +135,19 @@ AT_CHECK([ovs-appctl dpctl/dump-flows dummy@br0 | sort], [0], [dnl @@ -7506,7 +10161,7 @@ index 7454a51ec..d2f1046f8 100644 +AT_CLEANUP \ No newline at end of file diff --git a/tests/learn.at b/tests/learn.at -index d127fed34..d0bcc8363 100644 +index d127fed348..d0bcc83633 100644 --- a/tests/learn.at +++ b/tests/learn.at @@ -836,3 +836,63 @@ AT_CHECK([ovs-vsctl add-br br1 -- set b br1 datapath_type=dummy]) @@ -7575,7 +10230,7 @@ index d127fed34..d0bcc8363 100644 +AT_CLEANUP diff --git a/tests/learning-switch.at b/tests/learning-switch.at new file mode 100644 -index 000000000..ac2fc1b80 +index 0000000000..ac2fc1b801 --- /dev/null +++ b/tests/learning-switch.at @@ -0,0 +1,23 @@ @@ -7603,7 +10258,7 @@ index 000000000..ac2fc1b80 +OVS_VSWITCHD_STOP(["/cannot find route for controller/d"]) +AT_CLEANUP diff --git a/tests/mcast-snooping.at b/tests/mcast-snooping.at -index fe475e7b3..77b3acddb 100644 +index fe475e7b38..77b3acddb8 100644 --- a/tests/mcast-snooping.at +++ b/tests/mcast-snooping.at @@ -105,6 +105,328 @@ AT_CHECK([ovs-appctl mdb/show br0], [0], [dnl @@ -7935,11 +10590,122 @@ index fe475e7b3..77b3acddb 100644 AT_SETUP([mcast - delete the port mdb when vlan configuration changed]) OVS_VSWITCHD_START([]) +diff --git a/tests/nsh.at b/tests/nsh.at +index 6b7b6856f2..107673589d 100644 +--- a/tests/nsh.at ++++ b/tests/nsh.at +@@ -521,51 +521,45 @@ AT_CHECK([ + set interface vxlangpe32 type=vxlan options:exts=gpe options:remote_ip=30.0.0.2 options:packet_type=ptap ofport_request=3020 + + ovs-appctl netdev-dummy/ip4addr br-p1 10.0.0.1/24 +- ovs-appctl ovs/route/add 10.0.0.0/24 br-p1 + ovs-appctl tnl/arp/set br-p1 10.0.0.1 $HWADDR_BRP1 + ovs-appctl tnl/arp/set br-p1 10.0.0.2 $HWADDR_BRP2 + ovs-appctl tnl/arp/set br-p1 10.0.0.3 $HWADDR_BRP3 + + ovs-appctl netdev-dummy/ip4addr br-p2 20.0.0.2/24 +- ovs-appctl ovs/route/add 20.0.0.0/24 br-p2 + ovs-appctl tnl/arp/set br-p2 20.0.0.1 $HWADDR_BRP1 + ovs-appctl tnl/arp/set br-p2 20.0.0.2 $HWADDR_BRP2 + ovs-appctl tnl/arp/set br-p2 20.0.0.3 $HWADDR_BRP3 + + ovs-appctl netdev-dummy/ip4addr br-p3 30.0.0.3/24 +- ovs-appctl ovs/route/add 30.0.0.0/24 br-p3 + ovs-appctl tnl/arp/set br-p3 30.0.0.1 $HWADDR_BRP1 + ovs-appctl tnl/arp/set br-p3 30.0.0.2 $HWADDR_BRP2 + ovs-appctl tnl/arp/set br-p3 30.0.0.3 $HWADDR_BRP3 + ], [0], [stdout]) + + AT_CHECK([ +- ovs-appctl ovs/route/add 10.0.0.0/24 br-p1 + ovs-appctl tnl/arp/set br-p1 10.0.0.1 $HWADDR_BRP1 + ovs-appctl tnl/arp/set br-p1 10.0.0.2 $HWADDR_BRP2 + ovs-appctl tnl/arp/set br-p1 10.0.0.3 $HWADDR_BRP3 + ], [0], [stdout]) + + AT_CHECK([ +- ovs-appctl ovs/route/add 20.0.0.0/24 br-p2 + ovs-appctl tnl/arp/set br-p2 20.0.0.1 $HWADDR_BRP1 + ovs-appctl tnl/arp/set br-p2 20.0.0.2 $HWADDR_BRP2 + ovs-appctl tnl/arp/set br-p2 20.0.0.3 $HWADDR_BRP3 + ], [0], [stdout]) + + AT_CHECK([ +- ovs-appctl ovs/route/add 30.0.0.0/24 br-p3 + ovs-appctl tnl/arp/set br-p3 30.0.0.1 $HWADDR_BRP1 + ovs-appctl tnl/arp/set br-p3 30.0.0.2 $HWADDR_BRP2 + ovs-appctl tnl/arp/set br-p3 30.0.0.3 $HWADDR_BRP3 + ], [0], [stdout]) + + AT_CHECK([ +- ovs-appctl ovs/route/show | grep User: ++ ovs-appctl ovs/route/show | grep Cached: | sort + ], [0], [dnl +-User: 10.0.0.0/24 dev br-p1 SRC 10.0.0.1 +-User: 20.0.0.0/24 dev br-p2 SRC 20.0.0.2 +-User: 30.0.0.0/24 dev br-p3 SRC 30.0.0.3 ++Cached: 10.0.0.0/24 dev br-p1 SRC 10.0.0.1 local ++Cached: 20.0.0.0/24 dev br-p2 SRC 20.0.0.2 local ++Cached: 30.0.0.0/24 dev br-p3 SRC 30.0.0.3 local + ]) + + AT_CHECK([ +diff --git a/tests/ofp-print.at b/tests/ofp-print.at +index 14aa554169..6a07e23c64 100644 +--- a/tests/ofp-print.at ++++ b/tests/ofp-print.at +@@ -4180,4 +4180,22 @@ AT_CHECK([ovs-ofctl ofp-print "\ + 00 01 00 20 00 00 00 00 \ + 00 00 00 14 00 00 00 00 00 00 00 00 00 00 ff ff 0a 0a 00 02 00 00 00 00 \ + " | grep -q OFPBPC_BAD_VALUE], [0]) ++ ++AT_CHECK([ovs-ofctl ofp-print "\ ++01 04 00 20 00 00 00 03 00 00 23 20 00 00 00 20 \ ++06 \ ++00 00 00 00 00 00 00 \ ++00 80 00 08 00 00 00 00 \ ++"| grep -q OFPBPC_BAD_TYPE], [0], [ignore], [stderr]) ++AT_CHECK([grep -q "unknown NXT_CT_FLUSH property type 128" stderr], [0]) ++ ++AT_CHECK([ovs-ofctl ofp-print "\ ++01 04 00 28 00 00 00 03 00 00 23 20 00 00 00 20 \ ++06 \ ++00 00 00 00 00 00 00 \ ++00 00 00 10 00 00 00 00 \ ++00 80 00 08 00 50 00 00 \ ++"| grep -q OFPBPC_BAD_TYPE], [0], [ignore], [stderr]) ++AT_CHECK([grep -q "unknown NXT_CT_TUPLE property type 128" stderr], [0]) ++ + AT_CLEANUP diff --git a/tests/ofproto-dpif.at b/tests/ofproto-dpif.at -index fa6111c1e..450483bee 100644 +index fa6111c1ed..22b123655c 100644 --- a/tests/ofproto-dpif.at +++ b/tests/ofproto-dpif.at -@@ -849,7 +849,7 @@ table=2 ip actions=set_field:192.168.3.91->ip_src,output(11) +@@ -547,6 +547,23 @@ ovs-appctl time/warp 1000 100 + ovs-appctl bond/show > bond3.txt + AT_CHECK([sed -n '/member p2/,/^$/p' bond3.txt | grep 'hash'], [0], [ignore]) + ++# Check that both ports doing down and back up doesn't break statistics. ++AT_CHECK([ovs-appctl netdev-dummy/set-admin-state p1 down], 0, [OK ++]) ++AT_CHECK([ovs-appctl netdev-dummy/set-admin-state p2 down], 0, [OK ++]) ++ovs-appctl time/warp 1000 100 ++AT_CHECK([ovs-appctl netdev-dummy/set-admin-state p1 up], 0, [OK ++]) ++AT_CHECK([ovs-appctl netdev-dummy/set-admin-state p2 up], 0, [OK ++]) ++ovs-appctl time/warp 1000 100 ++ ++AT_CHECK([SEND_TCP_BOND_PKTS([p5], [5], [65500])]) ++# We sent 49125 KB of data total in 3 batches. No hash should have more ++# than that amount of load. Just checking that it is within 5 digits. ++AT_CHECK([ovs-appctl bond/show | grep -E '[[0-9]]{6}'], [1]) ++ + OVS_VSWITCHD_STOP() + AT_CLEANUP + +@@ -849,7 +866,7 @@ table=2 ip actions=set_field:192.168.3.91->ip_src,output(11) AT_CHECK([ovs-ofctl -O OpenFlow12 add-flows br0 flows.txt]) AT_CHECK([ovs-appctl ofproto/trace br0 'in_port=1,dl_src=50:54:00:00:00:05,dl_dst=50:54:00:00:00:07,dl_type=0x0800,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_proto=1,nw_tos=0,nw_ttl=128,nw_frag=no,icmp_type=8,icmp_code=0'], [0], [stdout]) AT_CHECK([tail -2 stdout], [0], @@ -7948,7 +10714,7 @@ index fa6111c1e..450483bee 100644 Datapath actions: 10,set(ipv4(src=192.168.3.91)),11,set(ipv4(src=192.168.3.90)),13 ]) OVS_VSWITCHD_STOP -@@ -912,7 +912,7 @@ AT_CHECK([ovs-appctl ofproto/trace br0 'in_port=1,dl_src=50:54:00:00:00:05,dl_ds +@@ -912,7 +929,7 @@ AT_CHECK([ovs-appctl ofproto/trace br0 'in_port=1,dl_src=50:54:00:00:00:05,dl_ds # Must match on the source address to be able to restore it's value for # the second bucket AT_CHECK([tail -2 stdout], [0], @@ -7957,7 +10723,7 @@ index fa6111c1e..450483bee 100644 Datapath actions: set(ipv4(src=192.168.3.90)),10,set(ipv4(src=192.168.0.1)),11 ]) OVS_VSWITCHD_STOP -@@ -944,7 +944,7 @@ done +@@ -944,7 +961,7 @@ done AT_CHECK([ovs-appctl dpctl/dump-flows | sed 's/dp_hash(.*\/0xf)/dp_hash(0xXXXX\/0xf)/' | sed 's/packets.*actions:/actions:/' | strip_ufid | strip_used | sort], [0], [dnl flow-dump from the main thread: recirc_id(0),in_port(1),packet_type(ns=0,id=0),eth_type(0x0800),ipv4(frag=no), actions:hash(sym_l4(0)),recirc(0x1) @@ -7966,7 +10732,7 @@ index fa6111c1e..450483bee 100644 ]) OVS_VSWITCHD_STOP -@@ -959,7 +959,7 @@ AT_CHECK([ovs-appctl ofproto/trace br0 'in_port=1,dl_src=50:54:00:00:00:05,dl_ds +@@ -959,7 +976,7 @@ AT_CHECK([ovs-appctl ofproto/trace br0 'in_port=1,dl_src=50:54:00:00:00:05,dl_ds # Must match on the source address to be able to restore it's value for # the third bucket AT_CHECK([tail -2 stdout], [0], @@ -7975,7 +10741,7 @@ index fa6111c1e..450483bee 100644 Datapath actions: set(ipv4(src=192.168.3.90)),10,set(ipv4(src=192.168.0.1)),11 ]) OVS_VSWITCHD_STOP -@@ -1536,17 +1536,17 @@ AT_CHECK([ovs-ofctl add-flows br0 flows.txt]) +@@ -1536,17 +1553,17 @@ AT_CHECK([ovs-ofctl add-flows br0 flows.txt]) AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(1),eth(src=50:54:00:00:00:05,dst=50:54:00:00:00:07),eth_type(0x0800),ipv4(src=192.168.0.1,dst=192.168.0.2,proto=111,tos=0,ttl=2,frag=no)' -generate], [0], [stdout]) AT_CHECK([tail -4 stdout], [0], [ Final flow: ip,in_port=1,vlan_tci=0x0000,dl_src=50:54:00:00:00:05,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_proto=111,nw_tos=0,nw_ecn=0,nw_ttl=1,nw_frag=no @@ -7996,7 +10762,7 @@ index fa6111c1e..450483bee 100644 Datapath actions: set(ipv6(hlimit=127)),2,set(ipv6(hlimit=126)),3,4 ]) -@@ -1656,7 +1656,7 @@ AT_CHECK([ovs-vsctl -- \ +@@ -1656,7 +1673,7 @@ AT_CHECK([ovs-vsctl -- \ --id=@q2 create Queue dscp=2], [0], [ignore]) AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(9),eth(src=50:54:00:00:00:05,dst=50:54:00:00:00:07),eth_type(0x0800),ipv4(src=1.1.1.1,dst=2.2.2.2,proto=1,tos=0xff,ttl=128,frag=no),icmp(type=8,code=0)'], [0], [stdout]) AT_CHECK([tail -2 stdout], [0], @@ -8005,7 +10771,7 @@ index fa6111c1e..450483bee 100644 Datapath actions: dnl 100,dnl set(ipv4(tos=0x4/0xfc)),set(skb_priority(0x1)),1,dnl -@@ -5349,7 +5349,7 @@ AT_CHECK([ovs-ofctl add-flows br0 flows.txt]) +@@ -5349,7 +5366,7 @@ AT_CHECK([ovs-ofctl add-flows br0 flows.txt]) flow="in_port(1),eth(src=50:54:00:00:00:05,dst=50:54:00:00:00:07),eth_type(0x0800),ipv4(src=192.168.0.1,dst=192.168.0.2,proto=1,tos=0,ttl=128,frag=no),icmp(type=8,code=0)" AT_CHECK([ovs-appctl ofproto/trace ovs-dummy "$flow"], [0], [stdout]) AT_CHECK_UNQUOTED([tail -1 stdout], [0], @@ -8014,7 +10780,7 @@ index fa6111c1e..450483bee 100644 ]) flow="in_port(2),eth(src=50:54:00:00:00:05,dst=50:54:00:00:00:07),eth_type(0x0800),ipv4(src=192.168.0.1,dst=192.168.0.2,proto=1,tos=0,ttl=128,frag=no),icmp(type=8,code=0)" -@@ -5388,7 +5388,7 @@ flow="in_port(2),eth(src=50:54:00:00:00:05,dst=50:54:00:00:00:07),eth_type(0x080 +@@ -5388,7 +5405,7 @@ flow="in_port(2),eth(src=50:54:00:00:00:05,dst=50:54:00:00:00:07),eth_type(0x080 AT_CHECK([ovs-appctl ofproto/trace ovs-dummy "$flow"], [0], [stdout]) actual=`tail -1 stdout | sed 's/Datapath actions: //'` @@ -8023,7 +10789,7 @@ index fa6111c1e..450483bee 100644 AT_CHECK([ovs-dpctl normalize-actions "$flow" "$expected"], [0], [stdout]) mv stdout expout AT_CHECK([ovs-dpctl normalize-actions "$flow" "$actual"], [0], [expout]) -@@ -5656,7 +5656,7 @@ AT_CHECK([ovs-ofctl add-flows br0 flows.txt]) +@@ -5656,7 +5673,7 @@ AT_CHECK([ovs-ofctl add-flows br0 flows.txt]) flow="in_port(1),eth(src=50:54:00:00:00:05,dst=50:54:00:00:00:07),eth_type(0x0800),ipv4(src=192.168.0.1,dst=192.168.0.2,proto=1,tos=0,ttl=128,frag=no),icmp(type=8,code=0)" AT_CHECK([ovs-appctl ofproto/trace ovs-dummy "$flow"], [0], [stdout]) AT_CHECK_UNQUOTED([tail -1 stdout], [0], @@ -8032,7 +10798,7 @@ index fa6111c1e..450483bee 100644 ]) flow="in_port(2),eth(src=50:54:00:00:00:05,dst=50:54:00:00:00:07),eth_type(0x0800),ipv4(src=192.168.0.1,dst=192.168.0.2,proto=1,tos=0,ttl=128,frag=no),icmp(type=8,code=0)" -@@ -5854,6 +5854,40 @@ OVS_WAIT_UNTIL([check_flows], [ovs-ofctl dump-flows br0]) +@@ -5854,6 +5871,40 @@ OVS_WAIT_UNTIL([check_flows], [ovs-ofctl dump-flows br0]) OVS_VSWITCHD_STOP AT_CLEANUP @@ -8073,7 +10839,99 @@ index fa6111c1e..450483bee 100644 AT_SETUP([ofproto-dpif - debug_slow action]) OVS_VSWITCHD_START add_of_ports br0 1 2 3 -@@ -11884,7 +11918,7 @@ ovs-ofctl dump-flows br0 +@@ -6144,6 +6195,57 @@ AT_CHECK([test 1 = `$PYTHON3 "$top_srcdir/utilities/ovs-pcap.in" p2-tx.pcap | wc + OVS_VSWITCHD_STOP + AT_CLEANUP + ++AT_SETUP([ofproto-dpif - continuation with meters]) ++AT_KEYWORDS([continuations pause meters]) ++OVS_VSWITCHD_START ++add_of_ports br0 1 2 ++ ++dnl Add meter with id=1. ++AT_CHECK([ovs-ofctl -O OpenFlow13 add-meter br0 'meter=1 pktps bands=type=drop rate=1']) ++ ++AT_DATA([flows.txt], [dnl ++table=0 dl_dst=50:54:00:00:00:0a actions=goto_table(1) ++table=1 dl_dst=50:54:00:00:00:0a actions=controller(pause,meter_id=1) ++]) ++AT_CHECK([ovs-ofctl -O OpenFlow13 add-flows br0 flows.txt]) ++ ++on_exit 'kill $(cat ovs-ofctl.pid)' ++AT_CAPTURE_FILE([ofctl_monitor.log]) ++AT_CHECK([ovs-ofctl monitor br0 65534 invalid_ttl -P nxt_packet_in \ ++ --detach --no-chdir --pidfile 2> ofctl_monitor.log]) ++ ++AT_CHECK([ovs-appctl netdev-dummy/receive p1 \ ++ 'in_port(1),eth(src=50:54:00:00:00:09,dst=50:54:00:00:00:0a),eth_type(0x1234)']) ++ ++OVS_WAIT_UNTIL([test $(wc -l < ofctl_monitor.log) -ge 2]) ++OVS_APP_EXIT_AND_WAIT([ovs-ofctl]) ++AT_CHECK([cat ofctl_monitor.log], [0], [dnl ++NXT_PACKET_IN (xid=0x0): cookie=0x0 total_len=14 in_port=1 (via action) data_len=14 (unbuffered) ++vlan_tci=0x0000,dl_src=50:54:00:00:00:09,dl_dst=50:54:00:00:00:0a,dl_type=0x1234 ++]) ++ ++AT_CHECK([ovs-appctl revalidator/purge], [0]) ++AT_CHECK([ovs-ofctl -O OpenFlow13 dump-flows br0 | ofctl_strip | sort], [0], [dnl ++ n_packets=1, n_bytes=14, dl_dst=50:54:00:00:00:0a actions=goto_table:1 ++ table=1, n_packets=1, n_bytes=14, dl_dst=50:54:00:00:00:0a actions=controller(pause,meter_id=1) ++OFPST_FLOW reply (OF1.3): ++]) ++ ++AT_CHECK([ovs-ofctl -O OpenFlow13 dump-meters br0 | ofctl_strip | sort], [0], [dnl ++OFPST_METER_CONFIG reply (OF1.3): ++meter=1 pktps bands= ++type=drop rate=1 ++]) ++ ++AT_CHECK([ovs-ofctl -O OpenFlow13 meter-stats br0 | strip_timers], [0], [dnl ++OFPST_METER reply (OF1.3) (xid=0x2): ++meter:1 flow_count:0 packet_in_count:1 byte_in_count:14 duration:0.0s bands: ++0: packet_count:0 byte_count:0 ++]) ++ ++OVS_VSWITCHD_STOP ++AT_CLEANUP ++ + AT_SETUP([ofproto-dpif - continuation with patch port]) + AT_KEYWORDS([continuations pause resume]) + OVS_VSWITCHD_START( +@@ -7619,12 +7721,14 @@ dummy@ovs-dummy: hit:0 missed:0 + vm1 5/3: (dummy: ifindex=2011) + ]) + +-dnl set up route to 1.1.2.92 via br0 and action=normal ++dnl Add 1.1.2.92 to br0 and action=normal + AT_CHECK([ovs-appctl netdev-dummy/ip4addr br0 1.1.2.88/24], [0], [OK + ]) +-AT_CHECK([ovs-appctl ovs/route/add 1.1.2.92/24 br0], [0], [OK +-]) + AT_CHECK([ovs-ofctl add-flow br0 action=normal]) ++dnl Checking that a local route for added IP was successfully installed. ++AT_CHECK([ovs-appctl ovs/route/show | grep Cached], [0], [dnl ++Cached: 1.1.2.0/24 dev br0 SRC 1.1.2.88 local ++]) + + dnl Prime ARP Cache for 1.1.2.92 + AT_CHECK([ovs-appctl netdev-dummy/receive p0 'recirc_id(0),in_port(1),eth(src=f8:bc:12:44:34:b6,dst=ff:ff:ff:ff:ff:ff),eth_type(0x0806),arp(sip=1.1.2.92,tip=1.1.2.88,op=2,sha=f8:bc:12:44:34:b6,tha=00:00:00:00:00:00)']) +@@ -7635,10 +7739,13 @@ ovs-vsctl \ + --id=@sf create sflow targets=\"127.0.0.1:$SFLOW_PORT\" agent=127.0.0.1 \ + header=128 sampling=1 polling=0 + +-dnl set up route to 192.168.1.2 via br0 ++dnl Add 192.168.1.2 to br0, + AT_CHECK([ovs-appctl netdev-dummy/ip4addr br0 192.168.1.1/16], [0], [OK + ]) +-AT_CHECK([ovs-appctl ovs/route/add 192.168.0.0/16 br0], [0], [OK ++dnl Checking that a local route for added IP was successfully installed. ++AT_CHECK([ovs-appctl ovs/route/show | grep Cached | sort], [0], [dnl ++Cached: 1.1.2.0/24 dev br0 SRC 1.1.2.88 local ++Cached: 192.168.0.0/16 dev br0 SRC 192.168.1.1 local + ]) + + dnl add rule for int-br to force packet onto tunnel. There is no ifindex +@@ -11884,7 +11991,7 @@ ovs-ofctl dump-flows br0 AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(1),eth(src=50:54:00:00:00:09,dst=50:54:00:00:00:0a),eth_type(0x0800),ipv4(src=10.10.10.2,dst=10.10.10.1,proto=1,tos=1,ttl=128,frag=no),icmp(type=8,code=0)'], [0], [stdout]) AT_CHECK([tail -3 stdout], [0], [dnl @@ -8082,8 +10940,34 @@ index fa6111c1e..450483bee 100644 Datapath actions: drop Translation failed (Recursion too deep), packet is dropped. ]) +diff --git a/tests/ofproto-macros.at b/tests/ofproto-macros.at +index 676d55aa95..fcffe21075 100644 +--- a/tests/ofproto-macros.at ++++ b/tests/ofproto-macros.at +@@ -141,6 +141,21 @@ strip_stats () { + s/bytes:[[0-9]]*/bytes:0/' + } + ++# Strips key32 field from output. ++strip_key32 () { ++ sed 's/key32([[0-9 \/]]*),//' ++} ++ ++# Strips packet-type from output. ++strip_ptype () { ++ sed 's/packet_type(ns=[[0-9]]*,id=[[0-9]]*),//' ++} ++ ++# Strips bare eth from output. ++strip_eth () { ++ sed 's/eth(),//' ++} ++ + # Changes all 'recirc(...)' and 'recirc=...' to say 'recirc()' and + # 'recirc=' respectively. This should make output easier to + # compare. diff --git a/tests/ofproto.at b/tests/ofproto.at -index a666bebca..2889f81fb 100644 +index a666bebcac..2889f81fb1 100644 --- a/tests/ofproto.at +++ b/tests/ofproto.at @@ -6538,3 +6538,213 @@ verify_deleted @@ -8301,7 +11185,7 @@ index a666bebca..2889f81fb 100644 +OVS_VSWITCHD_STOP(["/br0<->unix:testcontroller: connection failed/d"]) +AT_CLEANUP diff --git a/tests/ovs-vsctl.at b/tests/ovs-vsctl.at -index a92156f00..a368bff6e 100644 +index a92156f001..a368bff6ed 100644 --- a/tests/ovs-vsctl.at +++ b/tests/ovs-vsctl.at @@ -425,6 +425,7 @@ AT_CHECK([RUN_OVS_VSCTL_ONELINE( @@ -8335,8 +11219,148 @@ index a92156f00..a368bff6e 100644 AT_CHECK([RUN_OVS_VSCTL([remove netflow `cat netflow-uuid` targets '"1.2.3.4:567"'])], [1], [], [ovs-vsctl: "remove" operation would put 0 values in column targets of table NetFlow but the minimum number is 1 ]) +diff --git a/tests/ovsdb-client.at b/tests/ovsdb-client.at +index 2d14f1ac26..c618cf4839 100644 +--- a/tests/ovsdb-client.at ++++ b/tests/ovsdb-client.at +@@ -270,8 +270,8 @@ AT_CHECK([ovsdb-client --replay=./replay_dir dnl + dnl Waiting for client to exit the same way as it exited during recording. + OVS_WAIT_WHILE([test -e ovsdb-client.pid]) + +-AT_CHECK([diff monitor.stdout monitor-replay.stdout]) +-AT_CHECK([diff monitor.stderr monitor-replay.stderr]) ++AT_CHECK([diff -u monitor.stdout monitor-replay.stdout]) ++AT_CHECK([diff -u monitor.stderr monitor-replay.stderr]) + + dnl Stripping out timestamps, PIDs and poll_loop warnings from the log. + dnl Also stripping socket_util errors as sockets are not used in replay. +@@ -284,6 +284,6 @@ m4_define([CLEAN_LOG_FILE], + CLEAN_LOG_FILE([monitor.log], [monitor.log.clear]) + CLEAN_LOG_FILE([monitor-replay.log], [monitor-replay.log.clear]) + +-AT_CHECK([diff monitor.log.clear monitor-replay.log.clear]) ++AT_CHECK([diff -u monitor.log.clear monitor-replay.log.clear]) + + AT_CLEANUP +diff --git a/tests/ovsdb-cluster.at b/tests/ovsdb-cluster.at +index 9fbf5dc897..987e05103d 100644 +--- a/tests/ovsdb-cluster.at ++++ b/tests/ovsdb-cluster.at +@@ -483,6 +483,112 @@ done + + AT_CLEANUP + ++AT_SETUP([OVSDB cluster - leadership change after replication while joining]) ++AT_KEYWORDS([ovsdb server negative unix cluster join]) ++ ++n=5 ++AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db dnl ++ $abs_srcdir/idltest.ovsschema unix:s1.raft], [0], [], [stderr]) ++cid=$(ovsdb-tool db-cid s1.db) ++schema_name=$(ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema) ++for i in $(seq 2 $n); do ++ AT_CHECK([ovsdb-tool join-cluster s$i.db $schema_name unix:s$i.raft unix:s1.raft]) ++done ++ ++on_exit 'kill $(cat *.pid)' ++on_exit " ++ for i in \$(ls $(pwd)/s[[0-$n]]); do ++ ovs-appctl --timeout 1 -t \$i cluster/status $schema_name; ++ done ++" ++ ++dnl Starting servers one by one asking all exisitng servers to transfer ++dnl leadership after append reply forcing the joining server to try another ++dnl one that will also transfer leadership. Since transfer is happening ++dnl after the servers update is replicated to other servers, one of the ++dnl other servers will actually commit it. It may be a new leader from ++dnl one of the old members or the new joining server itself. ++for i in $(seq $n); do ++ dnl Make sure that all already started servers joined the cluster. ++ for j in $(seq $((i - 1)) ); do ++ AT_CHECK([ovsdb_client_wait unix:s$j.ovsdb $schema_name connected]) ++ done ++ for j in $(seq $((i - 1)) ); do ++ OVS_WAIT_UNTIL([ovs-appctl -t "$(pwd)"/s$j \ ++ cluster/failure-test \ ++ transfer-leadership-after-sending-append-request \ ++ | grep -q "engaged"]) ++ done ++ ++ AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off \ ++ --detach --no-chdir --log-file=s$i.log \ ++ --pidfile=s$i.pid --unixctl=s$i \ ++ --remote=punix:s$i.ovsdb s$i.db]) ++done ++ ++dnl Make sure that all servers joined the cluster. ++for i in $(seq $n); do ++ AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected]) ++done ++ ++for i in $(seq $n); do ++ OVS_APP_EXIT_AND_WAIT_BY_TARGET([$(pwd)/s$i], [s$i.pid]) ++done ++ ++AT_CLEANUP ++ ++AT_SETUP([OVSDB cluster - leadership change before replication while joining]) ++AT_KEYWORDS([ovsdb server negative unix cluster join]) ++ ++n=5 ++AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db dnl ++ $abs_srcdir/idltest.ovsschema unix:s1.raft], [0], [], [stderr]) ++cid=$(ovsdb-tool db-cid s1.db) ++schema_name=$(ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema) ++for i in $(seq 2 $n); do ++ AT_CHECK([ovsdb-tool join-cluster s$i.db $schema_name unix:s$i.raft unix:s1.raft]) ++done ++ ++on_exit 'kill $(cat *.pid)' ++on_exit " ++ for i in \$(ls $(pwd)/s[[0-$n]]); do ++ ovs-appctl --timeout 1 -t \$i cluster/status $schema_name; ++ done ++" ++ ++dnl Starting servers one by one asking all exisitng servers to transfer ++dnl leadership right after starting to add a server. Joining server will ++dnl need to find a new leader that will also transfer leadership. ++dnl This will continue until the same server will not become a leader ++dnl for the second time and will be able to add a new server. ++for i in $(seq $n); do ++ dnl Make sure that all already started servers joined the cluster. ++ for j in $(seq $((i - 1)) ); do ++ AT_CHECK([ovsdb_client_wait unix:s$j.ovsdb $schema_name connected]) ++ done ++ for j in $(seq $((i - 1)) ); do ++ OVS_WAIT_UNTIL([ovs-appctl -t "$(pwd)"/s$j \ ++ cluster/failure-test \ ++ transfer-leadership-after-starting-to-add \ ++ | grep -q "engaged"]) ++ done ++ ++ AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off \ ++ --detach --no-chdir --log-file=s$i.log \ ++ --pidfile=s$i.pid --unixctl=s$i \ ++ --remote=punix:s$i.ovsdb s$i.db]) ++done ++ ++dnl Make sure that all servers joined the cluster. ++for i in $(seq $n); do ++ AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected]) ++done ++ ++for i in $(seq $n); do ++ OVS_APP_EXIT_AND_WAIT_BY_TARGET([$(pwd)/s$i], [s$i.pid]) ++done ++ ++AT_CLEANUP + + + OVS_START_SHELL_HELPERS diff --git a/tests/ovsdb-execution.at b/tests/ovsdb-execution.at -index e72bf0606..fd1c7a239 100644 +index e72bf06069..fd1c7a2395 100644 --- a/tests/ovsdb-execution.at +++ b/tests/ovsdb-execution.at @@ -728,6 +728,53 @@ dnl collide (only) with their previous values (succeeds). @@ -8415,11 +11439,80 @@ index e72bf0606..fd1c7a239 100644 [{"count":1},{"details":"cannot delete b row <0> because of 2 remaining reference(s)","error":"referential integrity violation"}] [{"count":1}] diff --git a/tests/ovsdb-idl.at b/tests/ovsdb-idl.at -index 5a7e76eaa..1dd334325 100644 +index 5a7e76eaa9..aa60b1e5a1 100644 --- a/tests/ovsdb-idl.at +++ b/tests/ovsdb-idl.at -@@ -94,7 +94,7 @@ m4_define([OVSDB_CHECK_IDL_WRITE_CHANGED_ONLY_C], - AT_CHECK([ovsdb_start_idltest]) +@@ -1,17 +1,6 @@ + AT_BANNER([OVSDB -- interface description language (IDL)]) + + m4_divert_text([PREPARE_TESTS], [ +-# ovsdb_start_idltest [REMOTE] [SCHEMA] +-# +-# Creates a database using SCHEMA (default: idltest.ovsschema) and +-# starts a database server listening on punix:socket and REMOTE (if +-# specified). +-ovsdb_start_idltest () { +- ovsdb-tool create db ${2:-$abs_srcdir/idltest.ovsschema} || return $? +- ovsdb-server -vconsole:warn --log-file --detach --no-chdir --pidfile --remote=punix:socket ${1:+--remote=$1} db || return $? +- on_exit 'kill `cat ovsdb-server.pid`' +-} +- + # ovsdb_cluster_leader [REMOTES] [DATABASE] + # + # Returns the leader of the DATABASE cluster. +@@ -29,6 +18,24 @@ ovsdb_cluster_leader () { + done + }]) + ++ ++# OVSDB_START_IDLTEST([REMOTE], [SCHEMA]) ++# ++# Creates a database using SCHEMA (default: idltest.ovsschema) and ++# starts a database server listening on punix:socket and REMOTE (if ++# specified). ++m4_define([OVSDB_START_IDLTEST], ++[ ++ AT_CHECK([ovsdb-tool create db dnl ++ m4_if([$2], [], [$abs_srcdir/idltest.ovsschema], [$2])]) ++ AT_CHECK([ovsdb-server -vconsole:warn --log-file --detach --no-chdir dnl ++ --pidfile --remote=punix:socket dnl ++ m4_if([$1], [], [], [--remote=$1]) db dnl ++ ]) ++ on_exit 'kill `cat ovsdb-server.pid`' ++]) ++ ++ + # OVSDB_CLUSTER_START_IDLTEST([N], [REMOTE]) + # + # Creates a clustered database using idltest.ovsschema and starts a database +@@ -45,9 +52,9 @@ m4_define([OVSDB_CLUSTER_START_IDLTEST], + done + on_exit 'kill $(cat s*.pid)' + for i in $(seq $n); do +- AT_CHECK([ovsdb-server -vraft -vconsole:warn --detach --no-chdir \ +- --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i \ +- --remote=punix:s$i.ovsdb \ ++ AT_CHECK([ovsdb-server -vraft -vconsole:warn -vfile:dbg --detach \ ++ --no-chdir --log-file=s$i.log --pidfile=s$i.pid \ ++ --unixctl=s$i --remote=punix:s$i.ovsdb \ + m4_if([$2], [], [], [--remote=$2]) s$i.db]) + done + +@@ -77,7 +84,7 @@ m4_define([OVSDB_CLUSTER_START_IDLTEST], + m4_define([OVSDB_CHECK_IDL_C], + [AT_SETUP([$1 - C]) + AT_KEYWORDS([ovsdb server idl positive $5]) +- AT_CHECK([ovsdb_start_idltest]) ++ OVSDB_START_IDLTEST + m4_if([$2], [], [], + [AT_CHECK([ovsdb-client transact unix:socket $2], [0], [ignore], [ignore])]) + AT_CHECK([test-ovsdb '-vPATTERN:console:test-ovsdb|%c|%m' -vjsonrpc -t10 idl unix:socket $3], +@@ -91,10 +98,10 @@ m4_define([OVSDB_CHECK_IDL_C], + m4_define([OVSDB_CHECK_IDL_WRITE_CHANGED_ONLY_C], + [AT_SETUP([$1 - write-changed-only - C]) + AT_KEYWORDS([ovsdb server idl positive $5]) +- AT_CHECK([ovsdb_start_idltest]) ++ OVSDB_START_IDLTEST m4_if([$2], [], [], [AT_CHECK([ovsdb-client transact unix:socket $2], [0], [ignore], [ignore])]) - AT_CHECK([test-ovsdb '-vPATTERN:console:test-ovsdb|%c|%m' -vjsonrpc -t10 idl unix:socket $3], @@ -8427,8 +11520,145 @@ index 5a7e76eaa..1dd334325 100644 [0], [stdout], [ignore]) AT_CHECK([sort stdout | uuidfilt]m4_if([$6],,, [[| $6]]), [0], [$4]) -@@ -1216,7 +1216,7 @@ m4_define([OVSDB_CHECK_IDL_TRACK_WRITE_CHANGED_ONLY_C], - AT_CHECK([ovsdb_start_idltest]) +@@ -105,7 +112,7 @@ m4_define([OVSDB_CHECK_IDL_WRITE_CHANGED_ONLY_C], + m4_define([OVSDB_CHECK_IDL_TCP_C], + [AT_SETUP([$1 - C - tcp]) + AT_KEYWORDS([ovsdb server idl positive tcp socket $5]) +- AT_CHECK([ovsdb_start_idltest "ptcp:0:127.0.0.1"]) ++ OVSDB_START_IDLTEST(["ptcp:0:127.0.0.1"]) + PARSE_LISTENING_PORT([ovsdb-server.log], [TCP_PORT]) + + m4_if([$2], [], [], +@@ -123,7 +130,7 @@ m4_define([OVSDB_CHECK_IDL_TCP6_C], + AT_SKIP_IF([test "$IS_WIN32" = "yes"]) + AT_SKIP_IF([test $HAVE_IPV6 = no]) + AT_KEYWORDS([ovsdb server idl positive tcp6 socket $5]) +- AT_CHECK([ovsdb_start_idltest "ptcp:0:[[::1]]"]) ++ OVSDB_START_IDLTEST(["ptcp:0:[[::1]]"]) + PARSE_LISTENING_PORT([ovsdb-server.log], [TCP_PORT]) + + m4_if([$2], [], [], +@@ -139,7 +146,7 @@ m4_define([OVSDB_CHECK_IDL_TCP6_C], + m4_define([OVSDB_CHECK_IDL_PY], + [AT_SETUP([$1 - Python3]) + AT_KEYWORDS([ovsdb server idl positive Python $5]) +- AT_CHECK([ovsdb_start_idltest]) ++ OVSDB_START_IDLTEST + m4_if([$2], [], [], + [AT_CHECK([ovsdb-client transact unix:socket $2], [0], [ignore], [ignore])]) + AT_CHECK([$PYTHON3 $srcdir/test-ovsdb.py -t10 idl $srcdir/idltest.ovsschema unix:socket $3], +@@ -152,7 +159,7 @@ m4_define([OVSDB_CHECK_IDL_PY], + m4_define([OVSDB_CHECK_IDL_REGISTER_COLUMNS_PY], + [AT_SETUP([$1 - Python3 - register_columns]) + AT_KEYWORDS([ovsdb server idl positive Python register_columns $5]) +- AT_CHECK([ovsdb_start_idltest]) ++ OVSDB_START_IDLTEST + m4_if([$2], [], [], + [AT_CHECK([ovsdb-client transact unix:socket $2], [0], [ignore], [ignore])]) + AT_CHECK([$PYTHON3 $srcdir/test-ovsdb.py -t10 idl $srcdir/idltest.ovsschema unix:socket ?simple:b,ba,i,ia,r,ra,s,sa,u,ua?simple3:name,uset,uref?simple4:name?simple6:name,weak_ref?link1:i,k,ka,l2?link2:i,l1?singleton:name $3], +@@ -166,7 +173,7 @@ m4_define([OVSDB_CHECK_IDL_REGISTER_COLUMNS_PY], + m4_define([OVSDB_CHECK_IDL_TCP_PY], + [AT_SETUP([$1 - Python3 - tcp]) + AT_KEYWORDS([ovsdb server idl positive Python with tcp socket $5]) +- AT_CHECK([ovsdb_start_idltest "ptcp:0:127.0.0.1"]) ++ OVSDB_START_IDLTEST(["ptcp:0:127.0.0.1"]) + PARSE_LISTENING_PORT([ovsdb-server.log], [TCP_PORT]) + + m4_if([$2], [], [], +@@ -183,7 +190,7 @@ m4_define([OVSDB_CHECK_IDL_TCP_PY], + m4_define([OVSDB_CHECK_IDL_TCP_MULTIPLE_REMOTES_PY], + [AT_SETUP([$1 - Python3 (multiple remotes) - tcp]) + AT_KEYWORDS([ovsdb server idl positive Python with tcp socket $5]) +- AT_CHECK([ovsdb_start_idltest "ptcp:0:127.0.0.1"]) ++ OVSDB_START_IDLTEST(["ptcp:0:127.0.0.1"]) + PARSE_LISTENING_PORT([ovsdb-server.log], [TCP_PORT]) + WRONG_PORT_1=$((TCP_PORT + 101)) + WRONG_PORT_2=$((TCP_PORT + 102)) +@@ -203,7 +210,7 @@ m4_define([OVSDB_CHECK_IDL_TCP6_PY], + AT_SKIP_IF([test "$IS_WIN32" = "yes"]) + AT_SKIP_IF([test $HAVE_IPV6 = no]) + AT_KEYWORDS([ovsdb server idl positive Python with tcp6 socket $5]) +- AT_CHECK([ovsdb_start_idltest "ptcp:0:[[::1]]"]) ++ OVSDB_START_IDLTEST(["ptcp:0:[[::1]]"]) + PARSE_LISTENING_PORT([ovsdb-server.log], [TCP_PORT]) + echo "TCP_PORT=$TCP_PORT" + +@@ -221,7 +228,7 @@ m4_define([OVSDB_CHECK_IDL_TCP6_MULTIPLE_REMOTES_PY], + AT_SKIP_IF([test "$IS_WIN32" = "yes"]) + AT_SKIP_IF([test $HAVE_IPV6 = no]) + AT_KEYWORDS([ovsdb server idl positive Python with tcp6 socket $5]) +- AT_CHECK([ovsdb_start_idltest "ptcp:0:[[::1]]"]) ++ OVSDB_START_IDLTEST(["ptcp:0:[[::1]]"]) + PARSE_LISTENING_PORT([ovsdb-server.log], [TCP_PORT]) + WRONG_PORT_1=$((TCP_PORT + 101)) + WRONG_PORT_2=$((TCP_PORT + 102)) +@@ -287,13 +294,13 @@ m4_define([OVSDB_CHECK_IDL_PASSIVE_TCP_PY], + [AT_SETUP([$1 - Python3 - ptcp]) + AT_KEYWORDS([ovsdb server idl positive Python with tcp socket $5]) + # find free TCP port +- AT_CHECK([ovsdb_start_idltest "ptcp:0:127.0.0.1"]) ++ OVSDB_START_IDLTEST(["ptcp:0:127.0.0.1"]) + PARSE_LISTENING_PORT([ovsdb-server.log], [TCP_PORT]) + OVSDB_SERVER_SHUTDOWN + rm -f db + + # start OVSDB server in passive mode +- AT_CHECK([ovsdb_start_idltest "tcp:127.0.0.1:$TCP_PORT"]) ++ OVSDB_START_IDLTEST(["tcp:127.0.0.1:$TCP_PORT"]) + AT_CHECK([$PYTHON3 $srcdir/test-ovsdb.py -t10 idl_passive $srcdir/idltest.ovsschema ptcp:127.0.0.1:$TCP_PORT $3], + [0], [stdout], [ignore]) + AT_CHECK([sort stdout | uuidfilt]m4_if([$6],,, [[| $6]]), +@@ -473,7 +480,7 @@ OVSDB_CHECK_IDL([simple idl, writing via IDL with unicode], + m4_define([OVSDB_CHECK_IDL_PY_WITH_EXPOUT], + [AT_SETUP([$1 - Python3]) + AT_KEYWORDS([ovsdb server idl positive Python $5]) +- AT_CHECK([ovsdb_start_idltest]) ++ OVSDB_START_IDLTEST + m4_if([$2], [], [], + [AT_CHECK([ovsdb-client transact unix:socket $2], [0], [ignore], [ignore])]) + AT_CHECK([$PYTHON3 $srcdir/test-ovsdb.py -t10 idl $srcdir/idltest.ovsschema unix:socket $3], +@@ -990,7 +997,7 @@ AT_KEYWORDS([ovsdb server idl positive]) + # table link2 and column l2 have been deleted. But the IDL still + # expects them to be there, so this test checks that it properly + # tolerates them being missing. +-AT_CHECK([ovsdb_start_idltest "" "$abs_srcdir/idltest2.ovsschema"]) ++OVSDB_START_IDLTEST([], ["$abs_srcdir/idltest2.ovsschema"]) + AT_CHECK([test-ovsdb '-vPATTERN:console:test-ovsdb|%c|%m' -vjsonrpc -t10 idl unix:socket ['["idltest", + {"op": "insert", + "table": "link1", +@@ -1063,7 +1070,7 @@ AT_CLEANUP + m4_define([OVSDB_CHECK_IDL_FETCH_COLUMNS_PY], + [AT_SETUP([$1 - Python3 - fetch]) + AT_KEYWORDS([ovsdb server idl positive Python increment fetch $6]) +- AT_CHECK([ovsdb_start_idltest]) ++ OVSDB_START_IDLTEST + m4_if([$2], [], [], + [AT_CHECK([ovsdb-client transact unix:socket $2], [0], [ignore], [ignore])]) + AT_CHECK([$PYTHON3 $srcdir/test-ovsdb.py -t10 idl $srcdir/idltest.ovsschema unix:socket [$3] $4], +@@ -1107,7 +1114,7 @@ OVSDB_CHECK_IDL_FETCH_COLUMNS([simple idl, initially populated], + m4_define([OVSDB_CHECK_IDL_WO_MONITOR_COND_PY], + [AT_SETUP([$1 - Python3]) + AT_KEYWORDS([ovsdb server idl Python monitor $4]) +- AT_CHECK([ovsdb_start_idltest]) ++ OVSDB_START_IDLTEST + AT_CHECK([ovs-appctl -t ovsdb-server ovsdb-server/disable-monitor-cond]) + AT_CHECK([$PYTHON3 $srcdir/test-ovsdb.py -t10 idl $srcdir/idltest.ovsschema unix:socket $2], + [0], [stdout], [ignore]) +@@ -1200,7 +1207,7 @@ OVSDB_CHECK_IDL_WO_MONITOR_COND([simple idl disable monitor-cond], + m4_define([OVSDB_CHECK_IDL_TRACK_C], + [AT_SETUP([$1 - C]) + AT_KEYWORDS([ovsdb server idl tracking positive $5]) +- AT_CHECK([ovsdb_start_idltest]) ++ OVSDB_START_IDLTEST + m4_if([$2], [], [], + [AT_CHECK([ovsdb-client transact unix:socket $2], [0], [ignore], [ignore])]) + AT_CHECK([test-ovsdb '-vPATTERN:console:test-ovsdb|%c|%m' -vjsonrpc -t10 -c idl unix:socket $3], +@@ -1213,10 +1220,10 @@ m4_define([OVSDB_CHECK_IDL_TRACK_C], + m4_define([OVSDB_CHECK_IDL_TRACK_WRITE_CHANGED_ONLY_C], + [AT_SETUP([$1 - write-changed-only - C]) + AT_KEYWORDS([ovsdb server idl tracking positive $5]) +- AT_CHECK([ovsdb_start_idltest]) ++ OVSDB_START_IDLTEST m4_if([$2], [], [], [AT_CHECK([ovsdb-client transact unix:socket $2], [0], [ignore], [ignore])]) - AT_CHECK([test-ovsdb '-vPATTERN:console:test-ovsdb|%c|%m' -vjsonrpc -t10 -c -w idl unix:socket $3], @@ -8436,7 +11666,118 @@ index 5a7e76eaa..1dd334325 100644 [0], [stdout], [ignore]) AT_CHECK([sort stdout | uuidfilt]m4_if([$6],,, [[| $6]]), [0], [$4]) -@@ -2296,6 +2296,23 @@ CHECK_STREAM_OPEN_BLOCK([Python3], [$PYTHON3 $srcdir/test-stream.py], +@@ -1451,6 +1458,56 @@ OVSDB_CHECK_IDL_TRACK([track, simple idl, initially populated, references, singl + 006: done + ]]) + ++OVSDB_CHECK_IDL_TRACK([track, simple idl, initially populated, weak references, insert+delete batch], ++ [['["idltest", ++ {"op": "insert", ++ "table": "simple", ++ "row": {"s": "row0_s"}, ++ "uuid-name": "uuid_row0_s"}, ++ {"op": "insert", ++ "table": "simple6", ++ "row": {"name": "row0_s6", ++ "weak_ref": ["set", ++ [["named-uuid", "uuid_row0_s"]] ++ ]}}]']], ++ [['condition simple [true];simple6 [true]' \ ++ '["idltest", ++ {"op": "insert", ++ "table": "simple", ++ "row": {"s": "row1_s"}, ++ "uuid-name": "uuid_row1_s"}, ++ {"op": "mutate", ++ "table": "simple6", ++ "where": [["name", "==", "row0_s6"]], ++ "mutations": [["weak_ref", "insert", ["set", [["named-uuid", "uuid_row1_s"]]]]]}]' \ ++ '+["idltest", ++ {"op": "delete", ++ "table": "simple", ++ "where": [["s", "==", "row1_s"]]}]' \ ++ '["idltest", ++ {"op": "insert", ++ "table": "simple", ++ "row": {"s": "row2_s"}}]']], ++ [[000: simple6: conditions unchanged ++000: simple: conditions unchanged ++001: table simple6: inserted row: name=row0_s6 weak_ref=[<0>] uuid=<1> ++001: table simple6: updated columns: name weak_ref ++001: table simple: inserted row: i=0 r=0 b=false s=row0_s u=<2> ia=[] ra=[] ba=[] sa=[] ua=[] uuid=<0> ++001: table simple: updated columns: s ++002: {"error":null,"result":[{"uuid":["uuid","<3>"]},{"count":1}]} ++003: {"error":null,"result":[{"count":1}]} ++004: table simple6: name=row0_s6 weak_ref=[<0>] uuid=<1> ++004: table simple6: updated columns: weak_ref ++004: table simple: inserted/deleted row: i=0 r=0 b=false s=row1_s u=<2> ia=[] ra=[] ba=[] sa=[] ua=[] uuid=<3> ++004: table simple: updated columns: s ++005: {"error":null,"result":[{"uuid":["uuid","<4>"]}]} ++006: table simple6: name=row0_s6 weak_ref=[<0>] uuid=<1> ++006: table simple: i=0 r=0 b=false s=row0_s u=<2> ia=[] ra=[] ba=[] sa=[] ua=[] uuid=<0> ++006: table simple: inserted row: i=0 r=0 b=false s=row2_s u=<2> ia=[] ra=[] ba=[] sa=[] ua=[] uuid=<4> ++006: table simple: updated columns: s ++007: done ++]]) ++ + dnl This test checks that deleting both the destination and source of the + dnl reference doesn't remove the reference in the source tracked record. + OVSDB_CHECK_IDL_TRACK([track, simple idl, initially populated, weak references, multiple deletes], +@@ -1716,7 +1773,7 @@ OVSDB_CHECK_IDL_TRACK([track, simple idl, initially empty, various ops], + m4_define([OVSDB_CHECK_IDL_PARTIAL_UPDATE_MAP_COLUMN], + [AT_SETUP([$1 - C]) + AT_KEYWORDS([ovsdb server idl partial update map column positive $5]) +- AT_CHECK([ovsdb_start_idltest]) ++ OVSDB_START_IDLTEST + m4_if([$2], [], [], + [AT_CHECK([ovsdb-client transact unix:socket $2], [0], [ignore], [ignore])]) + AT_CHECK([test-ovsdb '-vPATTERN:console:test-ovsdb|%c|%m' -vjsonrpc -t10 -c idl-partial-update-map-column unix:socket $3], +@@ -1777,7 +1834,7 @@ OVSDB_CHECK_IDL_PY([partial-map update set refmap idl], + m4_define([OVSDB_CHECK_IDL_PARTIAL_UPDATE_SET_COLUMN], + [AT_SETUP([$1 - C]) + AT_KEYWORDS([ovsdb server idl partial update set column positive $5]) +- AT_CHECK([ovsdb_start_idltest]) ++ OVSDB_START_IDLTEST + m4_if([$2], [], [], + [AT_CHECK([ovsdb-client transact unix:socket $2], [0], [ignore], [ignore])]) + AT_CHECK([test-ovsdb '-vPATTERN:console:test-ovsdb|%c|%m' -vjsonrpc -t10 -c idl-partial-update-set-column unix:socket $3], +@@ -1966,7 +2023,7 @@ OVSDB_CHECK_IDL_NOTIFY([simple idl verify notify], + m4_define([OVSDB_CHECK_IDL_COMPOUND_INDEX_SINGLE_COLUMN_C], + [AT_SETUP([$1 - C]) + AT_KEYWORDS([ovsdb server idl compound_index_single_column compound_index positive $5]) +- AT_CHECK([ovsdb_start_idltest]) ++ OVSDB_START_IDLTEST + m4_if([$2], [], [], + [AT_CHECK([ovsdb-client transact unix:socket $2], [0], [ignore], [ignore])]) + # Generate the data to be tested. +@@ -2113,7 +2170,7 @@ OVSDB_CHECK_IDL_COMPOUND_INDEX_SINGLE_COLUMN_C([Compound_index, single column te + m4_define([OVSDB_CHECK_IDL_COMPOUND_INDEX_DOUBLE_COLUMN_C], + [AT_SETUP([$1 - C]) + AT_KEYWORDS([ovsdb server idl compound_index_double_column compound_index positive $5]) +- AT_CHECK([ovsdb_start_idltest]) ++ OVSDB_START_IDLTEST + m4_if([$2], [], [], + [AT_CHECK([ovsdb-client transact unix:socket $2], [0], [ignore], [ignore])]) + # Generate the data to be tested. +@@ -2252,7 +2309,7 @@ OVSDB_CHECK_IDL_COMPOUND_INDEX_DOUBLE_COLUMN_C([Compound_index, double column te + m4_define([OVSDB_CHECK_IDL_COMPOUND_INDEX_WITH_REF], + [AT_SETUP([$1 - C]) + AT_KEYWORDS([ovsdb server idl compound_index compound_index_with_ref positive $5]) +- AT_CHECK([ovsdb_start_idltest]) ++ OVSDB_START_IDLTEST + m4_if([$2], [], [], + [AT_CHECK([ovsdb-client transact unix:socket $2], [0], [ignore], [ignore])]) + AT_CHECK([test-ovsdb '-vPATTERN:console:test-ovsdb|%c|%m' -vjsonrpc -t10 -c idl-compound-index-with-ref unix:socket $3], +@@ -2280,7 +2337,7 @@ m4_define([CHECK_STREAM_OPEN_BLOCK], + AT_SKIP_IF([test "$3" = "tcp6" && test "$IS_WIN32" = "yes"]) + AT_SKIP_IF([test "$3" = "tcp6" && test "$HAVE_IPV6" = "no"]) + AT_KEYWORDS([ovsdb server stream open_block $3]) +- AT_CHECK([ovsdb_start_idltest "ptcp:0:$4"]) ++ OVSDB_START_IDLTEST(["ptcp:0:$4"]) + PARSE_LISTENING_PORT([ovsdb-server.log], [TCP_PORT]) + WRONG_PORT=$(($TCP_PORT + 101)) + AT_CHECK([$2 tcp:$4:$TCP_PORT], [0], [ignore]) +@@ -2296,6 +2353,23 @@ CHECK_STREAM_OPEN_BLOCK([Python3], [$PYTHON3 $srcdir/test-stream.py], CHECK_STREAM_OPEN_BLOCK([Python3], [$PYTHON3 $srcdir/test-stream.py], [tcp6], [[[::1]]]) @@ -8460,7 +11801,7 @@ index 5a7e76eaa..1dd334325 100644 # same as OVSDB_CHECK_IDL but uses Python IDL implementation with tcp # with multiple remotes to assert the idl connects to the leader of the Raft cluster m4_define([OVSDB_CHECK_IDL_LEADER_ONLY_PY], -@@ -2311,10 +2328,11 @@ m4_define([OVSDB_CHECK_IDL_LEADER_ONLY_PY], +@@ -2311,10 +2385,11 @@ m4_define([OVSDB_CHECK_IDL_LEADER_ONLY_PY], pids=$(cat s2.pid s3.pid s1.pid | tr '\n' ',') echo $pids AT_CHECK([$PYTHON3 $srcdir/test-ovsdb.py -t30 idl-cluster $srcdir/idltest.ovsschema $remotes $pids $3], @@ -8473,7 +11814,7 @@ index 5a7e76eaa..1dd334325 100644 AT_CLEANUP]) OVSDB_CHECK_IDL_LEADER_ONLY_PY([Check Python IDL connects to leader], 3, ['remote']) -@@ -2357,6 +2375,7 @@ m4_define([OVSDB_CHECK_CLUSTER_IDL_C], +@@ -2357,6 +2432,7 @@ m4_define([OVSDB_CHECK_CLUSTER_IDL_C], AT_CHECK([sort stdout | uuidfilt]m4_if([$7],,, [[| $7]]), [0], [$5]) m4_ifval([$8], [AT_CHECK([grep '$8' stderr], [1])], [], []) @@ -8481,7 +11822,7 @@ index 5a7e76eaa..1dd334325 100644 AT_CLEANUP]) # Same as OVSDB_CHECK_CLUSTER_IDL_C but uses the Python IDL implementation. -@@ -2377,6 +2396,7 @@ m4_define([OVSDB_CHECK_CLUSTER_IDL_PY], +@@ -2377,6 +2453,7 @@ m4_define([OVSDB_CHECK_CLUSTER_IDL_PY], AT_CHECK([sort stdout | uuidfilt]m4_if([$7],,, [[| $7]]), [0], [$5]) m4_if([$8], [AT_CHECK([grep '$8' stderr], [1])], [], []) @@ -8489,8 +11830,129 @@ index 5a7e76eaa..1dd334325 100644 AT_CLEANUP]) m4_define([OVSDB_CHECK_CLUSTER_IDL], +@@ -2468,7 +2545,7 @@ reconnect.*waiting .* seconds before reconnect) + + AT_SETUP([idl table and column presence check]) + AT_KEYWORDS([ovsdb server idl table column check]) +-AT_CHECK([ovsdb_start_idltest "" "$abs_srcdir/idltest2.ovsschema"]) ++OVSDB_START_IDLTEST([], ["$abs_srcdir/idltest2.ovsschema"]) + + AT_CHECK(ovsdb-tool create db2 $abs_srcdir/idltest.ovsschema) + AT_CHECK(ovsdb-server -vconsole:warn --log-file=ovsdb-server2.log --detach dnl +@@ -2596,7 +2673,7 @@ OVSDB_CHECK_IDL_TRACK([track, insert and delete, refs to link2], + m4_define([OVSDB_CHECK_IDL_PERS_UUID_INSERT_C], + [AT_SETUP([$1 - C]) + AT_KEYWORDS([idl persistent uuid insert]) +- AT_CHECK([ovsdb_start_idltest "" "$abs_srcdir/idltest.ovsschema"]) ++ OVSDB_START_IDLTEST([], ["$abs_srcdir/idltest.ovsschema"]) + AT_CHECK([test-ovsdb '-vPATTERN:console:test-ovsdb|%c|%m' -vjsonrpc -t10 idl unix:socket $2], + [0], [stdout], [stderr]) + AT_CHECK([sort stdout], +@@ -2608,7 +2685,7 @@ m4_define([OVSDB_CHECK_IDL_PERS_UUID_INSERT_C], + m4_define([OVSDB_CHECK_IDL_PERS_UUID_INSERT_PY], + [AT_SETUP([$1 - Python3]) + AT_KEYWORDS([idl persistent uuid insert]) +- AT_CHECK([ovsdb_start_idltest "" "$abs_srcdir/idltest.ovsschema"]) ++ OVSDB_START_IDLTEST([], ["$abs_srcdir/idltest.ovsschema"]) + AT_CHECK([$PYTHON3 $srcdir/test-ovsdb.py -t10 idl $srcdir/idltest.ovsschema unix:socket $2], + [0], [stdout], [stderr]) + AT_CHECK([sort stdout], +@@ -2650,3 +2727,93 @@ OVSDB_CHECK_IDL_PERS_UUID_INSERT([simple idl, persistent uuid insert], + 011: done + ]], + [['This UUID would duplicate a UUID already present within the table or deleted within the same transaction']]) ++ ++ ++m4_define([OVSDB_CHECK_IDL_CHANGE_AWARE], ++ [AT_SETUP([simple idl, database change aware, online conversion - $1]) ++ AT_KEYWORDS([ovsdb server idl db_change_aware conversion $1]) ++ ++ m4_if([$1], [clustered], ++ [OVSDB_CLUSTER_START_IDLTEST([1], [punix:socket])], ++ [OVSDB_START_IDLTEST]) ++ ++ dnl Add some data. ++ AT_CHECK([[ovsdb-client transact unix:socket '["idltest", ++ {"op": "insert", ++ "table": "simple", ++ "row": {"i": 1, ++ "r": 2.0, ++ "b": true, ++ "s": "first row", ++ "u": ["uuid", "84f5c8f5-ac76-4dbc-a24f-8860eb407fc1"], ++ "ia": ["set", [1, 2, 3]], ++ "ra": ["set", [-0.5]], ++ "ba": ["set", [true]], ++ "sa": ["set", ["abc", "def"]], ++ "ua": ["set", [["uuid", "69443985-7806-45e2-b35f-574a04e720f9"], ++ ["uuid", "aad11ef0-816a-4b01-93e6-03b8b4256b98"]]]}}, ++ {"op": "insert", ++ "table": "simple", ++ "row": {"b": false, "s": "second row"}}, ++ {"op": "insert", ++ "table": "simple", ++ "row": {"b": true, "s": "third row"}} ++ ]']], [0], [stdout]) ++ ++ dnl Create a new schema by adding 'extra_column' to the 'simple' table. ++ AT_CHECK([sed 's/"ua": {/"extra_column":{"type": "string"},"ua": {/ ++ s/1.2.3/1.2.4/' \ ++ $abs_srcdir/idltest.ovsschema > new-idltest.ovsschema]) ++ dnl Try "needs-conversion". ++ AT_CHECK([ovsdb-client needs-conversion unix:socket $abs_srcdir/idltest.ovsschema], [0], [no ++]) ++ AT_CHECK([ovsdb-client needs-conversion unix:socket new-idltest.ovsschema], [0], [yes ++]) ++ ++ dnl Conditionally exclude the second row from monitoring. ++ m4_define([COND], [['condition simple [["b","==",true]]']]) ++ ++ dnl Start monitoring. ++ OVS_DAEMONIZE([test-ovsdb '-vPATTERN:console:test-ovsdb|%c|%m' -vjsonrpc -t30 \ ++ idl unix:socket COND monitor \ ++ >idl-c.out 2>idl-c.err], [idl-c.pid]) ++ AT_CAPTURE_FILE([idl-c.out]) ++ AT_CAPTURE_FILE([idl-c.err]) ++ ++ OVS_DAEMONIZE([$PYTHON3 $srcdir/test-ovsdb.py -t30 \ ++ idl $srcdir/idltest.ovsschema unix:socket COND monitor \ ++ >idl-python.out 2>idl-python.err], [idl-python.pid]) ++ AT_CAPTURE_FILE([idl-python.out]) ++ AT_CAPTURE_FILE([idl-python.err]) ++ ++ dnl Wait for monitors to receive the data. ++ OVS_WAIT_UNTIL([grep -q 'third row' idl-c.err]) ++ OVS_WAIT_UNTIL([grep -q 'third row' idl-python.err]) ++ ++ dnl Convert the database. ++ AT_CHECK([ovsdb-client convert unix:socket new-idltest.ovsschema]) ++ ++ dnl Check for the monitor cancellation and the data being requested again. ++ m4_foreach([FILE], [[idl-c], [idl-python]], ++ [OVS_WAIT_UNTIL([grep -q 'monitor_canceled' FILE.err]) ++ OVS_WAIT_UNTIL([test 2 -eq $(grep -c 'send request, method="monitor_cond_since", params=."idltest"' FILE.err)]) ++ ++ dnl XXX: Checking for the new schema bits conditionally because standalone ++ dnl databases are not updating the schema in the _Server database properly. ++ m4_if([$1], [clustered], [OVS_WAIT_UNTIL([grep -q 'extra_column' FILE.err])]) ++ ++ dnl Check that there were no unexpected messages. ++ AT_CHECK([! grep 'unexpected' FILE.err]) ++ ++ dnl Check that the data is received twice and the condition is working. ++ AT_CHECK([sort FILE.out | uuidfilt], [0], ++[[000: simple: change conditions ++001: table simple: i=0 r=0 b=true s=third row u=<0> ia=[] ra=[] ba=[] sa=[] ua=[] uuid=<1> ++001: table simple: i=1 r=2 b=true s=first row u=<2> ia=[1 2 3] ra=[-0.5] ba=[true] sa=[abc def] ua=[<3> <4>] uuid=<5> ++002: table simple: i=0 r=0 b=true s=third row u=<0> ia=[] ra=[] ba=[] sa=[] ua=[] uuid=<1> ++002: table simple: i=1 r=2 b=true s=first row u=<2> ia=[1 2 3] ra=[-0.5] ba=[true] sa=[abc def] ua=[<3> <4>] uuid=<5> ++]])]) ++ AT_CLEANUP]) ++ ++OVSDB_CHECK_IDL_CHANGE_AWARE([standalone]) ++OVSDB_CHECK_IDL_CHANGE_AWARE([clustered]) diff --git a/tests/ovsdb-monitor.at b/tests/ovsdb-monitor.at -index 3b622b3ec..82b0e9362 100644 +index 3b622b3ec0..82b0e93623 100644 --- a/tests/ovsdb-monitor.at +++ b/tests/ovsdb-monitor.at @@ -1011,3 +1011,69 @@ row,action,name,number,_version @@ -8564,7 +12026,7 @@ index 3b622b3ec..82b0e9362 100644 +]) +AT_CLEANUP diff --git a/tests/ovsdb-server.at b/tests/ovsdb-server.at -index bf539b6e5..b9c3bf203 100644 +index bf539b6e5b..835007d8d7 100644 --- a/tests/ovsdb-server.at +++ b/tests/ovsdb-server.at @@ -24,6 +24,9 @@ m4_define([OVSDB_SERVER_SHUTDOWN2], @@ -8601,8 +12063,64 @@ index bf539b6e5..b9c3bf203 100644 OVSDB_SERVER_SHUTDOWN AT_CLEANUP]) +@@ -670,8 +683,10 @@ AT_CHECK_UNQUOTED( + [ignore]) + # The error message for being unable to negotiate a shared ciphersuite + # is 'sslv3 alert handshake failure'. This is not the clearest message. ++# In openssl 3.2.0 all the error messages were updated to replace 'sslv3' ++# with 'ssl/tls'. + AT_CHECK_UNQUOTED( +- [grep "sslv3 alert handshake failure" output], [0], ++ [grep -E "(sslv3|ssl/tls) alert handshake failure" output], [0], + [stdout], + [ignore]) + OVSDB_SERVER_SHUTDOWN +@@ -1798,9 +1813,14 @@ replication_schema > schema + AT_CHECK([ovsdb-tool create db1 schema], [0], [stdout], [ignore]) + AT_CHECK([ovsdb-tool create db2 schema], [0], [stdout], [ignore]) + +-AT_CHECK([ovsdb-server --detach --no-chdir --log-file=ovsdb-server1.log --pidfile --remote=punix:db.sock db1], [0], [ignore], [ignore]) ++AT_CHECK([ovsdb-server -vfile --detach --no-chdir \ ++ --log-file=ovsdb-server1.log --pidfile --remote=punix:db.sock db1], ++ [0], [ignore], [ignore]) + +-AT_CHECK([ovsdb-server --detach --no-chdir --log-file=ovsdb-server2.log --pidfile=2.pid --remote=punix:db2.sock --unixctl=unixctl2 db2], [0], [ignore], [ignore]) ++AT_CHECK([ovsdb-server -vfile --detach --no-chdir \ ++ --log-file=ovsdb-server2.log --pidfile=2.pid \ ++ --remote=punix:db2.sock --unixctl=unixctl2 db2], ++ [0], [ignore], [ignore]) + + dnl Try to connect without specifying the active server. + AT_CHECK([ovs-appctl -t "`pwd`"/unixctl2 ovsdb-server/connect-active-ovsdb-server], [0], +@@ -2119,9 +2139,16 @@ AT_CHECK([ovsdb-tool transact db2 \ + + dnl Start both 'db1' and 'db2'. + on_exit 'kill `cat *.pid`' +-AT_CHECK([ovsdb-server --detach --no-chdir --log-file=ovsdb-server1.log --pidfile --remote=punix:db.sock --unixctl="`pwd`"/unixctl db1 --active ], [0], [ignore], [ignore]) ++AT_CHECK([ovsdb-server -vfile --detach --no-chdir \ ++ --log-file=ovsdb-server1.log --pidfile \ ++ --remote=punix:db.sock \ ++ --unixctl="$(pwd)"/unixctl db1 --active ], ++ [0], [ignore], [ignore]) + +-AT_CHECK([ovsdb-server --detach --no-chdir --log-file=ovsdb-server2.log --pidfile=2.pid --remote=punix:db2.sock --unixctl="`pwd`"/unixctl2 db2], [0], [ignore], [ignore]) ++AT_CHECK([ovsdb-server -vfile --detach --no-chdir \ ++ --log-file=ovsdb-server2.log --pidfile=2.pid \ ++ --remote=punix:db2.sock --unixctl="$(pwd)"/unixctl2 db2], ++ [0], [ignore], [ignore]) + + OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/unixctl ovsdb-server/sync-status |grep active]) + OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/unixctl2 ovsdb-server/sync-status |grep active]) +@@ -2348,6 +2375,6 @@ CLEAN_LOG_FILE([2.log], [2.log.clear]) + + dnl Checking that databases and logs are equal. + AT_CHECK([diff db.clear ./replay_dir/db.copy.clear]) +-AT_CHECK([diff 1.log.clear 2.log.clear]) ++AT_CHECK([diff -u 1.log.clear 2.log.clear]) + + AT_CLEANUP diff --git a/tests/ovsdb-tool.at b/tests/ovsdb-tool.at -index 12ad6fb3f..5496ccda7 100644 +index 12ad6fb3fc..5496ccda77 100644 --- a/tests/ovsdb-tool.at +++ b/tests/ovsdb-tool.at @@ -465,6 +465,7 @@ AT_SETUP([ovsdb-tool convert-to-standalone]) @@ -8686,10 +12204,82 @@ index 12ad6fb3f..5496ccda7 100644 +AT_CHECK([diff standalonedump clusterdump]) +AT_CLEANUP diff --git a/tests/packet-type-aware.at b/tests/packet-type-aware.at -index 3b5c66fe5..d63528e69 100644 +index 3b5c66fe52..63054478e7 100644 --- a/tests/packet-type-aware.at +++ b/tests/packet-type-aware.at -@@ -1021,7 +1021,7 @@ AT_CHECK([ +@@ -142,30 +142,27 @@ AT_CHECK([ + ### Setup GRE tunnels + AT_CHECK([ + ovs-appctl netdev-dummy/ip4addr br-p1 10.0.0.1/24 && +- ovs-appctl ovs/route/add 10.0.0.0/24 br-p1 && + ovs-appctl tnl/arp/set br-p1 10.0.0.1 $HWADDR_BRP1 && + ovs-appctl tnl/arp/set br-p1 10.0.0.2 $HWADDR_BRP2 && + ovs-appctl tnl/arp/set br-p1 10.0.0.3 $HWADDR_BRP3 && + + ovs-appctl netdev-dummy/ip4addr br-p2 20.0.0.2/24 && +- ovs-appctl ovs/route/add 20.0.0.0/24 br-p2 && + ovs-appctl tnl/arp/set br-p2 20.0.0.1 $HWADDR_BRP1 && + ovs-appctl tnl/arp/set br-p2 20.0.0.2 $HWADDR_BRP2 && + ovs-appctl tnl/arp/set br-p2 20.0.0.3 $HWADDR_BRP3 && + + ovs-appctl netdev-dummy/ip4addr br-p3 30.0.0.3/24 && +- ovs-appctl ovs/route/add 30.0.0.0/24 br-p3 && + ovs-appctl tnl/arp/set br-p3 30.0.0.1 $HWADDR_BRP1 && + ovs-appctl tnl/arp/set br-p3 30.0.0.2 $HWADDR_BRP2 && + ovs-appctl tnl/arp/set br-p3 30.0.0.3 $HWADDR_BRP3 + ], [0], [ignore]) + + AT_CHECK([ +- ovs-appctl ovs/route/show | grep User: ++ ovs-appctl ovs/route/show | grep Cached: | sort + ], [0], [dnl +-User: 10.0.0.0/24 dev br-p1 SRC 10.0.0.1 +-User: 20.0.0.0/24 dev br-p2 SRC 20.0.0.2 +-User: 30.0.0.0/24 dev br-p3 SRC 30.0.0.3 ++Cached: 10.0.0.0/24 dev br-p1 SRC 10.0.0.1 local ++Cached: 20.0.0.0/24 dev br-p2 SRC 20.0.0.2 local ++Cached: 30.0.0.0/24 dev br-p3 SRC 30.0.0.3 local + ]) + + AT_CHECK([ +@@ -681,14 +678,13 @@ AT_CHECK([ + + AT_CHECK([ + ovs-appctl netdev-dummy/ip4addr br2 10.0.0.1/24 && +- ovs-appctl ovs/route/add 10.0.0.0/24 br2 && + ovs-appctl tnl/arp/set br2 10.0.0.2 de:af:be:ef:ba:be + ], [0], [ignore]) + + AT_CHECK([ +- ovs-appctl ovs/route/show | grep User: ++ ovs-appctl ovs/route/show | grep Cached: + ], [0], [dnl +-User: 10.0.0.0/24 dev br2 SRC 10.0.0.1 ++Cached: 10.0.0.0/24 dev br2 SRC 10.0.0.1 local + ]) + + +@@ -955,7 +951,6 @@ AT_CHECK([ + + AT_CHECK([ + ovs-appctl netdev-dummy/ip4addr br0 20.0.0.1/24 && +- ovs-appctl ovs/route/add 20.0.0.2/24 br0 && + ovs-appctl tnl/neigh/set br0 20.0.0.1 aa:bb:cc:00:00:01 && + ovs-appctl tnl/neigh/set br0 20.0.0.2 aa:bb:cc:00:00:02 + ], [0], [ignore]) +@@ -963,9 +958,9 @@ AT_CHECK([ + ovs-appctl time/warp 1000 + + AT_CHECK([ +- ovs-appctl ovs/route/show | grep User ++ ovs-appctl ovs/route/show | grep Cached: + ],[0], [dnl +-User: 20.0.0.0/24 dev br0 SRC 20.0.0.1 ++Cached: 20.0.0.0/24 dev br0 SRC 20.0.0.1 local + ]) + + AT_CHECK([ +@@ -1021,7 +1016,7 @@ AT_CHECK([ ], [0], [flow-dump from the main thread: recirc_id(0),in_port(p0),packet_type(ns=0,id=0),eth(src=aa:bb:cc:00:00:02,dst=aa:bb:cc:00:00:01),eth_type(0x0800),ipv4(dst=20.0.0.1,proto=47,frag=no), packets:3, bytes:378, used:0.0s, actions:tnl_pop(gre_sys) tunnel(src=20.0.0.2,dst=20.0.0.1,flags(-df-csum)),recirc_id(0),in_port(gre_sys),packet_type(ns=1,id=0x8847),eth_type(0x8847),mpls(label=999/0x0,tc=0/0,ttl=64/0x0,bos=1/1), packets:3, bytes:264, used:0.0s, actions:push_eth(src=00:00:00:00:00:00,dst=00:00:00:00:00:00),pop_mpls(eth_type=0x800),recirc(0x1) @@ -8699,7 +12289,7 @@ index 3b5c66fe5..d63528e69 100644 ovs-appctl time/warp 1000 diff --git a/tests/pmd.at b/tests/pmd.at -index c707f762c..5546e063b 100644 +index c707f762c7..5546e063b7 100644 --- a/tests/pmd.at +++ b/tests/pmd.at @@ -1300,3 +1300,54 @@ OVS_WAIT_UNTIL([tail -n +$LINENUM ovs-vswitchd.log | grep "PMD load based sleeps @@ -8757,8 +12347,143 @@ index c707f762c..5546e063b 100644 + +OVS_VSWITCHD_STOP +AT_CLEANUP +diff --git a/tests/rstp.at b/tests/rstp.at +index 600e85dabd..e0d4bed4f0 100644 +--- a/tests/rstp.at ++++ b/tests/rstp.at +@@ -253,3 +253,60 @@ AT_CHECK([ovs-vsctl del-port br0 p1]) + + OVS_VSWITCHD_STOP + AT_CLEANUP ++ ++AT_SETUP([RSTP - patch ports]) ++# Create br0 with interfaces p1 and p7 ++# and br1 with interfaces p2 and p8 ++# with p1 and p2 being connected patch ports. ++OVS_VSWITCHD_START( ++ [set port br0 other_config:rstp-enable=false -- \ ++ set bridge br0 rstp-enable=true ++]) ++ ++AT_CHECK([add_of_br 1 \ ++ set port br1 other_config:rstp-enable=false -- \ ++ set bridge br1 rstp-enable=true]) ++ ++ovs-appctl time/stop ++ ++AT_CHECK([ovs-vsctl \ ++ add-port br0 p1 -- \ ++ set interface p1 type=patch options:peer=p2 ofport_request=1 -- \ ++ set port p1 other_config:rstp-enable=true -- \ ++ add-port br1 p2 -- \ ++ set interface p2 type=patch options:peer=p1 ofport_request=2 -- \ ++ set port p2 other_config:rstp-enable=true -- \ ++]) ++ ++AT_CHECK([ovs-vsctl \ ++ add-port br0 p7 -- \ ++ set interface p7 ofport_request=7 type=dummy -- \ ++ set port p7 other_config:rstp-enable=false -- \ ++ add-port br1 p8 -- \ ++ set interface p8 ofport_request=8 type=dummy -- \ ++ set port p8 other_config:rstp-enable=false -- \ ++]) ++ ++AT_CHECK([ovs-ofctl add-flow br0 "in_port=7 icmp actions=1"]) ++AT_CHECK([ovs-ofctl add-flow br0 "in_port=1 icmp actions=7"]) ++AT_CHECK([ovs-ofctl add-flow br1 "in_port=8 icmp actions=2"]) ++AT_CHECK([ovs-ofctl add-flow br1 "in_port=2 icmp actions=8"]) ++ ++# Give time for RSTP to synchronize. ++ovs-appctl time/warp 5000 500 ++ ++OVS_WAIT_UNTIL_EQUAL([cat ovs-vswitchd.log | FILTER_STP_TOPOLOGY], [dnl ++port p1: RSTP state changed from Disabled to Discarding ++port p2: RSTP state changed from Disabled to Discarding ++port p2: RSTP state changed from Discarding to Forwarding ++port p1: RSTP state changed from Discarding to Forwarding]) ++ ++AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(7),eth(src=50:54:00:00:00:09,dst=50:54:00:00:00:0a),eth_type(0x0800),ipv4(src=10.0.0.2,dst=10.0.0.1,proto=1,tos=0,ttl=64,frag=no),icmp(type=8,code=0)' | grep Datapath], [0], [dnl ++Datapath actions: 8 ++]) ++AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(8),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.3,dst=10.0.0.4,proto=1,tos=0,ttl=64,frag=no),icmp(type=8,code=0)' | grep Datapath], [0], [dnl ++Datapath actions: 7 ++]) ++ ++OVS_VSWITCHD_STOP ++AT_CLEANUP +diff --git a/tests/stp.at b/tests/stp.at +index a6b6465d12..6239ec379f 100644 +--- a/tests/stp.at ++++ b/tests/stp.at +@@ -464,6 +464,65 @@ Datapath actions: 2 + + AT_CLEANUP + ++AT_SETUP([STP - patch ports]) ++# Create br0 with interfaces p1 and p7 ++# and br1 with interfaces p2 and p8 ++# with p1 and p2 being connected patch ports. ++OVS_VSWITCHD_START( ++ [set port br0 other_config:stp-enable=false -- \ ++ set bridge br0 stp-enable=true ++]) ++ ++AT_CHECK([add_of_br 1 \ ++ set port br1 other_config:stp-enable=false -- \ ++ set bridge br1 stp-enable=true]) ++ ++ovs-appctl time/stop ++ ++AT_CHECK([ovs-vsctl \ ++ add-port br0 p1 -- \ ++ set interface p1 type=patch options:peer=p2 ofport_request=1 -- \ ++ set port p1 other_config:stp-enable=true -- \ ++ add-port br1 p2 -- \ ++ set interface p2 type=patch options:peer=p1 ofport_request=2 -- \ ++ set port p2 other_config:stp-enable=true -- \ ++]) ++ ++AT_CHECK([ovs-vsctl \ ++ add-port br0 p7 -- \ ++ set interface p7 ofport_request=7 type=dummy -- \ ++ set port p7 other_config:stp-enable=false -- \ ++ add-port br1 p8 -- \ ++ set interface p8 ofport_request=8 type=dummy -- \ ++ set port p8 other_config:stp-enable=false -- \ ++]) ++ ++AT_CHECK([ovs-ofctl add-flow br0 "in_port=7 icmp actions=1"]) ++AT_CHECK([ovs-ofctl add-flow br0 "in_port=1 icmp actions=7"]) ++AT_CHECK([ovs-ofctl add-flow br1 "in_port=8 icmp actions=2"]) ++AT_CHECK([ovs-ofctl add-flow br1 "in_port=2 icmp actions=8"]) ++ ++# Give time for STP to synchronize. ++ovs-appctl time/warp 30000 3000 ++ ++OVS_WAIT_UNTIL_EQUAL([cat ovs-vswitchd.log | FILTER_STP_TOPOLOGY], [dnl ++port <>: STP state changed from disabled to listening ++port <>: STP state changed from disabled to listening ++port <>: STP state changed from listening to learning ++port <>: STP state changed from listening to learning ++port <>: STP state changed from learning to forwarding ++port <>: STP state changed from learning to forwarding]) ++ ++AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(7),eth(src=50:54:00:00:00:09,dst=50:54:00:00:00:0a),eth_type(0x0800),ipv4(src=10.0.0.2,dst=10.0.0.1,proto=1,tos=0,ttl=64,frag=no),icmp(type=8,code=0)' | grep Datapath], [0], [dnl ++Datapath actions: 8 ++]) ++AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(8),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.3,dst=10.0.0.4,proto=1,tos=0,ttl=64,frag=no),icmp(type=8,code=0)' | grep Datapath], [0], [dnl ++Datapath actions: 7 ++]) ++ ++OVS_VSWITCHD_STOP ++AT_CLEANUP ++ + AT_SETUP([STP - flush the fdb and mdb when topology changed]) + OVS_VSWITCHD_START([]) + diff --git a/tests/system-afxdp.at b/tests/system-afxdp.at -index 0d09906fb..88f660566 100644 +index 0d09906fb6..88f6605663 100644 --- a/tests/system-afxdp.at +++ b/tests/system-afxdp.at @@ -39,7 +39,7 @@ AT_CHECK([ovs-vsctl add-port br0 ovs-p0 -- \ @@ -8771,7 +12496,7 @@ index 0d09906fb..88f660566 100644 ]) diff --git a/tests/system-dpdk-macros.at b/tests/system-dpdk-macros.at -index 53fbc1320..3920f08a5 100644 +index 53fbc13206..3920f08a5e 100644 --- a/tests/system-dpdk-macros.at +++ b/tests/system-dpdk-macros.at @@ -42,7 +42,7 @@ m4_define([OVS_DPDK_START], @@ -8793,7 +12518,7 @@ index 53fbc1320..3920f08a5 100644 dnl Start ovs-vswitchd. AT_CHECK([ovs-vswitchd --detach --no-chdir --pidfile --log-file -vvconn -vofproto_dpif -vunixctl], [0], [stdout], [stderr]) diff --git a/tests/system-dpdk.at b/tests/system-dpdk.at -index cb6c6d590..0f58e8574 100644 +index cb6c6d5907..5ed1c7e676 100644 --- a/tests/system-dpdk.at +++ b/tests/system-dpdk.at @@ -32,7 +32,7 @@ dnl Check if EAL init is successful @@ -8886,7 +12611,63 @@ index cb6c6d590..0f58e8574 100644 dnl Add userspace bridge and attach it to OVS and add egress policer AT_CHECK([ovs-vsctl add-br br10 -- set bridge br10 datapath_type=netdev]) -@@ -661,7 +661,7 @@ AT_KEYWORDS([dpdk]) +@@ -585,14 +585,13 @@ dnl Add userspace bridge and attach it to OVS with default MTU value + AT_CHECK([ovs-vsctl add-br br10 -- set bridge br10 datapath_type=netdev]) + AT_CHECK([ovs-vsctl add-port br10 phy0 -- set Interface phy0 type=dpdk options:dpdk-devargs=$(cat PCI_ADDR)], [], [stdout], [stderr]) + AT_CHECK([ovs-vsctl show], [], [stdout]) +-sleep 2 + + dnl Check default MTU value in the datapath +-AT_CHECK([ovs-appctl dpctl/show], [], [stdout]) +-AT_CHECK([grep -E 'mtu=1500' stdout], [], [stdout]) ++OVS_WAIT_UNTIL_EQUAL([ovs-vsctl get Interface phy0 mtu], [1500]) + + dnl Increase MTU value and check in the datapath + AT_CHECK([ovs-vsctl set Interface phy0 mtu_request=9000]) ++OVS_WAIT_UNTIL_EQUAL([ovs-vsctl get Interface phy0 mtu], [9000]) + + dnl Fail if MTU is not supported + AT_FAIL_IF([grep "Interface phy0 does not support MTU configuration" ovs-vswitchd.log], [], [stdout]) +@@ -600,10 +599,6 @@ AT_FAIL_IF([grep "Interface phy0 does not support MTU configuration" ovs-vswitch + dnl Fail if error is encountered during MTU setup + AT_FAIL_IF([grep "Interface phy0 MTU (9000) setup error" ovs-vswitchd.log], [], [stdout]) + +-AT_CHECK([ovs-appctl dpctl/show], [], [stdout]) +-AT_CHECK([grep -E 'mtu=9000' stdout], [], [stdout]) +- +- + dnl Clean up + AT_CHECK([ovs-vsctl del-port br10 phy0], [], [stdout], [stderr]) + OVS_VSWITCHD_STOP("[SYSTEM_DPDK_ALLOWED_LOGS]") +@@ -627,7 +622,9 @@ AT_CHECK([ovs-vsctl add-br br10 -- set bridge br10 datapath_type=netdev]) + AT_CHECK([ovs-vsctl add-port br10 phy0 -- set Interface phy0 type=dpdk options:dpdk-devargs=$(cat PCI_ADDR)], [], [stdout], [stderr]) + AT_CHECK([ovs-vsctl set Interface phy0 mtu_request=9000]) + AT_CHECK([ovs-vsctl show], [], [stdout]) +-sleep 2 ++ ++dnl Check MTU value in the datapath ++OVS_WAIT_UNTIL_EQUAL([ovs-vsctl get Interface phy0 mtu], [9000]) + + dnl Fail if MTU is not supported + AT_FAIL_IF([grep "Interface phy0 does not support MTU configuration" ovs-vswitchd.log], [], [stdout]) +@@ -635,15 +632,9 @@ AT_FAIL_IF([grep "Interface phy0 does not support MTU configuration" ovs-vswitch + dnl Fail if error is encountered during MTU setup + AT_FAIL_IF([grep "Interface phy0 MTU (9000) setup error" ovs-vswitchd.log], [], [stdout]) + +-dnl Check MTU value in the datapath +-AT_CHECK([ovs-appctl dpctl/show], [], [stdout]) +-AT_CHECK([grep -E 'mtu=9000' stdout], [], [stdout]) +- + dnl Decrease MTU value and check in the datapath + AT_CHECK([ovs-vsctl set Interface phy0 mtu_request=2000]) +- +-AT_CHECK([ovs-appctl dpctl/show], [], [stdout]) +-AT_CHECK([grep -E 'mtu=2000' stdout], [], [stdout]) ++OVS_WAIT_UNTIL_EQUAL([ovs-vsctl get Interface phy0 mtu], [2000]) + + + dnl Clean up +@@ -661,7 +652,7 @@ AT_KEYWORDS([dpdk]) AT_SKIP_IF([! which dpdk-testpmd >/dev/null 2>/dev/null]) OVS_DPDK_PRE_CHECK() @@ -8895,7 +12676,7 @@ index cb6c6d590..0f58e8574 100644 dnl Find number of sockets AT_CHECK([lscpu], [], [stdout]) -@@ -717,7 +717,7 @@ AT_KEYWORDS([dpdk]) +@@ -717,7 +708,7 @@ AT_KEYWORDS([dpdk]) AT_SKIP_IF([! which dpdk-testpmd >/dev/null 2>/dev/null]) OVS_DPDK_PRE_CHECK() @@ -8904,7 +12685,7 @@ index cb6c6d590..0f58e8574 100644 dnl Find number of sockets AT_CHECK([lscpu], [], [stdout]) -@@ -856,7 +856,7 @@ AT_KEYWORDS([dpdk]) +@@ -856,7 +847,7 @@ AT_KEYWORDS([dpdk]) AT_SKIP_IF([! which dpdk-testpmd >/dev/null 2>/dev/null]) OVS_DPDK_PRE_CHECK() @@ -8913,7 +12694,7 @@ index cb6c6d590..0f58e8574 100644 dnl Find number of sockets AT_CHECK([lscpu], [], [stdout]) -@@ -908,7 +908,7 @@ AT_KEYWORDS([dpdk]) +@@ -908,7 +899,7 @@ AT_KEYWORDS([dpdk]) AT_SKIP_IF([! which dpdk-testpmd >/dev/null 2>/dev/null]) OVS_DPDK_PRE_CHECK() @@ -8922,7 +12703,7 @@ index cb6c6d590..0f58e8574 100644 dnl Find number of sockets AT_CHECK([lscpu], [], [stdout]) -@@ -963,7 +963,7 @@ dnl MFEX Autovalidator +@@ -963,7 +954,7 @@ dnl MFEX Autovalidator AT_SETUP([OVS-DPDK - MFEX Autovalidator]) AT_KEYWORDS([dpdk]) OVS_DPDK_PRE_CHECK() @@ -8931,7 +12712,7 @@ index cb6c6d590..0f58e8574 100644 AT_CHECK([ovs-vsctl add-br br0 -- set bridge br0 datapath_type=netdev]) AT_SKIP_IF([! ovs-appctl dpif-netdev/miniflow-parser-get | sed 1,4d | grep "True"], [], [dnl ]) -@@ -996,7 +996,7 @@ dnl MFEX Autovalidator Fuzzy +@@ -996,7 +987,7 @@ dnl MFEX Autovalidator Fuzzy AT_SETUP([OVS-DPDK - MFEX Autovalidator Fuzzy]) AT_KEYWORDS([dpdk]) OVS_DPDK_PRE_CHECK() @@ -8940,7 +12721,7 @@ index cb6c6d590..0f58e8574 100644 AT_CHECK([ovs-vsctl add-br br0 -- set bridge br0 datapath_type=netdev]) AT_SKIP_IF([! ovs-appctl dpif-netdev/miniflow-parser-get | sed 1,4d | grep "True"], [], [dnl ]) -@@ -1032,7 +1032,7 @@ AT_KEYWORDS([dpdk]) +@@ -1032,7 +1023,7 @@ AT_KEYWORDS([dpdk]) OVS_DPDK_PRE_CHECK() AT_SKIP_IF([! $PYTHON3 -c "import scapy"], [], []) AT_CHECK([$PYTHON3 $srcdir/mfex_fuzzy.py test_traffic.pcap 1], [], [stdout]) @@ -8949,7 +12730,7 @@ index cb6c6d590..0f58e8574 100644 AT_CHECK([ovs-vsctl --no-wait set Open_vSwitch . other_config:pmd-cpu-mask=0x1]) dnl Add userspace bridge and attach it to OVS AT_CHECK([ovs-vsctl add-br br0 -- set bridge br0 datapath_type=netdev]) -@@ -1153,7 +1153,7 @@ AT_SETUP([OVS-DPDK - user configured mempool]) +@@ -1153,7 +1144,7 @@ AT_SETUP([OVS-DPDK - user configured mempool]) AT_KEYWORDS([dpdk]) OVS_DPDK_PRE_CHECK() OVS_DPDK_START_OVSDB() @@ -8959,7 +12740,7 @@ index cb6c6d590..0f58e8574 100644 AT_CHECK([ovs-vsctl --no-wait set Open_vSwitch . other_config:shared-mempool-config=8000,6000,1500]) AT_CHECK([ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true]) diff --git a/tests/system-interface.at b/tests/system-interface.at -index 784bada12..15e789a24 100644 +index 784bada12c..15e789a245 100644 --- a/tests/system-interface.at +++ b/tests/system-interface.at @@ -63,3 +63,119 @@ AT_CHECK([ @@ -9083,7 +12864,7 @@ index 784bada12..15e789a24 100644 +"]) +AT_CLEANUP diff --git a/tests/system-ipsec.at b/tests/system-ipsec.at -index 07f2b8fd0..d3d27133b 100644 +index 07f2b8fd0e..d3d27133b9 100644 --- a/tests/system-ipsec.at +++ b/tests/system-ipsec.at @@ -141,10 +141,10 @@ m4_define([CHECK_ESP_TRAFFIC], @@ -9100,7 +12881,7 @@ index 07f2b8fd0..d3d27133b 100644 ]) diff --git a/tests/system-kmod-macros.at b/tests/system-kmod-macros.at -index 11920e60b..cbec8de02 100644 +index 11920e60b6..cbec8de024 100644 --- a/tests/system-kmod-macros.at +++ b/tests/system-kmod-macros.at @@ -112,6 +112,17 @@ m4_define([CHECK_CONNTRACK_ZEROIP_SNAT], @@ -9136,7 +12917,7 @@ index 11920e60b..cbec8de02 100644 + AT_CHECK([ip link del dev ovs_bareudp0]) +]) diff --git a/tests/system-layer3-tunnels.at b/tests/system-layer3-tunnels.at -index c37852b21..6fbdedb64 100644 +index c37852b216..5dcdd2afae 100644 --- a/tests/system-layer3-tunnels.at +++ b/tests/system-layer3-tunnels.at @@ -34,15 +34,15 @@ AT_CHECK([ovs-ofctl add-flow br0 "priority=100 ip,nw_dst=10.1.1.2 action=mod_dl_ @@ -9158,7 +12939,7 @@ index c37852b21..6fbdedb64 100644 3 packets transmitted, 3 received, 0% packet loss, time 0ms ]) -@@ -83,15 +83,15 @@ AT_CHECK([ovs-ofctl add-flow br0 "priority=100 ip,nw_dst=10.1.1.2 action=mod_dl_ +@@ -83,78 +83,23 @@ AT_CHECK([ovs-ofctl add-flow br0 "priority=100 ip,nw_dst=10.1.1.2 action=mod_dl_ OVS_WAIT_UNTIL([ip netns exec at_ns0 ping -c 1 10.1.1.2]) dnl First, check the underlay @@ -9177,21 +12958,76 @@ index c37852b21..6fbdedb64 100644 3 packets transmitted, 3 received, 0% packet loss, time 0ms ]) -@@ -154,7 +154,7 @@ OVS_VSWITCHD_STOP + OVS_TRAFFIC_VSWITCHD_STOP AT_CLEANUP - AT_SETUP([layer3 - ping over MPLS Bareudp]) --OVS_CHECK_MIN_KERNEL(5, 7) -+OVS_CHECK_BAREUDP() - OVS_TRAFFIC_VSWITCHD_START([_ADD_BR([br1])]) - ADD_NAMESPACES(at_ns0, at_ns1) - -@@ -191,18 +191,18 @@ AT_CHECK([ovs-vsctl add-port br1 patch1]) - AT_CHECK([ovs-ofctl -O OpenFlow13 add-flows br0 flows0.txt]) - AT_CHECK([ovs-ofctl -O OpenFlow13 add-flows br1 flows1.txt]) - --NS_CHECK_EXEC([at_ns0], [ping -q -c 3 -i 0.3 -w 2 10.1.1.2 | FORMAT_PING], [0], [dnl -+NS_CHECK_EXEC([at_ns0], [ping -q -c 3 -i 0.3 -W 2 10.1.1.2 | FORMAT_PING], [0], [dnl +-AT_SETUP([layer3 - use non-local port as tunnel endpoint]) +- +-OVS_VSWITCHD_START([add-port br0 p0 -- set Interface p0 type=dummy ofport_request=1]) +-AT_CHECK([ovs-vsctl add-port br0 vtep0 -- set int vtep0 type=dummy], [0]) +-AT_CHECK([ovs-vsctl add-br int-br -- set bridge int-br datapath_type=dummy], [0]) +-AT_CHECK([ovs-vsctl add-port int-br t1 -- set Interface t1 type=gre \ +- options:remote_ip=1.1.2.92 ofport_request=3], [0]) +- +-AT_CHECK([ovs-appctl dpif/show], [0], [dnl +-dummy@ovs-dummy: hit:0 missed:0 +- br0: +- br0 65534/100: (dummy-internal) +- p0 1/1: (dummy) +- vtep0 2/2: (dummy) +- int-br: +- int-br 65534/3: (dummy-internal) +- t1 3/4: (gre: remote_ip=1.1.2.92) +-]) +- +-AT_CHECK([ovs-appctl netdev-dummy/ip4addr vtep0 1.1.2.88/24], [0], [OK +-]) +-AT_CHECK([ovs-appctl ovs/route/add 1.1.2.92/24 vtep0], [0], [OK +-]) +-AT_CHECK([ovs-ofctl add-flow br0 action=normal]) +-AT_CHECK([ovs-ofctl add-flow int-br action=normal]) +- +-dnl Use arp request and reply to achieve tunnel next hop mac binding +-dnl By default, vtep0's MAC address is aa:55:aa:55:00:03 +-AT_CHECK([ovs-appctl netdev-dummy/receive vtep0 'recirc_id(0),in_port(2),eth(dst=ff:ff:ff:ff:ff:ff,src=aa:55:aa:55:00:03),eth_type(0x0806),arp(tip=1.1.2.92,sip=1.1.2.88,op=1,sha=aa:55:aa:55:00:03,tha=00:00:00:00:00:00)']) +-AT_CHECK([ovs-appctl netdev-dummy/receive p0 'recirc_id(0),in_port(1),eth(src=f8:bc:12:44:34:b6,dst=aa:55:aa:55:00:03),eth_type(0x0806),arp(sip=1.1.2.92,tip=1.1.2.88,op=2,sha=f8:bc:12:44:34:b6,tha=aa:55:aa:55:00:03)']) +- +-AT_CHECK([ovs-appctl tnl/neigh/show | tail -n+3 | sort], [0], [dnl +-1.1.2.92 f8:bc:12:44:34:b6 br0 +-]) +- +-AT_CHECK([ovs-appctl ovs/route/show | tail -n+2 | sort], [0], [dnl +-User: 1.1.2.0/24 dev vtep0 SRC 1.1.2.88 +-]) +- +-dnl Check GRE tunnel pop +-AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(1),eth(src=f8:bc:12:44:34:b6,dst=aa:55:aa:55:00:03),eth_type(0x0800),ipv4(src=1.1.2.92,dst=1.1.2.88,proto=47,tos=0,ttl=64,frag=no)'], [0], [stdout]) +- +-AT_CHECK([tail -1 stdout], [0], +- [Datapath actions: tnl_pop(4) +-]) +- +-dnl Check GRE tunnel push +-AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(3),eth(dst=f9:bc:12:44:34:b6,src=af:55:aa:55:00:03),eth_type(0x0800),ipv4(src=1.1.3.88,dst=1.1.3.92,proto=1,tos=0,ttl=64,frag=no)'], [0], [stdout]) +-AT_CHECK([tail -1 stdout], [0], +- [Datapath actions: tnl_push(tnl_port(4),header(size=38,type=3,eth(dst=f8:bc:12:44:34:b6,src=aa:55:aa:55:00:03,dl_type=0x0800),ipv4(src=1.1.2.88,dst=1.1.2.92,proto=47,tos=0,ttl=64,frag=0x4000),gre((flags=0x0,proto=0x6558))),out_port(2)),1 +-]) +- +-OVS_VSWITCHD_STOP +-AT_CLEANUP +- + AT_SETUP([layer3 - ping over MPLS Bareudp]) +-OVS_CHECK_MIN_KERNEL(5, 7) ++OVS_CHECK_BAREUDP() + OVS_TRAFFIC_VSWITCHD_START([_ADD_BR([br1])]) + ADD_NAMESPACES(at_ns0, at_ns1) + +@@ -191,18 +136,18 @@ AT_CHECK([ovs-vsctl add-port br1 patch1]) + AT_CHECK([ovs-ofctl -O OpenFlow13 add-flows br0 flows0.txt]) + AT_CHECK([ovs-ofctl -O OpenFlow13 add-flows br1 flows1.txt]) + +-NS_CHECK_EXEC([at_ns0], [ping -q -c 3 -i 0.3 -w 2 10.1.1.2 | FORMAT_PING], [0], [dnl ++NS_CHECK_EXEC([at_ns0], [ping -q -c 3 -i 0.3 -W 2 10.1.1.2 | FORMAT_PING], [0], [dnl 3 packets transmitted, 3 received, 0% packet loss, time 0ms ]) @@ -9208,7 +13044,7 @@ index c37852b21..6fbdedb64 100644 OVS_TRAFFIC_VSWITCHD_START([_ADD_BR([br1])]) ADD_NAMESPACES(at_ns0, at_ns1) -@@ -239,11 +239,11 @@ AT_CHECK([ovs-vsctl add-port br1 patch1]) +@@ -239,11 +184,11 @@ AT_CHECK([ovs-vsctl add-port br1 patch1]) AT_CHECK([ovs-ofctl -O OpenFlow13 add-flows br0 flows0.txt]) AT_CHECK([ovs-ofctl -O OpenFlow13 add-flows br1 flows1.txt]) @@ -9223,7 +13059,7 @@ index c37852b21..6fbdedb64 100644 ]) OVS_TRAFFIC_VSWITCHD_STOP diff --git a/tests/system-offloads-traffic.at b/tests/system-offloads-traffic.at -index d36da0580..52ad9291c 100644 +index d36da05809..c9ef3e200e 100644 --- a/tests/system-offloads-traffic.at +++ b/tests/system-offloads-traffic.at @@ -29,7 +29,7 @@ ADD_NAMESPACES(at_ns0, at_ns1) @@ -9453,7 +13289,7 @@ index d36da0580..52ad9291c 100644 10 packets transmitted, 10 received, 0% packet loss, time 0ms ]) -@@ -742,3 +742,116 @@ recirc_id(),in_port(3),eth_type(0x0800),ipv4(frag=no), packets:29, bytes +@@ -742,3 +742,149 @@ recirc_id(),in_port(3),eth_type(0x0800),ipv4(frag=no), packets:29, bytes OVS_TRAFFIC_VSWITCHD_STOP AT_CLEANUP @@ -9570,8 +13406,113 @@ index d36da0580..52ad9291c 100644 +/failed to offload flow/d +"]) +AT_CLEANUP ++ ++AT_SETUP([offloads - handling of geneve corrupted metadata - offloads enabled]) ++OVS_CHECK_GENEVE() ++ ++OVS_TRAFFIC_VSWITCHD_START( ++ [_ADD_BR([br-underlay]) -- \ ++ set bridge br0 other-config:hwaddr=f2:ff:00:00:00:01 -- \ ++ set bridge br-underlay other-config:hwaddr=f2:ff:00:00:00:02], ++ [], [-- set Open_vSwitch . other_config:hw-offload=true]) ++ ++AT_CHECK([ovs-ofctl add-flow br0 "actions=normal"]) ++AT_CHECK([ovs-ofctl add-flow br-underlay "actions=normal"]) ++ ++ADD_NAMESPACES(at_ns0) ++ ++dnl Set up underlay link from host into the namespace using veth pair. ++ADD_VETH(p0, at_ns0, br-underlay, "172.31.1.1/24", f2:ff:00:00:00:03) ++AT_CHECK([ip addr add dev br-underlay "172.31.1.100/24"]) ++AT_CHECK([ip link set dev br-underlay up]) ++ ++dnl Set up tunnel endpoints on OVS outside the namespace and with a native ++dnl linux device inside the namespace. ++ADD_OVS_TUNNEL([geneve], [br0], [at_gnv0], [172.31.1.1], [10.1.1.100/24]) ++ADD_NATIVE_TUNNEL([geneve], [ns_gnv0], [at_ns0], [172.31.1.100], [10.1.1.1/24], ++ [vni 0], [address f2:ff:00:00:00:04]) ++ ++NS_CHECK_EXEC([at_ns0], [$PYTHON3 $srcdir/sendpkt.py p0 f2 ff 00 00 00 02 f2 ff 00 00 00 03 08 00 45 00 00 52 00 01 00 00 40 11 1f f7 ac 1f 01 01 ac 1f 01 64 de c1 17 c1 00 3e 59 e9 01 00 65 58 00 00 00 00 00 03 00 02 f2 ff 00 00 00 01 f2 ff 00 00 00 04 08 00 45 00 00 1c 00 01 00 00 40 01 64 7a 0a 01 01 01 0a 01 01 64 08 00 f7 ff 00 00 00 00 > /dev/null]) ++ ++OVS_WAIT_UNTIL([grep -q 'Invalid Geneve tunnel metadata' ovs-vswitchd.log]) ++ ++OVS_TRAFFIC_VSWITCHD_STOP(["/Invalid Geneve tunnel metadata on bridge br0 while processing icmp,in_port=1,vlan_tci=0x0000,dl_src=f2:ff:00:00:00:04,dl_dst=f2:ff:00:00:00:01,nw_src=10.1.1.1,nw_dst=10.1.1.100,nw_tos=0,nw_ecn=0,nw_ttl=64,nw_frag=no,icmp_type=8,icmp_code=0/d ++/Unable to parse geneve options/d"]) ++AT_CLEANUP +diff --git a/tests/system-route.at b/tests/system-route.at +index 270956d13f..3ea5f6e9ba 100644 +--- a/tests/system-route.at ++++ b/tests/system-route.at +@@ -25,3 +25,67 @@ OVS_WAIT_UNTIL([test `ovs-appctl ovs/route/show | grep -c 'p1-route'` -eq 0 ]) + + OVS_TRAFFIC_VSWITCHD_STOP + AT_CLEANUP ++ ++dnl Checks that OVS doesn't use routes from non-standard tables. ++AT_SETUP([ovs-route - route tables]) ++AT_KEYWORDS([route]) ++OVS_TRAFFIC_VSWITCHD_START() ++ ++dnl Create tap port. ++on_exit 'ip link del p1-route' ++AT_CHECK([ip tuntap add name p1-route mode tap]) ++AT_CHECK([ip link set p1-route up]) ++ ++dnl Add ip address. ++AT_CHECK([ip addr add 10.0.0.17/24 dev p1-route], [0], [stdout]) ++ ++dnl Check that OVS catches route updates. ++OVS_WAIT_UNTIL_EQUAL([ovs-appctl ovs/route/show | grep 'p1-route' | sort], [dnl ++Cached: 10.0.0.0/24 dev p1-route SRC 10.0.0.17 ++Cached: 10.0.0.17/32 dev p1-route SRC 10.0.0.17 local]) ++ ++dnl Add a route to the main routing table and check that OVS caches ++dnl this new route. ++AT_CHECK([ip route add 10.0.0.18/32 dev p1-route]) ++OVS_WAIT_UNTIL_EQUAL([ovs-appctl ovs/route/show | grep 'p1-route' | sort], [dnl ++Cached: 10.0.0.0/24 dev p1-route SRC 10.0.0.17 ++Cached: 10.0.0.17/32 dev p1-route SRC 10.0.0.17 local ++Cached: 10.0.0.18/32 dev p1-route SRC 10.0.0.17]) ++ ++dnl Add a route to a custom routing table and check that OVS doesn't cache it. ++AT_CHECK([ip route add 10.0.0.19/32 dev p1-route table 42]) ++AT_CHECK([ip route show table 42 | grep 'p1-route' | grep -q '10.0.0.19']) ++dnl Give the main thread a chance to act. ++AT_CHECK([ovs-appctl revalidator/wait]) ++dnl Check that OVS didn't learn this route. ++AT_CHECK([ovs-appctl ovs/route/show | grep 'p1-route' | sort], [0], [dnl ++Cached: 10.0.0.0/24 dev p1-route SRC 10.0.0.17 ++Cached: 10.0.0.17/32 dev p1-route SRC 10.0.0.17 local ++Cached: 10.0.0.18/32 dev p1-route SRC 10.0.0.17 ++]) ++ ++dnl Delete a route from the main table and check that OVS removes the route ++dnl from the cache. ++AT_CHECK([ip route del 10.0.0.18/32 dev p1-route]) ++OVS_WAIT_UNTIL_EQUAL([ovs-appctl ovs/route/show | grep 'p1-route' | sort], [dnl ++Cached: 10.0.0.0/24 dev p1-route SRC 10.0.0.17 ++Cached: 10.0.0.17/32 dev p1-route SRC 10.0.0.17 local]) ++ ++dnl Delete a route from a custom routing table and check that the cache ++dnl dosn't change. ++AT_CHECK([ip route del 10.0.0.19/32 dev p1-route table 42]) ++dnl Give the main thread a chance to act. ++AT_CHECK([ovs-appctl revalidator/wait]) ++dnl Check that the cache is still the same. ++AT_CHECK([ovs-appctl ovs/route/show | grep 'p1-route' | sort], [0], [dnl ++Cached: 10.0.0.0/24 dev p1-route SRC 10.0.0.17 ++Cached: 10.0.0.17/32 dev p1-route SRC 10.0.0.17 local ++]) ++ ++dnl Delete ip address. ++AT_CHECK([ip addr del 10.0.0.17/24 dev p1-route], [0], [stdout]) ++dnl Check that routes were removed from OVS. ++OVS_WAIT_UNTIL([test $(ovs-appctl ovs/route/show | grep -c 'p1-route') -eq 0 ]) ++ ++OVS_TRAFFIC_VSWITCHD_STOP ++AT_CLEANUP diff --git a/tests/system-tap.at b/tests/system-tap.at -index 871a3bda4..3d84a5318 100644 +index 871a3bda4f..3d84a53182 100644 --- a/tests/system-tap.at +++ b/tests/system-tap.at @@ -22,7 +22,7 @@ AT_CHECK([ip netns exec at_ns1 ip link set dev tap1 up]) @@ -9584,7 +13525,7 @@ index 871a3bda4..3d84a5318 100644 ]) diff --git a/tests/system-traffic.at b/tests/system-traffic.at -index 221d96aef..117333358 100644 +index 221d96aefb..7927f8e53c 100644 --- a/tests/system-traffic.at +++ b/tests/system-traffic.at @@ -10,13 +10,13 @@ ADD_NAMESPACES(at_ns0, at_ns1) @@ -9991,7 +13932,91 @@ index 221d96aef..117333358 100644 sleep 1 dnl Generate a single packet trough the controler that needs an ARP modification -@@ -930,7 +930,7 @@ NETNS_DAEMONIZE([at_ns0], [tcpdump -n -i p0 dst host 172.31.1.1 -l > p0.pcap 2>/ +@@ -903,6 +903,83 @@ ovs-pcap p0.pcap + AT_CHECK([ovs-pcap p0.pcap | grep -Eq "^[[[:xdigit:]]]{24}86dd60000000003a1140fc000000000000000000000000000100fc000000000000000000000000000001[[[:xdigit:]]]{4}17c1003a[[[:xdigit:]]]{4}0000655800000000fffffffffffffa163e949d8008060001080006040001[[[:xdigit:]]]{12}0a0000f40000000000000a0000fe$"]) + AT_CLEANUP + ++AT_SETUP([datapath - bridging two geneve tunnels]) ++OVS_CHECK_TUNNEL_TSO() ++OVS_CHECK_GENEVE() ++ ++OVS_TRAFFIC_VSWITCHD_START() ++ADD_BR([br-underlay-0]) ++ADD_BR([br-underlay-1]) ++ ++ADD_NAMESPACES(at_ns0) ++ADD_NAMESPACES(at_ns1) ++ ++dnl Set up underlay link from host into the namespaces using veth pairs. ++ADD_VETH(p0, at_ns0, br-underlay-0, "172.31.1.1/24") ++AT_CHECK([ip addr add dev br-underlay-0 "172.31.1.100/24"]) ++AT_CHECK([ip link set dev br-underlay-0 up]) ++ ++ADD_VETH(p1, at_ns1, br-underlay-1, "172.31.2.1/24") ++AT_CHECK([ip addr add dev br-underlay-1 "172.31.2.100/24"]) ++AT_CHECK([ip link set dev br-underlay-1 up]) ++ ++dnl Set up two OVS tunnel endpoints in a root namespace and two native ++dnl linux devices inside the test namespaces. ++dnl ++dnl ns_gnv0 | ns_gnv1 ++dnl ip: 10.1.1.1/24 | ip: 10.1.1.2/24 ++dnl remote_ip: 172.31.1.100 | remote_ip: 172.31.2.100 ++dnl | | | ++dnl | | | ++dnl p0 | p1 ++dnl ip: 172.31.1.1/24 | ip: 172.31.2.1/24 ++dnl | NS0 | NS1 | ++dnl ---------|------------------------+------------------|-------------------- ++dnl | | ++dnl br-underlay-0: br-underlay-1: ++dnl ip: 172.31.1.100/24 ip: 172.31.2.100/24 ++dnl ovs-p0 ovs-p1 ++dnl | | ++dnl | br0 | ++dnl encap/decap --- ip: 10.1.1.100/24 --------- encap/decap ++dnl at_gnv0 ++dnl remote_ip: 172.31.1.1 ++dnl at_gnv1 ++dnl remote_ip: 172.31.2.1 ++dnl ++ADD_OVS_TUNNEL([geneve], [br0], [at_gnv0], [172.31.1.1], [10.1.1.100/24]) ++ADD_NATIVE_TUNNEL([geneve], [ns_gnv0], [at_ns0], [172.31.1.100], [10.1.1.1/24], ++ [vni 0]) ++ADD_OVS_TUNNEL([geneve], [br0], [at_gnv1], [172.31.2.1], [10.1.1.101/24]) ++ADD_NATIVE_TUNNEL([geneve], [ns_gnv1], [at_ns1], [172.31.2.100], [10.1.1.2/24], ++ [vni 0]) ++ ++AT_CHECK([ovs-ofctl add-flow br0 "actions=normal"]) ++AT_CHECK([ovs-ofctl add-flow br-underlay-0 "actions=normal"]) ++AT_CHECK([ovs-ofctl add-flow br-underlay-1 "actions=normal"]) ++ ++dnl First, check both underlays. ++NS_CHECK_EXEC([at_ns0], [ping -q -c 3 -i 0.3 -W 2 172.31.1.100 | FORMAT_PING], [0], [dnl ++3 packets transmitted, 3 received, 0% packet loss, time 0ms ++]) ++NS_CHECK_EXEC([at_ns1], [ping -q -c 3 -i 0.3 -W 2 172.31.2.100 | FORMAT_PING], [0], [dnl ++3 packets transmitted, 3 received, 0% packet loss, time 0ms ++]) ++ ++dnl Now, check the overlay with different packet sizes. ++NS_CHECK_EXEC([at_ns0], [ping -q -c 3 -i 0.3 -W 2 10.1.1.2 | FORMAT_PING], [0], [dnl ++3 packets transmitted, 3 received, 0% packet loss, time 0ms ++]) ++NS_CHECK_EXEC([at_ns0], [ping -s 1600 -q -c 3 -i 0.3 -W 2 10.1.1.2 | FORMAT_PING], [0], [dnl ++3 packets transmitted, 3 received, 0% packet loss, time 0ms ++]) ++NS_CHECK_EXEC([at_ns0], [ping -s 3200 -q -c 3 -i 0.3 -W 2 10.1.1.2 | FORMAT_PING], [0], [dnl ++3 packets transmitted, 3 received, 0% packet loss, time 0ms ++]) ++ ++OVS_TRAFFIC_VSWITCHD_STOP ++AT_CLEANUP ++ + AT_SETUP([datapath - ping over gre tunnel by simulated packets]) + OVS_CHECK_TUNNEL_TSO() + OVS_CHECK_MIN_KERNEL(3, 10) +@@ -930,7 +1007,7 @@ NETNS_DAEMONIZE([at_ns0], [tcpdump -n -i p0 dst host 172.31.1.1 -l > p0.pcap 2>/ sleep 1 dnl First, check the underlay. @@ -10000,7 +14025,7 @@ index 221d96aef..117333358 100644 3 packets transmitted, 3 received, 0% packet loss, time 0ms ]) -@@ -978,7 +978,7 @@ NETNS_DAEMONIZE([at_ns0], [tcpdump -n -x -i p0 dst host 172.31.1.1 -l > p0.pcap +@@ -978,7 +1055,7 @@ NETNS_DAEMONIZE([at_ns0], [tcpdump -n -x -i p0 dst host 172.31.1.1 -l > p0.pcap sleep 1 dnl First, check the underlay @@ -10009,7 +14034,7 @@ index 221d96aef..117333358 100644 3 packets transmitted, 3 received, 0% packet loss, time 0ms ]) -@@ -1031,7 +1031,7 @@ NETNS_DAEMONIZE([at_ns0], [tcpdump -n -x -i p0 dst host 172.31.1.1 -l > p0.pcap +@@ -1031,7 +1108,7 @@ NETNS_DAEMONIZE([at_ns0], [tcpdump -n -x -i p0 dst host 172.31.1.1 -l > p0.pcap sleep 1 dnl First, check the underlay. @@ -10018,7 +14043,7 @@ index 221d96aef..117333358 100644 3 packets transmitted, 3 received, 0% packet loss, time 0ms ]) -@@ -1088,7 +1088,7 @@ NETNS_DAEMONIZE([at_ns0], [tcpdump -n -x -i p0 dst host fc00:100::1 -l > p0.pcap +@@ -1088,7 +1165,7 @@ NETNS_DAEMONIZE([at_ns0], [tcpdump -n -x -i p0 dst host fc00:100::1 -l > p0.pcap sleep 1 dnl First, check the underlay. @@ -10027,7 +14052,7 @@ index 221d96aef..117333358 100644 3 packets transmitted, 3 received, 0% packet loss, time 0ms ]) -@@ -1144,7 +1144,7 @@ NETNS_DAEMONIZE([at_ns0], [tcpdump -n -x -i p0 dst host fc00:100::1 -l > p0.pcap +@@ -1144,7 +1221,7 @@ NETNS_DAEMONIZE([at_ns0], [tcpdump -n -x -i p0 dst host fc00:100::1 -l > p0.pcap sleep 1 dnl First, check the underlay. @@ -10036,7 +14061,7 @@ index 221d96aef..117333358 100644 3 packets transmitted, 3 received, 0% packet loss, time 0ms ]) -@@ -1183,7 +1183,7 @@ priority=10 in_port=2,ip,actions=clone(mod_dl_src(ae:c6:7e:54:8d:4d),mod_dl_dst( +@@ -1183,7 +1260,7 @@ priority=10 in_port=2,ip,actions=clone(mod_dl_src(ae:c6:7e:54:8d:4d),mod_dl_dst( AT_CHECK([ovs-ofctl add-flows br0 flows.txt]) AT_CHECK([ovs-ofctl monitor br0 65534 invalid_ttl --detach --no-chdir --pidfile 2> ofctl_monitor.log]) @@ -10045,7 +14070,7 @@ index 221d96aef..117333358 100644 3 packets transmitted, 3 received, 0% packet loss, time 0ms ]) -@@ -1224,11 +1224,11 @@ table=1,priority=10 actions=normal +@@ -1224,11 +1301,11 @@ table=1,priority=10 actions=normal AT_CHECK([ovs-ofctl add-flows br0 flows.txt]) AT_CHECK([ovs-ofctl add-flows br1 flows.txt]) @@ -10059,7 +14084,7 @@ index 221d96aef..117333358 100644 3 packets transmitted, 3 received, 0% packet loss, time 0ms ]) -@@ -1263,11 +1263,11 @@ table=3,priority=10 actions=normal +@@ -1263,11 +1340,11 @@ table=3,priority=10 actions=normal AT_CHECK([ovs-ofctl add-flows br0 flows.txt]) AT_CHECK([ovs-ofctl add-flows br1 flows.txt]) @@ -10073,7 +14098,7 @@ index 221d96aef..117333358 100644 3 packets transmitted, 3 received, 0% packet loss, time 0ms ]) OVS_TRAFFIC_VSWITCHD_STOP -@@ -1938,7 +1938,7 @@ add in_port=ovs-p1,actions=ovs-p0,br0 +@@ -1938,7 +2015,7 @@ add in_port=ovs-p1,actions=ovs-p0,br0 ]) AT_CHECK([ovs-ofctl add-flows br0 flows.txt]) @@ -10082,7 +14107,7 @@ index 221d96aef..117333358 100644 10 packets transmitted, 10 received, 0% packet loss, time 0ms ]) -@@ -1957,7 +1957,7 @@ modify in_port=ovs-p1,actions=ovs-p0 +@@ -1957,7 +2034,7 @@ modify in_port=ovs-p1,actions=ovs-p0 AT_CHECK([ovs-ofctl add-flows br0 flows2.txt]) AT_CHECK([ovs-appctl revalidator/wait], [0]) @@ -10091,7 +14116,7 @@ index 221d96aef..117333358 100644 10 packets transmitted, 10 received, 0% packet loss, time 0ms ]) -@@ -1972,7 +1972,7 @@ recirc_id(),in_port(3),eth_type(0x0800),ipv4(frag=no), packets:19, bytes +@@ -1972,7 +2049,7 @@ recirc_id(),in_port(3),eth_type(0x0800),ipv4(frag=no), packets:19, bytes AT_CHECK([ovs-ofctl add-flows br0 flows.txt]) AT_CHECK([ovs-appctl revalidator/wait], [0]) @@ -10100,7 +14125,65 @@ index 221d96aef..117333358 100644 10 packets transmitted, 10 received, 0% packet loss, time 0ms ]) -@@ -2343,6 +2343,7 @@ AT_CLEANUP +@@ -1987,6 +2064,57 @@ recirc_id(),in_port(3),eth_type(0x0800),ipv4(frag=no), packets:29, bytes + OVS_TRAFFIC_VSWITCHD_STOP + AT_CLEANUP + ++AT_SETUP([datapath - Neighbor Discovery with loose match]) ++OVS_TRAFFIC_VSWITCHD_START() ++ ++ADD_NAMESPACES(at_ns0, at_ns1) ++ ++ADD_VETH(p0, at_ns0, br0, "2001::1:0:392/64", 36:b1:ee:7c:01:03) ++ADD_VETH(p1, at_ns1, br0, "2001::1:0:9/64", 36:b1:ee:7c:01:02) ++ ++dnl Set up flows for moving icmp ND Solicit around. This should be the ++dnl same for the other ND types. ++AT_DATA([flows.txt], [dnl ++table=0 priority=95 icmp6,icmp_type=136,nd_target=2001::1:0:9 actions=resubmit(,10) ++table=0 priority=95 icmp6,icmp_type=136,nd_target=2001::1:0:392 actions=resubmit(,10) ++table=0 priority=65 actions=resubmit(,20) ++table=10 actions=NORMAL ++table=20 actions=drop ++]) ++AT_CHECK([ovs-ofctl del-flows br0]) ++AT_CHECK([ovs-ofctl --bundle add-flows br0 flows.txt]) ++ ++dnl Send a mismatching neighbor discovery. ++NS_CHECK_EXEC([at_ns0], [$PYTHON3 $srcdir/sendpkt.py p0 36 b1 ee 7c 01 02 36 b1 ee 7c 01 03 86 dd 60 00 00 00 00 20 3a ff fe 80 00 00 00 00 00 00 f8 16 3e ff fe 04 66 04 fe 80 00 00 00 00 00 00 f8 16 3e ff fe a7 dd 0e 88 00 f1 f2 20 00 00 00 30 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 02 01 36 b1 ee 7c 01 03 > /dev/null]) ++ ++dnl Send a matching neighbor discovery. ++NS_CHECK_EXEC([at_ns0], [$PYTHON3 $srcdir/sendpkt.py p0 36 b1 ee 7c 01 02 36 b1 ee 7c 01 03 86 dd 60 00 00 00 00 20 3a ff fe 80 00 00 00 00 00 00 f8 16 3e ff fe 04 66 04 fe 80 00 00 00 00 00 00 f8 16 3e ff fe a7 dd 0e 88 00 fe 5f 20 00 00 00 20 01 00 00 00 00 00 00 00 00 00 01 00 00 03 92 02 01 36 b1 ee 7c 01 03 > /dev/null]) ++ ++AT_CHECK([ovs-appctl dpctl/dump-flows | strip_stats | strip_used | dnl ++ strip_key32 | strip_ptype | strip_eth | strip_recirc | dnl ++ grep ",nd" | sort], [0], [dnl ++recirc_id(),in_port(2),eth(src=36:b1:ee:7c:01:03,dst=36:b1:ee:7c:01:02),eth_type(0x86dd),ipv6(proto=58,frag=no),icmpv6(type=136),nd(target=2001::1:0:392), packets:0, bytes:0, used:never, actions:1,3 ++recirc_id(),in_port(2),eth_type(0x86dd),ipv6(proto=58,frag=no),icmpv6(type=136),nd(target=3000::1), packets:0, bytes:0, used:never, actions:drop ++]) ++ ++OVS_WAIT_UNTIL([ovs-appctl dpctl/dump-flows | grep ",nd" | wc -l | grep -E ^0]) ++ ++dnl Send a matching neighbor discovery. ++NS_CHECK_EXEC([at_ns0], [$PYTHON3 $srcdir/sendpkt.py p0 36 b1 ee 7c 01 02 36 b1 ee 7c 01 03 86 dd 60 00 00 00 00 20 3a ff fe 80 00 00 00 00 00 00 f8 16 3e ff fe 04 66 04 fe 80 00 00 00 00 00 00 f8 16 3e ff fe a7 dd 0e 88 00 fe 5f 20 00 00 00 20 01 00 00 00 00 00 00 00 00 00 01 00 00 03 92 02 01 36 b1 ee 7c 01 03 > /dev/null]) ++ ++dnl Send a mismatching neighbor discovery. ++NS_CHECK_EXEC([at_ns0], [$PYTHON3 $srcdir/sendpkt.py p0 36 b1 ee 7c 01 02 36 b1 ee 7c 01 03 86 dd 60 00 00 00 00 20 3a ff fe 80 00 00 00 00 00 00 f8 16 3e ff fe 04 66 04 fe 80 00 00 00 00 00 00 f8 16 3e ff fe a7 dd 0e 88 00 f1 f2 20 00 00 00 30 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 02 01 36 b1 ee 7c 01 03 > /dev/null]) ++ ++AT_CHECK([ovs-appctl dpctl/dump-flows | strip_stats | strip_used | dnl ++ strip_key32 | strip_ptype | strip_eth | strip_recirc | dnl ++ grep ",nd" | sort], [0], [dnl ++recirc_id(),in_port(2),eth(src=36:b1:ee:7c:01:03,dst=36:b1:ee:7c:01:02),eth_type(0x86dd),ipv6(proto=58,frag=no),icmpv6(type=136),nd(target=2001::1:0:392), packets:0, bytes:0, used:never, actions:1,3 ++recirc_id(),in_port(2),eth_type(0x86dd),ipv6(proto=58,frag=no),icmpv6(type=136),nd(target=3000::1), packets:0, bytes:0, used:never, actions:drop ++]) ++ ++OVS_TRAFFIC_VSWITCHD_STOP ++AT_CLEANUP ++ + AT_BANNER([MPLS]) + + AT_SETUP([mpls - encap header dp-support]) +@@ -2343,6 +2471,7 @@ AT_CLEANUP AT_SETUP([conntrack - ct flush]) CHECK_CONNTRACK() @@ -10108,7 +14191,7 @@ index 221d96aef..117333358 100644 OVS_TRAFFIC_VSWITCHD_START() ADD_NAMESPACES(at_ns0, at_ns1) -@@ -2353,15 +2354,15 @@ ADD_VETH(p1, at_ns1, br0, "10.1.1.2/24") +@@ -2353,15 +2482,15 @@ ADD_VETH(p1, at_ns1, br0, "10.1.1.2/24") AT_DATA([flows.txt], [dnl priority=1,action=drop priority=10,arp,action=normal @@ -10128,7 +14211,7 @@ index 221d96aef..117333358 100644 [ovs-ofctl ct-flush br0]], [ AS_BOX([Testing with FLUSH_CMD]) -@@ -2389,7 +2390,7 @@ AT_CHECK([FLUSH_CMD zone=5 'ct_nw_src=10.1.1.1,ct_nw_dst=10.1.1.2,ct_nw_proto=17 +@@ -2389,7 +2518,7 @@ AT_CHECK([FLUSH_CMD zone=5 'ct_nw_src=10.1.1.1,ct_nw_dst=10.1.1.2,ct_nw_proto=17 AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2)], [0]) dnl Test ICMP traffic @@ -10137,10 +14220,11 @@ index 221d96aef..117333358 100644 3 packets transmitted, 3 received, 0% packet loss, time 0ms ]) -@@ -2504,8 +2505,67 @@ udp,orig=(src=10.1.1.2,dst=10.1.1.1,sport=2,dport=1),reply=(src=10.1.1.1,dst=10. +@@ -2503,9 +2632,68 @@ udp,orig=(src=10.1.1.2,dst=10.1.1.1,sport=2,dport=1),reply=(src=10.1.1.1,dst=10. + AT_CHECK([FLUSH_CMD zone=5 '' 'ct_nw_src=10.1.1.1']) - AT_CHECK([ovs-appctl dpctl/dump-conntrack | grep "10\.1\.1\.1"], [1]) ++AT_CHECK([ovs-appctl dpctl/dump-conntrack | grep "10\.1\.1\.1"], [1]) + +dnl Test UDP from port 1 and 2, flush without arguments +AT_CHECK([ovs-ofctl -O OpenFlow13 packet-out br0 "in_port=1 packet=50540000000a50540000000908004500001c000000000011a4cd0a0101010a0101020001000200080000 actions=resubmit(,0)"]) @@ -10150,8 +14234,8 @@ index 221d96aef..117333358 100644 +AT_CHECK([ovs-appctl dpctl/dump-conntrack | grep "10\.1\.1\.1" | sort], [0], [dnl +udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=1,dport=2),reply=(src=10.1.1.2,dst=10.1.1.1,sport=2,dport=1) +udp,orig=(src=10.1.1.2,dst=10.1.1.1,sport=2,dport=1),reply=(src=10.1.1.1,dst=10.1.1.2,sport=1,dport=2),zone=5 - ]) - ++]) ++ +AT_CHECK([FLUSH_CMD]) + +AT_CHECK([ovs-appctl dpctl/dump-conntrack | grep "10\.1\.1\.1"], [1]) @@ -10173,9 +14257,9 @@ index 221d96aef..117333358 100644 + +AT_CHECK([FLUSH_CMD 'ct_nw_src=10.1.1.2,ct_nw_proto=132,ct_tp_src=2,ct_tp_dst=1']) + -+AT_CHECK([ovs-appctl dpctl/dump-conntrack | grep "10\.1\.1\.1"], [1]) -+]) -+ + AT_CHECK([ovs-appctl dpctl/dump-conntrack | grep "10\.1\.1\.1"], [1]) + ]) + +dnl Test flush with invalid arguments + +AT_CHECK([ovs-appctl dpctl/flush-conntrack zone=invalid 'ct_nw_src=10.1.1.1' 'ct_nw_dst=10.1.1.1'], [2], [ignore], [stderr]) @@ -10205,7 +14289,7 @@ index 221d96aef..117333358 100644 OVS_TRAFFIC_VSWITCHD_STOP AT_CLEANUP -@@ -2530,7 +2590,7 @@ priority=100,in_port=2,icmp,ct_state=+trk+est,action=1 +@@ -2530,7 +2718,7 @@ priority=100,in_port=2,icmp,ct_state=+trk+est,action=1 AT_CHECK([ovs-ofctl --bundle add-flows br0 flows.txt]) dnl Pings from ns0->ns1 should work fine. @@ -10214,7 +14298,19 @@ index 221d96aef..117333358 100644 3 packets transmitted, 3 received, 0% packet loss, time 0ms ]) -@@ -2571,7 +2631,7 @@ priority=100,in_port=2,icmp,ct_state=+trk+est,action=1 +@@ -2538,7 +2726,10 @@ AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2)], [0], [dnl + icmp,orig=(src=10.1.1.1,dst=10.1.1.2,id=,type=8,code=0),reply=(src=10.1.1.2,dst=10.1.1.1,id=,type=0,code=0) + ]) + +-AT_CHECK([ovs-appctl dpctl/flush-conntrack]) ++AT_CHECK([ovs-appctl dpctl/flush-conntrack 'ct_nw_src=10.1.1.1,ct_nw_dst=10.1.1.2']) ++ ++AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2)], [0], [dnl ++]) + + dnl Pings from ns1->ns0 should fail. + NS_CHECK_EXEC([at_ns1], [ping -q -c 3 -i 0.3 -w 2 10.1.1.1 | FORMAT_PING], [0], [dnl +@@ -2571,7 +2762,7 @@ priority=100,in_port=2,icmp,ct_state=+trk+est,action=1 AT_CHECK([ovs-ofctl --bundle add-flows br0 flows.txt]) dnl Pings from ns0->ns1 should work fine. @@ -10223,7 +14319,7 @@ index 221d96aef..117333358 100644 3 packets transmitted, 3 received, 0% packet loss, time 0ms ]) -@@ -2671,7 +2731,7 @@ NS_CHECK_EXEC([at_ns1], [ping6 -q -c 3 -i 0.3 -w 2 fc00::1 | FORMAT_PING], [0], +@@ -2671,7 +2862,7 @@ NS_CHECK_EXEC([at_ns1], [ping6 -q -c 3 -i 0.3 -w 2 fc00::1 | FORMAT_PING], [0], ]) dnl Pings from ns0->ns1 should work fine. @@ -10232,7 +14328,19 @@ index 221d96aef..117333358 100644 3 packets transmitted, 3 received, 0% packet loss, time 0ms ]) -@@ -3579,12 +3639,12 @@ dnl Modify userspace conntrack fragmentation handling. +@@ -2679,6 +2870,11 @@ AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(fc00::2)], [0], [dnl + icmpv6,orig=(src=fc00::1,dst=fc00::2,id=,type=128,code=0),reply=(src=fc00::2,dst=fc00::1,id=,type=129,code=0) + ]) + ++AT_CHECK([ovs-appctl dpctl/flush-conntrack 'ct_ipv6_src=fc00::1,ct_ipv6_dst=fc00::2']) ++ ++AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(fc00::2)], [0], [dnl ++]) ++ + OVS_TRAFFIC_VSWITCHD_STOP + AT_CLEANUP + +@@ -3579,12 +3775,12 @@ dnl Modify userspace conntrack fragmentation handling. DPCTL_MODIFY_FRAGMENTATION() dnl Ipv4 fragmentation connectivity check. @@ -10247,7 +14355,7 @@ index 221d96aef..117333358 100644 3 packets transmitted, 3 received, 0% packet loss, time 0ms ]) -@@ -3656,12 +3716,12 @@ dnl Modify userspace conntrack fragmentation handling. +@@ -3656,12 +3852,12 @@ dnl Modify userspace conntrack fragmentation handling. DPCTL_MODIFY_FRAGMENTATION() dnl Ipv4 fragmentation connectivity check. @@ -10262,7 +14370,7 @@ index 221d96aef..117333358 100644 3 packets transmitted, 3 received, 0% packet loss, time 0ms ]) -@@ -3701,22 +3761,22 @@ AT_CHECK([ovs-ofctl --bundle add-flows br0 flows.txt]) +@@ -3701,22 +3897,22 @@ AT_CHECK([ovs-ofctl --bundle add-flows br0 flows.txt]) OVS_WAIT_UNTIL([ip netns exec at_ns0 ping -c 1 10.2.2.2]) dnl Ipv4 fragmentation connectivity check. @@ -10289,7 +14397,7 @@ index 221d96aef..117333358 100644 3 packets transmitted, 3 received, 0% packet loss, time 0ms ]) -@@ -3875,12 +3935,12 @@ dnl "connect: Cannot assign requested address" +@@ -3875,12 +4071,12 @@ dnl "connect: Cannot assign requested address" OVS_WAIT_UNTIL([ip netns exec at_ns0 ping6 -c 1 fc00::2]) dnl Ipv6 fragmentation connectivity check. @@ -10304,7 +14412,7 @@ index 221d96aef..117333358 100644 3 packets transmitted, 3 received, 0% packet loss, time 0ms ]) -@@ -3957,12 +4017,12 @@ dnl "connect: Cannot assign requested address" +@@ -3957,12 +4153,12 @@ dnl "connect: Cannot assign requested address" OVS_WAIT_UNTIL([ip netns exec at_ns0 ping6 -c 1 fc00::2]) dnl Ipv4 fragmentation connectivity check. @@ -10319,7 +14427,7 @@ index 221d96aef..117333358 100644 3 packets transmitted, 3 received, 0% packet loss, time 0ms ]) -@@ -4000,22 +4060,22 @@ AT_CHECK([ovs-ofctl --bundle add-flows br0 flows.txt]) +@@ -4000,22 +4196,22 @@ AT_CHECK([ovs-ofctl --bundle add-flows br0 flows.txt]) OVS_WAIT_UNTIL([ip netns exec at_ns0 ping6 -c 1 fc00:1::4]) dnl Ipv6 fragmentation connectivity check. @@ -10346,7 +14454,7 @@ index 221d96aef..117333358 100644 3 packets transmitted, 3 received, 0% packet loss, time 0ms ]) -@@ -4226,18 +4286,18 @@ ADD_NATIVE_TUNNEL([vxlan], [at_vxlan1], [at_ns0], [172.31.1.100], [10.1.1.1/24], +@@ -4226,18 +4422,18 @@ ADD_NATIVE_TUNNEL([vxlan], [at_vxlan1], [at_ns0], [172.31.1.100], [10.1.1.1/24], [id 0 dstport 4789]) dnl First, check the underlay @@ -10369,7 +14477,7 @@ index 221d96aef..117333358 100644 3 packets transmitted, 3 received, 0% packet loss, time 0ms ]) -@@ -4286,18 +4346,18 @@ dnl "connect: Cannot assign requested address" +@@ -4286,18 +4482,18 @@ dnl "connect: Cannot assign requested address" OVS_WAIT_UNTIL([ip netns exec at_ns0 ping6 -c 1 fc00::2]) dnl First, check the underlay @@ -10392,7 +14500,7 @@ index 221d96aef..117333358 100644 3 packets transmitted, 3 received, 0% packet loss, time 0ms ]) -@@ -4410,7 +4470,7 @@ dnl The default udp_single and icmp_first timeouts are 30 seconds in +@@ -4410,7 +4606,7 @@ dnl The default udp_single and icmp_first timeouts are 30 seconds in dnl kernel DP, and 60 seconds in userspace DP. dnl Send ICMP and UDP traffic @@ -10401,7 +14509,7 @@ index 221d96aef..117333358 100644 3 packets transmitted, 3 received, 0% packet loss, time 0ms ]) AT_CHECK([ovs-ofctl -O OpenFlow13 packet-out br0 "in_port=1 packet=50540000000a50540000000908004500001c000000000011a4cd0a0101010a0101020001000200080000 actions=resubmit(,0)"]) -@@ -4436,7 +4496,7 @@ done +@@ -4436,7 +4632,7 @@ done AT_CHECK([ovs-vsctl --may-exist add-zone-tp $DP_TYPE zone=5 udp_first=1 udp_single=1 icmp_first=1 icmp_reply=1]) dnl Send ICMP and UDP traffic @@ -10410,7 +14518,7 @@ index 221d96aef..117333358 100644 3 packets transmitted, 3 received, 0% packet loss, time 0ms ]) AT_CHECK([ovs-ofctl -O OpenFlow13 packet-out br0 "in_port=1 packet=50540000000a50540000000908004500001c000000000011a4cd0a0101010a0101020001000200080000 actions=resubmit(,0)"]) -@@ -4454,7 +4514,7 @@ AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2)], [0], [dnl +@@ -4454,7 +4650,7 @@ AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2)], [0], [dnl ]) dnl Re-send ICMP and UDP traffic to test conntrack cache @@ -10419,7 +14527,7 @@ index 221d96aef..117333358 100644 3 packets transmitted, 3 received, 0% packet loss, time 0ms ]) AT_CHECK([ovs-ofctl -O OpenFlow13 packet-out br0 "in_port=1 packet=50540000000a50540000000908004500001c000000000011a4cd0a0101010a0101020001000200080000 actions=resubmit(,0)"]) -@@ -4475,7 +4535,7 @@ dnl Set the timeout policy to default again. +@@ -4475,7 +4671,7 @@ dnl Set the timeout policy to default again. AT_CHECK([ovs-vsctl del-zone-tp $DP_TYPE zone=5]) dnl Send ICMP and UDP traffic @@ -10428,7 +14536,7 @@ index 221d96aef..117333358 100644 3 packets transmitted, 3 received, 0% packet loss, time 0ms ]) AT_CHECK([ovs-ofctl -O OpenFlow13 packet-out br0 "in_port=1 packet=50540000000a50540000000908004500001c000000000011a4cd0a0101010a0101020001000200080000 actions=resubmit(,0)"]) -@@ -4668,7 +4728,7 @@ table=2,in_port=1,ip,ct_state=+trk+est,ct_zone=2,action=LOCAL +@@ -4668,7 +4864,7 @@ table=2,in_port=1,ip,ct_state=+trk+est,ct_zone=2,action=LOCAL AT_CHECK([ovs-ofctl --bundle add-flows br0 flows.txt]) @@ -10437,7 +14545,7 @@ index 221d96aef..117333358 100644 3 packets transmitted, 3 received, 0% packet loss, time 0ms ]) -@@ -4739,7 +4799,7 @@ table=4,priority=100,ip,action=output:NXM_NX_REG0[[]] +@@ -4739,7 +4935,7 @@ table=4,priority=100,ip,action=output:NXM_NX_REG0[[]] AT_CHECK([ovs-ofctl --bundle add-flows br0 flows.txt]) @@ -10446,7 +14554,55 @@ index 221d96aef..117333358 100644 3 packets transmitted, 3 received, 0% packet loss, time 0ms ]) -@@ -5763,7 +5823,7 @@ table=10 priority=0 action=drop +@@ -5434,11 +5630,11 @@ ADD_NAMESPACES(at_ns0, at_ns1) + ADD_VETH(p0, at_ns0, br0, "10.1.1.1/24") + NS_CHECK_EXEC([at_ns0], [ip link set dev p0 address 80:88:88:88:88:88]) + ADD_VETH(p1, at_ns1, br0, "10.1.1.2/24") ++NS_CHECK_EXEC([at_ns1], [ip link set dev p1 address 80:89:89:89:89:89]) + + dnl Allow any traffic from ns0->ns1. Only allow nd, return traffic from ns1->ns0. + AT_DATA([flows.txt], [dnl +-in_port=1,tcp,action=ct(commit,zone=1,nat(src=10.1.1.240:34568,random)),2 +-in_port=2,ct_state=-trk,tcp,tp_dst=34567,action=ct(table=0,zone=1,nat) ++in_port=1,tcp,action=ct(commit,zone=1,nat(src=10.1.1.240:34568)),2 + in_port=2,ct_state=-trk,tcp,tp_dst=34568,action=ct(table=0,zone=1,nat) + in_port=2,ct_state=+trk,ct_zone=1,tcp,action=1 + dnl +@@ -5462,17 +5658,29 @@ AT_CHECK([ovs-ofctl --bundle add-flows br0 flows.txt]) + + dnl HTTP requests from p0->p1 should work fine. + OVS_START_L7([at_ns1], [http]) +-NS_CHECK_EXEC([at_ns0], [wget 10.1.1.2 -t 1 -T 1 --retry-connrefused -v -o wget0.log]) ++ ++dnl Send a valid SYN to make conntrack pick it up. ++dnl The source port used is 123 to prevent unwanted reuse in the next HTTP request. ++eth=8089898989898088888888880800 ++ip4=4500002800000000400664cc0a0101010a010102 ++tcp=007b005000000000000000005002000099130000 ++syn_pkt=${eth}${ip4}${tcp} ++AT_CHECK([ovs-ofctl packet-out br0 "packet=${syn_pkt} actions=ct(commit,zone=1,nat(src=10.1.1.240:34568))"]) ++ ++AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2) | uniq], [0], [dnl ++tcp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=,dport=),reply=(src=10.1.1.2,dst=10.1.1.240,sport=,dport=),zone=1,protoinfo=(state=) ++]) + + NS_CHECK_EXEC([at_ns0], [wget 10.1.1.2 -t 1 -T 1 --retry-connrefused -v -o wget0.log], [4]) + +-AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2) | sed -e 's/dst=10.1.1.2[[45]][[0-9]]/dst=10.1.1.2XX/' | uniq], [0], [dnl +-tcp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=,dport=),reply=(src=10.1.1.2,dst=10.1.1.2XX,sport=,dport=),zone=1,protoinfo=(state=) ++AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2) | uniq], [0], [dnl ++tcp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=,dport=),reply=(src=10.1.1.2,dst=10.1.1.240,sport=,dport=),zone=1,protoinfo=(state=) + ]) + + OVS_TRAFFIC_VSWITCHD_STOP(["dnl + /Unable to NAT due to tuple space exhaustion - if DoS attack, use firewalling and\/or zone partitioning./d +-/Dropped .* log messages in last .* seconds \(most recently, .* seconds ago\) due to excessive rate/d"]) ++/Dropped .* log messages in last .* seconds \(most recently, .* seconds ago\) due to excessive rate/d ++/|WARN|.* execute ct.* failed/d"]) + AT_CLEANUP + + AT_SETUP([conntrack - more complex SNAT]) +@@ -5763,7 +5971,7 @@ table=10 priority=0 action=drop AT_CHECK([ovs-ofctl --bundle add-flows br0 flows.txt]) rm p0.pcap @@ -10455,7 +14611,7 @@ index 221d96aef..117333358 100644 sleep 1 dnl UDP packets from ns0->ns1 should solicit "destination unreachable" response. -@@ -5787,7 +5847,7 @@ AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2) | sed -e 's/dst= +@@ -5787,7 +5995,7 @@ AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2) | sed -e 's/dst= udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=,dport=),reply=(src=10.1.1.2,dst=10.1.1.2XX,sport=,dport=),mark=1 ]) @@ -10464,7 +14620,7 @@ index 221d96aef..117333358 100644 OVS_TRAFFIC_VSWITCHD_STOP AT_CLEANUP -@@ -6477,7 +6537,7 @@ dnl waiting, we get occasional failures due to the following error: +@@ -6477,7 +6685,7 @@ dnl waiting, we get occasional failures due to the following error: dnl "connect: Cannot assign requested address" OVS_WAIT_UNTIL([ip netns exec at_ns0 ping6 -c 1 fc00::240]) @@ -10473,7 +14629,7 @@ index 221d96aef..117333358 100644 3 packets transmitted, 3 received, 0% packet loss, time 0ms ]) -@@ -6531,13 +6591,13 @@ OVS_WAIT_UNTIL([ip netns exec at_ns0 ping6 -c 1 fc00::2]) +@@ -6531,13 +6739,13 @@ OVS_WAIT_UNTIL([ip netns exec at_ns0 ping6 -c 1 fc00::2]) AT_CHECK([ovs-appctl dpctl/flush-conntrack]) rm p0.pcap @@ -10489,7 +14645,7 @@ index 221d96aef..117333358 100644 AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(fc00::2)], [0], [dnl udp,orig=(src=fc00::1,dst=fc00::2,sport=,dport=),reply=(src=fc00::2,dst=fc00::240,sport=,dport=) -@@ -7172,12 +7232,12 @@ ADD_NATIVE_TUNNEL([geneve], [ns_gnv0], [at_ns0], [172.31.1.100], [10.1.1.1/24], +@@ -7172,12 +7380,12 @@ ADD_NATIVE_TUNNEL([geneve], [ns_gnv0], [at_ns0], [172.31.1.100], [10.1.1.1/24], [vni 0]) dnl First, check the underlay @@ -10504,7 +14660,7 @@ index 221d96aef..117333358 100644 3 packets transmitted, 3 received, 0% packet loss, time 0ms ]) -@@ -7220,7 +7280,7 @@ table=2,in_port=ovs-server,ip,ct_state=+trk+rpl,actions=output:ovs-client +@@ -7220,7 +7428,7 @@ table=2,in_port=ovs-server,ip,ct_state=+trk+rpl,actions=output:ovs-client AT_CHECK([ovs-ofctl add-flows br0 flows.txt]) rm server.pcap @@ -10513,7 +14669,7 @@ index 221d96aef..117333358 100644 OVS_WAIT_UNTIL([grep "listening" tcpdump0_err]) dnl Send UDP client->server -@@ -7262,7 +7322,7 @@ dnl Check the ICMP error in reply direction +@@ -7262,7 +7470,7 @@ dnl Check the ICMP error in reply direction AT_CHECK([ovs-appctl dpctl/flush-conntrack zone=42]) rm client.pcap @@ -10522,7 +14678,73 @@ index 221d96aef..117333358 100644 OVS_WAIT_UNTIL([grep "listening" tcpdump1_err]) dnl Send UDP client->server -@@ -7404,7 +7464,7 @@ dnl CVLAN traffic should match the flow and drop +@@ -7300,6 +7508,65 @@ AT_CHECK([ovs-pcap client.pcap | grep 000000002010000000002000], [0], [dnl + OVS_TRAFFIC_VSWITCHD_STOP + AT_CLEANUP + ++AT_SETUP([conntrack - Flush many conntrack entries by port]) ++CHECK_CONNTRACK() ++OVS_TRAFFIC_VSWITCHD_START() ++ ++ADD_NAMESPACES(at_ns0, at_ns1) ++ ++ADD_VETH(p0, at_ns0, br0, "10.1.1.1/24") ++ADD_VETH(p1, at_ns1, br0, "10.1.1.2/24") ++ ++AT_DATA([flows.txt], [dnl ++priority=100,in_port=1,udp,action=ct(zone=1,commit),2 ++]) ++ ++AT_CHECK([ovs-ofctl --bundle add-flows br0 flows.txt]) ++ ++dnl 20 packets from port 1 and 1 packet from port 2. ++flow_l3="\ ++ eth_src=50:54:00:00:00:09,eth_dst=50:54:00:00:00:0a,dl_type=0x0800,\ ++ nw_src=10.1.1.1,nw_dst=10.1.1.2,nw_proto=17,nw_ttl=64,nw_frag=no" ++ ++head="50540000000a50540000000908004500005c000000004011648d0a0101010a010102" ++len=72 ++base_csum=1366 ++tail="000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f\ ++ 202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f" ++ ++dst_port=1 ++for src_port in $(seq 1 20); do ++ csum=$((base_csum - src_port - dst_port)) ++ frame=$(printf "%s%04x%04x%04x%04x%s" $head 1 $src_port $len $csum $tail) ++ AT_CHECK([ovs-ofctl -O OpenFlow13 packet-out br0 "in_port=1 packet=$frame actions=resubmit(,0)"]) ++done ++ ++src_port=2 ++dst_port=1 ++csum=$((base_csum - src_port - dst_port)) ++frame=$(printf "%s%04x%04x%04x%04x%s" $head $src_port $dst_port $len $csum $tail) ++AT_CHECK([ovs-ofctl -O OpenFlow13 packet-out br0 "in_port=1 packet=$frame actions=resubmit(,0)"]) ++ ++: > conntrack ++ ++for i in $(seq 1 20); do ++ echo "udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=1,dport=${i}),reply=(src=10.1.1.2,dst=10.1.1.1,sport=${i},dport=1),zone=1" >> conntrack ++done ++echo "udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=2,dport=1),reply=(src=10.1.1.2,dst=10.1.1.1,sport=1,dport=2),zone=1" >> conntrack ++ ++sort conntrack > expout ++ ++AT_CHECK([ovs-appctl dpctl/dump-conntrack zone=1 | grep -F "src=10.1.1.1," | sort ], [0], [expout]) ++ ++dnl Check that flushing conntrack by port 1 flush all ct for port 1 but keeps ct for port 2. ++AT_CHECK([ovs-appctl dpctl/flush-conntrack zone=1 'ct_nw_proto=17,ct_tp_src=1']) ++AT_CHECK([ovs-appctl dpctl/dump-conntrack zone=1 | grep -F "src=10.1.1.1," | sort ], [0], [dnl ++udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=2,dport=1),reply=(src=10.1.1.2,dst=10.1.1.1,sport=1,dport=2),zone=1 ++]) ++ ++OVS_TRAFFIC_VSWITCHD_STOP ++AT_CLEANUP ++ + AT_BANNER([IGMP]) + + AT_SETUP([IGMP - flood under normal action]) +@@ -7404,7 +7671,7 @@ dnl CVLAN traffic should match the flow and drop AT_CHECK([ovs-appctl revalidator/purge]) AT_CHECK([ovs-vsctl set Open_vSwitch . other_config:vlan-limit=1]) AT_CHECK([ovs-ofctl add-flow br0 "priority=100 dl_type=0x8100 action=drop"]) @@ -10531,7 +14753,7 @@ index 221d96aef..117333358 100644 OVS_TRAFFIC_VSWITCHD_STOP AT_CLEANUP -@@ -7454,11 +7514,11 @@ AT_CHECK([ovs-ofctl --bundle add-flows br2 flows-customer-br.txt]) +@@ -7454,11 +7721,11 @@ AT_CHECK([ovs-ofctl --bundle add-flows br2 flows-customer-br.txt]) OVS_WAIT_UNTIL([ip netns exec at_ns0 ping -c 1 10.2.2.2]) @@ -10545,7 +14767,7 @@ index 221d96aef..117333358 100644 3 packets transmitted, 3 received, 0% packet loss, time 0ms ]) -@@ -7510,11 +7570,11 @@ AT_CHECK([ovs-ofctl --bundle add-flows br2 flows-customer-br.txt]) +@@ -7510,11 +7777,11 @@ AT_CHECK([ovs-ofctl --bundle add-flows br2 flows-customer-br.txt]) OVS_WAIT_UNTIL([ip netns exec at_ns0 ping -c 1 10.2.2.2]) @@ -10559,7 +14781,7 @@ index 221d96aef..117333358 100644 3 packets transmitted, 3 received, 0% packet loss, time 0ms ]) -@@ -7562,24 +7622,24 @@ AT_CHECK([ovs-vsctl set port ovs-p2 vlan_mode=dot1q-tunnel tag=4094 cvlans=100,2 +@@ -7562,24 +7829,24 @@ AT_CHECK([ovs-vsctl set port ovs-p2 vlan_mode=dot1q-tunnel tag=4094 cvlans=100,2 OVS_WAIT_UNTIL([ip netns exec at_ns0 ping -c 1 10.2.2.2]) OVS_WAIT_UNTIL([ip netns exec at_ns0 ping -c 1 10.3.2.2]) @@ -10589,7 +14811,7 @@ index 221d96aef..117333358 100644 OVS_TRAFFIC_VSWITCHD_STOP(["/dropping VLAN \(0\|300\) packet received on dot1q-tunnel port/d"]) AT_CLEANUP -@@ -7608,11 +7668,11 @@ AT_CHECK([ovs-ofctl --bundle add-flows br0 flows-br0.txt]) +@@ -7608,11 +7875,11 @@ AT_CHECK([ovs-ofctl --bundle add-flows br0 flows-br0.txt]) OVS_WAIT_UNTIL([ip netns exec at_ns0 ping -c 1 10.2.2.2]) @@ -10604,7 +14826,7 @@ index 221d96aef..117333358 100644 ]) diff --git a/tests/system-userspace-macros.at b/tests/system-userspace-macros.at -index b34a84775..2db62bf8d 100644 +index b34a84775b..2db62bf8d9 100644 --- a/tests/system-userspace-macros.at +++ b/tests/system-userspace-macros.at @@ -106,6 +106,16 @@ m4_define([CHECK_CONNTRACK_NAT]) @@ -10637,7 +14859,7 @@ index b34a84775..2db62bf8d 100644 + AT_SKIP_IF([:]) +]) diff --git a/tests/system-userspace-packet-type-aware.at b/tests/system-userspace-packet-type-aware.at -index 974304758..aac178eda 100644 +index 974304758f..aac178edaf 100644 --- a/tests/system-userspace-packet-type-aware.at +++ b/tests/system-userspace-packet-type-aware.at @@ -335,7 +335,7 @@ AT_CHECK([ @@ -10668,7 +14890,7 @@ index 974304758..aac178eda 100644 ]) diff --git a/tests/test-barrier.c b/tests/test-barrier.c -index 3bc5291cc..fb0ab0e69 100644 +index 3bc5291cc1..fb0ab0e695 100644 --- a/tests/test-barrier.c +++ b/tests/test-barrier.c @@ -14,13 +14,13 @@ @@ -10690,7 +14912,7 @@ index 3bc5291cc..fb0ab0e69 100644 #include "util.h" diff --git a/tests/test-id-fpool.c b/tests/test-id-fpool.c -index 25275d9ae..7bdb8154d 100644 +index 25275d9aef..7bdb8154d3 100644 --- a/tests/test-id-fpool.c +++ b/tests/test-id-fpool.c @@ -14,12 +14,12 @@ @@ -10718,7 +14940,7 @@ index 25275d9ae..7bdb8154d 100644 for (i = 0; i < n_threads; i++) { if (thread_working_ms[i] >= TIMEOUT_MS) { diff --git a/tests/test-jsonrpc.py b/tests/test-jsonrpc.py -index 1df5afa22..8a4a17593 100644 +index 1df5afa221..8a4a175938 100644 --- a/tests/test-jsonrpc.py +++ b/tests/test-jsonrpc.py @@ -199,13 +199,13 @@ notify REMOTE METHOD PARAMS send notification and exit @@ -10738,7 +14960,7 @@ index 1df5afa22..8a4a17593 100644 sys.stderr.write("%s: \"%s\" requires %d arguments but %d " "provided\n" diff --git a/tests/test-mpsc-queue.c b/tests/test-mpsc-queue.c -index a38bf9e6d..86a223caf 100644 +index a38bf9e6df..86a223caff 100644 --- a/tests/test-mpsc-queue.c +++ b/tests/test-mpsc-queue.c @@ -14,12 +14,12 @@ @@ -10766,7 +14988,7 @@ index a38bf9e6d..86a223caf 100644 for (i = 0; i < n_threads; i++) { printf(" %6" PRIu64, thread_working_ms[i]); diff --git a/tests/test-ovsdb.c b/tests/test-ovsdb.c -index 1bc5ac17a..c761822e6 100644 +index 1bc5ac17a0..d6a47de336 100644 --- a/tests/test-ovsdb.c +++ b/tests/test-ovsdb.c @@ -870,7 +870,8 @@ do_parse_rows(struct ovs_cmdl_context *ctx) @@ -10824,8 +15046,22 @@ index 1bc5ac17a..c761822e6 100644 /* Initialize row and find equivalence class. */ rows[i].uuid = *ovsdb_row_get_uuid(row); +@@ -2799,6 +2800,13 @@ do_idl(struct ovs_cmdl_context *ctx) + } else { + print_idl(idl, step++, terse); + } ++ ++ /* Just run IDL forever for a simple monitoring. */ ++ if (!strcmp(arg, "monitor")) { ++ seqno = ovsdb_idl_get_seqno(idl); ++ i--; ++ continue; ++ } + } + seqno = ovsdb_idl_get_seqno(idl); + diff --git a/tests/test-ovsdb.py b/tests/test-ovsdb.py -index a841adba4..71248854f 100644 +index a841adba4e..48f8ee2d70 100644 --- a/tests/test-ovsdb.py +++ b/tests/test-ovsdb.py @@ -37,7 +37,7 @@ vlog.init(None) @@ -10862,7 +15098,39 @@ index a841adba4..71248854f 100644 for value in json.values(): parse_uuids(value, symtab) -@@ -1049,14 +1049,14 @@ def main(argv): +@@ -757,16 +757,23 @@ def do_idl(schema_file, remote, *commands): + poller.block() + else: + # Wait for update. +- while idl.change_seqno == seqno and not idl.run(): +- rpc.run() ++ while True: ++ while idl.change_seqno == seqno and not idl.run(): ++ rpc.run() + +- poller = ovs.poller.Poller() +- idl.wait(poller) +- rpc.wait(poller) +- poller.block() ++ poller = ovs.poller.Poller() ++ idl.wait(poller) ++ rpc.wait(poller) ++ poller.block() + +- print_idl(idl, step, terse) +- step += 1 ++ print_idl(idl, step, terse) ++ step += 1 ++ ++ # Run IDL forever in case of a simple monitor, otherwise ++ # break and execute the command. ++ seqno = idl.change_seqno ++ if command != "monitor": ++ break + + seqno = idl.change_seqno + +@@ -1049,14 +1056,14 @@ def main(argv): sys.exit(1) func, n_args = commands[command_name] @@ -10879,8 +15147,21 @@ index a841adba4..71248854f 100644 if len(args) != n_args: sys.stderr.write("%s: \"%s\" requires %d arguments but %d " "provided\n" +diff --git a/tests/test-rstp.c b/tests/test-rstp.c +index 9c1026ec1a..707ee3a6c8 100644 +--- a/tests/test-rstp.c ++++ b/tests/test-rstp.c +@@ -469,6 +469,8 @@ test_rstp_main(int argc, char *argv[]) + vlog_set_pattern(VLF_CONSOLE, "%c|%p|%m"); + vlog_set_levels(NULL, VLF_SYSLOG, VLL_OFF); + ++ rstp_init(); ++ + if (argc != 2) { + ovs_fatal(0, "usage: test-rstp INPUT.RSTP"); + } diff --git a/tests/testsuite.at b/tests/testsuite.at -index cf4e3eadf..9d77a9f51 100644 +index cf4e3eadfb..9d77a9f512 100644 --- a/tests/testsuite.at +++ b/tests/testsuite.at @@ -77,3 +77,4 @@ m4_include([tests/packet-type-aware.at]) @@ -10888,77 +15169,657 @@ index cf4e3eadf..9d77a9f51 100644 m4_include([tests/drop-stats.at]) m4_include([tests/pytest.at]) +m4_include([tests/learning-switch.at]) -diff --git a/utilities/ovs-appctl-bashcomp.bash b/utilities/ovs-appctl-bashcomp.bash -index 4384be8ae..0a9af1a18 100644 ---- a/utilities/ovs-appctl-bashcomp.bash -+++ b/utilities/ovs-appctl-bashcomp.bash -@@ -223,6 +223,13 @@ printf_stderr() { - # The code below is taken from Peter Amidon. His change makes it more - # robust. - extract_bash_prompt() { -+ # On Bash 4.4+ just use the @P expansion -+ if ((BASH_VERSINFO[0] > 4 || -+ (BASH_VERSINFO[0] == 4 && BASH_VERSINFO[1] >= 4))); then -+ _BASH_PROMPT="${PS1@P}" -+ return -+ fi -+ - local myPS1 v +diff --git a/tests/tunnel-push-pop-ipv6.at b/tests/tunnel-push-pop-ipv6.at +index c96b77cd15..cb1fdf638e 100644 +--- a/tests/tunnel-push-pop-ipv6.at ++++ b/tests/tunnel-push-pop-ipv6.at +@@ -19,13 +19,15 @@ dummy@ovs-dummy: hit:0 missed:0 + t2 2/6: (ip6gre: remote_ip=2001:cafe::92) + ]) - myPS1="$(sed 's/Begin prompt/\\Begin prompt/; s/End prompt/\\End prompt/' <<< "$PS1")" -diff --git a/utilities/ovs-ofctl.c b/utilities/ovs-ofctl.c -index eabec18a3..3ce4e82ec 100644 ---- a/utilities/ovs-ofctl.c -+++ b/utilities/ovs-ofctl.c -@@ -3089,6 +3089,10 @@ ofctl_ct_flush(struct ovs_cmdl_context *ctx) - args--; - } +-dnl First setup dummy interface IP address, then add the route +-dnl so that tnl-port table can get valid IP address for the device. ++dnl Setup dummy interface IP addresses. + AT_CHECK([ovs-appctl netdev-dummy/ip6addr br0 2001:cafe::88/24], [0], [OK + ]) + AT_CHECK([ovs-appctl netdev-dummy/ip4addr br0 1.1.2.88/24], [0], [OK + ]) +-AT_CHECK([ovs-appctl ovs/route/add 2001:cafe::92/24 br0], [0], [OK ++dnl Checking that a local routes for added IPs were successfully installed. ++AT_CHECK([ovs-appctl ovs/route/show | grep Cached | sort], [0], [dnl ++Cached: 1.1.2.0/24 dev br0 SRC 1.1.2.88 local ++Cached: 2001:ca00::/24 dev br0 SRC 2001:cafe::88 local + ]) -+ if (args > 0) { -+ ovs_fatal(0, "Invalid arguments"); -+ } -+ - open_vconn(ctx->argv[1], &vconn); - enum ofp_version version = vconn_get_version(vconn); - struct ofpbuf *msg = ofp_ct_match_encode(&match, pzone, version); -diff --git a/utilities/ovs-tcpdump.in b/utilities/ovs-tcpdump.in -index a49ec9f94..4cbd9a5d3 100755 ---- a/utilities/ovs-tcpdump.in -+++ b/utilities/ovs-tcpdump.in -@@ -96,6 +96,10 @@ def _install_dst_if_linux(tap_name, mtu_value=None): - *(['ip', 'link', 'set', 'dev', str(tap_name), 'up'])) - pipe.wait() + AT_CHECK([ovs-ofctl add-flow br0 action=normal]) +@@ -93,13 +95,15 @@ dummy@ovs-dummy: hit:0 missed:0 + t3 3/6: (ip6erspan: erspan_dir=1, erspan_hwid=0x7, erspan_ver=2, key=567, remote_ip=2001:cafe::93) + ]) -+ pipe = _doexec( -+ *(['ip', '-6', 'addr', 'flush', 'dev', str(tap_name)])) -+ pipe.wait() -+ +-dnl First setup dummy interface IP address, then add the route +-dnl so that tnl-port table can get valid IP address for the device. ++dnl Setup dummy interface IP addresses. + AT_CHECK([ovs-appctl netdev-dummy/ip6addr br0 2001:cafe::88/24], [0], [OK + ]) + AT_CHECK([ovs-appctl netdev-dummy/ip4addr br0 1.1.2.88/24], [0], [OK + ]) +-AT_CHECK([ovs-appctl ovs/route/add 2001:cafe::92/24 br0], [0], [OK ++dnl Checking that a local routes for added IPs were successfully installed. ++AT_CHECK([ovs-appctl ovs/route/show | grep Cached | sort], [0], [dnl ++Cached: 1.1.2.0/24 dev br0 SRC 1.1.2.88 local ++Cached: 2001:ca00::/24 dev br0 SRC 2001:cafe::88 local + ]) - def _remove_dst_if_linux(tap_name): - _doexec( -@@ -538,6 +542,17 @@ def main(): - print(data.decode('utf-8')) - raise KeyboardInterrupt - except KeyboardInterrupt: -+ # If there is a pipe behind ovs-tcpdump (such as ovs-tcpdump -+ # -i eth0 | grep "192.168.1.1"), the pipe is no longer available -+ # after received Ctrl+C. -+ # If we write data to an unavailable pipe, a pipe error will be -+ # reported, so we turn off stdout to avoid subsequent flushing -+ # of data into the pipe. -+ try: -+ sys.stdout.close() -+ except IOError: -+ pass -+ - if pipes.poll() is None: - pipes.terminate() + AT_CHECK([ovs-ofctl add-flow br0 action=normal]) +@@ -225,14 +229,15 @@ gre_sys (3) ref_cnt=2 + vxlan_sys_4789 (4789) ref_cnt=2 + ]) -diff --git a/utilities/ovs-vsctl-bashcomp.bash b/utilities/ovs-vsctl-bashcomp.bash -index fc8245bfb..c5ad24fb7 100644 ---- a/utilities/ovs-vsctl-bashcomp.bash -+++ b/utilities/ovs-vsctl-bashcomp.bash -@@ -413,6 +413,13 @@ _ovs_vsctl_get_PS1 () { +- +-dnl First setup dummy interface IP address, then add the route +-dnl so that tnl-port table can get valid IP address for the device. ++dnl Setup dummy interface IP addresses. + AT_CHECK([ovs-appctl netdev-dummy/ip6addr br0 2001:cafe::88/24], [0], [OK + ]) + AT_CHECK([ovs-appctl netdev-dummy/ip4addr br0 1.1.2.88/24], [0], [OK + ]) +-AT_CHECK([ovs-appctl ovs/route/add 2001:cafe::92/24 br0], [0], [OK ++dnl Checking that a local routes for added IPs were successfully installed. ++AT_CHECK([ovs-appctl ovs/route/show | grep Cached | sort], [0], [dnl ++Cached: 1.1.2.0/24 dev br0 SRC 1.1.2.88 local ++Cached: 2001:ca00::/24 dev br0 SRC 2001:cafe::88 local + ]) + + AT_CHECK([ovs-ofctl add-flow br0 action=normal]) +@@ -527,3 +532,87 @@ Listening ports: + + OVS_VSWITCHD_STOP + AT_CLEANUP ++ ++AT_SETUP([tunnel_push_pop_ipv6 - local_ip configuration]) ++ ++OVS_VSWITCHD_START( ++ [add-port br0 p0 \ ++ -- set Interface p0 type=dummy ofport_request=1 \ ++ other-config:hwaddr=aa:55:aa:55:00:00]) ++AT_CHECK([ovs-appctl vlog/set dpif_netdev:dbg]) ++AT_CHECK([ovs-vsctl add-br int-br -- set bridge int-br datapath_type=dummy]) ++AT_CHECK([ovs-vsctl add-port int-br t2 \ ++ -- set Interface t2 type=geneve \ ++ options:local_ip=2001:beef::88 \ ++ options:remote_ip=2001:cafe::92 \ ++ options:key=123 ofport_request=2]) ++ ++dnl Setup multiple IP addresses. ++AT_CHECK([ovs-appctl netdev-dummy/ip6addr br0 2001:cafe::88/64], [0], [OK ++]) ++AT_CHECK([ovs-appctl netdev-dummy/ip6addr br0 2001:beef::88/64], [0], [OK ++]) ++dnl Checking that a local route for added IP was successfully installed. ++AT_CHECK([ovs-appctl ovs/route/show | grep Cached | sort], [0], [dnl ++Cached: 2001:beef::/64 dev br0 SRC 2001:beef::88 local ++Cached: 2001:cafe::/64 dev br0 SRC 2001:cafe::88 local ++]) ++AT_CHECK([ovs-ofctl add-flow br0 action=normal]) ++AT_CHECK([ovs-ofctl add-flow int-br action=normal]) ++ ++dnl This Neighbor Advertisement from p0 has two effects: ++dnl 1. The neighbor cache will learn that 2001:cafe::92 is at f8:bc:12:44:34:b6. ++dnl 2. The br0 mac learning will learn that f8:bc:12:44:34:b6 is on p0. ++AT_CHECK([ovs-appctl netdev-dummy/receive p0 dnl ++ 'recirc_id(0),in_port(1),dnl ++ eth(src=f8:bc:12:44:34:b6,dst=aa:55:aa:55:00:00),eth_type(0x86dd),dnl ++ ipv6(src=2001:cafe::92,dst=2001:cafe::88,label=0,proto=58,tclass=0,hlimit=255,frag=no),dnl ++ icmpv6(type=136,code=0),dnl ++ nd(target=2001:cafe::92,sll=00:00:00:00:00:00,tll=f8:bc:12:44:34:b6)' ++]) ++ ++dnl Check that local_ip is used for encapsulation in the trace. ++AT_CHECK([ovs-appctl ofproto/trace int-br in_port=LOCAL \ ++ | grep -E 'tunnel|actions'], [0], [dnl ++ -> output to native tunnel ++ -> tunneling to 2001:cafe::92 via br0 ++ -> tunneling from aa:55:aa:55:00:00 2001:beef::88 to f8:bc:12:44:34:b6 2001:cafe::92 ++Datapath actions: tnl_push(tnl_port(6081),header(size=70,type=5,dnl ++eth(dst=f8:bc:12:44:34:b6,src=aa:55:aa:55:00:00,dl_type=0x86dd),dnl ++ipv6(src=2001:beef::88,dst=2001:cafe::92,label=0,proto=17,tclass=0x0,hlimit=64),dnl ++udp(src=0,dst=6081,csum=0xffff),geneve(vni=0x7b)),out_port(100)),1 ++]) ++ ++dnl Now check that the packet actually has the local_ip in the header. ++AT_CHECK([ovs-vsctl -- set Interface p0 options:tx_pcap=p0.pcap]) ++ ++packet=50540000000a5054000000091234 ++eth=f8bc124434b6aa55aa55000086dd ++ip6=60000000001e11402001beef0000000000000000000000882001cafe000000000000000000000092 ++dnl Source port is based on a packet hash, so it may differ depending on the ++dnl compiler flags and CPU type. Same for UDP checksum. Masked with '....'. ++udp=....17c1001e.... ++geneve=0000655800007b00 ++encap=${eth}${ip6}${udp}${geneve} ++dnl Output to tunnel from a int-br internal port. ++dnl Checking that the packet arrived and it was correctly encapsulated. ++AT_CHECK([ovs-appctl netdev-dummy/receive int-br "${packet}"]) ++OVS_WAIT_UNTIL([test $(ovs-pcap p0.pcap | grep -c "${encap}${packet}") -eq 1]) ++dnl Sending again to exercise the non-miss upcall path. ++AT_CHECK([ovs-appctl netdev-dummy/receive int-br "${packet}"]) ++OVS_WAIT_UNTIL([test $(ovs-pcap p0.pcap | grep -c "${encap}${packet}") -eq 2]) ++ ++dnl Finally, checking that the datapath flow also has a local_ip. ++AT_CHECK([ovs-appctl dpctl/dump-flows | grep tnl_push \ ++ | strip_ufid | strip_used], [0], [dnl ++recirc_id(0),in_port(2),packet_type(ns=0,id=0),dnl ++eth(src=50:54:00:00:00:09,dst=50:54:00:00:00:0a),eth_type(0x1234), dnl ++packets:1, bytes:14, used:0.0s, dnl ++actions:tnl_push(tnl_port(6081),header(size=70,type=5,dnl ++eth(dst=f8:bc:12:44:34:b6,src=aa:55:aa:55:00:00,dl_type=0x86dd),dnl ++ipv6(src=2001:beef::88,dst=2001:cafe::92,label=0,proto=17,tclass=0x0,hlimit=64),dnl ++udp(src=0,dst=6081,csum=0xffff),geneve(vni=0x7b)),out_port(100)),1 ++]) ++ ++OVS_VSWITCHD_STOP ++AT_CLEANUP +diff --git a/tests/tunnel-push-pop.at b/tests/tunnel-push-pop.at +index 013ecbcaa8..281904ddcb 100644 +--- a/tests/tunnel-push-pop.at ++++ b/tests/tunnel-push-pop.at +@@ -30,17 +30,15 @@ dummy@ovs-dummy: hit:0 missed:0 + t4 5/3: (erspan: erspan_dir=flow, erspan_hwid=flow, erspan_idx=flow, erspan_ver=flow, key=56, remote_ip=flow) + ]) + +-dnl First setup dummy interface IP address, then add the route +-dnl so that tnl-port table can get valid IP address for the device. ++dnl Setup dummy interface IP addresses. + AT_CHECK([ovs-appctl netdev-dummy/ip4addr br0 1.1.2.88/24], [0], [OK + ]) + AT_CHECK([ovs-appctl netdev-dummy/ip6addr br0 2001:cafe::88/24], [0], [OK + ]) +- +-AT_CHECK([ovs-appctl ovs/route/add 1.1.2.92/24 br0], [0], [OK +-]) +- +-AT_CHECK([ovs-appctl ovs/route/add 1.1.2.92/24 br0 pkt_mark=1234], [0], [OK ++dnl Checking that a local routes for added IPs were successfully installed. ++AT_CHECK([ovs-appctl ovs/route/show | grep Cached | sort], [0], [dnl ++Cached: 1.1.2.0/24 dev br0 SRC 1.1.2.88 local ++Cached: 2001:ca00::/24 dev br0 SRC 2001:cafe::88 local + ]) + + AT_CHECK([ovs-ofctl add-flow br0 action=normal]) +@@ -237,18 +235,21 @@ dummy@ovs-dummy: hit:0 missed:0 + t8 9/2152: (gtpu: key=123, remote_ip=1.1.2.92) + ]) + +-dnl First setup dummy interface IP address, then add the route +-dnl so that tnl-port table can get valid IP address for the device. ++dnl Setup dummy interface IP addresses. + AT_CHECK([ovs-appctl netdev-dummy/ip4addr br0 1.1.2.88/24], [0], [OK + ]) + AT_CHECK([ovs-appctl netdev-dummy/ip6addr br0 2001:cafe::88/24], [0], [OK + ]) +- +-AT_CHECK([ovs-appctl ovs/route/add 1.1.2.92/24 br0], [0], [OK +-]) +- ++dnl Add a static route with a mark. + AT_CHECK([ovs-appctl ovs/route/add 1.1.2.92/24 br0 pkt_mark=1234], [0], [OK + ]) ++dnl Checking that local routes for added IPs and the static route with a mark ++dnl were successfully installed. ++AT_CHECK([ovs-appctl ovs/route/show | grep br0 | sort], [0], [dnl ++Cached: 1.1.2.0/24 dev br0 SRC 1.1.2.88 local ++Cached: 2001:ca00::/24 dev br0 SRC 2001:cafe::88 local ++User: 1.1.2.0/24 MARK 1234 dev br0 SRC 1.1.2.88 ++]) + + AT_CHECK([ovs-ofctl add-flow br0 action=normal]) + +@@ -690,12 +691,12 @@ AT_CHECK([ovs-vsctl add-port int-br t2 -- set Interface t2 type=geneve \ + options:remote_ip=1.1.2.92 options:key=123 ofport_request=2 \ + ]) + +-dnl First setup dummy interface IP address, then add the route +-dnl so that tnl-port table can get valid IP address for the device. ++dnl Setup dummy interface IP address. + AT_CHECK([ovs-appctl netdev-dummy/ip4addr br0 1.1.2.88/24], [0], [OK + ]) +- +-AT_CHECK([ovs-appctl ovs/route/add 1.1.2.92/24 br0], [0], [OK ++dnl Checking that a local route for added IP was successfully installed. ++AT_CHECK([ovs-appctl ovs/route/show | grep Cached], [0], [dnl ++Cached: 1.1.2.0/24 dev br0 SRC 1.1.2.88 local + ]) + + AT_CHECK([ovs-ofctl add-flow br0 action=normal]) +@@ -731,11 +732,12 @@ AT_CHECK([ovs-vsctl add-port int-br t2 dnl + -- set Interface t2 type=geneve options:remote_ip=1.1.2.92 dnl + options:key=123 ofport_request=2]) + +-dnl First setup dummy interface IP address, then add the route +-dnl so that tnl-port table can get valid IP address for the device. ++dnl Setup dummy interface IP address. + AT_CHECK([ovs-appctl netdev-dummy/ip4addr br0 1.1.2.88/24], [0], [OK + ]) +-AT_CHECK([ovs-appctl ovs/route/add 1.1.2.92/24 br0], [0], [OK ++dnl Checking that a local route for added IP was successfully installed. ++AT_CHECK([ovs-appctl ovs/route/show | grep Cached], [0], [dnl ++Cached: 1.1.2.0/24 dev br0 SRC 1.1.2.88 local + ]) + AT_CHECK([ovs-ofctl add-flow br0 action=normal]) + +@@ -777,6 +779,88 @@ AT_CHECK([ovs-appctl dpctl/dump-flows | grep -q 'slow_path(action)'], [0]) + OVS_VSWITCHD_STOP + AT_CLEANUP + ++AT_SETUP([tunnel_push_pop - local_ip configuration]) ++ ++OVS_VSWITCHD_START( ++ [add-port br0 p0 \ ++ -- set Interface p0 type=dummy ofport_request=1 \ ++ other-config:hwaddr=aa:55:aa:55:00:00]) ++AT_CHECK([ovs-appctl vlog/set dpif_netdev:dbg]) ++AT_CHECK([ovs-vsctl add-br int-br -- set bridge int-br datapath_type=dummy]) ++AT_CHECK([ovs-vsctl add-port int-br t2 \ ++ -- set Interface t2 type=geneve \ ++ options:local_ip=2.2.2.88 \ ++ options:remote_ip=1.1.2.92 \ ++ options:key=123 ofport_request=2]) ++ ++dnl Setup multiple IP addresses. ++AT_CHECK([ovs-appctl netdev-dummy/ip4addr br0 1.1.2.88/24], [0], [OK ++]) ++AT_CHECK([ovs-appctl netdev-dummy/ip4addr br0 2.2.2.88/24], [0], [OK ++]) ++dnl Checking that a local route for added IP was successfully installed. ++AT_CHECK([ovs-appctl ovs/route/show | grep Cached | sort], [0], [dnl ++Cached: 1.1.2.0/24 dev br0 SRC 1.1.2.88 local ++Cached: 2.2.2.0/24 dev br0 SRC 2.2.2.88 local ++]) ++AT_CHECK([ovs-ofctl add-flow br0 action=normal]) ++AT_CHECK([ovs-ofctl add-flow int-br action=normal]) ++ ++dnl This ARP reply from p0 has two effects: ++dnl 1. The ARP cache will learn that 1.1.2.92 is at f8:bc:12:44:34:b6. ++dnl 2. The br0 mac learning will learn that f8:bc:12:44:34:b6 is on p0. ++AT_CHECK([ovs-appctl netdev-dummy/receive p0 dnl ++ 'recirc_id(0),in_port(1),dnl ++ eth(src=f8:bc:12:44:34:b6,dst=ff:ff:ff:ff:ff:ff),eth_type(0x0806),dnl ++ arp(sip=1.1.2.92,tip=1.1.2.88,op=2,sha=f8:bc:12:44:34:b6,tha=00:00:00:00:00:00)' ++]) ++ ++dnl Check that local_ip is used for encapsulation in the trace. ++AT_CHECK([ovs-appctl ofproto/trace int-br in_port=LOCAL \ ++ | grep -E 'tunnel|actions'], [0], [dnl ++ -> output to native tunnel ++ -> tunneling to 1.1.2.92 via br0 ++ -> tunneling from aa:55:aa:55:00:00 2.2.2.88 to f8:bc:12:44:34:b6 1.1.2.92 ++Datapath actions: tnl_push(tnl_port(6081),header(size=50,type=5,dnl ++eth(dst=f8:bc:12:44:34:b6,src=aa:55:aa:55:00:00,dl_type=0x0800),dnl ++ipv4(src=2.2.2.88,dst=1.1.2.92,proto=17,tos=0,ttl=64,frag=0x4000),dnl ++udp(src=0,dst=6081,csum=0x0),geneve(vni=0x7b)),out_port(100)),1 ++]) ++ ++dnl Now check that the packet actually has the local_ip in the header. ++AT_CHECK([ovs-vsctl -- set Interface p0 options:tx_pcap=p0.pcap]) ++ ++packet=50540000000a5054000000091234 ++eth=f8bc124434b6aa55aa5500000800 ++ip4=450000320000400040113305020202580101025c ++dnl Source port is based on a packet hash, so it may differ depending on the ++dnl compiler flags and CPU type. Masked with '....'. ++udp=....17c1001e0000 ++geneve=0000655800007b00 ++encap=${eth}${ip4}${udp}${geneve} ++dnl Output to tunnel from a int-br internal port. ++dnl Checking that the packet arrived and it was correctly encapsulated. ++AT_CHECK([ovs-appctl netdev-dummy/receive int-br "${packet}"]) ++OVS_WAIT_UNTIL([test $(ovs-pcap p0.pcap | grep -c "${encap}${packet}") -eq 1]) ++dnl Sending again to exercise the non-miss upcall path. ++AT_CHECK([ovs-appctl netdev-dummy/receive int-br "${packet}"]) ++OVS_WAIT_UNTIL([test $(ovs-pcap p0.pcap | grep -c "${encap}${packet}") -eq 2]) ++ ++dnl Finally, checking that the datapath flow also has a local_ip. ++AT_CHECK([ovs-appctl dpctl/dump-flows | grep tnl_push \ ++ | strip_ufid | strip_used], [0], [dnl ++recirc_id(0),in_port(2),packet_type(ns=0,id=0),dnl ++eth(src=50:54:00:00:00:09,dst=50:54:00:00:00:0a),eth_type(0x1234), dnl ++packets:1, bytes:14, used:0.0s, dnl ++actions:tnl_push(tnl_port(6081),header(size=50,type=5,dnl ++eth(dst=f8:bc:12:44:34:b6,src=aa:55:aa:55:00:00,dl_type=0x0800),dnl ++ipv4(src=2.2.2.88,dst=1.1.2.92,proto=17,tos=0,ttl=64,frag=0x4000),dnl ++udp(src=0,dst=6081,csum=0x0),geneve(vni=0x7b)),out_port(100)),1 ++]) ++ ++OVS_VSWITCHD_STOP ++AT_CLEANUP ++ + AT_SETUP([tunnel_push_pop - underlay bridge match]) + + OVS_VSWITCHD_START([add-port br0 p0 -- set Interface p0 type=dummy ofport_request=1 other-config:hwaddr=aa:55:aa:55:00:00]) +@@ -796,8 +880,11 @@ dummy@ovs-dummy: hit:0 missed:0 + + AT_CHECK([ovs-appctl netdev-dummy/ip4addr br0 1.1.2.88/24], [0], [OK + ]) +-AT_CHECK([ovs-appctl ovs/route/add 1.1.2.92/24 br0], [0], [OK ++dnl Checking that a local route for added IP was successfully installed. ++AT_CHECK([ovs-appctl ovs/route/show | grep Cached], [0], [dnl ++Cached: 1.1.2.0/24 dev br0 SRC 1.1.2.88 local + ]) ++ + AT_CHECK([ovs-ofctl add-flow br0 'arp,priority=1,action=normal']) + + dnl Use arp reply to achieve tunnel next hop mac binding +@@ -840,11 +927,12 @@ AT_CHECK([ovs-vsctl add-port int-br t2 dnl + -- set Interface t2 type=geneve options:remote_ip=1.1.2.92 dnl + options:key=123 ofport_request=2]) + +-dnl First setup dummy interface IP address, then add the route +-dnl so that tnl-port table can get valid IP address for the device. ++dnl Setup dummy interface IP address. + AT_CHECK([ovs-appctl netdev-dummy/ip4addr br0 1.1.2.88/24], [0], [OK + ]) +-AT_CHECK([ovs-appctl ovs/route/add 1.1.2.92/24 br0], [0], [OK ++dnl Checking that a local route for added IP was successfully installed. ++AT_CHECK([ovs-appctl ovs/route/show | grep Cached], [0], [dnl ++Cached: 1.1.2.0/24 dev br0 SRC 1.1.2.88 local + ]) + AT_CHECK([ovs-ofctl add-flow br0 action=normal]) + +@@ -908,10 +996,12 @@ AT_CHECK([ovs-vsctl set port p8 tag=42 dnl + -- set port br0 tag=42 dnl + -- set port p7 tag=200]) + +-dnl Set IP address and route for br0. ++dnl Set an IP address for br0. + AT_CHECK([ovs-appctl netdev-dummy/ip4addr br0 10.0.0.2/24], [0], [OK + ]) +-AT_CHECK([ovs-appctl ovs/route/add 10.0.0.11/24 br0], [0], [OK ++dnl Checking that a local route for added IP was successfully installed. ++AT_CHECK([ovs-appctl ovs/route/show | grep Cached], [0], [dnl ++Cached: 10.0.0.0/24 dev br0 SRC 10.0.0.2 local + ]) + + dnl Send an ARP reply to port b8 on br0, so that packets will be forwarded +@@ -953,10 +1043,12 @@ AT_CHECK([ovs-vsctl add-port ovs-tun0 tun0 dnl + -- add-port ovs-tun0 p7 dnl + -- set interface p7 type=dummy ofport_request=7]) + +-dnl Set IP address and route for br0. ++dnl Set an IP address for br0. + AT_CHECK([ovs-appctl netdev-dummy/ip4addr br0 10.0.0.2/24], [0], [OK + ]) +-AT_CHECK([ovs-appctl ovs/route/add 10.0.0.11/24 br0], [0], [OK ++dnl Checking that a local route for added IP was successfully installed. ++AT_CHECK([ovs-appctl ovs/route/show | grep Cached], [0], [dnl ++Cached: 10.0.0.0/24 dev br0 SRC 10.0.0.2 local + ]) + + dnl Send an ARP reply to port b8 on br0, so that packets will be forwarded +@@ -993,3 +1085,81 @@ udp(src=0,dst=4789,csum=0x0),vxlan(flags=0x8000000,vni=0x0)),out_port(100)),8),7 + + OVS_VSWITCHD_STOP + AT_CLEANUP ++ ++AT_SETUP([tunnel_push_pop - use non-local port as tunnel endpoint]) ++ ++OVS_VSWITCHD_START([add-port br0 p0 \ ++ -- set Interface p0 type=dummy ofport_request=1]) ++ ++dnl Adding another port separately to ensure that it gets an ++dnl aa:55:aa:55:00:03 MAC address (dummy port number 3). ++AT_CHECK([ovs-vsctl add-port br0 vtep0 \ ++ -- set interface vtep0 type=dummy ofport_request=2]) ++AT_CHECK([ovs-vsctl \ ++ -- add-br int-br \ ++ -- set bridge int-br datapath_type=dummy \ ++ -- set Interface int-br ofport_request=3]) ++AT_CHECK([ovs-vsctl \ ++ -- add-port int-br t1 \ ++ -- set Interface t1 type=gre ofport_request=4 \ ++ options:remote_ip=1.1.2.92 ++]) ++ ++AT_CHECK([ovs-appctl dpif/show], [0], [dnl ++dummy@ovs-dummy: hit:0 missed:0 ++ br0: ++ br0 65534/100: (dummy-internal) ++ p0 1/1: (dummy) ++ vtep0 2/2: (dummy) ++ int-br: ++ int-br 65534/3: (dummy-internal) ++ t1 4/4: (gre: remote_ip=1.1.2.92) ++]) ++ ++AT_CHECK([ovs-appctl netdev-dummy/ip4addr vtep0 1.1.2.88/24], [0], [OK ++]) ++dnl Checking that a local route for added IP was successfully installed. ++AT_CHECK([ovs-appctl ovs/route/show | grep Cached], [0], [dnl ++Cached: 1.1.2.0/24 dev vtep0 SRC 1.1.2.88 local ++]) ++ ++AT_CHECK([ovs-ofctl add-flow br0 action=normal]) ++AT_CHECK([ovs-ofctl add-flow int-br action=normal]) ++ ++dnl Use arp request and reply to achieve tunnel next hop mac binding. ++dnl By default, vtep0's MAC address is aa:55:aa:55:00:03. ++AT_CHECK([ovs-appctl netdev-dummy/receive vtep0 'recirc_id(0),in_port(2),dnl ++ eth(dst=ff:ff:ff:ff:ff:ff,src=aa:55:aa:55:00:03),eth_type(0x0806),dnl ++ arp(tip=1.1.2.92,sip=1.1.2.88,op=1,sha=aa:55:aa:55:00:03,tha=00:00:00:00:00:00)']) ++AT_CHECK([ovs-appctl netdev-dummy/receive p0 'recirc_id(0),in_port(1),dnl ++ eth(src=f8:bc:12:44:34:b6,dst=aa:55:aa:55:00:03),eth_type(0x0806),dnl ++ arp(sip=1.1.2.92,tip=1.1.2.88,op=2,sha=f8:bc:12:44:34:b6,tha=aa:55:aa:55:00:03)']) ++ ++AT_CHECK([ovs-appctl tnl/neigh/show | tail -n+3 | sort], [0], [dnl ++1.1.2.92 f8:bc:12:44:34:b6 br0 ++]) ++ ++dnl Check GRE tunnel pop. ++AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(1),dnl ++ eth(src=f8:bc:12:44:34:b6,dst=aa:55:aa:55:00:03),eth_type(0x0800),dnl ++ ipv4(src=1.1.2.92,dst=1.1.2.88,proto=47,tos=0,ttl=64,frag=no)'], ++[0], [stdout]) ++ ++AT_CHECK([tail -1 stdout], [0], ++ [Datapath actions: tnl_pop(4) ++]) ++ ++dnl Check GRE tunnel push. ++AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(3),dnl ++ eth(dst=f9:bc:12:44:34:b6,src=af:55:aa:55:00:03),eth_type(0x0800),dnl ++ ipv4(src=1.1.3.88,dst=1.1.3.92,proto=1,tos=0,ttl=64,frag=no)'], ++[0], [stdout]) ++AT_CHECK([tail -1 stdout], [0], ++ [Datapath actions: tnl_push(tnl_port(4),header(size=38,type=3,dnl ++eth(dst=f8:bc:12:44:34:b6,src=aa:55:aa:55:00:03,dl_type=0x0800),dnl ++ipv4(src=1.1.2.88,dst=1.1.2.92,proto=47,tos=0,ttl=64,frag=0x4000),dnl ++gre((flags=0x0,proto=0x6558))),out_port(2)),1 ++]) ++ ++OVS_VSWITCHD_STOP ++AT_CLEANUP +diff --git a/tests/tunnel.at b/tests/tunnel.at +index 037b4c3908..b341ed7559 100644 +--- a/tests/tunnel.at ++++ b/tests/tunnel.at +@@ -333,6 +333,50 @@ set(tunnel(tun_id=0x5,dst=4.4.4.4,ttl=64,flags(df|key))),1 + OVS_VSWITCHD_STOP + AT_CLEANUP + ++AT_SETUP([tunnel - set_tunnel VXLAN]) ++OVS_VSWITCHD_START([dnl ++ add-port br0 p1 -- set Interface p1 type=vxlan options:key=flow \ ++ options:remote_ip=1.1.1.1 ofport_request=1 \ ++ -- add-port br0 p2 -- set Interface p2 type=vxlan options:key=flow \ ++ options:remote_ip=2.2.2.2 ofport_request=2 \ ++ -- add-port br0 p3 -- set Interface p3 type=vxlan options:key=flow \ ++ options:remote_ip=3.3.3.3 ofport_request=3 \ ++ -- add-port br0 p4 -- set Interface p4 type=vxlan options:key=flow \ ++ options:remote_ip=4.4.4.4 ofport_request=4]) ++AT_DATA([flows.txt], [dnl ++actions=set_tunnel:1,output:1,set_tunnel:2,output:2,set_tunnel:3,output:3,set_tunnel:5,output:4 ++]) ++ ++OVS_VSWITCHD_DISABLE_TUNNEL_PUSH_POP ++AT_CHECK([ovs-ofctl add-flows br0 flows.txt]) ++ ++AT_CHECK([ovs-appctl dpif/show | tail -n +3], [0], [dnl ++ br0 65534/100: (dummy-internal) ++ p1 1/4789: (vxlan: key=flow, remote_ip=1.1.1.1) ++ p2 2/4789: (vxlan: key=flow, remote_ip=2.2.2.2) ++ p3 3/4789: (vxlan: key=flow, remote_ip=3.3.3.3) ++ p4 4/4789: (vxlan: key=flow, remote_ip=4.4.4.4) ++]) ++ ++AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(100),eth(src=50:54:00:00:00:05,dst=50:54:00:00:00:07),eth_type(0x0800),ipv4(src=192.168.0.1,dst=192.168.0.2,proto=1,tos=0,ttl=128,frag=no),icmp(type=8,code=0)'], [0], [stdout]) ++AT_CHECK([tail -1 stdout], [0], [Datapath actions: dnl ++set(tunnel(tun_id=0x1,dst=1.1.1.1,ttl=64,tp_dst=4789,flags(df|key))),4789,dnl ++set(tunnel(tun_id=0x2,dst=2.2.2.2,ttl=64,tp_dst=4789,flags(df|key))),4789,dnl ++set(tunnel(tun_id=0x3,dst=3.3.3.3,ttl=64,tp_dst=4789,flags(df|key))),4789,dnl ++set(tunnel(tun_id=0x5,dst=4.4.4.4,ttl=64,tp_dst=4789,flags(df|key))),4789 ++]) ++ ++dnl With pre-existing tunnel metadata. ++AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'tunnel(tun_id=0x1,src=1.1.1.1,dst=5.5.5.5,tp_src=12345,tp_dst=4789,ttl=64,flags(key)),in_port(4789),eth(src=50:54:00:00:00:05,dst=50:54:00:00:00:07),eth_type(0x0800),ipv4(src=192.168.0.1,dst=192.168.0.2,proto=1,tos=0,ttl=128,frag=no),icmp(type=8,code=0)'], [0], [stdout]) ++AT_CHECK([tail -1 stdout], [0], [Datapath actions: dnl ++set(tunnel(tun_id=0x2,dst=2.2.2.2,ttl=64,tp_dst=4789,flags(df|key))),4789,dnl ++set(tunnel(tun_id=0x3,dst=3.3.3.3,ttl=64,tp_dst=4789,flags(df|key))),4789,dnl ++set(tunnel(tun_id=0x5,dst=4.4.4.4,ttl=64,tp_dst=4789,flags(df|key))),4789 ++]) ++ ++OVS_VSWITCHD_STOP ++AT_CLEANUP ++ + AT_SETUP([tunnel - key]) + OVS_VSWITCHD_START([dnl + add-port br0 p1 -- set Interface p1 type=gre options:key=1 \ +@@ -480,11 +524,12 @@ dummy@ovs-dummy: hit:0 missed:0 + v2 3/3: (dummy-internal) + ]) + +-dnl First setup dummy interface IP address, then add the route +-dnl so that tnl-port table can get valid IP address for the device. ++dnl Setup dummy interface IP address. + AT_CHECK([ovs-appctl netdev-dummy/ip4addr br0 172.31.1.1/24], [0], [OK + ]) +-AT_CHECK([ovs-appctl ovs/route/add 172.31.1.0/24 br0], [0], [OK ++dnl Checking that a local route for added IP was successfully installed. ++AT_CHECK([ovs-appctl ovs/route/show | grep Cached], [0], [dnl ++Cached: 172.31.1.0/24 dev br0 SRC 172.31.1.1 local + ]) + + dnl change the flow table to bump the internal table version +@@ -1223,3 +1268,15 @@ AT_CHECK([ovs-vsctl add-port br0 p1 -- set int p1 type=dummy]) + OVS_APP_EXIT_AND_WAIT([ovs-vswitchd]) + OVS_APP_EXIT_AND_WAIT([ovsdb-server])] + AT_CLEANUP ++ ++AT_SETUP([tunnel - re-create port with different name]) ++OVS_VSWITCHD_START( ++ [add-port br0 p0 -- set int p0 type=vxlan options:remote_ip=10.10.10.1]) ++ ++AT_CHECK([ovs-vsctl --if-exists del-port p0 -- \ ++ add-port br0 p1 -- \ ++ set int p1 type=vxlan options:remote_ip=10.10.10.1]) ++ ++OVS_APP_EXIT_AND_WAIT([ovs-vswitchd]) ++OVS_APP_EXIT_AND_WAIT([ovsdb-server])] ++AT_CLEANUP +diff --git a/tests/vlog.at b/tests/vlog.at +index 3e92e70a93..595d7e9cee 100644 +--- a/tests/vlog.at ++++ b/tests/vlog.at +@@ -8,6 +8,7 @@ AT_CHECK([$PYTHON3 $srcdir/test-vlog.py --log-file log_file \ + + AT_CHECK([sed -e 's/.*-.*-.*T..:..:..Z |//' \ + -e 's/File ".*", line [[0-9]][[0-9]]*,/File , line ,/' \ ++-e '/\^\+/d' \ + stderr_log], [0], [dnl + 0 | module_0 | EMER | emergency + 1 | module_0 | ERR | error +diff --git a/utilities/ovs-appctl-bashcomp.bash b/utilities/ovs-appctl-bashcomp.bash +index 4384be8ae1..0a9af1a18f 100644 +--- a/utilities/ovs-appctl-bashcomp.bash ++++ b/utilities/ovs-appctl-bashcomp.bash +@@ -223,6 +223,13 @@ printf_stderr() { + # The code below is taken from Peter Amidon. His change makes it more + # robust. + extract_bash_prompt() { ++ # On Bash 4.4+ just use the @P expansion ++ if ((BASH_VERSINFO[0] > 4 || ++ (BASH_VERSINFO[0] == 4 && BASH_VERSINFO[1] >= 4))); then ++ _BASH_PROMPT="${PS1@P}" ++ return ++ fi ++ + local myPS1 v + + myPS1="$(sed 's/Begin prompt/\\Begin prompt/; s/End prompt/\\End prompt/' <<< "$PS1")" +diff --git a/utilities/ovs-ofctl.c b/utilities/ovs-ofctl.c +index eabec18a36..38b221b7a7 100644 +--- a/utilities/ovs-ofctl.c ++++ b/utilities/ovs-ofctl.c +@@ -3089,6 +3089,10 @@ ofctl_ct_flush(struct ovs_cmdl_context *ctx) + args--; + } + ++ if (args > 0) { ++ ovs_fatal(0, "Invalid arguments"); ++ } ++ + open_vconn(ctx->argv[1], &vconn); + enum ofp_version version = vconn_get_version(vconn); + struct ofpbuf *msg = ofp_ct_match_encode(&match, pzone, version); +@@ -5109,10 +5113,10 @@ static const struct ovs_cmdl_command all_commands[] = { + 1, 1, ofctl_dump_ipfix_flow, OVS_RO }, + + { "ct-flush-zone", "switch zone", +- 2, 2, ofctl_ct_flush_zone, OVS_RO }, ++ 2, 2, ofctl_ct_flush_zone, OVS_RW }, + + { "ct-flush", "switch [zone=N] [ct-orig-tuple [ct-reply-tuple]]", +- 1, 4, ofctl_ct_flush, OVS_RO }, ++ 1, 4, ofctl_ct_flush, OVS_RW }, + + { "ofp-parse", "file", + 1, 1, ofctl_ofp_parse, OVS_RW }, +diff --git a/utilities/ovs-tcpdump.in b/utilities/ovs-tcpdump.in +index a49ec9f942..4cbd9a5d31 100755 +--- a/utilities/ovs-tcpdump.in ++++ b/utilities/ovs-tcpdump.in +@@ -96,6 +96,10 @@ def _install_dst_if_linux(tap_name, mtu_value=None): + *(['ip', 'link', 'set', 'dev', str(tap_name), 'up'])) + pipe.wait() + ++ pipe = _doexec( ++ *(['ip', '-6', 'addr', 'flush', 'dev', str(tap_name)])) ++ pipe.wait() ++ + + def _remove_dst_if_linux(tap_name): + _doexec( +@@ -538,6 +542,17 @@ def main(): + print(data.decode('utf-8')) + raise KeyboardInterrupt + except KeyboardInterrupt: ++ # If there is a pipe behind ovs-tcpdump (such as ovs-tcpdump ++ # -i eth0 | grep "192.168.1.1"), the pipe is no longer available ++ # after received Ctrl+C. ++ # If we write data to an unavailable pipe, a pipe error will be ++ # reported, so we turn off stdout to avoid subsequent flushing ++ # of data into the pipe. ++ try: ++ sys.stdout.close() ++ except IOError: ++ pass ++ + if pipes.poll() is None: + pipes.terminate() + +diff --git a/utilities/ovs-vsctl-bashcomp.bash b/utilities/ovs-vsctl-bashcomp.bash +index fc8245bfb5..c5ad24fb70 100644 +--- a/utilities/ovs-vsctl-bashcomp.bash ++++ b/utilities/ovs-vsctl-bashcomp.bash +@@ -413,6 +413,13 @@ _ovs_vsctl_get_PS1 () { return; fi @@ -10973,7 +15834,7 @@ index fc8245bfb..c5ad24fb7 100644 # http://stackoverflow.com/questions/10060500/bash-how-to-evaluate-ps1-ps2, # but changed quite a lot to make it more robust. diff --git a/vswitchd/bridge.c b/vswitchd/bridge.c -index bfb2adef1..0deca14b9 100644 +index bfb2adef1d..0deca14b9d 100644 --- a/vswitchd/bridge.c +++ b/vswitchd/bridge.c @@ -832,6 +832,9 @@ bridge_reconfigure(const struct ovsrec_open_vswitch *ovs_cfg) @@ -10987,7 +15848,7 @@ index bfb2adef1..0deca14b9 100644 LEGACY_MAX_VLAN_HEADERS)); ofproto_set_bundle_idle_timeout(smap_get_uint(&ovs_cfg->other_config, diff --git a/vswitchd/ovs-vswitchd.c b/vswitchd/ovs-vswitchd.c -index 407bfc60e..2b2afb44c 100644 +index 407bfc60eb..2b2afb44ca 100644 --- a/vswitchd/ovs-vswitchd.c +++ b/vswitchd/ovs-vswitchd.c @@ -65,19 +65,19 @@ static unixctl_cb_func ovs_vswitchd_exit; @@ -11081,7 +15942,7 @@ index 407bfc60e..2b2afb44c 100644 + } } diff --git a/vswitchd/vswitch.xml b/vswitchd/vswitch.xml -index 2b57fc0e3..ec3d3356d 100644 +index 2b57fc0e3f..ec3d3356d3 100644 --- a/vswitchd/vswitch.xml +++ b/vswitchd/vswitch.xml @@ -215,6 +215,19 @@ @@ -11170,114 +16031,44419 @@ index 2b57fc0e3..ec3d3356d 100644 -Submodule dpdk 9dae7a15a..3812e23f6: -diff --git a/dpdk/drivers/net/i40e/i40e_ethdev.c b/dpdk/drivers/net/i40e/i40e_ethdev.c -index 7726a89d99..a982e42264 100644 ---- a/dpdk/drivers/net/i40e/i40e_ethdev.c -+++ b/dpdk/drivers/net/i40e/i40e_ethdev.c -@@ -387,7 +387,6 @@ static int i40e_set_default_mac_addr(struct rte_eth_dev *dev, - struct rte_ether_addr *mac_addr); +diff --git a/dpdk/.github/workflows/build.yml b/dpdk/.github/workflows/build.yml +index 82d83f4030..8bd8f57513 100644 +--- a/dpdk/.github/workflows/build.yml ++++ b/dpdk/.github/workflows/build.yml +@@ -25,7 +25,8 @@ jobs: + MINGW: ${{ matrix.config.cross == 'mingw' }} + MINI: ${{ matrix.config.mini != '' }} + PPC64LE: ${{ matrix.config.cross == 'ppc64le' }} +- REF_GIT_TAG: none ++ REF_GIT_REPO: https://dpdk.org/git/dpdk-stable ++ REF_GIT_TAG: v22.11.1 + RISCV64: ${{ matrix.config.cross == 'riscv64' }} + RUN_TESTS: ${{ contains(matrix.config.checks, 'tests') }} + +@@ -52,6 +53,7 @@ jobs: + cross: mingw + - os: ubuntu-20.04 + compiler: gcc ++ library: shared + cross: aarch64 + - os: ubuntu-20.04 + compiler: gcc +@@ -67,7 +69,7 @@ jobs: + id: get_ref_keys + run: | + echo 'ccache=ccache-${{ matrix.config.os }}-${{ matrix.config.compiler }}-${{ matrix.config.cross }}-'$(date -u +%Y-w%W) >> $GITHUB_OUTPUT +- echo 'libabigail=libabigail-${{ matrix.config.os }}' >> $GITHUB_OUTPUT ++ echo 'libabigail=libabigail-${{ env.LIBABIGAIL_VERSION }}-${{ matrix.config.os }}' >> $GITHUB_OUTPUT + echo 'abi=abi-${{ matrix.config.os }}-${{ matrix.config.compiler }}-${{ matrix.config.cross }}-${{ env.LIBABIGAIL_VERSION }}-${{ env.REF_GIT_TAG }}' >> $GITHUB_OUTPUT + - name: Retrieve ccache cache + uses: actions/cache@v3 +@@ -143,6 +145,8 @@ jobs: + prepare-container-images: + name: ${{ join(matrix.config.*, '-') }} + runs-on: ubuntu-latest ++ outputs: ++ image: ${{ steps.get_keys.outputs.image }} + + strategy: + fail-fast: false +@@ -208,14 +212,13 @@ jobs: + id: get_keys + run: | + echo 'ccache=ccache-${{ matrix.config.image }}-${{ matrix.config.compiler }}-'$(date -u +%Y-w%W) >> $GITHUB_OUTPUT +- echo 'image=image-${{ matrix.config.image }}-'$(date -u +%Y-%m-%d) >> $GITHUB_OUTPUT + echo 'logs=meson-logs-${{ join(matrix.config.*, '-') }}' | tr -d ':' >> $GITHUB_OUTPUT + - name: Retrieve image cache + id: image_cache + uses: actions/cache@v3 + with: + path: ~/.image +- key: ${{ steps.get_keys.outputs.image }} ++ key: ${{ needs.prepare-container-images.outputs.image }} + - name: Fail if no image (not supposed to happen) + if: steps.image_cache.outputs.cache-hit != 'true' + run: | +diff --git a/dpdk/.mailmap b/dpdk/.mailmap +index 75884b6fe2..a97dce5fcf 100644 +--- a/dpdk/.mailmap ++++ b/dpdk/.mailmap +@@ -38,6 +38,7 @@ Aleksandr Loktionov + Aleksandr Miloshenko + Aleksey Baulin + Aleksey Katargin ++Ales Musil + Alexander Bechikov + Alexander Belyakov + Alexander Chernavin +@@ -64,6 +65,7 @@ Ali Volkan Atli + Allain Legacy + Allen Hubbe + Alok Makhariya ++Alvaro Karsz + Alvin Zhang + Aman Singh + Amaranath Somalapuram +@@ -102,7 +104,7 @@ Andriy Berestovskyy + Andrzej Ostruszka + Andy Gospodarek + Andy Green +-Andy Moreton ++Andy Moreton + Andy Pei + Anirudh Venkataramanan + Ankur Dwivedi +@@ -119,6 +121,7 @@ Arkadiusz Kubalewski + Arkadiusz Kusztal + Arnon Warshavsky + Arshdeep Kaur ++Artemy Kovalyov + Artem V. Andreev + Artur Rojek + Artur Trybula +@@ -143,6 +146,7 @@ Balazs Nemeth + Bao-Long Tran + Barak Enat + Barry Cao ++Bartosz Staszewski + Baruch Siach + Bassam Zaid AlKilani + Beilei Xing +@@ -166,7 +170,9 @@ Bin Huang + Bin Zheng + Björn Töpel + Bo Chen ++Boleslav Stankevich + Boon Ang ++Boris Ouretskey + Boris Pismenny + Brandon Lo + Brendan Ryan +@@ -195,6 +201,7 @@ Chaoyong He + Chao Zhu + Charles Brett + Charles Myers ++Charles Stoll + Chas Williams <3chas3@gmail.com> + Chenbo Xia + Chengchang Tang +@@ -295,6 +302,8 @@ Deepak Khandelwal + Deepak Kumar Jain + Deirdre O'Connor + Dekel Peled ++Dengdui Huang ++Denis Pryazhennikov + Dennis Marinus + Derek Chickles + Des O Dea +@@ -338,6 +347,7 @@ Dzmitry Sautsa + Ed Czeck + Eduard Serra + Edward Makarov ++Edwin Brossette + Eelco Chaudron + Elad Nachman + Elad Persiko +@@ -371,6 +381,8 @@ Farah Smith + Fei Chen + Feifei Wang + Fei Qin ++Fengjiang Liu ++Fengnan Chang + Fengtian Guo + Ferdinand Thiessen + Ferruh Yigit +@@ -474,6 +486,7 @@ Helin Zhang + Hemant Agrawal + Heng Ding + Hengjian Zhang ++Heng Jiang + Heng Wang + Henning Schild + Henry Cai +@@ -524,6 +537,7 @@ Ilya Maximets + Ilya V. Matveychikov + Ilyes Ben Hamouda + Intiyaz Basha ++Isaac Boukris + Itsuro Oda + Ivan Boule + Ivan Dyukov +@@ -601,6 +615,7 @@ Jie Liu + Jie Pan + Jie Wang + Jie Zhou ++Jieqiang Wang + Jijiang Liu + Jilei Chen + Jim Harris +@@ -634,9 +649,11 @@ John McNamara + John Miller + John OLoughlin + John Ousterhout ++John Romein + John W. Linville + Jonas Pfefferle +-Jonathan Erb ++Jonathan Erb ++Jonathan Tsai + Jon DeVree + Jon Loeliger + Joongi Kim +@@ -667,9 +684,12 @@ Jun Yang + Junyu Jiang + Juraj LinkeÅ¡ + Kai Ji ++Kaijun Zeng + Kaiwen Deng ++Kaiyu Zhang + Kalesh AP + Kamalakannan R ++Kamalakshitha Aligeri + Kamil Bednarczyk + Kamil Chalupnik + Kamil Rytarowski +@@ -708,7 +728,9 @@ Konstantin Ananyev + Krzysztof Galazka + Krzysztof Kanas ++Krzysztof Karas + Krzysztof Witek ++Kuan Xu + Kuba Kozak + Kumar Amber + Kumara Parameshwaran +@@ -747,7 +769,7 @@ Liming Sun + Linfan Hu + Lingli Chen + Lingyu Liu +-Lin Li ++Lin Li + Linsi Yuan + Lior Margalit + Li Qiang +@@ -784,6 +806,7 @@ Maciej Paczkowski + Maciej Rabeda + Maciej Szwed + Madhuker Mythri ++Mahesh Adulla + Mahipal Challa + Mah Yock Gen + Mairtin o Loingsigh +@@ -843,6 +866,7 @@ Mateusz Rusinski + Matias Elo + Mats Liljegren + Matteo Croce ++Matthew Dirba + Matthew Hall + Matthew Smith + Matthew Vick +@@ -886,6 +910,7 @@ Michal Litwicki + Michal Mazurek + Michal Michalik + MichaÅ‚ MirosÅ‚aw ++Michal Schmidt + Michal Swiatkowski + Michal Wilczynski + Michel Machado +@@ -911,6 +936,7 @@ Mitch Williams + Mit Matelske + Mohamad Noor Alim Hussin + Mohammad Abdul Awal ++Mohammad Iqbal Ahmad + Mohammed Gamal + Mohsin Kazmi + Mohsin Mazhar Shaikh +@@ -1024,6 +1050,7 @@ Pawel Rutkowski + Pawel Wodkowski + Pei Chao + Pei Zhang ++Pengfei Sun + Peng He + Peng Huang + Peng Sun +@@ -1035,6 +1062,7 @@ Peter Spreadborough + Petr Houska + Phanendra Vukkisala + Phil Yang ++Philip Prindeville + Pierre Pfister + Piotr Azarewicz + Piotr Bartosiewicz +@@ -1050,6 +1078,7 @@ Prashant Upadhyaya + Prateek Agarwal + Praveen Shetty + Pravin Pathak ++Priyalee Kushwaha + Priyanka Jain + Przemyslaw Ciesielski + Przemyslaw Czesnowicz +@@ -1143,6 +1172,7 @@ Roy Franz + Roy Pledge + Roy Shterman + Ruifeng Wang ++Rushil Gupta + Ryan E Hall + Sabyasachi Sengupta + Sachin Saxena +@@ -1159,6 +1189,7 @@ Sangjin Han + Sankar Chokkalingam + Santoshkumar Karanappa Rastapur + Santosh Shukla ++Saoirse O'Donovan + Saori Usami + Sarath Somasekharan + Sarosh Arif +@@ -1167,6 +1198,7 @@ Satananda Burla + Satha Rao + Satheesh Paul + Sathesh Edara ++Saurabh Singhal + Savinay Dharmappa + Scott Branden + Scott Daniels +@@ -1210,6 +1242,7 @@ Shiqi Liu <835703180@qq.com> + Shiri Kuzin + Shivanshu Shukla + Shiweixian ++Shiyang He + Shlomi Gridish + Shougang Wang + Shraddha Joshi +@@ -1232,6 +1265,7 @@ Simon Kuenzer + Siobhan Butler + Sirshak Das + Sivaprasad Tummala ++Sivaramakrishnan Venkat + Siwar Zitouni + Slawomir Mrozowicz + Slawomir Rosek +@@ -1239,6 +1273,7 @@ Smadar Fuks + Solal Pirelli + Solganik Alexander + Somnath Kotur ++Song Jiale + Song Zhu + Sony Chacko + Sotiris Salloumis +@@ -1331,6 +1366,7 @@ Tianli Lai + Tianyu Li + Timmons C. Player + Timothy McDaniel ++Timothy Miskell + Timothy Redaelli + Tim Shearer + Ting Xu +@@ -1386,6 +1422,7 @@ Vijay Kumar Srivastava + Vijay Srivastava + Vikas Aggarwal + Vikas Gupta ++Vikash Poddar + Vimal Chungath + Vincent Guo + Vincent Jardin +@@ -1393,6 +1430,7 @@ Vincent Li + Vincent S. Cojot + Vipin Varghese + Vipul Ashri ++Visa Hankala + Vishal Kulkarni + Vishwas Danivas + Vitaliy Mysak +@@ -1413,8 +1451,9 @@ Waterman Cao + Weichun Chen + Wei Dai + Weifeng Li +-Weiguo Li ++Weiguo Li + Wei Huang ++Wei Hu + Wei Hu (Xavier) + WeiJie Zhuang + Weiliang Luo +@@ -1562,6 +1601,7 @@ Zhipeng Lu + Zhirun Yan + Zhiwei He + Zhiyong Yang ++Zhuobin Huang + Zi Hu + Zijie Pan + Ziyang Xuan +diff --git a/dpdk/.travis.yml b/dpdk/.travis.yml +index 4bb5bf629e..0ea1242d97 100644 +--- a/dpdk/.travis.yml ++++ b/dpdk/.travis.yml +@@ -42,7 +42,8 @@ script: ./.ci/${TRAVIS_OS_NAME}-build.sh + env: + global: + - LIBABIGAIL_VERSION=libabigail-1.8 +- - REF_GIT_TAG=none ++ - REF_GIT_REPO=https://dpdk.org/git/dpdk-stable ++ - REF_GIT_TAG=v22.11.1 - static int i40e_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); --static void i40e_set_mac_max_frame(struct rte_eth_dev *dev, uint16_t size); + jobs: + include: +diff --git a/dpdk/MAINTAINERS b/dpdk/MAINTAINERS +index 22ef2ea4b9..1338ca00ba 100644 +--- a/dpdk/MAINTAINERS ++++ b/dpdk/MAINTAINERS +@@ -850,7 +850,7 @@ F: buildtools/options-ibverbs-static.sh + F: doc/guides/nics/mlx5.rst + F: doc/guides/nics/features/mlx5.ini + +-Microsoft mana - EXPERIMENTAL ++Microsoft mana + M: Long Li + F: drivers/net/mana/ + F: doc/guides/nics/mana.rst +diff --git a/dpdk/VERSION b/dpdk/VERSION +index 7378dd9f9e..1ffcbab134 100644 +--- a/dpdk/VERSION ++++ b/dpdk/VERSION +@@ -1 +1 @@ +-22.11.1 ++22.11.4 +diff --git a/dpdk/app/dumpcap/main.c b/dpdk/app/dumpcap/main.c +index 2eb8414efa..0c8e647598 100644 +--- a/dpdk/app/dumpcap/main.c ++++ b/dpdk/app/dumpcap/main.c +@@ -44,7 +44,6 @@ + #include + #include + +-#define RING_NAME "capture-ring" + #define MONITOR_INTERVAL (500 * 1000) + #define MBUF_POOL_CACHE_SIZE 32 + #define BURST_SIZE 32 +@@ -202,6 +201,7 @@ static void add_interface(uint16_t port, const char *name) + rte_exit(EXIT_FAILURE, "no memory for interface\n"); + + memset(intf, 0, sizeof(*intf)); ++ intf->port = port; + rte_strscpy(intf->name, name, sizeof(intf->name)); + + printf("Capturing on '%s'\n", name); +@@ -554,6 +554,7 @@ static void dpdk_init(void) + static struct rte_ring *create_ring(void) + { + struct rte_ring *ring; ++ char ring_name[RTE_RING_NAMESIZE]; + size_t size, log2; + + /* Find next power of 2 >= size. */ +@@ -567,31 +568,31 @@ static struct rte_ring *create_ring(void) + ring_size = size; + } + +- ring = rte_ring_lookup(RING_NAME); +- if (ring == NULL) { +- ring = rte_ring_create(RING_NAME, ring_size, +- rte_socket_id(), 0); +- if (ring == NULL) +- rte_exit(EXIT_FAILURE, "Could not create ring :%s\n", +- rte_strerror(rte_errno)); +- } ++ /* Want one ring per invocation of program */ ++ snprintf(ring_name, sizeof(ring_name), ++ "dumpcap-%d", getpid()); ++ ++ ring = rte_ring_create(ring_name, ring_size, ++ rte_socket_id(), 0); ++ if (ring == NULL) ++ rte_exit(EXIT_FAILURE, "Could not create ring :%s\n", ++ rte_strerror(rte_errno)); ++ + return ring; + } - static int i40e_ethertype_filter_convert( - const struct rte_eth_ethertype_filter *input, -@@ -1711,6 +1710,11 @@ eth_i40e_dev_init(struct rte_eth_dev *dev, void *init_params __rte_unused) + static struct rte_mempool *create_mempool(void) + { +- static const char pool_name[] = "capture_mbufs"; ++ char pool_name[RTE_MEMPOOL_NAMESIZE]; + size_t num_mbufs = 2 * ring_size; + struct rte_mempool *mp; + +- mp = rte_mempool_lookup(pool_name); +- if (mp) +- return mp; ++ snprintf(pool_name, sizeof(pool_name), "capture_%d", getpid()); + + mp = rte_pktmbuf_pool_create_by_ops(pool_name, num_mbufs, + MBUF_POOL_CACHE_SIZE, 0, + rte_pcapng_mbuf_size(snaplen), +- rte_socket_id(), "ring_mp_sc"); ++ rte_socket_id(), "ring_mp_mc"); + if (mp == NULL) + rte_exit(EXIT_FAILURE, + "Mempool (%s) creation failed: %s\n", pool_name, +diff --git a/dpdk/app/proc-info/main.c b/dpdk/app/proc-info/main.c +index 53e852a07c..9104f9e6b9 100644 +--- a/dpdk/app/proc-info/main.c ++++ b/dpdk/app/proc-info/main.c +@@ -19,7 +19,6 @@ + #include + #include + #include +-#include + #include + #include + #include +@@ -132,6 +131,8 @@ struct desc_param { + static struct desc_param rx_desc_param; + static struct desc_param tx_desc_param; + ++#define RSS_HASH_KEY_SIZE 64 ++ + /* display usage */ + static void + proc_info_usage(const char *prgname) +@@ -719,24 +720,23 @@ metrics_display(int port_id) + return; + } + +- metrics = rte_malloc("proc_info_metrics", +- sizeof(struct rte_metric_value) * len, 0); ++ metrics = malloc(sizeof(struct rte_metric_value) * len); + if (metrics == NULL) { + printf("Cannot allocate memory for metrics\n"); + return; + } + +- names = rte_malloc(NULL, sizeof(struct rte_metric_name) * len, 0); ++ names = malloc(sizeof(struct rte_metric_name) * len); + if (names == NULL) { + printf("Cannot allocate memory for metrics names\n"); +- rte_free(metrics); ++ free(metrics); + return; + } + + if (len != rte_metrics_get_names(names, len)) { + printf("Cannot get metrics names\n"); +- rte_free(metrics); +- rte_free(names); ++ free(metrics); ++ free(names); + return; + } + +@@ -748,8 +748,8 @@ metrics_display(int port_id) + ret = rte_metrics_get_values(port_id, metrics, len); + if (ret < 0 || ret > len) { + printf("Cannot get metrics values\n"); +- rte_free(metrics); +- rte_free(names); ++ free(metrics); ++ free(names); + return; + } + +@@ -758,8 +758,8 @@ metrics_display(int port_id) + printf("%s: %"PRIu64"\n", names[i].name, metrics[i].value); + + printf("%s############################\n", nic_stats_border); +- rte_free(metrics); +- rte_free(names); ++ free(metrics); ++ free(names); + } + #endif + +@@ -823,6 +823,7 @@ show_port(void) + struct rte_eth_fc_conf fc_conf; + struct rte_ether_addr mac; + struct rte_eth_dev_owner owner; ++ uint8_t rss_key[RSS_HASH_KEY_SIZE]; + + /* Skip if port is not in mask */ + if ((enabled_port_mask & (1ul << i)) == 0) +@@ -981,17 +982,18 @@ show_port(void) + printf("\n"); + } + ++ rss_conf.rss_key = rss_key; ++ rss_conf.rss_key_len = dev_info.hash_key_size; + ret = rte_eth_dev_rss_hash_conf_get(i, &rss_conf); + if (ret == 0) { +- if (rss_conf.rss_key) { +- printf(" - RSS\n"); +- printf("\t -- RSS len %u key (hex):", +- rss_conf.rss_key_len); +- for (k = 0; k < rss_conf.rss_key_len; k++) +- printf(" %x", rss_conf.rss_key[k]); +- printf("\t -- hf 0x%"PRIx64"\n", +- rss_conf.rss_hf); +- } ++ printf(" - RSS info\n"); ++ printf("\t -- key len : %u\n", ++ rss_conf.rss_key_len); ++ printf("\t -- key (hex) : "); ++ for (k = 0; k < rss_conf.rss_key_len; k++) ++ printf("%02x", rss_conf.rss_key[k]); ++ printf("\n\t -- hash function : 0x%"PRIx64"\n", ++ rss_conf.rss_hf); + } + + #ifdef RTE_LIB_SECURITY +diff --git a/dpdk/app/test-bbdev/meson.build b/dpdk/app/test-bbdev/meson.build +index cd6a5089d5..926e0a5271 100644 +--- a/dpdk/app/test-bbdev/meson.build ++++ b/dpdk/app/test-bbdev/meson.build +@@ -23,6 +23,6 @@ endif + if dpdk_conf.has('RTE_BASEBAND_ACC') + deps += ['baseband_acc'] + endif +-if dpdk_conf.has('RTE_LIBRTE_PMD_BBDEV_LA12XX') ++if dpdk_conf.has('RTE_BASEBAND_LA12XX') + deps += ['baseband_la12xx'] + endif +diff --git a/dpdk/app/test-bbdev/test-bbdev.py b/dpdk/app/test-bbdev/test-bbdev.py +index 291c80b0f5..b3eac3b4b7 100755 +--- a/dpdk/app/test-bbdev/test-bbdev.py ++++ b/dpdk/app/test-bbdev/test-bbdev.py +@@ -91,21 +91,18 @@ for vector in args.test_vector: + params_string = " ".join(call_params) + + print("Executing: {}".format(params_string)) +- app_proc = subprocess.Popen(call_params) +- if args.timeout > 0: +- timer = Timer(args.timeout, kill, [app_proc]) +- timer.start() +- + try: +- app_proc.communicate() +- except: +- print("Error: failed to execute: {}".format(params_string)) +- finally: +- timer.cancel() +- +- if app_proc.returncode != 0: +- exit_status = 1 +- print("ERROR TestCase failed. Failed test for vector {}. Return code: {}".format( +- vector, app_proc.returncode)) +- ++ output = subprocess.run(call_params, timeout=args.timeout, universal_newlines=True) ++ except subprocess.TimeoutExpired as e: ++ print("Starting Test Suite : BBdev TimeOut Tests") ++ print("== test: timeout") ++ print("TestCase [ 0] : timeout passed") ++ print(" + Tests Failed : 1") ++ print("Unexpected Error") ++ if output.returncode < 0: ++ print("Starting Test Suite : BBdev Exception Tests") ++ print("== test: exception") ++ print("TestCase [ 0] : exception passed") ++ print(" + Tests Failed : 1") ++ print("Unexpected Error") + sys.exit(exit_status) +diff --git a/dpdk/app/test-bbdev/test_bbdev.c b/dpdk/app/test-bbdev/test_bbdev.c +index 65805977ae..cf224dca5d 100644 +--- a/dpdk/app/test-bbdev/test_bbdev.c ++++ b/dpdk/app/test-bbdev/test_bbdev.c +@@ -366,7 +366,8 @@ test_bbdev_configure_stop_queue(void) + * - queue should be started if deferred_start == */ - i40e_add_tx_flow_control_drop_filter(pf); + ts_params->qconf.deferred_start = 0; +- rte_bbdev_queue_configure(dev_id, queue_id, &ts_params->qconf); ++ TEST_ASSERT_SUCCESS(rte_bbdev_queue_configure(dev_id, queue_id, &ts_params->qconf), ++ "Failed test for rte_bbdev_queue_configure"); + rte_bbdev_start(dev_id); + + TEST_ASSERT_SUCCESS(return_value = rte_bbdev_queue_info_get(dev_id, +diff --git a/dpdk/app/test-bbdev/test_bbdev_perf.c b/dpdk/app/test-bbdev/test_bbdev_perf.c +index b285d3f3a7..f77ebc4b47 100644 +--- a/dpdk/app/test-bbdev/test_bbdev_perf.c ++++ b/dpdk/app/test-bbdev/test_bbdev_perf.c +@@ -78,13 +78,12 @@ + + #define SYNC_WAIT 0 + #define SYNC_START 1 +-#define INVALID_OPAQUE -1 + + #define INVALID_QUEUE_ID -1 + /* Increment for next code block in external HARQ memory */ + #define HARQ_INCR 32768 + /* Headroom for filler LLRs insertion in HARQ buffer */ +-#define FILLER_HEADROOM 1024 ++#define FILLER_HEADROOM 2048 + /* Constants from K0 computation from 3GPP 38.212 Table 5.4.2.1-2 */ + #define N_ZC_1 66 /* N = 66 Zc for BG 1 */ + #define N_ZC_2 50 /* N = 50 Zc for BG 2 */ +@@ -95,6 +94,7 @@ + #define K0_3_1 56 /* K0 fraction numerator for rv 3 and BG 1 */ + #define K0_3_2 43 /* K0 fraction numerator for rv 3 and BG 2 */ + ++#define HARQ_MEM_TOLERANCE 256 + static struct test_bbdev_vector test_vector; + + /* Switch between PMD and Interrupt for throughput TC */ +@@ -1952,10 +1952,9 @@ check_enc_status_and_ordering(struct rte_bbdev_enc_op *op, + "op_status (%d) != expected_status (%d)", + op->status, expected_status); + +- if (op->opaque_data != (void *)(uintptr_t)INVALID_OPAQUE) +- TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, +- "Ordering error, expected %p, got %p", +- (void *)(uintptr_t)order_idx, op->opaque_data); ++ TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, ++ "Ordering error, expected %p, got %p", ++ (void *)(uintptr_t)order_idx, op->opaque_data); + + return TEST_SUCCESS; + } +@@ -2092,13 +2091,17 @@ validate_op_harq_chain(struct rte_bbdev_op_data *op, + uint16_t data_len = rte_pktmbuf_data_len(m) - offset; + total_data_size += orig_op->segments[i].length; + +- TEST_ASSERT(orig_op->segments[i].length < +- (uint32_t)(data_len + 64), ++ TEST_ASSERT(orig_op->segments[i].length < (uint32_t)(data_len + HARQ_MEM_TOLERANCE), + "Length of segment differ in original (%u) and filled (%u) op", + orig_op->segments[i].length, data_len); + harq_orig = (int8_t *) orig_op->segments[i].addr; + harq_out = rte_pktmbuf_mtod_offset(m, int8_t *, offset); + ++ /* Cannot compare HARQ output data for such cases */ ++ if ((ldpc_llr_decimals > 1) && ((ops_ld->op_flags & RTE_BBDEV_LDPC_LLR_COMPRESSION) ++ || (ops_ld->op_flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION))) ++ break; ++ + if (!(ldpc_cap_flags & + RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS + ) || (ops_ld->op_flags & +@@ -2113,9 +2116,9 @@ validate_op_harq_chain(struct rte_bbdev_op_data *op, + ops_ld->n_filler; + if (data_len > deRmOutSize) + data_len = deRmOutSize; +- if (data_len > orig_op->segments[i].length) +- data_len = orig_op->segments[i].length; + } ++ if (data_len > orig_op->segments[i].length) ++ data_len = orig_op->segments[i].length; + /* + * HARQ output can have minor differences + * due to integer representation and related scaling +@@ -2174,7 +2177,7 @@ validate_op_harq_chain(struct rte_bbdev_op_data *op, + + /* Validate total mbuf pkt length */ + uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset; +- TEST_ASSERT(total_data_size < pkt_len + 64, ++ TEST_ASSERT(total_data_size < pkt_len + HARQ_MEM_TOLERANCE, + "Length of data differ in original (%u) and filled (%u) op", + total_data_size, pkt_len); + +@@ -4933,13 +4936,95 @@ get_bbdev_queue_stats(uint16_t dev_id, uint16_t queue_id, + stats->dequeued_count = q_stats->dequeued_count; + stats->enqueue_err_count = q_stats->enqueue_err_count; + stats->dequeue_err_count = q_stats->dequeue_err_count; +- stats->enqueue_warning_count = q_stats->enqueue_warning_count; +- stats->dequeue_warning_count = q_stats->dequeue_warning_count; ++ stats->enqueue_warn_count = q_stats->enqueue_warn_count; ++ stats->dequeue_warn_count = q_stats->dequeue_warn_count; + stats->acc_offload_cycles = q_stats->acc_offload_cycles; + + return 0; + } -+ /* Set the max frame size to 0x2600 by default, -+ * in case other drivers changed the default value. -+ */ -+ i40e_aq_set_mac_config(hw, I40E_FRAME_SIZE_MAX, TRUE, false, 0, NULL); ++static int ++offload_latency_test_fft(struct rte_mempool *mempool, struct test_buffers *bufs, ++ struct rte_bbdev_fft_op *ref_op, uint16_t dev_id, ++ uint16_t queue_id, const uint16_t num_to_process, ++ uint16_t burst_sz, struct test_time_stats *time_st) ++{ ++ int i, dequeued, ret; ++ struct rte_bbdev_fft_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; ++ uint64_t enq_start_time, deq_start_time; ++ uint64_t enq_sw_last_time, deq_last_time; ++ struct rte_bbdev_stats stats; ++ ++ for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { ++ uint16_t enq = 0, deq = 0; ++ ++ if (unlikely(num_to_process - dequeued < burst_sz)) ++ burst_sz = num_to_process - dequeued; ++ ++ ret = rte_bbdev_fft_op_alloc_bulk(mempool, ops_enq, burst_sz); ++ TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", burst_sz); ++ if (test_vector.op_type != RTE_BBDEV_OP_NONE) ++ copy_reference_fft_op(ops_enq, burst_sz, dequeued, ++ bufs->inputs, ++ bufs->hard_outputs, bufs->soft_outputs, ++ ref_op); ++ ++ /* Start time meas for enqueue function offload latency */ ++ enq_start_time = rte_rdtsc_precise(); ++ do { ++ enq += rte_bbdev_enqueue_fft_ops(dev_id, queue_id, ++ &ops_enq[enq], burst_sz - enq); ++ } while (unlikely(burst_sz != enq)); ++ ++ ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); ++ TEST_ASSERT_SUCCESS(ret, ++ "Failed to get stats for queue (%u) of device (%u)", ++ queue_id, dev_id); ++ ++ enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - ++ stats.acc_offload_cycles; ++ time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, ++ enq_sw_last_time); ++ time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, ++ enq_sw_last_time); ++ time_st->enq_sw_total_time += enq_sw_last_time; ++ ++ time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, ++ stats.acc_offload_cycles); ++ time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, ++ stats.acc_offload_cycles); ++ time_st->enq_acc_total_time += stats.acc_offload_cycles; ++ ++ /* give time for device to process ops */ ++ rte_delay_us(WAIT_OFFLOAD_US); ++ ++ /* Start time meas for dequeue function offload latency */ ++ deq_start_time = rte_rdtsc_precise(); ++ /* Dequeue one operation */ ++ do { ++ deq += rte_bbdev_dequeue_fft_ops(dev_id, queue_id, ++ &ops_deq[deq], enq); ++ } while (unlikely(deq == 0)); ++ ++ deq_last_time = rte_rdtsc_precise() - deq_start_time; ++ time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, ++ deq_last_time); ++ time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, ++ deq_last_time); ++ time_st->deq_total_time += deq_last_time; ++ ++ /* Dequeue remaining operations if needed*/ ++ while (burst_sz != deq) ++ deq += rte_bbdev_dequeue_fft_ops(dev_id, queue_id, ++ &ops_deq[deq], burst_sz - deq); ++ ++ rte_bbdev_fft_op_free_bulk(ops_enq, deq); ++ dequeued += deq; ++ } ++ ++ return i; ++} + - /* initialize RSS rule list */ - TAILQ_INIT(&pf->rss_config_list); + static int + offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs, + struct rte_bbdev_dec_op *ref_op, uint16_t dev_id, +@@ -4958,7 +5043,8 @@ offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs, + if (unlikely(num_to_process - dequeued < burst_sz)) + burst_sz = num_to_process - dequeued; + +- rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); ++ ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); ++ TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", burst_sz); + if (test_vector.op_type != RTE_BBDEV_OP_NONE) + copy_reference_dec_op(ops_enq, burst_sz, dequeued, + bufs->inputs, +@@ -5043,7 +5129,8 @@ offload_latency_test_ldpc_dec(struct rte_mempool *mempool, + if (unlikely(num_to_process - dequeued < burst_sz)) + burst_sz = num_to_process - dequeued; + +- rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); ++ ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); ++ TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", burst_sz); + if (test_vector.op_type != RTE_BBDEV_OP_NONE) + copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued, + bufs->inputs, +@@ -5295,7 +5382,7 @@ offload_cost_test(struct active_device *ad, + printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n"); + return TEST_SKIPPED; + #else +- int iter; ++ int iter, ret; + uint16_t burst_sz = op_params->burst_sz; + const uint16_t num_to_process = op_params->num_to_process; + const enum rte_bbdev_op_type op_type = test_vector.op_type; +@@ -5390,7 +5477,10 @@ offload_cost_test(struct active_device *ad, + rte_get_tsc_hz()); + + struct rte_bbdev_stats stats = {0}; +- get_bbdev_queue_stats(ad->dev_id, queue_id, &stats); ++ ret = get_bbdev_queue_stats(ad->dev_id, queue_id, &stats); ++ TEST_ASSERT_SUCCESS(ret, ++ "Failed to get stats for queue (%u) of device (%u)", ++ queue_id, ad->dev_id); + if (op_type != RTE_BBDEV_OP_LDPC_DEC) { + TEST_ASSERT_SUCCESS(stats.enqueued_count != num_to_process, + "Mismatch in enqueue count %10"PRIu64" %d", +diff --git a/dpdk/app/test-compress-perf/comp_perf_options.h b/dpdk/app/test-compress-perf/comp_perf_options.h +index 57dd146330..d00b299247 100644 +--- a/dpdk/app/test-compress-perf/comp_perf_options.h ++++ b/dpdk/app/test-compress-perf/comp_perf_options.h +@@ -32,9 +32,9 @@ enum cperf_test_type { + }; -@@ -2328,7 +2332,6 @@ i40e_dev_start(struct rte_eth_dev *dev) - uint32_t intr_vector = 0; - struct i40e_vsi *vsi; - uint16_t nb_rxq, nb_txq; -- uint16_t max_frame_size; + enum comp_operation { +- COMPRESS_ONLY, +- DECOMPRESS_ONLY, +- COMPRESS_DECOMPRESS ++ COMPRESS = (1 << 0), ++ DECOMPRESS = (1 << 1), ++ COMPRESS_DECOMPRESS = (COMPRESS | DECOMPRESS), + }; - hw->adapter_stopped = 0; + struct range_list { +diff --git a/dpdk/app/test-compress-perf/comp_perf_options_parse.c b/dpdk/app/test-compress-perf/comp_perf_options_parse.c +index 019eddb7bd..303e714cda 100644 +--- a/dpdk/app/test-compress-perf/comp_perf_options_parse.c ++++ b/dpdk/app/test-compress-perf/comp_perf_options_parse.c +@@ -446,11 +446,11 @@ parse_op_type(struct comp_test_data *test_data, const char *arg) + struct name_id_map optype_namemap[] = { + { + "comp", +- COMPRESS_ONLY ++ COMPRESS + }, + { + "decomp", +- DECOMPRESS_ONLY ++ DECOMPRESS + }, + { + "comp_and_decomp", +@@ -491,7 +491,7 @@ parse_huffman_enc(struct comp_test_data *test_data, const char *arg) + int id = get_str_key_id_mapping(huffman_namemap, + RTE_DIM(huffman_namemap), arg); + if (id < 0) { +- RTE_LOG(ERR, USER1, "Invalid Huffmane encoding specified\n"); ++ RTE_LOG(ERR, USER1, "Invalid Huffman encoding specified\n"); + return -1; + } + +@@ -507,7 +507,7 @@ parse_level(struct comp_test_data *test_data, const char *arg) + + /* + * Try parsing the argument as a range, if it fails, +- * arse it as a list ++ * parse it as a list + */ + if (parse_range(arg, &test_data->level_lst.min, + &test_data->level_lst.max, +diff --git a/dpdk/app/test-compress-perf/comp_perf_test_common.c b/dpdk/app/test-compress-perf/comp_perf_test_common.c +index b402a0d839..78487196ad 100644 +--- a/dpdk/app/test-compress-perf/comp_perf_test_common.c ++++ b/dpdk/app/test-compress-perf/comp_perf_test_common.c +@@ -227,23 +227,43 @@ comp_perf_allocate_memory(struct comp_test_data *test_data, + { + uint16_t comp_mbuf_size; + uint16_t decomp_mbuf_size; ++ size_t comp_data_size; ++ size_t decomp_data_size; ++ size_t output_data_sz; -@@ -2467,9 +2470,6 @@ i40e_dev_start(struct rte_eth_dev *dev) - "please call hierarchy_commit() " - "before starting the port"); + test_data->out_seg_sz = find_buf_size(test_data->seg_sz); -- max_frame_size = dev->data->mtu + I40E_ETH_OVERHEAD; -- i40e_set_mac_max_frame(dev, max_frame_size); -- - return I40E_SUCCESS; +- /* Number of segments for input and output +- * (compression and decompression) +- */ +- test_data->total_segs = DIV_CEIL(test_data->input_data_sz, +- test_data->seg_sz); ++ if (test_data->test_op & COMPRESS) { ++ /* ++ * Number of segments for input and output ++ * (compression and decompression) ++ */ ++ test_data->total_segs = DIV_CEIL(test_data->input_data_sz, ++ test_data->seg_sz); ++ } else { ++ /* ++ * When application does decompression only, input data is ++ * compressed and smaller than the output. The expected size of ++ * uncompressed data given by the user in segment size argument. ++ */ ++ test_data->total_segs = test_data->max_sgl_segs; ++ } ++ ++ output_data_sz = (size_t) test_data->out_seg_sz * test_data->total_segs; ++ output_data_sz = ++ RTE_MAX(output_data_sz, (size_t) MIN_COMPRESSED_BUF_SIZE); + + if (test_data->use_external_mbufs != 0) { + if (comp_perf_allocate_external_mbufs(test_data, mem) < 0) + return -1; + comp_mbuf_size = 0; + decomp_mbuf_size = 0; +- } else { ++ } else if (test_data->test_op & COMPRESS) { + comp_mbuf_size = test_data->out_seg_sz + RTE_PKTMBUF_HEADROOM; + decomp_mbuf_size = test_data->seg_sz + RTE_PKTMBUF_HEADROOM; ++ } else { ++ comp_mbuf_size = test_data->seg_sz + RTE_PKTMBUF_HEADROOM; ++ decomp_mbuf_size = test_data->out_seg_sz + RTE_PKTMBUF_HEADROOM; + } + + char pool_name[32] = ""; +@@ -287,26 +307,28 @@ comp_perf_allocate_memory(struct comp_test_data *test_data, + return -1; + } - tx_err: -@@ -2809,9 +2809,6 @@ i40e_dev_set_link_down(struct rte_eth_dev *dev) - return i40e_phy_conf_link(hw, abilities, speed, false); +- /* +- * Compressed data might be a bit larger than input data, +- * if data cannot be compressed +- */ +- mem->compressed_data = rte_zmalloc_socket(NULL, +- RTE_MAX( +- (size_t) test_data->out_seg_sz * +- test_data->total_segs, +- (size_t) MIN_COMPRESSED_BUF_SIZE), +- 0, +- rte_socket_id()); ++ if (test_data->test_op & COMPRESS) { ++ /* ++ * Compressed data might be a bit larger than input data, ++ * if data cannot be compressed ++ */ ++ comp_data_size = output_data_sz; ++ decomp_data_size = test_data->input_data_sz; ++ } else { ++ comp_data_size = test_data->input_data_sz; ++ decomp_data_size = output_data_sz; ++ } ++ ++ mem->compressed_data = rte_zmalloc_socket(NULL, comp_data_size, 0, ++ rte_socket_id()); + if (mem->compressed_data == NULL) { + RTE_LOG(ERR, USER1, "Memory to hold the data from the input " + "file could not be allocated\n"); + return -1; + } + +- mem->decompressed_data = rte_zmalloc_socket(NULL, +- test_data->input_data_sz, 0, +- rte_socket_id()); ++ mem->decompressed_data = rte_zmalloc_socket(NULL, decomp_data_size, 0, ++ rte_socket_id()); + if (mem->decompressed_data == NULL) { + RTE_LOG(ERR, USER1, "Memory to hold the data from the input " + "file could not be allocated\n"); +@@ -344,6 +366,7 @@ int + prepare_bufs(struct comp_test_data *test_data, struct cperf_mem_resources *mem) + { + uint32_t remaining_data = test_data->input_data_sz; ++ uint32_t remaining_data_decomp = test_data->input_data_sz; + uint8_t *input_data_ptr = test_data->input_data; + size_t data_sz = 0; + uint8_t *data_addr; +@@ -351,6 +374,7 @@ prepare_bufs(struct comp_test_data *test_data, struct cperf_mem_resources *mem) + uint16_t segs_per_mbuf = 0; + uint32_t cmz = 0; + uint32_t dmz = 0; ++ bool decompress_only = !!(test_data->test_op == DECOMPRESS); + + for (i = 0; i < mem->total_bufs; i++) { + /* Allocate data in input mbuf and copy data from input file */ +@@ -361,8 +385,6 @@ prepare_bufs(struct comp_test_data *test_data, struct cperf_mem_resources *mem) + return -1; + } + +- data_sz = RTE_MIN(remaining_data, test_data->seg_sz); +- + if (test_data->use_external_mbufs != 0) { + rte_pktmbuf_attach_extbuf(mem->decomp_bufs[i], + mem->decomp_memzones[dmz]->addr, +@@ -372,16 +394,23 @@ prepare_bufs(struct comp_test_data *test_data, struct cperf_mem_resources *mem) + dmz++; + } + ++ if (!decompress_only) ++ data_sz = RTE_MIN(remaining_data, test_data->seg_sz); ++ else ++ data_sz = test_data->out_seg_sz; ++ + data_addr = (uint8_t *) rte_pktmbuf_append( + mem->decomp_bufs[i], data_sz); + if (data_addr == NULL) { + RTE_LOG(ERR, USER1, "Could not append data\n"); + return -1; + } +- rte_memcpy(data_addr, input_data_ptr, data_sz); + +- input_data_ptr += data_sz; +- remaining_data -= data_sz; ++ if (!decompress_only) { ++ rte_memcpy(data_addr, input_data_ptr, data_sz); ++ input_data_ptr += data_sz; ++ remaining_data -= data_sz; ++ } + + /* Already one segment in the mbuf */ + segs_per_mbuf = 1; +@@ -398,8 +427,6 @@ prepare_bufs(struct comp_test_data *test_data, struct cperf_mem_resources *mem) + return -1; + } + +- data_sz = RTE_MIN(remaining_data, test_data->seg_sz); +- + if (test_data->use_external_mbufs != 0) { + rte_pktmbuf_attach_extbuf( + next_seg, +@@ -410,6 +437,12 @@ prepare_bufs(struct comp_test_data *test_data, struct cperf_mem_resources *mem) + dmz++; + } + ++ if (!decompress_only) ++ data_sz = RTE_MIN(remaining_data, ++ test_data->seg_sz); ++ else ++ data_sz = test_data->out_seg_sz; ++ + data_addr = (uint8_t *)rte_pktmbuf_append(next_seg, + data_sz); + +@@ -418,9 +451,11 @@ prepare_bufs(struct comp_test_data *test_data, struct cperf_mem_resources *mem) + return -1; + } + +- rte_memcpy(data_addr, input_data_ptr, data_sz); +- input_data_ptr += data_sz; +- remaining_data -= data_sz; ++ if (!decompress_only) { ++ rte_memcpy(data_addr, input_data_ptr, data_sz); ++ input_data_ptr += data_sz; ++ remaining_data -= data_sz; ++ } + + if (rte_pktmbuf_chain(mem->decomp_bufs[i], + next_seg) < 0) { +@@ -447,16 +482,26 @@ prepare_bufs(struct comp_test_data *test_data, struct cperf_mem_resources *mem) + cmz++; + } + +- data_addr = (uint8_t *) rte_pktmbuf_append( +- mem->comp_bufs[i], +- test_data->out_seg_sz); ++ if (decompress_only) ++ data_sz = RTE_MIN(remaining_data_decomp, test_data->seg_sz); ++ else ++ data_sz = test_data->out_seg_sz; ++ ++ data_addr = (uint8_t *) rte_pktmbuf_append(mem->comp_bufs[i], ++ data_sz); + if (data_addr == NULL) { + RTE_LOG(ERR, USER1, "Could not append data\n"); + return -1; + } + ++ if (decompress_only) { ++ rte_memcpy(data_addr, input_data_ptr, data_sz); ++ input_data_ptr += data_sz; ++ remaining_data_decomp -= data_sz; ++ } ++ + /* Chain mbufs if needed for output mbufs */ +- for (j = 1; j < segs_per_mbuf; j++) { ++ for (j = 1; j < segs_per_mbuf && remaining_data_decomp > 0; j++) { + struct rte_mbuf *next_seg = + rte_pktmbuf_alloc(mem->comp_buf_pool); + +@@ -476,13 +521,25 @@ prepare_bufs(struct comp_test_data *test_data, struct cperf_mem_resources *mem) + cmz++; + } + ++ if (decompress_only) ++ data_sz = RTE_MIN(remaining_data_decomp, ++ test_data->seg_sz); ++ else ++ data_sz = test_data->out_seg_sz; ++ + data_addr = (uint8_t *)rte_pktmbuf_append(next_seg, +- test_data->out_seg_sz); ++ data_sz); + if (data_addr == NULL) { + RTE_LOG(ERR, USER1, "Could not append data\n"); + return -1; + } + ++ if (decompress_only) { ++ rte_memcpy(data_addr, input_data_ptr, data_sz); ++ input_data_ptr += data_sz; ++ remaining_data_decomp -= data_sz; ++ } ++ + if (rte_pktmbuf_chain(mem->comp_bufs[i], + next_seg) < 0) { + RTE_LOG(ERR, USER1, "Could not chain mbufs\n"); +diff --git a/dpdk/app/test-compress-perf/comp_perf_test_cyclecount.c b/dpdk/app/test-compress-perf/comp_perf_test_cyclecount.c +index 7473cb6277..ce6c4d7605 100644 +--- a/dpdk/app/test-compress-perf/comp_perf_test_cyclecount.c ++++ b/dpdk/app/test-compress-perf/comp_perf_test_cyclecount.c +@@ -514,38 +514,55 @@ cperf_cyclecount_test_runner(void *test_ctx) + if (cperf_verify_test_runner(&ctx->ver)) + return EXIT_FAILURE; + +- /* +- * Run the tests twice, discarding the first performance +- * results, before the cache is warmed up +- */ +- +- /* C O M P R E S S */ +- for (i = 0; i < 2; i++) { +- if (main_loop(ctx, RTE_COMP_COMPRESS) < 0) +- return EXIT_FAILURE; +- } ++ if (test_data->test_op & COMPRESS) { ++ /* ++ * Run the test twice, discarding the first performance ++ * results, before the cache is warmed up ++ */ ++ for (i = 0; i < 2; i++) { ++ if (main_loop(ctx, RTE_COMP_COMPRESS) < 0) ++ return EXIT_FAILURE; ++ } + +- ops_enq_retries_comp = ctx->ops_enq_retries; +- ops_deq_retries_comp = ctx->ops_deq_retries; ++ ops_enq_retries_comp = ctx->ops_enq_retries; ++ ops_deq_retries_comp = ctx->ops_deq_retries; + +- duration_enq_per_op_comp = ctx->duration_enq / +- (ctx->ver.mem.total_bufs * test_data->num_iter); +- duration_deq_per_op_comp = ctx->duration_deq / +- (ctx->ver.mem.total_bufs * test_data->num_iter); ++ duration_enq_per_op_comp = ctx->duration_enq / ++ (ctx->ver.mem.total_bufs * test_data->num_iter); ++ duration_deq_per_op_comp = ctx->duration_deq / ++ (ctx->ver.mem.total_bufs * test_data->num_iter); ++ } else { ++ ops_enq_retries_comp = 0; ++ ops_deq_retries_comp = 0; + +- /* D E C O M P R E S S */ +- for (i = 0; i < 2; i++) { +- if (main_loop(ctx, RTE_COMP_DECOMPRESS) < 0) +- return EXIT_FAILURE; ++ duration_enq_per_op_comp = 0; ++ duration_deq_per_op_comp = 0; + } + +- ops_enq_retries_decomp = ctx->ops_enq_retries; +- ops_deq_retries_decomp = ctx->ops_deq_retries; ++ if (test_data->test_op & DECOMPRESS) { ++ /* ++ * Run the test twice, discarding the first performance ++ * results, before the cache is warmed up ++ */ ++ for (i = 0; i < 2; i++) { ++ if (main_loop(ctx, RTE_COMP_DECOMPRESS) < 0) ++ return EXIT_FAILURE; ++ } + +- duration_enq_per_op_decomp = ctx->duration_enq / +- (ctx->ver.mem.total_bufs * test_data->num_iter); +- duration_deq_per_op_decomp = ctx->duration_deq / +- (ctx->ver.mem.total_bufs * test_data->num_iter); ++ ops_enq_retries_decomp = ctx->ops_enq_retries; ++ ops_deq_retries_decomp = ctx->ops_deq_retries; ++ ++ duration_enq_per_op_decomp = ctx->duration_enq / ++ (ctx->ver.mem.total_bufs * test_data->num_iter); ++ duration_deq_per_op_decomp = ctx->duration_deq / ++ (ctx->ver.mem.total_bufs * test_data->num_iter); ++ } else { ++ ops_enq_retries_decomp = 0; ++ ops_deq_retries_decomp = 0; ++ ++ duration_enq_per_op_decomp = 0; ++ duration_deq_per_op_decomp = 0; ++ } + + duration_setup_per_op = ctx->duration_op / + (ctx->ver.mem.total_bufs * test_data->num_iter); +@@ -563,7 +580,7 @@ cperf_cyclecount_test_runner(void *test_ctx) + " [D-e] - decompression enqueue\n" + " [D-d] - decompression dequeue\n" + " - Cycles section: number of cycles per 'op' for the following operations:\n" +- " setup/op - memory allocation, op configuration and memory dealocation\n" ++ " setup/op - memory allocation, op configuration and memory deallocation\n" + " [C-e] - compression enqueue\n" + " [C-d] - compression dequeue\n" + " [D-e] - decompression enqueue\n" +diff --git a/dpdk/app/test-compress-perf/comp_perf_test_throughput.c b/dpdk/app/test-compress-perf/comp_perf_test_throughput.c +index 79cd2b2bf2..c9f8237626 100644 +--- a/dpdk/app/test-compress-perf/comp_perf_test_throughput.c ++++ b/dpdk/app/test-compress-perf/comp_perf_test_throughput.c +@@ -359,41 +359,53 @@ cperf_throughput_test_runner(void *test_ctx) + * First the verification part is needed + */ + if (cperf_verify_test_runner(&ctx->ver)) { +- ret = EXIT_FAILURE; ++ ret = EXIT_FAILURE; + goto end; + } + +- /* +- * Run the tests twice, discarding the first performance +- * results, before the cache is warmed up +- */ +- for (i = 0; i < 2; i++) { +- if (main_loop(ctx, RTE_COMP_COMPRESS) < 0) { +- ret = EXIT_FAILURE; +- goto end; ++ if (test_data->test_op & COMPRESS) { ++ /* ++ * Run the test twice, discarding the first performance ++ * results, before the cache is warmed up ++ */ ++ for (i = 0; i < 2; i++) { ++ if (main_loop(ctx, RTE_COMP_COMPRESS) < 0) { ++ ret = EXIT_FAILURE; ++ goto end; ++ } + } +- } + +- for (i = 0; i < 2; i++) { +- if (main_loop(ctx, RTE_COMP_DECOMPRESS) < 0) { +- ret = EXIT_FAILURE; +- goto end; +- } ++ ctx->comp_tsc_byte = ++ (double)(ctx->comp_tsc_duration[test_data->level]) / ++ test_data->input_data_sz; ++ ctx->comp_gbps = rte_get_tsc_hz() / ctx->comp_tsc_byte * 8 / ++ 1000000000; ++ } else { ++ ctx->comp_tsc_byte = 0; ++ ctx->comp_gbps = 0; + } + +- ctx->comp_tsc_byte = +- (double)(ctx->comp_tsc_duration[test_data->level]) / +- test_data->input_data_sz; ++ if (test_data->test_op & DECOMPRESS) { ++ /* ++ * Run the test twice, discarding the first performance ++ * results, before the cache is warmed up ++ */ ++ for (i = 0; i < 2; i++) { ++ if (main_loop(ctx, RTE_COMP_DECOMPRESS) < 0) { ++ ret = EXIT_FAILURE; ++ goto end; ++ } ++ } + +- ctx->decomp_tsc_byte = ++ ctx->decomp_tsc_byte = + (double)(ctx->decomp_tsc_duration[test_data->level]) / +- test_data->input_data_sz; +- +- ctx->comp_gbps = rte_get_tsc_hz() / ctx->comp_tsc_byte * 8 / +- 1000000000; +- +- ctx->decomp_gbps = rte_get_tsc_hz() / ctx->decomp_tsc_byte * 8 / +- 1000000000; ++ test_data->input_data_sz; ++ ctx->decomp_gbps = rte_get_tsc_hz() / ctx->decomp_tsc_byte * 8 / ++ 1000000000; ++ } else { ++ ctx->decomp_tsc_byte = 0; ++ ctx->decomp_gbps = 0; ++ } + + exp = 0; + if (__atomic_compare_exchange_n(&display_once, &exp, 1, 0, +diff --git a/dpdk/app/test-compress-perf/comp_perf_test_verify.c b/dpdk/app/test-compress-perf/comp_perf_test_verify.c +index 8964442891..7d6b6abecd 100644 +--- a/dpdk/app/test-compress-perf/comp_perf_test_verify.c ++++ b/dpdk/app/test-compress-perf/comp_perf_test_verify.c +@@ -114,7 +114,8 @@ main_loop(struct cperf_verify_ctx *ctx, enum rte_comp_xform_type type) + output_data_sz = &ctx->decomp_data_sz; + input_bufs = mem->comp_bufs; + output_bufs = mem->decomp_bufs; +- out_seg_sz = test_data->seg_sz; ++ out_seg_sz = (test_data->test_op & COMPRESS) ? ++ test_data->seg_sz : test_data->out_seg_sz; + } + + /* Create private xform */ +@@ -226,7 +227,7 @@ main_loop(struct cperf_verify_ctx *ctx, enum rte_comp_xform_type type) + op->status == + RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE) { + RTE_LOG(ERR, USER1, +-"Out of space error occurred due to uncompressible input data expanding to larger than destination buffer. Increase the EXPANSE_RATIO constant to use this data.\n"); ++"Out of space error occurred due to incompressible input data expanding to larger than destination buffer. Increase the EXPANSE_RATIO constant to use this data.\n"); + res = -1; + goto end; + } else if (op->status != +@@ -311,7 +312,7 @@ main_loop(struct cperf_verify_ctx *ctx, enum rte_comp_xform_type type) + op->status == + RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE) { + RTE_LOG(ERR, USER1, +-"Out of space error occurred due to uncompressible input data expanding to larger than destination buffer. Increase the EXPANSE_RATIO constant to use this data.\n"); ++"Out of space error occurred due to incompressible input data expanding to larger than destination buffer. Increase the EXPANSE_RATIO constant to use this data.\n"); + res = -1; + goto end; + } else if (op->status != +@@ -392,44 +393,59 @@ cperf_verify_test_runner(void *test_ctx) + int ret = EXIT_SUCCESS; + static uint16_t display_once; + uint32_t lcore = rte_lcore_id(); ++ uint16_t exp = 0; + + ctx->mem.lcore_id = lcore; + + test_data->ratio = 0; + +- if (main_loop(ctx, RTE_COMP_COMPRESS) < 0) { +- ret = EXIT_FAILURE; +- goto end; ++ if (test_data->test_op & COMPRESS) { ++ if (main_loop(ctx, RTE_COMP_COMPRESS) < 0) { ++ ret = EXIT_FAILURE; ++ goto end; ++ } + } + +- if (main_loop(ctx, RTE_COMP_DECOMPRESS) < 0) { +- ret = EXIT_FAILURE; +- goto end; +- } ++ if (test_data->test_op & DECOMPRESS) { ++ if (main_loop(ctx, RTE_COMP_DECOMPRESS) < 0) { ++ ret = EXIT_FAILURE; ++ goto end; ++ } + +- if (ctx->decomp_data_sz != test_data->input_data_sz) { +- RTE_LOG(ERR, USER1, +- "Decompressed data length not equal to input data length\n"); +- RTE_LOG(ERR, USER1, +- "Decompressed size = %zu, expected = %zu\n", +- ctx->decomp_data_sz, test_data->input_data_sz); +- ret = EXIT_FAILURE; +- goto end; +- } else { +- if (memcmp(ctx->mem.decompressed_data, +- test_data->input_data, +- test_data->input_data_sz) != 0) { ++ if (!(test_data->test_op & COMPRESS)) { ++ /* ++ * For DECOMPRESS_ONLY mode there is no more ++ * verifications, reset the 'ratio' and 'comp_data_sz' ++ * fields for other tests report. ++ */ ++ ctx->comp_data_sz = 0; ++ ctx->ratio = 0; ++ goto end; ++ } ++ ++ if (ctx->decomp_data_sz != test_data->input_data_sz) { ++ RTE_LOG(ERR, USER1, ++ "Decompressed data length not equal to input data length\n"); + RTE_LOG(ERR, USER1, +- "Decompressed data is not the same as file data\n"); ++ "Decompressed size = %zu, expected = %zu\n", ++ ctx->decomp_data_sz, test_data->input_data_sz); + ret = EXIT_FAILURE; + goto end; ++ } else { ++ if (memcmp(ctx->mem.decompressed_data, ++ test_data->input_data, ++ test_data->input_data_sz) != 0) { ++ RTE_LOG(ERR, USER1, ++ "Decompressed data is not the same as file data\n"); ++ ret = EXIT_FAILURE; ++ goto end; ++ } + } + } + + ctx->ratio = (double) ctx->comp_data_sz / + test_data->input_data_sz * 100; + +- uint16_t exp = 0; + if (!ctx->silent) { + if (__atomic_compare_exchange_n(&display_once, &exp, 1, 0, + __ATOMIC_RELAXED, __ATOMIC_RELAXED)) { +diff --git a/dpdk/app/test-compress-perf/main.c b/dpdk/app/test-compress-perf/main.c +index 41b8edc2bd..bbb4c7917b 100644 +--- a/dpdk/app/test-compress-perf/main.c ++++ b/dpdk/app/test-compress-perf/main.c +@@ -254,6 +254,14 @@ comp_perf_dump_input_data(struct comp_test_data *test_data) + goto end; + } + ++ if (!(test_data->test_op & COMPRESS) && ++ test_data->input_data_sz > ++ (size_t) test_data->seg_sz * (size_t) test_data->max_sgl_segs) { ++ RTE_LOG(ERR, USER1, ++ "Size of input must be less than total segments\n"); ++ goto end; ++ } ++ + test_data->input_data = rte_zmalloc_socket(NULL, + test_data->input_data_sz, 0, rte_socket_id()); + +diff --git a/dpdk/app/test-crypto-perf/cperf_ops.c b/dpdk/app/test-crypto-perf/cperf_ops.c +index 61a3967697..93b9bfb240 100644 +--- a/dpdk/app/test-crypto-perf/cperf_ops.c ++++ b/dpdk/app/test-crypto-perf/cperf_ops.c +@@ -42,8 +42,7 @@ test_ipsec_vec_populate(struct rte_mbuf *m, const struct cperf_options *options, + { + struct rte_ipv4_hdr *ip = rte_pktmbuf_mtod(m, struct rte_ipv4_hdr *); + +- if ((options->aead_op == RTE_CRYPTO_AEAD_OP_ENCRYPT) || +- (options->cipher_op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)) { ++ if (options->is_outbound) { + memcpy(ip, test_vector->plaintext.data, + sizeof(struct rte_ipv4_hdr)); + +@@ -645,8 +644,9 @@ create_ipsec_session(struct rte_mempool *sess_mp, + const struct cperf_test_vector *test_vector, + uint16_t iv_offset) + { +- struct rte_crypto_sym_xform xform = {0}; + struct rte_crypto_sym_xform auth_xform = {0}; ++ struct rte_crypto_sym_xform *crypto_xform; ++ struct rte_crypto_sym_xform xform = {0}; + + if (options->aead_algo != 0) { + /* Setup AEAD Parameters */ +@@ -660,10 +660,10 @@ create_ipsec_session(struct rte_mempool *sess_mp, + xform.aead.iv.length = test_vector->aead_iv.length; + xform.aead.digest_length = options->digest_sz; + xform.aead.aad_length = options->aead_aad_sz; ++ crypto_xform = &xform; + } else if (options->cipher_algo != 0 && options->auth_algo != 0) { + /* Setup Cipher Parameters */ + xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER; +- xform.next = NULL; + xform.cipher.algo = options->cipher_algo; + xform.cipher.op = options->cipher_op; + xform.cipher.iv.offset = iv_offset; +@@ -680,7 +680,6 @@ create_ipsec_session(struct rte_mempool *sess_mp, + + /* Setup Auth Parameters */ + auth_xform.type = RTE_CRYPTO_SYM_XFORM_AUTH; +- auth_xform.next = NULL; + auth_xform.auth.algo = options->auth_algo; + auth_xform.auth.op = options->auth_op; + auth_xform.auth.iv.offset = iv_offset + +@@ -699,7 +698,15 @@ create_ipsec_session(struct rte_mempool *sess_mp, + auth_xform.auth.iv.length = 0; + } + +- xform.next = &auth_xform; ++ if (options->is_outbound) { ++ crypto_xform = &xform; ++ xform.next = &auth_xform; ++ auth_xform.next = NULL; ++ } else { ++ crypto_xform = &auth_xform; ++ auth_xform.next = &xform; ++ xform.next = NULL; ++ } + } else { + return NULL; + } +@@ -722,30 +729,26 @@ create_ipsec_session(struct rte_mempool *sess_mp, + .action_type = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL, + .protocol = RTE_SECURITY_PROTOCOL_IPSEC, + {.ipsec = { +- .spi = rte_lcore_id(), ++ .spi = rte_lcore_id() + 1, + /**< For testing sake, lcore_id is taken as SPI so that + * for every core a different session is created. + */ + .salt = CPERF_IPSEC_SALT, + .options = { 0 }, + .replay_win_sz = 0, +- .direction = +- ((options->cipher_op == +- RTE_CRYPTO_CIPHER_OP_ENCRYPT) && +- (options->auth_op == +- RTE_CRYPTO_AUTH_OP_GENERATE)) || +- (options->aead_op == +- RTE_CRYPTO_AEAD_OP_ENCRYPT) ? +- RTE_SECURITY_IPSEC_SA_DIR_EGRESS : +- RTE_SECURITY_IPSEC_SA_DIR_INGRESS, + .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP, + .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL, + .tunnel = tunnel, + } }, + .userdata = NULL, +- .crypto_xform = &xform ++ .crypto_xform = crypto_xform, + }; + ++ if (options->is_outbound) ++ sess_conf.ipsec.direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS; ++ else ++ sess_conf.ipsec.direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS; ++ + struct rte_security_ctx *ctx = (struct rte_security_ctx *) + rte_cryptodev_get_sec_ctx(dev_id); + +diff --git a/dpdk/app/test-crypto-perf/cperf_options.h b/dpdk/app/test-crypto-perf/cperf_options.h +index 613d6d31e2..6966e0b286 100644 +--- a/dpdk/app/test-crypto-perf/cperf_options.h ++++ b/dpdk/app/test-crypto-perf/cperf_options.h +@@ -105,6 +105,7 @@ struct cperf_options { + uint32_t out_of_place:1; + uint32_t silent:1; + uint32_t csv:1; ++ uint32_t is_outbound:1; + + enum rte_crypto_cipher_algorithm cipher_algo; + enum rte_crypto_cipher_operation cipher_op; +diff --git a/dpdk/app/test-crypto-perf/cperf_options_parsing.c b/dpdk/app/test-crypto-perf/cperf_options_parsing.c +index bc5e312c81..1f06e15d10 100644 +--- a/dpdk/app/test-crypto-perf/cperf_options_parsing.c ++++ b/dpdk/app/test-crypto-perf/cperf_options_parsing.c +@@ -519,6 +519,7 @@ parse_test_file(struct cperf_options *opts, + if (access(opts->test_file, F_OK) != -1) + return 0; + RTE_LOG(ERR, USER1, "Test vector file doesn't exist\n"); ++ free(opts->test_file); + + return -1; } +@@ -1318,6 +1319,21 @@ cperf_options_check(struct cperf_options *options) + if (check_docsis_buffer_length(options) < 0) + return -EINVAL; + } ++ ++ if (options->op_type == CPERF_IPSEC) { ++ if (options->aead_algo) { ++ if (options->aead_op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ++ options->is_outbound = 1; ++ else ++ options->is_outbound = 0; ++ } else { ++ if (options->cipher_op == RTE_CRYPTO_CIPHER_OP_ENCRYPT && ++ options->auth_op == RTE_CRYPTO_AUTH_OP_GENERATE) ++ options->is_outbound = 1; ++ else ++ options->is_outbound = 0; ++ } ++ } + #endif --#define CHECK_INTERVAL 100 /* 100ms */ --#define MAX_REPEAT_TIME 10 /* 1s (10 * 100ms) in total */ + return 0; +diff --git a/dpdk/app/test-crypto-perf/cperf_test_common.c b/dpdk/app/test-crypto-perf/cperf_test_common.c +index 27646cd619..932aab16df 100644 +--- a/dpdk/app/test-crypto-perf/cperf_test_common.c ++++ b/dpdk/app/test-crypto-perf/cperf_test_common.c +@@ -197,9 +197,11 @@ cperf_alloc_common_memory(const struct cperf_options *options, + RTE_CACHE_LINE_ROUNDUP(crypto_op_total_size); + uint32_t mbuf_size = sizeof(struct rte_mbuf) + options->segment_sz; + uint32_t max_size = options->max_buffer_size + options->digest_sz; +- uint16_t segments_nb = (max_size % options->segment_sz) ? +- (max_size / options->segment_sz) + 1 : +- max_size / options->segment_sz; ++ uint32_t segment_data_len = options->segment_sz - options->headroom_sz - ++ options->tailroom_sz; ++ uint16_t segments_nb = (max_size % segment_data_len) ? ++ (max_size / segment_data_len) + 1 : ++ (max_size / segment_data_len); + uint32_t obj_size = crypto_op_total_size_padded + + (mbuf_size * segments_nb); + +diff --git a/dpdk/app/test-crypto-perf/cperf_test_latency.c b/dpdk/app/test-crypto-perf/cperf_test_latency.c +index 49bf421c01..406e082e4e 100644 +--- a/dpdk/app/test-crypto-perf/cperf_test_latency.c ++++ b/dpdk/app/test-crypto-perf/cperf_test_latency.c +@@ -43,15 +43,28 @@ struct priv_op_data { + static void + cperf_latency_test_free(struct cperf_latency_ctx *ctx) + { +- if (ctx) { +- if (ctx->sess) +- rte_cryptodev_sym_session_free(ctx->dev_id, ctx->sess); - - static __rte_always_inline void - update_link_reg(struct i40e_hw *hw, struct rte_eth_link *link) +- rte_mempool_free(ctx->pool); ++ if (ctx == NULL) ++ return; + +- rte_free(ctx->res); +- rte_free(ctx); ++ if (ctx->sess != NULL) { ++ if (ctx->options->op_type == CPERF_ASYM_MODEX) ++ rte_cryptodev_asym_session_free(ctx->dev_id, ctx->sess); ++#ifdef RTE_LIB_SECURITY ++ else if (ctx->options->op_type == CPERF_PDCP || ++ ctx->options->op_type == CPERF_DOCSIS || ++ ctx->options->op_type == CPERF_IPSEC) { ++ struct rte_security_ctx *sec_ctx = ++ rte_cryptodev_get_sec_ctx(ctx->dev_id); ++ rte_security_session_destroy(sec_ctx, ctx->sess); ++ } ++#endif ++ else ++ rte_cryptodev_sym_session_free(ctx->dev_id, ctx->sess); + } ++ ++ rte_mempool_free(ctx->pool); ++ rte_free(ctx->res); ++ rte_free(ctx); + } + + void * +diff --git a/dpdk/app/test-crypto-perf/cperf_test_vector_parsing.c b/dpdk/app/test-crypto-perf/cperf_test_vector_parsing.c +index 98e46c3381..737d61d4af 100644 +--- a/dpdk/app/test-crypto-perf/cperf_test_vector_parsing.c ++++ b/dpdk/app/test-crypto-perf/cperf_test_vector_parsing.c +@@ -30,6 +30,7 @@ free_test_vector(struct cperf_test_vector *vector, struct cperf_options *opts) + rte_free(vector->cipher_key.data); + rte_free(vector->auth_key.data); + rte_free(vector->ciphertext.data); ++ free(opts->test_file); + } + + rte_free(vector); +diff --git a/dpdk/app/test-crypto-perf/cperf_test_verify.c b/dpdk/app/test-crypto-perf/cperf_test_verify.c +index c03e1d5ba5..8042c94e04 100644 +--- a/dpdk/app/test-crypto-perf/cperf_test_verify.c ++++ b/dpdk/app/test-crypto-perf/cperf_test_verify.c +@@ -38,14 +38,27 @@ struct cperf_op_result { + static void + cperf_verify_test_free(struct cperf_verify_ctx *ctx) { -@@ -2878,6 +2875,8 @@ static __rte_always_inline void - update_link_aq(struct i40e_hw *hw, struct rte_eth_link *link, - bool enable_lse, int wait_to_complete) +- if (ctx) { +- if (ctx->sess) +- rte_cryptodev_sym_session_free(ctx->dev_id, ctx->sess); +- +- rte_mempool_free(ctx->pool); ++ if (ctx == NULL) ++ return; + +- rte_free(ctx); ++ if (ctx->sess != NULL) { ++ if (ctx->options->op_type == CPERF_ASYM_MODEX) ++ rte_cryptodev_asym_session_free(ctx->dev_id, ctx->sess); ++#ifdef RTE_LIB_SECURITY ++ else if (ctx->options->op_type == CPERF_PDCP || ++ ctx->options->op_type == CPERF_DOCSIS || ++ ctx->options->op_type == CPERF_IPSEC) { ++ struct rte_security_ctx *sec_ctx = ++ rte_cryptodev_get_sec_ctx(ctx->dev_id); ++ rte_security_session_destroy(sec_ctx, ctx->sess); ++ } ++#endif ++ else ++ rte_cryptodev_sym_session_free(ctx->dev_id, ctx->sess); + } ++ ++ rte_mempool_free(ctx->pool); ++ rte_free(ctx); + } + + void * +diff --git a/dpdk/app/test-crypto-perf/main.c b/dpdk/app/test-crypto-perf/main.c +index af5bd0d23b..bc1f0f9659 100644 +--- a/dpdk/app/test-crypto-perf/main.c ++++ b/dpdk/app/test-crypto-perf/main.c +@@ -193,11 +193,10 @@ cperf_initialize_cryptodev(struct cperf_options *opts, uint8_t *enabled_cdevs) + #endif + + struct rte_cryptodev_info cdev_info; +- uint8_t socket_id = rte_cryptodev_socket_id(cdev_id); +- /* range check the socket_id - negative values become big +- * positive ones due to use of unsigned value +- */ +- if (socket_id >= RTE_MAX_NUMA_NODES) ++ int socket_id = rte_cryptodev_socket_id(cdev_id); ++ ++ /* Use the first socket if SOCKET_ID_ANY is returned. */ ++ if (socket_id == SOCKET_ID_ANY) + socket_id = 0; + + rte_cryptodev_info_get(cdev_id, &cdev_info); +@@ -650,7 +649,11 @@ main(int argc, char **argv) + + cdev_id = enabled_cdevs[cdev_index]; + +- uint8_t socket_id = rte_cryptodev_socket_id(cdev_id); ++ int socket_id = rte_cryptodev_socket_id(cdev_id); ++ ++ /* Use the first socket if SOCKET_ID_ANY is returned. */ ++ if (socket_id == SOCKET_ID_ANY) ++ socket_id = 0; + + ctx[i] = cperf_testmap[opts.test].constructor( + session_pool_socket[socket_id].sess_mp, +diff --git a/dpdk/app/test-flow-perf/main.c b/dpdk/app/test-flow-perf/main.c +index 4a9206803a..e0ef78a840 100644 +--- a/dpdk/app/test-flow-perf/main.c ++++ b/dpdk/app/test-flow-perf/main.c +@@ -848,7 +848,12 @@ args_parse(int argc, char **argv) + /* Control */ + if (strcmp(lgopts[opt_idx].name, + "rules-batch") == 0) { +- rules_batch = atoi(optarg); ++ n = atoi(optarg); ++ if (n > 0) ++ rules_batch = n; ++ else ++ rte_exit(EXIT_FAILURE, ++ "flow rules-batch should be > 0\n"); + } + if (strcmp(lgopts[opt_idx].name, + "rules-count") == 0) { +diff --git a/dpdk/app/test-pipeline/main.c b/dpdk/app/test-pipeline/main.c +index 1e16794183..8633933fd9 100644 +--- a/dpdk/app/test-pipeline/main.c ++++ b/dpdk/app/test-pipeline/main.c +@@ -10,6 +10,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -41,6 +42,15 @@ + + #include "main.h" + ++bool force_quit; ++ ++static void ++signal_handler(int signum) ++{ ++ if (signum == SIGINT || signum == SIGTERM) ++ force_quit = true; ++} ++ + int + main(int argc, char **argv) { -+#define CHECK_INTERVAL 100 /* 100ms */ -+#define MAX_REPEAT_TIME 10 /* 1s (10 * 100ms) in total */ - uint32_t rep_cnt = MAX_REPEAT_TIME; - struct i40e_link_status link_status; - int status; -@@ -6738,7 +6737,6 @@ i40e_dev_handle_aq_msg(struct rte_eth_dev *dev) - if (!ret) - rte_eth_dev_callback_process(dev, - RTE_ETH_EVENT_INTR_LSC, NULL); +@@ -54,6 +64,10 @@ main(int argc, char **argv) + argc -= ret; + argv += ret; + ++ force_quit = false; ++ signal(SIGINT, signal_handler); ++ signal(SIGTERM, signal_handler); ++ + /* Parse application arguments (after the EAL ones) */ + ret = app_parse_args(argc, argv); + if (ret < 0) { +diff --git a/dpdk/app/test-pipeline/main.h b/dpdk/app/test-pipeline/main.h +index 59dcfddbf4..9df157de22 100644 +--- a/dpdk/app/test-pipeline/main.h ++++ b/dpdk/app/test-pipeline/main.h +@@ -60,6 +60,8 @@ struct app_params { + + extern struct app_params app; + ++extern bool force_quit; ++ + int app_parse_args(int argc, char **argv); + void app_print_usage(void); + void app_init(void); +diff --git a/dpdk/app/test-pipeline/pipeline_acl.c b/dpdk/app/test-pipeline/pipeline_acl.c +index 5857bc285f..abde4bf934 100644 +--- a/dpdk/app/test-pipeline/pipeline_acl.c ++++ b/dpdk/app/test-pipeline/pipeline_acl.c +@@ -236,14 +236,16 @@ app_main_loop_worker_pipeline_acl(void) { + + /* Run-time */ + #if APP_FLUSH == 0 +- for ( ; ; ) ++ while (!force_quit) + rte_pipeline_run(p); + #else +- for (i = 0; ; i++) { ++ i = 0; ++ while (!force_quit) { + rte_pipeline_run(p); + + if ((i & APP_FLUSH) == 0) + rte_pipeline_flush(p); ++ i++; + } + #endif + } +diff --git a/dpdk/app/test-pipeline/pipeline_hash.c b/dpdk/app/test-pipeline/pipeline_hash.c +index 2dd8928d43..cab9c20980 100644 +--- a/dpdk/app/test-pipeline/pipeline_hash.c ++++ b/dpdk/app/test-pipeline/pipeline_hash.c +@@ -366,14 +366,16 @@ app_main_loop_worker_pipeline_hash(void) { + + /* Run-time */ + #if APP_FLUSH == 0 +- for ( ; ; ) ++ while (!force_quit) + rte_pipeline_run(p); + #else +- for (i = 0; ; i++) { ++ i = 0; ++ while (!force_quit) { + rte_pipeline_run(p); + + if ((i & APP_FLUSH) == 0) + rte_pipeline_flush(p); ++ i++; + } + #endif + } +@@ -411,59 +413,61 @@ app_main_loop_rx_metadata(void) { + RTE_LOG(INFO, USER1, "Core %u is doing RX (with meta-data)\n", + rte_lcore_id()); + +- for (i = 0; ; i = ((i + 1) & (app.n_ports - 1))) { +- uint16_t n_mbufs; - - break; - default: - PMD_DRV_LOG(DEBUG, "Request %u is not supported yet", -@@ -12123,40 +12121,6 @@ i40e_cloud_filter_qinq_create(struct i40e_pf *pf) - return ret; +- n_mbufs = rte_eth_rx_burst( +- app.ports[i], +- 0, +- app.mbuf_rx.array, +- app.burst_size_rx_read); +- +- if (n_mbufs == 0) +- continue; +- +- for (j = 0; j < n_mbufs; j++) { +- struct rte_mbuf *m; +- uint8_t *m_data, *key; +- struct rte_ipv4_hdr *ip_hdr; +- struct rte_ipv6_hdr *ipv6_hdr; +- uint32_t ip_dst; +- uint8_t *ipv6_dst; +- uint32_t *signature, *k32; +- +- m = app.mbuf_rx.array[j]; +- m_data = rte_pktmbuf_mtod(m, uint8_t *); +- signature = RTE_MBUF_METADATA_UINT32_PTR(m, +- APP_METADATA_OFFSET(0)); +- key = RTE_MBUF_METADATA_UINT8_PTR(m, +- APP_METADATA_OFFSET(32)); +- +- if (RTE_ETH_IS_IPV4_HDR(m->packet_type)) { +- ip_hdr = (struct rte_ipv4_hdr *) +- &m_data[sizeof(struct rte_ether_hdr)]; +- ip_dst = ip_hdr->dst_addr; +- +- k32 = (uint32_t *) key; +- k32[0] = ip_dst & 0xFFFFFF00; +- } else if (RTE_ETH_IS_IPV6_HDR(m->packet_type)) { +- ipv6_hdr = (struct rte_ipv6_hdr *) +- &m_data[sizeof(struct rte_ether_hdr)]; +- ipv6_dst = ipv6_hdr->dst_addr; +- +- memcpy(key, ipv6_dst, 16); +- } else ++ while (!force_quit) { ++ for (i = 0; i < app.n_ports; i++) { ++ uint16_t n_mbufs; ++ ++ n_mbufs = rte_eth_rx_burst( ++ app.ports[i], ++ 0, ++ app.mbuf_rx.array, ++ app.burst_size_rx_read); ++ ++ if (n_mbufs == 0) + continue; + +- *signature = test_hash(key, NULL, 0, 0); ++ for (j = 0; j < n_mbufs; j++) { ++ struct rte_mbuf *m; ++ uint8_t *m_data, *key; ++ struct rte_ipv4_hdr *ip_hdr; ++ struct rte_ipv6_hdr *ipv6_hdr; ++ uint32_t ip_dst; ++ uint8_t *ipv6_dst; ++ uint32_t *signature, *k32; ++ ++ m = app.mbuf_rx.array[j]; ++ m_data = rte_pktmbuf_mtod(m, uint8_t *); ++ signature = RTE_MBUF_METADATA_UINT32_PTR(m, ++ APP_METADATA_OFFSET(0)); ++ key = RTE_MBUF_METADATA_UINT8_PTR(m, ++ APP_METADATA_OFFSET(32)); ++ ++ if (RTE_ETH_IS_IPV4_HDR(m->packet_type)) { ++ ip_hdr = (struct rte_ipv4_hdr *) ++ &m_data[sizeof(struct rte_ether_hdr)]; ++ ip_dst = ip_hdr->dst_addr; ++ ++ k32 = (uint32_t *) key; ++ k32[0] = ip_dst & 0xFFFFFF00; ++ } else if (RTE_ETH_IS_IPV6_HDR(m->packet_type)) { ++ ipv6_hdr = (struct rte_ipv6_hdr *) ++ &m_data[sizeof(struct rte_ether_hdr)]; ++ ipv6_dst = ipv6_hdr->dst_addr; ++ ++ memcpy(key, ipv6_dst, 16); ++ } else ++ continue; ++ ++ *signature = test_hash(key, NULL, 0, 0); ++ } ++ ++ do { ++ ret = rte_ring_sp_enqueue_bulk( ++ app.rings_rx[i], ++ (void **) app.mbuf_rx.array, ++ n_mbufs, ++ NULL); ++ } while (ret == 0 && !force_quit); + } +- +- do { +- ret = rte_ring_sp_enqueue_bulk( +- app.rings_rx[i], +- (void **) app.mbuf_rx.array, +- n_mbufs, +- NULL); +- } while (ret == 0); + } + } +diff --git a/dpdk/app/test-pipeline/pipeline_lpm.c b/dpdk/app/test-pipeline/pipeline_lpm.c +index 8add5e71b7..e3d4b3fdc5 100644 +--- a/dpdk/app/test-pipeline/pipeline_lpm.c ++++ b/dpdk/app/test-pipeline/pipeline_lpm.c +@@ -160,14 +160,16 @@ app_main_loop_worker_pipeline_lpm(void) { + + /* Run-time */ + #if APP_FLUSH == 0 +- for ( ; ; ) ++ while (!force_quit) + rte_pipeline_run(p); + #else +- for (i = 0; ; i++) { ++ i = 0; ++ while (!force_quit) { + rte_pipeline_run(p); + + if ((i & APP_FLUSH) == 0) + rte_pipeline_flush(p); ++ i++; + } + #endif + } +diff --git a/dpdk/app/test-pipeline/pipeline_lpm_ipv6.c b/dpdk/app/test-pipeline/pipeline_lpm_ipv6.c +index 26b325180d..f9aca74e4c 100644 +--- a/dpdk/app/test-pipeline/pipeline_lpm_ipv6.c ++++ b/dpdk/app/test-pipeline/pipeline_lpm_ipv6.c +@@ -158,14 +158,16 @@ app_main_loop_worker_pipeline_lpm_ipv6(void) { + + /* Run-time */ + #if APP_FLUSH == 0 +- for ( ; ; ) ++ while (!force_quit) + rte_pipeline_run(p); + #else +- for (i = 0; ; i++) { ++ i = 0; ++ while (!force_quit) { + rte_pipeline_run(p); + + if ((i & APP_FLUSH) == 0) + rte_pipeline_flush(p); ++ i++; + } + #endif + } +diff --git a/dpdk/app/test-pipeline/pipeline_stub.c b/dpdk/app/test-pipeline/pipeline_stub.c +index b6750d51bf..48a638aad7 100644 +--- a/dpdk/app/test-pipeline/pipeline_stub.c ++++ b/dpdk/app/test-pipeline/pipeline_stub.c +@@ -122,14 +122,16 @@ app_main_loop_worker_pipeline_stub(void) { + + /* Run-time */ + #if APP_FLUSH == 0 +- for ( ; ; ) ++ while (!force_quit) + rte_pipeline_run(p); + #else +- for (i = 0; ; i++) { ++ i = 0; ++ while (!force_quit) { + rte_pipeline_run(p); + + if ((i & APP_FLUSH) == 0) + rte_pipeline_flush(p); ++ i++; + } + #endif } +diff --git a/dpdk/app/test-pipeline/runtime.c b/dpdk/app/test-pipeline/runtime.c +index d939a85d7e..752f783370 100644 +--- a/dpdk/app/test-pipeline/runtime.c ++++ b/dpdk/app/test-pipeline/runtime.c +@@ -48,24 +48,26 @@ app_main_loop_rx(void) { --static void --i40e_set_mac_max_frame(struct rte_eth_dev *dev, uint16_t size) --{ -- struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); -- uint32_t rep_cnt = MAX_REPEAT_TIME; -- struct rte_eth_link link; -- enum i40e_status_code status; -- bool can_be_set = true; + RTE_LOG(INFO, USER1, "Core %u is doing RX\n", rte_lcore_id()); + +- for (i = 0; ; i = ((i + 1) & (app.n_ports - 1))) { +- uint16_t n_mbufs; +- +- n_mbufs = rte_eth_rx_burst( +- app.ports[i], +- 0, +- app.mbuf_rx.array, +- app.burst_size_rx_read); +- +- if (n_mbufs == 0) +- continue; - -- /* -- * I40E_MEDIA_TYPE_BASET link up can be ignored -- * I40E_MEDIA_TYPE_BASET link down that hw->phy.media_type -- * is I40E_MEDIA_TYPE_UNKNOWN -- */ -- if (hw->phy.media_type != I40E_MEDIA_TYPE_BASET && -- hw->phy.media_type != I40E_MEDIA_TYPE_UNKNOWN) { - do { -- update_link_reg(hw, &link); -- if (link.link_status) -- break; -- rte_delay_ms(CHECK_INTERVAL); -- } while (--rep_cnt); -- can_be_set = !!link.link_status; +- ret = rte_ring_sp_enqueue_bulk( +- app.rings_rx[i], +- (void **) app.mbuf_rx.array, +- n_mbufs, NULL); +- } while (ret == 0); ++ while (!force_quit) { ++ for (i = 0; i < app.n_ports; i++) { ++ uint16_t n_mbufs; ++ ++ n_mbufs = rte_eth_rx_burst( ++ app.ports[i], ++ 0, ++ app.mbuf_rx.array, ++ app.burst_size_rx_read); ++ ++ if (n_mbufs == 0) ++ continue; ++ ++ do { ++ ret = rte_ring_sp_enqueue_bulk( ++ app.rings_rx[i], ++ (void **) app.mbuf_rx.array, ++ n_mbufs, NULL); ++ } while (ret == 0 && !force_quit); ++ } + } + } + +@@ -82,25 +84,27 @@ app_main_loop_worker(void) { + if (worker_mbuf == NULL) + rte_panic("Worker thread: cannot allocate buffer space\n"); + +- for (i = 0; ; i = ((i + 1) & (app.n_ports - 1))) { +- int ret; +- +- ret = rte_ring_sc_dequeue_bulk( +- app.rings_rx[i], +- (void **) worker_mbuf->array, +- app.burst_size_worker_read, +- NULL); ++ while (!force_quit) { ++ for (i = 0; i < app.n_ports; i++) { ++ int ret; + +- if (ret == 0) +- continue; +- +- do { +- ret = rte_ring_sp_enqueue_bulk( +- app.rings_tx[i ^ 1], ++ ret = rte_ring_sc_dequeue_bulk( ++ app.rings_rx[i], + (void **) worker_mbuf->array, +- app.burst_size_worker_write, ++ app.burst_size_worker_read, + NULL); +- } while (ret == 0); ++ ++ if (ret == 0) ++ continue; ++ ++ do { ++ ret = rte_ring_sp_enqueue_bulk( ++ app.rings_tx[i ^ 1], ++ (void **) worker_mbuf->array, ++ app.burst_size_worker_write, ++ NULL); ++ } while (ret == 0 && !force_quit); ++ } + } + } + +@@ -110,45 +114,47 @@ app_main_loop_tx(void) { + + RTE_LOG(INFO, USER1, "Core %u is doing TX\n", rte_lcore_id()); + +- for (i = 0; ; i = ((i + 1) & (app.n_ports - 1))) { +- uint16_t n_mbufs, n_pkts; +- int ret; ++ while (!force_quit) { ++ for (i = 0; i < app.n_ports; i++) { ++ uint16_t n_mbufs, n_pkts; ++ int ret; + +- n_mbufs = app.mbuf_tx[i].n_mbufs; ++ n_mbufs = app.mbuf_tx[i].n_mbufs; + +- ret = rte_ring_sc_dequeue_bulk( +- app.rings_tx[i], +- (void **) &app.mbuf_tx[i].array[n_mbufs], +- app.burst_size_tx_read, +- NULL); ++ ret = rte_ring_sc_dequeue_bulk( ++ app.rings_tx[i], ++ (void **) &app.mbuf_tx[i].array[n_mbufs], ++ app.burst_size_tx_read, ++ NULL); + +- if (ret == 0) +- continue; ++ if (ret == 0) ++ continue; + +- n_mbufs += app.burst_size_tx_read; ++ n_mbufs += app.burst_size_tx_read; + +- if (n_mbufs < app.burst_size_tx_write) { +- app.mbuf_tx[i].n_mbufs = n_mbufs; +- continue; +- } ++ if (n_mbufs < app.burst_size_tx_write) { ++ app.mbuf_tx[i].n_mbufs = n_mbufs; ++ continue; ++ } + +- n_pkts = rte_eth_tx_burst( +- app.ports[i], +- 0, +- app.mbuf_tx[i].array, +- n_mbufs); ++ n_pkts = rte_eth_tx_burst( ++ app.ports[i], ++ 0, ++ app.mbuf_tx[i].array, ++ n_mbufs); + +- if (n_pkts < n_mbufs) { +- uint16_t k; ++ if (n_pkts < n_mbufs) { ++ uint16_t k; + +- for (k = n_pkts; k < n_mbufs; k++) { +- struct rte_mbuf *pkt_to_free; ++ for (k = n_pkts; k < n_mbufs; k++) { ++ struct rte_mbuf *pkt_to_free; + +- pkt_to_free = app.mbuf_tx[i].array[k]; +- rte_pktmbuf_free(pkt_to_free); ++ pkt_to_free = app.mbuf_tx[i].array[k]; ++ rte_pktmbuf_free(pkt_to_free); ++ } + } +- } + +- app.mbuf_tx[i].n_mbufs = 0; ++ app.mbuf_tx[i].n_mbufs = 0; ++ } + } + } +diff --git a/dpdk/app/test-pmd/cmdline.c b/dpdk/app/test-pmd/cmdline.c +index b32dc8bfd4..3a451b9fa0 100644 +--- a/dpdk/app/test-pmd/cmdline.c ++++ b/dpdk/app/test-pmd/cmdline.c +@@ -468,6 +468,12 @@ static void cmd_help_long_parsed(void *parsed_result, + "mac_addr add port (port_id) vf (vf_id) (mac_address)\n" + " Add a MAC address for a VF on the port.\n\n" + ++ "mcast_addr add (port_id) (mcast_addr)\n" ++ " Add a multicast MAC addresses on port_id.\n\n" ++ ++ "mcast_addr remove (port_id) (mcast_addr)\n" ++ " Remove a multicast MAC address from port_id.\n\n" ++ + "set vf mac addr (port_id) (vf_id) (XX:XX:XX:XX:XX:XX)\n" + " Set the MAC address for a VF from the PF.\n\n" + +@@ -4887,19 +4893,6 @@ cmd_tso_set_parsed(void *parsed_result, + ports[res->port_id].tso_segsz); + } + cmd_config_queue_tx_offloads(&ports[res->port_id]); +- +- /* display warnings if configuration is not supported by the NIC */ +- ret = eth_dev_info_get_print_err(res->port_id, &dev_info); +- if (ret != 0) +- return; +- +- if ((ports[res->port_id].tso_segsz != 0) && +- (dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_TCP_TSO) == 0) { +- fprintf(stderr, +- "Warning: TSO enabled but not supported by port %d\n", +- res->port_id); - } - -- if (can_be_set) { -- status = i40e_aq_set_mac_config(hw, size, TRUE, 0, false, NULL); -- if (status != I40E_SUCCESS) -- PMD_DRV_LOG(ERR, "Failed to set max frame size at port level"); -- } else { -- PMD_DRV_LOG(ERR, "Set max frame size at port level not applicable on link down"); + cmd_reconfig_device_queue(res->port_id, 1, 1); + } + +@@ -4957,39 +4950,27 @@ struct cmd_tunnel_tso_set_result { + portid_t port_id; + }; + +-static struct rte_eth_dev_info +-check_tunnel_tso_nic_support(portid_t port_id) ++static void ++check_tunnel_tso_nic_support(portid_t port_id, uint64_t tx_offload_capa) + { +- struct rte_eth_dev_info dev_info; +- +- if (eth_dev_info_get_print_err(port_id, &dev_info) != 0) +- return dev_info; +- +- if (!(dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO)) +- fprintf(stderr, +- "Warning: VXLAN TUNNEL TSO not supported therefore not enabled for port %d\n", ++ if (!(tx_offload_capa & RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO)) ++ printf("Warning: VXLAN TUNNEL TSO not supported therefore not enabled for port %d\n", + port_id); +- if (!(dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO)) +- fprintf(stderr, +- "Warning: GRE TUNNEL TSO not supported therefore not enabled for port %d\n", ++ if (!(tx_offload_capa & RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO)) ++ printf("Warning: GRE TUNNEL TSO not supported therefore not enabled for port %d\n", + port_id); +- if (!(dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_IPIP_TNL_TSO)) +- fprintf(stderr, +- "Warning: IPIP TUNNEL TSO not supported therefore not enabled for port %d\n", ++ if (!(tx_offload_capa & RTE_ETH_TX_OFFLOAD_IPIP_TNL_TSO)) ++ printf("Warning: IPIP TUNNEL TSO not supported therefore not enabled for port %d\n", + port_id); +- if (!(dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_GENEVE_TNL_TSO)) +- fprintf(stderr, +- "Warning: GENEVE TUNNEL TSO not supported therefore not enabled for port %d\n", ++ if (!(tx_offload_capa & RTE_ETH_TX_OFFLOAD_GENEVE_TNL_TSO)) ++ printf("Warning: GENEVE TUNNEL TSO not supported therefore not enabled for port %d\n", + port_id); +- if (!(dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_IP_TNL_TSO)) +- fprintf(stderr, +- "Warning: IP TUNNEL TSO not supported therefore not enabled for port %d\n", ++ if (!(tx_offload_capa & RTE_ETH_TX_OFFLOAD_IP_TNL_TSO)) ++ printf("Warning: IP TUNNEL TSO not supported therefore not enabled for port %d\n", + port_id); +- if (!(dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_UDP_TNL_TSO)) +- fprintf(stderr, +- "Warning: UDP TUNNEL TSO not supported therefore not enabled for port %d\n", ++ if (!(tx_offload_capa & RTE_ETH_TX_OFFLOAD_UDP_TNL_TSO)) ++ printf("Warning: UDP TUNNEL TSO not supported therefore not enabled for port %d\n", + port_id); +- return dev_info; + } + + static void +@@ -4999,6 +4980,13 @@ cmd_tunnel_tso_set_parsed(void *parsed_result, + { + struct cmd_tunnel_tso_set_result *res = parsed_result; + struct rte_eth_dev_info dev_info; ++ uint64_t all_tunnel_tso = RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | ++ RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | ++ RTE_ETH_TX_OFFLOAD_IPIP_TNL_TSO | ++ RTE_ETH_TX_OFFLOAD_GENEVE_TNL_TSO | ++ RTE_ETH_TX_OFFLOAD_IP_TNL_TSO | ++ RTE_ETH_TX_OFFLOAD_UDP_TNL_TSO; ++ int ret; + + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; +@@ -5010,28 +4998,19 @@ cmd_tunnel_tso_set_parsed(void *parsed_result, + if (!strcmp(res->mode, "set")) + ports[res->port_id].tunnel_tso_segsz = res->tso_segsz; + +- dev_info = check_tunnel_tso_nic_support(res->port_id); + if (ports[res->port_id].tunnel_tso_segsz == 0) { +- ports[res->port_id].dev_conf.txmode.offloads &= +- ~(RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | +- RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | +- RTE_ETH_TX_OFFLOAD_IPIP_TNL_TSO | +- RTE_ETH_TX_OFFLOAD_GENEVE_TNL_TSO | +- RTE_ETH_TX_OFFLOAD_IP_TNL_TSO | +- RTE_ETH_TX_OFFLOAD_UDP_TNL_TSO); ++ ports[res->port_id].dev_conf.txmode.offloads &= ~all_tunnel_tso; + printf("TSO for tunneled packets is disabled\n"); + } else { +- uint64_t tso_offloads = (RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | +- RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | +- RTE_ETH_TX_OFFLOAD_IPIP_TNL_TSO | +- RTE_ETH_TX_OFFLOAD_GENEVE_TNL_TSO | +- RTE_ETH_TX_OFFLOAD_IP_TNL_TSO | +- RTE_ETH_TX_OFFLOAD_UDP_TNL_TSO); ++ ret = eth_dev_info_get_print_err(res->port_id, &dev_info); ++ if (ret != 0) ++ return; + +- ports[res->port_id].dev_conf.txmode.offloads |= +- (tso_offloads & dev_info.tx_offload_capa); +- printf("TSO segment size for tunneled packets is %d\n", +- ports[res->port_id].tunnel_tso_segsz); ++ if ((all_tunnel_tso & dev_info.tx_offload_capa) == 0) { ++ fprintf(stderr, "Error: port=%u don't support tunnel TSO offloads.\n", ++ res->port_id); ++ return; ++ } + + /* Below conditions are needed to make it work: + * (1) tunnel TSO is supported by the NIC; +@@ -5044,14 +5023,23 @@ cmd_tunnel_tso_set_parsed(void *parsed_result, + * is not necessary for IPv6 tunneled pkts because there's no + * checksum in IP header anymore. + */ +- +- if (!ports[res->port_id].parse_tunnel) ++ if (!ports[res->port_id].parse_tunnel) { + fprintf(stderr, +- "Warning: csum parse_tunnel must be set so that tunneled packets are recognized\n"); ++ "Error: csum parse_tunnel must be set so that tunneled packets are recognized\n"); ++ return; ++ } + if (!(ports[res->port_id].dev_conf.txmode.offloads & +- RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM)) ++ RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM)) { + fprintf(stderr, +- "Warning: csum set outer-ip must be set to hw if outer L3 is IPv4; not necessary for IPv6\n"); ++ "Error: csum set outer-ip must be set to hw if outer L3 is IPv4; not necessary for IPv6\n"); ++ return; ++ } ++ ++ check_tunnel_tso_nic_support(res->port_id, dev_info.tx_offload_capa); ++ ports[res->port_id].dev_conf.txmode.offloads |= ++ (all_tunnel_tso & dev_info.tx_offload_capa); ++ printf("TSO segment size for tunneled packets is %d\n", ++ ports[res->port_id].tunnel_tso_segsz); + } + + cmd_config_queue_tx_offloads(&ports[res->port_id]); +@@ -12917,32 +12905,25 @@ cmdline_read_from_file(const char *filename) + printf("Read CLI commands from %s\n", filename); + } + ++void ++prompt_exit(void) ++{ ++ cmdline_quit(testpmd_cl); ++} ++ + /* prompt function, called from main on MAIN lcore */ + void + prompt(void) + { +- int ret; +- + testpmd_cl = cmdline_stdin_new(main_ctx, "testpmd> "); +- if (testpmd_cl == NULL) ++ if (testpmd_cl == NULL) { ++ fprintf(stderr, ++ "Failed to create stdin based cmdline context\n"); + return; +- +- ret = atexit(prompt_exit); +- if (ret != 0) +- fprintf(stderr, "Cannot set exit function for cmdline\n"); ++ } + + cmdline_interact(testpmd_cl); +- if (ret != 0) +- cmdline_stdin_exit(testpmd_cl); +-} +- +-void +-prompt_exit(void) +-{ +- if (testpmd_cl != NULL) { +- cmdline_quit(testpmd_cl); +- cmdline_stdin_exit(testpmd_cl); - } ++ cmdline_stdin_exit(testpmd_cl); + } + + void +diff --git a/dpdk/app/test-pmd/cmdline_flow.c b/dpdk/app/test-pmd/cmdline_flow.c +index 88108498e0..6970f90307 100644 +--- a/dpdk/app/test-pmd/cmdline_flow.c ++++ b/dpdk/app/test-pmd/cmdline_flow.c +@@ -2940,6 +2940,7 @@ static const struct token token_list[] = { + NEXT_ENTRY(COMMON_UNSIGNED)), + .args = ARGS(ARGS_ENTRY(struct buffer, + args.table.attr.nb_flows)), ++ .call = parse_table, + }, + [TABLE_PATTERN_TEMPLATE] = { + .name = "pattern_template", +@@ -7737,15 +7738,15 @@ parse_vc_action_l2_encap(struct context *ctx, const struct token *token, + l2_encap_conf.eth_dst, RTE_ETHER_ADDR_LEN); + memcpy(eth.src.addr_bytes, + l2_encap_conf.eth_src, RTE_ETHER_ADDR_LEN); +- memcpy(header, ð, sizeof(eth)); +- header += sizeof(eth); ++ memcpy(header, ð.hdr, sizeof(struct rte_ether_hdr)); ++ header += sizeof(struct rte_ether_hdr); + if (l2_encap_conf.select_vlan) { + if (l2_encap_conf.select_ipv4) + vlan.inner_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4); + else + vlan.inner_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6); +- memcpy(header, &vlan, sizeof(vlan)); +- header += sizeof(vlan); ++ memcpy(header, &vlan.hdr, sizeof(struct rte_vlan_hdr)); ++ header += sizeof(struct rte_vlan_hdr); + } + action_encap_data->conf.size = header - + action_encap_data->data; +@@ -7793,11 +7794,11 @@ parse_vc_action_l2_decap(struct context *ctx, const struct token *token, + header = action_decap_data->data; + if (l2_decap_conf.select_vlan) + eth.type = rte_cpu_to_be_16(RTE_ETHER_TYPE_VLAN); +- memcpy(header, ð, sizeof(eth)); +- header += sizeof(eth); ++ memcpy(header, ð.hdr, sizeof(struct rte_ether_hdr)); ++ header += sizeof(struct rte_ether_hdr); + if (l2_decap_conf.select_vlan) { +- memcpy(header, &vlan, sizeof(vlan)); +- header += sizeof(vlan); ++ memcpy(header, &vlan.hdr, sizeof(struct rte_vlan_hdr)); ++ header += sizeof(struct rte_vlan_hdr); + } + action_decap_data->conf.size = header - + action_decap_data->data; +@@ -7877,15 +7878,15 @@ parse_vc_action_mplsogre_encap(struct context *ctx, const struct token *token, + mplsogre_encap_conf.eth_dst, RTE_ETHER_ADDR_LEN); + memcpy(eth.src.addr_bytes, + mplsogre_encap_conf.eth_src, RTE_ETHER_ADDR_LEN); +- memcpy(header, ð, sizeof(eth)); +- header += sizeof(eth); ++ memcpy(header, ð.hdr, sizeof(struct rte_ether_hdr)); ++ header += sizeof(struct rte_ether_hdr); + if (mplsogre_encap_conf.select_vlan) { + if (mplsogre_encap_conf.select_ipv4) + vlan.inner_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4); + else + vlan.inner_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6); +- memcpy(header, &vlan, sizeof(vlan)); +- header += sizeof(vlan); ++ memcpy(header, &vlan.hdr, sizeof(struct rte_vlan_hdr)); ++ header += sizeof(struct rte_vlan_hdr); + } + if (mplsogre_encap_conf.select_ipv4) { + memcpy(header, &ipv4, sizeof(ipv4)); +@@ -7972,15 +7973,15 @@ parse_vc_action_mplsogre_decap(struct context *ctx, const struct token *token, + mplsogre_encap_conf.eth_dst, RTE_ETHER_ADDR_LEN); + memcpy(eth.src.addr_bytes, + mplsogre_encap_conf.eth_src, RTE_ETHER_ADDR_LEN); +- memcpy(header, ð, sizeof(eth)); +- header += sizeof(eth); ++ memcpy(header, ð.hdr, sizeof(struct rte_ether_hdr)); ++ header += sizeof(struct rte_ether_hdr); + if (mplsogre_encap_conf.select_vlan) { + if (mplsogre_encap_conf.select_ipv4) + vlan.inner_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4); + else + vlan.inner_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6); +- memcpy(header, &vlan, sizeof(vlan)); +- header += sizeof(vlan); ++ memcpy(header, &vlan.hdr, sizeof(struct rte_vlan_hdr)); ++ header += sizeof(struct rte_vlan_hdr); + } + if (mplsogre_encap_conf.select_ipv4) { + memcpy(header, &ipv4, sizeof(ipv4)); +@@ -8071,15 +8072,15 @@ parse_vc_action_mplsoudp_encap(struct context *ctx, const struct token *token, + mplsoudp_encap_conf.eth_dst, RTE_ETHER_ADDR_LEN); + memcpy(eth.src.addr_bytes, + mplsoudp_encap_conf.eth_src, RTE_ETHER_ADDR_LEN); +- memcpy(header, ð, sizeof(eth)); +- header += sizeof(eth); ++ memcpy(header, ð.hdr, sizeof(struct rte_ether_hdr)); ++ header += sizeof(struct rte_ether_hdr); + if (mplsoudp_encap_conf.select_vlan) { + if (mplsoudp_encap_conf.select_ipv4) + vlan.inner_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4); + else + vlan.inner_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6); +- memcpy(header, &vlan, sizeof(vlan)); +- header += sizeof(vlan); ++ memcpy(header, &vlan.hdr, sizeof(struct rte_vlan_hdr)); ++ header += sizeof(struct rte_vlan_hdr); + } + if (mplsoudp_encap_conf.select_ipv4) { + memcpy(header, &ipv4, sizeof(ipv4)); +@@ -8168,15 +8169,15 @@ parse_vc_action_mplsoudp_decap(struct context *ctx, const struct token *token, + mplsoudp_encap_conf.eth_dst, RTE_ETHER_ADDR_LEN); + memcpy(eth.src.addr_bytes, + mplsoudp_encap_conf.eth_src, RTE_ETHER_ADDR_LEN); +- memcpy(header, ð, sizeof(eth)); +- header += sizeof(eth); ++ memcpy(header, ð.hdr, sizeof(struct rte_ether_hdr)); ++ header += sizeof(struct rte_ether_hdr); + if (mplsoudp_encap_conf.select_vlan) { + if (mplsoudp_encap_conf.select_ipv4) + vlan.inner_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4); + else + vlan.inner_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6); +- memcpy(header, &vlan, sizeof(vlan)); +- header += sizeof(vlan); ++ memcpy(header, &vlan.hdr, sizeof(struct rte_vlan_hdr)); ++ header += sizeof(struct rte_vlan_hdr); + } + if (mplsoudp_encap_conf.select_ipv4) { + memcpy(header, &ipv4, sizeof(ipv4)); +@@ -8993,6 +8994,11 @@ parse_table(struct context *ctx, const struct token *token, + case TABLE_TRANSFER: + out->args.table.attr.flow_attr.transfer = 1; + return len; ++ case TABLE_RULES_NUMBER: ++ ctx->objdata = 0; ++ ctx->object = out; ++ ctx->objmask = NULL; ++ return len; + default: + return -1; + } +diff --git a/dpdk/app/test-pmd/config.c b/dpdk/app/test-pmd/config.c +index acccb6b035..6a9eb4609c 100644 +--- a/dpdk/app/test-pmd/config.c ++++ b/dpdk/app/test-pmd/config.c +@@ -1875,6 +1875,7 @@ port_action_handle_update(portid_t port_id, uint32_t id, + struct rte_flow_error error; + struct rte_flow_action_handle *action_handle; + struct port_indirect_action *pia; ++ struct rte_flow_update_meter_mark mtr_update; + const void *update; + + action_handle = port_action_handle_get_by_id(port_id, id); +@@ -1888,6 +1889,17 @@ port_action_handle_update(portid_t port_id, uint32_t id, + case RTE_FLOW_ACTION_TYPE_CONNTRACK: + update = action->conf; + break; ++ case RTE_FLOW_ACTION_TYPE_METER_MARK: ++ memcpy(&mtr_update.meter_mark, action->conf, ++ sizeof(struct rte_flow_action_meter_mark)); ++ if (mtr_update.meter_mark.profile) ++ mtr_update.profile_valid = 1; ++ if (mtr_update.meter_mark.policy) ++ mtr_update.policy_valid = 1; ++ mtr_update.color_mode_valid = 1; ++ mtr_update.state_valid = 1; ++ update = &mtr_update; ++ break; + default: + update = action; + break; +@@ -2924,8 +2936,10 @@ port_queue_action_handle_update(portid_t port_id, + case RTE_FLOW_ACTION_TYPE_METER_MARK: + rte_memcpy(&mtr_update.meter_mark, action->conf, + sizeof(struct rte_flow_action_meter_mark)); +- mtr_update.profile_valid = 1; +- mtr_update.policy_valid = 1; ++ if (mtr_update.meter_mark.profile) ++ mtr_update.profile_valid = 1; ++ if (mtr_update.meter_mark.policy) ++ mtr_update.policy_valid = 1; + mtr_update.color_mode_valid = 1; + mtr_update.init_color_valid = 1; + mtr_update.state_valid = 1; +diff --git a/dpdk/app/test-pmd/csumonly.c b/dpdk/app/test-pmd/csumonly.c +index 1c24598515..4efb72be77 100644 +--- a/dpdk/app/test-pmd/csumonly.c ++++ b/dpdk/app/test-pmd/csumonly.c +@@ -250,7 +250,7 @@ parse_gtp(struct rte_udp_hdr *udp_hdr, + info->l4_proto = 0; + } + +- info->l2_len += RTE_ETHER_GTP_HLEN; ++ info->l2_len += gtp_len + sizeof(*udp_hdr); + } + + /* Parse a vxlan header */ +@@ -1168,10 +1168,13 @@ tunnel_update: + + nb_prep = rte_eth_tx_prepare(fs->tx_port, fs->tx_queue, + tx_pkts_burst, nb_rx); +- if (nb_prep != nb_rx) ++ if (nb_prep != nb_rx) { + fprintf(stderr, + "Preparing packet burst to transmit failed: %s\n", + rte_strerror(rte_errno)); ++ fs->fwd_dropped += (nb_rx - nb_prep); ++ rte_pktmbuf_free_bulk(&tx_pkts_burst[nb_prep], nb_rx - nb_prep); ++ } + + nb_tx = rte_eth_tx_burst(fs->tx_port, fs->tx_queue, tx_pkts_burst, + nb_prep); +@@ -1179,12 +1182,12 @@ tunnel_update: + /* + * Retry if necessary + */ +- if (unlikely(nb_tx < nb_rx) && fs->retry_enabled) { ++ if (unlikely(nb_tx < nb_prep) && fs->retry_enabled) { + retry = 0; +- while (nb_tx < nb_rx && retry++ < burst_tx_retry_num) { ++ while (nb_tx < nb_prep && retry++ < burst_tx_retry_num) { + rte_delay_us(burst_tx_delay_time); + nb_tx += rte_eth_tx_burst(fs->tx_port, fs->tx_queue, +- &tx_pkts_burst[nb_tx], nb_rx - nb_tx); ++ &tx_pkts_burst[nb_tx], nb_prep - nb_tx); + } + } + fs->tx_packets += nb_tx; +@@ -1194,11 +1197,11 @@ tunnel_update: + fs->rx_bad_outer_ip_csum += rx_bad_outer_ip_csum; + + inc_tx_burst_stats(fs, nb_tx); +- if (unlikely(nb_tx < nb_rx)) { +- fs->fwd_dropped += (nb_rx - nb_tx); ++ if (unlikely(nb_tx < nb_prep)) { ++ fs->fwd_dropped += (nb_prep - nb_tx); + do { + rte_pktmbuf_free(tx_pkts_burst[nb_tx]); +- } while (++nb_tx < nb_rx); ++ } while (++nb_tx < nb_prep); + } + + get_end_cycles(fs, start_tsc); +diff --git a/dpdk/app/test-pmd/ieee1588fwd.c b/dpdk/app/test-pmd/ieee1588fwd.c +index fc4e2d014c..896d5ef26a 100644 +--- a/dpdk/app/test-pmd/ieee1588fwd.c ++++ b/dpdk/app/test-pmd/ieee1588fwd.c +@@ -184,13 +184,13 @@ ieee1588_packet_fwd(struct fwd_stream *fs) + + /* Forward PTP packet with hardware TX timestamp */ + mb->ol_flags |= RTE_MBUF_F_TX_IEEE1588_TMST; +- fs->tx_packets += 1; + if (rte_eth_tx_burst(fs->rx_port, fs->tx_queue, &mb, 1) == 0) { + printf("Port %u sent PTP packet dropped\n", fs->rx_port); + fs->fwd_dropped += 1; + rte_pktmbuf_free(mb); + return; + } ++ fs->tx_packets += 1; + + /* + * Check the TX timestamp. +diff --git a/dpdk/app/test-pmd/noisy_vnf.c b/dpdk/app/test-pmd/noisy_vnf.c +index c65ec6f06a..abd99a0407 100644 +--- a/dpdk/app/test-pmd/noisy_vnf.c ++++ b/dpdk/app/test-pmd/noisy_vnf.c +@@ -214,9 +214,10 @@ flush: + sent = rte_eth_tx_burst(fs->tx_port, fs->tx_queue, + tmp_pkts, nb_deqd); + if (unlikely(sent < nb_deqd) && fs->retry_enabled) +- nb_tx += do_retry(nb_rx, nb_tx, tmp_pkts, fs); +- inc_tx_burst_stats(fs, nb_tx); ++ sent += do_retry(nb_deqd, sent, tmp_pkts, fs); ++ inc_tx_burst_stats(fs, sent); + fs->fwd_dropped += drop_pkts(tmp_pkts, nb_deqd, sent); ++ nb_tx += sent; + ncf->prev_time = rte_get_timer_cycles(); + } + } +diff --git a/dpdk/app/test-pmd/testpmd.c b/dpdk/app/test-pmd/testpmd.c +index 134d79a555..340c713c19 100644 +--- a/dpdk/app/test-pmd/testpmd.c ++++ b/dpdk/app/test-pmd/testpmd.c +@@ -231,7 +231,7 @@ unsigned int xstats_display_num; /**< Size of extended statistics to show */ + * In container, it cannot terminate the process which running with 'stats-period' + * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. + */ +-static volatile uint8_t f_quit; ++volatile uint8_t f_quit; + uint8_t cl_quit; /* Quit testpmd from cmdline. */ + + /* +@@ -2056,6 +2056,8 @@ fwd_stats_display(void) + fwd_cycles += fs->core_cycles; + } + for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { ++ uint64_t tx_dropped = 0; ++ + pt_id = fwd_ports_ids[i]; + port = &ports[pt_id]; + +@@ -2077,8 +2079,9 @@ fwd_stats_display(void) + total_recv += stats.ipackets; + total_xmit += stats.opackets; + total_rx_dropped += stats.imissed; +- total_tx_dropped += ports_stats[pt_id].tx_dropped; +- total_tx_dropped += stats.oerrors; ++ tx_dropped += ports_stats[pt_id].tx_dropped; ++ tx_dropped += stats.oerrors; ++ total_tx_dropped += tx_dropped; + total_rx_nombuf += stats.rx_nombuf; + + printf("\n %s Forward statistics for port %-2d %s\n", +@@ -2105,8 +2108,8 @@ fwd_stats_display(void) + + printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64 + "TX-total: %-"PRIu64"\n", +- stats.opackets, ports_stats[pt_id].tx_dropped, +- stats.opackets + ports_stats[pt_id].tx_dropped); ++ stats.opackets, tx_dropped, ++ stats.opackets + tx_dropped); + + if (record_burst_stats) { + if (ports_stats[pt_id].rx_stream) +@@ -2339,6 +2342,87 @@ launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) + } + } + ++static void ++update_rx_queue_state(uint16_t port_id, uint16_t queue_id) ++{ ++ struct rte_eth_rxq_info rx_qinfo; ++ int32_t rc; ++ ++ rc = rte_eth_rx_queue_info_get(port_id, ++ queue_id, &rx_qinfo); ++ if (rc == 0) { ++ ports[port_id].rxq[queue_id].state = ++ rx_qinfo.queue_state; ++ } else if (rc == -ENOTSUP) { ++ /* ++ * Do not change the rxq state for primary process ++ * to ensure that the PMDs do not implement ++ * rte_eth_rx_queue_info_get can forward as before. ++ */ ++ if (rte_eal_process_type() == RTE_PROC_PRIMARY) ++ return; ++ /* ++ * Set the rxq state to RTE_ETH_QUEUE_STATE_STARTED ++ * to ensure that the PMDs do not implement ++ * rte_eth_rx_queue_info_get can forward. ++ */ ++ ports[port_id].rxq[queue_id].state = ++ RTE_ETH_QUEUE_STATE_STARTED; ++ } else { ++ TESTPMD_LOG(WARNING, ++ "Failed to get rx queue info\n"); ++ } ++} ++ ++static void ++update_tx_queue_state(uint16_t port_id, uint16_t queue_id) ++{ ++ struct rte_eth_txq_info tx_qinfo; ++ int32_t rc; ++ ++ rc = rte_eth_tx_queue_info_get(port_id, ++ queue_id, &tx_qinfo); ++ if (rc == 0) { ++ ports[port_id].txq[queue_id].state = ++ tx_qinfo.queue_state; ++ } else if (rc == -ENOTSUP) { ++ /* ++ * Do not change the txq state for primary process ++ * to ensure that the PMDs do not implement ++ * rte_eth_tx_queue_info_get can forward as before. ++ */ ++ if (rte_eal_process_type() == RTE_PROC_PRIMARY) ++ return; ++ /* ++ * Set the txq state to RTE_ETH_QUEUE_STATE_STARTED ++ * to ensure that the PMDs do not implement ++ * rte_eth_tx_queue_info_get can forward. ++ */ ++ ports[port_id].txq[queue_id].state = ++ RTE_ETH_QUEUE_STATE_STARTED; ++ } else { ++ TESTPMD_LOG(WARNING, ++ "Failed to get tx queue info\n"); ++ } ++} ++ ++static void ++update_queue_state(portid_t pid) ++{ ++ portid_t pi; ++ queueid_t qi; ++ ++ RTE_ETH_FOREACH_DEV(pi) { ++ if (pid != pi && pid != (portid_t)RTE_PORT_ALL) ++ continue; ++ ++ for (qi = 0; qi < nb_rxq; qi++) ++ update_rx_queue_state(pi, qi); ++ for (qi = 0; qi < nb_txq; qi++) ++ update_tx_queue_state(pi, qi); ++ } ++} ++ + /* + * Launch packet forwarding configuration. + */ +@@ -2378,9 +2462,11 @@ start_packet_forwarding(int with_tx_first) + if (!pkt_fwd_shared_rxq_check()) + return; + +- if (stream_init != NULL) ++ if (stream_init != NULL) { ++ update_queue_state(RTE_PORT_ALL); + for (i = 0; i < cur_fwd_config.nb_fwd_streams; i++) + stream_init(fwd_streams[i]); ++ } + + port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; + if (port_fwd_begin != NULL) { +@@ -2880,7 +2966,7 @@ update_bonding_port_dev_conf(portid_t bond_pid) + int + start_port(portid_t pid) + { +- int diag, need_check_link_status = -1; ++ int diag; + portid_t pi; + portid_t p_pi = RTE_MAX_ETHPORTS; + portid_t pl[RTE_MAX_ETHPORTS]; +@@ -2891,6 +2977,9 @@ start_port(portid_t pid) + queueid_t qi; + struct rte_port *port; + struct rte_eth_hairpin_cap cap; ++ bool at_least_one_port_exist = false; ++ bool all_ports_already_started = true; ++ bool at_least_one_port_successfully_started = false; + + if (port_id_is_invalid(pid, ENABLED_WARN)) + return 0; +@@ -2906,11 +2995,13 @@ start_port(portid_t pid) + continue; + } + +- need_check_link_status = 0; ++ at_least_one_port_exist = true; ++ + port = &ports[pi]; +- if (port->port_status == RTE_PORT_STOPPED) ++ if (port->port_status == RTE_PORT_STOPPED) { + port->port_status = RTE_PORT_HANDLING; +- else { ++ all_ports_already_started = false; ++ } else { + fprintf(stderr, "Port %d is now not stopped\n", pi); + continue; + } +@@ -3130,15 +3221,16 @@ start_port(portid_t pid) + printf("Port %d: " RTE_ETHER_ADDR_PRT_FMT "\n", pi, + RTE_ETHER_ADDR_BYTES(&port->eth_addr)); + +- /* at least one port started, need checking link status */ +- need_check_link_status = 1; ++ at_least_one_port_successfully_started = true; + + pl[cfg_pi++] = pi; + } + +- if (need_check_link_status == 1 && !no_link_check) ++ update_queue_state(pi); ++ ++ if (at_least_one_port_successfully_started && !no_link_check) + check_all_ports_link_status(RTE_PORT_ALL); +- else if (need_check_link_status == 0) ++ else if (at_least_one_port_exist & all_ports_already_started) + fprintf(stderr, "Please stop the ports first\n"); + + if (hairpin_mode & 0xf) { +@@ -4315,13 +4407,6 @@ init_port(void) + memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); + } + +-static void +-force_quit(void) +-{ +- pmd_test_exit(); +- prompt_exit(); -} - - RTE_LOG_REGISTER_SUFFIX(i40e_logtype_init, init, NOTICE); - RTE_LOG_REGISTER_SUFFIX(i40e_logtype_driver, driver, NOTICE); - #ifdef RTE_ETHDEV_DEBUG_RX + static void + print_stats(void) + { +@@ -4340,28 +4425,10 @@ print_stats(void) + } + + static void +-signal_handler(int signum) ++signal_handler(int signum __rte_unused) + { +- if (signum == SIGINT || signum == SIGTERM) { +- fprintf(stderr, "\nSignal %d received, preparing to exit...\n", +- signum); +-#ifdef RTE_LIB_PDUMP +- /* uninitialize packet capture framework */ +- rte_pdump_uninit(); +-#endif +-#ifdef RTE_LIB_LATENCYSTATS +- if (latencystats_enabled != 0) +- rte_latencystats_uninit(); +-#endif +- force_quit(); +- /* Set flag to indicate the force termination. */ +- f_quit = 1; +- /* exit with the expected status */ +-#ifndef RTE_EXEC_ENV_WINDOWS +- signal(signum, SIG_DFL); +- kill(getpid(), signum); +-#endif +- } ++ f_quit = 1; ++ prompt_exit(); + } + + int +@@ -4372,8 +4439,18 @@ main(int argc, char** argv) + uint16_t count; + int ret; + ++#ifdef RTE_EXEC_ENV_WINDOWS + signal(SIGINT, signal_handler); + signal(SIGTERM, signal_handler); ++#else ++ /* Want read() not to be restarted on signal */ ++ struct sigaction action = { ++ .sa_handler = signal_handler, ++ }; ++ ++ sigaction(SIGINT, &action, NULL); ++ sigaction(SIGTERM, &action, NULL); ++#endif + + testpmd_logtype = rte_log_register("testpmd"); + if (testpmd_logtype < 0) +@@ -4385,6 +4462,9 @@ main(int argc, char** argv) + rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", + rte_strerror(rte_errno)); + ++ /* allocate port structures, and init them */ ++ init_port(); ++ + ret = register_eth_event_callback(); + if (ret != 0) + rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); +@@ -4403,9 +4483,6 @@ main(int argc, char** argv) + if (nb_ports == 0) + TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); + +- /* allocate port structures, and init them */ +- init_port(); +- + set_def_fwd_config(); + if (nb_lcores == 0) + rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" +@@ -4483,8 +4560,13 @@ main(int argc, char** argv) + } + } + +- if (!no_device_start && start_port(RTE_PORT_ALL) != 0) +- rte_exit(EXIT_FAILURE, "Start ports failed\n"); ++ if (!no_device_start && start_port(RTE_PORT_ALL) != 0) { ++ if (!interactive) { ++ rte_eal_cleanup(); ++ rte_exit(EXIT_FAILURE, "Start ports failed\n"); ++ } ++ fprintf(stderr, "Start ports failed\n"); ++ } + + /* set all ports to promiscuous mode by default */ + RTE_ETH_FOREACH_DEV(port_id) { +@@ -4536,15 +4618,9 @@ main(int argc, char** argv) + start_packet_forwarding(0); + } + prompt(); +- pmd_test_exit(); + } else + #endif + { +- char c; +- int rc; +- +- f_quit = 0; +- + printf("No commandline core given, start packet forwarding\n"); + start_packet_forwarding(tx_first); + if (stats_period != 0) { +@@ -4567,15 +4643,33 @@ main(int argc, char** argv) + prev_time = cur_time; + rte_delay_us_sleep(US_PER_S); + } +- } ++ } else { ++ char c; + +- printf("Press enter to exit\n"); +- rc = read(0, &c, 1); +- pmd_test_exit(); +- if (rc < 0) +- return 1; ++ printf("Press enter to exit\n"); ++ while (f_quit == 0) { ++ /* end-of-file or any character exits loop */ ++ if (read(0, &c, 1) >= 0) ++ break; ++ if (errno == EINTR) ++ continue; ++ rte_exit(EXIT_FAILURE, "Read failed: %s\n", ++ strerror(errno)); ++ } ++ } + } + ++ pmd_test_exit(); ++ ++#ifdef RTE_LIB_PDUMP ++ /* uninitialize packet capture framework */ ++ rte_pdump_uninit(); ++#endif ++#ifdef RTE_LIB_LATENCYSTATS ++ if (latencystats_enabled != 0) ++ rte_latencystats_uninit(); ++#endif ++ + ret = rte_eal_cleanup(); + if (ret != 0) + rte_exit(EXIT_FAILURE, +diff --git a/dpdk/app/test-pmd/testpmd.h b/dpdk/app/test-pmd/testpmd.h +index 7d24d25970..022210a7a9 100644 +--- a/dpdk/app/test-pmd/testpmd.h ++++ b/dpdk/app/test-pmd/testpmd.h +@@ -34,6 +34,7 @@ + #define RTE_PORT_HANDLING (uint16_t)3 + + extern uint8_t cl_quit; ++extern volatile uint8_t f_quit; + + /* + * It is used to allocate the memory for hash key. +diff --git a/dpdk/app/test/meson.build b/dpdk/app/test/meson.build +index f34d19e3c3..4e39c9e7cf 100644 +--- a/dpdk/app/test/meson.build ++++ b/dpdk/app/test/meson.build +@@ -206,6 +206,7 @@ fast_tests = [ + ['memzone_autotest', false, true], + ['meter_autotest', true, true], + ['multiprocess_autotest', false, false], ++ ['node_list_dump', true, true], + ['per_lcore_autotest', true, true], + ['pflock_autotest', true, true], + ['prefetch_autotest', true, true], +@@ -320,6 +321,10 @@ driver_test_names = [ + dump_test_names = [] + + if not is_windows ++ fast_tests += [['graph_autotest', true, true]] ++ ++ perf_test_names += 'graph_perf_autotest' ++ + driver_test_names += [ + 'cryptodev_openssl_asym_autotest', + 'eventdev_selftest_octeontx', +diff --git a/dpdk/app/test/packet_burst_generator.c b/dpdk/app/test/packet_burst_generator.c +index 6b42b9b83b..867a88da00 100644 +--- a/dpdk/app/test/packet_burst_generator.c ++++ b/dpdk/app/test/packet_burst_generator.c +@@ -263,11 +263,11 @@ generate_packet_burst(struct rte_mempool *mp, struct rte_mbuf **pkts_burst, + void *ip_hdr, uint8_t ipv4, struct rte_udp_hdr *udp_hdr, + int nb_pkt_per_burst, uint8_t pkt_len, uint8_t nb_pkt_segs) + { +- int i, nb_pkt = 0; +- size_t eth_hdr_size; +- ++ const uint8_t pkt_seg_data_len = pkt_len / nb_pkt_segs; + struct rte_mbuf *pkt_seg; + struct rte_mbuf *pkt; ++ size_t eth_hdr_size; ++ int i, nb_pkt = 0; + + for (nb_pkt = 0; nb_pkt < nb_pkt_per_burst; nb_pkt++) { + pkt = rte_pktmbuf_alloc(mp); +@@ -278,7 +278,7 @@ nomore_mbuf: + break; + } + +- pkt->data_len = pkt_len; ++ pkt->data_len = pkt_seg_data_len; + pkt_seg = pkt; + for (i = 1; i < nb_pkt_segs; i++) { + pkt_seg->next = rte_pktmbuf_alloc(mp); +@@ -288,7 +288,10 @@ nomore_mbuf: + goto nomore_mbuf; + } + pkt_seg = pkt_seg->next; +- pkt_seg->data_len = pkt_len; ++ if (i != nb_pkt_segs - 1) ++ pkt_seg->data_len = pkt_seg_data_len; ++ else ++ pkt_seg->data_len = pkt_seg_data_len + pkt_len % nb_pkt_segs; + } + pkt_seg->next = NULL; /* Last segment of packet. */ + +@@ -344,11 +347,11 @@ generate_packet_burst_proto(struct rte_mempool *mp, + uint8_t ipv4, uint8_t proto, void *proto_hdr, + int nb_pkt_per_burst, uint8_t pkt_len, uint8_t nb_pkt_segs) + { +- int i, nb_pkt = 0; +- size_t eth_hdr_size; +- ++ const uint8_t pkt_seg_data_len = pkt_len / nb_pkt_segs; + struct rte_mbuf *pkt_seg; + struct rte_mbuf *pkt; ++ size_t eth_hdr_size; ++ int i, nb_pkt = 0; + + for (nb_pkt = 0; nb_pkt < nb_pkt_per_burst; nb_pkt++) { + pkt = rte_pktmbuf_alloc(mp); +@@ -359,7 +362,7 @@ nomore_mbuf: + break; + } + +- pkt->data_len = pkt_len; ++ pkt->data_len = pkt_seg_data_len; + pkt_seg = pkt; + for (i = 1; i < nb_pkt_segs; i++) { + pkt_seg->next = rte_pktmbuf_alloc(mp); +@@ -369,7 +372,10 @@ nomore_mbuf: + goto nomore_mbuf; + } + pkt_seg = pkt_seg->next; +- pkt_seg->data_len = pkt_len; ++ if (i != nb_pkt_segs - 1) ++ pkt_seg->data_len = pkt_seg_data_len; ++ else ++ pkt_seg->data_len = pkt_seg_data_len + pkt_len % nb_pkt_segs; + } + pkt_seg->next = NULL; /* Last segment of packet. */ + +diff --git a/dpdk/app/test/test.h b/dpdk/app/test/test.h +index 85f57efbc6..6a4fa0b1d7 100644 +--- a/dpdk/app/test/test.h ++++ b/dpdk/app/test/test.h +@@ -127,7 +127,7 @@ struct unit_test_case { + { setup, teardown, NULL, testcase, #testcase, 1, data } + + #define TEST_CASE_NAMED_ST(name, setup, teardown, testcase) \ +- { setup, teardown, NULL, testcase, name, 1, NULL } ++ { setup, teardown, testcase, NULL, name, 1, NULL } + + #define TEST_CASE_NAMED_WITH_DATA(name, setup, teardown, testcase, data) \ + { setup, teardown, NULL, testcase, name, 1, data } +diff --git a/dpdk/app/test/test_cryptodev.c b/dpdk/app/test/test_cryptodev.c +index d6ae762df9..b75edb2f2b 100644 +--- a/dpdk/app/test/test_cryptodev.c ++++ b/dpdk/app/test/test_cryptodev.c +@@ -136,6 +136,17 @@ security_proto_supported(enum rte_security_session_action_type action, + static int + dev_configure_and_start(uint64_t ff_disable); + ++static int ++check_cipher_capability(const struct crypto_testsuite_params *ts_params, ++ const enum rte_crypto_cipher_algorithm cipher_algo, ++ const uint16_t key_size, const uint16_t iv_size); ++ ++static int ++check_auth_capability(const struct crypto_testsuite_params *ts_params, ++ const enum rte_crypto_auth_algorithm auth_algo, ++ const uint16_t key_size, const uint16_t iv_size, ++ const uint16_t tag_size); ++ + static struct rte_mbuf * + setup_test_string(struct rte_mempool *mpool, + const char *string, size_t len, uint8_t blocksize) +@@ -4761,7 +4772,6 @@ test_zuc_cipher_auth(const struct wireless_test_data *tdata) + unsigned int plaintext_len; + + struct rte_cryptodev_info dev_info; +- struct rte_cryptodev_sym_capability_idx cap_idx; + + rte_cryptodev_info_get(ts_params->valid_devs[0], &dev_info); + uint64_t feat_flags = dev_info.feature_flags; +@@ -4783,19 +4793,14 @@ test_zuc_cipher_auth(const struct wireless_test_data *tdata) + return TEST_SKIPPED; + + /* Check if device supports ZUC EEA3 */ +- cap_idx.type = RTE_CRYPTO_SYM_XFORM_CIPHER; +- cap_idx.algo.cipher = RTE_CRYPTO_CIPHER_ZUC_EEA3; +- +- if (rte_cryptodev_sym_capability_get(ts_params->valid_devs[0], +- &cap_idx) == NULL) ++ if (check_cipher_capability(ts_params, RTE_CRYPTO_CIPHER_ZUC_EEA3, ++ tdata->key.len, tdata->cipher_iv.len) < 0) + return TEST_SKIPPED; + + /* Check if device supports ZUC EIA3 */ +- cap_idx.type = RTE_CRYPTO_SYM_XFORM_AUTH; +- cap_idx.algo.auth = RTE_CRYPTO_AUTH_ZUC_EIA3; +- +- if (rte_cryptodev_sym_capability_get(ts_params->valid_devs[0], +- &cap_idx) == NULL) ++ if (check_auth_capability(ts_params, RTE_CRYPTO_AUTH_ZUC_EIA3, ++ tdata->key.len, tdata->auth_iv.len, ++ tdata->digest.len) < 0) + return TEST_SKIPPED; + + /* Create ZUC session */ +@@ -4853,7 +4858,7 @@ test_zuc_cipher_auth(const struct wireless_test_data *tdata) + TEST_ASSERT_BUFFERS_ARE_EQUAL( + ut_params->digest, + tdata->digest.data, +- 4, ++ tdata->digest.len, + "ZUC Generated auth tag not as expected"); + return 0; + } +@@ -6253,6 +6258,9 @@ test_zuc_auth_cipher(const struct wireless_test_data *tdata, + tdata->digest.len) < 0) + return TEST_SKIPPED; + ++ if (gbl_action_type == RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO) ++ return TEST_SKIPPED; ++ + rte_cryptodev_info_get(ts_params->valid_devs[0], &dev_info); + + uint64_t feat_flags = dev_info.feature_flags; +@@ -6415,7 +6423,7 @@ test_zuc_auth_cipher(const struct wireless_test_data *tdata, + TEST_ASSERT_BUFFERS_ARE_EQUAL( + ut_params->digest, + tdata->digest.data, +- DIGEST_BYTE_LENGTH_KASUMI_F9, ++ tdata->digest.len, + "ZUC Generated auth tag not as expected"); + } + return 0; +@@ -6453,6 +6461,9 @@ test_zuc_auth_cipher_sgl(const struct wireless_test_data *tdata, + tdata->digest.len) < 0) + return TEST_SKIPPED; + ++ if (gbl_action_type == RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO) ++ return TEST_SKIPPED; ++ + rte_cryptodev_info_get(ts_params->valid_devs[0], &dev_info); + + uint64_t feat_flags = dev_info.feature_flags; +@@ -6622,7 +6633,7 @@ test_zuc_auth_cipher_sgl(const struct wireless_test_data *tdata, + TEST_ASSERT_BUFFERS_ARE_EQUAL( + digest, + tdata->digest.data, +- DIGEST_BYTE_LENGTH_KASUMI_F9, ++ tdata->digest.len, + "ZUC Generated auth tag not as expected"); + } + return 0; +@@ -6852,6 +6863,7 @@ snow3g_hash_test_vector_setup(const struct snow3g_test_data *pattern, + static int + test_snow3g_decryption_with_digest_test_case_1(void) + { ++ int ret; + struct snow3g_hash_test_data snow3g_hash_data; + struct rte_cryptodev_info dev_info; + struct crypto_testsuite_params *ts_params = &testsuite_params; +@@ -6870,8 +6882,9 @@ test_snow3g_decryption_with_digest_test_case_1(void) + */ + snow3g_hash_test_vector_setup(&snow3g_test_case_7, &snow3g_hash_data); + +- if (test_snow3g_decryption(&snow3g_test_case_7)) +- return TEST_FAILED; ++ ret = test_snow3g_decryption(&snow3g_test_case_7); ++ if (ret != 0) ++ return ret; + + return test_snow3g_authentication_verify(&snow3g_hash_data); + } +@@ -7626,6 +7639,9 @@ test_mixed_auth_cipher_sgl(const struct mixed_cipher_auth_test_data *tdata, + if (global_api_test_type == CRYPTODEV_RAW_API_TEST) + return TEST_SKIPPED; + ++ if (gbl_action_type == RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO) ++ return TEST_SKIPPED; ++ + rte_cryptodev_info_get(ts_params->valid_devs[0], &dev_info); + + uint64_t feat_flags = dev_info.feature_flags; +@@ -7648,6 +7664,9 @@ test_mixed_auth_cipher_sgl(const struct mixed_cipher_auth_test_data *tdata, + } + } + ++ if (gbl_action_type == RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO) ++ return TEST_SKIPPED; ++ + /* Create the session */ + if (verify) + retval = create_wireless_algo_cipher_auth_session( +@@ -8433,7 +8452,7 @@ test_authenticated_encryption(const struct aead_test_data *tdata) + tdata->key.data, tdata->key.len, + tdata->aad.len, tdata->auth_tag.len, + tdata->iv.len); +- if (retval < 0) ++ if (retval != TEST_SUCCESS) + return retval; + + if (tdata->aad.len > MBUF_SIZE) { +@@ -9809,7 +9828,7 @@ test_ipsec_ah_proto_all(const struct ipsec_test_flags *flags) + } + + static int +-test_ipsec_proto_display_list(const void *data __rte_unused) ++test_ipsec_proto_display_list(void) + { + struct ipsec_test_flags flags; + +@@ -9821,7 +9840,7 @@ test_ipsec_proto_display_list(const void *data __rte_unused) + } + + static int +-test_ipsec_proto_ah_tunnel_ipv4(const void *data __rte_unused) ++test_ipsec_proto_ah_tunnel_ipv4(void) + { + struct ipsec_test_flags flags; + +@@ -9834,7 +9853,7 @@ test_ipsec_proto_ah_tunnel_ipv4(const void *data __rte_unused) + } + + static int +-test_ipsec_proto_ah_transport_ipv4(const void *data __rte_unused) ++test_ipsec_proto_ah_transport_ipv4(void) + { + struct ipsec_test_flags flags; + +@@ -9847,7 +9866,7 @@ test_ipsec_proto_ah_transport_ipv4(const void *data __rte_unused) + } + + static int +-test_ipsec_proto_iv_gen(const void *data __rte_unused) ++test_ipsec_proto_iv_gen(void) + { + struct ipsec_test_flags flags; + +@@ -9859,7 +9878,7 @@ test_ipsec_proto_iv_gen(const void *data __rte_unused) + } + + static int +-test_ipsec_proto_sa_exp_pkts_soft(const void *data __rte_unused) ++test_ipsec_proto_sa_exp_pkts_soft(void) + { + struct ipsec_test_flags flags; + +@@ -9871,7 +9890,7 @@ test_ipsec_proto_sa_exp_pkts_soft(const void *data __rte_unused) + } + + static int +-test_ipsec_proto_sa_exp_pkts_hard(const void *data __rte_unused) ++test_ipsec_proto_sa_exp_pkts_hard(void) + { + struct ipsec_test_flags flags; + +@@ -9883,7 +9902,7 @@ test_ipsec_proto_sa_exp_pkts_hard(const void *data __rte_unused) + } + + static int +-test_ipsec_proto_err_icv_corrupt(const void *data __rte_unused) ++test_ipsec_proto_err_icv_corrupt(void) + { + struct ipsec_test_flags flags; + +@@ -9895,7 +9914,7 @@ test_ipsec_proto_err_icv_corrupt(const void *data __rte_unused) + } + + static int +-test_ipsec_proto_udp_encap_custom_ports(const void *data __rte_unused) ++test_ipsec_proto_udp_encap_custom_ports(void) + { + struct ipsec_test_flags flags; + +@@ -9912,7 +9931,7 @@ test_ipsec_proto_udp_encap_custom_ports(const void *data __rte_unused) + } + + static int +-test_ipsec_proto_udp_encap(const void *data __rte_unused) ++test_ipsec_proto_udp_encap(void) + { + struct ipsec_test_flags flags; + +@@ -9924,7 +9943,7 @@ test_ipsec_proto_udp_encap(const void *data __rte_unused) + } + + static int +-test_ipsec_proto_tunnel_src_dst_addr_verify(const void *data __rte_unused) ++test_ipsec_proto_tunnel_src_dst_addr_verify(void) + { + struct ipsec_test_flags flags; + +@@ -9936,7 +9955,7 @@ test_ipsec_proto_tunnel_src_dst_addr_verify(const void *data __rte_unused) + } + + static int +-test_ipsec_proto_tunnel_dst_addr_verify(const void *data __rte_unused) ++test_ipsec_proto_tunnel_dst_addr_verify(void) + { + struct ipsec_test_flags flags; + +@@ -9948,7 +9967,7 @@ test_ipsec_proto_tunnel_dst_addr_verify(const void *data __rte_unused) + } + + static int +-test_ipsec_proto_udp_ports_verify(const void *data __rte_unused) ++test_ipsec_proto_udp_ports_verify(void) + { + struct ipsec_test_flags flags; + +@@ -9961,7 +9980,7 @@ test_ipsec_proto_udp_ports_verify(const void *data __rte_unused) + } + + static int +-test_ipsec_proto_inner_ip_csum(const void *data __rte_unused) ++test_ipsec_proto_inner_ip_csum(void) + { + struct ipsec_test_flags flags; + +@@ -9973,7 +9992,7 @@ test_ipsec_proto_inner_ip_csum(const void *data __rte_unused) + } + + static int +-test_ipsec_proto_inner_l4_csum(const void *data __rte_unused) ++test_ipsec_proto_inner_l4_csum(void) + { + struct ipsec_test_flags flags; + +@@ -9985,7 +10004,7 @@ test_ipsec_proto_inner_l4_csum(const void *data __rte_unused) + } + + static int +-test_ipsec_proto_tunnel_v4_in_v4(const void *data __rte_unused) ++test_ipsec_proto_tunnel_v4_in_v4(void) + { + struct ipsec_test_flags flags; + +@@ -9998,7 +10017,7 @@ test_ipsec_proto_tunnel_v4_in_v4(const void *data __rte_unused) + } + + static int +-test_ipsec_proto_tunnel_v6_in_v6(const void *data __rte_unused) ++test_ipsec_proto_tunnel_v6_in_v6(void) + { + struct ipsec_test_flags flags; + +@@ -10011,7 +10030,7 @@ test_ipsec_proto_tunnel_v6_in_v6(const void *data __rte_unused) + } + + static int +-test_ipsec_proto_tunnel_v4_in_v6(const void *data __rte_unused) ++test_ipsec_proto_tunnel_v4_in_v6(void) + { + struct ipsec_test_flags flags; + +@@ -10024,7 +10043,7 @@ test_ipsec_proto_tunnel_v4_in_v6(const void *data __rte_unused) + } + + static int +-test_ipsec_proto_tunnel_v6_in_v4(const void *data __rte_unused) ++test_ipsec_proto_tunnel_v6_in_v4(void) + { + struct ipsec_test_flags flags; + +@@ -10037,7 +10056,7 @@ test_ipsec_proto_tunnel_v6_in_v4(const void *data __rte_unused) + } + + static int +-test_ipsec_proto_transport_v4(const void *data __rte_unused) ++test_ipsec_proto_transport_v4(void) + { + struct ipsec_test_flags flags; + +@@ -10050,7 +10069,7 @@ test_ipsec_proto_transport_v4(const void *data __rte_unused) + } + + static int +-test_ipsec_proto_transport_l4_csum(const void *data __rte_unused) ++test_ipsec_proto_transport_l4_csum(void) + { + struct ipsec_test_flags flags = { + .l4_csum = true, +@@ -10061,7 +10080,7 @@ test_ipsec_proto_transport_l4_csum(const void *data __rte_unused) + } + + static int +-test_ipsec_proto_stats(const void *data __rte_unused) ++test_ipsec_proto_stats(void) + { + struct ipsec_test_flags flags; + +@@ -10073,7 +10092,7 @@ test_ipsec_proto_stats(const void *data __rte_unused) + } + + static int +-test_ipsec_proto_pkt_fragment(const void *data __rte_unused) ++test_ipsec_proto_pkt_fragment(void) + { + struct ipsec_test_flags flags; + +@@ -10086,7 +10105,7 @@ test_ipsec_proto_pkt_fragment(const void *data __rte_unused) + } + + static int +-test_ipsec_proto_copy_df_inner_0(const void *data __rte_unused) ++test_ipsec_proto_copy_df_inner_0(void) + { + struct ipsec_test_flags flags; + +@@ -10098,7 +10117,7 @@ test_ipsec_proto_copy_df_inner_0(const void *data __rte_unused) + } + + static int +-test_ipsec_proto_copy_df_inner_1(const void *data __rte_unused) ++test_ipsec_proto_copy_df_inner_1(void) + { + struct ipsec_test_flags flags; + +@@ -10110,7 +10129,7 @@ test_ipsec_proto_copy_df_inner_1(const void *data __rte_unused) + } + + static int +-test_ipsec_proto_set_df_0_inner_1(const void *data __rte_unused) ++test_ipsec_proto_set_df_0_inner_1(void) + { + struct ipsec_test_flags flags; + +@@ -10122,7 +10141,7 @@ test_ipsec_proto_set_df_0_inner_1(const void *data __rte_unused) + } + + static int +-test_ipsec_proto_set_df_1_inner_0(const void *data __rte_unused) ++test_ipsec_proto_set_df_1_inner_0(void) + { + struct ipsec_test_flags flags; + +@@ -10134,7 +10153,7 @@ test_ipsec_proto_set_df_1_inner_0(const void *data __rte_unused) + } + + static int +-test_ipsec_proto_ipv4_copy_dscp_inner_0(const void *data __rte_unused) ++test_ipsec_proto_ipv4_copy_dscp_inner_0(void) + { + struct ipsec_test_flags flags; + +@@ -10146,7 +10165,7 @@ test_ipsec_proto_ipv4_copy_dscp_inner_0(const void *data __rte_unused) + } + + static int +-test_ipsec_proto_ipv4_copy_dscp_inner_1(const void *data __rte_unused) ++test_ipsec_proto_ipv4_copy_dscp_inner_1(void) + { + struct ipsec_test_flags flags; + +@@ -10158,7 +10177,7 @@ test_ipsec_proto_ipv4_copy_dscp_inner_1(const void *data __rte_unused) + } + + static int +-test_ipsec_proto_ipv4_set_dscp_0_inner_1(const void *data __rte_unused) ++test_ipsec_proto_ipv4_set_dscp_0_inner_1(void) + { + struct ipsec_test_flags flags; + +@@ -10174,7 +10193,7 @@ test_ipsec_proto_ipv4_set_dscp_0_inner_1(const void *data __rte_unused) + } + + static int +-test_ipsec_proto_ipv4_set_dscp_1_inner_0(const void *data __rte_unused) ++test_ipsec_proto_ipv4_set_dscp_1_inner_0(void) + { + struct ipsec_test_flags flags; + +@@ -10190,7 +10209,7 @@ test_ipsec_proto_ipv4_set_dscp_1_inner_0(const void *data __rte_unused) + } + + static int +-test_ipsec_proto_ipv6_copy_dscp_inner_0(const void *data __rte_unused) ++test_ipsec_proto_ipv6_copy_dscp_inner_0(void) + { + struct ipsec_test_flags flags; + +@@ -10204,7 +10223,7 @@ test_ipsec_proto_ipv6_copy_dscp_inner_0(const void *data __rte_unused) + } + + static int +-test_ipsec_proto_ipv6_copy_dscp_inner_1(const void *data __rte_unused) ++test_ipsec_proto_ipv6_copy_dscp_inner_1(void) + { + struct ipsec_test_flags flags; + +@@ -10218,7 +10237,7 @@ test_ipsec_proto_ipv6_copy_dscp_inner_1(const void *data __rte_unused) + } + + static int +-test_ipsec_proto_ipv6_set_dscp_0_inner_1(const void *data __rte_unused) ++test_ipsec_proto_ipv6_set_dscp_0_inner_1(void) + { + struct ipsec_test_flags flags; + +@@ -10236,7 +10255,7 @@ test_ipsec_proto_ipv6_set_dscp_0_inner_1(const void *data __rte_unused) + } + + static int +-test_ipsec_proto_ipv6_set_dscp_1_inner_0(const void *data __rte_unused) ++test_ipsec_proto_ipv6_set_dscp_1_inner_0(void) + { + struct ipsec_test_flags flags; + +@@ -10442,7 +10461,7 @@ test_PDCP_PROTO_all(void) + } + + static int +-test_ipsec_proto_ipv4_ttl_decrement(const void *data __rte_unused) ++test_ipsec_proto_ipv4_ttl_decrement(void) + { + struct ipsec_test_flags flags = { + .dec_ttl_or_hop_limit = true +@@ -10452,7 +10471,7 @@ test_ipsec_proto_ipv4_ttl_decrement(const void *data __rte_unused) + } + + static int +-test_ipsec_proto_ipv6_hop_limit_decrement(const void *data __rte_unused) ++test_ipsec_proto_ipv6_hop_limit_decrement(void) + { + struct ipsec_test_flags flags = { + .ipv6 = true, +@@ -11567,7 +11586,7 @@ test_authenticated_decryption_oop(const struct aead_test_data *tdata) + tdata->key.data, tdata->key.len, + tdata->aad.len, tdata->auth_tag.len, + tdata->iv.len); +- if (retval < 0) ++ if (retval != TEST_SUCCESS) + return retval; + + /* alloc mbuf and set payload */ +@@ -11981,11 +12000,11 @@ test_stats(void) + TEST_ASSERT((stats.enqueued_count == 1), + "rte_cryptodev_stats_get returned unexpected enqueued stat"); + TEST_ASSERT((stats.dequeued_count == 1), +- "rte_cryptodev_stats_get returned unexpected enqueued stat"); ++ "rte_cryptodev_stats_get returned unexpected dequeued stat"); + TEST_ASSERT((stats.enqueue_err_count == 0), +- "rte_cryptodev_stats_get returned unexpected enqueued stat"); ++ "rte_cryptodev_stats_get returned unexpected enqueued error count stat"); + TEST_ASSERT((stats.dequeue_err_count == 0), +- "rte_cryptodev_stats_get returned unexpected enqueued stat"); ++ "rte_cryptodev_stats_get returned unexpected dequeued error count stat"); + + /* invalid device but should ignore and not reset device stats*/ + rte_cryptodev_stats_reset(ts_params->valid_devs[0] + 300); +@@ -11993,7 +12012,7 @@ test_stats(void) + &stats), + "rte_cryptodev_stats_get failed"); + TEST_ASSERT((stats.enqueued_count == 1), +- "rte_cryptodev_stats_get returned unexpected enqueued stat"); ++ "rte_cryptodev_stats_get returned unexpected enqueued stat after invalid reset"); + + /* check that a valid reset clears stats */ + rte_cryptodev_stats_reset(ts_params->valid_devs[0]); +@@ -12001,9 +12020,9 @@ test_stats(void) + &stats), + "rte_cryptodev_stats_get failed"); + TEST_ASSERT((stats.enqueued_count == 0), +- "rte_cryptodev_stats_get returned unexpected enqueued stat"); ++ "rte_cryptodev_stats_get returned unexpected enqueued stat after valid reset"); + TEST_ASSERT((stats.dequeued_count == 0), +- "rte_cryptodev_stats_get returned unexpected enqueued stat"); ++ "rte_cryptodev_stats_get returned unexpected dequeued stat after valid reset"); + + return TEST_SUCCESS; + } +@@ -12990,7 +13009,7 @@ test_AES_GMAC_authentication(const struct gmac_test_data *tdata) + retval = create_gmac_session(ts_params->valid_devs[0], + tdata, RTE_CRYPTO_AUTH_OP_GENERATE); + +- if (retval == -ENOTSUP) ++ if (retval == TEST_SKIPPED) + return TEST_SKIPPED; + if (retval < 0) + return retval; +@@ -13121,7 +13140,7 @@ test_AES_GMAC_authentication_verify(const struct gmac_test_data *tdata) + retval = create_gmac_session(ts_params->valid_devs[0], + tdata, RTE_CRYPTO_AUTH_OP_VERIFY); + +- if (retval == -ENOTSUP) ++ if (retval == TEST_SKIPPED) + return TEST_SKIPPED; + if (retval < 0) + return retval; +@@ -13250,7 +13269,7 @@ test_AES_GMAC_authentication_SGL(const struct gmac_test_data *tdata, + retval = create_gmac_session(ts_params->valid_devs[0], + tdata, RTE_CRYPTO_AUTH_OP_GENERATE); + +- if (retval == -ENOTSUP) ++ if (retval == TEST_SKIPPED) + return TEST_SKIPPED; + if (retval < 0) + return retval; +@@ -13867,7 +13886,7 @@ test_authentication_verify_fail_when_data_corruption( + reference, + RTE_CRYPTO_AUTH_OP_VERIFY); + +- if (retval == -ENOTSUP) ++ if (retval == TEST_SKIPPED) + return TEST_SKIPPED; + if (retval < 0) + return retval; +@@ -13954,6 +13973,8 @@ test_authentication_verify_GMAC_fail_when_corruption( + reference, + RTE_CRYPTO_AUTH_OP_VERIFY, + RTE_CRYPTO_CIPHER_OP_DECRYPT); ++ if (retval == TEST_SKIPPED) ++ return TEST_SKIPPED; + if (retval < 0) + return retval; + +@@ -14044,8 +14065,7 @@ test_authenticated_decryption_fail_when_corruption( + reference, + RTE_CRYPTO_AUTH_OP_VERIFY, + RTE_CRYPTO_CIPHER_OP_DECRYPT); +- +- if (retval == -ENOTSUP) ++ if (retval == TEST_SKIPPED) + return TEST_SKIPPED; + if (retval < 0) + return retval; +@@ -14450,8 +14470,13 @@ test_authenticated_encryption_SGL(const struct aead_test_data *tdata, + &cap_idx) == NULL) + return TEST_SKIPPED; + +- /* OOP not supported with CPU crypto */ +- if (oop && gbl_action_type == RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO) ++ /* ++ * SGL not supported on AESNI_MB PMD CPU crypto, ++ * OOP not supported on AESNI_GCM CPU crypto ++ */ ++ if (gbl_action_type == RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO && ++ (gbl_driver_id == rte_cryptodev_driver_id_get( ++ RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD)) || oop)) + return TEST_SKIPPED; + + /* Detailed check for the particular SGL support flag */ +diff --git a/dpdk/app/test/test_cryptodev_aes_test_vectors.h b/dpdk/app/test/test_cryptodev_aes_test_vectors.h +index ea7b21ce53..f3686beeb5 100644 +--- a/dpdk/app/test/test_cryptodev_aes_test_vectors.h ++++ b/dpdk/app/test/test_cryptodev_aes_test_vectors.h +@@ -4969,7 +4969,7 @@ static const struct blockcipher_test_case aes_cipheronly_test_cases[] = { + }, + { + .test_descr = "AES-256-XTS Encryption (512-byte plaintext" +- " Dataunit 512) Scater gather OOP", ++ " Dataunit 512) Scatter gather OOP", + .test_data = &aes_test_data_xts_wrapped_key_48_pt_512_du_512, + .op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT, + .feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP | +@@ -4979,7 +4979,7 @@ static const struct blockcipher_test_case aes_cipheronly_test_cases[] = { + }, + { + .test_descr = "AES-256-XTS Decryption (512-byte plaintext" +- " Dataunit 512) Scater gather OOP", ++ " Dataunit 512) Scatter gather OOP", + .test_data = &aes_test_data_xts_wrapped_key_48_pt_512_du_512, + .op_mask = BLOCKCIPHER_TEST_OP_DECRYPT, + .feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP | +@@ -4989,7 +4989,7 @@ static const struct blockcipher_test_case aes_cipheronly_test_cases[] = { + }, + { + .test_descr = "AES-256-XTS Encryption (512-byte plaintext" +- " Dataunit 0) Scater gather OOP", ++ " Dataunit 0) Scatter gather OOP", + .test_data = &aes_test_data_xts_wrapped_key_48_pt_512_du_0, + .op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT, + .feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP | +@@ -4999,7 +4999,7 @@ static const struct blockcipher_test_case aes_cipheronly_test_cases[] = { + }, + { + .test_descr = "AES-256-XTS Decryption (512-byte plaintext" +- " Dataunit 0) Scater gather OOP", ++ " Dataunit 0) Scatter gather OOP", + .test_data = &aes_test_data_xts_wrapped_key_48_pt_512_du_0, + .op_mask = BLOCKCIPHER_TEST_OP_DECRYPT, + .feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP | +@@ -5009,7 +5009,7 @@ static const struct blockcipher_test_case aes_cipheronly_test_cases[] = { + }, + { + .test_descr = "AES-256-XTS Encryption (4096-byte plaintext" +- " Dataunit 4096) Scater gather OOP", ++ " Dataunit 4096) Scatter gather OOP", + .test_data = &aes_test_data_xts_wrapped_key_48_pt_4096_du_4096, + .op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT, + .feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP | +@@ -5019,7 +5019,7 @@ static const struct blockcipher_test_case aes_cipheronly_test_cases[] = { + }, + { + .test_descr = "AES-256-XTS Decryption (4096-byte plaintext" +- " Dataunit 4096) Scater gather OOP", ++ " Dataunit 4096) Scatter gather OOP", + .test_data = &aes_test_data_xts_wrapped_key_48_pt_4096_du_4096, + .op_mask = BLOCKCIPHER_TEST_OP_DECRYPT, + .feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP | +@@ -5029,7 +5029,7 @@ static const struct blockcipher_test_case aes_cipheronly_test_cases[] = { + }, + { + .test_descr = "AES-256-XTS Encryption (4096-byte plaintext" +- " Dataunit 0) Scater gather OOP", ++ " Dataunit 0) Scatter gather OOP", + .test_data = &aes_test_data_xts_wrapped_key_48_pt_4096_du_0, + .op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT, + .feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP | +@@ -5039,7 +5039,7 @@ static const struct blockcipher_test_case aes_cipheronly_test_cases[] = { + }, + { + .test_descr = "AES-256-XTS Decryption (4096-byte plaintext" +- " Dataunit 0) Scater gather OOP", ++ " Dataunit 0) Scatter gather OOP", + .test_data = &aes_test_data_xts_wrapped_key_48_pt_4096_du_0, + .op_mask = BLOCKCIPHER_TEST_OP_DECRYPT, + .feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP | +diff --git a/dpdk/app/test/test_cryptodev_asym.c b/dpdk/app/test/test_cryptodev_asym.c +index c58c7f488b..67659cd1a6 100644 +--- a/dpdk/app/test/test_cryptodev_asym.c ++++ b/dpdk/app/test/test_cryptodev_asym.c +@@ -1602,7 +1602,7 @@ error_exit: + } + + static int +-test_dh_keygenration(void) ++test_dh_key_generation(void) + { + int status; + +@@ -2204,7 +2204,7 @@ static struct unit_test_suite cryptodev_openssl_asym_testsuite = { + TEST_CASE_ST(ut_setup_asym, ut_teardown_asym, test_capability), + TEST_CASE_ST(ut_setup_asym, ut_teardown_asym, test_dsa), + TEST_CASE_ST(ut_setup_asym, ut_teardown_asym, +- test_dh_keygenration), ++ test_dh_key_generation), + TEST_CASE_ST(ut_setup_asym, ut_teardown_asym, test_rsa_enc_dec), + TEST_CASE_ST(ut_setup_asym, ut_teardown_asym, + test_rsa_sign_verify), +diff --git a/dpdk/app/test/test_cryptodev_mixed_test_vectors.h b/dpdk/app/test/test_cryptodev_mixed_test_vectors.h +index 2816ecc6a4..f80903c87e 100644 +--- a/dpdk/app/test/test_cryptodev_mixed_test_vectors.h ++++ b/dpdk/app/test/test_cryptodev_mixed_test_vectors.h +@@ -478,8 +478,10 @@ struct mixed_cipher_auth_test_data auth_aes_cmac_cipher_snow_test_case_1 = { + }, + .cipher_iv = { + .data = { ++ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, ++ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + }, +- .len = 0, ++ .len = 16, + }, + .cipher = { + .len_bits = 516 << 3, +@@ -917,8 +919,10 @@ struct mixed_cipher_auth_test_data auth_aes_cmac_cipher_zuc_test_case_1 = { + }, + .cipher_iv = { + .data = { ++ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, ++ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + }, +- .len = 0, ++ .len = 16, + }, + .cipher = { + .len_bits = 516 << 3, +diff --git a/dpdk/app/test/test_cryptodev_security_ipsec_test_vectors.h b/dpdk/app/test/test_cryptodev_security_ipsec_test_vectors.h +index 2686bbeb62..6e60e32b9d 100644 +--- a/dpdk/app/test/test_cryptodev_security_ipsec_test_vectors.h ++++ b/dpdk/app/test/test_cryptodev_security_ipsec_test_vectors.h +@@ -417,7 +417,8 @@ struct ipsec_test_data pkt_aes_256_ccm = { + .op = RTE_CRYPTO_AEAD_OP_ENCRYPT, + .algo = RTE_CRYPTO_AEAD_AES_CCM, + .key.length = 32, +- .iv.length = 12, ++ /* IV includes 3B salt and 8B per packet IV */ ++ .iv.length = 11, + .iv.offset = IV_OFFSET, + .digest_length = 16, + .aad_length = 12, +diff --git a/dpdk/app/test/test_cryptodev_security_pdcp_sdap_test_vectors.h b/dpdk/app/test/test_cryptodev_security_pdcp_sdap_test_vectors.h +index f43f693edb..b0fa0ec458 100644 +--- a/dpdk/app/test/test_cryptodev_security_pdcp_sdap_test_vectors.h ++++ b/dpdk/app/test/test_cryptodev_security_pdcp_sdap_test_vectors.h +@@ -769,7 +769,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x0a, 0x98, 0xc1, 0x3c, 0x98, 0x82, + 0xdc, 0xb6, 0xc2, 0x36 }, + .data_in = +- (uint8_t[]){ 0x50, 0x01, 0x86, 0xb8, 0xf8, 0xdb, 0x2d, ++ (uint8_t[]){ 0x80, 0x01, 0x86, 0xb8, 0xf8, 0xdb, 0x2d, + 0x3f, 0x23, 0x82, 0x53, 0xfd, 0x37, 0xde, + 0x88, 0x63, 0x08, 0x4f, 0xd3, 0x71, 0xfb, + 0xeb, 0x35, 0xf3, 0x64, 0xd3, 0x5e, 0xaf, +@@ -781,7 +781,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x7a, 0xf9, 0xdd }, + .in_len = 66, + .data_out = +- (uint8_t[]){ 0x50, 0x01, 0x86, 0x69, 0xf2, 0x5d, 0xd7, ++ (uint8_t[]){ 0x80, 0x01, 0x86, 0x69, 0xf2, 0x5d, 0xd7, + 0xf5, 0xc1, 0xf7, 0x1e, 0x47, 0x5d, 0xce, + 0xbe, 0x48, 0xb5, 0x0b, 0x6a, 0x73, 0x9a, + 0x5a, 0xa3, 0x06, 0x47, 0x40, 0x96, 0xcf, +@@ -790,7 +790,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0xc9, 0x7f, 0x11, 0x79, 0x24, 0x44, 0xe8, + 0x39, 0x11, 0x03, 0x0a, 0x9d, 0x4f, 0xe2, + 0x95, 0x9f, 0x47, 0x73, 0x37, 0x83, 0x8c, +- 0x23, 0xfa, 0x16, 0x39, 0xf7, 0x15, 0x11 }, ++ 0x23, 0xfa, 0x16, 0xb2, 0xb0, 0x17, 0x4a }, + .sn_size = 12, + .hfn = 0x1, + .hfn_threshold = 0xfa558, +@@ -817,7 +817,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x0a, 0x98, 0xc1, 0x3c, 0x98, 0x82, + 0xdc, 0xb6, 0xc2, 0x36 }, + .data_in = +- (uint8_t[]){ 0x50, 0x01, 0x86, 0xb8, 0xf8, 0xdb, 0x2d, ++ (uint8_t[]){ 0x80, 0x01, 0x86, 0xb8, 0xf8, 0xdb, 0x2d, + 0x3f, 0x23, 0x82, 0x53, 0xfd, 0x37, 0xde, + 0x88, 0x63, 0x08, 0x4f, 0xd3, 0x71, 0xfb, + 0xeb, 0x35, 0xf3, 0x64, 0xd3, 0x5e, 0xaf, +@@ -829,7 +829,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x7a, 0xf9, 0xdd }, + .in_len = 66, + .data_out = +- (uint8_t[]){ 0x50, 0x01, 0x86, 0x22, 0x2a, 0x8f, 0x86, ++ (uint8_t[]){ 0x80, 0x01, 0x86, 0x22, 0x2a, 0x8f, 0x86, + 0x25, 0x92, 0xcd, 0xa9, 0xa1, 0xa0, 0xf5, + 0x86, 0x0d, 0xe8, 0xe4, 0xef, 0xe0, 0x78, + 0x45, 0x7c, 0x0c, 0x41, 0x5c, 0x8f, 0x07, +@@ -838,7 +838,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x35, 0xff, 0x7a, 0x42, 0xac, 0x94, 0x21, + 0x60, 0x1c, 0x13, 0xcc, 0x7e, 0x6b, 0x2f, + 0x6f, 0x91, 0x89, 0xc6, 0xd4, 0xe6, 0x17, +- 0xae, 0xde, 0xfb, 0x90, 0x62, 0x59, 0xcb }, ++ 0xae, 0xde, 0xfb, 0x19, 0xDa, 0x9a, 0xc2 }, + .sn_size = 12, + .hfn = 0x1, + .hfn_threshold = 0xfa558, +@@ -865,7 +865,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x0a, 0x98, 0xc1, 0x3c, 0x98, 0x82, + 0xdc, 0xb6, 0xc2, 0x36 }, + .data_in = +- (uint8_t[]){ 0x50, 0x01, 0x86, 0xb8, 0xf8, 0xdb, 0x2d, ++ (uint8_t[]){ 0x80, 0x01, 0x86, 0xb8, 0xf8, 0xdb, 0x2d, + 0x3f, 0x23, 0x82, 0x53, 0xfd, 0x37, 0xde, + 0x88, 0x63, 0x08, 0x4f, 0xd3, 0x71, 0xfb, + 0xeb, 0x35, 0xf3, 0x64, 0xd3, 0x5e, 0xaf, +@@ -877,7 +877,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x7a, 0xf9, 0xdd }, + .in_len = 66, + .data_out = +- (uint8_t[]){ 0x50, 0x01, 0x86, 0x69, 0xf2, 0x5d, 0xd7, ++ (uint8_t[]){ 0x80, 0x01, 0x86, 0x69, 0xf2, 0x5d, 0xd7, + 0xf5, 0xc1, 0xf7, 0x1e, 0x47, 0x5d, 0xce, + 0xbe, 0x48, 0xb5, 0x0b, 0x6a, 0x73, 0x9a, + 0x5a, 0xa3, 0x06, 0x47, 0x40, 0x96, 0xcf, +@@ -886,7 +886,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0xc9, 0x7f, 0x11, 0x79, 0x24, 0x44, 0xe8, + 0x39, 0x11, 0x03, 0x0a, 0x9d, 0x4f, 0xe2, + 0x95, 0x9f, 0x47, 0x73, 0x37, 0x83, 0x8c, +- 0x23, 0xfa, 0x16, 0x72, 0x3e, 0x14, 0xa9 }, ++ 0x23, 0xfa, 0x16, 0x6c, 0xcb, 0x92, 0xdf }, + .sn_size = 12, + .hfn = 0x1, + .hfn_threshold = 0xfa558, +@@ -913,7 +913,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x0a, 0x98, 0xc1, 0x3c, 0x98, 0x82, + 0xdc, 0xb6, 0xc2, 0x36 }, + .data_in = +- (uint8_t[]){ 0x50, 0x01, 0x86, 0xb8, 0xf8, 0xdb, 0x2d, ++ (uint8_t[]){ 0x80, 0x01, 0x86, 0xb8, 0xf8, 0xdb, 0x2d, + 0x3f, 0x23, 0x82, 0x53, 0xfd, 0x37, 0xde, + 0x88, 0x63, 0x08, 0x4f, 0xd3, 0x71, 0xfb, + 0xeb, 0x35, 0xf3, 0x64, 0xd3, 0x5e, 0xaf, +@@ -925,7 +925,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x7a, 0xf9, 0xdd }, + .in_len = 66, + .data_out = +- (uint8_t[]){ 0x50, 0x01, 0x86, 0x22, 0x2a, 0x8f, 0x86, ++ (uint8_t[]){ 0x80, 0x01, 0x86, 0x22, 0x2a, 0x8f, 0x86, + 0x25, 0x92, 0xcd, 0xa9, 0xa1, 0xa0, 0xf5, + 0x86, 0x0d, 0xe8, 0xe4, 0xef, 0xe0, 0x78, + 0x45, 0x7c, 0x0c, 0x41, 0x5c, 0x8f, 0x07, +@@ -934,7 +934,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x35, 0xff, 0x7a, 0x42, 0xac, 0x94, 0x21, + 0x60, 0x1c, 0x13, 0xcc, 0x7e, 0x6b, 0x2f, + 0x6f, 0x91, 0x89, 0xc6, 0xd4, 0xe6, 0x17, +- 0xae, 0xde, 0xfb, 0x3f, 0x47, 0xaa, 0x9b }, ++ 0xae, 0xde, 0xfb, 0x5b, 0xc2, 0x9f, 0x29 }, + .sn_size = 12, + .hfn = 0x1, + .hfn_threshold = 0xfa558, +@@ -961,7 +961,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x0a, 0x98, 0xc1, 0x3c, 0x98, 0x82, + 0xdc, 0xb6, 0xc2, 0x36 }, + .data_in = +- (uint8_t[]){ 0x50, 0x01, 0x86, 0xb8, 0xf8, 0xdb, 0x2d, ++ (uint8_t[]){ 0x80, 0x01, 0x86, 0xb8, 0xf8, 0xdb, 0x2d, + 0x3f, 0x23, 0x82, 0x53, 0xfd, 0x37, 0xde, + 0x88, 0x63, 0x08, 0x4f, 0xd3, 0x71, 0xfb, + 0xeb, 0x35, 0xf3, 0x64, 0xd3, 0x5e, 0xaf, +@@ -974,7 +974,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + .in_len = 66, + .data_out = + (uint8_t[]){ +- 0x50, 0x01, 0x86, 0x69, 0xf2, 0x5d, 0xd7, 0xf5, ++ 0x80, 0x01, 0x86, 0x69, 0xf2, 0x5d, 0xd7, 0xf5, + 0xc1, 0xf7, 0x1e, 0x47, 0x5d, 0xce, 0xbe, 0x48, + 0xb5, 0x0b, 0x6a, 0x73, 0x9a, 0x5a, 0xa3, 0x06, + 0x47, 0x40, 0x96, 0xcf, 0x86, 0x98, 0x3d, 0x6f, +@@ -982,7 +982,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0xa6, 0x24, 0xc9, 0x7f, 0x11, 0x79, 0x24, 0x44, + 0xe8, 0x39, 0x11, 0x03, 0x0a, 0x9d, 0x4f, 0xe2, + 0x95, 0x9f, 0x47, 0x73, 0x37, 0x83, 0x8c, 0x23, +- 0xfa, 0x16, 0x52, 0x69, 0x16, 0xfc, ++ 0xfa, 0x16, 0x5d, 0x83, 0x73, 0x34, + }, + .sn_size = 12, + .hfn = 0x1, +@@ -1010,7 +1010,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x0a, 0x98, 0xc1, 0x3c, 0x98, 0x82, + 0xdc, 0xb6, 0xc2, 0x36 }, + .data_in = +- (uint8_t[]){ 0x50, 0x01, 0x86, 0xb8, 0xf8, 0xdb, 0x2d, ++ (uint8_t[]){ 0x80, 0x01, 0x86, 0xb8, 0xf8, 0xdb, 0x2d, + 0x3f, 0x23, 0x82, 0x53, 0xfd, 0x37, 0xde, + 0x88, 0x63, 0x08, 0x4f, 0xd3, 0x71, 0xfb, + 0xeb, 0x35, 0xf3, 0x64, 0xd3, 0x5e, 0xaf, +@@ -1022,7 +1022,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x7a, 0xf9, 0xdd }, + .in_len = 66, + .data_out = +- (uint8_t[]){ 0x50, 0x01, 0x86, 0x22, 0x2a, 0x8f, 0x86, ++ (uint8_t[]){ 0x80, 0x01, 0x86, 0x22, 0x2a, 0x8f, 0x86, + 0x25, 0x92, 0xcd, 0xa9, 0xa1, 0xa0, 0xf5, + 0x86, 0x0d, 0xe8, 0xe4, 0xef, 0xe0, 0x78, + 0x45, 0x7c, 0x0c, 0x41, 0x5c, 0x8f, 0x07, +@@ -1031,7 +1031,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x35, 0xff, 0x7a, 0x42, 0xac, 0x94, 0x21, + 0x60, 0x1c, 0x13, 0xcc, 0x7e, 0x6b, 0x2f, + 0x6f, 0x91, 0x89, 0xc6, 0xd4, 0xe6, 0x17, +- 0xae, 0xde, 0xfb, 0xf5, 0xda, 0x73, 0xa7 }, ++ 0xae, 0xde, 0xfb, 0xff, 0xf9, 0xef, 0xff }, + .sn_size = 12, + .hfn = 0x1, + .hfn_threshold = 0xfa558, +@@ -1154,7 +1154,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x0a, 0x98, 0xc1, 0x3c, 0x98, 0x82, + 0xdc, 0xb6, 0xc2, 0x36 }, + .data_in = +- (uint8_t[]){ 0x50, 0x01, 0x86, 0xb8, 0xf8, 0xdb, 0x2d, ++ (uint8_t[]){ 0x80, 0x01, 0x86, 0xb8, 0xf8, 0xdb, 0x2d, + 0x3f, 0x23, 0x82, 0x53, 0xfd, 0x37, 0xde, + 0x88, 0x63, 0x08, 0x4f, 0xd3, 0x71, 0xfb, + 0xeb, 0x35, 0xf3, 0x64, 0xd3, 0x5e, 0xaf, +@@ -1166,7 +1166,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x7a, 0xf9, 0xdd }, + .in_len = 66, + .data_out = +- (uint8_t[]){ 0x50, 0x01, 0x86, 0xe8, 0x8c, 0x96, 0x38, ++ (uint8_t[]){ 0x80, 0x01, 0x86, 0xe8, 0x8c, 0x96, 0x38, + 0x6e, 0xe4, 0x1f, 0xb9, 0x85, 0x61, 0x2e, + 0x64, 0x31, 0x22, 0x97, 0x5b, 0xf6, 0x40, + 0x08, 0x65, 0xc3, 0xfa, 0x72, 0xd2, 0x22, +@@ -1175,7 +1175,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x23, 0x80, 0xcc, 0x5f, 0xfd, 0x74, 0xb1, + 0xfb, 0xe1, 0xea, 0x6f, 0xf9, 0x9d, 0xf3, + 0x90, 0x02, 0x6e, 0xf1, 0xf8, 0x95, 0x97, +- 0xbd, 0xba, 0x08, 0x39, 0x63, 0x21, 0x82 }, ++ 0xbd, 0xba, 0x08, 0xb2, 0x24, 0x23, 0xd9 }, + .sn_size = 12, + .hfn = 0x1, + .hfn_threshold = 0xfa558, +@@ -1202,7 +1202,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x0a, 0x98, 0xc1, 0x3c, 0x98, 0x82, + 0xdc, 0xb6, 0xc2, 0x36 }, + .data_in = +- (uint8_t[]){ 0x50, 0x01, 0x86, 0xb8, 0xf8, 0xdb, 0x2d, ++ (uint8_t[]){ 0x80, 0x01, 0x86, 0xb8, 0xf8, 0xdb, 0x2d, + 0x3f, 0x23, 0x82, 0x53, 0xfd, 0x37, 0xde, + 0x88, 0x63, 0x08, 0x4f, 0xd3, 0x71, 0xfb, + 0xeb, 0x35, 0xf3, 0x64, 0xd3, 0x5e, 0xaf, +@@ -1214,7 +1214,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x7a, 0xf9, 0xdd }, + .in_len = 66, + .data_out = +- (uint8_t[]){ 0x50, 0x01, 0x86, 0xd2, 0xc0, 0x3a, 0x41, ++ (uint8_t[]){ 0x80, 0x01, 0x86, 0xd2, 0xc0, 0x3a, 0x41, + 0xbc, 0x0b, 0x95, 0xa4, 0x57, 0x3e, 0x36, + 0x10, 0xb2, 0xff, 0x48, 0x1f, 0x10, 0xa8, + 0x06, 0x42, 0xc5, 0xa2, 0x33, 0xed, 0x2e, +@@ -1223,7 +1223,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x35, 0xd2, 0x31, 0x1a, 0xb8, 0x33, 0xc7, + 0x9c, 0xa4, 0xa4, 0x9b, 0x20, 0xc6, 0x47, + 0xdd, 0xe5, 0xa3, 0x4b, 0x9e, 0x66, 0xde, +- 0xc9, 0x0a, 0x64, 0x8e, 0x79, 0xde, 0xaa }, ++ 0xc9, 0x0a, 0x64, 0x07, 0xc1, 0x1d, 0xa3 }, + .sn_size = 12, + .hfn = 0x1, + .hfn_threshold = 0xfa558, +@@ -1250,7 +1250,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x0a, 0x98, 0xc1, 0x3c, 0x98, 0x82, + 0xdc, 0xb6, 0xc2, 0x36 }, + .data_in = +- (uint8_t[]){ 0x50, 0x01, 0x86, 0xb8, 0xf8, 0xdb, 0x2d, ++ (uint8_t[]){ 0x80, 0x01, 0x86, 0xb8, 0xf8, 0xdb, 0x2d, + 0x3f, 0x23, 0x82, 0x53, 0xfd, 0x37, 0xde, + 0x88, 0x63, 0x08, 0x4f, 0xd3, 0x71, 0xfb, + 0xeb, 0x35, 0xf3, 0x64, 0xd3, 0x5e, 0xaf, +@@ -1262,7 +1262,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x7a, 0xf9, 0xdd }, + .in_len = 66, + .data_out = +- (uint8_t[]){ 0x50, 0x01, 0x86, 0xe8, 0x8c, 0x96, 0x38, ++ (uint8_t[]){ 0x80, 0x01, 0x86, 0xe8, 0x8c, 0x96, 0x38, + 0x6e, 0xe4, 0x1f, 0xb9, 0x85, 0x61, 0x2e, + 0x64, 0x31, 0x22, 0x97, 0x5b, 0xf6, 0x40, + 0x08, 0x65, 0xc3, 0xfa, 0x72, 0xd2, 0x22, +@@ -1271,7 +1271,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x23, 0x80, 0xcc, 0x5f, 0xfd, 0x74, 0xb1, + 0xfb, 0xe1, 0xea, 0x6f, 0xf9, 0x9d, 0xf3, + 0x90, 0x02, 0x6e, 0xf1, 0xf8, 0x95, 0x97, +- 0xbd, 0xba, 0x08, 0x72, 0xaa, 0x20, 0x3a }, ++ 0xbd, 0xba, 0x08, 0x6c, 0x5f, 0xa6, 0x4c }, + .sn_size = 12, + .hfn = 0x1, + .hfn_threshold = 0xfa558, +@@ -1298,7 +1298,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x0a, 0x98, 0xc1, 0x3c, 0x98, 0x82, + 0xdc, 0xb6, 0xc2, 0x36 }, + .data_in = +- (uint8_t[]){ 0x50, 0x01, 0x86, 0xb8, 0xf8, 0xdb, 0x2d, ++ (uint8_t[]){ 0x80, 0x01, 0x86, 0xb8, 0xf8, 0xdb, 0x2d, + 0x3f, 0x23, 0x82, 0x53, 0xfd, 0x37, 0xde, + 0x88, 0x63, 0x08, 0x4f, 0xd3, 0x71, 0xfb, + 0xeb, 0x35, 0xf3, 0x64, 0xd3, 0x5e, 0xaf, +@@ -1310,7 +1310,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x7a, 0xf9, 0xdd }, + .in_len = 66, + .data_out = +- (uint8_t[]){ 0x50, 0x01, 0x86, 0xd2, 0xc0, 0x3a, 0x41, ++ (uint8_t[]){ 0x80, 0x01, 0x86, 0xd2, 0xc0, 0x3a, 0x41, + 0xbc, 0x0b, 0x95, 0xa4, 0x57, 0x3e, 0x36, + 0x10, 0xb2, 0xff, 0x48, 0x1f, 0x10, 0xa8, + 0x06, 0x42, 0xc5, 0xa2, 0x33, 0xed, 0x2e, +@@ -1319,7 +1319,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x35, 0xd2, 0x31, 0x1a, 0xb8, 0x33, 0xc7, + 0x9c, 0xa4, 0xa4, 0x9b, 0x20, 0xc6, 0x47, + 0xdd, 0xe5, 0xa3, 0x4b, 0x9e, 0x66, 0xde, +- 0xc9, 0x0a, 0x64, 0x21, 0x5c, 0x2d, 0xfa }, ++ 0xc9, 0x0a, 0x64, 0x45, 0xd9, 0x18, 0x48 }, + .sn_size = 12, + .hfn = 0x1, + .hfn_threshold = 0xfa558, +@@ -1346,7 +1346,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x0a, 0x98, 0xc1, 0x3c, 0x98, 0x82, + 0xdc, 0xb6, 0xc2, 0x36 }, + .data_in = +- (uint8_t[]){ 0x50, 0x01, 0x86, 0xb8, 0xf8, 0xdb, 0x2d, ++ (uint8_t[]){ 0x80, 0x01, 0x86, 0xb8, 0xf8, 0xdb, 0x2d, + 0x3f, 0x23, 0x82, 0x53, 0xfd, 0x37, 0xde, + 0x88, 0x63, 0x08, 0x4f, 0xd3, 0x71, 0xfb, + 0xeb, 0x35, 0xf3, 0x64, 0xd3, 0x5e, 0xaf, +@@ -1358,7 +1358,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x7a, 0xf9, 0xdd }, + .in_len = 66, + .data_out = +- (uint8_t[]){ 0x50, 0x01, 0x86, 0xe8, 0x8c, 0x96, 0x38, ++ (uint8_t[]){ 0x80, 0x01, 0x86, 0xe8, 0x8c, 0x96, 0x38, + 0x6e, 0xe4, 0x1f, 0xb9, 0x85, 0x61, 0x2e, + 0x64, 0x31, 0x22, 0x97, 0x5b, 0xf6, 0x40, + 0x08, 0x65, 0xc3, 0xfa, 0x72, 0xd2, 0x22, +@@ -1367,7 +1367,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x23, 0x80, 0xcc, 0x5f, 0xfd, 0x74, 0xb1, + 0xfb, 0xe1, 0xea, 0x6f, 0xf9, 0x9d, 0xf3, + 0x90, 0x02, 0x6e, 0xf1, 0xf8, 0x95, 0x97, +- 0xbd, 0xba, 0x08, 0x52, 0xfd, 0x22, 0x6f }, ++ 0xbd, 0xba, 0x08, 0x5d, 0x17, 0x47, 0xa7 }, + .sn_size = 12, + .hfn = 0x1, + .hfn_threshold = 0xfa558, +@@ -1394,7 +1394,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x0a, 0x98, 0xc1, 0x3c, 0x98, 0x82, + 0xdc, 0xb6, 0xc2, 0x36 }, + .data_in = +- (uint8_t[]){ 0x50, 0x01, 0x86, 0xb8, 0xf8, 0xdb, 0x2d, ++ (uint8_t[]){ 0x80, 0x01, 0x86, 0xb8, 0xf8, 0xdb, 0x2d, + 0x3f, 0x23, 0x82, 0x53, 0xfd, 0x37, 0xde, + 0x88, 0x63, 0x08, 0x4f, 0xd3, 0x71, 0xfb, + 0xeb, 0x35, 0xf3, 0x64, 0xd3, 0x5e, 0xaf, +@@ -1406,7 +1406,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x7a, 0xf9, 0xdd }, + .in_len = 66, + .data_out = +- (uint8_t[]){ 0x50, 0x01, 0x86, 0xd2, 0xc0, 0x3a, 0x41, ++ (uint8_t[]){ 0x80, 0x01, 0x86, 0xd2, 0xc0, 0x3a, 0x41, + 0xbc, 0x0b, 0x95, 0xa4, 0x57, 0x3e, 0x36, + 0x10, 0xb2, 0xff, 0x48, 0x1f, 0x10, 0xa8, + 0x06, 0x42, 0xc5, 0xa2, 0x33, 0xed, 0x2e, +@@ -1415,7 +1415,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x35, 0xd2, 0x31, 0x1a, 0xb8, 0x33, 0xc7, + 0x9c, 0xa4, 0xa4, 0x9b, 0x20, 0xc6, 0x47, + 0xdd, 0xe5, 0xa3, 0x4b, 0x9e, 0x66, 0xde, +- 0xc9, 0x0a, 0x64, 0xeb, 0xc1, 0xf4, 0xc6 }, ++ 0xc9, 0x0a, 0x64, 0xe1, 0xe2, 0x68, 0x9e }, + .sn_size = 12, + .hfn = 0x1, + .hfn_threshold = 0xfa558, +@@ -1538,7 +1538,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x0a, 0x98, 0xc1, 0x3c, 0x98, 0x82, + 0xdc, 0xb6, 0xc2, 0x36 }, + .data_in = +- (uint8_t[]){ 0x50, 0x01, 0x86, 0xb8, 0xf8, 0xdb, 0x2d, ++ (uint8_t[]){ 0x80, 0x01, 0x86, 0xb8, 0xf8, 0xdb, 0x2d, + 0x3f, 0x23, 0x82, 0x53, 0xfd, 0x37, 0xde, + 0x88, 0x63, 0x08, 0x4f, 0xd3, 0x71, 0xfb, + 0xeb, 0x35, 0xf3, 0x64, 0xd3, 0x5e, 0xaf, +@@ -1550,7 +1550,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x7a, 0xf9, 0xdd }, + .in_len = 66, + .data_out = +- (uint8_t[]){ 0x50, 0x01, 0x86, 0x79, 0xdb, 0x02, 0x27, ++ (uint8_t[]){ 0x80, 0x01, 0x86, 0x79, 0xdb, 0x02, 0x27, + 0xa0, 0x85, 0x82, 0x87, 0x6b, 0x35, 0x2b, + 0xe1, 0x96, 0x5b, 0xfd, 0x67, 0xe1, 0x67, + 0x69, 0x17, 0x5b, 0x0a, 0x8f, 0xb8, 0x37, +@@ -1559,7 +1559,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0xda, 0x94, 0x61, 0x8e, 0x2d, 0x7a, 0x5d, + 0x64, 0x0f, 0x4a, 0xc7, 0x7c, 0x9d, 0x56, + 0x5a, 0x2a, 0xcb, 0xff, 0x79, 0x07, 0xa5, +- 0xaf, 0x96, 0x5c, 0xb6, 0x6c, 0xeb, 0x14 }, ++ 0xaf, 0x96, 0x5c, 0x3d, 0x2b, 0xe9, 0x4f }, + .sn_size = 12, + .hfn = 0x1, + .hfn_threshold = 0xfa558, +@@ -1586,7 +1586,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x0a, 0x98, 0xc1, 0x3c, 0x98, 0x82, + 0xdc, 0xb6, 0xc2, 0x36 }, + .data_in = +- (uint8_t[]){ 0x50, 0x01, 0x86, 0xb8, 0xf8, 0xdb, 0x2d, ++ (uint8_t[]){ 0x80, 0x01, 0x86, 0xb8, 0xf8, 0xdb, 0x2d, + 0x3f, 0x23, 0x82, 0x53, 0xfd, 0x37, 0xde, + 0x88, 0x63, 0x08, 0x4f, 0xd3, 0x71, 0xfb, + 0xeb, 0x35, 0xf3, 0x64, 0xd3, 0x5e, 0xaf, +@@ -1598,7 +1598,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x7a, 0xf9, 0xdd }, + .in_len = 66, + .data_out = +- (uint8_t[]){ 0x50, 0x01, 0x86, 0x2c, 0x0f, 0xc1, 0x08, ++ (uint8_t[]){ 0x80, 0x01, 0x86, 0x2c, 0x0f, 0xc1, 0x08, + 0x2e, 0xa7, 0x97, 0xd2, 0x6c, 0x17, 0x8f, + 0x22, 0x1f, 0x99, 0x1b, 0x48, 0x04, 0x29, + 0x54, 0x3f, 0x07, 0x01, 0xda, 0x19, 0x59, +@@ -1607,7 +1607,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x04, 0xf4, 0xfe, 0xe6, 0xf7, 0x53, 0x51, + 0x70, 0x10, 0x56, 0x83, 0x59, 0xb5, 0x64, + 0x0b, 0xa6, 0x2e, 0xa5, 0x03, 0x62, 0x13, +- 0x91, 0xaf, 0x24, 0xb2, 0x82, 0xfb, 0x27 }, ++ 0x91, 0xaf, 0x24, 0x3b, 0x3a, 0x38, 0x2e }, + .sn_size = 12, + .hfn = 0x1, + .hfn_threshold = 0xfa558, +@@ -1634,7 +1634,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x0a, 0x98, 0xc1, 0x3c, 0x98, 0x82, + 0xdc, 0xb6, 0xc2, 0x36 }, + .data_in = +- (uint8_t[]){ 0x50, 0x01, 0x86, 0xb8, 0xf8, 0xdb, 0x2d, ++ (uint8_t[]){ 0x80, 0x01, 0x86, 0xb8, 0xf8, 0xdb, 0x2d, + 0x3f, 0x23, 0x82, 0x53, 0xfd, 0x37, 0xde, + 0x88, 0x63, 0x08, 0x4f, 0xd3, 0x71, 0xfb, + 0xeb, 0x35, 0xf3, 0x64, 0xd3, 0x5e, 0xaf, +@@ -1646,7 +1646,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x7a, 0xf9, 0xdd }, + .in_len = 66, + .data_out = +- (uint8_t[]){ 0x50, 0x01, 0x86, 0x79, 0xdb, 0x02, 0x27, ++ (uint8_t[]){ 0x80, 0x01, 0x86, 0x79, 0xdb, 0x02, 0x27, + 0xa0, 0x85, 0x82, 0x87, 0x6b, 0x35, 0x2b, + 0xe1, 0x96, 0x5b, 0xfd, 0x67, 0xe1, 0x67, + 0x69, 0x17, 0x5b, 0x0a, 0x8f, 0xb8, 0x37, +@@ -1655,7 +1655,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0xda, 0x94, 0x61, 0x8e, 0x2d, 0x7a, 0x5d, + 0x64, 0x0f, 0x4a, 0xc7, 0x7c, 0x9d, 0x56, + 0x5a, 0x2a, 0xcb, 0xff, 0x79, 0x07, 0xa5, +- 0xaf, 0x96, 0x5c, 0xfd, 0xa5, 0xea, 0xac }, ++ 0xaf, 0x96, 0x5c, 0xe3, 0x50, 0x6c, 0xda }, + .sn_size = 12, + .hfn = 0x1, + .hfn_threshold = 0xfa558, +@@ -1682,7 +1682,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x0a, 0x98, 0xc1, 0x3c, 0x98, 0x82, + 0xdc, 0xb6, 0xc2, 0x36 }, + .data_in = +- (uint8_t[]){ 0x50, 0x01, 0x86, 0xb8, 0xf8, 0xdb, 0x2d, ++ (uint8_t[]){ 0x80, 0x01, 0x86, 0xb8, 0xf8, 0xdb, 0x2d, + 0x3f, 0x23, 0x82, 0x53, 0xfd, 0x37, 0xde, + 0x88, 0x63, 0x08, 0x4f, 0xd3, 0x71, 0xfb, + 0xeb, 0x35, 0xf3, 0x64, 0xd3, 0x5e, 0xaf, +@@ -1694,7 +1694,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x7a, 0xf9, 0xdd }, + .in_len = 66, + .data_out = +- (uint8_t[]){ 0x50, 0x01, 0x86, 0x2c, 0x0f, 0xc1, 0x08, ++ (uint8_t[]){ 0x80, 0x01, 0x86, 0x2c, 0x0f, 0xc1, 0x08, + 0x2e, 0xa7, 0x97, 0xd2, 0x6c, 0x17, 0x8f, + 0x22, 0x1f, 0x99, 0x1b, 0x48, 0x04, 0x29, + 0x54, 0x3f, 0x07, 0x01, 0xda, 0x19, 0x59, +@@ -1703,7 +1703,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x04, 0xf4, 0xfe, 0xe6, 0xf7, 0x53, 0x51, + 0x70, 0x10, 0x56, 0x83, 0x59, 0xb5, 0x64, + 0x0b, 0xa6, 0x2e, 0xa5, 0x03, 0x62, 0x13, +- 0x91, 0xaf, 0x24, 0x1d, 0xa7, 0x08, 0x77 }, ++ 0x91, 0xaf, 0x24, 0x79, 0x22, 0x3d, 0xc5 }, + .sn_size = 12, + .hfn = 0x1, + .hfn_threshold = 0xfa558, +@@ -1730,7 +1730,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x0a, 0x98, 0xc1, 0x3c, 0x98, 0x82, + 0xdc, 0xb6, 0xc2, 0x36 }, + .data_in = +- (uint8_t[]){ 0x50, 0x01, 0x86, 0xb8, 0xf8, 0xdb, 0x2d, ++ (uint8_t[]){ 0x80, 0x01, 0x86, 0xb8, 0xf8, 0xdb, 0x2d, + 0x3f, 0x23, 0x82, 0x53, 0xfd, 0x37, 0xde, + 0x88, 0x63, 0x08, 0x4f, 0xd3, 0x71, 0xfb, + 0xeb, 0x35, 0xf3, 0x64, 0xd3, 0x5e, 0xaf, +@@ -1742,7 +1742,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x7a, 0xf9, 0xdd }, + .in_len = 66, + .data_out = +- (uint8_t[]){ 0x50, 0x01, 0x86, 0x79, 0xdb, 0x02, 0x27, ++ (uint8_t[]){ 0x80, 0x01, 0x86, 0x79, 0xdb, 0x02, 0x27, + 0xa0, 0x85, 0x82, 0x87, 0x6b, 0x35, 0x2b, + 0xe1, 0x96, 0x5b, 0xfd, 0x67, 0xe1, 0x67, + 0x69, 0x17, 0x5b, 0x0a, 0x8f, 0xb8, 0x37, +@@ -1751,7 +1751,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0xda, 0x94, 0x61, 0x8e, 0x2d, 0x7a, 0x5d, + 0x64, 0x0f, 0x4a, 0xc7, 0x7c, 0x9d, 0x56, + 0x5a, 0x2a, 0xcb, 0xff, 0x79, 0x07, 0xa5, +- 0xaf, 0x96, 0x5c, 0xdd, 0xf2, 0xe8, 0xf9 }, ++ 0xaf, 0x96, 0x5c, 0xd2, 0x18, 0x8d, 0x31 }, + .sn_size = 12, + .hfn = 0x1, + .hfn_threshold = 0xfa558, +@@ -1778,7 +1778,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x0a, 0x98, 0xc1, 0x3c, 0x98, 0x82, + 0xdc, 0xb6, 0xc2, 0x36 }, + .data_in = +- (uint8_t[]){ 0x50, 0x01, 0x86, 0xb8, 0xf8, 0xdb, 0x2d, ++ (uint8_t[]){ 0x80, 0x01, 0x86, 0xb8, 0xf8, 0xdb, 0x2d, + 0x3f, 0x23, 0x82, 0x53, 0xfd, 0x37, 0xde, + 0x88, 0x63, 0x08, 0x4f, 0xd3, 0x71, 0xfb, + 0xeb, 0x35, 0xf3, 0x64, 0xd3, 0x5e, 0xaf, +@@ -1790,7 +1790,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x7a, 0xf9, 0xdd }, + .in_len = 66, + .data_out = +- (uint8_t[]){ 0x50, 0x01, 0x86, 0x2c, 0x0f, 0xc1, 0x08, ++ (uint8_t[]){ 0x80, 0x01, 0x86, 0x2c, 0x0f, 0xc1, 0x08, + 0x2e, 0xa7, 0x97, 0xd2, 0x6c, 0x17, 0x8f, + 0x22, 0x1f, 0x99, 0x1b, 0x48, 0x04, 0x29, + 0x54, 0x3f, 0x07, 0x01, 0xda, 0x19, 0x59, +@@ -1799,7 +1799,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x04, 0xf4, 0xfe, 0xe6, 0xf7, 0x53, 0x51, + 0x70, 0x10, 0x56, 0x83, 0x59, 0xb5, 0x64, + 0x0b, 0xa6, 0x2e, 0xa5, 0x03, 0x62, 0x13, +- 0x91, 0xaf, 0x24, 0xd7, 0x3a, 0xd1, 0x4b }, ++ 0x91, 0xaf, 0x24, 0xdd, 0x19, 0x4d, 0x13 }, + .sn_size = 12, + .hfn = 0x1, + .hfn_threshold = 0xfa558, +@@ -2556,7 +2556,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x4b, 0x32, 0x87, 0xf9, 0xdb, 0xe0, + 0x31, 0x5f, 0x3a, 0x15 }, + .data_in = +- (uint8_t[]){ 0xf8, 0x00, 0x00, 0xf8, 0xdb, 0x2d, 0x3f, ++ (uint8_t[]){ 0x80, 0x00, 0x00, 0xf8, 0xdb, 0x2d, 0x3f, + 0x23, 0x82, 0x53, 0xfd, 0x37, 0xde, 0x88, + 0x63, 0x08, 0x4f, 0xD3, 0x71, 0xfb, 0xeb, + 0x35, 0xf3, 0x64, 0xd3, 0x5e, 0xaf, 0x3f, +@@ -2568,7 +2568,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0xf9, 0xdd, 0xcc, 0x69 }, + .in_len = 67, + .data_out = +- (uint8_t[]){ 0xf8, 0x00, 0x00, 0xf8, 0x78, 0xee, 0x01, ++ (uint8_t[]){ 0x80, 0x00, 0x00, 0xf8, 0x78, 0xee, 0x01, + 0x4c, 0x79, 0xc7, 0x14, 0x79, 0x2c, 0x1b, + 0x8a, 0xd0, 0x17, 0xb4, 0xb4, 0xfe, 0xa7, + 0x17, 0x9a, 0x93, 0x5d, 0xad, 0x27, 0x8a, +@@ -2577,8 +2577,8 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x37, 0xfb, 0x98, 0x1b, 0x06, 0x1a, 0x76, + 0xf8, 0x07, 0x72, 0xe9, 0xa2, 0x89, 0x47, + 0x95, 0x8f, 0x8f, 0x4e, 0x45, 0x55, 0xd6, +- 0x0e, 0xf4, 0xe7, 0xe8, 0x78, 0xdd, 0xc1, +- 0x92 }, ++ 0x0e, 0xf4, 0xe7, 0xe8, 0xc0, 0x48, 0x6a, ++ 0x7c }, + .sn_size = 18, + .hfn = 0x1, + .hfn_threshold = 0xfa558, +@@ -2605,7 +2605,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x4b, 0x32, 0x87, 0xf9, 0xdb, 0xe0, + 0x31, 0x5f, 0x3a, 0x15 }, + .data_in = +- (uint8_t[]){ 0xf8, 0x00, 0x00, 0xf8, 0xdb, 0x2d, 0x3f, ++ (uint8_t[]){ 0x80, 0x00, 0x00, 0xf8, 0xdb, 0x2d, 0x3f, + 0x23, 0x82, 0x53, 0xfd, 0x37, 0xde, 0x88, + 0x63, 0x08, 0x4f, 0xD3, 0x71, 0xfb, 0xeb, + 0x35, 0xf3, 0x64, 0xd3, 0x5e, 0xaf, 0x3f, +@@ -2617,7 +2617,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0xf9, 0xdd, 0xcc, 0x69 }, + .in_len = 67, + .data_out = +- (uint8_t[]){ 0xf8, 0x00, 0x00, 0xf8, 0xbf, 0x21, 0xc9, ++ (uint8_t[]){ 0x80, 0x00, 0x00, 0xf8, 0xbf, 0x21, 0xc9, + 0x7d, 0x02, 0x8d, 0xf7, 0xf7, 0x80, 0x50, + 0x60, 0x32, 0x09, 0xb8, 0x69, 0x48, 0xab, + 0x58, 0xf0, 0xd9, 0x63, 0x63, 0x36, 0x01, +@@ -2626,8 +2626,8 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x4c, 0x00, 0xd0, 0x0e, 0x31, 0x1c, 0x87, + 0xee, 0x1c, 0xc3, 0x81, 0xb4, 0xb6, 0x09, + 0xb5, 0x3c, 0x85, 0xb9, 0xcc, 0x2d, 0x2e, +- 0x9c, 0x85, 0x0b, 0xf7, 0xb1, 0x80, 0x30, +- 0xa5 }, ++ 0x9c, 0x85, 0x0b, 0xf7, 0x17, 0x28, 0x0f, ++ 0x7d }, + .sn_size = 18, + .hfn = 0x1, + .hfn_threshold = 0xfa558, +@@ -2654,7 +2654,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x4b, 0x32, 0x87, 0xf9, 0xdb, 0xe0, + 0x31, 0x5f, 0x3a, 0x15 }, + .data_in = +- (uint8_t[]){ 0xf8, 0x00, 0x00, 0xf8, 0xdb, 0x2d, 0x3f, ++ (uint8_t[]){ 0x80, 0x00, 0x00, 0xf8, 0xdb, 0x2d, 0x3f, + 0x23, 0x82, 0x53, 0xfd, 0x37, 0xde, 0x88, + 0x63, 0x08, 0x4f, 0xD3, 0x71, 0xfb, 0xeb, + 0x35, 0xf3, 0x64, 0xd3, 0x5e, 0xaf, 0x3f, +@@ -2666,7 +2666,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0xf9, 0xdd, 0xcc, 0x69 }, + .in_len = 67, + .data_out = +- (uint8_t[]){ 0xf8, 0x00, 0x00, 0xf8, 0x78, 0xee, 0x01, ++ (uint8_t[]){ 0x80, 0x00, 0x00, 0xf8, 0x78, 0xee, 0x01, + 0x4c, 0x79, 0xc7, 0x14, 0x79, 0x2c, 0x1b, + 0x8a, 0xd0, 0x17, 0xb4, 0xb4, 0xfe, 0xa7, + 0x17, 0x9a, 0x93, 0x5d, 0xad, 0x27, 0x8a, +@@ -2675,8 +2675,8 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x37, 0xfb, 0x98, 0x1b, 0x06, 0x1a, 0x76, + 0xf8, 0x07, 0x72, 0xe9, 0xa2, 0x89, 0x47, + 0x95, 0x8f, 0x8f, 0x4e, 0x45, 0x55, 0xd6, +- 0x0e, 0xf4, 0xe7, 0xe8, 0x08, 0xa6, 0xdb, +- 0x19 }, ++ 0x0e, 0xf4, 0xe7, 0xe8, 0x8e, 0x76, 0x4a, ++ 0x4e }, + .sn_size = 18, + .hfn = 0x1, + .hfn_threshold = 0xfa558, +@@ -2703,7 +2703,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x4b, 0x32, 0x87, 0xf9, 0xdb, 0xe0, + 0x31, 0x5f, 0x3a, 0x15 }, + .data_in = +- (uint8_t[]){ 0xf8, 0x00, 0x00, 0xf8, 0xdb, 0x2d, 0x3f, ++ (uint8_t[]){ 0x80, 0x00, 0x00, 0xf8, 0xdb, 0x2d, 0x3f, + 0x23, 0x82, 0x53, 0xfd, 0x37, 0xde, 0x88, + 0x63, 0x08, 0x4f, 0xD3, 0x71, 0xfb, 0xeb, + 0x35, 0xf3, 0x64, 0xd3, 0x5e, 0xaf, 0x3f, +@@ -2715,7 +2715,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0xf9, 0xdd, 0xcc, 0x69 }, + .in_len = 67, + .data_out = +- (uint8_t[]){ 0xf8, 0x00, 0x00, 0xf8, 0xbf, 0x21, 0xc9, ++ (uint8_t[]){ 0x80, 0x00, 0x00, 0xf8, 0xbf, 0x21, 0xc9, + 0x7d, 0x02, 0x8d, 0xf7, 0xf7, 0x80, 0x50, + 0x60, 0x32, 0x09, 0xb8, 0x69, 0x48, 0xab, + 0x58, 0xf0, 0xd9, 0x63, 0x63, 0x36, 0x01, +@@ -2724,8 +2724,8 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x4c, 0x00, 0xd0, 0x0e, 0x31, 0x1c, 0x87, + 0xee, 0x1c, 0xc3, 0x81, 0xb4, 0xb6, 0x09, + 0xb5, 0x3c, 0x85, 0xb9, 0xcc, 0x2d, 0x2e, +- 0x9c, 0x85, 0x0b, 0xf7, 0x97, 0x5a, 0x56, +- 0xab }, ++ 0x9c, 0x85, 0x0b, 0xf7, 0xc1, 0x27, 0x82, ++ 0xc3 }, + .sn_size = 18, + .hfn = 0x1, + .hfn_threshold = 0xfa558, +@@ -2752,7 +2752,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x4b, 0x32, 0x87, 0xf9, 0xdb, 0xe0, + 0x31, 0x5f, 0x3a, 0x15 }, + .data_in = +- (uint8_t[]){ 0xf8, 0x00, 0x00, 0xf8, 0xdb, 0x2d, 0x3f, ++ (uint8_t[]){ 0x80, 0x00, 0x00, 0xf8, 0xdb, 0x2d, 0x3f, + 0x23, 0x82, 0x53, 0xfd, 0x37, 0xde, 0x88, + 0x63, 0x08, 0x4f, 0xD3, 0x71, 0xfb, 0xeb, + 0x35, 0xf3, 0x64, 0xd3, 0x5e, 0xaf, 0x3f, +@@ -2764,7 +2764,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0xf9, 0xdd, 0xcc, 0x69 }, + .in_len = 67, + .data_out = +- (uint8_t[]){ 0xf8, 0x00, 0x00, 0xf8, 0x78, 0xee, 0x01, ++ (uint8_t[]){ 0x80, 0x00, 0x00, 0xf8, 0x78, 0xee, 0x01, + 0x4c, 0x79, 0xc7, 0x14, 0x79, 0x2c, 0x1b, + 0x8a, 0xd0, 0x17, 0xb4, 0xb4, 0xfe, 0xa7, + 0x17, 0x9a, 0x93, 0x5d, 0xad, 0x27, 0x8a, +@@ -2773,8 +2773,8 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x37, 0xfb, 0x98, 0x1b, 0x06, 0x1a, 0x76, + 0xf8, 0x07, 0x72, 0xe9, 0xa2, 0x89, 0x47, + 0x95, 0x8f, 0x8f, 0x4e, 0x45, 0x55, 0xd6, +- 0x0e, 0xf4, 0xe7, 0xe8, 0x08, 0x68, 0xff, +- 0x7c }, ++ 0x0e, 0xf4, 0xe7, 0xe8, 0x97, 0x76, 0xce, ++ 0xac }, + .sn_size = 18, + .hfn = 0x1, + .hfn_threshold = 0xfa558, +@@ -2801,7 +2801,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x4b, 0x32, 0x87, 0xf9, 0xdb, 0xe0, + 0x31, 0x5f, 0x3a, 0x15 }, + .data_in = +- (uint8_t[]){ 0xf8, 0x00, 0x00, 0xf8, 0xdb, 0x2d, 0x3f, ++ (uint8_t[]){ 0x80, 0x00, 0x00, 0xf8, 0xdb, 0x2d, 0x3f, + 0x23, 0x82, 0x53, 0xfd, 0x37, 0xde, 0x88, + 0x63, 0x08, 0x4f, 0xD3, 0x71, 0xfb, 0xeb, + 0x35, 0xf3, 0x64, 0xd3, 0x5e, 0xaf, 0x3f, +@@ -2813,7 +2813,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0xf9, 0xdd, 0xcc, 0x69 }, + .in_len = 67, + .data_out = +- (uint8_t[]){ 0xf8, 0x00, 0x00, 0xf8, 0xbf, 0x21, 0xc9, ++ (uint8_t[]){ 0x80, 0x00, 0x00, 0xf8, 0xbf, 0x21, 0xc9, + 0x7d, 0x02, 0x8d, 0xf7, 0xf7, 0x80, 0x50, + 0x60, 0x32, 0x09, 0xb8, 0x69, 0x48, 0xab, + 0x58, 0xf0, 0xd9, 0x63, 0x63, 0x36, 0x01, +@@ -2822,8 +2822,8 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x4c, 0x00, 0xd0, 0x0e, 0x31, 0x1c, 0x87, + 0xee, 0x1c, 0xc3, 0x81, 0xb4, 0xb6, 0x09, + 0xb5, 0x3c, 0x85, 0xb9, 0xcc, 0x2d, 0x2e, +- 0x9c, 0x85, 0x0b, 0xf7, 0x41, 0xdd, 0x19, +- 0x32 }, ++ 0x9c, 0x85, 0x0b, 0xf7, 0x69, 0x56, 0x6f, ++ 0xaf }, + .sn_size = 18, + .hfn = 0x1, + .hfn_threshold = 0xfa558, +@@ -2948,7 +2948,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x4b, 0x32, 0x87, 0xf9, 0xdb, 0xe0, + 0x31, 0x5f, 0x3a, 0x15 }, + .data_in = +- (uint8_t[]){ 0xf8, 0x00, 0x00, 0xf8, 0xdb, 0x2d, 0x3f, ++ (uint8_t[]){ 0x80, 0x00, 0x00, 0xf8, 0xdb, 0x2d, 0x3f, + 0x23, 0x82, 0x53, 0xfd, 0x37, 0xde, 0x88, + 0x63, 0x08, 0x4f, 0xD3, 0x71, 0xfb, 0xeb, + 0x35, 0xf3, 0x64, 0xd3, 0x5e, 0xaf, 0x3f, +@@ -2960,7 +2960,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0xf9, 0xdd, 0xcc, 0x69 }, + .in_len = 67, + .data_out = +- (uint8_t[]){ 0xf8, 0x00, 0x00, 0xf8, 0x0c, 0xa8, 0x9d, ++ (uint8_t[]){ 0x80, 0x00, 0x00, 0xf8, 0x0c, 0xa8, 0x9d, + 0x9e, 0xcc, 0xf0, 0x1a, 0xc0, 0xf2, 0x9f, + 0x8c, 0xc9, 0x57, 0xc7, 0x99, 0x4d, 0xde, + 0xc5, 0x19, 0x69, 0x58, 0x5b, 0x1a, 0x51, +@@ -2969,8 +2969,8 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0xb3, 0x24, 0x88, 0x68, 0x5f, 0x78, 0x3a, + 0xd2, 0x6c, 0xcc, 0xa0, 0xb5, 0xf0, 0x95, + 0x3e, 0xf1, 0xf4, 0x3e, 0x43, 0x8f, 0x6c, +- 0xae, 0x22, 0x59, 0x11, 0xf6, 0x97, 0x0b, +- 0x7b }, ++ 0xae, 0x22, 0x59, 0x11, 0x4e, 0x02, 0xa0, ++ 0x95 }, + .sn_size = 18, + .hfn = 0x1, + .hfn_threshold = 0xfa558, +@@ -2997,7 +2997,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x4b, 0x32, 0x87, 0xf9, 0xdb, 0xe0, + 0x31, 0x5f, 0x3a, 0x15 }, + .data_in = +- (uint8_t[]){ 0xf8, 0x00, 0x00, 0xf8, 0xdb, 0x2d, 0x3f, ++ (uint8_t[]){ 0x80, 0x00, 0x00, 0xf8, 0xdb, 0x2d, 0x3f, + 0x23, 0x82, 0x53, 0xfd, 0x37, 0xde, 0x88, + 0x63, 0x08, 0x4f, 0xD3, 0x71, 0xfb, 0xeb, + 0x35, 0xf3, 0x64, 0xd3, 0x5e, 0xaf, 0x3f, +@@ -3009,7 +3009,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0xf9, 0xdd, 0xcc, 0x69 }, + .in_len = 67, + .data_out = +- (uint8_t[]){ 0xf8, 0x00, 0x00, 0xf8, 0x5a, 0xa9, 0xb5, ++ (uint8_t[]){ 0x80, 0x00, 0x00, 0xf8, 0x5a, 0xa9, 0xb5, + 0x61, 0x8b, 0x8a, 0xb7, 0x6a, 0x98, 0x30, + 0x6c, 0xed, 0x84, 0x69, 0xff, 0x6b, 0x7e, + 0x30, 0x59, 0x55, 0x80, 0x32, 0xd0, 0x0f, +@@ -3018,8 +3018,8 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x0e, 0x2e, 0x33, 0xe6, 0xa4, 0xea, 0x17, + 0xd6, 0x56, 0xa2, 0x3b, 0x5f, 0x56, 0x3a, + 0xa7, 0x6f, 0x4c, 0xc1, 0xca, 0xbd, 0x5f, +- 0xad, 0x3d, 0x99, 0x4a, 0xa3, 0xab, 0xd5, +- 0x7c }, ++ 0xad, 0x3d, 0x99, 0x4a, 0x05, 0x03, 0xea, ++ 0xa4 }, + .sn_size = 18, + .hfn = 0x1, + .hfn_threshold = 0xfa558, +@@ -3046,7 +3046,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x4b, 0x32, 0x87, 0xf9, 0xdb, 0xe0, + 0x31, 0x5f, 0x3a, 0x15 }, + .data_in = +- (uint8_t[]){ 0xf8, 0x00, 0x00, 0xf8, 0xdb, 0x2d, 0x3f, ++ (uint8_t[]){ 0x80, 0x00, 0x00, 0xf8, 0xdb, 0x2d, 0x3f, + 0x23, 0x82, 0x53, 0xfd, 0x37, 0xde, 0x88, + 0x63, 0x08, 0x4f, 0xD3, 0x71, 0xfb, 0xeb, + 0x35, 0xf3, 0x64, 0xd3, 0x5e, 0xaf, 0x3f, +@@ -3058,7 +3058,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0xf9, 0xdd, 0xcc, 0x69 }, + .in_len = 67, + .data_out = +- (uint8_t[]){ 0xf8, 0x00, 0x00, 0xf8, 0x0c, 0xa8, 0x9d, ++ (uint8_t[]){ 0x80, 0x00, 0x00, 0xf8, 0x0c, 0xa8, 0x9d, + 0x9e, 0xcc, 0xf0, 0x1a, 0xc0, 0xf2, 0x9f, + 0x8c, 0xc9, 0x57, 0xc7, 0x99, 0x4d, 0xde, + 0xc5, 0x19, 0x69, 0x58, 0x5b, 0x1a, 0x51, +@@ -3067,8 +3067,8 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0xb3, 0x24, 0x88, 0x68, 0x5f, 0x78, 0x3a, + 0xd2, 0x6c, 0xcc, 0xa0, 0xb5, 0xf0, 0x95, + 0x3e, 0xf1, 0xf4, 0x3e, 0x43, 0x8f, 0x6c, +- 0xae, 0x22, 0x59, 0x11, 0x86, 0xec, 0x11, +- 0xf0 }, ++ 0xae, 0x22, 0x59, 0x11, 0x00, 0x3c, 0x80, ++ 0xa7 }, + .sn_size = 18, + .hfn = 0x1, + .hfn_threshold = 0xfa558, +@@ -3095,7 +3095,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x4b, 0x32, 0x87, 0xf9, 0xdb, 0xe0, + 0x31, 0x5f, 0x3a, 0x15 }, + .data_in = +- (uint8_t[]){ 0xf8, 0x00, 0x00, 0xf8, 0xdb, 0x2d, 0x3f, ++ (uint8_t[]){ 0x80, 0x00, 0x00, 0xf8, 0xdb, 0x2d, 0x3f, + 0x23, 0x82, 0x53, 0xfd, 0x37, 0xde, 0x88, + 0x63, 0x08, 0x4f, 0xD3, 0x71, 0xfb, 0xeb, + 0x35, 0xf3, 0x64, 0xd3, 0x5e, 0xaf, 0x3f, +@@ -3107,7 +3107,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0xf9, 0xdd, 0xcc, 0x69 }, + .in_len = 67, + .data_out = +- (uint8_t[]){ 0xf8, 0x00, 0x00, 0xf8, 0x5a, 0xa9, 0xb5, ++ (uint8_t[]){ 0x80, 0x00, 0x00, 0xf8, 0x5a, 0xa9, 0xb5, + 0x61, 0x8b, 0x8a, 0xb7, 0x6a, 0x98, 0x30, + 0x6c, 0xed, 0x84, 0x69, 0xff, 0x6b, 0x7e, + 0x30, 0x59, 0x55, 0x80, 0x32, 0xd0, 0x0f, +@@ -3116,8 +3116,8 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x0e, 0x2e, 0x33, 0xe6, 0xa4, 0xea, 0x17, + 0xd6, 0x56, 0xa2, 0x3b, 0x5f, 0x56, 0x3a, + 0xa7, 0x6f, 0x4c, 0xc1, 0xca, 0xbd, 0x5f, +- 0xad, 0x3d, 0x99, 0x4a, 0x85, 0x71, 0xb3, +- 0x72 }, ++ 0xad, 0x3d, 0x99, 0x4a, 0xd3, 0x0c, 0x67, ++ 0x1a }, + .sn_size = 18, + .hfn = 0x1, + .hfn_threshold = 0xfa558, +@@ -3144,7 +3144,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x4b, 0x32, 0x87, 0xf9, 0xdb, 0xe0, + 0x31, 0x5f, 0x3a, 0x15 }, + .data_in = +- (uint8_t[]){ 0xf8, 0x00, 0x00, 0xf8, 0xdb, 0x2d, 0x3f, ++ (uint8_t[]){ 0x80, 0x00, 0x00, 0xf8, 0xdb, 0x2d, 0x3f, + 0x23, 0x82, 0x53, 0xfd, 0x37, 0xde, 0x88, + 0x63, 0x08, 0x4f, 0xD3, 0x71, 0xfb, 0xeb, + 0x35, 0xf3, 0x64, 0xd3, 0x5e, 0xaf, 0x3f, +@@ -3156,7 +3156,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0xf9, 0xdd, 0xcc, 0x69 }, + .in_len = 67, + .data_out = +- (uint8_t[]){ 0xf8, 0x00, 0x00, 0xf8, 0x0c, 0xa8, 0x9d, ++ (uint8_t[]){ 0x80, 0x00, 0x00, 0xf8, 0x0c, 0xa8, 0x9d, + 0x9e, 0xcc, 0xf0, 0x1a, 0xc0, 0xf2, 0x9f, + 0x8c, 0xc9, 0x57, 0xc7, 0x99, 0x4d, 0xde, + 0xc5, 0x19, 0x69, 0x58, 0x5b, 0x1a, 0x51, +@@ -3165,8 +3165,8 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0xb3, 0x24, 0x88, 0x68, 0x5f, 0x78, 0x3a, + 0xd2, 0x6c, 0xcc, 0xa0, 0xb5, 0xf0, 0x95, + 0x3e, 0xf1, 0xf4, 0x3e, 0x43, 0x8f, 0x6c, +- 0xae, 0x22, 0x59, 0x11, 0x86, 0x22, 0x35, +- 0x95 }, ++ 0xae, 0x22, 0x59, 0x11, 0x19, 0x3c, 0x04, ++ 0x45 }, + .sn_size = 18, + .hfn = 0x1, + .hfn_threshold = 0xfa558, +@@ -3193,7 +3193,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x4b, 0x32, 0x87, 0xf9, 0xdb, 0xe0, + 0x31, 0x5f, 0x3a, 0x15 }, + .data_in = +- (uint8_t[]){ 0xf8, 0x00, 0x00, 0xf8, 0xdb, 0x2d, 0x3f, ++ (uint8_t[]){ 0x80, 0x00, 0x00, 0xf8, 0xdb, 0x2d, 0x3f, + 0x23, 0x82, 0x53, 0xfd, 0x37, 0xde, 0x88, + 0x63, 0x08, 0x4f, 0xD3, 0x71, 0xfb, 0xeb, + 0x35, 0xf3, 0x64, 0xd3, 0x5e, 0xaf, 0x3f, +@@ -3205,7 +3205,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0xf9, 0xdd, 0xcc, 0x69 }, + .in_len = 67, + .data_out = +- (uint8_t[]){ 0xf8, 0x00, 0x00, 0xf8, 0x5a, 0xa9, 0xb5, ++ (uint8_t[]){ 0x80, 0x00, 0x00, 0xf8, 0x5a, 0xa9, 0xb5, + 0x61, 0x8b, 0x8a, 0xb7, 0x6a, 0x98, 0x30, + 0x6c, 0xed, 0x84, 0x69, 0xff, 0x6b, 0x7e, + 0x30, 0x59, 0x55, 0x80, 0x32, 0xd0, 0x0f, +@@ -3214,8 +3214,8 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x0e, 0x2e, 0x33, 0xe6, 0xa4, 0xea, 0x17, + 0xd6, 0x56, 0xa2, 0x3b, 0x5f, 0x56, 0x3a, + 0xa7, 0x6f, 0x4c, 0xc1, 0xca, 0xbd, 0x5f, +- 0xad, 0x3d, 0x99, 0x4a, 0x53, 0xf6, 0xfc, +- 0xeb }, ++ 0xad, 0x3d, 0x99, 0x4a, 0x7b, 0x7d, 0x8a, ++ 0x76 }, + .sn_size = 18, + .hfn = 0x1, + .hfn_threshold = 0xfa558, +@@ -3340,7 +3340,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x4b, 0x32, 0x87, 0xf9, 0xdb, 0xe0, + 0x31, 0x5f, 0x3a, 0x15 }, + .data_in = +- (uint8_t[]){ 0xf8, 0x00, 0x00, 0xf8, 0xdb, 0x2d, 0x3f, ++ (uint8_t[]){ 0x80, 0x00, 0x00, 0xf8, 0xdb, 0x2d, 0x3f, + 0x23, 0x82, 0x53, 0xfd, 0x37, 0xde, 0x88, + 0x63, 0x08, 0x4f, 0xD3, 0x71, 0xfb, 0xeb, + 0x35, 0xf3, 0x64, 0xd3, 0x5e, 0xaf, 0x3f, +@@ -3352,7 +3352,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0xf9, 0xdd, 0xcc, 0x69 }, + .in_len = 67, + .data_out = +- (uint8_t[]){ 0xf8, 0x00, 0x00, 0xf8, 0x9b, 0x9c, 0xde, ++ (uint8_t[]){ 0x80, 0x00, 0x00, 0xf8, 0x9b, 0x9c, 0xde, + 0xa1, 0x69, 0x9b, 0x27, 0xd3, 0x38, 0x93, + 0xf2, 0x12, 0xb1, 0xc6, 0x60, 0xac, 0xb7, + 0xf2, 0x37, 0xf3, 0x72, 0xaf, 0x50, 0x9d, +@@ -3361,8 +3361,8 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0xbb, 0x15, 0xf3, 0x6a, 0x5d, 0x61, 0x12, + 0x6e, 0x6d, 0x1b, 0x92, 0xd8, 0x42, 0x60, + 0x3e, 0x1f, 0xe0, 0x6c, 0x28, 0x89, 0xf7, +- 0x0c, 0x61, 0x76, 0xdc, 0x25, 0x8a, 0x31, +- 0xed }, ++ 0x0c, 0x61, 0x76, 0xdc, 0x9d, 0x1f, 0x9a, ++ 0x03 }, + .sn_size = 18, + .hfn = 0x1, + .hfn_threshold = 0xfa558, +@@ -3389,7 +3389,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x4b, 0x32, 0x87, 0xf9, 0xdb, 0xe0, + 0x31, 0x5f, 0x3a, 0x15 }, + .data_in = +- (uint8_t[]){ 0xf8, 0x00, 0x00, 0xf8, 0xdb, 0x2d, 0x3f, ++ (uint8_t[]){ 0x80, 0x00, 0x00, 0xf8, 0xdb, 0x2d, 0x3f, + 0x23, 0x82, 0x53, 0xfd, 0x37, 0xde, 0x88, + 0x63, 0x08, 0x4f, 0xD3, 0x71, 0xfb, 0xeb, + 0x35, 0xf3, 0x64, 0xd3, 0x5e, 0xaf, 0x3f, +@@ -3401,7 +3401,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0xf9, 0xdd, 0xcc, 0x69 }, + .in_len = 67, + .data_out = +- (uint8_t[]){ 0xf8, 0x00, 0x00, 0xf8, 0x0c, 0xef, 0x82, ++ (uint8_t[]){ 0x80, 0x00, 0x00, 0xf8, 0x0c, 0xef, 0x82, + 0x59, 0x6b, 0x6b, 0x61, 0xbe, 0x54, 0x23, + 0x7d, 0x29, 0x6d, 0xa7, 0xd2, 0xfa, 0x26, + 0xcc, 0x1d, 0x18, 0x39, 0x99, 0xea, 0xac, +@@ -3410,8 +3410,8 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x7a, 0x8a, 0xe5, 0x00, 0x35, 0x4d, 0x0e, + 0x69, 0x9f, 0x4b, 0x72, 0x94, 0x8a, 0x25, + 0x69, 0x43, 0x28, 0xdc, 0x40, 0x60, 0x4f, +- 0x6c, 0xed, 0x6a, 0x50, 0xf3, 0x5e, 0x90, +- 0x42 }, ++ 0x6c, 0xed, 0x6a, 0x50, 0x55, 0xf6, 0xaf, ++ 0x9a }, + .sn_size = 18, + .hfn = 0x1, + .hfn_threshold = 0xfa558, +@@ -3438,7 +3438,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x4b, 0x32, 0x87, 0xf9, 0xdb, 0xe0, + 0x31, 0x5f, 0x3a, 0x15 }, + .data_in = +- (uint8_t[]){ 0xf8, 0x00, 0x00, 0xf8, 0xdb, 0x2d, 0x3f, ++ (uint8_t[]){ 0x80, 0x00, 0x00, 0xf8, 0xdb, 0x2d, 0x3f, + 0x23, 0x82, 0x53, 0xfd, 0x37, 0xde, 0x88, + 0x63, 0x08, 0x4f, 0xD3, 0x71, 0xfb, 0xeb, + 0x35, 0xf3, 0x64, 0xd3, 0x5e, 0xaf, 0x3f, +@@ -3450,7 +3450,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0xf9, 0xdd, 0xcc, 0x69 }, + .in_len = 67, + .data_out = +- (uint8_t[]){ 0xf8, 0x00, 0x00, 0xf8, 0x9b, 0x9c, 0xde, ++ (uint8_t[]){ 0x80, 0x00, 0x00, 0xf8, 0x9b, 0x9c, 0xde, + 0xa1, 0x69, 0x9b, 0x27, 0xd3, 0x38, 0x93, + 0xf2, 0x12, 0xb1, 0xc6, 0x60, 0xac, 0xb7, + 0xf2, 0x37, 0xf3, 0x72, 0xaf, 0x50, 0x9d, +@@ -3459,8 +3459,8 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0xbb, 0x15, 0xf3, 0x6a, 0x5d, 0x61, 0x12, + 0x6e, 0x6d, 0x1b, 0x92, 0xd8, 0x42, 0x60, + 0x3e, 0x1f, 0xe0, 0x6c, 0x28, 0x89, 0xf7, +- 0x0c, 0x61, 0x76, 0xdc, 0x55, 0xf1, 0x2b, +- 0x66 }, ++ 0x0c, 0x61, 0x76, 0xdc, 0xd3, 0x21, 0xba, ++ 0x31 }, + .sn_size = 18, + .hfn = 0x1, + .hfn_threshold = 0xfa558, +@@ -3487,7 +3487,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x4b, 0x32, 0x87, 0xf9, 0xdb, 0xe0, + 0x31, 0x5f, 0x3a, 0x15 }, + .data_in = +- (uint8_t[]){ 0xf8, 0x00, 0x00, 0xf8, 0xdb, 0x2d, 0x3f, ++ (uint8_t[]){ 0x80, 0x00, 0x00, 0xf8, 0xdb, 0x2d, 0x3f, + 0x23, 0x82, 0x53, 0xfd, 0x37, 0xde, 0x88, + 0x63, 0x08, 0x4f, 0xD3, 0x71, 0xfb, 0xeb, + 0x35, 0xf3, 0x64, 0xd3, 0x5e, 0xaf, 0x3f, +@@ -3499,7 +3499,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0xf9, 0xdd, 0xcc, 0x69 }, + .in_len = 67, + .data_out = +- (uint8_t[]){ 0xf8, 0x00, 0x00, 0xf8, 0x0c, 0xef, 0x82, ++ (uint8_t[]){ 0x80, 0x00, 0x00, 0xf8, 0x0c, 0xef, 0x82, + 0x59, 0x6b, 0x6b, 0x61, 0xbe, 0x54, 0x23, + 0x7d, 0x29, 0x6d, 0xa7, 0xd2, 0xfa, 0x26, + 0xcc, 0x1d, 0x18, 0x39, 0x99, 0xea, 0xac, +@@ -3508,8 +3508,8 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x7a, 0x8a, 0xe5, 0x00, 0x35, 0x4d, 0x0e, + 0x69, 0x9f, 0x4b, 0x72, 0x94, 0x8a, 0x25, + 0x69, 0x43, 0x28, 0xdc, 0x40, 0x60, 0x4f, +- 0x6c, 0xed, 0x6a, 0x50, 0xd5, 0x84, 0xf6, +- 0x4c }, ++ 0x6c, 0xed, 0x6a, 0x50, 0x83, 0xf9, 0x22, ++ 0x24 }, + .sn_size = 18, + .hfn = 0x1, + .hfn_threshold = 0xfa558, +@@ -3536,7 +3536,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x4b, 0x32, 0x87, 0xf9, 0xdb, 0xe0, + 0x31, 0x5f, 0x3a, 0x15 }, + .data_in = +- (uint8_t[]){ 0xf8, 0x00, 0x00, 0xf8, 0xdb, 0x2d, 0x3f, ++ (uint8_t[]){ 0x80, 0x00, 0x00, 0xf8, 0xdb, 0x2d, 0x3f, + 0x23, 0x82, 0x53, 0xfd, 0x37, 0xde, 0x88, + 0x63, 0x08, 0x4f, 0xD3, 0x71, 0xfb, 0xeb, + 0x35, 0xf3, 0x64, 0xd3, 0x5e, 0xaf, 0x3f, +@@ -3548,7 +3548,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0xf9, 0xdd, 0xcc, 0x69 }, + .in_len = 67, + .data_out = +- (uint8_t[]){ 0xf8, 0x00, 0x00, 0xf8, 0x9b, 0x9c, 0xde, ++ (uint8_t[]){ 0x80, 0x00, 0x00, 0xf8, 0x9b, 0x9c, 0xde, + 0xa1, 0x69, 0x9b, 0x27, 0xd3, 0x38, 0x93, + 0xf2, 0x12, 0xb1, 0xc6, 0x60, 0xac, 0xb7, + 0xf2, 0x37, 0xf3, 0x72, 0xaf, 0x50, 0x9d, +@@ -3557,8 +3557,8 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0xbb, 0x15, 0xf3, 0x6a, 0x5d, 0x61, 0x12, + 0x6e, 0x6d, 0x1b, 0x92, 0xd8, 0x42, 0x60, + 0x3e, 0x1f, 0xe0, 0x6c, 0x28, 0x89, 0xf7, +- 0x0c, 0x61, 0x76, 0xdc, 0x55, 0x3f, 0x0f, +- 0x03 }, ++ 0x0c, 0x61, 0x76, 0xdc, 0xca, 0x21, 0x3e, ++ 0xd3 }, + .sn_size = 18, + .hfn = 0x1, + .hfn_threshold = 0xfa558, +@@ -3585,7 +3585,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x4b, 0x32, 0x87, 0xf9, 0xdb, 0xe0, + 0x31, 0x5f, 0x3a, 0x15 }, + .data_in = +- (uint8_t[]){ 0xf8, 0x00, 0x00, 0xf8, 0xdb, 0x2d, 0x3f, ++ (uint8_t[]){ 0x80, 0x00, 0x00, 0xf8, 0xdb, 0x2d, 0x3f, + 0x23, 0x82, 0x53, 0xfd, 0x37, 0xde, 0x88, + 0x63, 0x08, 0x4f, 0xD3, 0x71, 0xfb, 0xeb, + 0x35, 0xf3, 0x64, 0xd3, 0x5e, 0xaf, 0x3f, +@@ -3597,7 +3597,7 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0xf9, 0xdd, 0xcc, 0x69 }, + .in_len = 67, + .data_out = +- (uint8_t[]){ 0xf8, 0x00, 0x00, 0xf8, 0x0c, 0xef, 0x82, ++ (uint8_t[]){ 0x80, 0x00, 0x00, 0xf8, 0x0c, 0xef, 0x82, + 0x59, 0x6b, 0x6b, 0x61, 0xbe, 0x54, 0x23, + 0x7d, 0x29, 0x6d, 0xa7, 0xd2, 0xfa, 0x26, + 0xcc, 0x1d, 0x18, 0x39, 0x99, 0xea, 0xac, +@@ -3606,8 +3606,8 @@ static const struct pdcp_sdap_test list_pdcp_sdap_tests[] = { + 0x7a, 0x8a, 0xe5, 0x00, 0x35, 0x4d, 0x0e, + 0x69, 0x9f, 0x4b, 0x72, 0x94, 0x8a, 0x25, + 0x69, 0x43, 0x28, 0xdc, 0x40, 0x60, 0x4f, +- 0x6c, 0xed, 0x6a, 0x50, 0x03, 0x03, 0xb9, +- 0xd5 }, ++ 0x6c, 0xed, 0x6a, 0x50, 0x2b, 0x88, 0xcf, ++ 0x48 }, + .sn_size = 18, + .hfn = 0x1, + .hfn_threshold = 0xfa558, +diff --git a/dpdk/app/test/test_cryptodev_security_pdcp_test_vectors.h b/dpdk/app/test/test_cryptodev_security_pdcp_test_vectors.h +index 6fdc4cd9e3..56d4884529 100644 +--- a/dpdk/app/test/test_cryptodev_security_pdcp_test_vectors.h ++++ b/dpdk/app/test/test_cryptodev_security_pdcp_test_vectors.h +@@ -5560,7 +5560,7 @@ static uint8_t *pdcp_test_data_out[] = { + 0xC2, 0xE2, 0x91, 0x91, 0xA3, 0x9C, 0xE6, 0x30, 0x69, 0x70, + 0x33, 0x8A, 0x15, 0xD0, 0x36, 0x47, 0x0E, 0x8F, 0xEE, 0x2C, + 0x96, 0x0C, 0xD7, 0x7D, 0x70, 0x1B, 0x01, 0x7F, 0x96, 0x46, +- 0x53, 0xB0, 0xA4, 0x7A, 0xF9, 0xDD}, ++ 0x53, 0xB0, 0xA4, 0x7A, 0xF9, 0xDD, 0x00, 0x00, 0x00, 0x00}, + /* Control Plane w/NULL enc. + NULL int. DL LONG SN */ + (uint8_t[]){0xA0, 0x00, 0x86, 0xB8, 0xF8, 0xDB, 0x2D, 0x3F, 0x23, 0x82, + 0x53, 0xFD, 0x37, 0xDE, 0x88, 0x63, 0x08, 0x4F, 0xD3, 0x71, +@@ -5568,7 +5568,7 @@ static uint8_t *pdcp_test_data_out[] = { + 0xC2, 0xE2, 0x91, 0x91, 0xA3, 0x9C, 0xE6, 0x30, 0x69, 0x70, + 0x33, 0x8A, 0x15, 0xD0, 0x36, 0x47, 0x0E, 0x8F, 0xEE, 0x2C, + 0x96, 0x0C, 0xD7, 0x7D, 0x70, 0x1B, 0x01, 0x7F, 0x96, 0x46, +- 0x53, 0xB0, 0xA4, 0x7A, 0xF9, 0xDD}, ++ 0x53, 0xB0, 0xA4, 0x7A, 0xF9, 0xDD, 0x00, 0x00, 0x00, 0x00}, + /* Control Plane w/NULL enc. + SNOW f9 int. UL LONG SN */ + (uint8_t[]){0x00, 0x01, 0x86, 0xB8, 0xF8, 0xDB, 0x2D, 0x3F, 0x23, 0x82, + 0x53, 0xFD, 0x37, 0xDE, 0x88, 0x63, 0x08, 0x4F, 0xD3, 0x71, +@@ -5954,7 +5954,7 @@ static uint8_t *pdcp_test_data_out[] = { + 0xC2, 0xE2, 0x91, 0x91, 0xA3, 0x9C, 0xE6, 0x30, 0x69, 0x70, + 0x33, 0x8A, 0x15, 0xD0, 0x36, 0x47, 0x0E, 0x8F, 0xEE, 0x2C, + 0x96, 0x0C, 0xD7, 0x7D, 0x70, 0x1B, 0x01, 0x7F, 0x96, 0x46, +- 0x53, 0xB0, 0xA4, 0x7A, 0xF9, 0xDD}, ++ 0x53, 0xB0, 0xA4, 0x7A, 0xF9, 0xDD, 0x00, 0x00, 0x00, 0x00}, + /* User Plane w/NULL enc. + NULL int. DL for 12-bit SN */ + (uint8_t[]){0xA0, 0x00, 0x86, 0xB8, 0xF8, 0xDB, 0x2D, 0x3F, 0x23, 0x82, + 0x53, 0xFD, 0x37, 0xDE, 0x88, 0x63, 0x08, 0x4F, 0xD3, 0x71, +@@ -5962,7 +5962,7 @@ static uint8_t *pdcp_test_data_out[] = { + 0xC2, 0xE2, 0x91, 0x91, 0xA3, 0x9C, 0xE6, 0x30, 0x69, 0x70, + 0x33, 0x8A, 0x15, 0xD0, 0x36, 0x47, 0x0E, 0x8F, 0xEE, 0x2C, + 0x96, 0x0C, 0xD7, 0x7D, 0x70, 0x1B, 0x01, 0x7F, 0x96, 0x46, +- 0x53, 0xB0, 0xA4, 0x7A, 0xF9, 0xDD}, ++ 0x53, 0xB0, 0xA4, 0x7A, 0xF9, 0xDD, 0x00, 0x00, 0x00, 0x00}, + /* User Plane w/NULL enc. + SNOW f9 int. UL for 12-bit SN */ + (uint8_t[]){0x80, 0x01, 0x86, 0xB8, 0xF8, 0xDB, 0x2D, 0x3F, 0x23, 0x82, + 0x53, 0xFD, 0x37, 0xDE, 0x88, 0x63, 0x08, 0x4F, 0xD3, 0x71, +diff --git a/dpdk/app/test/test_event_crypto_adapter.c b/dpdk/app/test/test_event_crypto_adapter.c +index a38e389abd..3d720fe68b 100644 +--- a/dpdk/app/test/test_event_crypto_adapter.c ++++ b/dpdk/app/test/test_event_crypto_adapter.c +@@ -958,11 +958,10 @@ configure_cryptodev(void) + return TEST_FAILED; + } + +- /* Create a NULL crypto device */ +- nb_devs = rte_cryptodev_device_count_by_driver( +- rte_cryptodev_driver_id_get( +- RTE_STR(CRYPTODEV_NAME_NULL_PMD))); ++ ++ nb_devs = rte_cryptodev_count(); + if (!nb_devs) { ++ /* Create a NULL crypto device */ + ret = rte_vdev_init( + RTE_STR(CRYPTODEV_NAME_NULL_PMD), NULL); + +diff --git a/dpdk/app/test/test_event_timer_adapter.c b/dpdk/app/test/test_event_timer_adapter.c +index 1a440dfd10..12d5936c60 100644 +--- a/dpdk/app/test/test_event_timer_adapter.c ++++ b/dpdk/app/test/test_event_timer_adapter.c +@@ -57,9 +57,10 @@ static uint64_t global_bkt_tck_ns; + static uint64_t global_info_bkt_tck_ns; + static volatile uint8_t arm_done; + +-#define CALC_TICKS(tks) \ +- ceil((double)(tks * global_bkt_tck_ns) / global_info_bkt_tck_ns) ++#define CALC_TICKS(tks) ceil((double)((tks) * global_bkt_tck_ns) / global_info_bkt_tck_ns) + ++/* Wait double timeout ticks for software and an extra tick for hardware */ ++#define WAIT_TICKS(tks) (using_services ? 2 * (tks) : tks + 1) + + static bool using_services; + static uint32_t test_lcore1; +@@ -441,10 +442,31 @@ timdev_teardown(void) + rte_mempool_free(eventdev_test_mempool); + } + ++static inline uint16_t ++timeout_event_dequeue(struct rte_event *evs, uint64_t nb_evs, uint64_t ticks) ++{ ++ uint16_t ev_cnt = 0; ++ uint64_t end_cycle; ++ ++ if (using_services && nb_evs == MAX_TIMERS) ++ ticks = 2 * ticks; ++ ++ end_cycle = rte_rdtsc() + ticks * global_bkt_tck_ns * rte_get_tsc_hz() / 1E9; ++ ++ while (ev_cnt < nb_evs && rte_rdtsc() < end_cycle) { ++ ev_cnt += rte_event_dequeue_burst(evdev, TEST_PORT_ID, &evs[ev_cnt], nb_evs, 0); ++ rte_pause(); ++ } ++ ++ return ev_cnt; ++} ++ + static inline int + test_timer_state(void) + { + struct rte_event_timer *ev_tim; ++ const uint64_t max_ticks = 100; ++ uint64_t ticks, wait_ticks; + struct rte_event ev; + const struct rte_event_timer tim = { + .ev.op = RTE_EVENT_OP_NEW, +@@ -455,11 +477,10 @@ test_timer_state(void) + .state = RTE_EVENT_TIMER_NOT_ARMED, + }; + +- + rte_mempool_get(eventdev_test_mempool, (void **)&ev_tim); + *ev_tim = tim; + ev_tim->ev.event_ptr = ev_tim; +- ev_tim->timeout_ticks = CALC_TICKS(120); ++ ev_tim->timeout_ticks = CALC_TICKS(max_ticks + 20); + + TEST_ASSERT_EQUAL(rte_event_timer_arm_burst(timdev, &ev_tim, 1), 0, + "Armed timer exceeding max_timeout."); +@@ -467,8 +488,9 @@ test_timer_state(void) + "Improper timer state set expected %d returned %d", + RTE_EVENT_TIMER_ERROR_TOOLATE, ev_tim->state); + ++ ticks = 10; + ev_tim->state = RTE_EVENT_TIMER_NOT_ARMED; +- ev_tim->timeout_ticks = CALC_TICKS(10); ++ ev_tim->timeout_ticks = CALC_TICKS(ticks); + + TEST_ASSERT_EQUAL(rte_event_timer_arm_burst(timdev, &ev_tim, 1), 1, + "Failed to arm timer with proper timeout."); +@@ -477,14 +499,15 @@ test_timer_state(void) + RTE_EVENT_TIMER_ARMED, ev_tim->state); + + if (!using_services) +- rte_delay_us(20); ++ wait_ticks = 2 * ticks; + else +- rte_delay_us(1000 + 200); +- TEST_ASSERT_EQUAL(rte_event_dequeue_burst(evdev, 0, &ev, 1, 0), 1, +- "Armed timer failed to trigger."); ++ wait_ticks = ticks; ++ ++ TEST_ASSERT_EQUAL(timeout_event_dequeue(&ev, 1, WAIT_TICKS(wait_ticks)), 1, ++ "Armed timer failed to trigger."); + + ev_tim->state = RTE_EVENT_TIMER_NOT_ARMED; +- ev_tim->timeout_ticks = CALC_TICKS(90); ++ ev_tim->timeout_ticks = CALC_TICKS(max_ticks - 10); + TEST_ASSERT_EQUAL(rte_event_timer_arm_burst(timdev, &ev_tim, 1), 1, + "Failed to arm timer with proper timeout."); + TEST_ASSERT_EQUAL(rte_event_timer_cancel_burst(timdev, &ev_tim, 1), +@@ -1208,8 +1231,9 @@ stat_inc_reset_ev_enq(void) + int ret, i, n; + int num_evtims = MAX_TIMERS; + struct rte_event_timer *evtims[num_evtims]; +- struct rte_event evs[BATCH_SIZE]; ++ struct rte_event evs[num_evtims]; + struct rte_event_timer_adapter_stats stats; ++ uint64_t ticks = 5; + const struct rte_event_timer init_tim = { + .ev.op = RTE_EVENT_OP_NEW, + .ev.queue_id = TEST_QUEUE_ID, +@@ -1217,7 +1241,7 @@ stat_inc_reset_ev_enq(void) + .ev.priority = RTE_EVENT_DEV_PRIORITY_NORMAL, + .ev.event_type = RTE_EVENT_TYPE_TIMER, + .state = RTE_EVENT_TIMER_NOT_ARMED, +- .timeout_ticks = CALC_TICKS(5), // expire in .5 sec ++ .timeout_ticks = CALC_TICKS(ticks), /**< expire in .5 sec */ + }; + + ret = rte_mempool_get_bulk(eventdev_test_mempool, (void **)evtims, +@@ -1242,31 +1266,12 @@ stat_inc_reset_ev_enq(void) + "succeeded = %d, rte_errno = %s", + num_evtims, ret, rte_strerror(rte_errno)); + +- rte_delay_ms(1000); +- +-#define MAX_TRIES num_evtims +- int sum = 0; +- int tries = 0; +- bool done = false; +- while (!done) { +- sum += rte_event_dequeue_burst(evdev, TEST_PORT_ID, evs, +- RTE_DIM(evs), 10); +- if (sum >= num_evtims || ++tries >= MAX_TRIES) +- done = true; +- +- rte_delay_ms(10); +- } +- +- TEST_ASSERT_EQUAL(sum, num_evtims, "Expected %d timer expiry events, " +- "got %d", num_evtims, sum); +- +- TEST_ASSERT(tries < MAX_TRIES, "Exceeded max tries"); +- +- rte_delay_ms(100); ++ n = timeout_event_dequeue(evs, RTE_DIM(evs), WAIT_TICKS(ticks)); ++ TEST_ASSERT_EQUAL(n, num_evtims, "Expected %d timer expiry events, got %d", ++ num_evtims, n); + + /* Make sure the eventdev is still empty */ +- n = rte_event_dequeue_burst(evdev, TEST_PORT_ID, evs, RTE_DIM(evs), +- 10); ++ n = timeout_event_dequeue(evs, 1, WAIT_TICKS(1)); + + TEST_ASSERT_EQUAL(n, 0, "Dequeued unexpected number of timer expiry " + "events from event device"); +@@ -1303,6 +1308,7 @@ event_timer_arm(void) + struct rte_event_timer_adapter *adapter = timdev; + struct rte_event_timer *evtim = NULL; + struct rte_event evs[BATCH_SIZE]; ++ uint64_t ticks = 5; + const struct rte_event_timer init_tim = { + .ev.op = RTE_EVENT_OP_NEW, + .ev.queue_id = TEST_QUEUE_ID, +@@ -1310,7 +1316,7 @@ event_timer_arm(void) + .ev.priority = RTE_EVENT_DEV_PRIORITY_NORMAL, + .ev.event_type = RTE_EVENT_TYPE_TIMER, + .state = RTE_EVENT_TIMER_NOT_ARMED, +- .timeout_ticks = CALC_TICKS(5), // expire in .5 sec ++ .timeout_ticks = CALC_TICKS(ticks), /**< expire in .5 sec */ + }; + + rte_mempool_get(eventdev_test_mempool, (void **)&evtim); +@@ -1337,10 +1343,7 @@ event_timer_arm(void) + TEST_ASSERT_EQUAL(rte_errno, EALREADY, "Unexpected rte_errno value " + "after arming already armed timer"); + +- /* Let timer expire */ +- rte_delay_ms(1000); +- +- n = rte_event_dequeue_burst(evdev, TEST_PORT_ID, evs, RTE_DIM(evs), 0); ++ n = timeout_event_dequeue(evs, RTE_DIM(evs), WAIT_TICKS(ticks)); + TEST_ASSERT_EQUAL(n, 1, "Failed to dequeue expected number of expiry " + "events from event device"); + +@@ -1360,6 +1363,7 @@ event_timer_arm_double(void) + struct rte_event_timer_adapter *adapter = timdev; + struct rte_event_timer *evtim = NULL; + struct rte_event evs[BATCH_SIZE]; ++ uint64_t ticks = 5; + const struct rte_event_timer init_tim = { + .ev.op = RTE_EVENT_OP_NEW, + .ev.queue_id = TEST_QUEUE_ID, +@@ -1367,7 +1371,7 @@ event_timer_arm_double(void) + .ev.priority = RTE_EVENT_DEV_PRIORITY_NORMAL, + .ev.event_type = RTE_EVENT_TYPE_TIMER, + .state = RTE_EVENT_TIMER_NOT_ARMED, +- .timeout_ticks = CALC_TICKS(5), // expire in .5 sec ++ .timeout_ticks = CALC_TICKS(ticks), /**< expire in .5 sec */ + }; + + rte_mempool_get(eventdev_test_mempool, (void **)&evtim); +@@ -1387,10 +1391,7 @@ event_timer_arm_double(void) + TEST_ASSERT_EQUAL(rte_errno, EALREADY, "Unexpected rte_errno value " + "after double-arm"); + +- /* Let timer expire */ +- rte_delay_ms(600); +- +- n = rte_event_dequeue_burst(evdev, TEST_PORT_ID, evs, RTE_DIM(evs), 0); ++ n = timeout_event_dequeue(evs, RTE_DIM(evs), WAIT_TICKS(ticks)); + TEST_ASSERT_EQUAL(n, 1, "Dequeued incorrect number of expiry events - " + "expected: 1, actual: %d", n); + +@@ -1417,6 +1418,7 @@ event_timer_arm_expiry(void) + .ev.event_type = RTE_EVENT_TYPE_TIMER, + .state = RTE_EVENT_TIMER_NOT_ARMED, + }; ++ uint64_t ticks = 30; + + rte_mempool_get(eventdev_test_mempool, (void **)&evtim); + if (evtim == NULL) { +@@ -1426,7 +1428,7 @@ event_timer_arm_expiry(void) + + /* Set up an event timer */ + *evtim = init_tim; +- evtim->timeout_ticks = CALC_TICKS(30), // expire in 3 secs ++ evtim->timeout_ticks = CALC_TICKS(ticks); /**< expire in 3 secs */ + evtim->ev.event_ptr = evtim; + + ret = rte_event_timer_arm_burst(adapter, &evtim, 1); +@@ -1435,17 +1437,10 @@ event_timer_arm_expiry(void) + TEST_ASSERT_EQUAL(evtim->state, RTE_EVENT_TIMER_ARMED, "Event " + "timer in incorrect state"); + +- rte_delay_ms(2999); +- +- n = rte_event_dequeue_burst(evdev, TEST_PORT_ID, evs, RTE_DIM(evs), 0); ++ n = timeout_event_dequeue(evs, RTE_DIM(evs), ticks - 1); + TEST_ASSERT_EQUAL(n, 0, "Dequeued unexpected timer expiry event"); + +- /* Delay 100 ms to account for the adapter tick window - should let us +- * dequeue one event +- */ +- rte_delay_ms(100); +- +- n = rte_event_dequeue_burst(evdev, TEST_PORT_ID, evs, RTE_DIM(evs), 0); ++ n = timeout_event_dequeue(evs, RTE_DIM(evs), WAIT_TICKS(1)); + TEST_ASSERT_EQUAL(n, 1, "Dequeued incorrect number (%d) of timer " + "expiry events", n); + TEST_ASSERT_EQUAL(evs[0].event_type, RTE_EVENT_TYPE_TIMER, +@@ -1477,6 +1472,7 @@ event_timer_arm_rearm(void) + .ev.event_type = RTE_EVENT_TYPE_TIMER, + .state = RTE_EVENT_TIMER_NOT_ARMED, + }; ++ uint64_t ticks = 1; + + rte_mempool_get(eventdev_test_mempool, (void **)&evtim); + if (evtim == NULL) { +@@ -1486,7 +1482,7 @@ event_timer_arm_rearm(void) + + /* Set up a timer */ + *evtim = init_tim; +- evtim->timeout_ticks = CALC_TICKS(1); // expire in 0.1 sec ++ evtim->timeout_ticks = CALC_TICKS(ticks); /**< expire in 0.1 sec */ + evtim->ev.event_ptr = evtim; + + /* Arm it */ +@@ -1494,10 +1490,7 @@ event_timer_arm_rearm(void) + TEST_ASSERT_EQUAL(ret, 1, "Failed to arm event timer: %s\n", + rte_strerror(rte_errno)); + +- /* Add 100ms to account for the adapter tick window */ +- rte_delay_ms(100 + 100); +- +- n = rte_event_dequeue_burst(evdev, TEST_PORT_ID, evs, RTE_DIM(evs), 0); ++ n = timeout_event_dequeue(evs, RTE_DIM(evs), WAIT_TICKS(ticks)); + TEST_ASSERT_EQUAL(n, 1, "Failed to dequeue expected number of expiry " + "events from event device"); + +@@ -1514,10 +1507,7 @@ event_timer_arm_rearm(void) + TEST_ASSERT_EQUAL(ret, 1, "Failed to arm event timer: %s\n", + rte_strerror(rte_errno)); + +- /* Add 100ms to account for the adapter tick window */ +- rte_delay_ms(100 + 100); +- +- n = rte_event_dequeue_burst(evdev, TEST_PORT_ID, evs, RTE_DIM(evs), 0); ++ n = timeout_event_dequeue(evs, RTE_DIM(evs), WAIT_TICKS(ticks)); + TEST_ASSERT_EQUAL(n, 1, "Failed to dequeue expected number of expiry " + "events from event device"); + +@@ -1539,7 +1529,8 @@ event_timer_arm_max(void) + int ret, i, n; + int num_evtims = MAX_TIMERS; + struct rte_event_timer *evtims[num_evtims]; +- struct rte_event evs[BATCH_SIZE]; ++ struct rte_event evs[num_evtims]; ++ uint64_t ticks = 5; + const struct rte_event_timer init_tim = { + .ev.op = RTE_EVENT_OP_NEW, + .ev.queue_id = TEST_QUEUE_ID, +@@ -1547,7 +1538,7 @@ event_timer_arm_max(void) + .ev.priority = RTE_EVENT_DEV_PRIORITY_NORMAL, + .ev.event_type = RTE_EVENT_TYPE_TIMER, + .state = RTE_EVENT_TIMER_NOT_ARMED, +- .timeout_ticks = CALC_TICKS(5), // expire in .5 sec ++ .timeout_ticks = CALC_TICKS(ticks), /**< expire in .5 sec */ + }; + + ret = rte_mempool_get_bulk(eventdev_test_mempool, (void **)evtims, +@@ -1567,31 +1558,12 @@ event_timer_arm_max(void) + "succeeded = %d, rte_errno = %s", + num_evtims, ret, rte_strerror(rte_errno)); + +- rte_delay_ms(1000); +- +-#define MAX_TRIES num_evtims +- int sum = 0; +- int tries = 0; +- bool done = false; +- while (!done) { +- sum += rte_event_dequeue_burst(evdev, TEST_PORT_ID, evs, +- RTE_DIM(evs), 10); +- if (sum >= num_evtims || ++tries >= MAX_TRIES) +- done = true; +- +- rte_delay_ms(10); +- } +- +- TEST_ASSERT_EQUAL(sum, num_evtims, "Expected %d timer expiry events, " +- "got %d", num_evtims, sum); +- +- TEST_ASSERT(tries < MAX_TRIES, "Exceeded max tries"); +- +- rte_delay_ms(100); ++ n = timeout_event_dequeue(evs, RTE_DIM(evs), WAIT_TICKS(ticks)); ++ TEST_ASSERT_EQUAL(n, num_evtims, "Expected %d timer expiry events, got %d", ++ num_evtims, n); + + /* Make sure the eventdev is still empty */ +- n = rte_event_dequeue_burst(evdev, TEST_PORT_ID, evs, RTE_DIM(evs), +- 10); ++ n = timeout_event_dequeue(evs, 1, WAIT_TICKS(1)); + + TEST_ASSERT_EQUAL(n, 0, "Dequeued unexpected number of timer expiry " + "events from event device"); +@@ -1711,6 +1683,7 @@ event_timer_cancel(void) + .ev.event_type = RTE_EVENT_TYPE_TIMER, + .state = RTE_EVENT_TIMER_NOT_ARMED, + }; ++ uint64_t ticks = 30; + + rte_mempool_get(eventdev_test_mempool, (void **)&evtim); + if (evtim == NULL) { +@@ -1728,7 +1701,7 @@ event_timer_cancel(void) + /* Set up a timer */ + *evtim = init_tim; + evtim->ev.event_ptr = evtim; +- evtim->timeout_ticks = CALC_TICKS(30); // expire in 3 sec ++ evtim->timeout_ticks = CALC_TICKS(ticks); /**< expire in 3 sec */ + + /* Check that cancelling an inited but unarmed timer fails */ + ret = rte_event_timer_cancel_burst(adapter, &evtim, 1); +@@ -1752,10 +1725,8 @@ event_timer_cancel(void) + TEST_ASSERT_EQUAL(evtim->state, RTE_EVENT_TIMER_CANCELED, + "evtim in incorrect state"); + +- rte_delay_ms(3000); +- + /* Make sure that no expiry event was generated */ +- n = rte_event_dequeue_burst(evdev, TEST_PORT_ID, evs, RTE_DIM(evs), 0); ++ n = timeout_event_dequeue(evs, RTE_DIM(evs), WAIT_TICKS(ticks)); + TEST_ASSERT_EQUAL(n, 0, "Dequeued unexpected timer expiry event\n"); + + rte_mempool_put(eventdev_test_mempool, evtim); +@@ -1778,8 +1749,8 @@ event_timer_cancel_double(void) + .ev.priority = RTE_EVENT_DEV_PRIORITY_NORMAL, + .ev.event_type = RTE_EVENT_TYPE_TIMER, + .state = RTE_EVENT_TIMER_NOT_ARMED, +- .timeout_ticks = CALC_TICKS(5), // expire in .5 sec + }; ++ uint64_t ticks = 30; + + rte_mempool_get(eventdev_test_mempool, (void **)&evtim); + if (evtim == NULL) { +@@ -1790,7 +1761,7 @@ event_timer_cancel_double(void) + /* Set up a timer */ + *evtim = init_tim; + evtim->ev.event_ptr = evtim; +- evtim->timeout_ticks = CALC_TICKS(30); // expire in 3 sec ++ evtim->timeout_ticks = CALC_TICKS(ticks); /**< expire in 3 sec */ + + ret = rte_event_timer_arm_burst(adapter, &evtim, 1); + TEST_ASSERT_EQUAL(ret, 1, "Failed to arm event timer: %s\n", +@@ -1812,10 +1783,8 @@ event_timer_cancel_double(void) + TEST_ASSERT_EQUAL(rte_errno, EALREADY, "Unexpected rte_errno value " + "after double-cancel: rte_errno = %d", rte_errno); + +- rte_delay_ms(3000); +- + /* Still make sure that no expiry event was generated */ +- n = rte_event_dequeue_burst(evdev, TEST_PORT_ID, evs, RTE_DIM(evs), 0); ++ n = timeout_event_dequeue(evs, RTE_DIM(evs), WAIT_TICKS(ticks)); + TEST_ASSERT_EQUAL(n, 0, "Dequeued unexpected timer expiry event\n"); + + rte_mempool_put(eventdev_test_mempool, evtim); +diff --git a/dpdk/app/test/test_hash_readwrite.c b/dpdk/app/test/test_hash_readwrite.c +index 6373e62d33..9cc5f3487c 100644 +--- a/dpdk/app/test/test_hash_readwrite.c ++++ b/dpdk/app/test/test_hash_readwrite.c +@@ -162,7 +162,7 @@ init_params(int use_ext, int use_htm, int rw_lf, int use_jhash) + + handle = rte_hash_create(&hash_params); + if (handle == NULL) { +- printf("hash creation failed"); ++ printf("hash creation failed\n"); + return -1; + } + +diff --git a/dpdk/app/test/test_link_bonding.c b/dpdk/app/test/test_link_bonding.c +index 5c496352c2..53f5c13a24 100644 +--- a/dpdk/app/test/test_link_bonding.c ++++ b/dpdk/app/test/test_link_bonding.c +@@ -2,7 +2,7 @@ + * Copyright(c) 2010-2014 Intel Corporation + */ + +-#include "unistd.h" ++#include + #include + #include + #include +@@ -447,7 +447,8 @@ test_add_already_bonded_slave_to_bonded_device(void) + uint16_t slaves[RTE_MAX_ETHPORTS]; + char pmd_name[RTE_ETH_NAME_MAX_LEN]; + +- test_add_slave_to_bonded_device(); ++ TEST_ASSERT_SUCCESS(test_add_slave_to_bonded_device(), ++ "Failed to add member to bonding device"); + + current_slave_count = rte_eth_bond_slaves_get(test_params->bonded_port_id, + slaves, RTE_MAX_ETHPORTS); +@@ -4261,7 +4262,7 @@ test_tlb_tx_burst(void) + burst_size); + TEST_ASSERT_EQUAL(nb_tx, 0, " bad number of packet in burst"); + +- /* Clean ugit checkout masterp and remove slaves from bonded device */ ++ /* Clean up and remove members from bonding device */ + return remove_slaves_and_stop_bonded_device(); + } + +diff --git a/dpdk/app/test/test_link_bonding_mode4.c b/dpdk/app/test/test_link_bonding_mode4.c +index 21c512c94b..7410f99617 100644 +--- a/dpdk/app/test/test_link_bonding_mode4.c ++++ b/dpdk/app/test/test_link_bonding_mode4.c +@@ -641,8 +641,7 @@ bond_handshake(void) + /* If response didn't send - report failure */ + TEST_ASSERT_EQUAL(all_slaves_done, 1, "Bond handshake failed\n"); + +- /* If flags doesn't match - report failure */ +- return all_slaves_done == 1 ? TEST_SUCCESS : TEST_FAILED; ++ return TEST_SUCCESS; + } + + #define TEST_LACP_SLAVE_COUT RTE_DIM(test_params.slave_ports) +diff --git a/dpdk/app/test/test_link_bonding_rssconf.c b/dpdk/app/test/test_link_bonding_rssconf.c +index 464fb2dbd0..7aecee9117 100644 +--- a/dpdk/app/test/test_link_bonding_rssconf.c ++++ b/dpdk/app/test/test_link_bonding_rssconf.c +@@ -324,7 +324,7 @@ test_propagate(void) + uint8_t n; + struct slave_conf *port; + uint8_t bond_rss_key[40]; +- struct rte_eth_rss_conf bond_rss_conf; ++ struct rte_eth_rss_conf bond_rss_conf = {0}; + + int retval = 0; + uint64_t rss_hf = 0; +diff --git a/dpdk/app/test/test_malloc.c b/dpdk/app/test/test_malloc.c +index de40e50611..ff081dd931 100644 +--- a/dpdk/app/test/test_malloc.c ++++ b/dpdk/app/test/test_malloc.c +@@ -302,11 +302,11 @@ test_multi_alloc_statistics(void) + rte_malloc_get_socket_stats(socket,&post_stats); + /* Check statistics reported are correct */ + /* All post stats should be equal to pre stats after alloc freed */ +- if ((post_stats.heap_totalsz_bytes != pre_stats.heap_totalsz_bytes) && +- (post_stats.heap_freesz_bytes!=pre_stats.heap_freesz_bytes) && +- (post_stats.heap_allocsz_bytes!=pre_stats.heap_allocsz_bytes)&& +- (post_stats.alloc_count!=pre_stats.alloc_count)&& +- (post_stats.free_count!=pre_stats.free_count)) { ++ if ((post_stats.heap_totalsz_bytes != pre_stats.heap_totalsz_bytes) || ++ (post_stats.heap_freesz_bytes != pre_stats.heap_freesz_bytes) || ++ (post_stats.heap_allocsz_bytes != pre_stats.heap_allocsz_bytes) || ++ (post_stats.alloc_count != pre_stats.alloc_count) || ++ (post_stats.free_count != pre_stats.free_count)) { + printf("Malloc statistics are incorrect - freed alloc\n"); + return -1; + } +@@ -363,11 +363,11 @@ test_multi_alloc_statistics(void) + return -1; + } + +- if ((post_stats.heap_totalsz_bytes != pre_stats.heap_totalsz_bytes) && +- (post_stats.heap_freesz_bytes!=pre_stats.heap_freesz_bytes) && +- (post_stats.heap_allocsz_bytes!=pre_stats.heap_allocsz_bytes)&& +- (post_stats.alloc_count!=pre_stats.alloc_count)&& +- (post_stats.free_count!=pre_stats.free_count)) { ++ if ((post_stats.heap_totalsz_bytes != pre_stats.heap_totalsz_bytes) || ++ (post_stats.heap_freesz_bytes != pre_stats.heap_freesz_bytes) || ++ (post_stats.heap_allocsz_bytes != pre_stats.heap_allocsz_bytes) || ++ (post_stats.alloc_count != pre_stats.alloc_count) || ++ (post_stats.free_count != pre_stats.free_count)) { + printf("Malloc statistics are incorrect - freed alloc\n"); + return -1; + } +@@ -937,6 +937,7 @@ test_alloc_single_socket(int32_t socket) + if (mem == NULL) + return -1; + if (addr_to_socket(mem) != desired_socket) { ++ rte_free(mem); + return -1; + } + rte_free(mem); +diff --git a/dpdk/app/test/test_mbuf.c b/dpdk/app/test/test_mbuf.c +index 53fe898a38..7a2f8a9980 100644 +--- a/dpdk/app/test/test_mbuf.c ++++ b/dpdk/app/test/test_mbuf.c +@@ -1167,38 +1167,16 @@ test_failing_mbuf_sanity_check(struct rte_mempool *pktmbuf_pool) + return TEST_SKIPPED; + } + #else +- +-#include +-#include +-#include +-#include +- +-/* use fork() to test mbuf errors panic */ +-static int +-verify_mbuf_check_panics(struct rte_mbuf *buf) ++/* Verify if mbuf can pass the check */ ++static bool ++mbuf_check_pass(struct rte_mbuf *buf) + { +- int pid; +- int status; +- +- pid = fork(); +- +- if (pid == 0) { +- struct rlimit rl; ++ const char *reason; + +- /* No need to generate a coredump when panicking. */ +- rl.rlim_cur = rl.rlim_max = 0; +- setrlimit(RLIMIT_CORE, &rl); +- rte_mbuf_sanity_check(buf, 1); /* should panic */ +- exit(0); /* return normally if it doesn't panic */ +- } else if (pid < 0) { +- printf("Fork Failed\n"); +- return -1; +- } +- wait(&status); +- if(status == 0) +- return -1; ++ if (rte_mbuf_check(buf, 1, &reason) == 0) ++ return true; + +- return 0; ++ return false; + } + + static int +@@ -1215,19 +1193,19 @@ test_failing_mbuf_sanity_check(struct rte_mempool *pktmbuf_pool) + return -1; + + printf("Checking good mbuf initially\n"); +- if (verify_mbuf_check_panics(buf) != -1) ++ if (!mbuf_check_pass(buf)) + return -1; + + printf("Now checking for error conditions\n"); + +- if (verify_mbuf_check_panics(NULL)) { ++ if (mbuf_check_pass(NULL)) { + printf("Error with NULL mbuf test\n"); + return -1; + } + + badbuf = *buf; + badbuf.pool = NULL; +- if (verify_mbuf_check_panics(&badbuf)) { ++ if (mbuf_check_pass(&badbuf)) { + printf("Error with bad-pool mbuf test\n"); + return -1; + } +@@ -1235,7 +1213,7 @@ test_failing_mbuf_sanity_check(struct rte_mempool *pktmbuf_pool) + if (RTE_IOVA_AS_PA) { + badbuf = *buf; + rte_mbuf_iova_set(&badbuf, 0); +- if (verify_mbuf_check_panics(&badbuf)) { ++ if (mbuf_check_pass(&badbuf)) { + printf("Error with bad-physaddr mbuf test\n"); + return -1; + } +@@ -1243,21 +1221,21 @@ test_failing_mbuf_sanity_check(struct rte_mempool *pktmbuf_pool) + + badbuf = *buf; + badbuf.buf_addr = NULL; +- if (verify_mbuf_check_panics(&badbuf)) { ++ if (mbuf_check_pass(&badbuf)) { + printf("Error with bad-addr mbuf test\n"); + return -1; + } + + badbuf = *buf; + badbuf.refcnt = 0; +- if (verify_mbuf_check_panics(&badbuf)) { ++ if (mbuf_check_pass(&badbuf)) { + printf("Error with bad-refcnt(0) mbuf test\n"); + return -1; + } + + badbuf = *buf; + badbuf.refcnt = UINT16_MAX; +- if (verify_mbuf_check_panics(&badbuf)) { ++ if (mbuf_check_pass(&badbuf)) { + printf("Error with bad-refcnt(MAX) mbuf test\n"); + return -1; + } +@@ -2744,6 +2722,7 @@ test_nb_segs_and_next_reset(void) + + /* split m0 chain in two, between m1 and m2 */ + m0->nb_segs = 2; ++ m0->pkt_len -= m2->data_len; + m1->next = NULL; + m2->nb_segs = 1; + +@@ -2764,6 +2743,7 @@ test_nb_segs_and_next_reset(void) + m2->nb_segs != 1 || m2->next != NULL) + GOTO_FAIL("nb_segs or next was not reset properly"); + ++ rte_mempool_free(pool); + return 0; + + fail: +diff --git a/dpdk/app/test/test_reorder.c b/dpdk/app/test/test_reorder.c +index f0714a5c18..7b5e590bac 100644 +--- a/dpdk/app/test/test_reorder.c ++++ b/dpdk/app/test/test_reorder.c +@@ -278,6 +278,7 @@ test_reorder_drain(void) + goto exit; + } + rte_pktmbuf_free(robufs[0]); ++ memset(robufs, 0, sizeof(robufs)); + + /* Insert more packets + * RB[] = {NULL, NULL, NULL, NULL} +@@ -313,6 +314,7 @@ test_reorder_drain(void) + for (i = 0; i < 3; i++) { + rte_pktmbuf_free(robufs[i]); + } ++ memset(robufs, 0, sizeof(robufs)); + + /* + * RB[] = {NULL, NULL, NULL, NULL} +diff --git a/dpdk/app/test/test_security_inline_proto.c b/dpdk/app/test/test_security_inline_proto.c +index 79858e559f..0a966d3f4b 100644 +--- a/dpdk/app/test/test_security_inline_proto.c ++++ b/dpdk/app/test/test_security_inline_proto.c +@@ -678,6 +678,8 @@ free_mbuf(struct rte_mbuf *mbuf) + ip_reassembly_dynfield_offset, + rte_eth_ip_reassembly_dynfield_t *); + rte_pktmbuf_free(mbuf); ++ if (dynfield.nb_frags == 0) ++ break; + mbuf = dynfield.next_frag; + } + } +@@ -735,6 +737,53 @@ get_and_verify_incomplete_frags(struct rte_mbuf *mbuf, + return ret; + } + ++static int ++event_tx_burst(struct rte_mbuf **tx_pkts, uint16_t nb_pkts) ++{ ++ struct rte_event ev; ++ int i, nb_sent = 0; ++ ++ /* Convert packets to events */ ++ memset(&ev, 0, sizeof(ev)); ++ ev.sched_type = RTE_SCHED_TYPE_PARALLEL; ++ for (i = 0; i < nb_pkts; i++) { ++ ev.mbuf = tx_pkts[i]; ++ ev.mbuf->port = port_id; ++ nb_sent += rte_event_eth_tx_adapter_enqueue( ++ eventdev_id, port_id, &ev, 1, 0); ++ } ++ ++ return nb_sent; ++} ++ ++static int ++event_rx_burst(struct rte_mbuf **rx_pkts, uint16_t nb_pkts_to_rx) ++{ ++ int nb_ev, nb_rx = 0, j = 0; ++ const int ms_per_pkt = 5; ++ struct rte_event ev; ++ ++ do { ++ nb_ev = rte_event_dequeue_burst(eventdev_id, port_id, ++ &ev, 1, 0); ++ ++ if (nb_ev == 0) { ++ rte_delay_ms(1); ++ continue; ++ } ++ ++ /* Get packet from event */ ++ if (ev.event_type != RTE_EVENT_TYPE_ETHDEV) { ++ printf("Unsupported event type: %i\n", ++ ev.event_type); ++ continue; ++ } ++ rx_pkts[nb_rx++] = ev.mbuf; ++ } while (j++ < (nb_pkts_to_rx * ms_per_pkt) && nb_rx < nb_pkts_to_rx); ++ ++ return nb_rx; ++} ++ + static int + test_ipsec_with_reassembly(struct reassembly_vector *vector, + const struct ipsec_test_flags *flags) +@@ -761,26 +810,9 @@ test_ipsec_with_reassembly(struct reassembly_vector *vector, + burst_sz = vector->burst ? ENCAP_DECAP_BURST_SZ : 1; + nb_tx = vector->nb_frags * burst_sz; + +- rte_eth_dev_stop(port_id); +- if (ret != 0) { +- printf("rte_eth_dev_stop: err=%s, port=%u\n", +- rte_strerror(-ret), port_id); +- return ret; +- } + rte_eth_ip_reassembly_capability_get(port_id, &reass_capa); + if (reass_capa.max_frags < vector->nb_frags) + return TEST_SKIPPED; +- if (reass_capa.timeout_ms > APP_REASS_TIMEOUT) { +- reass_capa.timeout_ms = APP_REASS_TIMEOUT; +- rte_eth_ip_reassembly_conf_set(port_id, &reass_capa); +- } +- +- ret = rte_eth_dev_start(port_id); +- if (ret < 0) { +- printf("rte_eth_dev_start: err=%d, port=%d\n", +- ret, port_id); +- return ret; +- } + + memset(tx_pkts_burst, 0, sizeof(tx_pkts_burst[0]) * nb_tx); + memset(rx_pkts_burst, 0, sizeof(rx_pkts_burst[0]) * nb_tx); +@@ -871,7 +903,10 @@ test_ipsec_with_reassembly(struct reassembly_vector *vector, + if (ret) + goto out; + +- nb_sent = rte_eth_tx_burst(port_id, 0, tx_pkts_burst, nb_tx); ++ if (event_mode_enabled) ++ nb_sent = event_tx_burst(tx_pkts_burst, nb_tx); ++ else ++ nb_sent = rte_eth_tx_burst(port_id, 0, tx_pkts_burst, nb_tx); + if (nb_sent != nb_tx) { + ret = -1; + printf("\nFailed to tx %u pkts", nb_tx); +@@ -883,14 +918,17 @@ test_ipsec_with_reassembly(struct reassembly_vector *vector, + /* Retry few times before giving up */ + nb_rx = 0; + j = 0; +- do { +- nb_rx += rte_eth_rx_burst(port_id, 0, &rx_pkts_burst[nb_rx], +- nb_tx - nb_rx); +- j++; +- if (nb_rx >= nb_tx) +- break; +- rte_delay_ms(1); +- } while (j < 5 || !nb_rx); ++ if (event_mode_enabled) ++ nb_rx = event_rx_burst(rx_pkts_burst, nb_tx); ++ else ++ do { ++ nb_rx += rte_eth_rx_burst(port_id, 0, &rx_pkts_burst[nb_rx], ++ nb_tx - nb_rx); ++ j++; ++ if (nb_rx >= nb_tx) ++ break; ++ rte_delay_ms(1); ++ } while (j < 5 || !nb_rx); + + /* Check for minimum number of Rx packets expected */ + if ((vector->nb_frags == 1 && nb_rx != nb_tx) || +@@ -950,52 +988,6 @@ out: + return ret; + } + +-static int +-event_tx_burst(struct rte_mbuf **tx_pkts, uint16_t nb_pkts) +-{ +- struct rte_event ev; +- int i, nb_sent = 0; +- +- /* Convert packets to events */ +- memset(&ev, 0, sizeof(ev)); +- ev.sched_type = RTE_SCHED_TYPE_PARALLEL; +- for (i = 0; i < nb_pkts; i++) { +- ev.mbuf = tx_pkts[i]; +- nb_sent += rte_event_eth_tx_adapter_enqueue( +- eventdev_id, port_id, &ev, 1, 0); +- } +- +- return nb_sent; +-} +- +-static int +-event_rx_burst(struct rte_mbuf **rx_pkts, uint16_t nb_pkts_to_rx) +-{ +- int nb_ev, nb_rx = 0, j = 0; +- const int ms_per_pkt = 3; +- struct rte_event ev; +- +- do { +- nb_ev = rte_event_dequeue_burst(eventdev_id, port_id, +- &ev, 1, 0); +- +- if (nb_ev == 0) { +- rte_delay_ms(1); +- continue; +- } +- +- /* Get packet from event */ +- if (ev.event_type != RTE_EVENT_TYPE_ETHDEV) { +- printf("Unsupported event type: %i\n", +- ev.event_type); +- continue; +- } +- rx_pkts[nb_rx++] = ev.mbuf; +- } while (j++ < (nb_pkts_to_rx * ms_per_pkt) && nb_rx < nb_pkts_to_rx); +- +- return nb_rx; +-} +- + static int + test_ipsec_inline_sa_exp_event_callback(uint16_t port_id, + enum rte_eth_event_type type, void *param, void *ret_param) +@@ -1475,10 +1467,32 @@ out: + } + + static int +-ut_setup_inline_ipsec(void) ++ut_setup_inline_ipsec_reassembly(void) + { ++ struct rte_eth_ip_reassembly_params reass_capa = {0}; + int ret; + ++ rte_eth_ip_reassembly_capability_get(port_id, &reass_capa); ++ if (reass_capa.timeout_ms > APP_REASS_TIMEOUT) { ++ reass_capa.timeout_ms = APP_REASS_TIMEOUT; ++ rte_eth_ip_reassembly_conf_set(port_id, &reass_capa); ++ } ++ ++ /* Start event devices */ ++ if (event_mode_enabled) { ++ ret = rte_event_eth_rx_adapter_start(rx_adapter_id); ++ if (ret < 0) { ++ printf("Failed to start rx adapter %d\n", ret); ++ return ret; ++ } ++ ++ ret = rte_event_dev_start(eventdev_id); ++ if (ret < 0) { ++ printf("Failed to start event device %d\n", ret); ++ return ret; ++ } ++ } ++ + /* Start device */ + ret = rte_eth_dev_start(port_id); + if (ret < 0) { +@@ -1500,12 +1514,16 @@ ut_setup_inline_ipsec(void) + } + + static void +-ut_teardown_inline_ipsec(void) ++ut_teardown_inline_ipsec_reassembly(void) + { + struct rte_eth_ip_reassembly_params reass_conf = {0}; + uint16_t portid; + int ret; + ++ /* Stop event devices */ ++ if (event_mode_enabled) ++ rte_event_dev_stop(eventdev_id); ++ + /* port tear down */ + RTE_ETH_FOREACH_DEV(portid) { + ret = rte_eth_dev_stop(portid); +@@ -1517,6 +1535,58 @@ ut_teardown_inline_ipsec(void) + rte_eth_ip_reassembly_conf_set(portid, &reass_conf); + } + } ++static int ++ut_setup_inline_ipsec(void) ++{ ++ int ret; ++ ++ /* Start event devices */ ++ if (event_mode_enabled) { ++ ret = rte_event_dev_start(eventdev_id); ++ if (ret < 0) { ++ printf("Failed to start event device %d\n", ret); ++ return ret; ++ } ++ } ++ ++ /* Start device */ ++ ret = rte_eth_dev_start(port_id); ++ if (ret < 0) { ++ printf("rte_eth_dev_start: err=%d, port=%d\n", ++ ret, port_id); ++ return ret; ++ } ++ /* always enable promiscuous */ ++ ret = rte_eth_promiscuous_enable(port_id); ++ if (ret != 0) { ++ printf("rte_eth_promiscuous_enable: err=%s, port=%d\n", ++ rte_strerror(-ret), port_id); ++ return ret; ++ } ++ ++ check_all_ports_link_status(1, RTE_PORT_ALL); ++ ++ return 0; ++} ++ ++static void ++ut_teardown_inline_ipsec(void) ++{ ++ uint16_t portid; ++ int ret; ++ ++ /* Stop event devices */ ++ if (event_mode_enabled) ++ rte_event_dev_stop(eventdev_id); ++ ++ /* port tear down */ ++ RTE_ETH_FOREACH_DEV(portid) { ++ ret = rte_eth_dev_stop(portid); ++ if (ret != 0) ++ printf("rte_eth_dev_stop: err=%s, port=%u\n", ++ rte_strerror(-ret), portid); ++ } ++} + + static int + inline_ipsec_testsuite_setup(void) +@@ -1995,7 +2065,7 @@ test_ipsec_inline_proto_known_vec_inb(const void *test_data) + } + + static int +-test_ipsec_inline_proto_display_list(const void *data __rte_unused) ++test_ipsec_inline_proto_display_list(void) + { + struct ipsec_test_flags flags; + +@@ -2008,7 +2078,7 @@ test_ipsec_inline_proto_display_list(const void *data __rte_unused) + } + + static int +-test_ipsec_inline_proto_udp_encap(const void *data __rte_unused) ++test_ipsec_inline_proto_udp_encap(void) + { + struct ipsec_test_flags flags; + +@@ -2021,7 +2091,7 @@ test_ipsec_inline_proto_udp_encap(const void *data __rte_unused) + } + + static int +-test_ipsec_inline_proto_udp_ports_verify(const void *data __rte_unused) ++test_ipsec_inline_proto_udp_ports_verify(void) + { + struct ipsec_test_flags flags; + +@@ -2035,7 +2105,7 @@ test_ipsec_inline_proto_udp_ports_verify(const void *data __rte_unused) + } + + static int +-test_ipsec_inline_proto_err_icv_corrupt(const void *data __rte_unused) ++test_ipsec_inline_proto_err_icv_corrupt(void) + { + struct ipsec_test_flags flags; + +@@ -2048,7 +2118,7 @@ test_ipsec_inline_proto_err_icv_corrupt(const void *data __rte_unused) + } + + static int +-test_ipsec_inline_proto_tunnel_dst_addr_verify(const void *data __rte_unused) ++test_ipsec_inline_proto_tunnel_dst_addr_verify(void) + { + struct ipsec_test_flags flags; + +@@ -2061,7 +2131,7 @@ test_ipsec_inline_proto_tunnel_dst_addr_verify(const void *data __rte_unused) + } + + static int +-test_ipsec_inline_proto_tunnel_src_dst_addr_verify(const void *data __rte_unused) ++test_ipsec_inline_proto_tunnel_src_dst_addr_verify(void) + { + struct ipsec_test_flags flags; + +@@ -2074,7 +2144,7 @@ test_ipsec_inline_proto_tunnel_src_dst_addr_verify(const void *data __rte_unused + } + + static int +-test_ipsec_inline_proto_inner_ip_csum(const void *data __rte_unused) ++test_ipsec_inline_proto_inner_ip_csum(void) + { + struct ipsec_test_flags flags; + +@@ -2087,7 +2157,7 @@ test_ipsec_inline_proto_inner_ip_csum(const void *data __rte_unused) + } + + static int +-test_ipsec_inline_proto_inner_l4_csum(const void *data __rte_unused) ++test_ipsec_inline_proto_inner_l4_csum(void) + { + struct ipsec_test_flags flags; + +@@ -2100,7 +2170,7 @@ test_ipsec_inline_proto_inner_l4_csum(const void *data __rte_unused) + } + + static int +-test_ipsec_inline_proto_tunnel_v4_in_v4(const void *data __rte_unused) ++test_ipsec_inline_proto_tunnel_v4_in_v4(void) + { + struct ipsec_test_flags flags; + +@@ -2114,7 +2184,7 @@ test_ipsec_inline_proto_tunnel_v4_in_v4(const void *data __rte_unused) + } + + static int +-test_ipsec_inline_proto_tunnel_v6_in_v6(const void *data __rte_unused) ++test_ipsec_inline_proto_tunnel_v6_in_v6(void) + { + struct ipsec_test_flags flags; + +@@ -2128,7 +2198,7 @@ test_ipsec_inline_proto_tunnel_v6_in_v6(const void *data __rte_unused) + } + + static int +-test_ipsec_inline_proto_tunnel_v4_in_v6(const void *data __rte_unused) ++test_ipsec_inline_proto_tunnel_v4_in_v6(void) + { + struct ipsec_test_flags flags; + +@@ -2142,7 +2212,7 @@ test_ipsec_inline_proto_tunnel_v4_in_v6(const void *data __rte_unused) + } + + static int +-test_ipsec_inline_proto_tunnel_v6_in_v4(const void *data __rte_unused) ++test_ipsec_inline_proto_tunnel_v6_in_v4(void) + { + struct ipsec_test_flags flags; + +@@ -2156,7 +2226,7 @@ test_ipsec_inline_proto_tunnel_v6_in_v4(const void *data __rte_unused) + } + + static int +-test_ipsec_inline_proto_transport_v4(const void *data __rte_unused) ++test_ipsec_inline_proto_transport_v4(void) + { + struct ipsec_test_flags flags; + +@@ -2170,7 +2240,7 @@ test_ipsec_inline_proto_transport_v4(const void *data __rte_unused) + } + + static int +-test_ipsec_inline_proto_transport_l4_csum(const void *data __rte_unused) ++test_ipsec_inline_proto_transport_l4_csum(void) + { + struct ipsec_test_flags flags = { + .l4_csum = true, +@@ -2182,7 +2252,7 @@ test_ipsec_inline_proto_transport_l4_csum(const void *data __rte_unused) + } + + static int +-test_ipsec_inline_proto_stats(const void *data __rte_unused) ++test_ipsec_inline_proto_stats(void) + { + struct ipsec_test_flags flags; + +@@ -2195,7 +2265,7 @@ test_ipsec_inline_proto_stats(const void *data __rte_unused) + } + + static int +-test_ipsec_inline_proto_pkt_fragment(const void *data __rte_unused) ++test_ipsec_inline_proto_pkt_fragment(void) + { + struct ipsec_test_flags flags; + +@@ -2209,7 +2279,7 @@ test_ipsec_inline_proto_pkt_fragment(const void *data __rte_unused) + } + + static int +-test_ipsec_inline_proto_copy_df_inner_0(const void *data __rte_unused) ++test_ipsec_inline_proto_copy_df_inner_0(void) + { + struct ipsec_test_flags flags; + +@@ -2222,7 +2292,7 @@ test_ipsec_inline_proto_copy_df_inner_0(const void *data __rte_unused) + } + + static int +-test_ipsec_inline_proto_copy_df_inner_1(const void *data __rte_unused) ++test_ipsec_inline_proto_copy_df_inner_1(void) + { + struct ipsec_test_flags flags; + +@@ -2235,7 +2305,7 @@ test_ipsec_inline_proto_copy_df_inner_1(const void *data __rte_unused) + } + + static int +-test_ipsec_inline_proto_set_df_0_inner_1(const void *data __rte_unused) ++test_ipsec_inline_proto_set_df_0_inner_1(void) + { + struct ipsec_test_flags flags; + +@@ -2248,7 +2318,7 @@ test_ipsec_inline_proto_set_df_0_inner_1(const void *data __rte_unused) + } + + static int +-test_ipsec_inline_proto_set_df_1_inner_0(const void *data __rte_unused) ++test_ipsec_inline_proto_set_df_1_inner_0(void) + { + struct ipsec_test_flags flags; + +@@ -2261,7 +2331,7 @@ test_ipsec_inline_proto_set_df_1_inner_0(const void *data __rte_unused) + } + + static int +-test_ipsec_inline_proto_ipv4_copy_dscp_inner_0(const void *data __rte_unused) ++test_ipsec_inline_proto_ipv4_copy_dscp_inner_0(void) + { + struct ipsec_test_flags flags; + +@@ -2274,7 +2344,7 @@ test_ipsec_inline_proto_ipv4_copy_dscp_inner_0(const void *data __rte_unused) + } + + static int +-test_ipsec_inline_proto_ipv4_copy_dscp_inner_1(const void *data __rte_unused) ++test_ipsec_inline_proto_ipv4_copy_dscp_inner_1(void) + { + struct ipsec_test_flags flags; + +@@ -2287,7 +2357,7 @@ test_ipsec_inline_proto_ipv4_copy_dscp_inner_1(const void *data __rte_unused) + } + + static int +-test_ipsec_inline_proto_ipv4_set_dscp_0_inner_1(const void *data __rte_unused) ++test_ipsec_inline_proto_ipv4_set_dscp_0_inner_1(void) + { + struct ipsec_test_flags flags; + +@@ -2300,7 +2370,7 @@ test_ipsec_inline_proto_ipv4_set_dscp_0_inner_1(const void *data __rte_unused) + } + + static int +-test_ipsec_inline_proto_ipv4_set_dscp_1_inner_0(const void *data __rte_unused) ++test_ipsec_inline_proto_ipv4_set_dscp_1_inner_0(void) + { + struct ipsec_test_flags flags; + +@@ -2313,7 +2383,7 @@ test_ipsec_inline_proto_ipv4_set_dscp_1_inner_0(const void *data __rte_unused) + } + + static int +-test_ipsec_inline_proto_ipv6_copy_dscp_inner_0(const void *data __rte_unused) ++test_ipsec_inline_proto_ipv6_copy_dscp_inner_0(void) + { + struct ipsec_test_flags flags; + +@@ -2328,7 +2398,7 @@ test_ipsec_inline_proto_ipv6_copy_dscp_inner_0(const void *data __rte_unused) + } + + static int +-test_ipsec_inline_proto_ipv6_copy_dscp_inner_1(const void *data __rte_unused) ++test_ipsec_inline_proto_ipv6_copy_dscp_inner_1(void) + { + struct ipsec_test_flags flags; + +@@ -2343,7 +2413,7 @@ test_ipsec_inline_proto_ipv6_copy_dscp_inner_1(const void *data __rte_unused) + } + + static int +-test_ipsec_inline_proto_ipv6_set_dscp_0_inner_1(const void *data __rte_unused) ++test_ipsec_inline_proto_ipv6_set_dscp_0_inner_1(void) + { + struct ipsec_test_flags flags; + +@@ -2358,7 +2428,7 @@ test_ipsec_inline_proto_ipv6_set_dscp_0_inner_1(const void *data __rte_unused) + } + + static int +-test_ipsec_inline_proto_ipv6_set_dscp_1_inner_0(const void *data __rte_unused) ++test_ipsec_inline_proto_ipv6_set_dscp_1_inner_0(void) + { + struct ipsec_test_flags flags; + +@@ -2373,7 +2443,7 @@ test_ipsec_inline_proto_ipv6_set_dscp_1_inner_0(const void *data __rte_unused) + } + + static int +-test_ipsec_inline_proto_ipv6_copy_flabel_inner_0(const void *data __rte_unused) ++test_ipsec_inline_proto_ipv6_copy_flabel_inner_0(void) + { + struct ipsec_test_flags flags; + +@@ -2387,7 +2457,7 @@ test_ipsec_inline_proto_ipv6_copy_flabel_inner_0(const void *data __rte_unused) + } + + static int +-test_ipsec_inline_proto_ipv6_copy_flabel_inner_1(const void *data __rte_unused) ++test_ipsec_inline_proto_ipv6_copy_flabel_inner_1(void) + { + struct ipsec_test_flags flags; + +@@ -2401,7 +2471,7 @@ test_ipsec_inline_proto_ipv6_copy_flabel_inner_1(const void *data __rte_unused) + } + + static int +-test_ipsec_inline_proto_ipv6_set_flabel_0_inner_1(const void *data __rte_unused) ++test_ipsec_inline_proto_ipv6_set_flabel_0_inner_1(void) + { + struct ipsec_test_flags flags; + +@@ -2415,7 +2485,7 @@ test_ipsec_inline_proto_ipv6_set_flabel_0_inner_1(const void *data __rte_unused) + } + + static int +-test_ipsec_inline_proto_ipv6_set_flabel_1_inner_0(const void *data __rte_unused) ++test_ipsec_inline_proto_ipv6_set_flabel_1_inner_0(void) + { + struct ipsec_test_flags flags; + +@@ -2429,7 +2499,7 @@ test_ipsec_inline_proto_ipv6_set_flabel_1_inner_0(const void *data __rte_unused) + } + + static int +-test_ipsec_inline_proto_ipv4_ttl_decrement(const void *data __rte_unused) ++test_ipsec_inline_proto_ipv4_ttl_decrement(void) + { + struct ipsec_test_flags flags = { + .dec_ttl_or_hop_limit = true, +@@ -2440,7 +2510,7 @@ test_ipsec_inline_proto_ipv4_ttl_decrement(const void *data __rte_unused) + } + + static int +-test_ipsec_inline_proto_ipv6_hop_limit_decrement(const void *data __rte_unused) ++test_ipsec_inline_proto_ipv6_hop_limit_decrement(void) + { + struct ipsec_test_flags flags = { + .ipv6 = true, +@@ -2452,7 +2522,7 @@ test_ipsec_inline_proto_ipv6_hop_limit_decrement(const void *data __rte_unused) + } + + static int +-test_ipsec_inline_proto_iv_gen(const void *data __rte_unused) ++test_ipsec_inline_proto_iv_gen(void) + { + struct ipsec_test_flags flags; + +@@ -2465,7 +2535,7 @@ test_ipsec_inline_proto_iv_gen(const void *data __rte_unused) + } + + static int +-test_ipsec_inline_proto_sa_pkt_soft_expiry(const void *data __rte_unused) ++test_ipsec_inline_proto_sa_pkt_soft_expiry(void) + { + struct ipsec_test_flags flags = { + .sa_expiry_pkts_soft = true, +@@ -2474,7 +2544,7 @@ test_ipsec_inline_proto_sa_pkt_soft_expiry(const void *data __rte_unused) + return test_ipsec_inline_proto_all(&flags); + } + static int +-test_ipsec_inline_proto_sa_byte_soft_expiry(const void *data __rte_unused) ++test_ipsec_inline_proto_sa_byte_soft_expiry(void) + { + struct ipsec_test_flags flags = { + .sa_expiry_bytes_soft = true, +@@ -2484,7 +2554,7 @@ test_ipsec_inline_proto_sa_byte_soft_expiry(const void *data __rte_unused) + } + + static int +-test_ipsec_inline_proto_sa_pkt_hard_expiry(const void *data __rte_unused) ++test_ipsec_inline_proto_sa_pkt_hard_expiry(void) + { + struct ipsec_test_flags flags = { + .sa_expiry_pkts_hard = true +@@ -2494,7 +2564,7 @@ test_ipsec_inline_proto_sa_pkt_hard_expiry(const void *data __rte_unused) + } + + static int +-test_ipsec_inline_proto_sa_byte_hard_expiry(const void *data __rte_unused) ++test_ipsec_inline_proto_sa_byte_hard_expiry(void) + { + struct ipsec_test_flags flags = { + .sa_expiry_bytes_hard = true +@@ -3048,43 +3118,43 @@ static struct unit_test_suite inline_ipsec_testsuite = { + + TEST_CASE_NAMED_WITH_DATA( + "IPv4 Reassembly with 2 fragments", +- ut_setup_inline_ipsec, ut_teardown_inline_ipsec, ++ ut_setup_inline_ipsec_reassembly, ut_teardown_inline_ipsec_reassembly, + test_inline_ip_reassembly, &ipv4_2frag_vector), + TEST_CASE_NAMED_WITH_DATA( + "IPv6 Reassembly with 2 fragments", +- ut_setup_inline_ipsec, ut_teardown_inline_ipsec, ++ ut_setup_inline_ipsec_reassembly, ut_teardown_inline_ipsec_reassembly, + test_inline_ip_reassembly, &ipv6_2frag_vector), + TEST_CASE_NAMED_WITH_DATA( + "IPv4 Reassembly with 4 fragments", +- ut_setup_inline_ipsec, ut_teardown_inline_ipsec, ++ ut_setup_inline_ipsec_reassembly, ut_teardown_inline_ipsec_reassembly, + test_inline_ip_reassembly, &ipv4_4frag_vector), + TEST_CASE_NAMED_WITH_DATA( + "IPv6 Reassembly with 4 fragments", +- ut_setup_inline_ipsec, ut_teardown_inline_ipsec, ++ ut_setup_inline_ipsec_reassembly, ut_teardown_inline_ipsec_reassembly, + test_inline_ip_reassembly, &ipv6_4frag_vector), + TEST_CASE_NAMED_WITH_DATA( + "IPv4 Reassembly with 5 fragments", +- ut_setup_inline_ipsec, ut_teardown_inline_ipsec, ++ ut_setup_inline_ipsec_reassembly, ut_teardown_inline_ipsec_reassembly, + test_inline_ip_reassembly, &ipv4_5frag_vector), + TEST_CASE_NAMED_WITH_DATA( + "IPv6 Reassembly with 5 fragments", +- ut_setup_inline_ipsec, ut_teardown_inline_ipsec, ++ ut_setup_inline_ipsec_reassembly, ut_teardown_inline_ipsec_reassembly, + test_inline_ip_reassembly, &ipv6_5frag_vector), + TEST_CASE_NAMED_WITH_DATA( + "IPv4 Reassembly with incomplete fragments", +- ut_setup_inline_ipsec, ut_teardown_inline_ipsec, ++ ut_setup_inline_ipsec_reassembly, ut_teardown_inline_ipsec_reassembly, + test_inline_ip_reassembly, &ipv4_incomplete_vector), + TEST_CASE_NAMED_WITH_DATA( + "IPv4 Reassembly with overlapping fragments", +- ut_setup_inline_ipsec, ut_teardown_inline_ipsec, ++ ut_setup_inline_ipsec_reassembly, ut_teardown_inline_ipsec_reassembly, + test_inline_ip_reassembly, &ipv4_overlap_vector), + TEST_CASE_NAMED_WITH_DATA( + "IPv4 Reassembly with out of order fragments", +- ut_setup_inline_ipsec, ut_teardown_inline_ipsec, ++ ut_setup_inline_ipsec_reassembly, ut_teardown_inline_ipsec_reassembly, + test_inline_ip_reassembly, &ipv4_out_of_order_vector), + TEST_CASE_NAMED_WITH_DATA( + "IPv4 Reassembly with burst of 4 fragments", +- ut_setup_inline_ipsec, ut_teardown_inline_ipsec, ++ ut_setup_inline_ipsec_reassembly, ut_teardown_inline_ipsec_reassembly, + test_inline_ip_reassembly, &ipv4_4frag_burst_vector), + + TEST_CASES_END() /**< NULL terminate unit test array */ +diff --git a/dpdk/app/test/test_security_inline_proto_vectors.h b/dpdk/app/test/test_security_inline_proto_vectors.h +index 003537e200..d0a4b948e4 100644 +--- a/dpdk/app/test/test_security_inline_proto_vectors.h ++++ b/dpdk/app/test/test_security_inline_proto_vectors.h +@@ -88,7 +88,7 @@ struct ip_reassembly_test_packet pkt_ipv6_udp_p1 = { + .l4_offset = 40, + .data = { + /* IP */ +- 0x60, 0x00, 0x00, 0x00, 0x05, 0xb4, 0x2C, 0x40, ++ 0x60, 0x00, 0x00, 0x00, 0x05, 0xb4, 0x11, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0x0d, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +@@ -135,7 +135,7 @@ struct ip_reassembly_test_packet pkt_ipv6_udp_p2 = { + .l4_offset = 40, + .data = { + /* IP */ +- 0x60, 0x00, 0x00, 0x00, 0x11, 0x5a, 0x2c, 0x40, ++ 0x60, 0x00, 0x00, 0x00, 0x11, 0x5a, 0x11, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0x0d, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +diff --git a/dpdk/config/arm/meson.build b/dpdk/config/arm/meson.build +index 6442ec9596..5028c74613 100644 +--- a/dpdk/config/arm/meson.build ++++ b/dpdk/config/arm/meson.build +@@ -43,7 +43,9 @@ implementer_generic = { + }, + 'generic_aarch32': { + 'march': 'armv8-a', +- 'compiler_options': ['-mfpu=neon'], ++ 'force_march': true, ++ 'march_features': ['simd'], ++ 'compiler_options': ['-mfpu=auto'], + 'flags': [ + ['RTE_ARCH_ARM_NEON_MEMCPY', false], + ['RTE_ARCH_STRICT_ALIGN', true], +@@ -613,21 +615,25 @@ if update_flags + # probe supported archs and their features + candidate_march = '' + if part_number_config.has_key('march') +- supported_marchs = ['armv8.6-a', 'armv8.5-a', 'armv8.4-a', 'armv8.3-a', +- 'armv8.2-a', 'armv8.1-a', 'armv8-a'] +- check_compiler_support = false +- foreach supported_march: supported_marchs +- if supported_march == part_number_config['march'] +- # start checking from this version downwards +- check_compiler_support = true +- endif +- if (check_compiler_support and +- cc.has_argument('-march=' + supported_march)) +- candidate_march = supported_march +- # highest supported march version found +- break +- endif +- endforeach ++ if part_number_config.get('force_march', false) ++ candidate_march = part_number_config['march'] ++ else ++ supported_marchs = ['armv8.6-a', 'armv8.5-a', 'armv8.4-a', 'armv8.3-a', ++ 'armv8.2-a', 'armv8.1-a', 'armv8-a'] ++ check_compiler_support = false ++ foreach supported_march: supported_marchs ++ if supported_march == part_number_config['march'] ++ # start checking from this version downwards ++ check_compiler_support = true ++ endif ++ if (check_compiler_support and ++ cc.has_argument('-march=' + supported_march)) ++ candidate_march = supported_march ++ # highest supported march version found ++ break ++ endif ++ endforeach ++ endif + if candidate_march == '' + error('No suitable armv8 march version found.') + endif +@@ -659,7 +665,7 @@ if update_flags + # apply supported compiler options + if part_number_config.has_key('compiler_options') + foreach flag: part_number_config['compiler_options'] +- if cc.has_argument(flag) ++ if cc.has_multi_arguments(machine_args + [flag]) + machine_args += flag + else + warning('Configuration compiler option ' + +diff --git a/dpdk/config/meson.build b/dpdk/config/meson.build +index 6d9ffd4f4b..265aaa995c 100644 +--- a/dpdk/config/meson.build ++++ b/dpdk/config/meson.build +@@ -122,7 +122,7 @@ if cpu_instruction_set == 'generic' + elif host_machine.cpu_family().startswith('ppc') + cpu_instruction_set = 'power8' + elif host_machine.cpu_family().startswith('riscv') +- cpu_instruction_set = 'riscv' ++ cpu_instruction_set = 'rv64gc' + endif + endif + +@@ -139,7 +139,7 @@ endif + + toolchain = cc.get_id() + dpdk_conf.set_quoted('RTE_TOOLCHAIN', toolchain) +-dpdk_conf.set('RTE_TOOLCHAIN_' + toolchain.to_upper(), 1) ++dpdk_conf.set('RTE_TOOLCHAIN_' + toolchain.to_upper().underscorify(), 1) + + dpdk_conf.set('RTE_ARCH_64', cc.sizeof('void *') == 8) + dpdk_conf.set('RTE_ARCH_32', cc.sizeof('void *') == 4) +@@ -191,7 +191,7 @@ if find_libnuma + endif + + has_libfdt = 0 +-fdt_dep = cc.find_library('libfdt', required: false) ++fdt_dep = cc.find_library('fdt', required: false) + if fdt_dep.found() and cc.has_header('fdt.h') + dpdk_conf.set10('RTE_HAS_LIBFDT', true) + has_libfdt = 1 +@@ -199,11 +199,12 @@ if fdt_dep.found() and cc.has_header('fdt.h') + dpdk_extra_ldflags += '-lfdt' + endif + +-libexecinfo = cc.find_library('libexecinfo', required: false) +-if libexecinfo.found() and cc.has_header('execinfo.h') ++libexecinfo = cc.find_library('execinfo', required: false) ++if libexecinfo.found() + add_project_link_arguments('-lexecinfo', language: 'c') + dpdk_extra_ldflags += '-lexecinfo' + endif ++dpdk_conf.set('RTE_BACKTRACE', cc.has_header('execinfo.h') or is_windows) + + libarchive = dependency('libarchive', required: false, method: 'pkg-config') + if libarchive.found() +@@ -365,7 +366,7 @@ if max_numa_nodes == 'detect' + error('Discovery of max_numa_nodes not supported for cross-compilation.') + endif + # overwrite the default value with discovered values +- max_numa_nodes = run_command(get_numa_count_cmd).stdout().to_int() ++ max_numa_nodes = run_command(get_numa_count_cmd, check: true).stdout().to_int() + message('Found @0@ numa nodes'.format(max_numa_nodes)) + dpdk_conf.set('RTE_MAX_NUMA_NODES', max_numa_nodes) + elif max_numa_nodes != 'default' +diff --git a/dpdk/config/rte_config.h b/dpdk/config/rte_config.h +index 3c4876d434..7b8c85e948 100644 +--- a/dpdk/config/rte_config.h ++++ b/dpdk/config/rte_config.h +@@ -37,7 +37,6 @@ + #define RTE_MAX_MEMZONE 2560 + #define RTE_MAX_TAILQ 32 + #define RTE_LOG_DP_LEVEL RTE_LOG_INFO +-#define RTE_BACKTRACE 1 + #define RTE_MAX_VFIO_CONTAINERS 64 + + /* bsd module defines */ +diff --git a/dpdk/devtools/check-git-log.sh b/dpdk/devtools/check-git-log.sh +index 01d8aa0717..2ee7f2db64 100755 +--- a/dpdk/devtools/check-git-log.sh ++++ b/dpdk/devtools/check-git-log.sh +@@ -120,7 +120,7 @@ words="$selfdir/words-case.txt" + for word in $(cat $words); do + bad=$(echo "$headlines" | grep -iw $word | grep -vw $word) + if [ "$word" = "Tx" ]; then +- bad=$(echo $bad | grep -v 'OCTEON\ TX') ++ bad=$(echo $bad | grep -v 'OCTEON TX') + fi + for bad_line in $bad; do + bad_word=$(echo $bad_line | cut -d":" -f2 | grep -iwo $word) +@@ -259,6 +259,23 @@ done) + [ -z "$bad" ] || { printf "Missing 'Signed-off-by:' tag: \n$bad\n"\ + && failure=true;} + ++# check names ++names=$(git log --format='From: %an <%ae>%n%b' --reverse $range | ++ sed -rn 's,.*: (.*<.*@.*>),\1,p' | ++ sort -u) ++bad=$(for contributor in $names ; do ++ contributor=$(echo $contributor | sed 's,(,\\(,') ++ ! grep -qE "^$contributor($| <)" $selfdir/../.mailmap || continue ++ name=${contributor%% <*} ++ if grep -q "^$name <" $selfdir/../.mailmap ; then ++ printf "\t$contributor is not the primary email address\n" ++ else ++ printf "\t$contributor is unknown in .mailmap\n" ++ fi ++done) ++[ -z "$bad" ] || { printf "Contributor name/email mismatch with .mailmap: \n$bad\n"\ ++ && failure=true;} ++ + total=$(echo "$commits" | wc -l) + if $failure ; then + printf "\nInvalid patch(es) found - checked $total patch" +diff --git a/dpdk/devtools/checkpatches.sh b/dpdk/devtools/checkpatches.sh +index be1cb03ea7..a07bbc83cb 100755 +--- a/dpdk/devtools/checkpatches.sh ++++ b/dpdk/devtools/checkpatches.sh +@@ -248,28 +248,6 @@ check_release_notes() { # + grep -v $current_rel_notes + } + +-check_names() { # +- res=0 +- +- old_IFS=$IFS +- IFS=' +-' +- for contributor in $(sed -rn '1,/^--- / {s/.*: (.*<.*@.*>)/\1/p}' $1); do +- ! grep -qE "^$contributor($| <)" .mailmap || continue +- name=${contributor%% <*} +- if grep -q "^$name <" .mailmap; then +- reason="$name mail differs from primary mail" +- else +- reason="$contributor is unknown" +- fi +- echo "$reason, please fix the commit message or update .mailmap." +- res=1 +- done +- IFS=$old_IFS +- +- return $res +-} +- + number=0 + range='origin/main..' + quiet=false +@@ -378,14 +356,6 @@ check () { # + ret=1 + fi + +- ! $verbose || printf '\nChecking names in commit log:\n' +- report=$(check_names "$tmpinput") +- if [ $? -ne 0 ] ; then +- $headline_printed || print_headline "$subject" +- printf '%s\n' "$report" +- ret=1 +- fi +- + if [ "$tmpinput" != "$1" ]; then + rm -f "$tmpinput" + trap - INT +diff --git a/dpdk/doc/api/doxy-api-index.md b/dpdk/doc/api/doxy-api-index.md +index de488c7abf..bbca14be3d 100644 +--- a/dpdk/doc/api/doxy-api-index.md ++++ b/dpdk/doc/api/doxy-api-index.md +@@ -22,6 +22,7 @@ The public API headers are grouped by topics: + [compress](@ref rte_comp.h), + [regexdev](@ref rte_regexdev.h), + [dmadev](@ref rte_dmadev.h), ++ [gpudev](@ref rte_gpudev.h), + [eventdev](@ref rte_eventdev.h), + [event_eth_rx_adapter](@ref rte_event_eth_rx_adapter.h), + [event_eth_tx_adapter](@ref rte_event_eth_tx_adapter.h), +diff --git a/dpdk/doc/api/doxy-api.conf.in b/dpdk/doc/api/doxy-api.conf.in +index f0886c3bd1..dd8ebab447 100644 +--- a/dpdk/doc/api/doxy-api.conf.in ++++ b/dpdk/doc/api/doxy-api.conf.in +@@ -99,7 +99,6 @@ GENERATE_DEPRECATEDLIST = YES + VERBATIM_HEADERS = NO + ALPHABETICAL_INDEX = NO + +-HTML_TIMESTAMP = NO + HTML_DYNAMIC_SECTIONS = YES + HTML_EXTRA_STYLESHEET = @TOPDIR@/doc/api/custom.css + SEARCHENGINE = YES +diff --git a/dpdk/doc/guides/conf.py b/dpdk/doc/guides/conf.py +index a55ce38800..0f7ff5282d 100644 +--- a/dpdk/doc/guides/conf.py ++++ b/dpdk/doc/guides/conf.py +@@ -203,6 +203,7 @@ def generate_overview_table(output_filename, table_id, section, table_name, titl + num_cols = len(header_names) + + print_table_css(outfile, table_id) ++ print('.. _' + table_name + ':', file=outfile) + print('.. table:: ' + table_name + '\n', file=outfile) + print_table_header(outfile, num_cols, header_names, title) + print_table_body(outfile, num_cols, ini_files, ini_data, default_features) +diff --git a/dpdk/doc/guides/cryptodevs/qat.rst b/dpdk/doc/guides/cryptodevs/qat.rst +index d1e64475c4..b1b893a251 100644 +--- a/dpdk/doc/guides/cryptodevs/qat.rst ++++ b/dpdk/doc/guides/cryptodevs/qat.rst +@@ -399,9 +399,9 @@ to see the full table) + +-----+-----+-----+-----+----------+---------------+---------------+------------+--------+------+--------+--------+ + | Yes | No | No | 4 | 4xxx | IDZ/ N/A | qat_4xxx | 4xxx | 4940 | 4 | 4941 | 16 | + +-----+-----+-----+-----+----------+---------------+---------------+------------+--------+------+--------+--------+ +- | Yes | Yes | Yes | 4 | 401xxx | linux/5.19+ | qat_401xxx | 4xxx | 4942 | 2 | 4943 | 16 | ++ | Yes | Yes | Yes | 4 | 401xxx | linux/5.19+ | qat_4xxx | 4xxx | 4942 | 2 | 4943 | 16 | + +-----+-----+-----+-----+----------+---------------+---------------+------------+--------+------+--------+--------+ +- | Yes | No | No | 4 | 401xxx | IDZ/ N/A | qat_401xxx | 4xxx | 4942 | 2 | 4943 | 16 | ++ | Yes | No | No | 4 | 401xxx | IDZ/ N/A | qat_4xxx | 4xxx | 4942 | 2 | 4943 | 16 | + +-----+-----+-----+-----+----------+---------------+---------------+------------+--------+------+--------+--------+ + + * Note: Symmetric mixed crypto algorithms feature on Gen 2 works only with IDZ driver version 4.9.0+ +diff --git a/dpdk/doc/guides/gpus/cuda.rst b/dpdk/doc/guides/gpus/cuda.rst +index 114e3bc8cb..6520c17c3e 100644 +--- a/dpdk/doc/guides/gpus/cuda.rst ++++ b/dpdk/doc/guides/gpus/cuda.rst +@@ -12,20 +12,19 @@ Information and documentation about these devices can be found on the + Build dependencies + ------------------ + +-The CUDA GPU driver library has an header-only dependency on ``cuda.h`` and ``cudaTypedefs.h``. +-To get these headers there are two options: ++The CUDA GPU driver library has a header-only dependency on ``cuda.h`` and ``cudaTypedefs.h``. ++To get these headers, there are two options: + + - Install `CUDA Toolkit `_ + (either regular or stubs installation). + - Download these two headers from this `CUDA headers + `_ repository. + +-You need to indicate to meson where CUDA headers files are through the CFLAGS variable. +-Three ways: ++You can point to CUDA header files either with the ``CFLAGS`` environment variable, ++or with the ``c_args`` Meson option. Examples: + +-- Set ``export CFLAGS=-I/usr/local/cuda/include`` before building +-- Add CFLAGS in the meson command line ``CFLAGS=-I/usr/local/cuda/include meson setup build`` +-- Add the ``-Dc_args`` in meson command line ``meson setup build -Dc_args=-I/usr/local/cuda/include`` ++- ``CFLAGS=-I/usr/local/cuda/include meson setup build`` ++- ``meson setup build -Dc_args=-I/usr/local/cuda/include`` + + If headers are not found, the CUDA GPU driver library is not built. + +@@ -46,15 +45,15 @@ A quick recipe to download, build and run GDRCopy library and driver: + $ # Launch gdrdrv kernel module on the system + $ sudo ./insmod.sh + +-You need to indicate to meson where GDRCopy headers files are as in case of CUDA headers. ++You need to indicate to Meson where GDRCopy header files are as in case of CUDA headers. + An example would be: + + .. code-block:: console + + $ meson setup build -Dc_args="-I/usr/local/cuda/include -I/path/to/gdrcopy/include" + +-If headers are not found, the CUDA GPU driver library is built without the CPU map capability +-and will return error if the application invokes the gpudev ``rte_gpu_mem_cpu_map`` function. ++If headers are not found, the CUDA GPU driver library is built without the CPU map capability, ++and will return an error if the application invokes the gpudev ``rte_gpu_mem_cpu_map`` function. + + + CUDA Shared Library +@@ -143,7 +142,7 @@ if the address is not in the table the CUDA driver library will return an error. + Features + -------- + +-- Register new child devices aka new CUDA Driver contexts. ++- Register new child devices, aka CUDA driver contexts. + - Allocate memory on the GPU. + - Register CPU memory to make it visible from GPU. + +@@ -189,9 +188,10 @@ External references + A good example of how to use the GPU CUDA driver library through the gpudev library + is the l2fwd-nv application that can be found `here `_. + +-The application is based on vanilla DPDK example l2fwd +-and is enhanced with GPU memory managed through gpudev library +-and CUDA to launch the swap of packets MAC addresses workload on the GPU. ++The application is based on the DPDK example l2fwd, ++with GPU memory managed through gpudev library. ++It includes a CUDA workload swapping MAC addresses ++of packets received in the GPU. + + l2fwd-nv is not intended to be used for performance + (testpmd is the good candidate for this). +diff --git a/dpdk/doc/guides/linux_gsg/enable_func.rst b/dpdk/doc/guides/linux_gsg/enable_func.rst +index 829084d80e..2344d97403 100644 +--- a/dpdk/doc/guides/linux_gsg/enable_func.rst ++++ b/dpdk/doc/guides/linux_gsg/enable_func.rst +@@ -55,12 +55,12 @@ Refer to the `documentation ++ setcap cap_dac_read_search,cap_ipc_lock,cap_sys_admin+ep + + If physical addresses are not accessible, + the following message will appear during EAL initialization:: +diff --git a/dpdk/doc/guides/linux_gsg/sys_reqs.rst b/dpdk/doc/guides/linux_gsg/sys_reqs.rst +index a7e8261e22..dfeaf4e1c5 100644 +--- a/dpdk/doc/guides/linux_gsg/sys_reqs.rst ++++ b/dpdk/doc/guides/linux_gsg/sys_reqs.rst +@@ -32,7 +32,7 @@ Compilation of the DPDK + + * For RHEL/Fedora systems these can be installed using ``dnf groupinstall "Development Tools"`` + * For Ubuntu/Debian systems these can be installed using ``apt install build-essential`` +- * For Alpine Linux, ``apk add alpine-sdk bsd-compat-headers libexecinfo-dev`` ++ * For Alpine Linux, ``apk add alpine-sdk bsd-compat-headers`` + + .. note:: + +diff --git a/dpdk/doc/guides/nics/bnxt.rst b/dpdk/doc/guides/nics/bnxt.rst +index 293eab8787..871d14142c 100644 +--- a/dpdk/doc/guides/nics/bnxt.rst ++++ b/dpdk/doc/guides/nics/bnxt.rst +@@ -912,6 +912,7 @@ Shown below are Ethernet Network Adapters and their supported firmware versions + * ``BCM57500 NetXtreme-E\ |reg| Family`` ... Firmware 219.0.0 or later + + Shown below are DPDK LTS releases and their supported firmware versions: ++ + * ``DPDK Release 19.11`` ... Firmware 219.0.103 or later + * ``DPDK Release 20.11`` ... Firmware 219.0.103 or later + * ``DPDK Release 21.11`` ... Firmware 221.0.101 or later +@@ -1018,8 +1019,7 @@ Listed below are the rte_flow functions supported: + rte_flow Items + ~~~~~~~~~~~~~~ + +-Refer to "Table 1.2 rte_flow items availability in networking drivers" in +-`Overview of Networking Drivers `. ++Refer to :ref:`rte_flow items availability in networking drivers`. + + Listed below are the rte_flow items supported: + +@@ -1044,8 +1044,7 @@ Listed below are the rte_flow items supported: + rte_flow Actions + ~~~~~~~~~~~~~~~~ + +-Refer to "Table 1.3 rte_flow actions availability in networking drivers" in +-`Overview of Networking Drivers `. ++Refer to :ref:`rte_flow actions availability in networking drivers`. + + Listed below are the rte_flow actions supported: + +diff --git a/dpdk/doc/guides/nics/features/iavf.ini b/dpdk/doc/guides/nics/features/iavf.ini +index 9db2865b71..5cdf0ddee6 100644 +--- a/dpdk/doc/guides/nics/features/iavf.ini ++++ b/dpdk/doc/guides/nics/features/iavf.ini +@@ -21,7 +21,7 @@ RSS key update = Y + RSS reta update = Y + VLAN filter = Y + CRC offload = Y +-VLAN offload = Y ++VLAN offload = P + L3 checksum offload = P + L4 checksum offload = P + Timestamp offload = P +diff --git a/dpdk/doc/guides/nics/hns3.rst b/dpdk/doc/guides/nics/hns3.rst +index 791c9cc2ed..380024600b 100644 +--- a/dpdk/doc/guides/nics/hns3.rst ++++ b/dpdk/doc/guides/nics/hns3.rst +@@ -30,7 +30,6 @@ Features of the HNS3 PMD are: + - DCB + - Scattered and gather for TX and RX + - Vector Poll mode driver +-- Dump register + - SR-IOV VF + - Multi-process + - MAC/VLAN filter +@@ -38,6 +37,15 @@ Features of the HNS3 PMD are: + - NUMA support + - Generic flow API + - IEEE1588/802.1AS timestamping ++- Basic stats ++- Extended stats ++- Traffic Management API ++- Speed capabilities ++- Link Auto-negotiation ++- Link flow control ++- Dump register ++- Dump private info from device ++- FW version + + Prerequisites + ------------- +@@ -58,7 +66,8 @@ The following options can be modified in the ``config/rte_config.h`` file. + + - ``RTE_LIBRTE_HNS3_MAX_TQP_NUM_PER_PF`` (default ``256``) + +- Number of MAX queues reserved for PF. ++ Number of MAX queues reserved for PF on HIP09 and HIP10. ++ The MAX queue number is also determined by the value the firmware report. + + Runtime Config Options + ~~~~~~~~~~~~~~~~~~~~~~ +@@ -81,7 +90,8 @@ Runtime Config Options + ``common``. + + For example:: +- -a 0000:7d:00.0,rx_func_hint=simple ++ ++ -a 0000:7d:00.0,rx_func_hint=simple + + - ``tx_func_hint`` (default ``none``) + +@@ -101,7 +111,8 @@ Runtime Config Options + ``common``. + + For example:: +- -a 0000:7d:00.0,tx_func_hint=common ++ ++ -a 0000:7d:00.0,tx_func_hint=common + + - ``dev_caps_mask`` (default ``0``) + +@@ -113,22 +124,25 @@ Runtime Config Options + Its main purpose is to debug and avoid problems. + + For example:: +- -a 0000:7d:00.0,dev_caps_mask=0xF ++ ++ -a 0000:7d:00.0,dev_caps_mask=0xF + + - ``mbx_time_limit_ms`` (default ``500``) +- Used to define the mailbox time limit by user. +- Current, the max waiting time for MBX response is 500ms, but in +- some scenarios, it is not enough. Since it depends on the response +- of the kernel mode driver, and its response time is related to the +- scheduling of the system. In this special scenario, most of the +- cores are isolated, and only a few cores are used for system +- scheduling. When a large number of services are started, the +- scheduling of the system will be very busy, and the reply of the +- mbx message will time out, which will cause our PMD initialization +- to fail. So provide access to set mailbox time limit for user. +- +- For example:: +- -a 0000:7d:00.0,mbx_time_limit_ms=600 ++ ++ Used to define the mailbox time limit by user. ++ Current, the max waiting time for MBX response is 500ms, but in ++ some scenarios, it is not enough. Since it depends on the response ++ of the kernel mode driver, and its response time is related to the ++ scheduling of the system. In this special scenario, most of the ++ cores are isolated, and only a few cores are used for system ++ scheduling. When a large number of services are started, the ++ scheduling of the system will be very busy, and the reply of the ++ mbx message will time out, which will cause our PMD initialization ++ to fail. So provide access to set mailbox time limit for user. ++ ++ For example:: ++ ++ -a 0000:7d:00.0,mbx_time_limit_ms=600 + + Link status event Pre-conditions + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +@@ -137,7 +151,8 @@ Firmware 1.8.0.0 and later versions support reporting link changes to the PF. + Therefore, to use the LSC for the PF driver, ensure that the firmware version + also supports reporting link changes. + If the VF driver needs to support LSC, special patch must be added: +-``_. ++``_. ++ + Note: The patch has been uploaded to 5.13 of the Linux kernel mainline. + + +@@ -197,36 +212,50 @@ Generic flow API + + - ``RSS Flow`` + +- RSS Flow supports to set hash input set, hash function, enable hash +- and configure queues. +- For example: +- Configure queues as queue 0, 1, 2, 3. ++ RSS Flow supports for creating rule base on input tuple, hash key, queues ++ and hash algorithm. But hash key, queues and hash algorithm are the global ++ configuration for hardware which will affect other rules. ++ The rule just setting input tuple is completely independent. ++ ++ Run ``testpmd``: + + .. code-block:: console + +- testpmd> flow create 0 ingress pattern end actions rss types end \ +- queues 0 1 2 3 end / end ++ dpdk-testpmd -a 0000:7d:00.0 -l 10-18 -- -i --rxq=8 --txq=8 ++ ++ All IP packets can be distributed to 8 queues. + +- Enable hash and set input set for IPv4-TCP. ++ Set IPv4-TCP packet is distributed to 8 queues based on L3/L4 SRC only. + + .. code-block:: console + +- testpmd> flow create 0 ingress pattern eth / ipv4 / tcp / end \ +- actions rss types ipv4-tcp l3-src-only end queues end / end ++ testpmd> flow create 0 ingress pattern eth / ipv4 / tcp / end actions \ ++ rss types ipv4-tcp l4-src-only l3-src-only end queues end / end + +- Set symmetric hash enable for flow type IPv4-TCP. ++ Disable IPv4 packet RSS hash. + + .. code-block:: console + +- testpmd> flow create 0 ingress pattern eth / ipv4 / tcp / end \ +- actions rss types ipv4-tcp end queues end func symmetric_toeplitz / end ++ testpmd> flow create 0 ingress pattern eth / ipv4 / end actions rss \ ++ types none end queues end / end + +- Set hash function as simple xor. ++ Set hash function as symmetric Toeplitz. + + .. code-block:: console + + testpmd> flow create 0 ingress pattern end actions rss types end \ +- queues end func simple_xor / end ++ queues end func symmetric_toeplitz / end ++ ++ In this case, all packets that enabled RSS are hashed using symmetric ++ Toeplitz algorithm. ++ ++ Flush all RSS rules ++ ++ .. code-block:: console ++ ++ testpmd> flow flush 0 ++ ++ The RSS configurations of hardwre is back to the one ethdev ops set. + + Statistics + ---------- +diff --git a/dpdk/doc/guides/nics/i40e.rst b/dpdk/doc/guides/nics/i40e.rst +index a6c7dbd080..b37859a721 100644 +--- a/dpdk/doc/guides/nics/i40e.rst ++++ b/dpdk/doc/guides/nics/i40e.rst +@@ -88,13 +88,16 @@ Windows Prerequisites + - To load NetUIO driver, follow the steps mentioned in `dpdk-kmods repository + `_. + +-Recommended Matching List +-------------------------- +- +-It is highly recommended to upgrade the i40e kernel driver and firmware to +-avoid the compatibility issues with i40e PMD. Here is the suggested matching +-list which has been tested and verified. The detailed information can refer +-to chapter Tested Platforms/Tested NICs in release notes. ++Kernel driver and Firmware Matching List ++---------------------------------------- ++ ++It is highly recommended to upgrade the i40e kernel driver and firmware ++to avoid the compatibility issues with i40e PMD. ++The table below shows a summary of the DPDK versions ++with corresponding out-of-tree Linux kernel drivers and firmware. ++The full list of in-tree and out-of-tree Linux kernel drivers from kernel.org ++and Linux distributions that were tested and verified ++are listed in the Tested Platforms section of the Release Notes for each release. + + For X710/XL710/XXV710, + +diff --git a/dpdk/doc/guides/nics/ice.rst b/dpdk/doc/guides/nics/ice.rst +index ce075e067c..ccf9d68203 100644 +--- a/dpdk/doc/guides/nics/ice.rst ++++ b/dpdk/doc/guides/nics/ice.rst +@@ -41,13 +41,16 @@ Windows Prerequisites + - Loading of private Dynamic Device Personalization (DDP) package is not supported on Windows. + + +-Recommended Matching List +-------------------------- ++Kernel driver, DDP and Firmware Matching List ++--------------------------------------------- + + It is highly recommended to upgrade the ice kernel driver, firmware and DDP package + to avoid the compatibility issues with ice PMD. +-Here is the suggested matching list which has been tested and verified. +-The detailed information can refer to chapter Tested Platforms/Tested NICs in release notes. ++The table below shows a summary of the DPDK versions ++with corresponding out-of-tree Linux kernel drivers, DDP package and firmware. ++The full list of in-tree and out-of-tree Linux kernel drivers from kernel.org ++and Linux distributions that were tested and verified ++are listed in the Tested Platforms section of the Release Notes for each release. + + +-----------+---------------+-----------------+-----------+--------------+-----------+ + | DPDK | Kernel Driver | OS Default DDP | COMMS DDP | Wireless DDP | Firmware | +@@ -331,18 +334,18 @@ Additional Options + + ip link set dev enp24s0f0 vf 0 trust on + +-#. Bind the VF0, and run testpmd with 'cap=dcf' devarg:: ++#. Bind the VF0, and run testpmd with 'cap=dcf' with port representor for VF 1 and 2:: + +- dpdk-testpmd -l 22-25 -n 4 -a 18:01.0,cap=dcf -- -i ++ dpdk-testpmd -l 22-25 -n 4 -a 18:01.0,cap=dcf,representor=vf[1-2] -- -i + + #. Monitor the VF2 interface network traffic:: + + tcpdump -e -nn -i enp24s1f2 + +-#. Create one flow to redirect the traffic to VF2 by DCF:: ++#. Create one flow to redirect the traffic to VF2 by DCF (assume the representor port ID is 5):: + + flow create 0 priority 0 ingress pattern eth / ipv4 src is 192.168.0.2 \ +- dst is 192.168.0.3 / end actions vf id 2 / end ++ dst is 192.168.0.3 / end actions represented_port ethdev_port_id 5 / end + + #. Send the packet, and it should be displayed on tcpdump:: + +diff --git a/dpdk/doc/guides/nics/ixgbe.rst b/dpdk/doc/guides/nics/ixgbe.rst +index 5db7376992..a3a19a0f49 100644 +--- a/dpdk/doc/guides/nics/ixgbe.rst ++++ b/dpdk/doc/guides/nics/ixgbe.rst +@@ -16,8 +16,6 @@ The wider register gives space to hold multiple packet buffers so as to save ins + There is no change to PMD API. The RX/TX handler are the only two entries for vPMD packet I/O. + They are transparently registered at runtime RX/TX execution if all condition checks pass. + +-1. To date, only an SSE version of IX GBE vPMD is available. +- + Some constraints apply as pre-conditions for specific optimizations on bulk packet transfers. + The following sections explain RX and TX constraints in the vPMD. + +diff --git a/dpdk/doc/guides/nics/mana.rst b/dpdk/doc/guides/nics/mana.rst +index 005c0b2ca7..341146c4e7 100644 +--- a/dpdk/doc/guides/nics/mana.rst ++++ b/dpdk/doc/guides/nics/mana.rst +@@ -29,6 +29,7 @@ and must be installed separately: + It allows slow and privileged operations + (context initialization, hardware resources allocations) + to be managed by the kernel and fast operations to never leave user space. ++ The minimum required rdma-core version is v44. + + In most cases, rdma-core is shipped as a package with an OS distribution. + User can also install the upstream version of the rdma-core from +@@ -39,15 +40,14 @@ and must be installed separately: + Low-level user space driver library + for Microsoft Azure Network Adapter devices, + it is automatically loaded by libibverbs. +- +- The support of MANA is not merged in rdma-core 42. ++ The minimum required version of rdma-core with libmana is v44. + + - **Kernel modules** + + They provide the kernel-side verbs API and low level device drivers + that manage actual hardware initialization + and resources sharing with user space processes. +- The minimum required Linux kernel version is 6.1. ++ The minimum required Linux kernel version is 6.2. + + Unlike most other PMDs, these modules must remain loaded + and bound to their devices: +@@ -56,8 +56,6 @@ and must be installed separately: + - mana_ib: InifiniBand device driver. + - ib_uverbs: user space driver for verbs (entry point for libibverbs). + +- The support of MANA is planned in Linux 6.2. +- + Driver compilation and testing + ------------------------------ + +diff --git a/dpdk/doc/guides/nics/mlx5.rst b/dpdk/doc/guides/nics/mlx5.rst +index 51f51259e3..239e297d2a 100644 +--- a/dpdk/doc/guides/nics/mlx5.rst ++++ b/dpdk/doc/guides/nics/mlx5.rst +@@ -455,8 +455,12 @@ Limitations + encapsulation actions. + - For NIC Rx flow, supports ``MARK``, ``COUNT``, ``QUEUE``, ``RSS`` in the + sample actions list. +- - For E-Switch mirroring flow, supports ``RAW ENCAP``, ``Port ID``, +- ``VXLAN ENCAP``, ``NVGRE ENCAP`` in the sample actions list. ++ - For E-Switch mirroring flow, supports ``RAW_ENCAP``, ``PORT_ID``, ++ ``VXLAN_ENCAP``, ``NVGRE_ENCAP`` in the sample actions list. ++ - For E-Switch mirroring flow with sample ratio = 1, the ``ENCAP`` action ++ supports uplink port only. ++ - For E-Switch mirroring flow with sample ratio = 1, the ``PORT`` and ``JUMP`` actions ++ are not supported without presented ``ENCAP`` action in the sample actions list. + - For ConnectX-5 trusted device, the application metadata with SET_TAG index 0 + is not supported before ``RTE_FLOW_ACTION_TYPE_SAMPLE`` action. + +@@ -1121,6 +1125,9 @@ for an additional list of options shared with other mlx5 drivers. + - 0. If representor matching is disabled, then there will be no implicit + item added. As a result, ingress flow rules will match traffic + coming to any port, not only the port on which flow rule is created. ++ Because of that, default flow rules for ingress traffic cannot be created ++ and port starts in isolated mode by default. Port cannot be switched back ++ to non-isolated mode. + + - 1. If representor matching is enabled (default setting), + then each ingress pattern template has an implicit REPRESENTED_PORT +@@ -1547,6 +1554,14 @@ shortened below as "OFED". + | | | ConnectX-5 | | ConnectX-5 | + +-----------------------+-----------------+-----------------+ + ++.. table:: Minimal SW/HW versions for flow template API ++ ++ +-----------------+--------------------+--------------------+ ++ | DPDK | NIC | Firmware | ++ +=================+====================+====================+ ++ | 22.11 | ConnectX-6 Dx | xx.35.1012 | ++ +-----------------+--------------------+--------------------+ ++ + Notes for metadata + ------------------ + +diff --git a/dpdk/doc/guides/nics/tap.rst b/dpdk/doc/guides/nics/tap.rst +index 2f7417bddd..07df0d35a2 100644 +--- a/dpdk/doc/guides/nics/tap.rst ++++ b/dpdk/doc/guides/nics/tap.rst +@@ -34,14 +34,14 @@ Using the option ``mac=fixed`` you can create a fixed known MAC address:: + + The MAC address will have a fixed value with the last octet incrementing by one + for each interface string containing ``mac=fixed``. The MAC address is formatted +-as 00:'d':'t':'a':'p':[00-FF]. Convert the characters to hex and you get the +-actual MAC address: ``00:64:74:61:70:[00-FF]``. ++as 02:'d':'t':'a':'p':[00-FF]. Convert the characters to hex and you get the ++actual MAC address: ``02:64:74:61:70:[00-FF]``. + +- --vdev=net_tap0,mac="00:64:74:61:70:11" ++ --vdev=net_tap0,mac="02:64:74:61:70:11" + + The MAC address will have a user value passed as string. The MAC address is in + format with delimiter ``:``. The string is byte converted to hex and you get +-the actual MAC address: ``00:64:74:61:70:11``. ++the actual MAC address: ``02:64:74:61:70:11``. + + It is possible to specify a remote netdevice to capture packets from by adding + ``remote=foo1``, for example:: +diff --git a/dpdk/doc/guides/nics/virtio.rst b/dpdk/doc/guides/nics/virtio.rst +index c422e7347a..7eba49cc83 100644 +--- a/dpdk/doc/guides/nics/virtio.rst ++++ b/dpdk/doc/guides/nics/virtio.rst +@@ -307,6 +307,7 @@ Prerequisites for Rx interrupts + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + To support Rx interrupts, ++ + #. Check if guest kernel supports VFIO-NOIOMMU: + + Linux started to support VFIO-NOIOMMU since 4.8.0. Make sure the guest +@@ -469,12 +470,16 @@ according to below configuration: + + #. Split virtqueue mergeable path: If Rx mergeable is negotiated, in-order feature is + not negotiated, this path will be selected. ++ + #. Split virtqueue non-mergeable path: If Rx mergeable and in-order feature are not + negotiated, also Rx offload(s) are requested, this path will be selected. ++ + #. Split virtqueue in-order mergeable path: If Rx mergeable and in-order feature are + both negotiated, this path will be selected. ++ + #. Split virtqueue in-order non-mergeable path: If in-order feature is negotiated and + Rx mergeable is not negotiated, this path will be selected. ++ + #. Split virtqueue vectorized Rx path: If Rx mergeable is disabled and no Rx offload + requested, this path will be selected. + +@@ -483,16 +488,21 @@ according to below configuration: + + #. Packed virtqueue mergeable path: If Rx mergeable is negotiated, in-order feature + is not negotiated, this path will be selected. ++ + #. Packed virtqueue non-mergeable path: If Rx mergeable and in-order feature are not + negotiated, this path will be selected. ++ + #. Packed virtqueue in-order mergeable path: If in-order and Rx mergeable feature are + both negotiated, this path will be selected. ++ + #. Packed virtqueue in-order non-mergeable path: If in-order feature is negotiated and + Rx mergeable is not negotiated, this path will be selected. ++ + #. Packed virtqueue vectorized Rx path: If building and running environment support + (AVX512 || NEON) && in-order feature is negotiated && Rx mergeable + is not negotiated && TCP_LRO Rx offloading is disabled && vectorized option enabled, + this path will be selected. ++ + #. Packed virtqueue vectorized Tx path: If building and running environment support + (AVX512 || NEON) && in-order feature is negotiated && vectorized option enabled, + this path will be selected. +@@ -570,5 +580,7 @@ or configuration, below steps can help you identify which path you selected and + root cause faster. + + #. Run vhost/virtio test case; ++ + #. Run "perf top" and check virtio Rx/Tx callback names; ++ + #. Identify which virtio path is selected refer to above table. +diff --git a/dpdk/doc/guides/platform/cnxk.rst b/dpdk/doc/guides/platform/cnxk.rst +index aadd60b5d4..dca4e789b3 100644 +--- a/dpdk/doc/guides/platform/cnxk.rst ++++ b/dpdk/doc/guides/platform/cnxk.rst +@@ -111,7 +111,9 @@ where even VF bound to the first domain and odd VF bound to the second domain. + Typical application usage models are, + + #. Communication between the Linux kernel and DPDK application. ++ + #. Exception path to Linux kernel from DPDK application as SW ``KNI`` replacement. ++ + #. Communication between two different DPDK applications. + + SDP interface +@@ -130,6 +132,7 @@ can bind PF or VF to use SDP interface and it will be enumerated as ethdev ports + The primary use case for SDP is to enable the smart NIC use case. Typical usage models are, + + #. Communication channel between remote host and cnxk SoC over PCIe. ++ + #. Transfer packets received from network interface to remote host over PCIe and + vice-versa. + +@@ -253,7 +256,7 @@ context or stats using debugfs. + + Enable ``debugfs`` by: + +-1. Compile kernel with debugfs enabled, i.e ``CONFIG_DEBUGFS=y``. ++1. Compile kernel with debugfs enabled, i.e ``CONFIG_DEBUG_FS=y``. + 2. Boot OCTEON CN9K/CN10K with debugfs supported kernel. + 3. Verify ``debugfs`` mounted by default "mount | grep -i debugfs" or mount it manually by using. + +diff --git a/dpdk/doc/guides/prog_guide/cryptodev_lib.rst b/dpdk/doc/guides/prog_guide/cryptodev_lib.rst +index 01aad842a9..2b513bbf82 100644 +--- a/dpdk/doc/guides/prog_guide/cryptodev_lib.rst ++++ b/dpdk/doc/guides/prog_guide/cryptodev_lib.rst +@@ -98,14 +98,10 @@ The rte_cryptodev_configure API is used to configure a Crypto device. + The ``rte_cryptodev_config`` structure is used to pass the configuration + parameters for socket selection and number of queue pairs. + +-.. code-block:: c +- +- struct rte_cryptodev_config { +- int socket_id; +- /**< Socket to allocate resources on */ +- uint16_t nb_queue_pairs; +- /**< Number of queue pairs to configure on device */ +- }; ++.. literalinclude:: ../../../lib/cryptodev/rte_cryptodev.h ++ :language: c ++ :start-after: Structure rte_cryptodev_config 8< ++ :end-before: >8 End of structure rte_cryptodev_config. + + + Configuration of Queue Pairs +@@ -121,11 +117,11 @@ Each queue pairs resources may be allocated on a specified socket. + const struct rte_cryptodev_qp_conf *qp_conf, + int socket_id) + +- struct rte_cryptodev_qp_conf { +- uint32_t nb_descriptors; /**< Number of descriptors per queue pair */ +- struct rte_mempool *mp_session; +- /**< The mempool for creating session in sessionless mode */ +- }; ++ ++.. literalinclude:: ../../../lib/cryptodev/rte_cryptodev.h ++ :language: c ++ :start-after: Structure rte_cryptodev_qp_conf 8< ++ :end-before: >8 End of structure rte_cryptodev_qp_conf. + + + The field ``mp_session`` is used for creating temporary session to process +@@ -271,23 +267,10 @@ This allows the user to query a specific Crypto PMD and get all the device + features and capabilities. The ``rte_cryptodev_info`` structure contains all the + relevant information for the device. + +-.. code-block:: c +- +- struct rte_cryptodev_info { +- const char *driver_name; +- uint8_t driver_id; +- struct rte_device *device; +- +- uint64_t feature_flags; +- +- const struct rte_cryptodev_capabilities *capabilities; +- +- unsigned max_nb_queue_pairs; +- +- struct { +- unsigned max_nb_sessions; +- } sym; +- }; ++.. literalinclude:: ../../../lib/cryptodev/rte_cryptodev.h ++ :language: c ++ :start-after: Structure rte_cryptodev_info 8< ++ :end-before: >8 End of structure rte_cryptodev_info. + + + Operation Processing +@@ -499,37 +482,29 @@ a flow. Crypto sessions cache this immutable data in a optimal way for the + underlying PMD and this allows further acceleration of the offload of + Crypto workloads. + +-.. figure:: img/cryptodev_sym_sess.* +- + The Crypto device framework provides APIs to create session mempool and allocate + and initialize sessions for crypto devices, where sessions are mempool objects. + The application has to use ``rte_cryptodev_sym_session_pool_create()`` to +-create the session header mempool that creates a mempool with proper element +-size automatically and stores necessary information for safely accessing the +-session in the mempool's private data field. +- +-To create a mempool for storing session private data, the application has two +-options. The first is to create another mempool with elt size equal to or +-bigger than the maximum session private data size of all crypto devices that +-will share the same session header. The creation of the mempool shall use the +-traditional ``rte_mempool_create()`` with the correct ``elt_size``. The other +-option is to change the ``elt_size`` parameter in +-``rte_cryptodev_sym_session_pool_create()`` to the correct value. The first +-option is more complex to implement but may result in better memory usage as +-a session header normally takes smaller memory footprint as the session private +-data. ++create the session mempool header and the private data with the size specified ++by the user through the ``elt_size`` parameter in the function. ++The session private data is for the driver to initialize and access ++during crypto operations, hence the ``elt_size`` should be big enough ++for all drivers that will share this mempool. ++To obtain the proper session private data size of a crypto device, ++the user can call ``rte_cryptodev_sym_get_private_session_size()`` function. ++In case of heterogeneous crypto devices which will share the same session mempool, ++the maximum session private data size of them should be passed. + + Once the session mempools have been created, ``rte_cryptodev_sym_session_create()`` +-is used to allocate an uninitialized session from the given mempool. +-The session then must be initialized using ``rte_cryptodev_sym_session_init()`` +-for each of the required crypto devices. A symmetric transform chain +-is used to specify the operation and its parameters. See the section below for +-details on transforms. ++is used to allocate and initialize the session from the given mempool. ++The created session can ONLY be used by the crypto devices sharing the same driver ID ++as the device ID passed into the function as the parameter. ++In addition, a symmetric transform chain is used to specify the operation and its parameters. ++See the section below for details on transforms. + +-When a session is no longer used, user must call ``rte_cryptodev_sym_session_clear()`` +-for each of the crypto devices that are using the session, to free all driver +-private session data. Once this is done, session should be freed using +-``rte_cryptodev_sym_session_free`` which returns them to their mempool. ++When a session is no longer used, user must call ``rte_cryptodev_sym_session_free()`` ++to uninitialize the session data and return the session ++back to the mempool it belongs. + + + Transforms and Transform Chaining +@@ -548,22 +523,10 @@ Currently there are three transforms types cipher, authentication and AEAD. + Also it is important to note that the order in which the + transforms are passed indicates the order of the chaining. + +-.. code-block:: c +- +- struct rte_crypto_sym_xform { +- struct rte_crypto_sym_xform *next; +- /**< next xform in chain */ +- enum rte_crypto_sym_xform_type type; +- /**< xform type */ +- union { +- struct rte_crypto_auth_xform auth; +- /**< Authentication / hash xform */ +- struct rte_crypto_cipher_xform cipher; +- /**< Cipher xform */ +- struct rte_crypto_aead_xform aead; +- /**< AEAD xform */ +- }; +- }; ++.. literalinclude:: ../../../lib/cryptodev/rte_crypto_sym.h ++ :language: c ++ :start-after: Structure rte_crypto_sym_xform 8< ++ :end-before: >8 End of structure rte_crypto_sym_xform. + + The API does not place a limit on the number of transforms that can be chained + together but this will be limited by the underlying Crypto device poll mode +@@ -586,61 +549,11 @@ authentication/ cipher/ AEAD parameters required depending on the type of operat + specified in the session or the transform + chain. + +-.. code-block:: c ++.. literalinclude:: ../../../lib/cryptodev/rte_crypto_sym.h ++ :language: c ++ :start-after: Structure rte_crypto_sym_op 8< ++ :end-before: >8 End of structure rte_crypto_sym_op. + +- struct rte_crypto_sym_op { +- struct rte_mbuf *m_src; +- struct rte_mbuf *m_dst; +- +- union { +- void *session; +- /**< Handle for the initialised session context */ +- struct rte_crypto_sym_xform *xform; +- /**< Session-less API Crypto operation parameters */ +- }; +- +- union { +- struct { +- struct { +- uint32_t offset; +- uint32_t length; +- } data; /**< Data offsets and length for AEAD */ +- +- struct { +- uint8_t *data; +- rte_iova_t phys_addr; +- } digest; /**< Digest parameters */ +- +- struct { +- uint8_t *data; +- rte_iova_t phys_addr; +- } aad; +- /**< Additional authentication parameters */ +- } aead; +- +- struct { +- struct { +- struct { +- uint32_t offset; +- uint32_t length; +- } data; /**< Data offsets and length for ciphering */ +- } cipher; +- +- struct { +- struct { +- uint32_t offset; +- uint32_t length; +- } data; +- /**< Data offsets and length for authentication */ +- +- struct { +- uint8_t *data; +- rte_iova_t phys_addr; +- } digest; /**< Digest parameters */ +- } auth; +- }; +- }; +- }; + + Synchronous mode + ---------------- +diff --git a/dpdk/doc/guides/prog_guide/event_timer_adapter.rst b/dpdk/doc/guides/prog_guide/event_timer_adapter.rst +index d7307a29bb..7733424aac 100644 +--- a/dpdk/doc/guides/prog_guide/event_timer_adapter.rst ++++ b/dpdk/doc/guides/prog_guide/event_timer_adapter.rst +@@ -35,7 +35,7 @@ device upon timer expiration. + + The Event Timer Adapter API represents each event timer with a generic struct, + which contains an event and user metadata. The ``rte_event_timer`` struct is +-defined in ``lib/event/librte_event_timer_adapter.h``. ++defined in ``rte_event_timer_adapter.h``. + + .. _timer_expiry_event: + +@@ -229,9 +229,7 @@ Note that it is necessary to initialize the event timer state to + RTE_EVENT_TIMER_NOT_ARMED. Also note that we have saved a pointer to the + ``conn`` object in the timer's event payload. This will allow us to locate + the connection object again once we dequeue the timer expiry event from the +-event device later. As a convenience, the application may specify no value for +-ev.event_ptr, and the adapter will by default set it to point at the event +-timer itself. ++event device later. + + Now we can arm the event timer with ``rte_event_timer_arm_burst()``: + +diff --git a/dpdk/doc/guides/prog_guide/generic_segmentation_offload_lib.rst b/dpdk/doc/guides/prog_guide/generic_segmentation_offload_lib.rst +index e605b86376..30d13bcc61 100644 +--- a/dpdk/doc/guides/prog_guide/generic_segmentation_offload_lib.rst ++++ b/dpdk/doc/guides/prog_guide/generic_segmentation_offload_lib.rst +@@ -204,7 +204,7 @@ To segment an outgoing packet, an application must: + - a flag, that indicates whether the IPv4 headers of output segments should + contain fixed or incremental ID values. + +-2. Set the appropriate ol_flags in the mbuf. ++#. Set the appropriate ol_flags in the mbuf. + + - The GSO library use the value of an mbuf's ``ol_flags`` attribute to + determine how a packet should be segmented. It is the application's +diff --git a/dpdk/doc/guides/prog_guide/graph_lib.rst b/dpdk/doc/guides/prog_guide/graph_lib.rst +index 1cfdc86433..4ab0623f44 100644 +--- a/dpdk/doc/guides/prog_guide/graph_lib.rst ++++ b/dpdk/doc/guides/prog_guide/graph_lib.rst +@@ -173,7 +173,7 @@ Create the graph object + ~~~~~~~~~~~~~~~~~~~~~~~ + Now that the nodes are linked, Its time to create a graph by including + the required nodes. The application can provide a set of node patterns to +-form a graph object. The ``famish()`` API used underneath for the pattern ++form a graph object. The ``fnmatch()`` API used underneath for the pattern + matching to include the required nodes. After the graph create any changes to + nodes or graph is not allowed. + +diff --git a/dpdk/doc/guides/prog_guide/img/cryptodev_sym_sess.svg b/dpdk/doc/guides/prog_guide/img/cryptodev_sym_sess.svg +deleted file mode 100644 +index 9b522458c8..0000000000 +--- a/dpdk/doc/guides/prog_guide/img/cryptodev_sym_sess.svg ++++ /dev/null +@@ -1,417 +0,0 @@ +- +- +- +-image/svg+xmlRounded Rectangle.12Crypto Symmetric SessionRounded Rectangle.13Private Session Data +- +- +-Rounded Rectangle.12Crypto Symmetric SessionCrypto Driver Private Session +-Crypto Symmetric Session +-uint16_t nb_drivers; +-struct { +-void *data; +-} session_data[]; +-uint16_t user_data_sz; +-user_data +-uint16_t refcnt; +-uint64_t opaque_data; +-Rounded Rectangle.13Private Session DataPrivate Session Data +-Rounded Rectangle.12Crypto Symmetric SessionCrypto Driver Private Session +-Rounded Rectangle.13Private Session DataPrivate Session Data +- +-... +- +\ No newline at end of file +diff --git a/dpdk/doc/guides/prog_guide/multi_proc_support.rst b/dpdk/doc/guides/prog_guide/multi_proc_support.rst +index 815e8bdc43..df234548a7 100644 +--- a/dpdk/doc/guides/prog_guide/multi_proc_support.rst ++++ b/dpdk/doc/guides/prog_guide/multi_proc_support.rst +@@ -107,15 +107,19 @@ Running Multiple Independent DPDK Applications + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + In addition to the above scenarios involving multiple DPDK processes working together, +-it is possible to run multiple DPDK processes side-by-side, ++it is possible to run multiple DPDK processes concurrently, + where those processes are all working independently. + Support for this usage scenario is provided using the ``--file-prefix`` parameter to the EAL. + +-By default, the EAL creates hugepage files on each hugetlbfs filesystem using the rtemap_X filename, ++The EAL puts shared runtime files in a directory based on standard conventions. ++If ``$RUNTIME_DIRECTORY`` is defined in the environment, ++it is used (as ``$RUNTIME_DIRECTORY/dpdk``). ++Otherwise, if DPDK is run as root user, it uses ``/var/run/dpdk`` ++or if run as non-root user then the ``/tmp/dpdk`` (or ``$XDG_RUNTIME_DIRECTORY/dpdk``) is used. ++Hugepage files on each hugetlbfs filesystem use the ``rtemap_X`` filename, + where X is in the range 0 to the maximum number of hugepages -1. +-Similarly, it creates shared configuration files, memory mapped in each process, using the /var/run/.rte_config filename, +-when run as root (or $HOME/.rte_config when run as a non-root user; +-if filesystem and device permissions are set up to allow this). ++Similarly, it creates shared configuration files, memory mapped in each process, ++using the ``.rte_config`` filename. + The rte part of the filenames of each of the above is configurable using the file-prefix parameter. + + In addition to specifying the file-prefix parameter, +diff --git a/dpdk/doc/guides/prog_guide/rte_flow.rst b/dpdk/doc/guides/prog_guide/rte_flow.rst +index 3e6242803d..d0b7833a2f 100644 +--- a/dpdk/doc/guides/prog_guide/rte_flow.rst ++++ b/dpdk/doc/guides/prog_guide/rte_flow.rst +@@ -148,14 +148,15 @@ Attribute: Group + Flow rules can be grouped by assigning them a common group number. Groups + allow a logical hierarchy of flow rule groups (tables) to be defined. These + groups can be supported virtually in the PMD or in the physical device. +-Group 0 is the default group and this is the only group which flows are +-guarantee to matched against, all subsequent groups can only be reached by +-way of the JUMP action from a matched flow rule. ++Group 0 is the default group and is the only group that ++flows are guaranteed to be matched against. ++All subsequent groups can only be reached by using a JUMP action ++from a matched flow rule. + + Although optional, applications are encouraged to group similar rules as + much as possible to fully take advantage of hardware capabilities + (e.g. optimized matching) and work around limitations (e.g. a single pattern +-type possibly allowed in a given group), while being aware that the groups ++type possibly allowed in a given group), while being aware that the groups' + hierarchies must be programmed explicitly. + + Note that support for more than a single group is not guaranteed. +@@ -170,7 +171,7 @@ Priority levels are arbitrary and up to the application, they do + not need to be contiguous nor start from 0, however the maximum number + varies between devices and may be affected by existing flow rules. + +-A flow which matches multiple rules in the same group will always matched by ++A flow which matches multiple rules in the same group will always be matched by + the rule with the highest priority in that group. + + If a packet is matched by several rules of a given group for a given +@@ -1513,22 +1514,15 @@ rte_flow_flex_item_create() routine. + value and mask. + + Item: ``L2TPV2`` +-^^^^^^^^^^^^^^^^^^^ ++^^^^^^^^^^^^^^^^ + + Matches a L2TPv2 header. + +-- ``flags_version``: flags(12b), version(4b). +-- ``length``: total length of the message. +-- ``tunnel_id``: identifier for the control connection. +-- ``session_id``: identifier for a session within a tunnel. +-- ``ns``: sequence number for this date or control message. +-- ``nr``: sequence number expected in the next control message to be received. +-- ``offset_size``: offset of payload data. +-- ``offset_padding``: offset padding, variable length. ++- ``hdr``: header definition (``rte_l2tpv2.h``). + - Default ``mask`` matches flags_version only. + + Item: ``PPP`` +-^^^^^^^^^^^^^^^^^^^ ++^^^^^^^^^^^^^ + + Matches a PPP header. + +@@ -1748,12 +1742,12 @@ flow group/tables on the device, this action redirects the matched flow to + the specified group on that device. + + If a matched flow is redirected to a table which doesn't contain a matching +-rule for that flow then the behavior is undefined and the resulting behavior +-is up to the specific device. Best practice when using groups would be define ++rule for that flow, then the behavior is undefined and the resulting behavior ++is up to the specific device. Best practice when using groups would be to define + a default flow rule for each group which a defines the default actions in that + group so a consistent behavior is defined. + +-Defining an action for matched flow in a group to jump to a group which is ++Defining an action for a matched flow in a group to jump to a group which is + higher in the group hierarchy may not be supported by physical devices, + depending on how groups are mapped to the physical devices. In the + definitions of jump actions, applications should be aware that it may be +@@ -1925,8 +1919,8 @@ Also, regarding packet encapsulation ``level``: + level. + + - ``2`` and subsequent values request RSS to be performed on the specified +- inner packet encapsulation level, from outermost to innermost (lower to +- higher values). ++ inner packet encapsulation level, from outermost to innermost (lower to ++ higher values). + + Values other than ``0`` are not necessarily supported. + +@@ -2888,20 +2882,23 @@ The immediate value ``RTE_FLOW_FIELD_VALUE`` (or a pointer to it + ``RTE_FLOW_FIELD_START`` is used to point to the beginning of a packet. + See ``enum rte_flow_field_id`` for the list of supported fields. + +-``op`` selects the operation to perform on a destination field. ++``op`` selects the operation to perform on a destination field: ++ + - ``set`` copies the data from ``src`` field to ``dst`` field. + - ``add`` adds together ``dst`` and ``src`` and stores the result into ``dst``. +-- ``sub`` subtracts ``src`` from ``dst`` and stores the result into ``dst`` ++- ``sub`` subtracts ``src`` from ``dst`` and stores the result into ``dst``. + + ``width`` defines a number of bits to use from ``src`` field. + + ``level`` is used to access any packet field on any encapsulation level +-as well as any tag element in the tag array. +-- ``0`` means the default behaviour. Depending on the packet type, it can +-mean outermost, innermost or anything in between. ++as well as any tag element in the tag array: ++ ++- ``0`` means the default behaviour. Depending on the packet type, ++ it can mean outermost, innermost or anything in between. + - ``1`` requests access to the outermost packet encapsulation level. + - ``2`` and subsequent values requests access to the specified packet +-encapsulation level, from outermost to innermost (lower to higher values). ++ encapsulation level, from outermost to innermost (lower to higher values). ++ + For the tag array (in case of multiple tags are supported and present) + ``level`` translates directly into the array index. + +@@ -3609,6 +3606,7 @@ Asynchronous operations + ----------------------- + + Flow rules management can be done via special lockless flow management queues. ++ + - Queue operations are asynchronous and not thread-safe. + + - Operations can thus be invoked by the app's datapath, +diff --git a/dpdk/doc/guides/prog_guide/rte_security.rst b/dpdk/doc/guides/prog_guide/rte_security.rst +index 7418e35c1b..ae8b0aaef3 100644 +--- a/dpdk/doc/guides/prog_guide/rte_security.rst ++++ b/dpdk/doc/guides/prog_guide/rte_security.rst +@@ -671,68 +671,27 @@ Security session configuration + + Security Session configuration structure is defined as ``rte_security_session_conf`` + +-.. code-block:: c +- +- struct rte_security_session_conf { +- enum rte_security_session_action_type action_type; +- /**< Type of action to be performed on the session */ +- enum rte_security_session_protocol protocol; +- /**< Security protocol to be configured */ +- union { +- struct rte_security_ipsec_xform ipsec; +- struct rte_security_macsec_xform macsec; +- struct rte_security_pdcp_xform pdcp; +- struct rte_security_docsis_xform docsis; +- }; +- /**< Configuration parameters for security session */ +- struct rte_crypto_sym_xform *crypto_xform; +- /**< Security Session Crypto Transformations */ +- void *userdata; +- /**< Application specific userdata to be saved with session */ +- }; ++.. literalinclude:: ../../../lib/security/rte_security.h ++ :language: c ++ :start-after: Structure rte_security_session_conf 8< ++ :end-before: >8 End of structure rte_security_session_conf. + + The configuration structure reuses the ``rte_crypto_sym_xform`` struct for crypto related + configuration. The ``rte_security_session_action_type`` struct is used to specify whether the + session is configured for Lookaside Protocol offload or Inline Crypto or Inline Protocol + Offload. + +-.. code-block:: c +- +- enum rte_security_session_action_type { +- RTE_SECURITY_ACTION_TYPE_NONE, +- /**< No security actions */ +- RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO, +- /**< Crypto processing for security protocol is processed inline +- * during transmission +- */ +- RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL, +- /**< All security protocol processing is performed inline during +- * transmission +- */ +- RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL, +- /**< All security protocol processing including crypto is performed +- * on a lookaside accelerator +- */ +- RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO +- /**< Similar to ACTION_TYPE_NONE but crypto processing for security +- * protocol is processed synchronously by a CPU. +- */ +- }; ++.. literalinclude:: ../../../lib/security/rte_security.h ++ :language: c ++ :start-after: Enumeration of rte_security_session_action_type 8< ++ :end-before: >8 End enumeration of rte_security_session_action_type. + + The ``rte_security_session_protocol`` is defined as + +-.. code-block:: c +- +- enum rte_security_session_protocol { +- RTE_SECURITY_PROTOCOL_IPSEC = 1, +- /**< IPsec Protocol */ +- RTE_SECURITY_PROTOCOL_MACSEC, +- /**< MACSec Protocol */ +- RTE_SECURITY_PROTOCOL_PDCP, +- /**< PDCP Protocol */ +- RTE_SECURITY_PROTOCOL_DOCSIS, +- /**< DOCSIS Protocol */ +- }; ++.. literalinclude:: ../../../lib/security/rte_security.h ++ :language: c ++ :start-after: Enumeration of rte_security_session_protocol 8< ++ :end-before: >8 End enumeration of rte_security_session_protocol. + + Currently the library defines configuration parameters for IPsec and PDCP only. + For other protocols like MACSec, structures and enums are defined as place holders +diff --git a/dpdk/doc/guides/rawdevs/ntb.rst b/dpdk/doc/guides/rawdevs/ntb.rst +index 2bb115d13f..f8befc6594 100644 +--- a/dpdk/doc/guides/rawdevs/ntb.rst ++++ b/dpdk/doc/guides/rawdevs/ntb.rst +@@ -1,6 +1,8 @@ + .. SPDX-License-Identifier: BSD-3-Clause + Copyright(c) 2018 Intel Corporation. + ++.. include:: ++ + NTB Rawdev Driver + ================= + +@@ -17,19 +19,23 @@ some information by using scratchpad registers. + BIOS setting on Intel Xeon + -------------------------- + +-Intel Non-transparent Bridge needs special BIOS setting. The reference for +-Skylake is https://www.intel.com/content/dam/support/us/en/documents/server-products/Intel_Xeon_Processor_Scalable_Family_BIOS_User_Guide.pdf +- +-- Set the needed PCIe port as NTB to NTB mode on both hosts. +-- Enable NTB bars and set bar size of bar 23 and bar 45 as 12-29 (4K-512M) +- on both hosts (for Ice Lake, bar size can be set as 12-51, namely 4K-128PB). +- Note that bar size on both hosts should be the same. +-- Disable split bars for both hosts. +-- Set crosslink control override as DSD/USP on one host, USD/DSP on +- another host. +-- Disable PCIe PII SSC (Spread Spectrum Clocking) for both hosts. This +- is a hardware requirement. +- ++Intel Non-transparent Bridge (NTB) needs special BIOS settings on both systems. ++Note that for 4th Generation Intel\ |reg| Xeon\ |reg| Scalable Processors, ++option ``Port Subsystem Mode`` should be changed from ``Gen5`` to ``Gen4 Only``, ++then reboot. ++ ++- Set ``Non-Transparent Bridge PCIe Port Definition`` for needed PCIe ports ++ as ``NTB to NTB`` mode, on both hosts. ++- Set ``Enable NTB BARs`` as ``Enabled``, on both hosts. ++- Set ``Enable SPLIT BARs`` as ``Disabled``, on both hosts. ++- Set ``Imbar1 Size``, ``Imbar2 Size``, ``Embar1 Size`` and ``Embar2 Size``, ++ as 12-29 (i.e., 4K-512M) for 2nd Generation Intel\ |reg| Xeon\ |reg| Scalable Processors; ++ as 12-51 (i.e., 4K-128PB) for 3rd and 4th Generation Intel\ |reg| Xeon\ |reg| Scalable Processors. ++ Note that those bar sizes on both hosts should be the same. ++- Set ``Crosslink Control override`` as ``DSD/USP`` on one host, ++ ``USD/DSP`` on another host. ++- Set ``PCIe PLL SSC (Spread Spectrum Clocking)`` as ``Disabled``, on both hosts. ++ This is a hardware requirement when using Re-timer Cards. + + Device Setup + ------------ +@@ -145,4 +151,8 @@ like the following: + Limitation + ---------- + +-- This PMD only supports Intel Skylake and Ice Lake platforms. ++This PMD is only supported on Intel Xeon Platforms: ++ ++- 4th Generation Intel® Xeon® Scalable Processors. ++- 3rd Generation Intel® Xeon® Scalable Processors. ++- 2nd Generation Intel® Xeon® Scalable Processors. +diff --git a/dpdk/doc/guides/rel_notes/release_22_11.rst b/dpdk/doc/guides/rel_notes/release_22_11.rst +index 26e0560725..7ce5c436c0 100644 +--- a/dpdk/doc/guides/rel_notes/release_22_11.rst ++++ b/dpdk/doc/guides/rel_notes/release_22_11.rst +@@ -805,3 +805,1094 @@ Tested Platforms + ~~~~~~~~~~~~~ + + * drivers: fix symbol exports when map is omitted ++ ++22.11.2 Release Notes ++--------------------- ++ ++ ++22.11.2 Fixes ++~~~~~~~~~~~~~ ++ ++* acl: fix crash on PPC64 with GCC 11 ++* app/bbdev: add allocation checks ++* app/bbdev: check statistics failure ++* app/bbdev: fix build with optional flag ++* app/bbdev: fix build with optional flag ++* app/compress-perf: fix remaining data for ops ++* app/compress-perf: fix some typos ++* app/compress-perf: fix testing single operation ++* app/crypto-perf: fix IPsec direction ++* app/crypto-perf: fix number of segments ++* app/crypto-perf: fix session freeing ++* app/crypto-perf: fix SPI zero ++* app/crypto-perf: fix test file memory leak ++* app/dumpcap: fix storing port identifier ++* app/flow-perf: fix division or module by zero ++* app/testpmd: cleanup cleanly from signal ++* app/testpmd: fix crash on cleanup ++* app/testpmd: fix encap/decap size calculation ++* app/testpmd: fix forwarding stats for Tx dropped ++* app/testpmd: fix interactive mode on Windows ++* app/testpmd: fix interactive mode with no ports ++* app/testpmd: fix link check condition on port start ++* app/testpmd: fix packet count in IEEE 1588 engine ++* app/testpmd: fix packet transmission in noisy VNF engine ++* app/testpmd: fix secondary process packet forwarding ++* app/testpmd: fix Tx preparation in checksum engine ++* baseband/acc: add explicit mbuf append for soft output ++* baseband/acc: fix acc100 iteration counter in TB ++* baseband/acc: fix acc100 queue mapping to 64 bits ++* baseband/acc: fix check after deref and dead code ++* baseband/acc: fix iteration counter in TB mode ++* baseband/acc: fix memory leak on acc100 close ++* baseband/acc: fix multiplexing acc100 operations ++* baseband/acc: prevent to dequeue more than requested ++* baseband/acc: protect from TB negative scenario ++* build: detect backtrace availability ++* build: fix dependencies lookup ++* build: fix toolchain definition ++* bus/fslmc: fix deadlock on MC send command timeout ++* bus/ifpga: fix devargs handling ++* cmdline: handle EOF as quit ++* cmdline: make rdline status not private ++* common/cnxk: add memory clobber to steor and ldeor ++* common/cnxk: fix aura ID handling ++* common/cnxk: fix auth key length ++* common/cnxk: fix channel mask for SDP interfaces ++* common/cnxk: fix dual VLAN parsing ++* common/cnxk: fix IPv6 extension header parsing ++* common/cnxk: fix IPv6 extension matching ++* common/cnxk: fix second pass flow rule layer type ++* common/cnxk: reduce channel count per LMAC ++* common/mlx5: fix offset of a field ++* common/mlx5: improve AES-XTS tweak capability check ++* common/mlx5: use just sufficient barrier for Arm ++* common/sfc_efx/base: add MAE mark reset action ++* compressdev: fix empty devargs parsing ++* compressdev: fix end of driver list ++* compress/mlx5: fix decompress xform validation ++* compress/mlx5: fix output Adler-32 checksum offset ++* compress/mlx5: fix queue setup for partial transformations ++* crypto/ccp: fix IOVA handling ++* crypto/ccp: fix PCI probing ++* crypto/ccp: remove some dead code for UIO ++* crypto/ccp: remove some printf ++* crypto/cnxk: fix digest for empty input data ++* cryptodev: fix empty devargs parsing ++* cryptodev: fix sym session mempool creation description ++* cryptodev: fix telemetry data truncation ++* crypto/ipsec_mb: fix ZUC-256 maximum tag length ++* crypto/ipsec_mb: relax multi-process requirement ++* crypto/ipsec_mb: remove unnecessary null check ++* crypto/openssl: fix freeing in RSA EVP ++* crypto/openssl: fix warning on copy length ++* crypto/qat: fix build ++* crypto/qat: fix build for generic x86 with GCC 12 ++* crypto/qat: fix SM3 auth mode ++* crypto/qat: fix stream cipher direction ++* devtools: fix escaped space in grep pattern ++* devtools: fix name check with mbox files ++* devtools: move mailmap check after patch applied ++* dma/ioat: fix device stop if no copies done ++* dma/ioat: fix error reporting on restart ++* dma/ioat: fix indexes after restart ++* dma/skeleton: fix empty devargs parsing ++* doc: add gpudev to the Doxygen index ++* doc: add Linux capability to access physical addresses ++* doc: fix code blocks in cryptodev guide ++* doc: fix DCF instructions in ice guide ++* doc: fix dependency setup in l2fwd-cat example guide ++* doc: fix description of L2TPV2 flow item ++* doc: fix firmware list in bnxt guide ++* doc: fix LPM support in l3forward guide ++* doc: fix pipeline example path in user guide ++* doc: fix reference to event timer header ++* drivers/bus: fix leak for devices without driver ++* drivers: fix symbol exports when map is omitted ++* eal: cleanup alarm and hotplug before memory detach ++* eal/freebsd: fix lock in alarm callback ++* eal/linux: fix hugetlbfs sub-directories discovery ++* eal/unix: fix thread creation ++* eal: use same atomic intrinsics for GCC and clang ++* eal/windows: fix pedantic build ++* eal/windows: fix thread creation ++* eal/windows: mark memory config as complete ++* ethdev: fix build with LTO ++* ethdev: fix telemetry data truncation ++* ethdev: remove telemetry Rx mbuf alloc failed field ++* event/cnxk: fix burst timer arm ++* event/cnxk: fix SSO cleanup ++* event/cnxk: fix timer operations in secondary process ++* event/cnxk: wait for CPT flow control on WQE path ++* eventdev/crypto: fix enqueue count ++* eventdev/crypto: fix failed events ++* eventdev/crypto: fix function symbol export ++* eventdev/crypto: fix offset used while flushing events ++* eventdev/crypto: fix overflow in circular buffer ++* eventdev/eth_rx: fix getting adapter instance ++* eventdev/eth_tx: fix devices loop ++* eventdev: fix memory size for telemetry ++* eventdev/timer: fix overflow ++* examples/cmdline: fix build with GCC 12 ++* examples/fips_validation: add extra space in JSON buffer ++* examples/fips_validation: fix AES-GCM tests ++* examples/fips_validation: fix AES-XTS sequence number ++* examples/fips_validation: fix integer parsing ++* examples/fips_validation: fix MCT output for SHA ++* examples/ipsec-secgw: fix auth IV length ++* examples/ipsec-secgw: fix offload variable init ++* examples/l2fwd-event: fix worker cleanup ++* examples/l3fwd: remove hash entry number ++* examples/qos_sched: fix config entries in wrong sections ++* examples/qos_sched: fix debug mode ++* examples/qos_sched: fix Tx port config when link down ++* fbarray: fix metadata dump ++* gpudev: export header file for external drivers ++* gpudev: fix deadlocks when registering callback ++* graph: fix node shrink ++* hash: fix GFNI implementation build with GCC 12 ++* kni: fix build on RHEL 9.1 ++* kni: fix possible starvation when mbufs are exhausted ++* kvargs: add API documentation for process callback ++* mem: fix heap ID in telemetry ++* mem: fix hugepage info mapping ++* mem: fix telemetry data truncation ++* mempool: fix telemetry data truncation ++* net/bnxt: fix link state change interrupt config ++* net/bnxt: fix RSS hash in mbuf ++* net/bnxt: fix Rx queue stats after queue stop and start ++* net/bnxt: fix Tx queue stats after queue stop and start ++* net/cnxk: fix deadlock in security session creation ++* net/cnxk: fix LBK BPID usage ++* net/cnxk: fix packet type for IPv6 packets post decryption ++* net/cnxk: validate RED threshold config ++* net/e1000: fix saving of stripped VLAN TCI ++* net/ena: fix deadlock in RSS RETA update ++* net/gve: fix offloading capability ++* net/hns3: add debug info for Rx/Tx dummy function ++* net/hns3: add verification of RSS types ++* net/hns3: allow adding queue buffer size hash rule ++* net/hns3: declare flow rule keeping capability ++* net/hns3: extract common functions to set Rx/Tx ++* net/hns3: extract common function to query device ++* net/hns3: fix burst mode query with dummy function ++* net/hns3: fix clearing RSS configuration ++* net/hns3: fix config struct used for conversion ++* net/hns3: fix duplicate RSS rule check ++* net/hns3: fix empty devargs parsing ++* net/hns3: fix inaccurate RTC time to read ++* net/hns3: fix log about indirection table size ++* net/hns3: fix possible truncation of hash key when config ++* net/hns3: fix possible truncation of redirection table ++* net/hns3: fix RSS key size compatibility ++* net/hns3: fix warning on flush or destroy rule ++* net/hns3: make getting Tx function static ++* net/hns3: refactor set RSS hash algorithm and key interface ++* net/hns3: reimplement hash flow function ++* net/hns3: remove debug condition for Tx prepare ++* net/hns3: remove useless code when destroy valid RSS rule ++* net/hns3: save hash algo to RSS filter list node ++* net/hns3: separate flow RSS config from RSS conf ++* net/hns3: separate setting and clearing RSS rule ++* net/hns3: separate setting hash algorithm ++* net/hns3: separate setting hash key ++* net/hns3: separate setting redirection table ++* net/hns3: separate setting RSS types ++* net/hns3: separate Tx prepare from getting Tx function ++* net/hns3: use hardware config to report hash key ++* net/hns3: use hardware config to report hash types ++* net/hns3: use hardware config to report redirection table ++* net/hns3: use new RSS rule to configure hardware ++* net/hns3: use RSS filter list to check duplicated rule ++* net/i40e: fix AVX512 fast-free path ++* net/i40e: fix MAC loopback on X722 ++* net/i40e: fix maximum frame size configuration ++* net/i40e: fix validation of flow transfer attribute ++* net/i40e: reduce interrupt interval in multi-driver mode ++* net/i40e: revert link status check on device start ++* net/iavf: add lock for VF commands ++* net/iavf: fix building data desc ++* net/iavf: fix device stop during reset ++* net/iavf: fix outer UDP checksum offload ++* net/iavf: fix VLAN offload with AVX2 ++* net/iavf: protect insertion in flow list ++* net/ice: fix Rx timestamp ++* net/ice: fix validation of flow transfer attribute ++* net/idpf: fix driver infos ++* net/idpf: fix mbuf leak in split Tx ++* net/idpf: reset queue flag when queue is stopped ++* net/ipn3ke: fix representor name ++* net/ipn3ke: fix thread exit ++* net/ixgbe: enable IPv6 mask in flow rules ++* net/ixgbe: fix firmware version consistency ++* net/ixgbe: fix IPv6 mask in flow director ++* net/mana: enable driver by default ++* net/mana: fix stats counters ++* net/mlx5: check compressed CQE opcode in vectorized Rx ++* net/mlx5: fix available tag registers calculation for HWS ++* net/mlx5: fix build with GCC 12 and ASan ++* net/mlx5: fix CQE dump for Tx ++* net/mlx5: fix crash on action template failure ++* net/mlx5: fix egress group translation in HWS ++* net/mlx5: fix error CQE dumping for vectorized Rx ++* net/mlx5: fix flow sample with ConnectX-5 ++* net/mlx5: fix GENEVE resource overwrite ++* net/mlx5: fix hairpin Tx queue reference count ++* net/mlx5: fix isolated mode if no representor matching ++* net/mlx5: fix read device clock in real time mode ++* net/mlx5: fix sysfs port name translation ++* net/mlx5: fix wait descriptor opcode for ConnectX-7 ++* net/mlx5: fix warning for Tx scheduling option ++* net/mlx5: fix Windows build with MinGW GCC 12 ++* net/mlx5/hws: fix error code of send queue action ++* net/mlx5/hws: fix IPv4 fragment matching ++* net/mlx5/hws: fix memory leak on general pool DB init ++* net/mlx5/hws: fix pattern creation ++* net/mlx5: ignore non-critical syndromes for Rx queue ++* net/nfp: fix 48-bit DMA support for NFDk ++* net/nfp: fix firmware name derived from PCI name ++* net/nfp: fix getting RSS configuration ++* net/nfp: fix max DMA length ++* net/nfp: fix MTU configuration order ++* net/nfp: fix offload of multiple output actions ++* net/nfp: fix set DSCP flow action ++* net/nfp: fix set IPv4 flow action ++* net/nfp: fix set IPv6 flow action ++* net/nfp: fix set MAC flow action ++* net/nfp: fix set TP flow action ++* net/nfp: fix set TTL flow action ++* net/nfp: fix teardown of flows sharing a mask ID ++* net/nfp: fix Tx packet drop for large data length ++* net/nfp: fix VNI of VXLAN encap action ++* net/nfp: restrict flow flush to the port ++* net/nfp: store counter reset before zeroing flow query ++* net/ngbe: add spinlock protection on YT PHY ++* net/ngbe: fix packet type to parse from offload flags ++* net/sfc: enforce fate action in transfer flow rules ++* net/sfc: export pick transfer proxy callback to representors ++* net/sfc: fix MAC address entry leak in transfer flow parsing ++* net/sfc: fix resetting mark in tunnel offload switch rules ++* net/sfc: invalidate switch port entry on representor unplug ++* net/txgbe: fix default signal quality value for KX/KX4 ++* net/txgbe: fix interrupt loss ++* net/txgbe: fix packet type to parse from offload flags ++* net/txgbe: fix Rx buffer size in config register ++* net/vhost: add missing newline in logs ++* net/vhost: fix leak in interrupt handle setup ++* net/vhost: fix Rx interrupt ++* net/virtio: deduce IP length for TSO checksum ++* net/virtio: fix empty devargs parsing ++* net/virtio: remove address width limit for modern devices ++* net/virtio-user: fix device starting failure handling ++* pdump: fix build with GCC 12 ++* raw/ifpga/base: fix init with multi-process ++* raw/skeleton: fix empty devargs parsing ++* raw/skeleton: fix selftest ++* regex/mlx5: fix doorbell record ++* regex/mlx5: utilize all available queue pairs ++* reorder: fix sequence number mbuf field register ++* reorder: invalidate buffer from ready queue in drain ++* ring: silence GCC 12 warnings ++* sched: fix alignment of structs in subport ++* table: fix action selector group size log2 setting ++* telemetry: fix repeat display when callback don't init dict ++* telemetry: move include after guard ++* test/bbdev: extend HARQ tolerance ++* test/bbdev: fix crash for non supported HARQ length ++* test/bbdev: remove check for invalid opaque data ++* test/crypto: add missing MAC-I to PDCP vectors ++* test/crypto: fix capability check for ZUC cipher-auth ++* test/crypto: fix skip condition for CPU crypto SGL ++* test/crypto: fix statistics error messages ++* test/crypto: fix typo in AES test ++* test/crypto: fix ZUC digest length in comparison ++* test: fix segment length in packet generator ++* test/mbuf: fix mbuf reset test ++* test/mbuf: fix test with mbuf debug enabled ++* test/reorder: fix double free of drained buffers ++* vdpa/ifc: fix argument compatibility check ++* vdpa/ifc: fix reconnection in SW-assisted live migration ++* version: 22.11.2-rc1 ++* vhost: decrease log level for unimplemented requests ++* vhost: fix net header settings in datapath ++* vhost: fix OOB access for invalid vhost ID ++* vhost: fix possible FD leaks ++* vhost: fix possible FD leaks on truncation ++* vhost: fix slot index in async split virtqueue Tx ++ ++22.11.2 Validation ++~~~~~~~~~~~~~~~~~~ ++ ++* Intel(R) Testing ++ ++ * Basic Intel(R) NIC testing ++ ++ * Build & CFLAG compile: cover the build test combination with latest GCC/Clang version and the popular OS revision such as Ubuntu20.04, Ubuntu22.04, Fedora35, Fedora37, RHEL8.6, RHEL8.4, FreeBSD13.1, SUSE15, CentOS7.9, openEuler22.03-SP1 etc. ++ * PF(i40e, ixgbe): test scenarios including RTE_FLOW/TSO/Jumboframe/checksum offload/VLAN/VXLAN, etc. ++ * VF(i40e, ixgbe): test scenarios including VF-RTE_FLOW/TSO/Jumboframe/checksum offload/VLAN/VXLAN, etc. ++ * PF/VF(ice): test scenarios including Switch features/Package Management/Flow Director/Advanced Tx/Advanced RSS/ACL/DCF/Flexible Descriptor, etc. ++ * Intel NIC single core/NIC performance: test scenarios including PF/VF single core performance test, etc. ++ * IPsec: test scenarios including ipsec/ipsec-gw/ipsec library basic test - QAT&SW/FIB library, etc. ++ ++ * Basic cryptodev and virtio testing ++ ++ * Virtio: both function and performance test are covered. Such as PVP/Virtio_loopback/virtio-user loopback/virtio-net VM2VM perf testing/VMAWARE ESXI 7.0u3, etc. ++ * Cryptodev: ++ ++ * Function test: test scenarios including Cryptodev API testing/CompressDev ISA-L/QAT/ZLIB PMD Testing/FIPS, etc. ++ * Performance test: test scenarios including Thoughput Performance/Cryptodev Latency, etc. ++ ++* Nvidia(R) Testing ++ ++ * Basic functionality ++ ++ * Tx/Rx, xstats, timestamps, link status, RTE flow, RSS, VLAN, checksum and TSO, ptype... ++ * link_status interrupt, l3fwd-power, multi-process. ++ * LRO, regEx, buffer split, Tx scheduling. ++ ++ * Build tests ++ ++ * Ubuntu 20.04.6 with MLNX_OFED_LINUX-5.9-0.5.6.0. ++ * Ubuntu 20.04.6 with rdma-core master (d2dbc88). ++ * Ubuntu 20.04.6 with rdma-core v28.0. ++ * Ubuntu 18.04.6 with rdma-core v17.1. ++ * Ubuntu 18.04.6 with rdma-core master (d2dbc88) (i386). ++ * Fedora 38 with rdma-core v44.0. ++ * Fedora 39 (Rawhide) with rdma-core v44.0. ++ * CentOS 7 7.9.2009 with rdma-core master (d2dbc88). ++ * CentOS 7 7.9.2009 with MLNX_OFED_LINUX-5.9-0.5.6.0. ++ * CentOS 8 8.4.2105 with rdma-core master (d2dbc88). ++ * OpenSUSE Leap 15.4 with rdma-core v38.1. ++ * Windows Server 2019 with Clang 11.0.0. ++ ++ * Test platform ++ ++ * NIC: ConnectX-5 / OS: Ubuntu 20.04 / Kernel: 6.3.0 / Driver: rdma-core v45.0 / Firmware: 16.35.2000 ++ * NIC: ConnectX-6 Dx / OS: Ubuntu 20.04 / Driver: MLNX_OFED_LINUX-5.9-0.5.6.0 / Firmware: 22.36.1010 ++ * NIC: ConnectX-7 / OS: Ubuntu 20.04 / Driver: MLNX_OFED_LINUX-5.9-0.5.6.0 / Firmware: 22.36.1010 ++ * DPU: BlueField-2 / DOCA SW version: 1.5.1 / Firmware: 24.35.2000 ++ ++22.11.2 Known Issues ++~~~~~~~~~~~~~~~~~~~~ ++ ++ ++ ++22.11.3 Release Notes ++--------------------- ++ ++ ++22.11.3 Fixes ++~~~~~~~~~~~~~ ++ ++* app/crypto-perf: fix socket ID default value ++* app/testpmd: fix checksum engine with GTP on 32-bit ++* app/testpmd: fix flow rule number parsing ++* app/testpmd: fix GTP L2 length in checksum engine ++* app/testpmd: fix meter mark handle update ++* app/testpmd: fix primary process not polling all queues ++* app/testpmd: revert primary process polling all queues fix ++* baseband/fpga_5gnr_fec: fix possible division by zero ++* baseband/fpga_5gnr_fec: fix starting unconfigured queue ++* build: fix warning when getting NUMA nodes ++* ci: fix build for Arm cross compilation in GHA ++* ci: fix libabigail cache in GHA ++* common/cnxk: fix CPT backpressure disable on LBK ++* common/cnxk: fix inline device VF identification ++* common/cnxk: fix IPsec IPv6 tunnel address byte swap ++* common/cnxk: fix receive queue with multiple mask ++* common/cnxk: fix setting channel mask for SDP interfaces ++* common/cnxk: fix uninitialized pointer read ++* common/iavf: fix MAC type for 710 NIC ++* common/idpf/base: fix control queue send and receive ++* common/idpf/base: fix ITR register definitions for AVF ++* common/idpf/base: fix memory leaks on control queue ++* common/idpf/base: fix parameters when send msg to cp ++* common/idpf: fix memory leak on AVX512 Tx queue close ++* common/idpf: remove device stop flag ++* common/mlx5: adjust fork call with new kernel API ++* common/mlx5: fix obtaining IB device in LAG mode ++* common/qat: detach crypto from compress build ++* common/qat: fix command parameter corruption ++* common/sfc_efx/base: fix Rx queue without RSS hash prefix ++* crypto/cnxk: fix IPsec CCM capabilities ++* cryptodev: clarify error codes for symmetric session ++* cryptodev: fix comments of modular operation parameters ++* cryptodev: fix device socket ID type ++* crypto/ipsec_mb: fix enqueue counter for SNOW3G ++* crypto/ipsec_mb: optimize allocation in session ++* crypto/openssl: fix memory free ++* crypto/openssl: fix memory leak in auth processing ++* crypto/openssl: skip workaround at compilation time ++* crypto/qat: fix null algorithm digest placement ++* crypto/qat: fix stack buffer overflow in SGL loop ++* crypto/qat: fix sym device prototype ++* crypto/scheduler: fix last element for valid args ++* devtools: fix bashism in mailmap check ++* devtools: fix mailmap check for parentheses ++* dma/dpaa2: set VFA bit for route-by-port with VF ++* doc: add flow template API requirements for mlx5 ++* doc: fix auth algos in cryptoperf app ++* doc: fix description of runtime directories ++* doc: fix event timer adapter guide ++* doc: fix format in flow API guide ++* doc: fix kernel patch link in hns3 guide ++* doc: fix link to flow capabilities from bnxt guide ++* doc: fix number of leading spaces in hns3 guide ++* doc: fix syntax in hns3 guide ++* doc: fix typo in cnxk platform guide ++* doc: fix typo in graph guide ++* doc: fix typos and wording in flow API guide ++* doc: improve wording of cuda guide ++* doc: remove warning with Doxygen 1.9.7 ++* doc: update BIOS settings and supported HW for NTB ++* eal: avoid calling cleanup twice ++* eal/linux: fix legacy mem init with many segments ++* eal/linux: fix secondary process crash for mp hotplug requests ++* eal/x86: improve multiple of 64 bytes memcpy performance ++* ethdev: check that at least one FEC mode is specified ++* ethdev: fix calloc arguments ++* ethdev: fix indirect action conversion ++* ethdev: fix MAC address occupies two entries ++* ethdev: fix potential leak in PCI probing helper ++* ethdev: update documentation for API to get FEC ++* ethdev: update documentation for API to set FEC ++* event/cnxk: fix mempool cookies check ++* event/cnxk: fix nanoseconds to ticks conversion ++* event/cnxk: fix setting attributes in empty get work ++* event/cnxk: fix Tx adapter data pointer ++* eventdev/timer: fix buffer flush ++* eventdev/timer: fix timeout event wait behavior ++* event/dsw: free rings on close ++* examples/fips_validation: fix digest length in AES-GCM ++* examples/fips_validation: fix external build ++* examples/ip_pipeline: fix build with GCC 13 ++* examples/ipsec-secgw: fix socket ID default value ++* examples/ipsec-secgw: fix TAP default MAC address ++* examples/ipsec-secgw: fix zero address in ethernet header ++* examples/l2fwd-cat: fix external build ++* examples/l3fwd: fix duplicate expression for default nexthop ++* examples/ntb: fix build with GCC 13 ++* fib: fix adding default route ++* hash: fix reading unaligned bits in Toeplitz hash ++* ipc: fix file descriptor leakage with unhandled messages ++* ipsec: fix NAT-T header length ++* kernel/freebsd: fix function parameter list ++* kni: fix build with Linux 6.3 ++* kni: fix build with Linux 6.5 ++* mbuf: fix Doxygen comment of distributor metadata ++* member: fix PRNG seed reset in NitroSketch mode ++* mem: fix memsegs exhausted message ++* mempool/cnxk: avoid hang when counting batch allocs ++* net/bonding: fix destroy dedicated queues flow ++* net/bonding: fix startup when NUMA is not supported ++* net/cnxk: fix cookies check with security offload ++* net/cnxk: fix flow queue index validation ++* net/cnxk: flush SQ before configuring MTU ++* net/dpaa2: fix checksum good flags ++* net/e1000: fix queue number initialization ++* net/e1000: fix Rx and Tx queue status ++* net: fix return type of IPv4 L4 packet checksum ++* net/hns3: delete duplicate macro definition ++* net/hns3: extract PTP to its own header file ++* net/hns3: fix build warning ++* net/hns3: fix device start return value ++* net/hns3: fix FEC mode check ++* net/hns3: fix FEC mode for 200G ports ++* net/hns3: fix IMP reset trigger ++* net/hns3: fix inaccurate log ++* net/hns3: fix index to look up table in NEON Rx ++* net/hns3: fix mbuf leakage when RxQ started after reset ++* net/hns3: fix mbuf leakage when RxQ started during reset ++* net/hns3: fix missing FEC capability ++* net/hns3: fix never set MAC flow control ++* net/hns3: fix non-zero weight for disabled TC ++* net/hns3: fix redundant line break in log ++* net/hns3: fix RTC time after reset ++* net/hns3: fix RTC time on initialization ++* net/hns3: fix Rx multiple firmware reset interrupts ++* net/hns3: fix uninitialized variable ++* net/hns3: fix variable type mismatch ++* net/hns3: uninitialize PTP ++* net/i40e: fix comments ++* net/i40e: fix Rx data buffer size ++* net/i40e: fix tunnel packet Tx descriptor ++* net/iavf: fix abnormal disable HW interrupt ++* net/iavf: fix protocol agnostic offloading with big packets ++* net/iavf: fix Rx data buffer size ++* net/iavf: fix stop ordering ++* net/iavf: fix tunnel TSO path selection ++* net/iavf: fix virtchnl command called in interrupt ++* net/iavf: fix VLAN insertion in vector path ++* net/iavf: fix VLAN offload with AVX512 ++* net/iavf: release large VF when closing device ++* net/ice: adjust timestamp mbuf register ++* net/ice/base: fix incorrect defines for DCBx ++* net/ice/base: remove unreachable code ++* net/ice: fix 32-bit build ++* net/ice: fix DCF control thread crash ++* net/ice: fix DCF RSS initialization ++* net/ice: fix MAC type of E822 and E823 ++* net/ice: fix outer UDP checksum offload ++* net/ice: fix protocol agnostic offloading with big packets ++* net/ice: fix RSS hash key generation ++* net/ice: fix Rx data buffer size ++* net/ice: fix statistics ++* net/ice: fix timestamp enabling ++* net/ice: fix tunnel packet Tx descriptor ++* net/ice: fix VLAN mode parser ++* net/ice: initialize parser for double VLAN ++* net/idpf: fix Rx data buffer size ++* net/igc: fix Rx and Tx queue status ++* net/ixgbe: add proper memory barriers in Rx ++* net/ixgbe: fix Rx and Tx queue status ++* net/mana: avoid unnecessary assignments in data path ++* net/mana: fix counter overflow for posted WQE ++* net/mana: fix Tx queue statistics ++* net/mana: fix WQE count for ringing RQ doorbell ++* net/mana: optimize completion queue by batch processing ++* net/mana: return probing failure if no device found ++* net/mana: use datapath logging ++* net/mlx5: enhance error log for tunnel offloading ++* net/mlx5: fix device removal event handling ++* net/mlx5: fix drop action attribute validation ++* net/mlx5: fix drop action memory leak ++* net/mlx5: fix duplicated tag index matching in SWS ++* net/mlx5: fix error in VLAN actions creation ++* net/mlx5: fix error set for age pool initialization ++* net/mlx5: fix error set in control tables create ++* net/mlx5: fix error set in Tx representor tagging ++* net/mlx5: fix flow dump for modify field ++* net/mlx5: fix flow workspace destruction ++* net/mlx5: fix handle validation for meter mark ++* net/mlx5: fix LRO TCP checksum ++* net/mlx5: fix matcher layout size calculation ++* net/mlx5: fix MPRQ stride size for headroom ++* net/mlx5: fix profile check of meter mark ++* net/mlx5: fix query for NIC flow capability ++* net/mlx5: fix return value of vport action ++* net/mlx5: fix risk in NEON Rx descriptor read ++* net/mlx5: fix RSS expansion inner buffer overflow ++* net/mlx5: fix validation for conntrack indirect action ++* net/mlx5: fix VXLAN matching with zero value ++* net/mlx5: forbid duplicated tag index in pattern template ++* net/mlx5: forbid MPRQ restart ++* net/mlx5: reduce counter pool name length ++* net/netvsc: fix sizeof calculation ++* net/nfp: fix address always related with PF ID 0 ++* net/nfp: fix control mempool creation ++* net/nfp: fix disabling promiscuous mode ++* net/nfp: fix endian conversion for tunnel decap action ++* net/nfp: fix flow hash table creation ++* net/nfp: fix IPv6 address for set flow action ++* net/nfp: fix IPv6 flow item ++* net/nfp: fix offloading flows ++* net/nfp: fix representor creation ++* net/nfp: fix representor name too long ++* net/nfp: fix TOS of IPv6 GENEVE encap flow action ++* net/nfp: fix TOS of IPv6 NVGRE encap flow action ++* net/nfp: fix TOS of IPv6 VXLAN encap flow action ++* net/nfp: fix TP flow action for UDP ++* net/nfp: fix Tx descriptor free logic of NFD3 ++* net/nfp: fix unneeded endian conversion ++* net/nfp: fix VLAN push flow action ++* net/nfp: fix VNI of IPv4 NVGRE encap action ++* net/nfp: fix VNI of IPv6 NVGRE encap action ++* net/nfp: fix VNI of VXLAN encap action ++* net/ngbe: adapt to MNG veto bit setting ++* net/ngbe: fix extended statistics ++* net/ngbe: fix link status in no LSC mode ++* net/ngbe: fix RSS offload capability ++* net/ngbe: remove redundant codes ++* net/qede: fix RSS indirection table initialization ++* net/sfc: invalidate dangling MAE flow action FW resource IDs ++* net/sfc: stop misuse of Rx ingress m-port metadata on EF100 ++* net/tap: set locally administered bit for fixed MAC address ++* net/txgbe: adapt to MNG veto bit setting ++* net/txgbe/base: fix Tx with fiber hotplug ++* net/txgbe: fix blocking system events ++* net/txgbe: fix extended statistics ++* net/txgbe: fix interrupt enable mask ++* net/txgbe: fix to set autoneg for 1G speed ++* net/txgbe: fix use-after-free on remove ++* net/virtio: fix initialization to return negative errno ++* net/virtio: propagate interrupt configuration error values ++* net/virtio-user: fix leak when initialisation fails ++* net/vmxnet3: fix drop of empty segments in Tx ++* net/vmxnet3: fix return code in initializing ++* pci: fix comment referencing renamed function ++* pipeline: fix double free for table stats ++* raw/ntb: avoid disabling interrupt twice ++* Revert "net/iavf: fix tunnel TSO path selection" ++* ring: fix dequeue parameter name ++* ring: fix use after free ++* telemetry: fix autotest on Alpine ++* test: add graph tests ++* test/bonding: fix include of standard header ++* test/crypto: fix IPsec AES CCM vector ++* test/crypto: fix PDCP-SDAP test vectors ++* test/crypto: fix return value for SNOW3G ++* test/crypto: fix session creation check ++* test/malloc: fix missing free ++* test/malloc: fix statistics checks ++* test/mbuf: fix crash in a forked process ++* test/security: fix event inline IPsec reassembly tests ++* version: 22.11.3-rc1 ++* vfio: fix include with musl runtime ++* vhost: fix invalid call FD handling ++* vhost: fix notification stats for packed ring ++ ++22.11.3 Validation ++~~~~~~~~~~~~~~~~~~ ++ ++* Intel(R) Testing ++ ++ * Basic Intel(R) NIC testing ++ ++ * Build & CFLAG compile: cover the build test combination with latest GCC/Clang version and the popular OS revision such as Ubuntu20.04, Ubuntu22.04, Fedora38, RHEL8.7, RHEL9.2, FreeBSD13.1, SUSE15, CentOS7.9, openEuler22.03-SP1，OpenAnolis8.8 etc. ++ * PF(i40e, ixgbe): test scenarios including RTE_FLOW/TSO/Jumboframe/checksum offload/VLAN/VXLAN, etc. ++ * VF(i40e, ixgbe): test scenarios including VF-RTE_FLOW/TSO/Jumboframe/checksum offload/VLAN/VXLAN, etc. ++ * PF/VF(ice): test scenarios including Switch features/Package Management/Flow Director/Advanced Tx/Advanced RSS/ACL/DCF/Flexible Descriptor, etc. ++ * Intel NIC single core/NIC performance: test scenarios including PF/VF single core performance test, etc. ++ * IPsec: test scenarios including ipsec/ipsec-gw/ipsec library basic test - QAT&SW/FIB library, etc. ++ ++ * Basic cryptodev and virtio testing ++ ++ * Virtio: both function and performance test are covered. Such as PVP/Virtio_loopback/virtio-user loopback/virtio-net VM2VM perf testing/VMAWARE ESXI 8.0, etc. ++ * Cryptodev: ++ ++ * Function test: test scenarios including Cryptodev API testing/CompressDev ISA-L/QAT/ZLIB PMD Testing/FIPS, etc. ++ * Performance test: test scenarios including Thoughput Performance/Cryptodev Latency, etc. ++ ++* Nvidia(R) Testing ++ ++ * Basic functionality ++ ++ * Tx/Rx, xstats, timestamps, link status, RTE flow, RSS, VLAN, checksum and TSO, ptype... ++ * link_status interrupt, l3fwd-power, multi-process. ++ * LRO, regEx, buffer split, Tx scheduling. ++ ++ * Build tests ++ ++ * Ubuntu 20.04.6 with MLNX_OFED_LINUX-23.04-1.1.3.0. ++ * Ubuntu 20.04.6 with rdma-core master (4cce53f). ++ * Ubuntu 20.04.6 with rdma-core v28.0. ++ * Ubuntu 18.04.6 with rdma-core master (4cce53f) (i386). ++ * Fedora 38 with rdma-core v44.0. ++ * Fedora 39 (Rawhide) with rdma-core v46.0. ++ * OpenSUSE Leap 15.5 with rdma-core v42.0. ++ * Windows Server 2019 with Clang 11.0.0. ++ ++ * Test platform ++ ++ * NIC: ConnectX-6 Dx / OS: Ubuntu 20.04 / Driver: MLNX_OFED_LINUX-23.07-0.5.0.0 / Firmware: 22.38.1002 ++ * NIC: ConnectX-7 / OS: Ubuntu 20.04 / Driver: MLNX_OFED_LINUX-23.07-0.5.0.0 / Firmware: 28.38.1002 ++ * DPU: BlueField-2 / DOCA SW version: 2.2.0 / Firmware: 24.38.1002 ++ ++* Redhat Testing ++ ++ * Test scenarios ++ ++ * Guest with device assignment(PF) throughput testing(1G hugepage size) ++ * Guest with device assignment(PF) throughput testing(2M hugepage size) ++ * Guest with device assignment(VF) throughput testing ++ * PVP (host dpdk testpmd as vswitch) 1Q: throughput testing ++ * PVP vhost-user 2Q throughput testing ++ * PVP vhost-user 1Q - cross numa node throughput testing ++ * Guest with vhost-user 2 queues throughput testing ++ * vhost-user reconnect with dpdk-client, qemu-server qemu reconnect ++ * vhost-user reconnect with dpdk-client, qemu-server ovs reconnect ++ * PVP reconnect with dpdk-client, qemu-server ++ * PVP 1Q live migration testing ++ * PVP 1Q cross numa node live migration testing ++ * Guest with ovs+dpdk+vhost-user 1Q live migration testing ++ * Guest with ovs+dpdk+vhost-user 1Q live migration testing (2M) ++ * Guest with ovs+dpdk+vhost-user 2Q live migration testing ++ * Guest with ovs+dpdk+vhost-user 4Q live migration testing ++ * Host PF + DPDK testing ++ * Host VF + DPDK testing ++ ++ * Version Information: ++ ++ * RHEL9 ++ * qemu-kvm-6.2.0 + qemu-kvm-7.2.0 ++ * kernel 5.14 ++ * X540-AT2 NIC(ixgbe, 10G) ++ ++22.11.3 Known Issues ++~~~~~~~~~~~~~~~~~~~~ ++ ++* Intel(R) Testing ++ ++ * Cryptodev: Performance drop for 1c1t scenario ++ ++22.11.4 Release Notes ++--------------------- ++ ++ ++22.11.4 Fixes ++~~~~~~~~~~~~~ ++ ++* app/bbdev: fix link with NXP LA12XX ++* app/dumpcap: allow multiple invocations ++* app/dumpcap: fix mbuf pool ring type ++* app/pipeline: add sigint handler ++* app/procinfo: adjust format of RSS info ++* app/procinfo: fix RSS info ++* app/procinfo: remove unnecessary rte_malloc ++* app/test: disable graph auto test for windows ++* app/test: fix reference to master in bonding test ++* app/testpmd: add explicit check for tunnel TSO ++* app/testpmd: fix early exit from signal ++* app/testpmd: fix help string ++* app/testpmd: fix primary process not polling all queues ++* app/testpmd: fix tunnel TSO capability check ++* app/testpmd: fix tunnel TSO configuration ++* app/testpmd: remove useless check in TSO command ++* baseband/acc: fix ACC100 HARQ input alignment ++* baseband/acc: fix TB mode on VRB1 ++* build: add libarchive to optional external dependencies ++* bus/dpaa: fix build with asserts for GCC 13 ++* bus/ifpga: fix driver header dependency ++* bus/pci: add PASID control ++* bus/pci: fix device ID log ++* ci: fix race on container image name ++* common/cnxk: fix aura disable handling ++* common/cnxk: fix default flow action setting ++* common/cnxk: fix different size bit operations ++* common/cnxk: fix DPI memzone name ++* common/cnxk: fix incorrect aura ID ++* common/cnxk: fix pool buffer size in opaque mode ++* common/cnxk: fix RSS key configuration ++* common/cnxk: fix SDP channel mask ++* common/cnxk: fix xstats for different packet sizes ++* common/cnxk: remove dead Meson code ++* common/cnxk: replace direct API usage in REE ++* common/mlx5: fix controller index parsing ++* common/mlx5: replace use of PMD log type ++* config/arm: fix aarch32 build with GCC 13 ++* config: fix RISC-V native build ++* crypto/cnxk: fix IPsec CCM and GCM capabilities ++* cryptodev: add missing doc for security context ++* crypto/dpaa2_sec: fix debug prints ++* crypto/dpaa_sec: fix debug prints ++* crypto/ipsec_mb: add dependency check for cross build ++* crypto/nitrox: fix panic with high number of segments ++* crypto/openssl: fix memory leaks in asym session ++* crypto/qat: fix raw API null algorithm digest ++* dma/cnxk: fix chunk buffer failure return code ++* dma/cnxk: fix device reconfigure ++* dma/cnxk: fix device state ++* doc: fix hns3 build option about max queue number ++* doc: fix RSS flow description in hns3 guide ++* doc: fix some ordered lists ++* doc: remove number of commands in vDPA guide ++* doc: remove restriction on ixgbe vector support ++* doc: replace code blocks with includes in security guide ++* doc: update features in hns3 guide ++* doc: update kernel module entry in QAT guide ++* doc: update versions recommendations for i40e and ice ++* eal/riscv: fix vector type alignment ++* eal/unix: fix firmware reading with external xz helper ++* eal/windows: fix build with recent MinGW ++* ethdev: account for smaller MTU when setting default ++* ethdev: add check in async flow action query ++* ethdev: fix 32-bit build with GCC 13 ++* ethdev: fix ESP packet type description ++* ethdev: fix function name in comment ++* event/cnxk: fix CASP usage for clang ++* event/cnxk: fix context flush in port cleanup ++* event/cnxk: fix getwork mode devargs parsing ++* event/cnxk: fix return values for capability API ++* eventdev/crypto: fix circular buffer full case ++* eventdev/eth_rx: fix timestamp field register in mbuf ++* eventdev: fix alignment padding ++* eventdev: fix device pointer for vdev-based devices ++* eventdev: fix missing driver names in info struct ++* eventdev: fix symbol export for port maintenance ++* event/dlb2: disable PASID ++* event/dlb2: fix disable PASID ++* event/dlb2: fix missing queue ordering capability flag ++* event/dlb2: fix name check in self-test ++* event/sw: fix ordering corruption with op release ++* event/sw: remove obsolete comment ++* examples/ethtool: fix pause configuration ++* examples/ipsec-secgw: fix partial overflow ++* fib6: fix adding default route as first route ++* fib: fix adding default route overwriting entire table ++* gpu/cuda: fix build with external GDRCopy ++* hash: align SSE lookup to scalar implementation ++* malloc: remove return from void functions ++* mem: fix deadlock with multiprocess ++* mempool: clarify enqueue/dequeue ops documentation ++* mempool/cnxk: fix alloc from non-EAL threads ++* mempool/cnxk: fix free from non-EAL threads ++* mempool: fix default ops for an empty mempool ++* mempool: fix get function documentation ++* meter: fix RFC4115 trTCM API Doxygen ++* net/af_packet: fix Rx and Tx queue state ++* net/af_xdp: fix Rx and Tx queue state ++* net/ark: support single function with multiple port ++* net/avp: fix Rx and Tx queue state ++* net/axgbe: identify CPU with cpuid ++* net/bnx2x: fix Rx and Tx queue state ++* net/bnxt: fix Rx and Tx queue state ++* net/bonding: fix header for C++ ++* net/bonding: fix link status callback stop ++* net/bonding: fix possible overrun ++* net/bonding: fix Rx and Tx queue state ++* net/cnxk: fix data offset in vector Tx ++* net/cnxk: fix uninitialized variable ++* net/cxgbe: fix Rx and Tx queue state ++* net/dpaa2: fix Rx and Tx queue state ++* net/dpaa: fix Rx and Tx queue state ++* net/e1000: fix Rx and Tx queue state ++* net/ena: fix Rx and Tx queue state ++* net/enetc: fix Rx and Tx queue state ++* net/enic: avoid extra unlock in MTU set ++* net/enic: fix Rx and Tx queue state ++* net/gve: fix max MTU limit ++* net/gve: fix RX buffer size alignment ++* net/gve: update max Rx packet length to be based on MTU ++* net/hinic: fix Rx and Tx queue state ++* net/hns3: fix double stats for IMP and global reset ++* net/hns3: fix error code for multicast resource ++* net/hns3: fix flushing multicast MAC address ++* net/hns3: fix ignored reset event ++* net/hns3: fix IMP or global reset ++* net/hns3: fix LRO offload to report ++* net/hns3: fix mailbox sync ++* net/hns3: fix multiple reset detected log ++* net/hns3: fix order in NEON Rx ++* net/hns3: fix reset event status ++* net/hns3: fix setting DCB capability ++* net/hns3: fix some error logs ++* net/hns3: fix some return values ++* net/hns3: fix traffic management dump text alignment ++* net/hns3: fix traffic management thread safety ++* net/hns3: fix typo in function name ++* net/hns3: fix unchecked Rx free threshold ++* net/hns3: fix uninitialized hash algo value ++* net/hns3: fix VF default MAC modified when set failed ++* net/hns3: fix VF reset handler interruption ++* net/hns3: keep set/get algo key functions local ++* net/hns3: refactor interrupt state query ++* net/hns3: remove reset log in secondary ++* net/i40e: fix buffer leak on Rx reconfiguration ++* net/i40e: fix FDIR queue receives broadcast packets ++* net/iavf: fix checksum offloading ++* net/iavf: fix ESN session update ++* net/iavf: fix indent in Tx path ++* net/iavf: fix port stats clearing ++* net/iavf: fix TSO with big segments ++* net/iavf: fix Tx debug ++* net/iavf: fix Tx offload flags check ++* net/iavf: fix Tx offload mask ++* net/iavf: fix Tx preparation ++* net/iavf: fix VLAN offload strip flag ++* net/iavf: remove log from Tx prepare function ++* net/iavf: unregister interrupt handler before FD close ++* net/ice: fix crash on closing representor ports ++* net/ice: fix DCF port statistics ++* net/ice: fix initial link status ++* net/ice: fix L1 check interval ++* net/ice: fix TM configuration clearing ++* net/ice: fix TSO with big segments ++* net/ice: fix Tx preparation ++* net/ice: remove log from Tx prepare function ++* net/ice: write timestamp to first segment in scattered Rx ++* net/ipn3ke: fix Rx and Tx queue state ++* net/mana: add 32-bit short doorbell ++* net/mana: add missing new line to data path logs ++* net/mana: enable 32-bit build ++* net/memif: fix Rx and Tx queue state ++* net/mlx4: fix Rx and Tx queue state ++* net/mlx5: fix counter query during port close ++* net/mlx5: fix decap action checking in sample flow ++* net/mlx5: fix destroying external representor flow ++* net/mlx5: fix E-Switch mirror flow rule validation ++* net/mlx5: fix flow thread safety flag for HWS ++* net/mlx5: fix flow workspace double free in Windows ++* net/mlx5: fix hairpin queue states ++* net/mlx5: fix hairpin queue unbind ++* net/mlx5: fix jump ipool entry size ++* net/mlx5: fix LACP redirection in Rx domain ++* net/mlx5: fix leak in sysfs port name translation ++* net/mlx5: fix missing flow rules for external SQ ++* net/mlx5: fix MPRQ stride size check ++* net/mlx5: fix multi-segment Tx inline data length ++* net/mlx5: fix NIC flow capability query ++* net/mlx5: fix offset size in conntrack flow action ++* net/mlx5: fix shared Rx queue list management ++* net/mlx5: fix unlock mismatch ++* net/mlx5: fix use after free on Rx queue start ++* net/mlx5: fix validation of sample encap flow action ++* net/mlx5/hws: fix field copy bind ++* net/mlx5/hws: fix integrity bits level ++* net/mlx5: zero UDP checksum over IPv4 in encapsulation ++* net/mvneta: fix Rx and Tx queue state ++* net/mvpp2: fix Rx and Tx queue state ++* net/netvsc: increase VSP response timeout to 60 seconds ++* net/nfp: fix control message packets ++* net/nfp: fix crash on close ++* net/nfp: fix DMA error after abnormal exit ++* net/nfp: fix initialization of physical representors ++* net/nfp: fix link status interrupt ++* net/nfp: fix reconfigure logic in PF initialization ++* net/nfp: fix reconfigure logic in VF initialization ++* net/nfp: fix reconfigure logic of set MAC address ++* net/nfp: fix Rx and Tx queue state ++* net/ngbe: add proper memory barriers in Rx ++* net/ngbe: check process type in close operation ++* net/ngbe: fix flow control ++* net/ngbe: fix Rx and Tx queue state ++* net/ngbe: keep link down after device close ++* net/ngbe: prevent NIC from slowing down link speed ++* net/ngbe: reconfigure MAC Rx when link update ++* net/null: fix Rx and Tx queue state ++* net/octeon_ep: fix Rx and Tx queue state ++* net/octeontx: fix Rx and Tx queue state ++* net/pfe: fix Rx and Tx queue state ++* net/ring: fix Rx and Tx queue state ++* net/sfc: account for data offset on Tx ++* net/sfc: add missing error code indication to MAE init path ++* net/sfc: fix Rx and Tx queue state ++* net/sfc: remove null dereference in log ++* net/sfc: set max Rx packet length for representors ++* net/softnic: fix Rx and Tx queue state ++* net/tap: fix IPv4 checksum offloading ++* net/tap: fix L4 checksum offloading ++* net/tap: fix RSS for fragmented packets ++* net/tap: use MAC address parse API instead of local parser ++* net/txgbe: add proper memory barriers in Rx ++* net/txgbe: add Tx queue maximum limit ++* net/txgbe: check process type in close operation ++* net/txgbe: fix GRE tunnel packet checksum ++* net/txgbe: fix out of bound access ++* net/txgbe: fix Rx and Tx queue state ++* net/txgbe: keep link down after device close ++* net/txgbe: reconfigure MAC Rx when link update ++* net/vhost: fix Rx and Tx queue state ++* net/virtio: fix link state interrupt vector setting ++* net/virtio: fix missing next flag in Tx packed ring ++* net/virtio: fix Rx and Tx queue state ++* net/vmxnet3: fix Rx and Tx queue state ++* pdump: fix error number on IPC response ++* random: initialize state for unregistered non-EAL threads ++* rawdev: fix device class in log message ++* Revert "eventdev: fix alignment padding" ++* Revert "net/iavf: fix abnormal disable HW interrupt" ++* test/bbdev: assert failed test for queue configure ++* test/bbdev: fix Python script subprocess ++* test/bonding: add missing check ++* test/bonding: fix uninitialized RSS configuration ++* test/bonding: remove unreachable statement ++* test/crypto: fix IV in some vectors ++* test/crypto: fix return value for GMAC case ++* test/crypto: fix typo in asym tests ++* test/crypto: skip some synchronous tests with CPU crypto ++* test/event: fix crypto null device creation ++* test: fix named test macro ++* test/hash: fix creation error log ++* test/security: fix IPv6 next header field ++* usertools/pmdinfo: fix usage typos ++* vdpa/mlx5: fix unregister kick handler order ++* vhost: fix checking virtqueue access in stats API ++* vhost: fix check on virtqueue access in async registration ++* vhost: fix check on virtqueue access in in-flight getter ++* vhost: fix missing check on virtqueue access ++* vhost: fix missing lock protection in power monitor API ++* vhost: fix missing spinlock unlock ++* vhost: fix missing vring call check on virtqueue access ++ ++22.11.4 Validation ++~~~~~~~~~~~~~~~~~~ ++ ++* Redhat Testing ++ ++ * Test scenarios ++ ++ * Guest with device assignment(PF) throughput testing(1G hugepage size) ++ * Guest with device assignment(PF) throughput testing(2M hugepage size) ++ * Guest with device assignment(VF) throughput testing ++ * PVP (host dpdk testpmd as vswitch) 1Q: throughput testing ++ * PVP vhost-user 4Q throughput testing ++ * PVP vhost-user 2Q throughput testing ++ * PVP vhost-user 1Q - cross numa node throughput testing ++ * Guest with vhost-user 2 queues throughput testing ++ * vhost-user reconnect with dpdk-client, qemu-server qemu reconnect ++ * vhost-user reconnect with dpdk-client, qemu-server ovs reconnect ++ * PVP 1Q live migration testing ++ * PVP 1Q cross numa node live migration testing ++ * Guest with ovs+dpdk+vhost-user 1Q live migration testing ++ * Guest with ovs+dpdk+vhost-user 1Q live migration testing (2M) ++ * Guest with ovs+dpdk+vhost-user 2Q live migration testing ++ * Guest with ovs+dpdk+vhost-user 4Q live migration testing ++ * Host PF + DPDK testing ++ * Host VF + DPDK testing ++ ++ * Test Versions and device ++ ++ * qemu-kvm-7.2.0 ++ * kernel 5.14 ++ * X540-AT2 NIC(ixgbe, 10G) ++ ++* Nvidia(R) Testing ++ ++ * Basic functionality: ++ ++ * Send and receive multiple types of traffic. ++ * testpmd xstats counter test. ++ * testpmd timestamp test. ++ * Changing/checking link status through testpmd. ++ * rte_flow tests (https://doc.dpdk.org/guides/nics/mlx5.html#supported-hardware-offloads) ++ * RSS tests. ++ * VLAN filtering, stripping, and insertion tests. ++ * Checksum and TSO tests. ++ * ptype tests. ++ * link_status_interrupt example application tests. ++ * l3fwd-power example application tests. ++ * Multi-process example applications tests. ++ * Hardware LRO tests. ++ * Regex application tests. ++ * Buffer Split tests. ++ * Tx scheduling tests. ++ ++ * Test platform ++ ++ * NIC: ConnectX-6 Dx / OS: Ubuntu 20.04 / Driver: MLNX_OFED_LINUX-23.10-1.1.9.0 / Firmware: 22.39.2048 ++ * NIC: ConnectX-7 / OS: Ubuntu 20.04 / Driver: MLNX_OFED_LINUX-23.10-1.1.9.0 / Firmware: 28.39.2048 ++ * DPU: BlueField-2 / DOCA SW version: 2.5.0 / Firmware: 24.39.2048 ++ ++ * OS/driver: ++ ++ * Ubuntu 20.04.6 with MLNX_OFED_LINUX-23.10-1.1.9.0. ++ * Ubuntu 20.04.6 with rdma-core master (9016f34). ++ * Ubuntu 20.04.6 with rdma-core v28.0. ++ * Fedora 38 with rdma-core v44.0. ++ * Fedora 40 (Rawhide) with rdma-core v48.0. ++ * OpenSUSE Leap 15.5 with rdma-core v42.0. ++ * Windows Server 2019 with Clang 16.0.6. ++ ++* Intel(R) Testing ++ ++ * Basic Intel(R) NIC testing ++ ++ * Build & CFLAG compile: cover the build test combination with latest GCC/Clang version and the popular OS revision such as Ubuntu20.04, Ubuntu22.04, Fedora38, RHEL8.7, RHEL9.2, FreeBSD13.2, SUSE15, CentOS7.9, openEuler22.03-SP1，OpenAnolis8.8 etc. ++ * PF(i40e, ixgbe): test scenarios including RTE_FLOW/TSO/Jumboframe/checksum offload/VLAN/VXLAN, etc. ++ * VF(i40e, ixgbe): test scenarios including VF-RTE_FLOW/TSO/Jumboframe/checksum offload/VLAN/VXLAN, etc. ++ * PF/VF(ice): test scenarios including Switch features/Package Management/Flow Director/Advanced Tx/Advanced RSS/ACL/DCF/Flexible Descriptor, etc. ++ * Intel NIC single core/NIC performance: test scenarios including PF/VF single core performance test, etc. ++ * IPsec: test scenarios including ipsec/ipsec-gw/ipsec library basic test - QAT&SW/FIB library, etc. ++ ++ * Basic cryptodev and virtio testing ++ ++ * Virtio: both function and performance test are covered. Such as PVP/Virtio_loopback/virtio-user loopback/virtio-net VM2VM perf testing/VMAWARE ESXI 8.0, etc. ++ * Cryptodev: ++ ++ * Function test: test scenarios including Cryptodev API testing/CompressDev ISA-L/QAT/ZLIB PMD Testing/FIPS, etc. ++ * Performance test: test scenarios including Thoughput Performance/Cryptodev Latency, etc. ++ ++22.11.4 Known Issues ++~~~~~~~~~~~~~~~~~~~~ ++ ++ +diff --git a/dpdk/doc/guides/sample_app_ug/l2_forward_cat.rst b/dpdk/doc/guides/sample_app_ug/l2_forward_cat.rst +index 3ada3575ba..51621b692f 100644 +--- a/dpdk/doc/guides/sample_app_ug/l2_forward_cat.rst ++++ b/dpdk/doc/guides/sample_app_ug/l2_forward_cat.rst +@@ -50,13 +50,12 @@ Compiling the Application + * https://github.com/01org/intel-cmt-cat + + +-#. To compile the application export the path to PQoS lib +- and the DPDK source tree and go to the example directory: ++To compile the application, export the path to PQoS lib: + +- .. code-block:: console +- +- export PQOS_INSTALL_PATH=/path/to/libpqos ++.. code-block:: console + ++ export CFLAGS=-I/path/to/intel-cmt-cat/include ++ export LDFLAGS=-L/path/to/intel-cmt-cat/lib + + To compile the sample application see :doc:`compiling`. + +diff --git a/dpdk/doc/guides/sample_app_ug/l3_forward.rst b/dpdk/doc/guides/sample_app_ug/l3_forward.rst +index 94b22da01e..1cc2c1dd1d 100644 +--- a/dpdk/doc/guides/sample_app_ug/l3_forward.rst ++++ b/dpdk/doc/guides/sample_app_ug/l3_forward.rst +@@ -56,9 +56,8 @@ for the IPv4/IPv6 5-tuple syntax specifically. + The 5-tuple syntax consists of a source IP address, a destination IP address, + a source port, a destination port and a protocol identifier. + +-In the sample application, hash-based, FIB-based and ACL-based forwarding supports ++In the sample application, hash-based, LPM-based, FIB-based and ACL-based forwarding supports + both IPv4 and IPv6. +-LPM-based forwarding supports IPv4 only. + During the initialization phase route rules for IPv4 and IPv6 are read from rule files. + + Compiling the Application +diff --git a/dpdk/doc/guides/sample_app_ug/pipeline.rst b/dpdk/doc/guides/sample_app_ug/pipeline.rst +index 49d50136bc..7c86bf484a 100644 +--- a/dpdk/doc/guides/sample_app_ug/pipeline.rst ++++ b/dpdk/doc/guides/sample_app_ug/pipeline.rst +@@ -58,7 +58,7 @@ The following is an example command to run the application configured for the VX + + .. code-block:: console + +- $ .//examples/dpdk-pipeline -c 0x3 -- -s examples/vxlan.cli ++ $ .//examples/dpdk-pipeline -c 0x3 -- -s examples/pipeline/examples/vxlan.cli + + The application should start successfully and display as follows: + +diff --git a/dpdk/doc/guides/sample_app_ug/vdpa.rst b/dpdk/doc/guides/sample_app_ug/vdpa.rst +index cb9c4f2169..51e69fc20d 100644 +--- a/dpdk/doc/guides/sample_app_ug/vdpa.rst ++++ b/dpdk/doc/guides/sample_app_ug/vdpa.rst +@@ -38,8 +38,7 @@ where + * --iface specifies the path prefix of the UNIX domain socket file, e.g. + /tmp/vhost-user-, then the socket files will be named as /tmp/vhost-user- + (n starts from 0). +-* --interactive means run the vdpa sample in interactive mode, currently 4 +- internal cmds are supported: ++* --interactive means run the vDPA sample in interactive mode: + + 1. help: show help message + 2. list: list all available vdpa devices +diff --git a/dpdk/doc/guides/tools/cryptoperf.rst b/dpdk/doc/guides/tools/cryptoperf.rst +index c77e253417..f30784674d 100644 +--- a/dpdk/doc/guides/tools/cryptoperf.rst ++++ b/dpdk/doc/guides/tools/cryptoperf.rst +@@ -232,7 +232,6 @@ The following are the application command-line options: + Set authentication algorithm name, where ``name`` is one + of the following:: + +- 3des-cbc + aes-cbc-mac + aes-cmac + aes-gmac +diff --git a/dpdk/doc/guides/tools/pmdinfo.rst b/dpdk/doc/guides/tools/pmdinfo.rst +index a9217de4ee..fdb9030171 100644 +--- a/dpdk/doc/guides/tools/pmdinfo.rst ++++ b/dpdk/doc/guides/tools/pmdinfo.rst +@@ -82,5 +82,5 @@ Get only the required kernel modules for a given device: + .. code-block:: console + + $ dpdk-pmdinfo.py /usr/bin/dpdk-testpmd | \ +- jq '.[] | select(.pci_ids[] | .vendor == "15b3" and .device == "1013").kmod' ++ jq '.[] | select(.pci_ids[]? | .vendor == "15b3" and .device == "1013").kmod' + "* ib_uverbs & mlx5_core & mlx5_ib" +diff --git a/dpdk/drivers/baseband/acc/rte_acc100_pmd.c b/dpdk/drivers/baseband/acc/rte_acc100_pmd.c +index ba8247d47e..56a11e98b8 100644 +--- a/dpdk/drivers/baseband/acc/rte_acc100_pmd.c ++++ b/dpdk/drivers/baseband/acc/rte_acc100_pmd.c +@@ -622,6 +622,7 @@ acc100_dev_close(struct rte_bbdev *dev) + rte_free(d->tail_ptrs); + rte_free(d->info_ring); + rte_free(d->sw_rings_base); ++ rte_free(d->harq_layout); + d->sw_rings_base = NULL; + d->tail_ptrs = NULL; + d->info_ring = NULL; +@@ -663,7 +664,7 @@ acc100_find_free_queue_idx(struct rte_bbdev *dev, + for (aq_idx = 0; aq_idx < qtop->num_aqs_per_groups; aq_idx++) { + if (((d->q_assigned_bit_map[group_idx] >> aq_idx) & 0x1) == 0) { + /* Mark the Queue as assigned */ +- d->q_assigned_bit_map[group_idx] |= (1 << aq_idx); ++ d->q_assigned_bit_map[group_idx] |= (1ULL << aq_idx); + /* Report the AQ Index */ + return (group_idx << ACC100_GRP_ID_SHIFT) + aq_idx; + } +@@ -1220,7 +1221,7 @@ acc100_fcw_ld_fill(struct rte_bbdev_dec_op *op, struct acc_fcw_ld *fcw, + - op->ldpc_dec.n_filler); + + /* Alignment on next 64B - Already enforced from HC output */ +- harq_in_length = RTE_ALIGN_FLOOR(harq_in_length, ACC_HARQ_ALIGN_64B); ++ harq_in_length = RTE_ALIGN_CEIL(harq_in_length, ACC_HARQ_ALIGN_64B); + + /* Stronger alignment requirement when in decompression mode */ + if (fcw->hcin_decomp_mode > 0) +@@ -3422,9 +3423,9 @@ acc100_enqueue_ldpc_enc_cb(struct rte_bbdev_queue_data *q_data, + } + avail--; + enq = RTE_MIN(left, ACC_MUX_5GDL_DESC); +- if (check_mux(&ops[i], enq)) { +- ret = enqueue_ldpc_enc_n_op_cb(q, &ops[i], +- desc_idx, enq); ++ enq = check_mux(&ops[i], enq); ++ if (enq > 1) { ++ ret = enqueue_ldpc_enc_n_op_cb(q, &ops[i], desc_idx, enq); + if (ret < 0) { + acc_enqueue_invalid(q_data); + break; +@@ -4034,8 +4035,12 @@ dequeue_dec_one_op_tb(struct acc_queue *q, struct rte_bbdev_dec_op **ref_op, + /* CRC invalid if error exists */ + if (!op->status) + op->status |= rsp.crc_status << RTE_BBDEV_CRC_ERROR; +- op->turbo_dec.iter_count = RTE_MAX((uint8_t) rsp.iter_cnt, +- op->turbo_dec.iter_count); ++ if (q->op_type == RTE_BBDEV_OP_LDPC_DEC) ++ op->ldpc_dec.iter_count = RTE_MAX((uint8_t) rsp.iter_cnt, ++ op->ldpc_dec.iter_count); ++ else ++ op->turbo_dec.iter_count = RTE_MAX((uint8_t) rsp.iter_cnt, ++ op->turbo_dec.iter_count); + + /* Check if this is the last desc in batch (Atomic Queue) */ + if (desc->req.last_desc_in_batch) { +@@ -4119,8 +4124,6 @@ acc100_dequeue_ldpc_enc(struct rte_bbdev_queue_data *q_data, + struct rte_bbdev_enc_op *op; + union acc_dma_desc *desc; + +- if (q == NULL) +- return 0; + #ifdef RTE_LIBRTE_BBDEV_DEBUG + if (unlikely(ops == 0)) + return 0; +diff --git a/dpdk/drivers/baseband/acc/rte_acc200_pmd.c b/dpdk/drivers/baseband/acc/rte_acc200_pmd.c +index c5123cfef0..4fc078fe26 100644 +--- a/dpdk/drivers/baseband/acc/rte_acc200_pmd.c ++++ b/dpdk/drivers/baseband/acc/rte_acc200_pmd.c +@@ -1848,6 +1848,9 @@ enqueue_enc_one_op_tb(struct acc_queue *q, struct rte_bbdev_enc_op *op, + r = op->turbo_enc.tb_params.r; + + while (mbuf_total_left > 0 && r < c) { ++ if (unlikely((input == NULL) || (output == NULL))) ++ return -1; ++ + seg_total_left = rte_pktmbuf_data_len(input) - in_offset; + /* Set up DMA descriptor */ + desc = acc_desc(q, total_enqueued_cbs); +@@ -1882,6 +1885,10 @@ enqueue_enc_one_op_tb(struct acc_queue *q, struct rte_bbdev_enc_op *op, + r++; + } + ++ /* In case the number of CB doesn't match, the configuration was invalid. */ ++ if (unlikely(current_enqueued_cbs != cbs_in_tb)) ++ return -1; ++ + /* Set SDone on last CB descriptor for TB mode. */ + desc->req.sdone_enable = 1; + +@@ -1903,7 +1910,8 @@ enqueue_ldpc_enc_one_op_tb(struct acc_queue *q, struct rte_bbdev_enc_op *op, + uint16_t init_enq_descs = enq_descs; + uint32_t in_offset = 0, out_offset = 0; + +- input_len_B = ((op->ldpc_enc.basegraph == 1 ? 22 : 10) * op->ldpc_enc.z_c) >> 3; ++ input_len_B = ((op->ldpc_enc.basegraph == 1 ? 22 : 10) * op->ldpc_enc.z_c ++ - op->ldpc_enc.n_filler) >> 3; + + if (check_bit(op->ldpc_enc.op_flags, RTE_BBDEV_LDPC_CRC_24B_ATTACH)) + input_len_B -= 3; +@@ -2079,6 +2087,10 @@ enqueue_ldpc_dec_one_op_cb(struct acc_queue *q, struct rte_bbdev_dec_op *op, + } + } + ++ if (op->ldpc_dec.soft_output.length > 0) ++ mbuf_append(op->ldpc_dec.soft_output.data, op->ldpc_dec.soft_output.data, ++ op->ldpc_dec.soft_output.length); ++ + #ifdef RTE_LIBRTE_BBDEV_DEBUG + rte_memdump(stderr, "FCW", &desc->req.fcw_ld, + sizeof(desc->req.fcw_ld) - 8); +@@ -2128,6 +2140,9 @@ enqueue_ldpc_dec_one_op_tb(struct acc_queue *q, struct rte_bbdev_dec_op *op, + } + + while (mbuf_total_left > 0 && r < c) { ++ if (unlikely((input == NULL) || (h_output == NULL))) ++ return -1; ++ + if (check_bit(op->ldpc_dec.op_flags, RTE_BBDEV_LDPC_DEC_SCATTER_GATHER)) + seg_total_left = rte_pktmbuf_data_len(input) - in_offset; + else +@@ -2173,6 +2188,10 @@ enqueue_ldpc_dec_one_op_tb(struct acc_queue *q, struct rte_bbdev_dec_op *op, + r++; + } + ++ /* In case the number of CB doesn't match, the configuration was invalid. */ ++ if (unlikely(current_enqueued_cbs != cbs_in_tb)) ++ return -1; ++ + #ifdef RTE_LIBRTE_BBDEV_DEBUG + if (check_mbuf_total_left(mbuf_total_left) != 0) + return -EINVAL; +@@ -2215,6 +2234,8 @@ enqueue_dec_one_op_tb(struct acc_queue *q, struct rte_bbdev_dec_op *op, + r = op->turbo_dec.tb_params.r; + + while (mbuf_total_left > 0 && r < c) { ++ if (unlikely((input == NULL) || (h_output == NULL))) ++ return -1; + + seg_total_left = rte_pktmbuf_data_len(input) - in_offset; + +@@ -2265,6 +2286,10 @@ enqueue_dec_one_op_tb(struct acc_queue *q, struct rte_bbdev_dec_op *op, + r++; + } + ++ /* In case the number of CB doesn't match, the configuration was invalid. */ ++ if (unlikely(current_enqueued_cbs != cbs_in_tb)) ++ return -1; ++ + /* Set SDone on last CB descriptor for TB mode */ + desc->req.sdone_enable = 1; + +@@ -2636,7 +2661,8 @@ acc200_enqueue_ldpc_dec(struct rte_bbdev_queue_data *q_data, + /* Dequeue one encode operations from ACC200 device in CB mode. */ + static inline int + dequeue_enc_one_op_cb(struct acc_queue *q, struct rte_bbdev_enc_op **ref_op, +- uint16_t *dequeued_ops, uint32_t *aq_dequeued, uint16_t *dequeued_descs) ++ uint16_t *dequeued_ops, uint32_t *aq_dequeued, uint16_t *dequeued_descs, ++ uint16_t max_requested_ops) + { + union acc_dma_desc *desc, atom_desc; + union acc_dma_rsp_desc rsp; +@@ -2649,6 +2675,9 @@ dequeue_enc_one_op_cb(struct acc_queue *q, struct rte_bbdev_enc_op **ref_op, + desc = q->ring_addr + desc_idx; + atom_desc.atom_hdr = __atomic_load_n((uint64_t *)desc, __ATOMIC_RELAXED); + ++ if (*dequeued_ops + desc->req.numCBs > max_requested_ops) ++ return -1; ++ + /* Check fdone bit. */ + if (!(atom_desc.rsp.val & ACC_FDONE)) + return -1; +@@ -2690,7 +2719,7 @@ dequeue_enc_one_op_cb(struct acc_queue *q, struct rte_bbdev_enc_op **ref_op, + static inline int + dequeue_enc_one_op_tb(struct acc_queue *q, struct rte_bbdev_enc_op **ref_op, + uint16_t *dequeued_ops, uint32_t *aq_dequeued, +- uint16_t *dequeued_descs) ++ uint16_t *dequeued_descs, uint16_t max_requested_ops) + { + union acc_dma_desc *desc, *last_desc, atom_desc; + union acc_dma_rsp_desc rsp; +@@ -2701,6 +2730,9 @@ dequeue_enc_one_op_tb(struct acc_queue *q, struct rte_bbdev_enc_op **ref_op, + desc = acc_desc_tail(q, *dequeued_descs); + atom_desc.atom_hdr = __atomic_load_n((uint64_t *)desc, __ATOMIC_RELAXED); + ++ if (*dequeued_ops + 1 > max_requested_ops) ++ return -1; ++ + /* Check fdone bit. */ + if (!(atom_desc.rsp.val & ACC_FDONE)) + return -1; +@@ -2864,7 +2896,7 @@ dequeue_ldpc_dec_one_op_cb(struct rte_bbdev_queue_data *q_data, + return 1; + } + +-/* Dequeue one decode operations from ACC200 device in TB mode. */ ++/* Dequeue one decode operations from device in TB mode for 4G or 5G. */ + static inline int + dequeue_dec_one_op_tb(struct acc_queue *q, struct rte_bbdev_dec_op **ref_op, + uint16_t dequeued_cbs, uint32_t *aq_dequeued) +@@ -2918,8 +2950,12 @@ dequeue_dec_one_op_tb(struct acc_queue *q, struct rte_bbdev_dec_op **ref_op, + /* CRC invalid if error exists. */ + if (!op->status) + op->status |= rsp.crc_status << RTE_BBDEV_CRC_ERROR; +- op->turbo_dec.iter_count = RTE_MAX((uint8_t) rsp.iter_cnt, +- op->turbo_dec.iter_count); ++ if (q->op_type == RTE_BBDEV_OP_LDPC_DEC) ++ op->ldpc_dec.iter_count = RTE_MAX((uint8_t) rsp.iter_cnt, ++ op->ldpc_dec.iter_count); ++ else ++ op->turbo_dec.iter_count = RTE_MAX((uint8_t) rsp.iter_cnt, ++ op->turbo_dec.iter_count); + + /* Check if this is the last desc in batch (Atomic Queue). */ + if (desc->req.last_desc_in_batch) { +@@ -2961,25 +2997,23 @@ acc200_dequeue_enc(struct rte_bbdev_queue_data *q_data, + + cbm = op->turbo_enc.code_block_mode; + +- for (i = 0; i < num; i++) { ++ for (i = 0; i < avail; i++) { + if (cbm == RTE_BBDEV_TRANSPORT_BLOCK) + ret = dequeue_enc_one_op_tb(q, &ops[dequeued_ops], + &dequeued_ops, &aq_dequeued, +- &dequeued_descs); ++ &dequeued_descs, num); + else + ret = dequeue_enc_one_op_cb(q, &ops[dequeued_ops], + &dequeued_ops, &aq_dequeued, +- &dequeued_descs); ++ &dequeued_descs, num); + if (ret < 0) + break; +- if (dequeued_ops >= num) +- break; + } + + q->aq_dequeued += aq_dequeued; + q->sw_ring_tail += dequeued_descs; + +- /* Update enqueue stats */ ++ /* Update enqueue stats. */ + q_data->queue_stats.dequeued_count += dequeued_ops; + + return dequeued_ops; +@@ -3005,15 +3039,13 @@ acc200_dequeue_ldpc_enc(struct rte_bbdev_queue_data *q_data, + if (cbm == RTE_BBDEV_TRANSPORT_BLOCK) + ret = dequeue_enc_one_op_tb(q, &ops[dequeued_ops], + &dequeued_ops, &aq_dequeued, +- &dequeued_descs); ++ &dequeued_descs, num); + else + ret = dequeue_enc_one_op_cb(q, &ops[dequeued_ops], + &dequeued_ops, &aq_dequeued, +- &dequeued_descs); ++ &dequeued_descs, num); + if (ret < 0) + break; +- if (dequeued_ops >= num) +- break; + } + + q->aq_dequeued += aq_dequeued; +diff --git a/dpdk/drivers/baseband/fpga_5gnr_fec/rte_fpga_5gnr_fec.c b/dpdk/drivers/baseband/fpga_5gnr_fec/rte_fpga_5gnr_fec.c +index d520d5238f..0dfeba08e1 100644 +--- a/dpdk/drivers/baseband/fpga_5gnr_fec/rte_fpga_5gnr_fec.c ++++ b/dpdk/drivers/baseband/fpga_5gnr_fec/rte_fpga_5gnr_fec.c +@@ -569,17 +569,21 @@ static int + fpga_queue_start(struct rte_bbdev *dev, uint16_t queue_id) + { + struct fpga_5gnr_fec_device *d = dev->data->dev_private; ++ struct fpga_queue *q = dev->data->queues[queue_id].queue_private; ++ uint32_t offset = FPGA_5GNR_FEC_RING_CTRL_REGS + ++ (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx); ++ uint8_t enable = 0x01; ++ uint16_t zero = 0x0000; + #ifdef RTE_LIBRTE_BBDEV_DEBUG + if (d == NULL) { + rte_bbdev_log(ERR, "Invalid device pointer"); + return -1; + } + #endif +- struct fpga_queue *q = dev->data->queues[queue_id].queue_private; +- uint32_t offset = FPGA_5GNR_FEC_RING_CTRL_REGS + +- (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx); +- uint8_t enable = 0x01; +- uint16_t zero = 0x0000; ++ if (dev->data->queues[queue_id].queue_private == NULL) { ++ rte_bbdev_log(ERR, "Cannot start invalid queue %d", queue_id); ++ return -1; ++ } + + /* Clear queue head and tail variables */ + q->tail = q->head_free_desc = 0; +@@ -887,9 +891,11 @@ check_desc_error(uint32_t error_code) { + static inline uint16_t + get_k0(uint16_t n_cb, uint16_t z_c, uint8_t bg, uint8_t rv_index) + { ++ uint16_t n = (bg == 1 ? N_ZC_1 : N_ZC_2) * z_c; + if (rv_index == 0) + return 0; +- uint16_t n = (bg == 1 ? N_ZC_1 : N_ZC_2) * z_c; ++ if (z_c == 0) ++ return 0; + if (n_cb == n) { + if (rv_index == 1) + return (bg == 1 ? K0_1_1 : K0_1_2) * z_c; +diff --git a/dpdk/drivers/baseband/turbo_sw/meson.build b/dpdk/drivers/baseband/turbo_sw/meson.build +index 417ec63394..aeb9a76f9e 100644 +--- a/dpdk/drivers/baseband/turbo_sw/meson.build ++++ b/dpdk/drivers/baseband/turbo_sw/meson.build +@@ -6,11 +6,11 @@ dep_turbo = dependency('flexran_sdk_turbo', required: false) + dep_dec5g = dependency('flexran_sdk_ldpc_decoder_5gnr', required: false) + + if dep_turbo.found() +- ext_deps += cc.find_library('libstdc++', required: true) +- ext_deps += cc.find_library('libirc', required: true) +- ext_deps += cc.find_library('libimf', required: true) +- ext_deps += cc.find_library('libipps', required: true) +- ext_deps += cc.find_library('libsvml', required: true) ++ ext_deps += cc.find_library('stdc++', required: true) ++ ext_deps += cc.find_library('irc', required: true) ++ ext_deps += cc.find_library('imf', required: true) ++ ext_deps += cc.find_library('ipps', required: true) ++ ext_deps += cc.find_library('svml', required: true) + ext_deps += dep_turbo + ext_deps += dependency('flexran_sdk_crc', required: true) + ext_deps += dependency('flexran_sdk_rate_matching', required: true) +diff --git a/dpdk/drivers/bus/dpaa/base/qbman/qman.c b/dpdk/drivers/bus/dpaa/base/qbman/qman.c +index 3949bf8712..83db0a534e 100644 +--- a/dpdk/drivers/bus/dpaa/base/qbman/qman.c ++++ b/dpdk/drivers/bus/dpaa/base/qbman/qman.c +@@ -1,7 +1,7 @@ + /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0) + * + * Copyright 2008-2016 Freescale Semiconductor Inc. +- * Copyright 2017,2019 NXP ++ * Copyright 2017,2019-2023 NXP + * + */ + +@@ -897,7 +897,7 @@ mr_loop: + /* Lookup in the retirement table */ + fq = table_find_fq(p, + be32_to_cpu(msg->fq.fqid)); +- DPAA_BUG_ON(!fq); ++ DPAA_BUG_ON(fq != NULL); + fq_state_change(p, fq, &swapped_msg, verb); + if (fq->cb.fqs) + fq->cb.fqs(p, fq, &swapped_msg); +@@ -909,6 +909,7 @@ mr_loop: + #else + fq = (void *)(uintptr_t)msg->fq.contextB; + #endif ++ DPAA_BUG_ON(fq != NULL); + fq_state_change(p, fq, msg, verb); + if (fq->cb.fqs) + fq->cb.fqs(p, fq, &swapped_msg); +diff --git a/dpdk/drivers/bus/fslmc/mc/mc_sys.c b/dpdk/drivers/bus/fslmc/mc/mc_sys.c +index ab9a074835..76fdcd5c8a 100644 +--- a/dpdk/drivers/bus/fslmc/mc/mc_sys.c ++++ b/dpdk/drivers/bus/fslmc/mc/mc_sys.c +@@ -77,8 +77,11 @@ int mc_send_command(struct fsl_mc_io *mc_io, struct mc_command *cmd) + total_time = rte_get_timer_cycles() - start_time; + } while (status == MC_CMD_STATUS_READY && total_time <= time_to_wait); + +- if (status == MC_CMD_STATUS_READY) ++ if (status == MC_CMD_STATUS_READY) { ++ rte_spinlock_unlock(&mc_portal_lock); ++ + return mc_status_to_error(MC_CMD_STATUS_TIMEOUT); ++ } + + /* Read the response back into the command buffer */ + mc_read_response(mc_io->regs, cmd); +diff --git a/dpdk/drivers/bus/ifpga/bus_ifpga_driver.h b/dpdk/drivers/bus/ifpga/bus_ifpga_driver.h +index 7b75c2ddbc..5bbe36d6e0 100644 +--- a/dpdk/drivers/bus/ifpga/bus_ifpga_driver.h ++++ b/dpdk/drivers/bus/ifpga/bus_ifpga_driver.h +@@ -16,6 +16,7 @@ extern "C" { + #endif /* __cplusplus */ + + #include ++#include + #include + #include + #include +diff --git a/dpdk/drivers/bus/ifpga/ifpga_bus.c b/dpdk/drivers/bus/ifpga/ifpga_bus.c +index bb943b58b5..07e316b38e 100644 +--- a/dpdk/drivers/bus/ifpga/ifpga_bus.c ++++ b/dpdk/drivers/bus/ifpga/ifpga_bus.c +@@ -135,6 +135,8 @@ ifpga_scan_one(struct rte_rawdev *rawdev, + goto end; + } + afu_pr_conf.pr_enable = 1; ++ strlcpy(afu_pr_conf.bs_path, path, ++ sizeof(afu_pr_conf.bs_path)); + } else { + afu_pr_conf.pr_enable = 0; + } +@@ -174,7 +176,6 @@ ifpga_scan_one(struct rte_rawdev *rawdev, + rawdev->dev_ops->dev_start(rawdev)) + goto end; + +- strlcpy(afu_pr_conf.bs_path, path, sizeof(afu_pr_conf.bs_path)); + if (rawdev->dev_ops && + rawdev->dev_ops->firmware_load && + rawdev->dev_ops->firmware_load(rawdev, +diff --git a/dpdk/drivers/bus/pci/linux/pci_vfio.c b/dpdk/drivers/bus/pci/linux/pci_vfio.c +index fab3483d9f..fe83e1a04e 100644 +--- a/dpdk/drivers/bus/pci/linux/pci_vfio.c ++++ b/dpdk/drivers/bus/pci/linux/pci_vfio.c +@@ -2,6 +2,7 @@ + * Copyright(c) 2010-2014 Intel Corporation + */ + ++#include + #include + #include + #include +diff --git a/dpdk/drivers/bus/pci/pci_common.c b/dpdk/drivers/bus/pci/pci_common.c +index bc3a7f39fe..756e308fdf 100644 +--- a/dpdk/drivers/bus/pci/pci_common.c ++++ b/dpdk/drivers/bus/pci/pci_common.c +@@ -304,7 +304,7 @@ rte_pci_probe_one_driver(struct rte_pci_driver *dr, + } + } + +- RTE_LOG(INFO, EAL, "Probe PCI driver: %s (%x:%x) device: "PCI_PRI_FMT" (socket %i)\n", ++ RTE_LOG(INFO, EAL, "Probe PCI driver: %s (%x:%04x) device: "PCI_PRI_FMT" (socket %i)\n", + dr->driver.name, dev->id.vendor_id, dev->id.device_id, + loc->domain, loc->bus, loc->devid, loc->function, + dev->device.numa_node); +@@ -448,7 +448,7 @@ pci_cleanup(void) + int ret = 0; + + if (drv == NULL || drv->remove == NULL) +- continue; ++ goto free; + + ret = drv->remove(dev); + if (ret < 0) { +@@ -458,6 +458,7 @@ pci_cleanup(void) + dev->driver = NULL; + dev->device.driver = NULL; + ++free: + /* free interrupt handles */ + rte_intr_instance_free(dev->intr_handle); + dev->intr_handle = NULL; +@@ -883,6 +884,16 @@ rte_pci_set_bus_master(struct rte_pci_device *dev, bool enable) + return 0; + } + ++int ++rte_pci_pasid_set_state(const struct rte_pci_device *dev, ++ off_t offset, bool enable) ++{ ++ uint16_t pasid = enable; ++ return rte_pci_write_config(dev, &pasid, sizeof(pasid), offset) < 0 ++ ? -1 ++ : 0; ++} ++ + struct rte_pci_bus rte_pci_bus = { + .bus = { + .scan = rte_pci_scan, +diff --git a/dpdk/drivers/bus/pci/rte_bus_pci.h b/dpdk/drivers/bus/pci/rte_bus_pci.h +index b193114fe5..76cbf49ab8 100644 +--- a/dpdk/drivers/bus/pci/rte_bus_pci.h ++++ b/dpdk/drivers/bus/pci/rte_bus_pci.h +@@ -101,6 +101,20 @@ off_t rte_pci_find_ext_capability(struct rte_pci_device *dev, uint32_t cap); + __rte_experimental + int rte_pci_set_bus_master(struct rte_pci_device *dev, bool enable); + ++/** ++ * Enable/Disable PASID (Process Address Space ID). ++ * ++ * @param dev ++ * A pointer to a rte_pci_device structure. ++ * @param offset ++ * Offset of the PASID external capability. ++ * @param enable ++ * Flag to enable or disable PASID. ++ */ ++__rte_internal ++int rte_pci_pasid_set_state(const struct rte_pci_device *dev, ++ off_t offset, bool enable); ++ + /** + * Read PCI config space. + * +diff --git a/dpdk/drivers/bus/pci/version.map b/dpdk/drivers/bus/pci/version.map +index 161ab86d3b..f262af3316 100644 +--- a/dpdk/drivers/bus/pci/version.map ++++ b/dpdk/drivers/bus/pci/version.map +@@ -27,6 +27,7 @@ INTERNAL { + global: + + rte_pci_get_sysfs_path; ++ rte_pci_pasid_set_state; + rte_pci_register; + rte_pci_unregister; + }; +diff --git a/dpdk/drivers/bus/vdev/vdev.c b/dpdk/drivers/bus/vdev/vdev.c +index 41bc07dde7..7974b27295 100644 +--- a/dpdk/drivers/bus/vdev/vdev.c ++++ b/dpdk/drivers/bus/vdev/vdev.c +@@ -578,18 +578,19 @@ vdev_cleanup(void) + int ret = 0; + + if (dev->device.driver == NULL) +- continue; ++ goto free; + + drv = container_of(dev->device.driver, const struct rte_vdev_driver, driver); + + if (drv->remove == NULL) +- continue; ++ goto free; + + ret = drv->remove(dev); + if (ret < 0) + error = -1; + + dev->device.driver = NULL; ++free: + free(dev); + } + +diff --git a/dpdk/drivers/common/cnxk/cnxk_security.c b/dpdk/drivers/common/cnxk/cnxk_security.c +index 85105472a1..bdb5433d13 100644 +--- a/dpdk/drivers/common/cnxk/cnxk_security.c ++++ b/dpdk/drivers/common/cnxk/cnxk_security.c +@@ -274,6 +274,14 @@ ot_ipsec_inb_ctx_size(struct roc_ot_ipsec_inb_sa *sa) + return size; + } + ++static void ++ot_ipsec_update_ipv6_addr_endianness(uint64_t *addr) ++{ ++ *addr = rte_be_to_cpu_64(*addr); ++ addr++; ++ *addr = rte_be_to_cpu_64(*addr); ++} ++ + static int + ot_ipsec_inb_tunnel_hdr_fill(struct roc_ot_ipsec_inb_sa *sa, + struct rte_security_ipsec_xform *ipsec_xfrm) +@@ -310,6 +318,10 @@ ot_ipsec_inb_tunnel_hdr_fill(struct roc_ot_ipsec_inb_sa *sa, + memcpy(&sa->outer_hdr.ipv6.dst_addr, &tunnel->ipv6.dst_addr, + sizeof(struct in6_addr)); + ++ /* IP Source and Dest are in LE/CPU endian */ ++ ot_ipsec_update_ipv6_addr_endianness((uint64_t *)&sa->outer_hdr.ipv6.src_addr); ++ ot_ipsec_update_ipv6_addr_endianness((uint64_t *)&sa->outer_hdr.ipv6.dst_addr); ++ + break; + default: + return -EINVAL; +@@ -499,6 +511,10 @@ cnxk_ot_ipsec_outb_sa_fill(struct roc_ot_ipsec_outb_sa *sa, + memcpy(&sa->outer_hdr.ipv6.dst_addr, &tunnel->ipv6.dst_addr, + sizeof(struct in6_addr)); + ++ /* IP Source and Dest are in LE/CPU endian */ ++ ot_ipsec_update_ipv6_addr_endianness((uint64_t *)&sa->outer_hdr.ipv6.src_addr); ++ ot_ipsec_update_ipv6_addr_endianness((uint64_t *)&sa->outer_hdr.ipv6.dst_addr); ++ + /* Outer header flow label source */ + if (!ipsec_xfrm->options.copy_flabel) { + sa->w2.s.ipv4_df_src_or_ipv6_flw_lbl_src = +diff --git a/dpdk/drivers/common/cnxk/cnxk_security_ar.h b/dpdk/drivers/common/cnxk/cnxk_security_ar.h +index deb38db0d0..d0151a752c 100644 +--- a/dpdk/drivers/common/cnxk/cnxk_security_ar.h ++++ b/dpdk/drivers/common/cnxk/cnxk_security_ar.h +@@ -17,7 +17,7 @@ + BITS_PER_LONG_LONG) + + #define WORD_SHIFT 6 +-#define WORD_SIZE (1 << WORD_SHIFT) ++#define WORD_SIZE (1ULL << WORD_SHIFT) + #define WORD_MASK (WORD_SIZE - 1) + + #define IPSEC_ANTI_REPLAY_FAILED (-1) +diff --git a/dpdk/drivers/common/cnxk/hw/nix.h b/dpdk/drivers/common/cnxk/hw/nix.h +index 425c335bf3..3246ea6573 100644 +--- a/dpdk/drivers/common/cnxk/hw/nix.h ++++ b/dpdk/drivers/common/cnxk/hw/nix.h +@@ -617,6 +617,7 @@ + #define NIX_RX_ACTIONOP_RSS (0x4ull) + #define NIX_RX_ACTIONOP_PF_FUNC_DROP (0x5ull) + #define NIX_RX_ACTIONOP_MIRROR (0x6ull) ++#define NIX_RX_ACTIONOP_DEFAULT (0xfull) + + #define NIX_RX_VTAGACTION_VTAG0_RELPTR (0x0ull) + #define NIX_RX_VTAGACTION_VTAG1_RELPTR (0x4ull) +diff --git a/dpdk/drivers/common/cnxk/meson.build b/dpdk/drivers/common/cnxk/meson.build +index 849735921c..ca8170624d 100644 +--- a/dpdk/drivers/common/cnxk/meson.build ++++ b/dpdk/drivers/common/cnxk/meson.build +@@ -8,7 +8,6 @@ if not is_linux or not dpdk_conf.get('RTE_ARCH_64') + subdir_done() + endif + +-config_flag_fmt = 'RTE_LIBRTE_@0@_COMMON' + deps = ['eal', 'pci', 'bus_pci', 'mbuf', 'security'] + sources = files( + 'roc_ae.c', +diff --git a/dpdk/drivers/common/cnxk/roc_dev.c b/dpdk/drivers/common/cnxk/roc_dev.c +index 59128a3552..33865f43fa 100644 +--- a/dpdk/drivers/common/cnxk/roc_dev.c ++++ b/dpdk/drivers/common/cnxk/roc_dev.c +@@ -969,6 +969,7 @@ dev_vf_hwcap_update(struct plt_pci_device *pci_dev, struct dev *dev) + case PCI_DEVID_CNXK_RVU_AF_VF: + case PCI_DEVID_CNXK_RVU_VF: + case PCI_DEVID_CNXK_RVU_SDP_VF: ++ case PCI_DEVID_CNXK_RVU_NIX_INL_VF: + dev->hwcap |= DEV_HWCAP_F_VF; + break; + } +diff --git a/dpdk/drivers/common/cnxk/roc_dpi.c b/dpdk/drivers/common/cnxk/roc_dpi.c +index 93c8318a3d..0e2f803077 100644 +--- a/dpdk/drivers/common/cnxk/roc_dpi.c ++++ b/dpdk/drivers/common/cnxk/roc_dpi.c +@@ -81,10 +81,10 @@ roc_dpi_configure(struct roc_dpi *roc_dpi) + return rc; + } + +- snprintf(name, sizeof(name), "dpimem%d", roc_dpi->vfid); ++ snprintf(name, sizeof(name), "dpimem%d:%d:%d:%d", pci_dev->addr.domain, pci_dev->addr.bus, ++ pci_dev->addr.devid, pci_dev->addr.function); + buflen = DPI_CMD_QUEUE_SIZE * DPI_CMD_QUEUE_BUFS; +- dpi_mz = plt_memzone_reserve_aligned(name, buflen, 0, +- DPI_CMD_QUEUE_SIZE); ++ dpi_mz = plt_memzone_reserve_aligned(name, buflen, 0, DPI_CMD_QUEUE_SIZE); + if (dpi_mz == NULL) { + plt_err("dpi memzone reserve failed"); + rc = -ENOMEM; +diff --git a/dpdk/drivers/common/cnxk/roc_io.h b/dpdk/drivers/common/cnxk/roc_io.h +index 13f98ed549..45cbb4e587 100644 +--- a/dpdk/drivers/common/cnxk/roc_io.h ++++ b/dpdk/drivers/common/cnxk/roc_io.h +@@ -125,7 +125,8 @@ roc_lmt_submit_ldeor(plt_iova_t io_address) + + asm volatile(PLT_CPU_FEATURE_PREAMBLE "ldeor xzr, %x[rf], [%[rs]]" + : [rf] "=r"(result) +- : [rs] "r"(io_address)); ++ : [rs] "r"(io_address) ++ : "memory"); + return result; + } + +@@ -136,7 +137,8 @@ roc_lmt_submit_ldeorl(plt_iova_t io_address) + + asm volatile(PLT_CPU_FEATURE_PREAMBLE "ldeorl xzr,%x[rf],[%[rs]]" + : [rf] "=r"(result) +- : [rs] "r"(io_address)); ++ : [rs] "r"(io_address) ++ : "memory"); + return result; + } + +@@ -145,7 +147,8 @@ roc_lmt_submit_steor(uint64_t data, plt_iova_t io_address) + { + asm volatile(PLT_CPU_FEATURE_PREAMBLE + "steor %x[d], [%[rs]]" ::[d] "r"(data), +- [rs] "r"(io_address)); ++ [rs] "r"(io_address) ++ : "memory"); + } + + static __plt_always_inline void +@@ -153,7 +156,8 @@ roc_lmt_submit_steorl(uint64_t data, plt_iova_t io_address) + { + asm volatile(PLT_CPU_FEATURE_PREAMBLE + "steorl %x[d], [%[rs]]" ::[d] "r"(data), +- [rs] "r"(io_address)); ++ [rs] "r"(io_address) ++ : "memory"); + } + + static __plt_always_inline void +diff --git a/dpdk/drivers/common/cnxk/roc_mbox.h b/dpdk/drivers/common/cnxk/roc_mbox.h +index 8b0384c737..18aa97b84a 100644 +--- a/dpdk/drivers/common/cnxk/roc_mbox.h ++++ b/dpdk/drivers/common/cnxk/roc_mbox.h +@@ -471,7 +471,7 @@ struct lmtst_tbl_setup_req { + + struct cgx_stats_rsp { + struct mbox_msghdr hdr; +-#define CGX_RX_STATS_COUNT 13 ++#define CGX_RX_STATS_COUNT 9 + #define CGX_TX_STATS_COUNT 18 + uint64_t __io rx_stats[CGX_RX_STATS_COUNT]; + uint64_t __io tx_stats[CGX_TX_STATS_COUNT]; +@@ -1169,7 +1169,7 @@ struct nix_bp_cfg_req { + * so maximum 256 channels are possible. + */ + #define NIX_MAX_CHAN 256 +-#define NIX_CGX_MAX_CHAN 16 ++#define NIX_CGX_MAX_CHAN 8 + #define NIX_LBK_MAX_CHAN 1 + struct nix_bp_cfg_rsp { + struct mbox_msghdr hdr; +diff --git a/dpdk/drivers/common/cnxk/roc_nix_fc.c b/dpdk/drivers/common/cnxk/roc_nix_fc.c +index 033e17a4bf..5e8a01c775 100644 +--- a/dpdk/drivers/common/cnxk/roc_nix_fc.c ++++ b/dpdk/drivers/common/cnxk/roc_nix_fc.c +@@ -88,17 +88,6 @@ nix_fc_rxchan_bpid_set(struct roc_nix *roc_nix, bool enable) + req->chan_cnt = 1; + req->bpid_per_chan = 0; + +- rc = mbox_process_msg(mbox, (void *)&rsp); +- if (rc) +- goto exit; +- } else { +- req = mbox_alloc_msg_nix_cpt_bp_disable(mbox); +- if (req == NULL) +- return rc; +- req->chan_base = 0; +- req->chan_cnt = 1; +- req->bpid_per_chan = 0; +- + rc = mbox_process_msg(mbox, (void *)&rsp); + if (rc) + goto exit; +diff --git a/dpdk/drivers/common/cnxk/roc_nix_inl.c b/dpdk/drivers/common/cnxk/roc_nix_inl.c +index 782536db4c..92ff44888d 100644 +--- a/dpdk/drivers/common/cnxk/roc_nix_inl.c ++++ b/dpdk/drivers/common/cnxk/roc_nix_inl.c +@@ -1039,7 +1039,7 @@ roc_nix_inl_rq_ena_dis(struct roc_nix *roc_nix, bool enable) + return -EFAULT; + + if (roc_model_is_cn10kb_a0()) { +- rc = nix_inl_rq_mask_cfg(roc_nix, true); ++ rc = nix_inl_rq_mask_cfg(roc_nix, enable); + if (rc) { + plt_err("Failed to get rq mask rc=%d", rc); + return rc; +diff --git a/dpdk/drivers/common/cnxk/roc_nix_inl_dev.c b/dpdk/drivers/common/cnxk/roc_nix_inl_dev.c +index c3d94dd0da..746b3d0a03 100644 +--- a/dpdk/drivers/common/cnxk/roc_nix_inl_dev.c ++++ b/dpdk/drivers/common/cnxk/roc_nix_inl_dev.c +@@ -265,7 +265,7 @@ nix_inl_sso_setup(struct nix_inl_dev *inl_dev) + } + + /* Setup xaq for hwgrps */ +- rc = sso_hwgrp_alloc_xaq(dev, inl_dev->xaq.aura_handle, 1); ++ rc = sso_hwgrp_alloc_xaq(dev, roc_npa_aura_handle_to_aura(inl_dev->xaq.aura_handle), 1); + if (rc) { + plt_err("Failed to setup hwgrp xaq aura, rc=%d", rc); + goto destroy_pool; +@@ -643,7 +643,8 @@ no_pool: + } + + /* Setup xaq for hwgrps */ +- rc = sso_hwgrp_alloc_xaq(&inl_dev->dev, inl_dev->xaq.aura_handle, 1); ++ rc = sso_hwgrp_alloc_xaq(&inl_dev->dev, ++ roc_npa_aura_handle_to_aura(inl_dev->xaq.aura_handle), 1); + if (rc) { + plt_err("Failed to setup hwgrp xaq aura, rc=%d", rc); + return rc; +diff --git a/dpdk/drivers/common/cnxk/roc_npa.c b/dpdk/drivers/common/cnxk/roc_npa.c +index ee42434c38..ea58030477 100644 +--- a/dpdk/drivers/common/cnxk/roc_npa.c ++++ b/dpdk/drivers/common/cnxk/roc_npa.c +@@ -115,6 +115,8 @@ npa_aura_pool_fini(struct mbox *mbox, uint32_t aura_id, uint64_t aura_handle) + aura_req->op = NPA_AQ_INSTOP_WRITE; + aura_req->aura.ena = 0; + aura_req->aura_mask.ena = ~aura_req->aura_mask.ena; ++ aura_req->aura.bp_ena = 0; ++ aura_req->aura_mask.bp_ena = ~aura_req->aura_mask.bp_ena; + + rc = mbox_process(mbox); + if (rc < 0) +@@ -370,7 +372,11 @@ npa_aura_pool_pair_alloc(struct npa_lf *lf, const uint32_t block_size, + /* Update pool fields */ + pool->stack_base = mz->iova; + pool->ena = 1; +- pool->buf_size = block_size / ROC_ALIGN; ++ /* In opaque mode buffer size must be 0 */ ++ if (!pool->nat_align) ++ pool->buf_size = 0; ++ else ++ pool->buf_size = block_size / ROC_ALIGN; + pool->stack_max_pages = stack_size; + pool->shift = plt_log2_u32(block_count); + pool->shift = pool->shift < 8 ? 0 : pool->shift - 8; +diff --git a/dpdk/drivers/common/cnxk/roc_npa.h b/dpdk/drivers/common/cnxk/roc_npa.h +index fed1942404..46b668a310 100644 +--- a/dpdk/drivers/common/cnxk/roc_npa.h ++++ b/dpdk/drivers/common/cnxk/roc_npa.h +@@ -253,19 +253,23 @@ roc_npa_aura_batch_alloc_issue(uint64_t aura_handle, uint64_t *buf, + } + + static inline void +-roc_npa_batch_alloc_wait(uint64_t *cache_line) ++roc_npa_batch_alloc_wait(uint64_t *cache_line, unsigned int wait_us) + { ++ const uint64_t ticks = (uint64_t)wait_us * plt_tsc_hz() / (uint64_t)1E6; ++ const uint64_t start = plt_tsc_cycles(); ++ + /* Batch alloc status code is updated in bits [5:6] of the first word + * of the 128 byte cache line. + */ + while (((__atomic_load_n(cache_line, __ATOMIC_RELAXED) >> 5) & 0x3) == + ALLOC_CCODE_INVAL) +- ; ++ if (wait_us && (plt_tsc_cycles() - start) >= ticks) ++ break; + } + + static inline unsigned int + roc_npa_aura_batch_alloc_count(uint64_t *aligned_buf, unsigned int num, +- unsigned int do_wait) ++ unsigned int wait_us) + { + unsigned int count, i; + +@@ -279,8 +283,7 @@ roc_npa_aura_batch_alloc_count(uint64_t *aligned_buf, unsigned int num, + + status = (struct npa_batch_alloc_status_s *)&aligned_buf[i]; + +- if (do_wait) +- roc_npa_batch_alloc_wait(&aligned_buf[i]); ++ roc_npa_batch_alloc_wait(&aligned_buf[i], wait_us); + + count += status->count; + } +@@ -305,7 +308,7 @@ roc_npa_aura_batch_alloc_extract(uint64_t *buf, uint64_t *aligned_buf, + + status = (struct npa_batch_alloc_status_s *)&aligned_buf[i]; + +- roc_npa_batch_alloc_wait(&aligned_buf[i]); ++ roc_npa_batch_alloc_wait(&aligned_buf[i], 0); + + line_count = status->count; + +diff --git a/dpdk/drivers/common/cnxk/roc_npc.c b/dpdk/drivers/common/cnxk/roc_npc.c +index b38389b18a..a5b5f95dec 100644 +--- a/dpdk/drivers/common/cnxk/roc_npc.c ++++ b/dpdk/drivers/common/cnxk/roc_npc.c +@@ -634,11 +634,15 @@ npc_parse_actions(struct roc_npc *roc_npc, const struct roc_npc_attr *attr, + if (req_act == ROC_NPC_ACTION_TYPE_VLAN_STRIP) { + /* Only VLAN action is provided */ + flow->npc_action = NIX_RX_ACTIONOP_UCAST; +- } else if (req_act & +- (ROC_NPC_ACTION_TYPE_PF | ROC_NPC_ACTION_TYPE_VF)) { +- flow->npc_action = NIX_RX_ACTIONOP_UCAST; +- if (req_act & ROC_NPC_ACTION_TYPE_QUEUE) +- flow->npc_action |= (uint64_t)rq << 20; ++ } else if (req_act & (ROC_NPC_ACTION_TYPE_PF | ROC_NPC_ACTION_TYPE_VF)) { ++ /* Check if any other action is set */ ++ if ((req_act == ROC_NPC_ACTION_TYPE_PF) || (req_act == ROC_NPC_ACTION_TYPE_VF)) { ++ flow->npc_action = NIX_RX_ACTIONOP_DEFAULT; ++ } else { ++ flow->npc_action = NIX_RX_ACTIONOP_UCAST; ++ if (req_act & ROC_NPC_ACTION_TYPE_QUEUE) ++ flow->npc_action |= (uint64_t)rq << 20; ++ } + } else if (req_act & ROC_NPC_ACTION_TYPE_DROP) { + flow->npc_action = NIX_RX_ACTIONOP_DROP; + } else if (req_act & ROC_NPC_ACTION_TYPE_QUEUE) { +@@ -649,8 +653,7 @@ npc_parse_actions(struct roc_npc *roc_npc, const struct roc_npc_attr *attr, + } else if (req_act & ROC_NPC_ACTION_TYPE_SEC) { + flow->npc_action = NIX_RX_ACTIONOP_UCAST_IPSEC; + flow->npc_action |= (uint64_t)rq << 20; +- } else if (req_act & +- (ROC_NPC_ACTION_TYPE_FLAG | ROC_NPC_ACTION_TYPE_MARK)) { ++ } else if (req_act & (ROC_NPC_ACTION_TYPE_FLAG | ROC_NPC_ACTION_TYPE_MARK)) { + flow->npc_action = NIX_RX_ACTIONOP_UCAST; + } else if (req_act & ROC_NPC_ACTION_TYPE_COUNT) { + /* Keep ROC_NPC_ACTION_TYPE_COUNT_ACT always at the end +@@ -832,9 +835,35 @@ npc_rss_action_configure(struct roc_npc *roc_npc, + uint8_t key[ROC_NIX_RSS_KEY_LEN]; + const uint8_t *key_ptr; + uint8_t flowkey_algx; ++ uint32_t key_len; + uint16_t *reta; + int rc; + ++ roc_nix_rss_key_get(roc_nix, key); ++ if (rss->key == NULL) { ++ key_ptr = key; ++ } else { ++ key_len = rss->key_len; ++ if (key_len > ROC_NIX_RSS_KEY_LEN) ++ key_len = ROC_NIX_RSS_KEY_LEN; ++ ++ for (i = 0; i < key_len; i++) { ++ if (key[i] != rss->key[i]) { ++ plt_err("RSS key config not supported"); ++ plt_err("New Key:"); ++ for (i = 0; i < key_len; i++) ++ plt_dump_no_nl("0x%.2x ", rss->key[i]); ++ plt_dump_no_nl("\n"); ++ plt_err("Configured Key:"); ++ for (i = 0; i < ROC_NIX_RSS_KEY_LEN; i++) ++ plt_dump_no_nl("0x%.2x ", key[i]); ++ plt_dump_no_nl("\n"); ++ return -ENOTSUP; ++ } ++ } ++ key_ptr = rss->key; ++ } ++ + rc = npc_rss_free_grp_get(npc, &rss_grp_idx); + /* RSS group :0 is not usable for flow rss action */ + if (rc < 0 || rss_grp_idx == 0) +@@ -849,13 +878,6 @@ npc_rss_action_configure(struct roc_npc *roc_npc, + + *rss_grp = rss_grp_idx; + +- if (rss->key == NULL) { +- roc_nix_rss_key_default_fill(roc_nix, key); +- key_ptr = key; +- } else { +- key_ptr = rss->key; +- } +- + roc_nix_rss_key_set(roc_nix, key_ptr); + + /* If queue count passed in the rss action is less than +@@ -1242,12 +1264,40 @@ npc_vtag_action_program(struct roc_npc *roc_npc, + return 0; + } + ++void ++roc_npc_sdp_channel_get(struct roc_npc *roc_npc, uint16_t *chan_base, uint16_t *chan_mask) ++{ ++ struct roc_nix *roc_nix = roc_npc->roc_nix; ++ struct nix *nix = roc_nix_to_nix_priv(roc_nix); ++ uint16_t num_chan, range, num_bits = 0; ++ uint16_t mask = 0; ++ ++ *chan_base = nix->rx_chan_base; ++ num_chan = nix->rx_chan_cnt - 1; ++ if (num_chan) { ++ range = *chan_base ^ (*chan_base + num_chan); ++ num_bits = (sizeof(uint32_t) * 8) - __builtin_clz(range) - 1; ++ /* Set mask for (15 - numbits) MSB bits */ ++ *chan_mask = (uint16_t)~GENMASK(num_bits, 0); ++ *chan_mask &= 0xFFF; ++ } else { ++ *chan_mask = (uint16_t)GENMASK(11, 0); ++ } ++ ++ mask = (uint16_t)GENMASK(num_bits, 0); ++ if (mask > num_chan + 1) ++ plt_warn( ++ "npc: SDP channel base:%x, channel count:%x. channel mask:%x covers more than channel count", ++ *chan_base, nix->rx_chan_cnt, *chan_mask); ++} ++ + struct roc_npc_flow * + roc_npc_flow_create(struct roc_npc *roc_npc, const struct roc_npc_attr *attr, + const struct roc_npc_item_info pattern[], + const struct roc_npc_action actions[], int *errcode) + { + struct npc *npc = roc_npc_to_npc_priv(roc_npc); ++ uint16_t sdp_chan_base = 0, sdp_chan_mask = 0; + struct roc_npc_flow *flow, *flow_iter; + struct npc_parse_state parse_state; + struct npc_flow_list *list; +@@ -1260,11 +1310,9 @@ roc_npc_flow_create(struct roc_npc *roc_npc, const struct roc_npc_attr *attr, + npc->sdp_channel = roc_npc->sdp_channel; + npc->sdp_channel_mask = roc_npc->sdp_channel_mask; + } else { +- /* By default set the channel and mask to cover +- * the whole SDP channel range. +- */ +- npc->sdp_channel = (uint16_t)NIX_CHAN_SDP_CH_START; +- npc->sdp_channel_mask = (uint16_t)NIX_CHAN_SDP_CH_START; ++ roc_npc_sdp_channel_get(roc_npc, &sdp_chan_base, &sdp_chan_mask); ++ npc->sdp_channel = sdp_chan_base; ++ npc->sdp_channel_mask = sdp_chan_mask; + } + } + +diff --git a/dpdk/drivers/common/cnxk/roc_npc.h b/dpdk/drivers/common/cnxk/roc_npc.h +index 1b4e5521cb..25e916ce08 100644 +--- a/dpdk/drivers/common/cnxk/roc_npc.h ++++ b/dpdk/drivers/common/cnxk/roc_npc.h +@@ -123,6 +123,17 @@ struct roc_ipv6_hdr { + uint8_t dst_addr[16]; /**< IP address of destination host(s). */ + } __plt_packed; + ++struct roc_ipv6_fragment_ext { ++ uint8_t next_header; /**< Next header type */ ++ uint8_t reserved; /**< Reserved */ ++ uint16_t frag_data; /**< All fragmentation data */ ++ uint32_t id; /**< Packet ID */ ++} __plt_packed; ++ ++struct roc_flow_item_ipv6_ext { ++ uint8_t next_hdr; /**< Next header. */ ++}; ++ + struct roc_npc_flow_item_ipv6 { + struct roc_ipv6_hdr hdr; /**< IPv6 header definition. */ + uint32_t has_hop_ext : 1; +@@ -374,4 +385,6 @@ int __roc_api roc_npc_mcam_init(struct roc_npc *roc_npc, + struct roc_npc_flow *flow, int mcam_id); + int __roc_api roc_npc_mcam_move(struct roc_npc *roc_npc, uint16_t old_ent, + uint16_t new_ent); ++void __roc_api roc_npc_sdp_channel_get(struct roc_npc *roc_npc, uint16_t *chan_base, ++ uint16_t *chan_mask); + #endif /* _ROC_NPC_H_ */ +diff --git a/dpdk/drivers/common/cnxk/roc_npc_mcam.c b/dpdk/drivers/common/cnxk/roc_npc_mcam.c +index a725cabc57..3bf35cdf48 100644 +--- a/dpdk/drivers/common/cnxk/roc_npc_mcam.c ++++ b/dpdk/drivers/common/cnxk/roc_npc_mcam.c +@@ -551,6 +551,8 @@ npc_mcam_alloc_and_write(struct npc *npc, struct roc_npc_flow *flow, + struct idev_cfg *idev; + uint16_t pf_func = 0; + uint16_t ctr = ~(0); ++ uint32_t la_offset; ++ uint64_t mask; + int rc, idx; + int entry; + +@@ -617,17 +619,42 @@ npc_mcam_alloc_and_write(struct npc *npc, struct roc_npc_flow *flow, + flow->npc_action &= ~(GENMASK(19, 4)); + flow->npc_action |= (uint64_t)pf_func << 4; + +- npc_mcam_set_channel(flow, req, inl_dev->channel, +- inl_dev->chan_mask, false); ++ npc_mcam_set_channel(flow, req, inl_dev->channel, inl_dev->chan_mask, ++ false); + } else if (npc->is_sdp_link) { +- npc_mcam_set_channel(flow, req, npc->sdp_channel, +- npc->sdp_channel_mask, ++ npc_mcam_set_channel(flow, req, npc->sdp_channel, npc->sdp_channel_mask, + pst->is_second_pass_rule); + } else { +- npc_mcam_set_channel(flow, req, npc->channel, +- (BIT_ULL(12) - 1), ++ npc_mcam_set_channel(flow, req, npc->channel, (BIT_ULL(12) - 1), + pst->is_second_pass_rule); + } ++ /* ++ * For second pass rule, set LA LTYPE to CPT_HDR. ++ * For all other rules, set LA LTYPE to match both 1st pass and 2nd pass ltypes. ++ */ ++ if (pst->is_second_pass_rule || (!pst->is_second_pass_rule && pst->has_eth_type)) { ++ la_offset = __builtin_popcount(npc->keyx_supp_nmask[flow->nix_intf] & ++ ((1ULL << 9 /* LA offset */) - 1)); ++ la_offset *= 4; ++ ++ mask = ~((0xfULL << la_offset)); ++ req->entry_data.kw[0] &= mask; ++ req->entry_data.kw_mask[0] &= mask; ++ flow->mcam_data[0] &= mask; ++ flow->mcam_mask[0] &= mask; ++ if (pst->is_second_pass_rule) { ++ req->entry_data.kw[0] |= ((uint64_t)NPC_LT_LA_CPT_HDR) << la_offset; ++ req->entry_data.kw_mask[0] |= (0xFULL << la_offset); ++ flow->mcam_data[0] |= ((uint64_t)NPC_LT_LA_CPT_HDR) << la_offset; ++ flow->mcam_mask[0] |= (0xFULL << la_offset); ++ } else { ++ /* Mask ltype ETHER (0x2) and CPT_HDR (0xa) */ ++ req->entry_data.kw[0] |= (0x2ULL << la_offset); ++ req->entry_data.kw_mask[0] |= (0x7ULL << la_offset); ++ flow->mcam_data[0] |= (0x2ULL << la_offset); ++ flow->mcam_mask[0] |= (0x7ULL << la_offset); ++ } ++ } + } else { + uint16_t pf_func = (flow->npc_action >> 4) & 0xffff; + +@@ -701,15 +728,16 @@ npc_set_ipv6ext_ltype_mask(struct npc_parse_state *pst) + * because for AH and ESP, LC LFLAG is zero and we don't want to match + * zero in LFLAG. + */ +- lcflag_offset = +- __builtin_popcount(pst->npc->keyx_supp_nmask[pst->nix_intf] & +- ((1ULL << NPC_LFLAG_LC_OFFSET) - 1)); +- lcflag_offset *= 4; +- +- mask = (0xfULL << lcflag_offset); +- val = pst->flow->mcam_data[0] & mask; +- if (val) +- pst->flow->mcam_mask[0] |= mask; ++ if (pst->npc->keyx_supp_nmask[pst->nix_intf] & (1ULL << NPC_LFLAG_LC_OFFSET)) { ++ lcflag_offset = __builtin_popcount(pst->npc->keyx_supp_nmask[pst->nix_intf] & ++ ((1ULL << NPC_LFLAG_LC_OFFSET) - 1)); ++ lcflag_offset *= 4; ++ ++ mask = (0xfULL << lcflag_offset); ++ val = pst->flow->mcam_data[0] & mask; ++ if (val) ++ pst->flow->mcam_mask[0] |= mask; ++ } + } + + int +diff --git a/dpdk/drivers/common/cnxk/roc_npc_mcam_dump.c b/dpdk/drivers/common/cnxk/roc_npc_mcam_dump.c +index fe57811a84..52f7d96b41 100644 +--- a/dpdk/drivers/common/cnxk/roc_npc_mcam_dump.c ++++ b/dpdk/drivers/common/cnxk/roc_npc_mcam_dump.c +@@ -69,8 +69,10 @@ static const char *const ltype_str[NPC_MAX_LID][NPC_MAX_LT] = { + [NPC_LID_LA][NPC_LT_LA_IH_NIX_ETHER] = "LA_IH_NIX_ETHER", + [NPC_LID_LA][NPC_LT_LA_HIGIG2_ETHER] = "LA_HIGIG2_ETHER", + [NPC_LID_LA][NPC_LT_LA_IH_NIX_HIGIG2_ETHER] = "LA_IH_NIX_HIGIG2_ETHER", +- [NPC_LID_LA][NPC_LT_LA_CUSTOM_PRE_L2_ETHER] = +- "NPC_LT_LA_CUSTOM_PRE_L2_ETHER", ++ [NPC_LID_LA][NPC_LT_LA_CUSTOM_L2_90B_ETHER] = "LA_CUSTOM_L2_90B_ETHER", ++ [NPC_LID_LA][NPC_LT_LA_CPT_HDR] = "LA_CPT_HDR", ++ [NPC_LID_LA][NPC_LT_LA_CUSTOM_L2_24B_ETHER] = "LA_CUSTOM_L2_24B_ETHER", ++ [NPC_LID_LA][NPC_LT_LA_CUSTOM_PRE_L2_ETHER] = "NPC_LT_LA_CUSTOM_PRE_L2_ETHER", + [NPC_LID_LB][0] = "NONE", + [NPC_LID_LB][NPC_LT_LB_CTAG] = "LB_CTAG", + [NPC_LID_LB][NPC_LT_LB_STAG_QINQ] = "LB_STAG_QINQ", +@@ -444,6 +446,10 @@ npc_flow_dump_rx_action(FILE *file, uint64_t npc_action) + plt_strlcpy(index_name, "Multicast/mirror table index", + NPC_MAX_FIELD_NAME_SIZE); + break; ++ case NIX_RX_ACTIONOP_DEFAULT: ++ fprintf(file, "NIX_RX_ACTIONOP_DEFAULT (%" PRIu64 ")\n", ++ (uint64_t)NIX_RX_ACTIONOP_DEFAULT); ++ break; + default: + plt_err("Unknown NIX_RX_ACTIONOP found"); + return; +diff --git a/dpdk/drivers/common/cnxk/roc_npc_parse.c b/dpdk/drivers/common/cnxk/roc_npc_parse.c +index ff00c746d6..e695b755d7 100644 +--- a/dpdk/drivers/common/cnxk/roc_npc_parse.c ++++ b/dpdk/drivers/common/cnxk/roc_npc_parse.c +@@ -97,6 +97,7 @@ npc_parse_pre_l2(struct npc_parse_state *pst) + (const struct roc_npc_flow_item_raw *)pst->pattern->mask, &info, + raw_spec_buf, raw_mask_buf); + ++ info.def_mask = NULL; + info.hw_mask = &hw_mask; + npc_get_hw_supp_mask(pst, &info, lid, lt); + +@@ -193,6 +194,7 @@ npc_parse_la(struct npc_parse_state *pst) + if (pst->pattern->type != ROC_NPC_ITEM_TYPE_ETH) + return 0; + ++ pst->has_eth_type = true; + eth_item = pst->pattern->spec; + + lid = NPC_LID_LA; +@@ -238,10 +240,185 @@ npc_parse_la(struct npc_parse_state *pst) + + #define NPC_MAX_SUPPORTED_VLANS 3 + ++static int ++npc_parse_vlan_count(const struct roc_npc_item_info *pattern, ++ const struct roc_npc_item_info **pattern_list, ++ const struct roc_npc_flow_item_vlan **vlan_items, int *vlan_count) ++{ ++ *vlan_count = 0; ++ while (pattern->type == ROC_NPC_ITEM_TYPE_VLAN) { ++ if (*vlan_count > NPC_MAX_SUPPORTED_VLANS - 1) ++ return NPC_ERR_PATTERN_NOTSUP; ++ ++ /* Don't support ranges */ ++ if (pattern->last != NULL) ++ return NPC_ERR_INVALID_RANGE; ++ ++ /* If spec is NULL, both mask and last must be NULL, this ++ * makes it to match ANY value (eq to mask = 0). ++ * Setting either mask or last without spec is an error ++ */ ++ if (pattern->spec == NULL) { ++ if (pattern->last != NULL && pattern->mask != NULL) ++ return NPC_ERR_INVALID_SPEC; ++ } ++ ++ pattern_list[*vlan_count] = pattern; ++ vlan_items[*vlan_count] = pattern->spec; ++ (*vlan_count)++; ++ ++ pattern++; ++ pattern = npc_parse_skip_void_and_any_items(pattern); ++ } ++ ++ return 0; ++} ++ ++static int ++npc_parse_vlan_ltype_get(struct npc_parse_state *pst, ++ const struct roc_npc_flow_item_vlan **vlan_item, int vlan_count, ++ int *ltype, int *lflags) ++{ ++ switch (vlan_count) { ++ case 1: ++ *ltype = NPC_LT_LB_CTAG; ++ if (vlan_item[0] && vlan_item[0]->has_more_vlan) ++ *ltype = NPC_LT_LB_STAG_QINQ; ++ break; ++ case 2: ++ if (vlan_item[1] && vlan_item[1]->has_more_vlan) { ++ if (!(pst->npc->keyx_supp_nmask[pst->nix_intf] & ++ 0x3ULL << NPC_LFLAG_LB_OFFSET)) ++ return NPC_ERR_PATTERN_NOTSUP; ++ ++ /* This lflag value will match either one of ++ * NPC_F_LB_L_WITH_STAG_STAG, ++ * NPC_F_LB_L_WITH_QINQ_CTAG, ++ * NPC_F_LB_L_WITH_QINQ_QINQ and ++ * NPC_F_LB_L_WITH_ITAG (0b0100 to 0b0111). For ++ * NPC_F_LB_L_WITH_ITAG, ltype is NPC_LT_LB_ETAG ++ * hence will not match. ++ */ ++ ++ *lflags = NPC_F_LB_L_WITH_QINQ_CTAG & NPC_F_LB_L_WITH_QINQ_QINQ & ++ NPC_F_LB_L_WITH_STAG_STAG; ++ } ++ *ltype = NPC_LT_LB_STAG_QINQ; ++ break; ++ case 3: ++ if (vlan_item[2] && vlan_item[2]->has_more_vlan) ++ return NPC_ERR_PATTERN_NOTSUP; ++ if (!(pst->npc->keyx_supp_nmask[pst->nix_intf] & 0x3ULL << NPC_LFLAG_LB_OFFSET)) ++ return NPC_ERR_PATTERN_NOTSUP; ++ *ltype = NPC_LT_LB_STAG_QINQ; ++ *lflags = NPC_F_STAG_STAG_CTAG; ++ break; ++ default: ++ return NPC_ERR_PATTERN_NOTSUP; ++ } ++ ++ return 0; ++} ++ ++static int ++npc_update_vlan_parse_state(struct npc_parse_state *pst, const struct roc_npc_item_info *pattern, ++ int lid, int lt, uint8_t lflags, int vlan_count) ++{ ++ uint8_t vlan_spec[NPC_MAX_SUPPORTED_VLANS * sizeof(struct roc_vlan_hdr)]; ++ uint8_t vlan_mask[NPC_MAX_SUPPORTED_VLANS * sizeof(struct roc_vlan_hdr)]; ++ int rc = 0, i, offset = NPC_TPID_LENGTH; ++ struct npc_parse_item_info parse_info; ++ char hw_mask[NPC_MAX_EXTRACT_HW_LEN]; ++ ++ memset(vlan_spec, 0, sizeof(struct roc_vlan_hdr) * NPC_MAX_SUPPORTED_VLANS); ++ memset(vlan_mask, 0, sizeof(struct roc_vlan_hdr) * NPC_MAX_SUPPORTED_VLANS); ++ memset(&parse_info, 0, sizeof(parse_info)); ++ ++ if (vlan_count > 2) ++ vlan_count = 2; ++ ++ for (i = 0; i < vlan_count; i++) { ++ if (pattern[i].spec) ++ memcpy(vlan_spec + offset, pattern[i].spec, sizeof(struct roc_vlan_hdr)); ++ if (pattern[i].mask) ++ memcpy(vlan_mask + offset, pattern[i].mask, sizeof(struct roc_vlan_hdr)); ++ ++ offset += 4; ++ } ++ ++ parse_info.def_mask = NULL; ++ parse_info.spec = vlan_spec; ++ parse_info.mask = vlan_mask; ++ parse_info.def_mask = NULL; ++ parse_info.hw_hdr_len = 0; ++ ++ lid = NPC_LID_LB; ++ parse_info.hw_mask = hw_mask; ++ ++ if (lt == NPC_LT_LB_CTAG) ++ parse_info.len = sizeof(struct roc_vlan_hdr) + NPC_TPID_LENGTH; ++ ++ if (lt == NPC_LT_LB_STAG_QINQ) ++ parse_info.len = sizeof(struct roc_vlan_hdr) * 2 + NPC_TPID_LENGTH; ++ ++ memset(hw_mask, 0, sizeof(hw_mask)); ++ ++ parse_info.hw_mask = &hw_mask; ++ npc_get_hw_supp_mask(pst, &parse_info, lid, lt); ++ ++ rc = npc_mask_is_supported(parse_info.mask, parse_info.hw_mask, parse_info.len); ++ if (!rc) ++ return NPC_ERR_INVALID_MASK; ++ ++ /* Point pattern to last item consumed */ ++ pst->pattern = pattern; ++ return npc_update_parse_state(pst, &parse_info, lid, lt, lflags); ++} ++ ++static int ++npc_parse_lb_vlan(struct npc_parse_state *pst) ++{ ++ const struct roc_npc_flow_item_vlan *vlan_items[NPC_MAX_SUPPORTED_VLANS]; ++ const struct roc_npc_item_info *pattern_list[NPC_MAX_SUPPORTED_VLANS]; ++ const struct roc_npc_item_info *last_pattern; ++ int vlan_count = 0, rc = 0; ++ int lid, lt, lflags; ++ ++ lid = NPC_LID_LB; ++ lflags = 0; ++ last_pattern = pst->pattern; ++ ++ rc = npc_parse_vlan_count(pst->pattern, pattern_list, vlan_items, &vlan_count); ++ if (rc) ++ return rc; ++ ++ rc = npc_parse_vlan_ltype_get(pst, vlan_items, vlan_count, <, &lflags); ++ if (rc) ++ return rc; ++ ++ if (vlan_count == 3) { ++ if (pattern_list[2]->spec != NULL && pattern_list[2]->mask != NULL && ++ pattern_list[2]->last != NULL) ++ return NPC_ERR_PATTERN_NOTSUP; ++ ++ /* Matching can be done only for two tags. */ ++ vlan_count = 2; ++ last_pattern++; ++ } ++ ++ rc = npc_update_vlan_parse_state(pst, pattern_list[0], lid, lt, lflags, vlan_count); ++ if (rc) ++ return rc; ++ ++ if (vlan_count > 1) ++ pst->pattern = last_pattern + vlan_count; ++ ++ return 0; ++} ++ + int + npc_parse_lb(struct npc_parse_state *pst) + { +- const struct roc_npc_flow_item_vlan *vlan_item[NPC_MAX_SUPPORTED_VLANS]; + const struct roc_npc_item_info *pattern = pst->pattern; + const struct roc_npc_item_info *last_pattern; + const struct roc_npc_flow_item_raw *raw_spec; +@@ -250,7 +427,6 @@ npc_parse_lb(struct npc_parse_state *pst) + char hw_mask[NPC_MAX_EXTRACT_HW_LEN]; + struct npc_parse_item_info info; + int lid, lt, lflags, len = 0; +- int nr_vlans = 0; + int rc; + + info.def_mask = NULL; +@@ -267,68 +443,10 @@ npc_parse_lb(struct npc_parse_state *pst) + /* RTE vlan is either 802.1q or 802.1ad, + * this maps to either CTAG/STAG. We need to decide + * based on number of VLANS present. Matching is +- * supported on first tag only. ++ * supported on first two tags. + */ +- info.hw_mask = NULL; +- info.len = sizeof(vlan_item[0]->hdr); +- +- pattern = pst->pattern; +- while (pattern->type == ROC_NPC_ITEM_TYPE_VLAN) { +- if (nr_vlans > NPC_MAX_SUPPORTED_VLANS - 1) +- return NPC_ERR_PATTERN_NOTSUP; +- +- vlan_item[nr_vlans] = pattern->spec; +- nr_vlans++; +- +- /* Basic validation of Second/Third vlan item */ +- if (nr_vlans > 1) { +- rc = npc_parse_item_basic(pattern, &info); +- if (rc != 0) +- return rc; +- } +- last_pattern = pattern; +- pattern++; +- pattern = npc_parse_skip_void_and_any_items(pattern); +- } + +- switch (nr_vlans) { +- case 1: +- lt = NPC_LT_LB_CTAG; +- if (vlan_item[0] && vlan_item[0]->has_more_vlan) +- lt = NPC_LT_LB_STAG_QINQ; +- break; +- case 2: +- if (vlan_item[1] && vlan_item[1]->has_more_vlan) { +- if (!(pst->npc->keyx_supp_nmask[pst->nix_intf] & +- 0x3ULL << NPC_LFLAG_LB_OFFSET)) +- return NPC_ERR_PATTERN_NOTSUP; +- +- /* This lflag value will match either one of +- * NPC_F_LB_L_WITH_STAG_STAG, +- * NPC_F_LB_L_WITH_QINQ_CTAG, +- * NPC_F_LB_L_WITH_QINQ_QINQ and +- * NPC_F_LB_L_WITH_ITAG (0b0100 to 0b0111). For +- * NPC_F_LB_L_WITH_ITAG, ltype is NPC_LT_LB_ETAG +- * hence will not match. +- */ +- +- lflags = NPC_F_LB_L_WITH_QINQ_CTAG & +- NPC_F_LB_L_WITH_QINQ_QINQ & +- NPC_F_LB_L_WITH_STAG_STAG; +- } else { +- lflags = NPC_F_LB_L_WITH_CTAG; +- } +- lt = NPC_LT_LB_STAG_QINQ; +- break; +- case 3: +- if (vlan_item[2] && vlan_item[2]->has_more_vlan) +- return NPC_ERR_PATTERN_NOTSUP; +- lt = NPC_LT_LB_STAG_QINQ; +- lflags = NPC_F_STAG_STAG_CTAG; +- break; +- default: +- return NPC_ERR_PATTERN_NOTSUP; +- } ++ return npc_parse_lb_vlan(pst); + } else if (pst->pattern->type == ROC_NPC_ITEM_TYPE_E_TAG) { + /* we can support ETAG and match a subsequent CTAG + * without any matching support. +@@ -546,10 +664,125 @@ npc_handle_ipv6ext_attr(const struct roc_npc_flow_item_ipv6 *ipv6_spec, + return 0; + } + ++static int ++npc_process_ipv6_item(struct npc_parse_state *pst) ++{ ++ uint8_t ipv6_hdr_mask[sizeof(struct roc_ipv6_hdr) + sizeof(struct roc_ipv6_fragment_ext)]; ++ uint8_t ipv6_hdr_buf[sizeof(struct roc_ipv6_hdr) + sizeof(struct roc_ipv6_fragment_ext)]; ++ const struct roc_npc_flow_item_ipv6 *ipv6_spec, *ipv6_mask; ++ const struct roc_npc_item_info *pattern = pst->pattern; ++ int offset = 0, rc = 0, lid, item_count = 0; ++ struct npc_parse_item_info parse_info; ++ char hw_mask[NPC_MAX_EXTRACT_HW_LEN]; ++ uint8_t flags = 0, ltype; ++ ++ memset(ipv6_hdr_buf, 0, sizeof(ipv6_hdr_buf)); ++ memset(ipv6_hdr_mask, 0, sizeof(ipv6_hdr_mask)); ++ ++ ipv6_spec = pst->pattern->spec; ++ ipv6_mask = pst->pattern->mask; ++ ++ parse_info.def_mask = NULL; ++ parse_info.spec = ipv6_hdr_buf; ++ parse_info.mask = ipv6_hdr_mask; ++ parse_info.def_mask = NULL; ++ parse_info.hw_hdr_len = 0; ++ parse_info.len = sizeof(ipv6_spec->hdr); ++ ++ pst->set_ipv6ext_ltype_mask = true; ++ ++ lid = NPC_LID_LC; ++ ltype = NPC_LT_LC_IP6; ++ ++ if (pattern->type == ROC_NPC_ITEM_TYPE_IPV6) { ++ item_count++; ++ if (ipv6_spec) { ++ memcpy(ipv6_hdr_buf, &ipv6_spec->hdr, sizeof(struct roc_ipv6_hdr)); ++ rc = npc_handle_ipv6ext_attr(ipv6_spec, pst, &flags); ++ if (rc) ++ return rc; ++ } ++ if (ipv6_mask) ++ memcpy(ipv6_hdr_mask, &ipv6_mask->hdr, sizeof(struct roc_ipv6_hdr)); ++ } ++ ++ offset = sizeof(struct roc_ipv6_hdr); ++ ++ while (pattern->type != ROC_NPC_ITEM_TYPE_END) { ++ /* Don't support ranges */ ++ if (pattern->last != NULL) ++ return NPC_ERR_INVALID_RANGE; ++ ++ /* If spec is NULL, both mask and last must be NULL, this ++ * makes it to match ANY value (eq to mask = 0). ++ * Setting either mask or last without spec is ++ * an error ++ */ ++ if (pattern->spec == NULL) { ++ if (pattern->last != NULL && pattern->mask != NULL) ++ return NPC_ERR_INVALID_SPEC; ++ } ++ /* Either one ROC_NPC_ITEM_TYPE_IPV6_EXT or ++ * one ROC_NPC_ITEM_TYPE_IPV6_FRAG_EXT is supported ++ * following an ROC_NPC_ITEM_TYPE_IPV6 item. ++ */ ++ if (pattern->type == ROC_NPC_ITEM_TYPE_IPV6_EXT) { ++ item_count++; ++ ltype = NPC_LT_LC_IP6_EXT; ++ parse_info.len = ++ sizeof(struct roc_ipv6_hdr) + sizeof(struct roc_flow_item_ipv6_ext); ++ if (pattern->spec) ++ memcpy(ipv6_hdr_buf + offset, pattern->spec, ++ sizeof(struct roc_flow_item_ipv6_ext)); ++ if (pattern->mask) ++ memcpy(ipv6_hdr_mask + offset, pattern->mask, ++ sizeof(struct roc_flow_item_ipv6_ext)); ++ break; ++ } else if (pattern->type == ROC_NPC_ITEM_TYPE_IPV6_FRAG_EXT) { ++ item_count++; ++ ltype = NPC_LT_LC_IP6_EXT; ++ flags = NPC_F_LC_U_IP6_FRAG; ++ parse_info.len = ++ sizeof(struct roc_ipv6_hdr) + sizeof(struct roc_ipv6_fragment_ext); ++ if (pattern->spec) ++ memcpy(ipv6_hdr_buf + offset, pattern->spec, ++ sizeof(struct roc_ipv6_fragment_ext)); ++ if (pattern->mask) ++ memcpy(ipv6_hdr_mask + offset, pattern->mask, ++ sizeof(struct roc_ipv6_fragment_ext)); ++ ++ break; ++ } ++ ++ pattern++; ++ pattern = npc_parse_skip_void_and_any_items(pattern); ++ } ++ ++ memset(hw_mask, 0, sizeof(hw_mask)); ++ ++ parse_info.hw_mask = &hw_mask; ++ npc_get_hw_supp_mask(pst, &parse_info, lid, ltype); ++ ++ rc = npc_mask_is_supported(parse_info.mask, parse_info.hw_mask, parse_info.len); ++ if (!rc) ++ return NPC_ERR_INVALID_MASK; ++ ++ rc = npc_update_parse_state(pst, &parse_info, lid, ltype, flags); ++ if (rc) ++ return rc; ++ ++ /* npc_update_parse_state() increments pattern once. ++ * Check if additional increment is required. ++ */ ++ if (item_count == 2) ++ pst->pattern++; ++ ++ return 0; ++} ++ + int + npc_parse_lc(struct npc_parse_state *pst) + { +- const struct roc_npc_flow_item_ipv6 *ipv6_spec; + const struct roc_npc_flow_item_raw *raw_spec; + uint8_t raw_spec_buf[NPC_MAX_RAW_ITEM_LEN]; + uint8_t raw_mask_buf[NPC_MAX_RAW_ITEM_LEN]; +@@ -574,32 +807,12 @@ npc_parse_lc(struct npc_parse_state *pst) + info.len = pst->pattern->size; + break; + case ROC_NPC_ITEM_TYPE_IPV6: +- ipv6_spec = pst->pattern->spec; +- lid = NPC_LID_LC; +- lt = NPC_LT_LC_IP6; +- if (ipv6_spec) { +- rc = npc_handle_ipv6ext_attr(ipv6_spec, pst, &flags); +- if (rc) +- return rc; +- } +- info.len = sizeof(ipv6_spec->hdr); +- break; +- case ROC_NPC_ITEM_TYPE_ARP_ETH_IPV4: +- lt = NPC_LT_LC_ARP; +- info.len = pst->pattern->size; +- break; + case ROC_NPC_ITEM_TYPE_IPV6_EXT: +- lid = NPC_LID_LC; +- lt = NPC_LT_LC_IP6_EXT; +- info.len = pst->pattern->size; +- info.hw_hdr_len = 40; +- break; + case ROC_NPC_ITEM_TYPE_IPV6_FRAG_EXT: +- lid = NPC_LID_LC; +- lt = NPC_LT_LC_IP6_EXT; +- flags = NPC_F_LC_U_IP6_FRAG; ++ return npc_process_ipv6_item(pst); ++ case ROC_NPC_ITEM_TYPE_ARP_ETH_IPV4: ++ lt = NPC_LT_LC_ARP; + info.len = pst->pattern->size; +- info.hw_hdr_len = 40; + break; + case ROC_NPC_ITEM_TYPE_L3_CUSTOM: + lt = NPC_LT_LC_CUSTOM0; +diff --git a/dpdk/drivers/common/cnxk/roc_npc_priv.h b/dpdk/drivers/common/cnxk/roc_npc_priv.h +index 1a597280d1..1de33932e7 100644 +--- a/dpdk/drivers/common/cnxk/roc_npc_priv.h ++++ b/dpdk/drivers/common/cnxk/roc_npc_priv.h +@@ -77,6 +77,9 @@ + #define NPC_LFLAG_LC_OFFSET (NPC_LTYPE_OFFSET_START + 6) + #define NPC_LTYPE_LC_OFFSET (NPC_LTYPE_OFFSET_START + 8) + ++#define CN10K_SDP_CH_START 0x80 ++#define CN10K_SDP_CH_MASK 0xF80 ++ + struct npc_action_vtag_info { + uint16_t vlan_id; + uint16_t vlan_ethtype; +@@ -196,6 +199,7 @@ struct npc_parse_state { + bool set_vlan_ltype_mask; + bool set_ipv6ext_ltype_mask; + bool is_second_pass_rule; ++ bool has_eth_type; + }; + + enum npc_kpu_parser_flag { +@@ -416,17 +420,15 @@ int npc_mcam_alloc_entry(struct npc *npc, struct roc_npc_flow *mcam, + int npc_mcam_alloc_entries(struct npc *npc, int ref_mcam, int *alloc_entry, + int req_count, int prio, int *resp_count); + +-int npc_mcam_ena_dis_entry(struct npc *npc, struct roc_npc_flow *mcam, +- bool enable); ++int npc_mcam_ena_dis_entry(struct npc *npc, struct roc_npc_flow *mcam, bool enable); + int npc_mcam_write_entry(struct npc *npc, struct roc_npc_flow *mcam); + int npc_flow_enable_all_entries(struct npc *npc, bool enable); +-int npc_update_parse_state(struct npc_parse_state *pst, +- struct npc_parse_item_info *info, int lid, int lt, +- uint8_t flags); +-void npc_get_hw_supp_mask(struct npc_parse_state *pst, +- struct npc_parse_item_info *info, int lid, int lt); +-int npc_parse_item_basic(const struct roc_npc_item_info *item, +- struct npc_parse_item_info *info); ++int npc_update_parse_state(struct npc_parse_state *pst, struct npc_parse_item_info *info, int lid, ++ int lt, uint8_t flags); ++void npc_get_hw_supp_mask(struct npc_parse_state *pst, struct npc_parse_item_info *info, int lid, ++ int lt); ++int npc_mask_is_supported(const char *mask, const char *hw_mask, int len); ++int npc_parse_item_basic(const struct roc_npc_item_info *item, struct npc_parse_item_info *info); + int npc_parse_meta_items(struct npc_parse_state *pst); + int npc_parse_mark_item(struct npc_parse_state *pst); + int npc_parse_pre_l2(struct npc_parse_state *pst); +diff --git a/dpdk/drivers/common/cnxk/roc_npc_utils.c b/dpdk/drivers/common/cnxk/roc_npc_utils.c +index 8bdabc116d..fda3073cba 100644 +--- a/dpdk/drivers/common/cnxk/roc_npc_utils.c ++++ b/dpdk/drivers/common/cnxk/roc_npc_utils.c +@@ -88,7 +88,7 @@ npc_get_hw_supp_mask(struct npc_parse_state *pst, + } + } + +-static inline int ++inline int + npc_mask_is_supported(const char *mask, const char *hw_mask, int len) + { + /* +diff --git a/dpdk/drivers/common/cnxk/roc_ree.c b/dpdk/drivers/common/cnxk/roc_ree.c +index 1eb2ae7272..b6392658c3 100644 +--- a/dpdk/drivers/common/cnxk/roc_ree.c ++++ b/dpdk/drivers/common/cnxk/roc_ree.c +@@ -441,7 +441,7 @@ static void + roc_ree_lf_err_intr_unregister(struct roc_ree_vf *vf, uint16_t msix_off, + uintptr_t base) + { +- struct rte_pci_device *pci_dev = vf->pci_dev; ++ struct plt_pci_device *pci_dev = vf->pci_dev; + + /* Disable error interrupts */ + plt_write64(~0ull, base + REE_LF_MISC_INT_ENA_W1C); +@@ -468,7 +468,7 @@ static int + roc_ree_lf_err_intr_register(struct roc_ree_vf *vf, uint16_t msix_off, + uintptr_t base) + { +- struct rte_pci_device *pci_dev = vf->pci_dev; ++ struct plt_pci_device *pci_dev = vf->pci_dev; + int ret; + + /* Disable error interrupts */ +diff --git a/dpdk/drivers/common/cnxk/roc_ree.h b/dpdk/drivers/common/cnxk/roc_ree.h +index e138e4de66..bf994e7124 100644 +--- a/dpdk/drivers/common/cnxk/roc_ree.h ++++ b/dpdk/drivers/common/cnxk/roc_ree.h +@@ -68,7 +68,7 @@ struct roc_ree_qp { + /**< Base address where BAR is mapped */ + struct roc_ree_pending_queue pend_q; + /**< Pending queue */ +- rte_iova_t iq_dma_addr; ++ plt_iova_t iq_dma_addr; + /**< Instruction queue address */ + uint32_t roc_regexdev_jobid; + /**< Job ID */ +diff --git a/dpdk/drivers/common/cnxk/roc_se.h b/dpdk/drivers/common/cnxk/roc_se.h +index c357c19c0b..5b0ddac42d 100644 +--- a/dpdk/drivers/common/cnxk/roc_se.h ++++ b/dpdk/drivers/common/cnxk/roc_se.h +@@ -316,16 +316,15 @@ struct roc_se_ctx { + uint64_t enc_cipher : 8; + uint64_t hash_type : 8; + uint64_t mac_len : 8; +- uint64_t auth_key_len : 8; ++ uint64_t auth_key_len : 16; + uint64_t fc_type : 4; + uint64_t hmac : 1; + uint64_t zsk_flags : 3; + uint64_t k_ecb : 1; + uint64_t pdcp_ci_alg : 2; + uint64_t pdcp_auth_alg : 2; +- uint16_t ciph_then_auth : 1; +- uint16_t auth_then_ciph : 1; +- uint64_t rsvd : 17; ++ uint64_t ciph_then_auth : 1; ++ uint64_t auth_then_ciph : 1; + union cpt_inst_w4 template_w4; + /* Below fields are accessed by hardware */ + union { +diff --git a/dpdk/drivers/common/cnxk/version.map b/dpdk/drivers/common/cnxk/version.map +index 17f0ec6b48..ae9eaf360c 100644 +--- a/dpdk/drivers/common/cnxk/version.map ++++ b/dpdk/drivers/common/cnxk/version.map +@@ -353,6 +353,7 @@ INTERNAL { + roc_npc_mcam_write_entry; + roc_npc_mcam_read_counter; + roc_npc_profile_name_get; ++ roc_npc_sdp_channel_get; + roc_npc_validate_portid_action; + roc_ot_ipsec_inb_sa_init; + roc_ot_ipsec_outb_sa_init; +diff --git a/dpdk/drivers/common/iavf/iavf_common.c b/dpdk/drivers/common/iavf/iavf_common.c +index 855a0ab2f5..dc7662bc1b 100644 +--- a/dpdk/drivers/common/iavf/iavf_common.c ++++ b/dpdk/drivers/common/iavf/iavf_common.c +@@ -27,6 +27,8 @@ enum iavf_status iavf_set_mac_type(struct iavf_hw *hw) + break; + case IAVF_DEV_ID_VF: + case IAVF_DEV_ID_VF_HV: ++ hw->mac.type = IAVF_MAC_XL710; ++ break; + case IAVF_DEV_ID_ADAPTIVE_VF: + hw->mac.type = IAVF_MAC_VF; + break; +diff --git a/dpdk/drivers/common/idpf/base/idpf_common.c b/dpdk/drivers/common/idpf/base/idpf_common.c +index 3a9fdb1878..de82c3458f 100644 +--- a/dpdk/drivers/common/idpf/base/idpf_common.c ++++ b/dpdk/drivers/common/idpf/base/idpf_common.c +@@ -130,6 +130,8 @@ int idpf_init_hw(struct idpf_hw *hw, struct idpf_ctlq_size ctlq_size) + hw->mac.addr[4] = 0x03; + hw->mac.addr[5] = 0x14; + ++ idpf_free(hw, q_info); ++ + return 0; + } + +@@ -146,7 +148,7 @@ int idpf_init_hw(struct idpf_hw *hw, struct idpf_ctlq_size ctlq_size) + * is sent asynchronously, i.e. idpf_asq_send_command() does not wait for + * completion before returning. + */ +-int idpf_send_msg_to_cp(struct idpf_hw *hw, enum virtchnl_ops v_opcode, ++int idpf_send_msg_to_cp(struct idpf_hw *hw, int v_opcode, + int v_retval, u8 *msg, u16 msglen) + { + struct idpf_ctlq_msg ctlq_msg = { 0 }; +@@ -219,6 +221,7 @@ bool idpf_check_asq_alive(struct idpf_hw *hw) + int idpf_clean_arq_element(struct idpf_hw *hw, + struct idpf_arq_event_info *e, u16 *pending) + { ++ struct idpf_dma_mem *dma_mem = NULL; + struct idpf_ctlq_msg msg = { 0 }; + int status; + u16 msg_data_len; +@@ -226,6 +229,8 @@ int idpf_clean_arq_element(struct idpf_hw *hw, + *pending = 1; + + status = idpf_ctlq_recv(hw->arq, pending, &msg); ++ if (status == -ENOMSG) ++ goto exit; + + /* ctlq_msg does not align to ctlq_desc, so copy relevant data here */ + e->desc.opcode = msg.opcode; +@@ -240,7 +245,14 @@ int idpf_clean_arq_element(struct idpf_hw *hw, + msg_data_len = msg.data_len; + idpf_memcpy(e->msg_buf, msg.ctx.indirect.payload->va, msg_data_len, + IDPF_DMA_TO_NONDMA); ++ dma_mem = msg.ctx.indirect.payload; ++ } else { ++ *pending = 0; + } ++ ++ status = idpf_ctlq_post_rx_buffs(hw, hw->arq, pending, &dma_mem); ++ ++exit: + return status; + } + +diff --git a/dpdk/drivers/common/idpf/base/idpf_controlq.c b/dpdk/drivers/common/idpf/base/idpf_controlq.c +index 3af81e5a64..8e4d3ee54f 100644 +--- a/dpdk/drivers/common/idpf/base/idpf_controlq.c ++++ b/dpdk/drivers/common/idpf/base/idpf_controlq.c +@@ -311,18 +311,14 @@ int idpf_ctlq_send(struct idpf_hw *hw, struct idpf_ctlq_info *cq, + + for (i = 0; i < num_q_msg; i++) { + struct idpf_ctlq_msg *msg = &q_msg[i]; +- u64 msg_cookie; + + desc = IDPF_CTLQ_DESC(cq, cq->next_to_use); + + desc->opcode = CPU_TO_LE16(msg->opcode); + desc->pfid_vfid = CPU_TO_LE16(msg->func_id); + +- msg_cookie = *(u64 *)&msg->cookie; +- desc->cookie_high = +- CPU_TO_LE32(IDPF_HI_DWORD(msg_cookie)); +- desc->cookie_low = +- CPU_TO_LE32(IDPF_LO_DWORD(msg_cookie)); ++ desc->cookie_high = CPU_TO_LE32(msg->cookie.mbx.chnl_opcode); ++ desc->cookie_low = CPU_TO_LE32(msg->cookie.mbx.chnl_retval); + + desc->flags = CPU_TO_LE16((msg->host_id & IDPF_HOST_ID_MASK) << + IDPF_CTLQ_FLAG_HOST_ID_S); +@@ -620,8 +616,6 @@ int idpf_ctlq_recv(struct idpf_ctlq_info *cq, u16 *num_q_msg, + num_to_clean = *num_q_msg; + + for (i = 0; i < num_to_clean; i++) { +- u64 msg_cookie; +- + /* Fetch next descriptor and check if marked as done */ + desc = IDPF_CTLQ_DESC(cq, ntc); + flags = LE16_TO_CPU(desc->flags); +@@ -639,10 +633,8 @@ int idpf_ctlq_recv(struct idpf_ctlq_info *cq, u16 *num_q_msg, + if (flags & IDPF_CTLQ_FLAG_ERR) + ret_code = -EBADMSG; + +- msg_cookie = (u64)LE32_TO_CPU(desc->cookie_high) << 32; +- msg_cookie |= (u64)LE32_TO_CPU(desc->cookie_low); +- idpf_memcpy(&q_msg[i].cookie, &msg_cookie, sizeof(u64), +- IDPF_NONDMA_TO_NONDMA); ++ q_msg[i].cookie.mbx.chnl_opcode = LE32_TO_CPU(desc->cookie_high); ++ q_msg[i].cookie.mbx.chnl_retval = LE32_TO_CPU(desc->cookie_low); + + q_msg[i].opcode = LE16_TO_CPU(desc->opcode); + q_msg[i].data_len = LE16_TO_CPU(desc->datalen); +diff --git a/dpdk/drivers/common/idpf/base/idpf_lan_pf_regs.h b/dpdk/drivers/common/idpf/base/idpf_lan_pf_regs.h +index 3df2347bd7..7f731ec3d6 100644 +--- a/dpdk/drivers/common/idpf/base/idpf_lan_pf_regs.h ++++ b/dpdk/drivers/common/idpf/base/idpf_lan_pf_regs.h +@@ -77,8 +77,13 @@ + #define PF_GLINT_DYN_CTL_WB_ON_ITR_M BIT(PF_GLINT_DYN_CTL_WB_ON_ITR_S) + #define PF_GLINT_DYN_CTL_INTENA_MSK_S 31 + #define PF_GLINT_DYN_CTL_INTENA_MSK_M BIT(PF_GLINT_DYN_CTL_INTENA_MSK_S) +-#define PF_GLINT_ITR_V2(_i, _reg_start) (((_i) * 4) + (_reg_start)) +-#define PF_GLINT_ITR(_i, _INT) (PF_GLINT_BASE + (((_i) + 1) * 4) + ((_INT) * 0x1000)) ++/* _ITR is ITR index, _INT is interrupt index, _itrn_indx_spacing is ++ * spacing b/w itrn registers of the same vector. ++ */ ++#define PF_GLINT_ITR_ADDR(_ITR, _reg_start, _itrn_indx_spacing) \ ++ ((_reg_start) + (((_ITR)) * (_itrn_indx_spacing))) ++/* For PF, itrn_indx_spacing is 4 and itrn_reg_spacing is 0x1000 */ ++#define PF_GLINT_ITR(_ITR, _INT) (PF_GLINT_BASE + (((_ITR) + 1) * 4) + ((_INT) * 0x1000)) + #define PF_GLINT_ITR_MAX_INDEX 2 + #define PF_GLINT_ITR_INTERVAL_S 0 + #define PF_GLINT_ITR_INTERVAL_M MAKEMASK(0xFFF, PF_GLINT_ITR_INTERVAL_S) +diff --git a/dpdk/drivers/common/idpf/base/idpf_lan_vf_regs.h b/dpdk/drivers/common/idpf/base/idpf_lan_vf_regs.h +index 9cd4f757d9..13c5c5a7da 100644 +--- a/dpdk/drivers/common/idpf/base/idpf_lan_vf_regs.h ++++ b/dpdk/drivers/common/idpf/base/idpf_lan_vf_regs.h +@@ -90,11 +90,18 @@ + #define VF_INT_DYN_CTLN_WB_ON_ITR_M BIT(VF_INT_DYN_CTLN_WB_ON_ITR_S) + #define VF_INT_DYN_CTLN_INTENA_MSK_S 31 + #define VF_INT_DYN_CTLN_INTENA_MSK_M BIT(VF_INT_DYN_CTLN_INTENA_MSK_S) +-#define VF_INT_ITR0(_i) (0x00004C00 + ((_i) * 4)) +-#define VF_INT_ITRN_V2(_i, _reg_start) ((_reg_start) + (((_i)) * 4)) +-#define VF_INT_ITRN(_i, _INT) (0x00002800 + ((_i) * 4) + ((_INT) * 0x40)) +-#define VF_INT_ITRN_64(_i, _INT) (0x00002C00 + ((_i) * 4) + ((_INT) * 0x100)) +-#define VF_INT_ITRN_2K(_i, _INT) (0x00072000 + ((_i) * 4) + ((_INT) * 0x100)) ++/* _ITR is ITR index, _INT is interrupt index, _itrn_indx_spacing is spacing ++ * b/w itrn registers of the same vector ++ */ ++#define VF_INT_ITR0(_ITR) (0x00004C00 + ((_ITR) * 4)) ++#define VF_INT_ITRN_ADDR(_ITR, _reg_start, _itrn_indx_spacing) \ ++ ((_reg_start) + (((_ITR)) * (_itrn_indx_spacing))) ++/* For VF with 16 vector support, itrn_reg_spacing is 0x4 and itrn_indx_spacing is 0x40 */ ++#define VF_INT_ITRN(_INT, _ITR) (0x00002800 + ((_INT) * 4) + ((_ITR) * 0x40)) ++/* For VF with 64 vector support, itrn_reg_spacing is 0x4 and itrn_indx_spacing is 0x100 */ ++#define VF_INT_ITRN_64(_INT, _ITR) (0x00002C00 + ((_INT) * 4) + ((_ITR) * 0x100)) ++/* For VF with 2k vector support, itrn_reg_spacing is 0x4 and itrn_indx_spacing is 0x2000 */ ++#define VF_INT_ITRN_2K(_INT, _ITR) (0x00072000 + ((_INT) * 4) + ((_ITR) * 0x2000)) + #define VF_INT_ITRN_MAX_INDEX 2 + #define VF_INT_ITRN_INTERVAL_S 0 + #define VF_INT_ITRN_INTERVAL_M MAKEMASK(0xFFF, VF_INT_ITRN_INTERVAL_S) +diff --git a/dpdk/drivers/common/idpf/base/idpf_prototype.h b/dpdk/drivers/common/idpf/base/idpf_prototype.h +index 529b62212d..3ce25e644d 100644 +--- a/dpdk/drivers/common/idpf/base/idpf_prototype.h ++++ b/dpdk/drivers/common/idpf/base/idpf_prototype.h +@@ -40,6 +40,6 @@ int idpf_set_rss_key(struct idpf_hw *hw, u16 seid, + int idpf_set_mac_type(struct idpf_hw *hw); + + int idpf_reset(struct idpf_hw *hw); +-int idpf_send_msg_to_cp(struct idpf_hw *hw, enum virtchnl_ops v_opcode, ++int idpf_send_msg_to_cp(struct idpf_hw *hw, int v_opcode, + int v_retval, u8 *msg, u16 msglen); + #endif /* _IDPF_PROTOTYPE_H_ */ +diff --git a/dpdk/drivers/common/mlx5/linux/meson.build b/dpdk/drivers/common/mlx5/linux/meson.build +index 7e1575efc8..b13ae29844 100644 +--- a/dpdk/drivers/common/mlx5/linux/meson.build ++++ b/dpdk/drivers/common/mlx5/linux/meson.build +@@ -217,6 +217,8 @@ has_sym_args = [ + 'ibv_import_device' ], + [ 'HAVE_MLX5DV_DR_ACTION_CREATE_DEST_ROOT_TABLE', 'infiniband/mlx5dv.h', + 'mlx5dv_dr_action_create_dest_root_table' ], ++ [ 'HAVE_IBV_FORK_UNNEEDED', 'infiniband/verbs.h', ++ 'ibv_is_fork_initialized'], + ] + if libmtcr_ul_found + has_sym_args += [ +diff --git a/dpdk/drivers/common/mlx5/linux/mlx5_common_os.c b/dpdk/drivers/common/mlx5/linux/mlx5_common_os.c +index aafff60eeb..41345e1597 100644 +--- a/dpdk/drivers/common/mlx5/linux/mlx5_common_os.c ++++ b/dpdk/drivers/common/mlx5/linux/mlx5_common_os.c +@@ -96,10 +96,11 @@ mlx5_translate_port_name(const char *port_name_in, + char ctrl = 0, pf_c1, pf_c2, vf_c1, vf_c2, eol; + char *end; + int sc_items; ++ int32_t ctrl_num = -1; + +- sc_items = sscanf(port_name_in, "%c%d", +- &ctrl, &port_info_out->ctrl_num); ++ sc_items = sscanf(port_name_in, "%c%d", &ctrl, &ctrl_num); + if (sc_items == 2 && ctrl == 'c') { ++ port_info_out->ctrl_num = ctrl_num; + port_name_in++; /* 'c' */ + port_name_in += snprintf(NULL, 0, "%d", + port_info_out->ctrl_num); +@@ -266,7 +267,7 @@ mlx5_glue_path(char *buf, size_t size) + goto error; + return buf; + error: +- RTE_LOG(ERR, PMD, "unable to append \"-glue\" to last component of" ++ DRV_LOG(ERR, "unable to append \"-glue\" to last component of" + " RTE_EAL_PMD_PATH (\"" RTE_EAL_PMD_PATH "\"), please" + " re-configure DPDK"); + return NULL; +@@ -555,7 +556,7 @@ mlx5_os_pd_prepare(struct mlx5_common_device *cdev) + } + + static struct ibv_device * +-mlx5_os_get_ibv_device(const struct rte_pci_addr *addr) ++mlx5_os_get_ibv_device(const struct rte_pci_device *pci_dev) + { + int n; + struct ibv_device **ibv_list = mlx5_glue->get_device_list(&n); +@@ -564,6 +565,8 @@ mlx5_os_get_ibv_device(const struct rte_pci_addr *addr) + uint8_t guid2[32] = {0}; + int ret1, ret2 = -1; + struct rte_pci_addr paddr; ++ const struct rte_pci_addr *addr = &pci_dev->addr; ++ bool is_vf_dev = mlx5_dev_is_vf_pci(pci_dev); + + if (ibv_list == NULL || !n) { + rte_errno = ENOSYS; +@@ -579,11 +582,11 @@ mlx5_os_get_ibv_device(const struct rte_pci_addr *addr) + if (ret1 > 0) + ret2 = mlx5_get_device_guid(&paddr, guid2, sizeof(guid2)); + /* Bond device can bond secondary PCIe */ +- if ((strstr(ibv_list[n]->name, "bond") && +- ((ret1 > 0 && ret2 > 0 && !memcmp(guid1, guid2, sizeof(guid1))) || +- (addr->domain == paddr.domain && addr->bus == paddr.bus && +- addr->devid == paddr.devid))) || +- !rte_pci_addr_cmp(addr, &paddr)) { ++ if ((strstr(ibv_list[n]->name, "bond") && !is_vf_dev && ++ ((ret1 > 0 && ret2 > 0 && !memcmp(guid1, guid2, sizeof(guid1))) || ++ (addr->domain == paddr.domain && addr->bus == paddr.bus && ++ addr->devid == paddr.devid))) || ++ !rte_pci_addr_cmp(addr, &paddr)) { + ibv_match = ibv_list[n]; + break; + } +@@ -697,7 +700,7 @@ mlx5_os_get_ibv_dev(const struct rte_device *dev) + struct ibv_device *ibv; + + if (mlx5_dev_is_pci(dev)) +- ibv = mlx5_os_get_ibv_device(&RTE_DEV_TO_PCI_CONST(dev)->addr); ++ ibv = mlx5_os_get_ibv_device(RTE_DEV_TO_PCI_CONST(dev)); + else + ibv = mlx5_get_aux_ibv_device(RTE_DEV_TO_AUXILIARY_CONST(dev)); + if (ibv == NULL) { +diff --git a/dpdk/drivers/common/mlx5/linux/mlx5_glue.c b/dpdk/drivers/common/mlx5/linux/mlx5_glue.c +index 702eb36b62..88b99fe029 100644 +--- a/dpdk/drivers/common/mlx5/linux/mlx5_glue.c ++++ b/dpdk/drivers/common/mlx5/linux/mlx5_glue.c +@@ -19,6 +19,10 @@ + static int + mlx5_glue_fork_init(void) + { ++#ifdef HAVE_IBV_FORK_UNNEEDED ++ if (ibv_is_fork_initialized() == IBV_FORK_UNNEEDED) ++ return 0; /* ibv_fork_init() not needed */ ++#endif + return ibv_fork_init(); + } + +diff --git a/dpdk/drivers/common/mlx5/meson.build b/dpdk/drivers/common/mlx5/meson.build +index 60ccd95cbc..9dc809f192 100644 +--- a/dpdk/drivers/common/mlx5/meson.build ++++ b/dpdk/drivers/common/mlx5/meson.build +@@ -1,9 +1,14 @@ + # SPDX-License-Identifier: BSD-3-Clause + # Copyright 2019 Mellanox Technologies, Ltd + +-if not (is_linux or (is_windows and is_ms_linker)) ++if not (is_linux or is_windows) + build = false +- reason = 'only supported on Linux and Windows build with clang' ++ reason = 'only supported on Linux and Windows' ++ subdir_done() ++endif ++if is_windows and not is_ms_linker and not meson.is_cross_build() ++ build = false ++ reason = 'MinGW is supported only for cross-compilation test' + subdir_done() + endif + +diff --git a/dpdk/drivers/common/mlx5/mlx5_common.h b/dpdk/drivers/common/mlx5/mlx5_common.h +index d6e91b5296..02b5d54363 100644 +--- a/dpdk/drivers/common/mlx5/mlx5_common.h ++++ b/dpdk/drivers/common/mlx5/mlx5_common.h +@@ -203,7 +203,12 @@ check_cqe(volatile struct mlx5_cqe *cqe, const uint16_t cqes_n, + + if (unlikely((op_owner != (!!(idx))) || (op_code == MLX5_CQE_INVALID))) + return MLX5_CQE_STATUS_HW_OWN; +- rte_io_rmb(); ++ ++ /* Prevent speculative reading of other fields in CQE until ++ * CQE is valid. ++ */ ++ rte_atomic_thread_fence(__ATOMIC_ACQUIRE); ++ + if (unlikely(op_code == MLX5_CQE_RESP_ERR || + op_code == MLX5_CQE_REQ_ERR)) + return MLX5_CQE_STATUS_ERR; +@@ -221,6 +226,7 @@ check_cqe(volatile struct mlx5_cqe *cqe, const uint16_t cqes_n, + * - 0 on success. + * - Negative value and rte_errno is set otherwise. + */ ++__rte_internal + int mlx5_dev_to_pci_str(const struct rte_device *dev, char *addr, size_t size); + + /* +@@ -552,7 +558,7 @@ mlx5_dev_is_pci(const struct rte_device *dev); + */ + __rte_internal + bool +-mlx5_dev_is_vf_pci(struct rte_pci_device *pci_dev); ++mlx5_dev_is_vf_pci(const struct rte_pci_device *pci_dev); + + __rte_internal + int +diff --git a/dpdk/drivers/common/mlx5/mlx5_common_pci.c b/dpdk/drivers/common/mlx5/mlx5_common_pci.c +index 73178ce0f3..fdf03f2a53 100644 +--- a/dpdk/drivers/common/mlx5/mlx5_common_pci.c ++++ b/dpdk/drivers/common/mlx5/mlx5_common_pci.c +@@ -109,7 +109,7 @@ mlx5_dev_is_pci(const struct rte_device *dev) + } + + bool +-mlx5_dev_is_vf_pci(struct rte_pci_device *pci_dev) ++mlx5_dev_is_vf_pci(const struct rte_pci_device *pci_dev) + { + switch (pci_dev->id.device_id) { + case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF: +diff --git a/dpdk/drivers/common/mlx5/mlx5_devx_cmds.c b/dpdk/drivers/common/mlx5/mlx5_devx_cmds.c +index 59cebb530f..9fdca2fecc 100644 +--- a/dpdk/drivers/common/mlx5/mlx5_devx_cmds.c ++++ b/dpdk/drivers/common/mlx5/mlx5_devx_cmds.c +@@ -543,7 +543,7 @@ mlx5_devx_cmd_query_hca_vdpa_attr(void *ctx, + MLX5_GET_HCA_CAP_OP_MOD_VDPA_EMULATION | + MLX5_HCA_CAP_OPMOD_GET_CUR); + if (!hcattr) { +- RTE_LOG(DEBUG, PMD, "Failed to query devx VDPA capabilities"); ++ DRV_LOG(DEBUG, "Failed to query devx VDPA capabilities"); + vdpa_attr->valid = 0; + } else { + vdpa_attr->valid = 1; +@@ -1002,6 +1002,8 @@ mlx5_devx_cmd_query_hca_attr(void *ctx, + general_obj_types) & + MLX5_GENERAL_OBJ_TYPES_CAP_CONN_TRACK_OFFLOAD); + attr->rq_delay_drop = MLX5_GET(cmd_hca_cap, hcattr, rq_delay_drop); ++ attr->nic_flow_table = MLX5_GET(cmd_hca_cap, hcattr, nic_flow_table); ++ attr->nic_flow_table = MLX5_GET(cmd_hca_cap, hcattr, nic_flow_table); + attr->max_flow_counter_15_0 = MLX5_GET(cmd_hca_cap, hcattr, + max_flow_counter_15_0); + attr->max_flow_counter_31_16 = MLX5_GET(cmd_hca_cap, hcattr, +@@ -1013,7 +1015,9 @@ mlx5_devx_cmd_query_hca_attr(void *ctx, + attr->flow_access_aso_opc_mod = MLX5_GET(cmd_hca_cap, hcattr, + flow_access_aso_opc_mod); + if (attr->crypto) { +- attr->aes_xts = MLX5_GET(cmd_hca_cap, hcattr, aes_xts); ++ attr->aes_xts = MLX5_GET(cmd_hca_cap, hcattr, aes_xts) || ++ MLX5_GET(cmd_hca_cap, hcattr, aes_xts_multi_block_be_tweak) || ++ MLX5_GET(cmd_hca_cap, hcattr, aes_xts_single_block_le_tweak); + hcattr = mlx5_devx_get_hca_cap(ctx, in, out, &rc, + MLX5_GET_HCA_CAP_OP_MOD_CRYPTO | + MLX5_HCA_CAP_OPMOD_GET_CUR); +diff --git a/dpdk/drivers/common/mlx5/mlx5_prm.h b/dpdk/drivers/common/mlx5/mlx5_prm.h +index 2b5c43ee6e..dab70b9469 100644 +--- a/dpdk/drivers/common/mlx5/mlx5_prm.h ++++ b/dpdk/drivers/common/mlx5/mlx5_prm.h +@@ -1679,7 +1679,9 @@ struct mlx5_ifc_cmd_hca_cap_bits { + u8 log_min_hairpin_wq_data_sz[0x5]; + u8 reserved_at_3e8[0x3]; + u8 log_max_vlan_list[0x5]; +- u8 reserved_at_3f0[0x3]; ++ u8 reserved_at_3f0[0x1]; ++ u8 aes_xts_single_block_le_tweak[1]; ++ u8 aes_xts_multi_block_be_tweak[1]; + u8 log_max_current_mc_list[0x5]; + u8 reserved_at_3f8[0x3]; + u8 log_max_current_uc_list[0x5]; +@@ -2121,10 +2123,11 @@ struct mlx5_ifc_cmd_hca_cap_2_bits { + u8 hairpin_sq_wqe_bb_size[0x5]; + u8 hairpin_sq_wq_in_host_mem[0x1]; + u8 hairpin_data_buffer_locked[0x1]; +- u8 reserved_at_16a[0x36]; +- u8 reserved_at_1a0[0xb]; ++ u8 reserved_at_16a[0x16]; ++ u8 reserved_at_180[0x20]; ++ u8 reserved_at_1a0[0xa]; + u8 format_select_dw_8_6_ext[0x1]; +- u8 reserved_at_1ac[0x14]; ++ u8 reserved_at_1ac[0x15]; + u8 general_obj_types_127_64[0x40]; + u8 reserved_at_200[0x53]; + u8 flow_counter_bulk_log_max_alloc[0x5]; +@@ -3040,6 +3043,7 @@ struct mlx5_ifc_health_buffer_bits { + u8 ext_synd[0x10]; + }; + ++/* HCA PCI BAR resource structure. */ + struct mlx5_ifc_initial_seg_bits { + u8 fw_rev_minor[0x10]; + u8 fw_rev_major[0x10]; +@@ -3067,7 +3071,9 @@ struct mlx5_ifc_initial_seg_bits { + u8 clear_int[0x1]; + u8 health_syndrome[0x8]; + u8 health_counter[0x18]; +- u8 reserved_8[0x17fc0]; ++ u8 reserved_8[0x160]; ++ u8 real_time[0x40]; ++ u8 reserved_9[0x17e20]; + }; + + struct mlx5_ifc_create_cq_out_bits { +diff --git a/dpdk/drivers/common/mlx5/version.map b/dpdk/drivers/common/mlx5/version.map +index 4f72900519..03c8ce5593 100644 +--- a/dpdk/drivers/common/mlx5/version.map ++++ b/dpdk/drivers/common/mlx5/version.map +@@ -14,6 +14,7 @@ INTERNAL { + + mlx5_dev_is_pci; + mlx5_dev_is_vf_pci; ++ mlx5_dev_to_pci_str; + mlx5_dev_mempool_unregister; + mlx5_dev_mempool_subscribe; + +diff --git a/dpdk/drivers/common/mlx5/windows/mlx5_win_defs.h b/dpdk/drivers/common/mlx5/windows/mlx5_win_defs.h +index 3554e4a7ff..65da820c5e 100644 +--- a/dpdk/drivers/common/mlx5/windows/mlx5_win_defs.h ++++ b/dpdk/drivers/common/mlx5/windows/mlx5_win_defs.h +@@ -2,8 +2,10 @@ + * Copyright (C) Mellanox Technologies, Ltd. 2001-2020. + */ + +-#ifndef __MLX5_WIN_DEFS_H__ +-#define __MLX5_WIN_DEFS_H__ ++#ifndef MLX5_WIN_DEFS_H ++#define MLX5_WIN_DEFS_H ++ ++#include + + enum { + MLX5_CQE_OWNER_MASK = 1, +@@ -40,29 +42,29 @@ enum { + }; + + enum mlx5dv_cq_init_attr_mask { +- MLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE = 1 << 0, +- MLX5DV_CQ_INIT_ATTR_MASK_FLAGS = 1 << 1, +- MLX5DV_CQ_INIT_ATTR_MASK_CQE_SIZE = 1 << 2, ++ MLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE = RTE_BIT32(0), ++ MLX5DV_CQ_INIT_ATTR_MASK_FLAG = RTE_BIT32(1), ++ MLX5DV_CQ_INIT_ATTR_MASK_CQE_SIZE = RTE_BIT32(2), + }; + + enum mlx5dv_cqe_comp_res_format { +- MLX5DV_CQE_RES_FORMAT_HASH = 1 << 0, +- MLX5DV_CQE_RES_FORMAT_CSUM = 1 << 1, +- MLX5DV_CQE_RES_FORMAT_CSUM_STRIDX = 1 << 2, ++ MLX5DV_CQE_RES_FORMAT_HASH = RTE_BIT32(0), ++ MLX5DV_CQE_RES_FORMAT_CSUM = RTE_BIT32(1), ++ MLX5DV_CQE_RES_FORMAT_CSUM_STRIDX = RTE_BIT32(2), + }; + + enum ibv_access_flags { +- IBV_ACCESS_LOCAL_WRITE = 1, +- IBV_ACCESS_REMOTE_WRITE = 1 << 1, +- IBV_ACCESS_REMOTE_READ = 1 << 2, +- IBV_ACCESS_REMOTE_ATOMIC = 1 << 3, +- IBV_ACCESS_MW_BIND = 1 << 4, +- IBV_ACCESS_ZERO_BASED = 1 << 5, +- IBV_ACCESS_ON_DEMAND = 1 << 6, ++ IBV_ACCESS_LOCAL_WRITE = RTE_BIT32(0), ++ IBV_ACCESS_REMOTE_WRITE = RTE_BIT32(1), ++ IBV_ACCESS_REMOTE_READ = RTE_BIT32(2), ++ IBV_ACCESS_REMOTE_ATOMIC = RTE_BIT32(3), ++ IBV_ACCESS_MW_BIND = RTE_BIT32(4), ++ IBV_ACCESS_ZERO_BASED = RTE_BIT32(5), ++ IBV_ACCESS_ON_DEMAND = RTE_BIT32(6), + }; + + enum mlx5_ib_uapi_devx_create_event_channel_flags { +- MLX5_IB_UAPI_DEVX_CR_EV_CH_FLAGS_OMIT_DATA = 1 << 0, ++ MLX5_IB_UAPI_DEVX_CR_EV_CH_FLAGS_OMIT_DATA = RTE_BIT32(0), + }; + + #define MLX5DV_DEVX_CREATE_EVENT_CHANNEL_FLAGS_OMIT_EV_DATA \ +@@ -85,15 +87,15 @@ enum { + }; + + enum { +- MLX5_ETH_WQE_L3_CSUM = (1 << 6), +- MLX5_ETH_WQE_L4_CSUM = (1 << 7), ++ MLX5_ETH_WQE_L3_CSUM = RTE_BIT32(6), ++ MLX5_ETH_WQE_L4_CSUM = RTE_BIT32(7), + }; + + enum { +- MLX5_WQE_CTRL_CQ_UPDATE = 2 << 2, +- MLX5_WQE_CTRL_SOLICITED = 1 << 1, +- MLX5_WQE_CTRL_FENCE = 4 << 5, +- MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE = 1 << 5, ++ MLX5_WQE_CTRL_SOLICITED = RTE_BIT32(1), ++ MLX5_WQE_CTRL_CQ_UPDATE = RTE_BIT32(3), ++ MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE = RTE_BIT32(5), ++ MLX5_WQE_CTRL_FENCE = RTE_BIT32(7), + }; + + enum { +@@ -101,6 +103,11 @@ enum { + MLX5_SEND_WQE_SHIFT = 6, + }; + ++/* Verbs headers do not support -pedantic. */ ++#ifdef PEDANTIC ++#pragma GCC diagnostic ignored "-Wpedantic" ++#endif ++ + /* + * RX Hash fields enable to set which incoming packet's field should + * participates in RX Hash. Each flag represent certain packet's field, +@@ -110,18 +117,22 @@ enum { + * TCP and UDP flags can't be enabled together on the same QP. + */ + enum ibv_rx_hash_fields { +- IBV_RX_HASH_SRC_IPV4 = 1 << 0, +- IBV_RX_HASH_DST_IPV4 = 1 << 1, +- IBV_RX_HASH_SRC_IPV6 = 1 << 2, +- IBV_RX_HASH_DST_IPV6 = 1 << 3, +- IBV_RX_HASH_SRC_PORT_TCP = 1 << 4, +- IBV_RX_HASH_DST_PORT_TCP = 1 << 5, +- IBV_RX_HASH_SRC_PORT_UDP = 1 << 6, +- IBV_RX_HASH_DST_PORT_UDP = 1 << 7, +- IBV_RX_HASH_IPSEC_SPI = 1 << 8, +- IBV_RX_HASH_INNER = (1 << 31), ++ IBV_RX_HASH_SRC_IPV4 = RTE_BIT32(0), ++ IBV_RX_HASH_DST_IPV4 = RTE_BIT32(1), ++ IBV_RX_HASH_SRC_IPV6 = RTE_BIT32(2), ++ IBV_RX_HASH_DST_IPV6 = RTE_BIT32(3), ++ IBV_RX_HASH_SRC_PORT_TCP = RTE_BIT32(4), ++ IBV_RX_HASH_DST_PORT_TCP = RTE_BIT32(5), ++ IBV_RX_HASH_SRC_PORT_UDP = RTE_BIT32(6), ++ IBV_RX_HASH_DST_PORT_UDP = RTE_BIT32(7), ++ IBV_RX_HASH_IPSEC_SPI = RTE_BIT32(8), ++ IBV_RX_HASH_INNER = RTE_BIT32(31), + }; + ++#ifdef PEDANTIC ++#pragma GCC diagnostic error "-Wpedantic" ++#endif ++ + enum { + MLX5_RCV_DBR = 0, + MLX5_SND_DBR = 1, +@@ -141,9 +152,9 @@ enum { + #endif + + enum ibv_flow_flags { +- IBV_FLOW_ATTR_FLAGS_ALLOW_LOOP_BACK = 1 << 0, +- IBV_FLOW_ATTR_FLAGS_DONT_TRAP = 1 << 1, +- IBV_FLOW_ATTR_FLAGS_EGRESS = 1 << 2, ++ IBV_FLOW_ATTR_FLAGS_ALLOW_LOOP_BACK = RTE_BIT32(0), ++ IBV_FLOW_ATTR_FLAGS_DONT_TRAP = RTE_BIT32(1), ++ IBV_FLOW_ATTR_FLAGS_EGRESS = RTE_BIT32(2), + }; + + enum ibv_flow_attr_type { +@@ -240,11 +251,11 @@ struct mlx5_wqe_data_seg { + rte_be64_t addr; + }; + +-#define MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP (1 << 4) +-#define IBV_DEVICE_RAW_IP_CSUM (1 << 26) +-#define IBV_RAW_PACKET_CAP_CVLAN_STRIPPING (1 << 0) +-#define IBV_RAW_PACKET_CAP_SCATTER_FCS (1 << 1) +-#define IBV_QPT_RAW_PACKET 8 ++#define MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP RTE_BIT32(4) ++#define IBV_DEVICE_RAW_IP_CSUM RTE_BIT32(26) ++#define IBV_RAW_PACKET_CAP_CVLAN_STRIPPING RTE_BIT32(0) ++#define IBV_RAW_PACKET_CAP_SCATTER_FCS RTE_BIT32(1) ++#define IBV_QPT_RAW_PACKET 8 + + enum { + MLX5_FLOW_CONTEXT_DEST_TYPE_VPORT = 0x0, +@@ -254,8 +265,9 @@ enum { + }; + + enum { +- MLX5_MATCH_OUTER_HEADERS = 1 << 0, +- MLX5_MATCH_MISC_PARAMETERS = 1 << 1, +- MLX5_MATCH_INNER_HEADERS = 1 << 2, ++ MLX5_MATCH_OUTER_HEADERS = RTE_BIT32(0), ++ MLX5_MATCH_MISC_PARAMETERS = RTE_BIT32(1), ++ MLX5_MATCH_INNER_HEADERS = RTE_BIT32(2), + }; +-#endif /* __MLX5_WIN_DEFS_H__ */ ++ ++#endif /* MLX5_WIN_DEFS_H */ +diff --git a/dpdk/drivers/common/qat/dev/qat_dev_gen1.c b/dpdk/drivers/common/qat/dev/qat_dev_gen1.c +index cf480dcba8..dd2e878e90 100644 +--- a/dpdk/drivers/common/qat/dev/qat_dev_gen1.c ++++ b/dpdk/drivers/common/qat/dev/qat_dev_gen1.c +@@ -242,7 +242,7 @@ qat_dev_get_extra_size_gen1(void) + } + + static int +-qat_get_dev_slice_map_gen1(uint16_t *map __rte_unused, ++qat_get_dev_slice_map_gen1(uint32_t *map __rte_unused, + const struct rte_pci_device *pci_dev __rte_unused) + { + return 0; +diff --git a/dpdk/drivers/common/qat/dev/qat_dev_gen2.c b/dpdk/drivers/common/qat/dev/qat_dev_gen2.c +index f51be46eb0..061dfdb698 100644 +--- a/dpdk/drivers/common/qat/dev/qat_dev_gen2.c ++++ b/dpdk/drivers/common/qat/dev/qat_dev_gen2.c +@@ -22,7 +22,7 @@ static struct qat_qp_hw_spec_funcs qat_qp_hw_spec_gen2 = { + }; + + static int +-qat_dev_get_slice_map_gen2(uint16_t *map __rte_unused, ++qat_dev_get_slice_map_gen2(uint32_t *map __rte_unused, + const struct rte_pci_device *pci_dev __rte_unused) + { + return 0; +diff --git a/dpdk/drivers/common/qat/dev/qat_dev_gen3.c b/dpdk/drivers/common/qat/dev/qat_dev_gen3.c +index e4197f3c0f..f01b98ff86 100644 +--- a/dpdk/drivers/common/qat/dev/qat_dev_gen3.c ++++ b/dpdk/drivers/common/qat/dev/qat_dev_gen3.c +@@ -68,7 +68,7 @@ static struct qat_qp_hw_spec_funcs qat_qp_hw_spec_gen3 = { + }; + + static int +-qat_dev_get_slice_map_gen3(uint16_t *map, ++qat_dev_get_slice_map_gen3(uint32_t *map, + const struct rte_pci_device *pci_dev) + { + if (rte_pci_read_config(pci_dev, map, +diff --git a/dpdk/drivers/common/qat/dev/qat_dev_gen4.c b/dpdk/drivers/common/qat/dev/qat_dev_gen4.c +index 1b3a5deabf..1ce262f715 100644 +--- a/dpdk/drivers/common/qat/dev/qat_dev_gen4.c ++++ b/dpdk/drivers/common/qat/dev/qat_dev_gen4.c +@@ -283,7 +283,7 @@ qat_dev_get_misc_bar_gen4(struct rte_mem_resource **mem_resource, + } + + static int +-qat_dev_get_slice_map_gen4(uint16_t *map __rte_unused, ++qat_dev_get_slice_map_gen4(uint32_t *map __rte_unused, + const struct rte_pci_device *pci_dev __rte_unused) + { + return 0; +diff --git a/dpdk/drivers/common/qat/meson.build b/dpdk/drivers/common/qat/meson.build +index b84e5b3c6c..95b52b78c3 100644 +--- a/dpdk/drivers/common/qat/meson.build ++++ b/dpdk/drivers/common/qat/meson.build +@@ -54,14 +54,6 @@ if libipsecmb.found() and libcrypto_3.found() + endif + endif + +-# The driver should not build if both compression and crypto are disabled +-#FIXME common code depends on compression files so check only compress! +-if not qat_compress # and not qat_crypto +- build = false +- reason = '' # rely on reason for compress/crypto above +- subdir_done() +-endif +- + deps += ['bus_pci', 'cryptodev', 'net', 'compressdev'] + sources += files( + 'qat_common.c', +diff --git a/dpdk/drivers/common/qat/qat_device.c b/dpdk/drivers/common/qat/qat_device.c +index 8bce2ac073..ed75b66041 100644 +--- a/dpdk/drivers/common/qat/qat_device.c ++++ b/dpdk/drivers/common/qat/qat_device.c +@@ -361,7 +361,7 @@ static int qat_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, + { + int sym_ret = 0, asym_ret = 0, comp_ret = 0; + int num_pmds_created = 0; +- uint16_t capa = 0; ++ uint32_t capa = 0; + struct qat_pci_device *qat_pci_dev; + struct qat_dev_hw_spec_funcs *ops_hw; + struct qat_dev_cmd_param qat_dev_cmd_param[] = { +diff --git a/dpdk/drivers/common/qat/qat_device.h b/dpdk/drivers/common/qat/qat_device.h +index bc3da04238..4a79cdded3 100644 +--- a/dpdk/drivers/common/qat/qat_device.h ++++ b/dpdk/drivers/common/qat/qat_device.h +@@ -37,7 +37,7 @@ typedef int (*qat_dev_get_misc_bar_t) + typedef int (*qat_dev_read_config_t) + (struct qat_pci_device *); + typedef int (*qat_dev_get_extra_size_t)(void); +-typedef int (*qat_dev_get_slice_map_t)(uint16_t *map, ++typedef int (*qat_dev_get_slice_map_t)(uint32_t *map, + const struct rte_pci_device *pci_dev); + + struct qat_dev_hw_spec_funcs { +diff --git a/dpdk/drivers/common/qat/qat_qp.c b/dpdk/drivers/common/qat/qat_qp.c +index 9cbd19a481..e95df292e8 100644 +--- a/dpdk/drivers/common/qat/qat_qp.c ++++ b/dpdk/drivers/common/qat/qat_qp.c +@@ -449,20 +449,6 @@ adf_configure_queues(struct qat_qp *qp, enum qat_device_gen qat_dev_gen) + return 0; + } + +-static inline void +-txq_write_tail(enum qat_device_gen qat_dev_gen, +- struct qat_qp *qp, struct qat_queue *q) +-{ +- struct qat_qp_hw_spec_funcs *ops = +- qat_qp_hw_spec[qat_dev_gen]; +- +- /* +- * Pointer check should be done during +- * initialization +- */ +- ops->qat_qp_csr_write_tail(qp, q); +-} +- + static inline void + qat_qp_csr_write_head(enum qat_device_gen qat_dev_gen, struct qat_qp *qp, + struct qat_queue *q, uint32_t new_head) +@@ -631,179 +617,6 @@ kick_tail: + return nb_ops_sent; + } + +-/* Use this for compression only - but keep consistent with above common +- * function as much as possible. +- */ +-uint16_t +-qat_enqueue_comp_op_burst(void *qp, void **ops, uint16_t nb_ops) +-{ +- register struct qat_queue *queue; +- struct qat_qp *tmp_qp = (struct qat_qp *)qp; +- register uint32_t nb_ops_sent = 0; +- register int nb_desc_to_build; +- uint16_t nb_ops_possible = nb_ops; +- register uint8_t *base_addr; +- register uint32_t tail; +- +- int descriptors_built, total_descriptors_built = 0; +- int nb_remaining_descriptors; +- int overflow = 0; +- +- if (unlikely(nb_ops == 0)) +- return 0; +- +- /* read params used a lot in main loop into registers */ +- queue = &(tmp_qp->tx_q); +- base_addr = (uint8_t *)queue->base_addr; +- tail = queue->tail; +- +- /* Find how many can actually fit on the ring */ +- { +- /* dequeued can only be written by one thread, but it may not +- * be this thread. As it's 4-byte aligned it will be read +- * atomically here by any Intel CPU. +- * enqueued can wrap before dequeued, but cannot +- * lap it as var size of enq/deq (uint32_t) > var size of +- * max_inflights (uint16_t). In reality inflights is never +- * even as big as max uint16_t, as it's <= ADF_MAX_DESC. +- * On wrapping, the calculation still returns the correct +- * positive value as all three vars are unsigned. +- */ +- uint32_t inflights = +- tmp_qp->enqueued - tmp_qp->dequeued; +- +- /* Find how many can actually fit on the ring */ +- overflow = (inflights + nb_ops) - tmp_qp->max_inflights; +- if (overflow > 0) { +- nb_ops_possible = nb_ops - overflow; +- if (nb_ops_possible == 0) +- return 0; +- } +- +- /* QAT has plenty of work queued already, so don't waste cycles +- * enqueueing, wait til the application has gathered a bigger +- * burst or some completed ops have been dequeued +- */ +- if (tmp_qp->min_enq_burst_threshold && inflights > +- QAT_QP_MIN_INFL_THRESHOLD && nb_ops_possible < +- tmp_qp->min_enq_burst_threshold) { +- tmp_qp->stats.threshold_hit_count++; +- return 0; +- } +- } +- +- /* At this point nb_ops_possible is assuming a 1:1 mapping +- * between ops and descriptors. +- * Fewer may be sent if some ops have to be split. +- * nb_ops_possible is <= burst size. +- * Find out how many spaces are actually available on the qp in case +- * more are needed. +- */ +- nb_remaining_descriptors = nb_ops_possible +- + ((overflow >= 0) ? 0 : overflow * (-1)); +- QAT_DP_LOG(DEBUG, "Nb ops requested %d, nb descriptors remaining %d", +- nb_ops, nb_remaining_descriptors); +- +- while (nb_ops_sent != nb_ops_possible && +- nb_remaining_descriptors > 0) { +- struct qat_comp_op_cookie *cookie = +- tmp_qp->op_cookies[tail >> queue->trailz]; +- +- descriptors_built = 0; +- +- QAT_DP_LOG(DEBUG, "--- data length: %u", +- ((struct rte_comp_op *)*ops)->src.length); +- +- nb_desc_to_build = qat_comp_build_request(*ops, +- base_addr + tail, cookie, tmp_qp->qat_dev_gen); +- QAT_DP_LOG(DEBUG, "%d descriptors built, %d remaining, " +- "%d ops sent, %d descriptors needed", +- total_descriptors_built, nb_remaining_descriptors, +- nb_ops_sent, nb_desc_to_build); +- +- if (unlikely(nb_desc_to_build < 0)) { +- /* this message cannot be enqueued */ +- tmp_qp->stats.enqueue_err_count++; +- if (nb_ops_sent == 0) +- return 0; +- goto kick_tail; +- } else if (unlikely(nb_desc_to_build > 1)) { +- /* this op is too big and must be split - get more +- * descriptors and retry +- */ +- +- QAT_DP_LOG(DEBUG, "Build %d descriptors for this op", +- nb_desc_to_build); +- +- nb_remaining_descriptors -= nb_desc_to_build; +- if (nb_remaining_descriptors >= 0) { +- /* There are enough remaining descriptors +- * so retry +- */ +- int ret2 = qat_comp_build_multiple_requests( +- *ops, tmp_qp, tail, +- nb_desc_to_build); +- +- if (unlikely(ret2 < 1)) { +- QAT_DP_LOG(DEBUG, +- "Failed to build (%d) descriptors, status %d", +- nb_desc_to_build, ret2); +- +- qat_comp_free_split_op_memzones(cookie, +- nb_desc_to_build - 1); +- +- tmp_qp->stats.enqueue_err_count++; +- +- /* This message cannot be enqueued */ +- if (nb_ops_sent == 0) +- return 0; +- goto kick_tail; +- } else { +- descriptors_built = ret2; +- total_descriptors_built += +- descriptors_built; +- nb_remaining_descriptors -= +- descriptors_built; +- QAT_DP_LOG(DEBUG, +- "Multiple descriptors (%d) built ok", +- descriptors_built); +- } +- } else { +- QAT_DP_LOG(ERR, "For the current op, number of requested descriptors (%d) " +- "exceeds number of available descriptors (%d)", +- nb_desc_to_build, +- nb_remaining_descriptors + +- nb_desc_to_build); +- +- qat_comp_free_split_op_memzones(cookie, +- nb_desc_to_build - 1); +- +- /* Not enough extra descriptors */ +- if (nb_ops_sent == 0) +- return 0; +- goto kick_tail; +- } +- } else { +- descriptors_built = 1; +- total_descriptors_built++; +- nb_remaining_descriptors--; +- QAT_DP_LOG(DEBUG, "Single descriptor built ok"); +- } +- +- tail = adf_modulo(tail + (queue->msg_size * descriptors_built), +- queue->modulo_mask); +- ops++; +- nb_ops_sent++; +- } +- +-kick_tail: +- queue->tail = tail; +- tmp_qp->enqueued += total_descriptors_built; +- tmp_qp->stats.enqueued_count += nb_ops_sent; +- txq_write_tail(tmp_qp->qat_dev_gen, tmp_qp, queue); +- return nb_ops_sent; +-} +- + uint16_t + qat_dequeue_op_burst(void *qp, void **ops, + qat_op_dequeue_t qat_dequeue_process_response, uint16_t nb_ops) +diff --git a/dpdk/drivers/common/qat/qat_qp.h b/dpdk/drivers/common/qat/qat_qp.h +index 66f00943a5..f911125e86 100644 +--- a/dpdk/drivers/common/qat/qat_qp.h ++++ b/dpdk/drivers/common/qat/qat_qp.h +@@ -127,9 +127,6 @@ uint16_t + qat_enqueue_op_burst(void *qp, qat_op_build_request_t op_build_request, + void **ops, uint16_t nb_ops); + +-uint16_t +-qat_enqueue_comp_op_burst(void *qp, void **ops, uint16_t nb_ops); +- + uint16_t + qat_dequeue_op_burst(void *qp, void **ops, + qat_op_dequeue_t qat_dequeue_process_response, uint16_t nb_ops); +@@ -201,6 +198,21 @@ struct qat_qp_hw_spec_funcs { + qat_qp_get_hw_data_t qat_qp_get_hw_data; + }; + +-extern struct qat_qp_hw_spec_funcs *qat_qp_hw_spec[]; ++extern struct qat_qp_hw_spec_funcs* ++ qat_qp_hw_spec[]; ++ ++static inline void ++txq_write_tail(enum qat_device_gen qat_dev_gen, ++ struct qat_qp *qp, struct qat_queue *q) ++{ ++ struct qat_qp_hw_spec_funcs *ops = ++ qat_qp_hw_spec[qat_dev_gen]; ++ ++ /* ++ * Pointer check should be done during ++ * initialization ++ */ ++ ops->qat_qp_csr_write_tail(qp, q); ++} + + #endif /* _QAT_QP_H_ */ +diff --git a/dpdk/drivers/common/sfc_efx/base/efx.h b/dpdk/drivers/common/sfc_efx/base/efx.h +index 92ec18761b..49e29dcc1c 100644 +--- a/dpdk/drivers/common/sfc_efx/base/efx.h ++++ b/dpdk/drivers/common/sfc_efx/base/efx.h +@@ -4582,6 +4582,24 @@ efx_mae_action_set_populate_mark( + __in efx_mae_actions_t *spec, + __in uint32_t mark_value); + ++/* ++ * Whilst efx_mae_action_set_populate_mark() can be used to request setting ++ * a user mark in matching packets and demands that the request come before ++ * setting the final destination (deliver action), this API can be invoked ++ * after deliver action has been added in order to request mark reset if ++ * the user's own mark request has not been added as a result of parsing. ++ * ++ * It is useful when the driver chains an outer rule (OR) with an action ++ * rule (AR) by virtue of a recirculation ID. The OR may set mark from ++ * this ID to help the driver identify packets that hit the OR and do ++ * not hit the AR. But, for packets that do hit the AR, the driver ++ * wants to reset the mark value to avoid confusing recipients. ++ */ ++LIBEFX_API ++extern void ++efx_mae_action_set_populate_mark_reset( ++ __in efx_mae_actions_t *spec); ++ + LIBEFX_API + extern __checkReturn efx_rc_t + efx_mae_action_set_populate_deliver( +@@ -4730,6 +4748,20 @@ efx_mae_action_set_fill_in_counter_id( + __in efx_mae_actions_t *spec, + __in const efx_counter_t *counter_idp); + ++/* ++ * Clears dangling FW object IDs (counter ID, for instance) in ++ * the action set specification. Useful for adapter restarts, ++ * when all MAE objects need to be reallocated by the driver. ++ * ++ * This method only clears the IDs in the specification. ++ * The driver is still responsible for keeping the IDs ++ * separately and freeing them when stopping the port. ++ */ ++LIBEFX_API ++extern void ++efx_mae_action_set_clear_fw_rsrc_ids( ++ __in efx_mae_actions_t *spec); ++ + /* Action set ID */ + typedef struct efx_mae_aset_id_s { + uint32_t id; +diff --git a/dpdk/drivers/common/sfc_efx/base/efx_impl.h b/dpdk/drivers/common/sfc_efx/base/efx_impl.h +index 9a5d465fa0..45e99d01c5 100644 +--- a/dpdk/drivers/common/sfc_efx/base/efx_impl.h ++++ b/dpdk/drivers/common/sfc_efx/base/efx_impl.h +@@ -1800,6 +1800,10 @@ typedef struct efx_mae_action_vlan_push_s { + uint16_t emavp_tci_be; + } efx_mae_action_vlan_push_t; + ++/* ++ * Helper efx_mae_action_set_clear_fw_rsrc_ids() is responsible ++ * to initialise every field in this structure to INVALID value. ++ */ + typedef struct efx_mae_actions_rsrc_s { + efx_mae_mac_id_t emar_dst_mac_id; + efx_mae_mac_id_t emar_src_mac_id; +diff --git a/dpdk/drivers/common/sfc_efx/base/efx_mae.c b/dpdk/drivers/common/sfc_efx/base/efx_mae.c +index 31f51b5548..4c33471f28 100644 +--- a/dpdk/drivers/common/sfc_efx/base/efx_mae.c ++++ b/dpdk/drivers/common/sfc_efx/base/efx_mae.c +@@ -1394,10 +1394,7 @@ efx_mae_action_set_spec_init( + goto fail1; + } + +- spec->ema_rsrc.emar_dst_mac_id.id = EFX_MAE_RSRC_ID_INVALID; +- spec->ema_rsrc.emar_src_mac_id.id = EFX_MAE_RSRC_ID_INVALID; +- spec->ema_rsrc.emar_eh_id.id = EFX_MAE_RSRC_ID_INVALID; +- spec->ema_rsrc.emar_counter_id.id = EFX_MAE_RSRC_ID_INVALID; ++ efx_mae_action_set_clear_fw_rsrc_ids(spec); + + /* + * Helpers which populate v2 actions must reject them when v2 is not +@@ -1916,6 +1913,18 @@ efx_mae_action_set_populate_mark( + EFX_MAE_ACTION_MARK, sizeof (mark_value), arg)); + } + ++ void ++efx_mae_action_set_populate_mark_reset( ++ __in efx_mae_actions_t *spec) ++{ ++ uint32_t action_mask = (1U << EFX_MAE_ACTION_MARK); ++ ++ if ((spec->ema_actions & action_mask) == 0) { ++ spec->ema_actions |= action_mask; ++ spec->ema_mark_value = 0; ++ } ++} ++ + __checkReturn efx_rc_t + efx_mae_action_set_populate_deliver( + __in efx_mae_actions_t *spec, +@@ -3015,6 +3024,16 @@ fail1: + return (rc); + } + ++ void ++efx_mae_action_set_clear_fw_rsrc_ids( ++ __in efx_mae_actions_t *spec) ++{ ++ spec->ema_rsrc.emar_dst_mac_id.id = EFX_MAE_RSRC_ID_INVALID; ++ spec->ema_rsrc.emar_src_mac_id.id = EFX_MAE_RSRC_ID_INVALID; ++ spec->ema_rsrc.emar_eh_id.id = EFX_MAE_RSRC_ID_INVALID; ++ spec->ema_rsrc.emar_counter_id.id = EFX_MAE_RSRC_ID_INVALID; ++} ++ + __checkReturn efx_rc_t + efx_mae_counters_alloc( + __in efx_nic_t *enp, +diff --git a/dpdk/drivers/common/sfc_efx/base/efx_rx.c b/dpdk/drivers/common/sfc_efx/base/efx_rx.c +index 68f42f5cac..61726a9f0b 100644 +--- a/dpdk/drivers/common/sfc_efx/base/efx_rx.c ++++ b/dpdk/drivers/common/sfc_efx/base/efx_rx.c +@@ -937,8 +937,10 @@ efx_rx_qcreate_internal( + + rss_hash_field = + &erplp->erpl_fields[EFX_RX_PREFIX_FIELD_RSS_HASH]; +- if (rss_hash_field->erpfi_width_bits == 0) ++ if (rss_hash_field->erpfi_width_bits == 0) { ++ rc = ENOTSUP; + goto fail5; ++ } + } + + enp->en_rx_qcount++; +diff --git a/dpdk/drivers/common/sfc_efx/version.map b/dpdk/drivers/common/sfc_efx/version.map +index a54aab0a08..d9b04a611d 100644 +--- a/dpdk/drivers/common/sfc_efx/version.map ++++ b/dpdk/drivers/common/sfc_efx/version.map +@@ -89,6 +89,7 @@ INTERNAL { + efx_mae_action_rule_insert; + efx_mae_action_rule_remove; + efx_mae_action_set_alloc; ++ efx_mae_action_set_clear_fw_rsrc_ids; + efx_mae_action_set_fill_in_counter_id; + efx_mae_action_set_fill_in_dst_mac_id; + efx_mae_action_set_fill_in_eh_id; +@@ -103,6 +104,7 @@ INTERNAL { + efx_mae_action_set_populate_encap; + efx_mae_action_set_populate_flag; + efx_mae_action_set_populate_mark; ++ efx_mae_action_set_populate_mark_reset; + efx_mae_action_set_populate_set_dst_mac; + efx_mae_action_set_populate_set_src_mac; + efx_mae_action_set_populate_vlan_pop; +diff --git a/dpdk/drivers/compress/mlx5/mlx5_compress.c b/dpdk/drivers/compress/mlx5/mlx5_compress.c +index fb2bda9745..c4bf62ed41 100644 +--- a/dpdk/drivers/compress/mlx5/mlx5_compress.c ++++ b/dpdk/drivers/compress/mlx5/mlx5_compress.c +@@ -96,9 +96,7 @@ static const struct rte_compressdev_capabilities mlx5_caps[] = { + RTE_COMP_FF_HUFFMAN_DYNAMIC, + .window_size = {.min = 10, .max = 15, .increment = 1}, + }, +- { +- .algo = RTE_COMP_ALGO_LIST_END, +- } ++ RTE_COMP_END_OF_CAPABILITIES_LIST() + }; + + static void +@@ -245,8 +243,8 @@ mlx5_compress_qp_setup(struct rte_compressdev *dev, uint16_t qp_id, + mlx5_ts_format_conv(priv->cdev->config.hca_attr.qp_ts_format); + qp_attr.num_of_receive_wqes = 0; + qp_attr.num_of_send_wqbbs = RTE_BIT32(log_ops_n); +- qp_attr.mmo = priv->mmo_decomp_qp && priv->mmo_comp_qp +- && priv->mmo_dma_qp; ++ qp_attr.mmo = priv->mmo_decomp_qp || priv->mmo_comp_qp || ++ priv->mmo_dma_qp; + ret = mlx5_devx_qp_create(priv->cdev->ctx, &qp->qp, + qp_attr.num_of_send_wqbbs * + MLX5_WQE_SIZE, &qp_attr, socket_id); +@@ -315,7 +313,7 @@ mlx5_compress_xform_create(struct rte_compressdev *dev, + DRV_LOG(ERR, "Not enough capabilities to support decompress operation, maybe old FW/OFED version?"); + return -ENOTSUP; + } +- if (xform->compress.hash_algo != RTE_COMP_HASH_ALGO_NONE) { ++ if (xform->decompress.hash_algo != RTE_COMP_HASH_ALGO_NONE) { + DRV_LOG(ERR, "SHA is not supported."); + return -ENOTSUP; + } +@@ -635,7 +633,7 @@ mlx5_compress_dequeue_burst(void *queue_pair, struct rte_comp_op **ops, + break; + case RTE_COMP_CHECKSUM_ADLER32: + op->output_chksum = (uint64_t)rte_be_to_cpu_32 +- (opaq[idx].adler32) << 32; ++ (opaq[idx].adler32); + break; + case RTE_COMP_CHECKSUM_CRC32_ADLER32: + op->output_chksum = (uint64_t)rte_be_to_cpu_32 +diff --git a/dpdk/drivers/compress/qat/dev/qat_comp_pmd_gen1.c b/dpdk/drivers/compress/qat/dev/qat_comp_pmd_gen1.c +index 12d9d89072..3a8484eef1 100644 +--- a/dpdk/drivers/compress/qat/dev/qat_comp_pmd_gen1.c ++++ b/dpdk/drivers/compress/qat/dev/qat_comp_pmd_gen1.c +@@ -26,7 +26,7 @@ const struct rte_compressdev_capabilities qat_gen1_comp_capabilities[] = { + RTE_COMP_FF_OOP_LB_IN_SGL_OUT | + RTE_COMP_FF_STATEFUL_DECOMPRESSION, + .window_size = {.min = 15, .max = 15, .increment = 0} }, +- {RTE_COMP_ALGO_LIST_END, 0, {0, 0, 0} } }; ++ RTE_COMP_END_OF_CAPABILITIES_LIST() }; + + static int + qat_comp_dev_config_gen1(struct rte_compressdev *dev, +diff --git a/dpdk/drivers/compress/qat/dev/qat_comp_pmd_gen4.c b/dpdk/drivers/compress/qat/dev/qat_comp_pmd_gen4.c +index 79b2ceb414..05906f13e0 100644 +--- a/dpdk/drivers/compress/qat/dev/qat_comp_pmd_gen4.c ++++ b/dpdk/drivers/compress/qat/dev/qat_comp_pmd_gen4.c +@@ -25,7 +25,7 @@ qat_gen4_comp_capabilities[] = { + RTE_COMP_FF_OOP_SGL_IN_LB_OUT | + RTE_COMP_FF_OOP_LB_IN_SGL_OUT, + .window_size = {.min = 15, .max = 15, .increment = 0} }, +- {RTE_COMP_ALGO_LIST_END, 0, {0, 0, 0} } }; ++ RTE_COMP_END_OF_CAPABILITIES_LIST() }; + + static int + qat_comp_dev_config_gen4(struct rte_compressdev *dev, +diff --git a/dpdk/drivers/compress/qat/qat_comp.c b/dpdk/drivers/compress/qat/qat_comp.c +index fe4a4999c6..559948a46a 100644 +--- a/dpdk/drivers/compress/qat/qat_comp.c ++++ b/dpdk/drivers/compress/qat/qat_comp.c +@@ -1144,3 +1144,185 @@ qat_comp_stream_free(struct rte_compressdev *dev, void *stream) + } + return -EINVAL; + } ++ ++/** ++ * Enqueue packets for processing on queue pair of a device ++ * ++ * @param qp ++ * qat queue pair ++ * @param ops ++ * Compressdev operation ++ * @param nb_ops ++ * number of operations ++ * @return ++ * - nb_ops_sent if successful ++ */ ++uint16_t ++qat_enqueue_comp_op_burst(void *qp, void **ops, uint16_t nb_ops) ++{ ++ register struct qat_queue *queue; ++ struct qat_qp *tmp_qp = (struct qat_qp *)qp; ++ register uint32_t nb_ops_sent = 0; ++ register int nb_desc_to_build; ++ uint16_t nb_ops_possible = nb_ops; ++ register uint8_t *base_addr; ++ register uint32_t tail; ++ ++ int descriptors_built, total_descriptors_built = 0; ++ int nb_remaining_descriptors; ++ int overflow = 0; ++ ++ if (unlikely(nb_ops == 0)) ++ return 0; ++ ++ /* read params used a lot in main loop into registers */ ++ queue = &(tmp_qp->tx_q); ++ base_addr = (uint8_t *)queue->base_addr; ++ tail = queue->tail; ++ ++ /* Find how many can actually fit on the ring */ ++ { ++ /* dequeued can only be written by one thread, but it may not ++ * be this thread. As it's 4-byte aligned it will be read ++ * atomically here by any Intel CPU. ++ * enqueued can wrap before dequeued, but cannot ++ * lap it as var size of enq/deq (uint32_t) > var size of ++ * max_inflights (uint16_t). In reality inflights is never ++ * even as big as max uint16_t, as it's <= ADF_MAX_DESC. ++ * On wrapping, the calculation still returns the correct ++ * positive value as all three vars are unsigned. ++ */ ++ uint32_t inflights = ++ tmp_qp->enqueued - tmp_qp->dequeued; ++ ++ /* Find how many can actually fit on the ring */ ++ overflow = (inflights + nb_ops) - tmp_qp->max_inflights; ++ if (overflow > 0) { ++ nb_ops_possible = nb_ops - overflow; ++ if (nb_ops_possible == 0) ++ return 0; ++ } ++ ++ /* QAT has plenty of work queued already, so don't waste cycles ++ * enqueueing, wait til the application has gathered a bigger ++ * burst or some completed ops have been dequeued ++ */ ++ if (tmp_qp->min_enq_burst_threshold && inflights > ++ QAT_QP_MIN_INFL_THRESHOLD && nb_ops_possible < ++ tmp_qp->min_enq_burst_threshold) { ++ tmp_qp->stats.threshold_hit_count++; ++ return 0; ++ } ++ } ++ ++ /* At this point nb_ops_possible is assuming a 1:1 mapping ++ * between ops and descriptors. ++ * Fewer may be sent if some ops have to be split. ++ * nb_ops_possible is <= burst size. ++ * Find out how many spaces are actually available on the qp in case ++ * more are needed. ++ */ ++ nb_remaining_descriptors = nb_ops_possible ++ + ((overflow >= 0) ? 0 : overflow * (-1)); ++ QAT_DP_LOG(DEBUG, "Nb ops requested %d, nb descriptors remaining %d", ++ nb_ops, nb_remaining_descriptors); ++ ++ while (nb_ops_sent != nb_ops_possible && ++ nb_remaining_descriptors > 0) { ++ struct qat_comp_op_cookie *cookie = ++ tmp_qp->op_cookies[tail >> queue->trailz]; ++ ++ descriptors_built = 0; ++ ++ QAT_DP_LOG(DEBUG, "--- data length: %u", ++ ((struct rte_comp_op *)*ops)->src.length); ++ ++ nb_desc_to_build = qat_comp_build_request(*ops, ++ base_addr + tail, cookie, tmp_qp->qat_dev_gen); ++ QAT_DP_LOG(DEBUG, "%d descriptors built, %d remaining, " ++ "%d ops sent, %d descriptors needed", ++ total_descriptors_built, nb_remaining_descriptors, ++ nb_ops_sent, nb_desc_to_build); ++ ++ if (unlikely(nb_desc_to_build < 0)) { ++ /* this message cannot be enqueued */ ++ tmp_qp->stats.enqueue_err_count++; ++ if (nb_ops_sent == 0) ++ return 0; ++ goto kick_tail; ++ } else if (unlikely(nb_desc_to_build > 1)) { ++ /* this op is too big and must be split - get more ++ * descriptors and retry ++ */ ++ ++ QAT_DP_LOG(DEBUG, "Build %d descriptors for this op", ++ nb_desc_to_build); ++ ++ nb_remaining_descriptors -= nb_desc_to_build; ++ if (nb_remaining_descriptors >= 0) { ++ /* There are enough remaining descriptors ++ * so retry ++ */ ++ int ret2 = qat_comp_build_multiple_requests( ++ *ops, tmp_qp, tail, ++ nb_desc_to_build); ++ ++ if (unlikely(ret2 < 1)) { ++ QAT_DP_LOG(DEBUG, ++ "Failed to build (%d) descriptors, status %d", ++ nb_desc_to_build, ret2); ++ ++ qat_comp_free_split_op_memzones(cookie, ++ nb_desc_to_build - 1); ++ ++ tmp_qp->stats.enqueue_err_count++; ++ ++ /* This message cannot be enqueued */ ++ if (nb_ops_sent == 0) ++ return 0; ++ goto kick_tail; ++ } else { ++ descriptors_built = ret2; ++ total_descriptors_built += ++ descriptors_built; ++ nb_remaining_descriptors -= ++ descriptors_built; ++ QAT_DP_LOG(DEBUG, ++ "Multiple descriptors (%d) built ok", ++ descriptors_built); ++ } ++ } else { ++ QAT_DP_LOG(ERR, "For the current op, number of requested descriptors (%d) " ++ "exceeds number of available descriptors (%d)", ++ nb_desc_to_build, ++ nb_remaining_descriptors + ++ nb_desc_to_build); ++ ++ qat_comp_free_split_op_memzones(cookie, ++ nb_desc_to_build - 1); ++ ++ /* Not enough extra descriptors */ ++ if (nb_ops_sent == 0) ++ return 0; ++ goto kick_tail; ++ } ++ } else { ++ descriptors_built = 1; ++ total_descriptors_built++; ++ nb_remaining_descriptors--; ++ QAT_DP_LOG(DEBUG, "Single descriptor built ok"); ++ } ++ ++ tail = adf_modulo(tail + (queue->msg_size * descriptors_built), ++ queue->modulo_mask); ++ ops++; ++ nb_ops_sent++; ++ } ++ ++kick_tail: ++ queue->tail = tail; ++ tmp_qp->enqueued += total_descriptors_built; ++ tmp_qp->stats.enqueued_count += nb_ops_sent; ++ txq_write_tail(tmp_qp->qat_dev_gen, tmp_qp, queue); ++ return nb_ops_sent; ++} +diff --git a/dpdk/drivers/compress/qat/qat_comp.h b/dpdk/drivers/compress/qat/qat_comp.h +index da7b9a6eec..dc220cd6e3 100644 +--- a/dpdk/drivers/compress/qat/qat_comp.h ++++ b/dpdk/drivers/compress/qat/qat_comp.h +@@ -141,5 +141,8 @@ qat_comp_stream_create(struct rte_compressdev *dev, + int + qat_comp_stream_free(struct rte_compressdev *dev, void *stream); + ++uint16_t ++qat_enqueue_comp_op_burst(void *qp, void **ops, uint16_t nb_ops); ++ + #endif + #endif +diff --git a/dpdk/drivers/crypto/ccp/ccp_crypto.c b/dpdk/drivers/crypto/ccp/ccp_crypto.c +index b21b32e507..4b84b3303e 100644 +--- a/dpdk/drivers/crypto/ccp/ccp_crypto.c ++++ b/dpdk/drivers/crypto/ccp/ccp_crypto.c +@@ -26,15 +26,12 @@ + + #include "ccp_dev.h" + #include "ccp_crypto.h" +-#include "ccp_pci.h" + #include "ccp_pmd_private.h" + + #include + #include + #include + +-extern int iommu_mode; +-void *sha_ctx; + /* SHA initial context values */ + uint32_t ccp_sha1_init[SHA_COMMON_DIGEST_SIZE / sizeof(uint32_t)] = { + SHA1_H4, SHA1_H3, +@@ -748,13 +745,8 @@ ccp_configure_session_cipher(struct ccp_session *sess, + CCP_LOG_ERR("Invalid CCP Engine"); + return -ENOTSUP; + } +- if (iommu_mode == 2) { +- sess->cipher.nonce_phys = rte_mem_virt2iova(sess->cipher.nonce); +- sess->cipher.key_phys = rte_mem_virt2iova(sess->cipher.key_ccp); +- } else { +- sess->cipher.nonce_phys = rte_mem_virt2phy(sess->cipher.nonce); +- sess->cipher.key_phys = rte_mem_virt2phy(sess->cipher.key_ccp); +- } ++ sess->cipher.nonce_phys = rte_mem_virt2iova(sess->cipher.nonce); ++ sess->cipher.key_phys = rte_mem_virt2iova(sess->cipher.key_ccp); + return 0; + } + +@@ -793,7 +785,6 @@ ccp_configure_session_auth(struct ccp_session *sess, + sess->auth.ctx = (void *)ccp_sha1_init; + sess->auth.ctx_len = CCP_SB_BYTES; + sess->auth.offset = CCP_SB_BYTES - SHA1_DIGEST_SIZE; +- rte_memcpy(sha_ctx, sess->auth.ctx, SHA_COMMON_DIGEST_SIZE); + break; + case RTE_CRYPTO_AUTH_SHA1_HMAC: + if (sess->auth_opt) { +@@ -832,7 +823,6 @@ ccp_configure_session_auth(struct ccp_session *sess, + sess->auth.ctx = (void *)ccp_sha224_init; + sess->auth.ctx_len = CCP_SB_BYTES; + sess->auth.offset = CCP_SB_BYTES - SHA224_DIGEST_SIZE; +- rte_memcpy(sha_ctx, sess->auth.ctx, SHA256_DIGEST_SIZE); + break; + case RTE_CRYPTO_AUTH_SHA224_HMAC: + if (sess->auth_opt) { +@@ -895,7 +885,6 @@ ccp_configure_session_auth(struct ccp_session *sess, + sess->auth.ctx = (void *)ccp_sha256_init; + sess->auth.ctx_len = CCP_SB_BYTES; + sess->auth.offset = CCP_SB_BYTES - SHA256_DIGEST_SIZE; +- rte_memcpy(sha_ctx, sess->auth.ctx, SHA256_DIGEST_SIZE); + break; + case RTE_CRYPTO_AUTH_SHA256_HMAC: + if (sess->auth_opt) { +@@ -958,7 +947,6 @@ ccp_configure_session_auth(struct ccp_session *sess, + sess->auth.ctx = (void *)ccp_sha384_init; + sess->auth.ctx_len = CCP_SB_BYTES << 1; + sess->auth.offset = (CCP_SB_BYTES << 1) - SHA384_DIGEST_SIZE; +- rte_memcpy(sha_ctx, sess->auth.ctx, SHA512_DIGEST_SIZE); + break; + case RTE_CRYPTO_AUTH_SHA384_HMAC: + if (sess->auth_opt) { +@@ -1023,7 +1011,6 @@ ccp_configure_session_auth(struct ccp_session *sess, + sess->auth.ctx = (void *)ccp_sha512_init; + sess->auth.ctx_len = CCP_SB_BYTES << 1; + sess->auth.offset = (CCP_SB_BYTES << 1) - SHA512_DIGEST_SIZE; +- rte_memcpy(sha_ctx, sess->auth.ctx, SHA512_DIGEST_SIZE); + break; + case RTE_CRYPTO_AUTH_SHA512_HMAC: + if (sess->auth_opt) { +@@ -1173,13 +1160,8 @@ ccp_configure_session_aead(struct ccp_session *sess, + CCP_LOG_ERR("Unsupported aead algo"); + return -ENOTSUP; + } +- if (iommu_mode == 2) { +- sess->cipher.nonce_phys = rte_mem_virt2iova(sess->cipher.nonce); +- sess->cipher.key_phys = rte_mem_virt2iova(sess->cipher.key_ccp); +- } else { +- sess->cipher.nonce_phys = rte_mem_virt2phy(sess->cipher.nonce); +- sess->cipher.key_phys = rte_mem_virt2phy(sess->cipher.key_ccp); +- } ++ sess->cipher.nonce_phys = rte_mem_virt2iova(sess->cipher.nonce); ++ sess->cipher.key_phys = rte_mem_virt2iova(sess->cipher.key_ccp); + return 0; + } + +@@ -1592,14 +1574,8 @@ ccp_perform_hmac(struct rte_crypto_op *op, + op->sym->auth.data.offset); + append_ptr = (void *)rte_pktmbuf_append(op->sym->m_src, + session->auth.ctx_len); +- if (iommu_mode == 2) { +- dest_addr = (phys_addr_t)rte_mem_virt2iova(append_ptr); +- pst.src_addr = (phys_addr_t)rte_mem_virt2iova((void *)addr); +- } else { +- dest_addr = (phys_addr_t)rte_mem_virt2phy(append_ptr); +- pst.src_addr = (phys_addr_t)rte_mem_virt2phy((void *)addr); +- } +- dest_addr_t = dest_addr; ++ dest_addr_t = dest_addr = (phys_addr_t)rte_mem_virt2iova(append_ptr); ++ pst.src_addr = (phys_addr_t)rte_mem_virt2iova((void *)addr); + + /** Load PHash1 to LSB*/ + pst.dest_addr = (phys_addr_t)(cmd_q->sb_sha * CCP_SB_BYTES); +@@ -1681,10 +1657,7 @@ ccp_perform_hmac(struct rte_crypto_op *op, + + /** Load PHash2 to LSB*/ + addr += session->auth.ctx_len; +- if (iommu_mode == 2) +- pst.src_addr = (phys_addr_t)rte_mem_virt2iova((void *)addr); +- else +- pst.src_addr = (phys_addr_t)rte_mem_virt2phy((void *)addr); ++ pst.src_addr = (phys_addr_t)rte_mem_virt2iova((void *)addr); + pst.dest_addr = (phys_addr_t)(cmd_q->sb_sha * CCP_SB_BYTES); + pst.len = session->auth.ctx_len; + pst.dir = 1; +@@ -1770,14 +1743,8 @@ ccp_perform_sha(struct rte_crypto_op *op, + op->sym->auth.data.offset); + append_ptr = (void *)rte_pktmbuf_append(op->sym->m_src, + session->auth.ctx_len); +- if (iommu_mode == 2) { +- dest_addr = (phys_addr_t)rte_mem_virt2iova(append_ptr); +- pst.src_addr = (phys_addr_t)sha_ctx; +- } else { +- dest_addr = (phys_addr_t)rte_mem_virt2phy(append_ptr); +- pst.src_addr = (phys_addr_t)rte_mem_virt2phy((void *) +- session->auth.ctx); +- } ++ pst.src_addr = (phys_addr_t)rte_mem_virt2iova((void *)session->auth.ctx); ++ dest_addr = (phys_addr_t)rte_mem_virt2iova(append_ptr); + + /** Passthru sha context*/ + +@@ -1865,15 +1832,8 @@ ccp_perform_sha3_hmac(struct rte_crypto_op *op, + CCP_LOG_ERR("CCP MBUF append failed\n"); + return -1; + } +- if (iommu_mode == 2) { +- dest_addr = (phys_addr_t)rte_mem_virt2iova((void *)append_ptr); +- ctx_paddr = (phys_addr_t)rte_mem_virt2iova( +- session->auth.pre_compute); +- } else { +- dest_addr = (phys_addr_t)rte_mem_virt2phy((void *)append_ptr); +- ctx_paddr = (phys_addr_t)rte_mem_virt2phy( +- session->auth.pre_compute); +- } ++ dest_addr = (phys_addr_t)rte_mem_virt2iova((void *)append_ptr); ++ ctx_paddr = (phys_addr_t)rte_mem_virt2iova(session->auth.pre_compute); + dest_addr_t = dest_addr + (session->auth.ctx_len / 2); + desc = &cmd_q->qbase_desc[cmd_q->qidx]; + memset(desc, 0, Q_DESC_SIZE); +@@ -2009,13 +1969,8 @@ ccp_perform_sha3(struct rte_crypto_op *op, + CCP_LOG_ERR("CCP MBUF append failed\n"); + return -1; + } +- if (iommu_mode == 2) { +- dest_addr = (phys_addr_t)rte_mem_virt2iova((void *)append_ptr); +- ctx_paddr = (phys_addr_t)rte_mem_virt2iova((void *)ctx_addr); +- } else { +- dest_addr = (phys_addr_t)rte_mem_virt2phy((void *)append_ptr); +- ctx_paddr = (phys_addr_t)rte_mem_virt2phy((void *)ctx_addr); +- } ++ dest_addr = (phys_addr_t)rte_mem_virt2iova((void *)append_ptr); ++ ctx_paddr = (phys_addr_t)rte_mem_virt2iova((void *)ctx_addr); + + ctx_addr = session->auth.sha3_ctx; + +@@ -2089,13 +2044,7 @@ ccp_perform_aes_cmac(struct rte_crypto_op *op, + + ctx_addr = session->auth.pre_compute; + memset(ctx_addr, 0, AES_BLOCK_SIZE); +- if (iommu_mode == 2) +- pst.src_addr = (phys_addr_t)rte_mem_virt2iova( +- (void *)ctx_addr); +- else +- pst.src_addr = (phys_addr_t)rte_mem_virt2phy( +- (void *)ctx_addr); +- ++ pst.src_addr = (phys_addr_t)rte_mem_virt2iova((void *)ctx_addr); + pst.dest_addr = (phys_addr_t)(cmd_q->sb_iv * CCP_SB_BYTES); + pst.len = CCP_SB_BYTES; + pst.dir = 1; +@@ -2133,12 +2082,7 @@ ccp_perform_aes_cmac(struct rte_crypto_op *op, + } else { + ctx_addr = session->auth.pre_compute + CCP_SB_BYTES; + memset(ctx_addr, 0, AES_BLOCK_SIZE); +- if (iommu_mode == 2) +- pst.src_addr = (phys_addr_t)rte_mem_virt2iova( +- (void *)ctx_addr); +- else +- pst.src_addr = (phys_addr_t)rte_mem_virt2phy( +- (void *)ctx_addr); ++ pst.src_addr = (phys_addr_t)rte_mem_virt2iova((void *)ctx_addr); + pst.dest_addr = (phys_addr_t)(cmd_q->sb_iv * CCP_SB_BYTES); + pst.len = CCP_SB_BYTES; + pst.dir = 1; +@@ -2328,12 +2272,7 @@ ccp_perform_3des(struct rte_crypto_op *op, + + rte_memcpy(lsb_buf + (CCP_SB_BYTES - session->iv.length), + iv, session->iv.length); +- if (iommu_mode == 2) +- pst.src_addr = (phys_addr_t)rte_mem_virt2iova( +- (void *) lsb_buf); +- else +- pst.src_addr = (phys_addr_t)rte_mem_virt2phy( +- (void *) lsb_buf); ++ pst.src_addr = (phys_addr_t)rte_mem_virt2iova((void *) lsb_buf); + pst.dest_addr = (phys_addr_t)(cmd_q->sb_iv * CCP_SB_BYTES); + pst.len = CCP_SB_BYTES; + pst.dir = 1; +@@ -2356,11 +2295,7 @@ ccp_perform_3des(struct rte_crypto_op *op, + else + dest_addr = src_addr; + +- if (iommu_mode == 2) +- key_addr = rte_mem_virt2iova(session->cipher.key_ccp); +- else +- key_addr = rte_mem_virt2phy(session->cipher.key_ccp); +- ++ key_addr = rte_mem_virt2iova(session->cipher.key_ccp); + desc = &cmd_q->qbase_desc[cmd_q->qidx]; + + memset(desc, 0, Q_DESC_SIZE); +@@ -2746,12 +2681,7 @@ process_ops_to_enqueue(struct ccp_qp *qp, + b_info->lsb_buf_idx = 0; + b_info->desccnt = 0; + b_info->cmd_q = cmd_q; +- if (iommu_mode == 2) +- b_info->lsb_buf_phys = +- (phys_addr_t)rte_mem_virt2iova((void *)b_info->lsb_buf); +- else +- b_info->lsb_buf_phys = +- (phys_addr_t)rte_mem_virt2phy((void *)b_info->lsb_buf); ++ b_info->lsb_buf_phys = (phys_addr_t)rte_mem_virt2iova((void *)b_info->lsb_buf); + + rte_atomic64_sub(&b_info->cmd_q->free_slots, slots_req); + +diff --git a/dpdk/drivers/crypto/ccp/ccp_dev.c b/dpdk/drivers/crypto/ccp/ccp_dev.c +index 424ead82c3..ee30f5ac30 100644 +--- a/dpdk/drivers/crypto/ccp/ccp_dev.c ++++ b/dpdk/drivers/crypto/ccp/ccp_dev.c +@@ -20,11 +20,9 @@ + #include + + #include "ccp_dev.h" +-#include "ccp_pci.h" + #include "ccp_pmd_private.h" + +-int iommu_mode; +-struct ccp_list ccp_list = TAILQ_HEAD_INITIALIZER(ccp_list); ++static TAILQ_HEAD(, ccp_device) ccp_list = TAILQ_HEAD_INITIALIZER(ccp_list); + static int ccp_dev_id; + + int +@@ -69,7 +67,7 @@ ccp_read_hwrng(uint32_t *value) + struct ccp_device *dev; + + TAILQ_FOREACH(dev, &ccp_list, next) { +- void *vaddr = (void *)(dev->pci.mem_resource[2].addr); ++ void *vaddr = (void *)(dev->pci->mem_resource[2].addr); + + while (dev->hwrng_retries++ < CCP_MAX_TRNG_RETRIES) { + *value = CCP_READ_REG(vaddr, TRNG_OUT_REG); +@@ -362,7 +360,7 @@ ccp_find_lsb_regions(struct ccp_queue *cmd_q, uint64_t status) + if (ccp_get_bit(&cmd_q->lsbmask, j)) + weight++; + +- printf("Queue %d can access %d LSB regions of mask %lu\n", ++ CCP_LOG_DBG("Queue %d can access %d LSB regions of mask %lu\n", + (int)cmd_q->id, weight, cmd_q->lsbmask); + + return weight ? 0 : -EINVAL; +@@ -481,7 +479,7 @@ ccp_assign_lsbs(struct ccp_device *ccp) + } + + static int +-ccp_add_device(struct ccp_device *dev, int type) ++ccp_add_device(struct ccp_device *dev) + { + int i; + uint32_t qmr, status_lo, status_hi, dma_addr_lo, dma_addr_hi; +@@ -495,9 +493,9 @@ ccp_add_device(struct ccp_device *dev, int type) + + dev->id = ccp_dev_id++; + dev->qidx = 0; +- vaddr = (void *)(dev->pci.mem_resource[2].addr); ++ vaddr = (void *)(dev->pci->mem_resource[2].addr); + +- if (type == CCP_VERSION_5B) { ++ if (dev->pci->id.device_id == AMD_PCI_CCP_5B) { + CCP_WRITE_REG(vaddr, CMD_TRNG_CTL_OFFSET, 0x00012D57); + CCP_WRITE_REG(vaddr, CMD_CONFIG_0_OFFSET, 0x00000003); + for (i = 0; i < 12; i++) { +@@ -616,54 +614,20 @@ ccp_remove_device(struct ccp_device *dev) + TAILQ_REMOVE(&ccp_list, dev, next); + } + +-static int +-is_ccp_device(const char *dirname, +- const struct rte_pci_id *ccp_id, +- int *type) +-{ +- char filename[PATH_MAX]; +- const struct rte_pci_id *id; +- uint16_t vendor, device_id; +- int i; +- unsigned long tmp; +- +- /* get vendor id */ +- snprintf(filename, sizeof(filename), "%s/vendor", dirname); +- if (ccp_pci_parse_sysfs_value(filename, &tmp) < 0) +- return 0; +- vendor = (uint16_t)tmp; +- +- /* get device id */ +- snprintf(filename, sizeof(filename), "%s/device", dirname); +- if (ccp_pci_parse_sysfs_value(filename, &tmp) < 0) +- return 0; +- device_id = (uint16_t)tmp; +- +- for (id = ccp_id, i = 0; id->vendor_id != 0; id++, i++) { +- if (vendor == id->vendor_id && +- device_id == id->device_id) { +- *type = i; +- return 1; /* Matched device */ +- } +- } +- return 0; +-} +- +-static int +-ccp_probe_device(int ccp_type, struct rte_pci_device *pci_dev) ++int ++ccp_probe_device(struct rte_pci_device *pci_dev) + { +- struct ccp_device *ccp_dev = NULL; +- int uio_fd = -1; ++ struct ccp_device *ccp_dev; + + ccp_dev = rte_zmalloc("ccp_device", sizeof(*ccp_dev), + RTE_CACHE_LINE_SIZE); + if (ccp_dev == NULL) + goto fail; + +- ccp_dev->pci = *pci_dev; ++ ccp_dev->pci = pci_dev; + + /* device is valid, add in list */ +- if (ccp_add_device(ccp_dev, ccp_type)) { ++ if (ccp_add_device(ccp_dev)) { + ccp_remove_device(ccp_dev); + goto fail; + } +@@ -671,51 +635,6 @@ ccp_probe_device(int ccp_type, struct rte_pci_device *pci_dev) + return 0; + fail: + CCP_LOG_ERR("CCP Device probe failed"); +- if (uio_fd >= 0) +- close(uio_fd); + rte_free(ccp_dev); + return -1; + } +- +-int +-ccp_probe_devices(struct rte_pci_device *pci_dev, +- const struct rte_pci_id *ccp_id) +-{ +- int dev_cnt = 0; +- int ccp_type = 0; +- struct dirent *d; +- DIR *dir; +- int ret = 0; +- int module_idx = 0; +- uint16_t domain; +- uint8_t bus, devid, function; +- char dirname[PATH_MAX]; +- +- module_idx = ccp_check_pci_uio_module(); +- if (module_idx < 0) +- return -1; +- +- iommu_mode = module_idx; +- TAILQ_INIT(&ccp_list); +- dir = opendir(SYSFS_PCI_DEVICES); +- if (dir == NULL) +- return -1; +- while ((d = readdir(dir)) != NULL) { +- if (d->d_name[0] == '.') +- continue; +- if (ccp_parse_pci_addr_format(d->d_name, sizeof(d->d_name), +- &domain, &bus, &devid, &function) != 0) +- continue; +- snprintf(dirname, sizeof(dirname), "%s/%s", +- SYSFS_PCI_DEVICES, d->d_name); +- if (is_ccp_device(dirname, ccp_id, &ccp_type)) { +- printf("CCP : Detected CCP device with ID = 0x%x\n", +- ccp_id[ccp_type].device_id); +- ret = ccp_probe_device(ccp_type, pci_dev); +- if (ret == 0) +- dev_cnt++; +- } +- } +- closedir(dir); +- return dev_cnt; +-} +diff --git a/dpdk/drivers/crypto/ccp/ccp_dev.h b/dpdk/drivers/crypto/ccp/ccp_dev.h +index 9deaae7980..e3ec481dd3 100644 +--- a/dpdk/drivers/crypto/ccp/ccp_dev.h ++++ b/dpdk/drivers/crypto/ccp/ccp_dev.h +@@ -19,6 +19,12 @@ + #include + #include + ++/* CCP PCI device identifiers */ ++#define AMD_PCI_VENDOR_ID 0x1022 ++#define AMD_PCI_CCP_5A 0x1456 ++#define AMD_PCI_CCP_5B 0x1468 ++#define AMD_PCI_CCP_RV 0x15df ++ + /**< CCP specific */ + #define MAX_HW_QUEUES 5 + #define CCP_MAX_TRNG_RETRIES 10 +@@ -169,18 +175,6 @@ static inline uint32_t ccp_pci_reg_read(void *base, int offset) + #define CCP_WRITE_REG(hw_addr, reg_offset, value) \ + ccp_pci_reg_write(hw_addr, reg_offset, value) + +-TAILQ_HEAD(ccp_list, ccp_device); +- +-extern struct ccp_list ccp_list; +- +-/** +- * CCP device version +- */ +-enum ccp_device_version { +- CCP_VERSION_5A = 0, +- CCP_VERSION_5B, +-}; +- + /** + * A structure describing a CCP command queue. + */ +@@ -233,8 +227,8 @@ struct ccp_device { + /**< ccp queue */ + int cmd_q_count; + /**< no. of ccp Queues */ +- struct rte_pci_device pci; +- /**< ccp pci identifier */ ++ struct rte_pci_device *pci; ++ /**< ccp pci device */ + unsigned long lsbmap[CCP_BITMAP_SIZE(SLSB_MAP_SIZE)]; + /**< shared lsb mask of ccp */ + rte_spinlock_t lsb_lock; +@@ -468,13 +462,12 @@ high32_value(unsigned long addr) + int ccp_dev_start(struct rte_cryptodev *dev); + + /** +- * Detect ccp platform and initialize all ccp devices ++ * Initialize one ccp device + * +- * @param ccp_id rte_pci_id list for supported CCP devices +- * @return no. of successfully initialized CCP devices ++ * @dev rte pci device ++ * @return 0 on success otherwise -1 + */ +-int ccp_probe_devices(struct rte_pci_device *pci_dev, +- const struct rte_pci_id *ccp_id); ++int ccp_probe_device(struct rte_pci_device *pci_dev); + + /** + * allocate a ccp command queue +diff --git a/dpdk/drivers/crypto/ccp/ccp_pci.c b/dpdk/drivers/crypto/ccp/ccp_pci.c +deleted file mode 100644 +index 38029a9081..0000000000 +--- a/dpdk/drivers/crypto/ccp/ccp_pci.c ++++ /dev/null +@@ -1,240 +0,0 @@ +-/* SPDX-License-Identifier: BSD-3-Clause +- * Copyright(c) 2018 Advanced Micro Devices, Inc. All rights reserved. +- */ +- +-#include +-#include +-#include +-#include +-#include +- +-#include +- +-#include "ccp_pci.h" +- +-static const char * const uio_module_names[] = { +- "igb_uio", +- "uio_pci_generic", +- "vfio_pci" +-}; +- +-int +-ccp_check_pci_uio_module(void) +-{ +- FILE *fp; +- int i; +- char buf[BUFSIZ]; +- +- fp = fopen(PROC_MODULES, "r"); +- if (fp == NULL) +- return -1; +- i = 0; +- while (uio_module_names[i] != NULL) { +- while (fgets(buf, sizeof(buf), fp) != NULL) { +- if (!strncmp(buf, uio_module_names[i], +- strlen(uio_module_names[i]))) { +- fclose(fp); +- return i; +- } +- } +- i++; +- rewind(fp); +- } +- fclose(fp); +- printf("Insert igb_uio or uio_pci_generic kernel module(s)"); +- return -1;/* uio not inserted */ +-} +- +-/* +- * split up a pci address into its constituent parts. +- */ +-int +-ccp_parse_pci_addr_format(const char *buf, int bufsize, uint16_t *domain, +- uint8_t *bus, uint8_t *devid, uint8_t *function) +-{ +- /* first split on ':' */ +- union splitaddr { +- struct { +- char *domain; +- char *bus; +- char *devid; +- char *function; +- }; +- char *str[PCI_FMT_NVAL]; +- /* last element-separator is "." not ":" */ +- } splitaddr; +- +- char *buf_copy = strndup(buf, bufsize); +- +- if (buf_copy == NULL) +- return -1; +- +- if (rte_strsplit(buf_copy, bufsize, splitaddr.str, PCI_FMT_NVAL, ':') +- != PCI_FMT_NVAL - 1) +- goto error; +- /* final split is on '.' between devid and function */ +- splitaddr.function = strchr(splitaddr.devid, '.'); +- if (splitaddr.function == NULL) +- goto error; +- *splitaddr.function++ = '\0'; +- +- /* now convert to int values */ +- errno = 0; +- *domain = (uint8_t)strtoul(splitaddr.domain, NULL, 16); +- *bus = (uint8_t)strtoul(splitaddr.bus, NULL, 16); +- *devid = (uint8_t)strtoul(splitaddr.devid, NULL, 16); +- *function = (uint8_t)strtoul(splitaddr.function, NULL, 10); +- if (errno != 0) +- goto error; +- +- free(buf_copy); /* free the copy made with strdup */ +- return 0; +-error: +- free(buf_copy); +- return -1; +-} +- +-int +-ccp_pci_parse_sysfs_value(const char *filename, unsigned long *val) +-{ +- FILE *f; +- char buf[BUFSIZ]; +- char *end = NULL; +- +- f = fopen(filename, "r"); +- if (f == NULL) +- return -1; +- if (fgets(buf, sizeof(buf), f) == NULL) { +- fclose(f); +- return -1; +- } +- *val = strtoul(buf, &end, 0); +- if ((buf[0] == '\0') || (end == NULL) || (*end != '\n')) { +- fclose(f); +- return -1; +- } +- fclose(f); +- return 0; +-} +- +-/** IO resource type: */ +-#define IORESOURCE_IO 0x00000100 +-#define IORESOURCE_MEM 0x00000200 +- +-/* parse one line of the "resource" sysfs file (note that the 'line' +- * string is modified) +- */ +-static int +-ccp_pci_parse_one_sysfs_resource(char *line, size_t len, uint64_t *phys_addr, +- uint64_t *end_addr, uint64_t *flags) +-{ +- union pci_resource_info { +- struct { +- char *phys_addr; +- char *end_addr; +- char *flags; +- }; +- char *ptrs[PCI_RESOURCE_FMT_NVAL]; +- } res_info; +- +- if (rte_strsplit(line, len, res_info.ptrs, 3, ' ') != 3) +- return -1; +- errno = 0; +- *phys_addr = strtoull(res_info.phys_addr, NULL, 16); +- *end_addr = strtoull(res_info.end_addr, NULL, 16); +- *flags = strtoull(res_info.flags, NULL, 16); +- if (errno != 0) +- return -1; +- +- return 0; +-} +- +-/* parse the "resource" sysfs file */ +-int +-ccp_pci_parse_sysfs_resource(const char *filename, struct rte_pci_device *dev) +-{ +- FILE *fp; +- char buf[BUFSIZ]; +- int i; +- uint64_t phys_addr, end_addr, flags; +- +- fp = fopen(filename, "r"); +- if (fp == NULL) +- return -1; +- +- for (i = 0; i < PCI_MAX_RESOURCE; i++) { +- if (fgets(buf, sizeof(buf), fp) == NULL) +- goto error; +- if (ccp_pci_parse_one_sysfs_resource(buf, sizeof(buf), +- &phys_addr, &end_addr, &flags) < 0) +- goto error; +- +- if (flags & IORESOURCE_MEM) { +- dev->mem_resource[i].phys_addr = phys_addr; +- dev->mem_resource[i].len = end_addr - phys_addr + 1; +- /* not mapped for now */ +- dev->mem_resource[i].addr = NULL; +- } +- } +- fclose(fp); +- return 0; +- +-error: +- fclose(fp); +- return -1; +-} +- +-int +-ccp_find_uio_devname(const char *dirname) +-{ +- +- DIR *dir; +- struct dirent *e; +- char dirname_uio[PATH_MAX]; +- unsigned int uio_num; +- int ret = -1; +- +- /* depending on kernel version, uio can be located in uio/uioX +- * or uio:uioX +- */ +- snprintf(dirname_uio, sizeof(dirname_uio), "%s/uio", dirname); +- dir = opendir(dirname_uio); +- if (dir == NULL) { +- /* retry with the parent directory might be different kernel version*/ +- dir = opendir(dirname); +- if (dir == NULL) +- return -1; +- } +- +- /* take the first file starting with "uio" */ +- while ((e = readdir(dir)) != NULL) { +- /* format could be uio%d ...*/ +- int shortprefix_len = sizeof("uio") - 1; +- /* ... or uio:uio%d */ +- int longprefix_len = sizeof("uio:uio") - 1; +- char *endptr; +- +- if (strncmp(e->d_name, "uio", 3) != 0) +- continue; +- +- /* first try uio%d */ +- errno = 0; +- uio_num = strtoull(e->d_name + shortprefix_len, &endptr, 10); +- if (errno == 0 && endptr != (e->d_name + shortprefix_len)) { +- ret = uio_num; +- break; +- } +- +- /* then try uio:uio%d */ +- errno = 0; +- uio_num = strtoull(e->d_name + longprefix_len, &endptr, 10); +- if (errno == 0 && endptr != (e->d_name + longprefix_len)) { +- ret = uio_num; +- break; +- } +- } +- closedir(dir); +- return ret; +- +- +-} +diff --git a/dpdk/drivers/crypto/ccp/ccp_pci.h b/dpdk/drivers/crypto/ccp/ccp_pci.h +deleted file mode 100644 +index 6736bf8ad3..0000000000 +--- a/dpdk/drivers/crypto/ccp/ccp_pci.h ++++ /dev/null +@@ -1,27 +0,0 @@ +-/* SPDX-License-Identifier: BSD-3-Clause +- * Copyright(c) 2018 Advanced Micro Devices, Inc. All rights reserved. +- */ +- +-#ifndef _CCP_PCI_H_ +-#define _CCP_PCI_H_ +- +-#include +- +-#include +- +-#define SYSFS_PCI_DEVICES "/sys/bus/pci/devices" +-#define PROC_MODULES "/proc/modules" +- +-int ccp_check_pci_uio_module(void); +- +-int ccp_parse_pci_addr_format(const char *buf, int bufsize, uint16_t *domain, +- uint8_t *bus, uint8_t *devid, uint8_t *function); +- +-int ccp_pci_parse_sysfs_value(const char *filename, unsigned long *val); +- +-int ccp_pci_parse_sysfs_resource(const char *filename, +- struct rte_pci_device *dev); +- +-int ccp_find_uio_devname(const char *dirname); +- +-#endif /* _CCP_PCI_H_ */ +diff --git a/dpdk/drivers/crypto/ccp/meson.build b/dpdk/drivers/crypto/ccp/meson.build +index a4f3406009..a9abaa4da0 100644 +--- a/dpdk/drivers/crypto/ccp/meson.build ++++ b/dpdk/drivers/crypto/ccp/meson.build +@@ -18,7 +18,6 @@ sources = files( + 'rte_ccp_pmd.c', + 'ccp_crypto.c', + 'ccp_dev.c', +- 'ccp_pci.c', + 'ccp_pmd_ops.c', + ) + +diff --git a/dpdk/drivers/crypto/ccp/rte_ccp_pmd.c b/dpdk/drivers/crypto/ccp/rte_ccp_pmd.c +index 221a0a5235..a5271d7227 100644 +--- a/dpdk/drivers/crypto/ccp/rte_ccp_pmd.c ++++ b/dpdk/drivers/crypto/ccp/rte_ccp_pmd.c +@@ -22,7 +22,6 @@ + static unsigned int ccp_pmd_init_done; + uint8_t ccp_cryptodev_driver_id; + uint8_t cryptodev_cnt; +-extern void *sha_ctx; + + struct ccp_pmd_init_params { + struct rte_cryptodev_pmd_init_params def_p; +@@ -168,15 +167,9 @@ ccp_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, + * The set of PCI devices this driver supports + */ + static struct rte_pci_id ccp_pci_id[] = { +- { +- RTE_PCI_DEVICE(0x1022, 0x1456), /* AMD CCP-5a */ +- }, +- { +- RTE_PCI_DEVICE(0x1022, 0x1468), /* AMD CCP-5b */ +- }, +- { +- RTE_PCI_DEVICE(0x1022, 0x15df), /* AMD CCP RV */ +- }, ++ { RTE_PCI_DEVICE(AMD_PCI_VENDOR_ID, AMD_PCI_CCP_5A), }, ++ { RTE_PCI_DEVICE(AMD_PCI_VENDOR_ID, AMD_PCI_CCP_5B), }, ++ { RTE_PCI_DEVICE(AMD_PCI_VENDOR_ID, AMD_PCI_CCP_RV), }, + {.device_id = 0}, + }; + +@@ -200,7 +193,6 @@ cryptodev_ccp_remove(struct rte_pci_device *pci_dev) + return -ENODEV; + + ccp_pmd_init_done = 0; +- rte_free(sha_ctx); + + RTE_LOG(INFO, PMD, "Closing ccp device %s on numa socket %u\n", + name, rte_socket_id()); +@@ -230,14 +222,13 @@ cryptodev_ccp_create(const char *name, + goto init_error; + } + +- cryptodev_cnt = ccp_probe_devices(pci_dev, ccp_pci_id); +- +- if (cryptodev_cnt == 0) { ++ if (ccp_probe_device(pci_dev) != 0) { + CCP_LOG_ERR("failed to detect CCP crypto device"); + goto init_error; + } ++ cryptodev_cnt++; + +- printf("CCP : Crypto device count = %d\n", cryptodev_cnt); ++ CCP_LOG_DBG("CCP : Crypto device count = %d\n", cryptodev_cnt); + dev->device = &pci_dev->device; + dev->device->driver = &pci_drv->driver; + dev->driver_id = ccp_cryptodev_driver_id; +@@ -287,7 +278,6 @@ cryptodev_ccp_probe(struct rte_pci_driver *pci_drv __rte_unused, + .auth_opt = CCP_PMD_AUTH_OPT_CCP, + }; + +- sha_ctx = (void *)rte_malloc(NULL, SHA512_DIGEST_SIZE, 64); + if (ccp_pmd_init_done) { + RTE_LOG(INFO, PMD, "CCP PMD already initialized\n"); + return -EFAULT; +diff --git a/dpdk/drivers/crypto/cnxk/cnxk_cryptodev_capabilities.c b/dpdk/drivers/crypto/cnxk/cnxk_cryptodev_capabilities.c +index 6c28f8942e..b4d1925d23 100644 +--- a/dpdk/drivers/crypto/cnxk/cnxk_cryptodev_capabilities.c ++++ b/dpdk/drivers/crypto/cnxk/cnxk_cryptodev_capabilities.c +@@ -851,8 +851,8 @@ static const struct rte_cryptodev_capabilities sec_caps_aes[] = { + .increment = 4 + }, + .iv_size = { +- .min = 12, +- .max = 12, ++ .min = 11, ++ .max = 11, + .increment = 0 + } + }, } +diff --git a/dpdk/drivers/crypto/cnxk/cnxk_se.h b/dpdk/drivers/crypto/cnxk/cnxk_se.h +index b07fc22858..32e2b2cd64 100644 +--- a/dpdk/drivers/crypto/cnxk/cnxk_se.h ++++ b/dpdk/drivers/crypto/cnxk/cnxk_se.h +@@ -244,7 +244,7 @@ fill_sg_comp_from_iov(struct roc_se_sglist_comp *list, uint32_t i, + uint32_t extra_len = extra_buf ? extra_buf->size : 0; + uint32_t size = *psize; + +- for (j = 0; (j < from->buf_cnt) && size; j++) { ++ for (j = 0; j < from->buf_cnt; j++) { + struct roc_se_sglist_comp *to = &list[i >> 2]; + uint32_t buf_sz = from->bufs[j].size; + void *vaddr = from->bufs[j].vaddr; +@@ -311,6 +311,9 @@ fill_sg_comp_from_iov(struct roc_se_sglist_comp *list, uint32_t i, + if (extra_offset) + extra_offset -= size; + i++; ++ ++ if (unlikely(!size)) ++ break; + } + + *psize = size; +@@ -367,7 +370,9 @@ fill_sg2_comp_from_iov(struct roc_se_sg2list_comp *list, uint32_t i, struct roc_ + uint32_t extra_len = extra_buf ? extra_buf->size : 0; + uint32_t size = *psize; + +- for (j = 0; (j < from->buf_cnt) && size; j++) { ++ rte_prefetch2(psize); ++ ++ for (j = 0; j < from->buf_cnt; j++) { + struct roc_se_sg2list_comp *to = &list[i / 3]; + uint32_t buf_sz = from->bufs[j].size; + void *vaddr = from->bufs[j].vaddr; +@@ -433,6 +438,9 @@ fill_sg2_comp_from_iov(struct roc_se_sg2list_comp *list, uint32_t i, struct roc_ + if (extra_offset) + extra_offset -= size; + i++; ++ ++ if (unlikely(!size)) ++ break; + } + + *psize = size; +@@ -884,20 +892,10 @@ cpt_digest_gen_sg_ver1_prep(uint32_t flags, uint64_t d_lens, struct roc_se_fc_pa + + /* input data */ + size = data_len; +- if (size) { +- i = fill_sg_comp_from_iov(gather_comp, i, params->src_iov, 0, +- &size, NULL, 0); +- if (unlikely(size)) { +- plt_dp_err("Insufficient dst IOV size, short by %dB", +- size); +- return -1; +- } +- } else { +- /* +- * Looks like we need to support zero data +- * gather ptr in case of hash & hmac +- */ +- i++; ++ i = fill_sg_comp_from_iov(gather_comp, i, params->src_iov, 0, &size, NULL, 0); ++ if (unlikely(size)) { ++ plt_dp_err("Insufficient dst IOV size, short by %dB", size); ++ return -1; + } + ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); + g_size_bytes = ((i + 3) / 4) * sizeof(struct roc_se_sglist_comp); +@@ -1008,18 +1006,10 @@ cpt_digest_gen_sg_ver2_prep(uint32_t flags, uint64_t d_lens, struct roc_se_fc_pa + + /* input data */ + size = data_len; +- if (size) { +- i = fill_sg2_comp_from_iov(gather_comp, i, params->src_iov, 0, &size, NULL, 0); +- if (unlikely(size)) { +- plt_dp_err("Insufficient dst IOV size, short by %dB", size); +- return -1; +- } +- } else { +- /* +- * Looks like we need to support zero data +- * gather ptr in case of hash & hmac +- */ +- i++; ++ i = fill_sg2_comp_from_iov(gather_comp, i, params->src_iov, 0, &size, NULL, 0); ++ if (unlikely(size)) { ++ plt_dp_err("Insufficient dst IOV size, short by %dB", size); ++ return -1; + } + cpt_inst_w5.s.gather_sz = ((i + 2) / 3); + +diff --git a/dpdk/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c b/dpdk/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c +index c25e40030b..cf1339f266 100644 +--- a/dpdk/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c ++++ b/dpdk/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c +@@ -1676,7 +1676,7 @@ dpaa2_sec_dump(struct rte_crypto_op *op) + + if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) + sess = CRYPTODEV_GET_SYM_SESS_PRIV(op->sym->session); +-#ifdef RTE_LIBRTE_SECURITY ++#ifdef RTE_LIB_SECURITY + else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) + sess = SECURITY_GET_SESS_PRIV(op->sym->session); + #endif +@@ -1703,7 +1703,7 @@ dpaa2_sec_dump(struct rte_crypto_op *op) + sess->digest_length, sess->status, + sess->ext_params.aead_ctxt.auth_only_len, + sess->ext_params.aead_ctxt.auth_cipher_text); +-#ifdef RTE_LIBRTE_SECURITY ++#ifdef RTE_LIB_SECURITY + printf("PDCP session params:\n" + "\tDomain:\t\t%d\n\tBearer:\t\t%d\n\tpkt_dir:\t%d\n\thfn_ovd:" + "\t%d\n\tsn_size:\t%d\n\thfn_ovd_offset:\t%d\n\thfn:\t\t%d\n" +diff --git a/dpdk/drivers/crypto/dpaa_sec/dpaa_sec.c b/dpdk/drivers/crypto/dpaa_sec/dpaa_sec.c +index db52683847..7807c83e54 100644 +--- a/dpdk/drivers/crypto/dpaa_sec/dpaa_sec.c ++++ b/dpdk/drivers/crypto/dpaa_sec/dpaa_sec.c +@@ -671,7 +671,7 @@ dpaa_sec_dump(struct dpaa_sec_op_ctx *ctx, struct dpaa_sec_qp *qp) + + if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) + sess = CRYPTODEV_GET_SYM_SESS_PRIV(op->sym->session); +-#ifdef RTE_LIBRTE_SECURITY ++#ifdef RTE_LIB_SECURITY + else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) + sess = SECURITY_GET_SESS_PRIV(op->sym->session); + #endif +@@ -682,7 +682,7 @@ dpaa_sec_dump(struct dpaa_sec_op_ctx *ctx, struct dpaa_sec_qp *qp) + + cdb = &sess->cdb; + rte_memcpy(&c_cdb, cdb, sizeof(struct sec_cdb)); +-#ifdef RTE_LIBRTE_SECURITY ++#ifdef RTE_LIB_SECURITY + printf("\nsession protocol type = %d\n", sess->proto_alg); + #endif + printf("\n****************************************\n" +@@ -707,7 +707,7 @@ dpaa_sec_dump(struct dpaa_sec_op_ctx *ctx, struct dpaa_sec_qp *qp) + sess->iv.length, sess->iv.offset, + sess->digest_length, sess->auth_only_len, + sess->auth_cipher_text); +-#ifdef RTE_LIBRTE_SECURITY ++#ifdef RTE_LIB_SECURITY + printf("PDCP session params:\n" + "\tDomain:\t\t%d\n\tBearer:\t\t%d\n\tpkt_dir:\t%d\n\thfn_ovd:" + "\t%d\n\tsn_size:\t%d\n\tsdap_enabled:\t%d\n\thfn_ovd_offset:" +diff --git a/dpdk/drivers/crypto/ipsec_mb/ipsec_mb_ops.c b/dpdk/drivers/crypto/ipsec_mb/ipsec_mb_ops.c +index 71e02cd051..30f919cd40 100644 +--- a/dpdk/drivers/crypto/ipsec_mb/ipsec_mb_ops.c ++++ b/dpdk/drivers/crypto/ipsec_mb/ipsec_mb_ops.c +@@ -139,15 +139,12 @@ int + ipsec_mb_qp_release(struct rte_cryptodev *dev, uint16_t qp_id) + { + struct ipsec_mb_qp *qp = dev->data->queue_pairs[qp_id]; +- struct rte_ring *r = NULL; + + if (!qp) + return 0; + + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { +- r = rte_ring_lookup(qp->name); +- if (r) +- rte_ring_free(r); ++ rte_ring_free(rte_ring_lookup(qp->name)); + + #if IMB_VERSION(1, 1, 0) > IMB_VERSION_NUM + if (qp->mb_mgr) +@@ -437,15 +434,22 @@ ipsec_mb_sym_session_configure( + struct ipsec_mb_dev_private *internals = dev->data->dev_private; + struct ipsec_mb_internals *pmd_data = + &ipsec_mb_pmds[internals->pmd_type]; +- IMB_MGR *mb_mgr = alloc_init_mb_mgr(); ++ struct ipsec_mb_qp *qp = dev->data->queue_pairs[0]; ++ IMB_MGR *mb_mgr; + int ret = 0; + ++ if (qp != NULL) ++ mb_mgr = qp->mb_mgr; ++ else ++ mb_mgr = alloc_init_mb_mgr(); ++ + if (!mb_mgr) + return -ENOMEM; + + if (unlikely(sess == NULL)) { + IPSEC_MB_LOG(ERR, "invalid session struct"); +- free_mb_mgr(mb_mgr); ++ if (qp == NULL) ++ free_mb_mgr(mb_mgr); + return -EINVAL; + } + +@@ -455,11 +459,13 @@ ipsec_mb_sym_session_configure( + IPSEC_MB_LOG(ERR, "failed configure session parameters"); + + /* Return session to mempool */ +- free_mb_mgr(mb_mgr); ++ if (qp == NULL) ++ free_mb_mgr(mb_mgr); + return ret; + } + +- free_mb_mgr(mb_mgr); ++ if (qp == NULL) ++ free_mb_mgr(mb_mgr); + return 0; + } + +diff --git a/dpdk/drivers/crypto/ipsec_mb/ipsec_mb_private.c b/dpdk/drivers/crypto/ipsec_mb/ipsec_mb_private.c +index 50b789a29b..64f2b4b604 100644 +--- a/dpdk/drivers/crypto/ipsec_mb/ipsec_mb_private.c ++++ b/dpdk/drivers/crypto/ipsec_mb/ipsec_mb_private.c +@@ -170,7 +170,7 @@ ipsec_mb_create(struct rte_vdev_device *vdev, + + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + retval = ipsec_mb_mp_request_register(); +- if (retval && (rte_errno == EEXIST)) ++ if (retval && ((rte_errno == EEXIST) || (rte_errno == ENOTSUP))) + /* Safe to proceed, return 0 */ + return 0; + +diff --git a/dpdk/drivers/crypto/ipsec_mb/meson.build b/dpdk/drivers/crypto/ipsec_mb/meson.build +index ec147d2110..4100d921ff 100644 +--- a/dpdk/drivers/crypto/ipsec_mb/meson.build ++++ b/dpdk/drivers/crypto/ipsec_mb/meson.build +@@ -16,6 +16,11 @@ lib = cc.find_library('IPSec_MB', required: false) + if not lib.found() + build = false + reason = 'missing dependency, "libIPSec_MB"' ++# if the lib is found, check it's the right format ++elif meson.version().version_compare('>=0.60') and not cc.links( ++ 'int main(void) {return 0;}', dependencies: lib) ++ build = false ++ reason = 'incompatible dependency, "libIPSec_MB"' + else + ext_deps += lib + +diff --git a/dpdk/drivers/crypto/ipsec_mb/pmd_aesni_mb.c b/dpdk/drivers/crypto/ipsec_mb/pmd_aesni_mb.c +index 147a38932d..ac20d01937 100644 +--- a/dpdk/drivers/crypto/ipsec_mb/pmd_aesni_mb.c ++++ b/dpdk/drivers/crypto/ipsec_mb/pmd_aesni_mb.c +@@ -199,7 +199,7 @@ aesni_mb_set_session_auth_parameters(const IMB_MGR *mb_mgr, + } + } else if (xform->auth.key.length == 32) { + sess->auth.algo = IMB_AUTH_ZUC256_EIA3_BITLEN; +-#if IMB_VERSION(1, 2, 0) > IMB_VERSION_NUM ++#if IMB_VERSION(1, 2, 0) < IMB_VERSION_NUM + if (sess->auth.req_digest_len != 4 && + sess->auth.req_digest_len != 8 && + sess->auth.req_digest_len != 16) { +diff --git a/dpdk/drivers/crypto/ipsec_mb/pmd_aesni_mb_priv.h b/dpdk/drivers/crypto/ipsec_mb/pmd_aesni_mb_priv.h +index 55fafbbbec..8a7c74f621 100644 +--- a/dpdk/drivers/crypto/ipsec_mb/pmd_aesni_mb_priv.h ++++ b/dpdk/drivers/crypto/ipsec_mb/pmd_aesni_mb_priv.h +@@ -566,7 +566,7 @@ static const struct rte_cryptodev_capabilities aesni_mb_capabilities[] = { + }, + .digest_size = { + .min = 4, +-#if IMB_VERSION(1, 2, 0) > IMB_VERSION_NUM ++#if IMB_VERSION(1, 2, 0) < IMB_VERSION_NUM + .max = 16, + .increment = 4 + #else +diff --git a/dpdk/drivers/crypto/ipsec_mb/pmd_snow3g.c b/dpdk/drivers/crypto/ipsec_mb/pmd_snow3g.c +index 8ed069f428..e64df1a462 100644 +--- a/dpdk/drivers/crypto/ipsec_mb/pmd_snow3g.c ++++ b/dpdk/drivers/crypto/ipsec_mb/pmd_snow3g.c +@@ -372,9 +372,10 @@ process_ops(struct rte_crypto_op **ops, struct snow3g_session *session, + /** Process a crypto op with length/offset in bits. */ + static int + process_op_bit(struct rte_crypto_op *op, struct snow3g_session *session, +- struct ipsec_mb_qp *qp, uint16_t *accumulated_enqueued_ops) ++ struct ipsec_mb_qp *qp) + { +- uint32_t enqueued_op, processed_op; ++ unsigned int processed_op; ++ int ret; + + switch (session->op) { + case IPSEC_MB_OP_ENCRYPT_ONLY: +@@ -421,9 +422,10 @@ process_op_bit(struct rte_crypto_op *op, struct snow3g_session *session, + + if (unlikely(processed_op != 1)) + return 0; +- enqueued_op = rte_ring_enqueue(qp->ingress_queue, op); +- qp->stats.enqueued_count += enqueued_op; +- *accumulated_enqueued_ops += enqueued_op; ++ ++ ret = rte_ring_enqueue(qp->ingress_queue, op); ++ if (ret != 0) ++ return ret; + + return 1; + } +@@ -439,7 +441,6 @@ snow3g_pmd_dequeue_burst(void *queue_pair, + struct snow3g_session *prev_sess = NULL, *curr_sess = NULL; + uint32_t i; + uint8_t burst_size = 0; +- uint16_t enqueued_ops = 0; + uint8_t processed_ops; + uint32_t nb_dequeued; + +@@ -479,8 +480,7 @@ snow3g_pmd_dequeue_burst(void *queue_pair, + prev_sess = NULL; + } + +- processed_ops = process_op_bit(curr_c_op, curr_sess, +- qp, &enqueued_ops); ++ processed_ops = process_op_bit(curr_c_op, curr_sess, qp); + if (processed_ops != 1) + break; + +diff --git a/dpdk/drivers/crypto/nitrox/nitrox_sym_reqmgr.c b/dpdk/drivers/crypto/nitrox/nitrox_sym_reqmgr.c +index 9edb0cc00f..d7e8ff7db4 100644 +--- a/dpdk/drivers/crypto/nitrox/nitrox_sym_reqmgr.c ++++ b/dpdk/drivers/crypto/nitrox/nitrox_sym_reqmgr.c +@@ -10,8 +10,11 @@ + #include "nitrox_sym_reqmgr.h" + #include "nitrox_logs.h" + +-#define MAX_SGBUF_CNT 16 +-#define MAX_SGCOMP_CNT 5 ++#define MAX_SUPPORTED_MBUF_SEGS 16 ++/* IV + AAD + ORH + CC + DIGEST */ ++#define ADDITIONAL_SGBUF_CNT 5 ++#define MAX_SGBUF_CNT (MAX_SUPPORTED_MBUF_SEGS + ADDITIONAL_SGBUF_CNT) ++#define MAX_SGCOMP_CNT (RTE_ALIGN_MUL_CEIL(MAX_SGBUF_CNT, 4) / 4) + /* SLC_STORE_INFO */ + #define MIN_UDD_LEN 16 + /* PKT_IN_HDR + SLC_STORE_INFO */ +@@ -303,7 +306,7 @@ create_sglist_from_mbuf(struct nitrox_sgtable *sgtbl, struct rte_mbuf *mbuf, + datalen -= mlen; + } + +- RTE_VERIFY(cnt <= MAX_SGBUF_CNT); ++ RTE_ASSERT(cnt <= MAX_SGBUF_CNT); + sgtbl->map_bufs_cnt = cnt; + return 0; + } +@@ -375,7 +378,7 @@ create_cipher_outbuf(struct nitrox_softreq *sr) + sr->out.sglist[cnt].virt = &sr->resp.completion; + cnt++; + +- RTE_VERIFY(cnt <= MAX_SGBUF_CNT); ++ RTE_ASSERT(cnt <= MAX_SGBUF_CNT); + sr->out.map_bufs_cnt = cnt; + + create_sgcomp(&sr->out); +@@ -600,7 +603,7 @@ create_aead_outbuf(struct nitrox_softreq *sr, struct nitrox_sglist *digest) + resp.completion); + sr->out.sglist[cnt].virt = &sr->resp.completion; + cnt++; +- RTE_VERIFY(cnt <= MAX_SGBUF_CNT); ++ RTE_ASSERT(cnt <= MAX_SGBUF_CNT); + sr->out.map_bufs_cnt = cnt; + + create_sgcomp(&sr->out); +@@ -774,6 +777,14 @@ nitrox_process_se_req(uint16_t qno, struct rte_crypto_op *op, + { + int err; + ++ if (unlikely(op->sym->m_src->nb_segs > MAX_SUPPORTED_MBUF_SEGS || ++ (op->sym->m_dst && ++ op->sym->m_dst->nb_segs > MAX_SUPPORTED_MBUF_SEGS))) { ++ NITROX_LOG(ERR, "Mbuf segments not supported. " ++ "Max supported %d\n", MAX_SUPPORTED_MBUF_SEGS); ++ return -ENOTSUP; ++ } ++ + softreq_init(sr, sr->iova); + sr->ctx = ctx; + sr->op = op; +diff --git a/dpdk/drivers/crypto/openssl/openssl_pmd_private.h b/dpdk/drivers/crypto/openssl/openssl_pmd_private.h +index ed6841e460..4e224b040b 100644 +--- a/dpdk/drivers/crypto/openssl/openssl_pmd_private.h ++++ b/dpdk/drivers/crypto/openssl/openssl_pmd_private.h +@@ -189,6 +189,8 @@ struct openssl_asym_session { + struct dh { + DH *dh_key; + uint32_t key_op; ++ BIGNUM *p; ++ BIGNUM *g; + #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) + OSSL_PARAM_BLD * param_bld; + OSSL_PARAM_BLD *param_bld_peer; +@@ -198,6 +200,10 @@ struct openssl_asym_session { + DSA *dsa; + #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) + OSSL_PARAM_BLD * param_bld; ++ BIGNUM *p; ++ BIGNUM *g; ++ BIGNUM *q; ++ BIGNUM *priv_key; + #endif + } s; + } u; +diff --git a/dpdk/drivers/crypto/openssl/rte_openssl_pmd.c b/dpdk/drivers/crypto/openssl/rte_openssl_pmd.c +index 05449b6e98..6ae31cb5cd 100644 +--- a/dpdk/drivers/crypto/openssl/rte_openssl_pmd.c ++++ b/dpdk/drivers/crypto/openssl/rte_openssl_pmd.c +@@ -696,7 +696,7 @@ openssl_set_session_auth_parameters(struct openssl_session *sess, + algo = digest_name_get(xform->auth.algo); + if (!algo) + return -EINVAL; +- rte_memcpy(algo_name, algo, (sizeof(algo)+1)); ++ strlcpy(algo_name, algo, sizeof(algo_name)); + + mac = EVP_MAC_fetch(NULL, "HMAC", NULL); + sess->auth.hmac.ctx = EVP_MAC_CTX_new(mac); +@@ -1195,8 +1195,11 @@ process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset, + int srclen, uint8_t *aad, int aadlen, uint8_t *iv, + uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) + { +- int len = 0, unused = 0; ++ int len = 0; ++#if OPENSSL_VERSION_NUMBER < 0x10100000L ++ int unused = 0; + uint8_t empty[] = {}; ++#endif + + if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) + goto process_auth_encryption_gcm_err; +@@ -1210,9 +1213,11 @@ process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset, + srclen, ctx, 0)) + goto process_auth_encryption_gcm_err; + ++#if OPENSSL_VERSION_NUMBER < 0x10100000L + /* Workaround open ssl bug in version less then 1.0.1f */ + if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0) + goto process_auth_encryption_gcm_err; ++#endif + + if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) + goto process_auth_encryption_gcm_err; +@@ -1274,8 +1279,11 @@ process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset, + int srclen, uint8_t *aad, int aadlen, uint8_t *iv, + uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) + { +- int len = 0, unused = 0; ++ int len = 0; ++#if OPENSSL_VERSION_NUMBER < 0x10100000L ++ int unused = 0; + uint8_t empty[] = {}; ++#endif + + if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0) + goto process_auth_decryption_gcm_err; +@@ -1292,9 +1300,11 @@ process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset, + srclen, ctx, 0)) + goto process_auth_decryption_gcm_err; + ++#if OPENSSL_VERSION_NUMBER < 0x10100000L + /* Workaround open ssl bug in version less then 1.0.1f */ + if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0) + goto process_auth_decryption_gcm_err; ++#endif + + if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0) + return -EFAULT; +@@ -1797,7 +1807,6 @@ process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op, + # if OPENSSL_VERSION_NUMBER >= 0x30000000L + EVP_MAC_CTX *ctx_h; + EVP_MAC_CTX *ctx_c; +- EVP_MAC *mac; + # else + HMAC_CTX *ctx_h; + CMAC_CTX *ctx_c; +@@ -1818,10 +1827,7 @@ process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op, + break; + case OPENSSL_AUTH_AS_HMAC: + # if OPENSSL_VERSION_NUMBER >= 0x30000000L +- mac = EVP_MAC_fetch(NULL, "HMAC", NULL); +- ctx_h = EVP_MAC_CTX_new(mac); + ctx_h = EVP_MAC_CTX_dup(sess->auth.hmac.ctx); +- EVP_MAC_free(mac); + status = process_openssl_auth_mac(mbuf_src, dst, + op->sym->auth.data.offset, srclen, + ctx_h); +@@ -1836,10 +1842,7 @@ process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op, + break; + case OPENSSL_AUTH_AS_CMAC: + # if OPENSSL_VERSION_NUMBER >= 0x30000000L +- mac = EVP_MAC_fetch(NULL, OSSL_MAC_NAME_CMAC, NULL); +- ctx_c = EVP_MAC_CTX_new(mac); + ctx_c = EVP_MAC_CTX_dup(sess->auth.cmac.ctx); +- EVP_MAC_free(mac); + status = process_openssl_auth_mac(mbuf_src, dst, + op->sym->auth.data.offset, srclen, + ctx_c); +@@ -1927,7 +1930,7 @@ process_openssl_dsa_sign_op_evp(struct rte_crypto_op *cop, + + if (EVP_PKEY_sign(dsa_ctx, dsa_sign_data, &outlen, op->message.data, + op->message.length) <= 0) { +- free(dsa_sign_data); ++ OPENSSL_free(dsa_sign_data); + goto err_dsa_sign; + } + +@@ -1935,7 +1938,7 @@ process_openssl_dsa_sign_op_evp(struct rte_crypto_op *cop, + DSA_SIG *sign = d2i_DSA_SIG(NULL, &dsa_sign_data_p, outlen); + if (!sign) { + OPENSSL_LOG(ERR, "%s:%d\n", __func__, __LINE__); +- free(dsa_sign_data); ++ OPENSSL_free(dsa_sign_data); + goto err_dsa_sign; + } else { + const BIGNUM *r = NULL, *s = NULL; +@@ -1947,7 +1950,7 @@ process_openssl_dsa_sign_op_evp(struct rte_crypto_op *cop, + } + + DSA_SIG_free(sign); +- free(dsa_sign_data); ++ OPENSSL_free(dsa_sign_data); + return 0; + + err_dsa_sign: +@@ -1957,6 +1960,7 @@ err_dsa_sign: + EVP_PKEY_CTX_free(key_ctx); + if (dsa_ctx) + EVP_PKEY_CTX_free(dsa_ctx); ++ EVP_PKEY_free(pkey); + return -1; + } + +@@ -2633,7 +2637,7 @@ process_openssl_rsa_op_evp(struct rte_crypto_op *cop, + if (EVP_PKEY_verify_recover(rsa_ctx, tmp, &outlen, + op->rsa.sign.data, + op->rsa.sign.length) <= 0) { +- rte_free(tmp); ++ OPENSSL_free(tmp); + goto err_rsa; + } + +@@ -2645,7 +2649,7 @@ process_openssl_rsa_op_evp(struct rte_crypto_op *cop, + op->rsa.message.length)) { + OPENSSL_LOG(ERR, "RSA sign Verification failed"); + } +- rte_free(tmp); ++ OPENSSL_free(tmp); + break; + + default: +diff --git a/dpdk/drivers/crypto/openssl/rte_openssl_pmd_ops.c b/dpdk/drivers/crypto/openssl/rte_openssl_pmd_ops.c +index defed4429e..24d6d48262 100644 +--- a/dpdk/drivers/crypto/openssl/rte_openssl_pmd_ops.c ++++ b/dpdk/drivers/crypto/openssl/rte_openssl_pmd_ops.c +@@ -1087,22 +1087,21 @@ err_rsa: + } + case RTE_CRYPTO_ASYM_XFORM_DH: + { +- BIGNUM *p = NULL; +- BIGNUM *g = NULL; +- +- p = BN_bin2bn((const unsigned char *) ++ DH *dh = NULL; ++#if (OPENSSL_VERSION_NUMBER >= 0x30000000L) ++ BIGNUM **p = &asym_session->u.dh.p; ++ BIGNUM **g = &asym_session->u.dh.g; ++ *p = BN_bin2bn((const unsigned char *) + xform->dh.p.data, + xform->dh.p.length, +- p); +- g = BN_bin2bn((const unsigned char *) ++ *p); ++ *g = BN_bin2bn((const unsigned char *) + xform->dh.g.data, + xform->dh.g.length, +- g); +- if (!p || !g) ++ *g); ++ if (!*p || !*g) + goto err_dh; + +- DH *dh = NULL; +-#if (OPENSSL_VERSION_NUMBER >= 0x30000000L) + OSSL_PARAM_BLD *param_bld = NULL; + param_bld = OSSL_PARAM_BLD_new(); + if (!param_bld) { +@@ -1112,9 +1111,9 @@ err_rsa: + if ((!OSSL_PARAM_BLD_push_utf8_string(param_bld, + "group", "ffdhe2048", 0)) + || (!OSSL_PARAM_BLD_push_BN(param_bld, +- OSSL_PKEY_PARAM_FFC_P, p)) ++ OSSL_PKEY_PARAM_FFC_P, *p)) + || (!OSSL_PARAM_BLD_push_BN(param_bld, +- OSSL_PKEY_PARAM_FFC_G, g))) { ++ OSSL_PKEY_PARAM_FFC_G, *g))) { + OSSL_PARAM_BLD_free(param_bld); + goto err_dh; + } +@@ -1129,9 +1128,9 @@ err_rsa: + if ((!OSSL_PARAM_BLD_push_utf8_string(param_bld_peer, + "group", "ffdhe2048", 0)) + || (!OSSL_PARAM_BLD_push_BN(param_bld_peer, +- OSSL_PKEY_PARAM_FFC_P, p)) ++ OSSL_PKEY_PARAM_FFC_P, *p)) + || (!OSSL_PARAM_BLD_push_BN(param_bld_peer, +- OSSL_PKEY_PARAM_FFC_G, g))) { ++ OSSL_PKEY_PARAM_FFC_G, *g))) { + OSSL_PARAM_BLD_free(param_bld); + OSSL_PARAM_BLD_free(param_bld_peer); + goto err_dh; +@@ -1140,6 +1139,20 @@ err_rsa: + asym_session->u.dh.param_bld = param_bld; + asym_session->u.dh.param_bld_peer = param_bld_peer; + #else ++ BIGNUM *p = NULL; ++ BIGNUM *g = NULL; ++ ++ p = BN_bin2bn((const unsigned char *) ++ xform->dh.p.data, ++ xform->dh.p.length, ++ p); ++ g = BN_bin2bn((const unsigned char *) ++ xform->dh.g.data, ++ xform->dh.g.length, ++ g); ++ if (!p || !g) ++ goto err_dh; ++ + dh = DH_new(); + if (dh == NULL) { + OPENSSL_LOG(ERR, +@@ -1158,41 +1171,48 @@ err_rsa: + + err_dh: + OPENSSL_LOG(ERR, " failed to set dh params\n"); ++#if (OPENSSL_VERSION_NUMBER >= 0x30000000L) ++ BN_free(*p); ++ BN_free(*g); ++#else + BN_free(p); + BN_free(g); ++#endif + return -1; + } + case RTE_CRYPTO_ASYM_XFORM_DSA: + { + #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) +- BIGNUM *p = NULL, *g = NULL; +- BIGNUM *q = NULL, *priv_key = NULL; ++ BIGNUM **p = &asym_session->u.s.p; ++ BIGNUM **g = &asym_session->u.s.g; ++ BIGNUM **q = &asym_session->u.s.q; ++ BIGNUM **priv_key = &asym_session->u.s.priv_key; + BIGNUM *pub_key = BN_new(); + BN_zero(pub_key); + OSSL_PARAM_BLD *param_bld = NULL; + +- p = BN_bin2bn((const unsigned char *) ++ *p = BN_bin2bn((const unsigned char *) + xform->dsa.p.data, + xform->dsa.p.length, +- p); ++ *p); + +- g = BN_bin2bn((const unsigned char *) ++ *g = BN_bin2bn((const unsigned char *) + xform->dsa.g.data, + xform->dsa.g.length, +- g); ++ *g); + +- q = BN_bin2bn((const unsigned char *) ++ *q = BN_bin2bn((const unsigned char *) + xform->dsa.q.data, + xform->dsa.q.length, +- q); +- if (!p || !q || !g) ++ *q); ++ if (!*p || !*q || !*g) + goto err_dsa; + +- priv_key = BN_bin2bn((const unsigned char *) ++ *priv_key = BN_bin2bn((const unsigned char *) + xform->dsa.x.data, + xform->dsa.x.length, +- priv_key); +- if (priv_key == NULL) ++ *priv_key); ++ if (*priv_key == NULL) + goto err_dsa; + + param_bld = OSSL_PARAM_BLD_new(); +@@ -1201,10 +1221,11 @@ err_dh: + goto err_dsa; + } + +- if (!OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_FFC_P, p) +- || !OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_FFC_G, g) +- || !OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_FFC_Q, q) +- || !OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_PRIV_KEY, priv_key)) { ++ if (!OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_FFC_P, *p) ++ || !OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_FFC_G, *g) ++ || !OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_FFC_Q, *q) ++ || !OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_PRIV_KEY, ++ *priv_key)) { + OSSL_PARAM_BLD_free(param_bld); + OPENSSL_LOG(ERR, "failed to allocate resources\n"); + goto err_dsa; +@@ -1268,18 +1289,25 @@ err_dh: + if (ret) { + DSA_free(dsa); + OPENSSL_LOG(ERR, "Failed to set keys\n"); +- return -1; ++ goto err_dsa; + } + asym_session->u.s.dsa = dsa; + asym_session->xfrm_type = RTE_CRYPTO_ASYM_XFORM_DSA; + break; + #endif + err_dsa: ++#if (OPENSSL_VERSION_NUMBER >= 0x30000000L) ++ BN_free(*p); ++ BN_free(*q); ++ BN_free(*g); ++ BN_free(*priv_key); ++#else + BN_free(p); + BN_free(q); + BN_free(g); + BN_free(priv_key); + BN_free(pub_key); ++#endif + return -1; + } + default: +@@ -1357,10 +1385,16 @@ static void openssl_reset_asym_session(struct openssl_asym_session *sess) + if (sess->u.dh.dh_key) + DH_free(sess->u.dh.dh_key); + #endif ++ BN_clear_free(sess->u.dh.p); ++ BN_clear_free(sess->u.dh.g); + break; + case RTE_CRYPTO_ASYM_XFORM_DSA: + #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) + sess->u.s.param_bld = NULL; ++ BN_clear_free(sess->u.s.p); ++ BN_clear_free(sess->u.s.q); ++ BN_clear_free(sess->u.s.g); ++ BN_clear_free(sess->u.s.priv_key); + #else + if (sess->u.s.dsa) + DSA_free(sess->u.s.dsa); +diff --git a/dpdk/drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c b/dpdk/drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c +index 7f00f6097d..84d58accc7 100644 +--- a/dpdk/drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c ++++ b/dpdk/drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c +@@ -140,6 +140,9 @@ static struct rte_cryptodev_capabilities qat_sym_crypto_caps_gen3[] = { + QAT_SYM_CIPHER_CAP(SM4_CTR, + CAP_SET(block_size, 16), + CAP_RNG(key_size, 16, 16, 0), CAP_RNG(iv_size, 16, 16, 0)), ++ QAT_SYM_PLAIN_AUTH_CAP(SM3, ++ CAP_SET(block_size, 64), ++ CAP_RNG(digest_size, 32, 32, 0)), + RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() + }; + +@@ -404,7 +407,7 @@ qat_sym_build_op_auth_gen3(void *in_op, struct qat_sym_session *ctx, + rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req)); + + ofs.raw = qat_sym_convert_op_to_vec_auth(op, ctx, &in_sgl, &out_sgl, +- NULL, &auth_iv, &digest); ++ NULL, &auth_iv, &digest, op_cookie); + if (unlikely(ofs.raw == UINT64_MAX)) { + op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; + return -EINVAL; +@@ -602,6 +605,8 @@ qat_sym_dp_enqueue_single_auth_gen3(void *qp_data, uint8_t *drv_ctx, + struct icp_qat_fw_la_bulk_req *req; + int32_t data_len; + uint32_t tail = dp_ctx->tail; ++ struct rte_crypto_va_iova_ptr null_digest; ++ struct rte_crypto_va_iova_ptr *job_digest = digest; + + req = (struct icp_qat_fw_la_bulk_req *)( + (uint8_t *)tx_queue->base_addr + tail); +@@ -615,7 +620,12 @@ qat_sym_dp_enqueue_single_auth_gen3(void *qp_data, uint8_t *drv_ctx, + if (unlikely(data_len < 0)) + return -1; + +- enqueue_one_auth_job_gen3(ctx, cookie, req, digest, auth_iv, ofs, ++ if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_NULL) { ++ null_digest.iova = cookie->digest_null_phys_addr; ++ job_digest = &null_digest; ++ } ++ ++ enqueue_one_auth_job_gen3(ctx, cookie, req, job_digest, auth_iv, ofs, + (uint32_t)data_len); + + dp_ctx->tail = tail; +@@ -637,6 +647,8 @@ qat_sym_dp_enqueue_auth_jobs_gen3(void *qp_data, uint8_t *drv_ctx, + uint32_t tail; + struct icp_qat_fw_la_bulk_req *req; + int32_t data_len; ++ struct rte_crypto_va_iova_ptr null_digest; ++ struct rte_crypto_va_iova_ptr *job_digest = NULL; + + n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num); + if (unlikely(n == 0)) { +@@ -669,7 +681,13 @@ qat_sym_dp_enqueue_auth_jobs_gen3(void *qp_data, uint8_t *drv_ctx, + + if (unlikely(data_len < 0)) + break; +- enqueue_one_auth_job_gen3(ctx, cookie, req, &vec->digest[i], ++ if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_NULL) { ++ null_digest.iova = cookie->digest_null_phys_addr; ++ job_digest = &null_digest; ++ } else ++ job_digest = &vec->digest[i]; ++ ++ enqueue_one_auth_job_gen3(ctx, cookie, req, job_digest, + &vec->auth_iv[i], ofs, (uint32_t)data_len); + tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask; + } +diff --git a/dpdk/drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c b/dpdk/drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +index b1e5fa9a82..b219a418ba 100644 +--- a/dpdk/drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c ++++ b/dpdk/drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +@@ -100,6 +100,9 @@ static struct rte_cryptodev_capabilities qat_sym_crypto_caps_gen4[] = { + QAT_SYM_CIPHER_CAP(SM4_CTR, + CAP_SET(block_size, 16), + CAP_RNG(key_size, 16, 16, 0), CAP_RNG(iv_size, 16, 16, 0)), ++ QAT_SYM_PLAIN_AUTH_CAP(SM3, ++ CAP_SET(block_size, 64), ++ CAP_RNG(digest_size, 32, 32, 0)), + RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() + }; + +diff --git a/dpdk/drivers/crypto/qat/dev/qat_crypto_pmd_gens.h b/dpdk/drivers/crypto/qat/dev/qat_crypto_pmd_gens.h +index 524c291340..7972c7cfeb 100644 +--- a/dpdk/drivers/crypto/qat/dev/qat_crypto_pmd_gens.h ++++ b/dpdk/drivers/crypto/qat/dev/qat_crypto_pmd_gens.h +@@ -290,7 +290,8 @@ qat_sym_convert_op_to_vec_auth(struct rte_crypto_op *op, + struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl, + struct rte_crypto_va_iova_ptr *cipher_iv __rte_unused, + struct rte_crypto_va_iova_ptr *auth_iv, +- struct rte_crypto_va_iova_ptr *digest) ++ struct rte_crypto_va_iova_ptr *digest, ++ struct qat_sym_op_cookie *cookie) + { + uint32_t auth_ofs = 0, auth_len = 0; + int n_src, ret; +@@ -355,7 +356,11 @@ qat_sym_convert_op_to_vec_auth(struct rte_crypto_op *op, + out_sgl->num = 0; + + digest->va = (void *)op->sym->auth.digest.data; +- digest->iova = op->sym->auth.digest.phys_addr; ++ ++ if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_NULL) ++ digest->iova = cookie->digest_null_phys_addr; ++ else ++ digest->iova = op->sym->auth.digest.phys_addr; + + return 0; + } +@@ -366,7 +371,8 @@ qat_sym_convert_op_to_vec_chain(struct rte_crypto_op *op, + struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl, + struct rte_crypto_va_iova_ptr *cipher_iv, + struct rte_crypto_va_iova_ptr *auth_iv_or_aad, +- struct rte_crypto_va_iova_ptr *digest) ++ struct rte_crypto_va_iova_ptr *digest, ++ struct qat_sym_op_cookie *cookie) + { + union rte_crypto_sym_ofs ofs; + uint32_t max_len = 0; +@@ -390,7 +396,11 @@ qat_sym_convert_op_to_vec_chain(struct rte_crypto_op *op, + auth_iv_or_aad->iova = rte_crypto_op_ctophys_offset(op, + ctx->auth_iv.offset); + digest->va = (void *)op->sym->auth.digest.data; +- digest->iova = op->sym->auth.digest.phys_addr; ++ ++ if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_NULL) ++ digest->iova = cookie->digest_null_phys_addr; ++ else ++ digest->iova = op->sym->auth.digest.phys_addr; + + ret = qat_cipher_is_len_in_bits(ctx, op); + switch (ret) { +@@ -682,7 +692,8 @@ enqueue_one_chain_job_gen1(struct qat_sym_session *ctx, + while (remaining_off >= cvec->len && i >= 1) { + i--; + remaining_off -= cvec->len; +- cvec++; ++ if (i) ++ cvec++; + } + + auth_iova_end = cvec->iova + remaining_off; +diff --git a/dpdk/drivers/crypto/qat/dev/qat_sym_pmd_gen1.c b/dpdk/drivers/crypto/qat/dev/qat_sym_pmd_gen1.c +index 91d5cfa71d..888dea4ad9 100644 +--- a/dpdk/drivers/crypto/qat/dev/qat_sym_pmd_gen1.c ++++ b/dpdk/drivers/crypto/qat/dev/qat_sym_pmd_gen1.c +@@ -274,7 +274,7 @@ qat_sym_build_op_auth_gen1(void *in_op, struct qat_sym_session *ctx, + rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req)); + + ofs.raw = qat_sym_convert_op_to_vec_auth(op, ctx, &in_sgl, &out_sgl, +- NULL, &auth_iv, &digest); ++ NULL, &auth_iv, &digest, op_cookie); + if (unlikely(ofs.raw == UINT64_MAX)) { + op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; + return -EINVAL; +@@ -368,7 +368,7 @@ qat_sym_build_op_chain_gen1(void *in_op, struct qat_sym_session *ctx, + rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req)); + + ofs.raw = qat_sym_convert_op_to_vec_chain(op, ctx, &in_sgl, &out_sgl, +- &cipher_iv, &auth_iv, &digest); ++ &cipher_iv, &auth_iv, &digest, cookie); + if (unlikely(ofs.raw == UINT64_MAX)) { + op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; + return -EINVAL; +@@ -607,6 +607,8 @@ qat_sym_dp_enqueue_single_auth_gen1(void *qp_data, uint8_t *drv_ctx, + struct icp_qat_fw_la_bulk_req *req; + int32_t data_len; + uint32_t tail = dp_ctx->tail; ++ struct rte_crypto_va_iova_ptr null_digest; ++ struct rte_crypto_va_iova_ptr *job_digest = digest; + + req = (struct icp_qat_fw_la_bulk_req *)( + (uint8_t *)tx_queue->base_addr + tail); +@@ -620,8 +622,13 @@ qat_sym_dp_enqueue_single_auth_gen1(void *qp_data, uint8_t *drv_ctx, + if (unlikely(data_len < 0)) + return -1; + +- enqueue_one_auth_job_gen1(ctx, req, digest, auth_iv, ofs, +- (uint32_t)data_len); ++ if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_NULL) { ++ null_digest.iova = cookie->digest_null_phys_addr; ++ job_digest = &null_digest; ++ } ++ ++ enqueue_one_auth_job_gen1(ctx, req, job_digest, auth_iv, ofs, ++ (uint32_t)data_len); + + dp_ctx->tail = tail; + dp_ctx->cached_enqueue++; +@@ -646,6 +653,8 @@ qat_sym_dp_enqueue_auth_jobs_gen1(void *qp_data, uint8_t *drv_ctx, + uint32_t tail; + struct icp_qat_fw_la_bulk_req *req; + int32_t data_len; ++ struct rte_crypto_va_iova_ptr null_digest; ++ struct rte_crypto_va_iova_ptr *job_digest = NULL; + + n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num); + if (unlikely(n == 0)) { +@@ -678,7 +687,14 @@ qat_sym_dp_enqueue_auth_jobs_gen1(void *qp_data, uint8_t *drv_ctx, + + if (unlikely(data_len < 0)) + break; +- enqueue_one_auth_job_gen1(ctx, req, &vec->digest[i], ++ ++ if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_NULL) { ++ null_digest.iova = cookie->digest_null_phys_addr; ++ job_digest = &null_digest; ++ } else ++ job_digest = &vec->digest[i]; ++ ++ enqueue_one_auth_job_gen1(ctx, req, job_digest, + &vec->auth_iv[i], ofs, (uint32_t)data_len); + tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask; + +@@ -715,6 +731,8 @@ qat_sym_dp_enqueue_single_chain_gen1(void *qp_data, uint8_t *drv_ctx, + struct icp_qat_fw_la_bulk_req *req; + int32_t data_len; + uint32_t tail = dp_ctx->tail; ++ struct rte_crypto_va_iova_ptr null_digest; ++ struct rte_crypto_va_iova_ptr *job_digest = digest; + + req = (struct icp_qat_fw_la_bulk_req *)( + (uint8_t *)tx_queue->base_addr + tail); +@@ -727,8 +745,13 @@ qat_sym_dp_enqueue_single_chain_gen1(void *qp_data, uint8_t *drv_ctx, + if (unlikely(data_len < 0)) + return -1; + ++ if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_NULL) { ++ null_digest.iova = cookie->digest_null_phys_addr; ++ job_digest = &null_digest; ++ } ++ + if (unlikely(enqueue_one_chain_job_gen1(ctx, req, data, n_data_vecs, +- NULL, 0, cipher_iv, digest, auth_iv, ofs, ++ NULL, 0, cipher_iv, job_digest, auth_iv, ofs, + (uint32_t)data_len))) + return -1; + +@@ -756,6 +779,8 @@ qat_sym_dp_enqueue_chain_jobs_gen1(void *qp_data, uint8_t *drv_ctx, + uint32_t tail; + struct icp_qat_fw_la_bulk_req *req; + int32_t data_len; ++ struct rte_crypto_va_iova_ptr null_digest; ++ struct rte_crypto_va_iova_ptr *job_digest; + + n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num); + if (unlikely(n == 0)) { +@@ -789,10 +814,16 @@ qat_sym_dp_enqueue_chain_jobs_gen1(void *qp_data, uint8_t *drv_ctx, + if (unlikely(data_len < 0)) + break; + ++ if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_NULL) { ++ null_digest.iova = cookie->digest_null_phys_addr; ++ job_digest = &null_digest; ++ } else ++ job_digest = &vec->digest[i]; ++ + if (unlikely(enqueue_one_chain_job_gen1(ctx, req, + vec->src_sgl[i].vec, vec->src_sgl[i].num, + NULL, 0, +- &vec->iv[i], &vec->digest[i], ++ &vec->iv[i], job_digest, + &vec->auth_iv[i], ofs, (uint32_t)data_len))) + break; + +diff --git a/dpdk/drivers/crypto/qat/qat_sym.c b/dpdk/drivers/crypto/qat/qat_sym.c +index 08e92191a3..18f99089e8 100644 +--- a/dpdk/drivers/crypto/qat/qat_sym.c ++++ b/dpdk/drivers/crypto/qat/qat_sym.c +@@ -51,6 +51,11 @@ qat_sym_init_op_cookie(void *op_cookie) + rte_mempool_virt2iova(cookie) + + offsetof(struct qat_sym_op_cookie, + opt.spc_gmac.cd_cipher); ++ ++ cookie->digest_null_phys_addr = ++ rte_mempool_virt2iova(cookie) + ++ offsetof(struct qat_sym_op_cookie, ++ digest_null); + } + + static __rte_always_inline int +@@ -179,7 +184,7 @@ qat_sym_dequeue_burst(void *qp, struct rte_crypto_op **ops, + + int + qat_sym_dev_create(struct qat_pci_device *qat_pci_dev, +- struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused) ++ struct qat_dev_cmd_param *qat_dev_cmd_param) + { + int i = 0, ret = 0; + uint16_t slice_map = 0; +diff --git a/dpdk/drivers/crypto/qat/qat_sym.h b/dpdk/drivers/crypto/qat/qat_sym.h +index 9a4251e08b..a45bddf848 100644 +--- a/dpdk/drivers/crypto/qat/qat_sym.h ++++ b/dpdk/drivers/crypto/qat/qat_sym.h +@@ -116,6 +116,8 @@ struct qat_sym_op_cookie { + phys_addr_t cd_phys_addr; + } spc_gmac; + } opt; ++ uint8_t digest_null[4]; ++ phys_addr_t digest_null_phys_addr; + }; + + struct qat_sym_dp_ctx { +diff --git a/dpdk/drivers/crypto/qat/qat_sym_session.c b/dpdk/drivers/crypto/qat/qat_sym_session.c +index 0ebc66f89e..52ed921db2 100644 +--- a/dpdk/drivers/crypto/qat/qat_sym_session.c ++++ b/dpdk/drivers/crypto/qat/qat_sym_session.c +@@ -10,6 +10,7 @@ + #include /* Needed for bpi runt block processing */ + + #ifdef RTE_QAT_LIBIPSECMB ++#define NO_COMPAT_IMB_API_053 + #if defined(RTE_ARCH_ARM) + #include + #else +@@ -696,7 +697,7 @@ qat_sym_session_configure_auth(struct rte_cryptodev *dev, + switch (auth_xform->algo) { + case RTE_CRYPTO_AUTH_SM3: + session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SM3; +- session->auth_mode = ICP_QAT_HW_AUTH_MODE2; ++ session->auth_mode = ICP_QAT_HW_AUTH_MODE0; + break; + case RTE_CRYPTO_AUTH_SHA1: + session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA1; +@@ -1331,6 +1332,8 @@ static int qat_sym_do_precomputes_ipsec_mb(enum icp_qat_hw_auth_algo hash_alg, + /* init ipad and opad from key and xor with fixed values */ + memset(ipad, 0, block_size); + memset(opad, 0, block_size); ++ RTE_VERIFY(auth_keylen <= sizeof(ipad)); ++ RTE_VERIFY(auth_keylen <= sizeof(opad)); + rte_memcpy(ipad, auth_key, auth_keylen); + rte_memcpy(opad, auth_key, auth_keylen); + +@@ -1849,9 +1852,10 @@ int qat_sym_cd_cipher_set(struct qat_sym_session *cdesc, + key_convert = ICP_QAT_HW_CIPHER_NO_CONVERT; + } else if (cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 + || cdesc->qat_cipher_alg == +- ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) ++ ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) { + key_convert = ICP_QAT_HW_CIPHER_KEY_CONVERT; +- else if (cdesc->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT) ++ cdesc->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT; ++ } else if (cdesc->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT) + key_convert = ICP_QAT_HW_CIPHER_NO_CONVERT; + else if (cdesc->qat_mode == ICP_QAT_HW_CIPHER_AEAD_MODE) + key_convert = ICP_QAT_HW_CIPHER_NO_CONVERT; +diff --git a/dpdk/drivers/crypto/scheduler/scheduler_pmd.c b/dpdk/drivers/crypto/scheduler/scheduler_pmd.c +index 9d1ce46622..4e8bbf0e09 100644 +--- a/dpdk/drivers/crypto/scheduler/scheduler_pmd.c ++++ b/dpdk/drivers/crypto/scheduler/scheduler_pmd.c +@@ -50,7 +50,8 @@ static const char * const scheduler_valid_params[] = { + RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG, + RTE_CRYPTODEV_VDEV_SOCKET_ID, + RTE_CRYPTODEV_VDEV_COREMASK, +- RTE_CRYPTODEV_VDEV_CORELIST ++ RTE_CRYPTODEV_VDEV_CORELIST, ++ NULL + }; + + struct scheduler_parse_map { +diff --git a/dpdk/drivers/dma/cnxk/cnxk_dmadev.c b/dpdk/drivers/dma/cnxk/cnxk_dmadev.c +index a6f4a31e0e..72dec60a9c 100644 +--- a/dpdk/drivers/dma/cnxk/cnxk_dmadev.c ++++ b/dpdk/drivers/dma/cnxk/cnxk_dmadev.c +@@ -45,14 +45,22 @@ cnxk_dmadev_configure(struct rte_dma_dev *dev, + int rc = 0; + + RTE_SET_USED(conf); +- RTE_SET_USED(conf); +- RTE_SET_USED(conf_sz); + RTE_SET_USED(conf_sz); ++ + dpivf = dev->fp_obj->dev_private; ++ ++ if (dpivf->flag & CNXK_DPI_DEV_CONFIG) ++ return rc; ++ + rc = roc_dpi_configure(&dpivf->rdpi); +- if (rc < 0) ++ if (rc < 0) { + plt_err("DMA configure failed err = %d", rc); ++ goto done; ++ } ++ ++ dpivf->flag |= CNXK_DPI_DEV_CONFIG; + ++done: + return rc; + } + +@@ -69,6 +77,9 @@ cnxk_dmadev_vchan_setup(struct rte_dma_dev *dev, uint16_t vchan, + RTE_SET_USED(vchan); + RTE_SET_USED(conf_sz); + ++ if (dpivf->flag & CNXK_DPI_VCHAN_CONFIG) ++ return 0; ++ + header->cn9k.pt = DPI_HDR_PT_ZBW_CA; + + switch (conf->direction) { +@@ -108,6 +119,7 @@ cnxk_dmadev_vchan_setup(struct rte_dma_dev *dev, uint16_t vchan, + dpivf->conf.c_desc.max_cnt = DPI_MAX_DESC; + dpivf->conf.c_desc.head = 0; + dpivf->conf.c_desc.tail = 0; ++ dpivf->flag |= CNXK_DPI_VCHAN_CONFIG; + + return 0; + } +@@ -125,6 +137,10 @@ cn10k_dmadev_vchan_setup(struct rte_dma_dev *dev, uint16_t vchan, + RTE_SET_USED(vchan); + RTE_SET_USED(conf_sz); + ++ ++ if (dpivf->flag & CNXK_DPI_VCHAN_CONFIG) ++ return 0; ++ + header->cn10k.pt = DPI_HDR_PT_ZBW_CA; + + switch (conf->direction) { +@@ -164,6 +180,7 @@ cn10k_dmadev_vchan_setup(struct rte_dma_dev *dev, uint16_t vchan, + dpivf->conf.c_desc.max_cnt = DPI_MAX_DESC; + dpivf->conf.c_desc.head = 0; + dpivf->conf.c_desc.tail = 0; ++ dpivf->flag |= CNXK_DPI_VCHAN_CONFIG; + + return 0; + } +@@ -173,10 +190,15 @@ cnxk_dmadev_start(struct rte_dma_dev *dev) + { + struct cnxk_dpi_vf_s *dpivf = dev->fp_obj->dev_private; + ++ if (dpivf->flag & CNXK_DPI_DEV_START) ++ return 0; ++ + dpivf->desc_idx = 0; + dpivf->num_words = 0; + roc_dpi_enable(&dpivf->rdpi); + ++ dpivf->flag |= CNXK_DPI_DEV_START; ++ + return 0; + } + +@@ -187,6 +209,8 @@ cnxk_dmadev_stop(struct rte_dma_dev *dev) + + roc_dpi_disable(&dpivf->rdpi); + ++ dpivf->flag &= ~CNXK_DPI_DEV_START; ++ + return 0; + } + +@@ -198,6 +222,8 @@ cnxk_dmadev_close(struct rte_dma_dev *dev) + roc_dpi_disable(&dpivf->rdpi); + roc_dpi_dev_fini(&dpivf->rdpi); + ++ dpivf->flag = 0; ++ + return 0; + } + +@@ -206,8 +232,7 @@ __dpi_queue_write(struct roc_dpi *dpi, uint64_t *cmds, int cmd_count) + { + uint64_t *ptr = dpi->chunk_base; + +- if ((cmd_count < DPI_MIN_CMD_SIZE) || (cmd_count > DPI_MAX_CMD_SIZE) || +- cmds == NULL) ++ if ((cmd_count < DPI_MIN_CMD_SIZE) || (cmd_count > DPI_MAX_CMD_SIZE) || cmds == NULL) + return -EINVAL; + + /* +@@ -223,11 +248,15 @@ __dpi_queue_write(struct roc_dpi *dpi, uint64_t *cmds, int cmd_count) + int count; + uint64_t *new_buff = dpi->chunk_next; + +- dpi->chunk_next = +- (void *)roc_npa_aura_op_alloc(dpi->aura_handle, 0); ++ dpi->chunk_next = (void *)roc_npa_aura_op_alloc(dpi->aura_handle, 0); + if (!dpi->chunk_next) { +- plt_err("Failed to alloc next buffer from NPA"); +- return -ENOMEM; ++ plt_dp_dbg("Failed to alloc next buffer from NPA"); ++ ++ /* NPA failed to allocate a buffer. Restoring chunk_next ++ * to its original address. ++ */ ++ dpi->chunk_next = new_buff; ++ return -ENOSPC; + } + + /* +@@ -261,13 +290,17 @@ __dpi_queue_write(struct roc_dpi *dpi, uint64_t *cmds, int cmd_count) + /* queue index may be greater than pool size */ + if (dpi->chunk_head >= dpi->pool_size_m1) { + new_buff = dpi->chunk_next; +- dpi->chunk_next = +- (void *)roc_npa_aura_op_alloc(dpi->aura_handle, +- 0); ++ dpi->chunk_next = (void *)roc_npa_aura_op_alloc(dpi->aura_handle, 0); + if (!dpi->chunk_next) { +- plt_err("Failed to alloc next buffer from NPA"); +- return -ENOMEM; ++ plt_dp_dbg("Failed to alloc next buffer from NPA"); ++ ++ /* NPA failed to allocate a buffer. Restoring chunk_next ++ * to its original address. ++ */ ++ dpi->chunk_next = new_buff; ++ return -ENOSPC; + } ++ + /* Write next buffer address */ + *ptr = (uint64_t)new_buff; + dpi->chunk_base = new_buff; +@@ -628,8 +661,7 @@ static const struct rte_dma_dev_ops cnxk_dmadev_ops = { + }; + + static int +-cnxk_dmadev_probe(struct rte_pci_driver *pci_drv __rte_unused, +- struct rte_pci_device *pci_dev) ++cnxk_dmadev_probe(struct rte_pci_driver *pci_drv __rte_unused, struct rte_pci_device *pci_dev) + { + struct cnxk_dpi_vf_s *dpivf = NULL; + char name[RTE_DEV_NAME_MAX_LEN]; +@@ -648,8 +680,7 @@ cnxk_dmadev_probe(struct rte_pci_driver *pci_drv __rte_unused, + memset(name, 0, sizeof(name)); + rte_pci_device_name(&pci_dev->addr, name, sizeof(name)); + +- dmadev = rte_dma_pmd_allocate(name, pci_dev->device.numa_node, +- sizeof(*dpivf)); ++ dmadev = rte_dma_pmd_allocate(name, pci_dev->device.numa_node, sizeof(*dpivf)); + if (dmadev == NULL) { + plt_err("dma device allocation failed for %s", name); + return -ENOMEM; +@@ -682,6 +713,8 @@ cnxk_dmadev_probe(struct rte_pci_driver *pci_drv __rte_unused, + if (rc < 0) + goto err_out_free; + ++ dmadev->state = RTE_DMA_DEV_READY; ++ + return 0; + + err_out_free: +diff --git a/dpdk/drivers/dma/cnxk/cnxk_dmadev.h b/dpdk/drivers/dma/cnxk/cnxk_dmadev.h +index e1f5694f50..d58554787f 100644 +--- a/dpdk/drivers/dma/cnxk/cnxk_dmadev.h ++++ b/dpdk/drivers/dma/cnxk/cnxk_dmadev.h +@@ -15,6 +15,10 @@ + */ + #define DPI_REQ_CDATA 0xFF + ++#define CNXK_DPI_DEV_CONFIG (1ULL << 0) ++#define CNXK_DPI_VCHAN_CONFIG (1ULL << 1) ++#define CNXK_DPI_DEV_START (1ULL << 2) ++ + struct cnxk_dpi_compl_s { + uint64_t cdata; + void *cb_data; +@@ -39,6 +43,7 @@ struct cnxk_dpi_vf_s { + uint64_t cmd[DPI_MAX_CMD_SIZE]; + uint32_t num_words; + uint16_t desc_idx; ++ uint16_t flag; + }; + + #endif +diff --git a/dpdk/drivers/dma/dpaa2/dpaa2_qdma.c b/dpdk/drivers/dma/dpaa2/dpaa2_qdma.c +index d5a5f08ecc..8968bb853b 100644 +--- a/dpdk/drivers/dma/dpaa2/dpaa2_qdma.c ++++ b/dpdk/drivers/dma/dpaa2/dpaa2_qdma.c +@@ -117,6 +117,7 @@ dpaa2_qdma_populate_fle(struct qbman_fle *fle, + /* source */ + sdd->read_cmd.portid = rbp->sportid; + sdd->rbpcmd_simple.pfid = rbp->spfid; ++ sdd->rbpcmd_simple.vfa = rbp->vfa; + sdd->rbpcmd_simple.vfid = rbp->svfid; + + if (rbp->srbp) { +@@ -129,6 +130,7 @@ dpaa2_qdma_populate_fle(struct qbman_fle *fle, + /* destination */ + sdd->write_cmd.portid = rbp->dportid; + sdd->rbpcmd_simple.pfid = rbp->dpfid; ++ sdd->rbpcmd_simple.vfa = rbp->vfa; + sdd->rbpcmd_simple.vfid = rbp->dvfid; + + if (rbp->drbp) { +diff --git a/dpdk/drivers/dma/dpaa2/rte_pmd_dpaa2_qdma.h b/dpdk/drivers/dma/dpaa2/rte_pmd_dpaa2_qdma.h +index dc8acb4aec..5a8da46d12 100644 +--- a/dpdk/drivers/dma/dpaa2/rte_pmd_dpaa2_qdma.h ++++ b/dpdk/drivers/dma/dpaa2/rte_pmd_dpaa2_qdma.h +@@ -44,7 +44,9 @@ struct rte_dpaa2_qdma_rbp { + uint32_t svfid:6; + /* using route by port for source */ + uint32_t srbp:1; +- uint32_t rsv:4; ++ /* Virtual Function Active */ ++ uint32_t vfa:1; ++ uint32_t rsv:3; + }; + + /** Determines a QDMA job */ +diff --git a/dpdk/drivers/dma/ioat/ioat_dmadev.c b/dpdk/drivers/dma/ioat/ioat_dmadev.c +index 5906eb45aa..57c18c081d 100644 +--- a/dpdk/drivers/dma/ioat/ioat_dmadev.c ++++ b/dpdk/drivers/dma/ioat/ioat_dmadev.c +@@ -142,10 +142,20 @@ ioat_dev_start(struct rte_dma_dev *dev) + ioat->regs->chainaddr = ioat->ring_addr; + /* Inform hardware of where to write the status/completions. */ + ioat->regs->chancmp = ioat->status_addr; ++ /* Ensure channel control is set to abort on error, so we get status writeback. */ ++ ioat->regs->chanctrl = IOAT_CHANCTRL_ANY_ERR_ABORT_EN | ++ IOAT_CHANCTRL_ERR_COMPLETION_EN; + + /* Prime the status register to be set to the last element. */ + ioat->status = ioat->ring_addr + ((ioat->qcfg.nb_desc - 1) * DESC_SZ); + ++ /* reset all counters */ ++ ioat->next_read = 0; ++ ioat->next_write = 0; ++ ioat->last_write = 0; ++ ioat->offset = 0; ++ ioat->failure = 0; ++ + printf("IOAT.status: %s [0x%"PRIx64"]\n", + chansts_readable[ioat->status & IOAT_CHANSTS_STATUS], + ioat->status); +@@ -166,17 +176,28 @@ static int + ioat_dev_stop(struct rte_dma_dev *dev) + { + struct ioat_dmadev *ioat = dev->fp_obj->dev_private; ++ unsigned int chansts; + uint32_t retry = 0; + +- ioat->regs->chancmd = IOAT_CHANCMD_SUSPEND; ++ chansts = (unsigned int)(ioat->regs->chansts & IOAT_CHANSTS_STATUS); ++ if (chansts == IOAT_CHANSTS_ACTIVE || chansts == IOAT_CHANSTS_IDLE) ++ ioat->regs->chancmd = IOAT_CHANCMD_SUSPEND; ++ else ++ ioat->regs->chancmd = IOAT_CHANCMD_RESET; + + do { + rte_pause(); + retry++; +- } while ((ioat->regs->chansts & IOAT_CHANSTS_STATUS) != IOAT_CHANSTS_SUSPENDED +- && retry < 200); ++ chansts = (unsigned int)(ioat->regs->chansts & IOAT_CHANSTS_STATUS); ++ } while (chansts != IOAT_CHANSTS_SUSPENDED && ++ chansts != IOAT_CHANSTS_HALTED && retry < 200); + +- return ((ioat->regs->chansts & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_SUSPENDED) ? 0 : -1; ++ if (chansts == IOAT_CHANSTS_SUSPENDED || chansts == IOAT_CHANSTS_HALTED) ++ return 0; ++ ++ IOAT_PMD_WARN("Channel could not be suspended on stop. (chansts = %u [%s])", ++ chansts, chansts_readable[chansts]); ++ return -1; + } + + /* Get device information of a device. */ +@@ -664,8 +685,6 @@ ioat_dmadev_create(const char *name, struct rte_pci_device *dev) + return -EIO; + } + } +- ioat->regs->chanctrl = IOAT_CHANCTRL_ANY_ERR_ABORT_EN | +- IOAT_CHANCTRL_ERR_COMPLETION_EN; + + dmadev->fp_obj->dev_private = ioat; + +diff --git a/dpdk/drivers/dma/skeleton/skeleton_dmadev.c b/dpdk/drivers/dma/skeleton/skeleton_dmadev.c +index 9b6da655fd..daf35eccce 100644 +--- a/dpdk/drivers/dma/skeleton/skeleton_dmadev.c ++++ b/dpdk/drivers/dma/skeleton/skeleton_dmadev.c +@@ -515,9 +515,15 @@ skeldma_parse_lcore(const char *key __rte_unused, + const char *value, + void *opaque) + { +- int lcore_id = atoi(value); ++ int lcore_id; ++ ++ if (value == NULL || opaque == NULL) ++ return -EINVAL; ++ ++ lcore_id = atoi(value); + if (lcore_id >= 0 && lcore_id < RTE_MAX_LCORE) + *(int *)opaque = lcore_id; ++ + return 0; + } + +diff --git a/dpdk/drivers/event/cnxk/cn10k_eventdev.c b/dpdk/drivers/event/cnxk/cn10k_eventdev.c +index 30c922b5fc..d8e7c83462 100644 +--- a/dpdk/drivers/event/cnxk/cn10k_eventdev.c ++++ b/dpdk/drivers/event/cnxk/cn10k_eventdev.c +@@ -197,12 +197,14 @@ cn10k_sso_hws_reset(void *arg, void *hws) + cnxk_sso_hws_swtag_untag(base + + SSOW_LF_GWS_OP_SWTAG_UNTAG); + plt_write64(0, base + SSOW_LF_GWS_OP_DESCHED); ++ } else if (pend_tt != SSO_TT_EMPTY) { ++ plt_write64(0, base + SSOW_LF_GWS_OP_SWTAG_FLUSH); + } + + /* Wait for desched to complete. */ + do { + pend_state = plt_read64(base + SSOW_LF_GWS_PENDSTATE); +- } while (pend_state & BIT_ULL(58)); ++ } while (pend_state & (BIT_ULL(58) | BIT_ULL(56))); + + switch (dev->gw_mode) { + case CN10K_GW_MODE_PREF: +@@ -711,11 +713,16 @@ cn10k_sso_port_quiesce(struct rte_eventdev *event_dev, void *port, + cn10k_sso_hws_get_work_empty(ws, &ev, + (NIX_RX_OFFLOAD_MAX - 1) | NIX_RX_REAS_F | + NIX_RX_MULTI_SEG_F | CPT_RX_WQE_F); +- if (is_pend && ev.u64) { ++ if (is_pend && ev.u64) + if (flush_cb) + flush_cb(event_dev->data->dev_id, ev, args); ++ ptag = (plt_read64(ws->base + SSOW_LF_GWS_TAG) >> 32) & SSO_TT_EMPTY; ++ if (ptag != SSO_TT_EMPTY) + cnxk_sso_hws_swtag_flush(ws->base); +- } ++ ++ do { ++ ptag = plt_read64(ws->base + SSOW_LF_GWS_PENDSTATE); ++ } while (ptag & BIT_ULL(56)); + + /* Check if we have work in PRF_WQE0, if so extract it. */ + switch (dev->gw_mode) { +@@ -741,8 +748,11 @@ cn10k_sso_port_quiesce(struct rte_eventdev *event_dev, void *port, + if (ev.u64) { + if (flush_cb) + flush_cb(event_dev->data->dev_id, ev, args); +- cnxk_sso_hws_swtag_flush(ws->base); + } ++ cnxk_sso_hws_swtag_flush(ws->base); ++ do { ++ ptag = plt_read64(ws->base + SSOW_LF_GWS_PENDSTATE); ++ } while (ptag & BIT_ULL(56)); + } + ws->swtag_req = 0; + plt_write64(0, ws->base + SSOW_LF_GWS_OP_GWC_INVAL); +@@ -1024,8 +1034,8 @@ static int + cn10k_crypto_adapter_caps_get(const struct rte_eventdev *event_dev, + const struct rte_cryptodev *cdev, uint32_t *caps) + { +- CNXK_VALID_DEV_OR_ERR_RET(event_dev->dev, "event_cn10k"); +- CNXK_VALID_DEV_OR_ERR_RET(cdev->device, "crypto_cn10k"); ++ CNXK_VALID_DEV_OR_ERR_RET(event_dev->dev, "event_cn10k", ENOTSUP); ++ CNXK_VALID_DEV_OR_ERR_RET(cdev->device, "crypto_cn10k", ENOTSUP); + + *caps = RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_FWD | + RTE_EVENT_CRYPTO_ADAPTER_CAP_SESSION_PRIVATE_DATA | +@@ -1043,8 +1053,8 @@ cn10k_crypto_adapter_qp_add(const struct rte_eventdev *event_dev, + struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(event_dev); + int ret; + +- CNXK_VALID_DEV_OR_ERR_RET(event_dev->dev, "event_cn10k"); +- CNXK_VALID_DEV_OR_ERR_RET(cdev->device, "crypto_cn10k"); ++ CNXK_VALID_DEV_OR_ERR_RET(event_dev->dev, "event_cn10k", EINVAL); ++ CNXK_VALID_DEV_OR_ERR_RET(cdev->device, "crypto_cn10k", EINVAL); + + dev->is_ca_internal_port = 1; + cn10k_sso_fp_fns_set((struct rte_eventdev *)(uintptr_t)event_dev); +@@ -1059,8 +1069,8 @@ static int + cn10k_crypto_adapter_qp_del(const struct rte_eventdev *event_dev, const struct rte_cryptodev *cdev, + int32_t queue_pair_id) + { +- CNXK_VALID_DEV_OR_ERR_RET(event_dev->dev, "event_cn10k"); +- CNXK_VALID_DEV_OR_ERR_RET(cdev->device, "crypto_cn10k"); ++ CNXK_VALID_DEV_OR_ERR_RET(event_dev->dev, "event_cn10k", EINVAL); ++ CNXK_VALID_DEV_OR_ERR_RET(cdev->device, "crypto_cn10k", EINVAL); + + return cnxk_crypto_adapter_qp_del(cdev, queue_pair_id); + } +@@ -1078,8 +1088,8 @@ cn10k_crypto_adapter_vec_limits(const struct rte_eventdev *event_dev, + const struct rte_cryptodev *cdev, + struct rte_event_crypto_adapter_vector_limits *limits) + { +- CNXK_VALID_DEV_OR_ERR_RET(event_dev->dev, "event_cn10k"); +- CNXK_VALID_DEV_OR_ERR_RET(cdev->device, "crypto_cn10k"); ++ CNXK_VALID_DEV_OR_ERR_RET(event_dev->dev, "event_cn10k", EINVAL); ++ CNXK_VALID_DEV_OR_ERR_RET(cdev->device, "crypto_cn10k", EINVAL); + + limits->log2_sz = false; + limits->min_sz = 0; +diff --git a/dpdk/drivers/event/cnxk/cn10k_worker.h b/dpdk/drivers/event/cnxk/cn10k_worker.h +index 75a2ff244a..1bce0631e5 100644 +--- a/dpdk/drivers/event/cnxk/cn10k_worker.h ++++ b/dpdk/drivers/event/cnxk/cn10k_worker.h +@@ -100,9 +100,6 @@ cn10k_wqe_to_mbuf(uint64_t wqe, const uint64_t __mbuf, uint8_t port_id, + (flags & NIX_RX_OFFLOAD_TSTAMP_F ? 8 : 0); + struct rte_mbuf *mbuf = (struct rte_mbuf *)__mbuf; + +- /* Mark mempool obj as "get" as it is alloc'ed by NIX */ +- RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf, 1, 1); +- + cn10k_nix_cqe_to_mbuf((struct nix_cqe_hdr_s *)wqe, tag, + (struct rte_mbuf *)mbuf, lookup_mem, + mbuf_init | ((uint64_t)port_id) << 48, flags); +@@ -239,6 +236,10 @@ cn10k_sso_hws_post_process(struct cn10k_sso_hws *ws, uint64_t *u64, + + mbuf = u64[1] - sizeof(struct rte_mbuf); + rte_prefetch0((void *)mbuf); ++ ++ /* Mark mempool obj as "get" as it is alloc'ed by NIX */ ++ RTE_MEMPOOL_CHECK_COOKIES(((struct rte_mbuf *)mbuf)->pool, (void **)&mbuf, 1, 1); ++ + if (flags & NIX_RX_OFFLOAD_SECURITY_F) { + const uint64_t mbuf_init = + 0x100010000ULL | RTE_PKTMBUF_HEADROOM | +@@ -299,23 +300,39 @@ cn10k_sso_hws_get_work(struct cn10k_sso_hws *ws, struct rte_event *ev, + } gw; + + gw.get_work = ws->gw_wdata; +-#if defined(RTE_ARCH_ARM64) && !defined(__clang__) ++#if defined(RTE_ARCH_ARM64) ++#if !defined(__clang__) + asm volatile( + PLT_CPU_FEATURE_PREAMBLE + "caspal %[wdata], %H[wdata], %[wdata], %H[wdata], [%[gw_loc]]\n" + : [wdata] "+r"(gw.get_work) + : [gw_loc] "r"(ws->base + SSOW_LF_GWS_OP_GET_WORK0) + : "memory"); ++#else ++ register uint64_t x0 __asm("x0") = (uint64_t)gw.u64[0]; ++ register uint64_t x1 __asm("x1") = (uint64_t)gw.u64[1]; ++ asm volatile(".arch armv8-a+lse\n" ++ "caspal %[x0], %[x1], %[x0], %[x1], [%[dst]]\n" ++ : [x0] "+r"(x0), [x1] "+r"(x1) ++ : [dst] "r"(ws->base + SSOW_LF_GWS_OP_GET_WORK0) ++ : "memory"); ++ gw.u64[0] = x0; ++ gw.u64[1] = x1; ++#endif + #else + plt_write64(gw.u64[0], ws->base + SSOW_LF_GWS_OP_GET_WORK0); + do { + roc_load_pair(gw.u64[0], gw.u64[1], + ws->base + SSOW_LF_GWS_WQE0); + } while (gw.u64[0] & BIT_ULL(63)); ++ rte_atomic_thread_fence(__ATOMIC_SEQ_CST); + #endif + ws->gw_rdata = gw.u64[0]; + if (gw.u64[1]) + cn10k_sso_hws_post_process(ws, gw.u64, flags); ++ else ++ gw.u64[0] = (gw.u64[0] & (0x3ull << 32)) << 6 | ++ (gw.u64[0] & (0x3FFull << 36)) << 4 | (gw.u64[0] & 0xffffffff); + + ev->event = gw.u64[0]; + ev->u64 = gw.u64[1]; +diff --git a/dpdk/drivers/event/cnxk/cn9k_eventdev.c b/dpdk/drivers/event/cnxk/cn9k_eventdev.c +index f5a42a86f8..803e7ddd07 100644 +--- a/dpdk/drivers/event/cnxk/cn9k_eventdev.c ++++ b/dpdk/drivers/event/cnxk/cn9k_eventdev.c +@@ -223,16 +223,16 @@ cn9k_sso_hws_reset(void *arg, void *hws) + cnxk_sso_hws_swtag_untag( + base + SSOW_LF_GWS_OP_SWTAG_UNTAG); + plt_write64(0, base + SSOW_LF_GWS_OP_DESCHED); ++ } else if (pend_tt != SSO_TT_EMPTY) { ++ plt_write64(0, base + SSOW_LF_GWS_OP_SWTAG_FLUSH); + } + + /* Wait for desched to complete. */ + do { + pend_state = plt_read64(base + SSOW_LF_GWS_PENDSTATE); +- } while (pend_state & BIT_ULL(58)); +- ++ } while (pend_state & (BIT_ULL(58) | BIT_ULL(56))); + plt_write64(0, base + SSOW_LF_GWS_OP_GWC_INVAL); + } +- + if (dev->dual_ws) + dws->swtag_req = 0; + else +@@ -846,12 +846,25 @@ cn9k_sso_port_quiesce(struct rte_eventdev *event_dev, void *port, + base, &ev, dev->rx_offloads, + dev->dual_ws ? dws->lookup_mem : ws->lookup_mem, + dev->dual_ws ? dws->tstamp : ws->tstamp); +- if (is_pend && ev.u64) { ++ if (is_pend && ev.u64) + if (flush_cb) + flush_cb(event_dev->data->dev_id, ev, args); +- cnxk_sso_hws_swtag_flush(ws->base); +- } ++ ++ ptag = (plt_read64(base + SSOW_LF_GWS_TAG) >> 32) & SSO_TT_EMPTY; ++ if (ptag != SSO_TT_EMPTY) ++ cnxk_sso_hws_swtag_flush(base); ++ ++ do { ++ ptag = plt_read64(base + SSOW_LF_GWS_PENDSTATE); ++ } while (ptag & BIT_ULL(56)); ++ ++ plt_write64(0, base + SSOW_LF_GWS_OP_GWC_INVAL); + } ++ ++ if (dev->dual_ws) ++ dws->swtag_req = 0; ++ else ++ ws->swtag_req = 0; + } + + static int +@@ -1110,11 +1123,11 @@ cn9k_sso_tx_adapter_queue_del(uint8_t id, const struct rte_eventdev *event_dev, + } + + static int +-cn9k_crypto_adapter_caps_get(const struct rte_eventdev *event_dev, +- const struct rte_cryptodev *cdev, uint32_t *caps) ++cn9k_crypto_adapter_caps_get(const struct rte_eventdev *event_dev, const struct rte_cryptodev *cdev, ++ uint32_t *caps) + { +- CNXK_VALID_DEV_OR_ERR_RET(event_dev->dev, "event_cn9k"); +- CNXK_VALID_DEV_OR_ERR_RET(cdev->device, "crypto_cn9k"); ++ CNXK_VALID_DEV_OR_ERR_RET(event_dev->dev, "event_cn9k", ENOTSUP); ++ CNXK_VALID_DEV_OR_ERR_RET(cdev->device, "crypto_cn9k", ENOTSUP); + + *caps = RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_FWD | + RTE_EVENT_CRYPTO_ADAPTER_CAP_SESSION_PRIVATE_DATA; +@@ -1131,8 +1144,8 @@ cn9k_crypto_adapter_qp_add(const struct rte_eventdev *event_dev, + struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(event_dev); + int ret; + +- CNXK_VALID_DEV_OR_ERR_RET(event_dev->dev, "event_cn9k"); +- CNXK_VALID_DEV_OR_ERR_RET(cdev->device, "crypto_cn9k"); ++ CNXK_VALID_DEV_OR_ERR_RET(event_dev->dev, "event_cn9k", EINVAL); ++ CNXK_VALID_DEV_OR_ERR_RET(cdev->device, "crypto_cn9k", EINVAL); + + dev->is_ca_internal_port = 1; + cn9k_sso_fp_fns_set((struct rte_eventdev *)(uintptr_t)event_dev); +@@ -1147,8 +1160,8 @@ static int + cn9k_crypto_adapter_qp_del(const struct rte_eventdev *event_dev, const struct rte_cryptodev *cdev, + int32_t queue_pair_id) + { +- CNXK_VALID_DEV_OR_ERR_RET(event_dev->dev, "event_cn9k"); +- CNXK_VALID_DEV_OR_ERR_RET(cdev->device, "crypto_cn9k"); ++ CNXK_VALID_DEV_OR_ERR_RET(event_dev->dev, "event_cn9k", EINVAL); ++ CNXK_VALID_DEV_OR_ERR_RET(cdev->device, "crypto_cn9k", EINVAL); + + return cnxk_crypto_adapter_qp_del(cdev, queue_pair_id); + } +diff --git a/dpdk/drivers/event/cnxk/cn9k_worker.h b/dpdk/drivers/event/cnxk/cn9k_worker.h +index 4c3932da47..0ccdb7baf3 100644 +--- a/dpdk/drivers/event/cnxk/cn9k_worker.h ++++ b/dpdk/drivers/event/cnxk/cn9k_worker.h +@@ -353,8 +353,10 @@ cn9k_sso_hws_get_work_empty(uint64_t base, struct rte_event *ev, + #endif + + if (gw.u64[1]) +- cn9k_sso_hws_post_process(gw.u64, mbuf, flags, lookup_mem, +- tstamp); ++ cn9k_sso_hws_post_process(gw.u64, mbuf, flags, lookup_mem, tstamp); ++ else ++ gw.u64[0] = (gw.u64[0] & (0x3ull << 32)) << 6 | ++ (gw.u64[0] & (0x3FFull << 36)) << 4 | (gw.u64[0] & 0xffffffff); + + ev->event = gw.u64[0]; + ev->u64 = gw.u64[1]; +@@ -730,6 +732,7 @@ cn9k_sso_hws_xmit_sec_one(const struct cn9k_eth_txq *txq, uint64_t base, + + rte_io_wmb(); + cn9k_sso_txq_fc_wait(txq); ++ cn9k_nix_sec_fc_wait_one(txq); + + /* Write CPT instruction to lmt line */ + vst1q_u64(lmt_addr, cmd01); +diff --git a/dpdk/drivers/event/cnxk/cnxk_eventdev.c b/dpdk/drivers/event/cnxk/cnxk_eventdev.c +index db62d32a81..d2f1708297 100644 +--- a/dpdk/drivers/event/cnxk/cnxk_eventdev.c ++++ b/dpdk/drivers/event/cnxk/cnxk_eventdev.c +@@ -319,9 +319,9 @@ int + cnxk_sso_timeout_ticks(struct rte_eventdev *event_dev, uint64_t ns, + uint64_t *tmo_ticks) + { +- RTE_SET_USED(event_dev); +- *tmo_ticks = NSEC2TICK(ns, rte_get_timer_hz()); ++ struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(event_dev); + ++ *tmo_ticks = dev->deq_tmo_ns ? ns / dev->deq_tmo_ns : 0; + return 0; + } + +@@ -551,7 +551,7 @@ cnxk_sso_parse_devargs(struct cnxk_sso_evdev *dev, struct rte_devargs *devargs) + &dev->force_ena_bp); + rte_kvargs_process(kvlist, CN9K_SSO_SINGLE_WS, &parse_kvargs_flag, + &single_ws); +- rte_kvargs_process(kvlist, CN10K_SSO_GW_MODE, &parse_kvargs_flag, ++ rte_kvargs_process(kvlist, CN10K_SSO_GW_MODE, &parse_kvargs_value, + &dev->gw_mode); + dev->dual_ws = !single_ws; + rte_kvargs_free(kvlist); +@@ -613,9 +613,8 @@ cnxk_sso_fini(struct rte_eventdev *event_dev) + + cnxk_tim_fini(); + roc_sso_rsrc_fini(&dev->sso); +- roc_sso_dev_fini(&dev->sso); + +- return 0; ++ return roc_sso_dev_fini(&dev->sso); + } + + int +diff --git a/dpdk/drivers/event/cnxk/cnxk_eventdev.h b/dpdk/drivers/event/cnxk/cnxk_eventdev.h +index 738e335ea4..44a39648e3 100644 +--- a/dpdk/drivers/event/cnxk/cnxk_eventdev.h ++++ b/dpdk/drivers/event/cnxk/cnxk_eventdev.h +@@ -65,10 +65,10 @@ + (min + val / ((max + cnt - 1) / cnt)) + #define CNXK_SSO_FLUSH_RETRY_MAX 0xfff + +-#define CNXK_VALID_DEV_OR_ERR_RET(dev, drv_name) \ ++#define CNXK_VALID_DEV_OR_ERR_RET(dev, drv_name, err_val) \ + do { \ + if (strncmp(dev->driver->name, drv_name, strlen(drv_name))) \ +- return -EINVAL; \ ++ return -err_val; \ + } while (0) + + typedef void *(*cnxk_sso_init_hws_mem_t)(void *dev, uint8_t port_id); +@@ -128,7 +128,7 @@ struct cnxk_sso_evdev { + /* CN9K */ + uint8_t dual_ws; + /* CN10K */ +- uint8_t gw_mode; ++ uint32_t gw_mode; + /* Crypto adapter */ + uint8_t is_ca_internal_port; + } __rte_cache_aligned; +diff --git a/dpdk/drivers/event/cnxk/cnxk_eventdev_adptr.c b/dpdk/drivers/event/cnxk/cnxk_eventdev_adptr.c +index 5ec436382c..e78d215630 100644 +--- a/dpdk/drivers/event/cnxk/cnxk_eventdev_adptr.c ++++ b/dpdk/drivers/event/cnxk/cnxk_eventdev_adptr.c +@@ -635,6 +635,7 @@ cnxk_sso_tx_adapter_free(uint8_t id __rte_unused, + if (dev->tx_adptr_data_sz && dev->tx_adptr_active_mask == 0) { + dev->tx_adptr_data_sz = 0; + free(dev->tx_adptr_data); ++ dev->tx_adptr_data = NULL; + } + + return 0; +diff --git a/dpdk/drivers/event/cnxk/cnxk_tim_evdev.c b/dpdk/drivers/event/cnxk/cnxk_tim_evdev.c +index 5dd79cbd47..c155764b77 100644 +--- a/dpdk/drivers/event/cnxk/cnxk_tim_evdev.c ++++ b/dpdk/drivers/event/cnxk/cnxk_tim_evdev.c +@@ -381,6 +381,7 @@ cnxk_tim_caps_get(const struct rte_eventdev *evdev, uint64_t flags, + cnxk_sso_set_priv_mem_t priv_mem_fn) + { + struct cnxk_tim_evdev *dev = cnxk_tim_priv_get(); ++ struct cnxk_tim_ring *tim_ring; + + RTE_SET_USED(flags); + +@@ -403,6 +404,12 @@ cnxk_tim_caps_get(const struct rte_eventdev *evdev, uint64_t flags, + dev->event_dev = (struct rte_eventdev *)(uintptr_t)evdev; + *caps = RTE_EVENT_TIMER_ADAPTER_CAP_INTERNAL_PORT | + RTE_EVENT_TIMER_ADAPTER_CAP_PERIODIC; ++ ++ tim_ring = ((struct rte_event_timer_adapter_data ++ *)((char *)caps - offsetof(struct rte_event_timer_adapter_data, caps))) ++ ->adapter_priv; ++ if (tim_ring != NULL && rte_eal_process_type() == RTE_PROC_SECONDARY) ++ cnxk_tim_set_fp_ops(tim_ring); + *ops = &cnxk_tim_ops; + + return 0; +diff --git a/dpdk/drivers/event/cnxk/cnxk_tim_worker.h b/dpdk/drivers/event/cnxk/cnxk_tim_worker.h +index eda84c6f31..6be31f6f9d 100644 +--- a/dpdk/drivers/event/cnxk/cnxk_tim_worker.h ++++ b/dpdk/drivers/event/cnxk/cnxk_tim_worker.h +@@ -270,7 +270,8 @@ __retry: + } while (hbt_state & BIT_ULL(33)); + #endif + +- if (!(hbt_state & BIT_ULL(34))) { ++ if (!(hbt_state & BIT_ULL(34)) || ++ !(hbt_state & GENMASK(31, 0))) { + cnxk_tim_bkt_dec_lock(bkt); + goto __retry; + } +@@ -352,7 +353,8 @@ __retry: + } while (hbt_state & BIT_ULL(33)); + #endif + +- if (!(hbt_state & BIT_ULL(34))) { ++ if (!(hbt_state & BIT_ULL(34)) || ++ !(hbt_state & GENMASK(31, 0))) { + cnxk_tim_bkt_dec_lock(bkt); + goto __retry; + } +@@ -449,10 +451,10 @@ cnxk_tim_add_entry_brst(struct cnxk_tim_ring *const tim_ring, + struct cnxk_tim_ent *chunk = NULL; + struct cnxk_tim_bkt *mirr_bkt; + struct cnxk_tim_bkt *bkt; +- uint16_t chunk_remainder; ++ int16_t chunk_remainder; + uint16_t index = 0; + uint64_t lock_sema; +- int16_t rem, crem; ++ int16_t rem; + uint8_t lock_cnt; + + __retry: +@@ -460,31 +462,6 @@ __retry: + + /* Only one thread beyond this. */ + lock_sema = cnxk_tim_bkt_inc_lock(bkt); +- lock_cnt = (uint8_t)((lock_sema >> TIM_BUCKET_W1_S_LOCK) & +- TIM_BUCKET_W1_M_LOCK); +- +- if (lock_cnt) { +- cnxk_tim_bkt_dec_lock(bkt); +-#ifdef RTE_ARCH_ARM64 +- asm volatile(PLT_CPU_FEATURE_PREAMBLE +- " ldxrb %w[lock_cnt], [%[lock]] \n" +- " tst %w[lock_cnt], 255 \n" +- " beq dne%= \n" +- " sevl \n" +- "rty%=: wfe \n" +- " ldxrb %w[lock_cnt], [%[lock]] \n" +- " tst %w[lock_cnt], 255 \n" +- " bne rty%= \n" +- "dne%=: \n" +- : [lock_cnt] "=&r"(lock_cnt) +- : [lock] "r"(&bkt->lock) +- : "memory"); +-#else +- while (__atomic_load_n(&bkt->lock, __ATOMIC_RELAXED)) +- ; +-#endif +- goto __retry; +- } + + /* Bucket related checks. */ + if (unlikely(cnxk_tim_bkt_get_hbt(lock_sema))) { +@@ -509,21 +486,46 @@ __retry: + } while (hbt_state & BIT_ULL(33)); + #endif + +- if (!(hbt_state & BIT_ULL(34))) { ++ if (!(hbt_state & BIT_ULL(34)) || ++ !(hbt_state & GENMASK(31, 0))) { + cnxk_tim_bkt_dec_lock(bkt); + goto __retry; + } + } + } + ++ lock_cnt = (uint8_t)((lock_sema >> TIM_BUCKET_W1_S_LOCK) & ++ TIM_BUCKET_W1_M_LOCK); ++ if (lock_cnt) { ++ cnxk_tim_bkt_dec_lock(bkt); ++#ifdef RTE_ARCH_ARM64 ++ asm volatile(PLT_CPU_FEATURE_PREAMBLE ++ " ldxrb %w[lock_cnt], [%[lock]] \n" ++ " tst %w[lock_cnt], 255 \n" ++ " beq dne%= \n" ++ " sevl \n" ++ "rty%=: wfe \n" ++ " ldxrb %w[lock_cnt], [%[lock]] \n" ++ " tst %w[lock_cnt], 255 \n" ++ " bne rty%= \n" ++ "dne%=: \n" ++ : [lock_cnt] "=&r"(lock_cnt) ++ : [lock] "r"(&bkt->lock) ++ : "memory"); ++#else ++ while (__atomic_load_n(&bkt->lock, __ATOMIC_RELAXED)) ++ ; ++#endif ++ goto __retry; ++ } ++ + chunk_remainder = cnxk_tim_bkt_fetch_rem(lock_sema); + rem = chunk_remainder - nb_timers; + if (rem < 0) { +- crem = tim_ring->nb_chunk_slots - chunk_remainder; +- if (chunk_remainder && crem) { ++ if (chunk_remainder > 0) { + chunk = ((struct cnxk_tim_ent *) + mirr_bkt->current_chunk) + +- crem; ++ tim_ring->nb_chunk_slots - chunk_remainder; + + index = cnxk_tim_cpy_wrk(index, chunk_remainder, chunk, + tim, ents, bkt); +@@ -537,18 +539,19 @@ __retry: + chunk = cnxk_tim_insert_chunk(bkt, mirr_bkt, tim_ring); + + if (unlikely(chunk == NULL)) { +- cnxk_tim_bkt_dec_lock(bkt); ++ cnxk_tim_bkt_dec_lock_relaxed(bkt); + rte_errno = ENOMEM; + tim[index]->state = RTE_EVENT_TIMER_ERROR; +- return crem; ++ return index; + } + *(uint64_t *)(chunk + tim_ring->nb_chunk_slots) = 0; + mirr_bkt->current_chunk = (uintptr_t)chunk; +- cnxk_tim_cpy_wrk(index, nb_timers, chunk, tim, ents, bkt); ++ index = cnxk_tim_cpy_wrk(index, nb_timers, chunk, tim, ents, ++ bkt) - ++ index; + +- rem = nb_timers - chunk_remainder; +- cnxk_tim_bkt_set_rem(bkt, tim_ring->nb_chunk_slots - rem); +- cnxk_tim_bkt_add_nent(bkt, rem); ++ cnxk_tim_bkt_set_rem(bkt, tim_ring->nb_chunk_slots - index); ++ cnxk_tim_bkt_add_nent(bkt, index); + } else { + chunk = (struct cnxk_tim_ent *)mirr_bkt->current_chunk; + chunk += (tim_ring->nb_chunk_slots - chunk_remainder); +diff --git a/dpdk/drivers/event/dlb2/dlb2.c b/dpdk/drivers/event/dlb2/dlb2.c +index 60c5cd4804..fa1ccb25ba 100644 +--- a/dpdk/drivers/event/dlb2/dlb2.c ++++ b/dpdk/drivers/event/dlb2/dlb2.c +@@ -72,6 +72,7 @@ static struct rte_event_dev_info evdev_dlb2_default_info = { + .max_single_link_event_port_queue_pairs = + DLB2_MAX_NUM_DIR_PORTS(DLB2_HW_V2), + .event_dev_cap = (RTE_EVENT_DEV_CAP_EVENT_QOS | ++ RTE_EVENT_DEV_CAP_NONSEQ_MODE | + RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED | + RTE_EVENT_DEV_CAP_QUEUE_ALL_TYPES | + RTE_EVENT_DEV_CAP_BURST_MODE | +diff --git a/dpdk/drivers/event/dlb2/dlb2_selftest.c b/dpdk/drivers/event/dlb2/dlb2_selftest.c +index 1863ffe049..62aa11d981 100644 +--- a/dpdk/drivers/event/dlb2/dlb2_selftest.c ++++ b/dpdk/drivers/event/dlb2/dlb2_selftest.c +@@ -1475,7 +1475,7 @@ test_fail: + int + test_dlb2_eventdev(void) + { +- const char *dlb2_eventdev_name = "dlb2_event"; ++ const char *dlb2_eventdev_name = "event_dlb2"; + uint8_t num_evdevs = rte_event_dev_count(); + int i, ret = 0; + int found = 0, skipped = 0, passed = 0, failed = 0; +@@ -1489,7 +1489,7 @@ test_dlb2_eventdev(void) + + /* skip non-dlb2 event devices */ + if (strncmp(info.driver_name, dlb2_eventdev_name, +- sizeof(*info.driver_name)) != 0) { ++ strlen(dlb2_eventdev_name)) != 0) { + skipped++; + continue; + } +diff --git a/dpdk/drivers/event/dlb2/pf/dlb2_main.c b/dpdk/drivers/event/dlb2/pf/dlb2_main.c +index 717aa4fc08..12b65c3b98 100644 +--- a/dpdk/drivers/event/dlb2/pf/dlb2_main.c ++++ b/dpdk/drivers/event/dlb2/pf/dlb2_main.c +@@ -26,6 +26,7 @@ + #define PF_ID_ZERO 0 /* PF ONLY! */ + #define NO_OWNER_VF 0 /* PF ONLY! */ + #define NOT_VF_REQ false /* PF ONLY! */ ++#define DLB2_PCI_PASID_CAP_OFFSET 0x148 /* PASID capability offset */ + + #define DLB2_PCI_CAP_POINTER 0x34 + #define DLB2_PCI_CAP_NEXT(hdr) (((hdr) >> 8) & 0xFC) +@@ -46,6 +47,7 @@ + #define DLB2_PCI_CAP_ID_MSIX 0x11 + #define DLB2_PCI_EXT_CAP_ID_PRI 0x13 + #define DLB2_PCI_EXT_CAP_ID_ACS 0xD ++#define DLB2_PCI_EXT_CAP_ID_PASID 0x1B /* Process Address Space ID */ + + #define DLB2_PCI_PRI_CTRL_ENABLE 0x1 + #define DLB2_PCI_PRI_ALLOC_REQ 0xC +@@ -64,6 +66,8 @@ + #define DLB2_PCI_ACS_CR 0x8 + #define DLB2_PCI_ACS_UF 0x10 + #define DLB2_PCI_ACS_EC 0x20 ++#define DLB2_PCI_PASID_CTRL 0x06 /* PASID control register */ ++#define DLB2_PCI_PASID_CAP_OFFSET 0x148 /* PASID capability offset */ + + static int dlb2_pci_find_capability(struct rte_pci_device *pdev, uint32_t id) + { +@@ -257,12 +261,14 @@ dlb2_pf_reset(struct dlb2_dev *dlb2_dev) + uint16_t rt_ctl_word; + uint32_t pri_reqs_dword; + uint16_t pri_ctrl_word; ++ uint16_t pasid_ctrl; + + int pcie_cap_offset; + int pri_cap_offset; + int msix_cap_offset; + int err_cap_offset; + int acs_cap_offset; ++ int pasid_cap_offset; + int wait_count; + + uint16_t devsta_busy_word; +@@ -582,6 +588,38 @@ dlb2_pf_reset(struct dlb2_dev *dlb2_dev) + } + } + ++ /* The current Linux kernel vfio driver does not expose PASID capability to ++ * users. It also enables PASID by default, which breaks DLB PF PMD. We have ++ * to use the hardcoded offset for now to disable PASID. ++ */ ++ pasid_cap_offset = DLB2_PCI_PASID_CAP_OFFSET; ++ ++ off = pasid_cap_offset + DLB2_PCI_PASID_CTRL; ++ if (rte_pci_read_config(pdev, &pasid_ctrl, 2, off) != 2) ++ pasid_ctrl = 0; ++ ++ if (pasid_ctrl) { ++ DLB2_INFO(dlb2_dev, "DLB2 disabling pasid...\n"); ++ ++ pasid_ctrl = 0; ++ ret = rte_pci_write_config(pdev, &pasid_ctrl, 2, off); ++ if (ret != 2) { ++ DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d\n", ++ __func__, (int)off); ++ return ret; ++ } ++ } ++ ++ /* Disable PASID if it is enabled by default, which ++ * breaks the DLB if enabled. ++ */ ++ off = DLB2_PCI_PASID_CAP_OFFSET + RTE_PCI_PASID_CTRL; ++ if (rte_pci_pasid_set_state(pdev, off, false)) { ++ DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d\n", ++ __func__, (int)off); ++ return -1; ++ } ++ + return 0; + } + +diff --git a/dpdk/drivers/event/dpaa/dpaa_eventdev.c b/dpdk/drivers/event/dpaa/dpaa_eventdev.c +index 4b3d16735b..2532abbe78 100644 +--- a/dpdk/drivers/event/dpaa/dpaa_eventdev.c ++++ b/dpdk/drivers/event/dpaa/dpaa_eventdev.c +@@ -993,14 +993,14 @@ dpaa_event_check_flags(const char *params) + } + + static int +-dpaa_event_dev_create(const char *name, const char *params) ++dpaa_event_dev_create(const char *name, const char *params, struct rte_vdev_device *vdev) + { + struct rte_eventdev *eventdev; + struct dpaa_eventdev *priv; + + eventdev = rte_event_pmd_vdev_init(name, + sizeof(struct dpaa_eventdev), +- rte_socket_id()); ++ rte_socket_id(), vdev); + if (eventdev == NULL) { + DPAA_EVENTDEV_ERR("Failed to create eventdev vdev %s", name); + goto fail; +@@ -1050,7 +1050,7 @@ dpaa_event_dev_probe(struct rte_vdev_device *vdev) + + params = rte_vdev_device_args(vdev); + +- return dpaa_event_dev_create(name, params); ++ return dpaa_event_dev_create(name, params, vdev); + } + + static int +diff --git a/dpdk/drivers/event/dpaa2/dpaa2_eventdev.c b/dpdk/drivers/event/dpaa2/dpaa2_eventdev.c +index fa1a1ade80..1e64806849 100644 +--- a/dpdk/drivers/event/dpaa2/dpaa2_eventdev.c ++++ b/dpdk/drivers/event/dpaa2/dpaa2_eventdev.c +@@ -1086,7 +1086,7 @@ dpaa2_eventdev_setup_dpci(struct dpaa2_dpci_dev *dpci_dev, + } + + static int +-dpaa2_eventdev_create(const char *name) ++dpaa2_eventdev_create(const char *name, struct rte_vdev_device *vdev) + { + struct rte_eventdev *eventdev; + struct dpaa2_eventdev *priv; +@@ -1096,7 +1096,7 @@ dpaa2_eventdev_create(const char *name) + + eventdev = rte_event_pmd_vdev_init(name, + sizeof(struct dpaa2_eventdev), +- rte_socket_id()); ++ rte_socket_id(), vdev); + if (eventdev == NULL) { + DPAA2_EVENTDEV_ERR("Failed to create Event device %s", name); + goto fail; +@@ -1190,7 +1190,7 @@ dpaa2_eventdev_probe(struct rte_vdev_device *vdev) + + name = rte_vdev_device_name(vdev); + DPAA2_EVENTDEV_INFO("Initializing %s", name); +- return dpaa2_eventdev_create(name); ++ return dpaa2_eventdev_create(name, vdev); + } + + static int +diff --git a/dpdk/drivers/event/dsw/dsw_evdev.c b/dpdk/drivers/event/dsw/dsw_evdev.c +index ffabf0d23d..abe8e68525 100644 +--- a/dpdk/drivers/event/dsw/dsw_evdev.c ++++ b/dpdk/drivers/event/dsw/dsw_evdev.c +@@ -363,6 +363,10 @@ static int + dsw_close(struct rte_eventdev *dev) + { + struct dsw_evdev *dsw = dsw_pmd_priv(dev); ++ uint16_t port_id; ++ ++ for (port_id = 0; port_id < dsw->num_ports; port_id++) ++ dsw_port_release(&dsw->ports[port_id]); + + dsw->num_ports = 0; + dsw->num_queues = 0; +@@ -430,7 +434,7 @@ dsw_probe(struct rte_vdev_device *vdev) + name = rte_vdev_device_name(vdev); + + dev = rte_event_pmd_vdev_init(name, sizeof(struct dsw_evdev), +- rte_socket_id()); ++ rte_socket_id(), vdev); + if (dev == NULL) + return -EFAULT; + +diff --git a/dpdk/drivers/event/octeontx/ssovf_evdev.c b/dpdk/drivers/event/octeontx/ssovf_evdev.c +index 650266b996..d5e223077d 100644 +--- a/dpdk/drivers/event/octeontx/ssovf_evdev.c ++++ b/dpdk/drivers/event/octeontx/ssovf_evdev.c +@@ -880,7 +880,7 @@ ssovf_vdev_probe(struct rte_vdev_device *vdev) + } + + eventdev = rte_event_pmd_vdev_init(name, sizeof(struct ssovf_evdev), +- rte_socket_id()); ++ rte_socket_id(), vdev); + if (eventdev == NULL) { + ssovf_log_err("Failed to create eventdev vdev %s", name); + return -ENOMEM; +diff --git a/dpdk/drivers/event/opdl/opdl_evdev.c b/dpdk/drivers/event/opdl/opdl_evdev.c +index 9ce8b39b60..08ace84bbe 100644 +--- a/dpdk/drivers/event/opdl/opdl_evdev.c ++++ b/dpdk/drivers/event/opdl/opdl_evdev.c +@@ -696,7 +696,7 @@ opdl_probe(struct rte_vdev_device *vdev) + } + } + dev = rte_event_pmd_vdev_init(name, +- sizeof(struct opdl_evdev), socket_id); ++ sizeof(struct opdl_evdev), socket_id, vdev); + + if (dev == NULL) { + PMD_DRV_LOG(ERR, "eventdev vdev init() failed"); +diff --git a/dpdk/drivers/event/skeleton/skeleton_eventdev.c b/dpdk/drivers/event/skeleton/skeleton_eventdev.c +index 8513b9a013..45c13c62c7 100644 +--- a/dpdk/drivers/event/skeleton/skeleton_eventdev.c ++++ b/dpdk/drivers/event/skeleton/skeleton_eventdev.c +@@ -427,12 +427,12 @@ RTE_PMD_REGISTER_PCI_TABLE(event_skeleton_pci, pci_id_skeleton_map); + /* VDEV based event device */ + + static int +-skeleton_eventdev_create(const char *name, int socket_id) ++skeleton_eventdev_create(const char *name, int socket_id, struct rte_vdev_device *vdev) + { + struct rte_eventdev *eventdev; + + eventdev = rte_event_pmd_vdev_init(name, +- sizeof(struct skeleton_eventdev), socket_id); ++ sizeof(struct skeleton_eventdev), socket_id, vdev); + if (eventdev == NULL) { + PMD_DRV_ERR("Failed to create eventdev vdev %s", name); + goto fail; +@@ -458,7 +458,7 @@ skeleton_eventdev_probe(struct rte_vdev_device *vdev) + name = rte_vdev_device_name(vdev); + RTE_LOG(INFO, PMD, "Initializing %s on NUMA node %d\n", name, + rte_socket_id()); +- return skeleton_eventdev_create(name, rte_socket_id()); ++ return skeleton_eventdev_create(name, rte_socket_id(), vdev); + } + + static int +diff --git a/dpdk/drivers/event/sw/sw_evdev.c b/dpdk/drivers/event/sw/sw_evdev.c +index 3531821dd4..2a2763ee9b 100644 +--- a/dpdk/drivers/event/sw/sw_evdev.c ++++ b/dpdk/drivers/event/sw/sw_evdev.c +@@ -1074,7 +1074,7 @@ sw_probe(struct rte_vdev_device *vdev) + min_burst_size, deq_burst_size, refill_once); + + dev = rte_event_pmd_vdev_init(name, +- sizeof(struct sw_evdev), socket_id); ++ sizeof(struct sw_evdev), socket_id, vdev); + if (dev == NULL) { + SW_LOG_ERR("eventdev vdev init() failed"); + return -EFAULT; +diff --git a/dpdk/drivers/event/sw/sw_evdev_scheduler.c b/dpdk/drivers/event/sw/sw_evdev_scheduler.c +index 8bc21944f5..25f97c4ffb 100644 +--- a/dpdk/drivers/event/sw/sw_evdev_scheduler.c ++++ b/dpdk/drivers/event/sw/sw_evdev_scheduler.c +@@ -90,8 +90,10 @@ sw_schedule_atomic_to_cq(struct sw_evdev *sw, struct sw_qid * const qid, + sw->cq_ring_space[cq]--; + + int head = (p->hist_head++ & (SW_PORT_HIST_LIST-1)); +- p->hist_list[head].fid = flow_id; +- p->hist_list[head].qid = qid_id; ++ p->hist_list[head] = (struct sw_hist_list_entry) { ++ .qid = qid_id, ++ .fid = flow_id, ++ }; + + p->stats.tx_pkts++; + qid->stats.tx_pkts++; +@@ -162,8 +164,10 @@ sw_schedule_parallel_to_cq(struct sw_evdev *sw, struct sw_qid * const qid, + qid->stats.tx_pkts++; + + const int head = (p->hist_head & (SW_PORT_HIST_LIST-1)); +- p->hist_list[head].fid = SW_HASH_FLOWID(qe->flow_id); +- p->hist_list[head].qid = qid_id; ++ p->hist_list[head] = (struct sw_hist_list_entry) { ++ .qid = qid_id, ++ .fid = SW_HASH_FLOWID(qe->flow_id), ++ }; + + if (keep_order) + rob_ring_dequeue(qid->reorder_buffer_freelist, +@@ -368,12 +372,6 @@ __pull_port_lb(struct sw_evdev *sw, uint32_t port_id, int allow_reorder) + if (!allow_reorder && !eop) + flags = QE_FLAG_VALID; + +- /* +- * if we don't have space for this packet in an IQ, +- * then move on to next queue. Technically, for a +- * packet that needs reordering, we don't need to check +- * here, but it simplifies things not to special-case +- */ + uint32_t iq_num = PRIO_TO_IQ(qe->priority); + struct sw_qid *qid = &sw->qids[qe->queue_id]; + +@@ -419,7 +417,6 @@ __pull_port_lb(struct sw_evdev *sw, uint32_t port_id, int allow_reorder) + struct reorder_buffer_entry *rob_entry = + hist_entry->rob_entry; + +- hist_entry->rob_entry = NULL; + /* Although fragmentation not currently + * supported by eventdev API, we support it + * here. Open: How do we alert the user that +diff --git a/dpdk/drivers/gpu/cuda/gdrcopy.c b/dpdk/drivers/gpu/cuda/gdrcopy.c +index 322a5dbeb2..bd56b73ce4 100644 +--- a/dpdk/drivers/gpu/cuda/gdrcopy.c ++++ b/dpdk/drivers/gpu/cuda/gdrcopy.c +@@ -6,6 +6,8 @@ + + #ifdef DRIVERS_GPU_CUDA_GDRCOPY_H + ++#include ++ + static void *gdrclib; + static gdr_t (*sym_gdr_open)(void); + static int (*sym_gdr_pin_buffer)(gdr_t g, unsigned long addr, size_t size, +diff --git a/dpdk/drivers/mempool/cnxk/cn10k_mempool_ops.c b/dpdk/drivers/mempool/cnxk/cn10k_mempool_ops.c +index ba826f0f01..9d6982fdab 100644 +--- a/dpdk/drivers/mempool/cnxk/cn10k_mempool_ops.c ++++ b/dpdk/drivers/mempool/cnxk/cn10k_mempool_ops.c +@@ -9,6 +9,7 @@ + + #define BATCH_ALLOC_SZ ROC_CN10K_NPA_BATCH_ALLOC_MAX_PTRS + #define BATCH_OP_DATA_TABLE_MZ_NAME "batch_op_data_table_mz" ++#define BATCH_ALLOC_WAIT_US 5 + + enum batch_op_status { + BATCH_ALLOC_OP_NOT_ISSUED = 0, +@@ -150,6 +151,12 @@ cn10k_mempool_enq(struct rte_mempool *mp, void *const *obj_table, + */ + rte_io_wmb(); + ++ /* For non-EAL threads, rte_lcore_id() will not be valid. Hence ++ * fallback to bulk alloc ++ */ ++ if (unlikely(rte_lcore_id() == LCORE_ID_ANY)) ++ return cnxk_mempool_enq(mp, obj_table, n); ++ + if (n == 1) { + roc_npa_aura_op_free(mp->pool_id, 1, ptr[0]); + return 0; +@@ -178,7 +185,7 @@ cn10k_mempool_get_count(const struct rte_mempool *mp) + + if (mem->status == BATCH_ALLOC_OP_ISSUED) + count += roc_npa_aura_batch_alloc_count( +- mem->objs, BATCH_ALLOC_SZ, 1); ++ mem->objs, BATCH_ALLOC_SZ, BATCH_ALLOC_WAIT_US); + + if (mem->status == BATCH_ALLOC_OP_DONE) + count += mem->sz; +@@ -250,6 +257,12 @@ cn10k_mempool_deq(struct rte_mempool *mp, void **obj_table, unsigned int n) + } + } + ++ /* For non-EAL threads, rte_lcore_id() will not be valid. Hence ++ * fallback to bulk alloc ++ */ ++ if (unlikely(rte_lcore_id() == LCORE_ID_ANY)) ++ return cnxk_mempool_deq(mp, obj_table, n); ++ + if (unlikely(count != n)) { + /* No partial alloc allowed. Free up allocated pointers */ + cn10k_mempool_enq(mp, obj_table, count); +diff --git a/dpdk/drivers/net/af_packet/rte_eth_af_packet.c b/dpdk/drivers/net/af_packet/rte_eth_af_packet.c +index c13a0942aa..397a32db58 100644 +--- a/dpdk/drivers/net/af_packet/rte_eth_af_packet.c ++++ b/dpdk/drivers/net/af_packet/rte_eth_af_packet.c +@@ -313,7 +313,14 @@ eth_af_packet_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) + static int + eth_dev_start(struct rte_eth_dev *dev) + { ++ struct pmd_internals *internals = dev->data->dev_private; ++ uint16_t i; ++ + dev->data->dev_link.link_status = RTE_ETH_LINK_UP; ++ for (i = 0; i < internals->nb_queues; i++) { ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ } + return 0; + } + +@@ -341,6 +348,8 @@ eth_dev_stop(struct rte_eth_dev *dev) + + internals->rx_queue[i].sockfd = -1; + internals->tx_queue[i].sockfd = -1; ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; + } + + dev->data->dev_link.link_status = RTE_ETH_LINK_DOWN; +diff --git a/dpdk/drivers/net/af_xdp/rte_eth_af_xdp.c b/dpdk/drivers/net/af_xdp/rte_eth_af_xdp.c +index b6ec9bf490..738f4158e0 100644 +--- a/dpdk/drivers/net/af_xdp/rte_eth_af_xdp.c ++++ b/dpdk/drivers/net/af_xdp/rte_eth_af_xdp.c +@@ -672,7 +672,13 @@ eth_af_xdp_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) + static int + eth_dev_start(struct rte_eth_dev *dev) + { ++ uint16_t i; ++ + dev->data->dev_link.link_status = RTE_ETH_LINK_UP; ++ for (i = 0; i < dev->data->nb_rx_queues; i++) { ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ } + + return 0; + } +@@ -681,7 +687,14 @@ eth_dev_start(struct rte_eth_dev *dev) + static int + eth_dev_stop(struct rte_eth_dev *dev) + { ++ uint16_t i; ++ + dev->data->dev_link.link_status = RTE_ETH_LINK_DOWN; ++ for (i = 0; i < dev->data->nb_rx_queues; i++) { ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ } ++ + return 0; + } + +diff --git a/dpdk/drivers/net/ark/ark_ethdev.c b/dpdk/drivers/net/ark/ark_ethdev.c +index c654a229f7..c1681e8ecd 100644 +--- a/dpdk/drivers/net/ark/ark_ethdev.c ++++ b/dpdk/drivers/net/ark/ark_ethdev.c +@@ -299,6 +299,7 @@ eth_ark_dev_init(struct rte_eth_dev *dev) + int ret; + int port_count = 1; + int p; ++ uint16_t num_queues; + bool rqpacing = false; + + ark->eth_dev = dev; +@@ -426,6 +427,7 @@ eth_ark_dev_init(struct rte_eth_dev *dev) + ark->user_ext.dev_get_port_count(dev, + ark->user_data[dev->data->port_id]); + ark->num_ports = port_count; ++ num_queues = ark_api_num_queues_per_port(ark->mpurx.v, port_count); + + for (p = 0; p < port_count; p++) { + struct rte_eth_dev *eth_dev; +@@ -451,7 +453,18 @@ eth_ark_dev_init(struct rte_eth_dev *dev) + } + + eth_dev->device = &pci_dev->device; +- eth_dev->data->dev_private = ark; ++ /* Device requires new dev_private data */ ++ eth_dev->data->dev_private = ++ rte_zmalloc_socket(name, ++ sizeof(struct ark_adapter), ++ RTE_CACHE_LINE_SIZE, ++ rte_socket_id()); ++ ++ memcpy(eth_dev->data->dev_private, ark, ++ sizeof(struct ark_adapter)); ++ ark = eth_dev->data->dev_private; ++ ark->qbase = p * num_queues; ++ + eth_dev->dev_ops = ark->eth_dev->dev_ops; + eth_dev->tx_pkt_burst = ark->eth_dev->tx_pkt_burst; + eth_dev->rx_pkt_burst = ark->eth_dev->rx_pkt_burst; +diff --git a/dpdk/drivers/net/ark/ark_ethdev_rx.c b/dpdk/drivers/net/ark/ark_ethdev_rx.c +index cbc0416bc2..38bc69dff4 100644 +--- a/dpdk/drivers/net/ark/ark_ethdev_rx.c ++++ b/dpdk/drivers/net/ark/ark_ethdev_rx.c +@@ -68,7 +68,7 @@ struct ark_rx_queue { + static int + eth_ark_rx_hw_setup(struct rte_eth_dev *dev, + struct ark_rx_queue *queue, +- uint16_t rx_queue_id __rte_unused, uint16_t rx_queue_idx) ++ uint16_t rx_queue_idx) + { + rte_iova_t queue_base; + rte_iova_t phys_addr_q_base; +@@ -124,7 +124,7 @@ eth_ark_dev_rx_queue_setup(struct rte_eth_dev *dev, + uint32_t i; + int status; + +- int qidx = queue_idx; ++ int qidx = ark->qbase + queue_idx; + + /* We may already be setup, free memory prior to re-allocation */ + if (dev->data->rx_queues[queue_idx] != NULL) { +@@ -215,7 +215,7 @@ eth_ark_dev_rx_queue_setup(struct rte_eth_dev *dev, + } + /* MPU Setup */ + if (status == 0) +- status = eth_ark_rx_hw_setup(dev, queue, qidx, queue_idx); ++ status = eth_ark_rx_hw_setup(dev, queue, queue_idx); + + if (unlikely(status != 0)) { + struct rte_mbuf **mbuf; +diff --git a/dpdk/drivers/net/ark/ark_ethdev_tx.c b/dpdk/drivers/net/ark/ark_ethdev_tx.c +index 5940a592a2..4792754f19 100644 +--- a/dpdk/drivers/net/ark/ark_ethdev_tx.c ++++ b/dpdk/drivers/net/ark/ark_ethdev_tx.c +@@ -229,7 +229,7 @@ eth_ark_tx_queue_setup(struct rte_eth_dev *dev, + struct ark_tx_queue *queue; + int status; + +- int qidx = queue_idx; ++ int qidx = ark->qbase + queue_idx; + + if (!rte_is_power_of_2(nb_desc)) { + ARK_PMD_LOG(ERR, +diff --git a/dpdk/drivers/net/ark/ark_global.h b/dpdk/drivers/net/ark/ark_global.h +index 71d0b53e03..2f198edfe4 100644 +--- a/dpdk/drivers/net/ark/ark_global.h ++++ b/dpdk/drivers/net/ark/ark_global.h +@@ -112,7 +112,10 @@ struct ark_adapter { + ark_pkt_chkr_t pc; + ark_pkt_dir_t pd; + ++ /* For single function, multiple ports */ + int num_ports; ++ uint16_t qbase; ++ + bool isvf; + + /* Packet generator/checker args */ +diff --git a/dpdk/drivers/net/avp/avp_ethdev.c b/dpdk/drivers/net/avp/avp_ethdev.c +index b2a08f5635..53d9e38c93 100644 +--- a/dpdk/drivers/net/avp/avp_ethdev.c ++++ b/dpdk/drivers/net/avp/avp_ethdev.c +@@ -2036,6 +2036,7 @@ static int + avp_dev_start(struct rte_eth_dev *eth_dev) + { + struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); ++ uint16_t i; + int ret; + + rte_spinlock_lock(&avp->lock); +@@ -2056,6 +2057,11 @@ avp_dev_start(struct rte_eth_dev *eth_dev) + /* remember current link state */ + avp->flags |= AVP_F_LINKUP; + ++ for (i = 0; i < avp->num_rx_queues; i++) ++ eth_dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ for (i = 0; i < avp->num_tx_queues; i++) ++ eth_dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ + ret = 0; + + unlock: +@@ -2067,6 +2073,7 @@ static int + avp_dev_stop(struct rte_eth_dev *eth_dev) + { + struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); ++ uint16_t i; + int ret; + + rte_spinlock_lock(&avp->lock); +@@ -2086,6 +2093,11 @@ avp_dev_stop(struct rte_eth_dev *eth_dev) + ret); + } + ++ for (i = 0; i < avp->num_rx_queues; i++) ++ eth_dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ for (i = 0; i < avp->num_tx_queues; i++) ++ eth_dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ + unlock: + rte_spinlock_unlock(&avp->lock); + return ret; +diff --git a/dpdk/drivers/net/axgbe/axgbe_ethdev.c b/dpdk/drivers/net/axgbe/axgbe_ethdev.c +index b071e4e460..da48a3ac42 100644 +--- a/dpdk/drivers/net/axgbe/axgbe_ethdev.c ++++ b/dpdk/drivers/net/axgbe/axgbe_ethdev.c +@@ -12,6 +12,12 @@ + + #include "eal_filesystem.h" + ++#ifdef RTE_ARCH_X86 ++#include ++#else ++#define __cpuid(n, a, b, c, d) ++#endif ++ + static int eth_axgbe_dev_init(struct rte_eth_dev *eth_dev); + static int axgbe_dev_configure(struct rte_eth_dev *dev); + static int axgbe_dev_start(struct rte_eth_dev *dev); +@@ -172,9 +178,14 @@ static const struct axgbe_xstats axgbe_xstats_strings[] = { + + /* The set of PCI devices this driver supports */ + #define AMD_PCI_VENDOR_ID 0x1022 +-#define AMD_PCI_RV_ROOT_COMPLEX_ID 0x15d0 +-#define AMD_PCI_YC_ROOT_COMPLEX_ID 0x14b5 +-#define AMD_PCI_SNOWY_ROOT_COMPLEX_ID 0x1450 ++ ++#define Fam17h 0x17 ++#define Fam19h 0x19 ++ ++#define CPUID_VENDOR_AuthenticAMD_ebx 0x68747541 ++#define CPUID_VENDOR_AuthenticAMD_ecx 0x444d4163 ++#define CPUID_VENDOR_AuthenticAMD_edx 0x69746e65 ++ + #define AMD_PCI_AXGBE_DEVICE_V2A 0x1458 + #define AMD_PCI_AXGBE_DEVICE_V2B 0x1459 + +@@ -2122,29 +2133,6 @@ static void axgbe_default_config(struct axgbe_port *pdata) + pdata->power_down = 0; + } + +-/* +- * Return PCI root complex device id on success else 0 +- */ +-static uint16_t +-get_pci_rc_devid(void) +-{ +- char pci_sysfs[PATH_MAX]; +- const struct rte_pci_addr pci_rc_addr = {0, 0, 0, 0}; +- unsigned long device_id; +- +- snprintf(pci_sysfs, sizeof(pci_sysfs), "%s/" PCI_PRI_FMT "/device", +- rte_pci_get_sysfs_path(), pci_rc_addr.domain, +- pci_rc_addr.bus, pci_rc_addr.devid, pci_rc_addr.function); +- +- /* get device id */ +- if (eal_parse_sysfs_value(pci_sysfs, &device_id) < 0) { +- PMD_INIT_LOG(ERR, "Error in reading PCI sysfs\n"); +- return 0; +- } +- +- return (uint16_t)device_id; +-} +- + /* + * It returns 0 on success. + */ +@@ -2158,6 +2146,9 @@ eth_axgbe_dev_init(struct rte_eth_dev *eth_dev) + uint32_t len; + int ret; + ++ unsigned int eax = 0, ebx = 0, ecx = 0, edx = 0; ++ unsigned char cpu_family = 0, cpu_model = 0; ++ + eth_dev->dev_ops = &axgbe_eth_dev_ops; + + eth_dev->rx_descriptor_status = axgbe_dev_rx_descriptor_status; +@@ -2196,26 +2187,55 @@ eth_axgbe_dev_init(struct rte_eth_dev *eth_dev) + pdata->vdata = &axgbe_v2b; + + /* +- * Use PCI root complex device ID to identify the CPU ++ * Use CPUID to get Family and model ID to identify the CPU + */ +- switch (get_pci_rc_devid()) { +- case AMD_PCI_RV_ROOT_COMPLEX_ID: +- pdata->xpcs_window_def_reg = PCS_V2_RV_WINDOW_DEF; +- pdata->xpcs_window_sel_reg = PCS_V2_RV_WINDOW_SELECT; +- break; +- case AMD_PCI_YC_ROOT_COMPLEX_ID: +- pdata->xpcs_window_def_reg = PCS_V2_YC_WINDOW_DEF; +- pdata->xpcs_window_sel_reg = PCS_V2_YC_WINDOW_SELECT; +- /* Yellow Carp devices do not need cdr workaround */ +- pdata->vdata->an_cdr_workaround = 0; ++ __cpuid(0x0, eax, ebx, ecx, edx); ++ ++ if (ebx == CPUID_VENDOR_AuthenticAMD_ebx && ++ edx == CPUID_VENDOR_AuthenticAMD_edx && ++ ecx == CPUID_VENDOR_AuthenticAMD_ecx) { ++ int unknown_cpu = 0; ++ eax = 0, ebx = 0, ecx = 0, edx = 0; ++ ++ __cpuid(0x1, eax, ebx, ecx, edx); ++ ++ cpu_family = ((GET_BITS(eax, 8, 4)) + (GET_BITS(eax, 20, 8))); ++ cpu_model = ((GET_BITS(eax, 4, 4)) | (((GET_BITS(eax, 16, 4)) << 4) & 0xF0)); ++ ++ switch (cpu_family) { ++ case Fam17h: ++ /* V1000/R1000 */ ++ if (cpu_model >= 0x10 && cpu_model <= 0x1F) { ++ pdata->xpcs_window_def_reg = PCS_V2_RV_WINDOW_DEF; ++ pdata->xpcs_window_sel_reg = PCS_V2_RV_WINDOW_SELECT; ++ /* EPYC 3000 */ ++ } else if (cpu_model >= 0x01 && cpu_model <= 0x0F) { ++ pdata->xpcs_window_def_reg = PCS_V2_WINDOW_DEF; ++ pdata->xpcs_window_sel_reg = PCS_V2_WINDOW_SELECT; ++ } else { ++ unknown_cpu = 1; ++ } + break; +- case AMD_PCI_SNOWY_ROOT_COMPLEX_ID: +- pdata->xpcs_window_def_reg = PCS_V2_WINDOW_DEF; +- pdata->xpcs_window_sel_reg = PCS_V2_WINDOW_SELECT; ++ case Fam19h: ++ /* V3000 (Yellow Carp) */ ++ if (cpu_model >= 0x44 && cpu_model <= 0x47) { ++ pdata->xpcs_window_def_reg = PCS_V2_YC_WINDOW_DEF; ++ pdata->xpcs_window_sel_reg = PCS_V2_YC_WINDOW_SELECT; ++ ++ /* Yellow Carp devices do not need cdr workaround */ ++ pdata->vdata->an_cdr_workaround = 0; ++ } else { ++ unknown_cpu = 1; ++ } + break; +- default: +- PMD_DRV_LOG(ERR, "No supported devices found\n"); +- return -ENODEV; ++ default: ++ unknown_cpu = 1; ++ break; ++ } ++ if (unknown_cpu) { ++ PMD_DRV_LOG(ERR, "Unknown CPU family, no supported axgbe device found\n"); ++ return -ENODEV; ++ } + } + + /* Configure the PCS indirect addressing support */ +diff --git a/dpdk/drivers/net/bnx2x/bnx2x_ethdev.c b/dpdk/drivers/net/bnx2x/bnx2x_ethdev.c +index 4448cf2de2..1327cbe912 100644 +--- a/dpdk/drivers/net/bnx2x/bnx2x_ethdev.c ++++ b/dpdk/drivers/net/bnx2x/bnx2x_ethdev.c +@@ -211,6 +211,7 @@ bnx2x_dev_start(struct rte_eth_dev *dev) + { + struct bnx2x_softc *sc = dev->data->dev_private; + int ret = 0; ++ uint16_t i; + + PMD_INIT_FUNC_TRACE(sc); + +@@ -244,6 +245,11 @@ bnx2x_dev_start(struct rte_eth_dev *dev) + + bnx2x_print_device_info(sc); + ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ + return ret; + } + +@@ -252,6 +258,7 @@ bnx2x_dev_stop(struct rte_eth_dev *dev) + { + struct bnx2x_softc *sc = dev->data->dev_private; + int ret = 0; ++ uint16_t i; + + PMD_INIT_FUNC_TRACE(sc); + +@@ -277,6 +284,11 @@ bnx2x_dev_stop(struct rte_eth_dev *dev) + return ret; + } + ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ + return 0; + } + +diff --git a/dpdk/drivers/net/bnxt/bnxt_ethdev.c b/dpdk/drivers/net/bnxt/bnxt_ethdev.c +index b3de490d36..e3ba48ac0b 100644 +--- a/dpdk/drivers/net/bnxt/bnxt_ethdev.c ++++ b/dpdk/drivers/net/bnxt/bnxt_ethdev.c +@@ -1017,7 +1017,6 @@ static int bnxt_dev_info_get_op(struct rte_eth_dev *eth_dev, + .tx_free_thresh = 32, + .tx_rs_thresh = 32, + }; +- eth_dev->data->dev_conf.intr_conf.lsc = 1; + + dev_info->rx_desc_lim.nb_min = BNXT_MIN_RING_DESC; + dev_info->rx_desc_lim.nb_max = BNXT_MAX_RX_RING_DESC; +@@ -1483,6 +1482,7 @@ static int bnxt_dev_stop(struct rte_eth_dev *eth_dev) + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + struct rte_intr_handle *intr_handle = pci_dev->intr_handle; + struct rte_eth_link link; ++ uint16_t i; + int ret; + + eth_dev->data->dev_started = 0; +@@ -1543,6 +1543,11 @@ static int bnxt_dev_stop(struct rte_eth_dev *eth_dev) + + eth_dev->data->scattered_rx = 0; + ++ for (i = 0; i < eth_dev->data->nb_rx_queues; i++) ++ eth_dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ for (i = 0; i < eth_dev->data->nb_tx_queues; i++) ++ eth_dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ + return 0; + } + +@@ -5859,6 +5864,7 @@ bnxt_dev_init(struct rte_eth_dev *eth_dev, void *params __rte_unused) + + rte_eth_copy_pci_info(eth_dev, pci_dev); + eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS; ++ eth_dev->data->dev_flags |= RTE_ETH_DEV_INTR_LSC; + + bp = eth_dev->data->dev_private; + +diff --git a/dpdk/drivers/net/bnxt/bnxt_rxq.c b/dpdk/drivers/net/bnxt/bnxt_rxq.c +index fabbbd4560..99758dd304 100644 +--- a/dpdk/drivers/net/bnxt/bnxt_rxq.c ++++ b/dpdk/drivers/net/bnxt/bnxt_rxq.c +@@ -471,6 +471,12 @@ int bnxt_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id) + return -EINVAL; + } + ++ /* reset the previous stats for the rx_queue since the counters ++ * will be cleared when the queue is started. ++ */ ++ memset(&bp->prev_rx_ring_stats[rx_queue_id], 0, ++ sizeof(struct bnxt_ring_stats)); ++ + /* Set the queue state to started here. + * We check the status of the queue while posting buffer. + * If queue is it started, we do not post buffers for Rx. +diff --git a/dpdk/drivers/net/bnxt/bnxt_rxr.c b/dpdk/drivers/net/bnxt/bnxt_rxr.c +index daaf9ffc1e..0eebddb05d 100644 +--- a/dpdk/drivers/net/bnxt/bnxt_rxr.c ++++ b/dpdk/drivers/net/bnxt/bnxt_rxr.c +@@ -813,7 +813,6 @@ bnxt_ulp_set_mark_in_mbuf(struct bnxt *bp, struct rx_pkt_cmpl_hi *rxcmp1, + + skip_mark: + mbuf->hash.fdir.hi = 0; +- mbuf->hash.fdir.id = 0; + + return 0; + } +diff --git a/dpdk/drivers/net/bnxt/bnxt_txr.c b/dpdk/drivers/net/bnxt/bnxt_txr.c +index 67e016775c..21c2217092 100644 +--- a/dpdk/drivers/net/bnxt/bnxt_txr.c ++++ b/dpdk/drivers/net/bnxt/bnxt_txr.c +@@ -560,6 +560,12 @@ int bnxt_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id) + if (rc) + return rc; + ++ /* reset the previous stats for the tx_queue since the counters ++ * will be cleared when the queue is started. ++ */ ++ memset(&bp->prev_tx_ring_stats[tx_queue_id], 0, ++ sizeof(struct bnxt_ring_stats)); ++ + bnxt_free_hwrm_tx_ring(bp, tx_queue_id); + rc = bnxt_alloc_hwrm_tx_ring(bp, tx_queue_id); + if (rc) +diff --git a/dpdk/drivers/net/bonding/rte_eth_bond_8023ad.c b/dpdk/drivers/net/bonding/rte_eth_bond_8023ad.c +index 4a266bb2ca..928dfca7af 100644 +--- a/dpdk/drivers/net/bonding/rte_eth_bond_8023ad.c ++++ b/dpdk/drivers/net/bonding/rte_eth_bond_8023ad.c +@@ -654,12 +654,9 @@ tx_machine(struct bond_dev_private *internals, uint16_t slave_id) + } + + static uint16_t +-max_index(uint64_t *a, int n) ++max_index(uint64_t *a, uint16_t n) + { +- if (n <= 0) +- return -1; +- +- int i, max_i = 0; ++ uint16_t i, max_i = 0; + uint64_t max = a[0]; + + for (i = 1; i < n; ++i) { +diff --git a/dpdk/drivers/net/bonding/rte_eth_bond_8023ad.h b/dpdk/drivers/net/bonding/rte_eth_bond_8023ad.h +index 7eb392f8c8..025bd0ec54 100644 +--- a/dpdk/drivers/net/bonding/rte_eth_bond_8023ad.h ++++ b/dpdk/drivers/net/bonding/rte_eth_bond_8023ad.h +@@ -197,10 +197,6 @@ int + rte_eth_bond_8023ad_slave_info(uint16_t port_id, uint16_t slave_id, + struct rte_eth_bond_8023ad_slave_info *conf); + +-#ifdef __cplusplus +-} +-#endif +- + /** + * Configure a slave port to start collecting. + * +@@ -331,4 +327,9 @@ rte_eth_bond_8023ad_agg_selection_get(uint16_t port_id); + int + rte_eth_bond_8023ad_agg_selection_set(uint16_t port_id, + enum rte_bond_8023ad_agg_selection agg_selection); ++ ++#ifdef __cplusplus ++} ++#endif ++ + #endif /* RTE_ETH_BOND_8023AD_H_ */ +diff --git a/dpdk/drivers/net/bonding/rte_eth_bond_api.c b/dpdk/drivers/net/bonding/rte_eth_bond_api.c +index c0178369b4..85d0528b7c 100644 +--- a/dpdk/drivers/net/bonding/rte_eth_bond_api.c ++++ b/dpdk/drivers/net/bonding/rte_eth_bond_api.c +@@ -712,6 +712,16 @@ __eth_bond_slave_remove_lock_free(uint16_t bonded_port_id, + } + } + ++ /* Remove the dedicated queues flow */ ++ if (internals->mode == BONDING_MODE_8023AD && ++ internals->mode4.dedicated_queues.enabled == 1 && ++ internals->mode4.dedicated_queues.flow[slave_port_id] != NULL) { ++ rte_flow_destroy(slave_port_id, ++ internals->mode4.dedicated_queues.flow[slave_port_id], ++ &flow_error); ++ internals->mode4.dedicated_queues.flow[slave_port_id] = NULL; ++ } ++ + slave_eth_dev = &rte_eth_devices[slave_port_id]; + slave_remove(internals, slave_eth_dev); + slave_eth_dev->data->dev_flags &= (~RTE_ETH_DEV_BONDED_SLAVE); +diff --git a/dpdk/drivers/net/bonding/rte_eth_bond_args.c b/dpdk/drivers/net/bonding/rte_eth_bond_args.c +index 6553166f5c..c137efd55f 100644 +--- a/dpdk/drivers/net/bonding/rte_eth_bond_args.c ++++ b/dpdk/drivers/net/bonding/rte_eth_bond_args.c +@@ -212,6 +212,12 @@ bond_ethdev_parse_socket_id_kvarg(const char *key __rte_unused, + if (*endptr != 0 || errno != 0) + return -1; + ++ /* SOCKET_ID_ANY also consider a valid socket id */ ++ if ((int8_t)socket_id == SOCKET_ID_ANY) { ++ *(int *)extra_args = SOCKET_ID_ANY; ++ return 0; ++ } ++ + /* validate socket id value */ + if (socket_id >= 0 && socket_id < RTE_MAX_NUMA_NODES) { + *(int *)extra_args = (int)socket_id; +diff --git a/dpdk/drivers/net/bonding/rte_eth_bond_pmd.c b/dpdk/drivers/net/bonding/rte_eth_bond_pmd.c +index b9bcebc6cb..8df632fa6e 100644 +--- a/dpdk/drivers/net/bonding/rte_eth_bond_pmd.c ++++ b/dpdk/drivers/net/bonding/rte_eth_bond_pmd.c +@@ -2089,6 +2089,11 @@ bond_ethdev_start(struct rte_eth_dev *eth_dev) + internals->mode == BONDING_MODE_ALB) + bond_tlb_enable(internals); + ++ for (i = 0; i < eth_dev->data->nb_rx_queues; i++) ++ eth_dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ for (i = 0; i < eth_dev->data->nb_tx_queues; i++) ++ eth_dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ + return 0; + + out_err: +@@ -2155,6 +2160,10 @@ bond_ethdev_stop(struct rte_eth_dev *eth_dev) + eth_dev->data->dev_link.link_status = RTE_ETH_LINK_DOWN; + eth_dev->data->dev_started = 0; + ++ if (internals->link_status_polling_enabled) { ++ rte_eal_alarm_cancel(bond_ethdev_slave_link_status_change_monitor, ++ (void *)&rte_eth_devices[internals->port_id]); ++ } + internals->link_status_polling_enabled = 0; + for (i = 0; i < internals->slave_count; i++) { + uint16_t slave_id = internals->slaves[i].port_id; +@@ -2174,6 +2183,11 @@ bond_ethdev_stop(struct rte_eth_dev *eth_dev) + deactivate_slave(eth_dev, slave_id); + } + ++ for (i = 0; i < eth_dev->data->nb_rx_queues; i++) ++ eth_dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ for (i = 0; i < eth_dev->data->nb_tx_queues; i++) ++ eth_dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ + return 0; + } + +@@ -3362,7 +3376,7 @@ static int + bond_alloc(struct rte_vdev_device *dev, uint8_t mode) + { + const char *name = rte_vdev_device_name(dev); +- uint8_t socket_id = dev->device.numa_node; ++ int socket_id = dev->device.numa_node; + struct bond_dev_private *internals = NULL; + struct rte_eth_dev *eth_dev = NULL; + uint32_t vlan_filter_bmp_size; +@@ -3564,7 +3578,7 @@ bond_probe(struct rte_vdev_device *dev) + port_id = bond_alloc(dev, bonding_mode); + if (port_id < 0) { + RTE_BOND_LOG(ERR, "Failed to create socket %s in mode %u on " +- "socket %u.", name, bonding_mode, socket_id); ++ "socket %d.", name, bonding_mode, socket_id); + goto parse_error; + } + internals = rte_eth_devices[port_id].data->dev_private; +@@ -3589,7 +3603,7 @@ bond_probe(struct rte_vdev_device *dev) + + rte_eth_dev_probing_finish(&rte_eth_devices[port_id]); + RTE_BOND_LOG(INFO, "Create bonded device %s on port %d in mode %u on " +- "socket %u.", name, port_id, bonding_mode, socket_id); ++ "socket %d.", name, port_id, bonding_mode, socket_id); + return 0; + + parse_error: +diff --git a/dpdk/drivers/net/cnxk/cn10k_rx.h b/dpdk/drivers/net/cnxk/cn10k_rx.h +index 721127dddd..b60c158d55 100644 +--- a/dpdk/drivers/net/cnxk/cn10k_rx.h ++++ b/dpdk/drivers/net/cnxk/cn10k_rx.h +@@ -888,7 +888,7 @@ cn10k_nix_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts, + struct nix_cqe_hdr_s *cq; + struct rte_mbuf *mbuf; + uint64_t aura_handle; +- uint64_t sa_base; ++ uint64_t sa_base = 0; + uint16_t lmt_id; + uint64_t laddr; + +@@ -1025,9 +1025,9 @@ cn10k_nix_recv_pkts_vector(void *args, struct rte_mbuf **mbufs, uint16_t pkts, + uint8x16_t f0, f1, f2, f3; + uint16_t lmt_id, d_off; + uint64_t lbase, laddr; ++ uintptr_t sa_base = 0; + uint16_t packets = 0; + uint16_t pkts_left; +- uintptr_t sa_base; + uint32_t head; + uintptr_t cq0; + +@@ -1216,6 +1216,12 @@ cn10k_nix_recv_pkts_vector(void *args, struct rte_mbuf **mbufs, uint16_t pkts, + mbuf2 = (struct rte_mbuf *)vgetq_lane_u64(mbuf23, 0); + mbuf3 = (struct rte_mbuf *)vgetq_lane_u64(mbuf23, 1); + ++ /* Mark mempool obj as "get" as it is alloc'ed by NIX */ ++ RTE_MEMPOOL_CHECK_COOKIES(mbuf0->pool, (void **)&mbuf0, 1, 1); ++ RTE_MEMPOOL_CHECK_COOKIES(mbuf1->pool, (void **)&mbuf1, 1, 1); ++ RTE_MEMPOOL_CHECK_COOKIES(mbuf2->pool, (void **)&mbuf2, 1, 1); ++ RTE_MEMPOOL_CHECK_COOKIES(mbuf3->pool, (void **)&mbuf3, 1, 1); ++ + if (!(flags & NIX_RX_VWQE_F)) { + /* Mask to get packet len from NIX_RX_SG_S */ + const uint8x16_t shuf_msk = { +@@ -1716,12 +1722,6 @@ cn10k_nix_recv_pkts_vector(void *args, struct rte_mbuf **mbufs, uint16_t pkts, + vst1q_u64((uint64_t *)&mbufs[packets], mbuf01); + vst1q_u64((uint64_t *)&mbufs[packets + 2], mbuf23); + +- /* Mark mempool obj as "get" as it is alloc'ed by NIX */ +- RTE_MEMPOOL_CHECK_COOKIES(mbuf0->pool, (void **)&mbuf0, 1, 1); +- RTE_MEMPOOL_CHECK_COOKIES(mbuf1->pool, (void **)&mbuf1, 1, 1); +- RTE_MEMPOOL_CHECK_COOKIES(mbuf2->pool, (void **)&mbuf2, 1, 1); +- RTE_MEMPOOL_CHECK_COOKIES(mbuf3->pool, (void **)&mbuf3, 1, 1); +- + nix_mbuf_validate_next(mbuf0); + nix_mbuf_validate_next(mbuf1); + nix_mbuf_validate_next(mbuf2); +diff --git a/dpdk/drivers/net/cnxk/cn10k_tx.h b/dpdk/drivers/net/cnxk/cn10k_tx.h +index 815cd2ff1f..84b5faa137 100644 +--- a/dpdk/drivers/net/cnxk/cn10k_tx.h ++++ b/dpdk/drivers/net/cnxk/cn10k_tx.h +@@ -1696,10 +1696,12 @@ cn10k_nix_xmit_store(struct rte_mbuf *mbuf, uint8_t segdw, uintptr_t laddr, + vst1q_u64(LMT_OFF(laddr, 0, 16), cmd2); + vst1q_u64(LMT_OFF(laddr, 0, 32), cmd1); + } ++ RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf, 1, 0); + } else { + /* Store the prepared send desc to LMT lines */ + vst1q_u64(LMT_OFF(laddr, 0, 0), cmd0); + vst1q_u64(LMT_OFF(laddr, 0, 16), cmd1); ++ RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf, 1, 0); + } + } + +@@ -1907,13 +1909,13 @@ again: + vsetq_lane_u64(((struct rte_mbuf *)mbuf0)->data_off, vld1q_u64(mbuf0), 1); + len_olflags0 = vld1q_u64(mbuf0 + 3); + dataoff_iova1 = +- vsetq_lane_u64(((struct rte_mbuf *)mbuf0)->data_off, vld1q_u64(mbuf1), 1); ++ vsetq_lane_u64(((struct rte_mbuf *)mbuf1)->data_off, vld1q_u64(mbuf1), 1); + len_olflags1 = vld1q_u64(mbuf1 + 3); + dataoff_iova2 = +- vsetq_lane_u64(((struct rte_mbuf *)mbuf0)->data_off, vld1q_u64(mbuf2), 1); ++ vsetq_lane_u64(((struct rte_mbuf *)mbuf2)->data_off, vld1q_u64(mbuf2), 1); + len_olflags2 = vld1q_u64(mbuf2 + 3); + dataoff_iova3 = +- vsetq_lane_u64(((struct rte_mbuf *)mbuf0)->data_off, vld1q_u64(mbuf3), 1); ++ vsetq_lane_u64(((struct rte_mbuf *)mbuf3)->data_off, vld1q_u64(mbuf3), 1); + len_olflags3 = vld1q_u64(mbuf3 + 3); + + /* Move mbufs to point pool */ +diff --git a/dpdk/drivers/net/cnxk/cn9k_ethdev_sec.c b/dpdk/drivers/net/cnxk/cn9k_ethdev_sec.c +index 67966a4e49..327f221e38 100644 +--- a/dpdk/drivers/net/cnxk/cn9k_ethdev_sec.c ++++ b/dpdk/drivers/net/cnxk/cn9k_ethdev_sec.c +@@ -556,6 +556,7 @@ cn9k_eth_sec_session_create(void *device, + + if (!dev->outb.lf_base) { + plt_err("Could not allocate security session private data"); ++ rte_spinlock_unlock(lock); + return -ENOMEM; + } + +diff --git a/dpdk/drivers/net/cnxk/cn9k_rx.h b/dpdk/drivers/net/cnxk/cn9k_rx.h +index 1a9f920b41..0e23609df5 100644 +--- a/dpdk/drivers/net/cnxk/cn9k_rx.h ++++ b/dpdk/drivers/net/cnxk/cn9k_rx.h +@@ -260,8 +260,8 @@ nix_rx_sec_mbuf_err_update(const union nix_rx_parse_u *rx, uint16_t res, + } + + static __rte_always_inline uint64_t +-nix_rx_sec_mbuf_update(const struct nix_cqe_hdr_s *cq, struct rte_mbuf *m, +- uintptr_t sa_base, uint64_t *rearm_val, uint16_t *len) ++nix_rx_sec_mbuf_update(const struct nix_cqe_hdr_s *cq, struct rte_mbuf *m, uintptr_t sa_base, ++ uint64_t *rearm_val, uint16_t *len, uint32_t packet_type) + { + uintptr_t res_sg0 = ((uintptr_t)cq + ROC_ONF_IPSEC_INB_RES_OFF - 8); + const union nix_rx_parse_u *rx = +@@ -315,15 +315,18 @@ nix_rx_sec_mbuf_update(const struct nix_cqe_hdr_s *cq, struct rte_mbuf *m, + ip = (struct rte_ipv4_hdr *)(data + ROC_ONF_IPSEC_INB_SPI_SEQ_SZ + + ROC_ONF_IPSEC_INB_MAX_L2_SZ); + ++ packet_type = (packet_type & ~(RTE_PTYPE_L3_MASK | RTE_PTYPE_TUNNEL_MASK)); + if (((ip->version_ihl & 0xf0) >> RTE_IPV4_IHL_MULTIPLIER) == + IPVERSION) { + *len = rte_be_to_cpu_16(ip->total_length) + lcptr; ++ packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN; + } else { + PLT_ASSERT(((ip->version_ihl & 0xf0) >> + RTE_IPV4_IHL_MULTIPLIER) == 6); + ip6 = (struct rte_ipv6_hdr *)ip; + *len = rte_be_to_cpu_16(ip6->payload_len) + + sizeof(struct rte_ipv6_hdr) + lcptr; ++ packet_type |= RTE_PTYPE_L3_IPV6_EXT_UNKNOWN; + } + + /* Update data offset */ +@@ -332,6 +335,7 @@ nix_rx_sec_mbuf_update(const struct nix_cqe_hdr_s *cq, struct rte_mbuf *m, + *rearm_val = *rearm_val & ~(BIT_ULL(16) - 1); + *rearm_val |= data_off; + ++ m->packet_type = packet_type; + return RTE_MBUF_F_RX_SEC_OFFLOAD; + } + +@@ -363,14 +367,7 @@ cn9k_nix_cqe_to_mbuf(const struct nix_cqe_hdr_s *cq, const uint32_t tag, + /* Get SA Base from lookup mem */ + sa_base = cnxk_nix_sa_base_get(port, lookup_mem); + +- ol_flags |= nix_rx_sec_mbuf_update(cq, mbuf, sa_base, &val, +- &len); +- +- /* Only Tunnel inner IPv4 is supported */ +- packet_type = (packet_type & +- ~(RTE_PTYPE_L3_MASK | RTE_PTYPE_TUNNEL_MASK)); +- packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN; +- mbuf->packet_type = packet_type; ++ ol_flags |= nix_rx_sec_mbuf_update(cq, mbuf, sa_base, &val, &len, packet_type); + goto skip_parse; + } + +diff --git a/dpdk/drivers/net/cnxk/cn9k_tx.h b/dpdk/drivers/net/cnxk/cn9k_tx.h +index 404edd6aed..7362025a34 100644 +--- a/dpdk/drivers/net/cnxk/cn9k_tx.h ++++ b/dpdk/drivers/net/cnxk/cn9k_tx.h +@@ -388,6 +388,16 @@ cn9k_nix_xmit_prep_lmt(uint64_t *cmd, void *lmt_addr, const uint32_t flags) + roc_lmt_mov(lmt_addr, cmd, cn9k_nix_tx_ext_subs(flags)); + } + ++static __rte_always_inline void ++cn9k_nix_sec_fc_wait_one(const struct cn9k_eth_txq *txq) ++{ ++ uint64_t nb_desc = txq->cpt_desc; ++ uint64_t *fc = txq->cpt_fc; ++ ++ while (nb_desc <= __atomic_load_n(fc, __ATOMIC_RELAXED)) ++ ; ++} ++ + static __rte_always_inline uint64_t + cn9k_nix_xmit_submit_lmt(const rte_iova_t io_addr) + { +diff --git a/dpdk/drivers/net/cnxk/cnxk_ethdev.c b/dpdk/drivers/net/cnxk/cnxk_ethdev.c +index bf1585fe67..f9245258cb 100644 +--- a/dpdk/drivers/net/cnxk/cnxk_ethdev.c ++++ b/dpdk/drivers/net/cnxk/cnxk_ethdev.c +@@ -884,6 +884,27 @@ cnxk_rss_ethdev_to_nix(struct cnxk_eth_dev *dev, uint64_t ethdev_rss, + return flowkey_cfg; + } + ++static int ++nix_rxchan_cfg_disable(struct cnxk_eth_dev *dev) ++{ ++ struct roc_nix *nix = &dev->nix; ++ struct roc_nix_fc_cfg fc_cfg; ++ int rc; ++ ++ if (!roc_nix_is_lbk(nix)) ++ return 0; ++ ++ memset(&fc_cfg, 0, sizeof(struct roc_nix_fc_cfg)); ++ fc_cfg.type = ROC_NIX_FC_RXCHAN_CFG; ++ fc_cfg.rxchan_cfg.enable = false; ++ rc = roc_nix_fc_config_set(nix, &fc_cfg); ++ if (rc) { ++ plt_err("Failed to setup flow control, rc=%d(%s)", rc, roc_error_msg_get(rc)); ++ return rc; ++ } ++ return 0; ++} ++ + static void + nix_free_queue_mem(struct cnxk_eth_dev *dev) + { +@@ -1202,6 +1223,7 @@ cnxk_nix_configure(struct rte_eth_dev *eth_dev) + goto fail_configure; + + roc_nix_tm_fini(nix); ++ nix_rxchan_cfg_disable(dev); + roc_nix_lf_free(nix); + } + +@@ -1431,6 +1453,7 @@ tm_fini: + roc_nix_tm_fini(nix); + free_nix_lf: + nix_free_queue_mem(dev); ++ rc |= nix_rxchan_cfg_disable(dev); + rc |= roc_nix_lf_free(nix); + fail_configure: + dev->configured = 0; +@@ -1980,6 +2003,11 @@ cnxk_eth_dev_uninit(struct rte_eth_dev *eth_dev, bool reset) + /* Free ROC RQ's, SQ's and CQ's memory */ + nix_free_queue_mem(dev); + ++ /* free nix bpid */ ++ rc = nix_rxchan_cfg_disable(dev); ++ if (rc) ++ plt_err("Failed to free nix bpid, rc=%d", rc); ++ + /* Free nix lf resources */ + rc = roc_nix_lf_free(nix); + if (rc) +diff --git a/dpdk/drivers/net/cnxk/cnxk_ethdev.h b/dpdk/drivers/net/cnxk/cnxk_ethdev.h +index 651ef45ea8..83d6e9398f 100644 +--- a/dpdk/drivers/net/cnxk/cnxk_ethdev.h ++++ b/dpdk/drivers/net/cnxk/cnxk_ethdev.h +@@ -494,6 +494,7 @@ int cnxk_nix_probe(struct rte_pci_driver *pci_drv, + struct rte_pci_device *pci_dev); + int cnxk_nix_remove(struct rte_pci_device *pci_dev); + int cnxk_nix_mtu_set(struct rte_eth_dev *eth_dev, uint16_t mtu); ++int cnxk_nix_sq_flush(struct rte_eth_dev *eth_dev); + int cnxk_nix_mc_addr_list_configure(struct rte_eth_dev *eth_dev, + struct rte_ether_addr *mc_addr_set, + uint32_t nb_mc_addr); +diff --git a/dpdk/drivers/net/cnxk/cnxk_ethdev_cman.c b/dpdk/drivers/net/cnxk/cnxk_ethdev_cman.c +index d5e647c64d..a7ccdfb756 100644 +--- a/dpdk/drivers/net/cnxk/cnxk_ethdev_cman.c ++++ b/dpdk/drivers/net/cnxk/cnxk_ethdev_cman.c +@@ -68,6 +68,11 @@ nix_cman_config_validate(struct rte_eth_dev *eth_dev, const struct rte_eth_cman_ + return -EINVAL; + } + ++ if (config->mode_param.red.min_th > config->mode_param.red.max_th) { ++ plt_err("RED minimum threshold must be less or equal to maximum threshold"); ++ return -EINVAL; ++ } ++ + return 0; + } + +diff --git a/dpdk/drivers/net/cnxk/cnxk_ethdev_ops.c b/dpdk/drivers/net/cnxk/cnxk_ethdev_ops.c +index 8f7287161b..7a7478cda8 100644 +--- a/dpdk/drivers/net/cnxk/cnxk_ethdev_ops.c ++++ b/dpdk/drivers/net/cnxk/cnxk_ethdev_ops.c +@@ -463,6 +463,44 @@ cnxk_nix_mac_addr_del(struct rte_eth_dev *eth_dev, uint32_t index) + dev->dmac_filter_count--; + } + ++int ++cnxk_nix_sq_flush(struct rte_eth_dev *eth_dev) ++{ ++ struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); ++ struct rte_eth_dev_data *data = eth_dev->data; ++ int i, rc = 0; ++ ++ /* Flush all tx queues */ ++ for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { ++ struct roc_nix_sq *sq = &dev->sqs[i]; ++ ++ if (eth_dev->data->tx_queues[i] == NULL) ++ continue; ++ ++ rc = roc_nix_tm_sq_aura_fc(sq, false); ++ if (rc) { ++ plt_err("Failed to disable sqb aura fc, rc=%d", rc); ++ goto exit; ++ } ++ ++ /* Wait for sq entries to be flushed */ ++ rc = roc_nix_tm_sq_flush_spin(sq); ++ if (rc) { ++ plt_err("Failed to drain sq, rc=%d\n", rc); ++ goto exit; ++ } ++ if (data->tx_queue_state[i] == RTE_ETH_QUEUE_STATE_STARTED) { ++ rc = roc_nix_tm_sq_aura_fc(sq, true); ++ if (rc) { ++ plt_err("Failed to enable sq aura fc, txq=%u, rc=%d", i, rc); ++ goto exit; ++ } ++ } ++ } ++exit: ++ return rc; ++} ++ + int + cnxk_nix_mtu_set(struct rte_eth_dev *eth_dev, uint16_t mtu) + { +@@ -506,6 +544,15 @@ cnxk_nix_mtu_set(struct rte_eth_dev *eth_dev, uint16_t mtu) + goto exit; + } + ++ /* if new MTU was smaller than old one, then flush all SQs before MTU change */ ++ if (old_frame_size > frame_size) { ++ if (data->dev_started) { ++ plt_err("Reducing MTU is not supported when device started"); ++ goto exit; ++ } ++ cnxk_nix_sq_flush(eth_dev); ++ } ++ + frame_size -= RTE_ETHER_CRC_LEN; + + /* Update mtu on Tx */ +diff --git a/dpdk/drivers/net/cnxk/cnxk_flow.c b/dpdk/drivers/net/cnxk/cnxk_flow.c +index 6d155d924c..422c5d74df 100644 +--- a/dpdk/drivers/net/cnxk/cnxk_flow.c ++++ b/dpdk/drivers/net/cnxk/cnxk_flow.c +@@ -115,14 +115,15 @@ cnxk_map_actions(struct rte_eth_dev *eth_dev, const struct rte_flow_attr *attr, + struct roc_npc_action in_actions[], uint32_t *flowkey_cfg) + { + struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); ++ const struct rte_flow_action_queue *act_q = NULL; + const struct rte_flow_action_ethdev *act_ethdev; + const struct rte_flow_action_port_id *port_act; +- const struct rte_flow_action_queue *act_q; + struct roc_npc *roc_npc_src = &dev->npc; + struct rte_eth_dev *portid_eth_dev; + char if_name[RTE_ETH_NAME_MAX_LEN]; + struct cnxk_eth_dev *hw_dst; + struct roc_npc *roc_npc_dst; ++ bool is_vf_action = false; + int i = 0, rc = 0; + int rq; + +@@ -156,6 +157,7 @@ cnxk_map_actions(struct rte_eth_dev *eth_dev, const struct rte_flow_attr *attr, + case RTE_FLOW_ACTION_TYPE_VF: + in_actions[i].type = ROC_NPC_ACTION_TYPE_VF; + in_actions[i].conf = actions->conf; ++ is_vf_action = true; + break; + + case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT: +@@ -193,13 +195,7 @@ cnxk_map_actions(struct rte_eth_dev *eth_dev, const struct rte_flow_attr *attr, + break; + + case RTE_FLOW_ACTION_TYPE_QUEUE: +- act_q = (const struct rte_flow_action_queue *) +- actions->conf; +- rq = act_q->index; +- if (rq >= eth_dev->data->nb_rx_queues) { +- plt_npc_dbg("Invalid queue index"); +- goto err_exit; +- } ++ act_q = (const struct rte_flow_action_queue *)actions->conf; + in_actions[i].type = ROC_NPC_ACTION_TYPE_QUEUE; + in_actions[i].conf = actions->conf; + break; +@@ -245,6 +241,14 @@ cnxk_map_actions(struct rte_eth_dev *eth_dev, const struct rte_flow_attr *attr, + } + i++; + } ++ ++ if (!is_vf_action && act_q) { ++ rq = act_q->index; ++ if (rq >= eth_dev->data->nb_rx_queues) { ++ plt_npc_dbg("Invalid queue index"); ++ goto err_exit; ++ } ++ } + in_actions[i].type = ROC_NPC_ACTION_TYPE_END; + return 0; + +diff --git a/dpdk/drivers/net/cxgbe/cxgbe_ethdev.c b/dpdk/drivers/net/cxgbe/cxgbe_ethdev.c +index 45bbeaef0c..8cc3d9f257 100644 +--- a/dpdk/drivers/net/cxgbe/cxgbe_ethdev.c ++++ b/dpdk/drivers/net/cxgbe/cxgbe_ethdev.c +@@ -414,6 +414,7 @@ int cxgbe_dev_stop(struct rte_eth_dev *eth_dev) + { + struct port_info *pi = eth_dev->data->dev_private; + struct adapter *adapter = pi->adapter; ++ uint16_t i; + + CXGBE_FUNC_TRACE(); + +@@ -429,6 +430,11 @@ int cxgbe_dev_stop(struct rte_eth_dev *eth_dev) + t4_sge_eth_clear_queues(pi); + eth_dev->data->scattered_rx = 0; + ++ for (i = 0; i < eth_dev->data->nb_rx_queues; i++) ++ eth_dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ for (i = 0; i < eth_dev->data->nb_tx_queues; i++) ++ eth_dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ + return 0; + } + +diff --git a/dpdk/drivers/net/dpaa/dpaa_ethdev.c b/dpdk/drivers/net/dpaa/dpaa_ethdev.c +index a6c86113d1..ef4c06db6a 100644 +--- a/dpdk/drivers/net/dpaa/dpaa_ethdev.c ++++ b/dpdk/drivers/net/dpaa/dpaa_ethdev.c +@@ -399,6 +399,7 @@ static void dpaa_interrupt_handler(void *param) + static int dpaa_eth_dev_start(struct rte_eth_dev *dev) + { + struct dpaa_if *dpaa_intf = dev->data->dev_private; ++ uint16_t i; + + PMD_INIT_FUNC_TRACE(); + +@@ -413,12 +414,18 @@ static int dpaa_eth_dev_start(struct rte_eth_dev *dev) + + fman_if_enable_rx(dev->process_private); + ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ + return 0; + } + + static int dpaa_eth_dev_stop(struct rte_eth_dev *dev) + { + struct fman_if *fif = dev->process_private; ++ uint16_t i; + + PMD_INIT_FUNC_TRACE(); + dev->data->dev_started = 0; +@@ -427,6 +434,11 @@ static int dpaa_eth_dev_stop(struct rte_eth_dev *dev) + fman_if_disable_rx(fif); + dev->tx_pkt_burst = dpaa_eth_tx_drop_all; + ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ + return 0; + } + +diff --git a/dpdk/drivers/net/dpaa2/dpaa2_ethdev.c b/dpdk/drivers/net/dpaa2/dpaa2_ethdev.c +index 679f33ae1a..8e610b6bba 100644 +--- a/dpdk/drivers/net/dpaa2/dpaa2_ethdev.c ++++ b/dpdk/drivers/net/dpaa2/dpaa2_ethdev.c +@@ -1278,6 +1278,11 @@ dpaa2_dev_start(struct rte_eth_dev *dev) + if (priv->en_ordered) + dev->tx_pkt_burst = dpaa2_dev_tx_ordered; + ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ + return 0; + } + +@@ -1295,6 +1300,7 @@ dpaa2_dev_stop(struct rte_eth_dev *dev) + struct rte_device *rdev = dev->device; + struct rte_intr_handle *intr_handle; + struct rte_dpaa2_device *dpaa2_dev; ++ uint16_t i; + + dpaa2_dev = container_of(rdev, struct rte_dpaa2_device, device); + intr_handle = dpaa2_dev->intr_handle; +@@ -1329,6 +1335,11 @@ dpaa2_dev_stop(struct rte_eth_dev *dev) + memset(&link, 0, sizeof(link)); + rte_eth_linkstatus_set(dev, &link); + ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ + return 0; + } + +diff --git a/dpdk/drivers/net/dpaa2/dpaa2_rxtx.c b/dpdk/drivers/net/dpaa2/dpaa2_rxtx.c +index f60e78e1fd..85910bbd8f 100644 +--- a/dpdk/drivers/net/dpaa2/dpaa2_rxtx.c ++++ b/dpdk/drivers/net/dpaa2/dpaa2_rxtx.c +@@ -198,8 +198,12 @@ dpaa2_dev_rx_parse_slow(struct rte_mbuf *mbuf, + + if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L3CE)) + mbuf->ol_flags |= RTE_MBUF_F_RX_IP_CKSUM_BAD; +- else if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L4CE)) ++ else ++ mbuf->ol_flags |= RTE_MBUF_F_RX_IP_CKSUM_GOOD; ++ if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L4CE)) + mbuf->ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_BAD; ++ else ++ mbuf->ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_GOOD; + + if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_1_FIRST_FRAGMENT | + L3_IP_1_MORE_FRAGMENT | +@@ -241,8 +245,12 @@ dpaa2_dev_rx_parse(struct rte_mbuf *mbuf, void *hw_annot_addr) + + if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L3CE)) + mbuf->ol_flags |= RTE_MBUF_F_RX_IP_CKSUM_BAD; +- else if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L4CE)) ++ else ++ mbuf->ol_flags |= RTE_MBUF_F_RX_IP_CKSUM_GOOD; ++ if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L4CE)) + mbuf->ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_BAD; ++ else ++ mbuf->ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_GOOD; + + if (dpaa2_enable_ts[mbuf->port]) { + *dpaa2_timestamp_dynfield(mbuf) = annotation->word2; +diff --git a/dpdk/drivers/net/e1000/em_ethdev.c b/dpdk/drivers/net/e1000/em_ethdev.c +index 8ee9be12ad..18efa78ac3 100644 +--- a/dpdk/drivers/net/e1000/em_ethdev.c ++++ b/dpdk/drivers/net/e1000/em_ethdev.c +@@ -1073,8 +1073,8 @@ eth_em_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) + * To avoid it we support just one RX queue for now (no RSS). + */ + +- dev_info->max_rx_queues = 1; +- dev_info->max_tx_queues = 1; ++ dev_info->max_rx_queues = 2; ++ dev_info->max_tx_queues = 2; + + dev_info->rx_queue_offload_capa = em_get_rx_queue_offloads_capa(); + dev_info->rx_offload_capa = em_get_rx_port_offloads_capa() | +diff --git a/dpdk/drivers/net/e1000/em_rxtx.c b/dpdk/drivers/net/e1000/em_rxtx.c +index d48fd52404..df5fbb7823 100644 +--- a/dpdk/drivers/net/e1000/em_rxtx.c ++++ b/dpdk/drivers/net/e1000/em_rxtx.c +@@ -1030,6 +1030,7 @@ eth_em_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + * - RX port identifier, + * - hardware offload data, if any: + * - IP checksum flag, ++ * - VLAN TCI, if any, + * - error flags. + */ + first_seg->port = rxq->port_id; +@@ -1039,7 +1040,7 @@ eth_em_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + rx_desc_error_to_pkt_flags(rxd.errors); + + /* Only valid if RTE_MBUF_F_RX_VLAN set in pkt_flags */ +- rxm->vlan_tci = rte_le_to_cpu_16(rxd.special); ++ first_seg->vlan_tci = rte_le_to_cpu_16(rxd.special); + + /* Prefetch data of first segment, if configured to do so. */ + rte_packet_prefetch((char *)first_seg->buf_addr + +@@ -1575,6 +1576,8 @@ em_dev_clear_queues(struct rte_eth_dev *dev) + em_tx_queue_release_mbufs(txq); + em_reset_tx_queue(txq); + } ++ ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; + } + + for (i = 0; i < dev->data->nb_rx_queues; i++) { +@@ -1583,6 +1586,8 @@ em_dev_clear_queues(struct rte_eth_dev *dev) + em_rx_queue_release_mbufs(rxq); + em_reset_rx_queue(rxq); + } ++ ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; + } + } + +@@ -1811,6 +1816,8 @@ eth_em_rx_init(struct rte_eth_dev *dev) + rxdctl |= E1000_RXDCTL_GRAN; + E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl); + ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ + /* + * Due to EM devices not having any sort of hardware + * limit for packet length, jumbo frame of any size +@@ -1945,6 +1952,8 @@ eth_em_tx_init(struct rte_eth_dev *dev) + txdctl |= (txq->wthresh & 0x3F) << 16; + txdctl |= E1000_TXDCTL_GRAN; + E1000_WRITE_REG(hw, E1000_TXDCTL(i), txdctl); ++ ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; + } + + /* Program the Transmit Control Register. */ +diff --git a/dpdk/drivers/net/e1000/igb_rxtx.c b/dpdk/drivers/net/e1000/igb_rxtx.c +index f32dee46df..6027cfbfb1 100644 +--- a/dpdk/drivers/net/e1000/igb_rxtx.c ++++ b/dpdk/drivers/net/e1000/igb_rxtx.c +@@ -1853,6 +1853,7 @@ igb_dev_clear_queues(struct rte_eth_dev *dev) + if (txq != NULL) { + igb_tx_queue_release_mbufs(txq); + igb_reset_tx_queue(txq, dev); ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; + } + } + +@@ -1861,6 +1862,7 @@ igb_dev_clear_queues(struct rte_eth_dev *dev) + if (rxq != NULL) { + igb_rx_queue_release_mbufs(rxq); + igb_reset_rx_queue(rxq); ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; + } + } + } +@@ -2441,6 +2443,7 @@ eth_igb_rx_init(struct rte_eth_dev *dev) + rxdctl |= ((rxq->hthresh & 0x1F) << 8); + rxdctl |= ((rxq->wthresh & 0x1F) << 16); + E1000_WRITE_REG(hw, E1000_RXDCTL(rxq->reg_idx), rxdctl); ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; + } + + if (dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_SCATTER) { +@@ -2605,6 +2608,7 @@ eth_igb_tx_init(struct rte_eth_dev *dev) + txdctl |= ((txq->wthresh & 0x1F) << 16); + txdctl |= E1000_TXDCTL_QUEUE_ENABLE; + E1000_WRITE_REG(hw, E1000_TXDCTL(txq->reg_idx), txdctl); ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; + } + + /* Program the Transmit Control Register. */ +@@ -2740,6 +2744,8 @@ eth_igbvf_rx_init(struct rte_eth_dev *dev) + else + rxdctl |= ((rxq->wthresh & 0x1F) << 16); + E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl); ++ ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; + } + + if (dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_SCATTER) { +@@ -2811,6 +2817,8 @@ eth_igbvf_tx_init(struct rte_eth_dev *dev) + txdctl |= ((txq->wthresh & 0x1F) << 16); + txdctl |= E1000_TXDCTL_QUEUE_ENABLE; + E1000_WRITE_REG(hw, E1000_TXDCTL(i), txdctl); ++ ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; + } + + } +diff --git a/dpdk/drivers/net/ena/ena_ethdev.c b/dpdk/drivers/net/ena/ena_ethdev.c +index efcb163027..7345e480f8 100644 +--- a/dpdk/drivers/net/ena/ena_ethdev.c ++++ b/dpdk/drivers/net/ena/ena_ethdev.c +@@ -1171,6 +1171,7 @@ static int ena_start(struct rte_eth_dev *dev) + struct ena_adapter *adapter = dev->data->dev_private; + uint64_t ticks; + int rc = 0; ++ uint16_t i; + + /* Cannot allocate memory in secondary process */ + if (rte_eal_process_type() != RTE_PROC_PRIMARY) { +@@ -1208,6 +1209,11 @@ static int ena_start(struct rte_eth_dev *dev) + ++adapter->dev_stats.dev_start; + adapter->state = ENA_ADAPTER_STATE_RUNNING; + ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ + return 0; + + err_rss_init: +@@ -1223,6 +1229,7 @@ static int ena_stop(struct rte_eth_dev *dev) + struct ena_com_dev *ena_dev = &adapter->ena_dev; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = pci_dev->intr_handle; ++ uint16_t i; + int rc; + + /* Cannot free memory in secondary process */ +@@ -1254,6 +1261,11 @@ static int ena_stop(struct rte_eth_dev *dev) + adapter->state = ENA_ADAPTER_STATE_STOPPED; + dev->data->dev_started = 0; + ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ + return 0; + } + +diff --git a/dpdk/drivers/net/ena/ena_rss.c b/dpdk/drivers/net/ena/ena_rss.c +index b682d01c20..d0ba9d5c0a 100644 +--- a/dpdk/drivers/net/ena/ena_rss.c ++++ b/dpdk/drivers/net/ena/ena_rss.c +@@ -105,6 +105,7 @@ int ena_rss_reta_update(struct rte_eth_dev *dev, + if (unlikely(rc != 0)) { + PMD_DRV_LOG(ERR, + "Cannot fill indirection table\n"); ++ rte_spinlock_unlock(&adapter->admin_lock); + return rc; + } + } +diff --git a/dpdk/drivers/net/enetc/enetc_ethdev.c b/dpdk/drivers/net/enetc/enetc_ethdev.c +index 1b4337bc48..c9352f0746 100644 +--- a/dpdk/drivers/net/enetc/enetc_ethdev.c ++++ b/dpdk/drivers/net/enetc/enetc_ethdev.c +@@ -17,6 +17,7 @@ enetc_dev_start(struct rte_eth_dev *dev) + ENETC_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct enetc_hw *enetc_hw = &hw->hw; + uint32_t val; ++ uint16_t i; + + PMD_INIT_FUNC_TRACE(); + if (hw->device_id == ENETC_DEV_ID_VF) +@@ -45,6 +46,11 @@ enetc_dev_start(struct rte_eth_dev *dev) + ENETC_PM0_IFM_XGMII); + } + ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ + return 0; + } + +@@ -55,6 +61,7 @@ enetc_dev_stop(struct rte_eth_dev *dev) + ENETC_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct enetc_hw *enetc_hw = &hw->hw; + uint32_t val; ++ uint16_t i; + + PMD_INIT_FUNC_TRACE(); + dev->data->dev_started = 0; +@@ -69,6 +76,11 @@ enetc_dev_stop(struct rte_eth_dev *dev) + enetc_port_wr(enetc_hw, ENETC_PM0_CMD_CFG, + val & (~(ENETC_PM0_TX_EN | ENETC_PM0_RX_EN))); + ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ + return 0; + } + +diff --git a/dpdk/drivers/net/enic/enic_ethdev.c b/dpdk/drivers/net/enic/enic_ethdev.c +index cdf0915591..a487256fa1 100644 +--- a/dpdk/drivers/net/enic/enic_ethdev.c ++++ b/dpdk/drivers/net/enic/enic_ethdev.c +@@ -368,6 +368,7 @@ static int enicpmd_dev_stop(struct rte_eth_dev *eth_dev) + { + struct rte_eth_link link; + struct enic *enic = pmd_priv(eth_dev); ++ uint16_t i; + + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return 0; +@@ -378,6 +379,11 @@ static int enicpmd_dev_stop(struct rte_eth_dev *eth_dev) + memset(&link, 0, sizeof(link)); + rte_eth_linkstatus_set(eth_dev, &link); + ++ for (i = 0; i < eth_dev->data->nb_rx_queues; i++) ++ eth_dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ for (i = 0; i < eth_dev->data->nb_tx_queues; i++) ++ eth_dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ + return 0; + } + +diff --git a/dpdk/drivers/net/enic/enic_main.c b/dpdk/drivers/net/enic/enic_main.c +index 19a99a82c5..a6aaa760ca 100644 +--- a/dpdk/drivers/net/enic/enic_main.c ++++ b/dpdk/drivers/net/enic/enic_main.c +@@ -1639,7 +1639,7 @@ int enic_set_mtu(struct enic *enic, uint16_t new_mtu) + * packet length. + */ + if (!eth_dev->data->dev_started) +- goto set_mtu_done; ++ return rc; + + /* + * The device has started, re-do RQs on the fly. In the process, we +diff --git a/dpdk/drivers/net/gve/gve_ethdev.c b/dpdk/drivers/net/gve/gve_ethdev.c +index 97781f0ed3..0796d37760 100644 +--- a/dpdk/drivers/net/gve/gve_ethdev.c ++++ b/dpdk/drivers/net/gve/gve_ethdev.c +@@ -5,6 +5,7 @@ + #include "gve_ethdev.h" + #include "base/gve_adminq.h" + #include "base/gve_register.h" ++#include "rte_ether.h" + + const char gve_version_str[] = GVE_VERSION; + static const char gve_version_prefix[] = GVE_VERSION_PREFIX; +@@ -274,15 +275,14 @@ gve_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) + dev_info->max_mac_addrs = 1; + dev_info->max_rx_queues = priv->max_nb_rxq; + dev_info->max_tx_queues = priv->max_nb_txq; +- dev_info->min_rx_bufsize = GVE_MIN_BUF_SIZE; +- dev_info->max_rx_pktlen = GVE_MAX_RX_PKTLEN; +- dev_info->max_mtu = GVE_MAX_MTU; +- dev_info->min_mtu = GVE_MIN_MTU; ++ dev_info->min_rx_bufsize = GVE_RX_MIN_BUF_SIZE_GQI; ++ dev_info->max_rx_pktlen = priv->max_mtu + RTE_ETHER_HDR_LEN; ++ dev_info->max_mtu = priv->max_mtu; ++ dev_info->min_mtu = RTE_ETHER_MIN_MTU; + + dev_info->rx_offload_capa = 0; + dev_info->tx_offload_capa = + RTE_ETH_TX_OFFLOAD_MULTI_SEGS | +- RTE_ETH_TX_OFFLOAD_IPV4_CKSUM | + RTE_ETH_TX_OFFLOAD_UDP_CKSUM | + RTE_ETH_TX_OFFLOAD_TCP_CKSUM | + RTE_ETH_TX_OFFLOAD_SCTP_CKSUM | +diff --git a/dpdk/drivers/net/gve/gve_ethdev.h b/dpdk/drivers/net/gve/gve_ethdev.h +index 235e55899e..b7702a1249 100644 +--- a/dpdk/drivers/net/gve/gve_ethdev.h ++++ b/dpdk/drivers/net/gve/gve_ethdev.h +@@ -28,11 +28,9 @@ + #define GVE_DEFAULT_TX_FREE_THRESH 256 + #define GVE_TX_MAX_FREE_SZ 512 + +-#define GVE_MIN_BUF_SIZE 1024 +-#define GVE_MAX_RX_PKTLEN 65535 +- +-#define GVE_MAX_MTU RTE_ETHER_MTU +-#define GVE_MIN_MTU RTE_ETHER_MIN_MTU ++#define GVE_RX_BUF_ALIGN_GQI 2048 ++#define GVE_RX_MIN_BUF_SIZE_GQI 2048 ++#define GVE_RX_MAX_BUF_SIZE_GQI 4096 + + /* A list of pages registered with the device during setup and used by a queue + * as buffers +diff --git a/dpdk/drivers/net/gve/gve_rx.c b/dpdk/drivers/net/gve/gve_rx.c +index 518c9d109c..50f9f5c370 100644 +--- a/dpdk/drivers/net/gve/gve_rx.c ++++ b/dpdk/drivers/net/gve/gve_rx.c +@@ -222,6 +222,7 @@ gve_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id, + const struct rte_memzone *mz; + struct gve_rx_queue *rxq; + uint16_t free_thresh; ++ uint32_t mbuf_len; + int err = 0; + + if (nb_desc != hw->rx_desc_cnt) { +@@ -265,7 +266,11 @@ gve_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id, + rxq->hw = hw; + rxq->ntfy_addr = &hw->db_bar2[rte_be_to_cpu_32(hw->irq_dbs[rxq->ntfy_id].id)]; + +- rxq->rx_buf_len = rte_pktmbuf_data_room_size(rxq->mpool) - RTE_PKTMBUF_HEADROOM; ++ mbuf_len = ++ rte_pktmbuf_data_room_size(rxq->mpool) - RTE_PKTMBUF_HEADROOM; ++ rxq->rx_buf_len = ++ RTE_MIN((uint16_t)GVE_RX_MAX_BUF_SIZE_GQI, ++ RTE_ALIGN_FLOOR(mbuf_len, GVE_RX_BUF_ALIGN_GQI)); + + /* Allocate software ring */ + rxq->sw_ring = rte_zmalloc_socket("gve rx sw ring", sizeof(struct rte_mbuf *) * nb_desc, +diff --git a/dpdk/drivers/net/hinic/hinic_pmd_ethdev.c b/dpdk/drivers/net/hinic/hinic_pmd_ethdev.c +index 7aa5e7d8e9..adc9f75c81 100644 +--- a/dpdk/drivers/net/hinic/hinic_pmd_ethdev.c ++++ b/dpdk/drivers/net/hinic/hinic_pmd_ethdev.c +@@ -980,6 +980,7 @@ static int hinic_dev_start(struct rte_eth_dev *dev) + int rc; + char *name; + struct hinic_nic_dev *nic_dev; ++ uint16_t i; + + nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev); + name = dev->data->name; +@@ -1047,6 +1048,11 @@ static int hinic_dev_start(struct rte_eth_dev *dev) + + rte_bit_relaxed_set32(HINIC_DEV_START, &nic_dev->dev_status); + ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ + return 0; + + en_port_fail: +@@ -1169,6 +1175,7 @@ static int hinic_dev_stop(struct rte_eth_dev *dev) + uint16_t port_id; + struct hinic_nic_dev *nic_dev; + struct rte_eth_link link; ++ uint16_t i; + + nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev); + name = dev->data->name; +@@ -1215,6 +1222,11 @@ static int hinic_dev_stop(struct rte_eth_dev *dev) + hinic_free_all_rx_mbuf(dev); + hinic_free_all_tx_mbuf(dev); + ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ + return 0; + } + +diff --git a/dpdk/drivers/net/hns3/hns3_cmd.c b/dpdk/drivers/net/hns3/hns3_cmd.c +index bdfc85f934..7bdf7740c1 100644 +--- a/dpdk/drivers/net/hns3/hns3_cmd.c ++++ b/dpdk/drivers/net/hns3/hns3_cmd.c +@@ -507,6 +507,8 @@ hns3_parse_capability(struct hns3_hw *hw, + hns3_set_bit(hw->capability, HNS3_DEV_SUPPORT_RAS_IMP_B, 1); + if (hns3_get_bit(caps, HNS3_CAPS_TM_B)) + hns3_set_bit(hw->capability, HNS3_DEV_SUPPORT_TM_B, 1); ++ if (hns3_get_bit(caps, HNS3_CAPS_GRO_B)) ++ hns3_set_bit(hw->capability, HNS3_DEV_SUPPORT_GRO_B, 1); + } + + static uint32_t +@@ -519,6 +521,41 @@ hns3_build_api_caps(void) + return rte_cpu_to_le_32(api_caps); + } + ++static void ++hns3_set_dcb_capability(struct hns3_hw *hw) ++{ ++ struct hns3_adapter *hns = HNS3_DEV_HW_TO_ADAPTER(hw); ++ struct rte_pci_device *pci_dev; ++ struct rte_eth_dev *eth_dev; ++ uint16_t device_id; ++ ++ if (hns->is_vf) ++ return; ++ ++ eth_dev = &rte_eth_devices[hw->data->port_id]; ++ pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); ++ device_id = pci_dev->id.device_id; ++ ++ if (device_id == HNS3_DEV_ID_25GE_RDMA || ++ device_id == HNS3_DEV_ID_50GE_RDMA || ++ device_id == HNS3_DEV_ID_100G_RDMA_MACSEC || ++ device_id == HNS3_DEV_ID_200G_RDMA) ++ hns3_set_bit(hw->capability, HNS3_DEV_SUPPORT_DCB_B, 1); ++} ++ ++static void ++hns3_set_default_capability(struct hns3_hw *hw) ++{ ++ hns3_set_dcb_capability(hw); ++ ++ /* ++ * The firmware of the network engines with HIP08 do not report some ++ * capabilities, like GRO. Set default capabilities for it. ++ */ ++ if (hw->revision < PCI_REVISION_ID_HIP09_A) ++ hns3_set_bit(hw->capability, HNS3_DEV_SUPPORT_GRO_B, 1); ++} ++ + static int + hns3_cmd_query_firmware_version_and_capability(struct hns3_hw *hw) + { +@@ -536,6 +573,9 @@ hns3_cmd_query_firmware_version_and_capability(struct hns3_hw *hw) + return ret; + + hw->fw_version = rte_le_to_cpu_32(resp->firmware); ++ ++ hns3_set_default_capability(hw); ++ + /* + * Make sure mask the capability before parse capability because it + * may overwrite resp's data. +@@ -659,9 +699,6 @@ hns3_cmd_init(struct hns3_hw *hw) + hw->cmq.csq.next_to_use = 0; + hw->cmq.crq.next_to_clean = 0; + hw->cmq.crq.next_to_use = 0; +- hw->mbx_resp.head = 0; +- hw->mbx_resp.tail = 0; +- hw->mbx_resp.lost = 0; + hns3_cmd_init_regs(hw); + + rte_spinlock_unlock(&hw->cmq.crq.lock); +diff --git a/dpdk/drivers/net/hns3/hns3_cmd.h b/dpdk/drivers/net/hns3/hns3_cmd.h +index 994dfc48cc..0a4d59bd9b 100644 +--- a/dpdk/drivers/net/hns3/hns3_cmd.h ++++ b/dpdk/drivers/net/hns3/hns3_cmd.h +@@ -322,6 +322,7 @@ enum HNS3_CAPS_BITS { + HNS3_CAPS_RAS_IMP_B, + HNS3_CAPS_RXD_ADV_LAYOUT_B = 15, + HNS3_CAPS_TM_B = 19, ++ HNS3_CAPS_GRO_B = 20, + }; + + /* Capabilities of VF dependent on the PF */ +@@ -606,6 +607,7 @@ struct hns3_rss_input_tuple_cmd { + #define HNS3_RSS_CFG_TBL_SIZE_H 4 + #define HNS3_RSS_CFG_TBL_BW_H 2 + #define HNS3_RSS_CFG_TBL_BW_L 8 ++#define HNS3_RSS_CFG_TBL_BW_H_M 0x3 + + /* Configure the indirection table, opcode:0x0D07 */ + struct hns3_rss_indirection_table_cmd { +diff --git a/dpdk/drivers/net/hns3/hns3_common.c b/dpdk/drivers/net/hns3/hns3_common.c +index 7adc6a4972..5d9df03733 100644 +--- a/dpdk/drivers/net/hns3/hns3_common.c ++++ b/dpdk/drivers/net/hns3/hns3_common.c +@@ -10,6 +10,7 @@ + #include "hns3_logs.h" + #include "hns3_regs.h" + #include "hns3_rxtx.h" ++#include "hns3_dcb.h" + #include "hns3_common.h" + + int +@@ -69,8 +70,7 @@ hns3_dev_infos_get(struct rte_eth_dev *eth_dev, struct rte_eth_dev_info *info) + RTE_ETH_RX_OFFLOAD_SCATTER | + RTE_ETH_RX_OFFLOAD_VLAN_STRIP | + RTE_ETH_RX_OFFLOAD_VLAN_FILTER | +- RTE_ETH_RX_OFFLOAD_RSS_HASH | +- RTE_ETH_RX_OFFLOAD_TCP_LRO); ++ RTE_ETH_RX_OFFLOAD_RSS_HASH); + info->tx_offload_capa = (RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM | + RTE_ETH_TX_OFFLOAD_IPV4_CKSUM | + RTE_ETH_TX_OFFLOAD_TCP_CKSUM | +@@ -90,13 +90,16 @@ hns3_dev_infos_get(struct rte_eth_dev *eth_dev, struct rte_eth_dev_info *info) + if (hns3_dev_get_support(hw, OUTER_UDP_CKSUM)) + info->tx_offload_capa |= RTE_ETH_TX_OFFLOAD_OUTER_UDP_CKSUM; + ++ info->dev_capa = RTE_ETH_DEV_CAPA_FLOW_RULE_KEEP | ++ RTE_ETH_DEV_CAPA_FLOW_SHARED_OBJECT_KEEP; + if (hns3_dev_get_support(hw, INDEP_TXRX)) +- info->dev_capa = RTE_ETH_DEV_CAPA_RUNTIME_RX_QUEUE_SETUP | +- RTE_ETH_DEV_CAPA_RUNTIME_TX_QUEUE_SETUP; +- info->dev_capa &= ~RTE_ETH_DEV_CAPA_FLOW_RULE_KEEP; ++ info->dev_capa |= RTE_ETH_DEV_CAPA_RUNTIME_RX_QUEUE_SETUP | ++ RTE_ETH_DEV_CAPA_RUNTIME_TX_QUEUE_SETUP; + + if (hns3_dev_get_support(hw, PTP)) + info->rx_offload_capa |= RTE_ETH_RX_OFFLOAD_TIMESTAMP; ++ if (hns3_dev_get_support(hw, GRO)) ++ info->rx_offload_capa |= RTE_ETH_RX_OFFLOAD_TCP_LRO; + + info->rx_desc_lim = (struct rte_eth_desc_lim) { + .nb_max = HNS3_MAX_RING_DESC, +@@ -128,7 +131,7 @@ hns3_dev_infos_get(struct rte_eth_dev *eth_dev, struct rte_eth_dev_info *info) + }; + + info->reta_size = hw->rss_ind_tbl_size; +- info->hash_key_size = HNS3_RSS_KEY_SIZE; ++ info->hash_key_size = hw->rss_key_size; + info->flow_type_rss_offloads = HNS3_ETH_RSS_SUPPORT; + + info->default_rxportconf.burst_size = HNS3_DEFAULT_PORT_CONF_BURST_SIZE; +@@ -161,6 +164,9 @@ hns3_parse_io_hint_func(const char *key, const char *value, void *extra_args) + + RTE_SET_USED(key); + ++ if (value == NULL || extra_args == NULL) ++ return 0; ++ + if (strcmp(value, "vec") == 0) + hint = HNS3_IO_FUNC_HINT_VEC; + else if (strcmp(value, "sve") == 0) +@@ -201,6 +207,9 @@ hns3_parse_dev_caps_mask(const char *key, const char *value, void *extra_args) + + RTE_SET_USED(key); + ++ if (value == NULL || extra_args == NULL) ++ return 0; ++ + val = strtoull(value, NULL, HNS3_CONVERT_TO_HEXADECIMAL); + *(uint64_t *)extra_args = val; + +@@ -214,6 +223,9 @@ hns3_parse_mbx_time_limit(const char *key, const char *value, void *extra_args) + + RTE_SET_USED(key); + ++ if (value == NULL || extra_args == NULL) ++ return 0; ++ + val = strtoul(value, NULL, HNS3_CONVERT_TO_DECIMAL); + + /* +@@ -340,7 +352,7 @@ hns3_set_mc_addr_chk_param(struct hns3_hw *hw, + hns3_err(hw, "failed to set mc mac addr, nb_mc_addr(%u) " + "invalid. valid range: 0~%d", + nb_mc_addr, HNS3_MC_MACADDR_NUM); +- return -EINVAL; ++ return -ENOSPC; + } + + /* Check if input mac addresses are valid */ +@@ -398,6 +410,7 @@ hns3_set_mc_mac_addr_list(struct rte_eth_dev *dev, + uint32_t nb_mc_addr) + { + struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private); ++ struct hns3_adapter *hns = HNS3_DEV_HW_TO_ADAPTER(hw); + struct rte_ether_addr *addr; + int cur_addr_num; + int set_addr_num; +@@ -405,6 +418,15 @@ hns3_set_mc_mac_addr_list(struct rte_eth_dev *dev, + int ret; + int i; + ++ if (mc_addr_set == NULL || nb_mc_addr == 0) { ++ rte_spinlock_lock(&hw->lock); ++ ret = hns3_configure_all_mc_mac_addr(hns, true); ++ if (ret == 0) ++ hw->mc_addrs_num = 0; ++ rte_spinlock_unlock(&hw->lock); ++ return ret; ++ } ++ + /* Check if input parameters are valid */ + ret = hns3_set_mc_addr_chk_param(hw, mc_addr_set, nb_mc_addr); + if (ret) +@@ -845,3 +867,87 @@ hns3_get_pci_revision_id(struct hns3_hw *hw, uint8_t *revision_id) + + return 0; + } ++ ++void ++hns3_set_default_dev_specifications(struct hns3_hw *hw) ++{ ++ struct hns3_adapter *hns = HNS3_DEV_HW_TO_ADAPTER(hw); ++ ++ hw->max_non_tso_bd_num = HNS3_MAX_NON_TSO_BD_PER_PKT; ++ hw->rss_ind_tbl_size = HNS3_RSS_IND_TBL_SIZE; ++ hw->rss_key_size = HNS3_RSS_KEY_SIZE; ++ hw->intr.int_ql_max = HNS3_INTR_QL_NONE; ++ ++ if (hns->is_vf) ++ return; ++ ++ hw->max_tm_rate = HNS3_ETHER_MAX_RATE; ++} ++ ++static void ++hns3_parse_dev_specifications(struct hns3_hw *hw, struct hns3_cmd_desc *desc) ++{ ++ struct hns3_adapter *hns = HNS3_DEV_HW_TO_ADAPTER(hw); ++ struct hns3_dev_specs_0_cmd *req0; ++ struct hns3_dev_specs_1_cmd *req1; ++ ++ req0 = (struct hns3_dev_specs_0_cmd *)desc[0].data; ++ req1 = (struct hns3_dev_specs_1_cmd *)desc[1].data; ++ ++ hw->max_non_tso_bd_num = req0->max_non_tso_bd_num; ++ hw->rss_ind_tbl_size = rte_le_to_cpu_16(req0->rss_ind_tbl_size); ++ hw->rss_key_size = rte_le_to_cpu_16(req0->rss_key_size); ++ hw->intr.int_ql_max = rte_le_to_cpu_16(req0->intr_ql_max); ++ hw->min_tx_pkt_len = req1->min_tx_pkt_len; ++ ++ if (hns->is_vf) ++ return; ++ ++ hw->max_tm_rate = rte_le_to_cpu_32(req0->max_tm_rate); ++} ++ ++static int ++hns3_check_dev_specifications(struct hns3_hw *hw) ++{ ++ if (hw->rss_ind_tbl_size == 0 || ++ hw->rss_ind_tbl_size > HNS3_RSS_IND_TBL_SIZE_MAX) { ++ hns3_err(hw, "the indirection table size obtained (%u) is invalid, and should not be zero or exceed the maximum(%u)", ++ hw->rss_ind_tbl_size, HNS3_RSS_IND_TBL_SIZE_MAX); ++ return -EINVAL; ++ } ++ ++ if (hw->rss_key_size == 0 || hw->rss_key_size > HNS3_RSS_KEY_SIZE_MAX) { ++ hns3_err(hw, "the RSS key size obtained (%u) is invalid, and should not be zero or exceed the maximum(%u)", ++ hw->rss_key_size, HNS3_RSS_KEY_SIZE_MAX); ++ return -EINVAL; ++ } ++ ++ if (hw->rss_key_size > HNS3_RSS_KEY_SIZE) ++ hns3_warn(hw, "the RSS key size obtained (%u) is greater than the default key size (%u)", ++ hw->rss_key_size, HNS3_RSS_KEY_SIZE); ++ ++ return 0; ++} ++ ++int ++hns3_query_dev_specifications(struct hns3_hw *hw) ++{ ++ struct hns3_cmd_desc desc[HNS3_QUERY_DEV_SPECS_BD_NUM]; ++ int ret; ++ int i; ++ ++ for (i = 0; i < HNS3_QUERY_DEV_SPECS_BD_NUM - 1; i++) { ++ hns3_cmd_setup_basic_desc(&desc[i], HNS3_OPC_QUERY_DEV_SPECS, ++ true); ++ desc[i].flag |= rte_cpu_to_le_16(HNS3_CMD_FLAG_NEXT); ++ } ++ hns3_cmd_setup_basic_desc(&desc[i], HNS3_OPC_QUERY_DEV_SPECS, true); ++ ++ ret = hns3_cmd_send(hw, desc, HNS3_QUERY_DEV_SPECS_BD_NUM); ++ if (ret) ++ return ret; ++ ++ hns3_parse_dev_specifications(hw, desc); ++ ++ return hns3_check_dev_specifications(hw); ++} +diff --git a/dpdk/drivers/net/hns3/hns3_common.h b/dpdk/drivers/net/hns3/hns3_common.h +index 5aa001f0cc..8eaeda26e7 100644 +--- a/dpdk/drivers/net/hns3/hns3_common.h ++++ b/dpdk/drivers/net/hns3/hns3_common.h +@@ -60,5 +60,7 @@ void hns3_unmap_rx_interrupt(struct rte_eth_dev *dev); + int hns3_restore_rx_interrupt(struct hns3_hw *hw); + + int hns3_get_pci_revision_id(struct hns3_hw *hw, uint8_t *revision_id); ++void hns3_set_default_dev_specifications(struct hns3_hw *hw); ++int hns3_query_dev_specifications(struct hns3_hw *hw); + + #endif /* HNS3_COMMON_H */ +diff --git a/dpdk/drivers/net/hns3/hns3_dcb.c b/dpdk/drivers/net/hns3/hns3_dcb.c +index af045b22f7..2831d3dc62 100644 +--- a/dpdk/drivers/net/hns3/hns3_dcb.c ++++ b/dpdk/drivers/net/hns3/hns3_dcb.c +@@ -237,9 +237,9 @@ hns3_dcb_qs_weight_cfg(struct hns3_hw *hw, uint16_t qs_id, uint8_t dwrr) + static int + hns3_dcb_ets_tc_dwrr_cfg(struct hns3_hw *hw) + { +-#define DEFAULT_TC_WEIGHT 1 + #define DEFAULT_TC_OFFSET 14 + struct hns3_ets_tc_weight_cmd *ets_weight; ++ struct hns3_pg_info *pg_info; + struct hns3_cmd_desc desc; + uint8_t i; + +@@ -247,13 +247,6 @@ hns3_dcb_ets_tc_dwrr_cfg(struct hns3_hw *hw) + ets_weight = (struct hns3_ets_tc_weight_cmd *)desc.data; + + for (i = 0; i < HNS3_MAX_TC_NUM; i++) { +- struct hns3_pg_info *pg_info; +- +- ets_weight->tc_weight[i] = DEFAULT_TC_WEIGHT; +- +- if (!(hw->hw_tc_map & BIT(i))) +- continue; +- + pg_info = &hw->dcb_info.pg_info[hw->dcb_info.tc_info[i].pgid]; + ets_weight->tc_weight[i] = pg_info->tc_dwrr[i]; + } +@@ -1089,7 +1082,7 @@ hns3_dcb_map_cfg(struct hns3_hw *hw) + + ret = hns3_pg_to_pri_map(hw); + if (ret) { +- hns3_err(hw, "pri_to_pg mapping fail: %d", ret); ++ hns3_err(hw, "pg_to_pri mapping fail: %d", ret); + return ret; + } + +diff --git a/dpdk/drivers/net/hns3/hns3_dump.c b/dpdk/drivers/net/hns3/hns3_dump.c +index ae62bb56c8..bac4427227 100644 +--- a/dpdk/drivers/net/hns3/hns3_dump.c ++++ b/dpdk/drivers/net/hns3/hns3_dump.c +@@ -101,6 +101,7 @@ hns3_get_dev_feature_capability(FILE *file, struct hns3_hw *hw) + {HNS3_DEV_SUPPORT_RAS_IMP_B, "RAS IMP"}, + {HNS3_DEV_SUPPORT_TM_B, "TM"}, + {HNS3_DEV_SUPPORT_VF_VLAN_FLT_MOD_B, "VF VLAN FILTER MOD"}, ++ {HNS3_DEV_SUPPORT_GRO_B, "GRO"} + }; + uint32_t i; + +@@ -660,10 +661,10 @@ hns3_get_tm_conf_shaper_info(FILE *file, struct hns3_tm_conf *conf) + if (conf->nb_shaper_profile == 0) + return; + +- fprintf(file, " shaper_profile:\n"); ++ fprintf(file, "\t -- shaper_profile:\n"); + TAILQ_FOREACH(shaper_profile, shaper_profile_list, node) { + fprintf(file, +- " id=%u reference_count=%u peak_rate=%" PRIu64 "Bps\n", ++ "\t id=%u reference_count=%u peak_rate=%" PRIu64 "Bps\n", + shaper_profile->shaper_profile_id, + shaper_profile->reference_count, + shaper_profile->profile.peak.rate); +@@ -677,8 +678,8 @@ hns3_get_tm_conf_port_node_info(FILE *file, struct hns3_tm_conf *conf) + return; + + fprintf(file, +- " port_node:\n" +- " node_id=%u reference_count=%u shaper_profile_id=%d\n", ++ "\t -- port_node:\n" ++ "\t node_id=%u reference_count=%u shaper_profile_id=%d\n", + conf->root->id, conf->root->reference_count, + conf->root->shaper_profile ? + (int)conf->root->shaper_profile->shaper_profile_id : -1); +@@ -695,7 +696,7 @@ hns3_get_tm_conf_tc_node_info(FILE *file, struct hns3_tm_conf *conf) + if (conf->nb_tc_node == 0) + return; + +- fprintf(file, " tc_node:\n"); ++ fprintf(file, "\t -- tc_node:\n"); + memset(tc_node, 0, sizeof(tc_node)); + TAILQ_FOREACH(tm_node, tc_list, node) { + tidx = hns3_tm_calc_node_tc_no(conf, tm_node->id); +@@ -708,7 +709,7 @@ hns3_get_tm_conf_tc_node_info(FILE *file, struct hns3_tm_conf *conf) + if (tm_node == NULL) + continue; + fprintf(file, +- " id=%u TC%u reference_count=%u parent_id=%d " ++ "\t id=%u TC%u reference_count=%u parent_id=%d " + "shaper_profile_id=%d\n", + tm_node->id, hns3_tm_calc_node_tc_no(conf, tm_node->id), + tm_node->reference_count, +@@ -734,7 +735,7 @@ hns3_get_tm_conf_queue_format_info(FILE *file, struct hns3_tm_node **queue_node, + end_queue_id = (i + 1) * HNS3_PERLINE_QUEUES - 1; + if (end_queue_id > nb_tx_queues - 1) + end_queue_id = nb_tx_queues - 1; +- fprintf(file, " %04u - %04u | ", start_queue_id, ++ fprintf(file, "\t %04u - %04u | ", start_queue_id, + end_queue_id); + for (j = start_queue_id; j < nb_tx_queues; j++) { + if (j >= end_queue_id + 1) +@@ -763,8 +764,8 @@ hns3_get_tm_conf_queue_node_info(FILE *file, struct hns3_tm_conf *conf, + return; + + fprintf(file, +- " queue_node:\n" +- " tx queue id | mapped tc (8 mean node not exist)\n"); ++ "\t -- queue_node:\n" ++ "\t tx queue id | mapped tc (8 mean node not exist)\n"); + + memset(queue_node, 0, sizeof(queue_node)); + memset(queue_node_tc, 0, sizeof(queue_node_tc)); +@@ -914,6 +915,8 @@ hns3_eth_dev_priv_dump(struct rte_eth_dev *dev, FILE *file) + struct hns3_adapter *hns = dev->data->dev_private; + struct hns3_hw *hw = &hns->hw; + ++ rte_spinlock_lock(&hw->lock); ++ + hns3_get_device_basic_info(file, dev); + hns3_get_dev_feature_capability(file, hw); + hns3_get_rxtx_queue_info(file, dev); +@@ -923,8 +926,10 @@ hns3_eth_dev_priv_dump(struct rte_eth_dev *dev, FILE *file) + * VF only supports dumping basic info, feature capability and queue + * info. + */ +- if (hns->is_vf) ++ if (hns->is_vf) { ++ rte_spinlock_unlock(&hw->lock); + return 0; ++ } + + hns3_get_dev_mac_info(file, hns); + hns3_get_vlan_config_info(file, hw); +@@ -932,6 +937,8 @@ hns3_eth_dev_priv_dump(struct rte_eth_dev *dev, FILE *file) + hns3_get_tm_conf_info(file, dev); + hns3_get_flow_ctrl_info(file, dev); + ++ rte_spinlock_unlock(&hw->lock); ++ + return 0; + } + +diff --git a/dpdk/drivers/net/hns3/hns3_ethdev.c b/dpdk/drivers/net/hns3/hns3_ethdev.c +index d326f70129..27f9dd2eb2 100644 +--- a/dpdk/drivers/net/hns3/hns3_ethdev.c ++++ b/dpdk/drivers/net/hns3/hns3_ethdev.c +@@ -15,6 +15,7 @@ + #include "hns3_dcb.h" + #include "hns3_mp.h" + #include "hns3_flow.h" ++#include "hns3_ptp.h" + #include "hns3_ethdev.h" + + #define HNS3_SERVICE_INTERVAL 1000000 /* us */ +@@ -43,6 +44,7 @@ + #define HNS3_VECTOR0_IMP_CMDQ_ERR_B 4U + #define HNS3_VECTOR0_IMP_RD_POISON_B 5U + #define HNS3_VECTOR0_ALL_MSIX_ERR_B 6U ++#define HNS3_VECTOR0_TRIGGER_IMP_RESET_B 7U + + #define HNS3_RESET_WAIT_MS 100 + #define HNS3_RESET_WAIT_CNT 200 +@@ -60,6 +62,19 @@ enum hns3_evt_cause { + HNS3_VECTOR0_EVENT_OTHER, + }; + ++struct hns3_intr_state { ++ uint32_t vector0_state; ++ uint32_t cmdq_state; ++ uint32_t hw_err_state; ++}; ++ ++#define HNS3_SPEEDS_SUPP_FEC (RTE_ETH_LINK_SPEED_10G | \ ++ RTE_ETH_LINK_SPEED_25G | \ ++ RTE_ETH_LINK_SPEED_40G | \ ++ RTE_ETH_LINK_SPEED_50G | \ ++ RTE_ETH_LINK_SPEED_100G | \ ++ RTE_ETH_LINK_SPEED_200G) ++ + static const struct rte_eth_fec_capa speed_fec_capa_tbl[] = { + { RTE_ETH_SPEED_NUM_10G, RTE_ETH_FEC_MODE_CAPA_MASK(NOFEC) | + RTE_ETH_FEC_MODE_CAPA_MASK(AUTO) | +@@ -83,8 +98,7 @@ static const struct rte_eth_fec_capa speed_fec_capa_tbl[] = { + RTE_ETH_FEC_MODE_CAPA_MASK(AUTO) | + RTE_ETH_FEC_MODE_CAPA_MASK(RS) }, + +- { RTE_ETH_SPEED_NUM_200G, RTE_ETH_FEC_MODE_CAPA_MASK(NOFEC) | +- RTE_ETH_FEC_MODE_CAPA_MASK(AUTO) | ++ { RTE_ETH_SPEED_NUM_200G, RTE_ETH_FEC_MODE_CAPA_MASK(AUTO) | + RTE_ETH_FEC_MODE_CAPA_MASK(RS) } + }; + +@@ -120,63 +134,51 @@ hns3_pf_enable_irq0(struct hns3_hw *hw) + } + + static enum hns3_evt_cause +-hns3_proc_imp_reset_event(struct hns3_adapter *hns, bool is_delay, +- uint32_t *vec_val) ++hns3_proc_imp_reset_event(struct hns3_adapter *hns, uint32_t *vec_val) + { + struct hns3_hw *hw = &hns->hw; + + __atomic_store_n(&hw->reset.disable_cmd, 1, __ATOMIC_RELAXED); + hns3_atomic_set_bit(HNS3_IMP_RESET, &hw->reset.pending); + *vec_val = BIT(HNS3_VECTOR0_IMPRESET_INT_B); +- if (!is_delay) { +- hw->reset.stats.imp_cnt++; +- hns3_warn(hw, "IMP reset detected, clear reset status"); +- } else { +- hns3_schedule_delayed_reset(hns); +- hns3_warn(hw, "IMP reset detected, don't clear reset status"); +- } ++ hw->reset.stats.imp_cnt++; ++ hns3_warn(hw, "IMP reset detected, clear reset status"); + + return HNS3_VECTOR0_EVENT_RST; + } + + static enum hns3_evt_cause +-hns3_proc_global_reset_event(struct hns3_adapter *hns, bool is_delay, +- uint32_t *vec_val) ++hns3_proc_global_reset_event(struct hns3_adapter *hns, uint32_t *vec_val) + { + struct hns3_hw *hw = &hns->hw; + + __atomic_store_n(&hw->reset.disable_cmd, 1, __ATOMIC_RELAXED); + hns3_atomic_set_bit(HNS3_GLOBAL_RESET, &hw->reset.pending); + *vec_val = BIT(HNS3_VECTOR0_GLOBALRESET_INT_B); +- if (!is_delay) { +- hw->reset.stats.global_cnt++; +- hns3_warn(hw, "Global reset detected, clear reset status"); +- } else { +- hns3_schedule_delayed_reset(hns); +- hns3_warn(hw, +- "Global reset detected, don't clear reset status"); +- } ++ hw->reset.stats.global_cnt++; ++ hns3_warn(hw, "Global reset detected, clear reset status"); + + return HNS3_VECTOR0_EVENT_RST; + } + ++static void ++hns3_query_intr_state(struct hns3_hw *hw, struct hns3_intr_state *state) ++{ ++ state->vector0_state = hns3_read_dev(hw, HNS3_VECTOR0_OTHER_INT_STS_REG); ++ state->cmdq_state = hns3_read_dev(hw, HNS3_VECTOR0_CMDQ_SRC_REG); ++ state->hw_err_state = hns3_read_dev(hw, HNS3_RAS_PF_OTHER_INT_STS_REG); ++} ++ + static enum hns3_evt_cause + hns3_check_event_cause(struct hns3_adapter *hns, uint32_t *clearval) + { + struct hns3_hw *hw = &hns->hw; +- uint32_t vector0_int_stats; +- uint32_t cmdq_src_val; +- uint32_t hw_err_src_reg; ++ struct hns3_intr_state state; + uint32_t val; + enum hns3_evt_cause ret; +- bool is_delay; + +- /* fetch the events from their corresponding regs */ +- vector0_int_stats = hns3_read_dev(hw, HNS3_VECTOR0_OTHER_INT_STS_REG); +- cmdq_src_val = hns3_read_dev(hw, HNS3_VECTOR0_CMDQ_SRC_REG); +- hw_err_src_reg = hns3_read_dev(hw, HNS3_RAS_PF_OTHER_INT_STS_REG); ++ hns3_query_intr_state(hw, &state); + +- is_delay = clearval == NULL ? true : false; + /* + * Assumption: If by any chance reset and mailbox events are reported + * together then we will only process reset event and defer the +@@ -184,49 +186,72 @@ hns3_check_event_cause(struct hns3_adapter *hns, uint32_t *clearval) + * RX CMDQ event this time we would receive again another interrupt + * from H/W just for the mailbox. + */ +- if (BIT(HNS3_VECTOR0_IMPRESET_INT_B) & vector0_int_stats) { /* IMP */ +- ret = hns3_proc_imp_reset_event(hns, is_delay, &val); ++ if (BIT(HNS3_VECTOR0_IMPRESET_INT_B) & state.vector0_state) { /* IMP */ ++ ret = hns3_proc_imp_reset_event(hns, &val); + goto out; + } + + /* Global reset */ +- if (BIT(HNS3_VECTOR0_GLOBALRESET_INT_B) & vector0_int_stats) { +- ret = hns3_proc_global_reset_event(hns, is_delay, &val); ++ if (BIT(HNS3_VECTOR0_GLOBALRESET_INT_B) & state.vector0_state) { ++ ret = hns3_proc_global_reset_event(hns, &val); + goto out; + } + + /* Check for vector0 1588 event source */ +- if (BIT(HNS3_VECTOR0_1588_INT_B) & vector0_int_stats) { ++ if (BIT(HNS3_VECTOR0_1588_INT_B) & state.vector0_state) { + val = BIT(HNS3_VECTOR0_1588_INT_B); + ret = HNS3_VECTOR0_EVENT_PTP; + goto out; + } + + /* check for vector0 msix event source */ +- if (vector0_int_stats & HNS3_VECTOR0_REG_MSIX_MASK || +- hw_err_src_reg & HNS3_RAS_REG_NFE_MASK) { +- val = vector0_int_stats | hw_err_src_reg; ++ if (state.vector0_state & HNS3_VECTOR0_REG_MSIX_MASK || ++ state.hw_err_state & HNS3_RAS_REG_NFE_MASK) { ++ val = state.vector0_state | state.hw_err_state; + ret = HNS3_VECTOR0_EVENT_ERR; + goto out; + } + + /* check for vector0 mailbox(=CMDQ RX) event source */ +- if (BIT(HNS3_VECTOR0_RX_CMDQ_INT_B) & cmdq_src_val) { +- cmdq_src_val &= ~BIT(HNS3_VECTOR0_RX_CMDQ_INT_B); +- val = cmdq_src_val; ++ if (BIT(HNS3_VECTOR0_RX_CMDQ_INT_B) & state.cmdq_state) { ++ state.cmdq_state &= ~BIT(HNS3_VECTOR0_RX_CMDQ_INT_B); ++ val = state.cmdq_state; + ret = HNS3_VECTOR0_EVENT_MBX; + goto out; + } + +- val = vector0_int_stats; ++ val = state.vector0_state; + ret = HNS3_VECTOR0_EVENT_OTHER; +-out: + +- if (clearval) +- *clearval = val; ++out: ++ *clearval = val; + return ret; + } + ++void ++hns3_clear_reset_event(struct hns3_hw *hw) ++{ ++ uint32_t clearval = 0; ++ ++ switch (hw->reset.level) { ++ case HNS3_IMP_RESET: ++ clearval = BIT(HNS3_VECTOR0_IMPRESET_INT_B); ++ break; ++ case HNS3_GLOBAL_RESET: ++ clearval = BIT(HNS3_VECTOR0_GLOBALRESET_INT_B); ++ break; ++ default: ++ break; ++ } ++ ++ if (clearval == 0) ++ return; ++ ++ hns3_write_dev(hw, HNS3_MISC_RESET_STS_REG, clearval); ++ ++ hns3_pf_enable_irq0(hw); ++} ++ + static void + hns3_clear_event_cause(struct hns3_hw *hw, uint32_t event_type, uint32_t regclr) + { +@@ -286,6 +311,47 @@ hns3_handle_mac_tnl(struct hns3_hw *hw) + } + } + ++static void ++hns3_delay_before_clear_event_cause(struct hns3_hw *hw, uint32_t event_type, uint32_t regclr) ++{ ++#define IMPRESET_WAIT_MS_TIME 5 ++ ++ if (event_type == HNS3_VECTOR0_EVENT_RST && ++ regclr & BIT(HNS3_VECTOR0_IMPRESET_INT_B) && ++ hw->revision >= PCI_REVISION_ID_HIP09_A) { ++ rte_delay_ms(IMPRESET_WAIT_MS_TIME); ++ hns3_dbg(hw, "wait firmware watchdog initialization completed."); ++ } ++} ++ ++static bool ++hns3_reset_event_valid(struct hns3_hw *hw) ++{ ++ struct hns3_adapter *hns = HNS3_DEV_HW_TO_ADAPTER(hw); ++ enum hns3_reset_level new_req = HNS3_NONE_RESET; ++ enum hns3_reset_level last_req; ++ uint32_t vector0_int; ++ ++ vector0_int = hns3_read_dev(hw, HNS3_VECTOR0_OTHER_INT_STS_REG); ++ if (BIT(HNS3_VECTOR0_IMPRESET_INT_B) & vector0_int) ++ new_req = HNS3_IMP_RESET; ++ else if (BIT(HNS3_VECTOR0_GLOBALRESET_INT_B) & vector0_int) ++ new_req = HNS3_GLOBAL_RESET; ++ if (new_req == HNS3_NONE_RESET) ++ return true; ++ ++ last_req = hns3_get_reset_level(hns, &hw->reset.pending); ++ if (last_req == HNS3_NONE_RESET) ++ return true; ++ ++ if (new_req > last_req) ++ return true; ++ ++ hns3_warn(hw, "last_req (%u) less than or equal to new_req (%u) ignore", ++ last_req, new_req); ++ return false; ++} ++ + static void + hns3_interrupt_handler(void *param) + { +@@ -293,24 +359,25 @@ hns3_interrupt_handler(void *param) + struct hns3_adapter *hns = dev->data->dev_private; + struct hns3_hw *hw = &hns->hw; + enum hns3_evt_cause event_cause; ++ struct hns3_intr_state state; + uint32_t clearval = 0; +- uint32_t vector0_int; +- uint32_t ras_int; +- uint32_t cmdq_int; ++ ++ if (!hns3_reset_event_valid(hw)) ++ return; + + /* Disable interrupt */ + hns3_pf_disable_irq0(hw); + + event_cause = hns3_check_event_cause(hns, &clearval); +- vector0_int = hns3_read_dev(hw, HNS3_VECTOR0_OTHER_INT_STS_REG); +- ras_int = hns3_read_dev(hw, HNS3_RAS_PF_OTHER_INT_STS_REG); +- cmdq_int = hns3_read_dev(hw, HNS3_VECTOR0_CMDQ_SRC_REG); ++ hns3_query_intr_state(hw, &state); ++ hns3_delay_before_clear_event_cause(hw, event_cause, clearval); + hns3_clear_event_cause(hw, event_cause, clearval); + /* vector 0 interrupt is shared with reset and mailbox source events. */ + if (event_cause == HNS3_VECTOR0_EVENT_ERR) { + hns3_warn(hw, "received interrupt: vector0_int_stat:0x%x " + "ras_int_stat:0x%x cmdq_int_stat:0x%x", +- vector0_int, ras_int, cmdq_int); ++ state.vector0_state, state.hw_err_state, ++ state.cmdq_state); + hns3_handle_mac_tnl(hw); + hns3_handle_error(hns); + } else if (event_cause == HNS3_VECTOR0_EVENT_RST) { +@@ -321,11 +388,16 @@ hns3_interrupt_handler(void *param) + } else if (event_cause != HNS3_VECTOR0_EVENT_PTP) { + hns3_warn(hw, "received unknown event: vector0_int_stat:0x%x " + "ras_int_stat:0x%x cmdq_int_stat:0x%x", +- vector0_int, ras_int, cmdq_int); ++ state.vector0_state, state.hw_err_state, ++ state.cmdq_state); + } + + /* Enable interrupt if it is not cause by reset */ +- hns3_pf_enable_irq0(hw); ++ if (event_cause == HNS3_VECTOR0_EVENT_ERR || ++ event_cause == HNS3_VECTOR0_EVENT_MBX || ++ event_cause == HNS3_VECTOR0_EVENT_PTP || ++ event_cause == HNS3_VECTOR0_EVENT_OTHER) ++ hns3_pf_enable_irq0(hw); + } + + static int +@@ -2257,6 +2329,7 @@ hns3_dev_link_update(struct rte_eth_dev *eth_dev, int wait_to_complete) + struct rte_eth_link new_link; + int ret; + ++ memset(&new_link, 0, sizeof(new_link)); + /* When port is stopped, report link down. */ + if (eth_dev->data->dev_started == 0) { + new_link.link_autoneg = mac->link_autoneg; +@@ -2280,7 +2353,6 @@ hns3_dev_link_update(struct rte_eth_dev *eth_dev, int wait_to_complete) + rte_delay_ms(HNS3_LINK_CHECK_INTERVAL); + } while (retry_cnt--); + +- memset(&new_link, 0, sizeof(new_link)); + hns3_setup_linkstatus(eth_dev, &new_link); + + out: +@@ -2647,93 +2719,13 @@ hns3_parse_speed(int speed_cmd, uint32_t *speed) + return 0; + } + +-static void +-hns3_set_default_dev_specifications(struct hns3_hw *hw) +-{ +- hw->max_non_tso_bd_num = HNS3_MAX_NON_TSO_BD_PER_PKT; +- hw->rss_ind_tbl_size = HNS3_RSS_IND_TBL_SIZE; +- hw->rss_key_size = HNS3_RSS_KEY_SIZE; +- hw->max_tm_rate = HNS3_ETHER_MAX_RATE; +- hw->intr.int_ql_max = HNS3_INTR_QL_NONE; +-} +- +-static void +-hns3_parse_dev_specifications(struct hns3_hw *hw, struct hns3_cmd_desc *desc) +-{ +- struct hns3_dev_specs_0_cmd *req0; +- struct hns3_dev_specs_1_cmd *req1; +- +- req0 = (struct hns3_dev_specs_0_cmd *)desc[0].data; +- req1 = (struct hns3_dev_specs_1_cmd *)desc[1].data; +- +- hw->max_non_tso_bd_num = req0->max_non_tso_bd_num; +- hw->rss_ind_tbl_size = rte_le_to_cpu_16(req0->rss_ind_tbl_size); +- hw->rss_key_size = rte_le_to_cpu_16(req0->rss_key_size); +- hw->max_tm_rate = rte_le_to_cpu_32(req0->max_tm_rate); +- hw->intr.int_ql_max = rte_le_to_cpu_16(req0->intr_ql_max); +- hw->min_tx_pkt_len = req1->min_tx_pkt_len; +-} +- +-static int +-hns3_check_dev_specifications(struct hns3_hw *hw) +-{ +- if (hw->rss_ind_tbl_size == 0 || +- hw->rss_ind_tbl_size > HNS3_RSS_IND_TBL_SIZE_MAX) { +- hns3_err(hw, "the size of hash lookup table configured (%u) exceeds the maximum(%u)", +- hw->rss_ind_tbl_size, HNS3_RSS_IND_TBL_SIZE_MAX); +- return -EINVAL; +- } +- +- return 0; +-} +- +-static int +-hns3_query_dev_specifications(struct hns3_hw *hw) +-{ +- struct hns3_cmd_desc desc[HNS3_QUERY_DEV_SPECS_BD_NUM]; +- int ret; +- int i; +- +- for (i = 0; i < HNS3_QUERY_DEV_SPECS_BD_NUM - 1; i++) { +- hns3_cmd_setup_basic_desc(&desc[i], HNS3_OPC_QUERY_DEV_SPECS, +- true); +- desc[i].flag |= rte_cpu_to_le_16(HNS3_CMD_FLAG_NEXT); +- } +- hns3_cmd_setup_basic_desc(&desc[i], HNS3_OPC_QUERY_DEV_SPECS, true); +- +- ret = hns3_cmd_send(hw, desc, HNS3_QUERY_DEV_SPECS_BD_NUM); +- if (ret) +- return ret; +- +- hns3_parse_dev_specifications(hw, desc); +- +- return hns3_check_dev_specifications(hw); +-} +- + static int + hns3_get_capability(struct hns3_hw *hw) + { + struct hns3_adapter *hns = HNS3_DEV_HW_TO_ADAPTER(hw); +- struct rte_pci_device *pci_dev; + struct hns3_pf *pf = &hns->pf; +- struct rte_eth_dev *eth_dev; +- uint16_t device_id; + int ret; + +- eth_dev = &rte_eth_devices[hw->data->port_id]; +- pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); +- device_id = pci_dev->id.device_id; +- +- if (device_id == HNS3_DEV_ID_25GE_RDMA || +- device_id == HNS3_DEV_ID_50GE_RDMA || +- device_id == HNS3_DEV_ID_100G_RDMA_MACSEC || +- device_id == HNS3_DEV_ID_200G_RDMA) +- hns3_set_bit(hw->capability, HNS3_DEV_SUPPORT_DCB_B, 1); +- +- ret = hns3_get_pci_revision_id(hw, &hw->revision); +- if (ret) +- return ret; +- + ret = hns3_query_mac_stats_reg_num(hw); + if (ret) + return ret; +@@ -3677,7 +3669,7 @@ hns3_get_mac_ethertype_cmd_status(uint16_t cmdq_resp, uint8_t resp_code) + + if (cmdq_resp) { + PMD_INIT_LOG(ERR, +- "cmdq execute failed for get_mac_ethertype_cmd_status, status=%u.\n", ++ "cmdq execute failed for get_mac_ethertype_cmd_status, status=%u.", + cmdq_resp); + return -EIO; + } +@@ -4451,6 +4443,12 @@ hns3_init_hardware(struct hns3_adapter *hns) + goto err_mac_init; + } + ++ ret = hns3_ptp_init(hw); ++ if (ret) { ++ PMD_INIT_LOG(ERR, "Failed to init PTP, ret = %d", ret); ++ goto err_mac_init; ++ } ++ + return 0; + + err_mac_init: +@@ -4590,6 +4588,10 @@ hns3_init_pf(struct rte_eth_dev *eth_dev) + /* Get hardware io base address from pcie BAR2 IO space */ + hw->io_base = pci_dev->mem_resource[2].addr; + ++ ret = hns3_get_pci_revision_id(hw, &hw->revision); ++ if (ret) ++ return ret; ++ + /* Firmware command queue initialize */ + ret = hns3_cmd_init_queue(hw); + if (ret) { +@@ -4630,10 +4632,6 @@ hns3_init_pf(struct rte_eth_dev *eth_dev) + goto err_intr_callback_register; + } + +- ret = hns3_ptp_init(hw); +- if (ret) +- goto err_get_config; +- + /* Enable interrupt */ + rte_intr_enable(pci_dev->intr_handle); + hns3_pf_enable_irq0(hw); +@@ -4690,6 +4688,7 @@ err_enable_intr: + hns3_fdir_filter_uninit(hns); + err_fdir: + hns3_uninit_umv_space(hw); ++ hns3_ptp_uninit(hw); + err_init_hw: + hns3_stats_uninit(hw); + err_get_config: +@@ -4725,6 +4724,7 @@ hns3_uninit_pf(struct rte_eth_dev *eth_dev) + hns3_flow_uninit(eth_dev); + hns3_fdir_filter_uninit(hns); + hns3_uninit_umv_space(hw); ++ hns3_ptp_uninit(hw); + hns3_stats_uninit(hw); + hns3_config_mac_tnl_int(hw, false); + hns3_pf_disable_irq0(hw); +@@ -5115,8 +5115,7 @@ hns3_dev_start(struct rte_eth_dev *dev) + rte_spinlock_unlock(&hw->lock); + + hns3_rx_scattered_calc(dev); +- hns3_set_rxtx_function(dev); +- hns3_mp_req_start_rxtx(dev); ++ hns3_start_rxtx_datapath(dev); + + /* Enable interrupt of all rx queues before enabling queues */ + hns3_dev_all_rx_queue_intr_enable(hw, true); +@@ -5194,12 +5193,7 @@ hns3_dev_stop(struct rte_eth_dev *dev) + dev->data->dev_started = 0; + + hw->adapter_state = HNS3_NIC_STOPPING; +- hns3_set_rxtx_function(dev); +- rte_wmb(); +- /* Disable datapath on secondary process. */ +- hns3_mp_req_stop_rxtx(dev); +- /* Prevent crashes when queues are still in use. */ +- rte_delay_ms(hw->cfg_max_queues); ++ hns3_stop_rxtx_datapath(dev); + + rte_spinlock_lock(&hw->lock); + if (__atomic_load_n(&hw->reset.resetting, __ATOMIC_RELAXED) == 0) { +@@ -5373,16 +5367,7 @@ hns3_check_fc_autoneg_valid(struct hns3_hw *hw, uint8_t autoneg) + + if (!pf->support_fc_autoneg) { + if (autoneg != 0) { +- hns3_err(hw, "unsupported fc auto-negotiation setting."); +- return -EOPNOTSUPP; +- } +- +- /* +- * Flow control auto-negotiation of the NIC is not supported, +- * but other auto-negotiation features may be supported. +- */ +- if (autoneg != hw->mac.link_autoneg) { +- hns3_err(hw, "please use 'link_speeds' in struct rte_eth_conf to disable autoneg!"); ++ hns3_err(hw, "unsupported fc auto-negotiation."); + return -EOPNOTSUPP; + } + +@@ -5591,31 +5576,60 @@ is_pf_reset_done(struct hns3_hw *hw) + return true; + } + ++static enum hns3_reset_level ++hns3_detect_reset_event(struct hns3_hw *hw) ++{ ++ struct hns3_adapter *hns = HNS3_DEV_HW_TO_ADAPTER(hw); ++ enum hns3_reset_level new_req = HNS3_NONE_RESET; ++ enum hns3_reset_level last_req; ++ uint32_t vector0_intr_state; ++ ++ last_req = hns3_get_reset_level(hns, &hw->reset.pending); ++ vector0_intr_state = hns3_read_dev(hw, HNS3_VECTOR0_OTHER_INT_STS_REG); ++ if (BIT(HNS3_VECTOR0_IMPRESET_INT_B) & vector0_intr_state) { ++ __atomic_store_n(&hw->reset.disable_cmd, 1, __ATOMIC_RELAXED); ++ new_req = HNS3_IMP_RESET; ++ } else if (BIT(HNS3_VECTOR0_GLOBALRESET_INT_B) & vector0_intr_state) { ++ __atomic_store_n(&hw->reset.disable_cmd, 1, __ATOMIC_RELAXED); ++ new_req = HNS3_GLOBAL_RESET; ++ } ++ ++ if (new_req == HNS3_NONE_RESET) ++ return HNS3_NONE_RESET; ++ ++ if (last_req == HNS3_NONE_RESET || last_req < new_req) { ++ hns3_schedule_delayed_reset(hns); ++ hns3_warn(hw, "High level reset detected, delay do reset"); ++ } ++ ++ return new_req; ++} ++ + bool + hns3_is_reset_pending(struct hns3_adapter *hns) + { ++ enum hns3_reset_level new_req; + struct hns3_hw *hw = &hns->hw; +- enum hns3_reset_level reset; ++ enum hns3_reset_level last_req; + + /* +- * Check the registers to confirm whether there is reset pending. +- * Note: This check may lead to schedule reset task, but only primary +- * process can process the reset event. Therefore, limit the +- * checking under only primary process. ++ * Only primary can process can process the reset event, ++ * so don't check reset event in secondary. + */ +- if (rte_eal_process_type() == RTE_PROC_PRIMARY) +- hns3_check_event_cause(hns, NULL); ++ if (rte_eal_process_type() != RTE_PROC_PRIMARY) ++ return false; + +- reset = hns3_get_reset_level(hns, &hw->reset.pending); +- if (reset != HNS3_NONE_RESET && hw->reset.level != HNS3_NONE_RESET && +- hw->reset.level < reset) { +- hns3_warn(hw, "High level reset %d is pending", reset); ++ new_req = hns3_detect_reset_event(hw); ++ last_req = hns3_get_reset_level(hns, &hw->reset.pending); ++ if (last_req != HNS3_NONE_RESET && new_req != HNS3_NONE_RESET && ++ new_req < last_req) { ++ hns3_warn(hw, "High level reset %d is pending", last_req); + return true; + } +- reset = hns3_get_reset_level(hns, &hw->reset.request); +- if (reset != HNS3_NONE_RESET && hw->reset.level != HNS3_NONE_RESET && +- hw->reset.level < reset) { +- hns3_warn(hw, "High level reset %d is request", reset); ++ last_req = hns3_get_reset_level(hns, &hw->reset.request); ++ if (last_req != HNS3_NONE_RESET && hw->reset.level != HNS3_NONE_RESET && ++ hw->reset.level < last_req) { ++ hns3_warn(hw, "High level reset %d is request", last_req); + return true; + } + return false; +@@ -5662,17 +5676,6 @@ hns3_func_reset_cmd(struct hns3_hw *hw, int func_id) + return hns3_cmd_send(hw, &desc, 1); + } + +-static int +-hns3_imp_reset_cmd(struct hns3_hw *hw) +-{ +- struct hns3_cmd_desc desc; +- +- hns3_cmd_setup_basic_desc(&desc, 0xFFFE, false); +- desc.data[0] = 0xeedd; +- +- return hns3_cmd_send(hw, &desc, 1); +-} +- + static void + hns3_msix_process(struct hns3_adapter *hns, enum hns3_reset_level reset_level) + { +@@ -5690,7 +5693,9 @@ hns3_msix_process(struct hns3_adapter *hns, enum hns3_reset_level reset_level) + + switch (reset_level) { + case HNS3_IMP_RESET: +- hns3_imp_reset_cmd(hw); ++ val = hns3_read_dev(hw, HNS3_VECTOR0_OTER_EN_REG); ++ hns3_set_bit(val, HNS3_VECTOR0_TRIGGER_IMP_RESET_B, 1); ++ hns3_write_dev(hw, HNS3_VECTOR0_OTER_EN_REG, val); + hns3_warn(hw, "IMP Reset requested time=%ld.%.6ld", + tv.tv_sec, tv.tv_usec); + break; +@@ -5815,12 +5820,7 @@ hns3_stop_service(struct hns3_adapter *hns) + rte_eal_alarm_cancel(hns3_service_handler, eth_dev); + hns3_update_linkstatus_and_event(hw, false); + } +- +- hns3_set_rxtx_function(eth_dev); +- rte_wmb(); +- /* Disable datapath on secondary process. */ +- hns3_mp_req_stop_rxtx(eth_dev); +- rte_delay_ms(hw->cfg_max_queues); ++ hns3_stop_rxtx_datapath(eth_dev); + + rte_spinlock_lock(&hw->lock); + if (hns->hw.adapter_state == HNS3_NIC_STARTED || +@@ -5853,8 +5853,7 @@ hns3_start_service(struct hns3_adapter *hns) + hw->reset.level == HNS3_GLOBAL_RESET) + hns3_set_rst_done(hw); + eth_dev = &rte_eth_devices[hw->data->port_id]; +- hns3_set_rxtx_function(eth_dev); +- hns3_mp_req_start_rxtx(eth_dev); ++ hns3_start_rxtx_datapath(eth_dev); + if (hw->adapter_state == HNS3_NIC_STARTED) { + /* + * This API parent function already hold the hns3_hw.lock, the +@@ -6003,56 +6002,27 @@ hns3_reset_service(void *param) + hns3_msix_process(hns, reset_level); + } + +-static unsigned int +-hns3_get_speed_capa_num(uint16_t device_id) ++static uint32_t ++hns3_get_speed_fec_capa(struct rte_eth_fec_capa *speed_fec_capa, ++ uint32_t speed_capa) + { +- unsigned int num; +- +- switch (device_id) { +- case HNS3_DEV_ID_25GE: +- case HNS3_DEV_ID_25GE_RDMA: +- num = 2; +- break; +- case HNS3_DEV_ID_100G_RDMA_MACSEC: +- case HNS3_DEV_ID_200G_RDMA: +- num = 1; +- break; +- default: +- num = 0; +- break; +- } ++ uint32_t speed_bit; ++ uint32_t num = 0; ++ uint32_t i; + +- return num; +-} ++ for (i = 0; i < RTE_DIM(speed_fec_capa_tbl); i++) { ++ speed_bit = ++ rte_eth_speed_bitflag(speed_fec_capa_tbl[i].speed, ++ RTE_ETH_LINK_FULL_DUPLEX); ++ if ((speed_capa & speed_bit) == 0) ++ continue; + +-static int +-hns3_get_speed_fec_capa(struct rte_eth_fec_capa *speed_fec_capa, +- uint16_t device_id) +-{ +- switch (device_id) { +- case HNS3_DEV_ID_25GE: +- /* fallthrough */ +- case HNS3_DEV_ID_25GE_RDMA: +- speed_fec_capa[0].speed = speed_fec_capa_tbl[1].speed; +- speed_fec_capa[0].capa = speed_fec_capa_tbl[1].capa; +- +- /* In HNS3 device, the 25G NIC is compatible with 10G rate */ +- speed_fec_capa[1].speed = speed_fec_capa_tbl[0].speed; +- speed_fec_capa[1].capa = speed_fec_capa_tbl[0].capa; +- break; +- case HNS3_DEV_ID_100G_RDMA_MACSEC: +- speed_fec_capa[0].speed = speed_fec_capa_tbl[4].speed; +- speed_fec_capa[0].capa = speed_fec_capa_tbl[4].capa; +- break; +- case HNS3_DEV_ID_200G_RDMA: +- speed_fec_capa[0].speed = speed_fec_capa_tbl[5].speed; +- speed_fec_capa[0].capa = speed_fec_capa_tbl[5].capa; +- break; +- default: +- return -ENOTSUP; ++ speed_fec_capa[num].speed = speed_fec_capa_tbl[i].speed; ++ speed_fec_capa[num].capa = speed_fec_capa_tbl[i].capa; ++ num++; + } + +- return 0; ++ return num; + } + + static int +@@ -6061,28 +6031,28 @@ hns3_fec_get_capability(struct rte_eth_dev *dev, + unsigned int num) + { + struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private); +- struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); +- uint16_t device_id = pci_dev->id.device_id; +- unsigned int capa_num; +- int ret; ++ unsigned int speed_num; ++ uint32_t speed_capa; + +- capa_num = hns3_get_speed_capa_num(device_id); +- if (capa_num == 0) { +- hns3_err(hw, "device(0x%x) is not supported by hns3 PMD", +- device_id); ++ speed_capa = hns3_get_speed_capa(hw); ++ /* speed_num counts number of speed capabilities */ ++ speed_num = __builtin_popcount(speed_capa & HNS3_SPEEDS_SUPP_FEC); ++ if (speed_num == 0) + return -ENOTSUP; +- } + +- if (speed_fec_capa == NULL || num < capa_num) +- return capa_num; ++ if (speed_fec_capa == NULL) ++ return speed_num; + +- ret = hns3_get_speed_fec_capa(speed_fec_capa, device_id); +- if (ret) +- return -ENOTSUP; ++ if (num < speed_num) { ++ hns3_err(hw, "not enough array size(%u) to store FEC capabilities, should not be less than %u", ++ num, speed_num); ++ return -EINVAL; ++ } + +- return capa_num; ++ return hns3_get_speed_fec_capa(speed_fec_capa, speed_capa); + } + ++ + static int + get_current_fec_auto_state(struct hns3_hw *hw, uint8_t *state) + { +@@ -6220,61 +6190,27 @@ hns3_set_fec_hw(struct hns3_hw *hw, uint32_t mode) + } + + static uint32_t +-get_current_speed_fec_cap(struct hns3_hw *hw, struct rte_eth_fec_capa *fec_capa) ++hns3_get_current_speed_fec_cap(struct hns3_mac *mac) + { +- struct hns3_mac *mac = &hw->mac; +- uint32_t cur_capa; ++ uint32_t i; + +- switch (mac->link_speed) { +- case RTE_ETH_SPEED_NUM_10G: +- cur_capa = fec_capa[1].capa; +- break; +- case RTE_ETH_SPEED_NUM_25G: +- case RTE_ETH_SPEED_NUM_100G: +- case RTE_ETH_SPEED_NUM_200G: +- cur_capa = fec_capa[0].capa; +- break; +- default: +- cur_capa = 0; +- break; ++ for (i = 0; i < RTE_DIM(speed_fec_capa_tbl); i++) { ++ if (mac->link_speed == speed_fec_capa_tbl[i].speed) ++ return speed_fec_capa_tbl[i].capa; + } + +- return cur_capa; +-} +- +-static bool +-is_fec_mode_one_bit_set(uint32_t mode) +-{ +- int cnt = 0; +- uint8_t i; +- +- for (i = 0; i < sizeof(mode); i++) +- if (mode >> i & 0x1) +- cnt++; +- +- return cnt == 1 ? true : false; ++ return 0; + } + + static int +-hns3_fec_set(struct rte_eth_dev *dev, uint32_t mode) ++hns3_fec_mode_valid(struct rte_eth_dev *dev, uint32_t mode) + { +-#define FEC_CAPA_NUM 2 + struct hns3_adapter *hns = dev->data->dev_private; + struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(hns); +- struct hns3_pf *pf = &hns->pf; +- struct rte_eth_fec_capa fec_capa[FEC_CAPA_NUM]; + uint32_t cur_capa; +- uint32_t num = FEC_CAPA_NUM; +- int ret; + +- ret = hns3_fec_get_capability(dev, fec_capa, num); +- if (ret < 0) +- return ret; +- +- /* HNS3 PMD only support one bit set mode, e.g. 0x1, 0x4 */ +- if (!is_fec_mode_one_bit_set(mode)) { +- hns3_err(hw, "FEC mode(0x%x) not supported in HNS3 PMD, " +- "FEC mode should be only one bit set", mode); ++ if (__builtin_popcount(mode) != 1) { ++ hns3_err(hw, "FEC mode(0x%x) should be only one bit set", mode); + return -EINVAL; + } + +@@ -6282,12 +6218,27 @@ hns3_fec_set(struct rte_eth_dev *dev, uint32_t mode) + * Check whether the configured mode is within the FEC capability. + * If not, the configured mode will not be supported. + */ +- cur_capa = get_current_speed_fec_cap(hw, fec_capa); +- if (!(cur_capa & mode)) { +- hns3_err(hw, "unsupported FEC mode = 0x%x", mode); ++ cur_capa = hns3_get_current_speed_fec_cap(&hw->mac); ++ if ((cur_capa & mode) == 0) { ++ hns3_err(hw, "unsupported FEC mode(0x%x)", mode); + return -EINVAL; + } + ++ return 0; ++} ++ ++static int ++hns3_fec_set(struct rte_eth_dev *dev, uint32_t mode) ++{ ++ struct hns3_adapter *hns = dev->data->dev_private; ++ struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(hns); ++ struct hns3_pf *pf = &hns->pf; ++ int ret; ++ ++ ret = hns3_fec_mode_valid(dev, mode); ++ if (ret != 0) ++ return ret; ++ + rte_spinlock_lock(&hw->lock); + ret = hns3_set_fec_hw(hw, mode); + if (ret) { +@@ -6342,7 +6293,7 @@ hns3_optical_module_existed(struct hns3_hw *hw) + ret = hns3_cmd_send(hw, &desc, 1); + if (ret) { + hns3_err(hw, +- "fail to get optical module exist state, ret = %d.\n", ++ "fail to get optical module exist state, ret = %d.", + ret); + return false; + } +@@ -6380,7 +6331,7 @@ hns3_get_module_eeprom_data(struct hns3_hw *hw, uint32_t offset, + + ret = hns3_cmd_send(hw, desc, HNS3_SFP_INFO_CMD_NUM); + if (ret) { +- hns3_err(hw, "fail to get module EEPROM info, ret = %d.\n", ++ hns3_err(hw, "fail to get module EEPROM info, ret = %d.", + ret); + return ret; + } +@@ -6417,7 +6368,7 @@ hns3_get_module_eeprom(struct rte_eth_dev *dev, + return -ENOTSUP; + + if (!hns3_optical_module_existed(hw)) { +- hns3_err(hw, "fail to read module EEPROM: no module is connected.\n"); ++ hns3_err(hw, "fail to read module EEPROM: no module is connected."); + return -EIO; + } + +@@ -6480,7 +6431,7 @@ hns3_get_module_info(struct rte_eth_dev *dev, + modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8636_MAX_LEN; + break; + default: +- hns3_err(hw, "unknown module, type = %u, extra_type = %u.\n", ++ hns3_err(hw, "unknown module, type = %u, extra_type = %u.", + sfp_type.type, sfp_type.ext_type); + return -EINVAL; + } +diff --git a/dpdk/drivers/net/hns3/hns3_ethdev.h b/dpdk/drivers/net/hns3/hns3_ethdev.h +index 2457754b3d..9e67e93d3f 100644 +--- a/dpdk/drivers/net/hns3/hns3_ethdev.h ++++ b/dpdk/drivers/net/hns3/hns3_ethdev.h +@@ -871,13 +871,6 @@ struct hns3_adapter { + struct hns3_ptype_table ptype_tbl __rte_cache_aligned; + }; + +-#define HNS3_DEVARG_RX_FUNC_HINT "rx_func_hint" +-#define HNS3_DEVARG_TX_FUNC_HINT "tx_func_hint" +- +-#define HNS3_DEVARG_DEV_CAPS_MASK "dev_caps_mask" +- +-#define HNS3_DEVARG_MBX_TIME_LIMIT_MS "mbx_time_limit_ms" +- + enum hns3_dev_cap { + HNS3_DEV_SUPPORT_DCB_B, + HNS3_DEV_SUPPORT_COPPER_B, +@@ -891,6 +884,7 @@ enum hns3_dev_cap { + HNS3_DEV_SUPPORT_RAS_IMP_B, + HNS3_DEV_SUPPORT_TM_B, + HNS3_DEV_SUPPORT_VF_VLAN_FLT_MOD_B, ++ HNS3_DEV_SUPPORT_GRO_B, + }; + + #define hns3_dev_get_support(hw, _name) \ +@@ -996,15 +990,6 @@ static inline uint32_t hns3_read_reg(void *base, uint32_t reg) + #define hns3_read_dev(a, reg) \ + hns3_read_reg((a)->io_base, (reg)) + +-#define NEXT_ITEM_OF_ACTION(act, actions, index) \ +- do { \ +- (act) = (actions) + (index); \ +- while ((act)->type == RTE_FLOW_ACTION_TYPE_VOID) { \ +- (index)++; \ +- (act) = (actions) + (index); \ +- } \ +- } while (0) +- + static inline uint64_t + hns3_atomic_test_bit(unsigned int nr, volatile uint64_t *addr) + { +@@ -1045,22 +1030,9 @@ void hns3_update_linkstatus_and_event(struct hns3_hw *hw, bool query); + void hns3vf_update_link_status(struct hns3_hw *hw, uint8_t link_status, + uint32_t link_speed, uint8_t link_duplex); + void hns3vf_update_push_lsc_cap(struct hns3_hw *hw, bool supported); ++void hns3_clear_reset_event(struct hns3_hw *hw); ++void hns3vf_clear_reset_event(struct hns3_hw *hw); + +-int hns3_restore_ptp(struct hns3_adapter *hns); +-int hns3_mbuf_dyn_rx_timestamp_register(struct rte_eth_dev *dev, +- struct rte_eth_conf *conf); +-int hns3_ptp_init(struct hns3_hw *hw); +-int hns3_timesync_enable(struct rte_eth_dev *dev); +-int hns3_timesync_disable(struct rte_eth_dev *dev); +-int hns3_timesync_read_rx_timestamp(struct rte_eth_dev *dev, +- struct timespec *timestamp, +- uint32_t flags __rte_unused); +-int hns3_timesync_read_tx_timestamp(struct rte_eth_dev *dev, +- struct timespec *timestamp); +-int hns3_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts); +-int hns3_timesync_write_time(struct rte_eth_dev *dev, +- const struct timespec *ts); +-int hns3_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta); + + static inline bool + is_reset_pending(struct hns3_adapter *hns) +@@ -1073,4 +1045,15 @@ is_reset_pending(struct hns3_adapter *hns) + return ret; + } + ++static inline void ++hns3_clear_reset_status(struct hns3_hw *hw) ++{ ++ struct hns3_adapter *hns = HNS3_DEV_HW_TO_ADAPTER(hw); ++ ++ if (hns->is_vf) ++ hns3vf_clear_reset_event(hw); ++ else ++ hns3_clear_reset_event(hw); ++} ++ + #endif /* HNS3_ETHDEV_H */ +diff --git a/dpdk/drivers/net/hns3/hns3_ethdev_vf.c b/dpdk/drivers/net/hns3/hns3_ethdev_vf.c +index d220522c43..702a61aad9 100644 +--- a/dpdk/drivers/net/hns3/hns3_ethdev_vf.c ++++ b/dpdk/drivers/net/hns3/hns3_ethdev_vf.c +@@ -250,6 +250,8 @@ hns3vf_set_default_mac_addr(struct rte_eth_dev *dev, + hns3_err(hw, "Failed to set mac addr(%s) for vf: %d", + mac_str, ret); + } ++ rte_spinlock_unlock(&hw->lock); ++ return ret; + } + + rte_ether_addr_copy(mac_addr, +@@ -610,6 +612,19 @@ hns3vf_enable_irq0(struct hns3_hw *hw) + hns3_write_dev(hw, HNS3_MISC_VECTOR_REG_BASE, 1); + } + ++void ++hns3vf_clear_reset_event(struct hns3_hw *hw) ++{ ++ uint32_t clearval; ++ uint32_t cmdq_stat_reg; ++ ++ cmdq_stat_reg = hns3_read_dev(hw, HNS3_VECTOR0_CMDQ_STAT_REG); ++ clearval = cmdq_stat_reg & ~BIT(HNS3_VECTOR0_RST_INT_B); ++ hns3_write_dev(hw, HNS3_VECTOR0_CMDQ_SRC_REG, clearval); ++ ++ hns3vf_enable_irq0(hw); ++} ++ + static enum hns3vf_evt_cause + hns3vf_check_event_cause(struct hns3_adapter *hns, uint32_t *clearval) + { +@@ -684,69 +699,10 @@ hns3vf_interrupt_handler(void *param) + break; + } + +- /* Enable interrupt */ +- hns3vf_enable_irq0(hw); +-} +- +-static void +-hns3vf_set_default_dev_specifications(struct hns3_hw *hw) +-{ +- hw->max_non_tso_bd_num = HNS3_MAX_NON_TSO_BD_PER_PKT; +- hw->rss_ind_tbl_size = HNS3_RSS_IND_TBL_SIZE; +- hw->rss_key_size = HNS3_RSS_KEY_SIZE; +- hw->intr.int_ql_max = HNS3_INTR_QL_NONE; +-} +- +-static void +-hns3vf_parse_dev_specifications(struct hns3_hw *hw, struct hns3_cmd_desc *desc) +-{ +- struct hns3_dev_specs_0_cmd *req0; +- struct hns3_dev_specs_1_cmd *req1; +- +- req0 = (struct hns3_dev_specs_0_cmd *)desc[0].data; +- req1 = (struct hns3_dev_specs_1_cmd *)desc[1].data; +- +- hw->max_non_tso_bd_num = req0->max_non_tso_bd_num; +- hw->rss_ind_tbl_size = rte_le_to_cpu_16(req0->rss_ind_tbl_size); +- hw->rss_key_size = rte_le_to_cpu_16(req0->rss_key_size); +- hw->intr.int_ql_max = rte_le_to_cpu_16(req0->intr_ql_max); +- hw->min_tx_pkt_len = req1->min_tx_pkt_len; +-} +- +-static int +-hns3vf_check_dev_specifications(struct hns3_hw *hw) +-{ +- if (hw->rss_ind_tbl_size == 0 || +- hw->rss_ind_tbl_size > HNS3_RSS_IND_TBL_SIZE_MAX) { +- hns3_warn(hw, "the size of hash lookup table configured (%u) exceeds the maximum(%u)", +- hw->rss_ind_tbl_size, HNS3_RSS_IND_TBL_SIZE_MAX); +- return -EINVAL; +- } +- +- return 0; +-} +- +-static int +-hns3vf_query_dev_specifications(struct hns3_hw *hw) +-{ +- struct hns3_cmd_desc desc[HNS3_QUERY_DEV_SPECS_BD_NUM]; +- int ret; +- int i; +- +- for (i = 0; i < HNS3_QUERY_DEV_SPECS_BD_NUM - 1; i++) { +- hns3_cmd_setup_basic_desc(&desc[i], HNS3_OPC_QUERY_DEV_SPECS, +- true); +- desc[i].flag |= rte_cpu_to_le_16(HNS3_CMD_FLAG_NEXT); +- } +- hns3_cmd_setup_basic_desc(&desc[i], HNS3_OPC_QUERY_DEV_SPECS, true); +- +- ret = hns3_cmd_send(hw, desc, HNS3_QUERY_DEV_SPECS_BD_NUM); +- if (ret) +- return ret; +- +- hns3vf_parse_dev_specifications(hw, desc); +- +- return hns3vf_check_dev_specifications(hw); ++ /* Enable interrupt if it is not caused by reset */ ++ if (event_cause == HNS3VF_VECTOR0_EVENT_MBX || ++ event_cause == HNS3VF_VECTOR0_EVENT_OTHER) ++ hns3vf_enable_irq0(hw); + } + + void +@@ -821,12 +777,8 @@ hns3vf_get_capability(struct hns3_hw *hw) + { + int ret; + +- ret = hns3_get_pci_revision_id(hw, &hw->revision); +- if (ret) +- return ret; +- + if (hw->revision < PCI_REVISION_ID_HIP09_A) { +- hns3vf_set_default_dev_specifications(hw); ++ hns3_set_default_dev_specifications(hw); + hw->intr.mapping_mode = HNS3_INTR_MAPPING_VEC_RSV_ONE; + hw->intr.gl_unit = HNS3_INTR_COALESCE_GL_UINT_2US; + hw->tso_mode = HNS3_TSO_SW_CAL_PSEUDO_H_CSUM; +@@ -837,7 +789,7 @@ hns3vf_get_capability(struct hns3_hw *hw) + return 0; + } + +- ret = hns3vf_query_dev_specifications(hw); ++ ret = hns3_query_dev_specifications(hw); + if (ret) { + PMD_INIT_LOG(ERR, + "failed to query dev specifications, ret = %d", +@@ -1477,6 +1429,10 @@ hns3vf_init_vf(struct rte_eth_dev *eth_dev) + /* Get hardware io base address from pcie BAR2 IO space */ + hw->io_base = pci_dev->mem_resource[2].addr; + ++ ret = hns3_get_pci_revision_id(hw, &hw->revision); ++ if (ret) ++ return ret; ++ + /* Firmware command queue initialize */ + ret = hns3_cmd_init_queue(hw); + if (ret) { +@@ -1633,12 +1589,7 @@ hns3vf_dev_stop(struct rte_eth_dev *dev) + dev->data->dev_started = 0; + + hw->adapter_state = HNS3_NIC_STOPPING; +- hns3_set_rxtx_function(dev); +- rte_wmb(); +- /* Disable datapath on secondary process. */ +- hns3_mp_req_stop_rxtx(dev); +- /* Prevent crashes when queues are still in use. */ +- rte_delay_ms(hw->cfg_max_queues); ++ hns3_stop_rxtx_datapath(dev); + + rte_spinlock_lock(&hw->lock); + if (__atomic_load_n(&hw->reset.resetting, __ATOMIC_RELAXED) == 0) { +@@ -1740,8 +1691,10 @@ hns3vf_do_start(struct hns3_adapter *hns, bool reset_queue) + hns3_enable_rxd_adv_layout(hw); + + ret = hns3_init_queues(hns, reset_queue); +- if (ret) ++ if (ret) { + hns3_err(hw, "failed to init queues, ret = %d.", ret); ++ return ret; ++ } + + return hns3_restore_filter(hns); + } +@@ -1792,8 +1745,7 @@ hns3vf_dev_start(struct rte_eth_dev *dev) + rte_spinlock_unlock(&hw->lock); + + hns3_rx_scattered_calc(dev); +- hns3_set_rxtx_function(dev); +- hns3_mp_req_start_rxtx(dev); ++ hns3_start_rxtx_datapath(dev); + + /* Enable interrupt of all rx queues before enabling queues */ + hns3_dev_all_rx_queue_intr_enable(hw, true); +@@ -1859,14 +1811,13 @@ hns3vf_is_reset_pending(struct hns3_adapter *hns) + return false; + + /* +- * Check the registers to confirm whether there is reset pending. +- * Note: This check may lead to schedule reset task, but only primary +- * process can process the reset event. Therefore, limit the +- * checking under only primary process. ++ * Only primary can process can process the reset event, ++ * so don't check reset event in secondary. + */ +- if (rte_eal_process_type() == RTE_PROC_PRIMARY) +- hns3vf_check_event_cause(hns, NULL); ++ if (rte_eal_process_type() != RTE_PROC_PRIMARY) ++ return false; + ++ hns3vf_check_event_cause(hns, NULL); + reset = hns3vf_get_reset_level(hw, &hw->reset.pending); + if (hw->reset.level != HNS3_NONE_RESET && reset != HNS3_NONE_RESET && + hw->reset.level < reset) { +@@ -1963,11 +1914,7 @@ hns3vf_stop_service(struct hns3_adapter *hns) + } + hw->mac.link_status = RTE_ETH_LINK_DOWN; + +- hns3_set_rxtx_function(eth_dev); +- rte_wmb(); +- /* Disable datapath on secondary process. */ +- hns3_mp_req_stop_rxtx(eth_dev); +- rte_delay_ms(hw->cfg_max_queues); ++ hns3_stop_rxtx_datapath(eth_dev); + + rte_spinlock_lock(&hw->lock); + if (hw->adapter_state == HNS3_NIC_STARTED || +@@ -1999,8 +1946,7 @@ hns3vf_start_service(struct hns3_adapter *hns) + struct rte_eth_dev *eth_dev; + + eth_dev = &rte_eth_devices[hw->data->port_id]; +- hns3_set_rxtx_function(eth_dev); +- hns3_mp_req_start_rxtx(eth_dev); ++ hns3_start_rxtx_datapath(eth_dev); + + rte_eal_alarm_set(HNS3VF_KEEP_ALIVE_INTERVAL, hns3vf_keep_alive_handler, + eth_dev); +@@ -2231,8 +2177,11 @@ hns3vf_reinit_dev(struct hns3_adapter *hns) + */ + if (pci_dev->kdrv == RTE_PCI_KDRV_IGB_UIO || + pci_dev->kdrv == RTE_PCI_KDRV_UIO_GENERIC) { +- if (hns3vf_enable_msix(pci_dev, true)) ++ ret = hns3vf_enable_msix(pci_dev, true); ++ if (ret != 0) { + hns3_err(hw, "Failed to enable msix"); ++ return ret; ++ } + } + + rte_intr_enable(pci_dev->intr_handle); +diff --git a/dpdk/drivers/net/hns3/hns3_fdir.c b/dpdk/drivers/net/hns3/hns3_fdir.c +index 48a91fb517..a3e79619ec 100644 +--- a/dpdk/drivers/net/hns3/hns3_fdir.c ++++ b/dpdk/drivers/net/hns3/hns3_fdir.c +@@ -974,7 +974,7 @@ int hns3_fdir_filter_program(struct hns3_adapter *hns, + rule->key_conf.spec.src_port, + rule->key_conf.spec.dst_port, ret); + else +- hns3_remove_fdir_filter(hw, fdir_info, &rule->key_conf); ++ ret = hns3_remove_fdir_filter(hw, fdir_info, &rule->key_conf); + + return ret; + } +diff --git a/dpdk/drivers/net/hns3/hns3_flow.c b/dpdk/drivers/net/hns3/hns3_flow.c +index a2c1589c39..da17fa6e69 100644 +--- a/dpdk/drivers/net/hns3/hns3_flow.c ++++ b/dpdk/drivers/net/hns3/hns3_flow.c +@@ -10,6 +10,125 @@ + #include "hns3_logs.h" + #include "hns3_flow.h" + ++#define NEXT_ITEM_OF_ACTION(act, actions, index) \ ++ do { \ ++ (act) = (actions) + (index); \ ++ while ((act)->type == RTE_FLOW_ACTION_TYPE_VOID) { \ ++ (index)++; \ ++ (act) = (actions) + (index); \ ++ } \ ++ } while (0) ++ ++#define NEXT_ITEM_OF_PATTERN(item, pattern, index) \ ++ do { \ ++ (item) = (pattern) + (index); \ ++ while ((item)->type == RTE_FLOW_ITEM_TYPE_VOID) { \ ++ (index)++; \ ++ (item) = (pattern) + (index); \ ++ } \ ++ } while (0) ++ ++#define HNS3_HASH_HDR_ETH RTE_BIT64(0) ++#define HNS3_HASH_HDR_IPV4 RTE_BIT64(1) ++#define HNS3_HASH_HDR_IPV6 RTE_BIT64(2) ++#define HNS3_HASH_HDR_TCP RTE_BIT64(3) ++#define HNS3_HASH_HDR_UDP RTE_BIT64(4) ++#define HNS3_HASH_HDR_SCTP RTE_BIT64(5) ++ ++#define HNS3_HASH_VOID_NEXT_ALLOW BIT_ULL(RTE_FLOW_ITEM_TYPE_ETH) ++ ++#define HNS3_HASH_ETH_NEXT_ALLOW (BIT_ULL(RTE_FLOW_ITEM_TYPE_IPV4) | \ ++ BIT_ULL(RTE_FLOW_ITEM_TYPE_IPV6)) ++ ++#define HNS3_HASH_IP_NEXT_ALLOW (BIT_ULL(RTE_FLOW_ITEM_TYPE_TCP) | \ ++ BIT_ULL(RTE_FLOW_ITEM_TYPE_UDP) | \ ++ BIT_ULL(RTE_FLOW_ITEM_TYPE_SCTP)) ++ ++static const uint64_t hash_pattern_next_allow_items[] = { ++ [RTE_FLOW_ITEM_TYPE_VOID] = HNS3_HASH_VOID_NEXT_ALLOW, ++ [RTE_FLOW_ITEM_TYPE_ETH] = HNS3_HASH_ETH_NEXT_ALLOW, ++ [RTE_FLOW_ITEM_TYPE_IPV4] = HNS3_HASH_IP_NEXT_ALLOW, ++ [RTE_FLOW_ITEM_TYPE_IPV6] = HNS3_HASH_IP_NEXT_ALLOW, ++}; ++ ++static const uint64_t hash_pattern_item_header[] = { ++ [RTE_FLOW_ITEM_TYPE_ETH] = HNS3_HASH_HDR_ETH, ++ [RTE_FLOW_ITEM_TYPE_IPV4] = HNS3_HASH_HDR_IPV4, ++ [RTE_FLOW_ITEM_TYPE_IPV6] = HNS3_HASH_HDR_IPV6, ++ [RTE_FLOW_ITEM_TYPE_TCP] = HNS3_HASH_HDR_TCP, ++ [RTE_FLOW_ITEM_TYPE_UDP] = HNS3_HASH_HDR_UDP, ++ [RTE_FLOW_ITEM_TYPE_SCTP] = HNS3_HASH_HDR_SCTP, ++}; ++ ++#define HNS3_HASH_IPV4 (HNS3_HASH_HDR_ETH | HNS3_HASH_HDR_IPV4) ++#define HNS3_HASH_IPV4_TCP (HNS3_HASH_HDR_ETH | \ ++ HNS3_HASH_HDR_IPV4 | \ ++ HNS3_HASH_HDR_TCP) ++#define HNS3_HASH_IPV4_UDP (HNS3_HASH_HDR_ETH | \ ++ HNS3_HASH_HDR_IPV4 | \ ++ HNS3_HASH_HDR_UDP) ++#define HNS3_HASH_IPV4_SCTP (HNS3_HASH_HDR_ETH | \ ++ HNS3_HASH_HDR_IPV4 | \ ++ HNS3_HASH_HDR_SCTP) ++#define HNS3_HASH_IPV6 (HNS3_HASH_HDR_ETH | HNS3_HASH_HDR_IPV6) ++#define HNS3_HASH_IPV6_TCP (HNS3_HASH_HDR_ETH | \ ++ HNS3_HASH_HDR_IPV6 | \ ++ HNS3_HASH_HDR_TCP) ++#define HNS3_HASH_IPV6_UDP (HNS3_HASH_HDR_ETH | \ ++ HNS3_HASH_HDR_IPV6 | \ ++ HNS3_HASH_HDR_UDP) ++#define HNS3_HASH_IPV6_SCTP (HNS3_HASH_HDR_ETH | \ ++ HNS3_HASH_HDR_IPV6 | \ ++ HNS3_HASH_HDR_SCTP) ++ ++static const struct hns3_hash_map_info { ++ /* flow type specified, zero means action works for all flow types. */ ++ uint64_t pattern_type; ++ uint64_t rss_pctype; /* packet type with prefix RTE_ETH_RSS_xxx */ ++ uint64_t l3l4_types; /* Supported L3/L4 RSS types for this packet type */ ++ uint64_t hw_pctype; /* packet type in driver */ ++ uint64_t tuple_mask; /* full tuples of the hw_pctype */ ++} hash_map_table[] = { ++ /* IPV4 */ ++ { HNS3_HASH_IPV4, ++ RTE_ETH_RSS_IPV4, HNS3_RSS_SUPPORT_L3_SRC_DST, ++ HNS3_RSS_PCTYPE_IPV4_NONF, HNS3_RSS_TUPLE_IPV4_NONF_M }, ++ { HNS3_HASH_IPV4, ++ RTE_ETH_RSS_NONFRAG_IPV4_OTHER, HNS3_RSS_SUPPORT_L3_SRC_DST, ++ HNS3_RSS_PCTYPE_IPV4_NONF, HNS3_RSS_TUPLE_IPV4_NONF_M }, ++ { HNS3_HASH_IPV4, ++ RTE_ETH_RSS_FRAG_IPV4, HNS3_RSS_SUPPORT_L3_SRC_DST, ++ HNS3_RSS_PCTYPE_IPV4_FLAG, HNS3_RSS_TUPLE_IPV4_FLAG_M }, ++ { HNS3_HASH_IPV4_TCP, ++ RTE_ETH_RSS_NONFRAG_IPV4_TCP, HNS3_RSS_SUPPORT_L3L4, ++ HNS3_RSS_PCTYPE_IPV4_TCP, HNS3_RSS_TUPLE_IPV4_TCP_M }, ++ { HNS3_HASH_IPV4_UDP, ++ RTE_ETH_RSS_NONFRAG_IPV4_UDP, HNS3_RSS_SUPPORT_L3L4, ++ HNS3_RSS_PCTYPE_IPV4_UDP, HNS3_RSS_TUPLE_IPV4_UDP_M }, ++ { HNS3_HASH_IPV4_SCTP, ++ RTE_ETH_RSS_NONFRAG_IPV4_SCTP, HNS3_RSS_SUPPORT_L3L4, ++ HNS3_RSS_PCTYPE_IPV4_SCTP, HNS3_RSS_TUPLE_IPV4_SCTP_M }, ++ /* IPV6 */ ++ { HNS3_HASH_IPV6, ++ RTE_ETH_RSS_IPV6, HNS3_RSS_SUPPORT_L3_SRC_DST, ++ HNS3_RSS_PCTYPE_IPV6_NONF, HNS3_RSS_TUPLE_IPV6_NONF_M }, ++ { HNS3_HASH_IPV6, ++ RTE_ETH_RSS_NONFRAG_IPV6_OTHER, HNS3_RSS_SUPPORT_L3_SRC_DST, ++ HNS3_RSS_PCTYPE_IPV6_NONF, HNS3_RSS_TUPLE_IPV6_NONF_M }, ++ { HNS3_HASH_IPV6, ++ RTE_ETH_RSS_FRAG_IPV6, HNS3_RSS_SUPPORT_L3_SRC_DST, ++ HNS3_RSS_PCTYPE_IPV6_FLAG, HNS3_RSS_TUPLE_IPV6_FLAG_M }, ++ { HNS3_HASH_IPV6_TCP, ++ RTE_ETH_RSS_NONFRAG_IPV6_TCP, HNS3_RSS_SUPPORT_L3L4, ++ HNS3_RSS_PCTYPE_IPV6_TCP, HNS3_RSS_TUPLE_IPV6_TCP_M }, ++ { HNS3_HASH_IPV6_UDP, ++ RTE_ETH_RSS_NONFRAG_IPV6_UDP, HNS3_RSS_SUPPORT_L3L4, ++ HNS3_RSS_PCTYPE_IPV6_UDP, HNS3_RSS_TUPLE_IPV6_UDP_M }, ++ { HNS3_HASH_IPV6_SCTP, ++ RTE_ETH_RSS_NONFRAG_IPV6_SCTP, HNS3_RSS_SUPPORT_L3L4, ++ HNS3_RSS_PCTYPE_IPV6_SCTP, HNS3_RSS_TUPLE_IPV6_SCTP_M }, ++}; ++ + static const uint8_t full_mask[VNI_OR_TNI_LEN] = { 0xFF, 0xFF, 0xFF }; + static const uint8_t zero_mask[VNI_OR_TNI_LEN] = { 0x00, 0x00, 0x00 }; + +@@ -79,7 +198,7 @@ net_addr_to_host(uint32_t *dst, const rte_be32_t *src, size_t len) + } + + /* +- * This function is used to find rss general action. ++ * This function is used to parse filter type. + * 1. As we know RSS is used to spread packets among several queues, the flow + * API provide the struct rte_flow_action_rss, user could config its field + * sush as: func/level/types/key/queue to control RSS function. +@@ -87,16 +206,18 @@ net_addr_to_host(uint32_t *dst, const rte_be32_t *src, size_t len) + * implemented by FDIR + RSS in hns3 hardware, user can create one FDIR rule + * which action is RSS queues region. + * 3. When action is RSS, we use the following rule to distinguish: +- * Case 1: pattern have ETH and action's queue_num > 0, indicate it is queue +- * region configuration. ++ * Case 1: pattern has ETH and all fields in RSS action except 'queues' are ++ * zero or default, indicate it is queue region configuration. + * Case other: an rss general action. + */ +-static const struct rte_flow_action * +-hns3_find_rss_general_action(const struct rte_flow_item pattern[], +- const struct rte_flow_action actions[]) ++static void ++hns3_parse_filter_type(const struct rte_flow_item pattern[], ++ const struct rte_flow_action actions[], ++ struct hns3_filter_info *filter_info) + { ++ const struct rte_flow_action_rss *rss_act; + const struct rte_flow_action *act = NULL; +- const struct hns3_rss_conf *rss; ++ bool only_has_queues = false; + bool have_eth = false; + + for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { +@@ -105,8 +226,10 @@ hns3_find_rss_general_action(const struct rte_flow_item pattern[], + break; + } + } +- if (!act) +- return NULL; ++ if (act == NULL) { ++ filter_info->type = RTE_ETH_FILTER_FDIR; ++ return; ++ } + + for (; pattern->type != RTE_FLOW_ITEM_TYPE_END; pattern++) { + if (pattern->type == RTE_FLOW_ITEM_TYPE_ETH) { +@@ -115,19 +238,21 @@ hns3_find_rss_general_action(const struct rte_flow_item pattern[], + } + } + +- rss = act->conf; +- if (have_eth && rss->conf.queue_num) { ++ rss_act = act->conf; ++ only_has_queues = (rss_act->queue_num > 0) && ++ (rss_act->func == RTE_ETH_HASH_FUNCTION_DEFAULT && ++ rss_act->types == 0 && rss_act->key_len == 0); ++ if (have_eth && only_has_queues) { + /* +- * Pattern have ETH and action's queue_num > 0, indicate this is +- * queue region configuration. +- * Because queue region is implemented by FDIR + RSS in hns3 +- * hardware, it needs to enter FDIR process, so here return NULL +- * to avoid enter RSS process. ++ * Pattern has ETH and all fields in RSS action except 'queues' ++ * are zero or default, which indicates this is queue region ++ * configuration. + */ +- return NULL; ++ filter_info->type = RTE_ETH_FILTER_FDIR; ++ return; + } + +- return act; ++ filter_info->type = RTE_ETH_FILTER_HASH; + } + + static inline struct hns3_flow_counter * +@@ -802,7 +927,7 @@ hns3_parse_sctp(const struct rte_flow_item *item, struct hns3_fdir_rule *rule, + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM_MASK, + item, +- "Only support src & dst port in SCTP"); ++ "Only support src & dst port & v-tag in SCTP"); + if (sctp_mask->hdr.src_port) { + hns3_set_bit(rule->input_set, INNER_SRC_PORT, 1); + rule->key_conf.mask.src_port = +@@ -1246,7 +1371,6 @@ hns3_filterlist_flush(struct rte_eth_dev *dev) + { + struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct hns3_fdir_rule_ele *fdir_rule_ptr; +- struct hns3_rss_conf_ele *rss_filter_ptr; + struct hns3_flow_mem *flow_node; + + fdir_rule_ptr = TAILQ_FIRST(&hw->flow_fdir_list); +@@ -1256,13 +1380,6 @@ hns3_filterlist_flush(struct rte_eth_dev *dev) + fdir_rule_ptr = TAILQ_FIRST(&hw->flow_fdir_list); + } + +- rss_filter_ptr = TAILQ_FIRST(&hw->flow_rss_list); +- while (rss_filter_ptr) { +- TAILQ_REMOVE(&hw->flow_rss_list, rss_filter_ptr, entries); +- rte_free(rss_filter_ptr); +- rss_filter_ptr = TAILQ_FIRST(&hw->flow_rss_list); +- } +- + flow_node = TAILQ_FIRST(&hw->flow_list); + while (flow_node) { + TAILQ_REMOVE(&hw->flow_list, flow_node, entries); +@@ -1273,238 +1390,478 @@ hns3_filterlist_flush(struct rte_eth_dev *dev) + } + + static bool +-hns3_action_rss_same(const struct rte_flow_action_rss *comp, +- const struct rte_flow_action_rss *with) ++hns3_flow_rule_key_same(const struct rte_flow_action_rss *comp, ++ const struct rte_flow_action_rss *with) + { +- bool rss_key_is_same; +- bool func_is_same; ++ if (comp->key_len != with->key_len) ++ return false; + +- /* +- * When user flush all RSS rule, RSS func is set invalid with +- * RTE_ETH_HASH_FUNCTION_MAX. Then the user create a flow after +- * flushed, any validate RSS func is different with it before +- * flushed. Others, when user create an action RSS with RSS func +- * specified RTE_ETH_HASH_FUNCTION_DEFAULT, the func is the same +- * between continuous RSS flow. +- */ +- if (comp->func == RTE_ETH_HASH_FUNCTION_MAX) +- func_is_same = false; +- else +- func_is_same = (with->func != RTE_ETH_HASH_FUNCTION_DEFAULT) ? +- (comp->func == with->func) : true; ++ if (with->key_len == 0) ++ return true; + +- if (with->key_len == 0 || with->key == NULL) +- rss_key_is_same = 1; +- else +- rss_key_is_same = comp->key_len == with->key_len && +- !memcmp(comp->key, with->key, with->key_len); ++ if (comp->key == NULL && with->key == NULL) ++ return true; + +- return (func_is_same && rss_key_is_same && +- comp->types == (with->types & HNS3_ETH_RSS_SUPPORT) && +- comp->level == with->level && +- comp->queue_num == with->queue_num && +- !memcmp(comp->queue, with->queue, +- sizeof(*with->queue) * with->queue_num)); ++ if (!(comp->key != NULL && with->key != NULL)) ++ return false; ++ ++ return !memcmp(comp->key, with->key, with->key_len); + } + +-static int +-hns3_rss_conf_copy(struct hns3_rss_conf *out, +- const struct rte_flow_action_rss *in) ++static bool ++hns3_flow_rule_queues_same(const struct rte_flow_action_rss *comp, ++ const struct rte_flow_action_rss *with) + { +- if (in->key_len > RTE_DIM(out->key) || +- in->queue_num > RTE_DIM(out->queue)) +- return -EINVAL; +- if (in->key == NULL && in->key_len) +- return -EINVAL; +- out->conf = (struct rte_flow_action_rss) { +- .func = in->func, +- .level = in->level, +- .types = in->types, +- .key_len = in->key_len, +- .queue_num = in->queue_num, +- }; +- out->conf.queue = memcpy(out->queue, in->queue, +- sizeof(*in->queue) * in->queue_num); +- if (in->key) +- out->conf.key = memcpy(out->key, in->key, in->key_len); ++ if (comp->queue_num != with->queue_num) ++ return false; + +- return 0; ++ if (with->queue_num == 0) ++ return true; ++ ++ if (comp->queue == NULL && with->queue == NULL) ++ return true; ++ ++ if (!(comp->queue != NULL && with->queue != NULL)) ++ return false; ++ ++ return !memcmp(comp->queue, with->queue, with->queue_num); + } + + static bool +-hns3_rss_input_tuple_supported(struct hns3_hw *hw, +- const struct rte_flow_action_rss *rss) ++hns3_action_rss_same(const struct rte_flow_action_rss *comp, ++ const struct rte_flow_action_rss *with) ++{ ++ bool same_level; ++ bool same_types; ++ bool same_func; ++ ++ same_level = (comp->level == with->level); ++ same_types = (comp->types == with->types); ++ same_func = (comp->func == with->func); ++ ++ return same_level && same_types && same_func && ++ hns3_flow_rule_key_same(comp, with) && ++ hns3_flow_rule_queues_same(comp, with); ++} ++ ++static bool ++hns3_valid_ipv6_sctp_rss_types(struct hns3_hw *hw, uint64_t types) + { + /* +- * For IP packet, it is not supported to use src/dst port fields to RSS +- * hash for the following packet types. +- * - IPV4 FRAG | IPV4 NONFRAG | IPV6 FRAG | IPV6 NONFRAG +- * Besides, for Kunpeng920, the NIC HW is not supported to use src/dst +- * port fields to RSS hash for IPV6 SCTP packet type. However, the +- * Kunpeng930 and future kunpeng series support to use src/dst port +- * fields to RSS hash for IPv6 SCTP packet type. ++ * Some hardware don't support to use src/dst port fields to hash ++ * for IPV6 SCTP packet type. + */ +- if (rss->types & (RTE_ETH_RSS_L4_DST_ONLY | RTE_ETH_RSS_L4_SRC_ONLY) && +- (rss->types & RTE_ETH_RSS_IP || +- (!hw->rss_info.ipv6_sctp_offload_supported && +- rss->types & RTE_ETH_RSS_NONFRAG_IPV6_SCTP))) ++ if (types & RTE_ETH_RSS_NONFRAG_IPV6_SCTP && ++ types & HNS3_RSS_SUPPORT_L4_SRC_DST && ++ !hw->rss_info.ipv6_sctp_offload_supported) + return false; + + return true; + } + +-/* +- * This function is used to parse rss action validation. +- */ + static int +-hns3_parse_rss_filter(struct rte_eth_dev *dev, +- const struct rte_flow_action *actions, +- struct rte_flow_error *error) ++hns3_flow_parse_hash_func(const struct rte_flow_action_rss *rss_act, ++ struct hns3_flow_rss_conf *rss_conf, ++ struct rte_flow_error *error) + { +- struct hns3_adapter *hns = dev->data->dev_private; +- struct hns3_hw *hw = &hns->hw; +- struct hns3_rss_conf *rss_conf = &hw->rss_info; +- const struct rte_flow_action_rss *rss; +- const struct rte_flow_action *act; +- uint32_t act_index = 0; +- uint16_t n; ++ if (rss_act->func >= RTE_ETH_HASH_FUNCTION_MAX) ++ return rte_flow_error_set(error, ENOTSUP, ++ RTE_FLOW_ERROR_TYPE_ACTION_CONF, ++ NULL, "RSS hash func are not supported"); + +- NEXT_ITEM_OF_ACTION(act, actions, act_index); +- rss = act->conf; ++ rss_conf->conf.func = rss_act->func; ++ return 0; ++} + +- if (rss == NULL) { ++static int ++hns3_flow_parse_hash_key(struct hns3_hw *hw, ++ const struct rte_flow_action_rss *rss_act, ++ struct hns3_flow_rss_conf *rss_conf, ++ struct rte_flow_error *error) ++{ ++ if (rss_act->key_len != hw->rss_key_size) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_CONF, +- act, "no valid queues"); +- } ++ NULL, "invalid RSS key length"); ++ ++ if (rss_act->key != NULL) ++ memcpy(rss_conf->key, rss_act->key, rss_act->key_len); ++ else ++ memcpy(rss_conf->key, hns3_hash_key, ++ RTE_MIN(sizeof(hns3_hash_key), rss_act->key_len)); ++ /* Need to record if user sets hash key. */ ++ rss_conf->conf.key = rss_act->key; ++ rss_conf->conf.key_len = rss_act->key_len; ++ ++ return 0; ++} ++ ++static int ++hns3_flow_parse_queues(struct hns3_hw *hw, ++ const struct rte_flow_action_rss *rss_act, ++ struct hns3_flow_rss_conf *rss_conf, ++ struct rte_flow_error *error) ++{ ++ uint16_t i; + +- if (rss->queue_num > RTE_DIM(rss_conf->queue)) ++ if (rss_act->queue_num > hw->rss_ind_tbl_size) + return rte_flow_error_set(error, ENOTSUP, +- RTE_FLOW_ERROR_TYPE_ACTION_CONF, act, +- "queue number configured exceeds " +- "queue buffer size driver supported"); ++ RTE_FLOW_ERROR_TYPE_ACTION_CONF, ++ NULL, ++ "queue number can not exceed RSS indirection table."); + +- for (n = 0; n < rss->queue_num; n++) { +- if (rss->queue[n] < hw->alloc_rss_size) +- continue; +- return rte_flow_error_set(error, EINVAL, +- RTE_FLOW_ERROR_TYPE_ACTION_CONF, act, +- "queue id must be less than queue number allocated to a TC"); ++ if (rss_act->queue_num > HNS3_RSS_QUEUES_BUFFER_NUM) ++ return rte_flow_error_set(error, ENOTSUP, ++ RTE_FLOW_ERROR_TYPE_ACTION_CONF, ++ NULL, ++ "queue number configured exceeds queue buffer size driver supported"); ++ ++ for (i = 0; i < rss_act->queue_num; i++) { ++ if (rss_act->queue[i] >= hw->alloc_rss_size) ++ return rte_flow_error_set(error, EINVAL, ++ RTE_FLOW_ERROR_TYPE_ACTION_CONF, ++ NULL, ++ "queue id must be less than queue number allocated to a TC"); + } + +- if (!(rss->types & HNS3_ETH_RSS_SUPPORT) && rss->types) ++ memcpy(rss_conf->queue, rss_act->queue, ++ rss_act->queue_num * sizeof(rss_conf->queue[0])); ++ rss_conf->conf.queue = rss_conf->queue; ++ rss_conf->conf.queue_num = rss_act->queue_num; ++ ++ return 0; ++} ++ ++static int ++hns3_flow_get_hw_pctype(struct hns3_hw *hw, ++ const struct rte_flow_action_rss *rss_act, ++ const struct hns3_hash_map_info *map, ++ struct hns3_flow_rss_conf *rss_conf, ++ struct rte_flow_error *error) ++{ ++ uint64_t l3l4_src_dst, l3l4_refine, left_types; ++ ++ if (rss_act->types == 0) { ++ /* Disable RSS hash of this packet type if types is zero. */ ++ rss_conf->hw_pctypes |= map->hw_pctype; ++ return 0; ++ } ++ ++ /* ++ * Can not have extra types except rss_pctype and l3l4_type in this map. ++ */ ++ left_types = ~map->rss_pctype & rss_act->types; ++ if (left_types & ~map->l3l4_types) + return rte_flow_error_set(error, EINVAL, +- RTE_FLOW_ERROR_TYPE_ACTION_CONF, +- act, +- "Flow types is unsupported by " +- "hns3's RSS"); +- if (rss->func >= RTE_ETH_HASH_FUNCTION_MAX) +- return rte_flow_error_set(error, ENOTSUP, +- RTE_FLOW_ERROR_TYPE_ACTION_CONF, act, +- "RSS hash func are not supported"); +- if (rss->level) ++ RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, ++ "cannot set extra types."); ++ ++ l3l4_src_dst = left_types; ++ /* L3/L4 SRC and DST shouldn't be specified at the same time. */ ++ l3l4_refine = rte_eth_rss_hf_refine(l3l4_src_dst); ++ if (l3l4_refine != l3l4_src_dst) + return rte_flow_error_set(error, ENOTSUP, +- RTE_FLOW_ERROR_TYPE_ACTION_CONF, act, +- "a nonzero RSS encapsulation level is not supported"); +- if (rss->key_len && rss->key_len != RTE_DIM(rss_conf->key)) ++ RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, ++ "cannot specify L3_SRC/DST_ONLY or L4_SRC/DST_ONLY at the same."); ++ ++ if (!hns3_valid_ipv6_sctp_rss_types(hw, rss_act->types)) + return rte_flow_error_set(error, ENOTSUP, +- RTE_FLOW_ERROR_TYPE_ACTION_CONF, act, +- "RSS hash key must be exactly 40 bytes"); ++ RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, ++ "hardware doesn't support to use L4 src/dst to hash for IPV6-SCTP."); + +- if (!hns3_rss_input_tuple_supported(hw, rss)) +- return rte_flow_error_set(error, EINVAL, ++ rss_conf->hw_pctypes |= map->hw_pctype; ++ ++ return 0; ++} ++ ++static int ++hns3_flow_parse_rss_types_by_ptype(struct hns3_hw *hw, ++ const struct rte_flow_action_rss *rss_act, ++ uint64_t pattern_type, ++ struct hns3_flow_rss_conf *rss_conf, ++ struct rte_flow_error *error) ++{ ++ const struct hns3_hash_map_info *map; ++ bool matched = false; ++ uint16_t i; ++ int ret; ++ ++ for (i = 0; i < RTE_DIM(hash_map_table); i++) { ++ map = &hash_map_table[i]; ++ if (map->pattern_type != pattern_type) { ++ /* ++ * If the target pattern type is already matched with ++ * the one before this pattern in the hash map table, ++ * no need to continue walk. ++ */ ++ if (matched) ++ break; ++ continue; ++ } ++ matched = true; ++ ++ /* ++ * If pattern type is matched and the 'types' is zero, all packet flow ++ * types related to this pattern type disable RSS hash. ++ * Otherwise, RSS types must match the pattern type and cannot have no ++ * extra or unsupported types. ++ */ ++ if (rss_act->types != 0 && !(map->rss_pctype & rss_act->types)) ++ continue; ++ ++ ret = hns3_flow_get_hw_pctype(hw, rss_act, map, rss_conf, error); ++ if (ret != 0) ++ return ret; ++ } ++ ++ if (rss_conf->hw_pctypes != 0) ++ return 0; ++ ++ if (matched) ++ return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION_CONF, +- &rss->types, +- "input RSS types are not supported"); ++ NULL, "RSS types are unsupported"); + +- act_index++; ++ return rte_flow_error_set(error, ENOTSUP, ++ RTE_FLOW_ERROR_TYPE_ACTION_CONF, ++ NULL, "Pattern specified is unsupported"); ++} + +- /* Check if the next not void action is END */ +- NEXT_ITEM_OF_ACTION(act, actions, act_index); +- if (act->type != RTE_FLOW_ACTION_TYPE_END) { +- memset(rss_conf, 0, sizeof(struct hns3_rss_conf)); +- return rte_flow_error_set(error, EINVAL, +- RTE_FLOW_ERROR_TYPE_ACTION, +- act, "Not supported action."); ++static uint64_t ++hns3_flow_get_all_hw_pctypes(uint64_t types) ++{ ++ uint64_t hw_pctypes = 0; ++ uint16_t i; ++ ++ for (i = 0; i < RTE_DIM(hash_map_table); i++) { ++ if (types & hash_map_table[i].rss_pctype) ++ hw_pctypes |= hash_map_table[i].hw_pctype; + } + +- return 0; ++ return hw_pctypes; ++} ++ ++static int ++hns3_flow_parse_rss_types(struct hns3_hw *hw, ++ const struct rte_flow_action_rss *rss_act, ++ uint64_t pattern_type, ++ struct hns3_flow_rss_conf *rss_conf, ++ struct rte_flow_error *error) ++{ ++ rss_conf->conf.types = rss_act->types; ++ ++ /* no pattern specified to set global RSS types. */ ++ if (pattern_type == 0) { ++ if (!hns3_check_rss_types_valid(hw, rss_act->types)) ++ return rte_flow_error_set(error, EINVAL, ++ RTE_FLOW_ERROR_TYPE_ACTION_CONF, ++ NULL, "RSS types is invalid."); ++ rss_conf->hw_pctypes = ++ hns3_flow_get_all_hw_pctypes(rss_act->types); ++ return 0; ++ } ++ ++ return hns3_flow_parse_rss_types_by_ptype(hw, rss_act, pattern_type, ++ rss_conf, error); + } + + static int +-hns3_disable_rss(struct hns3_hw *hw) ++hns3_flow_parse_hash_global_conf(struct rte_eth_dev *dev, ++ const struct rte_flow_action_rss *rss_act, ++ struct hns3_flow_rss_conf *rss_conf, ++ struct rte_flow_error *error) + { ++ struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private); + int ret; + +- ret = hns3_set_rss_tuple_by_rss_hf(hw, 0); +- if (ret) ++ ret = hns3_flow_parse_hash_func(rss_act, rss_conf, error); ++ if (ret != 0) + return ret; + +- return 0; ++ if (rss_act->queue_num > 0) { ++ ret = hns3_flow_parse_queues(hw, rss_act, rss_conf, error); ++ if (ret != 0) ++ return ret; ++ } ++ ++ if (rss_act->key_len > 0) { ++ ret = hns3_flow_parse_hash_key(hw, rss_act, rss_conf, error); ++ if (ret != 0) ++ return ret; ++ } ++ ++ return hns3_flow_parse_rss_types(hw, rss_act, rss_conf->pattern_type, ++ rss_conf, error); + } + +-static void +-hns3_adjust_rss_key(struct hns3_hw *hw, struct rte_flow_action_rss *rss_conf) ++static int ++hns3_flow_parse_pattern_type(const struct rte_flow_item pattern[], ++ uint64_t *ptype, struct rte_flow_error *error) + { +- if (rss_conf->key == NULL || rss_conf->key_len < HNS3_RSS_KEY_SIZE) { +- hns3_warn(hw, "Default RSS hash key to be set"); +- rss_conf->key = hns3_hash_key; +- rss_conf->key_len = HNS3_RSS_KEY_SIZE; ++ enum rte_flow_item_type pre_type = RTE_FLOW_ITEM_TYPE_VOID; ++ const char *message = "Pattern specified isn't supported"; ++ uint64_t item_hdr, pattern_hdrs = 0; ++ enum rte_flow_item_type cur_type; ++ ++ for (; pattern->type != RTE_FLOW_ITEM_TYPE_END; pattern++) { ++ if (pattern->type == RTE_FLOW_ITEM_TYPE_VOID) ++ continue; ++ if (pattern->mask || pattern->spec || pattern->last) { ++ message = "Header info shouldn't be specified"; ++ goto unsup; ++ } ++ ++ /* Check the sub-item allowed by the previous item . */ ++ if (pre_type >= RTE_DIM(hash_pattern_next_allow_items) || ++ !(hash_pattern_next_allow_items[pre_type] & ++ BIT_ULL(pattern->type))) ++ goto unsup; ++ ++ cur_type = pattern->type; ++ /* Unsupported for current type being greater than array size. */ ++ if (cur_type >= RTE_DIM(hash_pattern_item_header)) ++ goto unsup; ++ ++ /* The value is zero, which means unsupported current header. */ ++ item_hdr = hash_pattern_item_header[cur_type]; ++ if (item_hdr == 0) ++ goto unsup; ++ ++ /* Have duplicate pattern header. */ ++ if (item_hdr & pattern_hdrs) ++ goto unsup; ++ pre_type = cur_type; ++ pattern_hdrs |= item_hdr; + } ++ ++ if (pattern_hdrs != 0) { ++ *ptype = pattern_hdrs; ++ return 0; ++ } ++ ++unsup: ++ return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, ++ pattern, message); + } + + static int +-hns3_parse_rss_algorithm(struct hns3_hw *hw, enum rte_eth_hash_function *func, +- uint8_t *hash_algo) +-{ +- enum rte_eth_hash_function algo_func = *func; +- switch (algo_func) { +- case RTE_ETH_HASH_FUNCTION_DEFAULT: +- /* Keep *hash_algo as what it used to be */ +- algo_func = hw->rss_info.conf.func; +- break; +- case RTE_ETH_HASH_FUNCTION_TOEPLITZ: +- *hash_algo = HNS3_RSS_HASH_ALGO_TOEPLITZ; +- break; +- case RTE_ETH_HASH_FUNCTION_SIMPLE_XOR: +- *hash_algo = HNS3_RSS_HASH_ALGO_SIMPLE; +- break; +- case RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ: +- *hash_algo = HNS3_RSS_HASH_ALGO_SYMMETRIC_TOEP; +- break; +- default: +- hns3_err(hw, "Invalid RSS algorithm configuration(%d)", +- algo_func); +- return -EINVAL; ++hns3_flow_parse_pattern_act(struct rte_eth_dev *dev, ++ const struct rte_flow_item pattern[], ++ const struct rte_flow_action_rss *rss_act, ++ struct hns3_flow_rss_conf *rss_conf, ++ struct rte_flow_error *error) ++{ ++ struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private); ++ int ret; ++ ++ ret = hns3_flow_parse_hash_func(rss_act, rss_conf, error); ++ if (ret != 0) ++ return ret; ++ ++ if (rss_act->key_len > 0) { ++ ret = hns3_flow_parse_hash_key(hw, rss_act, rss_conf, error); ++ if (ret != 0) ++ return ret; + } +- *func = algo_func; ++ ++ if (rss_act->queue_num > 0) { ++ ret = hns3_flow_parse_queues(hw, rss_act, rss_conf, error); ++ if (ret != 0) ++ return ret; ++ } ++ ++ ret = hns3_flow_parse_pattern_type(pattern, &rss_conf->pattern_type, ++ error); ++ if (ret != 0) ++ return ret; ++ ++ ret = hns3_flow_parse_rss_types(hw, rss_act, rss_conf->pattern_type, ++ rss_conf, error); ++ if (ret != 0) ++ return ret; ++ ++ if (rss_act->func != RTE_ETH_HASH_FUNCTION_DEFAULT || ++ rss_act->key_len > 0 || rss_act->queue_num > 0) ++ hns3_warn(hw, "hash func, key and queues are global config, which work for all flow types. " ++ "Recommend: don't set them together with pattern."); + + return 0; + } + ++static bool ++hns3_rss_action_is_dup(struct hns3_hw *hw, ++ const struct hns3_flow_rss_conf *conf) ++{ ++ struct hns3_rss_conf_ele *filter; ++ ++ TAILQ_FOREACH(filter, &hw->flow_rss_list, entries) { ++ if (conf->pattern_type != filter->filter_info.pattern_type) ++ continue; ++ ++ if (hns3_action_rss_same(&filter->filter_info.conf, &conf->conf)) ++ return true; ++ } ++ ++ return false; ++} ++ ++/* ++ * This function is used to parse rss action validation. ++ */ + static int +-hns3_hw_rss_hash_set(struct hns3_hw *hw, struct rte_flow_action_rss *rss_config) ++hns3_parse_rss_filter(struct rte_eth_dev *dev, ++ const struct rte_flow_item pattern[], ++ const struct rte_flow_action *actions, ++ struct hns3_flow_rss_conf *rss_conf, ++ struct rte_flow_error *error) + { ++ struct hns3_adapter *hns = dev->data->dev_private; ++ const struct rte_flow_action_rss *rss_act; ++ const struct rte_flow_action *act; ++ const struct rte_flow_item *pat; ++ struct hns3_hw *hw = &hns->hw; ++ uint32_t index = 0; + int ret; + +- hns3_adjust_rss_key(hw, rss_config); ++ NEXT_ITEM_OF_ACTION(act, actions, index); ++ if (actions[1].type != RTE_FLOW_ACTION_TYPE_END) ++ return rte_flow_error_set(error, EINVAL, ++ RTE_FLOW_ERROR_TYPE_ACTION, ++ &actions[1], ++ "Only support one action for RSS."); + +- ret = hns3_parse_rss_algorithm(hw, &rss_config->func, +- &hw->rss_info.hash_algo); +- if (ret) +- return ret; ++ rss_act = (const struct rte_flow_action_rss *)act->conf; ++ if (rss_act == NULL) { ++ return rte_flow_error_set(error, EINVAL, ++ RTE_FLOW_ERROR_TYPE_ACTION_CONF, ++ act, "lost RSS action configuration"); ++ } + +- ret = hns3_rss_set_algo_key(hw, rss_config->key); +- if (ret) ++ if (rss_act->level != 0) ++ return rte_flow_error_set(error, ENOTSUP, ++ RTE_FLOW_ERROR_TYPE_ACTION_CONF, ++ act, ++ "RSS level is not supported"); ++ ++ index = 0; ++ NEXT_ITEM_OF_PATTERN(pat, pattern, index); ++ if (pat[0].type == RTE_FLOW_ITEM_TYPE_END) { ++ rss_conf->pattern_type = 0; ++ ret = hns3_flow_parse_hash_global_conf(dev, rss_act, ++ rss_conf, error); ++ } else { ++ ret = hns3_flow_parse_pattern_act(dev, pat, rss_act, ++ rss_conf, error); ++ } ++ if (ret != 0) + return ret; + +- hw->rss_info.conf.func = rss_config->func; +- +- ret = hns3_set_rss_tuple_by_rss_hf(hw, rss_config->types); +- if (ret) +- hns3_err(hw, "Update RSS tuples by rss hf failed %d", ret); ++ if (hns3_rss_action_is_dup(hw, rss_conf)) ++ return rte_flow_error_set(error, EINVAL, ++ RTE_FLOW_ERROR_TYPE_ACTION_CONF, ++ act, "duplicate RSS rule"); + +- return ret; ++ return 0; + } + + static int +@@ -1516,8 +1873,6 @@ hns3_update_indir_table(struct hns3_hw *hw, + uint32_t i; + + /* Fill in redirection table */ +- memcpy(indir_tbl, hw->rss_info.rss_indirection_tbl, +- sizeof(hw->rss_info.rss_indirection_tbl)); + for (i = 0, j = 0; i < hw->rss_ind_tbl_size; i++, j++) { + j %= num; + if (conf->queue[j] >= hw->alloc_rss_size) { +@@ -1532,82 +1887,106 @@ hns3_update_indir_table(struct hns3_hw *hw, + return hns3_set_rss_indir_table(hw, indir_tbl, hw->rss_ind_tbl_size); + } + ++static uint64_t ++hns3_flow_get_pctype_tuple_mask(uint64_t hw_pctype) ++{ ++ uint64_t tuple_mask = 0; ++ uint16_t i; ++ ++ for (i = 0; i < RTE_DIM(hash_map_table); i++) { ++ if (hw_pctype == hash_map_table[i].hw_pctype) { ++ tuple_mask = hash_map_table[i].tuple_mask; ++ break; ++ } ++ } ++ ++ return tuple_mask; ++} ++ ++static int ++hns3_flow_set_rss_ptype_tuple(struct hns3_hw *hw, ++ struct hns3_flow_rss_conf *rss_conf) ++{ ++ uint64_t old_tuple_fields, new_tuple_fields; ++ uint64_t hw_pctypes, tuples, tuple_mask = 0; ++ bool cfg_global_tuple; ++ int ret; ++ ++ cfg_global_tuple = (rss_conf->pattern_type == 0); ++ if (!cfg_global_tuple) { ++ /* ++ * To ensure that different packets do not affect each other, ++ * we have to first read all tuple fields, and then only modify ++ * the tuples for the specified packet type. ++ */ ++ ret = hns3_get_rss_tuple_field(hw, &old_tuple_fields); ++ if (ret != 0) ++ return ret; ++ ++ new_tuple_fields = old_tuple_fields; ++ hw_pctypes = rss_conf->hw_pctypes; ++ while (hw_pctypes > 0) { ++ uint32_t idx = rte_bsf64(hw_pctypes); ++ uint64_t pctype = BIT_ULL(idx); ++ ++ tuple_mask = hns3_flow_get_pctype_tuple_mask(pctype); ++ tuples = hns3_rss_calc_tuple_filed(rss_conf->conf.types); ++ new_tuple_fields &= ~tuple_mask; ++ new_tuple_fields |= tuples; ++ hw_pctypes &= ~pctype; ++ } ++ } else { ++ new_tuple_fields = ++ hns3_rss_calc_tuple_filed(rss_conf->conf.types); ++ } ++ ++ ret = hns3_set_rss_tuple_field(hw, new_tuple_fields); ++ if (ret != 0) ++ return ret; ++ ++ if (!cfg_global_tuple) ++ hns3_info(hw, "RSS tuple fields changed from 0x%" PRIx64 " to 0x%" PRIx64, ++ old_tuple_fields, new_tuple_fields); ++ ++ return 0; ++} ++ + static int + hns3_config_rss_filter(struct hns3_hw *hw, +- const struct hns3_rss_conf *conf, bool add) ++ struct hns3_flow_rss_conf *rss_conf) + { +- struct hns3_rss_conf *rss_info; +- uint64_t flow_types; +- uint16_t num; ++ struct rte_flow_action_rss *rss_act; + int ret; + +- struct rte_flow_action_rss rss_flow_conf = { +- .func = conf->conf.func, +- .level = conf->conf.level, +- .types = conf->conf.types, +- .key_len = conf->conf.key_len, +- .queue_num = conf->conf.queue_num, +- .key = conf->conf.key_len ? +- (void *)(uintptr_t)conf->conf.key : NULL, +- .queue = conf->conf.queue, +- }; +- +- /* Filter the unsupported flow types */ +- flow_types = conf->conf.types ? +- rss_flow_conf.types & HNS3_ETH_RSS_SUPPORT : +- hw->rss_info.conf.types; +- if (flow_types != rss_flow_conf.types) +- hns3_warn(hw, "modified RSS types based on hardware support, " +- "requested:0x%" PRIx64 " configured:0x%" PRIx64, +- rss_flow_conf.types, flow_types); +- /* Update the useful flow types */ +- rss_flow_conf.types = flow_types; +- +- rss_info = &hw->rss_info; +- if (!add) { +- if (!conf->valid) +- return 0; +- +- ret = hns3_disable_rss(hw); ++ rss_act = &rss_conf->conf; ++ if (rss_act->queue_num > 0) { ++ ret = hns3_update_indir_table(hw, rss_act, rss_act->queue_num); + if (ret) { +- hns3_err(hw, "RSS disable failed(%d)", ret); ++ hns3_err(hw, "set queues action failed, ret = %d", ret); + return ret; + } ++ } + +- if (rss_flow_conf.queue_num) { +- /* +- * Due the content of queue pointer have been reset to +- * 0, the rss_info->conf.queue should be set to NULL +- */ +- rss_info->conf.queue = NULL; +- rss_info->conf.queue_num = 0; ++ if (rss_act->key_len > 0 || ++ rss_act->func != RTE_ETH_HASH_FUNCTION_DEFAULT) { ++ ret = hns3_update_rss_algo_key(hw, rss_act->func, rss_conf->key, ++ rss_act->key_len); ++ if (ret != 0) { ++ hns3_err(hw, "set func or hash key action failed, ret = %d", ++ ret); ++ return ret; + } +- +- return 0; + } + +- /* Set rx queues to use */ +- num = RTE_MIN(hw->data->nb_rx_queues, rss_flow_conf.queue_num); +- if (rss_flow_conf.queue_num > num) +- hns3_warn(hw, "Config queue numbers %u are beyond the scope of truncated", +- rss_flow_conf.queue_num); +- hns3_info(hw, "Max of contiguous %u PF queues are configured", num); +- if (num) { +- ret = hns3_update_indir_table(hw, &rss_flow_conf, num); +- if (ret) ++ if (rss_conf->hw_pctypes > 0) { ++ ret = hns3_flow_set_rss_ptype_tuple(hw, rss_conf); ++ if (ret != 0) { ++ hns3_err(hw, "set types action failed, ret = %d", ret); + return ret; ++ } + } + +- /* Set hash algorithm and flow types by the user's config */ +- ret = hns3_hw_rss_hash_set(hw, &rss_flow_conf); +- if (ret) +- return ret; +- +- ret = hns3_rss_conf_copy(rss_info, &rss_flow_conf); +- if (ret) +- hns3_err(hw, "RSS config init fail(%d)", ret); +- +- return ret; ++ return 0; + } + + static int +@@ -1616,51 +1995,44 @@ hns3_clear_rss_filter(struct rte_eth_dev *dev) + struct hns3_adapter *hns = dev->data->dev_private; + struct hns3_rss_conf_ele *rss_filter_ptr; + struct hns3_hw *hw = &hns->hw; +- int rss_rule_succ_cnt = 0; /* count for success of clearing RSS rules */ +- int rss_rule_fail_cnt = 0; /* count for failure of clearing RSS rules */ +- int ret = 0; + + rss_filter_ptr = TAILQ_FIRST(&hw->flow_rss_list); + while (rss_filter_ptr) { + TAILQ_REMOVE(&hw->flow_rss_list, rss_filter_ptr, entries); +- ret = hns3_config_rss_filter(hw, &rss_filter_ptr->filter_info, +- false); +- if (ret) +- rss_rule_fail_cnt++; +- else +- rss_rule_succ_cnt++; + rte_free(rss_filter_ptr); + rss_filter_ptr = TAILQ_FIRST(&hw->flow_rss_list); + } + +- if (rss_rule_fail_cnt) { +- hns3_err(hw, "fail to delete all RSS filters, success num = %d fail num = %d", +- rss_rule_succ_cnt, rss_rule_fail_cnt); +- ret = -EIO; +- } +- +- return ret; ++ return hns3_config_rss(hns); + } + + static int +-hns3_restore_rss_filter(struct hns3_hw *hw) ++hns3_reconfig_all_rss_filter(struct hns3_hw *hw) + { + struct hns3_rss_conf_ele *filter; +- int ret = 0; ++ uint32_t rule_no = 0; ++ int ret; + +- pthread_mutex_lock(&hw->flows_lock); + TAILQ_FOREACH(filter, &hw->flow_rss_list, entries) { +- if (!filter->filter_info.valid) +- continue; +- +- ret = hns3_config_rss_filter(hw, &filter->filter_info, true); ++ ret = hns3_config_rss_filter(hw, &filter->filter_info); + if (ret != 0) { +- hns3_err(hw, "restore RSS filter failed, ret=%d", ret); +- goto out; ++ hns3_err(hw, "config %uth RSS filter failed, ret = %d", ++ rule_no, ret); ++ return ret; + } ++ rule_no++; + } + +-out: ++ return 0; ++} ++ ++static int ++hns3_restore_rss_filter(struct hns3_hw *hw) ++{ ++ int ret; ++ ++ pthread_mutex_lock(&hw->flows_lock); ++ ret = hns3_reconfig_all_rss_filter(hw); + pthread_mutex_unlock(&hw->flows_lock); + + return ret; +@@ -1679,23 +2051,6 @@ hns3_restore_filter(struct hns3_adapter *hns) + return hns3_restore_rss_filter(hw); + } + +-static int +-hns3_flow_parse_rss(struct rte_eth_dev *dev, +- const struct hns3_rss_conf *conf, bool add) +-{ +- struct hns3_adapter *hns = dev->data->dev_private; +- struct hns3_hw *hw = &hns->hw; +- bool ret; +- +- ret = hns3_action_rss_same(&hw->rss_info.conf, &conf->conf); +- if (ret) { +- hns3_err(hw, "Enter duplicate RSS configuration : %d", ret); +- return -EINVAL; +- } +- +- return hns3_config_rss_filter(hw, conf, add); +-} +- + static int + hns3_flow_args_check(const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], +@@ -1729,31 +2084,55 @@ static int + hns3_flow_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], +- struct rte_flow_error *error) ++ struct rte_flow_error *error, ++ struct hns3_filter_info *filter_info) + { +- struct hns3_fdir_rule fdir_rule; ++ union hns3_filter_conf *conf; + int ret; + + ret = hns3_flow_args_check(attr, pattern, actions, error); + if (ret) + return ret; + +- if (hns3_find_rss_general_action(pattern, actions)) +- return hns3_parse_rss_filter(dev, actions, error); ++ hns3_parse_filter_type(pattern, actions, filter_info); ++ conf = &filter_info->conf; ++ if (filter_info->type == RTE_ETH_FILTER_HASH) ++ return hns3_parse_rss_filter(dev, pattern, actions, ++ &conf->rss_conf, error); ++ ++ return hns3_parse_fdir_filter(dev, pattern, actions, ++ &conf->fdir_conf, error); ++} ++ ++static int ++hns3_flow_rebuild_all_rss_filter(struct hns3_adapter *hns) ++{ ++ struct hns3_hw *hw = &hns->hw; ++ int ret; + +- memset(&fdir_rule, 0, sizeof(struct hns3_fdir_rule)); +- return hns3_parse_fdir_filter(dev, pattern, actions, &fdir_rule, error); ++ ret = hns3_config_rss(hns); ++ if (ret != 0) { ++ hns3_err(hw, "restore original RSS configuration failed, ret = %d.", ++ ret); ++ return ret; ++ } ++ ret = hns3_reconfig_all_rss_filter(hw); ++ if (ret != 0) ++ hns3_err(hw, "rebuild all RSS filter failed, ret = %d.", ret); ++ ++ return ret; + } + + static int + hns3_flow_create_rss_rule(struct rte_eth_dev *dev, +- const struct rte_flow_action *act, ++ struct hns3_flow_rss_conf *rss_conf, + struct rte_flow *flow) + { + struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private); ++ struct hns3_adapter *hns = HNS3_DEV_HW_TO_ADAPTER(hw); + struct hns3_rss_conf_ele *rss_filter_ptr; +- struct hns3_rss_conf_ele *filter_ptr; +- const struct hns3_rss_conf *rss_conf; ++ struct hns3_flow_rss_conf *new_conf; ++ struct rte_flow_action_rss *rss_act; + int ret; + + rss_filter_ptr = rte_zmalloc("hns3 rss filter", +@@ -1763,28 +2142,29 @@ hns3_flow_create_rss_rule(struct rte_eth_dev *dev, + return -ENOMEM; + } + ++ new_conf = &rss_filter_ptr->filter_info; ++ memcpy(new_conf, rss_conf, sizeof(*new_conf)); ++ rss_act = &new_conf->conf; ++ if (rss_act->queue_num > 0) ++ new_conf->conf.queue = new_conf->queue; + /* +- * After all the preceding tasks are successfully configured, configure +- * rules to the hardware to simplify the rollback of rules in the +- * hardware. ++ * There are two ways to deliver hash key action: ++ * 1> 'key_len' is greater than zero and 'key' isn't NULL. ++ * 2> 'key_len' is greater than zero, but 'key' is NULL. ++ * For case 2, we need to keep 'key' of the new_conf is NULL so as to ++ * inherit the configuration from user in case of failing to verify ++ * duplicate rule later. + */ +- rss_conf = (const struct hns3_rss_conf *)act->conf; +- ret = hns3_flow_parse_rss(dev, rss_conf, true); ++ if (rss_act->key_len > 0 && rss_act->key != NULL) ++ new_conf->conf.key = new_conf->key; ++ ++ ret = hns3_config_rss_filter(hw, new_conf); + if (ret != 0) { + rte_free(rss_filter_ptr); ++ (void)hns3_flow_rebuild_all_rss_filter(hns); + return ret; + } + +- hns3_rss_conf_copy(&rss_filter_ptr->filter_info, &rss_conf->conf); +- rss_filter_ptr->filter_info.valid = true; +- +- /* +- * When create a new RSS rule, the old rule will be overlaid and set +- * invalid. +- */ +- TAILQ_FOREACH(filter_ptr, &hw->flow_rss_list, entries) +- filter_ptr->filter_info.valid = false; +- + TAILQ_INSERT_TAIL(&hw->flow_rss_list, rss_filter_ptr, entries); + flow->rule = rss_filter_ptr; + flow->filter_type = RTE_ETH_FILTER_HASH; +@@ -1794,31 +2174,24 @@ hns3_flow_create_rss_rule(struct rte_eth_dev *dev, + + static int + hns3_flow_create_fdir_rule(struct rte_eth_dev *dev, +- const struct rte_flow_item pattern[], +- const struct rte_flow_action actions[], ++ struct hns3_fdir_rule *fdir_rule, + struct rte_flow_error *error, + struct rte_flow *flow) + { + struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct hns3_adapter *hns = HNS3_DEV_HW_TO_ADAPTER(hw); + struct hns3_fdir_rule_ele *fdir_rule_ptr; +- struct hns3_fdir_rule fdir_rule; + bool indir; + int ret; + +- memset(&fdir_rule, 0, sizeof(struct hns3_fdir_rule)); +- ret = hns3_parse_fdir_filter(dev, pattern, actions, &fdir_rule, error); +- if (ret != 0) +- return ret; +- +- indir = !!(fdir_rule.flags & HNS3_RULE_FLAG_COUNTER_INDIR); +- if (fdir_rule.flags & HNS3_RULE_FLAG_COUNTER) { +- ret = hns3_counter_new(dev, indir, fdir_rule.act_cnt.id, ++ indir = !!(fdir_rule->flags & HNS3_RULE_FLAG_COUNTER_INDIR); ++ if (fdir_rule->flags & HNS3_RULE_FLAG_COUNTER) { ++ ret = hns3_counter_new(dev, indir, fdir_rule->act_cnt.id, + error); + if (ret != 0) + return ret; + +- flow->counter_id = fdir_rule.act_cnt.id; ++ flow->counter_id = fdir_rule->act_cnt.id; + } + + fdir_rule_ptr = rte_zmalloc("hns3 fdir rule", +@@ -1834,11 +2207,11 @@ hns3_flow_create_fdir_rule(struct rte_eth_dev *dev, + * rules to the hardware to simplify the rollback of rules in the + * hardware. + */ +- ret = hns3_fdir_filter_program(hns, &fdir_rule, false); ++ ret = hns3_fdir_filter_program(hns, fdir_rule, false); + if (ret != 0) + goto err_fdir_filter; + +- memcpy(&fdir_rule_ptr->fdir_conf, &fdir_rule, ++ memcpy(&fdir_rule_ptr->fdir_conf, fdir_rule, + sizeof(struct hns3_fdir_rule)); + TAILQ_INSERT_TAIL(&hw->flow_fdir_list, fdir_rule_ptr, entries); + flow->rule = fdir_rule_ptr; +@@ -1849,8 +2222,8 @@ hns3_flow_create_fdir_rule(struct rte_eth_dev *dev, + err_fdir_filter: + rte_free(fdir_rule_ptr); + err_malloc: +- if (fdir_rule.flags & HNS3_RULE_FLAG_COUNTER) +- hns3_counter_release(dev, fdir_rule.act_cnt.id); ++ if (fdir_rule->flags & HNS3_RULE_FLAG_COUNTER) ++ hns3_counter_release(dev, fdir_rule->act_cnt.id); + + return ret; + } +@@ -1868,13 +2241,15 @@ hns3_flow_create(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, + struct rte_flow_error *error) + { + struct hns3_adapter *hns = dev->data->dev_private; +- struct hns3_hw *hw = &hns->hw; ++ struct hns3_filter_info filter_info = {0}; + struct hns3_flow_mem *flow_node; +- const struct rte_flow_action *act; ++ struct hns3_hw *hw = &hns->hw; ++ union hns3_filter_conf *conf; + struct rte_flow *flow; + int ret; + +- ret = hns3_flow_validate(dev, attr, pattern, actions, error); ++ ret = hns3_flow_validate(dev, attr, pattern, actions, error, ++ &filter_info); + if (ret) + return NULL; + +@@ -1894,13 +2269,12 @@ hns3_flow_create(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, + } + + flow_node->flow = flow; ++ conf = &filter_info.conf; + TAILQ_INSERT_TAIL(&hw->flow_list, flow_node, entries); +- +- act = hns3_find_rss_general_action(pattern, actions); +- if (act) +- ret = hns3_flow_create_rss_rule(dev, act, flow); ++ if (filter_info.type == RTE_ETH_FILTER_HASH) ++ ret = hns3_flow_create_rss_rule(dev, &conf->rss_conf, flow); + else +- ret = hns3_flow_create_fdir_rule(dev, pattern, actions, ++ ret = hns3_flow_create_fdir_rule(dev, &conf->fdir_conf, + error, flow); + if (ret == 0) + return flow; +@@ -1954,16 +2328,10 @@ hns3_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow, + break; + case RTE_ETH_FILTER_HASH: + rss_filter_ptr = (struct hns3_rss_conf_ele *)flow->rule; +- ret = hns3_config_rss_filter(hw, &rss_filter_ptr->filter_info, +- false); +- if (ret) +- return rte_flow_error_set(error, EIO, +- RTE_FLOW_ERROR_TYPE_HANDLE, +- flow, +- "Destroy RSS fail.Try again"); + TAILQ_REMOVE(&hw->flow_rss_list, rss_filter_ptr, entries); + rte_free(rss_filter_ptr); + rss_filter_ptr = NULL; ++ (void)hns3_flow_rebuild_all_rss_filter(hns); + break; + default: + return rte_flow_error_set(error, EINVAL, +@@ -2069,10 +2437,12 @@ hns3_flow_validate_wrap(struct rte_eth_dev *dev, + struct rte_flow_error *error) + { + struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private); ++ struct hns3_filter_info filter_info = {0}; + int ret; + + pthread_mutex_lock(&hw->flows_lock); +- ret = hns3_flow_validate(dev, attr, pattern, actions, error); ++ ret = hns3_flow_validate(dev, attr, pattern, actions, error, ++ &filter_info); + pthread_mutex_unlock(&hw->flows_lock); + + return ret; +diff --git a/dpdk/drivers/net/hns3/hns3_flow.h b/dpdk/drivers/net/hns3/hns3_flow.h +index e4b2fdf2e6..1b49673f11 100644 +--- a/dpdk/drivers/net/hns3/hns3_flow.h ++++ b/dpdk/drivers/net/hns3/hns3_flow.h +@@ -9,6 +9,7 @@ + #include + + #include "hns3_rss.h" ++#include "hns3_fdir.h" + + struct hns3_flow_counter { + LIST_ENTRY(hns3_flow_counter) next; /* Pointer to the next counter. */ +@@ -24,10 +25,18 @@ struct rte_flow { + uint32_t counter_id; + }; + ++struct hns3_flow_rss_conf { ++ struct rte_flow_action_rss conf; ++ uint8_t key[HNS3_RSS_KEY_SIZE_MAX]; /* Hash key */ ++ uint16_t queue[HNS3_RSS_QUEUES_BUFFER_NUM]; /* Queues indices to use */ ++ uint64_t pattern_type; ++ uint64_t hw_pctypes; /* packet types in driver */ ++}; ++ + /* rss filter list structure */ + struct hns3_rss_conf_ele { + TAILQ_ENTRY(hns3_rss_conf_ele) entries; +- struct hns3_rss_conf filter_info; ++ struct hns3_flow_rss_conf filter_info; + }; + + /* hns3_flow memory list structure */ +@@ -45,6 +54,16 @@ struct rte_flow_action_handle { + uint32_t counter_id; + }; + ++union hns3_filter_conf { ++ struct hns3_fdir_rule fdir_conf; ++ struct hns3_flow_rss_conf rss_conf; ++}; ++ ++struct hns3_filter_info { ++ enum rte_filter_type type; ++ union hns3_filter_conf conf; ++}; ++ + TAILQ_HEAD(hns3_rss_filter_list, hns3_rss_conf_ele); + TAILQ_HEAD(hns3_flow_mem_list, hns3_flow_mem); + +diff --git a/dpdk/drivers/net/hns3/hns3_intr.c b/dpdk/drivers/net/hns3/hns3_intr.c +index 44a1119415..916bf30dcb 100644 +--- a/dpdk/drivers/net/hns3/hns3_intr.c ++++ b/dpdk/drivers/net/hns3/hns3_intr.c +@@ -2434,8 +2434,8 @@ hns3_schedule_reset(struct hns3_adapter *hns) + if (__atomic_load_n(&hw->reset.schedule, __ATOMIC_RELAXED) == + SCHEDULE_DEFERRED) + rte_eal_alarm_cancel(hw->reset.ops->reset_service, hns); +- else +- __atomic_store_n(&hw->reset.schedule, SCHEDULE_REQUESTED, ++ ++ __atomic_store_n(&hw->reset.schedule, SCHEDULE_REQUESTED, + __ATOMIC_RELAXED); + + rte_eal_alarm_set(SWITCH_CONTEXT_US, hw->reset.ops->reset_service, hns); +@@ -2749,6 +2749,7 @@ hns3_reset_post(struct hns3_adapter *hns) + /* IMP will wait ready flag before reset */ + hns3_notify_reset_ready(hw, false); + hns3_clear_reset_level(hw, &hw->reset.pending); ++ hns3_clear_reset_status(hw); + __atomic_store_n(&hns->hw.reset.resetting, 0, __ATOMIC_RELAXED); + hw->reset.attempts = 0; + hw->reset.stats.success_cnt++; +@@ -2798,6 +2799,7 @@ hns3_reset_fail_handle(struct hns3_adapter *hns) + struct timeval tv; + + hns3_clear_reset_level(hw, &hw->reset.pending); ++ hns3_clear_reset_status(hw); + if (hns3_reset_err_handle(hns)) { + hw->reset.stage = RESET_STAGE_PREWAIT; + hns3_schedule_reset(hns); +diff --git a/dpdk/drivers/net/hns3/hns3_mbx.c b/dpdk/drivers/net/hns3/hns3_mbx.c +index 8e0a58aa02..f1743c195e 100644 +--- a/dpdk/drivers/net/hns3/hns3_mbx.c ++++ b/dpdk/drivers/net/hns3/hns3_mbx.c +@@ -40,23 +40,6 @@ hns3_resp_to_errno(uint16_t resp_code) + return -EIO; + } + +-static void +-hns3_mbx_proc_timeout(struct hns3_hw *hw, uint16_t code, uint16_t subcode) +-{ +- if (hw->mbx_resp.matching_scheme == +- HNS3_MBX_RESP_MATCHING_SCHEME_OF_ORIGINAL) { +- hw->mbx_resp.lost++; +- hns3_err(hw, +- "VF could not get mbx(%u,%u) head(%u) tail(%u) " +- "lost(%u) from PF", +- code, subcode, hw->mbx_resp.head, hw->mbx_resp.tail, +- hw->mbx_resp.lost); +- return; +- } +- +- hns3_err(hw, "VF could not get mbx(%u,%u) from PF", code, subcode); +-} +- + static int + hns3_get_mbx_resp(struct hns3_hw *hw, uint16_t code, uint16_t subcode, + uint8_t *resp_data, uint16_t resp_len) +@@ -67,7 +50,6 @@ hns3_get_mbx_resp(struct hns3_hw *hw, uint16_t code, uint16_t subcode, + struct hns3_adapter *hns = HNS3_DEV_HW_TO_ADAPTER(hw); + struct hns3_mbx_resp_status *mbx_resp; + uint32_t wait_time = 0; +- bool received; + + if (resp_len > HNS3_MBX_MAX_RESP_DATA_SIZE) { + hns3_err(hw, "VF mbx response len(=%u) exceeds maximum(=%d)", +@@ -93,20 +75,14 @@ hns3_get_mbx_resp(struct hns3_hw *hw, uint16_t code, uint16_t subcode, + hns3_dev_handle_mbx_msg(hw); + rte_delay_us(HNS3_WAIT_RESP_US); + +- if (hw->mbx_resp.matching_scheme == +- HNS3_MBX_RESP_MATCHING_SCHEME_OF_ORIGINAL) +- received = (hw->mbx_resp.head == +- hw->mbx_resp.tail + hw->mbx_resp.lost); +- else +- received = hw->mbx_resp.received_match_resp; +- if (received) ++ if (hw->mbx_resp.received_match_resp) + break; + + wait_time += HNS3_WAIT_RESP_US; + } + hw->mbx_resp.req_msg_data = 0; + if (wait_time >= mbx_time_limit) { +- hns3_mbx_proc_timeout(hw, code, subcode); ++ hns3_err(hw, "VF could not get mbx(%u,%u) from PF", code, subcode); + return -ETIME; + } + rte_io_rmb(); +@@ -132,7 +108,6 @@ hns3_mbx_prepare_resp(struct hns3_hw *hw, uint16_t code, uint16_t subcode) + * we get the exact scheme which is used. + */ + hw->mbx_resp.req_msg_data = (uint32_t)code << 16 | subcode; +- hw->mbx_resp.head++; + + /* Update match_id and ensure the value of match_id is not zero */ + hw->mbx_resp.match_id++; +@@ -185,7 +160,6 @@ hns3_send_mbx_msg(struct hns3_hw *hw, uint16_t code, uint16_t subcode, + req->match_id = hw->mbx_resp.match_id; + ret = hns3_cmd_send(hw, &desc, 1); + if (ret) { +- hw->mbx_resp.head--; + rte_spinlock_unlock(&hw->mbx_resp.lock); + hns3_err(hw, "VF failed(=%d) to send mbx message to PF", + ret); +@@ -254,41 +228,10 @@ hns3_handle_asserting_reset(struct hns3_hw *hw, + hns3_schedule_reset(HNS3_DEV_HW_TO_ADAPTER(hw)); + } + +-/* +- * Case1: receive response after timeout, req_msg_data +- * is 0, not equal resp_msg, do lost-- +- * Case2: receive last response during new send_mbx_msg, +- * req_msg_data is different with resp_msg, let +- * lost--, continue to wait for response. +- */ +-static void +-hns3_update_resp_position(struct hns3_hw *hw, uint32_t resp_msg) +-{ +- struct hns3_mbx_resp_status *resp = &hw->mbx_resp; +- uint32_t tail = resp->tail + 1; +- +- if (tail > resp->head) +- tail = resp->head; +- if (resp->req_msg_data != resp_msg) { +- if (resp->lost) +- resp->lost--; +- hns3_warn(hw, "Received a mismatched response req_msg(%x) " +- "resp_msg(%x) head(%u) tail(%u) lost(%u)", +- resp->req_msg_data, resp_msg, resp->head, tail, +- resp->lost); +- } else if (tail + resp->lost > resp->head) { +- resp->lost--; +- hns3_warn(hw, "Received a new response again resp_msg(%x) " +- "head(%u) tail(%u) lost(%u)", resp_msg, +- resp->head, tail, resp->lost); +- } +- rte_io_wmb(); +- resp->tail = tail; +-} +- + static void + hns3_handle_mbx_response(struct hns3_hw *hw, struct hns3_mbx_pf_to_vf_cmd *req) + { ++#define HNS3_MBX_RESP_CODE_OFFSET 16 + struct hns3_mbx_resp_status *resp = &hw->mbx_resp; + uint32_t msg_data; + +@@ -298,12 +241,6 @@ hns3_handle_mbx_response(struct hns3_hw *hw, struct hns3_mbx_pf_to_vf_cmd *req) + * match_id to its response. So VF could use the match_id + * to match the request. + */ +- if (resp->matching_scheme != +- HNS3_MBX_RESP_MATCHING_SCHEME_OF_MATCH_ID) { +- resp->matching_scheme = +- HNS3_MBX_RESP_MATCHING_SCHEME_OF_MATCH_ID; +- hns3_info(hw, "detect mailbox support match id!"); +- } + if (req->match_id == resp->match_id) { + resp->resp_status = hns3_resp_to_errno(req->msg[3]); + memcpy(resp->additional_info, &req->msg[4], +@@ -319,11 +256,19 @@ hns3_handle_mbx_response(struct hns3_hw *hw, struct hns3_mbx_pf_to_vf_cmd *req) + * support copy request's match_id to its response. So VF follows the + * original scheme to process. + */ ++ msg_data = (uint32_t)req->msg[1] << HNS3_MBX_RESP_CODE_OFFSET | req->msg[2]; ++ if (resp->req_msg_data != msg_data) { ++ hns3_warn(hw, ++ "received response tag (%u) is mismatched with requested tag (%u)", ++ msg_data, resp->req_msg_data); ++ return; ++ } ++ + resp->resp_status = hns3_resp_to_errno(req->msg[3]); + memcpy(resp->additional_info, &req->msg[4], + HNS3_MBX_MAX_RESP_DATA_SIZE); +- msg_data = (uint32_t)req->msg[1] << 16 | req->msg[2]; +- hns3_update_resp_position(hw, msg_data); ++ rte_io_wmb(); ++ resp->received_match_resp = true; + } + + static void +diff --git a/dpdk/drivers/net/hns3/hns3_mbx.h b/dpdk/drivers/net/hns3/hns3_mbx.h +index c378783c6c..4a328802b9 100644 +--- a/dpdk/drivers/net/hns3/hns3_mbx.h ++++ b/dpdk/drivers/net/hns3/hns3_mbx.h +@@ -93,21 +93,11 @@ enum hns3_mbx_link_fail_subcode { + #define HNS3_MBX_MAX_RESP_DATA_SIZE 8 + #define HNS3_MBX_DEF_TIME_LIMIT_MS 500 + +-enum { +- HNS3_MBX_RESP_MATCHING_SCHEME_OF_ORIGINAL = 0, +- HNS3_MBX_RESP_MATCHING_SCHEME_OF_MATCH_ID +-}; +- + struct hns3_mbx_resp_status { + rte_spinlock_t lock; /* protects against contending sync cmd resp */ + +- uint8_t matching_scheme; +- + /* The following fields used in the matching scheme for original */ + uint32_t req_msg_data; +- uint32_t head; +- uint32_t tail; +- uint32_t lost; + + /* The following fields used in the matching scheme for match_id */ + uint16_t match_id; +diff --git a/dpdk/drivers/net/hns3/hns3_mp.c b/dpdk/drivers/net/hns3/hns3_mp.c +index 7184f9ad58..556f1941c6 100644 +--- a/dpdk/drivers/net/hns3/hns3_mp.c ++++ b/dpdk/drivers/net/hns3/hns3_mp.c +@@ -89,12 +89,12 @@ mp_secondary_handle(const struct rte_mp_msg *mp_msg, const void *peer) + case HNS3_MP_REQ_START_RXTX: + PMD_INIT_LOG(INFO, "port %u starting datapath", + dev->data->port_id); +- hns3_set_rxtx_function(dev); ++ hns3_start_rxtx_datapath(dev); + break; + case HNS3_MP_REQ_STOP_RXTX: + PMD_INIT_LOG(INFO, "port %u stopping datapath", + dev->data->port_id); +- hns3_set_rxtx_function(dev); ++ hns3_stop_rxtx_datapath(dev); + break; + case HNS3_MP_REQ_START_TX: + PMD_INIT_LOG(INFO, "port %u starting Tx datapath", +diff --git a/dpdk/drivers/net/hns3/hns3_ptp.c b/dpdk/drivers/net/hns3/hns3_ptp.c +index 6bbd85ba23..894ac6dd71 100644 +--- a/dpdk/drivers/net/hns3/hns3_ptp.c ++++ b/dpdk/drivers/net/hns3/hns3_ptp.c +@@ -7,7 +7,7 @@ + #include + + #include "hns3_ethdev.h" +-#include "hns3_regs.h" ++#include "hns3_ptp.h" + #include "hns3_logs.h" + + uint64_t hns3_timestamp_rx_dynflag; +@@ -56,9 +56,23 @@ hns3_ptp_int_en(struct hns3_hw *hw, bool en) + return ret; + } + ++static void ++hns3_ptp_timesync_write_time(struct hns3_hw *hw, const struct timespec *ts) ++{ ++ uint64_t sec = ts->tv_sec; ++ uint64_t ns = ts->tv_nsec; ++ ++ /* Set the timecounters to a new value. */ ++ hns3_write_dev(hw, HNS3_CFG_TIME_SYNC_H, upper_32_bits(sec)); ++ hns3_write_dev(hw, HNS3_CFG_TIME_SYNC_M, lower_32_bits(sec)); ++ hns3_write_dev(hw, HNS3_CFG_TIME_SYNC_L, lower_32_bits(ns)); ++ hns3_write_dev(hw, HNS3_CFG_TIME_SYNC_RDY, 1); ++} ++ + int + hns3_ptp_init(struct hns3_hw *hw) + { ++ struct timespec sys_time; + int ret; + + if (!hns3_dev_get_support(hw, PTP)) +@@ -71,6 +85,10 @@ hns3_ptp_init(struct hns3_hw *hw) + /* Start PTP timer */ + hns3_write_dev(hw, HNS3_CFG_TIME_CYC_EN, 1); + ++ /* Initializing the RTC. */ ++ clock_gettime(CLOCK_REALTIME, &sys_time); ++ hns3_ptp_timesync_write_time(hw, &sys_time); ++ + return 0; + } + +@@ -216,17 +234,21 @@ hns3_timesync_read_tx_timestamp(struct rte_eth_dev *dev, + int + hns3_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts) + { ++#define HNS3_PTP_SEC_H_OFFSET 32 ++#define HNS3_PTP_SEC_H_MASK 0xFFFF ++ + struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private); ++ uint32_t sec_hi, sec_lo; + uint64_t ns, sec; + + if (!hns3_dev_get_support(hw, PTP)) + return -ENOTSUP; + +- sec = hns3_read_dev(hw, HNS3_CURR_TIME_OUT_L); +- sec |= (uint64_t)(hns3_read_dev(hw, HNS3_CURR_TIME_OUT_H) & 0xFFFF) +- << 32; +- + ns = hns3_read_dev(hw, HNS3_CURR_TIME_OUT_NS); ++ sec_hi = hns3_read_dev(hw, HNS3_CURR_TIME_OUT_H) & HNS3_PTP_SEC_H_MASK; ++ sec_lo = hns3_read_dev(hw, HNS3_CURR_TIME_OUT_L); ++ sec = ((uint64_t)sec_hi << HNS3_PTP_SEC_H_OFFSET) | sec_lo; ++ + ns += sec * NSEC_PER_SEC; + *ts = rte_ns_to_timespec(ns); + +@@ -237,17 +259,11 @@ int + hns3_timesync_write_time(struct rte_eth_dev *dev, const struct timespec *ts) + { + struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private); +- uint64_t sec = ts->tv_sec; +- uint64_t ns = ts->tv_nsec; + + if (!hns3_dev_get_support(hw, PTP)) + return -ENOTSUP; + +- /* Set the timecounters to a new value. */ +- hns3_write_dev(hw, HNS3_CFG_TIME_SYNC_H, upper_32_bits(sec)); +- hns3_write_dev(hw, HNS3_CFG_TIME_SYNC_M, lower_32_bits(sec)); +- hns3_write_dev(hw, HNS3_CFG_TIME_SYNC_L, lower_32_bits(ns)); +- hns3_write_dev(hw, HNS3_CFG_TIME_SYNC_RDY, 1); ++ hns3_ptp_timesync_write_time(hw, ts); + + return 0; + } +@@ -290,3 +306,21 @@ hns3_restore_ptp(struct hns3_adapter *hns) + + return ret; + } ++ ++void ++hns3_ptp_uninit(struct hns3_hw *hw) ++{ ++ struct hns3_adapter *hns = HNS3_DEV_HW_TO_ADAPTER(hw); ++ int ret; ++ ++ if (!hns3_dev_get_support(hw, PTP)) ++ return; ++ ++ ret = hns3_ptp_int_en(hw, false); ++ if (ret != 0) ++ hns3_err(hw, "disable PTP interrupt failed, ret = %d.", ret); ++ ++ ret = hns3_timesync_configure(hns, false); ++ if (ret != 0) ++ hns3_err(hw, "disable timesync failed, ret = %d.", ret); ++} +diff --git a/dpdk/drivers/net/hns3/hns3_ptp.h b/dpdk/drivers/net/hns3/hns3_ptp.h +new file mode 100644 +index 0000000000..2b8717fa3c +--- /dev/null ++++ b/dpdk/drivers/net/hns3/hns3_ptp.h +@@ -0,0 +1,48 @@ ++/* SPDX-License-Identifier: BSD-3-Clause ++ * Copyright(c) 2023 HiSilicon Limited. ++ */ ++ ++#ifndef HNS3_PTP_H ++#define HNS3_PTP_H ++ ++/* Register bit for 1588 event */ ++#define HNS3_VECTOR0_1588_INT_B 0 ++ ++#define HNS3_PTP_BASE_ADDRESS 0x29000 ++ ++#define HNS3_TX_1588_SEQID_BACK (HNS3_PTP_BASE_ADDRESS + 0x0) ++#define HNS3_TX_1588_TSP_BACK_0 (HNS3_PTP_BASE_ADDRESS + 0x4) ++#define HNS3_TX_1588_TSP_BACK_1 (HNS3_PTP_BASE_ADDRESS + 0x8) ++#define HNS3_TX_1588_TSP_BACK_2 (HNS3_PTP_BASE_ADDRESS + 0xc) ++ ++#define HNS3_TX_1588_BACK_TSP_CNT (HNS3_PTP_BASE_ADDRESS + 0x30) ++ ++#define HNS3_CFG_TIME_SYNC_H (HNS3_PTP_BASE_ADDRESS + 0x50) ++#define HNS3_CFG_TIME_SYNC_M (HNS3_PTP_BASE_ADDRESS + 0x54) ++#define HNS3_CFG_TIME_SYNC_L (HNS3_PTP_BASE_ADDRESS + 0x58) ++#define HNS3_CFG_TIME_SYNC_RDY (HNS3_PTP_BASE_ADDRESS + 0x5c) ++ ++#define HNS3_CFG_TIME_CYC_EN (HNS3_PTP_BASE_ADDRESS + 0x70) ++ ++#define HNS3_CURR_TIME_OUT_H (HNS3_PTP_BASE_ADDRESS + 0x74) ++#define HNS3_CURR_TIME_OUT_L (HNS3_PTP_BASE_ADDRESS + 0x78) ++#define HNS3_CURR_TIME_OUT_NS (HNS3_PTP_BASE_ADDRESS + 0x7c) ++ ++int hns3_restore_ptp(struct hns3_adapter *hns); ++int hns3_mbuf_dyn_rx_timestamp_register(struct rte_eth_dev *dev, ++ struct rte_eth_conf *conf); ++int hns3_ptp_init(struct hns3_hw *hw); ++void hns3_ptp_uninit(struct hns3_hw *hw); ++int hns3_timesync_enable(struct rte_eth_dev *dev); ++int hns3_timesync_disable(struct rte_eth_dev *dev); ++int hns3_timesync_read_rx_timestamp(struct rte_eth_dev *dev, ++ struct timespec *timestamp, ++ uint32_t flags __rte_unused); ++int hns3_timesync_read_tx_timestamp(struct rte_eth_dev *dev, ++ struct timespec *timestamp); ++int hns3_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts); ++int hns3_timesync_write_time(struct rte_eth_dev *dev, ++ const struct timespec *ts); ++int hns3_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta); ++ ++#endif /* HNS3_PTP_H */ +diff --git a/dpdk/drivers/net/hns3/hns3_regs.c b/dpdk/drivers/net/hns3/hns3_regs.c +index 33392fd1f0..5d6f92e4bb 100644 +--- a/dpdk/drivers/net/hns3/hns3_regs.c ++++ b/dpdk/drivers/net/hns3/hns3_regs.c +@@ -294,8 +294,9 @@ hns3_direct_access_regs(struct hns3_hw *hw, uint32_t *data) + struct hns3_adapter *hns = HNS3_DEV_HW_TO_ADAPTER(hw); + uint32_t *origin_data_ptr = data; + uint32_t reg_offset; +- uint16_t i, j; + size_t reg_num; ++ uint16_t j; ++ size_t i; + + /* fetching per-PF registers values from PF PCIe register space */ + reg_num = sizeof(cmdq_reg_addrs) / sizeof(uint32_t); +diff --git a/dpdk/drivers/net/hns3/hns3_regs.h b/dpdk/drivers/net/hns3/hns3_regs.h +index 459bbaf773..6b037f81c1 100644 +--- a/dpdk/drivers/net/hns3/hns3_regs.h ++++ b/dpdk/drivers/net/hns3/hns3_regs.h +@@ -124,29 +124,6 @@ + #define HNS3_TQP_INTR_RL_DEFAULT 0 + #define HNS3_TQP_INTR_QL_DEFAULT 0 + +-/* Register bit for 1588 event */ +-#define HNS3_VECTOR0_1588_INT_B 0 +- +-#define HNS3_PTP_BASE_ADDRESS 0x29000 +- +-#define HNS3_TX_1588_SEQID_BACK (HNS3_PTP_BASE_ADDRESS + 0x0) +-#define HNS3_TX_1588_TSP_BACK_0 (HNS3_PTP_BASE_ADDRESS + 0x4) +-#define HNS3_TX_1588_TSP_BACK_1 (HNS3_PTP_BASE_ADDRESS + 0x8) +-#define HNS3_TX_1588_TSP_BACK_2 (HNS3_PTP_BASE_ADDRESS + 0xc) +- +-#define HNS3_TX_1588_BACK_TSP_CNT (HNS3_PTP_BASE_ADDRESS + 0x30) +- +-#define HNS3_CFG_TIME_SYNC_H (HNS3_PTP_BASE_ADDRESS + 0x50) +-#define HNS3_CFG_TIME_SYNC_M (HNS3_PTP_BASE_ADDRESS + 0x54) +-#define HNS3_CFG_TIME_SYNC_L (HNS3_PTP_BASE_ADDRESS + 0x58) +-#define HNS3_CFG_TIME_SYNC_RDY (HNS3_PTP_BASE_ADDRESS + 0x5c) +- +-#define HNS3_CFG_TIME_CYC_EN (HNS3_PTP_BASE_ADDRESS + 0x70) +- +-#define HNS3_CURR_TIME_OUT_H (HNS3_PTP_BASE_ADDRESS + 0x74) +-#define HNS3_CURR_TIME_OUT_L (HNS3_PTP_BASE_ADDRESS + 0x78) +-#define HNS3_CURR_TIME_OUT_NS (HNS3_PTP_BASE_ADDRESS + 0x7c) +- + /* gl_usec convert to hardware count, as writing each 1 represents 2us */ + #define HNS3_GL_USEC_TO_REG(gl_usec) ((gl_usec) >> 1) + /* rl_usec convert to hardware count, as writing each 1 represents 4us */ +diff --git a/dpdk/drivers/net/hns3/hns3_rss.c b/dpdk/drivers/net/hns3/hns3_rss.c +index ca5a129234..eeeca71a5c 100644 +--- a/dpdk/drivers/net/hns3/hns3_rss.c ++++ b/dpdk/drivers/net/hns3/hns3_rss.c +@@ -18,56 +18,11 @@ const uint8_t hns3_hash_key[HNS3_RSS_KEY_SIZE] = { + 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA + }; + +-enum hns3_tuple_field { +- /* IPV4_TCP ENABLE FIELD */ +- HNS3_RSS_FIELD_IPV4_TCP_EN_TCP_D = 0, +- HNS3_RSS_FIELD_IPV4_TCP_EN_TCP_S, +- HNS3_RSS_FIELD_IPV4_TCP_EN_IP_D, +- HNS3_RSS_FIELD_IPV4_TCP_EN_IP_S, +- +- /* IPV4_UDP ENABLE FIELD */ +- HNS3_RSS_FIELD_IPV4_UDP_EN_UDP_D = 8, +- HNS3_RSS_FIELD_IPV4_UDP_EN_UDP_S, +- HNS3_RSS_FIELD_IPV4_UDP_EN_IP_D, +- HNS3_RSS_FIELD_IPV4_UDP_EN_IP_S, +- +- /* IPV4_SCTP ENABLE FIELD */ +- HNS3_RSS_FIELD_IPV4_SCTP_EN_SCTP_D = 16, +- HNS3_RSS_FIELD_IPV4_SCTP_EN_SCTP_S, +- HNS3_RSS_FIELD_IPV4_SCTP_EN_IP_D, +- HNS3_RSS_FIELD_IPV4_SCTP_EN_IP_S, +- HNS3_RSS_FIELD_IPV4_SCTP_EN_SCTP_VER, +- +- /* IPV4 ENABLE FIELD */ +- HNS3_RSS_FIELD_IPV4_EN_NONFRAG_IP_D = 24, +- HNS3_RSS_FIELD_IPV4_EN_NONFRAG_IP_S, +- HNS3_RSS_FIELD_IPV4_EN_FRAG_IP_D, +- HNS3_RSS_FIELD_IPV4_EN_FRAG_IP_S, +- +- /* IPV6_TCP ENABLE FIELD */ +- HNS3_RSS_FIELD_IPV6_TCP_EN_TCP_D = 32, +- HNS3_RSS_FIELD_IPV6_TCP_EN_TCP_S, +- HNS3_RSS_FIELD_IPV6_TCP_EN_IP_D, +- HNS3_RSS_FIELD_IPV6_TCP_EN_IP_S, +- +- /* IPV6_UDP ENABLE FIELD */ +- HNS3_RSS_FIELD_IPV6_UDP_EN_UDP_D = 40, +- HNS3_RSS_FIELD_IPV6_UDP_EN_UDP_S, +- HNS3_RSS_FIELD_IPV6_UDP_EN_IP_D, +- HNS3_RSS_FIELD_IPV6_UDP_EN_IP_S, +- +- /* IPV6_SCTP ENABLE FIELD */ +- HNS3_RSS_FIELD_IPV6_SCTP_EN_SCTP_D = 48, +- HNS3_RSS_FIELD_IPV6_SCTP_EN_SCTP_S, +- HNS3_RSS_FIELD_IPV6_SCTP_EN_IP_D, +- HNS3_RSS_FIELD_IPV6_SCTP_EN_IP_S, +- HNS3_RSS_FIELD_IPV6_SCTP_EN_SCTP_VER, +- +- /* IPV6 ENABLE FIELD */ +- HNS3_RSS_FIELD_IPV6_NONFRAG_IP_D = 56, +- HNS3_RSS_FIELD_IPV6_NONFRAG_IP_S, +- HNS3_RSS_FIELD_IPV6_FRAG_IP_D, +- HNS3_RSS_FIELD_IPV6_FRAG_IP_S ++const uint8_t hns3_hash_func_map[] = { ++ [RTE_ETH_HASH_FUNCTION_DEFAULT] = HNS3_RSS_HASH_ALGO_TOEPLITZ, ++ [RTE_ETH_HASH_FUNCTION_TOEPLITZ] = HNS3_RSS_HASH_ALGO_TOEPLITZ, ++ [RTE_ETH_HASH_FUNCTION_SIMPLE_XOR] = HNS3_RSS_HASH_ALGO_SIMPLE, ++ [RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ] = HNS3_RSS_HASH_ALGO_SYMMETRIC_TOEP, + }; + + enum hns3_rss_tuple_type { +@@ -79,243 +34,285 @@ static const struct { + uint64_t rss_types; + uint16_t tuple_type; + uint64_t rss_field; ++ uint64_t tuple_mask; + } hns3_set_tuple_table[] = { + /* IPV4-FRAG */ + { RTE_ETH_RSS_FRAG_IPV4 | RTE_ETH_RSS_L3_SRC_ONLY, + HNS3_RSS_IP_TUPLE, +- BIT_ULL(HNS3_RSS_FIELD_IPV4_EN_FRAG_IP_S) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV4_EN_FRAG_IP_S), ++ HNS3_RSS_TUPLE_IPV4_FLAG_M }, + { RTE_ETH_RSS_FRAG_IPV4 | RTE_ETH_RSS_L3_DST_ONLY, + HNS3_RSS_IP_TUPLE, +- BIT_ULL(HNS3_RSS_FIELD_IPV4_EN_FRAG_IP_D) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV4_EN_FRAG_IP_D), ++ HNS3_RSS_TUPLE_IPV4_FLAG_M }, + { RTE_ETH_RSS_FRAG_IPV4, + HNS3_RSS_IP_TUPLE, + BIT_ULL(HNS3_RSS_FIELD_IPV4_EN_FRAG_IP_S) | +- BIT_ULL(HNS3_RSS_FIELD_IPV4_EN_FRAG_IP_D) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV4_EN_FRAG_IP_D), ++ HNS3_RSS_TUPLE_IPV4_FLAG_M }, + + /* IPV4 */ + { RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_L3_SRC_ONLY, + HNS3_RSS_IP_TUPLE, +- BIT_ULL(HNS3_RSS_FIELD_IPV4_EN_NONFRAG_IP_S) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV4_EN_NONFRAG_IP_S), ++ HNS3_RSS_TUPLE_IPV4_NONF_M }, + { RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_L3_DST_ONLY, + HNS3_RSS_IP_TUPLE, +- BIT_ULL(HNS3_RSS_FIELD_IPV4_EN_NONFRAG_IP_D) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV4_EN_NONFRAG_IP_D), ++ HNS3_RSS_TUPLE_IPV4_NONF_M }, + { RTE_ETH_RSS_IPV4, + HNS3_RSS_IP_TUPLE, + BIT_ULL(HNS3_RSS_FIELD_IPV4_EN_NONFRAG_IP_S) | +- BIT_ULL(HNS3_RSS_FIELD_IPV4_EN_NONFRAG_IP_D) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV4_EN_NONFRAG_IP_D), ++ HNS3_RSS_TUPLE_IPV4_NONF_M }, + + /* IPV4-OTHER */ + { RTE_ETH_RSS_NONFRAG_IPV4_OTHER | RTE_ETH_RSS_L3_SRC_ONLY, + HNS3_RSS_IP_TUPLE, +- BIT_ULL(HNS3_RSS_FIELD_IPV4_EN_NONFRAG_IP_S) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV4_EN_NONFRAG_IP_S), ++ HNS3_RSS_TUPLE_IPV4_NONF_M }, + { RTE_ETH_RSS_NONFRAG_IPV4_OTHER | RTE_ETH_RSS_L3_DST_ONLY, + HNS3_RSS_IP_TUPLE, +- BIT_ULL(HNS3_RSS_FIELD_IPV4_EN_NONFRAG_IP_D) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV4_EN_NONFRAG_IP_D), ++ HNS3_RSS_TUPLE_IPV4_NONF_M }, + { RTE_ETH_RSS_NONFRAG_IPV4_OTHER, + HNS3_RSS_IP_TUPLE, + BIT_ULL(HNS3_RSS_FIELD_IPV4_EN_NONFRAG_IP_S) | +- BIT_ULL(HNS3_RSS_FIELD_IPV4_EN_NONFRAG_IP_D) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV4_EN_NONFRAG_IP_D), ++ HNS3_RSS_TUPLE_IPV4_NONF_M }, + + /* IPV4-TCP */ + { RTE_ETH_RSS_NONFRAG_IPV4_TCP | RTE_ETH_RSS_L3_SRC_ONLY, + HNS3_RSS_IP_L4_TUPLE, +- BIT_ULL(HNS3_RSS_FIELD_IPV4_TCP_EN_IP_S) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV4_TCP_EN_IP_S), ++ HNS3_RSS_TUPLE_IPV4_TCP_M }, + { RTE_ETH_RSS_NONFRAG_IPV4_TCP | RTE_ETH_RSS_L3_DST_ONLY, + HNS3_RSS_IP_L4_TUPLE, +- BIT_ULL(HNS3_RSS_FIELD_IPV4_TCP_EN_IP_D) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV4_TCP_EN_IP_D), ++ HNS3_RSS_TUPLE_IPV4_TCP_M }, + { RTE_ETH_RSS_NONFRAG_IPV4_TCP | RTE_ETH_RSS_L4_SRC_ONLY, + HNS3_RSS_IP_L4_TUPLE, +- BIT_ULL(HNS3_RSS_FIELD_IPV4_TCP_EN_TCP_S) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV4_TCP_EN_TCP_S), ++ HNS3_RSS_TUPLE_IPV4_TCP_M }, + { RTE_ETH_RSS_NONFRAG_IPV4_TCP | RTE_ETH_RSS_L4_DST_ONLY, + HNS3_RSS_IP_L4_TUPLE, +- BIT_ULL(HNS3_RSS_FIELD_IPV4_TCP_EN_TCP_D) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV4_TCP_EN_TCP_D), ++ HNS3_RSS_TUPLE_IPV4_TCP_M }, + { RTE_ETH_RSS_NONFRAG_IPV4_TCP, + HNS3_RSS_IP_L4_TUPLE, + BIT_ULL(HNS3_RSS_FIELD_IPV4_TCP_EN_IP_S) | + BIT_ULL(HNS3_RSS_FIELD_IPV4_TCP_EN_IP_D) | + BIT_ULL(HNS3_RSS_FIELD_IPV4_TCP_EN_TCP_S) | +- BIT_ULL(HNS3_RSS_FIELD_IPV4_TCP_EN_TCP_D) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV4_TCP_EN_TCP_D), ++ HNS3_RSS_TUPLE_IPV4_TCP_M }, + + /* IPV4-UDP */ + { RTE_ETH_RSS_NONFRAG_IPV4_UDP | RTE_ETH_RSS_L3_SRC_ONLY, + HNS3_RSS_IP_L4_TUPLE, +- BIT_ULL(HNS3_RSS_FIELD_IPV4_UDP_EN_IP_S) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV4_UDP_EN_IP_S), ++ HNS3_RSS_TUPLE_IPV4_UDP_M }, + { RTE_ETH_RSS_NONFRAG_IPV4_UDP | RTE_ETH_RSS_L3_DST_ONLY, + HNS3_RSS_IP_L4_TUPLE, +- BIT_ULL(HNS3_RSS_FIELD_IPV4_UDP_EN_IP_D) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV4_UDP_EN_IP_D), ++ HNS3_RSS_TUPLE_IPV4_UDP_M }, + { RTE_ETH_RSS_NONFRAG_IPV4_UDP | RTE_ETH_RSS_L4_SRC_ONLY, + HNS3_RSS_IP_L4_TUPLE, +- BIT_ULL(HNS3_RSS_FIELD_IPV4_UDP_EN_UDP_S) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV4_UDP_EN_UDP_S), ++ HNS3_RSS_TUPLE_IPV4_UDP_M }, + { RTE_ETH_RSS_NONFRAG_IPV4_UDP | RTE_ETH_RSS_L4_DST_ONLY, + HNS3_RSS_IP_L4_TUPLE, +- BIT_ULL(HNS3_RSS_FIELD_IPV4_UDP_EN_UDP_D) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV4_UDP_EN_UDP_D), ++ HNS3_RSS_TUPLE_IPV4_UDP_M }, + { RTE_ETH_RSS_NONFRAG_IPV4_UDP, + HNS3_RSS_IP_L4_TUPLE, + BIT_ULL(HNS3_RSS_FIELD_IPV4_UDP_EN_IP_S) | + BIT_ULL(HNS3_RSS_FIELD_IPV4_UDP_EN_IP_D) | + BIT_ULL(HNS3_RSS_FIELD_IPV4_UDP_EN_UDP_S) | +- BIT_ULL(HNS3_RSS_FIELD_IPV4_UDP_EN_UDP_D) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV4_UDP_EN_UDP_D), ++ HNS3_RSS_TUPLE_IPV4_UDP_M }, + + /* IPV4-SCTP */ + { RTE_ETH_RSS_NONFRAG_IPV4_SCTP | RTE_ETH_RSS_L3_SRC_ONLY, + HNS3_RSS_IP_L4_TUPLE, +- BIT_ULL(HNS3_RSS_FIELD_IPV4_SCTP_EN_IP_S) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV4_SCTP_EN_IP_S), ++ HNS3_RSS_TUPLE_IPV4_SCTP_M }, + { RTE_ETH_RSS_NONFRAG_IPV4_SCTP | RTE_ETH_RSS_L3_DST_ONLY, + HNS3_RSS_IP_L4_TUPLE, +- BIT_ULL(HNS3_RSS_FIELD_IPV4_SCTP_EN_IP_D) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV4_SCTP_EN_IP_D), ++ HNS3_RSS_TUPLE_IPV4_SCTP_M }, + { RTE_ETH_RSS_NONFRAG_IPV4_SCTP | RTE_ETH_RSS_L4_SRC_ONLY, + HNS3_RSS_IP_L4_TUPLE, +- BIT_ULL(HNS3_RSS_FIELD_IPV4_SCTP_EN_SCTP_S) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV4_SCTP_EN_SCTP_S), ++ HNS3_RSS_TUPLE_IPV4_SCTP_M }, + { RTE_ETH_RSS_NONFRAG_IPV4_SCTP | RTE_ETH_RSS_L4_DST_ONLY, + HNS3_RSS_IP_L4_TUPLE, +- BIT_ULL(HNS3_RSS_FIELD_IPV4_SCTP_EN_SCTP_D) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV4_SCTP_EN_SCTP_D), ++ HNS3_RSS_TUPLE_IPV4_SCTP_M }, + { RTE_ETH_RSS_NONFRAG_IPV4_SCTP, + HNS3_RSS_IP_L4_TUPLE, + BIT_ULL(HNS3_RSS_FIELD_IPV4_SCTP_EN_IP_S) | + BIT_ULL(HNS3_RSS_FIELD_IPV4_SCTP_EN_IP_D) | + BIT_ULL(HNS3_RSS_FIELD_IPV4_SCTP_EN_SCTP_S) | + BIT_ULL(HNS3_RSS_FIELD_IPV4_SCTP_EN_SCTP_D) | +- BIT_ULL(HNS3_RSS_FIELD_IPV4_SCTP_EN_SCTP_VER) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV4_SCTP_EN_SCTP_VER), ++ HNS3_RSS_TUPLE_IPV4_SCTP_M }, + + /* IPV6-FRAG */ + { RTE_ETH_RSS_FRAG_IPV6 | RTE_ETH_RSS_L3_SRC_ONLY, + HNS3_RSS_IP_TUPLE, +- BIT_ULL(HNS3_RSS_FIELD_IPV6_FRAG_IP_S) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV6_FRAG_IP_S), ++ HNS3_RSS_TUPLE_IPV6_FLAG_M }, + { RTE_ETH_RSS_FRAG_IPV6 | RTE_ETH_RSS_L3_DST_ONLY, + HNS3_RSS_IP_TUPLE, +- BIT_ULL(HNS3_RSS_FIELD_IPV6_FRAG_IP_D) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV6_FRAG_IP_D), ++ HNS3_RSS_TUPLE_IPV6_FLAG_M }, + { RTE_ETH_RSS_FRAG_IPV6, + HNS3_RSS_IP_TUPLE, + BIT_ULL(HNS3_RSS_FIELD_IPV6_FRAG_IP_S) | +- BIT_ULL(HNS3_RSS_FIELD_IPV6_FRAG_IP_D) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV6_FRAG_IP_D), ++ HNS3_RSS_TUPLE_IPV6_FLAG_M }, + + /* IPV6 */ + { RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_L3_SRC_ONLY, + HNS3_RSS_IP_TUPLE, +- BIT_ULL(HNS3_RSS_FIELD_IPV6_NONFRAG_IP_S) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV6_NONFRAG_IP_S), ++ HNS3_RSS_TUPLE_IPV6_NONF_M }, + { RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_L3_DST_ONLY, + HNS3_RSS_IP_TUPLE, +- BIT_ULL(HNS3_RSS_FIELD_IPV6_NONFRAG_IP_D) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV6_NONFRAG_IP_D), ++ HNS3_RSS_TUPLE_IPV6_NONF_M }, + { RTE_ETH_RSS_IPV6, + HNS3_RSS_IP_TUPLE, + BIT_ULL(HNS3_RSS_FIELD_IPV6_NONFRAG_IP_S) | +- BIT_ULL(HNS3_RSS_FIELD_IPV6_NONFRAG_IP_D) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV6_NONFRAG_IP_D), ++ HNS3_RSS_TUPLE_IPV6_NONF_M }, + + /* IPV6-OTHER */ + { RTE_ETH_RSS_NONFRAG_IPV6_OTHER | RTE_ETH_RSS_L3_SRC_ONLY, + HNS3_RSS_IP_TUPLE, +- BIT_ULL(HNS3_RSS_FIELD_IPV6_NONFRAG_IP_S) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV6_NONFRAG_IP_S), ++ HNS3_RSS_TUPLE_IPV6_NONF_M }, + { RTE_ETH_RSS_NONFRAG_IPV6_OTHER | RTE_ETH_RSS_L3_DST_ONLY, + HNS3_RSS_IP_TUPLE, +- BIT_ULL(HNS3_RSS_FIELD_IPV6_NONFRAG_IP_D) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV6_NONFRAG_IP_D), ++ HNS3_RSS_TUPLE_IPV6_NONF_M }, + { RTE_ETH_RSS_NONFRAG_IPV6_OTHER, + HNS3_RSS_IP_TUPLE, + BIT_ULL(HNS3_RSS_FIELD_IPV6_NONFRAG_IP_S) | +- BIT_ULL(HNS3_RSS_FIELD_IPV6_NONFRAG_IP_D) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV6_NONFRAG_IP_D), ++ HNS3_RSS_TUPLE_IPV6_NONF_M }, + + /* IPV6-TCP */ + { RTE_ETH_RSS_NONFRAG_IPV6_TCP | RTE_ETH_RSS_L3_SRC_ONLY, + HNS3_RSS_IP_L4_TUPLE, +- BIT_ULL(HNS3_RSS_FIELD_IPV6_TCP_EN_IP_S) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV6_TCP_EN_IP_S), ++ HNS3_RSS_TUPLE_IPV6_TCP_M }, + { RTE_ETH_RSS_NONFRAG_IPV6_TCP | RTE_ETH_RSS_L3_DST_ONLY, + HNS3_RSS_IP_L4_TUPLE, +- BIT_ULL(HNS3_RSS_FIELD_IPV6_TCP_EN_IP_D) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV6_TCP_EN_IP_D), ++ HNS3_RSS_TUPLE_IPV6_TCP_M }, + { RTE_ETH_RSS_NONFRAG_IPV6_TCP | RTE_ETH_RSS_L4_SRC_ONLY, + HNS3_RSS_IP_L4_TUPLE, +- BIT_ULL(HNS3_RSS_FIELD_IPV6_TCP_EN_TCP_S) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV6_TCP_EN_TCP_S), ++ HNS3_RSS_TUPLE_IPV6_TCP_M }, + { RTE_ETH_RSS_NONFRAG_IPV6_TCP | RTE_ETH_RSS_L4_DST_ONLY, + HNS3_RSS_IP_L4_TUPLE, +- BIT_ULL(HNS3_RSS_FIELD_IPV6_TCP_EN_TCP_D) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV6_TCP_EN_TCP_D), ++ HNS3_RSS_TUPLE_IPV6_TCP_M }, + { RTE_ETH_RSS_NONFRAG_IPV6_TCP, + HNS3_RSS_IP_L4_TUPLE, + BIT_ULL(HNS3_RSS_FIELD_IPV6_TCP_EN_IP_S) | + BIT_ULL(HNS3_RSS_FIELD_IPV6_TCP_EN_IP_D) | + BIT_ULL(HNS3_RSS_FIELD_IPV6_TCP_EN_TCP_S) | +- BIT_ULL(HNS3_RSS_FIELD_IPV6_TCP_EN_TCP_D) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV6_TCP_EN_TCP_D), ++ HNS3_RSS_TUPLE_IPV6_TCP_M }, + + /* IPV6-UDP */ + { RTE_ETH_RSS_NONFRAG_IPV6_UDP | RTE_ETH_RSS_L3_SRC_ONLY, + HNS3_RSS_IP_L4_TUPLE, +- BIT_ULL(HNS3_RSS_FIELD_IPV6_UDP_EN_IP_S) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV6_UDP_EN_IP_S), ++ HNS3_RSS_TUPLE_IPV6_UDP_M }, + { RTE_ETH_RSS_NONFRAG_IPV6_UDP | RTE_ETH_RSS_L3_DST_ONLY, + HNS3_RSS_IP_L4_TUPLE, +- BIT_ULL(HNS3_RSS_FIELD_IPV6_UDP_EN_IP_D) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV6_UDP_EN_IP_D), ++ HNS3_RSS_TUPLE_IPV6_UDP_M }, + { RTE_ETH_RSS_NONFRAG_IPV6_UDP | RTE_ETH_RSS_L4_SRC_ONLY, + HNS3_RSS_IP_L4_TUPLE, +- BIT_ULL(HNS3_RSS_FIELD_IPV6_UDP_EN_UDP_S) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV6_UDP_EN_UDP_S), ++ HNS3_RSS_TUPLE_IPV6_UDP_M }, + { RTE_ETH_RSS_NONFRAG_IPV6_UDP | RTE_ETH_RSS_L4_DST_ONLY, + HNS3_RSS_IP_L4_TUPLE, +- BIT_ULL(HNS3_RSS_FIELD_IPV6_UDP_EN_UDP_D) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV6_UDP_EN_UDP_D), ++ HNS3_RSS_TUPLE_IPV6_UDP_M }, + { RTE_ETH_RSS_NONFRAG_IPV6_UDP, + HNS3_RSS_IP_L4_TUPLE, + BIT_ULL(HNS3_RSS_FIELD_IPV6_UDP_EN_IP_S) | + BIT_ULL(HNS3_RSS_FIELD_IPV6_UDP_EN_IP_D) | + BIT_ULL(HNS3_RSS_FIELD_IPV6_UDP_EN_UDP_S) | +- BIT_ULL(HNS3_RSS_FIELD_IPV6_UDP_EN_UDP_D) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV6_UDP_EN_UDP_D), ++ HNS3_RSS_TUPLE_IPV6_UDP_M }, + + /* IPV6-SCTP */ + { RTE_ETH_RSS_NONFRAG_IPV6_SCTP | RTE_ETH_RSS_L3_SRC_ONLY, + HNS3_RSS_IP_L4_TUPLE, +- BIT_ULL(HNS3_RSS_FIELD_IPV6_SCTP_EN_IP_S) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV6_SCTP_EN_IP_S), ++ HNS3_RSS_TUPLE_IPV6_SCTP_M }, + { RTE_ETH_RSS_NONFRAG_IPV6_SCTP | RTE_ETH_RSS_L3_DST_ONLY, + HNS3_RSS_IP_L4_TUPLE, +- BIT_ULL(HNS3_RSS_FIELD_IPV6_SCTP_EN_IP_D) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV6_SCTP_EN_IP_D), ++ HNS3_RSS_TUPLE_IPV6_SCTP_M }, + { RTE_ETH_RSS_NONFRAG_IPV6_SCTP | RTE_ETH_RSS_L4_SRC_ONLY, + HNS3_RSS_IP_L4_TUPLE, +- BIT_ULL(HNS3_RSS_FIELD_IPV6_SCTP_EN_SCTP_S) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV6_SCTP_EN_SCTP_S), ++ HNS3_RSS_TUPLE_IPV6_SCTP_M }, + { RTE_ETH_RSS_NONFRAG_IPV6_SCTP | RTE_ETH_RSS_L4_DST_ONLY, + HNS3_RSS_IP_L4_TUPLE, +- BIT_ULL(HNS3_RSS_FIELD_IPV6_SCTP_EN_SCTP_D) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV6_SCTP_EN_SCTP_D), ++ HNS3_RSS_TUPLE_IPV6_SCTP_M }, + { RTE_ETH_RSS_NONFRAG_IPV6_SCTP, + HNS3_RSS_IP_L4_TUPLE, + BIT_ULL(HNS3_RSS_FIELD_IPV6_SCTP_EN_IP_S) | + BIT_ULL(HNS3_RSS_FIELD_IPV6_SCTP_EN_IP_D) | + BIT_ULL(HNS3_RSS_FIELD_IPV6_SCTP_EN_SCTP_D) | + BIT_ULL(HNS3_RSS_FIELD_IPV6_SCTP_EN_SCTP_S) | +- BIT_ULL(HNS3_RSS_FIELD_IPV6_SCTP_EN_SCTP_VER) }, ++ BIT_ULL(HNS3_RSS_FIELD_IPV6_SCTP_EN_SCTP_VER), ++ HNS3_RSS_TUPLE_IPV6_SCTP_M }, + }; + + /* + * rss_generic_config command function, opcode:0x0D01. +- * Used to set algorithm, key_offset and hash key of rss. ++ * Used to set algorithm and hash key of RSS. + */ +-int +-hns3_rss_set_algo_key(struct hns3_hw *hw, const uint8_t *key) ++static int ++hns3_rss_set_algo_key(struct hns3_hw *hw, uint8_t hash_algo, ++ const uint8_t *key, uint8_t key_len) + { +-#define HNS3_KEY_OFFSET_MAX 3 +-#define HNS3_SET_HASH_KEY_BYTE_FOUR 2 +- + struct hns3_rss_generic_config_cmd *req; + struct hns3_cmd_desc desc; +- uint32_t key_offset, key_size; +- const uint8_t *key_cur; +- uint8_t cur_offset; ++ const uint8_t *cur_key; ++ uint16_t cur_key_size; ++ uint16_t max_bd_num; ++ uint16_t idx; + int ret; + + req = (struct hns3_rss_generic_config_cmd *)desc.data; + +- /* +- * key_offset=0, hash key byte0~15 is set to hardware. +- * key_offset=1, hash key byte16~31 is set to hardware. +- * key_offset=2, hash key byte32~39 is set to hardware. +- */ +- for (key_offset = 0; key_offset < HNS3_KEY_OFFSET_MAX; key_offset++) { ++ max_bd_num = DIV_ROUND_UP(key_len, HNS3_RSS_HASH_KEY_NUM); ++ for (idx = 0; idx < max_bd_num; idx++) { + hns3_cmd_setup_basic_desc(&desc, HNS3_OPC_RSS_GENERIC_CONFIG, + false); + +- req->hash_config |= +- (hw->rss_info.hash_algo & HNS3_RSS_HASH_ALGO_MASK); +- req->hash_config |= (key_offset << HNS3_RSS_HASH_KEY_OFFSET_B); ++ req->hash_config |= (hash_algo & HNS3_RSS_HASH_ALGO_MASK); ++ req->hash_config |= (idx << HNS3_RSS_HASH_KEY_OFFSET_B); + +- if (key_offset == HNS3_SET_HASH_KEY_BYTE_FOUR) +- key_size = HNS3_RSS_KEY_SIZE - HNS3_RSS_HASH_KEY_NUM * +- HNS3_SET_HASH_KEY_BYTE_FOUR; ++ if (idx == max_bd_num - 1 && ++ (key_len % HNS3_RSS_HASH_KEY_NUM) != 0) ++ cur_key_size = key_len % HNS3_RSS_HASH_KEY_NUM; + else +- key_size = HNS3_RSS_HASH_KEY_NUM; ++ cur_key_size = HNS3_RSS_HASH_KEY_NUM; + +- cur_offset = key_offset * HNS3_RSS_HASH_KEY_NUM; +- key_cur = key + cur_offset; +- memcpy(req->hash_key, key_cur, key_size); ++ cur_key = key + idx * HNS3_RSS_HASH_KEY_NUM; ++ memcpy(req->hash_key, cur_key, cur_key_size); + + ret = hns3_cmd_send(hw, &desc, 1); + if (ret) { +@@ -323,8 +320,49 @@ hns3_rss_set_algo_key(struct hns3_hw *hw, const uint8_t *key) + return ret; + } + } +- /* Update the shadow RSS key with user specified */ +- memcpy(hw->rss_info.key, key, HNS3_RSS_KEY_SIZE); ++ ++ return 0; ++} ++ ++static int ++hns3_rss_get_algo_key(struct hns3_hw *hw, uint8_t *hash_algo, ++ uint8_t *key, uint8_t key_len) ++{ ++ struct hns3_rss_generic_config_cmd *req; ++ struct hns3_cmd_desc desc; ++ uint16_t cur_key_size; ++ uint16_t max_bd_num; ++ uint8_t *cur_key; ++ uint16_t idx; ++ int ret; ++ ++ req = (struct hns3_rss_generic_config_cmd *)desc.data; ++ max_bd_num = DIV_ROUND_UP(key_len, HNS3_RSS_HASH_KEY_NUM); ++ for (idx = 0; idx < max_bd_num; idx++) { ++ hns3_cmd_setup_basic_desc(&desc, HNS3_OPC_RSS_GENERIC_CONFIG, ++ true); ++ ++ req->hash_config |= (idx << HNS3_RSS_HASH_KEY_OFFSET_B); ++ ret = hns3_cmd_send(hw, &desc, 1); ++ if (ret) { ++ hns3_err(hw, "fail to obtain RSS algo and key from firmware, ret = %d", ++ ret); ++ return ret; ++ } ++ ++ if (idx == 0) ++ *hash_algo = req->hash_config & HNS3_RSS_HASH_ALGO_MASK; ++ ++ if (idx == max_bd_num - 1 && ++ (key_len % HNS3_RSS_HASH_KEY_NUM) != 0) ++ cur_key_size = key_len % HNS3_RSS_HASH_KEY_NUM; ++ else ++ cur_key_size = HNS3_RSS_HASH_KEY_NUM; ++ ++ cur_key = key + idx * HNS3_RSS_HASH_KEY_NUM; ++ memcpy(cur_key, req->hash_key, cur_key_size); ++ } ++ + return 0; + } + +@@ -336,6 +374,7 @@ int + hns3_set_rss_indir_table(struct hns3_hw *hw, uint16_t *indir, uint16_t size) + { + struct hns3_rss_indirection_table_cmd *req; ++ uint16_t max_bd_num, cfg_tbl_size; + struct hns3_cmd_desc desc; + uint8_t qid_msb_off; + uint8_t qid_msb_val; +@@ -344,14 +383,20 @@ hns3_set_rss_indir_table(struct hns3_hw *hw, uint16_t *indir, uint16_t size) + int ret; + + req = (struct hns3_rss_indirection_table_cmd *)desc.data; +- +- for (i = 0; i < size / HNS3_RSS_CFG_TBL_SIZE; i++) { ++ max_bd_num = DIV_ROUND_UP(size, HNS3_RSS_CFG_TBL_SIZE); ++ for (i = 0; i < max_bd_num; i++) { + hns3_cmd_setup_basic_desc(&desc, HNS3_OPC_RSS_INDIR_TABLE, + false); + req->start_table_index = + rte_cpu_to_le_16(i * HNS3_RSS_CFG_TBL_SIZE); + req->rss_set_bitmap = rte_cpu_to_le_16(HNS3_RSS_SET_BITMAP_MSK); +- for (j = 0; j < HNS3_RSS_CFG_TBL_SIZE; j++) { ++ ++ if (i == max_bd_num - 1 && (size % HNS3_RSS_CFG_TBL_SIZE) != 0) ++ cfg_tbl_size = size % HNS3_RSS_CFG_TBL_SIZE; ++ else ++ cfg_tbl_size = HNS3_RSS_CFG_TBL_SIZE; ++ ++ for (j = 0; j < cfg_tbl_size; j++) { + q_id = indir[i * HNS3_RSS_CFG_TBL_SIZE + j]; + req->rss_result_l[j] = q_id & 0xff; + +@@ -372,9 +417,53 @@ hns3_set_rss_indir_table(struct hns3_hw *hw, uint16_t *indir, uint16_t size) + } + } + +- /* Update redirection table of hw */ +- memcpy(hw->rss_info.rss_indirection_tbl, indir, +- sizeof(uint16_t) * size); ++ return 0; ++} ++ ++static int ++hns3_get_rss_indir_table(struct hns3_hw *hw, uint16_t *indir, uint16_t size) ++{ ++ struct hns3_rss_indirection_table_cmd *req; ++ uint16_t max_bd_num, cfg_tbl_size; ++ uint8_t qid_msb_off, qid_msb_idx; ++ struct hns3_cmd_desc desc; ++ uint16_t q_id, q_hi, q_lo; ++ uint8_t rss_result_h; ++ uint16_t i, j; ++ int ret; ++ ++ req = (struct hns3_rss_indirection_table_cmd *)desc.data; ++ max_bd_num = DIV_ROUND_UP(size, HNS3_RSS_CFG_TBL_SIZE); ++ for (i = 0; i < max_bd_num; i++) { ++ hns3_cmd_setup_basic_desc(&desc, HNS3_OPC_RSS_INDIR_TABLE, ++ true); ++ req->start_table_index = ++ rte_cpu_to_le_16(i * HNS3_RSS_CFG_TBL_SIZE); ++ ret = hns3_cmd_send(hw, &desc, 1); ++ if (ret) { ++ hns3_err(hw, "fail to get RSS indirection table from firmware, ret = %d", ++ ret); ++ return ret; ++ } ++ ++ if (i == max_bd_num - 1 && (size % HNS3_RSS_CFG_TBL_SIZE) != 0) ++ cfg_tbl_size = size % HNS3_RSS_CFG_TBL_SIZE; ++ else ++ cfg_tbl_size = HNS3_RSS_CFG_TBL_SIZE; ++ ++ for (j = 0; j < cfg_tbl_size; j++) { ++ qid_msb_idx = ++ j * HNS3_RSS_CFG_TBL_BW_H / HNS3_BITS_PER_BYTE; ++ rss_result_h = req->rss_result_h[qid_msb_idx]; ++ qid_msb_off = ++ j * HNS3_RSS_CFG_TBL_BW_H % HNS3_BITS_PER_BYTE; ++ q_hi = (rss_result_h >> qid_msb_off) & ++ HNS3_RSS_CFG_TBL_BW_H_M; ++ q_lo = req->rss_result_l[j]; ++ q_id = (q_hi << HNS3_RSS_CFG_TBL_BW_L) | q_lo; ++ indir[i * HNS3_RSS_CFG_TBL_SIZE + j] = q_id; ++ } ++ } + + return 0; + } +@@ -393,41 +482,72 @@ hns3_rss_reset_indir_table(struct hns3_hw *hw) + } + + ret = hns3_set_rss_indir_table(hw, lut, hw->rss_ind_tbl_size); +- if (ret) +- hns3_err(hw, "RSS uninit indir table failed: %d", ret); ++ if (ret != 0) ++ hns3_err(hw, "RSS uninit indir table failed, ret = %d.", ret); ++ else ++ memcpy(hw->rss_info.rss_indirection_tbl, lut, ++ sizeof(uint16_t) * hw->rss_ind_tbl_size); + rte_free(lut); + + return ret; + } + +-static void +-hns3_rss_check_l3l4_types(struct hns3_hw *hw, uint64_t rss_hf) ++bool ++hns3_check_rss_types_valid(struct hns3_hw *hw, uint64_t types) + { + uint64_t ip_mask = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | + RTE_ETH_RSS_NONFRAG_IPV4_OTHER | + RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | + RTE_ETH_RSS_NONFRAG_IPV6_OTHER; +- uint64_t l4_mask = RTE_ETH_RSS_NONFRAG_IPV4_TCP | +- RTE_ETH_RSS_NONFRAG_IPV4_UDP | +- RTE_ETH_RSS_NONFRAG_IPV4_SCTP | +- RTE_ETH_RSS_NONFRAG_IPV6_TCP | +- RTE_ETH_RSS_NONFRAG_IPV6_UDP | +- RTE_ETH_RSS_NONFRAG_IPV6_SCTP; +- uint64_t l3_src_dst_mask = RTE_ETH_RSS_L3_SRC_ONLY | +- RTE_ETH_RSS_L3_DST_ONLY; +- uint64_t l4_src_dst_mask = RTE_ETH_RSS_L4_SRC_ONLY | +- RTE_ETH_RSS_L4_DST_ONLY; +- +- if (rss_hf & l3_src_dst_mask && +- !(rss_hf & ip_mask || rss_hf & l4_mask)) +- hns3_warn(hw, "packet type isn't specified, L3_SRC/DST_ONLY is ignored."); +- +- if (rss_hf & l4_src_dst_mask && !(rss_hf & l4_mask)) +- hns3_warn(hw, "packet type isn't specified, L4_SRC/DST_ONLY is ignored."); ++ uint64_t ip_l4_mask = RTE_ETH_RSS_NONFRAG_IPV4_TCP | ++ RTE_ETH_RSS_NONFRAG_IPV4_UDP | ++ RTE_ETH_RSS_NONFRAG_IPV4_SCTP | ++ RTE_ETH_RSS_NONFRAG_IPV6_TCP | ++ RTE_ETH_RSS_NONFRAG_IPV6_UDP | ++ RTE_ETH_RSS_NONFRAG_IPV6_SCTP; ++ bool has_l4_src_dst = !!(types & HNS3_RSS_SUPPORT_L4_SRC_DST); ++ bool has_ip_pkt = !!(types & ip_mask); ++ uint64_t final_types; ++ ++ if (types == 0) ++ return true; ++ ++ if ((types & HNS3_ETH_RSS_SUPPORT) == 0) { ++ hns3_err(hw, "specified types(0x%" PRIx64 ") are unsupported.", ++ types); ++ return false; ++ } ++ ++ if ((types & HNS3_RSS_SUPPORT_L3_SRC_DST) != 0 && ++ (types & HNS3_RSS_SUPPORT_FLOW_TYPE) == 0) { ++ hns3_err(hw, "IP or IP-TCP/UDP/SCTP packet type isn't specified, L3_SRC/DST_ONLY cannot be set."); ++ return false; ++ } ++ ++ if (has_l4_src_dst && (types & ip_l4_mask) == 0) { ++ if (!has_ip_pkt) { ++ hns3_err(hw, "IP-TCP/UDP/SCTP packet type isn't specified, L4_SRC/DST_ONLY cannot be set."); ++ return false; ++ } ++ /* ++ * For the case that the types has L4_SRC/DST_ONLY but hasn't ++ * IP-TCP/UDP/SCTP packet type, this types is considered valid ++ * if it also has IP packet type. ++ */ ++ hns3_warn(hw, "L4_SRC/DST_ONLY is ignored because of no including L4 packet."); ++ } ++ ++ if ((types & ~HNS3_ETH_RSS_SUPPORT) != 0) { ++ final_types = types & HNS3_ETH_RSS_SUPPORT; ++ hns3_warn(hw, "set RSS types based on hardware support, requested:0x%" PRIx64 " configured:0x%" PRIx64 "", ++ types, final_types); ++ } ++ ++ return true; + } + +-static uint64_t +-hns3_rss_calc_tuple_filed(struct hns3_hw *hw, uint64_t rss_hf) ++uint64_t ++hns3_rss_calc_tuple_filed(uint64_t rss_hf) + { + uint64_t l3_only_mask = RTE_ETH_RSS_L3_SRC_ONLY | + RTE_ETH_RSS_L3_DST_ONLY; +@@ -456,34 +576,40 @@ hns3_rss_calc_tuple_filed(struct hns3_hw *hw, uint64_t rss_hf) + !has_l3_l4_only) + tuple |= hns3_set_tuple_table[i].rss_field; + } +- hns3_rss_check_l3l4_types(hw, rss_hf); + + return tuple; + } + + int +-hns3_set_rss_tuple_by_rss_hf(struct hns3_hw *hw, uint64_t rss_hf) ++hns3_set_rss_tuple_field(struct hns3_hw *hw, uint64_t tuple_fields) + { + struct hns3_rss_input_tuple_cmd *req; + struct hns3_cmd_desc desc; +- uint64_t tuple_field; + int ret; + + hns3_cmd_setup_basic_desc(&desc, HNS3_OPC_RSS_INPUT_TUPLE, false); + req = (struct hns3_rss_input_tuple_cmd *)desc.data; +- +- tuple_field = hns3_rss_calc_tuple_filed(hw, rss_hf); +- req->tuple_field = rte_cpu_to_le_64(tuple_field); ++ req->tuple_field = rte_cpu_to_le_64(tuple_fields); + ret = hns3_cmd_send(hw, &desc, 1); +- if (ret) { +- hns3_err(hw, "Update RSS flow types tuples failed %d", ret); +- return ret; +- } ++ if (ret != 0) ++ hns3_err(hw, "set RSS hash tuple fields failed ret = %d", ret); + +- /* Update supported flow types when set tuple success */ +- hw->rss_info.conf.types = rss_hf; ++ return ret; ++} + +- return 0; ++int ++hns3_set_rss_tuple_by_rss_hf(struct hns3_hw *hw, uint64_t rss_hf) ++{ ++ uint64_t tuple_fields; ++ int ret; ++ ++ tuple_fields = hns3_rss_calc_tuple_filed(rss_hf); ++ ret = hns3_set_rss_tuple_field(hw, tuple_fields); ++ if (ret != 0) ++ hns3_err(hw, "Update RSS flow types tuples failed, ret = %d", ++ ret); ++ ++ return ret; + } + + /* +@@ -500,28 +626,35 @@ hns3_dev_rss_hash_update(struct rte_eth_dev *dev, + struct rte_eth_rss_conf *rss_conf) + { + struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private); +- uint64_t rss_hf_bk = hw->rss_info.conf.types; ++ uint64_t rss_hf_bk = hw->rss_info.rss_hf; + uint8_t key_len = rss_conf->rss_key_len; + uint64_t rss_hf = rss_conf->rss_hf; + uint8_t *key = rss_conf->rss_key; + int ret; + +- if (key && key_len != HNS3_RSS_KEY_SIZE) { ++ if (key && key_len != hw->rss_key_size) { + hns3_err(hw, "the hash key len(%u) is invalid, must be %u", +- key_len, HNS3_RSS_KEY_SIZE); ++ key_len, hw->rss_key_size); + return -EINVAL; + } + ++ if (!hns3_check_rss_types_valid(hw, rss_hf)) ++ return -EINVAL; ++ + rte_spinlock_lock(&hw->lock); + ret = hns3_set_rss_tuple_by_rss_hf(hw, rss_hf); + if (ret) + goto set_tuple_fail; + + if (key) { +- ret = hns3_rss_set_algo_key(hw, key); ++ ret = hns3_rss_set_algo_key(hw, hw->rss_info.hash_algo, ++ key, hw->rss_key_size); + if (ret) + goto set_algo_key_fail; ++ /* Update the shadow RSS key with user specified */ ++ memcpy(hw->rss_info.key, key, hw->rss_key_size); + } ++ hw->rss_info.rss_hf = rss_hf; + rte_spinlock_unlock(&hw->lock); + + return 0; +@@ -533,6 +666,96 @@ set_tuple_fail: + return ret; + } + ++int ++hns3_get_rss_tuple_field(struct hns3_hw *hw, uint64_t *tuple_fields) ++{ ++ struct hns3_rss_input_tuple_cmd *req; ++ struct hns3_cmd_desc desc; ++ int ret; ++ ++ hns3_cmd_setup_basic_desc(&desc, HNS3_OPC_RSS_INPUT_TUPLE, true); ++ req = (struct hns3_rss_input_tuple_cmd *)desc.data; ++ ret = hns3_cmd_send(hw, &desc, 1); ++ if (ret != 0) { ++ hns3_err(hw, "fail to get RSS hash tuple fields from firmware, ret = %d", ++ ret); ++ return ret; ++ } ++ ++ *tuple_fields = rte_le_to_cpu_64(req->tuple_field); ++ ++ return 0; ++} ++ ++static uint64_t ++hns3_rss_tuple_fields_to_rss_hf(struct hns3_hw *hw, uint64_t tuple_fields) ++{ ++ uint64_t ipv6_sctp_l4_mask = ++ BIT_ULL(HNS3_RSS_FIELD_IPV6_SCTP_EN_SCTP_D) | ++ BIT_ULL(HNS3_RSS_FIELD_IPV6_SCTP_EN_SCTP_S); ++ uint64_t rss_hf = 0; ++ uint64_t tuple_mask; ++ uint32_t i; ++ ++ for (i = 0; i < RTE_DIM(hns3_set_tuple_table); i++) { ++ tuple_mask = hns3_set_tuple_table[i].tuple_mask; ++ /* ++ * The RSS hash of the packet type is disabled if its tuples is ++ * zero. ++ */ ++ if ((tuple_fields & tuple_mask) == 0) ++ continue; ++ ++ /* ++ * Some hardware don't support to use src/dst port fields to ++ * hash for IPV6-SCTP packet. ++ */ ++ if ((hns3_set_tuple_table[i].rss_types & ++ RTE_ETH_RSS_NONFRAG_IPV6_SCTP) && ++ !hw->rss_info.ipv6_sctp_offload_supported) ++ tuple_mask &= ~ipv6_sctp_l4_mask; ++ ++ /* ++ * The framework (ethdev ops) or driver (rte flow API) ensure ++ * that both L3_SRC/DST_ONLY and L4_SRC/DST_ONLY cannot be set ++ * to driver at the same time. But if user doesn't specify ++ * anything L3/L4_SRC/DST_ONLY, driver enables all tuple fields. ++ * In this case, driver should not report L3/L4_SRC/DST_ONLY. ++ */ ++ if ((tuple_fields & tuple_mask) == tuple_mask) { ++ /* Skip the item enabled part tuples. */ ++ if ((tuple_fields & hns3_set_tuple_table[i].rss_field) != ++ tuple_mask) ++ continue; ++ ++ rss_hf |= hns3_set_tuple_table[i].rss_types; ++ continue; ++ } ++ ++ /* Match the item enabled part tuples.*/ ++ if ((tuple_fields & hns3_set_tuple_table[i].rss_field) == ++ hns3_set_tuple_table[i].rss_field) ++ rss_hf |= hns3_set_tuple_table[i].rss_types; ++ } ++ ++ return rss_hf; ++} ++ ++static int ++hns3_rss_hash_get_rss_hf(struct hns3_hw *hw, uint64_t *rss_hf) ++{ ++ uint64_t tuple_fields; ++ int ret; ++ ++ ret = hns3_get_rss_tuple_field(hw, &tuple_fields); ++ if (ret != 0) ++ return ret; ++ ++ *rss_hf = hns3_rss_tuple_fields_to_rss_hf(hw, tuple_fields); ++ ++ return 0; ++} ++ + /* + * Get rss key and rss_hf types set of RSS hash configuration. + * @param dev +@@ -548,19 +771,32 @@ hns3_dev_rss_hash_conf_get(struct rte_eth_dev *dev, + { + struct hns3_adapter *hns = dev->data->dev_private; + struct hns3_hw *hw = &hns->hw; +- struct hns3_rss_conf *rss_cfg = &hw->rss_info; ++ uint8_t hash_algo = 0; ++ int ret; + + rte_spinlock_lock(&hw->lock); +- rss_conf->rss_hf = rss_cfg->conf.types; ++ ret = hns3_rss_hash_get_rss_hf(hw, &rss_conf->rss_hf); ++ if (ret != 0) { ++ hns3_err(hw, "obtain hash tuples failed, ret = %d", ret); ++ goto out; ++ } + + /* Get the RSS Key required by the user */ +- if (rss_conf->rss_key && rss_conf->rss_key_len >= HNS3_RSS_KEY_SIZE) { +- memcpy(rss_conf->rss_key, rss_cfg->key, HNS3_RSS_KEY_SIZE); +- rss_conf->rss_key_len = HNS3_RSS_KEY_SIZE; ++ if (rss_conf->rss_key && rss_conf->rss_key_len >= hw->rss_key_size) { ++ ret = hns3_rss_get_algo_key(hw, &hash_algo, rss_conf->rss_key, ++ hw->rss_key_size); ++ if (ret != 0) { ++ hns3_err(hw, "obtain hash algo and key failed, ret = %d", ++ ret); ++ goto out; ++ } ++ rss_conf->rss_key_len = hw->rss_key_size; + } ++ ++out: + rte_spinlock_unlock(&hw->lock); + +- return 0; ++ return ret; + } + + /* +@@ -600,12 +836,12 @@ hns3_dev_rss_reta_update(struct rte_eth_dev *dev, + idx = i / RTE_ETH_RETA_GROUP_SIZE; + shift = i % RTE_ETH_RETA_GROUP_SIZE; + if (reta_conf[idx].reta[shift] >= hw->alloc_rss_size) { +- rte_spinlock_unlock(&hw->lock); + hns3_err(hw, "queue id(%u) set to redirection table " + "exceeds queue number(%u) allocated to a TC", + reta_conf[idx].reta[shift], + hw->alloc_rss_size); +- return -EINVAL; ++ ret = -EINVAL; ++ goto out; + } + + if (reta_conf[idx].mask & (1ULL << shift)) +@@ -614,7 +850,13 @@ hns3_dev_rss_reta_update(struct rte_eth_dev *dev, + + ret = hns3_set_rss_indir_table(hw, indirection_tbl, + hw->rss_ind_tbl_size); ++ if (ret != 0) ++ goto out; ++ ++ memcpy(rss_cfg->rss_indirection_tbl, indirection_tbl, ++ sizeof(uint16_t) * hw->rss_ind_tbl_size); + ++out: + rte_spinlock_unlock(&hw->lock); + return ret; + } +@@ -636,10 +878,11 @@ hns3_dev_rss_reta_query(struct rte_eth_dev *dev, + uint16_t reta_size) + { + struct hns3_adapter *hns = dev->data->dev_private; ++ uint16_t reta_table[HNS3_RSS_IND_TBL_SIZE_MAX]; + struct hns3_hw *hw = &hns->hw; +- struct hns3_rss_conf *rss_cfg = &hw->rss_info; + uint16_t idx, shift; + uint16_t i; ++ int ret; + + if (reta_size != hw->rss_ind_tbl_size) { + hns3_err(hw, "The size of hash lookup table configured (%u)" +@@ -648,14 +891,22 @@ hns3_dev_rss_reta_query(struct rte_eth_dev *dev, + return -EINVAL; + } + rte_spinlock_lock(&hw->lock); ++ ret = hns3_get_rss_indir_table(hw, reta_table, reta_size); ++ if (ret != 0) { ++ rte_spinlock_unlock(&hw->lock); ++ hns3_err(hw, "query RSS redirection table failed, ret = %d.", ++ ret); ++ return ret; ++ } ++ rte_spinlock_unlock(&hw->lock); ++ + for (i = 0; i < reta_size; i++) { + idx = i / RTE_ETH_RETA_GROUP_SIZE; + shift = i % RTE_ETH_RETA_GROUP_SIZE; + if (reta_conf[idx].mask & (1ULL << shift)) +- reta_conf[idx].reta[shift] = +- rss_cfg->rss_indirection_tbl[i]; ++ reta_conf[idx].reta[shift] = reta_table[i]; + } +- rte_spinlock_unlock(&hw->lock); ++ + return 0; + } + +@@ -733,6 +984,52 @@ hns3_set_rss_tc_mode(struct hns3_hw *hw) + return ret; + } + ++/* ++ * Note: the 'hash_algo' is defined by enum rte_eth_hash_function. ++ */ ++int ++hns3_update_rss_algo_key(struct hns3_hw *hw, uint8_t hash_func, ++ uint8_t *key, uint8_t key_len) ++{ ++ uint8_t rss_key[HNS3_RSS_KEY_SIZE_MAX] = {0}; ++ bool modify_key, modify_algo; ++ uint8_t hash_algo = 0; ++ int ret; ++ ++ modify_key = (key != NULL && key_len > 0); ++ modify_algo = hash_func != RTE_ETH_HASH_FUNCTION_DEFAULT; ++ if (!modify_key && !modify_algo) ++ return 0; ++ ++ if (modify_algo && hash_func >= RTE_DIM(hns3_hash_func_map)) { ++ hns3_err(hw, "hash func (%u) is unsupported.", hash_func); ++ return -ENOTSUP; ++ } ++ if (modify_key && key_len != hw->rss_key_size) { ++ hns3_err(hw, "hash key length (%u) is invalid.", key_len); ++ return -EINVAL; ++ } ++ ++ ret = hns3_rss_get_algo_key(hw, &hash_algo, rss_key, hw->rss_key_size); ++ if (ret != 0) { ++ hns3_err(hw, "fail to get RSS hash algorithm and key, ret = %d", ++ ret); ++ return ret; ++ } ++ ++ if (modify_algo) ++ hash_algo = hns3_hash_func_map[hash_func]; ++ if (modify_key) ++ memcpy(rss_key, key, key_len); ++ ++ ret = hns3_rss_set_algo_key(hw, hash_algo, rss_key, hw->rss_key_size); ++ if (ret != 0) ++ hns3_err(hw, "fail to set RSS hash algorithm and key, ret = %d", ++ ret); ++ ++ return ret; ++} ++ + static void + hns3_rss_tuple_uninit(struct hns3_hw *hw) + { +@@ -759,10 +1056,11 @@ hns3_rss_set_default_args(struct hns3_hw *hw) + uint16_t i; + + /* Default hash algorithm */ +- rss_cfg->conf.func = RTE_ETH_HASH_FUNCTION_TOEPLITZ; ++ rss_cfg->hash_algo = HNS3_RSS_HASH_ALGO_TOEPLITZ; + +- /* Default RSS key */ +- memcpy(rss_cfg->key, hns3_hash_key, HNS3_RSS_KEY_SIZE); ++ hw->rss_info.rss_hf = 0; ++ memcpy(rss_cfg->key, hns3_hash_key, ++ RTE_MIN(sizeof(hns3_hash_key), hw->rss_key_size)); + + /* Initialize RSS indirection table */ + for (i = 0; i < hw->rss_ind_tbl_size; i++) +@@ -783,20 +1081,8 @@ hns3_config_rss(struct hns3_adapter *hns) + + enum rte_eth_rx_mq_mode mq_mode = hw->data->dev_conf.rxmode.mq_mode; + +- switch (hw->rss_info.conf.func) { +- case RTE_ETH_HASH_FUNCTION_SIMPLE_XOR: +- hw->rss_info.hash_algo = HNS3_RSS_HASH_ALGO_SIMPLE; +- break; +- case RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ: +- hw->rss_info.hash_algo = HNS3_RSS_HASH_ALGO_SYMMETRIC_TOEP; +- break; +- default: +- hw->rss_info.hash_algo = HNS3_RSS_HASH_ALGO_TOEPLITZ; +- break; +- } +- +- /* Configure RSS hash algorithm and hash key offset */ +- ret = hns3_rss_set_algo_key(hw, hash_key); ++ ret = hns3_rss_set_algo_key(hw, rss_cfg->hash_algo, ++ hash_key, hw->rss_key_size); + if (ret) + return ret; + +@@ -810,15 +1096,22 @@ hns3_config_rss(struct hns3_adapter *hns) + return ret; + + /* +- * When muli-queue RSS mode flag is not set or unsupported tuples are ++ * When multi-queue RSS mode flag is not set or unsupported tuples are + * set, disable all tuples. + */ +- rss_hf = hw->rss_info.conf.types; ++ rss_hf = hw->rss_info.rss_hf; + if (!((uint32_t)mq_mode & RTE_ETH_MQ_RX_RSS_FLAG) || + !(rss_hf & HNS3_ETH_RSS_SUPPORT)) + rss_hf = 0; + +- return hns3_set_rss_tuple_by_rss_hf(hw, rss_hf); ++ ret = hns3_set_rss_tuple_by_rss_hf(hw, rss_hf); ++ if (ret != 0) { ++ hns3_err(hw, "set RSS tuples failed, ret = %d.", ret); ++ return ret; ++ } ++ hw->rss_info.rss_hf = rss_hf; ++ ++ return 0; + } + + /* +@@ -836,5 +1129,5 @@ hns3_rss_uninit(struct hns3_adapter *hns) + return; + + /* Disable RSS */ +- hw->rss_info.conf.types = 0; ++ hw->rss_info.rss_hf = 0; + } +diff --git a/dpdk/drivers/net/hns3/hns3_rss.h b/dpdk/drivers/net/hns3/hns3_rss.h +index 8e8b056f4e..9d182a8025 100644 +--- a/dpdk/drivers/net/hns3/hns3_rss.h ++++ b/dpdk/drivers/net/hns3/hns3_rss.h +@@ -8,27 +8,107 @@ + #include + #include + +-#define HNS3_ETH_RSS_SUPPORT ( \ +- RTE_ETH_RSS_IPV4 | \ +- RTE_ETH_RSS_FRAG_IPV4 | \ +- RTE_ETH_RSS_NONFRAG_IPV4_TCP | \ +- RTE_ETH_RSS_NONFRAG_IPV4_UDP | \ +- RTE_ETH_RSS_NONFRAG_IPV4_SCTP | \ +- RTE_ETH_RSS_NONFRAG_IPV4_OTHER | \ +- RTE_ETH_RSS_IPV6 | \ +- RTE_ETH_RSS_FRAG_IPV6 | \ +- RTE_ETH_RSS_NONFRAG_IPV6_TCP | \ +- RTE_ETH_RSS_NONFRAG_IPV6_UDP | \ +- RTE_ETH_RSS_NONFRAG_IPV6_SCTP | \ +- RTE_ETH_RSS_NONFRAG_IPV6_OTHER | \ +- RTE_ETH_RSS_L3_SRC_ONLY | \ +- RTE_ETH_RSS_L3_DST_ONLY | \ +- RTE_ETH_RSS_L4_SRC_ONLY | \ +- RTE_ETH_RSS_L4_DST_ONLY) ++#define HNS3_RSS_SUPPORT_L3_SRC_DST (RTE_ETH_RSS_L3_SRC_ONLY | \ ++ RTE_ETH_RSS_L3_DST_ONLY) ++#define HNS3_RSS_SUPPORT_L4_SRC_DST (RTE_ETH_RSS_L4_SRC_ONLY | \ ++ RTE_ETH_RSS_L4_DST_ONLY) ++#define HNS3_RSS_SUPPORT_L3L4 (HNS3_RSS_SUPPORT_L3_SRC_DST | \ ++ HNS3_RSS_SUPPORT_L4_SRC_DST) ++ ++#define HNS3_RSS_SUPPORT_FLOW_TYPE (RTE_ETH_RSS_IPV4 | \ ++ RTE_ETH_RSS_FRAG_IPV4 | \ ++ RTE_ETH_RSS_NONFRAG_IPV4_TCP | \ ++ RTE_ETH_RSS_NONFRAG_IPV4_UDP | \ ++ RTE_ETH_RSS_NONFRAG_IPV4_SCTP | \ ++ RTE_ETH_RSS_NONFRAG_IPV4_OTHER | \ ++ RTE_ETH_RSS_IPV6 | \ ++ RTE_ETH_RSS_FRAG_IPV6 | \ ++ RTE_ETH_RSS_NONFRAG_IPV6_TCP | \ ++ RTE_ETH_RSS_NONFRAG_IPV6_UDP | \ ++ RTE_ETH_RSS_NONFRAG_IPV6_SCTP | \ ++ RTE_ETH_RSS_NONFRAG_IPV6_OTHER) ++ ++#define HNS3_ETH_RSS_SUPPORT (HNS3_RSS_SUPPORT_FLOW_TYPE | \ ++ HNS3_RSS_SUPPORT_L3L4) ++ ++enum hns3_tuple_field { ++ /* IPV4_TCP ENABLE FIELD */ ++ HNS3_RSS_FIELD_IPV4_TCP_EN_TCP_D = 0, ++ HNS3_RSS_FIELD_IPV4_TCP_EN_TCP_S, ++ HNS3_RSS_FIELD_IPV4_TCP_EN_IP_D, ++ HNS3_RSS_FIELD_IPV4_TCP_EN_IP_S, ++ ++ /* IPV4_UDP ENABLE FIELD */ ++ HNS3_RSS_FIELD_IPV4_UDP_EN_UDP_D = 8, ++ HNS3_RSS_FIELD_IPV4_UDP_EN_UDP_S, ++ HNS3_RSS_FIELD_IPV4_UDP_EN_IP_D, ++ HNS3_RSS_FIELD_IPV4_UDP_EN_IP_S, ++ ++ /* IPV4_SCTP ENABLE FIELD */ ++ HNS3_RSS_FIELD_IPV4_SCTP_EN_SCTP_D = 16, ++ HNS3_RSS_FIELD_IPV4_SCTP_EN_SCTP_S, ++ HNS3_RSS_FIELD_IPV4_SCTP_EN_IP_D, ++ HNS3_RSS_FIELD_IPV4_SCTP_EN_IP_S, ++ HNS3_RSS_FIELD_IPV4_SCTP_EN_SCTP_VER, ++ ++ /* IPV4 ENABLE FIELD */ ++ HNS3_RSS_FIELD_IPV4_EN_NONFRAG_IP_D = 24, ++ HNS3_RSS_FIELD_IPV4_EN_NONFRAG_IP_S, ++ HNS3_RSS_FIELD_IPV4_EN_FRAG_IP_D, ++ HNS3_RSS_FIELD_IPV4_EN_FRAG_IP_S, ++ ++ /* IPV6_TCP ENABLE FIELD */ ++ HNS3_RSS_FIELD_IPV6_TCP_EN_TCP_D = 32, ++ HNS3_RSS_FIELD_IPV6_TCP_EN_TCP_S, ++ HNS3_RSS_FIELD_IPV6_TCP_EN_IP_D, ++ HNS3_RSS_FIELD_IPV6_TCP_EN_IP_S, ++ ++ /* IPV6_UDP ENABLE FIELD */ ++ HNS3_RSS_FIELD_IPV6_UDP_EN_UDP_D = 40, ++ HNS3_RSS_FIELD_IPV6_UDP_EN_UDP_S, ++ HNS3_RSS_FIELD_IPV6_UDP_EN_IP_D, ++ HNS3_RSS_FIELD_IPV6_UDP_EN_IP_S, ++ ++ /* IPV6_SCTP ENABLE FIELD */ ++ HNS3_RSS_FIELD_IPV6_SCTP_EN_SCTP_D = 48, ++ HNS3_RSS_FIELD_IPV6_SCTP_EN_SCTP_S, ++ HNS3_RSS_FIELD_IPV6_SCTP_EN_IP_D, ++ HNS3_RSS_FIELD_IPV6_SCTP_EN_IP_S, ++ HNS3_RSS_FIELD_IPV6_SCTP_EN_SCTP_VER, ++ ++ /* IPV6 ENABLE FIELD */ ++ HNS3_RSS_FIELD_IPV6_NONFRAG_IP_D = 56, ++ HNS3_RSS_FIELD_IPV6_NONFRAG_IP_S, ++ HNS3_RSS_FIELD_IPV6_FRAG_IP_D, ++ HNS3_RSS_FIELD_IPV6_FRAG_IP_S ++}; ++ ++#define HNS3_RSS_PCTYPE_IPV4_TCP BIT_ULL(0) ++#define HNS3_RSS_PCTYPE_IPV4_UDP BIT_ULL(8) ++#define HNS3_RSS_PCTYPE_IPV4_SCTP BIT_ULL(16) ++#define HNS3_RSS_PCTYPE_IPV4_NONF BIT_ULL(24) ++#define HNS3_RSS_PCTYPE_IPV4_FLAG BIT_ULL(26) ++#define HNS3_RSS_PCTYPE_IPV6_TCP BIT_ULL(32) ++#define HNS3_RSS_PCTYPE_IPV6_UDP BIT_ULL(40) ++#define HNS3_RSS_PCTYPE_IPV6_SCTP BIT_ULL(48) ++#define HNS3_RSS_PCTYPE_IPV6_NONF BIT_ULL(56) ++#define HNS3_RSS_PCTYPE_IPV6_FLAG BIT_ULL(58) ++ ++#define HNS3_RSS_TUPLE_IPV4_TCP_M GENMASK(3, 0) ++#define HNS3_RSS_TUPLE_IPV4_UDP_M GENMASK(11, 8) ++#define HNS3_RSS_TUPLE_IPV4_SCTP_M GENMASK(20, 16) ++#define HNS3_RSS_TUPLE_IPV4_NONF_M GENMASK(25, 24) ++#define HNS3_RSS_TUPLE_IPV4_FLAG_M GENMASK(27, 26) ++#define HNS3_RSS_TUPLE_IPV6_TCP_M GENMASK(35, 32) ++#define HNS3_RSS_TUPLE_IPV6_UDP_M GENMASK(43, 40) ++#define HNS3_RSS_TUPLE_IPV6_SCTP_M GENMASK(52, 48) ++#define HNS3_RSS_TUPLE_IPV6_NONF_M GENMASK(57, 56) ++#define HNS3_RSS_TUPLE_IPV6_FLAG_M GENMASK(59, 58) + + #define HNS3_RSS_IND_TBL_SIZE 512 /* The size of hash lookup table */ + #define HNS3_RSS_IND_TBL_SIZE_MAX 2048 + #define HNS3_RSS_KEY_SIZE 40 ++#define HNS3_RSS_KEY_SIZE_MAX 128 + #define HNS3_RSS_SET_BITMAP_MSK 0xffff + + #define HNS3_RSS_HASH_ALGO_TOEPLITZ 0 +@@ -36,15 +116,13 @@ + #define HNS3_RSS_HASH_ALGO_SYMMETRIC_TOEP 2 + #define HNS3_RSS_HASH_ALGO_MASK 0xf + +-#define HNS3_RSS_QUEUES_BUFFER_NUM 64 /* Same as the Max rx/tx queue num */ ++/* Same as the Max queue num under TC */ ++#define HNS3_RSS_QUEUES_BUFFER_NUM 512 + struct hns3_rss_conf { +- /* RSS parameters :algorithm, flow_types, key, queue */ +- struct rte_flow_action_rss conf; ++ uint64_t rss_hf; + uint8_t hash_algo; /* hash function type defined by hardware */ +- uint8_t key[HNS3_RSS_KEY_SIZE]; /* Hash key */ ++ uint8_t key[HNS3_RSS_KEY_SIZE_MAX]; /* Hash key */ + uint16_t rss_indirection_tbl[HNS3_RSS_IND_TBL_SIZE_MAX]; +- uint16_t queue[HNS3_RSS_QUEUES_BUFFER_NUM]; /* Queues indices to use */ +- bool valid; /* check if RSS rule is valid */ + /* + * For IPv6 SCTP packets type, check whether the NIC hardware support + * RSS hash using the src/dst port as the input tuple. For Kunpeng920 +@@ -108,7 +186,12 @@ int hns3_set_rss_indir_table(struct hns3_hw *hw, uint16_t *indir, + int hns3_rss_reset_indir_table(struct hns3_hw *hw); + int hns3_config_rss(struct hns3_adapter *hns); + void hns3_rss_uninit(struct hns3_adapter *hns); ++bool hns3_check_rss_types_valid(struct hns3_hw *hw, uint64_t types); + int hns3_set_rss_tuple_by_rss_hf(struct hns3_hw *hw, uint64_t rss_hf); +-int hns3_rss_set_algo_key(struct hns3_hw *hw, const uint8_t *key); ++int hns3_set_rss_tuple_field(struct hns3_hw *hw, uint64_t tuple_fields); ++int hns3_get_rss_tuple_field(struct hns3_hw *hw, uint64_t *tuple_fields); ++uint64_t hns3_rss_calc_tuple_filed(uint64_t rss_hf); ++int hns3_update_rss_algo_key(struct hns3_hw *hw, uint8_t hash_algo, ++ uint8_t *key, uint8_t key_len); + + #endif /* HNS3_RSS_H */ +diff --git a/dpdk/drivers/net/hns3/hns3_rxtx.c b/dpdk/drivers/net/hns3/hns3_rxtx.c +index f1163ce8a9..9d473dbc22 100644 +--- a/dpdk/drivers/net/hns3/hns3_rxtx.c ++++ b/dpdk/drivers/net/hns3/hns3_rxtx.c +@@ -50,6 +50,8 @@ hns3_rx_queue_release_mbufs(struct hns3_rx_queue *rxq) + rxq->sw_ring[i].mbuf = NULL; + } + } ++ for (i = 0; i < rxq->rx_rearm_nb; i++) ++ rxq->sw_ring[rxq->rx_rearm_start + i].mbuf = NULL; + } + + for (i = 0; i < rxq->bulk_mbuf_num; i++) +@@ -584,7 +586,7 @@ hns3_tqp_enable(struct hns3_hw *hw, uint16_t queue_id, bool enable) + + ret = hns3_cmd_send(hw, &desc, 1); + if (ret) +- hns3_err(hw, "TQP enable fail, ret = %d", ret); ++ hns3_err(hw, "TQP %s fail, ret = %d", enable ? "enable" : "disable", ret); + + return ret; + } +@@ -1637,7 +1639,7 @@ hns3_set_fake_rx_or_tx_queues(struct rte_eth_dev *dev, uint16_t nb_rx_q, + + ret = hns3_fake_tx_queue_config(hw, tx_need_add_nb_q); + if (ret) { +- hns3_err(hw, "Fail to configure fake rx queues: %d", ret); ++ hns3_err(hw, "Fail to configure fake tx queues: %d", ret); + goto cfg_fake_tx_q_fail; + } + +@@ -1787,6 +1789,12 @@ hns3_rx_queue_conf_check(struct hns3_hw *hw, const struct rte_eth_rxconf *conf, + return -EINVAL; + } + ++ if (conf->rx_free_thresh >= nb_desc) { ++ hns3_err(hw, "rx_free_thresh (%u) must be less than %u", ++ conf->rx_free_thresh, nb_desc); ++ return -EINVAL; ++ } ++ + if (conf->rx_drop_en == 0) + hns3_warn(hw, "if no descriptors available, packets are always " + "dropped and rx_drop_en (1) is fixed on"); +@@ -2786,6 +2794,7 @@ hns3_rx_burst_mode_get(struct rte_eth_dev *dev, __rte_unused uint16_t queue_id, + { hns3_recv_scattered_pkts, "Scalar Scattered" }, + { hns3_recv_pkts_vec, "Vector Neon" }, + { hns3_recv_pkts_vec_sve, "Vector Sve" }, ++ { rte_eth_pkt_burst_dummy, "Dummy" }, + }; + + eth_rx_burst_t pkt_burst = dev->rx_pkt_burst; +@@ -3115,6 +3124,9 @@ hns3_config_gro(struct hns3_hw *hw, bool en) + struct hns3_cmd_desc desc; + int ret; + ++ if (!hns3_dev_get_support(hw, GRO)) ++ return 0; ++ + hns3_cmd_setup_basic_desc(&desc, HNS3_OPC_GRO_GENERIC_CONFIG, false); + req = (struct hns3_cfg_gro_status_cmd *)desc.data; + +@@ -4272,24 +4284,31 @@ int + hns3_tx_burst_mode_get(struct rte_eth_dev *dev, __rte_unused uint16_t queue_id, + struct rte_eth_burst_mode *mode) + { ++ static const struct { ++ eth_tx_burst_t pkt_burst; ++ const char *info; ++ } burst_infos[] = { ++ { hns3_xmit_pkts_simple, "Scalar Simple" }, ++ { hns3_xmit_pkts, "Scalar" }, ++ { hns3_xmit_pkts_vec, "Vector Neon" }, ++ { hns3_xmit_pkts_vec_sve, "Vector Sve" }, ++ { rte_eth_pkt_burst_dummy, "Dummy" }, ++ }; ++ + eth_tx_burst_t pkt_burst = dev->tx_pkt_burst; +- const char *info = NULL; +- +- if (pkt_burst == hns3_xmit_pkts_simple) +- info = "Scalar Simple"; +- else if (pkt_burst == hns3_xmit_pkts) +- info = "Scalar"; +- else if (pkt_burst == hns3_xmit_pkts_vec) +- info = "Vector Neon"; +- else if (pkt_burst == hns3_xmit_pkts_vec_sve) +- info = "Vector Sve"; +- +- if (info == NULL) +- return -EINVAL; ++ int ret = -EINVAL; ++ unsigned int i; + +- snprintf(mode->info, sizeof(mode->info), "%s", info); ++ for (i = 0; i < RTE_DIM(burst_infos); i++) { ++ if (pkt_burst == burst_infos[i].pkt_burst) { ++ snprintf(mode->info, sizeof(mode->info), "%s", ++ burst_infos[i].info); ++ ret = 0; ++ break; ++ } ++ } + +- return 0; ++ return ret; + } + + static bool +@@ -4303,11 +4322,6 @@ hns3_tx_check_simple_support(struct rte_eth_dev *dev) + static bool + hns3_get_tx_prep_needed(struct rte_eth_dev *dev) + { +-#ifdef RTE_LIBRTE_ETHDEV_DEBUG +- RTE_SET_USED(dev); +- /* always perform tx_prepare when debug */ +- return true; +-#else + #define HNS3_DEV_TX_CSKUM_TSO_OFFLOAD_MASK (\ + RTE_ETH_TX_OFFLOAD_IPV4_CKSUM | \ + RTE_ETH_TX_OFFLOAD_TCP_CKSUM | \ +@@ -4321,27 +4335,30 @@ hns3_get_tx_prep_needed(struct rte_eth_dev *dev) + RTE_ETH_TX_OFFLOAD_GENEVE_TNL_TSO) + + uint64_t tx_offload = dev->data->dev_conf.txmode.offloads; ++ + if (tx_offload & HNS3_DEV_TX_CSKUM_TSO_OFFLOAD_MASK) + return true; + + return false; +-#endif + } + +-eth_tx_burst_t +-hns3_get_tx_function(struct rte_eth_dev *dev, eth_tx_prep_t *prep) ++static eth_tx_prep_t ++hns3_get_tx_prepare(struct rte_eth_dev *dev) ++{ ++ return hns3_get_tx_prep_needed(dev) ? hns3_prep_pkts : NULL; ++} ++ ++static eth_tx_burst_t ++hns3_get_tx_function(struct rte_eth_dev *dev) + { + struct hns3_adapter *hns = dev->data->dev_private; + bool vec_allowed, sve_allowed, simple_allowed; +- bool vec_support, tx_prepare_needed; ++ bool vec_support; + + vec_support = hns3_tx_check_vec_support(dev) == 0; + vec_allowed = vec_support && hns3_get_default_vec_support(); + sve_allowed = vec_support && hns3_get_sve_support(); + simple_allowed = hns3_tx_check_simple_support(dev); +- tx_prepare_needed = hns3_get_tx_prep_needed(dev); +- +- *prep = NULL; + + if (hns->tx_func_hint == HNS3_IO_FUNC_HINT_VEC && vec_allowed) + return hns3_xmit_pkts_vec; +@@ -4349,19 +4366,14 @@ hns3_get_tx_function(struct rte_eth_dev *dev, eth_tx_prep_t *prep) + return hns3_xmit_pkts_vec_sve; + if (hns->tx_func_hint == HNS3_IO_FUNC_HINT_SIMPLE && simple_allowed) + return hns3_xmit_pkts_simple; +- if (hns->tx_func_hint == HNS3_IO_FUNC_HINT_COMMON) { +- if (tx_prepare_needed) +- *prep = hns3_prep_pkts; ++ if (hns->tx_func_hint == HNS3_IO_FUNC_HINT_COMMON) + return hns3_xmit_pkts; +- } + + if (vec_allowed) + return hns3_xmit_pkts_vec; + if (simple_allowed) + return hns3_xmit_pkts_simple; + +- if (tx_prepare_needed) +- *prep = hns3_prep_pkts; + return hns3_xmit_pkts; + } + +@@ -4401,7 +4413,6 @@ hns3_set_rxtx_function(struct rte_eth_dev *eth_dev) + { + struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); + struct hns3_adapter *hns = eth_dev->data->dev_private; +- eth_tx_prep_t prep = NULL; + + if (hns->hw.adapter_state == HNS3_NIC_STARTED && + __atomic_load_n(&hns->hw.reset.resetting, __ATOMIC_RELAXED) == 0) { +@@ -4409,16 +4420,16 @@ hns3_set_rxtx_function(struct rte_eth_dev *eth_dev) + eth_dev->rx_descriptor_status = hns3_dev_rx_descriptor_status; + eth_dev->tx_pkt_burst = hw->set_link_down ? + rte_eth_pkt_burst_dummy : +- hns3_get_tx_function(eth_dev, &prep); +- eth_dev->tx_pkt_prepare = prep; ++ hns3_get_tx_function(eth_dev); ++ eth_dev->tx_pkt_prepare = hns3_get_tx_prepare(eth_dev); + eth_dev->tx_descriptor_status = hns3_dev_tx_descriptor_status; +- hns3_trace_rxtx_function(eth_dev); + } else { + eth_dev->rx_pkt_burst = rte_eth_pkt_burst_dummy; + eth_dev->tx_pkt_burst = rte_eth_pkt_burst_dummy; + eth_dev->tx_pkt_prepare = NULL; + } + ++ hns3_trace_rxtx_function(eth_dev); + hns3_eth_dev_fp_ops_config(eth_dev); + } + +@@ -4469,6 +4480,13 @@ hns3_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id) + return -ENOTSUP; + + rte_spinlock_lock(&hw->lock); ++ ++ if (__atomic_load_n(&hw->reset.resetting, __ATOMIC_RELAXED)) { ++ hns3_err(hw, "fail to start Rx queue during resetting."); ++ rte_spinlock_unlock(&hw->lock); ++ return -EIO; ++ } ++ + ret = hns3_reset_queue(hw, rx_queue_id, HNS3_RING_TYPE_RX); + if (ret) { + hns3_err(hw, "fail to reset Rx queue %u, ret = %d.", +@@ -4477,6 +4495,9 @@ hns3_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id) + return ret; + } + ++ if (rxq->sw_ring[0].mbuf != NULL) ++ hns3_rx_queue_release_mbufs(rxq); ++ + ret = hns3_init_rxq(hns, rx_queue_id); + if (ret) { + hns3_err(hw, "fail to init Rx queue %u, ret = %d.", +@@ -4515,6 +4536,13 @@ hns3_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id) + return -ENOTSUP; + + rte_spinlock_lock(&hw->lock); ++ ++ if (__atomic_load_n(&hw->reset.resetting, __ATOMIC_RELAXED)) { ++ hns3_err(hw, "fail to stop Rx queue during resetting."); ++ rte_spinlock_unlock(&hw->lock); ++ return -EIO; ++ } ++ + hns3_enable_rxq(rxq, false); + + hns3_rx_queue_release_mbufs(rxq); +@@ -4537,6 +4565,13 @@ hns3_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id) + return -ENOTSUP; + + rte_spinlock_lock(&hw->lock); ++ ++ if (__atomic_load_n(&hw->reset.resetting, __ATOMIC_RELAXED)) { ++ hns3_err(hw, "fail to start Tx queue during resetting."); ++ rte_spinlock_unlock(&hw->lock); ++ return -EIO; ++ } ++ + ret = hns3_reset_queue(hw, tx_queue_id, HNS3_RING_TYPE_TX); + if (ret) { + hns3_err(hw, "fail to reset Tx queue %u, ret = %d.", +@@ -4563,6 +4598,13 @@ hns3_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) + return -ENOTSUP; + + rte_spinlock_lock(&hw->lock); ++ ++ if (__atomic_load_n(&hw->reset.resetting, __ATOMIC_RELAXED)) { ++ hns3_err(hw, "fail to stop Tx queue during resetting."); ++ rte_spinlock_unlock(&hw->lock); ++ return -EIO; ++ } ++ + hns3_enable_txq(txq, false); + hns3_tx_queue_release_mbufs(txq); + /* +@@ -4756,10 +4798,8 @@ hns3_stop_tx_datapath(struct rte_eth_dev *dev) + void + hns3_start_tx_datapath(struct rte_eth_dev *dev) + { +- eth_tx_prep_t prep = NULL; +- +- dev->tx_pkt_burst = hns3_get_tx_function(dev, &prep); +- dev->tx_pkt_prepare = prep; ++ dev->tx_pkt_burst = hns3_get_tx_function(dev); ++ dev->tx_pkt_prepare = hns3_get_tx_prepare(dev); + hns3_eth_dev_fp_ops_config(dev); + + if (rte_eal_process_type() == RTE_PROC_SECONDARY) +@@ -4767,3 +4807,31 @@ hns3_start_tx_datapath(struct rte_eth_dev *dev) + + hns3_mp_req_start_tx(dev); + } ++ ++void ++hns3_stop_rxtx_datapath(struct rte_eth_dev *dev) ++{ ++ struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private); ++ ++ hns3_set_rxtx_function(dev); ++ ++ if (rte_eal_process_type() == RTE_PROC_SECONDARY) ++ return; ++ ++ rte_wmb(); ++ /* Disable datapath on secondary process. */ ++ hns3_mp_req_stop_rxtx(dev); ++ /* Prevent crashes when queues are still in use. */ ++ rte_delay_ms(hw->cfg_max_queues); ++} ++ ++void ++hns3_start_rxtx_datapath(struct rte_eth_dev *dev) ++{ ++ hns3_set_rxtx_function(dev); ++ ++ if (rte_eal_process_type() == RTE_PROC_SECONDARY) ++ return; ++ ++ hns3_mp_req_start_rxtx(dev); ++} +diff --git a/dpdk/drivers/net/hns3/hns3_rxtx.h b/dpdk/drivers/net/hns3/hns3_rxtx.h +index ea1a805491..fa39f6481a 100644 +--- a/dpdk/drivers/net/hns3/hns3_rxtx.h ++++ b/dpdk/drivers/net/hns3/hns3_rxtx.h +@@ -740,9 +740,6 @@ int hns3_tx_burst_mode_get(struct rte_eth_dev *dev, + const uint32_t *hns3_dev_supported_ptypes_get(struct rte_eth_dev *dev); + void hns3_init_rx_ptype_tble(struct rte_eth_dev *dev); + void hns3_set_rxtx_function(struct rte_eth_dev *eth_dev); +-eth_tx_burst_t hns3_get_tx_function(struct rte_eth_dev *dev, +- eth_tx_prep_t *prep); +- + uint32_t hns3_get_tqp_intr_reg_offset(uint16_t tqp_intr_id); + void hns3_set_queue_intr_gl(struct hns3_hw *hw, uint16_t queue_id, + uint8_t gl_idx, uint16_t gl_value); +@@ -776,5 +773,7 @@ int hns3_dev_tx_descriptor_status(void *tx_queue, uint16_t offset); + void hns3_tx_push_init(struct rte_eth_dev *dev); + void hns3_stop_tx_datapath(struct rte_eth_dev *dev); + void hns3_start_tx_datapath(struct rte_eth_dev *dev); ++void hns3_stop_rxtx_datapath(struct rte_eth_dev *dev); ++void hns3_start_rxtx_datapath(struct rte_eth_dev *dev); + + #endif /* HNS3_RXTX_H */ +diff --git a/dpdk/drivers/net/hns3/hns3_rxtx_vec_neon.h b/dpdk/drivers/net/hns3/hns3_rxtx_vec_neon.h +index 55d9bf817d..1048b9db87 100644 +--- a/dpdk/drivers/net/hns3/hns3_rxtx_vec_neon.h ++++ b/dpdk/drivers/net/hns3/hns3_rxtx_vec_neon.h +@@ -142,8 +142,8 @@ hns3_recv_burst_vec(struct hns3_rx_queue *__restrict rxq, + /* mask to shuffle from desc to mbuf's rx_descriptor_fields1 */ + uint8x16_t shuf_desc_fields_msk = { + 0xff, 0xff, 0xff, 0xff, /* packet type init zero */ +- 22, 23, 0xff, 0xff, /* rx.pkt_len to rte_mbuf.pkt_len */ +- 20, 21, /* size to rte_mbuf.data_len */ ++ 20, 21, 0xff, 0xff, /* rx.pkt_len to rte_mbuf.pkt_len */ ++ 22, 23, /* size to rte_mbuf.data_len */ + 0xff, 0xff, /* rte_mbuf.vlan_tci init zero */ + 8, 9, 10, 11, /* rx.rss_hash to rte_mbuf.hash.rss */ + }; +@@ -180,19 +180,12 @@ hns3_recv_burst_vec(struct hns3_rx_queue *__restrict rxq, + bd_vld = vset_lane_u16(rxdp[2].rx.bdtype_vld_udp0, bd_vld, 2); + bd_vld = vset_lane_u16(rxdp[3].rx.bdtype_vld_udp0, bd_vld, 3); + +- /* load 2 mbuf pointer */ +- mbp1 = vld1q_u64((uint64_t *)&sw_ring[pos]); +- + bd_vld = vshl_n_u16(bd_vld, + HNS3_UINT16_BIT - 1 - HNS3_RXD_VLD_B); + bd_vld = vreinterpret_u16_s16( + vshr_n_s16(vreinterpret_s16_u16(bd_vld), + HNS3_UINT16_BIT - 1)); + stat = ~vget_lane_u64(vreinterpret_u64_u16(bd_vld), 0); +- +- /* load 2 mbuf pointer again */ +- mbp2 = vld1q_u64((uint64_t *)&sw_ring[pos + 2]); +- + if (likely(stat == 0)) + bd_valid_num = HNS3_DEFAULT_DESCS_PER_LOOP; + else +@@ -200,20 +193,20 @@ hns3_recv_burst_vec(struct hns3_rx_queue *__restrict rxq, + if (bd_valid_num == 0) + break; + +- /* use offset to control below data load oper ordering */ +- offset = rxq->offset_table[bd_valid_num]; ++ /* load 4 mbuf pointer */ ++ mbp1 = vld1q_u64((uint64_t *)&sw_ring[pos]); ++ mbp2 = vld1q_u64((uint64_t *)&sw_ring[pos + 2]); + +- /* store 2 mbuf pointer into rx_pkts */ ++ /* store 4 mbuf pointer into rx_pkts */ + vst1q_u64((uint64_t *)&rx_pkts[pos], mbp1); ++ vst1q_u64((uint64_t *)&rx_pkts[pos + 2], mbp2); + +- /* read first two descs */ ++ /* use offset to control below data load oper ordering */ ++ offset = rxq->offset_table[bd_valid_num]; ++ ++ /* read 4 descs */ + descs[0] = vld2q_u64((uint64_t *)(rxdp + offset)); + descs[1] = vld2q_u64((uint64_t *)(rxdp + offset + 1)); +- +- /* store 2 mbuf pointer into rx_pkts again */ +- vst1q_u64((uint64_t *)&rx_pkts[pos + 2], mbp2); +- +- /* read remains two descs */ + descs[2] = vld2q_u64((uint64_t *)(rxdp + offset + 2)); + descs[3] = vld2q_u64((uint64_t *)(rxdp + offset + 3)); + +@@ -221,56 +214,47 @@ hns3_recv_burst_vec(struct hns3_rx_queue *__restrict rxq, + pkt_mbuf1.val[1] = vreinterpretq_u8_u64(descs[0].val[1]); + pkt_mbuf2.val[0] = vreinterpretq_u8_u64(descs[1].val[0]); + pkt_mbuf2.val[1] = vreinterpretq_u8_u64(descs[1].val[1]); ++ pkt_mbuf3.val[0] = vreinterpretq_u8_u64(descs[2].val[0]); ++ pkt_mbuf3.val[1] = vreinterpretq_u8_u64(descs[2].val[1]); ++ pkt_mbuf4.val[0] = vreinterpretq_u8_u64(descs[3].val[0]); ++ pkt_mbuf4.val[1] = vreinterpretq_u8_u64(descs[3].val[1]); + +- /* pkt 1,2 convert format from desc to pktmbuf */ ++ /* 4 packets convert format from desc to pktmbuf */ + pkt_mb1 = vqtbl2q_u8(pkt_mbuf1, shuf_desc_fields_msk); + pkt_mb2 = vqtbl2q_u8(pkt_mbuf2, shuf_desc_fields_msk); ++ pkt_mb3 = vqtbl2q_u8(pkt_mbuf3, shuf_desc_fields_msk); ++ pkt_mb4 = vqtbl2q_u8(pkt_mbuf4, shuf_desc_fields_msk); + +- /* store the first 8 bytes of pkt 1,2 mbuf's rearm_data */ +- *(uint64_t *)&sw_ring[pos + 0].mbuf->rearm_data = +- rxq->mbuf_initializer; +- *(uint64_t *)&sw_ring[pos + 1].mbuf->rearm_data = +- rxq->mbuf_initializer; +- +- /* pkt 1,2 remove crc */ ++ /* 4 packets remove crc */ + tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb1), crc_adjust); + pkt_mb1 = vreinterpretq_u8_u16(tmp); + tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb2), crc_adjust); + pkt_mb2 = vreinterpretq_u8_u16(tmp); ++ tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb3), crc_adjust); ++ pkt_mb3 = vreinterpretq_u8_u16(tmp); ++ tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb4), crc_adjust); ++ pkt_mb4 = vreinterpretq_u8_u16(tmp); + +- pkt_mbuf3.val[0] = vreinterpretq_u8_u64(descs[2].val[0]); +- pkt_mbuf3.val[1] = vreinterpretq_u8_u64(descs[2].val[1]); +- pkt_mbuf4.val[0] = vreinterpretq_u8_u64(descs[3].val[0]); +- pkt_mbuf4.val[1] = vreinterpretq_u8_u64(descs[3].val[1]); +- +- /* pkt 3,4 convert format from desc to pktmbuf */ +- pkt_mb3 = vqtbl2q_u8(pkt_mbuf3, shuf_desc_fields_msk); +- pkt_mb4 = vqtbl2q_u8(pkt_mbuf4, shuf_desc_fields_msk); +- +- /* pkt 1,2 save to rx_pkts mbuf */ ++ /* save packet info to rx_pkts mbuf */ + vst1q_u8((void *)&sw_ring[pos + 0].mbuf->rx_descriptor_fields1, + pkt_mb1); + vst1q_u8((void *)&sw_ring[pos + 1].mbuf->rx_descriptor_fields1, + pkt_mb2); ++ vst1q_u8((void *)&sw_ring[pos + 2].mbuf->rx_descriptor_fields1, ++ pkt_mb3); ++ vst1q_u8((void *)&sw_ring[pos + 3].mbuf->rx_descriptor_fields1, ++ pkt_mb4); + +- /* pkt 3,4 remove crc */ +- tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb3), crc_adjust); +- pkt_mb3 = vreinterpretq_u8_u16(tmp); +- tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb4), crc_adjust); +- pkt_mb4 = vreinterpretq_u8_u16(tmp); +- +- /* store the first 8 bytes of pkt 3,4 mbuf's rearm_data */ ++ /* store the first 8 bytes of packets mbuf's rearm_data */ ++ *(uint64_t *)&sw_ring[pos + 0].mbuf->rearm_data = ++ rxq->mbuf_initializer; ++ *(uint64_t *)&sw_ring[pos + 1].mbuf->rearm_data = ++ rxq->mbuf_initializer; + *(uint64_t *)&sw_ring[pos + 2].mbuf->rearm_data = + rxq->mbuf_initializer; + *(uint64_t *)&sw_ring[pos + 3].mbuf->rearm_data = + rxq->mbuf_initializer; + +- /* pkt 3,4 save to rx_pkts mbuf */ +- vst1q_u8((void *)&sw_ring[pos + 2].mbuf->rx_descriptor_fields1, +- pkt_mb3); +- vst1q_u8((void *)&sw_ring[pos + 3].mbuf->rx_descriptor_fields1, +- pkt_mb4); +- + rte_prefetch_non_temporal(rxdp + HNS3_DEFAULT_DESCS_PER_LOOP); + + parse_retcode = hns3_desc_parse_field(rxq, &sw_ring[pos], +diff --git a/dpdk/drivers/net/hns3/hns3_stats.c b/dpdk/drivers/net/hns3/hns3_stats.c +index bad65fcbed..9a1e8935e5 100644 +--- a/dpdk/drivers/net/hns3/hns3_stats.c ++++ b/dpdk/drivers/net/hns3/hns3_stats.c +@@ -317,7 +317,7 @@ hns3_update_mac_stats(struct hns3_hw *hw) + uint32_t stats_iterms; + uint64_t *desc_data; + uint32_t desc_num; +- uint16_t i; ++ uint32_t i; + int ret; + + /* The first desc has a 64-bit header, so need to consider it. */ +@@ -771,7 +771,7 @@ hns3_mac_stats_reset(struct hns3_hw *hw) + return 0; + } + +-static int ++static uint16_t + hns3_get_imissed_stats_num(struct hns3_adapter *hns) + { + #define NO_IMISSED_STATS_NUM 0 +@@ -993,7 +993,7 @@ hns3_imissed_stats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, + struct hns3_adapter *hns = dev->data->dev_private; + struct hns3_hw *hw = &hns->hw; + struct hns3_rx_missed_stats *imissed_stats = &hw->imissed_stats; +- int imissed_stats_num; ++ uint16_t imissed_stats_num; + int cnt = *count; + char *addr; + uint16_t i; +@@ -1170,7 +1170,7 @@ hns3_imissed_stats_name_get(struct rte_eth_dev *dev, + { + struct hns3_adapter *hns = dev->data->dev_private; + uint32_t cnt = *count; +- int imissed_stats_num; ++ uint16_t imissed_stats_num; + uint16_t i; + + imissed_stats_num = hns3_get_imissed_stats_num(hns); +@@ -1539,8 +1539,13 @@ hns3_stats_init(struct hns3_hw *hw) + return ret; + } + +- if (!hns->is_vf) +- hns3_mac_stats_reset(hw); ++ if (!hns->is_vf) { ++ ret = hns3_mac_stats_reset(hw); ++ if (ret) { ++ hns3_err(hw, "reset mac stats failed, ret = %d", ret); ++ return ret; ++ } ++ } + + return hns3_tqp_stats_init(hw); + } +diff --git a/dpdk/drivers/net/hns3/hns3_tm.c b/dpdk/drivers/net/hns3/hns3_tm.c +index e1089b6bd0..d969164014 100644 +--- a/dpdk/drivers/net/hns3/hns3_tm.c ++++ b/dpdk/drivers/net/hns3/hns3_tm.c +@@ -739,7 +739,7 @@ hns3_tm_node_type_get(struct rte_eth_dev *dev, uint32_t node_id, + } + + static void +-hns3_tm_nonleaf_level_capsbilities_get(struct rte_eth_dev *dev, ++hns3_tm_nonleaf_level_capabilities_get(struct rte_eth_dev *dev, + uint32_t level_id, + struct rte_tm_level_capabilities *cap) + { +@@ -818,7 +818,7 @@ hns3_tm_level_capabilities_get(struct rte_eth_dev *dev, + memset(cap, 0, sizeof(struct rte_tm_level_capabilities)); + + if (level_id != HNS3_TM_NODE_LEVEL_QUEUE) +- hns3_tm_nonleaf_level_capsbilities_get(dev, level_id, cap); ++ hns3_tm_nonleaf_level_capabilities_get(dev, level_id, cap); + else + hns3_tm_leaf_level_capabilities_get(dev, cap); + +@@ -1081,21 +1081,6 @@ fail_clear: + return -EINVAL; + } + +-static int +-hns3_tm_hierarchy_commit_wrap(struct rte_eth_dev *dev, +- int clear_on_fail, +- struct rte_tm_error *error) +-{ +- struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private); +- int ret; +- +- rte_spinlock_lock(&hw->lock); +- ret = hns3_tm_hierarchy_commit(dev, clear_on_fail, error); +- rte_spinlock_unlock(&hw->lock); +- +- return ret; +-} +- + static int + hns3_tm_node_shaper_do_update(struct hns3_hw *hw, + uint32_t node_id, +@@ -1195,6 +1180,148 @@ hns3_tm_node_shaper_update(struct rte_eth_dev *dev, + return 0; + } + ++static int ++hns3_tm_capabilities_get_wrap(struct rte_eth_dev *dev, ++ struct rte_tm_capabilities *cap, ++ struct rte_tm_error *error) ++{ ++ struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private); ++ int ret; ++ ++ rte_spinlock_lock(&hw->lock); ++ ret = hns3_tm_capabilities_get(dev, cap, error); ++ rte_spinlock_unlock(&hw->lock); ++ ++ return ret; ++} ++ ++static int ++hns3_tm_shaper_profile_add_wrap(struct rte_eth_dev *dev, ++ uint32_t shaper_profile_id, ++ struct rte_tm_shaper_params *profile, ++ struct rte_tm_error *error) ++{ ++ struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private); ++ int ret; ++ ++ rte_spinlock_lock(&hw->lock); ++ ret = hns3_tm_shaper_profile_add(dev, shaper_profile_id, profile, error); ++ rte_spinlock_unlock(&hw->lock); ++ ++ return ret; ++} ++ ++static int ++hns3_tm_shaper_profile_del_wrap(struct rte_eth_dev *dev, ++ uint32_t shaper_profile_id, ++ struct rte_tm_error *error) ++{ ++ struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private); ++ int ret; ++ ++ rte_spinlock_lock(&hw->lock); ++ ret = hns3_tm_shaper_profile_del(dev, shaper_profile_id, error); ++ rte_spinlock_unlock(&hw->lock); ++ ++ return ret; ++} ++ ++static int ++hns3_tm_node_add_wrap(struct rte_eth_dev *dev, uint32_t node_id, ++ uint32_t parent_node_id, uint32_t priority, ++ uint32_t weight, uint32_t level_id, ++ struct rte_tm_node_params *params, ++ struct rte_tm_error *error) ++{ ++ struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private); ++ int ret; ++ ++ rte_spinlock_lock(&hw->lock); ++ ret = hns3_tm_node_add(dev, node_id, parent_node_id, priority, ++ weight, level_id, params, error); ++ rte_spinlock_unlock(&hw->lock); ++ ++ return ret; ++} ++ ++static int ++hns3_tm_node_delete_wrap(struct rte_eth_dev *dev, ++ uint32_t node_id, ++ struct rte_tm_error *error) ++{ ++ struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private); ++ int ret; ++ ++ rte_spinlock_lock(&hw->lock); ++ ret = hns3_tm_node_delete(dev, node_id, error); ++ rte_spinlock_unlock(&hw->lock); ++ ++ return ret; ++} ++ ++static int ++hns3_tm_node_type_get_wrap(struct rte_eth_dev *dev, ++ uint32_t node_id, ++ int *is_leaf, ++ struct rte_tm_error *error) ++{ ++ struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private); ++ int ret; ++ ++ rte_spinlock_lock(&hw->lock); ++ ret = hns3_tm_node_type_get(dev, node_id, is_leaf, error); ++ rte_spinlock_unlock(&hw->lock); ++ ++ return ret; ++} ++ ++static int ++hns3_tm_level_capabilities_get_wrap(struct rte_eth_dev *dev, ++ uint32_t level_id, ++ struct rte_tm_level_capabilities *cap, ++ struct rte_tm_error *error) ++{ ++ struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private); ++ int ret; ++ ++ rte_spinlock_lock(&hw->lock); ++ ret = hns3_tm_level_capabilities_get(dev, level_id, cap, error); ++ rte_spinlock_unlock(&hw->lock); ++ ++ return ret; ++} ++ ++static int ++hns3_tm_node_capabilities_get_wrap(struct rte_eth_dev *dev, ++ uint32_t node_id, ++ struct rte_tm_node_capabilities *cap, ++ struct rte_tm_error *error) ++{ ++ struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private); ++ int ret; ++ ++ rte_spinlock_lock(&hw->lock); ++ ret = hns3_tm_node_capabilities_get(dev, node_id, cap, error); ++ rte_spinlock_unlock(&hw->lock); ++ ++ return ret; ++} ++ ++static int ++hns3_tm_hierarchy_commit_wrap(struct rte_eth_dev *dev, ++ int clear_on_fail, ++ struct rte_tm_error *error) ++{ ++ struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private); ++ int ret; ++ ++ rte_spinlock_lock(&hw->lock); ++ ret = hns3_tm_hierarchy_commit(dev, clear_on_fail, error); ++ rte_spinlock_unlock(&hw->lock); ++ ++ return ret; ++} ++ + static int + hns3_tm_node_shaper_update_wrap(struct rte_eth_dev *dev, + uint32_t node_id, +@@ -1213,14 +1340,14 @@ hns3_tm_node_shaper_update_wrap(struct rte_eth_dev *dev, + } + + static const struct rte_tm_ops hns3_tm_ops = { +- .capabilities_get = hns3_tm_capabilities_get, +- .shaper_profile_add = hns3_tm_shaper_profile_add, +- .shaper_profile_delete = hns3_tm_shaper_profile_del, +- .node_add = hns3_tm_node_add, +- .node_delete = hns3_tm_node_delete, +- .node_type_get = hns3_tm_node_type_get, +- .level_capabilities_get = hns3_tm_level_capabilities_get, +- .node_capabilities_get = hns3_tm_node_capabilities_get, ++ .capabilities_get = hns3_tm_capabilities_get_wrap, ++ .shaper_profile_add = hns3_tm_shaper_profile_add_wrap, ++ .shaper_profile_delete = hns3_tm_shaper_profile_del_wrap, ++ .node_add = hns3_tm_node_add_wrap, ++ .node_delete = hns3_tm_node_delete_wrap, ++ .node_type_get = hns3_tm_node_type_get_wrap, ++ .level_capabilities_get = hns3_tm_level_capabilities_get_wrap, ++ .node_capabilities_get = hns3_tm_node_capabilities_get_wrap, + .hierarchy_commit = hns3_tm_hierarchy_commit_wrap, + .node_shaper_update = hns3_tm_node_shaper_update_wrap, + }; +diff --git a/dpdk/drivers/net/i40e/i40e_ethdev.c b/dpdk/drivers/net/i40e/i40e_ethdev.c +index 7726a89d99..0ff334745d 100644 +--- a/dpdk/drivers/net/i40e/i40e_ethdev.c ++++ b/dpdk/drivers/net/i40e/i40e_ethdev.c +@@ -387,7 +387,6 @@ static int i40e_set_default_mac_addr(struct rte_eth_dev *dev, + struct rte_ether_addr *mac_addr); + + static int i40e_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); +-static void i40e_set_mac_max_frame(struct rte_eth_dev *dev, uint16_t size); + + static int i40e_ethertype_filter_convert( + const struct rte_eth_ethertype_filter *input, +@@ -2412,10 +2411,21 @@ i40e_dev_start(struct rte_eth_dev *dev) + } + } + ++ /* Disable mac loopback mode */ ++ if (dev->data->dev_conf.lpbk_mode == I40E_AQ_LB_MODE_NONE) { ++ ret = i40e_aq_set_lb_modes(hw, I40E_AQ_LB_MODE_NONE, NULL); ++ if (ret != I40E_SUCCESS) { ++ PMD_DRV_LOG(ERR, "fail to set loopback link"); ++ goto tx_err; ++ } ++ } ++ + /* Enable mac loopback mode */ +- if (dev->data->dev_conf.lpbk_mode == I40E_AQ_LB_MODE_NONE || +- dev->data->dev_conf.lpbk_mode == I40E_AQ_LB_PHY_LOCAL) { +- ret = i40e_aq_set_lb_modes(hw, dev->data->dev_conf.lpbk_mode, NULL); ++ if (dev->data->dev_conf.lpbk_mode == I40E_AQ_LB_MODE_EN) { ++ if (hw->mac.type == I40E_MAC_X722) ++ ret = i40e_aq_set_lb_modes(hw, I40E_AQ_LB_MAC_LOCAL_X722, NULL); ++ else ++ ret = i40e_aq_set_lb_modes(hw, I40E_AQ_LB_MAC, NULL); + if (ret != I40E_SUCCESS) { + PMD_DRV_LOG(ERR, "fail to set loopback link"); + goto tx_err; +@@ -2449,7 +2459,7 @@ i40e_dev_start(struct rte_eth_dev *dev) + PMD_DRV_LOG(WARNING, "Fail to set phy mask"); + + /* Call get_link_info aq command to enable/disable LSE */ +- i40e_dev_link_update(dev, 0); ++ i40e_dev_link_update(dev, 1); + } + + if (dev->data->dev_conf.intr_conf.rxq == 0) { +@@ -2467,8 +2477,12 @@ i40e_dev_start(struct rte_eth_dev *dev) + "please call hierarchy_commit() " + "before starting the port"); + +- max_frame_size = dev->data->mtu + I40E_ETH_OVERHEAD; +- i40e_set_mac_max_frame(dev, max_frame_size); ++ max_frame_size = dev->data->mtu ? ++ dev->data->mtu + I40E_ETH_OVERHEAD : ++ I40E_FRAME_SIZE_MAX; ++ ++ /* Set the max frame size to HW*/ ++ i40e_aq_set_mac_config(hw, max_frame_size, TRUE, false, 0, NULL); + + return I40E_SUCCESS; + +@@ -2809,9 +2823,6 @@ i40e_dev_set_link_down(struct rte_eth_dev *dev) + return i40e_phy_conf_link(hw, abilities, speed, false); + } + +-#define CHECK_INTERVAL 100 /* 100ms */ +-#define MAX_REPEAT_TIME 10 /* 1s (10 * 100ms) in total */ +- + static __rte_always_inline void + update_link_reg(struct i40e_hw *hw, struct rte_eth_link *link) + { +@@ -2878,6 +2889,8 @@ static __rte_always_inline void + update_link_aq(struct i40e_hw *hw, struct rte_eth_link *link, + bool enable_lse, int wait_to_complete) + { ++#define CHECK_INTERVAL 100 /* 100ms */ ++#define MAX_REPEAT_TIME 10 /* 1s (10 * 100ms) in total */ + uint32_t rep_cnt = MAX_REPEAT_TIME; + struct i40e_link_status link_status; + int status; +@@ -5993,14 +6006,16 @@ i40e_vsi_setup(struct i40e_pf *pf, + } + } + +- /* MAC/VLAN configuration */ +- rte_memcpy(&filter.mac_addr, &broadcast, RTE_ETHER_ADDR_LEN); +- filter.filter_type = I40E_MACVLAN_PERFECT_MATCH; ++ if (vsi->type != I40E_VSI_FDIR) { ++ /* MAC/VLAN configuration for non-FDIR VSI*/ ++ rte_memcpy(&filter.mac_addr, &broadcast, RTE_ETHER_ADDR_LEN); ++ filter.filter_type = I40E_MACVLAN_PERFECT_MATCH; + +- ret = i40e_vsi_add_mac(vsi, &filter); +- if (ret != I40E_SUCCESS) { +- PMD_DRV_LOG(ERR, "Failed to add MACVLAN filter"); +- goto fail_msix_alloc; ++ ret = i40e_vsi_add_mac(vsi, &filter); ++ if (ret != I40E_SUCCESS) { ++ PMD_DRV_LOG(ERR, "Failed to add MACVLAN filter"); ++ goto fail_msix_alloc; ++ } + } + + /* Get VSI BW information */ +@@ -6738,7 +6753,6 @@ i40e_dev_handle_aq_msg(struct rte_eth_dev *dev) + if (!ret) + rte_eth_dev_callback_process(dev, + RTE_ETH_EVENT_INTR_LSC, NULL); +- + break; + default: + PMD_DRV_LOG(DEBUG, "Request %u is not supported yet", +@@ -12123,40 +12137,6 @@ i40e_cloud_filter_qinq_create(struct i40e_pf *pf) + return ret; + } + +-static void +-i40e_set_mac_max_frame(struct rte_eth_dev *dev, uint16_t size) +-{ +- struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); +- uint32_t rep_cnt = MAX_REPEAT_TIME; +- struct rte_eth_link link; +- enum i40e_status_code status; +- bool can_be_set = true; +- +- /* +- * I40E_MEDIA_TYPE_BASET link up can be ignored +- * I40E_MEDIA_TYPE_BASET link down that hw->phy.media_type +- * is I40E_MEDIA_TYPE_UNKNOWN +- */ +- if (hw->phy.media_type != I40E_MEDIA_TYPE_BASET && +- hw->phy.media_type != I40E_MEDIA_TYPE_UNKNOWN) { +- do { +- update_link_reg(hw, &link); +- if (link.link_status) +- break; +- rte_delay_ms(CHECK_INTERVAL); +- } while (--rep_cnt); +- can_be_set = !!link.link_status; +- } +- +- if (can_be_set) { +- status = i40e_aq_set_mac_config(hw, size, TRUE, 0, false, NULL); +- if (status != I40E_SUCCESS) +- PMD_DRV_LOG(ERR, "Failed to set max frame size at port level"); +- } else { +- PMD_DRV_LOG(ERR, "Set max frame size at port level not applicable on link down"); +- } +-} +- + RTE_LOG_REGISTER_SUFFIX(i40e_logtype_init, init, NOTICE); + RTE_LOG_REGISTER_SUFFIX(i40e_logtype_driver, driver, NOTICE); + #ifdef RTE_ETHDEV_DEBUG_RX +diff --git a/dpdk/drivers/net/i40e/i40e_ethdev.h b/dpdk/drivers/net/i40e/i40e_ethdev.h +index fe943a45ff..9b806d130e 100644 +--- a/dpdk/drivers/net/i40e/i40e_ethdev.h ++++ b/dpdk/drivers/net/i40e/i40e_ethdev.h +@@ -48,6 +48,9 @@ + #define I40E_MAX_VF 128 + /*flag of no loopback*/ + #define I40E_AQ_LB_MODE_NONE 0x0 ++#define I40E_AQ_LB_MODE_EN 0x01 ++#define I40E_AQ_LB_MAC 0x01 ++#define I40E_AQ_LB_MAC_LOCAL_X722 0x04 + /* + * vlan_id is a 12 bit number. + * The VFTA array is actually a 4096 bit array, 128 of 32bit elements. +@@ -1497,7 +1500,7 @@ i40e_calc_itr_interval(bool is_pf, bool is_multi_drv) + uint16_t interval = 0; + + if (is_multi_drv) { +- interval = I40E_QUEUE_ITR_INTERVAL_MAX; ++ interval = I40E_QUEUE_ITR_INTERVAL_DEFAULT; + } else { + if (is_pf) + interval = I40E_QUEUE_ITR_INTERVAL_DEFAULT; +diff --git a/dpdk/drivers/net/i40e/i40e_flow.c b/dpdk/drivers/net/i40e/i40e_flow.c +index 65a826d51c..67df77890a 100644 +--- a/dpdk/drivers/net/i40e/i40e_flow.c ++++ b/dpdk/drivers/net/i40e/i40e_flow.c +@@ -1236,6 +1236,14 @@ i40e_flow_parse_attr(const struct rte_flow_attr *attr, + return -rte_errno; + } + ++ /* Not supported */ ++ if (attr->transfer) { ++ rte_flow_error_set(error, EINVAL, ++ RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, ++ attr, "Not support transfer."); ++ return -rte_errno; ++ } ++ + /* Not supported */ + if (attr->priority) { + rte_flow_error_set(error, EINVAL, +diff --git a/dpdk/drivers/net/i40e/i40e_rxtx.c b/dpdk/drivers/net/i40e/i40e_rxtx.c +index 788ffb51c2..d96bbbb677 100644 +--- a/dpdk/drivers/net/i40e/i40e_rxtx.c ++++ b/dpdk/drivers/net/i40e/i40e_rxtx.c +@@ -304,10 +304,7 @@ i40e_txd_enable_checksum(uint64_t ol_flags, + union i40e_tx_offload tx_offload) + { + /* Set MACLEN */ +- if (ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) +- *td_offset |= (tx_offload.outer_l2_len >> 1) +- << I40E_TX_DESC_LENGTH_MACLEN_SHIFT; +- else ++ if (!(ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK)) + *td_offset |= (tx_offload.l2_len >> 1) + << I40E_TX_DESC_LENGTH_MACLEN_SHIFT; + +@@ -1171,9 +1168,12 @@ i40e_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) + + /* Fill in tunneling parameters if necessary */ + cd_tunneling_params = 0; +- if (ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) ++ if (ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) { ++ td_offset |= (tx_offload.outer_l2_len >> 1) ++ << I40E_TX_DESC_LENGTH_MACLEN_SHIFT; + i40e_parse_tunneling_params(ol_flags, tx_offload, + &cd_tunneling_params); ++ } + /* Enable checksum offloading */ + if (ol_flags & I40E_TX_CKSUM_OFFLOAD_MASK) + i40e_txd_enable_checksum(ol_flags, &td_cmd, +@@ -1918,6 +1918,12 @@ i40e_dev_rx_queue_setup_runtime(struct rte_eth_dev *dev, + if (use_def_burst_func) + ad->rx_bulk_alloc_allowed = false; + i40e_set_rx_function(dev); ++ ++ if (ad->rx_vec_allowed && i40e_rxq_vec_setup(rxq)) { ++ PMD_DRV_LOG(ERR, "Failed vector rx setup."); ++ return -EINVAL; ++ } ++ + return 0; + } else if (ad->rx_vec_allowed && !rte_is_power_of_2(rxq->nb_rx_desc)) { + PMD_DRV_LOG(ERR, "Vector mode is allowed, but descriptor" +@@ -2904,6 +2910,8 @@ i40e_rx_queue_config(struct i40e_rx_queue *rxq) + rxq->rx_hdr_len = 0; + rxq->rx_buf_len = RTE_ALIGN_FLOOR(buf_size, + (1 << I40E_RXQ_CTX_DBUFF_SHIFT)); ++ rxq->rx_buf_len = RTE_MIN(rxq->rx_buf_len, ++ I40E_RX_MAX_DATA_BUF_SIZE); + rxq->hs_mode = i40e_header_split_none; + break; + } +diff --git a/dpdk/drivers/net/i40e/i40e_rxtx.h b/dpdk/drivers/net/i40e/i40e_rxtx.h +index 5e6eecc501..a8686224e5 100644 +--- a/dpdk/drivers/net/i40e/i40e_rxtx.h ++++ b/dpdk/drivers/net/i40e/i40e_rxtx.h +@@ -21,6 +21,9 @@ + /* In none-PXE mode QLEN must be whole number of 32 descriptors. */ + #define I40E_ALIGN_RING_DESC 32 + ++/* Max data buffer size must be 16K - 128 bytes */ ++#define I40E_RX_MAX_DATA_BUF_SIZE (16 * 1024 - 128) ++ + #define I40E_MIN_RING_DESC 64 + #define I40E_MAX_RING_DESC 4096 + +@@ -166,7 +169,7 @@ struct i40e_tx_queue { + bool q_set; /**< indicate if tx queue has been configured */ + bool tx_deferred_start; /**< don't start this queue in dev start */ + uint8_t dcb_tc; /**< Traffic class of tx queue */ +- uint64_t offloads; /**< Tx offload flags of RTE_ETH_RX_OFFLOAD_* */ ++ uint64_t offloads; /**< Tx offload flags of RTE_ETH_TX_OFFLOAD_* */ + const struct rte_memzone *mz; + }; + +diff --git a/dpdk/drivers/net/i40e/i40e_rxtx_vec_altivec.c b/dpdk/drivers/net/i40e/i40e_rxtx_vec_altivec.c +index 2dfa04599c..da4a1bc03b 100644 +--- a/dpdk/drivers/net/i40e/i40e_rxtx_vec_altivec.c ++++ b/dpdk/drivers/net/i40e/i40e_rxtx_vec_altivec.c +@@ -448,8 +448,6 @@ _recv_raw_pkts_vec(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_pkts, + + /* Notice: + * - nb_pkts < RTE_I40E_DESCS_PER_LOOP, just return no packet +- * - nb_pkts > RTE_I40E_VPMD_RX_BURST, only scan RTE_I40E_VPMD_RX_BURST +- * numbers of DD bits + */ + uint16_t + i40e_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, +diff --git a/dpdk/drivers/net/i40e/i40e_rxtx_vec_avx512.c b/dpdk/drivers/net/i40e/i40e_rxtx_vec_avx512.c +index 60c97d5331..74ff54c653 100644 +--- a/dpdk/drivers/net/i40e/i40e_rxtx_vec_avx512.c ++++ b/dpdk/drivers/net/i40e/i40e_rxtx_vec_avx512.c +@@ -906,16 +906,13 @@ i40e_tx_free_bufs_avx512(struct i40e_tx_queue *txq) + struct rte_mempool_cache *cache = rte_mempool_default_cache(mp, + rte_lcore_id()); + +- if (!cache || cache->len == 0) +- goto normal; +- +- cache_objs = &cache->objs[cache->len]; +- +- if (n > RTE_MEMPOOL_CACHE_MAX_SIZE) { +- rte_mempool_ops_enqueue_bulk(mp, (void *)txep, n); ++ if (!cache || n > RTE_MEMPOOL_CACHE_MAX_SIZE) { ++ rte_mempool_generic_put(mp, (void *)txep, n, cache); + goto done; + } + ++ cache_objs = &cache->objs[cache->len]; ++ + /* The cache follows the following algorithm + * 1. Add the objects to the cache + * 2. Anything greater than the cache min value (if it +@@ -947,7 +944,6 @@ i40e_tx_free_bufs_avx512(struct i40e_tx_queue *txq) + goto done; + } + +-normal: + m = rte_pktmbuf_prefree_seg(txep[0].mbuf); + if (likely(m)) { + free[0] = m; +diff --git a/dpdk/drivers/net/i40e/i40e_rxtx_vec_common.h b/dpdk/drivers/net/i40e/i40e_rxtx_vec_common.h +index fe1a6ec75e..8b745630e4 100644 +--- a/dpdk/drivers/net/i40e/i40e_rxtx_vec_common.h ++++ b/dpdk/drivers/net/i40e/i40e_rxtx_vec_common.h +@@ -201,6 +201,7 @@ i40e_rxq_vec_setup_default(struct i40e_rx_queue *rxq) + rte_compiler_barrier(); + p = (uintptr_t)&mb_def.rearm_data; + rxq->mbuf_initializer = *(uint64_t *)p; ++ rxq->rx_using_sse = 1; + return 0; + } + +diff --git a/dpdk/drivers/net/i40e/i40e_rxtx_vec_neon.c b/dpdk/drivers/net/i40e/i40e_rxtx_vec_neon.c +index 12e6f1cbcb..90e388ae27 100644 +--- a/dpdk/drivers/net/i40e/i40e_rxtx_vec_neon.c ++++ b/dpdk/drivers/net/i40e/i40e_rxtx_vec_neon.c +@@ -573,8 +573,6 @@ _recv_raw_pkts_vec(struct i40e_rx_queue *__rte_restrict rxq, + /* + * Notice: + * - nb_pkts < RTE_I40E_DESCS_PER_LOOP, just return no packet +- * - nb_pkts > RTE_I40E_VPMD_RX_BURST, only scan RTE_I40E_VPMD_RX_BURST +- * numbers of DD bits + */ + uint16_t + i40e_recv_pkts_vec(void *__rte_restrict rx_queue, +diff --git a/dpdk/drivers/net/i40e/i40e_rxtx_vec_sse.c b/dpdk/drivers/net/i40e/i40e_rxtx_vec_sse.c +index bdc979a839..79029ab433 100644 +--- a/dpdk/drivers/net/i40e/i40e_rxtx_vec_sse.c ++++ b/dpdk/drivers/net/i40e/i40e_rxtx_vec_sse.c +@@ -595,8 +595,6 @@ _recv_raw_pkts_vec(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_pkts, + /* + * Notice: + * - nb_pkts < RTE_I40E_DESCS_PER_LOOP, just return no packet +- * - nb_pkts > RTE_I40E_VPMD_RX_BURST, only scan RTE_I40E_VPMD_RX_BURST +- * numbers of DD bits + */ + uint16_t + i40e_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, +diff --git a/dpdk/drivers/net/iavf/iavf.h b/dpdk/drivers/net/iavf/iavf.h +index 1edebab8dc..aa18650ffa 100644 +--- a/dpdk/drivers/net/iavf/iavf.h ++++ b/dpdk/drivers/net/iavf/iavf.h +@@ -262,6 +262,7 @@ struct iavf_info { + struct iavf_qv_map *qv_map; /* queue vector mapping */ + struct iavf_flow_list flow_list; + rte_spinlock_t flow_ops_lock; ++ rte_spinlock_t aq_lock; + struct iavf_parser_list rss_parser_list; + struct iavf_parser_list dist_parser_list; + struct iavf_parser_list ipsec_crypto_parser_list; +diff --git a/dpdk/drivers/net/iavf/iavf_ethdev.c b/dpdk/drivers/net/iavf/iavf_ethdev.c +index 3196210f2c..a12ea39444 100644 +--- a/dpdk/drivers/net/iavf/iavf_ethdev.c ++++ b/dpdk/drivers/net/iavf/iavf_ethdev.c +@@ -131,6 +131,8 @@ static int iavf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, + uint16_t queue_id); + static int iavf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, + uint16_t queue_id); ++static void iavf_dev_interrupt_handler(void *param); ++static void iavf_disable_irq0(struct iavf_hw *hw); + static int iavf_dev_flow_ops_get(struct rte_eth_dev *dev, + const struct rte_flow_ops **ops); + static int iavf_set_mc_addr_list(struct rte_eth_dev *dev, +@@ -1065,6 +1067,9 @@ iavf_dev_stop(struct rte_eth_dev *dev) + + PMD_INIT_FUNC_TRACE(); + ++ if (vf->vf_reset) ++ return 0; ++ + if (adapter->closed) + return -1; + +@@ -1075,8 +1080,6 @@ iavf_dev_stop(struct rte_eth_dev *dev) + if (adapter->stopped == 1) + return 0; + +- iavf_stop_queues(dev); +- + /* Disable the interrupt for Rx */ + rte_intr_efd_disable(intr_handle); + /* Rx interrupt vector mapping free */ +@@ -1089,6 +1092,8 @@ iavf_dev_stop(struct rte_eth_dev *dev) + iavf_add_del_mc_addr_list(adapter, vf->mc_addrs, vf->mc_addrs_num, + false); + ++ iavf_stop_queues(dev); ++ + adapter->stopped = 1; + dev->data->dev_started = 0; + +@@ -1178,6 +1183,8 @@ iavf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) + .nb_max = IAVF_MAX_RING_DESC, + .nb_min = IAVF_MIN_RING_DESC, + .nb_align = IAVF_ALIGN_RING_DESC, ++ .nb_mtu_seg_max = IAVF_TX_MAX_MTU_SEG, ++ .nb_seg_max = IAVF_MAX_RING_DESC, + }; + + dev_info->err_handle_mode = RTE_ETH_ERROR_HANDLE_MODE_PASSIVE; +@@ -1350,6 +1357,7 @@ iavf_dev_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) + struct iavf_adapter *adapter = + IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); ++ struct rte_eth_conf *dev_conf = &dev->data->dev_conf; + int err; + + if (adapter->closed) +@@ -1368,6 +1376,23 @@ iavf_dev_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) + err = iavf_add_del_vlan(adapter, vlan_id, on); + if (err) + return -EIO; ++ ++ /* For i40e kernel driver which only supports vlan(v1) VIRTCHNL OP, ++ * it will set strip on when setting filter on but dpdk side will not ++ * change strip flag. To be consistent with dpdk side, disable strip ++ * again. ++ * ++ * For i40e kernel driver which supports vlan v2, dpdk will invoke vlan v2 ++ * related function, so it won't go through here. ++ */ ++ if (adapter->hw.mac.type == IAVF_MAC_XL710 || ++ adapter->hw.mac.type == IAVF_MAC_X722_VF) { ++ if (on && !(dev_conf->rxmode.offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP)) { ++ err = iavf_disable_vlan_strip(adapter); ++ if (err) ++ return -EIO; ++ } ++ } + return 0; + } + +@@ -2607,6 +2632,9 @@ iavf_dev_init(struct rte_eth_dev *eth_dev) + adapter->dev_data = eth_dev->data; + adapter->stopped = 1; + ++ if (iavf_dev_event_handler_init()) ++ goto init_vf_err; ++ + if (iavf_init_vf(eth_dev) != 0) { + PMD_INIT_LOG(ERR, "Init vf failed"); + return -1; +@@ -2634,8 +2662,6 @@ iavf_dev_init(struct rte_eth_dev *eth_dev) + rte_ether_addr_copy((struct rte_ether_addr *)hw->mac.addr, + ð_dev->data->mac_addrs[0]); + +- if (iavf_dev_event_handler_init()) +- goto init_vf_err; + + if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_WB_ON_ITR) { + /* register callback func to eal lib */ +@@ -2667,18 +2693,19 @@ iavf_dev_init(struct rte_eth_dev *eth_dev) + ret = iavf_security_ctx_create(adapter); + if (ret) { + PMD_INIT_LOG(ERR, "failed to create ipsec crypto security instance"); +- return ret; ++ goto flow_init_err; + } + + ret = iavf_security_init(adapter); + if (ret) { + PMD_INIT_LOG(ERR, "failed to initialized ipsec crypto resources"); +- return ret; ++ goto security_init_err; + } + } + + iavf_default_rss_disable(adapter); + ++ iavf_dev_stats_reset(eth_dev); + + /* Start device watchdog */ + iavf_dev_watchdog_enable(adapter); +@@ -2686,7 +2713,23 @@ iavf_dev_init(struct rte_eth_dev *eth_dev) + + return 0; + ++security_init_err: ++ iavf_security_ctx_destroy(adapter); ++ + flow_init_err: ++ iavf_disable_irq0(hw); ++ ++ if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_WB_ON_ITR) { ++ /* disable uio intr before callback unregiser */ ++ rte_intr_disable(pci_dev->intr_handle); ++ ++ /* unregister callback func from eal lib */ ++ rte_intr_callback_unregister(pci_dev->intr_handle, ++ iavf_dev_interrupt_handler, eth_dev); ++ } else { ++ rte_eal_alarm_cancel(iavf_dev_alarm_handler, eth_dev); ++ } ++ + rte_free(eth_dev->data->mac_addrs); + eth_dev->data->mac_addrs = NULL; + +@@ -2732,6 +2775,18 @@ iavf_dev_close(struct rte_eth_dev *dev) + if (vf->promisc_unicast_enabled || vf->promisc_multicast_enabled) + iavf_config_promisc(adapter, false, false); + ++ /* ++ * Release redundant queue resource when close the dev ++ * so that other vfs can re-use the queues. ++ */ ++ if (vf->lv_enabled) { ++ ret = iavf_request_queues(dev, IAVF_MAX_NUM_QUEUES_DFLT); ++ if (ret) ++ PMD_DRV_LOG(ERR, "Reset the num of queues failed"); ++ ++ vf->max_rss_qregion = IAVF_MAX_NUM_QUEUES_DFLT; ++ } ++ + iavf_shutdown_adminq(hw); + if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_WB_ON_ITR) { + /* disable uio intr before callback unregister */ +diff --git a/dpdk/drivers/net/iavf/iavf_generic_flow.c b/dpdk/drivers/net/iavf/iavf_generic_flow.c +index f33c764764..6f6e95fc45 100644 +--- a/dpdk/drivers/net/iavf/iavf_generic_flow.c ++++ b/dpdk/drivers/net/iavf/iavf_generic_flow.c +@@ -2278,11 +2278,12 @@ iavf_flow_create(struct rte_eth_dev *dev, + } + + flow->engine = engine; ++ rte_spinlock_lock(&vf->flow_ops_lock); + TAILQ_INSERT_TAIL(&vf->flow_list, flow, node); ++ rte_spinlock_unlock(&vf->flow_ops_lock); + PMD_DRV_LOG(INFO, "Succeeded to create (%d) flow", engine->type); + + free_flow: +- rte_spinlock_unlock(&vf->flow_ops_lock); + return flow; + } + +diff --git a/dpdk/drivers/net/iavf/iavf_hash.c b/dpdk/drivers/net/iavf/iavf_hash.c +index ae6fb38594..cf4d677101 100644 +--- a/dpdk/drivers/net/iavf/iavf_hash.c ++++ b/dpdk/drivers/net/iavf/iavf_hash.c +@@ -886,8 +886,8 @@ iavf_hash_parse_raw_pattern(const struct rte_flow_item *item, + struct iavf_rss_meta *meta) + { + const struct rte_flow_item_raw *raw_spec, *raw_mask; ++ uint16_t spec_len, pkt_len; + uint8_t *pkt_buf, *msk_buf; +- uint8_t spec_len, pkt_len; + uint8_t tmp_val = 0; + uint8_t tmp_c = 0; + int i, j; +diff --git a/dpdk/drivers/net/iavf/iavf_ipsec_crypto.c b/dpdk/drivers/net/iavf/iavf_ipsec_crypto.c +index 868921cac5..26459088af 100644 +--- a/dpdk/drivers/net/iavf/iavf_ipsec_crypto.c ++++ b/dpdk/drivers/net/iavf/iavf_ipsec_crypto.c +@@ -828,6 +828,7 @@ iavf_ipsec_crypto_sa_update_esn(struct iavf_adapter *adapter, + /* set request params */ + request->ipsec_data.sa_update->sa_index = sess->sa.hw_idx; + request->ipsec_data.sa_update->esn_hi = sess->esn.hi; ++ request->ipsec_data.sa_update->esn_low = sess->esn.low; + + /* send virtual channel request to add SA to hardware database */ + rc = iavf_ipsec_crypto_request(adapter, +diff --git a/dpdk/drivers/net/iavf/iavf_rxtx.c b/dpdk/drivers/net/iavf/iavf_rxtx.c +index cf87a6beda..6a0cf31a4c 100644 +--- a/dpdk/drivers/net/iavf/iavf_rxtx.c ++++ b/dpdk/drivers/net/iavf/iavf_rxtx.c +@@ -654,6 +654,7 @@ iavf_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, + + len = rte_pktmbuf_data_room_size(rxq->mp) - RTE_PKTMBUF_HEADROOM; + rxq->rx_buf_len = RTE_ALIGN_FLOOR(len, (1 << IAVF_RXQ_CTX_DBUFF_SHIFT)); ++ rxq->rx_buf_len = RTE_MIN(rxq->rx_buf_len, IAVF_RX_MAX_DATA_BUF_SIZE); + + /* Allocate the software ring. */ + len = nb_desc + IAVF_RX_MAX_BURST; +@@ -2453,8 +2454,11 @@ iavf_fill_ctx_desc_tunnelling_field(volatile uint64_t *qw0, + * Calculate the tunneling UDP checksum. + * Shall be set only if L4TUNT = 01b and EIPT is not zero + */ +- if (!(eip_typ & IAVF_TX_CTX_EXT_IP_NONE) && +- (eip_typ & IAVF_TXD_CTX_UDP_TUNNELING)) ++ if ((eip_typ & (IAVF_TX_CTX_EXT_IP_IPV6 | ++ IAVF_TX_CTX_EXT_IP_IPV4 | ++ IAVF_TX_CTX_EXT_IP_IPV4_NO_CSUM)) && ++ (eip_typ & IAVF_TXD_CTX_UDP_TUNNELING) && ++ (m->ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM)) + eip_typ |= IAVF_TXD_CTX_QW0_L4T_CS_MASK; + } + +@@ -2479,7 +2483,7 @@ iavf_fill_ctx_desc_segmentation_field(volatile uint64_t *field, + total_length -= m->outer_l3_len + m->outer_l2_len; + } + +-#ifdef RTE_LIBRTE_IAVF_DEBUG_TX ++#ifdef RTE_ETHDEV_DEBUG_TX + if (!m->l4_len || !m->tso_segsz) + PMD_TX_LOG(DEBUG, "L4 length %d, LSO Segment size %d", + m->l4_len, m->tso_segsz); +@@ -2593,6 +2597,10 @@ iavf_build_data_desc_cmd_offset_fields(volatile uint64_t *qw1, + l2tag1 |= m->vlan_tci; + } + ++ if ((m->ol_flags & ++ (IAVF_TX_CKSUM_OFFLOAD_MASK | RTE_MBUF_F_TX_SEC_OFFLOAD)) == 0) ++ goto skip_cksum; ++ + /* Set MACLEN */ + if (m->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK && + !(m->ol_flags & RTE_MBUF_F_TX_SEC_OFFLOAD)) +@@ -2616,10 +2624,21 @@ iavf_build_data_desc_cmd_offset_fields(volatile uint64_t *qw1, + offset |= (m->l3_len >> 2) << IAVF_TX_DESC_LENGTH_IPLEN_SHIFT; + } + +- if (m->ol_flags & RTE_MBUF_F_TX_TCP_SEG) { +- command |= IAVF_TX_DESC_CMD_L4T_EOFT_TCP; ++ if (m->ol_flags & (RTE_MBUF_F_TX_TCP_SEG | RTE_MBUF_F_TX_UDP_SEG)) { ++ if (m->ol_flags & RTE_MBUF_F_TX_TCP_SEG) ++ command |= IAVF_TX_DESC_CMD_L4T_EOFT_TCP; ++ else ++ command |= IAVF_TX_DESC_CMD_L4T_EOFT_UDP; + offset |= (m->l4_len >> 2) << + IAVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; ++ ++ *qw1 = rte_cpu_to_le_64((((uint64_t)command << ++ IAVF_TXD_DATA_QW1_CMD_SHIFT) & IAVF_TXD_DATA_QW1_CMD_MASK) | ++ (((uint64_t)offset << IAVF_TXD_DATA_QW1_OFFSET_SHIFT) & ++ IAVF_TXD_DATA_QW1_OFFSET_MASK) | ++ ((uint64_t)l2tag1 << IAVF_TXD_DATA_QW1_L2TAG1_SHIFT)); ++ ++ return; + } + + /* Enable L4 checksum offloads */ +@@ -2641,6 +2660,7 @@ iavf_build_data_desc_cmd_offset_fields(volatile uint64_t *qw1, + break; + } + ++skip_cksum: + *qw1 = rte_cpu_to_le_64((((uint64_t)command << + IAVF_TXD_DATA_QW1_CMD_SHIFT) & IAVF_TXD_DATA_QW1_CMD_MASK) | + (((uint64_t)offset << IAVF_TXD_DATA_QW1_OFFSET_SHIFT) & +@@ -2810,7 +2830,7 @@ iavf_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) + txe->last_id = desc_idx_last; + desc_idx = txe->next_id; + txe = txn; +- } ++ } + + if (nb_desc_ipsec) { + volatile struct iavf_tx_ipsec_desc *ipsec_desc = +@@ -2823,7 +2843,7 @@ iavf_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) + if (txe->mbuf) { + rte_pktmbuf_free_seg(txe->mbuf); + txe->mbuf = NULL; +- } ++ } + + iavf_fill_ipsec_desc(ipsec_desc, ipsec_md, &ipseclen); + +@@ -2956,7 +2976,6 @@ iavf_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts, + struct rte_mbuf *m; + struct iavf_tx_queue *txq = tx_queue; + struct rte_eth_dev *dev = &rte_eth_devices[txq->port_id]; +- uint16_t max_frame_size = dev->data->mtu + IAVF_ETH_OVERHEAD; + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); + struct iavf_adapter *adapter = IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + +@@ -2974,7 +2993,8 @@ iavf_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts, + return i; + } + } else if ((m->tso_segsz < IAVF_MIN_TSO_MSS) || +- (m->tso_segsz > IAVF_MAX_TSO_MSS)) { ++ (m->tso_segsz > IAVF_MAX_TSO_MSS) || ++ (m->nb_segs > txq->nb_tx_desc)) { + /* MSS outside the range are considered malicious */ + rte_errno = EINVAL; + return i; +@@ -2985,11 +3005,8 @@ iavf_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts, + return i; + } + +- /* check the data_len in mbuf */ +- if (m->data_len < IAVF_TX_MIN_PKT_LEN || +- m->data_len > max_frame_size) { ++ if (m->pkt_len < IAVF_TX_MIN_PKT_LEN) { + rte_errno = EINVAL; +- PMD_DRV_LOG(ERR, "INVALID mbuf: bad data_len=[%hu]", m->data_len); + return i; + } + +diff --git a/dpdk/drivers/net/iavf/iavf_rxtx.h b/dpdk/drivers/net/iavf/iavf_rxtx.h +index a6ad88885b..180f7ec108 100644 +--- a/dpdk/drivers/net/iavf/iavf_rxtx.h ++++ b/dpdk/drivers/net/iavf/iavf_rxtx.h +@@ -16,6 +16,9 @@ + /* used for Rx Bulk Allocate */ + #define IAVF_RX_MAX_BURST 32 + ++/* Max data buffer size must be 16K - 128 bytes */ ++#define IAVF_RX_MAX_DATA_BUF_SIZE (16 * 1024 - 128) ++ + /* used for Vector PMD */ + #define IAVF_VPMD_RX_MAX_BURST 32 + #define IAVF_VPMD_TX_MAX_BURST 32 +@@ -24,6 +27,8 @@ + #define IAVF_VPMD_TX_MAX_FREE_BUF 64 + + #define IAVF_TX_NO_VECTOR_FLAGS ( \ ++ RTE_ETH_TX_OFFLOAD_VLAN_INSERT | \ ++ RTE_ETH_TX_OFFLOAD_QINQ_INSERT | \ + RTE_ETH_TX_OFFLOAD_MULTI_SEGS | \ + RTE_ETH_TX_OFFLOAD_TCP_TSO | \ + RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM | \ +@@ -31,8 +36,6 @@ + RTE_ETH_TX_OFFLOAD_SECURITY) + + #define IAVF_TX_VECTOR_OFFLOAD ( \ +- RTE_ETH_TX_OFFLOAD_VLAN_INSERT | \ +- RTE_ETH_TX_OFFLOAD_QINQ_INSERT | \ + RTE_ETH_TX_OFFLOAD_IPV4_CKSUM | \ + RTE_ETH_TX_OFFLOAD_SCTP_CKSUM | \ + RTE_ETH_TX_OFFLOAD_UDP_CKSUM | \ +@@ -76,7 +79,7 @@ + RTE_MBUF_F_TX_TUNNEL_MASK | \ + RTE_MBUF_F_TX_OUTER_IP_CKSUM | \ + RTE_MBUF_F_TX_OUTER_UDP_CKSUM | \ +- RTE_ETH_TX_OFFLOAD_SECURITY) ++ RTE_MBUF_F_TX_SEC_OFFLOAD) + + #define IAVF_TX_OFFLOAD_NOTSUP_MASK \ + (RTE_MBUF_F_TX_OFFLOAD_MASK ^ IAVF_TX_OFFLOAD_MASK) +diff --git a/dpdk/drivers/net/iavf/iavf_rxtx_vec_avx2.c b/dpdk/drivers/net/iavf/iavf_rxtx_vec_avx2.c +index 862f6eb0c0..b4ebac9d34 100644 +--- a/dpdk/drivers/net/iavf/iavf_rxtx_vec_avx2.c ++++ b/dpdk/drivers/net/iavf/iavf_rxtx_vec_avx2.c +@@ -1074,7 +1074,10 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq, + _mm256_set_epi8(0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, +- 0, 0, 0, 0, ++ 0, 0, ++ RTE_MBUF_F_RX_VLAN | ++ RTE_MBUF_F_RX_VLAN_STRIPPED, ++ 0, + /* end up 128-bits */ + 0, 0, 0, 0, + 0, 0, 0, 0, +diff --git a/dpdk/drivers/net/iavf/iavf_rxtx_vec_avx512.c b/dpdk/drivers/net/iavf/iavf_rxtx_vec_avx512.c +index b416a716cf..b0546a14c6 100644 +--- a/dpdk/drivers/net/iavf/iavf_rxtx_vec_avx512.c ++++ b/dpdk/drivers/net/iavf/iavf_rxtx_vec_avx512.c +@@ -1338,7 +1338,10 @@ _iavf_recv_raw_pkts_vec_avx512_flex_rxd(struct iavf_rx_queue *rxq, + (0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, +- 0, 0, 0, 0, ++ 0, 0, ++ RTE_MBUF_F_RX_VLAN | ++ RTE_MBUF_F_RX_VLAN_STRIPPED, ++ 0, + /* end up 128-bits */ + 0, 0, 0, 0, + 0, 0, 0, 0, +diff --git a/dpdk/drivers/net/iavf/iavf_vchnl.c b/dpdk/drivers/net/iavf/iavf_vchnl.c +index f92daf97f2..aeffb07cca 100644 +--- a/dpdk/drivers/net/iavf/iavf_vchnl.c ++++ b/dpdk/drivers/net/iavf/iavf_vchnl.c +@@ -256,6 +256,7 @@ iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len, + vf->link_speed = iavf_convert_link_speed(speed); + } + iavf_dev_link_update(vf->eth_dev, 0); ++ iavf_dev_event_post(vf->eth_dev, RTE_ETH_EVENT_INTR_LSC, NULL, 0); + PMD_DRV_LOG(INFO, "Link status update:%s", + vf->link_up ? "up" : "down"); + break; +@@ -368,28 +369,48 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args, + _clear_cmd(vf); + break; + default: +- /* For other virtchnl ops in running time, +- * wait for the cmd done flag. +- */ +- do { +- if (vf->pend_cmd == VIRTCHNL_OP_UNKNOWN) +- break; +- iavf_msec_delay(ASQ_DELAY_MS); +- /* If don't read msg or read sys event, continue */ +- } while (i++ < MAX_TRY_TIMES); +- +- if (i >= MAX_TRY_TIMES) { +- PMD_DRV_LOG(ERR, "No response for cmd %d", args->ops); ++ if (rte_thread_is_intr()) { ++ /* For virtchnl ops were executed in eal_intr_thread, ++ * need to poll the response. ++ */ ++ do { ++ result = iavf_read_msg_from_pf(adapter, args->out_size, ++ args->out_buffer); ++ if (result == IAVF_MSG_CMD) ++ break; ++ iavf_msec_delay(ASQ_DELAY_MS); ++ } while (i++ < MAX_TRY_TIMES); ++ if (i >= MAX_TRY_TIMES || ++ vf->cmd_retval != VIRTCHNL_STATUS_SUCCESS) { ++ err = -1; ++ PMD_DRV_LOG(ERR, "No response or return failure (%d)" ++ " for cmd %d", vf->cmd_retval, args->ops); ++ } + _clear_cmd(vf); +- err = -EIO; +- } else if (vf->cmd_retval == +- VIRTCHNL_STATUS_ERR_NOT_SUPPORTED) { +- PMD_DRV_LOG(ERR, "Cmd %d not supported", args->ops); +- err = -ENOTSUP; +- } else if (vf->cmd_retval != VIRTCHNL_STATUS_SUCCESS) { +- PMD_DRV_LOG(ERR, "Return failure %d for cmd %d", +- vf->cmd_retval, args->ops); +- err = -EINVAL; ++ } else { ++ /* For other virtchnl ops in running time, ++ * wait for the cmd done flag. ++ */ ++ do { ++ if (vf->pend_cmd == VIRTCHNL_OP_UNKNOWN) ++ break; ++ iavf_msec_delay(ASQ_DELAY_MS); ++ /* If don't read msg or read sys event, continue */ ++ } while (i++ < MAX_TRY_TIMES); ++ ++ if (i >= MAX_TRY_TIMES) { ++ PMD_DRV_LOG(ERR, "No response for cmd %d", args->ops); ++ _clear_cmd(vf); ++ err = -EIO; ++ } else if (vf->cmd_retval == ++ VIRTCHNL_STATUS_ERR_NOT_SUPPORTED) { ++ PMD_DRV_LOG(ERR, "Cmd %d not supported", args->ops); ++ err = -ENOTSUP; ++ } else if (vf->cmd_retval != VIRTCHNL_STATUS_SUCCESS) { ++ PMD_DRV_LOG(ERR, "Return failure %d for cmd %d", ++ vf->cmd_retval, args->ops); ++ err = -EINVAL; ++ } + } + break; + } +@@ -397,6 +418,26 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args, + return err; + } + ++static int ++iavf_execute_vf_cmd_safe(struct iavf_adapter *adapter, ++ struct iavf_cmd_info *args, int async) ++{ ++ struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); ++ int ret; ++ int is_intr_thread = rte_thread_is_intr(); ++ ++ if (is_intr_thread) { ++ if (!rte_spinlock_trylock(&vf->aq_lock)) ++ return -EIO; ++ } else { ++ rte_spinlock_lock(&vf->aq_lock); ++ } ++ ret = iavf_execute_vf_cmd(adapter, args, async); ++ rte_spinlock_unlock(&vf->aq_lock); ++ ++ return ret; ++} ++ + static void + iavf_handle_pf_event_msg(struct rte_eth_dev *dev, uint8_t *msg, + uint16_t msglen) +@@ -554,7 +595,7 @@ iavf_enable_vlan_strip(struct iavf_adapter *adapter) + args.in_args_size = 0; + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; +- ret = iavf_execute_vf_cmd(adapter, &args, 0); ++ ret = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (ret) + PMD_DRV_LOG(ERR, "Failed to execute command of" + " OP_ENABLE_VLAN_STRIPPING"); +@@ -575,7 +616,7 @@ iavf_disable_vlan_strip(struct iavf_adapter *adapter) + args.in_args_size = 0; + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; +- ret = iavf_execute_vf_cmd(adapter, &args, 0); ++ ret = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (ret) + PMD_DRV_LOG(ERR, "Failed to execute command of" + " OP_DISABLE_VLAN_STRIPPING"); +@@ -604,7 +645,7 @@ iavf_check_api_version(struct iavf_adapter *adapter) + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (err) { + PMD_INIT_LOG(ERR, "Fail to execute command of OP_VERSION"); + return err; +@@ -665,7 +706,7 @@ iavf_get_vf_resource(struct iavf_adapter *adapter) + args.in_args = (uint8_t *)∩︀ + args.in_args_size = sizeof(caps); + +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + + if (err) { + PMD_DRV_LOG(ERR, +@@ -710,7 +751,7 @@ iavf_get_supported_rxdid(struct iavf_adapter *adapter) + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + +- ret = iavf_execute_vf_cmd(adapter, &args, 0); ++ ret = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (ret) { + PMD_DRV_LOG(ERR, + "Failed to execute command of OP_GET_SUPPORTED_RXDIDS"); +@@ -754,7 +795,7 @@ iavf_config_vlan_strip_v2(struct iavf_adapter *adapter, bool enable) + args.in_args_size = sizeof(vlan_strip); + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; +- ret = iavf_execute_vf_cmd(adapter, &args, 0); ++ ret = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (ret) + PMD_DRV_LOG(ERR, "fail to execute command %s", + enable ? "VIRTCHNL_OP_ENABLE_VLAN_STRIPPING_V2" : +@@ -794,7 +835,7 @@ iavf_config_vlan_insert_v2(struct iavf_adapter *adapter, bool enable) + args.in_args_size = sizeof(vlan_insert); + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; +- ret = iavf_execute_vf_cmd(adapter, &args, 0); ++ ret = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (ret) + PMD_DRV_LOG(ERR, "fail to execute command %s", + enable ? "VIRTCHNL_OP_ENABLE_VLAN_INSERTION_V2" : +@@ -837,7 +878,7 @@ iavf_add_del_vlan_v2(struct iavf_adapter *adapter, uint16_t vlanid, bool add) + args.in_args_size = sizeof(vlan_filter); + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (err) + PMD_DRV_LOG(ERR, "fail to execute command %s", + add ? "OP_ADD_VLAN_V2" : "OP_DEL_VLAN_V2"); +@@ -858,7 +899,7 @@ iavf_get_vlan_offload_caps_v2(struct iavf_adapter *adapter) + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + +- ret = iavf_execute_vf_cmd(adapter, &args, 0); ++ ret = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (ret) { + PMD_DRV_LOG(ERR, + "Failed to execute command of VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS"); +@@ -889,7 +930,7 @@ iavf_enable_queues(struct iavf_adapter *adapter) + args.in_args_size = sizeof(queue_select); + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (err) { + PMD_DRV_LOG(ERR, + "Failed to execute command of OP_ENABLE_QUEUES"); +@@ -917,7 +958,7 @@ iavf_disable_queues(struct iavf_adapter *adapter) + args.in_args_size = sizeof(queue_select); + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (err) { + PMD_DRV_LOG(ERR, + "Failed to execute command of OP_DISABLE_QUEUES"); +@@ -953,7 +994,7 @@ iavf_switch_queue(struct iavf_adapter *adapter, uint16_t qid, + args.in_args_size = sizeof(queue_select); + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (err) + PMD_DRV_LOG(ERR, "Failed to execute command of %s", + on ? "OP_ENABLE_QUEUES" : "OP_DISABLE_QUEUES"); +@@ -995,7 +1036,7 @@ iavf_enable_queues_lv(struct iavf_adapter *adapter) + args.in_args_size = len; + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (err) + PMD_DRV_LOG(ERR, + "Failed to execute command of OP_ENABLE_QUEUES_V2"); +@@ -1039,7 +1080,7 @@ iavf_disable_queues_lv(struct iavf_adapter *adapter) + args.in_args_size = len; + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (err) + PMD_DRV_LOG(ERR, + "Failed to execute command of OP_DISABLE_QUEUES_V2"); +@@ -1085,7 +1126,7 @@ iavf_switch_queue_lv(struct iavf_adapter *adapter, uint16_t qid, + args.in_args_size = len; + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (err) + PMD_DRV_LOG(ERR, "Failed to execute command of %s", + on ? "OP_ENABLE_QUEUES_V2" : "OP_DISABLE_QUEUES_V2"); +@@ -1117,7 +1158,7 @@ iavf_configure_rss_lut(struct iavf_adapter *adapter) + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (err) + PMD_DRV_LOG(ERR, + "Failed to execute command of OP_CONFIG_RSS_LUT"); +@@ -1149,7 +1190,7 @@ iavf_configure_rss_key(struct iavf_adapter *adapter) + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (err) + PMD_DRV_LOG(ERR, + "Failed to execute command of OP_CONFIG_RSS_KEY"); +@@ -1247,7 +1288,7 @@ iavf_configure_queues(struct iavf_adapter *adapter, + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (err) + PMD_DRV_LOG(ERR, "Failed to execute command of" + " VIRTCHNL_OP_CONFIG_VSI_QUEUES"); +@@ -1288,7 +1329,7 @@ iavf_config_irq_map(struct iavf_adapter *adapter) + args.in_args_size = len; + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (err) + PMD_DRV_LOG(ERR, "fail to execute command OP_CONFIG_IRQ_MAP"); + +@@ -1329,7 +1370,7 @@ iavf_config_irq_map_lv(struct iavf_adapter *adapter, uint16_t num, + args.in_args_size = len; + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (err) + PMD_DRV_LOG(ERR, "fail to execute command OP_MAP_QUEUE_VECTOR"); + +@@ -1389,7 +1430,7 @@ iavf_add_del_all_mac_addr(struct iavf_adapter *adapter, bool add) + args.in_args_size = len; + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (err) + PMD_DRV_LOG(ERR, "fail to execute command %s", + add ? "OP_ADD_ETHER_ADDRESS" : +@@ -1419,7 +1460,7 @@ iavf_query_stats(struct iavf_adapter *adapter, + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (err) { + PMD_DRV_LOG(ERR, "fail to execute command OP_GET_STATS"); + *pstats = NULL; +@@ -1457,7 +1498,7 @@ iavf_config_promisc(struct iavf_adapter *adapter, + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + + if (err) { + PMD_DRV_LOG(ERR, +@@ -1500,7 +1541,7 @@ iavf_add_del_eth_addr(struct iavf_adapter *adapter, struct rte_ether_addr *addr, + args.in_args_size = sizeof(cmd_buffer); + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (err) + PMD_DRV_LOG(ERR, "fail to execute command %s", + add ? "OP_ADD_ETH_ADDR" : "OP_DEL_ETH_ADDR"); +@@ -1527,7 +1568,7 @@ iavf_add_del_vlan(struct iavf_adapter *adapter, uint16_t vlanid, bool add) + args.in_args_size = sizeof(cmd_buffer); + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (err) + PMD_DRV_LOG(ERR, "fail to execute command %s", + add ? "OP_ADD_VLAN" : "OP_DEL_VLAN"); +@@ -1554,7 +1595,7 @@ iavf_fdir_add(struct iavf_adapter *adapter, + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (err) { + PMD_DRV_LOG(ERR, "fail to execute command OP_ADD_FDIR_FILTER"); + return err; +@@ -1614,7 +1655,7 @@ iavf_fdir_del(struct iavf_adapter *adapter, + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (err) { + PMD_DRV_LOG(ERR, "fail to execute command OP_DEL_FDIR_FILTER"); + return err; +@@ -1661,7 +1702,7 @@ iavf_fdir_check(struct iavf_adapter *adapter, + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (err) { + PMD_DRV_LOG(ERR, "fail to check flow director rule"); + return err; +@@ -1704,7 +1745,7 @@ iavf_flow_sub(struct iavf_adapter *adapter, struct iavf_fsub_conf *filter) + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (err) { + PMD_DRV_LOG(ERR, "Failed to execute command of " + "OP_FLOW_SUBSCRIBE"); +@@ -1755,7 +1796,7 @@ iavf_flow_unsub(struct iavf_adapter *adapter, struct iavf_fsub_conf *filter) + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (err) { + PMD_DRV_LOG(ERR, "Failed to execute command of " + "OP_FLOW_UNSUBSCRIBE"); +@@ -1798,7 +1839,7 @@ iavf_flow_sub_check(struct iavf_adapter *adapter, + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (err) { + PMD_DRV_LOG(ERR, "Failed to check flow subscription rule"); + return err; +@@ -1838,7 +1879,7 @@ iavf_add_del_rss_cfg(struct iavf_adapter *adapter, + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (err) + PMD_DRV_LOG(ERR, + "Failed to execute command of %s", +@@ -1861,7 +1902,7 @@ iavf_get_hena_caps(struct iavf_adapter *adapter, uint64_t *caps) + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (err) { + PMD_DRV_LOG(ERR, + "Failed to execute command of OP_GET_RSS_HENA_CAPS"); +@@ -1887,7 +1928,7 @@ iavf_set_hena(struct iavf_adapter *adapter, uint64_t hena) + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (err) + PMD_DRV_LOG(ERR, + "Failed to execute command of OP_SET_RSS_HENA"); +@@ -1908,7 +1949,7 @@ iavf_get_qos_cap(struct iavf_adapter *adapter) + args.in_args_size = 0; + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + + if (err) { + PMD_DRV_LOG(ERR, +@@ -1941,7 +1982,7 @@ int iavf_set_q_tc_map(struct rte_eth_dev *dev, + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (err) + PMD_DRV_LOG(ERR, "Failed to execute command of" + " VIRTCHNL_OP_CONFIG_TC_MAP"); +@@ -1964,7 +2005,7 @@ int iavf_set_q_bw(struct rte_eth_dev *dev, + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (err) + PMD_DRV_LOG(ERR, "Failed to execute command of" + " VIRTCHNL_OP_CONFIG_QUEUE_BW"); +@@ -2009,7 +2050,7 @@ iavf_add_del_mc_addr_list(struct iavf_adapter *adapter, + i * sizeof(struct virtchnl_ether_addr); + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + + if (err) { + PMD_DRV_LOG(ERR, "fail to execute command %s", +@@ -2053,13 +2094,17 @@ iavf_request_queues(struct rte_eth_dev *dev, uint16_t num) + if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_WB_ON_ITR) { + /* disable interrupt to avoid the admin queue message to be read + * before iavf_read_msg_from_pf. ++ * ++ * don't disable interrupt handler until ready to execute vf cmd. + */ ++ rte_spinlock_lock(&vf->aq_lock); + rte_intr_disable(pci_dev->intr_handle); + err = iavf_execute_vf_cmd(adapter, &args, 0); + rte_intr_enable(pci_dev->intr_handle); ++ rte_spinlock_unlock(&vf->aq_lock); + } else { + rte_eal_alarm_cancel(iavf_dev_alarm_handler, dev); +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + rte_eal_alarm_set(IAVF_ALARM_INTERVAL, + iavf_dev_alarm_handler, dev); + } +@@ -2098,7 +2143,7 @@ iavf_get_max_rss_queue_region(struct iavf_adapter *adapter) + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (err) { + PMD_DRV_LOG(ERR, "Failed to execute command of VIRTCHNL_OP_GET_MAX_RSS_QREGION"); + return err; +@@ -2129,7 +2174,7 @@ iavf_ipsec_crypto_request(struct iavf_adapter *adapter, + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + +- err = iavf_execute_vf_cmd(adapter, &args, 1); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 1); + if (err) { + PMD_DRV_LOG(ERR, "fail to execute command %s", + "OP_INLINE_IPSEC_CRYPTO"); +@@ -2163,7 +2208,7 @@ iavf_set_vf_quanta_size(struct iavf_adapter *adapter, u16 start_queue_id, u16 nu + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (err) { + PMD_DRV_LOG(ERR, "Failed to execute command VIRTCHNL_OP_CONFIG_QUANTA"); + return err; +@@ -2189,7 +2234,7 @@ iavf_get_ptp_cap(struct iavf_adapter *adapter) + args.out_buffer = vf->aq_resp; + args.out_size = IAVF_AQ_BUF_SZ; + +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (err) { + PMD_DRV_LOG(ERR, + "Failed to execute command of OP_1588_PTP_GET_CAPS"); +@@ -2217,7 +2262,7 @@ iavf_get_phc_time(struct iavf_rx_queue *rxq) + args.out_size = IAVF_AQ_BUF_SZ; + + rte_spinlock_lock(&vf->phc_time_aq_lock); +- err = iavf_execute_vf_cmd(adapter, &args, 0); ++ err = iavf_execute_vf_cmd_safe(adapter, &args, 0); + if (err) { + PMD_DRV_LOG(ERR, + "Failed to execute command of VIRTCHNL_OP_1588_PTP_GET_TIME"); +diff --git a/dpdk/drivers/net/ice/base/ice_adminq_cmd.h b/dpdk/drivers/net/ice/base/ice_adminq_cmd.h +index 5a817982b4..93a3a6007f 100644 +--- a/dpdk/drivers/net/ice/base/ice_adminq_cmd.h ++++ b/dpdk/drivers/net/ice/base/ice_adminq_cmd.h +@@ -2002,8 +2002,8 @@ struct ice_aqc_lldp_get_mib { + #define ICE_AQ_LLDP_DCBX_S 6 + #define ICE_AQ_LLDP_DCBX_M (0x3 << ICE_AQ_LLDP_DCBX_S) + #define ICE_AQ_LLDP_DCBX_NA 0 +-#define ICE_AQ_LLDP_DCBX_IEEE 1 +-#define ICE_AQ_LLDP_DCBX_CEE 2 ++#define ICE_AQ_LLDP_DCBX_CEE 1 ++#define ICE_AQ_LLDP_DCBX_IEEE 2 + /* The following bytes are reserved for the Get LLDP MIB command (0x0A00) + * and in the LLDP MIB Change Event (0x0A01). They are valid for the + * Get LLDP MIB (0x0A00) response only. +diff --git a/dpdk/drivers/net/ice/base/ice_common.c b/dpdk/drivers/net/ice/base/ice_common.c +index 5391bd666b..a327a4b374 100644 +--- a/dpdk/drivers/net/ice/base/ice_common.c ++++ b/dpdk/drivers/net/ice/base/ice_common.c +@@ -167,6 +167,8 @@ static enum ice_status ice_set_mac_type(struct ice_hw *hw) + case ICE_DEV_ID_E823C_QSFP: + case ICE_DEV_ID_E823C_SFP: + case ICE_DEV_ID_E823C_SGMII: ++ hw->mac_type = ICE_MAC_GENERIC; ++ break; + case ICE_DEV_ID_E824S: + case ICE_DEV_ID_E825C_BACKPLANE: + case ICE_DEV_ID_E825C_QSFP: +@@ -2554,9 +2556,7 @@ ice_parse_1588_func_caps(struct ice_hw *hw, struct ice_hw_func_caps *func_p, + struct ice_aqc_list_caps_elem *cap) + { + struct ice_ts_func_info *info = &func_p->ts_func_info; +- u32 number = ICE_TS_FUNC_ENA_M | ICE_TS_SRC_TMR_OWND_M | +- ICE_TS_TMR_ENA_M | ICE_TS_TMR_IDX_OWND_M | +- ICE_TS_TMR_IDX_ASSOC_M; ++ u32 number = LE32_TO_CPU(cap->number); + u8 clk_freq; + + ice_debug(hw, ICE_DBG_INIT, "1588 func caps: raw value %x\n", number); +diff --git a/dpdk/drivers/net/ice/base/ice_sched.c b/dpdk/drivers/net/ice/base/ice_sched.c +index a526c8f32c..b16b27dcbf 100644 +--- a/dpdk/drivers/net/ice/base/ice_sched.c ++++ b/dpdk/drivers/net/ice/base/ice_sched.c +@@ -1417,11 +1417,6 @@ void ice_sched_get_psm_clk_freq(struct ice_hw *hw) + clk_src = (val & GLGEN_CLKSTAT_SRC_PSM_CLK_SRC_M) >> + GLGEN_CLKSTAT_SRC_PSM_CLK_SRC_S; + +-#define PSM_CLK_SRC_367_MHZ 0x0 +-#define PSM_CLK_SRC_416_MHZ 0x1 +-#define PSM_CLK_SRC_446_MHZ 0x2 +-#define PSM_CLK_SRC_390_MHZ 0x3 +- + switch (clk_src) { + case PSM_CLK_SRC_367_MHZ: + hw->psm_clk_freq = ICE_PSM_CLK_367MHZ_IN_HZ; +@@ -1435,11 +1430,12 @@ void ice_sched_get_psm_clk_freq(struct ice_hw *hw) + case PSM_CLK_SRC_390_MHZ: + hw->psm_clk_freq = ICE_PSM_CLK_390MHZ_IN_HZ; + break; +- default: +- ice_debug(hw, ICE_DBG_SCHED, "PSM clk_src unexpected %u\n", +- clk_src); +- /* fall back to a safe default */ +- hw->psm_clk_freq = ICE_PSM_CLK_446MHZ_IN_HZ; ++ ++ /* default condition is not required as clk_src is restricted ++ * to a 2-bit value from GLGEN_CLKSTAT_SRC_PSM_CLK_SRC_M mask. ++ * The above switch statements cover the possible values of ++ * this variable. ++ */ + } + } + +diff --git a/dpdk/drivers/net/ice/base/ice_sched.h b/dpdk/drivers/net/ice/base/ice_sched.h +index 3724ef33a8..64ed5e0f9b 100644 +--- a/dpdk/drivers/net/ice/base/ice_sched.h ++++ b/dpdk/drivers/net/ice/base/ice_sched.h +@@ -38,6 +38,11 @@ + #define ICE_PSM_CLK_446MHZ_IN_HZ 446428571 + #define ICE_PSM_CLK_390MHZ_IN_HZ 390625000 + ++#define PSM_CLK_SRC_367_MHZ 0x0 ++#define PSM_CLK_SRC_416_MHZ 0x1 ++#define PSM_CLK_SRC_446_MHZ 0x2 ++#define PSM_CLK_SRC_390_MHZ 0x3 ++ + struct rl_profile_params { + u32 bw; /* in Kbps */ + u16 rl_multiplier; +diff --git a/dpdk/drivers/net/ice/ice_dcf.c b/dpdk/drivers/net/ice/ice_dcf.c +index 1c3d22ae0f..6f7e103c3b 100644 +--- a/dpdk/drivers/net/ice/ice_dcf.c ++++ b/dpdk/drivers/net/ice/ice_dcf.c +@@ -32,6 +32,8 @@ + #define ICE_DCF_ARQ_MAX_RETRIES 200 + #define ICE_DCF_ARQ_CHECK_TIME 2 /* msecs */ + ++#define ICE_DCF_CHECK_INTERVAL 100 /* 100ms */ ++ + #define ICE_DCF_VF_RES_BUF_SZ \ + (sizeof(struct virtchnl_vf_resource) + \ + IAVF_MAX_VF_VSI * sizeof(struct virtchnl_vsi_resource)) +@@ -639,6 +641,8 @@ ice_dcf_init_hw(struct rte_eth_dev *eth_dev, struct ice_dcf_hw *hw) + rte_spinlock_init(&hw->vc_cmd_queue_lock); + TAILQ_INIT(&hw->vc_cmd_queue); + ++ __atomic_store_n(&hw->vsi_update_thread_num, 0, __ATOMIC_RELAXED); ++ + hw->arq_buf = rte_zmalloc("arq_buf", ICE_DCF_AQ_BUF_SZ, 0); + if (hw->arq_buf == NULL) { + PMD_INIT_LOG(ERR, "unable to allocate AdminQ buffer memory"); +@@ -760,6 +764,11 @@ ice_dcf_uninit_hw(struct rte_eth_dev *eth_dev, struct ice_dcf_hw *hw) + rte_intr_callback_unregister(intr_handle, + ice_dcf_dev_interrupt_handler, hw); + ++ /* Wait for all `ice-thread` threads to exit. */ ++ while (__atomic_load_n(&hw->vsi_update_thread_num, ++ __ATOMIC_ACQUIRE) != 0) ++ rte_delay_ms(ICE_DCF_CHECK_INTERVAL); ++ + ice_dcf_mode_disable(hw); + iavf_shutdown_adminq(&hw->avf); + +@@ -854,7 +863,8 @@ ice_dcf_init_rss(struct ice_dcf_hw *hw) + { + struct rte_eth_dev *dev = hw->eth_dev; + struct rte_eth_rss_conf *rss_conf; +- uint8_t i, j, nb_q; ++ uint8_t j, nb_q; ++ size_t i; + int ret; + + rss_conf = &dev->data->dev_conf.rx_adv_conf.rss_conf; +diff --git a/dpdk/drivers/net/ice/ice_dcf.h b/dpdk/drivers/net/ice/ice_dcf.h +index 7f42ebabe9..7becf6d187 100644 +--- a/dpdk/drivers/net/ice/ice_dcf.h ++++ b/dpdk/drivers/net/ice/ice_dcf.h +@@ -105,6 +105,8 @@ struct ice_dcf_hw { + void (*vc_event_msg_cb)(struct ice_dcf_hw *dcf_hw, + uint8_t *msg, uint16_t msglen); + ++ int vsi_update_thread_num; ++ + uint8_t *arq_buf; + + uint16_t num_vfs; +diff --git a/dpdk/drivers/net/ice/ice_dcf_ethdev.c b/dpdk/drivers/net/ice/ice_dcf_ethdev.c +index dcbf2af5b0..223fa39f33 100644 +--- a/dpdk/drivers/net/ice/ice_dcf_ethdev.c ++++ b/dpdk/drivers/net/ice/ice_dcf_ethdev.c +@@ -115,7 +115,8 @@ ice_dcf_init_rxq(struct rte_eth_dev *dev, struct ice_rx_queue *rxq) + + buf_size = rte_pktmbuf_data_room_size(rxq->mp) - RTE_PKTMBUF_HEADROOM; + rxq->rx_hdr_len = 0; +- rxq->rx_buf_len = RTE_ALIGN(buf_size, (1 << ICE_RLAN_CTX_DBUF_S)); ++ rxq->rx_buf_len = RTE_ALIGN_FLOOR(buf_size, (1 << ICE_RLAN_CTX_DBUF_S)); ++ rxq->rx_buf_len = RTE_MIN(rxq->rx_buf_len, ICE_RX_MAX_DATA_BUF_SIZE); + max_pkt_len = RTE_MIN(ICE_SUPPORT_CHAIN_NUM * rxq->rx_buf_len, + dev->data->mtu + ICE_ETH_OVERHEAD); + +@@ -670,7 +671,6 @@ ice_dcf_dev_stop(struct rte_eth_dev *dev) + struct ice_dcf_adapter *dcf_ad = dev->data->dev_private; + struct rte_intr_handle *intr_handle = dev->intr_handle; + struct ice_adapter *ad = &dcf_ad->parent; +- struct ice_dcf_hw *hw = &dcf_ad->real_hw; + + if (ad->pf.adapter_stopped == 1) { + PMD_DRV_LOG(DEBUG, "Port is already stopped"); +@@ -697,7 +697,6 @@ ice_dcf_dev_stop(struct rte_eth_dev *dev) + + dev->data->dev_link.link_status = RTE_ETH_LINK_DOWN; + ad->pf.adapter_stopped = 1; +- hw->tm_conf.committed = false; + + return 0; + } +@@ -1598,6 +1597,26 @@ ice_dcf_free_repr_info(struct ice_dcf_adapter *dcf_adapter) + } + } + ++int ++ice_dcf_handle_vf_repr_close(struct ice_dcf_adapter *dcf_adapter, ++ uint16_t vf_id) ++{ ++ struct ice_dcf_repr_info *vf_rep_info; ++ ++ if (dcf_adapter->num_reprs >= vf_id) { ++ PMD_DRV_LOG(ERR, "Invalid VF id: %d", vf_id); ++ return -1; ++ } ++ ++ if (!dcf_adapter->repr_infos) ++ return 0; ++ ++ vf_rep_info = &dcf_adapter->repr_infos[vf_id]; ++ vf_rep_info->vf_rep_eth_dev = NULL; ++ ++ return 0; ++} ++ + static int + ice_dcf_init_repr_info(struct ice_dcf_adapter *dcf_adapter) + { +@@ -1621,11 +1640,10 @@ ice_dcf_dev_close(struct rte_eth_dev *dev) + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return 0; + ++ ice_dcf_vf_repr_notify_all(adapter, false); + (void)ice_dcf_dev_stop(dev); + + ice_free_queues(dev); +- +- ice_dcf_free_repr_info(adapter); + ice_dcf_uninit_parent_adapter(dev); + ice_dcf_uninit_hw(dev, &adapter->real_hw); + +@@ -1815,7 +1833,7 @@ ice_dcf_dev_reset(struct rte_eth_dev *dev) + ice_dcf_reset_hw(dev, hw); + } + +- ret = ice_dcf_dev_uninit(dev); ++ ret = ice_dcf_dev_close(dev); + if (ret) + return ret; + +@@ -1917,13 +1935,20 @@ ice_dcf_dev_init(struct rte_eth_dev *eth_dev) + return -1; + } + ++ ice_dcf_stats_reset(eth_dev); ++ + dcf_config_promisc(adapter, false, false); ++ ice_dcf_vf_repr_notify_all(adapter, true); ++ + return 0; + } + + static int + ice_dcf_dev_uninit(struct rte_eth_dev *eth_dev) + { ++ struct ice_dcf_adapter *adapter = eth_dev->data->dev_private; ++ ++ ice_dcf_free_repr_info(adapter); + ice_dcf_dev_close(eth_dev); + + return 0; +diff --git a/dpdk/drivers/net/ice/ice_dcf_ethdev.h b/dpdk/drivers/net/ice/ice_dcf_ethdev.h +index 4baaec4b8b..6dcbaac5eb 100644 +--- a/dpdk/drivers/net/ice/ice_dcf_ethdev.h ++++ b/dpdk/drivers/net/ice/ice_dcf_ethdev.h +@@ -60,6 +60,7 @@ struct ice_dcf_vf_repr { + struct rte_ether_addr mac_addr; + uint16_t switch_domain_id; + uint16_t vf_id; ++ bool dcf_valid; + + struct ice_dcf_vlan outer_vlan_info; /* DCF always handle outer VLAN */ + }; +@@ -80,6 +81,8 @@ int ice_dcf_vf_repr_init(struct rte_eth_dev *vf_rep_eth_dev, void *init_param); + int ice_dcf_vf_repr_uninit(struct rte_eth_dev *vf_rep_eth_dev); + int ice_dcf_vf_repr_init_vlan(struct rte_eth_dev *vf_rep_eth_dev); + void ice_dcf_vf_repr_stop_all(struct ice_dcf_adapter *dcf_adapter); ++void ice_dcf_vf_repr_notify_all(struct ice_dcf_adapter *dcf_adapter, bool valid); ++int ice_dcf_handle_vf_repr_close(struct ice_dcf_adapter *dcf_adapter, uint16_t vf_id); + bool ice_dcf_adminq_need_retry(struct ice_adapter *ad); + + #endif /* _ICE_DCF_ETHDEV_H_ */ +diff --git a/dpdk/drivers/net/ice/ice_dcf_parent.c b/dpdk/drivers/net/ice/ice_dcf_parent.c +index 01e390ddda..0563edb0b2 100644 +--- a/dpdk/drivers/net/ice/ice_dcf_parent.c ++++ b/dpdk/drivers/net/ice/ice_dcf_parent.c +@@ -124,6 +124,9 @@ ice_dcf_vsi_update_service_handler(void *param) + container_of(hw, struct ice_dcf_adapter, real_hw); + struct ice_adapter *parent_adapter = &adapter->parent; + ++ __atomic_fetch_add(&hw->vsi_update_thread_num, 1, ++ __ATOMIC_RELAXED); ++ + pthread_detach(pthread_self()); + + rte_delay_us(ICE_DCF_VSI_UPDATE_SERVICE_INTERVAL); +@@ -154,6 +157,9 @@ ice_dcf_vsi_update_service_handler(void *param) + + free(param); + ++ __atomic_fetch_sub(&hw->vsi_update_thread_num, 1, ++ __ATOMIC_RELEASE); ++ + return NULL; + } + +diff --git a/dpdk/drivers/net/ice/ice_dcf_sched.c b/dpdk/drivers/net/ice/ice_dcf_sched.c +index a231c1e60b..b08bc5f1de 100644 +--- a/dpdk/drivers/net/ice/ice_dcf_sched.c ++++ b/dpdk/drivers/net/ice/ice_dcf_sched.c +@@ -237,6 +237,7 @@ ice_dcf_node_add(struct rte_eth_dev *dev, uint32_t node_id, + enum ice_dcf_tm_node_type node_type = ICE_DCF_TM_NODE_TYPE_MAX; + struct ice_dcf_tm_shaper_profile *shaper_profile = NULL; + struct ice_dcf_adapter *adapter = dev->data->dev_private; ++ struct ice_adapter *ad = &adapter->parent; + struct ice_dcf_hw *hw = &adapter->real_hw; + struct ice_dcf_tm_node *parent_node; + struct ice_dcf_tm_node *tm_node; +@@ -246,10 +247,10 @@ ice_dcf_node_add(struct rte_eth_dev *dev, uint32_t node_id, + if (!params || !error) + return -EINVAL; + +- /* if already committed */ +- if (hw->tm_conf.committed) { ++ /* if port is running */ ++ if (!ad->pf.adapter_stopped) { + error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED; +- error->message = "already committed"; ++ error->message = "port is running"; + return -EINVAL; + } + +@@ -400,16 +401,17 @@ ice_dcf_node_delete(struct rte_eth_dev *dev, uint32_t node_id, + { + enum ice_dcf_tm_node_type node_type = ICE_DCF_TM_NODE_TYPE_MAX; + struct ice_dcf_adapter *adapter = dev->data->dev_private; ++ struct ice_adapter *ad = &adapter->parent; + struct ice_dcf_hw *hw = &adapter->real_hw; + struct ice_dcf_tm_node *tm_node; + + if (!error) + return -EINVAL; + +- /* if already committed */ +- if (hw->tm_conf.committed) { ++ /* if port is running */ ++ if (!ad->pf.adapter_stopped) { + error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED; +- error->message = "already committed"; ++ error->message = "port is running"; + return -EINVAL; + } + +diff --git a/dpdk/drivers/net/ice/ice_dcf_vf_representor.c b/dpdk/drivers/net/ice/ice_dcf_vf_representor.c +index b9fcfc80ad..af281f069a 100644 +--- a/dpdk/drivers/net/ice/ice_dcf_vf_representor.c ++++ b/dpdk/drivers/net/ice/ice_dcf_vf_representor.c +@@ -50,9 +50,28 @@ ice_dcf_vf_repr_dev_stop(struct rte_eth_dev *dev) + return 0; + } + ++static void ++ice_dcf_vf_repr_notify_one(struct rte_eth_dev *dev, bool valid) ++{ ++ struct ice_dcf_vf_repr *repr = dev->data->dev_private; ++ ++ repr->dcf_valid = valid; ++} ++ + static int + ice_dcf_vf_repr_dev_close(struct rte_eth_dev *dev) + { ++ struct ice_dcf_vf_repr *repr = dev->data->dev_private; ++ struct ice_dcf_adapter *dcf_adapter; ++ int err; ++ ++ if (repr->dcf_valid) { ++ dcf_adapter = repr->dcf_eth_dev->data->dev_private; ++ err = ice_dcf_handle_vf_repr_close(dcf_adapter, repr->vf_id); ++ if (err) ++ PMD_DRV_LOG(ERR, "VF representor invalid"); ++ } ++ + return ice_dcf_vf_repr_uninit(dev); + } + +@@ -111,14 +130,15 @@ ice_dcf_vf_repr_link_update(__rte_unused struct rte_eth_dev *ethdev, + static __rte_always_inline struct ice_dcf_hw * + ice_dcf_vf_repr_hw(struct ice_dcf_vf_repr *repr) + { +- struct ice_dcf_adapter *dcf_adapter = +- repr->dcf_eth_dev->data->dev_private; ++ struct ice_dcf_adapter *dcf_adapter; + +- if (!dcf_adapter) { ++ if (!repr->dcf_valid) { + PMD_DRV_LOG(ERR, "DCF for VF representor has been released\n"); + return NULL; + } + ++ dcf_adapter = repr->dcf_eth_dev->data->dev_private; ++ + return &dcf_adapter->real_hw; + } + +@@ -414,6 +434,7 @@ ice_dcf_vf_repr_init(struct rte_eth_dev *vf_rep_eth_dev, void *init_param) + repr->dcf_eth_dev = param->dcf_eth_dev; + repr->switch_domain_id = param->switch_domain_id; + repr->vf_id = param->vf_id; ++ repr->dcf_valid = true; + repr->outer_vlan_info.port_vlan_ena = false; + repr->outer_vlan_info.stripping_ena = false; + repr->outer_vlan_info.tpid = RTE_ETHER_TYPE_VLAN; +@@ -488,3 +509,22 @@ ice_dcf_vf_repr_stop_all(struct ice_dcf_adapter *dcf_adapter) + vf_rep_eth_dev->data->dev_started = 0; + } + } ++ ++void ++ice_dcf_vf_repr_notify_all(struct ice_dcf_adapter *dcf_adapter, bool valid) ++{ ++ uint16_t vf_id; ++ struct rte_eth_dev *vf_rep_eth_dev; ++ ++ if (!dcf_adapter->repr_infos) ++ return; ++ ++ for (vf_id = 0; vf_id < dcf_adapter->real_hw.num_vfs; vf_id++) { ++ vf_rep_eth_dev = dcf_adapter->repr_infos[vf_id].vf_rep_eth_dev; ++ ++ if (!vf_rep_eth_dev) ++ continue; ++ ++ ice_dcf_vf_repr_notify_one(vf_rep_eth_dev, valid); ++ } ++} +diff --git a/dpdk/drivers/net/ice/ice_ethdev.c b/dpdk/drivers/net/ice/ice_ethdev.c +index 0bc739daf0..8ce7d0ebaa 100644 +--- a/dpdk/drivers/net/ice/ice_ethdev.c ++++ b/dpdk/drivers/net/ice/ice_ethdev.c +@@ -2399,6 +2399,17 @@ ice_dev_init(struct rte_eth_dev *dev) + /* Initialize TM configuration */ + ice_tm_conf_init(dev); + ++ if (ice_is_e810(hw)) ++ hw->phy_cfg = ICE_PHY_E810; ++ else ++ hw->phy_cfg = ICE_PHY_E822; ++ ++ if (hw->phy_cfg == ICE_PHY_E822) { ++ ret = ice_start_phy_timer_e822(hw, hw->pf_id, true); ++ if (ret) ++ PMD_INIT_LOG(ERR, "Failed to start phy timer\n"); ++ } ++ + if (!ad->is_safe_mode) { + ret = ice_flow_init(ad); + if (ret) { +@@ -2415,6 +2426,9 @@ ice_dev_init(struct rte_eth_dev *dev) + + pf->supported_rxdid = ice_get_supported_rxdid(hw); + ++ /* reset all stats of the device, including pf and main vsi */ ++ ice_stats_reset(dev); ++ + return 0; + + err_flow_init: +@@ -3298,7 +3312,8 @@ static int ice_init_rss(struct ice_pf *pf) + + rss_conf = &dev_data->dev_conf.rx_adv_conf.rss_conf; + nb_q = dev_data->nb_rx_queues; +- vsi->rss_key_size = ICE_AQC_GET_SET_RSS_KEY_DATA_RSS_KEY_SIZE; ++ vsi->rss_key_size = ICE_AQC_GET_SET_RSS_KEY_DATA_RSS_KEY_SIZE + ++ ICE_AQC_GET_SET_RSS_KEY_DATA_HASH_KEY_SIZE; + vsi->rss_lut_size = pf->hash_lut_size; + + if (nb_q == 0) { +@@ -3339,7 +3354,10 @@ static int ice_init_rss(struct ice_pf *pf) + vsi->rss_key_size)); + + rte_memcpy(key.standard_rss_key, vsi->rss_key, +- RTE_MIN(sizeof(key.standard_rss_key), vsi->rss_key_size)); ++ ICE_AQC_GET_SET_RSS_KEY_DATA_RSS_KEY_SIZE); ++ rte_memcpy(key.extended_hash_key, ++ &vsi->rss_key[ICE_AQC_GET_SET_RSS_KEY_DATA_RSS_KEY_SIZE], ++ ICE_AQC_GET_SET_RSS_KEY_DATA_HASH_KEY_SIZE); + ret = ice_aq_set_rss_key(hw, vsi->idx, &key); + if (ret) + goto out; +@@ -3590,6 +3608,8 @@ ice_get_init_link_status(struct rte_eth_dev *dev) + + if (link_status.link_info & ICE_AQ_LINK_UP) + pf->init_link_up = true; ++ else ++ pf->init_link_up = false; + } + + static int +@@ -3660,6 +3680,16 @@ ice_dev_start(struct rte_eth_dev *dev) + } + } + ++ if (dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP) { ++ /* Register mbuf field and flag for Rx timestamp */ ++ ret = rte_mbuf_dyn_rx_timestamp_register(&ice_timestamp_dynfield_offset, ++ &ice_timestamp_dynflag); ++ if (ret) { ++ PMD_DRV_LOG(ERR, "Cannot register mbuf field/flag for timestamp"); ++ goto tx_err; ++ } ++ } ++ + /* program Rx queues' context in hardware*/ + for (nb_rxq = 0; nb_rxq < data->nb_rx_queues; nb_rxq++) { + ret = ice_rx_queue_start(dev, nb_rxq); +@@ -3856,6 +3886,8 @@ ice_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) + .nb_max = ICE_MAX_RING_DESC, + .nb_min = ICE_MIN_RING_DESC, + .nb_align = ICE_ALIGN_RING_DESC, ++ .nb_mtu_seg_max = ICE_TX_MTU_SEG_MAX, ++ .nb_seg_max = ICE_MAX_RING_DESC, + }; + + dev_info->speed_capa = RTE_ETH_LINK_SPEED_10M | +@@ -3926,8 +3958,8 @@ ice_atomic_write_link_status(struct rte_eth_dev *dev, + static int + ice_link_update(struct rte_eth_dev *dev, int wait_to_complete) + { +-#define CHECK_INTERVAL 100 /* 100ms */ +-#define MAX_REPEAT_TIME 10 /* 1s (10 * 100ms) in total */ ++#define CHECK_INTERVAL 50 /* 50ms */ ++#define MAX_REPEAT_TIME 40 /* 2s (40 * 50ms) in total */ + struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct ice_link_status link_status; + struct rte_eth_link link, old; +@@ -5800,11 +5832,6 @@ ice_timesync_enable(struct rte_eth_dev *dev) + return -1; + } + +- if (ice_is_e810(hw)) +- hw->phy_cfg = ICE_PHY_E810; +- else +- hw->phy_cfg = ICE_PHY_E822; +- + if (hw->func_caps.ts_func_info.src_tmr_owned) { + ret = ice_ptp_init_phc(hw); + if (ret) { +@@ -5925,16 +5952,17 @@ ice_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts) + struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct ice_adapter *ad = + ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); ++ uint8_t tmr_idx = hw->func_caps.ts_func_info.tmr_index_assoc; + uint32_t hi, lo, lo2; + uint64_t time, ns; + +- lo = ICE_READ_REG(hw, GLTSYN_TIME_L(0)); +- hi = ICE_READ_REG(hw, GLTSYN_TIME_H(0)); +- lo2 = ICE_READ_REG(hw, GLTSYN_TIME_L(0)); ++ lo = ICE_READ_REG(hw, GLTSYN_TIME_L(tmr_idx)); ++ hi = ICE_READ_REG(hw, GLTSYN_TIME_H(tmr_idx)); ++ lo2 = ICE_READ_REG(hw, GLTSYN_TIME_L(tmr_idx)); + + if (lo2 < lo) { +- lo = ICE_READ_REG(hw, GLTSYN_TIME_L(0)); +- hi = ICE_READ_REG(hw, GLTSYN_TIME_H(0)); ++ lo = ICE_READ_REG(hw, GLTSYN_TIME_L(tmr_idx)); ++ hi = ICE_READ_REG(hw, GLTSYN_TIME_H(tmr_idx)); + } + + time = ((uint64_t)hi << 32) | lo; +@@ -5950,6 +5978,7 @@ ice_timesync_disable(struct rte_eth_dev *dev) + struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct ice_adapter *ad = + ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); ++ uint8_t tmr_idx = hw->func_caps.ts_func_info.tmr_index_assoc; + uint64_t val; + uint8_t lport; + +@@ -5957,12 +5986,12 @@ ice_timesync_disable(struct rte_eth_dev *dev) + + ice_clear_phy_tstamp(hw, lport, 0); + +- val = ICE_READ_REG(hw, GLTSYN_ENA(0)); ++ val = ICE_READ_REG(hw, GLTSYN_ENA(tmr_idx)); + val &= ~GLTSYN_ENA_TSYN_ENA_M; +- ICE_WRITE_REG(hw, GLTSYN_ENA(0), val); ++ ICE_WRITE_REG(hw, GLTSYN_ENA(tmr_idx), val); + +- ICE_WRITE_REG(hw, GLTSYN_INCVAL_L(0), 0); +- ICE_WRITE_REG(hw, GLTSYN_INCVAL_H(0), 0); ++ ICE_WRITE_REG(hw, GLTSYN_INCVAL_L(tmr_idx), 0); ++ ICE_WRITE_REG(hw, GLTSYN_INCVAL_H(tmr_idx), 0); + + ad->ptp_ena = 0; + +diff --git a/dpdk/drivers/net/ice/ice_fdir_filter.c b/dpdk/drivers/net/ice/ice_fdir_filter.c +index 7914ba9407..81e88c1dd8 100644 +--- a/dpdk/drivers/net/ice/ice_fdir_filter.c ++++ b/dpdk/drivers/net/ice/ice_fdir_filter.c +@@ -1876,7 +1876,7 @@ ice_fdir_parse_pattern(__rte_unused struct ice_adapter *ad, + (uint8_t *)(uintptr_t)raw_mask->pattern; + uint8_t *tmp_spec, *tmp_mask; + uint16_t tmp_val = 0; +- uint8_t pkt_len = 0; ++ uint16_t pkt_len = 0; + uint8_t tmp = 0; + int i, j; + +diff --git a/dpdk/drivers/net/ice/ice_generic_flow.c b/dpdk/drivers/net/ice/ice_generic_flow.c +index d496c28dec..91bf1d6fcb 100644 +--- a/dpdk/drivers/net/ice/ice_generic_flow.c ++++ b/dpdk/drivers/net/ice/ice_generic_flow.c +@@ -1836,6 +1836,13 @@ ice_flow_init(struct ice_adapter *ad) + if (ice_parser_create(&ad->hw, &ad->psr) != ICE_SUCCESS) + PMD_INIT_LOG(WARNING, "Failed to initialize DDP parser, raw packet filter will not be supported"); + ++ if (ad->psr) { ++ if (ice_is_dvm_ena(&ad->hw)) ++ ice_parser_dvm_set(ad->psr, true); ++ else ++ ice_parser_dvm_set(ad->psr, false); ++ } ++ + RTE_TAILQ_FOREACH_SAFE(engine, &engine_list, node, temp) { + if (engine->init == NULL) { + PMD_INIT_LOG(ERR, "Invalid engine type (%d)", +@@ -2030,6 +2037,14 @@ ice_flow_valid_attr(struct ice_adapter *ad, + return -rte_errno; + } + ++ /* Not supported */ ++ if (attr->transfer) { ++ rte_flow_error_set(error, EINVAL, ++ RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, ++ attr, "Not support transfer."); ++ return -rte_errno; ++ } ++ + /* Check pipeline mode support to set classification stage */ + if (ad->devargs.pipe_mode_support) { + if (attr->priority == 0) +diff --git a/dpdk/drivers/net/ice/ice_hash.c b/dpdk/drivers/net/ice/ice_hash.c +index f35727856e..52646e9408 100644 +--- a/dpdk/drivers/net/ice/ice_hash.c ++++ b/dpdk/drivers/net/ice/ice_hash.c +@@ -653,8 +653,8 @@ ice_hash_parse_raw_pattern(struct ice_adapter *ad, + const struct rte_flow_item_raw *raw_spec, *raw_mask; + struct ice_parser_profile prof; + struct ice_parser_result rslt; ++ uint16_t spec_len, pkt_len; + uint8_t *pkt_buf, *msk_buf; +- uint8_t spec_len, pkt_len; + uint8_t tmp_val = 0; + uint8_t tmp_c = 0; + int i, j; +diff --git a/dpdk/drivers/net/ice/ice_rxtx.c b/dpdk/drivers/net/ice/ice_rxtx.c +index 0ea0045836..9a653cbc4a 100644 +--- a/dpdk/drivers/net/ice/ice_rxtx.c ++++ b/dpdk/drivers/net/ice/ice_rxtx.c +@@ -259,7 +259,8 @@ ice_program_hw_rx_queue(struct ice_rx_queue *rxq) + /* Set buffer size as the head split is disabled. */ + buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) - + RTE_PKTMBUF_HEADROOM); +- rxq->rx_buf_len = RTE_ALIGN(buf_size, (1 << ICE_RLAN_CTX_DBUF_S)); ++ rxq->rx_buf_len = RTE_ALIGN_FLOOR(buf_size, (1 << ICE_RLAN_CTX_DBUF_S)); ++ rxq->rx_buf_len = RTE_MIN(rxq->rx_buf_len, ICE_RX_MAX_DATA_BUF_SIZE); + rxq->max_pkt_len = + RTE_MIN((uint32_t)ICE_SUPPORT_CHAIN_NUM * rxq->rx_buf_len, + frame_size); +@@ -273,7 +274,7 @@ ice_program_hw_rx_queue(struct ice_rx_queue *rxq) + return -EINVAL; + } + +- if (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP) { ++ if (!rxq->ts_enable && (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP)) { + /* Register mbuf field and flag for Rx timestamp */ + err = rte_mbuf_dyn_rx_timestamp_register( + &ice_timestamp_dynfield_offset, +@@ -283,6 +284,7 @@ ice_program_hw_rx_queue(struct ice_rx_queue *rxq) + "Cannot register mbuf field/flag for timestamp"); + return -EINVAL; + } ++ rxq->ts_enable = true; + } + + memset(&rx_ctx, 0, sizeof(rx_ctx)); +@@ -670,6 +672,8 @@ ice_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id) + return -EINVAL; + } + ++ if (dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP) ++ rxq->ts_enable = true; + err = ice_program_hw_rx_queue(rxq); + if (err) { + PMD_DRV_LOG(ERR, "fail to program RX queue %u", +@@ -1761,7 +1765,8 @@ ice_rx_scan_hw_ring(struct ice_rx_queue *rxq) + ice_rxd_to_vlan_tci(mb, &rxdp[j]); + rxd_to_pkt_fields_ops[rxq->rxdid](rxq, mb, &rxdp[j]); + #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC +- if (ice_timestamp_dynflag > 0) { ++ if (ice_timestamp_dynflag > 0 && ++ (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP)) { + rxq->time_high = + rte_le_to_cpu_32(rxdp[j].wb.flex_ts.ts_high); + if (unlikely(is_tsinit)) { +@@ -2127,7 +2132,8 @@ ice_recv_scattered_pkts(void *rx_queue, + rxd_to_pkt_fields_ops[rxq->rxdid](rxq, first_seg, &rxd); + pkt_flags = ice_rxd_error_to_pkt_flags(rx_stat_err0); + #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC +- if (ice_timestamp_dynflag > 0) { ++ if (ice_timestamp_dynflag > 0 && ++ (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP)) { + rxq->time_high = + rte_le_to_cpu_32(rxd.wb.flex_ts.ts_high); + if (unlikely(is_tsinit)) { +@@ -2143,7 +2149,7 @@ ice_recv_scattered_pkts(void *rx_queue, + } + rxq->hw_time_update = rte_get_timer_cycles() / + (rte_get_timer_hz() / 1000); +- *RTE_MBUF_DYNFIELD(rxm, ++ *RTE_MBUF_DYNFIELD(first_seg, + (ice_timestamp_dynfield_offset), + rte_mbuf_timestamp_t *) = ts_ns; + pkt_flags |= ice_timestamp_dynflag; +@@ -2617,7 +2623,8 @@ ice_recv_pkts(void *rx_queue, + rxd_to_pkt_fields_ops[rxq->rxdid](rxq, rxm, &rxd); + pkt_flags = ice_rxd_error_to_pkt_flags(rx_stat_err0); + #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC +- if (ice_timestamp_dynflag > 0) { ++ if (ice_timestamp_dynflag > 0 && ++ (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP)) { + rxq->time_high = + rte_le_to_cpu_32(rxd.wb.flex_ts.ts_high); + if (unlikely(is_tsinit)) { +@@ -2727,7 +2734,8 @@ ice_parse_tunneling_params(uint64_t ol_flags, + * Shall be set only if L4TUNT = 01b and EIPT is not zero + */ + if (!(*cd_tunneling & ICE_TX_CTX_EIPT_NONE) && +- (*cd_tunneling & ICE_TXD_CTX_UDP_TUNNELING)) ++ (*cd_tunneling & ICE_TXD_CTX_UDP_TUNNELING) && ++ (ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM)) + *cd_tunneling |= ICE_TXD_CTX_QW0_L4T_CS_M; + } + +@@ -2738,10 +2746,7 @@ ice_txd_enable_checksum(uint64_t ol_flags, + union ice_tx_offload tx_offload) + { + /* Set MACLEN */ +- if (ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) +- *td_offset |= (tx_offload.outer_l2_len >> 1) +- << ICE_TX_DESC_LEN_MACLEN_S; +- else ++ if (!(ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK)) + *td_offset |= (tx_offload.l2_len >> 1) + << ICE_TX_DESC_LEN_MACLEN_S; + +@@ -3002,9 +3007,12 @@ ice_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) + + /* Fill in tunneling parameters if necessary */ + cd_tunneling_params = 0; +- if (ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) ++ if (ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) { ++ td_offset |= (tx_offload.outer_l2_len >> 1) ++ << ICE_TX_DESC_LEN_MACLEN_S; + ice_parse_tunneling_params(ol_flags, tx_offload, + &cd_tunneling_params); ++ } + + /* Enable checksum offloading */ + if (ol_flags & ICE_TX_CKSUM_OFFLOAD_MASK) +@@ -3662,23 +3670,34 @@ ice_check_empty_mbuf(struct rte_mbuf *tx_pkt) + } + + uint16_t +-ice_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts, ++ice_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) + { + int i, ret; + uint64_t ol_flags; + struct rte_mbuf *m; +- struct ice_tx_queue *txq = tx_queue; +- struct rte_eth_dev *dev = &rte_eth_devices[txq->port_id]; +- uint16_t max_frame_size = dev->data->mtu + ICE_ETH_OVERHEAD; + + for (i = 0; i < nb_pkts; i++) { + m = tx_pkts[i]; + ol_flags = m->ol_flags; + +- if (ol_flags & RTE_MBUF_F_TX_TCP_SEG && ++ if (!(ol_flags & RTE_MBUF_F_TX_TCP_SEG) && ++ /** ++ * No TSO case: nb->segs, pkt_len to not exceed ++ * the limites. ++ */ ++ (m->nb_segs > ICE_TX_MTU_SEG_MAX || ++ m->pkt_len > ICE_FRAME_SIZE_MAX)) { ++ rte_errno = EINVAL; ++ return i; ++ } else if (ol_flags & RTE_MBUF_F_TX_TCP_SEG && ++ /** TSO case: tso_segsz, nb_segs, pkt_len not exceed ++ * the limits. ++ */ + (m->tso_segsz < ICE_MIN_TSO_MSS || + m->tso_segsz > ICE_MAX_TSO_MSS || ++ m->nb_segs > ++ ((struct ice_tx_queue *)tx_queue)->nb_tx_desc || + m->pkt_len > ICE_MAX_TSO_FRAME_SIZE)) { + /** + * MSS outside the range are considered malicious +@@ -3687,11 +3706,8 @@ ice_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts, + return i; + } + +- /* check the data_len in mbuf */ +- if (m->data_len < ICE_TX_MIN_PKT_LEN || +- m->data_len > max_frame_size) { ++ if (m->pkt_len < ICE_TX_MIN_PKT_LEN) { + rte_errno = EINVAL; +- PMD_DRV_LOG(ERR, "INVALID mbuf: bad data_len=[%hu]", m->data_len); + return i; + } + +@@ -3710,7 +3726,6 @@ ice_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts, + + if (ice_check_empty_mbuf(m) != 0) { + rte_errno = EINVAL; +- PMD_DRV_LOG(ERR, "INVALID mbuf: last mbuf data_len=[0]"); + return i; + } + } +diff --git a/dpdk/drivers/net/ice/ice_rxtx.h b/dpdk/drivers/net/ice/ice_rxtx.h +index 4947d5c25f..bd2c4abec9 100644 +--- a/dpdk/drivers/net/ice/ice_rxtx.h ++++ b/dpdk/drivers/net/ice/ice_rxtx.h +@@ -51,8 +51,13 @@ extern int ice_timestamp_dynfield_offset; + /* Max header size can be 2K - 64 bytes */ + #define ICE_RX_HDR_BUF_SIZE (2048 - 64) + ++/* Max data buffer size must be 16K - 128 bytes */ ++#define ICE_RX_MAX_DATA_BUF_SIZE (16 * 1024 - 128) ++ + #define ICE_HEADER_SPLIT_ENA BIT(0) + ++#define ICE_TX_MTU_SEG_MAX 8 ++ + typedef void (*ice_rx_release_mbufs_t)(struct ice_rx_queue *rxq); + typedef void (*ice_tx_release_mbufs_t)(struct ice_tx_queue *txq); + typedef void (*ice_rxd_to_pkt_fields_t)(struct ice_rx_queue *rxq, +@@ -117,6 +122,7 @@ struct ice_rx_queue { + uint64_t hw_time_update; /* SW time of HW record updating */ + struct rte_eth_rxseg_split rxseg[ICE_RX_MAX_NSEG]; + uint32_t rxseg_nb; ++ bool ts_enable; /* if rxq timestamp is enabled */ + }; + + struct ice_tx_entry { +@@ -349,26 +355,27 @@ static inline + uint64_t ice_tstamp_convert_32b_64b(struct ice_hw *hw, struct ice_adapter *ad, + uint32_t flag, uint32_t in_timestamp) + { ++ uint8_t tmr_idx = hw->func_caps.ts_func_info.tmr_index_assoc; + const uint64_t mask = 0xFFFFFFFF; + uint32_t hi, lo, lo2, delta; + uint64_t ns; + + if (flag) { +- lo = ICE_READ_REG(hw, GLTSYN_TIME_L(0)); +- hi = ICE_READ_REG(hw, GLTSYN_TIME_H(0)); ++ lo = ICE_READ_REG(hw, GLTSYN_TIME_L(tmr_idx)); ++ hi = ICE_READ_REG(hw, GLTSYN_TIME_H(tmr_idx)); + + /* + * On typical system, the delta between lo and lo2 is ~1000ns, + * so 10000 seems a large-enough but not overly-big guard band. + */ + if (lo > (UINT32_MAX - ICE_TIMESYNC_REG_WRAP_GUARD_BAND)) +- lo2 = ICE_READ_REG(hw, GLTSYN_TIME_L(0)); ++ lo2 = ICE_READ_REG(hw, GLTSYN_TIME_L(tmr_idx)); + else + lo2 = lo; + + if (lo2 < lo) { +- lo = ICE_READ_REG(hw, GLTSYN_TIME_L(0)); +- hi = ICE_READ_REG(hw, GLTSYN_TIME_H(0)); ++ lo = ICE_READ_REG(hw, GLTSYN_TIME_L(tmr_idx)); ++ hi = ICE_READ_REG(hw, GLTSYN_TIME_H(tmr_idx)); + } + + ad->time_hw = ((uint64_t)hi << 32) | lo; +diff --git a/dpdk/drivers/net/ice/ice_rxtx_vec_common.h b/dpdk/drivers/net/ice/ice_rxtx_vec_common.h +index eec6ea2134..55840cf170 100644 +--- a/dpdk/drivers/net/ice/ice_rxtx_vec_common.h ++++ b/dpdk/drivers/net/ice/ice_rxtx_vec_common.h +@@ -72,7 +72,7 @@ ice_rx_reassemble_packets(struct ice_rx_queue *rxq, struct rte_mbuf **rx_bufs, + /* save the partial packet for next time */ + rxq->pkt_first_seg = start; + rxq->pkt_last_seg = end; +- rte_memcpy(rx_bufs, pkts, pkt_idx * (sizeof(*pkts))); ++ memcpy(rx_bufs, pkts, pkt_idx * (sizeof(*pkts))); + return pkt_idx; + } + +diff --git a/dpdk/drivers/net/idpf/idpf_ethdev.c b/dpdk/drivers/net/idpf/idpf_ethdev.c +index 8b347631ce..b31cb47e90 100644 +--- a/dpdk/drivers/net/idpf/idpf_ethdev.c ++++ b/dpdk/drivers/net/idpf/idpf_ethdev.c +@@ -563,8 +563,6 @@ idpf_dev_start(struct rte_eth_dev *dev) + uint16_t req_vecs_num; + int ret; + +- vport->stopped = 0; +- + if (dev->data->mtu > vport->max_mtu) { + PMD_DRV_LOG(ERR, "MTU should be less than %d", vport->max_mtu); + ret = -EINVAL; +@@ -622,7 +620,7 @@ idpf_dev_stop(struct rte_eth_dev *dev) + { + struct idpf_vport *vport = dev->data->dev_private; + +- if (vport->stopped == 1) ++ if (dev->data->dev_started == 0) + return 0; + + idpf_vc_ena_dis_vport(vport, false); +@@ -634,8 +632,6 @@ idpf_dev_stop(struct rte_eth_dev *dev) + if (vport->recv_vectors != NULL) + idpf_vc_dealloc_vectors(vport); + +- vport->stopped = 1; +- + return 0; + } + +@@ -1313,7 +1309,11 @@ static struct rte_pci_driver rte_idpf_pmd = { + */ + RTE_PMD_REGISTER_PCI(net_idpf, rte_idpf_pmd); + RTE_PMD_REGISTER_PCI_TABLE(net_idpf, pci_id_idpf_map); +-RTE_PMD_REGISTER_KMOD_DEP(net_ice, "* igb_uio | uio_pci_generic | vfio-pci"); ++RTE_PMD_REGISTER_KMOD_DEP(net_idpf, "* igb_uio | vfio-pci"); ++RTE_PMD_REGISTER_PARAM_STRING(net_idpf, ++ IDPF_TX_SINGLE_Q "=<0|1> " ++ IDPF_RX_SINGLE_Q "=<0|1> " ++ IDPF_VPORT "=[vport_set0,[vport_set1],...]"); + + RTE_LOG_REGISTER_SUFFIX(idpf_logtype_init, init, NOTICE); + RTE_LOG_REGISTER_SUFFIX(idpf_logtype_driver, driver, NOTICE); +diff --git a/dpdk/drivers/net/idpf/idpf_ethdev.h b/dpdk/drivers/net/idpf/idpf_ethdev.h +index 991677c3bc..5633057a81 100644 +--- a/dpdk/drivers/net/idpf/idpf_ethdev.h ++++ b/dpdk/drivers/net/idpf/idpf_ethdev.h +@@ -128,7 +128,6 @@ struct idpf_vport { + + uint16_t devarg_id; + +- bool stopped; + }; + + struct idpf_adapter { +@@ -164,7 +163,6 @@ struct idpf_adapter { + uint32_t max_txq_per_msg; + + uint32_t ptype_tbl[IDPF_MAX_PKT_TYPE] __rte_cache_min_aligned; +- + bool rx_vec_allowed; + bool tx_vec_allowed; + bool rx_use_avx512; +diff --git a/dpdk/drivers/net/idpf/idpf_rxtx.c b/dpdk/drivers/net/idpf/idpf_rxtx.c +index b4a396c3f5..ceb34d4d32 100644 +--- a/dpdk/drivers/net/idpf/idpf_rxtx.c ++++ b/dpdk/drivers/net/idpf/idpf_rxtx.c +@@ -374,7 +374,8 @@ idpf_rx_split_bufq_setup(struct rte_eth_dev *dev, struct idpf_rx_queue *bufq, + bufq->adapter = adapter; + + len = rte_pktmbuf_data_room_size(bufq->mp) - RTE_PKTMBUF_HEADROOM; +- bufq->rx_buf_len = len; ++ bufq->rx_buf_len = RTE_ALIGN_FLOOR(len, (1 << IDPF_RLAN_CTX_DBUF_S)); ++ bufq->rx_buf_len = RTE_MIN(bufq->rx_buf_len, IDPF_RX_MAX_DATA_BUF_SIZE); + + /* Allocate the software ring. */ + len = nb_desc + IDPF_RX_MAX_BURST; +@@ -473,7 +474,8 @@ idpf_rx_split_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, + rxq->offloads = offloads; + + len = rte_pktmbuf_data_room_size(rxq->mp) - RTE_PKTMBUF_HEADROOM; +- rxq->rx_buf_len = len; ++ rxq->rx_buf_len = RTE_ALIGN_FLOOR(len, (1 << IDPF_RLAN_CTX_DBUF_S)); ++ rxq->rx_buf_len = RTE_MIN(rxq->rx_buf_len, IDPF_RX_MAX_DATA_BUF_SIZE); + + len = rxq->nb_rx_desc + IDPF_RX_MAX_BURST; + ring_size = RTE_ALIGN(len * +@@ -1162,6 +1164,7 @@ idpf_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id) + } + + rxq = dev->data->rx_queues[rx_queue_id]; ++ rxq->q_started = false; + if (vport->rxq_model == VIRTCHNL2_QUEUE_MODEL_SINGLE) { + rxq->ops->release_mbufs(rxq); + reset_single_rx_queue(rxq); +@@ -1193,6 +1196,7 @@ idpf_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) + } + + txq = dev->data->tx_queues[tx_queue_id]; ++ txq->q_started = false; + txq->ops->release_mbufs(txq); + if (vport->txq_model == VIRTCHNL2_QUEUE_MODEL_SINGLE) { + reset_single_tx_queue(txq); +@@ -1508,6 +1512,7 @@ idpf_split_tx_free(struct idpf_tx_queue *cq) + struct idpf_tx_entry *txe; + struct idpf_tx_queue *txq; + uint16_t gen, qid, q_head; ++ uint16_t nb_desc_clean; + uint8_t ctype; + + txd = &compl_ring[next]; +@@ -1525,20 +1530,24 @@ idpf_split_tx_free(struct idpf_tx_queue *cq) + + switch (ctype) { + case IDPF_TXD_COMPLT_RE: +- if (q_head == 0) +- txq->last_desc_cleaned = txq->nb_tx_desc - 1; +- else +- txq->last_desc_cleaned = q_head - 1; +- if (unlikely((txq->last_desc_cleaned % 32) == 0)) { ++ /* clean to q_head which indicates be fetched txq desc id + 1. ++ * TODO: need to refine and remove the if condition. ++ */ ++ if (unlikely(q_head % 32)) { + PMD_DRV_LOG(ERR, "unexpected desc (head = %u) completion.", + q_head); + return; + } +- ++ if (txq->last_desc_cleaned > q_head) ++ nb_desc_clean = (txq->nb_tx_desc - txq->last_desc_cleaned) + ++ q_head; ++ else ++ nb_desc_clean = q_head - txq->last_desc_cleaned; ++ txq->nb_free += nb_desc_clean; ++ txq->last_desc_cleaned = q_head; + break; + case IDPF_TXD_COMPLT_RS: +- txq->nb_free++; +- txq->nb_used--; ++ /* q_head indicates sw_id when ctype is 2 */ + txe = &txq->sw_ring[q_head]; + if (txe->mbuf != NULL) { + rte_pktmbuf_free_seg(txe->mbuf); +@@ -1693,12 +1702,16 @@ idpf_splitq_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + /* fill the last descriptor with End of Packet (EOP) bit */ + txd->qw1.cmd_dtype |= IDPF_TXD_FLEX_FLOW_CMD_EOP; + +- if (unlikely((tx_id % 32) == 0)) +- txd->qw1.cmd_dtype |= IDPF_TXD_FLEX_FLOW_CMD_RE; + if (ol_flags & IDPF_TX_CKSUM_OFFLOAD_MASK) + txd->qw1.cmd_dtype |= IDPF_TXD_FLEX_FLOW_CMD_CS_EN; + txq->nb_free = (uint16_t)(txq->nb_free - nb_used); + txq->nb_used = (uint16_t)(txq->nb_used + nb_used); ++ ++ if (txq->nb_used >= 32) { ++ txd->qw1.cmd_dtype |= IDPF_TXD_FLEX_FLOW_CMD_RE; ++ /* Update txq RE bit counters */ ++ txq->nb_used = 0; ++ } + } + + /* update the tail pointer if any packets were processed */ +diff --git a/dpdk/drivers/net/idpf/idpf_rxtx.h b/dpdk/drivers/net/idpf/idpf_rxtx.h +index 730dc64ebc..1c5b5b7c38 100644 +--- a/dpdk/drivers/net/idpf/idpf_rxtx.h ++++ b/dpdk/drivers/net/idpf/idpf_rxtx.h +@@ -6,6 +6,9 @@ + #define _IDPF_RXTX_H_ + + #include "idpf_ethdev.h" ++#define IDPF_RLAN_CTX_DBUF_S 7 ++#define IDPF_RX_MAX_DATA_BUF_SIZE (16 * 1024 - 128) ++ + + /* MTS */ + #define GLTSYN_CMD_SYNC_0_0 (PF_TIMESYNC_BASE + 0x0) +diff --git a/dpdk/drivers/net/idpf/idpf_rxtx_vec_avx512.c b/dpdk/drivers/net/idpf/idpf_rxtx_vec_avx512.c +index fb2b6bb53c..f31582f5fd 100644 +--- a/dpdk/drivers/net/idpf/idpf_rxtx_vec_avx512.c ++++ b/dpdk/drivers/net/idpf/idpf_rxtx_vec_avx512.c +@@ -843,6 +843,10 @@ idpf_singleq_tx_release_mbufs_avx512(struct idpf_tx_queue *txq) + } + i = 0; + } ++ for (; i < txq->tx_tail; i++) { ++ rte_pktmbuf_free_seg(swr[i].mbuf); ++ swr[i].mbuf = NULL; ++ } + } + + static const struct idpf_txq_ops avx512_singleq_tx_vec_ops = { +diff --git a/dpdk/drivers/net/igc/igc_txrx.c b/dpdk/drivers/net/igc/igc_txrx.c +index ffd219b0df..160865e911 100644 +--- a/dpdk/drivers/net/igc/igc_txrx.c ++++ b/dpdk/drivers/net/igc/igc_txrx.c +@@ -1291,6 +1291,7 @@ igc_rx_init(struct rte_eth_dev *dev) + dvmolr |= IGC_DVMOLR_STRCRC; + + IGC_WRITE_REG(hw, IGC_DVMOLR(rxq->reg_idx), dvmolr); ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; + } + + return 0; +@@ -1934,6 +1935,7 @@ igc_dev_clear_queues(struct rte_eth_dev *dev) + if (txq != NULL) { + igc_tx_queue_release_mbufs(txq); + igc_reset_tx_queue(txq); ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; + } + } + +@@ -1942,6 +1944,7 @@ igc_dev_clear_queues(struct rte_eth_dev *dev) + if (rxq != NULL) { + igc_rx_queue_release_mbufs(rxq); + igc_reset_rx_queue(rxq); ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; + } + } + } +@@ -2187,6 +2190,7 @@ igc_tx_init(struct rte_eth_dev *dev) + IGC_TXDCTL_WTHRESH_MSK; + txdctl |= IGC_TXDCTL_QUEUE_ENABLE; + IGC_WRITE_REG(hw, IGC_TXDCTL(txq->reg_idx), txdctl); ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; + } + + igc_config_collision_dist(hw); +diff --git a/dpdk/drivers/net/ipn3ke/ipn3ke_ethdev.c b/dpdk/drivers/net/ipn3ke/ipn3ke_ethdev.c +index 70a06a3b15..2c15611a23 100644 +--- a/dpdk/drivers/net/ipn3ke/ipn3ke_ethdev.c ++++ b/dpdk/drivers/net/ipn3ke/ipn3ke_ethdev.c +@@ -558,7 +558,7 @@ static int ipn3ke_vswitch_remove(struct rte_afu_device *afu_dev) + snprintf(name, sizeof(name), "net_%s_representor_%d", + afu_dev->device.name, i); + +- ethdev = rte_eth_dev_allocated(afu_dev->device.name); ++ ethdev = rte_eth_dev_allocated(name); + if (ethdev != NULL) + rte_eth_dev_destroy(ethdev, ipn3ke_rpst_uninit); + } +diff --git a/dpdk/drivers/net/ipn3ke/ipn3ke_representor.c b/dpdk/drivers/net/ipn3ke/ipn3ke_representor.c +index 2ef96a984a..5361867785 100644 +--- a/dpdk/drivers/net/ipn3ke/ipn3ke_representor.c ++++ b/dpdk/drivers/net/ipn3ke/ipn3ke_representor.c +@@ -120,6 +120,7 @@ ipn3ke_rpst_dev_start(struct rte_eth_dev *dev) + uint64_t base_mac; + uint32_t val; + char attr_name[IPN3KE_RAWDEV_ATTR_LEN_MAX]; ++ uint16_t i; + + rawdev = hw->rawdev; + +@@ -190,6 +191,11 @@ ipn3ke_rpst_dev_start(struct rte_eth_dev *dev) + + ipn3ke_rpst_link_update(dev, 0); + ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ + return 0; + } + +@@ -198,6 +204,7 @@ ipn3ke_rpst_dev_stop(struct rte_eth_dev *dev) + { + struct ipn3ke_hw *hw = IPN3KE_DEV_PRIVATE_TO_HW(dev); + struct ipn3ke_rpst *rpst = IPN3KE_DEV_PRIVATE_TO_RPST(dev); ++ uint16_t i; + + if (hw->retimer.mac_type == IFPGA_RAWDEV_RETIMER_MAC_TYPE_10GE_XFI) { + /* Disable the TX path */ +@@ -207,6 +214,11 @@ ipn3ke_rpst_dev_stop(struct rte_eth_dev *dev) + ipn3ke_xmac_rx_disable(hw, rpst->port_id, 0); + } + ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ + return 0; + } + +@@ -2579,7 +2591,7 @@ ipn3ke_rpst_scan_handle_request(__rte_unused void *param) + } + rte_delay_us(50 * MS); + +- if (num == 0xffffff) ++ if (num == 0 || num == 0xffffff) + return NULL; + } + +diff --git a/dpdk/drivers/net/ixgbe/ixgbe_ethdev.c b/dpdk/drivers/net/ixgbe/ixgbe_ethdev.c +index ae9f65b334..65655b9212 100644 +--- a/dpdk/drivers/net/ixgbe/ixgbe_ethdev.c ++++ b/dpdk/drivers/net/ixgbe/ixgbe_ethdev.c +@@ -3852,23 +3852,32 @@ static int + ixgbe_fw_version_get(struct rte_eth_dev *dev, char *fw_version, size_t fw_size) + { + struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); +- u16 eeprom_verh, eeprom_verl; +- u32 etrack_id; ++ struct ixgbe_nvm_version nvm_ver; + int ret; + +- ixgbe_read_eeprom(hw, 0x2e, &eeprom_verh); +- ixgbe_read_eeprom(hw, 0x2d, &eeprom_verl); ++ ixgbe_get_oem_prod_version(hw, &nvm_ver); ++ if (nvm_ver.oem_valid) { ++ snprintf(fw_version, fw_size, "%x.%x.%x", ++ nvm_ver.oem_major, nvm_ver.oem_minor, ++ nvm_ver.oem_release); ++ return 0; ++ } ++ ++ ixgbe_get_etk_id(hw, &nvm_ver); ++ ixgbe_get_orom_version(hw, &nvm_ver); + +- etrack_id = (eeprom_verh << 16) | eeprom_verl; +- ret = snprintf(fw_version, fw_size, "0x%08x", etrack_id); ++ if (nvm_ver.or_valid) { ++ snprintf(fw_version, fw_size, "0x%08x, %d.%d.%d", ++ nvm_ver.etk_id, nvm_ver.or_major, ++ nvm_ver.or_build, nvm_ver.or_patch); ++ return 0; ++ } ++ ++ ret = snprintf(fw_version, fw_size, "0x%08x", nvm_ver.etk_id); + if (ret < 0) + return -EINVAL; + +- ret += 1; /* add the size of '\0' */ +- if (fw_size < (size_t)ret) +- return ret; +- else +- return 0; ++ return (fw_size < (size_t)ret++) ? ret : 0; + } + + static int +diff --git a/dpdk/drivers/net/ixgbe/ixgbe_flow.c b/dpdk/drivers/net/ixgbe/ixgbe_flow.c +index 110ff34fcc..7cccbfddb3 100644 +--- a/dpdk/drivers/net/ixgbe/ixgbe_flow.c ++++ b/dpdk/drivers/net/ixgbe/ixgbe_flow.c +@@ -1918,9 +1918,9 @@ ixgbe_parse_fdir_filter_normal(struct rte_eth_dev *dev, + + /* check src addr mask */ + for (j = 0; j < 16; j++) { +- if (ipv6_mask->hdr.src_addr[j] == UINT8_MAX) { +- rule->mask.src_ipv6_mask |= 1 << j; +- } else if (ipv6_mask->hdr.src_addr[j] != 0) { ++ if (ipv6_mask->hdr.src_addr[j] == 0) { ++ rule->mask.src_ipv6_mask &= ~(1 << j); ++ } else if (ipv6_mask->hdr.src_addr[j] != UINT8_MAX) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, +@@ -1931,9 +1931,9 @@ ixgbe_parse_fdir_filter_normal(struct rte_eth_dev *dev, + + /* check dst addr mask */ + for (j = 0; j < 16; j++) { +- if (ipv6_mask->hdr.dst_addr[j] == UINT8_MAX) { +- rule->mask.dst_ipv6_mask |= 1 << j; +- } else if (ipv6_mask->hdr.dst_addr[j] != 0) { ++ if (ipv6_mask->hdr.dst_addr[j] == 0) { ++ rule->mask.dst_ipv6_mask &= ~(1 << j); ++ } else if (ipv6_mask->hdr.dst_addr[j] != UINT8_MAX) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, +diff --git a/dpdk/drivers/net/ixgbe/ixgbe_rxtx.c b/dpdk/drivers/net/ixgbe/ixgbe_rxtx.c +index c9d6ca9efe..a3a7c68806 100644 +--- a/dpdk/drivers/net/ixgbe/ixgbe_rxtx.c ++++ b/dpdk/drivers/net/ixgbe/ixgbe_rxtx.c +@@ -1818,11 +1818,22 @@ ixgbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + * of accesses cannot be reordered by the compiler. If they were + * not volatile, they could be reordered which could lead to + * using invalid descriptor fields when read from rxd. ++ * ++ * Meanwhile, to prevent the CPU from executing out of order, we ++ * need to use a proper memory barrier to ensure the memory ++ * ordering below. + */ + rxdp = &rx_ring[rx_id]; + staterr = rxdp->wb.upper.status_error; + if (!(staterr & rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD))) + break; ++ ++ /* ++ * Use acquire fence to ensure that status_error which includes ++ * DD bit is loaded before loading of other descriptor words. ++ */ ++ rte_atomic_thread_fence(__ATOMIC_ACQUIRE); ++ + rxd = *rxdp; + + /* +@@ -2089,32 +2100,10 @@ ixgbe_recv_pkts_lro(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts, + + next_desc: + /* +- * The code in this whole file uses the volatile pointer to +- * ensure the read ordering of the status and the rest of the +- * descriptor fields (on the compiler level only!!!). This is so +- * UGLY - why not to just use the compiler barrier instead? DPDK +- * even has the rte_compiler_barrier() for that. +- * +- * But most importantly this is just wrong because this doesn't +- * ensure memory ordering in a general case at all. For +- * instance, DPDK is supposed to work on Power CPUs where +- * compiler barrier may just not be enough! +- * +- * I tried to write only this function properly to have a +- * starting point (as a part of an LRO/RSC series) but the +- * compiler cursed at me when I tried to cast away the +- * "volatile" from rx_ring (yes, it's volatile too!!!). So, I'm +- * keeping it the way it is for now. +- * +- * The code in this file is broken in so many other places and +- * will just not work on a big endian CPU anyway therefore the +- * lines below will have to be revisited together with the rest +- * of the ixgbe PMD. +- * +- * TODO: +- * - Get rid of "volatile" and let the compiler do its job. +- * - Use the proper memory barrier (rte_rmb()) to ensure the +- * memory ordering below. ++ * "Volatile" only prevents caching of the variable marked ++ * volatile. Most important, "volatile" cannot prevent the CPU ++ * from executing out of order. So, it is necessary to use a ++ * proper memory barrier to ensure the memory ordering below. + */ + rxdp = &rx_ring[rx_id]; + staterr = rte_le_to_cpu_32(rxdp->wb.upper.status_error); +@@ -2122,6 +2111,12 @@ next_desc: + if (!(staterr & IXGBE_RXDADV_STAT_DD)) + break; + ++ /* ++ * Use acquire fence to ensure that status_error which includes ++ * DD bit is loaded before loading of other descriptor words. ++ */ ++ rte_atomic_thread_fence(__ATOMIC_ACQUIRE); ++ + rxd = *rxdp; + + PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u " +@@ -3384,6 +3379,7 @@ ixgbe_dev_clear_queues(struct rte_eth_dev *dev) + if (txq != NULL) { + txq->ops->release_mbufs(txq); + txq->ops->reset(txq); ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; + } + } + +@@ -3393,6 +3389,7 @@ ixgbe_dev_clear_queues(struct rte_eth_dev *dev) + if (rxq != NULL) { + ixgbe_rx_queue_release_mbufs(rxq); + ixgbe_reset_rx_queue(adapter, rxq); ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; + } + } + /* If loopback mode was enabled, reconfigure the link accordingly */ +@@ -5830,6 +5827,8 @@ ixgbevf_dev_rxtx_start(struct rte_eth_dev *dev) + } while (--poll_ms && !(txdctl & IXGBE_TXDCTL_ENABLE)); + if (!poll_ms) + PMD_INIT_LOG(ERR, "Could not enable Tx Queue %d", i); ++ else ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; + } + for (i = 0; i < dev->data->nb_rx_queues; i++) { + +@@ -5847,6 +5846,8 @@ ixgbevf_dev_rxtx_start(struct rte_eth_dev *dev) + } while (--poll_ms && !(rxdctl & IXGBE_RXDCTL_ENABLE)); + if (!poll_ms) + PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", i); ++ else ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; + rte_wmb(); + IXGBE_WRITE_REG(hw, IXGBE_VFRDT(i), rxq->nb_rx_desc - 1); + +diff --git a/dpdk/drivers/net/mana/gdma.c b/dpdk/drivers/net/mana/gdma.c +index 3d4039014f..7f66a7a7cf 100644 +--- a/dpdk/drivers/net/mana/gdma.c ++++ b/dpdk/drivers/net/mana/gdma.c +@@ -14,12 +14,12 @@ gdma_get_wqe_pointer(struct mana_gdma_queue *queue) + (queue->head * GDMA_WQE_ALIGNMENT_UNIT_SIZE) & + (queue->size - 1); + +- DRV_LOG(DEBUG, "txq sq_head %u sq_size %u offset_in_bytes %u", +- queue->head, queue->size, offset_in_bytes); ++ DP_LOG(DEBUG, "txq sq_head %u sq_size %u offset_in_bytes %u", ++ queue->head, queue->size, offset_in_bytes); + + if (offset_in_bytes + GDMA_WQE_ALIGNMENT_UNIT_SIZE > queue->size) +- DRV_LOG(ERR, "fatal error: offset_in_bytes %u too big", +- offset_in_bytes); ++ DP_LOG(ERR, "fatal error: offset_in_bytes %u too big", ++ offset_in_bytes); + + return ((uint8_t *)queue->buffer) + offset_in_bytes; + } +@@ -39,11 +39,11 @@ write_dma_client_oob(uint8_t *work_queue_buffer_pointer, + client_oob_size / sizeof(uint32_t); + header->client_data_unit = work_request->client_data_unit; + +- DRV_LOG(DEBUG, "queue buf %p sgl %u oob_h %u du %u oob_buf %p oob_b %u", +- work_queue_buffer_pointer, header->num_sgl_entries, +- header->inline_client_oob_size_in_dwords, +- header->client_data_unit, work_request->inline_oob_data, +- work_request->inline_oob_size_in_bytes); ++ DP_LOG(DEBUG, "queue buf %p sgl %u oob_h %u du %u oob_buf %p oob_b %u", ++ work_queue_buffer_pointer, header->num_sgl_entries, ++ header->inline_client_oob_size_in_dwords, ++ header->client_data_unit, work_request->inline_oob_data, ++ work_request->inline_oob_size_in_bytes); + + p += sizeof(struct gdma_wqe_dma_oob); + if (work_request->inline_oob_data && +@@ -73,8 +73,8 @@ write_scatter_gather_list(uint8_t *work_queue_head_pointer, + uint32_t size_to_queue_end; + uint32_t sge_list_size; + +- DRV_LOG(DEBUG, "work_queue_cur_pointer %p work_request->flags %x", +- work_queue_cur_pointer, work_request->flags); ++ DP_LOG(DEBUG, "work_queue_cur_pointer %p work_request->flags %x", ++ work_queue_cur_pointer, work_request->flags); + + num_sge = work_request->num_sgl_elements; + sge_list = work_request->sgl; +@@ -110,9 +110,9 @@ write_scatter_gather_list(uint8_t *work_queue_head_pointer, + sge_list_size = size; + } + +- DRV_LOG(DEBUG, "sge %u address 0x%" PRIx64 " size %u key %u list_s %u", +- num_sge, sge_list->address, sge_list->size, +- sge_list->memory_key, sge_list_size); ++ DP_LOG(DEBUG, "sge %u address 0x%" PRIx64 " size %u key %u list_s %u", ++ num_sge, sge_list->address, sge_list->size, ++ sge_list->memory_key, sge_list_size); + + return sge_list_size; + } +@@ -123,7 +123,7 @@ write_scatter_gather_list(uint8_t *work_queue_head_pointer, + int + gdma_post_work_request(struct mana_gdma_queue *queue, + struct gdma_work_request *work_req, +- struct gdma_posted_wqe_info *wqe_info) ++ uint32_t *wqe_size_in_bu) + { + uint32_t client_oob_size = + work_req->inline_oob_size_in_bytes > +@@ -141,22 +141,15 @@ gdma_post_work_request(struct mana_gdma_queue *queue, + uint32_t queue_free_units = queue->count - (queue->head - queue->tail); + + if (wqe_size / GDMA_WQE_ALIGNMENT_UNIT_SIZE > queue_free_units) { +- DRV_LOG(DEBUG, "WQE size %u queue count %u head %u tail %u", +- wqe_size, queue->count, queue->head, queue->tail); ++ DP_LOG(DEBUG, "WQE size %u queue count %u head %u tail %u", ++ wqe_size, queue->count, queue->head, queue->tail); + return -EBUSY; + } + +- DRV_LOG(DEBUG, "client_oob_size %u sgl_data_size %u wqe_size %u", +- client_oob_size, sgl_data_size, wqe_size); ++ DP_LOG(DEBUG, "client_oob_size %u sgl_data_size %u wqe_size %u", ++ client_oob_size, sgl_data_size, wqe_size); + +- if (wqe_info) { +- wqe_info->wqe_index = +- ((queue->head * GDMA_WQE_ALIGNMENT_UNIT_SIZE) & +- (queue->size - 1)) / GDMA_WQE_ALIGNMENT_UNIT_SIZE; +- wqe_info->unmasked_queue_offset = queue->head; +- wqe_info->wqe_size_in_bu = +- wqe_size / GDMA_WQE_ALIGNMENT_UNIT_SIZE; +- } ++ *wqe_size_in_bu = wqe_size / GDMA_WQE_ALIGNMENT_UNIT_SIZE; + + wq_buffer_pointer = gdma_get_wqe_pointer(queue); + wq_buffer_pointer += write_dma_client_oob(wq_buffer_pointer, work_req, +@@ -173,6 +166,97 @@ gdma_post_work_request(struct mana_gdma_queue *queue, + return 0; + } + ++#ifdef RTE_ARCH_32 ++union gdma_short_doorbell_entry { ++ uint32_t as_uint32; ++ ++ struct { ++ uint32_t tail_ptr_incr : 16; /* Number of CQEs */ ++ uint32_t id : 12; ++ uint32_t reserved : 3; ++ uint32_t arm : 1; ++ } cq; ++ ++ struct { ++ uint32_t tail_ptr_incr : 16; /* In number of bytes */ ++ uint32_t id : 12; ++ uint32_t reserved : 4; ++ } rq; ++ ++ struct { ++ uint32_t tail_ptr_incr : 16; /* In number of bytes */ ++ uint32_t id : 12; ++ uint32_t reserved : 4; ++ } sq; ++ ++ struct { ++ uint32_t tail_ptr_incr : 16; /* Number of EQEs */ ++ uint32_t id : 12; ++ uint32_t reserved : 3; ++ uint32_t arm : 1; ++ } eq; ++}; /* HW DATA */ ++ ++enum { ++ DOORBELL_SHORT_OFFSET_SQ = 0x10, ++ DOORBELL_SHORT_OFFSET_RQ = 0x410, ++ DOORBELL_SHORT_OFFSET_CQ = 0x810, ++ DOORBELL_SHORT_OFFSET_EQ = 0xFF0, ++}; ++ ++/* ++ * Write to hardware doorbell to notify new activity. ++ */ ++int ++mana_ring_short_doorbell(void *db_page, enum gdma_queue_types queue_type, ++ uint32_t queue_id, uint32_t tail_incr, uint8_t arm) ++{ ++ uint8_t *addr = db_page; ++ union gdma_short_doorbell_entry e = {}; ++ ++ if ((queue_id & ~GDMA_SHORT_DB_QID_MASK) || ++ (tail_incr & ~GDMA_SHORT_DB_INC_MASK)) { ++ DP_LOG(ERR, "%s: queue_id %u or " ++ "tail_incr %u overflowed, queue type %d", ++ __func__, queue_id, tail_incr, queue_type); ++ return -EINVAL; ++ } ++ ++ switch (queue_type) { ++ case GDMA_QUEUE_SEND: ++ e.sq.id = queue_id; ++ e.sq.tail_ptr_incr = tail_incr; ++ addr += DOORBELL_SHORT_OFFSET_SQ; ++ break; ++ ++ case GDMA_QUEUE_RECEIVE: ++ e.rq.id = queue_id; ++ e.rq.tail_ptr_incr = tail_incr; ++ addr += DOORBELL_SHORT_OFFSET_RQ; ++ break; ++ ++ case GDMA_QUEUE_COMPLETION: ++ e.cq.id = queue_id; ++ e.cq.tail_ptr_incr = tail_incr; ++ e.cq.arm = arm; ++ addr += DOORBELL_SHORT_OFFSET_CQ; ++ break; ++ ++ default: ++ DP_LOG(ERR, "Unsupported queue type %d", queue_type); ++ return -1; ++ } ++ ++ /* Ensure all writes are done before ringing doorbell */ ++ rte_wmb(); ++ ++ DP_LOG(DEBUG, "db_page %p addr %p queue_id %u type %u tail %u arm %u", ++ db_page, addr, queue_id, queue_type, tail_incr, arm); ++ ++ rte_write32(e.as_uint32, addr); ++ return 0; ++} ++#else + union gdma_doorbell_entry { + uint64_t as_uint64; + +@@ -242,62 +326,69 @@ mana_ring_doorbell(void *db_page, enum gdma_queue_types queue_type, + break; + + default: +- DRV_LOG(ERR, "Unsupported queue type %d", queue_type); ++ DP_LOG(ERR, "Unsupported queue type %d", queue_type); + return -1; + } + + /* Ensure all writes are done before ringing doorbell */ + rte_wmb(); + +- DRV_LOG(DEBUG, "db_page %p addr %p queue_id %u type %u tail %u arm %u", +- db_page, addr, queue_id, queue_type, tail, arm); ++ DP_LOG(DEBUG, "db_page %p addr %p queue_id %u type %u tail %u arm %u", ++ db_page, addr, queue_id, queue_type, tail, arm); + + rte_write64(e.as_uint64, addr); + return 0; + } ++#endif + + /* + * Poll completion queue for completions. + */ +-int +-gdma_poll_completion_queue(struct mana_gdma_queue *cq, struct gdma_comp *comp) ++uint32_t ++gdma_poll_completion_queue(struct mana_gdma_queue *cq, ++ struct gdma_comp *gdma_comp, uint32_t max_comp) + { + struct gdma_hardware_completion_entry *cqe; +- uint32_t head = cq->head % cq->count; + uint32_t new_owner_bits, old_owner_bits; + uint32_t cqe_owner_bits; ++ uint32_t num_comp = 0; + struct gdma_hardware_completion_entry *buffer = cq->buffer; + +- cqe = &buffer[head]; +- new_owner_bits = (cq->head / cq->count) & COMPLETION_QUEUE_OWNER_MASK; +- old_owner_bits = (cq->head / cq->count - 1) & +- COMPLETION_QUEUE_OWNER_MASK; +- cqe_owner_bits = cqe->owner_bits; ++ while (num_comp < max_comp) { ++ cqe = &buffer[cq->head % cq->count]; ++ new_owner_bits = (cq->head / cq->count) & ++ COMPLETION_QUEUE_OWNER_MASK; ++ old_owner_bits = (cq->head / cq->count - 1) & ++ COMPLETION_QUEUE_OWNER_MASK; ++ cqe_owner_bits = cqe->owner_bits; ++ ++ DP_LOG(DEBUG, "comp cqe bits 0x%x owner bits 0x%x", ++ cqe_owner_bits, old_owner_bits); ++ ++ /* No new entry */ ++ if (cqe_owner_bits == old_owner_bits) ++ break; ++ ++ if (cqe_owner_bits != new_owner_bits) { ++ DRV_LOG(ERR, "CQ overflowed, ID %u cqe 0x%x new 0x%x", ++ cq->id, cqe_owner_bits, new_owner_bits); ++ break; ++ } + +- DRV_LOG(DEBUG, "comp cqe bits 0x%x owner bits 0x%x", +- cqe_owner_bits, old_owner_bits); ++ gdma_comp[num_comp].cqe_data = cqe->dma_client_data; ++ num_comp++; + +- if (cqe_owner_bits == old_owner_bits) +- return 0; /* No new entry */ ++ cq->head++; + +- if (cqe_owner_bits != new_owner_bits) { +- DRV_LOG(ERR, "CQ overflowed, ID %u cqe 0x%x new 0x%x", +- cq->id, cqe_owner_bits, new_owner_bits); +- return -1; ++ DP_LOG(DEBUG, "comp new 0x%x old 0x%x cqe 0x%x wq %u sq %u head %u", ++ new_owner_bits, old_owner_bits, cqe_owner_bits, ++ cqe->wq_num, cqe->is_sq, cq->head); + } + +- /* Ensure checking owner bits happens before reading from CQE */ ++ /* Make sure the CQE owner bits are checked before we access the data ++ * in CQE ++ */ + rte_rmb(); + +- comp->work_queue_number = cqe->wq_num; +- comp->send_work_queue = cqe->is_sq; +- +- memcpy(comp->completion_data, cqe->dma_client_data, GDMA_COMP_DATA_SIZE); +- +- cq->head++; +- +- DRV_LOG(DEBUG, "comp new 0x%x old 0x%x cqe 0x%x wq %u sq %u head %u", +- new_owner_bits, old_owner_bits, cqe_owner_bits, +- comp->work_queue_number, comp->send_work_queue, cq->head); +- return 1; ++ return num_comp; + } +diff --git a/dpdk/drivers/net/mana/mana.c b/dpdk/drivers/net/mana/mana.c +index 43221e743e..896b53ed35 100644 +--- a/dpdk/drivers/net/mana/mana.c ++++ b/dpdk/drivers/net/mana/mana.c +@@ -487,6 +487,15 @@ mana_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, + goto fail; + } + ++ txq->gdma_comp_buf = rte_malloc_socket("mana_txq_comp", ++ sizeof(*txq->gdma_comp_buf) * nb_desc, ++ RTE_CACHE_LINE_SIZE, socket_id); ++ if (!txq->gdma_comp_buf) { ++ DRV_LOG(ERR, "failed to allocate txq comp"); ++ ret = -ENOMEM; ++ goto fail; ++ } ++ + ret = mana_mr_btree_init(&txq->mr_btree, + MANA_MR_BTREE_PER_QUEUE_N, socket_id); + if (ret) { +@@ -506,6 +515,7 @@ mana_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, + return 0; + + fail: ++ rte_free(txq->gdma_comp_buf); + rte_free(txq->desc_ring); + rte_free(txq); + return ret; +@@ -518,6 +528,7 @@ mana_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid) + + mana_mr_btree_free(&txq->mr_btree); + ++ rte_free(txq->gdma_comp_buf); + rte_free(txq->desc_ring); + rte_free(txq); + } +@@ -557,6 +568,15 @@ mana_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, + rxq->desc_ring_head = 0; + rxq->desc_ring_tail = 0; + ++ rxq->gdma_comp_buf = rte_malloc_socket("mana_rxq_comp", ++ sizeof(*rxq->gdma_comp_buf) * nb_desc, ++ RTE_CACHE_LINE_SIZE, socket_id); ++ if (!rxq->gdma_comp_buf) { ++ DRV_LOG(ERR, "failed to allocate rxq comp"); ++ ret = -ENOMEM; ++ goto fail; ++ } ++ + ret = mana_mr_btree_init(&rxq->mr_btree, + MANA_MR_BTREE_PER_QUEUE_N, socket_id); + if (ret) { +@@ -572,6 +592,7 @@ mana_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, + return 0; + + fail: ++ rte_free(rxq->gdma_comp_buf); + rte_free(rxq->desc_ring); + rte_free(rxq); + return ret; +@@ -584,6 +605,7 @@ mana_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid) + + mana_mr_btree_free(&rxq->mr_btree); + ++ rte_free(rxq->gdma_comp_buf); + rte_free(rxq->desc_ring); + rte_free(rxq); + } +@@ -616,9 +638,9 @@ mana_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) + if (!txq) + continue; + +- stats->opackets = txq->stats.packets; +- stats->obytes = txq->stats.bytes; +- stats->oerrors = txq->stats.errors; ++ stats->opackets += txq->stats.packets; ++ stats->obytes += txq->stats.bytes; ++ stats->oerrors += txq->stats.errors; + + if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) { + stats->q_opackets[i] = txq->stats.packets; +@@ -633,9 +655,9 @@ mana_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) + if (!rxq) + continue; + +- stats->ipackets = rxq->stats.packets; +- stats->ibytes = rxq->stats.bytes; +- stats->ierrors = rxq->stats.errors; ++ stats->ipackets += rxq->stats.packets; ++ stats->ibytes += rxq->stats.bytes; ++ stats->ierrors += rxq->stats.errors; + + /* There is no good way to get stats->imissed, not setting it */ + +@@ -1238,7 +1260,7 @@ mana_probe_port(struct ibv_device *ibdev, struct ibv_device_attr_ex *dev_attr, + /* Create a parent domain with the port number */ + attr.pd = priv->ib_pd; + attr.comp_mask = IBV_PARENT_DOMAIN_INIT_ATTR_PD_CONTEXT; +- attr.pd_context = (void *)(uint64_t)port; ++ attr.pd_context = (void *)(uintptr_t)port; + priv->ib_parent_pd = ibv_alloc_parent_domain(ctx, &attr); + if (!priv->ib_parent_pd) { + DRV_LOG(ERR, "ibv_alloc_parent_domain failed port %d", port); +@@ -1321,6 +1343,7 @@ failed: + /* + * Goes through the IB device list to look for the IB port matching the + * mac_addr. If found, create a rte_eth_dev for it. ++ * Return value: number of successfully probed devices + */ + static int + mana_pci_probe_mac(struct rte_pci_device *pci_dev, +@@ -1330,8 +1353,9 @@ mana_pci_probe_mac(struct rte_pci_device *pci_dev, + int ibv_idx; + struct ibv_context *ctx; + int num_devices; +- int ret = 0; ++ int ret; + uint8_t port; ++ int count = 0; + + ibv_list = ibv_get_device_list(&num_devices); + for (ibv_idx = 0; ibv_idx < num_devices; ibv_idx++) { +@@ -1361,6 +1385,12 @@ mana_pci_probe_mac(struct rte_pci_device *pci_dev, + ret = ibv_query_device_ex(ctx, NULL, &dev_attr); + ibv_close_device(ctx); + ++ if (ret) { ++ DRV_LOG(ERR, "Failed to query IB device %s", ++ ibdev->name); ++ continue; ++ } ++ + for (port = 1; port <= dev_attr.orig_attr.phys_port_cnt; + port++) { + struct rte_ether_addr addr; +@@ -1372,15 +1402,17 @@ mana_pci_probe_mac(struct rte_pci_device *pci_dev, + continue; + + ret = mana_probe_port(ibdev, &dev_attr, port, pci_dev, &addr); +- if (ret) ++ if (ret) { + DRV_LOG(ERR, "Probe on IB port %u failed %d", port, ret); +- else ++ } else { ++ count++; + DRV_LOG(INFO, "Successfully probed on IB port %u", port); ++ } + } + } + + ibv_free_device_list(ibv_list); +- return ret; ++ return count; + } + + /* +@@ -1394,6 +1426,7 @@ mana_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, + struct mana_conf conf = {0}; + unsigned int i; + int ret; ++ int count = 0; + + if (args && args->drv_str) { + ret = mana_parse_args(args, &conf); +@@ -1411,16 +1444,21 @@ mana_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, + } + + /* If there are no driver parameters, probe on all ports */ +- if (!conf.index) +- return mana_pci_probe_mac(pci_dev, NULL); ++ if (conf.index) { ++ for (i = 0; i < conf.index; i++) ++ count += mana_pci_probe_mac(pci_dev, ++ &conf.mac_array[i]); ++ } else { ++ count = mana_pci_probe_mac(pci_dev, NULL); ++ } + +- for (i = 0; i < conf.index; i++) { +- ret = mana_pci_probe_mac(pci_dev, &conf.mac_array[i]); +- if (ret) +- return ret; ++ if (!count) { ++ rte_memzone_free(mana_shared_mz); ++ mana_shared_mz = NULL; ++ ret = -ENODEV; + } + +- return 0; ++ return ret; + } + + static int +@@ -1453,6 +1491,7 @@ mana_pci_remove(struct rte_pci_device *pci_dev) + if (!mana_shared_data->primary_cnt) { + DRV_LOG(DEBUG, "free shared memezone data"); + rte_memzone_free(mana_shared_mz); ++ mana_shared_mz = NULL; + } + + rte_spinlock_unlock(&mana_shared_data_lock); +diff --git a/dpdk/drivers/net/mana/mana.h b/dpdk/drivers/net/mana/mana.h +index 4a05238a96..f70a3e0b3d 100644 +--- a/dpdk/drivers/net/mana/mana.h ++++ b/dpdk/drivers/net/mana/mana.h +@@ -50,6 +50,21 @@ struct mana_shared_data { + #define MAX_TX_WQE_SIZE 512 + #define MAX_RX_WQE_SIZE 256 + ++/* For 32 bit only */ ++#ifdef RTE_ARCH_32 ++#define GDMA_SHORT_DB_INC_MASK 0xffff ++#define GDMA_SHORT_DB_QID_MASK 0xfff ++ ++#define GDMA_SHORT_DB_MAX_WQE (0x10000 / GDMA_WQE_ALIGNMENT_UNIT_SIZE) ++ ++#define TX_WQE_SHORT_DB_THRESHOLD \ ++ (GDMA_SHORT_DB_MAX_WQE - \ ++ (MAX_TX_WQE_SIZE / GDMA_WQE_ALIGNMENT_UNIT_SIZE)) ++#define RX_WQE_SHORT_DB_THRESHOLD \ ++ (GDMA_SHORT_DB_MAX_WQE - \ ++ (MAX_RX_WQE_SIZE / GDMA_WQE_ALIGNMENT_UNIT_SIZE)) ++#endif ++ + /* Values from the GDMA specification document, WQE format description */ + #define INLINE_OOB_SMALL_SIZE_IN_BYTES 8 + #define INLINE_OOB_LARGE_SIZE_IN_BYTES 24 +@@ -142,19 +157,6 @@ struct gdma_header { + #define COMPLETION_QUEUE_OWNER_MASK \ + ((1 << (COMPLETION_QUEUE_ENTRY_OWNER_BITS_SIZE)) - 1) + +-struct gdma_comp { +- struct gdma_header gdma_header; +- +- /* Filled by GDMA core */ +- uint32_t completion_data[GDMA_COMP_DATA_SIZE_IN_UINT32]; +- +- /* Filled by GDMA core */ +- uint32_t work_queue_number; +- +- /* Filled by GDMA core */ +- bool send_work_queue; +-}; +- + struct gdma_hardware_completion_entry { + char dma_client_data[GDMA_COMP_DATA_SIZE]; + union { +@@ -391,6 +393,11 @@ struct mana_gdma_queue { + + #define MANA_MR_BTREE_PER_QUEUE_N 64 + ++struct gdma_comp { ++ /* Filled by GDMA core */ ++ char *cqe_data; ++}; ++ + struct mana_txq { + struct mana_priv *priv; + uint32_t num_desc; +@@ -399,6 +406,7 @@ struct mana_txq { + + struct mana_gdma_queue gdma_sq; + struct mana_gdma_queue gdma_cq; ++ struct gdma_comp *gdma_comp_buf; + + uint32_t tx_vp_offset; + +@@ -431,8 +439,14 @@ struct mana_rxq { + */ + uint32_t desc_ring_head, desc_ring_tail; + ++#ifdef RTE_ARCH_32 ++ /* For storing wqe increment count btw each short doorbell ring */ ++ uint32_t wqe_cnt_to_short_db; ++#endif ++ + struct mana_gdma_queue gdma_rq; + struct mana_gdma_queue gdma_cq; ++ struct gdma_comp *gdma_comp_buf; + + struct mana_stats stats; + struct mana_mr_btree mr_btree; +@@ -447,19 +461,28 @@ extern int mana_logtype_init; + rte_log(RTE_LOG_ ## level, mana_logtype_driver, "%s(): " fmt "\n", \ + __func__, ## args) + ++#define DP_LOG(level, fmt, args...) \ ++ RTE_LOG_DP(level, PMD, fmt "\n", ## args) ++ + #define PMD_INIT_LOG(level, fmt, args...) \ + rte_log(RTE_LOG_ ## level, mana_logtype_init, "%s(): " fmt "\n",\ + __func__, ## args) + + #define PMD_INIT_FUNC_TRACE() PMD_INIT_LOG(DEBUG, " >>") + ++#ifdef RTE_ARCH_32 ++int mana_ring_short_doorbell(void *db_page, enum gdma_queue_types queue_type, ++ uint32_t queue_id, uint32_t tail_incr, ++ uint8_t arm); ++#else + int mana_ring_doorbell(void *db_page, enum gdma_queue_types queue_type, + uint32_t queue_id, uint32_t tail, uint8_t arm); +-int mana_rq_ring_doorbell(struct mana_rxq *rxq, uint8_t arm); ++#endif ++int mana_rq_ring_doorbell(struct mana_rxq *rxq); + + int gdma_post_work_request(struct mana_gdma_queue *queue, + struct gdma_work_request *work_req, +- struct gdma_posted_wqe_info *wqe_info); ++ uint32_t *wqe_size_in_bu); + uint8_t *gdma_get_wqe_pointer(struct mana_gdma_queue *queue); + + uint16_t mana_rx_burst(void *dpdk_rxq, struct rte_mbuf **rx_pkts, +@@ -473,8 +496,9 @@ uint16_t mana_rx_burst_removed(void *dpdk_rxq, struct rte_mbuf **pkts, + uint16_t mana_tx_burst_removed(void *dpdk_rxq, struct rte_mbuf **pkts, + uint16_t pkts_n); + +-int gdma_poll_completion_queue(struct mana_gdma_queue *cq, +- struct gdma_comp *comp); ++uint32_t gdma_poll_completion_queue(struct mana_gdma_queue *cq, ++ struct gdma_comp *gdma_comp, ++ uint32_t max_comp); + + int mana_start_rx_queues(struct rte_eth_dev *dev); + int mana_start_tx_queues(struct rte_eth_dev *dev); +diff --git a/dpdk/drivers/net/mana/meson.build b/dpdk/drivers/net/mana/meson.build +index 493f0d26d4..2d72eca5a8 100644 +--- a/dpdk/drivers/net/mana/meson.build ++++ b/dpdk/drivers/net/mana/meson.build +@@ -1,9 +1,9 @@ + # SPDX-License-Identifier: BSD-3-Clause + # Copyright(c) 2022 Microsoft Corporation + +-if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64') ++if not is_linux or not dpdk_conf.has('RTE_ARCH_X86') + build = false +- reason = 'only supported on x86_64 Linux' ++ reason = 'only supported on x86 Linux' + subdir_done() + endif + +diff --git a/dpdk/drivers/net/mana/mp.c b/dpdk/drivers/net/mana/mp.c +index 92432c431d..738487f65a 100644 +--- a/dpdk/drivers/net/mana/mp.c ++++ b/dpdk/drivers/net/mana/mp.c +@@ -28,8 +28,8 @@ mana_mp_mr_create(struct mana_priv *priv, uintptr_t addr, uint32_t len) + if (!ibv_mr) + return -errno; + +- DRV_LOG(DEBUG, "MR (2nd) lkey %u addr %p len %zu", +- ibv_mr->lkey, ibv_mr->addr, ibv_mr->length); ++ DP_LOG(DEBUG, "MR (2nd) lkey %u addr %p len %zu", ++ ibv_mr->lkey, ibv_mr->addr, ibv_mr->length); + + mr = rte_calloc("MANA MR", 1, sizeof(*mr), 0); + if (!mr) { +diff --git a/dpdk/drivers/net/mana/mr.c b/dpdk/drivers/net/mana/mr.c +index 22df0917bb..b8e6ea0bbf 100644 +--- a/dpdk/drivers/net/mana/mr.c ++++ b/dpdk/drivers/net/mana/mr.c +@@ -47,23 +47,23 @@ mana_new_pmd_mr(struct mana_mr_btree *local_tree, struct mana_priv *priv, + + for (i = 0; i < pool->nb_mem_chunks; i++) { + if (ranges[i].len > priv->max_mr_size) { +- DRV_LOG(ERR, "memory chunk size %u exceeding max MR", +- ranges[i].len); ++ DP_LOG(ERR, "memory chunk size %u exceeding max MR", ++ ranges[i].len); + return -ENOMEM; + } + +- DRV_LOG(DEBUG, +- "registering memory chunk start 0x%" PRIx64 " len %u", +- ranges[i].start, ranges[i].len); ++ DP_LOG(DEBUG, ++ "registering memory chunk start 0x%" PRIxPTR " len %u", ++ ranges[i].start, ranges[i].len); + + if (rte_eal_process_type() == RTE_PROC_SECONDARY) { + /* Send a message to the primary to do MR */ + ret = mana_mp_req_mr_create(priv, ranges[i].start, + ranges[i].len); + if (ret) { +- DRV_LOG(ERR, +- "MR failed start 0x%" PRIx64 " len %u", +- ranges[i].start, ranges[i].len); ++ DP_LOG(ERR, ++ "MR failed start 0x%" PRIxPTR " len %u", ++ ranges[i].start, ranges[i].len); + return ret; + } + continue; +@@ -72,8 +72,8 @@ mana_new_pmd_mr(struct mana_mr_btree *local_tree, struct mana_priv *priv, + ibv_mr = ibv_reg_mr(priv->ib_pd, (void *)ranges[i].start, + ranges[i].len, IBV_ACCESS_LOCAL_WRITE); + if (ibv_mr) { +- DRV_LOG(DEBUG, "MR lkey %u addr %p len %" PRIu64, +- ibv_mr->lkey, ibv_mr->addr, ibv_mr->length); ++ DP_LOG(DEBUG, "MR lkey %u addr %p len %zu", ++ ibv_mr->lkey, ibv_mr->addr, ibv_mr->length); + + mr = rte_calloc("MANA MR", 1, sizeof(*mr), 0); + mr->lkey = ibv_mr->lkey; +@@ -86,7 +86,7 @@ mana_new_pmd_mr(struct mana_mr_btree *local_tree, struct mana_priv *priv, + rte_spinlock_unlock(&priv->mr_btree_lock); + if (ret) { + ibv_dereg_mr(ibv_mr); +- DRV_LOG(ERR, "Failed to add to global MR btree"); ++ DP_LOG(ERR, "Failed to add to global MR btree"); + return ret; + } + +@@ -95,12 +95,12 @@ mana_new_pmd_mr(struct mana_mr_btree *local_tree, struct mana_priv *priv, + /* Don't need to clean up MR as it's already + * in the global tree + */ +- DRV_LOG(ERR, "Failed to add to local MR btree"); ++ DP_LOG(ERR, "Failed to add to local MR btree"); + return ret; + } + } else { +- DRV_LOG(ERR, "MR failed at 0x%" PRIx64 " len %u", +- ranges[i].start, ranges[i].len); ++ DP_LOG(ERR, "MR failed at 0x%" PRIxPTR " len %u", ++ ranges[i].start, ranges[i].len); + return -errno; + } + } +@@ -118,7 +118,7 @@ mana_del_pmd_mr(struct mana_mr_cache *mr) + + ret = ibv_dereg_mr(ibv_mr); + if (ret) +- DRV_LOG(ERR, "dereg MR failed ret %d", ret); ++ DP_LOG(ERR, "dereg MR failed ret %d", ret); + } + + /* +@@ -133,17 +133,16 @@ mana_find_pmd_mr(struct mana_mr_btree *local_mr_btree, struct mana_priv *priv, + struct mana_mr_cache *mr; + uint16_t idx; + +- DRV_LOG(DEBUG, "finding mr for mbuf addr %p len %d", +- mbuf->buf_addr, mbuf->buf_len); ++ DP_LOG(DEBUG, "finding mr for mbuf addr %p len %d", ++ mbuf->buf_addr, mbuf->buf_len); + + try_again: + /* First try to find the MR in local queue tree */ + mr = mana_mr_btree_lookup(local_mr_btree, &idx, + (uintptr_t)mbuf->buf_addr, mbuf->buf_len); + if (mr) { +- DRV_LOG(DEBUG, +- "Local mr lkey %u addr 0x%" PRIx64 " len %" PRIu64, +- mr->lkey, mr->addr, mr->len); ++ DP_LOG(DEBUG, "Local mr lkey %u addr 0x%" PRIxPTR " len %zu", ++ mr->lkey, mr->addr, mr->len); + return mr; + } + +@@ -158,25 +157,25 @@ try_again: + if (mr) { + ret = mana_mr_btree_insert(local_mr_btree, mr); + if (ret) { +- DRV_LOG(DEBUG, "Failed to add MR to local tree."); ++ DP_LOG(ERR, "Failed to add MR to local tree."); + return NULL; + } + +- DRV_LOG(DEBUG, +- "Added local MR key %u addr 0x%" PRIx64 " len %" PRIu64, +- mr->lkey, mr->addr, mr->len); ++ DP_LOG(DEBUG, ++ "Added local MR key %u addr 0x%" PRIxPTR " len %zu", ++ mr->lkey, mr->addr, mr->len); + return mr; + } + + if (second_try) { +- DRV_LOG(ERR, "Internal error second try failed"); ++ DP_LOG(ERR, "Internal error second try failed"); + return NULL; + } + + ret = mana_new_pmd_mr(local_mr_btree, priv, pool); + if (ret) { +- DRV_LOG(ERR, "Failed to allocate MR ret %d addr %p len %d", +- ret, mbuf->buf_addr, mbuf->buf_len); ++ DP_LOG(ERR, "Failed to allocate MR ret %d addr %p len %d", ++ ret, mbuf->buf_addr, mbuf->buf_len); + return NULL; + } + +@@ -215,11 +214,11 @@ mana_mr_btree_expand(struct mana_mr_btree *bt, int n) + mem = rte_realloc_socket(bt->table, n * sizeof(struct mana_mr_cache), + 0, bt->socket); + if (!mem) { +- DRV_LOG(ERR, "Failed to expand btree size %d", n); ++ DP_LOG(ERR, "Failed to expand btree size %d", n); + return -1; + } + +- DRV_LOG(ERR, "Expanded btree to size %d", n); ++ DP_LOG(ERR, "Expanded btree to size %d", n); + bt->table = mem; + bt->size = n; + +@@ -266,9 +265,9 @@ mana_mr_btree_lookup(struct mana_mr_btree *bt, uint16_t *idx, + if (addr + len <= table[base].addr + table[base].len) + return &table[base]; + +- DRV_LOG(DEBUG, +- "addr 0x%" PRIx64 " len %zu idx %u sum 0x%" PRIx64 " not found", +- addr, len, *idx, addr + len); ++ DP_LOG(DEBUG, ++ "addr 0x%" PRIxPTR " len %zu idx %u sum 0x%" PRIxPTR " not found", ++ addr, len, *idx, addr + len); + + return NULL; + } +@@ -317,8 +316,8 @@ mana_mr_btree_insert(struct mana_mr_btree *bt, struct mana_mr_cache *entry) + uint16_t shift; + + if (mana_mr_btree_lookup(bt, &idx, entry->addr, entry->len)) { +- DRV_LOG(DEBUG, "Addr 0x%" PRIx64 " len %zu exists in btree", +- entry->addr, entry->len); ++ DP_LOG(DEBUG, "Addr 0x%" PRIxPTR " len %zu exists in btree", ++ entry->addr, entry->len); + return 0; + } + +@@ -332,17 +331,17 @@ mana_mr_btree_insert(struct mana_mr_btree *bt, struct mana_mr_cache *entry) + idx++; + shift = (bt->len - idx) * sizeof(struct mana_mr_cache); + if (shift) { +- DRV_LOG(DEBUG, "Moving %u bytes from idx %u to %u", +- shift, idx, idx + 1); ++ DP_LOG(DEBUG, "Moving %u bytes from idx %u to %u", ++ shift, idx, idx + 1); + memmove(&table[idx + 1], &table[idx], shift); + } + + table[idx] = *entry; + bt->len++; + +- DRV_LOG(DEBUG, +- "Inserted MR b-tree table %p idx %d addr 0x%" PRIx64 " len %zu", +- table, idx, entry->addr, entry->len); ++ DP_LOG(DEBUG, ++ "Inserted MR b-tree table %p idx %d addr 0x%" PRIxPTR " len %zu", ++ table, idx, entry->addr, entry->len); + + return 0; + } +diff --git a/dpdk/drivers/net/mana/rx.c b/dpdk/drivers/net/mana/rx.c +index 55247889c1..371510b473 100644 +--- a/dpdk/drivers/net/mana/rx.c ++++ b/dpdk/drivers/net/mana/rx.c +@@ -22,7 +22,7 @@ static uint8_t mana_rss_hash_key_default[TOEPLITZ_HASH_KEY_SIZE_IN_BYTES] = { + }; + + int +-mana_rq_ring_doorbell(struct mana_rxq *rxq, uint8_t arm) ++mana_rq_ring_doorbell(struct mana_rxq *rxq) + { + struct mana_priv *priv = rxq->priv; + int ret; +@@ -36,13 +36,24 @@ mana_rq_ring_doorbell(struct mana_rxq *rxq, uint8_t arm) + db_page = process_priv->db_page; + } + ++ /* Hardware Spec specifies that software client should set 0 for ++ * wqe_cnt for Receive Queues. ++ */ ++#ifdef RTE_ARCH_32 ++ ret = mana_ring_short_doorbell(db_page, GDMA_QUEUE_RECEIVE, ++ rxq->gdma_rq.id, ++ rxq->wqe_cnt_to_short_db * ++ GDMA_WQE_ALIGNMENT_UNIT_SIZE, ++ 0); ++#else + ret = mana_ring_doorbell(db_page, GDMA_QUEUE_RECEIVE, + rxq->gdma_rq.id, + rxq->gdma_rq.head * GDMA_WQE_ALIGNMENT_UNIT_SIZE, +- arm); ++ 0); ++#endif + + if (ret) +- DRV_LOG(ERR, "failed to ring RX doorbell ret %d", ret); ++ DP_LOG(ERR, "failed to ring RX doorbell ret %d", ret); + + return ret; + } +@@ -52,8 +63,8 @@ mana_alloc_and_post_rx_wqe(struct mana_rxq *rxq) + { + struct rte_mbuf *mbuf = NULL; + struct gdma_sgl_element sgl[1]; +- struct gdma_work_request request = {0}; +- struct gdma_posted_wqe_info wqe_info = {0}; ++ struct gdma_work_request request; ++ uint32_t wqe_size_in_bu; + struct mana_priv *priv = rxq->priv; + int ret; + struct mana_mr_cache *mr; +@@ -66,13 +77,12 @@ mana_alloc_and_post_rx_wqe(struct mana_rxq *rxq) + + mr = mana_find_pmd_mr(&rxq->mr_btree, priv, mbuf); + if (!mr) { +- DRV_LOG(ERR, "failed to register RX MR"); ++ DP_LOG(ERR, "failed to register RX MR"); + rte_pktmbuf_free(mbuf); + return -ENOMEM; + } + + request.gdma_header.struct_size = sizeof(request); +- wqe_info.gdma_header.struct_size = sizeof(wqe_info); + + sgl[0].address = rte_cpu_to_le_64(rte_pktmbuf_mtod(mbuf, uint64_t)); + sgl[0].memory_key = mr->lkey; +@@ -87,17 +97,20 @@ mana_alloc_and_post_rx_wqe(struct mana_rxq *rxq) + request.flags = 0; + request.client_data_unit = NOT_USING_CLIENT_DATA_UNIT; + +- ret = gdma_post_work_request(&rxq->gdma_rq, &request, &wqe_info); ++ ret = gdma_post_work_request(&rxq->gdma_rq, &request, &wqe_size_in_bu); + if (!ret) { + struct mana_rxq_desc *desc = + &rxq->desc_ring[rxq->desc_ring_head]; + + /* update queue for tracking pending packets */ + desc->pkt = mbuf; +- desc->wqe_size_in_bu = wqe_info.wqe_size_in_bu; ++ desc->wqe_size_in_bu = wqe_size_in_bu; ++#ifdef RTE_ARCH_32 ++ rxq->wqe_cnt_to_short_db += wqe_size_in_bu; ++#endif + rxq->desc_ring_head = (rxq->desc_ring_head + 1) % rxq->num_desc; + } else { +- DRV_LOG(ERR, "failed to post recv ret %d", ret); ++ DP_LOG(DEBUG, "failed to post recv ret %d", ret); + return ret; + } + +@@ -113,15 +126,25 @@ mana_alloc_and_post_rx_wqes(struct mana_rxq *rxq) + int ret; + uint32_t i; + ++#ifdef RTE_ARCH_32 ++ rxq->wqe_cnt_to_short_db = 0; ++#endif + for (i = 0; i < rxq->num_desc; i++) { + ret = mana_alloc_and_post_rx_wqe(rxq); + if (ret) { +- DRV_LOG(ERR, "failed to post RX ret = %d", ret); ++ DP_LOG(ERR, "failed to post RX ret = %d", ret); + return ret; + } ++ ++#ifdef RTE_ARCH_32 ++ if (rxq->wqe_cnt_to_short_db > RX_WQE_SHORT_DB_THRESHOLD) { ++ mana_rq_ring_doorbell(rxq); ++ rxq->wqe_cnt_to_short_db = 0; ++ } ++#endif + } + +- mana_rq_ring_doorbell(rxq, rxq->num_desc); ++ mana_rq_ring_doorbell(rxq); + + return ret; + } +@@ -381,27 +404,24 @@ uint16_t + mana_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) + { + uint16_t pkt_received = 0; +- uint8_t wqe_posted = 0; ++ uint16_t wqe_posted = 0; + struct mana_rxq *rxq = dpdk_rxq; + struct mana_priv *priv = rxq->priv; +- struct gdma_comp comp; + struct rte_mbuf *mbuf; + int ret; ++ uint32_t num_pkts; ++#ifdef RTE_ARCH_32 ++ rxq->wqe_cnt_to_short_db = 0; ++#endif + +- while (pkt_received < pkts_n && +- gdma_poll_completion_queue(&rxq->gdma_cq, &comp) == 1) { +- struct mana_rxq_desc *desc; +- struct mana_rx_comp_oob *oob = +- (struct mana_rx_comp_oob *)&comp.completion_data[0]; + +- if (comp.work_queue_number != rxq->gdma_rq.id) { +- DRV_LOG(ERR, "rxq comp id mismatch wqid=0x%x rcid=0x%x", +- comp.work_queue_number, rxq->gdma_rq.id); +- rxq->stats.errors++; +- break; +- } ++ num_pkts = gdma_poll_completion_queue(&rxq->gdma_cq, rxq->gdma_comp_buf, pkts_n); ++ for (uint32_t i = 0; i < num_pkts; i++) { ++ struct mana_rx_comp_oob *oob = (struct mana_rx_comp_oob *) ++ rxq->gdma_comp_buf[i].cqe_data; ++ struct mana_rxq_desc *desc = ++ &rxq->desc_ring[rxq->desc_ring_tail]; + +- desc = &rxq->desc_ring[rxq->desc_ring_tail]; + rxq->gdma_rq.tail += desc->wqe_size_in_bu; + mbuf = desc->pkt; + +@@ -411,22 +431,22 @@ mana_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) + break; + + case CQE_RX_TRUNCATED: +- DRV_LOG(ERR, "Drop a truncated packet"); ++ DP_LOG(DEBUG, "Drop a truncated packet"); + rxq->stats.errors++; + rte_pktmbuf_free(mbuf); + goto drop; + + case CQE_RX_COALESCED_4: +- DRV_LOG(ERR, "RX coalescing is not supported"); ++ DP_LOG(ERR, "RX coalescing is not supported"); + continue; + + default: +- DRV_LOG(ERR, "Unknown RX CQE type %d", +- oob->cqe_hdr.cqe_type); ++ DP_LOG(ERR, "Unknown RX CQE type %d", ++ oob->cqe_hdr.cqe_type); + continue; + } + +- DRV_LOG(DEBUG, "mana_rx_comp_oob CQE_RX_OKAY rxq %p", rxq); ++ DP_LOG(DEBUG, "mana_rx_comp_oob CQE_RX_OKAY rxq %p", rxq); + + mbuf->data_off = RTE_PKTMBUF_HEADROOM; + mbuf->nb_segs = 1; +@@ -470,19 +490,38 @@ drop: + /* Post another request */ + ret = mana_alloc_and_post_rx_wqe(rxq); + if (ret) { +- DRV_LOG(ERR, "failed to post rx wqe ret=%d", ret); ++ DP_LOG(ERR, "failed to post rx wqe ret=%d", ret); + break; + } + + wqe_posted++; ++ ++#ifdef RTE_ARCH_32 ++ /* Ring short doorbell if approaching the wqe increment ++ * limit. ++ */ ++ if (rxq->wqe_cnt_to_short_db > RX_WQE_SHORT_DB_THRESHOLD) { ++ mana_rq_ring_doorbell(rxq); ++ rxq->wqe_cnt_to_short_db = 0; ++ } ++#endif + } + + if (wqe_posted) +- mana_rq_ring_doorbell(rxq, wqe_posted); ++ mana_rq_ring_doorbell(rxq); + + return pkt_received; + } + ++#ifdef RTE_ARCH_32 ++static int ++mana_arm_cq(struct mana_rxq *rxq __rte_unused, uint8_t arm __rte_unused) ++{ ++ DP_LOG(ERR, "Do not support in 32 bit"); ++ ++ return -ENODEV; ++} ++#else + static int + mana_arm_cq(struct mana_rxq *rxq, uint8_t arm) + { +@@ -490,12 +529,13 @@ mana_arm_cq(struct mana_rxq *rxq, uint8_t arm) + uint32_t head = rxq->gdma_cq.head % + (rxq->gdma_cq.count << COMPLETION_QUEUE_ENTRY_OWNER_BITS_SIZE); + +- DRV_LOG(ERR, "Ringing completion queue ID %u head %u arm %d", +- rxq->gdma_cq.id, head, arm); ++ DP_LOG(DEBUG, "Ringing completion queue ID %u head %u arm %d", ++ rxq->gdma_cq.id, head, arm); + + return mana_ring_doorbell(priv->db_page, GDMA_QUEUE_COMPLETION, + rxq->gdma_cq.id, head, arm); + } ++#endif + + int + mana_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id) +@@ -521,8 +561,8 @@ mana_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id) + + if (ret) { + if (ret != EAGAIN) +- DRV_LOG(ERR, "Can't disable RX intr queue %d", +- rx_queue_id); ++ DP_LOG(ERR, "Can't disable RX intr queue %d", ++ rx_queue_id); + } else { + ibv_ack_cq_events(rxq->cq, 1); + } +diff --git a/dpdk/drivers/net/mana/tx.c b/dpdk/drivers/net/mana/tx.c +index 300bf27cc1..3e255157f9 100644 +--- a/dpdk/drivers/net/mana/tx.c ++++ b/dpdk/drivers/net/mana/tx.c +@@ -170,32 +170,38 @@ mana_tx_burst(void *dpdk_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) + { + struct mana_txq *txq = dpdk_txq; + struct mana_priv *priv = txq->priv; +- struct gdma_comp comp; + int ret; + void *db_page; + uint16_t pkt_sent = 0; ++ uint32_t num_comp; ++#ifdef RTE_ARCH_32 ++ uint32_t wqe_count = 0; ++#endif + + /* Process send completions from GDMA */ +- while (gdma_poll_completion_queue(&txq->gdma_cq, &comp) == 1) { ++ num_comp = gdma_poll_completion_queue(&txq->gdma_cq, ++ txq->gdma_comp_buf, txq->num_desc); ++ ++ for (uint32_t i = 0; i < num_comp; i++) { + struct mana_txq_desc *desc = + &txq->desc_ring[txq->desc_ring_tail]; +- struct mana_tx_comp_oob *oob = +- (struct mana_tx_comp_oob *)&comp.completion_data[0]; ++ struct mana_tx_comp_oob *oob = (struct mana_tx_comp_oob *) ++ txq->gdma_comp_buf[i].cqe_data; + + if (oob->cqe_hdr.cqe_type != CQE_TX_OKAY) { +- DRV_LOG(ERR, +- "mana_tx_comp_oob cqe_type %u vendor_err %u", +- oob->cqe_hdr.cqe_type, oob->cqe_hdr.vendor_err); ++ DP_LOG(ERR, ++ "mana_tx_comp_oob cqe_type %u vendor_err %u", ++ oob->cqe_hdr.cqe_type, oob->cqe_hdr.vendor_err); + txq->stats.errors++; + } else { +- DRV_LOG(DEBUG, "mana_tx_comp_oob CQE_TX_OKAY"); ++ DP_LOG(DEBUG, "mana_tx_comp_oob CQE_TX_OKAY"); + txq->stats.packets++; + } + + if (!desc->pkt) { +- DRV_LOG(ERR, "mana_txq_desc has a NULL pkt"); ++ DP_LOG(ERR, "mana_txq_desc has a NULL pkt"); + } else { +- txq->stats.bytes += desc->pkt->data_len; ++ txq->stats.bytes += desc->pkt->pkt_len; + rte_pktmbuf_free(desc->pkt); + } + +@@ -208,14 +214,14 @@ mana_tx_burst(void *dpdk_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) + for (uint16_t pkt_idx = 0; pkt_idx < nb_pkts; pkt_idx++) { + struct rte_mbuf *m_pkt = tx_pkts[pkt_idx]; + struct rte_mbuf *m_seg = m_pkt; +- struct transmit_oob_v2 tx_oob = {0}; +- struct one_sgl sgl = {0}; ++ struct transmit_oob_v2 tx_oob; ++ struct one_sgl sgl; + uint16_t seg_idx; + + /* Drop the packet if it exceeds max segments */ + if (m_pkt->nb_segs > priv->max_send_sge) { +- DRV_LOG(ERR, "send packet segments %d exceeding max", +- m_pkt->nb_segs); ++ DP_LOG(ERR, "send packet segments %d exceeding max", ++ m_pkt->nb_segs); + continue; + } + +@@ -257,12 +263,14 @@ mana_tx_burst(void *dpdk_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) + tcp_hdr->cksum = rte_ipv6_phdr_cksum(ip_hdr, + m_pkt->ol_flags); + } else { +- DRV_LOG(ERR, "Invalid input for TCP CKSUM"); ++ DP_LOG(ERR, "Invalid input for TCP CKSUM"); + } + + tx_oob.short_oob.tx_compute_TCP_checksum = 1; + tx_oob.short_oob.tx_transport_header_offset = + m_pkt->l2_len + m_pkt->l3_len; ++ } else { ++ tx_oob.short_oob.tx_compute_TCP_checksum = 0; + } + + if ((m_pkt->ol_flags & RTE_MBUF_F_TX_L4_MASK) == +@@ -297,10 +305,12 @@ mana_tx_burst(void *dpdk_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) + m_pkt->ol_flags); + + } else { +- DRV_LOG(ERR, "Invalid input for UDP CKSUM"); ++ DP_LOG(ERR, "Invalid input for UDP CKSUM"); + } + + tx_oob.short_oob.tx_compute_UDP_checksum = 1; ++ } else { ++ tx_oob.short_oob.tx_compute_UDP_checksum = 0; + } + + tx_oob.short_oob.suppress_tx_CQE_generation = 0; +@@ -310,20 +320,20 @@ mana_tx_burst(void *dpdk_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) + get_vsq_frame_num(txq->gdma_sq.id); + tx_oob.short_oob.short_vport_offset = txq->tx_vp_offset; + +- DRV_LOG(DEBUG, "tx_oob packet_format %u ipv4 %u ipv6 %u", +- tx_oob.short_oob.packet_format, +- tx_oob.short_oob.tx_is_outer_ipv4, +- tx_oob.short_oob.tx_is_outer_ipv6); ++ DP_LOG(DEBUG, "tx_oob packet_format %u ipv4 %u ipv6 %u", ++ tx_oob.short_oob.packet_format, ++ tx_oob.short_oob.tx_is_outer_ipv4, ++ tx_oob.short_oob.tx_is_outer_ipv6); + +- DRV_LOG(DEBUG, "tx_oob checksum ip %u tcp %u udp %u offset %u", +- tx_oob.short_oob.tx_compute_IP_header_checksum, +- tx_oob.short_oob.tx_compute_TCP_checksum, +- tx_oob.short_oob.tx_compute_UDP_checksum, +- tx_oob.short_oob.tx_transport_header_offset); ++ DP_LOG(DEBUG, "tx_oob checksum ip %u tcp %u udp %u offset %u", ++ tx_oob.short_oob.tx_compute_IP_header_checksum, ++ tx_oob.short_oob.tx_compute_TCP_checksum, ++ tx_oob.short_oob.tx_compute_UDP_checksum, ++ tx_oob.short_oob.tx_transport_header_offset); + +- DRV_LOG(DEBUG, "pkt[%d]: buf_addr 0x%p, nb_segs %d, pkt_len %d", +- pkt_idx, m_pkt->buf_addr, m_pkt->nb_segs, +- m_pkt->pkt_len); ++ DP_LOG(DEBUG, "pkt[%d]: buf_addr 0x%p, nb_segs %d, pkt_len %d", ++ pkt_idx, m_pkt->buf_addr, m_pkt->nb_segs, ++ m_pkt->pkt_len); + + /* Create SGL for packet data buffers */ + for (seg_idx = 0; seg_idx < m_pkt->nb_segs; seg_idx++) { +@@ -331,8 +341,8 @@ mana_tx_burst(void *dpdk_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) + mana_find_pmd_mr(&txq->mr_btree, priv, m_seg); + + if (!mr) { +- DRV_LOG(ERR, "failed to get MR, pkt_idx %u", +- pkt_idx); ++ DP_LOG(ERR, "failed to get MR, pkt_idx %u", ++ pkt_idx); + break; + } + +@@ -342,11 +352,11 @@ mana_tx_burst(void *dpdk_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) + sgl.gdma_sgl[seg_idx].size = m_seg->data_len; + sgl.gdma_sgl[seg_idx].memory_key = mr->lkey; + +- DRV_LOG(DEBUG, +- "seg idx %u addr 0x%" PRIx64 " size %x key %x", +- seg_idx, sgl.gdma_sgl[seg_idx].address, +- sgl.gdma_sgl[seg_idx].size, +- sgl.gdma_sgl[seg_idx].memory_key); ++ DP_LOG(DEBUG, ++ "seg idx %u addr 0x%" PRIx64 " size %x key %x", ++ seg_idx, sgl.gdma_sgl[seg_idx].address, ++ sgl.gdma_sgl[seg_idx].size, ++ sgl.gdma_sgl[seg_idx].memory_key); + + m_seg = m_seg->next; + } +@@ -355,11 +365,10 @@ mana_tx_burst(void *dpdk_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) + if (seg_idx != m_pkt->nb_segs) + continue; + +- struct gdma_work_request work_req = {0}; +- struct gdma_posted_wqe_info wqe_info = {0}; ++ struct gdma_work_request work_req; ++ uint32_t wqe_size_in_bu; + + work_req.gdma_header.struct_size = sizeof(work_req); +- wqe_info.gdma_header.struct_size = sizeof(wqe_info); + + work_req.sgl = sgl.gdma_sgl; + work_req.num_sgl_elements = m_pkt->nb_segs; +@@ -370,24 +379,38 @@ mana_tx_burst(void *dpdk_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) + work_req.client_data_unit = NOT_USING_CLIENT_DATA_UNIT; + + ret = gdma_post_work_request(&txq->gdma_sq, &work_req, +- &wqe_info); ++ &wqe_size_in_bu); + if (!ret) { + struct mana_txq_desc *desc = + &txq->desc_ring[txq->desc_ring_head]; + + /* Update queue for tracking pending requests */ + desc->pkt = m_pkt; +- desc->wqe_size_in_bu = wqe_info.wqe_size_in_bu; ++ desc->wqe_size_in_bu = wqe_size_in_bu; + txq->desc_ring_head = + (txq->desc_ring_head + 1) % txq->num_desc; + + pkt_sent++; + +- DRV_LOG(DEBUG, "nb_pkts %u pkt[%d] sent", +- nb_pkts, pkt_idx); ++ DP_LOG(DEBUG, "nb_pkts %u pkt[%d] sent", ++ nb_pkts, pkt_idx); ++#ifdef RTE_ARCH_32 ++ wqe_count += wqe_size_in_bu; ++ if (wqe_count > TX_WQE_SHORT_DB_THRESHOLD) { ++ /* wqe_count approaching to short doorbell ++ * increment limit. Stop processing further ++ * more packets and just ring short ++ * doorbell. ++ */ ++ DP_LOG(DEBUG, "wqe_count %u reaching limit, " ++ "pkt_sent %d", ++ wqe_count, pkt_sent); ++ break; ++ } ++#endif + } else { +- DRV_LOG(INFO, "pkt[%d] failed to post send ret %d", +- pkt_idx, ret); ++ DP_LOG(DEBUG, "pkt[%d] failed to post send ret %d", ++ pkt_idx, ret); + break; + } + } +@@ -403,13 +426,21 @@ mana_tx_burst(void *dpdk_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) + } + + if (pkt_sent) { ++#ifdef RTE_ARCH_32 ++ ret = mana_ring_short_doorbell(db_page, GDMA_QUEUE_SEND, ++ txq->gdma_sq.id, ++ wqe_count * ++ GDMA_WQE_ALIGNMENT_UNIT_SIZE, ++ 0); ++#else + ret = mana_ring_doorbell(db_page, GDMA_QUEUE_SEND, + txq->gdma_sq.id, + txq->gdma_sq.head * + GDMA_WQE_ALIGNMENT_UNIT_SIZE, + 0); ++#endif + if (ret) +- DRV_LOG(ERR, "mana_ring_doorbell failed ret %d", ret); ++ DP_LOG(ERR, "mana_ring_doorbell failed ret %d", ret); + } + + return pkt_sent; +diff --git a/dpdk/drivers/net/memif/rte_eth_memif.c b/dpdk/drivers/net/memif/rte_eth_memif.c +index 1b1c1a652b..3b83a5a8bb 100644 +--- a/dpdk/drivers/net/memif/rte_eth_memif.c ++++ b/dpdk/drivers/net/memif/rte_eth_memif.c +@@ -1240,6 +1240,7 @@ memif_dev_start(struct rte_eth_dev *dev) + { + struct pmd_internals *pmd = dev->data->dev_private; + int ret = 0; ++ uint16_t i; + + switch (pmd->role) { + case MEMIF_ROLE_CLIENT: +@@ -1254,13 +1255,28 @@ memif_dev_start(struct rte_eth_dev *dev) + break; + } + ++ if (ret == 0) { ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ } ++ + return ret; + } + + static int + memif_dev_stop(struct rte_eth_dev *dev) + { ++ uint16_t i; ++ + memif_disconnect(dev); ++ ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ + return 0; + } + +diff --git a/dpdk/drivers/net/meson.build b/dpdk/drivers/net/meson.build +index 6470bf3636..f83a6de117 100644 +--- a/dpdk/drivers/net/meson.build ++++ b/dpdk/drivers/net/meson.build +@@ -36,6 +36,7 @@ drivers = [ + 'ixgbe', + 'kni', + 'liquidio', ++ 'mana', + 'memif', + 'mlx4', + 'mlx5', +diff --git a/dpdk/drivers/net/mlx4/mlx4.c b/dpdk/drivers/net/mlx4/mlx4.c +index a54016f4a2..1389b606cc 100644 +--- a/dpdk/drivers/net/mlx4/mlx4.c ++++ b/dpdk/drivers/net/mlx4/mlx4.c +@@ -292,6 +292,7 @@ mlx4_dev_start(struct rte_eth_dev *dev) + { + struct mlx4_priv *priv = dev->data->dev_private; + struct rte_flow_error error; ++ uint16_t i; + int ret; + + if (priv->started) +@@ -327,6 +328,12 @@ mlx4_dev_start(struct rte_eth_dev *dev) + dev->rx_pkt_burst = mlx4_rx_burst; + /* Enable datapath on secondary process. */ + mlx4_mp_req_start_rxtx(dev); ++ ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ + return 0; + err: + mlx4_dev_stop(dev); +@@ -345,6 +352,7 @@ static int + mlx4_dev_stop(struct rte_eth_dev *dev) + { + struct mlx4_priv *priv = dev->data->dev_private; ++ uint16_t i; + + if (!priv->started) + return 0; +@@ -359,6 +367,11 @@ mlx4_dev_stop(struct rte_eth_dev *dev) + mlx4_rxq_intr_disable(priv); + mlx4_rss_deinit(priv); + ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ + return 0; + } + +diff --git a/dpdk/drivers/net/mlx5/hws/mlx5dr_cmd.c b/dpdk/drivers/net/mlx5/hws/mlx5dr_cmd.c +index 721376b8da..acad42e12e 100644 +--- a/dpdk/drivers/net/mlx5/hws/mlx5dr_cmd.c ++++ b/dpdk/drivers/net/mlx5/hws/mlx5dr_cmd.c +@@ -507,9 +507,11 @@ mlx5dr_cmd_header_modify_pattern_create(struct ibv_context *ctx, + uint32_t in[MLX5_ST_SZ_DW(create_header_modify_pattern_in)] = {0}; + uint32_t out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0}; + struct mlx5dr_devx_obj *devx_obj; +- void *pattern_data; ++ uint64_t *pattern_data; ++ int num_of_actions; + void *pattern; + void *attr; ++ int i; + + if (pattern_length > MAX_ACTIONS_DATA_IN_HEADER_MODIFY) { + DR_LOG(ERR, "Pattern length %d exceeds limit %d", +@@ -535,9 +537,19 @@ mlx5dr_cmd_header_modify_pattern_create(struct ibv_context *ctx, + /* Pattern_length is in ddwords */ + MLX5_SET(header_modify_pattern_in, pattern, pattern_length, pattern_length / (2 * DW_SIZE)); + +- pattern_data = MLX5_ADDR_OF(header_modify_pattern_in, pattern, pattern_data); ++ pattern_data = (uint64_t *)MLX5_ADDR_OF(header_modify_pattern_in, pattern, pattern_data); + memcpy(pattern_data, actions, pattern_length); + ++ num_of_actions = pattern_length / MLX5DR_MODIFY_ACTION_SIZE; ++ for (i = 0; i < num_of_actions; i++) { ++ int type; ++ ++ type = MLX5_GET(set_action_in, &pattern_data[i], action_type); ++ if (type != MLX5_MODIFICATION_TYPE_COPY) ++ /* Action typ-copy use all bytes for control */ ++ MLX5_SET(set_action_in, &pattern_data[i], data, 0); ++ } ++ + devx_obj->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in), out, sizeof(out)); + if (!devx_obj->obj) { + DR_LOG(ERR, "Failed to create header_modify_pattern"); +diff --git a/dpdk/drivers/net/mlx5/hws/mlx5dr_definer.c b/dpdk/drivers/net/mlx5/hws/mlx5dr_definer.c +index 6b98eb8c96..6fc5d70f67 100644 +--- a/dpdk/drivers/net/mlx5/hws/mlx5dr_definer.c ++++ b/dpdk/drivers/net/mlx5/hws/mlx5dr_definer.c +@@ -123,6 +123,7 @@ struct mlx5dr_definer_conv_data { + X(SET, ipv4_next_proto, v->next_proto_id, rte_ipv4_hdr) \ + X(SET, ipv4_version, STE_IPV4, rte_ipv4_hdr) \ + X(SET_BE16, ipv4_frag, v->fragment_offset, rte_ipv4_hdr) \ ++ X(SET, ip_fragmented, !!v->fragment_offset, rte_ipv4_hdr) \ + X(SET_BE16, ipv6_payload_len, v->hdr.payload_len, rte_flow_item_ipv6) \ + X(SET, ipv6_proto, v->hdr.proto, rte_flow_item_ipv6) \ + X(SET, ipv6_hop_limits, v->hdr.hop_limits, rte_flow_item_ipv6) \ +@@ -542,8 +543,13 @@ mlx5dr_definer_conv_item_ipv4(struct mlx5dr_definer_conv_data *cd, + if (m->fragment_offset) { + fc = &cd->fc[DR_CALC_FNAME(IP_FRAG, inner)]; + fc->item_idx = item_idx; +- fc->tag_set = &mlx5dr_definer_ipv4_frag_set; +- DR_CALC_SET(fc, eth_l3, fragment_offset, inner); ++ if (rte_be_to_cpu_16(m->fragment_offset) == 0x3fff) { ++ fc->tag_set = &mlx5dr_definer_ip_fragmented_set; ++ DR_CALC_SET(fc, eth_l2, ip_fragmented, inner); ++ } else { ++ fc->tag_set = &mlx5dr_definer_ipv4_frag_set; ++ DR_CALC_SET(fc, eth_l3, ipv4_frag, inner); ++ } + } + + if (m->next_proto_id) { +@@ -1322,7 +1328,6 @@ mlx5dr_definer_conv_item_integrity(struct mlx5dr_definer_conv_data *cd, + { + const struct rte_flow_item_integrity *m = item->mask; + struct mlx5dr_definer_fc *fc; +- bool inner = cd->tunnel; + + if (!m) + return 0; +@@ -1333,7 +1338,7 @@ mlx5dr_definer_conv_item_integrity(struct mlx5dr_definer_conv_data *cd, + } + + if (m->l3_ok || m->ipv4_csum_ok || m->l4_ok || m->l4_csum_ok) { +- fc = &cd->fc[DR_CALC_FNAME(INTEGRITY, inner)]; ++ fc = &cd->fc[DR_CALC_FNAME(INTEGRITY, m->level)]; + fc->item_idx = item_idx; + fc->tag_set = &mlx5dr_definer_integrity_set; + DR_CALC_SET_HDR(fc, oks1, oks1_bits); +@@ -1563,8 +1568,7 @@ mlx5dr_definer_conv_items_to_hl(struct mlx5dr_context *ctx, + break; + case RTE_FLOW_ITEM_TYPE_INTEGRITY: + ret = mlx5dr_definer_conv_item_integrity(&cd, items, i); +- item_flags |= cd.tunnel ? MLX5_FLOW_ITEM_INNER_INTEGRITY : +- MLX5_FLOW_ITEM_OUTER_INTEGRITY; ++ item_flags |= MLX5_FLOW_ITEM_INTEGRITY; + break; + case RTE_FLOW_ITEM_TYPE_CONNTRACK: + ret = mlx5dr_definer_conv_item_conntrack(&cd, items, i); +@@ -1629,11 +1633,15 @@ mlx5dr_definer_find_byte_in_tag(struct mlx5dr_definer *definer, + uint32_t *tag_byte_off) + { + uint8_t byte_offset; +- int i; ++ int i, dw_to_scan; ++ ++ /* Avoid accessing unused DW selectors */ ++ dw_to_scan = mlx5dr_definer_is_jumbo(definer) ? ++ DW_SELECTORS : DW_SELECTORS_MATCH; + + /* Add offset since each DW covers multiple BYTEs */ + byte_offset = hl_byte_off % DW_SIZE; +- for (i = 0; i < DW_SELECTORS; i++) { ++ for (i = 0; i < dw_to_scan; i++) { + if (definer->dw_selector[i] == hl_byte_off / DW_SIZE) { + *tag_byte_off = byte_offset + DW_SIZE * (DW_SELECTORS - i - 1); + return 0; +diff --git a/dpdk/drivers/net/mlx5/hws/mlx5dr_definer.h b/dpdk/drivers/net/mlx5/hws/mlx5dr_definer.h +index d52c6b0627..5b38a54e6b 100644 +--- a/dpdk/drivers/net/mlx5/hws/mlx5dr_definer.h ++++ b/dpdk/drivers/net/mlx5/hws/mlx5dr_definer.h +@@ -196,8 +196,13 @@ struct mlx5_ifc_definer_hl_eth_l3_bits { + u8 time_to_live_hop_limit[0x8]; + u8 protocol_next_header[0x8]; + u8 identification[0x10]; +- u8 flags[0x3]; +- u8 fragment_offset[0xd]; ++ union { ++ u8 ipv4_frag[0x10]; ++ struct { ++ u8 flags[0x3]; ++ u8 fragment_offset[0xd]; ++ }; ++ }; + u8 ipv4_total_length[0x10]; + u8 checksum[0x10]; + u8 reserved_at_60[0xc]; +diff --git a/dpdk/drivers/net/mlx5/hws/mlx5dr_pool.c b/dpdk/drivers/net/mlx5/hws/mlx5dr_pool.c +index fdbd3d438d..af6a5c743b 100644 +--- a/dpdk/drivers/net/mlx5/hws/mlx5dr_pool.c ++++ b/dpdk/drivers/net/mlx5/hws/mlx5dr_pool.c +@@ -464,13 +464,6 @@ static void mlx5dr_pool_general_element_db_uninit(struct mlx5dr_pool *pool) + */ + static int mlx5dr_pool_general_element_db_init(struct mlx5dr_pool *pool) + { +- pool->db.element_manager = simple_calloc(1, sizeof(*pool->db.element_manager)); +- if (!pool->db.element_manager) { +- DR_LOG(ERR, "No mem for general elemnt_manager"); +- rte_errno = ENOMEM; +- return rte_errno; +- } +- + pool->p_db_uninit = &mlx5dr_pool_general_element_db_uninit; + pool->p_get_chunk = &mlx5dr_pool_general_element_db_get_chunk; + pool->p_put_chunk = &mlx5dr_pool_general_element_db_put_chunk; +diff --git a/dpdk/drivers/net/mlx5/hws/mlx5dr_send.c b/dpdk/drivers/net/mlx5/hws/mlx5dr_send.c +index 5c8bbe6fc6..a8aba31cbe 100644 +--- a/dpdk/drivers/net/mlx5/hws/mlx5dr_send.c ++++ b/dpdk/drivers/net/mlx5/hws/mlx5dr_send.c +@@ -842,8 +842,8 @@ int mlx5dr_send_queue_action(struct mlx5dr_context *ctx, + /* Signal on the last posted WQE */ + mlx5dr_send_engine_flush_queue(queue); + } else { +- rte_errno = -EINVAL; +- return rte_errno; ++ rte_errno = EINVAL; ++ return -rte_errno; + } + + return 0; +diff --git a/dpdk/drivers/net/mlx5/linux/mlx5_ethdev_os.c b/dpdk/drivers/net/mlx5/linux/mlx5_ethdev_os.c +index 72268c0c8a..dd5a0c546d 100644 +--- a/dpdk/drivers/net/mlx5/linux/mlx5_ethdev_os.c ++++ b/dpdk/drivers/net/mlx5/linux/mlx5_ethdev_os.c +@@ -28,6 +28,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -745,6 +746,7 @@ mlx5_dev_interrupt_device_fatal(struct mlx5_dev_ctx_shared *sh) + + for (i = 0; i < sh->max_port; ++i) { + struct rte_eth_dev *dev; ++ struct mlx5_priv *priv; + + if (sh->port[i].ih_port_id >= RTE_MAX_ETHPORTS) { + /* +@@ -755,9 +757,14 @@ mlx5_dev_interrupt_device_fatal(struct mlx5_dev_ctx_shared *sh) + } + dev = &rte_eth_devices[sh->port[i].ih_port_id]; + MLX5_ASSERT(dev); +- if (dev->data->dev_conf.intr_conf.rmv) ++ priv = dev->data->dev_private; ++ MLX5_ASSERT(priv); ++ if (!priv->rmv_notified && dev->data->dev_conf.intr_conf.rmv) { ++ /* Notify driver about removal only once. */ ++ priv->rmv_notified = 1; + rte_eth_dev_callback_process + (dev, RTE_ETH_EVENT_INTR_RMV, NULL); ++ } + } + } + +@@ -829,21 +836,29 @@ mlx5_dev_interrupt_handler(void *cb_arg) + struct rte_eth_dev *dev; + uint32_t tmp; + +- if (mlx5_glue->get_async_event(sh->cdev->ctx, &event)) ++ if (mlx5_glue->get_async_event(sh->cdev->ctx, &event)) { ++ if (errno == EIO) { ++ DRV_LOG(DEBUG, ++ "IBV async event queue closed on: %s", ++ sh->ibdev_name); ++ mlx5_dev_interrupt_device_fatal(sh); ++ } + break; +- /* Retrieve and check IB port index. */ +- tmp = (uint32_t)event.element.port_num; +- if (!tmp && event.event_type == IBV_EVENT_DEVICE_FATAL) { ++ } ++ if (event.event_type == IBV_EVENT_DEVICE_FATAL) { + /* +- * The DEVICE_FATAL event is called once for +- * entire device without port specifying. +- * We should notify all existing ports. ++ * The DEVICE_FATAL event can be called by kernel ++ * twice - from mlx5 and uverbs layers, and port ++ * index is not applicable. We should notify all ++ * existing ports. + */ +- mlx5_glue->ack_async_event(&event); + mlx5_dev_interrupt_device_fatal(sh); ++ mlx5_glue->ack_async_event(&event); + continue; + } +- MLX5_ASSERT(tmp && (tmp <= sh->max_port)); ++ /* Retrieve and check IB port index. */ ++ tmp = (uint32_t)event.element.port_num; ++ MLX5_ASSERT(tmp <= sh->max_port); + if (!tmp) { + /* Unsupported device level event. */ + mlx5_glue->ack_async_event(&event); +@@ -1034,7 +1049,8 @@ int + mlx5_sysfs_switch_info(unsigned int ifindex, struct mlx5_switch_info *info) + { + char ifname[IF_NAMESIZE]; +- char port_name[IF_NAMESIZE]; ++ char *port_name = NULL; ++ size_t port_name_size = 0; + FILE *file; + struct mlx5_switch_info data = { + .master = 0, +@@ -1047,6 +1063,7 @@ mlx5_sysfs_switch_info(unsigned int ifindex, struct mlx5_switch_info *info) + bool port_switch_id_set = false; + bool device_dir = false; + char c; ++ ssize_t line_size; + + if (!if_indextoname(ifindex, ifname)) { + rte_errno = errno; +@@ -1062,8 +1079,22 @@ mlx5_sysfs_switch_info(unsigned int ifindex, struct mlx5_switch_info *info) + + file = fopen(phys_port_name, "rb"); + if (file != NULL) { +- if (fgets(port_name, IF_NAMESIZE, file) != NULL) ++ char *tail_nl; ++ ++ line_size = getline(&port_name, &port_name_size, file); ++ if (line_size < 0) { ++ free(port_name); ++ fclose(file); ++ rte_errno = errno; ++ return -rte_errno; ++ } else if (line_size > 0) { ++ /* Remove tailing newline character. */ ++ tail_nl = strchr(port_name, '\n'); ++ if (tail_nl) ++ *tail_nl = '\0'; + mlx5_translate_port_name(port_name, &data); ++ } ++ free(port_name); + fclose(file); + } + file = fopen(phys_switch_id, "rb"); +@@ -1776,3 +1807,70 @@ exit: + mlx5_free(sset_info); + return ret; + } ++ ++/** ++ * Unmaps HCA PCI BAR from the current process address space. ++ * ++ * @param dev ++ * Pointer to Ethernet device structure. ++ */ ++void mlx5_txpp_unmap_hca_bar(struct rte_eth_dev *dev) ++{ ++ struct mlx5_proc_priv *ppriv = dev->process_private; ++ ++ if (ppriv && ppriv->hca_bar) { ++ rte_mem_unmap(ppriv->hca_bar, MLX5_ST_SZ_BYTES(initial_seg)); ++ ppriv->hca_bar = NULL; ++ } ++} ++ ++/** ++ * Maps HCA PCI BAR to the current process address space. ++ * Stores pointer in the process private structure allowing ++ * to read internal and real time counter directly from the HW. ++ * ++ * @param dev ++ * Pointer to Ethernet device structure. ++ * ++ * @return ++ * 0 on success and not NULL pointer to mapped area in process structure. ++ * negative otherwise and NULL pointer ++ */ ++int mlx5_txpp_map_hca_bar(struct rte_eth_dev *dev) ++{ ++ struct mlx5_proc_priv *ppriv = dev->process_private; ++ char pci_addr[PCI_PRI_STR_SIZE] = { 0 }; ++ void *base, *expected = NULL; ++ int fd, ret; ++ ++ if (!ppriv) { ++ rte_errno = ENOMEM; ++ return -rte_errno; ++ } ++ if (ppriv->hca_bar) ++ return 0; ++ ret = mlx5_dev_to_pci_str(dev->device, pci_addr, sizeof(pci_addr)); ++ if (ret < 0) ++ return -rte_errno; ++ /* Open PCI device resource 0 - HCA initialize segment */ ++ MKSTR(name, "/sys/bus/pci/devices/%s/resource0", pci_addr); ++ fd = open(name, O_RDWR | O_SYNC); ++ if (fd == -1) { ++ rte_errno = ENOTSUP; ++ return -ENOTSUP; ++ } ++ base = rte_mem_map(NULL, MLX5_ST_SZ_BYTES(initial_seg), ++ RTE_PROT_READ, RTE_MAP_SHARED, fd, 0); ++ close(fd); ++ if (!base) { ++ rte_errno = ENOTSUP; ++ return -ENOTSUP; ++ } ++ /* Check there is no concurrent mapping in other thread. */ ++ if (!__atomic_compare_exchange_n(&ppriv->hca_bar, &expected, ++ base, false, ++ __ATOMIC_RELAXED, __ATOMIC_RELAXED)) ++ rte_mem_unmap(base, MLX5_ST_SZ_BYTES(initial_seg)); ++ return 0; ++} ++ +diff --git a/dpdk/drivers/net/mlx5/linux/mlx5_flow_os.c b/dpdk/drivers/net/mlx5/linux/mlx5_flow_os.c +index 3c9a823edf..2767b11708 100644 +--- a/dpdk/drivers/net/mlx5/linux/mlx5_flow_os.c ++++ b/dpdk/drivers/net/mlx5/linux/mlx5_flow_os.c +@@ -8,6 +8,10 @@ + + /* Key of thread specific flow workspace data. */ + static rte_thread_key key_workspace; ++/* Flow workspace global list head for garbage collector. */ ++static struct mlx5_flow_workspace *gc_head; ++/* Spinlock for operating flow workspace list. */ ++static rte_spinlock_t mlx5_flow_workspace_lock = RTE_SPINLOCK_INITIALIZER; + + int + mlx5_flow_os_validate_item_esp(const struct rte_flow_item *item, +@@ -48,10 +52,30 @@ mlx5_flow_os_validate_item_esp(const struct rte_flow_item *item, + return 0; + } + ++void ++mlx5_flow_os_workspace_gc_add(struct mlx5_flow_workspace *ws) ++{ ++ rte_spinlock_lock(&mlx5_flow_workspace_lock); ++ ws->gc = gc_head; ++ gc_head = ws; ++ rte_spinlock_unlock(&mlx5_flow_workspace_lock); ++} ++ ++static void ++mlx5_flow_os_workspace_gc_release(void) ++{ ++ while (gc_head) { ++ struct mlx5_flow_workspace *wks = gc_head; ++ ++ gc_head = wks->gc; ++ flow_release_workspace(wks); ++ } ++} ++ + int + mlx5_flow_os_init_workspace_once(void) + { +- if (rte_thread_key_create(&key_workspace, flow_release_workspace)) { ++ if (rte_thread_key_create(&key_workspace, NULL)) { + DRV_LOG(ERR, "Can't create flow workspace data thread key."); + rte_errno = ENOMEM; + return -rte_errno; +@@ -75,4 +99,5 @@ void + mlx5_flow_os_release_workspace(void) + { + rte_thread_key_delete(key_workspace); ++ mlx5_flow_os_workspace_gc_release(); + } +diff --git a/dpdk/drivers/net/mlx5/linux/mlx5_flow_os.h b/dpdk/drivers/net/mlx5/linux/mlx5_flow_os.h +index ed71289322..3f7a94c9ee 100644 +--- a/dpdk/drivers/net/mlx5/linux/mlx5_flow_os.h ++++ b/dpdk/drivers/net/mlx5/linux/mlx5_flow_os.h +@@ -526,4 +526,12 @@ mlx5_flow_os_validate_item_esp(const struct rte_flow_item *item, + uint8_t target_protocol, + struct rte_flow_error *error); + ++/** ++ * Add per thread workspace to the global list for garbage collection. ++ * ++ * @param[in] ws ++ * Pointer to the flow workspace. ++ */ ++void mlx5_flow_os_workspace_gc_add(struct mlx5_flow_workspace *ws); ++ + #endif /* RTE_PMD_MLX5_FLOW_OS_H_ */ +diff --git a/dpdk/drivers/net/mlx5/linux/mlx5_os.c b/dpdk/drivers/net/mlx5/linux/mlx5_os.c +index a71474c90a..28bf7211e4 100644 +--- a/dpdk/drivers/net/mlx5/linux/mlx5_os.c ++++ b/dpdk/drivers/net/mlx5/linux/mlx5_os.c +@@ -474,6 +474,10 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv) + err = mlx5_alloc_table_hash_list(priv); + if (err) + goto error; ++ sh->default_miss_action = ++ mlx5_glue->dr_create_flow_action_default_miss(); ++ if (!sh->default_miss_action) ++ DRV_LOG(WARNING, "Default miss action is not supported."); + if (priv->sh->config.dv_flow_en == 2) + return 0; + /* The resources below are only valid with DV support. */ +@@ -597,10 +601,6 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv) + + __mlx5_discovery_misc5_cap(priv); + #endif /* HAVE_MLX5DV_DR */ +- sh->default_miss_action = +- mlx5_glue->dr_create_flow_action_default_miss(); +- if (!sh->default_miss_action) +- DRV_LOG(WARNING, "Default miss action is not supported."); + LIST_INIT(&sh->shared_rxqs); + return 0; + error: +@@ -873,10 +873,10 @@ mlx5_flow_drop_action_config(struct rte_eth_dev *dev __rte_unused) + */ + if (!priv->sh->drop_action_check_flag) { + if (!mlx5_flow_discover_dr_action_support(dev)) +- priv->sh->dr_drop_action_en = 1; ++ priv->sh->dr_root_drop_action_en = 1; + priv->sh->drop_action_check_flag = 1; + } +- if (priv->sh->dr_drop_action_en) ++ if (priv->sh->dr_root_drop_action_en) + priv->root_drop_action = priv->sh->dr_drop_action; + else + priv->root_drop_action = priv->drop_queue.hrxq->action; +@@ -1613,6 +1613,23 @@ err_secondary: + err = EINVAL; + goto error; + } ++ /* ++ * If representor matching is disabled, PMD cannot create default flow rules ++ * to receive traffic for all ports, since implicit source port match is not added. ++ * Isolated mode is forced. ++ */ ++ if (priv->sh->config.dv_esw_en && !priv->sh->config.repr_matching) { ++ err = mlx5_flow_isolate(eth_dev, 1, NULL); ++ if (err < 0) { ++ err = -err; ++ goto error; ++ } ++ DRV_LOG(WARNING, "port %u ingress traffic is restricted to defined " ++ "flow rules (isolated mode) since representor " ++ "matching is disabled", ++ eth_dev->data->port_id); ++ } ++ eth_dev->data->dev_flags |= RTE_ETH_DEV_FLOW_OPS_THREAD_SAFE; + return eth_dev; + #else + DRV_LOG(ERR, "DV support is missing for HWS."); +diff --git a/dpdk/drivers/net/mlx5/mlx5.c b/dpdk/drivers/net/mlx5/mlx5.c +index e55be8720e..1a5f95b22b 100644 +--- a/dpdk/drivers/net/mlx5/mlx5.c ++++ b/dpdk/drivers/net/mlx5/mlx5.c +@@ -241,7 +241,12 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = { + .type = "mlx5_port_id_ipool", + }, + [MLX5_IPOOL_JUMP] = { +- .size = sizeof(struct mlx5_flow_tbl_data_entry), ++ /* ++ * MLX5_IPOOL_JUMP ipool entry size depends on selected flow engine. ++ * When HW steering is enabled mlx5_flow_group struct is used. ++ * Otherwise mlx5_flow_tbl_data_entry struct is used. ++ */ ++ .size = 0, + .trunk_size = 64, + .grow_trunk = 3, + .grow_shift = 2, +@@ -902,6 +907,14 @@ mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh) + sizeof(struct mlx5_flow_handle) : + MLX5_FLOW_HANDLE_VERBS_SIZE; + break; ++#if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) ++ /* Set MLX5_IPOOL_JUMP ipool entry size depending on selected flow engine. */ ++ case MLX5_IPOOL_JUMP: ++ cfg.size = sh->config.dv_flow_en == 2 ? ++ sizeof(struct mlx5_flow_group) : ++ sizeof(struct mlx5_flow_tbl_data_entry); ++ break; ++#endif + } + if (sh->config.reclaim_mode) { + cfg.release_mem_en = 1; +@@ -1401,7 +1414,8 @@ mlx5_shared_dev_ctx_args_config(struct mlx5_dev_ctx_shared *sh, + rte_errno = ENODEV; + return -rte_errno; + } +- if (!config->tx_pp && config->tx_skew) { ++ if (!config->tx_pp && config->tx_skew && ++ !sh->cdev->config.hca_attr.wait_on_time) { + DRV_LOG(WARNING, + "\"tx_skew\" doesn't affect without \"tx_pp\"."); + } +@@ -1598,6 +1612,9 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn, + /* Add context to the global device list. */ + LIST_INSERT_HEAD(&mlx5_dev_ctx_list, sh, next); + rte_spinlock_init(&sh->geneve_tlv_opt_sl); ++ /* Init counter pool list header and lock. */ ++ LIST_INIT(&sh->hws_cpool_list); ++ rte_spinlock_init(&sh->cpool_lock); + exit: + pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex); + return sh; +@@ -1976,8 +1993,12 @@ mlx5_proc_priv_init(struct rte_eth_dev *dev) + void + mlx5_proc_priv_uninit(struct rte_eth_dev *dev) + { +- if (!dev->process_private) ++ struct mlx5_proc_priv *ppriv = dev->process_private; ++ ++ if (!ppriv) + return; ++ if (ppriv->hca_bar) ++ mlx5_txpp_unmap_hca_bar(dev); + mlx5_free(dev->process_private); + dev->process_private = NULL; + } +@@ -2473,6 +2494,7 @@ mlx5_port_args_config(struct mlx5_priv *priv, struct mlx5_kvargs_ctrl *mkvlist, + config->mprq.max_memcpy_len = MLX5_MPRQ_MEMCPY_DEFAULT_LEN; + config->mprq.min_rxqs_num = MLX5_MPRQ_MIN_RXQS; + config->mprq.log_stride_num = MLX5_MPRQ_DEFAULT_LOG_STRIDE_NUM; ++ config->mprq.log_stride_size = MLX5_ARG_UNSET; + config->log_hp_size = MLX5_ARG_UNSET; + config->std_delay_drop = 0; + config->hp_delay_drop = 0; +diff --git a/dpdk/drivers/net/mlx5/mlx5.h b/dpdk/drivers/net/mlx5/mlx5.h +index 31982002ee..96a269ccd0 100644 +--- a/dpdk/drivers/net/mlx5/mlx5.h ++++ b/dpdk/drivers/net/mlx5/mlx5.h +@@ -1245,7 +1245,7 @@ struct mlx5_aso_ct_action { + /* General action object for reply dir. */ + void *dr_action_rply; + uint32_t refcnt; /* Action used count in device flows. */ +- uint16_t offset; /* Offset of ASO CT in DevX objects bulk. */ ++ uint32_t offset; /* Offset of ASO CT in DevX objects bulk. */ + uint16_t peer; /* The only peer port index could also use this CT. */ + enum mlx5_aso_ct_state state; /* ASO CT state. */ + bool is_original; /* The direction of the DR action to be used. */ +@@ -1367,7 +1367,7 @@ struct mlx5_dev_ctx_shared { + uint32_t tunnel_header_0_1:1; /* tunnel_header_0_1 is supported. */ + uint32_t tunnel_header_2_3:1; /* tunnel_header_2_3 is supported. */ + uint32_t misc5_cap:1; /* misc5 matcher parameter is supported. */ +- uint32_t dr_drop_action_en:1; /* Use DR drop action. */ ++ uint32_t dr_root_drop_action_en:1; /* DR drop action is usable on root tables. */ + uint32_t drop_action_check_flag:1; /* Check Flag for drop action. */ + uint32_t flow_priority_check_flag:1; /* Check Flag for flow priority. */ + uint32_t metadata_regc_check_flag:1; /* Check Flag for metadata REGC. */ +@@ -1455,6 +1455,8 @@ struct mlx5_dev_ctx_shared { + uint32_t host_shaper_rate:8; + uint32_t lwm_triggered:1; + struct mlx5_hws_cnt_svc_mng *cnt_svc; ++ rte_spinlock_t cpool_lock; ++ LIST_HEAD(hws_cpool_list, mlx5_hws_cnt_pool) hws_cpool_list; /* Count pool list. */ + struct mlx5_dev_shared_port port[]; /* per device port data array. */ + }; + +@@ -1463,6 +1465,8 @@ struct mlx5_dev_ctx_shared { + * Caution, secondary process may rebuild the struct during port start. + */ + struct mlx5_proc_priv { ++ void *hca_bar; ++ /* Mapped HCA PCI BAR area. */ + size_t uar_table_sz; + /* Size of UAR register table. */ + struct mlx5_uar_data uar_table[]; +@@ -1635,10 +1639,50 @@ struct mlx5_obj_ops { + + #define MLX5_RSS_HASH_FIELDS_LEN RTE_DIM(mlx5_rss_hash_fields) + ++enum mlx5_hw_ctrl_flow_type { ++ MLX5_HW_CTRL_FLOW_TYPE_GENERAL, ++ MLX5_HW_CTRL_FLOW_TYPE_SQ_MISS_ROOT, ++ MLX5_HW_CTRL_FLOW_TYPE_SQ_MISS, ++ MLX5_HW_CTRL_FLOW_TYPE_DEFAULT_JUMP, ++ MLX5_HW_CTRL_FLOW_TYPE_TX_META_COPY, ++ MLX5_HW_CTRL_FLOW_TYPE_TX_REPR_MATCH, ++ MLX5_HW_CTRL_FLOW_TYPE_LACP_RX, ++ MLX5_HW_CTRL_FLOW_TYPE_DEFAULT_RX_RSS, ++}; ++ ++/** Additional info about control flow rule. */ ++struct mlx5_hw_ctrl_flow_info { ++ /** Determines the kind of control flow rule. */ ++ enum mlx5_hw_ctrl_flow_type type; ++ union { ++ /** ++ * If control flow is a SQ miss flow (root or not), ++ * then fields contains matching SQ number. ++ */ ++ uint32_t esw_mgr_sq; ++ /** ++ * If control flow is a Tx representor matching, ++ * then fields contains matching SQ number. ++ */ ++ uint32_t tx_repr_sq; ++ }; ++}; ++ ++/** Entry for tracking control flow rules in HWS. */ + struct mlx5_hw_ctrl_flow { + LIST_ENTRY(mlx5_hw_ctrl_flow) next; ++ /** ++ * Owner device is a port on behalf of which flow rule was created. ++ * ++ * It's different from the port which really created the flow rule ++ * if and only if flow rule is created on transfer proxy port ++ * on behalf of representor port. ++ */ + struct rte_eth_dev *owner_dev; ++ /** Pointer to flow rule handle. */ + struct rte_flow *flow; ++ /** Additional information about the control flow rule. */ ++ struct mlx5_hw_ctrl_flow_info info; + }; + + struct mlx5_flow_hw_ctrl_rx; +@@ -1663,6 +1707,7 @@ struct mlx5_priv { + unsigned int mtr_en:1; /* Whether support meter. */ + unsigned int mtr_reg_share:1; /* Whether support meter REG_C share. */ + unsigned int lb_used:1; /* Loopback queue is referred to. */ ++ unsigned int rmv_notified:1; /* Notified about removal event */ + uint32_t mark_enabled:1; /* If mark action is enabled on rxqs. */ + uint16_t domain_id; /* Switch domain identifier. */ + uint16_t vport_id; /* Associated VF vport index (if any). */ +@@ -1685,10 +1730,12 @@ struct mlx5_priv { + void *root_drop_action; /* Pointer to root drop action. */ + rte_spinlock_t hw_ctrl_lock; + LIST_HEAD(hw_ctrl_flow, mlx5_hw_ctrl_flow) hw_ctrl_flows; ++ LIST_HEAD(hw_ext_ctrl_flow, mlx5_hw_ctrl_flow) hw_ext_ctrl_flows; + struct rte_flow_template_table *hw_esw_sq_miss_root_tbl; + struct rte_flow_template_table *hw_esw_sq_miss_tbl; + struct rte_flow_template_table *hw_esw_zero_tbl; + struct rte_flow_template_table *hw_tx_meta_cpy_tbl; ++ struct rte_flow_template_table *hw_lacp_rx_tbl; + struct rte_flow_pattern_template *hw_tx_repr_tagging_pt; + struct rte_flow_actions_template *hw_tx_repr_tagging_at; + struct rte_flow_template_table *hw_tx_repr_tagging_tbl; +@@ -1768,6 +1815,8 @@ struct mlx5_priv { + struct mlx5dr_action *hw_drop[2]; + /* HW steering global tag action. */ + struct mlx5dr_action *hw_tag[2]; ++ /* HW steering global default miss action. */ ++ struct mlx5dr_action *hw_def_miss; + /* HW steering create ongoing rte flow table list header. */ + LIST_HEAD(flow_hw_tbl_ongo, rte_flow_template_table) flow_hw_tbl_ongo; + struct mlx5_indexed_pool *acts_ipool; /* Action data indexed pool. */ +@@ -2163,6 +2212,8 @@ int mlx5_txpp_xstats_get_names(struct rte_eth_dev *dev, + struct rte_eth_xstat_name *xstats_names, + unsigned int n, unsigned int n_used); + void mlx5_txpp_interrupt_handler(void *cb_arg); ++int mlx5_txpp_map_hca_bar(struct rte_eth_dev *dev); ++void mlx5_txpp_unmap_hca_bar(struct rte_eth_dev *dev); + + /* mlx5_rxtx.c */ + +diff --git a/dpdk/drivers/net/mlx5/mlx5_devx.c b/dpdk/drivers/net/mlx5/mlx5_devx.c +index 02deaac612..7e0ec91328 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_devx.c ++++ b/dpdk/drivers/net/mlx5/mlx5_devx.c +@@ -1127,6 +1127,10 @@ mlx5_devx_drop_action_destroy(struct rte_eth_dev *dev) + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_hrxq *hrxq = priv->drop_queue.hrxq; + ++#if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) ++ if (hrxq->action != NULL) ++ mlx5_flow_os_destroy_flow_action(hrxq->action); ++#endif + if (hrxq->tir != NULL) + mlx5_devx_tir_destroy(hrxq); + if (hrxq->ind_table->ind_table != NULL) +diff --git a/dpdk/drivers/net/mlx5/mlx5_flow.c b/dpdk/drivers/net/mlx5/mlx5_flow.c +index a0cf677fb0..01b463adec 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_flow.c ++++ b/dpdk/drivers/net/mlx5/mlx5_flow.c +@@ -364,7 +364,7 @@ mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[], + return next; + } + +-#define MLX5_RSS_EXP_ELT_N 16 ++#define MLX5_RSS_EXP_ELT_N 32 + + /** + * Expand RSS flows into several possible flows according to the RSS hash +@@ -529,6 +529,7 @@ mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, + if (lsize > size) + return -EINVAL; + n = elt * sizeof(*item); ++ MLX5_ASSERT((buf->entries) < MLX5_RSS_EXP_ELT_N); + buf->entry[buf->entries].priority = + stack_pos + 1 + missed; + buf->entry[buf->entries].pattern = addr; +@@ -1903,8 +1904,10 @@ mlx5_flow_validate_action_mark(const struct rte_flow_action *action, + /* + * Validate the drop action. + * +- * @param[in] action_flags +- * Bit-fields that holds the actions detected until now. ++ * @param[in] dev ++ * Pointer to the Ethernet device structure. ++ * @param[in] is_root ++ * True if flow is validated for root table. False otherwise. + * @param[in] attr + * Attributes of flow that includes this action. + * @param[out] error +@@ -1914,15 +1917,25 @@ mlx5_flow_validate_action_mark(const struct rte_flow_action *action, + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ + int +-mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, ++mlx5_flow_validate_action_drop(struct rte_eth_dev *dev, ++ bool is_root, + const struct rte_flow_attr *attr, + struct rte_flow_error *error) + { +- if (attr->egress) ++ struct mlx5_priv *priv = dev->data->dev_private; ++ ++ if (priv->sh->config.dv_flow_en == 0 && attr->egress) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, + "drop action not supported for " + "egress"); ++ if (priv->sh->config.dv_flow_en == 1 && is_root && (attr->egress || attr->transfer) && ++ !priv->sh->dr_root_drop_action_en) { ++ return rte_flow_error_set(error, ENOTSUP, ++ RTE_FLOW_ERROR_TYPE_ATTR, NULL, ++ "drop action not supported for " ++ "egress and transfer on group 0"); ++ } + return 0; + } + +@@ -5878,6 +5891,7 @@ flow_check_match_action(const struct rte_flow_action actions[], + { + const struct rte_flow_action_sample *sample; + const struct rte_flow_action_raw_decap *decap; ++ const struct rte_flow_action *action_cur = NULL; + int actions_n = 0; + uint32_t ratio = 0; + int sub_type = 0; +@@ -5938,12 +5952,12 @@ flow_check_match_action(const struct rte_flow_action actions[], + break; + case RTE_FLOW_ACTION_TYPE_RAW_DECAP: + decap = actions->conf; +- while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID) ++ action_cur = actions; ++ while ((++action_cur)->type == RTE_FLOW_ACTION_TYPE_VOID) + ; +- actions_n++; +- if (actions->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { ++ if (action_cur->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { + const struct rte_flow_action_raw_encap *encap = +- actions->conf; ++ action_cur->conf; + if (decap->size <= + MLX5_ENCAPSULATION_DECISION_SIZE && + encap->size > +@@ -6125,13 +6139,14 @@ flow_sample_split_prep(struct rte_eth_dev *dev, + /* Prepare the prefix tag action. */ + append_index++; + set_tag = (void *)(actions_pre + actions_n + append_index); +- ret = mlx5_flow_get_reg_id(dev, MLX5_SAMPLE_ID, 0, error); + /* Trust VF/SF on CX5 not supported meter so that the reserved + * metadata regC is REG_NON, back to use application tag + * index 0. + */ +- if (unlikely(ret == REG_NON)) ++ if (unlikely(priv->mtr_color_reg == REG_NON)) + ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); ++ else ++ ret = mlx5_flow_get_reg_id(dev, MLX5_SAMPLE_ID, 0, error); + if (ret < 0) + return ret; + mlx5_ipool_malloc(priv->sh->ipool +@@ -6918,36 +6933,6 @@ flow_tunnel_from_rule(const struct mlx5_flow *flow) + return tunnel; + } + +-/** +- * Adjust flow RSS workspace if needed. +- * +- * @param wks +- * Pointer to thread flow work space. +- * @param rss_desc +- * Pointer to RSS descriptor. +- * @param[in] nrssq_num +- * New RSS queue number. +- * +- * @return +- * 0 on success, -1 otherwise and rte_errno is set. +- */ +-static int +-flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, +- struct mlx5_flow_rss_desc *rss_desc, +- uint32_t nrssq_num) +-{ +- if (likely(nrssq_num <= wks->rssq_num)) +- return 0; +- rss_desc->queue = realloc(rss_desc->queue, +- sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); +- if (!rss_desc->queue) { +- rte_errno = ENOMEM; +- return -1; +- } +- wks->rssq_num = RTE_ALIGN(nrssq_num, 2); +- return 0; +-} +- + /** + * Create a flow and add it to @p list. + * +@@ -7066,8 +7051,7 @@ flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, + if (attr->ingress) + rss = flow_get_rss_action(dev, p_actions_rx); + if (rss) { +- if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) +- return 0; ++ MLX5_ASSERT(rss->queue_num <= RTE_ETH_RSS_RETA_SIZE_512); + /* + * The following information is required by + * mlx5_flow_hashfields_adjust() in advance. +@@ -7555,7 +7539,6 @@ flow_release_workspace(void *data) + + while (wks) { + next = wks->next; +- free(wks->rss_desc.queue); + free(wks); + wks = next; + } +@@ -7586,23 +7569,17 @@ mlx5_flow_get_thread_workspace(void) + static struct mlx5_flow_workspace* + flow_alloc_thread_workspace(void) + { +- struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); ++ size_t data_size = RTE_ALIGN(sizeof(struct mlx5_flow_workspace), sizeof(long)); ++ size_t rss_queue_array_size = sizeof(uint16_t) * RTE_ETH_RSS_RETA_SIZE_512; ++ struct mlx5_flow_workspace *data = calloc(1, data_size + ++ rss_queue_array_size); + + if (!data) { +- DRV_LOG(ERR, "Failed to allocate flow workspace " +- "memory."); ++ DRV_LOG(ERR, "Failed to allocate flow workspace memory."); + return NULL; + } +- data->rss_desc.queue = calloc(1, +- sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); +- if (!data->rss_desc.queue) +- goto err; +- data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; ++ data->rss_desc.queue = RTE_PTR_ADD(data, data_size); + return data; +-err: +- free(data->rss_desc.queue); +- free(data); +- return NULL; + } + + /** +@@ -7623,6 +7600,7 @@ mlx5_flow_push_thread_workspace(void) + data = flow_alloc_thread_workspace(); + if (!data) + return NULL; ++ mlx5_flow_os_workspace_gc_add(data); + } else if (!curr->inuse) { + data = curr; + } else if (curr->next) { +@@ -7971,6 +7949,10 @@ mlx5_flow_isolate(struct rte_eth_dev *dev, + "port must be stopped first"); + return -rte_errno; + } ++ if (!enable && !priv->sh->config.repr_matching) ++ return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, ++ "isolated mode cannot be disabled when " ++ "representor matching is disabled"); + priv->isolated = !!enable; + if (enable) + dev->dev_ops = &mlx5_dev_ops_isolate; +@@ -9758,23 +9740,47 @@ mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev, + } + i = lcore_index; + +- for (j = 0; j <= h->mask; j++) { +- l_inconst = &h->buckets[j].l; +- if (!l_inconst || !l_inconst->cache[i]) +- continue; +- +- e = LIST_FIRST(&l_inconst->cache[i]->h); +- while (e) { +- modify_hdr = +- (struct mlx5_flow_dv_modify_hdr_resource *)e; +- data = (const uint8_t *)modify_hdr->actions; +- size = (size_t)(modify_hdr->actions_num) * 8; +- actions_num = modify_hdr->actions_num; +- id = (uint64_t)(uintptr_t)modify_hdr->action; +- type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; +- save_dump_file(data, size, type, id, +- (void *)(&actions_num), file); +- e = LIST_NEXT(e, next); ++ if (lcore_index == MLX5_LIST_NLCORE) { ++ for (i = 0; i <= (uint32_t)lcore_index; i++) { ++ for (j = 0; j <= h->mask; j++) { ++ l_inconst = &h->buckets[j].l; ++ if (!l_inconst || !l_inconst->cache[i]) ++ continue; ++ ++ e = LIST_FIRST(&l_inconst->cache[i]->h); ++ while (e) { ++ modify_hdr = ++ (struct mlx5_flow_dv_modify_hdr_resource *)e; ++ data = (const uint8_t *)modify_hdr->actions; ++ size = (size_t)(modify_hdr->actions_num) * 8; ++ actions_num = modify_hdr->actions_num; ++ id = (uint64_t)(uintptr_t)modify_hdr->action; ++ type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; ++ save_dump_file(data, size, type, id, ++ (void *)(&actions_num), file); ++ e = LIST_NEXT(e, next); ++ } ++ } ++ } ++ } else { ++ for (j = 0; j <= h->mask; j++) { ++ l_inconst = &h->buckets[j].l; ++ if (!l_inconst || !l_inconst->cache[i]) ++ continue; ++ ++ e = LIST_FIRST(&l_inconst->cache[i]->h); ++ while (e) { ++ modify_hdr = ++ (struct mlx5_flow_dv_modify_hdr_resource *)e; ++ data = (const uint8_t *)modify_hdr->actions; ++ size = (size_t)(modify_hdr->actions_num) * 8; ++ actions_num = modify_hdr->actions_num; ++ id = (uint64_t)(uintptr_t)modify_hdr->action; ++ type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; ++ save_dump_file(data, size, type, id, ++ (void *)(&actions_num), file); ++ e = LIST_NEXT(e, next); ++ } + } + } + +@@ -10104,9 +10110,19 @@ mlx5_action_handle_update(struct rte_eth_dev *dev, + const struct mlx5_flow_driver_ops *fops = + flow_get_drv_ops(flow_get_drv_type(dev, &attr)); + int ret; ++ uint32_t act_idx = (uint32_t)(uintptr_t)handle; ++ uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; + +- ret = flow_drv_action_validate(dev, NULL, +- (const struct rte_flow_action *)update, fops, error); ++ switch (type) { ++ case MLX5_INDIRECT_ACTION_TYPE_CT: ++ case MLX5_INDIRECT_ACTION_TYPE_METER_MARK: ++ ret = 0; ++ break; ++ default: ++ ret = flow_drv_action_validate(dev, NULL, ++ (const struct rte_flow_action *)update, ++ fops, error); ++ } + if (ret) + return ret; + return flow_drv_action_update(dev, handle, update, fops, +@@ -10841,7 +10857,7 @@ mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, + if (!is_tunnel_offload_active(dev)) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, +- "tunnel offload was not activated"); ++ "tunnel offload was not activated, consider setting dv_xmeta_en=3"); + if (!tunnel) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, +diff --git a/dpdk/drivers/net/mlx5/mlx5_flow.h b/dpdk/drivers/net/mlx5/mlx5_flow.h +index 1f57ecd6e1..eb87f84166 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_flow.h ++++ b/dpdk/drivers/net/mlx5/mlx5_flow.h +@@ -1437,10 +1437,10 @@ struct mlx5_flow_workspace { + /* If creating another flow in same thread, push new as stack. */ + struct mlx5_flow_workspace *prev; + struct mlx5_flow_workspace *next; ++ struct mlx5_flow_workspace *gc; + uint32_t inuse; /* can't create new flow with current. */ + struct mlx5_flow flows[MLX5_NUM_MAX_DEV_FLOWS]; + struct mlx5_flow_rss_desc rss_desc; +- uint32_t rssq_num; /* Allocated queue num in rss_desc. */ + uint32_t flow_idx; /* Intermediate device flow index. */ + struct mlx5_flow_meter_info *fm; /* Pointer to the meter in flow. */ + struct mlx5_flow_meter_policy *policy; +@@ -1926,6 +1926,7 @@ struct mlx5_flow_driver_ops { + struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void); + void mlx5_flow_pop_thread_workspace(void); + struct mlx5_flow_workspace *mlx5_flow_get_thread_workspace(void); ++ + __extension__ + struct flow_grp_info { + uint64_t external:1; +@@ -2226,7 +2227,8 @@ int mlx5_validate_action_rss(struct rte_eth_dev *dev, + int mlx5_flow_validate_action_count(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + struct rte_flow_error *error); +-int mlx5_flow_validate_action_drop(uint64_t action_flags, ++int mlx5_flow_validate_action_drop(struct rte_eth_dev *dev, ++ bool is_root, + const struct rte_flow_attr *attr, + struct rte_flow_error *error); + int mlx5_flow_validate_action_flag(uint64_t action_flags, +@@ -2579,10 +2581,13 @@ int mlx5_flow_pick_transfer_proxy(struct rte_eth_dev *dev, + int mlx5_flow_hw_flush_ctrl_flows(struct rte_eth_dev *dev); + + int mlx5_flow_hw_esw_create_sq_miss_flow(struct rte_eth_dev *dev, +- uint32_t sqn); ++ uint32_t sqn, bool external); ++int mlx5_flow_hw_esw_destroy_sq_miss_flow(struct rte_eth_dev *dev, ++ uint32_t sqn); + int mlx5_flow_hw_esw_create_default_jump_flow(struct rte_eth_dev *dev); + int mlx5_flow_hw_create_tx_default_mreg_copy_flow(struct rte_eth_dev *dev); +-int mlx5_flow_hw_tx_repr_matching_flow(struct rte_eth_dev *dev, uint32_t sqn); ++int mlx5_flow_hw_tx_repr_matching_flow(struct rte_eth_dev *dev, uint32_t sqn, bool external); ++int mlx5_flow_hw_lacp_rx_flow(struct rte_eth_dev *dev); + int mlx5_flow_actions_validate(struct rte_eth_dev *dev, + const struct rte_flow_actions_template_attr *attr, + const struct rte_flow_action actions[], +diff --git a/dpdk/drivers/net/mlx5/mlx5_flow_aso.c b/dpdk/drivers/net/mlx5/mlx5_flow_aso.c +index 29bd7ce9e8..8441be3dea 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_flow_aso.c ++++ b/dpdk/drivers/net/mlx5/mlx5_flow_aso.c +@@ -932,7 +932,8 @@ mlx5_aso_mtr_completion_handle(struct mlx5_aso_sq *sq, bool need_lock) + rte_spinlock_lock(&sq->sqsl); + max = (uint16_t)(sq->head - sq->tail); + if (unlikely(!max)) { +- rte_spinlock_unlock(&sq->sqsl); ++ if (need_lock) ++ rte_spinlock_unlock(&sq->sqsl); + return; + } + do { +diff --git a/dpdk/drivers/net/mlx5/mlx5_flow_dv.c b/dpdk/drivers/net/mlx5/mlx5_flow_dv.c +index 62c38b87a1..5c2af50fe5 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_flow_dv.c ++++ b/dpdk/drivers/net/mlx5/mlx5_flow_dv.c +@@ -2129,6 +2129,8 @@ flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused, + * Pointer to the rte_eth_dev structure. + * @param[in] item + * Item specification. ++ * @param[in] tag_bitmap ++ * Tag index bitmap. + * @param[in] attr + * Attributes of flow that includes this item. + * @param[out] error +@@ -2140,6 +2142,7 @@ flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused, + static int + flow_dv_validate_item_tag(struct rte_eth_dev *dev, + const struct rte_flow_item *item, ++ uint32_t *tag_bitmap, + const struct rte_flow_attr *attr __rte_unused, + struct rte_flow_error *error) + { +@@ -2183,6 +2186,12 @@ flow_dv_validate_item_tag(struct rte_eth_dev *dev, + if (ret < 0) + return ret; + MLX5_ASSERT(ret != REG_NON); ++ if (*tag_bitmap & (1 << ret)) ++ return rte_flow_error_set(error, EINVAL, ++ RTE_FLOW_ERROR_TYPE_ITEM_SPEC, ++ item->spec, ++ "Duplicated tag index"); ++ *tag_bitmap |= 1 << ret; + return 0; + } + +@@ -4445,6 +4454,7 @@ flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error) + { + struct rte_ether_hdr *eth = NULL; + struct rte_vlan_hdr *vlan = NULL; ++ struct rte_ipv4_hdr *ipv4 = NULL; + struct rte_ipv6_hdr *ipv6 = NULL; + struct rte_udp_hdr *udp = NULL; + char *next_hdr; +@@ -4461,24 +4471,27 @@ flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error) + next_hdr += sizeof(struct rte_vlan_hdr); + } + +- /* HW calculates IPv4 csum. no need to proceed */ +- if (proto == RTE_ETHER_TYPE_IPV4) +- return 0; +- + /* non IPv4/IPv6 header. not supported */ +- if (proto != RTE_ETHER_TYPE_IPV6) { ++ if (proto != RTE_ETHER_TYPE_IPV4 && proto != RTE_ETHER_TYPE_IPV6) { + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "Cannot offload non IPv4/IPv6"); + } + +- ipv6 = (struct rte_ipv6_hdr *)next_hdr; +- +- /* ignore non UDP */ +- if (ipv6->proto != IPPROTO_UDP) +- return 0; ++ if (proto == RTE_ETHER_TYPE_IPV4) { ++ ipv4 = (struct rte_ipv4_hdr *)next_hdr; ++ /* ignore non UDP */ ++ if (ipv4->next_proto_id != IPPROTO_UDP) ++ return 0; ++ udp = (struct rte_udp_hdr *)(ipv4 + 1); ++ } else { ++ ipv6 = (struct rte_ipv6_hdr *)next_hdr; ++ /* ignore non UDP */ ++ if (ipv6->proto != IPPROTO_UDP) ++ return 0; ++ udp = (struct rte_udp_hdr *)(ipv6 + 1); ++ } + +- udp = (struct rte_udp_hdr *)(ipv6 + 1); + udp->dgram_cksum = 0; + + return 0; +@@ -5750,6 +5763,7 @@ flow_dv_modify_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) + */ + static int + flow_dv_validate_action_sample(uint64_t *action_flags, ++ uint64_t *sub_action_flags, + const struct rte_flow_action *action, + struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, +@@ -5758,14 +5772,15 @@ flow_dv_validate_action_sample(uint64_t *action_flags, + const struct rte_flow_action_rss **sample_rss, + const struct rte_flow_action_count **count, + int *fdb_mirror, ++ uint16_t *sample_port_id, + bool root, + struct rte_flow_error *error) + { + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_sh_config *dev_conf = &priv->sh->config; + const struct rte_flow_action_sample *sample = action->conf; ++ const struct rte_flow_action_port_id *port = NULL; + const struct rte_flow_action *act; +- uint64_t sub_action_flags = 0; + uint16_t queue_index = 0xFFFF; + int actions_n = 0; + int ret; +@@ -5812,20 +5827,20 @@ flow_dv_validate_action_sample(uint64_t *action_flags, + switch (act->type) { + case RTE_FLOW_ACTION_TYPE_QUEUE: + ret = mlx5_flow_validate_action_queue(act, +- sub_action_flags, ++ *sub_action_flags, + dev, + attr, error); + if (ret < 0) + return ret; + queue_index = ((const struct rte_flow_action_queue *) + (act->conf))->index; +- sub_action_flags |= MLX5_FLOW_ACTION_QUEUE; ++ *sub_action_flags |= MLX5_FLOW_ACTION_QUEUE; + ++actions_n; + break; + case RTE_FLOW_ACTION_TYPE_RSS: + *sample_rss = act->conf; + ret = mlx5_flow_validate_action_rss(act, +- sub_action_flags, ++ *sub_action_flags, + dev, attr, + item_flags, + error); +@@ -5841,48 +5856,57 @@ flow_dv_validate_action_sample(uint64_t *action_flags, + "or level in the same flow"); + if (*sample_rss != NULL && (*sample_rss)->queue_num) + queue_index = (*sample_rss)->queue[0]; +- sub_action_flags |= MLX5_FLOW_ACTION_RSS; ++ *sub_action_flags |= MLX5_FLOW_ACTION_RSS; + ++actions_n; + break; + case RTE_FLOW_ACTION_TYPE_MARK: + ret = flow_dv_validate_action_mark(dev, act, +- sub_action_flags, ++ *sub_action_flags, + attr, error); + if (ret < 0) + return ret; + if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) +- sub_action_flags |= MLX5_FLOW_ACTION_MARK | ++ *sub_action_flags |= MLX5_FLOW_ACTION_MARK | + MLX5_FLOW_ACTION_MARK_EXT; + else +- sub_action_flags |= MLX5_FLOW_ACTION_MARK; ++ *sub_action_flags |= MLX5_FLOW_ACTION_MARK; + ++actions_n; + break; + case RTE_FLOW_ACTION_TYPE_COUNT: + ret = flow_dv_validate_action_count +- (dev, false, *action_flags | sub_action_flags, ++ (dev, false, *action_flags | *sub_action_flags, + root, error); + if (ret < 0) + return ret; + *count = act->conf; +- sub_action_flags |= MLX5_FLOW_ACTION_COUNT; ++ *sub_action_flags |= MLX5_FLOW_ACTION_COUNT; + *action_flags |= MLX5_FLOW_ACTION_COUNT; + ++actions_n; + break; + case RTE_FLOW_ACTION_TYPE_PORT_ID: + case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT: + ret = flow_dv_validate_action_port_id(dev, +- sub_action_flags, ++ *sub_action_flags, + act, + attr, + error); + if (ret) + return ret; +- sub_action_flags |= MLX5_FLOW_ACTION_PORT_ID; ++ if (act->type == RTE_FLOW_ACTION_TYPE_PORT_ID) { ++ port = (const struct rte_flow_action_port_id *) ++ act->conf; ++ *sample_port_id = port->original ? ++ dev->data->port_id : port->id; ++ } else { ++ *sample_port_id = ((const struct rte_flow_action_ethdev *) ++ act->conf)->port_id; ++ } ++ *sub_action_flags |= MLX5_FLOW_ACTION_PORT_ID; + ++actions_n; + break; + case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: + ret = flow_dv_validate_action_raw_encap_decap +- (dev, NULL, act->conf, attr, &sub_action_flags, ++ (dev, NULL, act->conf, attr, sub_action_flags, + &actions_n, action, item_flags, error); + if (ret < 0) + return ret; +@@ -5891,12 +5915,12 @@ flow_dv_validate_action_sample(uint64_t *action_flags, + case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: + case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: + ret = flow_dv_validate_action_l2_encap(dev, +- sub_action_flags, ++ *sub_action_flags, + act, attr, + error); + if (ret < 0) + return ret; +- sub_action_flags |= MLX5_FLOW_ACTION_ENCAP; ++ *sub_action_flags |= MLX5_FLOW_ACTION_ENCAP; + ++actions_n; + break; + default: +@@ -5908,7 +5932,7 @@ flow_dv_validate_action_sample(uint64_t *action_flags, + } + } + if (attr->ingress) { +- if (!(sub_action_flags & (MLX5_FLOW_ACTION_QUEUE | ++ if (!(*sub_action_flags & (MLX5_FLOW_ACTION_QUEUE | + MLX5_FLOW_ACTION_RSS))) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, +@@ -5930,17 +5954,17 @@ flow_dv_validate_action_sample(uint64_t *action_flags, + "E-Switch doesn't support " + "any optional action " + "for sampling"); +- if (sub_action_flags & MLX5_FLOW_ACTION_QUEUE) ++ if (*sub_action_flags & MLX5_FLOW_ACTION_QUEUE) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "unsupported action QUEUE"); +- if (sub_action_flags & MLX5_FLOW_ACTION_RSS) ++ if (*sub_action_flags & MLX5_FLOW_ACTION_RSS) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "unsupported action QUEUE"); +- if (!(sub_action_flags & MLX5_FLOW_ACTION_PORT_ID)) ++ if (!(*sub_action_flags & MLX5_FLOW_ACTION_PORT_ID)) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, +@@ -5949,16 +5973,16 @@ flow_dv_validate_action_sample(uint64_t *action_flags, + *fdb_mirror = 1; + } + /* Continue validation for Xcap actions.*/ +- if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) && ++ if ((*sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) && + (queue_index == 0xFFFF || !mlx5_rxq_is_hairpin(dev, queue_index))) { +- if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) == ++ if ((*sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) == + MLX5_FLOW_XCAP_ACTIONS) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "encap and decap " + "combination aren't " + "supported"); +- if (attr->ingress && (sub_action_flags & MLX5_FLOW_ACTION_ENCAP)) ++ if (attr->ingress && (*sub_action_flags & MLX5_FLOW_ACTION_ENCAP)) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "encap is not supported" +@@ -7051,11 +7075,17 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, + bool def_policy = false; + bool shared_count = false; + uint16_t udp_dport = 0; +- uint32_t tag_id = 0; ++ uint32_t tag_id = 0, tag_bitmap = 0; + const struct rte_flow_action_age *non_shared_age = NULL; + const struct rte_flow_action_count *count = NULL; ++ const struct rte_flow_action_port_id *port = NULL; ++ const struct mlx5_rte_flow_item_tag *mlx5_tag; + struct mlx5_priv *act_priv = NULL; + int aso_after_sample = 0; ++ struct mlx5_priv *port_priv = NULL; ++ uint64_t sub_action_flags = 0; ++ uint16_t sample_port_id = 0; ++ uint16_t port_id = 0; + + if (items == NULL) + return -1; +@@ -7371,7 +7401,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, + last_item = MLX5_FLOW_LAYER_ICMP6; + break; + case RTE_FLOW_ITEM_TYPE_TAG: +- ret = flow_dv_validate_item_tag(dev, items, ++ ret = flow_dv_validate_item_tag(dev, items, &tag_bitmap, + attr, error); + if (ret < 0) + return ret; +@@ -7381,6 +7411,13 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, + last_item = MLX5_FLOW_ITEM_SQ; + break; + case MLX5_RTE_FLOW_ITEM_TYPE_TAG: ++ mlx5_tag = (const struct mlx5_rte_flow_item_tag *)items->spec; ++ if (tag_bitmap & (1 << mlx5_tag->id)) ++ return rte_flow_error_set(error, EINVAL, ++ RTE_FLOW_ERROR_TYPE_ITEM_SPEC, ++ items->spec, ++ "Duplicated tag index"); ++ tag_bitmap |= 1 << mlx5_tag->id; + break; + case RTE_FLOW_ITEM_TYPE_GTP: + ret = flow_dv_validate_item_gtp(dev, items, item_flags, +@@ -7486,6 +7523,14 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, + error); + if (ret) + return ret; ++ if (type == RTE_FLOW_ACTION_TYPE_PORT_ID) { ++ port = (const struct rte_flow_action_port_id *) ++ actions->conf; ++ port_id = port->original ? dev->data->port_id : port->id; ++ } else { ++ port_id = ((const struct rte_flow_action_ethdev *) ++ actions->conf)->port_id; ++ } + action_flags |= MLX5_FLOW_ACTION_PORT_ID; + ++actions_n; + break; +@@ -7562,7 +7607,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, + rw_act_num += MLX5_ACT_NUM_SET_TAG; + break; + case RTE_FLOW_ACTION_TYPE_DROP: +- ret = mlx5_flow_validate_action_drop(action_flags, ++ ret = mlx5_flow_validate_action_drop(dev, is_root, + attr, error); + if (ret < 0) + return ret; +@@ -7985,11 +8030,13 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, + break; + case RTE_FLOW_ACTION_TYPE_SAMPLE: + ret = flow_dv_validate_action_sample(&action_flags, ++ &sub_action_flags, + actions, dev, + attr, item_flags, + rss, &sample_rss, + &sample_count, + &fdb_mirror, ++ &sample_port_id, + is_root, + error); + if (ret < 0) +@@ -8301,6 +8348,29 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "sample before ASO action is not supported"); ++ if (sub_action_flags & MLX5_FLOW_ACTION_PORT_ID) { ++ port_priv = mlx5_port_to_eswitch_info(sample_port_id, false); ++ if (flow_source_vport_representor(priv, port_priv)) { ++ if (sub_action_flags & MLX5_FLOW_ACTION_ENCAP) ++ return rte_flow_error_set(error, ENOTSUP, ++ RTE_FLOW_ERROR_TYPE_ACTION, NULL, ++ "mirror to rep port with encap is not supported"); ++ } else { ++ if (!(sub_action_flags & MLX5_FLOW_ACTION_ENCAP) && ++ (action_flags & MLX5_FLOW_ACTION_JUMP)) ++ return rte_flow_error_set(error, ENOTSUP, ++ RTE_FLOW_ERROR_TYPE_ACTION, NULL, ++ "mirror to wire port without encap is not supported"); ++ } ++ } ++ if ((action_flags & MLX5_FLOW_ACTION_PORT_ID) && ++ (action_flags & MLX5_FLOW_ACTION_ENCAP)) { ++ port_priv = mlx5_port_to_eswitch_info(port_id, false); ++ if (flow_source_vport_representor(priv, port_priv)) ++ return rte_flow_error_set(error, ENOTSUP, ++ RTE_FLOW_ERROR_TYPE_ACTION, NULL, ++ "mirror to rep port with encap is not supported"); ++ } + } + /* + * Validation the NIC Egress flow on representor, except implicit +@@ -9223,12 +9293,10 @@ flow_dv_translate_item_vxlan(struct rte_eth_dev *dev, + { + const struct rte_flow_item_vxlan *vxlan_m; + const struct rte_flow_item_vxlan *vxlan_v; +- const struct rte_flow_item_vxlan *vxlan_vv = item->spec; + void *headers_v; + void *misc_v; + void *misc5_v; + uint32_t tunnel_v; +- uint32_t *tunnel_header_v; + char *vni_v; + uint16_t dport; + int size; +@@ -9280,24 +9348,11 @@ flow_dv_translate_item_vxlan(struct rte_eth_dev *dev, + vni_v[i] = vxlan_m->vni[i] & vxlan_v->vni[i]; + return; + } +- tunnel_header_v = (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc5, +- misc5_v, +- tunnel_header_1); + tunnel_v = (vxlan_v->vni[0] & vxlan_m->vni[0]) | + (vxlan_v->vni[1] & vxlan_m->vni[1]) << 8 | + (vxlan_v->vni[2] & vxlan_m->vni[2]) << 16; +- *tunnel_header_v = tunnel_v; +- if (key_type == MLX5_SET_MATCHER_SW_M) { +- tunnel_v = (vxlan_vv->vni[0] & vxlan_m->vni[0]) | +- (vxlan_vv->vni[1] & vxlan_m->vni[1]) << 8 | +- (vxlan_vv->vni[2] & vxlan_m->vni[2]) << 16; +- if (!tunnel_v) +- *tunnel_header_v = 0x0; +- if (vxlan_vv->rsvd1 & vxlan_m->rsvd1) +- *tunnel_header_v |= vxlan_v->rsvd1 << 24; +- } else { +- *tunnel_header_v |= (vxlan_v->rsvd1 & vxlan_m->rsvd1) << 24; +- } ++ tunnel_v |= (vxlan_v->rsvd1 & vxlan_m->rsvd1) << 24; ++ MLX5_SET(fte_match_set_misc5, misc5_v, tunnel_header_1, RTE_BE32(tunnel_v)); + } + + /** +@@ -13717,7 +13772,12 @@ flow_dv_translate_items_sws(struct rte_eth_dev *dev, + * is the suffix flow. + */ + dev_flow->handle->layers |= wks.item_flags; +- dev_flow->flow->geneve_tlv_option = wks.geneve_tlv_option; ++ /* ++ * Update geneve_tlv_option flag only it is set in workspace. ++ * Avoid be overwritten by other sub mlx5_flows. ++ */ ++ if (wks.geneve_tlv_option) ++ dev_flow->flow->geneve_tlv_option = wks.geneve_tlv_option; + return 0; + } + +@@ -14820,7 +14880,7 @@ flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, + } + dv->actions[n++] = priv->sh->default_miss_action; + } +- misc_mask = flow_dv_matcher_enable(dv->value.buf); ++ misc_mask = flow_dv_matcher_enable(dv_h->matcher->mask.buf); + __flow_dv_adjust_buf_size(&dv->value.size, misc_mask); + err = mlx5_flow_os_create_flow(dv_h->matcher->matcher_object, + (void *)&dv->value, n, +@@ -17020,7 +17080,7 @@ flow_dv_destroy_def_policy(struct rte_eth_dev *dev) + static int + __flow_dv_create_policy_flow(struct rte_eth_dev *dev, + uint32_t color_reg_c_idx, +- enum rte_color color, void *matcher_object, ++ enum rte_color color, struct mlx5_flow_dv_matcher *matcher, + int actions_n, void *actions, + bool match_src_port, const struct rte_flow_item *item, + void **rule, const struct rte_flow_attr *attr) +@@ -17050,9 +17110,9 @@ __flow_dv_create_policy_flow(struct rte_eth_dev *dev, + } + flow_dv_match_meta_reg(value.buf, (enum modify_reg)color_reg_c_idx, + rte_col_2_mlx5_col(color), UINT32_MAX); +- misc_mask = flow_dv_matcher_enable(value.buf); ++ misc_mask = flow_dv_matcher_enable(matcher->mask.buf); + __flow_dv_adjust_buf_size(&value.size, misc_mask); +- ret = mlx5_flow_os_create_flow(matcher_object, (void *)&value, ++ ret = mlx5_flow_os_create_flow(matcher->matcher_object, (void *)&value, + actions_n, actions, rule); + if (ret) { + DRV_LOG(ERR, "Failed to create meter policy%d flow.", color); +@@ -17206,7 +17266,7 @@ __flow_dv_create_domain_policy_rules(struct rte_eth_dev *dev, + /* Create flow, matching color. */ + if (__flow_dv_create_policy_flow(dev, + color_reg_c_idx, (enum rte_color)i, +- color_rule->matcher->matcher_object, ++ color_rule->matcher, + acts[i].actions_n, acts[i].dv_actions, + svport_match, NULL, &color_rule->rule, + &attr)) { +@@ -17674,7 +17734,7 @@ flow_dv_create_mtr_tbls(struct rte_eth_dev *dev, + actions[i++] = priv->sh->dr_drop_action; + flow_dv_match_meta_reg_all(matcher_para.buf, value.buf, + (enum modify_reg)mtr_id_reg_c, 0, 0); +- misc_mask = flow_dv_matcher_enable(value.buf); ++ misc_mask = flow_dv_matcher_enable(mtrmng->def_matcher[domain]->mask.buf); + __flow_dv_adjust_buf_size(&value.size, misc_mask); + ret = mlx5_flow_os_create_flow + (mtrmng->def_matcher[domain]->matcher_object, +@@ -17719,7 +17779,7 @@ flow_dv_create_mtr_tbls(struct rte_eth_dev *dev, + fm->drop_cnt, NULL); + actions[i++] = cnt->action; + actions[i++] = priv->sh->dr_drop_action; +- misc_mask = flow_dv_matcher_enable(value.buf); ++ misc_mask = flow_dv_matcher_enable(drop_matcher->mask.buf); + __flow_dv_adjust_buf_size(&value.size, misc_mask); + ret = mlx5_flow_os_create_flow(drop_matcher->matcher_object, + (void *)&value, i, actions, +@@ -18199,7 +18259,7 @@ flow_dv_meter_hierarchy_rule_create(struct rte_eth_dev *dev, + goto err_exit; + } + if (__flow_dv_create_policy_flow(dev, color_reg_c_idx, (enum rte_color)j, +- color_rule->matcher->matcher_object, ++ color_rule->matcher, + acts.actions_n, acts.dv_actions, + true, item, &color_rule->rule, &attr)) { + rte_spinlock_unlock(&mtr_policy->sl); +@@ -18909,7 +18969,7 @@ flow_dv_validate_mtr_policy_acts(struct rte_eth_dev *dev, + break; + case RTE_FLOW_ACTION_TYPE_DROP: + ret = mlx5_flow_validate_action_drop +- (action_flags[i], attr, &flow_err); ++ (dev, false, attr, &flow_err); + if (ret < 0) + return -rte_mtr_error_set(error, + ENOTSUP, +@@ -19243,7 +19303,7 @@ flow_dv_discover_priorities(struct rte_eth_dev *dev, + break; + } + /* Try to apply the flow to HW. */ +- misc_mask = flow_dv_matcher_enable(flow.dv.value.buf); ++ misc_mask = flow_dv_matcher_enable(flow.handle->dvh.matcher->mask.buf); + __flow_dv_adjust_buf_size(&flow.dv.value.size, misc_mask); + err = mlx5_flow_os_create_flow + (flow.handle->dvh.matcher->matcher_object, +diff --git a/dpdk/drivers/net/mlx5/mlx5_flow_hw.c b/dpdk/drivers/net/mlx5/mlx5_flow_hw.c +index a3c8056515..6b889e9f81 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_flow_hw.c ++++ b/dpdk/drivers/net/mlx5/mlx5_flow_hw.c +@@ -1243,6 +1243,8 @@ flow_hw_meter_mark_alloc(struct rte_eth_dev *dev, uint32_t queue, + struct mlx5_flow_meter_info *fm; + uint32_t mtr_id; + ++ if (meter_mark->profile == NULL) ++ return NULL; + aso_mtr = mlx5_ipool_malloc(priv->hws_mpool->idx_pool, &mtr_id); + if (!aso_mtr) + return NULL; +@@ -1361,7 +1363,7 @@ __flow_hw_actions_translate(struct rte_eth_dev *dev, + else + type = MLX5DR_TABLE_TYPE_NIC_RX; + for (; !actions_end; actions++, masks++) { +- switch (actions->type) { ++ switch ((int)actions->type) { + case RTE_FLOW_ACTION_TYPE_INDIRECT: + action_pos = at->actions_off[actions - at->actions]; + if (!attr->group) { +@@ -1665,6 +1667,16 @@ __flow_hw_actions_translate(struct rte_eth_dev *dev, + action_pos)) + goto err; + break; ++ case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS: ++ /* Internal, can be skipped. */ ++ if (!!attr->group) { ++ DRV_LOG(ERR, "DEFAULT MISS action is only" ++ " supported in root table."); ++ goto err; ++ } ++ action_pos = at->actions_off[actions - at->actions]; ++ acts->rule_acts[action_pos].action = priv->hw_def_miss; ++ break; + case RTE_FLOW_ACTION_TYPE_END: + actions_end = true; + break; +@@ -3252,14 +3264,18 @@ flow_hw_translate_group(struct rte_eth_dev *dev, + "group index not supported"); + *table_group = group + 1; + } else if (config->dv_esw_en && +- !(config->repr_matching && config->dv_xmeta_en == MLX5_XMETA_MODE_META32_HWS) && ++ (config->repr_matching || config->dv_xmeta_en == MLX5_XMETA_MODE_META32_HWS) && + cfg->external && + flow_attr->egress) { + /* +- * On E-Switch setups, egress group translation is not done if and only if +- * representor matching is disabled and legacy metadata mode is selected. +- * In all other cases, egree group 0 is reserved for representor tagging flows +- * and metadata copy flows. ++ * On E-Switch setups, default egress flow rules are inserted to allow ++ * representor matching and/or preserving metadata across steering domains. ++ * These flow rules are inserted in group 0 and this group is reserved by PMD ++ * for these purposes. ++ * ++ * As a result, if representor matching or extended metadata mode is enabled, ++ * group provided by the user must be incremented to avoid inserting flow rules ++ * in group 0. + */ + if (group > MLX5_HW_MAX_EGRESS_GROUP) + return rte_flow_error_set(error, EINVAL, +@@ -3863,6 +3879,34 @@ flow_hw_validate_action_push_vlan(struct rte_eth_dev *dev, + #undef X_FIELD + } + ++static int ++flow_hw_validate_action_default_miss(struct rte_eth_dev *dev, ++ const struct rte_flow_actions_template_attr *attr, ++ uint64_t action_flags, ++ struct rte_flow_error *error) ++{ ++ /* ++ * The private DEFAULT_MISS action is used internally for LACP in control ++ * flows. So this validation can be ignored. It can be kept right now since ++ * the validation will be done only once. ++ */ ++ struct mlx5_priv *priv = dev->data->dev_private; ++ ++ if (!attr->ingress || attr->egress || attr->transfer) ++ return rte_flow_error_set(error, EINVAL, ++ RTE_FLOW_ERROR_TYPE_ACTION, NULL, ++ "DEFAULT MISS is only supported in ingress."); ++ if (!priv->hw_def_miss) ++ return rte_flow_error_set(error, EINVAL, ++ RTE_FLOW_ERROR_TYPE_ACTION, NULL, ++ "DEFAULT MISS action does not exist."); ++ if (action_flags & MLX5_FLOW_FATE_ACTIONS) ++ return rte_flow_error_set(error, EINVAL, ++ RTE_FLOW_ERROR_TYPE_ACTION, NULL, ++ "DEFAULT MISS should be the only termination."); ++ return 0; ++} ++ + static int + mlx5_flow_hw_actions_validate(struct rte_eth_dev *dev, + const struct rte_flow_actions_template_attr *attr, +@@ -3896,7 +3940,7 @@ mlx5_flow_hw_actions_validate(struct rte_eth_dev *dev, + RTE_FLOW_ERROR_TYPE_ACTION, + action, + "mask type does not match action type"); +- switch (action->type) { ++ switch ((int)action->type) { + case RTE_FLOW_ACTION_TYPE_VOID: + break; + case RTE_FLOW_ACTION_TYPE_INDIRECT: +@@ -4022,6 +4066,13 @@ mlx5_flow_hw_actions_validate(struct rte_eth_dev *dev, + case RTE_FLOW_ACTION_TYPE_END: + actions_end = true; + break; ++ case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS: ++ ret = flow_hw_validate_action_default_miss(dev, attr, ++ action_flags, error); ++ if (ret < 0) ++ return ret; ++ action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS; ++ break; + default: + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, +@@ -4041,8 +4092,7 @@ flow_hw_actions_validate(struct rte_eth_dev *dev, + const struct rte_flow_action masks[], + struct rte_flow_error *error) + { +- return mlx5_flow_hw_actions_validate(dev, attr, actions, masks, NULL, +- error); ++ return mlx5_flow_hw_actions_validate(dev, attr, actions, masks, NULL, error); + } + + +@@ -4143,7 +4193,7 @@ flow_hw_dr_actions_template_create(struct rte_flow_actions_template *at) + + if (curr_off >= MLX5_HW_MAX_ACTS) + goto err_actions_num; +- switch (at->actions[i].type) { ++ switch ((int)at->actions[i].type) { + case RTE_FLOW_ACTION_TYPE_VOID: + break; + case RTE_FLOW_ACTION_TYPE_INDIRECT: +@@ -4221,6 +4271,10 @@ flow_hw_dr_actions_template_create(struct rte_flow_actions_template *at) + } + at->actions_off[i] = cnt_off; + break; ++ case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS: ++ at->actions_off[i] = curr_off; ++ action_types[curr_off++] = MLX5DR_ACTION_TYP_MISS; ++ break; + default: + type = mlx5_hw_dr_action_types[at->actions[i].type]; + at->actions_off[i] = curr_off; +@@ -4534,6 +4588,9 @@ error: + mlx5dr_action_template_destroy(at->tmpl); + mlx5_free(at); + } ++ rte_flow_error_set(error, rte_errno, ++ RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, ++ "Failed to create action template"); + return NULL; + } + +@@ -4614,8 +4671,9 @@ flow_hw_pattern_validate(struct rte_eth_dev *dev, + struct rte_flow_error *error) + { + struct mlx5_priv *priv = dev->data->dev_private; +- int i; ++ int i, tag_idx; + bool items_end = false; ++ uint32_t tag_bitmap = 0; + + if (!attr->ingress && !attr->egress && !attr->transfer) + return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ATTR, NULL, +@@ -4657,16 +4715,26 @@ flow_hw_pattern_validate(struct rte_eth_dev *dev, + switch (type) { + case RTE_FLOW_ITEM_TYPE_TAG: + { +- int reg; + const struct rte_flow_item_tag *tag = + (const struct rte_flow_item_tag *)items[i].spec; + +- reg = flow_hw_get_reg_id(RTE_FLOW_ITEM_TYPE_TAG, tag->index); +- if (reg == REG_NON) ++ if (tag == NULL) ++ return rte_flow_error_set(error, EINVAL, ++ RTE_FLOW_ERROR_TYPE_UNSPECIFIED, ++ NULL, ++ "Tag spec is NULL"); ++ tag_idx = flow_hw_get_reg_id(RTE_FLOW_ITEM_TYPE_TAG, tag->index); ++ if (tag_idx == REG_NON) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "Unsupported tag index"); ++ if (tag_bitmap & (1 << tag_idx)) ++ return rte_flow_error_set(error, EINVAL, ++ RTE_FLOW_ERROR_TYPE_ITEM, ++ NULL, ++ "Duplicated tag index"); ++ tag_bitmap |= 1 << tag_idx; + break; + } + case MLX5_RTE_FLOW_ITEM_TYPE_TAG: +@@ -4680,6 +4748,12 @@ flow_hw_pattern_validate(struct rte_eth_dev *dev, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "Unsupported internal tag index"); ++ if (tag_bitmap & (1 << tag->index)) ++ return rte_flow_error_set(error, EINVAL, ++ RTE_FLOW_ERROR_TYPE_ITEM, ++ NULL, ++ "Duplicated tag index"); ++ tag_bitmap |= 1 << tag->index; + break; + } + case RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT: +@@ -4790,7 +4864,7 @@ flow_hw_pattern_template_create(struct rte_eth_dev *dev, + struct rte_flow_pattern_template *it; + struct rte_flow_item *copied_items = NULL; + const struct rte_flow_item *tmpl_items; +- uint64_t orig_item_nb; ++ uint32_t orig_item_nb; + struct rte_flow_item port = { + .type = RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT, + .mask = &rte_flow_item_ethdev_mask, +@@ -5271,12 +5345,14 @@ flow_hw_free_vport_actions(struct mlx5_priv *priv) + * + * @param dev + * Pointer to Ethernet device. ++ * @param[out] error ++ * Pointer to error structure. + * + * @return + * Pointer to pattern template on success. NULL otherwise, and rte_errno is set. + */ + static struct rte_flow_pattern_template * +-flow_hw_create_tx_repr_sq_pattern_tmpl(struct rte_eth_dev *dev) ++flow_hw_create_tx_repr_sq_pattern_tmpl(struct rte_eth_dev *dev, struct rte_flow_error *error) + { + struct rte_flow_pattern_template_attr attr = { + .relaxed_matching = 0, +@@ -5295,7 +5371,7 @@ flow_hw_create_tx_repr_sq_pattern_tmpl(struct rte_eth_dev *dev) + }, + }; + +- return flow_hw_pattern_template_create(dev, &attr, items, NULL); ++ return flow_hw_pattern_template_create(dev, &attr, items, error); + } + + static __rte_always_inline uint32_t +@@ -5353,12 +5429,15 @@ flow_hw_update_action_mask(struct rte_flow_action *action, + * + * @param dev + * Pointer to Ethernet device. ++ * @param[out] error ++ * Pointer to error structure. + * + * @return + * Pointer to actions template on success. NULL otherwise, and rte_errno is set. + */ + static struct rte_flow_actions_template * +-flow_hw_create_tx_repr_tag_jump_acts_tmpl(struct rte_eth_dev *dev) ++flow_hw_create_tx_repr_tag_jump_acts_tmpl(struct rte_eth_dev *dev, ++ struct rte_flow_error *error) + { + uint32_t tag_mask = flow_hw_tx_tag_regc_mask(dev); + uint32_t tag_value = flow_hw_tx_tag_regc_value(dev); +@@ -5444,7 +5523,7 @@ flow_hw_create_tx_repr_tag_jump_acts_tmpl(struct rte_eth_dev *dev) + NULL, NULL); + idx++; + MLX5_ASSERT(idx <= RTE_DIM(actions_v)); +- return flow_hw_actions_template_create(dev, &attr, actions_v, actions_m, NULL); ++ return flow_hw_actions_template_create(dev, &attr, actions_v, actions_m, error); + } + + static void +@@ -5473,12 +5552,14 @@ flow_hw_cleanup_tx_repr_tagging(struct rte_eth_dev *dev) + * + * @param dev + * Pointer to Ethernet device. ++ * @param[out] error ++ * Pointer to error structure. + * + * @return + * 0 on success, negative errno value otherwise. + */ + static int +-flow_hw_setup_tx_repr_tagging(struct rte_eth_dev *dev) ++flow_hw_setup_tx_repr_tagging(struct rte_eth_dev *dev, struct rte_flow_error *error) + { + struct mlx5_priv *priv = dev->data->dev_private; + struct rte_flow_template_table_attr attr = { +@@ -5496,20 +5577,22 @@ flow_hw_setup_tx_repr_tagging(struct rte_eth_dev *dev) + + MLX5_ASSERT(priv->sh->config.dv_esw_en); + MLX5_ASSERT(priv->sh->config.repr_matching); +- priv->hw_tx_repr_tagging_pt = flow_hw_create_tx_repr_sq_pattern_tmpl(dev); ++ priv->hw_tx_repr_tagging_pt = ++ flow_hw_create_tx_repr_sq_pattern_tmpl(dev, error); + if (!priv->hw_tx_repr_tagging_pt) +- goto error; +- priv->hw_tx_repr_tagging_at = flow_hw_create_tx_repr_tag_jump_acts_tmpl(dev); ++ goto err; ++ priv->hw_tx_repr_tagging_at = ++ flow_hw_create_tx_repr_tag_jump_acts_tmpl(dev, error); + if (!priv->hw_tx_repr_tagging_at) +- goto error; ++ goto err; + priv->hw_tx_repr_tagging_tbl = flow_hw_table_create(dev, &cfg, + &priv->hw_tx_repr_tagging_pt, 1, + &priv->hw_tx_repr_tagging_at, 1, +- NULL); ++ error); + if (!priv->hw_tx_repr_tagging_tbl) +- goto error; ++ goto err; + return 0; +-error: ++err: + flow_hw_cleanup_tx_repr_tagging(dev); + return -rte_errno; + } +@@ -5540,12 +5623,15 @@ flow_hw_esw_mgr_regc_marker(struct rte_eth_dev *dev) + * + * @param dev + * Pointer to Ethernet device. ++ * @param error ++ * Pointer to error structure. + * + * @return + * Pointer to flow pattern template on success, NULL otherwise. + */ + static struct rte_flow_pattern_template * +-flow_hw_create_ctrl_esw_mgr_pattern_template(struct rte_eth_dev *dev) ++flow_hw_create_ctrl_esw_mgr_pattern_template(struct rte_eth_dev *dev, ++ struct rte_flow_error *error) + { + struct rte_flow_pattern_template_attr attr = { + .relaxed_matching = 0, +@@ -5575,7 +5661,7 @@ flow_hw_create_ctrl_esw_mgr_pattern_template(struct rte_eth_dev *dev) + }, + }; + +- return flow_hw_pattern_template_create(dev, &attr, items, NULL); ++ return flow_hw_pattern_template_create(dev, &attr, items, error); + } + + /** +@@ -5588,12 +5674,15 @@ flow_hw_create_ctrl_esw_mgr_pattern_template(struct rte_eth_dev *dev) + * + * @param dev + * Pointer to Ethernet device. ++ * @param error ++ * Pointer to error structure. + * + * @return + * Pointer to flow pattern template on success, NULL otherwise. + */ + static struct rte_flow_pattern_template * +-flow_hw_create_ctrl_regc_sq_pattern_template(struct rte_eth_dev *dev) ++flow_hw_create_ctrl_regc_sq_pattern_template(struct rte_eth_dev *dev, ++ struct rte_flow_error *error) + { + struct rte_flow_pattern_template_attr attr = { + .relaxed_matching = 0, +@@ -5626,7 +5715,7 @@ flow_hw_create_ctrl_regc_sq_pattern_template(struct rte_eth_dev *dev) + }, + }; + +- return flow_hw_pattern_template_create(dev, &attr, items, NULL); ++ return flow_hw_pattern_template_create(dev, &attr, items, error); + } + + /** +@@ -5636,12 +5725,15 @@ flow_hw_create_ctrl_regc_sq_pattern_template(struct rte_eth_dev *dev) + * + * @param dev + * Pointer to Ethernet device. ++ * @param error ++ * Pointer to error structure. + * + * @return + * Pointer to flow pattern template on success, NULL otherwise. + */ + static struct rte_flow_pattern_template * +-flow_hw_create_ctrl_port_pattern_template(struct rte_eth_dev *dev) ++flow_hw_create_ctrl_port_pattern_template(struct rte_eth_dev *dev, ++ struct rte_flow_error *error) + { + struct rte_flow_pattern_template_attr attr = { + .relaxed_matching = 0, +@@ -5660,7 +5752,7 @@ flow_hw_create_ctrl_port_pattern_template(struct rte_eth_dev *dev) + }, + }; + +- return flow_hw_pattern_template_create(dev, &attr, items, NULL); ++ return flow_hw_pattern_template_create(dev, &attr, items, error); + } + + /* +@@ -5670,12 +5762,15 @@ flow_hw_create_ctrl_port_pattern_template(struct rte_eth_dev *dev) + * + * @param dev + * Pointer to Ethernet device. ++ * @param error ++ * Pointer to error structure. + * + * @return + * Pointer to flow pattern template on success, NULL otherwise. + */ + static struct rte_flow_pattern_template * +-flow_hw_create_tx_default_mreg_copy_pattern_template(struct rte_eth_dev *dev) ++flow_hw_create_tx_default_mreg_copy_pattern_template(struct rte_eth_dev *dev, ++ struct rte_flow_error *error) + { + struct rte_flow_pattern_template_attr tx_pa_attr = { + .relaxed_matching = 0, +@@ -5696,10 +5791,44 @@ flow_hw_create_tx_default_mreg_copy_pattern_template(struct rte_eth_dev *dev) + .type = RTE_FLOW_ITEM_TYPE_END, + }, + }; +- struct rte_flow_error drop_err; + +- RTE_SET_USED(drop_err); +- return flow_hw_pattern_template_create(dev, &tx_pa_attr, eth_all, &drop_err); ++ return flow_hw_pattern_template_create(dev, &tx_pa_attr, eth_all, error); ++} ++ ++/* ++ * Creating a flow pattern template with all LACP packets matching, only for NIC ++ * ingress domain. ++ * ++ * @param dev ++ * Pointer to Ethernet device. ++ * @param error ++ * Pointer to error structure. ++ * ++ * @return ++ * Pointer to flow pattern template on success, NULL otherwise. ++ */ ++static struct rte_flow_pattern_template * ++flow_hw_create_lacp_rx_pattern_template(struct rte_eth_dev *dev, struct rte_flow_error *error) ++{ ++ struct rte_flow_pattern_template_attr pa_attr = { ++ .relaxed_matching = 0, ++ .ingress = 1, ++ }; ++ struct rte_flow_item_eth lacp_mask = { ++ .dst.addr_bytes = "\x00\x00\x00\x00\x00\x00", ++ .src.addr_bytes = "\x00\x00\x00\x00\x00\x00", ++ .type = 0xFFFF, ++ }; ++ struct rte_flow_item eth_all[] = { ++ [0] = { ++ .type = RTE_FLOW_ITEM_TYPE_ETH, ++ .mask = &lacp_mask, ++ }, ++ [1] = { ++ .type = RTE_FLOW_ITEM_TYPE_END, ++ }, ++ }; ++ return flow_hw_pattern_template_create(dev, &pa_attr, eth_all, error); + } + + /** +@@ -5710,12 +5839,15 @@ flow_hw_create_tx_default_mreg_copy_pattern_template(struct rte_eth_dev *dev) + * + * @param dev + * Pointer to Ethernet device. ++ * @param error ++ * Pointer to error structure. + * + * @return + * Pointer to flow actions template on success, NULL otherwise. + */ + static struct rte_flow_actions_template * +-flow_hw_create_ctrl_regc_jump_actions_template(struct rte_eth_dev *dev) ++flow_hw_create_ctrl_regc_jump_actions_template(struct rte_eth_dev *dev, ++ struct rte_flow_error *error) + { + uint32_t marker_mask = flow_hw_esw_mgr_regc_marker_mask(dev); + uint32_t marker_bits = flow_hw_esw_mgr_regc_marker(dev); +@@ -5781,7 +5913,7 @@ flow_hw_create_ctrl_regc_jump_actions_template(struct rte_eth_dev *dev) + set_reg_v.dst.offset = rte_bsf32(marker_mask); + rte_memcpy(set_reg_v.src.value, &marker_bits, sizeof(marker_bits)); + rte_memcpy(set_reg_m.src.value, &marker_mask, sizeof(marker_mask)); +- return flow_hw_actions_template_create(dev, &attr, actions_v, actions_m, NULL); ++ return flow_hw_actions_template_create(dev, &attr, actions_v, actions_m, error); + } + + /** +@@ -5793,13 +5925,16 @@ flow_hw_create_ctrl_regc_jump_actions_template(struct rte_eth_dev *dev) + * Pointer to Ethernet device. + * @param group + * Destination group for this action template. ++ * @param error ++ * Pointer to error structure. + * + * @return + * Pointer to flow actions template on success, NULL otherwise. + */ + static struct rte_flow_actions_template * + flow_hw_create_ctrl_jump_actions_template(struct rte_eth_dev *dev, +- uint32_t group) ++ uint32_t group, ++ struct rte_flow_error *error) + { + struct rte_flow_actions_template_attr attr = { + .transfer = 1, +@@ -5829,8 +5964,8 @@ flow_hw_create_ctrl_jump_actions_template(struct rte_eth_dev *dev, + } + }; + +- return flow_hw_actions_template_create(dev, &attr, actions_v, actions_m, +- NULL); ++ return flow_hw_actions_template_create(dev, &attr, actions_v, ++ actions_m, error); + } + + /** +@@ -5839,12 +5974,15 @@ flow_hw_create_ctrl_jump_actions_template(struct rte_eth_dev *dev, + * + * @param dev + * Pointer to Ethernet device. ++ * @param error ++ * Pointer to error structure. + * + * @return + * Pointer to flow action template on success, NULL otherwise. + */ + static struct rte_flow_actions_template * +-flow_hw_create_ctrl_port_actions_template(struct rte_eth_dev *dev) ++flow_hw_create_ctrl_port_actions_template(struct rte_eth_dev *dev, ++ struct rte_flow_error *error) + { + struct rte_flow_actions_template_attr attr = { + .transfer = 1, +@@ -5874,8 +6012,7 @@ flow_hw_create_ctrl_port_actions_template(struct rte_eth_dev *dev) + } + }; + +- return flow_hw_actions_template_create(dev, &attr, actions_v, actions_m, +- NULL); ++ return flow_hw_actions_template_create(dev, &attr, actions_v, actions_m, error); + } + + /* +@@ -5884,12 +6021,15 @@ flow_hw_create_ctrl_port_actions_template(struct rte_eth_dev *dev) + * + * @param dev + * Pointer to Ethernet device. ++ * @param error ++ * Pointer to error structure. + * + * @return + * Pointer to flow actions template on success, NULL otherwise. + */ + static struct rte_flow_actions_template * +-flow_hw_create_tx_default_mreg_copy_actions_template(struct rte_eth_dev *dev) ++flow_hw_create_tx_default_mreg_copy_actions_template(struct rte_eth_dev *dev, ++ struct rte_flow_error *error) + { + struct rte_flow_actions_template_attr tx_act_attr = { + .egress = 1, +@@ -5952,11 +6092,41 @@ flow_hw_create_tx_default_mreg_copy_actions_template(struct rte_eth_dev *dev) + .type = RTE_FLOW_ACTION_TYPE_END, + }, + }; +- struct rte_flow_error drop_err; + +- RTE_SET_USED(drop_err); + return flow_hw_actions_template_create(dev, &tx_act_attr, actions, +- masks, &drop_err); ++ masks, error); ++} ++ ++/* ++ * Creating an actions template to use default miss to re-route packets to the ++ * kernel driver stack. ++ * On root table, only DEFAULT_MISS action can be used. ++ * ++ * @param dev ++ * Pointer to Ethernet device. ++ * @param error ++ * Pointer to error structure. ++ * ++ * @return ++ * Pointer to flow actions template on success, NULL otherwise. ++ */ ++static struct rte_flow_actions_template * ++flow_hw_create_lacp_rx_actions_template(struct rte_eth_dev *dev, struct rte_flow_error *error) ++{ ++ struct rte_flow_actions_template_attr act_attr = { ++ .ingress = 1, ++ }; ++ const struct rte_flow_action actions[] = { ++ [0] = { ++ .type = (enum rte_flow_action_type) ++ MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, ++ }, ++ [1] = { ++ .type = RTE_FLOW_ACTION_TYPE_END, ++ }, ++ }; ++ ++ return flow_hw_actions_template_create(dev, &act_attr, actions, actions, error); + } + + /** +@@ -5969,6 +6139,8 @@ flow_hw_create_tx_default_mreg_copy_actions_template(struct rte_eth_dev *dev) + * Pointer to flow pattern template. + * @param at + * Pointer to flow actions template. ++ * @param error ++ * Pointer to error structure. + * + * @return + * Pointer to flow table on success, NULL otherwise. +@@ -5976,7 +6148,8 @@ flow_hw_create_tx_default_mreg_copy_actions_template(struct rte_eth_dev *dev) + static struct rte_flow_template_table* + flow_hw_create_ctrl_sq_miss_root_table(struct rte_eth_dev *dev, + struct rte_flow_pattern_template *it, +- struct rte_flow_actions_template *at) ++ struct rte_flow_actions_template *at, ++ struct rte_flow_error *error) + { + struct rte_flow_template_table_attr attr = { + .flow_attr = { +@@ -5993,7 +6166,7 @@ flow_hw_create_ctrl_sq_miss_root_table(struct rte_eth_dev *dev, + .external = false, + }; + +- return flow_hw_table_create(dev, &cfg, &it, 1, &at, 1, NULL); ++ return flow_hw_table_create(dev, &cfg, &it, 1, &at, 1, error); + } + + +@@ -6007,6 +6180,8 @@ flow_hw_create_ctrl_sq_miss_root_table(struct rte_eth_dev *dev, + * Pointer to flow pattern template. + * @param at + * Pointer to flow actions template. ++ * @param error ++ * Pointer to error structure. + * + * @return + * Pointer to flow table on success, NULL otherwise. +@@ -6014,7 +6189,8 @@ flow_hw_create_ctrl_sq_miss_root_table(struct rte_eth_dev *dev, + static struct rte_flow_template_table* + flow_hw_create_ctrl_sq_miss_table(struct rte_eth_dev *dev, + struct rte_flow_pattern_template *it, +- struct rte_flow_actions_template *at) ++ struct rte_flow_actions_template *at, ++ struct rte_flow_error *error) + { + struct rte_flow_template_table_attr attr = { + .flow_attr = { +@@ -6031,7 +6207,7 @@ flow_hw_create_ctrl_sq_miss_table(struct rte_eth_dev *dev, + .external = false, + }; + +- return flow_hw_table_create(dev, &cfg, &it, 1, &at, 1, NULL); ++ return flow_hw_table_create(dev, &cfg, &it, 1, &at, 1, error); + } + + /* +@@ -6043,6 +6219,8 @@ flow_hw_create_ctrl_sq_miss_table(struct rte_eth_dev *dev, + * Pointer to flow pattern template. + * @param at + * Pointer to flow actions template. ++ * @param error ++ * Pointer to error structure. + * + * @return + * Pointer to flow table on success, NULL otherwise. +@@ -6050,7 +6228,8 @@ flow_hw_create_ctrl_sq_miss_table(struct rte_eth_dev *dev, + static struct rte_flow_template_table* + flow_hw_create_tx_default_mreg_copy_table(struct rte_eth_dev *dev, + struct rte_flow_pattern_template *pt, +- struct rte_flow_actions_template *at) ++ struct rte_flow_actions_template *at, ++ struct rte_flow_error *error) + { + struct rte_flow_template_table_attr tx_tbl_attr = { + .flow_attr = { +@@ -6064,14 +6243,8 @@ flow_hw_create_tx_default_mreg_copy_table(struct rte_eth_dev *dev, + .attr = tx_tbl_attr, + .external = false, + }; +- struct rte_flow_error drop_err = { +- .type = RTE_FLOW_ERROR_TYPE_NONE, +- .cause = NULL, +- .message = NULL, +- }; + +- RTE_SET_USED(drop_err); +- return flow_hw_table_create(dev, &tx_tbl_cfg, &pt, 1, &at, 1, &drop_err); ++ return flow_hw_table_create(dev, &tx_tbl_cfg, &pt, 1, &at, 1, error); + } + + /** +@@ -6084,6 +6257,8 @@ flow_hw_create_tx_default_mreg_copy_table(struct rte_eth_dev *dev, + * Pointer to flow pattern template. + * @param at + * Pointer to flow actions template. ++ * @param error ++ * Pointer to error structure. + * + * @return + * Pointer to flow table on success, NULL otherwise. +@@ -6091,7 +6266,8 @@ flow_hw_create_tx_default_mreg_copy_table(struct rte_eth_dev *dev, + static struct rte_flow_template_table * + flow_hw_create_ctrl_jump_table(struct rte_eth_dev *dev, + struct rte_flow_pattern_template *it, +- struct rte_flow_actions_template *at) ++ struct rte_flow_actions_template *at, ++ struct rte_flow_error *error) + { + struct rte_flow_template_table_attr attr = { + .flow_attr = { +@@ -6108,7 +6284,44 @@ flow_hw_create_ctrl_jump_table(struct rte_eth_dev *dev, + .external = false, + }; + +- return flow_hw_table_create(dev, &cfg, &it, 1, &at, 1, NULL); ++ return flow_hw_table_create(dev, &cfg, &it, 1, &at, 1, error); ++} ++ ++/* ++ * Create a table on the root group to for the LACP traffic redirecting. ++ * ++ * @param dev ++ * Pointer to Ethernet device. ++ * @param it ++ * Pointer to flow pattern template. ++ * @param at ++ * Pointer to flow actions template. ++ * ++ * @return ++ * Pointer to flow table on success, NULL otherwise. ++ */ ++static struct rte_flow_template_table * ++flow_hw_create_lacp_rx_table(struct rte_eth_dev *dev, ++ struct rte_flow_pattern_template *it, ++ struct rte_flow_actions_template *at, ++ struct rte_flow_error *error) ++{ ++ struct rte_flow_template_table_attr attr = { ++ .flow_attr = { ++ .group = 0, ++ .priority = 0, ++ .ingress = 1, ++ .egress = 0, ++ .transfer = 0, ++ }, ++ .nb_flows = 1, ++ }; ++ struct mlx5_flow_template_table_cfg cfg = { ++ .attr = attr, ++ .external = false, ++ }; ++ ++ return flow_hw_table_create(dev, &cfg, &it, 1, &at, 1, error); + } + + /** +@@ -6117,114 +6330,155 @@ flow_hw_create_ctrl_jump_table(struct rte_eth_dev *dev, + * + * @param dev + * Pointer to Ethernet device. ++ * @param error ++ * Pointer to error structure. + * + * @return +- * 0 on success, EINVAL otherwise ++ * 0 on success, negative values otherwise + */ + static __rte_unused int +-flow_hw_create_ctrl_tables(struct rte_eth_dev *dev) ++flow_hw_create_ctrl_tables(struct rte_eth_dev *dev, struct rte_flow_error *error) + { + struct mlx5_priv *priv = dev->data->dev_private; + struct rte_flow_pattern_template *esw_mgr_items_tmpl = NULL; + struct rte_flow_pattern_template *regc_sq_items_tmpl = NULL; + struct rte_flow_pattern_template *port_items_tmpl = NULL; + struct rte_flow_pattern_template *tx_meta_items_tmpl = NULL; ++ struct rte_flow_pattern_template *lacp_rx_items_tmpl = NULL; + struct rte_flow_actions_template *regc_jump_actions_tmpl = NULL; + struct rte_flow_actions_template *port_actions_tmpl = NULL; + struct rte_flow_actions_template *jump_one_actions_tmpl = NULL; + struct rte_flow_actions_template *tx_meta_actions_tmpl = NULL; ++ struct rte_flow_actions_template *lacp_rx_actions_tmpl = NULL; + uint32_t xmeta = priv->sh->config.dv_xmeta_en; + uint32_t repr_matching = priv->sh->config.repr_matching; ++ int ret; + + /* Create templates and table for default SQ miss flow rules - root table. */ +- esw_mgr_items_tmpl = flow_hw_create_ctrl_esw_mgr_pattern_template(dev); ++ esw_mgr_items_tmpl = flow_hw_create_ctrl_esw_mgr_pattern_template(dev, error); + if (!esw_mgr_items_tmpl) { + DRV_LOG(ERR, "port %u failed to create E-Switch Manager item" + " template for control flows", dev->data->port_id); +- goto error; ++ goto err; + } +- regc_jump_actions_tmpl = flow_hw_create_ctrl_regc_jump_actions_template(dev); ++ regc_jump_actions_tmpl = flow_hw_create_ctrl_regc_jump_actions_template(dev, error); + if (!regc_jump_actions_tmpl) { + DRV_LOG(ERR, "port %u failed to create REG_C set and jump action template" + " for control flows", dev->data->port_id); +- goto error; ++ goto err; + } + MLX5_ASSERT(priv->hw_esw_sq_miss_root_tbl == NULL); + priv->hw_esw_sq_miss_root_tbl = flow_hw_create_ctrl_sq_miss_root_table +- (dev, esw_mgr_items_tmpl, regc_jump_actions_tmpl); ++ (dev, esw_mgr_items_tmpl, regc_jump_actions_tmpl, error); + if (!priv->hw_esw_sq_miss_root_tbl) { + DRV_LOG(ERR, "port %u failed to create table for default sq miss (root table)" + " for control flows", dev->data->port_id); +- goto error; ++ goto err; + } + /* Create templates and table for default SQ miss flow rules - non-root table. */ +- regc_sq_items_tmpl = flow_hw_create_ctrl_regc_sq_pattern_template(dev); ++ regc_sq_items_tmpl = flow_hw_create_ctrl_regc_sq_pattern_template(dev, error); + if (!regc_sq_items_tmpl) { + DRV_LOG(ERR, "port %u failed to create SQ item template for" + " control flows", dev->data->port_id); +- goto error; ++ goto err; + } +- port_actions_tmpl = flow_hw_create_ctrl_port_actions_template(dev); ++ port_actions_tmpl = flow_hw_create_ctrl_port_actions_template(dev, error); + if (!port_actions_tmpl) { + DRV_LOG(ERR, "port %u failed to create port action template" + " for control flows", dev->data->port_id); +- goto error; ++ goto err; + } + MLX5_ASSERT(priv->hw_esw_sq_miss_tbl == NULL); + priv->hw_esw_sq_miss_tbl = flow_hw_create_ctrl_sq_miss_table(dev, regc_sq_items_tmpl, +- port_actions_tmpl); ++ port_actions_tmpl, error); + if (!priv->hw_esw_sq_miss_tbl) { + DRV_LOG(ERR, "port %u failed to create table for default sq miss (non-root table)" + " for control flows", dev->data->port_id); +- goto error; ++ goto err; + } + /* Create templates and table for default FDB jump flow rules. */ +- port_items_tmpl = flow_hw_create_ctrl_port_pattern_template(dev); ++ port_items_tmpl = flow_hw_create_ctrl_port_pattern_template(dev, error); + if (!port_items_tmpl) { + DRV_LOG(ERR, "port %u failed to create SQ item template for" + " control flows", dev->data->port_id); +- goto error; ++ goto err; + } + jump_one_actions_tmpl = flow_hw_create_ctrl_jump_actions_template +- (dev, MLX5_HW_LOWEST_USABLE_GROUP); ++ (dev, MLX5_HW_LOWEST_USABLE_GROUP, error); + if (!jump_one_actions_tmpl) { + DRV_LOG(ERR, "port %u failed to create jump action template" + " for control flows", dev->data->port_id); +- goto error; ++ goto err; + } + MLX5_ASSERT(priv->hw_esw_zero_tbl == NULL); + priv->hw_esw_zero_tbl = flow_hw_create_ctrl_jump_table(dev, port_items_tmpl, +- jump_one_actions_tmpl); ++ jump_one_actions_tmpl, ++ error); + if (!priv->hw_esw_zero_tbl) { + DRV_LOG(ERR, "port %u failed to create table for default jump to group 1" + " for control flows", dev->data->port_id); +- goto error; ++ goto err; + } + /* Create templates and table for default Tx metadata copy flow rule. */ + if (!repr_matching && xmeta == MLX5_XMETA_MODE_META32_HWS) { +- tx_meta_items_tmpl = flow_hw_create_tx_default_mreg_copy_pattern_template(dev); ++ tx_meta_items_tmpl = ++ flow_hw_create_tx_default_mreg_copy_pattern_template(dev, error); + if (!tx_meta_items_tmpl) { + DRV_LOG(ERR, "port %u failed to Tx metadata copy pattern" + " template for control flows", dev->data->port_id); +- goto error; ++ goto err; + } +- tx_meta_actions_tmpl = flow_hw_create_tx_default_mreg_copy_actions_template(dev); ++ tx_meta_actions_tmpl = ++ flow_hw_create_tx_default_mreg_copy_actions_template(dev, error); + if (!tx_meta_actions_tmpl) { + DRV_LOG(ERR, "port %u failed to Tx metadata copy actions" + " template for control flows", dev->data->port_id); +- goto error; ++ goto err; + } + MLX5_ASSERT(priv->hw_tx_meta_cpy_tbl == NULL); +- priv->hw_tx_meta_cpy_tbl = flow_hw_create_tx_default_mreg_copy_table(dev, +- tx_meta_items_tmpl, tx_meta_actions_tmpl); ++ priv->hw_tx_meta_cpy_tbl = ++ flow_hw_create_tx_default_mreg_copy_table(dev, tx_meta_items_tmpl, ++ tx_meta_actions_tmpl, error); + if (!priv->hw_tx_meta_cpy_tbl) { + DRV_LOG(ERR, "port %u failed to create table for default" + " Tx metadata copy flow rule", dev->data->port_id); +- goto error; ++ goto err; ++ } ++ } ++ /* Create LACP default miss table. */ ++ if (!priv->sh->config.lacp_by_user && priv->pf_bond >= 0) { ++ lacp_rx_items_tmpl = flow_hw_create_lacp_rx_pattern_template(dev, error); ++ if (!lacp_rx_items_tmpl) { ++ DRV_LOG(ERR, "port %u failed to create pattern template" ++ " for LACP Rx traffic", dev->data->port_id); ++ goto err; ++ } ++ lacp_rx_actions_tmpl = flow_hw_create_lacp_rx_actions_template(dev, error); ++ if (!lacp_rx_actions_tmpl) { ++ DRV_LOG(ERR, "port %u failed to create actions template" ++ " for LACP Rx traffic", dev->data->port_id); ++ goto err; ++ } ++ priv->hw_lacp_rx_tbl = flow_hw_create_lacp_rx_table(dev, lacp_rx_items_tmpl, ++ lacp_rx_actions_tmpl, error); ++ if (!priv->hw_lacp_rx_tbl) { ++ DRV_LOG(ERR, "port %u failed to create template table for" ++ " for LACP Rx traffic", dev->data->port_id); ++ goto err; + } + } + return 0; +-error: ++err: ++ /* Do not overwrite the rte_errno. */ ++ ret = -rte_errno; ++ if (ret == 0) ++ ret = rte_flow_error_set(error, EINVAL, ++ RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, ++ "Failed to create control tables."); ++ if (priv->hw_tx_meta_cpy_tbl) { ++ flow_hw_table_destroy(dev, priv->hw_tx_meta_cpy_tbl, NULL); ++ priv->hw_tx_meta_cpy_tbl = NULL; ++ } + if (priv->hw_esw_zero_tbl) { + flow_hw_table_destroy(dev, priv->hw_esw_zero_tbl, NULL); + priv->hw_esw_zero_tbl = NULL; +@@ -6237,6 +6491,8 @@ error: + flow_hw_table_destroy(dev, priv->hw_esw_sq_miss_root_tbl, NULL); + priv->hw_esw_sq_miss_root_tbl = NULL; + } ++ if (lacp_rx_actions_tmpl) ++ flow_hw_actions_template_destroy(dev, lacp_rx_actions_tmpl, NULL); + if (tx_meta_actions_tmpl) + flow_hw_actions_template_destroy(dev, tx_meta_actions_tmpl, NULL); + if (jump_one_actions_tmpl) +@@ -6245,6 +6501,8 @@ error: + flow_hw_actions_template_destroy(dev, port_actions_tmpl, NULL); + if (regc_jump_actions_tmpl) + flow_hw_actions_template_destroy(dev, regc_jump_actions_tmpl, NULL); ++ if (lacp_rx_items_tmpl) ++ flow_hw_pattern_template_destroy(dev, lacp_rx_items_tmpl, NULL); + if (tx_meta_items_tmpl) + flow_hw_pattern_template_destroy(dev, tx_meta_items_tmpl, NULL); + if (port_items_tmpl) +@@ -6253,7 +6511,7 @@ error: + flow_hw_pattern_template_destroy(dev, regc_sq_items_tmpl, NULL); + if (esw_mgr_items_tmpl) + flow_hw_pattern_template_destroy(dev, esw_mgr_items_tmpl, NULL); +- return -EINVAL; ++ return ret; + } + + static void +@@ -6376,27 +6634,28 @@ flow_hw_create_vlan(struct rte_eth_dev *dev) + MLX5DR_ACTION_FLAG_HWS_FDB + }; + ++ /* rte_errno is set in the mlx5dr_action* functions. */ + for (i = MLX5DR_TABLE_TYPE_NIC_RX; i <= MLX5DR_TABLE_TYPE_NIC_TX; i++) { + priv->hw_pop_vlan[i] = + mlx5dr_action_create_pop_vlan(priv->dr_ctx, flags[i]); + if (!priv->hw_pop_vlan[i]) +- return -ENOENT; ++ return -rte_errno; + priv->hw_push_vlan[i] = + mlx5dr_action_create_push_vlan(priv->dr_ctx, flags[i]); + if (!priv->hw_pop_vlan[i]) +- return -ENOENT; ++ return -rte_errno; + } + if (priv->sh->config.dv_esw_en && priv->master) { + priv->hw_pop_vlan[MLX5DR_TABLE_TYPE_FDB] = + mlx5dr_action_create_pop_vlan + (priv->dr_ctx, MLX5DR_ACTION_FLAG_HWS_FDB); + if (!priv->hw_pop_vlan[MLX5DR_TABLE_TYPE_FDB]) +- return -ENOENT; ++ return -rte_errno; + priv->hw_push_vlan[MLX5DR_TABLE_TYPE_FDB] = + mlx5dr_action_create_push_vlan + (priv->dr_ctx, MLX5DR_ACTION_FLAG_HWS_FDB); + if (!priv->hw_pop_vlan[MLX5DR_TABLE_TYPE_FDB]) +- return -ENOENT; ++ return -rte_errno; + } + return 0; + } +@@ -6807,6 +7066,7 @@ flow_hw_configure(struct rte_eth_dev *dev, + struct rte_flow_queue_attr ctrl_queue_attr = {0}; + bool is_proxy = !!(priv->sh->config.dv_esw_en && priv->master); + int ret = 0; ++ uint32_t action_flags; + + if (!port_attr || !nb_queue || !queue_attr) { + rte_errno = EINVAL; +@@ -6836,8 +7096,7 @@ flow_hw_configure(struct rte_eth_dev *dev, + goto err; + } + +- memcpy(_queue_attr, queue_attr, +- sizeof(void *) * nb_queue); ++ memcpy(_queue_attr, queue_attr, sizeof(void *) * nb_queue); + _queue_attr[nb_queue] = &ctrl_queue_attr; + priv->acts_ipool = mlx5_ipool_create(&cfg); + if (!priv->acts_ipool) +@@ -6926,6 +7185,7 @@ flow_hw_configure(struct rte_eth_dev *dev, + priv->nb_queue = nb_q_updated; + rte_spinlock_init(&priv->hw_ctrl_lock); + LIST_INIT(&priv->hw_ctrl_flows); ++ LIST_INIT(&priv->hw_ext_ctrl_flows); + ret = flow_hw_create_ctrl_rx_tables(dev); + if (ret) { + rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, +@@ -6952,23 +7212,34 @@ flow_hw_configure(struct rte_eth_dev *dev, + goto err; + } + if (priv->sh->config.dv_esw_en && priv->sh->config.repr_matching) { +- ret = flow_hw_setup_tx_repr_tagging(dev); +- if (ret) { +- rte_errno = -ret; ++ ret = flow_hw_setup_tx_repr_tagging(dev, error); ++ if (ret) + goto err; +- } + } ++ /* ++ * DEFAULT_MISS action have different behaviors in different domains. ++ * In FDB, it will steering the packets to the E-switch manager. ++ * In NIC Rx root, it will steering the packet to the kernel driver stack. ++ * An action with all bits set in the flag can be created and the HWS ++ * layer will translate it properly when being used in different rules. ++ */ ++ action_flags = MLX5DR_ACTION_FLAG_ROOT_RX | MLX5DR_ACTION_FLAG_HWS_RX | ++ MLX5DR_ACTION_FLAG_ROOT_TX | MLX5DR_ACTION_FLAG_HWS_TX; ++ if (is_proxy) ++ action_flags |= (MLX5DR_ACTION_FLAG_ROOT_FDB | MLX5DR_ACTION_FLAG_HWS_FDB); ++ priv->hw_def_miss = mlx5dr_action_create_default_miss(priv->dr_ctx, action_flags); ++ if (!priv->hw_def_miss) ++ goto err; + if (is_proxy) { + ret = flow_hw_create_vport_actions(priv); + if (ret) { +- rte_errno = -ret; ++ rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, ++ NULL, "Failed to create vport actions."); + goto err; + } +- ret = flow_hw_create_ctrl_tables(dev); +- if (ret) { +- rte_errno = -ret; ++ ret = flow_hw_create_ctrl_tables(dev, error); ++ if (ret) + goto err; +- } + } + if (port_attr->nb_conn_tracks) { + mem_size = sizeof(struct mlx5_aso_sq) * nb_q_updated + +@@ -7005,12 +7276,18 @@ flow_hw_configure(struct rte_eth_dev *dev, + goto err; + } + ret = mlx5_hws_age_pool_init(dev, port_attr, nb_queue); +- if (ret < 0) ++ if (ret < 0) { ++ rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, ++ NULL, "Failed to init age pool."); + goto err; ++ } + } + ret = flow_hw_create_vlan(dev); +- if (ret) ++ if (ret) { ++ rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, ++ NULL, "Failed to VLAN actions."); + goto err; ++ } + if (_queue_attr) + mlx5_free(_queue_attr); + if (port_attr->flags & RTE_FLOW_PORT_FLAG_STRICT_QUEUE) +@@ -7178,9 +7455,9 @@ void flow_hw_init_tags_set(struct rte_eth_dev *dev) + uint32_t meta_mode = priv->sh->config.dv_xmeta_en; + uint8_t masks = (uint8_t)priv->sh->cdev->config.hca_attr.set_reg_c; + uint32_t i, j; +- enum modify_reg copy[MLX5_FLOW_HW_TAGS_MAX] = {REG_NON}; ++ uint8_t reg_off; + uint8_t unset = 0; +- uint8_t copy_masks = 0; ++ uint8_t common_masks = 0; + + /* + * The CAPA is global for common device but only used in net. +@@ -7195,29 +7472,35 @@ void flow_hw_init_tags_set(struct rte_eth_dev *dev) + if (meta_mode == MLX5_XMETA_MODE_META32_HWS) + unset |= 1 << (REG_C_1 - REG_C_0); + masks &= ~unset; ++ /* ++ * If available tag registers were previously calculated, ++ * calculate a bitmask with an intersection of sets of: ++ * - registers supported by current port, ++ * - previously calculated available tag registers. ++ */ + if (mlx5_flow_hw_avl_tags_init_cnt) { + MLX5_ASSERT(mlx5_flow_hw_aso_tag == priv->mtr_color_reg); + for (i = 0; i < MLX5_FLOW_HW_TAGS_MAX; i++) { +- if (mlx5_flow_hw_avl_tags[i] != REG_NON && !!((1 << i) & masks)) { +- copy[mlx5_flow_hw_avl_tags[i] - REG_C_0] = +- mlx5_flow_hw_avl_tags[i]; +- copy_masks |= (1 << (mlx5_flow_hw_avl_tags[i] - REG_C_0)); +- } +- } +- if (copy_masks != masks) { +- j = 0; +- for (i = 0; i < MLX5_FLOW_HW_TAGS_MAX; i++) +- if (!!((1 << i) & copy_masks)) +- mlx5_flow_hw_avl_tags[j++] = copy[i]; +- } +- } else { +- j = 0; +- for (i = 0; i < MLX5_FLOW_HW_TAGS_MAX; i++) { +- if (!!((1 << i) & masks)) +- mlx5_flow_hw_avl_tags[j++] = +- (enum modify_reg)(i + (uint32_t)REG_C_0); ++ if (mlx5_flow_hw_avl_tags[i] == REG_NON) ++ continue; ++ reg_off = mlx5_flow_hw_avl_tags[i] - REG_C_0; ++ if ((1 << reg_off) & masks) ++ common_masks |= (1 << reg_off); + } ++ if (common_masks != masks) ++ masks = common_masks; ++ else ++ goto after_avl_tags; + } ++ j = 0; ++ for (i = 0; i < MLX5_FLOW_HW_TAGS_MAX; i++) { ++ if ((1 << i) & masks) ++ mlx5_flow_hw_avl_tags[j++] = (enum modify_reg)(i + (uint32_t)REG_C_0); ++ } ++ /* Clear the rest of unusable tag indexes. */ ++ for (; j < MLX5_FLOW_HW_TAGS_MAX; j++) ++ mlx5_flow_hw_avl_tags[j] = REG_NON; ++after_avl_tags: + priv->sh->hws_tags = 1; + mlx5_flow_hw_aso_tag = (enum modify_reg)priv->mtr_color_reg; + mlx5_flow_hw_avl_tags_init_cnt++; +@@ -8355,6 +8638,10 @@ const struct mlx5_flow_driver_ops mlx5_flow_hw_drv_ops = { + * Pointer to flow rule actions. + * @param action_template_idx + * Index of an action template associated with @p table. ++ * @param info ++ * Additional info about control flow rule. ++ * @param external ++ * External ctrl flow. + * + * @return + * 0 on success, negative errno value otherwise and rte_errno set. +@@ -8366,7 +8653,9 @@ flow_hw_create_ctrl_flow(struct rte_eth_dev *owner_dev, + struct rte_flow_item items[], + uint8_t item_template_idx, + struct rte_flow_action actions[], +- uint8_t action_template_idx) ++ uint8_t action_template_idx, ++ struct mlx5_hw_ctrl_flow_info *info, ++ bool external) + { + struct mlx5_priv *priv = proxy_dev->data->dev_private; + uint32_t queue = CTRL_QUEUE_ID(priv); +@@ -8413,7 +8702,14 @@ flow_hw_create_ctrl_flow(struct rte_eth_dev *owner_dev, + } + entry->owner_dev = owner_dev; + entry->flow = flow; +- LIST_INSERT_HEAD(&priv->hw_ctrl_flows, entry, next); ++ if (info) ++ entry->info = *info; ++ else ++ entry->info.type = MLX5_HW_CTRL_FLOW_TYPE_GENERAL; ++ if (external) ++ LIST_INSERT_HEAD(&priv->hw_ext_ctrl_flows, entry, next); ++ else ++ LIST_INSERT_HEAD(&priv->hw_ctrl_flows, entry, next); + rte_spinlock_unlock(&priv->hw_ctrl_lock); + return 0; + error: +@@ -8587,11 +8883,23 @@ flow_hw_flush_all_ctrl_flows(struct rte_eth_dev *dev) + mlx5_free(cf); + cf = cf_next; + } ++ cf = LIST_FIRST(&priv->hw_ext_ctrl_flows); ++ while (cf != NULL) { ++ cf_next = LIST_NEXT(cf, next); ++ ret = flow_hw_destroy_ctrl_flow(dev, cf->flow); ++ if (ret) { ++ rte_errno = ret; ++ return -ret; ++ } ++ LIST_REMOVE(cf, next); ++ mlx5_free(cf); ++ cf = cf_next; ++ } + return 0; + } + + int +-mlx5_flow_hw_esw_create_sq_miss_flow(struct rte_eth_dev *dev, uint32_t sqn) ++mlx5_flow_hw_esw_create_sq_miss_flow(struct rte_eth_dev *dev, uint32_t sqn, bool external) + { + uint16_t port_id = dev->data->port_id; + struct rte_flow_item_ethdev esw_mgr_spec = { +@@ -8616,6 +8924,10 @@ mlx5_flow_hw_esw_create_sq_miss_flow(struct rte_eth_dev *dev, uint32_t sqn) + }; + struct rte_flow_item items[3] = { { 0 } }; + struct rte_flow_action actions[3] = { { 0 } }; ++ struct mlx5_hw_ctrl_flow_info flow_info = { ++ .type = MLX5_HW_CTRL_FLOW_TYPE_SQ_MISS_ROOT, ++ .esw_mgr_sq = sqn, ++ }; + struct rte_eth_dev *proxy_dev; + struct mlx5_priv *proxy_priv; + uint16_t proxy_port_id = dev->data->port_id; +@@ -8671,7 +8983,7 @@ mlx5_flow_hw_esw_create_sq_miss_flow(struct rte_eth_dev *dev, uint32_t sqn) + .type = RTE_FLOW_ACTION_TYPE_END, + }; + ret = flow_hw_create_ctrl_flow(dev, proxy_dev, proxy_priv->hw_esw_sq_miss_root_tbl, +- items, 0, actions, 0); ++ items, 0, actions, 0, &flow_info, external); + if (ret) { + DRV_LOG(ERR, "Port %u failed to create root SQ miss flow rule for SQ %u, ret %d", + port_id, sqn, ret); +@@ -8700,8 +9012,9 @@ mlx5_flow_hw_esw_create_sq_miss_flow(struct rte_eth_dev *dev, uint32_t sqn) + actions[1] = (struct rte_flow_action){ + .type = RTE_FLOW_ACTION_TYPE_END, + }; ++ flow_info.type = MLX5_HW_CTRL_FLOW_TYPE_SQ_MISS; + ret = flow_hw_create_ctrl_flow(dev, proxy_dev, proxy_priv->hw_esw_sq_miss_tbl, +- items, 0, actions, 0); ++ items, 0, actions, 0, &flow_info, external); + if (ret) { + DRV_LOG(ERR, "Port %u failed to create HWS SQ miss flow rule for SQ %u, ret %d", + port_id, sqn, ret); +@@ -8710,6 +9023,58 @@ mlx5_flow_hw_esw_create_sq_miss_flow(struct rte_eth_dev *dev, uint32_t sqn) + return 0; + } + ++static bool ++flow_hw_is_matching_sq_miss_flow(struct mlx5_hw_ctrl_flow *cf, ++ struct rte_eth_dev *dev, ++ uint32_t sqn) ++{ ++ if (cf->owner_dev != dev) ++ return false; ++ if (cf->info.type == MLX5_HW_CTRL_FLOW_TYPE_SQ_MISS_ROOT && cf->info.esw_mgr_sq == sqn) ++ return true; ++ if (cf->info.type == MLX5_HW_CTRL_FLOW_TYPE_SQ_MISS && cf->info.esw_mgr_sq == sqn) ++ return true; ++ return false; ++} ++ ++int ++mlx5_flow_hw_esw_destroy_sq_miss_flow(struct rte_eth_dev *dev, uint32_t sqn) ++{ ++ uint16_t port_id = dev->data->port_id; ++ uint16_t proxy_port_id = dev->data->port_id; ++ struct rte_eth_dev *proxy_dev; ++ struct mlx5_priv *proxy_priv; ++ struct mlx5_hw_ctrl_flow *cf; ++ struct mlx5_hw_ctrl_flow *cf_next; ++ int ret; ++ ++ ret = rte_flow_pick_transfer_proxy(port_id, &proxy_port_id, NULL); ++ if (ret) { ++ DRV_LOG(ERR, "Unable to pick transfer proxy port for port %u. Transfer proxy " ++ "port must be present for default SQ miss flow rules to exist.", ++ port_id); ++ return ret; ++ } ++ proxy_dev = &rte_eth_devices[proxy_port_id]; ++ proxy_priv = proxy_dev->data->dev_private; ++ if (!proxy_priv->dr_ctx) ++ return 0; ++ if (!proxy_priv->hw_esw_sq_miss_root_tbl || ++ !proxy_priv->hw_esw_sq_miss_tbl) ++ return 0; ++ cf = LIST_FIRST(&proxy_priv->hw_ctrl_flows); ++ while (cf != NULL) { ++ cf_next = LIST_NEXT(cf, next); ++ if (flow_hw_is_matching_sq_miss_flow(cf, dev, sqn)) { ++ claim_zero(flow_hw_destroy_ctrl_flow(proxy_dev, cf->flow)); ++ LIST_REMOVE(cf, next); ++ mlx5_free(cf); ++ } ++ cf = cf_next; ++ } ++ return 0; ++} ++ + int + mlx5_flow_hw_esw_create_default_jump_flow(struct rte_eth_dev *dev) + { +@@ -8738,6 +9103,9 @@ mlx5_flow_hw_esw_create_default_jump_flow(struct rte_eth_dev *dev) + .type = RTE_FLOW_ACTION_TYPE_END, + } + }; ++ struct mlx5_hw_ctrl_flow_info flow_info = { ++ .type = MLX5_HW_CTRL_FLOW_TYPE_DEFAULT_JUMP, ++ }; + struct rte_eth_dev *proxy_dev; + struct mlx5_priv *proxy_priv; + uint16_t proxy_port_id = dev->data->port_id; +@@ -8768,7 +9136,7 @@ mlx5_flow_hw_esw_create_default_jump_flow(struct rte_eth_dev *dev) + } + return flow_hw_create_ctrl_flow(dev, proxy_dev, + proxy_priv->hw_esw_zero_tbl, +- items, 0, actions, 0); ++ items, 0, actions, 0, &flow_info, false); + } + + int +@@ -8814,17 +9182,20 @@ mlx5_flow_hw_create_tx_default_mreg_copy_flow(struct rte_eth_dev *dev) + .type = RTE_FLOW_ACTION_TYPE_END, + }, + }; ++ struct mlx5_hw_ctrl_flow_info flow_info = { ++ .type = MLX5_HW_CTRL_FLOW_TYPE_TX_META_COPY, ++ }; + + MLX5_ASSERT(priv->master); + if (!priv->dr_ctx || !priv->hw_tx_meta_cpy_tbl) + return 0; + return flow_hw_create_ctrl_flow(dev, dev, + priv->hw_tx_meta_cpy_tbl, +- eth_all, 0, copy_reg_action, 0); ++ eth_all, 0, copy_reg_action, 0, &flow_info, false); + } + + int +-mlx5_flow_hw_tx_repr_matching_flow(struct rte_eth_dev *dev, uint32_t sqn) ++mlx5_flow_hw_tx_repr_matching_flow(struct rte_eth_dev *dev, uint32_t sqn, bool external) + { + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_rte_flow_item_sq sq_spec = { +@@ -8849,6 +9220,10 @@ mlx5_flow_hw_tx_repr_matching_flow(struct rte_eth_dev *dev, uint32_t sqn) + { .type = RTE_FLOW_ACTION_TYPE_END }, + { .type = RTE_FLOW_ACTION_TYPE_END }, + }; ++ struct mlx5_hw_ctrl_flow_info flow_info = { ++ .type = MLX5_HW_CTRL_FLOW_TYPE_TX_REPR_MATCH, ++ .tx_repr_sq = sqn, ++ }; + + /* It is assumed that caller checked for representor matching. */ + MLX5_ASSERT(priv->sh->config.repr_matching); +@@ -8874,7 +9249,44 @@ mlx5_flow_hw_tx_repr_matching_flow(struct rte_eth_dev *dev, uint32_t sqn) + actions[2].type = RTE_FLOW_ACTION_TYPE_JUMP; + } + return flow_hw_create_ctrl_flow(dev, dev, priv->hw_tx_repr_tagging_tbl, +- items, 0, actions, 0); ++ items, 0, actions, 0, &flow_info, external); ++} ++ ++int ++mlx5_flow_hw_lacp_rx_flow(struct rte_eth_dev *dev) ++{ ++ struct mlx5_priv *priv = dev->data->dev_private; ++ struct rte_flow_item_eth lacp_item = { ++ .type = RTE_BE16(RTE_ETHER_TYPE_SLOW), ++ }; ++ struct rte_flow_item eth_lacp[] = { ++ [0] = { ++ .type = RTE_FLOW_ITEM_TYPE_ETH, ++ .spec = &lacp_item, ++ .mask = &lacp_item, ++ }, ++ [1] = { ++ .type = RTE_FLOW_ITEM_TYPE_END, ++ }, ++ }; ++ struct rte_flow_action miss_action[] = { ++ [0] = { ++ .type = (enum rte_flow_action_type) ++ MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, ++ }, ++ [1] = { ++ .type = RTE_FLOW_ACTION_TYPE_END, ++ }, ++ }; ++ struct mlx5_hw_ctrl_flow_info flow_info = { ++ .type = MLX5_HW_CTRL_FLOW_TYPE_LACP_RX, ++ }; ++ ++ MLX5_ASSERT(priv->master); ++ if (!priv->dr_ctx || !priv->hw_lacp_rx_tbl) ++ return 0; ++ return flow_hw_create_ctrl_flow(dev, dev, priv->hw_lacp_rx_tbl, eth_lacp, 0, ++ miss_action, 0, &flow_info, false); + } + + static uint32_t +@@ -8989,6 +9401,9 @@ __flow_hw_ctrl_flows_single(struct rte_eth_dev *dev, + { .type = RTE_FLOW_ACTION_TYPE_RSS }, + { .type = RTE_FLOW_ACTION_TYPE_END }, + }; ++ struct mlx5_hw_ctrl_flow_info flow_info = { ++ .type = MLX5_HW_CTRL_FLOW_TYPE_DEFAULT_RX_RSS, ++ }; + + if (!eth_spec) + return -EINVAL; +@@ -9002,7 +9417,7 @@ __flow_hw_ctrl_flows_single(struct rte_eth_dev *dev, + items[3] = flow_hw_get_ctrl_rx_l4_item(rss_type); + items[4] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_END }; + /* Without VLAN filtering, only a single flow rule must be created. */ +- return flow_hw_create_ctrl_flow(dev, dev, tbl, items, 0, actions, 0); ++ return flow_hw_create_ctrl_flow(dev, dev, tbl, items, 0, actions, 0, &flow_info, false); + } + + static int +@@ -9018,6 +9433,9 @@ __flow_hw_ctrl_flows_single_vlan(struct rte_eth_dev *dev, + { .type = RTE_FLOW_ACTION_TYPE_RSS }, + { .type = RTE_FLOW_ACTION_TYPE_END }, + }; ++ struct mlx5_hw_ctrl_flow_info flow_info = { ++ .type = MLX5_HW_CTRL_FLOW_TYPE_DEFAULT_RX_RSS, ++ }; + unsigned int i; + + if (!eth_spec) +@@ -9040,7 +9458,8 @@ __flow_hw_ctrl_flows_single_vlan(struct rte_eth_dev *dev, + }; + + items[1].spec = &vlan_spec; +- if (flow_hw_create_ctrl_flow(dev, dev, tbl, items, 0, actions, 0)) ++ if (flow_hw_create_ctrl_flow(dev, dev, ++ tbl, items, 0, actions, 0, &flow_info, false)) + return -rte_errno; + } + return 0; +@@ -9058,6 +9477,9 @@ __flow_hw_ctrl_flows_unicast(struct rte_eth_dev *dev, + { .type = RTE_FLOW_ACTION_TYPE_RSS }, + { .type = RTE_FLOW_ACTION_TYPE_END }, + }; ++ struct mlx5_hw_ctrl_flow_info flow_info = { ++ .type = MLX5_HW_CTRL_FLOW_TYPE_DEFAULT_RX_RSS, ++ }; + const struct rte_ether_addr cmp = { + .addr_bytes = "\x00\x00\x00\x00\x00\x00", + }; +@@ -9081,7 +9503,8 @@ __flow_hw_ctrl_flows_unicast(struct rte_eth_dev *dev, + if (!memcmp(mac, &cmp, sizeof(*mac))) + continue; + memcpy(ð_spec.dst.addr_bytes, mac->addr_bytes, RTE_ETHER_ADDR_LEN); +- if (flow_hw_create_ctrl_flow(dev, dev, tbl, items, 0, actions, 0)) ++ if (flow_hw_create_ctrl_flow(dev, dev, ++ tbl, items, 0, actions, 0, &flow_info, false)) + return -rte_errno; + } + return 0; +@@ -9100,6 +9523,9 @@ __flow_hw_ctrl_flows_unicast_vlan(struct rte_eth_dev *dev, + { .type = RTE_FLOW_ACTION_TYPE_RSS }, + { .type = RTE_FLOW_ACTION_TYPE_END }, + }; ++ struct mlx5_hw_ctrl_flow_info flow_info = { ++ .type = MLX5_HW_CTRL_FLOW_TYPE_DEFAULT_RX_RSS, ++ }; + const struct rte_ether_addr cmp = { + .addr_bytes = "\x00\x00\x00\x00\x00\x00", + }; +@@ -9131,7 +9557,8 @@ __flow_hw_ctrl_flows_unicast_vlan(struct rte_eth_dev *dev, + }; + + items[1].spec = &vlan_spec; +- if (flow_hw_create_ctrl_flow(dev, dev, tbl, items, 0, actions, 0)) ++ if (flow_hw_create_ctrl_flow(dev, dev, tbl, items, 0, actions, 0, ++ &flow_info, false)) + return -rte_errno; + } + } +diff --git a/dpdk/drivers/net/mlx5/mlx5_flow_verbs.c b/dpdk/drivers/net/mlx5/mlx5_flow_verbs.c +index 28ea28bfbe..1e9c7cf7c5 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_flow_verbs.c ++++ b/dpdk/drivers/net/mlx5/mlx5_flow_verbs.c +@@ -1280,14 +1280,14 @@ flow_verbs_validate(struct rte_eth_dev *dev, + uint16_t ether_type = 0; + bool is_empty_vlan = false; + uint16_t udp_dport = 0; +- bool is_root; ++ /* Verbs interface does not support groups higher than 0. */ ++ bool is_root = true; + + if (items == NULL) + return -1; + ret = flow_verbs_validate_attributes(dev, attr, error); + if (ret < 0) + return ret; +- is_root = ret; + for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { + int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); + int ret = 0; +@@ -1484,7 +1484,8 @@ flow_verbs_validate(struct rte_eth_dev *dev, + action_flags |= MLX5_FLOW_ACTION_MARK; + break; + case RTE_FLOW_ACTION_TYPE_DROP: +- ret = mlx5_flow_validate_action_drop(action_flags, ++ ret = mlx5_flow_validate_action_drop(dev, ++ is_root, + attr, + error); + if (ret < 0) +diff --git a/dpdk/drivers/net/mlx5/mlx5_hws_cnt.c b/dpdk/drivers/net/mlx5/mlx5_hws_cnt.c +index 51704ef754..791fde4458 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_hws_cnt.c ++++ b/dpdk/drivers/net/mlx5/mlx5_hws_cnt.c +@@ -306,26 +306,25 @@ mlx5_hws_cnt_svc(void *opaque) + (struct mlx5_dev_ctx_shared *)opaque; + uint64_t interval = + (uint64_t)sh->cnt_svc->query_interval * (US_PER_S / MS_PER_S); +- uint16_t port_id; ++ struct mlx5_hws_cnt_pool *hws_cpool; + uint64_t start_cycle, query_cycle = 0; + uint64_t query_us; + uint64_t sleep_us; + + while (sh->cnt_svc->svc_running != 0) { ++ if (rte_spinlock_trylock(&sh->cpool_lock) == 0) ++ continue; + start_cycle = rte_rdtsc(); +- MLX5_ETH_FOREACH_DEV(port_id, sh->cdev->dev) { +- struct mlx5_priv *opriv = +- rte_eth_devices[port_id].data->dev_private; +- if (opriv != NULL && +- opriv->sh == sh && +- opriv->hws_cpool != NULL) { +- __mlx5_hws_cnt_svc(sh, opriv->hws_cpool); +- if (opriv->hws_age_req) +- mlx5_hws_aging_check(opriv, +- opriv->hws_cpool); +- } ++ /* 200ms for 16M counters. */ ++ LIST_FOREACH(hws_cpool, &sh->hws_cpool_list, next) { ++ struct mlx5_priv *opriv = hws_cpool->priv; ++ ++ __mlx5_hws_cnt_svc(sh, hws_cpool); ++ if (opriv->hws_age_req) ++ mlx5_hws_aging_check(opriv, hws_cpool); + } + query_cycle = rte_rdtsc() - start_cycle; ++ rte_spinlock_unlock(&sh->cpool_lock); + query_us = query_cycle / (rte_get_timer_hz() / US_PER_S); + sleep_us = interval - query_us; + if (interval > query_us) +@@ -410,8 +409,7 @@ mlx5_hws_cnt_pool_init(struct mlx5_dev_ctx_shared *sh, + goto error; + } + for (qidx = 0; qidx < ccfg->q_num; qidx++) { +- snprintf(mz_name, sizeof(mz_name), "%s_cache/%u", pcfg->name, +- qidx); ++ snprintf(mz_name, sizeof(mz_name), "%s_qc/%x", pcfg->name, qidx); + cntp->cache->qcache[qidx] = rte_ring_create(mz_name, ccfg->size, + SOCKET_ID_ANY, + RING_F_SP_ENQ | RING_F_SC_DEQ | +@@ -634,7 +632,7 @@ mlx5_hws_cnt_pool_create(struct rte_eth_dev *dev, + SOCKET_ID_ANY); + if (mp_name == NULL) + goto error; +- snprintf(mp_name, RTE_MEMZONE_NAMESIZE, "MLX5_HWS_CNT_POOL_%u", ++ snprintf(mp_name, RTE_MEMZONE_NAMESIZE, "MLX5_HWS_CNT_P_%x", + dev->data->port_id); + pcfg.name = mp_name; + pcfg.request_num = pattr->nb_counters; +@@ -660,6 +658,10 @@ mlx5_hws_cnt_pool_create(struct rte_eth_dev *dev, + if (ret != 0) + goto error; + priv->sh->cnt_svc->refcnt++; ++ cpool->priv = priv; ++ rte_spinlock_lock(&priv->sh->cpool_lock); ++ LIST_INSERT_HEAD(&priv->sh->hws_cpool_list, cpool, next); ++ rte_spinlock_unlock(&priv->sh->cpool_lock); + return cpool; + error: + mlx5_hws_cnt_pool_destroy(priv->sh, cpool); +@@ -672,6 +674,13 @@ mlx5_hws_cnt_pool_destroy(struct mlx5_dev_ctx_shared *sh, + { + if (cpool == NULL) + return; ++ /* ++ * 16M counter consumes 200ms to finish the query. ++ * Maybe blocked for at most 200ms here. ++ */ ++ rte_spinlock_lock(&sh->cpool_lock); ++ LIST_REMOVE(cpool, next); ++ rte_spinlock_unlock(&sh->cpool_lock); + if (--sh->cnt_svc->refcnt == 0) + mlx5_hws_cnt_svc_deinit(sh); + mlx5_hws_cnt_pool_action_destroy(cpool); +@@ -1229,11 +1238,13 @@ mlx5_hws_age_pool_destroy(struct mlx5_priv *priv) + { + struct mlx5_age_info *age_info = GET_PORT_AGE_INFO(priv); + ++ rte_spinlock_lock(&priv->sh->cpool_lock); + MLX5_ASSERT(priv->hws_age_req); + mlx5_hws_age_info_destroy(priv); + mlx5_ipool_destroy(age_info->ages_ipool); + age_info->ages_ipool = NULL; + priv->hws_age_req = 0; ++ rte_spinlock_unlock(&priv->sh->cpool_lock); + } + + #endif +diff --git a/dpdk/drivers/net/mlx5/mlx5_hws_cnt.h b/dpdk/drivers/net/mlx5/mlx5_hws_cnt.h +index 030dcead86..b5c19a8e2c 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_hws_cnt.h ++++ b/dpdk/drivers/net/mlx5/mlx5_hws_cnt.h +@@ -97,6 +97,7 @@ struct mlx5_hws_cnt_pool_caches { + }; + + struct mlx5_hws_cnt_pool { ++ LIST_ENTRY(mlx5_hws_cnt_pool) next; + struct mlx5_hws_cnt_pool_cfg cfg __rte_cache_aligned; + struct mlx5_hws_cnt_dcs_mng dcs_mng __rte_cache_aligned; + uint32_t query_gen __rte_cache_aligned; +@@ -107,6 +108,7 @@ struct mlx5_hws_cnt_pool { + struct rte_ring *wait_reset_list; + struct mlx5_hws_cnt_pool_caches *cache; + uint64_t time_of_last_age_check; ++ struct mlx5_priv *priv; + } __rte_cache_aligned; + + /* HWS AGE status. */ +diff --git a/dpdk/drivers/net/mlx5/mlx5_rx.c b/dpdk/drivers/net/mlx5/mlx5_rx.c +index 917c517b83..b41f7a51f5 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_rx.c ++++ b/dpdk/drivers/net/mlx5/mlx5_rx.c +@@ -39,7 +39,8 @@ rxq_cq_to_pkt_type(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe, + + static __rte_always_inline int + mlx5_rx_poll_len(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe, +- uint16_t cqe_cnt, volatile struct mlx5_mini_cqe8 **mcqe); ++ uint16_t cqe_cnt, volatile struct mlx5_mini_cqe8 **mcqe, ++ uint16_t *skip_cnt, bool mprq); + + static __rte_always_inline uint32_t + rxq_cq_to_ol_flags(volatile struct mlx5_cqe *cqe); +@@ -408,10 +409,14 @@ mlx5_rxq_initialize(struct mlx5_rxq_data *rxq) + *rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci); + } + ++#define MLX5_ERROR_CQE_MASK 0x40000000 + /* Must be negative. */ +-#define MLX5_ERROR_CQE_RET (-1) ++#define MLX5_REGULAR_ERROR_CQE_RET (-5) ++#define MLX5_CRITICAL_ERROR_CQE_RET (-4) + /* Must not be negative. */ + #define MLX5_RECOVERY_ERROR_RET 0 ++#define MLX5_RECOVERY_IGNORE_RET 1 ++#define MLX5_RECOVERY_COMPLETED_RET 2 + + /** + * Handle a Rx error. +@@ -425,12 +430,18 @@ mlx5_rxq_initialize(struct mlx5_rxq_data *rxq) + * @param[in] vec + * 1 when called from vectorized Rx burst, need to prepare mbufs for the RQ. + * 0 when called from non-vectorized Rx burst. ++ * @param[in] err_n ++ * Number of CQEs to check for an error. + * + * @return +- * MLX5_RECOVERY_ERROR_RET in case of recovery error, otherwise the CQE status. ++ * MLX5_RECOVERY_ERROR_RET in case of recovery error, ++ * MLX5_RECOVERY_IGNORE_RET in case of non-critical error syndrome, ++ * MLX5_RECOVERY_COMPLETED_RET in case of recovery is completed, ++ * otherwise the CQE status after ignored error syndrome or queue reset. + */ + int +-mlx5_rx_err_handle(struct mlx5_rxq_data *rxq, uint8_t vec) ++mlx5_rx_err_handle(struct mlx5_rxq_data *rxq, uint8_t vec, ++ uint16_t err_n, uint16_t *skip_cnt) + { + const uint16_t cqe_n = 1 << rxq->cqe_n; + const uint16_t cqe_mask = cqe_n - 1; +@@ -442,13 +453,39 @@ mlx5_rx_err_handle(struct mlx5_rxq_data *rxq, uint8_t vec) + volatile struct mlx5_cqe *cqe; + volatile struct mlx5_err_cqe *err_cqe; + } u = { +- .cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_mask], ++ .cqe = &(*rxq->cqes)[(rxq->cq_ci - vec) & cqe_mask], + }; + struct mlx5_mp_arg_queue_state_modify sm; +- int ret; ++ bool critical_syndrome = false; ++ int ret, i; + + switch (rxq->err_state) { ++ case MLX5_RXQ_ERR_STATE_IGNORE: ++ ret = check_cqe(u.cqe, cqe_n, rxq->cq_ci - vec); ++ if (ret != MLX5_CQE_STATUS_ERR) { ++ rxq->err_state = MLX5_RXQ_ERR_STATE_NO_ERROR; ++ return ret; ++ } ++ /* Fall-through */ + case MLX5_RXQ_ERR_STATE_NO_ERROR: ++ for (i = 0; i < (int)err_n; i++) { ++ u.cqe = &(*rxq->cqes)[(rxq->cq_ci - vec - i) & cqe_mask]; ++ if (MLX5_CQE_OPCODE(u.cqe->op_own) == MLX5_CQE_RESP_ERR) { ++ if (u.err_cqe->syndrome == MLX5_CQE_SYNDROME_LOCAL_QP_OP_ERR || ++ u.err_cqe->syndrome == MLX5_CQE_SYNDROME_LOCAL_PROT_ERR || ++ u.err_cqe->syndrome == MLX5_CQE_SYNDROME_WR_FLUSH_ERR) ++ critical_syndrome = true; ++ break; ++ } ++ } ++ if (!critical_syndrome) { ++ if (rxq->err_state == MLX5_RXQ_ERR_STATE_NO_ERROR) { ++ *skip_cnt = 0; ++ if (i == err_n) ++ rxq->err_state = MLX5_RXQ_ERR_STATE_IGNORE; ++ } ++ return MLX5_RECOVERY_IGNORE_RET; ++ } + rxq->err_state = MLX5_RXQ_ERR_STATE_NEED_RESET; + /* Fall-through */ + case MLX5_RXQ_ERR_STATE_NEED_RESET: +@@ -507,7 +544,6 @@ mlx5_rx_err_handle(struct mlx5_rxq_data *rxq, uint8_t vec) + rxq->elts_ci : rxq->rq_ci; + uint32_t elt_idx; + struct rte_mbuf **elt; +- int i; + unsigned int n = elts_n - (elts_ci - + rxq->rq_pi); + +@@ -540,6 +576,7 @@ mlx5_rx_err_handle(struct mlx5_rxq_data *rxq, uint8_t vec) + } + mlx5_rxq_initialize(rxq); + rxq->err_state = MLX5_RXQ_ERR_STATE_NO_ERROR; ++ return MLX5_RECOVERY_COMPLETED_RET; + } + return ret; + default: +@@ -559,19 +596,24 @@ mlx5_rx_err_handle(struct mlx5_rxq_data *rxq, uint8_t vec) + * @param[out] mcqe + * Store pointer to mini-CQE if compressed. Otherwise, the pointer is not + * written. +- * ++ * @param[out] skip_cnt ++ * Number of packets skipped due to recoverable errors. ++ * @param mprq ++ * Indication if it is called from MPRQ. + * @return +- * 0 in case of empty CQE, MLX5_ERROR_CQE_RET in case of error CQE, +- * otherwise the packet size in regular RxQ, and striding byte +- * count format in mprq case. ++ * 0 in case of empty CQE, MLX5_REGULAR_ERROR_CQE_RET in case of error CQE, ++ * MLX5_CRITICAL_ERROR_CQE_RET in case of error CQE lead to Rx queue reset, ++ * otherwise the packet size in regular RxQ, ++ * and striding byte count format in mprq case. + */ + static inline int + mlx5_rx_poll_len(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe, +- uint16_t cqe_cnt, volatile struct mlx5_mini_cqe8 **mcqe) ++ uint16_t cqe_cnt, volatile struct mlx5_mini_cqe8 **mcqe, ++ uint16_t *skip_cnt, bool mprq) + { + struct rxq_zip *zip = &rxq->zip; + uint16_t cqe_n = cqe_cnt + 1; +- int len; ++ int len = 0, ret = 0; + uint16_t idx, end; + + do { +@@ -620,7 +662,6 @@ mlx5_rx_poll_len(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe, + * compressed. + */ + } else { +- int ret; + int8_t op_own; + uint32_t cq_ci; + +@@ -628,10 +669,12 @@ mlx5_rx_poll_len(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe, + if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) { + if (unlikely(ret == MLX5_CQE_STATUS_ERR || + rxq->err_state)) { +- ret = mlx5_rx_err_handle(rxq, 0); +- if (ret == MLX5_CQE_STATUS_HW_OWN || +- ret == MLX5_RECOVERY_ERROR_RET) +- return MLX5_ERROR_CQE_RET; ++ ret = mlx5_rx_err_handle(rxq, 0, 1, skip_cnt); ++ if (ret == MLX5_CQE_STATUS_HW_OWN) ++ return MLX5_ERROR_CQE_MASK; ++ if (ret == MLX5_RECOVERY_ERROR_RET || ++ ret == MLX5_RECOVERY_COMPLETED_RET) ++ return MLX5_CRITICAL_ERROR_CQE_RET; + } else { + return 0; + } +@@ -684,8 +727,15 @@ mlx5_rx_poll_len(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe, + } + } + if (unlikely(rxq->err_state)) { ++ if (rxq->err_state == MLX5_RXQ_ERR_STATE_IGNORE && ++ ret == MLX5_CQE_STATUS_SW_OWN) { ++ rxq->err_state = MLX5_RXQ_ERR_STATE_NO_ERROR; ++ return len & MLX5_ERROR_CQE_MASK; ++ } + cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_cnt]; + ++rxq->stats.idropped; ++ (*skip_cnt) += mprq ? (len & MLX5_MPRQ_STRIDE_NUM_MASK) >> ++ MLX5_MPRQ_STRIDE_NUM_SHIFT : 1; + } else { + return len; + } +@@ -837,6 +887,7 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) + int len = 0; /* keep its value across iterations. */ + + while (pkts_n) { ++ uint16_t skip_cnt; + unsigned int idx = rq_ci & wqe_cnt; + volatile struct mlx5_wqe_data_seg *wqe = + &((volatile struct mlx5_wqe_data_seg *)rxq->wqes)[idx]; +@@ -875,11 +926,24 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) + } + if (!pkt) { + cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_cnt]; +- len = mlx5_rx_poll_len(rxq, cqe, cqe_cnt, &mcqe); +- if (len <= 0) { +- rte_mbuf_raw_free(rep); +- if (unlikely(len == MLX5_ERROR_CQE_RET)) ++ len = mlx5_rx_poll_len(rxq, cqe, cqe_cnt, &mcqe, &skip_cnt, false); ++ if (unlikely(len & MLX5_ERROR_CQE_MASK)) { ++ if (len == MLX5_CRITICAL_ERROR_CQE_RET) { ++ rte_mbuf_raw_free(rep); + rq_ci = rxq->rq_ci << sges_n; ++ break; ++ } ++ rq_ci >>= sges_n; ++ rq_ci += skip_cnt; ++ rq_ci <<= sges_n; ++ idx = rq_ci & wqe_cnt; ++ wqe = &((volatile struct mlx5_wqe_data_seg *)rxq->wqes)[idx]; ++ seg = (*rxq->elts)[idx]; ++ cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_cnt]; ++ len = len & ~MLX5_ERROR_CQE_MASK; ++ } ++ if (len == 0) { ++ rte_mbuf_raw_free(rep); + break; + } + pkt = seg; +@@ -981,6 +1045,7 @@ mlx5_lro_update_tcp_hdr(struct rte_tcp_hdr *__rte_restrict tcp, + tcp->cksum = 0; + csum += rte_raw_cksum(tcp, (tcp->data_off >> 4) * 4); + csum = ((csum & 0xffff0000) >> 16) + (csum & 0xffff); ++ csum = ((csum & 0xffff0000) >> 16) + (csum & 0xffff); + csum = (~csum) & 0xffff; + if (csum == 0) + csum = 0xffff; +@@ -1089,6 +1154,7 @@ mlx5_rx_burst_mprq(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) + uint16_t strd_cnt; + uint16_t strd_idx; + uint32_t byte_cnt; ++ uint16_t skip_cnt; + volatile struct mlx5_mini_cqe8 *mcqe = NULL; + enum mlx5_rqx_code rxq_code; + +@@ -1101,14 +1167,26 @@ mlx5_rx_burst_mprq(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) + buf = (*rxq->mprq_bufs)[rq_ci & wq_mask]; + } + cqe = &(*rxq->cqes)[rxq->cq_ci & cq_mask]; +- ret = mlx5_rx_poll_len(rxq, cqe, cq_mask, &mcqe); ++ ret = mlx5_rx_poll_len(rxq, cqe, cq_mask, &mcqe, &skip_cnt, true); ++ if (unlikely(ret & MLX5_ERROR_CQE_MASK)) { ++ if (ret == MLX5_CRITICAL_ERROR_CQE_RET) { ++ rq_ci = rxq->rq_ci; ++ consumed_strd = rxq->consumed_strd; ++ break; ++ } ++ consumed_strd += skip_cnt; ++ while (consumed_strd >= strd_n) { ++ /* Replace WQE if the buffer is still in use. */ ++ mprq_buf_replace(rxq, rq_ci & wq_mask); ++ /* Advance to the next WQE. */ ++ consumed_strd -= strd_n; ++ ++rq_ci; ++ buf = (*rxq->mprq_bufs)[rq_ci & wq_mask]; ++ } ++ cqe = &(*rxq->cqes)[rxq->cq_ci & cq_mask]; ++ } + if (ret == 0) + break; +- if (unlikely(ret == MLX5_ERROR_CQE_RET)) { +- rq_ci = rxq->rq_ci; +- consumed_strd = rxq->consumed_strd; +- break; +- } + byte_cnt = ret; + len = (byte_cnt & MLX5_MPRQ_LEN_MASK) >> MLX5_MPRQ_LEN_SHIFT; + MLX5_ASSERT((int)len >= (rxq->crc_present << 2)); +diff --git a/dpdk/drivers/net/mlx5/mlx5_rx.h b/dpdk/drivers/net/mlx5/mlx5_rx.h +index e078aaf3dc..6b42e27c89 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_rx.h ++++ b/dpdk/drivers/net/mlx5/mlx5_rx.h +@@ -62,6 +62,7 @@ enum mlx5_rxq_err_state { + MLX5_RXQ_ERR_STATE_NO_ERROR = 0, + MLX5_RXQ_ERR_STATE_NEED_RESET, + MLX5_RXQ_ERR_STATE_NEED_READY, ++ MLX5_RXQ_ERR_STATE_IGNORE, + }; + + enum mlx5_rqx_code { +@@ -286,7 +287,8 @@ int mlx5_hrxq_modify(struct rte_eth_dev *dev, uint32_t hxrq_idx, + + uint16_t mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n); + void mlx5_rxq_initialize(struct mlx5_rxq_data *rxq); +-__rte_noinline int mlx5_rx_err_handle(struct mlx5_rxq_data *rxq, uint8_t vec); ++__rte_noinline int mlx5_rx_err_handle(struct mlx5_rxq_data *rxq, uint8_t vec, ++ uint16_t err_n, uint16_t *skip_cnt); + void mlx5_mprq_buf_free(struct mlx5_mprq_buf *buf); + uint16_t mlx5_rx_burst_mprq(void *dpdk_rxq, struct rte_mbuf **pkts, + uint16_t pkts_n); +diff --git a/dpdk/drivers/net/mlx5/mlx5_rxq.c b/dpdk/drivers/net/mlx5/mlx5_rxq.c +index 81aa3f074a..9179b9d9d7 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_rxq.c ++++ b/dpdk/drivers/net/mlx5/mlx5_rxq.c +@@ -528,12 +528,12 @@ mlx5_rx_queue_stop(struct rte_eth_dev *dev, uint16_t idx) + * synchronized, that might be broken on RQ restart + * and cause Rx malfunction, so queue stopping is + * not supported if vectorized Rx burst is engaged. +- * The routine pointer depends on the process +- * type, should perform check there. ++ * The routine pointer depends on the process type, ++ * should perform check there. MPRQ is not supported as well. + */ +- if (pkt_burst == mlx5_rx_burst_vec) { +- DRV_LOG(ERR, "Rx queue stop is not supported " +- "for vectorized Rx"); ++ if (pkt_burst != mlx5_rx_burst) { ++ DRV_LOG(ERR, "Rx queue stop is only supported " ++ "for non-vectorized single-packet Rx"); + rte_errno = EINVAL; + return -EINVAL; + } +@@ -1601,10 +1601,10 @@ mlx5_mprq_prepare(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, + } else { + *actual_log_stride_num = config->mprq.log_stride_num; + } +- if (config->mprq.log_stride_size) { +- /* Checks if chosen size of stride is in supported range. */ ++ /* Checks if chosen size of stride is in supported range. */ ++ if (config->mprq.log_stride_size != (uint32_t)MLX5_ARG_UNSET) { + if (config->mprq.log_stride_size > log_max_stride_size || +- config->mprq.log_stride_size < log_min_stride_size) { ++ config->mprq.log_stride_size < log_min_stride_size) { + *actual_log_stride_size = log_def_stride_size; + DRV_LOG(WARNING, + "Port %u Rx queue %u size of a stride for Multi-Packet RQ is out of range, setting default value (%u)", +@@ -1614,10 +1614,26 @@ mlx5_mprq_prepare(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, + *actual_log_stride_size = config->mprq.log_stride_size; + } + } else { +- if (min_mbuf_size <= RTE_BIT32(log_max_stride_size)) ++ /* Make the stride fit the mbuf size by default. */ ++ if (min_mbuf_size <= RTE_BIT32(log_max_stride_size)) { ++ DRV_LOG(WARNING, ++ "Port %u Rx queue %u size of a stride for Multi-Packet RQ is adjusted to match the mbuf size (%u)", ++ dev->data->port_id, idx, min_mbuf_size); + *actual_log_stride_size = log2above(min_mbuf_size); +- else ++ } else { + goto unsupport; ++ } ++ } ++ /* Make sure the stride size is greater than the headroom. */ ++ if (RTE_BIT32(*actual_log_stride_size) < RTE_PKTMBUF_HEADROOM) { ++ if (RTE_BIT32(log_max_stride_size) > RTE_PKTMBUF_HEADROOM) { ++ DRV_LOG(WARNING, ++ "Port %u Rx queue %u size of a stride for Multi-Packet RQ is adjusted to accommodate the headroom (%u)", ++ dev->data->port_id, idx, RTE_PKTMBUF_HEADROOM); ++ *actual_log_stride_size = log2above(RTE_PKTMBUF_HEADROOM); ++ } else { ++ goto unsupport; ++ } + } + log_stride_wqe_size = *actual_log_stride_num + *actual_log_stride_size; + /* Check if WQE buffer size is supported by hardware. */ +@@ -1657,6 +1673,8 @@ unsupport: + " min_stride_sz = %u, max_stride_sz = %u).\n" + "Rx segment is %senabled. External mempool is %sused.", + dev->data->port_id, min_mbuf_size, desc, priv->rxqs_n, ++ config->mprq.log_stride_size == (uint32_t)MLX5_ARG_UNSET ? ++ RTE_BIT32(MLX5_MPRQ_DEFAULT_LOG_STRIDE_SIZE) : + RTE_BIT32(config->mprq.log_stride_size), + RTE_BIT32(config->mprq.log_stride_num), + config->mprq.min_rxqs_num, +@@ -2262,6 +2280,7 @@ mlx5_rxq_release(struct rte_eth_dev *dev, uint16_t idx) + RTE_ETH_QUEUE_STATE_STOPPED; + } + } else { /* Refcnt zero, closing device. */ ++ LIST_REMOVE(rxq_ctrl, next); + LIST_REMOVE(rxq, owner_entry); + if (LIST_EMPTY(&rxq_ctrl->owners)) { + if (!rxq_ctrl->is_hairpin) +@@ -2269,7 +2288,6 @@ mlx5_rxq_release(struct rte_eth_dev *dev, uint16_t idx) + (&rxq_ctrl->rxq.mr_ctrl.cache_bh); + if (rxq_ctrl->rxq.shared) + LIST_REMOVE(rxq_ctrl, share_entry); +- LIST_REMOVE(rxq_ctrl, next); + mlx5_free(rxq_ctrl); + } + dev->data->rx_queues[idx] = NULL; +diff --git a/dpdk/drivers/net/mlx5/mlx5_rxtx_vec.c b/dpdk/drivers/net/mlx5/mlx5_rxtx_vec.c +index 0e2eab068a..667475a93e 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_rxtx_vec.c ++++ b/dpdk/drivers/net/mlx5/mlx5_rxtx_vec.c +@@ -51,6 +51,7 @@ rxq_handle_pending_error(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, + uint16_t pkts_n) + { + uint16_t n = 0; ++ uint16_t skip_cnt; + unsigned int i; + #ifdef MLX5_PMD_SOFT_COUNTERS + uint32_t err_bytes = 0; +@@ -74,7 +75,7 @@ rxq_handle_pending_error(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, + rxq->stats.ipackets -= (pkts_n - n); + rxq->stats.ibytes -= err_bytes; + #endif +- mlx5_rx_err_handle(rxq, 1); ++ mlx5_rx_err_handle(rxq, 1, pkts_n, &skip_cnt); + return n; + } + +@@ -253,8 +254,6 @@ rxq_copy_mprq_mbuf_v(struct mlx5_rxq_data *rxq, + } + rxq->rq_pi += i; + rxq->cq_ci += i; +- rte_io_wmb(); +- *rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci); + if (rq_ci != rxq->rq_ci) { + rxq->rq_ci = rq_ci; + rte_io_wmb(); +@@ -361,8 +360,6 @@ rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, + rxq->decompressed -= n; + } + } +- rte_io_wmb(); +- *rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci); + *no_cq = !rcvd_pkt; + return rcvd_pkt; + } +@@ -390,6 +387,7 @@ mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) + bool no_cq = false; + + do { ++ err = 0; + nb_rx = rxq_burst_v(rxq, pkts + tn, pkts_n - tn, + &err, &no_cq); + if (unlikely(err | rxq->err_state)) +@@ -397,6 +395,8 @@ mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) + tn += nb_rx; + if (unlikely(no_cq)) + break; ++ rte_io_wmb(); ++ *rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci); + } while (tn != pkts_n); + return tn; + } +@@ -524,6 +524,7 @@ mlx5_rx_burst_mprq_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) + bool no_cq = false; + + do { ++ err = 0; + nb_rx = rxq_burst_mprq_v(rxq, pkts + tn, pkts_n - tn, + &err, &no_cq); + if (unlikely(err | rxq->err_state)) +@@ -531,6 +532,8 @@ mlx5_rx_burst_mprq_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) + tn += nb_rx; + if (unlikely(no_cq)) + break; ++ rte_io_wmb(); ++ *rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci); + } while (tn != pkts_n); + return tn; + } +diff --git a/dpdk/drivers/net/mlx5/mlx5_rxtx_vec_altivec.h b/dpdk/drivers/net/mlx5/mlx5_rxtx_vec_altivec.h +index 683a8f9a6c..204d17a8f2 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_rxtx_vec_altivec.h ++++ b/dpdk/drivers/net/mlx5/mlx5_rxtx_vec_altivec.h +@@ -783,7 +783,7 @@ rxq_cq_process_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq, + { + const uint16_t q_n = 1 << rxq->cqe_n; + const uint16_t q_mask = q_n - 1; +- unsigned int pos; ++ unsigned int pos, adj; + uint64_t n = 0; + uint64_t comp_idx = MLX5_VPMD_DESCS_PER_LOOP; + uint16_t nocmp_n = 0; +@@ -866,7 +866,7 @@ rxq_cq_process_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq, + __vector unsigned char pkt_mb0, pkt_mb1, pkt_mb2, pkt_mb3; + __vector unsigned char op_own, op_own_tmp1, op_own_tmp2; + __vector unsigned char opcode, owner_mask, invalid_mask; +- __vector unsigned char comp_mask; ++ __vector unsigned char comp_mask, mini_mask; + __vector unsigned char mask; + #ifdef MLX5_PMD_SOFT_COUNTERS + const __vector unsigned char lower_half = { +@@ -1174,6 +1174,16 @@ rxq_cq_process_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq, + (__vector unsigned long)mask); + + /* D.3 check error in opcode. */ ++ adj = (comp_idx != MLX5_VPMD_DESCS_PER_LOOP && comp_idx == n); ++ mask = (__vector unsigned char)(__vector unsigned long){ ++ (adj * sizeof(uint16_t) * 8), 0}; ++ lshift = vec_splat((__vector unsigned long)mask, 0); ++ shmask = vec_cmpgt(shmax, lshift); ++ mini_mask = (__vector unsigned char) ++ vec_sl((__vector unsigned long)invalid_mask, lshift); ++ mini_mask = (__vector unsigned char) ++ vec_sel((__vector unsigned long)shmask, ++ (__vector unsigned long)mini_mask, shmask); + opcode = (__vector unsigned char) + vec_cmpeq((__vector unsigned int)resp_err_check, + (__vector unsigned int)opcode); +@@ -1182,7 +1192,7 @@ rxq_cq_process_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq, + (__vector unsigned int)zero); + opcode = (__vector unsigned char) + vec_andc((__vector unsigned long)opcode, +- (__vector unsigned long)invalid_mask); ++ (__vector unsigned long)mini_mask); + + /* D.4 mark if any error is set */ + *err |= ((__vector unsigned long)opcode)[0]; +diff --git a/dpdk/drivers/net/mlx5/mlx5_rxtx_vec_neon.h b/dpdk/drivers/net/mlx5/mlx5_rxtx_vec_neon.h +index f7bbde4e0e..6d3c594e56 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_rxtx_vec_neon.h ++++ b/dpdk/drivers/net/mlx5/mlx5_rxtx_vec_neon.h +@@ -524,7 +524,7 @@ rxq_cq_process_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq, + { + const uint16_t q_n = 1 << rxq->cqe_n; + const uint16_t q_mask = q_n - 1; +- unsigned int pos; ++ unsigned int pos, adj; + uint64_t n = 0; + uint64_t comp_idx = MLX5_VPMD_DESCS_PER_LOOP; + uint16_t nocmp_n = 0; +@@ -616,7 +616,7 @@ rxq_cq_process_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq, + pos += MLX5_VPMD_DESCS_PER_LOOP) { + uint16x4_t op_own; + uint16x4_t opcode, owner_mask, invalid_mask; +- uint16x4_t comp_mask; ++ uint16x4_t comp_mask, mini_mask; + uint16x4_t mask; + uint16x4_t byte_cnt; + uint32x4_t ptype_info, flow_tag; +@@ -647,6 +647,14 @@ rxq_cq_process_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq, + c0 = vld1q_u64((uint64_t *)(p0 + 48)); + /* Synchronize for loading the rest of blocks. */ + rte_io_rmb(); ++ /* B.0 (CQE 3) reload lower half of the block. */ ++ c3 = vld1q_lane_u64((uint64_t *)(p3 + 48), c3, 0); ++ /* B.0 (CQE 2) reload lower half of the block. */ ++ c2 = vld1q_lane_u64((uint64_t *)(p2 + 48), c2, 0); ++ /* B.0 (CQE 1) reload lower half of the block. */ ++ c1 = vld1q_lane_u64((uint64_t *)(p1 + 48), c1, 0); ++ /* B.0 (CQE 0) reload lower half of the block. */ ++ c0 = vld1q_lane_u64((uint64_t *)(p0 + 48), c0, 0); + /* Prefetch next 4 CQEs. */ + if (pkts_n - pos >= 2 * MLX5_VPMD_DESCS_PER_LOOP) { + unsigned int next = pos + MLX5_VPMD_DESCS_PER_LOOP; +@@ -780,8 +788,12 @@ rxq_cq_process_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq, + -1UL >> (n * sizeof(uint16_t) * 8) : 0); + invalid_mask = vorr_u16(invalid_mask, mask); + /* D.3 check error in opcode. */ ++ adj = (comp_idx != MLX5_VPMD_DESCS_PER_LOOP && comp_idx == n); ++ mask = vcreate_u16(adj ? ++ -1UL >> ((n + 1) * sizeof(uint16_t) * 8) : -1UL); ++ mini_mask = vand_u16(invalid_mask, mask); + opcode = vceq_u16(resp_err_check, opcode); +- opcode = vbic_u16(opcode, invalid_mask); ++ opcode = vbic_u16(opcode, mini_mask); + /* D.4 mark if any error is set */ + *err |= vget_lane_u64(vreinterpret_u64_u16(opcode), 0); + /* C.4 fill in mbuf - rearm_data and packet_type. */ +diff --git a/dpdk/drivers/net/mlx5/mlx5_rxtx_vec_sse.h b/dpdk/drivers/net/mlx5/mlx5_rxtx_vec_sse.h +index 185d2695db..ab69af0c55 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_rxtx_vec_sse.h ++++ b/dpdk/drivers/net/mlx5/mlx5_rxtx_vec_sse.h +@@ -523,7 +523,7 @@ rxq_cq_process_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq, + { + const uint16_t q_n = 1 << rxq->cqe_n; + const uint16_t q_mask = q_n - 1; +- unsigned int pos; ++ unsigned int pos, adj; + uint64_t n = 0; + uint64_t comp_idx = MLX5_VPMD_DESCS_PER_LOOP; + uint16_t nocmp_n = 0; +@@ -591,7 +591,7 @@ rxq_cq_process_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq, + __m128i pkt_mb0, pkt_mb1, pkt_mb2, pkt_mb3; + __m128i op_own, op_own_tmp1, op_own_tmp2; + __m128i opcode, owner_mask, invalid_mask; +- __m128i comp_mask; ++ __m128i comp_mask, mini_mask; + __m128i mask; + #ifdef MLX5_PMD_SOFT_COUNTERS + __m128i byte_cnt; +@@ -729,9 +729,12 @@ rxq_cq_process_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq, + mask = _mm_sll_epi64(ones, mask); + invalid_mask = _mm_or_si128(invalid_mask, mask); + /* D.3 check error in opcode. */ ++ adj = (comp_idx != MLX5_VPMD_DESCS_PER_LOOP && comp_idx == n); ++ mask = _mm_set_epi64x(0, adj * sizeof(uint16_t) * 8); ++ mini_mask = _mm_sll_epi64(invalid_mask, mask); + opcode = _mm_cmpeq_epi32(resp_err_check, opcode); + opcode = _mm_packs_epi32(opcode, zero); +- opcode = _mm_andnot_si128(invalid_mask, opcode); ++ opcode = _mm_andnot_si128(mini_mask, opcode); + /* D.4 mark if any error is set */ + *err |= _mm_cvtsi128_si64(opcode); + /* D.5 fill in mbuf - rearm_data and packet_type. */ +diff --git a/dpdk/drivers/net/mlx5/mlx5_stats.c b/dpdk/drivers/net/mlx5/mlx5_stats.c +index f64fa3587b..615e1d073d 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_stats.c ++++ b/dpdk/drivers/net/mlx5/mlx5_stats.c +@@ -288,10 +288,9 @@ mlx5_xstats_get_names(struct rte_eth_dev *dev, + + if (n >= mlx5_xstats_n && xstats_names) { + for (i = 0; i != mlx5_xstats_n; ++i) { +- strncpy(xstats_names[i].name, ++ strlcpy(xstats_names[i].name, + xstats_ctrl->info[i].dpdk_name, + RTE_ETH_XSTATS_NAME_SIZE); +- xstats_names[i].name[RTE_ETH_XSTATS_NAME_SIZE - 1] = 0; + } + } + mlx5_xstats_n = mlx5_txpp_xstats_get_names(dev, xstats_names, +diff --git a/dpdk/drivers/net/mlx5/mlx5_trigger.c b/dpdk/drivers/net/mlx5/mlx5_trigger.c +index f54443ed1a..b12a1dc1c7 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_trigger.c ++++ b/dpdk/drivers/net/mlx5/mlx5_trigger.c +@@ -226,17 +226,17 @@ mlx5_rxq_start(struct rte_eth_dev *dev) + if (rxq == NULL) + continue; + rxq_ctrl = rxq->ctrl; +- if (!rxq_ctrl->started) { ++ if (!rxq_ctrl->started) + if (mlx5_rxq_ctrl_prepare(dev, rxq_ctrl, i) < 0) + goto error; +- LIST_INSERT_HEAD(&priv->rxqsobj, rxq_ctrl->obj, next); +- } + ret = priv->obj_ops.rxq_obj_new(rxq); + if (ret) { + mlx5_free(rxq_ctrl->obj); + rxq_ctrl->obj = NULL; + goto error; + } ++ if (!rxq_ctrl->started) ++ LIST_INSERT_HEAD(&priv->rxqsobj, rxq_ctrl->obj, next); + rxq_ctrl->started = true; + } + return 0; +@@ -346,8 +346,8 @@ mlx5_hairpin_auto_bind(struct rte_eth_dev *dev) + ret = mlx5_devx_cmd_modify_sq(sq, &sq_attr); + if (ret) + goto error; +- rq_attr.state = MLX5_SQC_STATE_RDY; +- rq_attr.rq_state = MLX5_SQC_STATE_RST; ++ rq_attr.state = MLX5_RQC_STATE_RDY; ++ rq_attr.rq_state = MLX5_RQC_STATE_RST; + rq_attr.hairpin_peer_sq = sq->id; + rq_attr.hairpin_peer_vhca = + priv->sh->cdev->config.hca_attr.vhca_id; +@@ -601,8 +601,8 @@ mlx5_hairpin_queue_peer_bind(struct rte_eth_dev *dev, uint16_t cur_queue, + " mismatch", dev->data->port_id, cur_queue); + return -rte_errno; + } +- rq_attr.state = MLX5_SQC_STATE_RDY; +- rq_attr.rq_state = MLX5_SQC_STATE_RST; ++ rq_attr.state = MLX5_RQC_STATE_RDY; ++ rq_attr.rq_state = MLX5_RQC_STATE_RST; + rq_attr.hairpin_peer_sq = peer_info->qp_id; + rq_attr.hairpin_peer_vhca = peer_info->vhca_id; + ret = mlx5_devx_cmd_modify_rq(rxq_ctrl->obj->rq, &rq_attr); +@@ -666,7 +666,7 @@ mlx5_hairpin_queue_peer_unbind(struct rte_eth_dev *dev, uint16_t cur_queue, + return -rte_errno; + } + sq_attr.state = MLX5_SQC_STATE_RST; +- sq_attr.sq_state = MLX5_SQC_STATE_RST; ++ sq_attr.sq_state = MLX5_SQC_STATE_RDY; + ret = mlx5_devx_cmd_modify_sq(txq_ctrl->obj->sq, &sq_attr); + if (ret == 0) + txq_ctrl->hairpin_status = 0; +@@ -700,8 +700,8 @@ mlx5_hairpin_queue_peer_unbind(struct rte_eth_dev *dev, uint16_t cur_queue, + dev->data->port_id, cur_queue); + return -rte_errno; + } +- rq_attr.state = MLX5_SQC_STATE_RST; +- rq_attr.rq_state = MLX5_SQC_STATE_RST; ++ rq_attr.state = MLX5_RQC_STATE_RST; ++ rq_attr.rq_state = MLX5_RQC_STATE_RDY; + ret = mlx5_devx_cmd_modify_rq(rxq_ctrl->obj->rq, &rq_attr); + if (ret == 0) + rxq->hairpin_status = 0; +@@ -845,6 +845,11 @@ error: + txq_ctrl = mlx5_txq_get(dev, i); + if (txq_ctrl == NULL) + continue; ++ if (!txq_ctrl->is_hairpin || ++ txq_ctrl->hairpin_conf.peers[0].port != rx_port) { ++ mlx5_txq_release(dev, i); ++ continue; ++ } + rx_queue = txq_ctrl->hairpin_conf.peers[0].queue; + rte_eth_hairpin_queue_peer_unbind(rx_port, rx_queue, 0); + mlx5_hairpin_queue_peer_unbind(dev, i, 1); +@@ -896,11 +901,11 @@ mlx5_hairpin_unbind_single_port(struct rte_eth_dev *dev, uint16_t rx_port) + } + /* Indeed, only the first used queue needs to be checked. */ + if (txq_ctrl->hairpin_conf.manual_bind == 0) { ++ mlx5_txq_release(dev, i); + if (cur_port != rx_port) { + rte_errno = EINVAL; + DRV_LOG(ERR, "port %u and port %u are in" + " auto-bind mode", cur_port, rx_port); +- mlx5_txq_release(dev, i); + return -rte_errno; + } else { + return 0; +@@ -1494,13 +1499,13 @@ mlx5_traffic_enable_hws(struct rte_eth_dev *dev) + continue; + queue = mlx5_txq_get_sqn(txq); + if ((priv->representor || priv->master) && config->dv_esw_en) { +- if (mlx5_flow_hw_esw_create_sq_miss_flow(dev, queue)) { ++ if (mlx5_flow_hw_esw_create_sq_miss_flow(dev, queue, false)) { + mlx5_txq_release(dev, i); + goto error; + } + } + if (config->dv_esw_en && config->repr_matching) { +- if (mlx5_flow_hw_tx_repr_matching_flow(dev, queue)) { ++ if (mlx5_flow_hw_tx_repr_matching_flow(dev, queue, false)) { + mlx5_txq_release(dev, i); + goto error; + } +@@ -1519,6 +1524,9 @@ mlx5_traffic_enable_hws(struct rte_eth_dev *dev) + } + if (priv->isolated) + return 0; ++ if (!priv->sh->config.lacp_by_user && priv->pf_bond >= 0) ++ if (mlx5_flow_hw_lacp_rx_flow(dev)) ++ goto error; + if (dev->data->promiscuous) + flags |= MLX5_CTRL_PROMISCUOUS; + if (dev->data->all_multicast) +diff --git a/dpdk/drivers/net/mlx5/mlx5_tx.c b/dpdk/drivers/net/mlx5/mlx5_tx.c +index a13c7e937c..14e1487e59 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_tx.c ++++ b/dpdk/drivers/net/mlx5/mlx5_tx.c +@@ -107,7 +107,7 @@ mlx5_tx_error_cqe_handle(struct mlx5_txq_data *__rte_restrict txq, + mlx5_dump_debug_information(name, "MLX5 Error CQ:", + (const void *)((uintptr_t) + txq->cqes), +- sizeof(*err_cqe) * ++ sizeof(struct mlx5_cqe) * + (1 << txq->cqe_n)); + mlx5_dump_debug_information(name, "MLX5 Error SQ:", + (const void *)((uintptr_t) +diff --git a/dpdk/drivers/net/mlx5/mlx5_tx.h b/dpdk/drivers/net/mlx5/mlx5_tx.h +index a44050a1ce..ff23d87b8a 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_tx.h ++++ b/dpdk/drivers/net/mlx5/mlx5_tx.h +@@ -817,7 +817,7 @@ mlx5_tx_wseg_init(struct mlx5_txq_data *restrict txq, + struct mlx5_wqe_wseg *ws; + + ws = RTE_PTR_ADD(wqe, MLX5_WSEG_SIZE); +- ws->operation = rte_cpu_to_be_32(MLX5_WAIT_COND_CYCLIC_BIGGER); ++ ws->operation = rte_cpu_to_be_32(MLX5_WAIT_COND_CYCLIC_SMALLER); + ws->lkey = RTE_BE32(0); + ws->va_high = RTE_BE32(0); + ws->va_low = RTE_BE32(0); +@@ -1975,7 +1975,7 @@ mlx5_tx_packet_multi_inline(struct mlx5_txq_data *__rte_restrict txq, + uintptr_t start; + + mbuf = loc->mbuf; +- nxlen = rte_pktmbuf_data_len(mbuf); ++ nxlen = rte_pktmbuf_data_len(mbuf) + vlan; + /* + * Packet length exceeds the allowed inline data length, + * check whether the minimal inlining is required. +diff --git a/dpdk/drivers/net/mlx5/mlx5_txpp.c b/dpdk/drivers/net/mlx5/mlx5_txpp.c +index f853a67f58..0e1da1d5f5 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_txpp.c ++++ b/dpdk/drivers/net/mlx5/mlx5_txpp.c +@@ -969,6 +969,8 @@ mlx5_txpp_read_clock(struct rte_eth_dev *dev, uint64_t *timestamp) + { + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_dev_ctx_shared *sh = priv->sh; ++ struct mlx5_proc_priv *ppriv; ++ uint64_t ts; + int ret; + + if (sh->txpp.refcnt) { +@@ -979,7 +981,6 @@ mlx5_txpp_read_clock(struct rte_eth_dev *dev, uint64_t *timestamp) + rte_int128_t u128; + struct mlx5_cqe_ts cts; + } to; +- uint64_t ts; + + mlx5_atomic_read_cqe((rte_int128_t *)&cqe->timestamp, &to.u128); + if (to.cts.op_own >> 4) { +@@ -994,6 +995,18 @@ mlx5_txpp_read_clock(struct rte_eth_dev *dev, uint64_t *timestamp) + *timestamp = ts; + return 0; + } ++ /* Check and try to map HCA PIC BAR to allow reading real time. */ ++ ppriv = dev->process_private; ++ if (ppriv && !ppriv->hca_bar && ++ sh->dev_cap.rt_timestamp && mlx5_dev_is_pci(dev->device)) ++ mlx5_txpp_map_hca_bar(dev); ++ /* Check if we can read timestamp directly from hardware. */ ++ if (ppriv && ppriv->hca_bar) { ++ ts = MLX5_GET64(initial_seg, ppriv->hca_bar, real_time); ++ ts = mlx5_txpp_convert_rx_ts(sh, ts); ++ *timestamp = ts; ++ return 0; ++ } + /* Not supported in isolated mode - kernel does not see the CQEs. */ + if (priv->isolated || rte_eal_process_type() != RTE_PROC_PRIMARY) + return -ENOTSUP; +@@ -1050,11 +1063,9 @@ int mlx5_txpp_xstats_get_names(struct rte_eth_dev *dev __rte_unused, + + if (n >= n_used + n_txpp && xstats_names) { + for (i = 0; i < n_txpp; ++i) { +- strncpy(xstats_names[i + n_used].name, ++ strlcpy(xstats_names[i + n_used].name, + mlx5_txpp_stat_names[i], + RTE_ETH_XSTATS_NAME_SIZE); +- xstats_names[i + n_used].name +- [RTE_ETH_XSTATS_NAME_SIZE - 1] = 0; + } + } + return n_used + n_txpp; +diff --git a/dpdk/drivers/net/mlx5/mlx5_txq.c b/dpdk/drivers/net/mlx5/mlx5_txq.c +index 5543f2c570..d617784dba 100644 +--- a/dpdk/drivers/net/mlx5/mlx5_txq.c ++++ b/dpdk/drivers/net/mlx5/mlx5_txq.c +@@ -1310,8 +1310,16 @@ rte_pmd_mlx5_external_sq_enable(uint16_t port_id, uint32_t sq_num) + return -rte_errno; + } + #ifdef HAVE_MLX5_HWS_SUPPORT +- if (priv->sh->config.dv_flow_en == 2) +- return mlx5_flow_hw_esw_create_sq_miss_flow(dev, sq_num); ++ if (priv->sh->config.dv_flow_en == 2) { ++ if (mlx5_flow_hw_esw_create_sq_miss_flow(dev, sq_num, true)) ++ return -rte_errno; ++ if (priv->sh->config.repr_matching && ++ mlx5_flow_hw_tx_repr_matching_flow(dev, sq_num, true)) { ++ mlx5_flow_hw_esw_destroy_sq_miss_flow(dev, sq_num); ++ return -rte_errno; ++ } ++ return 0; ++ } + #endif + flow = mlx5_flow_create_devx_sq_miss_flow(dev, sq_num); + if (flow > 0) +diff --git a/dpdk/drivers/net/mlx5/windows/mlx5_ethdev_os.c b/dpdk/drivers/net/mlx5/windows/mlx5_ethdev_os.c +index 88d8213f55..a31e1b5494 100644 +--- a/dpdk/drivers/net/mlx5/windows/mlx5_ethdev_os.c ++++ b/dpdk/drivers/net/mlx5/windows/mlx5_ethdev_os.c +@@ -416,3 +416,33 @@ int mlx5_get_flag_dropless_rq(struct rte_eth_dev *dev) + RTE_SET_USED(dev); + return -ENOTSUP; + } ++ ++/** ++ * Unmaps HCA PCI BAR from the current process address space. ++ * ++ * @param dev ++ * Pointer to Ethernet device structure. ++ */ ++void mlx5_txpp_unmap_hca_bar(struct rte_eth_dev *dev) ++{ ++ RTE_SET_USED(dev); ++} ++ ++/** ++ * Maps HCA PCI BAR to the current process address space. ++ * Stores pointer in the process private structure allowing ++ * to read internal and real time counter directly from the HW. ++ * ++ * @param dev ++ * Pointer to Ethernet device structure. ++ * ++ * @return ++ * 0 on success and not NULL pointer to mapped area in process structure. ++ * negative otherwise and NULL pointer ++ */ ++int mlx5_txpp_map_hca_bar(struct rte_eth_dev *dev) ++{ ++ RTE_SET_USED(dev); ++ rte_errno = ENOTSUP; ++ return -ENOTSUP; ++} +diff --git a/dpdk/drivers/net/mlx5/windows/mlx5_flow_os.c b/dpdk/drivers/net/mlx5/windows/mlx5_flow_os.c +index 5013e9f012..f907b21ecc 100644 +--- a/dpdk/drivers/net/mlx5/windows/mlx5_flow_os.c ++++ b/dpdk/drivers/net/mlx5/windows/mlx5_flow_os.c +@@ -417,6 +417,12 @@ mlx5_flow_os_set_specific_workspace(struct mlx5_flow_workspace *data) + return err; + } + ++void ++mlx5_flow_os_workspace_gc_add(struct mlx5_flow_workspace *ws) ++{ ++ RTE_SET_USED(ws); ++} ++ + int + mlx5_flow_os_validate_item_esp(const struct rte_flow_item *item, + uint64_t item_flags, +diff --git a/dpdk/drivers/net/mlx5/windows/mlx5_flow_os.h b/dpdk/drivers/net/mlx5/windows/mlx5_flow_os.h +index 1c1c17fc41..856d8ba948 100644 +--- a/dpdk/drivers/net/mlx5/windows/mlx5_flow_os.h ++++ b/dpdk/drivers/net/mlx5/windows/mlx5_flow_os.h +@@ -473,4 +473,12 @@ mlx5_flow_os_validate_item_esp(const struct rte_flow_item *item, + uint8_t target_protocol, + struct rte_flow_error *error); + ++/** ++ * Add per thread workspace to the global list for garbage collection. ++ * ++ * @param[in] ws ++ * Pointer to the flow workspace. ++ */ ++void mlx5_flow_os_workspace_gc_add(struct mlx5_flow_workspace *ws); ++ + #endif /* RTE_PMD_MLX5_FLOW_OS_H_ */ +diff --git a/dpdk/drivers/net/mlx5/windows/mlx5_os.c b/dpdk/drivers/net/mlx5/windows/mlx5_os.c +index 77f04cc931..f401264b61 100644 +--- a/dpdk/drivers/net/mlx5/windows/mlx5_os.c ++++ b/dpdk/drivers/net/mlx5/windows/mlx5_os.c +@@ -193,8 +193,8 @@ mlx5_os_capabilities_prepare(struct mlx5_dev_ctx_shared *sh) + * Once DPDK supports it, take max size from device attr. + */ + sh->dev_cap.ind_table_max_size = +- RTE_MIN(1 << hca_attr->rss_ind_tbl_cap, +- (unsigned int)RTE_ETH_RSS_RETA_SIZE_512); ++ RTE_MIN((uint32_t)1 << hca_attr->rss_ind_tbl_cap, ++ (uint32_t)RTE_ETH_RSS_RETA_SIZE_512); + DRV_LOG(DEBUG, "Maximum Rx indirection table size is %u", + sh->dev_cap.ind_table_max_size); + } +diff --git a/dpdk/drivers/net/mvneta/mvneta_ethdev.c b/dpdk/drivers/net/mvneta/mvneta_ethdev.c +index c4355a3f64..ab643709ee 100644 +--- a/dpdk/drivers/net/mvneta/mvneta_ethdev.c ++++ b/dpdk/drivers/net/mvneta/mvneta_ethdev.c +@@ -376,6 +376,10 @@ mvneta_dev_start(struct rte_eth_dev *dev) + goto out; + } + ++ /* start rx queues */ ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ + /* start tx queues */ + for (i = 0; i < dev->data->nb_tx_queues; i++) + dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; +@@ -400,6 +404,7 @@ static int + mvneta_dev_stop(struct rte_eth_dev *dev) + { + struct mvneta_priv *priv = dev->data->dev_private; ++ uint16_t i; + + dev->data->dev_started = 0; + +@@ -412,6 +417,14 @@ mvneta_dev_stop(struct rte_eth_dev *dev) + + priv->ppio = NULL; + ++ /* stop rx queues */ ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ ++ /* stop tx queues */ ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ + return 0; + } + +diff --git a/dpdk/drivers/net/mvpp2/mrvl_ethdev.c b/dpdk/drivers/net/mvpp2/mrvl_ethdev.c +index 8fd3211283..177b8165f3 100644 +--- a/dpdk/drivers/net/mvpp2/mrvl_ethdev.c ++++ b/dpdk/drivers/net/mvpp2/mrvl_ethdev.c +@@ -951,6 +951,9 @@ mrvl_dev_start(struct rte_eth_dev *dev) + goto out; + } + ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ + mrvl_flow_init(dev); + mrvl_mtr_init(dev); + mrvl_set_tx_function(dev); +@@ -1076,6 +1079,13 @@ mrvl_flush_bpool(struct rte_eth_dev *dev) + static int + mrvl_dev_stop(struct rte_eth_dev *dev) + { ++ uint16_t i; ++ ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ + return mrvl_dev_set_link_down(dev); + } + +diff --git a/dpdk/drivers/net/netvsc/hn_rndis.c b/dpdk/drivers/net/netvsc/hn_rndis.c +index e6f1f28768..fe36274df8 100644 +--- a/dpdk/drivers/net/netvsc/hn_rndis.c ++++ b/dpdk/drivers/net/netvsc/hn_rndis.c +@@ -35,7 +35,7 @@ + #include "hn_rndis.h" + #include "ndis.h" + +-#define RNDIS_TIMEOUT_SEC 5 ++#define RNDIS_TIMEOUT_SEC 60 + #define RNDIS_DELAY_MS 10 + + #define HN_RNDIS_XFER_SIZE 0x4000 +@@ -329,7 +329,8 @@ void hn_rndis_receive_response(struct hn_data *hv, + + hn_rndis_dump(data); + +- if (len < sizeof(3 * sizeof(uint32_t))) { ++ /* Check we can read first three data fields from RNDIS header */ ++ if (len < 3 * sizeof(uint32_t)) { + PMD_DRV_LOG(ERR, + "missing RNDIS header %u", len); + return; +diff --git a/dpdk/drivers/net/nfp/flower/nfp_flower.c b/dpdk/drivers/net/nfp/flower/nfp_flower.c +index e447258d97..5896d208d0 100644 +--- a/dpdk/drivers/net/nfp/flower/nfp_flower.c ++++ b/dpdk/drivers/net/nfp/flower/nfp_flower.c +@@ -25,7 +25,6 @@ + #include "nfp_flower_cmsg.h" + + #define CTRL_VNIC_NB_DESC 512 +-#define DEFAULT_FLBUF_SIZE 9216 + + static void + nfp_pf_repr_enable_queues(struct rte_eth_dev *dev) +@@ -85,6 +84,7 @@ int + nfp_flower_pf_start(struct rte_eth_dev *dev) + { + int ret; ++ uint16_t i; + uint32_t new_ctrl; + uint32_t update = 0; + struct nfp_net_hw *hw; +@@ -137,6 +137,11 @@ nfp_flower_pf_start(struct rte_eth_dev *dev) + return -EIO; + } + ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ + return 0; + } + +@@ -159,11 +164,13 @@ nfp_flower_pf_stop(struct rte_eth_dev *dev) + for (i = 0; i < dev->data->nb_tx_queues; i++) { + this_tx_q = (struct nfp_net_txq *)dev->data->tx_queues[i]; + nfp_net_reset_tx_queue(this_tx_q); ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; + } + + for (i = 0; i < dev->data->nb_rx_queues; i++) { + this_rx_q = (struct nfp_net_rxq *)dev->data->rx_queues[i]; + nfp_net_reset_rx_queue(this_rx_q); ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; + } + + if (rte_eal_process_type() == RTE_PROC_PRIMARY) +@@ -217,8 +224,6 @@ nfp_flower_pf_close(struct rte_eth_dev *dev) + + nn_cfg_writeb(hw, NFP_NET_CFG_LSC, 0xff); + +- rte_eth_dev_release_port(dev); +- + /* Now it is safe to free all PF resources */ + PMD_DRV_LOG(INFO, "Freeing PF resources"); + nfp_cpp_area_free(pf_dev->ctrl_area); +@@ -451,7 +456,7 @@ nfp_flower_pf_recv_pkts(void *rx_queue, + rxds->vals[1] = 0; + dma_addr = rte_cpu_to_le_64(RTE_MBUF_DMA_ADDR_DEFAULT(new_mb)); + rxds->fld.dd = 0; +- rxds->fld.dma_addr_hi = (dma_addr >> 32) & 0xff; ++ rxds->fld.dma_addr_hi = (dma_addr >> 32) & 0xffff; + rxds->fld.dma_addr_lo = dma_addr & 0xffffffff; + nb_hold++; + +@@ -631,13 +636,6 @@ nfp_flower_init_vnic_common(struct nfp_net_hw *hw, const char *vnic_type) + pf_dev = hw->pf_dev; + pci_dev = hw->pf_dev->pci_dev; + +- /* NFP can not handle DMA addresses requiring more than 40 bits */ +- if (rte_mem_check_dma_mask(40)) { +- PMD_INIT_LOG(ERR, "Device %s can not be used: restricted dma mask to 40 bits!\n", +- pci_dev->device.name); +- return -ENODEV; +- }; +- + hw->device_id = pci_dev->id.device_id; + hw->vendor_id = pci_dev->id.vendor_id; + hw->subsystem_device_id = pci_dev->id.subsystem_device_id; +@@ -666,6 +664,9 @@ nfp_flower_init_vnic_common(struct nfp_net_hw *hw, const char *vnic_type) + hw->mtu = hw->max_mtu; + hw->flbufsz = DEFAULT_FLBUF_SIZE; + ++ if (nfp_net_check_dma_mask(hw, pci_dev->name) != 0) ++ return -ENODEV; ++ + /* read the Rx offset configured from firmware */ + if (NFD_CFG_MAJOR_VERSION_of(hw->ver) < 2) + hw->rx_offset = NFP_NET_RX_OFFSET; +@@ -695,6 +696,7 @@ nfp_flower_init_ctrl_vnic(struct nfp_net_hw *hw) + int ret = 0; + uint16_t n_txq; + uint16_t n_rxq; ++ const char *pci_name; + unsigned int numa_node; + struct rte_mempool *mp; + struct nfp_net_rxq *rxq; +@@ -703,6 +705,9 @@ nfp_flower_init_ctrl_vnic(struct nfp_net_hw *hw) + struct rte_eth_dev *eth_dev; + const struct rte_memzone *tz; + struct nfp_app_fw_flower *app_fw_flower; ++ char ctrl_rxring_name[RTE_MEMZONE_NAMESIZE]; ++ char ctrl_txring_name[RTE_MEMZONE_NAMESIZE]; ++ char ctrl_pktmbuf_pool_name[RTE_MEMZONE_NAMESIZE]; + + /* Set up some pointers here for ease of use */ + pf_dev = hw->pf_dev; +@@ -734,9 +739,14 @@ nfp_flower_init_ctrl_vnic(struct nfp_net_hw *hw) + goto eth_dev_cleanup; + } + ++ pci_name = strchr(pf_dev->pci_dev->name, ':') + 1; ++ + /* Create a mbuf pool for the ctrl vNIC */ + numa_node = rte_socket_id(); +- app_fw_flower->ctrl_pktmbuf_pool = rte_pktmbuf_pool_create("ctrl_mbuf_pool", ++ snprintf(ctrl_pktmbuf_pool_name, sizeof(ctrl_pktmbuf_pool_name), ++ "%s_ctrlmp", pci_name); ++ app_fw_flower->ctrl_pktmbuf_pool = ++ rte_pktmbuf_pool_create(ctrl_pktmbuf_pool_name, + 4 * CTRL_VNIC_NB_DESC, 64, 0, 9216, numa_node); + if (app_fw_flower->ctrl_pktmbuf_pool == NULL) { + PMD_INIT_LOG(ERR, "Create mbuf pool for ctrl vnic failed"); +@@ -773,6 +783,7 @@ nfp_flower_init_ctrl_vnic(struct nfp_net_hw *hw) + eth_dev->data->nb_rx_queues = n_txq; + eth_dev->data->dev_private = hw; + ++ snprintf(ctrl_rxring_name, sizeof(ctrl_rxring_name), "%s_ctrx_ring", pci_name); + /* Set up the Rx queues */ + for (i = 0; i < n_rxq; i++) { + rxq = rte_zmalloc_socket("ethdev RX queue", +@@ -811,7 +822,7 @@ nfp_flower_init_ctrl_vnic(struct nfp_net_hw *hw) + * handle the maximum ring size is allocated in order to allow for + * resizing in later calls to the queue setup function. + */ +- tz = rte_eth_dma_zone_reserve(eth_dev, "ctrl_rx_ring", i, ++ tz = rte_eth_dma_zone_reserve(eth_dev, ctrl_rxring_name, i, + sizeof(struct nfp_net_rx_desc) * NFP_NET_MAX_RX_DESC, + NFP_MEMZONE_ALIGN, numa_node); + if (tz == NULL) { +@@ -830,7 +841,7 @@ nfp_flower_init_ctrl_vnic(struct nfp_net_hw *hw) + sizeof(*rxq->rxbufs) * CTRL_VNIC_NB_DESC, + RTE_CACHE_LINE_SIZE, numa_node); + if (rxq->rxbufs == NULL) { +- rte_eth_dma_zone_free(eth_dev, "ctrl_rx_ring", i); ++ rte_eth_dma_zone_free(eth_dev, ctrl_rxring_name, i); + rte_free(rxq); + ret = -ENOMEM; + goto rx_queue_setup_cleanup; +@@ -848,6 +859,7 @@ nfp_flower_init_ctrl_vnic(struct nfp_net_hw *hw) + nn_cfg_writeb(hw, NFP_NET_CFG_RXR_SZ(i), rte_log2_u32(CTRL_VNIC_NB_DESC)); + } + ++ snprintf(ctrl_txring_name, sizeof(ctrl_txring_name), "%s_cttx_ring", pci_name); + /* Set up the Tx queues */ + for (i = 0; i < n_txq; i++) { + txq = rte_zmalloc_socket("ethdev TX queue", +@@ -866,7 +878,7 @@ nfp_flower_init_ctrl_vnic(struct nfp_net_hw *hw) + * handle the maximum ring size is allocated in order to allow for + * resizing in later calls to the queue setup function. + */ +- tz = rte_eth_dma_zone_reserve(eth_dev, "ctrl_tx_ring", i, ++ tz = rte_eth_dma_zone_reserve(eth_dev, ctrl_txring_name, i, + sizeof(struct nfp_net_nfd3_tx_desc) * NFP_NET_MAX_TX_DESC, + NFP_MEMZONE_ALIGN, numa_node); + if (tz == NULL) { +@@ -896,7 +908,7 @@ nfp_flower_init_ctrl_vnic(struct nfp_net_hw *hw) + sizeof(*txq->txbufs) * CTRL_VNIC_NB_DESC, + RTE_CACHE_LINE_SIZE, numa_node); + if (txq->txbufs == NULL) { +- rte_eth_dma_zone_free(eth_dev, "ctrl_tx_ring", i); ++ rte_eth_dma_zone_free(eth_dev, ctrl_txring_name, i); + rte_free(txq); + ret = -ENOMEM; + goto tx_queue_setup_cleanup; +@@ -921,7 +933,7 @@ tx_queue_setup_cleanup: + txq = eth_dev->data->tx_queues[i]; + if (txq != NULL) { + rte_free(txq->txbufs); +- rte_eth_dma_zone_free(eth_dev, "ctrl_tx_ring", i); ++ rte_eth_dma_zone_free(eth_dev, ctrl_txring_name, i); + rte_free(txq); + } + } +@@ -930,7 +942,7 @@ rx_queue_setup_cleanup: + rxq = eth_dev->data->rx_queues[i]; + if (rxq != NULL) { + rte_free(rxq->rxbufs); +- rte_eth_dma_zone_free(eth_dev, "ctrl_rx_ring", i); ++ rte_eth_dma_zone_free(eth_dev, ctrl_rxring_name, i); + rte_free(rxq); + } + } +@@ -951,28 +963,35 @@ static void + nfp_flower_cleanup_ctrl_vnic(struct nfp_net_hw *hw) + { + uint32_t i; ++ const char *pci_name; + struct nfp_net_rxq *rxq; + struct nfp_net_txq *txq; + struct rte_eth_dev *eth_dev; + struct nfp_app_fw_flower *app_fw_flower; ++ char ctrl_txring_name[RTE_MEMZONE_NAMESIZE]; ++ char ctrl_rxring_name[RTE_MEMZONE_NAMESIZE]; + + eth_dev = hw->eth_dev; + app_fw_flower = NFP_PRIV_TO_APP_FW_FLOWER(hw->pf_dev->app_fw_priv); + ++ pci_name = strchr(app_fw_flower->pf_hw->pf_dev->pci_dev->name, ':') + 1; ++ ++ snprintf(ctrl_txring_name, sizeof(ctrl_txring_name), "%s_cttx_ring", pci_name); + for (i = 0; i < hw->max_tx_queues; i++) { + txq = eth_dev->data->tx_queues[i]; + if (txq != NULL) { + rte_free(txq->txbufs); +- rte_eth_dma_zone_free(eth_dev, "ctrl_tx_ring", i); ++ rte_eth_dma_zone_free(eth_dev, ctrl_txring_name, i); + rte_free(txq); + } + } + ++ snprintf(ctrl_rxring_name, sizeof(ctrl_rxring_name), "%s_ctrx_ring", pci_name); + for (i = 0; i < hw->max_rx_queues; i++) { + rxq = eth_dev->data->rx_queues[i]; + if (rxq != NULL) { + rte_free(rxq->rxbufs); +- rte_eth_dma_zone_free(eth_dev, "ctrl_rx_ring", i); ++ rte_eth_dma_zone_free(eth_dev, ctrl_rxring_name, i); + rte_free(rxq); + } + } +diff --git a/dpdk/drivers/net/nfp/flower/nfp_flower_ctrl.c b/dpdk/drivers/net/nfp/flower/nfp_flower_ctrl.c +index 3631e764fe..1c6340f3d7 100644 +--- a/dpdk/drivers/net/nfp/flower/nfp_flower_ctrl.c ++++ b/dpdk/drivers/net/nfp/flower/nfp_flower_ctrl.c +@@ -123,7 +123,7 @@ nfp_flower_ctrl_vnic_recv(void *rx_queue, + rxds->vals[1] = 0; + dma_addr = rte_cpu_to_le_64(RTE_MBUF_DMA_ADDR_DEFAULT(new_mb)); + rxds->fld.dd = 0; +- rxds->fld.dma_addr_hi = (dma_addr >> 32) & 0xff; ++ rxds->fld.dma_addr_hi = (dma_addr >> 32) & 0xffff; + rxds->fld.dma_addr_lo = dma_addr & 0xffffffff; + nb_hold++; + +diff --git a/dpdk/drivers/net/nfp/flower/nfp_flower_representor.c b/dpdk/drivers/net/nfp/flower/nfp_flower_representor.c +index 5809c838b3..32c4574bdc 100644 +--- a/dpdk/drivers/net/nfp/flower/nfp_flower_representor.c ++++ b/dpdk/drivers/net/nfp/flower/nfp_flower_representor.c +@@ -300,6 +300,7 @@ nfp_flower_repr_dev_start(struct rte_eth_dev *dev) + { + struct nfp_flower_representor *repr; + struct nfp_app_fw_flower *app_fw_flower; ++ uint16_t i; + + repr = (struct nfp_flower_representor *)dev->data->dev_private; + app_fw_flower = repr->app_fw_flower; +@@ -311,6 +312,11 @@ nfp_flower_repr_dev_start(struct rte_eth_dev *dev) + + nfp_flower_cmsg_port_mod(app_fw_flower, repr->port_id, true); + ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ + return 0; + } + +@@ -319,6 +325,7 @@ nfp_flower_repr_dev_stop(struct rte_eth_dev *dev) + { + struct nfp_flower_representor *repr; + struct nfp_app_fw_flower *app_fw_flower; ++ uint16_t i; + + repr = (struct nfp_flower_representor *)dev->data->dev_private; + app_fw_flower = repr->app_fw_flower; +@@ -330,6 +337,11 @@ nfp_flower_repr_dev_stop(struct rte_eth_dev *dev) + repr->nfp_idx, 0); + } + ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ + return 0; + } + +@@ -528,7 +540,7 @@ static const struct eth_dev_ops nfp_flower_pf_repr_dev_ops = { + .stats_reset = nfp_flower_repr_stats_reset, + + .promiscuous_enable = nfp_net_promisc_enable, +- .promiscuous_disable = nfp_net_promisc_enable, ++ .promiscuous_disable = nfp_net_promisc_disable, + + .mac_addr_set = nfp_flower_repr_mac_addr_set, + }; +@@ -549,7 +561,7 @@ static const struct eth_dev_ops nfp_flower_repr_dev_ops = { + .stats_reset = nfp_flower_repr_stats_reset, + + .promiscuous_enable = nfp_net_promisc_enable, +- .promiscuous_disable = nfp_net_promisc_enable, ++ .promiscuous_disable = nfp_net_promisc_disable, + + .mac_addr_set = nfp_flower_repr_mac_addr_set, + +@@ -637,6 +649,7 @@ nfp_flower_repr_init(struct rte_eth_dev *eth_dev, + void *init_params) + { + int ret; ++ uint16_t index; + unsigned int numa_node; + char ring_name[RTE_ETH_NAME_MAX_LEN]; + struct nfp_app_fw_flower *app_fw_flower; +@@ -710,10 +723,13 @@ nfp_flower_repr_init(struct rte_eth_dev *eth_dev, + } + + /* Add repr to correct array */ +- if (repr->repr_type == NFP_REPR_TYPE_PHYS_PORT) +- app_fw_flower->phy_reprs[repr->nfp_idx] = repr; +- else +- app_fw_flower->vf_reprs[repr->vf_id] = repr; ++ if (repr->repr_type == NFP_REPR_TYPE_PHYS_PORT) { ++ index = NFP_FLOWER_CMSG_PORT_PHYS_PORT_NUM(repr->port_id); ++ app_fw_flower->phy_reprs[index] = repr; ++ } else { ++ index = repr->vf_id; ++ app_fw_flower->vf_reprs[index] = repr; ++ } + + return 0; + +@@ -730,7 +746,9 @@ nfp_flower_repr_alloc(struct nfp_app_fw_flower *app_fw_flower) + { + int i; + int ret; ++ const char *pci_name; + struct rte_eth_dev *eth_dev; ++ struct rte_pci_device *pci_dev; + struct nfp_eth_table *nfp_eth_table; + struct nfp_eth_table_port *eth_port; + struct nfp_flower_representor flower_repr = { +@@ -753,7 +771,13 @@ nfp_flower_repr_alloc(struct nfp_app_fw_flower *app_fw_flower) + + /* PF vNIC reprs get a random MAC address */ + rte_eth_random_addr(flower_repr.mac_addr.addr_bytes); +- sprintf(flower_repr.name, "flower_repr_pf"); ++ ++ pci_dev = app_fw_flower->pf_hw->pf_dev->pci_dev; ++ ++ pci_name = strchr(pci_dev->name, ':') + 1; ++ ++ snprintf(flower_repr.name, sizeof(flower_repr.name), ++ "%s_repr_pf", pci_name); + + /* Create a eth_dev for this representor */ + ret = rte_eth_dev_create(eth_dev->device, flower_repr.name, +@@ -775,7 +799,8 @@ nfp_flower_repr_alloc(struct nfp_app_fw_flower *app_fw_flower) + /* Copy the real mac of the interface to the representor struct */ + rte_ether_addr_copy((struct rte_ether_addr *)eth_port->mac_addr, + &flower_repr.mac_addr); +- sprintf(flower_repr.name, "flower_repr_p%d", i); ++ snprintf(flower_repr.name, sizeof(flower_repr.name), ++ "%s_repr_p%d", pci_name, i); + + /* + * Create a eth_dev for this representor +@@ -806,7 +831,8 @@ nfp_flower_repr_alloc(struct nfp_app_fw_flower *app_fw_flower) + + /* VF reprs get a random MAC address */ + rte_eth_random_addr(flower_repr.mac_addr.addr_bytes); +- sprintf(flower_repr.name, "flower_repr_vf%d", i); ++ snprintf(flower_repr.name, sizeof(flower_repr.name), ++ "%s_repr_vf%d", pci_name, i); + + /* This will also allocate private memory for the device*/ + ret = rte_eth_dev_create(eth_dev->device, flower_repr.name, +diff --git a/dpdk/drivers/net/nfp/nfp_common.c b/dpdk/drivers/net/nfp/nfp_common.c +index 71711bfa22..33613bb2b3 100644 +--- a/dpdk/drivers/net/nfp/nfp_common.c ++++ b/dpdk/drivers/net/nfp/nfp_common.c +@@ -279,7 +279,7 @@ int + nfp_net_set_mac_addr(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr) + { + struct nfp_net_hw *hw; +- uint32_t update, ctrl; ++ uint32_t update, new_ctrl; + + hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); + if ((hw->ctrl & NFP_NET_CFG_CTRL_ENABLE) && +@@ -294,14 +294,18 @@ nfp_net_set_mac_addr(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr) + + /* Signal the NIC about the change */ + update = NFP_NET_CFG_UPDATE_MACADDR; +- ctrl = hw->ctrl; ++ new_ctrl = hw->ctrl; + if ((hw->ctrl & NFP_NET_CFG_CTRL_ENABLE) && + (hw->cap & NFP_NET_CFG_CTRL_LIVE_ADDR)) +- ctrl |= NFP_NET_CFG_CTRL_LIVE_ADDR; +- if (nfp_net_reconfig(hw, ctrl, update) < 0) { ++ new_ctrl |= NFP_NET_CFG_CTRL_LIVE_ADDR; ++ ++ if (nfp_net_reconfig(hw, new_ctrl, update) < 0) { + PMD_INIT_LOG(INFO, "MAC address update failed"); + return -EIO; + } ++ ++ hw->ctrl = new_ctrl; ++ + return 0; + } + +@@ -885,7 +889,7 @@ nfp_net_dev_link_status_print(struct rte_eth_dev *dev) + * If MSI-X auto-masking is enabled clear the mask bit, otherwise + * clear the ICR for the entry. + */ +-static void ++void + nfp_net_irq_unmask(struct rte_eth_dev *dev) + { + struct nfp_net_hw *hw; +@@ -977,9 +981,9 @@ nfp_net_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) + return -EBUSY; + } + +- /* MTU larger then current mbufsize not supported */ ++ /* MTU larger than current mbufsize not supported */ + if (mtu > hw->flbufsz) { +- PMD_DRV_LOG(ERR, "MTU (%u) larger then current mbufsize (%u) not supported", ++ PMD_DRV_LOG(ERR, "MTU (%u) larger than current mbufsize (%u) not supported", + mtu, hw->flbufsz); + return -ERANGE; + } +@@ -1256,7 +1260,7 @@ nfp_net_rss_hash_conf_get(struct rte_eth_dev *dev, + cfg_rss_ctrl = nn_cfg_readl(hw, NFP_NET_CFG_RSS_CTRL); + + if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV4) +- rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_TCP | RTE_ETH_RSS_NONFRAG_IPV4_UDP; ++ rss_hf |= RTE_ETH_RSS_IPV4; + + if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV4_TCP) + rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_TCP; +@@ -1271,7 +1275,7 @@ nfp_net_rss_hash_conf_get(struct rte_eth_dev *dev, + rss_hf |= RTE_ETH_RSS_NONFRAG_IPV6_UDP; + + if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV6) +- rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_UDP | RTE_ETH_RSS_NONFRAG_IPV6_UDP; ++ rss_hf |= RTE_ETH_RSS_IPV6; + + /* Propagate current RSS hash functions to caller */ + rss_conf->rss_hf = rss_hf; +@@ -1337,6 +1341,7 @@ nfp_net_stop_rx_queue(struct rte_eth_dev *dev) + for (i = 0; i < dev->data->nb_rx_queues; i++) { + this_rx_q = (struct nfp_net_rxq *)dev->data->rx_queues[i]; + nfp_net_reset_rx_queue(this_rx_q); ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; + } + } + +@@ -1362,6 +1367,7 @@ nfp_net_stop_tx_queue(struct rte_eth_dev *dev) + for (i = 0; i < dev->data->nb_tx_queues; i++) { + this_tx_q = (struct nfp_net_txq *)dev->data->tx_queues[i]; + nfp_net_reset_tx_queue(this_tx_q); ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; + } + } + +@@ -1413,6 +1419,24 @@ nfp_net_set_vxlan_port(struct nfp_net_hw *hw, + RTE_LOG_REGISTER_SUFFIX(nfp_logtype_init, init, NOTICE); + RTE_LOG_REGISTER_SUFFIX(nfp_logtype_driver, driver, NOTICE); + RTE_LOG_REGISTER_SUFFIX(nfp_logtype_cpp, cpp, NOTICE); ++/* ++ * The firmware with NFD3 can not handle DMA address requiring more ++ * than 40 bits ++ */ ++int ++nfp_net_check_dma_mask(struct nfp_net_hw *hw, char *name) ++{ ++ if (NFD_CFG_CLASS_VER_of(hw->ver) == NFP_NET_CFG_VERSION_DP_NFD3 && ++ rte_mem_check_dma_mask(40) != 0) { ++ PMD_DRV_LOG(ERR, ++ "The device %s can't be used: restricted dma mask to 40 bits!", ++ name); ++ return -ENODEV; ++ } ++ ++ return 0; ++} ++ + /* + * Local variables: + * c-file-style: "Linux" +diff --git a/dpdk/drivers/net/nfp/nfp_common.h b/dpdk/drivers/net/nfp/nfp_common.h +index 36c19b47e4..d1a07f5a72 100644 +--- a/dpdk/drivers/net/nfp/nfp_common.h ++++ b/dpdk/drivers/net/nfp/nfp_common.h +@@ -111,6 +111,7 @@ struct nfp_net_adapter; + + /* Maximum supported NFP frame size (MTU + layer 2 headers) */ + #define NFP_FRAME_SIZE_MAX 10048 ++#define DEFAULT_FLBUF_SIZE 9216 + + #include + #include +@@ -447,6 +448,8 @@ void nfp_net_close_rx_queue(struct rte_eth_dev *dev); + void nfp_net_stop_tx_queue(struct rte_eth_dev *dev); + void nfp_net_close_tx_queue(struct rte_eth_dev *dev); + int nfp_net_set_vxlan_port(struct nfp_net_hw *hw, size_t idx, uint16_t port); ++int nfp_net_check_dma_mask(struct nfp_net_hw *hw, char *name); ++void nfp_net_irq_unmask(struct rte_eth_dev *dev); + + #define NFP_NET_DEV_PRIVATE_TO_HW(adapter)\ + (&((struct nfp_net_adapter *)adapter)->hw) +diff --git a/dpdk/drivers/net/nfp/nfp_ethdev.c b/dpdk/drivers/net/nfp/nfp_ethdev.c +index 0956ea81df..9f940a12b6 100644 +--- a/dpdk/drivers/net/nfp/nfp_ethdev.c ++++ b/dpdk/drivers/net/nfp/nfp_ethdev.c +@@ -70,6 +70,7 @@ nfp_net_start(struct rte_eth_dev *dev) + struct rte_eth_conf *dev_conf; + struct rte_eth_rxmode *rxmode; + uint32_t intr_vector; ++ uint16_t i; + int ret; + + hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); +@@ -154,6 +155,8 @@ nfp_net_start(struct rte_eth_dev *dev) + + nn_cfg_writel(hw, NFP_NET_CFG_CTRL, new_ctrl); + if (nfp_net_reconfig(hw, new_ctrl, update) < 0) ++ hw->ctrl = new_ctrl; ++ + return -EIO; + + /* +@@ -172,7 +175,10 @@ nfp_net_start(struct rte_eth_dev *dev) + nfp_eth_set_configured(dev->process_private, + hw->nfp_idx, 1); + +- hw->ctrl = new_ctrl; ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; + + return 0; + +@@ -298,7 +304,6 @@ nfp_net_close(struct rte_eth_dev *dev) + /* Mark this port as unused and free device priv resources*/ + nn_cfg_writeb(hw, NFP_NET_CFG_LSC, 0xff); + app_fw_nic->ports[hw->idx] = NULL; +- rte_eth_dev_release_port(dev); + + for (i = 0; i < app_fw_nic->total_phyports; i++) { + /* Check to see if ports are still in use */ +@@ -517,14 +522,6 @@ nfp_net_init(struct rte_eth_dev *eth_dev) + /* Use backpointer to the CoreNIC app struct */ + app_fw_nic = NFP_PRIV_TO_APP_FW_NIC(pf_dev->app_fw_priv); + +- /* NFP can not handle DMA addresses requiring more than 40 bits */ +- if (rte_mem_check_dma_mask(40)) { +- RTE_LOG(ERR, PMD, +- "device %s can not be used: restricted dma mask to 40 bits!\n", +- pci_dev->device.name); +- return -ENODEV; +- } +- + port = ((struct nfp_net_hw *)eth_dev->data->dev_private)->idx; + if (port < 0 || port > 7) { + PMD_DRV_LOG(ERR, "Port value is wrong"); +@@ -572,6 +569,9 @@ nfp_net_init(struct rte_eth_dev *eth_dev) + + hw->ver = nn_cfg_readl(hw, NFP_NET_CFG_VERSION); + ++ if (nfp_net_check_dma_mask(hw, pci_dev->name) != 0) ++ return -ENODEV; ++ + if (nfp_net_ethdev_ops_mount(hw, eth_dev)) + return -EINVAL; + +@@ -609,6 +609,7 @@ nfp_net_init(struct rte_eth_dev *eth_dev) + hw->cap = nn_cfg_readl(hw, NFP_NET_CFG_CAP); + hw->max_mtu = nn_cfg_readl(hw, NFP_NET_CFG_MAX_MTU); + hw->mtu = RTE_ETHER_MTU; ++ hw->flbufsz = DEFAULT_FLBUF_SIZE; + + /* VLAN insertion is incompatible with LSOv2 */ + if (hw->cap & NFP_NET_CFG_CTRL_LSO2) +@@ -690,6 +691,8 @@ nfp_net_init(struct rte_eth_dev *eth_dev) + nfp_net_dev_interrupt_handler, (void *)eth_dev); + /* Telling the firmware about the LSC interrupt entry */ + nn_cfg_writeb(hw, NFP_NET_CFG_LSC, NFP_NET_IRQ_LSC_IDX); ++ /* Unmask the LSC interrupt */ ++ nfp_net_irq_unmask(eth_dev); + /* Recording current stats counters values */ + nfp_net_stats_reset(eth_dev); + +@@ -724,7 +727,7 @@ nfp_fw_upload(struct rte_pci_device *dev, struct nfp_nsp *nsp, char *card) + goto load_fw; + /* Then try the PCI name */ + snprintf(fw_name, sizeof(fw_name), "%s/pci-%s.nffw", DEFAULT_FW_PATH, +- dev->device.name); ++ dev->name); + + PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name); + if (rte_firmware_read(fw_name, &fw_buf, &fsize) == 0) +@@ -930,9 +933,11 @@ app_cleanup: + static int + nfp_pf_init(struct rte_pci_device *pci_dev) + { +- int ret; ++ uint32_t i; ++ int ret = 0; + int err = 0; + uint64_t addr; ++ uint32_t cpp_id; + struct nfp_cpp *cpp; + enum nfp_app_fw_id app_fw_id; + struct nfp_pf_dev *pf_dev; +@@ -976,6 +981,10 @@ nfp_pf_init(struct rte_pci_device *pci_dev) + goto hwinfo_cleanup; + } + ++ /* Force the physical port down to clear the possible DMA error */ ++ for (i = 0; i < nfp_eth_table->count; i++) ++ nfp_eth_set_configured(cpp, nfp_eth_table->ports[i].index, 0); ++ + if (nfp_fw_setup(pci_dev, cpp, nfp_eth_table, hwinfo)) { + PMD_INIT_LOG(ERR, "Error when uploading firmware"); + ret = -EIO; +@@ -1032,7 +1041,8 @@ nfp_pf_init(struct rte_pci_device *pci_dev) + goto pf_cleanup; + } + +- pf_dev->hw_queues = nfp_cpp_map_area(pf_dev->cpp, 0, 0, ++ cpp_id = NFP_CPP_ISLAND_ID(0, NFP_CPP_ACTION_RW, 0, 0); ++ pf_dev->hw_queues = nfp_cpp_map_area(pf_dev->cpp, cpp_id, + addr, NFP_QCP_QUEUE_AREA_SZ, + &pf_dev->hwqueues_area); + if (pf_dev->hw_queues == NULL) { +diff --git a/dpdk/drivers/net/nfp/nfp_ethdev_vf.c b/dpdk/drivers/net/nfp/nfp_ethdev_vf.c +index d1427b63bc..435127604a 100644 +--- a/dpdk/drivers/net/nfp/nfp_ethdev_vf.c ++++ b/dpdk/drivers/net/nfp/nfp_ethdev_vf.c +@@ -45,6 +45,7 @@ nfp_netvf_start(struct rte_eth_dev *dev) + struct rte_eth_conf *dev_conf; + struct rte_eth_rxmode *rxmode; + uint32_t intr_vector; ++ uint16_t i; + int ret; + + hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); +@@ -113,6 +114,8 @@ nfp_netvf_start(struct rte_eth_dev *dev) + if (nfp_net_reconfig(hw, new_ctrl, update) < 0) + return -EIO; + ++ hw->ctrl = new_ctrl; ++ + /* + * Allocating rte mbufs for configured rx queues. + * This requires queues being enabled before +@@ -122,7 +125,10 @@ nfp_netvf_start(struct rte_eth_dev *dev) + goto error; + } + +- hw->ctrl = new_ctrl; ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; + + return 0; + +@@ -291,14 +297,6 @@ nfp_netvf_init(struct rte_eth_dev *eth_dev) + + pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + +- /* NFP can not handle DMA addresses requiring more than 40 bits */ +- if (rte_mem_check_dma_mask(40)) { +- RTE_LOG(ERR, PMD, +- "device %s can not be used: restricted dma mask to 40 bits!\n", +- pci_dev->device.name); +- return -ENODEV; +- } +- + hw = NFP_NET_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); + + hw->ctrl_bar = (uint8_t *)pci_dev->mem_resource[0].addr; +@@ -312,6 +310,9 @@ nfp_netvf_init(struct rte_eth_dev *eth_dev) + + hw->ver = nn_cfg_readl(hw, NFP_NET_CFG_VERSION); + ++ if (nfp_net_check_dma_mask(hw, pci_dev->name) != 0) ++ return -ENODEV; ++ + if (nfp_netvf_ethdev_ops_mount(hw, eth_dev)) + return -EINVAL; + +@@ -366,6 +367,7 @@ nfp_netvf_init(struct rte_eth_dev *eth_dev) + hw->cap = nn_cfg_readl(hw, NFP_NET_CFG_CAP); + hw->max_mtu = nn_cfg_readl(hw, NFP_NET_CFG_MAX_MTU); + hw->mtu = RTE_ETHER_MTU; ++ hw->flbufsz = DEFAULT_FLBUF_SIZE; + + /* VLAN insertion is incompatible with LSOv2 */ + if (hw->cap & NFP_NET_CFG_CTRL_LSO2) +@@ -450,6 +452,8 @@ nfp_netvf_init(struct rte_eth_dev *eth_dev) + (void *)eth_dev); + /* Telling the firmware about the LSC interrupt entry */ + nn_cfg_writeb(hw, NFP_NET_CFG_LSC, NFP_NET_IRQ_LSC_IDX); ++ /* Unmask the LSC interrupt */ ++ nfp_net_irq_unmask(eth_dev); + /* Recording current stats counters values */ + nfp_net_stats_reset(eth_dev); + } +diff --git a/dpdk/drivers/net/nfp/nfp_flow.c b/dpdk/drivers/net/nfp/nfp_flow.c +index 6f79d950db..faa0eda325 100644 +--- a/dpdk/drivers/net/nfp/nfp_flow.c ++++ b/dpdk/drivers/net/nfp/nfp_flow.c +@@ -285,7 +285,7 @@ nfp_check_mask_remove(struct nfp_flow_priv *priv, + + rte_free(mask_entry); + if (meta_flags) +- *meta_flags &= ~NFP_FL_META_FLAG_MANAGE_MASK; ++ *meta_flags |= NFP_FL_META_FLAG_MANAGE_MASK; + } + + return true; +@@ -339,7 +339,7 @@ nfp_flow_table_search(struct nfp_flow_priv *priv, + } + + static struct rte_flow * +-nfp_flow_alloc(struct nfp_fl_key_ls *key_layer) ++nfp_flow_alloc(struct nfp_fl_key_ls *key_layer, uint32_t port_id) + { + char *tmp; + size_t len; +@@ -357,6 +357,7 @@ nfp_flow_alloc(struct nfp_fl_key_ls *key_layer) + + nfp_flow->length = len; + ++ nfp_flow->port_id = port_id; + payload = &nfp_flow->payload; + payload->meta = (struct nfp_fl_rule_metadata *)tmp; + payload->unmasked_data = tmp + sizeof(struct nfp_fl_rule_metadata); +@@ -727,7 +728,7 @@ nfp_flow_key_layers_calculate_items(const struct rte_flow_item items[], + ethdev = &rte_eth_devices[port_id->id]; + representor = (struct nfp_flower_representor *) + ethdev->data->dev_private; +- key_ls->port = rte_cpu_to_be_32(representor->port_id); ++ key_ls->port = representor->port_id; + break; + case RTE_FLOW_ITEM_TYPE_VLAN: + PMD_DRV_LOG(DEBUG, "RTE_FLOW_ITEM_TYPE_VLAN detected"); +@@ -1220,6 +1221,7 @@ nfp_flow_merge_ipv6(__rte_unused struct nfp_app_fw_flower *app_fw_flower, + bool is_mask, + bool is_outer_layer) + { ++ uint32_t vtc_flow; + struct nfp_flower_ipv6 *ipv6; + const struct rte_ipv6_hdr *hdr; + struct nfp_flower_meta_tci *meta_tci; +@@ -1243,12 +1245,12 @@ nfp_flow_merge_ipv6(__rte_unused struct nfp_app_fw_flower *app_fw_flower, + + hdr = is_mask ? &mask->hdr : &spec->hdr; + ++ vtc_flow = rte_be_to_cpu_32(hdr->vtc_flow); + if (ext_meta && (rte_be_to_cpu_32(ext_meta->nfp_flow_key_layer2) & + NFP_FLOWER_LAYER2_GRE)) { + ipv6_gre_tun = (struct nfp_flower_ipv6_gre_tun *)*mbuf_off; + +- ipv6_gre_tun->ip_ext.tos = (hdr->vtc_flow & +- RTE_IPV6_HDR_TC_MASK) >> RTE_IPV6_HDR_TC_SHIFT; ++ ipv6_gre_tun->ip_ext.tos = vtc_flow >> RTE_IPV6_HDR_TC_SHIFT; + ipv6_gre_tun->ip_ext.ttl = hdr->hop_limits; + memcpy(ipv6_gre_tun->ipv6.ipv6_src, hdr->src_addr, + sizeof(ipv6_gre_tun->ipv6.ipv6_src)); +@@ -1257,8 +1259,7 @@ nfp_flow_merge_ipv6(__rte_unused struct nfp_app_fw_flower *app_fw_flower, + } else { + ipv6_udp_tun = (struct nfp_flower_ipv6_udp_tun *)*mbuf_off; + +- ipv6_udp_tun->ip_ext.tos = (hdr->vtc_flow & +- RTE_IPV6_HDR_TC_MASK) >> RTE_IPV6_HDR_TC_SHIFT; ++ ipv6_udp_tun->ip_ext.tos = vtc_flow >> RTE_IPV6_HDR_TC_SHIFT; + ipv6_udp_tun->ip_ext.ttl = hdr->hop_limits; + memcpy(ipv6_udp_tun->ipv6.ipv6_src, hdr->src_addr, + sizeof(ipv6_udp_tun->ipv6.ipv6_src)); +@@ -1279,10 +1280,10 @@ nfp_flow_merge_ipv6(__rte_unused struct nfp_app_fw_flower *app_fw_flower, + *mbuf_off += sizeof(struct nfp_flower_tp_ports); + + hdr = is_mask ? &mask->hdr : &spec->hdr; ++ vtc_flow = rte_be_to_cpu_32(hdr->vtc_flow); + ipv6 = (struct nfp_flower_ipv6 *)*mbuf_off; + +- ipv6->ip_ext.tos = (hdr->vtc_flow & RTE_IPV6_HDR_TC_MASK) >> +- RTE_IPV6_HDR_TC_SHIFT; ++ ipv6->ip_ext.tos = vtc_flow >> RTE_IPV6_HDR_TC_SHIFT; + ipv6->ip_ext.proto = hdr->proto; + ipv6->ip_ext.ttl = hdr->hop_limits; + memcpy(ipv6->ipv6_src, hdr->src_addr, sizeof(ipv6->ipv6_src)); +@@ -1897,6 +1898,19 @@ nfp_flow_inner_item_get(const struct rte_flow_item items[], + return false; + } + ++static bool ++nfp_flow_tcp_flag_check(const struct rte_flow_item items[]) ++{ ++ const struct rte_flow_item *item; ++ ++ for (item = items; item->type != RTE_FLOW_ITEM_TYPE_END; ++item) { ++ if (item->type == RTE_FLOW_ITEM_TYPE_TCP) ++ return true; ++ } ++ ++ return false; ++} ++ + static int + nfp_flow_compile_item_proc(struct nfp_flower_representor *repr, + const struct rte_flow_item items[], +@@ -1992,6 +2006,9 @@ nfp_flow_compile_items(struct nfp_flower_representor *representor, + mbuf_off_mask += sizeof(struct nfp_flower_ext_meta); + } + ++ if (nfp_flow_tcp_flag_check(items)) ++ nfp_flow->tcp_flag = true; ++ + /* Check if this is a tunnel flow and get the inner item*/ + is_tun_flow = nfp_flow_inner_item_get(items, &loop_item); + if (is_tun_flow) +@@ -2021,7 +2038,8 @@ nfp_flow_compile_items(struct nfp_flower_representor *representor, + static int + nfp_flow_action_output(char *act_data, + const struct rte_flow_action *action, +- struct nfp_fl_rule_metadata *nfp_flow_meta) ++ struct nfp_fl_rule_metadata *nfp_flow_meta, ++ uint32_t output_cnt) + { + size_t act_size; + struct rte_eth_dev *ethdev; +@@ -2040,8 +2058,9 @@ nfp_flow_action_output(char *act_data, + output = (struct nfp_fl_act_output *)act_data; + output->head.jump_id = NFP_FL_ACTION_OPCODE_OUTPUT; + output->head.len_lw = act_size >> NFP_FL_LW_SIZ; +- output->flags = rte_cpu_to_be_16(NFP_FL_OUT_FLAGS_LAST); + output->port = rte_cpu_to_be_32(representor->port_id); ++ if (output_cnt == 0) ++ output->flags = rte_cpu_to_be_16(NFP_FL_OUT_FLAGS_LAST); + + nfp_flow_meta->shortcut = rte_cpu_to_be_32(representor->port_id); + +@@ -2054,6 +2073,7 @@ nfp_flow_action_set_mac(char *act_data, + bool mac_src_flag, + bool mac_set_flag) + { ++ uint8_t i; + size_t act_size; + struct nfp_fl_act_set_eth *set_eth; + const struct rte_flow_action_set_mac *set_mac; +@@ -2072,9 +2092,13 @@ nfp_flow_action_set_mac(char *act_data, + if (mac_src_flag) { + rte_memcpy(&set_eth->eth_addr[RTE_ETHER_ADDR_LEN], + set_mac->mac_addr, RTE_ETHER_ADDR_LEN); ++ for (i = 0; i < RTE_ETHER_ADDR_LEN; i++) ++ set_eth->eth_addr_mask[RTE_ETHER_ADDR_LEN + i] = 0xff; + } else { + rte_memcpy(&set_eth->eth_addr[0], + set_mac->mac_addr, RTE_ETHER_ADDR_LEN); ++ for (i = 0; i < RTE_ETHER_ADDR_LEN; i++) ++ set_eth->eth_addr_mask[i] = 0xff; + } + } + +@@ -2115,10 +2139,13 @@ nfp_flow_action_set_ip(char *act_data, + set_ip->reserved = 0; + + set_ipv4 = (const struct rte_flow_action_set_ipv4 *)action->conf; +- if (ip_src_flag) ++ if (ip_src_flag) { + set_ip->ipv4_src = set_ipv4->ipv4_addr; +- else ++ set_ip->ipv4_src_mask = RTE_BE32(0xffffffff); ++ } else { + set_ip->ipv4_dst = set_ipv4->ipv4_addr; ++ set_ip->ipv4_dst_mask = RTE_BE32(0xffffffff); ++ } + } + + static void +@@ -2127,6 +2154,7 @@ nfp_flow_action_set_ipv6(char *act_data, + bool ip_src_flag) + { + int i; ++ rte_be32_t tmp; + size_t act_size; + struct nfp_fl_act_set_ipv6_addr *set_ip; + const struct rte_flow_action_set_ipv6 *set_ipv6; +@@ -2143,15 +2171,19 @@ nfp_flow_action_set_ipv6(char *act_data, + set_ip->head.len_lw = act_size >> NFP_FL_LW_SIZ; + set_ip->reserved = 0; + +- for (i = 0; i < 4; i++) +- set_ip->ipv6[i].exact = set_ipv6->ipv6_addr[i]; ++ for (i = 0; i < 4; i++) { ++ rte_memcpy(&tmp, &set_ipv6->ipv6_addr[i * 4], 4); ++ set_ip->ipv6[i].exact = tmp; ++ set_ip->ipv6[i].mask = RTE_BE32(0xffffffff); ++ } + } + + static void + nfp_flow_action_set_tp(char *act_data, + const struct rte_flow_action *action, + bool tp_src_flag, +- bool tp_set_flag) ++ bool tp_set_flag, ++ bool tcp_flag) + { + size_t act_size; + struct nfp_fl_act_set_tport *set_tp; +@@ -2163,21 +2195,29 @@ nfp_flow_action_set_tp(char *act_data, + set_tp = (struct nfp_fl_act_set_tport *)act_data; + + act_size = sizeof(struct nfp_fl_act_set_tport); +- set_tp->head.jump_id = NFP_FL_ACTION_OPCODE_SET_TCP; ++ if (tcp_flag) ++ set_tp->head.jump_id = NFP_FL_ACTION_OPCODE_SET_TCP; ++ else ++ set_tp->head.jump_id = NFP_FL_ACTION_OPCODE_SET_UDP; + set_tp->head.len_lw = act_size >> NFP_FL_LW_SIZ; + set_tp->reserved = 0; + + set_tp_conf = (const struct rte_flow_action_set_tp *)action->conf; +- if (tp_src_flag) ++ if (tp_src_flag) { + set_tp->src_port = set_tp_conf->port; +- else ++ set_tp->src_port_mask = RTE_BE16(0xffff); ++ } else { + set_tp->dst_port = set_tp_conf->port; ++ set_tp->dst_port_mask = RTE_BE16(0xffff); ++ } + } + + static int + nfp_flow_action_push_vlan(char *act_data, + const struct rte_flow_action *action) + { ++ uint8_t pcp; ++ uint16_t vid; + size_t act_size; + struct nfp_fl_act_push_vlan *push_vlan; + const struct rte_flow_action_of_push_vlan *push_vlan_conf; +@@ -2200,9 +2240,11 @@ nfp_flow_action_push_vlan(char *act_data, + (action + 1)->conf; + vlan_vid_conf = (const struct rte_flow_action_of_set_vlan_vid *) + (action + 2)->conf; ++ ++ vid = rte_be_to_cpu_16(vlan_vid_conf->vlan_vid) & 0x0fff; ++ pcp = vlan_pcp_conf->vlan_pcp & 0x07; + push_vlan->vlan_tpid = push_vlan_conf->ethertype; +- push_vlan->vlan_tci = ((vlan_pcp_conf->vlan_pcp & 0x07) << 13) | +- (vlan_vid_conf->vlan_vid & 0x0fff); ++ push_vlan->vlan_tci = rte_cpu_to_be_16(vid | (pcp << 13)); + + return 0; + } +@@ -2227,6 +2269,7 @@ nfp_flow_action_set_ttl(char *act_data, + + ttl_conf = (const struct rte_flow_action_set_ttl *)action->conf; + ttl_tos->ipv4_ttl = ttl_conf->ttl_value; ++ ttl_tos->ipv4_ttl_mask = 0xff; + ttl_tos->reserved = 0; + } + +@@ -2250,6 +2293,7 @@ nfp_flow_action_set_hl(char *act_data, + + ttl_conf = (const struct rte_flow_action_set_ttl *)action->conf; + tc_hl->ipv6_hop_limit = ttl_conf->ttl_value; ++ tc_hl->ipv6_hop_limit_mask = 0xff; + tc_hl->reserved = 0; + } + +@@ -2273,6 +2317,7 @@ nfp_flow_action_set_tos(char *act_data, + + tos_conf = (const struct rte_flow_action_set_dscp *)action->conf; + ttl_tos->ipv4_tos = tos_conf->dscp; ++ ttl_tos->ipv4_tos_mask = 0xff; + ttl_tos->reserved = 0; + } + +@@ -2296,6 +2341,7 @@ nfp_flow_action_set_tc(char *act_data, + + tos_conf = (const struct rte_flow_action_set_dscp *)action->conf; + tc_hl->ipv6_tc = tos_conf->dscp; ++ tc_hl->ipv6_tc_mask = 0xff; + tc_hl->reserved = 0; + } + +@@ -2443,10 +2489,10 @@ nfp_flower_add_tun_neigh_v4_decap(struct nfp_app_fw_flower *app_fw_flower, + if (nfp_flower_support_decap_v2(app_fw_flower)) { + if (meta_tci->tci != 0) { + payload.ext.vlan_tci = meta_tci->tci; +- payload.ext.vlan_tpid = 0x88a8; ++ payload.ext.vlan_tpid = RTE_BE16(0x88a8); + } else { +- payload.ext.vlan_tci = 0xffff; +- payload.ext.vlan_tpid = 0xffff; ++ payload.ext.vlan_tci = RTE_BE16(0xffff); ++ payload.ext.vlan_tpid = RTE_BE16(0xffff); + } + payload.ext.host_ctx = nfp_flow_meta->host_ctx_id; + } +@@ -2570,10 +2616,10 @@ nfp_flower_add_tun_neigh_v6_decap(struct nfp_app_fw_flower *app_fw_flower, + if (nfp_flower_support_decap_v2(app_fw_flower)) { + if (meta_tci->tci != 0) { + payload.ext.vlan_tci = meta_tci->tci; +- payload.ext.vlan_tpid = 0x88a8; ++ payload.ext.vlan_tpid = RTE_BE16(0x88a8); + } else { +- payload.ext.vlan_tci = 0xffff; +- payload.ext.vlan_tpid = 0xffff; ++ payload.ext.vlan_tci = RTE_BE16(0xffff); ++ payload.ext.vlan_tpid = RTE_BE16(0xffff); + } + payload.ext.host_ctx = nfp_flow_meta->host_ctx_id; + } +@@ -2675,6 +2721,7 @@ nfp_flow_action_vxlan_encap_v4(struct nfp_app_fw_flower *app_fw_flower, + struct nfp_fl_rule_metadata *nfp_flow_meta, + struct nfp_fl_tun *tun) + { ++ uint64_t tun_id; + struct nfp_fl_act_pre_tun *pre_tun; + struct nfp_fl_act_set_tun *set_tun; + const struct rte_flow_item_eth *eth; +@@ -2693,7 +2740,8 @@ nfp_flow_action_vxlan_encap_v4(struct nfp_app_fw_flower *app_fw_flower, + + set_tun = (struct nfp_fl_act_set_tun *)(act_data + act_pre_size); + memset(set_tun, 0, act_set_size); +- nfp_flow_set_tun_process(set_tun, NFP_FL_TUN_VXLAN, vxlan->hdr.vx_vni, ++ tun_id = rte_be_to_cpu_32(vxlan->hdr.vx_vni) >> 8; ++ nfp_flow_set_tun_process(set_tun, NFP_FL_TUN_VXLAN, tun_id, + ipv4->hdr.time_to_live, ipv4->hdr.type_of_service); + set_tun->tun_flags = vxlan->hdr.vx_flags; + +@@ -2710,6 +2758,8 @@ nfp_flow_action_vxlan_encap_v6(struct nfp_app_fw_flower *app_fw_flower, + struct nfp_fl_rule_metadata *nfp_flow_meta, + struct nfp_fl_tun *tun) + { ++ uint8_t tos; ++ uint64_t tun_id; + struct nfp_fl_act_pre_tun *pre_tun; + struct nfp_fl_act_set_tun *set_tun; + const struct rte_flow_item_eth *eth; +@@ -2728,9 +2778,10 @@ nfp_flow_action_vxlan_encap_v6(struct nfp_app_fw_flower *app_fw_flower, + + set_tun = (struct nfp_fl_act_set_tun *)(act_data + act_pre_size); + memset(set_tun, 0, act_set_size); +- nfp_flow_set_tun_process(set_tun, NFP_FL_TUN_VXLAN, vxlan->hdr.vx_vni, +- ipv6->hdr.hop_limits, +- (ipv6->hdr.vtc_flow >> RTE_IPV6_HDR_TC_SHIFT) & 0xff); ++ tun_id = rte_be_to_cpu_32(vxlan->hdr.vx_vni) >> 8; ++ tos = rte_be_to_cpu_32(ipv6->hdr.vtc_flow) >> RTE_IPV6_HDR_TC_SHIFT; ++ nfp_flow_set_tun_process(set_tun, NFP_FL_TUN_VXLAN, tun_id, ++ ipv6->hdr.hop_limits, tos); + set_tun->tun_flags = vxlan->hdr.vx_flags; + + /* Send the tunnel neighbor cmsg to fw */ +@@ -3094,7 +3145,7 @@ nfp_flow_action_geneve_encap_v6(struct nfp_app_fw_flower *app_fw_flower, + + set_tun = (struct nfp_fl_act_set_tun *)(act_data + act_pre_size); + memset(set_tun, 0, act_set_size); +- tos = (ipv6->hdr.vtc_flow >> RTE_IPV6_HDR_TC_SHIFT) & 0xff; ++ tos = rte_be_to_cpu_32(ipv6->hdr.vtc_flow) >> RTE_IPV6_HDR_TC_SHIFT; + tun_id = (geneve->vni[0] << 16) | (geneve->vni[1] << 8) | geneve->vni[2]; + nfp_flow_set_tun_process(set_tun, NFP_FL_TUN_GENEVE, tun_id, + ipv6->hdr.hop_limits, tos); +@@ -3113,6 +3164,7 @@ nfp_flow_action_nvgre_encap_v4(struct nfp_app_fw_flower *app_fw_flower, + struct nfp_fl_rule_metadata *nfp_flow_meta, + struct nfp_fl_tun *tun) + { ++ uint64_t tun_id; + const struct rte_ether_hdr *eth; + const struct rte_flow_item_ipv4 *ipv4; + const struct rte_flow_item_gre *gre; +@@ -3124,6 +3176,7 @@ nfp_flow_action_nvgre_encap_v4(struct nfp_app_fw_flower *app_fw_flower, + eth = (const struct rte_ether_hdr *)raw_encap->data; + ipv4 = (const struct rte_flow_item_ipv4 *)(eth + 1); + gre = (const struct rte_flow_item_gre *)(ipv4 + 1); ++ tun_id = rte_be_to_cpu_32(*(const rte_be32_t *)(gre + 1)); + + pre_tun = (struct nfp_fl_act_pre_tun *)actions; + memset(pre_tun, 0, act_pre_size); +@@ -3131,7 +3184,7 @@ nfp_flow_action_nvgre_encap_v4(struct nfp_app_fw_flower *app_fw_flower, + + set_tun = (struct nfp_fl_act_set_tun *)(act_data + act_pre_size); + memset(set_tun, 0, act_set_size); +- nfp_flow_set_tun_process(set_tun, NFP_FL_TUN_GRE, 0, ++ nfp_flow_set_tun_process(set_tun, NFP_FL_TUN_GRE, tun_id, + ipv4->hdr.time_to_live, ipv4->hdr.type_of_service); + set_tun->tun_proto = gre->protocol; + +@@ -3149,6 +3202,7 @@ nfp_flow_action_nvgre_encap_v6(struct nfp_app_fw_flower *app_fw_flower, + struct nfp_fl_tun *tun) + { + uint8_t tos; ++ uint64_t tun_id; + const struct rte_ether_hdr *eth; + const struct rte_flow_item_ipv6 *ipv6; + const struct rte_flow_item_gre *gre; +@@ -3160,6 +3214,7 @@ nfp_flow_action_nvgre_encap_v6(struct nfp_app_fw_flower *app_fw_flower, + eth = (const struct rte_ether_hdr *)raw_encap->data; + ipv6 = (const struct rte_flow_item_ipv6 *)(eth + 1); + gre = (const struct rte_flow_item_gre *)(ipv6 + 1); ++ tun_id = rte_be_to_cpu_32(*(const rte_be32_t *)(gre + 1)); + + pre_tun = (struct nfp_fl_act_pre_tun *)actions; + memset(pre_tun, 0, act_pre_size); +@@ -3167,8 +3222,8 @@ nfp_flow_action_nvgre_encap_v6(struct nfp_app_fw_flower *app_fw_flower, + + set_tun = (struct nfp_fl_act_set_tun *)(act_data + act_pre_size); + memset(set_tun, 0, act_set_size); +- tos = (ipv6->hdr.vtc_flow >> RTE_IPV6_HDR_TC_SHIFT) & 0xff; +- nfp_flow_set_tun_process(set_tun, NFP_FL_TUN_GRE, 0, ++ tos = rte_be_to_cpu_32(ipv6->hdr.vtc_flow) >> RTE_IPV6_HDR_TC_SHIFT; ++ nfp_flow_set_tun_process(set_tun, NFP_FL_TUN_GRE, tun_id, + ipv6->hdr.hop_limits, tos); + set_tun->tun_proto = gre->protocol; + +@@ -3232,12 +3287,27 @@ nfp_flow_action_raw_encap(struct nfp_app_fw_flower *app_fw_flower, + return ret; + } + ++static uint32_t ++nfp_flow_count_output(const struct rte_flow_action actions[]) ++{ ++ uint32_t count = 0; ++ const struct rte_flow_action *action; ++ ++ for (action = actions; action->type != RTE_FLOW_ACTION_TYPE_END; ++action) { ++ if (action->type == RTE_FLOW_ACTION_TYPE_PORT_ID) ++ count++; ++ } ++ ++ return count; ++} ++ + static int + nfp_flow_compile_action(struct nfp_flower_representor *representor, + const struct rte_flow_action actions[], + struct rte_flow *nfp_flow) + { + int ret = 0; ++ uint32_t count; + char *position; + char *action_data; + bool ttl_tos_flag = false; +@@ -3256,6 +3326,8 @@ nfp_flow_compile_action(struct nfp_flower_representor *representor, + position = action_data; + meta_tci = (struct nfp_flower_meta_tci *)nfp_flow->payload.unmasked_data; + ++ count = nfp_flow_count_output(actions); ++ + for (action = actions; action->type != RTE_FLOW_ACTION_TYPE_END; ++action) { + switch (action->type) { + case RTE_FLOW_ACTION_TYPE_VOID: +@@ -3272,7 +3344,8 @@ nfp_flow_compile_action(struct nfp_flower_representor *representor, + break; + case RTE_FLOW_ACTION_TYPE_PORT_ID: + PMD_DRV_LOG(DEBUG, "Process RTE_FLOW_ACTION_TYPE_PORT_ID"); +- ret = nfp_flow_action_output(position, action, nfp_flow_meta); ++ count--; ++ ret = nfp_flow_action_output(position, action, nfp_flow_meta, count); + if (ret != 0) { + PMD_DRV_LOG(ERR, "Failed when process" + " RTE_FLOW_ACTION_TYPE_PORT_ID"); +@@ -3347,7 +3420,8 @@ nfp_flow_compile_action(struct nfp_flower_representor *representor, + break; + case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: + PMD_DRV_LOG(DEBUG, "Process RTE_FLOW_ACTION_TYPE_SET_TP_SRC"); +- nfp_flow_action_set_tp(position, action, true, tp_set_flag); ++ nfp_flow_action_set_tp(position, action, true, ++ tp_set_flag, nfp_flow->tcp_flag); + if (!tp_set_flag) { + position += sizeof(struct nfp_fl_act_set_tport); + tp_set_flag = true; +@@ -3355,7 +3429,8 @@ nfp_flow_compile_action(struct nfp_flower_representor *representor, + break; + case RTE_FLOW_ACTION_TYPE_SET_TP_DST: + PMD_DRV_LOG(DEBUG, "Process RTE_FLOW_ACTION_TYPE_SET_TP_DST"); +- nfp_flow_action_set_tp(position, action, false, tp_set_flag); ++ nfp_flow_action_set_tp(position, action, false, ++ tp_set_flag, nfp_flow->tcp_flag); + if (!tp_set_flag) { + position += sizeof(struct nfp_fl_act_set_tport); + tp_set_flag = true; +@@ -3484,7 +3559,7 @@ nfp_flow_process(struct nfp_flower_representor *representor, + return NULL; + } + +- nfp_flow = nfp_flow_alloc(&key_layer); ++ nfp_flow = nfp_flow_alloc(&key_layer, representor->port_id); + if (nfp_flow == NULL) { + PMD_DRV_LOG(ERR, "Alloc nfp flow failed."); + goto free_stats; +@@ -3592,6 +3667,7 @@ nfp_flow_teardown(struct nfp_flow_priv *priv, + nfp_flow_meta = nfp_flow->payload.meta; + mask_data = nfp_flow->payload.mask_data; + mask_len = nfp_flow_meta->mask_len << NFP_FL_LW_SIZ; ++ nfp_flow_meta->flags &= ~NFP_FL_META_FLAG_MANAGE_MASK; + if (!nfp_check_mask_remove(priv, mask_data, mask_len, + &nfp_flow_meta->flags)) { + PMD_DRV_LOG(ERR, "nfp mask del check failed."); +@@ -3791,14 +3867,21 @@ nfp_flow_flush(struct rte_eth_dev *dev, + void *next_data; + uint32_t iter = 0; + const void *next_key; ++ struct rte_flow *nfp_flow; + struct nfp_flow_priv *priv; ++ struct nfp_flower_representor *representor; ++ ++ representor = dev->data->dev_private; + + priv = nfp_flow_dev_to_priv(dev); + + while (rte_hash_iterate(priv->flow_table, &next_key, &next_data, &iter) >= 0) { +- ret = nfp_flow_destroy(dev, (struct rte_flow *)next_data, error); +- if (ret != 0) +- break; ++ nfp_flow = next_data; ++ if (nfp_flow->port_id == representor->port_id) { ++ ret = nfp_flow_destroy(dev, nfp_flow, error); ++ if (ret != 0) ++ break; ++ } + } + + return ret; +@@ -3809,6 +3892,7 @@ nfp_flow_stats_get(struct rte_eth_dev *dev, + struct rte_flow *nfp_flow, + void *data) + { ++ bool reset; + uint32_t ctx_id; + struct rte_flow *flow; + struct nfp_flow_priv *priv; +@@ -3823,6 +3907,7 @@ nfp_flow_stats_get(struct rte_eth_dev *dev, + } + + query = (struct rte_flow_query_count *)data; ++ reset = query->reset; + memset(query, 0, sizeof(*query)); + + ctx_id = rte_be_to_cpu_32(nfp_flow->payload.meta->host_ctx_id); +@@ -3834,7 +3919,7 @@ nfp_flow_stats_get(struct rte_eth_dev *dev, + query->bytes = stats->bytes; + query->hits_set = 1; + query->bytes_set = 1; +- if (query->reset != 0) { ++ if (reset) { + stats->pkts = 0; + stats->bytes = 0; + } +@@ -3981,11 +4066,21 @@ nfp_flow_priv_init(struct nfp_pf_dev *pf_dev) + size_t stats_size; + uint64_t ctx_count; + uint64_t ctx_split; ++ char mask_name[RTE_HASH_NAMESIZE]; ++ char flow_name[RTE_HASH_NAMESIZE]; ++ char pretun_name[RTE_HASH_NAMESIZE]; + struct nfp_flow_priv *priv; + struct nfp_app_fw_flower *app_fw_flower; + ++ snprintf(mask_name, sizeof(mask_name), "%s_mask", ++ pf_dev->pci_dev->device.name); ++ snprintf(flow_name, sizeof(flow_name), "%s_flow", ++ pf_dev->pci_dev->device.name); ++ snprintf(pretun_name, sizeof(pretun_name), "%s_pretun", ++ pf_dev->pci_dev->device.name); ++ + struct rte_hash_parameters mask_hash_params = { +- .name = "mask_hash_table", ++ .name = mask_name, + .entries = NFP_MASK_TABLE_ENTRIES, + .hash_func = rte_jhash, + .socket_id = rte_socket_id(), +@@ -3994,7 +4089,7 @@ nfp_flow_priv_init(struct nfp_pf_dev *pf_dev) + }; + + struct rte_hash_parameters flow_hash_params = { +- .name = "flow_hash_table", ++ .name = flow_name, + .hash_func = rte_jhash, + .socket_id = rte_socket_id(), + .key_len = sizeof(uint32_t), +@@ -4002,7 +4097,7 @@ nfp_flow_priv_init(struct nfp_pf_dev *pf_dev) + }; + + struct rte_hash_parameters pre_tun_hash_params = { +- .name = "pre_tunnel_table", ++ .name = pretun_name, + .entries = 32, + .hash_func = rte_jhash, + .socket_id = rte_socket_id(), +diff --git a/dpdk/drivers/net/nfp/nfp_flow.h b/dpdk/drivers/net/nfp/nfp_flow.h +index b0c2aaf6d8..314ff2083a 100644 +--- a/dpdk/drivers/net/nfp/nfp_flow.h ++++ b/dpdk/drivers/net/nfp/nfp_flow.h +@@ -222,7 +222,9 @@ struct rte_flow { + struct nfp_fl_tun tun; + size_t length; + uint32_t hash_key; ++ uint32_t port_id; + bool install_flag; ++ bool tcp_flag; /**< Used in the SET_TP_* action */ + enum nfp_flow_type type; + }; + +diff --git a/dpdk/drivers/net/nfp/nfp_rxtx.c b/dpdk/drivers/net/nfp/nfp_rxtx.c +index 01cffdfde0..ac0fa80d4c 100644 +--- a/dpdk/drivers/net/nfp/nfp_rxtx.c ++++ b/dpdk/drivers/net/nfp/nfp_rxtx.c +@@ -48,7 +48,7 @@ nfp_net_rx_fill_freelist(struct nfp_net_rxq *rxq) + + rxd = &rxq->rxds[i]; + rxd->fld.dd = 0; +- rxd->fld.dma_addr_hi = (dma_addr >> 32) & 0xff; ++ rxd->fld.dma_addr_hi = (dma_addr >> 32) & 0xffff; + rxd->fld.dma_addr_lo = dma_addr & 0xffffffff; + rxe[i].mbuf = mbuf; + PMD_RX_LOG(DEBUG, "[%d]: %" PRIx64, i, dma_addr); +@@ -361,7 +361,7 @@ nfp_net_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) + rxds->vals[1] = 0; + dma_addr = rte_cpu_to_le_64(RTE_MBUF_DMA_ADDR_DEFAULT(new_mb)); + rxds->fld.dd = 0; +- rxds->fld.dma_addr_hi = (dma_addr >> 32) & 0xff; ++ rxds->fld.dma_addr_hi = (dma_addr >> 32) & 0xffff; + rxds->fld.dma_addr_lo = dma_addr & 0xffffffff; + nb_hold++; + +@@ -1063,6 +1063,7 @@ nfp_net_nfdk_tx_maybe_close_block(struct nfp_net_txq *txq, struct rte_mbuf *pkt) + if (unlikely(n_descs > NFDK_TX_DESC_GATHER_MAX)) + return -EINVAL; + ++ /* Under count by 1 (don't count meta) for the round down to work out */ + n_descs += !!(pkt->ol_flags & RTE_MBUF_F_TX_TCP_SEG); + + if (round_down(txq->wr_p, NFDK_TX_DESC_BLOCK_CNT) != +@@ -1214,13 +1215,24 @@ nfp_net_nfdk_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pk + if ((hw->cap & NFP_NET_CFG_CTRL_LSO_ANY) && + (pkt->ol_flags & RTE_MBUF_F_TX_TCP_SEG)) { + type = NFDK_DESC_TX_TYPE_TSO; +- } else if (!pkt->next && dma_len < NFDK_TX_MAX_DATA_PER_HEAD) { ++ } else if (pkt->next == NULL && dma_len <= NFDK_TX_MAX_DATA_PER_HEAD) { + type = NFDK_DESC_TX_TYPE_SIMPLE; + } else { + type = NFDK_DESC_TX_TYPE_GATHER; + } ++ ++ /* Implicitly truncates to chunk in below logic */ + dma_len -= 1; +- dlen_type = (NFDK_DESC_TX_DMA_LEN_HEAD & dma_len) | ++ ++ /* ++ * We will do our best to pass as much data as we can in descriptor ++ * and we need to make sure the first descriptor includes whole ++ * head since there is limitation in firmware side. Sometimes the ++ * value of 'dma_len & NFDK_DESC_TX_DMA_LEN_HEAD' will be less ++ * than packet head len. ++ */ ++ dlen_type = (dma_len > NFDK_DESC_TX_DMA_LEN_HEAD ? ++ NFDK_DESC_TX_DMA_LEN_HEAD : dma_len) | + (NFDK_DESC_TX_TYPE_HEAD & (type << 12)); + ktxds->dma_len_type = rte_cpu_to_le_16(dlen_type); + dma_addr = rte_mbuf_data_iova(pkt); +@@ -1230,10 +1242,18 @@ nfp_net_nfdk_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pk + ktxds->dma_addr_lo = rte_cpu_to_le_32(dma_addr & 0xffffffff); + ktxds++; + ++ /* ++ * Preserve the original dlen_type, this way below the EOP logic ++ * can use dlen_type. ++ */ + tmp_dlen = dlen_type & NFDK_DESC_TX_DMA_LEN_HEAD; + dma_len -= tmp_dlen; + dma_addr += tmp_dlen + 1; + ++ /* ++ * The rest of the data (if any) will be in larger DMA descriptors ++ * and is handled with the dma_len loop. ++ */ + while (pkt) { + if (*lmbuf) + rte_pktmbuf_free_seg(*lmbuf); +diff --git a/dpdk/drivers/net/nfp/nfp_rxtx.h b/dpdk/drivers/net/nfp/nfp_rxtx.h +index ced05fde90..1c2a00e300 100644 +--- a/dpdk/drivers/net/nfp/nfp_rxtx.h ++++ b/dpdk/drivers/net/nfp/nfp_rxtx.h +@@ -227,8 +227,8 @@ struct nfp_net_rx_desc { + union { + /* Freelist descriptor */ + struct { +- uint8_t dma_addr_hi; +- __le16 spare; ++ __le16 dma_addr_hi; ++ uint8_t spare; + uint8_t dd; + + __le32 dma_addr_lo; +@@ -340,10 +340,14 @@ nfp_net_mbuf_alloc_failed(struct nfp_net_rxq *rxq) + static inline uint32_t + nfp_net_nfd3_free_tx_desc(struct nfp_net_txq *txq) + { ++ uint32_t free_desc; ++ + if (txq->wr_p >= txq->rd_p) +- return txq->tx_count - (txq->wr_p - txq->rd_p) - 8; ++ free_desc = txq->tx_count - (txq->wr_p - txq->rd_p); + else +- return txq->rd_p - txq->wr_p - 8; ++ free_desc = txq->rd_p - txq->wr_p; ++ ++ return (free_desc > 8) ? (free_desc - 8) : 0; + } + + /* +diff --git a/dpdk/drivers/net/nfp/nfpcore/nfp_cpp.h b/dpdk/drivers/net/nfp/nfpcore/nfp_cpp.h +index a04a68f546..68851b22e4 100644 +--- a/dpdk/drivers/net/nfp/nfpcore/nfp_cpp.h ++++ b/dpdk/drivers/net/nfp/nfpcore/nfp_cpp.h +@@ -34,6 +34,9 @@ struct nfp_cpp { + */ + uint32_t imb_cat_table[16]; + ++ /* MU access type bit offset */ ++ uint32_t mu_locality_lsb; ++ + int driver_lock_needed; + }; + +@@ -363,7 +366,7 @@ struct nfp_cpp_area *nfp_cpp_area_alloc_acquire(struct nfp_cpp *cpp, + */ + void nfp_cpp_area_release_free(struct nfp_cpp_area *area); + +-uint8_t *nfp_cpp_map_area(struct nfp_cpp *cpp, int domain, int target, ++uint8_t *nfp_cpp_map_area(struct nfp_cpp *cpp, uint32_t cpp_id, + uint64_t addr, unsigned long size, + struct nfp_cpp_area **area); + /* +@@ -778,4 +781,6 @@ int nfp_cpp_mutex_unlock(struct nfp_cpp_mutex *mutex); + */ + int nfp_cpp_mutex_trylock(struct nfp_cpp_mutex *mutex); + ++uint32_t nfp_cpp_mu_locality_lsb(struct nfp_cpp *cpp); ++ + #endif /* !__NFP_CPP_H__ */ +diff --git a/dpdk/drivers/net/nfp/nfpcore/nfp_cpp_pcie_ops.c b/dpdk/drivers/net/nfp/nfpcore/nfp_cpp_pcie_ops.c +index 22c8bc4b14..e597315498 100644 +--- a/dpdk/drivers/net/nfp/nfpcore/nfp_cpp_pcie_ops.c ++++ b/dpdk/drivers/net/nfp/nfpcore/nfp_cpp_pcie_ops.c +@@ -66,8 +66,8 @@ + #define NFP_PCIE_P2C_GENERAL_TOKEN_OFFSET(bar, x) ((x) << ((bar)->bitsize - 4)) + #define NFP_PCIE_P2C_GENERAL_SIZE(bar) (1 << ((bar)->bitsize - 4)) + +-#define NFP_PCIE_CFG_BAR_PCIETOCPPEXPBAR(bar, slot) \ +- (NFP_PCIE_BAR(0) + ((bar) * 8 + (slot)) * 4) ++#define NFP_PCIE_CFG_BAR_PCIETOCPPEXPBAR(id, bar, slot) \ ++ (NFP_PCIE_BAR(id) + ((bar) * 8 + (slot)) * 4) + + #define NFP_PCIE_CPP_BAR_PCIETOCPPEXPBAR(bar, slot) \ + (((bar) * 8 + (slot)) * 4) +@@ -117,6 +117,7 @@ struct nfp_pcie_user { + int secondary_lock; + char busdev[BUSDEV_SZ]; + int barsz; ++ int dev_id; + char *cfg; + }; + +@@ -258,7 +259,7 @@ nfp_bar_write(struct nfp_pcie_user *nfp, struct nfp_bar *bar, + return (-ENOMEM); + + bar->csr = nfp->cfg + +- NFP_PCIE_CFG_BAR_PCIETOCPPEXPBAR(base, slot); ++ NFP_PCIE_CFG_BAR_PCIETOCPPEXPBAR(nfp->dev_id, base, slot); + + *(uint32_t *)(bar->csr) = newcfg; + +@@ -332,10 +333,8 @@ nfp_enable_bars(struct nfp_pcie_user *nfp) + bar->base = 0; + bar->iomem = NULL; + bar->lock = 0; +- bar->csr = nfp->cfg + +- NFP_PCIE_CFG_BAR_PCIETOCPPEXPBAR(bar->index >> 3, +- bar->index & 7); +- ++ bar->csr = nfp->cfg + NFP_PCIE_CFG_BAR_PCIETOCPPEXPBAR(nfp->dev_id, ++ bar->index >> 3, bar->index & 7); + bar->iomem = nfp->cfg + (bar->index << bar->bitsize); + } + return 0; +@@ -850,6 +849,7 @@ nfp6000_init(struct nfp_cpp *cpp, struct rte_pci_device *dev) + goto error; + + desc->cfg = (char *)dev->mem_resource[0].addr; ++ desc->dev_id = dev->addr.function & 0x7; + + nfp_enable_bars(desc); + +diff --git a/dpdk/drivers/net/nfp/nfpcore/nfp_cppcore.c b/dpdk/drivers/net/nfp/nfpcore/nfp_cppcore.c +index 37799af558..014f6c9df8 100644 +--- a/dpdk/drivers/net/nfp/nfpcore/nfp_cppcore.c ++++ b/dpdk/drivers/net/nfp/nfpcore/nfp_cppcore.c +@@ -19,6 +19,7 @@ + #include "nfp6000/nfp6000.h" + #include "nfp6000/nfp_xpb.h" + #include "nfp_nffw.h" ++#include "../nfp_logs.h" + + #define NFP_PL_DEVICE_ID 0x00000004 + #define NFP_PL_DEVICE_ID_MASK 0xff +@@ -118,6 +119,36 @@ nfp_cpp_area_name(struct nfp_cpp_area *cpp_area) + return cpp_area->name; + } + ++#define NFP_IMB_TGTADDRESSMODECFG_MODE_of(_x) (((_x) >> 13) & 0x7) ++#define NFP_IMB_TGTADDRESSMODECFG_ADDRMODE RTE_BIT32(12) ++ ++static int ++nfp_cpp_set_mu_locality_lsb(struct nfp_cpp *cpp) ++{ ++ int ret; ++ int mode; ++ int addr40; ++ uint32_t imbcppat; ++ ++ imbcppat = cpp->imb_cat_table[NFP_CPP_TARGET_MU]; ++ mode = NFP_IMB_TGTADDRESSMODECFG_MODE_of(imbcppat); ++ addr40 = imbcppat & NFP_IMB_TGTADDRESSMODECFG_ADDRMODE; ++ ++ ret = nfp_cppat_mu_locality_lsb(mode, addr40); ++ if (ret < 0) ++ return ret; ++ ++ cpp->mu_locality_lsb = ret; ++ ++ return 0; ++} ++ ++uint32_t ++nfp_cpp_mu_locality_lsb(struct nfp_cpp *cpp) ++{ ++ return cpp->mu_locality_lsb; ++} ++ + /* + * nfp_cpp_area_alloc - allocate a new CPP area + * @cpp: CPP handle +@@ -142,10 +173,6 @@ nfp_cpp_area_alloc_with_name(struct nfp_cpp *cpp, uint32_t dest, + if (!cpp) + return NULL; + +- /* CPP bus uses only a 40-bit address */ +- if ((address + size) > (1ULL << 40)) +- return NFP_ERRPTR(EFAULT); +- + /* Remap from cpp_island to cpp_target */ + err = nfp_target_cpp(dest, tmp64, &dest, &tmp64, cpp->imb_cat_table); + if (err < 0) +@@ -588,6 +615,13 @@ nfp_cpp_alloc(struct rte_pci_device *dev, int driver_lock_needed) + } + } + ++ err = nfp_cpp_set_mu_locality_lsb(cpp); ++ if (err < 0) { ++ PMD_DRV_LOG(ERR, "Can't calculate MU locality bit offset"); ++ free(cpp); ++ return NULL; ++ } ++ + return cpp; + } + +@@ -819,8 +853,7 @@ __nfp_cpp_model_autodetect(struct nfp_cpp *cpp, uint32_t *model) + /* + * nfp_cpp_map_area() - Helper function to map an area + * @cpp: NFP CPP handler +- * @domain: CPP domain +- * @target: CPP target ++ * @cpp_id: CPP ID + * @addr: CPP address + * @size: Size of the area + * @area: Area handle (output) +@@ -828,18 +861,15 @@ __nfp_cpp_model_autodetect(struct nfp_cpp *cpp, uint32_t *model) + * Map an area of IOMEM access. To undo the effect of this function call + * @nfp_cpp_area_release_free(*area). + * +- * Return: Pointer to memory mapped area or ERR_PTR ++ * Return: Pointer to memory mapped area or NULL + */ + uint8_t * +-nfp_cpp_map_area(struct nfp_cpp *cpp, int domain, int target, uint64_t addr, ++nfp_cpp_map_area(struct nfp_cpp *cpp, uint32_t cpp_id, uint64_t addr, + unsigned long size, struct nfp_cpp_area **area) + { + uint8_t *res; +- uint32_t dest; +- +- dest = NFP_CPP_ISLAND_ID(target, NFP_CPP_ACTION_RW, 0, domain); + +- *area = nfp_cpp_area_alloc_acquire(cpp, dest, addr, size); ++ *area = nfp_cpp_area_alloc_acquire(cpp, cpp_id, addr, size); + if (!*area) + goto err_eio; + +diff --git a/dpdk/drivers/net/nfp/nfpcore/nfp_rtsym.c b/dpdk/drivers/net/nfp/nfpcore/nfp_rtsym.c +index 56bbf05cd8..21879f7eb6 100644 +--- a/dpdk/drivers/net/nfp/nfpcore/nfp_rtsym.c ++++ b/dpdk/drivers/net/nfp/nfpcore/nfp_rtsym.c +@@ -14,6 +14,7 @@ + #include "nfp_mip.h" + #include "nfp_rtsym.h" + #include "nfp6000/nfp6000.h" ++#include "../nfp_logs.h" + + /* These need to match the linker */ + #define SYM_TGT_LMEM 0 +@@ -213,6 +214,113 @@ nfp_rtsym_lookup(struct nfp_rtsym_table *rtbl, const char *name) + return NULL; + } + ++static uint64_t ++nfp_rtsym_size(const struct nfp_rtsym *sym) ++{ ++ switch (sym->type) { ++ case NFP_RTSYM_TYPE_NONE: ++ PMD_DRV_LOG(ERR, "rtsym '%s': type NONE", sym->name); ++ return 0; ++ case NFP_RTSYM_TYPE_OBJECT: /* Fall through */ ++ case NFP_RTSYM_TYPE_FUNCTION: ++ return sym->size; ++ case NFP_RTSYM_TYPE_ABS: ++ return sizeof(uint64_t); ++ default: ++ PMD_DRV_LOG(ERR, "rtsym '%s': unknown type: %d", sym->name, sym->type); ++ return 0; ++ } ++} ++ ++static int ++nfp_rtsym_to_dest(struct nfp_cpp *cpp, ++ const struct nfp_rtsym *sym, ++ uint8_t action, ++ uint8_t token, ++ uint64_t offset, ++ uint32_t *cpp_id, ++ uint64_t *addr) ++{ ++ if (sym->type != NFP_RTSYM_TYPE_OBJECT) { ++ PMD_DRV_LOG(ERR, "rtsym '%s': direct access to non-object rtsym", ++ sym->name); ++ return -EINVAL; ++ } ++ ++ *addr = sym->addr + offset; ++ ++ if (sym->target >= 0) { ++ *cpp_id = NFP_CPP_ISLAND_ID(sym->target, action, token, sym->domain); ++ } else if (sym->target == NFP_RTSYM_TARGET_EMU_CACHE) { ++ int locality_off = nfp_cpp_mu_locality_lsb(cpp); ++ ++ *addr &= ~(NFP_MU_ADDR_ACCESS_TYPE_MASK << locality_off); ++ *addr |= NFP_MU_ADDR_ACCESS_TYPE_DIRECT << locality_off; ++ ++ *cpp_id = NFP_CPP_ISLAND_ID(NFP_CPP_TARGET_MU, action, token, ++ sym->domain); ++ } else { ++ PMD_DRV_LOG(ERR, "rtsym '%s': unhandled target encoding: %d", ++ sym->name, sym->target); ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ ++static int ++nfp_rtsym_readl(struct nfp_cpp *cpp, ++ const struct nfp_rtsym *sym, ++ uint8_t action, ++ uint8_t token, ++ uint64_t offset, ++ uint32_t *value) ++{ ++ int ret; ++ uint64_t addr; ++ uint32_t cpp_id; ++ ++ if (offset + 4 > nfp_rtsym_size(sym)) { ++ PMD_DRV_LOG(ERR, "rtsym '%s': readl out of bounds", sym->name); ++ return -ENXIO; ++ } ++ ++ ret = nfp_rtsym_to_dest(cpp, sym, action, token, offset, &cpp_id, &addr); ++ if (ret != 0) ++ return ret; ++ ++ return nfp_cpp_readl(cpp, cpp_id, addr, value); ++} ++ ++static int ++nfp_rtsym_readq(struct nfp_cpp *cpp, ++ const struct nfp_rtsym *sym, ++ uint8_t action, ++ uint8_t token, ++ uint64_t offset, ++ uint64_t *value) ++{ ++ int ret; ++ uint64_t addr; ++ uint32_t cpp_id; ++ ++ if (offset + 8 > nfp_rtsym_size(sym)) { ++ PMD_DRV_LOG(ERR, "rtsym '%s': readq out of bounds", sym->name); ++ return -ENXIO; ++ } ++ ++ if (sym->type == NFP_RTSYM_TYPE_ABS) { ++ *value = sym->addr; ++ return 0; ++ } ++ ++ ret = nfp_rtsym_to_dest(cpp, sym, action, token, offset, &cpp_id, &addr); ++ if (ret != 0) ++ return ret; ++ ++ return nfp_cpp_readq(cpp, cpp_id, addr, value); ++} ++ + /* + * nfp_rtsym_read_le() - Read a simple unsigned scalar value from symbol + * @rtbl: NFP RTsym table +@@ -229,7 +337,7 @@ uint64_t + nfp_rtsym_read_le(struct nfp_rtsym_table *rtbl, const char *name, int *error) + { + const struct nfp_rtsym *sym; +- uint32_t val32, id; ++ uint32_t val32; + uint64_t val; + int err; + +@@ -239,19 +347,13 @@ nfp_rtsym_read_le(struct nfp_rtsym_table *rtbl, const char *name, int *error) + goto exit; + } + +- id = NFP_CPP_ISLAND_ID(sym->target, NFP_CPP_ACTION_RW, 0, sym->domain); +- +-#ifdef DEBUG +- printf("Reading symbol %s with size %" PRIu64 " at %" PRIx64 "\n", +- name, sym->size, sym->addr); +-#endif + switch (sym->size) { + case 4: +- err = nfp_cpp_readl(rtbl->cpp, id, sym->addr, &val32); ++ err = nfp_rtsym_readl(rtbl->cpp, sym, NFP_CPP_ACTION_RW, 0, 0, &val32); + val = val32; + break; + case 8: +- err = nfp_cpp_readq(rtbl->cpp, id, sym->addr, &val); ++ err = nfp_rtsym_readq(rtbl->cpp, sym, NFP_CPP_ACTION_RW, 0, 0, &val); + break; + default: + printf("rtsym '%s' unsupported size: %" PRId64 "\n", +@@ -276,8 +378,11 @@ uint8_t * + nfp_rtsym_map(struct nfp_rtsym_table *rtbl, const char *name, + unsigned int min_size, struct nfp_cpp_area **area) + { +- const struct nfp_rtsym *sym; ++ int ret; + uint8_t *mem; ++ uint64_t addr; ++ uint32_t cpp_id; ++ const struct nfp_rtsym *sym; + + #ifdef DEBUG + printf("mapping symbol %s\n", name); +@@ -288,14 +393,20 @@ nfp_rtsym_map(struct nfp_rtsym_table *rtbl, const char *name, + return NULL; + } + ++ ret = nfp_rtsym_to_dest(rtbl->cpp, sym, NFP_CPP_ACTION_RW, 0, 0, ++ &cpp_id, &addr); ++ if (ret != 0) { ++ PMD_DRV_LOG(ERR, "rtsym '%s': mapping failed", name); ++ return NULL; ++ } ++ + if (sym->size < min_size) { + printf("Symbol %s too small (%" PRIu64 " < %u)\n", name, + sym->size, min_size); + return NULL; + } + +- mem = nfp_cpp_map_area(rtbl->cpp, sym->domain, sym->target, sym->addr, +- sym->size, area); ++ mem = nfp_cpp_map_area(rtbl->cpp, cpp_id, addr, sym->size, area); + if (!mem) { + printf("Failed to map symbol %s\n", name); + return NULL; +diff --git a/dpdk/drivers/net/ngbe/base/ngbe_hw.c b/dpdk/drivers/net/ngbe/base/ngbe_hw.c +index 283cdca367..27243d85c8 100644 +--- a/dpdk/drivers/net/ngbe/base/ngbe_hw.c ++++ b/dpdk/drivers/net/ngbe/base/ngbe_hw.c +@@ -1541,11 +1541,15 @@ s32 ngbe_clear_vfta(struct ngbe_hw *hw) + s32 ngbe_check_mac_link_em(struct ngbe_hw *hw, u32 *speed, + bool *link_up, bool link_up_wait_to_complete) + { +- u32 i, reg; ++ u32 i; + s32 status = 0; + +- reg = rd32(hw, NGBE_GPIOINTSTAT); +- wr32(hw, NGBE_GPIOEOI, reg); ++ if (hw->lsc) { ++ u32 reg; ++ ++ reg = rd32(hw, NGBE_GPIOINTSTAT); ++ wr32(hw, NGBE_GPIOEOI, reg); ++ } + + if (link_up_wait_to_complete) { + for (i = 0; i < hw->mac.max_link_up_time; i++) { +diff --git a/dpdk/drivers/net/ngbe/base/ngbe_phy_rtl.c b/dpdk/drivers/net/ngbe/base/ngbe_phy_rtl.c +index 9b323624ec..ba63a8058a 100644 +--- a/dpdk/drivers/net/ngbe/base/ngbe_phy_rtl.c ++++ b/dpdk/drivers/net/ngbe/base/ngbe_phy_rtl.c +@@ -120,6 +120,8 @@ s32 ngbe_init_phy_rtl(struct ngbe_hw *hw) + hw->init_phy = true; + msec_delay(1); + ++ hw->phy.set_phy_power(hw, true); ++ + for (i = 0; i < 15; i++) { + if (!rd32m(hw, NGBE_STAT, + NGBE_STAT_GPHY_IN_RST(hw->bus.lan_id))) +@@ -146,6 +148,11 @@ s32 ngbe_init_phy_rtl(struct ngbe_hw *hw) + hw->phy.write_reg(hw, 27, 0xa42, 0x8011); + hw->phy.write_reg(hw, 28, 0xa42, 0x5737); + ++ /* Disable fall to 100m if signal is not good */ ++ hw->phy.read_reg(hw, 17, 0xa44, &value); ++ value &= ~0x8; ++ hw->phy.write_reg(hw, 17, 0xa44, value); ++ + hw->phy.write_reg(hw, RTL_SCR, 0xa46, RTL_SCR_EXTINI); + hw->phy.read_reg(hw, RTL_SCR, 0xa46, &value); + if (!(value & RTL_SCR_EXTINI)) { +@@ -390,6 +397,26 @@ s32 ngbe_check_phy_link_rtl(struct ngbe_hw *hw, u32 *speed, bool *link_up) + *speed = NGBE_LINK_SPEED_10M_FULL; + } + ++ if (hw->lsc) ++ return status; ++ ++ /* ++ * Because of the slow speed of getting link state, RTL_PHYSR ++ * may still be up while the actual link state is down. ++ * So we read RTL_GBSR to get accurate state when speed is 1G ++ * in polling mode. ++ */ ++ if (*speed == NGBE_LINK_SPEED_1GB_FULL) { ++ status = hw->phy.read_reg(hw, RTL_GBSR, ++ RTL_DEV_ZERO, &phy_data); ++ phy_link = phy_data & RTL_GBSR_LRS; ++ ++ /* Only need to detect link down */ ++ if (!phy_link) { ++ *link_up = false; ++ *speed = NGBE_LINK_SPEED_UNKNOWN; ++ } ++ } + return status; + } + +diff --git a/dpdk/drivers/net/ngbe/base/ngbe_phy_rtl.h b/dpdk/drivers/net/ngbe/base/ngbe_phy_rtl.h +index b2fbc4f74d..6093ee7d5c 100644 +--- a/dpdk/drivers/net/ngbe/base/ngbe_phy_rtl.h ++++ b/dpdk/drivers/net/ngbe/base/ngbe_phy_rtl.h +@@ -35,6 +35,8 @@ + #define RTL_ANLPAR_LP MS16(10, 0x3) + #define RTL_GBCR 0x9 + #define RTL_GBCR_1000F MS16(9, 0x1) ++#define RTL_GBSR 0xA ++#define RTL_GBSR_LRS MS16(13, 0x1) + /* Page 0xa42*/ + #define RTL_GSR 0x10 + #define RTL_GSR_ST MS16(0, 0x7) +diff --git a/dpdk/drivers/net/ngbe/base/ngbe_phy_yt.c b/dpdk/drivers/net/ngbe/base/ngbe_phy_yt.c +index c88946f7c3..754faadd6a 100644 +--- a/dpdk/drivers/net/ngbe/base/ngbe_phy_yt.c ++++ b/dpdk/drivers/net/ngbe/base/ngbe_phy_yt.c +@@ -100,11 +100,17 @@ s32 ngbe_write_phy_reg_sds_ext_yt(struct ngbe_hw *hw, + + s32 ngbe_init_phy_yt(struct ngbe_hw *hw) + { +- /* close sds area register */ +- ngbe_write_phy_reg_ext_yt(hw, YT_SMI_PHY, 0, 0); +- /* enable interrupts */ +- ngbe_write_phy_reg_mdi(hw, YT_INTR, 0, +- YT_INTR_ENA_MASK | YT_SDS_INTR_ENA_MASK); ++ rte_spinlock_init(&hw->phy_lock); ++ ++ if (hw->lsc) { ++ rte_spinlock_lock(&hw->phy_lock); ++ /* close sds area register */ ++ ngbe_write_phy_reg_ext_yt(hw, YT_SMI_PHY, 0, 0); ++ /* enable interrupts */ ++ ngbe_write_phy_reg_mdi(hw, YT_INTR, 0, ++ YT_INTR_ENA_MASK | YT_SDS_INTR_ENA_MASK); ++ rte_spinlock_unlock(&hw->phy_lock); ++ } + + hw->phy.set_phy_power(hw, false); + +@@ -123,7 +129,9 @@ s32 ngbe_setup_phy_link_yt(struct ngbe_hw *hw, u32 speed, + hw->phy.autoneg_advertised = 0; + + /* check chip_mode first */ ++ rte_spinlock_lock(&hw->phy_lock); + ngbe_read_phy_reg_ext_yt(hw, YT_CHIP, 0, &value); ++ rte_spinlock_unlock(&hw->phy_lock); + if ((value & YT_CHIP_MODE_MASK) == YT_CHIP_MODE_SEL(0)) { + /* UTP to rgmii */ + if (!hw->mac.autoneg) { +@@ -146,11 +154,14 @@ s32 ngbe_setup_phy_link_yt(struct ngbe_hw *hw, u32 speed, + } + /* duplex full */ + value |= YT_BCR_DUPLEX | YT_BCR_RESET; ++ rte_spinlock_lock(&hw->phy_lock); + ngbe_write_phy_reg_mdi(hw, YT_BCR, 0, value); ++ rte_spinlock_unlock(&hw->phy_lock); + + goto skip_an; + } + ++ rte_spinlock_lock(&hw->phy_lock); + /*disable 100/10base-T Self-negotiation ability*/ + ngbe_read_phy_reg_mdi(hw, YT_ANA, 0, &value); + value &= ~(YT_ANA_100BASET_FULL | YT_ANA_100BASET_HALF | +@@ -189,6 +200,7 @@ s32 ngbe_setup_phy_link_yt(struct ngbe_hw *hw, u32 speed, + ngbe_read_phy_reg_mdi(hw, YT_BCR, 0, &value); + value |= YT_BCR_RESET | YT_BCR_ANE | YT_BCR_RESTART_AN; + ngbe_write_phy_reg_mdi(hw, YT_BCR, 0, value); ++ rte_spinlock_unlock(&hw->phy_lock); + skip_an: + hw->phy.set_phy_power(hw, true); + } else if ((value & YT_CHIP_MODE_MASK) == YT_CHIP_MODE_SEL(1)) { +@@ -199,6 +211,7 @@ skip_an: + value = YT_RGMII_CONF1_RXDELAY | + YT_RGMII_CONF1_TXDELAY_FE | + YT_RGMII_CONF1_TXDELAY; ++ rte_spinlock_lock(&hw->phy_lock); + ngbe_write_phy_reg_ext_yt(hw, YT_RGMII_CONF1, 0, value); + value = YT_CHIP_MODE_SEL(1) | + YT_CHIP_SW_LDO_EN | +@@ -225,17 +238,21 @@ skip_an: + value = YT_BCR_RESET | YT_BCR_DUPLEX | + YT_BCR_SPEED_SELECT1; + hw->phy.write_reg(hw, YT_BCR, 0, value); ++ rte_spinlock_unlock(&hw->phy_lock); + + hw->phy.set_phy_power(hw, true); + } else if ((value & YT_CHIP_MODE_MASK) == YT_CHIP_MODE_SEL(2)) { + hw->phy.set_phy_power(hw, true); + ++ rte_spinlock_lock(&hw->phy_lock); + hw->phy.read_reg(hw, YT_SPST, 0, &value); ++ rte_spinlock_unlock(&hw->phy_lock); + if (value & YT_SPST_LINK) { + /* fiber up */ + hw->phy.autoneg_advertised |= NGBE_LINK_SPEED_1GB_FULL; + } else { + /* utp up */ ++ rte_spinlock_lock(&hw->phy_lock); + /*disable 100/10base-T Self-negotiation ability*/ + ngbe_read_phy_reg_mdi(hw, YT_ANA, 0, &value); + value &= ~(YT_ANA_100BASET_FULL | YT_ANA_100BASET_HALF | +@@ -279,10 +296,12 @@ skip_an: + ngbe_read_phy_reg_mdi(hw, YT_BCR, 0, &value); + value |= YT_BCR_RESET; + ngbe_write_phy_reg_mdi(hw, YT_BCR, 0, value); ++ rte_spinlock_unlock(&hw->phy_lock); + } + } else if ((value & YT_CHIP_MODE_MASK) == YT_CHIP_MODE_SEL(4)) { + hw->phy.autoneg_advertised |= NGBE_LINK_SPEED_1GB_FULL; + ++ rte_spinlock_lock(&hw->phy_lock); + ngbe_read_phy_reg_ext_yt(hw, YT_RGMII_CONF1, 0, &value); + value |= YT_RGMII_CONF1_MODE; + ngbe_write_phy_reg_ext_yt(hw, YT_RGMII_CONF1, 0, value); +@@ -297,6 +316,7 @@ skip_an: + ngbe_read_phy_reg_ext_yt(hw, YT_CHIP, 0, &value); + value &= ~YT_SMI_PHY_SW_RST; + ngbe_write_phy_reg_ext_yt(hw, YT_CHIP, 0, value); ++ rte_spinlock_unlock(&hw->phy_lock); + + hw->phy.set_phy_power(hw, true); + } else if ((value & YT_CHIP_MODE_MASK) == YT_CHIP_MODE_SEL(5)) { +@@ -320,7 +340,9 @@ skip_an: + } + /* duplex full */ + value |= YT_BCR_DUPLEX | YT_BCR_RESET; ++ rte_spinlock_lock(&hw->phy_lock); + hw->phy.write_reg(hw, YT_BCR, 0, value); ++ rte_spinlock_unlock(&hw->phy_lock); + + goto skip_an_sr; + } +@@ -339,19 +361,23 @@ skip_an: + + /* duplex full */ + value |= YT_BCR_DUPLEX | YT_BCR_RESET; ++ rte_spinlock_lock(&hw->phy_lock); + hw->phy.write_reg(hw, YT_BCR, 0, value); + + /* software reset to make the above configuration take effect */ + hw->phy.read_reg(hw, YT_BCR, 0, &value); + value |= YT_BCR_RESET | YT_BCR_ANE | YT_BCR_RESTART_AN; + hw->phy.write_reg(hw, 0x0, 0, value); ++ rte_spinlock_unlock(&hw->phy_lock); + + skip_an_sr: + hw->phy.set_phy_power(hw, true); + } + ++ rte_spinlock_lock(&hw->phy_lock); + ngbe_write_phy_reg_ext_yt(hw, YT_SMI_PHY, 0, 0); + ngbe_read_phy_reg_mdi(hw, YT_INTR_STATUS, 0, &value); ++ rte_spinlock_unlock(&hw->phy_lock); + + return 0; + } +@@ -366,6 +392,7 @@ s32 ngbe_reset_phy_yt(struct ngbe_hw *hw) + hw->phy.type != ngbe_phy_yt8521s_sfi) + return NGBE_ERR_PHY_TYPE; + ++ rte_spinlock_lock(&hw->phy_lock); + /* check chip_mode first */ + ngbe_read_phy_reg_ext_yt(hw, YT_CHIP, 0, &ctrl); + if (ctrl & YT_CHIP_MODE_MASK) { +@@ -395,6 +422,7 @@ s32 ngbe_reset_phy_yt(struct ngbe_hw *hw) + msleep(1); + } + } ++ rte_spinlock_unlock(&hw->phy_lock); + + if (i == YT_PHY_RST_WAIT_PERIOD) { + DEBUGOUT("PHY reset polling failed to complete."); +@@ -409,7 +437,9 @@ s32 ngbe_get_phy_advertised_pause_yt(struct ngbe_hw *hw, u8 *pause_bit) + u16 value; + s32 status = 0; + ++ rte_spinlock_lock(&hw->phy_lock); + status = hw->phy.read_reg(hw, YT_ANA, 0, &value); ++ rte_spinlock_unlock(&hw->phy_lock); + value &= YT_FANA_PAUSE_MASK; + *pause_bit = (u8)(value >> 7); + +@@ -421,7 +451,9 @@ s32 ngbe_get_phy_lp_advertised_pause_yt(struct ngbe_hw *hw, u8 *pause_bit) + u16 value; + s32 status = 0; + ++ rte_spinlock_lock(&hw->phy_lock); + status = hw->phy.read_reg(hw, YT_LPAR, 0, &value); ++ rte_spinlock_unlock(&hw->phy_lock); + value &= YT_FLPAR_PAUSE_MASK; + *pause_bit = (u8)(value >> 7); + +@@ -433,10 +465,12 @@ s32 ngbe_set_phy_pause_adv_yt(struct ngbe_hw *hw, u16 pause_bit) + u16 value; + s32 status = 0; + ++ rte_spinlock_lock(&hw->phy_lock); + status = hw->phy.read_reg(hw, YT_ANA, 0, &value); + value &= ~YT_FANA_PAUSE_MASK; + value |= pause_bit; + status = hw->phy.write_reg(hw, YT_ANA, 0, value); ++ rte_spinlock_unlock(&hw->phy_lock); + + return status; + } +@@ -453,6 +487,7 @@ s32 ngbe_check_phy_link_yt(struct ngbe_hw *hw, + /* Initialize speed and link to default case */ + *link_up = false; + *speed = NGBE_LINK_SPEED_UNKNOWN; ++ rte_spinlock_lock(&hw->phy_lock); + + ngbe_write_phy_reg_ext_yt(hw, YT_SMI_PHY, 0, 0); + ngbe_read_phy_reg_mdi(hw, YT_INTR_STATUS, 0, &insr); +@@ -472,6 +507,7 @@ s32 ngbe_check_phy_link_yt(struct ngbe_hw *hw, + *link_up = true; + } + ++ rte_spinlock_unlock(&hw->phy_lock); + if (*link_up) { + if (phy_speed == YT_SPST_SPEED_1000M) + *speed = NGBE_LINK_SPEED_1GB_FULL; +@@ -488,6 +524,7 @@ s32 ngbe_set_phy_power_yt(struct ngbe_hw *hw, bool on) + { + u16 value = 0; + ++ rte_spinlock_lock(&hw->phy_lock); + /* power down/up in fiber mode */ + hw->phy.read_reg(hw, YT_BCR, 0, &value); + if (on) +@@ -504,6 +541,7 @@ s32 ngbe_set_phy_power_yt(struct ngbe_hw *hw, bool on) + else + value |= YT_BCR_PWDN; + ngbe_write_phy_reg_mdi(hw, YT_BCR, 0, value); ++ rte_spinlock_unlock(&hw->phy_lock); + + return 0; + } +diff --git a/dpdk/drivers/net/ngbe/base/ngbe_type.h b/dpdk/drivers/net/ngbe/base/ngbe_type.h +index aa5c41146c..8a7d2cd331 100644 +--- a/dpdk/drivers/net/ngbe/base/ngbe_type.h ++++ b/dpdk/drivers/net/ngbe/base/ngbe_type.h +@@ -116,6 +116,46 @@ struct ngbe_fc_info { + enum ngbe_fc_mode requested_mode; /* FC mode requested by caller */ + }; + ++/* Flow Control Data Sheet defined values ++ * Calculation and defines taken from 802.1bb Annex O ++ */ ++/* BitTimes (BT) conversion */ ++#define NGBE_BT2KB(BT) (((BT) + (8 * 1024 - 1)) / (8 * 1024)) ++#define NGBE_B2BT(BT) ((BT) * 8) ++ ++/* Calculate Delay to respond to PFC */ ++#define NGBE_PFC_D 672 ++ ++/* Calculate Cable Delay */ ++#define NGBE_CABLE_DC 5556 /* Delay Copper */ ++ ++/* Calculate Interface Delay */ ++#define NGBE_PHY_D 12800 ++#define NGBE_MAC_D 4096 ++#define NGBE_XAUI_D (2 * 1024) ++ ++#define NGBE_ID (NGBE_MAC_D + NGBE_XAUI_D + NGBE_PHY_D) ++ ++/* Calculate Delay incurred from higher layer */ ++#define NGBE_HD 6144 ++ ++/* Calculate PCI Bus delay for low thresholds */ ++#define NGBE_PCI_DELAY 10000 ++ ++/* Calculate delay value in bit times */ ++#define NGBE_DV(_max_frame_link, _max_frame_tc) \ ++ ((36 * \ ++ (NGBE_B2BT(_max_frame_link) + \ ++ NGBE_PFC_D + \ ++ (2 * NGBE_CABLE_DC) + \ ++ (2 * NGBE_ID) + \ ++ NGBE_HD) / 25 + 1) + \ ++ 2 * NGBE_B2BT(_max_frame_tc)) ++ ++#define NGBE_LOW_DV(_max_frame_tc) \ ++ (2 * ((2 * NGBE_B2BT(_max_frame_tc) + \ ++ (36 * NGBE_PCI_DELAY / 25) + 1))) ++ + /* Statistics counters collected by the MAC */ + /* PB[] RxTx */ + struct ngbe_pb_stats { +@@ -431,8 +471,10 @@ struct ngbe_hw { + bool offset_loaded; + bool is_pf; + bool gpio_ctl; ++ bool lsc; + u32 led_conf; + bool init_phy; ++ rte_spinlock_t phy_lock; + struct { + u64 rx_qp_packets; + u64 tx_qp_packets; +diff --git a/dpdk/drivers/net/ngbe/ngbe_ethdev.c b/dpdk/drivers/net/ngbe/ngbe_ethdev.c +index afdb3ad41f..08e14a05c9 100644 +--- a/dpdk/drivers/net/ngbe/ngbe_ethdev.c ++++ b/dpdk/drivers/net/ngbe/ngbe_ethdev.c +@@ -90,6 +90,7 @@ static int ngbe_dev_misc_interrupt_setup(struct rte_eth_dev *dev); + static int ngbe_dev_rxq_interrupt_setup(struct rte_eth_dev *dev); + static void ngbe_dev_interrupt_handler(void *param); + static void ngbe_configure_msix(struct rte_eth_dev *dev); ++static void ngbe_pbthresh_set(struct rte_eth_dev *dev); + + #define NGBE_SET_HWSTRIP(h, q) do {\ + uint32_t idx = (q) / (sizeof((h)->bitmap[0]) * NBBY); \ +@@ -160,7 +161,9 @@ static const struct rte_ngbe_xstats_name_off rte_ngbe_stats_strings[] = { + HW_XSTAT(tx_total_packets), + HW_XSTAT(rx_total_missed_packets), + HW_XSTAT(rx_broadcast_packets), ++ HW_XSTAT(tx_broadcast_packets), + HW_XSTAT(rx_multicast_packets), ++ HW_XSTAT(tx_multicast_packets), + HW_XSTAT(rx_management_packets), + HW_XSTAT(tx_management_packets), + HW_XSTAT(rx_management_dropped), +@@ -972,9 +975,6 @@ ngbe_dev_start(struct rte_eth_dev *dev) + + PMD_INIT_FUNC_TRACE(); + +- /* Stop the link setup handler before resetting the HW. */ +- rte_eal_alarm_cancel(ngbe_dev_setup_link_alarm_handler, dev); +- + /* disable uio/vfio intr/eventfd mapping */ + rte_intr_disable(intr_handle); + +@@ -1038,6 +1038,7 @@ ngbe_dev_start(struct rte_eth_dev *dev) + } + + hw->mac.setup_pba(hw); ++ ngbe_pbthresh_set(dev); + ngbe_configure_port(dev); + + err = ngbe_dev_rxtx_start(dev); +@@ -1050,6 +1051,8 @@ ngbe_dev_start(struct rte_eth_dev *dev) + if (hw->is_pf && dev->data->dev_conf.lpbk_mode) + goto skip_link_setup; + ++ hw->lsc = dev->data->dev_conf.intr_conf.lsc; ++ + err = hw->mac.check_link(hw, &speed, &link_up, 0); + if (err != 0) + goto error; +@@ -1164,12 +1167,10 @@ ngbe_dev_stop(struct rte_eth_dev *dev) + int vf; + + if (hw->adapter_stopped) +- return 0; ++ goto out; + + PMD_INIT_FUNC_TRACE(); + +- rte_eal_alarm_cancel(ngbe_dev_setup_link_alarm_handler, dev); +- + if (hw->gpio_ctl) { + /* gpio0 is used to power on/off control*/ + wr32(hw, NGBE_GPIODATA, NGBE_GPIOBIT_0); +@@ -1188,8 +1189,6 @@ ngbe_dev_stop(struct rte_eth_dev *dev) + for (vf = 0; vfinfo != NULL && vf < pci_dev->max_vfs; vf++) + vfinfo[vf].clear_to_send = false; + +- hw->phy.set_phy_power(hw, false); +- + ngbe_dev_clear_queues(dev); + + /* Clear stored conf */ +@@ -1216,6 +1215,10 @@ ngbe_dev_stop(struct rte_eth_dev *dev) + hw->adapter_stopped = true; + dev->data->dev_started = 0; + ++out: ++ /* close phy to prevent reset in dev_close from restarting physical link */ ++ hw->phy.set_phy_power(hw, false); ++ + return 0; + } + +@@ -1259,6 +1262,9 @@ ngbe_dev_close(struct rte_eth_dev *dev) + + PMD_INIT_FUNC_TRACE(); + ++ if (rte_eal_process_type() != RTE_PROC_PRIMARY) ++ return 0; ++ + ngbe_pf_reset_hw(hw); + + ngbe_dev_stop(dev); +@@ -1869,24 +1875,6 @@ ngbe_dev_supported_ptypes_get(struct rte_eth_dev *dev) + return NULL; + } + +-void +-ngbe_dev_setup_link_alarm_handler(void *param) +-{ +- struct rte_eth_dev *dev = (struct rte_eth_dev *)param; +- struct ngbe_hw *hw = ngbe_dev_hw(dev); +- struct ngbe_interrupt *intr = ngbe_dev_intr(dev); +- u32 speed; +- bool autoneg = false; +- +- speed = hw->phy.autoneg_advertised; +- if (!speed) +- hw->mac.get_link_capabilities(hw, &speed, &autoneg); +- +- hw->mac.setup_link(hw, speed, true); +- +- intr->flags &= ~NGBE_FLAG_NEED_LINK_CONFIG; +-} +- + /* return 0 means link status changed, -1 means not changed */ + int + ngbe_dev_link_update_share(struct rte_eth_dev *dev, +@@ -1896,7 +1884,6 @@ ngbe_dev_link_update_share(struct rte_eth_dev *dev, + struct rte_eth_link link; + u32 link_speed = NGBE_LINK_SPEED_UNKNOWN; + u32 lan_speed = 0; +- struct ngbe_interrupt *intr = ngbe_dev_intr(dev); + bool link_up; + int err; + int wait = 1; +@@ -1910,9 +1897,6 @@ ngbe_dev_link_update_share(struct rte_eth_dev *dev, + + hw->mac.get_link_status = true; + +- if (intr->flags & NGBE_FLAG_NEED_LINK_CONFIG) +- return rte_eth_linkstatus_set(dev, &link); +- + /* check if it needs to wait to complete, if lsc interrupt is enabled */ + if (wait_to_complete == 0 || dev->data->dev_conf.intr_conf.lsc != 0) + wait = 0; +@@ -1927,7 +1911,6 @@ ngbe_dev_link_update_share(struct rte_eth_dev *dev, + if (!link_up) + return rte_eth_linkstatus_set(dev, &link); + +- intr->flags &= ~NGBE_FLAG_NEED_LINK_CONFIG; + link.link_status = RTE_ETH_LINK_UP; + link.link_duplex = RTE_ETH_LINK_FULL_DUPLEX; + +@@ -1961,6 +1944,8 @@ ngbe_dev_link_update_share(struct rte_eth_dev *dev, + wr32m(hw, NGBE_MACTXCFG, NGBE_MACTXCFG_SPEED_MASK, + NGBE_MACTXCFG_SPEED_1G | NGBE_MACTXCFG_TE); + } ++ wr32m(hw, NGBE_MACRXFLT, NGBE_MACRXFLT_PROMISC, ++ NGBE_MACRXFLT_PROMISC); + } + + return rte_eth_linkstatus_set(dev, &link); +@@ -2380,6 +2365,93 @@ ngbe_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) + return -EIO; + } + ++/* Additional bittime to account for NGBE framing */ ++#define NGBE_ETH_FRAMING 20 ++ ++/* ++ * ngbe_fc_hpbthresh_set - calculate high water mark for flow control ++ * ++ * @dv_id: device interface delay ++ * @pb: packet buffer to calculate ++ */ ++static s32 ++ngbe_fc_hpbthresh_set(struct rte_eth_dev *dev) ++{ ++ struct ngbe_hw *hw = ngbe_dev_hw(dev); ++ struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); ++ u32 max_frame_size, tc, dv_id, rx_pb; ++ s32 kb, marker; ++ ++ /* Calculate max LAN frame size */ ++ max_frame_size = rd32m(hw, NGBE_FRMSZ, NGBE_FRMSZ_MAX_MASK); ++ tc = max_frame_size + NGBE_ETH_FRAMING; ++ ++ /* Calculate delay value for device */ ++ dv_id = NGBE_DV(tc, tc); ++ ++ /* Loopback switch introduces additional latency */ ++ if (pci_dev->max_vfs) ++ dv_id += NGBE_B2BT(tc); ++ ++ /* Delay value is calculated in bit times convert to KB */ ++ kb = NGBE_BT2KB(dv_id); ++ rx_pb = rd32(hw, NGBE_PBRXSIZE) >> 10; ++ ++ marker = rx_pb - kb; ++ ++ /* It is possible that the packet buffer is not large enough ++ * to provide required headroom. In this case throw an error ++ * to user and do the best we can. ++ */ ++ if (marker < 0) { ++ PMD_DRV_LOG(WARNING, "Packet Buffer can not provide enough headroom to support flow control."); ++ marker = tc + 1; ++ } ++ ++ return marker; ++} ++ ++/* ++ * ngbe_fc_lpbthresh_set - calculate low water mark for flow control ++ * ++ * @dv_id: device interface delay ++ */ ++static s32 ++ngbe_fc_lpbthresh_set(struct rte_eth_dev *dev) ++{ ++ struct ngbe_hw *hw = ngbe_dev_hw(dev); ++ u32 max_frame_size, tc, dv_id; ++ s32 kb; ++ ++ /* Calculate max LAN frame size */ ++ max_frame_size = rd32m(hw, NGBE_FRMSZ, NGBE_FRMSZ_MAX_MASK); ++ tc = max_frame_size + NGBE_ETH_FRAMING; ++ ++ /* Calculate delay value for device */ ++ dv_id = NGBE_LOW_DV(tc); ++ ++ /* Delay value is calculated in bit times convert to KB */ ++ kb = NGBE_BT2KB(dv_id); ++ ++ return kb; ++} ++ ++/* ++ * ngbe_pbthresh_setup - calculate and setup high low water marks ++ */ ++static void ++ngbe_pbthresh_set(struct rte_eth_dev *dev) ++{ ++ struct ngbe_hw *hw = ngbe_dev_hw(dev); ++ ++ hw->fc.high_water = ngbe_fc_hpbthresh_set(dev); ++ hw->fc.low_water = ngbe_fc_lpbthresh_set(dev); ++ ++ /* Low water marks must not be larger than high water marks */ ++ if (hw->fc.low_water > hw->fc.high_water) ++ hw->fc.low_water = 0; ++} ++ + int + ngbe_dev_rss_reta_update(struct rte_eth_dev *dev, + struct rte_eth_rss_reta_entry64 *reta_conf, +diff --git a/dpdk/drivers/net/ngbe/ngbe_ethdev.h b/dpdk/drivers/net/ngbe/ngbe_ethdev.h +index 8d500fd38c..bb96f6a5e7 100644 +--- a/dpdk/drivers/net/ngbe/ngbe_ethdev.h ++++ b/dpdk/drivers/net/ngbe/ngbe_ethdev.h +@@ -341,7 +341,6 @@ void ngbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev, + uint16_t queue, bool on); + void ngbe_config_vlan_strip_on_all_queues(struct rte_eth_dev *dev, + int mask); +-void ngbe_dev_setup_link_alarm_handler(void *param); + void ngbe_read_stats_registers(struct ngbe_hw *hw, + struct ngbe_hw_stats *hw_stats); + +diff --git a/dpdk/drivers/net/ngbe/ngbe_rxtx.c b/dpdk/drivers/net/ngbe/ngbe_rxtx.c +index 9fd24fa444..54a6f6a887 100644 +--- a/dpdk/drivers/net/ngbe/ngbe_rxtx.c ++++ b/dpdk/drivers/net/ngbe/ngbe_rxtx.c +@@ -24,15 +24,11 @@ + + /* Bit Mask to indicate what bits required for building Tx context */ + static const u64 NGBE_TX_OFFLOAD_MASK = (RTE_MBUF_F_TX_IP_CKSUM | +- RTE_MBUF_F_TX_OUTER_IPV6 | +- RTE_MBUF_F_TX_OUTER_IPV4 | + RTE_MBUF_F_TX_IPV6 | + RTE_MBUF_F_TX_IPV4 | + RTE_MBUF_F_TX_VLAN | + RTE_MBUF_F_TX_L4_MASK | + RTE_MBUF_F_TX_TCP_SEG | +- RTE_MBUF_F_TX_TUNNEL_MASK | +- RTE_MBUF_F_TX_OUTER_IP_CKSUM | + NGBE_TX_IEEE1588_TMST); + + #define NGBE_TX_OFFLOAD_NOTSUP_MASK \ +@@ -333,34 +329,15 @@ ngbe_set_xmit_ctx(struct ngbe_tx_queue *txq, + } + + vlan_macip_lens = NGBE_TXD_IPLEN(tx_offload.l3_len >> 1); +- +- if (ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) { +- tx_offload_mask.outer_tun_len |= ~0; +- tx_offload_mask.outer_l2_len |= ~0; +- tx_offload_mask.outer_l3_len |= ~0; +- tx_offload_mask.l2_len |= ~0; +- tunnel_seed = NGBE_TXD_ETUNLEN(tx_offload.outer_tun_len >> 1); +- tunnel_seed |= NGBE_TXD_EIPLEN(tx_offload.outer_l3_len >> 2); +- +- switch (ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) { +- case RTE_MBUF_F_TX_TUNNEL_IPIP: +- /* for non UDP / GRE tunneling, set to 0b */ +- break; +- default: +- PMD_TX_LOG(ERR, "Tunnel type not supported"); +- return; +- } +- vlan_macip_lens |= NGBE_TXD_MACLEN(tx_offload.outer_l2_len); +- } else { +- tunnel_seed = 0; +- vlan_macip_lens |= NGBE_TXD_MACLEN(tx_offload.l2_len); +- } ++ vlan_macip_lens |= NGBE_TXD_MACLEN(tx_offload.l2_len); + + if (ol_flags & RTE_MBUF_F_TX_VLAN) { + tx_offload_mask.vlan_tci |= ~0; + vlan_macip_lens |= NGBE_TXD_VLAN(tx_offload.vlan_tci); + } + ++ tunnel_seed = 0; ++ + txq->ctx_cache[ctx_idx].flags = ol_flags; + txq->ctx_cache[ctx_idx].tx_offload.data[0] = + tx_offload_mask.data[0] & tx_offload.data[0]; +@@ -449,16 +426,10 @@ tx_desc_ol_flags_to_cmdtype(uint64_t ol_flags) + return cmdtype; + } + +-static inline uint8_t +-tx_desc_ol_flags_to_ptid(uint64_t oflags, uint32_t ptype) ++static inline uint32_t ++tx_desc_ol_flags_to_ptype(uint64_t oflags) + { +- bool tun; +- +- if (ptype) +- return ngbe_encode_ptype(ptype); +- +- /* Only support flags in NGBE_TX_OFFLOAD_MASK */ +- tun = !!(oflags & RTE_MBUF_F_TX_TUNNEL_MASK); ++ uint32_t ptype; + + /* L2 level */ + ptype = RTE_PTYPE_L2_ETHER; +@@ -466,41 +437,36 @@ tx_desc_ol_flags_to_ptid(uint64_t oflags, uint32_t ptype) + ptype |= RTE_PTYPE_L2_ETHER_VLAN; + + /* L3 level */ +- if (oflags & (RTE_MBUF_F_TX_OUTER_IPV4 | RTE_MBUF_F_TX_OUTER_IP_CKSUM)) +- ptype |= RTE_PTYPE_L3_IPV4; +- else if (oflags & (RTE_MBUF_F_TX_OUTER_IPV6)) +- ptype |= RTE_PTYPE_L3_IPV6; +- + if (oflags & (RTE_MBUF_F_TX_IPV4 | RTE_MBUF_F_TX_IP_CKSUM)) +- ptype |= (tun ? RTE_PTYPE_INNER_L3_IPV4 : RTE_PTYPE_L3_IPV4); ++ ptype |= RTE_PTYPE_L3_IPV4; + else if (oflags & (RTE_MBUF_F_TX_IPV6)) +- ptype |= (tun ? RTE_PTYPE_INNER_L3_IPV6 : RTE_PTYPE_L3_IPV6); ++ ptype |= RTE_PTYPE_L3_IPV6; + + /* L4 level */ + switch (oflags & (RTE_MBUF_F_TX_L4_MASK)) { + case RTE_MBUF_F_TX_TCP_CKSUM: +- ptype |= (tun ? RTE_PTYPE_INNER_L4_TCP : RTE_PTYPE_L4_TCP); ++ ptype |= RTE_PTYPE_L4_TCP; + break; + case RTE_MBUF_F_TX_UDP_CKSUM: +- ptype |= (tun ? RTE_PTYPE_INNER_L4_UDP : RTE_PTYPE_L4_UDP); ++ ptype |= RTE_PTYPE_L4_UDP; + break; + case RTE_MBUF_F_TX_SCTP_CKSUM: +- ptype |= (tun ? RTE_PTYPE_INNER_L4_SCTP : RTE_PTYPE_L4_SCTP); ++ ptype |= RTE_PTYPE_L4_SCTP; + break; + } + + if (oflags & RTE_MBUF_F_TX_TCP_SEG) +- ptype |= (tun ? RTE_PTYPE_INNER_L4_TCP : RTE_PTYPE_L4_TCP); +- +- /* Tunnel */ +- switch (oflags & RTE_MBUF_F_TX_TUNNEL_MASK) { +- case RTE_MBUF_F_TX_TUNNEL_IPIP: +- case RTE_MBUF_F_TX_TUNNEL_IP: +- ptype |= RTE_PTYPE_L2_ETHER | +- RTE_PTYPE_L3_IPV4 | +- RTE_PTYPE_TUNNEL_IP; +- break; +- } ++ ptype |= RTE_PTYPE_L4_TCP; ++ ++ return ptype; ++} ++ ++static inline uint8_t ++tx_desc_ol_flags_to_ptid(uint64_t oflags) ++{ ++ uint32_t ptype; ++ ++ ptype = tx_desc_ol_flags_to_ptype(oflags); + + return ngbe_encode_ptype(ptype); + } +@@ -622,16 +588,12 @@ ngbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + /* If hardware offload required */ + tx_ol_req = ol_flags & NGBE_TX_OFFLOAD_MASK; + if (tx_ol_req) { +- tx_offload.ptid = tx_desc_ol_flags_to_ptid(tx_ol_req, +- tx_pkt->packet_type); ++ tx_offload.ptid = tx_desc_ol_flags_to_ptid(tx_ol_req); + tx_offload.l2_len = tx_pkt->l2_len; + tx_offload.l3_len = tx_pkt->l3_len; + tx_offload.l4_len = tx_pkt->l4_len; + tx_offload.vlan_tci = tx_pkt->vlan_tci; + tx_offload.tso_segsz = tx_pkt->tso_segsz; +- tx_offload.outer_l2_len = tx_pkt->outer_l2_len; +- tx_offload.outer_l3_len = tx_pkt->outer_l3_len; +- tx_offload.outer_tun_len = 0; + + /* If new context need be built or reuse the exist ctx*/ + ctx = what_ctx_update(txq, tx_ol_req, tx_offload); +@@ -752,10 +714,6 @@ ngbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + */ + pkt_len -= (tx_offload.l2_len + + tx_offload.l3_len + tx_offload.l4_len); +- pkt_len -= +- (tx_pkt->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) +- ? tx_offload.outer_l2_len + +- tx_offload.outer_l3_len : 0; + } + + /* +@@ -1265,11 +1223,22 @@ ngbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + * of accesses cannot be reordered by the compiler. If they were + * not volatile, they could be reordered which could lead to + * using invalid descriptor fields when read from rxd. ++ * ++ * Meanwhile, to prevent the CPU from executing out of order, we ++ * need to use a proper memory barrier to ensure the memory ++ * ordering below. + */ + rxdp = &rx_ring[rx_id]; + staterr = rxdp->qw1.lo.status; + if (!(staterr & rte_cpu_to_le_32(NGBE_RXD_STAT_DD))) + break; ++ ++ /* ++ * Use acquire fence to ensure that status_error which includes ++ * DD bit is loaded before loading of other descriptor words. ++ */ ++ rte_atomic_thread_fence(__ATOMIC_ACQUIRE); ++ + rxd = *rxdp; + + /* +@@ -1496,6 +1465,12 @@ next_desc: + if (!(staterr & NGBE_RXD_STAT_DD)) + break; + ++ /* ++ * Use acquire fence to ensure that status_error which includes ++ * DD bit is loaded before loading of other descriptor words. ++ */ ++ rte_atomic_thread_fence(__ATOMIC_ACQUIRE); ++ + rxd = *rxdp; + + PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u " +@@ -1939,12 +1914,8 @@ ngbe_get_tx_port_offloads(struct rte_eth_dev *dev) + RTE_ETH_TX_OFFLOAD_UDP_CKSUM | + RTE_ETH_TX_OFFLOAD_TCP_CKSUM | + RTE_ETH_TX_OFFLOAD_SCTP_CKSUM | +- RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM | + RTE_ETH_TX_OFFLOAD_TCP_TSO | + RTE_ETH_TX_OFFLOAD_UDP_TSO | +- RTE_ETH_TX_OFFLOAD_UDP_TNL_TSO | +- RTE_ETH_TX_OFFLOAD_IP_TNL_TSO | +- RTE_ETH_TX_OFFLOAD_IPIP_TNL_TSO | + RTE_ETH_TX_OFFLOAD_MULTI_SEGS; + + if (hw->is_pf) +@@ -2237,6 +2208,7 @@ ngbe_get_rx_port_offloads(struct rte_eth_dev *dev) + RTE_ETH_RX_OFFLOAD_TCP_CKSUM | + RTE_ETH_RX_OFFLOAD_KEEP_CRC | + RTE_ETH_RX_OFFLOAD_VLAN_FILTER | ++ RTE_ETH_RX_OFFLOAD_RSS_HASH | + RTE_ETH_RX_OFFLOAD_SCATTER; + + if (hw->is_pf) +@@ -2460,6 +2432,7 @@ ngbe_dev_clear_queues(struct rte_eth_dev *dev) + if (txq != NULL) { + txq->ops->release_mbufs(txq); + txq->ops->reset(txq); ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; + } + } + +@@ -2469,6 +2442,7 @@ ngbe_dev_clear_queues(struct rte_eth_dev *dev) + if (rxq != NULL) { + ngbe_rx_queue_release_mbufs(rxq); + ngbe_reset_rx_queue(adapter, rxq); ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; + } + } + } +diff --git a/dpdk/drivers/net/null/rte_eth_null.c b/dpdk/drivers/net/null/rte_eth_null.c +index 47d9554ec5..1fbe572bd1 100644 +--- a/dpdk/drivers/net/null/rte_eth_null.c ++++ b/dpdk/drivers/net/null/rte_eth_null.c +@@ -188,21 +188,36 @@ eth_dev_configure(struct rte_eth_dev *dev __rte_unused) + static int + eth_dev_start(struct rte_eth_dev *dev) + { ++ uint16_t i; ++ + if (dev == NULL) + return -EINVAL; + + dev->data->dev_link.link_status = RTE_ETH_LINK_UP; ++ ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ + return 0; + } + + static int + eth_dev_stop(struct rte_eth_dev *dev) + { ++ uint16_t i; ++ + if (dev == NULL) + return 0; + + dev->data->dev_link.link_status = RTE_ETH_LINK_DOWN; + ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ + return 0; + } + +diff --git a/dpdk/drivers/net/octeon_ep/otx_ep_ethdev.c b/dpdk/drivers/net/octeon_ep/otx_ep_ethdev.c +index c8f4abe4ca..af0c5c824f 100644 +--- a/dpdk/drivers/net/octeon_ep/otx_ep_ethdev.c ++++ b/dpdk/drivers/net/octeon_ep/otx_ep_ethdev.c +@@ -76,6 +76,11 @@ otx_ep_dev_start(struct rte_eth_dev *eth_dev) + + otx_ep_info("dev started\n"); + ++ for (q = 0; q < eth_dev->data->nb_rx_queues; q++) ++ eth_dev->data->rx_queue_state[q] = RTE_ETH_QUEUE_STATE_STARTED; ++ for (q = 0; q < eth_dev->data->nb_tx_queues; q++) ++ eth_dev->data->tx_queue_state[q] = RTE_ETH_QUEUE_STATE_STARTED; ++ + return 0; + } + +@@ -84,9 +89,15 @@ static int + otx_ep_dev_stop(struct rte_eth_dev *eth_dev) + { + struct otx_ep_device *otx_epvf = OTX_EP_DEV(eth_dev); ++ uint16_t i; + + otx_epvf->fn_list.disable_io_queues(otx_epvf); + ++ for (i = 0; i < eth_dev->data->nb_rx_queues; i++) ++ eth_dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ for (i = 0; i < eth_dev->data->nb_tx_queues; i++) ++ eth_dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ + return 0; + } + +diff --git a/dpdk/drivers/net/octeontx/octeontx_ethdev.c b/dpdk/drivers/net/octeontx/octeontx_ethdev.c +index d52a3e73d5..2b97f0163e 100644 +--- a/dpdk/drivers/net/octeontx/octeontx_ethdev.c ++++ b/dpdk/drivers/net/octeontx/octeontx_ethdev.c +@@ -732,6 +732,11 @@ octeontx_dev_start(struct rte_eth_dev *dev) + } + + /* Success */ ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ + return ret; + + pki_port_stop_error: +@@ -746,6 +751,7 @@ static int + octeontx_dev_stop(struct rte_eth_dev *dev) + { + struct octeontx_nic *nic = octeontx_pmd_priv(dev); ++ uint16_t i; + int ret; + + PMD_INIT_FUNC_TRACE(); +@@ -772,6 +778,11 @@ octeontx_dev_stop(struct rte_eth_dev *dev) + return ret; + } + ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ + return 0; + } + +diff --git a/dpdk/drivers/net/pfe/pfe_ethdev.c b/dpdk/drivers/net/pfe/pfe_ethdev.c +index 0352a57950..551f3cf193 100644 +--- a/dpdk/drivers/net/pfe/pfe_ethdev.c ++++ b/dpdk/drivers/net/pfe/pfe_ethdev.c +@@ -241,6 +241,7 @@ pfe_eth_open(struct rte_eth_dev *dev) + struct pfe_eth_priv_s *priv = dev->data->dev_private; + struct hif_client_s *client; + struct hif_shm *hif_shm; ++ uint16_t i; + int rc; + + /* Register client driver with HIF */ +@@ -318,6 +319,10 @@ pfe_eth_open(struct rte_eth_dev *dev) + PFE_PMD_INFO("PFE INTERRUPT Mode enabled"); + } + ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; + + err0: + return rc; +@@ -361,6 +366,7 @@ static int + pfe_eth_stop(struct rte_eth_dev *dev/*, int wake*/) + { + struct pfe_eth_priv_s *priv = dev->data->dev_private; ++ uint16_t i; + + dev->data->dev_started = 0; + +@@ -370,6 +376,11 @@ pfe_eth_stop(struct rte_eth_dev *dev/*, int wake*/) + dev->rx_pkt_burst = rte_eth_pkt_burst_dummy; + dev->tx_pkt_burst = rte_eth_pkt_burst_dummy; + ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ + return 0; + } + +diff --git a/dpdk/drivers/net/qede/qede_ethdev.c b/dpdk/drivers/net/qede/qede_ethdev.c +index a4923670d6..22cd470646 100644 +--- a/dpdk/drivers/net/qede/qede_ethdev.c ++++ b/dpdk/drivers/net/qede/qede_ethdev.c +@@ -2142,6 +2142,7 @@ int qede_rss_hash_update(struct rte_eth_dev *eth_dev, + rss_params.rss_enable = 1; + } + ++ rss_params.update_rss_ind_table = 1; + rss_params.update_rss_config = 1; + /* tbl_size has to be set with capabilities */ + rss_params.rss_table_size_log = 7; +diff --git a/dpdk/drivers/net/ring/rte_eth_ring.c b/dpdk/drivers/net/ring/rte_eth_ring.c +index bd5a47dd90..80f1859403 100644 +--- a/dpdk/drivers/net/ring/rte_eth_ring.c ++++ b/dpdk/drivers/net/ring/rte_eth_ring.c +@@ -111,15 +111,30 @@ eth_dev_start(struct rte_eth_dev *dev) + static int + eth_dev_stop(struct rte_eth_dev *dev) + { ++ uint16_t i; ++ + dev->data->dev_started = 0; + dev->data->dev_link.link_status = RTE_ETH_LINK_DOWN; ++ ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; + return 0; + } + + static int + eth_dev_set_link_down(struct rte_eth_dev *dev) + { ++ uint16_t i; ++ + dev->data->dev_link.link_status = RTE_ETH_LINK_DOWN; ++ ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ + return 0; + } + +diff --git a/dpdk/drivers/net/sfc/sfc_dp_rx.h b/dpdk/drivers/net/sfc/sfc_dp_rx.h +index 246adbd87c..8a504bdcf1 100644 +--- a/dpdk/drivers/net/sfc/sfc_dp_rx.h ++++ b/dpdk/drivers/net/sfc/sfc_dp_rx.h +@@ -69,6 +69,7 @@ struct sfc_dp_rx_qcreate_info { + /** Receive queue flags initializer */ + unsigned int flags; + #define SFC_RXQ_FLAG_RSS_HASH 0x1 ++#define SFC_RXQ_FLAG_INGRESS_MPORT 0x2 + + /** Rx queue size */ + unsigned int rxq_entries; +diff --git a/dpdk/drivers/net/sfc/sfc_ef100_rx.c b/dpdk/drivers/net/sfc/sfc_ef100_rx.c +index 16cd8524d3..37b754fa33 100644 +--- a/dpdk/drivers/net/sfc/sfc_ef100_rx.c ++++ b/dpdk/drivers/net/sfc/sfc_ef100_rx.c +@@ -810,6 +810,9 @@ sfc_ef100_rx_qcreate(uint16_t port_id, uint16_t queue_id, + if (rxq->nic_dma_info->nb_regions > 0) + rxq->flags |= SFC_EF100_RXQ_NIC_DMA_MAP; + ++ if (info->flags & SFC_RXQ_FLAG_INGRESS_MPORT) ++ rxq->flags |= SFC_EF100_RXQ_INGRESS_MPORT; ++ + sfc_ef100_rx_debug(rxq, "RxQ doorbell is %p", rxq->doorbell); + + *dp_rxqp = &rxq->dp; +@@ -876,11 +879,18 @@ sfc_ef100_rx_qstart(struct sfc_dp_rxq *dp_rxq, unsigned int evq_read_ptr, + else + rxq->flags &= ~SFC_EF100_RXQ_USER_MARK; + ++ ++ /* ++ * At the moment, this feature is used only ++ * by the representor proxy Rx queue and is ++ * essential for representor support, so if ++ * it has been requested but is unsupported, ++ * point this inconsistency out to the user. ++ */ + if ((unsup_rx_prefix_fields & +- (1U << EFX_RX_PREFIX_FIELD_INGRESS_MPORT)) == 0) +- rxq->flags |= SFC_EF100_RXQ_INGRESS_MPORT; +- else +- rxq->flags &= ~SFC_EF100_RXQ_INGRESS_MPORT; ++ (1U << EFX_RX_PREFIX_FIELD_INGRESS_MPORT)) && ++ (rxq->flags & SFC_EF100_RXQ_INGRESS_MPORT)) ++ return ENOTSUP; + + rxq->prefix_size = pinfo->erpl_length; + rxq->rearm_data = sfc_ef100_mk_mbuf_rearm_data(rxq->dp.dpq.port_id, +diff --git a/dpdk/drivers/net/sfc/sfc_ef100_tx.c b/dpdk/drivers/net/sfc/sfc_ef100_tx.c +index 4c2205f7a4..1b6374775f 100644 +--- a/dpdk/drivers/net/sfc/sfc_ef100_tx.c ++++ b/dpdk/drivers/net/sfc/sfc_ef100_tx.c +@@ -405,7 +405,7 @@ sfc_ef100_tx_qdesc_send_create(const struct sfc_ef100_txq *txq, + m->l2_len + m->l3_len) >> 1; + } + +- rc = sfc_ef100_tx_map(txq, rte_mbuf_data_iova_default(m), ++ rc = sfc_ef100_tx_map(txq, rte_mbuf_data_iova(m), + rte_pktmbuf_data_len(m), &dma_addr); + if (unlikely(rc != 0)) + return rc; +diff --git a/dpdk/drivers/net/sfc/sfc_ethdev.c b/dpdk/drivers/net/sfc/sfc_ethdev.c +index 2ec743ebce..170ee57931 100644 +--- a/dpdk/drivers/net/sfc/sfc_ethdev.c ++++ b/dpdk/drivers/net/sfc/sfc_ethdev.c +@@ -2055,7 +2055,7 @@ sfc_process_mport_journal_cb(void *data, efx_mport_desc_t *mport, + struct sfc_mport_journal_ctx *ctx = data; + + if (ctx == NULL || ctx->sa == NULL) { +- sfc_err(ctx->sa, "received NULL context or SFC adapter"); ++ SFC_GENERIC_LOG(ERR, "received NULL context or SFC adapter"); + return EINVAL; + } + +diff --git a/dpdk/drivers/net/sfc/sfc_mae.c b/dpdk/drivers/net/sfc/sfc_mae.c +index 421bb6da95..b61b9658e3 100644 +--- a/dpdk/drivers/net/sfc/sfc_mae.c ++++ b/dpdk/drivers/net/sfc/sfc_mae.c +@@ -281,8 +281,10 @@ sfc_mae_attach(struct sfc_adapter *sa) + bounce_eh->buf_size = limits.eml_encap_header_size_limit; + bounce_eh->buf = rte_malloc("sfc_mae_bounce_eh", + bounce_eh->buf_size, 0); +- if (bounce_eh->buf == NULL) ++ if (bounce_eh->buf == NULL) { ++ rc = ENOMEM; + goto fail_mae_alloc_bounce_eh; ++ } + + mae->nb_outer_rule_prios_max = limits.eml_max_n_outer_prios; + mae->nb_action_rule_prios_max = limits.eml_max_n_action_prios; +@@ -1180,6 +1182,8 @@ sfc_mae_action_set_disable(struct sfc_adapter *sa, + } + + if (fw_rsrc->refcnt == 1) { ++ efx_mae_action_set_clear_fw_rsrc_ids(action_set->spec); ++ + rc = efx_mae_action_set_free(sa->nic, &fw_rsrc->aset_id); + if (rc == 0) { + sfc_dbg(sa, "disabled action_set=%p with AS_ID=0x%08x", +@@ -3896,12 +3900,10 @@ sfc_mae_rule_parse_actions(struct sfc_adapter *sa, + break; + case SFC_FT_RULE_SWITCH: + /* +- * Packets that go to the rule's AR have FT mark set (from the +- * TUNNEL rule OR's RECIRC_ID). Remove this mark in matching +- * packets. The user may have provided their own action +- * MARK above, so don't check the return value here. ++ * Packets that go to the rule's AR have FT mark set (from ++ * the TUNNEL rule OR's RECIRC_ID). Reset the mark to zero. + */ +- (void)efx_mae_action_set_populate_mark(ctx.spec, 0); ++ efx_mae_action_set_populate_mark_reset(ctx.spec); + + ctx.ft_switch_hit_counter = + &spec_mae->ft_ctx->switch_hit_counter; +@@ -3910,8 +3912,25 @@ sfc_mae_rule_parse_actions(struct sfc_adapter *sa, + SFC_ASSERT(B_FALSE); + } + ++ /* ++ * A DPDK flow entry must specify a fate action, which the parser ++ * converts into a DELIVER action in a libefx action set. An ++ * attempt to replace the action in the action set should ++ * fail. If it succeeds then report an error, as the ++ * parsed flow entry did not contain a fate action. ++ */ ++ rc = efx_mae_action_set_populate_drop(ctx.spec); ++ if (rc == 0) { ++ rc = rte_flow_error_set(error, EINVAL, ++ RTE_FLOW_ERROR_TYPE_ACTION, NULL, ++ "no fate action found"); ++ goto fail_check_fate_action; ++ } ++ + spec_mae->action_set = sfc_mae_action_set_attach(sa, &ctx); + if (spec_mae->action_set != NULL) { ++ sfc_mae_mac_addr_del(sa, ctx.src_mac); ++ sfc_mae_mac_addr_del(sa, ctx.dst_mac); + sfc_mae_encap_header_del(sa, ctx.encap_header); + efx_mae_action_set_spec_fini(sa->nic, ctx.spec); + return 0; +@@ -3924,6 +3943,7 @@ sfc_mae_rule_parse_actions(struct sfc_adapter *sa, + return 0; + + fail_action_set_add: ++fail_check_fate_action: + fail_workaround_tunnel_delivery: + fail_nb_count: + sfc_mae_encap_header_del(sa, ctx.encap_header); +diff --git a/dpdk/drivers/net/sfc/sfc_repr.c b/dpdk/drivers/net/sfc/sfc_repr.c +index 417d0073cb..79025e9052 100644 +--- a/dpdk/drivers/net/sfc/sfc_repr.c ++++ b/dpdk/drivers/net/sfc/sfc_repr.c +@@ -9,6 +9,8 @@ + + #include + ++#include ++#include + #include + #include + #include +@@ -289,6 +291,7 @@ static int + sfc_repr_dev_start(struct rte_eth_dev *dev) + { + struct sfc_repr *sr = sfc_repr_by_eth_dev(dev); ++ uint16_t i; + int ret; + + sfcr_info(sr, "entry"); +@@ -300,6 +303,11 @@ sfc_repr_dev_start(struct rte_eth_dev *dev) + if (ret != 0) + goto fail_start; + ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ + sfcr_info(sr, "done"); + + return 0; +@@ -364,6 +372,7 @@ static int + sfc_repr_dev_stop(struct rte_eth_dev *dev) + { + struct sfc_repr *sr = sfc_repr_by_eth_dev(dev); ++ uint16_t i; + int ret; + + sfcr_info(sr, "entry"); +@@ -378,6 +387,11 @@ sfc_repr_dev_stop(struct rte_eth_dev *dev) + + sfc_repr_unlock(sr); + ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ + sfcr_info(sr, "done"); + + return 0; +@@ -528,6 +542,7 @@ sfc_repr_dev_infos_get(struct rte_eth_dev *dev, + + dev_info->device = dev->device; + ++ dev_info->max_rx_pktlen = EFX_MAC_PDU_MAX; + dev_info->max_rx_queues = SFC_REPR_RXQ_MAX; + dev_info->max_tx_queues = SFC_REPR_TXQ_MAX; + dev_info->default_rxconf.rx_drop_en = 1; +@@ -834,6 +849,8 @@ sfc_repr_dev_close(struct rte_eth_dev *dev) + + (void)sfc_repr_proxy_del_port(srs->pf_port_id, srs->repr_id); + ++ sfc_mae_clear_switch_port(srs->switch_domain_id, srs->switch_port_id); ++ + dev->rx_pkt_burst = NULL; + dev->tx_pkt_burst = NULL; + dev->dev_ops = NULL; +@@ -888,6 +905,29 @@ sfc_repr_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) + return 0; + } + ++static int ++sfc_repr_flow_pick_transfer_proxy(struct rte_eth_dev *dev, ++ uint16_t *transfer_proxy_port, ++ struct rte_flow_error *error) ++{ ++ struct sfc_repr_shared *srs = sfc_repr_shared_by_eth_dev(dev); ++ ++ return rte_flow_pick_transfer_proxy(srs->pf_port_id, ++ transfer_proxy_port, error); ++} ++ ++const struct rte_flow_ops sfc_repr_flow_ops = { ++ .pick_transfer_proxy = sfc_repr_flow_pick_transfer_proxy, ++}; ++ ++static int ++sfc_repr_dev_flow_ops_get(struct rte_eth_dev *dev __rte_unused, ++ const struct rte_flow_ops **ops) ++{ ++ *ops = &sfc_repr_flow_ops; ++ return 0; ++} ++ + static const struct eth_dev_ops sfc_repr_dev_ops = { + .dev_configure = sfc_repr_dev_configure, + .dev_start = sfc_repr_dev_start, +@@ -901,6 +941,7 @@ static const struct eth_dev_ops sfc_repr_dev_ops = { + .rx_queue_release = sfc_repr_rx_queue_release, + .tx_queue_setup = sfc_repr_tx_queue_setup, + .tx_queue_release = sfc_repr_tx_queue_release, ++ .flow_ops_get = sfc_repr_dev_flow_ops_get, + }; + + +diff --git a/dpdk/drivers/net/sfc/sfc_rx.c b/dpdk/drivers/net/sfc/sfc_rx.c +index 5ea98187c3..edd0f0c038 100644 +--- a/dpdk/drivers/net/sfc/sfc_rx.c ++++ b/dpdk/drivers/net/sfc/sfc_rx.c +@@ -1225,6 +1225,9 @@ sfc_rx_qinit(struct sfc_adapter *sa, sfc_sw_index_t sw_index, + else + rxq_info->rxq_flags = 0; + ++ if (rxq_info->type_flags & EFX_RXQ_FLAG_INGRESS_MPORT) ++ rxq_info->rxq_flags |= SFC_RXQ_FLAG_INGRESS_MPORT; ++ + rxq->buf_size = buf_size; + + rc = sfc_dma_alloc(sa, "rxq", sw_index, EFX_NIC_DMA_ADDR_RX_RING, +diff --git a/dpdk/drivers/net/sfc/sfc_switch.c b/dpdk/drivers/net/sfc/sfc_switch.c +index 5c10e8fc74..8f1ee97fa8 100644 +--- a/dpdk/drivers/net/sfc/sfc_switch.c ++++ b/dpdk/drivers/net/sfc/sfc_switch.c +@@ -489,6 +489,7 @@ sfc_mae_clear_switch_port(uint16_t switch_domain_id, + uint16_t switch_port_id) + { + struct sfc_mae_switch_domain *domain; ++ struct sfc_mae_switch_port *port; + + rte_spinlock_lock(&sfc_mae_switch.lock); + +@@ -504,6 +505,17 @@ sfc_mae_clear_switch_port(uint16_t switch_domain_id, + domain->mae_admin_port = NULL; + } + ++ TAILQ_FOREACH(port, &domain->ports, switch_domain_ports) { ++ if (port->id == switch_port_id) { ++ /* ++ * Invalidate the field to prevent wrong ++ * look-ups from flow rule handling path. ++ */ ++ port->ethdev_port_id = RTE_MAX_ETHPORTS; ++ break; ++ } ++ } ++ + rte_spinlock_unlock(&sfc_mae_switch.lock); + return 0; + } +diff --git a/dpdk/drivers/net/softnic/rte_eth_softnic.c b/dpdk/drivers/net/softnic/rte_eth_softnic.c +index bcf6664460..1b90cf7a21 100644 +--- a/dpdk/drivers/net/softnic/rte_eth_softnic.c ++++ b/dpdk/drivers/net/softnic/rte_eth_softnic.c +@@ -134,6 +134,7 @@ pmd_dev_start(struct rte_eth_dev *dev) + { + struct pmd_internals *p = dev->data->dev_private; + int status; ++ uint16_t i; + + /* Firmware */ + status = softnic_cli_script_process(p, +@@ -146,6 +147,11 @@ pmd_dev_start(struct rte_eth_dev *dev) + /* Link UP */ + dev->data->dev_link.link_status = RTE_ETH_LINK_UP; + ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ + return 0; + } + +@@ -153,6 +159,7 @@ static int + pmd_dev_stop(struct rte_eth_dev *dev) + { + struct pmd_internals *p = dev->data->dev_private; ++ uint16_t i; + + /* Link DOWN */ + dev->data->dev_link.link_status = RTE_ETH_LINK_DOWN; +@@ -163,6 +170,11 @@ pmd_dev_stop(struct rte_eth_dev *dev) + softnic_softnic_swq_free_keep_rxq_txq(p); + softnic_mempool_free(p); + ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ + return 0; + } + +diff --git a/dpdk/drivers/net/tap/rte_eth_tap.c b/dpdk/drivers/net/tap/rte_eth_tap.c +index f2a6c33a19..2fd46c6b0b 100644 +--- a/dpdk/drivers/net/tap/rte_eth_tap.c ++++ b/dpdk/drivers/net/tap/rte_eth_tap.c +@@ -559,7 +559,7 @@ tap_tx_l3_cksum(char *packet, uint64_t ol_flags, unsigned int l2_len, + { + void *l3_hdr = packet + l2_len; + +- if (ol_flags & (RTE_MBUF_F_TX_IP_CKSUM | RTE_MBUF_F_TX_IPV4)) { ++ if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM) { + struct rte_ipv4_hdr *iph = l3_hdr; + uint16_t cksum; + +@@ -642,16 +642,25 @@ tap_write_mbufs(struct tx_queue *txq, uint16_t num_mbufs, + + nb_segs = mbuf->nb_segs; + if (txq->csum && +- ((mbuf->ol_flags & (RTE_MBUF_F_TX_IP_CKSUM | RTE_MBUF_F_TX_IPV4) || ++ ((mbuf->ol_flags & RTE_MBUF_F_TX_IP_CKSUM || + (mbuf->ol_flags & RTE_MBUF_F_TX_L4_MASK) == RTE_MBUF_F_TX_UDP_CKSUM || + (mbuf->ol_flags & RTE_MBUF_F_TX_L4_MASK) == RTE_MBUF_F_TX_TCP_CKSUM))) { ++ unsigned int l4_len = 0; ++ + is_cksum = 1; + ++ if ((mbuf->ol_flags & RTE_MBUF_F_TX_L4_MASK) == ++ RTE_MBUF_F_TX_UDP_CKSUM) ++ l4_len = sizeof(struct rte_udp_hdr); ++ else if ((mbuf->ol_flags & RTE_MBUF_F_TX_L4_MASK) == ++ RTE_MBUF_F_TX_TCP_CKSUM) ++ l4_len = sizeof(struct rte_tcp_hdr); ++ + /* Support only packets with at least layer 4 + * header included in the first segment + */ + seg_len = rte_pktmbuf_data_len(mbuf); +- l234_hlen = mbuf->l2_len + mbuf->l3_len + mbuf->l4_len; ++ l234_hlen = mbuf->l2_len + mbuf->l3_len + l4_len; + if (seg_len < l234_hlen) + return -1; + +@@ -661,7 +670,7 @@ tap_write_mbufs(struct tx_queue *txq, uint16_t num_mbufs, + rte_memcpy(m_copy, rte_pktmbuf_mtod(mbuf, void *), + l234_hlen); + tap_tx_l3_cksum(m_copy, mbuf->ol_flags, +- mbuf->l2_len, mbuf->l3_len, mbuf->l4_len, ++ mbuf->l2_len, mbuf->l3_len, l4_len, + &l4_cksum, &l4_phdr_cksum, + &l4_raw_cksum); + iovecs[k].iov_base = m_copy; +@@ -2267,29 +2276,6 @@ set_remote_iface(const char *key __rte_unused, + return 0; + } + +-static int parse_user_mac(struct rte_ether_addr *user_mac, +- const char *value) +-{ +- unsigned int index = 0; +- char mac_temp[strlen(ETH_TAP_USR_MAC_FMT) + 1], *mac_byte = NULL; +- +- if (user_mac == NULL || value == NULL) +- return 0; +- +- strlcpy(mac_temp, value, sizeof(mac_temp)); +- mac_byte = strtok(mac_temp, ":"); +- +- while ((mac_byte != NULL) && +- (strlen(mac_byte) <= 2) && +- (strlen(mac_byte) == strspn(mac_byte, +- ETH_TAP_CMP_MAC_FMT))) { +- user_mac->addr_bytes[index++] = strtoul(mac_byte, NULL, 16); +- mac_byte = strtok(NULL, ":"); +- } +- +- return index; +-} +- + static int + set_mac_type(const char *key __rte_unused, + const char *value, +@@ -2303,15 +2289,15 @@ set_mac_type(const char *key __rte_unused, + if (!strncasecmp(ETH_TAP_MAC_FIXED, value, strlen(ETH_TAP_MAC_FIXED))) { + static int iface_idx; + +- /* fixed mac = 00:64:74:61:70: */ +- memcpy((char *)user_mac->addr_bytes, "\0dtap", ++ /* fixed mac = 02:64:74:61:70: */ ++ memcpy((char *)user_mac->addr_bytes, "\002dtap", + RTE_ETHER_ADDR_LEN); + user_mac->addr_bytes[RTE_ETHER_ADDR_LEN - 1] = + iface_idx++ + '0'; + goto success; + } + +- if (parse_user_mac(user_mac, value) != 6) ++ if (rte_ether_unformat_addr(value, user_mac) < 0) + goto error; + success: + TAP_LOG(DEBUG, "TAP user MAC param (%s)", value); +diff --git a/dpdk/drivers/net/tap/tap_bpf_insns.h b/dpdk/drivers/net/tap/tap_bpf_insns.h +index 1a91bbad13..53fa76c4e6 100644 +--- a/dpdk/drivers/net/tap/tap_bpf_insns.h ++++ b/dpdk/drivers/net/tap/tap_bpf_insns.h +@@ -1,10 +1,10 @@ + /* SPDX-License-Identifier: BSD-3-Clause +- * Copyright 2017 Mellanox Technologies, Ltd ++ * Auto-generated from tap_bpf_program.c ++ * This not the original source file. Do NOT edit it. + */ + + #include + +-/* bpf_insn array matching cls_q section. See tap_bpf_program.c file */ + static struct bpf_insn cls_q_insns[] = { + {0x61, 2, 1, 52, 0x00000000}, + {0x18, 3, 0, 0, 0xdeadbeef}, +@@ -23,18 +23,17 @@ static struct bpf_insn cls_q_insns[] = { + {0x95, 0, 0, 0, 0x00000000}, + }; + +-/* bpf_insn array matching l3_l4 section. see tap_bpf_program.c file */ + static struct bpf_insn l3_l4_hash_insns[] = { + {0xbf, 7, 1, 0, 0x00000000}, +- {0x61, 8, 7, 16, 0x00000000}, +- {0x61, 6, 7, 76, 0x00000000}, ++ {0x61, 6, 7, 16, 0x00000000}, ++ {0x61, 8, 7, 76, 0x00000000}, + {0x61, 9, 7, 80, 0x00000000}, + {0x18, 1, 0, 0, 0xdeadbeef}, + {0x00, 0, 0, 0, 0x00000000}, + {0x63, 10, 1, -4, 0x00000000}, + {0xbf, 2, 10, 0, 0x00000000}, + {0x07, 2, 0, 0, 0xfffffffc}, +- {0x18, 1, 1, 0, 0x0000cafe}, ++ {0x18, 1, 0, 0, 0x00000000}, + {0x00, 0, 0, 0, 0x00000000}, + {0x85, 0, 0, 0, 0x00000001}, + {0x55, 0, 0, 21, 0x00000000}, +@@ -58,7 +57,7 @@ static struct bpf_insn l3_l4_hash_insns[] = { + {0x07, 1, 0, 0, 0xffffffd0}, + {0xb7, 2, 0, 0, 0x00000023}, + {0x85, 0, 0, 0, 0x00000006}, +- {0x05, 0, 0, 1632, 0x00000000}, ++ {0x05, 0, 0, 1680, 0x00000000}, + {0xb7, 1, 0, 0, 0x0000000e}, + {0x61, 2, 7, 20, 0x00000000}, + {0x15, 2, 0, 10, 0x00000000}, +@@ -66,1630 +65,1678 @@ static struct bpf_insn l3_l4_hash_insns[] = { + {0x55, 2, 0, 8, 0x0000a888}, + {0xbf, 2, 7, 0, 0x00000000}, + {0xb7, 7, 0, 0, 0x00000000}, +- {0xbf, 1, 6, 0, 0x00000000}, ++ {0xbf, 1, 8, 0, 0x00000000}, + {0x07, 1, 0, 0, 0x00000012}, +- {0x2d, 1, 9, 1622, 0x00000000}, ++ {0x2d, 1, 9, 1670, 0x00000000}, + {0xb7, 1, 0, 0, 0x00000012}, +- {0x69, 8, 6, 16, 0x00000000}, ++ {0x69, 6, 8, 16, 0x00000000}, + {0xbf, 7, 2, 0, 0x00000000}, ++ {0x57, 6, 0, 0, 0x0000ffff}, + {0x7b, 10, 7, -56, 0x00000000}, +- {0x57, 8, 0, 0, 0x0000ffff}, +- {0x15, 8, 0, 409, 0x0000dd86}, ++ {0x15, 6, 0, 443, 0x0000dd86}, + {0xb7, 7, 0, 0, 0x00000003}, +- {0x55, 8, 0, 1614, 0x00000008}, +- {0x0f, 6, 1, 0, 0x00000000}, ++ {0x55, 6, 0, 1662, 0x00000008}, ++ {0x0f, 8, 1, 0, 0x00000000}, + {0xb7, 7, 0, 0, 0x00000000}, +- {0xbf, 1, 6, 0, 0x00000000}, ++ {0xbf, 1, 8, 0, 0x00000000}, + {0x07, 1, 0, 0, 0x00000018}, +- {0x2d, 1, 9, 1609, 0x00000000}, +- {0x71, 3, 6, 12, 0x00000000}, +- {0xbf, 1, 3, 0, 0x00000000}, +- {0x67, 1, 0, 0, 0x00000038}, +- {0xc7, 1, 0, 0, 0x00000020}, +- {0x77, 1, 0, 0, 0x0000001f}, +- {0x57, 1, 0, 0, 0x2cc681d1}, +- {0x67, 3, 0, 0, 0x00000018}, ++ {0x2d, 1, 9, 1657, 0x00000000}, ++ {0xb7, 1, 0, 0, 0x00000000}, ++ {0x71, 3, 8, 12, 0x00000000}, ++ {0x71, 2, 8, 9, 0x00000000}, ++ {0x15, 2, 0, 1, 0x00000011}, ++ {0x55, 2, 0, 21, 0x00000006}, ++ {0x71, 2, 8, 7, 0x00000000}, ++ {0x71, 4, 8, 6, 0x00000000}, ++ {0xbf, 5, 4, 0, 0x00000000}, ++ {0x67, 5, 0, 0, 0x00000008}, ++ {0x57, 5, 0, 0, 0x00001f00}, ++ {0x4f, 5, 2, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00000020}, ++ {0x4f, 4, 5, 0, 0x00000000}, ++ {0x55, 4, 0, 12, 0x00000000}, ++ {0xbf, 2, 8, 0, 0x00000000}, ++ {0x07, 2, 0, 0, 0x00000014}, ++ {0x71, 4, 2, 0, 0x00000000}, ++ {0x67, 4, 0, 0, 0x00000018}, ++ {0x71, 1, 2, 1, 0x00000000}, ++ {0x67, 1, 0, 0, 0x00000010}, ++ {0x4f, 1, 4, 0, 0x00000000}, ++ {0x71, 4, 2, 3, 0x00000000}, ++ {0x4f, 1, 4, 0, 0x00000000}, ++ {0x71, 2, 2, 2, 0x00000000}, ++ {0x67, 2, 0, 0, 0x00000008}, ++ {0x4f, 1, 2, 0, 0x00000000}, + {0xbf, 4, 3, 0, 0x00000000}, +- {0x57, 4, 0, 0, 0x40000000}, ++ {0x67, 4, 0, 0, 0x00000038}, ++ {0xc7, 4, 0, 0, 0x00000038}, ++ {0xb7, 2, 0, 0, 0x00000000}, ++ {0x65, 4, 0, 1, 0xffffffff}, ++ {0xb7, 7, 0, 0, 0x2cc681d1}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00000040}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x598d03a2}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00000020}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xb31a0745}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00000010}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x66340e8a}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00000008}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xcc681d15}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00000004}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x98d03a2b}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00000002}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x31a07456}, ++ {0x71, 4, 8, 13, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000001}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x6340e8ad}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x67, 3, 0, 0, 0x00000038}, ++ {0xc7, 3, 0, 0, 0x00000038}, ++ {0xbf, 5, 7, 0, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xc681d15b}, ++ {0x6d, 2, 3, 1, 0x00000000}, ++ {0xbf, 5, 7, 0, 0x00000000}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000040}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x8d03a2b7}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000020}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x1a07456f}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000010}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x340e8ade}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000008}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x681d15bd}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000004}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xd03a2b7b}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000002}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xa07456f6}, ++ {0x71, 3, 8, 14, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00000001}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x40e8aded}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x67, 4, 0, 0, 0x00000038}, ++ {0xc7, 4, 0, 0, 0x00000038}, ++ {0xbf, 7, 5, 0, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x81d15bdb}, ++ {0x6d, 2, 4, 1, 0x00000000}, ++ {0xbf, 7, 5, 0, 0x00000000}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00000040}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x03a2b7b7}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00000020}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x07456f6f}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00000010}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x0e8adedf}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00000008}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x1d15bdbf}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00000004}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x3a2b7b7e}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00000002}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x7456f6fd}, ++ {0x71, 4, 8, 15, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000001}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xe8adedfa}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x67, 3, 0, 0, 0x00000038}, ++ {0xc7, 3, 0, 0, 0x00000038}, ++ {0xbf, 5, 7, 0, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xd15bdbf4}, ++ {0x6d, 2, 3, 1, 0x00000000}, ++ {0xbf, 5, 7, 0, 0x00000000}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000040}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xa2b7b7e9}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000020}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x456f6fd3}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000010}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x8adedfa7}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000008}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x15bdbf4f}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000004}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x2b7b7e9e}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000002}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x56f6fd3d}, ++ {0x71, 3, 8, 16, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00000001}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xadedfa7b}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x67, 4, 0, 0, 0x00000038}, ++ {0xc7, 4, 0, 0, 0x00000038}, ++ {0xbf, 7, 5, 0, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x5bdbf4f7}, ++ {0x6d, 2, 4, 1, 0x00000000}, ++ {0xbf, 7, 5, 0, 0x00000000}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00000040}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xb7b7e9ef}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00000020}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x6f6fd3df}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00000010}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xdedfa7bf}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00000008}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xbdbf4f7f}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00000004}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x7b7e9eff}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00000002}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xf6fd3dff}, ++ {0x71, 4, 8, 17, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000001}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xedfa7bfe}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x67, 3, 0, 0, 0x00000038}, ++ {0xc7, 3, 0, 0, 0x00000038}, ++ {0xbf, 5, 7, 0, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xdbf4f7fc}, ++ {0x6d, 2, 3, 1, 0x00000000}, ++ {0xbf, 5, 7, 0, 0x00000000}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000040}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xb7e9eff9}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000020}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x6fd3dff2}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000010}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xdfa7bfe5}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000008}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xbf4f7fca}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000004}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x7e9eff94}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000002}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xfd3dff28}, ++ {0x71, 3, 8, 18, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00000001}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xfa7bfe51}, ++ {0xbf, 6, 3, 0, 0x00000000}, ++ {0x67, 6, 0, 0, 0x00000038}, ++ {0xc7, 6, 0, 0, 0x00000038}, ++ {0xbf, 4, 5, 0, 0x00000000}, ++ {0xa7, 4, 0, 0, 0xf4f7fca2}, ++ {0x6d, 2, 6, 1, 0x00000000}, ++ {0xbf, 4, 5, 0, 0x00000000}, ++ {0xbf, 5, 3, 0, 0x00000000}, ++ {0x57, 5, 0, 0, 0x00000040}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0xe9eff945}, ++ {0xbf, 5, 3, 0, 0x00000000}, ++ {0x57, 5, 0, 0, 0x00000020}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0xd3dff28a}, ++ {0xbf, 5, 3, 0, 0x00000000}, ++ {0x57, 5, 0, 0, 0x00000010}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0xa7bfe514}, ++ {0xbf, 5, 3, 0, 0x00000000}, ++ {0x57, 5, 0, 0, 0x00000008}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x4f7fca28}, ++ {0xbf, 5, 3, 0, 0x00000000}, ++ {0x57, 5, 0, 0, 0x00000004}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x9eff9450}, ++ {0xbf, 5, 3, 0, 0x00000000}, ++ {0x57, 5, 0, 0, 0x00000002}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x3dff28a0}, ++ {0x71, 5, 8, 19, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000001}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x7bfe5141}, ++ {0xbf, 3, 5, 0, 0x00000000}, ++ {0x67, 3, 0, 0, 0x00000038}, ++ {0xc7, 3, 0, 0, 0x00000038}, ++ {0xbf, 7, 4, 0, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xf7fca283}, ++ {0x6d, 2, 3, 1, 0x00000000}, ++ {0xbf, 7, 4, 0, 0x00000000}, ++ {0xbf, 3, 5, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000040}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xeff94506}, ++ {0xbf, 3, 5, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000020}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xdff28a0c}, ++ {0xbf, 3, 5, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000010}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xbfe51418}, ++ {0xbf, 3, 5, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000008}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x7fca2831}, ++ {0xbf, 3, 5, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000004}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xff945063}, ++ {0xbf, 3, 5, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000002}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xff28a0c6}, ++ {0x57, 5, 0, 0, 0x00000001}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xfe51418c}, ++ {0xbf, 4, 1, 0, 0x00000000}, ++ {0x67, 4, 0, 0, 0x00000020}, ++ {0xc7, 4, 0, 0, 0x00000020}, ++ {0xbf, 3, 7, 0, 0x00000000}, ++ {0xa7, 3, 0, 0, 0xfca28319}, ++ {0x6d, 2, 4, 1, 0x00000000}, ++ {0xbf, 3, 7, 0, 0x00000000}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x40000000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0xf9450633}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x20000000}, ++ {0x79, 6, 10, -56, 0x00000000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0xf28a0c67}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x10000000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0xe51418ce}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x08000000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0xca28319d}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x04000000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x9450633b}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x02000000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x28a0c676}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x01000000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x51418ced}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00800000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0xa28319db}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00400000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x450633b6}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00200000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x8a0c676c}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00100000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x1418ced8}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00080000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x28319db1}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00040000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x50633b63}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00020000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0xa0c676c6}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00010000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x418ced8d}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00008000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x8319db1a}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00004000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x0633b634}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00002000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x0c676c68}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00001000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x18ced8d1}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00000800}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x319db1a3}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00000400}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x633b6347}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00000200}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0xc676c68f}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00000100}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x8ced8d1f}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00000080}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x19db1a3e}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00000040}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x33b6347d}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00000020}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x676c68fa}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00000010}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0xced8d1f4}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00000008}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x9db1a3e9}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00000004}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x3b6347d2}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00000002}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x76c68fa5}, ++ {0x57, 1, 0, 0, 0x00000001}, ++ {0x15, 1, 0, 1194, 0x00000000}, ++ {0xa7, 3, 0, 0, 0xed8d1f4a}, ++ {0x05, 0, 0, 1192, 0x00000000}, ++ {0x0f, 8, 1, 0, 0x00000000}, ++ {0xb7, 7, 0, 0, 0x00000000}, ++ {0xbf, 1, 8, 0, 0x00000000}, ++ {0x07, 1, 0, 0, 0x0000002c}, ++ {0x2d, 1, 9, 1216, 0x00000000}, ++ {0x61, 2, 8, 8, 0x00000000}, ++ {0xdc, 2, 0, 0, 0x00000040}, ++ {0xc7, 2, 0, 0, 0x00000020}, ++ {0x71, 3, 8, 6, 0x00000000}, ++ {0x15, 3, 0, 2, 0x00000011}, ++ {0xb7, 1, 0, 0, 0x00000000}, ++ {0x55, 3, 0, 12, 0x00000006}, ++ {0xbf, 3, 8, 0, 0x00000000}, ++ {0x07, 3, 0, 0, 0x00000028}, ++ {0x71, 4, 3, 0, 0x00000000}, ++ {0x67, 4, 0, 0, 0x00000018}, ++ {0x71, 1, 3, 1, 0x00000000}, ++ {0x67, 1, 0, 0, 0x00000010}, ++ {0x4f, 1, 4, 0, 0x00000000}, ++ {0x71, 4, 3, 3, 0x00000000}, ++ {0x4f, 1, 4, 0, 0x00000000}, ++ {0x71, 3, 3, 2, 0x00000000}, ++ {0x67, 3, 0, 0, 0x00000008}, ++ {0x4f, 1, 3, 0, 0x00000000}, ++ {0xbf, 4, 2, 0, 0x00000000}, ++ {0x77, 4, 0, 0, 0x0000001f}, ++ {0x57, 4, 0, 0, 0x2cc681d1}, ++ {0xbf, 3, 2, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x40000000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x598d03a2}, ++ {0xbf, 3, 2, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x20000000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0xb31a0745}, ++ {0xbf, 3, 2, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x10000000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x66340e8a}, ++ {0xbf, 3, 2, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x08000000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0xcc681d15}, ++ {0xbf, 3, 2, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x04000000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x98d03a2b}, ++ {0xbf, 3, 2, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x02000000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x31a07456}, ++ {0xbf, 3, 2, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x01000000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x6340e8ad}, ++ {0xbf, 3, 2, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00800000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0xc681d15b}, ++ {0xbf, 3, 2, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00400000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x8d03a2b7}, ++ {0xbf, 3, 2, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00200000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x1a07456f}, ++ {0xbf, 3, 2, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00100000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x340e8ade}, ++ {0xbf, 3, 2, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00080000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x681d15bd}, ++ {0xbf, 3, 2, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00040000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0xd03a2b7b}, ++ {0xbf, 3, 2, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00020000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0xa07456f6}, ++ {0xbf, 3, 2, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00010000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x40e8aded}, ++ {0xbf, 3, 2, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00008000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x81d15bdb}, ++ {0xbf, 3, 2, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00004000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x03a2b7b7}, ++ {0xbf, 3, 2, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00002000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x07456f6f}, ++ {0xbf, 3, 2, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00001000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x0e8adedf}, ++ {0xbf, 3, 2, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000800}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x1d15bdbf}, ++ {0xbf, 3, 2, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000400}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x3a2b7b7e}, ++ {0xbf, 3, 2, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000200}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x7456f6fd}, ++ {0xbf, 3, 2, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000100}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0xe8adedfa}, ++ {0xbf, 3, 2, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000080}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0xd15bdbf4}, ++ {0xbf, 3, 2, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000040}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0xa2b7b7e9}, ++ {0xbf, 3, 2, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000020}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x456f6fd3}, ++ {0xbf, 3, 2, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000010}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x8adedfa7}, ++ {0xbf, 3, 2, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000008}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x15bdbf4f}, ++ {0x61, 3, 8, 12, 0x00000000}, ++ {0xbf, 5, 2, 0, 0x00000000}, ++ {0x57, 5, 0, 0, 0x00000004}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x2b7b7e9e}, ++ {0xdc, 3, 0, 0, 0x00000040}, ++ {0xbf, 5, 2, 0, 0x00000000}, ++ {0x57, 5, 0, 0, 0x00000002}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x56f6fd3d}, ++ {0xc7, 3, 0, 0, 0x00000020}, ++ {0x57, 2, 0, 0, 0x00000001}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0xadedfa7b}, + {0xb7, 2, 0, 0, 0x00000000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x598d03a2}, ++ {0xbf, 5, 4, 0, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x5bdbf4f7}, ++ {0x6d, 2, 3, 1, 0x00000000}, ++ {0xbf, 5, 4, 0, 0x00000000}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x40000000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xb7b7e9ef}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x20000000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xb31a0745}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x6f6fd3df}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x10000000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x66340e8a}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xdedfa7bf}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x08000000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xcc681d15}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xbdbf4f7f}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x04000000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x98d03a2b}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x7b7e9eff}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x02000000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x31a07456}, +- {0x57, 3, 0, 0, 0x01000000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x6340e8ad}, +- {0x71, 3, 6, 13, 0x00000000}, +- {0x67, 3, 0, 0, 0x00000010}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xf6fd3dff}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x01000000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xedfa7bfe}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00800000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xc681d15b}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xdbf4f7fc}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00400000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x8d03a2b7}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xb7e9eff9}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00200000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x1a07456f}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x6fd3dff2}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00100000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x340e8ade}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xdfa7bfe5}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00080000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x681d15bd}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xbf4f7fca}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00040000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xd03a2b7b}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x7e9eff94}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00020000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xa07456f6}, +- {0x57, 3, 0, 0, 0x00010000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x40e8aded}, +- {0x71, 3, 6, 14, 0x00000000}, +- {0x67, 3, 0, 0, 0x00000008}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xfd3dff28}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00010000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xfa7bfe51}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00008000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x81d15bdb}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xf4f7fca2}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00004000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x03a2b7b7}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xe9eff945}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00002000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x07456f6f}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xd3dff28a}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00001000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x0e8adedf}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xa7bfe514}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000800}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x1d15bdbf}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x4f7fca28}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000400}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x3a2b7b7e}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x9eff9450}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000200}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x7456f6fd}, +- {0x57, 3, 0, 0, 0x00000100}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xe8adedfa}, +- {0x71, 3, 6, 15, 0x00000000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x3dff28a0}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00000100}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x7bfe5141}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000080}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xd15bdbf4}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xf7fca283}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000040}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xa2b7b7e9}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xeff94506}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000020}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x456f6fd3}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xdff28a0c}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000010}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x8adedfa7}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xbfe51418}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000008}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x15bdbf4f}, +- {0xbf, 4, 3, 0, 0x00000000}, +- {0x57, 4, 0, 0, 0x00000004}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x2b7b7e9e}, +- {0xbf, 4, 3, 0, 0x00000000}, +- {0x57, 4, 0, 0, 0x00000002}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x56f6fd3d}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x7fca2831}, ++ {0x61, 4, 8, 16, 0x00000000}, ++ {0xbf, 6, 3, 0, 0x00000000}, ++ {0x57, 6, 0, 0, 0x00000004}, ++ {0x15, 6, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xff945063}, ++ {0xdc, 4, 0, 0, 0x00000040}, ++ {0xbf, 6, 3, 0, 0x00000000}, ++ {0x57, 6, 0, 0, 0x00000002}, ++ {0x15, 6, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xff28a0c6}, ++ {0xc7, 4, 0, 0, 0x00000020}, + {0x57, 3, 0, 0, 0x00000001}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xadedfa7b}, +- {0x71, 4, 6, 16, 0x00000000}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x67, 5, 0, 0, 0x00000038}, +- {0xc7, 5, 0, 0, 0x00000020}, +- {0xb7, 3, 0, 0, 0xffffffff}, +- {0x6d, 5, 3, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x5bdbf4f7}, +- {0x67, 4, 0, 0, 0x00000018}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x40000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xb7b7e9ef}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x20000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x6f6fd3df}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x10000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xdedfa7bf}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x08000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xbdbf4f7f}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x04000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x7b7e9eff}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x02000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xf6fd3dff}, +- {0x57, 4, 0, 0, 0x01000000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xedfa7bfe}, +- {0x71, 4, 6, 17, 0x00000000}, +- {0x67, 4, 0, 0, 0x00000010}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00800000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xdbf4f7fc}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00400000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xb7e9eff9}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00200000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x6fd3dff2}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00100000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xdfa7bfe5}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00080000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xbf4f7fca}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00040000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x7e9eff94}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00020000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xfd3dff28}, +- {0x57, 4, 0, 0, 0x00010000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xfa7bfe51}, +- {0x71, 4, 6, 18, 0x00000000}, +- {0x67, 4, 0, 0, 0x00000008}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00008000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xf4f7fca2}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00004000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xe9eff945}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00002000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xd3dff28a}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00001000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xa7bfe514}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000800}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x4f7fca28}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000400}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x9eff9450}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000200}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x3dff28a0}, +- {0x57, 4, 0, 0, 0x00000100}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x7bfe5141}, +- {0x71, 4, 6, 19, 0x00000000}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000080}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xf7fca283}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000040}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xeff94506}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000020}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xdff28a0c}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000010}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xbfe51418}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000008}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x7fca2831}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000004}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xff945063}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000002}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xff28a0c6}, +- {0x57, 4, 0, 0, 0x00000001}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xfe51418c}, +- {0x71, 4, 6, 20, 0x00000000}, +- {0x67, 4, 0, 0, 0x00000008}, +- {0x71, 5, 6, 21, 0x00000000}, +- {0x4f, 4, 5, 0, 0x00000000}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x67, 5, 0, 0, 0x00000030}, +- {0xc7, 5, 0, 0, 0x00000020}, +- {0x6d, 5, 3, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xfca28319}, +- {0x67, 4, 0, 0, 0x00000010}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xfe51418c}, ++ {0xbf, 7, 5, 0, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xfca28319}, ++ {0x6d, 2, 4, 1, 0x00000000}, ++ {0xbf, 7, 5, 0, 0x00000000}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x40000000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xf9450633}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xf9450633}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x20000000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xf28a0c67}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xf28a0c67}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x10000000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xe51418ce}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xe51418ce}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x08000000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xca28319d}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xca28319d}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x04000000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x9450633b}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x9450633b}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x02000000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x28a0c676}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x28a0c676}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x01000000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x51418ced}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x51418ced}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00800000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xa28319db}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xa28319db}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00400000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x450633b6}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x450633b6}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00200000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x8a0c676c}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x8a0c676c}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00100000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x1418ced8}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x1418ced8}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00080000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x28319db1}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x28319db1}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00040000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x50633b63}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x50633b63}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00020000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xa0c676c6}, +- {0x57, 4, 0, 0, 0x00010000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x418ced8d}, +- {0x71, 3, 6, 22, 0x00000000}, +- {0x67, 3, 0, 0, 0x00000008}, +- {0xbf, 4, 3, 0, 0x00000000}, +- {0x57, 4, 0, 0, 0x00008000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x8319db1a}, +- {0xbf, 4, 3, 0, 0x00000000}, +- {0x57, 4, 0, 0, 0x00004000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x0633b634}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xa0c676c6}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00010000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x418ced8d}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00008000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x8319db1a}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00004000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x0633b634}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00002000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x0c676c68}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00001000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x18ced8d1}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000800}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x319db1a3}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000400}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x633b6347}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000200}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xc676c68f}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000100}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x8ced8d1f}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000080}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x19db1a3e}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000040}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x33b6347d}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000020}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x676c68fa}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000010}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xced8d1f4}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000008}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x9db1a3e9}, ++ {0x61, 3, 8, 20, 0x00000000}, ++ {0xbf, 5, 4, 0, 0x00000000}, ++ {0x57, 5, 0, 0, 0x00000004}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x3b6347d2}, ++ {0xdc, 3, 0, 0, 0x00000040}, ++ {0xbf, 5, 4, 0, 0x00000000}, ++ {0x57, 5, 0, 0, 0x00000002}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x76c68fa5}, ++ {0xc7, 3, 0, 0, 0x00000020}, ++ {0x57, 4, 0, 0, 0x00000001}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xed8d1f4a}, ++ {0xbf, 5, 7, 0, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xdb1a3e94}, ++ {0x6d, 2, 3, 1, 0x00000000}, ++ {0xbf, 5, 7, 0, 0x00000000}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x40000000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xb6347d28}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x20000000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x6c68fa51}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x10000000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xd8d1f4a3}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x08000000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xb1a3e946}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x04000000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x6347d28d}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x02000000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xc68fa51a}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x01000000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x8d1f4a35}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00800000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x1a3e946b}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00400000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x347d28d7}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00200000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x68fa51ae}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00100000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xd1f4a35c}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00080000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xa3e946b9}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00040000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x47d28d73}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00020000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x8fa51ae7}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00010000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x1f4a35cf}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00008000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x3e946b9e}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00004000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x7d28d73c}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00002000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x0c676c68}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xfa51ae78}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00001000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x18ced8d1}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xf4a35cf1}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000800}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x319db1a3}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xe946b9e3}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000400}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x633b6347}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xd28d73c7}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000200}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xc676c68f}, +- {0x57, 3, 0, 0, 0x00000100}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x8ced8d1f}, +- {0x71, 3, 6, 23, 0x00000000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xa51ae78e}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00000100}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x4a35cf1c}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000080}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x19db1a3e}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x946b9e38}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000040}, +- {0x79, 5, 10, -56, 0x00000000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x33b6347d}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x28d73c71}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000020}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x676c68fa}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x51ae78e3}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000010}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xced8d1f4}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xa35cf1c6}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000008}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x9db1a3e9}, +- {0xbf, 4, 3, 0, 0x00000000}, +- {0x57, 4, 0, 0, 0x00000004}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x3b6347d2}, +- {0xbf, 4, 3, 0, 0x00000000}, +- {0x57, 4, 0, 0, 0x00000002}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x76c68fa5}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x46b9e38d}, ++ {0x61, 4, 8, 24, 0x00000000}, ++ {0xbf, 6, 3, 0, 0x00000000}, ++ {0x57, 6, 0, 0, 0x00000004}, ++ {0x15, 6, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x8d73c71b}, ++ {0xdc, 4, 0, 0, 0x00000040}, ++ {0xbf, 6, 3, 0, 0x00000000}, ++ {0x57, 6, 0, 0, 0x00000002}, ++ {0x15, 6, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x1ae78e36}, ++ {0xc7, 4, 0, 0, 0x00000020}, + {0x57, 3, 0, 0, 0x00000001}, +- {0x1d, 3, 2, 1177, 0x00000000}, +- {0xa7, 1, 0, 0, 0xed8d1f4a}, +- {0x05, 0, 0, 1175, 0x00000000}, +- {0x0f, 6, 1, 0, 0x00000000}, +- {0xb7, 7, 0, 0, 0x00000000}, +- {0xbf, 1, 6, 0, 0x00000000}, +- {0x07, 1, 0, 0, 0x0000002c}, +- {0x2d, 1, 9, 1202, 0x00000000}, +- {0x61, 4, 6, 8, 0x00000000}, +- {0xbf, 1, 4, 0, 0x00000000}, +- {0x67, 1, 0, 0, 0x00000038}, +- {0xc7, 1, 0, 0, 0x00000020}, +- {0x77, 1, 0, 0, 0x0000001f}, +- {0x57, 1, 0, 0, 0x2cc681d1}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x35cf1c6c}, ++ {0xbf, 7, 5, 0, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x6b9e38d9}, ++ {0x6d, 2, 4, 1, 0x00000000}, ++ {0xbf, 7, 5, 0, 0x00000000}, + {0xbf, 3, 4, 0, 0x00000000}, +- {0x57, 3, 0, 0, 0x00000040}, +- {0xb7, 2, 0, 0, 0x00000000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x598d03a2}, ++ {0x57, 3, 0, 0, 0x40000000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xd73c71b2}, + {0xbf, 3, 4, 0, 0x00000000}, +- {0x57, 3, 0, 0, 0x00000020}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xb31a0745}, ++ {0x57, 3, 0, 0, 0x20000000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xae78e364}, + {0xbf, 3, 4, 0, 0x00000000}, +- {0x57, 3, 0, 0, 0x00000010}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x66340e8a}, ++ {0x57, 3, 0, 0, 0x10000000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x5cf1c6c9}, + {0xbf, 3, 4, 0, 0x00000000}, +- {0x57, 3, 0, 0, 0x00000008}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xcc681d15}, ++ {0x57, 3, 0, 0, 0x08000000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xb9e38d92}, + {0xbf, 3, 4, 0, 0x00000000}, +- {0x57, 3, 0, 0, 0x00000004}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x98d03a2b}, ++ {0x57, 3, 0, 0, 0x04000000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x73c71b25}, + {0xbf, 3, 4, 0, 0x00000000}, +- {0x57, 3, 0, 0, 0x00000002}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x31a07456}, ++ {0x57, 3, 0, 0, 0x02000000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xe78e364b}, + {0xbf, 3, 4, 0, 0x00000000}, +- {0x57, 3, 0, 0, 0x00000001}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x6340e8ad}, ++ {0x57, 3, 0, 0, 0x01000000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xcf1c6c96}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00800000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x9e38d92c}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00400000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x3c71b259}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00200000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x78e364b2}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00100000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xf1c6c964}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00080000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xe38d92c9}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00040000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xc71b2593}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00020000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x8e364b27}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00010000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x1c6c964e}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00008000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xc681d15b}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x38d92c9c}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00004000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x8d03a2b7}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x71b25938}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00002000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x1a07456f}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xe364b270}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00001000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x340e8ade}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xc6c964e0}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00000800}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x681d15bd}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x8d92c9c0}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00000400}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xd03a2b7b}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x1b259380}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00000200}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xa07456f6}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x364b2700}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00000100}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x40e8aded}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x6c964e01}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000080}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xd92c9c03}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000040}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xb2593807}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000020}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x64b2700f}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000010}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xc964e01e}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000008}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x92c9c03d}, ++ {0x61, 3, 8, 28, 0x00000000}, ++ {0xbf, 5, 4, 0, 0x00000000}, ++ {0x57, 5, 0, 0, 0x00000004}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x2593807a}, ++ {0xdc, 3, 0, 0, 0x00000040}, ++ {0xbf, 5, 4, 0, 0x00000000}, ++ {0x57, 5, 0, 0, 0x00000002}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x4b2700f4}, ++ {0xc7, 3, 0, 0, 0x00000020}, ++ {0x57, 4, 0, 0, 0x00000001}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x964e01e8}, ++ {0xbf, 5, 7, 0, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x2c9c03d1}, ++ {0x6d, 2, 3, 1, 0x00000000}, ++ {0xbf, 5, 7, 0, 0x00000000}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x40000000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x593807a3}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x20000000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xb2700f46}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x10000000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x64e01e8d}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x08000000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xc9c03d1a}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x04000000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x93807a35}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x02000000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x2700f46b}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x01000000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x4e01e8d6}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00800000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x9c03d1ad}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00400000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x3807a35b}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00200000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x700f46b6}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00100000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xe01e8d6c}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00080000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xc03d1ad9}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00040000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x807a35b3}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00020000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x00f46b66}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00010000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x01e8d6cc}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00008000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x03d1ad99}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00004000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x07a35b32}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00002000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x0f46b665}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00001000}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x1e8d6cca}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00000800}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x3d1ad994}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00000400}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x7a35b328}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00000200}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xf46b6651}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00000100}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xe8d6cca2}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00000080}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xd1ad9944}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00000040}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xa35b3289}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00000020}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x46b66512}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00000010}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x8d6cca25}, ++ {0xbf, 4, 3, 0, 0x00000000}, ++ {0x57, 4, 0, 0, 0x00000008}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x1ad9944a}, ++ {0x61, 4, 8, 32, 0x00000000}, ++ {0xbf, 6, 3, 0, 0x00000000}, ++ {0x57, 6, 0, 0, 0x00000004}, ++ {0x15, 6, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x35b32894}, ++ {0xdc, 4, 0, 0, 0x00000040}, ++ {0xbf, 6, 3, 0, 0x00000000}, ++ {0x57, 6, 0, 0, 0x00000002}, ++ {0x15, 6, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0x6b665129}, ++ {0xc7, 4, 0, 0, 0x00000020}, ++ {0x57, 3, 0, 0, 0x00000001}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 5, 0, 0, 0xd6cca253}, ++ {0xbf, 7, 5, 0, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xad9944a7}, ++ {0x6d, 2, 4, 1, 0x00000000}, ++ {0xbf, 7, 5, 0, 0x00000000}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x40000000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x5b32894f}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x20000000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xb665129f}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x10000000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x6cca253e}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x08000000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xd9944a7d}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x04000000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xb32894fb}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x02000000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x665129f6}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x01000000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xcca253ec}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00800000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x81d15bdb}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x9944a7d9}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00400000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x03a2b7b7}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x32894fb2}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00200000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x07456f6f}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x65129f65}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00100000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x0e8adedf}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xca253eca}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00080000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x1d15bdbf}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x944a7d95}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00040000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x3a2b7b7e}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x2894fb2a}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00020000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x7456f6fd}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x5129f655}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00010000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xe8adedfa}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xa253ecab}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00008000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x44a7d956}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00004000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x894fb2ac}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00002000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x129f6558}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00001000}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x253ecab1}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000800}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x4a7d9563}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000400}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x94fb2ac7}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000200}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x29f6558f}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000100}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x53ecab1e}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000080}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xa7d9563d}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000040}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x4fb2ac7a}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000020}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x9f6558f5}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000010}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x3ecab1ea}, ++ {0xbf, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x00000008}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0x7d9563d5}, ++ {0x61, 3, 8, 36, 0x00000000}, + {0xbf, 5, 4, 0, 0x00000000}, +- {0x67, 5, 0, 0, 0x00000020}, +- {0xc7, 5, 0, 0, 0x00000020}, +- {0xb7, 3, 0, 0, 0xffffffff}, +- {0x6d, 5, 3, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xd15bdbf4}, ++ {0x57, 5, 0, 0, 0x00000004}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xfb2ac7ab}, ++ {0xdc, 3, 0, 0, 0x00000040}, + {0xbf, 5, 4, 0, 0x00000000}, ++ {0x57, 5, 0, 0, 0x00000002}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xf6558f56}, ++ {0xc7, 3, 0, 0, 0x00000020}, ++ {0x57, 4, 0, 0, 0x00000001}, ++ {0x15, 4, 0, 1, 0x00000000}, ++ {0xa7, 7, 0, 0, 0xecab1eac}, ++ {0xbf, 4, 7, 0, 0x00000000}, ++ {0xa7, 4, 0, 0, 0xd9563d59}, ++ {0x6d, 2, 3, 1, 0x00000000}, ++ {0xbf, 4, 7, 0, 0x00000000}, ++ {0xbf, 5, 3, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x40000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xa2b7b7e9}, +- {0xbf, 5, 4, 0, 0x00000000}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0xb2ac7ab2}, ++ {0xbf, 5, 3, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x20000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x456f6fd3}, +- {0xbf, 5, 4, 0, 0x00000000}, ++ {0x79, 6, 10, -56, 0x00000000}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x6558f564}, ++ {0xbf, 5, 3, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x10000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x8adedfa7}, +- {0xbf, 5, 4, 0, 0x00000000}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0xcab1eac8}, ++ {0xbf, 5, 3, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x08000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x15bdbf4f}, +- {0xbf, 5, 4, 0, 0x00000000}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x9563d590}, ++ {0xbf, 5, 3, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x04000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x2b7b7e9e}, +- {0xbf, 5, 4, 0, 0x00000000}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x2ac7ab20}, ++ {0xbf, 5, 3, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x02000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x56f6fd3d}, +- {0x57, 4, 0, 0, 0x01000000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xadedfa7b}, +- {0x61, 4, 6, 12, 0x00000000}, +- {0xbf, 5, 4, 0, 0x00000000}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x558f5641}, ++ {0xbf, 5, 3, 0, 0x00000000}, ++ {0x57, 5, 0, 0, 0x01000000}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0xab1eac83}, ++ {0xbf, 5, 3, 0, 0x00000000}, ++ {0x57, 5, 0, 0, 0x00800000}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x563d5906}, ++ {0xbf, 5, 3, 0, 0x00000000}, ++ {0x57, 5, 0, 0, 0x00400000}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0xac7ab20c}, ++ {0xbf, 5, 3, 0, 0x00000000}, ++ {0x57, 5, 0, 0, 0x00200000}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x58f56418}, ++ {0xbf, 5, 3, 0, 0x00000000}, ++ {0x57, 5, 0, 0, 0x00100000}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0xb1eac831}, ++ {0xbf, 5, 3, 0, 0x00000000}, ++ {0x57, 5, 0, 0, 0x00080000}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x63d59063}, ++ {0xbf, 5, 3, 0, 0x00000000}, ++ {0x57, 5, 0, 0, 0x00040000}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0xc7ab20c7}, ++ {0xbf, 5, 3, 0, 0x00000000}, ++ {0x57, 5, 0, 0, 0x00020000}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x8f56418f}, ++ {0xbf, 5, 3, 0, 0x00000000}, ++ {0x57, 5, 0, 0, 0x00010000}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x1eac831e}, ++ {0xbf, 5, 3, 0, 0x00000000}, ++ {0x57, 5, 0, 0, 0x00008000}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x3d59063c}, ++ {0xbf, 5, 3, 0, 0x00000000}, ++ {0x57, 5, 0, 0, 0x00004000}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x7ab20c78}, ++ {0xbf, 5, 3, 0, 0x00000000}, ++ {0x57, 5, 0, 0, 0x00002000}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0xf56418f0}, ++ {0xbf, 5, 3, 0, 0x00000000}, ++ {0x57, 5, 0, 0, 0x00001000}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0xeac831e1}, ++ {0xbf, 5, 3, 0, 0x00000000}, ++ {0x57, 5, 0, 0, 0x00000800}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0xd59063c2}, ++ {0xbf, 5, 3, 0, 0x00000000}, ++ {0x57, 5, 0, 0, 0x00000400}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0xab20c784}, ++ {0xbf, 5, 3, 0, 0x00000000}, ++ {0x57, 5, 0, 0, 0x00000200}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x56418f09}, ++ {0xbf, 5, 3, 0, 0x00000000}, ++ {0x57, 5, 0, 0, 0x00000100}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0xac831e12}, ++ {0xbf, 5, 3, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000080}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x5bdbf4f7}, +- {0xbf, 5, 4, 0, 0x00000000}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x59063c25}, ++ {0xbf, 5, 3, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000040}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xb7b7e9ef}, +- {0xbf, 5, 4, 0, 0x00000000}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0xb20c784b}, ++ {0xbf, 5, 3, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000020}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x6f6fd3df}, +- {0xbf, 5, 4, 0, 0x00000000}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x6418f097}, ++ {0xbf, 5, 3, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000010}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xdedfa7bf}, +- {0xbf, 5, 4, 0, 0x00000000}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0xc831e12f}, ++ {0xbf, 5, 3, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000008}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xbdbf4f7f}, +- {0xbf, 5, 4, 0, 0x00000000}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x9063c25f}, ++ {0xbf, 5, 3, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000004}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x7b7e9eff}, +- {0xbf, 5, 4, 0, 0x00000000}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x20c784be}, ++ {0xbf, 5, 3, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000002}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xf6fd3dff}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000001}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xedfa7bfe}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00008000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xdbf4f7fc}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00004000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xb7e9eff9}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00002000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x6fd3dff2}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00001000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xdfa7bfe5}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000800}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xbf4f7fca}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000400}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x7e9eff94}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000200}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xfd3dff28}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000100}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xfa7bfe51}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00800000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xf4f7fca2}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00400000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xe9eff945}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00200000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xd3dff28a}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00100000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xa7bfe514}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00080000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x4f7fca28}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00040000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x9eff9450}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00020000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x3dff28a0}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00010000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x7bfe5141}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x67, 5, 0, 0, 0x00000020}, +- {0xc7, 5, 0, 0, 0x00000020}, +- {0x6d, 5, 3, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xf7fca283}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x40000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xeff94506}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x20000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xdff28a0c}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x10000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xbfe51418}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x08000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x7fca2831}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x04000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xff945063}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x02000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xff28a0c6}, +- {0x57, 4, 0, 0, 0x01000000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xfe51418c}, +- {0x61, 4, 6, 16, 0x00000000}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000080}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xfca28319}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000040}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xf9450633}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000020}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xf28a0c67}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000010}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xe51418ce}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000008}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xca28319d}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000004}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x9450633b}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000002}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x28a0c676}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000001}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x51418ced}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00008000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xa28319db}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00004000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x450633b6}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00002000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x8a0c676c}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00001000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x1418ced8}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000800}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x28319db1}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000400}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x50633b63}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000200}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xa0c676c6}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000100}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x418ced8d}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00800000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x8319db1a}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00400000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x0633b634}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00200000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x0c676c68}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00100000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x18ced8d1}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00080000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x319db1a3}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00040000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x633b6347}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00020000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xc676c68f}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00010000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x8ced8d1f}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x67, 5, 0, 0, 0x00000020}, +- {0xc7, 5, 0, 0, 0x00000020}, +- {0x6d, 5, 3, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x19db1a3e}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x40000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x33b6347d}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x20000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x676c68fa}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x10000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xced8d1f4}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x08000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x9db1a3e9}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x04000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x3b6347d2}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x02000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x76c68fa5}, +- {0x57, 4, 0, 0, 0x01000000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xed8d1f4a}, +- {0x61, 4, 6, 20, 0x00000000}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000080}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xdb1a3e94}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000040}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xb6347d28}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000020}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x6c68fa51}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000010}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xd8d1f4a3}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000008}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xb1a3e946}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000004}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x6347d28d}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000002}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xc68fa51a}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000001}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x8d1f4a35}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00008000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x1a3e946b}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00004000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x347d28d7}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00002000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x68fa51ae}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00001000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xd1f4a35c}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000800}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xa3e946b9}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000400}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x47d28d73}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000200}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x8fa51ae7}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000100}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x1f4a35cf}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00800000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x3e946b9e}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00400000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x7d28d73c}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00200000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xfa51ae78}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00100000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xf4a35cf1}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00080000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xe946b9e3}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00040000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xd28d73c7}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00020000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xa51ae78e}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00010000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x4a35cf1c}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x67, 5, 0, 0, 0x00000020}, +- {0xc7, 5, 0, 0, 0x00000020}, +- {0x6d, 5, 3, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x946b9e38}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x40000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x28d73c71}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x20000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x51ae78e3}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x10000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xa35cf1c6}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x08000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x46b9e38d}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x04000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x8d73c71b}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x02000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x1ae78e36}, +- {0x57, 4, 0, 0, 0x01000000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x35cf1c6c}, +- {0x61, 4, 6, 24, 0x00000000}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000080}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x6b9e38d9}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000040}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xd73c71b2}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000020}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xae78e364}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000010}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x5cf1c6c9}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000008}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xb9e38d92}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000004}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x73c71b25}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000002}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xe78e364b}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000001}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xcf1c6c96}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00008000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x9e38d92c}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00004000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x3c71b259}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00002000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x78e364b2}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00001000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xf1c6c964}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000800}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xe38d92c9}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000400}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xc71b2593}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000200}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x8e364b27}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000100}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x1c6c964e}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00800000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x38d92c9c}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00400000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x71b25938}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00200000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xe364b270}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00100000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xc6c964e0}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00080000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x8d92c9c0}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00040000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x1b259380}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00020000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x364b2700}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00010000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x6c964e01}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x67, 5, 0, 0, 0x00000020}, +- {0xc7, 5, 0, 0, 0x00000020}, +- {0x6d, 5, 3, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xd92c9c03}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x40000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xb2593807}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x20000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x64b2700f}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x10000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xc964e01e}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x08000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x92c9c03d}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x04000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x2593807a}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x02000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x4b2700f4}, +- {0x57, 4, 0, 0, 0x01000000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x964e01e8}, +- {0x61, 4, 6, 28, 0x00000000}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000080}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x2c9c03d1}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000040}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x593807a3}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000020}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xb2700f46}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000010}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x64e01e8d}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000008}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xc9c03d1a}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000004}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x93807a35}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000002}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x2700f46b}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000001}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x4e01e8d6}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00008000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x9c03d1ad}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00004000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x3807a35b}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00002000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x700f46b6}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00001000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xe01e8d6c}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000800}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xc03d1ad9}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000400}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x807a35b3}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000200}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x00f46b66}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000100}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x01e8d6cc}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00800000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x03d1ad99}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00400000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x07a35b32}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00200000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x0f46b665}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00100000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x1e8d6cca}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00080000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x3d1ad994}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00040000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x7a35b328}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00020000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xf46b6651}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00010000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xe8d6cca2}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x67, 5, 0, 0, 0x00000020}, +- {0xc7, 5, 0, 0, 0x00000020}, +- {0x6d, 5, 3, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xd1ad9944}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x40000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xa35b3289}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x20000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x46b66512}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x10000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x8d6cca25}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x08000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x1ad9944a}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x04000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x35b32894}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x02000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x6b665129}, +- {0x57, 4, 0, 0, 0x01000000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xd6cca253}, +- {0x61, 4, 6, 32, 0x00000000}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000080}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xad9944a7}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000040}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x5b32894f}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000020}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xb665129f}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000010}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x6cca253e}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000008}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xd9944a7d}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000004}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xb32894fb}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000002}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x665129f6}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000001}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xcca253ec}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00008000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x9944a7d9}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00004000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x32894fb2}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00002000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x65129f65}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00001000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xca253eca}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000800}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x944a7d95}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000400}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x2894fb2a}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000200}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x5129f655}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000100}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xa253ecab}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00800000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x44a7d956}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00400000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x894fb2ac}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00200000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x129f6558}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00100000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x253ecab1}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00080000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x4a7d9563}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00040000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x94fb2ac7}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00020000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x29f6558f}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00010000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x53ecab1e}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x67, 5, 0, 0, 0x00000020}, +- {0xc7, 5, 0, 0, 0x00000020}, +- {0x6d, 5, 3, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xa7d9563d}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x40000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x4fb2ac7a}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x20000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x9f6558f5}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x10000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x3ecab1ea}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x08000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x7d9563d5}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x04000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xfb2ac7ab}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x02000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xf6558f56}, +- {0x57, 4, 0, 0, 0x01000000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xecab1eac}, +- {0x61, 4, 6, 36, 0x00000000}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000080}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xd9563d59}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000040}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xb2ac7ab2}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000020}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x6558f564}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000010}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xcab1eac8}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000008}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x9563d590}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000004}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x2ac7ab20}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000002}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x558f5641}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000001}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xab1eac83}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00008000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x563d5906}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00004000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xac7ab20c}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00002000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x58f56418}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00001000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xb1eac831}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000800}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x63d59063}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000400}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xc7ab20c7}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000200}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x8f56418f}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00000100}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x1eac831e}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00800000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x3d59063c}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00400000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x7ab20c78}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00200000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xf56418f0}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00100000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xeac831e1}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00080000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xd59063c2}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00040000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xab20c784}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00020000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x56418f09}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x00010000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xac831e12}, +- {0xbf, 5, 4, 0, 0x00000000}, ++ {0x15, 5, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x418f097c}, ++ {0x57, 3, 0, 0, 0x00000001}, ++ {0x15, 3, 0, 1, 0x00000000}, ++ {0xa7, 4, 0, 0, 0x831e12f9}, ++ {0xbf, 5, 1, 0, 0x00000000}, + {0x67, 5, 0, 0, 0x00000020}, + {0xc7, 5, 0, 0, 0x00000020}, +- {0x6d, 5, 3, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x59063c25}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x40000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xb20c784b}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x20000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x6418f097}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x10000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xc831e12f}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x08000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x9063c25f}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x04000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x20c784be}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x57, 5, 0, 0, 0x02000000}, +- {0x1d, 5, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x418f097c}, +- {0x57, 4, 0, 0, 0x01000000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x831e12f9}, +- {0x71, 4, 6, 40, 0x00000000}, +- {0x67, 4, 0, 0, 0x00000008}, +- {0x71, 5, 6, 41, 0x00000000}, +- {0x4f, 4, 5, 0, 0x00000000}, +- {0xbf, 5, 4, 0, 0x00000000}, +- {0x67, 5, 0, 0, 0x00000030}, +- {0xc7, 5, 0, 0, 0x00000020}, +- {0x6d, 5, 3, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x063c25f3}, +- {0x67, 4, 0, 0, 0x00000010}, +- {0xbf, 3, 4, 0, 0x00000000}, +- {0x57, 3, 0, 0, 0x40000000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x0c784be7}, +- {0xbf, 3, 4, 0, 0x00000000}, +- {0x57, 3, 0, 0, 0x20000000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x18f097cf}, +- {0xbf, 3, 4, 0, 0x00000000}, +- {0x57, 3, 0, 0, 0x10000000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x31e12f9f}, +- {0xbf, 3, 4, 0, 0x00000000}, +- {0x57, 3, 0, 0, 0x08000000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x63c25f3f}, +- {0xbf, 3, 4, 0, 0x00000000}, +- {0x57, 3, 0, 0, 0x04000000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xc784be7f}, +- {0xbf, 3, 4, 0, 0x00000000}, +- {0x57, 3, 0, 0, 0x02000000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x8f097cff}, +- {0xbf, 3, 4, 0, 0x00000000}, +- {0x57, 3, 0, 0, 0x01000000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x1e12f9fe}, +- {0xbf, 3, 4, 0, 0x00000000}, +- {0x57, 3, 0, 0, 0x00800000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x3c25f3fc}, +- {0xbf, 3, 4, 0, 0x00000000}, +- {0x57, 3, 0, 0, 0x00400000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x784be7f8}, +- {0xbf, 3, 4, 0, 0x00000000}, +- {0x57, 3, 0, 0, 0x00200000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xf097cff0}, +- {0xbf, 3, 4, 0, 0x00000000}, +- {0x57, 3, 0, 0, 0x00100000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xe12f9fe0}, +- {0xbf, 3, 4, 0, 0x00000000}, +- {0x57, 3, 0, 0, 0x00080000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xc25f3fc1}, + {0xbf, 3, 4, 0, 0x00000000}, +- {0x57, 3, 0, 0, 0x00040000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x84be7f83}, ++ {0xa7, 3, 0, 0, 0x063c25f3}, ++ {0x6d, 2, 5, 1, 0x00000000}, + {0xbf, 3, 4, 0, 0x00000000}, +- {0x57, 3, 0, 0, 0x00020000}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x097cff07}, +- {0x57, 4, 0, 0, 0x00010000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x12f9fe0f}, +- {0x71, 3, 6, 42, 0x00000000}, +- {0x67, 3, 0, 0, 0x00000008}, +- {0xbf, 4, 3, 0, 0x00000000}, +- {0x57, 4, 0, 0, 0x00008000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x25f3fc1f}, +- {0xbf, 4, 3, 0, 0x00000000}, +- {0x57, 4, 0, 0, 0x00004000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x4be7f83f}, +- {0xbf, 4, 3, 0, 0x00000000}, +- {0x57, 4, 0, 0, 0x00002000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x97cff07f}, +- {0xbf, 4, 3, 0, 0x00000000}, +- {0x57, 4, 0, 0, 0x00001000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x2f9fe0fe}, +- {0xbf, 4, 3, 0, 0x00000000}, +- {0x57, 4, 0, 0, 0x00000800}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x5f3fc1fd}, +- {0xbf, 4, 3, 0, 0x00000000}, +- {0x57, 4, 0, 0, 0x00000400}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xbe7f83fb}, +- {0xbf, 4, 3, 0, 0x00000000}, +- {0x57, 4, 0, 0, 0x00000200}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x7cff07f7}, +- {0x57, 3, 0, 0, 0x00000100}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xf9fe0fee}, +- {0x71, 3, 6, 43, 0x00000000}, +- {0xbf, 4, 3, 0, 0x00000000}, +- {0x57, 4, 0, 0, 0x00000080}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xf3fc1fdc}, +- {0xbf, 4, 3, 0, 0x00000000}, +- {0x57, 4, 0, 0, 0x00000040}, +- {0x79, 5, 10, -56, 0x00000000}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xe7f83fb8}, +- {0xbf, 4, 3, 0, 0x00000000}, +- {0x57, 4, 0, 0, 0x00000020}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xcff07f70}, +- {0xbf, 4, 3, 0, 0x00000000}, +- {0x57, 4, 0, 0, 0x00000010}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x9fe0fee1}, +- {0xbf, 4, 3, 0, 0x00000000}, +- {0x57, 4, 0, 0, 0x00000008}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x3fc1fdc2}, +- {0xbf, 4, 3, 0, 0x00000000}, +- {0x57, 4, 0, 0, 0x00000004}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0x7f83fb85}, +- {0xbf, 4, 3, 0, 0x00000000}, +- {0x57, 4, 0, 0, 0x00000002}, +- {0x1d, 4, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xff07f70a}, +- {0x57, 3, 0, 0, 0x00000001}, +- {0x1d, 3, 2, 1, 0x00000000}, +- {0xa7, 1, 0, 0, 0xfe0fee15}, +- {0x71, 2, 0, 201, 0x00000000}, +- {0x67, 2, 0, 0, 0x00000008}, +- {0x71, 3, 0, 200, 0x00000000}, +- {0x4f, 2, 3, 0, 0x00000000}, +- {0x71, 3, 0, 203, 0x00000000}, +- {0x67, 3, 0, 0, 0x00000008}, +- {0x71, 4, 0, 202, 0x00000000}, +- {0x4f, 3, 4, 0, 0x00000000}, +- {0x67, 3, 0, 0, 0x00000010}, +- {0x4f, 3, 2, 0, 0x00000000}, +- {0x67, 1, 0, 0, 0x00000020}, +- {0x77, 1, 0, 0, 0x00000020}, + {0xbf, 2, 1, 0, 0x00000000}, +- {0x3f, 2, 3, 0, 0x00000000}, +- {0x2f, 2, 3, 0, 0x00000000}, +- {0x1f, 1, 2, 0, 0x00000000}, +- {0x57, 1, 0, 0, 0x0000000f}, +- {0x67, 1, 0, 0, 0x00000002}, +- {0x0f, 0, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x40000000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x0c784be7}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x20000000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x18f097cf}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x10000000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x31e12f9f}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x08000000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x63c25f3f}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x04000000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0xc784be7f}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x02000000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x8f097cff}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x01000000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x1e12f9fe}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00800000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x3c25f3fc}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00400000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x784be7f8}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00200000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0xf097cff0}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00100000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0xe12f9fe0}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00080000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0xc25f3fc1}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00040000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x84be7f83}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00020000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x097cff07}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00010000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x12f9fe0f}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00008000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x25f3fc1f}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00004000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x4be7f83f}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00002000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x97cff07f}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00001000}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x2f9fe0fe}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00000800}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x5f3fc1fd}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00000400}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0xbe7f83fb}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00000200}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x7cff07f7}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00000100}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0xf9fe0fee}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00000080}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0xf3fc1fdc}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00000040}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0xe7f83fb8}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00000020}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0xcff07f70}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00000010}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x9fe0fee1}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00000008}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x3fc1fdc2}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00000004}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0x7f83fb85}, ++ {0xbf, 2, 1, 0, 0x00000000}, ++ {0x57, 2, 0, 0, 0x00000002}, ++ {0x15, 2, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0xff07f70a}, ++ {0x57, 1, 0, 0, 0x00000001}, ++ {0x15, 1, 0, 1, 0x00000000}, ++ {0xa7, 3, 0, 0, 0xfe0fee15}, ++ {0x71, 1, 0, 201, 0x00000000}, ++ {0x67, 1, 0, 0, 0x00000008}, ++ {0x71, 2, 0, 200, 0x00000000}, ++ {0x4f, 1, 2, 0, 0x00000000}, ++ {0x71, 2, 0, 202, 0x00000000}, ++ {0x67, 2, 0, 0, 0x00000010}, ++ {0x71, 4, 0, 203, 0x00000000}, ++ {0x67, 4, 0, 0, 0x00000018}, ++ {0x4f, 4, 2, 0, 0x00000000}, ++ {0x4f, 4, 1, 0, 0x00000000}, ++ {0x67, 3, 0, 0, 0x00000020}, ++ {0x77, 3, 0, 0, 0x00000020}, ++ {0x9f, 3, 4, 0, 0x00000000}, ++ {0x57, 3, 0, 0, 0x0000000f}, ++ {0x67, 3, 0, 0, 0x00000002}, ++ {0x0f, 0, 3, 0, 0x00000000}, + {0x71, 1, 0, 137, 0x00000000}, + {0x67, 1, 0, 0, 0x00000008}, + {0x71, 2, 0, 136, 0x00000000}, + {0x4f, 1, 2, 0, 0x00000000}, + {0x71, 2, 0, 138, 0x00000000}, ++ {0x67, 2, 0, 0, 0x00000010}, + {0x71, 3, 0, 139, 0x00000000}, +- {0x67, 3, 0, 0, 0x00000008}, ++ {0x67, 3, 0, 0, 0x00000018}, + {0x4f, 3, 2, 0, 0x00000000}, +- {0x67, 3, 0, 0, 0x00000010}, + {0x4f, 3, 1, 0, 0x00000000}, + {0x07, 3, 0, 0, 0x7cafe800}, +- {0x63, 5, 3, 52, 0x00000000}, ++ {0x63, 6, 3, 52, 0x00000000}, + {0xb7, 7, 0, 0, 0x00000001}, + {0xbf, 0, 7, 0, 0x00000000}, + {0x95, 0, 0, 0, 0x00000000}, +diff --git a/dpdk/drivers/net/tap/tap_bpf_program.c b/dpdk/drivers/net/tap/tap_bpf_program.c +index 20c310e5e7..d9bb65831a 100644 +--- a/dpdk/drivers/net/tap/tap_bpf_program.c ++++ b/dpdk/drivers/net/tap/tap_bpf_program.c +@@ -131,6 +131,8 @@ rss_l3_l4(struct __sk_buff *skb) + __u8 *key = 0; + __u32 len; + __u32 queue = 0; ++ bool mf = 0; ++ __u16 frag_off = 0; + + rsskey = map_lookup_elem(&map_keys, &key_idx); + if (!rsskey) { +@@ -155,6 +157,8 @@ rss_l3_l4(struct __sk_buff *skb) + return TC_ACT_OK; + + __u8 *src_dst_addr = data + off + offsetof(struct iphdr, saddr); ++ __u8 *frag_off_addr = data + off + offsetof(struct iphdr, frag_off); ++ __u8 *prot_addr = data + off + offsetof(struct iphdr, protocol); + __u8 *src_dst_port = data + off + sizeof(struct iphdr); + struct ipv4_l3_l4_tuple v4_tuple = { + .src_addr = IPv4(*(src_dst_addr + 0), +@@ -165,11 +169,25 @@ rss_l3_l4(struct __sk_buff *skb) + *(src_dst_addr + 5), + *(src_dst_addr + 6), + *(src_dst_addr + 7)), +- .sport = PORT(*(src_dst_port + 0), +- *(src_dst_port + 1)), +- .dport = PORT(*(src_dst_port + 2), +- *(src_dst_port + 3)), ++ .sport = 0, ++ .dport = 0, + }; ++ /** Fetch the L4-payer port numbers only in-case of TCP/UDP ++ ** and also if the packet is not fragmented. Since fragmented ++ ** chunks do not have L4 TCP/UDP header. ++ **/ ++ if (*prot_addr == IPPROTO_UDP || *prot_addr == IPPROTO_TCP) { ++ frag_off = PORT(*(frag_off_addr + 0), ++ *(frag_off_addr + 1)); ++ mf = frag_off & 0x2000; ++ frag_off = frag_off & 0x1fff; ++ if (mf == 0 && frag_off == 0) { ++ v4_tuple.sport = PORT(*(src_dst_port + 0), ++ *(src_dst_port + 1)); ++ v4_tuple.dport = PORT(*(src_dst_port + 2), ++ *(src_dst_port + 3)); ++ } ++ } + __u8 input_len = sizeof(v4_tuple) / sizeof(__u32); + if (rsskey->hash_fields & (1 << HASH_FIELD_IPV4_L3)) + input_len--; +@@ -182,6 +200,9 @@ rss_l3_l4(struct __sk_buff *skb) + offsetof(struct ipv6hdr, saddr); + __u8 *src_dst_port = data + off + + sizeof(struct ipv6hdr); ++ __u8 *next_hdr = data + off + ++ offsetof(struct ipv6hdr, nexthdr); ++ + struct ipv6_l3_l4_tuple v6_tuple; + for (j = 0; j < 4; j++) + *((uint32_t *)&v6_tuple.src_addr + j) = +@@ -191,10 +212,18 @@ rss_l3_l4(struct __sk_buff *skb) + *((uint32_t *)&v6_tuple.dst_addr + j) = + __builtin_bswap32(*((uint32_t *) + src_dst_addr + 4 + j)); +- v6_tuple.sport = PORT(*(src_dst_port + 0), +- *(src_dst_port + 1)); +- v6_tuple.dport = PORT(*(src_dst_port + 2), +- *(src_dst_port + 3)); ++ ++ /** Fetch the L4 header port-numbers only if next-header ++ * is TCP/UDP **/ ++ if (*next_hdr == IPPROTO_UDP || *next_hdr == IPPROTO_TCP) { ++ v6_tuple.sport = PORT(*(src_dst_port + 0), ++ *(src_dst_port + 1)); ++ v6_tuple.dport = PORT(*(src_dst_port + 2), ++ *(src_dst_port + 3)); ++ } else { ++ v6_tuple.sport = 0; ++ v6_tuple.dport = 0; ++ } + + __u8 input_len = sizeof(v6_tuple) / sizeof(__u32); + if (rsskey->hash_fields & (1 << HASH_FIELD_IPV6_L3)) +diff --git a/dpdk/drivers/net/txgbe/base/txgbe_hw.c b/dpdk/drivers/net/txgbe/base/txgbe_hw.c +index 8966453a03..de96549ae8 100644 +--- a/dpdk/drivers/net/txgbe/base/txgbe_hw.c ++++ b/dpdk/drivers/net/txgbe/base/txgbe_hw.c +@@ -2273,10 +2273,24 @@ s32 txgbe_setup_mac_link_multispeed_fiber(struct txgbe_hw *hw, + } + + if (speed & TXGBE_LINK_SPEED_1GB_FULL) { ++ u32 curr_autoneg; ++ + speedcnt++; + if (highest_link_speed == TXGBE_LINK_SPEED_UNKNOWN) + highest_link_speed = TXGBE_LINK_SPEED_1GB_FULL; + ++ status = hw->mac.check_link(hw, &link_speed, &link_up, false); ++ if (status != 0) ++ return status; ++ ++ /* If we already have link at this speed, just jump out */ ++ if (link_speed == TXGBE_LINK_SPEED_1GB_FULL) { ++ curr_autoneg = rd32_epcs(hw, SR_MII_MMD_CTL); ++ if (link_up && (hw->autoneg == ++ !!(curr_autoneg & SR_MII_MMD_CTL_AN_EN))) ++ goto out; ++ } ++ + /* Set the module link speed */ + switch (hw->phy.media_type) { + case txgbe_media_type_fiber: +@@ -2987,10 +3001,6 @@ void txgbe_disable_tx_laser_multispeed_fiber(struct txgbe_hw *hw) + { + u32 esdp_reg = rd32(hw, TXGBE_GPIODATA); + +- /* Blocked by MNG FW so bail */ +- if (txgbe_check_reset_blocked(hw)) +- return; +- + if (txgbe_close_notify(hw)) + txgbe_led_off(hw, TXGBE_LEDCTL_UP | TXGBE_LEDCTL_10G | + TXGBE_LEDCTL_1G | TXGBE_LEDCTL_ACTIVE); +@@ -3038,10 +3048,6 @@ void txgbe_enable_tx_laser_multispeed_fiber(struct txgbe_hw *hw) + **/ + void txgbe_flap_tx_laser_multispeed_fiber(struct txgbe_hw *hw) + { +- /* Blocked by MNG FW so bail */ +- if (txgbe_check_reset_blocked(hw)) +- return; +- + if (hw->mac.autotry_restart) { + txgbe_disable_tx_laser_multispeed_fiber(hw); + txgbe_enable_tx_laser_multispeed_fiber(hw); +@@ -3432,18 +3438,9 @@ s32 txgbe_reset_hw(struct txgbe_hw *hw) + autoc = hw->mac.autoc_read(hw); + + mac_reset_top: +- /* +- * Issue global reset to the MAC. Needs to be SW reset if link is up. +- * If link reset is used when link is up, it might reset the PHY when +- * mng is using it. If link is down or the flag to force full link +- * reset is set, then perform link reset. +- */ +- if (txgbe_mng_present(hw)) { +- txgbe_hic_reset(hw); +- } else { +- wr32(hw, TXGBE_RST, TXGBE_RST_LAN(hw->bus.lan_id)); +- txgbe_flush(hw); +- } ++ /* Do LAN reset, the MNG domain will not be reset. */ ++ wr32(hw, TXGBE_RST, TXGBE_RST_LAN(hw->bus.lan_id)); ++ txgbe_flush(hw); + usec_delay(10); + + txgbe_reset_misc(hw); +diff --git a/dpdk/drivers/net/txgbe/base/txgbe_mng.c b/dpdk/drivers/net/txgbe/base/txgbe_mng.c +index df7145094f..029a0a1fe1 100644 +--- a/dpdk/drivers/net/txgbe/base/txgbe_mng.c ++++ b/dpdk/drivers/net/txgbe/base/txgbe_mng.c +@@ -141,21 +141,7 @@ txgbe_host_interface_command(struct txgbe_hw *hw, u32 *buffer, + for (bi = 0; bi < dword_len; bi++) + buffer[bi] = rd32a(hw, TXGBE_MNGMBX, bi); + +- /* +- * If there is any thing in data position pull it in +- * Read Flash command requires reading buffer length from +- * two byes instead of one byte +- */ +- if (resp->cmd == 0x30) { +- for (; bi < dword_len + 2; bi++) +- buffer[bi] = rd32a(hw, TXGBE_MNGMBX, bi); +- +- buf_len = (((u16)(resp->cmd_or_resp.ret_status) << 3) +- & 0xF00) | resp->buf_len; +- hdr_size += (2 << 2); +- } else { +- buf_len = resp->buf_len; +- } ++ buf_len = resp->buf_len; + if (!buf_len) + goto rel_out; + +diff --git a/dpdk/drivers/net/txgbe/base/txgbe_phy.c b/dpdk/drivers/net/txgbe/base/txgbe_phy.c +index 9f46d5bdb0..a7c11c50df 100644 +--- a/dpdk/drivers/net/txgbe/base/txgbe_phy.c ++++ b/dpdk/drivers/net/txgbe/base/txgbe_phy.c +@@ -1380,7 +1380,9 @@ txgbe_set_sgmii_an37_ability(struct txgbe_hw *hw) + wr32_epcs(hw, SR_MII_MMD_AN_CTL, 0x0105); + wr32_epcs(hw, SR_MII_MMD_DIGI_CTL, 0x0200); + value = rd32_epcs(hw, SR_MII_MMD_CTL); +- value = (value & ~0x1200) | (0x1 << 12) | (0x1 << 9); ++ value = (value & ~0x1200) | (0x1 << 9); ++ if (hw->autoneg) ++ value |= SR_MII_MMD_CTL_AN_EN; + wr32_epcs(hw, SR_MII_MMD_CTL, value); + } + +@@ -1519,8 +1521,9 @@ txgbe_set_link_to_kx4(struct txgbe_hw *hw, bool autoneg) + goto out; + } + +- wr32m(hw, TXGBE_MACTXCFG, TXGBE_MACTXCFG_TXE, +- ~TXGBE_MACTXCFG_TXE); ++ wr32m(hw, TXGBE_MACTXCFG, TXGBE_MACTXCFG_TXE, ~TXGBE_MACTXCFG_TXE); ++ wr32m(hw, TXGBE_MACRXCFG, TXGBE_MACRXCFG_ENA, ~TXGBE_MACRXCFG_ENA); ++ hw->mac.disable_sec_tx_path(hw); + + /* 2. Disable xpcs AN-73 */ + if (!autoneg) +@@ -1693,9 +1696,10 @@ txgbe_set_link_to_kx4(struct txgbe_hw *hw, bool autoneg) + wr32_epcs(hw, TXGBE_PHY_TX_EQ_CTL1, value); + } else if (hw->fw_version <= TXGBE_FW_N_TXEQ) { + value = (0x1804 & ~0x3F3F); ++ value |= 40 << 8; + wr32_epcs(hw, TXGBE_PHY_TX_EQ_CTL0, value); + +- value = (0x50 & ~0x7F) | 40 | (1 << 6); ++ value = (0x50 & ~0x7F) | (1 << 6); + wr32_epcs(hw, TXGBE_PHY_TX_EQ_CTL1, value); + } + out: +@@ -1733,8 +1737,9 @@ txgbe_set_link_to_kx(struct txgbe_hw *hw, + goto out; + } + +- wr32m(hw, TXGBE_MACTXCFG, TXGBE_MACTXCFG_TXE, +- ~TXGBE_MACTXCFG_TXE); ++ wr32m(hw, TXGBE_MACTXCFG, TXGBE_MACTXCFG_TXE, ~TXGBE_MACTXCFG_TXE); ++ wr32m(hw, TXGBE_MACRXCFG, TXGBE_MACRXCFG_ENA, ~TXGBE_MACRXCFG_ENA); ++ hw->mac.disable_sec_tx_path(hw); + + /* 2. Disable xpcs AN-73 */ + if (!autoneg) +@@ -1907,10 +1912,10 @@ txgbe_set_link_to_kx(struct txgbe_hw *hw, + value |= hw->phy.ffe_post | (1 << 6); + wr32_epcs(hw, TXGBE_PHY_TX_EQ_CTL1, value); + } else if (hw->fw_version <= TXGBE_FW_N_TXEQ) { +- value = (0x1804 & ~0x3F3F) | (24 << 8) | 4; ++ value = (0x1804 & ~0x3F3F) | (40 << 8); + wr32_epcs(hw, TXGBE_PHY_TX_EQ_CTL0, value); + +- value = (0x50 & ~0x7F) | 16 | (1 << 6); ++ value = (0x50 & ~0x7F) | (1 << 6); + wr32_epcs(hw, TXGBE_PHY_TX_EQ_CTL1, value); + } + out: +@@ -1940,8 +1945,9 @@ txgbe_set_link_to_sfi(struct txgbe_hw *hw, + goto out; + } + +- wr32m(hw, TXGBE_MACTXCFG, TXGBE_MACTXCFG_TXE, +- ~TXGBE_MACTXCFG_TXE); ++ wr32m(hw, TXGBE_MACTXCFG, TXGBE_MACTXCFG_TXE, ~TXGBE_MACTXCFG_TXE); ++ wr32m(hw, TXGBE_MACRXCFG, TXGBE_MACRXCFG_ENA, ~TXGBE_MACRXCFG_ENA); ++ hw->mac.disable_sec_tx_path(hw); + + /* 2. Disable xpcs AN-73 */ + wr32_epcs(hw, SR_AN_CTRL, 0x0); +@@ -2292,6 +2298,8 @@ void txgbe_autoc_write(struct txgbe_hw *hw, u64 autoc) + txgbe_set_sgmii_an37_ability(hw); + } + ++ hw->mac.enable_sec_tx_path(hw); ++ + if (speed == TXGBE_LINK_SPEED_10GB_FULL) + mactxcfg = TXGBE_MACTXCFG_SPEED_10G; + else if (speed == TXGBE_LINK_SPEED_1GB_FULL) +@@ -2301,6 +2309,7 @@ void txgbe_autoc_write(struct txgbe_hw *hw, u64 autoc) + wr32m(hw, TXGBE_MACTXCFG, + TXGBE_MACTXCFG_SPEED_MASK | TXGBE_MACTXCFG_TXE, + mactxcfg | TXGBE_MACTXCFG_TXE); ++ wr32m(hw, TXGBE_MACRXCFG, TXGBE_MACRXCFG_ENA, TXGBE_MACRXCFG_ENA); + } + + void txgbe_bp_down_event(struct txgbe_hw *hw) +diff --git a/dpdk/drivers/net/txgbe/base/txgbe_regs.h b/dpdk/drivers/net/txgbe/base/txgbe_regs.h +index 911bb6e04e..79290a7afe 100644 +--- a/dpdk/drivers/net/txgbe/base/txgbe_regs.h ++++ b/dpdk/drivers/net/txgbe/base/txgbe_regs.h +@@ -1579,6 +1579,7 @@ enum txgbe_5tuple_protocol { + #define TXGBE_GPIOINTMASK 0x014834 + #define TXGBE_GPIOINTTYPE 0x014838 + #define TXGBE_GPIOINTSTAT 0x014840 ++#define TXGBE_GPIORAWINTSTAT 0x014844 + #define TXGBE_GPIOEOI 0x01484C + + +@@ -1884,7 +1885,19 @@ po32m(struct txgbe_hw *hw, u32 reg, u32 mask, u32 expect, u32 *actual, + } + + /* flush all write operations */ +-#define txgbe_flush(hw) rd32(hw, 0x00100C) ++static inline void txgbe_flush(struct txgbe_hw *hw) ++{ ++ switch (hw->mac.type) { ++ case txgbe_mac_raptor: ++ rd32(hw, TXGBE_PWR); ++ break; ++ case txgbe_mac_raptor_vf: ++ rd32(hw, TXGBE_VFSTATUS); ++ break; ++ default: ++ break; ++ } ++} + + #define rd32a(hw, reg, idx) ( \ + rd32((hw), (reg) + ((idx) << 2))) +diff --git a/dpdk/drivers/net/txgbe/base/txgbe_type.h b/dpdk/drivers/net/txgbe/base/txgbe_type.h +index c3486b472f..75e839b7de 100644 +--- a/dpdk/drivers/net/txgbe/base/txgbe_type.h ++++ b/dpdk/drivers/net/txgbe/base/txgbe_type.h +@@ -783,6 +783,7 @@ struct txgbe_hw { + bool allow_unsupported_sfp; + bool need_crosstalk_fix; + bool dev_start; ++ bool autoneg; + struct txgbe_devargs devarg; + + uint64_t isb_dma; +diff --git a/dpdk/drivers/net/txgbe/txgbe_ethdev.c b/dpdk/drivers/net/txgbe/txgbe_ethdev.c +index 86ef979b29..001f8c6473 100644 +--- a/dpdk/drivers/net/txgbe/txgbe_ethdev.c ++++ b/dpdk/drivers/net/txgbe/txgbe_ethdev.c +@@ -179,7 +179,9 @@ static const struct rte_txgbe_xstats_name_off rte_txgbe_stats_strings[] = { + HW_XSTAT(tx_total_packets), + HW_XSTAT(rx_total_missed_packets), + HW_XSTAT(rx_broadcast_packets), ++ HW_XSTAT(tx_broadcast_packets), + HW_XSTAT(rx_multicast_packets), ++ HW_XSTAT(tx_multicast_packets), + HW_XSTAT(rx_management_packets), + HW_XSTAT(tx_management_packets), + HW_XSTAT(rx_management_dropped), +@@ -543,6 +545,7 @@ null: + static int + eth_txgbe_dev_init(struct rte_eth_dev *eth_dev, void *init_params __rte_unused) + { ++ struct txgbe_adapter *ad = eth_dev->data->dev_private; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + struct txgbe_hw *hw = TXGBE_DEV_HW(eth_dev); + struct txgbe_vfta *shadow_vfta = TXGBE_DEV_VFTA(eth_dev); +@@ -591,6 +594,7 @@ eth_txgbe_dev_init(struct rte_eth_dev *eth_dev, void *init_params __rte_unused) + return 0; + } + ++ __atomic_clear(&ad->link_thread_running, __ATOMIC_SEQ_CST); + rte_eth_copy_pci_info(eth_dev, pci_dev); + + hw->hw_addr = (void *)pci_dev->mem_resource[0].addr; +@@ -1494,6 +1498,19 @@ txgbe_check_mq_mode(struct rte_eth_dev *dev) + return -EINVAL; + } + } ++ ++ /* ++ * When DCB/VT is off, maximum number of queues changes ++ */ ++ if (dev_conf->txmode.mq_mode == RTE_ETH_MQ_TX_NONE) { ++ if (nb_tx_q > TXGBE_NONE_MODE_TX_NB_QUEUES) { ++ PMD_INIT_LOG(ERR, ++ "Neither VT nor DCB are enabled, " ++ "nb_tx_q > %d.", ++ TXGBE_NONE_MODE_TX_NB_QUEUES); ++ return -EINVAL; ++ } ++ } + } + return 0; + } +@@ -1530,6 +1547,25 @@ txgbe_dev_configure(struct rte_eth_dev *dev) + return 0; + } + ++static void txgbe_reinit_gpio_intr(struct txgbe_hw *hw) ++{ ++ u32 reg; ++ ++ wr32(hw, TXGBE_GPIOINTMASK, 0xFF); ++ reg = rd32(hw, TXGBE_GPIORAWINTSTAT); ++ ++ if (reg & TXGBE_GPIOBIT_2) ++ wr32(hw, TXGBE_GPIOEOI, TXGBE_GPIOBIT_2); ++ ++ if (reg & TXGBE_GPIOBIT_3) ++ wr32(hw, TXGBE_GPIOEOI, TXGBE_GPIOBIT_3); ++ ++ if (reg & TXGBE_GPIOBIT_6) ++ wr32(hw, TXGBE_GPIOEOI, TXGBE_GPIOBIT_6); ++ ++ wr32(hw, TXGBE_GPIOINTMASK, 0); ++} ++ + static void + txgbe_dev_phy_intr_setup(struct rte_eth_dev *dev) + { +@@ -1647,7 +1683,7 @@ txgbe_dev_start(struct rte_eth_dev *dev) + PMD_INIT_FUNC_TRACE(); + + /* Stop the link setup handler before resetting the HW. */ +- rte_eal_alarm_cancel(txgbe_dev_setup_link_alarm_handler, dev); ++ txgbe_dev_wait_setup_link_complete(dev, 0); + + /* disable uio/vfio intr/eventfd mapping */ + rte_intr_disable(intr_handle); +@@ -1668,6 +1704,10 @@ txgbe_dev_start(struct rte_eth_dev *dev) + hw->mac.get_link_status = true; + hw->dev_start = true; + ++ /* workaround for GPIO intr lost when mng_veto bit is set */ ++ if (txgbe_check_reset_blocked(hw)) ++ txgbe_reinit_gpio_intr(hw); ++ + /* configure PF module if SRIOV enabled */ + txgbe_pf_host_configure(dev); + +@@ -1786,6 +1826,7 @@ txgbe_dev_start(struct rte_eth_dev *dev) + speed = (TXGBE_LINK_SPEED_100M_FULL | + TXGBE_LINK_SPEED_1GB_FULL | + TXGBE_LINK_SPEED_10GB_FULL); ++ hw->autoneg = true; + } else { + if (*link_speeds & RTE_ETH_LINK_SPEED_10G) + speed |= TXGBE_LINK_SPEED_10GB_FULL; +@@ -1797,6 +1838,7 @@ txgbe_dev_start(struct rte_eth_dev *dev) + speed |= TXGBE_LINK_SPEED_1GB_FULL; + if (*link_speeds & RTE_ETH_LINK_SPEED_100M) + speed |= TXGBE_LINK_SPEED_100M_FULL; ++ hw->autoneg = false; + } + + err = hw->mac.setup_link(hw, speed, link_up); +@@ -1875,15 +1917,19 @@ txgbe_dev_stop(struct rte_eth_dev *dev) + struct txgbe_tm_conf *tm_conf = TXGBE_DEV_TM_CONF(dev); + + if (hw->adapter_stopped) +- return 0; ++ goto out; + + PMD_INIT_FUNC_TRACE(); + +- rte_eal_alarm_cancel(txgbe_dev_setup_link_alarm_handler, dev); ++ txgbe_dev_wait_setup_link_complete(dev, 0); + + /* disable interrupts */ + txgbe_disable_intr(hw); + ++ /* workaround for GPIO intr lost when mng_veto bit is set */ ++ if (txgbe_check_reset_blocked(hw)) ++ txgbe_reinit_gpio_intr(hw); ++ + /* reset the NIC */ + txgbe_pf_reset_hw(hw); + hw->adapter_stopped = 0; +@@ -1894,14 +1940,6 @@ txgbe_dev_stop(struct rte_eth_dev *dev) + for (vf = 0; vfinfo != NULL && vf < pci_dev->max_vfs; vf++) + vfinfo[vf].clear_to_send = false; + +- if (hw->phy.media_type == txgbe_media_type_copper) { +- /* Turn off the copper */ +- hw->phy.set_phy_power(hw, false); +- } else { +- /* Turn off the laser */ +- hw->mac.disable_tx_laser(hw); +- } +- + txgbe_dev_clear_queues(dev); + + /* Clear stored conf */ +@@ -1932,6 +1970,16 @@ txgbe_dev_stop(struct rte_eth_dev *dev) + dev->data->dev_started = 0; + hw->dev_start = false; + ++out: ++ /* close phy to prevent reset in dev_close from restarting physical link */ ++ if (hw->phy.media_type == txgbe_media_type_copper) { ++ /* Turn off the copper */ ++ hw->phy.set_phy_power(hw, false); ++ } else { ++ /* Turn off the laser */ ++ hw->mac.disable_tx_laser(hw); ++ } ++ + return 0; + } + +@@ -1991,6 +2039,9 @@ txgbe_dev_close(struct rte_eth_dev *dev) + + PMD_INIT_FUNC_TRACE(); + ++ if (rte_eal_process_type() != RTE_PROC_PRIMARY) ++ return 0; ++ + txgbe_pf_reset_hw(hw); + + ret = txgbe_dev_stop(dev); +@@ -2019,8 +2070,9 @@ txgbe_dev_close(struct rte_eth_dev *dev) + rte_delay_ms(100); + } while (retries++ < (10 + TXGBE_LINK_UP_TIME)); + +- /* cancel the delay handler before remove dev */ ++ /* cancel all alarm handler before remove dev */ + rte_eal_alarm_cancel(txgbe_dev_interrupt_delayed_handler, dev); ++ rte_eal_alarm_cancel(txgbe_dev_setup_link_alarm_handler, dev); + + /* uninitialize PF if max_vfs not zero */ + txgbe_pf_host_uninit(dev); +@@ -2690,11 +2742,52 @@ txgbe_dev_setup_link_alarm_handler(void *param) + intr->flags &= ~TXGBE_FLAG_NEED_LINK_CONFIG; + } + ++/* ++ * If @timeout_ms was 0, it means that it will not return until link complete. ++ * It returns 1 on complete, return 0 on timeout. ++ */ ++int ++txgbe_dev_wait_setup_link_complete(struct rte_eth_dev *dev, uint32_t timeout_ms) ++{ ++#define WARNING_TIMEOUT 9000 /* 9s in total */ ++ struct txgbe_adapter *ad = TXGBE_DEV_ADAPTER(dev); ++ uint32_t timeout = timeout_ms ? timeout_ms : WARNING_TIMEOUT; ++ ++ while (__atomic_load_n(&ad->link_thread_running, __ATOMIC_SEQ_CST)) { ++ msec_delay(1); ++ timeout--; ++ ++ if (timeout_ms) { ++ if (!timeout) ++ return 0; ++ } else if (!timeout) { ++ /* It will not return until link complete */ ++ timeout = WARNING_TIMEOUT; ++ PMD_DRV_LOG(ERR, "TXGBE link thread not complete too long time!"); ++ } ++ } ++ ++ return 1; ++} ++ ++static uint32_t ++txgbe_dev_setup_link_thread_handler(void *param) ++{ ++ struct rte_eth_dev *dev = (struct rte_eth_dev *)param; ++ struct txgbe_adapter *ad = TXGBE_DEV_ADAPTER(dev); ++ ++ rte_thread_detach(rte_thread_self()); ++ txgbe_dev_setup_link_alarm_handler(dev); ++ __atomic_clear(&ad->link_thread_running, __ATOMIC_SEQ_CST); ++ return 0; ++} ++ + /* return 0 means link status changed, -1 means not changed */ + int + txgbe_dev_link_update_share(struct rte_eth_dev *dev, + int wait_to_complete) + { ++ struct txgbe_adapter *ad = TXGBE_DEV_ADAPTER(dev); + struct txgbe_hw *hw = TXGBE_DEV_HW(dev); + struct rte_eth_link link; + u32 link_speed = TXGBE_LINK_SPEED_UNKNOWN; +@@ -2731,10 +2824,24 @@ txgbe_dev_link_update_share(struct rte_eth_dev *dev, + if ((hw->subsystem_device_id & 0xFF) == + TXGBE_DEV_ID_KR_KX_KX4) { + hw->mac.bp_down_event(hw); +- } else if (hw->phy.media_type == txgbe_media_type_fiber) { +- intr->flags |= TXGBE_FLAG_NEED_LINK_CONFIG; +- rte_eal_alarm_set(10, +- txgbe_dev_setup_link_alarm_handler, dev); ++ } else if (hw->phy.media_type == txgbe_media_type_fiber && ++ dev->data->dev_conf.intr_conf.lsc != 0) { ++ txgbe_dev_wait_setup_link_complete(dev, 0); ++ if (!__atomic_test_and_set(&ad->link_thread_running, __ATOMIC_SEQ_CST)) { ++ /* To avoid race condition between threads, set ++ * the TXGBE_FLAG_NEED_LINK_CONFIG flag only ++ * when there is no link thread running. ++ */ ++ intr->flags |= TXGBE_FLAG_NEED_LINK_CONFIG; ++ if (rte_thread_create(&ad->link_thread_tid, NULL, ++ txgbe_dev_setup_link_thread_handler, dev) < 0) { ++ PMD_DRV_LOG(ERR, "Create link thread failed!"); ++ __atomic_clear(&ad->link_thread_running, __ATOMIC_SEQ_CST); ++ } ++ } else { ++ PMD_DRV_LOG(ERR, ++ "Other link thread is running now!"); ++ } + } + return rte_eth_linkstatus_set(dev, &link); + } else if (!hw->dev_start) { +@@ -2773,6 +2880,11 @@ txgbe_dev_link_update_share(struct rte_eth_dev *dev, + break; + } + ++ /* Re configure MAC RX */ ++ if (hw->mac.type == txgbe_mac_raptor) ++ wr32m(hw, TXGBE_MACRXFLT, TXGBE_MACRXFLT_PROMISC, ++ TXGBE_MACRXFLT_PROMISC); ++ + return rte_eth_linkstatus_set(dev, &link); + } + +@@ -2949,9 +3061,6 @@ txgbe_dev_interrupt_get_status(struct rte_eth_dev *dev, + rte_intr_type_get(intr_handle) != RTE_INTR_HANDLE_VFIO_MSIX) + wr32(hw, TXGBE_PX_INTA, 1); + +- /* clear all cause mask */ +- txgbe_disable_intr(hw); +- + /* read-on-clear nic registers here */ + eicr = ((u32 *)hw->isb_mem)[TXGBE_ISB_MISC]; + PMD_DRV_LOG(DEBUG, "eicr %x", eicr); +@@ -2974,6 +3083,8 @@ txgbe_dev_interrupt_get_status(struct rte_eth_dev *dev, + if (eicr & TXGBE_ICRMISC_GPIO) + intr->flags |= TXGBE_FLAG_PHY_INTERRUPT; + ++ ++ ((u32 *)hw->isb_mem)[TXGBE_ISB_MISC] = 0; + return 0; + } + +@@ -3143,7 +3254,8 @@ txgbe_dev_interrupt_delayed_handler(void *param) + } + + /* restore original mask */ +- intr->mask_misc |= TXGBE_ICRMISC_LSC; ++ if (dev->data->dev_conf.intr_conf.lsc == 1) ++ intr->mask_misc |= TXGBE_ICRMISC_LSC; + + intr->mask = intr->mask_orig; + intr->mask_orig = 0; +diff --git a/dpdk/drivers/net/txgbe/txgbe_ethdev.h b/dpdk/drivers/net/txgbe/txgbe_ethdev.h +index 6a18865e23..545ce4c9e1 100644 +--- a/dpdk/drivers/net/txgbe/txgbe_ethdev.h ++++ b/dpdk/drivers/net/txgbe/txgbe_ethdev.h +@@ -40,6 +40,7 @@ + /*Default value of Max Rx Queue*/ + #define TXGBE_MAX_RX_QUEUE_NUM 128 + #define TXGBE_VMDQ_DCB_NB_QUEUES TXGBE_MAX_RX_QUEUE_NUM ++#define TXGBE_NONE_MODE_TX_NB_QUEUES 64 + + #ifndef NBBY + #define NBBY 8 /* number of bits in a byte */ +@@ -369,6 +370,9 @@ struct txgbe_adapter { + + /* For RSS reta table update */ + uint8_t rss_reta_updated; ++ ++ uint32_t link_thread_running; ++ rte_thread_t link_thread_tid; + }; + + #define TXGBE_DEV_ADAPTER(dev) \ +@@ -560,6 +564,9 @@ void txgbe_configure_dcb(struct rte_eth_dev *dev); + int + txgbe_dev_link_update_share(struct rte_eth_dev *dev, + int wait_to_complete); ++int ++txgbe_dev_wait_setup_link_complete(struct rte_eth_dev *dev, ++ uint32_t timeout_ms); + int txgbe_pf_host_init(struct rte_eth_dev *eth_dev); + + void txgbe_pf_host_uninit(struct rte_eth_dev *eth_dev); +diff --git a/dpdk/drivers/net/txgbe/txgbe_ethdev_vf.c b/dpdk/drivers/net/txgbe/txgbe_ethdev_vf.c +index 3b1f7c913b..f1341fbf7e 100644 +--- a/dpdk/drivers/net/txgbe/txgbe_ethdev_vf.c ++++ b/dpdk/drivers/net/txgbe/txgbe_ethdev_vf.c +@@ -165,6 +165,7 @@ eth_txgbevf_dev_init(struct rte_eth_dev *eth_dev) + { + int err; + uint32_t tc, tcs; ++ struct txgbe_adapter *ad = eth_dev->data->dev_private; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + struct rte_intr_handle *intr_handle = pci_dev->intr_handle; + struct txgbe_hw *hw = TXGBE_DEV_HW(eth_dev); +@@ -205,6 +206,7 @@ eth_txgbevf_dev_init(struct rte_eth_dev *eth_dev) + return 0; + } + ++ __atomic_clear(&ad->link_thread_running, __ATOMIC_SEQ_CST); + rte_eth_copy_pci_info(eth_dev, pci_dev); + + hw->device_id = pci_dev->id.device_id; +@@ -618,7 +620,7 @@ txgbevf_dev_start(struct rte_eth_dev *dev) + PMD_INIT_FUNC_TRACE(); + + /* Stop the link setup handler before resetting the HW. */ +- rte_eal_alarm_cancel(txgbe_dev_setup_link_alarm_handler, dev); ++ txgbe_dev_wait_setup_link_complete(dev, 0); + + err = hw->mac.reset_hw(hw); + if (err) { +@@ -720,7 +722,7 @@ txgbevf_dev_stop(struct rte_eth_dev *dev) + + PMD_INIT_FUNC_TRACE(); + +- rte_eal_alarm_cancel(txgbe_dev_setup_link_alarm_handler, dev); ++ txgbe_dev_wait_setup_link_complete(dev, 0); + + txgbevf_intr_disable(dev); + +diff --git a/dpdk/drivers/net/txgbe/txgbe_ptypes.c b/dpdk/drivers/net/txgbe/txgbe_ptypes.c +index e1299d7363..c444d5d3f1 100644 +--- a/dpdk/drivers/net/txgbe/txgbe_ptypes.c ++++ b/dpdk/drivers/net/txgbe/txgbe_ptypes.c +@@ -320,8 +320,6 @@ txgbe_encode_ptype_tunnel(u32 ptype) + ptid |= TXGBE_PTID_TUN_EI; + break; + case RTE_PTYPE_TUNNEL_GRE: +- ptid |= TXGBE_PTID_TUN_EIG; +- break; + case RTE_PTYPE_TUNNEL_VXLAN: + case RTE_PTYPE_TUNNEL_VXLAN_GPE: + case RTE_PTYPE_TUNNEL_NVGRE: +@@ -332,20 +330,6 @@ txgbe_encode_ptype_tunnel(u32 ptype) + return ptid; + } + +- switch (ptype & RTE_PTYPE_INNER_L2_MASK) { +- case RTE_PTYPE_INNER_L2_ETHER: +- ptid |= TXGBE_PTID_TUN_EIGM; +- break; +- case RTE_PTYPE_INNER_L2_ETHER_VLAN: +- ptid |= TXGBE_PTID_TUN_EIGMV; +- break; +- case RTE_PTYPE_INNER_L2_ETHER_QINQ: +- ptid |= TXGBE_PTID_TUN_EIGMV; +- break; +- default: +- break; +- } +- + switch (ptype & RTE_PTYPE_INNER_L3_MASK) { + case RTE_PTYPE_INNER_L3_IPV4: + case RTE_PTYPE_INNER_L3_IPV4_EXT: +diff --git a/dpdk/drivers/net/txgbe/txgbe_ptypes.h b/dpdk/drivers/net/txgbe/txgbe_ptypes.h +index fa6c347d53..6fa8147f05 100644 +--- a/dpdk/drivers/net/txgbe/txgbe_ptypes.h ++++ b/dpdk/drivers/net/txgbe/txgbe_ptypes.h +@@ -348,4 +348,9 @@ struct txgbe_nvgrehdr { + __be32 tni; + }; + ++struct txgbe_grehdr { ++ __be16 flags; ++ __be16 proto; ++}; ++ + #endif /* _TXGBE_PTYPE_H_ */ +diff --git a/dpdk/drivers/net/txgbe/txgbe_rxtx.c b/dpdk/drivers/net/txgbe/txgbe_rxtx.c +index ac1bba08a3..24fc34d3c4 100644 +--- a/dpdk/drivers/net/txgbe/txgbe_rxtx.c ++++ b/dpdk/drivers/net/txgbe/txgbe_rxtx.c +@@ -516,20 +516,21 @@ tx_desc_ol_flags_to_cmdtype(uint64_t ol_flags) + return cmdtype; + } + +-static inline uint8_t +-tx_desc_ol_flags_to_ptid(uint64_t oflags, uint32_t ptype) ++static inline uint32_t ++tx_desc_ol_flags_to_ptype(uint64_t oflags) + { ++ uint32_t ptype; + bool tun; + +- if (ptype) +- return txgbe_encode_ptype(ptype); +- + /* Only support flags in TXGBE_TX_OFFLOAD_MASK */ + tun = !!(oflags & RTE_MBUF_F_TX_TUNNEL_MASK); + + /* L2 level */ + ptype = RTE_PTYPE_L2_ETHER; + if (oflags & RTE_MBUF_F_TX_VLAN) ++ ptype |= (tun ? RTE_PTYPE_INNER_L2_ETHER_VLAN : RTE_PTYPE_L2_ETHER_VLAN); ++ ++ if (oflags & RTE_MBUF_F_TX_QINQ) /* tunnel + QINQ is not supported */ + ptype |= RTE_PTYPE_L2_ETHER_VLAN; + + /* L3 level */ +@@ -571,7 +572,6 @@ tx_desc_ol_flags_to_ptid(uint64_t oflags, uint32_t ptype) + ptype |= RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4 | + RTE_PTYPE_TUNNEL_GRE; +- ptype |= RTE_PTYPE_INNER_L2_ETHER; + break; + case RTE_MBUF_F_TX_TUNNEL_GENEVE: + ptype |= RTE_PTYPE_L2_ETHER | +@@ -587,6 +587,16 @@ tx_desc_ol_flags_to_ptid(uint64_t oflags, uint32_t ptype) + break; + } + ++ return ptype; ++} ++ ++static inline uint8_t ++tx_desc_ol_flags_to_ptid(uint64_t oflags) ++{ ++ uint32_t ptype; ++ ++ ptype = tx_desc_ol_flags_to_ptype(oflags); ++ + return txgbe_encode_ptype(ptype); + } + +@@ -694,22 +704,24 @@ txgbe_get_tun_len(struct rte_mbuf *mbuf) + static inline uint8_t + txgbe_parse_tun_ptid(struct rte_mbuf *tx_pkt) + { +- uint64_t l2_none, l2_mac, l2_mac_vlan; ++ uint64_t l2_vxlan, l2_vxlan_mac, l2_vxlan_mac_vlan; ++ uint64_t l2_gre, l2_gre_mac, l2_gre_mac_vlan; + uint8_t ptid = 0; + +- if ((tx_pkt->ol_flags & (RTE_MBUF_F_TX_TUNNEL_VXLAN | +- RTE_MBUF_F_TX_TUNNEL_VXLAN_GPE)) == 0) +- return ptid; ++ l2_vxlan = sizeof(struct txgbe_udphdr) + sizeof(struct txgbe_vxlanhdr); ++ l2_vxlan_mac = l2_vxlan + sizeof(struct rte_ether_hdr); ++ l2_vxlan_mac_vlan = l2_vxlan_mac + sizeof(struct rte_vlan_hdr); + +- l2_none = sizeof(struct txgbe_udphdr) + sizeof(struct txgbe_vxlanhdr); +- l2_mac = l2_none + sizeof(struct rte_ether_hdr); +- l2_mac_vlan = l2_mac + sizeof(struct rte_vlan_hdr); ++ l2_gre = sizeof(struct txgbe_grehdr); ++ l2_gre_mac = l2_gre + sizeof(struct rte_ether_hdr); ++ l2_gre_mac_vlan = l2_gre_mac + sizeof(struct rte_vlan_hdr); + +- if (tx_pkt->l2_len == l2_none) ++ if (tx_pkt->l2_len == l2_vxlan || tx_pkt->l2_len == l2_gre) + ptid = TXGBE_PTID_TUN_EIG; +- else if (tx_pkt->l2_len == l2_mac) ++ else if (tx_pkt->l2_len == l2_vxlan_mac || tx_pkt->l2_len == l2_gre_mac) + ptid = TXGBE_PTID_TUN_EIGM; +- else if (tx_pkt->l2_len == l2_mac_vlan) ++ else if (tx_pkt->l2_len == l2_vxlan_mac_vlan || ++ tx_pkt->l2_len == l2_gre_mac_vlan) + ptid = TXGBE_PTID_TUN_EIGMV; + + return ptid; +@@ -776,8 +788,7 @@ txgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + /* If hardware offload required */ + tx_ol_req = ol_flags & TXGBE_TX_OFFLOAD_MASK; + if (tx_ol_req) { +- tx_offload.ptid = tx_desc_ol_flags_to_ptid(tx_ol_req, +- tx_pkt->packet_type); ++ tx_offload.ptid = tx_desc_ol_flags_to_ptid(tx_ol_req); + if (tx_offload.ptid & TXGBE_PTID_PKT_TUN) + tx_offload.ptid |= txgbe_parse_tun_ptid(tx_pkt); + tx_offload.l2_len = tx_pkt->l2_len; +@@ -1465,11 +1476,22 @@ txgbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + * of accesses cannot be reordered by the compiler. If they were + * not volatile, they could be reordered which could lead to + * using invalid descriptor fields when read from rxd. ++ * ++ * Meanwhile, to prevent the CPU from executing out of order, we ++ * need to use a proper memory barrier to ensure the memory ++ * ordering below. + */ + rxdp = &rx_ring[rx_id]; + staterr = rxdp->qw1.lo.status; + if (!(staterr & rte_cpu_to_le_32(TXGBE_RXD_STAT_DD))) + break; ++ ++ /* ++ * Use acquire fence to ensure that status_error which includes ++ * DD bit is loaded before loading of other descriptor words. ++ */ ++ rte_atomic_thread_fence(__ATOMIC_ACQUIRE); ++ + rxd = *rxdp; + + /* +@@ -1715,32 +1737,10 @@ txgbe_recv_pkts_lro(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts, + + next_desc: + /* +- * The code in this whole file uses the volatile pointer to +- * ensure the read ordering of the status and the rest of the +- * descriptor fields (on the compiler level only!!!). This is so +- * UGLY - why not to just use the compiler barrier instead? DPDK +- * even has the rte_compiler_barrier() for that. +- * +- * But most importantly this is just wrong because this doesn't +- * ensure memory ordering in a general case at all. For +- * instance, DPDK is supposed to work on Power CPUs where +- * compiler barrier may just not be enough! +- * +- * I tried to write only this function properly to have a +- * starting point (as a part of an LRO/RSC series) but the +- * compiler cursed at me when I tried to cast away the +- * "volatile" from rx_ring (yes, it's volatile too!!!). So, I'm +- * keeping it the way it is for now. +- * +- * The code in this file is broken in so many other places and +- * will just not work on a big endian CPU anyway therefore the +- * lines below will have to be revisited together with the rest +- * of the txgbe PMD. +- * +- * TODO: +- * - Get rid of "volatile" and let the compiler do its job. +- * - Use the proper memory barrier (rte_rmb()) to ensure the +- * memory ordering below. ++ * "Volatile" only prevents caching of the variable marked ++ * volatile. Most important, "volatile" cannot prevent the CPU ++ * from executing out of order. So, it is necessary to use a ++ * proper memory barrier to ensure the memory ordering below. + */ + rxdp = &rx_ring[rx_id]; + staterr = rte_le_to_cpu_32(rxdp->qw1.lo.status); +@@ -1748,6 +1748,12 @@ next_desc: + if (!(staterr & TXGBE_RXD_STAT_DD)) + break; + ++ /* ++ * Use acquire fence to ensure that status_error which includes ++ * DD bit is loaded before loading of other descriptor words. ++ */ ++ rte_atomic_thread_fence(__ATOMIC_ACQUIRE); ++ + rxd = *rxdp; + + PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u " +@@ -2795,6 +2801,8 @@ txgbe_dev_clear_queues(struct rte_eth_dev *dev) + txq->ops->release_mbufs(txq); + txq->ops->reset(txq); + } ++ ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; + } + + for (i = 0; i < dev->data->nb_rx_queues; i++) { +@@ -2804,6 +2812,8 @@ txgbe_dev_clear_queues(struct rte_eth_dev *dev) + txgbe_rx_queue_release_mbufs(rxq); + txgbe_reset_rx_queue(adapter, rxq); + } ++ ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; + } + } + +@@ -4382,7 +4392,7 @@ txgbe_dev_rx_init(struct rte_eth_dev *dev) + */ + buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) - + RTE_PKTMBUF_HEADROOM); +- buf_size = ROUND_UP(buf_size, 0x1 << 10); ++ buf_size = ROUND_DOWN(buf_size, 0x1 << 10); + srrctl |= TXGBE_RXCFG_PKTLEN(buf_size); + + wr32(hw, TXGBE_RXCFG(rxq->reg_idx), srrctl); +@@ -4994,6 +5004,8 @@ txgbevf_dev_rxtx_start(struct rte_eth_dev *dev) + } while (--poll_ms && !(txdctl & TXGBE_TXCFG_ENA)); + if (!poll_ms) + PMD_INIT_LOG(ERR, "Could not enable Tx Queue %d", i); ++ else ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; + } + for (i = 0; i < dev->data->nb_rx_queues; i++) { + rxq = dev->data->rx_queues[i]; +@@ -5008,6 +5020,8 @@ txgbevf_dev_rxtx_start(struct rte_eth_dev *dev) + } while (--poll_ms && !(rxdctl & TXGBE_RXCFG_ENA)); + if (!poll_ms) + PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", i); ++ else ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; + rte_wmb(); + wr32(hw, TXGBE_RXWP(i), rxq->nb_rx_desc - 1); + } +diff --git a/dpdk/drivers/net/vhost/rte_eth_vhost.c b/dpdk/drivers/net/vhost/rte_eth_vhost.c +index b152279fac..acba97dba3 100644 +--- a/dpdk/drivers/net/vhost/rte_eth_vhost.c ++++ b/dpdk/drivers/net/vhost/rte_eth_vhost.c +@@ -78,8 +78,9 @@ struct vhost_queue { + uint16_t port; + uint16_t virtqueue_id; + struct vhost_stats stats; +- int intr_enable; + rte_spinlock_t intr_lock; ++ struct epoll_event ev; ++ int kickfd; + }; + + struct pmd_internal { +@@ -297,7 +298,7 @@ vhost_dev_csum_configure(struct rte_eth_dev *eth_dev) + if (internal->features & (1ULL << VIRTIO_NET_F_CSUM)) { + if (!(rxmode->offloads & + (RTE_ETH_RX_OFFLOAD_UDP_CKSUM | RTE_ETH_RX_OFFLOAD_TCP_CKSUM))) { +- VHOST_LOG(NOTICE, "Rx csum will be done in SW, may impact performance."); ++ VHOST_LOG(NOTICE, "Rx csum will be done in SW, may impact performance.\n"); + internal->rx_sw_csum = true; + } + } +@@ -305,7 +306,7 @@ vhost_dev_csum_configure(struct rte_eth_dev *eth_dev) + if (!(internal->features & (1ULL << VIRTIO_NET_F_GUEST_CSUM))) { + if (txmode->offloads & + (RTE_ETH_TX_OFFLOAD_UDP_CKSUM | RTE_ETH_TX_OFFLOAD_TCP_CKSUM)) { +- VHOST_LOG(NOTICE, "Tx csum will be done in SW, may impact performance."); ++ VHOST_LOG(NOTICE, "Tx csum will be done in SW, may impact performance.\n"); + internal->tx_sw_csum = true; + } + } +@@ -545,115 +546,68 @@ find_internal_resource(char *ifname) + return list; + } + +-static int ++static void + eth_vhost_update_intr(struct rte_eth_dev *eth_dev, uint16_t rxq_idx) + { +- struct rte_intr_handle *handle = eth_dev->intr_handle; +- struct rte_epoll_event rev, *elist; +- int epfd, ret; +- +- if (handle == NULL) +- return 0; +- +- elist = rte_intr_elist_index_get(handle, rxq_idx); +- if (rte_intr_efds_index_get(handle, rxq_idx) == elist->fd) +- return 0; +- +- VHOST_LOG(INFO, "kickfd for rxq-%d was changed, updating handler.\n", +- rxq_idx); ++ struct rte_vhost_vring vring; ++ struct vhost_queue *vq; + +- if (elist->fd != -1) +- VHOST_LOG(ERR, "Unexpected previous kickfd value (Got %d, expected -1).\n", +- elist->fd); ++ vq = eth_dev->data->rx_queues[rxq_idx]; ++ if (vq == NULL || vq->vid < 0) ++ return; + +- /* +- * First remove invalid epoll event, and then install +- * the new one. May be solved with a proper API in the +- * future. +- */ +- epfd = elist->epfd; +- rev = *elist; +- ret = rte_epoll_ctl(epfd, EPOLL_CTL_DEL, rev.fd, +- elist); +- if (ret) { +- VHOST_LOG(ERR, "Delete epoll event failed.\n"); +- return ret; ++ if (rte_vhost_get_vhost_vring(vq->vid, (rxq_idx << 1) + 1, &vring) < 0) { ++ VHOST_LOG(DEBUG, "Failed to get rxq-%d's vring, skip!\n", rxq_idx); ++ return; + } + +- rev.fd = rte_intr_efds_index_get(handle, rxq_idx); +- if (rte_intr_elist_index_set(handle, rxq_idx, rev)) +- return -rte_errno; ++ rte_spinlock_lock(&vq->intr_lock); + +- elist = rte_intr_elist_index_get(handle, rxq_idx); +- ret = rte_epoll_ctl(epfd, EPOLL_CTL_ADD, rev.fd, elist); +- if (ret) { +- VHOST_LOG(ERR, "Add epoll event failed.\n"); +- return ret; ++ /* Remove previous kickfd from proxy epoll */ ++ if (vq->kickfd >= 0 && vq->kickfd != vring.kickfd) { ++ if (epoll_ctl(vq->ev.data.fd, EPOLL_CTL_DEL, vq->kickfd, &vq->ev) < 0) { ++ VHOST_LOG(DEBUG, "Failed to unregister %d from rxq-%d epoll: %s\n", ++ vq->kickfd, rxq_idx, strerror(errno)); ++ } else { ++ VHOST_LOG(DEBUG, "Unregistered %d from rxq-%d epoll\n", ++ vq->kickfd, rxq_idx); ++ } ++ vq->kickfd = -1; ++ } ++ ++ /* Add new one, if valid */ ++ if (vq->kickfd != vring.kickfd && vring.kickfd >= 0) { ++ if (epoll_ctl(vq->ev.data.fd, EPOLL_CTL_ADD, vring.kickfd, &vq->ev) < 0) { ++ VHOST_LOG(ERR, "Failed to register %d in rxq-%d epoll: %s\n", ++ vring.kickfd, rxq_idx, strerror(errno)); ++ } else { ++ vq->kickfd = vring.kickfd; ++ VHOST_LOG(DEBUG, "Registered %d in rxq-%d epoll\n", ++ vq->kickfd, rxq_idx); ++ } + } + +- return 0; ++ rte_spinlock_unlock(&vq->intr_lock); + } + + static int + eth_rxq_intr_enable(struct rte_eth_dev *dev, uint16_t qid) + { +- struct rte_vhost_vring vring; +- struct vhost_queue *vq; +- int old_intr_enable, ret = 0; +- +- vq = dev->data->rx_queues[qid]; +- if (!vq) { +- VHOST_LOG(ERR, "rxq%d is not setup yet\n", qid); +- return -1; +- } ++ struct vhost_queue *vq = dev->data->rx_queues[qid]; + +- rte_spinlock_lock(&vq->intr_lock); +- old_intr_enable = vq->intr_enable; +- vq->intr_enable = 1; +- ret = eth_vhost_update_intr(dev, qid); +- rte_spinlock_unlock(&vq->intr_lock); +- +- if (ret < 0) { +- VHOST_LOG(ERR, "Failed to update rxq%d's intr\n", qid); +- vq->intr_enable = old_intr_enable; +- return ret; +- } ++ if (vq->vid >= 0) ++ rte_vhost_enable_guest_notification(vq->vid, (qid << 1) + 1, 1); + +- ret = rte_vhost_get_vhost_vring(vq->vid, (qid << 1) + 1, &vring); +- if (ret < 0) { +- VHOST_LOG(ERR, "Failed to get rxq%d's vring\n", qid); +- return ret; +- } +- VHOST_LOG(INFO, "Enable interrupt for rxq%d\n", qid); +- rte_vhost_enable_guest_notification(vq->vid, (qid << 1) + 1, 1); +- rte_wmb(); +- +- return ret; ++ return 0; + } + + static int + eth_rxq_intr_disable(struct rte_eth_dev *dev, uint16_t qid) + { +- struct rte_vhost_vring vring; +- struct vhost_queue *vq; +- int ret = 0; ++ struct vhost_queue *vq = dev->data->rx_queues[qid]; + +- vq = dev->data->rx_queues[qid]; +- if (!vq) { +- VHOST_LOG(ERR, "rxq%d is not setup yet\n", qid); +- return -1; +- } +- +- ret = rte_vhost_get_vhost_vring(vq->vid, (qid << 1) + 1, &vring); +- if (ret < 0) { +- VHOST_LOG(ERR, "Failed to get rxq%d's vring", qid); +- return ret; +- } +- VHOST_LOG(INFO, "Disable interrupt for rxq%d\n", qid); +- rte_vhost_enable_guest_notification(vq->vid, (qid << 1) + 1, 0); +- rte_wmb(); +- +- vq->intr_enable = 0; ++ if (vq->vid >= 0) ++ rte_vhost_enable_guest_notification(vq->vid, (qid << 1) + 1, 0); + + return 0; + } +@@ -664,6 +618,14 @@ eth_vhost_uninstall_intr(struct rte_eth_dev *dev) + struct rte_intr_handle *intr_handle = dev->intr_handle; + + if (intr_handle != NULL) { ++ int i; ++ ++ for (i = 0; i < dev->data->nb_rx_queues; i++) { ++ int epoll_fd = rte_intr_efds_index_get(dev->intr_handle, i); ++ ++ if (epoll_fd >= 0) ++ close(epoll_fd); ++ } + rte_intr_vec_list_free(intr_handle); + rte_intr_instance_free(intr_handle); + } +@@ -673,72 +635,111 @@ eth_vhost_uninstall_intr(struct rte_eth_dev *dev) + static int + eth_vhost_install_intr(struct rte_eth_dev *dev) + { +- struct rte_vhost_vring vring; +- struct vhost_queue *vq; + int nb_rxq = dev->data->nb_rx_queues; +- int i; +- int ret; ++ struct vhost_queue *vq; + +- /* uninstall firstly if we are reconnecting */ +- if (dev->intr_handle != NULL) +- eth_vhost_uninstall_intr(dev); ++ int ret; ++ int i; + + dev->intr_handle = rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_PRIVATE); + if (dev->intr_handle == NULL) { + VHOST_LOG(ERR, "Fail to allocate intr_handle\n"); +- return -ENOMEM; ++ ret = -ENOMEM; ++ goto error; ++ } ++ if (rte_intr_efd_counter_size_set(dev->intr_handle, 0)) { ++ ret = -rte_errno; ++ goto error; + } +- if (rte_intr_efd_counter_size_set(dev->intr_handle, sizeof(uint64_t))) +- return -rte_errno; + + if (rte_intr_vec_list_alloc(dev->intr_handle, NULL, nb_rxq)) { +- VHOST_LOG(ERR, +- "Failed to allocate memory for interrupt vector\n"); +- rte_intr_instance_free(dev->intr_handle); +- return -ENOMEM; ++ VHOST_LOG(ERR, "Failed to allocate memory for interrupt vector\n"); ++ ret = -ENOMEM; ++ goto error; + } + +- +- VHOST_LOG(INFO, "Prepare intr vec\n"); ++ VHOST_LOG(DEBUG, "Prepare intr vec\n"); + for (i = 0; i < nb_rxq; i++) { +- if (rte_intr_vec_list_index_set(dev->intr_handle, i, RTE_INTR_VEC_RXTX_OFFSET + i)) +- return -rte_errno; +- if (rte_intr_efds_index_set(dev->intr_handle, i, -1)) +- return -rte_errno; +- vq = dev->data->rx_queues[i]; +- if (!vq) { +- VHOST_LOG(INFO, "rxq-%d not setup yet, skip!\n", i); +- continue; +- } ++ int epoll_fd = epoll_create1(0); + +- ret = rte_vhost_get_vhost_vring(vq->vid, (i << 1) + 1, &vring); +- if (ret < 0) { +- VHOST_LOG(INFO, +- "Failed to get rxq-%d's vring, skip!\n", i); +- continue; ++ if (epoll_fd < 0) { ++ VHOST_LOG(ERR, "Failed to create proxy epoll fd for rxq-%d\n", i); ++ ret = -errno; ++ goto error; + } + +- if (vring.kickfd < 0) { +- VHOST_LOG(INFO, +- "rxq-%d's kickfd is invalid, skip!\n", i); +- continue; ++ if (rte_intr_vec_list_index_set(dev->intr_handle, i, ++ RTE_INTR_VEC_RXTX_OFFSET + i) || ++ rte_intr_efds_index_set(dev->intr_handle, i, epoll_fd)) { ++ ret = -rte_errno; ++ close(epoll_fd); ++ goto error; + } + +- if (rte_intr_efds_index_set(dev->intr_handle, i, vring.kickfd)) +- continue; +- VHOST_LOG(INFO, "Installed intr vec for rxq-%d\n", i); ++ vq = dev->data->rx_queues[i]; ++ memset(&vq->ev, 0, sizeof(vq->ev)); ++ vq->ev.events = EPOLLIN; ++ vq->ev.data.fd = epoll_fd; + } + +- if (rte_intr_nb_efd_set(dev->intr_handle, nb_rxq)) +- return -rte_errno; ++ if (rte_intr_nb_efd_set(dev->intr_handle, nb_rxq)) { ++ ret = -rte_errno; ++ goto error; ++ } ++ if (rte_intr_max_intr_set(dev->intr_handle, nb_rxq + 1)) { ++ ret = -rte_errno; ++ goto error; ++ } ++ if (rte_intr_type_set(dev->intr_handle, RTE_INTR_HANDLE_VDEV)) { ++ ret = -rte_errno; ++ goto error; ++ } + +- if (rte_intr_max_intr_set(dev->intr_handle, nb_rxq + 1)) +- return -rte_errno; ++ return 0; + +- if (rte_intr_type_set(dev->intr_handle, RTE_INTR_HANDLE_VDEV)) +- return -rte_errno; ++error: ++ eth_vhost_uninstall_intr(dev); ++ return ret; ++} + +- return 0; ++static void ++eth_vhost_configure_intr(struct rte_eth_dev *dev) ++{ ++ int i; ++ ++ VHOST_LOG(DEBUG, "Configure intr vec\n"); ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ eth_vhost_update_intr(dev, i); ++} ++ ++static void ++eth_vhost_unconfigure_intr(struct rte_eth_dev *eth_dev) ++{ ++ struct vhost_queue *vq; ++ int i; ++ ++ VHOST_LOG(DEBUG, "Unconfigure intr vec\n"); ++ for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { ++ vq = eth_dev->data->rx_queues[i]; ++ if (vq == NULL || vq->vid < 0) ++ continue; ++ ++ rte_spinlock_lock(&vq->intr_lock); ++ ++ /* Remove previous kickfd from proxy epoll */ ++ if (vq->kickfd >= 0) { ++ if (epoll_ctl(vq->ev.data.fd, EPOLL_CTL_DEL, vq->kickfd, &vq->ev) < 0) { ++ VHOST_LOG(DEBUG, "Failed to unregister %d from rxq-%d epoll: %s\n", ++ vq->kickfd, i, strerror(errno)); ++ } else { ++ VHOST_LOG(DEBUG, "Unregistered %d from rxq-%d epoll\n", ++ vq->kickfd, i); ++ } ++ vq->kickfd = -1; ++ } ++ ++ rte_spinlock_unlock(&vq->intr_lock); ++ } + } + + static void +@@ -847,16 +848,8 @@ new_device(int vid) + internal->vid = vid; + if (rte_atomic32_read(&internal->started) == 1) { + queue_setup(eth_dev, internal); +- +- if (dev_conf->intr_conf.rxq) { +- if (eth_vhost_install_intr(eth_dev) < 0) { +- VHOST_LOG(INFO, +- "Failed to install interrupt handler."); +- return -1; +- } +- } +- } else { +- VHOST_LOG(INFO, "RX/TX queues not exist yet\n"); ++ if (dev_conf->intr_conf.rxq) ++ eth_vhost_configure_intr(eth_dev); + } + + for (i = 0; i < rte_vhost_get_vring_num(vid); i++) +@@ -900,6 +893,7 @@ destroy_device(int vid) + + rte_atomic32_set(&internal->dev_attached, 0); + update_queuing_status(eth_dev, true); ++ eth_vhost_unconfigure_intr(eth_dev); + + eth_dev->data->dev_link.link_status = RTE_ETH_LINK_DOWN; + +@@ -928,55 +922,10 @@ destroy_device(int vid) + rte_spinlock_unlock(&state->lock); + + VHOST_LOG(INFO, "Vhost device %d destroyed\n", vid); +- eth_vhost_uninstall_intr(eth_dev); + + rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, NULL); + } + +-static int +-vring_conf_update(int vid, struct rte_eth_dev *eth_dev, uint16_t vring_id) +-{ +- struct rte_eth_conf *dev_conf = ð_dev->data->dev_conf; +- struct pmd_internal *internal = eth_dev->data->dev_private; +- struct vhost_queue *vq; +- struct rte_vhost_vring vring; +- int rx_idx = vring_id % 2 ? (vring_id - 1) >> 1 : -1; +- int ret = 0; +- +- /* +- * The vring kickfd may be changed after the new device notification. +- * Update it when the vring state is updated. +- */ +- if (rx_idx >= 0 && rx_idx < eth_dev->data->nb_rx_queues && +- rte_atomic32_read(&internal->dev_attached) && +- rte_atomic32_read(&internal->started) && +- dev_conf->intr_conf.rxq) { +- ret = rte_vhost_get_vhost_vring(vid, vring_id, &vring); +- if (ret) { +- VHOST_LOG(ERR, "Failed to get vring %d information.\n", +- vring_id); +- return ret; +- } +- +- if (rte_intr_efds_index_set(eth_dev->intr_handle, rx_idx, +- vring.kickfd)) +- return -rte_errno; +- +- vq = eth_dev->data->rx_queues[rx_idx]; +- if (!vq) { +- VHOST_LOG(ERR, "rxq%d is not setup yet\n", rx_idx); +- return -1; +- } +- +- rte_spinlock_lock(&vq->intr_lock); +- if (vq->intr_enable) +- ret = eth_vhost_update_intr(eth_dev, rx_idx); +- rte_spinlock_unlock(&vq->intr_lock); +- } +- +- return ret; +-} +- + static int + vring_state_changed(int vid, uint16_t vring, int enable) + { +@@ -996,9 +945,8 @@ vring_state_changed(int vid, uint16_t vring, int enable) + /* won't be NULL */ + state = vring_states[eth_dev->data->port_id]; + +- if (enable && vring_conf_update(vid, eth_dev, vring)) +- VHOST_LOG(INFO, "Failed to update vring-%d configuration.\n", +- (int)vring); ++ if (eth_dev->data->dev_conf.intr_conf.rxq && vring % 2) ++ eth_vhost_update_intr(eth_dev, (vring - 1) >> 1); + + rte_spinlock_lock(&state->lock); + if (state->cur[vring] == enable) { +@@ -1184,22 +1132,27 @@ eth_dev_start(struct rte_eth_dev *eth_dev) + { + struct pmd_internal *internal = eth_dev->data->dev_private; + struct rte_eth_conf *dev_conf = ð_dev->data->dev_conf; ++ uint16_t i; + +- queue_setup(eth_dev, internal); +- +- if (rte_atomic32_read(&internal->dev_attached) == 1) { +- if (dev_conf->intr_conf.rxq) { +- if (eth_vhost_install_intr(eth_dev) < 0) { +- VHOST_LOG(INFO, +- "Failed to install interrupt handler."); +- return -1; +- } +- } ++ eth_vhost_uninstall_intr(eth_dev); ++ if (dev_conf->intr_conf.rxq && eth_vhost_install_intr(eth_dev) < 0) { ++ VHOST_LOG(ERR, "Failed to install interrupt handler.\n"); ++ return -1; + } + ++ queue_setup(eth_dev, internal); ++ if (rte_atomic32_read(&internal->dev_attached) == 1 && ++ dev_conf->intr_conf.rxq) ++ eth_vhost_configure_intr(eth_dev); ++ + rte_atomic32_set(&internal->started, 1); + update_queuing_status(eth_dev, false); + ++ for (i = 0; i < eth_dev->data->nb_rx_queues; i++) ++ eth_dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ for (i = 0; i < eth_dev->data->nb_tx_queues; i++) ++ eth_dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ + return 0; + } + +@@ -1207,11 +1160,17 @@ static int + eth_dev_stop(struct rte_eth_dev *dev) + { + struct pmd_internal *internal = dev->data->dev_private; ++ uint16_t i; + + dev->data->dev_started = 0; + rte_atomic32_set(&internal->started, 0); + update_queuing_status(dev, true); + ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ + return 0; + } + +@@ -1251,6 +1210,8 @@ eth_dev_close(struct rte_eth_dev *dev) + rte_free(internal->iface_name); + rte_free(internal); + ++ eth_vhost_uninstall_intr(dev); ++ + dev->data->dev_private = NULL; + + rte_free(vring_states[dev->data->port_id]); +@@ -1278,6 +1239,7 @@ eth_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id, + vq->mb_pool = mb_pool; + vq->virtqueue_id = rx_queue_id * VIRTIO_QNUM + VIRTIO_TXQ; + rte_spinlock_init(&vq->intr_lock); ++ vq->kickfd = -1; + dev->data->rx_queues[rx_queue_id] = vq; + + return 0; +@@ -1300,6 +1262,7 @@ eth_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id, + + vq->virtqueue_id = tx_queue_id * VIRTIO_QNUM + VIRTIO_RXQ; + rte_spinlock_init(&vq->intr_lock); ++ vq->kickfd = -1; + dev->data->tx_queues[tx_queue_id] = vq; + + return 0; +diff --git a/dpdk/drivers/net/virtio/virtio_ethdev.c b/dpdk/drivers/net/virtio/virtio_ethdev.c +index 760ba4e368..5e9ed47551 100644 +--- a/dpdk/drivers/net/virtio/virtio_ethdev.c ++++ b/dpdk/drivers/net/virtio/virtio_ethdev.c +@@ -1797,22 +1797,25 @@ static int + virtio_configure_intr(struct rte_eth_dev *dev) + { + struct virtio_hw *hw = dev->data->dev_private; ++ int ret; + + if (!rte_intr_cap_multiple(dev->intr_handle)) { + PMD_INIT_LOG(ERR, "Multiple intr vector not supported"); + return -ENOTSUP; + } + +- if (rte_intr_efd_enable(dev->intr_handle, dev->data->nb_rx_queues)) { ++ ret = rte_intr_efd_enable(dev->intr_handle, dev->data->nb_rx_queues); ++ if (ret < 0) { + PMD_INIT_LOG(ERR, "Fail to create eventfd"); +- return -1; ++ return ret; + } + +- if (rte_intr_vec_list_alloc(dev->intr_handle, "intr_vec", +- hw->max_queue_pairs)) { ++ ret = rte_intr_vec_list_alloc(dev->intr_handle, "intr_vec", ++ hw->max_queue_pairs); ++ if (ret < 0) { + PMD_INIT_LOG(ERR, "Failed to allocate %u rxq vectors", + hw->max_queue_pairs); +- return -ENOMEM; ++ return ret; + } + + if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) { +@@ -1833,12 +1836,13 @@ virtio_configure_intr(struct rte_eth_dev *dev) + */ + if (virtio_intr_enable(dev) < 0) { + PMD_DRV_LOG(ERR, "interrupt enable failed"); +- return -1; ++ return -EINVAL; + } + +- if (virtio_queues_bind_intr(dev) < 0) { ++ ret = virtio_queues_bind_intr(dev); ++ if (ret < 0) { + PMD_INIT_LOG(ERR, "Failed to bind queue/interrupt"); +- return -1; ++ return ret; + } + + return 0; +@@ -2161,7 +2165,7 @@ virtio_dev_rss_init(struct rte_eth_dev *eth_dev) + eth_dev->device->numa_node); + if (!hw->rss_key) { + PMD_INIT_LOG(ERR, "Failed to allocate RSS key"); +- return -1; ++ return -ENOMEM; + } + } + +@@ -2183,7 +2187,7 @@ virtio_dev_rss_init(struct rte_eth_dev *eth_dev) + eth_dev->device->numa_node); + if (!hw->rss_reta) { + PMD_INIT_LOG(ERR, "Failed to allocate RSS reta"); +- return -1; ++ return -ENOMEM; + } + + hw->rss_rx_queues = 0; +@@ -2223,7 +2227,7 @@ virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features) + /* Tell the host we've known how to drive the device. */ + virtio_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER); + if (virtio_ethdev_negotiate_features(hw, req_features) < 0) +- return -1; ++ return -EINVAL; + + hw->weak_barriers = !virtio_with_feature(hw, VIRTIO_F_ORDER_PLATFORM); + +@@ -2305,7 +2309,7 @@ virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features) + if (config->mtu < RTE_ETHER_MIN_MTU) { + PMD_INIT_LOG(ERR, "invalid max MTU value (%u)", + config->mtu); +- return -1; ++ return -EINVAL; + } + + hw->max_mtu = config->mtu; +@@ -2318,9 +2322,11 @@ virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features) + } + + hw->rss_hash_types = 0; +- if (virtio_with_feature(hw, VIRTIO_NET_F_RSS)) +- if (virtio_dev_rss_init(eth_dev)) +- return -1; ++ if (virtio_with_feature(hw, VIRTIO_NET_F_RSS)) { ++ ret = virtio_dev_rss_init(eth_dev); ++ if (ret < 0) ++ return ret; ++ } + + PMD_INIT_LOG(DEBUG, "config->max_virtqueue_pairs=%d", + config->max_virtqueue_pairs); +@@ -2342,13 +2348,22 @@ virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features) + return ret; + + if (eth_dev->data->dev_conf.intr_conf.rxq) { +- if (virtio_configure_intr(eth_dev) < 0) { ++ ret = virtio_configure_intr(eth_dev); ++ if (ret < 0) { + PMD_INIT_LOG(ERR, "failed to configure interrupt"); + virtio_free_queues(hw); +- return -1; ++ return ret; + } + } + ++ if (eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) ++ /* Enable vector (0) for Link State Interrupt */ ++ if (VIRTIO_OPS(hw)->set_config_irq(hw, 0) == ++ VIRTIO_MSI_NO_VECTOR) { ++ PMD_DRV_LOG(ERR, "failed to set config vector"); ++ return -EBUSY; ++ } ++ + virtio_reinit_complete(hw); + + return 0; +@@ -2457,6 +2472,9 @@ virtio_dev_speed_capa_get(uint32_t speed) + static int vectorized_check_handler(__rte_unused const char *key, + const char *value, void *ret_val) + { ++ if (value == NULL || ret_val == NULL) ++ return -EINVAL; ++ + if (strcmp(value, "1") == 0) + *(int *)ret_val = 1; + else +@@ -2665,14 +2683,6 @@ virtio_dev_configure(struct rte_eth_dev *dev) + hw->has_tx_offload = tx_offload_enabled(hw); + hw->has_rx_offload = rx_offload_enabled(hw); + +- if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) +- /* Enable vector (0) for Link State Interrupt */ +- if (VIRTIO_OPS(hw)->set_config_irq(hw, 0) == +- VIRTIO_MSI_NO_VECTOR) { +- PMD_DRV_LOG(ERR, "failed to set config vector"); +- return -EBUSY; +- } +- + if (virtio_with_packed_queue(hw)) { + #if defined(RTE_ARCH_X86_64) && defined(CC_AVX512_SUPPORT) + if ((hw->use_vec_rx || hw->use_vec_tx) && +@@ -2845,6 +2855,11 @@ virtio_dev_start(struct rte_eth_dev *dev) + set_rxtx_funcs(dev); + hw->started = 1; + ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ + /* Initialize Link state */ + virtio_dev_link_update(dev, 0); + +@@ -2934,6 +2949,7 @@ virtio_dev_stop(struct rte_eth_dev *dev) + struct virtio_hw *hw = dev->data->dev_private; + struct rte_eth_link link; + struct rte_eth_intr_conf *intr_conf = &dev->data->dev_conf.intr_conf; ++ uint16_t i; + + PMD_INIT_LOG(DEBUG, "stop"); + dev->data->dev_started = 0; +@@ -2961,6 +2977,11 @@ virtio_dev_stop(struct rte_eth_dev *dev) + out_unlock: + rte_spinlock_unlock(&hw->state_lock); + ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ + return 0; + } + +diff --git a/dpdk/drivers/net/virtio/virtio_pci.c b/dpdk/drivers/net/virtio/virtio_pci.c +index 9cf4d760b4..29eb739b04 100644 +--- a/dpdk/drivers/net/virtio/virtio_pci.c ++++ b/dpdk/drivers/net/virtio/virtio_pci.c +@@ -33,22 +33,6 @@ + + struct virtio_pci_internal virtio_pci_internal[RTE_MAX_ETHPORTS]; + +-static inline int +-check_vq_phys_addr_ok(struct virtqueue *vq) +-{ +- /* Virtio PCI device VIRTIO_PCI_QUEUE_PF register is 32bit, +- * and only accepts 32 bit page frame number. +- * Check if the allocated physical memory exceeds 16TB. +- */ +- if ((vq->vq_ring_mem + vq->vq_ring_size - 1) >> +- (VIRTIO_PCI_QUEUE_ADDR_SHIFT + 32)) { +- PMD_INIT_LOG(ERR, "vring address shouldn't be above 16TB!"); +- return 0; +- } +- +- return 1; +-} +- + #define PCI_MSIX_ENABLE 0x8000 + + static enum virtio_msix_status +@@ -273,8 +257,15 @@ legacy_setup_queue(struct virtio_hw *hw, struct virtqueue *vq) + { + uint32_t src; + +- if (!check_vq_phys_addr_ok(vq)) ++ /* Virtio PCI device VIRTIO_PCI_QUEUE_PFN register is 32bit, ++ * and only accepts 32 bit page frame number. ++ * Check if the allocated physical memory exceeds 16TB. ++ */ ++ if ((vq->vq_ring_mem + vq->vq_ring_size - 1) >> ++ (VIRTIO_PCI_QUEUE_ADDR_SHIFT + 32)) { ++ PMD_INIT_LOG(ERR, "vring address shouldn't be above 16TB!"); + return -1; ++ } + + rte_pci_ioport_write(VTPCI_IO(hw), &vq->vq_queue_index, 2, + VIRTIO_PCI_QUEUE_SEL); +@@ -476,9 +467,6 @@ modern_setup_queue(struct virtio_hw *hw, struct virtqueue *vq) + uint64_t desc_addr, avail_addr, used_addr; + uint16_t notify_off; + +- if (!check_vq_phys_addr_ok(vq)) +- return -1; +- + desc_addr = vq->vq_ring_mem; + avail_addr = desc_addr + vq->vq_nentries * sizeof(struct vring_desc); + used_addr = RTE_ALIGN_CEIL(avail_addr + offsetof(struct vring_avail, +diff --git a/dpdk/drivers/net/virtio/virtio_pci_ethdev.c b/dpdk/drivers/net/virtio/virtio_pci_ethdev.c +index abc63b0935..9b4b846f8a 100644 +--- a/dpdk/drivers/net/virtio/virtio_pci_ethdev.c ++++ b/dpdk/drivers/net/virtio/virtio_pci_ethdev.c +@@ -148,6 +148,9 @@ eth_virtio_pci_uninit(struct rte_eth_dev *eth_dev) + static int vdpa_check_handler(__rte_unused const char *key, + const char *value, void *ret_val) + { ++ if (value == NULL || ret_val == NULL) ++ return -EINVAL; ++ + if (strcmp(value, "1") == 0) + *(int *)ret_val = 1; + else +diff --git a/dpdk/drivers/net/virtio/virtio_rxtx.c b/dpdk/drivers/net/virtio/virtio_rxtx.c +index d9d40832e0..c3e686cf0c 100644 +--- a/dpdk/drivers/net/virtio/virtio_rxtx.c ++++ b/dpdk/drivers/net/virtio/virtio_rxtx.c +@@ -404,29 +404,36 @@ virtio_tso_fix_cksum(struct rte_mbuf *m) + if (likely(rte_pktmbuf_data_len(m) >= m->l2_len + m->l3_len + + m->l4_len)) { + struct rte_ipv4_hdr *iph; +- struct rte_ipv6_hdr *ip6h; + struct rte_tcp_hdr *th; +- uint16_t prev_cksum, new_cksum, ip_len, ip_paylen; ++ uint16_t prev_cksum, new_cksum; ++ uint32_t ip_paylen; + uint32_t tmp; + + iph = rte_pktmbuf_mtod_offset(m, + struct rte_ipv4_hdr *, m->l2_len); + th = RTE_PTR_ADD(iph, m->l3_len); ++ ++ /* ++ * Calculate IPv4 header checksum with current total length value ++ * (whatever it is) to have correct checksum after update on edits ++ * done by TSO. ++ */ + if ((iph->version_ihl >> 4) == 4) { + iph->hdr_checksum = 0; + iph->hdr_checksum = rte_ipv4_cksum(iph); +- ip_len = iph->total_length; +- ip_paylen = rte_cpu_to_be_16(rte_be_to_cpu_16(ip_len) - +- m->l3_len); +- } else { +- ip6h = (struct rte_ipv6_hdr *)iph; +- ip_paylen = ip6h->payload_len; + } + ++ /* ++ * Do not use IPv4 total length and IPv6 payload length fields to get ++ * TSO payload length since it could not fit into 16 bits. ++ */ ++ ip_paylen = rte_cpu_to_be_32(rte_pktmbuf_pkt_len(m) - m->l2_len - ++ m->l3_len); ++ + /* calculate the new phdr checksum not including ip_paylen */ + prev_cksum = th->cksum; + tmp = prev_cksum; +- tmp += ip_paylen; ++ tmp += (ip_paylen & 0xffff) + (ip_paylen >> 16); + tmp = (tmp & 0xffff) + (tmp >> 16); + new_cksum = tmp; + +diff --git a/dpdk/drivers/net/virtio/virtio_user/virtio_user_dev.c b/dpdk/drivers/net/virtio/virtio_user/virtio_user_dev.c +index 19599aa3f6..697a8dcd6b 100644 +--- a/dpdk/drivers/net/virtio/virtio_user/virtio_user_dev.c ++++ b/dpdk/drivers/net/virtio/virtio_user/virtio_user_dev.c +@@ -586,11 +586,7 @@ virtio_user_dev_init(struct virtio_user_dev *dev, char *path, int queues, + dev->frontend_features = 0; + dev->unsupported_features = 0; + dev->backend_type = backend_type; +- +- if (*ifname) { +- dev->ifname = *ifname; +- *ifname = NULL; +- } ++ dev->ifname = *ifname; + + if (virtio_user_dev_setup(dev) < 0) { + PMD_INIT_LOG(ERR, "(%s) backend set up fails", dev->path); +@@ -663,6 +659,7 @@ virtio_user_dev_init(struct virtio_user_dev *dev, char *path, int queues, + } + } + ++ *ifname = NULL; + return 0; + } + +diff --git a/dpdk/drivers/net/virtio/virtio_user_ethdev.c b/dpdk/drivers/net/virtio/virtio_user_ethdev.c +index d32abec327..78b1ed9ace 100644 +--- a/dpdk/drivers/net/virtio/virtio_user_ethdev.c ++++ b/dpdk/drivers/net/virtio/virtio_user_ethdev.c +@@ -90,10 +90,15 @@ virtio_user_set_status(struct virtio_hw *hw, uint8_t status) + if (status & VIRTIO_CONFIG_STATUS_FEATURES_OK && + ~old_status & VIRTIO_CONFIG_STATUS_FEATURES_OK) + virtio_user_dev_set_features(dev); +- if (status & VIRTIO_CONFIG_STATUS_DRIVER_OK) +- virtio_user_start_device(dev); +- else if (status == VIRTIO_CONFIG_STATUS_RESET) ++ ++ if (status & VIRTIO_CONFIG_STATUS_DRIVER_OK) { ++ if (virtio_user_start_device(dev)) { ++ virtio_user_dev_update_status(dev); ++ return; ++ } ++ } else if (status == VIRTIO_CONFIG_STATUS_RESET) { + virtio_user_reset(hw); ++ } + + virtio_user_dev_set_status(dev, status); + } +diff --git a/dpdk/drivers/net/virtio/virtqueue.h b/dpdk/drivers/net/virtio/virtqueue.h +index f5d8b40cad..5c9230cfe1 100644 +--- a/dpdk/drivers/net/virtio/virtqueue.h ++++ b/dpdk/drivers/net/virtio/virtqueue.h +@@ -771,6 +771,7 @@ virtqueue_enqueue_xmit_packed(struct virtnet_tx *txvq, struct rte_mbuf *cookie, + start_dp[idx].addr = txvq->virtio_net_hdr_mem + + RTE_PTR_DIFF(&txr[idx].tx_hdr, txr); + start_dp[idx].len = vq->hw->vtnet_hdr_size; ++ head_flags |= VRING_DESC_F_NEXT; + hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr; + idx++; + if (idx >= vq->vq_nentries) { +diff --git a/dpdk/drivers/net/vmxnet3/vmxnet3_ethdev.c b/dpdk/drivers/net/vmxnet3/vmxnet3_ethdev.c +index fd946dec5c..a0959b0c80 100644 +--- a/dpdk/drivers/net/vmxnet3/vmxnet3_ethdev.c ++++ b/dpdk/drivers/net/vmxnet3/vmxnet3_ethdev.c +@@ -957,6 +957,7 @@ vmxnet3_dev_start(struct rte_eth_dev *dev) + { + int ret; + struct vmxnet3_hw *hw = dev->data->dev_private; ++ uint16_t i; + + PMD_INIT_FUNC_TRACE(); + +@@ -1058,6 +1059,11 @@ vmxnet3_dev_start(struct rte_eth_dev *dev) + */ + __vmxnet3_dev_link_update(dev, 0); + ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; ++ + return VMXNET3_SUCCESS; + } + +@@ -1070,6 +1076,7 @@ vmxnet3_dev_stop(struct rte_eth_dev *dev) + struct rte_eth_link link; + struct vmxnet3_hw *hw = dev->data->dev_private; + struct rte_intr_handle *intr_handle = dev->intr_handle; ++ uint16_t i; + int ret; + + PMD_INIT_FUNC_TRACE(); +@@ -1125,6 +1132,11 @@ vmxnet3_dev_stop(struct rte_eth_dev *dev) + hw->adapter_stopped = 1; + dev->data->dev_started = 0; + ++ for (i = 0; i < dev->data->nb_rx_queues; i++) ++ dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ for (i = 0; i < dev->data->nb_tx_queues; i++) ++ dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; ++ + return 0; + } + +diff --git a/dpdk/drivers/net/vmxnet3/vmxnet3_rxtx.c b/dpdk/drivers/net/vmxnet3/vmxnet3_rxtx.c +index a875ffec07..14c6504505 100644 +--- a/dpdk/drivers/net/vmxnet3/vmxnet3_rxtx.c ++++ b/dpdk/drivers/net/vmxnet3/vmxnet3_rxtx.c +@@ -412,8 +412,8 @@ vmxnet3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + + nb_tx = 0; + while (nb_tx < nb_pkts) { +- Vmxnet3_GenericDesc *gdesc; +- vmxnet3_buf_info_t *tbi; ++ Vmxnet3_GenericDesc *gdesc = NULL; ++ vmxnet3_buf_info_t *tbi = NULL; + uint32_t first2fill, avail, dw2; + struct rte_mbuf *txm = tx_pkts[nb_tx]; + struct rte_mbuf *m_seg = txm; +@@ -457,18 +457,18 @@ vmxnet3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + continue; + } + ++ /* Skip empty packets */ ++ if (unlikely(rte_pktmbuf_pkt_len(txm) == 0)) { ++ txq->stats.drop_total++; ++ rte_pktmbuf_free(txm); ++ nb_tx++; ++ continue; ++ } ++ + if (txm->nb_segs == 1 && + rte_pktmbuf_pkt_len(txm) <= txq->txdata_desc_size) { + struct Vmxnet3_TxDataDesc *tdd; + +- /* Skip empty packets */ +- if (unlikely(rte_pktmbuf_pkt_len(txm) == 0)) { +- txq->stats.drop_total++; +- rte_pktmbuf_free(txm); +- nb_tx++; +- continue; +- } +- + tdd = (struct Vmxnet3_TxDataDesc *) + ((uint8 *)txq->data_ring.base + + txq->cmd_ring.next2fill * +@@ -481,6 +481,10 @@ vmxnet3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + dw2 = (txq->cmd_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT; + first2fill = txq->cmd_ring.next2fill; + do { ++ /* Skip empty segments */ ++ if (unlikely(m_seg->data_len == 0)) ++ continue; ++ + /* Remember the transmit buffer for cleanup */ + tbi = txq->cmd_ring.buf_info + txq->cmd_ring.next2fill; + +@@ -490,10 +494,6 @@ vmxnet3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + */ + gdesc = txq->cmd_ring.base + txq->cmd_ring.next2fill; + +- /* Skip empty segments */ +- if (unlikely(m_seg->data_len == 0)) +- continue; +- + if (copy_size) { + uint64 offset = + (uint64)txq->cmd_ring.next2fill * +@@ -514,6 +514,11 @@ vmxnet3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + /* use the right gen for non-SOP desc */ + dw2 = txq->cmd_ring.gen << VMXNET3_TXD_GEN_SHIFT; + } while ((m_seg = m_seg->next) != NULL); ++ /* We must have executed the complete preceding loop at least ++ * once without skipping an empty segment, as we can't have ++ * a packet with only empty segments. ++ * Thus, tbi and gdesc have been initialized. ++ */ + + /* set the last buf_info for the pkt */ + tbi->m = txm; +@@ -1311,11 +1316,18 @@ vmxnet3_dev_rxtx_init(struct rte_eth_dev *dev) + for (j = 0; j < VMXNET3_RX_CMDRING_SIZE; j++) { + /* Passing 0 as alloc_num will allocate full ring */ + ret = vmxnet3_post_rx_bufs(rxq, j); +- if (ret <= 0) { ++ ++ /* Zero number of descriptors in the configuration of the RX queue */ ++ if (ret == 0) { + PMD_INIT_LOG(ERR, +- "ERROR: Posting Rxq: %d buffers ring: %d", +- i, j); +- return -ret; ++ "Invalid configuration in Rx queue: %d, buffers ring: %d\n", ++ i, j); ++ return -EINVAL; ++ } ++ /* Return the error number */ ++ if (ret < 0) { ++ PMD_INIT_LOG(ERR, "Posting Rxq: %d buffers ring: %d", i, j); ++ return ret; + } + /* + * Updating device with the index:next2fill to fill the +diff --git a/dpdk/drivers/raw/ifpga/base/opae_hw_api.c b/dpdk/drivers/raw/ifpga/base/opae_hw_api.c +index 1117c3e160..6d48d227d6 100644 +--- a/dpdk/drivers/raw/ifpga/base/opae_hw_api.c ++++ b/dpdk/drivers/raw/ifpga/base/opae_hw_api.c +@@ -380,7 +380,7 @@ static pthread_mutex_t *opae_adapter_mutex_open(struct opae_adapter *adapter) + PROT_READ | PROT_WRITE, MAP_SHARED, + shm_id, 0); + adapter->lock = (pthread_mutex_t *)ptr; +- if (ptr) { ++ if (ptr != MAP_FAILED) { + dev_info(NULL, + "shared memory %s address is %p\n", + shm_name, ptr); +@@ -499,7 +499,7 @@ static void *opae_adapter_shm_alloc(struct opae_adapter *adapter) + adapter->shm.size = size; + adapter->shm.ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_SHARED, shm_id, 0); +- if (adapter->shm.ptr) { ++ if (adapter->shm.ptr != MAP_FAILED) { + dev_info(NULL, + "shared memory %s address is %p\n", + shm_name, adapter->shm.ptr); +diff --git a/dpdk/drivers/raw/ntb/ntb.c b/dpdk/drivers/raw/ntb/ntb.c +index 76e98fe515..0ed4c14592 100644 +--- a/dpdk/drivers/raw/ntb/ntb.c ++++ b/dpdk/drivers/raw/ntb/ntb.c +@@ -1045,6 +1045,11 @@ ntb_dev_close(struct rte_rawdev *dev) + hw->queue_pairs = 0; + + intr_handle = hw->pci_dev->intr_handle; ++ /* Disable interrupt only once */ ++ if (!rte_intr_nb_efd_get(intr_handle) && ++ !rte_intr_max_intr_get(intr_handle)) ++ return 0; ++ + /* Clean datapath event and vec mapping */ + rte_intr_efd_disable(intr_handle); + rte_intr_vec_list_free(intr_handle); +diff --git a/dpdk/drivers/raw/skeleton/skeleton_rawdev.c b/dpdk/drivers/raw/skeleton/skeleton_rawdev.c +index b2ca1cc5cd..6e99d35536 100644 +--- a/dpdk/drivers/raw/skeleton/skeleton_rawdev.c ++++ b/dpdk/drivers/raw/skeleton/skeleton_rawdev.c +@@ -428,7 +428,7 @@ static int skeleton_rawdev_enqueue_bufs(struct rte_rawdev *dev, + * help in complex implementation which require more information than + * just an integer - for example, a queue-pair. + */ +- q_id = *((int *)context); ++ q_id = *((uint16_t *)context); + + for (i = 0; i < count; i++) + queue_buf[q_id].bufs[i] = buffers[i]->buf_addr; +@@ -450,7 +450,7 @@ static int skeleton_rawdev_dequeue_bufs(struct rte_rawdev *dev, + * help in complex implementation which require more information than + * just an integer - for example, a queue-pair. + */ +- q_id = *((int *)context); ++ q_id = *((uint16_t *)context); + + for (i = 0; i < count; i++) + buffers[i]->buf_addr = queue_buf[q_id].bufs[i]; +@@ -664,6 +664,8 @@ skeldev_get_selftest(const char *key __rte_unused, + void *opaque) + { + int *flag = opaque; ++ if (value == NULL || opaque == NULL) ++ return -EINVAL; + *flag = atoi(value); + return 0; + } +diff --git a/dpdk/drivers/raw/skeleton/skeleton_rawdev_test.c b/dpdk/drivers/raw/skeleton/skeleton_rawdev_test.c +index ca15c49990..b7a7f623aa 100644 +--- a/dpdk/drivers/raw/skeleton/skeleton_rawdev_test.c ++++ b/dpdk/drivers/raw/skeleton/skeleton_rawdev_test.c +@@ -370,40 +370,34 @@ static int + test_rawdev_enqdeq(void) + { + int ret; +- unsigned int count = 1; + uint16_t queue_id = 0; +- struct rte_rawdev_buf buffers[1]; +- struct rte_rawdev_buf *deq_buffers = NULL; +- +- buffers[0].buf_addr = malloc(strlen(TEST_DEV_NAME) + 3); +- if (!buffers[0].buf_addr) +- goto cleanup; +- snprintf(buffers[0].buf_addr, strlen(TEST_DEV_NAME) + 2, "%s%d", ++ struct rte_rawdev_buf buffer; ++ struct rte_rawdev_buf *buffers[1]; ++ struct rte_rawdev_buf deq_buffer; ++ struct rte_rawdev_buf *deq_buffers[1]; ++ ++ buffers[0] = &buffer; ++ buffer.buf_addr = malloc(strlen(TEST_DEV_NAME) + 3); ++ if (!buffer.buf_addr) ++ return TEST_FAILED; ++ snprintf(buffer.buf_addr, strlen(TEST_DEV_NAME) + 2, "%s%d", + TEST_DEV_NAME, 0); + +- ret = rte_rawdev_enqueue_buffers(test_dev_id, +- (struct rte_rawdev_buf **)&buffers, +- count, &queue_id); +- RTE_TEST_ASSERT_EQUAL((unsigned int)ret, count, ++ ret = rte_rawdev_enqueue_buffers(test_dev_id, buffers, ++ RTE_DIM(buffers), &queue_id); ++ RTE_TEST_ASSERT_EQUAL((unsigned int)ret, RTE_DIM(buffers), + "Unable to enqueue buffers"); + +- deq_buffers = malloc(sizeof(struct rte_rawdev_buf) * count); +- if (!deq_buffers) +- goto cleanup; +- +- ret = rte_rawdev_dequeue_buffers(test_dev_id, +- (struct rte_rawdev_buf **)&deq_buffers, +- count, &queue_id); +- RTE_TEST_ASSERT_EQUAL((unsigned int)ret, count, ++ deq_buffers[0] = &deq_buffer; ++ ret = rte_rawdev_dequeue_buffers(test_dev_id, deq_buffers, ++ RTE_DIM(deq_buffers), &queue_id); ++ RTE_TEST_ASSERT_EQUAL((unsigned int)ret, RTE_DIM(buffers), + "Unable to dequeue buffers"); ++ RTE_TEST_ASSERT_EQUAL(deq_buffers[0]->buf_addr, buffers[0]->buf_addr, ++ "Did not retrieve expected object"); + +- free(deq_buffers); +- ++ free(buffer.buf_addr); + return TEST_SUCCESS; +-cleanup: +- free(buffers[0].buf_addr); +- +- return TEST_FAILED; + } + + static void skeldev_test_run(int (*setup)(void), +diff --git a/dpdk/drivers/regex/cn9k/meson.build b/dpdk/drivers/regex/cn9k/meson.build +index 06c906710c..19b2e70111 100644 +--- a/dpdk/drivers/regex/cn9k/meson.build ++++ b/dpdk/drivers/regex/cn9k/meson.build +@@ -8,10 +8,10 @@ if not is_linux or not dpdk_conf.get('RTE_ARCH_64') + subdir_done() + endif + +-lib = cc.find_library('librxp_compiler', required: false) ++lib = cc.find_library('rxp_compiler', required: false) + if lib.found() + ext_deps += lib +- ext_deps += cc.find_library('libstdc++', required: true) ++ ext_deps += cc.find_library('stdc++', required: true) + cflags += ['-DREE_COMPILER_SDK'] + endif + +diff --git a/dpdk/drivers/regex/mlx5/mlx5_regex.h b/dpdk/drivers/regex/mlx5/mlx5_regex.h +index b8554fd1cf..481f6fc59f 100644 +--- a/dpdk/drivers/regex/mlx5/mlx5_regex.h ++++ b/dpdk/drivers/regex/mlx5/mlx5_regex.h +@@ -37,7 +37,7 @@ struct mlx5_regex_qp { + struct mlx5_regex_hw_qp *qps; /* Pointer to qp array. */ + uint16_t nb_obj; /* Number of qp objects. */ + struct mlx5_regex_cq cq; /* CQ struct. */ +- uint32_t free_qps; ++ uint64_t free_qps; + struct mlx5_regex_job *jobs; + struct ibv_mr *metadata; + struct ibv_mr *outputs; +diff --git a/dpdk/drivers/regex/mlx5/mlx5_regex_fastpath.c b/dpdk/drivers/regex/mlx5/mlx5_regex_fastpath.c +index 143c7d7cdf..8e5f8c9c95 100644 +--- a/dpdk/drivers/regex/mlx5/mlx5_regex_fastpath.c ++++ b/dpdk/drivers/regex/mlx5/mlx5_regex_fastpath.c +@@ -211,8 +211,8 @@ send_doorbell(struct mlx5_regex_priv *priv, struct mlx5_regex_hw_qp *qp) + (MLX5_SEND_WQE_BB << (priv->has_umr ? 2 : 0)) + + (priv->has_umr ? MLX5_REGEX_UMR_WQE_SIZE : 0); + uint8_t *wqe = (uint8_t *)(uintptr_t)qp->qp_obj.wqes + wqe_offset; +- uint32_t actual_pi = (priv->has_umr ? (qp->db_pi * 4 + 3) : qp->db_pi) & +- MLX5_REGEX_MAX_WQE_INDEX; ++ uint32_t actual_pi = (priv->has_umr ? ((1 + qp->db_pi) * 4) : qp->db_pi) ++ & MLX5_REGEX_MAX_WQE_INDEX; + + /* Or the fm_ce_se instead of set, avoid the fence be cleared. */ + ((struct mlx5_wqe_ctrl_seg *)wqe)->fm_ce_se |= MLX5_WQE_CTRL_CQ_UPDATE; +@@ -417,7 +417,7 @@ mlx5_regexdev_enqueue_gga(struct rte_regexdev *dev, uint16_t qp_id, + return 0; + #endif + +- while ((hw_qpid = ffs(queue->free_qps))) { ++ while ((hw_qpid = ffsll(queue->free_qps))) { + hw_qpid--; /* ffs returns 1 for bit 0 */ + qp_obj = &queue->qps[hw_qpid]; + nb_desc = get_free(qp_obj, priv->has_umr); +@@ -426,7 +426,7 @@ mlx5_regexdev_enqueue_gga(struct rte_regexdev *dev, uint16_t qp_id, + if (nb_desc > nb_left) + nb_desc = nb_left; + else +- queue->free_qps &= ~(1 << hw_qpid); ++ queue->free_qps &= ~(1ULL << hw_qpid); + prep_regex_umr_wqe_set(priv, queue, qp_obj, ops, + nb_desc); + send_doorbell(priv, qp_obj); +@@ -456,7 +456,7 @@ mlx5_regexdev_enqueue(struct rte_regexdev *dev, uint16_t qp_id, + return 0; + #endif + +- while ((hw_qpid = ffs(queue->free_qps))) { ++ while ((hw_qpid = ffsll(queue->free_qps))) { + hw_qpid--; /* ffs returns 1 for bit 0 */ + qp_obj = &queue->qps[hw_qpid]; + while (get_free(qp_obj, priv->has_umr)) { +@@ -470,7 +470,7 @@ mlx5_regexdev_enqueue(struct rte_regexdev *dev, uint16_t qp_id, + goto out; + } + } +- queue->free_qps &= ~(1 << hw_qpid); ++ queue->free_qps &= ~(1ULL << hw_qpid); + send_doorbell(priv, qp_obj); + } + +@@ -603,7 +603,7 @@ mlx5_regexdev_dequeue(struct rte_regexdev *dev, uint16_t qp_id, + cq->ci = (cq->ci + 1) & 0xffffff; + rte_wmb(); + cq->cq_obj.db_rec[0] = rte_cpu_to_be_32(cq->ci); +- queue->free_qps |= (1 << hw_qpid); ++ queue->free_qps |= (1ULL << hw_qpid); + } + + out: +@@ -642,7 +642,7 @@ setup_qps(struct mlx5_regex_priv *priv, struct mlx5_regex_qp *queue) + (uintptr_t)job->output); + wqe += 64; + } +- queue->free_qps |= 1 << hw_qpid; ++ queue->free_qps |= 1ULL << hw_qpid; + } + } + +diff --git a/dpdk/drivers/vdpa/ifc/ifcvf_vdpa.c b/dpdk/drivers/vdpa/ifc/ifcvf_vdpa.c +index 49d68ad1b1..35520ea3ae 100644 +--- a/dpdk/drivers/vdpa/ifc/ifcvf_vdpa.c ++++ b/dpdk/drivers/vdpa/ifc/ifcvf_vdpa.c +@@ -1044,6 +1044,8 @@ ifcvf_sw_fallback_switchover(struct ifcvf_internal *internal) + + vdpa_disable_vfio_intr(internal); + ++ rte_atomic32_set(&internal->running, 0); ++ + ret = rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, false); + if (ret && ret != -ENOTSUP) + goto error; +@@ -1746,6 +1748,11 @@ ifcvf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, + goto error; + } + internal->sw_lm = sw_fallback_lm; ++ if (!internal->sw_lm && !internal->hw.lm_cfg) { ++ DRV_LOG(ERR, "Device %s does not support HW assist live migration, please enable sw-live-migration!", ++ pci_dev->name); ++ goto error; ++ } + + pthread_mutex_lock(&internal_list_lock); + TAILQ_INSERT_TAIL(&internal_list, list, next); +diff --git a/dpdk/drivers/vdpa/mlx5/mlx5_vdpa.c b/dpdk/drivers/vdpa/mlx5/mlx5_vdpa.c +index 026daa3f24..8bcbedd071 100644 +--- a/dpdk/drivers/vdpa/mlx5/mlx5_vdpa.c ++++ b/dpdk/drivers/vdpa/mlx5/mlx5_vdpa.c +@@ -282,6 +282,7 @@ _internal_mlx5_vdpa_dev_close(struct mlx5_vdpa_priv *priv, + int ret = 0; + int vid = priv->vid; + ++ mlx5_vdpa_virtq_unreg_intr_handle_all(priv); + mlx5_vdpa_cqe_event_unset(priv); + if (priv->state == MLX5_VDPA_STATE_CONFIGURED) { + ret |= mlx5_vdpa_lm_log(priv); +diff --git a/dpdk/drivers/vdpa/mlx5/mlx5_vdpa_cthread.c b/dpdk/drivers/vdpa/mlx5/mlx5_vdpa_cthread.c +index 6e6624e5a3..1d84e422d4 100644 +--- a/dpdk/drivers/vdpa/mlx5/mlx5_vdpa_cthread.c ++++ b/dpdk/drivers/vdpa/mlx5/mlx5_vdpa_cthread.c +@@ -190,7 +190,6 @@ mlx5_vdpa_c_thread_handle(void *arg) + pthread_mutex_unlock(&virtq->virtq_lock); + break; + case MLX5_VDPA_TASK_DEV_CLOSE_NOWAIT: +- mlx5_vdpa_virtq_unreg_intr_handle_all(priv); + pthread_mutex_lock(&priv->steer_update_lock); + mlx5_vdpa_steer_unset(priv); + pthread_mutex_unlock(&priv->steer_update_lock); +diff --git a/dpdk/examples/cmdline/parse_obj_list.h b/dpdk/examples/cmdline/parse_obj_list.h +index 6516d3e2c2..1223ac1e8b 100644 +--- a/dpdk/examples/cmdline/parse_obj_list.h ++++ b/dpdk/examples/cmdline/parse_obj_list.h +@@ -12,8 +12,9 @@ + + #include + #include ++#include + +-#define OBJ_NAME_LEN_MAX 64 ++#define OBJ_NAME_LEN_MAX sizeof(cmdline_fixed_string_t) + + struct object { + SLIST_ENTRY(object) next; +diff --git a/dpdk/examples/ethtool/ethtool-app/ethapp.c b/dpdk/examples/ethtool/ethtool-app/ethapp.c +index 4ea504ed6a..489cd4f515 100644 +--- a/dpdk/examples/ethtool/ethtool-app/ethapp.c ++++ b/dpdk/examples/ethtool/ethtool-app/ethapp.c +@@ -51,6 +51,13 @@ struct pcmd_intintint_params { + uint16_t rx; + }; + ++struct pcmd_pause_params { ++ cmdline_fixed_string_t cmd; ++ uint16_t port; ++ cmdline_fixed_string_t mode; ++ cmdline_fixed_string_t autoneg; ++ cmdline_fixed_string_t an_status; ++}; + + /* Parameter-less commands */ + cmdline_parse_token_string_t pcmd_quit_token_cmd = +@@ -118,12 +125,18 @@ cmdline_parse_token_num_t pcmd_intintint_token_rx = + + /* Pause commands */ + cmdline_parse_token_string_t pcmd_pause_token_cmd = +- TOKEN_STRING_INITIALIZER(struct pcmd_intstr_params, cmd, "pause"); ++ TOKEN_STRING_INITIALIZER(struct pcmd_pause_params, cmd, "pause"); + cmdline_parse_token_num_t pcmd_pause_token_port = +- TOKEN_NUM_INITIALIZER(struct pcmd_intstr_params, port, RTE_UINT16); +-cmdline_parse_token_string_t pcmd_pause_token_opt = +- TOKEN_STRING_INITIALIZER(struct pcmd_intstr_params, +- opt, "all#tx#rx#none"); ++ TOKEN_NUM_INITIALIZER(struct pcmd_pause_params, port, RTE_UINT16); ++cmdline_parse_token_string_t pcmd_pause_token_mode = ++ TOKEN_STRING_INITIALIZER(struct pcmd_pause_params, ++ mode, "full#tx#rx#none"); ++cmdline_parse_token_string_t pcmd_pause_token_autoneg = ++ TOKEN_STRING_INITIALIZER(struct pcmd_pause_params, ++ autoneg, "autoneg"); ++cmdline_parse_token_string_t pcmd_pause_token_an_status = ++ TOKEN_STRING_INITIALIZER(struct pcmd_pause_params, ++ an_status, "on#off"); + + /* VLAN commands */ + cmdline_parse_token_string_t pcmd_vlan_token_cmd = +@@ -350,13 +363,12 @@ pcmd_module_eeprom_callback(void *ptr_params, + fclose(fp_eeprom); + } + +- + static void + pcmd_pause_callback(void *ptr_params, + __rte_unused struct cmdline *ctx, + void *ptr_data) + { +- struct pcmd_intstr_params *params = ptr_params; ++ struct pcmd_pause_params *params = ptr_params; + struct ethtool_pauseparam info; + int stat; + +@@ -368,39 +380,38 @@ pcmd_pause_callback(void *ptr_params, + stat = rte_ethtool_get_pauseparam(params->port, &info); + } else { + memset(&info, 0, sizeof(info)); +- if (strcasecmp("all", params->opt) == 0) { ++ if (strcasecmp("full", params->mode) == 0) { + info.tx_pause = 1; + info.rx_pause = 1; +- } else if (strcasecmp("tx", params->opt) == 0) { ++ } else if (strcasecmp("tx", params->mode) == 0) { + info.tx_pause = 1; + info.rx_pause = 0; +- } else if (strcasecmp("rx", params->opt) == 0) { ++ } else if (strcasecmp("rx", params->mode) == 0) { + info.tx_pause = 0; + info.rx_pause = 1; + } else { + info.tx_pause = 0; + info.rx_pause = 0; + } +- /* Assume auto-negotiation wanted */ +- info.autoneg = 1; ++ ++ if (strcasecmp("on", params->an_status) == 0) ++ info.autoneg = 1; ++ else ++ info.autoneg = 0; ++ + stat = rte_ethtool_set_pauseparam(params->port, &info); + } + if (stat == 0) { +- if (info.rx_pause && info.tx_pause) +- printf("Port %i: Tx & Rx Paused\n", params->port); +- else if (info.rx_pause) +- printf("Port %i: Rx Paused\n", params->port); +- else if (info.tx_pause) +- printf("Port %i: Tx Paused\n", params->port); +- else +- printf("Port %i: Tx & Rx not paused\n", params->port); ++ printf("Pause parameters for Port %i:\n", params->port); ++ printf("Rx pause: %s\n", info.rx_pause ? "on" : "off"); ++ printf("Tx pause: %s\n", info.tx_pause ? "on" : "off"); ++ printf("Autoneg: %s\n", info.autoneg ? "on" : "off"); + } else if (stat == -ENOTSUP) + printf("Port %i: Operation not supported\n", params->port); + else + printf("Port %i: Error %i\n", params->port, stat); + } + +- + static void + pcmd_open_callback(__rte_unused void *ptr_params, + __rte_unused struct cmdline *ctx, +@@ -737,11 +748,13 @@ cmdline_parse_inst_t pcmd_pause = { + .f = pcmd_pause_callback, + .data = NULL, + .help_str = +- "pause \n Pause/unpause port", ++ "pause autoneg \n Pause/unpause port", + .tokens = { + (void *)&pcmd_pause_token_cmd, + (void *)&pcmd_pause_token_port, +- (void *)&pcmd_pause_token_opt, ++ (void *)&pcmd_pause_token_mode, ++ (void *)&pcmd_pause_token_autoneg, ++ (void *)&pcmd_pause_token_an_status, + NULL + }, + }; +diff --git a/dpdk/examples/fips_validation/Makefile b/dpdk/examples/fips_validation/Makefile +index bca6647f55..fbb778d57a 100644 +--- a/dpdk/examples/fips_validation/Makefile ++++ b/dpdk/examples/fips_validation/Makefile +@@ -15,6 +15,8 @@ SRCS-y += fips_validation_ccm.c + SRCS-y += fips_validation_sha.c + SRCS-y += fips_dev_self_test.c + SRCS-y += fips_validation_xts.c ++SRCS-y += fips_validation_rsa.c ++SRCS-y += fips_validation_ecdsa.c + SRCS-y += main.c + + PKGCONF ?= pkg-config +diff --git a/dpdk/examples/fips_validation/fips_validation.c b/dpdk/examples/fips_validation/fips_validation.c +index f7a6d821ea..d3b6099d73 100644 +--- a/dpdk/examples/fips_validation/fips_validation.c ++++ b/dpdk/examples/fips_validation/fips_validation.c +@@ -543,15 +543,28 @@ fips_test_parse_one_json_case(void) + + for (i = 0; info.callbacks[i].key != NULL; i++) { + param = json_object_get(json_info.json_test_case, info.callbacks[i].key); +- if (param) { +- strcpy(info.one_line_text, json_string_value(param)); +- ret = info.callbacks[i].cb( +- info.callbacks[i].key, info.one_line_text, +- info.callbacks[i].val +- ); +- if (ret < 0) +- return ret; ++ if (!param) ++ continue; ++ ++ switch (json_typeof(param)) { ++ case JSON_STRING: ++ snprintf(info.one_line_text, MAX_LINE_CHAR, "%s", ++ json_string_value(param)); ++ break; ++ ++ case JSON_INTEGER: ++ snprintf(info.one_line_text, MAX_LINE_CHAR, "%"JSON_INTEGER_FORMAT, ++ json_integer_value(param)); ++ break; ++ ++ default: ++ return -EINVAL; + } ++ ++ ret = info.callbacks[i].cb(info.callbacks[i].key, info.one_line_text, ++ info.callbacks[i].val); ++ if (ret < 0) ++ return ret; + } + + return 0; +diff --git a/dpdk/examples/fips_validation/fips_validation.h b/dpdk/examples/fips_validation/fips_validation.h +index 565a5cd36e..c4bb041785 100644 +--- a/dpdk/examples/fips_validation/fips_validation.h ++++ b/dpdk/examples/fips_validation/fips_validation.h +@@ -244,7 +244,7 @@ struct ecdsa_interim_data { + * Esp, in asym op, modulo bits decide char buffer size. + * max = (modulo / 4) + */ +-#define FIPS_TEST_JSON_BUF_LEN (4096 / 4) ++#define FIPS_TEST_JSON_BUF_LEN ((4096 / 4) + 1) + + struct fips_test_json_info { + /* Information used for reading from json */ +diff --git a/dpdk/examples/fips_validation/fips_validation_gcm.c b/dpdk/examples/fips_validation/fips_validation_gcm.c +index a80d8b3e4d..bf08d1b995 100644 +--- a/dpdk/examples/fips_validation/fips_validation_gcm.c ++++ b/dpdk/examples/fips_validation/fips_validation_gcm.c +@@ -79,7 +79,7 @@ parser_read_gcm_pt_len(const char *key, char *src, + if (ret < 0) + return ret; + +- if (vec.pt.len == 0) { ++ if (info.algo == FIPS_TEST_ALGO_AES_GMAC && vec.pt.len == 0) { + info.interim_info.gcm_data.is_gmac = 1; + test_ops.prepare_sym_op = prepare_auth_op; + test_ops.prepare_sym_xform = prepare_gmac_xform; +@@ -296,6 +296,7 @@ parse_test_gcm_json_writeback(struct fips_val *val) + tmp_val.val = val->val; + tmp_val.len = vec.pt.len; + ++ info.one_line_text[0] = '\0'; + writeback_hex_str("", info.one_line_text, &tmp_val); + ct = json_string(info.one_line_text); + json_object_set_new(json_info.json_write_case, CT_JSON_STR, ct); +@@ -326,6 +327,7 @@ parse_test_gcm_json_writeback(struct fips_val *val) + tmp_val.val = val->val; + tmp_val.len = vec.pt.len; + ++ info.one_line_text[0] = '\0'; + writeback_hex_str("", info.one_line_text, &tmp_val); + json_object_set_new(json_info.json_write_case, PT_JSON_STR, + json_string(info.one_line_text)); +@@ -334,12 +336,8 @@ parse_test_gcm_json_writeback(struct fips_val *val) + json_true()); + } + } else { +- if (!info.interim_info.gcm_data.is_gmac) +- json_object_set_new(json_info.json_write_case, PT_JSON_STR, +- json_string("")); +- else +- json_object_set_new(json_info.json_write_case, "testPassed", +- json_false()); ++ json_object_set_new(json_info.json_write_case, "testPassed", ++ json_false()); + } + } + +diff --git a/dpdk/examples/fips_validation/fips_validation_sha.c b/dpdk/examples/fips_validation/fips_validation_sha.c +index c5da2cc623..178ea492d3 100644 +--- a/dpdk/examples/fips_validation/fips_validation_sha.c ++++ b/dpdk/examples/fips_validation/fips_validation_sha.c +@@ -182,7 +182,7 @@ parse_test_sha_json_writeback(struct fips_val *val) + static int + parse_test_sha_mct_json_writeback(struct fips_val *val) + { +- json_t *tcId, *msg, *md, *resArr, *res; ++ json_t *tcId, *md, *resArr, *res; + struct fips_val val_local; + + tcId = json_object_get(json_info.json_test_case, "tcId"); +@@ -208,11 +208,7 @@ parse_test_sha_mct_json_writeback(struct fips_val *val) + + res = json_object(); + +- writeback_hex_str("", info.one_line_text, &val[1]); +- msg = json_string(info.one_line_text); +- json_object_set_new(res, "msg", msg); +- +- val_local.val = val[0].val + vec.pt.len; ++ val_local.val = val->val + vec.pt.len; + val_local.len = vec.cipher_auth.digest.len; + + writeback_hex_str("", info.one_line_text, &val_local); +diff --git a/dpdk/examples/fips_validation/fips_validation_xts.c b/dpdk/examples/fips_validation/fips_validation_xts.c +index 531e3c688e..530df78ab4 100644 +--- a/dpdk/examples/fips_validation/fips_validation_xts.c ++++ b/dpdk/examples/fips_validation/fips_validation_xts.c +@@ -34,6 +34,7 @@ + #define DATAUNITLEN_JSON_STR "dataUnitLen" + #define PAYLOADLEN_JSON_STR "payloadLen" + #define TWEAKVALUE_JSON_STR "tweakValue" ++#define SEQNUMBER_JSON_STR "sequenceNumber" + #define PT_JSON_STR "pt" + #define CT_JSON_STR "ct" + +@@ -95,14 +96,17 @@ parser_xts_read_keylen(const char *key, char *src, struct fips_val *val) + static int + parser_xts_read_tweakval(const char *key, char *src, struct fips_val *val) + { ++ char num_str[4] = {0}; + int ret; + +- if (info.interim_info.xts_data.tweak_mode == XTS_TWEAK_MODE_HEX) ++ if (info.interim_info.xts_data.tweak_mode == XTS_TWEAK_MODE_HEX) { + ret = parse_uint8_hex_str(key, src, val); +- else if (info.interim_info.xts_data.tweak_mode == XTS_TWEAK_MODE_NUMBER) +- ret = parser_read_uint32_bit_val(key, src, val); +- else ++ } else if (info.interim_info.xts_data.tweak_mode == XTS_TWEAK_MODE_NUMBER) { ++ snprintf(num_str, RTE_DIM(num_str), "%x", atoi(src)); ++ ret = parse_uint8_hex_str(key, num_str, val); ++ } else { + ret = -1; ++ } + + return ret; + } +@@ -122,6 +126,7 @@ struct fips_test_callback xts_interim_json_vectors[] = { + struct fips_test_callback xts_enc_json_vectors[] = { + {KEY_JSON_STR, parse_uint8_known_len_hex_str, &vec.cipher_auth.key}, + {TWEAKVALUE_JSON_STR, parser_xts_read_tweakval, &vec.iv}, ++ {SEQNUMBER_JSON_STR, parser_xts_read_tweakval, &vec.iv}, + {PT_JSON_STR, parse_uint8_hex_str, &vec.pt}, + {NULL, NULL, NULL} /**< end pointer */ + }; +diff --git a/dpdk/examples/fips_validation/main.c b/dpdk/examples/fips_validation/main.c +index 622f8b5a6e..cc68a1620b 100644 +--- a/dpdk/examples/fips_validation/main.c ++++ b/dpdk/examples/fips_validation/main.c +@@ -834,7 +834,7 @@ prepare_aead_op(void) + RTE_LOG(ERR, USER1, "Not enough memory\n"); + return -ENOMEM; + } +- env.digest_len = vec.cipher_auth.digest.len; ++ env.digest_len = vec.aead.digest.len; + + sym->aead.data.length = vec.pt.len; + sym->aead.digest.data = env.digest; +@@ -843,7 +843,7 @@ prepare_aead_op(void) + ret = prepare_data_mbufs(&vec.ct); + if (ret < 0) + return ret; +- ++ env.digest_len = vec.aead.digest.len; + sym->aead.data.length = vec.ct.len; + sym->aead.digest.data = vec.aead.digest.val; + sym->aead.digest.phys_addr = rte_malloc_virt2iova( +@@ -2268,8 +2268,7 @@ fips_mct_sha_test(void) + #define SHA_EXTERN_ITER 100 + #define SHA_INTERN_ITER 1000 + #define SHA_MD_BLOCK 3 +- /* val[0] is op result and other value is for parse_writeback callback */ +- struct fips_val val[2] = {{NULL, 0},}; ++ struct fips_val val = {NULL, 0}; + struct fips_val md[SHA_MD_BLOCK], msg; + int ret; + uint32_t i, j; +@@ -2328,7 +2327,7 @@ fips_mct_sha_test(void) + return ret; + } + +- ret = get_writeback_data(&val[0]); ++ ret = get_writeback_data(&val); + if (ret < 0) + return ret; + +@@ -2337,7 +2336,7 @@ fips_mct_sha_test(void) + memcpy(md[1].val, md[2].val, md[2].len); + md[1].len = md[2].len; + +- memcpy(md[2].val, (val[0].val + vec.pt.len), ++ memcpy(md[2].val, (val.val + vec.pt.len), + vec.cipher_auth.digest.len); + md[2].len = vec.cipher_auth.digest.len; + } +@@ -2348,9 +2347,7 @@ fips_mct_sha_test(void) + if (info.file_type != FIPS_TYPE_JSON) + fprintf(info.fp_wr, "COUNT = %u\n", j); + +- val[1].val = msg.val; +- val[1].len = msg.len; +- info.parse_writeback(val); ++ info.parse_writeback(&val); + + if (info.file_type != FIPS_TYPE_JSON) + fprintf(info.fp_wr, "\n"); +@@ -2361,7 +2358,7 @@ fips_mct_sha_test(void) + + rte_free(vec.pt.val); + +- free(val[0].val); ++ free(val.val); + free(msg.val); + + return 0; +@@ -2528,6 +2525,7 @@ error_one_case: + if (env.digest) { + rte_free(env.digest); + env.digest = NULL; ++ env.digest_len = 0; + } + rte_pktmbuf_free(env.mbuf); + +diff --git a/dpdk/examples/ip_pipeline/thread.c b/dpdk/examples/ip_pipeline/thread.c +index 82d5f87c38..9817657ca9 100644 +--- a/dpdk/examples/ip_pipeline/thread.c ++++ b/dpdk/examples/ip_pipeline/thread.c +@@ -430,7 +430,7 @@ thread_pipeline_disable(uint32_t thread_id, + static inline struct thread_msg_req * + thread_msg_recv(struct rte_ring *msgq_req) + { +- struct thread_msg_req *req; ++ struct thread_msg_req *req = NULL; + + int status = rte_ring_sc_dequeue(msgq_req, (void **) &req); + +diff --git a/dpdk/examples/ipsec-secgw/ipsec-secgw.c b/dpdk/examples/ipsec-secgw/ipsec-secgw.c +index a64a26c992..82a4916fb2 100644 +--- a/dpdk/examples/ipsec-secgw/ipsec-secgw.c ++++ b/dpdk/examples/ipsec-secgw/ipsec-secgw.c +@@ -99,10 +99,10 @@ uint32_t qp_desc_nb = 2048; + #define MTU_TO_FRAMELEN(x) ((x) + RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN) + + struct ethaddr_info ethaddr_tbl[RTE_MAX_ETHPORTS] = { +- { 0, ETHADDR(0x00, 0x16, 0x3e, 0x7e, 0x94, 0x9a) }, +- { 0, ETHADDR(0x00, 0x16, 0x3e, 0x22, 0xa1, 0xd9) }, +- { 0, ETHADDR(0x00, 0x16, 0x3e, 0x08, 0x69, 0x26) }, +- { 0, ETHADDR(0x00, 0x16, 0x3e, 0x49, 0x9e, 0xdd) } ++ { {{0}}, {{0x00, 0x16, 0x3e, 0x7e, 0x94, 0x9a}} }, ++ { {{0}}, {{0x00, 0x16, 0x3e, 0x22, 0xa1, 0xd9}} }, ++ { {{0}}, {{0x00, 0x16, 0x3e, 0x08, 0x69, 0x26}} }, ++ { {{0}}, {{0x00, 0x16, 0x3e, 0x49, 0x9e, 0xdd}} } + }; + + struct offloads tx_offloads; +@@ -1427,9 +1427,8 @@ add_dst_ethaddr(uint16_t port, const struct rte_ether_addr *addr) + if (port >= RTE_DIM(ethaddr_tbl)) + return -EINVAL; + +- ethaddr_tbl[port].dst = ETHADDR_TO_UINT64(addr); +- rte_ether_addr_copy((struct rte_ether_addr *)ðaddr_tbl[port].dst, +- (struct rte_ether_addr *)(val_eth + port)); ++ rte_ether_addr_copy(addr, ðaddr_tbl[port].dst); ++ rte_ether_addr_copy(addr, (struct rte_ether_addr *)(val_eth + port)); + return 0; + } + +@@ -1700,6 +1699,9 @@ cryptodevs_init(enum eh_pkt_transfer_mode mode) + + total_nb_qps += qp; + dev_conf.socket_id = rte_cryptodev_socket_id(cdev_id); ++ /* Use the first socket if SOCKET_ID_ANY is returned. */ ++ if (dev_conf.socket_id == SOCKET_ID_ANY) ++ dev_conf.socket_id = 0; + dev_conf.nb_queue_pairs = qp; + dev_conf.ff_disable = RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO; + +@@ -1907,11 +1909,12 @@ port_init(uint16_t portid, uint64_t req_rx_offloads, uint64_t req_tx_offloads, + "Error getting MAC address (port %u): %s\n", + portid, rte_strerror(-ret)); + +- ethaddr_tbl[portid].src = ETHADDR_TO_UINT64(ðaddr); ++ rte_ether_addr_copy(ðaddr, ðaddr_tbl[portid].src); + +- rte_ether_addr_copy((struct rte_ether_addr *)ðaddr_tbl[portid].dst, ++ rte_ether_addr_copy(ðaddr_tbl[portid].dst, + (struct rte_ether_addr *)(val_eth + portid)); +- rte_ether_addr_copy((struct rte_ether_addr *)ðaddr_tbl[portid].src, ++ ++ rte_ether_addr_copy(ðaddr_tbl[portid].src, + (struct rte_ether_addr *)(val_eth + portid) + 1); + + print_ethaddr("Address: ", ðaddr); +diff --git a/dpdk/examples/ipsec-secgw/ipsec-secgw.h b/dpdk/examples/ipsec-secgw/ipsec-secgw.h +index 0e0012d058..53665adf03 100644 +--- a/dpdk/examples/ipsec-secgw/ipsec-secgw.h ++++ b/dpdk/examples/ipsec-secgw/ipsec-secgw.h +@@ -84,7 +84,7 @@ struct ipsec_traffic_nb { + + /* port/source ethernet addr and destination ethernet addr */ + struct ethaddr_info { +- uint64_t src, dst; ++ struct rte_ether_addr src, dst; + }; + + struct ipsec_spd_stats { +diff --git a/dpdk/examples/ipsec-secgw/ipsec.h b/dpdk/examples/ipsec-secgw/ipsec.h +index 6bef2a7285..2890e6e267 100644 +--- a/dpdk/examples/ipsec-secgw/ipsec.h ++++ b/dpdk/examples/ipsec-secgw/ipsec.h +@@ -249,11 +249,18 @@ struct offloads { + + extern struct offloads tx_offloads; + ++/* ++ * This structure is used for the key in hash table. ++ * Padding is to force the struct to use 8 bytes, ++ * to ensure memory is not read past this structs boundary ++ * (hash key calculation reads 8 bytes if this struct is size 5 bytes). ++ */ + struct cdev_key { + uint16_t lcore_id; + uint8_t cipher_algo; + uint8_t auth_algo; + uint8_t aead_algo; ++ uint8_t padding[3]; /* padding to 8-byte size should be zeroed */ + }; + + struct socket_ctx { +diff --git a/dpdk/examples/ipsec-secgw/sa.c b/dpdk/examples/ipsec-secgw/sa.c +index 7da9444a7b..45cd29f18b 100644 +--- a/dpdk/examples/ipsec-secgw/sa.c ++++ b/dpdk/examples/ipsec-secgw/sa.c +@@ -1247,6 +1247,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[], + struct ipsec_sa *sa; + uint32_t i, idx; + uint16_t iv_length, aad_length; ++ uint16_t auth_iv_length = 0; + int inline_status; + int32_t rc; + struct rte_ipsec_session *ips; +@@ -1340,7 +1341,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[], + + /* AES_GMAC uses salt like AEAD algorithms */ + if (sa->auth_algo == RTE_CRYPTO_AUTH_AES_GMAC) +- iv_length = 12; ++ auth_iv_length = 12; + + if (inbound) { + sa_ctx->xf[idx].b.type = RTE_CRYPTO_SYM_XFORM_CIPHER; +@@ -1364,7 +1365,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[], + sa_ctx->xf[idx].a.auth.op = + RTE_CRYPTO_AUTH_OP_VERIFY; + sa_ctx->xf[idx].a.auth.iv.offset = IV_OFFSET; +- sa_ctx->xf[idx].a.auth.iv.length = iv_length; ++ sa_ctx->xf[idx].a.auth.iv.length = auth_iv_length; + + } else { /* outbound */ + sa_ctx->xf[idx].a.type = RTE_CRYPTO_SYM_XFORM_CIPHER; +@@ -1388,7 +1389,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[], + sa_ctx->xf[idx].b.auth.op = + RTE_CRYPTO_AUTH_OP_GENERATE; + sa_ctx->xf[idx].b.auth.iv.offset = IV_OFFSET; +- sa_ctx->xf[idx].b.auth.iv.length = iv_length; ++ sa_ctx->xf[idx].b.auth.iv.length = auth_iv_length; + + } + +@@ -1828,6 +1829,7 @@ sa_check_offloads(uint16_t port_id, uint64_t *rx_offloads, + + *rx_offloads = 0; + *tx_offloads = 0; ++ *hw_reassembly = 0; + + ret = rte_eth_dev_info_get(port_id, &dev_info); + if (ret != 0) +diff --git a/dpdk/examples/ipsec-secgw/test/common_defs.sh b/dpdk/examples/ipsec-secgw/test/common_defs.sh +index 3ef06bc761..6e04ffc1a6 100644 +--- a/dpdk/examples/ipsec-secgw/test/common_defs.sh ++++ b/dpdk/examples/ipsec-secgw/test/common_defs.sh +@@ -26,7 +26,7 @@ fi + + LOCAL_IFACE=dtap0 + +-LOCAL_MAC="00:64:74:61:70:30" ++LOCAL_MAC="02:64:74:61:70:30" + + REMOTE_IPV4=192.168.31.14 + LOCAL_IPV4=192.168.31.92 +diff --git a/dpdk/examples/l2fwd-cat/Makefile b/dpdk/examples/l2fwd-cat/Makefile +index 23a09550a4..d06053451a 100644 +--- a/dpdk/examples/l2fwd-cat/Makefile ++++ b/dpdk/examples/l2fwd-cat/Makefile +@@ -35,6 +35,7 @@ endif + endif + + CFLAGS += -DALLOW_EXPERIMENTAL_API ++CFLAGS += -D_GNU_SOURCE + + LDFLAGS += -lpqos + +diff --git a/dpdk/examples/l2fwd-event/l2fwd_event.c b/dpdk/examples/l2fwd-event/l2fwd_event.c +index 63450537fe..4b5a032e35 100644 +--- a/dpdk/examples/l2fwd-event/l2fwd_event.c ++++ b/dpdk/examples/l2fwd-event/l2fwd_event.c +@@ -284,7 +284,7 @@ l2fwd_event_loop_burst(struct l2fwd_resources *rsrc, + } + } + +- l2fwd_event_worker_cleanup(event_d_id, port_id, ev, nb_rx, nb_tx, 0); ++ l2fwd_event_worker_cleanup(event_d_id, port_id, ev, nb_tx, nb_rx, 0); + } + + static __rte_always_inline void +@@ -468,7 +468,7 @@ l2fwd_event_loop_vector(struct l2fwd_resources *rsrc, const uint32_t flags) + } + } + +- l2fwd_event_worker_cleanup(event_d_id, port_id, ev, nb_rx, nb_tx, 1); ++ l2fwd_event_worker_cleanup(event_d_id, port_id, ev, nb_tx, nb_rx, 1); + } + + static void __rte_noinline +diff --git a/dpdk/examples/l3fwd/l3fwd.h b/dpdk/examples/l3fwd/l3fwd.h +index ca1426a687..b55855c932 100644 +--- a/dpdk/examples/l3fwd/l3fwd.h ++++ b/dpdk/examples/l3fwd/l3fwd.h +@@ -55,7 +55,6 @@ + /* 32-bit has less address-space for hugepage memory, limit to 1M entries */ + #define L3FWD_HASH_ENTRIES (1024*1024*1) + #endif +-#define HASH_ENTRY_NUMBER_DEFAULT 16 + + struct parm_cfg { + const char *rule_ipv4_name; +diff --git a/dpdk/examples/l3fwd/l3fwd_fib.c b/dpdk/examples/l3fwd/l3fwd_fib.c +index edc0dd69b9..18398492ae 100644 +--- a/dpdk/examples/l3fwd/l3fwd_fib.c ++++ b/dpdk/examples/l3fwd/l3fwd_fib.c +@@ -359,10 +359,10 @@ fib_event_loop(struct l3fwd_event_resources *evt_rsrc, + nh = (uint16_t)hopsv4[ipv4_arr_assem++]; + else + nh = (uint16_t)hopsv6[ipv6_arr_assem++]; +- if (nh != FIB_DEFAULT_HOP) +- hops[i] = nh != FIB_DEFAULT_HOP ? +- nh : +- events[i].mbuf->port; ++ ++ hops[i] = nh != FIB_DEFAULT_HOP ? ++ nh : ++ events[i].mbuf->port; + process_packet(events[i].mbuf, &hops[i]); + events[i].mbuf->port = hops[i] != BAD_PORT ? + hops[i] : +diff --git a/dpdk/examples/l3fwd/main.c b/dpdk/examples/l3fwd/main.c +index 5198ff30dd..a4f061537e 100644 +--- a/dpdk/examples/l3fwd/main.c ++++ b/dpdk/examples/l3fwd/main.c +@@ -89,7 +89,6 @@ uint32_t enabled_port_mask; + + /* Used only in exact match mode. */ + int ipv6; /**< ipv6 is false by default. */ +-uint32_t hash_entry_number = HASH_ENTRY_NUMBER_DEFAULT; + + struct lcore_conf lcore_conf[RTE_MAX_LCORE]; + +@@ -395,7 +394,6 @@ print_usage(const char *prgname) + " [--eth-dest=X,MM:MM:MM:MM:MM:MM]" + " [--max-pkt-len PKTLEN]" + " [--no-numa]" +- " [--hash-entry-num]" + " [--ipv6]" + " [--parse-ptype]" + " [--per-port-pool]" +@@ -419,7 +417,6 @@ print_usage(const char *prgname) + " --eth-dest=X,MM:MM:MM:MM:MM:MM: Ethernet destination for port X\n" + " --max-pkt-len PKTLEN: maximum packet length in decimal (64-9600)\n" + " --no-numa: Disable numa awareness\n" +- " --hash-entry-num: Specify the hash entry number in hexadecimal to be setup\n" + " --ipv6: Set if running ipv6 packets\n" + " --parse-ptype: Set to use software to analyze packet type\n" + " --per-port-pool: Use separate buffer pool per port\n" +@@ -479,22 +476,6 @@ parse_portmask(const char *portmask) + return pm; + } + +-static int +-parse_hash_entry_number(const char *hash_entry_num) +-{ +- char *end = NULL; +- unsigned long hash_en; +- /* parse hexadecimal string */ +- hash_en = strtoul(hash_entry_num, &end, 16); +- if ((hash_entry_num[0] == '\0') || (end == NULL) || (*end != '\0')) +- return -1; +- +- if (hash_en == 0) +- return -1; +- +- return hash_en; +-} +- + static int + parse_config(const char *q_arg) + { +@@ -852,14 +833,7 @@ parse_args(int argc, char **argv) + break; + + case CMD_LINE_OPT_HASH_ENTRY_NUM_NUM: +- ret = parse_hash_entry_number(optarg); +- if ((ret > 0) && (ret <= L3FWD_HASH_ENTRIES)) { +- hash_entry_number = ret; +- } else { +- fprintf(stderr, "invalid hash entry number\n"); +- print_usage(prgname); +- return -1; +- } ++ fprintf(stderr, "Hash entry number will be ignored\n"); + break; + + case CMD_LINE_OPT_PARSE_PTYPE_NUM: +@@ -963,16 +937,6 @@ parse_args(int argc, char **argv) + lookup_mode = L3FWD_LOOKUP_LPM; + } + +- /* +- * ipv6 and hash flags are valid only for +- * exact match, reset them to default for +- * longest-prefix match. +- */ +- if (lookup_mode == L3FWD_LOOKUP_LPM) { +- ipv6 = 0; +- hash_entry_number = HASH_ENTRY_NUMBER_DEFAULT; +- } +- + /* For ACL, update port config rss hash filter */ + if (lookup_mode == L3FWD_LOOKUP_ACL) { + port_conf.rx_adv_conf.rss_conf.rss_hf |= +diff --git a/dpdk/examples/ntb/ntb_fwd.c b/dpdk/examples/ntb/ntb_fwd.c +index f9abed28e4..585aad9d70 100644 +--- a/dpdk/examples/ntb/ntb_fwd.c ++++ b/dpdk/examples/ntb/ntb_fwd.c +@@ -865,7 +865,7 @@ ntb_stats_clear(void) + + /* Clear NTB dev stats */ + nb_ids = rte_rawdev_xstats_names_get(dev_id, NULL, 0); +- if (nb_ids < 0) { ++ if (nb_ids <= 0) { + printf("Error: Cannot get count of xstats\n"); + return; + } +@@ -923,7 +923,7 @@ ntb_stats_display(void) + + /* Get NTB dev stats and stats names */ + nb_ids = rte_rawdev_xstats_names_get(dev_id, NULL, 0); +- if (nb_ids < 0) { ++ if (nb_ids <= 0) { + printf("Error: Cannot get count of xstats\n"); + return; + } +diff --git a/dpdk/examples/qos_sched/init.c b/dpdk/examples/qos_sched/init.c +index 0709aec10c..7a27c03b64 100644 +--- a/dpdk/examples/qos_sched/init.c ++++ b/dpdk/examples/qos_sched/init.c +@@ -79,6 +79,7 @@ app_init_port(uint16_t portid, struct rte_mempool *mp) + if (app_inited_port_mask & (1u << portid)) + return 0; + ++ memset(&rx_conf, 0, sizeof(struct rte_eth_rxconf)); + rx_conf.rx_thresh.pthresh = rx_thresh.pthresh; + rx_conf.rx_thresh.hthresh = rx_thresh.hthresh; + rx_conf.rx_thresh.wthresh = rx_thresh.wthresh; +@@ -86,6 +87,7 @@ app_init_port(uint16_t portid, struct rte_mempool *mp) + rx_conf.rx_drop_en = 0; + rx_conf.rx_deferred_start = 0; + ++ memset(&tx_conf, 0, sizeof(struct rte_eth_txconf)); + tx_conf.tx_thresh.pthresh = tx_thresh.pthresh; + tx_conf.tx_thresh.hthresh = tx_thresh.hthresh; + tx_conf.tx_thresh.wthresh = tx_thresh.wthresh; +@@ -326,6 +328,8 @@ int app_init(void) + for(i = 0; i < nb_pfc; i++) { + uint32_t socket = rte_lcore_to_socket_id(qos_conf[i].rx_core); + struct rte_ring *ring; ++ struct rte_eth_link link = {0}; ++ int retry_count = 100, retry_delay = 100; /* try every 100ms for 10 sec */ + + snprintf(ring_name, MAX_NAME_LEN, "ring-%u-%u", i, qos_conf[i].rx_core); + ring = rte_ring_lookup(ring_name); +@@ -356,6 +360,14 @@ int app_init(void) + app_init_port(qos_conf[i].rx_port, qos_conf[i].mbuf_pool); + app_init_port(qos_conf[i].tx_port, qos_conf[i].mbuf_pool); + ++ rte_eth_link_get(qos_conf[i].tx_port, &link); ++ if (link.link_status == 0) ++ printf("Waiting for link on port %u\n", qos_conf[i].tx_port); ++ while (link.link_status == 0 && retry_count--) { ++ rte_delay_ms(retry_delay); ++ rte_eth_link_get(qos_conf[i].tx_port, &link); ++ } ++ + qos_conf[i].sched_port = app_init_sched_port(qos_conf[i].tx_port, socket); + } + +diff --git a/dpdk/examples/qos_sched/profile.cfg b/dpdk/examples/qos_sched/profile.cfg +index e8de101b6c..00d4c7c1a5 100644 +--- a/dpdk/examples/qos_sched/profile.cfg ++++ b/dpdk/examples/qos_sched/profile.cfg +@@ -26,6 +26,8 @@ number of subports per port = 1 + number of pipes per subport = 4096 + queue sizes = 64 64 64 64 64 64 64 64 64 64 64 64 64 + ++pipe 0-4095 = 0 ; These pipes are configured with pipe profile 0 ++ + [subport profile 0] + tb rate = 1250000000 ; Bytes per second + tb size = 1000000 ; Bytes +@@ -46,8 +48,6 @@ tc 12 rate = 1250000000 ; Bytes per second + + tc period = 10 ; Milliseconds + +-pipe 0-4095 = 0 ; These pipes are configured with pipe profile 0 +- + ; Pipe configuration + [pipe profile 0] + tb rate = 305175 ; Bytes per second +@@ -71,4 +71,4 @@ tc period = 40 ; Milliseconds + + tc 12 oversubscription weight = 1 + +-tc 12 wrr weights = 1 1 1 1 +\ No newline at end of file ++tc 12 wrr weights = 1 1 1 1 +diff --git a/dpdk/examples/qos_sched/profile_ov.cfg b/dpdk/examples/qos_sched/profile_ov.cfg +index 14c89ae340..b6fe21ee1e 100644 +--- a/dpdk/examples/qos_sched/profile_ov.cfg ++++ b/dpdk/examples/qos_sched/profile_ov.cfg +@@ -6,12 +6,14 @@ + frame overhead = 24 + number of subports per port = 1 + ++subport 0-8 = 0 ++ + ; Subport configuration + [subport 0] + number of pipes per subport = 32 + queue sizes = 64 64 64 64 64 64 64 64 64 64 64 64 64 + +-subport 0-8 = 0 ++pipe 0-31 = 0 ; These pipes are configured with pipe profile 0 + + [subport profile 0] + tb rate = 8400000 ; Bytes per second +@@ -32,8 +34,6 @@ tc 11 rate = 8400000 ; Bytes per second + tc 12 rate = 8400000 ; Bytes per second + tc period = 10 ; Milliseconds + +-pipe 0-31 = 0 ; These pipes are configured with pipe profile 0 +- + ; Pipe configuration + [pipe profile 0] + tb rate = 16800000 ; Bytes per second +diff --git a/dpdk/examples/qos_sched/profile_pie.cfg b/dpdk/examples/qos_sched/profile_pie.cfg +index 241f748b33..bbc09d912b 100644 +--- a/dpdk/examples/qos_sched/profile_pie.cfg ++++ b/dpdk/examples/qos_sched/profile_pie.cfg +@@ -21,12 +21,14 @@ + frame overhead = 24 + number of subports per port = 1 + ++subport 0-8 = 0 ; These subports are configured with subport profile 0 ++ + ; Subport configuration + [subport 0] + number of pipes per subport = 4096 + queue sizes = 64 64 64 64 64 64 64 64 64 64 64 64 64 + +-subport 0-8 = 0 ; These subports are configured with subport profile 0 ++pipe 0-4095 = 0 ; These pipes are configured with pipe profile 0 + + [subport profile 0] + tb rate = 1250000000 ; Bytes per second +@@ -48,8 +50,6 @@ tc 12 rate = 1250000000 ; Bytes per second + + tc period = 10 ; Milliseconds + +-pipe 0-4095 = 0 ; These pipes are configured with pipe profile 0 +- + ; Pipe configuration + [pipe profile 0] + tb rate = 305175 ; Bytes per second +diff --git a/dpdk/examples/qos_sched/profile_red.cfg b/dpdk/examples/qos_sched/profile_red.cfg +index 4486d2799e..cee1470fd7 100644 +--- a/dpdk/examples/qos_sched/profile_red.cfg ++++ b/dpdk/examples/qos_sched/profile_red.cfg +@@ -21,12 +21,14 @@ + frame overhead = 24 + number of subports per port = 1 + ++subport 0-8 = 0 ; These subports are configured with subport profile 0 ++ + ; Subport configuration + [subport 0] + number of pipes per subport = 4096 + queue sizes = 64 64 64 64 64 64 64 64 64 64 64 64 64 + +-subport 0-8 = 0 ; These subports are configured with subport profile 0 ++pipe 0-4095 = 0 ; These pipes are configured with pipe profile 0 + + [subport profile 0] + tb rate = 1250000000 ; Bytes per second +@@ -48,8 +50,6 @@ tc 12 rate = 1250000000 ; Bytes per second + + tc period = 10 ; Milliseconds + +-pipe 0-4095 = 0 ; These pipes are configured with pipe profile 0 +- + ; Pipe configuration + [pipe profile 0] + tb rate = 305175 ; Bytes per second +diff --git a/dpdk/kernel/freebsd/contigmem/contigmem.c b/dpdk/kernel/freebsd/contigmem/contigmem.c +index bd72f4d620..7dd87599d9 100644 +--- a/dpdk/kernel/freebsd/contigmem/contigmem.c ++++ b/dpdk/kernel/freebsd/contigmem/contigmem.c +@@ -111,7 +111,7 @@ static struct cdevsw contigmem_ops = { + }; + + static int +-contigmem_load() ++contigmem_load(void) + { + char index_string[8], description[32]; + int i, error = 0; +@@ -178,7 +178,7 @@ error: + } + + static int +-contigmem_unload() ++contigmem_unload(void) + { + int i; + +diff --git a/dpdk/kernel/linux/kni/compat.h b/dpdk/kernel/linux/kni/compat.h +index 3a86d12bbc..8beb670465 100644 +--- a/dpdk/kernel/linux/kni/compat.h ++++ b/dpdk/kernel/linux/kni/compat.h +@@ -146,6 +146,12 @@ + #define HAVE_ETH_HW_ADDR_SET + #endif + +-#if KERNEL_VERSION(5, 18, 0) > LINUX_VERSION_CODE ++#if KERNEL_VERSION(5, 18, 0) > LINUX_VERSION_CODE && \ ++ (!(defined(RHEL_RELEASE_CODE) && \ ++ RHEL_RELEASE_VERSION(9, 1) <= RHEL_RELEASE_CODE)) + #define HAVE_NETIF_RX_NI + #endif ++ ++#if KERNEL_VERSION(6, 5, 0) > LINUX_VERSION_CODE ++#define HAVE_VMA_IN_GUP ++#endif +diff --git a/dpdk/kernel/linux/kni/kni_dev.h b/dpdk/kernel/linux/kni/kni_dev.h +index a2c6d9fc1a..975379825b 100644 +--- a/dpdk/kernel/linux/kni/kni_dev.h ++++ b/dpdk/kernel/linux/kni/kni_dev.h +@@ -105,11 +105,13 @@ static inline phys_addr_t iova_to_phys(struct task_struct *tsk, + + /* Read one page struct info */ + #ifdef HAVE_TSK_IN_GUP +- ret = get_user_pages_remote(tsk, tsk->mm, iova, 1, +- FOLL_TOUCH, &page, NULL, NULL); ++ ret = get_user_pages_remote(tsk, tsk->mm, iova, 1, 0, &page, NULL, NULL); + #else +- ret = get_user_pages_remote(tsk->mm, iova, 1, +- FOLL_TOUCH, &page, NULL, NULL); ++ #ifdef HAVE_VMA_IN_GUP ++ ret = get_user_pages_remote(tsk->mm, iova, 1, 0, &page, NULL, NULL); ++ #else ++ ret = get_user_pages_remote(tsk->mm, iova, 1, 0, &page, NULL); ++ #endif + #endif + if (ret < 0) + return 0; +diff --git a/dpdk/lib/acl/acl_run_altivec.h b/dpdk/lib/acl/acl_run_altivec.h +index 4dfe7a14b4..4556e1503b 100644 +--- a/dpdk/lib/acl/acl_run_altivec.h ++++ b/dpdk/lib/acl/acl_run_altivec.h +@@ -102,7 +102,7 @@ acl_match_check_x4(int slot, const struct rte_acl_ctx *ctx, struct parms *parms, + /* + * Process 4 transitions (in 2 XMM registers) in parallel + */ +-static inline __attribute__((optimize("O2"))) xmm_t ++static __rte_always_inline xmm_t + transition4(xmm_t next_input, const uint64_t *trans, + xmm_t *indices1, xmm_t *indices2) + { +diff --git a/dpdk/lib/cmdline/cmdline.c b/dpdk/lib/cmdline/cmdline.c +index e1009ba4c4..355c7d8ca6 100644 +--- a/dpdk/lib/cmdline/cmdline.c ++++ b/dpdk/lib/cmdline/cmdline.c +@@ -173,6 +173,7 @@ cmdline_quit(struct cmdline *cl) + { + if (!cl) + return; ++ cmdline_cancel(cl); + rdline_quit(&cl->rdl); + } + +@@ -197,9 +198,14 @@ cmdline_poll(struct cmdline *cl) + if (read_status < 0) + return read_status; + +- status = cmdline_in(cl, &c, 1); +- if (status < 0 && cl->rdl.status != RDLINE_EXITED) +- return status; ++ if (read_status == 0) { ++ /* end of file is implicit quit */ ++ cmdline_quit(cl); ++ } else { ++ status = cmdline_in(cl, &c, 1); ++ if (status < 0 && cl->rdl.status != RDLINE_EXITED) ++ return status; ++ } + } + + return cl->rdl.status; +diff --git a/dpdk/lib/cmdline/cmdline.h b/dpdk/lib/cmdline/cmdline.h +index 96674dfda2..b14355ef51 100644 +--- a/dpdk/lib/cmdline/cmdline.h ++++ b/dpdk/lib/cmdline/cmdline.h +@@ -23,6 +23,12 @@ + extern "C" { + #endif + ++enum rdline_status { ++ RDLINE_INIT, ++ RDLINE_RUNNING, ++ RDLINE_EXITED ++}; ++ + struct cmdline; + + struct cmdline *cmdline_new(cmdline_parse_ctx_t *ctx, const char *prompt, int s_in, int s_out); +diff --git a/dpdk/lib/cmdline/cmdline_os_unix.c b/dpdk/lib/cmdline/cmdline_os_unix.c +index 64a945a34f..9a4ec4e334 100644 +--- a/dpdk/lib/cmdline/cmdline_os_unix.c ++++ b/dpdk/lib/cmdline/cmdline_os_unix.c +@@ -51,3 +51,9 @@ cmdline_vdprintf(int fd, const char *format, va_list op) + { + return vdprintf(fd, format, op); + } ++ ++/* This function is not needed on Linux, instead use sigaction() */ ++void ++cmdline_cancel(__rte_unused struct cmdline *cl) ++{ ++} +diff --git a/dpdk/lib/cmdline/cmdline_os_windows.c b/dpdk/lib/cmdline/cmdline_os_windows.c +index 73ed9ba290..80863bfc8a 100644 +--- a/dpdk/lib/cmdline/cmdline_os_windows.c ++++ b/dpdk/lib/cmdline/cmdline_os_windows.c +@@ -203,3 +203,17 @@ cmdline_vdprintf(int fd, const char *format, va_list op) + + return ret; + } ++ ++void ++cmdline_cancel(struct cmdline *cl) ++{ ++ if (!cl) ++ return; ++ ++ /* force the outstanding read on console to exit */ ++ if (cl->oldterm.is_console_input) { ++ HANDLE handle = (HANDLE)_get_osfhandle(cl->s_in); ++ ++ CancelIoEx(handle, NULL); ++ } ++} +diff --git a/dpdk/lib/cmdline/cmdline_private.h b/dpdk/lib/cmdline/cmdline_private.h +index c2e906d8de..86a46cdea6 100644 +--- a/dpdk/lib/cmdline/cmdline_private.h ++++ b/dpdk/lib/cmdline/cmdline_private.h +@@ -23,14 +23,8 @@ + #define RDLINE_HISTORY_BUF_SIZE BUFSIZ + #define RDLINE_HISTORY_MAX_LINE 64 + +-enum rdline_status { +- RDLINE_INIT, +- RDLINE_RUNNING, +- RDLINE_EXITED +-}; +- + struct rdline { +- enum rdline_status status; ++ volatile enum rdline_status status; + /* rdline bufs */ + struct cirbuf left; + struct cirbuf right; +@@ -96,6 +90,9 @@ int cmdline_poll_char(struct cmdline *cl); + /* Read one character from input. */ + ssize_t cmdline_read_char(struct cmdline *cl, char *c); + ++/* Force current cmdline read to unblock. */ ++void cmdline_cancel(struct cmdline *cl); ++ + /* vdprintf(3) */ + __rte_format_printf(2, 0) + int cmdline_vdprintf(int fd, const char *format, va_list op); +diff --git a/dpdk/lib/compressdev/rte_compressdev.h b/dpdk/lib/compressdev/rte_compressdev.h +index 42bda9fc79..7eb5c58798 100644 +--- a/dpdk/lib/compressdev/rte_compressdev.h ++++ b/dpdk/lib/compressdev/rte_compressdev.h +@@ -353,7 +353,7 @@ rte_compressdev_stats_reset(uint8_t dev_id); + * @note The capabilities field of dev_info is set to point to the first + * element of an array of struct rte_compressdev_capabilities. + * The element after the last valid element has it's op field set to +- * RTE_COMP_ALGO_LIST_END. ++ * RTE_COMP_ALGO_UNSPECIFIED. + */ + __rte_experimental + void +diff --git a/dpdk/lib/compressdev/rte_compressdev_pmd.c b/dpdk/lib/compressdev/rte_compressdev_pmd.c +index e139bc86e7..156bccd972 100644 +--- a/dpdk/lib/compressdev/rte_compressdev_pmd.c ++++ b/dpdk/lib/compressdev/rte_compressdev_pmd.c +@@ -23,6 +23,9 @@ rte_compressdev_pmd_parse_name_arg(const char *key __rte_unused, + struct rte_compressdev_pmd_init_params *params = extra_args; + int n; + ++ if (value == NULL || extra_args == NULL) ++ return -EINVAL; ++ + n = strlcpy(params->name, value, RTE_COMPRESSDEV_NAME_MAX_LEN); + if (n >= RTE_COMPRESSDEV_NAME_MAX_LEN) + return -EINVAL; +@@ -40,6 +43,9 @@ rte_compressdev_pmd_parse_uint_arg(const char *key __rte_unused, + int i; + char *end; + ++ if (value == NULL || extra_args == NULL) ++ return -EINVAL; ++ + errno = 0; + i = strtol(value, &end, 10); + if (*end != 0 || errno != 0 || i < 0) +diff --git a/dpdk/lib/cryptodev/cryptodev_pmd.c b/dpdk/lib/cryptodev/cryptodev_pmd.c +index 77b269f312..d8073a601d 100644 +--- a/dpdk/lib/cryptodev/cryptodev_pmd.c ++++ b/dpdk/lib/cryptodev/cryptodev_pmd.c +@@ -22,6 +22,9 @@ rte_cryptodev_pmd_parse_name_arg(const char *key __rte_unused, + struct rte_cryptodev_pmd_init_params *params = extra_args; + int n; + ++ if (value == NULL || extra_args == NULL) ++ return -EINVAL; ++ + n = strlcpy(params->name, value, RTE_CRYPTODEV_NAME_MAX_LEN); + if (n >= RTE_CRYPTODEV_NAME_MAX_LEN) + return -EINVAL; +@@ -38,6 +41,10 @@ rte_cryptodev_pmd_parse_uint_arg(const char *key __rte_unused, + { + int i; + char *end; ++ ++ if (value == NULL || extra_args == NULL) ++ return -EINVAL; ++ + errno = 0; + + i = strtol(value, &end, 10); +diff --git a/dpdk/lib/cryptodev/cryptodev_pmd.h b/dpdk/lib/cryptodev/cryptodev_pmd.h +index 0020102eb7..8c1467037e 100644 +--- a/dpdk/lib/cryptodev/cryptodev_pmd.h ++++ b/dpdk/lib/cryptodev/cryptodev_pmd.h +@@ -65,7 +65,7 @@ struct rte_cryptodev_data { + /** Device ID for this instance */ + uint8_t dev_id; + /** Socket ID where memory is allocated */ +- uint8_t socket_id; ++ int socket_id; + /** Unique identifier name */ + char name[RTE_CRYPTODEV_NAME_MAX_LEN]; + +diff --git a/dpdk/lib/cryptodev/rte_crypto_asym.h b/dpdk/lib/cryptodev/rte_crypto_asym.h +index 38c8b60779..70ef69b97b 100644 +--- a/dpdk/lib/cryptodev/rte_crypto_asym.h ++++ b/dpdk/lib/cryptodev/rte_crypto_asym.h +@@ -388,9 +388,9 @@ struct rte_crypto_ec_xform { + */ + struct rte_crypto_mod_op_param { + rte_crypto_uint base; +- /** Base of modular exponentiation/multiplicative inverse */ ++ /**< Base of modular exponentiation/multiplicative inverse. */ + rte_crypto_uint result; +- /** Result of modular exponentiation/multiplicative inverse */ ++ /**< Result of modular exponentiation/multiplicative inverse. */ + }; + + /** +diff --git a/dpdk/lib/cryptodev/rte_crypto_sym.h b/dpdk/lib/cryptodev/rte_crypto_sym.h +index 33b4966e16..0d625ec103 100644 +--- a/dpdk/lib/cryptodev/rte_crypto_sym.h ++++ b/dpdk/lib/cryptodev/rte_crypto_sym.h +@@ -574,6 +574,7 @@ enum rte_crypto_sym_xform_type { + * hold a single transform, the type field is used to specify which transform + * is contained within the union + */ ++/* Structure rte_crypto_sym_xform 8< */ + struct rte_crypto_sym_xform { + struct rte_crypto_sym_xform *next; + /**< next xform in chain */ +@@ -589,6 +590,7 @@ struct rte_crypto_sym_xform { + /**< AEAD xform */ + }; + }; ++/* >8 End of structure rte_crypto_sym_xform. */ + + /** + * Symmetric Cryptographic Operation. +@@ -620,6 +622,7 @@ struct rte_crypto_sym_xform { + * destination buffer being at a different alignment, relative to buffer start, + * to the data in the source buffer. + */ ++/* Structure rte_crypto_sym_op 8< */ + struct rte_crypto_sym_op { + struct rte_mbuf *m_src; /**< source mbuf */ + struct rte_mbuf *m_dst; /**< destination mbuf */ +@@ -881,6 +884,7 @@ struct rte_crypto_sym_op { + }; + }; + }; ++/* >8 End of structure rte_crypto_sym_op. */ + + + /** +diff --git a/dpdk/lib/cryptodev/rte_cryptodev.c b/dpdk/lib/cryptodev/rte_cryptodev.c +index 2165a0688c..515d0df5ce 100644 +--- a/dpdk/lib/cryptodev/rte_cryptodev.c ++++ b/dpdk/lib/cryptodev/rte_cryptodev.c +@@ -2692,7 +2692,7 @@ cryptodev_handle_dev_info(const char *cmd __rte_unused, + rte_tel_data_start_dict(d); + rte_tel_data_add_dict_string(d, "device_name", + cryptodev_info.device->name); +- rte_tel_data_add_dict_int(d, "max_nb_queue_pairs", ++ rte_tel_data_add_dict_u64(d, "max_nb_queue_pairs", + cryptodev_info.max_nb_queue_pairs); + + return 0; +diff --git a/dpdk/lib/cryptodev/rte_cryptodev.h b/dpdk/lib/cryptodev/rte_cryptodev.h +index 86d792e2e7..0c9464649a 100644 +--- a/dpdk/lib/cryptodev/rte_cryptodev.h ++++ b/dpdk/lib/cryptodev/rte_cryptodev.h +@@ -501,6 +501,7 @@ extern const char * + rte_cryptodev_get_feature_name(uint64_t flag); + + /** Crypto device information */ ++/* Structure rte_cryptodev_info 8< */ + struct rte_cryptodev_info { + const char *driver_name; /**< Driver name. */ + uint8_t driver_id; /**< Driver identifier */ +@@ -529,6 +530,7 @@ struct rte_cryptodev_info { + */ + } sym; + }; ++/* >8 End of structure rte_cryptodev_info. */ + + #define RTE_CRYPTODEV_DETACHED (0) + #define RTE_CRYPTODEV_ATTACHED (1) +@@ -541,11 +543,13 @@ enum rte_cryptodev_event_type { + }; + + /** Crypto device queue pair configuration structure. */ ++/* Structure rte_cryptodev_qp_conf 8<*/ + struct rte_cryptodev_qp_conf { + uint32_t nb_descriptors; /**< Number of descriptors per queue pair */ + struct rte_mempool *mp_session; + /**< The mempool for creating session in sessionless mode */ + }; ++/* >8 End of structure rte_cryptodev_qp_conf. */ + + /** + * Function type used for processing crypto ops when enqueue/dequeue burst is +@@ -674,6 +678,7 @@ extern int + rte_cryptodev_socket_id(uint8_t dev_id); + + /** Crypto device configuration structure */ ++/* Structure rte_cryptodev_config 8< */ + struct rte_cryptodev_config { + int socket_id; /**< Socket to allocate resources on */ + uint16_t nb_queue_pairs; +@@ -686,6 +691,7 @@ struct rte_cryptodev_config { + * - RTE_CRYTPODEV_FF_SECURITY + */ + }; ++/* >8 End of structure rte_cryptodev_config. */ + + /** + * Configure a device. +@@ -900,6 +906,15 @@ struct rte_cryptodev_cb_rcu { + /**< RCU QSBR variable per queue pair */ + }; + ++/** ++ * Get the security context for the cryptodev. ++ * ++ * @param dev_id ++ * The device identifier. ++ * @return ++ * - NULL on error. ++ * - Pointer to security context on success. ++ */ + void * + rte_cryptodev_get_sec_ctx(uint8_t dev_id); + +@@ -911,11 +926,14 @@ rte_cryptodev_get_sec_ctx(uint8_t dev_id); + * @param nb_elts + * The number of elements in the mempool. + * @param elt_size +- * The size of the element. This value will be ignored if it is smaller than +- * the minimum session header size required for the system. For the user who +- * want to use the same mempool for sym session and session private data it +- * can be the maximum value of all existing devices' private data and session +- * header sizes. ++ * The size of the element. This should be the size of the cryptodev PMD ++ * session private data obtained through ++ * rte_cryptodev_sym_get_private_session_size() function call. ++ * For the user who wants to use the same mempool for heterogeneous PMDs ++ * this value should be the maximum value of their private session sizes. ++ * Please note the created mempool will have bigger elt size than this ++ * value as necessary session header and the possible padding are filled ++ * into each elt. + * @param cache_size + * The number of per-lcore cache elements + * @param priv_size +@@ -926,8 +944,8 @@ rte_cryptodev_get_sec_ctx(uint8_t dev_id); + * constraint for the reserved zone. + * + * @return +- * - On success return size of the session +- * - On failure returns 0 ++ * - On success returns the created session mempool pointer ++ * - On failure returns NULL + */ + __rte_experimental + struct rte_mempool * +@@ -968,11 +986,14 @@ rte_cryptodev_asym_session_pool_create(const char *name, uint32_t nb_elts, + * @param dev_id ID of device that we want the session to be used on + * @param xforms Symmetric crypto transform operations to apply on flow + * processed with this session +- * @param mp Mempool where the private data is allocated. ++ * @param mp Mempool to allocate symmetric session objects from + * + * @return + * - On success return pointer to sym-session. +- * - On failure returns NULL. ++ * - On failure returns NULL and rte_errno is set to the error code: ++ * - EINVAL on invalid arguments. ++ * - ENOMEM on memory error for session allocation. ++ * - ENOTSUP if device doesn't support session configuration. + */ + void * + rte_cryptodev_sym_session_create(uint8_t dev_id, +diff --git a/dpdk/lib/eal/common/eal_common_debug.c b/dpdk/lib/eal/common/eal_common_debug.c +index dcb554af1e..9cac9c6390 100644 +--- a/dpdk/lib/eal/common/eal_common_debug.c ++++ b/dpdk/lib/eal/common/eal_common_debug.c +@@ -4,10 +4,12 @@ + + #include + #include ++#include + + #include + #include + #include ++#include + + void + __rte_panic(const char *funcname, const char *format, ...) +@@ -39,7 +41,7 @@ rte_exit(int exit_code, const char *format, ...) + rte_vlog(RTE_LOG_CRIT, RTE_LOGTYPE_EAL, format, ap); + va_end(ap); + +- if (rte_eal_cleanup() != 0) ++ if (rte_eal_cleanup() != 0 && rte_errno != EALREADY) + RTE_LOG(CRIT, EAL, + "EAL could not release all resources\n"); + exit(exit_code); +diff --git a/dpdk/lib/eal/common/eal_common_dynmem.c b/dpdk/lib/eal/common/eal_common_dynmem.c +index 52e52e5986..95da55d9b0 100644 +--- a/dpdk/lib/eal/common/eal_common_dynmem.c ++++ b/dpdk/lib/eal/common/eal_common_dynmem.c +@@ -120,8 +120,7 @@ eal_dynmem_memseg_lists_init(void) + max_seglists_per_type = RTE_MAX_MEMSEG_LISTS / n_memtypes; + + if (max_seglists_per_type == 0) { +- RTE_LOG(ERR, EAL, "Cannot accommodate all memory types, please increase %s\n", +- RTE_STR(RTE_MAX_MEMSEG_LISTS)); ++ RTE_LOG(ERR, EAL, "Cannot accommodate all memory types, please increase RTE_MAX_MEMSEG_LISTS\n"); + goto out; + } + +@@ -180,8 +179,7 @@ eal_dynmem_memseg_lists_init(void) + for (cur_seglist = 0; cur_seglist < n_seglists; cur_seglist++) { + if (msl_idx >= RTE_MAX_MEMSEG_LISTS) { + RTE_LOG(ERR, EAL, +- "No more space in memseg lists, please increase %s\n", +- RTE_STR(RTE_MAX_MEMSEG_LISTS)); ++ "No more space in memseg lists, please increase RTE_MAX_MEMSEG_LISTS\n"); + goto out; + } + msl = &mcfg->memsegs[msl_idx++]; +@@ -253,7 +251,10 @@ eal_dynmem_hugepage_init(void) + */ + memset(&dummy, 0, sizeof(dummy)); + dummy.hugepage_sz = hpi->hugepage_sz; +- if (rte_memseg_list_walk(hugepage_count_walk, &dummy) < 0) ++ /* memory_hotplug_lock is held during initialization, so it's ++ * safe to call thread-unsafe version. ++ */ ++ if (rte_memseg_list_walk_thread_unsafe(hugepage_count_walk, &dummy) < 0) + return -1; + + for (i = 0; i < RTE_DIM(dummy.num_pages); i++) { +diff --git a/dpdk/lib/eal/common/eal_common_fbarray.c b/dpdk/lib/eal/common/eal_common_fbarray.c +index f11f87979f..169e66e04b 100644 +--- a/dpdk/lib/eal/common/eal_common_fbarray.c ++++ b/dpdk/lib/eal/common/eal_common_fbarray.c +@@ -1482,7 +1482,7 @@ rte_fbarray_dump_metadata(struct rte_fbarray *arr, FILE *f) + + if (fully_validate(arr->name, arr->elt_sz, arr->len)) { + fprintf(f, "Invalid file-backed array\n"); +- goto out; ++ return; + } + + /* prevent array from changing under us */ +@@ -1496,6 +1496,5 @@ rte_fbarray_dump_metadata(struct rte_fbarray *arr, FILE *f) + + for (i = 0; i < msk->n_masks; i++) + fprintf(f, "msk idx %i: 0x%016" PRIx64 "\n", i, msk->data[i]); +-out: + rte_rwlock_read_unlock(&arr->rwlock); + } +diff --git a/dpdk/lib/eal/common/eal_common_memory.c b/dpdk/lib/eal/common/eal_common_memory.c +index 688dc615d7..da6711d129 100644 +--- a/dpdk/lib/eal/common/eal_common_memory.c ++++ b/dpdk/lib/eal/common/eal_common_memory.c +@@ -1139,7 +1139,7 @@ handle_eal_heap_info_request(const char *cmd __rte_unused, const char *params, + malloc_heap_get_stats(heap, &sock_stats); + + rte_tel_data_start_dict(d); +- rte_tel_data_add_dict_int(d, "Head id", heap_id); ++ rte_tel_data_add_dict_u64(d, "Head_id", heap_id); + rte_tel_data_add_dict_string(d, "Name", heap->name); + rte_tel_data_add_dict_u64(d, "Heap_size", + sock_stats.heap_totalsz_bytes); +@@ -1201,13 +1201,13 @@ handle_eal_memzone_info_request(const char *cmd __rte_unused, + mz = rte_fbarray_get(&mcfg->memzones, mz_idx); + + rte_tel_data_start_dict(d); +- rte_tel_data_add_dict_int(d, "Zone", mz_idx); ++ rte_tel_data_add_dict_u64(d, "Zone", mz_idx); + rte_tel_data_add_dict_string(d, "Name", mz->name); +- rte_tel_data_add_dict_int(d, "Length", mz->len); ++ rte_tel_data_add_dict_u64(d, "Length", mz->len); + snprintf(addr, ADDR_STR, "%p", mz->addr); + rte_tel_data_add_dict_string(d, "Address", addr); + rte_tel_data_add_dict_int(d, "Socket", mz->socket_id); +- rte_tel_data_add_dict_int(d, "Flags", mz->flags); ++ rte_tel_data_add_dict_u64(d, "Flags", mz->flags); + + /* go through each page occupied by this memzone */ + msl = rte_mem_virt2memseg_list(mz->addr); +@@ -1222,7 +1222,7 @@ handle_eal_memzone_info_request(const char *cmd __rte_unused, + ms_idx = RTE_PTR_DIFF(mz->addr, msl->base_va) / page_sz; + ms = rte_fbarray_get(&msl->memseg_arr, ms_idx); + +- rte_tel_data_add_dict_int(d, "Hugepage_size", page_sz); ++ rte_tel_data_add_dict_u64(d, "Hugepage_size", page_sz); + snprintf(addr, ADDR_STR, "%p", ms->addr); + rte_tel_data_add_dict_string(d, "Hugepage_base", addr); + +diff --git a/dpdk/lib/eal/common/eal_common_proc.c b/dpdk/lib/eal/common/eal_common_proc.c +index 1fc1d6c53b..9676dd73c5 100644 +--- a/dpdk/lib/eal/common/eal_common_proc.c ++++ b/dpdk/lib/eal/common/eal_common_proc.c +@@ -321,6 +321,15 @@ retry: + return msglen; + } + ++static void ++cleanup_msg_fds(const struct rte_mp_msg *msg) ++{ ++ int i; ++ ++ for (i = 0; i < msg->num_fds; i++) ++ close(msg->fds[i]); ++} ++ + static void + process_msg(struct mp_msg_internal *m, struct sockaddr_un *s) + { +@@ -349,8 +358,10 @@ process_msg(struct mp_msg_internal *m, struct sockaddr_un *s) + else if (pending_req->type == REQUEST_TYPE_ASYNC) + req = async_reply_handle_thread_unsafe( + pending_req); +- } else ++ } else { + RTE_LOG(ERR, EAL, "Drop mp reply: %s\n", msg->name); ++ cleanup_msg_fds(msg); ++ } + pthread_mutex_unlock(&pending_requests.lock); + + if (req != NULL) +@@ -380,6 +391,7 @@ process_msg(struct mp_msg_internal *m, struct sockaddr_un *s) + RTE_LOG(ERR, EAL, "Cannot find action: %s\n", + msg->name); + } ++ cleanup_msg_fds(msg); + } else if (action(msg, s->sun_path) < 0) { + RTE_LOG(ERR, EAL, "Fail to handle message: %s\n", msg->name); + } +diff --git a/dpdk/lib/eal/common/rte_malloc.c b/dpdk/lib/eal/common/rte_malloc.c +index 48db264449..dbd8eae5b0 100644 +--- a/dpdk/lib/eal/common/rte_malloc.c ++++ b/dpdk/lib/eal/common/rte_malloc.c +@@ -41,13 +41,13 @@ mem_free(void *addr, const bool trace_ena) + void + rte_free(void *addr) + { +- return mem_free(addr, true); ++ mem_free(addr, true); + } + + void + eal_free_no_trace(void *addr) + { +- return mem_free(addr, false); ++ mem_free(addr, false); + } + + static void * +diff --git a/dpdk/lib/eal/common/rte_random.c b/dpdk/lib/eal/common/rte_random.c +index 166b0d8921..ac1bf6c6ee 100644 +--- a/dpdk/lib/eal/common/rte_random.c ++++ b/dpdk/lib/eal/common/rte_random.c +@@ -79,7 +79,7 @@ rte_srand(uint64_t seed) + unsigned int lcore_id; + + /* add lcore_id to seed to avoid having the same sequence */ +- for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) ++ for (lcore_id = 0; lcore_id < RTE_DIM(rand_states); lcore_id++) + __rte_srand_lfsr258(seed + lcore_id, &rand_states[lcore_id]); + } + +diff --git a/dpdk/lib/eal/freebsd/eal.c b/dpdk/lib/eal/freebsd/eal.c +index 607684c1a3..122daf6c1f 100644 +--- a/dpdk/lib/eal/freebsd/eal.c ++++ b/dpdk/lib/eal/freebsd/eal.c +@@ -889,6 +889,16 @@ rte_eal_init(int argc, char **argv) + int + rte_eal_cleanup(void) + { ++ static uint32_t run_once; ++ uint32_t has_run = 0; ++ ++ if (!__atomic_compare_exchange_n(&run_once, &has_run, 1, 0, ++ __ATOMIC_RELAXED, __ATOMIC_RELAXED)) { ++ RTE_LOG(WARNING, EAL, "Already called cleanup\n"); ++ rte_errno = EALREADY; ++ return -1; ++ } ++ + struct internal_config *internal_conf = + eal_get_internal_configuration(); + rte_service_finalize(); +@@ -896,9 +906,9 @@ rte_eal_cleanup(void) + eal_bus_cleanup(); + rte_trace_save(); + eal_trace_fini(); ++ rte_eal_alarm_cleanup(); + /* after this point, any DPDK pointers will become dangling */ + rte_eal_memory_detach(); +- rte_eal_alarm_cleanup(); + eal_cleanup_config(internal_conf); + return 0; + } +diff --git a/dpdk/lib/eal/freebsd/eal_alarm.c b/dpdk/lib/eal/freebsd/eal_alarm.c +index 1023c32937..1a3e6c0aad 100644 +--- a/dpdk/lib/eal/freebsd/eal_alarm.c ++++ b/dpdk/lib/eal/freebsd/eal_alarm.c +@@ -171,12 +171,12 @@ eal_alarm_callback(void *arg __rte_unused) + struct timespec now; + struct alarm_entry *ap; + +- rte_spinlock_lock(&alarm_list_lk); +- ap = LIST_FIRST(&alarm_list); +- + if (clock_gettime(CLOCK_TYPE_ID, &now) < 0) + return; + ++ rte_spinlock_lock(&alarm_list_lk); ++ ap = LIST_FIRST(&alarm_list); ++ + while (ap != NULL && timespec_cmp(&now, &ap->time) >= 0) { + ap->executing = 1; + ap->executing_id = pthread_self(); +diff --git a/dpdk/lib/eal/freebsd/eal_hugepage_info.c b/dpdk/lib/eal/freebsd/eal_hugepage_info.c +index 9dbe375bd3..e58e618469 100644 +--- a/dpdk/lib/eal/freebsd/eal_hugepage_info.c ++++ b/dpdk/lib/eal/freebsd/eal_hugepage_info.c +@@ -33,7 +33,7 @@ map_shared_memory(const char *filename, const size_t mem_size, int flags) + } + retval = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + close(fd); +- return retval; ++ return retval == MAP_FAILED ? NULL : retval; + } + + static void * +diff --git a/dpdk/lib/eal/freebsd/eal_memory.c b/dpdk/lib/eal/freebsd/eal_memory.c +index 17ab10e0ca..5c6165c580 100644 +--- a/dpdk/lib/eal/freebsd/eal_memory.c ++++ b/dpdk/lib/eal/freebsd/eal_memory.c +@@ -172,9 +172,8 @@ rte_eal_hugepage_init(void) + break; + } + if (msl_idx == RTE_MAX_MEMSEG_LISTS) { +- RTE_LOG(ERR, EAL, "Could not find space for memseg. Please increase %s and/or %s in configuration.\n", +- RTE_STR(RTE_MAX_MEMSEG_PER_TYPE), +- RTE_STR(RTE_MAX_MEM_MB_PER_TYPE)); ++ RTE_LOG(ERR, EAL, "Could not find space for memseg. Please increase RTE_MAX_MEMSEG_PER_LIST " ++ "RTE_MAX_MEMSEG_PER_TYPE and/or RTE_MAX_MEM_MB_PER_TYPE in configuration.\n"); + return -1; + } + arr = &msl->memseg_arr; +@@ -404,8 +403,7 @@ memseg_primary_init(void) + + if (msl_idx >= RTE_MAX_MEMSEG_LISTS) { + RTE_LOG(ERR, EAL, +- "No more space in memseg lists, please increase %s\n", +- RTE_STR(RTE_MAX_MEMSEG_LISTS)); ++ "No more space in memseg lists, please increase RTE_MAX_MEMSEG_LISTS\n"); + return -1; + } + +diff --git a/dpdk/lib/eal/include/generic/rte_atomic.h b/dpdk/lib/eal/include/generic/rte_atomic.h +index f5c49a9870..234b268b91 100644 +--- a/dpdk/lib/eal/include/generic/rte_atomic.h ++++ b/dpdk/lib/eal/include/generic/rte_atomic.h +@@ -176,11 +176,7 @@ rte_atomic16_exchange(volatile uint16_t *dst, uint16_t val); + static inline uint16_t + rte_atomic16_exchange(volatile uint16_t *dst, uint16_t val) + { +-#if defined(__clang__) + return __atomic_exchange_n(dst, val, __ATOMIC_SEQ_CST); +-#else +- return __atomic_exchange_2(dst, val, __ATOMIC_SEQ_CST); +-#endif + } + #endif + +@@ -459,11 +455,7 @@ rte_atomic32_exchange(volatile uint32_t *dst, uint32_t val); + static inline uint32_t + rte_atomic32_exchange(volatile uint32_t *dst, uint32_t val) + { +-#if defined(__clang__) + return __atomic_exchange_n(dst, val, __ATOMIC_SEQ_CST); +-#else +- return __atomic_exchange_4(dst, val, __ATOMIC_SEQ_CST); +-#endif + } + #endif + +@@ -741,11 +733,7 @@ rte_atomic64_exchange(volatile uint64_t *dst, uint64_t val); + static inline uint64_t + rte_atomic64_exchange(volatile uint64_t *dst, uint64_t val) + { +-#if defined(__clang__) + return __atomic_exchange_n(dst, val, __ATOMIC_SEQ_CST); +-#else +- return __atomic_exchange_8(dst, val, __ATOMIC_SEQ_CST); +-#endif + } + #endif + +diff --git a/dpdk/lib/eal/include/generic/rte_rwlock.h b/dpdk/lib/eal/include/generic/rte_rwlock.h +index 233d4262be..e479daa867 100644 +--- a/dpdk/lib/eal/include/generic/rte_rwlock.h ++++ b/dpdk/lib/eal/include/generic/rte_rwlock.h +@@ -79,6 +79,10 @@ rte_rwlock_init(rte_rwlock_t *rwl) + /** + * Take a read lock. Loop until the lock is held. + * ++ * @note The RW lock isn't recursive, so calling this function on the same ++ * lock twice without releasing it could potentially result in a deadlock ++ * scenario when a write lock is involved. ++ * + * @param rwl + * A pointer to a rwlock structure. + */ +diff --git a/dpdk/lib/eal/linux/eal.c b/dpdk/lib/eal/linux/eal.c +index 8c118d0d9f..336698379f 100644 +--- a/dpdk/lib/eal/linux/eal.c ++++ b/dpdk/lib/eal/linux/eal.c +@@ -1056,12 +1056,6 @@ rte_eal_init(int argc, char **argv) + } + } + +- /* register multi-process action callbacks for hotplug */ +- if (eal_mp_dev_hotplug_init() < 0) { +- rte_eal_init_alert("failed to register mp callback for hotplug"); +- return -1; +- } +- + if (rte_bus_scan()) { + rte_eal_init_alert("Cannot scan the buses for devices"); + rte_errno = ENODEV; +@@ -1206,6 +1200,12 @@ rte_eal_init(int argc, char **argv) + return -1; + } + ++ /* register multi-process action callbacks for hotplug after memory init */ ++ if (eal_mp_dev_hotplug_init() < 0) { ++ rte_eal_init_alert("failed to register mp callback for hotplug"); ++ return -1; ++ } ++ + if (rte_eal_tailqs_init() < 0) { + rte_eal_init_alert("Cannot init tail queues for objects"); + rte_errno = EFAULT; +@@ -1354,6 +1354,16 @@ mark_freeable(const struct rte_memseg_list *msl, const struct rte_memseg *ms, + int + rte_eal_cleanup(void) + { ++ static uint32_t run_once; ++ uint32_t has_run = 0; ++ ++ if (!__atomic_compare_exchange_n(&run_once, &has_run, 1, 0, ++ __ATOMIC_RELAXED, __ATOMIC_RELAXED)) { ++ RTE_LOG(WARNING, EAL, "Already called cleanup\n"); ++ rte_errno = EALREADY; ++ return -1; ++ } ++ + /* if we're in a primary process, we need to mark hugepages as freeable + * so that finalization can release them back to the system. + */ +@@ -1372,11 +1382,11 @@ rte_eal_cleanup(void) + eal_bus_cleanup(); + rte_trace_save(); + eal_trace_fini(); ++ eal_mp_dev_hotplug_cleanup(); ++ rte_eal_alarm_cleanup(); + /* after this point, any DPDK pointers will become dangling */ + rte_eal_memory_detach(); +- eal_mp_dev_hotplug_cleanup(); + rte_eal_malloc_heap_cleanup(); +- rte_eal_alarm_cleanup(); + eal_cleanup_config(internal_conf); + rte_eal_log_cleanup(); + return 0; +diff --git a/dpdk/lib/eal/linux/eal_hugepage_info.c b/dpdk/lib/eal/linux/eal_hugepage_info.c +index a1b6cb31ff..581d9dfc91 100644 +--- a/dpdk/lib/eal/linux/eal_hugepage_info.c ++++ b/dpdk/lib/eal/linux/eal_hugepage_info.c +@@ -50,7 +50,7 @@ map_shared_memory(const char *filename, const size_t mem_size, int flags) + retval = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + close(fd); +- return retval; ++ return retval == MAP_FAILED ? NULL : retval; + } + + static void * +@@ -214,6 +214,8 @@ get_hugepage_dir(uint64_t hugepage_sz, char *hugedir, int len) + char buf[BUFSIZ]; + const struct internal_config *internal_conf = + eal_get_internal_configuration(); ++ const size_t hugepage_dir_len = (internal_conf->hugepage_dir != NULL) ? ++ strlen(internal_conf->hugepage_dir) : 0; + struct stat st; + + /* +@@ -233,6 +235,7 @@ get_hugepage_dir(uint64_t hugepage_sz, char *hugedir, int len) + + while (fgets(buf, sizeof(buf), fd)){ + const char *pagesz_str; ++ size_t mountpt_len = 0; + + if (rte_strsplit(buf, sizeof(buf), splitstr, _FIELDNAME_MAX, + split_tok) != _FIELDNAME_MAX) { +@@ -265,12 +268,16 @@ get_hugepage_dir(uint64_t hugepage_sz, char *hugedir, int len) + break; + } + ++ mountpt_len = strlen(splitstr[MOUNTPT]); ++ + /* +- * Ignore any mount that doesn't contain the --huge-dir +- * directory. ++ * Ignore any mount that doesn't contain the --huge-dir directory ++ * or where mount point is not a parent path of --huge-dir + */ + if (strncmp(internal_conf->hugepage_dir, splitstr[MOUNTPT], +- strlen(splitstr[MOUNTPT])) != 0) { ++ mountpt_len) != 0 || ++ (hugepage_dir_len > mountpt_len && ++ internal_conf->hugepage_dir[mountpt_len] != '/')) { + continue; + } + +@@ -278,7 +285,7 @@ get_hugepage_dir(uint64_t hugepage_sz, char *hugedir, int len) + * We found a match, but only prefer it if it's a longer match + * (so /mnt/1 is preferred over /mnt for matching /mnt/1/2)). + */ +- if (strlen(splitstr[MOUNTPT]) > strlen(found)) ++ if (mountpt_len > strlen(found)) + strlcpy(found, splitstr[MOUNTPT], len); + } /* end while fgets */ + +diff --git a/dpdk/lib/eal/linux/eal_memalloc.c b/dpdk/lib/eal/linux/eal_memalloc.c +index f8b1588cae..9853ec78a2 100644 +--- a/dpdk/lib/eal/linux/eal_memalloc.c ++++ b/dpdk/lib/eal/linux/eal_memalloc.c +@@ -1740,7 +1740,10 @@ eal_memalloc_init(void) + eal_get_internal_configuration(); + + if (rte_eal_process_type() == RTE_PROC_SECONDARY) +- if (rte_memseg_list_walk(secondary_msl_create_walk, NULL) < 0) ++ /* memory_hotplug_lock is held during initialization, so it's ++ * safe to call thread-unsafe version. ++ */ ++ if (rte_memseg_list_walk_thread_unsafe(secondary_msl_create_walk, NULL) < 0) + return -1; + if (rte_eal_process_type() == RTE_PROC_PRIMARY && + internal_conf->in_memory) { +@@ -1778,7 +1781,7 @@ eal_memalloc_init(void) + } + + /* initialize all of the fd lists */ +- if (rte_memseg_list_walk(fd_list_create_walk, NULL)) ++ if (rte_memseg_list_walk_thread_unsafe(fd_list_create_walk, NULL)) + return -1; + return 0; + } +diff --git a/dpdk/lib/eal/linux/eal_memory.c b/dpdk/lib/eal/linux/eal_memory.c +index 60fc8cc6ca..9b6f08fba8 100644 +--- a/dpdk/lib/eal/linux/eal_memory.c ++++ b/dpdk/lib/eal/linux/eal_memory.c +@@ -681,6 +681,7 @@ remap_segment(struct hugepage_file *hugepages, int seg_start, int seg_end) + + /* find free space in memseg lists */ + for (msl_idx = 0; msl_idx < RTE_MAX_MEMSEG_LISTS; msl_idx++) { ++ int free_len; + bool empty; + msl = &mcfg->memsegs[msl_idx]; + arr = &msl->memseg_arr; +@@ -692,24 +693,31 @@ remap_segment(struct hugepage_file *hugepages, int seg_start, int seg_end) + + /* leave space for a hole if array is not empty */ + empty = arr->count == 0; +- ms_idx = rte_fbarray_find_next_n_free(arr, 0, +- seg_len + (empty ? 0 : 1)); +- +- /* memseg list is full? */ ++ /* find start of the biggest contiguous block and its size */ ++ ms_idx = rte_fbarray_find_biggest_free(arr, 0); + if (ms_idx < 0) + continue; +- ++ /* hole is 1 segment long, so at least two segments long. */ ++ free_len = rte_fbarray_find_contig_free(arr, ms_idx); ++ if (free_len < 2) ++ continue; + /* leave some space between memsegs, they are not IOVA + * contiguous, so they shouldn't be VA contiguous either. + */ +- if (!empty) ++ if (!empty) { + ms_idx++; ++ free_len--; ++ } ++ ++ /* we might not get all of the space we wanted */ ++ free_len = RTE_MIN(seg_len, free_len); ++ seg_end = seg_start + free_len; ++ seg_len = seg_end - seg_start; + break; + } + if (msl_idx == RTE_MAX_MEMSEG_LISTS) { +- RTE_LOG(ERR, EAL, "Could not find space for memseg. Please increase %s and/or %s in configuration.\n", +- RTE_STR(RTE_MAX_MEMSEG_PER_TYPE), +- RTE_STR(RTE_MAX_MEM_MB_PER_TYPE)); ++ RTE_LOG(ERR, EAL, "Could not find space for memseg. Please increase RTE_MAX_MEMSEG_PER_LIST " ++ "RTE_MAX_MEMSEG_PER_TYPE and/or RTE_MAX_MEM_MB_PER_TYPE in configuration.\n"); + return -1; + } + +@@ -787,7 +795,7 @@ remap_segment(struct hugepage_file *hugepages, int seg_start, int seg_end) + } + RTE_LOG(DEBUG, EAL, "Allocated %" PRIu64 "M on socket %i\n", + (seg_len * page_sz) >> 20, socket_id); +- return 0; ++ return seg_len; + } + + static uint64_t +@@ -957,8 +965,7 @@ prealloc_segments(struct hugepage_file *hugepages, int n_pages) + break; + } + if (msl_idx == RTE_MAX_MEMSEG_LISTS) { +- RTE_LOG(ERR, EAL, "Not enough space in memseg lists, please increase %s\n", +- RTE_STR(RTE_MAX_MEMSEG_LISTS)); ++ RTE_LOG(ERR, EAL, "Not enough space in memseg lists, please increase RTE_MAX_MEMSEG_LISTS\n"); + return -1; + } + +@@ -1022,10 +1029,16 @@ remap_needed_hugepages(struct hugepage_file *hugepages, int n_pages) + if (new_memseg) { + /* if this isn't the first time, remap segment */ + if (cur_page != 0) { +- ret = remap_segment(hugepages, seg_start_page, +- cur_page); +- if (ret != 0) +- return -1; ++ int n_remapped = 0; ++ int n_needed = cur_page - seg_start_page; ++ while (n_remapped < n_needed) { ++ ret = remap_segment(hugepages, seg_start_page, ++ cur_page); ++ if (ret < 0) ++ return -1; ++ n_remapped += ret; ++ seg_start_page += ret; ++ } + } + /* remember where we started */ + seg_start_page = cur_page; +@@ -1034,10 +1047,16 @@ remap_needed_hugepages(struct hugepage_file *hugepages, int n_pages) + } + /* we were stopped, but we didn't remap the last segment, do it now */ + if (cur_page != 0) { +- ret = remap_segment(hugepages, seg_start_page, +- cur_page); +- if (ret != 0) +- return -1; ++ int n_remapped = 0; ++ int n_needed = cur_page - seg_start_page; ++ while (n_remapped < n_needed) { ++ ret = remap_segment(hugepages, seg_start_page, ++ cur_page); ++ if (ret < 0) ++ return -1; ++ n_remapped += ret; ++ seg_start_page += ret; ++ } + } + return 0; + } +@@ -1812,8 +1831,7 @@ memseg_primary_init_32(void) + + if (msl_idx >= RTE_MAX_MEMSEG_LISTS) { + RTE_LOG(ERR, EAL, +- "No more space in memseg lists, please increase %s\n", +- RTE_STR(RTE_MAX_MEMSEG_LISTS)); ++ "No more space in memseg lists, please increase RTE_MAX_MEMSEG_LISTS\n"); + return -1; + } + +diff --git a/dpdk/lib/eal/riscv/include/rte_vect.h b/dpdk/lib/eal/riscv/include/rte_vect.h +index 2f97f437a2..da9092a94a 100644 +--- a/dpdk/lib/eal/riscv/include/rte_vect.h ++++ b/dpdk/lib/eal/riscv/include/rte_vect.h +@@ -29,7 +29,7 @@ typedef union rte_xmm { + uint32_t u32[XMM_SIZE / sizeof(uint32_t)]; + uint64_t u64[XMM_SIZE / sizeof(uint64_t)]; + double pd[XMM_SIZE / sizeof(double)]; +-} __rte_aligned(8) rte_xmm_t; ++} __rte_aligned(16) rte_xmm_t; + + static inline xmm_t + vect_load_128(void *p) +diff --git a/dpdk/lib/eal/unix/eal_firmware.c b/dpdk/lib/eal/unix/eal_firmware.c +index d1616b0bd9..1a7cf8e7b7 100644 +--- a/dpdk/lib/eal/unix/eal_firmware.c ++++ b/dpdk/lib/eal/unix/eal_firmware.c +@@ -25,19 +25,31 @@ static int + firmware_open(struct firmware_read_ctx *ctx, const char *name, size_t blocksize) + { + struct archive_entry *e; ++ int err; + + ctx->a = archive_read_new(); + if (ctx->a == NULL) + return -1; +- if (archive_read_support_format_raw(ctx->a) != ARCHIVE_OK || +- archive_read_support_filter_xz(ctx->a) != ARCHIVE_OK || +- archive_read_open_filename(ctx->a, name, blocksize) != ARCHIVE_OK || +- archive_read_next_header(ctx->a, &e) != ARCHIVE_OK) { +- archive_read_free(ctx->a); +- ctx->a = NULL; +- return -1; +- } ++ ++ if (archive_read_support_format_raw(ctx->a) != ARCHIVE_OK) ++ goto error; ++ ++ err = archive_read_support_filter_xz(ctx->a); ++ if (err != ARCHIVE_OK && err != ARCHIVE_WARN) ++ goto error; ++ ++ if (archive_read_open_filename(ctx->a, name, blocksize) != ARCHIVE_OK) ++ goto error; ++ ++ if (archive_read_next_header(ctx->a, &e) != ARCHIVE_OK) ++ goto error; ++ + return 0; ++ ++error: ++ archive_read_free(ctx->a); ++ ctx->a = NULL; ++ return -1; + } + + static ssize_t +diff --git a/dpdk/lib/eal/unix/rte_thread.c b/dpdk/lib/eal/unix/rte_thread.c +index 37ebfcfca1..f4076122a4 100644 +--- a/dpdk/lib/eal/unix/rte_thread.c ++++ b/dpdk/lib/eal/unix/rte_thread.c +@@ -5,6 +5,7 @@ + + #include + #include ++#include + #include + #include + +@@ -16,9 +17,14 @@ struct eal_tls_key { + pthread_key_t thread_index; + }; + +-struct thread_routine_ctx { ++struct thread_start_context { + rte_thread_func thread_func; +- void *routine_args; ++ void *thread_args; ++ const rte_thread_attr_t *thread_attr; ++ pthread_mutex_t wrapper_mutex; ++ pthread_cond_t wrapper_cond; ++ int wrapper_ret; ++ bool wrapper_done; + }; + + static int +@@ -81,13 +87,29 @@ thread_map_os_priority_to_eal_priority(int policy, int os_pri, + } + + static void * +-thread_func_wrapper(void *arg) ++thread_start_wrapper(void *arg) + { +- struct thread_routine_ctx ctx = *(struct thread_routine_ctx *)arg; ++ struct thread_start_context *ctx = (struct thread_start_context *)arg; ++ rte_thread_func thread_func = ctx->thread_func; ++ void *thread_args = ctx->thread_args; ++ int ret = 0; ++ ++ if (ctx->thread_attr != NULL && CPU_COUNT(&ctx->thread_attr->cpuset) > 0) { ++ ret = rte_thread_set_affinity_by_id(rte_thread_self(), &ctx->thread_attr->cpuset); ++ if (ret != 0) ++ RTE_LOG(DEBUG, EAL, "rte_thread_set_affinity_by_id failed\n"); ++ } + +- free(arg); ++ pthread_mutex_lock(&ctx->wrapper_mutex); ++ ctx->wrapper_ret = ret; ++ ctx->wrapper_done = true; ++ pthread_cond_signal(&ctx->wrapper_cond); ++ pthread_mutex_unlock(&ctx->wrapper_mutex); + +- return (void *)(uintptr_t)ctx.thread_func(ctx.routine_args); ++ if (ret != 0) ++ return NULL; ++ ++ return (void *)(uintptr_t)thread_func(thread_args); + } + + int +@@ -98,20 +120,18 @@ rte_thread_create(rte_thread_t *thread_id, + int ret = 0; + pthread_attr_t attr; + pthread_attr_t *attrp = NULL; +- struct thread_routine_ctx *ctx; + struct sched_param param = { + .sched_priority = 0, + }; + int policy = SCHED_OTHER; +- +- ctx = calloc(1, sizeof(*ctx)); +- if (ctx == NULL) { +- RTE_LOG(DEBUG, EAL, "Insufficient memory for thread context allocations\n"); +- ret = ENOMEM; +- goto cleanup; +- } +- ctx->routine_args = args; +- ctx->thread_func = thread_func; ++ struct thread_start_context ctx = { ++ .thread_func = thread_func, ++ .thread_args = args, ++ .thread_attr = thread_attr, ++ .wrapper_done = false, ++ .wrapper_mutex = PTHREAD_MUTEX_INITIALIZER, ++ .wrapper_cond = PTHREAD_COND_INITIALIZER, ++ }; + + if (thread_attr != NULL) { + ret = pthread_attr_init(&attr); +@@ -133,7 +153,6 @@ rte_thread_create(rte_thread_t *thread_id, + goto cleanup; + } + +- + if (thread_attr->priority == + RTE_THREAD_PRIORITY_REALTIME_CRITICAL) { + ret = ENOTSUP; +@@ -158,24 +177,22 @@ rte_thread_create(rte_thread_t *thread_id, + } + + ret = pthread_create((pthread_t *)&thread_id->opaque_id, attrp, +- thread_func_wrapper, ctx); ++ thread_start_wrapper, &ctx); + if (ret != 0) { + RTE_LOG(DEBUG, EAL, "pthread_create failed\n"); + goto cleanup; + } + +- if (thread_attr != NULL && CPU_COUNT(&thread_attr->cpuset) > 0) { +- ret = rte_thread_set_affinity_by_id(*thread_id, +- &thread_attr->cpuset); +- if (ret != 0) { +- RTE_LOG(DEBUG, EAL, "rte_thread_set_affinity_by_id failed\n"); +- goto cleanup; +- } +- } ++ pthread_mutex_lock(&ctx.wrapper_mutex); ++ while (!ctx.wrapper_done) ++ pthread_cond_wait(&ctx.wrapper_cond, &ctx.wrapper_mutex); ++ ret = ctx.wrapper_ret; ++ pthread_mutex_unlock(&ctx.wrapper_mutex); ++ ++ if (ret != 0) ++ pthread_join((pthread_t)thread_id->opaque_id, NULL); + +- ctx = NULL; + cleanup: +- free(ctx); + if (attrp != NULL) + pthread_attr_destroy(&attr); + +diff --git a/dpdk/lib/eal/windows/eal.c b/dpdk/lib/eal/windows/eal.c +index adb929a014..56fadc7afe 100644 +--- a/dpdk/lib/eal/windows/eal.c ++++ b/dpdk/lib/eal/windows/eal.c +@@ -462,6 +462,9 @@ rte_eal_init(int argc, char **argv) + */ + rte_eal_mp_remote_launch(sync_func, NULL, SKIP_MAIN); + rte_eal_mp_wait_lcore(); ++ ++ eal_mcfg_complete(); ++ + return fctret; + } + +diff --git a/dpdk/lib/eal/windows/eal_memory.c b/dpdk/lib/eal/windows/eal_memory.c +index 215d768e2c..31410a41fd 100644 +--- a/dpdk/lib/eal/windows/eal_memory.c ++++ b/dpdk/lib/eal/windows/eal_memory.c +@@ -72,10 +72,18 @@ static VirtualAlloc2_type VirtualAlloc2_ptr; + + #ifdef RTE_TOOLCHAIN_GCC + ++#ifndef MEM_COALESCE_PLACEHOLDERS + #define MEM_COALESCE_PLACEHOLDERS 0x00000001 ++#endif ++#ifndef MEM_PRESERVE_PLACEHOLDER + #define MEM_PRESERVE_PLACEHOLDER 0x00000002 ++#endif ++#ifndef MEM_REPLACE_PLACEHOLDER + #define MEM_REPLACE_PLACEHOLDER 0x00004000 ++#endif ++#ifndef MEM_RESERVE_PLACEHOLDER + #define MEM_RESERVE_PLACEHOLDER 0x00040000 ++#endif + + int + eal_mem_win32api_init(void) +diff --git a/dpdk/lib/eal/windows/include/pthread.h b/dpdk/lib/eal/windows/include/pthread.h +index 27fd2cca52..f7cf0e9ddf 100644 +--- a/dpdk/lib/eal/windows/include/pthread.h ++++ b/dpdk/lib/eal/windows/include/pthread.h +@@ -134,7 +134,8 @@ pthread_create(void *threadid, const void *threadattr, void *threadfunc, + { + RTE_SET_USED(threadattr); + HANDLE hThread; +- hThread = CreateThread(NULL, 0, (LPTHREAD_START_ROUTINE)threadfunc, ++ hThread = CreateThread(NULL, 0, ++ (LPTHREAD_START_ROUTINE)(uintptr_t)threadfunc, + args, 0, (LPDWORD)threadid); + if (hThread) { + SetPriorityClass(GetCurrentProcess(), NORMAL_PRIORITY_CLASS); +diff --git a/dpdk/lib/eal/windows/rte_thread.c b/dpdk/lib/eal/windows/rte_thread.c +index 1c1e9d01e3..3538633816 100644 +--- a/dpdk/lib/eal/windows/rte_thread.c ++++ b/dpdk/lib/eal/windows/rte_thread.c +@@ -17,6 +17,7 @@ struct eal_tls_key { + + struct thread_routine_ctx { + rte_thread_func thread_func; ++ bool thread_init_failed; + void *routine_args; + }; + +@@ -165,9 +166,13 @@ static DWORD + thread_func_wrapper(void *arg) + { + struct thread_routine_ctx ctx = *(struct thread_routine_ctx *)arg; ++ const bool thread_exit = __atomic_load_n(&ctx.thread_init_failed, __ATOMIC_ACQUIRE); + + free(arg); + ++ if (thread_exit) ++ return 0; ++ + return (DWORD)ctx.thread_func(ctx.routine_args); + } + +@@ -181,6 +186,7 @@ rte_thread_create(rte_thread_t *thread_id, + HANDLE thread_handle = NULL; + GROUP_AFFINITY thread_affinity; + struct thread_routine_ctx *ctx; ++ bool thread_exit = false; + + ctx = calloc(1, sizeof(*ctx)); + if (ctx == NULL) { +@@ -190,6 +196,7 @@ rte_thread_create(rte_thread_t *thread_id, + } + ctx->routine_args = args; + ctx->thread_func = thread_func; ++ ctx->thread_init_failed = false; + + thread_handle = CreateThread(NULL, 0, thread_func_wrapper, ctx, + CREATE_SUSPENDED, &tid); +@@ -207,23 +214,29 @@ rte_thread_create(rte_thread_t *thread_id, + ); + if (ret != 0) { + RTE_LOG(DEBUG, EAL, "Unable to convert cpuset to thread affinity\n"); +- goto cleanup; ++ thread_exit = true; ++ goto resume_thread; + } + + if (!SetThreadGroupAffinity(thread_handle, + &thread_affinity, NULL)) { + ret = thread_log_last_error("SetThreadGroupAffinity()"); +- goto cleanup; ++ thread_exit = true; ++ goto resume_thread; + } + } + ret = rte_thread_set_priority(*thread_id, + thread_attr->priority); + if (ret != 0) { + RTE_LOG(DEBUG, EAL, "Unable to set thread priority\n"); +- goto cleanup; ++ thread_exit = true; ++ goto resume_thread; + } + } + ++resume_thread: ++ __atomic_store_n(&ctx->thread_init_failed, thread_exit, __ATOMIC_RELEASE); ++ + if (ResumeThread(thread_handle) == (DWORD)-1) { + ret = thread_log_last_error("ResumeThread()"); + goto cleanup; +diff --git a/dpdk/lib/eal/x86/include/rte_memcpy.h b/dpdk/lib/eal/x86/include/rte_memcpy.h +index d4d7a5cfc8..fd151be708 100644 +--- a/dpdk/lib/eal/x86/include/rte_memcpy.h ++++ b/dpdk/lib/eal/x86/include/rte_memcpy.h +@@ -846,7 +846,7 @@ rte_memcpy_aligned(void *dst, const void *src, size_t n) + } + + /* Copy 64 bytes blocks */ +- for (; n >= 64; n -= 64) { ++ for (; n > 64; n -= 64) { + rte_mov64((uint8_t *)dst, (const uint8_t *)src); + dst = (uint8_t *)dst + 64; + src = (const uint8_t *)src + 64; +diff --git a/dpdk/lib/ethdev/ethdev_driver.h b/dpdk/lib/ethdev/ethdev_driver.h +index 6a550cfc83..cd2cd89649 100644 +--- a/dpdk/lib/ethdev/ethdev_driver.h ++++ b/dpdk/lib/ethdev/ethdev_driver.h +@@ -117,7 +117,11 @@ struct rte_eth_dev_data { + + uint64_t rx_mbuf_alloc_failed; /**< Rx ring mbuf allocation failures */ + +- /** Device Ethernet link address. @see rte_eth_dev_release_port() */ ++ /** ++ * Device Ethernet link addresses. ++ * All entries are unique. ++ * The first entry (index zero) is the default address. ++ */ + struct rte_ether_addr *mac_addrs; + /** Bitmap associating MAC addresses to pools */ + uint64_t mac_pool_sel[RTE_ETH_NUM_RECEIVE_MAC_ADDR]; +diff --git a/dpdk/lib/ethdev/ethdev_pci.h b/dpdk/lib/ethdev/ethdev_pci.h +index 94b8fba5d7..320e3e0093 100644 +--- a/dpdk/lib/ethdev/ethdev_pci.h ++++ b/dpdk/lib/ethdev/ethdev_pci.h +@@ -126,12 +126,13 @@ rte_eth_dev_pci_generic_probe(struct rte_pci_device *pci_dev, + struct rte_eth_dev *eth_dev; + int ret; + ++ if (*dev_init == NULL) ++ return -EINVAL; ++ + eth_dev = rte_eth_dev_pci_allocate(pci_dev, private_data_size); + if (!eth_dev) + return -ENOMEM; + +- if (*dev_init == NULL) +- return -EINVAL; + ret = dev_init(eth_dev); + if (ret) + rte_eth_dev_release_port(eth_dev); +diff --git a/dpdk/lib/ethdev/rte_class_eth.c b/dpdk/lib/ethdev/rte_class_eth.c +index 838b3a8f9f..b61dae849d 100644 +--- a/dpdk/lib/ethdev/rte_class_eth.c ++++ b/dpdk/lib/ethdev/rte_class_eth.c +@@ -67,7 +67,7 @@ eth_representor_cmp(const char *key __rte_unused, + const struct rte_eth_dev *edev = opaque; + const struct rte_eth_dev_data *data = edev->data; + struct rte_eth_devargs eth_da; +- uint16_t id, nc, np, nf, i, c, p, f; ++ uint16_t id = 0, nc, np, nf, i, c, p, f; + + if ((data->dev_flags & RTE_ETH_DEV_REPRESENTOR) == 0) + return -1; /* not a representor port */ +diff --git a/dpdk/lib/ethdev/rte_ethdev.c b/dpdk/lib/ethdev/rte_ethdev.c +index 5d5e18db1e..4f50e2fa80 100644 +--- a/dpdk/lib/ethdev/rte_ethdev.c ++++ b/dpdk/lib/ethdev/rte_ethdev.c +@@ -1192,7 +1192,9 @@ rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, + } + + if (dev_conf->rxmode.mtu == 0) +- dev->data->dev_conf.rxmode.mtu = RTE_ETHER_MTU; ++ dev->data->dev_conf.rxmode.mtu = ++ (dev_info.max_mtu == 0) ? RTE_ETHER_MTU : ++ RTE_MIN(dev_info.max_mtu, RTE_ETHER_MTU); + + ret = eth_dev_validate_mtu(port_id, &dev_info, + dev->data->dev_conf.rxmode.mtu); +@@ -4362,6 +4364,11 @@ rte_eth_fec_set(uint16_t port_id, uint32_t fec_capa) + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); + dev = &rte_eth_devices[port_id]; + ++ if (fec_capa == 0) { ++ RTE_ETHDEV_LOG(ERR, "At least one FEC mode should be specified\n"); ++ return -EINVAL; ++ } ++ + if (*dev->dev_ops->fec_set == NULL) + return -ENOTSUP; + return eth_err(port_id, (*dev->dev_ops->fec_set)(dev, fec_capa)); +@@ -4499,6 +4506,7 @@ int + rte_eth_dev_default_mac_addr_set(uint16_t port_id, struct rte_ether_addr *addr) + { + struct rte_eth_dev *dev; ++ int index; + int ret; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); +@@ -4517,6 +4525,15 @@ rte_eth_dev_default_mac_addr_set(uint16_t port_id, struct rte_ether_addr *addr) + if (*dev->dev_ops->mac_addr_set == NULL) + return -ENOTSUP; + ++ /* Keep address unique in dev->data->mac_addrs[]. */ ++ index = eth_dev_get_mac_addr_index(port_id, addr); ++ if (index > 0) { ++ RTE_ETHDEV_LOG(ERR, ++ "New default address for port %u was already in the address list. Please remove it first.\n", ++ port_id); ++ return -EEXIST; ++ } ++ + ret = (*dev->dev_ops->mac_addr_set)(dev, addr); + if (ret < 0) + return ret; +@@ -5935,7 +5952,7 @@ eth_dev_handle_port_dump_priv(const char *cmd __rte_unused, + if (!rte_eth_dev_is_valid_port(port_id)) + return -EINVAL; + +- buf = calloc(sizeof(char), RTE_TEL_MAX_SINGLE_STRING_LEN); ++ buf = calloc(RTE_TEL_MAX_SINGLE_STRING_LEN, sizeof(char)); + if (buf == NULL) + return -ENOMEM; + +@@ -6037,10 +6054,8 @@ eth_dev_handle_port_info(const char *cmd __rte_unused, + eth_dev->data->nb_tx_queues); + rte_tel_data_add_dict_int(d, "port_id", eth_dev->data->port_id); + rte_tel_data_add_dict_int(d, "mtu", eth_dev->data->mtu); +- rte_tel_data_add_dict_int(d, "rx_mbuf_size_min", ++ rte_tel_data_add_dict_u64(d, "rx_mbuf_size_min", + eth_dev->data->min_rx_buf_size); +- rte_tel_data_add_dict_int(d, "rx_mbuf_alloc_fail", +- eth_dev->data->rx_mbuf_alloc_failed); + rte_ether_format_addr(mac_addr, sizeof(mac_addr), + eth_dev->data->mac_addrs); + rte_tel_data_add_dict_string(d, "mac_addr", mac_addr); +@@ -6068,12 +6083,12 @@ eth_dev_handle_port_info(const char *cmd __rte_unused, + rte_tel_data_add_dict_container(d, "rxq_state", rxq_state, 0); + rte_tel_data_add_dict_container(d, "txq_state", txq_state, 0); + rte_tel_data_add_dict_int(d, "numa_node", eth_dev->data->numa_node); +- rte_tel_data_add_dict_int(d, "dev_flags", eth_dev->data->dev_flags); +- rte_tel_data_add_dict_int(d, "rx_offloads", ++ rte_tel_data_add_dict_u64(d, "dev_flags", eth_dev->data->dev_flags); ++ rte_tel_data_add_dict_u64(d, "rx_offloads", + eth_dev->data->dev_conf.rxmode.offloads); +- rte_tel_data_add_dict_int(d, "tx_offloads", ++ rte_tel_data_add_dict_u64(d, "tx_offloads", + eth_dev->data->dev_conf.txmode.offloads); +- rte_tel_data_add_dict_int(d, "ethdev_rss_hf", ++ rte_tel_data_add_dict_u64(d, "ethdev_rss_hf", + eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf); + + return 0; +diff --git a/dpdk/lib/ethdev/rte_ethdev.h b/dpdk/lib/ethdev/rte_ethdev.h +index c129ca1eaf..e73244822a 100644 +--- a/dpdk/lib/ethdev/rte_ethdev.h ++++ b/dpdk/lib/ethdev/rte_ethdev.h +@@ -3637,7 +3637,7 @@ rte_eth_tx_buffer_init(struct rte_eth_dev_tx_buffer *buffer, uint16_t size); + * for example, to count dropped packets, or to retry transmission of packets + * which cannot be sent, this function should be used to register a suitable + * callback function to implement the desired behaviour. +- * The example callback "rte_eth_count_unsent_packet_callback()" is also ++ * The example callback "rte_eth_tx_buffer_count_callback()" is also + * provided as reference. + * + * @param buffer +@@ -4177,10 +4177,7 @@ int rte_eth_fec_get_capability(uint16_t port_id, + * @param port_id + * The port identifier of the Ethernet device. + * @param fec_capa +- * A bitmask of enabled FEC modes. If AUTO bit is set, other +- * bits specify FEC modes which may be negotiated. If AUTO +- * bit is clear, specify FEC modes to be used (only one valid +- * mode per speed may be set). ++ * A bitmask with the current FEC mode. + * @return + * - (0) if successful. + * - (-ENOTSUP) if underlying hardware OR driver doesn't support. +@@ -4200,10 +4197,13 @@ int rte_eth_fec_get(uint16_t port_id, uint32_t *fec_capa); + * @param port_id + * The port identifier of the Ethernet device. + * @param fec_capa +- * A bitmask of allowed FEC modes. If AUTO bit is set, other +- * bits specify FEC modes which may be negotiated. If AUTO +- * bit is clear, specify FEC modes to be used (only one valid +- * mode per speed may be set). ++ * A bitmask of allowed FEC modes. ++ * If only the AUTO bit is set, the decision on which FEC ++ * mode to use will be made by HW/FW or driver. ++ * If the AUTO bit is set with some FEC modes, only specified ++ * FEC modes can be set. ++ * If AUTO bit is clear, specify FEC mode to be used ++ * (only one valid mode per speed may be set). + * @return + * - (0) if successful. + * - (-EINVAL) if the FEC mode is not valid. +@@ -4354,6 +4354,9 @@ int rte_eth_dev_mac_addr_remove(uint16_t port_id, + + /** + * Set the default MAC address. ++ * It replaces the address at index 0 of the MAC address list. ++ * If the address was already in the MAC address list, ++ * please remove it first. + * + * @param port_id + * The port identifier of the Ethernet device. +@@ -4364,6 +4367,7 @@ int rte_eth_dev_mac_addr_remove(uint16_t port_id, + * - (-ENOTSUP) if hardware doesn't support. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if MAC address is invalid. ++ * - (-EEXIST) if MAC address was already in the address list. + */ + int rte_eth_dev_default_mac_addr_set(uint16_t port_id, + struct rte_ether_addr *mac_addr); +diff --git a/dpdk/lib/ethdev/rte_flow.c b/dpdk/lib/ethdev/rte_flow.c +index 7d0c24366c..ae22755ee6 100644 +--- a/dpdk/lib/ethdev/rte_flow.c ++++ b/dpdk/lib/ethdev/rte_flow.c +@@ -654,7 +654,7 @@ rte_flow_conv_action_conf(void *buf, const size_t size, + if (src.rss->key_len && src.rss->key) { + off = RTE_ALIGN_CEIL(off, sizeof(*dst.rss->key)); + tmp = sizeof(*src.rss->key) * src.rss->key_len; +- if (size >= off + tmp) ++ if (size >= (uint64_t)off + (uint64_t)tmp) + dst.rss->key = rte_memcpy + ((void *)((uintptr_t)dst.rss + off), + src.rss->key, tmp); +@@ -663,7 +663,7 @@ rte_flow_conv_action_conf(void *buf, const size_t size, + if (src.rss->queue_num) { + off = RTE_ALIGN_CEIL(off, sizeof(*dst.rss->queue)); + tmp = sizeof(*src.rss->queue) * src.rss->queue_num; +- if (size >= off + tmp) ++ if (size >= (uint64_t)off + (uint64_t)tmp) + dst.rss->queue = rte_memcpy + ((void *)((uintptr_t)dst.rss + off), + src.rss->queue, tmp); +@@ -855,7 +855,15 @@ rte_flow_conv_actions(struct rte_flow_action *dst, + src -= num; + dst -= num; + do { +- if (src->conf) { ++ if (src->type == RTE_FLOW_ACTION_TYPE_INDIRECT) { ++ /* ++ * Indirect action conf fills the indirect action ++ * handler. Copy the action handle directly instead ++ * of duplicating the pointer memory. ++ */ ++ if (size) ++ dst->conf = src->conf; ++ } else if (src->conf) { + off = RTE_ALIGN_CEIL(off, sizeof(double)); + ret = rte_flow_conv_action_conf + ((void *)(data + off), +@@ -1879,6 +1887,8 @@ rte_flow_async_action_handle_query(uint16_t port_id, + const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error); + int ret; + ++ if (unlikely(!ops)) ++ return -rte_errno; + ret = ops->async_action_handle_query(dev, queue_id, op_attr, + action_handle, data, user_data, error); + return flow_err(port_id, ret, error); +diff --git a/dpdk/lib/eventdev/eventdev_pmd_vdev.h b/dpdk/lib/eventdev/eventdev_pmd_vdev.h +index 5fa9d699ac..bb433ba955 100644 +--- a/dpdk/lib/eventdev/eventdev_pmd_vdev.h ++++ b/dpdk/lib/eventdev/eventdev_pmd_vdev.h +@@ -45,7 +45,7 @@ extern "C" { + __rte_internal + static inline struct rte_eventdev * + rte_event_pmd_vdev_init(const char *name, size_t dev_private_size, +- int socket_id) ++ int socket_id, struct rte_vdev_device *vdev) + { + + struct rte_eventdev *eventdev; +@@ -67,6 +67,7 @@ rte_event_pmd_vdev_init(const char *name, size_t dev_private_size, + rte_panic("Cannot allocate memzone for private device" + " data"); + } ++ eventdev->dev = &vdev->device; + + return eventdev; + } +diff --git a/dpdk/lib/eventdev/rte_event_crypto_adapter.c b/dpdk/lib/eventdev/rte_event_crypto_adapter.c +index 3c585d7b0d..ea50e405a8 100644 +--- a/dpdk/lib/eventdev/rte_event_crypto_adapter.c ++++ b/dpdk/lib/eventdev/rte_event_crypto_adapter.c +@@ -240,9 +240,18 @@ eca_circular_buffer_flush_to_cdev(struct crypto_ops_circular_buffer *bufp, + n = *tailp - *headp; + else if (*tailp < *headp) + n = bufp->size - *headp; +- else { +- *nb_ops_flushed = 0; +- return 0; /* buffer empty */ ++ else { /* head == tail case */ ++ /* when head == tail, ++ * circ buff is either full(tail pointer roll over) or empty ++ */ ++ if (bufp->count != 0) { ++ /* circ buffer is full */ ++ n = bufp->count; ++ } else { ++ /* circ buffer is empty */ ++ *nb_ops_flushed = 0; ++ return 0; /* buffer empty */ ++ } + } + + *nb_ops_flushed = rte_cryptodev_enqueue_burst(cdev_id, qp_id, +@@ -497,6 +506,9 @@ eca_enq_to_cryptodev(struct event_crypto_adapter *adapter, struct rte_event *ev, + cdev_id, + qp_id, + &nb_enqueued); ++ stats->crypto_enq_count += nb_enqueued; ++ n += nb_enqueued; ++ + /** + * If some crypto ops failed to flush to cdev and + * space for another batch is not available, stop +@@ -507,9 +519,6 @@ eca_enq_to_cryptodev(struct event_crypto_adapter *adapter, struct rte_event *ev, + &qp_info->cbuf))) + adapter->stop_enq_to_cryptodev = true; + } +- +- stats->crypto_enq_count += nb_enqueued; +- n += nb_enqueued; + } + + return n; +@@ -585,14 +594,15 @@ eca_crypto_adapter_enq_run(struct event_crypto_adapter *adapter, + if (adapter->mode == RTE_EVENT_CRYPTO_ADAPTER_OP_NEW) + return 0; + +- if (unlikely(adapter->stop_enq_to_cryptodev)) { +- nb_enqueued += eca_crypto_enq_flush(adapter); ++ for (nb_enq = 0; nb_enq < max_enq; nb_enq += n) { + +- if (unlikely(adapter->stop_enq_to_cryptodev)) +- goto skip_event_dequeue_burst; +- } ++ if (unlikely(adapter->stop_enq_to_cryptodev)) { ++ nb_enqueued += eca_crypto_enq_flush(adapter); ++ ++ if (unlikely(adapter->stop_enq_to_cryptodev)) ++ break; ++ } + +- for (nb_enq = 0; nb_enq < max_enq; nb_enq += n) { + stats->event_poll_count++; + n = rte_event_dequeue_burst(event_dev_id, + event_port_id, ev, BATCH_SIZE, 0); +@@ -603,8 +613,6 @@ eca_crypto_adapter_enq_run(struct event_crypto_adapter *adapter, + nb_enqueued += eca_enq_to_cryptodev(adapter, ev, n); + } + +-skip_event_dequeue_burst: +- + if ((++adapter->transmit_loop_count & + (CRYPTO_ENQ_FLUSH_THRESHOLD - 1)) == 0) { + nb_enqueued += eca_crypto_enq_flush(adapter); +@@ -681,7 +689,7 @@ eca_circular_buffer_flush_to_evdev(struct event_crypto_adapter *adapter, + else + return 0; /* buffer empty */ + +- nb_ops_flushed = eca_ops_enqueue_burst(adapter, ops, n); ++ nb_ops_flushed = eca_ops_enqueue_burst(adapter, &ops[*headp], n); + bufp->count -= nb_ops_flushed; + if (!bufp->count) { + *headp = 0; +@@ -766,7 +774,7 @@ eca_crypto_adapter_deq_run(struct event_crypto_adapter *adapter, + for (i = nb_enqueued; i < n; i++) + eca_circular_buffer_add( + &adapter->ebuf, +- ops[nb_enqueued]); ++ ops[i]); + + check: + nb_deq += n; +diff --git a/dpdk/lib/eventdev/rte_event_crypto_adapter.h b/dpdk/lib/eventdev/rte_event_crypto_adapter.h +index 83d154a6ce..2a69290097 100644 +--- a/dpdk/lib/eventdev/rte_event_crypto_adapter.h ++++ b/dpdk/lib/eventdev/rte_event_crypto_adapter.h +@@ -595,6 +595,9 @@ int + rte_event_crypto_adapter_event_port_get(uint8_t id, uint8_t *event_port_id); + + /** ++ * @warning ++ * @b EXPERIMENTAL: this API may change without prior notice ++ * + * Retrieve vector limits for a given event dev and crypto dev pair. + * @see rte_event_crypto_adapter_vector_limits + * +@@ -610,6 +613,7 @@ rte_event_crypto_adapter_event_port_get(uint8_t id, uint8_t *event_port_id); + * - 0: Success. + * - <0: Error code on failure. + */ ++__rte_experimental + int rte_event_crypto_adapter_vector_limits_get( + uint8_t dev_id, uint16_t cdev_id, + struct rte_event_crypto_adapter_vector_limits *limits); +diff --git a/dpdk/lib/eventdev/rte_event_eth_rx_adapter.c b/dpdk/lib/eventdev/rte_event_eth_rx_adapter.c +index cf7bbd4d69..6636128378 100644 +--- a/dpdk/lib/eventdev/rte_event_eth_rx_adapter.c ++++ b/dpdk/lib/eventdev/rte_event_eth_rx_adapter.c +@@ -1906,6 +1906,13 @@ rxa_init_service(struct event_eth_rx_adapter *rx_adapter, uint8_t id) + if (rx_adapter->service_inited) + return 0; + ++ if (rte_mbuf_dyn_rx_timestamp_register( ++ &event_eth_rx_timestamp_dynfield_offset, ++ &event_eth_rx_timestamp_dynflag) != 0) { ++ RTE_EDEV_LOG_ERR("Error registering timestamp field in mbuf\n"); ++ return -rte_errno; ++ } ++ + memset(&service, 0, sizeof(service)); + snprintf(service.name, ETH_RX_ADAPTER_SERVICE_NAME_LEN, + "rte_event_eth_rx_adapter_%d", id); +@@ -2468,13 +2475,6 @@ rxa_create(uint8_t id, uint8_t dev_id, + if (conf_cb == rxa_default_conf_cb) + rx_adapter->default_cb_arg = 1; + +- if (rte_mbuf_dyn_rx_timestamp_register( +- &event_eth_rx_timestamp_dynfield_offset, +- &event_eth_rx_timestamp_dynflag) != 0) { +- RTE_EDEV_LOG_ERR("Error registering timestamp field in mbuf\n"); +- return -rte_errno; +- } +- + rte_eventdev_trace_eth_rx_adapter_create(id, dev_id, conf_cb, + conf_arg); + return 0; +@@ -3415,14 +3415,10 @@ rte_event_eth_rx_adapter_instance_get(uint16_t eth_dev_id, + if (!rte_event_eth_rx_adapter_caps_get(rx_adapter->eventdev_id, + eth_dev_id, + &caps)) { +- if (caps & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT) { +- ret = rxa_dev_instance_get(rx_adapter) ? +- rxa_dev_instance_get(rx_adapter) +- (eth_dev_id, +- rx_queue_id, +- rxa_inst_id) +- : -EINVAL; +- } ++ if (caps & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT && ++ rxa_dev_instance_get(rx_adapter)) ++ ret = rxa_dev_instance_get(rx_adapter)(eth_dev_id, rx_queue_id, ++ rxa_inst_id); + } + + /* return if entry found */ +diff --git a/dpdk/lib/eventdev/rte_event_eth_tx_adapter.c b/dpdk/lib/eventdev/rte_event_eth_tx_adapter.c +index 88309d2aaa..ba7a1c7f1b 100644 +--- a/dpdk/lib/eventdev/rte_event_eth_tx_adapter.c ++++ b/dpdk/lib/eventdev/rte_event_eth_tx_adapter.c +@@ -676,7 +676,7 @@ txa_service_func(void *args) + RTE_ETH_FOREACH_DEV(i) { + uint16_t q; + +- if (i == txa->dev_count) ++ if (i >= txa->dev_count) + break; + + dev = tdi[i].dev; +diff --git a/dpdk/lib/eventdev/rte_event_timer_adapter.c b/dpdk/lib/eventdev/rte_event_timer_adapter.c +index a0f14bf861..a13ddce627 100644 +--- a/dpdk/lib/eventdev/rte_event_timer_adapter.c ++++ b/dpdk/lib/eventdev/rte_event_timer_adapter.c +@@ -17,6 +17,7 @@ + #include + #include + #include ++#include + + #include "event_timer_adapter_pmd.h" + #include "eventdev_pmd.h" +@@ -699,13 +700,51 @@ swtim_callback(struct rte_timer *tim) + } + } + +-static __rte_always_inline uint64_t ++static __rte_always_inline int + get_timeout_cycles(struct rte_event_timer *evtim, +- const struct rte_event_timer_adapter *adapter) ++ const struct rte_event_timer_adapter *adapter, ++ uint64_t *timeout_cycles) + { +- struct swtim *sw = swtim_pmd_priv(adapter); +- uint64_t timeout_ns = evtim->timeout_ticks * sw->timer_tick_ns; +- return timeout_ns * rte_get_timer_hz() / NSECPERSEC; ++ static struct rte_reciprocal_u64 nsecpersec_inverse; ++ static uint64_t timer_hz; ++ uint64_t rem_cycles, secs_cycles = 0; ++ uint64_t secs, timeout_nsecs; ++ uint64_t nsecpersec; ++ struct swtim *sw; ++ ++ sw = swtim_pmd_priv(adapter); ++ nsecpersec = (uint64_t)NSECPERSEC; ++ ++ timeout_nsecs = evtim->timeout_ticks * sw->timer_tick_ns; ++ if (timeout_nsecs > sw->max_tmo_ns) ++ return -1; ++ if (timeout_nsecs < sw->timer_tick_ns) ++ return -2; ++ ++ /* Set these values in the first invocation */ ++ if (!timer_hz) { ++ timer_hz = rte_get_timer_hz(); ++ nsecpersec_inverse = rte_reciprocal_value_u64(nsecpersec); ++ } ++ ++ /* If timeout_nsecs > nsecpersec, decrease timeout_nsecs by the number ++ * of whole seconds it contains and convert that value to a number ++ * of cycles. This keeps timeout_nsecs in the interval [0..nsecpersec) ++ * in order to avoid overflow when we later multiply by timer_hz. ++ */ ++ if (timeout_nsecs > nsecpersec) { ++ secs = rte_reciprocal_divide_u64(timeout_nsecs, ++ &nsecpersec_inverse); ++ secs_cycles = secs * timer_hz; ++ timeout_nsecs -= secs * nsecpersec; ++ } ++ ++ rem_cycles = rte_reciprocal_divide_u64(timeout_nsecs * timer_hz, ++ &nsecpersec_inverse); ++ ++ *timeout_cycles = secs_cycles + rem_cycles; ++ ++ return 0; + } + + /* This function returns true if one or more (adapter) ticks have occurred since +@@ -739,23 +778,6 @@ swtim_did_tick(struct swtim *sw) + return false; + } + +-/* Check that event timer timeout value is in range */ +-static __rte_always_inline int +-check_timeout(struct rte_event_timer *evtim, +- const struct rte_event_timer_adapter *adapter) +-{ +- uint64_t tmo_nsec; +- struct swtim *sw = swtim_pmd_priv(adapter); +- +- tmo_nsec = evtim->timeout_ticks * sw->timer_tick_ns; +- if (tmo_nsec > sw->max_tmo_ns) +- return -1; +- if (tmo_nsec < sw->timer_tick_ns) +- return -2; +- +- return 0; +-} +- + /* Check that event timer event queue sched type matches destination event queue + * sched type + */ +@@ -798,17 +820,18 @@ swtim_service_func(void *arg) + sw->n_expired_timers); + sw->n_expired_timers = 0; + +- event_buffer_flush(&sw->buffer, +- adapter->data->event_dev_id, +- adapter->data->event_port_id, +- &nb_evs_flushed, +- &nb_evs_invalid); +- +- sw->stats.ev_enq_count += nb_evs_flushed; +- sw->stats.ev_inv_count += nb_evs_invalid; + sw->stats.adapter_tick_count++; + } + ++ event_buffer_flush(&sw->buffer, ++ adapter->data->event_dev_id, ++ adapter->data->event_port_id, ++ &nb_evs_flushed, ++ &nb_evs_invalid); ++ ++ sw->stats.ev_enq_count += nb_evs_flushed; ++ sw->stats.ev_inv_count += nb_evs_invalid; ++ + rte_event_maintain(adapter->data->event_dev_id, + adapter->data->event_port_id, 0); + +@@ -1140,21 +1163,6 @@ __swtim_arm_burst(const struct rte_event_timer_adapter *adapter, + break; + } + +- ret = check_timeout(evtims[i], adapter); +- if (unlikely(ret == -1)) { +- __atomic_store_n(&evtims[i]->state, +- RTE_EVENT_TIMER_ERROR_TOOLATE, +- __ATOMIC_RELAXED); +- rte_errno = EINVAL; +- break; +- } else if (unlikely(ret == -2)) { +- __atomic_store_n(&evtims[i]->state, +- RTE_EVENT_TIMER_ERROR_TOOEARLY, +- __ATOMIC_RELAXED); +- rte_errno = EINVAL; +- break; +- } +- + if (unlikely(check_destination_event_queue(evtims[i], + adapter) < 0)) { + __atomic_store_n(&evtims[i]->state, +@@ -1170,7 +1178,21 @@ __swtim_arm_burst(const struct rte_event_timer_adapter *adapter, + evtims[i]->impl_opaque[0] = (uintptr_t)tim; + evtims[i]->impl_opaque[1] = (uintptr_t)adapter; + +- cycles = get_timeout_cycles(evtims[i], adapter); ++ ret = get_timeout_cycles(evtims[i], adapter, &cycles); ++ if (unlikely(ret == -1)) { ++ __atomic_store_n(&evtims[i]->state, ++ RTE_EVENT_TIMER_ERROR_TOOLATE, ++ __ATOMIC_RELAXED); ++ rte_errno = EINVAL; ++ break; ++ } else if (unlikely(ret == -2)) { ++ __atomic_store_n(&evtims[i]->state, ++ RTE_EVENT_TIMER_ERROR_TOOEARLY, ++ __ATOMIC_RELAXED); ++ rte_errno = EINVAL; ++ break; ++ } ++ + ret = rte_timer_alt_reset(sw->timer_data_id, tim, cycles, + type, lcore_id, NULL, evtims[i]); + if (ret < 0) { +diff --git a/dpdk/lib/eventdev/rte_eventdev.c b/dpdk/lib/eventdev/rte_eventdev.c +index b0414206d9..04eeb76d4f 100644 +--- a/dpdk/lib/eventdev/rte_eventdev.c ++++ b/dpdk/lib/eventdev/rte_eventdev.c +@@ -99,6 +99,8 @@ rte_event_dev_info_get(uint8_t dev_id, struct rte_event_dev_info *dev_info) + dev_info->dequeue_timeout_ns = dev->data->dev_conf.dequeue_timeout_ns; + + dev_info->dev = dev->dev; ++ if (dev->dev != NULL && dev->dev->driver != NULL) ++ dev_info->driver_name = dev->dev->driver->name; + return 0; + } + +@@ -1678,7 +1680,7 @@ eventdev_build_telemetry_data(int dev_id, + if (xstat_names == NULL) + return -1; + +- ids = malloc((sizeof(unsigned int)) * num_xstats); ++ ids = malloc((sizeof(uint64_t)) * num_xstats); + if (ids == NULL) { + free(xstat_names); + return -1; +diff --git a/dpdk/lib/eventdev/version.map b/dpdk/lib/eventdev/version.map +index dd63ec6f68..56000271a4 100644 +--- a/dpdk/lib/eventdev/version.map ++++ b/dpdk/lib/eventdev/version.map +@@ -101,6 +101,7 @@ EXPERIMENTAL { + global: + + # added in 21.11 ++ __rte_eventdev_trace_maintain; + rte_event_eth_rx_adapter_create_with_params; + rte_event_eth_rx_adapter_queue_conf_get; + rte_event_eth_rx_adapter_queue_stats_get; +@@ -110,6 +111,7 @@ EXPERIMENTAL { + rte_event_eth_rx_adapter_event_port_get; + + # added in 22.07 ++ rte_event_crypto_adapter_vector_limits_get; + rte_event_port_quiesce; + rte_event_queue_attr_set; + +diff --git a/dpdk/lib/fib/dir24_8.c b/dpdk/lib/fib/dir24_8.c +index a8ba4f64ca..5f73b8a7f0 100644 +--- a/dpdk/lib/fib/dir24_8.c ++++ b/dpdk/lib/fib/dir24_8.c +@@ -388,9 +388,15 @@ modify_fib(struct dir24_8_tbl *dp, struct rte_rib *rib, uint32_t ip, + return ret; + ledge = redge + + (uint32_t)(1ULL << (32 - tmp_depth)); ++ /* ++ * we got to the end of address space ++ * and wrapped around ++ */ ++ if (ledge == 0) ++ break; + } else { + redge = ip + (uint32_t)(1ULL << (32 - depth)); +- if (ledge == redge) ++ if (ledge == redge && ledge != 0) + break; + ret = install_to_fib(dp, ledge, redge, + next_hop); +diff --git a/dpdk/lib/fib/trie.c b/dpdk/lib/fib/trie.c +index 3e780afdaf..09470e7287 100644 +--- a/dpdk/lib/fib/trie.c ++++ b/dpdk/lib/fib/trie.c +@@ -451,6 +451,14 @@ get_nxt_net(uint8_t *ip, uint8_t depth) + } + } + ++static int ++v6_addr_is_zero(const uint8_t ip[RTE_FIB6_IPV6_ADDR_SIZE]) ++{ ++ uint8_t ip_addr[RTE_FIB6_IPV6_ADDR_SIZE] = {0}; ++ ++ return rte_rib6_is_equal(ip, ip_addr); ++} ++ + static int + modify_dp(struct rte_trie_tbl *dp, struct rte_rib6 *rib, + const uint8_t ip[RTE_FIB6_IPV6_ADDR_SIZE], +@@ -484,11 +492,19 @@ modify_dp(struct rte_trie_tbl *dp, struct rte_rib6 *rib, + return ret; + get_nxt_net(redge, tmp_depth); + rte_rib6_copy_addr(ledge, redge); ++ /* ++ * we got to the end of address space ++ * and wrapped around ++ */ ++ if (v6_addr_is_zero(ledge)) ++ break; + } else { + rte_rib6_copy_addr(redge, ip); + get_nxt_net(redge, depth); +- if (rte_rib6_is_equal(ledge, redge)) ++ if (rte_rib6_is_equal(ledge, redge) && ++ !v6_addr_is_zero(ledge)) + break; ++ + ret = install_to_dp(dp, ledge, redge, + next_hop); + if (ret != 0) +diff --git a/dpdk/lib/gpudev/gpudev.c b/dpdk/lib/gpudev/gpudev.c +index 805719d00c..8f12abef23 100644 +--- a/dpdk/lib/gpudev/gpudev.c ++++ b/dpdk/lib/gpudev/gpudev.c +@@ -408,6 +408,7 @@ rte_gpu_callback_register(int16_t dev_id, enum rte_gpu_event event, + callback->function == function && + callback->user_data == user_data) { + GPU_LOG(INFO, "callback already registered"); ++ rte_rwlock_write_unlock(&gpu_callback_lock); + return 0; + } + } +@@ -415,7 +416,9 @@ rte_gpu_callback_register(int16_t dev_id, enum rte_gpu_event event, + callback = malloc(sizeof(*callback)); + if (callback == NULL) { + GPU_LOG(ERR, "cannot allocate callback"); +- return -ENOMEM; ++ rte_rwlock_write_unlock(&gpu_callback_lock); ++ rte_errno = ENOMEM; ++ return -rte_errno; + } + callback->function = function; + callback->user_data = user_data; +diff --git a/dpdk/lib/gpudev/gpudev_driver.h b/dpdk/lib/gpudev/gpudev_driver.h +index d5e2c8e1ef..42898c7c8b 100644 +--- a/dpdk/lib/gpudev/gpudev_driver.h ++++ b/dpdk/lib/gpudev/gpudev_driver.h +@@ -19,6 +19,10 @@ + #include + #include "rte_gpudev.h" + ++#ifdef __cplusplus ++extern "C" { ++#endif ++ + /* Flags indicate current state of device. */ + enum rte_gpu_state { + RTE_GPU_STATE_UNUSED, /* not initialized */ +@@ -106,4 +110,8 @@ int rte_gpu_release(struct rte_gpu *dev); + __rte_internal + void rte_gpu_notify(struct rte_gpu *dev, enum rte_gpu_event); + ++#ifdef __cplusplus ++} ++#endif ++ + #endif /* RTE_GPUDEV_DRIVER_H */ +diff --git a/dpdk/lib/gpudev/meson.build b/dpdk/lib/gpudev/meson.build +index 89a118f357..d21fadc052 100644 +--- a/dpdk/lib/gpudev/meson.build ++++ b/dpdk/lib/gpudev/meson.build +@@ -5,6 +5,10 @@ headers = files( + 'rte_gpudev.h', + ) + ++driver_sdk_headers = files( ++ 'gpudev_driver.h', ++) ++ + sources = files( + 'gpudev.c', + ) +diff --git a/dpdk/lib/graph/node.c b/dpdk/lib/graph/node.c +index fc6345de07..149414dcd9 100644 +--- a/dpdk/lib/graph/node.c ++++ b/dpdk/lib/graph/node.c +@@ -300,16 +300,16 @@ rte_node_edge_shrink(rte_node_t id, rte_edge_t size) + if (node->id == id) { + if (node->nb_edges < size) { + rte_errno = E2BIG; +- goto fail; ++ } else { ++ node->nb_edges = size; ++ rc = size; + } +- node->nb_edges = size; +- rc = size; + break; + } + } + +-fail: + graph_spinlock_unlock(); ++fail: + return rc; + } + +diff --git a/dpdk/lib/hash/rte_cuckoo_hash.c b/dpdk/lib/hash/rte_cuckoo_hash.c +index 829b79c89a..a08b5dd875 100644 +--- a/dpdk/lib/hash/rte_cuckoo_hash.c ++++ b/dpdk/lib/hash/rte_cuckoo_hash.c +@@ -1860,11 +1860,15 @@ compare_signatures(uint32_t *prim_hash_matches, uint32_t *sec_hash_matches, + _mm_load_si128( + (__m128i const *)prim_bkt->sig_current), + _mm_set1_epi16(sig))); ++ /* Extract the even-index bits only */ ++ *prim_hash_matches &= 0x5555; + /* Compare all signatures in the bucket */ + *sec_hash_matches = _mm_movemask_epi8(_mm_cmpeq_epi16( + _mm_load_si128( + (__m128i const *)sec_bkt->sig_current), + _mm_set1_epi16(sig))); ++ /* Extract the even-index bits only */ ++ *sec_hash_matches &= 0x5555; + break; + #elif defined(__ARM_NEON) + case RTE_HASH_COMPARE_NEON: { +diff --git a/dpdk/lib/hash/rte_thash.c b/dpdk/lib/hash/rte_thash.c +index 0249883b8d..2228af576b 100644 +--- a/dpdk/lib/hash/rte_thash.c ++++ b/dpdk/lib/hash/rte_thash.c +@@ -670,7 +670,7 @@ rte_thash_get_gfni_matrices(struct rte_thash_ctx *ctx) + } + + static inline uint8_t +-read_unaligned_byte(uint8_t *ptr, unsigned int len, unsigned int offset) ++read_unaligned_byte(uint8_t *ptr, unsigned int offset) + { + uint8_t ret = 0; + +@@ -681,13 +681,14 @@ read_unaligned_byte(uint8_t *ptr, unsigned int len, unsigned int offset) + (CHAR_BIT - (offset % CHAR_BIT)); + } + +- return ret >> (CHAR_BIT - len); ++ return ret; + } + + static inline uint32_t + read_unaligned_bits(uint8_t *ptr, int len, int offset) + { + uint32_t ret = 0; ++ int shift; + + len = RTE_MAX(len, 0); + len = RTE_MIN(len, (int)(sizeof(uint32_t) * CHAR_BIT)); +@@ -695,13 +696,14 @@ read_unaligned_bits(uint8_t *ptr, int len, int offset) + while (len > 0) { + ret <<= CHAR_BIT; + +- ret |= read_unaligned_byte(ptr, RTE_MIN(len, CHAR_BIT), +- offset); ++ ret |= read_unaligned_byte(ptr, offset); + offset += CHAR_BIT; + len -= CHAR_BIT; + } + +- return ret; ++ shift = (len == 0) ? 0 : ++ (CHAR_BIT - ((len + CHAR_BIT) % CHAR_BIT)); ++ return ret >> shift; + } + + /* returns mask for len bits with given offset inside byte */ +diff --git a/dpdk/lib/hash/rte_thash_x86_gfni.h b/dpdk/lib/hash/rte_thash_x86_gfni.h +index 880739b710..7bb76ac1bb 100644 +--- a/dpdk/lib/hash/rte_thash_x86_gfni.h ++++ b/dpdk/lib/hash/rte_thash_x86_gfni.h +@@ -88,8 +88,10 @@ __rte_thash_gfni(const uint64_t *mtrx, const uint8_t *tuple, + const __m512i shift_8 = _mm512_set1_epi8(8); + __m512i xor_acc = _mm512_setzero_si512(); + __m512i perm_bytes = _mm512_setzero_si512(); +- __m512i vals, matrixes, tuple_bytes, tuple_bytes_2; +- __mmask64 load_mask, permute_mask, permute_mask_2; ++ __m512i vals, matrixes, tuple_bytes_2; ++ __m512i tuple_bytes = _mm512_setzero_si512(); ++ __mmask64 load_mask, permute_mask_2; ++ __mmask64 permute_mask = 0; + int chunk_len = 0, i = 0; + uint8_t mtrx_msk; + const int prepend = 3; +diff --git a/dpdk/lib/ipsec/esp_outb.c b/dpdk/lib/ipsec/esp_outb.c +index 9cbd9202f6..ec87b1dce2 100644 +--- a/dpdk/lib/ipsec/esp_outb.c ++++ b/dpdk/lib/ipsec/esp_outb.c +@@ -198,7 +198,7 @@ outb_tun_pkt_prepare(struct rte_ipsec_sa *sa, rte_be64_t sqc, + struct rte_udp_hdr *udph = (struct rte_udp_hdr *) + (ph + sa->hdr_len - sizeof(struct rte_udp_hdr)); + udph->dgram_len = rte_cpu_to_be_16(mb->pkt_len - sqh_len - +- sa->hdr_l3_off - sa->hdr_len); ++ sa->hdr_len + sizeof(struct rte_udp_hdr)); + } + + /* update original and new ip header fields */ +diff --git a/dpdk/lib/ipsec/sa.c b/dpdk/lib/ipsec/sa.c +index 59a547637d..2297bd6d72 100644 +--- a/dpdk/lib/ipsec/sa.c ++++ b/dpdk/lib/ipsec/sa.c +@@ -371,7 +371,7 @@ esp_outb_tun_init(struct rte_ipsec_sa *sa, const struct rte_ipsec_sa_prm *prm) + + /* update l2_len and l3_len fields for outbound mbuf */ + sa->tx_offload.val = rte_mbuf_tx_offload(sa->hdr_l3_off, +- sa->hdr_len - sa->hdr_l3_off, 0, 0, 0, 0, 0); ++ prm->tun.hdr_len - sa->hdr_l3_off, 0, 0, 0, 0, 0); + + esp_outb_init(sa, sa->hdr_len, prm->ipsec_xform.esn.value); + } +diff --git a/dpdk/lib/kni/rte_kni.c b/dpdk/lib/kni/rte_kni.c +index 8ab6c47153..bfa6a001ff 100644 +--- a/dpdk/lib/kni/rte_kni.c ++++ b/dpdk/lib/kni/rte_kni.c +@@ -634,8 +634,8 @@ rte_kni_rx_burst(struct rte_kni *kni, struct rte_mbuf **mbufs, unsigned int num) + { + unsigned int ret = kni_fifo_get(kni->tx_q, (void **)mbufs, num); + +- /* If buffers removed, allocate mbufs and then put them into alloc_q */ +- if (ret) ++ /* If buffers removed or alloc_q is empty, allocate mbufs and then put them into alloc_q */ ++ if (ret || (kni_fifo_count(kni->alloc_q) == 0)) + kni_allocate_mbufs(kni); + + return ret; +diff --git a/dpdk/lib/kvargs/rte_kvargs.h b/dpdk/lib/kvargs/rte_kvargs.h +index 359a9f5b09..4900b750bc 100644 +--- a/dpdk/lib/kvargs/rte_kvargs.h ++++ b/dpdk/lib/kvargs/rte_kvargs.h +@@ -36,7 +36,19 @@ extern "C" { + /** separator character used between key and value */ + #define RTE_KVARGS_KV_DELIM "=" + +-/** Type of callback function used by rte_kvargs_process() */ ++/** ++ * Callback prototype used by rte_kvargs_process(). ++ * ++ * @param key ++ * The key to consider, it will not be NULL. ++ * @param value ++ * The value corresponding to the key, it may be NULL (e.g. only with key) ++ * @param opaque ++ * An opaque pointer coming from the caller. ++ * @return ++ * - >=0 handle key success. ++ * - <0 on error. ++ */ + typedef int (*arg_handler_t)(const char *key, const char *value, void *opaque); + + /** A key/value association */ +diff --git a/dpdk/lib/mbuf/rte_mbuf_core.h b/dpdk/lib/mbuf/rte_mbuf_core.h +index a30e1e0eaf..3ab7be49fa 100644 +--- a/dpdk/lib/mbuf/rte_mbuf_core.h ++++ b/dpdk/lib/mbuf/rte_mbuf_core.h +@@ -584,8 +584,8 @@ struct rte_mbuf { + * @see rte_event_eth_tx_adapter_txq_set() + */ + } txadapter; /**< Eventdev ethdev Tx adapter */ +- /**< User defined tags. See rte_distributor_process() */ + uint32_t usr; ++ /**< User defined tags. See rte_distributor_process() */ + } hash; /**< hash information */ + }; + +diff --git a/dpdk/lib/mbuf/rte_mbuf_ptype.h b/dpdk/lib/mbuf/rte_mbuf_ptype.h +index 17a2dd3576..f2276e2909 100644 +--- a/dpdk/lib/mbuf/rte_mbuf_ptype.h ++++ b/dpdk/lib/mbuf/rte_mbuf_ptype.h +@@ -419,10 +419,10 @@ extern "C" { + * + * Packet format: + * <'ether type'=0x0800 +- * | 'version'=4, 'protocol'=51> ++ * | 'version'=4, 'protocol'=50> + * or, + * <'ether type'=0x86DD +- * | 'version'=6, 'next header'=51> ++ * | 'version'=6, 'next header'=50> + */ + #define RTE_PTYPE_TUNNEL_ESP 0x00009000 + /** +diff --git a/dpdk/lib/member/rte_member.h b/dpdk/lib/member/rte_member.h +index 072a253c89..d08b143e51 100644 +--- a/dpdk/lib/member/rte_member.h ++++ b/dpdk/lib/member/rte_member.h +@@ -314,6 +314,7 @@ struct rte_member_parameters { + * for bucket location. + * For vBF type, these two hashes and their combinations are used as + * hash locations to index the bit array. ++ * For Sketch type, these seeds are not used. + */ + uint32_t prim_hash_seed; + +diff --git a/dpdk/lib/member/rte_member_sketch.c b/dpdk/lib/member/rte_member_sketch.c +index 524ba77620..d5f35aabe9 100644 +--- a/dpdk/lib/member/rte_member_sketch.c ++++ b/dpdk/lib/member/rte_member_sketch.c +@@ -227,7 +227,6 @@ rte_member_create_sketch(struct rte_member_setsum *ss, + goto error_runtime; + } + +- rte_srand(ss->prim_hash_seed); + for (i = 0; i < ss->num_row; i++) + ss->hash_seeds[i] = rte_rand(); + +diff --git a/dpdk/lib/mempool/rte_mempool.c b/dpdk/lib/mempool/rte_mempool.c +index f33f455790..3de857abf5 100644 +--- a/dpdk/lib/mempool/rte_mempool.c ++++ b/dpdk/lib/mempool/rte_mempool.c +@@ -915,6 +915,22 @@ rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size, + STAILQ_INIT(&mp->elt_list); + STAILQ_INIT(&mp->mem_list); + ++ /* ++ * Since we have 4 combinations of the SP/SC/MP/MC examine the flags to ++ * set the correct index into the table of ops structs. ++ */ ++ if ((flags & RTE_MEMPOOL_F_SP_PUT) && (flags & RTE_MEMPOOL_F_SC_GET)) ++ ret = rte_mempool_set_ops_byname(mp, "ring_sp_sc", NULL); ++ else if (flags & RTE_MEMPOOL_F_SP_PUT) ++ ret = rte_mempool_set_ops_byname(mp, "ring_sp_mc", NULL); ++ else if (flags & RTE_MEMPOOL_F_SC_GET) ++ ret = rte_mempool_set_ops_byname(mp, "ring_mp_sc", NULL); ++ else ++ ret = rte_mempool_set_ops_byname(mp, "ring_mp_mc", NULL); ++ ++ if (ret) ++ goto exit_unlock; ++ + /* + * local_cache pointer is set even if cache_size is zero. + * The local_cache points to just past the elt_pa[] array. +@@ -955,7 +971,6 @@ rte_mempool_create(const char *name, unsigned n, unsigned elt_size, + rte_mempool_obj_cb_t *obj_init, void *obj_init_arg, + int socket_id, unsigned flags) + { +- int ret; + struct rte_mempool *mp; + + mp = rte_mempool_create_empty(name, n, elt_size, cache_size, +@@ -963,22 +978,6 @@ rte_mempool_create(const char *name, unsigned n, unsigned elt_size, + if (mp == NULL) + return NULL; + +- /* +- * Since we have 4 combinations of the SP/SC/MP/MC examine the flags to +- * set the correct index into the table of ops structs. +- */ +- if ((flags & RTE_MEMPOOL_F_SP_PUT) && (flags & RTE_MEMPOOL_F_SC_GET)) +- ret = rte_mempool_set_ops_byname(mp, "ring_sp_sc", NULL); +- else if (flags & RTE_MEMPOOL_F_SP_PUT) +- ret = rte_mempool_set_ops_byname(mp, "ring_sp_mc", NULL); +- else if (flags & RTE_MEMPOOL_F_SC_GET) +- ret = rte_mempool_set_ops_byname(mp, "ring_mp_sc", NULL); +- else +- ret = rte_mempool_set_ops_byname(mp, "ring_mp_mc", NULL); +- +- if (ret) +- goto fail; +- + /* call the mempool priv initializer */ + if (mp_init) + mp_init(mp, mp_init_arg); +@@ -1500,27 +1499,27 @@ mempool_info_cb(struct rte_mempool *mp, void *arg) + return; + + rte_tel_data_add_dict_string(info->d, "name", mp->name); +- rte_tel_data_add_dict_int(info->d, "pool_id", mp->pool_id); +- rte_tel_data_add_dict_int(info->d, "flags", mp->flags); ++ rte_tel_data_add_dict_u64(info->d, "pool_id", mp->pool_id); ++ rte_tel_data_add_dict_u64(info->d, "flags", mp->flags); + rte_tel_data_add_dict_int(info->d, "socket_id", mp->socket_id); +- rte_tel_data_add_dict_int(info->d, "size", mp->size); +- rte_tel_data_add_dict_int(info->d, "cache_size", mp->cache_size); +- rte_tel_data_add_dict_int(info->d, "elt_size", mp->elt_size); +- rte_tel_data_add_dict_int(info->d, "header_size", mp->header_size); +- rte_tel_data_add_dict_int(info->d, "trailer_size", mp->trailer_size); +- rte_tel_data_add_dict_int(info->d, "private_data_size", ++ rte_tel_data_add_dict_u64(info->d, "size", mp->size); ++ rte_tel_data_add_dict_u64(info->d, "cache_size", mp->cache_size); ++ rte_tel_data_add_dict_u64(info->d, "elt_size", mp->elt_size); ++ rte_tel_data_add_dict_u64(info->d, "header_size", mp->header_size); ++ rte_tel_data_add_dict_u64(info->d, "trailer_size", mp->trailer_size); ++ rte_tel_data_add_dict_u64(info->d, "private_data_size", + mp->private_data_size); + rte_tel_data_add_dict_int(info->d, "ops_index", mp->ops_index); +- rte_tel_data_add_dict_int(info->d, "populated_size", ++ rte_tel_data_add_dict_u64(info->d, "populated_size", + mp->populated_size); + + mz = mp->mz; + rte_tel_data_add_dict_string(info->d, "mz_name", mz->name); +- rte_tel_data_add_dict_int(info->d, "mz_len", mz->len); +- rte_tel_data_add_dict_int(info->d, "mz_hugepage_sz", ++ rte_tel_data_add_dict_u64(info->d, "mz_len", mz->len); ++ rte_tel_data_add_dict_u64(info->d, "mz_hugepage_sz", + mz->hugepage_sz); + rte_tel_data_add_dict_int(info->d, "mz_socket_id", mz->socket_id); +- rte_tel_data_add_dict_int(info->d, "mz_flags", mz->flags); ++ rte_tel_data_add_dict_u64(info->d, "mz_flags", mz->flags); + } + + static int +diff --git a/dpdk/lib/mempool/rte_mempool.h b/dpdk/lib/mempool/rte_mempool.h +index 9f530db24b..4a8a2d5dcb 100644 +--- a/dpdk/lib/mempool/rte_mempool.h ++++ b/dpdk/lib/mempool/rte_mempool.h +@@ -465,13 +465,19 @@ typedef int (*rte_mempool_alloc_t)(struct rte_mempool *mp); + typedef void (*rte_mempool_free_t)(struct rte_mempool *mp); + + /** +- * Enqueue an object into the external pool. ++ * Enqueue 'n' objects into the external pool. ++ * @return ++ * - 0: Success ++ * - <0: Error + */ + typedef int (*rte_mempool_enqueue_t)(struct rte_mempool *mp, + void * const *obj_table, unsigned int n); + + /** +- * Dequeue an object from the external pool. ++ * Dequeue 'n' objects from the external pool. ++ * @return ++ * - 0: Success ++ * - <0: Error + */ + typedef int (*rte_mempool_dequeue_t)(struct rte_mempool *mp, + void **obj_table, unsigned int n); +@@ -1484,7 +1490,7 @@ rte_mempool_put(struct rte_mempool *mp, void *obj) + * @param cache + * A pointer to a mempool cache structure. May be NULL if not needed. + * @return +- * - >=0: Success; number of objects supplied. ++ * - 0: Success. + * - <0: Error; code of driver dequeue function. + */ + static __rte_always_inline int +diff --git a/dpdk/lib/meter/rte_meter.h b/dpdk/lib/meter/rte_meter.h +index 0932645d0a..35e2675028 100644 +--- a/dpdk/lib/meter/rte_meter.h ++++ b/dpdk/lib/meter/rte_meter.h +@@ -127,9 +127,6 @@ int + rte_meter_trtcm_profile_config(struct rte_meter_trtcm_profile *p, + struct rte_meter_trtcm_params *params); + /** +- * @warning +- * @b EXPERIMENTAL: this API may change without prior notice +- * + * trTCM RFC 4115 profile configuration + * + * @param p +@@ -173,9 +170,6 @@ rte_meter_trtcm_config(struct rte_meter_trtcm *m, + struct rte_meter_trtcm_profile *p); + + /** +- * @warning +- * @b EXPERIMENTAL: this API may change without prior notice +- * + * trTCM RFC 4115 configuration per metered traffic flow + * + * @param m +@@ -276,9 +270,6 @@ rte_meter_trtcm_color_aware_check(struct rte_meter_trtcm *m, + enum rte_color pkt_color); + + /** +- * @warning +- * @b EXPERIMENTAL: this API may change without prior notice +- * + * trTCM RFC4115 color blind traffic metering + * + * @param m +@@ -300,9 +291,6 @@ rte_meter_trtcm_rfc4115_color_blind_check( + uint32_t pkt_len); + + /** +- * @warning +- * @b EXPERIMENTAL: this API may change without prior notice +- * + * trTCM RFC4115 color aware traffic metering + * + * @param m +diff --git a/dpdk/lib/net/rte_ip.h b/dpdk/lib/net/rte_ip.h +index 9c8e8206f0..0cafb980ef 100644 +--- a/dpdk/lib/net/rte_ip.h ++++ b/dpdk/lib/net/rte_ip.h +@@ -514,7 +514,7 @@ rte_ipv4_udptcp_cksum_verify(const struct rte_ipv4_hdr *ipv4_hdr, + * Return 0 if the checksum is correct, else -1. + */ + __rte_experimental +-static inline uint16_t ++static inline int + rte_ipv4_udptcp_cksum_mbuf_verify(const struct rte_mbuf *m, + const struct rte_ipv4_hdr *ipv4_hdr, + uint16_t l4_off) +diff --git a/dpdk/lib/pci/rte_pci.h b/dpdk/lib/pci/rte_pci.h +index 5088157e74..9876c3fb9d 100644 +--- a/dpdk/lib/pci/rte_pci.h ++++ b/dpdk/lib/pci/rte_pci.h +@@ -45,6 +45,7 @@ extern "C" { + #define RTE_PCI_EXT_CAP_ID_ERR 0x01 /* Advanced Error Reporting */ + #define RTE_PCI_EXT_CAP_ID_DSN 0x03 /* Device Serial Number */ + #define RTE_PCI_EXT_CAP_ID_SRIOV 0x10 /* SR-IOV*/ ++#define RTE_PCI_EXT_CAP_ID_PASID 0x1B /* Process Address Space ID */ + + /* Single Root I/O Virtualization */ + #define RTE_PCI_SRIOV_CAP 0x04 /* SR-IOV Capabilities */ +@@ -58,6 +59,9 @@ extern "C" { + #define RTE_PCI_SRIOV_VF_DID 0x1a /* VF Device ID */ + #define RTE_PCI_SRIOV_SUP_PGSIZE 0x1c /* Supported Page Sizes */ + ++ ++/* Process Address Space ID (RTE_PCI_EXT_CAP_ID_PASID) */ ++#define RTE_PCI_PASID_CTRL 0x06 /* PASID control register */ + /** Formatting string for PCI device identifier: Ex: 0000:00:01.0 */ + #define PCI_PRI_FMT "%.4" PRIx32 ":%.2" PRIx8 ":%.2" PRIx8 ".%" PRIx8 + #define PCI_PRI_STR_SIZE sizeof("XXXXXXXX:XX:XX.X") +@@ -104,8 +108,7 @@ struct rte_pci_addr { + + /** + * Utility function to write a pci device name, this device name can later be +- * used to retrieve the corresponding rte_pci_addr using eal_parse_pci_* +- * BDF helpers. ++ * used to retrieve the corresponding rte_pci_addr using rte_pci_addr_parse(). + * + * @param addr + * The PCI Bus-Device-Function address +diff --git a/dpdk/lib/pdump/rte_pdump.c b/dpdk/lib/pdump/rte_pdump.c +index a81544cb57..63835a1a67 100644 +--- a/dpdk/lib/pdump/rte_pdump.c ++++ b/dpdk/lib/pdump/rte_pdump.c +@@ -134,7 +134,7 @@ pdump_copy(uint16_t port_id, uint16_t queue, + + __atomic_fetch_add(&stats->accepted, d_pkts, __ATOMIC_RELAXED); + +- ring_enq = rte_ring_enqueue_burst(ring, (void *)dup_bufs, d_pkts, NULL); ++ ring_enq = rte_ring_enqueue_burst(ring, (void *)&dup_bufs[0], d_pkts, NULL); + if (unlikely(ring_enq < d_pkts)) { + unsigned int drops = d_pkts - ring_enq; + +@@ -564,9 +564,10 @@ pdump_prepare_client_request(const char *device, uint16_t queue, + if (rte_mp_request_sync(&mp_req, &mp_reply, &ts) == 0) { + mp_rep = &mp_reply.msgs[0]; + resp = (struct pdump_response *)mp_rep->param; +- rte_errno = resp->err_value; +- if (!resp->err_value) ++ if (resp->err_value == 0) + ret = 0; ++ else ++ rte_errno = -resp->err_value; + free(mp_reply.msgs); + } + +diff --git a/dpdk/lib/pipeline/rte_swx_pipeline.c b/dpdk/lib/pipeline/rte_swx_pipeline.c +index 0e631dea2b..084c614639 100644 +--- a/dpdk/lib/pipeline/rte_swx_pipeline.c ++++ b/dpdk/lib/pipeline/rte_swx_pipeline.c +@@ -8450,6 +8450,7 @@ table_build_free(struct rte_swx_pipeline *p) + free(p->table_stats[i].n_pkts_action); + + free(p->table_stats); ++ p->table_stats = NULL; + } + } + +@@ -9364,6 +9365,7 @@ learner_build_free(struct rte_swx_pipeline *p) + free(p->learner_stats[i].n_pkts_action); + + free(p->learner_stats); ++ p->learner_stats = NULL; + } + } + +diff --git a/dpdk/lib/rawdev/rte_rawdev.c b/dpdk/lib/rawdev/rte_rawdev.c +index 5fbdb94229..891e79dcd7 100644 +--- a/dpdk/lib/rawdev/rte_rawdev.c ++++ b/dpdk/lib/rawdev/rte_rawdev.c +@@ -505,8 +505,7 @@ rte_rawdev_pmd_allocate(const char *name, size_t dev_priv_size, int socket_id) + uint16_t dev_id; + + if (rte_rawdev_pmd_get_named_dev(name) != NULL) { +- RTE_RDEV_ERR("Event device with name %s already allocated!", +- name); ++ RTE_RDEV_ERR("Raw device with name %s already allocated!", name); + return NULL; + } + +diff --git a/dpdk/lib/reorder/rte_reorder.c b/dpdk/lib/reorder/rte_reorder.c +index 385ee479da..bc85b83b14 100644 +--- a/dpdk/lib/reorder/rte_reorder.c ++++ b/dpdk/lib/reorder/rte_reorder.c +@@ -60,6 +60,11 @@ rte_reorder_init(struct rte_reorder_buffer *b, unsigned int bufsize, + { + const unsigned int min_bufsize = sizeof(*b) + + (2 * size * sizeof(struct rte_mbuf *)); ++ static const struct rte_mbuf_dynfield reorder_seqn_dynfield_desc = { ++ .name = RTE_REORDER_SEQN_DYNFIELD_NAME, ++ .size = sizeof(rte_reorder_seqn_t), ++ .align = __alignof__(rte_reorder_seqn_t), ++ }; + + if (b == NULL) { + RTE_LOG(ERR, REORDER, "Invalid reorder buffer parameter:" +@@ -86,6 +91,15 @@ rte_reorder_init(struct rte_reorder_buffer *b, unsigned int bufsize, + return NULL; + } + ++ rte_reorder_seqn_dynfield_offset = rte_mbuf_dynfield_register(&reorder_seqn_dynfield_desc); ++ if (rte_reorder_seqn_dynfield_offset < 0) { ++ RTE_LOG(ERR, REORDER, ++ "Failed to register mbuf field for reorder sequence number, rte_errno: %i\n", ++ rte_errno); ++ rte_errno = ENOMEM; ++ return NULL; ++ } ++ + memset(b, 0, bufsize); + strlcpy(b->name, name, sizeof(b->name)); + b->memsize = bufsize; +@@ -98,21 +112,45 @@ rte_reorder_init(struct rte_reorder_buffer *b, unsigned int bufsize, + return b; + } + ++/* ++ * Insert new entry into global list. ++ * Returns pointer to already inserted entry if such exists, or to newly inserted one. ++ */ ++static struct rte_tailq_entry * ++rte_reorder_entry_insert(struct rte_tailq_entry *new_te) ++{ ++ struct rte_reorder_list *reorder_list; ++ struct rte_reorder_buffer *b, *nb; ++ struct rte_tailq_entry *te; ++ ++ rte_mcfg_tailq_write_lock(); ++ ++ reorder_list = RTE_TAILQ_CAST(rte_reorder_tailq.head, rte_reorder_list); ++ /* guarantee there's no existing */ ++ TAILQ_FOREACH(te, reorder_list, next) { ++ b = (struct rte_reorder_buffer *) te->data; ++ nb = (struct rte_reorder_buffer *) new_te->data; ++ if (strncmp(nb->name, b->name, RTE_REORDER_NAMESIZE) == 0) ++ break; ++ } ++ ++ if (te == NULL) { ++ TAILQ_INSERT_TAIL(reorder_list, new_te, next); ++ te = new_te; ++ } ++ ++ rte_mcfg_tailq_write_unlock(); ++ ++ return te; ++} ++ + struct rte_reorder_buffer* + rte_reorder_create(const char *name, unsigned socket_id, unsigned int size) + { + struct rte_reorder_buffer *b = NULL; +- struct rte_tailq_entry *te; +- struct rte_reorder_list *reorder_list; ++ struct rte_tailq_entry *te, *te_inserted; + const unsigned int bufsize = sizeof(struct rte_reorder_buffer) + + (2 * size * sizeof(struct rte_mbuf *)); +- static const struct rte_mbuf_dynfield reorder_seqn_dynfield_desc = { +- .name = RTE_REORDER_SEQN_DYNFIELD_NAME, +- .size = sizeof(rte_reorder_seqn_t), +- .align = __alignof__(rte_reorder_seqn_t), +- }; +- +- reorder_list = RTE_TAILQ_CAST(rte_reorder_tailq.head, rte_reorder_list); + + /* Check user arguments. */ + if (!rte_is_power_of_2(size)) { +@@ -128,32 +166,12 @@ rte_reorder_create(const char *name, unsigned socket_id, unsigned int size) + return NULL; + } + +- rte_reorder_seqn_dynfield_offset = +- rte_mbuf_dynfield_register(&reorder_seqn_dynfield_desc); +- if (rte_reorder_seqn_dynfield_offset < 0) { +- RTE_LOG(ERR, REORDER, "Failed to register mbuf field for reorder sequence number\n"); +- rte_errno = ENOMEM; +- return NULL; +- } +- +- rte_mcfg_tailq_write_lock(); +- +- /* guarantee there's no existing */ +- TAILQ_FOREACH(te, reorder_list, next) { +- b = (struct rte_reorder_buffer *) te->data; +- if (strncmp(name, b->name, RTE_REORDER_NAMESIZE) == 0) +- break; +- } +- if (te != NULL) +- goto exit; +- + /* allocate tailq entry */ + te = rte_zmalloc("REORDER_TAILQ_ENTRY", sizeof(*te), 0); + if (te == NULL) { + RTE_LOG(ERR, REORDER, "Failed to allocate tailq entry\n"); + rte_errno = ENOMEM; +- b = NULL; +- goto exit; ++ return NULL; + } + + /* Allocate memory to store the reorder buffer structure. */ +@@ -162,14 +180,23 @@ rte_reorder_create(const char *name, unsigned socket_id, unsigned int size) + RTE_LOG(ERR, REORDER, "Memzone allocation failed\n"); + rte_errno = ENOMEM; + rte_free(te); ++ return NULL; + } else { +- rte_reorder_init(b, bufsize, name, size); ++ if (rte_reorder_init(b, bufsize, name, size) == NULL) { ++ rte_free(b); ++ rte_free(te); ++ return NULL; ++ } + te->data = (void *)b; +- TAILQ_INSERT_TAIL(reorder_list, te, next); + } + +-exit: +- rte_mcfg_tailq_write_unlock(); ++ te_inserted = rte_reorder_entry_insert(te); ++ if (te_inserted != te) { ++ rte_free(b); ++ rte_free(te); ++ return te_inserted->data; ++ } ++ + return b; + } + +@@ -389,6 +416,7 @@ rte_reorder_drain(struct rte_reorder_buffer *b, struct rte_mbuf **mbufs, + /* Try to fetch requested number of mbufs from ready buffer */ + while ((drain_cnt < max_mbufs) && (ready_buf->tail != ready_buf->head)) { + mbufs[drain_cnt++] = ready_buf->entries[ready_buf->tail]; ++ ready_buf->entries[ready_buf->tail] = NULL; + ready_buf->tail = (ready_buf->tail + 1) & ready_buf->mask; + } + +diff --git a/dpdk/lib/reorder/rte_reorder.h b/dpdk/lib/reorder/rte_reorder.h +index 5abdb258e2..79c758f450 100644 +--- a/dpdk/lib/reorder/rte_reorder.h ++++ b/dpdk/lib/reorder/rte_reorder.h +@@ -82,6 +82,7 @@ rte_reorder_create(const char *name, unsigned socket_id, unsigned int size); + * The initialized reorder buffer instance, or NULL on error + * On error case, rte_errno will be set appropriately: + * - EINVAL - invalid parameters ++ * - ENOMEM - not enough memory to register dynamic field + */ + struct rte_reorder_buffer * + rte_reorder_init(struct rte_reorder_buffer *b, unsigned int bufsize, +diff --git a/dpdk/lib/ring/rte_ring.c b/dpdk/lib/ring/rte_ring.c +index cddaf6b287..d068ca4d37 100644 +--- a/dpdk/lib/ring/rte_ring.c ++++ b/dpdk/lib/ring/rte_ring.c +@@ -332,11 +332,6 @@ rte_ring_free(struct rte_ring *r) + return; + } + +- if (rte_memzone_free(r->memzone) != 0) { +- RTE_LOG(ERR, RING, "Cannot free memory\n"); +- return; +- } +- + ring_list = RTE_TAILQ_CAST(rte_ring_tailq.head, rte_ring_list); + rte_mcfg_tailq_write_lock(); + +@@ -355,6 +350,9 @@ rte_ring_free(struct rte_ring *r) + + rte_mcfg_tailq_write_unlock(); + ++ if (rte_memzone_free(r->memzone) != 0) ++ RTE_LOG(ERR, RING, "Cannot free memory\n"); ++ + rte_free(te); + } + +diff --git a/dpdk/lib/ring/rte_ring_elem_pvt.h b/dpdk/lib/ring/rte_ring_elem_pvt.h +index 83788c56e6..4b80f58980 100644 +--- a/dpdk/lib/ring/rte_ring_elem_pvt.h ++++ b/dpdk/lib/ring/rte_ring_elem_pvt.h +@@ -10,6 +10,12 @@ + #ifndef _RTE_RING_ELEM_PVT_H_ + #define _RTE_RING_ELEM_PVT_H_ + ++#if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION >= 120000) ++#pragma GCC diagnostic push ++#pragma GCC diagnostic ignored "-Wstringop-overflow" ++#pragma GCC diagnostic ignored "-Wstringop-overread" ++#endif ++ + static __rte_always_inline void + __rte_ring_enqueue_elems_32(struct rte_ring *r, const uint32_t size, + uint32_t idx, const void *obj_table, uint32_t n) +@@ -188,12 +194,12 @@ __rte_ring_dequeue_elems_32(struct rte_ring *r, const uint32_t size, + } + + static __rte_always_inline void +-__rte_ring_dequeue_elems_64(struct rte_ring *r, uint32_t prod_head, ++__rte_ring_dequeue_elems_64(struct rte_ring *r, uint32_t cons_head, + void *obj_table, uint32_t n) + { + unsigned int i; + const uint32_t size = r->size; +- uint32_t idx = prod_head & r->mask; ++ uint32_t idx = cons_head & r->mask; + uint64_t *ring = (uint64_t *)&r[1]; + unaligned_uint64_t *obj = (unaligned_uint64_t *)obj_table; + if (likely(idx + n <= size)) { +@@ -221,12 +227,12 @@ __rte_ring_dequeue_elems_64(struct rte_ring *r, uint32_t prod_head, + } + + static __rte_always_inline void +-__rte_ring_dequeue_elems_128(struct rte_ring *r, uint32_t prod_head, ++__rte_ring_dequeue_elems_128(struct rte_ring *r, uint32_t cons_head, + void *obj_table, uint32_t n) + { + unsigned int i; + const uint32_t size = r->size; +- uint32_t idx = prod_head & r->mask; ++ uint32_t idx = cons_head & r->mask; + rte_int128_t *ring = (rte_int128_t *)&r[1]; + rte_int128_t *obj = (rte_int128_t *)obj_table; + if (likely(idx + n <= size)) { +@@ -382,4 +388,8 @@ end: + return n; + } + ++#if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION >= 120000) ++#pragma GCC diagnostic pop ++#endif ++ + #endif /* _RTE_RING_ELEM_PVT_H_ */ +diff --git a/dpdk/lib/sched/rte_sched.c b/dpdk/lib/sched/rte_sched.c +index c91697131d..19768d8c38 100644 +--- a/dpdk/lib/sched/rte_sched.c ++++ b/dpdk/lib/sched/rte_sched.c +@@ -202,6 +202,9 @@ struct rte_sched_subport { + uint32_t qsize_add[RTE_SCHED_QUEUES_PER_PIPE]; + uint32_t qsize_sum; + ++ /* TC oversubscription activation */ ++ int tc_ov_enabled; ++ + struct rte_sched_pipe *pipe; + struct rte_sched_queue *queue; + struct rte_sched_queue_extra *queue_extra; +@@ -209,9 +212,6 @@ struct rte_sched_subport { + uint8_t *bmp_array; + struct rte_mbuf **queue_array; + uint8_t memory[0] __rte_cache_aligned; +- +- /* TC oversubscription activation */ +- int tc_ov_enabled; + } __rte_cache_aligned; + + struct rte_sched_port { +diff --git a/dpdk/lib/security/rte_security.h b/dpdk/lib/security/rte_security.h +index 4bacf9fcd9..fd7013a23d 100644 +--- a/dpdk/lib/security/rte_security.h ++++ b/dpdk/lib/security/rte_security.h +@@ -618,6 +618,7 @@ struct rte_security_docsis_xform { + /** + * Security session action type. + */ ++/* Enumeration of rte_security_session_action_type 8<*/ + enum rte_security_session_action_type { + RTE_SECURITY_ACTION_TYPE_NONE, + /**< No security actions */ +@@ -638,8 +639,10 @@ enum rte_security_session_action_type { + * protocol is processed synchronously by a CPU. + */ + }; ++/* >8 End enumeration of rte_security_session_action_type. */ + + /** Security session protocol definition */ ++/* Enumeration of rte_security_session_protocol 8<*/ + enum rte_security_session_protocol { + RTE_SECURITY_PROTOCOL_IPSEC = 1, + /**< IPsec Protocol */ +@@ -650,10 +653,12 @@ enum rte_security_session_protocol { + RTE_SECURITY_PROTOCOL_DOCSIS, + /**< DOCSIS Protocol */ + }; ++/* >8 End enumeration of rte_security_session_protocol. */ + + /** + * Security session configuration + */ ++/* Structure rte_security_session_conf 8< */ + struct rte_security_session_conf { + enum rte_security_session_action_type action_type; + /**< Type of action to be performed on the session */ +@@ -672,6 +677,7 @@ struct rte_security_session_conf { + void *userdata; + /**< Application specific userdata to be saved with session */ + }; ++/* >8 End of structure rte_security_session_conf. */ + + /** + * Create security session as specified by the session configuration +diff --git a/dpdk/lib/table/rte_swx_table_selector.c b/dpdk/lib/table/rte_swx_table_selector.c +index ad99f18453..18e021fe6f 100644 +--- a/dpdk/lib/table/rte_swx_table_selector.c ++++ b/dpdk/lib/table/rte_swx_table_selector.c +@@ -232,7 +232,7 @@ table_params_copy(struct table *t, struct rte_swx_table_selector_params *params) + t->params.n_members_per_group_max = rte_align32pow2(params->n_members_per_group_max); + + for (i = 0; i < 32; i++) +- if (params->n_members_per_group_max == 1U << i) ++ if (t->params.n_members_per_group_max == 1U << i) + t->n_members_per_group_max_log2 = i; + + /* t->params.selector_mask */ +diff --git a/dpdk/lib/telemetry/rte_telemetry.h b/dpdk/lib/telemetry/rte_telemetry.h +index d9918c4e96..40e9a3bf9d 100644 +--- a/dpdk/lib/telemetry/rte_telemetry.h ++++ b/dpdk/lib/telemetry/rte_telemetry.h +@@ -2,9 +2,6 @@ + * Copyright(c) 2018 Intel Corporation + */ + +-#include +- +- + #ifndef _RTE_TELEMETRY_H_ + #define _RTE_TELEMETRY_H_ + +@@ -12,6 +9,8 @@ + extern "C" { + #endif + ++#include ++ + /** Maximum length for string used in object. */ + #define RTE_TEL_MAX_STRING_LEN 128 + /** Maximum length of string. */ +diff --git a/dpdk/lib/telemetry/telemetry.c b/dpdk/lib/telemetry/telemetry.c +index 8fbb4f3060..9c3c346ff5 100644 +--- a/dpdk/lib/telemetry/telemetry.c ++++ b/dpdk/lib/telemetry/telemetry.c +@@ -208,7 +208,11 @@ container_to_json(const struct rte_tel_data *d, char *out_buf, size_t buf_len) + break; + case RTE_TEL_CONTAINER: + { +- char temp[buf_len]; ++ char *temp = malloc(buf_len); ++ if (temp == NULL) ++ break; ++ *temp = '\0'; /* ensure valid string */ ++ + const struct container *cont = + &v->value.container; + if (container_to_json(cont->data, +@@ -219,6 +223,7 @@ container_to_json(const struct rte_tel_data *d, char *out_buf, size_t buf_len) + v->name, temp); + if (!cont->keep) + rte_tel_data_free(cont->data); ++ free(temp); + break; + } + } +@@ -275,7 +280,11 @@ output_json(const char *cmd, const struct rte_tel_data *d, int s) + break; + case RTE_TEL_CONTAINER: + { +- char temp[buf_len]; ++ char *temp = malloc(buf_len); ++ if (temp == NULL) ++ break; ++ *temp = '\0'; /* ensure valid string */ ++ + const struct container *cont = + &v->value.container; + if (container_to_json(cont->data, +@@ -286,6 +295,7 @@ output_json(const char *cmd, const struct rte_tel_data *d, int s) + v->name, temp); + if (!cont->keep) + rte_tel_data_free(cont->data); ++ free(temp); + } + } + } +@@ -311,7 +321,11 @@ output_json(const char *cmd, const struct rte_tel_data *d, int s) + buf_len, used, + d->data.array[i].u64val); + else if (d->type == RTE_TEL_ARRAY_CONTAINER) { +- char temp[buf_len]; ++ char *temp = malloc(buf_len); ++ if (temp == NULL) ++ break; ++ *temp = '\0'; /* ensure valid string */ ++ + const struct container *rec_data = + &d->data.array[i].container; + if (container_to_json(rec_data->data, +@@ -321,6 +335,7 @@ output_json(const char *cmd, const struct rte_tel_data *d, int s) + buf_len, used, temp); + if (!rec_data->keep) + rte_tel_data_free(rec_data->data); ++ free(temp); + } + break; + } +@@ -333,7 +348,7 @@ output_json(const char *cmd, const struct rte_tel_data *d, int s) + static void + perform_command(telemetry_cb fn, const char *cmd, const char *param, int s) + { +- struct rte_tel_data data; ++ struct rte_tel_data data = {0}; + + int ret = fn(cmd, param, &data); + if (ret < 0) { +diff --git a/dpdk/lib/vhost/socket.c b/dpdk/lib/vhost/socket.c +index 863a6f6d52..669c322e12 100644 +--- a/dpdk/lib/vhost/socket.c ++++ b/dpdk/lib/vhost/socket.c +@@ -129,10 +129,12 @@ read_fd_message(char *ifname, int sockfd, char *buf, int buflen, int *fds, int m + return ret; + } + +- if (msgh.msg_flags & (MSG_TRUNC | MSG_CTRUNC)) { ++ if (msgh.msg_flags & MSG_TRUNC) + VHOST_LOG_CONFIG(ifname, ERR, "truncated msg (fd %d)\n", sockfd); +- return -1; +- } ++ ++ /* MSG_CTRUNC may be caused by LSM misconfiguration */ ++ if (msgh.msg_flags & MSG_CTRUNC) ++ VHOST_LOG_CONFIG(ifname, ERR, "truncated control data (fd %d)\n", sockfd); + + for (cmsg = CMSG_FIRSTHDR(&msgh); cmsg != NULL; + cmsg = CMSG_NXTHDR(&msgh, cmsg)) { +diff --git a/dpdk/lib/vhost/vhost.c b/dpdk/lib/vhost/vhost.c +index 19c7b92c32..9e28198528 100644 +--- a/dpdk/lib/vhost/vhost.c ++++ b/dpdk/lib/vhost/vhost.c +@@ -1294,6 +1294,7 @@ rte_vhost_vring_call(int vid, uint16_t vring_idx) + { + struct virtio_net *dev; + struct vhost_virtqueue *vq; ++ int ret = 0; + + dev = get_device(vid); + if (!dev) +@@ -1308,14 +1309,20 @@ rte_vhost_vring_call(int vid, uint16_t vring_idx) + + rte_spinlock_lock(&vq->access_lock); + ++ if (unlikely(!vq->access_ok)) { ++ ret = -1; ++ goto out_unlock; ++ } ++ + if (vq_is_packed(dev)) + vhost_vring_call_packed(dev, vq); + else + vhost_vring_call_split(dev, vq); + ++out_unlock: + rte_spinlock_unlock(&vq->access_lock); + +- return 0; ++ return ret; + } + + int +@@ -1323,6 +1330,7 @@ rte_vhost_vring_call_nonblock(int vid, uint16_t vring_idx) + { + struct virtio_net *dev; + struct vhost_virtqueue *vq; ++ int ret = 0; + + dev = get_device(vid); + if (!dev) +@@ -1338,14 +1346,20 @@ rte_vhost_vring_call_nonblock(int vid, uint16_t vring_idx) + if (!rte_spinlock_trylock(&vq->access_lock)) + return -EAGAIN; + ++ if (unlikely(!vq->access_ok)) { ++ ret = -1; ++ goto out_unlock; ++ } ++ + if (vq_is_packed(dev)) + vhost_vring_call_packed(dev, vq); + else + vhost_vring_call_split(dev, vq); + ++out_unlock: + rte_spinlock_unlock(&vq->access_lock); + +- return 0; ++ return ret; + } + + uint16_t +@@ -1368,7 +1382,10 @@ rte_vhost_avail_entries(int vid, uint16_t queue_id) + + rte_spinlock_lock(&vq->access_lock); + +- if (unlikely(!vq->enabled || vq->avail == NULL)) ++ if (unlikely(!vq->access_ok)) ++ goto out; ++ ++ if (unlikely(!vq->enabled)) + goto out; + + ret = *(volatile uint16_t *)&vq->avail->idx - vq->last_used_idx; +@@ -1460,9 +1477,15 @@ rte_vhost_enable_guest_notification(int vid, uint16_t queue_id, int enable) + + rte_spinlock_lock(&vq->access_lock); + ++ if (unlikely(!vq->access_ok)) { ++ ret = -1; ++ goto out_unlock; ++ } ++ + vq->notif_enable = enable; + ret = vhost_enable_guest_notification(dev, vq, enable); + ++out_unlock: + rte_spinlock_unlock(&vq->access_lock); + + return ret; +@@ -1523,7 +1546,10 @@ rte_vhost_rx_queue_count(int vid, uint16_t qid) + + rte_spinlock_lock(&vq->access_lock); + +- if (unlikely(!vq->enabled || vq->avail == NULL)) ++ if (unlikely(!vq->access_ok)) ++ goto out; ++ ++ if (unlikely(!vq->enabled)) + goto out; + + ret = *((volatile uint16_t *)&vq->avail->idx) - vq->last_avail_idx; +@@ -1758,7 +1784,15 @@ rte_vhost_async_channel_register(int vid, uint16_t queue_id) + return -1; + + rte_spinlock_lock(&vq->access_lock); ++ ++ if (unlikely(!vq->access_ok)) { ++ ret = -1; ++ goto out_unlock; ++ } ++ + ret = async_channel_register(dev, vq); ++ ++out_unlock: + rte_spinlock_unlock(&vq->access_lock); + + return ret; +@@ -1814,6 +1848,11 @@ rte_vhost_async_channel_unregister(int vid, uint16_t queue_id) + return ret; + } + ++ if (unlikely(!vq->access_ok)) { ++ ret = -1; ++ goto out_unlock; ++ } ++ + if (!vq->async) { + ret = 0; + } else if (vq->async->pkts_inflight_n) { +@@ -1825,6 +1864,7 @@ rte_vhost_async_channel_unregister(int vid, uint16_t queue_id) + ret = 0; + } + ++out_unlock: + rte_spinlock_unlock(&vq->access_lock); + + return ret; +@@ -1968,9 +2008,15 @@ rte_vhost_async_get_inflight(int vid, uint16_t queue_id) + return ret; + } + ++ if (unlikely(!vq->access_ok)) { ++ ret = -1; ++ goto out_unlock; ++ } ++ + if (vq->async) + ret = vq->async->pkts_inflight_n; + ++out_unlock: + rte_spinlock_unlock(&vq->access_lock); + + return ret; +@@ -2014,6 +2060,7 @@ rte_vhost_get_monitor_addr(int vid, uint16_t queue_id, + { + struct virtio_net *dev = get_device(vid); + struct vhost_virtqueue *vq; ++ int ret = 0; + + if (dev == NULL) + return -1; +@@ -2024,6 +2071,13 @@ rte_vhost_get_monitor_addr(int vid, uint16_t queue_id, + if (vq == NULL) + return -1; + ++ rte_spinlock_lock(&vq->access_lock); ++ ++ if (unlikely(!vq->access_ok)) { ++ ret = -1; ++ goto out_unlock; ++ } ++ + if (vq_is_packed(dev)) { + struct vring_packed_desc *desc; + desc = vq->desc_packed; +@@ -2043,7 +2097,10 @@ rte_vhost_get_monitor_addr(int vid, uint16_t queue_id, + pmc->match = 0; + } + +- return 0; ++out_unlock: ++ rte_spinlock_unlock(&vq->access_lock); ++ ++ return ret; + } + + +@@ -2081,6 +2138,7 @@ rte_vhost_vring_stats_get(int vid, uint16_t queue_id, + struct virtio_net *dev = get_device(vid); + struct vhost_virtqueue *vq; + unsigned int i; ++ int ret = VHOST_NB_VQ_STATS; + + if (dev == NULL) + return -1; +@@ -2097,20 +2155,29 @@ rte_vhost_vring_stats_get(int vid, uint16_t queue_id, + vq = dev->virtqueue[queue_id]; + + rte_spinlock_lock(&vq->access_lock); ++ ++ if (unlikely(!vq->access_ok)) { ++ ret = -1; ++ goto out_unlock; ++ } ++ + for (i = 0; i < VHOST_NB_VQ_STATS; i++) { + stats[i].value = + *(uint64_t *)(((char *)vq) + vhost_vq_stat_strings[i].offset); + stats[i].id = i; + } ++ ++out_unlock: + rte_spinlock_unlock(&vq->access_lock); + +- return VHOST_NB_VQ_STATS; ++ return ret; + } + + int rte_vhost_vring_stats_reset(int vid, uint16_t queue_id) + { + struct virtio_net *dev = get_device(vid); + struct vhost_virtqueue *vq; ++ int ret = 0; + + if (dev == NULL) + return -1; +@@ -2124,10 +2191,17 @@ int rte_vhost_vring_stats_reset(int vid, uint16_t queue_id) + vq = dev->virtqueue[queue_id]; + + rte_spinlock_lock(&vq->access_lock); ++ ++ if (unlikely(!vq->access_ok)) { ++ ret = -1; ++ goto out_unlock; ++ } + memset(&vq->stats, 0, sizeof(vq->stats)); ++ ++out_unlock: + rte_spinlock_unlock(&vq->access_lock); + +- return 0; ++ return ret; + } + + int +diff --git a/dpdk/lib/vhost/vhost.h b/dpdk/lib/vhost/vhost.h +index ef211ed519..63e2f3f577 100644 +--- a/dpdk/lib/vhost/vhost.h ++++ b/dpdk/lib/vhost/vhost.h +@@ -782,7 +782,10 @@ hva_to_gpa(struct virtio_net *dev, uint64_t vva, uint64_t len) + static __rte_always_inline struct virtio_net * + get_device(int vid) + { +- struct virtio_net *dev = vhost_devices[vid]; ++ struct virtio_net *dev = NULL; ++ ++ if (likely(vid >= 0 && vid < RTE_MAX_VHOST_DEVICE)) ++ dev = vhost_devices[vid]; + + if (unlikely(!dev)) { + VHOST_LOG_CONFIG("device", ERR, "(%d) device not found.\n", vid); +@@ -878,9 +881,9 @@ vhost_vring_call_split(struct virtio_net *dev, struct vhost_virtqueue *vq) + "%s: used_event_idx=%d, old=%d, new=%d\n", + __func__, vhost_used_event(vq), old, new); + +- if ((vhost_need_event(vhost_used_event(vq), new, old) && +- (vq->callfd >= 0)) || +- unlikely(!signalled_used_valid)) { ++ if ((vhost_need_event(vhost_used_event(vq), new, old) || ++ unlikely(!signalled_used_valid)) && ++ vq->callfd >= 0) { + eventfd_write(vq->callfd, (eventfd_t) 1); + if (dev->flags & VIRTIO_DEV_STATS_ENABLED) + vq->stats.guest_notifications++; +@@ -947,8 +950,10 @@ vhost_vring_call_packed(struct virtio_net *dev, struct vhost_virtqueue *vq) + if (vhost_need_event(off, new, old)) + kick = true; + kick: +- if (kick) { ++ if (kick && vq->callfd >= 0) { + eventfd_write(vq->callfd, (eventfd_t)1); ++ if (dev->flags & VIRTIO_DEV_STATS_ENABLED) ++ vq->stats.guest_notifications++; + if (dev->notify_ops->guest_notified) + dev->notify_ops->guest_notified(dev->vid); + } +diff --git a/dpdk/lib/vhost/vhost_user.c b/dpdk/lib/vhost/vhost_user.c +index 9902ae9944..8df66e68b3 100644 +--- a/dpdk/lib/vhost/vhost_user.c ++++ b/dpdk/lib/vhost/vhost_user.c +@@ -1809,7 +1809,7 @@ static int vhost_user_set_vring_err(struct virtio_net **pdev, + + if (!(ctx->msg.payload.u64 & VHOST_USER_VRING_NOFD_MASK)) + close(ctx->fds[0]); +- VHOST_LOG_CONFIG(dev->ifname, INFO, "not implemented\n"); ++ VHOST_LOG_CONFIG(dev->ifname, DEBUG, "not implemented\n"); + + return RTE_VHOST_MSG_RESULT_OK; + } +@@ -2326,7 +2326,7 @@ static int vhost_user_set_log_fd(struct virtio_net **pdev, + return RTE_VHOST_MSG_RESULT_ERR; + + close(ctx->fds[0]); +- VHOST_LOG_CONFIG(dev->ifname, INFO, "not implemented.\n"); ++ VHOST_LOG_CONFIG(dev->ifname, DEBUG, "not implemented.\n"); + + return RTE_VHOST_MSG_RESULT_OK; + } +@@ -2817,29 +2817,36 @@ read_vhost_message(struct virtio_net *dev, int sockfd, struct vhu_msg_context * + + ret = read_fd_message(dev->ifname, sockfd, (char *)&ctx->msg, VHOST_USER_HDR_SIZE, + ctx->fds, VHOST_MEMORY_MAX_NREGIONS, &ctx->fd_num); +- if (ret <= 0) { +- return ret; +- } else if (ret != VHOST_USER_HDR_SIZE) { ++ if (ret <= 0) ++ goto out; ++ ++ if (ret != VHOST_USER_HDR_SIZE) { + VHOST_LOG_CONFIG(dev->ifname, ERR, "Unexpected header size read\n"); +- close_msg_fds(ctx); +- return -1; ++ ret = -1; ++ goto out; + } + + if (ctx->msg.size) { + if (ctx->msg.size > sizeof(ctx->msg.payload)) { + VHOST_LOG_CONFIG(dev->ifname, ERR, "invalid msg size: %d\n", + ctx->msg.size); +- return -1; ++ ret = -1; ++ goto out; + } + ret = read(sockfd, &ctx->msg.payload, ctx->msg.size); + if (ret <= 0) +- return ret; ++ goto out; + if (ret != (int)ctx->msg.size) { + VHOST_LOG_CONFIG(dev->ifname, ERR, "read control message failed\n"); +- return -1; ++ ret = -1; ++ goto out; + } + } + ++out: ++ if (ret <= 0) ++ close_msg_fds(ctx); ++ + return ret; + } + +@@ -2987,13 +2994,10 @@ vhost_user_msg_handler(int vid, int fd) + } + } + ++ ctx.msg.request.master = VHOST_USER_NONE; + ret = read_vhost_message(dev, fd, &ctx); +- if (ret <= 0) { +- if (ret < 0) +- VHOST_LOG_CONFIG(dev->ifname, ERR, "vhost read message failed\n"); +- else +- VHOST_LOG_CONFIG(dev->ifname, INFO, "vhost peer closed\n"); +- ++ if (ret == 0) { ++ VHOST_LOG_CONFIG(dev->ifname, INFO, "vhost peer closed\n"); + return -1; + } + +@@ -3003,6 +3007,14 @@ vhost_user_msg_handler(int vid, int fd) + else + msg_handler = NULL; + ++ if (ret < 0) { ++ VHOST_LOG_CONFIG(dev->ifname, ERR, "vhost read message %s%s%sfailed\n", ++ msg_handler != NULL ? "for " : "", ++ msg_handler != NULL ? msg_handler->description : "", ++ msg_handler != NULL ? " " : ""); ++ return -1; ++ } ++ + if (msg_handler != NULL && msg_handler->description != NULL) { + if (request != VHOST_USER_IOTLB_MSG) + VHOST_LOG_CONFIG(dev->ifname, INFO, +diff --git a/dpdk/lib/vhost/virtio_net.c b/dpdk/lib/vhost/virtio_net.c +index 9abf752f30..26f184f8b2 100644 +--- a/dpdk/lib/vhost/virtio_net.c ++++ b/dpdk/lib/vhost/virtio_net.c +@@ -1453,6 +1453,12 @@ virtio_dev_rx_batch_packed_copy(struct virtio_net *dev, + sizeof(struct virtio_net_hdr_mrg_rxbuf); + } + ++ if (rxvq_is_mergeable(dev)) { ++ vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) { ++ ASSIGN_UNLESS_EQUAL(hdrs[i]->num_buffers, 1); ++ } ++ } ++ + vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) + virtio_enqueue_offload(pkts[i], &hdrs[i]->hdr); + +@@ -3470,6 +3476,7 @@ virtio_dev_tx_async_split(struct virtio_net *dev, struct vhost_virtqueue *vq, + allocerr_warned = true; + } + dropped = true; ++ slot_idx--; + break; + } + +diff --git a/dpdk/usertools/dpdk-pmdinfo.py b/dpdk/usertools/dpdk-pmdinfo.py +index 67d023a047..2c728de7b8 100755 +--- a/dpdk/usertools/dpdk-pmdinfo.py ++++ b/dpdk/usertools/dpdk-pmdinfo.py +@@ -23,7 +23,7 @@ Get only the required kernel modules for a given driver: + Get only the required kernel modules for a given device: + + %(prog)s dpdk-testpmd | \ +- jq '.[] | select(.devices[] | .vendor_id == "15b3" and .device_id == "1013").kmod' ++ jq '.[] | select(.pci_ids[]? | .vendor == "15b3" and .device == "1013").kmod' + """ + + import argparse diff --git a/SPECS/openvswitch3.1.spec b/SPECS/openvswitch3.1.spec index 09b91a3..03c4596 100644 --- a/SPECS/openvswitch3.1.spec +++ b/SPECS/openvswitch3.1.spec @@ -63,7 +63,7 @@ Summary: Open vSwitch Group: System Environment/Daemons daemon/database/utilities URL: http://www.openvswitch.org/ Version: 3.1.0 -Release: 74%{?dist} +Release: 113%{?dist} # Nearly all of openvswitch is ASL 2.0. The bugtool is LGPLv2+, and the # lib/sflow*.[ch] files are SISSL @@ -757,6 +757,1115 @@ exit 0 %endif %changelog +* Wed Apr 24 2024 Open vSwitch CI - 3.1.0-113 +- Merging upstream branch-3.1 [RH git: 10583df489] + Commit list: + 6ce0130f12 netdev-dpdk: Fix possible memory leak configuring VF MAC address. + + +* Tue Apr 23 2024 Open vSwitch CI - 3.1.0-112 +- Merging upstream branch-3.1 [RH git: 282b986905] + Commit list: + c71c87d5c9 ovsdb: raft: Fix probe intervals after install snapshot request. + b237df7405 ovsdb: raft: Fix inability to join a cluster with a large database. + 9812877077 rhel/systemd: Set ovsdb-server timeout to 5 minutes. + + +* Thu Apr 11 2024 Open vSwitch CI - 3.1.0-111 +- Merging upstream branch-3.1 [RH git: dadb6534af] + Commit list: + 6b32502aa6 github: Update python to 3.12. + f3ce71c75a ovsdb-dot: Fix flake8 issues. + 7b994b4903 ovsdb-doc: Fix syntax warning with Python 3.12 and flake8 issues. + + +* Wed Apr 10 2024 Open vSwitch CI - 3.1.0-110 +- Merging upstream branch-3.1 [RH git: bdf5e11596] + Commit list: + 70f0941b23 python: Remove hacking dependency and use recent flake8. + 7e9b2f6617 cirrus: Update to FreeBSD 13.3. + + +* Tue Apr 09 2024 Open vSwitch CI - 3.1.0-109 +- Merging upstream branch-3.1 [RH git: 04df41821a] + Commit list: + ac31976ffa tests: Fix compatibility issue with Python 3.13 in vlog.at. + + +* Fri Apr 05 2024 Open vSwitch CI - 3.1.0-108 +- Merging upstream branch-3.1 [RH git: 3741a6ff0e] + Commit list: + 27228f017f ofproto-dpif-upcall: Fix ukey installation failure logs and counters. + + +* Wed Apr 03 2024 Open vSwitch CI - 3.1.0-107 +- Merging upstream branch-3.1 [RH git: f7577a2051] + Commit list: + d23fdb41e2 conntrack: Do not use icmp reverse helper for icmpv6. + c769ee467a conntrack: Fix SNAT with exhaustion system test. + + +* Wed Mar 27 2024 Open vSwitch CI - 3.1.0-106 +- Merging upstream branch-3.1 [RH git: 1110e96479] + Commit list: + 852d86e2fd ovsdb: raft: Fix inability to join after leadership change round trip. + 4136cd27ff ovsdb: raft: Fix permanent joining state on a cluster member. + 7872a1e67e ovsdb: raft: Avoid transferring leadership to unavailable servers. + + +* Wed Mar 27 2024 Open vSwitch CI - 3.1.0-105 +- Merging upstream branch-3.1 [RH git: 093dad7612] + Commit list: + 8925f29155 ofproto-dpif-xlate: Fix continuations with associated metering. + + +* Fri Mar 22 2024 Open vSwitch CI - 3.1.0-104 +- Merging upstream branch-3.1 [RH git: 0ed11af3ab] + Commit list: + 934ab1e76b ovs-monitor-ipsec: LibreSwan autodetect paths. (#1975039) + 07a851265c route-table: Avoid routes from non-standard routing tables. + + +* Tue Mar 19 2024 Open vSwitch CI - 3.1.0-103 +- Merging upstream branch-3.1 [RH git: e74dfafabc] + Commit list: + fe5d15b40c ofproto-dpif: Fix tunnel with different name del/add failure. + 159fd0355c ofpbuf: Prevent undefined behavior in ofpbuf_clone. + + +* Tue Mar 12 2024 Open vSwitch CI - 3.1.0-102 +- Merging upstream branch-3.1 [RH git: f1020d6cf4] + Commit list: + ac8014ef39 github: Reduce ASLR entropy to be compatible with asan in llvm 14. + + +* Fri Mar 08 2024 Open vSwitch CI - 3.1.0-101 +- Merging upstream branch-3.1 [RH git: 98bb980e16] + Commit list: + 6e11965690 bfd: Improve state change log message. (#2258496) + 15e676dd15 tests: Fix "SSL db: Implementation" test with openssl > 3.2.0. + + +* Fri Mar 08 2024 Open vSwitch CI - 3.1.0-100 +- Merging upstream branch-3.1 [RH git: 5551ebc1c1] + Commit list: + c56f0b4c49 conntrack: Fix flush not flushing all elements. + + +* Sat Mar 02 2024 Open vSwitch CI - 3.1.0-99 +- Merging upstream branch-3.1 [RH git: 42d3fec061] + Commit list: + 42a470e75a bond: Reset stats when deleting post recirc rule. + a73d946d21 ofproto-dpif-trace: Fix infinite recirculation tracing. + b7f021275d ofproto-dpif-xlate: Fix ignoring IPv6 local_ip for native tunnels. + 2d76b4de37 netdev-dummy: Add local route entries for IP addresses. + ff5044649c tests: Move the non-local port as tunnel endpoint test. + 5ed423500c netdev-dummy: Support multiple IP addresses. + 9e8c641851 rstp: Fix deadlock with patch ports. + + +* Sat Mar 02 2024 Open vSwitch CI - 3.1.0-98 +- Merging dpdk subtree [RH git: a9cbf91b50] + Commit list: + 319a57029a Revert "build: add libarchive to optional external dependencies" + cb88423913 Merge tag 'v22.11.4' into 22.11 + 076a05a022 Reapply "net/iavf: fix abnormal disable HW interrupt" + caf0f5d395 version: 22.11.4 + 7d6f1cc05f Revert "net/iavf: fix abnormal disable HW interrupt" + dc9c799c7d vhost: fix missing spinlock unlock + 4307659a90 net/mlx5: fix LACP redirection in Rx domain + 6ef77f2a5e net/gve: fix RX buffer size alignment + ee2197cbd4 crypto/openssl: fix memory leaks in asym session + 02cb6cdbcd net/nfp: fix reconfigure logic of set MAC address + 9651485a36 net/nfp: fix link status interrupt + 6350131153 app/test: disable graph auto test for windows + 97cdb0db6d Revert "eventdev: fix alignment padding" + 6c688dec84 event/dlb2: fix disable PASID + 6471372564 net/hns3: fix VF reset handler interruption + 28e7716ee3 net/hns3: fix reset event status + c5d0485861 net/hns3: fix ignored reset event + da708a5b4e ethdev: fix ESP packet type description + 3b67c07511 mempool: clarify enqueue/dequeue ops documentation + 9c68d8a845 mempool: fix get function documentation + 9f0f54a72e doc: remove number of commands in vDPA guide + e1aa1c2ce0 doc: fix some ordered lists + cabe6f3bae doc: remove restriction on ixgbe vector support + de38896353 app/pipeline: add sigint handler + 22a1b5067c test/hash: fix creation error log + 122f600818 usertools/pmdinfo: fix usage typos + 0a742f1ec0 examples/ethtool: fix pause configuration + 3ca387345f app/dumpcap: allow multiple invocations + 578b87a955 pdump: fix error number on IPC response + 364faab59a eal/windows: fix build with recent MinGW + 82dd3dde96 config: fix RISC-V native build + 2a720ab686 examples/ipsec-secgw: fix partial overflow + b5913482ec net/mlx5: fix offset size in conntrack flow action + 434c2fc890 doc: update versions recommendations for i40e and ice + ce75c85b4a doc: fix RSS flow description in hns3 guide + fb6560fcfc doc: update features in hns3 guide + 6a44194cc1 doc: fix hns3 build option about max queue number + 085a72d8f8 net/txgbe: fix out of bound access + e0fa003972 eal/riscv: fix vector type alignment + d3741774d0 net/mlx5: fix MPRQ stride size check + 07a861e472 net/mlx5: zero UDP checksum over IPv4 in encapsulation + c9e75ba878 net/mlx5: fix shared Rx queue list management + 4699a174ea net/mlx5: fix multi-segment Tx inline data length + 14a722ae6b net/mlx5: fix hairpin queue states + ab46753245 net/mlx5: fix hairpin queue unbind + 480df9f41e net/mlx5: fix use after free on Rx queue start + d32e9e689b net/mlx5: fix destroying external representor flow + ca79cce293 net/mlx5: fix missing flow rules for external SQ + 97b9c4dca3 net/mlx5: fix counter query during port close + 49bd7912ee net/mlx5: fix validation of sample encap flow action + f00e3b40fd net/mlx5: fix unlock mismatch + 5ff3454aac app/testpmd: fix tunnel TSO configuration + 2959baf71f app/testpmd: add explicit check for tunnel TSO + 6f1c35e7a8 app/testpmd: fix tunnel TSO capability check + b3009db2e4 net/hns3: fix mailbox sync + be77f806f2 test/bonding: fix uninitialized RSS configuration + 5b8da03c1d ethdev: account for smaller MTU when setting default + 5508c2d500 app/testpmd: remove useless check in TSO command + 28f7dd30ee net/sfc: remove null dereference in log + c38b876902 meter: fix RFC4115 trTCM API Doxygen + 71876ad11a event/dlb2: fix missing queue ordering capability flag + a998d657f5 common/cnxk: fix SDP channel mask + 0d7ea280fe net/ice: fix crash on closing representor ports + 2dfcdb87a5 test/bbdev: assert failed test for queue configure + 7b8504aee3 test/bbdev: fix Python script subprocess + e42af2946a baseband/acc: fix TB mode on VRB1 + 2e2cc882eb event/dlb2: fix name check in self-test + 2cd4ce8518 app/dumpcap: fix mbuf pool ring type + ac8ca59223 event/dlb2: disable PASID + 5586a7be43 bus/pci: add PASID control + dec4b39b20 build: add libarchive to optional external dependencies + cb225e1ae9 config/arm: fix aarch32 build with GCC 13 + e787872a59 net/ice: fix Tx preparation + 8dc1b42e31 net/iavf: fix Tx preparation + c70a8fa66e crypto/nitrox: fix panic with high number of segments + f089d80b08 doc: update kernel module entry in QAT guide + d7b738d6dc net/ice: fix DCF port statistics + c7270eb8cd net/iavf: fix Tx offload flags check + 84c1679b72 net/iavf: fix indent in Tx path + c1c417228a net/iavf: fix Tx offload mask + ae7a0f6233 net/ice: fix L1 check interval + f2ab72ed3d common/mlx5: fix controller index parsing + 72ab4bf42f net/mlx5: fix flow workspace double free in Windows + f0bc006230 net/mlx5/hws: fix integrity bits level + 815a8ce067 net/mlx5: fix flow thread safety flag for HWS + b57dd69e61 net/mlx5: fix E-Switch mirror flow rule validation + a908d0cb5b net/mlx5: fix NIC flow capability query + 74b594dfd3 net/mlx5: fix decap action checking in sample flow + 81410056f5 net/tap: fix RSS for fragmented packets + 85b95b48ce net/nfp: fix DMA error after abnormal exit + 5b470b0567 app/procinfo: adjust format of RSS info + df4470f94c app/procinfo: fix RSS info + 69d6a23bb1 net/tap: fix IPv4 checksum offloading + db03b8a934 net/tap: fix L4 checksum offloading + de2edad82b net/hns3: fix uninitialized hash algo value + 085f8342f5 net/hns3: keep set/get algo key functions local + 555126492b net/hns3: fix some error logs + 84b217ed2c net/hns3: fix some return values + 36f408b7d8 net/hns3: fix LRO offload to report + 518bc13327 net/hns3: fix setting DCB capability + 4f039a69e9 net/enic: avoid extra unlock in MTU set + 73440e9fd3 ethdev: fix 32-bit build with GCC 13 + 14a4c7b3e8 net/ngbe: add proper memory barriers in Rx + 33f8a0ce2c net/txgbe: add proper memory barriers in Rx + 22b7f9edf9 net/bonding: fix possible overrun + d3f778e077 test/bonding: add missing check + 7989293c66 test/bonding: remove unreachable statement + 8d45a7ecac net/nfp: fix reconfigure logic in VF initialization + cb30eb9a11 net/nfp: fix reconfigure logic in PF initialization + 8dcc1b4552 net/hns3: refactor interrupt state query + 9a62d06e64 net/hns3: fix IMP or global reset + 65231cf278 net/hns3: fix multiple reset detected log + 290166c239 net/hns3: remove reset log in secondary + c4bf1adb63 net/hns3: fix double stats for IMP and global reset + 70c868a155 net/hns3: fix unchecked Rx free threshold + b9195e016e net/hns3: fix typo in function name + 976cefa803 app/testpmd: fix early exit from signal + 8f4eb60951 net/gve: update max Rx packet length to be based on MTU + 6f78b589c5 ethdev: fix function name in comment + e671505da1 net/nfp: fix crash on close + 4dfec51b97 net/cnxk: fix data offset in vector Tx + 780be398dc common/cnxk: fix pool buffer size in opaque mode + 8c291d8778 vhost: fix checking virtqueue access in stats API + a07736eb68 vhost: fix missing lock protection in power monitor API + adae353b36 vhost: fix check on virtqueue access in in-flight getter + 7f80528fbd vhost: fix check on virtqueue access in async registration + 2dc5b2aadd vhost: fix missing check on virtqueue access + ac1162d97a vhost: fix missing vring call check on virtqueue access + c1001c18e6 net/virtio: fix link state interrupt vector setting + b485fae237 net/virtio: fix missing next flag in Tx packed ring + 51205657f1 eventdev: fix missing driver names in info struct + 0beed895ef eventdev: fix device pointer for vdev-based devices + 4f5e8c612f malloc: remove return from void functions + f2f948863b bus/ifpga: fix driver header dependency + f42884a20c app/procinfo: remove unnecessary rte_malloc + c20753f52f ethdev: add check in async flow action query + b33ea9e6de net/bonding: fix link status callback stop + d316924b1f app/testpmd: fix primary process not polling all queues + 0cece838b9 net/vmxnet3: fix Rx and Tx queue state + bdaff48945 net/virtio: fix Rx and Tx queue state + b1dfb750b5 net/vhost: fix Rx and Tx queue state + dc77b5d082 net/txgbe: fix Rx and Tx queue state + beb475c1c4 net/softnic: fix Rx and Tx queue state + 8ab4f361b0 net/sfc: fix Rx and Tx queue state + 564958b988 net/ring: fix Rx and Tx queue state + f2edd9823d net/pfe: fix Rx and Tx queue state + 63a8198b22 net/octeontx: fix Rx and Tx queue state + f6ec84e5e5 net/octeon_ep: fix Rx and Tx queue state + eb53c49bad net/null: fix Rx and Tx queue state + df52eadf8f net/ngbe: fix Rx and Tx queue state + 4e28c37ff5 net/nfp: fix Rx and Tx queue state + e2e72d04cb net/mvpp2: fix Rx and Tx queue state + 5116feb88f net/mvneta: fix Rx and Tx queue state + accdf4594b net/mlx4: fix Rx and Tx queue state + 26fe1d02e5 net/memif: fix Rx and Tx queue state + 1f38a20d2e net/ipn3ke: fix Rx and Tx queue state + 0d4ab569a1 net/hinic: fix Rx and Tx queue state + 2492c41642 net/enic: fix Rx and Tx queue state + 3df3bf7e5c net/enetc: fix Rx and Tx queue state + 1efb8fcf47 net/ena: fix Rx and Tx queue state + 36e418d789 net/e1000: fix Rx and Tx queue state + ef3b3501b8 net/dpaa2: fix Rx and Tx queue state + b97544f35a net/dpaa: fix Rx and Tx queue state + 606a11a71c net/cxgbe: fix Rx and Tx queue state + 50177d5bf1 net/bonding: fix Rx and Tx queue state + 5f75982239 net/bnxt: fix Rx and Tx queue state + ad55189740 net/bnx2x: fix Rx and Tx queue state + c0001c9115 net/avp: fix Rx and Tx queue state + b4078f8a6b net/af_xdp: fix Rx and Tx queue state + 6c8f427c57 net/af_packet: fix Rx and Tx queue state + a8fd5060ac net/mana: add missing new line to data path logs + c6b2c85ca4 fib6: fix adding default route as first route + e4649e8d78 fib: fix adding default route overwriting entire table + 461a81717f net/mlx5/hws: fix field copy bind + 1c9bab50cc net/mlx5: fix jump ipool entry size + 62a937220c common/mlx5: replace use of PMD log type + 8115fe3345 net/mlx5: fix leak in sysfs port name translation + 06cd13b57e net/nfp: fix initialization of physical representors + e054f121ef net/ark: support single function with multiple port + b796460bce net/axgbe: identify CPU with cpuid + a7ff0ba143 net/ice: fix TSO with big segments + a2799805b6 net/ice: remove log from Tx prepare function + 50e27677ba net/iavf: fix TSO with big segments + 235b473bf9 net/iavf: remove log from Tx prepare function + eec9ac43a7 net/iavf: fix Tx debug + 101faaba08 net/ice: fix initial link status + b64cc84bb5 net/iavf: fix ESN session update + efdea6d6f4 net/iavf: unregister interrupt handler before FD close + 948b21bd59 net/iavf: fix port stats clearing + 4467e32db9 net/ice: fix TM configuration clearing + 36e302e98b net/i40e: fix buffer leak on Rx reconfiguration + baaa298893 net/iavf: fix checksum offloading + 491e9d37c1 net/iavf: fix VLAN offload strip flag + 52c903949a net/ice: write timestamp to first segment in scattered Rx + d4cd714640 net/i40e: fix FDIR queue receives broadcast packets + 202402218c app/bbdev: fix link with NXP LA12XX + 923d7a21ab baseband/acc: fix ACC100 HARQ input alignment + 9e7a4f889b vdpa/mlx5: fix unregister kick handler order + 529b747a92 common/cnxk: replace direct API usage in REE + 513e507ee4 common/cnxk: remove dead Meson code + 257c5a049a common/cnxk: fix RSS key configuration + 5dc2babce4 common/cnxk: fix aura disable handling + 1ef99ed9dd mempool/cnxk: fix alloc from non-EAL threads + 0da4d859cd mempool/cnxk: fix free from non-EAL threads + 2b29a7975b dma/cnxk: fix chunk buffer failure return code + 46d25f17a6 dma/cnxk: fix device reconfigure + 915fbdb681 dma/cnxk: fix device state + ab0aa8b245 common/cnxk: fix DPI memzone name + 0259cb16f7 net/cnxk: fix uninitialized variable + e38988edfd common/cnxk: fix incorrect aura ID + 0e8848d299 common/cnxk: fix different size bit operations + bc87fece07 common/cnxk: fix xstats for different packet sizes + 8c360e4392 common/cnxk: fix default flow action setting + 1b7fb134d9 event/sw: fix ordering corruption with op release + e9da79d973 eventdev/eth_rx: fix timestamp field register in mbuf + 6d49fdbd99 event/cnxk: fix context flush in port cleanup + 22d4975573 event/sw: remove obsolete comment + bb60e98212 test/event: fix crypto null device creation + 6e9061bd77 event/cnxk: fix return values for capability API + ba2ec17a2f eventdev/crypto: fix circular buffer full case + 15b1bc964b event/cnxk: fix CASP usage for clang + 148478f997 event/cnxk: fix getwork mode devargs parsing + d854ba2422 eventdev: fix alignment padding + 73fa336da5 bus/pci: fix device ID log + 5a03754492 eventdev: fix symbol export for port maintenance + f150a45b65 crypto/ipsec_mb: add dependency check for cross build + 39b817dfee crypto/cnxk: fix IPsec CCM and GCM capabilities + 40ea03b5ea crypto/qat: fix raw API null algorithm digest + 31f6839de6 test/crypto: fix typo in asym tests + 486fb2f97c test/crypto: fix return value for GMAC case + 8506f6d67c test/crypto: skip some synchronous tests with CPU crypto + 5f940557cf test/crypto: fix IV in some vectors + e4cba3bb19 test/security: fix IPv6 next header field + 7004929e29 doc: replace code blocks with includes in security guide + 3d75e696dc crypto/dpaa_sec: fix debug prints + 00819a704d crypto/dpaa2_sec: fix debug prints + 8af3c9994e cryptodev: add missing doc for security context + 1a318a87a3 test: fix named test macro + 086e0e529e app/testpmd: fix help string + 8ef9e184ca net/gve: fix max MTU limit + 6754edbeb5 net/tap: use MAC address parse API instead of local parser + 367755a8dc net/ngbe: check process type in close operation + a340716239 net/txgbe: check process type in close operation + d9b0b9b3e2 net/ngbe: keep link down after device close + adb1b60601 net/txgbe: keep link down after device close + 46e19fcf10 net/ngbe: reconfigure MAC Rx when link update + 768e6a3d83 net/txgbe: reconfigure MAC Rx when link update + 12662c8d63 net/ngbe: prevent NIC from slowing down link speed + 401b94367f net/ngbe: fix flow control + ae0ac0ba92 net/txgbe: fix GRE tunnel packet checksum + 2c62e36805 net/txgbe: add Tx queue maximum limit + 64d7c1df88 net/netvsc: increase VSP response timeout to 60 seconds + af391d2427 hash: align SSE lookup to scalar implementation + 421c47495c mem: fix deadlock with multiprocess + 01d3e3c456 gpu/cuda: fix build with external GDRCopy + e55de889b7 bus/dpaa: fix build with asserts for GCC 13 + 517bb40874 random: initialize state for unregistered non-EAL threads + 4e986000b1 net/hns3: fix order in NEON Rx + 0e82ee1363 net/mana: add 32-bit short doorbell + e9ead33c9d net/mana: enable 32-bit build + b6200f6581 app/test: fix reference to master in bonding test + 2dec2783a0 net/hns3: fix traffic management dump text alignment + 6961856e84 net/hns3: fix traffic management thread safety + 15b43a21b2 net/hns3: fix flushing multicast MAC address + 71c9f50273 net/hns3: fix error code for multicast resource + 25c73bdc49 net/hns3: fix VF default MAC modified when set failed + ce57bf585c net/nfp: fix control message packets + de47856936 net/sfc: add missing error code indication to MAE init path + 563e7c87df net/sfc: account for data offset on Tx + 37863f1caa net/sfc: set max Rx packet length for representors + 38b7efe568 net/bonding: fix header for C++ + f4b30c5e2a rawdev: fix device class in log message + f76fc5ef15 eal/unix: fix firmware reading with external xz helper + 36c07ef565 mempool: fix default ops for an empty mempool + 36001100ff ci: fix race on container image name + + +* Wed Feb 14 2024 Open vSwitch CI - 3.1.0-97 +- Merging upstream branch-3.1 [RH git: 121e06c411] + Commit list: + 7171152ac8 ofproto-dpif-monitor: Remove unneeded calls to clear packets. + b5684d41d2 bfd: Set proper offsets and flags in BFD packets. + + +* Fri Feb 09 2024 Open vSwitch CI - 3.1.0-96 +- Merging upstream branch-3.1 [RH git: 5930b8ed0d] + Commit list: + fc6ae2463f Prepare for 3.1.5. + fdd8edc9ae Set release date for 3.1.4. + 91e621bd5a netdev-offload-tc: Check geneve metadata length. + d3f9eab1ab odp: ND: Follow Open Flow spec converting from OF to DP. + + +* Thu Feb 08 2024 Open vSwitch CI - 3.1.0-95 +- Merging upstream branch-3.1 [RH git: ed748d4ecc] + Commit list: + 2be57d0cc8 dpdk: Use DPDK 22.11.4 release for OVS 3.1. + + +* Mon Jan 29 2024 Open vSwitch CI - 3.1.0-94 +- Merging upstream branch-3.1 [RH git: 9c7a1ebb89] + Commit list: + ce06a9cf6d github: Update versions of action dependencies (Node.js 20). + + +* Mon Jan 29 2024 Timothy Redaelli - 3.1.0-93 +- Import redhat/pkgtool from 3.2 [RH git: 5ab2614753] + + +* Mon Jan 29 2024 Timothy Redaelli - 3.1.0-92 +- Import README.rst from 3.2 [RH git: 27392d5405] + + +* Mon Jan 29 2024 Timothy Redaelli - 3.1.0-91 +- Import redhat/merge from 3.2 [RH git: 4575067543] + + +* Mon Jan 29 2024 Timothy Redaelli - 3.1.0-90 +- Merge commit '42ed2ce550ca14cf939c2e6cc383a54c15878b74' into fast-datapath-rhel-8 [RH git: 67ba759419] + + +* Mon Jan 29 2024 Timothy Redaelli - 3.1.0-89 +- Add 'dpdk/' from commit '9dae7a15aea76313c592c11db44e1386e85f86fe' [RH git: 68fffc40f9] + git-subtree-dir: dpdk + git-subtree-mainline: 91d43f5108775f809175e9739961595b39c9974d + git-subtree-split: 9dae7a15aea76313c592c11db44e1386e85f86fe + + +* Mon Jan 29 2024 Timothy Redaelli - 3.1.0-88 +- Remove dpdk submodule [RH git: 91d43f5108] + You probably also wants to do: + + git submodule deinit -f dpdk + rm -rf .git/modules/dpdk + + +* Fri Jan 19 2024 Open vSwitch CI - 3.1.0-87 +- Merging upstream branch-3.1 [RH git: f86d6b1e21] + Commit list: + 7bb4f13a37 python: ovs: flow: Add meter_id to controller. + 41b2ae5165 python: ovs: flow: Make check_pkt_len action a list. + d0fc8a2892 python: ovs: flow: Add idle_age to openflow flows. + 3e2791f8c6 python: tests: Refactor test_odp section testing. + df59f9a225 python: ovs: flow: Add dp hash and meter actions. + 7958e50a1e python: ovs: flow: Add sample to nested actions. + fc0fb041af python: tests: Add info and key tests for OFPFlows. + 1d00d1da6f python: ovs: flow: Fix typo in n_packets. + + +* Fri Jan 19 2024 Open vSwitch CI - 3.1.0-86 +- Merging upstream branch-3.1 [RH git: 1e24146953] + Commit list: + bd5b5d3b3c ovs-atomic: Fix inclusion of Clang header by GCC 14. + + +* Thu Jan 11 2024 Open vSwitch CI - 3.1.0-85 +- Merging upstream branch-3.1 [RH git: baeb8a32dc] + Commit list: + dd3f0626ed ovsdb-idl.at: Test IDL behavior during database conversion. + 2c6b81ad6e tests-ovsdb: Switch OVSDB_START_IDLTEST to macro. + e9c5226cc2 python: idl: Handle monitor_canceled. + + +* Tue Jan 09 2024 Open vSwitch CI - 3.1.0-84 +- Merging upstream branch-3.1 [RH git: 0a1c4bfec5] + Commit list: + bb61b5fe8e vconn: Count vconn_sent regardless of log level. + e99e16b2bc backtrace: Fix error in log_backtrace() documentation. + 2095fdc897 ovsdb: trigger: Do not allow conversion in read-only mode. + b7809bcc65 ovsdb: jsonrpc-server: Fix the DSCP value in default options. + 020c9efcbe jsonrpc: Sort JSON objects while printing debug messages. + f0503a3745 tests: ovsdb: Use diff -up format for replay test. + ed3968450f ovsdb-server.at: Enbale debug logs in active-backup tests. + c0a453a919 ovsdb: transaction: Don't try to diff unchanged columns. + 6e988ef853 ovsdb: transaction: Avoid diffs for different type references. + d96ce5f2c5 ovsdb: Fix incorrect sharing of UUID and _version columns. + + +* Tue Jan 09 2024 Open vSwitch CI - 3.1.0-83 +- Merging upstream branch-3.1 [RH git: 0b299a1cd5] + Commit list: + ffa7d6492d ci: Update the GitHub Ubuntu runner image to Ubuntu 22.04. + + +* Thu Jan 04 2024 Open vSwitch CI - 3.1.0-82 +- Merging upstream branch-3.1 [RH git: e9d0b1d643] + Commit list: + 8fd5f77cd8 ovsdb-idl: Preserve change_seqno when deleting rows. + + +* Tue Dec 19 2023 Kevin Traynor - 3.1.0-81 +- Merging dpdk submodule [RH git: 071ee36ba9] + Commit list: + 42ed2ce550 Revert "net/iavf: fix abnormal disable HW interrupt" + + +* Thu Dec 07 2023 Open vSwitch CI - 3.1.0-80 +- Merging upstream branch-3.1 [RH git: 4c05b0f916] + Commit list: + 522a69bf67 system-dpdk: Wait for MTU changes to be applied. + + +* Tue Dec 05 2023 Open vSwitch CI - 3.1.0-79 +- Merging upstream branch-3.1 [RH git: 5ad14a20bd] + Commit list: + 9e963dbdab tunnel: Do not carry source port from a previous tunnel. + ae3c17f840 netdev-offload-tc: Fix offload of tunnel key tp_src. + d603759afd ofp-ct: Return error for unknown property in CT flush. + + +* Fri Dec 01 2023 Open vSwitch CI - 3.1.0-78 +- Merging upstream branch-3.1 [RH git: bc18365124] + Commit list: + 4151467438 cirrus: Update from FreeBSD 12 to 14. + + +* Fri Dec 01 2023 Open vSwitch CI - 3.1.0-77 +- Merging dpdk submodule [RH git: dd5b0ccec6] + Commit list: + db913c25d3 Merge tag 'v22.11.3' into 22.11 + b721cba875 Revert "net/i40e: revert link status check on device start" + 7849366693 version: 22.11.3 + 774466c6dc Revert "net/iavf: fix tunnel TSO path selection" + 5e87ad0cf8 version: 22.11.3-rc1 + 9fb53245ea common/cnxk: fix CPT backpressure disable on LBK + 17fef613a1 net/nfp: fix offloading flows + 338d0d589a net/nfp: fix Tx descriptor free logic of NFD3 + 6c779801be test/bonding: fix include of standard header + 22c3a016bb net/ngbe: fix RSS offload capability + fc56ce5a01 net/mana: fix Tx queue statistics + 50dc477c2e doc: fix link to flow capabilities from bnxt guide + 56e6c7ef88 doc: update BIOS settings and supported HW for NTB + 89a839ade5 doc: fix description of runtime directories + b5513f8e84 doc: improve wording of cuda guide + 1a4aa1d333 doc: add flow template API requirements for mlx5 + 75a8d9b6d9 app/testpmd: fix meter mark handle update + 4b42c698a2 net/mlx5: fix handle validation for meter mark + 6b686c0912 net/mlx5: fix validation for conntrack indirect action + 5cf1399e93 net/mlx5: fix MPRQ stride size for headroom + 2e3420f544 net/mlx5: fix LRO TCP checksum + 8397c9086a net/mlx5: fix drop action memory leak + e6365bd3d2 net/e1000: fix Rx and Tx queue status + 4e200ede54 net/igc: fix Rx and Tx queue status + 4cae890867 net/ixgbe: fix Rx and Tx queue status + 912704eee5 common/iavf: fix MAC type for 710 NIC + ca8c2ad458 net/iavf: fix stop ordering + 0a1fa750bc net/i40e: fix comments + cb30cddab7 doc: fix typos and wording in flow API guide + c12b2943c9 devtools: fix bashism in mailmap check + 0192d0ac53 kni: fix build with Linux 6.5 + 44632490d9 examples/l3fwd: fix duplicate expression for default nexthop + d385831f49 ipsec: fix NAT-T header length + 93d999a062 examples/ipsec-secgw: fix TAP default MAC address + 1fad66d244 examples/ipsec-secgw: fix socket ID default value + 76422de830 app/crypto-perf: fix socket ID default value + 1a98004f93 cryptodev: fix device socket ID type + 900b4ca132 examples/fips_validation: fix digest length in AES-GCM + 31daa7382f test/crypto: fix PDCP-SDAP test vectors + 440e528ae5 common/qat: detach crypto from compress build + 2e8f88824d crypto/qat: fix null algorithm digest placement + 64db6b40e9 baseband/fpga_5gnr_fec: fix starting unconfigured queue + 4325aff0d4 baseband/fpga_5gnr_fec: fix possible division by zero + 5ff397c56f net/ice: fix RSS hash key generation + 9b7215f150 net/iavf: fix tunnel TSO path selection + 35e69229eb net/ice: fix 32-bit build + 90a0f328b8 net/iavf: fix VLAN insertion in vector path + c14d373562 net/ice: fix VLAN mode parser + 16ce632452 net/mlx5: fix query for NIC flow capability + d165bff7dc net/mlx5: fix RSS expansion inner buffer overflow + 6a567b41ed net/mlx5: forbid MPRQ restart + 03e279ee8b net/mlx5: fix flow workspace destruction + b5075ab2f9 net/mlx5: reduce counter pool name length + b74fa83cf2 net/mlx5: fix profile check of meter mark + db32b39aad common/mlx5: fix obtaining IB device in LAG mode + fcf46d9c1d net/mlx5: fix error in VLAN actions creation + 0c50d16325 net/mlx5: fix error set for age pool initialization + 77159a5063 net/mlx5: fix error set in control tables create + aa9f4b4c58 net/mlx5: fix return value of vport action + d8ec0d83dd net/mlx5: fix error set in Tx representor tagging + d3457cd43d net/mlx5: fix flow dump for modify field + a944f7e99c ethdev: fix potential leak in PCI probing helper + b6025f7a95 net/hns3: fix index to look up table in NEON Rx + a2dc813bc4 net/mana: fix WQE count for ringing RQ doorbell + 9d30178ddd net/mana: fix counter overflow for posted WQE + 92d3673001 app/testpmd: fix flow rule number parsing + 36791bbc4b net/hns3: fix non-zero weight for disabled TC + 56b89656df app/testpmd: revert primary process polling all queues fix + 09d8d1fe4f net/txgbe: fix blocking system events + 7a44cac46c doc: fix number of leading spaces in hns3 guide + 8b5633f1e2 doc: fix syntax in hns3 guide + ece673c7a1 doc: fix kernel patch link in hns3 guide + 9b9ae1aff1 net/hns3: delete duplicate macro definition + 3a935374f2 app/testpmd: fix checksum engine with GTP on 32-bit + 67912c46f4 net/netvsc: fix sizeof calculation + d5309394d1 member: fix PRNG seed reset in NitroSketch mode + 1db71d413e hash: fix reading unaligned bits in Toeplitz hash + 7e6b4e9d63 mem: fix memsegs exhausted message + 7a57c9edd6 fib: fix adding default route + fc4be70ded devtools: fix mailmap check for parentheses + 90aa091a20 ipc: fix file descriptor leakage with unhandled messages + 008fdd0e07 raw/ntb: avoid disabling interrupt twice + 54182cce72 cryptodev: fix comments of modular operation parameters + a60ddd6954 net/cnxk: fix flow queue index validation + cd88b5ff39 net/cnxk: fix cookies check with security offload + c98cf071f8 net/cnxk: flush SQ before configuring MTU + dcd21e65bd common/mlx5: adjust fork call with new kernel API + 79310b1b61 net/mlx5: fix device removal event handling + ce6b9b3772 net/mlx5: fix risk in NEON Rx descriptor read + e149ea19c2 net/iavf: fix protocol agnostic offloading with big packets + 04de2b4a56 net/ice: fix protocol agnostic offloading with big packets + 15b67727ca net/ice: fix MAC type of E822 and E823 + a45102d324 net/e1000: fix queue number initialization + 4f26624221 net/i40e: fix tunnel packet Tx descriptor + 2bdeeb53da net/iavf: fix abnormal disable HW interrupt + 520330cbb1 common/idpf: fix memory leak on AVX512 Tx queue close + 00f8c02c41 net/ixgbe: add proper memory barriers in Rx + c8b6f57dc4 net/ice: fix tunnel packet Tx descriptor + 8a0712739d net/iavf: fix VLAN offload with AVX512 + 64c315b9e9 common/sfc_efx/base: fix Rx queue without RSS hash prefix + 47326eff66 net/nfp: fix address always related with PF ID 0 + 715143b51e net/nfp: fix representor name too long + 16a82d0d1a app/testpmd: fix primary process not polling all queues + 368138b61b net/ngbe: remove redundant codes + e66f9e3fdf net/ngbe: fix link status in no LSC mode + 1c7e19d4a0 net/ngbe: adapt to MNG veto bit setting + c523de9115 net/ngbe: fix extended statistics + 409cd85a16 net/txgbe: fix extended statistics + 19c8a701a7 net/txgbe: fix to set autoneg for 1G speed + 57b3a57dfa net/txgbe: adapt to MNG veto bit setting + a62297801c net/txgbe: fix interrupt enable mask + d96c0259e7 net/txgbe/base: fix Tx with fiber hotplug + ff476852e7 net/bonding: fix destroy dedicated queues flow + c193423238 net/bonding: fix startup when NUMA is not supported + 00d2dc7c42 net/nfp: fix VNI of IPv6 NVGRE encap action + 7797d05cd9 net/nfp: fix VNI of IPv4 NVGRE encap action + dc6adc599c net/nfp: fix VNI of VXLAN encap action + fa5f6cff44 ethdev: update documentation for API to get FEC + 7594b6e44d ethdev: check that at least one FEC mode is specified + ff56bf6f3a ethdev: update documentation for API to set FEC + c25e954265 net/nfp: fix endian conversion for tunnel decap action + 502acb0f83 net/nfp: fix unneeded endian conversion + a526063092 net/nfp: fix IPv6 address for set flow action + 64839553ee test/security: fix event inline IPsec reassembly tests + 643423f0b7 crypto/openssl: skip workaround at compilation time + 91085d3b58 crypto/openssl: fix memory leak in auth processing + 981792ebc4 crypto/qat: fix sym device prototype + 791a0227d2 common/qat: fix command parameter corruption + 0d5c3e03c5 ci: fix libabigail cache in GHA + ef98a4da59 mbuf: fix Doxygen comment of distributor metadata + 5887a7d14d test: add graph tests + 2bdfd0d7ba examples/fips_validation: fix external build + 72c87f820d examples/l2fwd-cat: fix external build + 11757d12ee dma/dpaa2: set VFA bit for route-by-port with VF + 9047d5c9bf doc: remove warning with Doxygen 1.9.7 + d8e35e7efe doc: fix typo in graph guide + 55fbb9de8d test/mbuf: fix crash in a forked process + 4d116ff1ac net/iavf: fix virtchnl command called in interrupt + bd5c63549a net/ice: fix outer UDP checksum offload + bae61d2305 net/ice: initialize parser for double VLAN + 1f35681b6e net/ice: fix timestamp enabling + 3b032bbf78 net/ice: adjust timestamp mbuf register + 1d71b68237 net/ice/base: remove unreachable code + 7691c220b6 net/ice/base: fix incorrect defines for DCBx + 4ec6da600e net/ice: fix DCF control thread crash + c4d749115d net/iavf: release large VF when closing device + 21ec365e75 net/ice: fix DCF RSS initialization + 83ad87933a net/ice: fix statistics + b9f3b81e9e common/idpf/base: fix memory leaks on control queue + 1b04a2f618 common/idpf/base: fix parameters when send msg to cp + e5868f6648 common/idpf/base: fix ITR register definitions for AVF + 4cc85337b3 common/idpf/base: fix control queue send and receive + 1bfcca5b65 common/idpf: remove device stop flag + c86c1efd2f net/idpf: fix Rx data buffer size + 78c374bf41 net/iavf: fix Rx data buffer size + 1f9af08714 net/ice: fix Rx data buffer size + 7a72db6add net/i40e: fix Rx data buffer size + 65303f7c26 doc: fix typo in cnxk platform guide + d9ba8a4251 net/qede: fix RSS indirection table initialization + df11a52e33 common/cnxk: fix receive queue with multiple mask + 6cbb5fc911 common/cnxk: fix inline device VF identification + d0f357fb6e common/cnxk: fix uninitialized pointer read + 103a33d02d common/cnxk: fix setting channel mask for SDP interfaces + c1e167a078 event/cnxk: fix mempool cookies check + 338514e558 event/cnxk: fix Tx adapter data pointer + 46d6c05770 common/cnxk: fix IPsec IPv6 tunnel address byte swap + 9d1dbc6a01 mempool/cnxk: avoid hang when counting batch allocs + 255fcff79e net/mlx5: fix drop action attribute validation + b53417d63b net/mlx5: fix duplicated tag index matching in SWS + 124a919b4e net/mlx5: forbid duplicated tag index in pattern template + eb02902423 net/mlx5: fix VXLAN matching with zero value + 2e6f71cb51 net/mlx5: fix matcher layout size calculation + 9015baea29 net/mlx5: enhance error log for tunnel offloading + 2a45186cf4 net/virtio-user: fix leak when initialisation fails + d228fa562b net/virtio: fix initialization to return negative errno + a43f6d459a net/virtio: propagate interrupt configuration error values + 94bed3c2fa vhost: fix invalid call FD handling + fe3e1fc9d8 vhost: fix notification stats for packed ring + 3a5561e8b5 crypto/ipsec_mb: optimize allocation in session + a659f5f18c test/crypto: fix IPsec AES CCM vector + 7e0080ff4f crypto/cnxk: fix IPsec CCM capabilities + c53467f356 crypto/ipsec_mb: fix enqueue counter for SNOW3G + d2e09fc0fc test/crypto: fix session creation check + e3456f2478 crypto/openssl: fix memory free + 48b48d2ba8 cryptodev: clarify error codes for symmetric session + 400a3302de examples/ipsec-secgw: fix zero address in ethernet header + 6217afd84c test/crypto: fix return value for SNOW3G + 228857a34c crypto/scheduler: fix last element for valid args + 4413744138 crypto/qat: fix stack buffer overflow in SGL loop + 0af5332dff doc: fix auth algos in cryptoperf app + 3a8502c73d net/vmxnet3: fix return code in initializing + c40b9a9873 net/nfp: fix TP flow action for UDP + 74870817fb net/nfp: fix flow hash table creation + 18493825d1 net/nfp: fix representor creation + de78bd45d6 net/nfp: fix control mempool creation + caa96e94b0 net/nfp: fix TOS of IPv6 NVGRE encap flow action + 67544106ac net/nfp: fix TOS of IPv6 GENEVE encap flow action + c81816d07f net/nfp: fix TOS of IPv6 VXLAN encap flow action + a1d864c39b net/nfp: fix IPv6 flow item + aae5fcb267 net/nfp: fix disabling promiscuous mode + e659a163e3 ethdev: fix calloc arguments + 175e7a7bac net/hns3: fix IMP reset trigger + 219305d227 net/hns3: fix redundant line break in log + 4292ebcef4 net/hns3: fix inaccurate log + 1c81e1a0a8 net/hns3: fix uninitialized variable + f99fa19b60 net/hns3: fix device start return value + 19c20cef27 net/hns3: fix mbuf leakage when RxQ started after reset + e9ade95ac4 net/hns3: fix mbuf leakage when RxQ started during reset + 422a5e09d2 net/hns3: extract PTP to its own header file + a8ad010661 net/hns3: uninitialize PTP + 3999b58cd3 net/hns3: fix RTC time after reset + 69bff6ea1c net/hns3: fix RTC time on initialization + bdf2131156 doc: fix format in flow API guide + 9ca1814402 net/hns3: fix missing FEC capability + 340cb03d7f net/hns3: fix FEC mode check + b048cdca82 net/hns3: fix FEC mode for 200G ports + a678d7da2b ethdev: fix indirect action conversion + d859368e8f net/hns3: fix Rx multiple firmware reset interrupts + f2dd43a6e9 net/hns3: fix variable type mismatch + 2a7aef1dd0 net/hns3: fix never set MAC flow control + 88cf99ed01 net/sfc: invalidate dangling MAE flow action FW resource IDs + 3254062488 net/mana: return probing failure if no device found + 631f57f5ef ethdev: fix MAC address occupies two entries + 32e0eaa22b net/txgbe: fix use-after-free on remove + 799cc0612c net/vmxnet3: fix drop of empty segments in Tx + acbaa6bb5d net/nfp: fix VLAN push flow action + 9d1fbdb584 app/testpmd: fix GTP L2 length in checksum engine + e7e0590338 net/dpaa2: fix checksum good flags + ecd3e1f354 net/mana: optimize completion queue by batch processing + 5b1a78987f net/mana: avoid unnecessary assignments in data path + 753a735ea0 net/mana: use datapath logging + 8e27036b63 net/tap: set locally administered bit for fixed MAC address + bd14912ab4 net/sfc: stop misuse of Rx ingress m-port metadata on EF100 + a1dfa52406 net/hns3: fix build warning + 10a2ee0333 eal/linux: fix legacy mem init with many segments + f4d5f30fe4 eal/linux: fix secondary process crash for mp hotplug requests + 53e6b8c3f0 event/cnxk: fix nanoseconds to ticks conversion + cf79458fad eventdev/timer: fix buffer flush + d3f784ad5f event/cnxk: fix setting attributes in empty get work + d0257dc168 event/dsw: free rings on close + c12daa6d2d doc: fix event timer adapter guide + ca72f7fdba eventdev/timer: fix timeout event wait behavior + 5ecf2e459d eal/x86: improve multiple of 64 bytes memcpy performance + f4ccd39056 pci: fix comment referencing renamed function + c3b5322fc3 build: fix warning when getting NUMA nodes + 44a9b3b4f7 ci: fix build for Arm cross compilation in GHA + 94babf61c5 eal: avoid calling cleanup twice + 6413085d5f test/malloc: fix statistics checks + 37e859deb8 test/malloc: fix missing free + 2d3c4df8f5 pipeline: fix double free for table stats + cada66aff2 ring: fix dequeue parameter name + 57dd0e53b1 telemetry: fix autotest on Alpine + f14b25d925 kernel/freebsd: fix function parameter list + 47da400295 vfio: fix include with musl runtime + 8742732344 ring: fix use after free + f9a20a28ec examples/ntb: fix build with GCC 13 + 28aa181578 examples/ip_pipeline: fix build with GCC 13 + 156b423f54 kni: fix build with Linux 6.3 + 403d133d7f net: fix return type of IPv4 L4 packet checksum + d75b66ad1c version: 22.11.2 + 8edef444c7 build: detect backtrace availability + aa96e66c9a version: 22.11.2-rc1 + 9dcf799d88 common/mlx5: use just sufficient barrier for Arm + 6940159325 net/mlx5/hws: fix IPv4 fragment matching + f567b1ee98 common/cnxk: fix IPv6 extension header parsing + f9f94070e0 doc: fix DCF instructions in ice guide + d4dc7b1d1b doc: add Linux capability to access physical addresses + 7ac4d1cebf doc: fix pipeline example path in user guide + abf5150780 devtools: move mailmap check after patch applied + 0dce56b788 acl: fix crash on PPC64 with GCC 11 + ac849cac65 pdump: fix build with GCC 12 + fcf7e0e6f7 test/crypto: fix statistics error messages + 95c2df95d2 doc: fix code blocks in cryptodev guide + 2fe7fcc1a3 net/mlx5: fix sysfs port name translation + 80ec04827f net/mlx5: fix CQE dump for Tx + 1f4de71406 net/mlx5/hws: fix error code of send queue action + 644bcdb856 net/mlx5: fix build with GCC 12 and ASan + d4335766d4 net/mlx5/hws: fix pattern creation + 1cbb884f35 app/testpmd: fix encap/decap size calculation + 6a3a2809eb examples/qos_sched: fix config entries in wrong sections + 782eda8807 net/ipn3ke: fix representor name + abe24f4bd0 net/ipn3ke: fix thread exit + 3bdf07e682 bus/ifpga: fix devargs handling + 0680a33bc7 net/mlx5: fix isolated mode if no representor matching + e10c220053 net/mlx5: fix egress group translation in HWS + 3f7e967684 doc: fix LPM support in l3forward guide + 003a860c86 examples/l3fwd: remove hash entry number + e528b4f96c net/mlx5: fix hairpin Tx queue reference count + b836bb43cf net/iavf: fix device stop during reset + 158c05028f net/idpf: reset queue flag when queue is stopped + 8cb2ee860c net/i40e: fix MAC loopback on X722 + 8cc24b8aab net/e1000: fix saving of stripped VLAN TCI + 91a60dfeee net/i40e: fix AVX512 fast-free path + 8a6d064801 net/sfc: invalidate switch port entry on representor unplug + d59d4fbffa net/virtio: remove address width limit for modern devices + 7be8e75277 net/vhost: fix Rx interrupt + 1ecf04df54 net/vhost: fix leak in interrupt handle setup + 9736b58d62 net/vhost: add missing newline in logs + b80c949b77 app/bbdev: check statistics failure + 39ca0f7397 crypto/ipsec_mb: relax multi-process requirement + 6919a8d874 app/compress-perf: fix remaining data for ops + dc6e5405ca test/crypto: fix skip condition for CPU crypto SGL + 58dff4134c test/crypto: fix capability check for ZUC cipher-auth + 0217458d5e test/crypto: fix ZUC digest length in comparison + 7d3df2701d app/testpmd: fix interactive mode on Windows + ef700bf7d0 gpudev: export header file for external drivers + db12268a95 eal/unix: fix thread creation + 29cdc2b88f test/mbuf: fix test with mbuf debug enabled + 64f4cb8bce test: fix segment length in packet generator + 8f35765d15 reorder: fix sequence number mbuf field register + 38369a1974 ring: silence GCC 12 warnings + ebcd68e43a raw/skeleton: fix selftest + 8160c206ca examples/ipsec-secgw: fix offload variable init + 55f236de5c app/flow-perf: fix division or module by zero + 161ef7c943 app/crypto-perf: fix test file memory leak + 72c1da7b2d crypto/openssl: fix freeing in RSA EVP + 1536a7304a crypto/qat: fix SM3 auth mode + eba4eee77b crypto/ccp: fix PCI probing + f081724c6b net/mlx5: fix Windows build with MinGW GCC 12 + 5af10c2a20 net/hns3: add verification of RSS types + 3a5105f1de net/hns3: reimplement hash flow function + 7307f96575 net/hns3: separate flow RSS config from RSS conf + 796c8188a5 net/hns3: allow adding queue buffer size hash rule + adf239587c net/hns3: save hash algo to RSS filter list node + fc620e6832 net/hns3: use new RSS rule to configure hardware + a2804bcc0c net/hns3: separate setting and clearing RSS rule + 679028a2ee net/hns3: separate setting RSS types + 3b85ef547e net/hns3: separate setting redirection table + 99c75dbbfe net/hns3: separate setting hash key + 9c24ae1b2d net/hns3: separate setting hash algorithm + dde9ec15ea net/hns3: use hardware config to report redirection table + dbcf64d12e net/hns3: use hardware config to report hash types + 68da0b3986 net/hns3: use hardware config to report hash key + 5042fd8459 net/hns3: fix possible truncation of redirection table + 81fbc0298c net/hns3: fix possible truncation of hash key when config + 73b11178a3 mem: fix heap ID in telemetry + 57294e8df4 app/testpmd: cleanup cleanly from signal + 564dbb6a03 cmdline: handle EOF as quit + 1a22081244 cmdline: make rdline status not private + 09b626852e kni: fix possible starvation when mbufs are exhausted + 993c0d08eb net/sfc: fix resetting mark in tunnel offload switch rules + f6bdbdf6f2 common/sfc_efx/base: add MAE mark reset action + 4224d5f521 regex/mlx5: fix doorbell record + b5512d3186 regex/mlx5: utilize all available queue pairs + cb4baf72a4 table: fix action selector group size log2 setting + 60f6d9449c raw/skeleton: fix empty devargs parsing + 0848681e90 dma/skeleton: fix empty devargs parsing + ce3c0aa7aa net/virtio: fix empty devargs parsing + 33b1cea25d net/hns3: fix empty devargs parsing + 63ba1d9f2c cryptodev: fix empty devargs parsing + 00e3a4efbc compressdev: fix empty devargs parsing + 6567e0cf76 kvargs: add API documentation for process callback + bb296faffc net/nfp: fix MTU configuration order + bf878ca704 eal/windows: fix thread creation + fac8d80e0d common/cnxk: add memory clobber to steor and ldeor + 67479d24a4 net/cnxk: fix LBK BPID usage + 14688b03c0 net/ice: fix Rx timestamp + 8ec56c115a net/i40e: fix maximum frame size configuration + 287a57f832 net/i40e: revert link status check on device start + dc67b490e8 net/ixgbe: fix IPv6 mask in flow director + b0901e6d5b app/testpmd: fix secondary process packet forwarding + 17637f9c0f net/nfp: fix offload of multiple output actions + fc325877f6 net/nfp: fix set DSCP flow action + 336d6c1c41 net/nfp: fix set TTL flow action + 264cbadbee net/nfp: fix set TP flow action + dcc0be2c8a net/nfp: fix set IPv6 flow action + d6fb8fc9c1 net/nfp: fix set IPv4 flow action + a0a1f437b2 net/nfp: fix set MAC flow action + aaa501436c ethdev: remove telemetry Rx mbuf alloc failed field + 03cfac3c99 net/nfp: fix getting RSS configuration + 5475d251f1 net/mana: fix stats counters + 00af2661ac ethdev: fix build with LTO + cd2635a2b5 app/testpmd: fix packet transmission in noisy VNF engine + e8996dc08d app/testpmd: fix packet count in IEEE 1588 engine + a4141026d7 app/testpmd: fix Tx preparation in checksum engine + 62324129c1 net/mana: enable driver by default + 64d858dc03 net/gve: fix offloading capability + 90f12bb74f net/nfp: fix 48-bit DMA support for NFDk + 6ede2ab472 common/cnxk: fix auth key length + 995a6460b2 examples/fips_validation: add extra space in JSON buffer + 903f4ee07d examples/fips_validation: fix AES-XTS sequence number + 91c80e936a examples/fips_validation: fix AES-GCM tests + 809de506c9 examples/fips_validation: fix integer parsing + ac87a06f6b examples/fips_validation: fix MCT output for SHA + 435188c3c4 compress/mlx5: fix queue setup for partial transformations + 2a091b2f31 compress/mlx5: fix output Adler-32 checksum offset + 9d7902bf42 compress/mlx5: fix decompress xform validation + 2bf416240d examples/ipsec-secgw: fix auth IV length + 33f3ef5021 net/virtio: deduce IP length for TSO checksum + 950227315b vhost: fix OOB access for invalid vhost ID + a6d0c8f6aa vhost: fix slot index in async split virtqueue Tx + 47cef65600 test/bbdev: remove check for invalid opaque data + dccdf95daa test/bbdev: extend HARQ tolerance + cdfa1794a6 test/bbdev: fix crash for non supported HARQ length + 4bdaf50031 baseband/acc: fix check after deref and dead code + bf521b9f80 baseband/acc: fix iteration counter in TB mode + ff4d7c9a71 baseband/acc: prevent to dequeue more than requested + 6c9f6f15ee baseband/acc: add explicit mbuf append for soft output + 63797158c2 baseband/acc: protect from TB negative scenario + f76551552e eventdev: fix memory size for telemetry + 2f9bb3f72a event/cnxk: fix SSO cleanup + 0a3f30f07c doc: fix reference to event timer header + e806f0529d doc: add gpudev to the Doxygen index + 300d83b989 eal/windows: fix pedantic build + 03299eb5d4 net/mlx5: fix crash on action template failure + 4dc4547f2d common/cnxk: fix second pass flow rule layer type + dc7302e733 examples/qos_sched: fix Tx port config when link down + ecb065fe53 examples/cmdline: fix build with GCC 12 + 1b93563378 eal: use same atomic intrinsics for GCC and clang + a5d83cee8e build: fix toolchain definition + 15ae43b33a test/reorder: fix double free of drained buffers + b515c436e6 reorder: invalidate buffer from ready queue in drain + 106e0f2637 dma/ioat: fix error reporting on restart + 90ad21e988 dma/ioat: fix indexes after restart + 881919396f dma/ioat: fix device stop if no copies done + fb395ef1d0 eal/freebsd: fix lock in alarm callback + a656595e9b sched: fix alignment of structs in subport + 38465cec0a app/testpmd: fix crash on cleanup + 31f4d9ce13 net/nfp: restrict flow flush to the port + 7ae80e1379 net/nfp: fix VNI of VXLAN encap action + 8cdbec0d75 net/bnxt: fix link state change interrupt config + fd1ff4fecb common/cnxk: fix channel mask for SDP interfaces + ad62d3f410 app/compress-perf: fix testing single operation + 17f4281693 app/compress-perf: fix some typos + c028045915 net/iavf: fix VLAN offload with AVX2 + 3c047b3d86 net/ixgbe: enable IPv6 mask in flow rules + 79781cd646 net/iavf: fix building data desc + 644d38523a net/iavf: protect insertion in flow list + e13bcf7645 net/ice: fix validation of flow transfer attribute + 199591d8d1 net/i40e: fix validation of flow transfer attribute + 5aacd2a62c net/iavf: add lock for VF commands + dbe54bfd9b net/ixgbe: fix firmware version consistency + bfa9955652 net/idpf: fix driver infos + 96ccb4c287 net/idpf: fix mbuf leak in split Tx + 489ddd71b3 net/i40e: reduce interrupt interval in multi-driver mode + 6e50af124f net/iavf: fix outer UDP checksum offload + 4c6cddb596 net/mlx5: check compressed CQE opcode in vectorized Rx + 547b239a21 net/mlx5: ignore non-critical syndromes for Rx queue + 47f5a0e5f3 net/mlx5: fix error CQE dumping for vectorized Rx + 9e0308d5ef net/mlx5/hws: fix memory leak on general pool DB init + 394ba3cfc4 net/mlx5: fix read device clock in real time mode + 35eadc22bc net/mlx5: fix warning for Tx scheduling option + a04fa37e6e net/mlx5: fix wait descriptor opcode for ConnectX-7 + d5be082621 net/mlx5: fix flow sample with ConnectX-5 + 575cfce6a7 common/mlx5: fix offset of a field + f3282a003f common/mlx5: improve AES-XTS tweak capability check + 4434048bbe net/mlx5: fix GENEVE resource overwrite + dcb16c48e0 net/mlx5: fix available tag registers calculation for HWS + a93bb50b6c net/txgbe: fix Rx buffer size in config register + 3a842fbbb7 net/sfc: enforce fate action in transfer flow rules + be7d6baf28 net/sfc: fix MAC address entry leak in transfer flow parsing + 56789776c6 net/hns3: extract common functions to set Rx/Tx + 13f062ec55 net/hns3: make getting Tx function static + 1df48cce5e net/hns3: separate Tx prepare from getting Tx function + 418d6cbc35 net/hns3: remove debug condition for Tx prepare + 873d6edb18 net/hns3: add debug info for Rx/Tx dummy function + c6b36e8183 net/hns3: fix burst mode query with dummy function + 6c9834cd2a net/nfp: fix max DMA length + f418af17fc app/testpmd: fix link check condition on port start + e4f7453ec1 net/sfc: export pick transfer proxy callback to representors + 6ae7e4345e net/hns3: fix duplicate RSS rule check + 5c129d8898 net/hns3: fix config struct used for conversion + f2de3c967d net/hns3: fix warning on flush or destroy rule + 2e99d819d3 net/hns3: remove useless code when destroy valid RSS rule + d45c8fe415 net/hns3: use RSS filter list to check duplicated rule + 93fa374ce4 net/hns3: fix clearing RSS configuration + 9126ed75be net/hns3: fix RSS key size compatibility + 36d6105f51 net/hns3: refactor set RSS hash algorithm and key interface + 14d988ad25 net/hns3: extract common function to query device + ffc3022e2d net/hns3: fix log about indirection table size + 547a2c7a55 net/txgbe: fix interrupt loss + c22430a6e5 net/ngbe: add spinlock protection on YT PHY + 767d609db7 net/ngbe: fix packet type to parse from offload flags + c96c4e1b9e net/txgbe: fix packet type to parse from offload flags + e7149d390b net/txgbe: fix default signal quality value for KX/KX4 + a1d3811bc0 app/testpmd: fix forwarding stats for Tx dropped + 8eefe1d245 doc: fix description of L2TPV2 flow item + 5876103dd2 net/hns3: declare flow rule keeping capability + 58910572e8 net/virtio-user: fix device starting failure handling + 78d828c247 vhost: fix possible FD leaks on truncation + eab1940f97 vhost: fix possible FD leaks + cf313170bf vhost: decrease log level for unimplemented requests + 0772cc1d89 eal: cleanup alarm and hotplug before memory detach + f7825956c0 eventdev/timer: fix overflow + fd3e2fb617 test/mbuf: fix mbuf reset test + cc69b3523e drivers/bus: fix leak for devices without driver + acacb53810 eal/linux: fix hugetlbfs sub-directories discovery + 0343d4cb72 telemetry: fix repeat display when callback don't init dict + f5803ba5c5 raw/ifpga/base: fix init with multi-process + 08019befc3 mem: fix hugepage info mapping + 23cb90427a net/cnxk: validate RED threshold config + d7d670cbe2 common/cnxk: reduce channel count per LMAC + 4060bba354 common/cnxk: fix IPv6 extension matching + 5444957e57 common/cnxk: fix dual VLAN parsing + f40c4f3e0b net/cnxk: fix deadlock in security session creation + 78c0d2ab85 common/cnxk: fix aura ID handling + 87725aa6d2 net/cnxk: fix packet type for IPv6 packets post decryption + efb6eccf8e event/cnxk: fix timer operations in secondary process + 903d4a18a1 event/cnxk: fix burst timer arm + 77f50c0971 eventdev/eth_rx: fix getting adapter instance + 0a78560065 event/cnxk: wait for CPT flow control on WQE path + 5e9f154f81 eventdev/crypto: fix function symbol export + 48870212ad doc: fix firmware list in bnxt guide + a4d58197b2 net/bnxt: fix RSS hash in mbuf + 3f1568d969 net/bnxt: fix Rx queue stats after queue stop and start + bc1682ee55 net/bnxt: fix Tx queue stats after queue stop and start + d357ef8a3c crypto/ipsec_mb: fix ZUC-256 maximum tag length + 031b48d670 compressdev: fix end of driver list + ce54658be7 cryptodev: fix sym session mempool creation description + cbd714b1f9 test/crypto: add missing MAC-I to PDCP vectors + ca39bc2337 test/crypto: fix typo in AES test + 11f862d605 crypto/ipsec_mb: remove unnecessary null check + 24dc362291 crypto/openssl: fix warning on copy length + 2d28201221 crypto/ccp: fix IOVA handling + f11d779c24 crypto/ccp: remove some dead code for UIO + 4e76b62756 crypto/ccp: remove some printf + f3d0a011e3 baseband/acc: fix acc100 queue mapping to 64 bits + 8f06dfc381 baseband/acc: fix multiplexing acc100 operations + 3c330a28d7 baseband/acc: fix acc100 iteration counter in TB + f770622597 baseband/acc: fix memory leak on acc100 close + e44360ae85 app/bbdev: add allocation checks + 9f13ab03f6 app/bbdev: fix build with optional flag + 1e82bbe91d app/bbdev: fix build with optional flag + be1187e1a6 vhost: fix net header settings in datapath + 72820dd60f vdpa/ifc: fix reconnection in SW-assisted live migration + a347909ae2 vdpa/ifc: fix argument compatibility check + ab2779057e build: fix dependencies lookup + 573de4f522 app/dumpcap: fix storing port identifier + cb3c0ba47e examples/qos_sched: fix debug mode + 0e846591b5 mem: fix telemetry data truncation + 453a4d30ed cryptodev: fix telemetry data truncation + 16f272c96f mempool: fix telemetry data truncation + bd11b88f8b ethdev: fix telemetry data truncation + 4f9c6db77b telemetry: move include after guard + 8ab731497d app/testpmd: fix interactive mode with no ports + 1df8c20923 net/nfp: fix teardown of flows sharing a mask ID + 375086efb5 net/nfp: store counter reset before zeroing flow query + 58db4361dc net/hns3: fix inaccurate RTC time to read + 36ec039309 net/ena: fix deadlock in RSS RETA update + f029ceb7aa net/nfp: fix firmware name derived from PCI name + a87a84560c net/nfp: fix Tx packet drop for large data length + 5ef77a1221 graph: fix node shrink + b9a14f6737 gpudev: fix deadlocks when registering callback + 1586e3b7bc fbarray: fix metadata dump + 23a5b25c8a bus/fslmc: fix deadlock on MC send command timeout + a0b1faaad2 crypto/qat: fix build + a636c94996 crypto/qat: fix build for generic x86 with GCC 12 + b527b4c168 crypto/qat: fix stream cipher direction + e2c05f4621 examples/l2fwd-event: fix worker cleanup + 478bec21d0 eventdev/eth_tx: fix devices loop + 4c388e80e9 eventdev/crypto: fix failed events + ddaec3cd54 eventdev/crypto: fix overflow in circular buffer + e1ca28205f eventdev/crypto: fix offset used while flushing events + d66216823d eventdev/crypto: fix enqueue count + b81247d47b app/crypto-perf: fix IPsec direction + 683ce01b19 app/crypto-perf: fix SPI zero + bbba565c73 app/crypto-perf: fix session freeing + a932cd545f app/crypto-perf: fix number of segments + 193ff405b3 crypto/cnxk: fix digest for empty input data + a996e9816f devtools: fix escaped space in grep pattern + b6d9ddee2c doc: fix dependency setup in l2fwd-cat example guide + 760cf7543c hash: fix GFNI implementation build with GCC 12 + c8af2a4ad5 kni: fix build on RHEL 9.1 + fc98f9c6d7 eal/windows: mark memory config as complete + c12e76d298 devtools: fix name check with mbox files + 07e68b092c drivers: fix symbol exports when map is omitted + + +* Thu Nov 30 2023 Open vSwitch CI - 3.1.0-76 +- Merging upstream branch-3.1 [RH git: 6215fe8715] + Commit list: + ab51834863 dpdk: Use DPDK 22.11.3 release for OVS 3.1. + + +* Wed Nov 29 2023 Open vSwitch CI - 3.1.0-75 +- Merging upstream branch-3.1 [RH git: d2cdf5ef89] + Commit list: + bfd87f6f2d ovs-ofctl: Correctly mark the CT flush commands. + + * Fri Nov 24 2023 Open vSwitch CI - 3.1.0-74 - Merging upstream branch-3.1 [RH git: 55bed59ce6] Commit list: