From 0ea619c80146bffd4fef7ac1a8a2aa07f9003eb0 Mon Sep 17 00:00:00 2001

From: Dumitru Ceara <dceara@redhat.com>

Date: Fri, 5 Feb 2021 23:29:29 +0100

Subject: [PATCH 2/2] lflow: Use learn() action to generate LB hairpin reply

 flows.

The main trait of load balancer hairpin traffic is that it never leaves

the local hypervisor.  Essentially this means that only hairpin

openflows installed for logical switches that have at least one logical

switch port bound locally can ever be hit.

Until now, if a load balancer was applied on multiple logical switches

that are connected through a distributed router, ovn-controller would

install flows to detect hairpin replies for all logical switches. In

practice this leads to a very high number of openflows out of which

most will never be used.

Instead we now use an additional action, learn(), on flows that match on

packets that create the hairpin session.  The learn() action will then

generate the necessary flows to handle hairpin replies, but only for

the local datapaths which actually generate hairpin traffic.

For example, simulating how ovn-k8s uses load balancer for services,

in a "switch per node" scenario, the script below would generate

10K (n_nodes * n_vips * n_backends) openflows on every node in table=69

(hairpin reply).  With this patch the maximum number of openflows that

can be created for hairpin replies is 200 (n_vips * n_backends).

In general, for deployments that leverage switch-per-node topologies,

the number of openflows is reduced by a factor of N, where N is the

number of nodes.

  $ cat lbs.sh

  NODES=50

  VIPS=20

  BACKENDS=10

  ovn-nbctl lr-add rtr

  for ((i = 1; i <= $NODES; i++)); do

      ovn-nbctl \

          -- ls-add ls$i \

          -- lsp-add ls$i vm$i \

          -- lsp-add ls$i ls$i-rtr \

          -- lsp-set-type ls$i-rtr router \

          -- lsp-set-options ls$i-rtr router-port=rtr-ls$i \

          -- lrp-add rtr rtr-ls$i 00:00:00:00:01:00 42.42.42.$i/24

  done

  for ((i = 1; i <= $VIPS; i++)); do

      lb=lb$i

      vip=10.10.10.$i:1

      bip=20.20.20.1:2

      for ((j = 2; j <= $BACKENDS; j++)); do

          bip="$bip,20.20.20.$j:2"

      done

      ovn-nbctl lb-add $lb $vip $backends

  done

  for ((i = 1; i <= $NODES; i++)); do

      for ((j = 1; j <= $VIPS; j++)); do

          ovn-nbctl ls-lb-add ls$i lb$j

      done

  done

  ovs-vsctl add-port br-int vm1 \

      -- set interface vm1 type=internal \

      -- set interface vm1 external-ids:iface-id=vm1

Suggested-by: Ilya Maximets <i.maximets@ovn.org>

Signed-off-by: Dumitru Ceara <dceara@redhat.com>

Signed-off-by: Numan Siddique <numans@ovn.org>

(cherry picked from upstream commit 022ea339c8e22824ba6f6f1257da0d1b6c66d401)

Change-Id: Ic0c20047538d6881e8cb79e00c96da8afde67a72

---

 controller/lflow.c      | 204 ++++++++++++++++---

 tests/ofproto-macros.at |   5 +-

 tests/ovn.at            | 516 +++++++++++++++++++++++++-----------------------

 3 files changed, 455 insertions(+), 270 deletions(-)

diff --git a/controller/lflow.c b/controller/lflow.c

index 946c1e0..2b7d356 100644

--- a/controller/lflow.c

+++ b/controller/lflow.c

@@ -1171,6 +1171,178 @@ add_neighbor_flows(struct ovsdb_idl_index *sbrec_port_binding_by_name,

     }

 }

+/* Builds the "learn()" action to be triggered by packets initiating a

+ * hairpin session.

+ *

+ * This will generate flows in table OFTABLE_CHK_LB_HAIRPIN_REPLY of the form:

+ * - match:

+ *     metadata=<orig-pkt-metadata>,ip/ipv6,ip.src=<backend>,ip.dst=<vip>

+ *     nw_proto='lb_proto',tp_src_port=<backend-port>

+ * - action:

+ *     set MLF_LOOKUP_LB_HAIRPIN_BIT=1

+ */

+static void

+add_lb_vip_hairpin_reply_action(struct in6_addr *vip6, ovs_be32 vip,

+                                uint8_t lb_proto, bool has_l4_port,

+                                uint64_t cookie, struct ofpbuf *ofpacts)

+{

+    struct match match = MATCH_CATCHALL_INITIALIZER;

+    struct ofpact_learn *ol = ofpact_put_LEARN(ofpacts);

+    struct ofpact_learn_spec *ol_spec;

+    unsigned int imm_bytes;

+    uint8_t *src_imm;

+

+    /* Once learned, hairpin reply flows are permanent until the VIP/backend

+     * is removed.

+     */

+    ol->flags = NX_LEARN_F_DELETE_LEARNED;

+    ol->idle_timeout = OFP_FLOW_PERMANENT;

+    ol->hard_timeout = OFP_FLOW_PERMANENT;

+    ol->priority = OFP_DEFAULT_PRIORITY;

+    ol->table_id = OFTABLE_CHK_LB_HAIRPIN_REPLY;

+    ol->cookie = htonll(cookie);

+

+    /* Match on metadata of the packet that created the hairpin session. */

+    ol_spec = ofpbuf_put_zeros(ofpacts, sizeof *ol_spec);

+

+    ol_spec->dst.field = mf_from_id(MFF_METADATA);

+    ol_spec->dst.ofs = 0;

+    ol_spec->dst.n_bits = ol_spec->dst.field->n_bits;

+    ol_spec->n_bits = ol_spec->dst.n_bits;

+    ol_spec->dst_type = NX_LEARN_DST_MATCH;

+    ol_spec->src_type = NX_LEARN_SRC_FIELD;

+    ol_spec->src.field = mf_from_id(MFF_METADATA);

+

+    /* Match on the same ETH type as the packet that created the hairpin

+     * session.

+     */

+    ol_spec = ofpbuf_put_zeros(ofpacts, sizeof *ol_spec);

+    ol_spec->dst.field = mf_from_id(MFF_ETH_TYPE);

+    ol_spec->dst.ofs = 0;

+    ol_spec->dst.n_bits = ol_spec->dst.field->n_bits;

+    ol_spec->n_bits = ol_spec->dst.n_bits;

+    ol_spec->dst_type = NX_LEARN_DST_MATCH;

+    ol_spec->src_type = NX_LEARN_SRC_IMMEDIATE;

+    union mf_value imm_eth_type = {

+        .be16 = !vip6 ? htons(ETH_TYPE_IP) : htons(ETH_TYPE_IPV6)

+    };

+    mf_write_subfield_value(&ol_spec->dst, &imm_eth_type, &match);

+

+    /* Push value last, as this may reallocate 'ol_spec'. */

+    imm_bytes = DIV_ROUND_UP(ol_spec->dst.n_bits, 8);

+    src_imm = ofpbuf_put_zeros(ofpacts, OFPACT_ALIGN(imm_bytes));

+    memcpy(src_imm, &imm_eth_type, imm_bytes);

+

+    /* Hairpin replies have ip.src == <backend-ip>. */

+    ol_spec = ofpbuf_put_zeros(ofpacts, sizeof *ol_spec);

+    if (!vip6) {

+        ol_spec->dst.field = mf_from_id(MFF_IPV4_SRC);

+        ol_spec->src.field = mf_from_id(MFF_IPV4_SRC);

+    } else {

+        ol_spec->dst.field = mf_from_id(MFF_IPV6_SRC);

+        ol_spec->src.field = mf_from_id(MFF_IPV6_SRC);

+    }

+    ol_spec->dst.ofs = 0;

+    ol_spec->dst.n_bits = ol_spec->dst.field->n_bits;

+    ol_spec->n_bits = ol_spec->dst.n_bits;

+    ol_spec->dst_type = NX_LEARN_DST_MATCH;

+    ol_spec->src_type = NX_LEARN_SRC_FIELD;

+

+    /* Hairpin replies have ip.dst == <vip>. */

+    union mf_value imm_ip;

+    ol_spec = ofpbuf_put_zeros(ofpacts, sizeof *ol_spec);

+    if (!vip6) {

+        ol_spec->dst.field = mf_from_id(MFF_IPV4_DST);

+        imm_ip = (union mf_value) {

+            .be32 = vip

+        };

+    } else {

+        ol_spec->dst.field = mf_from_id(MFF_IPV6_DST);

+        imm_ip = (union mf_value) {

+            .ipv6 = *vip6

+        };

+    }

+    ol_spec->dst.ofs = 0;

+    ol_spec->dst.n_bits = ol_spec->dst.field->n_bits;

+    ol_spec->n_bits = ol_spec->dst.n_bits;

+    ol_spec->dst_type = NX_LEARN_DST_MATCH;

+    ol_spec->src_type = NX_LEARN_SRC_IMMEDIATE;

+    mf_write_subfield_value(&ol_spec->dst, &imm_ip, &match);

+

+    /* Push value last, as this may reallocate 'ol_spec' */

+    imm_bytes = DIV_ROUND_UP(ol_spec->dst.n_bits, 8);

+    src_imm = ofpbuf_put_zeros(ofpacts, OFPACT_ALIGN(imm_bytes));

+    memcpy(src_imm, &imm_ip, imm_bytes);

+

+    /* Hairpin replies have the same nw_proto as packets that created the

+     * session.

+     */

+    union mf_value imm_proto = {

+        .u8 = lb_proto,

+    };

+    ol_spec = ofpbuf_put_zeros(ofpacts, sizeof *ol_spec);

+    ol_spec->dst.field = mf_from_id(MFF_IP_PROTO);

+    ol_spec->src.field = mf_from_id(MFF_IP_PROTO);

+    ol_spec->dst.ofs = 0;

+    ol_spec->dst.n_bits = ol_spec->dst.field->n_bits;

+    ol_spec->n_bits = ol_spec->dst.n_bits;

+    ol_spec->dst_type = NX_LEARN_DST_MATCH;

+    ol_spec->src_type = NX_LEARN_SRC_IMMEDIATE;

+    mf_write_subfield_value(&ol_spec->dst, &imm_proto, &match);

+

+    /* Push value last, as this may reallocate 'ol_spec' */

+    imm_bytes = DIV_ROUND_UP(ol_spec->dst.n_bits, 8);

+    src_imm = ofpbuf_put_zeros(ofpacts, OFPACT_ALIGN(imm_bytes));

+    memcpy(src_imm, &imm_proto, imm_bytes);

+

+    /* Hairpin replies have source port == <backend-port>. */

+    if (has_l4_port) {

+        ol_spec = ofpbuf_put_zeros(ofpacts, sizeof *ol_spec);

+        switch (lb_proto) {

+        case IPPROTO_TCP:

+            ol_spec->dst.field = mf_from_id(MFF_TCP_SRC);

+            ol_spec->src.field = mf_from_id(MFF_TCP_DST);

+            break;

+        case IPPROTO_UDP:

+            ol_spec->dst.field = mf_from_id(MFF_UDP_SRC);

+            ol_spec->src.field = mf_from_id(MFF_UDP_DST);

+            break;

+        case IPPROTO_SCTP:

+            ol_spec->dst.field = mf_from_id(MFF_SCTP_SRC);

+            ol_spec->src.field = mf_from_id(MFF_SCTP_DST);

+            break;

+        default:

+            OVS_NOT_REACHED();

+            break;

+        }

+        ol_spec->dst.ofs = 0;

+        ol_spec->dst.n_bits = ol_spec->dst.field->n_bits;

+        ol_spec->n_bits = ol_spec->dst.n_bits;

+        ol_spec->dst_type = NX_LEARN_DST_MATCH;

+        ol_spec->src_type = NX_LEARN_SRC_FIELD;

+    }

+

+    /* Set MLF_LOOKUP_LB_HAIRPIN_BIT for hairpin replies. */

+    ol_spec = ofpbuf_put_zeros(ofpacts, sizeof *ol_spec);

+    ol_spec->dst.field = mf_from_id(MFF_LOG_FLAGS);

+    ol_spec->dst.ofs = MLF_LOOKUP_LB_HAIRPIN_BIT;

+    ol_spec->dst.n_bits = 1;

+    ol_spec->n_bits = ol_spec->dst.n_bits;

+    ol_spec->dst_type = NX_LEARN_DST_LOAD;

+    ol_spec->src_type = NX_LEARN_SRC_IMMEDIATE;

+    union mf_value imm_reg_value = {

+        .u8 = 1

+    };

+    mf_write_subfield_value(&ol_spec->dst, &imm_reg_value, &match);

+

+    /* Push value last, as this may reallocate 'ol_spec' */

+    imm_bytes = DIV_ROUND_UP(ol_spec->dst.n_bits, 8);

+    src_imm = ofpbuf_put_zeros(ofpacts, OFPACT_ALIGN(imm_bytes));

+    memcpy(src_imm, &imm_reg_value, imm_bytes);

+

+    ofpact_finish_LEARN(ofpacts, &ol);

+}

+

 static void

 add_lb_vip_hairpin_flows(struct ovn_controller_lb *lb,

                          struct ovn_lb_vip *lb_vip,

@@ -1180,14 +1352,12 @@ add_lb_vip_hairpin_flows(struct ovn_controller_lb *lb,

 {

     uint64_t stub[1024 / 8];

     struct ofpbuf ofpacts = OFPBUF_STUB_INITIALIZER(stub);

+    struct match hairpin_match = MATCH_CATCHALL_INITIALIZER;

     uint8_t value = 1;

     put_load(&value, sizeof value, MFF_LOG_FLAGS,

              MLF_LOOKUP_LB_HAIRPIN_BIT, 1, &ofpacts);

-    struct match hairpin_match = MATCH_CATCHALL_INITIALIZER;

-    struct match hairpin_reply_match = MATCH_CATCHALL_INITIALIZER;

-

     if (IN6_IS_ADDR_V4MAPPED(&lb_vip->vip)) {

         ovs_be32 bip4 = in6_addr_get_mapped_ipv4(&lb_backend->ip);

         ovs_be32 vip4 = lb->hairpin_snat_ips.n_ipv4_addrs

@@ -1198,9 +1368,10 @@ add_lb_vip_hairpin_flows(struct ovn_controller_lb *lb,

         match_set_nw_src(&hairpin_match, bip4);

         match_set_nw_dst(&hairpin_match, bip4);

-        match_set_dl_type(&hairpin_reply_match, htons(ETH_TYPE_IP));

-        match_set_nw_src(&hairpin_reply_match, bip4);

-        match_set_nw_dst(&hairpin_reply_match, vip4);

+        add_lb_vip_hairpin_reply_action(NULL, vip4, lb_proto,

+                                        lb_backend->port,

+                                        lb->slb->header_.uuid.parts[0],

+                                        &ofpacts);

     } else {

         struct in6_addr *bip6 = &lb_backend->ip;

         struct in6_addr *vip6 = lb->hairpin_snat_ips.n_ipv6_addrs

@@ -1210,17 +1381,15 @@ add_lb_vip_hairpin_flows(struct ovn_controller_lb *lb,

         match_set_ipv6_src(&hairpin_match, bip6);

         match_set_ipv6_dst(&hairpin_match, bip6);

-        match_set_dl_type(&hairpin_reply_match, htons(ETH_TYPE_IPV6));

-        match_set_ipv6_src(&hairpin_reply_match, bip6);

-        match_set_ipv6_dst(&hairpin_reply_match, vip6);

+        add_lb_vip_hairpin_reply_action(vip6, 0, lb_proto,

+                                        lb_backend->port,

+                                        lb->slb->header_.uuid.parts[0],

+                                        &ofpacts);

     }

     if (lb_backend->port) {

         match_set_nw_proto(&hairpin_match, lb_proto);

         match_set_tp_dst(&hairpin_match, htons(lb_backend->port));

-

-        match_set_nw_proto(&hairpin_reply_match, lb_proto);

-        match_set_tp_src(&hairpin_reply_match, htons(lb_backend->port));

     }

     /* In the original direction, only match on traffic that was already

@@ -1241,17 +1410,6 @@ add_lb_vip_hairpin_flows(struct ovn_controller_lb *lb,

     ofctrl_add_flow(flow_table, OFTABLE_CHK_LB_HAIRPIN, 100,

                     lb->slb->header_.uuid.parts[0], &hairpin_match,

                     &ofpacts, &lb->slb->header_.uuid);

-

-    for (size_t i = 0; i < lb->slb->n_datapaths; i++) {

-        match_set_metadata(&hairpin_reply_match,

-                           htonll(lb->slb->datapaths[i]->tunnel_key));

-

-        ofctrl_add_flow(flow_table, OFTABLE_CHK_LB_HAIRPIN_REPLY, 100,

-                        lb->slb->header_.uuid.parts[0],

-                        &hairpin_reply_match,

-                        &ofpacts, &lb->slb->header_.uuid);

-    }

-

     ofpbuf_uninit(&ofpacts);

 }

diff --git a/tests/ofproto-macros.at b/tests/ofproto-macros.at

index dd5d384..ff65d60 100644

--- a/tests/ofproto-macros.at

+++ b/tests/ofproto-macros.at

@@ -12,7 +12,10 @@ strip_n_bytes () {

 # Strips 'cookie=...' from ovs-ofctl output.

 strip_cookie () {

-    sed 's/ cookie=0x[0-9a-fA-F]*,//'

+    sed '

+s/ cookie=0x[0-9a-fA-F]*,//

+s/cookie=0x[0-9a-fA-F]*,//

+'

 }

 # Strips out uninteresting parts of ovs-ofctl output, as well as parts

diff --git a/tests/ovn.at b/tests/ovn.at

index 14072ec..6c9bda0 100644

--- a/tests/ovn.at

+++ b/tests/ovn.at

@@ -20822,6 +20822,8 @@ ovn-sbctl dump-flows > sbflows

 AT_CAPTURE_FILE([sbflows])

 > expected

+AS_BOX([IPv4 TCP Hairpin])

+

 # Inject IPv4 TCP packet from lsp.

 tcp_payload=$(build_tcp_syn 84d0 1f90 05a7)

 hp_tcp_payload=$(build_tcp_syn 84d0 0fc9 156e)

@@ -20835,9 +20837,15 @@ send_ipv4_pkt hv1 hv1-vif1 000000000001 000000000100 \

 # Check that traffic is hairpinned.

 OVN_CHECK_PACKETS([hv1/vif1-tx.pcap], [expected])

+# Check learned hairpin reply flows.

+OVS_WAIT_FOR_OUTPUT([as hv1 ovs-ofctl dump-flows br-int table=69 | ofctl_strip_all | grep -v NXST], [0], [dnl

+ table=69, tcp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.88,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]

+])

+

 # Change LB Hairpin SNAT IP.

 # Also flush conntrack to avoid reusing an existing entry.

 as hv1 ovs-appctl dpctl/flush-conntrack

+

 ovn-nbctl --wait=hv set load_balancer lb-ipv4-tcp options:hairpin_snat_ip="88.88.88.87"

 # Inject IPv4 TCP packet from lsp.

 hp_tcp_payload=$(build_tcp_syn 84d0 0fc9 156f)

@@ -20851,6 +20859,13 @@ send_ipv4_pkt hv1 hv1-vif1 000000000001 000000000100 \

 # Check that traffic is hairpinned.

 OVN_CHECK_PACKETS([hv1/vif1-tx.pcap], [expected])

+# Check learned hairpin reply flows.

+OVS_WAIT_FOR_OUTPUT([as hv1 ovs-ofctl dump-flows br-int table=69 | ofctl_strip_all | grep -v NXST], [0], [dnl

+ table=69, tcp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.87,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]

+])

+

+AS_BOX([IPv4 UDP Hairpin])

+

 # Inject IPv4 UDP packet from lsp.

 udp_payload=$(build_udp 84d0 0fc8 6666)

 hp_udp_payload=$(build_udp 84d0 07e5 6e49)

@@ -20864,6 +20879,12 @@ send_ipv4_pkt hv1 hv1-vif1 000000000001 000000000100 \

 # Check that traffic is hairpinned.

 OVN_CHECK_PACKETS([hv1/vif1-tx.pcap], [expected])

+# Check learned hairpin reply flows.

+OVS_WAIT_FOR_OUTPUT([as hv1 ovs-ofctl dump-flows br-int table=69 | ofctl_strip_all | grep -v NXST], [0], [dnl

+ table=69, tcp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.87,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]

+ table=69, udp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.88,tp_src=2021 actions=load:0x1->NXM_NX_REG10[[7]]

+])

+

 # Change LB Hairpin SNAT IP.

 # Also flush conntrack to avoid reusing an existing entry.

 as hv1 ovs-appctl dpctl/flush-conntrack

@@ -20880,6 +20901,14 @@ send_ipv4_pkt hv1 hv1-vif1 000000000001 000000000100 \

 # Check that traffic is hairpinned.

 OVN_CHECK_PACKETS([hv1/vif1-tx.pcap], [expected])

+# Check learned hairpin reply flows.

+OVS_WAIT_FOR_OUTPUT([as hv1 ovs-ofctl dump-flows br-int table=69 | ofctl_strip_all | grep -v NXST], [0], [dnl

+ table=69, tcp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.87,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]

+ table=69, udp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.87,tp_src=2021 actions=load:0x1->NXM_NX_REG10[[7]]

+])

+

+AS_BOX([IPv6 TCP Hairpin])

+

 # Inject IPv6 TCP packet from lsp.

 tcp_payload=$(build_tcp_syn 84d0 1f90 3ff9)

 hp_tcp_payload=$(build_tcp_syn 84d0 0fc9 4fc0)

@@ -20893,6 +20922,13 @@ send_ipv6_pkt hv1 hv1-vif1 000000000001 000000000100 \

 # Check that traffic is hairpinned.

 OVN_CHECK_PACKETS([hv1/vif1-tx.pcap], [expected])

+# Check learned hairpin reply flows.

+OVS_WAIT_FOR_OUTPUT([as hv1 ovs-ofctl dump-flows br-int table=69 | ofctl_strip_all | grep -v NXST], [0], [dnl

+ table=69, tcp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.87,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]

+ table=69, udp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.87,tp_src=2021 actions=load:0x1->NXM_NX_REG10[[7]]

+ table=69, tcp6,metadata=0x1,ipv6_src=4200::1,ipv6_dst=8800::88,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]

+])

+

 # Change LB Hairpin SNAT IP.

 # Also flush conntrack to avoid reusing an existing entry.

 as hv1 ovs-appctl dpctl/flush-conntrack

@@ -20910,6 +20946,15 @@ send_ipv6_pkt hv1 hv1-vif1 000000000001 000000000100 \

 # Check that traffic is hairpinned.

 OVN_CHECK_PACKETS([hv1/vif1-tx.pcap], [expected])

+# Check learned hairpin reply flows.

+OVS_WAIT_FOR_OUTPUT([as hv1 ovs-ofctl dump-flows br-int table=69 | ofctl_strip_all | grep -v NXST], [0], [dnl

+ table=69, tcp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.87,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]

+ table=69, udp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.87,tp_src=2021 actions=load:0x1->NXM_NX_REG10[[7]]

+ table=69, tcp6,metadata=0x1,ipv6_src=4200::1,ipv6_dst=8800::87,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]

+])

+

+AS_BOX([IPv6 UDP Hairpin])

+

 # Inject IPv6 UDP packet from lsp.

 udp_payload=$(build_udp 84d0 0fc8 a0b8)

 hp_udp_payload=$(build_udp 84d0 07e5 a89b)

@@ -20920,9 +20965,17 @@ send_ipv6_pkt hv1 hv1-vif1 000000000001 000000000100 \

     88000000000000000000000000000088 ${hp_udp_payload} \

     expected

-Check that traffic is hairpinned.

+# Check that traffic is hairpinned.

 OVN_CHECK_PACKETS([hv1/vif1-tx.pcap], [expected])

+# Check learned hairpin reply flows.

+OVS_WAIT_FOR_OUTPUT([as hv1 ovs-ofctl dump-flows br-int table=69 | ofctl_strip_all | grep -v NXST], [0], [dnl

+ table=69, tcp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.87,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]

+ table=69, udp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.87,tp_src=2021 actions=load:0x1->NXM_NX_REG10[[7]]

+ table=69, tcp6,metadata=0x1,ipv6_src=4200::1,ipv6_dst=8800::87,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]

+ table=69, udp6,metadata=0x1,ipv6_src=4200::1,ipv6_dst=8800::88,tp_src=2021 actions=load:0x1->NXM_NX_REG10[[7]]

+])

+

 # Change LB Hairpin SNAT IP.

 # Also flush conntrack to avoid reusing an existing entry.

 as hv1 ovs-appctl dpctl/flush-conntrack

@@ -20937,6 +20990,41 @@ send_ipv6_pkt hv1 hv1-vif1 000000000001 000000000100 \

     88000000000000000000000000000087 ${hp_udp_payload} \

     expected

+# Check learned hairpin reply flows.

+OVS_WAIT_FOR_OUTPUT([as hv1 ovs-ofctl dump-flows br-int table=69 | ofctl_strip_all | grep -v NXST], [0], [dnl

+ table=69, tcp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.87,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]

+ table=69, udp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.87,tp_src=2021 actions=load:0x1->NXM_NX_REG10[[7]]

+ table=69, tcp6,metadata=0x1,ipv6_src=4200::1,ipv6_dst=8800::87,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]

+ table=69, udp6,metadata=0x1,ipv6_src=4200::1,ipv6_dst=8800::87,tp_src=2021 actions=load:0x1->NXM_NX_REG10[[7]]

+])

+

+AS_BOX([Delete VIP])

+check ovn-nbctl --wait=hv set Load_Balancer lb-ipv4-tcp vips='"88.88.88.88:8080"=""'

+OVS_WAIT_FOR_OUTPUT([as hv1 ovs-ofctl dump-flows br-int table=69 | ofctl_strip_all | grep -v NXST], [0], [dnl

+ table=69, udp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.87,tp_src=2021 actions=load:0x1->NXM_NX_REG10[[7]]

+ table=69, tcp6,metadata=0x1,ipv6_src=4200::1,ipv6_dst=8800::87,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]

+ table=69, udp6,metadata=0x1,ipv6_src=4200::1,ipv6_dst=8800::87,tp_src=2021 actions=load:0x1->NXM_NX_REG10[[7]]

+])

+

+AS_BOX([Delete LB])

+check ovn-nbctl --wait=hv \

+    -- lb-del lb-ipv4-tcp \

+    -- lb-del lb-ipv4-udp

+

+OVS_WAIT_FOR_OUTPUT([as hv1 ovs-ofctl dump-flows br-int table=69 | ofctl_strip_all | grep -v NXST], [0], [dnl

+ table=69, tcp6,metadata=0x1,ipv6_src=4200::1,ipv6_dst=8800::87,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]

+ table=69, udp6,metadata=0x1,ipv6_src=4200::1,ipv6_dst=8800::87,tp_src=2021 actions=load:0x1->NXM_NX_REG10[[7]]

+])

+

+check ovn-nbctl --wait=hv lb-del lb-ipv6-tcp

+OVS_WAIT_FOR_OUTPUT([as hv1 ovs-ofctl dump-flows br-int table=69 | ofctl_strip_all | grep -v NXST], [0], [dnl

+ table=69, udp6,metadata=0x1,ipv6_src=4200::1,ipv6_dst=8800::87,tp_src=2021 actions=load:0x1->NXM_NX_REG10[[7]]

+])

+

+check ovn-nbctl --wait=hv lb-del lb-ipv6-udp

+OVS_WAIT_FOR_OUTPUT([as hv1 ovs-ofctl dump-flows br-int table=69 | ofctl_strip_all | grep -v NXST], [1], [dnl

+])

+

 OVN_CLEANUP([hv1])

 AT_CLEANUP

@@ -23188,93 +23276,79 @@ check ovn-nbctl lb-add lb-ipv4-udp 88.88.88.88:4040 42.42.42.1:2021 udp

 check ovn-nbctl lb-add lb-ipv6-tcp [[8800::0088]]:8080 [[4200::1]]:4041 tcp

 check ovn-nbctl --wait=hv lb-add lb-ipv6-udp [[8800::0088]]:4040 [[4200::1]]:2021 udp

-AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=68], [0], [dnl

-NXST_FLOW reply (xid=0x8):

+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=68 | grep -v NXST], [1], [dnl

 ])

-AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=69], [0], [dnl

-NXST_FLOW reply (xid=0x8):

+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=69 | grep -v NXST], [1], [dnl

 ])

-AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=70], [0], [dnl

-NXST_FLOW reply (xid=0x8):

+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=70 | grep -v NXST], [1], [dnl

 ])

-AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=68], [0], [dnl

-NXST_FLOW reply (xid=0x8):

+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=68 | grep -v NXST], [1], [dnl

 ])

-AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=69], [0], [dnl

-NXST_FLOW reply (xid=0x8):

+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=69 | grep -v NXST], [1], [dnl

 ])

-AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=70], [0], [dnl

-NXST_FLOW reply (xid=0x8):

+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=70 | grep -v NXST], [1], [dnl

 ])

 check ovn-nbctl --wait=hv ls-lb-add sw0 lb-ipv4-tcp

 OVS_WAIT_UNTIL(

-    [test $(as hv1 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | wc -l) -eq 1]

+    [test $(as hv1 ovs-ofctl dump-flows br-int table=68 | grep -c -v NXST) -eq 1]

 )

-AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | cut -d ' ' -f8-], [0], [dnl

-priority=100,ct_label=0x2/0x2,tcp,nw_src=42.42.42.1,nw_dst=42.42.42.1,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]

+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=68 | ofctl_strip_all | grep -v NXST | sort], [0], [dnl

+ table=68, priority=100,ct_label=0x2/0x2,tcp,nw_src=42.42.42.1,nw_dst=42.42.42.1,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]],learn(table=69,delete_learned,OXM_OF_METADATA[[]],eth_type=0x800,NXM_OF_IP_SRC[[]],ip_dst=88.88.88.88,nw_proto=6,NXM_OF_TCP_SRC[[]]=NXM_OF_TCP_DST[[]],load:0x1->NXM_NX_REG10[[7]])

 ])

-AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=69 | grep -v NXST | cut -d ' ' -f8-], [0], [dnl

-priority=100,tcp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.88,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]

+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=69], [0], [dnl

+NXST_FLOW reply (xid=0x8):

 ])

-AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=70 | grep -v NXST | cut -d ' ' -f8-], [0], [dnl

-priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.88,ct_nw_proto=6,ct_tp_dst=8080,tcp,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.88))

+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=70 | ofctl_strip_all | grep -v NXST | sort], [0], [dnl

+ table=70, priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.88,ct_nw_proto=6,ct_tp_dst=8080,tcp,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.88))

 ])

-AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=68], [0], [dnl

-NXST_FLOW reply (xid=0x8):

+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=68 | grep -v NXST], [1], [dnl

 ])

-AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=69], [0], [dnl

-NXST_FLOW reply (xid=0x8):

+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=69 | grep -v NXST], [1], [dnl

 ])

-AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=70], [0], [dnl

-NXST_FLOW reply (xid=0x8):

+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=70 | grep -v NXST], [1], [dnl

 ])

 check ovn-nbctl lb-add lb-ipv4-tcp 88.88.88.90:8080 42.42.42.42:4041,52.52.52.52:4042 tcp

 OVS_WAIT_UNTIL(

-    [test $(as hv1 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | wc -l) -eq 3]

+    [test $(as hv1 ovs-ofctl dump-flows br-int table=68 | grep -c -v NXST) -eq 3]

 )

-AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl

-priority=100,ct_label=0x2/0x2,tcp,nw_src=42.42.42.1,nw_dst=42.42.42.1,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]

-priority=100,ct_label=0x2/0x2,tcp,nw_src=42.42.42.42,nw_dst=42.42.42.42,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]

-priority=100,ct_label=0x2/0x2,tcp,nw_src=52.52.52.52,nw_dst=52.52.52.52,tp_dst=4042 actions=load:0x1->NXM_NX_REG10[[7]]

+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=68 | ofctl_strip_all | grep -v NXST | sort], [0], [dnl

+ table=68, priority=100,ct_label=0x2/0x2,tcp,nw_src=42.42.42.1,nw_dst=42.42.42.1,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]],learn(table=69,delete_learned,OXM_OF_METADATA[[]],eth_type=0x800,NXM_OF_IP_SRC[[]],ip_dst=88.88.88.88,nw_proto=6,NXM_OF_TCP_SRC[[]]=NXM_OF_TCP_DST[[]],load:0x1->NXM_NX_REG10[[7]])

+ table=68, priority=100,ct_label=0x2/0x2,tcp,nw_src=42.42.42.42,nw_dst=42.42.42.42,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]],learn(table=69,delete_learned,OXM_OF_METADATA[[]],eth_type=0x800,NXM_OF_IP_SRC[[]],ip_dst=88.88.88.90,nw_proto=6,NXM_OF_TCP_SRC[[]]=NXM_OF_TCP_DST[[]],load:0x1->NXM_NX_REG10[[7]])

+ table=68, priority=100,ct_label=0x2/0x2,tcp,nw_src=52.52.52.52,nw_dst=52.52.52.52,tp_dst=4042 actions=load:0x1->NXM_NX_REG10[[7]],learn(table=69,delete_learned,OXM_OF_METADATA[[]],eth_type=0x800,NXM_OF_IP_SRC[[]],ip_dst=88.88.88.90,nw_proto=6,NXM_OF_TCP_SRC[[]]=NXM_OF_TCP_DST[[]],load:0x1->NXM_NX_REG10[[7]])

 ])

-AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=69 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl

-priority=100,tcp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.88,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]

-priority=100,tcp,metadata=0x1,nw_src=42.42.42.42,nw_dst=88.88.88.90,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]

-priority=100,tcp,metadata=0x1,nw_src=52.52.52.52,nw_dst=88.88.88.90,tp_src=4042 actions=load:0x1->NXM_NX_REG10[[7]]

+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=69], [0], [dnl

+NXST_FLOW reply (xid=0x8):

 ])

-AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=70 | grep -v NXST | cut -d ' ' -f8-], [0], [dnl

-priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.88,ct_nw_proto=6,ct_tp_dst=8080,tcp,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.88))

-priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.90,ct_nw_proto=6,ct_tp_dst=8080,tcp,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.90))

+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=70 | ofctl_strip_all | grep -v NXST | sort], [0], [dnl

+ table=70, priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.88,ct_nw_proto=6,ct_tp_dst=8080,tcp,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.88))

+ table=70, priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.90,ct_nw_proto=6,ct_tp_dst=8080,tcp,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.90))

 ])

-AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=68], [0], [dnl

-NXST_FLOW reply (xid=0x8):

+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=68 | grep -v NXST], [1], [dnl

 ])

-AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=69], [0], [dnl

-NXST_FLOW reply (xid=0x8):

+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=69 | grep -v NXST], [1], [dnl

 ])

-AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=70], [0], [dnl

-NXST_FLOW reply (xid=0x8):

+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=70 | grep -v NXST], [1], [dnl

 ])

 check ovn-nbctl lsp-add sw0 sw0-p2

@@ -23282,192 +23356,159 @@ check ovn-nbctl lsp-add sw0 sw0-p2

 OVS_WAIT_UNTIL([test x$(ovn-nbctl lsp-get-up sw0-p2) = xup])

 OVS_WAIT_UNTIL(

-    [test $(as hv2 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | wc -l) -eq 3]

+    [test $(as hv2 ovs-ofctl dump-flows br-int table=68 | grep -c -v NXST) -eq 3]

 )

-AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl

-priority=100,ct_label=0x2/0x2,tcp,nw_src=42.42.42.1,nw_dst=42.42.42.1,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]

-priority=100,ct_label=0x2/0x2,tcp,nw_src=42.42.42.42,nw_dst=42.42.42.42,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]

-priority=100,ct_label=0x2/0x2,tcp,nw_src=52.52.52.52,nw_dst=52.52.52.52,tp_dst=4042 actions=load:0x1->NXM_NX_REG10[[7]]

+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=68 | ofctl_strip_all | grep -v NXST | sort], [0], [dnl

+ table=68, priority=100,ct_label=0x2/0x2,tcp,nw_src=42.42.42.1,nw_dst=42.42.42.1,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]],learn(table=69,delete_learned,OXM_OF_METADATA[[]],eth_type=0x800,NXM_OF_IP_SRC[[]],ip_dst=88.88.88.88,nw_proto=6,NXM_OF_TCP_SRC[[]]=NXM_OF_TCP_DST[[]],load:0x1->NXM_NX_REG10[[7]])

+ table=68, priority=100,ct_label=0x2/0x2,tcp,nw_src=42.42.42.42,nw_dst=42.42.42.42,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]],learn(table=69,delete_learned,OXM_OF_METADATA[[]],eth_type=0x800,NXM_OF_IP_SRC[[]],ip_dst=88.88.88.90,nw_proto=6,NXM_OF_TCP_SRC[[]]=NXM_OF_TCP_DST[[]],load:0x1->NXM_NX_REG10[[7]])

+ table=68, priority=100,ct_label=0x2/0x2,tcp,nw_src=52.52.52.52,nw_dst=52.52.52.52,tp_dst=4042 actions=load:0x1->NXM_NX_REG10[[7]],learn(table=69,delete_learned,OXM_OF_METADATA[[]],eth_type=0x800,NXM_OF_IP_SRC[[]],ip_dst=88.88.88.90,nw_proto=6,NXM_OF_TCP_SRC[[]]=NXM_OF_TCP_DST[[]],load:0x1->NXM_NX_REG10[[7]])

 ])

-AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=69 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl

-priority=100,tcp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.88,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]

-priority=100,tcp,metadata=0x1,nw_src=42.42.42.42,nw_dst=88.88.88.90,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]

-priority=100,tcp,metadata=0x1,nw_src=52.52.52.52,nw_dst=88.88.88.90,tp_src=4042 actions=load:0x1->NXM_NX_REG10[[7]]

+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=69 | grep -v NXST], [1], [dnl

 ])

-AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=70 | grep -v NXST | cut -d ' ' -f8-], [0], [dnl

-priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.88,ct_nw_proto=6,ct_tp_dst=8080,tcp,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.88))

-priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.90,ct_nw_proto=6,ct_tp_dst=8080,tcp,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.90))

+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=70 | ofctl_strip_all | grep -v NXST | sort], [0], [dnl

+ table=70, priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.88,ct_nw_proto=6,ct_tp_dst=8080,tcp,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.88))

+ table=70, priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.90,ct_nw_proto=6,ct_tp_dst=8080,tcp,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.90))

 ])

 check ovn-nbctl --wait=hv ls-lb-add sw0 lb-ipv4-udp

 OVS_WAIT_UNTIL(

-    [test $(as hv1 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | wc -l) -eq 4]

+    [test $(as hv1 ovs-ofctl dump-flows br-int table=68 | grep -c -v NXST) -eq 4]

 )

 OVS_WAIT_UNTIL(

-    [test $(as hv2 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | wc -l) -eq 4]

+    [test $(as hv2 ovs-ofctl dump-flows br-int table=68 | grep -c -v NXST) -eq 4]

 )

-AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl

-priority=100,ct_label=0x2/0x2,tcp,nw_src=42.42.42.1,nw_dst=42.42.42.1,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]

-priority=100,ct_label=0x2/0x2,tcp,nw_src=42.42.42.42,nw_dst=42.42.42.42,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]

-priority=100,ct_label=0x2/0x2,tcp,nw_src=52.52.52.52,nw_dst=52.52.52.52,tp_dst=4042 actions=load:0x1->NXM_NX_REG10[[7]]

-priority=100,ct_label=0x2/0x2,udp,nw_src=42.42.42.1,nw_dst=42.42.42.1,tp_dst=2021 actions=load:0x1->NXM_NX_REG10[[7]]

+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=68 | ofctl_strip_all | grep -v NXST | sort], [0], [dnl

+ table=68, priority=100,ct_label=0x2/0x2,tcp,nw_src=42.42.42.1,nw_dst=42.42.42.1,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]],learn(table=69,delete_learned,OXM_OF_METADATA[[]],eth_type=0x800,NXM_OF_IP_SRC[[]],ip_dst=88.88.88.88,nw_proto=6,NXM_OF_TCP_SRC[[]]=NXM_OF_TCP_DST[[]],load:0x1->NXM_NX_REG10[[7]])

+ table=68, priority=100,ct_label=0x2/0x2,tcp,nw_src=42.42.42.42,nw_dst=42.42.42.42,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]],learn(table=69,delete_learned,OXM_OF_METADATA[[]],eth_type=0x800,NXM_OF_IP_SRC[[]],ip_dst=88.88.88.90,nw_proto=6,NXM_OF_TCP_SRC[[]]=NXM_OF_TCP_DST[[]],load:0x1->NXM_NX_REG10[[7]])

+ table=68, priority=100,ct_label=0x2/0x2,tcp,nw_src=52.52.52.52,nw_dst=52.52.52.52,tp_dst=4042 actions=load:0x1->NXM_NX_REG10[[7]],learn(table=69,delete_learned,OXM_OF_METADATA[[]],eth_type=0x800,NXM_OF_IP_SRC[[]],ip_dst=88.88.88.90,nw_proto=6,NXM_OF_TCP_SRC[[]]=NXM_OF_TCP_DST[[]],load:0x1->NXM_NX_REG10[[7]])

+ table=68, priority=100,ct_label=0x2/0x2,udp,nw_src=42.42.42.1,nw_dst=42.42.42.1,tp_dst=2021 actions=load:0x1->NXM_NX_REG10[[7]],learn(table=69,delete_learned,OXM_OF_METADATA[[]],eth_type=0x800,NXM_OF_IP_SRC[[]],ip_dst=88.88.88.88,nw_proto=17,NXM_OF_UDP_SRC[[]]=NXM_OF_UDP_DST[[]],load:0x1->NXM_NX_REG10[[7]])

 ])

-AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=69 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl

-priority=100,tcp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.88,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]

-priority=100,tcp,metadata=0x1,nw_src=42.42.42.42,nw_dst=88.88.88.90,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]

-priority=100,tcp,metadata=0x1,nw_src=52.52.52.52,nw_dst=88.88.88.90,tp_src=4042 actions=load:0x1->NXM_NX_REG10[[7]]

-priority=100,udp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.88,tp_src=2021 actions=load:0x1->NXM_NX_REG10[[7]]

+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=69 | grep -v NXST], [1], [dnl

 ])

-AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=70 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl

-priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.88,ct_nw_proto=17,ct_tp_dst=4040,udp,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.88))

-priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.88,ct_nw_proto=6,ct_tp_dst=8080,tcp,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.88))

-priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.90,ct_nw_proto=6,ct_tp_dst=8080,tcp,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.90))

+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=70 | ofctl_strip_all | grep -v NXST | sort], [0], [dnl

+ table=70, priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.88,ct_nw_proto=17,ct_tp_dst=4040,udp,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.88))

+ table=70, priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.88,ct_nw_proto=6,ct_tp_dst=8080,tcp,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.88))

+ table=70, priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.90,ct_nw_proto=6,ct_tp_dst=8080,tcp,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.90))

 ])

-AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl

-priority=100,ct_label=0x2/0x2,tcp,nw_src=42.42.42.1,nw_dst=42.42.42.1,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]

-priority=100,ct_label=0x2/0x2,tcp,nw_src=42.42.42.42,nw_dst=42.42.42.42,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]

-priority=100,ct_label=0x2/0x2,tcp,nw_src=52.52.52.52,nw_dst=52.52.52.52,tp_dst=4042 actions=load:0x1->NXM_NX_REG10[[7]]

-priority=100,ct_label=0x2/0x2,udp,nw_src=42.42.42.1,nw_dst=42.42.42.1,tp_dst=2021 actions=load:0x1->NXM_NX_REG10[[7]]

+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=68 | ofctl_strip_all | grep -v NXST | sort], [0], [dnl

+ table=68, priority=100,ct_label=0x2/0x2,tcp,nw_src=42.42.42.1,nw_dst=42.42.42.1,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]],learn(table=69,delete_learned,OXM_OF_METADATA[[]],eth_type=0x800,NXM_OF_IP_SRC[[]],ip_dst=88.88.88.88,nw_proto=6,NXM_OF_TCP_SRC[[]]=NXM_OF_TCP_DST[[]],load:0x1->NXM_NX_REG10[[7]])

+ table=68, priority=100,ct_label=0x2/0x2,tcp,nw_src=42.42.42.42,nw_dst=42.42.42.42,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]],learn(table=69,delete_learned,OXM_OF_METADATA[[]],eth_type=0x800,NXM_OF_IP_SRC[[]],ip_dst=88.88.88.90,nw_proto=6,NXM_OF_TCP_SRC[[]]=NXM_OF_TCP_DST[[]],load:0x1->NXM_NX_REG10[[7]])

+ table=68, priority=100,ct_label=0x2/0x2,tcp,nw_src=52.52.52.52,nw_dst=52.52.52.52,tp_dst=4042 actions=load:0x1->NXM_NX_REG10[[7]],learn(table=69,delete_learned,OXM_OF_METADATA[[]],eth_type=0x800,NXM_OF_IP_SRC[[]],ip_dst=88.88.88.90,nw_proto=6,NXM_OF_TCP_SRC[[]]=NXM_OF_TCP_DST[[]],load:0x1->NXM_NX_REG10[[7]])

+ table=68, priority=100,ct_label=0x2/0x2,udp,nw_src=42.42.42.1,nw_dst=42.42.42.1,tp_dst=2021 actions=load:0x1->NXM_NX_REG10[[7]],learn(table=69,delete_learned,OXM_OF_METADATA[[]],eth_type=0x800,NXM_OF_IP_SRC[[]],ip_dst=88.88.88.88,nw_proto=17,NXM_OF_UDP_SRC[[]]=NXM_OF_UDP_DST[[]],load:0x1->NXM_NX_REG10[[7]])

 ])