From 4f81eab659b42aeb8b433783515b03b5772c68de Mon Sep 17 00:00:00 2001
From: Numan Siddique <numans@ovn.org>
Date: Thu, 12 Nov 2020 17:26:17 +0530
Subject: [PATCH 06/10] controller: Add load balancer hairpin OF flows.
Presently to handle the load balancer hairpin traffic (the traffic destined to the
load balancer VIP is dnatted to the backend which originated the traffic), ovn-northd
adds a lot of logical flows to check this scenario. This patch attempts to reduce the
these logical flows. Each ovn-controller will read the load balancers from
the newly added southbound Load_Balancer table and adds the load balancer hairpin OF
flows in the table 68, 69 and 70. If suppose a below load balancer is configured
10.0.0.10:80 = 10.0.0.4:8080, 10.0.0.5:8090, then the below flows are added
table=68, ip.src = 10.0.0.4,ip.dst=10.0.0.4,tcp.dst=8080 actions=load:1->NXM_NX_REG10[7]
table=68, ip.src = 10.0.0.5,ip.dst=10.0.0.5,tcp.dst=8090 actions=load:1->NXM_NX_REG10[7]
table=69, ip.src = 10.0.0.4,ip.dst=10.0.0.10,tcp.src=8080 actions=load:1->NXM_NX_REG10[7]
table=69, ip.src = 10.0.0.5,ip.dst=10.0.0.10,tcp.src=8090 actions=load:1->NXM_NX_REG10[7]
table=70, ct.trk && ct.dnat && ct.nw_dst == 10.0.0.10. actions=ct(commit, zone=reg12, nat(src=10.0.0.5))
Upcoming patch will add OVN actions which does the lookup in these tables to handle the
hairpin traffic.
Acked-by: Dumitru Ceara <dceara@redhat.com>
Acked-by: Mark Michelson <mmichels@redhat.com>
Signed-off-by: Numan Siddique <numans@ovn.org>
(cherry-picked from master commit 7da8145d9d4743b3d15f0d7f51b201e7ba87fe63)
Conflicts:
tests/ovn.at
---
controller/lflow.c | 230 +++++++++++++++++
controller/lflow.h | 6 +-
controller/ovn-controller.c | 27 +-
include/ovn/logical-fields.h | 3 +
tests/ovn.at | 469 +++++++++++++++++++++++++++++++++++
5 files changed, 733 insertions(+), 2 deletions(-)
diff --git a/controller/lflow.c b/controller/lflow.c
index 4d71dfddb..633fdfb7f 100644
--- a/controller/lflow.c
+++ b/controller/lflow.c
@@ -26,6 +26,7 @@
#include "ovn-controller.h"
#include "ovn/actions.h"
#include "ovn/expr.h"
+#include "lib/lb.h"
#include "lib/ovn-l7.h"
#include "lib/ovn-sb-idl.h"
#include "lib/extend-table.h"
@@ -1144,6 +1145,190 @@ add_neighbor_flows(struct ovsdb_idl_index *sbrec_port_binding_by_name,
}
}
+static void
+add_lb_vip_hairpin_flows(struct ovn_controller_lb *lb,
+ struct ovn_lb_vip *lb_vip,
+ struct ovn_lb_backend *lb_backend,
+ uint8_t lb_proto,
+ struct ovn_desired_flow_table *flow_table)
+{
+ uint64_t stub[1024 / 8];
+ struct ofpbuf ofpacts = OFPBUF_STUB_INITIALIZER(stub);
+
+ uint8_t value = 1;
+ put_load(&value, sizeof value, MFF_LOG_FLAGS,
+ MLF_LOOKUP_LB_HAIRPIN_BIT, 1, &ofpacts);
+
+ struct match hairpin_match = MATCH_CATCHALL_INITIALIZER;
+ struct match hairpin_reply_match = MATCH_CATCHALL_INITIALIZER;
+
+ if (IN6_IS_ADDR_V4MAPPED(&lb_vip->vip)) {
+ ovs_be32 ip4 = in6_addr_get_mapped_ipv4(&lb_backend->ip);
+
+ match_set_dl_type(&hairpin_match, htons(ETH_TYPE_IP));
+ match_set_nw_src(&hairpin_match, ip4);
+ match_set_nw_dst(&hairpin_match, ip4);
+
+ match_set_dl_type(&hairpin_reply_match,
+ htons(ETH_TYPE_IP));
+ match_set_nw_src(&hairpin_reply_match, ip4);
+ match_set_nw_dst(&hairpin_reply_match,
+ in6_addr_get_mapped_ipv4(&lb_vip->vip));
+ } else {
+ match_set_dl_type(&hairpin_match, htons(ETH_TYPE_IPV6));
+ match_set_ipv6_src(&hairpin_match, &lb_backend->ip);
+ match_set_ipv6_dst(&hairpin_match, &lb_backend->ip);
+
+ match_set_dl_type(&hairpin_reply_match,
+ htons(ETH_TYPE_IPV6));
+ match_set_ipv6_src(&hairpin_reply_match, &lb_backend->ip);
+ match_set_ipv6_dst(&hairpin_reply_match, &lb_vip->vip);
+ }
+
+ if (lb_backend->port) {
+ match_set_nw_proto(&hairpin_match, lb_proto);
+ match_set_tp_dst(&hairpin_match, htons(lb_backend->port));
+
+ match_set_nw_proto(&hairpin_reply_match, lb_proto);
+ match_set_tp_src(&hairpin_reply_match, htons(lb_backend->port));
+ }
+
+ for (size_t i = 0; i < lb->slb->n_datapaths; i++) {
+ match_set_metadata(&hairpin_match,
+ htonll(lb->slb->datapaths[i]->tunnel_key));
+ match_set_metadata(&hairpin_reply_match,
+ htonll(lb->slb->datapaths[i]->tunnel_key));
+
+ ofctrl_add_flow(flow_table, OFTABLE_CHK_LB_HAIRPIN, 100,
+ lb->slb->header_.uuid.parts[0], &hairpin_match,
+ &ofpacts, &lb->slb->header_.uuid);
+
+ ofctrl_add_flow(flow_table, OFTABLE_CHK_LB_HAIRPIN_REPLY, 100,
+ lb->slb->header_.uuid.parts[0],
+ &hairpin_reply_match,
+ &ofpacts, &lb->slb->header_.uuid);
+ }
+
+ ofpbuf_uninit(&ofpacts);
+}
+
+static void
+add_lb_ct_snat_vip_flows(struct ovn_controller_lb *lb,
+ struct ovn_lb_vip *lb_vip,
+ struct ovn_desired_flow_table *flow_table)
+{
+ uint64_t stub[1024 / 8];
+ struct ofpbuf ofpacts = OFPBUF_STUB_INITIALIZER(stub);
+
+ struct ofpact_conntrack *ct = ofpact_put_CT(&ofpacts);
+ ct->recirc_table = NX_CT_RECIRC_NONE;
+ ct->zone_src.field = mf_from_id(MFF_LOG_SNAT_ZONE);
+ ct->zone_src.ofs = 0;
+ ct->zone_src.n_bits = 16;
+ ct->flags = NX_CT_F_COMMIT;
+ ct->alg = 0;
+
+ size_t nat_offset;
+ nat_offset = ofpacts.size;
+ ofpbuf_pull(&ofpacts, nat_offset);
+
+ struct ofpact_nat *nat = ofpact_put_NAT(&ofpacts);
+ nat->flags = NX_NAT_F_SRC;
+ nat->range_af = AF_UNSPEC;
+
+ if (IN6_IS_ADDR_V4MAPPED(&lb_vip->vip)) {
+ nat->range_af = AF_INET;
+ nat->range.addr.ipv4.min = in6_addr_get_mapped_ipv4(&lb_vip->vip);
+ } else {
+ nat->range_af = AF_INET6;
+ nat->range.addr.ipv6.min = lb_vip->vip;
+ }
+ ofpacts.header = ofpbuf_push_uninit(&ofpacts, nat_offset);
+ ofpact_finish(&ofpacts, &ct->ofpact);
+
+ struct match match = MATCH_CATCHALL_INITIALIZER;
+ if (IN6_IS_ADDR_V4MAPPED(&lb_vip->vip)) {
+ match_set_dl_type(&match, htons(ETH_TYPE_IP));
+ match_set_ct_nw_dst(&match, nat->range.addr.ipv4.min);
+ } else {
+ match_set_dl_type(&match, htons(ETH_TYPE_IPV6));
+ match_set_ct_ipv6_dst(&match, &lb_vip->vip);
+ }
+
+ uint32_t ct_state = OVS_CS_F_TRACKED | OVS_CS_F_DST_NAT;
+ match_set_ct_state_masked(&match, ct_state, ct_state);
+
+ for (size_t i = 0; i < lb->slb->n_datapaths; i++) {
+ match_set_metadata(&match,
+ htonll(lb->slb->datapaths[i]->tunnel_key));
+
+ ofctrl_add_flow(flow_table, OFTABLE_CT_SNAT_FOR_VIP, 100,
+ lb->slb->header_.uuid.parts[0],
+ &match, &ofpacts, &lb->slb->header_.uuid);
+ }
+
+ ofpbuf_uninit(&ofpacts);
+}
+
+static void
+consider_lb_hairpin_flows(const struct sbrec_load_balancer *sbrec_lb,
+ const struct hmap *local_datapaths,
+ struct ovn_desired_flow_table *flow_table)
+{
+ /* Check if we need to add flows or not. If there is one datapath
+ * in the local_datapaths, it means all the datapaths of the lb
+ * will be in the local_datapaths. */
+ size_t i;
+ for (i = 0; i < sbrec_lb->n_datapaths; i++) {
+ if (get_local_datapath(local_datapaths,
+ sbrec_lb->datapaths[i]->tunnel_key)) {
+ break;
+ }
+ }
+
+ if (i == sbrec_lb->n_datapaths) {
+ return;
+ }
+
+ struct ovn_controller_lb *lb = ovn_controller_lb_create(sbrec_lb);
+ uint8_t lb_proto = IPPROTO_TCP;
+ if (lb->slb->protocol && lb->slb->protocol[0]) {
+ if (!strcmp(lb->slb->protocol, "udp")) {
+ lb_proto = IPPROTO_UDP;
+ } else if (!strcmp(lb->slb->protocol, "sctp")) {
+ lb_proto = IPPROTO_SCTP;
+ }
+ }
+
+ for (i = 0; i < lb->n_vips; i++) {
+ struct ovn_lb_vip *lb_vip = &lb->vips[i];
+
+ for (size_t j = 0; j < lb_vip->n_backends; j++) {
+ struct ovn_lb_backend *lb_backend = &lb_vip->backends[j];
+
+ add_lb_vip_hairpin_flows(lb, lb_vip, lb_backend, lb_proto,
+ flow_table);
+ }
+
+ add_lb_ct_snat_vip_flows(lb, lb_vip, flow_table);
+ }
+
+ ovn_controller_lb_destroy(lb);
+}
+
+/* Adds OpenFlow flows to flow tables for each Load balancer VIPs and
+ * backends to handle the load balanced hairpin traffic. */
+static void
+add_lb_hairpin_flows(const struct sbrec_load_balancer_table *lb_table,
+ const struct hmap *local_datapaths,
+ struct ovn_desired_flow_table *flow_table)
+{
+ const struct sbrec_load_balancer *lb;
+ SBREC_LOAD_BALANCER_TABLE_FOR_EACH (lb, lb_table) {
+ consider_lb_hairpin_flows(lb, local_datapaths, flow_table);
+ }
+}
+
/* Handles neighbor changes in mac_binding table. */
void
lflow_handle_changed_neighbors(
@@ -1203,6 +1388,8 @@ lflow_run(struct lflow_ctx_in *l_ctx_in, struct lflow_ctx_out *l_ctx_out)
add_neighbor_flows(l_ctx_in->sbrec_port_binding_by_name,
l_ctx_in->mac_binding_table, l_ctx_in->local_datapaths,
l_ctx_out->flow_table);
+ add_lb_hairpin_flows(l_ctx_in->lb_table, l_ctx_in->local_datapaths,
+ l_ctx_out->flow_table);
}
void
@@ -1262,6 +1449,15 @@ lflow_add_flows_for_datapath(const struct sbrec_datapath_binding *dp,
dhcp_opts_destroy(&dhcpv6_opts);
nd_ra_opts_destroy(&nd_ra_opts);
controller_event_opts_destroy(&controller_event_opts);
+
+ /* Add load balancer hairpin flows if the datapath has any load balancers
+ * associated. */
+ for (size_t i = 0; i < dp->n_load_balancers; i++) {
+ consider_lb_hairpin_flows(dp->load_balancers[i],
+ l_ctx_in->local_datapaths,
+ l_ctx_out->flow_table);
+ }
+
return handled;
}
@@ -1279,3 +1475,37 @@ lflow_handle_flows_for_lport(const struct sbrec_port_binding *pb,
return lflow_handle_changed_ref(REF_TYPE_PORTBINDING, pb_ref_name,
l_ctx_in, l_ctx_out, &changed);
}
+
+bool
+lflow_handle_changed_lbs(struct lflow_ctx_in *l_ctx_in,
+ struct lflow_ctx_out *l_ctx_out)
+{
+ const struct sbrec_load_balancer *lb;
+
+ SBREC_LOAD_BALANCER_TABLE_FOR_EACH_TRACKED (lb, l_ctx_in->lb_table) {
+ if (sbrec_load_balancer_is_deleted(lb)) {
+ VLOG_DBG("Remove hairpin flows for deleted load balancer "UUID_FMT,
+ UUID_ARGS(&lb->header_.uuid));
+ ofctrl_remove_flows(l_ctx_out->flow_table, &lb->header_.uuid);
+ }
+ }
+
+ SBREC_LOAD_BALANCER_TABLE_FOR_EACH_TRACKED (lb, l_ctx_in->lb_table) {
+ if (sbrec_load_balancer_is_deleted(lb)) {
+ continue;
+ }
+
+ if (!sbrec_load_balancer_is_new(lb)) {
+ VLOG_DBG("Remove hairpin flows for updated load balancer "UUID_FMT,
+ UUID_ARGS(&lb->header_.uuid));
+ ofctrl_remove_flows(l_ctx_out->flow_table, &lb->header_.uuid);
+ }
+
+ VLOG_DBG("Add load balancer hairpin flows for "UUID_FMT,
+ UUID_ARGS(&lb->header_.uuid));
+ consider_lb_hairpin_flows(lb, l_ctx_in->local_datapaths,
+ l_ctx_out->flow_table);
+ }
+
+ return true;
+}
diff --git a/controller/lflow.h b/controller/lflow.h
index 1251fb0f4..1225131de 100644
--- a/controller/lflow.h
+++ b/controller/lflow.h
@@ -68,6 +68,9 @@ struct uuid;
#define OFTABLE_LOG_TO_PHY 65
#define OFTABLE_MAC_BINDING 66
#define OFTABLE_MAC_LOOKUP 67
+#define OFTABLE_CHK_LB_HAIRPIN 68
+#define OFTABLE_CHK_LB_HAIRPIN_REPLY 69
+#define OFTABLE_CT_SNAT_FOR_VIP 70
/* The number of tables for the ingress and egress pipelines. */
#define LOG_PIPELINE_LEN 24
@@ -132,6 +135,7 @@ struct lflow_ctx_in {
const struct sbrec_logical_flow_table *logical_flow_table;
const struct sbrec_multicast_group_table *mc_group_table;
const struct sbrec_chassis *chassis;
+ const struct sbrec_load_balancer_table *lb_table;
const struct hmap *local_datapaths;
const struct shash *addr_sets;
const struct shash *port_groups;
@@ -160,7 +164,7 @@ void lflow_handle_changed_neighbors(
const struct sbrec_mac_binding_table *,
const struct hmap *local_datapaths,
struct ovn_desired_flow_table *);
-
+bool lflow_handle_changed_lbs(struct lflow_ctx_in *, struct lflow_ctx_out *);
void lflow_destroy(void);
void lflow_cache_init(struct hmap *);
diff --git a/controller/ovn-controller.c b/controller/ovn-controller.c
index 8d8c678e5..e5479cf3e 100644
--- a/controller/ovn-controller.c
+++ b/controller/ovn-controller.c
@@ -790,7 +790,8 @@ ctrl_register_ovs_idl(struct ovsdb_idl *ovs_idl)
SB_NODE(logical_flow, "logical_flow") \
SB_NODE(dhcp_options, "dhcp_options") \
SB_NODE(dhcpv6_options, "dhcpv6_options") \
- SB_NODE(dns, "dns")
+ SB_NODE(dns, "dns") \
+ SB_NODE(load_balancer, "load_balancer")
enum sb_engine_node {
#define SB_NODE(NAME, NAME_STR) SB_##NAME,
@@ -1682,6 +1683,10 @@ static void init_lflow_ctx(struct engine_node *node,
(struct sbrec_multicast_group_table *)EN_OVSDB_GET(
engine_get_input("SB_multicast_group", node));
+ struct sbrec_load_balancer_table *lb_table =
+ (struct sbrec_load_balancer_table *)EN_OVSDB_GET(
+ engine_get_input("SB_load_balancer", node));
+
const char *chassis_id = chassis_get_id();
const struct sbrec_chassis *chassis = NULL;
struct ovsdb_idl_index *sbrec_chassis_by_name =
@@ -1713,6 +1718,7 @@ static void init_lflow_ctx(struct engine_node *node,
l_ctx_in->logical_flow_table = logical_flow_table;
l_ctx_in->mc_group_table = multicast_group_table;
l_ctx_in->chassis = chassis;
+ l_ctx_in->lb_table = lb_table;
l_ctx_in->local_datapaths = &rt_data->local_datapaths;
l_ctx_in->addr_sets = addr_sets;
l_ctx_in->port_groups = port_groups;
@@ -2131,6 +2137,23 @@ flow_output_runtime_data_handler(struct engine_node *node,
return true;
}
+static bool
+flow_output_sb_load_balancer_handler(struct engine_node *node, void *data)
+{
+ struct ed_type_runtime_data *rt_data =
+ engine_get_input_data("runtime_data", node);
+
+ struct ed_type_flow_output *fo = data;
+ struct lflow_ctx_in l_ctx_in;
+ struct lflow_ctx_out l_ctx_out;
+ init_lflow_ctx(node, rt_data, fo, &l_ctx_in, &l_ctx_out);
+
+ bool handled = lflow_handle_changed_lbs(&l_ctx_in, &l_ctx_out);
+
+ engine_set_node_state(node, EN_UPDATED);
+ return handled;
+}
+
struct ovn_controller_exit_args {
bool *exiting;
bool *restart;
@@ -2327,6 +2350,8 @@ main(int argc, char *argv[])
engine_add_input(&en_flow_output, &en_sb_dhcp_options, NULL);
engine_add_input(&en_flow_output, &en_sb_dhcpv6_options, NULL);
engine_add_input(&en_flow_output, &en_sb_dns, NULL);
+ engine_add_input(&en_flow_output, &en_sb_load_balancer,
+ flow_output_sb_load_balancer_handler);
engine_add_input(&en_ct_zones, &en_ovs_open_vswitch, NULL);
engine_add_input(&en_ct_zones, &en_ovs_bridge, NULL);
diff --git a/include/ovn/logical-fields.h b/include/ovn/logical-fields.h
index ac6f2f909..0fe5bc3bb 100644
--- a/include/ovn/logical-fields.h
+++ b/include/ovn/logical-fields.h
@@ -57,6 +57,7 @@ enum mff_log_flags_bits {
MLF_LOCAL_ONLY_BIT = 4,
MLF_NESTED_CONTAINER_BIT = 5,
MLF_LOOKUP_MAC_BIT = 6,
+ MLF_LOOKUP_LB_HAIRPIN_BIT = 7,
};
/* MFF_LOG_FLAGS_REG flag assignments */
@@ -88,6 +89,8 @@ enum mff_log_flags {
/* Indicate that the lookup in the mac binding table was successful. */
MLF_LOOKUP_MAC = (1 << MLF_LOOKUP_MAC_BIT),
+
+ MLF_LOOKUP_LB_HAIRPIN = (1 << MLF_LOOKUP_LB_HAIRPIN_BIT),
};
/* OVN logical fields
diff --git a/tests/ovn.at b/tests/ovn.at
index ba17246d4..5cb96bae6 100644
--- a/tests/ovn.at
+++ b/tests/ovn.at
@@ -22727,3 +22727,472 @@ AT_CHECK([test "$encap_rec_mvtep" == "$encap_rec_mvtep1"], [0], [])
OVN_CLEANUP([hv1])
AT_CLEANUP
+
+AT_SETUP([ovn -- Load Balancer LS hairpin OF flows])
+ovn_start
+
+net_add n1
+
+sim_add hv1
+as hv1
+ovs-vsctl add-br br-phys
+ovn_attach n1 br-phys 192.168.0.1
+ovs-vsctl -- add-port br-int hv1-vif1 -- \
+ set interface hv1-vif1 external-ids:iface-id=sw0-p1 \
+ options:tx_pcap=hv1/vif1-tx.pcap \
+ options:rxq_pcap=hv1/vif1-rx.pcap \
+ ofport-request=1
+ovs-vsctl -- add-port br-int hv1-vif2 -- \
+ set interface hv1-vif2 external-ids:iface-id=sw1-p1 \
+ options:tx_pcap=hv1/vif2-tx.pcap \
+ options:rxq_pcap=hv1/vif2-rx.pcap \
+ ofport-request=2
+
+sim_add hv2
+as hv2
+ovs-vsctl add-br br-phys
+ovn_attach n1 br-phys 192.168.0.2
+ovs-vsctl -- add-port br-int hv2-vif1 -- \
+ set interface hv2-vif1 external-ids:iface-id=sw0-p2 \
+ options:tx_pcap=hv2/vif1-tx.pcap \
+ options:rxq_pcap=hv2/vif1-rx.pcap \
+ ofport-request=1
+ovs-vsctl -- add-port br-int hv1-vif2 -- \
+ set interface hv1-vif2 external-ids:iface-id=sw1-p2 \
+ options:tx_pcap=hv1/vif2-tx.pcap \
+ options:rxq_pcap=hv1/vif2-rx.pcap \
+ ofport-request=2
+
+check ovn-nbctl --wait=hv ls-add sw0
+check ovn-nbctl lsp-add sw0 sw0-p1 -- lsp-set-addresses sw0-p1 00:00:00:00:00:01
+
+check ovn-nbctl ls-add sw1
+check ovn-nbctl lsp-add sw1 sw1-p1 -- lsp-set-addresses sw1-p1 00:00:00:00:01:01
+
+OVS_WAIT_UNTIL([test x$(ovn-nbctl lsp-get-up sw0-p1) = xup])
+OVS_WAIT_UNTIL([test x$(ovn-nbctl lsp-get-up sw1-p1) = xup])
+
+check ovn-nbctl lb-add lb-ipv4-tcp 88.88.88.88:8080 42.42.42.1:4041 tcp
+check ovn-nbctl lb-add lb-ipv4-udp 88.88.88.88:4040 42.42.42.1:2021 udp
+check ovn-nbctl lb-add lb-ipv6-tcp [[8800::0088]]:8080 [[4200::1]]:4041 tcp
+check ovn-nbctl --wait=hv lb-add lb-ipv6-udp [[8800::0088]]:4040 [[4200::1]]:2021 udp
+
+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=68], [0], [dnl
+NXST_FLOW reply (xid=0x8):
+])
+
+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=69], [0], [dnl
+NXST_FLOW reply (xid=0x8):
+])
+
+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=70], [0], [dnl
+NXST_FLOW reply (xid=0x8):
+])
+
+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=68], [0], [dnl
+NXST_FLOW reply (xid=0x8):
+])
+
+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=69], [0], [dnl
+NXST_FLOW reply (xid=0x8):
+])
+
+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=70], [0], [dnl
+NXST_FLOW reply (xid=0x8):
+])
+
+check ovn-nbctl --wait=hv ls-lb-add sw0 lb-ipv4-tcp
+
+OVS_WAIT_UNTIL(
+ [test $(as hv1 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | wc -l) -eq 1]
+)
+
+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | cut -d ' ' -f8-], [0], [dnl
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.1,nw_dst=42.42.42.1,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+])
+
+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=69 | grep -v NXST | cut -d ' ' -f8-], [0], [dnl
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.88,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+])
+
+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=70 | grep -v NXST | cut -d ' ' -f8-], [0], [dnl
+priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.88,ip,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.88))
+])
+
+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=68], [0], [dnl
+NXST_FLOW reply (xid=0x8):
+])
+
+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=69], [0], [dnl
+NXST_FLOW reply (xid=0x8):
+])
+
+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=70], [0], [dnl
+NXST_FLOW reply (xid=0x8):
+])
+
+check ovn-nbctl lb-add lb-ipv4-tcp 88.88.88.90:8080 42.42.42.42:4041,52.52.52.52:4042 tcp
+
+OVS_WAIT_UNTIL(
+ [test $(as hv1 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | wc -l) -eq 3]
+)
+
+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.1,nw_dst=42.42.42.1,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.42,nw_dst=42.42.42.42,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=52.52.52.52,nw_dst=52.52.52.52,tp_dst=4042 actions=load:0x1->NXM_NX_REG10[[7]]
+])
+
+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=69 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.88,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.42,nw_dst=88.88.88.90,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=52.52.52.52,nw_dst=88.88.88.90,tp_src=4042 actions=load:0x1->NXM_NX_REG10[[7]]
+])
+
+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=70 | grep -v NXST | cut -d ' ' -f8-], [0], [dnl
+priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.88,ip,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.88))
+priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.90,ip,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.90))
+])
+
+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=68], [0], [dnl
+NXST_FLOW reply (xid=0x8):
+])
+
+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=69], [0], [dnl
+NXST_FLOW reply (xid=0x8):
+])
+
+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=70], [0], [dnl
+NXST_FLOW reply (xid=0x8):
+])
+
+check ovn-nbctl lsp-add sw0 sw0-p2
+# hv2 should bind sw0-p2 and it should install the LB hairpin flows.
+OVS_WAIT_UNTIL([test x$(ovn-nbctl lsp-get-up sw0-p2) = xup])
+
+OVS_WAIT_UNTIL(
+ [test $(as hv2 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | wc -l) -eq 3]
+)
+
+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.1,nw_dst=42.42.42.1,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.42,nw_dst=42.42.42.42,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=52.52.52.52,nw_dst=52.52.52.52,tp_dst=4042 actions=load:0x1->NXM_NX_REG10[[7]]
+])
+
+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=69 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.88,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.42,nw_dst=88.88.88.90,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=52.52.52.52,nw_dst=88.88.88.90,tp_src=4042 actions=load:0x1->NXM_NX_REG10[[7]]
+])
+
+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=70 | grep -v NXST | cut -d ' ' -f8-], [0], [dnl
+priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.88,ip,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.88))
+priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.90,ip,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.90))
+])
+
+check ovn-nbctl --wait=hv ls-lb-add sw0 lb-ipv4-udp
+
+OVS_WAIT_UNTIL(
+ [test $(as hv1 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | wc -l) -eq 4]
+)
+
+OVS_WAIT_UNTIL(
+ [test $(as hv2 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | wc -l) -eq 4]
+)
+
+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.1,nw_dst=42.42.42.1,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.42,nw_dst=42.42.42.42,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=52.52.52.52,nw_dst=52.52.52.52,tp_dst=4042 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,udp,metadata=0x1,nw_src=42.42.42.1,nw_dst=42.42.42.1,tp_dst=2021 actions=load:0x1->NXM_NX_REG10[[7]]
+])
+
+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=69 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.88,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.42,nw_dst=88.88.88.90,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=52.52.52.52,nw_dst=88.88.88.90,tp_src=4042 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,udp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.88,tp_src=2021 actions=load:0x1->NXM_NX_REG10[[7]]
+])
+
+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=70 | grep -v NXST | cut -d ' ' -f8-], [0], [dnl
+priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.88,ip,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.88))
+priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.90,ip,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.90))
+])
+
+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.1,nw_dst=42.42.42.1,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.42,nw_dst=42.42.42.42,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=52.52.52.52,nw_dst=52.52.52.52,tp_dst=4042 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,udp,metadata=0x1,nw_src=42.42.42.1,nw_dst=42.42.42.1,tp_dst=2021 actions=load:0x1->NXM_NX_REG10[[7]]
+])
+
+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=69 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.88,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.42,nw_dst=88.88.88.90,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=52.52.52.52,nw_dst=88.88.88.90,tp_src=4042 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,udp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.88,tp_src=2021 actions=load:0x1->NXM_NX_REG10[[7]]
+])
+
+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=70 | grep -v NXST | cut -d ' ' -f8-], [0], [dnl
+priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.88,ip,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.88))
+priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.90,ip,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.90))
+])
+
+check ovn-nbctl --wait=hv ls-lb-add sw0 lb-ipv6-tcp
+
+OVS_WAIT_UNTIL(
+ [test $(as hv1 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | wc -l) -eq 5]
+)
+
+OVS_WAIT_UNTIL(
+ [test $(as hv2 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | wc -l) -eq 5]
+)
+
+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.1,nw_dst=42.42.42.1,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.42,nw_dst=42.42.42.42,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=52.52.52.52,nw_dst=52.52.52.52,tp_dst=4042 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp6,metadata=0x1,ipv6_src=4200::1,ipv6_dst=4200::1,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,udp,metadata=0x1,nw_src=42.42.42.1,nw_dst=42.42.42.1,tp_dst=2021 actions=load:0x1->NXM_NX_REG10[[7]]
+])
+
+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=69 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.88,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.42,nw_dst=88.88.88.90,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=52.52.52.52,nw_dst=88.88.88.90,tp_src=4042 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp6,metadata=0x1,ipv6_src=4200::1,ipv6_dst=8800::88,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,udp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.88,tp_src=2021 actions=load:0x1->NXM_NX_REG10[[7]]
+])
+
+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=70 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl
+priority=100,ct_state=+trk+dnat,ct_ipv6_dst=8800::88,ipv6,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=8800::88))
+priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.88,ip,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.88))
+priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.90,ip,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.90))
+])
+
+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.1,nw_dst=42.42.42.1,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.42,nw_dst=42.42.42.42,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=52.52.52.52,nw_dst=52.52.52.52,tp_dst=4042 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp6,metadata=0x1,ipv6_src=4200::1,ipv6_dst=4200::1,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,udp,metadata=0x1,nw_src=42.42.42.1,nw_dst=42.42.42.1,tp_dst=2021 actions=load:0x1->NXM_NX_REG10[[7]]
+])
+
+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=69 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.88,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.42,nw_dst=88.88.88.90,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=52.52.52.52,nw_dst=88.88.88.90,tp_src=4042 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp6,metadata=0x1,ipv6_src=4200::1,ipv6_dst=8800::88,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,udp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.88,tp_src=2021 actions=load:0x1->NXM_NX_REG10[[7]]
+])
+
+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=70 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl
+priority=100,ct_state=+trk+dnat,ct_ipv6_dst=8800::88,ipv6,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=8800::88))
+priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.88,ip,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.88))
+priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.90,ip,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.90))
+])
+
+check ovn-nbctl --wait=hv ls-lb-add sw0 lb-ipv6-udp
+
+OVS_WAIT_UNTIL(
+ [test $(as hv1 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | wc -l) -eq 6]
+)
+
+OVS_WAIT_UNTIL(
+ [test $(as hv2 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | wc -l) -eq 6]
+)
+
+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.1,nw_dst=42.42.42.1,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.42,nw_dst=42.42.42.42,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=52.52.52.52,nw_dst=52.52.52.52,tp_dst=4042 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp6,metadata=0x1,ipv6_src=4200::1,ipv6_dst=4200::1,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,udp,metadata=0x1,nw_src=42.42.42.1,nw_dst=42.42.42.1,tp_dst=2021 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,udp6,metadata=0x1,ipv6_src=4200::1,ipv6_dst=4200::1,tp_dst=2021 actions=load:0x1->NXM_NX_REG10[[7]]
+])
+
+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=69 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.88,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.42,nw_dst=88.88.88.90,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=52.52.52.52,nw_dst=88.88.88.90,tp_src=4042 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp6,metadata=0x1,ipv6_src=4200::1,ipv6_dst=8800::88,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,udp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.88,tp_src=2021 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,udp6,metadata=0x1,ipv6_src=4200::1,ipv6_dst=8800::88,tp_src=2021 actions=load:0x1->NXM_NX_REG10[[7]]
+])
+
+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=70 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl
+priority=100,ct_state=+trk+dnat,ct_ipv6_dst=8800::88,ipv6,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=8800::88))
+priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.88,ip,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.88))
+priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.90,ip,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.90))
+])
+
+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.1,nw_dst=42.42.42.1,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.42,nw_dst=42.42.42.42,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=52.52.52.52,nw_dst=52.52.52.52,tp_dst=4042 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp6,metadata=0x1,ipv6_src=4200::1,ipv6_dst=4200::1,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,udp,metadata=0x1,nw_src=42.42.42.1,nw_dst=42.42.42.1,tp_dst=2021 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,udp6,metadata=0x1,ipv6_src=4200::1,ipv6_dst=4200::1,tp_dst=2021 actions=load:0x1->NXM_NX_REG10[[7]]
+])
+
+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=69 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.88,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.42,nw_dst=88.88.88.90,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=52.52.52.52,nw_dst=88.88.88.90,tp_src=4042 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp6,metadata=0x1,ipv6_src=4200::1,ipv6_dst=8800::88,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,udp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.88,tp_src=2021 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,udp6,metadata=0x1,ipv6_src=4200::1,ipv6_dst=8800::88,tp_src=2021 actions=load:0x1->NXM_NX_REG10[[7]]
+])
+
+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=70 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl
+priority=100,ct_state=+trk+dnat,ct_ipv6_dst=8800::88,ipv6,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=8800::88))
+priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.88,ip,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.88))
+priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.90,ip,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.90))
+])
+
+check ovn-nbctl --wait=hv ls-lb-add sw1 lb-ipv6-udp
+
+OVS_WAIT_UNTIL(
+ [test $(as hv1 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | wc -l) -eq 7]
+)
+
+OVS_WAIT_UNTIL(
+ [test $(as hv2 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | wc -l) -eq 7]
+)
+
+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.1,nw_dst=42.42.42.1,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.42,nw_dst=42.42.42.42,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=52.52.52.52,nw_dst=52.52.52.52,tp_dst=4042 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp6,metadata=0x1,ipv6_src=4200::1,ipv6_dst=4200::1,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,udp,metadata=0x1,nw_src=42.42.42.1,nw_dst=42.42.42.1,tp_dst=2021 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,udp6,metadata=0x1,ipv6_src=4200::1,ipv6_dst=4200::1,tp_dst=2021 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,udp6,metadata=0x2,ipv6_src=4200::1,ipv6_dst=4200::1,tp_dst=2021 actions=load:0x1->NXM_NX_REG10[[7]]
+])
+
+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=69 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.88,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.42,nw_dst=88.88.88.90,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=52.52.52.52,nw_dst=88.88.88.90,tp_src=4042 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp6,metadata=0x1,ipv6_src=4200::1,ipv6_dst=8800::88,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,udp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.88,tp_src=2021 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,udp6,metadata=0x1,ipv6_src=4200::1,ipv6_dst=8800::88,tp_src=2021 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,udp6,metadata=0x2,ipv6_src=4200::1,ipv6_dst=8800::88,tp_src=2021 actions=load:0x1->NXM_NX_REG10[[7]]
+])
+
+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=70 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl
+priority=100,ct_state=+trk+dnat,ct_ipv6_dst=8800::88,ipv6,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=8800::88))
+priority=100,ct_state=+trk+dnat,ct_ipv6_dst=8800::88,ipv6,metadata=0x2 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=8800::88))
+priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.88,ip,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.88))
+priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.90,ip,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.90))
+])
+
+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.1,nw_dst=42.42.42.1,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.42,nw_dst=42.42.42.42,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=52.52.52.52,nw_dst=52.52.52.52,tp_dst=4042 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp6,metadata=0x1,ipv6_src=4200::1,ipv6_dst=4200::1,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,udp,metadata=0x1,nw_src=42.42.42.1,nw_dst=42.42.42.1,tp_dst=2021 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,udp6,metadata=0x1,ipv6_src=4200::1,ipv6_dst=4200::1,tp_dst=2021 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,udp6,metadata=0x2,ipv6_src=4200::1,ipv6_dst=4200::1,tp_dst=2021 actions=load:0x1->NXM_NX_REG10[[7]]
+])
+
+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=69 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.88,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=42.42.42.42,nw_dst=88.88.88.90,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp,metadata=0x1,nw_src=52.52.52.52,nw_dst=88.88.88.90,tp_src=4042 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,tcp6,metadata=0x1,ipv6_src=4200::1,ipv6_dst=8800::88,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,udp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.88,tp_src=2021 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,udp6,metadata=0x1,ipv6_src=4200::1,ipv6_dst=8800::88,tp_src=2021 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,udp6,metadata=0x2,ipv6_src=4200::1,ipv6_dst=8800::88,tp_src=2021 actions=load:0x1->NXM_NX_REG10[[7]]
+])
+
+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=70 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl
+priority=100,ct_state=+trk+dnat,ct_ipv6_dst=8800::88,ipv6,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=8800::88))
+priority=100,ct_state=+trk+dnat,ct_ipv6_dst=8800::88,ipv6,metadata=0x2 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=8800::88))
+priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.88,ip,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.88))
+priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.90,ip,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.90))
+])
+
+as hv2 ovs-vsctl del-port hv2-vif1
+OVS_WAIT_UNTIL([test x$(ovn-nbctl lsp-get-up sw0-p2) = xdown])
+
+# Trigger recompute on hv2 as sw0 will not be cleared from local_datapaths.
+as hv2 ovn-appctl -t ovn-controller recompute
+
+OVS_WAIT_UNTIL(
+ [test $(as hv2 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | wc -l) -eq 0]
+)
+
+OVS_WAIT_UNTIL(
+ [test $(as hv2 ovs-ofctl dump-flows br-int table=69 | grep -v NXST | wc -l) -eq 0]
+)
+
+OVS_WAIT_UNTIL(
+ [test $(as hv2 ovs-ofctl dump-flows br-int table=70 | grep -v NXST | wc -l) -eq 0]
+)
+
+OVS_WAIT_UNTIL(
+ [test $(as hv1 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | wc -l) -eq 7]
+)
+
+check ovn-nbctl --wait=hv lb-del lb-ipv4-tcp
+
+OVS_WAIT_UNTIL(
+ [test $(as hv1 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | wc -l) -eq 4]
+)
+
+OVS_WAIT_UNTIL(
+ [test $(as hv2 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | wc -l) -eq 0]
+)
+
+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl
+priority=100,tcp6,metadata=0x1,ipv6_src=4200::1,ipv6_dst=4200::1,tp_dst=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,udp,metadata=0x1,nw_src=42.42.42.1,nw_dst=42.42.42.1,tp_dst=2021 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,udp6,metadata=0x1,ipv6_src=4200::1,ipv6_dst=4200::1,tp_dst=2021 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,udp6,metadata=0x2,ipv6_src=4200::1,ipv6_dst=4200::1,tp_dst=2021 actions=load:0x1->NXM_NX_REG10[[7]]
+])
+
+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=69 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl
+priority=100,tcp6,metadata=0x1,ipv6_src=4200::1,ipv6_dst=8800::88,tp_src=4041 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,udp,metadata=0x1,nw_src=42.42.42.1,nw_dst=88.88.88.88,tp_src=2021 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,udp6,metadata=0x1,ipv6_src=4200::1,ipv6_dst=8800::88,tp_src=2021 actions=load:0x1->NXM_NX_REG10[[7]]
+priority=100,udp6,metadata=0x2,ipv6_src=4200::1,ipv6_dst=8800::88,tp_src=2021 actions=load:0x1->NXM_NX_REG10[[7]]
+])
+
+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=70 | grep -v NXST | cut -d ' ' -f8- | sort], [0], [dnl
+priority=100,ct_state=+trk+dnat,ct_ipv6_dst=8800::88,ipv6,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=8800::88))
+priority=100,ct_state=+trk+dnat,ct_ipv6_dst=8800::88,ipv6,metadata=0x2 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=8800::88))
+priority=100,ct_state=+trk+dnat,ct_nw_dst=88.88.88.88,ip,metadata=0x1 actions=ct(commit,zone=NXM_NX_REG12[[0..15]],nat(src=88.88.88.88))
+])
+
+check ovn-nbctl --wait=hv ls-del sw0
+check ovn-nbctl --wait=hv ls-del sw1
+
+OVS_WAIT_UNTIL(
+ [test $(as hv1 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | wc -l) -eq 0]
+)
+
+OVS_WAIT_UNTIL(
+ [test $(as hv1 ovs-ofctl dump-flows br-int table=69 | grep -v NXST | wc -l) -eq 0]
+)
+
+OVS_WAIT_UNTIL(
+ [test $(as hv1 ovs-ofctl dump-flows br-int table=70 | grep -v NXST | wc -l) -eq 0]
+)
+
+OVS_WAIT_UNTIL(
+ [test $(as hv2 ovs-ofctl dump-flows br-int table=68 | grep -v NXST | wc -l) -eq 0]
+)
+
+OVS_WAIT_UNTIL(
+ [test $(as hv2 ovs-ofctl dump-flows br-int table=69 | grep -v NXST | wc -l) -eq 0]
+)
+
+OVS_WAIT_UNTIL(
+ [test $(as hv2 ovs-ofctl dump-flows br-int table=70 | grep -v NXST | wc -l) -eq 0]
+)
+
+OVN_CLEANUP([hv1], [hv2])
+AT_CLEANUP
--
2.28.0