From 0f2bc62c05f039f3311aebf33f7c01f49caabc5f Mon Sep 17 00:00:00 2001

From: Ben Pfaff <blp@ovn.org>

Date: Wed, 21 Oct 2020 15:01:35 -0700

Subject: [PATCH 03/10] tests: Introduce new testing helpers.

These simplify a lot of otherwise harder to understand checks within

the tests.  This commit should show how valuable they are, although I'm

sure scope remains to use them in more places.

Signed-off-by: Ben Pfaff <blp@ovn.org>

Acked-by: Numan Siddique <numans@ovn.org>

(cherry-picked from master commit 4afe409e95c72187a8f7a755fa19b17237d14818)

Conflicts:

	tests/ovn-northd.at

---

 tests/ovn-controller.at |   8 +-

 tests/ovn-ic.at         |  31 +--

 tests/ovn-macros.at     | 121 ++++++++++

 tests/ovn-nbctl.at      |  12 +-

 tests/ovn-northd.at     | 520 ++++++++++------------------------------

 tests/ovn.at            | 495 +++++++++++++-------------------------

 tests/ovs-macros.at     |  41 +++-

 7 files changed, 466 insertions(+), 762 deletions(-)

diff --git a/tests/ovn-controller.at b/tests/ovn-controller.at

index d8061345f..014a97760 100644

--- a/tests/ovn-controller.at

+++ b/tests/ovn-controller.at

@@ -201,9 +201,7 @@ OVS_WAIT_UNTIL([

 ])

 # Only one Chassis_Private record should exist.

-OVS_WAIT_UNTIL([

-    test $(ovn-sbctl --columns _uuid list chassis_private | wc -l) -eq 1

-])

+wait_row_count Chassis_Private 1

 # Simulate system-id changing while ovn-controller is disconnected from the

 # SB.

@@ -227,9 +225,7 @@ OVS_WAIT_UNTIL([

 ])

 # Only one Chassis_Private record should exist.

-OVS_WAIT_UNTIL([

-    test $(ovn-sbctl --columns _uuid list chassis_private | wc -l) -eq 1

-])

+wait_row_count Chassis_Private 1

 # Gracefully terminate daemons

 OVN_CLEANUP_SBOX([hv])

diff --git a/tests/ovn-ic.at b/tests/ovn-ic.at

index 6fb00319a..1d40ce958 100644

--- a/tests/ovn-ic.at

+++ b/tests/ovn-ic.at

@@ -5,7 +5,7 @@ ovn_init_ic_db

 ovn_start az1

 ovn_start az2

-OVS_WAIT_UNTIL([test `ovn-ic-sbctl show | wc -l` -eq 2])

+wait_row_count ic-sb:Availability_Zone 2

 AT_CHECK([ovn-ic-sbctl show], [0], [dnl

 availability-zone az1

 availability-zone az2

@@ -39,32 +39,21 @@ AT_CHECK([ovn-ic-nbctl ts-add ts1])

 AT_CHECK([ovn-ic-nbctl ts-add ts2])

 # Check ISB

-OVS_WAIT_UNTIL([ovn-ic-sbctl list datapath | grep ts2])

-AT_CHECK([ovn-ic-sbctl -f csv -d bare --no-headings --columns transit_switch list datapath | sort], [0], [dnl

-ts1

-ts2

-])

+wait_row_count ic-sb:Datapath_Binding 1 transit_switch=ts1

+wait_row_count ic-sb:Datapath_Binding 1 transit_switch=ts2

+check_column "ts1 ts2" ic-sb:Datapath_Binding transit_switch

+check_column "ts1 ts2" nb:Logical_Switch name

-# Check NB

-AT_CHECK([ovn-nbctl -f csv -d bare --no-headings --columns name list logical_switch | sort], [0], [dnl

-ts1

-ts2

-])

 # Check SB DP key

-ts1_key=$(ovn-ic-sbctl -f csv -d bare --no-headings --columns tunnel_key find datapath transit_switch=ts1)

-sb_ts1_key=$(ovn-sbctl -f csv -d bare --no-headings --columns tunnel_key find datapath_binding external_ids:interconn-ts=ts1)

-AT_CHECK([test $ts1_key = $sb_ts1_key])

+ts1_key=$(fetch_column ic-sb:Datapath_Binding tunnel_key transit_switch=ts1)

+check_column "$ts1_key" Datapath_Binding tunnel_key external_ids:interconn-ts=ts1

 # Test delete

 AT_CHECK([ovn-ic-nbctl ts-del ts1])

-OVS_WAIT_WHILE([ovn-ic-sbctl list datapath | grep ts1])

-AT_CHECK([ovn-ic-sbctl -f csv -d bare --no-headings --columns transit_switch list datapath], [0], [dnl

-ts2

-])

-AT_CHECK([ovn-nbctl -f csv -d bare --no-headings --columns name list logical_switch | sort], [0], [dnl

-ts2

-])

+wait_row_count ic-sb:Datapath_Binding 0 transit_switch=ts1

+check_column ts2 ic-sb:Datapath_Binding transit_switch

+check_column ts2 nb:Logical_Switch name

 OVN_CLEANUP_IC([az1])

diff --git a/tests/ovn-macros.at b/tests/ovn-macros.at

index a6719be83..be596caf3 100644

--- a/tests/ovn-macros.at

+++ b/tests/ovn-macros.at

@@ -286,4 +286,125 @@ ovn_populate_arp__() {

 }

 m4_divert_pop([PREPARE_TESTS])

+OVS_START_SHELL_HELPERS

+# check COMMAND...

+#

+# Runs COMMAND and checks that it succeeds without any output.

+check() {

+    echo "$@"

+    AT_CHECK(["$@"])

+}

+

+parse_db() {

+    case $1 in

+        (*:*) echo ${1%%:*} ;;

+        (*) echo sb ;;

+    esac

+}

+

+parse_table() {

+    case $1 in

+        (*:*) echo ${1##*:} ;;

+        (*) echo $1 ;;

+    esac

+}

+

+# count_rows TABLE [CONDITION...]

+#

+# Prints the number of rows in TABLE (that satisfy CONDITION).

+# Uses the southbound db by default; set DB=nb for the northbound database.

+count_rows() {

+    local db=$(parse_db $1) table=$(parse_table $1); shift

+    ovn-${db}ctl --format=table --no-headings find $table "$@" | wc -l

+}

+

+# check_row_count [DATABASE:]TABLE COUNT [CONDITION...]

+#

+# Checks that TABLE contains COUNT rows (that satisfy CONDITION).

+# The default DATABASE is "sb".

+check_row_count() {

+    local db=$(parse_db $1) table=$(parse_table $1); shift

+    local count=$1; shift

+    local found=$(count_rows $db:$table "$@")

+    echo

+    echo "Checking for $count rows in $db $table${1+ with $*}... found $found"

+    if test "$count" != "$found"; then

+        ovn-${db}ctl list $table

+        AT_FAIL_IF([:])

+    fi

+}

+

+# wait_row_count [DATABASE:]TABLE COUNT [CONDITION...]

+#

+# Waits until TABLE contains COUNT rows (that satisfy CONDITION).

+# The default DATABASE is "sb".

+wait_row_count() {

+    local db=$(parse_db $1) table=$(parse_table $1); shift

+    local count=$1; shift

+    local a=$1 b=$2 c=$3

+    echo "Waiting until $count rows in $db $table${1+ with $*}..."

+    OVS_WAIT_UNTIL([test $count = $(count_rows $db:$table $a $b $c)],[

+      echo "$db table $table has the following rows. $(count_rows $db:$table $a $b $c) rows match instead of expected $count:"

+      ovn-${db}ctl list $table])

+}

+

+# fetch_column [DATABASE:]TABLE COLUMN [CONDITION...]

+#

+# Fetches and prints all the values of COLUMN in the rows of TABLE

+# (that satisfy CONDITION), sorting the results lexicographically.

+# The default DATABASE is "sb".

+fetch_column() {

+    local db=$(parse_db $1) table=$(parse_table $1) column=${2-_uuid}; shift; shift

+    # Using "echo" removes spaces and newlines.

+    echo $(ovn-${db}ctl --bare --columns $column find $table "$@" | sort)

+}

+

+# check_column EXPECTED [DATABASE:]TABLE COLUMN [CONDITION...]

+#

+# Fetches all of the values of COLUMN in the rows of TABLE (that

+# satisfy CONDITION), and compares them against EXPECTED (ignoring

+# order).

+#

+# The default DATABASE is "sb".

+check_column() {

+    local expected=$1 db=$(parse_db $2) table=$(parse_table $2) column=${3-_uuid}; shift; shift; shift

+    local found=$(ovn-${db}ctl --bare --columns $column find $table "$@")

+

+    # Sort the expected and found values.

+    local found=$(for d in $found; do echo $d; done | sort)

+    local expected=$(for d in $expected; do echo $d; done | sort)

+

+    echo

+    echo "Checking values in $db $table${1+ with $*} against $expected... found $found"

+    if test "$found" != "$expected"; then

+        ovn-${db}ctl list $table

+        AT_FAIL_IF([:])

+    fi

+}

+

+# wait_column EXPECTED [DATABASE:]TABLE [COLUMN [CONDITION...]]

+#

+# Wait until all of the values of COLUMN in the rows of TABLE (that

+# satisfy CONDITION) equal EXPECTED (ignoring order).

+#

+# The default DATABASE is "sb".

+#

+# COLUMN defaults to _uuid if unspecified.

+wait_column() {

+    local expected=$(for d in $1; do echo $d; done | sort)

+    local db=$(parse_db $2) table=$(parse_table $2) column=${3-_uuid}; shift; shift; shift

+    local a=$1 b=$2 c=$3

+

+    echo

+    echo "Waiting until $column in $db $table${1+ with $*} is $expected..."

+    OVS_WAIT_UNTIL([

+      found=$(ovn-${db}ctl --bare --columns $column find $table $a $b $c)

+      found=$(for d in $found; do echo $d; done | sort)

+      test "$expected" = "$found"

+    ], [

+      echo "$column in $db table $table has value $found, from the following rows:"

+      ovn-${db}ctl list $table])

+}

+OVS_END_SHELL_HELPERS

+

 m4_define([OVN_POPULATE_ARP], [AT_CHECK(ovn_populate_arp__, [0], [ignore])])

diff --git a/tests/ovn-nbctl.at b/tests/ovn-nbctl.at

index 3dbedc843..79d580d3f 100644

--- a/tests/ovn-nbctl.at

+++ b/tests/ovn-nbctl.at

@@ -536,18 +536,12 @@ snat             30.0.0.1                            192.168.1.0/24

 snat             fd01::1                             fd11::/64

 ])

-AT_CHECK([ovn-nbctl --bare --columns=options list nat | grep stateless=true| wc -l], [0],

-[0

-])

+check_row_count nb:NAT 0 options:stateless=true

 AT_CHECK([ovn-nbctl --stateless lr-nat-add lr0 dnat_and_snat 40.0.0.2 192.168.1.4])

-AT_CHECK([ovn-nbctl --bare --columns=options list nat | grep stateless=true| wc -l], [0],

-[1

-])

+check_row_count nb:NAT 1 options:stateless=true

 AT_CHECK([ovn-nbctl --stateless lr-nat-add lr0 dnat_and_snat fd21::1 fd11::2])

-AT_CHECK([ovn-nbctl --bare --columns=options list nat | grep stateless=true| wc -l], [0],

-[2

-])

+check_row_count nb:NAT 2 options:stateless=true

 AT_CHECK([ovn-nbctl lr-nat-del lr0 dnat_and_snat fd21::1])

diff --git a/tests/ovn-northd.at b/tests/ovn-northd.at

index 94007fcac..49d74b08b 100644

--- a/tests/ovn-northd.at

+++ b/tests/ovn-northd.at

@@ -22,130 +22,47 @@ nb_gwc1_uuid=`ovn-nbctl --bare --columns _uuid find Gateway_Chassis name="alice_

 # With the new ha_chassis_group table added, there should be no rows in

 # gateway_chassis table in SB DB.

-AT_CHECK([ovn-sbctl list gateway_chassis | wc -l], [0], [0

-])

-

-# There should be one ha_chassis_group with the name "alice"

-ha_chassi_grp_name=`ovn-sbctl --bare --columns name find \

-ha_chassis_group name="alice"`

-

-AT_CHECK([test $ha_chassi_grp_name = alice])

-

-ha_chgrp_uuid=`ovn-sbctl --bare --columns _uuid find ha_chassis_group name=alice`

-

-AT_CHECK([ovn-sbctl --bare --columns ha_chassis_group find port_binding \

-logical_port="cr-alice" | grep $ha_chgrp_uuid | wc -l], [0], [1

-])

+check_row_count Gateway_Chassis 0

 # There should be one ha_chassis_group with the name "alice"

-ha_chassi_grp_name=`ovn-sbctl --bare --columns name find \

-ha_chassis_group name="alice"`

-

-AT_CHECK([test $ha_chassi_grp_name = alice])

-

-ha_chgrp_uuid=`ovn-sbctl --bare --columns _uuid find ha_chassis_group name=alice`

-

-AT_CHECK([ovn-sbctl --bare --columns ha_chassis_group find port_binding \

-logical_port="cr-alice" | grep $ha_chgrp_uuid | wc -l], [0], [1

-])

-

-ha_ch=`ovn-sbctl --bare --columns ha_chassis  find ha_chassis_group`

-# Trim the spaces.

-ha_ch=`echo $ha_ch | sed 's/ //g'`

-

-ha_ch_list=''

-for i in `ovn-sbctl --bare --columns _uuid list ha_chassis | sort`

-do

-    ha_ch_list="$ha_ch_list $i"

-done

+check_row_count HA_Chassis_Group 1 name=alice

+ha_chgrp_uuid=$(fetch_column HA_Chassis_Group _uuid name=alice)

+check_row_count Port_Binding 1 logical_port=cr-alice ha_chassis_group=$ha_chgrp_uuid

-# Trim the spaces.

-ha_ch_list=`echo $ha_ch_list | sed 's/ //g'`

-

-AT_CHECK([test "$ha_ch_list" = "$ha_ch"])

+ha_ch=$(fetch_column HA_Chassis_Group ha_chassis name=alice)

+check_column "$ha_ch" HA_Chassis _uuid

 # Delete chassis - gw2 in SB DB.

 # ovn-northd should not recreate ha_chassis rows

 # repeatedly when gw2 is deleted.

 ovn-sbctl chassis-del gw2

-ha_ch_list_1=''

-for i in `ovn-sbctl --bare --columns _uuid list ha_chassis | sort`

-do

-    ha_ch_list_1="$ha_ch_list_1 $i"

-done

-

-# Trim the spaces.

-ha_ch_list_1=`echo $ha_ch_list_1 | sed 's/ //g'`

-

-ha_ch_list_2=''

-for i in `ovn-sbctl --bare --columns _uuid list ha_chassis | sort`

-do

-    ha_ch_list_2="$ha_ch_list_2 $i"

-done

-

-# Trim the spaces.

-ha_ch_list_2=`echo $ha_ch_list_2 | sed 's/ //g'`

-

-AT_CHECK([test "$ha_ch_list_1" = "$ha_ch_list_2"])

+ha_ch_list=$(fetch_column HA_Chassis _uuid)

+check_column "$ha_ch_list" HA_Chassis _uuid

 # Add back the gw2 chassis

 ovn-sbctl chassis-add gw2 geneve 1.2.4.8

 # delete the 2nd Gateway_Chassis on NBDB for alice port

-gw_ch=`ovn-sbctl --bare --columns gateway_chassis find port_binding \

-logical_port="cr-alice"`

-AT_CHECK([test "$gw_ch" = ""])

+check_column '' Port_Binding gateway_chassis logical_port=cr-alice

-ha_ch=`ovn-sbctl --bare --columns ha_chassis  find ha_chassis_group`

-ha_ch=`echo $ha_ch | sed 's/ //g'`

-# Trim the spaces.

-echo "ha ch in grp = $ha_ch"

-

-ha_ch_list=''

-for i in `ovn-sbctl --bare --columns _uuid list ha_chassis | sort`

-do

-    ha_ch_list="$ha_ch_list $i"

-done

-

-# Trim the spaces.

-ha_ch_list=`echo $ha_ch_list | sed 's/ //g'`

-

-AT_CHECK([test "$ha_ch_list" = "$ha_ch"])

+ha_ch=$(fetch_column HA_Chassis_Group ha_chassis)

+check_column "$ha_ch" HA_Chassis _uuid

 # delete the 2nd Gateway_Chassis on NBDB for alice port

 ovn-nbctl --wait=sb set Logical_Router_Port alice gateway_chassis=${nb_gwc1_uuid}

 # There should be only 1 row in ha_chassis SB DB table.

-AT_CHECK([ovn-sbctl --bare --columns _uuid list ha_chassis | wc -l], [0], [1

-])

-

-AT_CHECK([ovn-sbctl list gateway_chassis | wc -l], [0], [0

-])

-

-# There should be only 1 row in ha_chassis SB DB table.

-AT_CHECK([ovn-sbctl --bare --columns _uuid list ha_chassis | wc -l], [0], [1

-])

+check_row_count HA_Chassis 1

+check_row_count Gateway_Chassis 0

 # delete all the gateway_chassis on NBDB for alice port

-

 ovn-nbctl --wait=sb clear Logical_Router_Port alice gateway_chassis

 # expect that the ha_chassis doesn't exist anymore

-AT_CHECK([ovn-sbctl list gateway_chassis | wc -l], [0], [0

-])

-

-AT_CHECK([ovn-sbctl list ha_chassis | wc -l], [0], [0

-])

-

-AT_CHECK([ovn-sbctl list ha_chassis_group | wc -l], [0], [0

-])

-

-# expect that the ha_chassis doesn't exist anymore

-AT_CHECK([ovn-sbctl list ha_chassis | wc -l], [0], [0

-])

-AT_CHECK([ovn-sbctl list ha_chassis_group | wc -l], [0], [0

-])

+check_row_count HA_Chassis 0

+check_row_count Gateway_Chassis 0

+check_row_count Ha_Chassis_Group 0

 AT_CLEANUP

@@ -202,11 +119,11 @@ ovn_start

 ovn-nbctl ls-add S1

 ovn-nbctl --wait=sb lsp-add S1 S1-vm1

-AT_CHECK([test x`ovn-nbctl lsp-get-up S1-vm1` = xdown])

+wait_row_count nb:Logical_Switch_Port 1 name=S1-vm1 'up!=true'

 ovn-sbctl chassis-add hv1 geneve 127.0.0.1

 ovn-sbctl lsp-bind S1-vm1 hv1

-AT_CHECK([test x`ovn-nbctl lsp-get-up S1-vm1` = xup])

+wait_row_count nb:Logical_Switch_Port 1 name=S1-vm1 'up=true'

 AT_CLEANUP

@@ -382,8 +299,7 @@ as northd start_daemon ovn-northd --unixctl="$ovs_base"/northd/ovn-northd.ctl --

 ovn-nbctl ls-add sw

 ovn-nbctl --wait=sb lsp-add sw p1

 # northd created with unixctl option successfully created port_binding entry

-AT_CHECK([ovn-sbctl --bare --columns datapath find port_binding logical_port="p1" | wc -l], [0], [1

-])

+check_row_count Port_Binding 1 logical_port=p1

 AT_CHECK([ovn-nbctl --wait=sb lsp-del p1])

 # ovs-appctl exit with unixctl option

@@ -392,15 +308,13 @@ OVS_APP_EXIT_AND_WAIT_BY_TARGET(["$ovs_base"/northd/ovn-northd.ctl], ["$ovs_base

 # Check no port_binding entry for new port as ovn-northd is not running

 ovn-nbctl lsp-add sw p2

 ovn-nbctl --timeout=10 --wait=sb sync

-AT_CHECK([ovn-sbctl --bare --columns datapath find port_binding logical_port="p2" | wc -l], [0], [0

-])

+check_row_count Port_Binding 0 logical_port=p2

 # test default unixctl path

 as northd start_daemon ovn-northd --ovnnb-db=unix:"$ovs_base"/ovn-nb/ovn-nb.sock --ovnsb-db=unix:"$ovs_base"/ovn-sb/ovn-sb.sock

 ovn-nbctl --wait=sb lsp-add sw p3

 # northd created with default unixctl path successfully created port_binding entry

-AT_CHECK([ovn-sbctl --bare --columns datapath find port_binding logical_port="p3" | wc -l], [0], [1

-])

+check_row_count Port_Binding 1 logical_port=p3

 as ovn-sb

 OVS_APP_EXIT_AND_WAIT([ovsdb-server])

@@ -419,28 +333,22 @@ ovn-nbctl --wait=sb ha-chassis-group-add hagrp1

 # ovn-northd should not create HA chassis group and HA chassis rows

 # unless the HA chassis group in OVN NB DB is associated to

 # a logical router port or logical port of type external.

-AT_CHECK([ovn-sbctl --bare --columns name find ha_chassis_group name="hagrp1" \

-| wc -l], [0], [0

-])

+check_row_count HA_Chassis_Group 0 name=hagrp1

 ovn-nbctl --wait=sb ha-chassis-group-add-chassis hagrp1 ch1 30

 ovn-nbctl --wait=sb ha-chassis-group-add-chassis hagrp1 ch2 20

 ovn-nbctl --wait=sb ha-chassis-group-add-chassis hagrp1 ch3 10

 # There should be no HA_Chassis rows in SB DB.

-AT_CHECK([ovn-sbctl list ha_chassis | grep chassis | awk '{print $3}' \

-| grep -v '-' | wc -l ], [0], [0

-])

+check_row_count HA_Chassis 0

 # Add chassis ch1.

 ovn-sbctl chassis-add ch1 geneve 127.0.0.2

-OVS_WAIT_UNTIL([test 1 = `ovn-sbctl list chassis | grep ch1 | wc -l`])

+wait_row_count Chassis 1 name=ch1

 # There should be no HA_Chassis rows

-AT_CHECK([ovn-sbctl list ha_chassis | grep chassis | awk '{print $3}' \

-| grep -v '-' | wc -l ], [0], [0

-])

+check_row_count HA_Chassis 0

 # Create a logical router port and attach ha chassis group.

 ovn-nbctl lr-add lr0

@@ -449,44 +357,21 @@ ovn-nbctl lrp-add lr0 lr0-public 00:00:20:20:12:13 172.168.0.100/24

 hagrp1_uuid=`ovn-nbctl --bare --columns _uuid find ha_chassis_group name=hagrp1`

 ovn-nbctl set logical_router_port lr0-public ha_chassis_group=$hagrp1_uuid

-OVS_WAIT_UNTIL([test 1 = `ovn-sbctl --bare --columns name find \

-ha_chassis_group name="hagrp1" | wc -l`])

+wait_row_count HA_Chassis_Group 1 name=hagrp1

-AT_CHECK([test 3 = `ovn-sbctl list ha_chassis | grep chassis | \

-grep -v chassis-name | wc -l`])

+check_row_count HA_Chassis 3

 # Make sure that ovn-northd doesn't recreate the ha_chassis

 # records if the chassis record is missing in SB DB.

-

-ha_ch_list_1=''

-for i in `ovn-sbctl --bare --columns _uuid list ha_chassis | sort`

-do

-    ha_ch_list_1="$ha_ch_list_1 $i"

-done

-

-# Trim the spaces.

-ha_ch_list_1=`echo $ha_ch_list_1 | sed 's/ //g'`

-

-ha_ch_list_2=''

-for i in `ovn-sbctl --bare --columns _uuid list ha_chassis | sort`

-do

-    ha_ch_list_2="$ha_ch_list_2 $i"

-done

-

-# Trim the spaces.

-ha_ch_list_2=`echo $ha_ch_list_2 | sed 's/ //g'`

-

-AT_CHECK([test "$ha_ch_list_1" = "$ha_ch_list_2"])

+ha_ch_list=$(fetch_column HA_Chassis _uuid)

+check_column "$ha_ch_list" HA_Chassis _uuid

 # 2 HA chassis should be created with 'chassis' column empty because

 # we have not added hv1 and hv2 chassis to the SB DB.

-AT_CHECK([test 2 = `ovn-sbctl list ha_chassis | grep chassis | awk '{print $3}' \

-| grep -v '-' | wc -l`])

+check_row_count HA_Chassis 2 'chassis=[[]]'

 # We should have 1 ha chassis with 'chassis' column set for hv1

-AT_CHECK([test 1 = `ovn-sbctl list ha_chassis | grep chassis | \

-grep -v chassis-name | awk '{print $3}' \

-| grep '-' | wc -l`])

+check_row_count HA_Chassis 1 'chassis!=[[]]'

 # Create another logical router port and associate to the same ha_chasis_group

 ovn-nbctl lr-add lr1

@@ -495,94 +380,68 @@ ovn-nbctl lrp-add lr1 lr1-public 00:00:20:20:12:14 182.168.0.100/24

 ovn-nbctl set logical_router_port lr1-public ha_chassis_group=$hagrp1_uuid

 # We should still have 1 HA chassis group and 3 HA chassis in SB DB.

-OVS_WAIT_UNTIL([test 1 = `ovn-sbctl --bare --columns name find \

-ha_chassis_group name="hagrp1" | wc -l`])

-

-AT_CHECK([test 3 = `ovn-sbctl list ha_chassis | grep chassis | \

-grep -v chassis-name | wc -l`])

+wait_row_count HA_Chassis_Group 1 name=hagrp1

+check_row_count HA_Chassis 3

 # Change the priority of ch1 - ha chassis in NB DB. It should get

 # reflected in SB DB.

 ovn-nbctl --wait=sb ha-chassis-group-add-chassis hagrp1 ch1 100

-OVS_WAIT_UNTIL([test 1 = `ovn-sbctl --bare --columns priority find \

-ha_chassis | grep 100 | wc -l`])

+wait_row_count HA_Chassis 1 priority=100

 # Delete ch1 HA chassis in NB DB.

 ovn-nbctl --wait=sb ha-chassis-group-remove-chassis hagrp1 ch1

-OVS_WAIT_UNTIL([test 2 = `ovn-sbctl list ha_chassis | grep chassis | \

-grep -v chassis-name | wc -l`])

+wait_row_count HA_Chassis 2

 # Add back the ha chassis

 ovn-nbctl --wait=sb ha-chassis-group-add-chassis hagrp1 ch1 40

-OVS_WAIT_UNTIL([test 3 = `ovn-sbctl list ha_chassis | grep chassis | \

-grep -v chassis-name | wc -l`])

+wait_row_count HA_Chassis 3

 # Delete lr0-public. We should still have 1 HA chassis group and

 # 3 HA chassis in SB DB.

 ovn-nbctl --wait=sb lrp-del lr0-public

-OVS_WAIT_UNTIL([test 1 = `ovn-sbctl --bare --columns name find \

-ha_chassis_group name="hagrp1" | wc -l`])

-

-AT_CHECK([test 3 = `ovn-sbctl list ha_chassis | grep chassis | \

-grep -v chassis-name | wc -l`])

+wait_row_count HA_Chassis_Group 1 name=hagrp1

+wait_row_count HA_Chassis 3

 # Delete lr1-public. There should be no HA chassis group in SB DB.

 ovn-nbctl --wait=sb lrp-del lr1-public

-OVS_WAIT_UNTIL([test 0 = `ovn-sbctl --bare --columns name find \

-ha_chassis_group name="hagrp1" | wc -l`])

-

-AT_CHECK([test 0 = `ovn-sbctl list ha_chassis | grep chassis | wc -l`])

+wait_row_count HA_Chassis_Group 0 name=hagrp1

+wait_row_count HA_Chassis 0

 # Add lr0-public again

 ovn-nbctl lrp-add lr0 lr0-public 00:00:20:20:12:13 172.168.0.100/24

 ovn-nbctl set logical_router_port lr0-public ha_chassis_group=$hagrp1_uuid

-OVS_WAIT_UNTIL([test 1 = `ovn-sbctl --bare --columns name find \

-ha_chassis_group name="hagrp1" | wc -l`])

-

-AT_CHECK([test 3 = `ovn-sbctl list ha_chassis | grep chassis | \

-grep -v chassis-name | wc -l`])

+wait_row_count HA_Chassis_Group 1 name=hagrp1

+wait_row_count HA_Chassis 3

 # Create a Gateway chassis. ovn-northd should ignore this.

 ovn-nbctl lrp-set-gateway-chassis lr0-public ch-1 20

 # There should be only 1 HA chassis group in SB DB with the

 # name hagrp1.

-OVS_WAIT_UNTIL([test 1 = `ovn-sbctl --bare --columns name find \

-ha_chassis_group | wc -l`])

-

-OVS_WAIT_UNTIL([test 1 = `ovn-sbctl --bare --columns name find \

-ha_chassis_group name="hagrp1" | wc -l`])

-

-AT_CHECK([test 3 = `ovn-sbctl list ha_chassis | grep chassis | \

-grep -v chassis-name | wc -l`])

+wait_row_count HA_Chassis_Group 1

+wait_row_count HA_Chassis_Group 1 name=hagrp1

+wait_row_count HA_Chassis 3

 # Now delete HA chassis group. ovn-northd should create HA chassis group

 # with the Gateway chassis name

 ovn-nbctl clear logical_router_port lr0-public ha_chassis_group

 ovn-nbctl ha-chassis-group-del hagrp1

-OVS_WAIT_UNTIL([test 0 = `ovn-sbctl --bare --columns name find \

-ha_chassis_group name="hagrp1" | wc -l`])

-

-OVS_WAIT_UNTIL([test 1 = `ovn-sbctl --bare --columns name find \

-ha_chassis_group name="lr0-public" | wc -l`])

-

-OVS_WAIT_UNTIL([test 1 = `ovn-sbctl --bare --columns _uuid \

-find ha_chassis | wc -l`])

+wait_row_count HA_Chassis_Group 0 name=hagrp1

+wait_row_count HA_Chassis_Group 1 name=lr0-public

+wait_row_count HA_Chassis 1

 ovn-nbctl lrp-set-gateway-chassis lr0-public ch2 10

-OVS_WAIT_UNTIL([test 1 = `ovn-sbctl --bare --columns name find \

-ha_chassis_group name="lr0-public" | wc -l`])

+wait_row_count HA_Chassis_Group 1 name=lr0-public

 ovn-sbctl --bare --columns _uuid find ha_chassis

-OVS_WAIT_UNTIL([test 2 = `ovn-sbctl list ha_chassis | grep chassis | \

-grep -v chassis-name | wc -l`])

+wait_row_count HA_Chassis 2

 # Test if 'ref_chassis' column is properly set or not in

 # SB DB ha_chassis_group.

@@ -601,35 +460,23 @@ ovn-nbctl lsp-set-addresses sw0-lr0 router

 ovn-nbctl lsp-set-options sw0-lr0 router-port=lr0-sw0

 ovn-sbctl lsp-bind sw0-p1 comp1

-OVS_WAIT_UNTIL([test x`ovn-nbctl lsp-get-up sw0-p1` = xup])

+wait_row_count nb:Logical_Switch_Port 1 name=sw0-p1 up=true

-comp1_ch_uuid=`ovn-sbctl --bare --columns _uuid find chassis name="comp1"`

-comp2_ch_uuid=`ovn-sbctl --bare --columns _uuid find chassis name="comp2"`

-ch2_ch_uuid=`ovn-sbctl --bare --columns _uuid find chassis name="comp1"`

+comp1_ch_uuid=$(fetch_column Chassis _uuid name=comp1)

+comp2_ch_uuid=$(fetch_column Chassis _uuid name=comp2)

+ch2_ch_uuid=$comp1_ch_uuid

 echo "comp1_ch_uuid = $comp1_ch_uuid"

-OVS_WAIT_UNTIL(

-    [ref_ch_list=`ovn-sbctl --bare --columns ref_chassis find ha_chassis_group | sort`

-     # Trim the spaces.

-     ref_ch_list=`echo $ref_ch_list | sed 's/ //g'`

-     test "$comp1_ch_uuid" = "$ref_ch_list"])

+wait_column "$comp1_ch_uuid" HA_Chassis_Group ref_chassis

 # unbind sw0-p1

 ovn-sbctl lsp-unbind sw0-p1

-OVS_WAIT_UNTIL([test x`ovn-nbctl lsp-get-up sw0-p1` = xdown])

-OVS_WAIT_UNTIL(

-    [ref_ch_list=`ovn-sbctl --bare --columns ref_chassis find ha_chassis_group | sort`

-     # Trim the spaces.

-     ref_ch_list=`echo $ref_ch_list | sed 's/ //g'`

-     test "" = "$ref_ch_list"])

+wait_row_count nb:Logical_Switch_Port 1 name=sw0-p1 up=false

+wait_column "" HA_Chassis_Group ref_chassis

 # Bind sw0-p1 in comp2

 ovn-sbctl lsp-bind sw0-p1 comp2

-OVS_WAIT_UNTIL(

-    [ref_ch_list=`ovn-sbctl --bare --columns ref_chassis find ha_chassis_group | sort`

-     # Trim the spaces.

-     ref_ch_list=`echo $ref_ch_list | sed 's/ //g'`

-     test "$comp2_ch_uuid" = "$ref_ch_list"])

+wait_column "$comp2_ch_uuid" HA_Chassis_Group ref_chassis

 ovn-nbctl ls-add sw1

 ovn-nbctl lsp-add sw1 sw1-p1

@@ -643,14 +490,10 @@ ovn-nbctl lsp-set-options sw1-lr1 router-port=lr1-sw1

 # Bind sw1-p1 in comp1.

 ovn-sbctl lsp-bind sw1-p1 comp1

 # Wait until sw1-p1 is up

-OVS_WAIT_UNTIL([test x`ovn-nbctl lsp-get-up sw1-p1` = xup])

+wait_row_count nb:Logical_Switch_Port 1 name=sw1-p1 up=true

 # sw1-p1 is not connected to lr0. So comp1 should not be in 'ref_chassis'

-OVS_WAIT_UNTIL(

-    [ref_ch_list=`ovn-sbctl --bare --columns ref_chassis find ha_chassis_group | sort`

-     # Trim the spaces.

-     ref_ch_list=`echo $ref_ch_list | sed 's/ //g'`

-     test "$comp2_ch_uuid" = "$ref_ch_list"])

+wait_column "$comp2_ch_uuid" HA_Chassis_Group ref_chassis

 # Now attach sw0 to lr1

 ovn-nbctl lrp-add lr1 lr1-sw0 00:00:20:20:12:16 10.0.0.10/24

@@ -661,30 +504,14 @@ ovn-nbctl lsp-set-options sw0-lr1 router-port=lr1-sw0

 # Both comp1 and comp2 should be in 'ref_chassis' as sw1 is indirectly

 # connected to lr0

-exp_ref_ch_list=''