233933
From 48f7be493588fdf5e99dff0c3b91327e07da05f3 Mon Sep 17 00:00:00 2001
233933
From: Mohammed Rafi KC <rkavunga@redhat.com>
233933
Date: Thu, 11 Jul 2019 12:48:34 +0530
233933
Subject: [PATCH 242/255] Revert "tests/shd: Add test coverage for shd mux"
233933
233933
This reverts commit b7f832288d2d2e57231d90765afc049ad7cb2f9d.
233933
233933
BUG: 1471742
233933
Change-Id: Ifccac5150f07b98006714e43c77c5a4b1fd38cb8
233933
Signed-off-by: Mohammed Rafi KC <rkavunga@redhat.com>
233933
Reviewed-on: https://code.engineering.redhat.com/gerrit/175951
233933
Tested-by: RHGS Build Bot <nigelb@redhat.com>
233933
Reviewed-by: Sunil Kumar Heggodu Gopala Acharya <sheggodu@redhat.com>
233933
---
233933
 tests/basic/glusterd-restart-shd-mux.t |  96 ---------------------
233933
 tests/basic/shd-mux.t                  | 149 ---------------------------------
233933
 tests/basic/volume-scale-shd-mux.t     | 112 -------------------------
233933
 tests/volume.rc                        |  15 ----
233933
 4 files changed, 372 deletions(-)
233933
 delete mode 100644 tests/basic/glusterd-restart-shd-mux.t
233933
 delete mode 100644 tests/basic/shd-mux.t
233933
 delete mode 100644 tests/basic/volume-scale-shd-mux.t
233933
233933
diff --git a/tests/basic/glusterd-restart-shd-mux.t b/tests/basic/glusterd-restart-shd-mux.t
233933
deleted file mode 100644
233933
index a50af9d..0000000
233933
--- a/tests/basic/glusterd-restart-shd-mux.t
233933
+++ /dev/null
233933
@@ -1,96 +0,0 @@
233933
-#!/bin/bash
233933
-
233933
-. $(dirname $0)/../include.rc
233933
-. $(dirname $0)/../volume.rc
233933
-
233933
-cleanup;
233933
-
233933
-TESTS_EXPECTED_IN_LOOP=20
233933
-
233933
-TEST glusterd
233933
-TEST pidof glusterd
233933
-TEST $CLI volume create $V0 replica 3 $H0:$B0/${V0}{0,1,2,3,4,5}
233933
-TEST $CLI volume set $V0 cluster.background-self-heal-count 0
233933
-TEST $CLI volume set $V0 cluster.eager-lock off
233933
-TEST $CLI volume set $V0 performance.flush-behind off
233933
-TEST $CLI volume start $V0
233933
-
233933
-for i in $(seq 1 3); do
233933
-   TEST $CLI volume create ${V0}_afr$i replica 3 $H0:$B0/${V0}_afr${i}{0,1,2,3,4,5}
233933
-   TEST $CLI volume start ${V0}_afr$i
233933
-   TEST $CLI volume create ${V0}_ec$i disperse 6 redundancy 2 $H0:$B0/${V0}_ec${i}{0,1,2,3,4,5}
233933
-   TEST $CLI volume start ${V0}_ec$i
233933
-done
233933
-
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^1$" shd_count
233933
-
233933
-#Stop the glusterd
233933
-TEST pkill glusterd
233933
-#Only stopping glusterd, so there will be one shd
233933
-EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT "^1$" shd_count
233933
-TEST glusterd
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^1$" shd_count
233933
-#Check the thread count become to number of volumes*number of ec subvolume (3*6=18)
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^18$" number_healer_threads_shd $V0 "__ec_shd_healer_wait"
233933
-#Check the thread count become to number of volumes*number of afr subvolume (4*6=24)
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^24$" number_healer_threads_shd $V0 "__afr_shd_healer_wait"
233933
-
233933
-shd_pid=$(get_shd_mux_pid $V0)
233933
-for i in $(seq 1 3); do
233933
-    afr_path="/var/run/gluster/shd/${V0}_afr$i/${V0}_afr$i-shd.pid"
233933
-    EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^${shd_pid}$" cat $afr_path
233933
-    ec_path="/var/run/gluster/shd/${V0}_ec$i/${V0}_ec${i}-shd.pid"
233933
-    EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^${shd_pid}$" cat $ec_path
233933
-done
233933
-
233933
-#Reboot a node scenario
233933
-TEST pkill gluster
233933
-#Only stopped glusterd, so there will be one shd
233933
-EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT "^0$" shd_count
233933
-
233933
-TEST glusterd
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^1$" shd_count
233933
-
233933
-#Check the thread count become to number of volumes*number of ec subvolume (3*6=18)
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^18$" number_healer_threads_shd $V0 "__ec_shd_healer_wait"
233933
-#Check the thread count become to number of volumes*number of afr subvolume (4*6=24)
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^24$" number_healer_threads_shd $V0 "__afr_shd_healer_wait"
233933
-
233933
-shd_pid=$(get_shd_mux_pid $V0)
233933
-for i in $(seq 1 3); do
233933
-    afr_path="/var/run/gluster/shd/${V0}_afr$i/${V0}_afr$i-shd.pid"
233933
-    EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^${shd_pid}$" cat $afr_path
233933
-    ec_path="/var/run/gluster/shd/${V0}_ec$i/${V0}_ec${i}-shd.pid"
233933
-    EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^${shd_pid}$" cat $ec_path
233933
-done
233933
-
233933
-for i in $(seq 1 3); do
233933
-   TEST $CLI volume stop ${V0}_afr$i
233933
-   TEST $CLI volume stop ${V0}_ec$i
233933
-done
233933
-
233933
-EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT "^6$" number_healer_threads_shd $V0 "__afr_shd_healer_wait"
233933
-
233933
-TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0
233933
-
233933
-TEST kill_brick $V0 $H0 $B0/${V0}0
233933
-TEST kill_brick $V0 $H0 $B0/${V0}3
233933
-
233933
-TEST touch $M0/foo{1..100}
233933
-
233933
-EXPECT_WITHIN $HEAL_TIMEOUT "^204$" get_pending_heal_count $V0
233933
-
233933
-TEST $CLI volume start ${V0} force
233933
-
233933
-EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0
233933
-
233933
-TEST rm -rf $M0/*
233933
-EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
233933
-
233933
-
233933
-TEST $CLI volume stop ${V0}
233933
-TEST $CLI volume delete ${V0}
233933
-
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^0$" shd_count
233933
-
233933
-cleanup
233933
diff --git a/tests/basic/shd-mux.t b/tests/basic/shd-mux.t
233933
deleted file mode 100644
233933
index e42a34a..0000000
233933
--- a/tests/basic/shd-mux.t
233933
+++ /dev/null
233933
@@ -1,149 +0,0 @@
233933
-#!/bin/bash
233933
-
233933
-. $(dirname $0)/../include.rc
233933
-. $(dirname $0)/../volume.rc
233933
-
233933
-cleanup;
233933
-
233933
-TESTS_EXPECTED_IN_LOOP=16
233933
-
233933
-TEST glusterd
233933
-TEST pidof glusterd
233933
-TEST $CLI volume create $V0 replica 3 $H0:$B0/${V0}{0,1,2,3,4,5}
233933
-TEST $CLI volume set $V0 cluster.background-self-heal-count 0
233933
-TEST $CLI volume set $V0 cluster.eager-lock off
233933
-TEST $CLI volume set $V0 performance.flush-behind off
233933
-TEST $CLI volume start $V0
233933
-TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0
233933
-
233933
-shd_pid=$(get_shd_mux_pid $V0)
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^6$" number_healer_threads_shd $V0 "__afr_shd_healer_wait"
233933
-
233933
-#Create a one more volume
233933
-TEST $CLI volume create ${V0}_1 replica 3 $H0:$B0/${V0}_1{0,1,2,3,4,5}
233933
-TEST $CLI volume start ${V0}_1
233933
-
233933
-#Check whether the shd has multiplexed or not
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^1$" shd_count
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^${shd_pid}$" get_shd_mux_pid ${V0}_1
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^${shd_pid}$" get_shd_mux_pid ${V0}
233933
-
233933
-TEST $CLI volume set ${V0}_1 cluster.background-self-heal-count 0
233933
-TEST $CLI volume set ${V0}_1 cluster.eager-lock off
233933
-TEST $CLI volume set ${V0}_1 performance.flush-behind off
233933
-TEST $GFS --volfile-id=/${V0}_1 --volfile-server=$H0 $M1
233933
-
233933
-TEST kill_brick $V0 $H0 $B0/${V0}0
233933
-TEST kill_brick $V0 $H0 $B0/${V0}4
233933
-TEST kill_brick ${V0}_1 $H0 $B0/${V0}_10
233933
-TEST kill_brick ${V0}_1 $H0 $B0/${V0}_14
233933
-
233933
-TEST touch $M0/foo{1..100}
233933
-TEST touch $M1/foo{1..100}
233933
-
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^204$" get_pending_heal_count $V0
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^204$" get_pending_heal_count ${V0}_1
233933
-
233933
-TEST $CLI volume start ${V0} force
233933
-TEST $CLI volume start ${V0}_1 force
233933
-
233933
-EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0
233933
-EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count ${V0}_1
233933
-
233933
-TEST rm -rf $M0/*
233933
-TEST rm -rf $M1/*
233933
-EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M1
233933
-
233933
-#Stop the volume
233933
-TEST $CLI volume stop ${V0}_1
233933
-TEST $CLI volume delete ${V0}_1
233933
-
233933
-#Check the stop succeeded and detached the volume with out restarting it
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^1$" shd_count
233933
-
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^${shd_pid}$" get_shd_mux_pid $V0
233933
-
233933
-#Check the thread count become to earlier number after stopping
233933
-EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT "^6$" number_healer_threads_shd $V0 "__afr_shd_healer_wait"
233933
-
233933
-
233933
-#Now create a  ec volume and check mux works
233933
-TEST $CLI volume create ${V0}_2 disperse 6 redundancy 2 $H0:$B0/${V0}_2{0,1,2,3,4,5}
233933
-TEST $CLI volume start ${V0}_2
233933
-
233933
-#Check whether the shd has multiplexed or not
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^1$" shd_count
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^${shd_pid}$" get_shd_mux_pid ${V0}_2
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^${shd_pid}$" get_shd_mux_pid ${V0}
233933
-
233933
-TEST $CLI volume set ${V0}_2 cluster.background-self-heal-count 0
233933
-TEST $CLI volume set ${V0}_2 cluster.eager-lock off
233933
-TEST $CLI volume set ${V0}_2 performance.flush-behind off
233933
-TEST $GFS --volfile-id=/${V0}_2 --volfile-server=$H0 $M1
233933
-
233933
-TEST kill_brick $V0 $H0 $B0/${V0}0
233933
-TEST kill_brick $V0 $H0 $B0/${V0}4
233933
-TEST kill_brick ${V0}_2 $H0 $B0/${V0}_20
233933
-TEST kill_brick ${V0}_2 $H0 $B0/${V0}_22
233933
-
233933
-TEST touch $M0/foo{1..100}
233933
-TEST touch $M1/foo{1..100}
233933
-
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^204$" get_pending_heal_count $V0
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^404$" get_pending_heal_count ${V0}_2
233933
-
233933
-TEST $CLI volume start ${V0} force
233933
-TEST $CLI volume start ${V0}_2 force
233933
-
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^6$" number_healer_threads_shd $V0 "__ec_shd_healer_wait"
233933
-
233933
-EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0
233933
-EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count ${V0}_2
233933
-
233933
-TEST rm -rf $M0/*
233933
-TEST rm -rf $M1/*
233933
-
233933
-
233933
-#Stop the volume
233933
-TEST $CLI volume stop ${V0}_2
233933
-TEST $CLI volume delete ${V0}_2
233933
-
233933
-#Check the stop succeeded and detached the volume with out restarting it
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^1$" shd_count
233933
-
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^${shd_pid}$" get_shd_mux_pid $V0
233933
-
233933
-#Check the thread count become to zero for ec related threads
233933
-EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT "^0$" number_healer_threads_shd $V0 "__ec_shd_healer_wait"
233933
-#Check the thread count become to earlier number after stopping
233933
-EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT "^6$" number_healer_threads_shd $V0 "__afr_shd_healer_wait"
233933
-
233933
-for i in $(seq 1 3); do
233933
-   TEST $CLI volume create ${V0}_afr$i replica 3 $H0:$B0/${V0}_afr${i}{0,1,2,3,4,5}
233933
-   TEST $CLI volume start ${V0}_afr$i
233933
-   TEST $CLI volume create ${V0}_ec$i disperse 6 redundancy 2 $H0:$B0/${V0}_ec${i}{0,1,2,3,4,5}
233933
-   TEST $CLI volume start ${V0}_ec$i
233933
-done
233933
-
233933
-#Check the thread count become to number of volumes*number of ec subvolume (3*6=18)
233933
-EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT "^18$" number_healer_threads_shd $V0 "__ec_shd_healer_wait"
233933
-#Check the thread count become to number of volumes*number of afr subvolume (4*6=24)
233933
-EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT "^24$" number_healer_threads_shd $V0 "__afr_shd_healer_wait"
233933
-#Delete the volumes
233933
-for i in $(seq 1 3); do
233933
-   TEST $CLI volume stop ${V0}_afr$i
233933
-   TEST $CLI volume stop ${V0}_ec$i
233933
-   TEST $CLI volume delete ${V0}_afr$i
233933
-   TEST $CLI volume delete ${V0}_ec$i
233933
-done
233933
-
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^${shd_pid}$" get_shd_mux_pid $V0
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^1$" shd_count
233933
-
233933
-EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT "^6$" number_healer_threads_shd $V0 "__afr_shd_healer_wait"
233933
-
233933
-TEST $CLI volume stop ${V0}
233933
-TEST $CLI volume delete ${V0}
233933
-EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT "^0$" shd_count
233933
-
233933
-cleanup
233933
diff --git a/tests/basic/volume-scale-shd-mux.t b/tests/basic/volume-scale-shd-mux.t
233933
deleted file mode 100644
233933
index dd9cf83..0000000
233933
--- a/tests/basic/volume-scale-shd-mux.t
233933
+++ /dev/null
233933
@@ -1,112 +0,0 @@
233933
-#!/bin/bash
233933
-
233933
-. $(dirname $0)/../include.rc
233933
-. $(dirname $0)/../volume.rc
233933
-
233933
-cleanup;
233933
-
233933
-TESTS_EXPECTED_IN_LOOP=6
233933
-
233933
-TEST glusterd
233933
-TEST pidof glusterd
233933
-TEST $CLI volume create $V0 replica 3 $H0:$B0/${V0}{0,1,2,3,4,5}
233933
-TEST $CLI volume set $V0 cluster.background-self-heal-count 0
233933
-TEST $CLI volume set $V0 cluster.eager-lock off
233933
-TEST $CLI volume set $V0 performance.flush-behind off
233933
-TEST $CLI volume start $V0
233933
-
233933
-for i in $(seq 1 2); do
233933
-   TEST $CLI volume create ${V0}_afr$i replica 3 $H0:$B0/${V0}_afr${i}{0,1,2,3,4,5}
233933
-   TEST $CLI volume start ${V0}_afr$i
233933
-   TEST $CLI volume create ${V0}_ec$i disperse 6 redundancy 2 $H0:$B0/${V0}_ec${i}{0,1,2,3,4,5}
233933
-   TEST $CLI volume start ${V0}_ec$i
233933
-done
233933
-
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^1$" shd_count
233933
-
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^1$" shd_count
233933
-#Check the thread count become to number of volumes*number of ec subvolume (2*6=12)
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^12$" number_healer_threads_shd $V0 "__ec_shd_healer_wait"
233933
-#Check the thread count become to number of volumes*number of afr subvolume (3*6=18)
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^18$" number_healer_threads_shd $V0 "__afr_shd_healer_wait"
233933
-
233933
-TEST $CLI volume add-brick $V0 replica 3 $H0:$B0/${V0}{6,7,8};
233933
-#Check the thread count become to number of volumes*number of afr subvolume plus 3 additional threads from newly added bricks (3*6+3=21)
233933
-
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^1$" shd_count
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^21$" number_healer_threads_shd $V0 "__afr_shd_healer_wait"
233933
-
233933
-#Remove the brick and check the detach is successful
233933
-$CLI volume remove-brick $V0 $H0:$B0/${V0}{6,7,8} force
233933
-
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^18$" number_healer_threads_shd $V0 "__afr_shd_healer_wait"
233933
-
233933
-TEST $CLI volume add-brick ${V0}_ec1 $H0:$B0/${V0}_ec1_add{0,1,2,3,4,5};
233933
-#Check the thread count become to number of volumes*number of ec subvolume plus 2 additional threads from newly added bricks (2*6+6=18)
233933
-
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^1$" shd_count
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^18$" number_healer_threads_shd $V0 "__ec_shd_healer_wait"
233933
-
233933
-#Remove the brick and check the detach is successful
233933
-$CLI volume remove-brick ${V0}_ec1 $H0:$B0/${V0}_ec1_add{0,1,2,3,4,5} force
233933
-
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^12$" number_healer_threads_shd $V0 "__ec_shd_healer_wait"
233933
-
233933
-
233933
-for i in $(seq 1 2); do
233933
-   TEST $CLI volume stop ${V0}_afr$i
233933
-   TEST $CLI volume stop ${V0}_ec$i
233933
-done
233933
-
233933
-EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT "^6$" number_healer_threads_shd $V0 "__afr_shd_healer_wait"
233933
-
233933
-TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0
233933
-
233933
-TEST kill_brick $V0 $H0 $B0/${V0}0
233933
-TEST kill_brick $V0 $H0 $B0/${V0}4
233933
-
233933
-TEST touch $M0/foo{1..100}
233933
-
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^204$" get_pending_heal_count $V0
233933
-
233933
-TEST $CLI volume start ${V0} force
233933
-
233933
-EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0
233933
-
233933
-TEST rm -rf $M0/*
233933
-EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
233933
-shd_pid=$(get_shd_mux_pid $V0)
233933
-TEST $CLI volume create ${V0}_distribute1 $H0:$B0/${V0}_distribute10
233933
-TEST $CLI volume start ${V0}_distribute1
233933
-
233933
-#Creating a non-replicate/non-ec volume should not have any effect in shd
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^6$" number_healer_threads_shd $V0 "__afr_shd_healer_wait"
233933
-EXPECT "^${shd_pid}$" get_shd_mux_pid $V0
233933
-
233933
-TEST mkdir $B0/add/
233933
-#Now convert the distributed volume to replicate
233933
-TEST $CLI volume add-brick ${V0}_distribute1 replica 3 $H0:$B0/add/{2..3}
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^9$" number_healer_threads_shd $V0 "__afr_shd_healer_wait"
233933
-
233933
-#scale down the volume
233933
-TEST $CLI volume remove-brick ${V0}_distribute1 replica 1 $H0:$B0/add/{2..3} force
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^6$" number_healer_threads_shd $V0 "__afr_shd_healer_wait"
233933
-
233933
-TEST $CLI volume stop ${V0}
233933
-TEST $CLI volume delete ${V0}
233933
-EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT "^0$" shd_count
233933
-
233933
-TEST rm -rf $B0/add/
233933
-TEST mkdir $B0/add/
233933
-#Now convert the distributed volume back to replicate and make sure that a new shd is spawned
233933
-TEST $CLI volume add-brick ${V0}_distribute1 replica 3 $H0:$B0/add/{2..3};
233933
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^1$" shd_count
233933
-EXPECT_WITHIN $HEAL_TIMEOUT "^3$" number_healer_threads_shd ${V0}_distribute1 "__afr_shd_healer_wait"
233933
-
233933
-#Now convert the replica volume to distribute again and make sure the shd is now stopped
233933
-TEST $CLI volume remove-brick ${V0}_distribute1 replica 1 $H0:$B0/add/{2..3} force
233933
-TEST rm -rf $B0/add/
233933
-
233933
-EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT "^0$" shd_count
233933
-
233933
-cleanup
233933
diff --git a/tests/volume.rc b/tests/volume.rc
233933
index 6a78c37..022d972 100644
233933
--- a/tests/volume.rc
233933
+++ b/tests/volume.rc
233933
@@ -913,21 +913,6 @@ function volgen_check_ancestry {
233933
         fi
233933
 }
233933
 
233933
-function get_shd_mux_pid {
233933
-   local volume=$1
233933
-   pid=`$CLI volume status $volume shd | awk '/Self-heal/{print $8}'`
233933
-   echo $pid
233933
-}
233933
-
233933
-function shd_count {
233933
-   ps aux | grep "glustershd" | grep -v grep | wc -l
233933
-}
233933
-
233933
-function number_healer_threads_shd {
233933
-   local pid=$(get_shd_mux_pid $1)
233933
-   pstack $pid | grep $2 | wc -l
233933
-}
233933
-
233933
 function get_mtime {
233933
     local time=$(get-mdata-xattr -m $1)
233933
     if [ $time == "-1" ];
233933
-- 
233933
1.8.3.1
233933