From 2e072bb19d47080ca91c181698cfcf1beb60b8b1 Mon Sep 17 00:00:00 2001

From: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>

Date: Fri, 16 Sep 2016 13:08:00 +0530

Subject: [PATCH 37/86] extras: kill processes gracefully

Currently all of brick, gsyncd and other glustershd services/processes

are killed without checking for ongoing tasks such as geo-rep,

self-heal, rebalance and etc. which may lead to inconsistency after

the node is brought back.

This patch introduce an option '-g' which ensures whether all the

gluster processes are ready (not busy) to be terminated before we

executing 'kill' on them

Usage: ./extras/stop-all-gluster-processes.sh [-g] [-h]

    options:

    -g  Terminate in graceful mode

    -h  Show this message, then exit

 eg:

  1. ./extras/stop-all-gluster-processes.sh

  2. ./extras/stop-all-gluster-processes.sh -g

By default, this script executes in force mode, processes are killed

without checking for ongoing tasks, on specifying '-g' option this

script works in graceful mode, which returns exitcode if some of gluster

processes are busy in doing their jobs.

exitcodes include:

   0       No errors/Success

   64      Rebalance is in progress

   65      Self-Heal is in progress

   66      Tier daemon running on this node

   127     option not found

Backport of:

> Change-Id: I2f924b2bf9f04a81788d0f5604895a42755b33a1

> BUG: 1367771

> Signed-off-by: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>

> Reviewed-on: http://review.gluster.org/15188

> Tested-by: Prasanna Kumar Kalever <pkalever@redhat.com>

> Smoke: Gluster Build System <jenkins@build.gluster.org>

> NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>

> Reviewed-by: Rajesh Joseph <rjoseph@redhat.com>

> CentOS-regression: Gluster Build System <jenkins@build.gluster.org>

> Reviewed-by: Ravishankar N <ravishankar@redhat.com>

> Reviewed-by: mohammed rafi  kc <rkavunga@redhat.com>

> Reviewed-by: Atin Mukherjee <amukherj@redhat.com>

Change-Id: I2f924b2bf9f04a81788d0f5604895a42755b33a1

BUG: 1294754

Signed-off-by: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>

Reviewed-on: https://code.engineering.redhat.com/gerrit/84768

Reviewed-by: Atin Mukherjee <amukherj@redhat.com>

---

 extras/stop-all-gluster-processes.sh |  111 +++++++++++++++++++++++++++++++++-

 1 files changed, 110 insertions(+), 1 deletions(-)

diff --git a/extras/stop-all-gluster-processes.sh b/extras/stop-all-gluster-processes.sh

index 356a2a6..f11c634 100755

--- a/extras/stop-all-gluster-processes.sh

+++ b/extras/stop-all-gluster-processes.sh

@@ -1,4 +1,37 @@

 #!/usr/bin/env bash

+#

+# Kill all the processes/services except glusterd

+#

+# Usage: ./extras/stop-all-gluster-processes.sh [-g] [-h]

+#    options:

+#    -g  Terminate in graceful mode

+#    -h  Show this message, then exit

+#

+# eg:

+#  1. ./extras/stop-all-gluster-processes.sh

+#  2. ./extras/stop-all-gluster-processes.sh -g

+#

+# By default, this script executes in force mode, i.e. all of brick, gsyncd

+# and other glustershd services/processes are killed without checking for

+# ongoing tasks such as geo-rep, self-heal, rebalance and etc. which may lead

+# to inconsistency after the node is brought back.

+#

+# On specifying '-g' option this script works in graceful mode, to maintain

+# data consistency the script fails with a valid exit code incase if any of

+# the gluster processes are busy in doing their jobs.

+#

+# The author of page [1] proposes user-defined exit codes to the range 64 - 113

+# Find the better explanation behind the choice in the link

+#

+# The exit code returned by stop-all-gluster-processes.sh:

+#   0       No errors/Success

+#   64      Rebalance is in progress

+#   65      Self-Heal is in progress

+#   66      Tier daemon running on this node

+#   127     option not found

+#

+# [1] http://www.tldp.org/LDP/abs/html/exitcodes.html

+

 # global

 errors=0

@@ -64,8 +97,84 @@ kill_georep_gsync()

     fi

 }

+# check if all processes are ready to die

+check_background_tasks()

+{

+    volumes=$(gluster vol list)

+    quit=0

+    for volname in ${volumes};

+    do

+        # tiering

+        if [[ $(gluster volume tier ${volname} status 2> /dev/null |

+                grep "localhost" | grep -c "in progress") -gt 0 ]]

+        then

+            quit=66

+            break;

+        fi

+

+        # rebalance

+        if [[ $(gluster volume rebalance ${volname} status 2> /dev/null |

+                grep -c "in progress") -gt 0 ]]

+        then

+            quit=64

+            break;

+        fi

+

+        # self heal

+        if [[ $(gluster volume heal ${volname} info | grep "Number of entries" |

+                awk '{ sum+=$4} END {print sum}') -gt 0 ]];

+        then

+            quit=65

+            break;

+        fi

+

+        # geo-rep, snapshot and quota doesn't need grace checks,

+        # as they ensures the consistancy on force kills

+    done

+

+    echo ${quit}

+}

+

+usage()

+{

+    cat <

+Usage: $0 [-g] [-h]

+    options:

+    -g  Terminate in graceful mode

+    -h  Show this message, then exit

+

+eg:

+ 1. $0

+ 2. $0 -g

+EOM

+}

+

 main()

 {

+    while getopts "gh" opt; do

+        case $opt in

+            g)

+                # graceful mode

+                quit=$(check_background_tasks)

+                if [[ ${quit} -ne 0 ]]

+                then

+                    exit ${quit};

+                fi

+                # else safe to kill

+                ;;

+            h)

+                usage

+                exit 0;

+                ;;

+            *)

+                usage

+                exit 127;

+                ;;

+        esac

+    done

+    # remove all the options that have been parsed by getopts

+    shift $((OPTIND-1))

+

     kill_mounts TERM

     kill_bricks_and_services TERM

     kill_georep_gsync TERM

@@ -81,4 +190,4 @@ main()

     exit ${errors};

 }

-main

+main "$@"

-- 

1.7.1