From 7554d75e97c36abe2121cca41d98a6e57f3d9640 Mon Sep 17 00:00:00 2001

From: Gaurav Kumar Garg <garg.gaurav52@gmail.com>

Date: Wed, 2 Mar 2016 17:42:07 +0530

Subject: [PATCH 602/602] glusterd: Gluster should keep PID file in correct

 location

Currently Gluster keeps process pid information of all the daemons

and brick processes in Gluster configuration file directory

(ie., /var/lib/glusterd/*).

These pid files should be seperate from configuration files.

Deletion of the configuration file directory might result into serious problems.

Also, /var/run/gluster is the default placeholder directory for pid files.

So, with this fix Gluster will keep all process pid information of all

processes in /var/run/gluster/* directory.

>Signed-off-by: Saravanakumar Arumugam <sarumuga@redhat.com>

>Reviewed-on: https://review.gluster.org/13580

>Tested-by: MOHIT AGRAWAL <moagrawa@redhat.com>

>Smoke: Gluster Build System <jenkins@build.gluster.org>

>CentOS-regression: Gluster Build System <jenkins@build.gluster.org>

>Reviewed-by: Atin Mukherjee <amukherj@redhat.com>

Change-Id: Idb09e3fccb6a7355fbac1df31082637c8d7ab5b4

BUG: 1480423

Signed-off-by: Gaurav Kumar Garg <ggarg@redhat.com>

Reviewed-on: https://code.engineering.redhat.com/gerrit/115068

Reviewed-by: Atin Mukherjee <amukherj@redhat.com>

Tested-by: Atin Mukherjee <amukherj@redhat.com>

---

 extras/ocf/volume.in                               | 18 +++--

 extras/stop-all-gluster-processes.sh               |  2 +-

 libglusterfs/src/store.c                           |  2 +-

 tests/basic/mount-nfs-auth.t                       |  2 +-

 tests/bugs/fuse/bug-858215.t                       |  4 +-

 ...-1225716-brick-online-validation-remove-brick.t |  1 -

 tests/bugs/glusterd/bug-916549.t                   |  4 +-

 tests/bugs/glusterfs-server/bug-861542.t           |  2 +-

 tests/bugs/snapshot/bug-1399598-uss-with-ssl.t     |  2 +-

 tests/bugs/trace/bug-797171.t                      |  4 +-

 tests/cluster.rc                                   | 10 +--

 tests/env.rc.in                                    |  3 +

 tests/include.rc                                   |  1 +

 tests/volume.rc                                    |  8 +--

 .../mgmt/glusterd/src/glusterd-snapd-svc-helper.c  |  5 +-

 xlators/mgmt/glusterd/src/glusterd-store.c         | 34 ++++++++++

 xlators/mgmt/glusterd/src/glusterd-svc-mgmt.c      |  9 +--

 xlators/mgmt/glusterd/src/glusterd-utils.c         | 14 ++--

 xlators/mgmt/glusterd/src/glusterd.c               | 77 ++++++++++++++++++++++

 xlators/mgmt/glusterd/src/glusterd.h               | 37 ++++++++---

 20 files changed, 190 insertions(+), 49 deletions(-)

diff --git a/extras/ocf/volume.in b/extras/ocf/volume.in

index 72fd121..de05373 100755

--- a/extras/ocf/volume.in

+++ b/extras/ocf/volume.in

@@ -78,6 +78,16 @@ volume_getdir() {

     return 0

 }

+volume_getpid_dir() {

+    local volpid_dir

+    volpid_dir="/var/run/gluster/vols/${OCF_RESKEY_volname}"

+

+    [ -d ${volpid_dir} ] || return 1

+

+    echo "${volpid_dir}"

+    return 0

+}

+

 volume_getbricks() {

     local infofile

     local voldir

@@ -92,17 +102,15 @@ volume_getbricks() {

 volume_getpids() {

     local bricks

-    local piddir

     local pidfile

     local infofile

-    local voldir

+    local volpid_dir

-    voldir=`volume_getdir`

+    volpid_dir=`volume_getpid_dir`

     bricks=`volume_getbricks`

-    piddir="${voldir}/run"

     for brick in ${bricks}; do

-	pidfile="${piddir}/${SHORTHOSTNAME}${brick}.pid"

+	pidfile="${volpid_dir}/${SHORTHOSTNAME}${brick}.pid"

 	[ -e $pidfile ] || return 1

 	cat $pidfile

     done

diff --git a/extras/stop-all-gluster-processes.sh b/extras/stop-all-gluster-processes.sh

index 69fd601..ea2b48e 100755

--- a/extras/stop-all-gluster-processes.sh

+++ b/extras/stop-all-gluster-processes.sh

@@ -72,7 +72,7 @@ kill_bricks_and_services()

     local pidfile

     local pid

-    for pidfile in $(find /var/lib/glusterd/ -name '*.pid');

+    for pidfile in $(find /var/run/gluster/ -name '*.pid');

     do

         local pid=$(cat ${pidfile});

         echo "sending SIG${signal} to pid: ${pid}";

diff --git a/libglusterfs/src/store.c b/libglusterfs/src/store.c

index 1591627..199ba29 100644

--- a/libglusterfs/src/store.c

+++ b/libglusterfs/src/store.c

@@ -24,7 +24,7 @@ gf_store_mkdir (char *path)

 {

         int32_t     ret = -1;

-        ret = sys_mkdir (path, 0777);

+        ret = mkdir_p (path, 0777, _gf_true);

         if ((-1 == ret) && (EEXIST != errno)) {

                 gf_msg ("", GF_LOG_ERROR, errno, LG_MSG_DIR_OP_FAILED, "mkdir()"

diff --git a/tests/basic/mount-nfs-auth.t b/tests/basic/mount-nfs-auth.t

index 9df5cb4..c4f7f65 100755

--- a/tests/basic/mount-nfs-auth.t

+++ b/tests/basic/mount-nfs-auth.t

@@ -132,7 +132,7 @@ function stat_nfs () {

 # Restarts the NFS server

 function restart_nfs () {

-        local NFS_PID=$(cat ${GLUSTERD_WORKDIR}/nfs/run/nfs.pid)

+        local NFS_PID=$(cat $GLUSTERD_PIDFILEDIR/nfs/nfs.pid)

         # kill the NFS-server if it is running

         while ps -q ${NFS_PID} 2>&1 > /dev/null; do

diff --git a/tests/bugs/fuse/bug-858215.t b/tests/bugs/fuse/bug-858215.t

index c8613ef..389783b 100755

--- a/tests/bugs/fuse/bug-858215.t

+++ b/tests/bugs/fuse/bug-858215.t

@@ -40,8 +40,8 @@ TEST touch $M0/newfile;

 TEST stat $M0/newfile;

 TEST rm $M0/newfile;

-nfs_pid=$(cat $GLUSTERD_WORKDIR/nfs/run/nfs.pid);

-glustershd_pid=$(cat $GLUSTERD_WORKDIR/glustershd/run/glustershd.pid);

+nfs_pid=$(cat $GLUSTERD_PIDFILEDIR/nfs/nfs.pid);

+glustershd_pid=$(cat $GLUSTERD_PIDFILEDIR/glustershd/glustershd.pid);

 pids=$(pidof glusterfs);

 for i in $pids

diff --git a/tests/bugs/glusterd/bug-1225716-brick-online-validation-remove-brick.t b/tests/bugs/glusterd/bug-1225716-brick-online-validation-remove-brick.t

index 47403b4..d168866 100644

--- a/tests/bugs/glusterd/bug-1225716-brick-online-validation-remove-brick.t

+++ b/tests/bugs/glusterd/bug-1225716-brick-online-validation-remove-brick.t

@@ -28,7 +28,6 @@ EXPECT_WITHIN $REBALANCE_TIMEOUT "completed" remove_brick_status_completed_field

 #kill a brick process

 kill_brick $V0 $H0 $B0/${V0}1

-EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT "0" brick_up_status $V0 $H0 $B0/${V0}1

 #remove-brick commit should pass even if the brick is down

 TEST $CLI volume remove-brick $V0 $H0:$B0/${V0}1 commit

diff --git a/tests/bugs/glusterd/bug-916549.t b/tests/bugs/glusterd/bug-916549.t

index bedbdd6..6e3612d 100755

--- a/tests/bugs/glusterd/bug-916549.t

+++ b/tests/bugs/glusterd/bug-916549.t

@@ -8,8 +8,8 @@ TEST glusterd;

 TEST $CLI volume create $V0 $H0:$B0/${V0}1;

 TEST $CLI volume start $V0;

-pid_file=$(ls $GLUSTERD_WORKDIR/vols/$V0/run);

-brick_pid=$(cat $GLUSTERD_WORKDIR/vols/$V0/run/$pid_file);

+pid_file=$(ls $GLUSTERD_PIDFILEDIR/vols/$V0/);

+brick_pid=$(cat $GLUSTERD_PIDFILEDIR/vols/$V0/$pid_file);

 kill -SIGKILL $brick_pid;

diff --git a/tests/bugs/glusterfs-server/bug-861542.t b/tests/bugs/glusterfs-server/bug-861542.t

index d335932..60d1b13 100755

--- a/tests/bugs/glusterfs-server/bug-861542.t

+++ b/tests/bugs/glusterfs-server/bug-861542.t

@@ -39,7 +39,7 @@ TEST $CLI --xml volume status $V0;

 TEST $CLI --xml volume status $V0 detail;

 # Kill the brick process. After this, port number for the killed (in this case brick) process must be "N/A".

-kill `cat $GLUSTERD_WORKDIR/vols/$V0/run/$H0-d-backends-brick0.pid`

+kill `cat $GLUSTERD_PIDFILEDIR/vols/$V0/$H0-d-backends-brick0.pid`

 EXPECT "N/A" port_field $V0 '0'; # volume status

 EXPECT "N/A" port_field $V0 '1'; # volume status detail

diff --git a/tests/bugs/snapshot/bug-1399598-uss-with-ssl.t b/tests/bugs/snapshot/bug-1399598-uss-with-ssl.t

index 7d62526..f4e4e6e 100755

--- a/tests/bugs/snapshot/bug-1399598-uss-with-ssl.t

+++ b/tests/bugs/snapshot/bug-1399598-uss-with-ssl.t

@@ -20,7 +20,7 @@ function total_online_bricks

 {

         # This will count snapd, which isn't really a brick, but callers can

         # account for that so it's OK.

-        find $GLUSTERD_WORKDIR -name '*.pid' | wc -l

+        find $GLUSTERD_PIDFILEDIR -name '*.pid' | wc -l

 }

 cleanup;

diff --git a/tests/bugs/trace/bug-797171.t b/tests/bugs/trace/bug-797171.t

index 29f96b1..b823e47 100755

--- a/tests/bugs/trace/bug-797171.t

+++ b/tests/bugs/trace/bug-797171.t

@@ -21,8 +21,8 @@ $M0;

 touch $M0/{1..22};

 rm -f $M0/*;

-pid_file=$(ls $GLUSTERD_WORKDIR/vols/$V0/run);

-brick_pid=$(cat $GLUSTERD_WORKDIR/vols/$V0/run/$pid_file);

+pid_file=$(ls $GLUSTERD_PIDFILEDIR/vols/$V0/);

+brick_pid=$(cat $GLUSTERD_PIDFILEDIR/vols/$V0/$pid_file);

 mkdir $statedumpdir/statedump_tmp/;

 echo "path=$statedumpdir/statedump_tmp" > $statedumpdir/glusterdump.options;

diff --git a/tests/cluster.rc b/tests/cluster.rc

index 467bbcb..7bb4aa3 100644

--- a/tests/cluster.rc

+++ b/tests/cluster.rc

@@ -27,6 +27,7 @@ function define_backends() {

     for i in `seq 1 $count`; do

         b="B$i";

         mkdir -p ${!b}/glusterd;

+        mkdir -p ${!b}/run;

     done

 }

@@ -43,6 +44,7 @@ function define_glusterds() {

         b="B$i";

         h="H$i";

         wopt="management.working-directory=${!b}/glusterd";

+        ropt="management.run-directory=${!b}/run/gluster";

         bopt="management.transport.socket.bind-address=${!h}";

         popt="--pid-file=${!b}/glusterd.pid";

         sopt="management.glusterd-sockfile=${!b}/glusterd/gd.sock"

@@ -52,11 +54,11 @@ function define_glusterds() {

         logfile=`echo ${0##*/}`_glusterd$i.log

         lopt="--log-file=$logdir/$logfile"

         if [ "$2" == "-LDEBUG" ]; then

-            eval "glusterd_$i='glusterd -LDEBUG --xlator-option $wopt --xlator-option $bopt --xlator-option $sopt $lopt $popt'";

-            eval "glusterd$i='glusterd -LDEBUG --xlator-option $wopt --xlator-option $bopt --xlator-option $sopt $lopt $popt'";

+            eval "glusterd_$i='glusterd -LDEBUG --xlator-option $wopt --xlator-option $bopt --xlator-option $ropt --xlator-option $sopt $lopt $popt'";

+            eval "glusterd$i='glusterd -LDEBUG --xlator-option $wopt --xlator-option $bopt --xlator-option $ropt --xlator-option $sopt $lopt $popt'";

         else

-            eval "glusterd_$i='glusterd --xlator-option $wopt --xlator-option $bopt --xlator-option $sopt $lopt $popt'";

-            eval "glusterd$i='glusterd --xlator-option $wopt --xlator-option $bopt --xlator-option $sopt $lopt $popt'";

+            eval "glusterd_$i='glusterd --xlator-option $wopt --xlator-option $bopt --xlator-option $ropt --xlator-option $sopt $lopt $popt'";

+            eval "glusterd$i='glusterd --xlator-option $wopt --xlator-option $bopt --xlator-option $ropt --xlator-option $sopt $lopt $popt'";

         fi

     done

 }

diff --git a/tests/env.rc.in b/tests/env.rc.in

index acd8f06..0ca98ca 100644

--- a/tests/env.rc.in

+++ b/tests/env.rc.in

@@ -4,6 +4,9 @@ exec_prefix=@exec_prefix@

 PATH=@sbindir@:$PATH

 export PATH

+GLUSTERD_PIDFILEDIR=@localstatedir@/run/gluster

+export GLUSTERD_PIDFILEDIR

+

 LD_LIBRARY_PATH=@libdir@:$LD_LIBRARY_PATH

 export LD_LIBRARY_PATH

diff --git a/tests/include.rc b/tests/include.rc

index 3e61ce7..e8c380f 100644

--- a/tests/include.rc

+++ b/tests/include.rc

@@ -561,6 +561,7 @@ function cleanup()

         # Complete cleanup time

         rm -rf "$B0/*" "/etc/glusterd/*";

         rm -rf $WORKDIRS

+        find $GLUSTERD_PIDFILEDIR -name "*.pid" | xargs rm -rf

         leftover=""

         for d in $WORKDIRS ; do

                 if test -d $d ; then

diff --git a/tests/volume.rc b/tests/volume.rc

index 495d8e5..5d6c96c 100644

--- a/tests/volume.rc

+++ b/tests/volume.rc

@@ -102,7 +102,7 @@ function get_nfs_pid ()

 function read_nfs_pidfile ()

 {

-        echo `cat $GLUSTERD_WORKDIR/nfs/run/nfs.pid`

+        echo `cat $GLUSTERD_PIDFILEDIR/nfs/nfs.pid`

 }

 function cleanup_statedump {

@@ -205,7 +205,7 @@ function ec_child_up_count_shd {

 }

 function get_shd_process_pid {

-        ps auxww | grep glusterfs | grep -E "glustershd/run/glustershd.pid" | awk '{print $2}' | head -1

+        ps auxww | grep glusterfs | grep -E "glustershd/glustershd.pid" | awk '{print $2}' | head -1

 }

 function generate_shd_statedump {

@@ -254,7 +254,7 @@ function get_brick_pidfile {

         local host=$2

         local brick=$3

         local brick_hiphenated=$(echo $brick | tr '/' '-')

-        echo $GLUSTERD_WORKDIR/vols/$vol/run/${host}${brick_hiphenated}.pid

+        echo $GLUSTERD_PIDFILEDIR/vols/$vol/${host}${brick_hiphenated}.pid

 }

 function get_brick_pid {

@@ -498,7 +498,7 @@ function volume_exists() {

 function killall_gluster() {

         pkill gluster

-        find $GLUSTERD_WORKDIR -name '*.pid' | xargs rm -f

+        find $GLUSTERD_PIDFILEDIR -name '*.pid' | xargs rm -f

         sleep 1

 }

diff --git a/xlators/mgmt/glusterd/src/glusterd-snapd-svc-helper.c b/xlators/mgmt/glusterd/src/glusterd-snapd-svc-helper.c

index 826b4ca..b2e0682 100644

--- a/xlators/mgmt/glusterd/src/glusterd-snapd-svc-helper.c

+++ b/xlators/mgmt/glusterd/src/glusterd-snapd-svc-helper.c

@@ -19,9 +19,8 @@ glusterd_svc_build_snapd_rundir (glusterd_volinfo_t *volinfo,

         char                    workdir[PATH_MAX]      = {0,};

         glusterd_conf_t        *priv                   = THIS->private;

-        GLUSTERD_GET_VOLUME_DIR (workdir, volinfo, priv);

-

-        snprintf (path, path_len, "%s/run", workdir);

+        GLUSTERD_GET_VOLUME_PID_DIR (workdir, volinfo, priv);

+        snprintf (path, path_len, "%s", workdir);

 }

 void

diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c

index 4f9144c..dc55fff 100644

--- a/xlators/mgmt/glusterd/src/glusterd-store.c

+++ b/xlators/mgmt/glusterd/src/glusterd-store.c

@@ -1063,6 +1063,19 @@ glusterd_store_voldirpath_set (glusterd_volinfo_t *volinfo, char *voldirpath,

         GLUSTERD_GET_VOLUME_DIR (voldirpath, volinfo, priv);

 }

+static void

+glusterd_store_piddirpath_set (glusterd_volinfo_t *volinfo, char *piddirpath,

+                               size_t len)

+{

+        glusterd_conf_t         *priv = NULL;

+

+        GF_ASSERT (volinfo);

+        priv = THIS->private;

+        GF_ASSERT (priv);

+

+        GLUSTERD_GET_VOLUME_PID_DIR (piddirpath, volinfo, priv);

+}

+

 static int32_t

 glusterd_store_create_volume_dir (glusterd_volinfo_t *volinfo)

 {

@@ -1079,6 +1092,23 @@ glusterd_store_create_volume_dir (glusterd_volinfo_t *volinfo)

         return ret;

 }

+static int32_t

+glusterd_store_create_volume_run_dir (glusterd_volinfo_t *volinfo)

+{

+        int32_t                 ret = -1;

+        char                    piddirpath[PATH_MAX] = {0,};

+

+        GF_ASSERT (volinfo);

+

+        glusterd_store_piddirpath_set (volinfo, piddirpath,

+                                       sizeof (piddirpath));

+

+        ret = gf_store_mkdir (piddirpath);

+

+        gf_msg_debug (THIS->name, 0, "Returning with %d", ret);

+        return ret;

+}

+

 int32_t

 glusterd_store_create_snap_dir (glusterd_snap_t *snap)

 {

@@ -1665,6 +1695,10 @@ glusterd_store_volinfo (glusterd_volinfo_t *volinfo, glusterd_volinfo_ver_ac_t a

         if (ret)

                 goto out;

+        ret = glusterd_store_create_volume_run_dir (volinfo);

+        if (ret)

+                goto out;

+

         ret = glusterd_store_create_vol_shandle_on_absence (volinfo);

         if (ret)

                 goto out;

diff --git a/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.c b/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.c

index d6e57a4..9dcf503 100644

--- a/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.c

+++ b/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.c

@@ -86,8 +86,9 @@ glusterd_svc_init_common (glusterd_svc_t *svc,

                 goto out;

         /* Initialize the process mgmt */

-        glusterd_svc_build_pidfile_path (svc_name, workdir, pidfile,

-                                         sizeof(pidfile));

+        glusterd_svc_build_pidfile_path (svc_name, priv->rundir,

+                                         pidfile, sizeof(pidfile));

+

         glusterd_svc_build_volfile_path (svc_name, workdir, volfile,

                                          sizeof (volfile));

@@ -133,7 +134,7 @@ int glusterd_svc_init (glusterd_svc_t *svc, char *svc_name)

         priv = this->private;

         GF_ASSERT (priv);

-        glusterd_svc_build_rundir (svc_name, priv->workdir, rundir,

+        glusterd_svc_build_rundir (svc_name, priv->rundir, rundir,

                                    sizeof (rundir));

         ret = glusterd_svc_init_common (svc, svc_name, priv->workdir, rundir,

                                         DEFAULT_LOG_FILE_DIRECTORY, NULL);

@@ -276,7 +277,7 @@ glusterd_svc_build_rundir (char *server, char *workdir, char *path, size_t len)

         GF_ASSERT (len == PATH_MAX);

         glusterd_svc_build_svcdir (server, workdir, dir, sizeof (dir));

-        snprintf (path, len, "%s/run", dir);

+        snprintf (path, len, "%s", dir);

 }

 int

diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c

index 407e5b8..02ba0d6 100644

--- a/xlators/mgmt/glusterd/src/glusterd-utils.c

+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c

@@ -1807,7 +1807,7 @@ glusterd_set_brick_socket_filepath (glusterd_volinfo_t *volinfo,

         priv = this->private;

-        GLUSTERD_GET_VOLUME_DIR (volume_dir, volinfo, priv);

+        GLUSTERD_GET_VOLUME_PID_DIR (volume_dir, volinfo, priv);

         GLUSTERD_REMOVE_SLASH_FROM_PATH (brickinfo->path, export_path);

         snprintf (sock_filepath, PATH_MAX, "%s/run/%s-%s",

                   volume_dir, brickinfo->hostname, export_path);

@@ -1866,7 +1866,6 @@ out:

 static int

 _mk_rundir_p (glusterd_volinfo_t *volinfo)

 {

-        char voldir[PATH_MAX]   = {0,};

         char rundir[PATH_MAX]   = {0,};

         glusterd_conf_t *priv   = NULL;

         xlator_t        *this   = NULL;

@@ -1874,8 +1873,7 @@ _mk_rundir_p (glusterd_volinfo_t *volinfo)

         this = THIS;

         priv = this->private;

-        GLUSTERD_GET_VOLUME_DIR (voldir, volinfo, priv);

-        snprintf (rundir, sizeof (rundir)-1, "%s/run", voldir);

+        GLUSTERD_GET_VOLUME_PID_DIR (rundir, volinfo, priv);

         ret =  mkdir_p (rundir, 0777, _gf_true);

         if (ret)

                 gf_msg (this->name, GF_LOG_ERROR, errno,

@@ -4953,8 +4951,8 @@ glusterd_add_node_to_dict (char *server, dict_t *dict, int count,

         priv = this->private;

         GF_ASSERT (priv);

-        glusterd_svc_build_pidfile_path (server, priv->workdir, pidfile,

-                                         sizeof (pidfile));

+        glusterd_svc_build_pidfile_path (server, priv->rundir,

+                                         pidfile, sizeof (pidfile));

         if (strcmp(server, priv->shd_svc.name) == 0)

                 svc = &(priv->shd_svc);

@@ -8042,7 +8040,7 @@ glusterd_nfs_statedump (char *options, int option_cnt, char **op_errstr)

         }

         GLUSTERD_GET_NFS_DIR (path, conf);

-        GLUSTERD_GET_NFS_PIDFILE (pidfile_path, path);

+        GLUSTERD_GET_NFS_PIDFILE (pidfile_path, path, conf);

         pidfile = fopen (pidfile_path, "r");

         if (!pidfile) {

@@ -8167,7 +8165,7 @@ glusterd_quotad_statedump (char *options, int option_cnt, char **op_errstr)

         }

         GLUSTERD_GET_QUOTAD_DIR (path, conf);

-        GLUSTERD_GET_QUOTAD_PIDFILE (pidfile_path, path);

+        GLUSTERD_GET_QUOTAD_PIDFILE (pidfile_path, path, conf);

         pidfile = fopen (pidfile_path, "r");

         if (!pidfile) {

diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c

index e8292cd..34182b0 100644

--- a/xlators/mgmt/glusterd/src/glusterd.c

+++ b/xlators/mgmt/glusterd/src/glusterd.c

@@ -1417,6 +1417,7 @@ init (xlator_t *this)

         struct stat        buf                        = {0,};

         char               storedir[PATH_MAX]         = {0,};

         char               workdir[PATH_MAX]          = {0,};

+        char               rundir[PATH_MAX]           = {0,};

         char               cmd_log_filename[PATH_MAX] = {0,};

         int                first_time                 = 0;

         char              *mountbroker_root           = NULL;

@@ -1449,6 +1450,17 @@ init (xlator_t *this)

         }

 #endif

+        dir_data = dict_get (this->options, "run-directory");

+

+        if (!dir_data) {

+                /* Use default working dir */

+                strncpy (rundir, DEFAULT_VAR_RUN_DIRECTORY, PATH_MAX);

+        } else {

+                strncpy (rundir, dir_data->data, PATH_MAX);

+        }

+

+        dir_data = NULL;

+

         dir_data = dict_get (this->options, "working-directory");

         if (!dir_data) {

@@ -1495,6 +1507,11 @@ init (xlator_t *this)

                 GD_MSG_CURR_WORK_DIR_INFO, "Using %s as working directory",

                 workdir);

+        setenv ("DEFAULT_VAR_RUN_DIRECTORY", rundir, 1);

+        gf_msg (this->name, GF_LOG_INFO, 0,

+                GD_MSG_CURR_WORK_DIR_INFO, "Using %s as pid file working "

+                "directory", rundir);

+

         ret = glusterd_find_correct_var_run_dir (this, var_run_dir);

         if (ret) {

                 gf_msg (this->name, GF_LOG_CRITICAL, 0,

@@ -1505,6 +1522,7 @@ init (xlator_t *this)

         ret = glusterd_init_var_run_dirs (this, var_run_dir,

                                       GLUSTERD_DEFAULT_SNAPS_BRICK_DIR);

+

         if (ret) {

                 gf_msg (this->name, GF_LOG_CRITICAL, 0,

                         GD_MSG_CREATE_DIR_FAILED, "Unable to create "

@@ -1524,6 +1542,51 @@ init (xlator_t *this)

                 exit (1);

         }

+        ret = glusterd_init_var_run_dirs (this, rundir,

+                                          GLUSTERD_BITD_RUN_DIR);

+        if (ret) {

+                gf_msg (this->name, GF_LOG_CRITICAL, 0,

+                        GD_MSG_CREATE_DIR_FAILED, "Unable to create "

+                        "bitd running directory");

+                exit (1);

+        }

+

+        ret = glusterd_init_var_run_dirs (this, rundir,

+                                          GLUSTERD_SCRUB_RUN_DIR);

+        if (ret) {

+                gf_msg (this->name, GF_LOG_CRITICAL, 0,

+                        GD_MSG_CREATE_DIR_FAILED, "Unable to create "

+                        "scrub running directory");

+                exit (1);

+        }

+

+        ret = glusterd_init_var_run_dirs (this, rundir,

+                                          GLUSTERD_GLUSTERSHD_RUN_DIR);

+        if (ret) {

+                gf_msg (this->name, GF_LOG_CRITICAL, 0,

+                        GD_MSG_CREATE_DIR_FAILED, "Unable to create "

+                        "glustershd running directory");

+                exit (1);

+        }

+

+        ret = glusterd_init_var_run_dirs (this, rundir,

+                                          GLUSTERD_NFS_RUN_DIR);

+        if (ret) {

+                gf_msg (this->name, GF_LOG_CRITICAL, 0,

+                        GD_MSG_CREATE_DIR_FAILED, "Unable to create "

+                        "nfs running directory");

+                exit (1);

+        }

+

+        ret = glusterd_init_var_run_dirs (this, rundir,

+                                          GLUSTERD_QUOTAD_RUN_DIR);

+        if (ret) {

+                gf_msg (this->name, GF_LOG_CRITICAL, 0,

+                        GD_MSG_CREATE_DIR_FAILED, "Unable to create "

+                        "quota running directory");

+                exit (1);

+        }

+

         snprintf (cmd_log_filename, PATH_MAX, "%s/cmd_history.log",

                   DEFAULT_LOG_FILE_DIRECTORY);

         ret = gf_cmd_log_init (cmd_log_filename);

@@ -1547,6 +1610,19 @@ init (xlator_t *this)

                 exit (1);

         }

+        /*keeping individual volume pid file information in /var/run/gluster* */

+        snprintf (storedir, PATH_MAX, "%s/vols", rundir);

+

+        ret = sys_mkdir (storedir, 0777);

+

+        if ((-1 == ret) && (errno != EEXIST)) {

+                gf_msg (this->name, GF_LOG_CRITICAL, errno,

+                        GD_MSG_CREATE_DIR_FAILED,

+                        "Unable to create volume directory %s"

+                        " ,errno = %d", storedir, errno);

+                exit (1);

+        }

+

         snprintf (storedir, PATH_MAX, "%s/snaps", workdir);

         ret = sys_mkdir (storedir, 0777);

@@ -1759,6 +1835,7 @@ init (xlator_t *this)

         conf->uds_rpc = uds_rpc;

         conf->gfs_mgmt = &gd_brick_prog;

         strncpy (conf->workdir, workdir, PATH_MAX);

+        strncpy (conf->rundir, rundir, PATH_MAX);

         synclock_init (&conf->big_lock, SYNC_LOCK_RECURSIVE);

         pthread_mutex_init (&conf->xprt_lock, NULL);

diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h

index 533f9a8..0beb6e1 100644

--- a/xlators/mgmt/glusterd/src/glusterd.h

+++ b/xlators/mgmt/glusterd/src/glusterd.h

@@ -141,6 +141,7 @@ typedef struct {

         gf_boolean_t             trace;

         uuid_t                   uuid;

         char                     workdir[PATH_MAX];

+        char                     rundir[PATH_MAX];

         rpcsvc_t                *rpc;

         glusterd_svc_t           shd_svc;

         glusterd_svc_t           nfs_svc;

@@ -562,6 +563,11 @@ typedef enum {

 #define GLUSTERD_VOL_SNAP_DIR_PREFIX "snaps"

 #define GLUSTERD_DEFAULT_SNAPS_BRICK_DIR     "/gluster/snaps"

+#define GLUSTERD_BITD_RUN_DIR                "/bitd"

+#define GLUSTERD_SCRUB_RUN_DIR               "/scrub"

+#define GLUSTERD_GLUSTERSHD_RUN_DIR          "/glustershd"

+#define GLUSTERD_NFS_RUN_DIR                 "/nfs"

+#define GLUSTERD_QUOTAD_RUN_DIR              "/quotad"

 #define GLUSTER_SHARED_STORAGE_BRICK_DIR     GLUSTERD_DEFAULT_WORKDIR"/ss_brick"

 #define GLUSTERD_VAR_RUN_DIR                 "/var/run"

 #define GLUSTERD_RUN_DIR                     "/run"

@@ -586,6 +592,19 @@ typedef ssize_t (*gd_serialize_t) (struct iovec outmsg, void *args);

                           volinfo->volname);                               \

         }

+#define GLUSTERD_GET_VOLUME_PID_DIR(path, volinfo, priv)                   \

+do {                                                                       \

+        if (volinfo->is_snap_volume) {                                     \

+                snprintf (path, PATH_MAX, "%s/snaps/%s/%s",                \

+                          priv->rundir,                                    \

+                          volinfo->snapshot->snapname, volinfo->volname);  \

+        } else {                                                           \

+                snprintf (path, PATH_MAX, "%s/vols/%s",                    \

+                          priv->rundir,                                    \

+                          volinfo->volname);                               \

+        }                                                                  \

+} while (0)

+

 #define GLUSTERD_GET_SNAP_DIR(path, snap, priv)                           \

                 snprintf (path, PATH_MAX, "%s/snaps/%s", priv->workdir,   \

                           snap->snapname);

@@ -638,20 +657,20 @@ typedef ssize_t (*gd_serialize_t) (struct iovec outmsg, void *args);

 #define GLUSTERD_GET_BRICK_PIDFILE(pidfile,volinfo,brickinfo, priv) do {      \

                 char exp_path[PATH_MAX] = {0,};                               \

                 char volpath[PATH_MAX]  = {0,};                               \

-                GLUSTERD_GET_VOLUME_DIR (volpath, volinfo, priv);             \

+                GLUSTERD_GET_VOLUME_PID_DIR (volpath, volinfo, priv);         \

                 GLUSTERD_REMOVE_SLASH_FROM_PATH (brickinfo->path, exp_path);  \

-                snprintf (pidfile, PATH_MAX, "%s/run/%s-%s.pid",              \

-                          volpath, brickinfo->hostname, exp_path);      \

+                snprintf (pidfile, PATH_MAX, "%s/%s-%s.pid",                  \

+                          volpath, brickinfo->hostname, exp_path);            \

         } while (0)

-#define GLUSTERD_GET_NFS_PIDFILE(pidfile,nfspath) {                     \

-                snprintf (pidfile, PATH_MAX, "%s/run/nfs.pid",          \

-                          nfspath);                                     \

+#define GLUSTERD_GET_NFS_PIDFILE(pidfile, nfspath, priv) {            \

+                snprintf (pidfile, PATH_MAX, "%s/nfs/nfs.pid",        \

+                          priv->rundir);                              \

         }

-#define GLUSTERD_GET_QUOTAD_PIDFILE(pidfile,quotadpath) {                     \

-                snprintf (pidfile, PATH_MAX, "%s/run/quotad.pid",          \

-                          quotadpath);                                     \

+#define GLUSTERD_GET_QUOTAD_PIDFILE(pidfile, quotadpath, priv) {         \

+                snprintf (pidfile, PATH_MAX, "%s/quotad/quotad.pid",     \

+                           priv->rundir);                                \

         }

 #define GLUSTERD_GET_QUOTA_CRAWL_PIDDIR(piddir, volinfo, type) do {           \

-- 

1.8.3.1