From 9cab3d31da4bfd5784215fe394b46bcfc6214bfd Mon Sep 17 00:00:00 2001

From: Jiffin Tony Thottan <jthottan@redhat.com>

Date: Tue, 12 Jul 2016 15:44:23 +0530

Subject: [PATCH 55/86] glusterd/ganesha : Move ganesha-ha.conf and ganesha.conf to shared storage

Currently all the ganesha related configuration files(ganesha.conf,

ganesha-ha.conf, export files, etc) is stored locally at /etc/ganesha

on a every node in ganesha cluster. Usually we end up in two issues by

doing so :

* difficult in modifiying ganesha related conf file

* diffciult to maintain consistency of conf file across ganesha cluster

To tackle this, we plan to move all the ganesha configuration to shared

storage. As a first step in this patch ganesha.conf and ganesha-ha.conf

move to shared storage. Here actual ganesha.conf will resides in shared

stoarge and symlinks will be created in /etc/ganesha when the option

"gluster nfs-ganesha enable" is executed and remove those during the

"disable" part.

Modified prerequisites to done before running globaloption:

	* enable shared storage

	* create nfs-ganesha folder in shared storage

	* create ganesha.conf and ganesha-ha.conf in it

More details can be found at http://review.gluster.org/#/c/15105/

Upstream reference :

>Change-Id: Ifabb6c5db50061f077a03932940190af74e2ca7f

>BUG: 1355956

>Signed-off-by: Jiffin Tony Thottan <jthottan@redhat.com>

>Reviewed-on: http://review.gluster.org/14906

>Reviewed-by: soumya k <skoduri@redhat.com>

>NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>

>Smoke: Gluster Build System <jenkins@build.gluster.org>

>Reviewed-by: Kaleb KEITHLEY <kkeithle@redhat.com>

>CentOS-regression: Gluster Build System <jenkins@build.gluster.org>

>Signed-off-by: Jiffin Tony Thottan <jthottan@redhat.com>

Change-Id: Ifabb6c5db50061f077a03932940190af74e2ca7f

BUG: 1348949

Signed-off-by: Jiffin Tony Thottan <jthottan@redhat.com>

Reviewed-on: https://code.engineering.redhat.com/gerrit/84776

Reviewed-by: Soumya Koduri <skoduri@redhat.com>

Reviewed-by: Atin Mukherjee <amukherj@redhat.com>

Tested-by: Atin Mukherjee <amukherj@redhat.com>

---

 extras/ganesha/scripts/ganesha-ha.sh         |   34 ++++++++--

 xlators/mgmt/glusterd/src/Makefile.am        |    2 +-

 xlators/mgmt/glusterd/src/glusterd-ganesha.c |  101 +++++++++++++++++++++-----

 xlators/mgmt/glusterd/src/glusterd-op-sm.c   |   19 ++++--

 4 files changed, 125 insertions(+), 31 deletions(-)

diff --git a/extras/ganesha/scripts/ganesha-ha.sh b/extras/ganesha/scripts/ganesha-ha.sh

index 8b55abb..31c0c39 100644

--- a/extras/ganesha/scripts/ganesha-ha.sh

+++ b/extras/ganesha/scripts/ganesha-ha.sh

@@ -22,9 +22,9 @@

 HA_NUM_SERVERS=0

 HA_SERVERS=""

-HA_CONFDIR="/etc/ganesha"

 HA_VOL_NAME="gluster_shared_storage"

 HA_VOL_MNT="/var/run/gluster/shared_storage"

+HA_CONFDIR=$HA_VOL_MNT"/nfs-ganesha"

 SERVICE_MAN="DISTRO_NOT_FOUND"

 RHEL6_PCS_CNAME_OPTION="--name"

@@ -416,12 +416,8 @@ teardown_cluster()

 cleanup_ganesha_config ()

 {

-       rm -rf ${HA_CONFDIR}/exports/*.conf

-       rm -rf ${HA_CONFDIR}/.export_added

        rm -rf /etc/cluster/cluster.conf*

        rm -rf /var/lib/pacemaker/cib/*

-       sed -r -i -e '/^%include[[:space:]]+".+\.conf"$/d' ${GANESHA_CONF}

-       rm -rf ${HA_VOL_MNT}/nfs-ganesha

 }

 do_create_virt_ip_constraints()

@@ -830,6 +826,29 @@ status()

     rm -f ${status_file}

 }

+create_ganesha_conf_file()

+{

+        if [ $1 == "yes" ];

+        then

+                if [  -e $GANESHA_CONF ];

+                then

+                        rm -rf $GANESHA_CONF

+                fi

+        # The symlink /etc/ganesha/ganesha.conf need to be

+        # created using ganesha conf file mentioned in the

+        # shared storage. Every node will only have this

+        # link and actual file will stored in shared storage,

+        # so that ganesha conf editing of ganesha conf will

+        # be easy as well as it become more consistent.

+

+                ln -s $HA_CONFDIR/ganesha.conf $GANESHA_CONF

+        else

+        # Restoring previous file

+                rm -rf $GANESHA_CONF

+                sed -r -i -e '/^%include[[:space:]]+".+\.conf"$/d' $HA_CONFDIR/ganesha.conf

+                cp $HA_CONFDIR/ganesha.conf $GANESHA_CONF

+        fi

+}

 main()

 {

@@ -971,6 +990,11 @@ $HA_CONFDIR/ganesha-ha.conf

         refresh_config ${VOL} ${HA_CONFDIR} ${HA_SERVERS}

         ;;

+    setup-ganesha-conf-files | --setup-ganesha-conf-files)

+

+        create_ganesha_conf_file ${1}

+        ;;

+

     *)

         # setup and teardown are not intended to be used by a

         # casual user

diff --git a/xlators/mgmt/glusterd/src/Makefile.am b/xlators/mgmt/glusterd/src/Makefile.am

index f3381e3..23840cd 100644

--- a/xlators/mgmt/glusterd/src/Makefile.am

+++ b/xlators/mgmt/glusterd/src/Makefile.am

@@ -47,7 +47,7 @@ AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \

 	-I$(CONTRIBDIR)/userspace-rcu \

 	-DSBIN_DIR=\"$(sbindir)\" -DDATADIR=\"$(localstatedir)\" \

 	-DGSYNCD_PREFIX=\"$(libexecdir)/glusterfs\" \

-	-DCONFDIR=\"$(sysconfdir)/ganesha\" \

+	-DCONFDIR=\"/$(runstatedir)/gluster/shared_storage/nfs-ganesha\" \

 	-DGANESHA_PREFIX=\"$(libexecdir)/ganesha\" \

 	-DSYNCDAEMON_COMPILE=$(SYNCDAEMON_COMPILE) $(XML_CPPFLAGS)

diff --git a/xlators/mgmt/glusterd/src/glusterd-ganesha.c b/xlators/mgmt/glusterd/src/glusterd-ganesha.c

index d34ec05..3d9a10e 100644

--- a/xlators/mgmt/glusterd/src/glusterd-ganesha.c

+++ b/xlators/mgmt/glusterd/src/glusterd-ganesha.c

@@ -22,8 +22,6 @@

 #include <ctype.h>

-#define SHARED_STORAGE_MNT "/var/run/gluster/shared_storage/nfs-ganesha"

-

 int start_ganesha (char **op_errstr);

@@ -267,6 +265,11 @@ glusterd_op_stage_set_ganesha (dict_t *dict, char **op_errstr)

                                 "Could not start NFS-Ganesha");

                 }

+        } else {

+                ret =  stop_ganesha (op_errstr);

+                if (ret)

+                        gf_msg_debug (THIS->name, 0, "Could not stop "

+                                                "NFS-Ganesha.");

         }

 out:

@@ -638,8 +641,12 @@ out:

 int

 tear_down_cluster(void)

 {

-        int     ret     =       0;

-        runner_t runner =       {0,};

+        int     ret                     = 0;

+        runner_t runner                 = {0,};

+        struct stat     st              = {0,};

+        DIR            *dir             = NULL;

+        struct dirent  *entry           = NULL;

+        char            path[PATH_MAX]  = {0,};

         if (is_ganesha_host()) {

                 runinit (&runner);

@@ -647,7 +654,55 @@ tear_down_cluster(void)

                                 GANESHA_PREFIX"/ganesha-ha.sh", "teardown",

                                 CONFDIR, NULL);

                 ret = runner_run(&runner);

+                /* *

+                 * Remove all the entries in CONFDIR expect ganesha.conf and

+                 * ganesha-ha.conf

+                 */

+                dir = sys_opendir (CONFDIR);

+                if (!dir) {

+                        gf_msg_debug (THIS->name, 0, "Failed to open directory %s. "

+                                      "Reason : %s", CONFDIR, strerror (errno));

+                        ret = 0;

+                        goto out;

+                }

+

+                GF_FOR_EACH_ENTRY_IN_DIR (entry, dir);

+                while (entry) {

+                        snprintf (path, PATH_MAX, "%s/%s", CONFDIR, entry->d_name);

+                        ret = sys_lstat (path, &st);

+                        if (ret == -1) {

+                                gf_msg_debug (THIS->name, 0, "Failed to stat entry %s :"

+                                              " %s", path, strerror (errno));

+                                goto out;

+                        }

+

+                        if (strcmp(entry->d_name, "ganesha.conf") == 0 ||

+                            strcmp(entry->d_name, "ganesha-ha.conf") == 0)

+                                gf_msg_debug (THIS->name, 0, " %s is not required"

+                                                " to remove", path);

+                        else if (S_ISDIR (st.st_mode))

+                                ret = recursive_rmdir (path);

+                        else

+                                ret = sys_unlink (path);

+

+                        if (ret) {

+                                gf_msg_debug (THIS->name, 0, " Failed to remove %s. "

+                                      "Reason : %s", path, strerror (errno));

+                        }

+

+                        gf_msg_debug (THIS->name, 0, "%s %s", ret ?

+                                      "Failed to remove" : "Removed", entry->d_name);

+                        GF_FOR_EACH_ENTRY_IN_DIR (entry, dir);

+                }

+

+                ret = sys_closedir (dir);

+                if (ret) {

+                        gf_msg_debug (THIS->name, 0, "Failed to close dir %s. Reason :"

+                                      " %s", CONFDIR, strerror (errno));

+                }

         }

+

+out:

         return ret;

 }

@@ -685,11 +740,6 @@ teardown (char **op_errstr)

                              " HA config failed.");

                 goto out;

         }

-        ret =  stop_ganesha (op_errstr);

-        if (ret) {

-                gf_asprintf (op_errstr, "Could not stop NFS-Ganesha.");

-                goto out;

-        }

         runinit (&runner);

         runner_add_args (&runner, "sh", GANESHA_PREFIX"/ganesha-ha.sh",

@@ -733,7 +783,17 @@ out:

 int

 stop_ganesha (char **op_errstr) {

-        int ret = 0;

+        int ret                 = 0;

+        runner_t runner         = {0,};

+

+        runinit (&runner);

+        runner_add_args (&runner, "sh", GANESHA_PREFIX"/ganesha-ha.sh",

+                         "--setup-ganesha-conf-files", CONFDIR, "no", NULL);

+        ret =  runner_run (&runner);

+        if (ret) {

+                gf_asprintf (op_errstr, "removal of symlink ganesha.conf "

+                             "in /etc/ganesha failed");

+        }

         if (check_host_list ()) {

                 ret = manage_service ("stop");

@@ -755,6 +815,7 @@ start_ganesha (char **op_errstr)

         int count                                          = 0;

         char *volname                                      = NULL;

         glusterd_conf_t *priv                              = NULL;

+        runner_t runner                                    = {0,};

         priv =  THIS->private;

         GF_ASSERT (priv);

@@ -789,6 +850,16 @@ start_ganesha (char **op_errstr)

                         goto out;

                 }

         }

+

+        runinit (&runner);

+        runner_add_args (&runner, "sh", GANESHA_PREFIX"/ganesha-ha.sh",

+                         "--setup-ganesha-conf-files", CONFDIR, "yes", NULL);

+        ret =  runner_run (&runner);

+        if (ret) {

+                gf_asprintf (op_errstr, "creation of symlink ganesha.conf "

+                             "in /etc/ganesha failed");

+                goto out;

+        }

         if (check_host_list()) {

                 ret = manage_service ("start");

                 if (ret)

@@ -805,15 +876,6 @@ pre_setup (char **op_errstr)

 {

         int    ret = 0;

-        ret = sys_mkdir (SHARED_STORAGE_MNT, 0775);

-

-        if ((-1 == ret) && (EEXIST != errno)) {

-                gf_msg ("THIS->name", GF_LOG_ERROR, errno,

-                        GD_MSG_CREATE_DIR_FAILED, "mkdir() failed on path %s,",

-                        SHARED_STORAGE_MNT);

-                goto out;

-        }

-

         ret = check_host_list();

         if (ret) {

@@ -824,7 +886,6 @@ pre_setup (char **op_errstr)

                                      "Please check the log file for details");

         }

-out:

         return ret;

 }

diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c

index d138e81..4b88570 100644

--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c

+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c

@@ -1422,10 +1422,14 @@ glusterd_op_stage_reset_volume (dict_t *dict, char **op_errstr)

         char                                    *key_fixed = NULL;

         glusterd_volinfo_t                      *volinfo       = NULL;

         xlator_t                                *this = NULL;

+        glusterd_conf_t                         *priv    = NULL;

         this = THIS;

         GF_ASSERT (this);

+        priv = this->private;

+        GF_ASSERT (priv);

+

         ret = dict_get_str (dict, "volname", &volname);

         if (ret) {

@@ -1452,6 +1456,16 @@ glusterd_op_stage_reset_volume (dict_t *dict, char **op_errstr)

                 ret = glusterd_validate_volume_id (dict, volinfo);

                 if (ret)

                         goto out;

+                ret = dict_get_str_boolean (priv->opts,

+                            GLUSTERD_STORE_KEY_GANESHA_GLOBAL, _gf_false);

+                if (ret) {

+                        ret =  stop_ganesha (op_errstr);

+                        if (ret)

+                                gf_msg (THIS->name, GF_LOG_WARNING, 0,

+                                        GD_MSG_NFS_GNS_STOP_FAIL,

+                                        "Could not stop NFS-Ganesha service");

+                }

+

         }

         ret = dict_get_str (dict, "key", &key);

@@ -2038,11 +2052,6 @@ glusterd_op_reset_all_volume_options (xlator_t *this, dict_t *dict)

                         gf_msg (THIS->name, GF_LOG_WARNING, errno,

                                 GD_MSG_DICT_GET_FAILED,

                                 "Could not tear down NFS-Ganesha cluster");

-                ret =  stop_ganesha (&op_errstr);

-                if (ret)

-                        gf_msg (THIS->name, GF_LOG_WARNING, 0,

-                                GD_MSG_NFS_GNS_STOP_FAIL,

-                                "Could not stop NFS-Ganesha service");

         }

         ret = -1;

-- 

1.7.1