From 9cab3d31da4bfd5784215fe394b46bcfc6214bfd Mon Sep 17 00:00:00 2001 From: Jiffin Tony Thottan Date: Tue, 12 Jul 2016 15:44:23 +0530 Subject: [PATCH 55/86] glusterd/ganesha : Move ganesha-ha.conf and ganesha.conf to shared storage Currently all the ganesha related configuration files(ganesha.conf, ganesha-ha.conf, export files, etc) is stored locally at /etc/ganesha on a every node in ganesha cluster. Usually we end up in two issues by doing so : * difficult in modifiying ganesha related conf file * diffciult to maintain consistency of conf file across ganesha cluster To tackle this, we plan to move all the ganesha configuration to shared storage. As a first step in this patch ganesha.conf and ganesha-ha.conf move to shared storage. Here actual ganesha.conf will resides in shared stoarge and symlinks will be created in /etc/ganesha when the option "gluster nfs-ganesha enable" is executed and remove those during the "disable" part. Modified prerequisites to done before running globaloption: * enable shared storage * create nfs-ganesha folder in shared storage * create ganesha.conf and ganesha-ha.conf in it More details can be found at http://review.gluster.org/#/c/15105/ Upstream reference : >Change-Id: Ifabb6c5db50061f077a03932940190af74e2ca7f >BUG: 1355956 >Signed-off-by: Jiffin Tony Thottan >Reviewed-on: http://review.gluster.org/14906 >Reviewed-by: soumya k >NetBSD-regression: NetBSD Build System >Smoke: Gluster Build System >Reviewed-by: Kaleb KEITHLEY >CentOS-regression: Gluster Build System >Signed-off-by: Jiffin Tony Thottan Change-Id: Ifabb6c5db50061f077a03932940190af74e2ca7f BUG: 1348949 Signed-off-by: Jiffin Tony Thottan Reviewed-on: https://code.engineering.redhat.com/gerrit/84776 Reviewed-by: Soumya Koduri Reviewed-by: Atin Mukherjee Tested-by: Atin Mukherjee --- extras/ganesha/scripts/ganesha-ha.sh | 34 ++++++++-- xlators/mgmt/glusterd/src/Makefile.am | 2 +- xlators/mgmt/glusterd/src/glusterd-ganesha.c | 101 +++++++++++++++++++++----- xlators/mgmt/glusterd/src/glusterd-op-sm.c | 19 ++++-- 4 files changed, 125 insertions(+), 31 deletions(-) diff --git a/extras/ganesha/scripts/ganesha-ha.sh b/extras/ganesha/scripts/ganesha-ha.sh index 8b55abb..31c0c39 100644 --- a/extras/ganesha/scripts/ganesha-ha.sh +++ b/extras/ganesha/scripts/ganesha-ha.sh @@ -22,9 +22,9 @@ HA_NUM_SERVERS=0 HA_SERVERS="" -HA_CONFDIR="/etc/ganesha" HA_VOL_NAME="gluster_shared_storage" HA_VOL_MNT="/var/run/gluster/shared_storage" +HA_CONFDIR=$HA_VOL_MNT"/nfs-ganesha" SERVICE_MAN="DISTRO_NOT_FOUND" RHEL6_PCS_CNAME_OPTION="--name" @@ -416,12 +416,8 @@ teardown_cluster() cleanup_ganesha_config () { - rm -rf ${HA_CONFDIR}/exports/*.conf - rm -rf ${HA_CONFDIR}/.export_added rm -rf /etc/cluster/cluster.conf* rm -rf /var/lib/pacemaker/cib/* - sed -r -i -e '/^%include[[:space:]]+".+\.conf"$/d' ${GANESHA_CONF} - rm -rf ${HA_VOL_MNT}/nfs-ganesha } do_create_virt_ip_constraints() @@ -830,6 +826,29 @@ status() rm -f ${status_file} } +create_ganesha_conf_file() +{ + if [ $1 == "yes" ]; + then + if [ -e $GANESHA_CONF ]; + then + rm -rf $GANESHA_CONF + fi + # The symlink /etc/ganesha/ganesha.conf need to be + # created using ganesha conf file mentioned in the + # shared storage. Every node will only have this + # link and actual file will stored in shared storage, + # so that ganesha conf editing of ganesha conf will + # be easy as well as it become more consistent. + + ln -s $HA_CONFDIR/ganesha.conf $GANESHA_CONF + else + # Restoring previous file + rm -rf $GANESHA_CONF + sed -r -i -e '/^%include[[:space:]]+".+\.conf"$/d' $HA_CONFDIR/ganesha.conf + cp $HA_CONFDIR/ganesha.conf $GANESHA_CONF + fi +} main() { @@ -971,6 +990,11 @@ $HA_CONFDIR/ganesha-ha.conf refresh_config ${VOL} ${HA_CONFDIR} ${HA_SERVERS} ;; + setup-ganesha-conf-files | --setup-ganesha-conf-files) + + create_ganesha_conf_file ${1} + ;; + *) # setup and teardown are not intended to be used by a # casual user diff --git a/xlators/mgmt/glusterd/src/Makefile.am b/xlators/mgmt/glusterd/src/Makefile.am index f3381e3..23840cd 100644 --- a/xlators/mgmt/glusterd/src/Makefile.am +++ b/xlators/mgmt/glusterd/src/Makefile.am @@ -47,7 +47,7 @@ AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \ -I$(CONTRIBDIR)/userspace-rcu \ -DSBIN_DIR=\"$(sbindir)\" -DDATADIR=\"$(localstatedir)\" \ -DGSYNCD_PREFIX=\"$(libexecdir)/glusterfs\" \ - -DCONFDIR=\"$(sysconfdir)/ganesha\" \ + -DCONFDIR=\"/$(runstatedir)/gluster/shared_storage/nfs-ganesha\" \ -DGANESHA_PREFIX=\"$(libexecdir)/ganesha\" \ -DSYNCDAEMON_COMPILE=$(SYNCDAEMON_COMPILE) $(XML_CPPFLAGS) diff --git a/xlators/mgmt/glusterd/src/glusterd-ganesha.c b/xlators/mgmt/glusterd/src/glusterd-ganesha.c index d34ec05..3d9a10e 100644 --- a/xlators/mgmt/glusterd/src/glusterd-ganesha.c +++ b/xlators/mgmt/glusterd/src/glusterd-ganesha.c @@ -22,8 +22,6 @@ #include -#define SHARED_STORAGE_MNT "/var/run/gluster/shared_storage/nfs-ganesha" - int start_ganesha (char **op_errstr); @@ -267,6 +265,11 @@ glusterd_op_stage_set_ganesha (dict_t *dict, char **op_errstr) "Could not start NFS-Ganesha"); } + } else { + ret = stop_ganesha (op_errstr); + if (ret) + gf_msg_debug (THIS->name, 0, "Could not stop " + "NFS-Ganesha."); } out: @@ -638,8 +641,12 @@ out: int tear_down_cluster(void) { - int ret = 0; - runner_t runner = {0,}; + int ret = 0; + runner_t runner = {0,}; + struct stat st = {0,}; + DIR *dir = NULL; + struct dirent *entry = NULL; + char path[PATH_MAX] = {0,}; if (is_ganesha_host()) { runinit (&runner); @@ -647,7 +654,55 @@ tear_down_cluster(void) GANESHA_PREFIX"/ganesha-ha.sh", "teardown", CONFDIR, NULL); ret = runner_run(&runner); + /* * + * Remove all the entries in CONFDIR expect ganesha.conf and + * ganesha-ha.conf + */ + dir = sys_opendir (CONFDIR); + if (!dir) { + gf_msg_debug (THIS->name, 0, "Failed to open directory %s. " + "Reason : %s", CONFDIR, strerror (errno)); + ret = 0; + goto out; + } + + GF_FOR_EACH_ENTRY_IN_DIR (entry, dir); + while (entry) { + snprintf (path, PATH_MAX, "%s/%s", CONFDIR, entry->d_name); + ret = sys_lstat (path, &st); + if (ret == -1) { + gf_msg_debug (THIS->name, 0, "Failed to stat entry %s :" + " %s", path, strerror (errno)); + goto out; + } + + if (strcmp(entry->d_name, "ganesha.conf") == 0 || + strcmp(entry->d_name, "ganesha-ha.conf") == 0) + gf_msg_debug (THIS->name, 0, " %s is not required" + " to remove", path); + else if (S_ISDIR (st.st_mode)) + ret = recursive_rmdir (path); + else + ret = sys_unlink (path); + + if (ret) { + gf_msg_debug (THIS->name, 0, " Failed to remove %s. " + "Reason : %s", path, strerror (errno)); + } + + gf_msg_debug (THIS->name, 0, "%s %s", ret ? + "Failed to remove" : "Removed", entry->d_name); + GF_FOR_EACH_ENTRY_IN_DIR (entry, dir); + } + + ret = sys_closedir (dir); + if (ret) { + gf_msg_debug (THIS->name, 0, "Failed to close dir %s. Reason :" + " %s", CONFDIR, strerror (errno)); + } } + +out: return ret; } @@ -685,11 +740,6 @@ teardown (char **op_errstr) " HA config failed."); goto out; } - ret = stop_ganesha (op_errstr); - if (ret) { - gf_asprintf (op_errstr, "Could not stop NFS-Ganesha."); - goto out; - } runinit (&runner); runner_add_args (&runner, "sh", GANESHA_PREFIX"/ganesha-ha.sh", @@ -733,7 +783,17 @@ out: int stop_ganesha (char **op_errstr) { - int ret = 0; + int ret = 0; + runner_t runner = {0,}; + + runinit (&runner); + runner_add_args (&runner, "sh", GANESHA_PREFIX"/ganesha-ha.sh", + "--setup-ganesha-conf-files", CONFDIR, "no", NULL); + ret = runner_run (&runner); + if (ret) { + gf_asprintf (op_errstr, "removal of symlink ganesha.conf " + "in /etc/ganesha failed"); + } if (check_host_list ()) { ret = manage_service ("stop"); @@ -755,6 +815,7 @@ start_ganesha (char **op_errstr) int count = 0; char *volname = NULL; glusterd_conf_t *priv = NULL; + runner_t runner = {0,}; priv = THIS->private; GF_ASSERT (priv); @@ -789,6 +850,16 @@ start_ganesha (char **op_errstr) goto out; } } + + runinit (&runner); + runner_add_args (&runner, "sh", GANESHA_PREFIX"/ganesha-ha.sh", + "--setup-ganesha-conf-files", CONFDIR, "yes", NULL); + ret = runner_run (&runner); + if (ret) { + gf_asprintf (op_errstr, "creation of symlink ganesha.conf " + "in /etc/ganesha failed"); + goto out; + } if (check_host_list()) { ret = manage_service ("start"); if (ret) @@ -805,15 +876,6 @@ pre_setup (char **op_errstr) { int ret = 0; - ret = sys_mkdir (SHARED_STORAGE_MNT, 0775); - - if ((-1 == ret) && (EEXIST != errno)) { - gf_msg ("THIS->name", GF_LOG_ERROR, errno, - GD_MSG_CREATE_DIR_FAILED, "mkdir() failed on path %s,", - SHARED_STORAGE_MNT); - goto out; - } - ret = check_host_list(); if (ret) { @@ -824,7 +886,6 @@ pre_setup (char **op_errstr) "Please check the log file for details"); } -out: return ret; } diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c index d138e81..4b88570 100644 --- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c +++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c @@ -1422,10 +1422,14 @@ glusterd_op_stage_reset_volume (dict_t *dict, char **op_errstr) char *key_fixed = NULL; glusterd_volinfo_t *volinfo = NULL; xlator_t *this = NULL; + glusterd_conf_t *priv = NULL; this = THIS; GF_ASSERT (this); + priv = this->private; + GF_ASSERT (priv); + ret = dict_get_str (dict, "volname", &volname); if (ret) { @@ -1452,6 +1456,16 @@ glusterd_op_stage_reset_volume (dict_t *dict, char **op_errstr) ret = glusterd_validate_volume_id (dict, volinfo); if (ret) goto out; + ret = dict_get_str_boolean (priv->opts, + GLUSTERD_STORE_KEY_GANESHA_GLOBAL, _gf_false); + if (ret) { + ret = stop_ganesha (op_errstr); + if (ret) + gf_msg (THIS->name, GF_LOG_WARNING, 0, + GD_MSG_NFS_GNS_STOP_FAIL, + "Could not stop NFS-Ganesha service"); + } + } ret = dict_get_str (dict, "key", &key); @@ -2038,11 +2052,6 @@ glusterd_op_reset_all_volume_options (xlator_t *this, dict_t *dict) gf_msg (THIS->name, GF_LOG_WARNING, errno, GD_MSG_DICT_GET_FAILED, "Could not tear down NFS-Ganesha cluster"); - ret = stop_ganesha (&op_errstr); - if (ret) - gf_msg (THIS->name, GF_LOG_WARNING, 0, - GD_MSG_NFS_GNS_STOP_FAIL, - "Could not stop NFS-Ganesha service"); } ret = -1; -- 1.7.1