From 9cab3d31da4bfd5784215fe394b46bcfc6214bfd Mon Sep 17 00:00:00 2001
From: Jiffin Tony Thottan <jthottan@redhat.com>
Date: Tue, 12 Jul 2016 15:44:23 +0530
Subject: [PATCH 55/86] glusterd/ganesha : Move ganesha-ha.conf and ganesha.conf to shared storage
Currently all the ganesha related configuration files(ganesha.conf,
ganesha-ha.conf, export files, etc) is stored locally at /etc/ganesha
on a every node in ganesha cluster. Usually we end up in two issues by
doing so :
* difficult in modifiying ganesha related conf file
* diffciult to maintain consistency of conf file across ganesha cluster
To tackle this, we plan to move all the ganesha configuration to shared
storage. As a first step in this patch ganesha.conf and ganesha-ha.conf
move to shared storage. Here actual ganesha.conf will resides in shared
stoarge and symlinks will be created in /etc/ganesha when the option
"gluster nfs-ganesha enable" is executed and remove those during the
"disable" part.
Modified prerequisites to done before running globaloption:
* enable shared storage
* create nfs-ganesha folder in shared storage
* create ganesha.conf and ganesha-ha.conf in it
More details can be found at http://review.gluster.org/#/c/15105/
Upstream reference :
>Change-Id: Ifabb6c5db50061f077a03932940190af74e2ca7f
>BUG: 1355956
>Signed-off-by: Jiffin Tony Thottan <jthottan@redhat.com>
>Reviewed-on: http://review.gluster.org/14906
>Reviewed-by: soumya k <skoduri@redhat.com>
>NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
>Smoke: Gluster Build System <jenkins@build.gluster.org>
>Reviewed-by: Kaleb KEITHLEY <kkeithle@redhat.com>
>CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
>Signed-off-by: Jiffin Tony Thottan <jthottan@redhat.com>
Change-Id: Ifabb6c5db50061f077a03932940190af74e2ca7f
BUG: 1348949
Signed-off-by: Jiffin Tony Thottan <jthottan@redhat.com>
Reviewed-on: https://code.engineering.redhat.com/gerrit/84776
Reviewed-by: Soumya Koduri <skoduri@redhat.com>
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>
Tested-by: Atin Mukherjee <amukherj@redhat.com>
---
extras/ganesha/scripts/ganesha-ha.sh | 34 ++++++++--
xlators/mgmt/glusterd/src/Makefile.am | 2 +-
xlators/mgmt/glusterd/src/glusterd-ganesha.c | 101 +++++++++++++++++++++-----
xlators/mgmt/glusterd/src/glusterd-op-sm.c | 19 ++++--
4 files changed, 125 insertions(+), 31 deletions(-)
diff --git a/extras/ganesha/scripts/ganesha-ha.sh b/extras/ganesha/scripts/ganesha-ha.sh
index 8b55abb..31c0c39 100644
--- a/extras/ganesha/scripts/ganesha-ha.sh
+++ b/extras/ganesha/scripts/ganesha-ha.sh
@@ -22,9 +22,9 @@
HA_NUM_SERVERS=0
HA_SERVERS=""
-HA_CONFDIR="/etc/ganesha"
HA_VOL_NAME="gluster_shared_storage"
HA_VOL_MNT="/var/run/gluster/shared_storage"
+HA_CONFDIR=$HA_VOL_MNT"/nfs-ganesha"
SERVICE_MAN="DISTRO_NOT_FOUND"
RHEL6_PCS_CNAME_OPTION="--name"
@@ -416,12 +416,8 @@ teardown_cluster()
cleanup_ganesha_config ()
{
- rm -rf ${HA_CONFDIR}/exports/*.conf
- rm -rf ${HA_CONFDIR}/.export_added
rm -rf /etc/cluster/cluster.conf*
rm -rf /var/lib/pacemaker/cib/*
- sed -r -i -e '/^%include[[:space:]]+".+\.conf"$/d' ${GANESHA_CONF}
- rm -rf ${HA_VOL_MNT}/nfs-ganesha
}
do_create_virt_ip_constraints()
@@ -830,6 +826,29 @@ status()
rm -f ${status_file}
}
+create_ganesha_conf_file()
+{
+ if [ $1 == "yes" ];
+ then
+ if [ -e $GANESHA_CONF ];
+ then
+ rm -rf $GANESHA_CONF
+ fi
+ # The symlink /etc/ganesha/ganesha.conf need to be
+ # created using ganesha conf file mentioned in the
+ # shared storage. Every node will only have this
+ # link and actual file will stored in shared storage,
+ # so that ganesha conf editing of ganesha conf will
+ # be easy as well as it become more consistent.
+
+ ln -s $HA_CONFDIR/ganesha.conf $GANESHA_CONF
+ else
+ # Restoring previous file
+ rm -rf $GANESHA_CONF
+ sed -r -i -e '/^%include[[:space:]]+".+\.conf"$/d' $HA_CONFDIR/ganesha.conf
+ cp $HA_CONFDIR/ganesha.conf $GANESHA_CONF
+ fi
+}
main()
{
@@ -971,6 +990,11 @@ $HA_CONFDIR/ganesha-ha.conf
refresh_config ${VOL} ${HA_CONFDIR} ${HA_SERVERS}
;;
+ setup-ganesha-conf-files | --setup-ganesha-conf-files)
+
+ create_ganesha_conf_file ${1}
+ ;;
+
*)
# setup and teardown are not intended to be used by a
# casual user
diff --git a/xlators/mgmt/glusterd/src/Makefile.am b/xlators/mgmt/glusterd/src/Makefile.am
index f3381e3..23840cd 100644
--- a/xlators/mgmt/glusterd/src/Makefile.am
+++ b/xlators/mgmt/glusterd/src/Makefile.am
@@ -47,7 +47,7 @@ AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-I$(CONTRIBDIR)/userspace-rcu \
-DSBIN_DIR=\"$(sbindir)\" -DDATADIR=\"$(localstatedir)\" \
-DGSYNCD_PREFIX=\"$(libexecdir)/glusterfs\" \
- -DCONFDIR=\"$(sysconfdir)/ganesha\" \
+ -DCONFDIR=\"/$(runstatedir)/gluster/shared_storage/nfs-ganesha\" \
-DGANESHA_PREFIX=\"$(libexecdir)/ganesha\" \
-DSYNCDAEMON_COMPILE=$(SYNCDAEMON_COMPILE) $(XML_CPPFLAGS)
diff --git a/xlators/mgmt/glusterd/src/glusterd-ganesha.c b/xlators/mgmt/glusterd/src/glusterd-ganesha.c
index d34ec05..3d9a10e 100644
--- a/xlators/mgmt/glusterd/src/glusterd-ganesha.c
+++ b/xlators/mgmt/glusterd/src/glusterd-ganesha.c
@@ -22,8 +22,6 @@
#include <ctype.h>
-#define SHARED_STORAGE_MNT "/var/run/gluster/shared_storage/nfs-ganesha"
-
int start_ganesha (char **op_errstr);
@@ -267,6 +265,11 @@ glusterd_op_stage_set_ganesha (dict_t *dict, char **op_errstr)
"Could not start NFS-Ganesha");
}
+ } else {
+ ret = stop_ganesha (op_errstr);
+ if (ret)
+ gf_msg_debug (THIS->name, 0, "Could not stop "
+ "NFS-Ganesha.");
}
out:
@@ -638,8 +641,12 @@ out:
int
tear_down_cluster(void)
{
- int ret = 0;
- runner_t runner = {0,};
+ int ret = 0;
+ runner_t runner = {0,};
+ struct stat st = {0,};
+ DIR *dir = NULL;
+ struct dirent *entry = NULL;
+ char path[PATH_MAX] = {0,};
if (is_ganesha_host()) {
runinit (&runner);
@@ -647,7 +654,55 @@ tear_down_cluster(void)
GANESHA_PREFIX"/ganesha-ha.sh", "teardown",
CONFDIR, NULL);
ret = runner_run(&runner);
+ /* *
+ * Remove all the entries in CONFDIR expect ganesha.conf and
+ * ganesha-ha.conf
+ */
+ dir = sys_opendir (CONFDIR);
+ if (!dir) {
+ gf_msg_debug (THIS->name, 0, "Failed to open directory %s. "
+ "Reason : %s", CONFDIR, strerror (errno));
+ ret = 0;
+ goto out;
+ }
+
+ GF_FOR_EACH_ENTRY_IN_DIR (entry, dir);
+ while (entry) {
+ snprintf (path, PATH_MAX, "%s/%s", CONFDIR, entry->d_name);
+ ret = sys_lstat (path, &st);
+ if (ret == -1) {
+ gf_msg_debug (THIS->name, 0, "Failed to stat entry %s :"
+ " %s", path, strerror (errno));
+ goto out;
+ }
+
+ if (strcmp(entry->d_name, "ganesha.conf") == 0 ||
+ strcmp(entry->d_name, "ganesha-ha.conf") == 0)
+ gf_msg_debug (THIS->name, 0, " %s is not required"
+ " to remove", path);
+ else if (S_ISDIR (st.st_mode))
+ ret = recursive_rmdir (path);
+ else
+ ret = sys_unlink (path);
+
+ if (ret) {
+ gf_msg_debug (THIS->name, 0, " Failed to remove %s. "
+ "Reason : %s", path, strerror (errno));
+ }
+
+ gf_msg_debug (THIS->name, 0, "%s %s", ret ?
+ "Failed to remove" : "Removed", entry->d_name);
+ GF_FOR_EACH_ENTRY_IN_DIR (entry, dir);
+ }
+
+ ret = sys_closedir (dir);
+ if (ret) {
+ gf_msg_debug (THIS->name, 0, "Failed to close dir %s. Reason :"
+ " %s", CONFDIR, strerror (errno));
+ }
}
+
+out:
return ret;
}
@@ -685,11 +740,6 @@ teardown (char **op_errstr)
" HA config failed.");
goto out;
}
- ret = stop_ganesha (op_errstr);
- if (ret) {
- gf_asprintf (op_errstr, "Could not stop NFS-Ganesha.");
- goto out;
- }
runinit (&runner);
runner_add_args (&runner, "sh", GANESHA_PREFIX"/ganesha-ha.sh",
@@ -733,7 +783,17 @@ out:
int
stop_ganesha (char **op_errstr) {
- int ret = 0;
+ int ret = 0;
+ runner_t runner = {0,};
+
+ runinit (&runner);
+ runner_add_args (&runner, "sh", GANESHA_PREFIX"/ganesha-ha.sh",
+ "--setup-ganesha-conf-files", CONFDIR, "no", NULL);
+ ret = runner_run (&runner);
+ if (ret) {
+ gf_asprintf (op_errstr, "removal of symlink ganesha.conf "
+ "in /etc/ganesha failed");
+ }
if (check_host_list ()) {
ret = manage_service ("stop");
@@ -755,6 +815,7 @@ start_ganesha (char **op_errstr)
int count = 0;
char *volname = NULL;
glusterd_conf_t *priv = NULL;
+ runner_t runner = {0,};
priv = THIS->private;
GF_ASSERT (priv);
@@ -789,6 +850,16 @@ start_ganesha (char **op_errstr)
goto out;
}
}
+
+ runinit (&runner);
+ runner_add_args (&runner, "sh", GANESHA_PREFIX"/ganesha-ha.sh",
+ "--setup-ganesha-conf-files", CONFDIR, "yes", NULL);
+ ret = runner_run (&runner);
+ if (ret) {
+ gf_asprintf (op_errstr, "creation of symlink ganesha.conf "
+ "in /etc/ganesha failed");
+ goto out;
+ }
if (check_host_list()) {
ret = manage_service ("start");
if (ret)
@@ -805,15 +876,6 @@ pre_setup (char **op_errstr)
{
int ret = 0;
- ret = sys_mkdir (SHARED_STORAGE_MNT, 0775);
-
- if ((-1 == ret) && (EEXIST != errno)) {
- gf_msg ("THIS->name", GF_LOG_ERROR, errno,
- GD_MSG_CREATE_DIR_FAILED, "mkdir() failed on path %s,",
- SHARED_STORAGE_MNT);
- goto out;
- }
-
ret = check_host_list();
if (ret) {
@@ -824,7 +886,6 @@ pre_setup (char **op_errstr)
"Please check the log file for details");
}
-out:
return ret;
}
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index d138e81..4b88570 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -1422,10 +1422,14 @@ glusterd_op_stage_reset_volume (dict_t *dict, char **op_errstr)
char *key_fixed = NULL;
glusterd_volinfo_t *volinfo = NULL;
xlator_t *this = NULL;
+ glusterd_conf_t *priv = NULL;
this = THIS;
GF_ASSERT (this);
+ priv = this->private;
+ GF_ASSERT (priv);
+
ret = dict_get_str (dict, "volname", &volname);
if (ret) {
@@ -1452,6 +1456,16 @@ glusterd_op_stage_reset_volume (dict_t *dict, char **op_errstr)
ret = glusterd_validate_volume_id (dict, volinfo);
if (ret)
goto out;
+ ret = dict_get_str_boolean (priv->opts,
+ GLUSTERD_STORE_KEY_GANESHA_GLOBAL, _gf_false);
+ if (ret) {
+ ret = stop_ganesha (op_errstr);
+ if (ret)
+ gf_msg (THIS->name, GF_LOG_WARNING, 0,
+ GD_MSG_NFS_GNS_STOP_FAIL,
+ "Could not stop NFS-Ganesha service");
+ }
+
}
ret = dict_get_str (dict, "key", &key);
@@ -2038,11 +2052,6 @@ glusterd_op_reset_all_volume_options (xlator_t *this, dict_t *dict)
gf_msg (THIS->name, GF_LOG_WARNING, errno,
GD_MSG_DICT_GET_FAILED,
"Could not tear down NFS-Ganesha cluster");
- ret = stop_ganesha (&op_errstr);
- if (ret)
- gf_msg (THIS->name, GF_LOG_WARNING, 0,
- GD_MSG_NFS_GNS_STOP_FAIL,
- "Could not stop NFS-Ganesha service");
}
ret = -1;
--
1.7.1