From 62ad86bf93d94212eddbb50b010f5bd2908e9600 Mon Sep 17 00:00:00 2001
From: Milind Changire <mchangir@redhat.com>
Date: Wed, 26 Oct 2016 15:28:14 +0530
Subject: [PATCH 145/157] tools/glusterfind: kill remote processes and separate run-time directories
Problem #1:
Hitting CTRL+C leaves stale processes on remote nodes if glusterfind pre
has been initiated.
Solution #1:
Adding "-t -t" to ssh command-line forces pseudo-terminal to be assigned
to remote process. When local process receives Keyboard Interrupt,
SIGHUP is immediately conveyed to the remote terminal causing remote
changelog.py process to terminate immediately.
Problem #2:
Concurrent glusterfind pre runs are not possible on the same glusterfind
session in case of a runaway process.
Solution #2:
glusterfind pre runs now add random directory name to the working
directory to store and manage temporary database and changelog
processing.
If KeyboardInterrupt is received, the function call
run_cmd_nodes("cleanup", args, tmpfilename=gtmpfilename)
cleans up the remote run specific directory.
Patch:
7571380 cli/xml: Fix wrong XML format in volume get command
broke "gluster volume get <vol> changelog.rollover-time --xml"
Now fixed function utils.py::get_changelog_rollover_time()
Fixed spurious trailing space getting written if second path is empty in
main.py::write_output()
Fixed repetitive changelog processing in changelog.py::get_changes()
mainline:
> > Reviewed-on: http://review.gluster.org/15609
> > Smoke: Gluster Build System <jenkins@build.gluster.org>
> > CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
> > NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
> > Reviewed-by: Aravinda VK <avishwan@redhat.com>
(cherry picked from commit feea851fad4f89b48bfe89fe3b75250cc7bd6501)
release-3.9:
> Reviewed-on: http://review.gluster.org/15729
> Smoke: Gluster Build System <jenkins@build.gluster.org>
> NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
> CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
> Reviewed-by: Aravinda VK <avishwan@redhat.com>
(cherry picked from commit 915ae56a65d5a96bfddf977193dca60535ac7c11)
Change-Id: Ia8d96e2cd47bf2a64416bece312e67631a1dbf29
BUG: 1379790
Signed-off-by: Milind Changire <mchangir@redhat.com>
Reviewed-on: https://code.engineering.redhat.com/gerrit/88272
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>
---
tools/glusterfind/src/changelog.py | 2 +-
tools/glusterfind/src/main.py | 68 +++++++++++++++++++++++++++++-------
tools/glusterfind/src/nodeagent.py | 4 ++-
tools/glusterfind/src/utils.py | 2 +-
4 files changed, 60 insertions(+), 16 deletions(-)
diff --git a/tools/glusterfind/src/changelog.py b/tools/glusterfind/src/changelog.py
index 283a035..721b8d0 100644
--- a/tools/glusterfind/src/changelog.py
+++ b/tools/glusterfind/src/changelog.py
@@ -284,7 +284,7 @@ def get_changes(brick, hash_dir, log_file, start, end, args):
# history_getchanges()
changes = []
while libgfchangelog.cl_history_scan() > 0:
- changes += libgfchangelog.cl_history_getchanges()
+ changes = libgfchangelog.cl_history_getchanges()
for change in changes:
# Ignore if last processed changelog comes
diff --git a/tools/glusterfind/src/main.py b/tools/glusterfind/src/main.py
index 37d6c38..0c993f5 100644
--- a/tools/glusterfind/src/main.py
+++ b/tools/glusterfind/src/main.py
@@ -18,6 +18,9 @@ import xml.etree.cElementTree as etree
from argparse import ArgumentParser, RawDescriptionHelpFormatter, Action
import logging
import shutil
+import tempfile
+import signal
+from datetime import datetime
from utils import execute, is_host_local, mkdirp, fail
from utils import setup_logger, human_time, handle_rm_error
@@ -34,6 +37,7 @@ ParseError = etree.ParseError if hasattr(etree, 'ParseError') else SyntaxError
logger = logging.getLogger()
node_outfiles = []
vol_statusStr = ""
+gtmpfilename = None
class StoreAbsPath(Action):
@@ -71,6 +75,8 @@ def node_cmd(host, host_uuid, task, cmd, args, opts):
cmd = ["ssh",
"-oNumberOfPasswordPrompts=0",
"-oStrictHostKeyChecking=no",
+ "-t",
+ "-t",
"-i", pem_key_path,
"root@%s" % host] + cmd
@@ -98,8 +104,13 @@ def run_cmd_nodes(task, args, **kwargs):
host_uuid = node[0]
cmd = []
opts = {}
+
+ # tmpfilename is valid only for tasks: pre, query and cleanup
+ tmpfilename = kwargs.get("tmpfilename", "BADNAME")
+
node_outfile = os.path.join(conf.get_opt("working_dir"),
args.session, args.volume,
+ tmpfilename,
"tmp_output_%s" % num)
if task == "pre":
@@ -117,6 +128,9 @@ def run_cmd_nodes(task, args, **kwargs):
tag = '""' if not is_host_local(host_uuid) else ""
node_outfiles.append(node_outfile)
+ # remote file will be copied into this directory
+ mkdirp(os.path.dirname(node_outfile),
+ exit_on_err=True, logger=logger)
cmd = [change_detector,
args.session,
@@ -144,6 +158,9 @@ def run_cmd_nodes(task, args, **kwargs):
tag = '""' if not is_host_local(host_uuid) else ""
node_outfiles.append(node_outfile)
+ # remote file will be copied into this directory
+ mkdirp(os.path.dirname(node_outfile),
+ exit_on_err=True, logger=logger)
cmd = [change_detector,
args.session,
@@ -162,8 +179,9 @@ def run_cmd_nodes(task, args, **kwargs):
opts["node_outfile"] = node_outfile
opts["copy_outfile"] = True
elif task == "cleanup":
- # After pre run, cleanup the working directory and other temp files
- # Remove the copied node_outfile in main node
+ # After pre/query run, cleanup the working directory and other
+ # temp files. Remove the directory to which node_outfile has
+ # been copied in main node
try:
os.remove(node_outfile)
except (OSError, IOError):
@@ -174,7 +192,9 @@ def run_cmd_nodes(task, args, **kwargs):
cmd = [conf.get_opt("nodeagent"),
"cleanup",
args.session,
- args.volume] + (["--debug"] if args.debug else [])
+ args.volume,
+ os.path.dirname(node_outfile)] + \
+ (["--debug"] if args.debug else [])
elif task == "create":
if vol_statusStr != "Started":
fail("Volume %s is not online" % args.volume,
@@ -422,8 +442,8 @@ def enable_volume_options(args):
% args.volume)
-def write_output(args, outfilemerger):
- with codecs.open(args.outfile, "a", encoding="utf-8") as f:
+def write_output(outfile, outfilemerger):
+ with codecs.open(outfile, "a", encoding="utf-8") as f:
for row in outfilemerger.get():
# Multiple paths in case of Hardlinks
paths = row[1].split(",")
@@ -438,9 +458,10 @@ def write_output(args, outfilemerger):
if p_rep == row_2_rep:
continue
- f.write(u"{0} {1} {2}\n".format(row[0],
- p_rep,
- row_2_rep))
+ if row_2_rep and row_2_rep != "":
+ f.write(u"{0} {1} {2}\n".format(row[0], p_rep, row_2_rep))
+ else:
+ f.write(u"{0} {1}\n".format(row[0], p_rep))
def mode_create(session_dir, args):
@@ -490,6 +511,8 @@ def mode_create(session_dir, args):
def mode_query(session_dir, args):
+ global gtmpfilename
+
# Verify volume status
cmd = ["gluster", 'volume', 'info', args.volume, "--xml"]
_, data, _ = execute(cmd,
@@ -533,7 +556,10 @@ def mode_query(session_dir, args):
"Start time: %s"
% ("default", args.volume, start))
- run_cmd_nodes("query", args, start=start)
+ prefix = datetime.now().strftime("%Y%m%d-%H%M%S-%f-")
+ gtmpfilename = prefix + next(tempfile._get_candidate_names())
+
+ run_cmd_nodes("query", args, start=start, tmpfilename=gtmpfilename)
# Merger
if args.full:
@@ -545,7 +571,7 @@ def mode_query(session_dir, args):
# Read each Changelogs db and generate finaldb
create_file(args.outfile, exit_on_err=True, logger=logger)
outfilemerger = OutputMerger(args.outfile + ".db", node_outfiles)
- write_output(args, outfilemerger)
+ write_output(args.outfile, outfilemerger)
try:
os.remove(args.outfile + ".db")
@@ -558,6 +584,8 @@ def mode_query(session_dir, args):
def mode_pre(session_dir, args):
+ global gtmpfilename
+
"""
Read from Session file and write to session.pre file
"""
@@ -587,7 +615,10 @@ def mode_pre(session_dir, args):
"Start time: %s, End time: %s"
% (args.session, args.volume, start, endtime_to_update))
- run_cmd_nodes("pre", args, start=start)
+ prefix = datetime.now().strftime("%Y%m%d-%H%M%S-%f-")
+ gtmpfilename = prefix + next(tempfile._get_candidate_names())
+
+ run_cmd_nodes("pre", args, start=start, tmpfilename=gtmpfilename)
# Merger
if args.full:
@@ -599,8 +630,7 @@ def mode_pre(session_dir, args):
# Read each Changelogs db and generate finaldb
create_file(args.outfile, exit_on_err=True, logger=logger)
outfilemerger = OutputMerger(args.outfile + ".db", node_outfiles)
-
- write_output(args, outfilemerger)
+ write_output(args.outfile, outfilemerger)
try:
os.remove(args.outfile + ".db")
@@ -713,6 +743,10 @@ def mode_list(session_dir, args):
def main():
+ global gtmpfilename
+
+ args = None
+
try:
args = _get_args()
mkdirp(conf.get_opt("session_dir"), exit_on_err=True)
@@ -756,5 +790,13 @@ def main():
# mode_<args.mode> will be the function name to be called
globals()["mode_" + args.mode](session_dir, args)
except KeyboardInterrupt:
+ if args is not None:
+ if args.mode == "pre" or args.mode == "query":
+ # cleanup session
+ if gtmpfilename is not None:
+ # no more interrupts until we clean up
+ signal.signal(signal.SIGINT, signal.SIG_IGN)
+ run_cmd_nodes("cleanup", args, tmpfilename=gtmpfilename)
+
# Interrupted, exit with non zero error code
sys.exit(2)
diff --git a/tools/glusterfind/src/nodeagent.py b/tools/glusterfind/src/nodeagent.py
index f707449..07d8282 100644
--- a/tools/glusterfind/src/nodeagent.py
+++ b/tools/glusterfind/src/nodeagent.py
@@ -26,7 +26,8 @@ logger = logging.getLogger()
def mode_cleanup(args):
working_dir = os.path.join(conf.get_opt("working_dir"),
args.session,
- args.volume)
+ args.volume,
+ args.tmpfilename)
mkdirp(os.path.join(conf.get_opt("log_dir"), args.session, args.volume),
exit_on_err=True)
@@ -98,6 +99,7 @@ def _get_args():
parser_cleanup = subparsers.add_parser('cleanup')
parser_cleanup.add_argument("session", help="Session Name")
parser_cleanup.add_argument("volume", help="Volume Name")
+ parser_cleanup.add_argument("tmpfilename", help="Temporary File Name")
parser_cleanup.add_argument("--debug", help="Debug", action="store_true")
parser_session_create = subparsers.add_parser('create')
diff --git a/tools/glusterfind/src/utils.py b/tools/glusterfind/src/utils.py
index 598cc9e..70737be 100644
--- a/tools/glusterfind/src/utils.py
+++ b/tools/glusterfind/src/utils.py
@@ -227,7 +227,7 @@ def get_changelog_rollover_time(volumename):
try:
tree = etree.fromstring(out)
- return int(tree.find('volGetopts/Value').text)
+ return int(tree.find('volGetopts/Opt/Value').text)
except ParseError:
return DEFAULT_CHANGELOG_INTERVAL
--
1.7.1