From f0a3a0de87cb215fb6b9d5451d04af720080decd Mon Sep 17 00:00:00 2001
From: Milind Changire <mchangir@redhat.com>
Date: Mon, 12 Jun 2017 14:06:50 +0530
Subject: [PATCH 497/509] tools/glusterfind: add --end-time option
Add optional --end-time argument to be used with --since-time
when using the "query" command.
"start" and "end" times are passed in the command-line to
changelog.py only if --full has not been specified on command-line.
-1 is passed to changelog.py as end time if user has not supplied
--end-time on command-line.
brickfind.py:
Remove unused "start" command-line argument.
Also:
Minor indentation changes to keep flake8-2.7 happy.
mainline:
> BUG: 1453151
> Reviewed-on: https://review.gluster.org/17439
> Smoke: Gluster Build System <jenkins@build.gluster.org>
> NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
> CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
> Reviewed-by: Aravinda VK <avishwan@redhat.com>
(cherry picked from commit dbdf56ae3f87fc2765ad9cb528c1cfbadfde96ec)
Change-Id: I063ef5459916f711503881ade5c4fc32374edad0
BUG: 1450722
Signed-off-by: Milind Changire <mchangir@redhat.com>
Reviewed-on: https://code.engineering.redhat.com/gerrit/108769
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>
---
tools/glusterfind/src/brickfind.py | 2 +-
tools/glusterfind/src/changelog.py | 27 ++++++++++++++-----------
tools/glusterfind/src/main.py | 40 ++++++++++++++++++++++++++------------
3 files changed, 45 insertions(+), 24 deletions(-)
diff --git a/tools/glusterfind/src/brickfind.py b/tools/glusterfind/src/brickfind.py
index 439c881..e914bac 100644
--- a/tools/glusterfind/src/brickfind.py
+++ b/tools/glusterfind/src/brickfind.py
@@ -62,9 +62,9 @@ def _get_args():
parser.add_argument("session", help="Session Name")
parser.add_argument("volume", help="Volume Name")
+ parser.add_argument("node", help="Node Name")
parser.add_argument("brick", help="Brick Name")
parser.add_argument("outfile", help="Output File")
- parser.add_argument("start", help="Start Time", type=float)
parser.add_argument("tag", help="Tag to prefix file name with")
parser.add_argument("--only-query", help="Only query, No session update",
action="store_true")
diff --git a/tools/glusterfind/src/changelog.py b/tools/glusterfind/src/changelog.py
index 721b8d0..ec38bfa 100644
--- a/tools/glusterfind/src/changelog.py
+++ b/tools/glusterfind/src/changelog.py
@@ -40,8 +40,6 @@ history_turn_time = 0
logger = logging.getLogger()
-
-
def pgfid_to_path(brick, changelog_data):
"""
For all the pgfids in table, converts into path using recursive
@@ -94,7 +92,7 @@ def populate_pgfid_and_inodegfid(brick, changelog_data):
path = symlink_gfid_to_path(brick, gfid)
path = output_path_prepare(path, args)
changelog_data.gfidpath_update({"path1": path},
- {"gfid": gfid})
+ {"gfid": gfid})
except (IOError, OSError) as e:
logger.warn("Error converting to path: %s" % e)
continue
@@ -160,10 +158,10 @@ def gfid_to_path_using_pgfid(brick, changelog_data, args):
try:
path = symlink_gfid_to_path(brick, row[0])
find(os.path.join(brick, path),
- callback_func=output_callback,
- filter_func=inode_filter,
- ignore_dirs=ignore_dirs,
- subdirs_crawl=False)
+ callback_func=output_callback,
+ filter_func=inode_filter,
+ ignore_dirs=ignore_dirs,
+ subdirs_crawl=False)
except (IOError, OSError) as e:
logger.warn("Error converting to path: %s" % e)
continue
@@ -272,8 +270,8 @@ def get_changes(brick, hash_dir, log_file, start, end, args):
actual_end = libgfchangelog.cl_history_changelog(
cl_path, start, end, CHANGELOGAPI_NUM_WORKERS)
except libgfchangelog.ChangelogException as e:
- fail("%s Historical Changelogs not available: %s" % (brick, e),
- logger=logger)
+ fail("%s: %s Historical Changelogs not available: %s" %
+ (args.node, brick, e), logger=logger)
try:
# scan followed by getchanges till scan returns zero.
@@ -296,7 +294,7 @@ def get_changes(brick, hash_dir, log_file, start, end, args):
libgfchangelog.cl_history_done(change)
except IOError as e:
logger.warn("Error parsing changelog file %s: %s" %
- (change, e))
+ (change, e))
changelog_data.commit()
except libgfchangelog.ChangelogException as e:
@@ -350,9 +348,11 @@ def _get_args():
parser.add_argument("session", help="Session Name")
parser.add_argument("volume", help="Volume Name")
+ parser.add_argument("node", help="Node Name")
parser.add_argument("brick", help="Brick Name")
parser.add_argument("outfile", help="Output File")
parser.add_argument("start", help="Start Time", type=int)
+ parser.add_argument("end", help="End Time", type=int)
parser.add_argument("--only-query", help="Query mode only (no session)",
action="store_true")
parser.add_argument("--debug", help="Debug", action="store_true")
@@ -387,6 +387,7 @@ if __name__ == "__main__":
if args.only_query:
start = args.start
+ end = args.end
else:
try:
with open(status_file) as f:
@@ -394,7 +395,11 @@ if __name__ == "__main__":
except (ValueError, OSError, IOError):
start = args.start
- end = int(time.time()) - get_changelog_rollover_time(args.volume)
+ # end time is optional; so a -1 may be sent to use the default method of
+ # identifying the end time
+ if end == -1:
+ end = int(time.time()) - get_changelog_rollover_time(args.volume)
+
logger.info("%s Started Changelog Crawl - Start: %s End: %s" % (args.brick,
start,
end))
diff --git a/tools/glusterfind/src/main.py b/tools/glusterfind/src/main.py
index 8e31a07..df49e3d 100644
--- a/tools/glusterfind/src/main.py
+++ b/tools/glusterfind/src/main.py
@@ -139,9 +139,11 @@ def run_cmd_nodes(task, args, **kwargs):
cmd = [change_detector,
args.session,
args.volume,
+ host,
brick,
- node_outfile,
- str(kwargs.get("start"))] + \
+ node_outfile] + \
+ ([str(kwargs.get("start")), str(kwargs.get("end"))]
+ if not args.full else []) + \
([tag] if tag is not None else []) + \
["--output-prefix", args.output_prefix] + \
(["--debug"] if args.debug else []) + \
@@ -174,9 +176,11 @@ def run_cmd_nodes(task, args, **kwargs):
cmd = [change_detector,
args.session,
args.volume,
+ host,
brick,
- node_outfile,
- str(kwargs.get("start"))] + \
+ node_outfile] + \
+ ([str(kwargs.get("start")), str(kwargs.get("end"))]
+ if not args.full else []) + \
([tag] if tag is not None else []) + \
["--only-query"] + \
["--output-prefix", args.output_prefix] + \
@@ -353,6 +357,8 @@ def _get_args():
action=StoreAbsPath)
parser_query.add_argument("--since-time", help="UNIX epoch time since "
"which listing is required", type=int)
+ parser_query.add_argument("--end-time", help="UNIX epoch time upto "
+ "which listing is required", type=int)
parser_query.add_argument("--no-encode",
help="Do not encode path in output file",
action="store_true")
@@ -560,27 +566,37 @@ def mode_query(session_dir, args):
enable_volume_options(args)
# Test options
- if not args.since_time and not args.full:
- fail("Please specify either --since-time or --full", logger=logger)
+ if not args.since_time and not args.end_time and not args.full:
+ fail("Please specify either {--since-time and optionally --end-time} "
+ "or --full", logger=logger)
- if args.since_time and args.full:
- fail("Please specify either --since-time or --full, but not both",
+ if args.since_time and args.end_time and args.full:
+ fail("Please specify either {--since-time and optionally --end-time} "
+ "or --full, but not both",
logger=logger)
+ if args.end_time and not args.since_time:
+ fail("Please specify --since-time as well", logger=logger)
+
# Start query command processing
+ start = -1
+ end = -1
if args.since_time:
start = args.since_time
+ if args.end_time:
+ end = args.end_time
else:
start = 0 # --full option is handled separately
logger.debug("Query is called - Session: %s, Volume: %s, "
- "Start time: %s"
- % ("default", args.volume, start))
+ "Start time: %s, End time: %s"
+ % ("default", args.volume, start, end))
prefix = datetime.now().strftime("%Y%m%d-%H%M%S-%f-")
gtmpfilename = prefix + next(tempfile._get_candidate_names())
- run_cmd_nodes("query", args, start=start, tmpfilename=gtmpfilename)
+ run_cmd_nodes("query", args, start=start, end=end,
+ tmpfilename=gtmpfilename)
# Merger
if args.full:
@@ -639,7 +655,7 @@ def mode_pre(session_dir, args):
prefix = datetime.now().strftime("%Y%m%d-%H%M%S-%f-")
gtmpfilename = prefix + next(tempfile._get_candidate_names())
- run_cmd_nodes("pre", args, start=start, tmpfilename=gtmpfilename)
+ run_cmd_nodes("pre", args, start=start, end=-1, tmpfilename=gtmpfilename)
# Merger
if args.full:
--
1.8.3.1