From f0a3a0de87cb215fb6b9d5451d04af720080decd Mon Sep 17 00:00:00 2001 From: Milind Changire Date: Mon, 12 Jun 2017 14:06:50 +0530 Subject: [PATCH 497/509] tools/glusterfind: add --end-time option Add optional --end-time argument to be used with --since-time when using the "query" command. "start" and "end" times are passed in the command-line to changelog.py only if --full has not been specified on command-line. -1 is passed to changelog.py as end time if user has not supplied --end-time on command-line. brickfind.py: Remove unused "start" command-line argument. Also: Minor indentation changes to keep flake8-2.7 happy. mainline: > BUG: 1453151 > Reviewed-on: https://review.gluster.org/17439 > Smoke: Gluster Build System > NetBSD-regression: NetBSD Build System > CentOS-regression: Gluster Build System > Reviewed-by: Aravinda VK (cherry picked from commit dbdf56ae3f87fc2765ad9cb528c1cfbadfde96ec) Change-Id: I063ef5459916f711503881ade5c4fc32374edad0 BUG: 1450722 Signed-off-by: Milind Changire Reviewed-on: https://code.engineering.redhat.com/gerrit/108769 Reviewed-by: Atin Mukherjee --- tools/glusterfind/src/brickfind.py | 2 +- tools/glusterfind/src/changelog.py | 27 ++++++++++++++----------- tools/glusterfind/src/main.py | 40 ++++++++++++++++++++++++++------------ 3 files changed, 45 insertions(+), 24 deletions(-) diff --git a/tools/glusterfind/src/brickfind.py b/tools/glusterfind/src/brickfind.py index 439c881..e914bac 100644 --- a/tools/glusterfind/src/brickfind.py +++ b/tools/glusterfind/src/brickfind.py @@ -62,9 +62,9 @@ def _get_args(): parser.add_argument("session", help="Session Name") parser.add_argument("volume", help="Volume Name") + parser.add_argument("node", help="Node Name") parser.add_argument("brick", help="Brick Name") parser.add_argument("outfile", help="Output File") - parser.add_argument("start", help="Start Time", type=float) parser.add_argument("tag", help="Tag to prefix file name with") parser.add_argument("--only-query", help="Only query, No session update", action="store_true") diff --git a/tools/glusterfind/src/changelog.py b/tools/glusterfind/src/changelog.py index 721b8d0..ec38bfa 100644 --- a/tools/glusterfind/src/changelog.py +++ b/tools/glusterfind/src/changelog.py @@ -40,8 +40,6 @@ history_turn_time = 0 logger = logging.getLogger() - - def pgfid_to_path(brick, changelog_data): """ For all the pgfids in table, converts into path using recursive @@ -94,7 +92,7 @@ def populate_pgfid_and_inodegfid(brick, changelog_data): path = symlink_gfid_to_path(brick, gfid) path = output_path_prepare(path, args) changelog_data.gfidpath_update({"path1": path}, - {"gfid": gfid}) + {"gfid": gfid}) except (IOError, OSError) as e: logger.warn("Error converting to path: %s" % e) continue @@ -160,10 +158,10 @@ def gfid_to_path_using_pgfid(brick, changelog_data, args): try: path = symlink_gfid_to_path(brick, row[0]) find(os.path.join(brick, path), - callback_func=output_callback, - filter_func=inode_filter, - ignore_dirs=ignore_dirs, - subdirs_crawl=False) + callback_func=output_callback, + filter_func=inode_filter, + ignore_dirs=ignore_dirs, + subdirs_crawl=False) except (IOError, OSError) as e: logger.warn("Error converting to path: %s" % e) continue @@ -272,8 +270,8 @@ def get_changes(brick, hash_dir, log_file, start, end, args): actual_end = libgfchangelog.cl_history_changelog( cl_path, start, end, CHANGELOGAPI_NUM_WORKERS) except libgfchangelog.ChangelogException as e: - fail("%s Historical Changelogs not available: %s" % (brick, e), - logger=logger) + fail("%s: %s Historical Changelogs not available: %s" % + (args.node, brick, e), logger=logger) try: # scan followed by getchanges till scan returns zero. @@ -296,7 +294,7 @@ def get_changes(brick, hash_dir, log_file, start, end, args): libgfchangelog.cl_history_done(change) except IOError as e: logger.warn("Error parsing changelog file %s: %s" % - (change, e)) + (change, e)) changelog_data.commit() except libgfchangelog.ChangelogException as e: @@ -350,9 +348,11 @@ def _get_args(): parser.add_argument("session", help="Session Name") parser.add_argument("volume", help="Volume Name") + parser.add_argument("node", help="Node Name") parser.add_argument("brick", help="Brick Name") parser.add_argument("outfile", help="Output File") parser.add_argument("start", help="Start Time", type=int) + parser.add_argument("end", help="End Time", type=int) parser.add_argument("--only-query", help="Query mode only (no session)", action="store_true") parser.add_argument("--debug", help="Debug", action="store_true") @@ -387,6 +387,7 @@ if __name__ == "__main__": if args.only_query: start = args.start + end = args.end else: try: with open(status_file) as f: @@ -394,7 +395,11 @@ if __name__ == "__main__": except (ValueError, OSError, IOError): start = args.start - end = int(time.time()) - get_changelog_rollover_time(args.volume) + # end time is optional; so a -1 may be sent to use the default method of + # identifying the end time + if end == -1: + end = int(time.time()) - get_changelog_rollover_time(args.volume) + logger.info("%s Started Changelog Crawl - Start: %s End: %s" % (args.brick, start, end)) diff --git a/tools/glusterfind/src/main.py b/tools/glusterfind/src/main.py index 8e31a07..df49e3d 100644 --- a/tools/glusterfind/src/main.py +++ b/tools/glusterfind/src/main.py @@ -139,9 +139,11 @@ def run_cmd_nodes(task, args, **kwargs): cmd = [change_detector, args.session, args.volume, + host, brick, - node_outfile, - str(kwargs.get("start"))] + \ + node_outfile] + \ + ([str(kwargs.get("start")), str(kwargs.get("end"))] + if not args.full else []) + \ ([tag] if tag is not None else []) + \ ["--output-prefix", args.output_prefix] + \ (["--debug"] if args.debug else []) + \ @@ -174,9 +176,11 @@ def run_cmd_nodes(task, args, **kwargs): cmd = [change_detector, args.session, args.volume, + host, brick, - node_outfile, - str(kwargs.get("start"))] + \ + node_outfile] + \ + ([str(kwargs.get("start")), str(kwargs.get("end"))] + if not args.full else []) + \ ([tag] if tag is not None else []) + \ ["--only-query"] + \ ["--output-prefix", args.output_prefix] + \ @@ -353,6 +357,8 @@ def _get_args(): action=StoreAbsPath) parser_query.add_argument("--since-time", help="UNIX epoch time since " "which listing is required", type=int) + parser_query.add_argument("--end-time", help="UNIX epoch time upto " + "which listing is required", type=int) parser_query.add_argument("--no-encode", help="Do not encode path in output file", action="store_true") @@ -560,27 +566,37 @@ def mode_query(session_dir, args): enable_volume_options(args) # Test options - if not args.since_time and not args.full: - fail("Please specify either --since-time or --full", logger=logger) + if not args.since_time and not args.end_time and not args.full: + fail("Please specify either {--since-time and optionally --end-time} " + "or --full", logger=logger) - if args.since_time and args.full: - fail("Please specify either --since-time or --full, but not both", + if args.since_time and args.end_time and args.full: + fail("Please specify either {--since-time and optionally --end-time} " + "or --full, but not both", logger=logger) + if args.end_time and not args.since_time: + fail("Please specify --since-time as well", logger=logger) + # Start query command processing + start = -1 + end = -1 if args.since_time: start = args.since_time + if args.end_time: + end = args.end_time else: start = 0 # --full option is handled separately logger.debug("Query is called - Session: %s, Volume: %s, " - "Start time: %s" - % ("default", args.volume, start)) + "Start time: %s, End time: %s" + % ("default", args.volume, start, end)) prefix = datetime.now().strftime("%Y%m%d-%H%M%S-%f-") gtmpfilename = prefix + next(tempfile._get_candidate_names()) - run_cmd_nodes("query", args, start=start, tmpfilename=gtmpfilename) + run_cmd_nodes("query", args, start=start, end=end, + tmpfilename=gtmpfilename) # Merger if args.full: @@ -639,7 +655,7 @@ def mode_pre(session_dir, args): prefix = datetime.now().strftime("%Y%m%d-%H%M%S-%f-") gtmpfilename = prefix + next(tempfile._get_candidate_names()) - run_cmd_nodes("pre", args, start=start, tmpfilename=gtmpfilename) + run_cmd_nodes("pre", args, start=start, end=-1, tmpfilename=gtmpfilename) # Merger if args.full: -- 1.8.3.1