From f0a3a0de87cb215fb6b9d5451d04af720080decd Mon Sep 17 00:00:00 2001

From: Milind Changire <mchangir@redhat.com>

Date: Mon, 12 Jun 2017 14:06:50 +0530

Subject: [PATCH 497/509] tools/glusterfind: add --end-time option

Add optional --end-time argument to be used with --since-time

when using the "query" command.

"start" and "end" times are passed in the command-line to

changelog.py only if --full has not been specified on command-line.

-1 is passed to changelog.py as end time if user has not supplied

--end-time on command-line.

brickfind.py:

Remove unused "start" command-line argument.

Also:

Minor indentation changes to keep flake8-2.7 happy.

mainline:

> BUG: 1453151

> Reviewed-on: https://review.gluster.org/17439

> Smoke: Gluster Build System <jenkins@build.gluster.org>

> NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>

> CentOS-regression: Gluster Build System <jenkins@build.gluster.org>

> Reviewed-by: Aravinda VK <avishwan@redhat.com>

(cherry picked from commit dbdf56ae3f87fc2765ad9cb528c1cfbadfde96ec)

Change-Id: I063ef5459916f711503881ade5c4fc32374edad0

BUG: 1450722

Signed-off-by: Milind Changire <mchangir@redhat.com>

Reviewed-on: https://code.engineering.redhat.com/gerrit/108769

Reviewed-by: Atin Mukherjee <amukherj@redhat.com>

---

 tools/glusterfind/src/brickfind.py |  2 +-

 tools/glusterfind/src/changelog.py | 27 ++++++++++++++-----------

 tools/glusterfind/src/main.py      | 40 ++++++++++++++++++++++++++------------

 3 files changed, 45 insertions(+), 24 deletions(-)

diff --git a/tools/glusterfind/src/brickfind.py b/tools/glusterfind/src/brickfind.py

index 439c881..e914bac 100644

--- a/tools/glusterfind/src/brickfind.py

+++ b/tools/glusterfind/src/brickfind.py

@@ -62,9 +62,9 @@ def _get_args():

     parser.add_argument("session", help="Session Name")

     parser.add_argument("volume", help="Volume Name")

+    parser.add_argument("node", help="Node Name")

     parser.add_argument("brick", help="Brick Name")

     parser.add_argument("outfile", help="Output File")

-    parser.add_argument("start", help="Start Time", type=float)

     parser.add_argument("tag", help="Tag to prefix file name with")

     parser.add_argument("--only-query", help="Only query, No session update",

                         action="store_true")

diff --git a/tools/glusterfind/src/changelog.py b/tools/glusterfind/src/changelog.py

index 721b8d0..ec38bfa 100644

--- a/tools/glusterfind/src/changelog.py

+++ b/tools/glusterfind/src/changelog.py

@@ -40,8 +40,6 @@ history_turn_time = 0

 logger = logging.getLogger()

-

-

 def pgfid_to_path(brick, changelog_data):

     """

     For all the pgfids in table, converts into path using recursive

@@ -94,7 +92,7 @@ def populate_pgfid_and_inodegfid(brick, changelog_data):

                 path = symlink_gfid_to_path(brick, gfid)

                 path = output_path_prepare(path, args)

                 changelog_data.gfidpath_update({"path1": path},

-                                                {"gfid": gfid})

+                                               {"gfid": gfid})

             except (IOError, OSError) as e:

                 logger.warn("Error converting to path: %s" % e)

                 continue

@@ -160,10 +158,10 @@ def gfid_to_path_using_pgfid(brick, changelog_data, args):

         try:

             path = symlink_gfid_to_path(brick, row[0])

             find(os.path.join(brick, path),

-                callback_func=output_callback,

-                filter_func=inode_filter,

-                ignore_dirs=ignore_dirs,

-                subdirs_crawl=False)

+                 callback_func=output_callback,

+                 filter_func=inode_filter,

+                 ignore_dirs=ignore_dirs,

+                 subdirs_crawl=False)

         except (IOError, OSError) as e:

             logger.warn("Error converting to path: %s" % e)

             continue

@@ -272,8 +270,8 @@ def get_changes(brick, hash_dir, log_file, start, end, args):

         actual_end = libgfchangelog.cl_history_changelog(

             cl_path, start, end, CHANGELOGAPI_NUM_WORKERS)

     except libgfchangelog.ChangelogException as e:

-        fail("%s Historical Changelogs not available: %s" % (brick, e),

-             logger=logger)

+        fail("%s: %s Historical Changelogs not available: %s" %

+             (args.node, brick, e), logger=logger)

     try:

         # scan followed by getchanges till scan returns zero.

@@ -296,7 +294,7 @@ def get_changes(brick, hash_dir, log_file, start, end, args):

                     libgfchangelog.cl_history_done(change)

                 except IOError as e:

                     logger.warn("Error parsing changelog file %s: %s" %

-                        (change, e))

+                                (change, e))

             changelog_data.commit()

     except libgfchangelog.ChangelogException as e:

@@ -350,9 +348,11 @@ def _get_args():

     parser.add_argument("session", help="Session Name")

     parser.add_argument("volume", help="Volume Name")

+    parser.add_argument("node", help="Node Name")

     parser.add_argument("brick", help="Brick Name")

     parser.add_argument("outfile", help="Output File")

     parser.add_argument("start", help="Start Time", type=int)

+    parser.add_argument("end", help="End Time", type=int)

     parser.add_argument("--only-query", help="Query mode only (no session)",

                         action="store_true")

     parser.add_argument("--debug", help="Debug", action="store_true")

@@ -387,6 +387,7 @@ if __name__ == "__main__":

     if args.only_query:

         start = args.start

+        end = args.end

     else:

         try:

             with open(status_file) as f:

@@ -394,7 +395,11 @@ if __name__ == "__main__":

         except (ValueError, OSError, IOError):

             start = args.start

-    end = int(time.time()) - get_changelog_rollover_time(args.volume)

+    # end time is optional; so a -1 may be sent to use the default method of

+    # identifying the end time

+    if end == -1:

+        end = int(time.time()) - get_changelog_rollover_time(args.volume)

+

     logger.info("%s Started Changelog Crawl - Start: %s End: %s" % (args.brick,

                                                                     start,

                                                                     end))

diff --git a/tools/glusterfind/src/main.py b/tools/glusterfind/src/main.py

index 8e31a07..df49e3d 100644

--- a/tools/glusterfind/src/main.py

+++ b/tools/glusterfind/src/main.py

@@ -139,9 +139,11 @@ def run_cmd_nodes(task, args, **kwargs):

             cmd = [change_detector,

                    args.session,

                    args.volume,

+                   host,

                    brick,

-                   node_outfile,

-                   str(kwargs.get("start"))] + \

+                   node_outfile] + \

+                ([str(kwargs.get("start")), str(kwargs.get("end"))]

+                    if not args.full else []) + \

                 ([tag] if tag is not None else []) + \

                 ["--output-prefix", args.output_prefix] + \

                 (["--debug"] if args.debug else []) + \

@@ -174,9 +176,11 @@ def run_cmd_nodes(task, args, **kwargs):

             cmd = [change_detector,

                    args.session,

                    args.volume,

+                   host,

                    brick,

-                   node_outfile,

-                   str(kwargs.get("start"))] + \

+                   node_outfile] + \

+                ([str(kwargs.get("start")), str(kwargs.get("end"))]

+                    if not args.full else []) + \

                 ([tag] if tag is not None else []) + \

                 ["--only-query"] + \

                 ["--output-prefix", args.output_prefix] + \

@@ -353,6 +357,8 @@ def _get_args():

                               action=StoreAbsPath)

     parser_query.add_argument("--since-time", help="UNIX epoch time since "

                               "which listing is required", type=int)

+    parser_query.add_argument("--end-time", help="UNIX epoch time upto "

+                              "which listing is required", type=int)

     parser_query.add_argument("--no-encode",

                               help="Do not encode path in output file",

                               action="store_true")

@@ -560,27 +566,37 @@ def mode_query(session_dir, args):

     enable_volume_options(args)

     # Test options

-    if not args.since_time and not args.full:

-        fail("Please specify either --since-time or --full", logger=logger)

+    if not args.since_time and not args.end_time and not args.full:

+        fail("Please specify either {--since-time and optionally --end-time} "

+             "or --full", logger=logger)

-    if args.since_time and args.full:

-        fail("Please specify either --since-time or --full, but not both",

+    if args.since_time and args.end_time and args.full:

+        fail("Please specify either {--since-time and optionally --end-time} "

+             "or --full, but not both",

              logger=logger)

+    if args.end_time and not args.since_time:

+        fail("Please specify --since-time as well", logger=logger)

+

     # Start query command processing

+    start = -1

+    end = -1

     if args.since_time:

         start = args.since_time

+        if args.end_time:

+            end = args.end_time

     else:

         start = 0  # --full option is handled separately

     logger.debug("Query is called - Session: %s, Volume: %s, "

-                 "Start time: %s"

-                 % ("default", args.volume, start))

+                 "Start time: %s, End time: %s"

+                 % ("default", args.volume, start, end))

     prefix = datetime.now().strftime("%Y%m%d-%H%M%S-%f-")

     gtmpfilename = prefix + next(tempfile._get_candidate_names())

-    run_cmd_nodes("query", args, start=start, tmpfilename=gtmpfilename)

+    run_cmd_nodes("query", args, start=start, end=end,

+                  tmpfilename=gtmpfilename)

     # Merger

     if args.full:

@@ -639,7 +655,7 @@ def mode_pre(session_dir, args):

     prefix = datetime.now().strftime("%Y%m%d-%H%M%S-%f-")

     gtmpfilename = prefix + next(tempfile._get_candidate_names())

-    run_cmd_nodes("pre", args, start=start, tmpfilename=gtmpfilename)

+    run_cmd_nodes("pre", args, start=start, end=-1, tmpfilename=gtmpfilename)

     # Merger

     if args.full:

-- 

1.8.3.1