commit 6596f1121b89162f96d1e1825c2905b83b59bec1
Author: Dave Anderson <anderson@redhat.com>
Date: Wed Jul 11 16:25:59 2018 -0400
The existing "list" command uses a hash table to detect duplicate
items as it traverses the list. The hash table approach has worked
well for many years. However, with increasing memory sizes and list
sizes, the overhead of the hash table can be substantial, often
leading to commands running for a very long time. For large lists,
we have found that the existing hash based approach may slow the
system to a crawl and possibly never complete. You can turn off
the hash with "set hash off" but then there is no loop detection; in
that case, loop detection must be done manually after dumping the
list to disk or some other method. This patch is an implementation
of the cycle detection algorithm from R. P. Brent as an alternative
algorithm for the "list" command. The algorithm both avoids the
overhead of the hash table and yet is able to detect a loop. In
addition, further loop characteristics are printed, such as the
distance to the start of the loop as well as the loop length.
An excellent description of the algorithm can be found here on
the crash-utility mailing list:
https://www.redhat.com/archives/crash-utility/2018-July/msg00019.html
A new "list -B" option has been added to the "list" command to
invoke this new algorithm rather than using the hash table. In
addition to low memory usage, the output of the list command is
slightly different when a loop is detected. In addition to printing
the first duplicate entry, the length of the loop, and the distance
to the loop is output.
(dwysocha@redhat.com)
diff --git a/defs.h b/defs.h
index b05aecc..5af82be 100644
--- a/defs.h
+++ b/defs.h
@@ -2491,6 +2491,7 @@ struct list_data { /* generic structure used by do_list() to walk */
#define CALLBACK_RETURN (VERBOSE << 12)
#define LIST_PARSE_MEMBER (VERBOSE << 13)
#define LIST_READ_MEMBER (VERBOSE << 14)
+#define LIST_BRENT_ALGO (VERBOSE << 15)
struct tree_data {
ulong flags;
@@ -4944,6 +4945,7 @@ char *shift_string_right(char *, int);
int bracketed(char *, char *, int);
void backspace(int);
int do_list(struct list_data *);
+int do_list_no_hash(struct list_data *);
struct radix_tree_ops {
void (*entry)(ulong node, ulong slot, const char *path,
ulong index, void *private);
diff --git a/help.c b/help.c
index 638c6ec..54bf9b4 100644
--- a/help.c
+++ b/help.c
@@ -5724,7 +5724,7 @@ char *help__list[] = {
"list",
"linked list",
"[[-o] offset][-e end][-[s|S] struct[.member[,member] [-l offset]] -[x|d]]"
-"\n [-r|-h|-H] start",
+"\n [-r|-B] [-h|-H] start",
" ",
" This command dumps the contents of a linked list. The entries in a linked",
" list are typically data structures that are tied together in one of two",
@@ -5822,6 +5822,12 @@ char *help__list[] = {
" -r For a list linked with list_head structures, traverse the list",
" in the reverse order by using the \"prev\" pointer instead",
" of \"next\".",
+" -B Use the algorithm from R. P. Brent to detect loops instead of",
+" using a hash table. This algorithm uses a tiny fixed amount of",
+" memory and so is especially helpful for longer lists. The output",
+" is slightly different than the normal list output as it will",
+" print the length of the loop, the start of the loop, and the",
+" first duplicate in the list.",
" ",
" The meaning of the \"start\" argument, which can be expressed symbolically,",
" in hexadecimal format, or an expression evaluating to an address, depends",
diff --git a/tools.c b/tools.c
index 1a83643..634aec6 100644
--- a/tools.c
+++ b/tools.c
@@ -3266,9 +3266,12 @@ cmd_list(void)
BZERO(ld, sizeof(struct list_data));
struct_list_offset = 0;
- while ((c = getopt(argcnt, args, "Hhrs:S:e:o:xdl:")) != EOF) {
+ while ((c = getopt(argcnt, args, "BHhrs:S:e:o:xdl:")) != EOF) {
switch(c)
{
+ case 'B':
+ ld->flags |= LIST_BRENT_ALGO;
+ break;
case 'H':
ld->flags |= LIST_HEAD_FORMAT;
ld->flags |= LIST_HEAD_POINTER;
@@ -3516,9 +3519,13 @@ next_arg:
ld->flags &= ~(LIST_OFFSET_ENTERED|LIST_START_ENTERED);
ld->flags |= VERBOSE;
- hq_open();
- c = do_list(ld);
- hq_close();
+ if (ld->flags & LIST_BRENT_ALGO)
+ c = do_list_no_hash(ld);
+ else {
+ hq_open();
+ c = do_list(ld);
+ hq_close();
+ }
if (ld->structname_args)
FREEBUF(ld->structname);
@@ -3862,6 +3869,283 @@ do_list(struct list_data *ld)
return count;
}
+static void
+do_list_debug_entry(struct list_data *ld)
+{
+ int i, others;
+
+ if (CRASHDEBUG(1)) {
+ others = 0;
+ console(" flags: %lx (", ld->flags);
+ if (ld->flags & VERBOSE)
+ console("%sVERBOSE", others++ ? "|" : "");
+ if (ld->flags & LIST_OFFSET_ENTERED)
+ console("%sLIST_OFFSET_ENTERED", others++ ? "|" : "");
+ if (ld->flags & LIST_START_ENTERED)
+ console("%sLIST_START_ENTERED", others++ ? "|" : "");
+ if (ld->flags & LIST_HEAD_FORMAT)
+ console("%sLIST_HEAD_FORMAT", others++ ? "|" : "");
+ if (ld->flags & LIST_HEAD_POINTER)
+ console("%sLIST_HEAD_POINTER", others++ ? "|" : "");
+ if (ld->flags & RETURN_ON_DUPLICATE)
+ console("%sRETURN_ON_DUPLICATE", others++ ? "|" : "");
+ if (ld->flags & RETURN_ON_LIST_ERROR)
+ console("%sRETURN_ON_LIST_ERROR", others++ ? "|" : "");
+ if (ld->flags & RETURN_ON_LIST_ERROR)
+ console("%sRETURN_ON_LIST_ERROR", others++ ? "|" : "");
+ if (ld->flags & LIST_STRUCT_RADIX_10)
+ console("%sLIST_STRUCT_RADIX_10", others++ ? "|" : "");
+ if (ld->flags & LIST_STRUCT_RADIX_16)
+ console("%sLIST_STRUCT_RADIX_16", others++ ? "|" : "");
+ if (ld->flags & LIST_ALLOCATE)
+ console("%sLIST_ALLOCATE", others++ ? "|" : "");
+ if (ld->flags & LIST_CALLBACK)
+ console("%sLIST_CALLBACK", others++ ? "|" : "");
+ if (ld->flags & CALLBACK_RETURN)
+ console("%sCALLBACK_RETURN", others++ ? "|" : "");
+ console(")\n");
+ console(" start: %lx\n", ld->start);
+ console(" member_offset: %ld\n", ld->member_offset);
+ console(" list_head_offset: %ld\n", ld->list_head_offset);
+ console(" end: %lx\n", ld->end);
+ console(" searchfor: %lx\n", ld->searchfor);
+ console(" structname_args: %lx\n", ld->structname_args);
+ if (!ld->structname_args)
+ console(" structname: (unused)\n");
+ for (i = 0; i < ld->structname_args; i++)
+ console(" structname[%d]: %s\n", i, ld->structname[i]);
+ console(" header: %s\n", ld->header);
+ console(" list_ptr: %lx\n", (ulong)ld->list_ptr);
+ console(" callback_func: %lx\n", (ulong)ld->callback_func);
+ console(" callback_data: %lx\n", (ulong)ld->callback_data);
+ console("struct_list_offset: %lx\n", ld->struct_list_offset);
+ }
+}
+
+
+static void
+do_list_output_struct(struct list_data *ld, ulong next, ulong offset,
+ unsigned int radix, struct req_entry **e)
+{
+ int i;
+
+ for (i = 0; i < ld->structname_args; i++) {
+ switch (count_chars(ld->structname[i], '.'))
+ {
+ case 0:
+ dump_struct(ld->structname[i],
+ next - offset, radix);
+ break;
+ default:
+ if (ld->flags & LIST_PARSE_MEMBER)
+ dump_struct_members(ld, i, next);
+ else if (ld->flags & LIST_READ_MEMBER)
+ dump_struct_members_fast(e[i],
+ radix, next - offset);
+ break;
+ }
+ }
+}
+
+static int
+do_list_no_hash_readmem(struct list_data *ld, ulong *next_ptr,
+ ulong readflag)
+{
+ if (!readmem(*next_ptr + ld->member_offset, KVADDR, next_ptr,
+ sizeof(void *), "list entry", readflag)) {
+ error(INFO, "\ninvalid list entry: %lx\n", *next_ptr);
+ return -1;
+ }
+ return 0;
+}
+
+static ulong brent_x; /* tortoise */
+static ulong brent_y; /* hare */
+static ulong brent_r; /* power */
+static ulong brent_lambda; /* loop length */
+static ulong brent_mu; /* distance to start of loop */
+static ulong brent_loop_detect;
+static ulong brent_loop_exit;
+/*
+ * 'ptr': representative of x or y; modified on return
+ */
+static int
+brent_f(ulong *ptr, struct list_data *ld, ulong readflag)
+{
+ return do_list_no_hash_readmem(ld, ptr, readflag);
+}
+
+/*
+ * Similar to do_list() but without the hash_table or LIST_ALLOCATE.
+ * Useful for the 'list' command and other callers needing faster list
+ * enumeration.
+ */
+int
+do_list_no_hash(struct list_data *ld)
+{
+ ulong next, last, first, offset;
+ ulong searchfor, readflag;
+ int i, count, ret;
+ unsigned int radix;
+ struct req_entry **e = NULL;
+
+ do_list_debug_entry(ld);
+
+ count = 0;
+ searchfor = ld->searchfor;
+ ld->searchfor = 0;
+ if (ld->flags & LIST_STRUCT_RADIX_10)
+ radix = 10;
+ else if (ld->flags & LIST_STRUCT_RADIX_16)
+ radix = 16;
+ else
+ radix = 0;
+ next = ld->start;
+
+ readflag = ld->flags & RETURN_ON_LIST_ERROR ?
+ (RETURN_ON_ERROR|QUIET) : FAULT_ON_ERROR;
+
+ if (!readmem(next + ld->member_offset, KVADDR, &first, sizeof(void *),
+ "first list entry", readflag)) {
+ error(INFO, "\ninvalid list entry: %lx\n", next);
+ return -1;
+ }
+
+ if (ld->header)
+ fprintf(fp, "%s", ld->header);
+
+ offset = ld->list_head_offset + ld->struct_list_offset;
+
+ if (ld->structname && (ld->flags & LIST_READ_MEMBER)) {
+ e = (struct req_entry **)GETBUF(sizeof(*e) * ld->structname_args);
+ for (i = 0; i < ld->structname_args; i++)
+ e[i] = fill_member_offsets(ld->structname[i]);
+ }
+
+ brent_loop_detect = brent_loop_exit = 0;
+ brent_lambda = 0;
+ brent_r = 2;
+ brent_x = brent_y = next;
+ ret = brent_f(&brent_y, ld, readflag);
+ if (ret == -1)
+ return -1;
+ while (1) {
+ if (!brent_loop_detect && ld->flags & VERBOSE) {
+ fprintf(fp, "%lx\n", next - ld->list_head_offset);
+ if (ld->structname) {
+ do_list_output_struct(ld, next, offset, radix, e);
+ }
+ }
+
+ if (next && brent_loop_exit) {
+ if (ld->flags &
+ (RETURN_ON_DUPLICATE|RETURN_ON_LIST_ERROR)) {
+ error(INFO, "\nduplicate list entry: %lx\n",
+ brent_x);
+ return -1;
+ }
+ error(FATAL, "\nduplicate list entry: %lx\n", brent_x);
+ }
+
+ if ((searchfor == next) ||
+ (searchfor == (next - ld->list_head_offset)))
+ ld->searchfor = searchfor;
+
+ count++;
+ last = next;
+
+ if ((ld->flags & LIST_CALLBACK) &&
+ ld->callback_func((void *)(next - ld->list_head_offset),
+ ld->callback_data) && (ld->flags & CALLBACK_RETURN))
+ break;
+
+ ret = do_list_no_hash_readmem(ld, &next, readflag);
+ if (ret == -1)
+ return -1;
+
+ if (!brent_loop_detect) {
+ if (brent_x == brent_y) {
+ brent_loop_detect = 1;
+ error(INFO, "loop detected, loop length: %lx\n", brent_lambda);
+ /* reset x and y to start; advance y loop length */
+ brent_mu = 0;
+ brent_x = brent_y = ld->start;
+ while (brent_lambda--) {
+ ret = brent_f(&brent_y, ld, readflag);
+ if (ret == -1)
+ return -1;
+ }
+ } else {
+ if (brent_r == brent_lambda) {
+ brent_x = brent_y;
+ brent_r *= 2;
+ brent_lambda = 0;
+ }
+ brent_y = next;
+ brent_lambda++;
+ }
+ } else {
+ if (!brent_loop_exit && brent_x == brent_y) {
+ brent_loop_exit = 1;
+ error(INFO, "length from start to loop: %lx",
+ brent_mu);
+ } else {
+ ret = brent_f(&brent_x, ld, readflag);
+ if (ret == -1)
+ return -1;
+ ret = brent_f(&brent_y, ld, readflag);
+ if (ret == -1)
+ return -1;
+ brent_mu++;
+ }
+ }
+
+ if (next == 0) {
+ if (ld->flags & LIST_HEAD_FORMAT) {
+ error(INFO, "\ninvalid list entry: 0\n");
+ return -1;
+ }
+ if (CRASHDEBUG(1))
+ console("do_list end: next:%lx\n", next);
+
+ break;
+ }
+
+ if (next == ld->end) {
+ if (CRASHDEBUG(1))
+ console("do_list end: next:%lx == end:%lx\n",
+ next, ld->end);
+ break;
+ }
+
+ if (next == ld->start) {
+ if (CRASHDEBUG(1))
+ console("do_list end: next:%lx == start:%lx\n",
+ next, ld->start);
+ break;
+ }
+
+ if (next == last) {
+ if (CRASHDEBUG(1))
+ console("do_list end: next:%lx == last:%lx\n",
+ next, last);
+ break;
+ }
+
+ if ((next == first) && (count != 1)) {
+ if (CRASHDEBUG(1))
+ console("do_list end: next:%lx == first:%lx (count %d)\n",
+ next, last, count);
+ break;
+ }
+ }
+
+ if (CRASHDEBUG(1))
+ console("do_list count: %d\n", count);
+
+ return count;
+}
+
/*
* Issue a dump_struct_member() call for one or more structure
* members. Multiple members are passed in a comma-separated