69ce21
--- crash-trace-command-2.0/trace.c.orig
69ce21
+++ crash-trace-command-2.0/trace.c
69ce21
@@ -38,6 +38,10 @@ static int trace_buffer_available;
69ce21
  * max_buffer is supported
69ce21
  */
69ce21
 static int max_buffer_available;
69ce21
+/*
69ce21
+ * multiple trace instances are supported
69ce21
+ */
69ce21
+static int multiple_instances_available;
69ce21
 
69ce21
 #define koffset(struct, member) struct##_##member##_offset
69ce21
 
69ce21
@@ -78,6 +82,8 @@ static int koffset(ftrace_event_field, o
69ce21
 static int koffset(ftrace_event_field, size);
69ce21
 static int koffset(ftrace_event_field, is_signed);
69ce21
 
69ce21
+static int koffset(trace_array, name);
69ce21
+
69ce21
 static int koffset(POINTER_SYM, POINTER) = 0;
69ce21
 
69ce21
 struct ring_buffer_per_cpu {
69ce21
@@ -101,16 +107,25 @@ struct ring_buffer_per_cpu {
69ce21
 };
69ce21
 
69ce21
 static ulong global_trace;
69ce21
-static ulong global_trace_buffer;
69ce21
-static ulong global_max_buffer;
69ce21
-static ulong global_ring_buffer;
69ce21
-static unsigned global_pages;
69ce21
-static struct ring_buffer_per_cpu *global_buffers;
69ce21
-
69ce21
 static ulong max_tr_trace;
69ce21
-static ulong max_tr_ring_buffer;
69ce21
-static unsigned max_tr_pages;
69ce21
-static struct ring_buffer_per_cpu *max_tr_buffers;
69ce21
+
69ce21
+struct trace_instance {
69ce21
+	char name[NAME_MAX + 1];
69ce21
+	ulong trace_buffer;
69ce21
+	ulong max_buffer;
69ce21
+	ulong ring_buffer;
69ce21
+	unsigned pages;
69ce21
+	struct ring_buffer_per_cpu *buffers;
69ce21
+
69ce21
+	ulong max_tr_ring_buffer;
69ce21
+	unsigned max_tr_pages;
69ce21
+	struct ring_buffer_per_cpu *max_tr_buffers;
69ce21
+};
69ce21
+
69ce21
+static ulong ftrace_trace_arrays;
69ce21
+static struct trace_instance global_trace_instance;
69ce21
+static struct trace_instance *trace_instances = NULL;
69ce21
+static int instance_count;
69ce21
 
69ce21
 static ulong ftrace_events;
69ce21
 static ulong current_trace;
69ce21
@@ -229,6 +244,9 @@ static int init_offsets(void)
69ce21
 	init_offset(ftrace_event_field, size);
69ce21
 	init_offset(ftrace_event_field, is_signed);
69ce21
 
69ce21
+	if (MEMBER_EXISTS("trace_array", "name"))
69ce21
+		init_offset(trace_array, name);
69ce21
+
69ce21
 	return 0;
69ce21
 #undef init_offset
69ce21
 }
69ce21
@@ -435,61 +453,140 @@ out_fail:
69ce21
 	return -1;
69ce21
 }
69ce21
 
69ce21
-static int ftrace_int_global_trace(void)
69ce21
+static int ftrace_init_trace(struct trace_instance *ti, ulong instance_addr)
69ce21
 {
69ce21
 	if (trace_buffer_available) {
69ce21
-		global_trace_buffer = global_trace + koffset(trace_array, trace_buffer);
69ce21
-		read_value(global_ring_buffer, global_trace_buffer, trace_buffer, buffer);
69ce21
+		ti->trace_buffer = instance_addr +
69ce21
+				koffset(trace_array, trace_buffer);
69ce21
+		read_value(ti->ring_buffer, ti->trace_buffer,
69ce21
+				trace_buffer, buffer);
69ce21
+
69ce21
+		if (max_buffer_available) {
69ce21
+			ti->max_buffer = instance_addr +
69ce21
+					koffset(trace_array, max_buffer);
69ce21
+			read_value(ti->max_tr_ring_buffer, ti->max_buffer,
69ce21
+					trace_buffer, buffer);
69ce21
+		}
69ce21
 	} else {
69ce21
-		read_value(global_ring_buffer, global_trace, trace_array, buffer);
69ce21
-		read_value(global_pages, global_ring_buffer, ring_buffer, pages);
69ce21
+		read_value(ti->ring_buffer, instance_addr, trace_array, buffer);
69ce21
+		read_value(ti->pages, ti->ring_buffer, ring_buffer, pages);
69ce21
+
69ce21
+		read_value(ti->max_tr_ring_buffer, max_tr_trace, trace_array, buffer);
69ce21
+		if (ti->max_tr_ring_buffer)
69ce21
+			read_value(ti->max_tr_pages, ti->max_tr_ring_buffer, ring_buffer, pages);
69ce21
 	}
69ce21
 
69ce21
-	global_buffers = calloc(sizeof(*global_buffers), nr_cpu_ids);
69ce21
-	if (global_buffers == NULL)
69ce21
+	ti->buffers = calloc(sizeof(*ti->buffers), nr_cpu_ids);
69ce21
+	if (ti->buffers == NULL)
69ce21
+		goto out_fail;
69ce21
+
69ce21
+	if (ftrace_init_buffers(ti->buffers, ti->ring_buffer,
69ce21
+			ti->pages) < 0)
69ce21
+		goto out_fail;
69ce21
+
69ce21
+	if (!ti->max_tr_ring_buffer)
69ce21
+		return 0;
69ce21
+
69ce21
+	ti->max_tr_buffers = calloc(sizeof(*ti->max_tr_buffers), nr_cpu_ids);
69ce21
+	if (ti->max_tr_buffers == NULL)
69ce21
 		goto out_fail;
69ce21
 
69ce21
-	if (ftrace_init_buffers(global_buffers, global_ring_buffer,
69ce21
-			global_pages) < 0)
69ce21
+	if (ftrace_init_buffers(ti->max_tr_buffers, ti->max_tr_ring_buffer,
69ce21
+			ti->max_tr_pages) < 0)
69ce21
 		goto out_fail;
69ce21
 
69ce21
 	return 0;
69ce21
 
69ce21
 out_fail:
69ce21
-	free(global_buffers);
69ce21
+	free(ti->max_tr_buffers);
69ce21
+	free(ti->buffers);
69ce21
 	return -1;
69ce21
 }
69ce21
 
69ce21
-static int ftrace_int_max_tr_trace(void)
69ce21
+static void ftrace_destroy_all_instance_buffers()
69ce21
 {
69ce21
-	if (trace_buffer_available) {
69ce21
-		if (!max_buffer_available)
69ce21
-			return 0;
69ce21
+	int i;
69ce21
 
69ce21
-		global_max_buffer = global_trace + koffset(trace_array, max_buffer);
69ce21
-		read_value(max_tr_ring_buffer, global_max_buffer, trace_buffer, buffer);
69ce21
-	} else {
69ce21
-		read_value(max_tr_ring_buffer, max_tr_trace, trace_array, buffer);
69ce21
+	for (i = 0; i < instance_count; i++)
69ce21
+	{
69ce21
+		struct trace_instance *ti = &trace_instances[i];
69ce21
 
69ce21
-		if (!max_tr_ring_buffer)
69ce21
-			return 0;
69ce21
+		if (ti->max_tr_ring_buffer) {
69ce21
+			ftrace_destroy_buffers(ti->max_tr_buffers);
69ce21
+			free(ti->max_tr_buffers);
69ce21
+		}
69ce21
 
69ce21
-		read_value(max_tr_pages, max_tr_ring_buffer, ring_buffer, pages);
69ce21
+		ftrace_destroy_buffers(ti->buffers);
69ce21
+		free(ti->buffers);
69ce21
 	}
69ce21
+}
69ce21
 
69ce21
-	max_tr_buffers = calloc(sizeof(*max_tr_buffers), nr_cpu_ids);
69ce21
-	if (max_tr_buffers == NULL)
69ce21
-		goto out_fail;
69ce21
+static void ftrace_destroy_instances()
69ce21
+{
69ce21
+	ftrace_destroy_all_instance_buffers();
69ce21
+	free(trace_instances);
69ce21
+}
69ce21
 
69ce21
-	if (ftrace_init_buffers(max_tr_buffers, max_tr_ring_buffer,
69ce21
-			max_tr_pages) < 0)
69ce21
-		goto out_fail;
69ce21
+static int ftrace_init_instances()
69ce21
+{
69ce21
+	int i;
69ce21
+	struct trace_instance *ti;
69ce21
+	struct list_data list_data;
69ce21
+	struct list_data *ld = &list_data;
69ce21
+
69ce21
+	if (!multiple_instances_available)
69ce21
+		return 0;
69ce21
+
69ce21
+	BZERO(ld, sizeof(struct list_data));
69ce21
+	ld->start = ftrace_trace_arrays;
69ce21
+	ld->end = global_trace;
69ce21
+	ld->flags = LIST_ALLOCATE;
69ce21
+	instance_count = do_list(ld);
69ce21
+
69ce21
+	/* The do_list count includes the list_head, which is not a
69ce21
+	 * proper instance */
69ce21
+	instance_count--;
69ce21
+	if (instance_count <= 0)
69ce21
+		return 0;
69ce21
+
69ce21
+	trace_instances = calloc(sizeof(struct trace_instance), instance_count);
69ce21
+
69ce21
+	/* We start i at 1 to skip over the list_head and continue to the last
69ce21
+	 * instance, which lies at index instance_count */
69ce21
+	for (i = 1; i <= instance_count; i++)
69ce21
+	{
69ce21
+		ulong instance_ptr;
69ce21
+		ulong name_addr;
69ce21
+		int ret;
69ce21
+
69ce21
+		ti = &trace_instances[i-1];
69ce21
+		instance_ptr = ld->list_ptr[i];
69ce21
+		read_value(name_addr, instance_ptr, trace_array, name);
69ce21
+		if (!name_addr)
69ce21
+		{
69ce21
+			console("Instance name is NULL\n");
69ce21
+		}
69ce21
+		else if (!read_string(name_addr, ti->name, sizeof(ti->name)))
69ce21
+		{
69ce21
+			console("Failed to read instance name at address %p\n", (void*)name_addr);
69ce21
+			goto out_fail;
69ce21
+		}
69ce21
+
69ce21
+		ret = ftrace_init_trace(ti, instance_ptr);
69ce21
+		if (ret < 0)
69ce21
+			goto out_fail;
69ce21
+	}
69ce21
+	FREEBUF(ld->list_ptr);
69ce21
 
69ce21
 	return 0;
69ce21
 
69ce21
 out_fail:
69ce21
-	free(max_tr_buffers);
69ce21
-	max_tr_ring_buffer = 0;
69ce21
+	/* We've already freed the current instance's trace buffer info, so
69ce21
+	 * we'll clear that out to avoid double freeing in
69ce21
+	 * ftrace_destroy_instances() */
69ce21
+	BZERO(ti, sizeof(struct trace_instance));
69ce21
+	ftrace_destroy_instances();
69ce21
+
69ce21
 	return -1;
69ce21
 }
69ce21
 
69ce21
@@ -504,7 +601,7 @@ static int ftrace_init_current_tracer(vo
69ce21
 	} else {
69ce21
 		read_value(addr, current_trace, POINTER_SYM, POINTER);
69ce21
 	}
69ce21
-	
69ce21
+
69ce21
 	read_value(addr, addr, tracer, name);
69ce21
 	read_string(addr, tmp, 128);
69ce21
 
69ce21
@@ -524,9 +621,11 @@ static int ftrace_init(void)
69ce21
 	struct syment *sym_max_tr_trace;
69ce21
 	struct syment *sym_ftrace_events;
69ce21
 	struct syment *sym_current_trace;
69ce21
+	struct syment *sym_ftrace_trace_arrays;
69ce21
 
69ce21
 	sym_global_trace = symbol_search("global_trace");
69ce21
 	sym_ftrace_events = symbol_search("ftrace_events");
69ce21
+	sym_ftrace_trace_arrays = symbol_search("ftrace_trace_arrays");
69ce21
 
69ce21
 	if (sym_global_trace == NULL || sym_ftrace_events == NULL)
69ce21
 		return -1;
69ce21
@@ -534,6 +633,13 @@ static int ftrace_init(void)
69ce21
 	global_trace = sym_global_trace->value;
69ce21
 	ftrace_events = sym_ftrace_events->value;
69ce21
 
69ce21
+	if (sym_ftrace_trace_arrays)
69ce21
+	{
69ce21
+		multiple_instances_available = 1;
69ce21
+		ftrace_trace_arrays = sym_ftrace_trace_arrays->value;
69ce21
+	}
69ce21
+
69ce21
+
69ce21
 	if (MEMBER_EXISTS("trace_array", "current_trace")) {
69ce21
 		encapsulated_current_trace = 1;
69ce21
 	} else {
69ce21
@@ -564,28 +670,31 @@ static int ftrace_init(void)
69ce21
 		return -1;
69ce21
 	print_offsets();
69ce21
 
69ce21
-	if (ftrace_int_global_trace() < 0)
69ce21
+	if (ftrace_init_trace(&global_trace_instance, global_trace) < 0)
69ce21
 		goto out_0;
69ce21
 
69ce21
-	ftrace_int_max_tr_trace();
69ce21
+	if (ftrace_init_instances() < 0)
69ce21
+		goto out_1;
69ce21
 
69ce21
 	if (ftrace_init_event_types() < 0)
69ce21
-		goto out_1;
69ce21
+		goto out_2;
69ce21
 
69ce21
 	if (ftrace_init_current_tracer() < 0)
69ce21
-		goto out_2;
69ce21
+		goto out_3;
69ce21
 
69ce21
 	return 0;
69ce21
 
69ce21
-out_2:
69ce21
+out_3:
69ce21
 	ftrace_destroy_event_types();
69ce21
+out_2:
69ce21
+	ftrace_destroy_instances();
69ce21
 out_1:
69ce21
-	if (max_tr_ring_buffer) {
69ce21
-		ftrace_destroy_buffers(max_tr_buffers);
69ce21
-		free(max_tr_buffers);
69ce21
+	if (global_trace_instance.max_tr_ring_buffer) {
69ce21
+		ftrace_destroy_buffers(global_trace_instance.max_tr_buffers);
69ce21
+		free(global_trace_instance.max_tr_buffers);
69ce21
 	}
69ce21
-	ftrace_destroy_buffers(global_buffers);
69ce21
-	free(global_buffers);
69ce21
+	ftrace_destroy_buffers(global_trace_instance.buffers);
69ce21
+	free(global_trace_instance.buffers);
69ce21
 out_0:
69ce21
 	return -1;
69ce21
 }
69ce21
@@ -595,13 +704,15 @@ static void ftrace_destroy(void)
69ce21
 	free(current_tracer_name);
69ce21
 	ftrace_destroy_event_types();
69ce21
 
69ce21
-	if (max_tr_ring_buffer) {
69ce21
-		ftrace_destroy_buffers(max_tr_buffers);
69ce21
-		free(max_tr_buffers);
69ce21
+	ftrace_destroy_instances();
69ce21
+
69ce21
+	if (global_trace_instance.max_tr_ring_buffer) {
69ce21
+		ftrace_destroy_buffers(global_trace_instance.max_tr_buffers);
69ce21
+		free(global_trace_instance.max_tr_buffers);
69ce21
 	}
69ce21
 
69ce21
-	ftrace_destroy_buffers(global_buffers);
69ce21
-	free(global_buffers);
69ce21
+	ftrace_destroy_buffers(global_trace_instance.buffers);
69ce21
+	free(global_trace_instance.buffers);
69ce21
 }
69ce21
 
69ce21
 static int ftrace_dump_page(int fd, ulong page, void *page_tmp)
69ce21
@@ -652,7 +763,8 @@ static int try_mkdir(const char *pathnam
69ce21
 	return 0;
69ce21
 }
69ce21
 
69ce21
-static int ftrace_dump_buffers(const char *per_cpu_path)
69ce21
+static int ftrace_dump_buffers(const char *per_cpu_path,
69ce21
+                                struct trace_instance *ti)
69ce21
 {
69ce21
 	int i;
69ce21
 	void *page_tmp;
69ce21
@@ -664,7 +776,7 @@ static int ftrace_dump_buffers(const cha
69ce21
 		return -1;
69ce21
 
69ce21
 	for (i = 0; i < nr_cpu_ids; i++) {
69ce21
-		struct ring_buffer_per_cpu *cpu_buffer = &global_buffers[i];
69ce21
+		struct ring_buffer_per_cpu *cpu_buffer = &ti->buffers[i];
69ce21
 
69ce21
 		if (!cpu_buffer->kaddr)
69ce21
 			continue;
69ce21
@@ -679,7 +791,7 @@ static int ftrace_dump_buffers(const cha
69ce21
 		if (fd < 0)
69ce21
 			goto out_fail;
69ce21
 
69ce21
-		ftrace_dump_buffer(fd, cpu_buffer, global_pages, page_tmp);
69ce21
+		ftrace_dump_buffer(fd, cpu_buffer, ti->pages, page_tmp);
69ce21
 		close(fd);
69ce21
 	}
69ce21
 
69ce21
@@ -1015,8 +1127,6 @@ static void ftrace_destroy_event_types(v
69ce21
 	free(ftrace_common_fields);
69ce21
 }
69ce21
 
69ce21
-#define TRACE_EVENT_FL_TRACEPOINT 0x40
69ce21
-
69ce21
 static
69ce21
 int ftrace_get_event_type_name(ulong call, char *name, int len)
69ce21
 {
69ce21
@@ -1024,34 +1134,35 @@ int ftrace_get_event_type_name(ulong cal
69ce21
 	static int name_offset;
69ce21
 	static int flags_offset;
69ce21
 	static int tp_name_offset;
69ce21
-	uint flags;
69ce21
+	static long tracepoint_flag;
69ce21
 
69ce21
+	uint flags;
69ce21
 	ulong name_addr;
69ce21
 
69ce21
 	if (inited)
69ce21
 		goto work;
69ce21
 
69ce21
-	inited = 1;
69ce21
-	name_offset = MAX(MEMBER_OFFSET("ftrace_event_call", "name"),
69ce21
-		MEMBER_OFFSET("trace_event_call", "name"));
69ce21
-	if (name_offset >= 0)
69ce21
-		goto work;
69ce21
-
69ce21
-	name_offset = MAX(ANON_MEMBER_OFFSET("ftrace_event_call", "name"),
69ce21
-		ANON_MEMBER_OFFSET("trace_event_call", "name"));
69ce21
-	if (name_offset < 0)
69ce21
-		return -1;
69ce21
+	name_offset = MAX(MEMBER_OFFSET("ftrace_event_call", "tp"),
69ce21
+		MEMBER_OFFSET("trace_event_call", "tp"));
69ce21
+	if (name_offset >= 0) {
69ce21
+		flags_offset = MAX(MEMBER_OFFSET("ftrace_event_call", "flags"),
69ce21
+			MEMBER_OFFSET("trace_event_call", "flags"));
69ce21
+		if (flags_offset < 0)
69ce21
+			return -1;
69ce21
 
69ce21
-	flags_offset = MAX(MEMBER_OFFSET("ftrace_event_call", "flags"),
69ce21
-		MEMBER_OFFSET("trace_event_call", "flags"));
69ce21
-	if (flags_offset < 0)
69ce21
-		return -1;
69ce21
+		tp_name_offset = MEMBER_OFFSET("tracepoint", "name");
69ce21
+		if (tp_name_offset < 0)
69ce21
+			return -1;
69ce21
 
69ce21
-	tp_name_offset = MEMBER_OFFSET("tracepoint", "name");
69ce21
-	if (tp_name_offset < 0)
69ce21
-		return -1;
69ce21
+		if (!enumerator_value("TRACE_EVENT_FL_TRACEPOINT", &tracepoint_flag))
69ce21
+			return -1;
69ce21
 
69ce21
-	inited = 2;
69ce21
+		inited = 2;
69ce21
+	} else {
69ce21
+		name_offset = MAX(MEMBER_OFFSET("ftrace_event_call", "name"),
69ce21
+			MEMBER_OFFSET("trace_event_call", "name"));
69ce21
+		inited = 1;
69ce21
+	}
69ce21
 
69ce21
 work:
69ce21
 	if (name_offset < 0)
69ce21
@@ -1067,7 +1178,7 @@ work:
69ce21
 			     RETURN_ON_ERROR))
69ce21
 			return -1;
69ce21
 
69ce21
-		if (flags & TRACE_EVENT_FL_TRACEPOINT) {
69ce21
+		if (flags & (uint)tracepoint_flag) {
69ce21
 			if (!readmem(name_addr + tp_name_offset, KVADDR,
69ce21
 				     &name_addr, sizeof(name_addr),
69ce21
 				     "read tracepoint name", RETURN_ON_ERROR))
69ce21
@@ -1476,26 +1587,72 @@ static int dump_kallsyms(const char *dum
69ce21
 
69ce21
 static int trace_cmd_data_output(int fd);
69ce21
 
69ce21
+#define	FTRACE_DUMP_SYMBOLS	(1 << 0)
69ce21
+#define	FTRACE_DUMP_META_DATA	(1 << 1)
69ce21
+
69ce21
+static int populate_ftrace_dir_tree(struct trace_instance *ti,
69ce21
+		char *root, uint flags)
69ce21
+{
69ce21
+	char path[PATH_MAX];
69ce21
+	int ret;
69ce21
+
69ce21
+	ret = mkdir(root, 0755);
69ce21
+	if (ret < 0) {
69ce21
+		if (errno == EEXIST)
69ce21
+			error(INFO, "mkdir: %s exists\n", root);
69ce21
+		return FALSE;
69ce21
+	}
69ce21
+
69ce21
+	snprintf(path, sizeof(path), "%s/per_cpu", root);
69ce21
+	if (try_mkdir(path, 0755) < 0)
69ce21
+		return FALSE;
69ce21
+
69ce21
+	if (ftrace_dump_buffers(path, ti) < 0)
69ce21
+		return FALSE;
69ce21
+
69ce21
+	if (flags & FTRACE_DUMP_META_DATA) {
69ce21
+		/* Dump event types */
69ce21
+		snprintf(path, sizeof(path), "%s/events", root);
69ce21
+		if (try_mkdir(path, 0755) < 0)
69ce21
+			return FALSE;
69ce21
+
69ce21
+		if (ftrace_dump_event_types(path) < 0)
69ce21
+			return FALSE;
69ce21
+
69ce21
+		/* Dump pids with corresponding cmdlines */
69ce21
+		if (dump_saved_cmdlines(root) < 0)
69ce21
+			return FALSE;
69ce21
+	}
69ce21
+
69ce21
+	if (flags & FTRACE_DUMP_SYMBOLS) {
69ce21
+		/* Dump all symbols of the kernel */
69ce21
+		dump_kallsyms(root);
69ce21
+	}
69ce21
+
69ce21
+	return TRUE;
69ce21
+}
69ce21
+
69ce21
 static void ftrace_dump(int argc, char *argv[])
69ce21
 {
69ce21
 	int c;
69ce21
-	int dump_meta_data = 0;
69ce21
-	int dump_symbols = 0;
69ce21
+	int i;
69ce21
+	uint flags = 0;
69ce21
 	char *dump_tracing_dir;
69ce21
-	char path[PATH_MAX];
69ce21
-	int ret;
69ce21
+	char instance_path[PATH_MAX];
69ce21
 
69ce21
         while ((c = getopt(argc, argv, "smt")) != EOF) {
69ce21
                 switch(c)
69ce21
 		{
69ce21
 		case 's':
69ce21
-			dump_symbols = 1;
69ce21
+			flags |= FTRACE_DUMP_SYMBOLS;
69ce21
 			break;
69ce21
 		case 'm':
69ce21
-			dump_meta_data = 1;
69ce21
+			flags |= FTRACE_DUMP_META_DATA;
69ce21
 			break;
69ce21
 		case 't':
69ce21
-			if (dump_symbols || dump_meta_data || argc - optind > 1)
69ce21
+			if (flags & FTRACE_DUMP_SYMBOLS ||
69ce21
+				flags & FTRACE_DUMP_META_DATA ||
69ce21
+				argc - optind > 1)
69ce21
 				cmd_usage(pc->curcmd, SYNOPSIS);
69ce21
 			else {
69ce21
 				char *trace_dat = "trace.dat";
69ce21
@@ -1526,38 +1683,34 @@ static void ftrace_dump(int argc, char *
69ce21
 		return;
69ce21
 	}
69ce21
 
69ce21
-	ret = mkdir(dump_tracing_dir, 0755);
69ce21
-	if (ret < 0) {
69ce21
-		if (errno == EEXIST)
69ce21
-			error(INFO, "mkdir: %s exists\n", dump_tracing_dir);
69ce21
+	if (!populate_ftrace_dir_tree(&global_trace_instance, dump_tracing_dir, flags))
69ce21
 		return;
69ce21
-	}
69ce21
 
69ce21
-	snprintf(path, sizeof(path), "%s/per_cpu", dump_tracing_dir);
69ce21
-	if (try_mkdir(path, 0755) < 0)
69ce21
+	if (!multiple_instances_available || instance_count == 0)
69ce21
 		return;
69ce21
 
69ce21
-	if (ftrace_dump_buffers(path) < 0)
69ce21
+	/* Create an instances directory, and dump instance data in there */
69ce21
+	snprintf(instance_path, sizeof(instance_path),
69ce21
+			"%s/instances", dump_tracing_dir);
69ce21
+	if (try_mkdir(instance_path, 0755) < 0)
69ce21
 		return;
69ce21
 
69ce21
-	if (dump_meta_data) {
69ce21
-		/* Dump event types */
69ce21
-		snprintf(path, sizeof(path), "%s/events", dump_tracing_dir);
69ce21
-		if (try_mkdir(path, 0755) < 0)
69ce21
-			return;
69ce21
+	/* Don't care about the flags anymore */
69ce21
+	flags = 0;
69ce21
 
69ce21
-		if (ftrace_dump_event_types(path) < 0)
69ce21
-			return;
69ce21
+	for (i = 0; i < instance_count; i++)
69ce21
+	{
69ce21
+		struct trace_instance *ti = &trace_instances[i];
69ce21
+
69ce21
+		snprintf(instance_path, sizeof(instance_path),
69ce21
+			"%s/instances/%s", dump_tracing_dir,
69ce21
+			ti->name);
69ce21
 
69ce21
-		/* Dump pids with corresponding cmdlines */
69ce21
-		if (dump_saved_cmdlines(dump_tracing_dir) < 0)
69ce21
-			return;
69ce21
+		if (populate_ftrace_dir_tree(ti, instance_path, flags) < 0)
69ce21
+			break;
69ce21
 	}
69ce21
 
69ce21
-	if (dump_symbols) {
69ce21
-		/* Dump all symbols of the kernel */
69ce21
-		dump_kallsyms(dump_tracing_dir);
69ce21
-	}
69ce21
+	return;
69ce21
 }
69ce21
 
69ce21
 static void ftrace_show(int argc, char *argv[])
69ce21
@@ -2161,26 +2314,69 @@ static int save_ftrace_cmdlines(int fd)
69ce21
 	return tmp_file_flush(fd);
69ce21
 }
69ce21
 
69ce21
-static int save_res_data(int fd, int nr_cpu_buffers)
69ce21
+/* From trace-cmd.h */
69ce21
+enum {
69ce21
+	TRACECMD_OPTION_DONE,         /* 0 */
69ce21
+	TRACECMD_OPTION_DATE,         /* 1 */
69ce21
+	TRACECMD_OPTION_CPUSTAT,      /* 2 */
69ce21
+	TRACECMD_OPTION_BUFFER,       /* 3 */
69ce21
+	TRACECMD_OPTION_TRACECLOCK,   /* 4 */
69ce21
+	TRACECMD_OPTION_UNAME,        /* 5 */
69ce21
+	TRACECMD_OPTION_HOOK,         /* 6 */
69ce21
+};
69ce21
+
69ce21
+static int write_options(int fd, unsigned long long *buffer_offsets)
69ce21
 {
69ce21
-	unsigned short option = 0;
69ce21
+	int i;
69ce21
+	unsigned short option;
69ce21
 
69ce21
-	if (write_and_check(fd, &nr_cpu_buffers, 4))
69ce21
-		return -1;
69ce21
+	if (!multiple_instances_available)
69ce21
+		return 0;
69ce21
 
69ce21
 	if (write_and_check(fd, "options  ", 10))
69ce21
 		return -1;
69ce21
 
69ce21
+	option = TRACECMD_OPTION_BUFFER;
69ce21
+	for (i = 0; i < instance_count; i++)
69ce21
+	{
69ce21
+		char *name = trace_instances[i].name;
69ce21
+		size_t name_size = strlen(name) + 1; /* Name length + '\0' */
69ce21
+		unsigned long long option_size = 8 + name_size;
69ce21
+		unsigned long long offset;
69ce21
+
69ce21
+		offset = buffer_offsets ? buffer_offsets[i] : 0;
69ce21
+		if (write_and_check(fd, &option, 2))
69ce21
+			return -1;
69ce21
+		if (write_and_check(fd, &option_size, 4))
69ce21
+			return -1;
69ce21
+		if (write_and_check(fd, &offset, 8))
69ce21
+			return -1;
69ce21
+		if (write_and_check(fd, name, name_size))
69ce21
+			return -1;
69ce21
+	}
69ce21
+
69ce21
+	option = TRACECMD_OPTION_DONE;
69ce21
 	if (write_and_check(fd, &option, 2))
69ce21
 		return -1;
69ce21
 
69ce21
+	return 0;
69ce21
+}
69ce21
+
69ce21
+static int save_res_data(int fd, int nr_cpu_buffers, unsigned long long *buffer_offsets)
69ce21
+{
69ce21
+	if (write_and_check(fd, &nr_cpu_buffers, 4))
69ce21
+		return -1;
69ce21
+
69ce21
+	if (write_options(fd, buffer_offsets))
69ce21
+		return -1;
69ce21
+
69ce21
 	if (write_and_check(fd, "flyrecord", 10))
69ce21
 		return -1;
69ce21
 
69ce21
 	return 0;
69ce21
 }
69ce21
 
69ce21
-static int save_record_data(int fd, int nr_cpu_buffers)
69ce21
+static int save_record_data(int fd, int nr_cpu_buffers, struct trace_instance *ti)
69ce21
 {
69ce21
 	int i, j;
69ce21
 	unsigned long long offset, buffer_offset;
69ce21
@@ -2192,7 +2388,7 @@ static int save_record_data(int fd, int
69ce21
 	buffer_offset = offset;
69ce21
 
69ce21
 	for (i = 0; i < nr_cpu_ids; i++) {
69ce21
-		struct ring_buffer_per_cpu *cpu_buffer = &global_buffers[i];
69ce21
+		struct ring_buffer_per_cpu *cpu_buffer = &ti->buffers[i];
69ce21
 		unsigned long long buffer_size;
69ce21
 
69ce21
 		if (!cpu_buffer->kaddr)
69ce21
@@ -2212,7 +2408,7 @@ static int save_record_data(int fd, int
69ce21
 
69ce21
 	lseek(fd, offset, SEEK_SET);
69ce21
 	for (i = 0; i < nr_cpu_ids; i++) {
69ce21
-		struct ring_buffer_per_cpu *cpu_buffer = &global_buffers[i];
69ce21
+		struct ring_buffer_per_cpu *cpu_buffer = &ti->buffers[i];
69ce21
 
69ce21
 		if (!cpu_buffer->kaddr)
69ce21
 			continue;
69ce21
@@ -2231,13 +2427,13 @@ static int save_record_data(int fd, int
69ce21
 	return 0;
69ce21
 }
69ce21
 
69ce21
-static int __trace_cmd_data_output(int fd)
69ce21
+static int get_nr_cpu_buffers(struct trace_instance *ti)
69ce21
 {
69ce21
 	int i;
69ce21
 	int nr_cpu_buffers = 0;
69ce21
 
69ce21
 	for (i = 0; i < nr_cpu_ids; i++) {
69ce21
-		struct ring_buffer_per_cpu *cpu_buffer = &global_buffers[i];
69ce21
+		struct ring_buffer_per_cpu *cpu_buffer = &ti->buffers[i];
69ce21
 
69ce21
 		if (!cpu_buffer->kaddr)
69ce21
 			continue;
69ce21
@@ -2245,6 +2441,19 @@ static int __trace_cmd_data_output(int f
69ce21
 		nr_cpu_buffers++;
69ce21
 	}
69ce21
 
69ce21
+	return nr_cpu_buffers;
69ce21
+}
69ce21
+
69ce21
+static int __trace_cmd_data_output(int fd)
69ce21
+{
69ce21
+	int nr_cpu_buffers;
69ce21
+	unsigned long long global_res_data_offset;
69ce21
+	unsigned long long *instance_offsets;
69ce21
+
69ce21
+	instance_offsets = calloc(sizeof(unsigned long long), instance_count);
69ce21
+
69ce21
+	nr_cpu_buffers = get_nr_cpu_buffers(&global_trace_instance);
69ce21
+
69ce21
 	if (save_initial_data(fd))
69ce21
 		return -1;
69ce21
 	if (save_header_files(fd))
69ce21
@@ -2257,9 +2466,38 @@ static int __trace_cmd_data_output(int f
69ce21
 		return -1;
69ce21
 	if (save_ftrace_cmdlines(fd))
69ce21
 		return -1;
69ce21
-	if (save_res_data(fd, nr_cpu_buffers))
69ce21
+
69ce21
+	/* We don't have the instance buffer offsets yet, so we'll write in 0s
69ce21
+	 * for now, and fix it up after we have that information available */
69ce21
+	global_res_data_offset = lseek(fd, 0, SEEK_CUR);
69ce21
+	if (save_res_data(fd, nr_cpu_buffers, NULL))
69ce21
 		return -1;
69ce21
-	if (save_record_data(fd, nr_cpu_buffers))
69ce21
+	if (save_record_data(fd, nr_cpu_buffers, &global_trace_instance))
69ce21
+		return -1;
69ce21
+
69ce21
+	if (multiple_instances_available)
69ce21
+	{
69ce21
+		int i;
69ce21
+
69ce21
+		for (i = 0; i < instance_count; i++)
69ce21
+		{
69ce21
+			struct trace_instance *ti = &trace_instances[i];
69ce21
+			nr_cpu_buffers = get_nr_cpu_buffers(ti);
69ce21
+
69ce21
+			/* Save off the instance offset for fixup later */
69ce21
+			instance_offsets[i] = lseek(fd, 0, SEEK_CUR);
69ce21
+
69ce21
+			if (write_and_check(fd, "flyrecord", 10))
69ce21
+				return -1;
69ce21
+			if (save_record_data(fd, nr_cpu_buffers, ti))
69ce21
+				return -1;
69ce21
+		}
69ce21
+	}
69ce21
+
69ce21
+	/* Fix up the global trace's options header with the instance offsets */
69ce21
+	lseek(fd, global_res_data_offset, SEEK_SET);
69ce21
+	nr_cpu_buffers = get_nr_cpu_buffers(&global_trace_instance);
69ce21
+	if (save_res_data(fd, nr_cpu_buffers, instance_offsets))
69ce21
 		return -1;
69ce21
 
69ce21
 	return 0;