From 8a9557041411dbb6739fb81c7ea992ef4109aa22 Mon Sep 17 00:00:00 2001

From: "Richard W.M. Jones" <rjones@redhat.com>

Date: Fri, 1 Apr 2016 19:19:51 +0100

Subject: [PATCH] tests/qemu: Add boot-benchmark.

Add a new test program called 'boot-benchmark'.  This is similar to

'boot-analysis' but it simply boots and shuts down the appliance

several times in a row and measures how long it takes, calculating

mean and standard deviation.

(cherry picked from commit 96ce2f9afedc6a7ecb2f7781958c3940255f453b)

---

 .gitignore                       |   1 +

 docs/guestfs-performance.pod     |  15 ++-

 tests/qemu/Makefile.am           |  30 ++++-

 tests/qemu/boot-analysis-utils.c |  47 ++++++++

 tests/qemu/boot-analysis-utils.h |  30 +++++

 tests/qemu/boot-analysis.c       |  21 +---

 tests/qemu/boot-benchmark.c      | 230 +++++++++++++++++++++++++++++++++++++++

 7 files changed, 346 insertions(+), 28 deletions(-)

 create mode 100644 tests/qemu/boot-analysis-utils.c

 create mode 100644 tests/qemu/boot-analysis-utils.h

 create mode 100644 tests/qemu/boot-benchmark.c

diff --git a/.gitignore b/.gitignore

index 9c45df7..46a6e65 100644

--- a/.gitignore

+++ b/.gitignore

@@ -510,6 +510,7 @@ Makefile.in

 /tests/parallel/test-parallel

 /tests/protocol/test-error-messages

 /tests/qemu/boot-analysis

+/tests/qemu/boot-benchmark

 /tests/qemu/qemu-boot

 /tests/qemu/qemu-speed-test

 /tests/regressions/rhbz501893

diff --git a/docs/guestfs-performance.pod b/docs/guestfs-performance.pod

index 4ba6faf..cf30fdc 100644

--- a/docs/guestfs-performance.pod

+++ b/docs/guestfs-performance.pod

@@ -29,11 +29,20 @@ appliance:

 Run this command several times in a row and discard the first few

 runs, so that you are measuring a typical "hot cache" case.

+I<Side note for developers:> If you are compiling libguestfs from

+source, there is a program called F<tests/qemu/boot-benchmark> which

+does the same thing, but performs multiple runs and prints the mean

+and standard deviation.  To run it, do:

+

+ make

+ make -C tests/qemu boot-benchmark

+ ./run ./tests/qemu/boot-benchmark

+

 =head3 Explanation

-This command starts up the libguestfs appliance on a null disk, and

-then immediately shuts it down.  The first time you run the command,

-it will create an appliance and cache it (usually under

+The guestfish command above starts up the libguestfs appliance on a

+null disk, and then immediately shuts it down.  The first time you run

+the command, it will create an appliance and cache it (usually under

 F).  Subsequent runs should reuse the cached

 appliance.

diff --git a/tests/qemu/Makefile.am b/tests/qemu/Makefile.am

index bea1c85..cc5cb6a 100644

--- a/tests/qemu/Makefile.am

+++ b/tests/qemu/Makefile.am

@@ -1,5 +1,5 @@

 # libguestfs

-# Copyright (C) 2011 Red Hat Inc.

+# Copyright (C) 2011-2016 Red Hat Inc.

 #

 # This program is free software; you can redistribute it and/or modify

 # it under the terms of the GNU General Public License as published by

@@ -33,10 +33,11 @@ EXTRA_DIST = \

 	qemu-boot.c \

 	qemu-speed-test.c

-# qemu-boot, qemu-speed-test and boot-analysis are built but not run

-# by default as they are mainly qemu & kernel diagnostic tools.

+# qemu-boot, qemu-speed-test, boot-analysis and boot-benchmark are

+# built but not run by default as they are mainly qemu & kernel

+# diagnostic tools.

-check_PROGRAMS = qemu-boot qemu-speed-test boot-analysis

+check_PROGRAMS = qemu-boot qemu-speed-test boot-analysis boot-benchmark

 qemu_boot_SOURCES = \

 	../../df/estimate-max-threads.c \

@@ -76,7 +77,9 @@ qemu_speed_test_LDADD = \

 boot_analysis_SOURCES = \

 	boot-analysis.c \

 	boot-analysis.h \

-	boot-analysis-timeline.c

+	boot-analysis-timeline.c \

+	boot-analysis-utils.c \

+	boot-analysis-utils.h

 boot_analysis_CPPFLAGS = \

 	-I$(top_srcdir)/gnulib/lib -I$(top_builddir)/gnulib/lib \

 	-I$(top_srcdir)/src -I$(top_builddir)/src

@@ -93,6 +96,23 @@ boot_analysis_LDADD = \

 	$(top_builddir)/gnulib/lib/libgnu.la \

 	-lm

+boot_benchmark_SOURCES = \

+	boot-benchmark.c \

+	boot-analysis-utils.c \

+	boot-analysis-utils.h

+boot_benchmark_CPPFLAGS = \

+	-I$(top_srcdir)/gnulib/lib -I$(top_builddir)/gnulib/lib \

+	-I$(top_srcdir)/src -I$(top_builddir)/src

+boot_benchmark_CFLAGS = \

+	$(WARN_CFLAGS) $(WERROR_CFLAGS)

+boot_benchmark_LDADD = \

+	$(top_builddir)/src/libutils.la \

+	$(top_builddir)/src/libguestfs.la \

+	$(LIBXML2_LIBS) \

+	$(LTLIBINTL) \

+	$(top_builddir)/gnulib/lib/libgnu.la \

+	-lm

+

 # Don't run these tests in parallel, since they are designed to check

 # the integrity of qemu.

 .NOTPARALLEL:

diff --git a/tests/qemu/boot-analysis-utils.c b/tests/qemu/boot-analysis-utils.c

new file mode 100644

index 0000000..e885f3b

--- /dev/null

+++ b/tests/qemu/boot-analysis-utils.c

@@ -0,0 +1,47 @@

+/* libguestfs

+ * Copyright (C) 2016 Red Hat Inc.

+ *

+ * This program is free software; you can redistribute it and/or modify

+ * it under the terms of the GNU General Public License as published by

+ * the Free Software Foundation; either version 2 of the License, or

+ * (at your option) any later version.

+ *

+ * This program is distributed in the hope that it will be useful,

+ * but WITHOUT ANY WARRANTY; without even the implied warranty of

+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

+ * GNU General Public License for more details.

+ *

+ * You should have received a copy of the GNU General Public License along

+ * with this program; if not, write to the Free Software Foundation, Inc.,

+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

+ */

+

+#include <config.h>

+

+#include <stdio.h>

+#include <stdlib.h>

+#include <time.h>

+#include <error.h>

+#include <errno.h>

+

+#include "guestfs.h"

+#include "guestfs-internal-frontend.h"

+

+#include "boot-analysis-utils.h"

+

+void

+get_time (struct timespec *ts)

+{

+  if (clock_gettime (CLOCK_REALTIME, ts) == -1)

+    error (EXIT_FAILURE, errno, "clock_gettime: CLOCK_REALTIME");

+}

+

+int64_t

+timespec_diff (const struct timespec *x, const struct timespec *y)

+{

+  int64_t nsec;

+

+  nsec = (y->tv_sec - x->tv_sec) * UINT64_C(1000000000);

+  nsec += y->tv_nsec - x->tv_nsec;

+  return nsec;

+}

diff --git a/tests/qemu/boot-analysis-utils.h b/tests/qemu/boot-analysis-utils.h

new file mode 100644

index 0000000..83fc494

--- /dev/null

+++ b/tests/qemu/boot-analysis-utils.h

@@ -0,0 +1,30 @@

+/* libguestfs

+ * Copyright (C) 2016 Red Hat Inc.

+ *

+ * This program is free software; you can redistribute it and/or modify

+ * it under the terms of the GNU General Public License as published by

+ * the Free Software Foundation; either version 2 of the License, or

+ * (at your option) any later version.

+ *

+ * This program is distributed in the hope that it will be useful,

+ * but WITHOUT ANY WARRANTY; without even the implied warranty of

+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

+ * GNU General Public License for more details.

+ *

+ * You should have received a copy of the GNU General Public License along

+ * with this program; if not, write to the Free Software Foundation, Inc.,

+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

+ */

+

+#ifndef GUESTFS_BOOT_ANALYSIS_UTILS_H_

+#define GUESTFS_BOOT_ANALYSIS_UTILS_H_

+

+/* Get current time, returning it in *ts.  If there is a system call

+ * failure, this exits.

+ */

+extern void get_time (struct timespec *ts);

+

+/* Computes Y - X, returning nanoseconds. */

+extern int64_t timespec_diff (const struct timespec *x, const struct timespec *y);

+

+#endif /* GUESTFS_BOOT_ANALYSIS_UTILS_H_ */

diff --git a/tests/qemu/boot-analysis.c b/tests/qemu/boot-analysis.c

index fc2c93b..022eaab 100644

--- a/tests/qemu/boot-analysis.c

+++ b/tests/qemu/boot-analysis.c

@@ -79,6 +79,7 @@

 #include "guestfs-internal-frontend.h"

 #include "boot-analysis.h"

+#include "boot-analysis-utils.h"

 /* Activities taking longer than this % of the total time, except

  * those flagged as LONG_ACTIVITY, are highlighted in red.

@@ -96,8 +97,6 @@ static int smp = 1;

 static int verbose = 0;

 static void run_test (void);

-static void get_time (struct timespec *ts);

-static int64_t timespec_diff (const struct timespec *x, const struct timespec *y);

 static struct event *add_event (struct pass_data *, uint64_t source);

 static guestfs_h *create_handle (void);

 static void set_up_event_handlers (guestfs_h *g, size_t pass);

@@ -267,24 +266,6 @@ run_test (void)

   free_final_timeline ();

 }

-static void

-get_time (struct timespec *ts)

-{

-  if (clock_gettime (CLOCK_REALTIME, ts) == -1)

-    error (EXIT_FAILURE, errno, "clock_gettime: CLOCK_REALTIME");

-}

-

-/* Computes Y - X, returning nanoseconds. */

-static int64_t

-timespec_diff (const struct timespec *x, const struct timespec *y)

-{

-  int64_t nsec;

-

-  nsec = (y->tv_sec - x->tv_sec) * UINT64_C(1000000000);

-  nsec += y->tv_nsec - x->tv_nsec;

-  return nsec;

-}

-

 static struct event *

 add_event (struct pass_data *data, uint64_t source)

 {

diff --git a/tests/qemu/boot-benchmark.c b/tests/qemu/boot-benchmark.c

new file mode 100644

index 0000000..2a6a038

--- /dev/null

+++ b/tests/qemu/boot-benchmark.c

@@ -0,0 +1,230 @@

+/* libguestfs

+ * Copyright (C) 2016 Red Hat Inc.

+ *

+ * This program is free software; you can redistribute it and/or modify

+ * it under the terms of the GNU General Public License as published by

+ * the Free Software Foundation; either version 2 of the License, or

+ * (at your option) any later version.

+ *

+ * This program is distributed in the hope that it will be useful,

+ * but WITHOUT ANY WARRANTY; without even the implied warranty of

+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

+ * GNU General Public License for more details.

+ *

+ * You should have received a copy of the GNU General Public License along

+ * with this program; if not, write to the Free Software Foundation, Inc.,

+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

+ */

+

+/* Benchmark the time taken to boot the libguestfs appliance. */

+

+#include <config.h>

+

+#include <stdio.h>

+#include <stdlib.h>

+#include <stdint.h>

+#include <inttypes.h>

+#include <string.h>

+#include <getopt.h>

+#include <limits.h>

+#include <time.h>

+#include <errno.h>

+#include <error.h>

+#include <assert.h>

+#include <math.h>

+

+#include "guestfs.h"

+#include "guestfs-internal-frontend.h"

+

+#include "boot-analysis-utils.h"

+

+#define NR_WARMUP_PASSES 3

+#define NR_TEST_PASSES   10

+

+static const char *append = NULL;

+static int memsize = 0;

+static int smp = 1;

+

+static void run_test (void);

+static guestfs_h *create_handle (void);

+static void add_drive (guestfs_h *g);

+

+static void

+usage (int exitcode)

+{

+  guestfs_h *g;

+  int default_memsize = -1;

+

+  g = guestfs_create ();

+  if (g) {

+    default_memsize = guestfs_get_memsize (g);

+    guestfs_close (g);

+  }

+

+  fprintf (stderr,

+           "boot-benchmark: Benchmark the time taken to boot the libguestfs appliance.\n"

+           "Usage:\n"

+           "  boot-benchmark [--options]\n"

+           "Options:\n"

+           "  --help         Display this usage text and exit.\n"

+           "  --append OPTS  Append OPTS to kernel command line.\n"

+           "  -m MB\n"

+           "  --memsize MB   Set memory size in MB (default: %d).\n"

+           "  --smp N        Enable N virtual CPUs (default: 1).\n",

+           default_memsize);

+  exit (exitcode);

+}

+

+int

+main (int argc, char *argv[])

+{

+  enum { HELP_OPTION = CHAR_MAX + 1 };

+  static const char *options = "m:";

+  static const struct option long_options[] = {

+    { "help", 0, 0, HELP_OPTION },

+    { "append", 1, 0, 0 },

+    { "memsize", 1, 0, 'm' },

+    { "smp", 1, 0, 0 },

+    { 0, 0, 0, 0 }

+  };

+  int c, option_index;

+

+  for (;;) {

+    c = getopt_long (argc, argv, options, long_options, &option_index);

+    if (c == -1) break;

+

+    switch (c) {

+    case 0:                     /* Options which are long only. */

+      if (STREQ (long_options[option_index].name, "append")) {

+        append = optarg;

+        break;

+      }

+      else if (STREQ (long_options[option_index].name, "smp")) {

+        if (sscanf (optarg, "%d", &smp) != 1) {

+          fprintf (stderr, "%s: could not parse smp parameter: %s\n",

+                   guestfs_int_program_name, optarg);

+          exit (EXIT_FAILURE);

+        }

+        break;

+      }

+      fprintf (stderr, "%s: unknown long option: %s (%d)\n",

+               guestfs_int_program_name, long_options[option_index].name, option_index);

+      exit (EXIT_FAILURE);

+

+    case 'm':

+      if (sscanf (optarg, "%d", &memsize) != 1) {

+        fprintf (stderr, "%s: could not parse memsize parameter: %s\n",

+                 guestfs_int_program_name, optarg);

+        exit (EXIT_FAILURE);

+      }

+      break;

+

+    case HELP_OPTION:

+      usage (EXIT_SUCCESS);

+

+    default:

+      usage (EXIT_FAILURE);

+    }

+  }

+

+  run_test ();

+}

+

+static void

+run_test (void)

+{

+  guestfs_h *g;

+  size_t i;

+  int64_t ns[NR_TEST_PASSES];

+  double mean;

+  double variance;

+  double sd;

+

+  printf ("Warming up the libguestfs cache ...\n");

+  for (i = 0; i < NR_WARMUP_PASSES; ++i) {

+    g = create_handle ();

+    add_drive (g);

+    if (guestfs_launch (g) == -1)

+      exit (EXIT_FAILURE);

+    guestfs_close (g);

+  }

+

+  printf ("Running the tests ...\n");

+  for (i = 0; i < NR_TEST_PASSES; ++i) {

+    struct timespec start_t, end_t;

+

+    g = create_handle ();

+    add_drive (g);

+    get_time (&start_t);

+    if (guestfs_launch (g) == -1)

+      exit (EXIT_FAILURE);

+    guestfs_close (g);

+    get_time (&end_t);

+

+    ns[i] = timespec_diff (&start_t, &end_t);

+  }

+

+  /* Calculate the mean. */

+  mean = 0;

+  for (i = 0; i < NR_TEST_PASSES; ++i)

+    mean += ns[i];

+  mean /= NR_TEST_PASSES;

+

+  /* Calculate the variance and standard deviation. */

+  variance = 0;

+  for (i = 0; i < NR_TEST_PASSES; ++i)

+    variance = pow (ns[i] - mean, 2);

+  variance /= NR_TEST_PASSES;

+  sd = sqrt (variance);

+

+  /* Print the test parameters. */

+  printf ("\n");

+  printf (" passes %d\n", NR_TEST_PASSES);

+  g = create_handle ();

+  printf (" append %s\n", guestfs_get_append (g) ? : "");

+  printf ("backend %s\n", guestfs_get_backend (g));

+  printf ("     hv %s\n", guestfs_get_hv (g));

+  printf ("memsize %d\n", guestfs_get_memsize (g));

+  printf ("    smp %d\n", guestfs_get_smp (g));

+  guestfs_close (g);

+

+  /* Print the result. */

+  printf ("\n");

+  printf ("Result: %.1fms ±%.1fms\n", mean / 1000000, sd / 1000000);

+}

+

+/* Common function to create the handle and set various defaults. */

+static guestfs_h *

+create_handle (void)

+{

+  guestfs_h *g;

+  CLEANUP_FREE char *full_append = NULL;

+

+  g = guestfs_create ();

+  if (!g) error (EXIT_FAILURE, errno, "guestfs_create");

+

+  if (memsize != 0)

+    if (guestfs_set_memsize (g, memsize) == -1)

+      exit (EXIT_FAILURE);

+

+  if (smp >= 2)

+    if (guestfs_set_smp (g, smp) == -1)

+      exit (EXIT_FAILURE);

+

+  if (append != NULL)

+    if (guestfs_set_append (g, full_append) == -1)

+      exit (EXIT_FAILURE);

+

+  return g;

+}

+

+/* Common function to add the /dev/null drive. */

+static void

+add_drive (guestfs_h *g)

+{

+  if (guestfs_add_drive_opts (g, "/dev/null",

+                              GUESTFS_ADD_DRIVE_OPTS_FORMAT, "raw",

+                              GUESTFS_ADD_DRIVE_OPTS_READONLY, 1,

+                              -1) == -1)

+    exit (EXIT_FAILURE);

+}

-- 

1.8.3.1