From af4d66e07c86d7593f7d18ae4b6a2151123b529b Mon Sep 17 00:00:00 2001

From: Eric Blake <eblake@redhat.com>

Date: Tue, 2 Jun 2020 02:34:17 +0100

Subject: [PATCH 12/26] qcow2: Expose bitmaps' size during measure

RH-Author: Eric Blake <eblake@redhat.com>

Message-id: <20200602023420.2133649-10-eblake@redhat.com>

Patchwork-id: 97072

O-Subject: [RHEL-AV-8.2.1 qemu-kvm PATCH 09/12] qcow2: Expose bitmaps' size during measure

Bugzilla: 1779893 1779904

RH-Acked-by: Stefan Hajnoczi <stefanha@redhat.com>

RH-Acked-by: Max Reitz <mreitz@redhat.com>

RH-Acked-by: Kevin Wolf <kwolf@redhat.com>

It's useful to know how much space can be occupied by qcow2 persistent

bitmaps, even though such metadata is unrelated to the guest-visible

data.  Report this value as an additional QMP field, present when

measuring an existing image and output format that both support

bitmaps.  Update iotest 178 and 190 to updated output, as well as new

coverage in 190 demonstrating non-zero values made possible with the

recently-added qemu-img bitmap command (see 3b51ab4b).

The new 'bitmaps size:' field is displayed automatically as part of

'qemu-img measure' any time it is present in QMP (that is, any time

both the source image being measured and destination format support

bitmaps, even if the measurement is 0 because there are no bitmaps

present).  If the field is absent, it means that no bitmaps can be

copied (source, destination, or both lack bitmaps, including when

measuring based on size rather than on a source image).  This behavior

is compatible with an upcoming patch adding 'qemu-img convert

--bitmaps': that command will fail in the same situations where this

patch omits the field.

The addition of a new field demonstrates why we should always

zero-initialize qapi C structs; while the qcow2 driver still fully

populates all fields, the raw and crypto drivers had to be tweaked to

avoid uninitialized data.

Consideration was also given towards having a 'qemu-img measure

--bitmaps' which errors out when bitmaps are not possible, and

otherwise sums the bitmaps into the existing allocation totals rather

than displaying as a separate field, as a potential convenience

factor.  But this was ultimately decided to be more complexity than

necessary when the QMP interface was sufficient enough with bitmaps

remaining a separate field.

See also: https://bugzilla.redhat.com/1779904

Reported-by: Nir Soffer <nsoffer@redhat.com>

Signed-off-by: Eric Blake <eblake@redhat.com>

Message-Id: <20200521192137.1120211-3-eblake@redhat.com>

Reviewed-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

(cherry picked from commit 5d72c68b49769c927e90b78af6d90f6a384b26ac)

Signed-off-by: Danilo C. L. de Paula <ddepaula@redhat.com>

Conflicts:

	block/crypto.c - commit a9da6e49 not present (no measure support)

	docs/tools/qemu-img.rst - changes in qemu-img.texi instead

Signed-off-by: Eric Blake <eblake@redhat.com>

Signed-off-by: Danilo C. L. de Paula <ddepaula@redhat.com>

---

 block/qcow2-bitmap.c             | 36 ++++++++++++++++++++++++++++++

 block/qcow2.c                    | 14 +++++++++---

 block/qcow2.h                    |  2 ++

 block/raw-format.c               |  2 +-

 qapi/block-core.json             | 16 +++++++++-----

 qemu-img.c                       |  3 +++

 qemu-img.texi                    |  7 ++++++

 tests/qemu-iotests/178.out.qcow2 | 16 ++++++++++++++

 tests/qemu-iotests/190           | 47 ++++++++++++++++++++++++++++++++++++++--

 tests/qemu-iotests/190.out       | 27 ++++++++++++++++++++++-

 10 files changed, 158 insertions(+), 12 deletions(-)

diff --git a/block/qcow2-bitmap.c b/block/qcow2-bitmap.c

index cbac905..10d1297 100644

--- a/block/qcow2-bitmap.c

+++ b/block/qcow2-bitmap.c

@@ -1766,3 +1766,39 @@ bool qcow2_supports_persistent_dirty_bitmap(BlockDriverState *bs)

     return s->qcow_version >= 3;

 }

+

+/*

+ * Compute the space required for bitmaps in @bs.

+ *

+ * The computation is based as if copying to a new image with the

+ * given @cluster_size, which may differ from the cluster size in @bs.

+ */

+uint64_t qcow2_get_persistent_dirty_bitmap_size(BlockDriverState *bs,

+                                                uint32_t cluster_size)

+{

+    uint64_t bitmaps_size = 0;

+    BdrvDirtyBitmap *bm;

+    size_t bitmap_dir_size = 0;

+

+    FOR_EACH_DIRTY_BITMAP(bs, bm) {

+        if (bdrv_dirty_bitmap_get_persistence(bm)) {

+            const char *name = bdrv_dirty_bitmap_name(bm);

+            uint32_t granularity = bdrv_dirty_bitmap_granularity(bm);

+            uint64_t bmbytes =

+                get_bitmap_bytes_needed(bdrv_dirty_bitmap_size(bm),

+                                        granularity);

+            uint64_t bmclusters = DIV_ROUND_UP(bmbytes, cluster_size);

+

+            /* Assume the entire bitmap is allocated */

+            bitmaps_size += bmclusters * cluster_size;

+            /* Also reserve space for the bitmap table entries */

+            bitmaps_size += ROUND_UP(bmclusters * sizeof(uint64_t),

+                                     cluster_size);

+            /* And space for contribution to bitmap directory size */

+            bitmap_dir_size += calc_dir_entry_size(strlen(name), 0);

+        }

+    }

+    bitmaps_size += ROUND_UP(bitmap_dir_size, cluster_size);

+

+    return bitmaps_size;

+}

diff --git a/block/qcow2.c b/block/qcow2.c

index 36b0f7d..dbd870a 100644

--- a/block/qcow2.c

+++ b/block/qcow2.c

@@ -4751,16 +4751,24 @@ static BlockMeasureInfo *qcow2_measure(QemuOpts *opts, BlockDriverState *in_bs,

         required = virtual_size;

     }

-    info = g_new(BlockMeasureInfo, 1);

+    info = g_new0(BlockMeasureInfo, 1);

     info->fully_allocated =

         qcow2_calc_prealloc_size(virtual_size, cluster_size,

                                  ctz32(refcount_bits)) + luks_payload_size;

-    /* Remove data clusters that are not required.  This overestimates the

+    /*

+     * Remove data clusters that are not required.  This overestimates the

      * required size because metadata needed for the fully allocated file is

-     * still counted.

+     * still counted.  Show bitmaps only if both source and destination

+     * would support them.

      */

     info->required = info->fully_allocated - virtual_size + required;

+    info->has_bitmaps = version >= 3 && in_bs &&

+        bdrv_supports_persistent_dirty_bitmap(in_bs);

+    if (info->has_bitmaps) {

+        info->bitmaps = qcow2_get_persistent_dirty_bitmap_size(in_bs,

+                                                               cluster_size);

+    }

     return info;

 err:

diff --git a/block/qcow2.h b/block/qcow2.h

index ceb1ceb..3297e6b 100644

--- a/block/qcow2.h

+++ b/block/qcow2.h

@@ -768,6 +768,8 @@ int qcow2_co_remove_persistent_dirty_bitmap(BlockDriverState *bs,

                                             const char *name,

                                             Error **errp);

 bool qcow2_supports_persistent_dirty_bitmap(BlockDriverState *bs);

+uint64_t qcow2_get_persistent_dirty_bitmap_size(BlockDriverState *bs,

+                                                uint32_t cluster_size);

 ssize_t coroutine_fn

 qcow2_co_compress(BlockDriverState *bs, void *dest, size_t dest_size,

diff --git a/block/raw-format.c b/block/raw-format.c

index 93b25e1..4bb54f4 100644

--- a/block/raw-format.c

+++ b/block/raw-format.c

@@ -346,7 +346,7 @@ static BlockMeasureInfo *raw_measure(QemuOpts *opts, BlockDriverState *in_bs,

                             BDRV_SECTOR_SIZE);

     }

-    info = g_new(BlockMeasureInfo, 1);

+    info = g_new0(BlockMeasureInfo, 1);

     info->required = required;

     /* Unallocated sectors count towards the file size in raw images */

diff --git a/qapi/block-core.json b/qapi/block-core.json

index a64ad81..2893209 100644

--- a/qapi/block-core.json

+++ b/qapi/block-core.json

@@ -689,18 +689,24 @@

 # efficiently so file size may be smaller than virtual disk size.

 #

 # The values are upper bounds that are guaranteed to fit the new image file.

-# Subsequent modification, such as internal snapshot or bitmap creation, may

-# require additional space and is not covered here.

+# Subsequent modification, such as internal snapshot or further bitmap

+# creation, may require additional space and is not covered here.

 #

-# @required: Size required for a new image file, in bytes.

+# @required: Size required for a new image file, in bytes, when copying just

+#            allocated guest-visible contents.

 #

 # @fully-allocated: Image file size, in bytes, once data has been written

-#                   to all sectors.

+#                   to all sectors, when copying just guest-visible contents.

+#

+# @bitmaps: Additional size required if all the top-level bitmap metadata

+#           in the source image were to be copied to the destination,

+#           present only when source and destination both support

+#           persistent bitmaps. (since 5.1)

 #

 # Since: 2.10

 ##

 { 'struct': 'BlockMeasureInfo',

-  'data': {'required': 'int', 'fully-allocated': 'int'} }

+  'data': {'required': 'int', 'fully-allocated': 'int', '*bitmaps': 'int'} }

 ##

 # @query-block:

diff --git a/qemu-img.c b/qemu-img.c

index 11a4537..b57856e 100644

--- a/qemu-img.c

+++ b/qemu-img.c

@@ -5212,6 +5212,9 @@ static int img_measure(int argc, char **argv)

     if (output_format == OFORMAT_HUMAN) {

         printf("required size: %" PRIu64 "\n", info->required);

         printf("fully allocated size: %" PRIu64 "\n", info->fully_allocated);

+        if (info->has_bitmaps) {

+            printf("bitmaps size: %" PRIu64 "\n", info->bitmaps);

+        }

     } else {

         dump_json_block_measure_info(info);

     }

diff --git a/qemu-img.texi b/qemu-img.texi

index abf2771..3670b96 100644

--- a/qemu-img.texi

+++ b/qemu-img.texi

@@ -576,6 +576,7 @@ The following fields are reported:

 @example

 required size: 524288

 fully allocated size: 1074069504

+bitmaps size: 0

 @end example

 The @code{required size} is the file size of the new image.  It may be smaller

@@ -586,6 +587,12 @@ been written to all sectors.  This is the maximum size that the image file can

 occupy with the exception of internal snapshots, dirty bitmaps, vmstate data,

 and other advanced image format features.

+The @code{bitmaps size} is the additional size required in order to

+copy bitmaps from a source image in addition to the guest-visible

+data; the line is omitted if either source or destination lacks

+bitmap support, or 0 if bitmaps are supported but there is nothing to

+copy.

+

 @item snapshot [--object @var{objectdef}] [--image-opts] [-U] [-q] [-l | -a @var{snapshot} | -c @var{snapshot} | -d @var{snapshot}] @var{filename}

 List, apply, create or delete snapshots in image @var{filename}.

diff --git a/tests/qemu-iotests/178.out.qcow2 b/tests/qemu-iotests/178.out.qcow2

index 345eab3..b9ed41b 100644

--- a/tests/qemu-iotests/178.out.qcow2

+++ b/tests/qemu-iotests/178.out.qcow2

@@ -37,6 +37,7 @@ qemu-img: The image size is too large (try using a larger cluster size)

 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=0

 required size: 196608

 fully allocated size: 196608

+bitmaps size: 0

 converted image file size in bytes: 196608

@@ -45,6 +46,7 @@ converted image file size in bytes: 196608

 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824

 required size: 393216

 fully allocated size: 1074135040

+bitmaps size: 0

 wrote 512/512 bytes at offset 512

 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)

 wrote 65536/65536 bytes at offset 65536

@@ -53,6 +55,7 @@ wrote 64512/64512 bytes at offset 134217728

 63 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)

 required size: 589824

 fully allocated size: 1074135040

+bitmaps size: 0

 converted image file size in bytes: 524288

@@ -60,6 +63,7 @@ converted image file size in bytes: 524288

 required size: 524288

 fully allocated size: 1074135040

+bitmaps size: 0

 converted image file size in bytes: 458752

@@ -67,16 +71,19 @@ converted image file size in bytes: 458752

 required size: 1074135040

 fully allocated size: 1074135040

+bitmaps size: 0

 == qcow2 input image and LUKS encryption ==

 required size: 2686976

 fully allocated size: 1076232192

+bitmaps size: 0

 == qcow2 input image and preallocation (human) ==

 required size: 1074135040

 fully allocated size: 1074135040

+bitmaps size: 0

 converted image file size in bytes: 1074135040

@@ -87,6 +94,7 @@ wrote 8388608/8388608 bytes at offset 0

 8 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)

 required size: 8716288

 fully allocated size: 8716288

+bitmaps size: 0

 converted image file size in bytes: 8716288

@@ -173,6 +181,7 @@ qemu-img: The image size is too large (try using a larger cluster size)

 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=0

 {

+    "bitmaps": 0,

     "required": 196608,

     "fully-allocated": 196608

 }

@@ -183,6 +192,7 @@ converted image file size in bytes: 196608

 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824

 {

+    "bitmaps": 0,

     "required": 393216,

     "fully-allocated": 1074135040

 }

@@ -193,6 +203,7 @@ wrote 65536/65536 bytes at offset 65536

 wrote 64512/64512 bytes at offset 134217728

 63 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)

 {

+    "bitmaps": 0,

     "required": 589824,

     "fully-allocated": 1074135040

 }

@@ -202,6 +213,7 @@ converted image file size in bytes: 524288

 == qcow2 input image with internal snapshot (json) ==

 {

+    "bitmaps": 0,

     "required": 524288,

     "fully-allocated": 1074135040

 }

@@ -211,6 +223,7 @@ converted image file size in bytes: 458752

 == qcow2 input image and a backing file (json) ==

 {

+    "bitmaps": 0,

     "required": 1074135040,

     "fully-allocated": 1074135040

 }

@@ -218,6 +231,7 @@ converted image file size in bytes: 458752

 == qcow2 input image and LUKS encryption ==

 {

+    "bitmaps": 0,

     "required": 2686976,

     "fully-allocated": 1076232192

 }

@@ -225,6 +239,7 @@ converted image file size in bytes: 458752

 == qcow2 input image and preallocation (json) ==

 {

+    "bitmaps": 0,

     "required": 1074135040,

     "fully-allocated": 1074135040

 }

@@ -237,6 +252,7 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=8388608

 wrote 8388608/8388608 bytes at offset 0

 8 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)

 {

+    "bitmaps": 0,

     "required": 8716288,

     "fully-allocated": 8716288

 }

diff --git a/tests/qemu-iotests/190 b/tests/qemu-iotests/190

index eb766ad..5084ccd 100755

--- a/tests/qemu-iotests/190

+++ b/tests/qemu-iotests/190

@@ -2,7 +2,7 @@

 #

 # qemu-img measure sub-command tests on huge qcow2 files

 #

-# Copyright (C) 2017 Red Hat, Inc.

+# Copyright (C) 2017-2020 Red Hat, Inc.

 #

 # This program is free software; you can redistribute it and/or modify

 # it under the terms of the GNU General Public License as published by

@@ -42,7 +42,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15

 _supported_fmt qcow2

 _supported_proto file

-echo "== Huge file =="

+echo "== Huge file without bitmaps =="

 echo

 IMGOPTS='cluster_size=2M' _make_test_img 2T

@@ -51,6 +51,49 @@ $QEMU_IMG measure -O raw -f qcow2 "$TEST_IMG"

 $QEMU_IMG measure -O qcow2 -o cluster_size=64k -f qcow2 "$TEST_IMG"

 $QEMU_IMG measure -O qcow2 -o cluster_size=2M -f qcow2 "$TEST_IMG"

+echo

+echo "== Huge file with bitmaps =="

+echo

+

+$QEMU_IMG bitmap --add --granularity 512 -f qcow2 "$TEST_IMG" b1

+$QEMU_IMG bitmap --add -g 2M -f qcow2 "$TEST_IMG" b2

+

+# No bitmap without a source

+$QEMU_IMG measure -O qcow2 --size 10M

+# No bitmap output, since raw does not support it

+$QEMU_IMG measure -O raw -f qcow2 "$TEST_IMG"

+# No bitmap output, since no bitmaps on raw source. Munge required size, as

+# some filesystems store the qcow2 file with less sparseness than others

+$QEMU_IMG measure -O qcow2 -f raw "$TEST_IMG" |

+    sed '/^required size:/ s/[0-9][0-9]*/SIZE/'

+# No bitmap output, since v2 does not support it

+$QEMU_IMG measure -O qcow2 -o compat=0.10 -f qcow2 "$TEST_IMG"

+

+# Compute expected output: bitmap clusters + bitmap tables + bitmaps directory

+echo

+val2T=$((2*1024*1024*1024*1024))

+cluster=$((64*1024))

+b1clusters=$(( (val2T/512/8 + cluster - 1) / cluster ))

+b2clusters=$(( (val2T/2/1024/1024/8 + cluster - 1) / cluster ))

+echo expected bitmap $((b1clusters * cluster +

+                        (b1clusters * 8 + cluster - 1) / cluster * cluster +

+                        b2clusters * cluster +

+                        (b2clusters * 8 + cluster - 1) / cluster * cluster +

+                        cluster))

+$QEMU_IMG measure -O qcow2 -o cluster_size=64k -f qcow2 "$TEST_IMG"

+

+# Compute expected output: bitmap clusters + bitmap tables + bitmaps directory

+echo

+cluster=$((2*1024*1024))

+b1clusters=$(( (val2T/512/8 + cluster - 1) / cluster ))

+b2clusters=$(( (val2T/2/1024/1024/8 + cluster - 1) / cluster ))

+echo expected bitmap $((b1clusters * cluster +

+                        (b1clusters * 8 + cluster - 1) / cluster * cluster +

+                        b2clusters * cluster +

+                        (b2clusters * 8 + cluster - 1) / cluster * cluster +

+                        cluster))

+$QEMU_IMG measure --output=json -O qcow2 -o cluster_size=2M -f qcow2 "$TEST_IMG"

+

 # success, all done

 echo "*** done"

 rm -f $seq.full

diff --git a/tests/qemu-iotests/190.out b/tests/qemu-iotests/190.out

index d001942..ed9d821 100644

--- a/tests/qemu-iotests/190.out

+++ b/tests/qemu-iotests/190.out

@@ -1,11 +1,36 @@

 QA output created by 190

-== Huge file ==

+== Huge file without bitmaps ==

 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2199023255552

 required size: 2199023255552

 fully allocated size: 2199023255552

 required size: 335806464

 fully allocated size: 2199359062016

+bitmaps size: 0

 required size: 18874368

 fully allocated size: 2199042129920

+bitmaps size: 0

+

+== Huge file with bitmaps ==

+

+required size: 327680

+fully allocated size: 10813440

+required size: 2199023255552

+fully allocated size: 2199023255552

+required size: SIZE

+fully allocated size: 17170432

+required size: 335806464

+fully allocated size: 2199359062016

+

+expected bitmap 537198592

+required size: 335806464

+fully allocated size: 2199359062016

+bitmaps size: 537198592

+

+expected bitmap 545259520

+{

+    "bitmaps": 545259520,

+    "required": 18874368,

+    "fully-allocated": 2199042129920

+}

 *** done

-- 

1.8.3.1