Blame SOURCES/kvm-iotests-108-Test-new-refcount-rebuild-algorithm.patch

4841a6
From d638552d76db0db9e2b6ae90a35f0b451b0cbaf8 Mon Sep 17 00:00:00 2001
6443c2
From: Hanna Reitz <hreitz@redhat.com>
6443c2
Date: Tue, 5 Apr 2022 15:46:51 +0200
4841a6
Subject: [PATCH 4/6] iotests/108: Test new refcount rebuild algorithm
6443c2
6443c2
RH-Author: Hanna Reitz <hreitz@redhat.com>
4841a6
RH-MergeRequest: 171: qcow2: Improve refcount structure rebuilding
4841a6
RH-Commit: [2/4] 2aa8c383f0c88c414f10ade8bd2e8af07c35f35b
4841a6
RH-Bugzilla: 1519071
6443c2
RH-Acked-by: Miroslav Rezanina <mrezanin@redhat.com>
6443c2
RH-Acked-by: Stefan Hajnoczi <stefanha@redhat.com>
4841a6
RH-Acked-by: Eric Blake <eblake@redhat.com>
6443c2
6443c2
One clear problem with how qcow2's refcount structure rebuild algorithm
6443c2
used to be before "qcow2: Improve refcount structure rebuilding" was
6443c2
that it is prone to failure for qcow2 images on block devices: There is
6443c2
generally unused space after the actual image, and if that exceeds what
6443c2
one refblock covers, the old algorithm would invariably write the
6443c2
reftable past the block device's end, which cannot work.  The new
6443c2
algorithm does not have this problem.
6443c2
6443c2
Test it with three tests:
6443c2
(1) Create an image with more empty space at the end than what one
6443c2
    refblock covers, see whether rebuilding the refcount structures
6443c2
    results in a change in the image file length.  (It should not.)
6443c2
6443c2
(2) Leave precisely enough space somewhere at the beginning of the image
6443c2
    for the new reftable (and the refblock for that place), see whether
6443c2
    the new algorithm puts the reftable there.  (It should.)
6443c2
6443c2
(3) Test the original problem: Create (something like) a block device
6443c2
    with a fixed size, then create a qcow2 image in there, write some
6443c2
    data, and then have qemu-img check rebuild the refcount structures.
6443c2
    Before HEAD^, the reftable would have been written past the image
6443c2
    file end, i.e. outside of what the block device provides, which
6443c2
    cannot work.  HEAD^ should have fixed that.
6443c2
    ("Something like a block device" means a loop device if we can use
6443c2
    one ("sudo -n losetup" works), or a FUSE block export with
6443c2
    growable=false otherwise.)
6443c2
6443c2
Reviewed-by: Eric Blake <eblake@redhat.com>
6443c2
Signed-off-by: Hanna Reitz <hreitz@redhat.com>
6443c2
Message-Id: <20220405134652.19278-3-hreitz@redhat.com>
6443c2
(cherry picked from commit 9ffd6d646d1d5ee9087a8cbf0b7d2f96c5656162)
6443c2
6443c2
Conflicts:
6443c2
- 108: The downstream qemu-storage-daemon does not support --daemonize,
6443c2
  so this switch has been replaced by a loop waiting for the PID file to
6443c2
  appear
6443c2
6443c2
Signed-off-by: Hanna Reitz <hreitz@redhat.com>
6443c2
---
6443c2
 tests/qemu-iotests/108     | 263 ++++++++++++++++++++++++++++++++++++-
6443c2
 tests/qemu-iotests/108.out |  81 ++++++++++++
6443c2
 2 files changed, 343 insertions(+), 1 deletion(-)
6443c2
6443c2
diff --git a/tests/qemu-iotests/108 b/tests/qemu-iotests/108
6443c2
index 8eaef0b8bf..23abbeaff0 100755
6443c2
--- a/tests/qemu-iotests/108
6443c2
+++ b/tests/qemu-iotests/108
6443c2
@@ -30,13 +30,20 @@ status=1	# failure is the default!
6443c2
 
6443c2
 _cleanup()
6443c2
 {
6443c2
-	_cleanup_test_img
6443c2
+    _cleanup_test_img
6443c2
+    if [ -f "$TEST_DIR/qsd.pid" ]; then
6443c2
+        qsd_pid=$(cat "$TEST_DIR/qsd.pid")
6443c2
+        kill -KILL "$qsd_pid"
6443c2
+        fusermount -u "$TEST_DIR/fuse-export" &>/dev/null
6443c2
+    fi
6443c2
+    rm -f "$TEST_DIR/fuse-export"
6443c2
 }
6443c2
 trap "_cleanup; exit \$status" 0 1 2 3 15
6443c2
 
6443c2
 # get standard environment, filters and checks
6443c2
 . ./common.rc
6443c2
 . ./common.filter
6443c2
+. ./common.qemu
6443c2
 
6443c2
 # This tests qcow2-specific low-level functionality
6443c2
 _supported_fmt qcow2
6443c2
@@ -47,6 +54,22 @@ _supported_os Linux
6443c2
 # files
6443c2
 _unsupported_imgopts 'refcount_bits=\([^1]\|.\([^6]\|$\)\)' data_file
6443c2
 
6443c2
+# This test either needs sudo -n losetup or FUSE exports to work
6443c2
+if sudo -n losetup &>/dev/null; then
6443c2
+    loopdev=true
6443c2
+else
6443c2
+    loopdev=false
6443c2
+
6443c2
+    # QSD --export fuse will either yield "Parameter 'id' is missing"
6443c2
+    # or "Invalid parameter 'fuse'", depending on whether there is
6443c2
+    # FUSE support or not.
6443c2
+    error=$($QSD --export fuse 2>&1)
6443c2
+    if [[ $error = *"'fuse'"* ]]; then
6443c2
+        _notrun 'Passwordless sudo for losetup or FUSE support required, but' \
6443c2
+                'neither is available'
6443c2
+    fi
6443c2
+fi
6443c2
+
6443c2
 echo
6443c2
 echo '=== Repairing an image without any refcount table ==='
6443c2
 echo
6443c2
@@ -138,6 +161,244 @@ _make_test_img 64M
6443c2
 poke_file "$TEST_IMG" $((0x10008)) "\xff\xff\xff\xff\xff\xff\x00\x00"
6443c2
 _check_test_img -r all
6443c2
 
6443c2
+echo
6443c2
+echo '=== Check rebuilt reftable location ==='
6443c2
+
6443c2
+# In an earlier version of the refcount rebuild algorithm, the
6443c2
+# reftable was generally placed at the image end (unless something was
6443c2
+# allocated in the area covered by the refblock right before the image
6443c2
+# file end, then we would try to place the reftable in that refblock).
6443c2
+# This was later changed so the reftable would be placed in the
6443c2
+# earliest possible location.  Test this.
6443c2
+
6443c2
+echo
6443c2
+echo '--- Does the image size increase? ---'
6443c2
+echo
6443c2
+
6443c2
+# First test: Just create some image, write some data to it, and
6443c2
+# resize it so there is free space at the end of the image (enough
6443c2
+# that it spans at least one full refblock, which for cluster_size=512
6443c2
+# images, spans 128k).  With the old algorithm, the reftable would
6443c2
+# have then been placed at the end of the image file, but with the new
6443c2
+# one, it will be put in that free space.
6443c2
+# We want to check whether the size of the image file increases due to
6443c2
+# rebuilding the refcount structures (it should not).
6443c2
+
6443c2
+_make_test_img -o 'cluster_size=512' 1M
6443c2
+# Write something
6443c2
+$QEMU_IO -c 'write 0 64k' "$TEST_IMG" | _filter_qemu_io
6443c2
+
6443c2
+# Add free space
6443c2
+file_len=$(stat -c '%s' "$TEST_IMG")
6443c2
+truncate -s $((file_len + 256 * 1024)) "$TEST_IMG"
6443c2
+
6443c2
+# Corrupt the image by saying the image header was not allocated
6443c2
+rt_offset=$(peek_file_be "$TEST_IMG" 48 8)
6443c2
+rb_offset=$(peek_file_be "$TEST_IMG" $rt_offset 8)
6443c2
+poke_file "$TEST_IMG" $rb_offset "\x00\x00"
6443c2
+
6443c2
+# Check whether rebuilding the refcount structures increases the image
6443c2
+# file size
6443c2
+file_len=$(stat -c '%s' "$TEST_IMG")
6443c2
+echo
6443c2
+# The only leaks there can be are the old refcount structures that are
6443c2
+# leaked during rebuilding, no need to clutter the output with them
6443c2
+_check_test_img -r all | grep -v '^Repairing cluster.*refcount=1 reference=0'
6443c2
+echo
6443c2
+post_repair_file_len=$(stat -c '%s' "$TEST_IMG")
6443c2
+
6443c2
+if [[ $file_len -eq $post_repair_file_len ]]; then
6443c2
+    echo 'OK: Image size did not change'
6443c2
+else
6443c2
+    echo 'ERROR: Image size differs' \
6443c2
+        "($file_len before, $post_repair_file_len after)"
6443c2
+fi
6443c2
+
6443c2
+echo
6443c2
+echo '--- Will the reftable occupy a hole specifically left for it?  ---'
6443c2
+echo
6443c2
+
6443c2
+# Note: With cluster_size=512, every refblock covers 128k.
6443c2
+# The reftable covers 8M per reftable cluster.
6443c2
+
6443c2
+# Create an image that requires two reftable clusters (just because
6443c2
+# this is more interesting than a single-clustered reftable).
6443c2
+_make_test_img -o 'cluster_size=512' 9M
6443c2
+$QEMU_IO -c 'write 0 8M' "$TEST_IMG" | _filter_qemu_io
6443c2
+
6443c2
+# Writing 8M will have resized the reftable.  Unfortunately, doing so
6443c2
+# will leave holes in the file, so we need to fill them up so we can
6443c2
+# be sure the whole file is allocated.  Do that by writing
6443c2
+# consecutively smaller chunks starting from 8 MB, until the file
6443c2
+# length increases even with a chunk size of 512.  Then we must have
6443c2
+# filled all holes.
6443c2
+ofs=$((8 * 1024 * 1024))
6443c2
+block_len=$((16 * 1024))
6443c2
+while [[ $block_len -ge 512 ]]; do
6443c2
+    file_len=$(stat -c '%s' "$TEST_IMG")
6443c2
+    while [[ $(stat -c '%s' "$TEST_IMG") -eq $file_len ]]; do
6443c2
+        # Do not include this in the reference output, it does not
6443c2
+        # really matter which qemu-io calls we do here exactly
6443c2
+        $QEMU_IO -c "write $ofs $block_len" "$TEST_IMG" >/dev/null
6443c2
+        ofs=$((ofs + block_len))
6443c2
+    done
6443c2
+    block_len=$((block_len / 2))
6443c2
+done
6443c2
+
6443c2
+# Fill up to 9M (do not include this in the reference output either,
6443c2
+# $ofs is random for all we know)
6443c2
+$QEMU_IO -c "write $ofs $((9 * 1024 * 1024 - ofs))" "$TEST_IMG" >/dev/null
6443c2
+
6443c2
+# Make space as follows:
6443c2
+# - For the first refblock: Right at the beginning of the image (this
6443c2
+#   refblock is placed in the first place possible),
6443c2
+# - For the reftable somewhere soon afterwards, still near the
6443c2
+#   beginning of the image (i.e. covered by the first refblock); the
6443c2
+#   reftable too is placed in the first place possible, but only after
6443c2
+#   all refblocks have been placed)
6443c2
+# No space is needed for the other refblocks, because no refblock is
6443c2
+# put before the space it covers.  In this test case, we do not mind
6443c2
+# if they are placed at the image file's end.
6443c2
+
6443c2
+# Before we make that space, we have to find out the host offset of
6443c2
+# the area that belonged to the two data clusters at guest offset 4k,
6443c2
+# because we expect the reftable to be placed there, and we will have
6443c2
+# to verify that it is.
6443c2
+
6443c2
+l1_offset=$(peek_file_be "$TEST_IMG" 40 8)
6443c2
+l2_offset=$(peek_file_be "$TEST_IMG" $l1_offset 8)
6443c2
+l2_offset=$((l2_offset & 0x00fffffffffffe00))
6443c2
+data_4k_offset=$(peek_file_be "$TEST_IMG" \
6443c2
+                 $((l2_offset + 4096 / 512 * 8)) 8)
6443c2
+data_4k_offset=$((data_4k_offset & 0x00fffffffffffe00))
6443c2
+
6443c2
+$QEMU_IO -c "discard 0 512" -c "discard 4k 1k" "$TEST_IMG" | _filter_qemu_io
6443c2
+
6443c2
+# Corrupt the image by saying the image header was not allocated
6443c2
+rt_offset=$(peek_file_be "$TEST_IMG" 48 8)
6443c2
+rb_offset=$(peek_file_be "$TEST_IMG" $rt_offset 8)
6443c2
+poke_file "$TEST_IMG" $rb_offset "\x00\x00"
6443c2
+
6443c2
+echo
6443c2
+# The only leaks there can be are the old refcount structures that are
6443c2
+# leaked during rebuilding, no need to clutter the output with them
6443c2
+_check_test_img -r all | grep -v '^Repairing cluster.*refcount=1 reference=0'
6443c2
+echo
6443c2
+
6443c2
+# Check whether the reftable was put where we expected
6443c2
+rt_offset=$(peek_file_be "$TEST_IMG" 48 8)
6443c2
+if [[ $rt_offset -eq $data_4k_offset ]]; then
6443c2
+    echo 'OK: Reftable is where we expect it'
6443c2
+else
6443c2
+    echo "ERROR: Reftable is at $rt_offset, but was expected at $data_4k_offset"
6443c2
+fi
6443c2
+
6443c2
+echo
6443c2
+echo '--- Rebuilding refcount structures on block devices ---'
6443c2
+echo
6443c2
+
6443c2
+# A block device cannot really grow, at least not during qemu-img
6443c2
+# check.  As mentioned in the above cases, rebuilding the refcount
6443c2
+# structure may lead to new refcount structures being written after
6443c2
+# the end of the image, and in the past that happened even if there
6443c2
+# was more than sufficient space in the image.  Such post-EOF writes
6443c2
+# will not work on block devices, so test that the new algorithm
6443c2
+# avoids it.
6443c2
+
6443c2
+# If we have passwordless sudo and losetup, we can use those to create
6443c2
+# a block device.  Otherwise, we can resort to qemu's FUSE export to
6443c2
+# create a file that isn't growable, which effectively tests the same
6443c2
+# thing.
6443c2
+
6443c2
+_cleanup_test_img
6443c2
+truncate -s $((64 * 1024 * 1024)) "$TEST_IMG"
6443c2
+
6443c2
+if $loopdev; then
6443c2
+    export_mp=$(sudo -n losetup --show -f "$TEST_IMG")
6443c2
+    export_mp_driver=host_device
6443c2
+    sudo -n chmod go+rw "$export_mp"
6443c2
+else
6443c2
+    # Create non-growable FUSE export that is a bit like an empty
6443c2
+    # block device
6443c2
+    export_mp="$TEST_DIR/fuse-export"
6443c2
+    export_mp_driver=file
6443c2
+    touch "$export_mp"
6443c2
+
6443c2
+    $QSD \
6443c2
+        --blockdev file,node-name=export-node,filename="$TEST_IMG" \
6443c2
+        --export fuse,id=fuse-export,node-name=export-node,mountpoint="$export_mp",writable=on,growable=off \
6443c2
+        --pidfile "$TEST_DIR/qsd.pid" \
6443c2
+        &
6443c2
+
6443c2
+    while [ ! -f "$TEST_DIR/qsd.pid" ]; do
6443c2
+        sleep 0.1
6443c2
+    done
6443c2
+fi
6443c2
+
6443c2
+# Now create a qcow2 image on the device -- unfortunately, qemu-img
6443c2
+# create force-creates the file, so we have to resort to the
6443c2
+# blockdev-create job.
6443c2
+_launch_qemu \
6443c2
+    --blockdev $export_mp_driver,node-name=file,filename="$export_mp"
6443c2
+
6443c2
+_send_qemu_cmd \
6443c2
+    $QEMU_HANDLE \
6443c2
+    '{ "execute": "qmp_capabilities" }' \
6443c2
+    'return'
6443c2
+
6443c2
+# Small cluster size again, so the image needs multiple refblocks
6443c2
+_send_qemu_cmd \
6443c2
+    $QEMU_HANDLE \
6443c2
+    '{ "execute": "blockdev-create",
6443c2
+       "arguments": {
6443c2
+           "job-id": "create",
6443c2
+           "options": {
6443c2
+               "driver": "qcow2",
6443c2
+               "file": "file",
6443c2
+               "size": '$((64 * 1024 * 1024))',
6443c2
+               "cluster-size": 512
6443c2
+           } } }' \
6443c2
+    '"concluded"'
6443c2
+
6443c2
+_send_qemu_cmd \
6443c2
+    $QEMU_HANDLE \
6443c2
+    '{ "execute": "job-dismiss", "arguments": { "id": "create" } }' \
6443c2
+    'return'
6443c2
+
6443c2
+_send_qemu_cmd \
6443c2
+    $QEMU_HANDLE \
6443c2
+    '{ "execute": "quit" }' \
6443c2
+    'return'
6443c2
+
6443c2
+wait=y _cleanup_qemu
6443c2
+echo
6443c2
+
6443c2
+# Write some data
6443c2
+$QEMU_IO -c 'write 0 64k' "$export_mp" | _filter_qemu_io
6443c2
+
6443c2
+# Corrupt the image by saying the image header was not allocated
6443c2
+rt_offset=$(peek_file_be "$export_mp" 48 8)
6443c2
+rb_offset=$(peek_file_be "$export_mp" $rt_offset 8)
6443c2
+poke_file "$export_mp" $rb_offset "\x00\x00"
6443c2
+
6443c2
+# Repairing such a simple case should just work
6443c2
+# (We used to put the reftable at the end of the image file, which can
6443c2
+# never work for non-growable devices.)
6443c2
+echo
6443c2
+TEST_IMG="$export_mp" _check_test_img -r all \
6443c2
+    | grep -v '^Repairing cluster.*refcount=1 reference=0'
6443c2
+
6443c2
+if $loopdev; then
6443c2
+    sudo -n losetup -d "$export_mp"
6443c2
+else
6443c2
+    qsd_pid=$(cat "$TEST_DIR/qsd.pid")
6443c2
+    kill -TERM "$qsd_pid"
6443c2
+    # Wait for process to exit (cannot `wait` because the QSD is daemonized)
6443c2
+    while [ -f "$TEST_DIR/qsd.pid" ]; do
6443c2
+        true
6443c2
+    done
6443c2
+fi
6443c2
+
6443c2
 # success, all done
6443c2
 echo '*** done'
6443c2
 rm -f $seq.full
6443c2
diff --git a/tests/qemu-iotests/108.out b/tests/qemu-iotests/108.out
6443c2
index 75bab8dc84..b5401d788d 100644
6443c2
--- a/tests/qemu-iotests/108.out
6443c2
+++ b/tests/qemu-iotests/108.out
6443c2
@@ -105,6 +105,87 @@ The following inconsistencies were found and repaired:
6443c2
     0 leaked clusters
6443c2
     1 corruptions
6443c2
 
6443c2
+Double checking the fixed image now...
6443c2
+No errors were found on the image.
6443c2
+
6443c2
+=== Check rebuilt reftable location ===
6443c2
+
6443c2
+--- Does the image size increase? ---
6443c2
+
6443c2
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576
6443c2
+wrote 65536/65536 bytes at offset 0
6443c2
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
6443c2
+
6443c2
+ERROR cluster 0 refcount=0 reference=1
6443c2
+Rebuilding refcount structure
6443c2
+The following inconsistencies were found and repaired:
6443c2
+
6443c2
+    0 leaked clusters
6443c2
+    1 corruptions
6443c2
+
6443c2
+Double checking the fixed image now...
6443c2
+No errors were found on the image.
6443c2
+
6443c2
+OK: Image size did not change
6443c2
+
6443c2
+--- Will the reftable occupy a hole specifically left for it?  ---
6443c2
+
6443c2
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=9437184
6443c2
+wrote 8388608/8388608 bytes at offset 0
6443c2
+8 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
6443c2
+discard 512/512 bytes at offset 0
6443c2
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
6443c2
+discard 1024/1024 bytes at offset 4096
6443c2
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
6443c2
+
6443c2
+ERROR cluster 0 refcount=0 reference=1
6443c2
+Rebuilding refcount structure
6443c2
+The following inconsistencies were found and repaired:
6443c2
+
6443c2
+    0 leaked clusters
6443c2
+    1 corruptions
6443c2
+
6443c2
+Double checking the fixed image now...
6443c2
+No errors were found on the image.
6443c2
+
6443c2
+OK: Reftable is where we expect it
6443c2
+
6443c2
+--- Rebuilding refcount structures on block devices ---
6443c2
+
6443c2
+{ "execute": "qmp_capabilities" }
6443c2
+{"return": {}}
6443c2
+{ "execute": "blockdev-create",
6443c2
+       "arguments": {
6443c2
+           "job-id": "create",
6443c2
+           "options": {
6443c2
+               "driver": "IMGFMT",
6443c2
+               "file": "file",
6443c2
+               "size": 67108864,
6443c2
+               "cluster-size": 512
6443c2
+           } } }
6443c2
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "created", "id": "create"}}
6443c2
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "running", "id": "create"}}
6443c2
+{"return": {}}
6443c2
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "waiting", "id": "create"}}
6443c2
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "pending", "id": "create"}}
6443c2
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "concluded", "id": "create"}}
6443c2
+{ "execute": "job-dismiss", "arguments": { "id": "create" } }
6443c2
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "null", "id": "create"}}
6443c2
+{"return": {}}
6443c2
+{ "execute": "quit" }
6443c2
+{"return": {}}
6443c2
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN", "data": {"guest": false, "reason": "host-qmp-quit"}}
6443c2
+
6443c2
+wrote 65536/65536 bytes at offset 0
6443c2
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
6443c2
+
6443c2
+ERROR cluster 0 refcount=0 reference=1
6443c2
+Rebuilding refcount structure
6443c2
+The following inconsistencies were found and repaired:
6443c2
+
6443c2
+    0 leaked clusters
6443c2
+    1 corruptions
6443c2
+
6443c2
 Double checking the fixed image now...
6443c2
 No errors were found on the image.
6443c2
 *** done
6443c2
-- 
6443c2
2.27.0
6443c2