From b4af1df08f9433d858ec01aff4ec9f06cb28128b Mon Sep 17 00:00:00 2001 From: Max Reitz Date: Tue, 27 Oct 2020 17:44:15 +0100 Subject: iotests/291: Filter irrelevant parts of img-info We need to let _img_info emit the format-specific information so we get the list of bitmaps we want, but we do not need anything but the bitmaps. So filter out everything that is irrelevant to us. (Ideally, this would be a generalized function in common.filters that takes a list of things to keep, but that would require implementing an anti-bitmap filter, which would be hard, and which we do not need here. So that is why this function is just a local hack.) This lets 291 pass with qcow2 options like refcount_bits or data_file again. Fixes: 14f16bf9474c860ecc127a66a86961942319f7af ("qemu-img: Support bitmap --merge into backing image") Signed-off-by: Max Reitz Message-Id: <20201027164416.144115-2-mreitz@redhat.com> Reviewed-by: Eric Blake Signed-off-by: Eric Blake --- tests/qemu-iotests/291 | 14 +++++++++++--- tests/qemu-iotests/291.out | 20 -------------------- 2 files changed, 11 insertions(+), 23 deletions(-) (limited to 'tests') diff --git a/tests/qemu-iotests/291 b/tests/qemu-iotests/291 index 4f837b2056..77fa38f93d 100755 --- a/tests/qemu-iotests/291 +++ b/tests/qemu-iotests/291 @@ -42,6 +42,14 @@ _require_command QEMU_NBD # compat=0.10 does not support bitmaps _unsupported_imgopts 'compat=0.10' +# Filter irrelevant format-specific information from the qemu-img info +# output (we only want the bitmaps, basically) +_filter_irrelevant_img_info() +{ + grep -v -e 'compat' -e 'compression type' -e 'data file' -e 'extended l2' \ + -e 'lazy refcounts' -e 'refcount bits' +} + echo echo "=== Initial image setup ===" echo @@ -79,7 +87,7 @@ echo # Only bitmaps from the active layer are copied $QEMU_IMG convert --bitmaps -O qcow2 "$TEST_IMG.orig" "$TEST_IMG" -_img_info --format-specific +_img_info --format-specific | _filter_irrelevant_img_info # But we can also merge in bitmaps from other layers. This test is a bit # contrived to cover more code paths, in reality, you could merge directly # into b0 without going through tmp @@ -89,7 +97,7 @@ $QEMU_IMG bitmap --add --merge b0 -b "$TEST_IMG.base" -F $IMGFMT \ $QEMU_IMG bitmap --merge tmp -f $IMGFMT "$TEST_IMG" b0 $QEMU_IMG bitmap --remove --image-opts \ driver=$IMGFMT,file.driver=file,file.filename="$TEST_IMG" tmp -_img_info --format-specific +_img_info --format-specific | _filter_irrelevant_img_info echo echo "=== Merge from top layer into backing image ===" @@ -98,7 +106,7 @@ echo $QEMU_IMG rebase -u -F qcow2 -b "$TEST_IMG.base" "$TEST_IMG" $QEMU_IMG bitmap --add --merge b2 -b "$TEST_IMG" -F $IMGFMT \ -f $IMGFMT "$TEST_IMG.base" b3 -_img_info --format-specific --backing-chain +_img_info --format-specific --backing-chain | _filter_irrelevant_img_info echo echo "=== Check bitmap contents ===" diff --git a/tests/qemu-iotests/291.out b/tests/qemu-iotests/291.out index 3990f7aacc..23411c0ff4 100644 --- a/tests/qemu-iotests/291.out +++ b/tests/qemu-iotests/291.out @@ -26,9 +26,6 @@ file format: IMGFMT virtual size: 10 MiB (10485760 bytes) cluster_size: 65536 Format specific information: - compat: 1.1 - compression type: zlib - lazy refcounts: false bitmaps: [0]: flags: @@ -39,17 +36,12 @@ Format specific information: [0]: auto name: b2 granularity: 65536 - refcount bits: 16 corrupt: false - extended l2: false image: TEST_DIR/t.IMGFMT file format: IMGFMT virtual size: 10 MiB (10485760 bytes) cluster_size: 65536 Format specific information: - compat: 1.1 - compression type: zlib - lazy refcounts: false bitmaps: [0]: flags: @@ -64,9 +56,7 @@ Format specific information: flags: name: b0 granularity: 65536 - refcount bits: 16 corrupt: false - extended l2: false === Merge from top layer into backing image === @@ -77,9 +67,6 @@ cluster_size: 65536 backing file: TEST_DIR/t.IMGFMT.base backing file format: IMGFMT Format specific information: - compat: 1.1 - compression type: zlib - lazy refcounts: false bitmaps: [0]: flags: @@ -94,18 +81,13 @@ Format specific information: flags: name: b0 granularity: 65536 - refcount bits: 16 corrupt: false - extended l2: false image: TEST_DIR/t.IMGFMT.base file format: IMGFMT virtual size: 10 MiB (10485760 bytes) cluster_size: 65536 Format specific information: - compat: 1.1 - compression type: zlib - lazy refcounts: false bitmaps: [0]: flags: @@ -117,9 +99,7 @@ Format specific information: [0]: auto name: b3 granularity: 65536 - refcount bits: 16 corrupt: false - extended l2: false === Check bitmap contents === -- cgit v1.2.3 From cb7223ac33fa35976d64f31145122965c1ba332d Mon Sep 17 00:00:00 2001 From: Max Reitz Date: Tue, 27 Oct 2020 17:44:16 +0100 Subject: iotests/291: Stop NBD server nbd_server_start_unix_socket() includes an implicit nbd_server_stop(), but we still need an explicit one at the end of the test (where there follows no next nbd_server_start_unix_socket()), or qemu-nbd will linger until the test exits. This will become important when enabling this test to run on FUSE exports, because then the export (which is the image used by qemu-nbd) will go away before qemu-nbd exits, which will lead to qemu-nbd complaining that it cannot flush the bitmaps in the image. Signed-off-by: Max Reitz Message-Id: <20201027164416.144115-3-mreitz@redhat.com> Reviewed-by: Eric Blake Signed-off-by: Eric Blake --- tests/qemu-iotests/291 | 2 ++ 1 file changed, 2 insertions(+) (limited to 'tests') diff --git a/tests/qemu-iotests/291 b/tests/qemu-iotests/291 index 77fa38f93d..b7320bc7ad 100755 --- a/tests/qemu-iotests/291 +++ b/tests/qemu-iotests/291 @@ -128,6 +128,8 @@ nbd_server_start_unix_socket -r -f qcow2 -B b3 "$TEST_IMG" $QEMU_IMG map --output=json --image-opts \ "$IMG,x-dirty-bitmap=qemu:dirty-bitmap:b3" | _filter_qemu_img_map +nbd_server_stop + # success, all done echo '*** done' rm -f $seq.full -- cgit v1.2.3 From 3b1f244c59b7045680e615d50dc444a316abd891 Mon Sep 17 00:00:00 2001 From: Eric Blake Date: Tue, 27 Oct 2020 00:05:52 -0500 Subject: nbd: Allow export of multiple bitmaps for one device With this, 'qemu-nbd -B b0 -B b1 -f qcow2 img.qcow2' can let you sniff out multiple bitmaps from one server. qemu-img as client can still only read one bitmap per client connection, but other NBD clients (hello libnbd) can now read multiple bitmaps in a single pass. Signed-off-by: Eric Blake Reviewed-by: Vladimir Sementsov-Ogievskiy Message-Id: <20201027050556.269064-8-eblake@redhat.com> --- tests/qemu-iotests/291 | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'tests') diff --git a/tests/qemu-iotests/291 b/tests/qemu-iotests/291 index b7320bc7ad..ecef9eec62 100755 --- a/tests/qemu-iotests/291 +++ b/tests/qemu-iotests/291 @@ -115,16 +115,14 @@ echo # x-dirty-bitmap is a hack for reading bitmaps; it abuses block status to # report "data":false for portions of the bitmap which are set IMG="driver=nbd,server.type=unix,server.path=$nbd_unix_socket" -nbd_server_start_unix_socket -r -f qcow2 -B b0 "$TEST_IMG" +nbd_server_start_unix_socket -r -f qcow2 \ + -B b0 -B b1 -B b2 -B b3 "$TEST_IMG" $QEMU_IMG map --output=json --image-opts \ "$IMG,x-dirty-bitmap=qemu:dirty-bitmap:b0" | _filter_qemu_img_map -nbd_server_start_unix_socket -r -f qcow2 -B b1 "$TEST_IMG" $QEMU_IMG map --output=json --image-opts \ "$IMG,x-dirty-bitmap=qemu:dirty-bitmap:b1" | _filter_qemu_img_map -nbd_server_start_unix_socket -r -f qcow2 -B b2 "$TEST_IMG" $QEMU_IMG map --output=json --image-opts \ "$IMG,x-dirty-bitmap=qemu:dirty-bitmap:b2" | _filter_qemu_img_map -nbd_server_start_unix_socket -r -f qcow2 -B b3 "$TEST_IMG" $QEMU_IMG map --output=json --image-opts \ "$IMG,x-dirty-bitmap=qemu:dirty-bitmap:b3" | _filter_qemu_img_map -- cgit v1.2.3 From dbc7b01492371e4a54b92d2b6d968f9b863cc794 Mon Sep 17 00:00:00 2001 From: Eric Blake Date: Tue, 27 Oct 2020 00:05:55 -0500 Subject: nbd: Add 'qemu-nbd -A' to expose allocation depth Allow the server to expose an additional metacontext to be requested by savvy clients. qemu-nbd adds a new option -A to expose the qemu:allocation-depth metacontext through NBD_CMD_BLOCK_STATUS; this can also be set via QMP when using block-export-add. qemu as client is hacked into viewing the key aspects of this new context by abusing the already-experimental x-dirty-bitmap option to collapse all depths greater than 2, which results in a tri-state value visible in the output of 'qemu-img map --output=json' (yes, that means x-dirty-bitmap is now a bit of a misnomer, but I didn't feel like renaming it as it would introduce a needless break of back-compat, even though we make no compat guarantees with x- members): unallocated (depth 0) => "zero":false, "data":true local (depth 1) => "zero":false, "data":false backing (depth 2+) => "zero":true, "data":true libnbd as client is probably a nicer way to get at the information without having to decipher such hacks in qemu as client. ;) Signed-off-by: Eric Blake Message-Id: <20201027050556.269064-11-eblake@redhat.com> Reviewed-by: Vladimir Sementsov-Ogievskiy --- tests/qemu-iotests/309 | 77 ++++++++++++++++++++++++++++++++++++++++++++++ tests/qemu-iotests/309.out | 22 +++++++++++++ tests/qemu-iotests/group | 1 + 3 files changed, 100 insertions(+) create mode 100755 tests/qemu-iotests/309 create mode 100644 tests/qemu-iotests/309.out (limited to 'tests') diff --git a/tests/qemu-iotests/309 b/tests/qemu-iotests/309 new file mode 100755 index 0000000000..fb61157c2e --- /dev/null +++ b/tests/qemu-iotests/309 @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# +# Test qemu-nbd -A +# +# Copyright (C) 2018-2020 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +seq="$(basename $0)" +echo "QA output created by $seq" + +status=1 # failure is the default! + +_cleanup() +{ + _cleanup_test_img + nbd_server_stop +} +trap "_cleanup; exit \$status" 0 1 2 3 15 + +# get standard environment, filters and checks +. ./common.rc +. ./common.filter +. ./common.nbd + +_supported_fmt qcow2 +_supported_proto file +_supported_os Linux +_require_command QEMU_NBD + +echo +echo "=== Initial image setup ===" +echo + +TEST_IMG="$TEST_IMG.base" _make_test_img 4M +$QEMU_IO -c 'w 0 2M' -f $IMGFMT "$TEST_IMG.base" | _filter_qemu_io +_make_test_img -b "$TEST_IMG.base" -F $IMGFMT 4M +$QEMU_IO -c 'w 1M 2M' -f $IMGFMT "$TEST_IMG" | _filter_qemu_io + +echo +echo "=== Check allocation over NBD ===" +echo + +$QEMU_IMG map --output=json -f qcow2 "$TEST_IMG" +IMG="driver=nbd,server.type=unix,server.path=$nbd_unix_socket" +nbd_server_start_unix_socket -r -f qcow2 -A "$TEST_IMG" +# Normal -f raw NBD block status loses access to allocation information +$QEMU_IMG map --output=json --image-opts \ + "$IMG" | _filter_qemu_img_map +# But when we use -A, coupled with x-dirty-bitmap in the client for feeding +# 2-bit block status from an alternative NBD metadata context (note that +# the client code for x-dirty-bitmap intentionally collapses all depths +# beyond 2 into a single value), we can determine: +# unallocated (depth 0) => "zero":false, "data":true +# local (depth 1) => "zero":false, "data":false +# backing (depth 2+) => "zero":true, "data":true +$QEMU_IMG map --output=json --image-opts \ + "$IMG,x-dirty-bitmap=qemu:allocation-depth" | _filter_qemu_img_map +# More accurate results can be obtained by other NBD clients such as +# libnbd, but this test works without such external dependencies. + +# success, all done +echo '*** done' +rm -f $seq.full +status=0 diff --git a/tests/qemu-iotests/309.out b/tests/qemu-iotests/309.out new file mode 100644 index 0000000000..db75bb6b0d --- /dev/null +++ b/tests/qemu-iotests/309.out @@ -0,0 +1,22 @@ +QA output created by 309 + +=== Initial image setup === + +Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=4194304 +wrote 2097152/2097152 bytes at offset 0 +2 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4194304 backing_file=TEST_DIR/t.IMGFMT.base backing_fmt=IMGFMT +wrote 2097152/2097152 bytes at offset 1048576 +2 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) + +=== Check allocation over NBD === + +[{ "start": 0, "length": 1048576, "depth": 1, "zero": false, "data": true, "offset": 327680}, +{ "start": 1048576, "length": 2097152, "depth": 0, "zero": false, "data": true, "offset": 327680}, +{ "start": 3145728, "length": 1048576, "depth": 1, "zero": true, "data": false}] +[{ "start": 0, "length": 3145728, "depth": 0, "zero": false, "data": true, "offset": OFFSET}, +{ "start": 3145728, "length": 1048576, "depth": 0, "zero": true, "data": false, "offset": OFFSET}] +[{ "start": 0, "length": 1048576, "depth": 0, "zero": true, "data": true, "offset": OFFSET}, +{ "start": 1048576, "length": 2097152, "depth": 0, "zero": false, "data": false}, +{ "start": 3145728, "length": 1048576, "depth": 0, "zero": false, "data": true, "offset": OFFSET}] +*** done diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group index 3432989283..2960dff728 100644 --- a/tests/qemu-iotests/group +++ b/tests/qemu-iotests/group @@ -315,3 +315,4 @@ 304 rw quick 305 rw quick 307 rw quick export +309 rw auto quick -- cgit v1.2.3