Merge remote-tracking branch 'remotes/ericb/tags/pull-nbd-2020-10-27-v2' into staging

nbd patches for 2020-10-27

- Tweak the new block-export-add QMP command
- Allow multiple -B options for qemu-nbd
- Add qemu:allocation-depth metadata context as qemu-nbd -A
- Improve iotest use of NBD

# gpg: Signature made Fri 30 Oct 2020 20:22:42 GMT
# gpg:                using RSA key 71C2CC22B1C4602927D2F3AAA7A16B4A2527436A
# gpg: Good signature from "Eric Blake <eblake@redhat.com>" [full]
# gpg:                 aka "Eric Blake (Free Software Programmer) <ebb9@byu.net>" [full]
# gpg:                 aka "[jpeg image of size 6874]" [full]
# Primary key fingerprint: 71C2 CC22 B1C4 6029 27D2  F3AA A7A1 6B4A 2527 436A

* remotes/ericb/tags/pull-nbd-2020-10-27-v2:
  nbd: Add 'qemu-nbd -A' to expose allocation depth
  nbd: Add new qemu:allocation-depth metadata context
  block: Return depth level during bdrv_is_allocated_above
  nbd: Allow export of multiple bitmaps for one device
  nbd: Refactor counting of metadata contexts
  nbd: Simplify qemu bitmap context name
  nbd: Update qapi to support exporting multiple bitmaps
  nbd: Utilize QAPI_CLONE for type conversion
  qapi: Add QAPI_LIST_PREPEND() macro
  block: Simplify QAPI_LIST_ADD
  iotests/291: Stop NBD server
  iotests/291: Filter irrelevant parts of img-info

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>

5 files changed, 115 insertions, 27 deletions

diff --git a/tests/qemu-iotests/291 b/tests/qemu-iotests/291
index 4f837b2056..ecef9eec62 100755
--- a/tests/qemu-iotests/291
+++ b/tests/qemu-iotests/291
@@ -42,6 +42,14 @@ _require_command QEMU_NBD
 # compat=0.10 does not support bitmaps
 _unsupported_imgopts 'compat=0.10'
 
+# Filter irrelevant format-specific information from the qemu-img info
+# output (we only want the bitmaps, basically)
+_filter_irrelevant_img_info()
+{
+    grep -v -e 'compat' -e 'compression type' -e 'data file' -e 'extended l2' \
+            -e 'lazy refcounts' -e 'refcount bits'
+}
+
 echo
 echo "=== Initial image setup ==="
 echo
@@ -79,7 +87,7 @@ echo
 
 # Only bitmaps from the active layer are copied
 $QEMU_IMG convert --bitmaps -O qcow2 "$TEST_IMG.orig" "$TEST_IMG"
-_img_info --format-specific
+_img_info --format-specific | _filter_irrelevant_img_info
 # But we can also merge in bitmaps from other layers.  This test is a bit
 # contrived to cover more code paths, in reality, you could merge directly
 # into b0 without going through tmp
@@ -89,7 +97,7 @@ $QEMU_IMG bitmap --add --merge b0 -b "$TEST_IMG.base" -F $IMGFMT \
 $QEMU_IMG bitmap --merge tmp -f $IMGFMT "$TEST_IMG" b0
 $QEMU_IMG bitmap --remove --image-opts \
     driver=$IMGFMT,file.driver=file,file.filename="$TEST_IMG" tmp
-_img_info --format-specific
+_img_info --format-specific | _filter_irrelevant_img_info
 
 echo
 echo "=== Merge from top layer into backing image ==="
@@ -98,7 +106,7 @@ echo
 $QEMU_IMG rebase -u -F qcow2 -b "$TEST_IMG.base" "$TEST_IMG"
 $QEMU_IMG bitmap --add --merge b2 -b "$TEST_IMG" -F $IMGFMT \
      -f $IMGFMT "$TEST_IMG.base" b3
-_img_info --format-specific --backing-chain
+_img_info --format-specific --backing-chain | _filter_irrelevant_img_info
 
 echo
 echo "=== Check bitmap contents ==="
@@ -107,19 +115,19 @@ echo
 # x-dirty-bitmap is a hack for reading bitmaps; it abuses block status to
 # report "data":false for portions of the bitmap which are set
 IMG="driver=nbd,server.type=unix,server.path=$nbd_unix_socket"
-nbd_server_start_unix_socket -r -f qcow2 -B b0 "$TEST_IMG"
+nbd_server_start_unix_socket -r -f qcow2 \
+    -B b0 -B b1 -B b2 -B b3 "$TEST_IMG"
 $QEMU_IMG map --output=json --image-opts \
     "$IMG,x-dirty-bitmap=qemu:dirty-bitmap:b0" | _filter_qemu_img_map
-nbd_server_start_unix_socket -r -f qcow2 -B b1 "$TEST_IMG"
 $QEMU_IMG map --output=json --image-opts \
     "$IMG,x-dirty-bitmap=qemu:dirty-bitmap:b1" | _filter_qemu_img_map
-nbd_server_start_unix_socket -r -f qcow2 -B b2 "$TEST_IMG"
 $QEMU_IMG map --output=json --image-opts \
     "$IMG,x-dirty-bitmap=qemu:dirty-bitmap:b2" | _filter_qemu_img_map
-nbd_server_start_unix_socket -r -f qcow2 -B b3 "$TEST_IMG"
 $QEMU_IMG map --output=json --image-opts \
     "$IMG,x-dirty-bitmap=qemu:dirty-bitmap:b3" | _filter_qemu_img_map
 
+nbd_server_stop
+
 # success, all done
 echo '*** done'
 rm -f $seq.full
diff --git a/tests/qemu-iotests/291.out b/tests/qemu-iotests/291.out
index 3990f7aacc..23411c0ff4 100644
--- a/tests/qemu-iotests/291.out
+++ b/tests/qemu-iotests/291.out
@@ -26,9 +26,6 @@ file format: IMGFMT
 virtual size: 10 MiB (10485760 bytes)
 cluster_size: 65536
 Format specific information:
-    compat: 1.1
-    compression type: zlib
-    lazy refcounts: false
     bitmaps:
         [0]:
             flags:
@@ -39,17 +36,12 @@ Format specific information:
                 [0]: auto
             name: b2
             granularity: 65536
-    refcount bits: 16
     corrupt: false
-    extended l2: false
 image: TEST_DIR/t.IMGFMT
 file format: IMGFMT
 virtual size: 10 MiB (10485760 bytes)
 cluster_size: 65536
 Format specific information:
-    compat: 1.1
-    compression type: zlib
-    lazy refcounts: false
     bitmaps:
         [0]:
             flags:
@@ -64,9 +56,7 @@ Format specific information:
             flags:
             name: b0
             granularity: 65536
-    refcount bits: 16
     corrupt: false
-    extended l2: false
 
 === Merge from top layer into backing image ===
 
@@ -77,9 +67,6 @@ cluster_size: 65536
 backing file: TEST_DIR/t.IMGFMT.base
 backing file format: IMGFMT
 Format specific information:
-    compat: 1.1
-    compression type: zlib
-    lazy refcounts: false
     bitmaps:
         [0]:
             flags:
@@ -94,18 +81,13 @@ Format specific information:
             flags:
             name: b0
             granularity: 65536
-    refcount bits: 16
     corrupt: false
-    extended l2: false
 
 image: TEST_DIR/t.IMGFMT.base
 file format: IMGFMT
 virtual size: 10 MiB (10485760 bytes)
 cluster_size: 65536
 Format specific information:
-    compat: 1.1
-    compression type: zlib
-    lazy refcounts: false
     bitmaps:
         [0]:
             flags:
@@ -117,9 +99,7 @@ Format specific information:
                 [0]: auto
             name: b3
             granularity: 65536
-    refcount bits: 16
     corrupt: false
-    extended l2: false
 
 === Check bitmap contents ===
 
diff --git a/tests/qemu-iotests/309 b/tests/qemu-iotests/309
new file mode 100755
index 0000000000..fb61157c2e
--- /dev/null
+++ b/tests/qemu-iotests/309
@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+#
+# Test qemu-nbd -A
+#
+# Copyright (C) 2018-2020 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+status=1 # failure is the default!
+
+_cleanup()
+{
+    _cleanup_test_img
+    nbd_server_stop
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+. ./common.nbd
+
+_supported_fmt qcow2
+_supported_proto file
+_supported_os Linux
+_require_command QEMU_NBD
+
+echo
+echo "=== Initial image setup ==="
+echo
+
+TEST_IMG="$TEST_IMG.base" _make_test_img 4M
+$QEMU_IO -c 'w 0 2M' -f $IMGFMT "$TEST_IMG.base" | _filter_qemu_io
+_make_test_img -b "$TEST_IMG.base" -F $IMGFMT 4M
+$QEMU_IO -c 'w 1M 2M' -f $IMGFMT "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo "=== Check allocation over NBD ==="
+echo
+
+$QEMU_IMG map --output=json -f qcow2 "$TEST_IMG"
+IMG="driver=nbd,server.type=unix,server.path=$nbd_unix_socket"
+nbd_server_start_unix_socket -r -f qcow2 -A "$TEST_IMG"
+# Normal -f raw NBD block status loses access to allocation information
+$QEMU_IMG map --output=json --image-opts \
+    "$IMG" | _filter_qemu_img_map
+# But when we use -A, coupled with x-dirty-bitmap in the client for feeding
+# 2-bit block status from an alternative NBD metadata context (note that
+# the client code for x-dirty-bitmap intentionally collapses all depths
+# beyond 2 into a single value), we can determine:
+#    unallocated (depth 0) => "zero":false, "data":true
+#    local (depth 1)       => "zero":false, "data":false
+#    backing (depth 2+)    => "zero":true,  "data":true
+$QEMU_IMG map --output=json --image-opts \
+    "$IMG,x-dirty-bitmap=qemu:allocation-depth" | _filter_qemu_img_map
+# More accurate results can be obtained by other NBD clients such as
+# libnbd, but this test works without such external dependencies.
+
+# success, all done
+echo '*** done'
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/309.out b/tests/qemu-iotests/309.out
new file mode 100644
index 0000000000..db75bb6b0d
--- /dev/null
+++ b/tests/qemu-iotests/309.out
@@ -0,0 +1,22 @@
+QA output created by 309
+
+=== Initial image setup ===
+
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=4194304
+wrote 2097152/2097152 bytes at offset 0
+2 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4194304 backing_file=TEST_DIR/t.IMGFMT.base backing_fmt=IMGFMT
+wrote 2097152/2097152 bytes at offset 1048576
+2 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+=== Check allocation over NBD ===
+
+[{ "start": 0, "length": 1048576, "depth": 1, "zero": false, "data": true, "offset": 327680},
+{ "start": 1048576, "length": 2097152, "depth": 0, "zero": false, "data": true, "offset": 327680},
+{ "start": 3145728, "length": 1048576, "depth": 1, "zero": true, "data": false}]
+[{ "start": 0, "length": 3145728, "depth": 0, "zero": false, "data": true, "offset": OFFSET},
+{ "start": 3145728, "length": 1048576, "depth": 0, "zero": true, "data": false, "offset": OFFSET}]
+[{ "start": 0, "length": 1048576, "depth": 0, "zero": true, "data": true, "offset": OFFSET},
+{ "start": 1048576, "length": 2097152, "depth": 0, "zero": false, "data": false},
+{ "start": 3145728, "length": 1048576, "depth": 0, "zero": false, "data": true, "offset": OFFSET}]
+*** done
diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group
index 3432989283..2960dff728 100644
--- a/tests/qemu-iotests/group
+++ b/tests/qemu-iotests/group
@@ -315,3 +315,4 @@
 304 rw quick
 305 rw quick
 307 rw quick export
+309 rw auto quick