Merge remote-tracking branch 'remotes/kevin/tags/for-upstream' into staging

Block layer patches (v2)

# gpg: Signature made Mon 14 Sep 2015 15:56:54 BST using RSA key ID C88F2FD6
# gpg: Good signature from "Kevin Wolf <kwolf@redhat.com>"

* remotes/kevin/tags/for-upstream: (23 commits)
  qcow2: Make qcow2_alloc_bytes() more explicit
  vmdk: Fix next_cluster_sector for compressed write
  iotests: Add test for checking large image files
  qcow2: Make size_to_clusters() return uint64_t
  qemu-iotests: More qcow2 reopen tests
  qemu-iotests: Reopen qcow2 with lazy-refcounts change
  qcow2: Support updating driver-specific options in reopen
  qcow2: Make qcow2_update_options() suitable for transactions
  qcow2: Fix memory leak in qcow2_update_options() error path
  qcow2: Leave s unchanged on qcow2_update_options() failure
  qcow2: Move rest of option handling to qcow2_update_options()
  qcow2: Move qcow2_update_options() call up
  qcow2: Factor out qcow2_update_options()
  qcow2: Improve error message
  qemu-io: Add command 'reopen'
  qemu-io: Remove duplicate 'open' error message
  block: Allow specifying driver-specific options to reopen
  qcow2: Rename BDRVQcowState to BDRVQcow2State
  block: Drop bdrv_find_whitelisted_format()
  block: Drop drv parameter from bdrv_fill_options()
  ...

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>

7 files changed, 316 insertions, 0 deletions

diff --git a/tests/qemu-iotests/039 b/tests/qemu-iotests/039
index 617f3977cc..9e9b379baa 100755
--- a/tests/qemu-iotests/039
+++ b/tests/qemu-iotests/039
@@ -147,6 +147,33 @@ $PYTHON qcow2.py "$TEST_IMG".base dump-header | grep incompatible_features
 _check_test_img
 TEST_IMG="$TEST_IMG".base _check_test_img
 
+echo
+echo "== Changing lazy_refcounts setting at runtime =="
+
+IMGOPTS="compat=1.1,lazy_refcounts=off"
+_make_test_img $size
+
+$QEMU_IO -c "reopen -o lazy-refcounts=on" \
+         -c "write -P 0x5a 0 512" \
+         -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 \
+    | _filter_qemu_io
+
+# The dirty bit must be set
+$PYTHON qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
+_check_test_img
+
+IMGOPTS="compat=1.1,lazy_refcounts=on"
+_make_test_img $size
+
+$QEMU_IO -c "reopen -o lazy-refcounts=off" \
+         -c "write -P 0x5a 0 512" \
+         -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 \
+    | _filter_qemu_io
+
+# The dirty bit must not be set
+$PYTHON qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
+_check_test_img
+
 
 # success, all done
 echo "*** done"
diff --git a/tests/qemu-iotests/039.out b/tests/qemu-iotests/039.out
index d8c5a44519..03a31c5943 100644
--- a/tests/qemu-iotests/039.out
+++ b/tests/qemu-iotests/039.out
@@ -74,4 +74,22 @@ incompatible_features     0x0
 incompatible_features     0x0
 No errors were found on the image.
 No errors were found on the image.
+
+== Changing lazy_refcounts setting at runtime ==
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
+wrote 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+./common.config: Killed                  ( exec "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@" )
+incompatible_features     0x1
+ERROR cluster 5 refcount=0 reference=1
+ERROR OFLAG_COPIED data cluster: l2_entry=8000000000050000 refcount=0
+
+2 errors were found on the image.
+Data may be corrupted, or further writes to the image may corrupt it.
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
+wrote 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+./common.config: Killed                  ( exec "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@" )
+incompatible_features     0x0
+No errors were found on the image.
 *** done
diff --git a/tests/qemu-iotests/137 b/tests/qemu-iotests/137
new file mode 100755
index 0000000000..9a6597cf9d
--- /dev/null
+++ b/tests/qemu-iotests/137
@@ -0,0 +1,145 @@
+#!/bin/bash
+#
+# Test qcow2 reopen
+#
+# Copyright (C) 2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=kwolf@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+tmp=/tmp/$$
+status=1	# failure is the default!
+
+_cleanup()
+{
+    _cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+. ./common.qemu
+
+_supported_fmt qcow2
+_supported_proto generic
+_supported_os Linux
+
+
+_make_test_img 64M
+
+echo === Try setting valid values for all options ===
+echo
+
+# Try all options and then check that all of the basic I/O operations still
+# work on this image.
+$QEMU_IO \
+    -c "reopen -o lazy-refcounts=on,pass-discard-request=on" \
+    -c "reopen -o lazy-refcounts=off,pass-discard-request=off" \
+    -c "reopen -o pass-discard-snapshot=on,pass-discard-other=on" \
+    -c "reopen -o pass-discard-snapshot=off,pass-discard-other=off" \
+    -c "reopen -o overlap-check=all" \
+    -c "reopen -o overlap-check=none" \
+    -c "reopen -o overlap-check=cached" \
+    -c "reopen -o overlap-check=constant" \
+    -c "reopen -o overlap-check.template=all" \
+    -c "reopen -o overlap-check.template=none" \
+    -c "reopen -o overlap-check.template=cached" \
+    -c "reopen -o overlap-check.template=constant" \
+    -c "reopen -o overlap-check.main-header=on" \
+    -c "reopen -o overlap-check.main-header=off" \
+    -c "reopen -o overlap-check.active-l1=on" \
+    -c "reopen -o overlap-check.active-l1=off" \
+    -c "reopen -o overlap-check.active-l2=on" \
+    -c "reopen -o overlap-check.active-l2=off" \
+    -c "reopen -o overlap-check.refcount-table=on" \
+    -c "reopen -o overlap-check.refcount-table=off" \
+    -c "reopen -o overlap-check.refcount-block=on" \
+    -c "reopen -o overlap-check.refcount-block=off" \
+    -c "reopen -o overlap-check.snapshot-table=on" \
+    -c "reopen -o overlap-check.snapshot-table=off" \
+    -c "reopen -o overlap-check.inactive-l1=on" \
+    -c "reopen -o overlap-check.inactive-l1=off" \
+    -c "reopen -o overlap-check.inactive-l2=on" \
+    -c "reopen -o overlap-check.inactive-l2=off" \
+    -c "reopen -o cache-size=1M" \
+    -c "reopen -o l2-cache-size=512k" \
+    -c "reopen -o refcount-cache-size=128k" \
+    -c "reopen -o cache-clean-interval=5" \
+    -c "reopen -o cache-clean-interval=0" \
+    -c "reopen -o cache-clean-interval=10" \
+    \
+    -c "write -P 55 0 32M" \
+    -c "read -P 55 0 32M" \
+    -c "discard 0 32M" \
+    -c "write -z 0 32M" \
+    -c "read -P 0 0 32M" \
+    \
+    "$TEST_IMG" | _filter_qemu_io
+
+
+echo
+echo === Try setting some invalid values ===
+echo
+
+$QEMU_IO \
+    -c "reopen -o lazy-refcounts=42" \
+    -c "reopen -o cache-size=1M,l2-cache-size=64k,refcount-cache-size=64k" \
+    -c "reopen -o cache-size=1M,l2-cache-size=2M" \
+    -c "reopen -o cache-size=1M,refcount-cache-size=2M" \
+    -c "reopen -o l2-cache-size=256T" \
+    -c "reopen -o refcount-cache-size=256T" \
+    -c "reopen -o overlap-check=constant,overlap-check.template=all" \
+    -c "reopen -o overlap-check=blubb" \
+    -c "reopen -o overlap-check.template=blubb" \
+    -c "reopen -o cache-clean-interval=-1" \
+    "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo === Test transaction semantics ===
+echo
+
+# Whether lazy-refcounts was actually enabled can easily be tested: Check if
+# the dirty bit is set after a crash
+$QEMU_IO \
+    -c "reopen -o lazy-refcounts=on,overlap-check=blubb" \
+    -c "write -P 0x5a 0 512" \
+    -c "sigraise $(kill -l KILL)" \
+    "$TEST_IMG" 2>&1 | _filter_qemu_io
+
+# The dirty bit must not be set
+$PYTHON qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
+
+# Similarly we can test whether corruption detection has been enabled:
+# Create L1/L2, overwrite first entry in refcount block, allocate something.
+# Disabling the checks should fail, so the corruption must be detected.
+_make_test_img 64M
+$QEMU_IO -c "write 0 64k" "$TEST_IMG" | _filter_qemu_io
+poke_file "$TEST_IMG" "$((0x20000))" "\x00\x00"
+$QEMU_IO \
+    -c "reopen -o overlap-check=none,lazy-refcounts=42" \
+    -c "write 64k 64k" \
+    "$TEST_IMG" 2>&1 | _filter_qemu_io
+
+# success, all done
+echo '*** done'
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/137.out b/tests/qemu-iotests/137.out
new file mode 100644
index 0000000000..cf55a41d8a
--- /dev/null
+++ b/tests/qemu-iotests/137.out
@@ -0,0 +1,42 @@
+QA output created by 137
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+=== Try setting valid values for all options ===
+
+wrote 33554432/33554432 bytes at offset 0
+32 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 33554432/33554432 bytes at offset 0
+32 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+discard 33554432/33554432 bytes at offset 0
+32 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 33554432/33554432 bytes at offset 0
+32 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 33554432/33554432 bytes at offset 0
+32 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+=== Try setting some invalid values ===
+
+Parameter 'lazy-refcounts' expects 'on' or 'off'
+cache-size, l2-cache-size and refcount-cache-size may not be set the same time
+l2-cache-size may not exceed cache-size
+refcount-cache-size may not exceed cache-size
+L2 cache size too big
+L2 cache size too big
+Conflicting values for qcow2 options 'overlap-check' ('constant') and 'overlap-check.template' ('all')
+Unsupported value 'blubb' for qcow2 option 'overlap-check'. Allowed are any of the following: none, constant, cached, all
+Unsupported value 'blubb' for qcow2 option 'overlap-check'. Allowed are any of the following: none, constant, cached, all
+Cache clean interval too big
+
+=== Test transaction semantics ===
+
+Unsupported value 'blubb' for qcow2 option 'overlap-check'. Allowed are any of the following: none, constant, cached, all
+wrote 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+./common.config: Killed                  ( exec "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@" )
+incompatible_features     0x0
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Parameter 'lazy-refcounts' expects 'on' or 'off'
+qcow2: Marking image as corrupt: Preventing invalid write on metadata (overlaps with qcow2_header); further corruption events will be suppressed
+write failed: Input/output error
+*** done
diff --git a/tests/qemu-iotests/138 b/tests/qemu-iotests/138
new file mode 100755
index 0000000000..a5c3464d58
--- /dev/null
+++ b/tests/qemu-iotests/138
@@ -0,0 +1,73 @@
+#!/bin/bash
+#
+# General test case for qcow2's image check
+#
+# Copyright (C) 2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+tmp=/tmp/$$
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+# This tests qocw2-specific low-level functionality
+_supported_fmt qcow2
+_supported_proto file
+_supported_os Linux
+
+echo
+echo '=== Check on an image with a multiple of 2^32 clusters ==='
+echo
+
+IMGOPTS=$(_optstr_add "$IMGOPTS" "cluster_size=512") \
+    _make_test_img 512
+
+# Allocate L2 table
+$QEMU_IO -c 'write 0 512' "$TEST_IMG" | _filter_qemu_io
+
+# Put the data cluster at a multiple of 2 TB, resulting in the image apparently
+# having a multiple of 2^32 clusters
+# (To be more specific: It is at 32 PB)
+poke_file "$TEST_IMG" 2048 "\x80\x80\x00\x00\x00\x00\x00\x00"
+
+# An offset of 32 PB results in qemu-img check having to allocate an in-memory
+# refcount table of 128 TB (16 bit refcounts, 512 byte clusters).
+# This should be generally too much for any system and thus fail.
+# What this test is checking is that the qcow2 driver actually tries to allocate
+# such a large amount of memory (and is consequently aborting) instead of having
+# truncated the cluster count somewhere (which would result in much less memory
+# being allocated and then a segfault occurring).
+_check_test_img
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/138.out b/tests/qemu-iotests/138.out
new file mode 100644
index 0000000000..3fe911f85a
--- /dev/null
+++ b/tests/qemu-iotests/138.out
@@ -0,0 +1,9 @@
+QA output created by 138
+
+=== Check on an image with a multiple of 2^32 clusters ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=512
+wrote 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-img: Check failed: Cannot allocate memory
+*** done
diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group
index c430b6c234..439b1d237d 100644
--- a/tests/qemu-iotests/group
+++ b/tests/qemu-iotests/group
@@ -134,3 +134,5 @@
 132 rw auto quick
 134 rw auto quick
 135 rw auto
+137 rw auto
+138 rw auto quick