Merge remote-tracking branch 'remotes/jnsnow/tags/bitmaps-pull-request' into staging

Pull request

# gpg: Signature made Tue 12 Mar 2019 20:23:08 GMT
# gpg:                using RSA key F9B7ABDBBCACDF95BE76CBD07DEF8106AAFC390E
# gpg: Good signature from "John Snow (John Huston) <jsnow@redhat.com>" [full]
# Primary key fingerprint: FAEB 9711 A12C F475 812F  18F2 88A9 064D 1835 61EB
#      Subkey fingerprint: F9B7 ABDB BCAC DF95 BE76  CBD0 7DEF 8106 AAFC 390E

* remotes/jnsnow/tags/bitmaps-pull-request: (22 commits)
  tests/qemu-iotests: add bitmap resize test 246
  block/qcow2-bitmap: Allow resizes with persistent bitmaps
  block/qcow2-bitmap: Don't check size for IN_USE bitmap
  docs/interop/qcow2: Improve bitmap flag in_use specification
  bitmaps: Fix typo in function name
  block/dirty-bitmaps: implement inconsistent bit
  block/dirty-bitmaps: disallow busy bitmaps as merge source
  block/dirty-bitmaps: prohibit removing readonly bitmaps
  block/dirty-bitmaps: prohibit readonly bitmaps for backups
  block/dirty-bitmaps: add block_dirty_bitmap_check function
  block/dirty-bitmap: add inconsistent status
  block/dirty-bitmaps: add inconsistent bit
  iotests: add busy/recording bit test to 124
  blockdev: remove unused paio parameter documentation
  block/dirty-bitmaps: move comment block
  block/dirty-bitmaps: unify qmp_locked and user_locked calls
  block/dirty-bitmap: explicitly lock bitmaps with successors
  nbd: change error checking order for bitmaps
  block/dirty-bitmap: change semantics of enabled predicate
  block/dirty-bitmap: remove set/reset assertions against enabled bit
  ...

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>

# Conflicts:
#	tests/qemu-iotests/group

6 files changed, 553 insertions, 2 deletions

diff --git a/tests/qemu-iotests/124 b/tests/qemu-iotests/124
index 5aa1bf1bd6..80b356f7bb 100755
--- a/tests/qemu-iotests/124
+++ b/tests/qemu-iotests/124
@@ -634,6 +634,119 @@ class TestIncrementalBackupBlkdebug(TestIncrementalBackupBase):
         self.vm.shutdown()
         self.check_backups()
 
+    def test_incremental_pause(self):
+        """
+        Test an incremental backup that errors into a pause and is resumed.
+        """
+
+        drive0 = self.drives[0]
+        # NB: The blkdebug script here looks for a "flush, read, read" pattern.
+        # The flush occurs in hmp_io_writes, the first read in device_add, and
+        # the last read during the block job.
+        result = self.vm.qmp('blockdev-add',
+                             node_name=drive0['id'],
+                             driver=drive0['fmt'],
+                             file={
+                                 'driver': 'blkdebug',
+                                 'image': {
+                                     'driver': 'file',
+                                     'filename': drive0['file']
+                                 },
+                                 'set-state': [{
+                                     'event': 'flush_to_disk',
+                                     'state': 1,
+                                     'new_state': 2
+                                 },{
+                                     'event': 'read_aio',
+                                     'state': 2,
+                                     'new_state': 3
+                                 }],
+                                 'inject-error': [{
+                                     'event': 'read_aio',
+                                     'errno': 5,
+                                     'state': 3,
+                                     'immediately': False,
+                                     'once': True
+                                 }],
+                             })
+        self.assert_qmp(result, 'return', {})
+        self.create_anchor_backup(drive0)
+        bitmap = self.add_bitmap('bitmap0', drive0)
+
+        # Emulate guest activity
+        self.hmp_io_writes(drive0['id'], (('0xab', 0, 512),
+                                          ('0xfe', '16M', '256k'),
+                                          ('0x64', '32736k', '64k')))
+
+        # For the purposes of query-block visibility of bitmaps, add a drive
+        # frontend after we've written data; otherwise we can't use hmp-io
+        result = self.vm.qmp("device_add",
+                             id="device0",
+                             drive=drive0['id'],
+                             driver="virtio-blk")
+        self.assert_qmp(result, 'return', {})
+
+        # Bitmap Status Check
+        query = self.vm.qmp('query-block')
+        ret = [bmap for bmap in query['return'][0]['dirty-bitmaps']
+               if bmap.get('name') == bitmap.name][0]
+        self.assert_qmp(ret, 'count', 458752)
+        self.assert_qmp(ret, 'granularity', 65536)
+        self.assert_qmp(ret, 'status', 'active')
+        self.assert_qmp(ret, 'busy', False)
+        self.assert_qmp(ret, 'recording', True)
+
+        # Start backup
+        parent, _ = bitmap.last_target()
+        target = self.prepare_backup(bitmap, parent)
+        res = self.vm.qmp('drive-backup',
+                          job_id=bitmap.drive['id'],
+                          device=bitmap.drive['id'],
+                          sync='incremental',
+                          bitmap=bitmap.name,
+                          format=bitmap.drive['fmt'],
+                          target=target,
+                          mode='existing',
+                          on_source_error='stop')
+        self.assert_qmp(res, 'return', {})
+
+        # Wait for the error
+        event = self.vm.event_wait(name="BLOCK_JOB_ERROR",
+                                   match={"data":{"device":bitmap.drive['id']}})
+        self.assert_qmp(event, 'data', {'device': bitmap.drive['id'],
+                                        'action': 'stop',
+                                        'operation': 'read'})
+
+        # Bitmap Status Check
+        query = self.vm.qmp('query-block')
+        ret = [bmap for bmap in query['return'][0]['dirty-bitmaps']
+               if bmap.get('name') == bitmap.name][0]
+        self.assert_qmp(ret, 'count', 458752)
+        self.assert_qmp(ret, 'granularity', 65536)
+        self.assert_qmp(ret, 'status', 'frozen')
+        self.assert_qmp(ret, 'busy', True)
+        self.assert_qmp(ret, 'recording', True)
+
+        # Resume and check incremental backup for consistency
+        res = self.vm.qmp('block-job-resume', device=bitmap.drive['id'])
+        self.assert_qmp(res, 'return', {})
+        self.wait_qmp_backup(bitmap.drive['id'])
+
+        # Bitmap Status Check
+        query = self.vm.qmp('query-block')
+        ret = [bmap for bmap in query['return'][0]['dirty-bitmaps']
+               if bmap.get('name') == bitmap.name][0]
+        self.assert_qmp(ret, 'count', 0)
+        self.assert_qmp(ret, 'granularity', 65536)
+        self.assert_qmp(ret, 'status', 'active')
+        self.assert_qmp(ret, 'busy', False)
+        self.assert_qmp(ret, 'recording', True)
+
+        # Finalize / Cleanup
+        self.make_reference_backup(bitmap)
+        self.vm.shutdown()
+        self.check_backups()
+
 
 if __name__ == '__main__':
     iotests.main(supported_fmts=['qcow2'])
diff --git a/tests/qemu-iotests/124.out b/tests/qemu-iotests/124.out
index e56cae021b..281b69efea 100644
--- a/tests/qemu-iotests/124.out
+++ b/tests/qemu-iotests/124.out
@@ -1,5 +1,5 @@
-...........
+............
 ----------------------------------------------------------------------
-Ran 11 tests
+Ran 12 tests
 
 OK
diff --git a/tests/qemu-iotests/236.out b/tests/qemu-iotests/236.out
index 5006f7bca1..815cd053f0 100644
--- a/tests/qemu-iotests/236.out
+++ b/tests/qemu-iotests/236.out
@@ -22,17 +22,21 @@ write -P0xcd 0x3ff0000 64k
   "bitmaps": {
     "drive0": [
       {
+        "busy": false,
         "count": 262144,
         "granularity": 65536,
         "name": "bitmapB",
         "persistent": false,
+        "recording": true,
         "status": "active"
       },
       {
+        "busy": false,
         "count": 262144,
         "granularity": 65536,
         "name": "bitmapA",
         "persistent": false,
+        "recording": true,
         "status": "active"
       }
     ]
@@ -84,17 +88,21 @@ write -P0xcd 0x3ff0000 64k
   "bitmaps": {
     "drive0": [
       {
+        "busy": false,
         "count": 262144,
         "granularity": 65536,
         "name": "bitmapB",
         "persistent": false,
+        "recording": true,
         "status": "active"
       },
       {
+        "busy": false,
         "count": 262144,
         "granularity": 65536,
         "name": "bitmapA",
         "persistent": false,
+        "recording": true,
         "status": "active"
       }
     ]
@@ -184,24 +192,30 @@ write -P0xea 0x3fe0000 64k
   "bitmaps": {
     "drive0": [
       {
+        "busy": false,
         "count": 393216,
         "granularity": 65536,
         "name": "bitmapC",
         "persistent": false,
+        "recording": false,
         "status": "disabled"
       },
       {
+        "busy": false,
         "count": 262144,
         "granularity": 65536,
         "name": "bitmapB",
         "persistent": false,
+        "recording": false,
         "status": "disabled"
       },
       {
+        "busy": false,
         "count": 458752,
         "granularity": 65536,
         "name": "bitmapA",
         "persistent": false,
+        "recording": false,
         "status": "disabled"
       }
     ]
@@ -251,24 +265,30 @@ write -P0xea 0x3fe0000 64k
   "bitmaps": {
     "drive0": [
       {
+        "busy": false,
         "count": 393216,
         "granularity": 65536,
         "name": "bitmapC",
         "persistent": false,
+        "recording": false,
         "status": "disabled"
       },
       {
+        "busy": false,
         "count": 262144,
         "granularity": 65536,
         "name": "bitmapB",
         "persistent": false,
+        "recording": false,
         "status": "disabled"
       },
       {
+        "busy": false,
         "count": 458752,
         "granularity": 65536,
         "name": "bitmapA",
         "persistent": false,
+        "recording": false,
         "status": "disabled"
       }
     ]
@@ -311,31 +331,39 @@ write -P0xea 0x3fe0000 64k
   "bitmaps": {
     "drive0": [
       {
+        "busy": false,
         "count": 458752,
         "granularity": 65536,
         "name": "bitmapD",
         "persistent": false,
+        "recording": false,
         "status": "disabled"
       },
       {
+        "busy": false,
         "count": 393216,
         "granularity": 65536,
         "name": "bitmapC",
         "persistent": false,
+        "recording": false,
         "status": "disabled"
       },
       {
+        "busy": false,
         "count": 262144,
         "granularity": 65536,
         "name": "bitmapB",
         "persistent": false,
+        "recording": false,
         "status": "disabled"
       },
       {
+        "busy": false,
         "count": 458752,
         "granularity": 65536,
         "name": "bitmapA",
         "persistent": false,
+        "recording": false,
         "status": "disabled"
       }
     ]
diff --git a/tests/qemu-iotests/246 b/tests/qemu-iotests/246
new file mode 100755
index 0000000000..b0997a392f
--- /dev/null
+++ b/tests/qemu-iotests/246
@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+#
+# Test persistent bitmap resizing.
+#
+# Copyright (c) 2019 John Snow for Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# owner=jsnow@redhat.com
+
+import iotests
+from iotests import log
+
+iotests.verify_image_format(supported_fmts=['qcow2'])
+size = 64 * 1024 * 1024 * 1024
+gran_small = 32 * 1024
+gran_large = 128 * 1024
+
+def query_bitmaps(vm):
+    res = vm.qmp("query-block")
+    return { "bitmaps": { device['device']: device.get('dirty-bitmaps', []) for
+                          device in res['return'] } }
+
+with iotests.FilePath('img') as img_path, \
+     iotests.VM() as vm:
+
+    log('--- Preparing image & VM ---\n')
+    iotests.qemu_img_create('-f', iotests.imgfmt, img_path, str(size))
+    vm.add_drive(img_path)
+
+
+    log('--- 1st Boot (Establish Baseline Image) ---\n')
+    vm.launch()
+
+    log('\n--- Adding bitmaps Small, Medium, Large, and Transient ---\n')
+    vm.qmp_log("block-dirty-bitmap-add", node="drive0",
+               name="Small", granularity=gran_small, persistent=True)
+    vm.qmp_log("block-dirty-bitmap-add", node="drive0",
+               name="Medium", persistent=True)
+    vm.qmp_log("block-dirty-bitmap-add", node="drive0",
+               name="Large", granularity=gran_large, persistent=True)
+    vm.qmp_log("block-dirty-bitmap-add", node="drive0",
+               name="Transient", persistent=False)
+
+    log('--- Forcing flush of bitmaps to disk ---\n')
+    log(query_bitmaps(vm), indent=2)
+    vm.shutdown()
+
+
+    log('--- 2nd Boot (Grow Image) ---\n')
+    vm.launch()
+    log(query_bitmaps(vm), indent=2)
+
+    log('--- Adding new bitmap, growing image, and adding 2nd new bitmap ---')
+    vm.qmp_log("block-dirty-bitmap-add", node="drive0",
+               name="New", persistent=True)
+    vm.qmp_log("human-monitor-command",
+               command_line="block_resize drive0 70G")
+    vm.qmp_log("block-dirty-bitmap-add", node="drive0",
+               name="Newtwo", persistent=True)
+    log(query_bitmaps(vm), indent=2)
+
+    log('--- Forcing flush of bitmaps to disk ---\n')
+    vm.shutdown()
+
+
+    log('--- 3rd Boot (Shrink Image) ---\n')
+    vm.launch()
+    log(query_bitmaps(vm), indent=2)
+
+    log('--- Adding "NewB" bitmap, removing "New" bitmap ---')
+    vm.qmp_log("block-dirty-bitmap-add", node="drive0",
+               name="NewB", persistent=True)
+    vm.qmp_log("block-dirty-bitmap-remove", node="drive0",
+               name="New")
+
+    log('--- Truncating image ---\n')
+    vm.qmp_log("human-monitor-command",
+               command_line="block_resize drive0 50G")
+
+    log('--- Adding "NewC" bitmap, removing "NewTwo" bitmap ---')
+    vm.qmp_log("block-dirty-bitmap-add", node="drive0",
+               name="NewC", persistent=True)
+    vm.qmp_log("block-dirty-bitmap-remove", node="drive0", name="Newtwo")
+
+    log('--- Forcing flush of bitmaps to disk ---\n')
+    vm.shutdown()
+
+
+    log('--- 4th Boot (Verification and Cleanup) ---\n')
+    vm.launch()
+    log(query_bitmaps(vm), indent=2)
+
+    log('--- Removing all Bitmaps ---\n')
+    vm.qmp_log("block-dirty-bitmap-remove", node="drive0", name="Small")
+    vm.qmp_log("block-dirty-bitmap-remove", node="drive0", name="Medium")
+    vm.qmp_log("block-dirty-bitmap-remove", node="drive0", name="Large")
+    vm.qmp_log("block-dirty-bitmap-remove", node="drive0", name="NewB")
+    vm.qmp_log("block-dirty-bitmap-remove", node="drive0", name="NewC")
+    log(query_bitmaps(vm), indent=2)
+
+    log('\n--- Done ---')
+    vm.shutdown()
diff --git a/tests/qemu-iotests/246.out b/tests/qemu-iotests/246.out
new file mode 100644
index 0000000000..6671a11fdd
--- /dev/null
+++ b/tests/qemu-iotests/246.out
@@ -0,0 +1,295 @@
+--- Preparing image & VM ---
+
+--- 1st Boot (Establish Baseline Image) ---
+
+
+--- Adding bitmaps Small, Medium, Large, and Transient ---
+
+{"execute": "block-dirty-bitmap-add", "arguments": {"granularity": 32768, "name": "Small", "node": "drive0", "persistent": true}}
+{"return": {}}
+{"execute": "block-dirty-bitmap-add", "arguments": {"name": "Medium", "node": "drive0", "persistent": true}}
+{"return": {}}
+{"execute": "block-dirty-bitmap-add", "arguments": {"granularity": 131072, "name": "Large", "node": "drive0", "persistent": true}}
+{"return": {}}
+{"execute": "block-dirty-bitmap-add", "arguments": {"name": "Transient", "node": "drive0", "persistent": false}}
+{"return": {}}
+--- Forcing flush of bitmaps to disk ---
+
+{
+  "bitmaps": {
+    "drive0": [
+      {
+        "busy": false,
+        "count": 0,
+        "granularity": 65536,
+        "name": "Transient",
+        "persistent": false,
+        "recording": true,
+        "status": "active"
+      },
+      {
+        "busy": false,
+        "count": 0,
+        "granularity": 131072,
+        "name": "Large",
+        "persistent": true,
+        "recording": true,
+        "status": "active"
+      },
+      {
+        "busy": false,
+        "count": 0,
+        "granularity": 65536,
+        "name": "Medium",
+        "persistent": true,
+        "recording": true,
+        "status": "active"
+      },
+      {
+        "busy": false,
+        "count": 0,
+        "granularity": 32768,
+        "name": "Small",
+        "persistent": true,
+        "recording": true,
+        "status": "active"
+      }
+    ]
+  }
+}
+--- 2nd Boot (Grow Image) ---
+
+{
+  "bitmaps": {
+    "drive0": [
+      {
+        "busy": false,
+        "count": 0,
+        "granularity": 32768,
+        "name": "Small",
+        "persistent": true,
+        "recording": true,
+        "status": "active"
+      },
+      {
+        "busy": false,
+        "count": 0,
+        "granularity": 65536,
+        "name": "Medium",
+        "persistent": true,
+        "recording": true,
+        "status": "active"
+      },
+      {
+        "busy": false,
+        "count": 0,
+        "granularity": 131072,
+        "name": "Large",
+        "persistent": true,
+        "recording": true,
+        "status": "active"
+      }
+    ]
+  }
+}
+--- Adding new bitmap, growing image, and adding 2nd new bitmap ---
+{"execute": "block-dirty-bitmap-add", "arguments": {"name": "New", "node": "drive0", "persistent": true}}
+{"return": {}}
+{"execute": "human-monitor-command", "arguments": {"command-line": "block_resize drive0 70G"}}
+{"return": ""}
+{"execute": "block-dirty-bitmap-add", "arguments": {"name": "Newtwo", "node": "drive0", "persistent": true}}
+{"return": {}}
+{
+  "bitmaps": {
+    "drive0": [
+      {
+        "busy": false,
+        "count": 0,
+        "granularity": 65536,
+        "name": "Newtwo",
+        "persistent": true,
+        "recording": true,
+        "status": "active"
+      },
+      {
+        "busy": false,
+        "count": 0,
+        "granularity": 65536,
+        "name": "New",
+        "persistent": true,
+        "recording": true,
+        "status": "active"
+      },
+      {
+        "busy": false,
+        "count": 0,
+        "granularity": 32768,
+        "name": "Small",
+        "persistent": true,
+        "recording": true,
+        "status": "active"
+      },
+      {
+        "busy": false,
+        "count": 0,
+        "granularity": 65536,
+        "name": "Medium",
+        "persistent": true,
+        "recording": true,
+        "status": "active"
+      },
+      {
+        "busy": false,
+        "count": 0,
+        "granularity": 131072,
+        "name": "Large",
+        "persistent": true,
+        "recording": true,
+        "status": "active"
+      }
+    ]
+  }
+}
+--- Forcing flush of bitmaps to disk ---
+
+--- 3rd Boot (Shrink Image) ---
+
+{
+  "bitmaps": {
+    "drive0": [
+      {
+        "busy": false,
+        "count": 0,
+        "granularity": 65536,
+        "name": "New",
+        "persistent": true,
+        "recording": true,
+        "status": "active"
+      },
+      {
+        "busy": false,
+        "count": 0,
+        "granularity": 65536,
+        "name": "Newtwo",
+        "persistent": true,
+        "recording": true,
+        "status": "active"
+      },
+      {
+        "busy": false,
+        "count": 0,
+        "granularity": 32768,
+        "name": "Small",
+        "persistent": true,
+        "recording": true,
+        "status": "active"
+      },
+      {
+        "busy": false,
+        "count": 0,
+        "granularity": 65536,
+        "name": "Medium",
+        "persistent": true,
+        "recording": true,
+        "status": "active"
+      },
+      {
+        "busy": false,
+        "count": 0,
+        "granularity": 131072,
+        "name": "Large",
+        "persistent": true,
+        "recording": true,
+        "status": "active"
+      }
+    ]
+  }
+}
+--- Adding "NewB" bitmap, removing "New" bitmap ---
+{"execute": "block-dirty-bitmap-add", "arguments": {"name": "NewB", "node": "drive0", "persistent": true}}
+{"return": {}}
+{"execute": "block-dirty-bitmap-remove", "arguments": {"name": "New", "node": "drive0"}}
+{"return": {}}
+--- Truncating image ---
+
+{"execute": "human-monitor-command", "arguments": {"command-line": "block_resize drive0 50G"}}
+{"return": ""}
+--- Adding "NewC" bitmap, removing "NewTwo" bitmap ---
+{"execute": "block-dirty-bitmap-add", "arguments": {"name": "NewC", "node": "drive0", "persistent": true}}
+{"return": {}}
+{"execute": "block-dirty-bitmap-remove", "arguments": {"name": "Newtwo", "node": "drive0"}}
+{"return": {}}
+--- Forcing flush of bitmaps to disk ---
+
+--- 4th Boot (Verification and Cleanup) ---
+
+{
+  "bitmaps": {
+    "drive0": [
+      {
+        "busy": false,
+        "count": 0,
+        "granularity": 65536,
+        "name": "NewB",
+        "persistent": true,
+        "recording": true,
+        "status": "active"
+      },
+      {
+        "busy": false,
+        "count": 0,
+        "granularity": 65536,
+        "name": "NewC",
+        "persistent": true,
+        "recording": true,
+        "status": "active"
+      },
+      {
+        "busy": false,
+        "count": 0,
+        "granularity": 32768,
+        "name": "Small",
+        "persistent": true,
+        "recording": true,
+        "status": "active"
+      },
+      {
+        "busy": false,
+        "count": 0,
+        "granularity": 65536,
+        "name": "Medium",
+        "persistent": true,
+        "recording": true,
+        "status": "active"
+      },
+      {
+        "busy": false,
+        "count": 0,
+        "granularity": 131072,
+        "name": "Large",
+        "persistent": true,
+        "recording": true,
+        "status": "active"
+      }
+    ]
+  }
+}
+--- Removing all Bitmaps ---
+
+{"execute": "block-dirty-bitmap-remove", "arguments": {"name": "Small", "node": "drive0"}}
+{"return": {}}
+{"execute": "block-dirty-bitmap-remove", "arguments": {"name": "Medium", "node": "drive0"}}
+{"return": {}}
+{"execute": "block-dirty-bitmap-remove", "arguments": {"name": "Large", "node": "drive0"}}
+{"return": {}}
+{"execute": "block-dirty-bitmap-remove", "arguments": {"name": "NewB", "node": "drive0"}}
+{"return": {}}
+{"execute": "block-dirty-bitmap-remove", "arguments": {"name": "NewC", "node": "drive0"}}
+{"return": {}}
+{
+  "bitmaps": {
+    "drive0": []
+  }
+}
+
+--- Done ---
diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group
index 66ab708d4a..7289309604 100644
--- a/tests/qemu-iotests/group
+++ b/tests/qemu-iotests/group
@@ -244,3 +244,4 @@
 243 rw auto quick
 244 rw auto quick
 245 rw auto
+246 rw auto quick