Merge remote-tracking branch 'remotes/stefanha/tags/block-pull-request' into staging

Pull request

Let's get the image fuzzer Python 3 changes merged in QEMU 4.2.

# gpg: Signature made Tue 05 Nov 2019 15:43:16 GMT
# gpg:                using RSA key 8695A8BFD3F97CDAAC35775A9CA4ABB381AB73C8
# gpg: Good signature from "Stefan Hajnoczi <stefanha@redhat.com>" [full]
# gpg:                 aka "Stefan Hajnoczi <stefanha@gmail.com>" [full]
# Primary key fingerprint: 8695 A8BF D3F9 7CDA AC35  775A 9CA4 ABB3 81AB 73C8

* remotes/stefanha/tags/block-pull-request:
  image-fuzzer: Use OSerror.strerror instead of tuple subscript
  image-fuzzer: Use errors parameter of subprocess.Popen()
  image-fuzzer: Run using python3
  image-fuzzer: Encode file name and file format to bytes
  image-fuzzer: Use bytes constant for field values
  image-fuzzer: Return bytes objects on string fuzzing functions
  image-fuzzer: Use %r for all fiels at Field.__repr__()
  image-fuzzer: Use io.StringIO
  image-fuzzer: Explicitly use integer division operator
  image-fuzzer: Write bytes instead of string to image file
  image-fuzzer: Open image files in binary mode

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>

4 files changed, 63 insertions, 65 deletions

diff --git a/tests/image-fuzzer/qcow2/__init__.py b/tests/image-fuzzer/qcow2/__init__.py
index 09ef59821b..ed3af5da86 100644
--- a/tests/image-fuzzer/qcow2/__init__.py
+++ b/tests/image-fuzzer/qcow2/__init__.py
@@ -1,2 +1 @@
-from __future__ import absolute_import
 from .layout import create_image
diff --git a/tests/image-fuzzer/qcow2/fuzz.py b/tests/image-fuzzer/qcow2/fuzz.py
index abc4f0635d..c58bf11005 100644
--- a/tests/image-fuzzer/qcow2/fuzz.py
+++ b/tests/image-fuzzer/qcow2/fuzz.py
@@ -27,20 +27,20 @@ UINT64 = 0xffffffffffffffff
 UINT32_M = 31
 UINT64_M = 63
 # Fuzz vectors
-UINT8_V = [0, 0x10, UINT8/4, UINT8/2 - 1, UINT8/2, UINT8/2 + 1, UINT8 - 1,
+UINT8_V = [0, 0x10, UINT8//4, UINT8//2 - 1, UINT8//2, UINT8//2 + 1, UINT8 - 1,
            UINT8]
-UINT16_V = [0, 0x100, 0x1000, UINT16/4, UINT16/2 - 1, UINT16/2, UINT16/2 + 1,
+UINT16_V = [0, 0x100, 0x1000, UINT16//4, UINT16//2 - 1, UINT16//2, UINT16//2 + 1,
             UINT16 - 1, UINT16]
-UINT32_V = [0, 0x100, 0x1000, 0x10000, 0x100000, UINT32/4, UINT32/2 - 1,
-            UINT32/2, UINT32/2 + 1, UINT32 - 1, UINT32]
-UINT64_V = UINT32_V + [0x1000000, 0x10000000, 0x100000000, UINT64/4,
-                       UINT64/2 - 1, UINT64/2, UINT64/2 + 1, UINT64 - 1,
+UINT32_V = [0, 0x100, 0x1000, 0x10000, 0x100000, UINT32//4, UINT32//2 - 1,
+            UINT32//2, UINT32//2 + 1, UINT32 - 1, UINT32]
+UINT64_V = UINT32_V + [0x1000000, 0x10000000, 0x100000000, UINT64//4,
+                       UINT64//2 - 1, UINT64//2, UINT64//2 + 1, UINT64 - 1,
                        UINT64]
-STRING_V = ['%s%p%x%d', '.1024d', '%.2049d', '%p%p%p%p', '%x%x%x%x',
-            '%d%d%d%d', '%s%s%s%s', '%99999999999s', '%08x', '%%20d', '%%20n',
-            '%%20x', '%%20s', '%s%s%s%s%s%s%s%s%s%s', '%p%p%p%p%p%p%p%p%p%p',
-            '%#0123456x%08x%x%s%p%d%n%o%u%c%h%l%q%j%z%Z%t%i%e%g%f%a%C%S%08x%%',
-            '%s x 129', '%x x 257']
+BYTES_V = [b'%s%p%x%d', b'.1024d', b'%.2049d', b'%p%p%p%p', b'%x%x%x%x',
+           b'%d%d%d%d', b'%s%s%s%s', b'%99999999999s', b'%08x', b'%%20d', b'%%20n',
+           b'%%20x', b'%%20s', b'%s%s%s%s%s%s%s%s%s%s', b'%p%p%p%p%p%p%p%p%p%p',
+           b'%#0123456x%08x%x%s%p%d%n%o%u%c%h%l%q%j%z%Z%t%i%e%g%f%a%C%S%08x%%',
+           b'%s x 129', b'%x x 257']
 
 
 def random_from_intervals(intervals):
@@ -76,12 +76,12 @@ def random_bits(bit_ranges):
     return val
 
 
-def truncate_string(strings, length):
-    """Return strings truncated to specified length."""
-    if type(strings) == list:
-        return [s[:length] for s in strings]
+def truncate_bytes(sequences, length):
+    """Return sequences truncated to specified length."""
+    if type(sequences) == list:
+        return [s[:length] for s in sequences]
     else:
-        return strings[:length]
+        return sequences[:length]
 
 
 def validator(current, pick, choices):
@@ -110,12 +110,12 @@ def bit_validator(current, bit_ranges):
     return validator(current, random_bits, bit_ranges)
 
 
-def string_validator(current, strings):
-    """Return a random string value from the list not equal to the current.
+def bytes_validator(current, sequences):
+    """Return a random bytes value from the list not equal to the current.
 
     This function is useful for selection from valid values except current one.
     """
-    return validator(current, random.choice, strings)
+    return validator(current, random.choice, sequences)
 
 
 def selector(current, constraints, validate=int_validator):
@@ -283,9 +283,9 @@ def header_length(current):
 def bf_name(current):
     """Fuzz the backing file name."""
     constraints = [
-        truncate_string(STRING_V, len(current))
+        truncate_bytes(BYTES_V, len(current))
     ]
-    return selector(current, constraints, string_validator)
+    return selector(current, constraints, bytes_validator)
 
 
 def ext_magic(current):
@@ -303,10 +303,10 @@ def ext_length(current):
 def bf_format(current):
     """Fuzz backing file format in the corresponding header extension."""
     constraints = [
-        truncate_string(STRING_V, len(current)),
-        truncate_string(STRING_V, (len(current) + 7) & ~7)  # Fuzz padding
+        truncate_bytes(BYTES_V, len(current)),
+        truncate_bytes(BYTES_V, (len(current) + 7) & ~7)  # Fuzz padding
     ]
-    return selector(current, constraints, string_validator)
+    return selector(current, constraints, bytes_validator)
 
 
 def feature_type(current):
@@ -324,10 +324,10 @@ def feature_bit_number(current):
 def feature_name(current):
     """Fuzz feature name field of a feature name table header extension."""
     constraints = [
-        truncate_string(STRING_V, len(current)),
-        truncate_string(STRING_V, 46)  # Fuzz padding (field length = 46)
+        truncate_bytes(BYTES_V, len(current)),
+        truncate_bytes(BYTES_V, 46)  # Fuzz padding (field length = 46)
     ]
-    return selector(current, constraints, string_validator)
+    return selector(current, constraints, bytes_validator)
 
 
 def l1_entry(current):
diff --git a/tests/image-fuzzer/qcow2/layout.py b/tests/image-fuzzer/qcow2/layout.py
index 675877da96..57ebe86e9a 100644
--- a/tests/image-fuzzer/qcow2/layout.py
+++ b/tests/image-fuzzer/qcow2/layout.py
@@ -16,7 +16,6 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-from __future__ import absolute_import
 import random
 import struct
 from . import fuzz
@@ -53,8 +52,8 @@ class Field(object):
         return iter([self.fmt, self.offset, self.value, self.name])
 
     def __repr__(self):
-        return "Field(fmt='%s', offset=%d, value=%s, name=%s)" % \
-            (self.fmt, self.offset, str(self.value), self.name)
+        return "Field(fmt=%r, offset=%r, value=%r, name=%r)" % \
+            (self.fmt, self.offset, self.value, self.name)
 
 
 class FieldsList(object):
@@ -122,7 +121,7 @@ class Image(object):
     def create_header(self, cluster_bits, backing_file_name=None):
         """Generate a random valid header."""
         meta_header = [
-            ['>4s', 0, "QFI\xfb", 'magic'],
+            ['>4s', 0, b"QFI\xfb", 'magic'],
             ['>I', 4, random.randint(2, 3), 'version'],
             ['>Q', 8, 0, 'backing_file_offset'],
             ['>I', 16, 0, 'backing_file_size'],
@@ -231,7 +230,7 @@ class Image(object):
             feature_tables = []
             feature_ids = []
             inner_offset = self.ext_offset + ext_header_len
-            feat_name = 'some cool feature'
+            feat_name = b'some cool feature'
             while len(feature_tables) < num_fnt_entries * 3:
                 feat_type, feat_bit = gen_feat_ids()
                 # Remove duplicates
@@ -253,7 +252,7 @@ class Image(object):
                 ['>I', self.ext_offset, 0x6803f857, 'ext_magic'],
                 # One feature table contains 3 fields and takes 48 bytes
                 ['>I', self.ext_offset + UINT32_S,
-                 len(feature_tables) / 3 * 48, 'ext_length']
+                 len(feature_tables) // 3 * 48, 'ext_length']
             ] + feature_tables)
             self.ext_offset = inner_offset
 
@@ -271,7 +270,7 @@ class Image(object):
         def create_l2_entry(host, guest, l2_cluster):
             """Generate one L2 entry."""
             offset = l2_cluster * self.cluster_size
-            l2_size = self.cluster_size / UINT64_S
+            l2_size = self.cluster_size // UINT64_S
             entry_offset = offset + UINT64_S * (guest % l2_size)
             cluster_descriptor = host * self.cluster_size
             if not self.header['version'][0].value == 2:
@@ -283,8 +282,8 @@ class Image(object):
 
         def create_l1_entry(l2_cluster, l1_offset, guest):
             """Generate one L1 entry."""
-            l2_size = self.cluster_size / UINT64_S
-            entry_offset = l1_offset + UINT64_S * (guest / l2_size)
+            l2_size = self.cluster_size // UINT64_S
+            entry_offset = l1_offset + UINT64_S * (guest // l2_size)
             # While snapshots are not supported bit #63 = 1
             entry_val = (1 << 63) + l2_cluster * self.cluster_size
             return ['>Q', entry_offset, entry_val, 'l1_entry']
@@ -298,11 +297,11 @@ class Image(object):
             l2 = []
         else:
             meta_data = self._get_metadata()
-            guest_clusters = random.sample(range(self.image_size /
+            guest_clusters = random.sample(range(self.image_size //
                                                  self.cluster_size),
                                            len(self.data_clusters))
             # Number of entries in a L1/L2 table
-            l_size = self.cluster_size / UINT64_S
+            l_size = self.cluster_size // UINT64_S
             # Number of clusters necessary for L1 table
             l1_size = int(ceil((max(guest_clusters) + 1) / float(l_size**2)))
             l1_start = self._get_adjacent_clusters(self.data_clusters |
@@ -318,7 +317,7 @@ class Image(object):
             # L2 entries
             l2 = []
             for host, guest in zip(self.data_clusters, guest_clusters):
-                l2_id = guest / l_size
+                l2_id = guest // l_size
                 if l2_id not in l2_ids:
                     l2_ids.append(l2_id)
                     l2_clusters.append(self._get_adjacent_clusters(
@@ -339,14 +338,14 @@ class Image(object):
         def allocate_rfc_blocks(data, size):
             """Return indices of clusters allocated for refcount blocks."""
             cluster_ids = set()
-            diff = block_ids = set([x / size for x in data])
+            diff = block_ids = set([x // size for x in data])
             while len(diff) != 0:
                 # Allocate all yet not allocated clusters
                 new = self._get_available_clusters(data | cluster_ids,
                                                    len(diff))
                 # Indices of new refcount blocks necessary to cover clusters
                 # in 'new'
-                diff = set([x / size for x in new]) - block_ids
+                diff = set([x // size for x in new]) - block_ids
                 cluster_ids |= new
                 block_ids |= diff
             return cluster_ids, block_ids
@@ -359,7 +358,7 @@ class Image(object):
             blocks = set(init_blocks)
             clusters = set()
             # Number of entries in one cluster of the refcount table
-            size = self.cluster_size / UINT64_S
+            size = self.cluster_size // UINT64_S
             # Number of clusters necessary for the refcount table based on
             # the current number of refcount blocks
             table_size = int(ceil((max(blocks) + 1) / float(size)))
@@ -373,7 +372,7 @@ class Image(object):
                                                  table_size + 1))
             # New refcount blocks necessary for clusters occupied by the
             # refcount table
-            diff = set([c / block_size for c in table_clusters]) - blocks
+            diff = set([c // block_size for c in table_clusters]) - blocks
             blocks |= diff
             while len(diff) != 0:
                 # Allocate clusters for new refcount blocks
@@ -382,12 +381,12 @@ class Image(object):
                                                    len(diff))
                 # Indices of new refcount blocks necessary to cover
                 # clusters in 'new'
-                diff = set([x / block_size for x in new]) - blocks
+                diff = set([x // block_size for x in new]) - blocks
                 clusters |= new
                 blocks |= diff
                 # Check if the refcount table needs one more cluster
                 if int(ceil((max(blocks) + 1) / float(size))) > table_size:
-                    new_block_id = (table_start + table_size) / block_size
+                    new_block_id = (table_start + table_size) // block_size
                     # Check if the additional table cluster needs
                     # one more refcount block
                     if new_block_id not in blocks:
@@ -399,13 +398,13 @@ class Image(object):
         def create_table_entry(table_offset, block_cluster, block_size,
                                cluster):
             """Generate a refcount table entry."""
-            offset = table_offset + UINT64_S * (cluster / block_size)
+            offset = table_offset + UINT64_S * (cluster // block_size)
             return ['>Q', offset, block_cluster * self.cluster_size,
                     'refcount_table_entry']
 
         def create_block_entry(block_cluster, block_size, cluster):
             """Generate a list of entries for the current block."""
-            entry_size = self.cluster_size / block_size
+            entry_size = self.cluster_size // block_size
             offset = block_cluster * self.cluster_size
             entry_offset = offset + entry_size * (cluster % block_size)
             # While snapshots are not supported all refcounts are set to 1
@@ -415,7 +414,7 @@ class Image(object):
         # Number of refcount entries per refcount block
         # Convert self.cluster_size from bytes to bits to have the same
         # base for the numerator and denominator
-        block_size = self.cluster_size * 8 / refcount_bits
+        block_size = self.cluster_size * 8 // refcount_bits
         meta_data = self._get_metadata()
         if len(self.data_clusters) == 0:
             # All metadata for an empty guest image needs 4 clusters:
@@ -452,8 +451,8 @@ class Image(object):
         rfc_blocks = []
 
         for cluster in sorted(self.data_clusters | meta_data):
-            if cluster / block_size != block_id:
-                block_id = cluster / block_size
+            if cluster // block_size != block_id:
+                block_id = cluster // block_size
                 block_cluster = block_clusters[block_ids.index(block_id)]
                 rfc_table.append(create_table_entry(table_offset,
                                                     block_cluster,
@@ -503,7 +502,7 @@ class Image(object):
 
     def write(self, filename):
         """Write an entire image to the file."""
-        image_file = open(filename, 'w')
+        image_file = open(filename, 'wb')
         for field in self:
             image_file.seek(field.offset)
             image_file.write(struct.pack(field.fmt, field.value))
@@ -518,7 +517,7 @@ class Image(object):
         rounded = (size + self.cluster_size - 1) & ~(self.cluster_size - 1)
         if rounded > size:
             image_file.seek(rounded - 1)
-            image_file.write("\0")
+            image_file.write(b'\x00')
         image_file.close()
 
     @staticmethod
@@ -587,7 +586,7 @@ class Image(object):
     def _alloc_data(img_size, cluster_size):
         """Return a set of random indices of clusters allocated for guest data.
         """
-        num_of_cls = img_size/cluster_size
+        num_of_cls = img_size // cluster_size
         return set(random.sample(range(1, num_of_cls + 1),
                                  random.randint(0, num_of_cls)))
 
@@ -595,15 +594,15 @@ class Image(object):
         """Return indices of clusters allocated for image metadata."""
         ids = set()
         for x in self:
-            ids.add(x.offset/self.cluster_size)
+            ids.add(x.offset // self.cluster_size)
         return ids
 
 
 def create_image(test_img_path, backing_file_name=None, backing_file_fmt=None,
                  fields_to_fuzz=None):
     """Create a fuzzed image and write it to the specified file."""
-    image = Image(backing_file_name)
-    image.set_backing_file_format(backing_file_fmt)
+    image = Image(backing_file_name.encode())
+    image.set_backing_file_format(backing_file_fmt.encode())
     image.create_feature_name_table()
     image.set_end_of_extension_area()
     image.create_l_structures()
diff --git a/tests/image-fuzzer/runner.py b/tests/image-fuzzer/runner.py
index 95d84f38f3..2fc010fd9d 100755
--- a/tests/image-fuzzer/runner.py
+++ b/tests/image-fuzzer/runner.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Tool for running fuzz tests
 #
@@ -18,7 +18,6 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-from __future__ import print_function
 import sys
 import os
 import signal
@@ -28,7 +27,7 @@ import shutil
 from itertools import count
 import time
 import getopt
-import StringIO
+import io
 import resource
 
 try:
@@ -80,7 +79,8 @@ def run_app(fd, q_args):
     devnull = open('/dev/null', 'r+')
     process = subprocess.Popen(q_args, stdin=devnull,
                                stdout=subprocess.PIPE,
-                               stderr=subprocess.PIPE)
+                               stderr=subprocess.PIPE,
+                               errors='replace')
     try:
         out, err = process.communicate()
         signal.alarm(0)
@@ -159,7 +159,7 @@ class TestEnv(object):
             os.makedirs(self.current_dir)
         except OSError as e:
             print("Error: The working directory '%s' cannot be used. Reason: %s"\
-                % (self.work_dir, e[1]), file=sys.stderr)
+                % (self.work_dir, e.strerror), file=sys.stderr)
             raise TestException
         self.log = open(os.path.join(self.current_dir, "test.log"), "w")
         self.parent_log = open(run_log, "a")
@@ -183,7 +183,7 @@ class TestEnv(object):
                                            MAX_BACKING_FILE_SIZE) * (1 << 20)
         cmd = self.qemu_img + ['create', '-f', backing_file_fmt,
                                backing_file_name, str(backing_file_size)]
-        temp_log = StringIO.StringIO()
+        temp_log = io.StringIO()
         retcode = run_app(temp_log, cmd)
         if retcode == 0:
             temp_log.close()
@@ -240,13 +240,13 @@ class TestEnv(object):
                            "Backing file: %s\n" \
                            % (self.seed, " ".join(current_cmd),
                               self.current_dir, backing_file_name)
-            temp_log = StringIO.StringIO()
+            temp_log = io.StringIO()
             try:
                 retcode = run_app(temp_log, current_cmd)
             except OSError as e:
                 multilog("%sError: Start of '%s' failed. Reason: %s\n\n"
                          % (test_summary, os.path.basename(current_cmd[0]),
-                            e[1]),
+                            e.strerror),
                          sys.stderr, self.log, self.parent_log)
                 raise TestException