Merge remote-tracking branch 'remotes/cleber/tags/python-next-pull-request' into staging

Python queue, 2019-02-22

Python:
* introduce "python" directory with module namespace
* log QEMU launch command line on qemu.QEMUMachine

Acceptance Tests:
* initrd 4GiB+ test
* migration test
* multi vm support in test class
* bump Avocado version and drop ":avocado: enable"

# gpg: Signature made Fri 22 Feb 2019 19:37:07 GMT
# gpg:                using RSA key 657E8D33A5F209F3
# gpg: Good signature from "Cleber Rosa <crosa@redhat.com>" [marginal]
# gpg: WARNING: This key is not certified with sufficiently trusted signatures!
# gpg:          It is not certain that the signature belongs to the owner.
# Primary key fingerprint: 7ABB 96EB 8B46 B94D 5E0F  E9BB 657E 8D33 A5F2 09F3

* remotes/cleber/tags/python-next-pull-request:
  Acceptance tests: expect boot to extract 2GiB+ initrd with linux-v4.16
  Acceptance tests: use linux-3.6 and set vm memory to 4GiB
  tests.acceptance: adds simple migration test
  tests.acceptance: adds multi vm capability for acceptance tests
  scripts/qemu.py: log QEMU launch command line
  Introduce a Python module structure
  Acceptance tests: drop usage of ":avocado: enable"

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>

13 files changed, 131 insertions, 29 deletions

diff --git a/tests/acceptance/avocado_qemu/__init__.py b/tests/acceptance/avocado_qemu/__init__.py
index 1e54fd5932..a66ec72daa 100644
--- a/tests/acceptance/avocado_qemu/__init__.py
+++ b/tests/acceptance/avocado_qemu/__init__.py
@@ -10,12 +10,12 @@
 
 import os
 import sys
+import uuid
 
 import avocado
 
-SRC_ROOT_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
-SRC_ROOT_DIR = os.path.abspath(os.path.dirname(SRC_ROOT_DIR))
-sys.path.append(os.path.join(SRC_ROOT_DIR, 'scripts'))
+SRC_ROOT_DIR = os.path.join(os.path.dirname(__file__), '..', '..', '..')
+sys.path.append(os.path.join(SRC_ROOT_DIR, 'python'))
 
 from qemu import QEMUMachine
 
@@ -42,13 +42,29 @@ def pick_default_qemu_bin():
 
 class Test(avocado.Test):
     def setUp(self):
-        self.vm = None
+        self._vms = {}
         self.qemu_bin = self.params.get('qemu_bin',
                                         default=pick_default_qemu_bin())
         if self.qemu_bin is None:
             self.cancel("No QEMU binary defined or found in the source tree")
-        self.vm = QEMUMachine(self.qemu_bin)
+
+    def _new_vm(self, *args):
+        vm = QEMUMachine(self.qemu_bin)
+        if args:
+            vm.add_args(*args)
+        return vm
+
+    @property
+    def vm(self):
+        return self.get_vm(name='default')
+
+    def get_vm(self, *args, name=None):
+        if not name:
+            name = str(uuid.uuid4())
+        if self._vms.get(name) is None:
+            self._vms[name] = self._new_vm(*args)
+        return self._vms[name]
 
     def tearDown(self):
-        if self.vm is not None:
-            self.vm.shutdown()
+        for vm in self._vms.values():
+            vm.shutdown()
diff --git a/tests/acceptance/boot_linux_console.py b/tests/acceptance/boot_linux_console.py
index 98324f7591..beeb1e59e8 100644
--- a/tests/acceptance/boot_linux_console.py
+++ b/tests/acceptance/boot_linux_console.py
@@ -18,7 +18,6 @@ class BootLinuxConsole(Test):
     Boots a x86_64 Linux kernel and checks that the console is operational
     and the kernel command line is properly passed from QEMU to the kernel
 
-    :avocado: enable
     :avocado: tags=x86_64
     """
 
diff --git a/tests/acceptance/linux_initrd.py b/tests/acceptance/linux_initrd.py
index 737355c2ef..fbdb48e43f 100644
--- a/tests/acceptance/linux_initrd.py
+++ b/tests/acceptance/linux_initrd.py
@@ -8,6 +8,7 @@
 # This work is licensed under the terms of the GNU GPL, version 2 or
 # later.  See the COPYING file in the top-level directory.
 
+import logging
 import tempfile
 from avocado.utils.process import run
 
@@ -18,20 +19,21 @@ class LinuxInitrd(Test):
     """
     Checks QEMU evaluates correctly the initrd file passed as -initrd option.
 
-    :avocado: enable
     :avocado: tags=x86_64
     """
 
-    timeout = 60
+    timeout = 300
 
-    def test_with_2gib_file_should_exit_error_msg(self):
+    def test_with_2gib_file_should_exit_error_msg_with_linux_v3_6(self):
         """
         Pretends to boot QEMU with an initrd file with size of 2GiB
         and expect it exits with error message.
+        Fedora-18 shipped with linux-3.6 which have not supported xloadflags
+        cannot support more than 2GiB initrd.
         """
-        kernel_url = ('https://mirrors.kernel.org/fedora/releases/28/'
-                      'Everything/x86_64/os/images/pxeboot/vmlinuz')
-        kernel_hash = '238e083e114c48200f80d889f7e32eeb2793e02a'
+        kernel_url = ('https://archives.fedoraproject.org/pub/archive/fedora/li'
+                      'nux/releases/18/Fedora/x86_64/os/images/pxeboot/vmlinuz')
+        kernel_hash = '41464f68efe42b9991250bed86c7081d2ccdbb21'
         kernel_path = self.fetch_asset(kernel_url, asset_hash=kernel_hash)
         max_size = 2 * (1024 ** 3) - 1
 
@@ -39,10 +41,44 @@ class LinuxInitrd(Test):
             initrd.seek(max_size)
             initrd.write(b'\0')
             initrd.flush()
-            cmd = "%s -kernel %s -initrd %s" % (self.qemu_bin, kernel_path,
-                                                initrd.name)
+            cmd = "%s -kernel %s -initrd %s -m 4096" % (
+                  self.qemu_bin, kernel_path, initrd.name)
             res = run(cmd, ignore_status=True)
             self.assertEqual(res.exit_status, 1)
             expected_msg = r'.*initrd is too large.*max: \d+, need %s.*' % (
                 max_size + 1)
             self.assertRegex(res.stderr_text, expected_msg)
+
+    def test_with_2gib_file_should_work_with_linux_v4_16(self):
+        """
+        QEMU has supported up to 4 GiB initrd for recent kernel
+        Expect guest can reach 'Unpacking initramfs...'
+        """
+        kernel_url = ('https://mirrors.kernel.org/fedora/releases/28/'
+                      'Everything/x86_64/os/images/pxeboot/vmlinuz')
+        kernel_hash = '238e083e114c48200f80d889f7e32eeb2793e02a'
+        kernel_path = self.fetch_asset(kernel_url, asset_hash=kernel_hash)
+        max_size = 2 * (1024 ** 3) + 1
+
+        with tempfile.NamedTemporaryFile() as initrd:
+            initrd.seek(max_size)
+            initrd.write(b'\0')
+            initrd.flush()
+
+            self.vm.set_machine('pc')
+            self.vm.set_console()
+            kernel_command_line = 'console=ttyS0'
+            self.vm.add_args('-kernel', kernel_path,
+                             '-append', kernel_command_line,
+                             '-initrd', initrd.name,
+                             '-m', '5120')
+            self.vm.launch()
+            console = self.vm.console_socket.makefile()
+            console_logger = logging.getLogger('console')
+            while True:
+                msg = console.readline()
+                console_logger.debug(msg.strip())
+                if 'Unpacking initramfs...' in msg:
+                    break
+                if 'Kernel panic - not syncing' in msg:
+                    self.fail("Kernel panic reached")
diff --git a/tests/acceptance/migration.py b/tests/acceptance/migration.py
new file mode 100644
index 0000000000..6115cf6c24
--- /dev/null
+++ b/tests/acceptance/migration.py
@@ -0,0 +1,53 @@
+# Migration test
+#
+# Copyright (c) 2019 Red Hat, Inc.
+#
+# Authors:
+#  Cleber Rosa <crosa@redhat.com>
+#  Caio Carrara <ccarrara@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2 or
+# later.  See the COPYING file in the top-level directory.
+
+
+from avocado_qemu import Test
+
+from avocado.utils import network
+from avocado.utils import wait
+
+
+class Migration(Test):
+    """
+    :avocado: enable
+    """
+
+    timeout = 10
+
+    @staticmethod
+    def migration_finished(vm):
+        return vm.command('query-migrate')['status'] in ('completed', 'failed')
+
+    def _get_free_port(self):
+        port = network.find_free_port()
+        if port is None:
+            self.cancel('Failed to find a free port')
+        return port
+
+
+    def test_migration_with_tcp_localhost(self):
+        source_vm = self.get_vm()
+        dest_uri = 'tcp:localhost:%u' % self._get_free_port()
+        dest_vm = self.get_vm('-incoming', dest_uri)
+        dest_vm.launch()
+        source_vm.launch()
+        source_vm.qmp('migrate', uri=dest_uri)
+        wait.wait_for(
+            self.migration_finished,
+            timeout=self.timeout,
+            step=0.1,
+            args=(source_vm,)
+        )
+        self.assertEqual(dest_vm.command('query-migrate')['status'], 'completed')
+        self.assertEqual(source_vm.command('query-migrate')['status'], 'completed')
+        self.assertEqual(dest_vm.command('query-status')['status'], 'running')
+        self.assertEqual(source_vm.command('query-status')['status'], 'postmigrate')
diff --git a/tests/acceptance/version.py b/tests/acceptance/version.py
index 13b0a7440d..67c2192c93 100644
--- a/tests/acceptance/version.py
+++ b/tests/acceptance/version.py
@@ -14,7 +14,6 @@ from avocado_qemu import Test
 
 class Version(Test):
     """
-    :avocado: enable
     :avocado: tags=quick
     """
     def test_qmp_human_info_version(self):
diff --git a/tests/acceptance/virtio_version.py b/tests/acceptance/virtio_version.py
index ce990250d8..37fc01ea18 100644
--- a/tests/acceptance/virtio_version.py
+++ b/tests/acceptance/virtio_version.py
@@ -11,7 +11,7 @@ Check compatibility of virtio device types
 import sys
 import os
 
-sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "scripts"))
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
 from qemu import QEMUMachine
 from avocado_qemu import Test
 
@@ -61,7 +61,6 @@ class VirtioVersionCheck(Test):
     same device tree created by `disable-modern` and
     `disable-legacy`.
 
-    :avocado: enable
     :avocado: tags=x86_64
     """
 
diff --git a/tests/acceptance/vnc.py b/tests/acceptance/vnc.py
index b1ef9d71b1..064ceabcc1 100644
--- a/tests/acceptance/vnc.py
+++ b/tests/acceptance/vnc.py
@@ -13,7 +13,6 @@ from avocado_qemu import Test
 
 class Vnc(Test):
     """
-    :avocado: enable
     :avocado: tags=vnc,quick
     """
     def test_no_vnc(self):
diff --git a/tests/migration/guestperf/engine.py b/tests/migration/guestperf/engine.py
index 398e3f2706..0e304660b8 100644
--- a/tests/migration/guestperf/engine.py
+++ b/tests/migration/guestperf/engine.py
@@ -24,13 +24,14 @@ import re
 import sys
 import time
 
-sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', '..', 'scripts'))
-import qemu
-import qmp.qmp
 from guestperf.progress import Progress, ProgressStats
 from guestperf.report import Report
 from guestperf.timings import TimingRecord, Timings
 
+sys.path.append(os.path.join(os.path.dirname(__file__),
+                             '..', '..', '..', 'python'))
+import qemu
+
 
 class Engine(object):
 
diff --git a/tests/qemu-iotests/235 b/tests/qemu-iotests/235
index d6edd97ab4..75c203b30c 100755
--- a/tests/qemu-iotests/235
+++ b/tests/qemu-iotests/235
@@ -23,7 +23,7 @@ import os
 import iotests
 from iotests import qemu_img_create, qemu_io, file_path, log
 
-sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'scripts'))
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
 
 from qemu import QEMUMachine
 
diff --git a/tests/qemu-iotests/238 b/tests/qemu-iotests/238
index f81ee1112f..688abc9acb 100755
--- a/tests/qemu-iotests/238
+++ b/tests/qemu-iotests/238
@@ -23,7 +23,7 @@ import os
 import iotests
 from iotests import log
 
-sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'scripts'))
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
 
 from qemu import QEMUMachine
 
diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
index 4910fb2005..3d15571688 100644
--- a/tests/qemu-iotests/iotests.py
+++ b/tests/qemu-iotests/iotests.py
@@ -32,8 +32,8 @@ import atexit
 import io
 from collections import OrderedDict
 
-sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'scripts'))
-import qtest
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
+from qemu import qtest
 
 
 # This will not work if arguments contain spaces but is necessary if we
diff --git a/tests/requirements.txt b/tests/requirements.txt
index 64c6e27a94..002ded6a22 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -1,4 +1,4 @@
 # Add Python module requirements, one per line, to be installed
 # in the tests/venv Python virtual environment. For more info,
 # refer to: https://pip.pypa.io/en/stable/user_guide/#id1
-avocado-framework==65.0
+avocado-framework==68.0
diff --git a/tests/vm/basevm.py b/tests/vm/basevm.py
index bdca6cb2fc..0556bdcf9e 100755
--- a/tests/vm/basevm.py
+++ b/tests/vm/basevm.py
@@ -17,7 +17,7 @@ import sys
 import logging
 import time
 import datetime
-sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "scripts"))
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
 from qemu import QEMUMachine, kvm_available
 import subprocess
 import hashlib