diff options
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/docker/Makefile.include | 1 | ||||
| -rwxr-xr-x | tests/docker/docker.py | 110 | ||||
| -rw-r--r-- | tests/docker/dockerfiles/debian-buster-arm64-cross.docker | 15 | ||||
| -rw-r--r-- | tests/docker/dockerfiles/debian10.docker | 31 | ||||
| -rw-r--r-- | tests/tcg/aarch64/Makefile.include | 2 | ||||
| -rw-r--r-- | tests/vm/Makefile.include | 22 | ||||
| -rwxr-xr-x | tests/vm/basevm.py | 3 | ||||
| -rwxr-xr-x | tests/vm/centos | 3 | ||||
| -rwxr-xr-x | tests/vm/freebsd | 6 | ||||
| -rwxr-xr-x | tests/vm/netbsd | 6 | ||||
| -rwxr-xr-x | tests/vm/openbsd | 3 | ||||
| -rwxr-xr-x | tests/vm/ubuntu.i386 | 3 |
12 files changed, 159 insertions, 46 deletions
diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include index 7032c68895..83d43c50e4 100644 --- a/tests/docker/Makefile.include +++ b/tests/docker/Makefile.include @@ -90,6 +90,7 @@ docker-image-debian-amd64: docker-image-debian9 docker-image-debian-armel-cross: docker-image-debian9 docker-image-debian-armhf-cross: docker-image-debian9 docker-image-debian-arm64-cross: docker-image-debian9 +docker-image-debian-buster-arm64-cross: docker-image-debian10 docker-image-debian-mips-cross: docker-image-debian9 docker-image-debian-mipsel-cross: docker-image-debian9 docker-image-debian-mips64el-cross: docker-image-debian9 diff --git a/tests/docker/docker.py b/tests/docker/docker.py index 02d8a83847..53a8c9c801 100755 --- a/tests/docker/docker.py +++ b/tests/docker/docker.py @@ -30,7 +30,7 @@ except ImportError: from io import StringIO from shutil import copy, rmtree from pwd import getpwuid -from datetime import datetime,timedelta +from datetime import datetime, timedelta FILTERED_ENV_NAMES = ['ftp_proxy', 'http_proxy', 'https_proxy'] @@ -43,9 +43,11 @@ def _text_checksum(text): """Calculate a digest string unique to the text content""" return hashlib.sha1(text).hexdigest() + def _file_checksum(filename): return _text_checksum(open(filename, 'rb').read()) + def _guess_docker_command(): """ Guess a working docker command or raise exception if not found""" commands = [["docker"], ["sudo", "-n", "docker"]] @@ -59,9 +61,10 @@ def _guess_docker_command(): except OSError: pass commands_txt = "\n".join([" " + " ".join(x) for x in commands]) - raise Exception("Cannot find working docker command. Tried:\n%s" % \ + raise Exception("Cannot find working docker command. Tried:\n%s" % commands_txt) + def _copy_with_mkdir(src, root_dir, sub_path='.'): """Copy src into root_dir, creating sub_path as needed.""" dest_dir = os.path.normpath("%s/%s" % (root_dir, sub_path)) @@ -96,19 +99,29 @@ def _get_so_libs(executable): return libs -def _copy_binary_with_libs(src, dest_dir): - """Copy a binary executable and all its dependent libraries. + +def _copy_binary_with_libs(src, bin_dest, dest_dir): + """Maybe copy a binary and all its dependent libraries. + + If bin_dest isn't set we only copy the support libraries because + we don't need qemu in the docker path to run (due to persistent + mapping). Indeed users may get confused if we aren't running what + is in the image. This does rely on the host file-system being fairly multi-arch - aware so the file don't clash with the guests layout.""" + aware so the file don't clash with the guests layout. + """ - _copy_with_mkdir(src, dest_dir, "/usr/bin") + if bin_dest: + _copy_with_mkdir(src, dest_dir, os.path.dirname(bin_dest)) + else: + print("only copying support libraries for %s" % (src)) libs = _get_so_libs(src) if libs: for l in libs: so_path = os.path.dirname(l) - _copy_with_mkdir(l , dest_dir, so_path) + _copy_with_mkdir(l, dest_dir, so_path) def _check_binfmt_misc(executable): @@ -116,23 +129,34 @@ def _check_binfmt_misc(executable): The details of setting up binfmt_misc are outside the scope of this script but we should at least fail early with a useful - message if it won't work.""" + message if it won't work. + + Returns the configured binfmt path and a valid flag. For + persistent configurations we will still want to copy and dependent + libraries. + """ binary = os.path.basename(executable) binfmt_entry = "/proc/sys/fs/binfmt_misc/%s" % (binary) if not os.path.exists(binfmt_entry): print ("No binfmt_misc entry for %s" % (binary)) - return False + return None, False with open(binfmt_entry) as x: entry = x.read() - qpath = "/usr/bin/%s" % (binary) - if not re.search("interpreter %s\n" % (qpath), entry): - print ("binfmt_misc for %s does not point to %s" % (binary, qpath)) - return False + if re.search("flags:.*F.*\n", entry): + print("binfmt_misc for %s uses persistent(F) mapping to host binary" % + (binary)) + return None, True - return True + m = re.search("interpreter (\S+)\n", entry) + interp = m.group(1) + if interp and interp != executable: + print("binfmt_misc for %s does not point to %s, using %s" % + (binary, executable, interp)) + + return interp, True def _read_qemu_dockerfile(img_name): @@ -144,6 +168,7 @@ def _read_qemu_dockerfile(img_name): img_name + ".docker") return open(df, "r").read() + def _dockerfile_preprocess(df): out = "" for l in df.splitlines(): @@ -161,6 +186,7 @@ def _dockerfile_preprocess(df): out += l + "\n" return out + class Docker(object): """ Running Docker commands """ def __init__(self): @@ -228,7 +254,7 @@ class Docker(object): def build_image(self, tag, docker_dir, dockerfile, quiet=True, user=False, argv=None, extra_files_cksum=[]): - if argv == None: + if argv is None: argv = [] tmp_df = tempfile.NamedTemporaryFile(dir=docker_dir, suffix=".docker") @@ -249,7 +275,7 @@ class Docker(object): tmp_df.flush() - self._do_check(["build", "-t", tag, "-f", tmp_df.name] + argv + \ + self._do_check(["build", "-t", tag, "-f", tmp_df.name] + argv + [docker_dir], quiet=quiet) @@ -279,9 +305,11 @@ class Docker(object): def command(self, cmd, argv, quiet): return self._do([cmd] + argv, quiet=quiet) + class SubCommand(object): """A SubCommand template base class""" - name = None # Subcommand name + name = None # Subcommand name + def shared_args(self, parser): parser.add_argument("--quiet", action="store_true", help="Run quietly unless an error occurred") @@ -289,6 +317,7 @@ class SubCommand(object): def args(self, parser): """Setup argument parser""" pass + def run(self, args, argv): """Run command. args: parsed argument by argument parser. @@ -296,18 +325,23 @@ class SubCommand(object): """ pass + class RunCommand(SubCommand): """Invoke docker run and take care of cleaning up""" name = "run" + def args(self, parser): parser.add_argument("--keep", action="store_true", help="Don't remove image when command completes") + def run(self, args, argv): return Docker().run(argv, args.keep, quiet=args.quiet) + class BuildCommand(SubCommand): - """ Build docker image out of a dockerfile. Arguments: <tag> <dockerfile>""" + """ Build docker image out of a dockerfile. Arg: <tag> <dockerfile>""" name = "build" + def args(self, parser): parser.add_argument("--include-executable", "-e", help="""Specify a binary that will be copied to the @@ -340,7 +374,8 @@ class BuildCommand(SubCommand): # Validate binfmt_misc will work if args.include_executable: - if not _check_binfmt_misc(args.include_executable): + qpath, enabled = _check_binfmt_misc(args.include_executable) + if not enabled: return 1 # Is there a .pre file to run in the build context? @@ -363,14 +398,16 @@ class BuildCommand(SubCommand): # FIXME: there is no checksum of this executable and the linked # libraries, once the image built any change of this executable # or any library won't trigger another build. - _copy_binary_with_libs(args.include_executable, docker_dir) + _copy_binary_with_libs(args.include_executable, + qpath, docker_dir) + for filename in args.extra_files or []: _copy_with_mkdir(filename, docker_dir) cksum += [(filename, _file_checksum(filename))] argv += ["--build-arg=" + k.lower() + "=" + v - for k, v in os.environ.iteritems() - if k.lower() in FILTERED_ENV_NAMES] + for k, v in os.environ.iteritems() + if k.lower() in FILTERED_ENV_NAMES] dkr.build_image(tag, docker_dir, dockerfile, quiet=args.quiet, user=args.user, argv=argv, extra_files_cksum=cksum) @@ -379,9 +416,11 @@ class BuildCommand(SubCommand): return 0 + class UpdateCommand(SubCommand): - """ Update a docker image with new executables. Arguments: <tag> <executable>""" + """ Update a docker image with new executables. Args: <tag> <executable>""" name = "update" + def args(self, parser): parser.add_argument("tag", help="Image Tag") @@ -394,10 +433,17 @@ class UpdateCommand(SubCommand): tmp = tempfile.NamedTemporaryFile(suffix="dckr.tar.gz") tmp_tar = TarFile(fileobj=tmp, mode='w') - # Add the executable to the tarball - bn = os.path.basename(args.executable) - ff = "/usr/bin/%s" % bn - tmp_tar.add(args.executable, arcname=ff) + # Add the executable to the tarball, using the current + # configured binfmt_misc path. If we don't get a path then we + # only need the support libraries copied + ff, enabled = _check_binfmt_misc(args.executable) + + if not enabled: + print("binfmt_misc not enabled, update disabled") + return 1 + + if ff: + tmp_tar.add(args.executable, arcname=ff) # Add any associated libraries libs = _get_so_libs(args.executable) @@ -427,16 +473,20 @@ class UpdateCommand(SubCommand): return 0 + class CleanCommand(SubCommand): """Clean up docker instances""" name = "clean" + def run(self, args, argv): Docker().clean() return 0 + class ImagesCommand(SubCommand): """Run "docker images" command""" name = "images" + def run(self, args, argv): return Docker().command("images", argv, args.quiet) @@ -509,7 +559,7 @@ class CheckCommand(SubCommand): try: dkr = Docker() - except: + except subprocess.CalledProcessError: print("Docker not set up") return 1 @@ -548,7 +598,8 @@ class CheckCommand(SubCommand): def main(): parser = argparse.ArgumentParser(description="A Docker helper", - usage="%s <subcommand> ..." % os.path.basename(sys.argv[0])) + usage="%s <subcommand> ..." % + os.path.basename(sys.argv[0])) subparsers = parser.add_subparsers(title="subcommands", help=None) for cls in SubCommand.__subclasses__(): cmd = cls() @@ -559,5 +610,6 @@ def main(): args, argv = parser.parse_known_args() return args.cmdobj.run(args, argv) + if __name__ == "__main__": sys.exit(main()) diff --git a/tests/docker/dockerfiles/debian-buster-arm64-cross.docker b/tests/docker/dockerfiles/debian-buster-arm64-cross.docker new file mode 100644 index 0000000000..33ada13806 --- /dev/null +++ b/tests/docker/dockerfiles/debian-buster-arm64-cross.docker @@ -0,0 +1,15 @@ +# +# Docker arm64 cross-compiler target +# +# This docker target builds on the Debian's Buster base image. There +# are no QEMU pre-requistes so this image can only be used to build +# test cases. +# +FROM qemu:debian10 + +# Add the foreign architecture we want and install dependencies +RUN dpkg --add-architecture arm64 +RUN apt update +RUN DEBIAN_FRONTEND=noninteractive eatmydata \ + apt-get install -y --no-install-recommends \ + crossbuild-essential-arm64 diff --git a/tests/docker/dockerfiles/debian10.docker b/tests/docker/dockerfiles/debian10.docker new file mode 100644 index 0000000000..aeeb151b52 --- /dev/null +++ b/tests/docker/dockerfiles/debian10.docker @@ -0,0 +1,31 @@ +# +# Docker multiarch cross-compiler target +# +# This docker target is builds on Debian cross compiler targets to build distro +# with a selection of cross compilers for building test binaries. +# +# On its own you can't build much but the docker-foo-cross targets +# build on top of the base debian image. +# +FROM debian:buster-slim + +# Duplicate deb line as deb-src +RUN cat /etc/apt/sources.list | sed "s/^deb\ /deb-src /" >> /etc/apt/sources.list + +# Install common build utilities +RUN apt update +RUN DEBIAN_FRONTEND=noninteractive apt install -yy eatmydata +RUN DEBIAN_FRONTEND=noninteractive eatmydata \ + apt install -y --no-install-recommends \ + bison \ + build-essential \ + ca-certificates \ + clang \ + flex \ + gettext \ + git \ + pkg-config \ + psmisc \ + python \ + texinfo \ + $(apt-get -s build-dep qemu | egrep ^Inst | fgrep '[all]' | cut -d\ -f2) diff --git a/tests/tcg/aarch64/Makefile.include b/tests/tcg/aarch64/Makefile.include index de32c91235..5d4e4c6f99 100644 --- a/tests/tcg/aarch64/Makefile.include +++ b/tests/tcg/aarch64/Makefile.include @@ -3,6 +3,6 @@ # We don't have any bigendian build tools so we only use this for AArch64 ifeq ($(TARGET_NAME),aarch64) -DOCKER_IMAGE=debian-arm64-cross +DOCKER_IMAGE=debian-buster-arm64-cross DOCKER_CROSS_COMPILER=aarch64-linux-gnu-gcc endif diff --git a/tests/vm/Makefile.include b/tests/vm/Makefile.include index a98fb3027f..992d823f6b 100644 --- a/tests/vm/Makefile.include +++ b/tests/vm/Makefile.include @@ -3,7 +3,8 @@ .PHONY: vm-build-all vm-clean-all IMAGES := ubuntu.i386 freebsd netbsd openbsd centos -IMAGE_FILES := $(patsubst %, tests/vm/%.img, $(IMAGES)) +IMAGES_DIR := $(HOME)/.cache/qemu-vm/images +IMAGE_FILES := $(patsubst %, $(IMAGES_DIR)/%.img, $(IMAGES)) .PRECIOUS: $(IMAGE_FILES) @@ -18,15 +19,21 @@ vm-test: @echo "" @echo " vm-build-all - Build QEMU in all VMs" @echo " vm-clean-all - Clean up VM images" + @echo + @echo "Special variables:" + @echo " BUILD_TARGET=foo - override the build target" + @echo " TARGET_LIST=a,b,c - Override target list in builds." + @echo ' EXTRA_CONFIGURE_OPTS="..."' vm-build-all: $(addprefix vm-build-, $(IMAGES)) vm-clean-all: rm -f $(IMAGE_FILES) -tests/vm/%.img: $(SRC_PATH)/tests/vm/% \ - $(SRC_PATH)/tests/vm/basevm.py \ - $(SRC_PATH)/tests/vm/Makefile.include +$(IMAGES_DIR)/%.img: $(SRC_PATH)/tests/vm/% \ + $(SRC_PATH)/tests/vm/basevm.py \ + $(SRC_PATH)/tests/vm/Makefile.include + @mkdir -p $(IMAGES_DIR) $(call quiet-command, \ $< \ $(if $(V)$(DEBUG), --debug) \ @@ -37,7 +44,7 @@ tests/vm/%.img: $(SRC_PATH)/tests/vm/% \ # Build in VM $(IMAGE) -vm-build-%: tests/vm/%.img +vm-build-%: $(IMAGES_DIR)/%.img $(call quiet-command, \ $(SRC_PATH)/tests/vm/$* \ $(if $(V)$(DEBUG), --debug) \ @@ -45,6 +52,9 @@ vm-build-%: tests/vm/%.img $(if $(J),--jobs $(J)) \ $(if $(V),--verbose) \ --image "$<" \ - --build-qemu $(SRC_PATH), \ + $(if $(BUILD_TARGET),--build-target $(BUILD_TARGET)) \ + --build-qemu $(SRC_PATH) -- \ + $(if $(TARGET_LIST),--target-list=$(TARGET_LIST)) \ + $(if $(EXTRA_CONFIGURE_OPTS),$(EXTRA_CONFIGURE_OPTS)), \ " VM-BUILD $*") diff --git a/tests/vm/basevm.py b/tests/vm/basevm.py index 5caf77d6b8..bdca6cb2fc 100755 --- a/tests/vm/basevm.py +++ b/tests/vm/basevm.py @@ -228,6 +228,8 @@ def parse_args(vmcls): help="build image") parser.add_option("--build-qemu", help="build QEMU from source in guest") + parser.add_option("--build-target", + help="QEMU build target", default="check") parser.add_option("--interactive", "-I", action="store_true", help="Interactively run command") parser.add_option("--snapshot", "-s", action="store_true", @@ -255,6 +257,7 @@ def main(vmcls): cmd = [vm.BUILD_SCRIPT.format( configure_opts = " ".join(argv), jobs=args.jobs, + target=args.build_target, verbose = "V=1" if args.verbose else "")] else: cmd = argv diff --git a/tests/vm/centos b/tests/vm/centos index daa2dbca03..ba133ea429 100755 --- a/tests/vm/centos +++ b/tests/vm/centos @@ -65,8 +65,9 @@ class CentosVM(basevm.BaseVM): def build_image(self, img): cimg = self._download_with_cache("https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-1802.qcow2.xz") img_tmp = img + ".tmp" + sys.stderr.write("Extracting the image...\n") subprocess.check_call(["cp", "-f", cimg, img_tmp + ".xz"]) - subprocess.check_call(["xz", "-df", img_tmp + ".xz"]) + subprocess.check_call(["xz", "-dvf", img_tmp + ".xz"]) subprocess.check_call(["qemu-img", "resize", img_tmp, "50G"]) self.boot(img_tmp, extra_args = ["-cdrom", self._gen_cloud_init_iso()]) self.wait_ssh() diff --git a/tests/vm/freebsd b/tests/vm/freebsd index 19a3729172..b0066017a6 100755 --- a/tests/vm/freebsd +++ b/tests/vm/freebsd @@ -25,8 +25,7 @@ class FreeBSDVM(basevm.BaseVM): cd $(mktemp -d /var/tmp/qemu-test.XXXXXX); tar -xf /dev/vtbd1; ./configure {configure_opts}; - gmake --output-sync -j{jobs} {verbose}; - gmake --output-sync -j{jobs} check {verbose}; + gmake --output-sync -j{jobs} {target} {verbose}; """ def build_image(self, img): @@ -34,8 +33,9 @@ class FreeBSDVM(basevm.BaseVM): sha256sum='adcb771549b37bc63826c501f05121a206ed3d9f55f49145908f7e1432d65891') img_tmp_xz = img + ".tmp.xz" img_tmp = img + ".tmp" + sys.stderr.write("Extracting the image...\n") subprocess.check_call(["cp", "-f", cimg, img_tmp_xz]) - subprocess.check_call(["xz", "-df", img_tmp_xz]) + subprocess.check_call(["xz", "-dvf", img_tmp_xz]) if os.path.exists(img): os.remove(img) os.rename(img_tmp, img) diff --git a/tests/vm/netbsd b/tests/vm/netbsd index fac6a7ce51..4c6624ea5e 100755 --- a/tests/vm/netbsd +++ b/tests/vm/netbsd @@ -25,8 +25,7 @@ class NetBSDVM(basevm.BaseVM): cd $(mktemp -d /var/tmp/qemu-test.XXXXXX); tar -xf /dev/rld1a; ./configure --python=python2.7 {configure_opts}; - gmake --output-sync -j{jobs} {verbose}; - gmake --output-sync -j{jobs} check {verbose}; + gmake --output-sync -j{jobs} {target} {verbose}; """ def build_image(self, img): @@ -34,8 +33,9 @@ class NetBSDVM(basevm.BaseVM): sha256sum='b633d565b0eac3d02015cd0c81440bd8a7a8df8512615ac1ee05d318be015732') img_tmp_xz = img + ".tmp.xz" img_tmp = img + ".tmp" + sys.stderr.write("Extracting the image...\n") subprocess.check_call(["cp", "-f", cimg, img_tmp_xz]) - subprocess.check_call(["xz", "-df", img_tmp_xz]) + subprocess.check_call(["xz", "-dvf", img_tmp_xz]) if os.path.exists(img): os.remove(img) os.rename(img_tmp, img) diff --git a/tests/vm/openbsd b/tests/vm/openbsd index cfe0572c59..2105c01a26 100755 --- a/tests/vm/openbsd +++ b/tests/vm/openbsd @@ -35,8 +35,9 @@ class OpenBSDVM(basevm.BaseVM): sha256sum='8c6cedc483e602cfee5e04f0406c64eb99138495e8ca580bc0293bcf0640c1bf') img_tmp_xz = img + ".tmp.xz" img_tmp = img + ".tmp" + sys.stderr.write("Extracting the image...\n") subprocess.check_call(["cp", "-f", cimg, img_tmp_xz]) - subprocess.check_call(["xz", "-df", img_tmp_xz]) + subprocess.check_call(["xz", "-dvf", img_tmp_xz]) if os.path.exists(img): os.remove(img) os.rename(img_tmp, img) diff --git a/tests/vm/ubuntu.i386 b/tests/vm/ubuntu.i386 index 1b7e1ab8f0..a22d137e76 100755 --- a/tests/vm/ubuntu.i386 +++ b/tests/vm/ubuntu.i386 @@ -26,8 +26,7 @@ class UbuntuX86VM(basevm.BaseVM): sudo chmod a+r /dev/vdb; tar -xf /dev/vdb; ./configure {configure_opts}; - make --output-sync -j{jobs}; - make --output-sync check -j{jobs} {verbose}; + make --output-sync {target} -j{jobs} {verbose}; """ def _gen_cloud_init_iso(self): |