Merge remote-tracking branch 'remotes/stsquad/tags/pull-testing-next-110219-1' into staging

Testing updates:

  - .travis.yml tweaks and optimisations
  - .cirrus.yml enabled for FreeBSD CI
  - docker.py clean-ups for binfmt_misc
  - more control of vm-test builds

# gpg: Signature made Mon 11 Feb 2019 13:03:14 GMT
# gpg:                using RSA key F715F7CD46F94435F4F588658E520D61289519AE
# gpg: Good signature from "Alex Bennée (Master Work Key) <alex.bennee@linaro.org>" [full]
# Primary key fingerprint: 6685 AE99 E751 67BC AFC8  DF35 FBD0 DB09 5A9E 2A44
#      Subkey fingerprint: F715 F7CD 46F9 4435 F4F5  8865 8E52 0D61 2895 19AE

* remotes/stsquad/tags/pull-testing-next-110219-1:
  tests/vm: Be verbose while extracting compressed images
  docs/devel/testing: Add -a option to usermod command on docker setup
  scripts/qemu.py: allow arches use KVM for their 32bit cousins
  tests/vm: expose BUILD_TARGET, TARGET_LIST and EXTRA_CONFIGURE_OPTS
  tests/vm: add --build-target option
  tests/vm: call make check directly for netbsd/freebsd/ubuntu.i386
  tests/vm: move images to $HOME/.cache/qemu-vm/images
  tests: PEP8 cleanup of docker.py, mostly white space
  tests: docker.py be even smarter with persistent binfmt_misc
  tests: make docker.py check for persistent configs
  tests: make docker.py update use configured binfmt path
  docker: add debian-buster-arm64-cross
  archive-source.sh: Clone the submodules locally
  MAINTAINERS: Add an entry for scripts/archive-source.sh
  .travis.yml: fold --disable-tcg into alternate coroutine builds
  .travis.yml: separate tools and docs into another entry
  .travis.yml: stop requesting libffi & gettext from homebrew
  .cirrus.yml: basic compile and test for FreeBSD

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>

1 files changed, 81 insertions, 29 deletions

diff --git a/tests/docker/docker.py b/tests/docker/docker.py
index 02d8a83847..53a8c9c801 100755
--- a/tests/docker/docker.py
+++ b/tests/docker/docker.py
@@ -30,7 +30,7 @@ except ImportError:
     from io import StringIO
 from shutil import copy, rmtree
 from pwd import getpwuid
-from datetime import datetime,timedelta
+from datetime import datetime, timedelta
 
 
 FILTERED_ENV_NAMES = ['ftp_proxy', 'http_proxy', 'https_proxy']
@@ -43,9 +43,11 @@ def _text_checksum(text):
     """Calculate a digest string unique to the text content"""
     return hashlib.sha1(text).hexdigest()
 
+
 def _file_checksum(filename):
     return _text_checksum(open(filename, 'rb').read())
 
+
 def _guess_docker_command():
     """ Guess a working docker command or raise exception if not found"""
     commands = [["docker"], ["sudo", "-n", "docker"]]
@@ -59,9 +61,10 @@ def _guess_docker_command():
         except OSError:
             pass
     commands_txt = "\n".join(["  " + " ".join(x) for x in commands])
-    raise Exception("Cannot find working docker command. Tried:\n%s" % \
+    raise Exception("Cannot find working docker command. Tried:\n%s" %
                     commands_txt)
 
+
 def _copy_with_mkdir(src, root_dir, sub_path='.'):
     """Copy src into root_dir, creating sub_path as needed."""
     dest_dir = os.path.normpath("%s/%s" % (root_dir, sub_path))
@@ -96,19 +99,29 @@ def _get_so_libs(executable):
 
     return libs
 
-def _copy_binary_with_libs(src, dest_dir):
-    """Copy a binary executable and all its dependent libraries.
+
+def _copy_binary_with_libs(src, bin_dest, dest_dir):
+    """Maybe copy a binary and all its dependent libraries.
+
+    If bin_dest isn't set we only copy the support libraries because
+    we don't need qemu in the docker path to run (due to persistent
+    mapping). Indeed users may get confused if we aren't running what
+    is in the image.
 
     This does rely on the host file-system being fairly multi-arch
-    aware so the file don't clash with the guests layout."""
+    aware so the file don't clash with the guests layout.
+    """
 
-    _copy_with_mkdir(src, dest_dir, "/usr/bin")
+    if bin_dest:
+        _copy_with_mkdir(src, dest_dir, os.path.dirname(bin_dest))
+    else:
+        print("only copying support libraries for %s" % (src))
 
     libs = _get_so_libs(src)
     if libs:
         for l in libs:
             so_path = os.path.dirname(l)
-            _copy_with_mkdir(l , dest_dir, so_path)
+            _copy_with_mkdir(l, dest_dir, so_path)
 
 
 def _check_binfmt_misc(executable):
@@ -116,23 +129,34 @@ def _check_binfmt_misc(executable):
 
     The details of setting up binfmt_misc are outside the scope of
     this script but we should at least fail early with a useful
-    message if it won't work."""
+    message if it won't work.
+
+    Returns the configured binfmt path and a valid flag. For
+    persistent configurations we will still want to copy and dependent
+    libraries.
+    """
 
     binary = os.path.basename(executable)
     binfmt_entry = "/proc/sys/fs/binfmt_misc/%s" % (binary)
 
     if not os.path.exists(binfmt_entry):
         print ("No binfmt_misc entry for %s" % (binary))
-        return False
+        return None, False
 
     with open(binfmt_entry) as x: entry = x.read()
 
-    qpath = "/usr/bin/%s" % (binary)
-    if not re.search("interpreter %s\n" % (qpath), entry):
-        print ("binfmt_misc for %s does not point to %s" % (binary, qpath))
-        return False
+    if re.search("flags:.*F.*\n", entry):
+        print("binfmt_misc for %s uses persistent(F) mapping to host binary" %
+              (binary))
+        return None, True
 
-    return True
+    m = re.search("interpreter (\S+)\n", entry)
+    interp = m.group(1)
+    if interp and interp != executable:
+        print("binfmt_misc for %s does not point to %s, using %s" %
+              (binary, executable, interp))
+
+    return interp, True
 
 
 def _read_qemu_dockerfile(img_name):
@@ -144,6 +168,7 @@ def _read_qemu_dockerfile(img_name):
                       img_name + ".docker")
     return open(df, "r").read()
 
+
 def _dockerfile_preprocess(df):
     out = ""
     for l in df.splitlines():
@@ -161,6 +186,7 @@ def _dockerfile_preprocess(df):
         out += l + "\n"
     return out
 
+
 class Docker(object):
     """ Running Docker commands """
     def __init__(self):
@@ -228,7 +254,7 @@ class Docker(object):
 
     def build_image(self, tag, docker_dir, dockerfile,
                     quiet=True, user=False, argv=None, extra_files_cksum=[]):
-        if argv == None:
+        if argv is None:
             argv = []
 
         tmp_df = tempfile.NamedTemporaryFile(dir=docker_dir, suffix=".docker")
@@ -249,7 +275,7 @@ class Docker(object):
 
         tmp_df.flush()
 
-        self._do_check(["build", "-t", tag, "-f", tmp_df.name] + argv + \
+        self._do_check(["build", "-t", tag, "-f", tmp_df.name] + argv +
                        [docker_dir],
                        quiet=quiet)
 
@@ -279,9 +305,11 @@ class Docker(object):
     def command(self, cmd, argv, quiet):
         return self._do([cmd] + argv, quiet=quiet)
 
+
 class SubCommand(object):
     """A SubCommand template base class"""
-    name = None # Subcommand name
+    name = None  # Subcommand name
+
     def shared_args(self, parser):
         parser.add_argument("--quiet", action="store_true",
                             help="Run quietly unless an error occurred")
@@ -289,6 +317,7 @@ class SubCommand(object):
     def args(self, parser):
         """Setup argument parser"""
         pass
+
     def run(self, args, argv):
         """Run command.
         args: parsed argument by argument parser.
@@ -296,18 +325,23 @@ class SubCommand(object):
         """
         pass
 
+
 class RunCommand(SubCommand):
     """Invoke docker run and take care of cleaning up"""
     name = "run"
+
     def args(self, parser):
         parser.add_argument("--keep", action="store_true",
                             help="Don't remove image when command completes")
+
     def run(self, args, argv):
         return Docker().run(argv, args.keep, quiet=args.quiet)
 
+
 class BuildCommand(SubCommand):
-    """ Build docker image out of a dockerfile. Arguments: <tag> <dockerfile>"""
+    """ Build docker image out of a dockerfile. Arg: <tag> <dockerfile>"""
     name = "build"
+
     def args(self, parser):
         parser.add_argument("--include-executable", "-e",
                             help="""Specify a binary that will be copied to the
@@ -340,7 +374,8 @@ class BuildCommand(SubCommand):
 
             # Validate binfmt_misc will work
             if args.include_executable:
-                if not _check_binfmt_misc(args.include_executable):
+                qpath, enabled = _check_binfmt_misc(args.include_executable)
+                if not enabled:
                     return 1
 
             # Is there a .pre file to run in the build context?
@@ -363,14 +398,16 @@ class BuildCommand(SubCommand):
                 # FIXME: there is no checksum of this executable and the linked
                 # libraries, once the image built any change of this executable
                 # or any library won't trigger another build.
-                _copy_binary_with_libs(args.include_executable, docker_dir)
+                _copy_binary_with_libs(args.include_executable,
+                                       qpath, docker_dir)
+
             for filename in args.extra_files or []:
                 _copy_with_mkdir(filename, docker_dir)
                 cksum += [(filename, _file_checksum(filename))]
 
             argv += ["--build-arg=" + k.lower() + "=" + v
-                        for k, v in os.environ.iteritems()
-                        if k.lower() in FILTERED_ENV_NAMES]
+                     for k, v in os.environ.iteritems()
+                     if k.lower() in FILTERED_ENV_NAMES]
             dkr.build_image(tag, docker_dir, dockerfile,
                             quiet=args.quiet, user=args.user, argv=argv,
                             extra_files_cksum=cksum)
@@ -379,9 +416,11 @@ class BuildCommand(SubCommand):
 
         return 0
 
+
 class UpdateCommand(SubCommand):
-    """ Update a docker image with new executables. Arguments: <tag> <executable>"""
+    """ Update a docker image with new executables. Args: <tag> <executable>"""
     name = "update"
+
     def args(self, parser):
         parser.add_argument("tag",
                             help="Image Tag")
@@ -394,10 +433,17 @@ class UpdateCommand(SubCommand):
         tmp = tempfile.NamedTemporaryFile(suffix="dckr.tar.gz")
         tmp_tar = TarFile(fileobj=tmp, mode='w')
 
-        # Add the executable to the tarball
-        bn = os.path.basename(args.executable)
-        ff = "/usr/bin/%s" % bn
-        tmp_tar.add(args.executable, arcname=ff)
+        # Add the executable to the tarball, using the current
+        # configured binfmt_misc path. If we don't get a path then we
+        # only need the support libraries copied
+        ff, enabled = _check_binfmt_misc(args.executable)
+
+        if not enabled:
+            print("binfmt_misc not enabled, update disabled")
+            return 1
+
+        if ff:
+            tmp_tar.add(args.executable, arcname=ff)
 
         # Add any associated libraries
         libs = _get_so_libs(args.executable)
@@ -427,16 +473,20 @@ class UpdateCommand(SubCommand):
 
         return 0
 
+
 class CleanCommand(SubCommand):
     """Clean up docker instances"""
     name = "clean"
+
     def run(self, args, argv):
         Docker().clean()
         return 0
 
+
 class ImagesCommand(SubCommand):
     """Run "docker images" command"""
     name = "images"
+
     def run(self, args, argv):
         return Docker().command("images", argv, args.quiet)
 
@@ -509,7 +559,7 @@ class CheckCommand(SubCommand):
 
         try:
             dkr = Docker()
-        except:
+        except subprocess.CalledProcessError:
             print("Docker not set up")
             return 1
 
@@ -548,7 +598,8 @@ class CheckCommand(SubCommand):
 
 def main():
     parser = argparse.ArgumentParser(description="A Docker helper",
-            usage="%s <subcommand> ..." % os.path.basename(sys.argv[0]))
+                                     usage="%s <subcommand> ..." %
+                                     os.path.basename(sys.argv[0]))
     subparsers = parser.add_subparsers(title="subcommands", help=None)
     for cls in SubCommand.__subclasses__():
         cmd = cls()
@@ -559,5 +610,6 @@ def main():
     args, argv = parser.parse_known_args()
     return args.cmdobj.run(args, argv)
 
+
 if __name__ == "__main__":
     sys.exit(main())