tests: integrate lcitool for generating build env manifests

This introduces

  https://gitlab.com/libvirt/libvirt-ci

as a git submodule at tests/lcitool/libvirt-ci

The 'lcitool' program within this submodule will be used to
automatically generate build environment manifests from a definition
of requirements in tests/lcitool/projects/qemu.yml

It will ultimately be capable of generating

 - Dockerfiles
 - Package lists for installation in VMs
 - Variables for configuring Cirrus CI environments

When a new build pre-requisite is needed for QEMU, if this package
is not currently known to libvirt-ci, it must first be added to the
'mappings.yml' file in the above git repo.

Then the submodule can be updated and the build pre-requisite added
to the tests/lcitool/projects/qemu.yml file. Now all the build env
manifests can be re-generated using  'make lcitool-refresh'

This ensures that when a new build pre-requisite is introduced, it
is added to all the different OS containers, VMs and Cirrus CI
environments consistently.

It also facilitates the addition of containers targetting new distros
or updating existing containers to new versions of the same distro,
where packages might have been renamed.

Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com>
Signed-off-by: Daniel P. Berrangé <berrange@redhat.com>
Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
Message-Id: <20211215141949.3512719-8-berrange@redhat.com>
Message-Id: <20220105135009.1584676-8-alex.bennee@linaro.org>

4 files changed, 199 insertions, 0 deletions

diff --git a/tests/lcitool/Makefile.include b/tests/lcitool/Makefile.include
new file mode 100644
index 0000000000..cff7c0b814
--- /dev/null
+++ b/tests/lcitool/Makefile.include
@@ -0,0 +1,17 @@
+
+LCITOOL_REFRESH = $(SRC_PATH)/tests/lcitool/refresh
+
+lcitool:
+	@echo 'Manage build environment manifests'
+	@echo
+	@echo 'Available targets:'
+	@echo
+	@echo '    lcitool:             Print this help.'
+	@echo '    lcitool-refresh:     Re-generate all build environment manifests.'
+	@echo
+
+lcitool-help: lcitool
+
+lcitool-refresh:
+	$(call quiet-command, git submodule update --init $(SRC_PATH)/tests/lcitool/libvirt-ci)
+	$(call quiet-command, $(LCITOOL_REFRESH))
diff --git a/tests/lcitool/libvirt-ci b/tests/lcitool/libvirt-ci
new file mode 160000
+Subproject 29cec2153b9a4dbb2e66f1cbc9866a4eff519cf
diff --git a/tests/lcitool/projects/qemu.yml b/tests/lcitool/projects/qemu.yml
new file mode 100644
index 0000000000..2e2271510e
--- /dev/null
+++ b/tests/lcitool/projects/qemu.yml
@@ -0,0 +1,115 @@
+---
+packages:
+ - alsa
+ - bash
+ - bc
+ - brlapi
+ - bzip2
+ - bzip2-libs
+ - capstone
+ - ccache
+ - clang
+ - column
+ - ctags
+ - cyrus-sasl
+ - daxctl
+ - dbus-daemon
+ - device-mapper-multipath
+ - diffutils
+ - dtrace
+ - findutils
+ - g++
+ - gcc
+ - gcovr
+ - gettext
+ - genisoimage
+ - glib2
+ - glib2-static
+ - glibc-static
+ - glusterfs
+ - gnutls
+ - gtk3
+ - hostname
+ - libaio
+ - libattr
+ - libasan
+ - libbpf
+ - libcacard
+ - libcap-ng
+ - libcurl
+ - libdrm
+ - libepoxy
+ - libfdt
+ - libffi
+ - libgcrypt
+ - libibverbs
+ - libiscsi
+ - libjemalloc
+ - libjpeg
+ - libnfs
+ - libnuma
+ - libpmem
+ - libpng
+ - librbd
+ - librdmacm
+ - libseccomp
+ - libselinux
+ - libslirp
+ - libssh
+ - libtasn1
+ - libubsan
+ - libudev
+ - liburing
+ - libusbx
+ - libvdeplug
+ - libxml2
+ - libzstd
+ - llvm
+ - lttng-ust
+ - lzo
+ - netcat
+ - nettle
+ - ninja
+ - nsis
+ - make
+ - mesa-libgbm
+ - meson
+ - ncursesw
+ - pam
+ - pcre-static
+ - perl
+ - perl-Test-Harness
+ - pixman
+ - pkg-config
+ - pulseaudio
+ - python3
+ - python3-PyYAML
+ - python3-numpy
+ - python3-opencv
+ - python3-pillow
+ - python3-pip
+ - python3-sphinx
+ - python3-sphinx-rtd-theme
+ - python3-virtualenv
+ - rpm2cpio
+ - sdl2
+ - sdl2-image
+ - sed
+ - snappy
+ - sparse
+ - spice-protocol
+ - spice-server
+ - ssh-client
+ - systemd
+ - tar
+ - tesseract
+ - tesseract-eng
+ - texinfo
+ - usbredir
+ - virglrenderer
+ - vte
+ - which
+ - xen
+ - xfsprogs
+ - zlib
+ - zlib-static
diff --git a/tests/lcitool/refresh b/tests/lcitool/refresh
new file mode 100755
index 0000000000..b47e25f64b
--- /dev/null
+++ b/tests/lcitool/refresh
@@ -0,0 +1,67 @@
+#!/usr/bin/python3
+#
+# Re-generate container recipes
+#
+# This script uses the "lcitool" available from
+#
+#   https://gitlab.com/libvirt/libvirt-ci
+#
+# Copyright (c) 2020 Red Hat Inc.
+#
+# This work is licensed under the terms of the GNU GPL, version 2
+# or (at your option) any later version. See the COPYING file in
+# the top-level directory.
+
+import sys
+import os
+import subprocess
+
+from pathlib import Path
+
+if len(sys.argv) != 1:
+   print("syntax: %s" % sys.argv[0], file=sys.stderr)
+   sys.exit(1)
+
+self_dir = Path(__file__).parent
+src_dir = self_dir.parent.parent
+dockerfiles_dir = Path(src_dir, "tests", "docker", "dockerfiles")
+
+lcitool_path = Path(self_dir, "libvirt-ci", "lcitool")
+
+lcitool_cmd = [lcitool_path, "--data-dir", self_dir]
+
+def atomic_write(filename, content):
+   tmp = filename.with_suffix(filename.suffix + ".tmp")
+   try:
+      with tmp.open("w") as fp:
+         print(content, file=fp, end="")
+         tmp.rename(filename)
+   except Exception as ex:
+      tmp.unlink()
+      raise
+
+def generate(filename, cmd, trailer):
+   print("Generate %s" % filename)
+   lcitool=subprocess.run(cmd, capture_output=True)
+
+   if lcitool.returncode != 0:
+      raise Exception("Failed to generate %s: %s" % (filename, lcitool.stderr))
+
+   content = lcitool.stdout.decode("utf8")
+   if trailer is not None:
+      content += trailer
+   atomic_write(filename, content)
+
+def generate_dockerfile(host, target, cross=None, trailer=None):
+   filename = Path(src_dir, "tests", "docker", "dockerfiles", host + ".docker")
+   cmd = lcitool_cmd + ["dockerfile"]
+   if cross is not None:
+      cmd.extend(["--cross", cross])
+   cmd.extend([target, "qemu"])
+   generate(filename, cmd, trailer)
+
+try:
+   sys.exit(0)
+except Exception as ex:
+   print(str(ex), file=sys.stderr)
+   sys.exit(1)