aboutsummaryrefslogtreecommitdiff
path: root/python/scripts
diff options
context:
space:
mode:
authorPaolo Bonzini <pbonzini@redhat.com>2023-08-08 10:03:42 +0200
committerPaolo Bonzini <pbonzini@redhat.com>2023-08-28 09:55:36 +0200
commit71ed611cd47d961e1897721d7d002ffb498221d4 (patch)
treea38c6821c99269bf242059c30d9ed33cc77bbb5d /python/scripts
parent0f1ec0705b92b79e5e5f69bed236639dae67d312 (diff)
python: mkvenv: add ensuregroup command
Introduce a new subcommand that retrieves the packages to be installed from a TOML file. This allows being more flexible in using the system version of a package, while at the same time using a known-good version when installing the package. This is important for packages that sometimes have backwards-incompatible changes or that depend on specific versions of their dependencies. Compared to JSON, TOML is more human readable and easier to edit. A parser is available in 3.11 but also available as a small (12k) package for older versions, tomli. While tomli is bundled with pip, this is only true of recent versions of pip. Of all the supported OSes pretty much only FreeBSD has a recent enough version of pip while staying on Python <3.11. So we cannot use the same trick that is in place for distlib. Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
Diffstat (limited to 'python/scripts')
-rw-r--r--python/scripts/mkvenv.py126
1 files changed, 125 insertions, 1 deletions
diff --git a/python/scripts/mkvenv.py b/python/scripts/mkvenv.py
index 96f506d7e2..02bcd9a8c9 100644
--- a/python/scripts/mkvenv.py
+++ b/python/scripts/mkvenv.py
@@ -14,6 +14,8 @@ Commands:
post_init
post-venv initialization
ensure Ensure that the specified package is installed.
+ ensuregroup
+ Ensure that the specified package group is installed.
--------------------------------------------------
@@ -44,6 +46,19 @@ options:
--online Install packages from PyPI, if necessary.
--dir DIR Path to vendored packages where we may install from.
+--------------------------------------------------
+
+usage: mkvenv ensuregroup [-h] [--online] [--dir DIR] file group...
+
+positional arguments:
+ file pointer to a TOML file
+ group section name in the TOML file
+
+options:
+ -h, --help show this help message and exit
+ --online Install packages from PyPI, if necessary.
+ --dir DIR Path to vendored packages where we may install from.
+
"""
# The duplication between importlib and pkg_resources does not help
@@ -99,6 +114,18 @@ except ImportError:
except ImportError:
HAVE_DISTLIB = False
+# Try to load tomllib, with a fallback to tomli.
+# HAVE_TOMLLIB is checked below, just-in-time, so that mkvenv does not fail
+# outside the venv or before a potential call to ensurepip in checkpip().
+HAVE_TOMLLIB = True
+try:
+ import tomllib
+except ImportError:
+ try:
+ import tomli as tomllib
+ except ImportError:
+ HAVE_TOMLLIB = False
+
# Do not add any mandatory dependencies from outside the stdlib:
# This script *must* be usable standalone!
@@ -837,6 +864,7 @@ def _do_ensure(
for name, info in group.items():
constraint = _make_version_constraint(info, False)
matcher = distlib.version.LegacyMatcher(name + constraint)
+ print(f"mkvenv: checking for {matcher}", file=sys.stderr)
ver = _get_version(name)
if (
ver is None
@@ -898,7 +926,6 @@ def ensure(
be presented to the user. e.g., 'sphinx-build' can be used as a
bellwether for the presence of 'sphinx'.
"""
- print(f"mkvenv: checking for {', '.join(dep_specs)}", file=sys.stderr)
if not HAVE_DISTLIB:
raise Ouch("a usable distlib could not be found, please install it")
@@ -928,6 +955,64 @@ def ensure(
raise SystemExit(f"\n{result[0]}\n\n")
+def _parse_groups(file: str) -> Dict[str, Dict[str, Any]]:
+ if not HAVE_TOMLLIB:
+ if sys.version_info < (3, 11):
+ raise Ouch("found no usable tomli, please install it")
+
+ raise Ouch(
+ "Python >=3.11 does not have tomllib... what have you done!?"
+ )
+
+ try:
+ # Use loads() to support both tomli v1.2.x (Ubuntu 22.04,
+ # Debian bullseye-backports) and v2.0.x
+ with open(file, "r", encoding="ascii") as depfile:
+ contents = depfile.read()
+ return tomllib.loads(contents) # type: ignore
+ except tomllib.TOMLDecodeError as exc:
+ raise Ouch(f"parsing {file} failed: {exc}") from exc
+
+
+def ensure_group(
+ file: str,
+ groups: Sequence[str],
+ online: bool = False,
+ wheels_dir: Optional[Union[str, Path]] = None,
+) -> None:
+ """
+ Use pip to ensure we have the package specified by @dep_specs.
+
+ If the package is already installed, do nothing. If online and
+ wheels_dir are both provided, prefer packages found in wheels_dir
+ first before connecting to PyPI.
+
+ :param dep_specs:
+ PEP 508 dependency specifications. e.g. ['meson>=0.61.5'].
+ :param online: If True, fall back to PyPI.
+ :param wheels_dir: If specified, search this path for packages.
+ """
+
+ if not HAVE_DISTLIB:
+ raise Ouch("found no usable distlib, please install it")
+
+ parsed_deps = _parse_groups(file)
+
+ to_install: Dict[str, Dict[str, str]] = {}
+ for group in groups:
+ try:
+ to_install.update(parsed_deps[group])
+ except KeyError as exc:
+ raise Ouch(f"group {group} not defined") from exc
+
+ result = _do_ensure(to_install, online, wheels_dir)
+ if result:
+ # Well, that's not good.
+ if result[1]:
+ raise Ouch(result[0])
+ raise SystemExit(f"\n{result[0]}\n\n")
+
+
def post_venv_setup() -> None:
"""
This is intended to be run *inside the venv* after it is created.
@@ -955,6 +1040,37 @@ def _add_post_init_subcommand(subparsers: Any) -> None:
subparsers.add_parser("post_init", help="post-venv initialization")
+def _add_ensuregroup_subcommand(subparsers: Any) -> None:
+ subparser = subparsers.add_parser(
+ "ensuregroup",
+ help="Ensure that the specified package group is installed.",
+ )
+ subparser.add_argument(
+ "--online",
+ action="store_true",
+ help="Install packages from PyPI, if necessary.",
+ )
+ subparser.add_argument(
+ "--dir",
+ type=str,
+ action="store",
+ help="Path to vendored packages where we may install from.",
+ )
+ subparser.add_argument(
+ "file",
+ type=str,
+ action="store",
+ help=("Path to a TOML file describing package groups"),
+ )
+ subparser.add_argument(
+ "group",
+ type=str,
+ action="store",
+ help="One or more package group names",
+ nargs="+",
+ )
+
+
def _add_ensure_subcommand(subparsers: Any) -> None:
subparser = subparsers.add_parser(
"ensure", help="Ensure that the specified package is installed."
@@ -1012,6 +1128,7 @@ def main() -> int:
_add_create_subcommand(subparsers)
_add_post_init_subcommand(subparsers)
_add_ensure_subcommand(subparsers)
+ _add_ensuregroup_subcommand(subparsers)
args = parser.parse_args()
try:
@@ -1030,6 +1147,13 @@ def main() -> int:
wheels_dir=args.dir,
prog=args.diagnose,
)
+ if args.command == "ensuregroup":
+ ensure_group(
+ file=args.file,
+ groups=args.group,
+ online=args.online,
+ wheels_dir=args.dir,
+ )
logger.debug("mkvenv.py %s: exiting", args.command)
except Ouch as exc:
print("\n*** Ouch! ***\n", file=sys.stderr)