diff options
| -rw-r--r-- | python/scripts/mkvenv.py | 126 | ||||
| -rw-r--r-- | python/setup.cfg | 6 | ||||
| -rw-r--r-- | pythondeps.toml | 17 |
3 files changed, 148 insertions, 1 deletions
diff --git a/python/scripts/mkvenv.py b/python/scripts/mkvenv.py index 96f506d7e2..02bcd9a8c9 100644 --- a/python/scripts/mkvenv.py +++ b/python/scripts/mkvenv.py @@ -14,6 +14,8 @@ Commands: post_init post-venv initialization ensure Ensure that the specified package is installed. + ensuregroup + Ensure that the specified package group is installed. -------------------------------------------------- @@ -44,6 +46,19 @@ options: --online Install packages from PyPI, if necessary. --dir DIR Path to vendored packages where we may install from. +-------------------------------------------------- + +usage: mkvenv ensuregroup [-h] [--online] [--dir DIR] file group... + +positional arguments: + file pointer to a TOML file + group section name in the TOML file + +options: + -h, --help show this help message and exit + --online Install packages from PyPI, if necessary. + --dir DIR Path to vendored packages where we may install from. + """ # The duplication between importlib and pkg_resources does not help @@ -99,6 +114,18 @@ except ImportError: except ImportError: HAVE_DISTLIB = False +# Try to load tomllib, with a fallback to tomli. +# HAVE_TOMLLIB is checked below, just-in-time, so that mkvenv does not fail +# outside the venv or before a potential call to ensurepip in checkpip(). +HAVE_TOMLLIB = True +try: + import tomllib +except ImportError: + try: + import tomli as tomllib + except ImportError: + HAVE_TOMLLIB = False + # Do not add any mandatory dependencies from outside the stdlib: # This script *must* be usable standalone! @@ -837,6 +864,7 @@ def _do_ensure( for name, info in group.items(): constraint = _make_version_constraint(info, False) matcher = distlib.version.LegacyMatcher(name + constraint) + print(f"mkvenv: checking for {matcher}", file=sys.stderr) ver = _get_version(name) if ( ver is None @@ -898,7 +926,6 @@ def ensure( be presented to the user. e.g., 'sphinx-build' can be used as a bellwether for the presence of 'sphinx'. """ - print(f"mkvenv: checking for {', '.join(dep_specs)}", file=sys.stderr) if not HAVE_DISTLIB: raise Ouch("a usable distlib could not be found, please install it") @@ -928,6 +955,64 @@ def ensure( raise SystemExit(f"\n{result[0]}\n\n") +def _parse_groups(file: str) -> Dict[str, Dict[str, Any]]: + if not HAVE_TOMLLIB: + if sys.version_info < (3, 11): + raise Ouch("found no usable tomli, please install it") + + raise Ouch( + "Python >=3.11 does not have tomllib... what have you done!?" + ) + + try: + # Use loads() to support both tomli v1.2.x (Ubuntu 22.04, + # Debian bullseye-backports) and v2.0.x + with open(file, "r", encoding="ascii") as depfile: + contents = depfile.read() + return tomllib.loads(contents) # type: ignore + except tomllib.TOMLDecodeError as exc: + raise Ouch(f"parsing {file} failed: {exc}") from exc + + +def ensure_group( + file: str, + groups: Sequence[str], + online: bool = False, + wheels_dir: Optional[Union[str, Path]] = None, +) -> None: + """ + Use pip to ensure we have the package specified by @dep_specs. + + If the package is already installed, do nothing. If online and + wheels_dir are both provided, prefer packages found in wheels_dir + first before connecting to PyPI. + + :param dep_specs: + PEP 508 dependency specifications. e.g. ['meson>=0.61.5']. + :param online: If True, fall back to PyPI. + :param wheels_dir: If specified, search this path for packages. + """ + + if not HAVE_DISTLIB: + raise Ouch("found no usable distlib, please install it") + + parsed_deps = _parse_groups(file) + + to_install: Dict[str, Dict[str, str]] = {} + for group in groups: + try: + to_install.update(parsed_deps[group]) + except KeyError as exc: + raise Ouch(f"group {group} not defined") from exc + + result = _do_ensure(to_install, online, wheels_dir) + if result: + # Well, that's not good. + if result[1]: + raise Ouch(result[0]) + raise SystemExit(f"\n{result[0]}\n\n") + + def post_venv_setup() -> None: """ This is intended to be run *inside the venv* after it is created. @@ -955,6 +1040,37 @@ def _add_post_init_subcommand(subparsers: Any) -> None: subparsers.add_parser("post_init", help="post-venv initialization") +def _add_ensuregroup_subcommand(subparsers: Any) -> None: + subparser = subparsers.add_parser( + "ensuregroup", + help="Ensure that the specified package group is installed.", + ) + subparser.add_argument( + "--online", + action="store_true", + help="Install packages from PyPI, if necessary.", + ) + subparser.add_argument( + "--dir", + type=str, + action="store", + help="Path to vendored packages where we may install from.", + ) + subparser.add_argument( + "file", + type=str, + action="store", + help=("Path to a TOML file describing package groups"), + ) + subparser.add_argument( + "group", + type=str, + action="store", + help="One or more package group names", + nargs="+", + ) + + def _add_ensure_subcommand(subparsers: Any) -> None: subparser = subparsers.add_parser( "ensure", help="Ensure that the specified package is installed." @@ -1012,6 +1128,7 @@ def main() -> int: _add_create_subcommand(subparsers) _add_post_init_subcommand(subparsers) _add_ensure_subcommand(subparsers) + _add_ensuregroup_subcommand(subparsers) args = parser.parse_args() try: @@ -1030,6 +1147,13 @@ def main() -> int: wheels_dir=args.dir, prog=args.diagnose, ) + if args.command == "ensuregroup": + ensure_group( + file=args.file, + groups=args.group, + online=args.online, + wheels_dir=args.dir, + ) logger.debug("mkvenv.py %s: exiting", args.command) except Ouch as exc: print("\n*** Ouch! ***\n", file=sys.stderr) diff --git a/python/setup.cfg b/python/setup.cfg index 5d7e95f5d2..e74b58a8c2 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -94,6 +94,12 @@ allow_subclassing_any = True [mypy-fuse] ignore_missing_imports = True +[mypy-tomli] +ignore_missing_imports = True + +[mypy-tomllib] +ignore_missing_imports = True + [mypy-urwid] ignore_missing_imports = True diff --git a/pythondeps.toml b/pythondeps.toml new file mode 100644 index 0000000000..362f63ff2c --- /dev/null +++ b/pythondeps.toml @@ -0,0 +1,17 @@ +# This file describes Python package requirements to be +# installed in the pyvenv Python virtual environment. +# +# Packages are placed in groups, which are installed using +# the ensuregroup subcommand of python/scripts/mkvenv.py. +# Each group forms a TOML section and each entry in the +# section is a TOML key-value list describing a package. +# All fields are optional; valid fields are: +# +# - accepted: accepted versions when using a system package +# - installed: fixed version to install in the virtual environment +# if a system package is not found; if not specified, +# the minimum and maximum +# - canary: if specified, use this program name to present more +# precise error diagnostics to the user. For example, +# 'sphinx-build' can be used as a bellwether for the +# presence of 'sphinx' in the system. |