diff options
Diffstat (limited to 'python')
| -rw-r--r-- | python/Makefile | 8 | ||||
| -rw-r--r-- | python/scripts/mkvenv.py | 272 | ||||
| -rw-r--r-- | python/setup.cfg | 16 | ||||
| -rw-r--r-- | python/tests/minreqs.txt | 2 |
4 files changed, 54 insertions, 244 deletions
diff --git a/python/Makefile b/python/Makefile index 7c70dcc8d1..1fa4ba2498 100644 --- a/python/Makefile +++ b/python/Makefile @@ -9,13 +9,13 @@ help: @echo "make check-minreqs:" @echo " Run tests in the minreqs virtual environment." @echo " These tests use the oldest dependencies." - @echo " Requires: Python 3.7" - @echo " Hint (Fedora): 'sudo dnf install python3.7'" + @echo " Requires: Python 3.8" + @echo " Hint (Fedora): 'sudo dnf install python3.8'" @echo "" @echo "make check-tox:" @echo " Run tests against multiple python versions." @echo " These tests use the newest dependencies." - @echo " Requires: Python 3.7 - 3.11, and tox." + @echo " Requires: Python 3.8 - 3.11, and tox." @echo " Hint (Fedora): 'sudo dnf install python3-tox python3.11'" @echo " The variable QEMU_TOX_EXTRA_ARGS can be use to pass extra" @echo " arguments to tox". @@ -59,7 +59,7 @@ PIP_INSTALL = pip install --disable-pip-version-check min-venv: $(QEMU_MINVENV_DIR) $(QEMU_MINVENV_DIR)/bin/activate $(QEMU_MINVENV_DIR) $(QEMU_MINVENV_DIR)/bin/activate: setup.cfg tests/minreqs.txt @echo "VENV $(QEMU_MINVENV_DIR)" - @python3.7 -m venv $(QEMU_MINVENV_DIR) + @python3.8 -m venv $(QEMU_MINVENV_DIR) @( \ echo "ACTIVATE $(QEMU_MINVENV_DIR)"; \ . $(QEMU_MINVENV_DIR)/bin/activate; \ diff --git a/python/scripts/mkvenv.py b/python/scripts/mkvenv.py index 4f2349fbb6..d0b9c215ca 100644 --- a/python/scripts/mkvenv.py +++ b/python/scripts/mkvenv.py @@ -61,9 +61,6 @@ options: """ -# The duplication between importlib and pkg_resources does not help -# pylint: disable=too-many-lines - # Copyright (C) 2022-2023 Red Hat, Inc. # # Authors: @@ -74,6 +71,13 @@ options: # later. See the COPYING file in the top-level directory. import argparse +from importlib.metadata import ( + Distribution, + EntryPoint, + PackageNotFoundError, + distribution, + version, +) from importlib.util import find_spec import logging import os @@ -189,7 +193,7 @@ class QemuEnvBuilder(venv.EnvBuilder): ): kwargs["with_pip"] = False else: - check_ensurepip(suggest_remedy=True) + check_ensurepip() super().__init__(*args, **kwargs) @@ -294,7 +298,7 @@ def need_ensurepip() -> bool: return True -def check_ensurepip(prefix: str = "", suggest_remedy: bool = False) -> None: +def check_ensurepip() -> None: """ Check that we have ensurepip. @@ -305,15 +309,12 @@ def check_ensurepip(prefix: str = "", suggest_remedy: bool = False) -> None: "Python's ensurepip module is not found.\n" "It's normally part of the Python standard library, " "maybe your distribution packages it separately?\n" - "(Debian puts ensurepip in its python3-venv package.)\n" + "Either install ensurepip, or alleviate the need for it in the " + "first place by installing pip and setuptools for " + f"'{sys.executable}'.\n" + "(Hint: Debian puts ensurepip in its python3-venv package.)" ) - if suggest_remedy: - msg += ( - "Either install ensurepip, or alleviate the need for it in the" - " first place by installing pip and setuptools for " - f"'{sys.executable}'.\n" - ) - raise Ouch(prefix + msg) + raise Ouch(msg) # ensurepip uses pyexpat, which can also go missing on us: if not find_spec("pyexpat"): @@ -321,15 +322,12 @@ def check_ensurepip(prefix: str = "", suggest_remedy: bool = False) -> None: "Python's pyexpat module is not found.\n" "It's normally part of the Python standard library, " "maybe your distribution packages it separately?\n" - "(NetBSD's pkgsrc debundles this to e.g. 'py310-expat'.)\n" + "Either install pyexpat, or alleviate the need for it in the " + "first place by installing pip and setuptools for " + f"'{sys.executable}'.\n\n" + "(Hint: NetBSD's pkgsrc debundles this to e.g. 'py310-expat'.)" ) - if suggest_remedy: - msg += ( - "Either install pyexpat, or alleviate the need for it in the " - "first place by installing pip and setuptools for " - f"'{sys.executable}'.\n" - ) - raise Ouch(prefix + msg) + raise Ouch(msg) def make_venv( # pylint: disable=too-many-arguments @@ -428,28 +426,13 @@ def make_venv( # pylint: disable=too-many-arguments print(builder.get_value("env_exe")) -def _gen_importlib(packages: Sequence[str]) -> Iterator[str]: - # pylint: disable=import-outside-toplevel - # pylint: disable=no-name-in-module - # pylint: disable=import-error - try: - # First preference: Python 3.8+ stdlib - from importlib.metadata import ( # type: ignore - PackageNotFoundError, - distribution, - ) - except ImportError as exc: - logger.debug("%s", str(exc)) - # Second preference: Commonly available PyPI backport - from importlib_metadata import ( # type: ignore - PackageNotFoundError, - distribution, - ) +def _get_entry_points(packages: Sequence[str]) -> Iterator[str]: def _generator() -> Iterator[str]: for package in packages: try: - entry_points = distribution(package).entry_points + entry_points: Iterator[EntryPoint] = \ + iter(distribution(package).entry_points) except PackageNotFoundError: continue @@ -465,24 +448,6 @@ def _gen_importlib(packages: Sequence[str]) -> Iterator[str]: return _generator() -def _gen_pkg_resources(packages: Sequence[str]) -> Iterator[str]: - # pylint: disable=import-outside-toplevel - # Bundled with setuptools; has a good chance of being available. - import pkg_resources - - def _generator() -> Iterator[str]: - for package in packages: - try: - eps = pkg_resources.get_entry_map(package, "console_scripts") - except pkg_resources.DistributionNotFound: - continue - - for entry_point in eps.values(): - yield str(entry_point) - - return _generator() - - def generate_console_scripts( packages: Sequence[str], python_path: Optional[str] = None, @@ -507,66 +472,15 @@ def generate_console_scripts( if not packages: return - def _get_entry_points() -> Iterator[str]: - """Python 3.7 compatibility shim for iterating entry points.""" - # Python 3.8+, or Python 3.7 with importlib_metadata installed. - try: - return _gen_importlib(packages) - except ImportError as exc: - logger.debug("%s", str(exc)) - - # Python 3.7 with setuptools installed. - try: - return _gen_pkg_resources(packages) - except ImportError as exc: - logger.debug("%s", str(exc)) - raise Ouch( - "Neither importlib.metadata nor pkg_resources found, " - "can't generate console script shims.\n" - "Use Python 3.8+, or install importlib-metadata or setuptools." - ) from exc - maker = distlib.scripts.ScriptMaker(None, bin_path) maker.variants = {""} maker.clobber = False - for entry_point in _get_entry_points(): + for entry_point in _get_entry_points(packages): for filename in maker.make(entry_point): logger.debug("wrote console_script '%s'", filename) -def checkpip() -> bool: - """ - Debian10 has a pip that's broken when used inside of a virtual environment. - - We try to detect and correct that case here. - """ - try: - # pylint: disable=import-outside-toplevel,unused-import,import-error - # pylint: disable=redefined-outer-name - import pip._internal # type: ignore # noqa: F401 - - logger.debug("pip appears to be working correctly.") - return False - except ModuleNotFoundError as exc: - if exc.name == "pip._internal": - # Uh, fair enough. They did say "internal". - # Let's just assume it's fine. - return False - logger.warning("pip appears to be malfunctioning: %s", str(exc)) - - check_ensurepip("pip appears to be non-functional, and ") - - logger.debug("Attempting to repair pip ...") - subprocess.run( - (sys.executable, "-m", "ensurepip"), - stdout=subprocess.DEVNULL, - check=True, - ) - logger.debug("Pip is now (hopefully) repaired!") - return True - - def pkgname_from_depspec(dep_spec: str) -> str: """ Parse package name out of a PEP-508 depspec. @@ -584,57 +498,6 @@ def pkgname_from_depspec(dep_spec: str) -> str: return match.group(0) -def _get_path_importlib(package: str) -> Optional[str]: - # pylint: disable=import-outside-toplevel - # pylint: disable=no-name-in-module - # pylint: disable=import-error - try: - # First preference: Python 3.8+ stdlib - from importlib.metadata import ( # type: ignore - PackageNotFoundError, - distribution, - ) - except ImportError as exc: - logger.debug("%s", str(exc)) - # Second preference: Commonly available PyPI backport - from importlib_metadata import ( # type: ignore - PackageNotFoundError, - distribution, - ) - - try: - return str(distribution(package).locate_file(".")) - except PackageNotFoundError: - return None - - -def _get_path_pkg_resources(package: str) -> Optional[str]: - # pylint: disable=import-outside-toplevel - # Bundled with setuptools; has a good chance of being available. - import pkg_resources - - try: - return str(pkg_resources.get_distribution(package).location) - except pkg_resources.DistributionNotFound: - return None - - -def _get_path(package: str) -> Optional[str]: - try: - return _get_path_importlib(package) - except ImportError as exc: - logger.debug("%s", str(exc)) - - try: - return _get_path_pkg_resources(package) - except ImportError as exc: - logger.debug("%s", str(exc)) - raise Ouch( - "Neither importlib.metadata nor pkg_resources found. " - "Use Python 3.8+, or install importlib-metadata or setuptools." - ) from exc - - def _path_is_prefix(prefix: Optional[str], path: str) -> bool: try: return ( @@ -644,65 +507,14 @@ def _path_is_prefix(prefix: Optional[str], path: str) -> bool: return False -def _is_system_package(package: str) -> bool: - path = _get_path(package) - return path is not None and not ( +def _is_system_package(dist: Distribution) -> bool: + path = str(dist.locate_file(".")) + return not ( _path_is_prefix(sysconfig.get_path("purelib"), path) or _path_is_prefix(sysconfig.get_path("platlib"), path) ) -def _get_version_importlib(package: str) -> Optional[str]: - # pylint: disable=import-outside-toplevel - # pylint: disable=no-name-in-module - # pylint: disable=import-error - try: - # First preference: Python 3.8+ stdlib - from importlib.metadata import ( # type: ignore - PackageNotFoundError, - distribution, - ) - except ImportError as exc: - logger.debug("%s", str(exc)) - # Second preference: Commonly available PyPI backport - from importlib_metadata import ( # type: ignore - PackageNotFoundError, - distribution, - ) - - try: - return str(distribution(package).version) - except PackageNotFoundError: - return None - - -def _get_version_pkg_resources(package: str) -> Optional[str]: - # pylint: disable=import-outside-toplevel - # Bundled with setuptools; has a good chance of being available. - import pkg_resources - - try: - return str(pkg_resources.get_distribution(package).version) - except pkg_resources.DistributionNotFound: - return None - - -def _get_version(package: str) -> Optional[str]: - try: - return _get_version_importlib(package) - except ImportError as exc: - logger.debug("%s", str(exc)) - - try: - return _get_version_pkg_resources(package) - except ImportError as exc: - logger.debug("%s", str(exc)) - raise Ouch( - "Neither importlib.metadata nor pkg_resources found. " - "Use Python 3.8+, or install importlib-metadata or setuptools." - ) from exc - - def diagnose( dep_spec: str, online: bool, @@ -728,7 +540,11 @@ def diagnose( bad = False pkg_name = pkgname_from_depspec(dep_spec) - pkg_version = _get_version(pkg_name) + pkg_version: Optional[str] = None + try: + pkg_version = version(pkg_name) + except PackageNotFoundError: + pass lines = [] @@ -865,19 +681,25 @@ def _do_ensure( constraint = _make_version_constraint(info, False) matcher = distlib.version.LegacyMatcher(name + constraint) print(f"mkvenv: checking for {matcher}", file=sys.stderr) - ver = _get_version(name) + + dist: Optional[Distribution] = None + try: + dist = distribution(matcher.name) + except PackageNotFoundError: + pass + if ( - ver is None + dist is None # Always pass installed package to pip, so that they can be # updated if the requested version changes - or not _is_system_package(name) - or not matcher.match(distlib.version.LegacyVersion(ver)) + or not _is_system_package(dist) + or not matcher.match(distlib.version.LegacyVersion(dist.version)) ): absent.append(name + _make_version_constraint(info, True)) if len(absent) == 1: canary = info.get("canary", None) else: - logger.info("found %s %s", name, ver) + logger.info("found %s %s", name, dist.version) present.append(name) if present: @@ -1015,12 +837,10 @@ def post_venv_setup() -> None: This is intended to be run *inside the venv* after it is created. """ logger.debug("post_venv_setup()") - # Test for a broken pip (Debian 10 or derivative?) and fix it if needed - if not checkpip(): - # Finally, generate a 'pip' script so the venv is usable in a normal - # way from the CLI. This only happens when we inherited pip from a - # parent/system-site and haven't run ensurepip in some way. - generate_console_scripts(["pip"]) + # Generate a 'pip' script so the venv is usable in a normal + # way from the CLI. This only happens when we inherited pip from a + # parent/system-site and haven't run ensurepip in some way. + generate_console_scripts(["pip"]) def _add_create_subcommand(subparsers: Any) -> None: diff --git a/python/setup.cfg b/python/setup.cfg index e74b58a8c2..8c67dce457 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -14,7 +14,6 @@ classifiers = Natural Language :: English Operating System :: OS Independent Programming Language :: Python :: 3 :: Only - Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 @@ -22,7 +21,7 @@ classifiers = Typing :: Typed [options] -python_requires = >= 3.7 +python_requires = >= 3.8 packages = qemu.qmp qemu.machine @@ -76,7 +75,7 @@ exclude = __pycache__, [mypy] strict = True -python_version = 3.7 +python_version = 3.8 warn_unused_configs = True namespace_packages = True warn_unused_ignores = False @@ -109,15 +108,6 @@ ignore_missing_imports = True [mypy-pygments] ignore_missing_imports = True -[mypy-importlib.metadata] -ignore_missing_imports = True - -[mypy-importlib_metadata] -ignore_missing_imports = True - -[mypy-pkg_resources] -ignore_missing_imports = True - [mypy-distlib] ignore_missing_imports = True @@ -192,7 +182,7 @@ multi_line_output=3 # of python available on your system to run this test. [tox:tox] -envlist = py37, py38, py39, py310, py311 +envlist = py38, py39, py310, py311 skip_missing_interpreters = true [testenv] diff --git a/python/tests/minreqs.txt b/python/tests/minreqs.txt index 979461be6b..a3f423efd8 100644 --- a/python/tests/minreqs.txt +++ b/python/tests/minreqs.txt @@ -1,5 +1,5 @@ # This file lists the ***oldest possible dependencies*** needed to run -# "make check" successfully under ***Python 3.7***. It is used primarily +# "make check" successfully under ***Python 3.8***. It is used primarily # by GitLab CI to ensure that our stated minimum versions in setup.cfg # are truthful and regularly validated. # |