From ca056f4499c259c0de68ed7cefad7ee7b62bfa43 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 3 May 2023 12:48:02 +0200 Subject: Python: Drop support for Python 3.7 Debian 10 is not anymore a supported distro, since Debian 12 was released on June 10, 2023. Our supported build platforms as of today all support at least 3.8 (and all of them except for Ubuntu 20.04 support 3.9): openSUSE Leap 15.5: 3.6.15 (3.11.2) CentOS Stream 8: 3.6.8 (3.8.13, 3.9.16, 3.11.4) CentOS Stream 9: 3.9.17 (3.11.4) Fedora 37: 3.11.4 Fedora 38: 3.11.4 Debian 11: 3.9.2 Debian 12: 3.11.2 Alpine 3.14, 3.15: 3.9.16 Alpine 3.16, 3.17: 3.10.10 Ubuntu 20.04 LTS: 3.8.10 Ubuntu 22.04 LTS: 3.10.12 NetBSD 9.3: 3.9.13* FreeBSD 12.4: 3.9.16 FreeBSD 13.1: 3.9.18 OpenBSD 7.2: 3.9.17 Note: NetBSD does not appear to have a default meta-package, but offers several options, the lowest of which is 3.7.15. However, "python39" appears to be a pre-requisite to one of the other packages we request in tests/vm/netbsd. Since it is safe under our supported platform policy, bump our minimum supported version of Python to 3.8. The two most interesting features to have by default include: - the importlib.metadata module, whose lack is responsible for over 100 lines of code in mkvenv.py - improvements to asyncio, for example asyncio.CancelledError inherits from BaseException rather than Exception In addition, code can now use the assignment operator ':=' Because mypy now learns about importlib.metadata, a small change to mkvenv.py is needed to pass type checking. Signed-off-by: Paolo Bonzini --- python/Makefile | 8 ++++---- python/scripts/mkvenv.py | 5 ++++- python/setup.cfg | 7 +++---- python/tests/minreqs.txt | 2 +- 4 files changed, 12 insertions(+), 10 deletions(-) (limited to 'python') diff --git a/python/Makefile b/python/Makefile index 7c70dcc8d1..1fa4ba2498 100644 --- a/python/Makefile +++ b/python/Makefile @@ -9,13 +9,13 @@ help: @echo "make check-minreqs:" @echo " Run tests in the minreqs virtual environment." @echo " These tests use the oldest dependencies." - @echo " Requires: Python 3.7" - @echo " Hint (Fedora): 'sudo dnf install python3.7'" + @echo " Requires: Python 3.8" + @echo " Hint (Fedora): 'sudo dnf install python3.8'" @echo "" @echo "make check-tox:" @echo " Run tests against multiple python versions." @echo " These tests use the newest dependencies." - @echo " Requires: Python 3.7 - 3.11, and tox." + @echo " Requires: Python 3.8 - 3.11, and tox." @echo " Hint (Fedora): 'sudo dnf install python3-tox python3.11'" @echo " The variable QEMU_TOX_EXTRA_ARGS can be use to pass extra" @echo " arguments to tox". @@ -59,7 +59,7 @@ PIP_INSTALL = pip install --disable-pip-version-check min-venv: $(QEMU_MINVENV_DIR) $(QEMU_MINVENV_DIR)/bin/activate $(QEMU_MINVENV_DIR) $(QEMU_MINVENV_DIR)/bin/activate: setup.cfg tests/minreqs.txt @echo "VENV $(QEMU_MINVENV_DIR)" - @python3.7 -m venv $(QEMU_MINVENV_DIR) + @python3.8 -m venv $(QEMU_MINVENV_DIR) @( \ echo "ACTIVATE $(QEMU_MINVENV_DIR)"; \ . $(QEMU_MINVENV_DIR)/bin/activate; \ diff --git a/python/scripts/mkvenv.py b/python/scripts/mkvenv.py index 4f2349fbb6..57a2494803 100644 --- a/python/scripts/mkvenv.py +++ b/python/scripts/mkvenv.py @@ -435,6 +435,7 @@ def _gen_importlib(packages: Sequence[str]) -> Iterator[str]: try: # First preference: Python 3.8+ stdlib from importlib.metadata import ( # type: ignore + EntryPoint, PackageNotFoundError, distribution, ) @@ -442,6 +443,7 @@ def _gen_importlib(packages: Sequence[str]) -> Iterator[str]: logger.debug("%s", str(exc)) # Second preference: Commonly available PyPI backport from importlib_metadata import ( # type: ignore + EntryPoint, PackageNotFoundError, distribution, ) @@ -449,7 +451,8 @@ def _gen_importlib(packages: Sequence[str]) -> Iterator[str]: def _generator() -> Iterator[str]: for package in packages: try: - entry_points = distribution(package).entry_points + entry_points: Iterator[EntryPoint] = \ + iter(distribution(package).entry_points) except PackageNotFoundError: continue diff --git a/python/setup.cfg b/python/setup.cfg index e74b58a8c2..f6d2d8a136 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -14,7 +14,6 @@ classifiers = Natural Language :: English Operating System :: OS Independent Programming Language :: Python :: 3 :: Only - Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 @@ -22,7 +21,7 @@ classifiers = Typing :: Typed [options] -python_requires = >= 3.7 +python_requires = >= 3.8 packages = qemu.qmp qemu.machine @@ -76,7 +75,7 @@ exclude = __pycache__, [mypy] strict = True -python_version = 3.7 +python_version = 3.8 warn_unused_configs = True namespace_packages = True warn_unused_ignores = False @@ -192,7 +191,7 @@ multi_line_output=3 # of python available on your system to run this test. [tox:tox] -envlist = py37, py38, py39, py310, py311 +envlist = py38, py39, py310, py311 skip_missing_interpreters = true [testenv] diff --git a/python/tests/minreqs.txt b/python/tests/minreqs.txt index 979461be6b..a3f423efd8 100644 --- a/python/tests/minreqs.txt +++ b/python/tests/minreqs.txt @@ -1,5 +1,5 @@ # This file lists the ***oldest possible dependencies*** needed to run -# "make check" successfully under ***Python 3.7***. It is used primarily +# "make check" successfully under ***Python 3.8***. It is used primarily # by GitLab CI to ensure that our stated minimum versions in setup.cfg # are truthful and regularly validated. # -- cgit v1.2.3 From 3e4b6b0ad9522bb641deeb9a74e24db7c82b3aca Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 6 Jun 2023 10:20:18 +0200 Subject: mkvenv: assume presence of importlib.metadata importlib.metadata is included in Python 3.8, so there is no need to fallback to either importlib-metadata or pkgresources when generating console script shims. Signed-off-by: Paolo Bonzini --- python/scripts/mkvenv.py | 197 +++++++---------------------------------------- python/setup.cfg | 9 --- 2 files changed, 28 insertions(+), 178 deletions(-) (limited to 'python') diff --git a/python/scripts/mkvenv.py b/python/scripts/mkvenv.py index 57a2494803..6797e12e34 100644 --- a/python/scripts/mkvenv.py +++ b/python/scripts/mkvenv.py @@ -61,7 +61,6 @@ options: """ -# The duplication between importlib and pkg_resources does not help # pylint: disable=too-many-lines # Copyright (C) 2022-2023 Red Hat, Inc. @@ -74,6 +73,13 @@ options: # later. See the COPYING file in the top-level directory. import argparse +from importlib.metadata import ( + Distribution, + EntryPoint, + PackageNotFoundError, + distribution, + version, +) from importlib.util import find_spec import logging import os @@ -428,25 +434,7 @@ def make_venv( # pylint: disable=too-many-arguments print(builder.get_value("env_exe")) -def _gen_importlib(packages: Sequence[str]) -> Iterator[str]: - # pylint: disable=import-outside-toplevel - # pylint: disable=no-name-in-module - # pylint: disable=import-error - try: - # First preference: Python 3.8+ stdlib - from importlib.metadata import ( # type: ignore - EntryPoint, - PackageNotFoundError, - distribution, - ) - except ImportError as exc: - logger.debug("%s", str(exc)) - # Second preference: Commonly available PyPI backport - from importlib_metadata import ( # type: ignore - EntryPoint, - PackageNotFoundError, - distribution, - ) +def _get_entry_points(packages: Sequence[str]) -> Iterator[str]: def _generator() -> Iterator[str]: for package in packages: @@ -468,24 +456,6 @@ def _gen_importlib(packages: Sequence[str]) -> Iterator[str]: return _generator() -def _gen_pkg_resources(packages: Sequence[str]) -> Iterator[str]: - # pylint: disable=import-outside-toplevel - # Bundled with setuptools; has a good chance of being available. - import pkg_resources - - def _generator() -> Iterator[str]: - for package in packages: - try: - eps = pkg_resources.get_entry_map(package, "console_scripts") - except pkg_resources.DistributionNotFound: - continue - - for entry_point in eps.values(): - yield str(entry_point) - - return _generator() - - def generate_console_scripts( packages: Sequence[str], python_path: Optional[str] = None, @@ -510,30 +480,11 @@ def generate_console_scripts( if not packages: return - def _get_entry_points() -> Iterator[str]: - """Python 3.7 compatibility shim for iterating entry points.""" - # Python 3.8+, or Python 3.7 with importlib_metadata installed. - try: - return _gen_importlib(packages) - except ImportError as exc: - logger.debug("%s", str(exc)) - - # Python 3.7 with setuptools installed. - try: - return _gen_pkg_resources(packages) - except ImportError as exc: - logger.debug("%s", str(exc)) - raise Ouch( - "Neither importlib.metadata nor pkg_resources found, " - "can't generate console script shims.\n" - "Use Python 3.8+, or install importlib-metadata or setuptools." - ) from exc - maker = distlib.scripts.ScriptMaker(None, bin_path) maker.variants = {""} maker.clobber = False - for entry_point in _get_entry_points(): + for entry_point in _get_entry_points(packages): for filename in maker.make(entry_point): logger.debug("wrote console_script '%s'", filename) @@ -587,57 +538,6 @@ def pkgname_from_depspec(dep_spec: str) -> str: return match.group(0) -def _get_path_importlib(package: str) -> Optional[str]: - # pylint: disable=import-outside-toplevel - # pylint: disable=no-name-in-module - # pylint: disable=import-error - try: - # First preference: Python 3.8+ stdlib - from importlib.metadata import ( # type: ignore - PackageNotFoundError, - distribution, - ) - except ImportError as exc: - logger.debug("%s", str(exc)) - # Second preference: Commonly available PyPI backport - from importlib_metadata import ( # type: ignore - PackageNotFoundError, - distribution, - ) - - try: - return str(distribution(package).locate_file(".")) - except PackageNotFoundError: - return None - - -def _get_path_pkg_resources(package: str) -> Optional[str]: - # pylint: disable=import-outside-toplevel - # Bundled with setuptools; has a good chance of being available. - import pkg_resources - - try: - return str(pkg_resources.get_distribution(package).location) - except pkg_resources.DistributionNotFound: - return None - - -def _get_path(package: str) -> Optional[str]: - try: - return _get_path_importlib(package) - except ImportError as exc: - logger.debug("%s", str(exc)) - - try: - return _get_path_pkg_resources(package) - except ImportError as exc: - logger.debug("%s", str(exc)) - raise Ouch( - "Neither importlib.metadata nor pkg_resources found. " - "Use Python 3.8+, or install importlib-metadata or setuptools." - ) from exc - - def _path_is_prefix(prefix: Optional[str], path: str) -> bool: try: return ( @@ -647,65 +547,14 @@ def _path_is_prefix(prefix: Optional[str], path: str) -> bool: return False -def _is_system_package(package: str) -> bool: - path = _get_path(package) - return path is not None and not ( +def _is_system_package(dist: Distribution) -> bool: + path = str(dist.locate_file(".")) + return not ( _path_is_prefix(sysconfig.get_path("purelib"), path) or _path_is_prefix(sysconfig.get_path("platlib"), path) ) -def _get_version_importlib(package: str) -> Optional[str]: - # pylint: disable=import-outside-toplevel - # pylint: disable=no-name-in-module - # pylint: disable=import-error - try: - # First preference: Python 3.8+ stdlib - from importlib.metadata import ( # type: ignore - PackageNotFoundError, - distribution, - ) - except ImportError as exc: - logger.debug("%s", str(exc)) - # Second preference: Commonly available PyPI backport - from importlib_metadata import ( # type: ignore - PackageNotFoundError, - distribution, - ) - - try: - return str(distribution(package).version) - except PackageNotFoundError: - return None - - -def _get_version_pkg_resources(package: str) -> Optional[str]: - # pylint: disable=import-outside-toplevel - # Bundled with setuptools; has a good chance of being available. - import pkg_resources - - try: - return str(pkg_resources.get_distribution(package).version) - except pkg_resources.DistributionNotFound: - return None - - -def _get_version(package: str) -> Optional[str]: - try: - return _get_version_importlib(package) - except ImportError as exc: - logger.debug("%s", str(exc)) - - try: - return _get_version_pkg_resources(package) - except ImportError as exc: - logger.debug("%s", str(exc)) - raise Ouch( - "Neither importlib.metadata nor pkg_resources found. " - "Use Python 3.8+, or install importlib-metadata or setuptools." - ) from exc - - def diagnose( dep_spec: str, online: bool, @@ -731,7 +580,11 @@ def diagnose( bad = False pkg_name = pkgname_from_depspec(dep_spec) - pkg_version = _get_version(pkg_name) + pkg_version: Optional[str] = None + try: + pkg_version = version(pkg_name) + except PackageNotFoundError: + pass lines = [] @@ -868,19 +721,25 @@ def _do_ensure( constraint = _make_version_constraint(info, False) matcher = distlib.version.LegacyMatcher(name + constraint) print(f"mkvenv: checking for {matcher}", file=sys.stderr) - ver = _get_version(name) + + dist: Optional[Distribution] = None + try: + dist = distribution(matcher.name) + except PackageNotFoundError: + pass + if ( - ver is None + dist is None # Always pass installed package to pip, so that they can be # updated if the requested version changes - or not _is_system_package(name) - or not matcher.match(distlib.version.LegacyVersion(ver)) + or not _is_system_package(dist) + or not matcher.match(distlib.version.LegacyVersion(dist.version)) ): absent.append(name + _make_version_constraint(info, True)) if len(absent) == 1: canary = info.get("canary", None) else: - logger.info("found %s %s", name, ver) + logger.info("found %s %s", name, dist.version) present.append(name) if present: diff --git a/python/setup.cfg b/python/setup.cfg index f6d2d8a136..8c67dce457 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -108,15 +108,6 @@ ignore_missing_imports = True [mypy-pygments] ignore_missing_imports = True -[mypy-importlib.metadata] -ignore_missing_imports = True - -[mypy-importlib_metadata] -ignore_missing_imports = True - -[mypy-pkg_resources] -ignore_missing_imports = True - [mypy-distlib] ignore_missing_imports = True -- cgit v1.2.3 From 0a88ac9662950cecac74b5de3056071a964e4fc4 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 16 May 2023 12:38:48 +0200 Subject: Revert "mkvenv: work around broken pip installations on Debian 10" Debian 10 has Python 3.7, so it is not possible to use it anymore now that Python 3.8 is required. Signed-off-by: Paolo Bonzini --- python/scripts/mkvenv.py | 74 +++++++++++------------------------------------- 1 file changed, 16 insertions(+), 58 deletions(-) (limited to 'python') diff --git a/python/scripts/mkvenv.py b/python/scripts/mkvenv.py index 6797e12e34..d0b9c215ca 100644 --- a/python/scripts/mkvenv.py +++ b/python/scripts/mkvenv.py @@ -61,8 +61,6 @@ options: """ -# pylint: disable=too-many-lines - # Copyright (C) 2022-2023 Red Hat, Inc. # # Authors: @@ -195,7 +193,7 @@ class QemuEnvBuilder(venv.EnvBuilder): ): kwargs["with_pip"] = False else: - check_ensurepip(suggest_remedy=True) + check_ensurepip() super().__init__(*args, **kwargs) @@ -300,7 +298,7 @@ def need_ensurepip() -> bool: return True -def check_ensurepip(prefix: str = "", suggest_remedy: bool = False) -> None: +def check_ensurepip() -> None: """ Check that we have ensurepip. @@ -311,15 +309,12 @@ def check_ensurepip(prefix: str = "", suggest_remedy: bool = False) -> None: "Python's ensurepip module is not found.\n" "It's normally part of the Python standard library, " "maybe your distribution packages it separately?\n" - "(Debian puts ensurepip in its python3-venv package.)\n" + "Either install ensurepip, or alleviate the need for it in the " + "first place by installing pip and setuptools for " + f"'{sys.executable}'.\n" + "(Hint: Debian puts ensurepip in its python3-venv package.)" ) - if suggest_remedy: - msg += ( - "Either install ensurepip, or alleviate the need for it in the" - " first place by installing pip and setuptools for " - f"'{sys.executable}'.\n" - ) - raise Ouch(prefix + msg) + raise Ouch(msg) # ensurepip uses pyexpat, which can also go missing on us: if not find_spec("pyexpat"): @@ -327,15 +322,12 @@ def check_ensurepip(prefix: str = "", suggest_remedy: bool = False) -> None: "Python's pyexpat module is not found.\n" "It's normally part of the Python standard library, " "maybe your distribution packages it separately?\n" - "(NetBSD's pkgsrc debundles this to e.g. 'py310-expat'.)\n" + "Either install pyexpat, or alleviate the need for it in the " + "first place by installing pip and setuptools for " + f"'{sys.executable}'.\n\n" + "(Hint: NetBSD's pkgsrc debundles this to e.g. 'py310-expat'.)" ) - if suggest_remedy: - msg += ( - "Either install pyexpat, or alleviate the need for it in the " - "first place by installing pip and setuptools for " - f"'{sys.executable}'.\n" - ) - raise Ouch(prefix + msg) + raise Ouch(msg) def make_venv( # pylint: disable=too-many-arguments @@ -489,38 +481,6 @@ def generate_console_scripts( logger.debug("wrote console_script '%s'", filename) -def checkpip() -> bool: - """ - Debian10 has a pip that's broken when used inside of a virtual environment. - - We try to detect and correct that case here. - """ - try: - # pylint: disable=import-outside-toplevel,unused-import,import-error - # pylint: disable=redefined-outer-name - import pip._internal # type: ignore # noqa: F401 - - logger.debug("pip appears to be working correctly.") - return False - except ModuleNotFoundError as exc: - if exc.name == "pip._internal": - # Uh, fair enough. They did say "internal". - # Let's just assume it's fine. - return False - logger.warning("pip appears to be malfunctioning: %s", str(exc)) - - check_ensurepip("pip appears to be non-functional, and ") - - logger.debug("Attempting to repair pip ...") - subprocess.run( - (sys.executable, "-m", "ensurepip"), - stdout=subprocess.DEVNULL, - check=True, - ) - logger.debug("Pip is now (hopefully) repaired!") - return True - - def pkgname_from_depspec(dep_spec: str) -> str: """ Parse package name out of a PEP-508 depspec. @@ -877,12 +837,10 @@ def post_venv_setup() -> None: This is intended to be run *inside the venv* after it is created. """ logger.debug("post_venv_setup()") - # Test for a broken pip (Debian 10 or derivative?) and fix it if needed - if not checkpip(): - # Finally, generate a 'pip' script so the venv is usable in a normal - # way from the CLI. This only happens when we inherited pip from a - # parent/system-site and haven't run ensurepip in some way. - generate_console_scripts(["pip"]) + # Generate a 'pip' script so the venv is usable in a normal + # way from the CLI. This only happens when we inherited pip from a + # parent/system-site and haven't run ensurepip in some way. + generate_console_scripts(["pip"]) def _add_create_subcommand(subparsers: Any) -> None: -- cgit v1.2.3