Merge pull request #18240 from fuzzard/depends-python3.8

[tools/depends] Python 3.8.5

18 files changed, 144 insertions, 183 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 3c01a9479a..2ccaaaf098 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -172,7 +172,7 @@ find_package(JsonSchemaBuilder REQUIRED)
 core_optional_dep(${optional_deps})
 core_require_dep(${required_deps})
 
-SET(PYTHON_VERSION 3.7)
+SET(PYTHON_VERSION 3.8)
 
 if(ENABLE_MARIADBCLIENT AND NOT ENABLE_MARIADBCLIENT STREQUAL AUTO AND ENABLE_MYSQLCLIENT AND NOT ENABLE_MYSQLCLIENT STREQUAL AUTO)
   MESSAGE(FATAL_ERROR "You can not use MySql and MariaDB at the same time. Disable one by adding -DENABLE_MYSQLCLIENT=OFF or -DENABLE_MARIADBCLIENT=OFF.")
diff --git a/tools/depends/native/python3/Makefile b/tools/depends/native/python3/Makefile
index 612ec89552..86931e4087 100644
--- a/tools/depends/native/python3/Makefile
+++ b/tools/depends/native/python3/Makefile
@@ -4,7 +4,7 @@ DEPS= ../../Makefile.include Makefile
 
 # lib name, version
 LIBNAME=Python
-VERSION=3.7.4
+VERSION=3.8.5
 SOURCE=$(LIBNAME)-$(VERSION)
 ARCHIVE=$(SOURCE).tar.xz
 CWD=$(shell pwd)
@@ -18,6 +18,7 @@ endif
 
 LIBDYLIB=$(PLATFORM)/python
 ifeq ($(NATIVE_OS), osx)
+    # Case insensitive FS will add .exe to buildexe regardless of --with-suffix=no
 	LIBDYLIB=$(PLATFORM)/python.exe
 endif
 
@@ -32,12 +33,10 @@ $(PLATFORM): $(TARBALLS_LOCATION)/$(ARCHIVE) $(DEPS)
 	cd $(PLATFORM); $(CONFIGURE)
 
 $(LIBDYLIB): $(PLATFORM)
-	cd $(PLATFORM); $(MAKE) regen-grammar
 	cd $(PLATFORM); $(MAKE)
 
 .installed-$(PLATFORM): $(LIBDYLIB)
 	cd $(PLATFORM); $(MAKE) install
-	install $(PLATFORM)/Parser/pgen $(NATIVEPREFIX)/bin
 	touch $(LIBDYLIB)
 	touch $@
 
diff --git a/tools/depends/target/config.site.in b/tools/depends/target/config.site.in
index 0dd36976fe..142624fcaa 100644
--- a/tools/depends/target/config.site.in
+++ b/tools/depends/target/config.site.in
@@ -34,7 +34,7 @@ LIBGCRYPT_CONFIG=@prefix@/@deps_dir@/bin/libgcrypt-config
 PATH=@prefix@/@tool_dir@/bin:@use_toolchain@/usr/bin:@use_toolchain@/bin:$PATH
 LD_LIBRARY_PATH=@prefix@/@tool_dir@/lib:$LD_LIBRARY_PATH
 
-PYTHON_VERSION=3.7
+PYTHON_VERSION=3.8
 PYTHON_LDFLAGS="-L@prefix@/@deps_dir@/lib -lpython${PYTHON_VERSION} -lc -ldl -lm -lexpat -lffi -lintl -lutil -lssl -lcrypto"
 PYTHON_CPPFLAGS=-I@prefix@/@deps_dir@/include/python${PYTHON_VERSION}
 PYTHON_SITE_PKG=@prefix@/@deps_dir@/lib/python${PYTHON_VERSION}/site-packages
diff --git a/tools/depends/target/expat/Makefile b/tools/depends/target/expat/Makefile
index 6681743df8..7e3ca40294 100644
--- a/tools/depends/target/expat/Makefile
+++ b/tools/depends/target/expat/Makefile
@@ -8,7 +8,7 @@ SOURCE=$(LIBNAME)-$(VERSION)
 ARCHIVE=$(SOURCE).tar.bz2
 
 # configuration settings
-CONFIGURE=cp -f $(CONFIG_SUB) $(CONFIG_GUESS) ./conftools; ./configure --prefix=$(PREFIX) --disable-shared --without-examples --without-tests --without-docbook
+CONFIGURE=cp -f $(CONFIG_SUB) $(CONFIG_GUESS) ./conftools; CFLAGS="-D_INTERNAL_trim_to_complete_utf8_characters=expat_INTERNAL_trim_to_complete_utf8_characters" ./configure --prefix=$(PREFIX) --disable-shared --without-examples --without-tests --without-docbook
 
 LIBDYLIB=$(PLATFORM)/lib/.libs/lib$(LIBNAME).a
 
diff --git a/tools/depends/target/python3/Makefile b/tools/depends/target/python3/Makefile
index 0772b08cc7..d3720b8163 100644
--- a/tools/depends/target/python3/Makefile
+++ b/tools/depends/target/python3/Makefile
@@ -4,16 +4,12 @@ DEPS= ../../Makefile.include Makefile apple.patch crosscompile.patch android.pat
 
 # lib name, version
 LIBNAME=Python
-VERSION=3.7.8
+VERSION=3.8.5
 SOURCE=$(LIBNAME)-$(VERSION)
 ARCHIVE=$(SOURCE).tar.xz
 BASE_URL=https://www.python.org/ftp/python/$(VERSION)
 
-ifeq ($(OS),osx)
-  HOSTPLATFORM=_PYTHON_HOST_PLATFORM="darwin"
-  LINK_ICONV=-framework CoreFoundation -liconv
-endif
-ifeq ($(OS),darwin_embedded)
+ifeq ($(findstring apple-darwin, $(HOST)), apple-darwin)
   HOSTPLATFORM=_PYTHON_HOST_PLATFORM="darwin"
   LINK_ICONV=-framework CoreFoundation -liconv
 endif
@@ -24,7 +20,8 @@ CONFIGURE=./configure --prefix=$(PREFIX) \
   --without-ensurepip \
   --disable-framework --with-system-expat --with-system-ffi --without-pymalloc --enable-ipv6
 
-LIBDYLIB=$(PLATFORM)/libpython3.7.a
+
+LIBDYLIB=$(PLATFORM)/libpython3.8.a
 
 all: .installed-$(PLATFORM)
 
@@ -40,29 +37,29 @@ $(PLATFORM): $(TARBALLS_LOCATION)/$(ARCHIVE) $(DEPS)
 ifeq ($(OS),darwin_embedded)
 	cd $(PLATFORM); patch -p1 -i ../darwin_embedded.patch
 endif
-	cp modules.setup $(PLATFORM)/Modules/Setup.dist
+	cp modules.setup $(PLATFORM)/Modules/Setup
 
 #Add -liconv as needed, and add the _scproxy module for darwin
 #  disable locale altogether for Android
 ifeq ($(OS),android)
-	cd $(PLATFORM); sed -ie 's|_locale _localemodule.c   -lintl|#_locale _localemodule.c   -lintl|' Modules/Setup.dist
+	cd $(PLATFORM); sed -ie 's|_locale -DPy_BUILD_CORE_BUILTIN _localemodule.c  # -lintl|#_locale -DPy_BUILD_CORE_BUILTIN _localemodule.c  # -lintl|' Modules/Setup
 else
-	cd $(PLATFORM); sed -ie 's|_locale _localemodule.c   -lintl|_locale _localemodule.c   -lintl $(LINK_ICONV)|' Modules/Setup.dist
+	cd $(PLATFORM); sed -ie 's|_locale -DPy_BUILD_CORE_BUILTIN _localemodule.c  # -lintl|_locale -DPy_BUILD_CORE_BUILTIN _localemodule.c   -lintl $(LINK_ICONV)|' Modules/Setup
 endif
 ifeq ($(OS),osx)
-	echo "_scproxy \$$(srcdir)/Modules/_scproxy.c -framework SystemConfiguration -framework CoreFoundation" >> $(PLATFORM)/Modules/Setup.dist
+	echo "_scproxy \$$(srcdir)/Modules/_scproxy.c -framework SystemConfiguration -framework CoreFoundation" >> $(PLATFORM)/Modules/Setup
 endif
 
 	cd $(PLATFORM); $(AUTORECONF)
 	cd $(PLATFORM); $(CONFIGURE)
 
 $(LIBDYLIB): $(PLATFORM)
-	$(MAKE) -C $(PLATFORM) $(HOSTPLATFORM) PYTHON_FOR_BUILD=$(NATIVEPREFIX)/bin/python3 NATIVE_PGEN=$(NATIVEPREFIX)/bin/pgen CROSS_COMPILE_TARGET=yes libpython3.7.a
+	$(MAKE) -C $(PLATFORM) $(HOSTPLATFORM) PYTHON_FOR_BUILD=$(NATIVEPREFIX)/bin/python3 CROSS_COMPILE_TARGET=yes libpython3.8.a
 	touch $@
 
 .installed-$(PLATFORM): $(LIBDYLIB)
-	$(MAKE) -C $(PLATFORM) $(HOSTPLATFORM) PYTHON_FOR_BUILD=$(NATIVEPREFIX)/bin/python3 NATIVE_PGEN=$(NATIVEPREFIX)/bin/pgen CROSS_COMPILE_TARGET=yes install
-	find $(PREFIX)/lib/python3.7 -type f -name "*.pyc" -delete
+	$(MAKE) -C $(PLATFORM) $(HOSTPLATFORM) PYTHON_FOR_BUILD=$(NATIVEPREFIX)/bin/python3 CROSS_COMPILE_TARGET=yes install
+	find $(PREFIX)/lib/python3.8 -type f -name "*.pyc" -delete
 	touch $(LIBDYLIB)
 	touch $@
 
diff --git a/tools/depends/target/python3/apple.patch b/tools/depends/target/python3/apple.patch
index 816cef40fb..8d63246eed 100644
--- a/tools/depends/target/python3/apple.patch
+++ b/tools/depends/target/python3/apple.patch
@@ -1,38 +1,30 @@
 --- a/configure.ac
 +++ b/configure.ac
-@@ -379,10 +379,12 @@
+@@ -379,6 +379,9 @@
  	*-*-cygwin*)
  		ac_sys_system=Cygwin
  		;;
 +	*-*-darwin*)
 +		ac_sys_system=Darwin
 +		;;
- 	*)
- 		# for now, limit cross builds to known configurations
- 		MACHDEP="unknown"
--		AC_MSG_ERROR([cross build not supported for $host])
- 	esac
- 	ac_sys_release=
-     else
-@@ -423,10 +425,11 @@
+ 	*-*-vxworks*)
+ 	    ac_sys_system=VxWorks
+ 	    ;;
+@@ -428,6 +431,8 @@
  	*-*-cygwin*)
  		_host_cpu=
  		;;
 +	*-*-darwin*)
 +		;;
- 	*)
- 		# for now, limit cross builds to known configurations
- 		MACHDEP="unknown"
--		AC_MSG_ERROR([cross build not supported for $host])
- 	esac
- 	_PYTHON_HOST_PLATFORM="$MACHDEP${_host_cpu:+-$_host_cpu}"
- fi
-@@ -492,6 +492,8 @@
+ 	*-*-vxworks*)
+ 		_host_cpu=$host_cpu
+ 		;;
+@@ -500,6 +505,8 @@
      define_xopen_source=no;;
    Darwin/@<:@[12]@:>@@<:@0-9@:>@.*)
      define_xopen_source=no;;
 +  Darwin/*)
-+    define_xopen_source=no;;  
++    define_xopen_source=no;;
    # On AIX 4 and 5.1, mbstate_t is defined only when _XOPEN_SOURCE == 500 but
    # used in wcsnrtombs() and mbsnrtowcs() even if _XOPEN_SOURCE is not defined
    # or has another value. By not (re)defining it, the defaults come in place.
diff --git a/tools/depends/target/python3/crosscompile.patch b/tools/depends/target/python3/crosscompile.patch
index 46e212f1f2..a65296ceb6 100644
--- a/tools/depends/target/python3/crosscompile.patch
+++ b/tools/depends/target/python3/crosscompile.patch
@@ -1,3 +1,21 @@
+--- a/configure.ac
++++ b/configure.ac
+@@ -1184,15 +1184,6 @@
+         ARFLAGS="rcs"
+ fi
+ 
+-AC_CHECK_TOOLS([READELF], [readelf], [:])
+-if test "$cross_compiling" = yes; then
+-    case "$READELF" in
+-	readelf|:)
+-	AC_MSG_ERROR([readelf for the host is required for cross builds])
+-	;;
+-    esac
+-fi
+-AC_SUBST(READELF)
+ 
+ 
+ case $MACHDEP in
 --- a/Makefile.pre.in
 +++ b/Makefile.pre.in
 @@ -38,7 +38,6 @@
@@ -7,17 +25,8 @@
 -READELF=	@READELF@
  SOABI=		@SOABI@
  LDVERSION=	@LDVERSION@
- GITVERSION=	@GITVERSION@
-@@ -295,7 +294,7 @@
- ##########################################################################
- # Parser
- PGEN=		Parser/pgen$(EXE)
--
-+PGEN_FOR_BUILD=$(NATIVE_PGEN)
- POBJS=		\
- 		Parser/acceler.o \
- 		Parser/grammar1.o \
-@@ -623,7 +623,7 @@
+ LIBPYTHON=	@LIBPYTHON@
+@@ -608,7 +607,7 @@
  		$(PYTHON_FOR_BUILD) $(srcdir)/setup.py $$quiet build"; \
  	$(RUNSHARED) CC='$(CC)' LDSHARED='$(BLDSHARED)' OPT='$(OPT)' \
  		_TCLTK_INCLUDES='$(TCLTK_INCLUDES)' _TCLTK_LIBS='$(TCLTK_LIBS)' \
@@ -26,13 +35,11 @@
  
  
  # Build static library
-@@ -1535,11 +1535,13 @@
- # Install the dynamically loadable modules
+@@ -1600,10 +1599,11 @@
  # This goes into $(exec_prefix)
  sharedinstall: sharedmods
-+	CC='$(CC)' LDSHARED='$(BLDSHARED)' OPT='$(OPT)' \
  	$(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/setup.py install \
-+	   	--skip-build \
++ 	    --skip-build \
  	   	--prefix=$(prefix) \
 -		--install-scripts=$(BINDIR) \
 -		--install-platlib=$(DESTSHARED) \
@@ -43,50 +50,31 @@
  	-rm $(DESTDIR)$(DESTSHARED)/_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).py
  	-rm -r $(DESTDIR)$(DESTSHARED)/__pycache__
  
---- a/configure.ac
-+++ b/configure.ac
-@@ -1200,15 +1200,6 @@
-         ARFLAGS="rcs"
- fi
- 
--AC_CHECK_TOOLS([READELF], [readelf], [:])
--if test "$cross_compiling" = yes; then
--    case "$READELF" in
--	readelf|:)
--	AC_MSG_ERROR([readelf for the host is required for cross builds])
--	;;
--    esac
--fi
--AC_SUBST(READELF)
- 
- 
- case $MACHDEP in
 --- a/setup.py
 +++ b/setup.py
-@@ -16,7 +16,7 @@
- from distutils.command.build_scripts import build_scripts
- from distutils.spawn import find_executable
+@@ -21,7 +21,7 @@
  
--cross_compiling = "_PYTHON_HOST_PLATFORM" in os.environ
-+cross_compiling = ("_PYTHON_HOST_PLATFORM" in os.environ) or ('CROSS_COMPILE_TARGET' in os.environ)
  
- # Set common compiler and linker flags derived from the Makefile,
- # reserved for building the interpreter and the stdlib modules.
-@@ -268,6 +268,7 @@
+ # Compile extensions used to test Python?
+-TEST_EXTENSIONS = True
++TEST_EXTENSIONS = False
  
-     def build_extensions(self):
+ # This global variable is used to hold the list of modules to be disabled.
+ DISABLED_MODULE_LIST = []
+@@ -38,7 +38,7 @@
+     return sys.platform
  
-+        return
-         # Detect which modules should be compiled
-         missing = self.detect_modules()
- 
-@@ -477,6 +478,9 @@
  
-         # Don't try to load extensions for cross builds
-         if cross_compiling:
-+            self.announce(
-+                'WARNING: skipping import check for cross-compiled: "%s"' %
-+                ext.name)
-             return
+-CROSS_COMPILING = ("_PYTHON_HOST_PLATFORM" in os.environ)
++CROSS_COMPILING = ("_PYTHON_HOST_PLATFORM" in os.environ) or ('CROSS_COMPILE_TARGET' in os.environ)
+ HOST_PLATFORM = get_platform()
+ MS_WINDOWS = (HOST_PLATFORM == 'win32')
+ CYGWIN = (HOST_PLATFORM == 'cygwin')
+@@ -310,6 +310,7 @@
+         self.extensions.append(ext)
  
-         loader = importlib.machinery.ExtensionFileLoader(ext.name, ext_filename)
+     def build_extensions(self):
++        return
+         self.srcdir = sysconfig.get_config_var('srcdir')
+         if not self.srcdir:
+             # Maybe running on Windows but not using CYGWIN?
diff --git a/tools/depends/target/python3/darwin_embedded.patch b/tools/depends/target/python3/darwin_embedded.patch
index 094e7f112e..8297615b81 100644
--- a/tools/depends/target/python3/darwin_embedded.patch
+++ b/tools/depends/target/python3/darwin_embedded.patch
@@ -1,6 +1,6 @@
 --- a/Lib/os.py
 +++ b/Lib/os.py
-@@ -602,6 +602,7 @@
+@@ -599,6 +599,7 @@
          fullname = path.join(dir, file)
          try:
              exec_func(fullname, *argrest)
@@ -10,7 +10,7 @@
          except OSError as e:
 --- a/Lib/platform.py
 +++ b/Lib/platform.py
-@@ -781,7 +781,7 @@
+@@ -604,7 +604,7 @@
  
      """ Interface to the system's uname command.
      """
@@ -18,8 +18,8 @@
 +    if sys.platform in ('dos', 'win32', 'win16', 'darwin'):
          # XXX Others too ?
          return default
-     try:
-@@ -804,7 +804,7 @@
+ 
+@@ -626,7 +626,7 @@
          default in case the command should fail.
  
      """
@@ -27,10 +27,10 @@
 +    if sys.platform in ('dos', 'win32', 'win16', 'darwin'):
          # XXX Others too ?
          return default
-     target = _follow_symlinks(target)
+ 
 --- a/Lib/subprocess.py
 +++ b/Lib/subprocess.py
-@@ -851,6 +851,7 @@
+@@ -936,6 +936,7 @@
          if not self._child_created:
              # We didn't get to successfully create a child process.
              return
@@ -38,7 +38,7 @@
          if self.returncode is None:
              # Not reading subprocess exit status creates a zombie process which
              # is only destroyed at the parent python process exit
-@@ -1394,6 +1395,7 @@
+@@ -1566,6 +1566,7 @@
              else:
                  args = list(args)
  
@@ -46,7 +46,7 @@
              if shell:
                  # On Android the default shell is at '/system/bin/sh'.
                  unix_shell = ('/system/bin/sh' if
-@@ -1452,6 +1454,7 @@
+@@ -1643,6 +1643,7 @@
                              errpipe_read, errpipe_write,
                              restore_signals, start_new_session, preexec_fn)
                      self._child_created = True
@@ -54,7 +54,7 @@
                  finally:
                      # be sure the FD is closed no matter what
                      os.close(errpipe_write)
-@@ -1480,6 +1483,7 @@
+@@ -1662,6 +1662,7 @@
              finally:
                  # be sure the FD is closed no matter what
                  os.close(errpipe_read)
@@ -64,7 +64,7 @@
                  try:
 --- a/Lib/urllib/request.py
 +++ b/Lib/urllib/request.py
-@@ -2580,11 +2580,9 @@
+@@ -2616,11 +2616,9 @@
  
  
  if sys.platform == 'darwin':
@@ -77,7 +77,7 @@
  
      def getproxies_macosx_sysconf():
          """Return a dictionary of scheme -> proxy server URL mappings.
-@@ -2592,7 +2590,7 @@
+@@ -2628,7 +2628,7 @@
          This function uses the MacOSX framework SystemConfiguration
          to fetch the proxy information.
          """
@@ -86,7 +86,7 @@
  
  
  
-@@ -2605,9 +2603,9 @@
+@@ -2641,9 +2641,9 @@
          """
          proxies = getproxies_environment()
          if proxies:
@@ -187,7 +187,7 @@
      /* Report the posix error to our parent process. */
      /* We ignore all write() return values as the total size of our writes is
         less than PIPEBUF and we cannot do anything about an error anyways.
-@@ -683,7 +702,8 @@
+@@ -690,7 +709,8 @@
          need_after_fork = 1;
      }
  
@@ -199,7 +199,7 @@
          /*
 --- a/Modules/getpath.c
 +++ b/Modules/getpath.c
-@@ -593,8 +593,10 @@
+@@ -806,8 +806,10 @@
        * will fail if a relative path was used. but in that case,
        * absolutize() should help us out below
        */
@@ -214,7 +214,7 @@
          wchar_t *path = Py_DecodeLocale(execpath, &len);
 --- a/Modules/posixmodule.c
 +++ b/Modules/posixmodule.c
-@@ -197,6 +197,19 @@
+@@ -216,6 +216,22 @@
  #endif  /* _MSC_VER */
  #endif  /* ! __WATCOMC__ || __QNX__ */
  
@@ -223,6 +223,8 @@
 +#  undef HAVE_FORK1
 +#  undef HAVE_FORKPTY
 +#  undef HAVE_GETGROUPS
++#  undef HAVE_POSIX_SPAWN
++#  undef HAVE_POSIX_SPAWNP
 +#  undef HAVE_SCHED_H
 +#  undef HAVE_SENDFILE
 +#  undef HAVE_SETPRIORITY
@@ -231,10 +233,11 @@
 +#  undef HAVE_WAIT
 +#  undef HAVE_WAIT3
 +#  undef HAVE_WAIT4
++#  undef HAVE_WAITPID
  
  /*[clinic input]
  # one of the few times we lie about this name!
-@@ -475,6 +488,10 @@
+@@ -488,6 +504,10 @@
  }
  #endif
  
@@ -245,7 +248,7 @@
  /* Legacy wrapper */
  void
  PyOS_AfterFork(void)
-@@ -5040,6 +5057,8 @@
+@@ -5239,6 +5259,8 @@
  #endif
      _Py_END_SUPPRESS_IPH
  
@@ -254,7 +257,7 @@
      /* If we get here it's definitely an error */
  
      free_string_array(argvlist, argc);
-@@ -5116,6 +5135,12 @@
+@@ -5319,6 +5341,12 @@
  #endif
      _Py_END_SUPPRESS_IPH
  
@@ -267,18 +270,18 @@
      /* If we get here it's definitely an error */
  
      posix_path_error(path);
-@@ -5416,7 +5441,7 @@
-     pid_t pid;
- 
+@@ -6135,7 +6163,7 @@
+         return NULL;
+     }
      PyOS_BeforeFork();
 -    pid = fork1();
 +    pid = -1;
      if (pid == 0) {
          /* child: this clobbers and resets the import lock. */
          PyOS_AfterFork_Child();
-@@ -5447,7 +5472,7 @@
-     pid_t pid;
- 
+@@ -6173,7 +6201,7 @@
+         return NULL;
+     }
      PyOS_BeforeFork();
 -    pid = fork();
 +    pid = -1;
diff --git a/tools/depends/target/python3/modules.setup b/tools/depends/target/python3/modules.setup
index 673f634087..b2068535b3 100644
--- a/tools/depends/target/python3/modules.setup
+++ b/tools/depends/target/python3/modules.setup
@@ -1,14 +1,11 @@
 # -*- makefile -*-
 # The file Setup is used by the makesetup script to construct the files
 # Makefile and config.c, from Makefile.pre and config.c.in,
-# respectively.  The file Setup itself is initially copied from
-# Setup.dist; once it exists it will not be overwritten, so you can edit
-# Setup to your heart's content.  Note that Makefile.pre is created
-# from Makefile.pre.in by the toplevel configure script.
+# respectively.  Note that Makefile.pre is created from Makefile.pre.in
+# by the toplevel configure script.
 
 # (VPATH notes: Setup and Makefile.pre are in the build directory, as
-# are Makefile and config.c; the *.in and *.dist files are in the source
-# directory.)
+# are Makefile and config.c; the *.in files are in the source directory.)
 
 # Each line in this file describes one or more optional modules.
 # Modules configured here will not be compiled by the setup.py script,
@@ -64,15 +61,11 @@
 # toplevel "make install" target.)  (For compatibility,
 # *noconfig* has the same effect as *shared*.)
 #
-# In addition, *static* explicitly declares the following modules to
-# be static.  Lines containing "*static*" and "*shared*" may thus
-# alternate throughout this file.
-
 # NOTE: As a standard policy, as many modules as can be supported by a
 # platform should be present.  The distribution comes with all modules
 # enabled that are supported by most platforms and don't require you
 # to ftp sources from elsewhere.
-*static*
+
 
 # Some special rules to define PYTHONPATH.
 # Edit the definitions below to indicate which options you are using.
@@ -108,38 +101,37 @@ PYTHONPATH=$(COREPYTHONPATH)
 # This only contains the minimal set of modules required to run the
 # setup.py script in the root of the Python source tree.
 
-posix -DPy_BUILD_CORE posixmodule.c	# posix (UNIX) system calls
+posix -DPy_BUILD_CORE_BUILTIN -I$(srcdir)/Include/internal posixmodule.c # posix (UNIX) system calls
 errno errnomodule.c			# posix (UNIX) errno values
 pwd pwdmodule.c				# this is needed to find out the user's home dir
 					# if $HOME is not set
 _sre _sre.c				# Fredrik Lundh's new regular expressions
 _codecs _codecsmodule.c			# access to the builtin codecs and codec registry
 _weakref _weakref.c			# weak references
-_functools -DPy_BUILD_CORE _functoolsmodule.c   # Tools for working with functions and callable objects
+_functools -DPy_BUILD_CORE_BUILTIN -I$(srcdir)/Include/internal _functoolsmodule.c   # Tools for working with functions and callable objects
 _operator _operator.c	        	# operator.add() and similar goodies
 _collections _collectionsmodule.c	# Container types
 _abc _abc.c				# Abstract base classes
 itertools itertoolsmodule.c		# Functions creating iterators for efficient looping
 atexit atexitmodule.c			# Register functions to be run at interpreter-shutdown
-_signal -DPy_BUILD_CORE signalmodule.c
+_signal -DPy_BUILD_CORE_BUILTIN -I$(srcdir)/Include/internal signalmodule.c
 _stat _stat.c				# stat.h interface
-time -DPy_BUILD_CORE timemodule.c	# -lm # time operations and variables
-_thread -DPy_BUILD_CORE _threadmodule.c	# low-level threading interface
+time -DPy_BUILD_CORE_BUILTIN -I$(srcdir)/Include/internal timemodule.c	# -lm # time operations and variables
+_thread -DPy_BUILD_CORE_BUILTIN -I$(srcdir)/Include/internal _threadmodule.c	# low-level threading interface
 
 # access to ISO C locale support
-_locale _localemodule.c   -lintl
+_locale -DPy_BUILD_CORE_BUILTIN _localemodule.c  # -lintl
 
 # Standard I/O baseline
-_io -DPy_BUILD_CORE -I$(srcdir)/Modules/_io _io/_iomodule.c _io/iobase.c _io/fileio.c _io/bytesio.c _io/bufferedio.c _io/textio.c _io/stringio.c
-
-# The zipimport module is always imported at startup. Having it as a
-# builtin module avoids some bootstrapping problems and reduces overhead.
-zipimport -DPy_BUILD_CORE zipimport.c
+_io -DPy_BUILD_CORE_BUILTIN -I$(srcdir)/Include/internal -I$(srcdir)/Modules/_io _io/_iomodule.c _io/iobase.c _io/fileio.c _io/bytesio.c _io/bufferedio.c _io/textio.c _io/stringio.c
 
 # faulthandler module
 faulthandler faulthandler.c
 
 # debug tool to trace memory blocks allocated by Python
+#
+# bpo-35053: The module must be builtin since _Py_NewReference()
+# can call _PyTraceMalloc_NewReference().
 _tracemalloc _tracemalloc.c hashtable.c
 
 # The rest of the modules listed in this file are all commented out by
@@ -180,7 +172,8 @@ math mathmodule.c _math.c # -lm # math library functions, e.g. sin()
 _contextvars _contextvarsmodule.c  # Context Variables
 _struct _struct.c	# binary structure packing/unpacking
 _weakref _weakref.c	# basic weak reference support
-_testcapi _testcapimodule.c    # Python C API test module
+#_testcapi _testcapimodule.c    # Python C API test module
+#_testinternalcapi _testinternalcapi.c -I$(srcdir)/Include/internal -DPy_BUILD_CORE_MODULE  # Python internal C API test module
 _random _randommodule.c	# Random number generator
 _elementtree -I$(srcdir)/Modules/expat -DHAVE_EXPAT_CONFIG_H -DUSE_PYEXPAT_CAPI _elementtree.c	# elementtree accelerator
 _pickle _pickle.c	# pickle accelerator
@@ -188,6 +181,8 @@ _datetime _datetimemodule.c	# datetime accelerator
 _bisect _bisectmodule.c	# Bisection algorithms
 _heapq _heapqmodule.c	# Heap queue algorithm
 _asyncio _asynciomodule.c  # Fast asyncio Future
+_json -I$(srcdir)/Include/internal -DPy_BUILD_CORE_BUILTIN _json.c	# _json speedups
+_statistics _statisticsmodule.c # statistics accelerator
 
 unicodedata unicodedata.c    # static Unicode character database
 
@@ -212,14 +207,10 @@ _socket socketmodule.c
 
 # Socket module helper for SSL support; you must comment out the other
 # socket line above, and possibly edit the SSL variable:
-SSLDEFINES='-DUSE_SSL'
+SSL=$(prefix)
 _ssl _ssl.c \
-	$(SSLDEFINES) -I$(prefix)/include -I$(prefix)/include/openssl \
-	-L$(prefix)/lib -lssl -lcrypto
-
-_hashlib _hashopenssl.c \
-        $(SSLDEFINES) -I$(prefix)/include -I$(prefix)/include/openssl \
-        -L$(prefix)/lib -lssl -lcrypto
+	-DUSE_SSL -I$(SSL)/include -I$(SSL)/include/openssl \
+	-L$(SSL)/lib -lssl -lcrypto
 
 # The crypt module is now disabled by default because it breaks builds
 # on many systems (where -lcrypt is needed), e.g. Linux (I believe).
@@ -350,7 +341,7 @@ zlib zlibmodule.c -I$(prefix)/include -L$(exec_prefix)/lib -lz
 # Interface to the Expat XML parser
 # More information on Expat can be found at www.libexpat.org.
 #
-pyexpat pyexpat.c -I$(prefix)/include -L$(prefix)/lib -lexpat -DHAVE_EXPAT_CONFIG_H -DUSE_PYEXPAT_CAPI
+pyexpat expat/xmlparse.c expat/xmlrole.c expat/xmltok.c pyexpat.c -I$(srcdir)/Modules/expat -DHAVE_EXPAT_CONFIG_H -DXML_POOR_ENTROPY -DUSE_PYEXPAT_CAPI
 
 # Hye-Shik Chang's CJKCodecs
 
diff --git a/tools/depends/target/pythonmodule-pil/Makefile b/tools/depends/target/pythonmodule-pil/Makefile
index 6633c23664..51297daf1c 100644
--- a/tools/depends/target/pythonmodule-pil/Makefile
+++ b/tools/depends/target/pythonmodule-pil/Makefile
@@ -10,7 +10,7 @@ VERSION=7.1.2
 SOURCE=$(LIBNAME)-$(VERSION)
 ARCHIVE=$(SOURCE).tar.gz
 
-PYTHONPATH=$(PREFIX)/lib/python3.7/site-packages/
+PYTHONPATH=$(PREFIX)/lib/python3.8/site-packages/
 PILPATH=$(PYTHONPATH)
 LDSHARED=$(CC) -shared
 
@@ -24,12 +24,12 @@ endif
 ifeq ($(OS),android)
 PILPATH=$(PREFIX)/share/$(APP_NAME)/addons/script.module.pil
 PILPATHLIB=$(PILPATH)/lib
-PYTHONPATH=$(PILPATH):$(PREFIX)/lib/python3.7/site-packages/
+PYTHONPATH=$(PILPATH):$(PREFIX)/lib/python3.8/site-packages/
 EXTRALDFLAGS=-L$(PREFIX)/lib/dummy-lib$(APP_NAME)/ -l$(APP_NAME) -lm
 else ifeq ($(OS),darwin_embedded)
 PILPATH=$(PREFIX)/share/$(APP_NAME)/addons/script.module.pil
 PILPATHLIB=$(PILPATH)/lib
-PYTHONPATH=$(PILPATH):$(PREFIX)/lib/python3.7/site-packages/
+PYTHONPATH=$(PILPATH):$(PREFIX)/lib/python3.8/site-packages/
 endif
 
 ifeq (darwin, $(findstring darwin, $(HOST)))
diff --git a/tools/depends/target/pythonmodule-pil/pillow-crosscompile.patch b/tools/depends/target/pythonmodule-pil/pillow-crosscompile.patch
index 57401a25c0..b62aa5522e 100644
--- a/tools/depends/target/pythonmodule-pil/pillow-crosscompile.patch
+++ b/tools/depends/target/pythonmodule-pil/pillow-crosscompile.patch
@@ -26,7 +26,7 @@
          if prefix:
              _add_directory(library_dirs, os.path.join(prefix, "lib"))
              _add_directory(include_dirs, os.path.join(prefix, "include"))
-+            _add_directory(include_dirs, os.path.join(prefix, "include/python3.7"))
++            _add_directory(include_dirs, os.path.join(prefix, "include/python3.8"))
  
          #
          # add platform directories
diff --git a/tools/depends/target/pythonmodule-pycryptodome/Makefile b/tools/depends/target/pythonmodule-pycryptodome/Makefile
index 759de0f3a6..34f20c7ec1 100644
--- a/tools/depends/target/pythonmodule-pycryptodome/Makefile
+++ b/tools/depends/target/pythonmodule-pycryptodome/Makefile
@@ -10,9 +10,9 @@ VERSION=3.4.3
 SOURCE=$(LIBNAME)-$(VERSION)
 ARCHIVE=$(SOURCE).tar.gz
 
-CROSSFLAGS=PYTHONXCPREFIX="$(PREFIX)" CC="$(CC) $(CFLAGS)" LDSHARED="$(CC) -shared" LDFLAGS="$(LDFLAGS)" PYTHONPATH="$(PREFIX)/lib/python3.7/site-packages/"
+CROSSFLAGS=PYTHONXCPREFIX="$(PREFIX)" CC="$(CC) $(CFLAGS)" LDSHARED="$(CC) -shared" LDFLAGS="$(LDFLAGS)" PYTHONPATH="$(PREFIX)/lib/python3.8/site-packages/"
 ifeq ($(OS),android)
-CROSSFLAGS=PYTHONXCPREFIX="$(PREFIX)" CC="$(CC) $(CFLAGS)" LDSHARED="$(CC) -shared" LDFLAGS="$(LDFLAGS) -L$(PREFIX)/lib/dummy-lib$(APP_NAME)/ -l$(APP_NAME) -lm" PYTHONPATH="$(PREFIX)/lib/python3.7/site-packages/"
+CROSSFLAGS=PYTHONXCPREFIX="$(PREFIX)" CC="$(CC) $(CFLAGS)" LDSHARED="$(CC) -shared" LDFLAGS="$(LDFLAGS) -L$(PREFIX)/lib/dummy-lib$(APP_NAME)/ -l$(APP_NAME) -lm" PYTHONPATH="$(PREFIX)/lib/python3.8/site-packages/"
 endif
 
 ifeq (darwin, $(findstring darwin, $(HOST)))
@@ -21,12 +21,12 @@ PYTHON_O=$(abs_top_srcdir)/target/python3/$(PLATFORM)/Programs/python.o
 endif
 #ensure that only our target ldflags are passed to the python build
 LDSHARED:=$(CC) -bundle -undefined dynamic_lookup $(LDFLAGS)
-CROSSFLAGS=PYTHONXCPREFIX="$(PREFIX)" CC="$(CC) $(CFLAGS)" CCSHARED="$(CC) $(CFLAGS) $(PYTHON_O)" LDFLAGS="$(LDFLAGS)" PYTHONPATH="$(PREFIX)/lib/python3.7/site-packages/" LDSHARED="$(LDSHARED)"
+CROSSFLAGS=PYTHONXCPREFIX="$(PREFIX)" CC="$(CC) $(CFLAGS)" CCSHARED="$(CC) $(CFLAGS) $(PYTHON_O)" LDFLAGS="$(LDFLAGS)" PYTHONPATH="$(PREFIX)/lib/python3.8/site-packages/" LDSHARED="$(LDSHARED)"
 endif
 
-LIBDYLIB=$(PLATFORM)/build/lib.$(OS)-$(CPU)-3.7/Cryptodome
+LIBDYLIB=$(PLATFORM)/build/lib.$(OS)-$(CPU)-3.8/Cryptodome
 ifeq ($(NATIVE_OS), osx)
-	LIBDYLIB=$(PLATFORM)/build/lib.macosx-10.4-x86_64-3.7/Cryptodome
+	LIBDYLIB=$(PLATFORM)/build/lib.macosx-10.4-x86_64-3.8/Cryptodome
 endif
 
 all: .installed-$(PLATFORM)
diff --git a/tools/depends/target/pythonmodule-setuptools/Makefile b/tools/depends/target/pythonmodule-setuptools/Makefile
index b629c42e97..fb7a757b78 100644
--- a/tools/depends/target/pythonmodule-setuptools/Makefile
+++ b/tools/depends/target/pythonmodule-setuptools/Makefile
@@ -10,20 +10,20 @@ VERSION=18.4
 SOURCE=$(LIBNAME)-$(VERSION)
 ARCHIVE=$(SOURCE).tar.gz
 
-CROSSFLAGS=PYTHONXCPREFIX="$(PREFIX)" CC="$(CC) $(CFLAGS)" LDSHARED="$(CC) -shared" LDFLAGS="$(LDFLAGS)" PYTHONPATH="$(PREFIX)/lib/python3.7/site-packages/"
+CROSSFLAGS=PYTHONXCPREFIX="$(PREFIX)" CC="$(CC) $(CFLAGS)" LDSHARED="$(CC) -shared" LDFLAGS="$(LDFLAGS)" PYTHONPATH="$(PREFIX)/lib/python3.8/site-packages/"
 ifeq ($(OS),android)
 #PYTHONPATH=$(PREFIX)/share/$(APP_NAME)/addons/script.module.pil
-CROSSFLAGS=PYTHONXCPREFIX="$(PREFIX)" CC="$(CC) $(CFLAGS)" LDSHARED="$(CC) -shared" LDFLAGS="$(LDFLAGS) -L$(PREFIX)/lib/dummy-lib$(APP_NAME)/ -l$(APP_NAME) -lm" PYTHONPATH="$(PREFIX)/lib/python3.7/site-packages/"
+CROSSFLAGS=PYTHONXCPREFIX="$(PREFIX)" CC="$(CC) $(CFLAGS)" LDSHARED="$(CC) -shared" LDFLAGS="$(LDFLAGS) -L$(PREFIX)/lib/dummy-lib$(APP_NAME)/ -l$(APP_NAME) -lm" PYTHONPATH="$(PREFIX)/lib/python3.8/site-packages/"
 endif
 ifeq ($(OS),osx)
-CROSSFLAGS=PYTHONXCPREFIX="$(PREFIX)" CC="$(CC)" LDSHARED="$(CC) -shared" LDFLAGS="$(LDFLAGS) -lpython" PYTHONPATH="$(PREFIX)/lib/python3.7/site-packages/"
+CROSSFLAGS=PYTHONXCPREFIX="$(PREFIX)" CC="$(CC)" LDSHARED="$(CC) -shared" LDFLAGS="$(LDFLAGS) -lpython" PYTHONPATH="$(PREFIX)/lib/python3.8/site-packages/"
 endif
 ifeq ($(OS),darwin_embedded)
 PYTHON_O=$(abs_top_srcdir)/target/python3/$(PLATFORM)/Programs/python.o
-CROSSFLAGS=PYTHONXCPREFIX="$(PREFIX)" CC="$(CC) $(CFLAGS)" CCSHARED="$(CC) $(CFLAGS) $(PYTHON_O)" LDFLAGS="$(LDFLAGS)" PYTHONPATH="$(PREFIX)/lib/python3.7/site-packages/"
+CROSSFLAGS=PYTHONXCPREFIX="$(PREFIX)" CC="$(CC) $(CFLAGS)" CCSHARED="$(CC) $(CFLAGS) $(PYTHON_O)" LDFLAGS="$(LDFLAGS)" PYTHONPATH="$(PREFIX)/lib/python3.8/site-packages/"
 endif
 
-LIBDYLIB=$(PLATFORM)/dist/$(LIBNAME)-$(VERSION)-py3.7.egg
+LIBDYLIB=$(PLATFORM)/dist/$(LIBNAME)-$(VERSION)-py3.8.egg
 
 all: .installed-$(PLATFORM)
 
diff --git a/xbmc/cores/DllLoader/exports/emu_msvcrt.cpp b/xbmc/cores/DllLoader/exports/emu_msvcrt.cpp
index 23bd096da1..b7585e5ffa 100644
--- a/xbmc/cores/DllLoader/exports/emu_msvcrt.cpp
+++ b/xbmc/cores/DllLoader/exports/emu_msvcrt.cpp
@@ -131,8 +131,8 @@ extern "C" void __stdcall init_emu_environ()
   // check if we are running as real xbmc.app or just binary
   if (!CUtil::GetFrameworksPath(true).empty())
   {
-    // using external python, it's build looking for xxx/lib/python3.7
-    // so point it to frameworks which is where python3.7 is located
+    // using external python, it's build looking for xxx/lib/python3.8
+    // so point it to frameworks which is where python3.8 is located
     dll_putenv(("PYTHONPATH=" +
       CSpecialProtocol::TranslatePath("special://frameworks")).c_str());
     dll_putenv(("PYTHONHOME=" +
@@ -154,7 +154,7 @@ extern "C" void __stdcall init_emu_environ()
 
 #if defined(TARGET_ANDROID)
   std::string apkPath = getenv("KODI_ANDROID_APK");
-  apkPath += "/assets/python3.7";
+  apkPath += "/assets/python3.8";
   dll_putenv(("PYTHONHOME=" + apkPath).c_str());
   dll_putenv("PYTHONOPTIMIZE=");
   dll_putenv("PYTHONNOUSERSITE=1");
diff --git a/xbmc/interfaces/python/PythonInvoker.cpp b/xbmc/interfaces/python/PythonInvoker.cpp
index d217d46f95..c559723a16 100644
--- a/xbmc/interfaces/python/PythonInvoker.cpp
+++ b/xbmc/interfaces/python/PythonInvoker.cpp
@@ -10,17 +10,6 @@
 #include <Python.h>
 #include <iterator>
 
-// This is a workaround to compile Kodi against python 3.8
-//! @todo implement a compliant way to get access to the chain of thread states
-#if PY_VERSION_HEX >= 0x03080000
-# define Py_BUILD_CORE
-# undef HAVE_STD_ATOMIC
-/* for access to the fields of PyInterpreterState */
-#  include <internal/pycore_pystate.h>
-# undef Py_BUILD_CORE
-# define HAVE_STD_ATOMIC
-#endif
-
 #include "Application.h"
 #include "PythonInvoker.h"
 #include "ServiceBroker.h"
@@ -406,9 +395,9 @@ bool CPythonInvoker::execute(const std::string& script, const std::vector<std::w
     // make sure all sub threads have finished
     for (PyThreadState* old = nullptr; m_threadState != nullptr;)
     {
-      PyThreadState* s = m_threadState->interp->tstate_head;
+      PyThreadState* s = PyInterpreterState_ThreadHead(m_threadState->interp);
       for (; s && s == m_threadState;)
-        s = s->next;
+        s = PyThreadState_Next(s);
 
       if (!s)
         break;
@@ -536,12 +525,15 @@ bool CPythonInvoker::stop(bool abort)
         PyEval_RestoreThread((PyThreadState*)m_threadState);
       }
 
-      for (PyThreadState* state = ((PyThreadState*)m_threadState)->interp->tstate_head; state; state = state->next)
+
+      PyThreadState* state = PyInterpreterState_ThreadHead(m_threadState->interp);
+      while (state)
       {
         // Raise a SystemExit exception in python threads
         Py_XDECREF(state->async_exc);
         state->async_exc = PyExc_SystemExit;
         Py_XINCREF(state->async_exc);
+        state = PyThreadState_Next(state);
       }
 
       // If a dialog entered its doModal(), we need to wake it to see the exception
diff --git a/xbmc/interfaces/python/PythonInvoker.h b/xbmc/interfaces/python/PythonInvoker.h
index 144de29fa2..0ea3bfa154 100644
--- a/xbmc/interfaces/python/PythonInvoker.h
+++ b/xbmc/interfaces/python/PythonInvoker.h
@@ -19,7 +19,6 @@
 #include <vector>
 
 typedef struct _object PyObject;
-struct _ts;
 
 class CPythonInvoker : public ILanguageInvoker
 {
@@ -65,7 +64,7 @@ private:
   FILE* PyFile_AsFileWithMode(PyObject* py_file, const char* mode);
 
   std::string m_pythonPath;
-  _ts* m_threadState;
+  PyThreadState* m_threadState;
   bool m_stop;
   CEvent m_stoppedEvent;
 
diff --git a/xbmc/interfaces/python/XBPython.cpp b/xbmc/interfaces/python/XBPython.cpp
index 199e5254eb..047896ba48 100644
--- a/xbmc/interfaces/python/XBPython.cpp
+++ b/xbmc/interfaces/python/XBPython.cpp
@@ -564,8 +564,8 @@ bool XBPython::OnScriptInitialized(ILanguageInvoker *invoker)
     // check if we are running as real xbmc.app or just binary
     if (!CUtil::GetFrameworksPath(true).empty())
     {
-      // using external python, it's build looking for xxx/lib/python3.7
-      // so point it to frameworks which is where python3.7 is located
+      // using external python, it's build looking for xxx/lib/python3.8
+      // so point it to frameworks which is where python3.8 is located
       setenv("PYTHONHOME", CSpecialProtocol::TranslatePath("special://frameworks").c_str(), 1);
       setenv("PYTHONPATH", CSpecialProtocol::TranslatePath("special://frameworks").c_str(), 1);
       CLog::Log(LOGDEBUG, "PYTHONHOME -> %s", CSpecialProtocol::TranslatePath("special://frameworks").c_str());
diff --git a/xbmc/platform/android/activity/XBMCApp.cpp b/xbmc/platform/android/activity/XBMCApp.cpp
index 755254fb22..03babf75bf 100644
--- a/xbmc/platform/android/activity/XBMCApp.cpp
+++ b/xbmc/platform/android/activity/XBMCApp.cpp
@@ -1360,7 +1360,7 @@ void CXBMCApp::SetupEnv()
   else
     setenv("HOME", getenv("KODI_TEMP"), 0);
 
-  std::string pythonPath = cacheDir + "/apk/assets/python3.7";
+  std::string pythonPath = cacheDir + "/apk/assets/python3.8";
   setenv("PYTHONHOME", pythonPath.c_str(), 1);
   setenv("PYTHONPATH", "", 1);
   setenv("PYTHONOPTIMIZE","", 1);