Merge #17919: depends: Allow building with system clang

de4fedb6c3a45e6a23f1bccd28045d76b5830afc depends: Add documentation for FORCE_USE_SYSTEM_CLANG make flag (Carl Dong)
fe98999dcf87ac056d0a2c9231fb3160abdf3417 depends: Reformat make options as definition list (Carl Dong)
60c55b1b9bab8c1e143e2f4c26d729bfa0bbcf09 depends: Add justifications for macOS clang flags (Carl Dong)
6b8e497eeaf38f272715c490f317fdc98a2174be depends: specify libc++ header location for darwin (Cory Fields)
156b604203ef17b2b77ee9dacf15e375c809242a depends: force a new host id string if FORCE_USE_SYSTEM_CLANG is in use (Cory Fields)
c9c572a367f08095a3e2c7c0723da9f6778b9378 depends: Allow building with system clang (Carl Dong)
e6e5c8d6caccb4648fc580e5a01448857c2fdf18 depends: Decouple toolchain + binutils (Carl Dong)

Pull request description:

  This replaces: #17099

  -----

  This patchset allows us to force depends to use system clang.

  Previously, #17099 removes our dependency on a specific clang we download from llvm.org, but theuni pointed out that since OSX builds are only ever built with a version of clang that is chosen and "blessed" by Apple, it is more likely that the user will encounter problems if they use their system clang. This patchset forces the user to set `FORCE_USE_SYSTEM_CLANG=1` in order to use their system clang (when they know what they're doing)

ACKs for top commit:
  theuni:
    ACK de4fedb6c3a45e6a23f1bccd28045d76b5830afc.

Tree-SHA512: 8774121e035f90c27030bcce06e1b79f7729b5e17802c718e49652ab06e19780632db974df47423c1d1b04f1ab1b7a763554fb922fec05d1cd6445b26578be1d

6 files changed, 121 insertions, 27 deletions

diff --git a/depends/Makefile b/depends/Makefile
index 3d0784cb6b..2bc5df974a 100644
--- a/depends/Makefile
+++ b/depends/Makefile
@@ -125,6 +125,11 @@ $(host_arch)_$(host_os)_id_string+=$(shell $(host_CXX) --version 2>/dev/null)
 $(host_arch)_$(host_os)_id_string+=$(shell $(host_RANLIB) --version 2>/dev/null)
 $(host_arch)_$(host_os)_id_string+=$(shell $(host_STRIP) --version 2>/dev/null)
 
+ifneq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
+build_id_string+=system_clang
+$(host_arch)_$(host_os)_id_string+=system_clang
+endif
+
 qrencode_packages_$(NO_QR) = $(qrencode_packages)
 
 qt_packages_$(NO_QT) = $(qt_packages) $(qt_$(host_os)_packages) $(qt_$(host_arch)_$(host_os)_packages) $(qrencode_packages_)
@@ -150,11 +155,17 @@ all_packages = $(packages) $(native_packages)
 
 meta_depends = Makefile funcs.mk builders/default.mk hosts/default.mk hosts/$(host_os).mk builders/$(build_os).mk
 
+$(host_arch)_$(host_os)_native_binutils?=$($(host_os)_native_binutils)
 $(host_arch)_$(host_os)_native_toolchain?=$($(host_os)_native_toolchain)
 
 include funcs.mk
 
+binutils_path=$($($(host_arch)_$(host_os)_native_binutils)_prefixbin)
+ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
 toolchain_path=$($($(host_arch)_$(host_os)_native_toolchain)_prefixbin)
+else
+toolchain_path=
+endif
 final_build_id_long+=$(shell $(build_SHA256SUM) config.site.in)
 final_build_id+=$(shell echo -n "$(final_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH))
 $(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages)
@@ -170,10 +181,10 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_
 	$(AT)sed -e 's|@HOST@|$(host)|' \
             -e 's|@CC@|$(toolchain_path)$(host_CC)|' \
             -e 's|@CXX@|$(toolchain_path)$(host_CXX)|' \
-            -e 's|@AR@|$(toolchain_path)$(host_AR)|' \
-            -e 's|@RANLIB@|$(toolchain_path)$(host_RANLIB)|' \
-            -e 's|@NM@|$(toolchain_path)$(host_NM)|' \
-            -e 's|@STRIP@|$(toolchain_path)$(host_STRIP)|' \
+            -e 's|@AR@|$(binutils_path)$(host_AR)|' \
+            -e 's|@RANLIB@|$(binutils_path)$(host_RANLIB)|' \
+            -e 's|@NM@|$(binutils_path)$(host_NM)|' \
+            -e 's|@STRIP@|$(binutils_path)$(host_STRIP)|' \
             -e 's|@build_os@|$(build_os)|' \
             -e 's|@host_os@|$(host_os)|' \
             -e 's|@CFLAGS@|$(strip $(host_CFLAGS) $(host_$(release_type)_CFLAGS))|' \
diff --git a/depends/README.md b/depends/README.md
index c12ea8bcb3..01276e48ee 100644
--- a/depends/README.md
+++ b/depends/README.md
@@ -80,21 +80,40 @@ For linux S390X cross compilation:
     sudo apt-get install g++-s390x-linux-gnu binutils-s390x-linux-gnu
 
 ### Dependency Options
-The following can be set when running make: make FOO=bar
-
-    SOURCES_PATH: downloaded sources will be placed here
-    BASE_CACHE: built packages will be placed here
-    SDK_PATH: Path where sdk's can be found (used by macOS)
-    FALLBACK_DOWNLOAD_PATH: If a source file can't be fetched, try here before giving up
-    NO_QT: Don't download/build/cache qt and its dependencies
-    NO_QR: Don't download/build/cache packages needed for enabling qrencode
-    NO_ZMQ: Don't download/build/cache packages needed for enabling zeromq
-    NO_WALLET: Don't download/build/cache libs needed to enable the wallet
-    NO_UPNP: Don't download/build/cache packages needed for enabling upnp
-    MULTIPROCESS: build libmultiprocess (experimental, requires cmake)
-    DEBUG: disable some optimizations and enable more runtime checking
-    HOST_ID_SALT: Optional salt to use when generating host package ids
-    BUILD_ID_SALT: Optional salt to use when generating build package ids
+The following can be set when running make: `make FOO=bar`
+
+<dl>
+<dt>SOURCES_PATH</dt>
+<dd>downloaded sources will be placed here</dd>
+<dt>BASE_CACHE</dt>
+<dd>built packages will be placed here</dd>
+<dt>SDK_PATH</dt>
+<dd>Path where sdk's can be found (used by macOS)</dd>
+<dt>FALLBACK_DOWNLOAD_PATH</dt>
+<dd>If a source file can't be fetched, try here before giving up</dd>
+<dt>NO_QT</dt>
+<dd>Don't download/build/cache qt and its dependencies</dd>
+<dt>NO_QR</dt>
+<dd>Don't download/build/cache packages needed for enabling qrencode</dd>
+<dt>NO_ZMQ</dt>
+<dd>Don't download/build/cache packages needed for enabling zeromq</dd>
+<dt>NO_WALLET</dt>
+<dd>Don't download/build/cache libs needed to enable the wallet</dd>
+<dt>NO_UPNP</dt>
+<dd>Don't download/build/cache packages needed for enabling upnp</dd>
+<dt>MULTIPROCESS</dt>
+<dd>build libmultiprocess (experimental, requires cmake)</dd>
+<dt>DEBUG</dt>
+<dd>disable some optimizations and enable more runtime checking</dd>
+<dt>HOST_ID_SALT</dt>
+<dd>Optional salt to use when generating host package ids</dd>
+<dt>BUILD_ID_SALT</dt>
+<dd>Optional salt to use when generating build package ids</dd>
+<dt>FORCE_USE_SYSTEM_CLANG</dt>
+<dd>(EXPERTS ONLY) When cross-compiling for macOS, use clang found in the
+system's <code>$PATH</code> rather than the default prebuilt release of clang
+from llvm.org</dd>
+</dl>
 
 If some packages are not built, for example `make NO_WALLET=1`, the appropriate
 options will be passed to bitcoin's configure. In this case, `--disable-wallet`.
diff --git a/depends/builders/darwin.mk b/depends/builders/darwin.mk
index 69c394ec1d..f4103fc1f2 100644
--- a/depends/builders/darwin.mk
+++ b/depends/builders/darwin.mk
@@ -19,4 +19,5 @@ darwin_LIBTOOL:=$(shell xcrun -f libtool)
 darwin_OTOOL:=$(shell xcrun -f otool)
 darwin_NM:=$(shell xcrun -f nm)
 darwin_INSTALL_NAME_TOOL:=$(shell xcrun -f install_name_tool)
+darwin_native_binutils=
 darwin_native_toolchain=
diff --git a/depends/funcs.mk b/depends/funcs.mk
index 135ebba9f8..6fc20543bb 100644
--- a/depends/funcs.mk
+++ b/depends/funcs.mk
@@ -41,7 +41,7 @@ endef
 
 define int_get_build_id
 $(eval $(1)_dependencies += $($(1)_$(host_arch)_$(host_os)_dependencies) $($(1)_$(host_os)_dependencies))
-$(eval $(1)_all_dependencies:=$(call int_get_all_dependencies,$(1),$($($(1)_type)_native_toolchain) $($(1)_dependencies)))
+$(eval $(1)_all_dependencies:=$(call int_get_all_dependencies,$(1),$($($(1)_type)_native_toolchain) $($($(1)_type)_native_binutils) $($(1)_dependencies)))
 $(foreach dep,$($(1)_all_dependencies),$(eval $(1)_build_id_deps+=$(dep)-$($(dep)_version)-$($(dep)_recipe_hash)))
 $(eval $(1)_build_id_long:=$(1)-$($(1)_version)-$($(1)_recipe_hash)-$(release_type) $($(1)_build_id_deps) $($($(1)_type)_id_string))
 $(eval $(1)_build_id:=$(shell echo -n "$($(1)_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH)))
@@ -260,4 +260,4 @@ $(foreach package,$(all_packages),$(eval $(call int_config_attach_build_config,$
 $(foreach package,$(all_packages),$(eval $(call int_add_cmds,$(package))))
 
 #special exception: if a toolchain package exists, all non-native packages depend on it
-$(foreach package,$(packages),$(eval $($(package)_unpacked): |$($($(host_arch)_$(host_os)_native_toolchain)_cached) ))
+$(foreach package,$(packages),$(eval $($(package)_unpacked): |$($($(host_arch)_$(host_os)_native_toolchain)_cached) $($($(host_arch)_$(host_os)_native_binutils)_cached) ))
diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk
index 0d4fab937d..6099fd4c71 100644
--- a/depends/hosts/darwin.mk
+++ b/depends/hosts/darwin.mk
@@ -6,10 +6,30 @@ LD64_VERSION=530
 
 OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-libcxx-headers
 
-# When cross-compiling for Darwin using Clang, -mlinker-version must be passed to
-# ensure that modern linker features are enabled.
-darwin_CC=clang -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -mlinker-version=$(LD64_VERSION)
-darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -stdlib=libc++ -mlinker-version=$(LD64_VERSION)
+# Flag explanations:
+#
+#     -mlinker-version
+#
+#         Ensures that modern linker features are enabled. See here for more
+#         details: https://github.com/bitcoin/bitcoin/pull/19407.
+#
+#     -B$(build_prefix)/bin
+#
+#         Explicitly point to our binaries (e.g. cctools) so that they are
+#         ensured to be found and preferred over other possibilities.
+#
+#     -nostdinc++ -isystem $(OSX_SDK)/usr/include/c++/v1
+#
+#         Forces clang to use the libc++ headers from our SDK and completely
+#         forget about the libc++ headers from the standard directories
+#
+#         TODO: Once we start requiring a clang version that has the
+#         -stdlib++-isystem<directory> flag first introduced here:
+#         https://reviews.llvm.org/D64089, we should use that instead. Read the
+#         differential summary there for more details.
+#
+darwin_CC=clang -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -mlinker-version=$(LD64_VERSION) -B$(build_prefix)/bin
+darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -stdlib=libc++ -mlinker-version=$(LD64_VERSION) -B$(build_prefix)/bin -nostdinc++ -isystem $(OSX_SDK)/usr/include/c++/v1
 
 darwin_CFLAGS=-pipe
 darwin_CXXFLAGS=$(darwin_CFLAGS)
@@ -20,5 +40,11 @@ darwin_release_CXXFLAGS=$(darwin_release_CFLAGS)
 darwin_debug_CFLAGS=-O1
 darwin_debug_CXXFLAGS=$(darwin_debug_CFLAGS)
 
+darwin_native_binutils=native_cctools
+ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
 darwin_native_toolchain=native_cctools
+else
+darwin_native_toolchain=
+endif
+
 darwin_cmake_system=Darwin
diff --git a/depends/packages/native_cctools.mk b/depends/packages/native_cctools.mk
index bdebd11862..072d3828a6 100644
--- a/depends/packages/native_cctools.mk
+++ b/depends/packages/native_cctools.mk
@@ -4,11 +4,13 @@ $(package)_download_path=https://github.com/tpoechtrager/cctools-port/archive
 $(package)_file_name=$($(package)_version).tar.gz
 $(package)_sha256_hash=a2d491c0981cef72fee2b833598f20f42a6c44a7614a61c439bda93d56446fec
 $(package)_build_subdir=cctools
+ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
 $(package)_clang_version=8.0.0
 $(package)_clang_download_path=https://releases.llvm.org/$($(package)_clang_version)
 $(package)_clang_download_file=clang+llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz
 $(package)_clang_file_name=clang-llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz
 $(package)_clang_sha256_hash=9ef854b71949f825362a119bf2597f744836cb571131ae6b721cd102ffea8cd0
+endif
 
 $(package)_libtapi_version=3efb201881e7a76a21e0554906cf306432539cef
 $(package)_libtapi_download_path=https://github.com/tpoechtrager/apple-libtapi/archive
@@ -16,15 +18,25 @@ $(package)_libtapi_download_file=$($(package)_libtapi_version).tar.gz
 $(package)_libtapi_file_name=$($(package)_libtapi_version).tar.gz
 $(package)_libtapi_sha256_hash=380c1ca37cfa04a8699d0887a8d3ee1ad27f3d08baba78887c73b09485c0fbd3
 
-$(package)_extra_sources=$($(package)_clang_file_name)
-$(package)_extra_sources += $($(package)_libtapi_file_name)
+$(package)_extra_sources=$($(package)_libtapi_file_name)
+ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
+$(package)_extra_sources += $($(package)_clang_file_name)
+endif
 
+ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
 define $(package)_fetch_cmds
 $(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \
 $(call fetch_file,$(package),$($(package)_clang_download_path),$($(package)_clang_download_file),$($(package)_clang_file_name),$($(package)_clang_sha256_hash)) && \
 $(call fetch_file,$(package),$($(package)_libtapi_download_path),$($(package)_libtapi_download_file),$($(package)_libtapi_file_name),$($(package)_libtapi_sha256_hash))
 endef
+else
+define $(package)_fetch_cmds
+$(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \
+$(call fetch_file,$(package),$($(package)_libtapi_download_path),$($(package)_libtapi_download_file),$($(package)_libtapi_file_name),$($(package)_libtapi_sha256_hash))
+endef
+endif
 
+ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
 define $(package)_extract_cmds
   mkdir -p $($(package)_extract_dir) && \
   echo "$($(package)_sha256_hash)  $($(package)_source)" > $($(package)_extract_dir)/.$($(package)_file_name).hash && \
@@ -38,12 +50,28 @@ define $(package)_extract_cmds
   rm -f toolchain/lib/libc++abi.so* && \
   tar --no-same-owner --strip-components=1 -xf $($(package)_source)
 endef
+else
+define $(package)_extract_cmds
+  mkdir -p $($(package)_extract_dir) && \
+  echo "$($(package)_sha256_hash)  $($(package)_source)" > $($(package)_extract_dir)/.$($(package)_file_name).hash && \
+  echo "$($(package)_libtapi_sha256_hash)  $($(package)_source_dir)/$($(package)_libtapi_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \
+  $(build_SHA256SUM) -c $($(package)_extract_dir)/.$($(package)_file_name).hash && \
+  mkdir -p libtapi && \
+  tar --no-same-owner --strip-components=1 -C libtapi -xf $($(package)_source_dir)/$($(package)_libtapi_file_name) && \
+  tar --no-same-owner --strip-components=1 -xf $($(package)_source)
+endef
+endif
 
 define $(package)_set_vars
   $(package)_config_opts=--target=$(host) --disable-lto-support --with-libtapi=$($(package)_extract_dir)
   $(package)_ldflags+=-Wl,-rpath=\\$$$$$$$$\$$$$$$$$ORIGIN/../lib
+  ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
   $(package)_cc=$($(package)_extract_dir)/toolchain/bin/clang
   $(package)_cxx=$($(package)_extract_dir)/toolchain/bin/clang++
+  else
+  $(package)_cc=clang
+  $(package)_cxx=clang++
+  endif
 endef
 
 define $(package)_preprocess_cmds
@@ -60,6 +88,7 @@ define $(package)_build_cmds
   $(MAKE)
 endef
 
+ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
 define $(package)_stage_cmds
   $(MAKE) DESTDIR=$($(package)_staging_dir) install && \
   mkdir -p $($(package)_staging_prefix_dir)/lib/ && \
@@ -74,3 +103,11 @@ define $(package)_stage_cmds
   cp -rf lib/clang/$($(package)_clang_version)/include/* $($(package)_staging_prefix_dir)/lib/clang/$($(package)_clang_version)/include/ && \
   cp bin/dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil
 endef
+else
+define $(package)_stage_cmds
+  $(MAKE) DESTDIR=$($(package)_staging_dir) install && \
+  mkdir -p $($(package)_staging_prefix_dir)/lib/ && \
+  cd $($(package)_extract_dir) && \
+  cp lib/libtapi.so.6 $($(package)_staging_prefix_dir)/lib/
+endef
+endif