Merge #21457: build: split libtapi and clang out of native_cctools

765e0be5347e9d85cee03255204d94119e4cbf38 build: split native_cctools (fanquake)

Pull request description:

  This splits our native cctools package into two additional packages: `native_clang` and `native_libtapi`. This is in an effort to not only make our mac toolchain more understandable, but also to reduce duplication, and as a nice side-effect, fix the issue mentioned [here](https://github.com/bitcoin/bitcoin/pull/19817#issuecomment-741763289).

  Everything about the current build process should remain the same. For gitian, that is:
  * Download LLVM Clang 8.0.0 binary.
  * Build libtapi using downloaded Clang.
  * Build cctools using downloaded Clang and libtapi.
  * Build the rest of depends..

  For Guix (`FORCE_USE_SYSTEM_CLANG=1`):
  * Build libtapi using using system Clang.
  * Build cctools using system Clang and libtapi.
  * Build the rest of depends..

  After this has been merged, my plan is to combine a modified version of #20454 and #21414 with #19817, and  from what I  understand that will be enough to support Apple Arm cross compilation.

  Builds at 477ed59f49f39ae4272219a1216ee19fb72fdde5:
  Guix:
  ```bash
  find output -type f -name *$(git rev-parse --short HEAD)*.* -print0 | env LC_ALL=C sort -z | xargs -r0 sha256sum
  7e57b9e5a2109d1a35f0091d86f975c1b1d73ac70ac2609cefbe1134efbf2c87  output/bitcoin-477ed59f49f3-osx-unsigned.dmg
  dd11e71c2634ac2fa883d1e45cbd6de194fad37624bb56b8b8a6213fd40d6050  output/bitcoin-477ed59f49f3-osx-unsigned.tar.gz
  64384eaa2fd9768992d86a06a1414c9e92e84ba21a875696483df2bb5828e2a2  output/bitcoin-477ed59f49f3-osx64.tar.gz
  8a889e88db952d2c82ef44713c04aba95b777441f578738ff6d8a0d251e51da3  output/src/bitcoin-477ed59f49f3.tar.gz
  ```

  Gitian:
  ```bash
  d0eee8542d5f3d662555ad7218d2dc9f3f862656e65bcb2f01f256bfa0deead2  bitcoin-477ed59f49f3-osx-unsigned.dmg
  ba7bc94897e42e7a037e352c4e4e1730f181c6d76b6d6a2785bbd7bf85614c83  bitcoin-477ed59f49f3-osx-unsigned.tar.gz
  c4426d1d310a2fbffcaf2b7df0da4ec97bd11aab07085006dae68777b03f6372  bitcoin-477ed59f49f3-osx64.tar.gz
  8a889e88db952d2c82ef44713c04aba95b777441f578738ff6d8a0d251e51da3  src/bitcoin-477ed59f49f3.tar.gz
  a746831467dc8ff17ec5df06fc9288a859c1961d8c0b632d97b42f080dbd825d  bitcoin-core-osx-22-res.yml
  ```

ACKs for top commit:
  dongcarl:
    reACK 765e0be5347e9d85cee03255204d94119e4cbf38
  hebasto:
    ACK 765e0be5347e9d85cee03255204d94119e4cbf38, verified building of the `native_cctools` package step-by-step.

Tree-SHA512: 61cf2b092fb8b9724adda1084e0cac9db889cd5e391914b43592aebc470fae3c1cbabc8b59a0abce6e7bad8c646694fe927f26f4deb18b60d7fd92f374f62feb

5 files changed, 63 insertions, 91 deletions

diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk
index 646e97837a..dd71697f0f 100644
--- a/depends/hosts/darwin.mk
+++ b/depends/hosts/darwin.mk
@@ -12,14 +12,13 @@ ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
 # FORCE_USE_SYSTEM_CLANG is empty, so we use our depends-managed, pinned clang
 # from llvm.org
 
-# The native_cctools package is what provides clang when FORCE_USE_SYSTEM_CLANG
-# is empty
+# Clang is a dependency of native_cctools when FORCE_USE_SYSTEM_CLANG is empty
 darwin_native_toolchain=native_cctools
 
 clang_prog=$(build_prefix)/bin/clang
 clangxx_prog=$(clang_prog)++
 
-clang_resource_dir=$(build_prefix)/lib/clang/$(native_cctools_clang_version)
+clang_resource_dir=$(build_prefix)/lib/clang/$(native_clang_version)
 else
 # FORCE_USE_SYSTEM_CLANG is non-empty, so we use the clang from the user's
 # system
diff --git a/depends/packages/native_cctools.mk b/depends/packages/native_cctools.mk
index d56b636695..c789dab74c 100644
--- a/depends/packages/native_cctools.mk
+++ b/depends/packages/native_cctools.mk
@@ -5,81 +5,19 @@ $(package)_file_name=$($(package)_version).tar.gz
 $(package)_sha256_hash=e51995a843533a3dac155dd0c71362dd471597a2d23f13dff194c6285362f875
 $(package)_build_subdir=cctools
 $(package)_patches=ld64_disable_threading.patch
-
-ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
-$(package)_clang_version=8.0.0
-$(package)_clang_download_path=https://releases.llvm.org/$($(package)_clang_version)
-$(package)_clang_download_file=clang+llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz
-$(package)_clang_file_name=clang-llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz
-$(package)_clang_sha256_hash=9ef854b71949f825362a119bf2597f744836cb571131ae6b721cd102ffea8cd0
-endif
-
-$(package)_libtapi_version=3efb201881e7a76a21e0554906cf306432539cef
-$(package)_libtapi_download_path=https://github.com/tpoechtrager/apple-libtapi/archive
-$(package)_libtapi_download_file=$($(package)_libtapi_version).tar.gz
-$(package)_libtapi_file_name=$($(package)_libtapi_version).tar.gz
-$(package)_libtapi_sha256_hash=380c1ca37cfa04a8699d0887a8d3ee1ad27f3d08baba78887c73b09485c0fbd3
-
-$(package)_extra_sources=$($(package)_libtapi_file_name)
-ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
-$(package)_extra_sources += $($(package)_clang_file_name)
-endif
-
-ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
-define $(package)_fetch_cmds
-$(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \
-$(call fetch_file,$(package),$($(package)_clang_download_path),$($(package)_clang_download_file),$($(package)_clang_file_name),$($(package)_clang_sha256_hash)) && \
-$(call fetch_file,$(package),$($(package)_libtapi_download_path),$($(package)_libtapi_download_file),$($(package)_libtapi_file_name),$($(package)_libtapi_sha256_hash))
-endef
-else
-define $(package)_fetch_cmds
-$(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \
-$(call fetch_file,$(package),$($(package)_libtapi_download_path),$($(package)_libtapi_download_file),$($(package)_libtapi_file_name),$($(package)_libtapi_sha256_hash))
-endef
-endif
-
-ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
-define $(package)_extract_cmds
-  mkdir -p $($(package)_extract_dir) && \
-  echo "$($(package)_sha256_hash)  $($(package)_source)" > $($(package)_extract_dir)/.$($(package)_file_name).hash && \
-  echo "$($(package)_clang_sha256_hash)  $($(package)_source_dir)/$($(package)_clang_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \
-  echo "$($(package)_libtapi_sha256_hash)  $($(package)_source_dir)/$($(package)_libtapi_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \
-  $(build_SHA256SUM) -c $($(package)_extract_dir)/.$($(package)_file_name).hash && \
-  mkdir -p toolchain/bin toolchain/lib/clang/$($(package)_clang_version)/include && \
-  mkdir -p libtapi && \
-  tar --no-same-owner --strip-components=1 -C libtapi -xf $($(package)_source_dir)/$($(package)_libtapi_file_name) && \
-  tar --no-same-owner --strip-components=1 -C toolchain -xf $($(package)_source_dir)/$($(package)_clang_file_name) && \
-  rm -f toolchain/lib/libc++abi.so* && \
-  tar --no-same-owner --strip-components=1 -xf $($(package)_source)
-endef
-else
-define $(package)_extract_cmds
-  mkdir -p $($(package)_extract_dir) && \
-  echo "$($(package)_sha256_hash)  $($(package)_source)" > $($(package)_extract_dir)/.$($(package)_file_name).hash && \
-  echo "$($(package)_libtapi_sha256_hash)  $($(package)_source_dir)/$($(package)_libtapi_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \
-  $(build_SHA256SUM) -c $($(package)_extract_dir)/.$($(package)_file_name).hash && \
-  mkdir -p libtapi && \
-  tar --no-same-owner --strip-components=1 -C libtapi -xf $($(package)_source_dir)/$($(package)_libtapi_file_name) && \
-  tar --no-same-owner --strip-components=1 -xf $($(package)_source)
-endef
-endif
+$(package)_dependencies=native_libtapi
 
 define $(package)_set_vars
-  $(package)_config_opts=--target=$(host) --with-libtapi=$($(package)_extract_dir)
+  $(package)_config_opts=--target=$(host)
   $(package)_ldflags+=-Wl,-rpath=\\$$$$$$$$\$$$$$$$$ORIGIN/../lib
   ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
-  $(package)_config_opts+=--enable-lto-support --with-llvm-config=$($(package)_extract_dir)/toolchain/bin/llvm-config
-  $(package)_cc=$($(package)_extract_dir)/toolchain/bin/clang
-  $(package)_cxx=$($(package)_extract_dir)/toolchain/bin/clang++
-  else
-  $(package)_cc=clang
-  $(package)_cxx=clang++
+  $(package)_config_opts+=--enable-lto-support --with-llvm-config=$(build_prefix)/bin/llvm-config
   endif
+  $(package)_cc=$(clang_prog)
+  $(package)_cxx=$(clangxx_prog)
 endef
 
 define $(package)_preprocess_cmds
-  CC=$($(package)_cc) CXX=$($(package)_cxx) INSTALLPREFIX=$($(package)_extract_dir) ./libtapi/build.sh && \
-  CC=$($(package)_cc) CXX=$($(package)_cxx) INSTALLPREFIX=$($(package)_extract_dir) ./libtapi/install.sh && \
   patch -p1 < $($(package)_patch_dir)/ld64_disable_threading.patch
 endef
 
@@ -91,26 +29,10 @@ define $(package)_build_cmds
   $(MAKE)
 endef
 
-ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
 define $(package)_stage_cmds
-  $(MAKE) DESTDIR=$($(package)_staging_dir) install && \
-  mkdir -p $($(package)_staging_prefix_dir)/lib/ && \
-  cd $($(package)_extract_dir) && \
-  cp lib/libtapi.so.6 $($(package)_staging_prefix_dir)/lib/ && \
-  cd $($(package)_extract_dir)/toolchain && \
-  mkdir -p $($(package)_staging_prefix_dir)/lib/clang/$($(package)_clang_version)/include && \
-  mkdir -p $($(package)_staging_prefix_dir)/bin $($(package)_staging_prefix_dir)/include && \
-  cp bin/clang $($(package)_staging_prefix_dir)/bin/ &&\
-  cp -P bin/clang++ $($(package)_staging_prefix_dir)/bin/ &&\
-  cp lib/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \
-  cp -rf lib/clang/$($(package)_clang_version)/include/* $($(package)_staging_prefix_dir)/lib/clang/$($(package)_clang_version)/include/ && \
-  cp bin/dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil
+  $(MAKE) DESTDIR=$($(package)_staging_dir) install
 endef
-else
-define $(package)_stage_cmds
-  $(MAKE) DESTDIR=$($(package)_staging_dir) install && \
-  mkdir -p $($(package)_staging_prefix_dir)/lib/ && \
-  cd $($(package)_extract_dir) && \
-  cp lib/libtapi.so.6 $($(package)_staging_prefix_dir)/lib/
+
+define $(package)_postprocess_cmds
+  rm -rf share
 endef
-endif
diff --git a/depends/packages/native_clang.mk b/depends/packages/native_clang.mk
new file mode 100644
index 0000000000..741ddacf09
--- /dev/null
+++ b/depends/packages/native_clang.mk
@@ -0,0 +1,26 @@
+package=native_clang
+$(package)_version=8.0.0
+$(package)_download_path=https://releases.llvm.org/$($(package)_version)
+$(package)_download_file=clang+llvm-$($(package)_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz
+$(package)_file_name=clang-llvm-$($(package)_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz
+$(package)_sha256_hash=9ef854b71949f825362a119bf2597f744836cb571131ae6b721cd102ffea8cd0
+
+define $(package)_preprocess_cmds
+  rm -f $($(package)_extract_dir)/lib/libc++abi.so*
+endef
+
+define $(package)_stage_cmds
+  mkdir -p $($(package)_staging_prefix_dir)/lib/clang/$($(package)_version)/include && \
+  mkdir -p $($(package)_staging_prefix_dir)/bin && \
+  mkdir -p $($(package)_staging_prefix_dir)/include && \
+  cp bin/clang $($(package)_staging_prefix_dir)/bin/ && \
+  cp -P bin/clang++ $($(package)_staging_prefix_dir)/bin/ && \
+  cp bin/dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil && \
+  cp bin/llvm-config $($(package)_staging_prefix_dir)/bin/ && \
+  cp lib/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \
+  cp -rf lib/clang/$($(package)_version)/include/* $($(package)_staging_prefix_dir)/lib/clang/$($(package)_version)/include/
+endef
+
+define $(package)_postprocess_cmds
+  rmdir include
+endef
diff --git a/depends/packages/native_libtapi.mk b/depends/packages/native_libtapi.mk
new file mode 100644
index 0000000000..d7ac4156a7
--- /dev/null
+++ b/depends/packages/native_libtapi.mk
@@ -0,0 +1,20 @@
+package=native_libtapi
+$(package)_version=3efb201881e7a76a21e0554906cf306432539cef
+$(package)_download_path=https://github.com/tpoechtrager/apple-libtapi/archive
+$(package)_download_file=$($(package)_version).tar.gz
+$(package)_file_name=$($(package)_version).tar.gz
+$(package)_sha256_hash=380c1ca37cfa04a8699d0887a8d3ee1ad27f3d08baba78887c73b09485c0fbd3
+
+ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
+$(package)_dependencies=native_clang
+endif
+
+define $(package)_build_cmds
+  CC=$(clang_prog) CXX=$(clangxx_prog) INSTALLPREFIX=$($(package)_staging_prefix_dir) ./build.sh
+endef
+
+define $(package)_stage_cmds
+  ./install.sh && \
+  mkdir -p $($(package)_staging_prefix_dir)/include/llvm-c && \
+  cp src/llvm/include/llvm-c/lto.h $($(package)_staging_prefix_dir)/include/llvm-c
+endef
diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk
index 810068b686..9094e327ef 100644
--- a/depends/packages/packages.mk
+++ b/depends/packages/packages.mk
@@ -24,5 +24,10 @@ darwin_native_packages = native_ds_store native_mac_alias
 $(host_arch)_$(host_os)_native_packages += native_b2
 
 ifneq ($(build_os),darwin)
-darwin_native_packages += native_cctools native_libdmg-hfsplus
+darwin_native_packages += native_cctools native_libtapi native_libdmg-hfsplus
+
+ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
+darwin_native_packages+= native_clang
+endif
+
 endif