diff options
Diffstat (limited to 'depends')
-rw-r--r-- | depends/Makefile | 40 | ||||
-rw-r--r-- | depends/README.md | 56 | ||||
-rw-r--r-- | depends/README.usage | 34 | ||||
-rw-r--r-- | depends/description.md (renamed from depends/README) | 18 | ||||
-rw-r--r-- | depends/funcs.mk | 13 | ||||
-rw-r--r-- | depends/packages.md (renamed from depends/README.packages) | 129 | ||||
-rw-r--r-- | depends/packages/native_cctools.mk | 2 |
7 files changed, 184 insertions, 108 deletions
diff --git a/depends/Makefile b/depends/Makefile index 05ef33f2ee..ef5a20e6c3 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -91,12 +91,12 @@ include funcs.mk toolchain_path=$($($(host_arch)_$(host_os)_native_toolchain)_prefixbin) final_build_id_long+=$(shell $(build_SHA256SUM) config.site.in) final_build_id+=$(shell echo -n $(final_build_id_long) | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH)) -$(host_prefix)/.stamp_$(final_build_id): | $(native_packages) $(packages) +$(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages) $(AT)rm -rf $(@D) $(AT)mkdir -p $(@D) - $(AT)echo copying packages: $| + $(AT)echo copying packages: $^ $(AT)echo to: $(@D) - $(AT)cd $(@D); $(foreach package,$|, tar xf $($(package)_cached); ) + $(AT)cd $(@D); $(foreach package,$^, tar xf $($(package)_cached); ) $(AT)touch $@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_build_id) @@ -121,8 +121,35 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_ $< > $@ $(AT)touch $@ -install: $(host_prefix)/share/config.site -download-one: $(all_sources) + +define check_or_remove_cached + mkdir -p $(BASE_CACHE)/$(host)/$(package) && cd $(BASE_CACHE)/$(host)/$(package); \ + $(build_SHA256SUM) -c $($(package)_cached_checksum) >/dev/null 2>/dev/null || \ + ( rm -f $($(package)_cached_checksum); \ + if test -f "$($(package)_cached)"; then echo "Checksum mismatch for $(package). Forcing rebuild.."; rm -f $($(package)_cached_checksum) $($(package)_cached); fi ) +endef + +define check_or_remove_sources + mkdir -p $($(package)_source_dir); cd $($(package)_source_dir); \ + $(build_SHA256SUM) -c $($(package)_fetched) >/dev/null 2>/dev/null || \ + ( if test -f $($(package)_all_sources); then echo "Checksum missing or mismatched for $(package) source. Forcing re-download."; fi; \ + rm -f $($(package)_all_sources) $($(1)_fetched)) +endef + +check-packages: + @$(foreach package,$(all_packages),$(call check_or_remove_cached,$(package));) +check-sources: + @$(foreach package,$(all_packages),$(call check_or_remove_sources,$(package));) + +$(host_prefix)/share/config.site: check-packages + +check-packages: check-sources + +install: check-packages $(host_prefix)/share/config.site + + +download-one: check-sources $(all_sources) + download-osx: @$(MAKE) -s HOST=x86_64-apple-darwin11 download-one download-linux: @@ -130,4 +157,5 @@ download-linux: download-win: @$(MAKE) -s HOST=x86_64-w64-mingw32 download-one download: download-osx download-linux download-win -.PHONY: install cached download-one download-osx download-linux download-win download + +.PHONY: install cached download-one download-osx download-linux download-win download check-packages check-sources diff --git a/depends/README.md b/depends/README.md new file mode 100644 index 0000000000..2dc0b9e47e --- /dev/null +++ b/depends/README.md @@ -0,0 +1,56 @@ +### Usage + +To build dependencies for the current arch+OS: + + make + +To build for another arch/OS: + + make HOST=host-platform-triplet + +For example: + + make HOST=x86_64-w64-mingw32 -j4 + +A prefix will be generated that's suitable for plugging into Bitcoin's +configure. In the above example, a dir named i686-w64-mingw32 will be +created. To use it for Bitcoin: + + ./configure --prefix=`pwd`/depends/x86_64-w64-mingw32 + +Common `host-platform-triplets` for cross compilation are: + +- `i686-w64-mingw32` for Win32 +- `x86_64-w64-mingw32` for Win64 +- `x86_64-apple-darwin11` for MacOSX +- `arm-linux-gnueabihf` for Linux ARM + +No other options are needed, the paths are automatically configured. + +Dependency Options: +The following can be set when running make: make FOO=bar + + SOURCES_PATH: downloaded sources will be placed here + BASE_CACHE: built packages will be placed here + SDK_PATH: Path where sdk's can be found (used by OSX) + FALLBACK_DOWNLOAD_PATH: If a source file can't be fetched, try here before giving up + NO_QT: Don't download/build/cache qt and its dependencies + NO_WALLET: Don't download/build/cache libs needed to enable the wallet + NO_UPNP: Don't download/build/cache packages needed for enabling upnp + DEBUG: disable some optimizations and enable more runtime checking + +If some packages are not built, for example `make NO_WALLET=1`, the appropriate +options will be passed to bitcoin's configure. In this case, `--disable-wallet`. + +Additional targets: + + download: run 'make download' to fetch all sources without building them + download-osx: run 'make download-osx' to fetch all sources needed for osx builds + download-win: run 'make download-win' to fetch all sources needed for win builds + download-linux: run 'make download-linux' to fetch all sources needed for linux builds + +### Other documentation + +- [description.md](description.md): General description of the depends system +- [packages.md](packages.md): Steps for adding packages + diff --git a/depends/README.usage b/depends/README.usage deleted file mode 100644 index 24e1231d82..0000000000 --- a/depends/README.usage +++ /dev/null @@ -1,34 +0,0 @@ -To build dependencies for the current arch+OS: - make -To build for another arch/OS: - make HOST=host-platform-triplet && make HOST=host-platform-triplet - (For example: make HOST=i686-w64-mingw32 -j4) - -A prefix will be generated that's suitable for plugging into Bitcoin's -configure. In the above example, a dir named i686-w64-mingw32 will be -created. To use it for Bitcoin: - -./configure --prefix=`pwd`/depends/i686-w64-mingw32 - -No other options are needed, the paths are automatically configured. - -Dependency Options: -The following can be set when running make: make FOO=bar - -SOURCES_PATH: downloaded sources will be placed here -BASE_CACHE: built packages will be placed here -SDK_PATH: Path where sdk's can be found (used by OSX) -FALLBACK_DOWNLOAD_PATH: If a source file can't be fetched, try here before giving up -NO_QT: Don't download/build/cache qt and its dependencies -NO_WALLET: Don't download/build/cache libs needed to enable the wallet -NO_UPNP: Don't download/build/cache packages needed for enabling upnp -DEBUG: disable some optimizations and enable more runtime checking - -If some packages are not built, for example 'make NO_WALLET=1', the appropriate -options will be passed to bitcoin's configure. In this case, --disable-wallet. - -Additional targets: -download: run 'make download' to fetch all sources without building them -download-osx: run 'make download-osx' to fetch all sources needed for osx builds -download-win: run 'make download-win' to fetch all sources needed for win builds -download-linux: run 'make download-linux' to fetch all sources needed for linux builds diff --git a/depends/README b/depends/description.md index 55e7222697..74f9ef3f20 100644 --- a/depends/README +++ b/depends/description.md @@ -1,9 +1,7 @@ -This is a system of building and caching dependencies necessary for building -Bitcoin. - +This is a system of building and caching dependencies necessary for building Bitcoin. There are several features that make it different from most similar systems: -- It is designed to be builder and host agnostic +### It is designed to be builder and host agnostic In theory, binaries for any target OS/architecture can be created, from a builder running any OS/architecture. In practice, build-side tools must be @@ -11,18 +9,18 @@ specified when the defaults don't fit, and packages must be amended to work on new hosts. For now, a build architecture of x86_64 is assumed, either on Linux or OSX. -- No reliance on timestamps +### No reliance on timestamps File presence is used to determine what needs to be built. This makes the results distributable and easily digestable by automated builders. -- Each build only has its specified dependencies available at build-time. +### Each build only has its specified dependencies available at build-time. For each build, the sysroot is wiped and the (recursive) dependencies are installed. This makes each build deterministic, since there will never be any unknown files available to cause side-effects. -- Each package is cached and only rebuilt as needed. +### Each package is cached and only rebuilt as needed. Before building, a unique build-id is generated for each package. This id consists of a hash of all files used to build the package (Makefiles, packages, @@ -32,7 +30,7 @@ any other package that depends on it. If any of the main makefiles (Makefile, funcs.mk, etc) are changed, all packages will be rebuilt. After building, the results are cached into a tarball that can be re-used and distributed. -- Package build results are (relatively) deterministic. +### Package build results are (relatively) deterministic. Each package is configured and patched so that it will yield the same build-results with each consequent build, within a reasonable set of @@ -41,13 +39,13 @@ beyond the scope of this system. Additionally, the toolchain itself must be capable of deterministic results. When revisions are properly bumped, a cached build should represent an exact single payload. -- Sources are fetched and verified automatically +### Sources are fetched and verified automatically Each package must define its source location and checksum. The build will fail if the fetched source does not match. Sources may be pre-seeded and/or cached as desired. -- Self-cleaning +### Self-cleaning Build and staging dirs are wiped after use, and any previous version of a cached result is removed following a successful build. Automated builders diff --git a/depends/funcs.mk b/depends/funcs.mk index b407737f7f..050a9b1321 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -53,12 +53,14 @@ $(1)_staging_prefix_dir:=$$($(1)_staging_dir)$($($(1)_type)_prefix) $(1)_extract_dir:=$(base_build_dir)/$(host)/$(1)/$($(1)_version)-$($(1)_build_id) $(1)_download_dir:=$(base_download_dir)/$(1)-$($(1)_version) $(1)_build_dir:=$$($(1)_extract_dir)/$$($(1)_build_subdir) +$(1)_cached_checksum:=$(BASE_CACHE)/$(host)/$(1)/$(1)-$($(1)_version)-$($(1)_build_id).tar.gz.hash $(1)_patch_dir:=$(base_build_dir)/$(host)/$(1)/$($(1)_version)-$($(1)_build_id)/.patches-$($(1)_build_id) $(1)_prefixbin:=$($($(1)_type)_prefix)/bin/ $(1)_cached:=$(BASE_CACHE)/$(host)/$(1)/$(1)-$($(1)_version)-$($(1)_build_id).tar.gz +$(1)_all_sources=$($(1)_file_name) $($(1)_extra_sources) #stamps -$(1)_fetched=$$($(1)_source_dir)/download-stamps/.stamp_fetched-$(1)-$($(1)_file_name) +$(1)_fetched=$(SOURCES_PATH)/download-stamps/.stamp_fetched-$(1)-$($(1)_file_name).hash $(1)_extracted=$$($(1)_extract_dir)/.stamp_extracted $(1)_preprocessed=$$($(1)_extract_dir)/.stamp_preprocessed $(1)_cleaned=$$($(1)_extract_dir)/.stamp_cleaned @@ -154,7 +156,10 @@ endef define int_add_cmds $($(1)_fetched): $(AT)mkdir -p $$(@D) $(SOURCES_PATH) + $(AT)rm -f $$@ + $(AT)touch $$@ $(AT)cd $$(@D); $(call $(1)_fetch_cmds,$(1)) + $(AT)cd $($(1)_source_dir); $(foreach source,$($(1)_all_sources),$(build_SHA256SUM) $(source) >> $$(@);) $(AT)touch $$@ $($(1)_extracted): | $($(1)_fetched) $(AT)echo Extracting $(1)... @@ -195,10 +200,12 @@ $($(1)_cached): | $($(1)_dependencies) $($(1)_postprocessed) $(AT)rm -rf $$(@D) && mkdir -p $$(@D) $(AT)mv $$($(1)_staging_dir)/$$(@F) $$(@) $(AT)rm -rf $($(1)_staging_dir) +$($(1)_cached_checksum): $($(1)_cached) + $(AT)cd $$(@D); $(build_SHA256SUM) $$(<F) > $$(@) .PHONY: $(1) -$(1): | $($(1)_cached) -.SECONDARY: $($(1)_postprocessed) $($(1)_staged) $($(1)_built) $($(1)_configured) $($(1)_preprocessed) $($(1)_extracted) $($(1)_fetched) +$(1): | $($(1)_cached_checksum) +.SECONDARY: $($(1)_cached) $($(1)_postprocessed) $($(1)_staged) $($(1)_built) $($(1)_configured) $($(1)_preprocessed) $($(1)_extracted) $($(1)_fetched) endef diff --git a/depends/README.packages b/depends/packages.md index 5ab7ed7dee..7c80362509 100644 --- a/depends/README.packages +++ b/depends/packages.md @@ -4,119 +4,137 @@ variables, and defining build commands. The package "mylib" will be used here as an example General tips: -mylib_foo is written as $(package)_foo in order to make recipes more similar. +- mylib_foo is written as $(package)_foo in order to make recipes more similar. -Identifiers: +## Identifiers Each package is required to define at least these variables: - $(package)_version: + + $(package)_version: Version of the upstream library or program. If there is no version, a placeholder such as 1.0 can be used. - $(package)_download_path: + + $(package)_download_path: Location of the upstream source, without the file-name. Usually http or ftp. - $(package)_file_name: + + $(package)_file_name: The upstream source filename available at the download path. - $(package)_sha256_hash: + + $(package)_sha256_hash: The sha256 hash of the upstream file These variables are optional: - $(package)_build_subdir: + + $(package)_build_subdir: cd to this dir before running configure/build/stage commands. - $(package)_download_file: + + $(package)_download_file: The file-name of the upstream source if it differs from how it should be stored locally. This can be used to avoid storing file-names with strange characters. - $(package)_dependencies: + + $(package)_dependencies: Names of any other packages that this one depends on. - $(package)_patches: + + $(package)_patches: Filenames of any patches needed to build the package + $(package)_extra_sources: + Any extra files that will be fetched via $(package)_fetch_cmds. These are + specified so that they can be fetched and verified via 'make download'. + -Build Variables: +## Build Variables: After defining the main identifiers, build variables may be added or customized before running the build commands. They should be added to a function called $(package)_set_vars. For example: -define $(package)_set_vars -... -endef + define $(package)_set_vars + ... + endef Most variables can be prefixed with the host, architecture, or both, to make the modifications specific to that case. For example: - Universal: $(package)_cc=gcc - Linux only: $(package)_linux_cc=gcc - x86_64 only: $(package)_x86_64_cc = gcc - x86_64 linux only: $(package)_x86_64_linux_cc = gcc + Universal: $(package)_cc=gcc + Linux only: $(package)_linux_cc=gcc + x86_64 only: $(package)_x86_64_cc = gcc + x86_64 linux only: $(package)_x86_64_linux_cc = gcc These variables may be set to override or append their default values. - $(package)_cc - $(package)_cxx - $(package)_objc - $(package)_objcxx - $(package)_ar - $(package)_ranlib - $(package)_libtool - $(package)_nm - $(package)_cflags - $(package)_cxxflags - $(package)_ldflags - $(package)_cppflags - $(package)_config_env - $(package)_build_env - $(package)_stage_env - $(package)_build_opts - $(package)_config_opts + + $(package)_cc + $(package)_cxx + $(package)_objc + $(package)_objcxx + $(package)_ar + $(package)_ranlib + $(package)_libtool + $(package)_nm + $(package)_cflags + $(package)_cxxflags + $(package)_ldflags + $(package)_cppflags + $(package)_config_env + $(package)_build_env + $(package)_stage_env + $(package)_build_opts + $(package)_config_opts The *_env variables are used to add environment variables to the respective commands. Many variables respect a debug/release suffix as well, in order to use them for only the appropriate build config. For example: - $(package)_cflags_release = -O3 - $(package)_cflags_i686_debug = -g - $(package)_config_opts_release = --disable-debug + + $(package)_cflags_release = -O3 + $(package)_cflags_i686_debug = -g + $(package)_config_opts_release = --disable-debug These will be used in addition to the options that do not specify debug/release. All builds are considered to be release unless DEBUG=1 is set by -the user. +the user. Other variables may be defined as needed. -Other variables may be defined as needed. - -Build commands: +## Build commands: For each build, a unique build dir and staging dir are created. For example, - work/build/mylib/1.0-1adac830f6e and work/staging/mylib/1.0-1adac830f6e. + `work/build/mylib/1.0-1adac830f6e` and `work/staging/mylib/1.0-1adac830f6e`. The following build commands are available for each recipe: - $(package)_fetch_cmds: + $(package)_fetch_cmds: Runs from: build dir Fetch the source file. If undefined, it will be fetched and verified against its hash. - $(package)_extract_cmds: + + $(package)_extract_cmds: Runs from: build dir Verify the source file against its hash and extract it. If undefined, the source is assumed to be a tarball. - $(package)_preprocess_cmds: + + $(package)_preprocess_cmds: Runs from: build dir/$(package)_build_subdir Preprocess the source as necessary. If undefined, does nothing. - $(package)_config_cmds: + + $(package)_config_cmds: Runs from: build dir/$(package)_build_subdir Configure the source. If undefined, does nothing. - $(package)_build_cmds: + + $(package)_build_cmds: Runs from: build dir/$(package)_build_subdir Build the source. If undefined, does nothing. - $(package)_stage_cmds: + + $(package)_stage_cmds: Runs from: build dir/$(package)_build_subdir Stage the build results. If undefined, does nothing. The following variables are available for each recipe: - $(1)_staging_dir: package's destination sysroot path - $(1)_staging_prefix_dir: prefix path inside of the package's staging dir - $(1)_extract_dir: path to the package's extracted sources - $(1)_build_dir: path where configure/build/stage commands will be run - $(1)_patch_dir: path where the package's patches (if any) are found + + $(1)_staging_dir: package's destination sysroot path + $(1)_staging_prefix_dir: prefix path inside of the package's staging dir + $(1)_extract_dir: path to the package's extracted sources + $(1)_build_dir: path where configure/build/stage commands will be run + $(1)_patch_dir: path where the package's patches (if any) are found Notes on build commands: @@ -125,4 +143,5 @@ configure step to (usually) correctly configure automatically. Any $($(package)_config_opts) will be appended. Most autotools projects can be properly staged using: - $(MAKE) DESTDIR=$($(package)_staging_dir) install + + $(MAKE) DESTDIR=$($(package)_staging_dir) install diff --git a/depends/packages/native_cctools.mk b/depends/packages/native_cctools.mk index 951ad4fb29..1c1bcf199a 100644 --- a/depends/packages/native_cctools.mk +++ b/depends/packages/native_cctools.mk @@ -9,6 +9,8 @@ $(package)_clang_download_path=http://llvm.org/releases/$($(package)_clang_versi $(package)_clang_download_file=clang+llvm-$($(package)_clang_version)-amd64-Ubuntu-12.04.2.tar.gz $(package)_clang_file_name=clang-llvm-$($(package)_clang_version)-amd64-Ubuntu-12.04.2.tar.gz $(package)_clang_sha256_hash=60d8f69f032d62ef61bf527857ebb933741ec3352d4d328c5516aa520662dab7 +$(package)_extra_sources=$($(package)_clang_file_name) + define $(package)_fetch_cmds $(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \ $(call fetch_file,$(package),$($(package)_clang_download_path),$($(package)_clang_download_file),$($(package)_clang_file_name),$($(package)_clang_sha256_hash)) |