aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorCory Fields <cory-nospam-@coryfields.com>2015-05-13 21:12:04 -0400
committerCory Fields <cory-nospam-@coryfields.com>2015-05-14 01:33:07 -0400
commit235b3a789d6c54b042a241ffcfaba4404f6245fa (patch)
tree3e96e49c64c89b5ad44389268dd5756ee1095560
parent351f73ecd5d44bf6b80f45ab2e1aaa25a08c5c8c (diff)
depends: sanity-check sources and cached builds
In some cases (Travis), sources and build caches may be moved around in-between builds, and we can't necessarily trust that everything is still intact. This introduces pre-build checks that verify against stashed checksums. Note that this will cause all sources to be re-downloaded, since cached sources weren't trustworthy before this.
-rw-r--r--depends/Makefile40
-rw-r--r--depends/README.packages4
-rw-r--r--depends/funcs.mk13
-rw-r--r--depends/packages/native_cctools.mk2
4 files changed, 49 insertions, 10 deletions
diff --git a/depends/Makefile b/depends/Makefile
index 05ef33f2ee..ef5a20e6c3 100644
--- a/depends/Makefile
+++ b/depends/Makefile
@@ -91,12 +91,12 @@ include funcs.mk
toolchain_path=$($($(host_arch)_$(host_os)_native_toolchain)_prefixbin)
final_build_id_long+=$(shell $(build_SHA256SUM) config.site.in)
final_build_id+=$(shell echo -n $(final_build_id_long) | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH))
-$(host_prefix)/.stamp_$(final_build_id): | $(native_packages) $(packages)
+$(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages)
$(AT)rm -rf $(@D)
$(AT)mkdir -p $(@D)
- $(AT)echo copying packages: $|
+ $(AT)echo copying packages: $^
$(AT)echo to: $(@D)
- $(AT)cd $(@D); $(foreach package,$|, tar xf $($(package)_cached); )
+ $(AT)cd $(@D); $(foreach package,$^, tar xf $($(package)_cached); )
$(AT)touch $@
$(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_build_id)
@@ -121,8 +121,35 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_
$< > $@
$(AT)touch $@
-install: $(host_prefix)/share/config.site
-download-one: $(all_sources)
+
+define check_or_remove_cached
+ mkdir -p $(BASE_CACHE)/$(host)/$(package) && cd $(BASE_CACHE)/$(host)/$(package); \
+ $(build_SHA256SUM) -c $($(package)_cached_checksum) >/dev/null 2>/dev/null || \
+ ( rm -f $($(package)_cached_checksum); \
+ if test -f "$($(package)_cached)"; then echo "Checksum mismatch for $(package). Forcing rebuild.."; rm -f $($(package)_cached_checksum) $($(package)_cached); fi )
+endef
+
+define check_or_remove_sources
+ mkdir -p $($(package)_source_dir); cd $($(package)_source_dir); \
+ $(build_SHA256SUM) -c $($(package)_fetched) >/dev/null 2>/dev/null || \
+ ( if test -f $($(package)_all_sources); then echo "Checksum missing or mismatched for $(package) source. Forcing re-download."; fi; \
+ rm -f $($(package)_all_sources) $($(1)_fetched))
+endef
+
+check-packages:
+ @$(foreach package,$(all_packages),$(call check_or_remove_cached,$(package));)
+check-sources:
+ @$(foreach package,$(all_packages),$(call check_or_remove_sources,$(package));)
+
+$(host_prefix)/share/config.site: check-packages
+
+check-packages: check-sources
+
+install: check-packages $(host_prefix)/share/config.site
+
+
+download-one: check-sources $(all_sources)
+
download-osx:
@$(MAKE) -s HOST=x86_64-apple-darwin11 download-one
download-linux:
@@ -130,4 +157,5 @@ download-linux:
download-win:
@$(MAKE) -s HOST=x86_64-w64-mingw32 download-one
download: download-osx download-linux download-win
-.PHONY: install cached download-one download-osx download-linux download-win download
+
+.PHONY: install cached download-one download-osx download-linux download-win download check-packages check-sources
diff --git a/depends/README.packages b/depends/README.packages
index 5ab7ed7dee..6d87402ebc 100644
--- a/depends/README.packages
+++ b/depends/README.packages
@@ -30,7 +30,9 @@ These variables are optional:
Names of any other packages that this one depends on.
$(package)_patches:
Filenames of any patches needed to build the package
-
+ $(package)_extra_sources:
+ Any extra files that will be fetched via $(package)_fetch_cmds. These are
+ specified so that they can be fetched and verified via 'make download'.
Build Variables:
After defining the main identifiers, build variables may be added or customized
diff --git a/depends/funcs.mk b/depends/funcs.mk
index b407737f7f..050a9b1321 100644
--- a/depends/funcs.mk
+++ b/depends/funcs.mk
@@ -53,12 +53,14 @@ $(1)_staging_prefix_dir:=$$($(1)_staging_dir)$($($(1)_type)_prefix)
$(1)_extract_dir:=$(base_build_dir)/$(host)/$(1)/$($(1)_version)-$($(1)_build_id)
$(1)_download_dir:=$(base_download_dir)/$(1)-$($(1)_version)
$(1)_build_dir:=$$($(1)_extract_dir)/$$($(1)_build_subdir)
+$(1)_cached_checksum:=$(BASE_CACHE)/$(host)/$(1)/$(1)-$($(1)_version)-$($(1)_build_id).tar.gz.hash
$(1)_patch_dir:=$(base_build_dir)/$(host)/$(1)/$($(1)_version)-$($(1)_build_id)/.patches-$($(1)_build_id)
$(1)_prefixbin:=$($($(1)_type)_prefix)/bin/
$(1)_cached:=$(BASE_CACHE)/$(host)/$(1)/$(1)-$($(1)_version)-$($(1)_build_id).tar.gz
+$(1)_all_sources=$($(1)_file_name) $($(1)_extra_sources)
#stamps
-$(1)_fetched=$$($(1)_source_dir)/download-stamps/.stamp_fetched-$(1)-$($(1)_file_name)
+$(1)_fetched=$(SOURCES_PATH)/download-stamps/.stamp_fetched-$(1)-$($(1)_file_name).hash
$(1)_extracted=$$($(1)_extract_dir)/.stamp_extracted
$(1)_preprocessed=$$($(1)_extract_dir)/.stamp_preprocessed
$(1)_cleaned=$$($(1)_extract_dir)/.stamp_cleaned
@@ -154,7 +156,10 @@ endef
define int_add_cmds
$($(1)_fetched):
$(AT)mkdir -p $$(@D) $(SOURCES_PATH)
+ $(AT)rm -f $$@
+ $(AT)touch $$@
$(AT)cd $$(@D); $(call $(1)_fetch_cmds,$(1))
+ $(AT)cd $($(1)_source_dir); $(foreach source,$($(1)_all_sources),$(build_SHA256SUM) $(source) >> $$(@);)
$(AT)touch $$@
$($(1)_extracted): | $($(1)_fetched)
$(AT)echo Extracting $(1)...
@@ -195,10 +200,12 @@ $($(1)_cached): | $($(1)_dependencies) $($(1)_postprocessed)
$(AT)rm -rf $$(@D) && mkdir -p $$(@D)
$(AT)mv $$($(1)_staging_dir)/$$(@F) $$(@)
$(AT)rm -rf $($(1)_staging_dir)
+$($(1)_cached_checksum): $($(1)_cached)
+ $(AT)cd $$(@D); $(build_SHA256SUM) $$(<F) > $$(@)
.PHONY: $(1)
-$(1): | $($(1)_cached)
-.SECONDARY: $($(1)_postprocessed) $($(1)_staged) $($(1)_built) $($(1)_configured) $($(1)_preprocessed) $($(1)_extracted) $($(1)_fetched)
+$(1): | $($(1)_cached_checksum)
+.SECONDARY: $($(1)_cached) $($(1)_postprocessed) $($(1)_staged) $($(1)_built) $($(1)_configured) $($(1)_preprocessed) $($(1)_extracted) $($(1)_fetched)
endef
diff --git a/depends/packages/native_cctools.mk b/depends/packages/native_cctools.mk
index 951ad4fb29..1c1bcf199a 100644
--- a/depends/packages/native_cctools.mk
+++ b/depends/packages/native_cctools.mk
@@ -9,6 +9,8 @@ $(package)_clang_download_path=http://llvm.org/releases/$($(package)_clang_versi
$(package)_clang_download_file=clang+llvm-$($(package)_clang_version)-amd64-Ubuntu-12.04.2.tar.gz
$(package)_clang_file_name=clang-llvm-$($(package)_clang_version)-amd64-Ubuntu-12.04.2.tar.gz
$(package)_clang_sha256_hash=60d8f69f032d62ef61bf527857ebb933741ec3352d4d328c5516aa520662dab7
+$(package)_extra_sources=$($(package)_clang_file_name)
+
define $(package)_fetch_cmds
$(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \
$(call fetch_file,$(package),$($(package)_clang_download_path),$($(package)_clang_download_file),$($(package)_clang_file_name),$($(package)_clang_sha256_hash))