diff options
Diffstat (limited to 'contrib')
| -rw-r--r-- | contrib/README.md | 3 | ||||
| -rw-r--r-- | contrib/devtools/README.md | 3 | ||||
| -rwxr-xr-x | contrib/devtools/fix-copyright-headers.py | 77 | ||||
| -rwxr-xr-x | contrib/gitian-build.sh | 388 | ||||
| -rwxr-xr-x | contrib/macdeploy/extract-osx-sdk.sh | 29 |
5 files changed, 471 insertions, 29 deletions
diff --git a/contrib/README.md b/contrib/README.md index a23b197cc6..3e3c83da5f 100644 --- a/contrib/README.md +++ b/contrib/README.md @@ -45,6 +45,9 @@ Scripts and notes for Mac builds. ### [RPM](/contrib/rpm) ### RPM spec file for building bitcoin-core on RPM based distributions +### [Gitian-build](/contrib/gitian-build.sh) ### +Script for running full gitian builds. + Test and Verify Tools --------------------- diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md index bb8b9246b8..af5c000b03 100644 --- a/contrib/devtools/README.md +++ b/contrib/devtools/README.md @@ -51,8 +51,9 @@ maintained: * for `src/secp256k1`: https://github.com/bitcoin-core/secp256k1.git (branch master) * for `src/leveldb`: https://github.com/bitcoin-core/leveldb.git (branch bitcoin-fork) * for `src/univalue`: https://github.com/bitcoin-core/univalue.git (branch master) +* for `src/crypto/ctaes`: https://github.com/bitcoin-core/ctaes.git (branch master) -Usage: `git-subtree-check.sh DIR COMMIT` +Usage: `git-subtree-check.sh DIR (COMMIT)` `COMMIT` may be omitted, in which case `HEAD` is used. diff --git a/contrib/devtools/fix-copyright-headers.py b/contrib/devtools/fix-copyright-headers.py index b6414a551f..54836bd83f 100755 --- a/contrib/devtools/fix-copyright-headers.py +++ b/contrib/devtools/fix-copyright-headers.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python -''' +#!/usr/bin/env python3 +""" Run this script to update all the copyright headers of files that were changed this year. @@ -10,37 +10,58 @@ For example: it will change it to // Copyright (c) 2009-2015 The Bitcoin Core developers -''' -import os +""" +import subprocess import time import re -year = time.gmtime()[0] -CMD_GIT_DATE = 'git log --format=@%%at -1 %s | date +"%%Y" -u -f -' -CMD_REGEX= "perl -pi -e 's/(20\d\d)(?:-20\d\d)? The Bitcoin/$1-%s The Bitcoin/' %s" -REGEX_CURRENT= re.compile("%s The Bitcoin" % year) -CMD_LIST_FILES= "find %s | grep %s" +CMD_GIT_LIST_FILES = ['git', 'ls-files'] +CMD_GIT_DATE = ['git', 'log', '--format=%ad', '--date=short', '-1'] +CMD_PERL_REGEX = ['perl', '-pi', '-e'] +REGEX_TEMPLATE = 's/(20\\d\\d)(?:-20\\d\\d)? The Bitcoin/$1-%s The Bitcoin/' -FOLDERS = ["./qa", "./src"] +FOLDERS = ["qa/", "src/"] EXTENSIONS = [".cpp",".h", ".py"] + def get_git_date(file_path): - r = os.popen(CMD_GIT_DATE % file_path) - for l in r: - # Result is one line, so just return - return l.replace("\n","") - return "" - -n=1 -for folder in FOLDERS: - for extension in EXTENSIONS: - for file_path in os.popen(CMD_LIST_FILES % (folder, extension)): - file_path = os.getcwd() + file_path[1:-1] - if file_path.endswith(extension): - git_date = get_git_date(file_path) - if str(year) == git_date: - # Only update if current year is not found - if REGEX_CURRENT.search(open(file_path, "r").read()) is None: - print n,"Last git edit", git_date, "-", file_path - os.popen(CMD_REGEX % (year,file_path)) + d = subprocess.run(CMD_GIT_DATE + [file_path], + stdout=subprocess.PIPE, + check=True, + universal_newlines=True).stdout + # yyyy-mm-dd + return d.split('-')[0] + + +def skip_file(file_path): + for ext in EXTENSIONS: + if file_path.endswith(ext): + return False + else: + return True + +if __name__ == "__main__": + year = str(time.gmtime()[0]) + regex_current = re.compile("%s The Bitcoin" % year) + n = 1 + for folder in FOLDERS: + for file_path in subprocess.run( + CMD_GIT_LIST_FILES + [folder], + stdout=subprocess.PIPE, + check=True, + universal_newlines=True + ).stdout.split("\n"): + if skip_file(file_path): + # print(file_path, "(skip)") + continue + git_date = get_git_date(file_path) + if not year == git_date: + # print(file_path, year, "(skip)") + continue + if regex_current.search(open(file_path, "r").read()) is not None: + # already up to date + # print(file_path, year, "(skip)") + continue + print(n, file_path, "(update to %s)" % year) + subprocess.run(CMD_PERL_REGEX + [REGEX_TEMPLATE % year, file_path], check=True) n = n + 1 diff --git a/contrib/gitian-build.sh b/contrib/gitian-build.sh new file mode 100755 index 0000000000..ee47d138f3 --- /dev/null +++ b/contrib/gitian-build.sh @@ -0,0 +1,388 @@ +# What to do +sign=false +verify=false +build=false +setupenv=false + +# Systems to build +linux=true +windows=true +osx=true + +# Other Basic variables +SIGNER= +VERSION= +commit=false +url=https://github.com/bitcoin/bitcoin +proc=2 +mem=2000 +lxc=true +osslTarUrl=http://downloads.sourceforge.net/project/osslsigncode/osslsigncode/osslsigncode-1.7.1.tar.gz +osslPatchUrl=https://bitcoincore.org/cfields/osslsigncode-Backports-to-1.7.1.patch +scriptName=$(basename -- "$0") +signProg="gpg --detach-sign" +commitFiles=true + +# Help Message +read -d '' usage <<- EOF +Usage: $scriptName [-c|u|v|b|s|B|o|h|j|m|] signer version + +Run this script from the directory containing the bitcoin, gitian-builder, gitian.sigs, and bitcoin-detached-sigs. + +Arguments: +signer GPG signer to sign each build assert file +version Version number, commit, or branch to build. If building a commit or branch, the -c option must be specified + +Options: +-c|--commit Indicate that the version argument is for a commit or branch +-u|--url Specify the URL of the repository. Default is https://github.com/bitcoin/bitcoin +-v|--verify Verify the gitian build +-b|--build Do a gitiain build +-s|--sign Make signed binaries for Windows and Mac OSX +-B|--buildsign Build both signed and unsigned binaries +-o|--os Specify which Operating Systems the build is for. Default is lwx. l for linux, w for windows, x for osx +-j Number of processes to use. Default 2 +-m Memory to allocate in MiB. Default 2000 +--kvm Use KVM instead of LXC +--setup Setup the gitian building environment. Uses KVM. If you want to use lxc, use the --lxc option. Only works on Debian-based systems (Ubuntu, Debian) +--detach-sign Create the assert file for detached signing. Will not commit anything. +--no-commit Do not commit anything to git +-h|--help Print this help message +EOF + +# Get options and arguments +while :; do + case $1 in + # Verify + -v|--verify) + verify=true + ;; + # Build + -b|--build) + build=true + ;; + # Sign binaries + -s|--sign) + sign=true + ;; + # Build then Sign + -B|--buildsign) + sign=true + build=true + ;; + # PGP Signer + -S|--signer) + if [ -n "$2" ] + then + SIGNER=$2 + shift + else + echo 'Error: "--signer" requires a non-empty argument.' + exit 1 + fi + ;; + # Operating Systems + -o|--os) + if [ -n "$2" ] + then + linux=false + windows=false + osx=false + if [[ "$2" = *"l"* ]] + then + linux=true + fi + if [[ "$2" = *"w"* ]] + then + windows=true + fi + if [[ "$2" = *"x"* ]] + then + osx=true + fi + shift + else + echo 'Error: "--os" requires an argument containing an l (for linux), w (for windows), or x (for Mac OSX)\n' + exit 1 + fi + ;; + # Help message + -h|--help) + echo "$usage" + exit 0 + ;; + # Commit or branch + -c|--commit) + commit=true + ;; + # Number of Processes + -j) + if [ -n "$2" ] + then + proc=$2 + shift + else + echo 'Error: "-j" requires an argument' + exit 1 + fi + ;; + # Memory to allocate + -m) + if [ -n "$2" ] + then + mem=$2 + shift + else + echo 'Error: "-m" requires an argument' + exit 1 + fi + ;; + # URL + -u) + if [ -n "$2" ] + then + url=$2 + shift + else + echo 'Error: "-u" requires an argument' + exit 1 + fi + ;; + # kvm + --kvm) + lxc=false + ;; + # Detach sign + --detach-sign) + signProg="true" + commitFiles=false + ;; + # Commit files + --no-commit) + commitFiles=false + ;; + # Setup + --setup) + setup=true + ;; + *) # Default case: If no more options then break out of the loop. + break + esac + shift +done + +# Set up LXC +if [[ $lxc = true ]] +then + export USE_LXC=1 + export LXC_BRIDGE=lxcbr0 + sudo ifconfig lxcbr0 up 10.0.2.2 +fi + +# Check for OSX SDK +if [[ ! -e "gitian-builder/inputs/MacOSX10.11.sdk.tar.gz" && $osx == true ]] +then + echo "Cannot build for OSX, SDK does not exist. Will build for other OSes" + osx=false +fi + +# Get signer +if [[ -n"$1" ]] +then + SIGNER=$1 + shift +fi + +# Get version +if [[ -n "$1" ]] +then + VERSION=$1 + COMMIT=$VERSION + shift +fi + +# Check that a signer is specified +if [[ $SIGNER == "" ]] +then + echo "$scriptName: Missing signer." + echo "Try $scriptName --help for more information" + exit 1 +fi + +# Check that a version is specified +if [[ $VERSION == "" ]] +then + echo "$scriptName: Missing version." + echo "Try $scriptName --help for more information" + exit 1 +fi + +# Add a "v" if no -c +if [[ $commit = false ]] +then + COMMIT="v${VERSION}" +fi +echo ${COMMIT} + +# Setup build environment +if [[ $setup = true ]] +then + sudo apt-get install ruby apache2 git apt-cacher-ng python-vm-builder qemu-kvm qemu-utils + git clone https://github.com/bitcoin-core/gitian.sigs.git + git clone https://github.com/bitcoin-core/bitcoin-detached-sigs.git + git clone https://github.com/devrandom/gitian-builder.git + pushd ./gitian-builder + if [[ -n "$USE_LXC" ]] + then + sudo apt-get install lxc + bin/make-base-vm --suite trusty --arch amd64 --lxc + else + bin/make-base-vm --suite trusty --arch amd64 + fi + popd +fi + +# Set up build +pushd ./bitcoin +git fetch +git checkout ${COMMIT} +popd + +# Build +if [[ $build = true ]] +then + # Make output folder + mkdir -p ./bitcoin-binaries/${VERSION} + + # Build Dependencies + echo "" + echo "Building Dependencies" + echo "" + pushd ./gitian-builder + mkdir -p inputs + wget -N -P inputs $osslPatchUrl + wget -N -P inputs $osslTarUrl + make -C ../bitcoin/depends download SOURCES_PATH=`pwd`/cache/common + + # Linux + if [[ $linux = true ]] + then + echo "" + echo "Compiling ${VERSION} Linux" + echo "" + ./bin/gbuild -j ${proc} -m ${mem} --commit bitcoin=${COMMIT} --url bitcoin=${url} ../bitcoin/contrib/gitian-descriptors/gitian-linux.yml + ./bin/gsign -p $signProg --signer $SIGNER --release ${VERSION}-linux --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-linux.yml + mv build/out/bitcoin-*.tar.gz build/out/src/bitcoin-*.tar.gz ../bitcoin-binaries/${VERSION} + fi + # Windows + if [[ $windows = true ]] + then + echo "" + echo "Compiling ${VERSION} Windows" + echo "" + ./bin/gbuild -j ${proc} -m ${mem} --commit bitcoin=${COMMIT} --url bitcoin=${url} ../bitcoin/contrib/gitian-descriptors/gitian-win.yml + ./bin/gsign -p $signProg --signer $SIGNER --release ${VERSION}-win-unsigned --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-win.yml + mv build/out/bitcoin-*-win-unsigned.tar.gz inputs/bitcoin-win-unsigned.tar.gz + mv build/out/bitcoin-*.zip build/out/bitcoin-*.exe ../bitcoin-binaries/${VERSION} + fi + # Mac OSX + if [[ $osx = true ]] + then + echo "" + echo "Compiling ${VERSION} Mac OSX" + echo "" + ./bin/gbuild -j ${proc} -m ${mem} --commit bitcoin=${COMMIT} --url bitcoin=${url} ../bitcoin/contrib/gitian-descriptors/gitian-osx.yml + ./bin/gsign -p $signProg --signer $SIGNER --release ${VERSION}-osx-unsigned --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-osx.yml + mv build/out/bitcoin-*-osx-unsigned.tar.gz inputs/bitcoin-osx-unsigned.tar.gz + mv build/out/bitcoin-*.tar.gz build/out/bitcoin-*.dmg ../bitcoin-binaries/${VERSION} + fi + popd + + if [[ $commitFiles = true ]] + then + # Commit to gitian.sigs repo + echo "" + echo "Committing ${VERSION} Unsigned Sigs" + echo "" + pushd gitian.sigs + git add ${VERSION}-linux/${SIGNER} + git add ${VERSION}-win-unsigned/${SIGNER} + git add ${VERSION}-osx-unsigned/${SIGNER} + git commit -a -m "Add ${VERSION} unsigned sigs for ${SIGNER}" + popd + fi +fi + +# Verify the build +if [[ $verify = true ]] +then + # Linux + pushd ./gitian-builder + echo "" + echo "Verifying v${VERSION} Linux" + echo "" + ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-linux ../bitcoin/contrib/gitian-descriptors/gitian-linux.yml + # Windows + echo "" + echo "Verifying v${VERSION} Windows" + echo "" + ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-win-unsigned ../bitcoin/contrib/gitian-descriptors/gitian-win.yml + # Mac OSX + echo "" + echo "Verifying v${VERSION} Mac OSX" + echo "" + ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-osx-unsigned ../bitcoin/contrib/gitian-descriptors/gitian-osx.yml + # Signed Windows + echo "" + echo "Verifying v${VERSION} Signed Windows" + echo "" + ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-osx-signed ../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml + # Signed Mac OSX + echo "" + echo "Verifying v${VERSION} Signed Mac OSX" + echo "" + ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-osx-signed ../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml + popd +fi + +# Sign binaries +if [[ $sign = true ]] +then + + pushd ./gitian-builder + # Sign Windows + if [[ $windows = true ]] + then + echo "" + echo "Signing ${VERSION} Windows" + echo "" + ./bin/gbuild -i --commit signature=${COMMIT} ../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml + ./bin/gsign -p $signProg --signer $SIGNER --release ${VERSION}-win-signed --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml + mv build/out/bitcoin-*win64-setup.exe ../bitcoin-binaries/${VERSION} + mv build/out/bitcoin-*win32-setup.exe ../bitcoin-binaries/${VERSION} + fi + # Sign Mac OSX + if [[ $osx = true ]] + then + echo "" + echo "Signing ${VERSION} Mac OSX" + echo "" + ./bin/gbuild -i --commit signature=${COMMIT} ../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml + ./bin/gsign -p $signProg --signer $SIGNER --release ${VERSION}-osx-signed --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml + mv build/out/bitcoin-osx-signed.dmg ../bitcoin-binaries/${VERSION}/bitcoin-${VERSION}-osx.dmg + fi + popd + + if [[ $commitFiles = true ]] + then + # Commit Sigs + pushd gitian.sigs + echo "" + echo "Committing ${VERSION} Signed Sigs" + echo "" + git add ${VERSION}-win-signed/${SIGNER} + git add ${VERSION}-osx-signed/${SIGNER} + git commit -a -m "Add ${VERSION} signed binary sigs for ${SIGNER}" + popd + fi +fi diff --git a/contrib/macdeploy/extract-osx-sdk.sh b/contrib/macdeploy/extract-osx-sdk.sh new file mode 100755 index 0000000000..46d2d825d4 --- /dev/null +++ b/contrib/macdeploy/extract-osx-sdk.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -e + +INPUTFILE="Xcode_7.3.1.dmg" +HFSFILENAME="5.hfs" +SDKDIR="Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.11.sdk" + +7z x "${INPUTFILE}" "${HFSFILENAME}" +SDKNAME="$(basename "${SDKDIR}")" +SDKDIRINODE=$(ifind -n "${SDKDIR}" "${HFSFILENAME}") +fls "${HFSFILENAME}" -rpF ${SDKDIRINODE} | + while read type inode filename; do + inode="${inode::-1}" + if [ "${filename:0:14}" = "usr/share/man/" ]; then + continue + fi + filename="${SDKNAME}/$filename" + echo "Extracting $filename ..." + mkdir -p "$(dirname "$filename")" + if [ "$type" = "l/l" ]; then + ln -s "$(icat "${HFSFILENAME}" $inode)" "$filename" + else + icat "${HFSFILENAME}" $inode >"$filename" + fi +done +echo "Building ${SDKNAME}.tar.gz ..." +MTIME="$(istat "${HFSFILENAME}" "${SDKDIRINODE}" | perl -nle 'm/Content Modified:\s+(.*?)\s\(/ && print $1')" +find "${SDKNAME}" | sort | tar --no-recursion --mtime="${MTIME}" --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > "${SDKNAME}.tar.gz" +echo 'All done!' |